From 9d1aa187ce200fd481b622f08a59c3c96f6600e7 Mon Sep 17 00:00:00 2001 From: rpriven Date: Tue, 4 Nov 2025 00:31:40 -0700 Subject: [PATCH] Initial commit: xactdiff tool for comparing Xactimate estimates --- README.md | 47 +++++++ xactdiff.py | 387 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 434 insertions(+) create mode 100644 README.md create mode 100644 xactdiff.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..db6b741 --- /dev/null +++ b/README.md @@ -0,0 +1,47 @@ +# xactdiff + +Compare two Xactimate estimates and identify missing line items. + +## Purpose + +Construction contractors use this tool to compare: +- Their estimate vs. Insurance estimate +- Find missing line items that should be included +- Identify discrepancies in scope + +## Usage + +```bash +xactdiff estimate1.pdf estimate2.pdf output.xlsx +``` + +The output Excel file contains: +- **Estimate1 sheet**: All line items from first PDF +- **Estimate2 sheet**: All line items from second PDF +- **Diff sheet**: Side-by-side comparison with fuzzy matching + - Highlighted rows show differences + - Missing items clearly marked + +## Features + +- Fuzzy text matching (handles slight wording differences) +- Color-coded differences (green = match, red = missing/different) +- Organized by trade category +- Sortable and filterable in Excel + +## Requirements + +```bash +python3 -m pip install --user pdfplumber pandas openpyxl rapidfuzz +``` + +## Installation + +```bash +chmod +x xactdiff.py +sudo ln -s $(pwd)/xactdiff.py /usr/local/bin/xactdiff +``` + +## Related Tools + +- **xactparse**: Extract and total line items from single estimate diff --git a/xactdiff.py b/xactdiff.py new file mode 100644 index 0000000..5f45141 --- /dev/null +++ b/xactdiff.py @@ -0,0 +1,387 @@ +import argparse +import logging +import pandas as pd +from openpyxl import load_workbook +from rapidfuzz import process, fuzz + +# ... (TRADE_KEYWORDS, assign_trade, is_line_item, extract_xactimate_items as before) ... +TRADE_KEYWORDS = { + "Floor Protection": ["floor protection", "cardboard", "protect floor", "mask floor"], + "Insulation": ["insulation", "batt", "fiberglass", "blown-in"], + "Drywall": ["drywall", "sheetrock", "tape joint", "texture", "patch", "mud", "repair wall", "joint compound", "corner bead"], + "Painting": ["paint", "painting", "primer", "prime", "seal", "coating", "enamel", "mask wall", "caulk", "caulking", "mirror", "towel bar", "toilet paper", "tp holder"], + "Baseboards, Trim, Casing": ["baseboard", "trim", "casing", "moulding", "crown", "shoe mould", "quarter round"], + "Doors": ["door", "door stop", "interior door", "slab"], + "Shower": ["shower", "shower pan", "shower door", "shower surround"], + "Laminate": ["laminate", "pergo", "engineered wood"], + "Vinyl Flooring": ["vinyl floor", "vinyl sheet"], + "Tile Flooring": ["tile floor", "ceramic tile", "porcelain tile", "grout", "thinset", "cement board", "grout", "clean floor and prep"], + "Carpet": ["carpet", "carpeting", "pad", "broadloom", "tack stip"], + "HVAC": ["hvac", "register", "ventilation"], + "Content Manipulation": ["content manipulation", "move contents", "protect contents", "cover contents", "contents"], + "Cleaning": ["clean", "cleaning", "final clean", "clean up", "final cleanup", "final cleaning", "construction clean"], + "Debris Removal": ["debris removal", "dump", "haul debris", "remove debris", "trash out"], + "Cabinets": ["cabinet", "vanity", "base cabinet", "wall cabinet", "countertop"], + "Electrical": ["electrical", "outlet", "switch", "receptacle", "breaker", "light fixture", "light", "ceiling fan"], + "Showers, Tubs, Tile": ["tub", "bathtub", "shower", "tile", "surround", "enclosure", "shower pan", "shower door", "shower surround"], + "Plumbing, Toilets, Sinks": ["plumbing", "toilet", "sink", "faucet", "supply line", "angle stop", "drain", "p-trap"], + "Labor Minimums": ["labor minimum"], + "Mitigation": ["water extraction", "remediation", "mitigation"] +} + +HEADERS = ["DESCRIPTION", "TRADE", "QUANTITY", "UNIT PRICE", "TAX", "O&P", "RCV", "DEPREC.", "ACV"] + + +def assign_trade(description): + desc = description.lower() + for trade, keywords in TRADE_KEYWORDS.items(): + if any(keyword in desc for keyword in keywords): + return trade + return "Other" + + +def clean_description(desc): + # Remove leading number and period, lowercase, and strip + desc = re.sub(r"^\d+\.\s*", "", desc) + desc = desc.lower().strip() + return desc + + +def write_masters_to_excel(items1, items2, excel_path): + df1 = pd.DataFrame(items1[1:], columns=items1[0]) + df2 = pd.DataFrame(items2[1:], columns=items2[0]) + with pd.ExcelWriter(excel_path, engine='openpyxl') as writer: + df1.to_excel(writer, sheet_name="Estimate1", index=False) + df2.to_excel(writer, sheet_name="Estimate2", index=False) + # Diff will be added in the next step + + +def extract_xactimate_items_with_fallback(pdf_path, assign_trade, HEADERS): + """ + Try robust extraction first; if no items, try classic extraction. + """ + # --- Robust multi-line regex extractor --- + def robust_extractor(): + import re + import pdfplumber + line_item_regex = re.compile( + r"^(\d+\.)\s+(.+?)\s+([\d,.]+(?:SF|DA|HR|LF|EA|SY|UN|MO|WK|DY|BD|FT|YD|IN|CM|M|MM|LB|KG|GM|L|GAL|PC|SET|SQ|BOX|ROLL)?)\s+" + r"([\d,.]+)\s+([\d,.]+)\s+([\d,.]+)\s+([\d,.]+)\s+[\(<]([\d,.]+)[\)>]\s+([\d,.]+)(?:\s|$)" + ) + extracted_items = [] + with pdfplumber.open(pdf_path) as pdf: + for page in pdf.pages: + text = page.extract_text() + if not text: + continue + lines = text.split('\n') + i = 0 + while i < len(lines): + line = lines[i].strip() + if re.match(r"^\d+\.\s", line): + combined_line = line + j = i + 1 + while j < len(lines) and not re.match(r"^\d+\.\s", lines[j]): + combined_line += " " + lines[j].strip() + j += 1 + match = line_item_regex.match(combined_line) + if match: + description = match.group(1) + " " + match.group(2) + quantity_unit = match.group(3) + unit_price = match.group(4) + tax = match.group(5) + o_p = match.group(6) + rcv = match.group(7) + deprec = match.group(8) + acv = match.group(9) + trade = assign_trade(description) + extracted_items.append([ + description, trade, quantity_unit, unit_price, tax, o_p, rcv, deprec, acv + ]) + i = j + else: + i += 1 + return [HEADERS] + extracted_items + + # --- Classic single-line extractor --- + def classic_extractor(): + import re + import pdfplumber + LINE_ITEM_REGEX = re.compile( + r"^(\d+\.\s+.+?)\s+(\d+\.\d+\s+(?:SF|LF|EA|HR|DA|SY))\s+([\d,.]+)\s+([\d,.]+)\s+([\d,.]+)\s+([\d,.]+)\s+\(([\d,.]+)\)\s+([\d,.]+)" + ) + extracted_items = [] + + def is_line_item(line): + return bool(LINE_ITEM_REGEX.match(line)) + with pdfplumber.open(pdf_path) as pdf: + for page in pdf.pages: + text = page.extract_text() + if not text: + continue + lines = text.split('\n') + i = 0 + while i < len(lines): + line = lines[i].strip() + if is_line_item(line): + match = LINE_ITEM_REGEX.match(line) + if match: + description = match.group(1) + quantity_unit = match.group(2) + unit_price = match.group(3) + tax = match.group(4) + o_p = match.group(5) + rcv = match.group(6) + deprec = match.group(7) + acv = match.group(8) + trade = assign_trade(description) + extracted_items.append([ + description, trade, quantity_unit, unit_price, tax, o_p, rcv, deprec, acv + ]) + else: + if i + 1 < len(lines) and not is_line_item(lines[i + 1]): + combined_line = line + " " + lines[i + 1].strip() + match = LINE_ITEM_REGEX.match(combined_line) + if match: + description = match.group(1) + quantity_unit = match.group(2) + unit_price = match.group(3) + tax = match.group(4) + o_p = match.group(5) + rcv = match.group(6) + deprec = match.group(7) + acv = match.group(8) + trade = assign_trade(description) + extracted_items.append([ + description, trade, quantity_unit, unit_price, tax, o_p, rcv, deprec, acv + ]) + i += 1 + i += 1 + return [HEADERS] + extracted_items + + # --- Try robust, fallback to classic if needed --- + try: + data = robust_extractor() + if len(data) > 1: + return data + except Exception as e: + print("Robust extractor failed:", e) + print("Trying classic extractor...") + try: + data = classic_extractor() + if len(data) > 1: + return data + except Exception as e: + print("Classic extractor failed:", e) + print("No items extracted from", pdf_path) + return [HEADERS] # Only headers if nothing found + + +def create_diff_tab(items1, items2, excel_path): + df1 = pd.DataFrame(items1[1:], columns=items1[0]) + df2 = pd.DataFrame(items2[1:], columns=items2[0]) + + # Merge on DESCRIPTION + merged = pd.merge( + df1, df2, on="DESCRIPTION", how="outer", suffixes=('_EST1', '_EST2'), indicator=True + ) + + # DEBUG + print("Estimate 1 sample:") + print(df1.head()) + print("Estimate 2 sample:") + print(df2.head()) + + # DEBUG + print("DF1 columns:", df1.columns) + print("DF2 columns:", df2.columns) + + # DEBUG + print("Estimate 1 DESCRIPTIONS:") + print(df1['DESCRIPTION'].head(20)) + print("Estimate 2 DESCRIPTIONS:") + print(df2['DESCRIPTION'].head(20)) + + # DEBUG + print("Estimate 1 TRADES:") + print(df1['TRADE'].head(20)) + print("Estimate 2 TRADES:") + print(df2['TRADE'].head(20)) + + # Mark differences + merged['DIFF'] = merged['_merge'].map({ + 'left_only': 'Only in Estimate1', + 'right_only': 'Only in Estimate2', + 'both': '' + }).astype(str) + + # For items in both, check if any fields differ + for col in ["QUANTITY", "UNIT PRICE", "TAX", "O&P", "RCV", "DEPREC.", "ACV"]: + est1_col = col + "_EST1" + est2_col = col + "_EST2" + mask = (merged['_merge'] == 'both') & (merged[est1_col] != merged[est2_col]) + merged.loc[mask, 'DIFF'] = 'Changed' + + # Write to Excel (append to previous file) + with pd.ExcelWriter(excel_path, engine='openpyxl', mode='a', if_sheet_exists='replace') as writer: + merged.to_excel(writer, sheet_name="Diff", index=False) + + +def highlight_diff_sheet(filename, diff_sheet_name="Diff"): + from openpyxl import load_workbook + from openpyxl.styles import PatternFill + + wb = load_workbook(filename) + ws = wb[diff_sheet_name] + + # Find the DIFF column index + diff_col = None + for idx, cell in enumerate(ws[1], 1): + if cell.value == "DIFF": + diff_col = idx + break + + yellow = PatternFill(start_color="FFFF00", end_color="FFFF00", fill_type="solid") + red = PatternFill(start_color="FFC7CE", end_color="FFC7CE", fill_type="solid") + green = PatternFill(start_color="C6EFCE", end_color="C6EFCE", fill_type="solid") + + for row in ws.iter_rows(min_row=2, max_row=ws.max_row): + diff_value = row[diff_col - 1].value + if diff_value == "Changed": + for cell in row: + cell.fill = yellow + elif diff_value == "Only in Estimate1": + for cell in row: + cell.fill = red + elif diff_value == "Only in Estimate2": + for cell in row: + cell.fill = green + + wb.save(filename) + + +def sort_and_freeze_diff(filename, diff_sheet_name="Diff"): + from openpyxl import load_workbook + + wb = load_workbook(filename) + ws = wb[diff_sheet_name] + + # Read all rows into a list + data = list(ws.values) + headers = data[0] + rows = data[1:] + + # Find DIFF and DESCRIPTION column indexes + diff_idx = headers.index("DIFF") + desc_idx = headers.index("DESCRIPTION") + + # Sort: DIFF, then DESCRIPTION + rows.sort(key=lambda x: (x[diff_idx], x[desc_idx])) + + # Write back to sheet + for i, row in enumerate([headers] + rows, 1): + for j, value in enumerate(row, 1): + ws.cell(row=i, column=j, value=value) + + # Freeze header row + ws.freeze_panes = ws["A2"] + + wb.save(filename) + + +def auto_fit_excel_columns(filename): + wb = load_workbook(filename) + for ws in wb.worksheets: + for column_cells in ws.columns: + max_length = 0 + column = column_cells[0].column_letter + for cell in column_cells: + try: + cell_length = len(str(cell.value)) + if cell_length > max_length: + max_length = cell_length + except: + pass + adjusted_width = max_length + 2 + ws.column_dimensions[column].width = adjusted_width + wb.save(filename) + + +def find_fuzzy_matches(df1, df2, threshold=85): + """ + For each item in df1, find the closest DESCRIPTION_CLEAN in df2. + Returns a DataFrame with best match and score. + """ + matches = [] + choices = df2['DESCRIPTION_CLEAN'].tolist() + for idx, row in df1.iterrows(): + desc = row['DESCRIPTION_CLEAN'] + # Find best match in df2 + best_match, score, match_idx = process.extractOne( + desc, choices, scorer=fuzz.token_sort_ratio) + matches.append((idx, best_match, score, match_idx)) + match_df = pd.DataFrame(matches, columns=['df1_idx', 'best_match', 'score', 'df2_idx']) + return match_df + + +def main(): + parser = argparse.ArgumentParser( + description="Compare two Xactimate estimates and highlight differences in Excel." + ) + parser.add_argument("estimate1", help="Path to the first PDF file") + parser.add_argument("estimate2", help="Path to the second PDF file") + parser.add_argument("excel_file", help="Path to the output Excel file") + args = parser.parse_args() + + logging.info(f"Extracting line items from {args.estimate1} ...") + items1 = extract_xactimate_items_with_fallback(args.estimate1, assign_trade, HEADERS) + + logging.info(f"Extracting line items from {args.estimate2} ...") + items2 = extract_xactimate_items_with_fallback(args.estimate2, assign_trade, HEADERS) + + write_masters_to_excel(items1, items2, args.excel_file) + create_diff_tab(items1, items2, args.excel_file) + + highlight_diff_sheet(args.excel_file) + sort_and_freeze_diff(args.excel_file) + + # --- ADDED: Find missing and extra items --- + import pandas as pd + import re + + df1 = pd.DataFrame(items1[1:], columns=items1[0]) + df2 = pd.DataFrame(items2[1:], columns=items2[0]) + + def clean_description(desc): + desc = re.sub(r"^\d+\.\s*", "", desc) + desc = desc.lower().strip() + return desc + + df1['DESCRIPTION_CLEAN'] = df1['DESCRIPTION'].apply(clean_description) + df2['DESCRIPTION_CLEAN'] = df2['DESCRIPTION'].apply(clean_description) + + # missing = df1[~df1['DESCRIPTION_CLEAN'].isin(df2['DESCRIPTION_CLEAN'])] + # extra = df2[~df2['DESCRIPTION_CLEAN'].isin(df1['DESCRIPTION_CLEAN'])] + + match_df = find_fuzzy_matches(df1, df2, threshold=85) + good_matches = match_df[match_df['score'] >= 85] + + matched_df1 = set(good_matches['df1_idx']) + matched_df2 = set(good_matches['df2_idx']) + + missing = df1.loc[~df1.index.isin(matched_df1)] + extra = df2.loc[~df2.index.isin(matched_df2)] + + with pd.ExcelWriter(args.excel_file, engine='openpyxl', mode='a', if_sheet_exists='overlay') as writer: + missing[['DESCRIPTION', 'TRADE', 'QUANTITY', 'UNIT PRICE', 'RCV', 'ACV'] + ].to_excel(writer, sheet_name="MissingFromInsurance", index=False) + extra[['DESCRIPTION', 'TRADE', 'QUANTITY', 'UNIT PRICE', 'RCV', 'ACV'] + ].to_excel(writer, sheet_name="ExtraInInsurance", index=False) + + auto_fit_excel_columns(args.excel_file) + + print(f"Comparison complete! Results saved to {args.excel_file}") + + +if __name__ == "__main__": + main()