[{"data":1,"prerenderedAt":2299},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fappend-new-rows-to-a-master-excel-file-without-duplicates":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fappend-new-rows-to-a-master-excel-file-without-duplicates":2268,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fappend-new-rows-to-a-master-excel-file-without-duplicates":2276,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fappend-new-rows-to-a-master-excel-file-without-duplicates":2280,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fappend-new-rows-to-a-master-excel-file-without-duplicates":2281},{"id":4,"title":5,"body":6,"dateModified":2233,"datePublished":2233,"description":2234,"extension":2235,"faq":2236,"keyword":2245,"kind":2246,"meta":2247,"navigation":104,"path":2257,"seo":2258,"slug":2260,"stem":2261,"topics":2262,"type":2266,"__hash__":2267},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fappend-new-rows-to-a-master-excel-file-without-duplicates\u002Findex.md","Append New Rows to a Master Excel File Without Duplicates",{"type":7,"value":8,"toc":2220},"minimark",[9,24,29,58,72,76,341,368,474,478,481,726,733,737,744,1083,1113,1375,1379,1382,1641,1644,1648,1651,1904,1911,2087,2091,2098,2102,2148,2152,2155,2159,2165,2171,2177,2183,2187,2216],[10,11,12,13,17,18,23],"p",{},"A common reporting setup: a master workbook that collects every invoice, ticket or shipment, and a weekly extract from a system that covers \"the last 14 days\" — so each extract overlaps the previous one. Appending each extract in full doubles the overlapping rows; appending only rows after the last date misses late-arriving records with earlier dates. The reliable approach is to compare on a ",[14,15,16],"em",{},"key"," — the invoice number, the ticket ID — and append only rows whose key is not already in the master. Two refinements make it production-ready: deciding what to do with rows that already exist but have changed, and appending without destroying the master's formatting, which a full rewrite with pandas would. This page belongs to ",[19,20,22],"a",{"href":21},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002F","Merging and Joining Excel DataFrames",".",[25,26,28],"h2",{"id":27},"prerequisites","Prerequisites",[30,31,36],"pre",{"className":32,"code":33,"language":34,"meta":35,"style":35},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[37,38,39],"code",{"__ignoreMap":35},[40,41,44,48,52,55],"span",{"class":42,"line":43},"line",1,[40,45,47],{"class":46},"sMTad","pip",[40,49,51],{"class":50},"srMev"," install",[40,53,54],{"class":50}," pandas",[40,56,57],{"class":50}," openpyxl\n",[10,59,60,63,64,67,68,71],{},[37,61,62],{},"master_invoices.xlsx"," holds three invoices in a formatted Excel table named ",[37,65,66],{},"Invoices",". ",[37,69,70],{},"extract_week39.xlsx"," is this week's system extract: it repeats INV-1002 (now paid) and INV-1003 (amount corrected from 980 to 1,010), adds two new invoices — and lists INV-1005 twice, because extracts are not always clean.",[25,73,75],{"id":74},"step-1-find-the-new-rows-with-an-anti-join","Step 1 — Find the new rows with an anti-join",[30,77,81],{"className":78,"code":79,"language":80,"meta":35,"style":35},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\nmaster = pd.read_excel(\"master_invoices.xlsx\", sheet_name=\"Invoices\")\nextract = pd.read_excel(\"extract_week39.xlsx\")\nKEY = \"Invoice\"\n\ndupes_in_extract = extract[extract.duplicated(KEY, keep=False)]\nprint(\"duplicate keys inside the extract:\", dupes_in_extract[KEY].unique().tolist())\nextract = extract.drop_duplicates(KEY, keep=\"last\")\n\nflagged = extract.merge(master[[KEY]], on=KEY, how=\"left\", indicator=True)\nnew_rows = flagged[flagged[\"_merge\"] == \"left_only\"].drop(columns=\"_merge\")\nprint(new_rows.to_string(index=False))\n","python",[37,82,83,99,106,136,151,164,169,195,215,238,243,288,323],{"__ignoreMap":35},[40,84,85,89,93,96],{"class":42,"line":43},[40,86,88],{"class":87},"s-kum","import",[40,90,92],{"class":91},"skGVy"," pandas ",[40,94,95],{"class":87},"as",[40,97,98],{"class":91}," pd\n",[40,100,102],{"class":42,"line":101},2,[40,103,105],{"emptyLinePlaceholder":104},true,"\n",[40,107,109,112,115,118,121,124,128,130,133],{"class":42,"line":108},3,[40,110,111],{"class":91},"master ",[40,113,114],{"class":87},"=",[40,116,117],{"class":91}," pd.read_excel(",[40,119,120],{"class":50},"\"master_invoices.xlsx\"",[40,122,123],{"class":91},", ",[40,125,127],{"class":126},"sa561","sheet_name",[40,129,114],{"class":87},[40,131,132],{"class":50},"\"Invoices\"",[40,134,135],{"class":91},")\n",[40,137,139,142,144,146,149],{"class":42,"line":138},4,[40,140,141],{"class":91},"extract ",[40,143,114],{"class":87},[40,145,117],{"class":91},[40,147,148],{"class":50},"\"extract_week39.xlsx\"",[40,150,135],{"class":91},[40,152,154,158,161],{"class":42,"line":153},5,[40,155,157],{"class":156},"sP0c6","KEY",[40,159,160],{"class":87}," =",[40,162,163],{"class":50}," \"Invoice\"\n",[40,165,167],{"class":42,"line":166},6,[40,168,105],{"emptyLinePlaceholder":104},[40,170,172,175,177,180,182,184,187,189,192],{"class":42,"line":171},7,[40,173,174],{"class":91},"dupes_in_extract ",[40,176,114],{"class":87},[40,178,179],{"class":91}," extract[extract.duplicated(",[40,181,157],{"class":156},[40,183,123],{"class":91},[40,185,186],{"class":126},"keep",[40,188,114],{"class":87},[40,190,191],{"class":156},"False",[40,193,194],{"class":91},")]\n",[40,196,198,201,204,207,210,212],{"class":42,"line":197},8,[40,199,200],{"class":156},"print",[40,202,203],{"class":91},"(",[40,205,206],{"class":50},"\"duplicate keys inside the extract:\"",[40,208,209],{"class":91},", dupes_in_extract[",[40,211,157],{"class":156},[40,213,214],{"class":91},"].unique().tolist())\n",[40,216,218,220,222,225,227,229,231,233,236],{"class":42,"line":217},9,[40,219,141],{"class":91},[40,221,114],{"class":87},[40,223,224],{"class":91}," extract.drop_duplicates(",[40,226,157],{"class":156},[40,228,123],{"class":91},[40,230,186],{"class":126},[40,232,114],{"class":87},[40,234,235],{"class":50},"\"last\"",[40,237,135],{"class":91},[40,239,241],{"class":42,"line":240},10,[40,242,105],{"emptyLinePlaceholder":104},[40,244,246,249,251,254,256,259,262,264,266,268,271,273,276,278,281,283,286],{"class":42,"line":245},11,[40,247,248],{"class":91},"flagged ",[40,250,114],{"class":87},[40,252,253],{"class":91}," extract.merge(master[[",[40,255,157],{"class":156},[40,257,258],{"class":91},"]], ",[40,260,261],{"class":126},"on",[40,263,114],{"class":87},[40,265,157],{"class":156},[40,267,123],{"class":91},[40,269,270],{"class":126},"how",[40,272,114],{"class":87},[40,274,275],{"class":50},"\"left\"",[40,277,123],{"class":91},[40,279,280],{"class":126},"indicator",[40,282,114],{"class":87},[40,284,285],{"class":156},"True",[40,287,135],{"class":91},[40,289,291,294,296,299,302,305,308,311,314,317,319,321],{"class":42,"line":290},12,[40,292,293],{"class":91},"new_rows ",[40,295,114],{"class":87},[40,297,298],{"class":91}," flagged[flagged[",[40,300,301],{"class":50},"\"_merge\"",[40,303,304],{"class":91},"] ",[40,306,307],{"class":87},"==",[40,309,310],{"class":50}," \"left_only\"",[40,312,313],{"class":91},"].drop(",[40,315,316],{"class":126},"columns",[40,318,114],{"class":87},[40,320,301],{"class":50},[40,322,135],{"class":91},[40,324,326,328,331,334,336,338],{"class":42,"line":325},13,[40,327,200],{"class":156},[40,329,330],{"class":91},"(new_rows.to_string(",[40,332,333],{"class":126},"index",[40,335,114],{"class":87},[40,337,191],{"class":156},[40,339,340],{"class":91},"))\n",[10,342,343,344,347,348,351,352,355,356,358,359,362,363,367],{},"First the extract is de-duplicated on its own key — otherwise INV-1005 would be appended twice. Then a left merge with ",[37,345,346],{},"indicator=True"," marks each extract row as present in the master (",[37,349,350],{},"both",") or not (",[37,353,354],{},"left_only","); the ",[37,357,354],{}," rows are the new ones. This is an ",[14,360,361],{},"anti-join",", the same technique used in ",[19,364,366],{"href":365},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Ffind-rows-in-one-excel-file-missing-from-another\u002F","Find Rows in One Excel File Missing from Another",". Comparing on the key rather than on whole rows matters: INV-1003 changed amount, and a whole-row comparison would treat it as new and add a second INV-1003.",[369,370,378,379,378,383,378,387,378,395,378,402,378,405,378,408,378,416,378,419,378,426,378,430,378,434,378,439,378,442,378,445,378,449,378,455,378,459,378,469],"svg",{"viewBox":371,"role":372,"ariaLabelledBy":373,"xmlns":376,"style":377},"162 0 436 258","img",[374,375],"ap-anti-t","ap-anti-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:436px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[380,381,382],"title",{"id":374},"New rows are extract keys not yet in the master",[384,385,386],"desc",{"id":375},"Two overlapping sets, the extract keys and the master keys. The overlap holds INV-1002 and INV-1003, which already exist. The extract-only region holds INV-1004 and INV-1005, the rows to append.",[388,389],"rect",{"x":390,"y":391,"width":392,"height":393,"fill":394},"162","0","436","258","#ffffff",[396,397],"path",{"d":398,"fill":399,"stroke":400,"style":401},"M380.0,39.0 A78,78 0 1 0 380.0,165.0 A78,78 0 0 0 380.0,39.0 Z","#d9f4f1","none","stroke-width:0px",[396,403],{"d":404,"fill":394,"stroke":400,"style":401},"M380.0,39.0 A78,78 0 0 1 380.0,165.0 A78,78 0 0 1 380.0,39.0 Z",[396,406],{"d":407,"fill":394,"stroke":400,"style":401},"M380.0,39.0 A78,78 0 1 1 380.0,165.0 A78,78 0 0 1 380.0,39.0 Z",[409,410],"circle",{"cx":411,"cy":412,"r":413,"fill":400,"stroke":414,"style":415},"334.0","102.0","78.0","var(--muted,#5b6780)","stroke-width:2px",[409,417],{"cx":418,"cy":412,"r":413,"fill":400,"stroke":414,"style":415},"426.0",[420,421,425],"text",{"x":422,"y":423,"style":424},"298.9","92.0","font-size:11.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","INV-1004,",[420,427,429],{"x":422,"y":428,"style":424},"106.0","INV-1005",[420,431,433],{"x":422,"y":432,"style":424},"120.0","— append",[420,435,438],{"x":436,"y":423,"style":437},"380.0","font-size:11.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","1002,",[420,440,441],{"x":436,"y":428,"style":437},"1003 —",[420,443,444],{"x":436,"y":432,"style":437},"exist",[420,446,448],{"x":447,"y":428,"style":437},"461.1","INV-1001",[420,450,454],{"x":451,"y":452,"style":453},"306.7","204.0","font-size:13px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","extract",[420,456,458],{"x":457,"y":452,"style":453},"453.3","master",[388,460],{"x":461,"y":462,"width":463,"height":464,"rx":465,"fill":466,"stroke":467,"style":468},"178.4","216.0","403.3","26.0","8","#f0f2f5","var(--line,#cdd5e6)","stroke-width:1px",[420,470,473],{"x":436,"y":471,"style":472},"233.5","font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace","merge(..., how='left', indicator=True) → left_only",[25,475,477],{"id":476},"step-2-detect-rows-that-changed","Step 2 — Detect rows that changed",[10,479,480],{},"Existing keys are not necessarily identical rows. Compare the overlapping rows column by column to see what changed:",[30,482,484],{"className":78,"code":483,"language":80,"meta":35,"style":35},"compare_cols = [\"Date\", \"Customer\", \"Amount\", \"Status\"]\nboth = extract.merge(master, on=KEY, suffixes=(\"_new\", \"_old\"))\nchanges = []\nfor col in compare_cols:\n    diff = both[both[f\"{col}_new\"] != both[f\"{col}_old\"]]\n    for _, r in diff.iterrows():\n        changes.append({KEY: r[KEY], \"Column\": col, \"Old\": r[f\"{col}_old\"], \"New\": r[f\"{col}_new\"]})\nchanges = pd.DataFrame(changes)\nprint(changes.to_string(index=False))\n",[37,485,486,517,552,562,576,629,642,702,711],{"__ignoreMap":35},[40,487,488,491,493,496,499,501,504,506,509,511,514],{"class":42,"line":43},[40,489,490],{"class":91},"compare_cols ",[40,492,114],{"class":87},[40,494,495],{"class":91}," [",[40,497,498],{"class":50},"\"Date\"",[40,500,123],{"class":91},[40,502,503],{"class":50},"\"Customer\"",[40,505,123],{"class":91},[40,507,508],{"class":50},"\"Amount\"",[40,510,123],{"class":91},[40,512,513],{"class":50},"\"Status\"",[40,515,516],{"class":91},"]\n",[40,518,519,522,524,527,529,531,533,535,538,540,542,545,547,550],{"class":42,"line":101},[40,520,521],{"class":91},"both ",[40,523,114],{"class":87},[40,525,526],{"class":91}," extract.merge(master, ",[40,528,261],{"class":126},[40,530,114],{"class":87},[40,532,157],{"class":156},[40,534,123],{"class":91},[40,536,537],{"class":126},"suffixes",[40,539,114],{"class":87},[40,541,203],{"class":91},[40,543,544],{"class":50},"\"_new\"",[40,546,123],{"class":91},[40,548,549],{"class":50},"\"_old\"",[40,551,340],{"class":91},[40,553,554,557,559],{"class":42,"line":108},[40,555,556],{"class":91},"changes ",[40,558,114],{"class":87},[40,560,561],{"class":91}," []\n",[40,563,564,567,570,573],{"class":42,"line":138},[40,565,566],{"class":87},"for",[40,568,569],{"class":91}," col ",[40,571,572],{"class":87},"in",[40,574,575],{"class":91}," compare_cols:\n",[40,577,578,581,583,586,589,592,596,599,602,605,607,610,613,615,617,619,621,623,626],{"class":42,"line":153},[40,579,580],{"class":91},"    diff ",[40,582,114],{"class":87},[40,584,585],{"class":91}," both[both[",[40,587,588],{"class":87},"f",[40,590,591],{"class":50},"\"",[40,593,595],{"class":594},"sSjpA","{",[40,597,598],{"class":91},"col",[40,600,601],{"class":594},"}",[40,603,604],{"class":50},"_new\"",[40,606,304],{"class":91},[40,608,609],{"class":87},"!=",[40,611,612],{"class":91}," both[",[40,614,588],{"class":87},[40,616,591],{"class":50},[40,618,595],{"class":594},[40,620,598],{"class":91},[40,622,601],{"class":594},[40,624,625],{"class":50},"_old\"",[40,627,628],{"class":91},"]]\n",[40,630,631,634,637,639],{"class":42,"line":166},[40,632,633],{"class":87},"    for",[40,635,636],{"class":91}," _, r ",[40,638,572],{"class":87},[40,640,641],{"class":91}," diff.iterrows():\n",[40,643,644,647,649,652,654,657,660,663,666,668,670,672,674,676,678,680,682,685,687,689,691,693,695,697,699],{"class":42,"line":171},[40,645,646],{"class":91},"        changes.append({",[40,648,157],{"class":156},[40,650,651],{"class":91},": r[",[40,653,157],{"class":156},[40,655,656],{"class":91},"], ",[40,658,659],{"class":50},"\"Column\"",[40,661,662],{"class":91},": col, ",[40,664,665],{"class":50},"\"Old\"",[40,667,651],{"class":91},[40,669,588],{"class":87},[40,671,591],{"class":50},[40,673,595],{"class":594},[40,675,598],{"class":91},[40,677,601],{"class":594},[40,679,625],{"class":50},[40,681,656],{"class":91},[40,683,684],{"class":50},"\"New\"",[40,686,651],{"class":91},[40,688,588],{"class":87},[40,690,591],{"class":50},[40,692,595],{"class":594},[40,694,598],{"class":91},[40,696,601],{"class":594},[40,698,604],{"class":50},[40,700,701],{"class":91},"]})\n",[40,703,704,706,708],{"class":42,"line":197},[40,705,556],{"class":91},[40,707,114],{"class":87},[40,709,710],{"class":91}," pd.DataFrame(changes)\n",[40,712,713,715,718,720,722,724],{"class":42,"line":217},[40,714,200],{"class":156},[40,716,717],{"class":91},"(changes.to_string(",[40,719,333],{"class":126},[40,721,114],{"class":87},[40,723,191],{"class":156},[40,725,340],{"class":91},[10,727,728,729,23],{},"INV-1002's status moved from Open to Paid; INV-1003's amount went from 980 to 1,010. Whether the master should take these updates is a policy decision: status changes are usually wanted, amount corrections in a finance master usually need review. Keeping them in a separate change list lets the job apply the safe ones and route the rest to a person; the general comparison technique is in ",[19,730,732],{"href":731},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcompare-two-excel-files-for-differences-with-python\u002F","Compare Two Excel Files for Differences with Python",[25,734,736],{"id":735},"step-3-append-with-openpyxl-to-keep-the-formatting","Step 3 — Append with openpyxl to keep the formatting",[10,738,739,740,743],{},"Rewriting the master with ",[37,741,742],{},"to_excel"," would discard the header colours, the number formats and the table. Append the new rows below the existing data instead, copying the formats from the last existing row, and extend the table's range:",[30,745,747],{"className":78,"code":746,"language":80,"meta":35,"style":35},"from copy import copy\nfrom openpyxl import load_workbook\n\ndef append_rows(path, sheet, frame, table_name):\n    wb = load_workbook(path)\n    ws = wb[sheet]\n    template_row = ws.max_row\n    for row in frame.itertuples(index=False):\n        values = [v.to_pydatetime() if isinstance(v, pd.Timestamp) else v for v in row]\n        ws.append(values)\n        for col, cell in enumerate(ws[ws.max_row], start=1):\n            src = ws.cell(template_row, col)\n            cell.number_format = src.number_format\n            cell.font, cell.alignment = copy(src.font), copy(src.alignment)\n    table = ws.tables[table_name]\n    first, _ = table.ref.split(\":\")\n    table.ref = f\"{first}:{ws.cell(ws.max_row, ws.max_column).coordinate}\"\n    wb.save(path)\n    return ws.max_row - 1\n\nrows_now = append_rows(\"master_invoices.xlsx\", \"Invoices\", new_rows[master.columns], \"Invoices\")\nprint(\"rows in master:\", rows_now)\nprint(load_workbook(\"master_invoices.xlsx\")[\"Invoices\"].tables[\"Invoices\"].ref)\n",[37,748,749,762,774,778,790,800,810,820,841,875,880,906,916,926,937,948,964,997,1003,1018,1023,1047,1060],{"__ignoreMap":35},[40,750,751,754,757,759],{"class":42,"line":43},[40,752,753],{"class":87},"from",[40,755,756],{"class":91}," copy ",[40,758,88],{"class":87},[40,760,761],{"class":91}," copy\n",[40,763,764,766,769,771],{"class":42,"line":101},[40,765,753],{"class":87},[40,767,768],{"class":91}," openpyxl ",[40,770,88],{"class":87},[40,772,773],{"class":91}," load_workbook\n",[40,775,776],{"class":42,"line":108},[40,777,105],{"emptyLinePlaceholder":104},[40,779,780,783,787],{"class":42,"line":138},[40,781,782],{"class":87},"def",[40,784,786],{"class":785},"s_Opv"," append_rows",[40,788,789],{"class":91},"(path, sheet, frame, table_name):\n",[40,791,792,795,797],{"class":42,"line":153},[40,793,794],{"class":91},"    wb ",[40,796,114],{"class":87},[40,798,799],{"class":91}," load_workbook(path)\n",[40,801,802,805,807],{"class":42,"line":166},[40,803,804],{"class":91},"    ws ",[40,806,114],{"class":87},[40,808,809],{"class":91}," wb[sheet]\n",[40,811,812,815,817],{"class":42,"line":171},[40,813,814],{"class":91},"    template_row ",[40,816,114],{"class":87},[40,818,819],{"class":91}," ws.max_row\n",[40,821,822,824,827,829,832,834,836,838],{"class":42,"line":197},[40,823,633],{"class":87},[40,825,826],{"class":91}," row ",[40,828,572],{"class":87},[40,830,831],{"class":91}," frame.itertuples(",[40,833,333],{"class":126},[40,835,114],{"class":87},[40,837,191],{"class":156},[40,839,840],{"class":91},"):\n",[40,842,843,846,848,851,854,857,860,863,866,868,870,872],{"class":42,"line":217},[40,844,845],{"class":91},"        values ",[40,847,114],{"class":87},[40,849,850],{"class":91}," [v.to_pydatetime() ",[40,852,853],{"class":87},"if",[40,855,856],{"class":156}," isinstance",[40,858,859],{"class":91},"(v, pd.Timestamp) ",[40,861,862],{"class":87},"else",[40,864,865],{"class":91}," v ",[40,867,566],{"class":87},[40,869,865],{"class":91},[40,871,572],{"class":87},[40,873,874],{"class":91}," row]\n",[40,876,877],{"class":42,"line":240},[40,878,879],{"class":91},"        ws.append(values)\n",[40,881,882,885,888,890,893,896,899,901,904],{"class":42,"line":245},[40,883,884],{"class":87},"        for",[40,886,887],{"class":91}," col, cell ",[40,889,572],{"class":87},[40,891,892],{"class":156}," enumerate",[40,894,895],{"class":91},"(ws[ws.max_row], ",[40,897,898],{"class":126},"start",[40,900,114],{"class":87},[40,902,903],{"class":156},"1",[40,905,840],{"class":91},[40,907,908,911,913],{"class":42,"line":290},[40,909,910],{"class":91},"            src ",[40,912,114],{"class":87},[40,914,915],{"class":91}," ws.cell(template_row, col)\n",[40,917,918,921,923],{"class":42,"line":325},[40,919,920],{"class":91},"            cell.number_format ",[40,922,114],{"class":87},[40,924,925],{"class":91}," src.number_format\n",[40,927,929,932,934],{"class":42,"line":928},14,[40,930,931],{"class":91},"            cell.font, cell.alignment ",[40,933,114],{"class":87},[40,935,936],{"class":91}," copy(src.font), copy(src.alignment)\n",[40,938,940,943,945],{"class":42,"line":939},15,[40,941,942],{"class":91},"    table ",[40,944,114],{"class":87},[40,946,947],{"class":91}," ws.tables[table_name]\n",[40,949,951,954,956,959,962],{"class":42,"line":950},16,[40,952,953],{"class":91},"    first, _ ",[40,955,114],{"class":87},[40,957,958],{"class":91}," table.ref.split(",[40,960,961],{"class":50},"\":\"",[40,963,135],{"class":91},[40,965,967,970,972,975,977,979,982,984,987,989,992,994],{"class":42,"line":966},17,[40,968,969],{"class":91},"    table.ref ",[40,971,114],{"class":87},[40,973,974],{"class":87}," f",[40,976,591],{"class":50},[40,978,595],{"class":594},[40,980,981],{"class":91},"first",[40,983,601],{"class":594},[40,985,986],{"class":50},":",[40,988,595],{"class":594},[40,990,991],{"class":91},"ws.cell(ws.max_row, ws.max_column).coordinate",[40,993,601],{"class":594},[40,995,996],{"class":50},"\"\n",[40,998,1000],{"class":42,"line":999},18,[40,1001,1002],{"class":91},"    wb.save(path)\n",[40,1004,1006,1009,1012,1015],{"class":42,"line":1005},19,[40,1007,1008],{"class":87},"    return",[40,1010,1011],{"class":91}," ws.max_row ",[40,1013,1014],{"class":87},"-",[40,1016,1017],{"class":156}," 1\n",[40,1019,1021],{"class":42,"line":1020},20,[40,1022,105],{"emptyLinePlaceholder":104},[40,1024,1026,1029,1031,1034,1036,1038,1040,1043,1045],{"class":42,"line":1025},21,[40,1027,1028],{"class":91},"rows_now ",[40,1030,114],{"class":87},[40,1032,1033],{"class":91}," append_rows(",[40,1035,120],{"class":50},[40,1037,123],{"class":91},[40,1039,132],{"class":50},[40,1041,1042],{"class":91},", new_rows[master.columns], ",[40,1044,132],{"class":50},[40,1046,135],{"class":91},[40,1048,1050,1052,1054,1057],{"class":42,"line":1049},22,[40,1051,200],{"class":156},[40,1053,203],{"class":91},[40,1055,1056],{"class":50},"\"rows in master:\"",[40,1058,1059],{"class":91},", rows_now)\n",[40,1061,1063,1065,1068,1070,1073,1075,1078,1080],{"class":42,"line":1062},23,[40,1064,200],{"class":156},[40,1066,1067],{"class":91},"(load_workbook(",[40,1069,120],{"class":50},[40,1071,1072],{"class":91},")[",[40,1074,132],{"class":50},[40,1076,1077],{"class":91},"].tables[",[40,1079,132],{"class":50},[40,1081,1082],{"class":91},"].ref)\n",[10,1084,1085,1088,1089,1092,1093,1096,1097,1100,1101,1104,1105,1108,1109,23],{},[37,1086,1087],{},"ws.append"," writes after the last used row; copying number format, font and alignment from the row above keeps dates as ",[37,1090,1091],{},"dd mmm yyyy"," and amounts with two decimals. The table's ",[37,1094,1095],{},"ref"," is widened from ",[37,1098,1099],{},"A1:E4"," to ",[37,1102,1103],{},"A1:E6",", so the new rows join the table — its banding, filters and any formulas that use structured references. Column order is forced with ",[37,1106,1107],{},"new_rows[master.columns]",", so a reordered extract cannot shift values into the wrong columns. Table resizing is covered in more detail in ",[19,1110,1112],{"href":1111},"\u002Fformatting-and-charting-excel-reports-with-python\u002Fcreating-excel-tables-and-autofilters-with-python\u002Fappend-rows-to-an-excel-table-and-resize-it\u002F","Append Rows to an Excel Table and Resize It",[369,1114,378,1120,378,1123,378,1126,378,1131,378,1138,378,1142,378,1148,378,1152,378,1156,378,1160,378,1164,378,1168,378,1172,378,1176,378,1180,378,1184,378,1188,378,1192,378,1196,378,1198,378,1201,378,1203,378,1206,378,1208,378,1211,378,1213,378,1216,378,1219,378,1223,378,1225,378,1228,378,1230,378,1233,378,1235,378,1238,378,1240,378,1243,378,1245,378,1248,378,1251,378,1255,378,1257,378,1260,378,1262,378,1265,378,1267,378,1270,378,1272,378,1275,378,1277,378,1280,378,1283,378,1287,378,1289,378,1292,378,1294,378,1297,378,1299,378,1302,378,1304,378,1307,378,1309,378,1311,378,1314,378,1318,378,1322,378,1326,378,1328,378,1331,378,1333,378,1336,378,1338,378,1341,378,1343,378,1345,378,1348,378,1352,378,1354,378,1356,378,1358,378,1361,378,1363,378,1366,378,1368,378,1371,378,1373],{"viewBox":1115,"role":372,"ariaLabelledBy":1116,"xmlns":376,"style":1119},"143 0 474 224",[1117,1118],"ap-master-t","ap-master-d","width:100%;max-width:474px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[380,1121,1122],{"id":1117},"The master after the weekly append",[384,1124,1125],{"id":1118},"The Invoices table with five rows. The original three invoices keep their values and formatting; INV-1004 and INV-1005 are appended below and the table range now ends at row 6.",[388,1127],{"x":1128,"y":391,"width":1129,"height":1130,"fill":394},"143","474","224",[388,1132],{"x":1133,"y":1134,"width":1135,"height":1136,"rx":391,"fill":1137,"stroke":467,"style":468},"159.1","16.0","32.0","24.0","#e7ebef",[388,1139],{"x":1140,"y":1134,"width":1141,"height":1136,"rx":391,"fill":1137,"stroke":467,"style":468},"191.1","76.0",[420,1143,1147],{"x":1144,"y":1145,"style":1146},"229.0","33.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[388,1149],{"x":1150,"y":1134,"width":1151,"height":1136,"rx":391,"fill":1137,"stroke":467,"style":468},"267.0","92.3",[420,1153,1155],{"x":1154,"y":1145,"style":1146},"313.2","B",[388,1157],{"x":1158,"y":1134,"width":1159,"height":1136,"rx":391,"fill":1137,"stroke":467,"style":468},"359.4","110.1",[420,1161,1163],{"x":1162,"y":1145,"style":1146},"414.4","C",[388,1165],{"x":1166,"y":1134,"width":1167,"height":1136,"rx":391,"fill":1137,"stroke":467,"style":468},"469.4","72.8",[420,1169,1171],{"x":1170,"y":1145,"style":1146},"505.8","D",[388,1173],{"x":1174,"y":1134,"width":1175,"height":1136,"rx":391,"fill":1137,"stroke":467,"style":468},"542.2","58.7",[420,1177,1179],{"x":1178,"y":1145,"style":1146},"571.6","E",[388,1181],{"x":1133,"y":1182,"width":1135,"height":1183,"rx":391,"fill":1137,"stroke":467,"style":468},"40.0","28.0",[420,1185,903],{"x":1186,"y":1187,"style":1146},"175.1","59.0",[388,1189],{"x":1140,"y":1182,"width":1141,"height":1183,"rx":391,"fill":399,"stroke":1190,"style":1191},"var(--teal,#0f9488)","stroke-width:1.5px",[420,1193,1195],{"x":1144,"y":1187,"style":1194},"font-size:10.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Invoice",[388,1197],{"x":1150,"y":1182,"width":1151,"height":1183,"rx":391,"fill":399,"stroke":1190,"style":1191},[420,1199,1200],{"x":1154,"y":1187,"style":1194},"Date",[388,1202],{"x":1158,"y":1182,"width":1159,"height":1183,"rx":391,"fill":399,"stroke":1190,"style":1191},[420,1204,1205],{"x":1162,"y":1187,"style":1194},"Customer",[388,1207],{"x":1166,"y":1182,"width":1167,"height":1183,"rx":391,"fill":399,"stroke":1190,"style":1191},[420,1209,1210],{"x":1170,"y":1187,"style":1194},"Amount",[388,1212],{"x":1174,"y":1182,"width":1175,"height":1183,"rx":391,"fill":399,"stroke":1190,"style":1191},[420,1214,1215],{"x":1178,"y":1187,"style":1194},"Status",[388,1217],{"x":1133,"y":1218,"width":1135,"height":1183,"rx":391,"fill":1137,"stroke":467,"style":468},"68.0",[420,1220,1222],{"x":1186,"y":1221,"style":1146},"87.0","2",[388,1224],{"x":1140,"y":1218,"width":1141,"height":1183,"rx":391,"fill":394,"stroke":467,"style":468},[420,1226,448],{"x":1144,"y":1221,"style":1227},"font-size:10.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle",[388,1229],{"x":1150,"y":1218,"width":1151,"height":1183,"rx":391,"fill":394,"stroke":467,"style":468},[420,1231,1232],{"x":1154,"y":1221,"style":1227},"14 Sep 2026",[388,1234],{"x":1158,"y":1218,"width":1159,"height":1183,"rx":391,"fill":394,"stroke":467,"style":468},[420,1236,1237],{"x":1162,"y":1221,"style":1227},"Müller GmbH",[388,1239],{"x":1166,"y":1218,"width":1167,"height":1183,"rx":391,"fill":394,"stroke":467,"style":468},[420,1241,1242],{"x":1170,"y":1221,"style":1227},"1,200.00",[388,1244],{"x":1174,"y":1218,"width":1175,"height":1183,"rx":391,"fill":394,"stroke":467,"style":468},[420,1246,1247],{"x":1178,"y":1221,"style":1227},"Paid",[388,1249],{"x":1133,"y":1250,"width":1135,"height":1183,"rx":391,"fill":1137,"stroke":467,"style":468},"96.0",[420,1252,1254],{"x":1186,"y":1253,"style":1146},"115.0","3",[388,1256],{"x":1140,"y":1250,"width":1141,"height":1183,"rx":391,"fill":394,"stroke":467,"style":468},[420,1258,1259],{"x":1144,"y":1253,"style":1227},"INV-1002",[388,1261],{"x":1150,"y":1250,"width":1151,"height":1183,"rx":391,"fill":394,"stroke":467,"style":468},[420,1263,1264],{"x":1154,"y":1253,"style":1227},"15 Sep 2026",[388,1266],{"x":1158,"y":1250,"width":1159,"height":1183,"rx":391,"fill":394,"stroke":467,"style":468},[420,1268,1269],{"x":1162,"y":1253,"style":1227},"Café Señor",[388,1271],{"x":1166,"y":1250,"width":1167,"height":1183,"rx":391,"fill":394,"stroke":467,"style":468},[420,1273,1274],{"x":1170,"y":1253,"style":1227},"240.00",[388,1276],{"x":1174,"y":1250,"width":1175,"height":1183,"rx":391,"fill":394,"stroke":467,"style":468},[420,1278,1279],{"x":1178,"y":1253,"style":1227},"Open",[388,1281],{"x":1133,"y":1282,"width":1135,"height":1183,"rx":391,"fill":1137,"stroke":467,"style":468},"124.0",[420,1284,1286],{"x":1186,"y":1285,"style":1146},"143.0","4",[388,1288],{"x":1140,"y":1282,"width":1141,"height":1183,"rx":391,"fill":394,"stroke":467,"style":468},[420,1290,1291],{"x":1144,"y":1285,"style":1227},"INV-1003",[388,1293],{"x":1150,"y":1282,"width":1151,"height":1183,"rx":391,"fill":394,"stroke":467,"style":468},[420,1295,1296],{"x":1154,"y":1285,"style":1227},"16 Sep 2026",[388,1298],{"x":1158,"y":1282,"width":1159,"height":1183,"rx":391,"fill":394,"stroke":467,"style":468},[420,1300,1301],{"x":1162,"y":1285,"style":1227},"Øresund ApS",[388,1303],{"x":1166,"y":1282,"width":1167,"height":1183,"rx":391,"fill":394,"stroke":467,"style":468},[420,1305,1306],{"x":1170,"y":1285,"style":1227},"980.00",[388,1308],{"x":1174,"y":1282,"width":1175,"height":1183,"rx":391,"fill":394,"stroke":467,"style":468},[420,1310,1279],{"x":1178,"y":1285,"style":1227},[388,1312],{"x":1133,"y":1313,"width":1135,"height":1183,"rx":391,"fill":1137,"stroke":467,"style":468},"152.0",[420,1315,1317],{"x":1186,"y":1316,"style":1146},"171.0","5",[388,1319],{"x":1140,"y":1313,"width":1141,"height":1183,"rx":391,"fill":1320,"stroke":1321,"style":1191},"#f0f4ff","var(--brand,#5b5cf0)",[420,1323,1325],{"x":1144,"y":1316,"style":1324},"font-size:10.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","INV-1004",[388,1327],{"x":1150,"y":1313,"width":1151,"height":1183,"rx":391,"fill":1320,"stroke":1321,"style":1191},[420,1329,1330],{"x":1154,"y":1316,"style":1324},"21 Sep 2026",[388,1332],{"x":1158,"y":1313,"width":1159,"height":1183,"rx":391,"fill":1320,"stroke":1321,"style":1191},[420,1334,1335],{"x":1162,"y":1316,"style":1324},"Oakridge Farms",[388,1337],{"x":1166,"y":1313,"width":1167,"height":1183,"rx":391,"fill":1320,"stroke":1321,"style":1191},[420,1339,1340],{"x":1170,"y":1316,"style":1324},"560.00",[388,1342],{"x":1174,"y":1313,"width":1175,"height":1183,"rx":391,"fill":1320,"stroke":1321,"style":1191},[420,1344,1279],{"x":1178,"y":1316,"style":1324},[388,1346],{"x":1133,"y":1347,"width":1135,"height":1183,"rx":391,"fill":1137,"stroke":467,"style":468},"180.0",[420,1349,1351],{"x":1186,"y":1350,"style":1146},"199.0","6",[388,1353],{"x":1140,"y":1347,"width":1141,"height":1183,"rx":391,"fill":1320,"stroke":1321,"style":1191},[420,1355,429],{"x":1144,"y":1350,"style":1324},[388,1357],{"x":1150,"y":1347,"width":1151,"height":1183,"rx":391,"fill":1320,"stroke":1321,"style":1191},[420,1359,1360],{"x":1154,"y":1350,"style":1324},"22 Sep 2026",[388,1362],{"x":1158,"y":1347,"width":1159,"height":1183,"rx":391,"fill":1320,"stroke":1321,"style":1191},[420,1364,1365],{"x":1162,"y":1350,"style":1324},"Zenith Logistics",[388,1367],{"x":1166,"y":1347,"width":1167,"height":1183,"rx":391,"fill":1320,"stroke":1321,"style":1191},[420,1369,1370],{"x":1170,"y":1350,"style":1324},"75.00",[388,1372],{"x":1174,"y":1347,"width":1175,"height":1183,"rx":391,"fill":1320,"stroke":1321,"style":1191},[420,1374,1279],{"x":1178,"y":1350,"style":1324},[25,1376,1378],{"id":1377},"step-4-apply-approved-updates-in-place","Step 4 — Apply approved updates in place",[10,1380,1381],{},"For the changes the policy allows — here, status changes — update the matching cells in the master directly, again without rewriting the sheet:",[30,1383,1385],{"className":78,"code":1384,"language":80,"meta":35,"style":35},"def apply_updates(path, sheet, changes, key_col=\"A\", allowed=(\"Status\",)):\n    wb = load_workbook(path)\n    ws = wb[sheet]\n    header = {c.value: c.column for c in ws[1]}\n    rows = {ws[f\"{key_col}{r}\"].value: r for r in range(2, ws.max_row + 1)}\n    applied = []\n    for ch in changes.itertuples(index=False):\n        if ch.Column in allowed and ch.Invoice in rows:\n            ws.cell(rows[ch.Invoice], header[ch.Column], ch.New)\n            applied.append((ch.Invoice, ch.Column, ch.New))\n    wb.save(path)\n    return applied\n\nprint(\"applied:\", apply_updates(\"master_invoices.xlsx\", \"Invoices\", changes))\nprint(\"needs review:\", changes[changes[\"Column\"] != \"Status\"].to_dict(\"records\"))\n",[37,1386,1387,1414,1422,1430,1455,1513,1522,1542,1566,1571,1576,1580,1587,1591,1612],{"__ignoreMap":35},[40,1388,1389,1391,1394,1397,1399,1402,1405,1407,1409,1411],{"class":42,"line":43},[40,1390,782],{"class":87},[40,1392,1393],{"class":785}," apply_updates",[40,1395,1396],{"class":91},"(path, sheet, changes, key_col",[40,1398,114],{"class":87},[40,1400,1401],{"class":50},"\"A\"",[40,1403,1404],{"class":91},", allowed",[40,1406,114],{"class":87},[40,1408,203],{"class":91},[40,1410,513],{"class":50},[40,1412,1413],{"class":91},",)):\n",[40,1415,1416,1418,1420],{"class":42,"line":101},[40,1417,794],{"class":91},[40,1419,114],{"class":87},[40,1421,799],{"class":91},[40,1423,1424,1426,1428],{"class":42,"line":108},[40,1425,804],{"class":91},[40,1427,114],{"class":87},[40,1429,809],{"class":91},[40,1431,1432,1435,1437,1440,1442,1445,1447,1450,1452],{"class":42,"line":138},[40,1433,1434],{"class":91},"    header ",[40,1436,114],{"class":87},[40,1438,1439],{"class":91}," {c.value: c.column ",[40,1441,566],{"class":87},[40,1443,1444],{"class":91}," c ",[40,1446,572],{"class":87},[40,1448,1449],{"class":91}," ws[",[40,1451,903],{"class":156},[40,1453,1454],{"class":91},"]}\n",[40,1456,1457,1460,1462,1465,1467,1469,1471,1474,1477,1480,1482,1484,1487,1489,1492,1494,1497,1499,1501,1504,1507,1510],{"class":42,"line":153},[40,1458,1459],{"class":91},"    rows ",[40,1461,114],{"class":87},[40,1463,1464],{"class":91}," {ws[",[40,1466,588],{"class":87},[40,1468,591],{"class":50},[40,1470,595],{"class":594},[40,1472,1473],{"class":91},"key_col",[40,1475,1476],{"class":594},"}{",[40,1478,1479],{"class":91},"r",[40,1481,601],{"class":594},[40,1483,591],{"class":50},[40,1485,1486],{"class":91},"].value: r ",[40,1488,566],{"class":87},[40,1490,1491],{"class":91}," r ",[40,1493,572],{"class":87},[40,1495,1496],{"class":156}," range",[40,1498,203],{"class":91},[40,1500,1222],{"class":156},[40,1502,1503],{"class":91},", ws.max_row ",[40,1505,1506],{"class":87},"+",[40,1508,1509],{"class":156}," 1",[40,1511,1512],{"class":91},")}\n",[40,1514,1515,1518,1520],{"class":42,"line":166},[40,1516,1517],{"class":91},"    applied ",[40,1519,114],{"class":87},[40,1521,561],{"class":91},[40,1523,1524,1526,1529,1531,1534,1536,1538,1540],{"class":42,"line":171},[40,1525,633],{"class":87},[40,1527,1528],{"class":91}," ch ",[40,1530,572],{"class":87},[40,1532,1533],{"class":91}," changes.itertuples(",[40,1535,333],{"class":126},[40,1537,114],{"class":87},[40,1539,191],{"class":156},[40,1541,840],{"class":91},[40,1543,1544,1547,1550,1552,1555,1558,1561,1563],{"class":42,"line":197},[40,1545,1546],{"class":87},"        if",[40,1548,1549],{"class":91}," ch.Column ",[40,1551,572],{"class":87},[40,1553,1554],{"class":91}," allowed ",[40,1556,1557],{"class":87},"and",[40,1559,1560],{"class":91}," ch.Invoice ",[40,1562,572],{"class":87},[40,1564,1565],{"class":91}," rows:\n",[40,1567,1568],{"class":42,"line":217},[40,1569,1570],{"class":91},"            ws.cell(rows[ch.Invoice], header[ch.Column], ch.New)\n",[40,1572,1573],{"class":42,"line":240},[40,1574,1575],{"class":91},"            applied.append((ch.Invoice, ch.Column, ch.New))\n",[40,1577,1578],{"class":42,"line":245},[40,1579,1002],{"class":91},[40,1581,1582,1584],{"class":42,"line":290},[40,1583,1008],{"class":87},[40,1585,1586],{"class":91}," applied\n",[40,1588,1589],{"class":42,"line":325},[40,1590,105],{"emptyLinePlaceholder":104},[40,1592,1593,1595,1597,1600,1603,1605,1607,1609],{"class":42,"line":928},[40,1594,200],{"class":156},[40,1596,203],{"class":91},[40,1598,1599],{"class":50},"\"applied:\"",[40,1601,1602],{"class":91},", apply_updates(",[40,1604,120],{"class":50},[40,1606,123],{"class":91},[40,1608,132],{"class":50},[40,1610,1611],{"class":91},", changes))\n",[40,1613,1614,1616,1618,1621,1624,1626,1628,1630,1633,1636,1639],{"class":42,"line":939},[40,1615,200],{"class":156},[40,1617,203],{"class":91},[40,1619,1620],{"class":50},"\"needs review:\"",[40,1622,1623],{"class":91},", changes[changes[",[40,1625,659],{"class":50},[40,1627,304],{"class":91},[40,1629,609],{"class":87},[40,1631,1632],{"class":50}," \"Status\"",[40,1634,1635],{"class":91},"].to_dict(",[40,1637,1638],{"class":50},"\"records\"",[40,1640,340],{"class":91},[10,1642,1643],{},"INV-1002 becomes Paid in the master; INV-1003's amount correction is reported for review instead of being applied. Mapping keys to row numbers and header names to column numbers makes the update independent of where things sit in the sheet.",[25,1645,1647],{"id":1646},"step-5-make-reruns-harmless-and-keep-a-log","Step 5 — Make reruns harmless and keep a log",[10,1649,1650],{},"Jobs get rerun — after a failure, by a colleague, by a scheduler that fired twice. Because new rows are found by key, a rerun finds nothing new:",[30,1652,1654],{"className":78,"code":1653,"language":80,"meta":35,"style":35},"import datetime as dt\nimport os\n\nmaster_after = pd.read_excel(\"master_invoices.xlsx\", sheet_name=\"Invoices\")\nrerun_new = extract[~extract[KEY].isin(master_after[KEY])]\nprint(\"new rows on rerun:\", len(rerun_new), \"| duplicate keys in master:\", master_after[KEY].duplicated().sum())\n\nlog = pd.DataFrame([{\"Run at\": dt.datetime(2026, 10, 2, 7, 30), \"Extract\": \"extract_week39.xlsx\",\n                     \"Appended\": len(new_rows), \"Status updates\": 1, \"For review\": len(changes) - 1}])\nlog.to_csv(\"master_update_log.csv\", mode=\"a\", index=False, header=not os.path.exists(\"master_update_log.csv\"))\nprint(log.to_string(index=False))\n",[37,1655,1656,1668,1675,1679,1700,1726,1754,1758,1810,1848,1889],{"__ignoreMap":35},[40,1657,1658,1660,1663,1665],{"class":42,"line":43},[40,1659,88],{"class":87},[40,1661,1662],{"class":91}," datetime ",[40,1664,95],{"class":87},[40,1666,1667],{"class":91}," dt\n",[40,1669,1670,1672],{"class":42,"line":101},[40,1671,88],{"class":87},[40,1673,1674],{"class":91}," os\n",[40,1676,1677],{"class":42,"line":108},[40,1678,105],{"emptyLinePlaceholder":104},[40,1680,1681,1684,1686,1688,1690,1692,1694,1696,1698],{"class":42,"line":138},[40,1682,1683],{"class":91},"master_after ",[40,1685,114],{"class":87},[40,1687,117],{"class":91},[40,1689,120],{"class":50},[40,1691,123],{"class":91},[40,1693,127],{"class":126},[40,1695,114],{"class":87},[40,1697,132],{"class":50},[40,1699,135],{"class":91},[40,1701,1702,1705,1707,1710,1713,1716,1718,1721,1723],{"class":42,"line":153},[40,1703,1704],{"class":91},"rerun_new ",[40,1706,114],{"class":87},[40,1708,1709],{"class":91}," extract[",[40,1711,1712],{"class":87},"~",[40,1714,1715],{"class":91},"extract[",[40,1717,157],{"class":156},[40,1719,1720],{"class":91},"].isin(master_after[",[40,1722,157],{"class":156},[40,1724,1725],{"class":91},"])]\n",[40,1727,1728,1730,1732,1735,1737,1740,1743,1746,1749,1751],{"class":42,"line":166},[40,1729,200],{"class":156},[40,1731,203],{"class":91},[40,1733,1734],{"class":50},"\"new rows on rerun:\"",[40,1736,123],{"class":91},[40,1738,1739],{"class":156},"len",[40,1741,1742],{"class":91},"(rerun_new), ",[40,1744,1745],{"class":50},"\"| duplicate keys in master:\"",[40,1747,1748],{"class":91},", master_after[",[40,1750,157],{"class":156},[40,1752,1753],{"class":91},"].duplicated().sum())\n",[40,1755,1756],{"class":42,"line":171},[40,1757,105],{"emptyLinePlaceholder":104},[40,1759,1760,1763,1765,1768,1771,1774,1777,1779,1782,1784,1786,1788,1791,1793,1796,1799,1802,1805,1807],{"class":42,"line":197},[40,1761,1762],{"class":91},"log ",[40,1764,114],{"class":87},[40,1766,1767],{"class":91}," pd.DataFrame([{",[40,1769,1770],{"class":50},"\"Run at\"",[40,1772,1773],{"class":91},": dt.datetime(",[40,1775,1776],{"class":156},"2026",[40,1778,123],{"class":91},[40,1780,1781],{"class":156},"10",[40,1783,123],{"class":91},[40,1785,1222],{"class":156},[40,1787,123],{"class":91},[40,1789,1790],{"class":156},"7",[40,1792,123],{"class":91},[40,1794,1795],{"class":156},"30",[40,1797,1798],{"class":91},"), ",[40,1800,1801],{"class":50},"\"Extract\"",[40,1803,1804],{"class":91},": ",[40,1806,148],{"class":50},[40,1808,1809],{"class":91},",\n",[40,1811,1812,1815,1817,1819,1822,1825,1827,1829,1831,1834,1836,1838,1841,1843,1845],{"class":42,"line":217},[40,1813,1814],{"class":50},"                     \"Appended\"",[40,1816,1804],{"class":91},[40,1818,1739],{"class":156},[40,1820,1821],{"class":91},"(new_rows), ",[40,1823,1824],{"class":50},"\"Status updates\"",[40,1826,1804],{"class":91},[40,1828,903],{"class":156},[40,1830,123],{"class":91},[40,1832,1833],{"class":50},"\"For review\"",[40,1835,1804],{"class":91},[40,1837,1739],{"class":156},[40,1839,1840],{"class":91},"(changes) ",[40,1842,1014],{"class":87},[40,1844,1509],{"class":156},[40,1846,1847],{"class":91},"}])\n",[40,1849,1850,1853,1856,1858,1861,1863,1866,1868,1870,1872,1874,1876,1879,1882,1885,1887],{"class":42,"line":240},[40,1851,1852],{"class":91},"log.to_csv(",[40,1854,1855],{"class":50},"\"master_update_log.csv\"",[40,1857,123],{"class":91},[40,1859,1860],{"class":126},"mode",[40,1862,114],{"class":87},[40,1864,1865],{"class":50},"\"a\"",[40,1867,123],{"class":91},[40,1869,333],{"class":126},[40,1871,114],{"class":87},[40,1873,191],{"class":156},[40,1875,123],{"class":91},[40,1877,1878],{"class":126},"header",[40,1880,1881],{"class":87},"=not",[40,1883,1884],{"class":91}," os.path.exists(",[40,1886,1855],{"class":50},[40,1888,340],{"class":91},[40,1890,1891,1893,1896,1898,1900,1902],{"class":42,"line":245},[40,1892,200],{"class":156},[40,1894,1895],{"class":91},"(log.to_string(",[40,1897,333],{"class":126},[40,1899,114],{"class":87},[40,1901,191],{"class":156},[40,1903,340],{"class":91},[10,1905,1906,1907,23],{},"The rerun appends zero rows and the master contains no duplicate invoice numbers — the property worth asserting in a test. The log records what each run did, which is what someone needs when a figure in the master is questioned weeks later. For locking against two runs at the same moment, see ",[19,1908,1910],{"href":1909},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fprevent-overlapping-runs-of-an-excel-report-job\u002F","Prevent Overlapping Runs of an Excel Report Job",[369,1912,378,1918,378,1921,378,1924,378,1928,378,1934,378,1943,378,1949,378,1952,378,1957,378,1962,378,1967,378,1970,378,1977,378,1982,378,1986,378,1989,378,1993,378,1996,378,1999,378,2006,378,2011,378,2015,378,2019,378,2023,378,2027,378,2030,378,2035,378,2042,378,2048,378,2051,378,2054,378,2057,378,2061,378,2065,378,2068,378,2071,378,2077,378,2081,378,2084],{"viewBox":1913,"role":372,"ariaLabelledBy":1914,"xmlns":376,"style":1917},"4 0 752 178",[1915,1916],"ap-flow-t","ap-flow-d","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[380,1919,1920],{"id":1915},"One safe update of the master workbook",[384,1922,1923],{"id":1916},"The extract is de-duplicated on its key, compared with the master to split new rows from existing ones, new rows are appended with formatting, approved changes are applied in place, the rest are sent for review, and the run is logged.",[388,1925],{"x":1286,"y":391,"width":1926,"height":1927,"fill":394},"752","178",[388,1929],{"x":1930,"y":1134,"width":1931,"height":1932,"rx":1933,"fill":1137,"stroke":467,"style":415},"20.0","115.2","145.5","12",[1935,1936,1938],"g",{"transform":1937},"translate(65.6,46.5) scale(1.000)",[396,1939],{"d":1940,"fill":400,"stroke":1941,"style":1942},"M3 5h18l-7 8v6l-4 2v-8z","var(--text,#172033)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[420,1944,1948],{"x":1945,"y":1946,"style":1947},"77.6","93.0","font-size:13.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","De-duplicate",[420,1950,454],{"x":1945,"y":1951,"style":1947},"110.5",[420,1953,1956],{"x":1945,"y":1954,"style":1955},"130.0","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","on the key",[42,1958],{"x1":1959,"y1":1960,"x2":1961,"y2":1960,"stroke":1321,"style":415},"140.2","88.8","162.2",[1963,1964],"polygon",{"points":1965,"fill":1966},"166.2,88.8 157.2,93.8 157.2,83.8","#5b5cf0",[388,1968],{"x":1969,"y":1134,"width":1931,"height":1932,"rx":1933,"fill":1320,"stroke":1321,"style":415},"171.2",[1935,1971,1973],{"transform":1972},"translate(216.8,46.5) scale(1.000)",[396,1974],{"d":1975,"fill":400,"stroke":1976,"style":1942},"M11 4a7 7 0 1 0 .01 0z M16 16l5 5","var(--brand-strong,#4338ca)",[420,1978,1981],{"x":1979,"y":1946,"style":1980},"228.8","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Anti-join",[420,1983,1985],{"x":1979,"y":1984,"style":1955},"112.5","new vs existing",[420,1987,1988],{"x":1979,"y":1954,"style":1955},"keys",[42,1990],{"x1":1991,"y1":1960,"x2":1992,"y2":1960,"stroke":1321,"style":415},"291.4","313.4",[1963,1994],{"points":1995,"fill":1966},"317.4,88.8 308.4,93.8 308.4,83.8",[388,1997],{"x":1998,"y":1134,"width":1931,"height":1932,"rx":1933,"fill":399,"stroke":1190,"style":415},"322.4",[1935,2000,2002],{"transform":2001},"translate(368.0,37.8) scale(1.000)",[396,2003],{"d":2004,"fill":400,"stroke":2005,"style":1942},"M3 5h18v14H3z M3 10h18 M3 15h18 M9 5v14","var(--teal-ink,#0b6157)",[420,2007,2010],{"x":436,"y":2008,"style":2009},"84.2","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Append new",[420,2012,2014],{"x":436,"y":2013,"style":2009},"101.8","rows",[420,2016,2018],{"x":436,"y":2017,"style":1955},"121.2","openpyxl +",[420,2020,2022],{"x":436,"y":2021,"style":1955},"138.8","table resize",[42,2024],{"x1":2025,"y1":1960,"x2":2026,"y2":1960,"stroke":1321,"style":415},"442.6","464.6",[1963,2028],{"points":2029,"fill":1966},"468.6,88.8 459.6,93.8 459.6,83.8",[388,2031],{"x":2032,"y":1134,"width":1931,"height":1932,"rx":1933,"fill":2033,"stroke":2034,"style":415},"473.6","#fdefd8","var(--gold,#b4740a)",[1935,2036,2038],{"transform":2037},"translate(519.2,29.0) scale(1.000)",[396,2039],{"d":2040,"fill":400,"stroke":2041,"style":1942},"M5 12.5l4.5 4.5L19 7","var(--gold-ink,#7a4e06)",[420,2043,2047],{"x":2044,"y":2045,"style":2046},"531.2","75.5","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Apply \u002F",[420,2049,2050],{"x":2044,"y":1946,"style":2046},"review",[420,2052,2053],{"x":2044,"y":1951,"style":2046},"changes",[420,2055,2056],{"x":2044,"y":1954,"style":1955},"per column",[420,2058,2060],{"x":2044,"y":2059,"style":1955},"147.5","policy",[42,2062],{"x1":2063,"y1":1960,"x2":2064,"y2":1960,"stroke":1321,"style":415},"593.8","615.8",[1963,2066],{"points":2067,"fill":1966},"619.8,88.8 610.8,93.8 610.8,83.8",[388,2069],{"x":2070,"y":1134,"width":1931,"height":1932,"rx":1933,"fill":1137,"stroke":467,"style":415},"624.8",[1935,2072,2074],{"transform":2073},"translate(670.4,46.5) scale(1.000)",[396,2075],{"d":2076,"fill":400,"stroke":1941,"style":1942},"M6 3h9l4 4v14H6z M15 3v4h4",[420,2078,2080],{"x":2079,"y":1946,"style":1947},"682.4","Log the run",[420,2082,2083],{"x":2079,"y":1984,"style":1955},"idempotent",[420,2085,2086],{"x":2079,"y":1954,"style":1955},"reruns",[25,2088,2090],{"id":2089},"performance-notes","Performance notes",[10,2092,2093,2094,23],{},"Reading the master's key column and anti-joining a weekly extract is fast at any realistic size. The cost that grows is the openpyxl load and save of the master: a workbook of a few thousand rows takes well under a second, one with several hundred thousand rows takes many seconds and a lot of memory every week, because openpyxl rewrites the whole file to add two rows. When a master grows that large, it has outgrown Excel as a store — keep the records in a database or Parquet file, append there, and generate the workbook from it as a report, as in ",[19,2095,2097],{"href":2096},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-an-excel-file-into-a-sql-database-with-pandas\u002F","Load an Excel File into a SQL Database with pandas",[25,2099,2101],{"id":2100},"common-pitfalls","Common pitfalls",[2103,2104,2105,2113,2119,2125,2131],"ul",{},[2106,2107,2108,2112],"li",{},[2109,2110,2111],"strong",{},"Comparing whole rows."," A changed row looks new and is appended a second time; compare on the key.",[2106,2114,2115,2118],{},[2109,2116,2117],{},"Trusting the extract to be unique."," De-duplicate it on the key before anything else.",[2106,2120,2121,2124],{},[2109,2122,2123],{},"Appending by date."," Late-arriving rows with earlier dates are missed; use keys.",[2106,2126,2127,2130],{},[2109,2128,2129],{},"Rewriting the master with to_excel."," Formatting, tables and other sheets are lost; append with openpyxl.",[2106,2132,2133,2136,2137,2140,2141,2143,2144,23],{},[2109,2134,2135],{},"Keys that differ only in formatting."," ",[37,2138,2139],{},"INV-1004 "," and ",[37,2142,1325],{}," do not match; normalise keys first, as in ",[19,2145,2147],{"href":2146},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fmerge-on-keys-that-differ-in-case-and-spacing-with-pandas\u002F","Merge on Keys That Differ in Case and Spacing with pandas",[25,2149,2151],{"id":2150},"conclusion","Conclusion",[10,2153,2154],{},"To update a master workbook from overlapping extracts, de-duplicate the extract on its key, anti-join it against the master to find genuinely new rows, and list existing rows whose values changed. Append the new rows with openpyxl below the existing data, copying formats and widening the table, apply only the changes your policy allows, and send the rest for review. Because everything is keyed, reruns add nothing, and a short log records what every run did.",[25,2156,2158],{"id":2157},"frequently-asked-questions","Frequently asked questions",[10,2160,2161,2164],{},[2109,2162,2163],{},"How do I append only new rows to an existing Excel file with pandas?","\nRead the master and the new data, merge the new data against the master's key column with how=\"left\" and indicator=True, keep the rows marked left_only, and append those rows to the master.",[10,2166,2167,2170],{},[2109,2168,2169],{},"How do I avoid duplicate rows when appending to Excel?","\nCompare on a unique key such as an invoice number rather than whole rows, remove duplicate keys inside the new data first, and check after appending that the master has no duplicated keys.",[10,2172,2173,2176],{},[2109,2174,2175],{},"How do I append rows without losing the master workbook's formatting?","\nLoad the master with openpyxl, write the new rows with ws.append below the existing data, copy the number formats from the row above, extend any Excel table's ref to include the new rows, and save.",[10,2178,2179,2182],{},[2109,2180,2181],{},"What should happen to rows that already exist but have changed?","\nThat depends on the column: status updates are usually applied automatically, while corrections to amounts often need review. Compare the overlapping rows column by column and apply only the changes your policy allows.",[25,2184,2186],{"id":2185},"related","Related",[2103,2188,2189,2195,2202,2209],{},[2106,2190,2191,2192,2194],{},"Up one level: ",[19,2193,22],{"href":21}," — combining data from several sources.",[2106,2196,2197,2201],{},[19,2198,2200],{"href":2199},"\u002Fgetting-started-with-python-excel-automation\u002Fwriting-dataframes-to-excel-with-pandas\u002Fappend-a-dataframe-to-an-existing-excel-file-with-pandas\u002F","Append a DataFrame to an Existing Excel File with pandas"," — the pandas writer options for appending.",[2106,2203,2204,2208],{},[19,2205,2207],{"href":2206},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fupdate-one-excel-file-from-another-with-pandas\u002F","Update One Excel File from Another with pandas"," — updating values by key.",[2106,2210,2211,2215],{},[19,2212,2214],{"href":2213},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fupsert-excel-rows-into-a-database-table\u002F","Upsert Excel Rows into a Database Table"," — the same pattern when the master is a database.",[2217,2218,2219],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}",{"title":35,"searchDepth":101,"depth":101,"links":2221},[2222,2223,2224,2225,2226,2227,2228,2229,2230,2231,2232],{"id":27,"depth":101,"text":28},{"id":74,"depth":101,"text":75},{"id":476,"depth":101,"text":477},{"id":735,"depth":101,"text":736},{"id":1377,"depth":101,"text":1378},{"id":1646,"depth":101,"text":1647},{"id":2089,"depth":101,"text":2090},{"id":2100,"depth":101,"text":2101},{"id":2150,"depth":101,"text":2151},{"id":2157,"depth":101,"text":2158},{"id":2185,"depth":101,"text":2186},"2026-10-02","Weekly extracts overlap — the same invoices appear in this week's file and last week's — and appending them blindly fills the master workbook with duplicates. Find genuinely new rows with a key-based anti-join, decide what to do with rows that changed, append with openpyxl so the master's formatting and table survive, keep reruns idempotent, and log every run.","md",[2237,2239,2241,2243],{"q":2163,"a":2238},"Read the master and the new data, merge the new data against the master's key column with how=\"left\" and indicator=True, keep the rows marked left_only, and append those rows to the master.",{"q":2169,"a":2240},"Compare on a unique key such as an invoice number rather than whole rows, remove duplicate keys inside the new data first, and check after appending that the master has no duplicated keys.",{"q":2175,"a":2242},"Load the master with openpyxl, write the new rows with ws.append below the existing data, copy the number formats from the row above, extend any Excel table's ref to include the new rows, and save.",{"q":2181,"a":2244},"That depends on the column: status updates are usually applied automatically, while corrections to amounts often need review. Compare the overlapping rows column by column and apply only the changes your policy allows.","append new rows to a master excel file without duplicates",null,{"breadcrumb":2248},[2249,2252,2255],{"name":2250,"item":2251},"Home","\u002F",{"name":2253,"item":2254},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":2256,"item":21},"Merging and Joining Excel DataFrames with Pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fappend-new-rows-to-a-master-excel-file-without-duplicates",{"title":5,"description":2259},"Update a master workbook from overlapping extracts: anti-join on a key to find new rows, detect changed rows, append below the existing data with openpyxl, extend the Excel table, make reruns safe, and record a run log.","append-new-rows-to-a-master-excel-file-without-duplicates","advanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fappend-new-rows-to-a-master-excel-file-without-duplicates\u002Findex",[2263,2264,2265],"joins-and-lookups","sheets-and-files","writing-excel","how-to","XWNf2QCaswrOiRz-ad66k1i3OjBhRp-IDRf0RCaAl2w",[2269,2272],{"title":2256,"path":2270,"stem":2271,"children":-1},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes","advanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Findex",{"title":2273,"path":2274,"stem":2275,"children":-1},"Concatenate Excel Sheets with Different Columns","\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fconcatenate-excel-sheets-with-different-columns","advanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fconcatenate-excel-sheets-with-different-columns\u002Findex",[2277,2279],{"path":2278,"title":2253},"\u002Fadvanced-data-transformation-and-cleaning",{"path":2270,"title":2256},[],[2282,2284,2287,2290,2293,2296],{"path":2283,"title":732,"kind":2266},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcompare-two-excel-files-for-differences-with-python",{"path":2285,"title":2286,"kind":2266},"\u002Fautomating-reporting-workflows\u002Fbuilding-finance-and-operations-reports\u002Fconvert-multi-currency-transactions-with-an-fx-rate-table","Convert Multi-Currency Transactions with an FX Rate Table",{"path":2288,"title":2289,"kind":2266},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-differences-between-two-columns-with-openpyxl","Highlight Differences Between Two Columns with openpyxl",{"path":2291,"title":2292,"kind":2266},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fmap-free-text-job-titles-to-standard-categories","Map Free-Text Job Titles to Standard Categories",{"path":2294,"title":2295,"kind":2266},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-category-values-with-a-mapping-table","Standardise Category Values with a Mapping Table in pandas",{"path":2297,"title":2298,"kind":2266},"\u002Fadvanced-data-transformation-and-cleaning\u002Fexcel-formula-equivalents-in-pandas\u002Fapproximate-match-vlookup-in-pandas","Approximate-Match VLOOKUP for Tiered Rates in pandas",1790972096145]