[{"data":1,"prerenderedAt":2037},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-references-between-excel-sheets":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-references-between-excel-sheets":2003,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-references-between-excel-sheets":2012,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-references-between-excel-sheets":2017,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-references-between-excel-sheets":2018},{"id":4,"title":5,"body":6,"dateModified":1968,"datePublished":1968,"description":1969,"extension":1970,"faq":1971,"keyword":1980,"kind":1981,"meta":1982,"navigation":220,"path":1991,"seo":1992,"slug":1995,"stem":1996,"topics":1997,"type":2001,"__hash__":2002},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-references-between-excel-sheets\u002Findex.md","Check References Between Excel Sheets with pandas",{"type":7,"value":8,"toc":1952},"minimark",[9,29,146,151,179,185,189,192,354,368,372,375,502,509,513,520,803,821,918,922,925,1085,1096,1100,1103,1225,1228,1232,1235,1478,1764,1768,1779,1783,1799,1803,1851,1855,1864,1868,1877,1881,1897,1903,1909,1915,1919,1948],[10,11,12,13,17,18,22,23,28],"p",{},"Workbooks that grow by hand end up as small databases: an Orders sheet whose Customer ID column refers to the Customers sheet, a SKU column that should exist on the Products sheet. Excel does not enforce any of those links. A VLOOKUP silently returns ",[14,15,16],"code",{},"#N\u002FA"," for an order whose customer was mistyped, or the ",[19,20,21],"em",{},"wrong"," customer when the lookup sheet has the same ID twice. Before a workbook like this feeds a report or a database load, check every reference the way a database would check a foreign key. This page, part of ",[24,25,27],"a",{"href":26},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002F","Validating Excel Data with Python",", builds those checks with pandas and turns the failures into a list of Excel rows to fix.",[30,31,39,40,39,44,39,48,39,56,39,66,39,77,39,84,39,89,39,97,39,104,39,110,39,115,39,118,39,123,39,126,39,132,39,136,39,140,39,143],"svg",{"viewBox":32,"role":33,"ariaLabelledBy":34,"xmlns":37,"style":38},"4 0 752 228","img",[35,36],"ref-map-t","ref-map-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[41,42,43],"title",{"id":35},"The references in one workbook",[45,46,47],"desc",{"id":36},"The Orders sheet points at two other sheets. Its Customer ID column must exist on the Customers sheet, where Customer ID must be unique and the customer active. Its SKU column must exist on the Products sheet, where SKU must be unique.",[49,50],"rect",{"x":51,"y":52,"width":53,"height":54,"fill":55},"4","0","752","228","#ffffff",[49,57],{"x":58,"y":59,"width":60,"height":61,"rx":62,"fill":63,"stroke":64,"style":65},"20.0","67.5","220.0","93.0","12","#f0f4ff","var(--brand,#5b5cf0)","stroke-width:2px",[67,68,70],"g",{"transform":69},"translate(118.0,80.5) scale(1.000)",[71,72],"path",{"d":73,"fill":74,"stroke":75,"style":76},"M4 4h16v16H4z M4 9h16 M4 14h16 M10 4v16","none","var(--brand-strong,#4338ca)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[78,79,83],"text",{"x":80,"y":81,"style":82},"130.0","127.0","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Orders sheet",[78,85,88],{"x":80,"y":86,"style":87},"146.5","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","Customer ID, SKU",[49,90],{"x":91,"y":92,"width":93,"height":94,"rx":62,"fill":95,"stroke":96,"style":65},"460.0","16.0","280.0","92.0","#d9f4f1","var(--teal,#0f9488)",[67,98,100],{"transform":99},"translate(588.0,29.0) scale(1.000)",[71,101],{"d":102,"fill":74,"stroke":103,"style":76},"M9 11a3.5 3.5 0 1 0 0-7 3.5 3.5 0 0 0 0 7z M2.5 20c0-3.6 2.9-6 6.5-6s6.5 2.4 6.5 6 M16 4.5a3.3 3.3 0 0 1 0 6.3 M18 14.3c2.2.7 3.5 2.8 3.5 5.7","var(--teal-ink,#0b6157)",[78,105,109],{"x":106,"y":107,"style":108},"600.0","75.0","font-size:13px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Customers sheet",[78,111,114],{"x":106,"y":112,"style":113},"94.0","font-size:11px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","ID exists, unique, active",[71,116],{"d":117,"fill":74,"stroke":64,"style":65},"M244.0,114.0 C349.0,114.0 349.0,62.0 450.0,62.0",[119,120],"polygon",{"points":121,"fill":122},"454.0,62.0 445.0,67.0 445.0,57.0","#5b5cf0",[49,124],{"x":91,"y":125,"width":93,"height":94,"rx":62,"fill":95,"stroke":96,"style":65},"120.0",[67,127,129],{"transform":128},"translate(588.0,133.0) scale(1.000)",[71,130],{"d":131,"fill":74,"stroke":103,"style":76},"M12 2.5l8.5 4.7v9.6L12 21.5l-8.5-4.7V7.2z M3.5 7.2l8.5 4.7 8.5-4.7 M12 11.9v9.6",[78,133,135],{"x":106,"y":134,"style":108},"179.0","Products sheet",[78,137,139],{"x":106,"y":138,"style":113},"198.0","SKU exists, unique",[71,141],{"d":142,"fill":74,"stroke":64,"style":65},"M244.0,114.0 C349.0,114.0 349.0,166.0 450.0,166.0",[119,144],{"points":145,"fill":122},"454.0,166.0 445.0,171.0 445.0,161.0",[147,148,150],"h2",{"id":149},"prerequisites","Prerequisites",[152,153,158],"pre",{"className":154,"code":155,"language":156,"meta":157,"style":157},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[14,159,160],{"__ignoreMap":157},[161,162,165,169,173,176],"span",{"class":163,"line":164},"line",1,[161,166,168],{"class":167},"sMTad","pip",[161,170,172],{"class":171},"srMev"," install",[161,174,175],{"class":171}," pandas",[161,177,178],{"class":171}," openpyxl\n",[10,180,181,184],{},[14,182,183],{},"sales_book.xlsx"," has three sheets: eight orders, five customer rows (one ID listed twice, one customer closed) and five products. Some orders use keys with the wrong case or a trailing space, one refers to a customer that does not exist, one to an unknown SKU, and one has no customer at all.",[147,186,188],{"id":187},"step-1-load-the-sheets-and-keep-excel-row-numbers","Step 1 — Load the sheets and keep Excel row numbers",[10,190,191],{},"Read every sheet in one call and add each row's Excel row number before any filtering, so every error can point at a cell:",[152,193,197],{"className":194,"code":195,"language":196,"meta":157,"style":157},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\nbook = pd.read_excel(\"sales_book.xlsx\", sheet_name=None, dtype=str)\nfor name, df in book.items():\n    df.insert(0, \"excel_row\", df.index + 2)\norders, customers, products = book[\"Orders\"], book[\"Customers\"], book[\"Products\"]\nprint({name: len(df) for name, df in book.items()})\n","python",[14,198,199,215,222,263,278,302,330],{"__ignoreMap":157},[161,200,201,205,209,212],{"class":163,"line":164},[161,202,204],{"class":203},"s-kum","import",[161,206,208],{"class":207},"skGVy"," pandas ",[161,210,211],{"class":203},"as",[161,213,214],{"class":207}," pd\n",[161,216,218],{"class":163,"line":217},2,[161,219,221],{"emptyLinePlaceholder":220},true,"\n",[161,223,225,228,231,234,237,240,244,246,250,252,255,257,260],{"class":163,"line":224},3,[161,226,227],{"class":207},"book ",[161,229,230],{"class":203},"=",[161,232,233],{"class":207}," pd.read_excel(",[161,235,236],{"class":171},"\"sales_book.xlsx\"",[161,238,239],{"class":207},", ",[161,241,243],{"class":242},"sa561","sheet_name",[161,245,230],{"class":203},[161,247,249],{"class":248},"sP0c6","None",[161,251,239],{"class":207},[161,253,254],{"class":242},"dtype",[161,256,230],{"class":203},[161,258,259],{"class":248},"str",[161,261,262],{"class":207},")\n",[161,264,266,269,272,275],{"class":163,"line":265},4,[161,267,268],{"class":203},"for",[161,270,271],{"class":207}," name, df ",[161,273,274],{"class":203},"in",[161,276,277],{"class":207}," book.items():\n",[161,279,281,284,286,288,291,294,297,300],{"class":163,"line":280},5,[161,282,283],{"class":207},"    df.insert(",[161,285,52],{"class":248},[161,287,239],{"class":207},[161,289,290],{"class":171},"\"excel_row\"",[161,292,293],{"class":207},", df.index ",[161,295,296],{"class":203},"+",[161,298,299],{"class":248}," 2",[161,301,262],{"class":207},[161,303,305,308,310,313,316,319,322,324,327],{"class":163,"line":304},6,[161,306,307],{"class":207},"orders, customers, products ",[161,309,230],{"class":203},[161,311,312],{"class":207}," book[",[161,314,315],{"class":171},"\"Orders\"",[161,317,318],{"class":207},"], book[",[161,320,321],{"class":171},"\"Customers\"",[161,323,318],{"class":207},[161,325,326],{"class":171},"\"Products\"",[161,328,329],{"class":207},"]\n",[161,331,333,336,339,342,345,347,349,351],{"class":163,"line":332},7,[161,334,335],{"class":248},"print",[161,337,338],{"class":207},"({name: ",[161,340,341],{"class":248},"len",[161,343,344],{"class":207},"(df) ",[161,346,268],{"class":203},[161,348,271],{"class":207},[161,350,274],{"class":203},[161,352,353],{"class":207}," book.items()})\n",[10,355,356,359,360,363,364,367],{},[14,357,358],{},"dtype=str"," keeps every key as text on both sides; a key read as a number on one sheet and as text on the other never matches. ",[14,361,362],{},"sheet_name=None"," returns a dict of all sheets. The ",[14,365,366],{},"+ 2"," accounts for the header row and Excel counting from 1.",[147,369,371],{"id":370},"step-2-check-the-lookup-sheets-have-unique-keys","Step 2 — Check the lookup sheets have unique keys",[10,373,374],{},"A reference can only be checked against a lookup sheet whose keys are unique. If a customer ID appears twice, an order pointing at it is ambiguous — Excel's VLOOKUP takes the first row, a pandas merge takes both and duplicates the order. Check that first:",[152,376,378],{"className":194,"code":377,"language":196,"meta":157,"style":157},"def duplicate_keys(df, key):\n    dupes = df[df[key].duplicated(keep=False)]\n    return dupes[[\"excel_row\", key]].assign(problem=f\"duplicate {key}\")\n\nprint(duplicate_keys(customers, \"Customer ID\").to_string(index=False))\nprint(len(duplicate_keys(products, \"SKU\")), \"duplicate SKUs\")\n",[14,379,380,392,413,452,456,479],{"__ignoreMap":157},[161,381,382,385,389],{"class":163,"line":164},[161,383,384],{"class":203},"def",[161,386,388],{"class":387},"s_Opv"," duplicate_keys",[161,390,391],{"class":207},"(df, key):\n",[161,393,394,397,399,402,405,407,410],{"class":163,"line":217},[161,395,396],{"class":207},"    dupes ",[161,398,230],{"class":203},[161,400,401],{"class":207}," df[df[key].duplicated(",[161,403,404],{"class":242},"keep",[161,406,230],{"class":203},[161,408,409],{"class":248},"False",[161,411,412],{"class":207},")]\n",[161,414,415,418,421,423,426,429,431,434,437,441,444,447,450],{"class":163,"line":224},[161,416,417],{"class":203},"    return",[161,419,420],{"class":207}," dupes[[",[161,422,290],{"class":171},[161,424,425],{"class":207},", key]].assign(",[161,427,428],{"class":242},"problem",[161,430,230],{"class":203},[161,432,433],{"class":203},"f",[161,435,436],{"class":171},"\"duplicate ",[161,438,440],{"class":439},"sSjpA","{",[161,442,443],{"class":207},"key",[161,445,446],{"class":439},"}",[161,448,449],{"class":171},"\"",[161,451,262],{"class":207},[161,453,454],{"class":163,"line":265},[161,455,221],{"emptyLinePlaceholder":220},[161,457,458,460,463,466,469,472,474,476],{"class":163,"line":280},[161,459,335],{"class":248},[161,461,462],{"class":207},"(duplicate_keys(customers, ",[161,464,465],{"class":171},"\"Customer ID\"",[161,467,468],{"class":207},").to_string(",[161,470,471],{"class":242},"index",[161,473,230],{"class":203},[161,475,409],{"class":248},[161,477,478],{"class":207},"))\n",[161,480,481,483,486,488,491,494,497,500],{"class":163,"line":304},[161,482,335],{"class":248},[161,484,485],{"class":207},"(",[161,487,341],{"class":248},[161,489,490],{"class":207},"(duplicate_keys(products, ",[161,492,493],{"class":171},"\"SKU\"",[161,495,496],{"class":207},")), ",[161,498,499],{"class":171},"\"duplicate SKUs\"",[161,501,262],{"class":207},[10,503,504,505,508],{},"C002 appears on rows 3 and 6 with different names — \"Brightside\" and \"Brightside Group\". That is a data problem to fix on the Customers sheet, not something a script should resolve by picking one; ",[14,506,507],{},"keep=False"," lists both rows so the person fixing it sees the conflict.",[147,510,512],{"id":511},"step-3-find-orders-that-point-at-nothing","Step 3 — Find orders that point at nothing",[10,514,515,516,519],{},"With unique lookup keys, a reference check is one ",[14,517,518],{},"isin",". Normalise both sides the same way first — strip spaces and upper-case — so the check catches real orphans rather than formatting differences:",[152,521,523],{"className":194,"code":522,"language":196,"meta":157,"style":157},"def norm(s):\n    return s.str.strip().str.upper()\n\ndef orphans(child, key, parent, parent_key=None):\n    parent_keys = set(norm(parent[parent_key or key]).dropna())\n    missing = child[key].isna()\n    unknown = ~norm(child[key]).isin(parent_keys) & ~missing\n    near_miss = norm(child[key]).isin(parent_keys) & ~child[key].isin(parent[parent_key or key])\n    out = pd.concat([\n        child[missing].assign(problem=f\"{key} is blank\"),\n        child[unknown].assign(problem=f\"{key} not found\"),\n        child[near_miss].assign(problem=f\"{key} differs only in case or spaces\"),\n    ])\n    return out[[\"excel_row\", \"Order\", key, \"problem\"]].sort_values(\"excel_row\")\n\nprint(orphans(orders, \"Customer ID\", customers).to_string(index=False))\nprint(orphans(orders, \"SKU\", products).to_string(index=False))\n",[14,524,525,535,542,546,563,582,592,613,636,647,673,698,723,729,757,762,783],{"__ignoreMap":157},[161,526,527,529,532],{"class":163,"line":164},[161,528,384],{"class":203},[161,530,531],{"class":387}," norm",[161,533,534],{"class":207},"(s):\n",[161,536,537,539],{"class":163,"line":217},[161,538,417],{"class":203},[161,540,541],{"class":207}," s.str.strip().str.upper()\n",[161,543,544],{"class":163,"line":224},[161,545,221],{"emptyLinePlaceholder":220},[161,547,548,550,553,556,558,560],{"class":163,"line":265},[161,549,384],{"class":203},[161,551,552],{"class":387}," orphans",[161,554,555],{"class":207},"(child, key, parent, parent_key",[161,557,230],{"class":203},[161,559,249],{"class":248},[161,561,562],{"class":207},"):\n",[161,564,565,568,570,573,576,579],{"class":163,"line":280},[161,566,567],{"class":207},"    parent_keys ",[161,569,230],{"class":203},[161,571,572],{"class":248}," set",[161,574,575],{"class":207},"(norm(parent[parent_key ",[161,577,578],{"class":203},"or",[161,580,581],{"class":207}," key]).dropna())\n",[161,583,584,587,589],{"class":163,"line":304},[161,585,586],{"class":207},"    missing ",[161,588,230],{"class":203},[161,590,591],{"class":207}," child[key].isna()\n",[161,593,594,597,599,602,605,608,610],{"class":163,"line":332},[161,595,596],{"class":207},"    unknown ",[161,598,230],{"class":203},[161,600,601],{"class":203}," ~",[161,603,604],{"class":207},"norm(child[key]).isin(parent_keys) ",[161,606,607],{"class":203},"&",[161,609,601],{"class":203},[161,611,612],{"class":207},"missing\n",[161,614,616,619,621,624,626,628,631,633],{"class":163,"line":615},8,[161,617,618],{"class":207},"    near_miss ",[161,620,230],{"class":203},[161,622,623],{"class":207}," norm(child[key]).isin(parent_keys) ",[161,625,607],{"class":203},[161,627,601],{"class":203},[161,629,630],{"class":207},"child[key].isin(parent[parent_key ",[161,632,578],{"class":203},[161,634,635],{"class":207}," key])\n",[161,637,639,642,644],{"class":163,"line":638},9,[161,640,641],{"class":207},"    out ",[161,643,230],{"class":203},[161,645,646],{"class":207}," pd.concat([\n",[161,648,650,653,655,657,659,661,663,665,667,670],{"class":163,"line":649},10,[161,651,652],{"class":207},"        child[missing].assign(",[161,654,428],{"class":242},[161,656,230],{"class":203},[161,658,433],{"class":203},[161,660,449],{"class":171},[161,662,440],{"class":439},[161,664,443],{"class":207},[161,666,446],{"class":439},[161,668,669],{"class":171}," is blank\"",[161,671,672],{"class":207},"),\n",[161,674,676,679,681,683,685,687,689,691,693,696],{"class":163,"line":675},11,[161,677,678],{"class":207},"        child[unknown].assign(",[161,680,428],{"class":242},[161,682,230],{"class":203},[161,684,433],{"class":203},[161,686,449],{"class":171},[161,688,440],{"class":439},[161,690,443],{"class":207},[161,692,446],{"class":439},[161,694,695],{"class":171}," not found\"",[161,697,672],{"class":207},[161,699,701,704,706,708,710,712,714,716,718,721],{"class":163,"line":700},12,[161,702,703],{"class":207},"        child[near_miss].assign(",[161,705,428],{"class":242},[161,707,230],{"class":203},[161,709,433],{"class":203},[161,711,449],{"class":171},[161,713,440],{"class":439},[161,715,443],{"class":207},[161,717,446],{"class":439},[161,719,720],{"class":171}," differs only in case or spaces\"",[161,722,672],{"class":207},[161,724,726],{"class":163,"line":725},13,[161,727,728],{"class":207},"    ])\n",[161,730,732,734,737,739,741,744,747,750,753,755],{"class":163,"line":731},14,[161,733,417],{"class":203},[161,735,736],{"class":207}," out[[",[161,738,290],{"class":171},[161,740,239],{"class":207},[161,742,743],{"class":171},"\"Order\"",[161,745,746],{"class":207},", key, ",[161,748,749],{"class":171},"\"problem\"",[161,751,752],{"class":207},"]].sort_values(",[161,754,290],{"class":171},[161,756,262],{"class":207},[161,758,760],{"class":163,"line":759},15,[161,761,221],{"emptyLinePlaceholder":220},[161,763,765,767,770,772,775,777,779,781],{"class":163,"line":764},16,[161,766,335],{"class":248},[161,768,769],{"class":207},"(orphans(orders, ",[161,771,465],{"class":171},[161,773,774],{"class":207},", customers).to_string(",[161,776,471],{"class":242},[161,778,230],{"class":203},[161,780,409],{"class":248},[161,782,478],{"class":207},[161,784,786,788,790,792,795,797,799,801],{"class":163,"line":785},17,[161,787,335],{"class":248},[161,789,769],{"class":207},[161,791,493],{"class":171},[161,793,794],{"class":207},", products).to_string(",[161,796,471],{"class":242},[161,798,230],{"class":203},[161,800,409],{"class":248},[161,802,478],{"class":207},[10,804,805,806,809,810,813,814,809,817,820],{},"The checks sort the problems into three kinds, which need different fixes. ",[14,807,808],{},"C009"," and ",[14,811,812],{},"X-999"," do not exist at all — someone mistyped or the master sheet is missing a row. ",[14,815,816],{},"c001 ",[14,818,819],{},"a-200"," match once normalised; a lookup in Excel would still fail on them, so they are reported but the fix is simple. And SO-1008 has no customer. Treating near-misses as their own category stops them from hiding among the real orphans, and stops a normalising script from silently \"fixing\" data that other tools will read raw.",[30,822,39,827,39,830,39,833,39,836,39,842,39,847,39,853,39,856,39,860,39,865,39,871,39,873,39,877,39,879,39,882,39,885,39,889,39,891,39,895,39,897,39,900,39,903,39,907,39,909,39,913,39,915],{"viewBox":823,"role":33,"ariaLabelledBy":824,"xmlns":37,"style":38},"4 0 752 174",[825,826],"ref-kinds-t","ref-kinds-d",[41,828,829],{"id":825},"Three kinds of broken reference",[45,831,832],{"id":826},"A table of broken reference kinds. Not found, such as C009 or X-999: the key does not exist on the lookup sheet; fix the key or add the missing master row. Near miss, such as c001 with a trailing space or a-200: the key matches only after trimming and upper-casing; clean the key. Blank, such as SO-1008 with no customer: the reference is missing; fill it in.",[49,834],{"x":51,"y":52,"width":53,"height":835,"fill":55},"174",[49,837],{"x":58,"y":92,"width":838,"height":839,"rx":52,"fill":55,"stroke":840,"style":841},"92.1","40.0","var(--line,#cdd5e6)","stroke-width:1px",[49,843],{"x":844,"y":92,"width":845,"height":839,"rx":52,"fill":63,"stroke":64,"style":846},"112.1","313.9","stroke-width:1.2px",[78,848,852],{"x":849,"y":850,"style":851},"269.1","41.0","font-size:13px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Example",[49,854],{"x":855,"y":92,"width":845,"height":839,"rx":52,"fill":63,"stroke":64,"style":846},"426.1",[78,857,859],{"x":858,"y":850,"style":851},"583.0","Fix",[49,861],{"x":58,"y":862,"width":838,"height":863,"rx":52,"fill":864,"stroke":840,"style":841},"56.0","34.0","#e7ebef",[78,866,870],{"x":867,"y":868,"style":869},"32.0","78.0","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:start","Not found",[49,872],{"x":844,"y":862,"width":845,"height":863,"rx":52,"fill":55,"stroke":840,"style":841},[78,874,876],{"x":849,"y":875,"style":87},"77.0","C009, X-999",[49,878],{"x":855,"y":862,"width":845,"height":863,"rx":52,"fill":55,"stroke":840,"style":841},[78,880,881],{"x":858,"y":875,"style":87},"correct key or add master row",[49,883],{"x":58,"y":884,"width":838,"height":863,"rx":52,"fill":864,"stroke":840,"style":841},"90.0",[78,886,888],{"x":867,"y":887,"style":869},"112.0","Near miss",[49,890],{"x":844,"y":884,"width":845,"height":863,"rx":52,"fill":55,"stroke":840,"style":841},[78,892,894],{"x":849,"y":893,"style":87},"111.0","\"c001 \", a-200",[49,896],{"x":855,"y":884,"width":845,"height":863,"rx":52,"fill":55,"stroke":840,"style":841},[78,898,899],{"x":858,"y":893,"style":87},"trim and fix case",[49,901],{"x":58,"y":902,"width":838,"height":863,"rx":52,"fill":864,"stroke":840,"style":841},"124.0",[78,904,906],{"x":867,"y":905,"style":869},"146.0","Blank",[49,908],{"x":844,"y":902,"width":845,"height":863,"rx":52,"fill":55,"stroke":840,"style":841},[78,910,912],{"x":849,"y":911,"style":87},"145.0","SO-1008",[49,914],{"x":855,"y":902,"width":845,"height":863,"rx":52,"fill":55,"stroke":840,"style":841},[78,916,917],{"x":858,"y":911,"style":87},"fill in the reference",[147,919,921],{"id":920},"step-4-check-the-reference-is-still-valid","Step 4 — Check the reference is still valid",[10,923,924],{},"Existing is not always enough. An order for a closed customer, or a product that has been discontinued, refers to a row that exists but should not be used. Merge the orders onto the lookup sheet and test the looked-up attribute — after removing the duplicate keys, so the merge cannot multiply rows:",[152,926,928],{"className":194,"code":927,"language":196,"meta":157,"style":157},"lookup = customers.drop_duplicates(\"Customer ID\", keep=False)\njoined = orders.assign(key=norm(orders[\"Customer ID\"])).merge(\n    lookup.assign(key=norm(lookup[\"Customer ID\"]))[[\"key\", \"Status\"]],\n    on=\"key\", how=\"left\", validate=\"many_to_one\")\nclosed = joined[joined[\"Status\"].eq(\"closed\")]\nprint(closed[[\"excel_row\", \"Order\", \"Customer ID\", \"Status\"]].to_string(index=False))\n",[14,929,930,952,974,1002,1033,1053],{"__ignoreMap":157},[161,931,932,935,937,940,942,944,946,948,950],{"class":163,"line":164},[161,933,934],{"class":207},"lookup ",[161,936,230],{"class":203},[161,938,939],{"class":207}," customers.drop_duplicates(",[161,941,465],{"class":171},[161,943,239],{"class":207},[161,945,404],{"class":242},[161,947,230],{"class":203},[161,949,409],{"class":248},[161,951,262],{"class":207},[161,953,954,957,959,962,964,966,969,971],{"class":163,"line":217},[161,955,956],{"class":207},"joined ",[161,958,230],{"class":203},[161,960,961],{"class":207}," orders.assign(",[161,963,443],{"class":242},[161,965,230],{"class":203},[161,967,968],{"class":207},"norm(orders[",[161,970,465],{"class":171},[161,972,973],{"class":207},"])).merge(\n",[161,975,976,979,981,983,986,988,991,994,996,999],{"class":163,"line":224},[161,977,978],{"class":207},"    lookup.assign(",[161,980,443],{"class":242},[161,982,230],{"class":203},[161,984,985],{"class":207},"norm(lookup[",[161,987,465],{"class":171},[161,989,990],{"class":207},"]))[[",[161,992,993],{"class":171},"\"key\"",[161,995,239],{"class":207},[161,997,998],{"class":171},"\"Status\"",[161,1000,1001],{"class":207},"]],\n",[161,1003,1004,1007,1009,1011,1013,1016,1018,1021,1023,1026,1028,1031],{"class":163,"line":265},[161,1005,1006],{"class":242},"    on",[161,1008,230],{"class":203},[161,1010,993],{"class":171},[161,1012,239],{"class":207},[161,1014,1015],{"class":242},"how",[161,1017,230],{"class":203},[161,1019,1020],{"class":171},"\"left\"",[161,1022,239],{"class":207},[161,1024,1025],{"class":242},"validate",[161,1027,230],{"class":203},[161,1029,1030],{"class":171},"\"many_to_one\"",[161,1032,262],{"class":207},[161,1034,1035,1038,1040,1043,1045,1048,1051],{"class":163,"line":280},[161,1036,1037],{"class":207},"closed ",[161,1039,230],{"class":203},[161,1041,1042],{"class":207}," joined[joined[",[161,1044,998],{"class":171},[161,1046,1047],{"class":207},"].eq(",[161,1049,1050],{"class":171},"\"closed\"",[161,1052,412],{"class":207},[161,1054,1055,1057,1060,1062,1064,1066,1068,1070,1072,1074,1077,1079,1081,1083],{"class":163,"line":304},[161,1056,335],{"class":248},[161,1058,1059],{"class":207},"(closed[[",[161,1061,290],{"class":171},[161,1063,239],{"class":207},[161,1065,743],{"class":171},[161,1067,239],{"class":207},[161,1069,465],{"class":171},[161,1071,239],{"class":207},[161,1073,998],{"class":171},[161,1075,1076],{"class":207},"]].to_string(",[161,1078,471],{"class":242},[161,1080,230],{"class":203},[161,1082,409],{"class":248},[161,1084,478],{"class":207},[10,1086,1087,1090,1091,1095],{},[14,1088,1089],{},"validate=\"many_to_one\""," makes pandas raise if the lookup still had a duplicate key — a guard worth keeping in any lookup merge; ",[24,1092,1094],{"href":1093},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvalidate-a-merge-to-avoid-duplicate-rows\u002F","Validate a Merge to Avoid Duplicate Rows"," explains the options. SO-1005 belongs to the closed customer C003. The orders for C002 get no status here, because C002 was excluded as ambiguous; they will pass once the Customers sheet is fixed.",[147,1097,1099],{"id":1098},"step-5-flag-master-rows-nothing-uses","Step 5 — Flag master rows nothing uses",[10,1101,1102],{},"The reverse check is informational rather than an error, but useful when tidying a workbook: customers and products that no order refers to. They may be new, retired, or duplicates under a slightly different ID:",[152,1104,1106],{"className":194,"code":1105,"language":196,"meta":157,"style":157},"used = set(norm(orders[\"SKU\"]).dropna())\nunused = products[~norm(products[\"SKU\"]).isin(used)]\nprint(unused[[\"excel_row\", \"SKU\", \"Description\"]].to_string(index=False))\nused_customers = set(norm(orders[\"Customer ID\"]).dropna())\nidle = customers[~norm(customers[\"Customer ID\"]).isin(used_customers)]\nprint(len(idle), \"customers without orders\")\n",[14,1107,1108,1125,1146,1174,1189,1209],{"__ignoreMap":157},[161,1109,1110,1113,1115,1117,1120,1122],{"class":163,"line":164},[161,1111,1112],{"class":207},"used ",[161,1114,230],{"class":203},[161,1116,572],{"class":248},[161,1118,1119],{"class":207},"(norm(orders[",[161,1121,493],{"class":171},[161,1123,1124],{"class":207},"]).dropna())\n",[161,1126,1127,1130,1132,1135,1138,1141,1143],{"class":163,"line":217},[161,1128,1129],{"class":207},"unused ",[161,1131,230],{"class":203},[161,1133,1134],{"class":207}," products[",[161,1136,1137],{"class":203},"~",[161,1139,1140],{"class":207},"norm(products[",[161,1142,493],{"class":171},[161,1144,1145],{"class":207},"]).isin(used)]\n",[161,1147,1148,1150,1153,1155,1157,1159,1161,1164,1166,1168,1170,1172],{"class":163,"line":224},[161,1149,335],{"class":248},[161,1151,1152],{"class":207},"(unused[[",[161,1154,290],{"class":171},[161,1156,239],{"class":207},[161,1158,493],{"class":171},[161,1160,239],{"class":207},[161,1162,1163],{"class":171},"\"Description\"",[161,1165,1076],{"class":207},[161,1167,471],{"class":242},[161,1169,230],{"class":203},[161,1171,409],{"class":248},[161,1173,478],{"class":207},[161,1175,1176,1179,1181,1183,1185,1187],{"class":163,"line":265},[161,1177,1178],{"class":207},"used_customers ",[161,1180,230],{"class":203},[161,1182,572],{"class":248},[161,1184,1119],{"class":207},[161,1186,465],{"class":171},[161,1188,1124],{"class":207},[161,1190,1191,1194,1196,1199,1201,1204,1206],{"class":163,"line":280},[161,1192,1193],{"class":207},"idle ",[161,1195,230],{"class":203},[161,1197,1198],{"class":207}," customers[",[161,1200,1137],{"class":203},[161,1202,1203],{"class":207},"norm(customers[",[161,1205,465],{"class":171},[161,1207,1208],{"class":207},"]).isin(used_customers)]\n",[161,1210,1211,1213,1215,1217,1220,1223],{"class":163,"line":304},[161,1212,335],{"class":248},[161,1214,485],{"class":207},[161,1216,341],{"class":248},[161,1218,1219],{"class":207},"(idle), ",[161,1221,1222],{"class":171},"\"customers without orders\"",[161,1224,262],{"class":207},[10,1226,1227],{},"The Latch, D-500 on row 6 of the Products sheet, has never been ordered. Every customer has at least one order — including C003, whose only order is the one to a closed account. Keep this list out of the error report, or give it its own sheet headed \"for information\", so that a clean workbook is not reported as failing because a new product has not sold yet.",[147,1229,1231],{"id":1230},"step-6-write-one-report-for-the-whole-workbook","Step 6 — Write one report for the whole workbook",[10,1233,1234],{},"Gather every check into a single sheet with the sheet name and row, so one file tells the owner everything to fix:",[152,1236,1238],{"className":194,"code":1237,"language":196,"meta":157,"style":157},"problems = pd.concat([\n    duplicate_keys(customers, \"Customer ID\").assign(sheet=\"Customers\").rename(columns={\"Customer ID\": \"value\"}),\n    orphans(orders, \"Customer ID\", customers).assign(sheet=\"Orders\").rename(columns={\"Customer ID\": \"value\"}),\n    orphans(orders, \"SKU\", products).assign(sheet=\"Orders\").rename(columns={\"SKU\": \"value\"}),\n    closed.assign(sheet=\"Orders\", problem=\"customer is closed\").rename(columns={\"Customer ID\": \"value\"}),\n])[[\"sheet\", \"excel_row\", \"value\", \"problem\"]].sort_values([\"sheet\", \"excel_row\"])\nproblems.to_excel(\"reference_check.xlsx\", index=False)\nprint(problems.to_string(index=False))\nprint(len(problems), \"problems in\", problems.groupby(\"sheet\").excel_row.nunique().to_dict(), \"rows\")\n",[14,1239,1240,1249,1287,1319,1350,1386,1418,1436,1451],{"__ignoreMap":157},[161,1241,1242,1245,1247],{"class":163,"line":164},[161,1243,1244],{"class":207},"problems ",[161,1246,230],{"class":203},[161,1248,646],{"class":207},[161,1250,1251,1254,1256,1259,1262,1264,1266,1269,1272,1274,1276,1278,1281,1284],{"class":163,"line":217},[161,1252,1253],{"class":207},"    duplicate_keys(customers, ",[161,1255,465],{"class":171},[161,1257,1258],{"class":207},").assign(",[161,1260,1261],{"class":242},"sheet",[161,1263,230],{"class":203},[161,1265,321],{"class":171},[161,1267,1268],{"class":207},").rename(",[161,1270,1271],{"class":242},"columns",[161,1273,230],{"class":203},[161,1275,440],{"class":207},[161,1277,465],{"class":171},[161,1279,1280],{"class":207},": ",[161,1282,1283],{"class":171},"\"value\"",[161,1285,1286],{"class":207},"}),\n",[161,1288,1289,1292,1294,1297,1299,1301,1303,1305,1307,1309,1311,1313,1315,1317],{"class":163,"line":224},[161,1290,1291],{"class":207},"    orphans(orders, ",[161,1293,465],{"class":171},[161,1295,1296],{"class":207},", customers).assign(",[161,1298,1261],{"class":242},[161,1300,230],{"class":203},[161,1302,315],{"class":171},[161,1304,1268],{"class":207},[161,1306,1271],{"class":242},[161,1308,230],{"class":203},[161,1310,440],{"class":207},[161,1312,465],{"class":171},[161,1314,1280],{"class":207},[161,1316,1283],{"class":171},[161,1318,1286],{"class":207},[161,1320,1321,1323,1325,1328,1330,1332,1334,1336,1338,1340,1342,1344,1346,1348],{"class":163,"line":265},[161,1322,1291],{"class":207},[161,1324,493],{"class":171},[161,1326,1327],{"class":207},", products).assign(",[161,1329,1261],{"class":242},[161,1331,230],{"class":203},[161,1333,315],{"class":171},[161,1335,1268],{"class":207},[161,1337,1271],{"class":242},[161,1339,230],{"class":203},[161,1341,440],{"class":207},[161,1343,493],{"class":171},[161,1345,1280],{"class":207},[161,1347,1283],{"class":171},[161,1349,1286],{"class":207},[161,1351,1352,1355,1357,1359,1361,1363,1365,1367,1370,1372,1374,1376,1378,1380,1382,1384],{"class":163,"line":280},[161,1353,1354],{"class":207},"    closed.assign(",[161,1356,1261],{"class":242},[161,1358,230],{"class":203},[161,1360,315],{"class":171},[161,1362,239],{"class":207},[161,1364,428],{"class":242},[161,1366,230],{"class":203},[161,1368,1369],{"class":171},"\"customer is closed\"",[161,1371,1268],{"class":207},[161,1373,1271],{"class":242},[161,1375,230],{"class":203},[161,1377,440],{"class":207},[161,1379,465],{"class":171},[161,1381,1280],{"class":207},[161,1383,1283],{"class":171},[161,1385,1286],{"class":207},[161,1387,1388,1391,1394,1396,1398,1400,1402,1404,1406,1409,1411,1413,1415],{"class":163,"line":304},[161,1389,1390],{"class":207},"])[[",[161,1392,1393],{"class":171},"\"sheet\"",[161,1395,239],{"class":207},[161,1397,290],{"class":171},[161,1399,239],{"class":207},[161,1401,1283],{"class":171},[161,1403,239],{"class":207},[161,1405,749],{"class":171},[161,1407,1408],{"class":207},"]].sort_values([",[161,1410,1393],{"class":171},[161,1412,239],{"class":207},[161,1414,290],{"class":171},[161,1416,1417],{"class":207},"])\n",[161,1419,1420,1423,1426,1428,1430,1432,1434],{"class":163,"line":332},[161,1421,1422],{"class":207},"problems.to_excel(",[161,1424,1425],{"class":171},"\"reference_check.xlsx\"",[161,1427,239],{"class":207},[161,1429,471],{"class":242},[161,1431,230],{"class":203},[161,1433,409],{"class":248},[161,1435,262],{"class":207},[161,1437,1438,1440,1443,1445,1447,1449],{"class":163,"line":615},[161,1439,335],{"class":248},[161,1441,1442],{"class":207},"(problems.to_string(",[161,1444,471],{"class":242},[161,1446,230],{"class":203},[161,1448,409],{"class":248},[161,1450,478],{"class":207},[161,1452,1453,1455,1457,1459,1462,1465,1468,1470,1473,1476],{"class":163,"line":638},[161,1454,335],{"class":248},[161,1456,485],{"class":207},[161,1458,341],{"class":248},[161,1460,1461],{"class":207},"(problems), ",[161,1463,1464],{"class":171},"\"problems in\"",[161,1466,1467],{"class":207},", problems.groupby(",[161,1469,1393],{"class":171},[161,1471,1472],{"class":207},").excel_row.nunique().to_dict(), ",[161,1474,1475],{"class":171},"\"rows\"",[161,1477,262],{"class":207},[30,1479,39,1485,39,1488,39,1491,39,1496,39,1502,39,1507,39,1511,39,1517,39,1521,39,1525,39,1529,39,1533,39,1537,39,1541,39,1545,39,1550,39,1552,39,1555,39,1557,39,1560,39,1562,39,1565,39,1567,39,1569,39,1572,39,1576,39,1578,39,1582,39,1584,39,1587,39,1589,39,1592,39,1594,39,1597,39,1600,39,1603,39,1605,39,1607,39,1609,39,1612,39,1614,39,1616,39,1618,39,1620,39,1623,39,1626,39,1628,39,1631,39,1633,39,1635,39,1637,39,1639,39,1641,39,1644,39,1647,39,1651,39,1653,39,1655,39,1657,39,1659,39,1664,39,1667,39,1669,39,1672,39,1675,39,1678,39,1680,39,1682,39,1684,39,1686,39,1688,39,1691,39,1693,39,1696,39,1699,39,1703,39,1705,39,1707,39,1709,39,1711,39,1713,39,1715,39,1717,39,1719,39,1722,39,1726,39,1728,39,1730,39,1732,39,1734,39,1736,39,1738,39,1740,39,1742,39,1745,39,1749,39,1751,39,1753,39,1755,39,1757,39,1759,39,1761],{"viewBox":1480,"role":33,"ariaLabelledBy":1481,"xmlns":37,"style":1484},"163 0 434 340",[1482,1483],"ref-report-t","ref-report-d","width:100%;max-width:434px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[41,1486,1487],{"id":1482},"The reference report for the whole workbook",[45,1489,1490],{"id":1483},"The reference_check.xlsx sheet. Customers rows 3 and 6 have the duplicate ID C002. Orders row 4 has c001 with a trailing space and row 8 a-200, which only match after cleaning. Row 5 refers to customer C009, which is not found. Row 6 refers to the closed customer C003. Row 7 has the unknown SKU X-999. Row 9 has a blank customer.",[49,1492],{"x":1493,"y":52,"width":1494,"height":1495,"fill":55},"163","434","340",[78,1497,1501],{"x":1498,"y":1499,"style":1500},"380.0","31.0","font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","reference_check.xlsx",[49,1503],{"x":1504,"y":1505,"width":867,"height":1506,"rx":52,"fill":864,"stroke":840,"style":841},"179.5","48.0","24.0",[49,1508],{"x":1509,"y":1505,"width":1510,"height":1506,"rx":52,"fill":864,"stroke":840,"style":841},"211.5","91.8",[78,1512,1516],{"x":1513,"y":1514,"style":1515},"257.4","65.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[49,1518],{"x":1519,"y":1505,"width":1520,"height":1506,"rx":52,"fill":864,"stroke":840,"style":841},"303.2","86.7",[78,1522,1524],{"x":1523,"y":1514,"style":1515},"346.6","B",[49,1526],{"x":1527,"y":1505,"width":1528,"height":1506,"rx":52,"fill":864,"stroke":840,"style":841},"390.0","63.7",[78,1530,1532],{"x":1531,"y":1514,"style":1515},"421.8","C",[49,1534],{"x":1535,"y":1505,"width":1536,"height":1506,"rx":52,"fill":864,"stroke":840,"style":841},"453.7","126.8",[78,1538,1540],{"x":1539,"y":1514,"style":1515},"517.1","D",[49,1542],{"x":1504,"y":1543,"width":867,"height":1544,"rx":52,"fill":864,"stroke":840,"style":841},"72.0","28.0",[78,1546,1549],{"x":1547,"y":1548,"style":1515},"195.5","91.0","1",[49,1551],{"x":1509,"y":1543,"width":1510,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1553,1261],{"x":1513,"y":1548,"style":1554},"font-size:12px;font-weight:700;fill:var(--text,#172033);text-anchor:middle",[49,1556],{"x":1519,"y":1543,"width":1520,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1558,1559],{"x":1523,"y":1548,"style":1554},"excel_row",[49,1561],{"x":1527,"y":1543,"width":1528,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1563,1564],{"x":1531,"y":1548,"style":1554},"value",[49,1566],{"x":1535,"y":1543,"width":1536,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1568,428],{"x":1539,"y":1548,"style":1554},[49,1570],{"x":1504,"y":1571,"width":867,"height":1544,"rx":52,"fill":864,"stroke":840,"style":841},"100.0",[78,1573,1575],{"x":1547,"y":1574,"style":1515},"119.0","2",[49,1577],{"x":1509,"y":1571,"width":1510,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1579,1581],{"x":1513,"y":1574,"style":1580},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","Customers",[49,1583],{"x":1519,"y":1571,"width":1520,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1585,1586],{"x":1523,"y":1574,"style":1580},"3",[49,1588],{"x":1527,"y":1571,"width":1528,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1590,1591],{"x":1531,"y":1574,"style":1580},"C002",[49,1593],{"x":1535,"y":1571,"width":1536,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1595,1596],{"x":1539,"y":1574,"style":1580},"duplicate ID",[49,1598],{"x":1504,"y":1599,"width":867,"height":1544,"rx":52,"fill":864,"stroke":840,"style":841},"128.0",[78,1601,1586],{"x":1547,"y":1602,"style":1515},"147.0",[49,1604],{"x":1509,"y":1599,"width":1510,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1606,1581],{"x":1513,"y":1602,"style":1580},[49,1608],{"x":1519,"y":1599,"width":1520,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1610,1611],{"x":1523,"y":1602,"style":1580},"6",[49,1613],{"x":1527,"y":1599,"width":1528,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1615,1591],{"x":1531,"y":1602,"style":1580},[49,1617],{"x":1535,"y":1599,"width":1536,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1619,1596],{"x":1539,"y":1602,"style":1580},[49,1621],{"x":1504,"y":1622,"width":867,"height":1544,"rx":52,"fill":864,"stroke":840,"style":841},"156.0",[78,1624,51],{"x":1547,"y":1625,"style":1515},"175.0",[49,1627],{"x":1509,"y":1622,"width":1510,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1629,1630],{"x":1513,"y":1625,"style":1580},"Orders",[49,1632],{"x":1519,"y":1622,"width":1520,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1634,51],{"x":1523,"y":1625,"style":1580},[49,1636],{"x":1527,"y":1622,"width":1528,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1638,816],{"x":1531,"y":1625,"style":1580},[49,1640],{"x":1535,"y":1622,"width":1536,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1642,1643],{"x":1539,"y":1625,"style":1580},"case or spaces",[49,1645],{"x":1504,"y":1646,"width":867,"height":1544,"rx":52,"fill":864,"stroke":840,"style":841},"184.0",[78,1648,1650],{"x":1547,"y":1649,"style":1515},"203.0","5",[49,1652],{"x":1509,"y":1646,"width":1510,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1654,1630],{"x":1513,"y":1649,"style":1580},[49,1656],{"x":1519,"y":1646,"width":1520,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1658,1650],{"x":1523,"y":1649,"style":1580},[49,1660],{"x":1527,"y":1646,"width":1528,"height":1544,"rx":52,"fill":1661,"stroke":1662,"style":1663},"#fee8f2","var(--accent,#f43f8f)","stroke-width:1.5px",[78,1665,808],{"x":1531,"y":1649,"style":1666},"font-size:12px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle",[49,1668],{"x":1535,"y":1646,"width":1536,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1670,1671],{"x":1539,"y":1649,"style":1580},"not found",[49,1673],{"x":1504,"y":1674,"width":867,"height":1544,"rx":52,"fill":864,"stroke":840,"style":841},"212.0",[78,1676,1611],{"x":1547,"y":1677,"style":1515},"231.0",[49,1679],{"x":1509,"y":1674,"width":1510,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1681,1630],{"x":1513,"y":1677,"style":1580},[49,1683],{"x":1519,"y":1674,"width":1520,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1685,1611],{"x":1523,"y":1677,"style":1580},[49,1687],{"x":1527,"y":1674,"width":1528,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1689,1690],{"x":1531,"y":1677,"style":1580},"C003",[49,1692],{"x":1535,"y":1674,"width":1536,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1694,1695],{"x":1539,"y":1677,"style":1580},"customer closed",[49,1697],{"x":1504,"y":1698,"width":867,"height":1544,"rx":52,"fill":864,"stroke":840,"style":841},"240.0",[78,1700,1702],{"x":1547,"y":1701,"style":1515},"259.0","7",[49,1704],{"x":1509,"y":1698,"width":1510,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1706,1630],{"x":1513,"y":1701,"style":1580},[49,1708],{"x":1519,"y":1698,"width":1520,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1710,1702],{"x":1523,"y":1701,"style":1580},[49,1712],{"x":1527,"y":1698,"width":1528,"height":1544,"rx":52,"fill":1661,"stroke":1662,"style":1663},[78,1714,812],{"x":1531,"y":1701,"style":1666},[49,1716],{"x":1535,"y":1698,"width":1536,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1718,1671],{"x":1539,"y":1701,"style":1580},[49,1720],{"x":1504,"y":1721,"width":867,"height":1544,"rx":52,"fill":864,"stroke":840,"style":841},"268.0",[78,1723,1725],{"x":1547,"y":1724,"style":1515},"287.0","8",[49,1727],{"x":1509,"y":1721,"width":1510,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1729,1630],{"x":1513,"y":1724,"style":1580},[49,1731],{"x":1519,"y":1721,"width":1520,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1733,1725],{"x":1523,"y":1724,"style":1580},[49,1735],{"x":1527,"y":1721,"width":1528,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1737,819],{"x":1531,"y":1724,"style":1580},[49,1739],{"x":1535,"y":1721,"width":1536,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1741,1643],{"x":1539,"y":1724,"style":1580},[49,1743],{"x":1504,"y":1744,"width":867,"height":1544,"rx":52,"fill":864,"stroke":840,"style":841},"296.0",[78,1746,1748],{"x":1547,"y":1747,"style":1515},"315.0","9",[49,1750],{"x":1509,"y":1744,"width":1510,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1752,1630],{"x":1513,"y":1747,"style":1580},[49,1754],{"x":1519,"y":1744,"width":1520,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1756,1748],{"x":1523,"y":1747,"style":1580},[49,1758],{"x":1527,"y":1744,"width":1528,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[49,1760],{"x":1535,"y":1744,"width":1536,"height":1544,"rx":52,"fill":55,"stroke":840,"style":841},[78,1762,1763],{"x":1539,"y":1747,"style":1580},"blank",[147,1765,1767],{"id":1766},"run-the-check-as-a-gate","Run the check as a gate",[10,1769,1770,1771,1774,1775,1778],{},"When the workbook feeds a load or a report, make the check stop the pipeline: exit with a non-zero status when ",[14,1772,1773],{},"problems"," is not empty, after writing the report, so a scheduler or CI job marks the run as failed and nobody works from a report built on broken references. The equivalent check inside Excel is a helper column of ",[14,1776,1777],{},"=COUNTIF(Customers!A:A, B2)=0"," — useful for the sheet's owner, but easy to forget to copy down, which is why the script-based check is the one to trust.",[147,1780,1782],{"id":1781},"references-across-workbooks","References across workbooks",[10,1784,1785,1786,1789,1790,1793,1794,1798],{},"The same checks work when the lookup lives in another file — a customer master maintained by another team, say. Read it with its own ",[14,1787,1788],{},"pd.read_excel",", keep its file name alongside the row numbers in the report, and run the checks unchanged. When a reference depends on two columns — a price list keyed by SKU and region — build the key from both columns on each side (",[14,1791,1792],{},"norm(df[\"SKU\"]) + \"|\" + norm(df[\"Region\"])",") and check that combined key. ",[24,1795,1797],{"href":1796},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fmerge-excel-data-on-multiple-key-columns\u002F","Merge Excel Data on Multiple Key Columns with pandas"," covers composite keys in merges.",[147,1800,1802],{"id":1801},"common-pitfalls","Common pitfalls",[1804,1805,1806,1817,1823,1829,1838],"ul",{},[1807,1808,1809,1813,1814,1816],"li",{},[1810,1811,1812],"strong",{},"Mixed key types."," Read keys with ",[14,1815,358],{}," on every sheet.",[1807,1818,1819,1822],{},[1810,1820,1821],{},"Skipping the uniqueness check."," Duplicate lookup keys make every downstream lookup ambiguous.",[1807,1824,1825,1828],{},[1810,1826,1827],{},"Silently normalising."," Report near-misses; tools that read the raw sheet still fail on them.",[1807,1830,1831,1834,1835,1837],{},[1810,1832,1833],{},"Losing row numbers."," Add ",[14,1836,1559],{}," before filtering or merging.",[1807,1839,1840,1846,1847,1850],{},[1810,1841,1842,1843,1845],{},"Merging without ",[14,1844,1025],{},"."," Use ",[14,1848,1849],{},"many_to_one"," so duplicate keys raise instead of multiplying rows.",[147,1852,1854],{"id":1853},"performance-notes","Performance notes",[10,1856,1857,1859,1860,1863],{},[14,1858,518],{}," against a set is fast for hundreds of thousands of rows. Normalise each key column once and reuse it rather than calling ",[14,1861,1862],{},"norm"," in every check when the sheets are large.",[147,1865,1867],{"id":1866},"conclusion","Conclusion",[10,1869,1870,1871,1873,1874,1876],{},"Load every sheet with keys as text and keep the Excel row number. Check each lookup sheet for duplicate keys first, then test every reference with ",[14,1872,518],{}," on normalised keys, separating missing, not-found and near-miss references. Merge with ",[14,1875,1089],{}," to check attributes such as a closed status, flag master rows nothing uses, and collect everything into one report that names the sheet and row to fix.",[147,1878,1880],{"id":1879},"frequently-asked-questions","Frequently asked questions",[10,1882,1883,1886,1887,1896],{},[1810,1884,1885],{},"How do I find rows whose ID is missing from another sheet?","\nLoad both sheets and test the key column with isin: orders",[161,1888,1889,1890,1892,1893,1895],{},"~orders",[161,1891,465],{},".isin(customers",[161,1894,465],{},")"," returns every order whose customer is not on the Customers sheet.",[10,1898,1899,1902],{},[1810,1900,1901],{},"Why do IDs that look identical not match?","\nUsually a trailing space, a case difference or a type difference — \"c001 \" versus \"C001\", or the number 1001 versus the text \"1001\". Normalise both sides the same way (strip, upper-case, one type) before comparing, and report which rows only matched after normalising.",[10,1904,1905,1908],{},[1810,1906,1907],{},"Why check the lookup sheet for duplicate keys?","\nA VLOOKUP returns the first match, and a pandas merge returns every match. If a customer ID appears twice with different names, reports built on the sheet disagree depending on which tool did the lookup. Duplicate keys in a lookup sheet should be an error.",[10,1910,1911,1914],{},[1810,1912,1913],{},"How do I report the Excel row number of a broken reference?","\nWhen the header is in row 1, the Excel row is the DataFrame index plus 2. Add it as a column before filtering so it survives the checks.",[147,1916,1918],{"id":1917},"related","Related",[1804,1920,1921,1927,1934,1941],{},[1807,1922,1923,1924,1926],{},"Up one level: ",[24,1925,27],{"href":26}," — checking data before it spreads.",[1807,1928,1929,1933],{},[24,1930,1932],{"href":1931},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Ffind-duplicate-rows-in-excel-with-python\u002F","Find Duplicate Rows in Excel with Python"," — duplicates in one sheet.",[1807,1935,1936,1940],{},[24,1937,1939],{"href":1938},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvlookup-equivalent-in-pandas-for-excel-files\u002F","VLOOKUP Equivalent in pandas for Excel Files"," — the lookup these checks protect.",[1807,1942,1943,1947],{},[24,1944,1946],{"href":1945},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fvalidate-excel-rows-with-pydantic\u002F","Validate Excel Rows with pydantic Models"," — rules within a row.",[1949,1950,1951],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}",{"title":157,"searchDepth":217,"depth":217,"links":1953},[1954,1955,1956,1957,1958,1959,1960,1961,1962,1963,1964,1965,1966,1967],{"id":149,"depth":217,"text":150},{"id":187,"depth":217,"text":188},{"id":370,"depth":217,"text":371},{"id":511,"depth":217,"text":512},{"id":920,"depth":217,"text":921},{"id":1098,"depth":217,"text":1099},{"id":1230,"depth":217,"text":1231},{"id":1766,"depth":217,"text":1767},{"id":1781,"depth":217,"text":1782},{"id":1801,"depth":217,"text":1802},{"id":1853,"depth":217,"text":1854},{"id":1866,"depth":217,"text":1867},{"id":1879,"depth":217,"text":1880},{"id":1917,"depth":217,"text":1918},"2026-09-25","Find orders that point at customers who do not exist, products missing from the price list and lookup sheets with duplicate keys. Check every reference from one sheet to another with pandas, normalise keys so near-misses are caught, report the Excel rows that break a reference, and flag master rows nothing uses.","md",[1972,1974,1976,1978],{"q":1885,"a":1973},"Load both sheets and test the key column with isin: orders[~orders[\"Customer ID\"].isin(customers[\"Customer ID\"])] returns every order whose customer is not on the Customers sheet.",{"q":1901,"a":1975},"Usually a trailing space, a case difference or a type difference — \"c001 \" versus \"C001\", or the number 1001 versus the text \"1001\". Normalise both sides the same way (strip, upper-case, one type) before comparing, and report which rows only matched after normalising.",{"q":1907,"a":1977},"A VLOOKUP returns the first match, and a pandas merge returns every match. If a customer ID appears twice with different names, reports built on the sheet disagree depending on which tool did the lookup. Duplicate keys in a lookup sheet should be an error.",{"q":1913,"a":1979},"When the header is in row 1, the Excel row is the DataFrame index plus 2. Add it as a column before filtering so it survives the checks.","check references between excel sheets",null,{"breadcrumb":1983},[1984,1987,1990],{"name":1985,"item":1986},"Home","\u002F",{"name":1988,"item":1989},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":27,"item":26},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-references-between-excel-sheets",{"title":1993,"description":1994},"Check Lookup References Between Excel Sheets (Orphan Keys)","Validate references between Excel sheets with pandas: orphan keys with isin and merge indicator, key normalisation, duplicate keys in lookup sheets, composite references, inactive records, unused master rows and a report with Excel row numbers.","check-references-between-excel-sheets","advanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-references-between-excel-sheets\u002Findex",[1998,1999,2000],"data-validation","joins-and-lookups","pandas","how-to","H2fIccakwTh7HlNK-TM4bxDBQ8q9KDJcESfqCXW8aO8",[2004,2008],{"title":2005,"path":2006,"stem":2007,"children":-1},"Check Excel Data Types with pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-excel-data-types-with-pandas","advanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-excel-data-types-with-pandas\u002Findex",{"title":2009,"path":2010,"stem":2011,"children":-1},"Check That Totals in a Report Add Up","\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-that-totals-in-a-report-add-up","advanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-that-totals-in-a-report-add-up\u002Findex",[2013,2015],{"path":2014,"title":1988},"\u002Fadvanced-data-transformation-and-cleaning",{"path":2016,"title":27},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python",[],[2019,2022,2025,2028,2031,2034],{"path":2020,"title":2021,"kind":2001},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fdetect-outliers-in-excel-data-with-pandas","Detect Outliers in Excel Data with pandas",{"path":2023,"title":2024,"kind":2001},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-cells-containing-text-with-openpyxl","Highlight Cells Containing Specific Text with openpyxl",{"path":2026,"title":2027,"kind":2001},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffind-near-duplicate-names-within-one-excel-column","Find Near-Duplicate Names Within One Excel Column",{"path":2029,"title":2030,"kind":2001},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fread-dropdown-lists-from-an-existing-workbook","Read Dropdown Lists and Validation Rules from an Existing Workbook",{"path":2032,"title":2033,"kind":2001},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffind-and-report-missing-values-in-an-excel-file","Find and Report Missing Values in an Excel File",{"path":2035,"title":2036,"kind":2001},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fflag-rows-with-too-many-missing-values","Flag Rows with Too Many Missing Values",1790972098509]