[{"data":1,"prerenderedAt":3954},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets":3916,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets":3924,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets":3928,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets":3939},{"id":4,"title":5,"body":6,"dateModified":3881,"datePublished":3881,"description":3882,"extension":3883,"faq":3884,"keyword":3895,"kind":3896,"meta":3897,"navigation":315,"path":3905,"seo":3906,"slug":3909,"stem":3910,"topics":3911,"type":3914,"__hash__":3915},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Findex.md","Extracting Data from Messy Excel Sheets",{"type":7,"value":8,"toc":3864},"minimark",[9,19,166,171,187,246,249,253,269,619,638,715,719,725,979,1006,1010,1017,1369,1378,1536,1540,1553,1841,1854,1858,1865,2148,2160,2245,2249,2262,2507,2516,2520,2531,2821,2830,2834,2841,3245,3252,3324,3328,3331,3335,3346,3470,3473,3477,3480,3663,3674,3678,3724,3728,3734,3740,3746,3752,3758,3762,3813,3816,3860],[10,11,12,13,18],"p",{},"Most tutorials on reading Excel assume a tidy sheet: one header in row 1, one record per row, nothing\nelse. The workbooks that arrive in a reporting job are rarely like that. They are exports with a\ncompany banner of unpredictable height, summary tabs holding three small tables, finance layouts with\na quarter label merged above each pair of columns, reports with a subtotal after every group, filtered\nviews mailed \"as attached\", and trackers where the status is a fill colour. This section of\n",[14,15,17],"a",{"href":16},"\u002Fadvanced-data-transformation-and-cleaning\u002F","Advanced Data Transformation and Cleaning"," is about\nturning those sheets into ordinary DataFrames — reliably, on every run, and with loud failures when the\nlayout changes in a way code should not guess about.",[20,21,29,30,29,34,29,38,29,45,29,54,29,61,29,66,29,73,29,78,29,82,29,87,29,91,29,95,29,99,29,102,29,105,29,109,29,112,29,115,29,119,29,122,29,127,29,132,29,135,29,138,29,142,29,145,29,150,29,155,29,158,29,161],"svg",{"viewBox":22,"role":23,"ariaLabelledBy":24,"xmlns":27,"style":28},"0 0 760 156","img",[25,26],"msy-overview-t","msy-overview-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:760px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[31,32,33],"title",{"id":25},"From a human-built worksheet to a clean table",[35,36,37],"desc",{"id":26},"Read the sheet raw, locate the table by its header labels or blank boundaries, repair the header, remove rows that are not records, recover meaning stored in formatting, then validate and hand a tidy DataFrame to the rest of the pipeline.",[39,40],"rect",{"x":41,"y":41,"width":42,"height":43,"fill":44},"0","760","156","#ffffff",[39,46],{"x":47,"y":47,"width":48,"height":49,"rx":50,"fill":51,"stroke":52,"style":53},"20","121.6","86","12","#e7ebef","var(--line,#cdd5e6)","stroke-width:2px",[55,56,60],"text",{"x":57,"y":58,"style":59},"80.8","50","font-size:14px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","read raw",[55,62,65],{"x":57,"y":63,"style":64},"72","font-size:11.5px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:middle","header=None",[67,68],"line",{"x1":69,"y1":70,"x2":71,"y2":70,"stroke":72,"style":53},"146.6","63","163.6","var(--brand,#5b5cf0)",[74,75],"polygon",{"points":76,"fill":77},"166.6,63 157.6,58 157.6,68","#5b5cf0",[39,79],{"x":80,"y":47,"width":48,"height":49,"rx":50,"fill":81,"stroke":72,"style":53},"169.6","#f0f4ff",[55,83,86],{"x":84,"y":58,"style":85},"230.4","font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","locate",[55,88,90],{"x":84,"y":63,"style":89},"font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","header row,",[55,92,94],{"x":84,"y":93,"style":89},"92","table edges",[67,96],{"x1":97,"y1":70,"x2":98,"y2":70,"stroke":72,"style":53},"296.2","313.2",[74,100],{"points":101,"fill":77},"316.2,63 307.2,58 307.2,68",[39,103],{"x":104,"y":47,"width":48,"height":49,"rx":50,"fill":81,"stroke":72,"style":53},"319.2",[55,106,108],{"x":107,"y":58,"style":85},"380","repair",[55,110,111],{"x":107,"y":63,"style":89},"two-row header,",[55,113,114],{"x":107,"y":93,"style":89},"ffill labels",[67,116],{"x1":117,"y1":70,"x2":118,"y2":70,"stroke":72,"style":53},"445.8","462.8",[74,120],{"points":121,"fill":77},"465.8,63 456.8,58 456.8,68",[39,123],{"x":124,"y":47,"width":48,"height":49,"rx":50,"fill":125,"stroke":126,"style":53},"468.8","#fdefd8","var(--gold,#b4740a)",[55,128,131],{"x":129,"y":58,"style":130},"529.6","font-size:14px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","filter",[55,133,134],{"x":129,"y":63,"style":89},"totals, hidden,",[55,136,137],{"x":129,"y":93,"style":89},"blank rows",[67,139],{"x1":140,"y1":70,"x2":141,"y2":70,"stroke":72,"style":53},"595.4","612.4",[74,143],{"points":144,"fill":77},"615.4,63 606.4,58 606.4,68",[39,146],{"x":147,"y":47,"width":48,"height":49,"rx":50,"fill":148,"stroke":149,"style":53},"618.4","#d9f4f1","var(--teal,#0f9488)",[55,151,154],{"x":152,"y":58,"style":153},"679.2","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","tidy table",[55,156,157],{"x":152,"y":63,"style":89},"typed and",[55,159,160],{"x":152,"y":93,"style":89},"validated",[55,162,165],{"x":107,"y":163,"style":164},"140","font-size:12.5px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:middle","every step replaces a fixed assumption with something the file itself says",[167,168,170],"h2",{"id":169},"what-this-section-covers","What this section covers",[10,172,173,174,178,179,182,183,186],{},"The techniques fall into three groups. The first ",[175,176,177],"strong",{},"locates"," the data: finding the header row by its\nlabels, and splitting a sheet that holds several tables. The second ",[175,180,181],{},"repairs structure",": combining\nmulti-row headers into usable column names and removing subtotal rows that would inflate every sum. The\nthird ",[175,184,185],{},"recovers meaning that is not stored as values",": which rows a filter hid, and what a colour was\nsupposed to say.",[188,189,190,203,210,221,232,239],"ul",{},[191,192,193,197,198,202],"li",{},[14,194,196],{"href":195},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Ffind-the-header-row-in-an-excel-sheet-with-a-shifting-layout\u002F","Find the Header Row in an Excel Sheet with a Shifting Layout"," — scan for required labels instead of trusting ",[199,200,201],"code",{},"skiprows",".",[191,204,205,209],{},[14,206,208],{"href":207},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fread-multiple-tables-from-one-excel-sheet\u002F","Read Multiple Tables from One Excel Sheet"," — split on blank rows and columns, or find blocks by anchor.",[191,211,212,216,217,220],{},[14,213,215],{"href":214},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fread-excel-with-multi-row-headers-in-pandas\u002F","Read Excel with Multi-Row Headers in pandas"," — ",[199,218,219],{},"header=[0, 1]",", flattening and stacking.",[191,222,223,227,228,231],{},[14,224,226],{"href":225},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fremove-subtotal-and-total-rows-from-an-excel-export\u002F","Remove Subtotal and Total Rows from an Excel Export"," — by label, by shape or by ",[199,229,230],{},"SUBTOTAL"," formula, then reconcile.",[191,233,234,238],{},[14,235,237],{"href":236},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fskip-hidden-and-filtered-rows-when-reading-excel\u002F","Skip Hidden and Filtered Rows When Reading Excel"," — read what the user actually saw.",[191,240,241,245],{},[14,242,244],{"href":243},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fread-cell-colours-and-font-styles-as-data-with-openpyxl\u002F","Read Cell Colours and Font Styles as Data with openpyxl"," — turn fills and strikethrough into columns.",[10,247,248],{},"Each task below builds a small sample workbook first, so every code block can be run as written.",[167,250,252],{"id":251},"the-first-rule-read-the-sheet-raw","The first rule: read the sheet raw",[10,254,255,256,259,260,262,263,265,266,268],{},"Every technique in this section starts from the same place. Instead of letting ",[199,257,258],{},"read_excel"," decide where the\nheader is and what type each column holds, read the whole used range with ",[199,261,65],{},". You get every cell\nexactly where it sits, with integer labels that map straight back to the worksheet: row label ",[199,264,41],{}," is Excel row\n1, column label ",[199,267,41],{}," is column A.",[270,271,276],"pre",{"className":272,"code":273,"language":274,"meta":275,"style":275},"language-python shiki shiki-themes github-light github-dark-high-contrast","from openpyxl import Workbook\nimport pandas as pd\n\nwb = Workbook()\nws = wb.active\nws.title = \"Export\"\nws.append([\"Northwind Trading — monthly export\"])\nws.append([\"Filter: all regions\"])\nws.append([])\nws.append([\"Order ID\", \"Region\", \"Rep\", \"Amount\"])\nfor row in [(501, \"North\", \"Ana\", 1200), (502, \"North\", \"Ben\", 800), (None, \"North Total\", None, 2000),\n            (503, \"South\", \"Cara\", 950), (504, \"South\", \"Dev\", 650), (None, \"South Total\", None, 1600)]:\n    ws.append(list(row))\nwb.save(\"export.xlsx\")\n\nraw = pd.read_excel(\"export.xlsx\", sheet_name=\"Export\", header=None)\nprint(raw)\n","python","",[199,277,278,296,310,317,329,340,352,364,374,380,406,482,546,558,570,575,610],{"__ignoreMap":275},[279,280,282,286,290,293],"span",{"class":67,"line":281},1,[279,283,285],{"class":284},"s-kum","from",[279,287,289],{"class":288},"skGVy"," openpyxl ",[279,291,292],{"class":284},"import",[279,294,295],{"class":288}," Workbook\n",[279,297,299,301,304,307],{"class":67,"line":298},2,[279,300,292],{"class":284},[279,302,303],{"class":288}," pandas ",[279,305,306],{"class":284},"as",[279,308,309],{"class":288}," pd\n",[279,311,313],{"class":67,"line":312},3,[279,314,316],{"emptyLinePlaceholder":315},true,"\n",[279,318,320,323,326],{"class":67,"line":319},4,[279,321,322],{"class":288},"wb ",[279,324,325],{"class":284},"=",[279,327,328],{"class":288}," Workbook()\n",[279,330,332,335,337],{"class":67,"line":331},5,[279,333,334],{"class":288},"ws ",[279,336,325],{"class":284},[279,338,339],{"class":288}," wb.active\n",[279,341,343,346,348],{"class":67,"line":342},6,[279,344,345],{"class":288},"ws.title ",[279,347,325],{"class":284},[279,349,351],{"class":350},"srMev"," \"Export\"\n",[279,353,355,358,361],{"class":67,"line":354},7,[279,356,357],{"class":288},"ws.append([",[279,359,360],{"class":350},"\"Northwind Trading — monthly export\"",[279,362,363],{"class":288},"])\n",[279,365,367,369,372],{"class":67,"line":366},8,[279,368,357],{"class":288},[279,370,371],{"class":350},"\"Filter: all regions\"",[279,373,363],{"class":288},[279,375,377],{"class":67,"line":376},9,[279,378,379],{"class":288},"ws.append([])\n",[279,381,383,385,388,391,394,396,399,401,404],{"class":67,"line":382},10,[279,384,357],{"class":288},[279,386,387],{"class":350},"\"Order ID\"",[279,389,390],{"class":288},", ",[279,392,393],{"class":350},"\"Region\"",[279,395,390],{"class":288},[279,397,398],{"class":350},"\"Rep\"",[279,400,390],{"class":288},[279,402,403],{"class":350},"\"Amount\"",[279,405,363],{"class":288},[279,407,409,412,415,418,421,425,427,430,432,435,437,440,443,446,448,450,452,455,457,460,462,465,467,470,472,474,476,479],{"class":67,"line":408},11,[279,410,411],{"class":284},"for",[279,413,414],{"class":288}," row ",[279,416,417],{"class":284},"in",[279,419,420],{"class":288}," [(",[279,422,424],{"class":423},"sP0c6","501",[279,426,390],{"class":288},[279,428,429],{"class":350},"\"North\"",[279,431,390],{"class":288},[279,433,434],{"class":350},"\"Ana\"",[279,436,390],{"class":288},[279,438,439],{"class":423},"1200",[279,441,442],{"class":288},"), (",[279,444,445],{"class":423},"502",[279,447,390],{"class":288},[279,449,429],{"class":350},[279,451,390],{"class":288},[279,453,454],{"class":350},"\"Ben\"",[279,456,390],{"class":288},[279,458,459],{"class":423},"800",[279,461,442],{"class":288},[279,463,464],{"class":423},"None",[279,466,390],{"class":288},[279,468,469],{"class":350},"\"North Total\"",[279,471,390],{"class":288},[279,473,464],{"class":423},[279,475,390],{"class":288},[279,477,478],{"class":423},"2000",[279,480,481],{"class":288},"),\n",[279,483,485,488,491,493,496,498,501,503,506,508,511,513,515,517,520,522,525,527,529,531,534,536,538,540,543],{"class":67,"line":484},12,[279,486,487],{"class":288},"            (",[279,489,490],{"class":423},"503",[279,492,390],{"class":288},[279,494,495],{"class":350},"\"South\"",[279,497,390],{"class":288},[279,499,500],{"class":350},"\"Cara\"",[279,502,390],{"class":288},[279,504,505],{"class":423},"950",[279,507,442],{"class":288},[279,509,510],{"class":423},"504",[279,512,390],{"class":288},[279,514,495],{"class":350},[279,516,390],{"class":288},[279,518,519],{"class":350},"\"Dev\"",[279,521,390],{"class":288},[279,523,524],{"class":423},"650",[279,526,442],{"class":288},[279,528,464],{"class":423},[279,530,390],{"class":288},[279,532,533],{"class":350},"\"South Total\"",[279,535,390],{"class":288},[279,537,464],{"class":423},[279,539,390],{"class":288},[279,541,542],{"class":423},"1600",[279,544,545],{"class":288},")]:\n",[279,547,549,552,555],{"class":67,"line":548},13,[279,550,551],{"class":288},"    ws.append(",[279,553,554],{"class":423},"list",[279,556,557],{"class":288},"(row))\n",[279,559,561,564,567],{"class":67,"line":560},14,[279,562,563],{"class":288},"wb.save(",[279,565,566],{"class":350},"\"export.xlsx\"",[279,568,569],{"class":288},")\n",[279,571,573],{"class":67,"line":572},15,[279,574,316],{"emptyLinePlaceholder":315},[279,576,578,581,583,586,588,590,594,596,599,601,604,606,608],{"class":67,"line":577},16,[279,579,580],{"class":288},"raw ",[279,582,325],{"class":284},[279,584,585],{"class":288}," pd.read_excel(",[279,587,566],{"class":350},[279,589,390],{"class":288},[279,591,593],{"class":592},"sa561","sheet_name",[279,595,325],{"class":284},[279,597,598],{"class":350},"\"Export\"",[279,600,390],{"class":288},[279,602,603],{"class":592},"header",[279,605,325],{"class":284},[279,607,464],{"class":423},[279,609,569],{"class":288},[279,611,613,616],{"class":67,"line":612},17,[279,614,615],{"class":423},"print",[279,617,618],{"class":288},"(raw)\n",[10,620,621,622,624,625,629,630,633,634,637],{},"Reading raw costs nothing extra — it is the same parse — and it separates two decisions that ",[199,623,258],{},"\notherwise makes together and silently: ",[626,627,628],"em",{},"where the table is"," and ",[626,631,632],{},"what the data means",". Everything that follows\nis ordinary DataFrame slicing on ",[199,635,636],{},"raw",", which you can repeat as many ways as you like without touching the file\nagain.",[20,639,29,644,29,647,29,650,29,653,29,660,29,666,29,672,29,676,29,680,29,684,29,688,29,691,29,695,29,699,29,702,29,705,29,708,29,711],{"viewBox":640,"role":23,"ariaLabelledBy":641,"xmlns":27,"style":28},"0 0 760 214",[642,643],"msy-raw-t","msy-raw-d",[31,645,646],{"id":642},"What read_excel decides for you, and what reading raw gives back",[35,648,649],{"id":643},"A default read assumes row one is the header and infers types across the banner and data together. A raw read keeps every cell in place so that the header position and types can be decided from the content.",[39,651],{"x":41,"y":41,"width":42,"height":652,"fill":44},"214",[39,654],{"x":47,"y":47,"width":655,"height":656,"rx":657,"fill":658,"stroke":659,"style":53},"350","150","14","#fee8f2","var(--accent,#f43f8f)",[55,661,665],{"x":662,"y":663,"style":664},"195","46","font-size:13px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","default read_excel",[67,667],{"x1":668,"y1":669,"x2":670,"y2":669,"stroke":659,"style":671},"36","56","354","stroke-width:1px",[55,673,675],{"x":662,"y":674,"style":89},"78","row 1 becomes the header",[55,677,679],{"x":662,"y":678,"style":89},"101","banner text becomes data",[55,681,683],{"x":662,"y":682,"style":89},"124","types inferred across both",[55,685,687],{"x":662,"y":686,"style":89},"147","Unnamed: 1, Unnamed: 2 …",[39,689],{"x":690,"y":47,"width":655,"height":656,"rx":657,"fill":148,"stroke":149,"style":53},"390",[55,692,65],{"x":693,"y":663,"style":694},"565","font-size:13px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle",[67,696],{"x1":697,"y1":669,"x2":698,"y2":669,"stroke":149,"style":671},"406","724",[55,700,701],{"x":693,"y":674,"style":89},"every cell kept in place",[55,703,704],{"x":693,"y":678,"style":89},"row label 0 = Excel row 1",[55,706,707],{"x":693,"y":682,"style":89},"header chosen by content",[55,709,710],{"x":693,"y":686,"style":89},"types fixed afterwards",[55,712,714],{"x":107,"y":713,"style":164},"200","decide where the table is before deciding what it means",[167,716,718],{"id":717},"finding-the-header-row-by-its-labels","Finding the header row by its labels",[10,720,721,722,724],{},"A banner that grows by a line breaks every ",[199,723,201],{}," value in your code, and it breaks it quietly: the banner\nbecomes the header and the real header becomes the first data row. The robust alternative is to look for the\nrow that contains the labels you need. Normalise each cell — trim, collapse internal whitespace, lower-case —\nand test whether your required set is a subset of the row.",[270,726,728],{"className":272,"code":727,"language":274,"meta":275,"style":275},"REQUIRED = {\"order id\", \"region\", \"amount\"}\n\ndef norm(value) -> str:\n    return \" \".join(str(value).split()).lower()\n\ndef header_position(frame: pd.DataFrame, required=REQUIRED) -> int:\n    for position, row in frame.iterrows():\n        if required \u003C= {norm(v) for v in row.dropna()}:\n            return position\n    raise ValueError(f\"no row contains {sorted(required)}\")\n\npos = header_position(raw)\ntable = raw.iloc[pos + 1:].set_axis(raw.iloc[pos], axis=1).rename_axis(None, axis=1).reset_index(drop=True)\nprint(pos, table.columns.tolist())   # 3 ['Order ID', 'Region', 'Rep', 'Amount']\n",[199,729,730,757,761,779,795,799,821,834,858,866,901,905,915,968],{"__ignoreMap":275},[279,731,732,735,738,741,744,746,749,751,754],{"class":67,"line":281},[279,733,734],{"class":423},"REQUIRED",[279,736,737],{"class":284}," =",[279,739,740],{"class":288}," {",[279,742,743],{"class":350},"\"order id\"",[279,745,390],{"class":288},[279,747,748],{"class":350},"\"region\"",[279,750,390],{"class":288},[279,752,753],{"class":350},"\"amount\"",[279,755,756],{"class":288},"}\n",[279,758,759],{"class":67,"line":298},[279,760,316],{"emptyLinePlaceholder":315},[279,762,763,766,770,773,776],{"class":67,"line":312},[279,764,765],{"class":284},"def",[279,767,769],{"class":768},"s_Opv"," norm",[279,771,772],{"class":288},"(value) -> ",[279,774,775],{"class":423},"str",[279,777,778],{"class":288},":\n",[279,780,781,784,787,790,792],{"class":67,"line":319},[279,782,783],{"class":284},"    return",[279,785,786],{"class":350}," \" \"",[279,788,789],{"class":288},".join(",[279,791,775],{"class":423},[279,793,794],{"class":288},"(value).split()).lower()\n",[279,796,797],{"class":67,"line":331},[279,798,316],{"emptyLinePlaceholder":315},[279,800,801,803,806,809,811,813,816,819],{"class":67,"line":342},[279,802,765],{"class":284},[279,804,805],{"class":768}," header_position",[279,807,808],{"class":288},"(frame: pd.DataFrame, required",[279,810,325],{"class":284},[279,812,734],{"class":423},[279,814,815],{"class":288},") -> ",[279,817,818],{"class":423},"int",[279,820,778],{"class":288},[279,822,823,826,829,831],{"class":67,"line":354},[279,824,825],{"class":284},"    for",[279,827,828],{"class":288}," position, row ",[279,830,417],{"class":284},[279,832,833],{"class":288}," frame.iterrows():\n",[279,835,836,839,842,845,848,850,853,855],{"class":67,"line":366},[279,837,838],{"class":284},"        if",[279,840,841],{"class":288}," required ",[279,843,844],{"class":284},"\u003C=",[279,846,847],{"class":288}," {norm(v) ",[279,849,411],{"class":284},[279,851,852],{"class":288}," v ",[279,854,417],{"class":284},[279,856,857],{"class":288}," row.dropna()}:\n",[279,859,860,863],{"class":67,"line":376},[279,861,862],{"class":284},"            return",[279,864,865],{"class":288}," position\n",[279,867,868,871,874,877,880,883,887,890,893,896,899],{"class":67,"line":382},[279,869,870],{"class":284},"    raise",[279,872,873],{"class":423}," ValueError",[279,875,876],{"class":288},"(",[279,878,879],{"class":284},"f",[279,881,882],{"class":350},"\"no row contains ",[279,884,886],{"class":885},"sSjpA","{",[279,888,889],{"class":423},"sorted",[279,891,892],{"class":288},"(required)",[279,894,895],{"class":885},"}",[279,897,898],{"class":350},"\"",[279,900,569],{"class":288},[279,902,903],{"class":67,"line":408},[279,904,316],{"emptyLinePlaceholder":315},[279,906,907,910,912],{"class":67,"line":484},[279,908,909],{"class":288},"pos ",[279,911,325],{"class":284},[279,913,914],{"class":288}," header_position(raw)\n",[279,916,917,920,922,925,928,931,934,937,939,942,945,947,949,951,953,955,958,961,963,966],{"class":67,"line":548},[279,918,919],{"class":288},"table ",[279,921,325],{"class":284},[279,923,924],{"class":288}," raw.iloc[pos ",[279,926,927],{"class":284},"+",[279,929,930],{"class":423}," 1",[279,932,933],{"class":288},":].set_axis(raw.iloc[pos], ",[279,935,936],{"class":592},"axis",[279,938,325],{"class":284},[279,940,941],{"class":423},"1",[279,943,944],{"class":288},").rename_axis(",[279,946,464],{"class":423},[279,948,390],{"class":288},[279,950,936],{"class":592},[279,952,325],{"class":284},[279,954,941],{"class":423},[279,956,957],{"class":288},").reset_index(",[279,959,960],{"class":592},"drop",[279,962,325],{"class":284},[279,964,965],{"class":423},"True",[279,967,569],{"class":288},[279,969,970,972,975],{"class":67,"line":560},[279,971,615],{"class":423},[279,973,974],{"class":288},"(pos, table.columns.tolist())   ",[279,976,978],{"class":977},"s-wDw","# 3 ['Order ID', 'Region', 'Rep', 'Amount']\n",[10,980,981,982,985,986,989,990,993,994,997,998,1001,1002,1005],{},"Two design choices make this dependable. Using a ",[626,983,984],{},"subset"," test means an extra column added to the export next\nquarter does not break detection. Requiring ",[626,987,988],{},"several"," labels means a banner line such as \"Amount by region\"\ncannot be mistaken for the header. When labels drift between sources — ",[199,991,992],{},"Order #"," in one system, ",[199,995,996],{},"Order ID"," in\nanother — add an alias table that maps every known spelling to a canonical name, as the full\n",[14,999,1000],{"href":195},"header detection guide","\nshows, together with a preview read that uses ",[199,1003,1004],{},"nrows"," so large files are only scanned at the top.",[167,1007,1009],{"id":1008},"splitting-a-sheet-that-holds-several-tables","Splitting a sheet that holds several tables",[10,1011,1012,1013,1016],{},"Summary tabs are built for reading, not parsing: a regional table, a gap, a product table, and perhaps a quota\nlist off to the right. In the raw frame, a row where every cell is empty marks a vertical boundary and a column\nwhere every cell is empty marks a horizontal one. A cumulative sum over the blank-row mask numbers each block,\nand ",[199,1014,1015],{},"groupby"," hands them back one by one.",[270,1018,1020],{"className":272,"code":1019,"language":274,"meta":275,"style":275},"import numpy as np\n\nwb = Workbook()\nws = wb.active\nws.append([\"Region\", \"Revenue\"])\nws.append([\"North\", 2000])\nws.append([\"South\", 1600])\nws.append([])\nws.append([\"Product\", \"Units\"])\nws.append([\"Widget\", 420])\nwb.save(\"summary.xlsx\")\n\nsheet = pd.read_excel(\"summary.xlsx\", header=None)\nsheet = sheet.replace(r\"^\\s*$\", np.nan, regex=True)       # whitespace is not content\n\nblank = sheet.isna().all(axis=1)\nblocks = []\nfor _, block in sheet[~blank].groupby(blank.cumsum()[~blank]):\n    block = block.dropna(axis=1, how=\"all\")\n    blocks.append(block.iloc[1:].set_axis(block.iloc[0], axis=1).rename_axis(None, axis=1).reset_index(drop=True))\n\nfor b in blocks:\n    print(b, end=\"\\n\\n\")\n",[199,1021,1022,1034,1038,1046,1054,1067,1079,1091,1095,1109,1123,1132,1136,1157,1198,1202,1220,1230,1254,1283,1328,1333,1346],{"__ignoreMap":275},[279,1023,1024,1026,1029,1031],{"class":67,"line":281},[279,1025,292],{"class":284},[279,1027,1028],{"class":288}," numpy ",[279,1030,306],{"class":284},[279,1032,1033],{"class":288}," np\n",[279,1035,1036],{"class":67,"line":298},[279,1037,316],{"emptyLinePlaceholder":315},[279,1039,1040,1042,1044],{"class":67,"line":312},[279,1041,322],{"class":288},[279,1043,325],{"class":284},[279,1045,328],{"class":288},[279,1047,1048,1050,1052],{"class":67,"line":319},[279,1049,334],{"class":288},[279,1051,325],{"class":284},[279,1053,339],{"class":288},[279,1055,1056,1058,1060,1062,1065],{"class":67,"line":331},[279,1057,357],{"class":288},[279,1059,393],{"class":350},[279,1061,390],{"class":288},[279,1063,1064],{"class":350},"\"Revenue\"",[279,1066,363],{"class":288},[279,1068,1069,1071,1073,1075,1077],{"class":67,"line":342},[279,1070,357],{"class":288},[279,1072,429],{"class":350},[279,1074,390],{"class":288},[279,1076,478],{"class":423},[279,1078,363],{"class":288},[279,1080,1081,1083,1085,1087,1089],{"class":67,"line":354},[279,1082,357],{"class":288},[279,1084,495],{"class":350},[279,1086,390],{"class":288},[279,1088,542],{"class":423},[279,1090,363],{"class":288},[279,1092,1093],{"class":67,"line":366},[279,1094,379],{"class":288},[279,1096,1097,1099,1102,1104,1107],{"class":67,"line":376},[279,1098,357],{"class":288},[279,1100,1101],{"class":350},"\"Product\"",[279,1103,390],{"class":288},[279,1105,1106],{"class":350},"\"Units\"",[279,1108,363],{"class":288},[279,1110,1111,1113,1116,1118,1121],{"class":67,"line":382},[279,1112,357],{"class":288},[279,1114,1115],{"class":350},"\"Widget\"",[279,1117,390],{"class":288},[279,1119,1120],{"class":423},"420",[279,1122,363],{"class":288},[279,1124,1125,1127,1130],{"class":67,"line":408},[279,1126,563],{"class":288},[279,1128,1129],{"class":350},"\"summary.xlsx\"",[279,1131,569],{"class":288},[279,1133,1134],{"class":67,"line":484},[279,1135,316],{"emptyLinePlaceholder":315},[279,1137,1138,1141,1143,1145,1147,1149,1151,1153,1155],{"class":67,"line":548},[279,1139,1140],{"class":288},"sheet ",[279,1142,325],{"class":284},[279,1144,585],{"class":288},[279,1146,1129],{"class":350},[279,1148,390],{"class":288},[279,1150,603],{"class":592},[279,1152,325],{"class":284},[279,1154,464],{"class":423},[279,1156,569],{"class":288},[279,1158,1159,1161,1163,1166,1169,1171,1174,1177,1180,1182,1185,1188,1190,1192,1195],{"class":67,"line":560},[279,1160,1140],{"class":288},[279,1162,325],{"class":284},[279,1164,1165],{"class":288}," sheet.replace(",[279,1167,1168],{"class":284},"r",[279,1170,898],{"class":350},[279,1172,1173],{"class":423},"^\\s",[279,1175,1176],{"class":284},"*",[279,1178,1179],{"class":423},"$",[279,1181,898],{"class":350},[279,1183,1184],{"class":288},", np.nan, ",[279,1186,1187],{"class":592},"regex",[279,1189,325],{"class":284},[279,1191,965],{"class":423},[279,1193,1194],{"class":288},")       ",[279,1196,1197],{"class":977},"# whitespace is not content\n",[279,1199,1200],{"class":67,"line":572},[279,1201,316],{"emptyLinePlaceholder":315},[279,1203,1204,1207,1209,1212,1214,1216,1218],{"class":67,"line":577},[279,1205,1206],{"class":288},"blank ",[279,1208,325],{"class":284},[279,1210,1211],{"class":288}," sheet.isna().all(",[279,1213,936],{"class":592},[279,1215,325],{"class":284},[279,1217,941],{"class":423},[279,1219,569],{"class":288},[279,1221,1222,1225,1227],{"class":67,"line":612},[279,1223,1224],{"class":288},"blocks ",[279,1226,325],{"class":284},[279,1228,1229],{"class":288}," []\n",[279,1231,1233,1235,1238,1240,1243,1246,1249,1251],{"class":67,"line":1232},18,[279,1234,411],{"class":284},[279,1236,1237],{"class":288}," _, block ",[279,1239,417],{"class":284},[279,1241,1242],{"class":288}," sheet[",[279,1244,1245],{"class":284},"~",[279,1247,1248],{"class":288},"blank].groupby(blank.cumsum()[",[279,1250,1245],{"class":284},[279,1252,1253],{"class":288},"blank]):\n",[279,1255,1257,1260,1262,1265,1267,1269,1271,1273,1276,1278,1281],{"class":67,"line":1256},19,[279,1258,1259],{"class":288},"    block ",[279,1261,325],{"class":284},[279,1263,1264],{"class":288}," block.dropna(",[279,1266,936],{"class":592},[279,1268,325],{"class":284},[279,1270,941],{"class":423},[279,1272,390],{"class":288},[279,1274,1275],{"class":592},"how",[279,1277,325],{"class":284},[279,1279,1280],{"class":350},"\"all\"",[279,1282,569],{"class":288},[279,1284,1286,1289,1291,1294,1296,1299,1301,1303,1305,1307,1309,1311,1313,1315,1317,1319,1321,1323,1325],{"class":67,"line":1285},20,[279,1287,1288],{"class":288},"    blocks.append(block.iloc[",[279,1290,941],{"class":423},[279,1292,1293],{"class":288},":].set_axis(block.iloc[",[279,1295,41],{"class":423},[279,1297,1298],{"class":288},"], ",[279,1300,936],{"class":592},[279,1302,325],{"class":284},[279,1304,941],{"class":423},[279,1306,944],{"class":288},[279,1308,464],{"class":423},[279,1310,390],{"class":288},[279,1312,936],{"class":592},[279,1314,325],{"class":284},[279,1316,941],{"class":423},[279,1318,957],{"class":288},[279,1320,960],{"class":592},[279,1322,325],{"class":284},[279,1324,965],{"class":423},[279,1326,1327],{"class":288},"))\n",[279,1329,1331],{"class":67,"line":1330},21,[279,1332,316],{"emptyLinePlaceholder":315},[279,1334,1336,1338,1341,1343],{"class":67,"line":1335},22,[279,1337,411],{"class":284},[279,1339,1340],{"class":288}," b ",[279,1342,417],{"class":284},[279,1344,1345],{"class":288}," blocks:\n",[279,1347,1349,1352,1355,1358,1360,1362,1365,1367],{"class":67,"line":1348},23,[279,1350,1351],{"class":423},"    print",[279,1353,1354],{"class":288},"(b, ",[279,1356,1357],{"class":592},"end",[279,1359,325],{"class":284},[279,1361,898],{"class":350},[279,1363,1364],{"class":885},"\\n\\n",[279,1366,898],{"class":350},[279,1368,569],{"class":288},[10,1370,1371,1372,1375,1376,202],{},"The ",[199,1373,1374],{},"replace"," line is the one that prevents most failures. Excel keeps a cell alive once it has been formatted\nor once somebody typed a space into it, and a \"blank\" row that holds a single space glues two tables together.\nWhen the layout is too irregular for blank-line splitting — a note typed into the gap column, for instance — find\neach table by its header label instead. Both approaches, plus reading named Excel tables that record their own\nranges, are in ",[14,1377,208],{"href":207},[20,1379,29,1385,29,1388,29,1391,29,1395,29,1398,29,1401,29,1406,29,1409,29,1413,29,1415,29,1419,29,1421,29,1425,29,1427,29,1430,29,1436,29,1441,29,1444,29,1448,29,1450,29,1453,29,1455,29,1457,29,1460,29,1464,29,1466,29,1469,29,1471,29,1473,29,1475,29,1479,29,1481,29,1483,29,1486,29,1491,29,1494,29,1498,29,1500,29,1504,29,1506,29,1509,29,1512,29,1516,29,1518,29,1522,29,1524,29,1527,29,1529,29,1531],{"viewBox":1380,"role":23,"ariaLabelledBy":1381,"xmlns":27,"style":1384},"0 0 426 284",[1382,1383],"msy-blocks-t","msy-blocks-d","width:100%;max-width:426px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[31,1386,1387],{"id":1382},"Blank rows as the boundary between stacked tables",[35,1389,1390],{"id":1383},"A region table occupies rows 1 to 3, row 4 is blank, and a product table occupies rows 5 and 6. The cumulative count of blank rows gives the first table id zero and the second id one.",[39,1392],{"x":41,"y":41,"width":1393,"height":1394,"fill":44},"426","284",[39,1396],{"x":47,"y":47,"width":668,"height":1397,"rx":41,"fill":51,"stroke":52,"style":671},"30",[39,1399],{"x":669,"y":47,"width":1400,"height":1397,"rx":41,"fill":51,"stroke":52,"style":671},"90",[55,1402,1405],{"x":678,"y":1403,"style":1404},"40","font-size:11.5px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[39,1407],{"x":1408,"y":47,"width":1400,"height":1397,"rx":41,"fill":51,"stroke":52,"style":671},"146",[55,1410,1412],{"x":1411,"y":1403,"style":1404},"191","B",[39,1414],{"x":47,"y":58,"width":668,"height":1397,"rx":41,"fill":51,"stroke":52,"style":671},[55,1416,941],{"x":1417,"y":1418,"style":1404},"38","70",[39,1420],{"x":669,"y":58,"width":1400,"height":1397,"rx":41,"fill":81,"stroke":72,"style":671},[55,1422,1424],{"x":678,"y":1418,"style":1423},"font-size:11.5px;font-weight:600;fill:var(--brand-strong,#4338ca);text-anchor:middle","Region",[39,1426],{"x":1408,"y":58,"width":1400,"height":1397,"rx":41,"fill":81,"stroke":72,"style":671},[55,1428,1429],{"x":1411,"y":1418,"style":1423},"Revenue",[39,1431],{"x":1432,"y":1433,"width":656,"height":1434,"rx":1435,"fill":81,"stroke":72,"style":671},"256","52","26","8",[55,1437,1440],{"x":1438,"y":1439,"style":1423},"331","69","block 0 header",[39,1442],{"x":47,"y":1443,"width":668,"height":1397,"rx":41,"fill":51,"stroke":52,"style":671},"80",[55,1445,1447],{"x":1417,"y":1446,"style":1404},"100","2",[39,1449],{"x":669,"y":1443,"width":1400,"height":1397,"rx":41,"fill":44,"stroke":52,"style":671},[55,1451,1452],{"x":678,"y":1446,"style":89},"North",[39,1454],{"x":1408,"y":1443,"width":1400,"height":1397,"rx":41,"fill":44,"stroke":52,"style":671},[55,1456,478],{"x":1411,"y":1446,"style":89},[39,1458],{"x":47,"y":1459,"width":668,"height":1397,"rx":41,"fill":51,"stroke":52,"style":671},"110",[55,1461,1463],{"x":1417,"y":1462,"style":1404},"130","3",[39,1465],{"x":669,"y":1459,"width":1400,"height":1397,"rx":41,"fill":44,"stroke":52,"style":671},[55,1467,1468],{"x":678,"y":1462,"style":89},"South",[39,1470],{"x":1408,"y":1459,"width":1400,"height":1397,"rx":41,"fill":44,"stroke":52,"style":671},[55,1472,542],{"x":1411,"y":1462,"style":89},[39,1474],{"x":47,"y":163,"width":668,"height":1397,"rx":41,"fill":51,"stroke":52,"style":671},[55,1476,1478],{"x":1417,"y":1477,"style":1404},"160","4",[39,1480],{"x":669,"y":163,"width":1400,"height":1397,"rx":41,"fill":51,"stroke":52,"style":671},[39,1482],{"x":1408,"y":163,"width":1400,"height":1397,"rx":41,"fill":51,"stroke":52,"style":671},[39,1484],{"x":1432,"y":1485,"width":656,"height":1434,"rx":1435,"fill":125,"stroke":126,"style":671},"142",[55,1487,1490],{"x":1438,"y":1488,"style":1489},"159","font-size:11.5px;font-weight:600;fill:var(--gold-ink,#7a4e06);text-anchor:middle","blank: counter +1",[39,1492],{"x":47,"y":1493,"width":668,"height":1397,"rx":41,"fill":51,"stroke":52,"style":671},"170",[55,1495,1497],{"x":1417,"y":1496,"style":1404},"190","5",[39,1499],{"x":669,"y":1493,"width":1400,"height":1397,"rx":41,"fill":148,"stroke":149,"style":671},[55,1501,1503],{"x":678,"y":1496,"style":1502},"font-size:11.5px;font-weight:600;fill:var(--teal-ink,#0b6157);text-anchor:middle","Product",[39,1505],{"x":1408,"y":1493,"width":1400,"height":1397,"rx":41,"fill":148,"stroke":149,"style":671},[55,1507,1508],{"x":1411,"y":1496,"style":1502},"Units",[39,1510],{"x":1432,"y":1511,"width":656,"height":1434,"rx":1435,"fill":148,"stroke":149,"style":671},"172",[55,1513,1515],{"x":1438,"y":1514,"style":1502},"189","block 1 header",[39,1517],{"x":47,"y":713,"width":668,"height":1397,"rx":41,"fill":51,"stroke":52,"style":671},[55,1519,1521],{"x":1417,"y":1520,"style":1404},"220","6",[39,1523],{"x":669,"y":713,"width":1400,"height":1397,"rx":41,"fill":44,"stroke":52,"style":671},[55,1525,1526],{"x":678,"y":1520,"style":89},"Widget",[39,1528],{"x":1408,"y":713,"width":1400,"height":1397,"rx":41,"fill":44,"stroke":52,"style":671},[55,1530,1120],{"x":1411,"y":1520,"style":89},[55,1532,1535],{"x":1533,"y":1534,"style":164},"213","262","blank.cumsum() turns separators into table ids",[167,1537,1539],{"id":1538},"repairing-headers-that-span-two-rows","Repairing headers that span two rows",[10,1541,1542,1543,1546,1547,1549,1550,202],{},"Finance and planning sheets put a merged label such as ",[199,1544,1545],{},"Q1"," above a pair of sub-columns. Pass both header\nrows to ",[199,1548,258],{}," and pandas builds a two-level column index, forward-filling the empty cells a merge\nleaves behind. Put the label column in the index so it does not end up called ",[199,1551,1552],{},"Unnamed: 0_level_1",[270,1554,1556],{"className":272,"code":1555,"language":274,"meta":275,"style":275},"wb = Workbook()\nws = wb.active\nws.append([\"Region\", \"Q1\", None, \"Q2\", None])\nws.append([None, \"Revenue\", \"Units\", \"Revenue\", \"Units\"])\nws.append([\"North\", 1200, 30, 1500, 34])\nws.append([\"South\", 900, 22, 1100, 25])\nws.merge_cells(\"B1:C1\")\nws.merge_cells(\"D1:E1\")\nwb.save(\"quarterly.xlsx\")\n\nwide = pd.read_excel(\"quarterly.xlsx\", header=[0, 1], index_col=0).rename_axis(\"Region\")\nrevenue = wide.xs(\"Revenue\", axis=1, level=1)            # one measure, every quarter\nlong = wide.stack(level=0, future_stack=True).rename_axis([\"Region\", \"Quarter\"]).reset_index()\nprint(revenue)\nprint(long)\n",[199,1557,1558,1566,1574,1600,1624,1650,1678,1688,1697,1706,1710,1751,1786,1824,1831],{"__ignoreMap":275},[279,1559,1560,1562,1564],{"class":67,"line":281},[279,1561,322],{"class":288},[279,1563,325],{"class":284},[279,1565,328],{"class":288},[279,1567,1568,1570,1572],{"class":67,"line":298},[279,1569,334],{"class":288},[279,1571,325],{"class":284},[279,1573,339],{"class":288},[279,1575,1576,1578,1580,1582,1585,1587,1589,1591,1594,1596,1598],{"class":67,"line":312},[279,1577,357],{"class":288},[279,1579,393],{"class":350},[279,1581,390],{"class":288},[279,1583,1584],{"class":350},"\"Q1\"",[279,1586,390],{"class":288},[279,1588,464],{"class":423},[279,1590,390],{"class":288},[279,1592,1593],{"class":350},"\"Q2\"",[279,1595,390],{"class":288},[279,1597,464],{"class":423},[279,1599,363],{"class":288},[279,1601,1602,1604,1606,1608,1610,1612,1614,1616,1618,1620,1622],{"class":67,"line":319},[279,1603,357],{"class":288},[279,1605,464],{"class":423},[279,1607,390],{"class":288},[279,1609,1064],{"class":350},[279,1611,390],{"class":288},[279,1613,1106],{"class":350},[279,1615,390],{"class":288},[279,1617,1064],{"class":350},[279,1619,390],{"class":288},[279,1621,1106],{"class":350},[279,1623,363],{"class":288},[279,1625,1626,1628,1630,1632,1634,1636,1638,1640,1643,1645,1648],{"class":67,"line":331},[279,1627,357],{"class":288},[279,1629,429],{"class":350},[279,1631,390],{"class":288},[279,1633,439],{"class":423},[279,1635,390],{"class":288},[279,1637,1397],{"class":423},[279,1639,390],{"class":288},[279,1641,1642],{"class":423},"1500",[279,1644,390],{"class":288},[279,1646,1647],{"class":423},"34",[279,1649,363],{"class":288},[279,1651,1652,1654,1656,1658,1661,1663,1666,1668,1671,1673,1676],{"class":67,"line":342},[279,1653,357],{"class":288},[279,1655,495],{"class":350},[279,1657,390],{"class":288},[279,1659,1660],{"class":423},"900",[279,1662,390],{"class":288},[279,1664,1665],{"class":423},"22",[279,1667,390],{"class":288},[279,1669,1670],{"class":423},"1100",[279,1672,390],{"class":288},[279,1674,1675],{"class":423},"25",[279,1677,363],{"class":288},[279,1679,1680,1683,1686],{"class":67,"line":354},[279,1681,1682],{"class":288},"ws.merge_cells(",[279,1684,1685],{"class":350},"\"B1:C1\"",[279,1687,569],{"class":288},[279,1689,1690,1692,1695],{"class":67,"line":366},[279,1691,1682],{"class":288},[279,1693,1694],{"class":350},"\"D1:E1\"",[279,1696,569],{"class":288},[279,1698,1699,1701,1704],{"class":67,"line":376},[279,1700,563],{"class":288},[279,1702,1703],{"class":350},"\"quarterly.xlsx\"",[279,1705,569],{"class":288},[279,1707,1708],{"class":67,"line":382},[279,1709,316],{"emptyLinePlaceholder":315},[279,1711,1712,1715,1717,1719,1721,1723,1725,1727,1730,1732,1734,1736,1738,1741,1743,1745,1747,1749],{"class":67,"line":408},[279,1713,1714],{"class":288},"wide ",[279,1716,325],{"class":284},[279,1718,585],{"class":288},[279,1720,1703],{"class":350},[279,1722,390],{"class":288},[279,1724,603],{"class":592},[279,1726,325],{"class":284},[279,1728,1729],{"class":288},"[",[279,1731,41],{"class":423},[279,1733,390],{"class":288},[279,1735,941],{"class":423},[279,1737,1298],{"class":288},[279,1739,1740],{"class":592},"index_col",[279,1742,325],{"class":284},[279,1744,41],{"class":423},[279,1746,944],{"class":288},[279,1748,393],{"class":350},[279,1750,569],{"class":288},[279,1752,1753,1756,1758,1761,1763,1765,1767,1769,1771,1773,1776,1778,1780,1783],{"class":67,"line":484},[279,1754,1755],{"class":288},"revenue ",[279,1757,325],{"class":284},[279,1759,1760],{"class":288}," wide.xs(",[279,1762,1064],{"class":350},[279,1764,390],{"class":288},[279,1766,936],{"class":592},[279,1768,325],{"class":284},[279,1770,941],{"class":423},[279,1772,390],{"class":288},[279,1774,1775],{"class":592},"level",[279,1777,325],{"class":284},[279,1779,941],{"class":423},[279,1781,1782],{"class":288},")            ",[279,1784,1785],{"class":977},"# one measure, every quarter\n",[279,1787,1788,1791,1793,1796,1798,1800,1802,1804,1807,1809,1811,1814,1816,1818,1821],{"class":67,"line":548},[279,1789,1790],{"class":592},"long",[279,1792,737],{"class":284},[279,1794,1795],{"class":288}," wide.stack(",[279,1797,1775],{"class":592},[279,1799,325],{"class":284},[279,1801,41],{"class":423},[279,1803,390],{"class":288},[279,1805,1806],{"class":592},"future_stack",[279,1808,325],{"class":284},[279,1810,965],{"class":423},[279,1812,1813],{"class":288},").rename_axis([",[279,1815,393],{"class":350},[279,1817,390],{"class":288},[279,1819,1820],{"class":350},"\"Quarter\"",[279,1822,1823],{"class":288},"]).reset_index()\n",[279,1825,1826,1828],{"class":67,"line":560},[279,1827,615],{"class":423},[279,1829,1830],{"class":288},"(revenue)\n",[279,1832,1833,1835,1837,1839],{"class":67,"line":572},[279,1834,615],{"class":423},[279,1836,876],{"class":288},[279,1838,1790],{"class":592},[279,1840,569],{"class":288},[10,1842,1843,1844,1847,1848,1851,1852,202],{},"Which shape to keep depends on where the data goes next. The MultiIndex is convenient for interactive\nselection; flattened names such as ",[199,1845,1846],{},"Q1 Revenue"," suit CSV files and SQL tables; and the long format produced by\n",[199,1849,1850],{},"stack"," — one row per region and quarter — is what grouping, charting and database loads prefer, because a new\nquarter adds rows rather than columns. The details, including building the header by hand when a title block\nsits above it and validating that each quarter carries the expected measures, are in\n",[14,1853,215],{"href":214},[167,1855,1857],{"id":1856},"removing-subtotal-and-total-rows","Removing subtotal and total rows",[10,1859,1860,1861,1864],{},"Rollup rows are the most expensive mess to miss, because nothing fails: every sum is just wrong. The export at\nthe top of this page has a subtotal after each region, so summing ",[199,1862,1863],{},"Amount"," without removing them doubles the\nresult. Detect rollups by their label, by an empty record identifier, or by both.",[270,1866,1868],{"className":272,"code":1867,"language":274,"meta":275,"style":275},"TOTAL = r\"\\b(?:sub-?)?total\\b\"\n\nlabel_hit = table[\"Region\"].astype(str).str.contains(TOTAL, case=False, regex=True)\nshape_hit = table[\"Order ID\"].isna() & table[\"Amount\"].notna()\nrollups = table[label_hit | shape_hit]\ndetail = table[~(label_hit | shape_hit)].reset_index(drop=True)\n\ncomputed = detail.groupby(\"Region\")[\"Amount\"].sum()\nreported = rollups.assign(Region=rollups[\"Region\"].str.replace(r\"\\s*total$\", \"\", case=False, regex=True)) \\\n                  .set_index(\"Region\")[\"Amount\"]\nassert (computed == reported.reindex(computed.index)).all(), \"subtotals do not reconcile\"\nprint(detail)\nprint(computed.to_dict())\n",[199,1869,1870,1905,1909,1951,1975,1991,2018,2022,2042,2103,2117,2134,2141],{"__ignoreMap":275},[279,1871,1872,1875,1877,1880,1882,1885,1888,1891,1894,1896,1899,1902],{"class":67,"line":281},[279,1873,1874],{"class":423},"TOTAL",[279,1876,737],{"class":284},[279,1878,1879],{"class":284}," r",[279,1881,898],{"class":350},[279,1883,1884],{"class":423},"\\b(?:",[279,1886,1887],{"class":350},"sub-",[279,1889,1890],{"class":284},"?",[279,1892,1893],{"class":423},")",[279,1895,1890],{"class":284},[279,1897,1898],{"class":350},"total",[279,1900,1901],{"class":423},"\\b",[279,1903,1904],{"class":350},"\"\n",[279,1906,1907],{"class":67,"line":298},[279,1908,316],{"emptyLinePlaceholder":315},[279,1910,1911,1914,1916,1919,1921,1924,1926,1929,1931,1933,1936,1938,1941,1943,1945,1947,1949],{"class":67,"line":312},[279,1912,1913],{"class":288},"label_hit ",[279,1915,325],{"class":284},[279,1917,1918],{"class":288}," table[",[279,1920,393],{"class":350},[279,1922,1923],{"class":288},"].astype(",[279,1925,775],{"class":423},[279,1927,1928],{"class":288},").str.contains(",[279,1930,1874],{"class":423},[279,1932,390],{"class":288},[279,1934,1935],{"class":592},"case",[279,1937,325],{"class":284},[279,1939,1940],{"class":423},"False",[279,1942,390],{"class":288},[279,1944,1187],{"class":592},[279,1946,325],{"class":284},[279,1948,965],{"class":423},[279,1950,569],{"class":288},[279,1952,1953,1956,1958,1960,1962,1965,1968,1970,1972],{"class":67,"line":319},[279,1954,1955],{"class":288},"shape_hit ",[279,1957,325],{"class":284},[279,1959,1918],{"class":288},[279,1961,387],{"class":350},[279,1963,1964],{"class":288},"].isna() ",[279,1966,1967],{"class":284},"&",[279,1969,1918],{"class":288},[279,1971,403],{"class":350},[279,1973,1974],{"class":288},"].notna()\n",[279,1976,1977,1980,1982,1985,1988],{"class":67,"line":331},[279,1978,1979],{"class":288},"rollups ",[279,1981,325],{"class":284},[279,1983,1984],{"class":288}," table[label_hit ",[279,1986,1987],{"class":284},"|",[279,1989,1990],{"class":288}," shape_hit]\n",[279,1992,1993,1996,1998,2000,2002,2005,2007,2010,2012,2014,2016],{"class":67,"line":342},[279,1994,1995],{"class":288},"detail ",[279,1997,325],{"class":284},[279,1999,1918],{"class":288},[279,2001,1245],{"class":284},[279,2003,2004],{"class":288},"(label_hit ",[279,2006,1987],{"class":284},[279,2008,2009],{"class":288}," shape_hit)].reset_index(",[279,2011,960],{"class":592},[279,2013,325],{"class":284},[279,2015,965],{"class":423},[279,2017,569],{"class":288},[279,2019,2020],{"class":67,"line":354},[279,2021,316],{"emptyLinePlaceholder":315},[279,2023,2024,2027,2029,2032,2034,2037,2039],{"class":67,"line":366},[279,2025,2026],{"class":288},"computed ",[279,2028,325],{"class":284},[279,2030,2031],{"class":288}," detail.groupby(",[279,2033,393],{"class":350},[279,2035,2036],{"class":288},")[",[279,2038,403],{"class":350},[279,2040,2041],{"class":288},"].sum()\n",[279,2043,2044,2047,2049,2052,2054,2056,2059,2061,2064,2066,2068,2071,2073,2075,2077,2079,2081,2084,2086,2088,2090,2092,2094,2096,2098,2100],{"class":67,"line":376},[279,2045,2046],{"class":288},"reported ",[279,2048,325],{"class":284},[279,2050,2051],{"class":288}," rollups.assign(",[279,2053,1424],{"class":592},[279,2055,325],{"class":284},[279,2057,2058],{"class":288},"rollups[",[279,2060,393],{"class":350},[279,2062,2063],{"class":288},"].str.replace(",[279,2065,1168],{"class":284},[279,2067,898],{"class":350},[279,2069,2070],{"class":423},"\\s",[279,2072,1176],{"class":284},[279,2074,1898],{"class":350},[279,2076,1179],{"class":423},[279,2078,898],{"class":350},[279,2080,390],{"class":288},[279,2082,2083],{"class":350},"\"\"",[279,2085,390],{"class":288},[279,2087,1935],{"class":592},[279,2089,325],{"class":284},[279,2091,1940],{"class":423},[279,2093,390],{"class":288},[279,2095,1187],{"class":592},[279,2097,325],{"class":284},[279,2099,965],{"class":423},[279,2101,2102],{"class":288},")) \\\n",[279,2104,2105,2108,2110,2112,2114],{"class":67,"line":382},[279,2106,2107],{"class":288},"                  .set_index(",[279,2109,393],{"class":350},[279,2111,2036],{"class":288},[279,2113,403],{"class":350},[279,2115,2116],{"class":288},"]\n",[279,2118,2119,2122,2125,2128,2131],{"class":67,"line":408},[279,2120,2121],{"class":284},"assert",[279,2123,2124],{"class":288}," (computed ",[279,2126,2127],{"class":284},"==",[279,2129,2130],{"class":288}," reported.reindex(computed.index)).all(), ",[279,2132,2133],{"class":350},"\"subtotals do not reconcile\"\n",[279,2135,2136,2138],{"class":67,"line":484},[279,2137,615],{"class":423},[279,2139,2140],{"class":288},"(detail)\n",[279,2142,2143,2145],{"class":67,"line":548},[279,2144,615],{"class":423},[279,2146,2147],{"class":288},"(computed.to_dict())\n",[10,2149,2150,2151,2153,2154,2157,2158,202],{},"The last three statements are the part worth copying. The totals you remove were computed by the system that\nproduced the export, so comparing them with your own group sums is a free test that the right rows were\ndropped and nothing went missing in the read. When the rollups are ",[199,2152,230],{}," formulas — the output of Excel's\n",[175,2155,2156],{},"Data → Subtotal"," command — openpyxl can identify them exactly from the formula text, as shown in\n",[14,2159,226],{"href":225},[20,2161,29,2167,29,2170,29,2173,29,2177,29,2181,29,2184,29,2190,29,2195,29,2201,29,2206,29,2208,29,2212,29,2215,29,2218,29,2222,29,2225,29,2229,29,2232,29,2236,29,2240],{"viewBox":2162,"role":23,"ariaLabelledBy":2163,"xmlns":27,"style":2166},"0 0 720 212",[2164,2165],"msy-inflate-t","msy-inflate-d","width:100%;max-width:720px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[31,2168,2169],{"id":2164},"How rollup rows inflate a naive column sum",[35,2171,2172],{"id":2165},"With detail rows only the sum is the true total. Adding one layer of subtotals doubles it, and adding a grand total on top triples it.",[39,2174],{"x":41,"y":41,"width":2175,"height":2176,"fill":44},"720","212",[55,2178,2180],{"x":47,"y":1397,"style":2179},"font-size:12px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:start","column sum relative to the true total",[39,2182],{"x":47,"y":663,"width":1493,"height":2183,"rx":1521,"fill":44,"stroke":52,"style":671},"28",[55,2185,2189],{"x":2186,"y":2187,"style":2188},"105","65","font-size:12px;font-weight:600;fill:var(--text,#172033);text-anchor:middle","detail rows only",[39,2191],{"x":2192,"y":2193,"width":2194,"height":1434,"rx":1521,"fill":51,"stroke":52,"style":671},"204","47","330",[39,2196],{"x":2197,"y":2198,"width":2199,"height":2200,"rx":1497,"fill":148,"stroke":149,"style":671},"205","48","97","24",[55,2202,2205],{"x":2203,"y":2187,"style":2204},"546","font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:start","1× (correct)",[39,2207],{"x":47,"y":1400,"width":1493,"height":2183,"rx":1521,"fill":44,"stroke":52,"style":671},[55,2209,2211],{"x":2186,"y":2210,"style":2188},"109","+ subtotals",[39,2213],{"x":2192,"y":2214,"width":2194,"height":1434,"rx":1521,"fill":51,"stroke":52,"style":671},"91",[39,2216],{"x":2197,"y":93,"width":2217,"height":2200,"rx":1497,"fill":125,"stroke":126,"style":671},"196",[55,2219,2221],{"x":2203,"y":2210,"style":2220},"font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:start","2×",[39,2223],{"x":47,"y":2224,"width":1493,"height":2183,"rx":1521,"fill":44,"stroke":52,"style":671},"134",[55,2226,2228],{"x":2186,"y":2227,"style":2188},"153","+ grand total",[39,2230],{"x":2192,"y":2231,"width":2194,"height":1434,"rx":1521,"fill":51,"stroke":52,"style":671},"135",[39,2233],{"x":2197,"y":2234,"width":2235,"height":2200,"rx":1497,"fill":658,"stroke":659,"style":671},"136","295",[55,2237,2239],{"x":2203,"y":2227,"style":2238},"font-size:12px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:start","3×",[55,2241,2244],{"x":2242,"y":2243,"style":164},"360","194","nothing errors; the numbers are simply wrong",[167,2246,2248],{"id":2247},"reading-only-what-the-user-saw","Reading only what the user saw",[10,2250,2251,2253,2254,2257,2258,2261],{},[199,2252,258],{}," returns every stored row, including rows hidden by an autofilter, rows hidden by hand and rows\ninside collapsed outline groups. For a report somebody prepared and filtered on purpose, those rows are not part\nof the answer. openpyxl exposes each row's visibility through ",[199,2255,2256],{},"row_dimensions",", and every one of those three\nmechanisms sets the same ",[199,2259,2260],{},"hidden"," flag.",[270,2263,2265],{"className":272,"code":2264,"language":274,"meta":275,"style":275},"from openpyxl import load_workbook\n\nwb = Workbook()\nws = wb.active\nws.append([\"Order\", \"Region\", \"Amount\"])\nfor r in [(1, \"North\", 250), (2, \"South\", 120), (3, \"West\", 400)]:\n    ws.append(list(r))\nws.row_dimensions[3].hidden = True                    # the South row\nwb.save(\"filtered.xlsx\")\n\nsheet = load_workbook(\"filtered.xlsx\").active\nhidden = {i for i, dim in sheet.row_dimensions.items() if dim.hidden}\nframe = pd.read_excel(\"filtered.xlsx\")\nvisible = frame[~(pd.RangeIndex(len(frame)) + 2).isin(hidden)]\nprint(sorted(hidden), visible[\"Amount\"].sum())       # [3] 650\n",[199,2266,2267,2278,2282,2290,2298,2315,2366,2375,2393,2402,2406,2420,2446,2459,2488],{"__ignoreMap":275},[279,2268,2269,2271,2273,2275],{"class":67,"line":281},[279,2270,285],{"class":284},[279,2272,289],{"class":288},[279,2274,292],{"class":284},[279,2276,2277],{"class":288}," load_workbook\n",[279,2279,2280],{"class":67,"line":298},[279,2281,316],{"emptyLinePlaceholder":315},[279,2283,2284,2286,2288],{"class":67,"line":312},[279,2285,322],{"class":288},[279,2287,325],{"class":284},[279,2289,328],{"class":288},[279,2291,2292,2294,2296],{"class":67,"line":319},[279,2293,334],{"class":288},[279,2295,325],{"class":284},[279,2297,339],{"class":288},[279,2299,2300,2302,2305,2307,2309,2311,2313],{"class":67,"line":331},[279,2301,357],{"class":288},[279,2303,2304],{"class":350},"\"Order\"",[279,2306,390],{"class":288},[279,2308,393],{"class":350},[279,2310,390],{"class":288},[279,2312,403],{"class":350},[279,2314,363],{"class":288},[279,2316,2317,2319,2322,2324,2326,2328,2330,2332,2334,2337,2339,2341,2343,2345,2347,2350,2352,2354,2356,2359,2361,2364],{"class":67,"line":342},[279,2318,411],{"class":284},[279,2320,2321],{"class":288}," r ",[279,2323,417],{"class":284},[279,2325,420],{"class":288},[279,2327,941],{"class":423},[279,2329,390],{"class":288},[279,2331,429],{"class":350},[279,2333,390],{"class":288},[279,2335,2336],{"class":423},"250",[279,2338,442],{"class":288},[279,2340,1447],{"class":423},[279,2342,390],{"class":288},[279,2344,495],{"class":350},[279,2346,390],{"class":288},[279,2348,2349],{"class":423},"120",[279,2351,442],{"class":288},[279,2353,1463],{"class":423},[279,2355,390],{"class":288},[279,2357,2358],{"class":350},"\"West\"",[279,2360,390],{"class":288},[279,2362,2363],{"class":423},"400",[279,2365,545],{"class":288},[279,2367,2368,2370,2372],{"class":67,"line":354},[279,2369,551],{"class":288},[279,2371,554],{"class":423},[279,2373,2374],{"class":288},"(r))\n",[279,2376,2377,2380,2382,2385,2387,2390],{"class":67,"line":366},[279,2378,2379],{"class":288},"ws.row_dimensions[",[279,2381,1463],{"class":423},[279,2383,2384],{"class":288},"].hidden ",[279,2386,325],{"class":284},[279,2388,2389],{"class":423}," True",[279,2391,2392],{"class":977},"                    # the South row\n",[279,2394,2395,2397,2400],{"class":67,"line":376},[279,2396,563],{"class":288},[279,2398,2399],{"class":350},"\"filtered.xlsx\"",[279,2401,569],{"class":288},[279,2403,2404],{"class":67,"line":382},[279,2405,316],{"emptyLinePlaceholder":315},[279,2407,2408,2410,2412,2415,2417],{"class":67,"line":408},[279,2409,1140],{"class":288},[279,2411,325],{"class":284},[279,2413,2414],{"class":288}," load_workbook(",[279,2416,2399],{"class":350},[279,2418,2419],{"class":288},").active\n",[279,2421,2422,2425,2427,2430,2432,2435,2437,2440,2443],{"class":67,"line":484},[279,2423,2424],{"class":288},"hidden ",[279,2426,325],{"class":284},[279,2428,2429],{"class":288}," {i ",[279,2431,411],{"class":284},[279,2433,2434],{"class":288}," i, dim ",[279,2436,417],{"class":284},[279,2438,2439],{"class":288}," sheet.row_dimensions.items() ",[279,2441,2442],{"class":284},"if",[279,2444,2445],{"class":288}," dim.hidden}\n",[279,2447,2448,2451,2453,2455,2457],{"class":67,"line":548},[279,2449,2450],{"class":288},"frame ",[279,2452,325],{"class":284},[279,2454,585],{"class":288},[279,2456,2399],{"class":350},[279,2458,569],{"class":288},[279,2460,2461,2464,2466,2469,2471,2474,2477,2480,2482,2485],{"class":67,"line":560},[279,2462,2463],{"class":288},"visible ",[279,2465,325],{"class":284},[279,2467,2468],{"class":288}," frame[",[279,2470,1245],{"class":284},[279,2472,2473],{"class":288},"(pd.RangeIndex(",[279,2475,2476],{"class":423},"len",[279,2478,2479],{"class":288},"(frame)) ",[279,2481,927],{"class":284},[279,2483,2484],{"class":423}," 2",[279,2486,2487],{"class":288},").isin(hidden)]\n",[279,2489,2490,2492,2494,2496,2499,2501,2504],{"class":67,"line":572},[279,2491,615],{"class":423},[279,2493,876],{"class":288},[279,2495,889],{"class":423},[279,2497,2498],{"class":288},"(hidden), visible[",[279,2500,403],{"class":350},[279,2502,2503],{"class":288},"].sum())       ",[279,2505,2506],{"class":977},"# [3] 650\n",[10,2508,2509,2510,2513,2514,202],{},"Whether visible-only is ",[626,2511,2512],{},"correct"," is a judgement, not a technical question. A filtered report a person sends is\nusually meant to be read as filtered; an extract a system produced should be read in full even if someone left a\nfilter on. Make the choice an explicit parameter and log how many hidden rows were skipped or included. Grouped\ncolumns, hidden sheets and the zero-height-row trap are covered in\n",[14,2515,237],{"href":236},[167,2517,2519],{"id":2518},"recovering-meaning-from-colours-and-fonts","Recovering meaning from colours and fonts",[10,2521,2522,2523,2526,2527,2530],{},"The last kind of mess is information that was never typed at all: a task shaded green when done, a name struck\nthrough when the person left. pandas discards formatting, so read it with openpyxl. A solid fill's colour lives\nin ",[199,2524,2525],{},"fill.fgColor",", and its ",[199,2528,2529],{},"type"," tells you whether it is an explicit RGB value, a theme slot with a tint, or a\nlegacy indexed colour.",[270,2532,2534],{"className":272,"code":2533,"language":274,"meta":275,"style":275},"from openpyxl.styles import PatternFill\n\nwb = Workbook()\nws = wb.active\nws.append([\"Task\"])\nfor task, colour in [(\"Invoice run\", \"C6EFCE\"), (\"Audit prep\", \"FFC7CE\")]:\n    ws.append([task])\n    ws.cell(ws.max_row, 1).fill = PatternFill(\"solid\", start_color=colour)\nwb.save(\"tracker.xlsx\")\n\nSTATUS = {\"C6EFCE\": \"done\", \"FFC7CE\": \"blocked\"}\nsheet = load_workbook(\"tracker.xlsx\").active\nrows = []\nfor (cell,) in sheet.iter_rows(min_row=2, max_col=1):\n    color = cell.fill.fgColor\n    key = color.rgb[-6:] if cell.fill.fill_type and color.type == \"rgb\" else None\n    rows.append({\"task\": cell.value, \"status\": STATUS.get(key, key or \"not started\")})\nprint(pd.DataFrame(rows))\n",[199,2535,2536,2548,2552,2560,2568,2577,2608,2613,2641,2650,2654,2682,2694,2703,2734,2744,2784,2814],{"__ignoreMap":275},[279,2537,2538,2540,2543,2545],{"class":67,"line":281},[279,2539,285],{"class":284},[279,2541,2542],{"class":288}," openpyxl.styles ",[279,2544,292],{"class":284},[279,2546,2547],{"class":288}," PatternFill\n",[279,2549,2550],{"class":67,"line":298},[279,2551,316],{"emptyLinePlaceholder":315},[279,2553,2554,2556,2558],{"class":67,"line":312},[279,2555,322],{"class":288},[279,2557,325],{"class":284},[279,2559,328],{"class":288},[279,2561,2562,2564,2566],{"class":67,"line":319},[279,2563,334],{"class":288},[279,2565,325],{"class":284},[279,2567,339],{"class":288},[279,2569,2570,2572,2575],{"class":67,"line":331},[279,2571,357],{"class":288},[279,2573,2574],{"class":350},"\"Task\"",[279,2576,363],{"class":288},[279,2578,2579,2581,2584,2586,2588,2591,2593,2596,2598,2601,2603,2606],{"class":67,"line":342},[279,2580,411],{"class":284},[279,2582,2583],{"class":288}," task, colour ",[279,2585,417],{"class":284},[279,2587,420],{"class":288},[279,2589,2590],{"class":350},"\"Invoice run\"",[279,2592,390],{"class":288},[279,2594,2595],{"class":350},"\"C6EFCE\"",[279,2597,442],{"class":288},[279,2599,2600],{"class":350},"\"Audit prep\"",[279,2602,390],{"class":288},[279,2604,2605],{"class":350},"\"FFC7CE\"",[279,2607,545],{"class":288},[279,2609,2610],{"class":67,"line":354},[279,2611,2612],{"class":288},"    ws.append([task])\n",[279,2614,2615,2618,2620,2623,2625,2628,2631,2633,2636,2638],{"class":67,"line":366},[279,2616,2617],{"class":288},"    ws.cell(ws.max_row, ",[279,2619,941],{"class":423},[279,2621,2622],{"class":288},").fill ",[279,2624,325],{"class":284},[279,2626,2627],{"class":288}," PatternFill(",[279,2629,2630],{"class":350},"\"solid\"",[279,2632,390],{"class":288},[279,2634,2635],{"class":592},"start_color",[279,2637,325],{"class":284},[279,2639,2640],{"class":288},"colour)\n",[279,2642,2643,2645,2648],{"class":67,"line":376},[279,2644,563],{"class":288},[279,2646,2647],{"class":350},"\"tracker.xlsx\"",[279,2649,569],{"class":288},[279,2651,2652],{"class":67,"line":382},[279,2653,316],{"emptyLinePlaceholder":315},[279,2655,2656,2659,2661,2663,2665,2668,2671,2673,2675,2677,2680],{"class":67,"line":408},[279,2657,2658],{"class":423},"STATUS",[279,2660,737],{"class":284},[279,2662,740],{"class":288},[279,2664,2595],{"class":350},[279,2666,2667],{"class":288},": ",[279,2669,2670],{"class":350},"\"done\"",[279,2672,390],{"class":288},[279,2674,2605],{"class":350},[279,2676,2667],{"class":288},[279,2678,2679],{"class":350},"\"blocked\"",[279,2681,756],{"class":288},[279,2683,2684,2686,2688,2690,2692],{"class":67,"line":484},[279,2685,1140],{"class":288},[279,2687,325],{"class":284},[279,2689,2414],{"class":288},[279,2691,2647],{"class":350},[279,2693,2419],{"class":288},[279,2695,2696,2699,2701],{"class":67,"line":548},[279,2697,2698],{"class":288},"rows ",[279,2700,325],{"class":284},[279,2702,1229],{"class":288},[279,2704,2705,2707,2710,2712,2715,2718,2720,2722,2724,2727,2729,2731],{"class":67,"line":560},[279,2706,411],{"class":284},[279,2708,2709],{"class":288}," (cell,) ",[279,2711,417],{"class":284},[279,2713,2714],{"class":288}," sheet.iter_rows(",[279,2716,2717],{"class":592},"min_row",[279,2719,325],{"class":284},[279,2721,1447],{"class":423},[279,2723,390],{"class":288},[279,2725,2726],{"class":592},"max_col",[279,2728,325],{"class":284},[279,2730,941],{"class":423},[279,2732,2733],{"class":288},"):\n",[279,2735,2736,2739,2741],{"class":67,"line":572},[279,2737,2738],{"class":288},"    color ",[279,2740,325],{"class":284},[279,2742,2743],{"class":288}," cell.fill.fgColor\n",[279,2745,2746,2749,2751,2754,2757,2759,2762,2764,2767,2770,2773,2775,2778,2781],{"class":67,"line":577},[279,2747,2748],{"class":288},"    key ",[279,2750,325],{"class":284},[279,2752,2753],{"class":288}," color.rgb[",[279,2755,2756],{"class":284},"-",[279,2758,1521],{"class":423},[279,2760,2761],{"class":288},":] ",[279,2763,2442],{"class":284},[279,2765,2766],{"class":288}," cell.fill.fill_type ",[279,2768,2769],{"class":284},"and",[279,2771,2772],{"class":288}," color.type ",[279,2774,2127],{"class":284},[279,2776,2777],{"class":350}," \"rgb\"",[279,2779,2780],{"class":284}," else",[279,2782,2783],{"class":423}," None\n",[279,2785,2786,2789,2792,2795,2798,2800,2802,2805,2808,2811],{"class":67,"line":612},[279,2787,2788],{"class":288},"    rows.append({",[279,2790,2791],{"class":350},"\"task\"",[279,2793,2794],{"class":288},": cell.value, ",[279,2796,2797],{"class":350},"\"status\"",[279,2799,2667],{"class":288},[279,2801,2658],{"class":423},[279,2803,2804],{"class":288},".get(key, key ",[279,2806,2807],{"class":284},"or",[279,2809,2810],{"class":350}," \"not started\"",[279,2812,2813],{"class":288},")})\n",[279,2815,2816,2818],{"class":67,"line":1232},[279,2817,615],{"class":423},[279,2819,2820],{"class":288},"(pd.DataFrame(rows))\n",[10,2822,2823,2824,2827,2828,202],{},"Before trusting any colour you read, check ",[199,2825,2826],{},"ws.conditional_formatting",". A cell painted by a conditional format\nrule has no fill of its own — the colour is computed by Excel at display time — so openpyxl reports it as\nunfilled. In that case, re-implement the rule's condition in pandas instead. Theme colours, font flags and\nwriting a real Status column back are covered in\n",[14,2829,244],{"href":243},[167,2831,2833],{"id":2832},"putting-it-together-a-defensive-loader","Putting it together: a defensive loader",[10,2835,2836,2837,2840],{},"In production these steps belong in one function per source, run in a fixed order, with each decision logged.\nThe order matters: locate the table before repairing it, remove rollups before typing columns (a ",[199,2838,2839],{},"Total","\nlabel in a numeric column blocks conversion), and validate last.",[270,2842,2844],{"className":272,"code":2843,"language":274,"meta":275,"style":275},"def load_export(path: str) -> pd.DataFrame:\n    raw = pd.read_excel(path, header=None).replace(r\"^\\s*$\", np.nan, regex=True)\n    pos = header_position(raw)\n    frame = raw.iloc[pos + 1:].set_axis(raw.iloc[pos], axis=1).rename_axis(None, axis=1)\n    frame = frame.dropna(axis=1, how=\"all\").dropna(axis=0, how=\"all\")\n\n    is_rollup = (frame[\"Region\"].astype(str).str.contains(TOTAL, case=False, regex=True)\n                 | frame[\"Order ID\"].isna())\n    detail = frame[~is_rollup].reset_index(drop=True)\n    detail = detail.astype({\"Order ID\": \"int64\", \"Amount\": \"float64\"})\n\n    missing = REQUIRED - {norm(c) for c in detail.columns}\n    if missing:\n        raise ValueError(f\"{path}: missing columns {sorted(missing)}\")\n    print(f\"{path}: header on row {pos + 1}, {len(detail)} records, {int(is_rollup.sum())} rollups removed\")\n    return detail\n\nprint(load_export(\"export.xlsx\"))\n",[199,2845,2846,2861,2902,2911,2946,2988,2992,3030,3042,3064,3092,3096,3122,3130,3166,3223,3230,3234],{"__ignoreMap":275},[279,2847,2848,2850,2853,2856,2858],{"class":67,"line":281},[279,2849,765],{"class":284},[279,2851,2852],{"class":768}," load_export",[279,2854,2855],{"class":288},"(path: ",[279,2857,775],{"class":423},[279,2859,2860],{"class":288},") -> pd.DataFrame:\n",[279,2862,2863,2866,2868,2871,2873,2875,2877,2880,2882,2884,2886,2888,2890,2892,2894,2896,2898,2900],{"class":67,"line":298},[279,2864,2865],{"class":288},"    raw ",[279,2867,325],{"class":284},[279,2869,2870],{"class":288}," pd.read_excel(path, ",[279,2872,603],{"class":592},[279,2874,325],{"class":284},[279,2876,464],{"class":423},[279,2878,2879],{"class":288},").replace(",[279,2881,1168],{"class":284},[279,2883,898],{"class":350},[279,2885,1173],{"class":423},[279,2887,1176],{"class":284},[279,2889,1179],{"class":423},[279,2891,898],{"class":350},[279,2893,1184],{"class":288},[279,2895,1187],{"class":592},[279,2897,325],{"class":284},[279,2899,965],{"class":423},[279,2901,569],{"class":288},[279,2903,2904,2907,2909],{"class":67,"line":312},[279,2905,2906],{"class":288},"    pos ",[279,2908,325],{"class":284},[279,2910,914],{"class":288},[279,2912,2913,2916,2918,2920,2922,2924,2926,2928,2930,2932,2934,2936,2938,2940,2942,2944],{"class":67,"line":319},[279,2914,2915],{"class":288},"    frame ",[279,2917,325],{"class":284},[279,2919,924],{"class":288},[279,2921,927],{"class":284},[279,2923,930],{"class":423},[279,2925,933],{"class":288},[279,2927,936],{"class":592},[279,2929,325],{"class":284},[279,2931,941],{"class":423},[279,2933,944],{"class":288},[279,2935,464],{"class":423},[279,2937,390],{"class":288},[279,2939,936],{"class":592},[279,2941,325],{"class":284},[279,2943,941],{"class":423},[279,2945,569],{"class":288},[279,2947,2948,2950,2952,2955,2957,2959,2961,2963,2965,2967,2969,2972,2974,2976,2978,2980,2982,2984,2986],{"class":67,"line":331},[279,2949,2915],{"class":288},[279,2951,325],{"class":284},[279,2953,2954],{"class":288}," frame.dropna(",[279,2956,936],{"class":592},[279,2958,325],{"class":284},[279,2960,941],{"class":423},[279,2962,390],{"class":288},[279,2964,1275],{"class":592},[279,2966,325],{"class":284},[279,2968,1280],{"class":350},[279,2970,2971],{"class":288},").dropna(",[279,2973,936],{"class":592},[279,2975,325],{"class":284},[279,2977,41],{"class":423},[279,2979,390],{"class":288},[279,2981,1275],{"class":592},[279,2983,325],{"class":284},[279,2985,1280],{"class":350},[279,2987,569],{"class":288},[279,2989,2990],{"class":67,"line":342},[279,2991,316],{"emptyLinePlaceholder":315},[279,2993,2994,2997,2999,3002,3004,3006,3008,3010,3012,3014,3016,3018,3020,3022,3024,3026,3028],{"class":67,"line":354},[279,2995,2996],{"class":288},"    is_rollup ",[279,2998,325],{"class":284},[279,3000,3001],{"class":288}," (frame[",[279,3003,393],{"class":350},[279,3005,1923],{"class":288},[279,3007,775],{"class":423},[279,3009,1928],{"class":288},[279,3011,1874],{"class":423},[279,3013,390],{"class":288},[279,3015,1935],{"class":592},[279,3017,325],{"class":284},[279,3019,1940],{"class":423},[279,3021,390],{"class":288},[279,3023,1187],{"class":592},[279,3025,325],{"class":284},[279,3027,965],{"class":423},[279,3029,569],{"class":288},[279,3031,3032,3035,3037,3039],{"class":67,"line":366},[279,3033,3034],{"class":284},"                 |",[279,3036,2468],{"class":288},[279,3038,387],{"class":350},[279,3040,3041],{"class":288},"].isna())\n",[279,3043,3044,3047,3049,3051,3053,3056,3058,3060,3062],{"class":67,"line":376},[279,3045,3046],{"class":288},"    detail ",[279,3048,325],{"class":284},[279,3050,2468],{"class":288},[279,3052,1245],{"class":284},[279,3054,3055],{"class":288},"is_rollup].reset_index(",[279,3057,960],{"class":592},[279,3059,325],{"class":284},[279,3061,965],{"class":423},[279,3063,569],{"class":288},[279,3065,3066,3068,3070,3073,3075,3077,3080,3082,3084,3086,3089],{"class":67,"line":382},[279,3067,3046],{"class":288},[279,3069,325],{"class":284},[279,3071,3072],{"class":288}," detail.astype({",[279,3074,387],{"class":350},[279,3076,2667],{"class":288},[279,3078,3079],{"class":350},"\"int64\"",[279,3081,390],{"class":288},[279,3083,403],{"class":350},[279,3085,2667],{"class":288},[279,3087,3088],{"class":350},"\"float64\"",[279,3090,3091],{"class":288},"})\n",[279,3093,3094],{"class":67,"line":408},[279,3095,316],{"emptyLinePlaceholder":315},[279,3097,3098,3101,3103,3106,3109,3112,3114,3117,3119],{"class":67,"line":484},[279,3099,3100],{"class":288},"    missing ",[279,3102,325],{"class":284},[279,3104,3105],{"class":423}," REQUIRED",[279,3107,3108],{"class":284}," -",[279,3110,3111],{"class":288}," {norm(c) ",[279,3113,411],{"class":284},[279,3115,3116],{"class":288}," c ",[279,3118,417],{"class":284},[279,3120,3121],{"class":288}," detail.columns}\n",[279,3123,3124,3127],{"class":67,"line":548},[279,3125,3126],{"class":284},"    if",[279,3128,3129],{"class":288}," missing:\n",[279,3131,3132,3135,3137,3139,3141,3143,3145,3148,3150,3153,3155,3157,3160,3162,3164],{"class":67,"line":560},[279,3133,3134],{"class":284},"        raise",[279,3136,873],{"class":423},[279,3138,876],{"class":288},[279,3140,879],{"class":284},[279,3142,898],{"class":350},[279,3144,886],{"class":885},[279,3146,3147],{"class":288},"path",[279,3149,895],{"class":885},[279,3151,3152],{"class":350},": missing columns ",[279,3154,886],{"class":885},[279,3156,889],{"class":423},[279,3158,3159],{"class":288},"(missing)",[279,3161,895],{"class":885},[279,3163,898],{"class":350},[279,3165,569],{"class":288},[279,3167,3168,3170,3172,3174,3176,3178,3180,3182,3185,3187,3189,3191,3193,3195,3197,3199,3201,3204,3206,3209,3211,3213,3216,3218,3221],{"class":67,"line":572},[279,3169,1351],{"class":423},[279,3171,876],{"class":288},[279,3173,879],{"class":284},[279,3175,898],{"class":350},[279,3177,886],{"class":885},[279,3179,3147],{"class":288},[279,3181,895],{"class":885},[279,3183,3184],{"class":350},": header on row ",[279,3186,886],{"class":885},[279,3188,909],{"class":288},[279,3190,927],{"class":284},[279,3192,930],{"class":423},[279,3194,895],{"class":885},[279,3196,390],{"class":350},[279,3198,886],{"class":885},[279,3200,2476],{"class":423},[279,3202,3203],{"class":288},"(detail)",[279,3205,895],{"class":885},[279,3207,3208],{"class":350}," records, ",[279,3210,886],{"class":885},[279,3212,818],{"class":423},[279,3214,3215],{"class":288},"(is_rollup.sum())",[279,3217,895],{"class":885},[279,3219,3220],{"class":350}," rollups removed\"",[279,3222,569],{"class":288},[279,3224,3225,3227],{"class":67,"line":577},[279,3226,783],{"class":284},[279,3228,3229],{"class":288}," detail\n",[279,3231,3232],{"class":67,"line":612},[279,3233,316],{"emptyLinePlaceholder":315},[279,3235,3236,3238,3241,3243],{"class":67,"line":1232},[279,3237,615],{"class":423},[279,3239,3240],{"class":288},"(load_export(",[279,3242,566],{"class":350},[279,3244,1327],{"class":288},[10,3246,3247,3248,202],{},"The log line is not decoration. When a total looks wrong next month, \"header on row 6\" instead of \"row 4\", or\n\"3 rollups removed\" instead of \"2\", points directly at what changed in the export. For formal schema checks on\nthe result — types, ranges, allowed values — hand the frame to the tools in\n",[14,3249,3251],{"href":3250},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002F","Validating Excel Data with Python",[20,3253,29,3258,29,3261,29,3264,29,3266,29,3269,29,3274,29,3278,29,3281,29,3286,29,3290,29,3293,29,3297,29,3300,29,3302,29,3307,29,3310,29,3313,29,3318,29,3321],{"viewBox":3254,"role":23,"ariaLabelledBy":3255,"xmlns":27,"style":2166},"0 0 720 380",[3256,3257],"msy-order-t","msy-order-d",[31,3259,3260],{"id":3256},"The order of operations in a defensive loader",[35,3262,3263],{"id":3257},"First read raw and normalise whitespace, then locate the header, then trim empty rows and columns, then remove rollup rows, then set types, and finally validate required columns and log the decisions.",[39,3265],{"x":41,"y":41,"width":2175,"height":107,"fill":44},[39,3267],{"x":47,"y":47,"width":3268,"height":1433,"rx":50,"fill":51,"stroke":52,"style":53},"680",[55,3270,3273],{"x":1403,"y":3271,"style":3272},"51","font-size:14px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:start","1 read raw",[55,3275,3277],{"x":2336,"y":3271,"style":3276},"font-size:12px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:start","header=None, whitespace-only cells become NaN",[39,3279],{"x":47,"y":3280,"width":3268,"height":1433,"rx":50,"fill":81,"stroke":72,"style":53},"84",[55,3282,3285],{"x":1403,"y":3283,"style":3284},"115","font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:start","2 locate",[55,3287,3289],{"x":2336,"y":3283,"style":3288},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:start","header row found by required labels",[39,3291],{"x":47,"y":3292,"width":3268,"height":1433,"rx":50,"fill":81,"stroke":72,"style":53},"148",[55,3294,3296],{"x":1403,"y":3295,"style":3284},"179","3 trim",[55,3298,3299],{"x":2336,"y":3295,"style":3288},"drop empty columns and rows around the table",[39,3301],{"x":47,"y":2176,"width":3268,"height":1433,"rx":50,"fill":125,"stroke":126,"style":53},[55,3303,3306],{"x":1403,"y":3304,"style":3305},"243","font-size:14px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:start","4 remove rollups",[55,3308,3309],{"x":2336,"y":3304,"style":3288},"label or missing record id, before any typing",[39,3311],{"x":47,"y":3312,"width":3268,"height":1433,"rx":50,"fill":148,"stroke":149,"style":53},"276",[55,3314,3317],{"x":1403,"y":3315,"style":3316},"307","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:start","5 type and validate",[55,3319,3320],{"x":2336,"y":3315,"style":3288},"astype, required columns, log what was decided",[55,3322,3323],{"x":2242,"y":2242,"style":164},"typing before removing totals fails on the word Total in a number column",[167,3325,3327],{"id":3326},"when-to-push-back-on-the-file-instead","When to push back on the file instead",[10,3329,3330],{},"Every technique here is a workaround for a workbook that was not designed to be read by code. Some are\npermanent — exports from systems you do not control will always have banners — but some are worth fixing at the\nsource. If a team shades rows to record status, a Status column with a conditional format looks identical to its\nreaders and removes the colour-decoding step entirely. If a summary tab stacks tables, each table can be an Excel\ntable object with a name, which openpyxl reads exactly. And if people keep filtering the file before sending it,\nask for the unfiltered extract and apply the filter in code, where it is visible and repeatable. The goal of a\nparser for messy sheets is to keep the job running today, not to make the mess permanent.",[167,3332,3334],{"id":3333},"matching-the-symptom-to-the-technique","Matching the symptom to the technique",[10,3336,3337,3338,3341,3342,3345],{},"Messy sheets rarely announce what is wrong with them. The symptom you see in pandas is usually indirect — a\ncolumn called ",[199,3339,3340],{},"Unnamed: 3",", a sum that is too high, a ",[199,3343,3344],{},"KeyError"," three functions away from the read — so it helps\nto work backwards from what the DataFrame looks like to the layout problem that caused it.",[3347,3348,3349,3365],"table",{},[3350,3351,3352],"thead",{},[3353,3354,3355,3359,3362],"tr",{},[3356,3357,3358],"th",{},"What you see in pandas",[3356,3360,3361],{},"Likely layout problem",[3356,3363,3364],{},"Technique",[3366,3367,3368,3387,3401,3416,3427,3440,3456],"tbody",{},[3353,3369,3370,3381,3384],{},[3371,3372,3373,3374,390,3377,3380],"td",{},"Columns named ",[199,3375,3376],{},"Unnamed: 1",[199,3378,3379],{},"Unnamed: 2"," and a title as a column name",[3371,3382,3383],{},"Banner rows above the header",[3371,3385,3386],{},"Header detection by labels",[3353,3388,3389,3392,3395],{},[3371,3390,3391],{},"Real column names appear in the first data row",[3371,3393,3394],{},"Header on a lower row, or two header rows",[3371,3396,3397,3398,3400],{},"Header detection, then ",[199,3399,219],{}," if two rows",[3353,3402,3403,3410,3413],{},[3371,3404,3405,3406,3409],{},"Large blocks of ",[199,3407,3408],{},"NaN"," splitting the frame",[3371,3411,3412],{},"Several tables on one sheet",[3371,3414,3415],{},"Blank-line split or anchor search",[3353,3417,3418,3421,3424],{},[3371,3419,3420],{},"Sums are a whole multiple of the expected total",[3371,3422,3423],{},"Subtotal and grand total rows",[3371,3425,3426],{},"Remove rollups, then reconcile",[3353,3428,3429,3432,3435],{},[3371,3430,3431],{},"Row count higher than the person who sent the file reports",[3371,3433,3434],{},"Filter, hidden rows or collapsed groups",[3371,3436,3437,3438],{},"Read visibility from ",[199,3439,2256],{},[3353,3441,3442,3445,3448],{},[3371,3443,3444],{},"A status everybody talks about has no column",[3371,3446,3447],{},"Status stored as colour or font",[3371,3449,3450,3451,629,3453],{},"Read ",[199,3452,2525],{},[199,3454,3455],{},"font",[3353,3457,3458,3461,3464],{},[3371,3459,3460],{},"Half the rows have an empty group label",[3371,3462,3463],{},"Label printed once per group",[3371,3465,3466,3469],{},[199,3467,3468],{},"ffill()"," after removing rollups",[10,3471,3472],{},"Several of these usually appear together. An export with a banner almost always has a footer; a report with\nsubtotals often prints its group label once. Work through them in the loader order shown above rather than fixing\nwhichever one you notice first.",[167,3474,3476],{"id":3475},"testing-a-loader-against-real-files","Testing a loader against real files",[10,3478,3479],{},"A messy-sheet loader is a set of rules about files you do not control, so the most valuable test data is the files\nthemselves. Keep a small folder of real examples — anonymised if necessary — including every layout variant that has\never broken the job, and assert the facts that must hold for each one: the columns found, the number of records, and\na control total.",[270,3481,3483],{"className":272,"code":3482,"language":274,"meta":275,"style":275},"import pytest\n\nCASES = [\n    (\"export.xlsx\", 4, 3600.0),\n]\n\n@pytest.mark.parametrize(\"path, records, total\", CASES)\ndef test_load_export(path, records, total):\n    frame = load_export(path)\n    assert {\"order id\", \"region\", \"amount\"} \u003C= {norm(c) for c in frame.columns}\n    assert len(frame) == records\n    assert frame[\"Amount\"].sum() == pytest.approx(total)\n\ntest_load_export(\"export.xlsx\", 4, 3600.0)\nprint(\"loader test passed\")\n",[199,3484,3485,3492,3496,3506,3524,3528,3532,3548,3558,3567,3600,3615,3631,3635,3652],{"__ignoreMap":275},[279,3486,3487,3489],{"class":67,"line":281},[279,3488,292],{"class":284},[279,3490,3491],{"class":288}," pytest\n",[279,3493,3494],{"class":67,"line":298},[279,3495,316],{"emptyLinePlaceholder":315},[279,3497,3498,3501,3503],{"class":67,"line":312},[279,3499,3500],{"class":423},"CASES",[279,3502,737],{"class":284},[279,3504,3505],{"class":288}," [\n",[279,3507,3508,3511,3513,3515,3517,3519,3522],{"class":67,"line":319},[279,3509,3510],{"class":288},"    (",[279,3512,566],{"class":350},[279,3514,390],{"class":288},[279,3516,1478],{"class":423},[279,3518,390],{"class":288},[279,3520,3521],{"class":423},"3600.0",[279,3523,481],{"class":288},[279,3525,3526],{"class":67,"line":331},[279,3527,2116],{"class":288},[279,3529,3530],{"class":67,"line":342},[279,3531,316],{"emptyLinePlaceholder":315},[279,3533,3534,3537,3539,3542,3544,3546],{"class":67,"line":354},[279,3535,3536],{"class":768},"@pytest.mark.parametrize",[279,3538,876],{"class":288},[279,3540,3541],{"class":350},"\"path, records, total\"",[279,3543,390],{"class":288},[279,3545,3500],{"class":423},[279,3547,569],{"class":288},[279,3549,3550,3552,3555],{"class":67,"line":366},[279,3551,765],{"class":284},[279,3553,3554],{"class":768}," test_load_export",[279,3556,3557],{"class":288},"(path, records, total):\n",[279,3559,3560,3562,3564],{"class":67,"line":376},[279,3561,2915],{"class":288},[279,3563,325],{"class":284},[279,3565,3566],{"class":288}," load_export(path)\n",[279,3568,3569,3572,3574,3576,3578,3580,3582,3584,3587,3589,3591,3593,3595,3597],{"class":67,"line":382},[279,3570,3571],{"class":284},"    assert",[279,3573,740],{"class":288},[279,3575,743],{"class":350},[279,3577,390],{"class":288},[279,3579,748],{"class":350},[279,3581,390],{"class":288},[279,3583,753],{"class":350},[279,3585,3586],{"class":288},"} ",[279,3588,844],{"class":284},[279,3590,3111],{"class":288},[279,3592,411],{"class":284},[279,3594,3116],{"class":288},[279,3596,417],{"class":284},[279,3598,3599],{"class":288}," frame.columns}\n",[279,3601,3602,3604,3607,3610,3612],{"class":67,"line":408},[279,3603,3571],{"class":284},[279,3605,3606],{"class":423}," len",[279,3608,3609],{"class":288},"(frame) ",[279,3611,2127],{"class":284},[279,3613,3614],{"class":288}," records\n",[279,3616,3617,3619,3621,3623,3626,3628],{"class":67,"line":484},[279,3618,3571],{"class":284},[279,3620,2468],{"class":288},[279,3622,403],{"class":350},[279,3624,3625],{"class":288},"].sum() ",[279,3627,2127],{"class":284},[279,3629,3630],{"class":288}," pytest.approx(total)\n",[279,3632,3633],{"class":67,"line":548},[279,3634,316],{"emptyLinePlaceholder":315},[279,3636,3637,3640,3642,3644,3646,3648,3650],{"class":67,"line":560},[279,3638,3639],{"class":288},"test_load_export(",[279,3641,566],{"class":350},[279,3643,390],{"class":288},[279,3645,1478],{"class":423},[279,3647,390],{"class":288},[279,3649,3521],{"class":423},[279,3651,569],{"class":288},[279,3653,3654,3656,3658,3661],{"class":67,"line":572},[279,3655,615],{"class":423},[279,3657,876],{"class":288},[279,3659,3660],{"class":350},"\"loader test passed\"",[279,3662,569],{"class":288},[10,3664,3665,3666,3669,3670,202],{},"Every time a new file breaks the loader, add it to the folder ",[626,3667,3668],{},"before"," fixing the rule. The fix then has a failing\ntest to turn green, and the old layouts stay covered so a rule tightened for April cannot silently break March. The\nbroader pattern for testing spreadsheet output is described in\n",[14,3671,3673],{"href":3672},"\u002Fautomating-reporting-workflows\u002Ftesting-and-packaging-excel-automation-scripts\u002Ftest-excel-output-with-pytest\u002F","Test Excel Output with pytest",[167,3675,3677],{"id":3676},"key-takeaways","Key takeaways",[188,3679,3680,3686,3691,3697,3706,3709,3715,3721],{},[191,3681,3682,3683,3685],{},"Read the sheet with ",[199,3684,65],{}," first; decide where the table is before deciding what it means.",[191,3687,3688,3689,202],{},"Find headers by required labels with a subset test, never by fixed ",[199,3690,201],{},[191,3692,3693,3694,3696],{},"Normalise whitespace to ",[199,3695,3408],{}," before treating any row or column as blank.",[191,3698,3699,3700,3702,3703,3705],{},"Use ",[199,3701,219],{}," with ",[199,3704,1740],{}," for two-row headers, and stack the top level for analysis.",[191,3707,3708],{},"Remove subtotal and total rows before summing or typing, then reconcile against them.",[191,3710,3711,3712,3714],{},"Hidden, filtered and grouped rows are all visible to pandas; openpyxl's ",[199,3713,2256],{}," tells them apart.",[191,3716,3717,3718,3720],{},"Colours are readable through ",[199,3719,2525],{},", except when conditional formatting paints them.",[191,3722,3723],{},"Log every decision a loader makes, so layout changes show up in the log before they show up in a report.",[167,3725,3727],{"id":3726},"frequently-asked-questions","Frequently asked questions",[10,3729,3730,3733],{},[175,3731,3732],{},"Is there a library that parses messy Excel sheets automatically?","\nNot reliably. Layout inference tools exist, but they guess, and a wrong guess produces plausible-looking data. Explicit rules — required labels, blank boundaries, rollup patterns — are short to write and fail loudly when the file changes.",[10,3735,3736,3739],{},[175,3737,3738],{},"Should I use pandas or openpyxl for messy sheets?","\nBoth. pandas with header=None is the fastest way to get values into a frame for locating and slicing. openpyxl is needed for anything stored outside values: hidden rows, fills, fonts, merged ranges, named tables and formula text.",[10,3741,3742,3745],{},[175,3743,3744],{},"How do I handle a sheet that changes layout every month?","\nReplace every positional assumption with a content rule — the header row by its labels, table edges by blank rows, totals by label or missing identifier — and log what each rule found so that a change is visible.",[10,3747,3748,3751],{},[175,3749,3750],{},"Why do my sums come out too high?","\nAlmost always because the sheet contains subtotal or grand total rows. Remove them before aggregating and compare your group sums with the removed totals to confirm.",[10,3753,3754,3757],{},[175,3755,3756],{},"Can these techniques handle .xls and .ods files?","\nEverything that runs on the pandas frame works for any engine pandas supports. The openpyxl techniques — visibility, colours, formulas, named tables — need .xlsx or .xlsm, so convert older formats first.",[167,3759,3761],{"id":3760},"related","Related",[188,3763,3764,3770,3774,3778,3782,3786,3790,3794,3801,3808],{},[191,3765,3766,3767,3769],{},"Up one level: ",[14,3768,17],{"href":16}," — the full section on reshaping, cleaning and validating spreadsheet data.",[191,3771,3772],{},[14,3773,196],{"href":195},[191,3775,3776],{},[14,3777,208],{"href":207},[191,3779,3780],{},[14,3781,215],{"href":214},[191,3783,3784],{},[14,3785,226],{"href":225},[191,3787,3788],{},[14,3789,237],{"href":236},[191,3791,3792],{},[14,3793,244],{"href":243},[191,3795,3796,3800],{},[14,3797,3799],{"href":3798},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002F","Cleaning Excel Data With Pandas"," — the next step once the table is out.",[191,3802,3803,3807],{},[14,3804,3806],{"href":3805},"\u002Fgetting-started-with-python-excel-automation\u002Freading-excel-files-with-pandas\u002F","Reading Excel Files with Pandas"," — the read_excel options these techniques build on.",[191,3809,3810,3812],{},[14,3811,3251],{"href":3250}," — checking the extracted table before it is trusted.",[10,3814,3815],{},"More in this section:",[188,3817,3818,3824,3830,3836,3842,3848,3854],{},[191,3819,3820],{},[14,3821,3823],{"href":3822},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fcombine-sheets-with-different-layouts-into-one-table\u002F","Combine Sheets with Different Layouts into One Table",[191,3825,3826],{},[14,3827,3829],{"href":3828},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fparse-numbers-with-units-in-excel-cells\u002F","Parse Numbers with Units in Excel Cells",[191,3831,3832],{},[14,3833,3835],{"href":3834},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fread-a-calendar-style-sheet-into-a-table\u002F","Read a Calendar-Style Sheet into a Table",[191,3837,3838],{},[14,3839,3841],{"href":3840},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fread-a-sheet-where-records-run-across-columns\u002F","Read an Excel Sheet Where Records Run Across Columns",[191,3843,3844],{},[14,3845,3847],{"href":3846},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fread-dropdown-lists-from-an-existing-workbook\u002F","Read Dropdown Lists and Validation Rules from an Existing Workbook",[191,3849,3850],{},[14,3851,3853],{"href":3852},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fstop-reading-at-notes-and-footers-below-the-data\u002F","Stop Reading at Notes and Footers Below the Data",[191,3855,3856],{},[14,3857,3859],{"href":3858},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fturn-a-pivot-table-export-back-into-tidy-data\u002F","Turn a Pivot Table Export Back into Tidy Data",[3861,3862,3863],"style",{},"html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}",{"title":275,"searchDepth":298,"depth":298,"links":3865},[3866,3867,3868,3869,3870,3871,3872,3873,3874,3875,3876,3877,3878,3879,3880],{"id":169,"depth":298,"text":170},{"id":251,"depth":298,"text":252},{"id":717,"depth":298,"text":718},{"id":1008,"depth":298,"text":1009},{"id":1538,"depth":298,"text":1539},{"id":1856,"depth":298,"text":1857},{"id":2247,"depth":298,"text":2248},{"id":2518,"depth":298,"text":2519},{"id":2832,"depth":298,"text":2833},{"id":3326,"depth":298,"text":3327},{"id":3333,"depth":298,"text":3334},{"id":3475,"depth":298,"text":3476},{"id":3676,"depth":298,"text":3677},{"id":3726,"depth":298,"text":3727},{"id":3760,"depth":298,"text":3761},"2026-09-17","Get clean tables out of worksheets built for people: shifting banners, several tables per sheet, two-row headers, subtotal rows, hidden rows and status colours — with pandas and openpyxl.","md",[3885,3887,3889,3891,3893],{"q":3732,"a":3886},"Not reliably. Layout inference tools exist, but they guess, and a wrong guess produces plausible-looking data. Explicit rules — required labels, blank boundaries, rollup patterns — are short to write and fail loudly when the file changes.",{"q":3738,"a":3888},"Both. pandas with header=None is the fastest way to get values into a frame for locating and slicing. openpyxl is needed for anything stored outside values: hidden rows, fills, fonts, merged ranges, named tables and formula text.",{"q":3744,"a":3890},"Replace every positional assumption with a content rule — the header row by its labels, table edges by blank rows, totals by label or missing identifier — and log what each rule found so that a change is visible.",{"q":3750,"a":3892},"Almost always because the sheet contains subtotal or grand total rows. Remove them before aggregating and compare your group sums with the removed totals to confirm.",{"q":3756,"a":3894},"Everything that runs on the pandas frame works for any engine pandas supports. The openpyxl techniques — visibility, colours, formulas, named tables — need .xlsx or .xlsm, so convert older formats first.","extracting data from messy excel sheets",null,{"breadcrumb":3898},[3899,3902,3903],{"name":3900,"item":3901},"Home","\u002F",{"name":17,"item":16},{"name":5,"item":3904},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002F","\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets",{"title":3907,"description":3908},"Extracting Data from Messy Excel Sheets with Python","A practical guide to parsing human-built Excel sheets in Python: detect header rows, split multiple tables, read multi-row headers, drop subtotals, skip hidden rows and read colours.","extracting-data-from-messy-excel-sheets","advanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Findex",[3912,3913],"messy-sheets","reading-excel","guide","mILIZH-QxLcv49i2vfqqb05o2f0Zq8DY8-1tI9EX62A",[3917,3921],{"title":3918,"path":3919,"stem":3920,"children":-1},"YEARFRAC and DAYS360 Equivalents in Python","\u002Fadvanced-data-transformation-and-cleaning\u002Fexcel-formula-equivalents-in-pandas\u002Fyearfrac-and-days360-equivalents-in-python","advanced-data-transformation-and-cleaning\u002Fexcel-formula-equivalents-in-pandas\u002Fyearfrac-and-days360-equivalents-in-python\u002Findex",{"title":3823,"path":3922,"stem":3923,"children":-1},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fcombine-sheets-with-different-layouts-into-one-table","advanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fcombine-sheets-with-different-layouts-into-one-table\u002Findex",[3925],{"path":3926,"title":3927},"\u002Fadvanced-data-transformation-and-cleaning","Advanced Data Transformation and Cleaning for Excel Automation",[3929,3933,3936],{"path":3930,"title":3931,"kind":3932},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fextract-hyperlink-urls-from-excel-cells","Extract Hyperlink URLs from Excel Cells with openpyxl","how-to",{"path":3934,"title":3935,"kind":3932},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fextract-report-header-fields-from-an-excel-sheet","Extract Report Header Fields from an Excel Sheet with Python",{"path":3937,"title":3938,"kind":3932},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fparse-records-that-span-multiple-rows-in-excel","Parse Records That Span Multiple Rows in Excel with pandas",[3940,3943,3946,3949,3950,3952],{"path":3941,"title":3942,"kind":3932},"\u002Fformatting-and-charting-excel-reports-with-python\u002Fstyling-excel-cells-with-openpyxl\u002Ffreeze-header-row-in-excel-with-openpyxl","Freeze the Header Row in Excel with openpyxl",{"path":3944,"title":3945,"kind":3932},"\u002Fgetting-started-with-python-excel-automation\u002Freading-excel-files-with-pandas\u002Fhandle-merged-cells-when-reading-excel-with-pandas","Handle Merged Cells When Reading Excel with pandas",{"path":3947,"title":3948,"kind":3932},"\u002Fgetting-started-with-python-excel-automation\u002Fusing-openpyxl-for-excel-file-manipulation\u002Funmerge-cells-and-fill-values-with-openpyxl","Unmerge Cells and Fill the Values with openpyxl",{"path":3922,"title":3823,"kind":3932},{"path":3951,"title":196,"kind":3932},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Ffind-the-header-row-in-an-excel-sheet-with-a-shifting-layout",{"path":3953,"title":3829,"kind":3932},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fparse-numbers-with-units-in-excel-cells",1790972053407]