[{"data":1,"prerenderedAt":2240},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fskip-hidden-and-filtered-rows-when-reading-excel":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fskip-hidden-and-filtered-rows-when-reading-excel":2205,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fskip-hidden-and-filtered-rows-when-reading-excel":2214,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fskip-hidden-and-filtered-rows-when-reading-excel":2220,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fskip-hidden-and-filtered-rows-when-reading-excel":2221},{"id":4,"title":5,"body":6,"dateModified":2170,"datePublished":2170,"description":2171,"extension":2172,"faq":2173,"keyword":2182,"kind":2183,"meta":2184,"navigation":346,"path":2193,"seo":2194,"slug":2197,"stem":2198,"topics":2199,"type":2203,"__hash__":2204},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fskip-hidden-and-filtered-rows-when-reading-excel\u002Findex.md","Skip Hidden and Filtered Rows When Reading Excel",{"type":7,"value":8,"toc":2157},"minimark",[9,24,278,283,311,318,640,652,656,667,799,806,896,900,903,1262,1285,1289,1304,1389,1392,1482,1486,1504,1588,1591,1595,1598,1827,1838,1842,1957,1960,1964,1980,2053,2060,2064,2082,2086,2093,2099,2105,2111,2115,2153],[10,11,12,13,17,18,23],"p",{},"A colleague filters a sheet down to the North and West regions, saves it, and sends it over with \"the\nnumbers are in the attached\". The total they saw on screen was 650. The total your script computes from\nthe same file is 1,170, because ",[14,15,16],"code",{},"pd.read_excel()"," ignores filters, hidden rows, collapsed groups and\nhidden columns — it reads every cell that holds a value. Whether that is a bug depends on what the file\nis meant to say. This guide from ",[19,20,22],"a",{"href":21},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002F","Extracting Data from Messy Excel Sheets","\nshows how to see what the user saw, and how to decide which version you want.",[25,26,34,35,34,39,34,43,34,50,34,57,34,66,34,70,34,75,34,79,34,83,34,86,34,90,34,94,34,98,34,100,34,105,34,107,34,111,34,113,34,116,34,118,34,121,34,123,34,126,34,128,34,132,34,134,34,138,34,140,34,143,34,145,34,148,34,150,34,153,34,157,34,161,34,165,34,167,34,170,34,172,34,175,34,177,34,183,34,188,34,191,34,195,34,199,34,203,34,205,34,207,34,209,34,212,34,214,34,217,34,221,34,224,34,228,34,230,34,233,34,235,34,238,34,240,34,243,34,245,34,248,34,251,34,253,34,256,34,258,34,260,34,262,34,265,34,267,34,270,34,273],"svg",{"viewBox":27,"role":28,"ariaLabelledBy":29,"xmlns":32,"style":33},"0 0 576 314","img",[30,31],"hid-sheet-t","hid-sheet-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:576px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[36,37,38],"title",{"id":30},"Rows hidden by an autofilter and by hand",[40,41,42],"desc",{"id":31},"An orders sheet with five data rows. Rows 3 and 6 are hidden by an autofilter on Region, row 4 is hidden manually, and column D is a hidden internal note column.",[44,45],"rect",{"x":46,"y":46,"width":47,"height":48,"fill":49},"0","576","314","#ffffff",[51,52,56],"text",{"x":53,"y":54,"style":55},"288","32","font-size:13px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","what is stored versus what is visible",[44,58],{"x":59,"y":60,"width":61,"height":62,"rx":46,"fill":63,"stroke":64,"style":65},"20","50","36","30","#e7ebef","var(--line,#cdd5e6)","stroke-width:1px",[44,67],{"x":68,"y":60,"width":69,"height":62,"rx":46,"fill":63,"stroke":64,"style":65},"56","70",[51,71,74],{"x":72,"y":69,"style":73},"91","font-size:11.5px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[44,76],{"x":77,"y":60,"width":78,"height":62,"rx":46,"fill":63,"stroke":64,"style":65},"126","80",[51,80,82],{"x":81,"y":69,"style":73},"166","B",[44,84],{"x":85,"y":60,"width":78,"height":62,"rx":46,"fill":63,"stroke":64,"style":65},"206",[51,87,89],{"x":88,"y":69,"style":73},"246","C",[44,91],{"x":92,"y":60,"width":93,"height":62,"rx":46,"fill":63,"stroke":64,"style":65},"286","110",[51,95,97],{"x":96,"y":69,"style":73},"341","D",[44,99],{"x":59,"y":78,"width":61,"height":62,"rx":46,"fill":63,"stroke":64,"style":65},[51,101,104],{"x":102,"y":103,"style":73},"38","100","1",[44,106],{"x":68,"y":78,"width":69,"height":62,"rx":46,"fill":63,"stroke":64,"style":65},[51,108,110],{"x":72,"y":103,"style":109},"font-size:11.5px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:middle","Order",[44,112],{"x":77,"y":78,"width":78,"height":62,"rx":46,"fill":63,"stroke":64,"style":65},[51,114,115],{"x":81,"y":103,"style":109},"Region",[44,117],{"x":85,"y":78,"width":78,"height":62,"rx":46,"fill":63,"stroke":64,"style":65},[51,119,120],{"x":88,"y":103,"style":109},"Amount",[44,122],{"x":92,"y":78,"width":93,"height":62,"rx":46,"fill":63,"stroke":64,"style":65},[51,124,125],{"x":96,"y":103,"style":109},"Note (hidden)",[44,127],{"x":59,"y":93,"width":61,"height":62,"rx":46,"fill":63,"stroke":64,"style":65},[51,129,131],{"x":102,"y":130,"style":73},"130","2",[44,133],{"x":68,"y":93,"width":69,"height":62,"rx":46,"fill":49,"stroke":64,"style":65},[51,135,137],{"x":72,"y":130,"style":136},"font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","1001",[44,139],{"x":77,"y":93,"width":78,"height":62,"rx":46,"fill":49,"stroke":64,"style":65},[51,141,142],{"x":81,"y":130,"style":136},"North",[44,144],{"x":85,"y":93,"width":78,"height":62,"rx":46,"fill":49,"stroke":64,"style":65},[51,146,147],{"x":88,"y":130,"style":136},"250",[44,149],{"x":92,"y":93,"width":93,"height":62,"rx":46,"fill":49,"stroke":64,"style":65},[44,151],{"x":59,"y":152,"width":61,"height":62,"rx":46,"fill":63,"stroke":64,"style":65},"140",[51,154,156],{"x":102,"y":155,"style":73},"160","3",[44,158],{"x":68,"y":152,"width":69,"height":62,"rx":46,"fill":159,"stroke":160,"style":65},"#fee8f2","var(--accent,#f43f8f)",[51,162,164],{"x":72,"y":155,"style":163},"font-size:11.5px;font-weight:600;fill:var(--accent-ink,#be185d);text-anchor:middle","1002",[44,166],{"x":77,"y":152,"width":78,"height":62,"rx":46,"fill":159,"stroke":160,"style":65},[51,168,169],{"x":81,"y":155,"style":163},"South",[44,171],{"x":85,"y":152,"width":78,"height":62,"rx":46,"fill":159,"stroke":160,"style":65},[51,173,174],{"x":88,"y":155,"style":163},"120",[44,176],{"x":92,"y":152,"width":93,"height":62,"rx":46,"fill":49,"stroke":64,"style":65},[44,178],{"x":179,"y":180,"width":152,"height":181,"rx":182,"fill":159,"stroke":160,"style":65},"416","142","26","8",[51,184,187],{"x":185,"y":186,"style":163},"486","159","filtered out",[44,189],{"x":59,"y":190,"width":61,"height":62,"rx":46,"fill":63,"stroke":64,"style":65},"170",[51,192,194],{"x":102,"y":193,"style":73},"190","4",[44,196],{"x":68,"y":190,"width":69,"height":62,"rx":46,"fill":197,"stroke":198,"style":65},"#fdefd8","var(--gold,#b4740a)",[51,200,202],{"x":72,"y":193,"style":201},"font-size:11.5px;font-weight:600;fill:var(--gold-ink,#7a4e06);text-anchor:middle","1003",[44,204],{"x":77,"y":190,"width":78,"height":62,"rx":46,"fill":197,"stroke":198,"style":65},[51,206,142],{"x":81,"y":193,"style":201},[44,208],{"x":85,"y":190,"width":78,"height":62,"rx":46,"fill":197,"stroke":198,"style":65},[51,210,211],{"x":88,"y":193,"style":201},"90",[44,213],{"x":92,"y":190,"width":93,"height":62,"rx":46,"fill":49,"stroke":64,"style":65},[44,215],{"x":179,"y":216,"width":152,"height":181,"rx":182,"fill":197,"stroke":198,"style":65},"172",[51,218,220],{"x":185,"y":219,"style":201},"189","hidden by hand",[44,222],{"x":59,"y":223,"width":61,"height":62,"rx":46,"fill":63,"stroke":64,"style":65},"200",[51,225,227],{"x":102,"y":226,"style":73},"220","5",[44,229],{"x":68,"y":223,"width":69,"height":62,"rx":46,"fill":49,"stroke":64,"style":65},[51,231,232],{"x":72,"y":226,"style":136},"1004",[44,234],{"x":77,"y":223,"width":78,"height":62,"rx":46,"fill":49,"stroke":64,"style":65},[51,236,237],{"x":81,"y":226,"style":136},"West",[44,239],{"x":85,"y":223,"width":78,"height":62,"rx":46,"fill":49,"stroke":64,"style":65},[51,241,242],{"x":88,"y":226,"style":136},"400",[44,244],{"x":92,"y":223,"width":93,"height":62,"rx":46,"fill":49,"stroke":64,"style":65},[44,246],{"x":59,"y":247,"width":61,"height":62,"rx":46,"fill":63,"stroke":64,"style":65},"230",[51,249,250],{"x":102,"y":147,"style":73},"6",[44,252],{"x":68,"y":247,"width":69,"height":62,"rx":46,"fill":159,"stroke":160,"style":65},[51,254,255],{"x":72,"y":147,"style":163},"1005",[44,257],{"x":77,"y":247,"width":78,"height":62,"rx":46,"fill":159,"stroke":160,"style":65},[51,259,169],{"x":81,"y":147,"style":163},[44,261],{"x":85,"y":247,"width":78,"height":62,"rx":46,"fill":159,"stroke":160,"style":65},[51,263,264],{"x":88,"y":147,"style":163},"310",[44,266],{"x":92,"y":247,"width":93,"height":62,"rx":46,"fill":49,"stroke":64,"style":65},[44,268],{"x":179,"y":269,"width":152,"height":181,"rx":182,"fill":159,"stroke":160,"style":65},"232",[51,271,187],{"x":185,"y":272,"style":163},"249",[51,274,277],{"x":53,"y":275,"style":276},"292","font-size:12.5px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:middle","read_excel returns all five rows and the hidden column",[279,280,282],"h2",{"id":281},"prerequisites","Prerequisites",[284,285,290],"pre",{"className":286,"code":287,"language":288,"meta":289,"style":289},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install openpyxl pandas\n","bash","",[14,291,292],{"__ignoreMap":289},[293,294,297,301,305,308],"span",{"class":295,"line":296},"line",1,[293,298,300],{"class":299},"sMTad","pip",[293,302,304],{"class":303},"srMev"," install",[293,306,307],{"class":303}," openpyxl",[293,309,310],{"class":303}," pandas\n",[10,312,313,314,317],{},"This builds ",[14,315,316],{},"orders.xlsx"," with an autofilter, a manually hidden row, a hidden column and a hidden sheet:",[284,319,323],{"className":320,"code":321,"language":322,"meta":289,"style":289},"language-python shiki shiki-themes github-light github-dark-high-contrast","from openpyxl import Workbook\n\nwb = Workbook()\nws = wb.active\nws.title = \"Orders\"\nws.append([\"Order\", \"Region\", \"Amount\", \"Internal note\"])\nfor order, region, amount in [(1001, \"North\", 250), (1002, \"South\", 120), (1003, \"North\", 90),\n                              (1004, \"West\", 400), (1005, \"South\", 310)]:\n    ws.append([order, region, amount, \"check\"])\n\nws.auto_filter.ref = \"A1:D6\"\nws.auto_filter.add_filter_column(1, [\"North\", \"West\"])\nws.row_dimensions[3].hidden = True      # filtered out: South\nws.row_dimensions[6].hidden = True      # filtered out: South\nws.row_dimensions[4].hidden = True      # hidden by hand\nws.column_dimensions[\"D\"].hidden = True\nwb.create_sheet(\"Archive\").sheet_state = \"hidden\"\nwb.save(\"orders.xlsx\")\n","python",[14,324,325,341,348,360,371,382,410,466,498,509,514,525,544,564,579,595,611,628],{"__ignoreMap":289},[293,326,327,331,335,338],{"class":295,"line":296},[293,328,330],{"class":329},"s-kum","from",[293,332,334],{"class":333},"skGVy"," openpyxl ",[293,336,337],{"class":329},"import",[293,339,340],{"class":333}," Workbook\n",[293,342,344],{"class":295,"line":343},2,[293,345,347],{"emptyLinePlaceholder":346},true,"\n",[293,349,351,354,357],{"class":295,"line":350},3,[293,352,353],{"class":333},"wb ",[293,355,356],{"class":329},"=",[293,358,359],{"class":333}," Workbook()\n",[293,361,363,366,368],{"class":295,"line":362},4,[293,364,365],{"class":333},"ws ",[293,367,356],{"class":329},[293,369,370],{"class":333}," wb.active\n",[293,372,374,377,379],{"class":295,"line":373},5,[293,375,376],{"class":333},"ws.title ",[293,378,356],{"class":329},[293,380,381],{"class":303}," \"Orders\"\n",[293,383,385,388,391,394,397,399,402,404,407],{"class":295,"line":384},6,[293,386,387],{"class":333},"ws.append([",[293,389,390],{"class":303},"\"Order\"",[293,392,393],{"class":333},", ",[293,395,396],{"class":303},"\"Region\"",[293,398,393],{"class":333},[293,400,401],{"class":303},"\"Amount\"",[293,403,393],{"class":333},[293,405,406],{"class":303},"\"Internal note\"",[293,408,409],{"class":333},"])\n",[293,411,413,416,419,422,425,428,430,433,435,437,440,442,444,447,449,451,453,455,457,459,461,463],{"class":295,"line":412},7,[293,414,415],{"class":329},"for",[293,417,418],{"class":333}," order, region, amount ",[293,420,421],{"class":329},"in",[293,423,424],{"class":333}," [(",[293,426,137],{"class":427},"sP0c6",[293,429,393],{"class":333},[293,431,432],{"class":303},"\"North\"",[293,434,393],{"class":333},[293,436,147],{"class":427},[293,438,439],{"class":333},"), (",[293,441,164],{"class":427},[293,443,393],{"class":333},[293,445,446],{"class":303},"\"South\"",[293,448,393],{"class":333},[293,450,174],{"class":427},[293,452,439],{"class":333},[293,454,202],{"class":427},[293,456,393],{"class":333},[293,458,432],{"class":303},[293,460,393],{"class":333},[293,462,211],{"class":427},[293,464,465],{"class":333},"),\n",[293,467,469,472,474,476,479,481,483,485,487,489,491,493,495],{"class":295,"line":468},8,[293,470,471],{"class":333},"                              (",[293,473,232],{"class":427},[293,475,393],{"class":333},[293,477,478],{"class":303},"\"West\"",[293,480,393],{"class":333},[293,482,242],{"class":427},[293,484,439],{"class":333},[293,486,255],{"class":427},[293,488,393],{"class":333},[293,490,446],{"class":303},[293,492,393],{"class":333},[293,494,264],{"class":427},[293,496,497],{"class":333},")]:\n",[293,499,501,504,507],{"class":295,"line":500},9,[293,502,503],{"class":333},"    ws.append([order, region, amount, ",[293,505,506],{"class":303},"\"check\"",[293,508,409],{"class":333},[293,510,512],{"class":295,"line":511},10,[293,513,347],{"emptyLinePlaceholder":346},[293,515,517,520,522],{"class":295,"line":516},11,[293,518,519],{"class":333},"ws.auto_filter.ref ",[293,521,356],{"class":329},[293,523,524],{"class":303}," \"A1:D6\"\n",[293,526,528,531,533,536,538,540,542],{"class":295,"line":527},12,[293,529,530],{"class":333},"ws.auto_filter.add_filter_column(",[293,532,104],{"class":427},[293,534,535],{"class":333},", [",[293,537,432],{"class":303},[293,539,393],{"class":333},[293,541,478],{"class":303},[293,543,409],{"class":333},[293,545,547,550,552,555,557,560],{"class":295,"line":546},13,[293,548,549],{"class":333},"ws.row_dimensions[",[293,551,156],{"class":427},[293,553,554],{"class":333},"].hidden ",[293,556,356],{"class":329},[293,558,559],{"class":427}," True",[293,561,563],{"class":562},"s-wDw","      # filtered out: South\n",[293,565,567,569,571,573,575,577],{"class":295,"line":566},14,[293,568,549],{"class":333},[293,570,250],{"class":427},[293,572,554],{"class":333},[293,574,356],{"class":329},[293,576,559],{"class":427},[293,578,563],{"class":562},[293,580,582,584,586,588,590,592],{"class":295,"line":581},15,[293,583,549],{"class":333},[293,585,194],{"class":427},[293,587,554],{"class":333},[293,589,356],{"class":329},[293,591,559],{"class":427},[293,593,594],{"class":562},"      # hidden by hand\n",[293,596,598,601,604,606,608],{"class":295,"line":597},16,[293,599,600],{"class":333},"ws.column_dimensions[",[293,602,603],{"class":303},"\"D\"",[293,605,554],{"class":333},[293,607,356],{"class":329},[293,609,610],{"class":427}," True\n",[293,612,614,617,620,623,625],{"class":295,"line":613},17,[293,615,616],{"class":333},"wb.create_sheet(",[293,618,619],{"class":303},"\"Archive\"",[293,621,622],{"class":333},").sheet_state ",[293,624,356],{"class":329},[293,626,627],{"class":303}," \"hidden\"\n",[293,629,631,634,637],{"class":295,"line":630},18,[293,632,633],{"class":333},"wb.save(",[293,635,636],{"class":303},"\"orders.xlsx\"",[293,638,639],{"class":333},")\n",[10,641,642,643,646,647,651],{},"Note the three ",[14,644,645],{},"row_dimensions"," lines. An autofilter in the file is only a ",[648,649,650],"em",{},"description"," of the filter;\nExcel records which rows it hid by marking each one hidden, and openpyxl does not apply filters itself.\nA workbook saved from Excel after filtering already has those flags set, which is what you will read.",[279,653,655],{"id":654},"step-1-find-the-hidden-rows","Step 1 — Find the hidden rows",[10,657,658,659,662,663,666],{},"Every row that is hidden — by a filter, by hand, or by a collapsed outline group — carries\n",[14,660,661],{},"hidden=True"," in ",[14,664,665],{},"ws.row_dimensions",". Rows with default formatting have no entry at all, so iterate over\nthe entries that exist rather than over every row number.",[284,668,670],{"className":320,"code":669,"language":322,"meta":289,"style":289},"from openpyxl import load_workbook\n\nwb = load_workbook(\"orders.xlsx\")\nws = wb[\"Orders\"]\n\nhidden_rows = {idx for idx, dim in ws.row_dimensions.items() if dim.hidden}\nprint(sorted(hidden_rows))          # [3, 4, 6]\n\nprint(ws.auto_filter.ref)           # A1:D6\nfor column in ws.auto_filter.filterColumn:\n    print(column.colId, column.filters.filter)   # 1 ['North', 'West']\n",[14,671,672,683,687,700,715,719,745,762,766,776,788],{"__ignoreMap":289},[293,673,674,676,678,680],{"class":295,"line":296},[293,675,330],{"class":329},[293,677,334],{"class":333},[293,679,337],{"class":329},[293,681,682],{"class":333}," load_workbook\n",[293,684,685],{"class":295,"line":343},[293,686,347],{"emptyLinePlaceholder":346},[293,688,689,691,693,696,698],{"class":295,"line":350},[293,690,353],{"class":333},[293,692,356],{"class":329},[293,694,695],{"class":333}," load_workbook(",[293,697,636],{"class":303},[293,699,639],{"class":333},[293,701,702,704,706,709,712],{"class":295,"line":362},[293,703,365],{"class":333},[293,705,356],{"class":329},[293,707,708],{"class":333}," wb[",[293,710,711],{"class":303},"\"Orders\"",[293,713,714],{"class":333},"]\n",[293,716,717],{"class":295,"line":373},[293,718,347],{"emptyLinePlaceholder":346},[293,720,721,724,726,729,731,734,736,739,742],{"class":295,"line":384},[293,722,723],{"class":333},"hidden_rows ",[293,725,356],{"class":329},[293,727,728],{"class":333}," {idx ",[293,730,415],{"class":329},[293,732,733],{"class":333}," idx, dim ",[293,735,421],{"class":329},[293,737,738],{"class":333}," ws.row_dimensions.items() ",[293,740,741],{"class":329},"if",[293,743,744],{"class":333}," dim.hidden}\n",[293,746,747,750,753,756,759],{"class":295,"line":412},[293,748,749],{"class":427},"print",[293,751,752],{"class":333},"(",[293,754,755],{"class":427},"sorted",[293,757,758],{"class":333},"(hidden_rows))          ",[293,760,761],{"class":562},"# [3, 4, 6]\n",[293,763,764],{"class":295,"line":468},[293,765,347],{"emptyLinePlaceholder":346},[293,767,768,770,773],{"class":295,"line":500},[293,769,749],{"class":427},[293,771,772],{"class":333},"(ws.auto_filter.ref)           ",[293,774,775],{"class":562},"# A1:D6\n",[293,777,778,780,783,785],{"class":295,"line":511},[293,779,415],{"class":329},[293,781,782],{"class":333}," column ",[293,784,421],{"class":329},[293,786,787],{"class":333}," ws.auto_filter.filterColumn:\n",[293,789,790,793,796],{"class":295,"line":516},[293,791,792],{"class":427},"    print",[293,794,795],{"class":333},"(column.colId, column.filters.filter)   ",[293,797,798],{"class":562},"# 1 ['North', 'West']\n",[10,800,801,802,805],{},"The filter definition is worth logging even when you only use the hidden flags. It tells a human reading\nyour job's output ",[648,803,804],{},"why"," rows were excluded, which matters the first time somebody asks where the South\norders went.",[25,807,34,813,34,816,34,819,34,823,34,829,34,835,34,838,34,842,34,846,34,850,34,853,34,857,34,861,34,864,34,867,34,870,34,875,34,880,34,884,34,887,34,889,34,892],{"viewBox":808,"role":28,"ariaLabelledBy":809,"xmlns":32,"style":812},"0 0 760 191",[810,811],"hid-sources-t","hid-sources-d","width:100%;max-width:760px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[36,814,815],{"id":810},"Three ways a row ends up hidden, one flag to read",[40,817,818],{"id":811},"An autofilter, a manual hide and a collapsed outline group all set the hidden flag on the row dimension, which is the single property to check.",[44,820],{"x":46,"y":46,"width":821,"height":822,"fill":49},"760","191",[44,824],{"x":59,"y":59,"width":825,"height":826,"rx":827,"fill":159,"stroke":160,"style":828},"226.7","127","14","stroke-width:2px",[51,830,834],{"x":831,"y":832,"style":833},"133.3","46","font-size:13px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","autofilter",[295,836],{"x1":61,"y1":68,"x2":837,"y2":68,"stroke":160,"style":65},"230.7",[51,839,841],{"x":831,"y":840,"style":136},"78","Excel applies the filter",[51,843,845],{"x":831,"y":844,"style":136},"101","marks rows hidden",[51,847,849],{"x":831,"y":848,"style":136},"124","filter kept in auto_filter",[44,851],{"x":852,"y":59,"width":825,"height":826,"rx":827,"fill":197,"stroke":198,"style":828},"266.7",[51,854,220],{"x":855,"y":832,"style":856},"380","font-size:13px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle",[295,858],{"x1":859,"y1":68,"x2":860,"y2":68,"stroke":198,"style":65},"282.7","477.3",[51,862,863],{"x":855,"y":840,"style":136},"right-click → Hide",[51,865,866],{"x":855,"y":844,"style":136},"marks row hidden",[51,868,869],{"x":855,"y":848,"style":136},"no other trace",[44,871],{"x":872,"y":59,"width":825,"height":826,"rx":827,"fill":873,"stroke":874,"style":828},"513.3","#f0f4ff","var(--brand,#5b5cf0)",[51,876,879],{"x":877,"y":832,"style":878},"626.7","font-size:13px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","collapsed group",[295,881],{"x1":882,"y1":68,"x2":883,"y2":68,"stroke":874,"style":65},"529.3","724",[51,885,886],{"x":877,"y":840,"style":136},"outline level set",[51,888,845],{"x":877,"y":844,"style":136},[51,890,891],{"x":877,"y":848,"style":136},"outlineLevel > 0",[51,893,895],{"x":855,"y":894,"style":276},"177","all three land in row_dimensions[n].hidden",[279,897,899],{"id":898},"step-2-build-a-dataframe-from-visible-rows-only","Step 2 — Build a DataFrame from visible rows only",[10,901,902],{},"The safest way to keep rows aligned with their flags is to build the DataFrame from openpyxl directly,\nskipping hidden rows as you go. Row numbers then never have to be translated between Excel's one-based\nnumbering and a pandas index.",[284,904,906],{"className":320,"code":905,"language":322,"meta":289,"style":289},"import pandas as pd\n\ndef read_visible(ws, header_row: int = 1, skip_hidden_columns: bool = True) -> pd.DataFrame:\n    hidden_r = {i for i, d in ws.row_dimensions.items() if d.hidden}\n    hidden_c = set()\n    for dim in ws.column_dimensions.values():\n        if dim.hidden:\n            hidden_c.update(range(dim.min, dim.max + 1))    # grouped columns share one entry\n\n    keep = [c for c in range(1, ws.max_column + 1)\n            if not (skip_hidden_columns and c in hidden_c)]\n    header = [ws.cell(header_row, c).value for c in keep]\n    rows = [\n        [ws.cell(r, c).value for c in keep]\n        for r in range(header_row + 1, ws.max_row + 1)\n        if r not in hidden_r\n    ]\n    return pd.DataFrame(rows, columns=header)\n\nvisible = read_visible(ws)\nprint(visible)\n#    Order Region  Amount\n# 0   1001  North     250\n# 1   1004   West     400\nprint(visible[\"Amount\"].sum())       # 650 — the number the colleague saw\n",[14,907,908,921,925,959,983,996,1009,1017,1039,1043,1076,1097,1116,1126,1139,1167,1182,1187,1204,1209,1220,1228,1234,1240,1246],{"__ignoreMap":289},[293,909,910,912,915,918],{"class":295,"line":296},[293,911,337],{"class":329},[293,913,914],{"class":333}," pandas ",[293,916,917],{"class":329},"as",[293,919,920],{"class":333}," pd\n",[293,922,923],{"class":295,"line":343},[293,924,347],{"emptyLinePlaceholder":346},[293,926,927,930,934,937,940,943,946,949,952,954,956],{"class":295,"line":350},[293,928,929],{"class":329},"def",[293,931,933],{"class":932},"s_Opv"," read_visible",[293,935,936],{"class":333},"(ws, header_row: ",[293,938,939],{"class":427},"int",[293,941,942],{"class":329}," =",[293,944,945],{"class":427}," 1",[293,947,948],{"class":333},", skip_hidden_columns: ",[293,950,951],{"class":427},"bool",[293,953,942],{"class":329},[293,955,559],{"class":427},[293,957,958],{"class":333},") -> pd.DataFrame:\n",[293,960,961,964,966,969,971,974,976,978,980],{"class":295,"line":362},[293,962,963],{"class":333},"    hidden_r ",[293,965,356],{"class":329},[293,967,968],{"class":333}," {i ",[293,970,415],{"class":329},[293,972,973],{"class":333}," i, d ",[293,975,421],{"class":329},[293,977,738],{"class":333},[293,979,741],{"class":329},[293,981,982],{"class":333}," d.hidden}\n",[293,984,985,988,990,993],{"class":295,"line":373},[293,986,987],{"class":333},"    hidden_c ",[293,989,356],{"class":329},[293,991,992],{"class":427}," set",[293,994,995],{"class":333},"()\n",[293,997,998,1001,1004,1006],{"class":295,"line":384},[293,999,1000],{"class":329},"    for",[293,1002,1003],{"class":333}," dim ",[293,1005,421],{"class":329},[293,1007,1008],{"class":333}," ws.column_dimensions.values():\n",[293,1010,1011,1014],{"class":295,"line":412},[293,1012,1013],{"class":329},"        if",[293,1015,1016],{"class":333}," dim.hidden:\n",[293,1018,1019,1022,1025,1028,1031,1033,1036],{"class":295,"line":468},[293,1020,1021],{"class":333},"            hidden_c.update(",[293,1023,1024],{"class":427},"range",[293,1026,1027],{"class":333},"(dim.min, dim.max ",[293,1029,1030],{"class":329},"+",[293,1032,945],{"class":427},[293,1034,1035],{"class":333},"))    ",[293,1037,1038],{"class":562},"# grouped columns share one entry\n",[293,1040,1041],{"class":295,"line":500},[293,1042,347],{"emptyLinePlaceholder":346},[293,1044,1045,1048,1050,1053,1055,1058,1060,1063,1065,1067,1070,1072,1074],{"class":295,"line":511},[293,1046,1047],{"class":333},"    keep ",[293,1049,356],{"class":329},[293,1051,1052],{"class":333}," [c ",[293,1054,415],{"class":329},[293,1056,1057],{"class":333}," c ",[293,1059,421],{"class":329},[293,1061,1062],{"class":427}," range",[293,1064,752],{"class":333},[293,1066,104],{"class":427},[293,1068,1069],{"class":333},", ws.max_column ",[293,1071,1030],{"class":329},[293,1073,945],{"class":427},[293,1075,639],{"class":333},[293,1077,1078,1081,1084,1087,1090,1092,1094],{"class":295,"line":516},[293,1079,1080],{"class":329},"            if",[293,1082,1083],{"class":329}," not",[293,1085,1086],{"class":333}," (skip_hidden_columns ",[293,1088,1089],{"class":329},"and",[293,1091,1057],{"class":333},[293,1093,421],{"class":329},[293,1095,1096],{"class":333}," hidden_c)]\n",[293,1098,1099,1102,1104,1107,1109,1111,1113],{"class":295,"line":527},[293,1100,1101],{"class":333},"    header ",[293,1103,356],{"class":329},[293,1105,1106],{"class":333}," [ws.cell(header_row, c).value ",[293,1108,415],{"class":329},[293,1110,1057],{"class":333},[293,1112,421],{"class":329},[293,1114,1115],{"class":333}," keep]\n",[293,1117,1118,1121,1123],{"class":295,"line":546},[293,1119,1120],{"class":333},"    rows ",[293,1122,356],{"class":329},[293,1124,1125],{"class":333}," [\n",[293,1127,1128,1131,1133,1135,1137],{"class":295,"line":566},[293,1129,1130],{"class":333},"        [ws.cell(r, c).value ",[293,1132,415],{"class":329},[293,1134,1057],{"class":333},[293,1136,421],{"class":329},[293,1138,1115],{"class":333},[293,1140,1141,1144,1147,1149,1151,1154,1156,1158,1161,1163,1165],{"class":295,"line":581},[293,1142,1143],{"class":329},"        for",[293,1145,1146],{"class":333}," r ",[293,1148,421],{"class":329},[293,1150,1062],{"class":427},[293,1152,1153],{"class":333},"(header_row ",[293,1155,1030],{"class":329},[293,1157,945],{"class":427},[293,1159,1160],{"class":333},", ws.max_row ",[293,1162,1030],{"class":329},[293,1164,945],{"class":427},[293,1166,639],{"class":333},[293,1168,1169,1171,1173,1176,1179],{"class":295,"line":597},[293,1170,1013],{"class":329},[293,1172,1146],{"class":333},[293,1174,1175],{"class":329},"not",[293,1177,1178],{"class":329}," in",[293,1180,1181],{"class":333}," hidden_r\n",[293,1183,1184],{"class":295,"line":613},[293,1185,1186],{"class":333},"    ]\n",[293,1188,1189,1192,1195,1199,1201],{"class":295,"line":630},[293,1190,1191],{"class":329},"    return",[293,1193,1194],{"class":333}," pd.DataFrame(rows, ",[293,1196,1198],{"class":1197},"sa561","columns",[293,1200,356],{"class":329},[293,1202,1203],{"class":333},"header)\n",[293,1205,1207],{"class":295,"line":1206},19,[293,1208,347],{"emptyLinePlaceholder":346},[293,1210,1212,1215,1217],{"class":295,"line":1211},20,[293,1213,1214],{"class":333},"visible ",[293,1216,356],{"class":329},[293,1218,1219],{"class":333}," read_visible(ws)\n",[293,1221,1223,1225],{"class":295,"line":1222},21,[293,1224,749],{"class":427},[293,1226,1227],{"class":333},"(visible)\n",[293,1229,1231],{"class":295,"line":1230},22,[293,1232,1233],{"class":562},"#    Order Region  Amount\n",[293,1235,1237],{"class":295,"line":1236},23,[293,1238,1239],{"class":562},"# 0   1001  North     250\n",[293,1241,1243],{"class":295,"line":1242},24,[293,1244,1245],{"class":562},"# 1   1004   West     400\n",[293,1247,1249,1251,1254,1256,1259],{"class":295,"line":1248},25,[293,1250,749],{"class":427},[293,1252,1253],{"class":333},"(visible[",[293,1255,401],{"class":303},[293,1257,1258],{"class":333},"].sum())       ",[293,1260,1261],{"class":562},"# 650 — the number the colleague saw\n",[10,1263,1264,1265,1268,1269,1272,1273,1276,1277,1280,1281,1284],{},"The column loop has one subtlety worth the comment. When columns are hidden as a ",[648,1266,1267],{},"group",", openpyxl stores\na single dimension entry keyed by the first letter with ",[14,1270,1271],{},"min"," and ",[14,1274,1275],{},"max"," spanning the whole range, so\nchecking only ",[14,1278,1279],{},"column_dimensions[\"C\"]"," would wrongly report column C as visible. Expanding ",[14,1282,1283],{},"min..max","\nhandles both single and grouped columns.",[279,1286,1288],{"id":1287},"step-3-or-filter-a-pandas-read-by-row-number","Step 3 — Or filter a pandas read by row number",[10,1290,1291,1292,1295,1296,1299,1300,1303],{},"If the rest of the pipeline already uses ",[14,1293,1294],{},"read_excel"," — for its dtype handling or its engine choice — keep\nit and drop hidden rows afterwards. The mapping from DataFrame position to Excel row is\n",[14,1297,1298],{},"position + header_row + 1",", which holds as long as you did not pass ",[14,1301,1302],{},"skiprows",".",[284,1305,1307],{"className":320,"code":1306,"language":322,"meta":289,"style":289},"df = pd.read_excel(\"orders.xlsx\", sheet_name=\"Orders\")\nexcel_row = pd.RangeIndex(len(df)) + 2          # header in row 1, data from row 2\nvisible_pd = df[~excel_row.isin(hidden_rows)].reset_index(drop=True)\nprint(visible_pd)\n",[14,1308,1309,1332,1356,1382],{"__ignoreMap":289},[293,1310,1311,1314,1316,1319,1321,1323,1326,1328,1330],{"class":295,"line":296},[293,1312,1313],{"class":333},"df ",[293,1315,356],{"class":329},[293,1317,1318],{"class":333}," pd.read_excel(",[293,1320,636],{"class":303},[293,1322,393],{"class":333},[293,1324,1325],{"class":1197},"sheet_name",[293,1327,356],{"class":329},[293,1329,711],{"class":303},[293,1331,639],{"class":333},[293,1333,1334,1337,1339,1342,1345,1348,1350,1353],{"class":295,"line":343},[293,1335,1336],{"class":333},"excel_row ",[293,1338,356],{"class":329},[293,1340,1341],{"class":333}," pd.RangeIndex(",[293,1343,1344],{"class":427},"len",[293,1346,1347],{"class":333},"(df)) ",[293,1349,1030],{"class":329},[293,1351,1352],{"class":427}," 2",[293,1354,1355],{"class":562},"          # header in row 1, data from row 2\n",[293,1357,1358,1361,1363,1366,1369,1372,1375,1377,1380],{"class":295,"line":350},[293,1359,1360],{"class":333},"visible_pd ",[293,1362,356],{"class":329},[293,1364,1365],{"class":333}," df[",[293,1367,1368],{"class":329},"~",[293,1370,1371],{"class":333},"excel_row.isin(hidden_rows)].reset_index(",[293,1373,1374],{"class":1197},"drop",[293,1376,356],{"class":329},[293,1378,1379],{"class":427},"True",[293,1381,639],{"class":333},[293,1383,1384,1386],{"class":295,"line":362},[293,1385,749],{"class":427},[293,1387,1388],{"class":333},"(visible_pd)\n",[10,1390,1391],{},"This is shorter but more fragile. Any option that changes which rows pandas reads shifts the mapping,\nso prefer Step 2 whenever the sheet has a title block or skipped rows above the data.",[25,1393,34,1398,34,1401,34,1404,34,1407,34,1412,34,1417,34,1421,34,1426,34,1431,34,1434,34,1439,34,1442,34,1446,34,1449,34,1451,34,1456,34,1459,34,1463,34,1466,34,1471,34,1476,34,1479],{"viewBox":1394,"role":28,"ariaLabelledBy":1395,"xmlns":32,"style":812},"0 0 760 136",[1396,1397],"hid-map-t","hid-map-d",[36,1399,1400],{"id":1396},"Mapping a pandas row position back to an Excel row",[40,1402,1403],{"id":1397},"DataFrame position zero corresponds to Excel row two when the header is in row one; add the header row plus one, then test membership in the set of hidden rows.",[44,1405],{"x":46,"y":46,"width":821,"height":1406,"fill":49},"136",[44,1408],{"x":59,"y":59,"width":1409,"height":1410,"rx":1411,"fill":63,"stroke":64,"style":828},"150","66","12",[51,1413,1416],{"x":1414,"y":60,"style":1415},"95","font-size:14px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","df position",[51,1418,1420],{"x":1414,"y":1419,"style":109},"72","0, 1, 2, …",[295,1422],{"x1":1423,"y1":1424,"x2":1425,"y2":1424,"stroke":874,"style":828},"175","53","204",[1427,1428],"polygon",{"points":1429,"fill":1430},"207,53 198,48 198,58","#5b5cf0",[44,1432],{"x":1433,"y":59,"width":1409,"height":1410,"rx":1411,"fill":873,"stroke":874,"style":828},"210",[51,1435,1438],{"x":1436,"y":60,"style":1437},"285","font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","+ header_row + 1",[51,1440,1441],{"x":1436,"y":1419,"style":136},"Excel row 2, 3, 4",[295,1443],{"x1":1444,"y1":1424,"x2":1445,"y2":1424,"stroke":874,"style":828},"365","394",[1427,1447],{"points":1448,"fill":1430},"397,53 388,48 388,58",[44,1450],{"x":242,"y":59,"width":1409,"height":1410,"rx":1411,"fill":159,"stroke":160,"style":828},[51,1452,1455],{"x":1453,"y":60,"style":1454},"475","font-size:14px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","isin(hidden)",[51,1457,1458],{"x":1453,"y":1419,"style":136},"row 3, 4, 6 dropped",[295,1460],{"x1":1461,"y1":1424,"x2":1462,"y2":1424,"stroke":874,"style":828},"555","584",[1427,1464],{"points":1465,"fill":1430},"587,53 578,48 578,58",[44,1467],{"x":1468,"y":59,"width":1409,"height":1410,"rx":1411,"fill":1469,"stroke":1470,"style":828},"590","#d9f4f1","var(--teal,#0f9488)",[51,1472,1475],{"x":1473,"y":60,"style":1474},"665","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","visible frame",[51,1477,1478],{"x":1473,"y":1419,"style":136},"what the user saw",[51,1480,1481],{"x":855,"y":174,"style":276},"skiprows breaks this arithmetic; build from openpyxl instead",[279,1483,1485],{"id":1484},"step-4-skip-hidden-and-very-hidden-sheets","Step 4 — Skip hidden and very hidden sheets",[10,1487,1488,1489,1492,1493,393,1496,1499,1500,1503],{},"Sheets can be hidden too, and ",[14,1490,1491],{},"pd.read_excel(sheet_name=None)"," returns them alongside the visible ones.\nopenpyxl exposes the state on each worksheet: ",[14,1494,1495],{},"\"visible\"",[14,1497,1498],{},"\"hidden\""," (unhide-able from Excel's menu) or\n",[14,1501,1502],{},"\"veryHidden\""," (only reachable through VBA, and usually holding lookup tables or configuration).",[284,1505,1507],{"className":320,"code":1506,"language":322,"meta":289,"style":289},"visible_sheets = [s.title for s in wb.worksheets if s.sheet_state == \"visible\"]\nprint(visible_sheets)                 # ['Orders']\n\nframes = pd.read_excel(\"orders.xlsx\", sheet_name=visible_sheets)\nprint(list(frames))\n",[14,1508,1509,1542,1552,1556,1576],{"__ignoreMap":289},[293,1510,1511,1514,1516,1519,1521,1524,1526,1529,1531,1534,1537,1540],{"class":295,"line":296},[293,1512,1513],{"class":333},"visible_sheets ",[293,1515,356],{"class":329},[293,1517,1518],{"class":333}," [s.title ",[293,1520,415],{"class":329},[293,1522,1523],{"class":333}," s ",[293,1525,421],{"class":329},[293,1527,1528],{"class":333}," wb.worksheets ",[293,1530,741],{"class":329},[293,1532,1533],{"class":333}," s.sheet_state ",[293,1535,1536],{"class":329},"==",[293,1538,1539],{"class":303}," \"visible\"",[293,1541,714],{"class":333},[293,1543,1544,1546,1549],{"class":295,"line":343},[293,1545,749],{"class":427},[293,1547,1548],{"class":333},"(visible_sheets)                 ",[293,1550,1551],{"class":562},"# ['Orders']\n",[293,1553,1554],{"class":295,"line":350},[293,1555,347],{"emptyLinePlaceholder":346},[293,1557,1558,1561,1563,1565,1567,1569,1571,1573],{"class":295,"line":362},[293,1559,1560],{"class":333},"frames ",[293,1562,356],{"class":329},[293,1564,1318],{"class":333},[293,1566,636],{"class":303},[293,1568,393],{"class":333},[293,1570,1325],{"class":1197},[293,1572,356],{"class":329},[293,1574,1575],{"class":333},"visible_sheets)\n",[293,1577,1578,1580,1582,1585],{"class":295,"line":373},[293,1579,749],{"class":427},[293,1581,752],{"class":333},[293,1583,1584],{"class":427},"list",[293,1586,1587],{"class":333},"(frames))\n",[10,1589,1590],{},"A very hidden sheet is a strong signal that the workbook author did not intend it to be data. It is\nstill worth listing in your job's log, because it is also exactly where a lookup table that your numbers\nsilently depend on tends to live.",[279,1592,1594],{"id":1593},"step-5-decide-which-version-you-actually-want","Step 5 — Decide which version you actually want",[10,1596,1597],{},"Visible-only is not automatically right. A filter is a view, and a person sending \"the attached\" may have\nleft a filter on by accident. Rows hidden by hand are sometimes corrections and sometimes just clutter.\nThe honest approach is to make the choice explicit and record it.",[284,1599,1601],{"className":320,"code":1600,"language":322,"meta":289,"style":289},"def load_orders(path: str, visible_only: bool) -> pd.DataFrame:\n    book = load_workbook(path)\n    sheet = book[\"Orders\"]\n    frame = read_visible(sheet) if visible_only else pd.read_excel(path, sheet_name=\"Orders\")\n    hidden = sum(1 for d in sheet.row_dimensions.values() if d.hidden)\n    print(f\"{path}: {len(frame)} rows read, {hidden} hidden rows \"\n          f\"{'skipped' if visible_only else 'included'}\")\n    return frame\n\nload_orders(\"orders.xlsx\", visible_only=True)\nload_orders(\"orders.xlsx\", visible_only=False)\n",[14,1602,1603,1623,1633,1647,1676,1706,1753,1781,1788,1792,1810],{"__ignoreMap":289},[293,1604,1605,1607,1610,1613,1616,1619,1621],{"class":295,"line":296},[293,1606,929],{"class":329},[293,1608,1609],{"class":932}," load_orders",[293,1611,1612],{"class":333},"(path: ",[293,1614,1615],{"class":427},"str",[293,1617,1618],{"class":333},", visible_only: ",[293,1620,951],{"class":427},[293,1622,958],{"class":333},[293,1624,1625,1628,1630],{"class":295,"line":343},[293,1626,1627],{"class":333},"    book ",[293,1629,356],{"class":329},[293,1631,1632],{"class":333}," load_workbook(path)\n",[293,1634,1635,1638,1640,1643,1645],{"class":295,"line":350},[293,1636,1637],{"class":333},"    sheet ",[293,1639,356],{"class":329},[293,1641,1642],{"class":333}," book[",[293,1644,711],{"class":303},[293,1646,714],{"class":333},[293,1648,1649,1652,1654,1657,1659,1662,1665,1668,1670,1672,1674],{"class":295,"line":362},[293,1650,1651],{"class":333},"    frame ",[293,1653,356],{"class":329},[293,1655,1656],{"class":333}," read_visible(sheet) ",[293,1658,741],{"class":329},[293,1660,1661],{"class":333}," visible_only ",[293,1663,1664],{"class":329},"else",[293,1666,1667],{"class":333}," pd.read_excel(path, ",[293,1669,1325],{"class":1197},[293,1671,356],{"class":329},[293,1673,711],{"class":303},[293,1675,639],{"class":333},[293,1677,1678,1681,1683,1686,1688,1690,1693,1696,1698,1701,1703],{"class":295,"line":373},[293,1679,1680],{"class":333},"    hidden ",[293,1682,356],{"class":329},[293,1684,1685],{"class":427}," sum",[293,1687,752],{"class":333},[293,1689,104],{"class":427},[293,1691,1692],{"class":329}," for",[293,1694,1695],{"class":333}," d ",[293,1697,421],{"class":329},[293,1699,1700],{"class":333}," sheet.row_dimensions.values() ",[293,1702,741],{"class":329},[293,1704,1705],{"class":333}," d.hidden)\n",[293,1707,1708,1710,1712,1715,1718,1722,1725,1728,1731,1733,1735,1738,1740,1743,1745,1748,1750],{"class":295,"line":384},[293,1709,792],{"class":427},[293,1711,752],{"class":333},[293,1713,1714],{"class":329},"f",[293,1716,1717],{"class":303},"\"",[293,1719,1721],{"class":1720},"sSjpA","{",[293,1723,1724],{"class":333},"path",[293,1726,1727],{"class":1720},"}",[293,1729,1730],{"class":303},": ",[293,1732,1721],{"class":1720},[293,1734,1344],{"class":427},[293,1736,1737],{"class":333},"(frame)",[293,1739,1727],{"class":1720},[293,1741,1742],{"class":303}," rows read, ",[293,1744,1721],{"class":1720},[293,1746,1747],{"class":333},"hidden",[293,1749,1727],{"class":1720},[293,1751,1752],{"class":303}," hidden rows \"\n",[293,1754,1755,1758,1760,1762,1765,1768,1770,1772,1775,1777,1779],{"class":295,"line":412},[293,1756,1757],{"class":329},"          f",[293,1759,1717],{"class":303},[293,1761,1721],{"class":1720},[293,1763,1764],{"class":303},"'skipped'",[293,1766,1767],{"class":329}," if",[293,1769,1661],{"class":333},[293,1771,1664],{"class":329},[293,1773,1774],{"class":303}," 'included'",[293,1776,1727],{"class":1720},[293,1778,1717],{"class":303},[293,1780,639],{"class":333},[293,1782,1783,1785],{"class":295,"line":468},[293,1784,1191],{"class":329},[293,1786,1787],{"class":333}," frame\n",[293,1789,1790],{"class":295,"line":500},[293,1791,347],{"emptyLinePlaceholder":346},[293,1793,1794,1797,1799,1801,1804,1806,1808],{"class":295,"line":511},[293,1795,1796],{"class":333},"load_orders(",[293,1798,636],{"class":303},[293,1800,393],{"class":333},[293,1802,1803],{"class":1197},"visible_only",[293,1805,356],{"class":329},[293,1807,1379],{"class":427},[293,1809,639],{"class":333},[293,1811,1812,1814,1816,1818,1820,1822,1825],{"class":295,"line":516},[293,1813,1796],{"class":333},[293,1815,636],{"class":303},[293,1817,393],{"class":333},[293,1819,1803],{"class":1197},[293,1821,356],{"class":329},[293,1823,1824],{"class":427},"False",[293,1826,639],{"class":333},[10,1828,1829,1830,1833,1834,1837],{},"A good rule of thumb: treat visible-only as correct for files that are ",[648,1831,1832],{},"reports"," a person prepared, and\nall rows as correct for files that are ",[648,1835,1836],{},"extracts"," a system produced. When in doubt, log both totals.",[279,1839,1841],{"id":1840},"common-pitfalls","Common pitfalls",[1843,1844,1845,1861],"table",{},[1846,1847,1848],"thead",{},[1849,1850,1851,1855,1858],"tr",{},[1852,1853,1854],"th",{},"Symptom",[1852,1856,1857],{},"Cause",[1852,1859,1860],{},"Fix",[1862,1863,1864,1879,1893,1912,1925,1942],"tbody",{},[1849,1865,1866,1870,1873],{},[1867,1868,1869],"td",{},"No rows reported hidden, but Excel shows a filter",[1867,1871,1872],{},"File saved by a tool that did not apply the filter",[1867,1874,1875,1876],{},"Re-apply the criteria in pandas from ",[14,1877,1878],{},"auto_filter.filterColumn",[1849,1880,1881,1884,1887],{},[1867,1882,1883],{},"Hidden column still read",[1867,1885,1886],{},"Column hidden as part of a group",[1867,1888,1889,1890],{},"Expand ",[14,1891,1892],{},"dim.min..dim.max",[1849,1894,1895,1903,1909],{},[1867,1896,1897,1900,1901],{},[14,1898,1899],{},"AttributeError"," on ",[14,1902,645],{},[1867,1904,1905,1906],{},"Loaded with ",[14,1907,1908],{},"read_only=True",[1867,1910,1911],{},"Load normally; read-only worksheets have no row or column dimensions",[1849,1913,1914,1917,1922],{},[1867,1915,1916],{},"Row numbers misaligned",[1867,1918,1919,1921],{},[14,1920,1302],{}," or a title block shifted the pandas read",[1867,1923,1924],{},"Build the frame from openpyxl",[1849,1926,1927,1930,1936],{},[1867,1928,1929],{},"Hidden sheet processed",[1867,1931,1932,1935],{},[14,1933,1934],{},"sheet_name=None"," reads every sheet",[1867,1937,1938,1939],{},"Filter on ",[14,1940,1941],{},"sheet_state == \"visible\"",[1849,1943,1944,1947,1950],{},[1867,1945,1946],{},"Rows with zero height still read",[1867,1948,1949],{},"Height set to 0 rather than hidden",[1867,1951,1952,1953,1956],{},"Treat ",[14,1954,1955],{},"dim.height == 0"," as hidden too",[10,1958,1959],{},"The last row of that table is more common than it should be. Dragging a row border until it disappears\ndoes not hide the row; it sets its height to zero. Excel shows nothing, and the hidden flag stays false.",[279,1961,1963],{"id":1962},"performance-and-scale-notes","Performance and scale notes",[10,1965,1966,1967,1970,1971,1974,1975,1979],{},"Reading row dimensions requires a full ",[14,1968,1969],{},"load_workbook",", and building the frame cell by cell is slower\nthan a pandas read. For a few thousand rows neither matters. For a large sheet, collect ",[14,1972,1973],{},"hidden_rows"," with\nopenpyxl once and read the values with a fast engine as in Step 3, or use\n",[19,1976,1978],{"href":1977},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fspeed-up-openpyxl-with-read-only-mode\u002F","Speed Up openpyxl with read_only Mode","\nfor the values and a separate normal load for the flags.",[25,1981,34,1987,34,1990,34,1993,34,1997,34,2001,34,2005,34,2011,34,2015,34,2018,34,2023,34,2026,34,2029,34,2034,34,2037,34,2040,34,2045,34,2049],{"viewBox":1982,"role":28,"ariaLabelledBy":1983,"xmlns":32,"style":1986},"0 0 720 346",[1984,1985],"hid-split-t","hid-split-d","width:100%;max-width:720px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[36,1988,1989],{"id":1984},"Splitting the work between two reads on a large sheet",[40,1991,1992],{"id":1985},"A normal openpyxl load collects only the set of hidden row numbers, a fast calamine read loads the values, and pandas drops the hidden rows by position before the rest of the pipeline runs.",[44,1994],{"x":46,"y":46,"width":1995,"height":1996,"fill":49},"720","346",[51,1998,2000],{"x":1999,"y":54,"style":55},"360","flags from one read, values from another",[44,2002],{"x":59,"y":60,"width":2003,"height":2004,"rx":1411,"fill":873,"stroke":874,"style":828},"680","52",[51,2006,2010],{"x":2007,"y":2008,"style":2009},"40","81","font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:start","openpyxl load",[51,2012,2014],{"x":147,"y":2008,"style":2013},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:start","collect hidden_rows and hidden column ranges once",[44,2016],{"x":59,"y":2017,"width":2003,"height":2004,"rx":1411,"fill":1469,"stroke":1470,"style":828},"114",[51,2019,2022],{"x":2007,"y":2020,"style":2021},"145","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:start","calamine read",[51,2024,2025],{"x":147,"y":2020,"style":2013},"read_excel(engine=\"calamine\") for the values",[44,2027],{"x":59,"y":2028,"width":2003,"height":2004,"rx":1411,"fill":197,"stroke":198,"style":828},"178",[51,2030,2033],{"x":2007,"y":2031,"style":2032},"209","font-size:14px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:start","position filter",[51,2035,2036],{"x":147,"y":2031,"style":2013},"drop rows whose Excel row number is hidden",[44,2038],{"x":59,"y":2039,"width":2003,"height":2004,"rx":1411,"fill":63,"stroke":64,"style":828},"242",[51,2041,2044],{"x":2007,"y":2042,"style":2043},"273","font-size:14px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:start","pipeline",[51,2046,2048],{"x":147,"y":2042,"style":2047},"font-size:12px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:start","the rest of the job sees only visible rows",[51,2050,2052],{"x":1999,"y":2051,"style":276},"326","the flag scan is the slow part; do it once per file",[10,2054,2055,2056,2059],{},"The two-read approach does not make the flag scan cheaper — a normal openpyxl load still builds every cell\nand style object — but it keeps that cost out of every later step, which can then use the fast engine and\nvectorised pandas operations instead of Python loops over cells. If you process the\nsame workbook repeatedly — a daily extract that only changes its values — cache the set of hidden rows\nalongside the file's modification time and skip the flag scan whenever the file has not changed. And when the\nsheet is truly large, remember that filtered views are usually small: filtering the pandas frame by position\nis a vectorised ",[14,2057,2058],{},"isin",", so the cost after both reads is negligible even at hundreds of thousands of rows.",[279,2061,2063],{"id":2062},"conclusion","Conclusion",[10,2065,2066,2067,393,2070,2073,2074,2077,2078,2081],{},"pandas reads stored cells, not the view a person saw. openpyxl exposes that view through\n",[14,2068,2069],{},"row_dimensions[n].hidden",[14,2071,2072],{},"column_dimensions"," (with grouped ranges expanded), ",[14,2075,2076],{},"auto_filter"," and\n",[14,2079,2080],{},"sheet_state",". Build the frame from visible rows when the file is a prepared report, keep every row when it\nis a system extract, and log which choice was made so that a mismatched total is explained rather than\nargued about.",[279,2083,2085],{"id":2084},"frequently-asked-questions","Frequently asked questions",[10,2087,2088,2092],{},[2089,2090,2091],"strong",{},"Does pandas have an option to skip hidden rows?","\nNo. read_excel has no visibility option for any engine. You need openpyxl to read the hidden flags and then drop those rows yourself.",[10,2094,2095,2098],{},[2089,2096,2097],{},"Will openpyxl apply the autofilter criteria for me?","\nNo. openpyxl reads and writes the filter definition but does not evaluate it. The rows are hidden only if the application that saved the file applied the filter, which Excel always does.",[10,2100,2101,2104],{},[2089,2102,2103],{},"How do I read hidden rows from a very hidden sheet?","\nThe same way as any sheet: index the loaded workbook by sheet name, which works regardless of sheet_state, and pandas can read it by name too. Visibility never blocks access from code.",[10,2106,2107,2110],{},[2089,2108,2109],{},"Can I unhide everything and save a clean copy?","\nYes. Set hidden to False on every row and column dimension, set sheet_state to \"visible\" on every worksheet, clear ws.auto_filter.ref, and save to a new file.",[279,2112,2114],{"id":2113},"related","Related",[2116,2117,2118,2125,2132,2139,2146],"ul",{},[2119,2120,2121,2122,2124],"li",{},"Up one level: ",[19,2123,22],{"href":21}," — the rest of the techniques for human-shaped workbooks.",[2119,2126,2127,2131],{},[19,2128,2130],{"href":2129},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fread-cell-colours-and-font-styles-as-data-with-openpyxl\u002F","Read Cell Colours and Font Styles as Data with openpyxl"," — more meaning stored outside the values.",[2119,2133,2134,2138],{},[19,2135,2137],{"href":2136},"\u002Fformatting-and-charting-excel-reports-with-python\u002Fprotecting-and-sharing-excel-workbooks\u002Fhide-sheets-rows-and-columns-with-openpyxl\u002F","Hide Sheets, Rows and Columns with openpyxl"," — the write side of the same flags.",[2119,2140,2141,2145],{},[19,2142,2144],{"href":2143},"\u002Fformatting-and-charting-excel-reports-with-python\u002Fcreating-excel-tables-and-autofilters-with-python\u002Fadd-an-autofilter-to-an-excel-sheet-with-openpyxl\u002F","Add an Autofilter to an Excel Sheet with openpyxl"," — how filter definitions are stored.",[2119,2147,2148,2152],{},[19,2149,2151],{"href":2150},"\u002Fformatting-and-charting-excel-reports-with-python\u002Fcreating-excel-tables-and-autofilters-with-python\u002Fgroup-and-outline-rows-in-excel-with-openpyxl\u002F","Group and Outline Rows in Excel with openpyxl"," — collapsed groups and outline levels.",[2154,2155,2156],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}",{"title":289,"searchDepth":343,"depth":343,"links":2158},[2159,2160,2161,2162,2163,2164,2165,2166,2167,2168,2169],{"id":281,"depth":343,"text":282},{"id":654,"depth":343,"text":655},{"id":898,"depth":343,"text":899},{"id":1287,"depth":343,"text":1288},{"id":1484,"depth":343,"text":1485},{"id":1593,"depth":343,"text":1594},{"id":1840,"depth":343,"text":1841},{"id":1962,"depth":343,"text":1963},{"id":2062,"depth":343,"text":2063},{"id":2084,"depth":343,"text":2085},{"id":2113,"depth":343,"text":2114},"2026-09-17","pandas reads every row, including the ones hidden by a filter or collapsed in a group. Use openpyxl to find hidden rows, columns and sheets, and read only what the user can see.","md",[2174,2176,2178,2180],{"q":2091,"a":2175},"No. read_excel has no visibility option for any engine. You need openpyxl to read the hidden flags and then drop those rows yourself.",{"q":2097,"a":2177},"No. openpyxl reads and writes the filter definition but does not evaluate it. The rows are hidden only if the application that saved the file applied the filter, which Excel always does.",{"q":2103,"a":2179},"The same way as any sheet: index the loaded workbook by sheet name, which works regardless of sheet_state, and pandas can read it by name too. Visibility never blocks access from code.",{"q":2109,"a":2181},"Yes. Set hidden to False on every row and column dimension, set sheet_state to \"visible\" on every worksheet, clear ws.auto_filter.ref, and save to a new file.","skip hidden and filtered rows when reading excel",null,{"breadcrumb":2185},[2186,2189,2192],{"name":2187,"item":2188},"Home","\u002F",{"name":2190,"item":2191},"Advanced Data Transformation and Cleaning","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":22,"item":21},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fskip-hidden-and-filtered-rows-when-reading-excel",{"title":2195,"description":2196},"Skip Hidden and Filtered Rows When Reading Excel in Python","Read only visible Excel rows in Python: detect hidden and autofiltered rows with openpyxl row_dimensions, expand grouped columns, skip hidden sheets, and build the DataFrame.","skip-hidden-and-filtered-rows-when-reading-excel","advanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fskip-hidden-and-filtered-rows-when-reading-excel\u002Findex",[2200,2201,2202],"messy-sheets","reading-excel","openpyxl","how-to","ygXR8FsuhQLqxOfnXuAA4O4lMuhw9IpnjGX2mhach_Y",[2206,2210],{"title":2207,"path":2208,"stem":2209,"children":-1},"Remove Subtotal and Total Rows from an Excel Export","\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fremove-subtotal-and-total-rows-from-an-excel-export","advanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fremove-subtotal-and-total-rows-from-an-excel-export\u002Findex",{"title":2211,"path":2212,"stem":2213,"children":-1},"Stop Reading at Notes and Footers Below the Data","\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fstop-reading-at-notes-and-footers-below-the-data","advanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fstop-reading-at-notes-and-footers-below-the-data\u002Findex",[2215,2217],{"path":2216,"title":22},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets",{"path":2218,"title":2219},"\u002Fadvanced-data-transformation-and-cleaning","Advanced Data Transformation and Cleaning for Excel Automation",[],[2222,2225,2228,2231,2234,2237],{"path":2223,"title":2224,"kind":2203},"\u002Fformatting-and-charting-excel-reports-with-python\u002Fstyling-excel-cells-with-openpyxl\u002Ffreeze-header-row-in-excel-with-openpyxl","Freeze the Header Row in Excel with openpyxl",{"path":2226,"title":2227,"kind":2203},"\u002Fgetting-started-with-python-excel-automation\u002Freading-excel-files-with-pandas\u002Fhandle-merged-cells-when-reading-excel-with-pandas","Handle Merged Cells When Reading Excel with pandas",{"path":2229,"title":2230,"kind":2203},"\u002Fgetting-started-with-python-excel-automation\u002Fusing-openpyxl-for-excel-file-manipulation\u002Funmerge-cells-and-fill-values-with-openpyxl","Unmerge Cells and Fill the Values with openpyxl",{"path":2232,"title":2233,"kind":2203},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fcombine-sheets-with-different-layouts-into-one-table","Combine Sheets with Different Layouts into One Table",{"path":2235,"title":2236,"kind":2203},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fextract-hyperlink-urls-from-excel-cells","Extract Hyperlink URLs from Excel Cells with openpyxl",{"path":2238,"title":2239,"kind":2203},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fextract-report-header-fields-from-an-excel-sheet","Extract Report Header Fields from an Excel Sheet with Python",1790972092946]