[{"data":1,"prerenderedAt":2087},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Freplace-placeholder-text-with-nan-in-pandas":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Freplace-placeholder-text-with-nan-in-pandas":2054,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Freplace-placeholder-text-with-nan-in-pandas":2063,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Freplace-placeholder-text-with-nan-in-pandas":2068,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Freplace-placeholder-text-with-nan-in-pandas":2069},{"id":4,"title":5,"body":6,"dateModified":2018,"datePublished":2018,"description":2019,"extension":2020,"faq":2021,"keyword":2030,"kind":2031,"meta":2032,"navigation":311,"path":2042,"seo":2043,"slug":2046,"stem":2047,"topics":2048,"type":2052,"__hash__":2053},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Freplace-placeholder-text-with-nan-in-pandas\u002Findex.md","Replace Placeholder Text with NaN in pandas",{"type":7,"value":8,"toc":2004},"minimark",[9,31,250,255,283,548,552,595,605,744,748,762,928,940,944,955,1042,1050,1054,1072,1189,1202,1206,1209,1297,1311,1461,1465,1474,1653,1662,1666,1682,1856,1863,1867,1916,1920,1935,1939,1949,1955,1961,1967,1971,2000],[10,11,12,13,17,18,21,22,24,25,30],"p",{},"People who maintain spreadsheets rarely leave a cell empty on purpose. They type \"n\u002Fa\", a dash, \"TBC\", \"unknown\", ",[14,15,16],"code",{},"-999"," in a numeric column, or ",[14,19,20],{},"31\u002F12\u002F9999"," as an end date that has not happened yet. To pandas these are ordinary values: a score column containing \"-\" is read as text, so it cannot be summed, and ",[14,23,16],{}," drags every average down. This guide, part of ",[26,27,29],"a",{"href":28},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002F","Handling Missing Data in Excel Reports",", turns placeholders into real missing values at read time and afterwards, and shows how to avoid the opposite mistake of destroying legitimate values.",[32,33,41,42,41,46,41,50,41,58,41,67,41,71,41,78,41,82,41,86,41,90,41,94,41,98,41,102,41,106,41,111,41,113,41,117,41,119,41,122,41,124,41,127,41,129,41,132,41,135,41,139,41,144,41,148,41,152,41,155,41,157,41,160,41,162,41,165,41,168,41,172,41,174,41,178,41,180,41,183,41,185,41,188,41,190,41,193,41,196,41,200,41,202,41,205,41,207,41,210,41,212,41,214,41,217,41,220,41,224,41,226,41,229,41,231,41,234,41,236,41,239,41,241,41,244],"svg",{"viewBox":34,"role":35,"ariaLabelledBy":36,"xmlns":39,"style":40},"121 0 518 222","img",[37,38],"ph-sheet-t","ph-sheet-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:518px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[43,44,45],"title",{"id":37},"A sheet with placeholders instead of blanks",[47,48,49],"desc",{"id":38},"The country column holds the code NA for Namibia and the placeholder N\u002FA. The score column mixes numbers with -999, a dash and TBC. The closed column has a 9999-12-31 end date and an empty cell. The comment column contains the word None, n\u002Fa, the error value",[51,52],"rect",{"x":53,"y":54,"width":55,"height":56,"fill":57},"121","0","518","222","#ffffff",[51,59],{"x":60,"y":61,"width":62,"height":63,"rx":54,"fill":64,"stroke":65,"style":66},"207.8","16.0","32.0","24.0","#e7ebef","var(--line,#cdd5e6)","stroke-width:1px",[51,68],{"x":69,"y":61,"width":70,"height":63,"rx":54,"fill":64,"stroke":65,"style":66},"239.8","72.2",[72,73,77],"text",{"x":74,"y":75,"style":76},"275.9","33.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[51,79],{"x":80,"y":61,"width":81,"height":63,"rx":54,"fill":64,"stroke":65,"style":66},"312.0","58.8",[72,83,85],{"x":84,"y":75,"style":76},"341.4","B",[51,87],{"x":88,"y":61,"width":89,"height":63,"rx":54,"fill":64,"stroke":65,"style":66},"370.8","99.3",[72,91,93],{"x":92,"y":75,"style":76},"420.4","C",[51,95],{"x":96,"y":61,"width":97,"height":63,"rx":54,"fill":64,"stroke":65,"style":66},"470.1","82.1",[72,99,101],{"x":100,"y":75,"style":76},"511.1","D",[51,103],{"x":60,"y":104,"width":62,"height":105,"rx":54,"fill":64,"stroke":65,"style":66},"40.0","28.0",[72,107,110],{"x":108,"y":109,"style":76},"223.8","59.0","1",[51,112],{"x":69,"y":104,"width":70,"height":105,"rx":54,"fill":57,"stroke":65,"style":66},[72,114,116],{"x":74,"y":109,"style":115},"font-size:12px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","country",[51,118],{"x":80,"y":104,"width":81,"height":105,"rx":54,"fill":57,"stroke":65,"style":66},[72,120,121],{"x":84,"y":109,"style":115},"score",[51,123],{"x":88,"y":104,"width":89,"height":105,"rx":54,"fill":57,"stroke":65,"style":66},[72,125,126],{"x":92,"y":109,"style":115},"closed",[51,128],{"x":96,"y":104,"width":97,"height":105,"rx":54,"fill":57,"stroke":65,"style":66},[72,130,131],{"x":100,"y":109,"style":115},"comment",[51,133],{"x":60,"y":134,"width":62,"height":105,"rx":54,"fill":64,"stroke":65,"style":66},"68.0",[72,136,138],{"x":108,"y":137,"style":76},"87.0","2",[51,140],{"x":69,"y":134,"width":70,"height":105,"rx":54,"fill":141,"stroke":142,"style":143},"#d9f4f1","var(--teal,#0f9488)","stroke-width:1.5px",[72,145,147],{"x":74,"y":137,"style":146},"font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","NA",[51,149],{"x":80,"y":134,"width":81,"height":105,"rx":54,"fill":150,"stroke":151,"style":143},"#fdefd8","var(--gold,#b4740a)",[72,153,16],{"x":84,"y":137,"style":154},"font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle",[51,156],{"x":88,"y":134,"width":89,"height":105,"rx":54,"fill":150,"stroke":151,"style":143},[72,158,159],{"x":92,"y":137,"style":154},"9999-12-31",[51,161],{"x":96,"y":134,"width":97,"height":105,"rx":54,"fill":141,"stroke":142,"style":143},[72,163,164],{"x":100,"y":137,"style":146},"None",[51,166],{"x":60,"y":167,"width":62,"height":105,"rx":54,"fill":64,"stroke":65,"style":66},"96.0",[72,169,171],{"x":108,"y":170,"style":76},"115.0","3",[51,173],{"x":69,"y":167,"width":70,"height":105,"rx":54,"fill":57,"stroke":65,"style":66},[72,175,177],{"x":74,"y":170,"style":176},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","DE",[51,179],{"x":80,"y":167,"width":81,"height":105,"rx":54,"fill":57,"stroke":65,"style":66},[72,181,182],{"x":84,"y":170,"style":176},"12",[51,184],{"x":88,"y":167,"width":89,"height":105,"rx":54,"fill":57,"stroke":65,"style":66},[72,186,187],{"x":92,"y":170,"style":176},"2026-03-01",[51,189],{"x":96,"y":167,"width":97,"height":105,"rx":54,"fill":150,"stroke":151,"style":143},[72,191,192],{"x":100,"y":170,"style":154},"n\u002Fa",[51,194],{"x":60,"y":195,"width":62,"height":105,"rx":54,"fill":64,"stroke":65,"style":66},"124.0",[72,197,199],{"x":108,"y":198,"style":76},"143.0","4",[51,201],{"x":69,"y":195,"width":70,"height":105,"rx":54,"fill":150,"stroke":151,"style":143},[72,203,204],{"x":74,"y":198,"style":154},"N\u002FA",[51,206],{"x":80,"y":195,"width":81,"height":105,"rx":54,"fill":150,"stroke":151,"style":143},[72,208,209],{"x":84,"y":198,"style":154},"-",[51,211],{"x":88,"y":195,"width":89,"height":105,"rx":54,"fill":57,"stroke":65,"style":66},[51,213],{"x":96,"y":195,"width":97,"height":105,"rx":54,"fill":150,"stroke":151,"style":143},[72,215,216],{"x":100,"y":198,"style":154},"#N\u002FA",[51,218],{"x":60,"y":219,"width":62,"height":105,"rx":54,"fill":64,"stroke":65,"style":66},"152.0",[72,221,223],{"x":108,"y":222,"style":76},"171.0","5",[51,225],{"x":69,"y":219,"width":70,"height":105,"rx":54,"fill":57,"stroke":65,"style":66},[72,227,228],{"x":74,"y":222,"style":176},"FR",[51,230],{"x":80,"y":219,"width":81,"height":105,"rx":54,"fill":150,"stroke":151,"style":143},[72,232,233],{"x":84,"y":222,"style":154},"TBC",[51,235],{"x":88,"y":219,"width":89,"height":105,"rx":54,"fill":57,"stroke":65,"style":66},[72,237,238],{"x":92,"y":222,"style":176},"2026-04-01",[51,240],{"x":96,"y":219,"width":97,"height":105,"rx":54,"fill":150,"stroke":151,"style":143},[72,242,243],{"x":100,"y":222,"style":154},"␠",[72,245,249],{"x":246,"y":247,"style":248},"380.0","202.0","font-size:12.5px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:middle","gold cells are placeholders; teal cells look like placeholders but are real values",[251,252,254],"h2",{"id":253},"prerequisites","Prerequisites",[256,257,262],"pre",{"className":258,"code":259,"language":260,"meta":261,"style":261},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[14,263,264],{"__ignoreMap":261},[265,266,269,273,277,280],"span",{"class":267,"line":268},"line",1,[265,270,272],{"class":271},"sMTad","pip",[265,274,276],{"class":275},"srMev"," install",[265,278,279],{"class":275}," pandas",[265,281,282],{"class":275}," openpyxl\n",[256,284,288],{"className":285,"code":286,"language":287,"meta":261,"style":261},"language-python shiki shiki-themes github-light github-dark-high-contrast","import datetime as dt\n\nimport numpy as np\nimport pandas as pd\nfrom openpyxl import Workbook\n\nwb = Workbook()\nws = wb.active\nws.append([\"country\", \"score\", \"closed\", \"comment\"])\nws.append([\"NA\", -999, dt.datetime(9999, 12, 31), \"None\"])\nws.append([\"DE\", 12, dt.datetime(2026, 3, 1), \"n\u002Fa\"])\nws.append([\"N\u002FA\", \"-\", None, \"#N\u002FA\"])\nws.append([\"FR\", \"TBC\", dt.datetime(2026, 4, 1), \" \"])\nwb.save(\"survey.xlsx\")\n","python",[14,289,290,306,313,326,339,353,358,370,381,409,448,480,504,536],{"__ignoreMap":261},[265,291,292,296,300,303],{"class":267,"line":268},[265,293,295],{"class":294},"s-kum","import",[265,297,299],{"class":298},"skGVy"," datetime ",[265,301,302],{"class":294},"as",[265,304,305],{"class":298}," dt\n",[265,307,309],{"class":267,"line":308},2,[265,310,312],{"emptyLinePlaceholder":311},true,"\n",[265,314,316,318,321,323],{"class":267,"line":315},3,[265,317,295],{"class":294},[265,319,320],{"class":298}," numpy ",[265,322,302],{"class":294},[265,324,325],{"class":298}," np\n",[265,327,329,331,334,336],{"class":267,"line":328},4,[265,330,295],{"class":294},[265,332,333],{"class":298}," pandas ",[265,335,302],{"class":294},[265,337,338],{"class":298}," pd\n",[265,340,342,345,348,350],{"class":267,"line":341},5,[265,343,344],{"class":294},"from",[265,346,347],{"class":298}," openpyxl ",[265,349,295],{"class":294},[265,351,352],{"class":298}," Workbook\n",[265,354,356],{"class":267,"line":355},6,[265,357,312],{"emptyLinePlaceholder":311},[265,359,361,364,367],{"class":267,"line":360},7,[265,362,363],{"class":298},"wb ",[265,365,366],{"class":294},"=",[265,368,369],{"class":298}," Workbook()\n",[265,371,373,376,378],{"class":267,"line":372},8,[265,374,375],{"class":298},"ws ",[265,377,366],{"class":294},[265,379,380],{"class":298}," wb.active\n",[265,382,384,387,390,393,396,398,401,403,406],{"class":267,"line":383},9,[265,385,386],{"class":298},"ws.append([",[265,388,389],{"class":275},"\"country\"",[265,391,392],{"class":298},", ",[265,394,395],{"class":275},"\"score\"",[265,397,392],{"class":298},[265,399,400],{"class":275},"\"closed\"",[265,402,392],{"class":298},[265,404,405],{"class":275},"\"comment\"",[265,407,408],{"class":298},"])\n",[265,410,412,414,417,419,421,425,428,431,433,435,437,440,443,446],{"class":267,"line":411},10,[265,413,386],{"class":298},[265,415,416],{"class":275},"\"NA\"",[265,418,392],{"class":298},[265,420,209],{"class":294},[265,422,424],{"class":423},"sP0c6","999",[265,426,427],{"class":298},", dt.datetime(",[265,429,430],{"class":423},"9999",[265,432,392],{"class":298},[265,434,182],{"class":423},[265,436,392],{"class":298},[265,438,439],{"class":423},"31",[265,441,442],{"class":298},"), ",[265,444,445],{"class":275},"\"None\"",[265,447,408],{"class":298},[265,449,451,453,456,458,460,462,465,467,469,471,473,475,478],{"class":267,"line":450},11,[265,452,386],{"class":298},[265,454,455],{"class":275},"\"DE\"",[265,457,392],{"class":298},[265,459,182],{"class":423},[265,461,427],{"class":298},[265,463,464],{"class":423},"2026",[265,466,392],{"class":298},[265,468,171],{"class":423},[265,470,392],{"class":298},[265,472,110],{"class":423},[265,474,442],{"class":298},[265,476,477],{"class":275},"\"n\u002Fa\"",[265,479,408],{"class":298},[265,481,483,485,488,490,493,495,497,499,502],{"class":267,"line":482},12,[265,484,386],{"class":298},[265,486,487],{"class":275},"\"N\u002FA\"",[265,489,392],{"class":298},[265,491,492],{"class":275},"\"-\"",[265,494,392],{"class":298},[265,496,164],{"class":423},[265,498,392],{"class":298},[265,500,501],{"class":275},"\"#N\u002FA\"",[265,503,408],{"class":298},[265,505,507,509,512,514,517,519,521,523,525,527,529,531,534],{"class":267,"line":506},13,[265,508,386],{"class":298},[265,510,511],{"class":275},"\"FR\"",[265,513,392],{"class":298},[265,515,516],{"class":275},"\"TBC\"",[265,518,427],{"class":298},[265,520,464],{"class":423},[265,522,392],{"class":298},[265,524,199],{"class":423},[265,526,392],{"class":298},[265,528,110],{"class":423},[265,530,442],{"class":298},[265,532,533],{"class":275},"\" \"",[265,535,408],{"class":298},[265,537,539,542,545],{"class":267,"line":538},14,[265,540,541],{"class":298},"wb.save(",[265,543,544],{"class":275},"\"survey.xlsx\"",[265,546,547],{"class":298},")\n",[251,549,551],{"id":550},"step-1-see-what-pandas-does-by-default","Step 1 — See what pandas does by default",[256,553,555],{"className":285,"code":554,"language":287,"meta":261,"style":261},"df = pd.read_excel(\"survey.xlsx\")\nprint(df)\nprint(df[\"score\"].dtype)          # object\n",[14,556,557,571,579],{"__ignoreMap":261},[265,558,559,562,564,567,569],{"class":267,"line":268},[265,560,561],{"class":298},"df ",[265,563,366],{"class":294},[265,565,566],{"class":298}," pd.read_excel(",[265,568,544],{"class":275},[265,570,547],{"class":298},[265,572,573,576],{"class":267,"line":308},[265,574,575],{"class":423},"print",[265,577,578],{"class":298},"(df)\n",[265,580,581,583,586,588,591],{"class":267,"line":315},[265,582,575],{"class":423},[265,584,585],{"class":298},"(df[",[265,587,395],{"class":275},[265,589,590],{"class":298},"].dtype)          ",[265,592,594],{"class":593},"s-wDw","# object\n",[10,596,597,598,601,602,604],{},"The default read already converts some placeholders — and gets two cells wrong. It recognises \"N\u002FA\", \"n\u002Fa\", \"#N\u002FA\" and the empty cell, because they are on pandas' built-in list of missing-value strings. It also converts \"NA\" (the country code for Namibia) and \"None\" (a real answer in the comment column) to ",[14,599,600],{},"NaN",", because those are on the list too. Meanwhile \"-\", \"TBC\" and ",[14,603,16],{}," survive, so the score column is text and cannot be summed. The single space in the last comment survives as well.",[32,606,41,612,41,615,41,618,41,622,41,629,41,640,41,646,41,651,41,657,41,663,41,667,41,672,41,676,41,683,41,687,41,692,41,695,41,702,41,707,41,711,41,715,41,720,41,723,41,728,41,731,41,736,41,739],{"viewBox":607,"role":35,"ariaLabelledBy":608,"xmlns":39,"style":611},"4 0 752 233",[609,610],"ph-default-t","ph-default-d","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[43,613,614],{"id":609},"What the default read gets right and wrong",[47,616,617],{"id":610},"By default pandas correctly turns N\u002FA, n\u002Fa,",[51,619],{"x":199,"y":54,"width":620,"height":621,"fill":57},"752","233",[51,623],{"x":624,"y":61,"width":625,"height":626,"rx":627,"fill":141,"stroke":142,"style":628},"20.0","351.0","201.0","14","stroke-width:2px",[630,631,633],"g",{"transform":632},"translate(182.5,29.0) scale(1.083)",[634,635],"path",{"d":636,"fill":637,"stroke":638,"style":639},"M5 12.5l4.5 4.5L19 7","none","var(--teal-ink,#0b6157)","stroke-width:1.66px;stroke-linecap:round;stroke-linejoin:round",[72,641,645],{"x":642,"y":643,"style":644},"195.5","78.0","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Converted to NaN",[267,647],{"x1":648,"y1":649,"x2":650,"y2":649,"stroke":142,"style":66},"36.0","91.0","355.0",[72,652,656],{"x":653,"y":654,"style":655},"60.0","111.0","font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:start","N\u002FA, n\u002Fa, #N\u002FA",[630,658,660],{"transform":659},"translate(38.5,99.5) scale(0.625)",[634,661],{"d":636,"fill":637,"stroke":142,"style":662},"stroke-width:2.88px;stroke-linecap:round;stroke-linejoin:round",[72,664,666],{"x":653,"y":665,"style":655},"137.0","empty cells",[630,668,670],{"transform":669},"translate(38.5,125.5) scale(0.625)",[634,671],{"d":636,"fill":637,"stroke":142,"style":662},[72,673,675],{"x":653,"y":674,"style":655},"163.0","the country code NA",[630,677,679],{"transform":678},"translate(38.5,151.5) scale(0.625)",[634,680],{"d":681,"fill":637,"stroke":682,"style":662},"M6 6l12 12 M18 6L6 18","var(--accent,#f43f8f)",[72,684,686],{"x":653,"y":685,"style":655},"189.0","the answer None",[630,688,690],{"transform":689},"translate(38.5,177.5) scale(0.625)",[634,691],{"d":681,"fill":637,"stroke":682,"style":662},[51,693],{"x":694,"y":61,"width":625,"height":626,"rx":627,"fill":150,"stroke":151,"style":628},"389.0",[630,696,698],{"transform":697},"translate(551.5,29.0) scale(1.083)",[634,699],{"d":700,"fill":637,"stroke":701,"style":639},"M12 3.5l9.5 17h-19z M12 10v5 M12 17.8v.4","var(--gold-ink,#7a4e06)",[72,703,706],{"x":704,"y":643,"style":705},"564.5","font-size:14px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Left as values",[267,708],{"x1":709,"y1":649,"x2":710,"y2":649,"stroke":151,"style":66},"405.0","724.0",[72,712,714],{"x":713,"y":654,"style":655},"429.0","dash and TBC",[630,716,718],{"transform":717},"translate(407.5,99.5) scale(0.625)",[634,719],{"d":681,"fill":637,"stroke":682,"style":662},[72,721,722],{"x":713,"y":665,"style":655},"-999 in a numeric column",[630,724,726],{"transform":725},"translate(407.5,125.5) scale(0.625)",[634,727],{"d":681,"fill":637,"stroke":682,"style":662},[72,729,730],{"x":713,"y":674,"style":655},"31\u002F12\u002F9999 end dates",[630,732,734],{"transform":733},"translate(407.5,151.5) scale(0.625)",[634,735],{"d":681,"fill":637,"stroke":682,"style":662},[72,737,738],{"x":713,"y":685,"style":655},"a single space",[630,740,742],{"transform":741},"translate(407.5,177.5) scale(0.625)",[634,743],{"d":681,"fill":637,"stroke":682,"style":662},[251,745,747],{"id":746},"step-2-take-control-with-per-column-na_values","Step 2 — Take control with per-column na_values",[10,749,750,753,754,757,758,761],{},[14,751,752],{},"read_excel"," accepts ",[14,755,756],{},"na_values"," as a dictionary keyed by column name. Combined with ",[14,759,760],{},"keep_default_na=False",", only the values you list count as missing, so you decide column by column:",[256,763,765],{"className":285,"code":764,"language":287,"meta":261,"style":261},"NA_RULES = {\n    \"country\": [\"\", \"N\u002FA\"],\n    \"score\": [\"\", \"-\", \"TBC\", -999],\n    \"closed\": [\"\"],\n    \"comment\": [\"\", \"n\u002Fa\", \"#N\u002FA\"],\n}\ndf = pd.read_excel(\"survey.xlsx\", keep_default_na=False, na_values=NA_RULES)\nprint(df[\"country\"].tolist())     # ['NA', 'DE', nan, 'FR']\nprint(df[\"score\"].tolist())       # [nan, 12.0, nan, nan]\nprint(df[\"comment\"].tolist())     # ['None', nan, nan, ' ']\n",[14,766,767,778,796,821,832,851,856,887,901,915],{"__ignoreMap":261},[265,768,769,772,775],{"class":267,"line":268},[265,770,771],{"class":423},"NA_RULES",[265,773,774],{"class":294}," =",[265,776,777],{"class":298}," {\n",[265,779,780,783,786,789,791,793],{"class":267,"line":308},[265,781,782],{"class":275},"    \"country\"",[265,784,785],{"class":298},": [",[265,787,788],{"class":275},"\"\"",[265,790,392],{"class":298},[265,792,487],{"class":275},[265,794,795],{"class":298},"],\n",[265,797,798,801,803,805,807,809,811,813,815,817,819],{"class":267,"line":315},[265,799,800],{"class":275},"    \"score\"",[265,802,785],{"class":298},[265,804,788],{"class":275},[265,806,392],{"class":298},[265,808,492],{"class":275},[265,810,392],{"class":298},[265,812,516],{"class":275},[265,814,392],{"class":298},[265,816,209],{"class":294},[265,818,424],{"class":423},[265,820,795],{"class":298},[265,822,823,826,828,830],{"class":267,"line":328},[265,824,825],{"class":275},"    \"closed\"",[265,827,785],{"class":298},[265,829,788],{"class":275},[265,831,795],{"class":298},[265,833,834,837,839,841,843,845,847,849],{"class":267,"line":341},[265,835,836],{"class":275},"    \"comment\"",[265,838,785],{"class":298},[265,840,788],{"class":275},[265,842,392],{"class":298},[265,844,477],{"class":275},[265,846,392],{"class":298},[265,848,501],{"class":275},[265,850,795],{"class":298},[265,852,853],{"class":267,"line":355},[265,854,855],{"class":298},"}\n",[265,857,858,860,862,864,866,868,872,874,877,879,881,883,885],{"class":267,"line":360},[265,859,561],{"class":298},[265,861,366],{"class":294},[265,863,566],{"class":298},[265,865,544],{"class":275},[265,867,392],{"class":298},[265,869,871],{"class":870},"sa561","keep_default_na",[265,873,366],{"class":294},[265,875,876],{"class":423},"False",[265,878,392],{"class":298},[265,880,756],{"class":870},[265,882,366],{"class":294},[265,884,771],{"class":423},[265,886,547],{"class":298},[265,888,889,891,893,895,898],{"class":267,"line":372},[265,890,575],{"class":423},[265,892,585],{"class":298},[265,894,389],{"class":275},[265,896,897],{"class":298},"].tolist())     ",[265,899,900],{"class":593},"# ['NA', 'DE', nan, 'FR']\n",[265,902,903,905,907,909,912],{"class":267,"line":383},[265,904,575],{"class":423},[265,906,585],{"class":298},[265,908,395],{"class":275},[265,910,911],{"class":298},"].tolist())       ",[265,913,914],{"class":593},"# [nan, 12.0, nan, nan]\n",[265,916,917,919,921,923,925],{"class":267,"line":411},[265,918,575],{"class":423},[265,920,585],{"class":298},[265,922,405],{"class":275},[265,924,897],{"class":298},[265,926,927],{"class":593},"# ['None', nan, nan, ' ']\n",[10,929,930,931,933,934,936,937,939],{},"Namibia and \"None\" survive, and the score column is now numeric. Note the empty string in every list: once ",[14,932,760],{}," is set, blank cells are no longer treated as missing unless you say so. Leave ",[14,935,788],{}," out of the ",[14,938,126],{}," list and the empty cell arrives as an empty string, turning the whole date column into text — a quiet failure that only shows up later when a date calculation breaks.",[251,941,943],{"id":942},"step-3-handle-whitespace-only-cells","Step 3 — Handle whitespace-only cells",[10,945,946,948,949,392,951,954],{},[14,947,756],{}," compares exact strings, so ",[14,950,533],{},[14,952,953],{},"\"  \""," and a non-breaking space would each need their own entry. Convert every whitespace-only text cell after reading instead:",[256,956,958],{"className":285,"code":957,"language":287,"meta":261,"style":261},"text_cols = df.select_dtypes(include=[\"object\", \"string\"]).columns\ndf[text_cols] = df[text_cols].replace(r\"^\\s*$\", np.nan, regex=True)\nprint(df[\"comment\"].tolist())     # ['None', nan, nan, nan]\n",[14,959,960,989,1029],{"__ignoreMap":261},[265,961,962,965,967,970,973,975,978,981,983,986],{"class":267,"line":268},[265,963,964],{"class":298},"text_cols ",[265,966,366],{"class":294},[265,968,969],{"class":298}," df.select_dtypes(",[265,971,972],{"class":870},"include",[265,974,366],{"class":294},[265,976,977],{"class":298},"[",[265,979,980],{"class":275},"\"object\"",[265,982,392],{"class":298},[265,984,985],{"class":275},"\"string\"",[265,987,988],{"class":298},"]).columns\n",[265,990,991,994,996,999,1002,1005,1008,1011,1014,1016,1019,1022,1024,1027],{"class":267,"line":308},[265,992,993],{"class":298},"df[text_cols] ",[265,995,366],{"class":294},[265,997,998],{"class":298}," df[text_cols].replace(",[265,1000,1001],{"class":294},"r",[265,1003,1004],{"class":275},"\"",[265,1006,1007],{"class":423},"^\\s",[265,1009,1010],{"class":294},"*",[265,1012,1013],{"class":423},"$",[265,1015,1004],{"class":275},[265,1017,1018],{"class":298},", np.nan, ",[265,1020,1021],{"class":870},"regex",[265,1023,366],{"class":294},[265,1025,1026],{"class":423},"True",[265,1028,547],{"class":298},[265,1030,1031,1033,1035,1037,1039],{"class":267,"line":315},[265,1032,575],{"class":423},[265,1034,585],{"class":298},[265,1036,405],{"class":275},[265,1038,897],{"class":298},[265,1040,1041],{"class":593},"# ['None', nan, nan, nan]\n",[10,1043,1044,1045,1049],{},"The anchored pattern only matches cells that contain nothing but whitespace, so \"None\" and other real text are untouched. Wider whitespace problems are covered in ",[26,1046,1048],{"href":1047},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstrip-whitespace-and-normalise-text-columns-with-pandas\u002F","Strip Whitespace and Normalise Text Columns with pandas",".",[251,1051,1053],{"id":1052},"step-4-mask-sentinel-numbers-and-dates","Step 4 — Mask sentinel numbers and dates",[10,1055,1056,1057,1059,1060,1062,1063,1059,1065,1068,1069,1071],{},"Some placeholders are values of the right type: ",[14,1058,16],{}," or ",[14,1061,430],{}," in a numeric column, ",[14,1064,20],{},[14,1066,1067],{},"01\u002F01\u002F1900"," in a date column. When they arrive through a path where ",[14,1070,756],{}," does not apply — a frame from another function, or a sentinel that depends on the column — mask them explicitly:",[256,1073,1075],{"className":285,"code":1074,"language":287,"meta":261,"style":261},"scores = pd.Series([-999, 12, 9999, 40])\nprint(scores.mask(scores.isin([-999, 9999])).tolist())   # [nan, 12.0, nan, 40.0]\n\nclosed = pd.read_excel(\"survey.xlsx\", keep_default_na=False, na_values=NA_RULES)[\"closed\"]\nfar_future = closed.dt.year >= 9999\nprint(closed.mask(far_future).tolist())\n",[14,1076,1077,1106,1127,1131,1166,1182],{"__ignoreMap":261},[265,1078,1079,1082,1084,1087,1089,1091,1093,1095,1097,1099,1101,1104],{"class":267,"line":268},[265,1080,1081],{"class":298},"scores ",[265,1083,366],{"class":294},[265,1085,1086],{"class":298}," pd.Series([",[265,1088,209],{"class":294},[265,1090,424],{"class":423},[265,1092,392],{"class":298},[265,1094,182],{"class":423},[265,1096,392],{"class":298},[265,1098,430],{"class":423},[265,1100,392],{"class":298},[265,1102,1103],{"class":423},"40",[265,1105,408],{"class":298},[265,1107,1108,1110,1113,1115,1117,1119,1121,1124],{"class":267,"line":308},[265,1109,575],{"class":423},[265,1111,1112],{"class":298},"(scores.mask(scores.isin([",[265,1114,209],{"class":294},[265,1116,424],{"class":423},[265,1118,392],{"class":298},[265,1120,430],{"class":423},[265,1122,1123],{"class":298},"])).tolist())   ",[265,1125,1126],{"class":593},"# [nan, 12.0, nan, 40.0]\n",[265,1128,1129],{"class":267,"line":315},[265,1130,312],{"emptyLinePlaceholder":311},[265,1132,1133,1136,1138,1140,1142,1144,1146,1148,1150,1152,1154,1156,1158,1161,1163],{"class":267,"line":328},[265,1134,1135],{"class":298},"closed ",[265,1137,366],{"class":294},[265,1139,566],{"class":298},[265,1141,544],{"class":275},[265,1143,392],{"class":298},[265,1145,871],{"class":870},[265,1147,366],{"class":294},[265,1149,876],{"class":423},[265,1151,392],{"class":298},[265,1153,756],{"class":870},[265,1155,366],{"class":294},[265,1157,771],{"class":423},[265,1159,1160],{"class":298},")[",[265,1162,400],{"class":275},[265,1164,1165],{"class":298},"]\n",[265,1167,1168,1171,1173,1176,1179],{"class":267,"line":341},[265,1169,1170],{"class":298},"far_future ",[265,1172,366],{"class":294},[265,1174,1175],{"class":298}," closed.dt.year ",[265,1177,1178],{"class":294},">=",[265,1180,1181],{"class":423}," 9999\n",[265,1183,1184,1186],{"class":267,"line":355},[265,1185,575],{"class":423},[265,1187,1188],{"class":298},"(closed.mask(far_future).tolist())\n",[10,1190,1191,1192,1194,1195,1198,1199,1049],{},"Before masking a date sentinel, decide what it means. ",[14,1193,20],{}," usually means \"still open\", which is a different fact from \"unknown\". If reports need to tell the two apart, keep a boolean ",[14,1196,1197],{},"is_open"," column before replacing the date with ",[14,1200,1201],{},"NaT",[251,1203,1205],{"id":1204},"step-5-build-the-rules-from-the-data-not-from-memory","Step 5 — Build the rules from the data, not from memory",[10,1207,1208],{},"Placeholder habits vary by team and by column. Find the candidates by listing the non-numeric values in columns that should be numeric, and the most frequent short strings in text columns:",[256,1210,1212],{"className":285,"code":1211,"language":287,"meta":261,"style":261},"raw = pd.read_excel(\"survey.xlsx\", keep_default_na=False, dtype=str)\nfor col in [\"score\"]:\n    as_num = pd.to_numeric(raw[col], errors=\"coerce\")\n    print(col, raw.loc[as_num.isna(), col].value_counts().to_dict())\n# score {'-': 1, 'TBC': 1}\n",[14,1213,1214,1245,1264,1284,1292],{"__ignoreMap":261},[265,1215,1216,1219,1221,1223,1225,1227,1229,1231,1233,1235,1238,1240,1243],{"class":267,"line":268},[265,1217,1218],{"class":298},"raw ",[265,1220,366],{"class":294},[265,1222,566],{"class":298},[265,1224,544],{"class":275},[265,1226,392],{"class":298},[265,1228,871],{"class":870},[265,1230,366],{"class":294},[265,1232,876],{"class":423},[265,1234,392],{"class":298},[265,1236,1237],{"class":870},"dtype",[265,1239,366],{"class":294},[265,1241,1242],{"class":423},"str",[265,1244,547],{"class":298},[265,1246,1247,1250,1253,1256,1259,1261],{"class":267,"line":308},[265,1248,1249],{"class":294},"for",[265,1251,1252],{"class":298}," col ",[265,1254,1255],{"class":294},"in",[265,1257,1258],{"class":298}," [",[265,1260,395],{"class":275},[265,1262,1263],{"class":298},"]:\n",[265,1265,1266,1269,1271,1274,1277,1279,1282],{"class":267,"line":315},[265,1267,1268],{"class":298},"    as_num ",[265,1270,366],{"class":294},[265,1272,1273],{"class":298}," pd.to_numeric(raw[col], ",[265,1275,1276],{"class":870},"errors",[265,1278,366],{"class":294},[265,1280,1281],{"class":275},"\"coerce\"",[265,1283,547],{"class":298},[265,1285,1286,1289],{"class":267,"line":328},[265,1287,1288],{"class":423},"    print",[265,1290,1291],{"class":298},"(col, raw.loc[as_num.isna(), col].value_counts().to_dict())\n",[265,1293,1294],{"class":267,"line":341},[265,1295,1296],{"class":593},"# score {'-': 1, 'TBC': 1}\n",[10,1298,1299,1300,1303,1304,1306,1307,1049],{},"Reading with ",[14,1301,1302],{},"dtype=str"," and ",[14,1305,760],{}," shows every value exactly as typed, so nothing is converted before you have looked at it. Review the list with whoever owns the sheet; \"TBC\" might need to become a separate status rather than a blank. Run the same check on every sheet you receive, because new placeholders appear whenever a new person starts filling in the workbook. A full missing-value report is covered in ",[26,1308,1310],{"href":1309},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffind-and-report-missing-values-in-an-excel-file\u002F","Find and Report Missing Values in an Excel File",[32,1312,41,1317,41,1320,41,1323,41,1326,41,1332,41,1340,41,1346,41,1350,41,1355,41,1360,41,1363,41,1369,41,1374,41,1378,41,1382,41,1386,41,1389,41,1392,41,1398,41,1402,41,1405,41,1409,41,1412,41,1415,41,1421,41,1426,41,1429,41,1432,41,1436,41,1439,41,1442,41,1447,41,1452,41,1455,41,1458],{"viewBox":1313,"role":35,"ariaLabelledBy":1314,"xmlns":39,"style":611},"4 0 752 160",[1315,1316],"ph-flow-t","ph-flow-d",[43,1318,1319],{"id":1315},"From raw placeholders to real missing values",[47,1321,1322],{"id":1316},"Read every column as text without conversion, list candidate placeholders per column, agree the rules with the sheet owner, read with keep_default_na False and per-column na_values, then blank whitespace-only cells and mask sentinel numbers and dates.",[51,1324],{"x":199,"y":54,"width":620,"height":1325,"fill":57},"160",[51,1327],{"x":624,"y":61,"width":1328,"height":1329,"rx":182,"fill":1330,"stroke":1331,"style":628},"115.2","128.0","#f0f4ff","var(--brand,#5b5cf0)",[630,1333,1335],{"transform":1334},"translate(65.6,46.5) scale(1.000)",[634,1336],{"d":1337,"fill":637,"stroke":1338,"style":1339},"M4 4h16v16H4z M4 9h16 M4 14h16 M10 4v16","var(--brand-strong,#4338ca)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[72,1341,1345],{"x":1342,"y":1343,"style":1344},"77.6","93.0","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Read as text",[72,1347,1302],{"x":1342,"y":1348,"style":1349},"112.5","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle",[267,1351],{"x1":1352,"y1":1353,"x2":1354,"y2":1353,"stroke":1331,"style":628},"140.2","80.0","162.2",[1356,1357],"polygon",{"points":1358,"fill":1359},"166.2,80.0 157.2,85.0 157.2,75.0","#5b5cf0",[51,1361],{"x":1362,"y":61,"width":1328,"height":1329,"rx":182,"fill":1330,"stroke":1331,"style":628},"171.2",[630,1364,1366],{"transform":1365},"translate(216.8,37.8) scale(1.000)",[634,1367],{"d":1368,"fill":637,"stroke":1338,"style":1339},"M11 4a7 7 0 1 0 .01 0z M16 16l5 5",[72,1370,1373],{"x":1371,"y":1372,"style":1344},"228.8","84.2","List",[72,1375,1377],{"x":1371,"y":1376,"style":1344},"101.8","candidates",[72,1379,1381],{"x":1371,"y":1380,"style":1349},"121.2","per column",[267,1383],{"x1":1384,"y1":1353,"x2":1385,"y2":1353,"stroke":1331,"style":628},"291.4","313.4",[1356,1387],{"points":1388,"fill":1359},"317.4,80.0 308.4,85.0 308.4,75.0",[51,1390],{"x":1391,"y":61,"width":1328,"height":1329,"rx":182,"fill":150,"stroke":151,"style":628},"322.4",[630,1393,1395],{"transform":1394},"translate(368.0,46.5) scale(1.000)",[634,1396],{"d":1397,"fill":637,"stroke":701,"style":1339},"M9 11a3.5 3.5 0 1 0 0-7 3.5 3.5 0 0 0 0 7z M2.5 20c0-3.6 2.9-6 6.5-6s6.5 2.4 6.5 6 M16 4.5a3.3 3.3 0 0 1 0 6.3 M18 14.3c2.2.7 3.5 2.8 3.5 5.7",[72,1399,1401],{"x":246,"y":1343,"style":1400},"font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Agree rules",[72,1403,1404],{"x":246,"y":1348,"style":1349},"with the owner",[267,1406],{"x1":1407,"y1":1353,"x2":1408,"y2":1353,"stroke":1331,"style":628},"442.6","464.6",[1356,1410],{"points":1411,"fill":1359},"468.6,80.0 459.6,85.0 459.6,75.0",[51,1413],{"x":1414,"y":61,"width":1328,"height":1329,"rx":182,"fill":141,"stroke":142,"style":628},"473.6",[630,1416,1418],{"transform":1417},"translate(519.2,37.8) scale(1.000)",[634,1419],{"d":1420,"fill":637,"stroke":638,"style":1339},"M3 5h18l-7 8v6l-4 2v-8z",[72,1422,1425],{"x":1423,"y":1372,"style":1424},"531.2","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Read with",[72,1427,1428],{"x":1423,"y":1376,"style":1424},"rules",[72,1430,1431],{"x":1423,"y":1380,"style":1349},"na_values dict",[267,1433],{"x1":1434,"y1":1353,"x2":1435,"y2":1353,"stroke":1331,"style":628},"593.8","615.8",[1356,1437],{"points":1438,"fill":1359},"619.8,80.0 610.8,85.0 610.8,75.0",[51,1440],{"x":1441,"y":61,"width":1328,"height":1329,"rx":182,"fill":1330,"stroke":1331,"style":628},"624.8",[630,1443,1445],{"transform":1444},"translate(670.4,29.0) scale(1.000)",[634,1446],{"d":636,"fill":637,"stroke":1338,"style":1339},[72,1448,1451],{"x":1449,"y":1450,"style":1344},"682.4","75.5","Mask",[72,1453,1454],{"x":1449,"y":1343,"style":1344},"sentinels",[72,1456,1457],{"x":1449,"y":1348,"style":1349},"-999,",[72,1459,159],{"x":1449,"y":1460,"style":1349},"130.0",[251,1462,1464],{"id":1463},"step-6-keep-the-meaning-in-a-status-column","Step 6 — Keep the meaning in a status column",[10,1466,1467,1468,1470,1471,1473],{},"Not every placeholder means \"unknown\". In the score column, \"TBC\" says a score is expected, \"-\" says the question was skipped, and ",[14,1469,16],{}," is what the survey tool writes when the respondent never reached the question. Once all three become ",[14,1472,600],{},", a report can no longer say how many scores are still to come. Record the reason in its own column before blanking the value:",[256,1475,1477],{"className":285,"code":1476,"language":287,"meta":261,"style":261},"raw_scores = pd.read_excel(\"survey.xlsx\", keep_default_na=False, dtype={\"score\": str})[\"score\"]\nSTATUS = {\"TBC\": \"pending\", \"-\": \"skipped\", \"-999\": \"not reached\"}\nstatus = raw_scores.map(STATUS).fillna(\"answered\")\nscore = pd.to_numeric(raw_scores.where(~raw_scores.isin(STATUS.keys())), errors=\"coerce\")\nresult = pd.DataFrame({\"score\": score, \"score_status\": status})\nprint(result.values.tolist())\n# [[nan, 'not reached'], [12.0, 'answered'], [nan, 'skipped'], [nan, 'pending']]\nprint(result[\"score_status\"].value_counts().to_dict())\n",[14,1478,1479,1521,1559,1579,1608,1629,1636,1641],{"__ignoreMap":261},[265,1480,1481,1484,1486,1488,1490,1492,1494,1496,1498,1500,1502,1504,1507,1509,1512,1514,1517,1519],{"class":267,"line":268},[265,1482,1483],{"class":298},"raw_scores ",[265,1485,366],{"class":294},[265,1487,566],{"class":298},[265,1489,544],{"class":275},[265,1491,392],{"class":298},[265,1493,871],{"class":870},[265,1495,366],{"class":294},[265,1497,876],{"class":423},[265,1499,392],{"class":298},[265,1501,1237],{"class":870},[265,1503,366],{"class":294},[265,1505,1506],{"class":298},"{",[265,1508,395],{"class":275},[265,1510,1511],{"class":298},": ",[265,1513,1242],{"class":423},[265,1515,1516],{"class":298},"})[",[265,1518,395],{"class":275},[265,1520,1165],{"class":298},[265,1522,1523,1526,1528,1531,1533,1535,1538,1540,1542,1544,1547,1549,1552,1554,1557],{"class":267,"line":308},[265,1524,1525],{"class":423},"STATUS",[265,1527,774],{"class":294},[265,1529,1530],{"class":298}," {",[265,1532,516],{"class":275},[265,1534,1511],{"class":298},[265,1536,1537],{"class":275},"\"pending\"",[265,1539,392],{"class":298},[265,1541,492],{"class":275},[265,1543,1511],{"class":298},[265,1545,1546],{"class":275},"\"skipped\"",[265,1548,392],{"class":298},[265,1550,1551],{"class":275},"\"-999\"",[265,1553,1511],{"class":298},[265,1555,1556],{"class":275},"\"not reached\"",[265,1558,855],{"class":298},[265,1560,1561,1564,1566,1569,1571,1574,1577],{"class":267,"line":315},[265,1562,1563],{"class":298},"status ",[265,1565,366],{"class":294},[265,1567,1568],{"class":298}," raw_scores.map(",[265,1570,1525],{"class":423},[265,1572,1573],{"class":298},").fillna(",[265,1575,1576],{"class":275},"\"answered\"",[265,1578,547],{"class":298},[265,1580,1581,1584,1586,1589,1592,1595,1597,1600,1602,1604,1606],{"class":267,"line":328},[265,1582,1583],{"class":298},"score ",[265,1585,366],{"class":294},[265,1587,1588],{"class":298}," pd.to_numeric(raw_scores.where(",[265,1590,1591],{"class":294},"~",[265,1593,1594],{"class":298},"raw_scores.isin(",[265,1596,1525],{"class":423},[265,1598,1599],{"class":298},".keys())), ",[265,1601,1276],{"class":870},[265,1603,366],{"class":294},[265,1605,1281],{"class":275},[265,1607,547],{"class":298},[265,1609,1610,1613,1615,1618,1620,1623,1626],{"class":267,"line":341},[265,1611,1612],{"class":298},"result ",[265,1614,366],{"class":294},[265,1616,1617],{"class":298}," pd.DataFrame({",[265,1619,395],{"class":275},[265,1621,1622],{"class":298},": score, ",[265,1624,1625],{"class":275},"\"score_status\"",[265,1627,1628],{"class":298},": status})\n",[265,1630,1631,1633],{"class":267,"line":355},[265,1632,575],{"class":423},[265,1634,1635],{"class":298},"(result.values.tolist())\n",[265,1637,1638],{"class":267,"line":360},[265,1639,1640],{"class":593},"# [[nan, 'not reached'], [12.0, 'answered'], [nan, 'skipped'], [nan, 'pending']]\n",[265,1642,1643,1645,1648,1650],{"class":267,"line":372},[265,1644,575],{"class":423},[265,1646,1647],{"class":298},"(result[",[265,1649,1625],{"class":275},[265,1651,1652],{"class":298},"].value_counts().to_dict())\n",[10,1654,1655,1656,1658,1659,1661],{},"The numeric column is clean for sums and averages, and the status column answers the questions the placeholders were really trying to express: three missing scores, but only one of them is still expected. Keep the ",[14,1657,1525],{}," dictionary next to your ",[14,1660,771],{}," so the two stay consistent; any value that is in one but not the other is a rule someone forgot to write.",[251,1663,1665],{"id":1664},"step-7-excel-error-cells","Step 7 — Excel error cells",[10,1667,1668,1669,392,1672,1059,1675,1678,1679,1681],{},"Cells that show ",[14,1670,1671],{},"#DIV\u002F0!",[14,1673,1674],{},"#VALUE!",[14,1676,1677],{},"#REF!"," in Excel are read as ",[14,1680,600],{}," by both the openpyxl and calamine engines — indistinguishable from blanks. If an error means the source workbook is broken and the report should not run, check for errors before reading with pandas:",[256,1683,1685],{"className":285,"code":1684,"language":287,"meta":261,"style":261},"wb = Workbook()\nws = wb.active\nws.append([\"value\"])\nfor v in [10, \"#DIV\u002F0!\", \"#REF!\", 7]:\n    ws.append([v])\nwb.save(\"errors.xlsx\")\n\nfrom openpyxl import load_workbook\n\nerrors = [c.coordinate for row in load_workbook(\"errors.xlsx\").active.iter_rows()\n          for c in row if c.data_type == \"e\"]\nprint(errors)                                         # ['A3', 'A4']\nprint(pd.read_excel(\"errors.xlsx\")[\"value\"].tolist())  # [10.0, nan, nan, 7.0]\n",[14,1686,1687,1695,1703,1712,1743,1748,1757,1761,1772,1776,1801,1827,1837],{"__ignoreMap":261},[265,1688,1689,1691,1693],{"class":267,"line":268},[265,1690,363],{"class":298},[265,1692,366],{"class":294},[265,1694,369],{"class":298},[265,1696,1697,1699,1701],{"class":267,"line":308},[265,1698,375],{"class":298},[265,1700,366],{"class":294},[265,1702,380],{"class":298},[265,1704,1705,1707,1710],{"class":267,"line":315},[265,1706,386],{"class":298},[265,1708,1709],{"class":275},"\"value\"",[265,1711,408],{"class":298},[265,1713,1714,1716,1719,1721,1723,1726,1728,1731,1733,1736,1738,1741],{"class":267,"line":328},[265,1715,1249],{"class":294},[265,1717,1718],{"class":298}," v ",[265,1720,1255],{"class":294},[265,1722,1258],{"class":298},[265,1724,1725],{"class":423},"10",[265,1727,392],{"class":298},[265,1729,1730],{"class":275},"\"#DIV\u002F0!\"",[265,1732,392],{"class":298},[265,1734,1735],{"class":275},"\"#REF!\"",[265,1737,392],{"class":298},[265,1739,1740],{"class":423},"7",[265,1742,1263],{"class":298},[265,1744,1745],{"class":267,"line":341},[265,1746,1747],{"class":298},"    ws.append([v])\n",[265,1749,1750,1752,1755],{"class":267,"line":355},[265,1751,541],{"class":298},[265,1753,1754],{"class":275},"\"errors.xlsx\"",[265,1756,547],{"class":298},[265,1758,1759],{"class":267,"line":360},[265,1760,312],{"emptyLinePlaceholder":311},[265,1762,1763,1765,1767,1769],{"class":267,"line":372},[265,1764,344],{"class":294},[265,1766,347],{"class":298},[265,1768,295],{"class":294},[265,1770,1771],{"class":298}," load_workbook\n",[265,1773,1774],{"class":267,"line":383},[265,1775,312],{"emptyLinePlaceholder":311},[265,1777,1778,1781,1783,1786,1788,1791,1793,1796,1798],{"class":267,"line":411},[265,1779,1780],{"class":298},"errors ",[265,1782,366],{"class":294},[265,1784,1785],{"class":298}," [c.coordinate ",[265,1787,1249],{"class":294},[265,1789,1790],{"class":298}," row ",[265,1792,1255],{"class":294},[265,1794,1795],{"class":298}," load_workbook(",[265,1797,1754],{"class":275},[265,1799,1800],{"class":298},").active.iter_rows()\n",[265,1802,1803,1806,1809,1811,1813,1816,1819,1822,1825],{"class":267,"line":450},[265,1804,1805],{"class":294},"          for",[265,1807,1808],{"class":298}," c ",[265,1810,1255],{"class":294},[265,1812,1790],{"class":298},[265,1814,1815],{"class":294},"if",[265,1817,1818],{"class":298}," c.data_type ",[265,1820,1821],{"class":294},"==",[265,1823,1824],{"class":275}," \"e\"",[265,1826,1165],{"class":298},[265,1828,1829,1831,1834],{"class":267,"line":482},[265,1830,575],{"class":423},[265,1832,1833],{"class":298},"(errors)                                         ",[265,1835,1836],{"class":593},"# ['A3', 'A4']\n",[265,1838,1839,1841,1844,1846,1848,1850,1853],{"class":267,"line":506},[265,1840,575],{"class":423},[265,1842,1843],{"class":298},"(pd.read_excel(",[265,1845,1754],{"class":275},[265,1847,1160],{"class":298},[265,1849,1709],{"class":275},[265,1851,1852],{"class":298},"].tolist())  ",[265,1854,1855],{"class":593},"# [10.0, nan, nan, 7.0]\n",[10,1857,1858,1859,1049],{},"Failing on errors rather than blanking them keeps a broken lookup in the source from turning into a quietly smaller total. Broken references themselves are covered in ",[26,1860,1862],{"href":1861},"\u002Fgetting-started-with-python-excel-automation\u002Fworking-with-excel-formulas-in-python\u002Ffind-external-links-and-broken-references-in-a-workbook\u002F","Find External Links and Broken References in a Workbook with Python",[251,1864,1866],{"id":1865},"common-pitfalls","Common pitfalls",[1868,1869,1870,1878,1884,1890,1899,1908],"ul",{},[1871,1872,1873,1877],"li",{},[1874,1875,1876],"strong",{},"Global na_values lists."," A flat list applies to every column, so a rule meant for scores (\"-\") also blanks a comment that is just a dash. Use a dictionary.",[1871,1879,1880,1883],{},[1874,1881,1882],{},"Forgetting \"\" with keep_default_na=False."," Blank cells become empty strings and date or number columns turn into text.",[1871,1885,1886,1889],{},[1874,1887,1888],{},"Placeholders with meaning."," \"TBC\", \"exempt\" and \"not applicable\" can be different facts. Blanking them loses information; map them to a status column first.",[1871,1891,1892,1895,1896,1898],{},[1874,1893,1894],{},"Rules that drift between scripts."," Two jobs reading the same sheet with different placeholder lists will disagree on every count. Keep ",[14,1897,771],{}," in one shared module and import it.",[1871,1900,1901,1904,1905,1907],{},[1874,1902,1903],{},"Numbers typed as placeholders."," A ",[14,1906,54],{}," can be a real zero or someone's way of saying \"no data\". Only the sheet owner knows; do not guess, ask.",[1871,1909,1910,1904,1913,1915],{},[1874,1911,1912],{},"Replacing after arithmetic.",[14,1914,16],{}," that entered a sum before being masked has already changed the result. Clean at read time.",[251,1917,1919],{"id":1918},"conclusion","Conclusion",[10,1921,1922,1923,1925,1926,1928,1929,1931,1932,1934],{},"Default missing-value handling in pandas is both too eager (\"NA\", \"None\") and too timid (\"-\", \"TBC\", ",[14,1924,16],{},"). Discover the placeholders each column actually uses, read with ",[14,1927,760],{}," and a per-column ",[14,1930,756],{}," dictionary that always includes ",[14,1933,788],{},", blank whitespace-only cells with an anchored pattern, and mask sentinel numbers and dates after deciding what they mean. Check for Excel error cells separately when they should stop the run.",[251,1936,1938],{"id":1937},"frequently-asked-questions","Frequently asked questions",[10,1940,1941,1944,1945],{},[1874,1942,1943],{},"Which strings does pandas treat as missing by default?","\nThe empty string plus \"#N\u002FA\", \"#N\u002FA N\u002FA\", \"#NA\", \"-1.#IND\", \"-1.#QNAN\", \"-NaN\", \"-nan\", \"1.#IND\", \"1.#QNAN\", \"",[1946,1947,1948],"na",{},"\", \"N\u002FA\", \"NA\", \"NULL\", \"NaN\", \"None\", \"n\u002Fa\", \"nan\" and \"null\". Anything else, such as \"-\" or \"TBC\", stays as text.",[10,1950,1951,1954],{},[1874,1952,1953],{},"Why did the country code NA disappear?","\nThe string \"NA\" is on pandas' default list of missing-value strings, so Namibia's code is read as NaN. Turn the defaults off for that column by passing keep_default_na=False with an explicit na_values dictionary.",[10,1956,1957,1960],{},[1874,1958,1959],{},"Why are my empty cells empty strings after using keep_default_na=False?","\nWith keep_default_na=False, only the values you list are treated as missing, and that includes the empty string. Add \"\" to the list for every column where blank cells should be NaN.",[10,1962,1963,1966],{},[1874,1964,1965],{},"How are Excel error values like #DIV\u002F0! read?","\nAs NaN, with both the openpyxl and calamine engines. If you need to know which cells held errors rather than blanks, inspect the workbook with openpyxl, where error cells have data_type \"e\".",[251,1968,1970],{"id":1969},"related","Related",[1868,1972,1973,1979,1986,1993],{},[1871,1974,1975,1976,1978],{},"Up one level: ",[26,1977,29],{"href":28}," — everything about gaps in Excel data.",[1871,1980,1981,1985],{},[26,1982,1984],{"href":1983},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffill-missing-values-in-excel-with-pandas-fillna\u002F","Fill Missing Values in Excel with pandas fillna"," — what to do once gaps are real NaN.",[1871,1987,1988,1992],{},[26,1989,1991],{"href":1990},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fdrop-rows-with-missing-required-fields-in-pandas\u002F","Drop Rows with Missing Required Fields in pandas"," — enforcing required values.",[1871,1994,1995,1999],{},[26,1996,1998],{"href":1997},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Freplace-values-in-an-excel-column-with-pandas\u002F","Replace Values in an Excel Column with pandas"," — recoding values that are not placeholders.",[2001,2002,2003],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}",{"title":261,"searchDepth":308,"depth":308,"links":2005},[2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017],{"id":253,"depth":308,"text":254},{"id":550,"depth":308,"text":551},{"id":746,"depth":308,"text":747},{"id":942,"depth":308,"text":943},{"id":1052,"depth":308,"text":1053},{"id":1204,"depth":308,"text":1205},{"id":1463,"depth":308,"text":1464},{"id":1664,"depth":308,"text":1665},{"id":1865,"depth":308,"text":1866},{"id":1918,"depth":308,"text":1919},{"id":1937,"depth":308,"text":1938},{"id":1969,"depth":308,"text":1970},"2026-09-23","Excel sheets mark missing data with \"n\u002Fa\", \"-\", \"TBC\", -999 or 31\u002F12\u002F9999. Turn those placeholders into real missing values with read_excel's na_values, per-column rules and sentinel masks — without losing genuine values such as the country code NA.","md",[2022,2024,2026,2028],{"q":1943,"a":2023},"The empty string plus \"#N\u002FA\", \"#N\u002FA N\u002FA\", \"#NA\", \"-1.#IND\", \"-1.#QNAN\", \"-NaN\", \"-nan\", \"1.#IND\", \"1.#QNAN\", \"\u003CNA>\", \"N\u002FA\", \"NA\", \"NULL\", \"NaN\", \"None\", \"n\u002Fa\", \"nan\" and \"null\". Anything else, such as \"-\" or \"TBC\", stays as text.",{"q":1953,"a":2025},"The string \"NA\" is on pandas' default list of missing-value strings, so Namibia's code is read as NaN. Turn the defaults off for that column by passing keep_default_na=False with an explicit na_values dictionary.",{"q":1959,"a":2027},"With keep_default_na=False, only the values you list are treated as missing, and that includes the empty string. Add \"\" to the list for every column where blank cells should be NaN.",{"q":1965,"a":2029},"As NaN, with both the openpyxl and calamine engines. If you need to know which cells held errors rather than blanks, inspect the workbook with openpyxl, where error cells have data_type \"e\".","replace placeholder text with nan in pandas",null,{"breadcrumb":2033},[2034,2037,2040],{"name":2035,"item":2036},"Home","\u002F",{"name":2038,"item":2039},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":2041,"item":28},"Handling Missing Data in Excel Reports with Pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Freplace-placeholder-text-with-nan-in-pandas",{"title":2044,"description":2045},"Replace Placeholder Text with NaN in pandas (na_values, sentinels)","Convert placeholders like n\u002Fa, -, TBC, -999 and 9999-12-31 in Excel data to NaN with pandas — default NA strings, per-column na_values, keep_default_na pitfalls, sentinel numbers and dates, and Excel error cells.","replace-placeholder-text-with-nan-in-pandas","advanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Freplace-placeholder-text-with-nan-in-pandas\u002Findex",[2049,2050,2051],"missing-data","data-cleaning","pandas","how-to","0_HcN_heqMRTXYNZpeCBqRzc5nxcg3R5vz_IRDJHAYg",[2055,2059],{"title":2056,"path":2057,"stem":2058,"children":-1},"Interpolate Missing Numeric Values in Excel Data","\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Finterpolate-missing-numeric-values-in-excel-data","advanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Finterpolate-missing-numeric-values-in-excel-data\u002Findex",{"title":2060,"path":2061,"stem":2062,"children":-1},"Tell Blank Cells, Empty Strings and Zeros Apart in Excel Data","\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ftell-blank-cells-empty-strings-and-zeros-apart-in-excel-data","advanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ftell-blank-cells-empty-strings-and-zeros-apart-in-excel-data\u002Findex",[2064,2066],{"path":2065,"title":2041},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports",{"path":2067,"title":2038},"\u002Fadvanced-data-transformation-and-cleaning",[],[2070,2073,2076,2079,2081,2084],{"path":2071,"title":2072,"kind":2052},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-blank-cells-with-openpyxl","Highlight Blank Cells in Excel with openpyxl",{"path":2074,"title":2075,"kind":2052},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fcoalesce-several-excel-columns-into-one-with-pandas","Coalesce Several Excel Columns into One with pandas",{"path":2077,"title":2078,"kind":2052},"\u002Fadvanced-data-transformation-and-cleaning\u002Fexcel-formula-equivalents-in-pandas\u002Fiferror-and-isna-equivalents-in-pandas","IFERROR, IFNA, ISNA and ISNUMBER Equivalents in pandas",{"path":2080,"title":1991,"kind":2052},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fdrop-rows-with-missing-required-fields-in-pandas",{"path":2082,"title":2083,"kind":2052},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffill-gaps-in-per-item-time-series-with-a-staleness-limit","Fill Gaps in Per-Item Time Series with a Staleness Limit",{"path":2085,"title":2086,"kind":2052},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffill-missing-values-in-excel-with-pandas-fillna","Fill Missing Values in Excel with Pandas fillna",1790972096074]