[{"data":1,"prerenderedAt":2420},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-non-printable-characters-from-excel-data":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-non-printable-characters-from-excel-data":2386,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-non-printable-characters-from-excel-data":2395,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-non-printable-characters-from-excel-data":2400,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-non-printable-characters-from-excel-data":2401},{"id":4,"title":5,"body":6,"dateModified":2351,"datePublished":2351,"description":2352,"extension":2353,"faq":2354,"keyword":2364,"kind":2365,"meta":2366,"navigation":263,"path":2376,"seo":2377,"slug":2379,"stem":2380,"topics":2381,"type":2384,"__hash__":2385},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-non-printable-characters-from-excel-data\u002Findex.md","Remove Non-Printable Characters from Excel Data with Python",{"type":7,"value":8,"toc":2336},"minimark",[9,24,190,195,230,233,481,485,488,787,842,851,865,940,943,1101,1111,1272,1276,1291,1454,1460,1464,1467,1753,1756,1760,1766,1814,1821,1825,1828,1886,1889,2045,2049,2052,2206,2213,2217,2257,2261,2267,2271,2283,2289,2295,2301,2305,2332],[10,11,12,13,18,19,23],"p",{},"Two product codes look identical in Excel, but a lookup says they are different. A name exports to CSV and breaks the file across two lines. A report shows \"Line one_x000D_\" where a line break should be. All three are caused by characters you cannot see: control codes, format characters and escape sequences. This guide, part of ",[14,15,17],"a",{"href":16},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002F","Cleaning Excel Data with pandas",", shows how to find them by Unicode category, decode the escapes openpyxl leaves behind, and remove what should go while keeping the line breaks and emoji you want. Spaces — ordinary, non-breaking and repeated — are covered separately in ",[14,20,22],{"href":21},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstrip-whitespace-and-normalise-text-columns-with-pandas\u002F","Strip Whitespace and Normalise Text Columns with pandas",".",[25,26,34,35,34,39,34,43,34,51,34,59,34,66,34,73,34,76,34,80,34,85,34,91,34,93,34,98,34,100,34,103,34,106,34,110,34,112,34,115,34,117,34,120,34,123,34,127,34,129,34,133,34,135,34,137,34,140,34,144,34,146,34,149,34,151,34,154,34,157,34,161,34,163,34,167,34,169,34,172,34,175,34,179,34,181,34,185,34,187],"svg",{"viewBox":27,"role":28,"ariaLabelledBy":29,"xmlns":32,"style":33},"4 0 752 276","img",[30,31],"np-kinds-t","np-kinds-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[36,37,38],"title",{"id":30},"Invisible characters in Excel text and what to do with them",[40,41,42],"desc",{"id":31},"Carriage returns and line feeds are control characters to normalise, not delete. Tabs and bell codes are control characters to remove or replace with a space. The soft hyphen, zero-width space and byte-order mark are format characters to remove. The zero-width joiner inside emoji is a format character to keep. The replacement character U+FFFD is a visible symbol that signals lost data and should be reported.",[44,45],"rect",{"x":46,"y":47,"width":48,"height":49,"fill":50},"4","0","752","276","#ffffff",[44,52],{"x":53,"y":54,"width":55,"height":56,"rx":47,"fill":50,"stroke":57,"style":58},"20.0","16.0","263.8","40.0","var(--line,#cdd5e6)","stroke-width:1px",[44,60],{"x":61,"y":54,"width":62,"height":56,"rx":47,"fill":63,"stroke":64,"style":65},"283.8","228.1","#f0f4ff","var(--brand,#5b5cf0)","stroke-width:1.2px",[67,68,72],"text",{"x":69,"y":70,"style":71},"397.9","41.0","font-size:13px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Category",[44,74],{"x":75,"y":54,"width":62,"height":56,"rx":47,"fill":63,"stroke":64,"style":65},"511.9",[67,77,79],{"x":78,"y":70,"style":71},"626.0","Action",[44,81],{"x":53,"y":82,"width":55,"height":83,"rx":47,"fill":84,"stroke":57,"style":58},"56.0","34.0","#e7ebef",[67,86,90],{"x":87,"y":88,"style":89},"32.0","78.0","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:start","\\r \\n inside a cell",[44,92],{"x":61,"y":82,"width":62,"height":83,"rx":47,"fill":50,"stroke":57,"style":58},[67,94,97],{"x":69,"y":95,"style":96},"77.0","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","Cc control",[44,99],{"x":75,"y":82,"width":62,"height":83,"rx":47,"fill":50,"stroke":57,"style":58},[67,101,102],{"x":78,"y":95,"style":96},"normalise",[44,104],{"x":53,"y":105,"width":55,"height":83,"rx":47,"fill":84,"stroke":57,"style":58},"90.0",[67,107,109],{"x":87,"y":108,"style":89},"112.0","\\t tab, \\x07 bell",[44,111],{"x":61,"y":105,"width":62,"height":83,"rx":47,"fill":50,"stroke":57,"style":58},[67,113,97],{"x":69,"y":114,"style":96},"111.0",[44,116],{"x":75,"y":105,"width":62,"height":83,"rx":47,"fill":50,"stroke":57,"style":58},[67,118,119],{"x":78,"y":114,"style":96},"remove",[44,121],{"x":53,"y":122,"width":55,"height":83,"rx":47,"fill":84,"stroke":57,"style":58},"124.0",[67,124,126],{"x":87,"y":125,"style":89},"146.0","soft hyphen, zero-width space, BOM",[44,128],{"x":61,"y":122,"width":62,"height":83,"rx":47,"fill":50,"stroke":57,"style":58},[67,130,132],{"x":69,"y":131,"style":96},"145.0","Cf format",[44,134],{"x":75,"y":122,"width":62,"height":83,"rx":47,"fill":50,"stroke":57,"style":58},[67,136,119],{"x":78,"y":131,"style":96},[44,138],{"x":53,"y":139,"width":55,"height":83,"rx":47,"fill":84,"stroke":57,"style":58},"158.0",[67,141,143],{"x":87,"y":142,"style":89},"180.0","zero-width joiner in emoji",[44,145],{"x":61,"y":139,"width":62,"height":83,"rx":47,"fill":50,"stroke":57,"style":58},[67,147,132],{"x":69,"y":148,"style":96},"179.0",[44,150],{"x":75,"y":139,"width":62,"height":83,"rx":47,"fill":50,"stroke":57,"style":58},[67,152,153],{"x":78,"y":148,"style":96},"keep",[44,155],{"x":53,"y":156,"width":55,"height":83,"rx":47,"fill":84,"stroke":57,"style":58},"192.0",[67,158,160],{"x":87,"y":159,"style":89},"214.0","_x000D_ escape text",[44,162],{"x":61,"y":156,"width":62,"height":83,"rx":47,"fill":50,"stroke":57,"style":58},[67,164,166],{"x":69,"y":165,"style":96},"213.0","plain letters",[44,168],{"x":75,"y":156,"width":62,"height":83,"rx":47,"fill":50,"stroke":57,"style":58},[67,170,171],{"x":78,"y":165,"style":96},"decode",[44,173],{"x":53,"y":174,"width":55,"height":83,"rx":47,"fill":84,"stroke":57,"style":58},"226.0",[67,176,178],{"x":87,"y":177,"style":89},"248.0","U+FFFD replacement",[44,180],{"x":61,"y":174,"width":62,"height":83,"rx":47,"fill":50,"stroke":57,"style":58},[67,182,184],{"x":69,"y":183,"style":96},"247.0","So symbol",[44,186],{"x":75,"y":174,"width":62,"height":83,"rx":47,"fill":50,"stroke":57,"style":58},[67,188,189],{"x":78,"y":183,"style":96},"report",[191,192,194],"h2",{"id":193},"prerequisites","Prerequisites",[196,197,202],"pre",{"className":198,"code":199,"language":200,"meta":201,"style":201},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl python-calamine xlsxwriter\n","bash","",[203,204,205],"code",{"__ignoreMap":201},[206,207,210,214,218,221,224,227],"span",{"class":208,"line":209},"line",1,[206,211,213],{"class":212},"sMTad","pip",[206,215,217],{"class":216},"srMev"," install",[206,219,220],{"class":216}," pandas",[206,222,223],{"class":216}," openpyxl",[206,225,226],{"class":216}," python-calamine",[206,228,229],{"class":216}," xlsxwriter\n",[10,231,232],{},"A sample workbook written with xlsxwriter, which stores control characters the same way Excel does:",[196,234,238],{"className":235,"code":236,"language":237,"meta":201,"style":201},"language-python shiki shiki-themes github-light github-dark-high-contrast","import re\nimport unicodedata\n\nimport pandas as pd\n\nraw = pd.DataFrame({\"sku\": [\"AB-100\", \"AB\\u200b-100\", \"\\ufeffAB-200\", \"AB-3\\xad00\"],\n                    \"note\": [\"Line one\\r\\nLine two\", \"tab\\there\", \"bell\\x07!\", \"caf\\ufffd 👩\\u200d💻\"]})\nraw.to_excel(\"invisible.xlsx\", index=False, engine=\"xlsxwriter\")\ndf = pd.read_excel(\"invisible.xlsx\")\nprint(df[\"sku\"].tolist())\nprint(df[\"sku\"].nunique())        # 4 — although they look like 3 codes\n","python",[203,239,240,250,258,265,279,284,343,402,436,451,465],{"__ignoreMap":201},[206,241,242,246],{"class":208,"line":209},[206,243,245],{"class":244},"s-kum","import",[206,247,249],{"class":248},"skGVy"," re\n",[206,251,253,255],{"class":208,"line":252},2,[206,254,245],{"class":244},[206,256,257],{"class":248}," unicodedata\n",[206,259,261],{"class":208,"line":260},3,[206,262,264],{"emptyLinePlaceholder":263},true,"\n",[206,266,268,270,273,276],{"class":208,"line":267},4,[206,269,245],{"class":244},[206,271,272],{"class":248}," pandas ",[206,274,275],{"class":244},"as",[206,277,278],{"class":248}," pd\n",[206,280,282],{"class":208,"line":281},5,[206,283,264],{"emptyLinePlaceholder":263},[206,285,287,290,293,296,299,302,305,308,311,315,318,320,323,326,329,331,334,337,340],{"class":208,"line":286},6,[206,288,289],{"class":248},"raw ",[206,291,292],{"class":244},"=",[206,294,295],{"class":248}," pd.DataFrame({",[206,297,298],{"class":216},"\"sku\"",[206,300,301],{"class":248},": [",[206,303,304],{"class":216},"\"AB-100\"",[206,306,307],{"class":248},", ",[206,309,310],{"class":216},"\"AB",[206,312,314],{"class":313},"sSjpA","\\u200b",[206,316,317],{"class":216},"-100\"",[206,319,307],{"class":248},[206,321,322],{"class":216},"\"",[206,324,325],{"class":313},"\\ufeff",[206,327,328],{"class":216},"AB-200\"",[206,330,307],{"class":248},[206,332,333],{"class":216},"\"AB-3",[206,335,336],{"class":313},"\\xad",[206,338,339],{"class":216},"00\"",[206,341,342],{"class":248},"],\n",[206,344,346,349,351,354,357,360,362,365,368,371,373,376,379,382,384,387,390,393,396,399],{"class":208,"line":345},7,[206,347,348],{"class":216},"                    \"note\"",[206,350,301],{"class":248},[206,352,353],{"class":216},"\"Line one",[206,355,356],{"class":313},"\\r\\n",[206,358,359],{"class":216},"Line two\"",[206,361,307],{"class":248},[206,363,364],{"class":216},"\"tab",[206,366,367],{"class":313},"\\t",[206,369,370],{"class":216},"here\"",[206,372,307],{"class":248},[206,374,375],{"class":216},"\"bell",[206,377,378],{"class":313},"\\x07",[206,380,381],{"class":216},"!\"",[206,383,307],{"class":248},[206,385,386],{"class":216},"\"caf",[206,388,389],{"class":313},"\\ufffd",[206,391,392],{"class":216}," 👩",[206,394,395],{"class":313},"\\u200d",[206,397,398],{"class":216},"💻\"",[206,400,401],{"class":248},"]})\n",[206,403,405,408,411,413,417,419,423,425,428,430,433],{"class":208,"line":404},8,[206,406,407],{"class":248},"raw.to_excel(",[206,409,410],{"class":216},"\"invisible.xlsx\"",[206,412,307],{"class":248},[206,414,416],{"class":415},"sa561","index",[206,418,292],{"class":244},[206,420,422],{"class":421},"sP0c6","False",[206,424,307],{"class":248},[206,426,427],{"class":415},"engine",[206,429,292],{"class":244},[206,431,432],{"class":216},"\"xlsxwriter\"",[206,434,435],{"class":248},")\n",[206,437,439,442,444,447,449],{"class":208,"line":438},9,[206,440,441],{"class":248},"df ",[206,443,292],{"class":244},[206,445,446],{"class":248}," pd.read_excel(",[206,448,410],{"class":216},[206,450,435],{"class":248},[206,452,454,457,460,462],{"class":208,"line":453},10,[206,455,456],{"class":421},"print",[206,458,459],{"class":248},"(df[",[206,461,298],{"class":216},[206,463,464],{"class":248},"].tolist())\n",[206,466,468,470,472,474,477],{"class":208,"line":467},11,[206,469,456],{"class":421},[206,471,459],{"class":248},[206,473,298],{"class":216},[206,475,476],{"class":248},"].nunique())        ",[206,478,480],{"class":479},"s-wDw","# 4 — although they look like 3 codes\n",[191,482,484],{"id":483},"step-1-audit-before-you-delete","Step 1 — Audit before you delete",[10,486,487],{},"Count the characters by Unicode category, per column, so you know what you are dealing with and can explain the change later:",[196,489,491],{"className":235,"code":490,"language":237,"meta":201,"style":201},"SUSPECT = {\"Cc\", \"Cf\", \"Co\", \"Cn\"}\n\ndef audit(frame):\n    rows = []\n    for col in frame.select_dtypes(include=[\"object\", \"string\"]).columns:\n        counts = {}\n        for value in frame[col].dropna().astype(str):\n            for ch in value:\n                cat = unicodedata.category(ch)\n                if cat in SUSPECT or ch == \"\\ufffd\":\n                    name = unicodedata.name(ch, f\"U+{ord(ch):04X}\")\n                    counts[name] = counts.get(name, 0) + 1\n        rows += [{\"column\": col, \"character\": k, \"count\": v} for k, v in counts.items()]\n    return pd.DataFrame(rows)\n\nprint(audit(df))\n",[203,492,493,525,529,541,551,584,594,613,626,636,667,702,724,765,774,779],{"__ignoreMap":201},[206,494,495,498,501,504,507,509,512,514,517,519,522],{"class":208,"line":209},[206,496,497],{"class":421},"SUSPECT",[206,499,500],{"class":244}," =",[206,502,503],{"class":248}," {",[206,505,506],{"class":216},"\"Cc\"",[206,508,307],{"class":248},[206,510,511],{"class":216},"\"Cf\"",[206,513,307],{"class":248},[206,515,516],{"class":216},"\"Co\"",[206,518,307],{"class":248},[206,520,521],{"class":216},"\"Cn\"",[206,523,524],{"class":248},"}\n",[206,526,527],{"class":208,"line":252},[206,528,264],{"emptyLinePlaceholder":263},[206,530,531,534,538],{"class":208,"line":260},[206,532,533],{"class":244},"def",[206,535,537],{"class":536},"s_Opv"," audit",[206,539,540],{"class":248},"(frame):\n",[206,542,543,546,548],{"class":208,"line":267},[206,544,545],{"class":248},"    rows ",[206,547,292],{"class":244},[206,549,550],{"class":248}," []\n",[206,552,553,556,559,562,565,568,570,573,576,578,581],{"class":208,"line":281},[206,554,555],{"class":244},"    for",[206,557,558],{"class":248}," col ",[206,560,561],{"class":244},"in",[206,563,564],{"class":248}," frame.select_dtypes(",[206,566,567],{"class":415},"include",[206,569,292],{"class":244},[206,571,572],{"class":248},"[",[206,574,575],{"class":216},"\"object\"",[206,577,307],{"class":248},[206,579,580],{"class":216},"\"string\"",[206,582,583],{"class":248},"]).columns:\n",[206,585,586,589,591],{"class":208,"line":286},[206,587,588],{"class":248},"        counts ",[206,590,292],{"class":244},[206,592,593],{"class":248}," {}\n",[206,595,596,599,602,604,607,610],{"class":208,"line":345},[206,597,598],{"class":244},"        for",[206,600,601],{"class":248}," value ",[206,603,561],{"class":244},[206,605,606],{"class":248}," frame[col].dropna().astype(",[206,608,609],{"class":421},"str",[206,611,612],{"class":248},"):\n",[206,614,615,618,621,623],{"class":208,"line":404},[206,616,617],{"class":244},"            for",[206,619,620],{"class":248}," ch ",[206,622,561],{"class":244},[206,624,625],{"class":248}," value:\n",[206,627,628,631,633],{"class":208,"line":438},[206,629,630],{"class":248},"                cat ",[206,632,292],{"class":244},[206,634,635],{"class":248}," unicodedata.category(ch)\n",[206,637,638,641,644,646,649,652,654,657,660,662,664],{"class":208,"line":453},[206,639,640],{"class":244},"                if",[206,642,643],{"class":248}," cat ",[206,645,561],{"class":244},[206,647,648],{"class":421}," SUSPECT",[206,650,651],{"class":244}," or",[206,653,620],{"class":248},[206,655,656],{"class":244},"==",[206,658,659],{"class":216}," \"",[206,661,389],{"class":313},[206,663,322],{"class":216},[206,665,666],{"class":248},":\n",[206,668,669,672,674,677,680,683,686,689,692,695,698,700],{"class":208,"line":467},[206,670,671],{"class":248},"                    name ",[206,673,292],{"class":244},[206,675,676],{"class":248}," unicodedata.name(ch, ",[206,678,679],{"class":244},"f",[206,681,682],{"class":216},"\"U+",[206,684,685],{"class":313},"{",[206,687,688],{"class":421},"ord",[206,690,691],{"class":248},"(ch)",[206,693,694],{"class":244},":04X",[206,696,697],{"class":313},"}",[206,699,322],{"class":216},[206,701,435],{"class":248},[206,703,705,708,710,713,715,718,721],{"class":208,"line":704},12,[206,706,707],{"class":248},"                    counts[name] ",[206,709,292],{"class":244},[206,711,712],{"class":248}," counts.get(name, ",[206,714,47],{"class":421},[206,716,717],{"class":248},") ",[206,719,720],{"class":244},"+",[206,722,723],{"class":421}," 1\n",[206,725,727,730,733,736,739,742,745,748,751,754,757,760,762],{"class":208,"line":726},13,[206,728,729],{"class":248},"        rows ",[206,731,732],{"class":244},"+=",[206,734,735],{"class":248}," [{",[206,737,738],{"class":216},"\"column\"",[206,740,741],{"class":248},": col, ",[206,743,744],{"class":216},"\"character\"",[206,746,747],{"class":248},": k, ",[206,749,750],{"class":216},"\"count\"",[206,752,753],{"class":248},": v} ",[206,755,756],{"class":244},"for",[206,758,759],{"class":248}," k, v ",[206,761,561],{"class":244},[206,763,764],{"class":248}," counts.items()]\n",[206,766,768,771],{"class":208,"line":767},14,[206,769,770],{"class":244},"    return",[206,772,773],{"class":248}," pd.DataFrame(rows)\n",[206,775,777],{"class":208,"line":776},15,[206,778,264],{"emptyLinePlaceholder":263},[206,780,782,784],{"class":208,"line":781},16,[206,783,456],{"class":421},[206,785,786],{"class":248},"(audit(df))\n",[10,788,789,790,307,793,307,796,799,800,307,803,806,807,810,811,814,815,818,819,822,823,826,827,830,831,834,835,810,838,841],{},"The report names each character where Unicode gives it a name — ",[203,791,792],{},"ZERO WIDTH SPACE",[203,794,795],{},"SOFT HYPHEN",[203,797,798],{},"ZERO WIDTH NO-BREAK SPACE"," (the byte-order mark), ",[203,801,802],{},"ZERO WIDTH JOINER",[203,804,805],{},"REPLACEMENT CHARACTER"," — rather than showing invisible output; control codes have no names, so the line feed and tab appear as ",[203,808,809],{},"U+000A"," and ",[203,812,813],{},"U+0009",". Category ",[203,816,817],{},"Cc"," holds control codes, ",[203,820,821],{},"Cf"," format characters, ",[203,824,825],{},"Co"," private-use characters from custom fonts, and ",[203,828,829],{},"Cn"," unassigned code points. The ",[203,832,833],{},"note"," column also contains the texts ",[203,836,837],{},"_x000D_",[203,839,840],{},"_x0007_"," (the carriage return and the bell), which the audit does not flag at all, because those are ordinary letters.",[191,843,845,846,850],{"id":844},"step-2-decode-the-x000d-escapes","Step 2 — Decode the ",[847,848,849],"em",{},"x000D"," escapes",[10,852,853,854,857,858,860,861,864],{},"Excel's file format cannot store most control characters directly, so it writes them as ",[203,855,856],{},"_xHHHH_"," escape sequences — a carriage return becomes ",[203,859,837],{},". The calamine engine decodes these; openpyxl, the default engine for ",[203,862,863],{},"read_excel",", leaves them in the text:",[196,866,868],{"className":235,"code":867,"language":237,"meta":201,"style":201},"print(repr(pd.read_excel(\"invisible.xlsx\")[\"note\"][0]))\n# 'Line one_x000D_\\nLine two'\nprint(repr(pd.read_excel(\"invisible.xlsx\", engine=\"calamine\")[\"note\"][0]))\n# 'Line one\\r\\nLine two'\n",[203,869,870,899,904,935],{"__ignoreMap":201},[206,871,872,874,877,880,883,885,888,891,894,896],{"class":208,"line":209},[206,873,456],{"class":421},[206,875,876],{"class":248},"(",[206,878,879],{"class":421},"repr",[206,881,882],{"class":248},"(pd.read_excel(",[206,884,410],{"class":216},[206,886,887],{"class":248},")[",[206,889,890],{"class":216},"\"note\"",[206,892,893],{"class":248},"][",[206,895,47],{"class":421},[206,897,898],{"class":248},"]))\n",[206,900,901],{"class":208,"line":252},[206,902,903],{"class":479},"# 'Line one_x000D_\\nLine two'\n",[206,905,906,908,910,912,914,916,918,920,922,925,927,929,931,933],{"class":208,"line":260},[206,907,456],{"class":421},[206,909,876],{"class":248},[206,911,879],{"class":421},[206,913,882],{"class":248},[206,915,410],{"class":216},[206,917,307],{"class":248},[206,919,427],{"class":415},[206,921,292],{"class":244},[206,923,924],{"class":216},"\"calamine\"",[206,926,887],{"class":248},[206,928,890],{"class":216},[206,930,893],{"class":248},[206,932,47],{"class":421},[206,934,898],{"class":248},[206,936,937],{"class":208,"line":267},[206,938,939],{"class":479},"# 'Line one\\r\\nLine two'\n",[10,941,942],{},"If you read with openpyxl, decode the escapes yourself before any other cleaning:",[196,944,946],{"className":235,"code":945,"language":237,"meta":201,"style":201},"ESCAPE = re.compile(r\"_x([0-9A-Fa-f]{4})_\")\n\ndef decode_escapes(text):\n    return ESCAPE.sub(lambda m: chr(int(m.group(1), 16)), text) if isinstance(text, str) else text\n\ndf[\"note\"] = df[\"note\"].map(decode_escapes)\nprint(repr(df[\"note\"][0]))        # 'Line one\\r\\nLine two'\n",[203,947,948,984,988,998,1056,1060,1080],{"__ignoreMap":201},[206,949,950,953,955,958,961,964,967,970,973,976,979,982],{"class":208,"line":209},[206,951,952],{"class":421},"ESCAPE",[206,954,500],{"class":244},[206,956,957],{"class":248}," re.compile(",[206,959,960],{"class":244},"r",[206,962,963],{"class":216},"\"_x",[206,965,966],{"class":421},"([",[206,968,969],{"class":313},"0-9A-Fa-f",[206,971,972],{"class":421},"]",[206,974,975],{"class":244},"{4}",[206,977,978],{"class":421},")",[206,980,981],{"class":216},"_\"",[206,983,435],{"class":248},[206,985,986],{"class":208,"line":252},[206,987,264],{"emptyLinePlaceholder":263},[206,989,990,992,995],{"class":208,"line":260},[206,991,533],{"class":244},[206,993,994],{"class":536}," decode_escapes",[206,996,997],{"class":248},"(text):\n",[206,999,1000,1002,1005,1008,1011,1014,1017,1019,1022,1025,1028,1031,1034,1037,1040,1043,1046,1048,1050,1053],{"class":208,"line":267},[206,1001,770],{"class":244},[206,1003,1004],{"class":421}," ESCAPE",[206,1006,1007],{"class":248},".sub(",[206,1009,1010],{"class":244},"lambda",[206,1012,1013],{"class":248}," m: ",[206,1015,1016],{"class":421},"chr",[206,1018,876],{"class":248},[206,1020,1021],{"class":421},"int",[206,1023,1024],{"class":248},"(m.group(",[206,1026,1027],{"class":421},"1",[206,1029,1030],{"class":248},"), ",[206,1032,1033],{"class":421},"16",[206,1035,1036],{"class":248},")), text) ",[206,1038,1039],{"class":244},"if",[206,1041,1042],{"class":421}," isinstance",[206,1044,1045],{"class":248},"(text, ",[206,1047,609],{"class":421},[206,1049,717],{"class":248},[206,1051,1052],{"class":244},"else",[206,1054,1055],{"class":248}," text\n",[206,1057,1058],{"class":208,"line":281},[206,1059,264],{"emptyLinePlaceholder":263},[206,1061,1062,1065,1067,1070,1072,1075,1077],{"class":208,"line":286},[206,1063,1064],{"class":248},"df[",[206,1066,890],{"class":216},[206,1068,1069],{"class":248},"] ",[206,1071,292],{"class":244},[206,1073,1074],{"class":248}," df[",[206,1076,890],{"class":216},[206,1078,1079],{"class":248},"].map(decode_escapes)\n",[206,1081,1082,1084,1086,1088,1090,1092,1094,1096,1099],{"class":208,"line":345},[206,1083,456],{"class":421},[206,1085,876],{"class":248},[206,1087,879],{"class":421},[206,1089,459],{"class":248},[206,1091,890],{"class":216},[206,1093,893],{"class":248},[206,1095,47],{"class":421},[206,1097,1098],{"class":248},"]))        ",[206,1100,939],{"class":479},[10,1102,1103,1104,1106,1107,1110],{},"There is one ambiguity: a cell whose real text is ",[203,1105,837],{}," — rare outside of documentation — is indistinguishable from an escaped carriage return once openpyxl has read it. When that matters, read with ",[203,1108,1109],{},"engine=\"calamine\"",", which keeps the two apart.",[25,1112,34,1117,34,1120,34,1123,34,1126,34,1132,34,1143,34,1149,34,1153,34,1157,34,1162,34,1167,34,1170,34,1176,34,1180,34,1184,34,1187,34,1191,34,1194,34,1197,34,1203,34,1207,34,1210,34,1213,34,1217,34,1220,34,1225,34,1232,34,1237,34,1240,34,1243,34,1247,34,1250,34,1255,34,1262,34,1268],{"viewBox":1113,"role":28,"ariaLabelledBy":1114,"xmlns":32,"style":33},"4 0 752 142",[1115,1116],"np-flow-t","np-flow-d",[36,1118,1119],{"id":1115},"Cleaning invisible characters in the right order",[40,1121,1122],{"id":1116},"Audit the text columns by Unicode category, decode _x000D_ escapes, normalise line breaks to a single line feed, remove control and format characters except an allow list, then flag replacement characters and compare the audit before and after.",[44,1124],{"x":46,"y":47,"width":48,"height":1125,"fill":50},"142",[44,1127],{"x":53,"y":54,"width":1128,"height":1129,"rx":1130,"fill":63,"stroke":64,"style":1131},"115.2","110.5","12","stroke-width:2px",[1133,1134,1136],"g",{"transform":1135},"translate(65.6,29.0) scale(1.000)",[1137,1138],"path",{"d":1139,"fill":1140,"stroke":1141,"style":1142},"M11 4a7 7 0 1 0 .01 0z M16 16l5 5","none","var(--brand-strong,#4338ca)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[67,1144,1148],{"x":1145,"y":1146,"style":1147},"77.6","75.5","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Audit",[67,1150,1152],{"x":1145,"y":1151,"style":96},"95.0","count by",[67,1154,1156],{"x":1145,"y":1155,"style":96},"112.5","category",[208,1158],{"x1":1159,"y1":1160,"x2":1161,"y2":1160,"stroke":64,"style":1131},"140.2","71.2","162.2",[1163,1164],"polygon",{"points":1165,"fill":1166},"166.2,71.2 157.2,76.2 157.2,66.2","#5b5cf0",[44,1168],{"x":1169,"y":54,"width":1128,"height":1129,"rx":1130,"fill":63,"stroke":64,"style":1131},"171.2",[1133,1171,1173],{"transform":1172},"translate(216.8,29.0) scale(1.000)",[1137,1174],{"d":1175,"fill":1140,"stroke":1141,"style":1142},"M9 8l-4 4 4 4 M15 8l4 4-4 4",[67,1177,1179],{"x":1178,"y":1146,"style":1147},"228.8","Decode",[67,1181,1183],{"x":1178,"y":1182,"style":1147},"93.0","escapes",[67,1185,1186],{"x":1178,"y":1155,"style":96},"_x000D_ → \\r",[208,1188],{"x1":1189,"y1":1160,"x2":1190,"y2":1160,"stroke":64,"style":1131},"291.4","313.4",[1163,1192],{"points":1193,"fill":1166},"317.4,71.2 308.4,76.2 308.4,66.2",[44,1195],{"x":1196,"y":54,"width":1128,"height":1129,"rx":1130,"fill":63,"stroke":64,"style":1131},"322.4",[1133,1198,1200],{"transform":1199},"translate(368.0,29.0) scale(1.000)",[1137,1201],{"d":1202,"fill":1140,"stroke":1141,"style":1142},"M20 11a8 8 0 1 0-2.3 5.7 M20 4v7h-7",[67,1204,1206],{"x":1205,"y":1146,"style":1147},"380.0","Normalise",[67,1208,1209],{"x":1205,"y":1182,"style":1147},"breaks",[67,1211,1212],{"x":1205,"y":1155,"style":96},"\\r\\n → \\n",[208,1214],{"x1":1215,"y1":1160,"x2":1216,"y2":1160,"stroke":64,"style":1131},"442.6","464.6",[1163,1218],{"points":1219,"fill":1166},"468.6,71.2 459.6,76.2 459.6,66.2",[44,1221],{"x":1222,"y":54,"width":1128,"height":1129,"rx":1130,"fill":1223,"stroke":1224,"style":1131},"473.6","#d9f4f1","var(--teal,#0f9488)",[1133,1226,1228],{"transform":1227},"translate(519.2,29.0) scale(1.000)",[1137,1229],{"d":1230,"fill":1140,"stroke":1231,"style":1142},"M3 5h18l-7 8v6l-4 2v-8z","var(--teal-ink,#0b6157)",[67,1233,1236],{"x":1234,"y":1146,"style":1235},"531.2","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Remove Cc",[67,1238,1239],{"x":1234,"y":1182,"style":1235},"and Cf",[67,1241,1242],{"x":1234,"y":1155,"style":96},"keep allow list",[208,1244],{"x1":1245,"y1":1160,"x2":1246,"y2":1160,"stroke":64,"style":1131},"593.8","615.8",[1163,1248],{"points":1249,"fill":1166},"619.8,71.2 610.8,76.2 610.8,66.2",[44,1251],{"x":1252,"y":54,"width":1128,"height":1129,"rx":1130,"fill":1253,"stroke":1254,"style":1131},"624.8","#fdefd8","var(--gold,#b4740a)",[1133,1256,1258],{"transform":1257},"translate(670.4,37.8) scale(1.000)",[1137,1259],{"d":1260,"fill":1140,"stroke":1261,"style":1142},"M12 3.5l9.5 17h-19z M12 10v5 M12 17.8v.4","var(--gold-ink,#7a4e06)",[67,1263,1267],{"x":1264,"y":1265,"style":1266},"682.4","84.2","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Flag U+FFFD",[67,1269,1271],{"x":1264,"y":1270,"style":96},"103.8","lost upstream",[191,1273,1275],{"id":1274},"step-3-decide-what-a-line-break-means","Step 3 — Decide what a line break means",[10,1277,1278,1279,1282,1283,307,1285,810,1288,1290],{},"A line break typed with Alt+Enter is intentional in an address or a comment, and wrong in a product code. Normalise every variant to ",[203,1280,1281],{},"\\n"," first — Excel files contain ",[203,1284,356],{},[203,1286,1287],{},"\\r",[203,1289,1281],{}," depending on where the text came from — then decide per column:",[196,1292,1294],{"className":235,"code":1293,"language":237,"meta":201,"style":201},"def normalise_breaks(text):\n    return re.sub(r\"\\r\\n?\", \"\\n\", text) if isinstance(text, str) else text\n\ndf[\"note\"] = df[\"note\"].map(normalise_breaks)\nsingle_line = df[\"note\"].str.replace(r\"\\s*\\n\\s*\", \" \", regex=True)\nprint(repr(df[\"note\"][0]), \"|\", single_line[0])\n# 'Line one\\nLine two' | Line one Line two\n",[203,1295,1296,1305,1349,1353,1370,1419,1449],{"__ignoreMap":201},[206,1297,1298,1300,1303],{"class":208,"line":209},[206,1299,533],{"class":244},[206,1301,1302],{"class":536}," normalise_breaks",[206,1304,997],{"class":248},[206,1306,1307,1309,1312,1314,1316,1319,1322,1324,1326,1328,1330,1332,1335,1337,1339,1341,1343,1345,1347],{"class":208,"line":252},[206,1308,770],{"class":244},[206,1310,1311],{"class":248}," re.sub(",[206,1313,960],{"class":244},[206,1315,322],{"class":216},[206,1317,356],{"class":1318},"s_b0D",[206,1320,1321],{"class":244},"?",[206,1323,322],{"class":216},[206,1325,307],{"class":248},[206,1327,322],{"class":216},[206,1329,1281],{"class":313},[206,1331,322],{"class":216},[206,1333,1334],{"class":248},", text) ",[206,1336,1039],{"class":244},[206,1338,1042],{"class":421},[206,1340,1045],{"class":248},[206,1342,609],{"class":421},[206,1344,717],{"class":248},[206,1346,1052],{"class":244},[206,1348,1055],{"class":248},[206,1350,1351],{"class":208,"line":260},[206,1352,264],{"emptyLinePlaceholder":263},[206,1354,1355,1357,1359,1361,1363,1365,1367],{"class":208,"line":267},[206,1356,1064],{"class":248},[206,1358,890],{"class":216},[206,1360,1069],{"class":248},[206,1362,292],{"class":244},[206,1364,1074],{"class":248},[206,1366,890],{"class":216},[206,1368,1369],{"class":248},"].map(normalise_breaks)\n",[206,1371,1372,1375,1377,1379,1381,1384,1386,1388,1391,1394,1396,1398,1400,1402,1404,1407,1409,1412,1414,1417],{"class":208,"line":281},[206,1373,1374],{"class":248},"single_line ",[206,1376,292],{"class":244},[206,1378,1074],{"class":248},[206,1380,890],{"class":216},[206,1382,1383],{"class":248},"].str.replace(",[206,1385,960],{"class":244},[206,1387,322],{"class":216},[206,1389,1390],{"class":421},"\\s",[206,1392,1393],{"class":244},"*",[206,1395,1281],{"class":1318},[206,1397,1390],{"class":421},[206,1399,1393],{"class":244},[206,1401,322],{"class":216},[206,1403,307],{"class":248},[206,1405,1406],{"class":216},"\" \"",[206,1408,307],{"class":248},[206,1410,1411],{"class":415},"regex",[206,1413,292],{"class":244},[206,1415,1416],{"class":421},"True",[206,1418,435],{"class":248},[206,1420,1421,1423,1425,1427,1429,1431,1433,1435,1438,1441,1444,1446],{"class":208,"line":286},[206,1422,456],{"class":421},[206,1424,876],{"class":248},[206,1426,879],{"class":421},[206,1428,459],{"class":248},[206,1430,890],{"class":216},[206,1432,893],{"class":248},[206,1434,47],{"class":421},[206,1436,1437],{"class":248},"]), ",[206,1439,1440],{"class":216},"\"|\"",[206,1442,1443],{"class":248},", single_line[",[206,1445,47],{"class":421},[206,1447,1448],{"class":248},"])\n",[206,1450,1451],{"class":208,"line":345},[206,1452,1453],{"class":479},"# 'Line one\\nLine two' | Line one Line two\n",[10,1455,1456,1457,1459],{},"Keep ",[203,1458,1281],{}," in columns that will be shown in a wrapped Excel cell; replace it with a space in columns bound for CSV files, database keys or single-line labels.",[191,1461,1463],{"id":1462},"step-4-remove-control-and-format-characters","Step 4 — Remove control and format characters",[10,1465,1466],{},"With escapes decoded and breaks normalised, remove every remaining control or format character, except for a short allow list of the ones your data genuinely needs:",[196,1468,1470],{"className":235,"code":1469,"language":237,"meta":201,"style":201},"KEEP = {\"\\n\", \"\\u200d\", \"\\u200e\", \"\\u200f\"}      # line feed, zero-width joiner, direction marks\n\ndef strip_invisible(text, keep=KEEP, tab=\" \"):\n    if not isinstance(text, str):\n        return text\n    out = []\n    for ch in text:\n        if ch == \"\\t\":\n            out.append(tab)\n        elif ch in keep or unicodedata.category(ch) not in {\"Cc\", \"Cf\", \"Co\", \"Cn\"}:\n            out.append(ch)\n    return \"\".join(out)\n\ntext_cols = df.select_dtypes(include=[\"object\", \"string\"]).columns\ndf[text_cols] = df[text_cols].apply(lambda s: s.map(strip_invisible))\nprint(df[\"sku\"].tolist())        # ['AB-100', 'AB-100', 'AB-200', 'AB-300']\nprint(df[\"sku\"].nunique())       # 3\nprint(df[\"note\"].tolist())\n",[203,1471,1472,1519,1523,1546,1562,1569,1578,1589,1606,1611,1654,1659,1669,1673,1698,1713,1727,1742],{"__ignoreMap":201},[206,1473,1474,1477,1479,1481,1483,1485,1487,1489,1491,1493,1495,1497,1499,1502,1504,1506,1508,1511,1513,1516],{"class":208,"line":209},[206,1475,1476],{"class":421},"KEEP",[206,1478,500],{"class":244},[206,1480,503],{"class":248},[206,1482,322],{"class":216},[206,1484,1281],{"class":313},[206,1486,322],{"class":216},[206,1488,307],{"class":248},[206,1490,322],{"class":216},[206,1492,395],{"class":313},[206,1494,322],{"class":216},[206,1496,307],{"class":248},[206,1498,322],{"class":216},[206,1500,1501],{"class":313},"\\u200e",[206,1503,322],{"class":216},[206,1505,307],{"class":248},[206,1507,322],{"class":216},[206,1509,1510],{"class":313},"\\u200f",[206,1512,322],{"class":216},[206,1514,1515],{"class":248},"}      ",[206,1517,1518],{"class":479},"# line feed, zero-width joiner, direction marks\n",[206,1520,1521],{"class":208,"line":252},[206,1522,264],{"emptyLinePlaceholder":263},[206,1524,1525,1527,1530,1533,1535,1537,1540,1542,1544],{"class":208,"line":260},[206,1526,533],{"class":244},[206,1528,1529],{"class":536}," strip_invisible",[206,1531,1532],{"class":248},"(text, keep",[206,1534,292],{"class":244},[206,1536,1476],{"class":421},[206,1538,1539],{"class":248},", tab",[206,1541,292],{"class":244},[206,1543,1406],{"class":216},[206,1545,612],{"class":248},[206,1547,1548,1551,1554,1556,1558,1560],{"class":208,"line":267},[206,1549,1550],{"class":244},"    if",[206,1552,1553],{"class":244}," not",[206,1555,1042],{"class":421},[206,1557,1045],{"class":248},[206,1559,609],{"class":421},[206,1561,612],{"class":248},[206,1563,1564,1567],{"class":208,"line":281},[206,1565,1566],{"class":244},"        return",[206,1568,1055],{"class":248},[206,1570,1571,1574,1576],{"class":208,"line":286},[206,1572,1573],{"class":248},"    out ",[206,1575,292],{"class":244},[206,1577,550],{"class":248},[206,1579,1580,1582,1584,1586],{"class":208,"line":345},[206,1581,555],{"class":244},[206,1583,620],{"class":248},[206,1585,561],{"class":244},[206,1587,1588],{"class":248}," text:\n",[206,1590,1591,1594,1596,1598,1600,1602,1604],{"class":208,"line":404},[206,1592,1593],{"class":244},"        if",[206,1595,620],{"class":248},[206,1597,656],{"class":244},[206,1599,659],{"class":216},[206,1601,367],{"class":313},[206,1603,322],{"class":216},[206,1605,666],{"class":248},[206,1607,1608],{"class":208,"line":438},[206,1609,1610],{"class":248},"            out.append(tab)\n",[206,1612,1613,1616,1618,1620,1623,1626,1629,1632,1635,1637,1639,1641,1643,1645,1647,1649,1651],{"class":208,"line":453},[206,1614,1615],{"class":244},"        elif",[206,1617,620],{"class":248},[206,1619,561],{"class":244},[206,1621,1622],{"class":248}," keep ",[206,1624,1625],{"class":244},"or",[206,1627,1628],{"class":248}," unicodedata.category(ch) ",[206,1630,1631],{"class":244},"not",[206,1633,1634],{"class":244}," in",[206,1636,503],{"class":248},[206,1638,506],{"class":216},[206,1640,307],{"class":248},[206,1642,511],{"class":216},[206,1644,307],{"class":248},[206,1646,516],{"class":216},[206,1648,307],{"class":248},[206,1650,521],{"class":216},[206,1652,1653],{"class":248},"}:\n",[206,1655,1656],{"class":208,"line":467},[206,1657,1658],{"class":248},"            out.append(ch)\n",[206,1660,1661,1663,1666],{"class":208,"line":704},[206,1662,770],{"class":244},[206,1664,1665],{"class":216}," \"\"",[206,1667,1668],{"class":248},".join(out)\n",[206,1670,1671],{"class":208,"line":726},[206,1672,264],{"emptyLinePlaceholder":263},[206,1674,1675,1678,1680,1683,1685,1687,1689,1691,1693,1695],{"class":208,"line":767},[206,1676,1677],{"class":248},"text_cols ",[206,1679,292],{"class":244},[206,1681,1682],{"class":248}," df.select_dtypes(",[206,1684,567],{"class":415},[206,1686,292],{"class":244},[206,1688,572],{"class":248},[206,1690,575],{"class":216},[206,1692,307],{"class":248},[206,1694,580],{"class":216},[206,1696,1697],{"class":248},"]).columns\n",[206,1699,1700,1703,1705,1708,1710],{"class":208,"line":776},[206,1701,1702],{"class":248},"df[text_cols] ",[206,1704,292],{"class":244},[206,1706,1707],{"class":248}," df[text_cols].apply(",[206,1709,1010],{"class":244},[206,1711,1712],{"class":248}," s: s.map(strip_invisible))\n",[206,1714,1715,1717,1719,1721,1724],{"class":208,"line":781},[206,1716,456],{"class":421},[206,1718,459],{"class":248},[206,1720,298],{"class":216},[206,1722,1723],{"class":248},"].tolist())        ",[206,1725,1726],{"class":479},"# ['AB-100', 'AB-100', 'AB-200', 'AB-300']\n",[206,1728,1730,1732,1734,1736,1739],{"class":208,"line":1729},17,[206,1731,456],{"class":421},[206,1733,459],{"class":248},[206,1735,298],{"class":216},[206,1737,1738],{"class":248},"].nunique())       ",[206,1740,1741],{"class":479},"# 3\n",[206,1743,1745,1747,1749,1751],{"class":208,"line":1744},18,[206,1746,456],{"class":421},[206,1748,459],{"class":248},[206,1750,890],{"class":216},[206,1752,464],{"class":248},[10,1754,1755],{},"The zero-width space, byte-order mark and soft hyphen are gone, so the four codes collapse to the three real ones. Tabs become spaces rather than vanishing, so \"tab↹here\" does not turn into \"tabhere\". The zero-width joiner is kept because it binds the woman and laptop emoji into a single \"technologist\" symbol; removing it splits them in two. Direction marks are kept for text in right-to-left scripts. Adjust the allow list to your data — for purely numeric codes, keep nothing.",[191,1757,1759],{"id":1758},"step-5-report-replacement-characters-do-not-delete-them","Step 5 — Report replacement characters, do not delete them",[10,1761,1762,1765],{},[203,1763,1764],{},"U+FFFD"," (�) is not invisible; it is what a program writes when it could not decode a byte. The original character is already lost — \"café\" became \"caf�\" somewhere upstream, usually in a CSV read with the wrong encoding before it reached Excel. Deleting it produces \"caf\", which is wrong in a different way. List the affected rows instead so the source can be fixed:",[196,1767,1769],{"className":235,"code":1768,"language":237,"meta":201,"style":201},"damaged = df[df[\"note\"].str.contains(\"\\ufffd\", na=False)]\nprint(damaged.index.tolist())    # [3]\n",[203,1770,1771,1804],{"__ignoreMap":201},[206,1772,1773,1776,1778,1781,1783,1786,1788,1790,1792,1794,1797,1799,1801],{"class":208,"line":209},[206,1774,1775],{"class":248},"damaged ",[206,1777,292],{"class":244},[206,1779,1780],{"class":248}," df[df[",[206,1782,890],{"class":216},[206,1784,1785],{"class":248},"].str.contains(",[206,1787,322],{"class":216},[206,1789,389],{"class":313},[206,1791,322],{"class":216},[206,1793,307],{"class":248},[206,1795,1796],{"class":415},"na",[206,1798,292],{"class":244},[206,1800,422],{"class":421},[206,1802,1803],{"class":248},")]\n",[206,1805,1806,1808,1811],{"class":208,"line":252},[206,1807,456],{"class":421},[206,1809,1810],{"class":248},"(damaged.index.tolist())    ",[206,1812,1813],{"class":479},"# [3]\n",[10,1815,1816,1817,23],{},"Encoding problems at the CSV stage are covered in ",[14,1818,1820],{"href":1819},"\u002Fgetting-started-with-python-excel-automation\u002Fhandling-excel-file-formats-and-conversions\u002Fconvert-csv-files-to-excel-with-python\u002F","Convert CSV Files to Excel with Python",[191,1822,1824],{"id":1823},"step-6-confirm-the-result","Step 6 — Confirm the result",[10,1826,1827],{},"Run the audit again and compare. Anything left should be on your allow list — here the kept line feed and joiner, plus the reported replacement character:",[196,1829,1831],{"className":235,"code":1830,"language":237,"meta":201,"style":201},"after = audit(df)\nprint(after[[\"column\", \"character\", \"count\"]].values.tolist())\n# [['note', 'U+000A', 1], ['note', 'REPLACEMENT CHARACTER', 1], ['note', 'ZERO WIDTH JOINER', 1]]\ndf.to_excel(\"clean.xlsx\", index=False)\n",[203,1832,1833,1843,1863,1868],{"__ignoreMap":201},[206,1834,1835,1838,1840],{"class":208,"line":209},[206,1836,1837],{"class":248},"after ",[206,1839,292],{"class":244},[206,1841,1842],{"class":248}," audit(df)\n",[206,1844,1845,1847,1850,1852,1854,1856,1858,1860],{"class":208,"line":252},[206,1846,456],{"class":421},[206,1848,1849],{"class":248},"(after[[",[206,1851,738],{"class":216},[206,1853,307],{"class":248},[206,1855,744],{"class":216},[206,1857,307],{"class":248},[206,1859,750],{"class":216},[206,1861,1862],{"class":248},"]].values.tolist())\n",[206,1864,1865],{"class":208,"line":260},[206,1866,1867],{"class":479},"# [['note', 'U+000A', 1], ['note', 'REPLACEMENT CHARACTER', 1], ['note', 'ZERO WIDTH JOINER', 1]]\n",[206,1869,1870,1873,1876,1878,1880,1882,1884],{"class":208,"line":267},[206,1871,1872],{"class":248},"df.to_excel(",[206,1874,1875],{"class":216},"\"clean.xlsx\"",[206,1877,307],{"class":248},[206,1879,416],{"class":415},[206,1881,292],{"class":244},[206,1883,422],{"class":421},[206,1885,435],{"class":248},[10,1887,1888],{},"Saving the before and after audits with the output gives you a record of exactly what changed, which is useful when someone asks why a code in the cleaned file differs from the source.",[25,1890,34,1896,34,1899,34,1902,34,1907,34,1911,34,1915,34,1921,34,1925,34,1929,34,1931,34,1935,34,1938,34,1941,34,1943,34,1947,34,1950,34,1954,34,1956,34,1959,34,1962,34,1965,34,1967,34,1970,34,1973,34,1977,34,1980,34,1982,34,1985,34,1987,34,1989,34,1991,34,1993,34,1995,34,1997,34,1999,34,2001,34,2003,34,2006,34,2008,34,2010,34,2012,34,2015,34,2021,34,2025,34,2030,34,2033,34,2040],{"viewBox":1891,"role":28,"ariaLabelledBy":1892,"xmlns":32,"style":1895},"132 0 495 192",[1893,1894],"np-result-t","np-result-d","width:100%;max-width:495px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[36,1897,1898],{"id":1893},"Four product codes become three",[40,1900,1901],{"id":1894},"Before cleaning, AB-100 appears twice, once with a zero-width space, AB-200 carries a byte-order mark and AB-300 contains a soft hyphen, so pandas counts four distinct codes. After removing format characters the codes are AB-100, AB-100, AB-200 and AB-300, and pandas counts three.",[44,1903],{"x":1904,"y":47,"width":1905,"height":1906,"fill":50},"132","495","192",[44,1908],{"x":1909,"y":56,"width":87,"height":1910,"rx":47,"fill":84,"stroke":57,"style":58},"148.7","24.0",[44,1912],{"x":1913,"y":56,"width":1914,"height":1910,"rx":47,"fill":84,"stroke":57,"style":58},"180.7","124.6",[67,1916,1920],{"x":1917,"y":1918,"style":1919},"243.0","57.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[44,1922],{"x":1909,"y":1923,"width":87,"height":1924,"rx":47,"fill":84,"stroke":57,"style":58},"64.0","28.0",[67,1926,1027],{"x":1927,"y":1928,"style":1919},"164.7","83.0",[44,1930],{"x":1913,"y":1923,"width":1914,"height":1924,"rx":47,"fill":50,"stroke":57,"style":58},[67,1932,1934],{"x":1917,"y":1928,"style":1933},"font-size:12px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","AB-100",[44,1936],{"x":1909,"y":1937,"width":87,"height":1924,"rx":47,"fill":84,"stroke":57,"style":58},"92.0",[67,1939,1940],{"x":1927,"y":114,"style":1919},"2",[44,1942],{"x":1913,"y":1937,"width":1914,"height":1924,"rx":47,"fill":50,"stroke":57,"style":58},[67,1944,1946],{"x":1917,"y":114,"style":1945},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","AB·-100 (ZWSP)",[44,1948],{"x":1909,"y":1949,"width":87,"height":1924,"rx":47,"fill":84,"stroke":57,"style":58},"120.0",[67,1951,1953],{"x":1927,"y":1952,"style":1919},"139.0","3",[44,1955],{"x":1913,"y":1949,"width":1914,"height":1924,"rx":47,"fill":50,"stroke":57,"style":58},[67,1957,1958],{"x":1917,"y":1952,"style":1945},"·AB-200 (BOM)",[44,1960],{"x":1909,"y":1961,"width":87,"height":1924,"rx":47,"fill":84,"stroke":57,"style":58},"148.0",[67,1963,46],{"x":1927,"y":1964,"style":1919},"167.0",[44,1966],{"x":1913,"y":1961,"width":1914,"height":1924,"rx":47,"fill":50,"stroke":57,"style":58},[67,1968,1969],{"x":1917,"y":1964,"style":1945},"AB-3·00 (SHY)",[44,1971],{"x":1972,"y":56,"width":87,"height":1910,"rx":47,"fill":84,"stroke":57,"style":58},"506.6",[44,1974],{"x":1975,"y":56,"width":1976,"height":1910,"rx":47,"fill":84,"stroke":57,"style":58},"538.6","72.8",[67,1978,1920],{"x":1979,"y":1918,"style":1919},"575.0",[44,1981],{"x":1972,"y":1923,"width":87,"height":1924,"rx":47,"fill":84,"stroke":57,"style":58},[67,1983,1027],{"x":1984,"y":1928,"style":1919},"522.6",[44,1986],{"x":1975,"y":1923,"width":1976,"height":1924,"rx":47,"fill":50,"stroke":57,"style":58},[67,1988,1934],{"x":1979,"y":1928,"style":1933},[44,1990],{"x":1972,"y":1937,"width":87,"height":1924,"rx":47,"fill":84,"stroke":57,"style":58},[67,1992,1940],{"x":1984,"y":114,"style":1919},[44,1994],{"x":1975,"y":1937,"width":1976,"height":1924,"rx":47,"fill":50,"stroke":57,"style":58},[67,1996,1934],{"x":1979,"y":114,"style":1945},[44,1998],{"x":1972,"y":1949,"width":87,"height":1924,"rx":47,"fill":84,"stroke":57,"style":58},[67,2000,1953],{"x":1984,"y":1952,"style":1919},[44,2002],{"x":1975,"y":1949,"width":1976,"height":1924,"rx":47,"fill":50,"stroke":57,"style":58},[67,2004,2005],{"x":1979,"y":1952,"style":1945},"AB-200",[44,2007],{"x":1972,"y":1961,"width":87,"height":1924,"rx":47,"fill":84,"stroke":57,"style":58},[67,2009,46],{"x":1984,"y":1964,"style":1919},[44,2011],{"x":1975,"y":1961,"width":1976,"height":1924,"rx":47,"fill":50,"stroke":57,"style":58},[67,2013,2014],{"x":1979,"y":1964,"style":1945},"AB-300",[67,2016,2020],{"x":2017,"y":2018,"style":2019},"227.0","29.0","font-size:12.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","before",[67,2022,2024],{"x":2023,"y":2018,"style":2019},"559.0","after",[208,2026],{"x1":2027,"y1":2028,"x2":2029,"y2":2028,"stroke":64,"style":1131},"315.3","108.0","492.6",[1163,2031],{"points":2032,"fill":1166},"496.6,108.0 487.6,113.0 487.6,103.0",[44,2034],{"x":2035,"y":2036,"width":2037,"height":2038,"rx":2039,"fill":50,"stroke":57,"style":65},"341.4","75.0","129.0","22.0","11",[67,2041,2044],{"x":2042,"y":105,"style":2043},"405.9","font-size:11.5px;font-weight:400;fill:var(--brand-strong,#4338ca);text-anchor:middle;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace","strip_invisible",[191,2046,2048],{"id":2047},"step-7-clean-every-sheet-in-one-pass","Step 7 — Clean every sheet in one pass",[10,2050,2051],{},"Invisible characters rarely stay in one column. Wrap the steps in a single function and apply it to every text column of every sheet, so the whole workbook is treated consistently:",[196,2053,2055],{"className":235,"code":2054,"language":237,"meta":201,"style":201},"def clean_text(value):\n    return strip_invisible(normalise_breaks(decode_escapes(value)))\n\nsheets = pd.read_excel(\"invisible.xlsx\", sheet_name=None)\nwith pd.ExcelWriter(\"invisible_clean.xlsx\") as writer:\n    for name, frame in sheets.items():\n        cols = frame.select_dtypes(include=[\"object\", \"string\"]).columns\n        frame[cols] = frame[cols].apply(lambda s: s.map(clean_text))\n        frame.to_excel(writer, sheet_name=name, index=False)\nprint(pd.read_excel(\"invisible_clean.xlsx\")[\"sku\"].nunique())   # 3\n",[203,2056,2057,2067,2074,2078,2101,2119,2131,2154,2169,2189],{"__ignoreMap":201},[206,2058,2059,2061,2064],{"class":208,"line":209},[206,2060,533],{"class":244},[206,2062,2063],{"class":536}," clean_text",[206,2065,2066],{"class":248},"(value):\n",[206,2068,2069,2071],{"class":208,"line":252},[206,2070,770],{"class":244},[206,2072,2073],{"class":248}," strip_invisible(normalise_breaks(decode_escapes(value)))\n",[206,2075,2076],{"class":208,"line":260},[206,2077,264],{"emptyLinePlaceholder":263},[206,2079,2080,2083,2085,2087,2089,2091,2094,2096,2099],{"class":208,"line":267},[206,2081,2082],{"class":248},"sheets ",[206,2084,292],{"class":244},[206,2086,446],{"class":248},[206,2088,410],{"class":216},[206,2090,307],{"class":248},[206,2092,2093],{"class":415},"sheet_name",[206,2095,292],{"class":244},[206,2097,2098],{"class":421},"None",[206,2100,435],{"class":248},[206,2102,2103,2106,2109,2112,2114,2116],{"class":208,"line":281},[206,2104,2105],{"class":244},"with",[206,2107,2108],{"class":248}," pd.ExcelWriter(",[206,2110,2111],{"class":216},"\"invisible_clean.xlsx\"",[206,2113,717],{"class":248},[206,2115,275],{"class":244},[206,2117,2118],{"class":248}," writer:\n",[206,2120,2121,2123,2126,2128],{"class":208,"line":286},[206,2122,555],{"class":244},[206,2124,2125],{"class":248}," name, frame ",[206,2127,561],{"class":244},[206,2129,2130],{"class":248}," sheets.items():\n",[206,2132,2133,2136,2138,2140,2142,2144,2146,2148,2150,2152],{"class":208,"line":345},[206,2134,2135],{"class":248},"        cols ",[206,2137,292],{"class":244},[206,2139,564],{"class":248},[206,2141,567],{"class":415},[206,2143,292],{"class":244},[206,2145,572],{"class":248},[206,2147,575],{"class":216},[206,2149,307],{"class":248},[206,2151,580],{"class":216},[206,2153,1697],{"class":248},[206,2155,2156,2159,2161,2164,2166],{"class":208,"line":404},[206,2157,2158],{"class":248},"        frame[cols] ",[206,2160,292],{"class":244},[206,2162,2163],{"class":248}," frame[cols].apply(",[206,2165,1010],{"class":244},[206,2167,2168],{"class":248}," s: s.map(clean_text))\n",[206,2170,2171,2174,2176,2178,2181,2183,2185,2187],{"class":208,"line":438},[206,2172,2173],{"class":248},"        frame.to_excel(writer, ",[206,2175,2093],{"class":415},[206,2177,292],{"class":244},[206,2179,2180],{"class":248},"name, ",[206,2182,416],{"class":415},[206,2184,292],{"class":244},[206,2186,422],{"class":421},[206,2188,435],{"class":248},[206,2190,2191,2193,2195,2197,2199,2201,2204],{"class":208,"line":453},[206,2192,456],{"class":421},[206,2194,882],{"class":248},[206,2196,2111],{"class":216},[206,2198,887],{"class":248},[206,2200,298],{"class":216},[206,2202,2203],{"class":248},"].nunique())   ",[206,2205,1741],{"class":479},[10,2207,2208,2209,2212],{},"The order inside ",[203,2210,2211],{},"clean_text"," is the order of the steps above, and it matters: escapes must become characters before line breaks can be normalised, and breaks must be normalised before the stripping step decides which characters to keep. Selecting only text columns leaves numbers and dates untouched, so the cleaning cannot change a figure.",[191,2214,2216],{"id":2215},"common-pitfalls","Common pitfalls",[2218,2219,2220,2228,2234,2243],"ul",{},[2221,2222,2223,2227],"li",{},[2224,2225,2226],"strong",{},"Relying on Excel's CLEAN."," It removes ASCII codes 0–31 only; the soft hyphen, zero-width space and byte-order mark survive it.",[2221,2229,2230,2233],{},[2224,2231,2232],{},"Deleting all Cf characters."," That breaks emoji sequences, flags and right-to-left text. Use an allow list.",[2221,2235,2236,2239,2240,2242],{},[2224,2237,2238],{},"Cleaning before decoding."," If you strip characters before decoding ",[203,2241,837],{},", the escape text survives as letters and appears in the report.",[2221,2244,2245,2248,2249,2252,2253,23],{},[2224,2246,2247],{},"Writing control characters back."," openpyxl refuses most of them with ",[203,2250,2251],{},"IllegalCharacterError","; clean first, as described in ",[14,2254,2256],{"href":2255},"\u002Fgetting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002Ffix-illegalcharactererror-when-writing-excel-with-openpyxl\u002F","Fix IllegalCharacterError When Writing Excel with openpyxl",[191,2258,2260],{"id":2259},"conclusion","Conclusion",[10,2262,2263,2264,2266],{},"Invisible characters make identical-looking values unequal and break exports. Audit text columns by Unicode category first, decode ",[203,2265,856],{}," escapes (or read with calamine), normalise line breaks and decide per column whether to keep them, then remove control and format characters with an allow list for joiners and direction marks. Report replacement characters instead of deleting them, and re-run the audit to confirm what is left is intentional.",[191,2268,2270],{"id":2269},"frequently-asked-questions","Frequently asked questions",[10,2272,2273,2279,2280,2282],{},[2224,2274,2275,2276,2278],{},"Why do I see ",[847,2277,849],{}," in text read from Excel?","\nExcel stores a carriage return inside a cell as the escape sequence ",[847,2281,849],{},". The calamine engine decodes it back to a character, but openpyxl — the default engine for pandas.read_excel — leaves the escape in the text. Decode it with a regular expression or read with engine=\"calamine\".",[10,2284,2285,2288],{},[2224,2286,2287],{},"What counts as a non-printable character?","\nIn Unicode terms, control characters (category Cc, such as tab, carriage return and bell) and format characters (category Cf, such as the soft hyphen, zero-width space and byte-order mark). They take no visible space but still make two strings unequal.",[10,2290,2291,2294],{},[2224,2292,2293],{},"Will removing format characters break emoji?","\nIt can. Some emoji are joined with a zero-width joiner (U+200D), and some scripts need direction marks. Keep an allow list of the format characters your data legitimately uses.",[10,2296,2297,2300],{},[2224,2298,2299],{},"Is Excel's CLEAN function the same as this?","\nCLEAN removes only the first 32 ASCII control codes. It leaves soft hyphens, zero-width characters, the byte-order mark and other Unicode format characters in place, which is why Python-side cleaning based on Unicode categories is more thorough.",[191,2302,2304],{"id":2303},"related","Related",[2218,2306,2307,2313,2318,2325],{},[2221,2308,2309,2310,2312],{},"Up one level: ",[14,2311,17],{"href":16}," — the full cleaning toolkit.",[2221,2314,2315,2317],{},[14,2316,22],{"href":21}," — spaces, case and Unicode normalisation.",[2221,2319,2320,2324],{},[14,2321,2323],{"href":2322},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fpandas-drop-duplicates-from-excel-column\u002F","Pandas: Drop Duplicates From an Excel Column"," — deduplicating once hidden differences are gone.",[2221,2326,2327,2331],{},[14,2328,2330],{"href":2329},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-excel-column-names-with-pandas\u002F","Standardise Excel Column Names with pandas"," — the same cleaning for headers.",[2333,2334,2335],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .s_b0D, html code.shiki .s_b0D{--shiki-default:#22863A;--shiki-default-font-weight:bold;--shiki-dark:#72F088;--shiki-dark-font-weight:bold}",{"title":201,"searchDepth":252,"depth":252,"links":2337},[2338,2339,2340,2342,2343,2344,2345,2346,2347,2348,2349,2350],{"id":193,"depth":252,"text":194},{"id":483,"depth":252,"text":484},{"id":844,"depth":252,"text":2341},"Step 2 — Decode the x000D escapes",{"id":1274,"depth":252,"text":1275},{"id":1462,"depth":252,"text":1463},{"id":1758,"depth":252,"text":1759},{"id":1823,"depth":252,"text":1824},{"id":2047,"depth":252,"text":2048},{"id":2215,"depth":252,"text":2216},{"id":2259,"depth":252,"text":2260},{"id":2269,"depth":252,"text":2270},{"id":2303,"depth":252,"text":2304},"2026-09-23","Find and remove the characters you cannot see in Excel text — carriage returns, tabs, bell and other control codes, soft hyphens, zero-width marks and leftover _x000D_ escapes — using Unicode categories, without damaging line breaks or emoji you want to keep.","md",[2355,2358,2360,2362],{"q":2356,"a":2357},"Why do I see _x000D_ in text read from Excel?","Excel stores a carriage return inside a cell as the escape sequence _x000D_. The calamine engine decodes it back to a character, but openpyxl — the default engine for pandas.read_excel — leaves the escape in the text. Decode it with a regular expression or read with engine=\"calamine\".",{"q":2287,"a":2359},"In Unicode terms, control characters (category Cc, such as tab, carriage return and bell) and format characters (category Cf, such as the soft hyphen, zero-width space and byte-order mark). They take no visible space but still make two strings unequal.",{"q":2293,"a":2361},"It can. Some emoji are joined with a zero-width joiner (U+200D), and some scripts need direction marks. Keep an allow list of the format characters your data legitimately uses.",{"q":2299,"a":2363},"CLEAN removes only the first 32 ASCII control codes. It leaves soft hyphens, zero-width characters, the byte-order mark and other Unicode format characters in place, which is why Python-side cleaning based on Unicode categories is more thorough.","remove non-printable characters from excel data with python",null,{"breadcrumb":2367},[2368,2371,2374],{"name":2369,"item":2370},"Home","\u002F",{"name":2372,"item":2373},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":2375,"item":16},"Cleaning Excel Data With Pandas: A Production Workflow","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-non-printable-characters-from-excel-data",{"title":5,"description":2378},"Remove invisible characters from Excel data with pandas — audit by Unicode category, decode _x000D_ escapes openpyxl leaves behind, normalise line breaks, strip control and format characters and flag U+FFFD.","remove-non-printable-characters-from-excel-data","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-non-printable-characters-from-excel-data\u002Findex",[2382,2383],"data-cleaning","pandas","how-to","RFNoVTZr-m1VN1Q8Z4rhaf8Ft1S0IC7etsR8WSzteCI",[2387,2391],{"title":2388,"path":2389,"stem":2390,"children":-1},"Remove Duplicate Words and Extra Punctuation from Text","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-duplicate-words-and-extra-punctuation-from-text","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-duplicate-words-and-extra-punctuation-from-text\u002Findex",{"title":2392,"path":2393,"stem":2394,"children":-1},"Reorder Columns to Match a Template Layout with pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Freorder-columns-to-match-a-template-layout-with-pandas","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Freorder-columns-to-match-a-template-layout-with-pandas\u002Findex",[2396,2398],{"path":2397,"title":2375},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas",{"path":2399,"title":2372},"\u002Fadvanced-data-transformation-and-cleaning",[],[2402,2405,2408,2411,2414,2417],{"path":2403,"title":2404,"kind":2384},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Ffind-duplicate-rows-in-excel-with-python","Find Duplicate Rows in Excel with Python",{"path":2406,"title":2407,"kind":2384},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-duplicate-values-in-excel-with-openpyxl","Highlight Duplicate Values in Excel with openpyxl",{"path":2409,"title":2410,"kind":2384},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fclean-mixed-currency-amounts-into-amount-and-currency-columns","Clean Mixed-Currency Amounts into Amount and Currency Columns",{"path":2412,"title":2413,"kind":2384},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fcoalesce-several-excel-columns-into-one-with-pandas","Coalesce Several Excel Columns into One with pandas",{"path":2415,"title":2416,"kind":2384},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-currency-strings-to-numbers-with-pandas","Convert Currency Strings Like \"$1,234.56\" and \"(500)\" to Numbers with pandas",{"path":2418,"title":2419,"kind":2384},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-excel-text-columns-to-numbers-with-pandas","Convert Excel Text Columns to Numbers with pandas",1790972085489]