[{"data":1,"prerenderedAt":2068},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-garbled-characters-in-excel-data":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-garbled-characters-in-excel-data":2034,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-garbled-characters-in-excel-data":2043,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-garbled-characters-in-excel-data":2048,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-garbled-characters-in-excel-data":2049},{"id":4,"title":5,"body":6,"dateModified":1999,"datePublished":1999,"description":2000,"extension":2001,"faq":2002,"keyword":2011,"kind":2012,"meta":2013,"navigation":295,"path":2022,"seo":2023,"slug":2026,"stem":2027,"topics":2028,"type":2032,"__hash__":2033},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-garbled-characters-in-excel-data\u002Findex.md","Fix Garbled Characters Like \"Café\" and \"’\" in Excel Data",{"type":7,"value":8,"toc":1984},"minimark",[9,37,192,197,227,241,245,259,415,428,432,435,574,604,786,790,797,943,954,957,979,989,993,1021,1068,1071,1256,1260,1263,1431,1434,1438,1441,1629,1632,1636,1639,1817,1841,1845,1880,1884,1887,1891,1919,1923,1929,1935,1941,1947,1951,1980],[10,11,12,13,17,18,21,22,25,26,30,31,36],"p",{},"A customer list comes back from somewhere with names like ",[14,15,16],"code",{},"Müller GmbH",", prices like ",[14,19,20],{},"€100"," and apostrophes turned into ",[14,23,24],{},"’",". This is ",[27,28,29],"em",{},"mojibake",", and it almost always has the same cause: text that was stored as UTF-8 was, at some step, read as Windows-1252 (or Latin-1). Each accented character is two or three bytes in UTF-8; decoded one byte at a time, those bytes become two or three unrelated characters. Because the damage follows a fixed rule, it can usually be reversed exactly — with care, because the same reversal applied to text that was never damaged can damage it. This page, part of ",[32,33,35],"a",{"href":34},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002F","Cleaning Excel Data With Pandas",", repairs the damage and shows how to stop it at the source.",[38,39,47,48,47,52,47,56,47,64,47,74,47,85,47,92,47,97,47,104,47,109,47,114,47,121,47,127,47,131,47,135,47,139,47,142,47,147,47,154,47,159,47,163,47,166,47,170,47,173,47,176,47,182,47,186,47,189],"svg",{"viewBox":40,"role":41,"ariaLabelledBy":42,"xmlns":45,"style":46},"4 0 752 142","img",[43,44],"moj-cause-t","moj-cause-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[49,50,51],"title",{"id":43},"How \"é\" turns into \"é\"",[53,54,55],"desc",{"id":44},"The character é is saved as UTF-8, which encodes it as two bytes, C3 and A9. A later step reads the bytes as Windows-1252, a single-byte encoding, and turns C3 into à and A9 into ©. The text now shows é where é should be. Encoding the text back to Windows-1252 bytes and decoding as UTF-8 reverses it.",[57,58],"rect",{"x":59,"y":60,"width":61,"height":62,"fill":63},"4","0","752","142","#ffffff",[57,65],{"x":66,"y":67,"width":68,"height":69,"rx":70,"fill":71,"stroke":72,"style":73},"20.0","16.0","153.0","110.5","12","#d9f4f1","var(--teal,#0f9488)","stroke-width:2px",[75,76,78],"g",{"transform":77},"translate(84.5,37.8) scale(1.000)",[79,80],"path",{"d":81,"fill":82,"stroke":83,"style":84},"M5 12.5l4.5 4.5L19 7","none","var(--teal-ink,#0b6157)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[86,87,91],"text",{"x":88,"y":89,"style":90},"96.5","84.2","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","é",[86,93,96],{"x":88,"y":94,"style":95},"103.8","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","the original character",[98,99],"line",{"x1":100,"y1":101,"x2":102,"y2":101,"stroke":103,"style":73},"178.0","71.2","200.0","var(--brand,#5b5cf0)",[105,106],"polygon",{"points":107,"fill":108},"204.0,71.2 195.0,76.2 195.0,66.2","#5b5cf0",[57,110],{"x":111,"y":67,"width":68,"height":69,"rx":70,"fill":112,"stroke":113,"style":73},"209.0","#e7ebef","var(--line,#cdd5e6)",[75,115,117],{"transform":116},"translate(273.5,29.0) scale(1.000)",[79,118],{"d":119,"fill":82,"stroke":120,"style":84},"M6 3h9l4 4v14H6z M15 3v4h4","var(--text,#172033)",[86,122,126],{"x":123,"y":124,"style":125},"285.5","75.5","font-size:13.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","UTF-8 bytes C3",[86,128,130],{"x":123,"y":129,"style":125},"93.0","A9",[86,132,134],{"x":123,"y":133,"style":95},"112.5","saved correctly",[98,136],{"x1":137,"y1":101,"x2":138,"y2":101,"stroke":103,"style":73},"367.0","389.0",[105,140],{"points":141,"fill":108},"393.0,71.2 384.0,76.2 384.0,66.2",[57,143],{"x":144,"y":67,"width":68,"height":69,"rx":70,"fill":145,"stroke":146,"style":73},"398.0","#fee8f2","var(--accent,#f43f8f)",[75,148,150],{"transform":149},"translate(462.5,29.0) scale(1.000)",[79,151],{"d":152,"fill":82,"stroke":153,"style":84},"M12 3.5l9.5 17h-19z M12 10v5 M12 17.8v.4","var(--accent-ink,#be185d)",[86,155,158],{"x":156,"y":124,"style":157},"474.5","font-size:13.5px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","Read as cp1252",[86,160,162],{"x":156,"y":161,"style":95},"95.0","one byte = one",[86,164,165],{"x":156,"y":133,"style":95},"character",[98,167],{"x1":168,"y1":101,"x2":169,"y2":101,"stroke":103,"style":73},"556.0","578.0",[105,171],{"points":172,"fill":108},"582.0,71.2 573.0,76.2 573.0,66.2",[57,174],{"x":175,"y":67,"width":68,"height":69,"rx":70,"fill":145,"stroke":146,"style":73},"587.0",[75,177,179],{"transform":178},"translate(651.5,29.0) scale(1.000)",[79,180],{"d":181,"fill":82,"stroke":153,"style":84},"M6 6l12 12 M18 6L6 18",[86,183,185],{"x":184,"y":124,"style":157},"663.5","é",[86,187,188],{"x":184,"y":161,"style":95},"what the cell now",[86,190,191],{"x":184,"y":133,"style":95},"shows",[193,194,196],"h2",{"id":195},"prerequisites","Prerequisites",[198,199,204],"pre",{"className":200,"code":201,"language":202,"meta":203,"style":203},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl ftfy\n","bash","",[14,205,206],{"__ignoreMap":203},[207,208,210,214,218,221,224],"span",{"class":98,"line":209},1,[207,211,213],{"class":212},"sMTad","pip",[207,215,217],{"class":216},"srMev"," install",[207,219,220],{"class":216}," pandas",[207,222,223],{"class":216}," openpyxl",[207,225,226],{"class":216}," ftfy\n",[10,228,229,232,233,236,237,240],{},[14,230,231],{},"customers.xlsx"," has seven names. Six were garbled by encoding them as UTF-8 and decoding as Windows-1252 — including one containing curly quotes — and one, ",[14,234,235],{},"à is a letter",", is correct text that merely contains an ",[14,238,239],{},"Ã",".",[193,242,244],{"id":243},"step-1-find-the-cells-that-look-garbled","Step 1 — Find the cells that look garbled",[10,246,247,248,250,251,254,255,258],{},"Repair only what is broken. Mojibake from UTF-8 read as Windows-1252 leaves recognisable traces: ",[14,249,239],{}," or ",[14,252,253],{},"Â"," followed by another Latin-1 character, and the sequence ",[14,256,257],{},"â€"," that starts every garbled curly quote, dash and euro sign:",[198,260,264],{"className":261,"code":262,"language":263,"meta":203,"style":203},"language-python shiki shiki-themes github-light github-dark-high-contrast","import re\nimport pandas as pd\n\ncustomers = pd.read_excel(\"customers.xlsx\")\nMOJIBAKE = re.compile(r\"[ÃÂ][\\u0080-¿]|â€|€|�\")\ncustomers[\"looks_garbled\"] = customers[\"customer\"].str.contains(MOJIBAKE)\nprint(customers.to_string(index=False))\n","python",[14,265,266,276,290,297,315,367,394],{"__ignoreMap":203},[207,267,268,272],{"class":98,"line":209},[207,269,271],{"class":270},"s-kum","import",[207,273,275],{"class":274},"skGVy"," re\n",[207,277,279,281,284,287],{"class":98,"line":278},2,[207,280,271],{"class":270},[207,282,283],{"class":274}," pandas ",[207,285,286],{"class":270},"as",[207,288,289],{"class":274}," pd\n",[207,291,293],{"class":98,"line":292},3,[207,294,296],{"emptyLinePlaceholder":295},true,"\n",[207,298,300,303,306,309,312],{"class":98,"line":299},4,[207,301,302],{"class":274},"customers ",[207,304,305],{"class":270},"=",[207,307,308],{"class":274}," pd.read_excel(",[207,310,311],{"class":216},"\"customers.xlsx\"",[207,313,314],{"class":274},")\n",[207,316,318,322,325,328,331,334,337,341,344,347,350,353,355,357,360,362,365],{"class":98,"line":317},5,[207,319,321],{"class":320},"sP0c6","MOJIBAKE",[207,323,324],{"class":270}," =",[207,326,327],{"class":274}," re.compile(",[207,329,330],{"class":270},"r",[207,332,333],{"class":216},"\"",[207,335,336],{"class":320},"[",[207,338,340],{"class":339},"sSjpA","ÃÂ",[207,342,343],{"class":320},"][",[207,345,346],{"class":339},"\\u0080-¿",[207,348,349],{"class":320},"]",[207,351,352],{"class":270},"|",[207,354,257],{"class":216},[207,356,352],{"class":270},[207,358,359],{"class":216},"€",[207,361,352],{"class":270},[207,363,364],{"class":216},"�\"",[207,366,314],{"class":274},[207,368,370,373,376,379,381,384,387,390,392],{"class":98,"line":369},6,[207,371,372],{"class":274},"customers[",[207,374,375],{"class":216},"\"looks_garbled\"",[207,377,378],{"class":274},"] ",[207,380,305],{"class":270},[207,382,383],{"class":274}," customers[",[207,385,386],{"class":216},"\"customer\"",[207,388,389],{"class":274},"].str.contains(",[207,391,321],{"class":320},[207,393,314],{"class":274},[207,395,397,400,403,407,409,412],{"class":98,"line":396},7,[207,398,399],{"class":320},"print",[207,401,402],{"class":274},"(customers.to_string(",[207,404,406],{"class":405},"sa561","index",[207,408,305],{"class":270},[207,410,411],{"class":320},"False",[207,413,414],{"class":274},"))\n",[10,416,417,418,420,421,423,424,427],{},"Six rows are flagged. ",[14,419,235],{}," is not: its ",[14,422,239],{}," is followed by a space, which is not a continuation byte. The pattern also catches the replacement character ",[14,425,426],{},"�",", which marks bytes that were lost entirely — Step 4 deals with those.",[193,429,431],{"id":430},"step-2-reverse-the-wrong-decoding","Step 2 — Reverse the wrong decoding",[10,433,434],{},"The repair is the damage run backwards: turn the text back into the bytes it was wrongly decoded from, then decode those bytes as UTF-8. If either step fails, the cell was not simple mojibake and is left alone:",[198,436,438],{"className":261,"code":437,"language":263,"meta":203,"style":203},"def repair(text):\n    try:\n        return text.encode(\"cp1252\").decode(\"utf-8\")\n    except (UnicodeEncodeError, UnicodeDecodeError):\n        return text\n\ncustomers[\"repaired\"] = customers[\"customer\"].where(~customers[\"looks_garbled\"],\n                                                    customers[\"customer\"].map(repair))\nprint(customers[[\"customer\", \"repaired\"]].to_string(index=False))\n",[14,439,440,452,460,479,499,506,510,538,549],{"__ignoreMap":203},[207,441,442,445,449],{"class":98,"line":209},[207,443,444],{"class":270},"def",[207,446,448],{"class":447},"s_Opv"," repair",[207,450,451],{"class":274},"(text):\n",[207,453,454,457],{"class":98,"line":278},[207,455,456],{"class":270},"    try",[207,458,459],{"class":274},":\n",[207,461,462,465,468,471,474,477],{"class":98,"line":292},[207,463,464],{"class":270},"        return",[207,466,467],{"class":274}," text.encode(",[207,469,470],{"class":216},"\"cp1252\"",[207,472,473],{"class":274},").decode(",[207,475,476],{"class":216},"\"utf-8\"",[207,478,314],{"class":274},[207,480,481,484,487,490,493,496],{"class":98,"line":299},[207,482,483],{"class":270},"    except",[207,485,486],{"class":274}," (",[207,488,489],{"class":320},"UnicodeEncodeError",[207,491,492],{"class":274},", ",[207,494,495],{"class":320},"UnicodeDecodeError",[207,497,498],{"class":274},"):\n",[207,500,501,503],{"class":98,"line":317},[207,502,464],{"class":270},[207,504,505],{"class":274}," text\n",[207,507,508],{"class":98,"line":369},[207,509,296],{"emptyLinePlaceholder":295},[207,511,512,514,517,519,521,523,525,528,531,533,535],{"class":98,"line":396},[207,513,372],{"class":274},[207,515,516],{"class":216},"\"repaired\"",[207,518,378],{"class":274},[207,520,305],{"class":270},[207,522,383],{"class":274},[207,524,386],{"class":216},[207,526,527],{"class":274},"].where(",[207,529,530],{"class":270},"~",[207,532,372],{"class":274},[207,534,375],{"class":216},[207,536,537],{"class":274},"],\n",[207,539,541,544,546],{"class":98,"line":540},8,[207,542,543],{"class":274},"                                                    customers[",[207,545,386],{"class":216},[207,547,548],{"class":274},"].map(repair))\n",[207,550,552,554,557,559,561,563,566,568,570,572],{"class":98,"line":551},9,[207,553,399],{"class":320},[207,555,556],{"class":274},"(customers[[",[207,558,386],{"class":216},[207,560,492],{"class":274},[207,562,516],{"class":216},[207,564,565],{"class":274},"]].to_string(",[207,567,406],{"class":405},[207,569,305],{"class":270},[207,571,411],{"class":320},[207,573,414],{"class":274},[10,575,576,577,492,580,492,583,492,586,492,589,592,593,595,596,599,600,603],{},"Five of the six flagged names come back exactly: ",[14,578,579],{},"Café Rouge",[14,581,582],{},"Müller GmbH",[14,584,585],{},"naïve – plan",[14,587,588],{},"€100 voucher",[14,590,591],{},"São Paulo",". The sixth, the quote, does not repair — it contains a ",[14,594,426],{},", and ",[14,597,598],{},"encode(\"cp1252\")"," cannot turn that back into a byte. Applying ",[14,601,602],{},"repair"," only to flagged rows matters: it is exact on mojibake, but on some correct text it can succeed and produce nonsense, so it should never be run on a whole column blindly.",[38,605,47,611,47,614,47,617,47,622,47,629,47,633,47,639,47,643,47,648,47,650,47,654,47,657,47,661,47,664,47,668,47,671,47,675,47,677,47,679,47,682,47,685,47,687,47,690,47,693,47,697,47,699,47,702,47,705,47,709,47,712,47,714,47,717,47,719,47,721,47,723,47,725,47,727,47,730,47,732,47,734,47,736,47,738,47,740,47,742,47,744,47,746,47,748,47,750,47,752,47,754,47,760,47,764,47,769,47,772,47,780],{"viewBox":606,"role":41,"ariaLabelledBy":607,"xmlns":45,"style":610},"147 0 466 220",[608,609],"moj-fix-t","moj-fix-d","width:100%;max-width:466px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[49,612,613],{"id":608},"Garbled names before and after repair",[53,615,616],{"id":609},"Left: Café Rouge, Müller GmbH, €100 voucher and São Paulo as they arrived. Right: the same names after encoding to Windows-1252 bytes and decoding as UTF-8: Café Rouge, Müller GmbH, €100 voucher and São Paulo.",[57,618],{"x":619,"y":60,"width":620,"height":621,"fill":63},"147","466","220",[57,623],{"x":624,"y":625,"width":626,"height":627,"rx":60,"fill":112,"stroke":113,"style":628},"163.1","40.0","32.0","24.0","stroke-width:1px",[57,630],{"x":631,"y":625,"width":632,"height":627,"rx":60,"fill":112,"stroke":113,"style":628},"195.1","123.1",[86,634,638],{"x":635,"y":636,"style":637},"256.6","57.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[57,640],{"x":624,"y":641,"width":626,"height":642,"rx":60,"fill":112,"stroke":113,"style":628},"64.0","28.0",[86,644,647],{"x":645,"y":646,"style":637},"179.1","83.0","1",[57,649],{"x":631,"y":641,"width":632,"height":642,"rx":60,"fill":63,"stroke":113,"style":628},[86,651,653],{"x":635,"y":646,"style":652},"font-size:12px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","customer",[57,655],{"x":624,"y":656,"width":626,"height":642,"rx":60,"fill":112,"stroke":113,"style":628},"92.0",[86,658,660],{"x":645,"y":659,"style":637},"111.0","2",[57,662],{"x":631,"y":656,"width":632,"height":642,"rx":60,"fill":145,"stroke":146,"style":663},"stroke-width:1.5px",[86,665,667],{"x":635,"y":659,"style":666},"font-size:12px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","Café Rouge",[57,669],{"x":624,"y":670,"width":626,"height":642,"rx":60,"fill":112,"stroke":113,"style":628},"120.0",[86,672,674],{"x":645,"y":673,"style":637},"139.0","3",[57,676],{"x":631,"y":670,"width":632,"height":642,"rx":60,"fill":145,"stroke":146,"style":663},[86,678,16],{"x":635,"y":673,"style":666},[57,680],{"x":624,"y":681,"width":626,"height":642,"rx":60,"fill":112,"stroke":113,"style":628},"148.0",[86,683,59],{"x":645,"y":684,"style":637},"167.0",[57,686],{"x":631,"y":681,"width":632,"height":642,"rx":60,"fill":145,"stroke":146,"style":663},[86,688,689],{"x":635,"y":684,"style":666},"€100 voucher",[57,691],{"x":624,"y":692,"width":626,"height":642,"rx":60,"fill":112,"stroke":113,"style":628},"176.0",[86,694,696],{"x":645,"y":695,"style":637},"195.0","5",[57,698],{"x":631,"y":692,"width":632,"height":642,"rx":60,"fill":145,"stroke":146,"style":663},[86,700,701],{"x":635,"y":695,"style":666},"São Paulo",[57,703],{"x":704,"y":625,"width":626,"height":627,"rx":60,"fill":112,"stroke":113,"style":628},"455.9",[57,706],{"x":707,"y":625,"width":708,"height":627,"rx":60,"fill":112,"stroke":113,"style":628},"487.9","109.1",[86,710,638],{"x":711,"y":636,"style":637},"542.4",[57,713],{"x":704,"y":641,"width":626,"height":642,"rx":60,"fill":112,"stroke":113,"style":628},[86,715,647],{"x":716,"y":646,"style":637},"471.9",[57,718],{"x":707,"y":641,"width":708,"height":642,"rx":60,"fill":63,"stroke":113,"style":628},[86,720,653],{"x":711,"y":646,"style":652},[57,722],{"x":704,"y":656,"width":626,"height":642,"rx":60,"fill":112,"stroke":113,"style":628},[86,724,660],{"x":716,"y":659,"style":637},[57,726],{"x":707,"y":656,"width":708,"height":642,"rx":60,"fill":71,"stroke":72,"style":663},[86,728,579],{"x":711,"y":659,"style":729},"font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle",[57,731],{"x":704,"y":670,"width":626,"height":642,"rx":60,"fill":112,"stroke":113,"style":628},[86,733,674],{"x":716,"y":673,"style":637},[57,735],{"x":707,"y":670,"width":708,"height":642,"rx":60,"fill":71,"stroke":72,"style":663},[86,737,582],{"x":711,"y":673,"style":729},[57,739],{"x":704,"y":681,"width":626,"height":642,"rx":60,"fill":112,"stroke":113,"style":628},[86,741,59],{"x":716,"y":684,"style":637},[57,743],{"x":707,"y":681,"width":708,"height":642,"rx":60,"fill":71,"stroke":72,"style":663},[86,745,588],{"x":711,"y":684,"style":729},[57,747],{"x":704,"y":692,"width":626,"height":642,"rx":60,"fill":112,"stroke":113,"style":628},[86,749,696],{"x":716,"y":695,"style":637},[57,751],{"x":707,"y":692,"width":708,"height":642,"rx":60,"fill":71,"stroke":72,"style":663},[86,753,591],{"x":711,"y":695,"style":729},[86,755,759],{"x":756,"y":757,"style":758},"240.6","29.0","font-size:12.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","as received",[86,761,763],{"x":762,"y":757,"style":758},"526.4","repaired",[98,765],{"x1":766,"y1":767,"x2":768,"y2":767,"stroke":103,"style":73},"328.2","122.0","441.9",[105,770],{"points":771,"fill":108},"445.9,122.0 436.9,127.0 436.9,117.0",[57,773],{"x":774,"y":775,"width":776,"height":777,"rx":778,"fill":63,"stroke":113,"style":779},"347.9","89.0","78.2","22.0","11","stroke-width:1.2px",[86,781,785],{"x":782,"y":783,"style":784},"387.0","104.0","font-size:11.5px;font-weight:400;fill:var(--brand-strong,#4338ca);text-anchor:middle;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace","repair()",[193,787,789],{"id":788},"step-3-use-ftfy-for-the-harder-cases-with-its-options-set","Step 3 — Use ftfy for the harder cases, with its options set",[10,791,792,793,796],{},"The ",[14,794,795],{},"ftfy"," library knows many more mojibake patterns — text garbled twice, mixtures of encodings, Latin-1 instead of Windows-1252 — and repairs them heuristically. Its defaults also do things you may not want in a data cleaning job, notably replacing curly quotes and apostrophes with straight ones:",[198,798,800],{"className":261,"code":799,"language":263,"meta":203,"style":203},"import ftfy\n\nbroken = customers.loc[customers[\"looks_garbled\"], \"customer\"]\nprint([ftfy.fix_text(t) for t in broken])\nprint([ftfy.fix_text(t, uncurl_quotes=False) for t in broken])\n\ntwice = \"Café\".encode(\"utf-8\").decode(\"cp1252\").encode(\"utf-8\").decode(\"cp1252\")\nprint(repr(twice), \"->\", repr(ftfy.fix_text(twice)), \"| repair():\", repr(repair(twice)))\n",[14,801,802,808,812,832,851,876,880,910],{"__ignoreMap":203},[207,803,804,806],{"class":98,"line":209},[207,805,271],{"class":270},[207,807,226],{"class":274},[207,809,810],{"class":98,"line":278},[207,811,296],{"emptyLinePlaceholder":295},[207,813,814,817,819,822,824,827,829],{"class":98,"line":292},[207,815,816],{"class":274},"broken ",[207,818,305],{"class":270},[207,820,821],{"class":274}," customers.loc[customers[",[207,823,375],{"class":216},[207,825,826],{"class":274},"], ",[207,828,386],{"class":216},[207,830,831],{"class":274},"]\n",[207,833,834,836,839,842,845,848],{"class":98,"line":299},[207,835,399],{"class":320},[207,837,838],{"class":274},"([ftfy.fix_text(t) ",[207,840,841],{"class":270},"for",[207,843,844],{"class":274}," t ",[207,846,847],{"class":270},"in",[207,849,850],{"class":274}," broken])\n",[207,852,853,855,858,861,863,865,868,870,872,874],{"class":98,"line":317},[207,854,399],{"class":320},[207,856,857],{"class":274},"([ftfy.fix_text(t, ",[207,859,860],{"class":405},"uncurl_quotes",[207,862,305],{"class":270},[207,864,411],{"class":320},[207,866,867],{"class":274},") ",[207,869,841],{"class":270},[207,871,844],{"class":274},[207,873,847],{"class":270},[207,875,850],{"class":274},[207,877,878],{"class":98,"line":369},[207,879,296],{"emptyLinePlaceholder":295},[207,881,882,885,887,890,893,895,897,899,902,904,906,908],{"class":98,"line":396},[207,883,884],{"class":274},"twice ",[207,886,305],{"class":270},[207,888,889],{"class":216}," \"Café\"",[207,891,892],{"class":274},".encode(",[207,894,476],{"class":216},[207,896,473],{"class":274},[207,898,470],{"class":216},[207,900,901],{"class":274},").encode(",[207,903,476],{"class":216},[207,905,473],{"class":274},[207,907,470],{"class":216},[207,909,314],{"class":274},[207,911,912,914,917,920,923,926,928,930,933,936,938,940],{"class":98,"line":540},[207,913,399],{"class":320},[207,915,916],{"class":274},"(",[207,918,919],{"class":320},"repr",[207,921,922],{"class":274},"(twice), ",[207,924,925],{"class":216},"\"->\"",[207,927,492],{"class":274},[207,929,919],{"class":320},[207,931,932],{"class":274},"(ftfy.fix_text(twice)), ",[207,934,935],{"class":216},"\"| repair():\"",[207,937,492],{"class":274},[207,939,919],{"class":320},[207,941,942],{"class":274},"(repair(twice)))\n",[10,944,945,946,949,950,953],{},"With ",[14,947,948],{},"uncurl_quotes=False",", ftfy leaves typographic quotes as they were meant to be. The double-encoded example shows where ftfy earns its place: text garbled twice (",[14,951,952],{},"Café",") is beyond a single re-encode, and ftfy unwinds it in one call.",[10,955,956],{},"ftfy can also be wrong. Run on the correct row, it changes the text:",[198,958,960],{"className":261,"code":959,"language":263,"meta":203,"style":203},"print(repr(ftfy.fix_text(\"à is a letter\")))\n",[14,961,962],{"__ignoreMap":203},[207,963,964,966,968,970,973,976],{"class":98,"line":209},[207,965,399],{"class":320},[207,967,916],{"class":274},[207,969,919],{"class":320},[207,971,972],{"class":274},"(ftfy.fix_text(",[207,974,975],{"class":216},"\"à is a letter\"",[207,977,978],{"class":274},")))\n",[10,980,981,982,985,986,988],{},"ftfy knows that a garbled ",[14,983,984],{},"à"," often arrives as ",[14,987,239],{}," followed by a plain space — the character's second byte is a non-breaking space, which many systems turn into an ordinary one — so it treats this perfectly correct text as damaged and \"repairs\" it. That is why the detection step comes first: apply ftfy to rows that match the mojibake pattern, and keep the original next to the repair so a reviewer can see every change.",[193,990,992],{"id":991},"step-4-recognise-what-cannot-be-recovered","Step 4 — Recognise what cannot be recovered",[10,994,995,996,492,999,492,1002,492,1005,1008,1009,1012,1013,1015,1016,1018,1019,240],{},"When UTF-8 bytes were decoded as Windows-1252 with errors replaced, any byte Windows-1252 does not define — ",[14,997,998],{},"0x81",[14,1000,1001],{},"0x8D",[14,1003,1004],{},"0x8F",[14,1006,1007],{},"0x90"," and ",[14,1010,1011],{},"0x9D"," — became ",[14,1014,426],{},". The closing curly quote is encoded with a ",[14,1017,1011],{}," byte, which is why the quote row cannot be restored: the byte is simply not there any more. Neither method brings it back; ftfy repairs the rest of the cell and leaves the ",[14,1020,426],{},[198,1022,1024],{"className":261,"code":1023,"language":263,"meta":203,"style":203},"unrecoverable = customers[\"customer\"].str.contains(\"�\")\nprint(customers.loc[unrecoverable, [\"id\", \"customer\"]].to_string(index=False))\n",[14,1025,1026,1044],{"__ignoreMap":203},[207,1027,1028,1031,1033,1035,1037,1039,1042],{"class":98,"line":209},[207,1029,1030],{"class":274},"unrecoverable ",[207,1032,305],{"class":270},[207,1034,383],{"class":274},[207,1036,386],{"class":216},[207,1038,389],{"class":274},[207,1040,1041],{"class":216},"\"�\"",[207,1043,314],{"class":274},[207,1045,1046,1048,1051,1054,1056,1058,1060,1062,1064,1066],{"class":98,"line":278},[207,1047,399],{"class":320},[207,1049,1050],{"class":274},"(customers.loc[unrecoverable, [",[207,1052,1053],{"class":216},"\"id\"",[207,1055,492],{"class":274},[207,1057,386],{"class":216},[207,1059,565],{"class":274},[207,1061,406],{"class":405},[207,1063,305],{"class":270},[207,1065,411],{"class":320},[207,1067,414],{"class":274},[10,1069,1070],{},"List these rows separately and fetch them again from the source system, or correct them by hand. Pretending to repair them — guessing a character — creates plausible-looking wrong data.",[38,1072,47,1077,47,1080,47,1083,47,1086,47,1089,47,1094,47,1100,47,1103,47,1107,47,1110,47,1114,47,1118,47,1123,47,1125,47,1131,47,1133,47,1138,47,1140,47,1145,47,1148,47,1152,47,1154,47,1160,47,1162,47,1167,47,1169,47,1174,47,1177,47,1181,47,1183,47,1188,47,1190,47,1197,47,1199,47,1204,47,1207,47,1211,47,1213,47,1218,47,1220,47,1225,47,1227,47,1232,47,1235,47,1239,47,1241,47,1246,47,1248,47,1252,47,1254],{"viewBox":1073,"role":41,"ariaLabelledBy":1074,"xmlns":45,"style":46},"4 0 752 242",[1075,1076],"moj-outcomes-t","moj-outcomes-d",[49,1078,1079],{"id":1075},"What each approach does to each kind of cell",[53,1081,1082],{"id":1076},"A grid of cell types against three approaches. Simple mojibake like Café is repaired by the re-encode and by ftfy. Double-encoded text is repaired only by ftfy. Cells containing the replacement character are not recoverable by either. Correct text that merely looks suspicious is left alone by detect-then-repair, but can be changed by ftfy run on everything. Curly quotes are straightened by ftfy defaults unless uncurl_quotes is False.",[57,1084],{"x":59,"y":60,"width":61,"height":1085,"fill":63},"242",[57,1087],{"x":66,"y":67,"width":1088,"height":625,"rx":60,"fill":63,"stroke":113,"style":628},"146.7",[57,1090],{"x":1091,"y":67,"width":1092,"height":625,"rx":60,"fill":1093,"stroke":103,"style":779},"166.7","191.1","#f0f4ff",[86,1095,1099],{"x":1096,"y":1097,"style":1098},"262.3","41.0","font-size:13px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","re-encode (flagged rows)",[57,1101],{"x":1102,"y":67,"width":1092,"height":625,"rx":60,"fill":1093,"stroke":103,"style":779},"357.8",[86,1104,1106],{"x":1105,"y":1097,"style":1098},"453.4","ftfy (flagged rows)",[57,1108],{"x":1109,"y":67,"width":1092,"height":625,"rx":60,"fill":1093,"stroke":103,"style":779},"548.9",[86,1111,1113],{"x":1112,"y":1097,"style":1098},"644.5","ftfy on every row",[57,1115],{"x":66,"y":1116,"width":1088,"height":1117,"rx":60,"fill":112,"stroke":113,"style":628},"56.0","34.0",[86,1119,1122],{"x":626,"y":1120,"style":1121},"78.0","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:start","Simple: Café",[57,1124],{"x":1091,"y":1116,"width":1092,"height":1117,"rx":60,"fill":63,"stroke":113,"style":628},[75,1126,1128],{"transform":1127},"translate(253.3,64.0) scale(0.750)",[79,1129],{"d":81,"fill":82,"stroke":72,"style":1130},"stroke-width:2.40px;stroke-linecap:round;stroke-linejoin:round",[57,1132],{"x":1102,"y":1116,"width":1092,"height":1117,"rx":60,"fill":63,"stroke":113,"style":628},[75,1134,1136],{"transform":1135},"translate(444.4,64.0) scale(0.750)",[79,1137],{"d":81,"fill":82,"stroke":72,"style":1130},[57,1139],{"x":1109,"y":1116,"width":1092,"height":1117,"rx":60,"fill":63,"stroke":113,"style":628},[75,1141,1143],{"transform":1142},"translate(635.5,64.0) scale(0.750)",[79,1144],{"d":81,"fill":82,"stroke":72,"style":1130},[57,1146],{"x":66,"y":1147,"width":1088,"height":1117,"rx":60,"fill":112,"stroke":113,"style":628},"90.0",[86,1149,1151],{"x":626,"y":1150,"style":1121},"112.0","Twice: Café",[57,1153],{"x":1091,"y":1147,"width":1092,"height":1117,"rx":60,"fill":63,"stroke":113,"style":628},[75,1155,1157],{"transform":1156},"translate(254.3,99.0) scale(0.667)",[79,1158],{"d":181,"fill":82,"stroke":146,"style":1159},"stroke-width:2.70px;stroke-linecap:round;stroke-linejoin:round",[57,1161],{"x":1102,"y":1147,"width":1092,"height":1117,"rx":60,"fill":63,"stroke":113,"style":628},[75,1163,1165],{"transform":1164},"translate(444.4,98.0) scale(0.750)",[79,1166],{"d":81,"fill":82,"stroke":72,"style":1130},[57,1168],{"x":1109,"y":1147,"width":1092,"height":1117,"rx":60,"fill":63,"stroke":113,"style":628},[75,1170,1172],{"transform":1171},"translate(635.5,98.0) scale(0.750)",[79,1173],{"d":81,"fill":82,"stroke":72,"style":1130},[57,1175],{"x":66,"y":1176,"width":1088,"height":1117,"rx":60,"fill":112,"stroke":113,"style":628},"124.0",[86,1178,1180],{"x":626,"y":1179,"style":1121},"146.0","Contains �",[57,1182],{"x":1091,"y":1176,"width":1092,"height":1117,"rx":60,"fill":63,"stroke":113,"style":628},[75,1184,1186],{"transform":1185},"translate(254.3,133.0) scale(0.667)",[79,1187],{"d":181,"fill":82,"stroke":146,"style":1159},[57,1189],{"x":1102,"y":1176,"width":1092,"height":1117,"rx":60,"fill":63,"stroke":113,"style":628},[75,1191,1193],{"transform":1192},"translate(444.9,132.5) scale(0.708)",[79,1194],{"d":152,"fill":82,"stroke":1195,"style":1196},"var(--gold,#b4740a)","stroke-width:2.54px;stroke-linecap:round;stroke-linejoin:round",[57,1198],{"x":1109,"y":1176,"width":1092,"height":1117,"rx":60,"fill":63,"stroke":113,"style":628},[75,1200,1202],{"transform":1201},"translate(636.0,132.5) scale(0.708)",[79,1203],{"d":152,"fill":82,"stroke":1195,"style":1196},[57,1205],{"x":66,"y":1206,"width":1088,"height":1117,"rx":60,"fill":112,"stroke":113,"style":628},"158.0",[86,1208,1210],{"x":626,"y":1209,"style":1121},"180.0","Correct text with Ã",[57,1212],{"x":1091,"y":1206,"width":1092,"height":1117,"rx":60,"fill":63,"stroke":113,"style":628},[75,1214,1216],{"transform":1215},"translate(253.3,166.0) scale(0.750)",[79,1217],{"d":81,"fill":82,"stroke":72,"style":1130},[57,1219],{"x":1102,"y":1206,"width":1092,"height":1117,"rx":60,"fill":63,"stroke":113,"style":628},[75,1221,1223],{"transform":1222},"translate(444.4,166.0) scale(0.750)",[79,1224],{"d":81,"fill":82,"stroke":72,"style":1130},[57,1226],{"x":1109,"y":1206,"width":1092,"height":1117,"rx":60,"fill":63,"stroke":113,"style":628},[75,1228,1230],{"transform":1229},"translate(636.5,167.0) scale(0.667)",[79,1231],{"d":181,"fill":82,"stroke":146,"style":1159},[57,1233],{"x":66,"y":1234,"width":1088,"height":1117,"rx":60,"fill":112,"stroke":113,"style":628},"192.0",[86,1236,1238],{"x":626,"y":1237,"style":1121},"214.0","Curly quotes kept",[57,1240],{"x":1091,"y":1234,"width":1092,"height":1117,"rx":60,"fill":63,"stroke":113,"style":628},[75,1242,1244],{"transform":1243},"translate(253.3,200.0) scale(0.750)",[79,1245],{"d":81,"fill":82,"stroke":72,"style":1130},[57,1247],{"x":1102,"y":1234,"width":1092,"height":1117,"rx":60,"fill":63,"stroke":113,"style":628},[86,1249,1251],{"x":1105,"y":1250,"style":95},"213.0","with uncurl_quotes=False",[57,1253],{"x":1109,"y":1234,"width":1092,"height":1117,"rx":60,"fill":63,"stroke":113,"style":628},[86,1255,1251],{"x":1112,"y":1250,"style":95},[193,1257,1259],{"id":1258},"scan-a-whole-workbook-before-cleaning-one-column","Scan a whole workbook before cleaning one column",[10,1261,1262],{},"Mojibake rarely affects one column only: if a name was garbled, the address and the notes usually were too. Before repairing, count flagged cells in every text column of every sheet, which also tells you whether the damage is systematic (the whole export) or local (one pasted block):",[198,1264,1266],{"className":261,"code":1265,"language":263,"meta":203,"style":203},"def mojibake_report(path):\n    sheets = pd.read_excel(path, sheet_name=None, dtype=str)\n    rows = []\n    for name, frame in sheets.items():\n        for col in frame.columns:\n            hits = frame[col].dropna().str.contains(MOJIBAKE).sum()\n            if hits:\n                rows.append({\"sheet\": name, \"column\": col, \"cells\": int(hits), \"of\": int(frame[col].notna().sum())})\n    return pd.DataFrame(rows)\n\nprint(mojibake_report(\"customers.xlsx\"))\n",[14,1267,1268,1278,1308,1318,1331,1344,1359,1367,1406,1414,1419],{"__ignoreMap":203},[207,1269,1270,1272,1275],{"class":98,"line":209},[207,1271,444],{"class":270},[207,1273,1274],{"class":447}," mojibake_report",[207,1276,1277],{"class":274},"(path):\n",[207,1279,1280,1283,1285,1288,1291,1293,1296,1298,1301,1303,1306],{"class":98,"line":278},[207,1281,1282],{"class":274},"    sheets ",[207,1284,305],{"class":270},[207,1286,1287],{"class":274}," pd.read_excel(path, ",[207,1289,1290],{"class":405},"sheet_name",[207,1292,305],{"class":270},[207,1294,1295],{"class":320},"None",[207,1297,492],{"class":274},[207,1299,1300],{"class":405},"dtype",[207,1302,305],{"class":270},[207,1304,1305],{"class":320},"str",[207,1307,314],{"class":274},[207,1309,1310,1313,1315],{"class":98,"line":292},[207,1311,1312],{"class":274},"    rows ",[207,1314,305],{"class":270},[207,1316,1317],{"class":274}," []\n",[207,1319,1320,1323,1326,1328],{"class":98,"line":299},[207,1321,1322],{"class":270},"    for",[207,1324,1325],{"class":274}," name, frame ",[207,1327,847],{"class":270},[207,1329,1330],{"class":274}," sheets.items():\n",[207,1332,1333,1336,1339,1341],{"class":98,"line":317},[207,1334,1335],{"class":270},"        for",[207,1337,1338],{"class":274}," col ",[207,1340,847],{"class":270},[207,1342,1343],{"class":274}," frame.columns:\n",[207,1345,1346,1349,1351,1354,1356],{"class":98,"line":369},[207,1347,1348],{"class":274},"            hits ",[207,1350,305],{"class":270},[207,1352,1353],{"class":274}," frame[col].dropna().str.contains(",[207,1355,321],{"class":320},[207,1357,1358],{"class":274},").sum()\n",[207,1360,1361,1364],{"class":98,"line":396},[207,1362,1363],{"class":270},"            if",[207,1365,1366],{"class":274}," hits:\n",[207,1368,1369,1372,1375,1378,1381,1384,1387,1390,1393,1396,1399,1401,1403],{"class":98,"line":540},[207,1370,1371],{"class":274},"                rows.append({",[207,1373,1374],{"class":216},"\"sheet\"",[207,1376,1377],{"class":274},": name, ",[207,1379,1380],{"class":216},"\"column\"",[207,1382,1383],{"class":274},": col, ",[207,1385,1386],{"class":216},"\"cells\"",[207,1388,1389],{"class":274},": ",[207,1391,1392],{"class":320},"int",[207,1394,1395],{"class":274},"(hits), ",[207,1397,1398],{"class":216},"\"of\"",[207,1400,1389],{"class":274},[207,1402,1392],{"class":320},[207,1404,1405],{"class":274},"(frame[col].notna().sum())})\n",[207,1407,1408,1411],{"class":98,"line":551},[207,1409,1410],{"class":270},"    return",[207,1412,1413],{"class":274}," pd.DataFrame(rows)\n",[207,1415,1417],{"class":98,"line":1416},10,[207,1418,296],{"emptyLinePlaceholder":295},[207,1420,1422,1424,1427,1429],{"class":98,"line":1421},11,[207,1423,399],{"class":320},[207,1425,1426],{"class":274},"(mojibake_report(",[207,1428,311],{"class":216},[207,1430,414],{"class":274},[10,1432,1433],{},"Six of seven cells in one column is systematic damage — the whole export was decoded wrongly and the fix belongs upstream (Step 6). A handful of cells scattered across a large sheet usually means text pasted in from another source, and a targeted repair is enough.",[193,1435,1437],{"id":1436},"step-5-write-the-repairs-with-an-audit-trail","Step 5 — Write the repairs with an audit trail",[10,1439,1440],{},"Write the repaired column next to the original, with a status that says what happened to each row:",[198,1442,1444],{"className":261,"code":1443,"language":263,"meta":203,"style":203},"def fix_cell(text):\n    if not isinstance(text, str) or not MOJIBAKE.search(text):\n        return text, \"unchanged\"\n    fixed = ftfy.fix_text(text, uncurl_quotes=False)\n    status = \"unrecoverable\" if \"�\" in fixed else \"repaired\"\n    return fixed, status\n\nout = customers[[\"id\", \"customer\"]].copy()\nout[[\"customer_fixed\", \"status\"]] = out[\"customer\"].map(fix_cell).tolist()\nout.to_excel(\"customers_fixed.xlsx\", index=False)\nprint(out.to_string(index=False))\n",[14,1445,1446,1455,1484,1494,1512,1540,1547,1551,1570,1596,1614],{"__ignoreMap":203},[207,1447,1448,1450,1453],{"class":98,"line":209},[207,1449,444],{"class":270},[207,1451,1452],{"class":447}," fix_cell",[207,1454,451],{"class":274},[207,1456,1457,1460,1463,1466,1469,1471,1473,1476,1478,1481],{"class":98,"line":278},[207,1458,1459],{"class":270},"    if",[207,1461,1462],{"class":270}," not",[207,1464,1465],{"class":320}," isinstance",[207,1467,1468],{"class":274},"(text, ",[207,1470,1305],{"class":320},[207,1472,867],{"class":274},[207,1474,1475],{"class":270},"or",[207,1477,1462],{"class":270},[207,1479,1480],{"class":320}," MOJIBAKE",[207,1482,1483],{"class":274},".search(text):\n",[207,1485,1486,1488,1491],{"class":98,"line":292},[207,1487,464],{"class":270},[207,1489,1490],{"class":274}," text, ",[207,1492,1493],{"class":216},"\"unchanged\"\n",[207,1495,1496,1499,1501,1504,1506,1508,1510],{"class":98,"line":299},[207,1497,1498],{"class":274},"    fixed ",[207,1500,305],{"class":270},[207,1502,1503],{"class":274}," ftfy.fix_text(text, ",[207,1505,860],{"class":405},[207,1507,305],{"class":270},[207,1509,411],{"class":320},[207,1511,314],{"class":274},[207,1513,1514,1517,1519,1522,1525,1528,1531,1534,1537],{"class":98,"line":317},[207,1515,1516],{"class":274},"    status ",[207,1518,305],{"class":270},[207,1520,1521],{"class":216}," \"unrecoverable\"",[207,1523,1524],{"class":270}," if",[207,1526,1527],{"class":216}," \"�\"",[207,1529,1530],{"class":270}," in",[207,1532,1533],{"class":274}," fixed ",[207,1535,1536],{"class":270},"else",[207,1538,1539],{"class":216}," \"repaired\"\n",[207,1541,1542,1544],{"class":98,"line":369},[207,1543,1410],{"class":270},[207,1545,1546],{"class":274}," fixed, status\n",[207,1548,1549],{"class":98,"line":396},[207,1550,296],{"emptyLinePlaceholder":295},[207,1552,1553,1556,1558,1561,1563,1565,1567],{"class":98,"line":540},[207,1554,1555],{"class":274},"out ",[207,1557,305],{"class":270},[207,1559,1560],{"class":274}," customers[[",[207,1562,1053],{"class":216},[207,1564,492],{"class":274},[207,1566,386],{"class":216},[207,1568,1569],{"class":274},"]].copy()\n",[207,1571,1572,1575,1578,1580,1583,1586,1588,1591,1593],{"class":98,"line":551},[207,1573,1574],{"class":274},"out[[",[207,1576,1577],{"class":216},"\"customer_fixed\"",[207,1579,492],{"class":274},[207,1581,1582],{"class":216},"\"status\"",[207,1584,1585],{"class":274},"]] ",[207,1587,305],{"class":270},[207,1589,1590],{"class":274}," out[",[207,1592,386],{"class":216},[207,1594,1595],{"class":274},"].map(fix_cell).tolist()\n",[207,1597,1598,1601,1604,1606,1608,1610,1612],{"class":98,"line":1416},[207,1599,1600],{"class":274},"out.to_excel(",[207,1602,1603],{"class":216},"\"customers_fixed.xlsx\"",[207,1605,492],{"class":274},[207,1607,406],{"class":405},[207,1609,305],{"class":270},[207,1611,411],{"class":320},[207,1613,314],{"class":274},[207,1615,1616,1618,1621,1623,1625,1627],{"class":98,"line":1421},[207,1617,399],{"class":320},[207,1619,1620],{"class":274},"(out.to_string(",[207,1622,406],{"class":405},[207,1624,305],{"class":270},[207,1626,411],{"class":320},[207,1628,414],{"class":274},[10,1630,1631],{},"The status column makes the result reviewable: five repaired, one unrecoverable, one unchanged. Once the reviewer is satisfied, replace the original column in the downstream data and keep this workbook as the record of what changed.",[193,1633,1635],{"id":1634},"step-6-fix-it-at-the-source","Step 6 — Fix it at the source",[10,1637,1638],{},"Downstream repair should be a one-off. Find the step that decodes with the wrong encoding and fix it there. The two usual culprits:",[198,1640,1642],{"className":261,"code":1641,"language":263,"meta":203,"style":203},"text = \"customer\\nCafé Rouge\\n\"\nopen(\"export.csv\", \"w\", encoding=\"utf-8\").write(text)\n\nprint(pd.read_csv(\"export.csv\", encoding=\"cp1252\")[\"customer\"].tolist())   # the bug\nprint(pd.read_csv(\"export.csv\", encoding=\"utf-8\")[\"customer\"].tolist())    # the fix\n\npd.DataFrame({\"customer\": [\"Café Rouge\"]}).to_csv(\"for_excel.csv\", index=False, encoding=\"utf-8-sig\")\nprint(open(\"for_excel.csv\", \"rb\").read()[:3])\n",[14,1643,1644,1664,1691,1695,1724,1750,1754,1792],{"__ignoreMap":203},[207,1645,1646,1649,1651,1654,1657,1659,1661],{"class":98,"line":209},[207,1647,1648],{"class":274},"text ",[207,1650,305],{"class":270},[207,1652,1653],{"class":216}," \"customer",[207,1655,1656],{"class":339},"\\n",[207,1658,579],{"class":216},[207,1660,1656],{"class":339},[207,1662,1663],{"class":216},"\"\n",[207,1665,1666,1669,1671,1674,1676,1679,1681,1684,1686,1688],{"class":98,"line":278},[207,1667,1668],{"class":320},"open",[207,1670,916],{"class":274},[207,1672,1673],{"class":216},"\"export.csv\"",[207,1675,492],{"class":274},[207,1677,1678],{"class":216},"\"w\"",[207,1680,492],{"class":274},[207,1682,1683],{"class":405},"encoding",[207,1685,305],{"class":270},[207,1687,476],{"class":216},[207,1689,1690],{"class":274},").write(text)\n",[207,1692,1693],{"class":98,"line":292},[207,1694,296],{"emptyLinePlaceholder":295},[207,1696,1697,1699,1702,1704,1706,1708,1710,1712,1715,1717,1720],{"class":98,"line":299},[207,1698,399],{"class":320},[207,1700,1701],{"class":274},"(pd.read_csv(",[207,1703,1673],{"class":216},[207,1705,492],{"class":274},[207,1707,1683],{"class":405},[207,1709,305],{"class":270},[207,1711,470],{"class":216},[207,1713,1714],{"class":274},")[",[207,1716,386],{"class":216},[207,1718,1719],{"class":274},"].tolist())   ",[207,1721,1723],{"class":1722},"s-wDw","# the bug\n",[207,1725,1726,1728,1730,1732,1734,1736,1738,1740,1742,1744,1747],{"class":98,"line":317},[207,1727,399],{"class":320},[207,1729,1701],{"class":274},[207,1731,1673],{"class":216},[207,1733,492],{"class":274},[207,1735,1683],{"class":405},[207,1737,305],{"class":270},[207,1739,476],{"class":216},[207,1741,1714],{"class":274},[207,1743,386],{"class":216},[207,1745,1746],{"class":274},"].tolist())    ",[207,1748,1749],{"class":1722},"# the fix\n",[207,1751,1752],{"class":98,"line":369},[207,1753,296],{"emptyLinePlaceholder":295},[207,1755,1756,1759,1761,1764,1767,1770,1773,1775,1777,1779,1781,1783,1785,1787,1790],{"class":98,"line":396},[207,1757,1758],{"class":274},"pd.DataFrame({",[207,1760,386],{"class":216},[207,1762,1763],{"class":274},": [",[207,1765,1766],{"class":216},"\"Café Rouge\"",[207,1768,1769],{"class":274},"]}).to_csv(",[207,1771,1772],{"class":216},"\"for_excel.csv\"",[207,1774,492],{"class":274},[207,1776,406],{"class":405},[207,1778,305],{"class":270},[207,1780,411],{"class":320},[207,1782,492],{"class":274},[207,1784,1683],{"class":405},[207,1786,305],{"class":270},[207,1788,1789],{"class":216},"\"utf-8-sig\"",[207,1791,314],{"class":274},[207,1793,1794,1796,1798,1800,1802,1804,1806,1809,1812,1814],{"class":98,"line":540},[207,1795,399],{"class":320},[207,1797,916],{"class":274},[207,1799,1668],{"class":320},[207,1801,916],{"class":274},[207,1803,1772],{"class":216},[207,1805,492],{"class":274},[207,1807,1808],{"class":216},"\"rb\"",[207,1810,1811],{"class":274},").read()[:",[207,1813,674],{"class":320},[207,1815,1816],{"class":274},"])\n",[10,1818,1819,1820,1823,1824,1827,1828,1831,1832,1835,1836,1840],{},"Reading a UTF-8 file with ",[14,1821,1822],{},"encoding=\"cp1252\""," produces exactly the mojibake this page repaired. And a UTF-8 CSV ",[27,1825,1826],{},"without"," a byte-order mark, opened by double-clicking in Excel on Windows, is decoded as the local code page — the same damage, done by Excel — which ",[14,1829,1830],{},"encoding=\"utf-8-sig\""," prevents by writing the ",[14,1833,1834],{},"EF BB BF"," marker. ",[32,1837,1839],{"href":1838},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fconvert-excel-to-csv-with-python\u002F","Convert Excel to CSV with Python"," covers encodings for CSV output.",[193,1842,1844],{"id":1843},"common-pitfalls","Common pitfalls",[1846,1847,1848,1856,1865,1874],"ul",{},[1849,1850,1851,1855],"li",{},[1852,1853,1854],"strong",{},"Repairing every row."," Detect first; repair only what matches the mojibake pattern.",[1849,1857,1858,1861,1862,1864],{},[1852,1859,1860],{},"ftfy's default quote handling."," Pass ",[14,1863,948],{}," unless straight quotes are what you want.",[1849,1866,1867,1870,1871,1873],{},[1852,1868,1869],{},"Guessing lost characters."," A ",[14,1872,426],{}," is missing data, not a puzzle to solve.",[1849,1875,1876,1879],{},[1852,1877,1878],{},"Fixing downstream for ever."," Correct the encoding at the step that caused it.",[193,1881,1883],{"id":1882},"performance-notes","Performance notes",[10,1885,1886],{},"The detection regex is vectorised and fast; ftfy is pure Python and processes tens of thousands of short strings per second, which is why it should run only on the flagged subset. For very large tables with little damage, detection first makes the whole job a matter of seconds.",[193,1888,1890],{"id":1889},"conclusion","Conclusion",[10,1892,1893,1894,1008,1896,1898,1899,1901,1902,1904,1905,1907,1908,1911,1912,1915,1916,1918],{},"Garbled text like ",[14,1895,185],{},[14,1897,24],{}," is UTF-8 that was decoded as Windows-1252. Find affected cells with a pattern for ",[14,1900,239],{},"\u002F",[14,1903,253],{}," followed by a continuation character and for ",[14,1906,257],{},", repair them with ",[14,1909,1910],{},"text.encode(\"cp1252\").decode(\"utf-8\")"," or with ",[14,1913,1914],{},"ftfy.fix_text(..., uncurl_quotes=False)"," for harder cases, keep the original and a status column, list cells containing ",[14,1917,426],{}," as unrecoverable, and fix the reading or writing step that caused the damage.",[193,1920,1922],{"id":1921},"frequently-asked-questions","Frequently asked questions",[10,1924,1925,1928],{},[1852,1926,1927],{},"What causes text like Café or ’?","\nText saved as UTF-8 was read back as Windows-1252 (or Latin-1). Each non-ASCII character is two or three bytes in UTF-8, and reading those bytes one at a time as single-byte characters produces the familiar pairs: é becomes é, the curly apostrophe becomes ’.",[10,1930,1931,1934],{},[1852,1932,1933],{},"Can garbled text always be repaired?","\nOnly if no bytes were lost. Where the wrong decoding hit a byte that Windows-1252 does not define, the byte was replaced with the replacement character �, and the original information is gone. Those cells need the source data.",[10,1936,1937,1940],{},[1852,1938,1939],{},"Is ftfy safe to run on a whole column?","\nIt is careful, but not perfect. By default it also straightens curly quotes, and it can misread legitimate text that happens to look like mojibake. Run it only on cells that match a mojibake pattern, and review the changes.",[10,1942,1943,1946],{},[1852,1944,1945],{},"How do I stop this happening?","\nFind the step that decodes with the wrong encoding — usually reading a UTF-8 CSV without encoding=\"utf-8\", or opening a UTF-8 CSV in Excel without a byte-order mark — and fix it there. Repairing downstream should be a one-off.",[193,1948,1950],{"id":1949},"related","Related",[1846,1952,1953,1959,1966,1973],{},[1849,1954,1955,1956,1958],{},"Up one level: ",[32,1957,35],{"href":34}," — the full cleaning workflow.",[1849,1960,1961,1965],{},[32,1962,1964],{"href":1963},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-non-printable-characters-from-excel-data\u002F","Remove Non-Printable Characters from Excel Data with Python"," — control characters and invisible text.",[1849,1967,1968,1972],{},[32,1969,1971],{"href":1970},"\u002Fgetting-started-with-python-excel-automation\u002Fhandling-excel-file-formats-and-conversions\u002Fread-csv-exports-with-semicolons-and-decimal-commas\u002F","Read European CSV Exports with Semicolons and Decimal Commas"," — choosing the right encoding when reading.",[1849,1974,1975,1979],{},[32,1976,1978],{"href":1977},"\u002Fgetting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002Ffix-illegalcharactererror-when-writing-excel-with-openpyxl\u002F","Fix IllegalCharacterError When Writing Excel with openpyxl"," — characters Excel refuses outright.",[1981,1982,1983],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}",{"title":203,"searchDepth":278,"depth":278,"links":1985},[1986,1987,1988,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998],{"id":195,"depth":278,"text":196},{"id":243,"depth":278,"text":244},{"id":430,"depth":278,"text":431},{"id":788,"depth":278,"text":789},{"id":991,"depth":278,"text":992},{"id":1258,"depth":278,"text":1259},{"id":1436,"depth":278,"text":1437},{"id":1634,"depth":278,"text":1635},{"id":1843,"depth":278,"text":1844},{"id":1882,"depth":278,"text":1883},{"id":1889,"depth":278,"text":1890},{"id":1921,"depth":278,"text":1922},{"id":1949,"depth":278,"text":1950},"2026-09-25","Text that shows \"Müller\", \"€100\" or \"’\" was UTF-8 read as Windows-1252 somewhere upstream. Detect the affected cells, repair them with a one-line re-encode or with ftfy, avoid the repairs that damage correct text, recognise what cannot be recovered, and stop it happening again.","md",[2003,2005,2007,2009],{"q":1927,"a":2004},"Text saved as UTF-8 was read back as Windows-1252 (or Latin-1). Each non-ASCII character is two or three bytes in UTF-8, and reading those bytes one at a time as single-byte characters produces the familiar pairs: é becomes é, the curly apostrophe becomes ’.",{"q":1933,"a":2006},"Only if no bytes were lost. Where the wrong decoding hit a byte that Windows-1252 does not define, the byte was replaced with the replacement character �, and the original information is gone. Those cells need the source data.",{"q":1939,"a":2008},"It is careful, but not perfect. By default it also straightens curly quotes, and it can misread legitimate text that happens to look like mojibake. Run it only on cells that match a mojibake pattern, and review the changes.",{"q":1945,"a":2010},"Find the step that decodes with the wrong encoding — usually reading a UTF-8 CSV without encoding=\"utf-8\", or opening a UTF-8 CSV in Excel without a byte-order mark — and fix it there. Repairing downstream should be a one-off.","fix mojibake garbled characters in excel data",null,{"breadcrumb":2014},[2015,2017,2020],{"name":2016,"item":1901},"Home",{"name":2018,"item":2019},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":2021,"item":34},"Cleaning Excel Data With Pandas: A Production Workflow","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-garbled-characters-in-excel-data",{"title":2024,"description":2025},"Fix Mojibake (é, ’, Â) in Excel and CSV Data with Python","Repair garbled characters in Excel data with Python: why UTF-8 read as cp1252 produces é and ’, detecting affected cells, re-encoding, ftfy and its side effects, unrecoverable replacement characters, and fixing the encoding at the source.","fix-garbled-characters-in-excel-data","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-garbled-characters-in-excel-data\u002Findex",[2029,2030,2031],"data-cleaning","file-formats","pandas","how-to","ZdtB7L0EgH6qgEsoY5gFsTLtBGYvrYeKjOe9Xa_UibU",[2035,2039],{"title":2036,"path":2037,"stem":2038,"children":-1},"Find Near-Duplicate Names Within One Excel Column","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffind-near-duplicate-names-within-one-excel-column","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffind-near-duplicate-names-within-one-excel-column\u002Findex",{"title":2040,"path":2041,"stem":2042,"children":-1},"Fix the Capitalisation of Names with Exceptions in pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-the-capitalisation-of-names-with-exceptions-in-pandas","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-the-capitalisation-of-names-with-exceptions-in-pandas\u002Findex",[2044,2046],{"path":2045,"title":2021},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas",{"path":2047,"title":2018},"\u002Fadvanced-data-transformation-and-cleaning",[],[2050,2053,2056,2059,2062,2065],{"path":2051,"title":2052,"kind":2032},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Ffind-duplicate-rows-in-excel-with-python","Find Duplicate Rows in Excel with Python",{"path":2054,"title":2055,"kind":2032},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-duplicate-values-in-excel-with-openpyxl","Highlight Duplicate Values in Excel with openpyxl",{"path":2057,"title":2058,"kind":2032},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fclean-mixed-currency-amounts-into-amount-and-currency-columns","Clean Mixed-Currency Amounts into Amount and Currency Columns",{"path":2060,"title":2061,"kind":2032},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fcoalesce-several-excel-columns-into-one-with-pandas","Coalesce Several Excel Columns into One with pandas",{"path":2063,"title":2064,"kind":2032},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-currency-strings-to-numbers-with-pandas","Convert Currency Strings Like \"$1,234.56\" and \"(500)\" to Numbers with pandas",{"path":2066,"title":2067,"kind":2032},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-excel-text-columns-to-numbers-with-pandas","Convert Excel Text Columns to Numbers with pandas",1790972080752]