[{"data":1,"prerenderedAt":1931},["ShallowReactive",2],{"doc:\u002Fgetting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002Ffix-unicodedecodeerror-when-reading-csv-files-saved-by-excel":3,"surround:\u002Fgetting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002Ffix-unicodedecodeerror-when-reading-csv-files-saved-by-excel":1895,"ancestors:\u002Fgetting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002Ffix-unicodedecodeerror-when-reading-csv-files-saved-by-excel":1904,"children:\u002Fgetting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002Ffix-unicodedecodeerror-when-reading-csv-files-saved-by-excel":1909,"related:\u002Fgetting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002Ffix-unicodedecodeerror-when-reading-csv-files-saved-by-excel":1910},{"id":4,"title":5,"body":6,"dateModified":1861,"datePublished":1861,"description":1862,"extension":1863,"faq":1864,"keyword":1873,"kind":1874,"meta":1875,"navigation":126,"path":1884,"seo":1885,"slug":1888,"stem":1889,"topics":1890,"type":1893,"__hash__":1894},"docs\u002Fgetting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002Ffix-unicodedecodeerror-when-reading-csv-files-saved-by-excel\u002Findex.md","Fix UnicodeDecodeError When Reading CSV Files Saved by Excel",{"type":7,"value":8,"toc":1847},"minimark",[9,23,33,46,51,77,91,95,98,225,228,278,448,452,633,678,803,807,810,894,927,1056,1060,1063,1346,1376,1380,1410,1545,1549,1563,1665,1682,1686,1697,1701,1754,1758,1784,1788,1794,1800,1806,1812,1816,1843],[10,11,12,13,17,18,22],"p",{},"Someone exports a sheet from Excel with ",[14,15,16],"em",{},"Save As › CSV"," and sends it over. ",[19,20,21],"code",{},"pd.read_csv(\"export.csv\")"," fails at once:",[24,25,31],"pre",{"className":26,"code":28,"language":29,"meta":30},[27],"language-text","UnicodeDecodeError: 'utf-8' codec can't decode byte 0xfc in position 22: invalid start byte\n","text","",[19,32,28],{"__ignoreMap":30},[10,34,35,36,39,40,45],{},"or, for a different export option, ",[19,37,38],{},"can't decode byte 0xff in position 0",". Python reads text files as UTF-8 by default, and Excel offers several \"CSV\" and text formats, most of which are not UTF-8. The byte in the message is a strong clue to which one you have. This page shows how to identify the format from the first bytes, read each correctly, and avoid the two \"fixes\" that make the error go away while quietly corrupting the data. It belongs to ",[41,42,44],"a",{"href":43},"\u002Fgetting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002F","Troubleshooting Common Python Excel Errors",".",[47,48,50],"h2",{"id":49},"prerequisites","Prerequisites",[24,52,56],{"className":53,"code":54,"language":55,"meta":30,"style":30},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas charset-normalizer\n","bash",[19,57,58],{"__ignoreMap":30},[59,60,63,67,71,74],"span",{"class":61,"line":62},"line",1,[59,64,66],{"class":65},"sMTad","pip",[59,68,70],{"class":69},"srMev"," install",[59,72,73],{"class":69}," pandas",[59,75,76],{"class":69}," charset-normalizer\n",[10,78,79,80,83,84,87,88,45],{},"The three sample files contain the same customers with accented names and euro prices, saved the way Excel saves its three text formats on a Western-European Windows machine: ",[14,81,82],{},"CSV (Comma delimited)",", ",[14,85,86],{},"CSV UTF-8 (Comma delimited)"," and ",[14,89,90],{},"Unicode Text",[47,92,94],{"id":93},"step-1-read-the-first-bytes","Step 1 — Read the first bytes",[10,96,97],{},"The quickest diagnosis needs no library: look at the start of the file.",[24,99,103],{"className":100,"code":101,"language":102,"meta":30,"style":30},"language-python shiki shiki-themes github-light github-dark-high-contrast","from pathlib import Path\n\nfor name in (\"excel_csv.csv\", \"excel_csv_utf8.csv\", \"excel_unicode.txt\"):\n    head = Path(name).read_bytes()[:16]\n    print(f\"{name:20} {head.hex(' ')}\")\n","python",[19,104,105,121,128,159,178],{"__ignoreMap":30},[59,106,107,111,115,118],{"class":61,"line":62},[59,108,110],{"class":109},"s-kum","from",[59,112,114],{"class":113},"skGVy"," pathlib ",[59,116,117],{"class":109},"import",[59,119,120],{"class":113}," Path\n",[59,122,124],{"class":61,"line":123},2,[59,125,127],{"emptyLinePlaceholder":126},true,"\n",[59,129,131,134,137,140,143,146,148,151,153,156],{"class":61,"line":130},3,[59,132,133],{"class":109},"for",[59,135,136],{"class":113}," name ",[59,138,139],{"class":109},"in",[59,141,142],{"class":113}," (",[59,144,145],{"class":69},"\"excel_csv.csv\"",[59,147,83],{"class":113},[59,149,150],{"class":69},"\"excel_csv_utf8.csv\"",[59,152,83],{"class":113},[59,154,155],{"class":69},"\"excel_unicode.txt\"",[59,157,158],{"class":113},"):\n",[59,160,162,165,168,171,175],{"class":61,"line":161},4,[59,163,164],{"class":113},"    head ",[59,166,167],{"class":109},"=",[59,169,170],{"class":113}," Path(name).read_bytes()[:",[59,172,174],{"class":173},"sP0c6","16",[59,176,177],{"class":113},"]\n",[59,179,181,184,187,190,193,197,200,203,206,209,212,215,218,220,222],{"class":61,"line":180},5,[59,182,183],{"class":173},"    print",[59,185,186],{"class":113},"(",[59,188,189],{"class":109},"f",[59,191,192],{"class":69},"\"",[59,194,196],{"class":195},"sSjpA","{",[59,198,199],{"class":113},"name",[59,201,202],{"class":109},":20",[59,204,205],{"class":195},"}",[59,207,208],{"class":195}," {",[59,210,211],{"class":113},"head.hex(",[59,213,214],{"class":69},"' '",[59,216,217],{"class":113},")",[59,219,205],{"class":195},[59,221,192],{"class":69},[59,223,224],{"class":113},")\n",[10,226,227],{},"Three patterns, three formats:",[229,230,231,241,255],"ul",{},[232,233,234,237,238,45],"li",{},[19,235,236],{},"ef bb bf"," at the start is the UTF-8 byte order mark — ",[14,239,240],{},"CSV UTF-8",[232,242,243,246,247,249,250,254],{},[19,244,245],{},"ff fe"," at the start is the UTF-16 little-endian byte order mark — ",[14,248,90],{},", which is also ",[251,252,253],"strong",{},"tab","-separated despite the name people give it.",[232,256,257,258,261,262,265,266,269,270,273,274,277],{},"No marker, and bytes such as ",[19,259,260],{},"fc"," (ü), ",[19,263,264],{},"e9"," (é) or ",[19,267,268],{},"80"," (€) later on, means a single-byte Windows code page — plain ",[14,271,272],{},"CSV",", written in the code page of the machine that saved it, usually Windows-1252 (",[19,275,276],{},"cp1252",") in Western Europe and the Americas.",[279,280,288,289,288,293,288,297,288,305,288,315,288,326,288,331,288,335,288,339,288,342,288,347,288,356,288,361,288,369,288,376,288,380,288,385,288,387,288,390,288,394,288,397,288,400,288,406,288,408,288,411,288,413,288,416,288,420,288,423,288,428,288,435,288,440,288,444],"svg",{"viewBox":281,"role":282,"ariaLabelledBy":283,"xmlns":286,"style":287},"4 0 752 290","img",[284,285],"enc-detect-t","enc-detect-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[290,291,292],"title",{"id":284},"Identify an Excel text export from its first bytes",[294,295,296],"desc",{"id":285},"If the file starts with EF BB BF it is CSV UTF-8, read with utf-8-sig. If it starts with FF FE it is Unicode Text, UTF-16 and tab separated. With no marker and bytes like FC or 80 that fail as UTF-8, it is plain CSV in the Windows code page, usually cp1252.",[298,299],"rect",{"x":300,"y":301,"width":302,"height":303,"fill":304},"4","0","752","290","#ffffff",[298,306],{"x":307,"y":308,"width":309,"height":310,"rx":311,"fill":312,"stroke":313,"style":314},"240.0","16.0","280.0","75.5","12","#f0f4ff","var(--brand,#5b5cf0)","stroke-width:2px",[316,317,319],"g",{"transform":318},"translate(368.0,29.0) scale(1.000)",[320,321],"path",{"d":322,"fill":323,"stroke":324,"style":325},"M11 4a7 7 0 1 0 .01 0z M16 16l5 5","none","var(--brand-strong,#4338ca)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[29,327,330],{"x":328,"y":310,"style":329},"380.0","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","How does the file start?",[61,332],{"x1":328,"y1":333,"x2":328,"y2":334,"stroke":313,"style":314},"91.5","111.5",[61,336],{"x1":337,"y1":334,"x2":338,"y2":334,"stroke":313,"style":314},"134.7","625.3",[61,340],{"x1":337,"y1":334,"x2":337,"y2":341,"stroke":313,"style":314},"157.5",[343,344],"polygon",{"points":345,"fill":346},"134.7,161.5 129.7,152.5 139.7,152.5","#5b5cf0",[298,348],{"x":349,"y":350,"width":351,"height":352,"rx":353,"fill":304,"stroke":354,"style":355},"100.1","121.5","69.2","21.0","10","var(--line,#cdd5e6)","stroke-width:1.2px",[29,357,360],{"x":337,"y":358,"style":359},"136.0","font-size:11.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","EF BB BF",[298,362],{"x":363,"y":364,"width":365,"height":366,"rx":311,"fill":367,"stroke":368,"style":314},"20.0","163.5","229.3","110.5","#d9f4f1","var(--teal,#0f9488)",[316,370,372],{"transform":371},"translate(122.7,185.2) scale(1.000)",[320,373],{"d":374,"fill":323,"stroke":375,"style":325},"M5 12.5l4.5 4.5L19 7","var(--teal-ink,#0b6157)",[29,377,240],{"x":337,"y":378,"style":379},"231.8","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle",[29,381,384],{"x":337,"y":382,"style":383},"251.2","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","encoding=\"utf-8-sig\"",[61,386],{"x1":328,"y1":334,"x2":328,"y2":341,"stroke":313,"style":314},[343,388],{"points":389,"fill":346},"380.0,161.5 375.0,152.5 385.0,152.5",[298,391],{"x":392,"y":350,"width":393,"height":352,"rx":353,"fill":304,"stroke":354,"style":355},"355.3","49.5",[29,395,396],{"x":328,"y":358,"style":359},"FF FE",[298,398],{"x":399,"y":364,"width":365,"height":366,"rx":311,"fill":312,"stroke":313,"style":314},"265.3",[316,401,403],{"transform":402},"translate(368.0,185.2) scale(1.000)",[320,404],{"d":405,"fill":323,"stroke":324,"style":325},"M6 3h9l4 4v14H6z M15 3v4h4",[29,407,90],{"x":328,"y":378,"style":329},[29,409,410],{"x":328,"y":382,"style":383},"encoding=\"utf-16\", sep=\"\\t\"",[61,412],{"x1":338,"y1":334,"x2":338,"y2":341,"stroke":313,"style":314},[343,414],{"points":415,"fill":346},"625.3,161.5 620.3,152.5 630.3,152.5",[298,417],{"x":418,"y":350,"width":419,"height":352,"rx":353,"fill":304,"stroke":354,"style":355},"586.4","77.9",[29,421,422],{"x":338,"y":358,"style":359},"no marker",[298,424],{"x":425,"y":364,"width":365,"height":366,"rx":311,"fill":426,"stroke":427,"style":314},"510.7","#fdefd8","var(--gold,#b4740a)",[316,429,431],{"transform":430},"translate(613.3,176.5) scale(1.000)",[320,432],{"d":433,"fill":323,"stroke":434,"style":325},"M3 5.5l8-1.2v7.2H3z M13 4l8-1.2v8.7h-8z M3 13.5h8v7.2l-8-1.2z M13 13.5h8v8.7l-8-1.2z","var(--gold-ink,#7a4e06)",[29,436,439],{"x":338,"y":437,"style":438},"223.0","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Plain CSV",[29,441,443],{"x":338,"y":442,"style":383},"242.5","encoding=\"cp1252\"",[29,445,447],{"x":338,"y":446,"style":383},"260.0","(or local code page)",[47,449,451],{"id":450},"step-2-read-each-format-correctly","Step 2 — Read each format correctly",[24,453,455],{"className":100,"code":454,"language":102,"meta":30,"style":30},"import pandas as pd\n\nansi = pd.read_csv(\"excel_csv.csv\", encoding=\"cp1252\")\nutf8 = pd.read_csv(\"excel_csv_utf8.csv\", encoding=\"utf-8-sig\")\nutf16 = pd.read_csv(\"excel_unicode.txt\", encoding=\"utf-16\", sep=\"\\t\")\nfor label, df in [(\"cp1252\", ansi), (\"utf-8-sig\", utf8), (\"utf-16\", utf16)]:\n    print(f\"{label:10} {list(df.columns)}  {df.iloc[0].tolist()}\")\n",[19,456,457,470,474,499,521,557,585],{"__ignoreMap":30},[59,458,459,461,464,467],{"class":61,"line":62},[59,460,117],{"class":109},[59,462,463],{"class":113}," pandas ",[59,465,466],{"class":109},"as",[59,468,469],{"class":113}," pd\n",[59,471,472],{"class":61,"line":123},[59,473,127],{"emptyLinePlaceholder":126},[59,475,476,479,481,484,486,488,492,494,497],{"class":61,"line":130},[59,477,478],{"class":113},"ansi ",[59,480,167],{"class":109},[59,482,483],{"class":113}," pd.read_csv(",[59,485,145],{"class":69},[59,487,83],{"class":113},[59,489,491],{"class":490},"sa561","encoding",[59,493,167],{"class":109},[59,495,496],{"class":69},"\"cp1252\"",[59,498,224],{"class":113},[59,500,501,504,506,508,510,512,514,516,519],{"class":61,"line":161},[59,502,503],{"class":113},"utf8 ",[59,505,167],{"class":109},[59,507,483],{"class":113},[59,509,150],{"class":69},[59,511,83],{"class":113},[59,513,491],{"class":490},[59,515,167],{"class":109},[59,517,518],{"class":69},"\"utf-8-sig\"",[59,520,224],{"class":113},[59,522,523,526,528,530,532,534,536,538,541,543,546,548,550,553,555],{"class":61,"line":180},[59,524,525],{"class":113},"utf16 ",[59,527,167],{"class":109},[59,529,483],{"class":113},[59,531,155],{"class":69},[59,533,83],{"class":113},[59,535,491],{"class":490},[59,537,167],{"class":109},[59,539,540],{"class":69},"\"utf-16\"",[59,542,83],{"class":113},[59,544,545],{"class":490},"sep",[59,547,167],{"class":109},[59,549,192],{"class":69},[59,551,552],{"class":195},"\\t",[59,554,192],{"class":69},[59,556,224],{"class":113},[59,558,560,562,565,567,570,572,575,577,580,582],{"class":61,"line":559},6,[59,561,133],{"class":109},[59,563,564],{"class":113}," label, df ",[59,566,139],{"class":109},[59,568,569],{"class":113}," [(",[59,571,496],{"class":69},[59,573,574],{"class":113},", ansi), (",[59,576,518],{"class":69},[59,578,579],{"class":113},", utf8), (",[59,581,540],{"class":69},[59,583,584],{"class":113},", utf16)]:\n",[59,586,588,590,592,594,596,598,601,604,606,608,611,614,616,619,622,624,627,629,631],{"class":61,"line":587},7,[59,589,183],{"class":173},[59,591,186],{"class":113},[59,593,189],{"class":109},[59,595,192],{"class":69},[59,597,196],{"class":195},[59,599,600],{"class":113},"label",[59,602,603],{"class":109},":10",[59,605,205],{"class":195},[59,607,208],{"class":195},[59,609,610],{"class":173},"list",[59,612,613],{"class":113},"(df.columns)",[59,615,205],{"class":195},[59,617,618],{"class":195},"  {",[59,620,621],{"class":113},"df.iloc[",[59,623,301],{"class":173},[59,625,626],{"class":113},"].tolist()",[59,628,205],{"class":195},[59,630,192],{"class":69},[59,632,224],{"class":113},[10,634,635,636,83,639,83,642,645,646,649,650,653,654,657,658,661,662,665,666,669,670,673,674,677],{},"All three now produce the same frame: ",[19,637,638],{},"Müller GmbH",[19,640,641],{},"Köln",[19,643,644],{},"€1250",". The UTF-16 file needs both the encoding and ",[19,647,648],{},"sep=\"\\t\"","; with only the encoding it reads as one column. The ",[19,651,652],{},"utf-8-sig"," codec decodes UTF-8 and strips the byte order mark if present. pandas actually strips a leading BOM even with the default ",[19,655,656],{},"utf-8",", but Python's ",[19,659,660],{},"csv"," module and plain ",[19,663,664],{},"open()"," do not, which leaves the first header as ",[19,667,668],{},"'\\ufeffCustomer'"," and makes ",[19,671,672],{},"row[\"Customer\"]"," raise a ",[19,675,676],{},"KeyError",":",[24,679,681],{"className":100,"code":680,"language":102,"meta":30,"style":30},"import csv\n\nwith open(\"excel_csv_utf8.csv\", encoding=\"utf-8\", newline=\"\") as fh:\n    print(next(csv.reader(fh))[0].encode(\"unicode_escape\"))\nwith open(\"excel_csv_utf8.csv\", encoding=\"utf-8-sig\", newline=\"\") as fh:\n    print(next(csv.reader(fh))[0])\n",[19,682,683,690,694,733,756,788],{"__ignoreMap":30},[59,684,685,687],{"class":61,"line":62},[59,686,117],{"class":109},[59,688,689],{"class":113}," csv\n",[59,691,692],{"class":61,"line":123},[59,693,127],{"emptyLinePlaceholder":126},[59,695,696,699,702,704,706,708,710,712,715,717,720,722,725,728,730],{"class":61,"line":130},[59,697,698],{"class":109},"with",[59,700,701],{"class":173}," open",[59,703,186],{"class":113},[59,705,150],{"class":69},[59,707,83],{"class":113},[59,709,491],{"class":490},[59,711,167],{"class":109},[59,713,714],{"class":69},"\"utf-8\"",[59,716,83],{"class":113},[59,718,719],{"class":490},"newline",[59,721,167],{"class":109},[59,723,724],{"class":69},"\"\"",[59,726,727],{"class":113},") ",[59,729,466],{"class":109},[59,731,732],{"class":113}," fh:\n",[59,734,735,737,739,742,745,747,750,753],{"class":61,"line":161},[59,736,183],{"class":173},[59,738,186],{"class":113},[59,740,741],{"class":173},"next",[59,743,744],{"class":113},"(csv.reader(fh))[",[59,746,301],{"class":173},[59,748,749],{"class":113},"].encode(",[59,751,752],{"class":69},"\"unicode_escape\"",[59,754,755],{"class":113},"))\n",[59,757,758,760,762,764,766,768,770,772,774,776,778,780,782,784,786],{"class":61,"line":180},[59,759,698],{"class":109},[59,761,701],{"class":173},[59,763,186],{"class":113},[59,765,150],{"class":69},[59,767,83],{"class":113},[59,769,491],{"class":490},[59,771,167],{"class":109},[59,773,518],{"class":69},[59,775,83],{"class":113},[59,777,719],{"class":490},[59,779,167],{"class":109},[59,781,724],{"class":69},[59,783,727],{"class":113},[59,785,466],{"class":109},[59,787,732],{"class":113},[59,789,790,792,794,796,798,800],{"class":61,"line":559},[59,791,183],{"class":173},[59,793,186],{"class":113},[59,795,741],{"class":173},[59,797,744],{"class":113},[59,799,301],{"class":173},[59,801,802],{"class":113},"])\n",[47,804,806],{"id":805},"step-3-avoid-the-two-silent-fixes","Step 3 — Avoid the two silent fixes",[10,808,809],{},"Two popular answers make the error disappear without making the data right:",[24,811,813],{"className":100,"code":812,"language":102,"meta":30,"style":30},"latin = pd.read_csv(\"excel_csv.csv\", encoding=\"latin-1\")\nreplaced = pd.read_csv(\"excel_csv.csv\", encoding_errors=\"replace\")\nprint(\"latin-1 :\", latin.iloc[0].tolist())\nprint(\"replace :\", replaced.iloc[0].tolist())\n",[19,814,815,837,860,878],{"__ignoreMap":30},[59,816,817,820,822,824,826,828,830,832,835],{"class":61,"line":62},[59,818,819],{"class":113},"latin ",[59,821,167],{"class":109},[59,823,483],{"class":113},[59,825,145],{"class":69},[59,827,83],{"class":113},[59,829,491],{"class":490},[59,831,167],{"class":109},[59,833,834],{"class":69},"\"latin-1\"",[59,836,224],{"class":113},[59,838,839,842,844,846,848,850,853,855,858],{"class":61,"line":123},[59,840,841],{"class":113},"replaced ",[59,843,167],{"class":109},[59,845,483],{"class":113},[59,847,145],{"class":69},[59,849,83],{"class":113},[59,851,852],{"class":490},"encoding_errors",[59,854,167],{"class":109},[59,856,857],{"class":69},"\"replace\"",[59,859,224],{"class":113},[59,861,862,865,867,870,873,875],{"class":61,"line":130},[59,863,864],{"class":173},"print",[59,866,186],{"class":113},[59,868,869],{"class":69},"\"latin-1 :\"",[59,871,872],{"class":113},", latin.iloc[",[59,874,301],{"class":173},[59,876,877],{"class":113},"].tolist())\n",[59,879,880,882,884,887,890,892],{"class":61,"line":161},[59,881,864],{"class":173},[59,883,186],{"class":113},[59,885,886],{"class":69},"\"replace :\"",[59,888,889],{"class":113},", replaced.iloc[",[59,891,301],{"class":173},[59,893,877],{"class":113},[10,895,896,899,900,903,904,907,908,911,912,915,916,919,920,922,923,926],{},[19,897,898],{},"latin-1"," never raises — it maps every byte to ",[14,901,902],{},"some"," character — so accented letters come out right, but the euro sign (byte ",[19,905,906],{},"0x80"," in Windows-1252) becomes the invisible control character ",[19,909,910],{},"\\x80",", and so do curly quotes and dashes. ",[19,913,914],{},"encoding_errors=\"replace\""," keeps reading as UTF-8 and substitutes ",[19,917,918],{},"�"," for every byte it cannot decode, so ",[19,921,641],{}," becomes ",[19,924,925],{},"K�ln"," and the original letter is gone for good. Both produce plausible-looking frames that fail later, in a lookup that never matches or a report that prints garbage.",[279,928,288,934,288,937,288,940,288,945,288,952,288,958,288,961,288,966,288,969,288,973,288,978,288,982,288,986,288,989,288,993,288,997,288,999,288,1001,288,1003,288,1005,288,1009,288,1012,288,1014,288,1016,288,1018,288,1021,288,1023,288,1026,288,1031,288,1035,288,1040,288,1043,288,1050],{"viewBox":929,"role":282,"ariaLabelledBy":930,"xmlns":286,"style":933},"125 0 510 140",[931,932],"enc-wrong-t","enc-wrong-d","width:100%;max-width:510px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[290,935,936],{"id":931},"The same bytes decoded three ways",[294,938,939],{"id":932},"The cp1252 bytes for Köln and €1250. Decoded as cp1252 they read correctly. Decoded as latin-1 the euro sign becomes an invisible control character. Decoded as UTF-8 with errors replaced, ö becomes the replacement character.",[298,941],{"x":942,"y":301,"width":943,"height":944,"fill":304},"125","510","140",[298,946],{"x":947,"y":948,"width":949,"height":950,"rx":301,"fill":304,"stroke":354,"style":951},"141.0","40.0","107.3","28.0","stroke-width:1px",[29,953,957],{"x":954,"y":955,"style":956},"194.7","59.0","font-size:11px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","raw bytes",[298,959],{"x":947,"y":960,"width":949,"height":950,"rx":301,"fill":304,"stroke":354,"style":951},"68.0",[29,962,965],{"x":954,"y":963,"style":964},"87.0","font-size:11px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","4b f6 6c 6e",[298,967],{"x":947,"y":968,"width":949,"height":950,"rx":301,"fill":304,"stroke":354,"style":951},"96.0",[29,970,972],{"x":954,"y":971,"style":964},"115.0","80 31 32 35 30",[298,974],{"x":975,"y":948,"width":976,"height":950,"rx":301,"fill":367,"stroke":368,"style":977},"367.9","64.8","stroke-width:1.5px",[29,979,276],{"x":980,"y":955,"style":981},"400.2","font-size:11px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle",[298,983],{"x":984,"y":948,"width":985,"height":950,"rx":301,"fill":304,"stroke":354,"style":951},"432.6","77.7",[29,987,898],{"x":988,"y":955,"style":956},"471.5",[298,990],{"x":991,"y":948,"width":992,"height":950,"rx":301,"fill":304,"stroke":354,"style":951},"510.3","108.6",[29,994,996],{"x":995,"y":955,"style":956},"564.7","utf-8 + replace",[298,998],{"x":975,"y":960,"width":976,"height":950,"rx":301,"fill":367,"stroke":368,"style":977},[29,1000,641],{"x":980,"y":963,"style":981},[298,1002],{"x":984,"y":960,"width":985,"height":950,"rx":301,"fill":304,"stroke":354,"style":951},[29,1004,641],{"x":988,"y":963,"style":964},[298,1006],{"x":991,"y":960,"width":992,"height":950,"rx":301,"fill":1007,"stroke":1008,"style":977},"#fce9e9","var(--danger,#dc2626)",[29,1010,925],{"x":995,"y":963,"style":1011},"font-size:11px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle",[298,1013],{"x":975,"y":968,"width":976,"height":950,"rx":301,"fill":367,"stroke":368,"style":977},[29,1015,644],{"x":980,"y":971,"style":981},[298,1017],{"x":984,"y":968,"width":985,"height":950,"rx":301,"fill":1007,"stroke":1008,"style":977},[29,1019,1020],{"x":988,"y":971,"style":1011},"\\x801250",[298,1022],{"x":991,"y":968,"width":992,"height":950,"rx":301,"fill":1007,"stroke":1008,"style":977},[29,1024,1025],{"x":995,"y":971,"style":1011},"�1250",[29,1027,1030],{"x":954,"y":1028,"style":1029},"29.0","font-size:12.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","bytes in the file",[29,1032,1034],{"x":1033,"y":1028,"style":1029},"493.4","cp1252 \u002F latin-1 \u002F utf-8 replace",[61,1036],{"x1":1037,"y1":1038,"x2":1039,"y2":1038,"stroke":313,"style":314},"258.3","82.0","353.9",[343,1041],{"points":1042,"fill":346},"357.9,82.0 348.9,87.0 348.9,77.0",[298,1044],{"x":1045,"y":1046,"width":1047,"height":1048,"rx":1049,"fill":304,"stroke":354,"style":355},"276.3","49.0","63.6","22.0","11",[29,1051,1055],{"x":1052,"y":1053,"style":1054},"308.1","64.0","font-size:11.5px;font-weight:400;fill:var(--brand-strong,#4338ca);text-anchor:middle;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace","decode",[47,1057,1059],{"id":1058},"step-4-when-you-do-not-know-the-encoding","Step 4 — When you do not know the encoding",[10,1061,1062],{},"For files from many sources, try the likely encodings in order of strictness and keep the first that decodes cleanly. UTF-8 is strict — Windows-1252 text almost never decodes as valid UTF-8 by accident — so trying it first is safe:",[24,1064,1066],{"className":100,"code":1065,"language":102,"meta":30,"style":30},"def read_excel_csv(path, candidates=(\"utf-8-sig\", \"cp1252\")):\n    head = Path(path).read_bytes()[:4]\n    if head.startswith((b\"\\xff\\xfe\", b\"\\xfe\\xff\")):\n        return pd.read_csv(path, encoding=\"utf-16\", sep=\"\\t\"), \"utf-16\"\n    for enc in candidates:\n        try:\n            return pd.read_csv(path, encoding=enc), enc\n        except UnicodeDecodeError:\n            continue\n    raise ValueError(f\"{path}: none of {candidates} decoded the file\")\n\nfor name in (\"excel_csv.csv\", \"excel_csv_utf8.csv\", \"excel_unicode.txt\"):\n    df, used = read_excel_csv(name)\n    print(f\"{name:20} read as {used:10} {df.iloc[0, 0]}\")\n",[19,1067,1068,1093,1106,1137,1169,1182,1190,1204,1215,1221,1257,1262,1285,1296],{"__ignoreMap":30},[59,1069,1070,1073,1077,1080,1082,1084,1086,1088,1090],{"class":61,"line":62},[59,1071,1072],{"class":109},"def",[59,1074,1076],{"class":1075},"s_Opv"," read_excel_csv",[59,1078,1079],{"class":113},"(path, candidates",[59,1081,167],{"class":109},[59,1083,186],{"class":113},[59,1085,518],{"class":69},[59,1087,83],{"class":113},[59,1089,496],{"class":69},[59,1091,1092],{"class":113},")):\n",[59,1094,1095,1097,1099,1102,1104],{"class":61,"line":123},[59,1096,164],{"class":113},[59,1098,167],{"class":109},[59,1100,1101],{"class":113}," Path(path).read_bytes()[:",[59,1103,300],{"class":173},[59,1105,177],{"class":113},[59,1107,1108,1111,1114,1117,1119,1122,1124,1126,1128,1130,1133,1135],{"class":61,"line":130},[59,1109,1110],{"class":109},"    if",[59,1112,1113],{"class":113}," head.startswith((",[59,1115,1116],{"class":109},"b",[59,1118,192],{"class":69},[59,1120,1121],{"class":195},"\\xff\\xfe",[59,1123,192],{"class":69},[59,1125,83],{"class":113},[59,1127,1116],{"class":109},[59,1129,192],{"class":69},[59,1131,1132],{"class":195},"\\xfe\\xff",[59,1134,192],{"class":69},[59,1136,1092],{"class":113},[59,1138,1139,1142,1145,1147,1149,1151,1153,1155,1157,1159,1161,1163,1166],{"class":61,"line":161},[59,1140,1141],{"class":109},"        return",[59,1143,1144],{"class":113}," pd.read_csv(path, ",[59,1146,491],{"class":490},[59,1148,167],{"class":109},[59,1150,540],{"class":69},[59,1152,83],{"class":113},[59,1154,545],{"class":490},[59,1156,167],{"class":109},[59,1158,192],{"class":69},[59,1160,552],{"class":195},[59,1162,192],{"class":69},[59,1164,1165],{"class":113},"), ",[59,1167,1168],{"class":69},"\"utf-16\"\n",[59,1170,1171,1174,1177,1179],{"class":61,"line":180},[59,1172,1173],{"class":109},"    for",[59,1175,1176],{"class":113}," enc ",[59,1178,139],{"class":109},[59,1180,1181],{"class":113}," candidates:\n",[59,1183,1184,1187],{"class":61,"line":559},[59,1185,1186],{"class":109},"        try",[59,1188,1189],{"class":113},":\n",[59,1191,1192,1195,1197,1199,1201],{"class":61,"line":587},[59,1193,1194],{"class":109},"            return",[59,1196,1144],{"class":113},[59,1198,491],{"class":490},[59,1200,167],{"class":109},[59,1202,1203],{"class":113},"enc), enc\n",[59,1205,1207,1210,1213],{"class":61,"line":1206},8,[59,1208,1209],{"class":109},"        except",[59,1211,1212],{"class":173}," UnicodeDecodeError",[59,1214,1189],{"class":113},[59,1216,1218],{"class":61,"line":1217},9,[59,1219,1220],{"class":109},"            continue\n",[59,1222,1224,1227,1230,1232,1234,1236,1238,1240,1242,1245,1247,1250,1252,1255],{"class":61,"line":1223},10,[59,1225,1226],{"class":109},"    raise",[59,1228,1229],{"class":173}," ValueError",[59,1231,186],{"class":113},[59,1233,189],{"class":109},[59,1235,192],{"class":69},[59,1237,196],{"class":195},[59,1239,320],{"class":113},[59,1241,205],{"class":195},[59,1243,1244],{"class":69},": none of ",[59,1246,196],{"class":195},[59,1248,1249],{"class":113},"candidates",[59,1251,205],{"class":195},[59,1253,1254],{"class":69}," decoded the file\"",[59,1256,224],{"class":113},[59,1258,1260],{"class":61,"line":1259},11,[59,1261,127],{"emptyLinePlaceholder":126},[59,1263,1265,1267,1269,1271,1273,1275,1277,1279,1281,1283],{"class":61,"line":1264},12,[59,1266,133],{"class":109},[59,1268,136],{"class":113},[59,1270,139],{"class":109},[59,1272,142],{"class":113},[59,1274,145],{"class":69},[59,1276,83],{"class":113},[59,1278,150],{"class":69},[59,1280,83],{"class":113},[59,1282,155],{"class":69},[59,1284,158],{"class":113},[59,1286,1288,1291,1293],{"class":61,"line":1287},13,[59,1289,1290],{"class":113},"    df, used ",[59,1292,167],{"class":109},[59,1294,1295],{"class":113}," read_excel_csv(name)\n",[59,1297,1299,1301,1303,1305,1307,1309,1311,1313,1315,1318,1320,1323,1325,1327,1329,1331,1333,1335,1337,1340,1342,1344],{"class":61,"line":1298},14,[59,1300,183],{"class":173},[59,1302,186],{"class":113},[59,1304,189],{"class":109},[59,1306,192],{"class":69},[59,1308,196],{"class":195},[59,1310,199],{"class":113},[59,1312,202],{"class":109},[59,1314,205],{"class":195},[59,1316,1317],{"class":69}," read as ",[59,1319,196],{"class":195},[59,1321,1322],{"class":113},"used",[59,1324,603],{"class":109},[59,1326,205],{"class":195},[59,1328,208],{"class":195},[59,1330,621],{"class":113},[59,1332,301],{"class":173},[59,1334,83],{"class":113},[59,1336,301],{"class":173},[59,1338,1339],{"class":113},"]",[59,1341,205],{"class":195},[59,1343,192],{"class":69},[59,1345,224],{"class":113},[10,1347,1348,1349,1351,1352,1355,1356,1359,1360,1363,1364,1367,1368,1371,1372,1375],{},"Note that ",[19,1350,276],{}," itself rarely fails — only five byte values are undefined in it — so it must come ",[14,1353,1354],{},"last","; put the code page your senders actually use there (",[19,1357,1358],{},"cp1250"," for Central Europe, ",[19,1361,1362],{},"cp1251"," for Cyrillic). Statistical detectors such as ",[19,1365,1366],{},"charset-normalizer"," can help with unknown files, but treat their answer as a guess: on this short sample it labelled the Windows-1252 file as ",[19,1369,1370],{},"mac_latin2",", an old Macintosh encoding that turns ",[19,1373,1374],{},"ü"," into a different letter altogether. Short files simply do not contain enough text for statistics to be reliable. Detection is no substitute for knowing where files come from.",[47,1377,1379],{"id":1378},"step-5-fix-it-at-the-source","Step 5 — Fix it at the source",[10,1381,1382,1383,1385,1386,1389,1390,1393,1394,1397,1398,1400,1401,1405,1406,45],{},"The durable fix is to ask for ",[14,1384,86],{}," when you can, which every current Excel offers in the Save As list, or better, for the ",[19,1387,1388],{},".xlsx"," itself, which has no encoding question at all — ",[19,1391,1392],{},"pd.read_excel"," reads text as Unicode. If you produce CSVs that go ",[14,1395,1396],{},"back"," to Excel users, write them with ",[19,1399,384],{}," so Excel decodes them correctly, as explained in ",[41,1402,1404],{"href":1403},"\u002Fgetting-started-with-python-excel-automation\u002Fhandling-excel-file-formats-and-conversions\u002Fwrite-csv-files-that-open-correctly-in-excel\u002F","Write CSV Files That Open Correctly in Excel",". European exports often bring a second problem with them — semicolons and decimal commas — covered in ",[41,1407,1409],{"href":1408},"\u002Fgetting-started-with-python-excel-automation\u002Fhandling-excel-file-formats-and-conversions\u002Fread-csv-exports-with-semicolons-and-decimal-commas\u002F","Read CSV Exports with Semicolons and Decimal Commas",[279,1411,288,1416,288,1419,288,1422,288,1425,288,1430,288,1435,288,1440,288,1446,288,1453,288,1457,288,1460,288,1465,288,1468,288,1473,288,1476,288,1479,288,1483,288,1487,288,1492,288,1496,288,1501,288,1504,288,1509,288,1512,288,1516,288,1520,288,1524,288,1529,288,1532,288,1537,288,1540],{"viewBox":1412,"role":282,"ariaLabelledBy":1413,"xmlns":286,"style":287},"4 0 752 190",[1414,1415],"enc-formats-t","enc-formats-d",[290,1417,1418],{"id":1414},"Excel's text export options and how to read them",[294,1420,1421],{"id":1415},"CSV (Comma delimited) uses the Windows code page and commas or the regional list separator. CSV UTF-8 uses UTF-8 with a byte order mark. Unicode Text uses UTF-16 with tabs. Each needs a different read_csv call.",[298,1423],{"x":300,"y":301,"width":302,"height":1424,"fill":304},"190",[298,1426],{"x":363,"y":308,"width":1427,"height":1428,"rx":1429,"fill":426,"stroke":427,"style":314},"228.0","158.0","14",[29,1431,82],{"x":1432,"y":1433,"style":1434},"134.0","44.0","font-size:14px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle",[61,1436],{"x1":1437,"y1":1438,"x2":1439,"y2":1438,"stroke":427,"style":951},"36.0","57.0","232.0",[29,1441,1445],{"x":1442,"y":1443,"style":1444},"60.0","77.0","font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:start","Windows code page",[316,1447,1449],{"transform":1448},"translate(38.5,65.5) scale(0.625)",[320,1450],{"d":1451,"fill":323,"stroke":427,"style":1452},"M12 3.5l9.5 17h-19z M12 10v5 M12 17.8v.4","stroke-width:2.88px;stroke-linecap:round;stroke-linejoin:round",[29,1454,1456],{"x":1442,"y":1455,"style":1444},"94.0","(cp1252…)",[29,1458,422],{"x":1442,"y":1459,"style":1444},"120.0",[316,1461,1463],{"transform":1462},"translate(38.5,108.5) scale(0.625)",[320,1464],{"d":1451,"fill":323,"stroke":427,"style":1452},[29,1466,443],{"x":1442,"y":1467,"style":1444},"146.0",[316,1469,1471],{"transform":1470},"translate(38.5,134.5) scale(0.625)",[320,1472],{"d":1451,"fill":323,"stroke":427,"style":1452},[298,1474],{"x":1475,"y":308,"width":1427,"height":1428,"rx":1429,"fill":367,"stroke":368,"style":314},"266.0",[29,1477,240],{"x":328,"y":1433,"style":1478},"font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle",[61,1480],{"x1":1481,"y1":1438,"x2":1482,"y2":1438,"stroke":368,"style":951},"282.0","478.0",[29,1484,1486],{"x":1485,"y":1443,"style":1444},"306.0","UTF-8 with BOM",[316,1488,1490],{"transform":1489},"translate(284.5,65.5) scale(0.625)",[320,1491],{"d":374,"fill":323,"stroke":368,"style":1452},[29,1493,1495],{"x":1485,"y":1494,"style":1444},"103.0","starts EF BB BF",[316,1497,1499],{"transform":1498},"translate(284.5,91.5) scale(0.625)",[320,1500],{"d":374,"fill":323,"stroke":368,"style":1452},[29,1502,384],{"x":1485,"y":1503,"style":1444},"129.0",[316,1505,1507],{"transform":1506},"translate(284.5,117.5) scale(0.625)",[320,1508],{"d":374,"fill":323,"stroke":368,"style":1452},[298,1510],{"x":1511,"y":308,"width":1427,"height":1428,"rx":1429,"fill":312,"stroke":313,"style":314},"512.0",[29,1513,90],{"x":1514,"y":1433,"style":1515},"626.0","font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle",[61,1517],{"x1":1518,"y1":1438,"x2":1519,"y2":1438,"stroke":313,"style":951},"528.0","724.0",[29,1521,1523],{"x":1522,"y":1443,"style":1444},"552.0","UTF-16 LE, tab separated",[316,1525,1527],{"transform":1526},"translate(530.5,65.5) scale(0.625)",[320,1528],{"d":1451,"fill":323,"stroke":427,"style":1452},[29,1530,1531],{"x":1522,"y":1494,"style":1444},"starts FF FE",[316,1533,1535],{"transform":1534},"translate(530.5,91.5) scale(0.625)",[320,1536],{"d":1451,"fill":323,"stroke":427,"style":1452},[29,1538,1539],{"x":1522,"y":1503,"style":1444},"utf-16 and sep=\"\\t\"",[316,1541,1543],{"transform":1542},"translate(530.5,117.5) scale(0.625)",[320,1544],{"d":1451,"fill":323,"stroke":427,"style":1452},[47,1546,1548],{"id":1547},"step-6-repair-text-that-was-already-decoded-wrongly","Step 6 — Repair text that was already decoded wrongly",[10,1550,1551,1552,87,1555,1558,1559,1562],{},"Sometimes the damage happened upstream: a system read a UTF-8 file as Windows-1252, saved the result, and the sheet now contains ",[19,1553,1554],{},"Müller",[19,1556,1557],{},"Café",". That pattern — ",[19,1560,1561],{},"Ã"," followed by another character — is the signature of UTF-8 bytes decoded with a single-byte code page, and it can usually be reversed by doing the opposite round trip:",[24,1564,1566],{"className":100,"code":1565,"language":102,"meta":30,"style":30},"garbled = pd.Series([\"Müller GmbH\", \"Café Señor\", \"Plain ASCII\"])\n\ndef undo_mojibake(text):\n    try:\n        return text.encode(\"cp1252\").decode(\"utf-8\")\n    except (UnicodeEncodeError, UnicodeDecodeError):\n        return text\n\nprint(garbled.map(undo_mojibake).tolist())\n",[19,1567,1568,1593,1597,1607,1614,1630,1647,1654,1658],{"__ignoreMap":30},[59,1569,1570,1573,1575,1578,1581,1583,1586,1588,1591],{"class":61,"line":62},[59,1571,1572],{"class":113},"garbled ",[59,1574,167],{"class":109},[59,1576,1577],{"class":113}," pd.Series([",[59,1579,1580],{"class":69},"\"Müller GmbH\"",[59,1582,83],{"class":113},[59,1584,1585],{"class":69},"\"Café Señor\"",[59,1587,83],{"class":113},[59,1589,1590],{"class":69},"\"Plain ASCII\"",[59,1592,802],{"class":113},[59,1594,1595],{"class":61,"line":123},[59,1596,127],{"emptyLinePlaceholder":126},[59,1598,1599,1601,1604],{"class":61,"line":130},[59,1600,1072],{"class":109},[59,1602,1603],{"class":1075}," undo_mojibake",[59,1605,1606],{"class":113},"(text):\n",[59,1608,1609,1612],{"class":61,"line":161},[59,1610,1611],{"class":109},"    try",[59,1613,1189],{"class":113},[59,1615,1616,1618,1621,1623,1626,1628],{"class":61,"line":180},[59,1617,1141],{"class":109},[59,1619,1620],{"class":113}," text.encode(",[59,1622,496],{"class":69},[59,1624,1625],{"class":113},").decode(",[59,1627,714],{"class":69},[59,1629,224],{"class":113},[59,1631,1632,1635,1637,1640,1642,1645],{"class":61,"line":559},[59,1633,1634],{"class":109},"    except",[59,1636,142],{"class":113},[59,1638,1639],{"class":173},"UnicodeEncodeError",[59,1641,83],{"class":113},[59,1643,1644],{"class":173},"UnicodeDecodeError",[59,1646,158],{"class":113},[59,1648,1649,1651],{"class":61,"line":587},[59,1650,1141],{"class":109},[59,1652,1653],{"class":113}," text\n",[59,1655,1656],{"class":61,"line":1206},[59,1657,127],{"emptyLinePlaceholder":126},[59,1659,1660,1662],{"class":61,"line":1217},[59,1661,864],{"class":173},[59,1663,1664],{"class":113},"(garbled.map(undo_mojibake).tolist())\n",[10,1666,1667,1668,1670,1671,1673,1674,1677,1678,45],{},"Re-encoding with ",[19,1669,276],{}," recovers the original bytes, and decoding them as UTF-8 recovers the text; values that were never garbled raise an error in one direction or the other and are returned unchanged. Check a sample by eye before applying it to a whole column, because a value that legitimately contains ",[19,1672,1561],{}," would be \"repaired\" too. The ",[19,1675,1676],{},"ftfy"," package automates this and handles many more patterns, as described in ",[41,1679,1681],{"href":1680},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-garbled-characters-in-excel-data\u002F","Fix Garbled Characters in Excel Data",[47,1683,1685],{"id":1684},"performance-notes","Performance notes",[10,1687,1688,1689,1692,1693,1696],{},"The encoding barely affects reading speed; decoding cp1252 or UTF-16 costs about the same as UTF-8. The try-in-order helper reads the file a second time when the first encoding fails, which only matters for very large files — for those, decode just the first megabyte to choose the encoding (",[19,1690,1691],{},"Path(p).read_bytes()[:1_000_000].decode(enc)","), then read the whole file once with the winner. A UTF-8 character can be cut in half at the one-megabyte boundary, so trim a few bytes from the end of the sample, or decode it with the incremental decoder from the ",[19,1694,1695],{},"codecs"," module, before treating a failure as proof.",[47,1698,1700],{"id":1699},"common-pitfalls","Common pitfalls",[229,1702,1703,1709,1716,1731,1741],{},[232,1704,1705,1708],{},[251,1706,1707],{},"Using latin-1 because it never fails."," It mis-decodes €, curly quotes and dashes from Windows files.",[232,1710,1711,1715],{},[251,1712,1713,45],{},[19,1714,914],{}," Destroys characters; use it only to locate the problem, never in production.",[232,1717,1718,1724,1725,1728,1729,45],{},[251,1719,1720,1721,1723],{},"Ignoring the BOM in the ",[19,1722,660],{}," module."," The first column name gains ",[19,1726,1727],{},"\\ufeff","; use ",[19,1730,652],{},[232,1732,1733,1736,1737,1740],{},[251,1734,1735],{},"Reading Unicode Text as CSV."," It is UTF-16 ",[14,1738,1739],{},"and"," tab-separated.",[232,1742,1743,1746,1747,1749,1750,1753],{},[251,1744,1745],{},"Assuming one code page."," A Polish or Greek colleague's \"CSV\" uses ",[19,1748,1358],{}," or ",[19,1751,1752],{},"cp1253","; ask, or record the source per sender.",[47,1755,1757],{"id":1756},"conclusion","Conclusion",[10,1759,1760,1761,1763,1764,1766,1767,1165,1769,1771,1772,1774,1775,1749,1777,1780,1781,1783],{},"A ",[19,1762,1644],{}," on a CSV from Excel means the file is not plain UTF-8. Look at the first bytes: ",[19,1765,360],{}," is CSV UTF-8 (read with ",[19,1768,652],{},[19,1770,396],{}," is Unicode Text (UTF-16 with tabs), and no marker means the Windows code page, usually ",[19,1773,276],{},". Read with that encoding, try strict encodings before permissive ones when the source is unknown, never paper over the error with ",[19,1776,898],{},[19,1778,1779],{},"errors=\"replace\"",", and ask for CSV UTF-8 or the ",[19,1782,1388],{}," itself wherever you can.",[47,1785,1787],{"id":1786},"frequently-asked-questions","Frequently asked questions",[10,1789,1790,1793],{},[251,1791,1792],{},"Why do I get \"utf-8 codec can't decode byte 0xfc\" reading an Excel CSV?","\nThe file was saved with Excel's plain CSV option, which uses the Windows code page rather than UTF-8. Byte 0xfc is ü in Windows-1252. Read it with pd.read_csv(path, encoding=\"cp1252\").",[10,1795,1796,1799],{},[251,1797,1798],{},"What does \"can't decode byte 0xff in position 0\" mean?","\nThe file starts with FF FE, the UTF-16 byte order mark, which Excel's Unicode Text format writes. Read it with encoding=\"utf-16\" and sep=\"\\t\", since that format is tab-separated.",[10,1801,1802,1805],{},[251,1803,1804],{},"Why does my first column name start with \\ufeff?","\nThe file is CSV UTF-8 with a byte order mark, and it was read with plain utf-8 by the csv module or open(). Use encoding=\"utf-8-sig\", which removes the mark.",[10,1807,1808,1811],{},[251,1809,1810],{},"Is encoding=\"latin-1\" a safe fix?","\nNo. It never raises an error, but it decodes Windows-specific characters such as the euro sign, curly quotes and dashes as invisible control characters. Use cp1252 for files saved by Western-European Excel.",[47,1813,1815],{"id":1814},"related","Related",[229,1817,1818,1824,1829,1836],{},[232,1819,1820,1821,1823],{},"Up one level: ",[41,1822,44],{"href":43}," — fixes for common failures.",[232,1825,1826,1828],{},[41,1827,1681],{"href":1680}," — repairing text that was decoded wrongly earlier.",[232,1830,1831,1835],{},[41,1832,1834],{"href":1833},"\u002Fgetting-started-with-python-excel-automation\u002Fhandling-excel-file-formats-and-conversions\u002Fconvert-csv-files-to-excel-with-python\u002F","Convert CSV Files to Excel with Python"," — once the CSV reads cleanly.",[232,1837,1838,1842],{},[41,1839,1841],{"href":1840},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-non-printable-characters-from-excel-data\u002F","Remove Non-Printable Characters from Excel Data"," — cleaning up stray control characters.",[1844,1845,1846],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}",{"title":30,"searchDepth":123,"depth":123,"links":1848},[1849,1850,1851,1852,1853,1854,1855,1856,1857,1858,1859,1860],{"id":49,"depth":123,"text":50},{"id":93,"depth":123,"text":94},{"id":450,"depth":123,"text":451},{"id":805,"depth":123,"text":806},{"id":1058,"depth":123,"text":1059},{"id":1378,"depth":123,"text":1379},{"id":1547,"depth":123,"text":1548},{"id":1684,"depth":123,"text":1685},{"id":1699,"depth":123,"text":1700},{"id":1756,"depth":123,"text":1757},{"id":1786,"depth":123,"text":1787},{"id":1814,"depth":123,"text":1815},"2026-10-02","\"utf-8 codec can't decode byte 0xfc\" or \"byte 0xff in position 0\" when reading a CSV someone saved from Excel means the file is not UTF-8: Excel's plain \"CSV\" format uses the Windows code page, \"Unicode Text\" is UTF-16, and \"CSV UTF-8\" adds a byte order mark. Identify which one you have, read it with the right encoding, and stop silent corruption from latin-1 and errors=\"replace\".","md",[1865,1867,1869,1871],{"q":1792,"a":1866},"The file was saved with Excel's plain CSV option, which uses the Windows code page rather than UTF-8. Byte 0xfc is ü in Windows-1252. Read it with pd.read_csv(path, encoding=\"cp1252\").",{"q":1798,"a":1868},"The file starts with FF FE, the UTF-16 byte order mark, which Excel's Unicode Text format writes. Read it with encoding=\"utf-16\" and sep=\"\\t\", since that format is tab-separated.",{"q":1804,"a":1870},"The file is CSV UTF-8 with a byte order mark, and it was read with plain utf-8 by the csv module or open(). Use encoding=\"utf-8-sig\", which removes the mark.",{"q":1810,"a":1872},"No. It never raises an error, but it decodes Windows-specific characters such as the euro sign, curly quotes and dashes as invisible control characters. Use cp1252 for files saved by Western-European Excel.","fix unicodedecodeerror reading csv saved by excel",null,{"breadcrumb":1876},[1877,1880,1883],{"name":1878,"item":1879},"Home","\u002F",{"name":1881,"item":1882},"Getting Started with Python Excel Automation","\u002Fgetting-started-with-python-excel-automation\u002F",{"name":44,"item":43},"\u002Fgetting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002Ffix-unicodedecodeerror-when-reading-csv-files-saved-by-excel",{"title":1886,"description":1887},"Fix UnicodeDecodeError Reading CSV Files Saved by Excel","Why pd.read_csv raises \"utf-8 codec cannot decode byte 0xfc \u002F 0xff\": Excel CSV in cp1252, Unicode Text in UTF-16, CSV UTF-8 with a BOM. Detect the encoding from the first bytes, read with cp1252, utf-16 or utf-8-sig, and avoid latin-1 and errors=\"replace\" traps.","fix-unicodedecodeerror-when-reading-csv-files-saved-by-excel","getting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002Ffix-unicodedecodeerror-when-reading-csv-files-saved-by-excel\u002Findex",[1891,1892],"troubleshooting","file-formats","how-to","KCqaL-6LyrKnSLnuloy9aL4keEpiJQCGANZ7sP134Us",[1896,1900],{"title":1897,"path":1898,"stem":1899,"children":-1},"Fix \"Object of type Timestamp is not JSON serializable\" When Exporting Excel Data","\u002Fgetting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002Ffix-timestamp-is-not-json-serializable-when-exporting-excel-data","getting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002Ffix-timestamp-is-not-json-serializable-when-exporting-excel-data\u002Findex",{"title":1901,"path":1902,"stem":1903,"children":-1},"Fix \"Unnamed: 0\" Columns When Reading Excel with pandas","\u002Fgetting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002Ffix-unnamed-0-columns-when-reading-excel","getting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002Ffix-unnamed-0-columns-when-reading-excel\u002Findex",[1905,1907],{"path":1906,"title":1881},"\u002Fgetting-started-with-python-excel-automation",{"path":1908,"title":44},"\u002Fgetting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors",[],[1911,1915,1918,1921,1924,1927],{"path":1912,"title":1913,"kind":1914},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fhandle-permission-denied-when-writing-excel-in-python","Handle Permission Denied When Writing Excel in Python","fix",{"path":1916,"title":1917,"kind":1893},"\u002Fgetting-started-with-python-excel-automation\u002Fusing-openpyxl-for-excel-file-manipulation\u002Fwhat-openpyxl-loses-when-saving-an-existing-workbook","What openpyxl Loses When You Save an Existing Workbook",{"path":1919,"title":1920,"kind":1914},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-dates-and-times-in-excel-data\u002Ffix-excel-serial-numbers-showing-instead-of-dates","Fix Excel Serial Numbers Showing Instead of Dates",{"path":1922,"title":1923,"kind":1893},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-dates-and-times-in-excel-data\u002Fhandle-daylight-saving-time-in-excel-timestamps","Handle Daylight Saving Time in Excel Timestamps",{"path":1925,"title":1926,"kind":1893},"\u002Fautomating-reporting-workflows\u002Fautomating-excel-online-with-microsoft-graph\u002Fhandle-microsoft-graph-throttling-for-excel","Handle Microsoft Graph Throttling in Excel Automation",{"path":1928,"title":1929,"kind":1930},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation","Error Handling and Logging in Excel Automation","guide",1790972067556]