[{"data":1,"prerenderedAt":2048},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-category-values-with-a-mapping-table":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-category-values-with-a-mapping-table":2014,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-category-values-with-a-mapping-table":2023,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-category-values-with-a-mapping-table":2028,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-category-values-with-a-mapping-table":2029},{"id":4,"title":5,"body":6,"dateModified":1979,"datePublished":1979,"description":1980,"extension":1981,"faq":1982,"keyword":1991,"kind":1992,"meta":1993,"navigation":311,"path":2002,"seo":2003,"slug":2006,"stem":2007,"topics":2008,"type":2012,"__hash__":2013},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-category-values-with-a-mapping-table\u002Findex.md","Standardise Category Values with a Mapping Table in pandas",{"type":7,"value":8,"toc":1964},"minimark",[9,24,210,215,243,261,265,275,360,386,469,475,479,482,608,619,623,626,806,809,1032,1036,1054,1153,1156,1160,1167,1326,1337,1341,1344,1497,1508,1668,1672,1675,1791,1794,1798,1862,1866,1883,1887,1899,1903,1909,1915,1921,1927,1931,1960],[10,11,12,13,17,18,23],"p",{},"Free-text categories drift. One region is entered as \"North America\", \"N. America\", \"north america \" with a trailing space, \"NA\", and once as \"Nroth America\". Every variant becomes a separate group in a pivot table, and the totals by region are wrong in a way that is easy to overlook. Fixing it with a chain of ",[14,15,16],"code",{},"replace"," calls in code works once and then rots, because the people who know that \"NA\" means the Americas — and who will invent the next variant — are not the people who edit the code. This page, part of ",[19,20,22],"a",{"href":21},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002F","Cleaning Excel Data With Pandas",", keeps the mapping in an Excel table the business can maintain, and builds a loop that catches every new variant.",[25,26,34,35,34,39,34,43,34,51,34,61,34,72,34,79,34,84,34,92,34,98,34,102,34,107,34,115,34,121,34,124,34,128,34,131,34,133,34,136,34,139,34,142,34,146,34,149,34,151,34,156,34,160,34,163,34,167,34,170,34,172,34,175,34,178,34,184,34,191,34,196,34,199,34,201,34,206],"svg",{"viewBox":27,"role":28,"ariaLabelledBy":29,"xmlns":32,"style":33},"4 0 752 355","img",[30,31],"map-fan-t","map-fan-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[36,37,38],"title",{"id":30},"Many spellings, one standard category",[40,41,42],"desc",{"id":31},"Five raw region values — North America, N. America, north america with a trailing space, NA and the typo Nroth America — fan in to a single standard value, Americas. The first four are matched through the mapping table after normalising; the typo is reported with a suggestion for review.",[44,45],"rect",{"x":46,"y":47,"width":48,"height":49,"fill":50},"4","0","752","355","#ffffff",[44,52],{"x":53,"y":54,"width":55,"height":56,"rx":57,"fill":58,"stroke":59,"style":60},"540.0","131.0","200.0","93.0","12","#d9f4f1","var(--teal,#0f9488)","stroke-width:2px",[62,63,65],"g",{"transform":64},"translate(628.0,144.0) scale(1.000)",[66,67],"path",{"d":68,"fill":69,"stroke":70,"style":71},"M5 12.5l4.5 4.5L19 7","none","var(--teal-ink,#0b6157)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[73,74,78],"text",{"x":75,"y":76,"style":77},"640.0","190.5","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Americas",[73,80,83],{"x":75,"y":81,"style":82},"210.0","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","the standard value",[44,85],{"x":86,"y":87,"width":88,"height":89,"rx":57,"fill":90,"stroke":91,"style":60},"20.0","16.0","230.0","50.0","#e7ebef","var(--line,#cdd5e6)",[73,93,97],{"x":94,"y":95,"style":96},"135.0","45.5","font-size:13px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","North America",[66,99],{"d":100,"fill":69,"stroke":101,"style":60},"M254.0,41.0 C394.0,41.0 394.0,177.5 530.0,177.5","var(--brand,#5b5cf0)",[103,104],"polygon",{"points":105,"fill":106},"534.0,177.5 525.0,182.5 525.0,172.5","#5b5cf0",[44,108],{"x":109,"y":110,"width":111,"height":112,"rx":113,"fill":50,"stroke":91,"style":114},"366.1","99.2","57.9","19.0","9","stroke-width:1px",[73,116,120],{"x":117,"y":118,"style":119},"395.0","112.8","font-size:10.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","mapped",[44,122],{"x":86,"y":123,"width":88,"height":89,"rx":57,"fill":90,"stroke":91,"style":60},"78.0",[73,125,127],{"x":94,"y":126,"style":96},"107.5","N. America",[66,129],{"d":130,"fill":69,"stroke":101,"style":60},"M254.0,103.0 C394.0,103.0 394.0,177.5 530.0,177.5",[103,132],{"points":105,"fill":106},[44,134],{"x":109,"y":135,"width":111,"height":112,"rx":113,"fill":50,"stroke":91,"style":114},"130.2",[73,137,120],{"x":117,"y":138,"style":119},"143.8",[44,140],{"x":86,"y":141,"width":88,"height":89,"rx":57,"fill":90,"stroke":91,"style":60},"140.0",[73,143,145],{"x":94,"y":144,"style":96},"169.5","\"north america \"",[66,147],{"d":148,"fill":69,"stroke":101,"style":60},"M254.0,165.0 C394.0,165.0 394.0,177.5 530.0,177.5",[103,150],{"points":105,"fill":106},[44,152],{"x":153,"y":154,"width":155,"height":112,"rx":113,"fill":50,"stroke":91,"style":114},"357.9","161.2","74.3",[73,157,159],{"x":117,"y":158,"style":119},"174.8","normalised",[44,161],{"x":86,"y":162,"width":88,"height":89,"rx":57,"fill":90,"stroke":91,"style":60},"202.0",[73,164,166],{"x":94,"y":165,"style":96},"231.5","NA",[66,168],{"d":169,"fill":69,"stroke":101,"style":60},"M254.0,227.0 C394.0,227.0 394.0,177.5 530.0,177.5",[103,171],{"points":105,"fill":106},[44,173],{"x":109,"y":174,"width":111,"height":112,"rx":113,"fill":50,"stroke":91,"style":114},"192.2",[73,176,120],{"x":117,"y":177,"style":119},"205.8",[44,179],{"x":86,"y":180,"width":88,"height":181,"rx":57,"fill":182,"stroke":183,"style":60},"264.0","75.0","#fdefd8","var(--gold,#b4740a)",[62,185,187],{"transform":186},"translate(123.0,277.0) scale(1.000)",[66,188],{"d":189,"fill":69,"stroke":190,"style":71},"M12 3.5l9.5 17h-19z M12 10v5 M12 17.8v.4","var(--gold-ink,#7a4e06)",[73,192,195],{"x":94,"y":193,"style":194},"323.0","font-size:13px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Nroth America",[66,197],{"d":198,"fill":69,"stroke":101,"style":60},"M254.0,301.5 C394.0,301.5 394.0,177.5 530.0,177.5",[103,200],{"points":105,"fill":106},[44,202],{"x":203,"y":204,"width":205,"height":112,"rx":113,"fill":50,"stroke":91,"style":114},"359.4","229.5","71.3",[73,207,209],{"x":117,"y":208,"style":119},"243.0","suggested",[211,212,214],"h2",{"id":213},"prerequisites","Prerequisites",[216,217,222],"pre",{"className":218,"code":219,"language":220,"meta":221,"style":221},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[14,223,224],{"__ignoreMap":221},[225,226,229,233,237,240],"span",{"class":227,"line":228},"line",1,[225,230,232],{"class":231},"sMTad","pip",[225,234,236],{"class":235},"srMev"," install",[225,238,239],{"class":235}," pandas",[225,241,242],{"class":235}," openpyxl\n",[10,244,245,248,249,252,253,256,257,260],{},[14,246,247],{},"sales.xlsx"," has ten orders with ten spellings of three regions. ",[14,250,251],{},"region_mapping.xlsx"," has a Mapping sheet with a ",[14,254,255],{},"raw"," column and a ",[14,258,259],{},"standard"," column — the table a sales operations person would own.",[211,262,264],{"id":263},"step-1-stop-pandas-turning-na-into-a-missing-value","Step 1 — Stop pandas turning \"NA\" into a missing value",[10,266,267,268,270,271,274],{},"Before any cleaning, look at order 4. Its region is the code ",[14,269,166],{},", meaning North America — and ",[14,272,273],{},"read_excel"," does not return it:",[216,276,280],{"className":277,"code":278,"language":279,"meta":221,"style":221},"language-python shiki shiki-themes github-light github-dark-high-contrast","import re\nimport pandas as pd\n\ndefault = pd.read_excel(\"sales.xlsx\")\nprint(default.loc[3, \"region\"])\n# nan\n","python",[14,281,282,292,306,313,331,353],{"__ignoreMap":221},[225,283,284,288],{"class":227,"line":228},[225,285,287],{"class":286},"s-kum","import",[225,289,291],{"class":290},"skGVy"," re\n",[225,293,295,297,300,303],{"class":227,"line":294},2,[225,296,287],{"class":286},[225,298,299],{"class":290}," pandas ",[225,301,302],{"class":286},"as",[225,304,305],{"class":290}," pd\n",[225,307,309],{"class":227,"line":308},3,[225,310,312],{"emptyLinePlaceholder":311},true,"\n",[225,314,316,319,322,325,328],{"class":227,"line":315},4,[225,317,318],{"class":290},"default ",[225,320,321],{"class":286},"=",[225,323,324],{"class":290}," pd.read_excel(",[225,326,327],{"class":235},"\"sales.xlsx\"",[225,329,330],{"class":290},")\n",[225,332,334,338,341,344,347,350],{"class":227,"line":333},5,[225,335,337],{"class":336},"sP0c6","print",[225,339,340],{"class":290},"(default.loc[",[225,342,343],{"class":336},"3",[225,345,346],{"class":290},", ",[225,348,349],{"class":235},"\"region\"",[225,351,352],{"class":290},"])\n",[225,354,356],{"class":227,"line":355},6,[225,357,359],{"class":358},"s-wDw","# nan\n",[10,361,362,363,346,365,346,368,346,371,346,374,377,378,380,381,385],{},"pandas treats a list of strings as missing by default — ",[14,364,166],{},[14,366,367],{},"N\u002FA",[14,369,370],{},"NULL",[14,372,373],{},"nan",[14,375,376],{},"#N\u002FA"," and others — and ",[14,379,166],{}," is on it. A region code, a country code for Namibia or a product code can disappear this way without a warning. For category columns, switch the default list off and treat only genuinely empty cells as missing, for the data ",[382,383,384],"em",{},"and"," the mapping table:",[216,387,389],{"className":277,"code":388,"language":279,"meta":221,"style":221},"READ = dict(keep_default_na=False, na_values=[\"\"])\nsales = pd.read_excel(\"sales.xlsx\", **READ)\nprint(sales.loc[3, \"region\"])\n# NA\n",[14,390,391,429,449,464],{"__ignoreMap":221},[225,392,393,396,399,402,405,409,411,414,416,419,421,424,427],{"class":227,"line":228},[225,394,395],{"class":336},"READ",[225,397,398],{"class":286}," =",[225,400,401],{"class":336}," dict",[225,403,404],{"class":290},"(",[225,406,408],{"class":407},"sa561","keep_default_na",[225,410,321],{"class":286},[225,412,413],{"class":336},"False",[225,415,346],{"class":290},[225,417,418],{"class":407},"na_values",[225,420,321],{"class":286},[225,422,423],{"class":290},"[",[225,425,426],{"class":235},"\"\"",[225,428,352],{"class":290},[225,430,431,434,436,438,440,442,445,447],{"class":227,"line":294},[225,432,433],{"class":290},"sales ",[225,435,321],{"class":286},[225,437,324],{"class":290},[225,439,327],{"class":235},[225,441,346],{"class":290},[225,443,444],{"class":286},"**",[225,446,395],{"class":336},[225,448,330],{"class":290},[225,450,451,453,456,458,460,462],{"class":227,"line":308},[225,452,337],{"class":336},[225,454,455],{"class":290},"(sales.loc[",[225,457,343],{"class":336},[225,459,346],{"class":290},[225,461,349],{"class":235},[225,463,352],{"class":290},[225,465,466],{"class":227,"line":315},[225,467,468],{"class":358},"# NA\n",[10,470,471,472,474],{},"Without this, the mapping table's own ",[14,473,166],{}," row would also be read as missing, and a missing key would match every missing region — a coincidence that looks like success.",[211,476,478],{"id":477},"step-2-normalise-the-key-on-both-sides","Step 2 — Normalise the key on both sides",[10,480,481],{},"Most variants differ only in case, spacing and punctuation. Normalise them to one key before looking anything up, on both the data and the mapping table, so the table needs one row per genuinely different spelling rather than one per typing habit:",[216,483,485],{"className":277,"code":484,"language":279,"meta":221,"style":221},"def key(value):\n    if pd.isna(value):\n        return value\n    s = str(value).casefold().strip()\n    s = re.sub(r\"[^\\w\\s]\", \" \", s)          # punctuation to spaces: \"N. America\" -> \"n  america\"\n    return re.sub(r\"\\s+\", \" \", s).strip()\n\nprint(sales[\"region\"].map(key).value_counts().to_dict())\n",[14,486,487,499,507,515,528,564,590,595],{"__ignoreMap":221},[225,488,489,492,496],{"class":227,"line":228},[225,490,491],{"class":286},"def",[225,493,495],{"class":494},"s_Opv"," key",[225,497,498],{"class":290},"(value):\n",[225,500,501,504],{"class":227,"line":294},[225,502,503],{"class":286},"    if",[225,505,506],{"class":290}," pd.isna(value):\n",[225,508,509,512],{"class":227,"line":308},[225,510,511],{"class":286},"        return",[225,513,514],{"class":290}," value\n",[225,516,517,520,522,525],{"class":227,"line":315},[225,518,519],{"class":290},"    s ",[225,521,321],{"class":286},[225,523,524],{"class":336}," str",[225,526,527],{"class":290},"(value).casefold().strip()\n",[225,529,530,532,534,537,540,543,545,548,551,553,555,558,561],{"class":227,"line":333},[225,531,519],{"class":290},[225,533,321],{"class":286},[225,535,536],{"class":290}," re.sub(",[225,538,539],{"class":286},"r",[225,541,542],{"class":235},"\"",[225,544,423],{"class":336},[225,546,547],{"class":286},"^",[225,549,550],{"class":336},"\\w\\s]",[225,552,542],{"class":235},[225,554,346],{"class":290},[225,556,557],{"class":235},"\" \"",[225,559,560],{"class":290},", s)          ",[225,562,563],{"class":358},"# punctuation to spaces: \"N. America\" -> \"n  america\"\n",[225,565,566,569,571,573,575,578,581,583,585,587],{"class":227,"line":355},[225,567,568],{"class":286},"    return",[225,570,536],{"class":290},[225,572,539],{"class":286},[225,574,542],{"class":235},[225,576,577],{"class":336},"\\s",[225,579,580],{"class":286},"+",[225,582,542],{"class":235},[225,584,346],{"class":290},[225,586,557],{"class":235},[225,588,589],{"class":290},", s).strip()\n",[225,591,593],{"class":227,"line":592},7,[225,594,312],{"emptyLinePlaceholder":311},[225,596,598,600,603,605],{"class":227,"line":597},8,[225,599,337],{"class":336},[225,601,602],{"class":290},"(sales[",[225,604,349],{"class":235},[225,606,607],{"class":290},"].map(key).value_counts().to_dict())\n",[10,609,610,611,614,615,618],{},"\"North America\" and \"north america \" now share a key, and so do \"EMEA\" and \"emea\". \"Asia-Pacific\" becomes ",[14,612,613],{},"asia pacific",", matching the mapping table's \"Asia Pacific\". Normalisation is intentionally mechanical — it never decides that two ",[382,616,617],{},"different"," words mean the same thing. That decision belongs to the mapping table.",[211,620,622],{"id":621},"step-3-load-the-mapping-table-and-check-it-for-conflicts","Step 3 — Load the mapping table and check it for conflicts",[10,624,625],{},"Read the table, build the same keys, and check the one thing that makes a mapping table unusable: a key that maps to two different standard values. Duplicates that agree are harmless; duplicates that disagree must be resolved by whoever owns the table:",[216,627,629],{"className":277,"code":628,"language":279,"meta":221,"style":221},"mapping = pd.read_excel(\"region_mapping.xlsx\", sheet_name=\"Mapping\", **READ)\nmapping[\"key\"] = mapping[\"raw\"].map(key)\n\nconflicts = mapping.groupby(\"key\")[\"standard\"].nunique()\nconflicts = conflicts[conflicts > 1]\nif not conflicts.empty:\n    raise ValueError(f\"mapping keys with more than one standard value: {list(conflicts.index)}\")\n\nlookup = mapping.drop_duplicates(\"key\").set_index(\"key\")[\"standard\"]\nprint(lookup.to_dict())\n",[14,630,631,661,683,687,708,726,737,770,774,798],{"__ignoreMap":221},[225,632,633,636,638,640,643,645,648,650,653,655,657,659],{"class":227,"line":228},[225,634,635],{"class":290},"mapping ",[225,637,321],{"class":286},[225,639,324],{"class":290},[225,641,642],{"class":235},"\"region_mapping.xlsx\"",[225,644,346],{"class":290},[225,646,647],{"class":407},"sheet_name",[225,649,321],{"class":286},[225,651,652],{"class":235},"\"Mapping\"",[225,654,346],{"class":290},[225,656,444],{"class":286},[225,658,395],{"class":336},[225,660,330],{"class":290},[225,662,663,666,669,672,674,677,680],{"class":227,"line":294},[225,664,665],{"class":290},"mapping[",[225,667,668],{"class":235},"\"key\"",[225,670,671],{"class":290},"] ",[225,673,321],{"class":286},[225,675,676],{"class":290}," mapping[",[225,678,679],{"class":235},"\"raw\"",[225,681,682],{"class":290},"].map(key)\n",[225,684,685],{"class":227,"line":308},[225,686,312],{"emptyLinePlaceholder":311},[225,688,689,692,694,697,699,702,705],{"class":227,"line":315},[225,690,691],{"class":290},"conflicts ",[225,693,321],{"class":286},[225,695,696],{"class":290}," mapping.groupby(",[225,698,668],{"class":235},[225,700,701],{"class":290},")[",[225,703,704],{"class":235},"\"standard\"",[225,706,707],{"class":290},"].nunique()\n",[225,709,710,712,714,717,720,723],{"class":227,"line":333},[225,711,691],{"class":290},[225,713,321],{"class":286},[225,715,716],{"class":290}," conflicts[conflicts ",[225,718,719],{"class":286},">",[225,721,722],{"class":336}," 1",[225,724,725],{"class":290},"]\n",[225,727,728,731,734],{"class":227,"line":355},[225,729,730],{"class":286},"if",[225,732,733],{"class":286}," not",[225,735,736],{"class":290}," conflicts.empty:\n",[225,738,739,742,745,747,750,753,757,760,763,766,768],{"class":227,"line":592},[225,740,741],{"class":286},"    raise",[225,743,744],{"class":336}," ValueError",[225,746,404],{"class":290},[225,748,749],{"class":286},"f",[225,751,752],{"class":235},"\"mapping keys with more than one standard value: ",[225,754,756],{"class":755},"sSjpA","{",[225,758,759],{"class":336},"list",[225,761,762],{"class":290},"(conflicts.index)",[225,764,765],{"class":755},"}",[225,767,542],{"class":235},[225,769,330],{"class":290},[225,771,772],{"class":227,"line":597},[225,773,312],{"emptyLinePlaceholder":311},[225,775,777,780,782,785,787,790,792,794,796],{"class":227,"line":776},9,[225,778,779],{"class":290},"lookup ",[225,781,321],{"class":286},[225,783,784],{"class":290}," mapping.drop_duplicates(",[225,786,668],{"class":235},[225,788,789],{"class":290},").set_index(",[225,791,668],{"class":235},[225,793,701],{"class":290},[225,795,704],{"class":235},[225,797,725],{"class":290},[225,799,801,803],{"class":227,"line":800},10,[225,802,337],{"class":336},[225,804,805],{"class":290},"(lookup.to_dict())\n",[10,807,808],{},"Failing loudly on a conflict is the right behaviour: a table that says \"NA\" is both \"Americas\" and \"Not applicable\" cannot be applied, and guessing which row wins would silently misfile orders.",[25,810,34,816,34,819,34,822,34,827,34,833,34,839,34,843,34,849,34,853,34,857,34,861,34,866,34,868,34,871,34,873,34,875,34,878,34,882,34,884,34,887,34,889,34,891,34,894,34,897,34,899,34,901,34,903,34,905,34,908,34,911,34,913,34,915,34,917,34,919,34,922,34,926,34,928,34,931,34,933,34,935,34,938,34,942,34,944,34,947,34,949,34,951,34,954,34,958,34,960,34,963,34,965,34,967,34,970,34,974,34,976,34,979,34,981,34,983,34,988,34,994,34,1000,34,1004,34,1007,34,1013,34,1019,34,1024,34,1028],{"viewBox":811,"role":28,"ariaLabelledBy":812,"xmlns":32,"style":815},"138 0 483 336",[813,814],"map-table-t","map-table-d","width:100%;max-width:483px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[36,817,818],{"id":813},"The mapping table the business maintains",[40,820,821],{"id":814},"The Mapping sheet in region_mapping.xlsx with raw and standard columns: North America, N. America and NA map to Americas; EMEA and Europe map to EMEA; APAC and Asia Pacific map to APAC. A note explains that keys are normalised before lookup, so case and spacing variants need no extra rows.",[44,823],{"x":824,"y":47,"width":825,"height":826,"fill":50},"138","483","336",[73,828,832],{"x":829,"y":830,"style":831},"380.0","31.0","font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","region_mapping.xlsx — Mapping",[44,834],{"x":835,"y":836,"width":837,"height":838,"rx":47,"fill":90,"stroke":91,"style":114},"154.5","48.0","32.0","24.0",[44,840],{"x":841,"y":836,"width":842,"height":838,"rx":47,"fill":90,"stroke":91,"style":114},"186.5","113.6",[73,844,848],{"x":845,"y":846,"style":847},"243.3","65.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[44,850],{"x":851,"y":836,"width":852,"height":838,"rx":47,"fill":90,"stroke":91,"style":114},"300.1","82.8",[73,854,856],{"x":855,"y":846,"style":847},"341.5","B",[44,858],{"x":835,"y":859,"width":837,"height":860,"rx":47,"fill":90,"stroke":91,"style":114},"72.0","28.0",[73,862,865],{"x":863,"y":864,"style":847},"170.5","91.0","1",[44,867],{"x":841,"y":859,"width":842,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,869,255],{"x":845,"y":864,"style":870},"font-size:12px;font-weight:700;fill:var(--text,#172033);text-anchor:middle",[44,872],{"x":851,"y":859,"width":852,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,874,259],{"x":855,"y":864,"style":870},[44,876],{"x":835,"y":877,"width":837,"height":860,"rx":47,"fill":90,"stroke":91,"style":114},"100.0",[73,879,881],{"x":863,"y":880,"style":847},"119.0","2",[44,883],{"x":841,"y":877,"width":842,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,885,97],{"x":845,"y":880,"style":886},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle",[44,888],{"x":851,"y":877,"width":852,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,890,78],{"x":855,"y":880,"style":886},[44,892],{"x":835,"y":893,"width":837,"height":860,"rx":47,"fill":90,"stroke":91,"style":114},"128.0",[73,895,343],{"x":863,"y":896,"style":847},"147.0",[44,898],{"x":841,"y":893,"width":842,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,900,127],{"x":845,"y":896,"style":886},[44,902],{"x":851,"y":893,"width":852,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,904,78],{"x":855,"y":896,"style":886},[44,906],{"x":835,"y":907,"width":837,"height":860,"rx":47,"fill":90,"stroke":91,"style":114},"156.0",[73,909,46],{"x":863,"y":910,"style":847},"175.0",[44,912],{"x":841,"y":907,"width":842,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,914,166],{"x":845,"y":910,"style":886},[44,916],{"x":851,"y":907,"width":852,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,918,78],{"x":855,"y":910,"style":886},[44,920],{"x":835,"y":921,"width":837,"height":860,"rx":47,"fill":90,"stroke":91,"style":114},"184.0",[73,923,925],{"x":863,"y":924,"style":847},"203.0","5",[44,927],{"x":841,"y":921,"width":842,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,929,930],{"x":845,"y":924,"style":886},"EMEA",[44,932],{"x":851,"y":921,"width":852,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,934,930],{"x":855,"y":924,"style":886},[44,936],{"x":835,"y":937,"width":837,"height":860,"rx":47,"fill":90,"stroke":91,"style":114},"212.0",[73,939,941],{"x":863,"y":940,"style":847},"231.0","6",[44,943],{"x":841,"y":937,"width":842,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,945,946],{"x":845,"y":940,"style":886},"Europe",[44,948],{"x":851,"y":937,"width":852,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,950,930],{"x":855,"y":940,"style":886},[44,952],{"x":835,"y":953,"width":837,"height":860,"rx":47,"fill":90,"stroke":91,"style":114},"240.0",[73,955,957],{"x":863,"y":956,"style":847},"259.0","7",[44,959],{"x":841,"y":953,"width":842,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,961,962],{"x":845,"y":956,"style":886},"APAC",[44,964],{"x":851,"y":953,"width":852,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,966,962],{"x":855,"y":956,"style":886},[44,968],{"x":835,"y":969,"width":837,"height":860,"rx":47,"fill":90,"stroke":91,"style":114},"268.0",[73,971,973],{"x":863,"y":972,"style":847},"287.0","8",[44,975],{"x":841,"y":969,"width":842,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,977,978],{"x":845,"y":972,"style":886},"Asia Pacific",[44,980],{"x":851,"y":969,"width":852,"height":860,"rx":47,"fill":50,"stroke":91,"style":114},[73,982,962],{"x":855,"y":972,"style":886},[44,984],{"x":985,"y":986,"width":987,"height":838,"rx":47,"fill":50,"stroke":91,"style":114},"162.5","296.0","75.9",[227,989],{"x1":990,"y1":991,"x2":992,"y2":991,"stroke":59,"style":993},"164.5","319.0","236.4","stroke-width:3px",[73,995,999],{"x":996,"y":997,"style":998},"200.5","312.5","font-size:11.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Mapping",[227,1001],{"x1":1002,"y1":921,"x2":1003,"y2":921,"stroke":101,"style":60},"390.9","422.9",[103,1005],{"points":1006,"fill":106},"426.9,184.0 417.9,189.0 417.9,179.0",[44,1008],{"x":1009,"y":1010,"width":1011,"height":1012,"rx":57,"fill":58,"stroke":59,"style":60},"432.9","128.8","172.6","110.5",[62,1014,1016],{"transform":1015},"translate(507.2,141.8) scale(1.000)",[66,1017],{"d":1018,"fill":69,"stroke":70,"style":71},"M3 5h18v14H3z M3 10h18 M3 15h18 M9 5v14",[73,1020,1023],{"x":1021,"y":1022,"style":77},"519.2","188.2","One row per spelling",[73,1025,1027],{"x":1021,"y":1026,"style":82},"207.8","case and spaces are",[73,1029,1031],{"x":1021,"y":1030,"style":82},"225.2","normalised first",[211,1033,1035],{"id":1034},"step-4-map-and-let-unmapped-values-show","Step 4 — Map, and let unmapped values show",[10,1037,1038,1039,1042,1043,1045,1046,1049,1050,1053],{},"Use ",[14,1040,1041],{},"Series.map"," with the lookup. Unlike ",[14,1044,16],{},", which leaves unknown values untouched and so hides them among the cleaned ones, ",[14,1047,1048],{},"map"," returns ",[14,1051,1052],{},"NaN"," for anything not in the table:",[216,1055,1057],{"className":277,"code":1056,"language":279,"meta":221,"style":221},"sales[\"region_key\"] = sales[\"region\"].map(key)\nsales[\"region_std\"] = sales[\"region_key\"].map(lookup)\nprint(sales[[\"order\", \"region\", \"region_std\"]].to_string(index=False))\nprint(sales.groupby(\"region_std\", dropna=False)[\"amount\"].sum())\n",[14,1058,1059,1078,1096,1127],{"__ignoreMap":221},[225,1060,1061,1064,1067,1069,1071,1074,1076],{"class":227,"line":228},[225,1062,1063],{"class":290},"sales[",[225,1065,1066],{"class":235},"\"region_key\"",[225,1068,671],{"class":290},[225,1070,321],{"class":286},[225,1072,1073],{"class":290}," sales[",[225,1075,349],{"class":235},[225,1077,682],{"class":290},[225,1079,1080,1082,1085,1087,1089,1091,1093],{"class":227,"line":294},[225,1081,1063],{"class":290},[225,1083,1084],{"class":235},"\"region_std\"",[225,1086,671],{"class":290},[225,1088,321],{"class":286},[225,1090,1073],{"class":290},[225,1092,1066],{"class":235},[225,1094,1095],{"class":290},"].map(lookup)\n",[225,1097,1098,1100,1103,1106,1108,1110,1112,1114,1117,1120,1122,1124],{"class":227,"line":308},[225,1099,337],{"class":336},[225,1101,1102],{"class":290},"(sales[[",[225,1104,1105],{"class":235},"\"order\"",[225,1107,346],{"class":290},[225,1109,349],{"class":235},[225,1111,346],{"class":290},[225,1113,1084],{"class":235},[225,1115,1116],{"class":290},"]].to_string(",[225,1118,1119],{"class":407},"index",[225,1121,321],{"class":286},[225,1123,413],{"class":336},[225,1125,1126],{"class":290},"))\n",[225,1128,1129,1131,1134,1136,1138,1141,1143,1145,1147,1150],{"class":227,"line":315},[225,1130,337],{"class":336},[225,1132,1133],{"class":290},"(sales.groupby(",[225,1135,1084],{"class":235},[225,1137,346],{"class":290},[225,1139,1140],{"class":407},"dropna",[225,1142,321],{"class":286},[225,1144,413],{"class":336},[225,1146,701],{"class":290},[225,1148,1149],{"class":235},"\"amount\"",[225,1151,1152],{"class":290},"].sum())\n",[10,1154,1155],{},"Nine of the ten orders now carry a standard region. The tenth, \"Nroth America\", has none, and the grouped totals show a separate missing-region line rather than quietly folding its 40 into nothing. That line is the signal that the table needs a new row.",[211,1157,1159],{"id":1158},"step-5-report-unmapped-values-with-suggestions","Step 5 — Report unmapped values with suggestions",[10,1161,1162,1163,1166],{},"Collect the unmapped raw values with how often each occurs — a variant used a hundred times deserves attention before one used once — and suggest the closest known key with the standard library's ",[14,1164,1165],{},"difflib",":",[216,1168,1170],{"className":277,"code":1169,"language":279,"meta":221,"style":221},"from difflib import get_close_matches\n\nunmapped = (sales[sales[\"region_std\"].isna() & sales[\"region\"].notna()]\n            .groupby(\"region\").size().rename(\"rows\").reset_index())\nunmapped[\"suggested_key\"] = unmapped[\"region\"].map(\n    lambda r: (get_close_matches(key(r), lookup.index, n=1, cutoff=0.8) or [None])[0])\nunmapped[\"suggested_standard\"] = unmapped[\"suggested_key\"].map(lookup)\nprint(unmapped.to_string(index=False))\n",[14,1171,1172,1185,1189,1214,1230,1250,1294,1311],{"__ignoreMap":221},[225,1173,1174,1177,1180,1182],{"class":227,"line":228},[225,1175,1176],{"class":286},"from",[225,1178,1179],{"class":290}," difflib ",[225,1181,287],{"class":286},[225,1183,1184],{"class":290}," get_close_matches\n",[225,1186,1187],{"class":227,"line":294},[225,1188,312],{"emptyLinePlaceholder":311},[225,1190,1191,1194,1196,1199,1201,1204,1207,1209,1211],{"class":227,"line":308},[225,1192,1193],{"class":290},"unmapped ",[225,1195,321],{"class":286},[225,1197,1198],{"class":290}," (sales[sales[",[225,1200,1084],{"class":235},[225,1202,1203],{"class":290},"].isna() ",[225,1205,1206],{"class":286},"&",[225,1208,1073],{"class":290},[225,1210,349],{"class":235},[225,1212,1213],{"class":290},"].notna()]\n",[225,1215,1216,1219,1221,1224,1227],{"class":227,"line":315},[225,1217,1218],{"class":290},"            .groupby(",[225,1220,349],{"class":235},[225,1222,1223],{"class":290},").size().rename(",[225,1225,1226],{"class":235},"\"rows\"",[225,1228,1229],{"class":290},").reset_index())\n",[225,1231,1232,1235,1238,1240,1242,1245,1247],{"class":227,"line":333},[225,1233,1234],{"class":290},"unmapped[",[225,1236,1237],{"class":235},"\"suggested_key\"",[225,1239,671],{"class":290},[225,1241,321],{"class":286},[225,1243,1244],{"class":290}," unmapped[",[225,1246,349],{"class":235},[225,1248,1249],{"class":290},"].map(\n",[225,1251,1252,1255,1258,1261,1263,1265,1267,1270,1272,1275,1278,1281,1284,1287,1290,1292],{"class":227,"line":355},[225,1253,1254],{"class":286},"    lambda",[225,1256,1257],{"class":290}," r: (get_close_matches(key(r), lookup.index, ",[225,1259,1260],{"class":407},"n",[225,1262,321],{"class":286},[225,1264,865],{"class":336},[225,1266,346],{"class":290},[225,1268,1269],{"class":407},"cutoff",[225,1271,321],{"class":286},[225,1273,1274],{"class":336},"0.8",[225,1276,1277],{"class":290},") ",[225,1279,1280],{"class":286},"or",[225,1282,1283],{"class":290}," [",[225,1285,1286],{"class":336},"None",[225,1288,1289],{"class":290},"])[",[225,1291,47],{"class":336},[225,1293,352],{"class":290},[225,1295,1296,1298,1301,1303,1305,1307,1309],{"class":227,"line":592},[225,1297,1234],{"class":290},[225,1299,1300],{"class":235},"\"suggested_standard\"",[225,1302,671],{"class":290},[225,1304,321],{"class":286},[225,1306,1244],{"class":290},[225,1308,1237],{"class":235},[225,1310,1095],{"class":290},[225,1312,1313,1315,1318,1320,1322,1324],{"class":227,"line":597},[225,1314,337],{"class":336},[225,1316,1317],{"class":290},"(unmapped.to_string(",[225,1319,1119],{"class":407},[225,1321,321],{"class":286},[225,1323,413],{"class":336},[225,1325,1126],{"class":290},[10,1327,1328,1329,1332,1333,1336],{},"\"Nroth America\" is suggested as ",[14,1330,1331],{},"north america"," → Americas. The suggestion is only that: a person confirms it, because the closest spelling is not always the right meaning — \"Austria\" is one letter from \"Australia\". ",[14,1334,1335],{},"cutoff=0.8"," keeps weak matches out of the list; lower it to see more candidates.",[211,1338,1340],{"id":1339},"step-6-append-new-values-to-the-table-for-review","Step 6 — Append new values to the table for review",[10,1342,1343],{},"Close the loop by writing unmapped values into the mapping workbook, in a Review sheet with the suggestion filled in and the decision left for the owner:",[216,1345,1347],{"className":277,"code":1346,"language":279,"meta":221,"style":221},"review = unmapped.rename(columns={\"region\": \"raw\", \"suggested_standard\": \"standard\"})[[\"raw\", \"standard\", \"rows\"]]\nreview[\"approved\"] = \"\"\n\nwith pd.ExcelWriter(\"region_mapping.xlsx\", mode=\"a\", engine=\"openpyxl\", if_sheet_exists=\"replace\") as writer:\n    review.to_excel(writer, sheet_name=\"Review\", index=False)\nprint(pd.ExcelFile(\"region_mapping.xlsx\").sheet_names)\n",[14,1348,1349,1397,1412,1416,1463,1485],{"__ignoreMap":221},[225,1350,1351,1354,1356,1359,1362,1364,1366,1368,1371,1373,1375,1377,1379,1381,1384,1386,1388,1390,1392,1394],{"class":227,"line":228},[225,1352,1353],{"class":290},"review ",[225,1355,321],{"class":286},[225,1357,1358],{"class":290}," unmapped.rename(",[225,1360,1361],{"class":407},"columns",[225,1363,321],{"class":286},[225,1365,756],{"class":290},[225,1367,349],{"class":235},[225,1369,1370],{"class":290},": ",[225,1372,679],{"class":235},[225,1374,346],{"class":290},[225,1376,1300],{"class":235},[225,1378,1370],{"class":290},[225,1380,704],{"class":235},[225,1382,1383],{"class":290},"})[[",[225,1385,679],{"class":235},[225,1387,346],{"class":290},[225,1389,704],{"class":235},[225,1391,346],{"class":290},[225,1393,1226],{"class":235},[225,1395,1396],{"class":290},"]]\n",[225,1398,1399,1402,1405,1407,1409],{"class":227,"line":294},[225,1400,1401],{"class":290},"review[",[225,1403,1404],{"class":235},"\"approved\"",[225,1406,671],{"class":290},[225,1408,321],{"class":286},[225,1410,1411],{"class":235}," \"\"\n",[225,1413,1414],{"class":227,"line":308},[225,1415,312],{"emptyLinePlaceholder":311},[225,1417,1418,1421,1424,1426,1428,1431,1433,1436,1438,1441,1443,1446,1448,1451,1453,1456,1458,1460],{"class":227,"line":315},[225,1419,1420],{"class":286},"with",[225,1422,1423],{"class":290}," pd.ExcelWriter(",[225,1425,642],{"class":235},[225,1427,346],{"class":290},[225,1429,1430],{"class":407},"mode",[225,1432,321],{"class":286},[225,1434,1435],{"class":235},"\"a\"",[225,1437,346],{"class":290},[225,1439,1440],{"class":407},"engine",[225,1442,321],{"class":286},[225,1444,1445],{"class":235},"\"openpyxl\"",[225,1447,346],{"class":290},[225,1449,1450],{"class":407},"if_sheet_exists",[225,1452,321],{"class":286},[225,1454,1455],{"class":235},"\"replace\"",[225,1457,1277],{"class":290},[225,1459,302],{"class":286},[225,1461,1462],{"class":290}," writer:\n",[225,1464,1465,1468,1470,1472,1475,1477,1479,1481,1483],{"class":227,"line":333},[225,1466,1467],{"class":290},"    review.to_excel(writer, ",[225,1469,647],{"class":407},[225,1471,321],{"class":286},[225,1473,1474],{"class":235},"\"Review\"",[225,1476,346],{"class":290},[225,1478,1119],{"class":407},[225,1480,321],{"class":286},[225,1482,413],{"class":336},[225,1484,330],{"class":290},[225,1486,1487,1489,1492,1494],{"class":227,"line":355},[225,1488,337],{"class":336},[225,1490,1491],{"class":290},"(pd.ExcelFile(",[225,1493,642],{"class":235},[225,1495,1496],{"class":290},").sheet_names)\n",[10,1498,1499,1500,1504,1505,1507],{},"The owner moves approved rows into the Mapping sheet, and the next run maps them. Over a few weeks the Review sheet empties: new variants still appear, but they are caught the first time. ",[19,1501,1503],{"href":1502},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fadd-dropdown-data-validation-to-excel-with-openpyxl\u002F","Add Dropdown Data Validation to Excel with openpyxl"," can put a dropdown of the allowed standard values on the Review sheet's ",[14,1506,259],{}," column, so approvals cannot introduce a new spelling of their own.",[25,1509,34,1514,34,1517,34,1520,34,1523,34,1526,34,1533,34,1539,34,1542,34,1546,34,1550,34,1555,34,1558,34,1562,34,1568,34,1573,34,1576,34,1579,34,1582,34,1586,34,1589,34,1592,34,1598,34,1602,34,1606,34,1610,34,1614,34,1617,34,1620,34,1625,34,1630,34,1633,34,1636,34,1639,34,1643,34,1646,34,1649,34,1655,34,1659,34,1662,34,1665],{"viewBox":1510,"role":28,"ariaLabelledBy":1511,"xmlns":32,"style":33},"4 0 752 160",[1512,1513],"map-loop-t","map-loop-d",[36,1515,1516],{"id":1512},"The standardisation loop",[40,1518,1519],{"id":1513},"A cycle of five steps. Normalise keys in data and table. Check the table for conflicting mappings. Map values, leaving unknown ones as NaN. Report unmapped values with counts and difflib suggestions to a Review sheet. The owner approves rows into the mapping table, and the next run maps them.",[44,1521],{"x":46,"y":47,"width":48,"height":1522,"fill":50},"160",[44,1524],{"x":86,"y":87,"width":1525,"height":893,"rx":57,"fill":90,"stroke":91,"style":60},"115.2",[62,1527,1529],{"transform":1528},"translate(65.6,29.0) scale(1.000)",[66,1530],{"d":1531,"fill":69,"stroke":1532,"style":71},"M3 5h18l-7 8v6l-4 2v-8z","var(--text,#172033)",[73,1534,1538],{"x":1535,"y":1536,"style":1537},"77.6","75.5","font-size:13.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","Normalise",[73,1540,1541],{"x":1535,"y":56,"style":1537},"keys",[73,1543,1545],{"x":1535,"y":1544,"style":82},"112.5","case, spaces,",[73,1547,1549],{"x":1535,"y":1548,"style":82},"130.0","punctuation",[227,1551],{"x1":1552,"y1":1553,"x2":1554,"y2":1553,"stroke":101,"style":60},"140.2","80.0","162.2",[103,1556],{"points":1557,"fill":106},"166.2,80.0 157.2,85.0 157.2,75.0",[44,1559],{"x":1560,"y":87,"width":1525,"height":893,"rx":57,"fill":1561,"stroke":101,"style":60},"171.2","#f0f4ff",[62,1563,1565],{"transform":1564},"translate(216.8,29.0) scale(1.000)",[66,1566],{"d":68,"fill":69,"stroke":1567,"style":71},"var(--brand-strong,#4338ca)",[73,1569,1572],{"x":1570,"y":1536,"style":1571},"228.8","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Check the",[73,1574,1575],{"x":1570,"y":56,"style":1571},"table",[73,1577,1578],{"x":1570,"y":1544,"style":82},"no conflicting",[73,1580,1581],{"x":1570,"y":1548,"style":82},"rows",[227,1583],{"x1":1584,"y1":1553,"x2":1585,"y2":1553,"stroke":101,"style":60},"291.4","313.4",[103,1587],{"points":1588,"fill":106},"317.4,80.0 308.4,85.0 308.4,75.0",[44,1590],{"x":1591,"y":87,"width":1525,"height":893,"rx":57,"fill":1561,"stroke":101,"style":60},"322.4",[62,1593,1595],{"transform":1594},"translate(368.0,37.8) scale(1.000)",[66,1596],{"d":1597,"fill":69,"stroke":1567,"style":71},"M10 14a4 4 0 0 0 5.7 0l3-3a4 4 0 0 0-5.7-5.7l-1 1 M14 10a4 4 0 0 0-5.7 0l-3 3a4 4 0 0 0 5.7 5.7l1-1",[73,1599,1601],{"x":829,"y":1600,"style":1571},"84.2","Map",[73,1603,1605],{"x":829,"y":1604,"style":82},"103.8","unknown values",[73,1607,1609],{"x":829,"y":1608,"style":82},"121.2","become NaN",[227,1611],{"x1":1612,"y1":1553,"x2":1613,"y2":1553,"stroke":101,"style":60},"442.6","464.6",[103,1615],{"points":1616,"fill":106},"468.6,80.0 459.6,85.0 459.6,75.0",[44,1618],{"x":1619,"y":87,"width":1525,"height":893,"rx":57,"fill":182,"stroke":183,"style":60},"473.6",[62,1621,1623],{"transform":1622},"translate(519.2,29.0) scale(1.000)",[66,1624],{"d":189,"fill":69,"stroke":190,"style":71},[73,1626,1629],{"x":1627,"y":1536,"style":1628},"531.2","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Report and",[73,1631,1632],{"x":1627,"y":56,"style":1628},"suggest",[73,1634,1635],{"x":1627,"y":1544,"style":82},"Review sheet",[73,1637,1638],{"x":1627,"y":1548,"style":82},"with counts",[227,1640],{"x1":1641,"y1":1553,"x2":1642,"y2":1553,"stroke":101,"style":60},"593.8","615.8",[103,1644],{"points":1645,"fill":106},"619.8,80.0 610.8,85.0 610.8,75.0",[44,1647],{"x":1648,"y":87,"width":1525,"height":893,"rx":57,"fill":58,"stroke":59,"style":60},"624.8",[62,1650,1652],{"transform":1651},"translate(670.4,29.0) scale(1.000)",[66,1653],{"d":1654,"fill":69,"stroke":70,"style":71},"M12 12a4 4 0 1 0 0-8 4 4 0 0 0 0 8z M4 21c0-4 3.6-7 8-7s8 3 8 7",[73,1656,1658],{"x":1657,"y":1536,"style":77},"682.4","Owner",[73,1660,1661],{"x":1657,"y":56,"style":77},"approves",[73,1663,1664],{"x":1657,"y":1544,"style":82},"next run maps",[73,1666,1667],{"x":1657,"y":1548,"style":82},"them",[211,1669,1671],{"id":1670},"step-7-fail-the-job-when-too-much-is-unmapped","Step 7 — Fail the job when too much is unmapped",[10,1673,1674],{},"A report built on data that is largely unmapped should not go out. Set a threshold on the share of rows — or of the amount — without a standard value:",[216,1676,1678],{"className":277,"code":1677,"language":279,"meta":221,"style":221},"share = sales[\"region_std\"].isna().mean()\namount_share = sales.loc[sales[\"region_std\"].isna(), \"amount\"].sum() \u002F sales[\"amount\"].sum()\nprint(f\"{share:.0%} of rows, {amount_share:.1%} of amount unmapped\")\nif amount_share > 0.05:\n    raise SystemExit(\"too much revenue has no standard region — review region_mapping.xlsx\")\n",[14,1679,1680,1694,1724,1762,1777],{"__ignoreMap":221},[225,1681,1682,1685,1687,1689,1691],{"class":227,"line":228},[225,1683,1684],{"class":290},"share ",[225,1686,321],{"class":286},[225,1688,1073],{"class":290},[225,1690,1084],{"class":235},[225,1692,1693],{"class":290},"].isna().mean()\n",[225,1695,1696,1699,1701,1704,1706,1709,1711,1714,1717,1719,1721],{"class":227,"line":294},[225,1697,1698],{"class":290},"amount_share ",[225,1700,321],{"class":286},[225,1702,1703],{"class":290}," sales.loc[sales[",[225,1705,1084],{"class":235},[225,1707,1708],{"class":290},"].isna(), ",[225,1710,1149],{"class":235},[225,1712,1713],{"class":290},"].sum() ",[225,1715,1716],{"class":286},"\u002F",[225,1718,1073],{"class":290},[225,1720,1149],{"class":235},[225,1722,1723],{"class":290},"].sum()\n",[225,1725,1726,1728,1730,1732,1734,1736,1739,1742,1744,1747,1749,1752,1755,1757,1760],{"class":227,"line":308},[225,1727,337],{"class":336},[225,1729,404],{"class":290},[225,1731,749],{"class":286},[225,1733,542],{"class":235},[225,1735,756],{"class":755},[225,1737,1738],{"class":290},"share",[225,1740,1741],{"class":286},":.0%",[225,1743,765],{"class":755},[225,1745,1746],{"class":235}," of rows, ",[225,1748,756],{"class":755},[225,1750,1751],{"class":290},"amount_share",[225,1753,1754],{"class":286},":.1%",[225,1756,765],{"class":755},[225,1758,1759],{"class":235}," of amount unmapped\"",[225,1761,330],{"class":290},[225,1763,1764,1766,1769,1771,1774],{"class":227,"line":315},[225,1765,730],{"class":286},[225,1767,1768],{"class":290}," amount_share ",[225,1770,719],{"class":286},[225,1772,1773],{"class":336}," 0.05",[225,1775,1776],{"class":290},":\n",[225,1778,1779,1781,1784,1786,1789],{"class":227,"line":333},[225,1780,741],{"class":286},[225,1782,1783],{"class":336}," SystemExit",[225,1785,404],{"class":290},[225,1787,1788],{"class":235},"\"too much revenue has no standard region — review region_mapping.xlsx\"",[225,1790,330],{"class":290},[10,1792,1793],{},"Here one order in ten is unmapped but it carries only 3.4% of the revenue, so the run continues and the Review sheet tells the owner what to add. Measuring by amount as well as by row count matters: one unmapped enterprise customer can carry more revenue than a hundred unmapped small orders.",[211,1795,1797],{"id":1796},"common-pitfalls","Common pitfalls",[1799,1800,1801,1821,1835,1841,1847,1856],"ul",{},[1802,1803,1804,1808,1809,346,1811,1813,1814,1816,1817,1820],"li",{},[1805,1806,1807],"strong",{},"Default NA strings."," ",[14,1810,166],{},[14,1812,367],{}," and ",[14,1815,370],{}," become missing on read; use ",[14,1818,1819],{},"keep_default_na=False"," for code and category columns.",[1802,1822,1823,1828,1829,1831,1832,1834],{},[1805,1824,1825,1827],{},[14,1826,16],{}," for standardisation."," It hides unknown values; use ",[14,1830,1048],{}," and look at the ",[14,1833,1052],{},"s.",[1802,1836,1837,1840],{},[1805,1838,1839],{},"Mapping raw spellings without normalising."," The table fills with case and spacing variants.",[1802,1842,1843,1846],{},[1805,1844,1845],{},"Auto-applying fuzzy matches."," Close spelling is not the same meaning.",[1802,1848,1849,1852,1853,1855],{},[1805,1850,1851],{},"Editing the mapping in two places."," Once the table lives in Excel, remove any leftover ",[14,1854,16],{}," calls from the code, or the two will disagree the first time someone updates only one of them.",[1802,1857,1858,1861],{},[1805,1859,1860],{},"Silently picking one of two conflicting rows."," Fail and let the owner decide.",[211,1863,1865],{"id":1864},"performance-notes","Performance notes",[10,1867,1868,1869,1871,1872,1875,1876,1879,1880,1882],{},"A normalised-key lookup is a dictionary lookup per row, which pandas' ",[14,1870,1048],{}," performs quickly even on millions of rows; the normalisation function is the slower part and can be applied to the column's ",[382,1873,1874],{},"unique"," values only (",[14,1877,1878],{},"sales[\"region\"].drop_duplicates()","), then joined back. ",[14,1881,1165],{}," is slow on very large vocabularies but only ever runs on the handful of unmapped values.",[211,1884,1886],{"id":1885},"conclusion","Conclusion",[10,1888,1889,1890,1892,1893,1895,1896,1898],{},"Keep category mappings in an Excel table the business owns, normalise keys on both sides so the table only needs genuinely different spellings, and fail on conflicting rows. Map with ",[14,1891,1041],{}," so unknown values surface as ",[14,1894,1052],{},", report them with counts and ",[14,1897,1165],{}," suggestions to a Review sheet for approval, and stop the job when an unacceptable share of rows or revenue is unmapped.",[211,1900,1902],{"id":1901},"frequently-asked-questions","Frequently asked questions",[10,1904,1905,1908],{},[1805,1906,1907],{},"Should the mapping live in code or in a spreadsheet?","\nIn a spreadsheet when the people who decide what the categories mean are not programmers. A small Excel table that the sales team can edit keeps the code stable and puts decisions with the people who own them.",[10,1910,1911,1914],{},[1805,1912,1913],{},"What is the difference between map and replace?","\nSeries.map returns NaN for any value not in the mapping, which makes unmapped values visible. Series.replace leaves unmapped values unchanged, which hides them among the cleaned ones. For standardisation, map and then report the NaNs.",[10,1916,1917,1920],{},[1805,1918,1919],{},"How do I match \"north america \" to \"North America\"?","\nNormalise both sides before looking up: strip spaces, collapse repeated spaces, casefold and remove punctuation. Build the lookup on the normalised key, so the mapping table only needs one row per genuinely different spelling.",[10,1922,1923,1926],{},[1805,1924,1925],{},"Can pandas suggest the right category for a typo?","\nThe standard library's difflib.get_close_matches finds the closest known keys, so \"nroth america\" suggests \"north america\". Use it to propose mappings for a person to accept, not to apply them automatically.",[211,1928,1930],{"id":1929},"related","Related",[1799,1932,1933,1939,1946,1953],{},[1802,1934,1935,1936,1938],{},"Up one level: ",[19,1937,22],{"href":21}," — the full cleaning workflow.",[1802,1940,1941,1945],{},[19,1942,1944],{"href":1943},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Freplace-values-in-an-excel-column-with-pandas\u002F","Replace Values in an Excel Column with pandas"," — replace for one-off fixes.",[1802,1947,1948,1952],{},[19,1949,1951],{"href":1950},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Ffuzzy-match-names-between-two-excel-files-with-python\u002F","Fuzzy Match Names Between Two Excel Files with Python"," — fuzzy matching at scale.",[1802,1954,1955,1959],{},[19,1956,1958],{"href":1957},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvlookup-equivalent-in-pandas-for-excel-files\u002F","The VLOOKUP Equivalent in pandas for Excel Files"," — lookups against other tables.",[1961,1962,1963],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}",{"title":221,"searchDepth":294,"depth":294,"links":1965},[1966,1967,1968,1969,1970,1971,1972,1973,1974,1975,1976,1977,1978],{"id":213,"depth":294,"text":214},{"id":263,"depth":294,"text":264},{"id":477,"depth":294,"text":478},{"id":621,"depth":294,"text":622},{"id":1034,"depth":294,"text":1035},{"id":1158,"depth":294,"text":1159},{"id":1339,"depth":294,"text":1340},{"id":1670,"depth":294,"text":1671},{"id":1796,"depth":294,"text":1797},{"id":1864,"depth":294,"text":1865},{"id":1885,"depth":294,"text":1886},{"id":1901,"depth":294,"text":1902},{"id":1929,"depth":294,"text":1930},"2026-09-25","Turn \"N. America\", \"north america \", \"NA\" and \"USA\" into one region name using a mapping table kept in Excel, where the people who own the categories can maintain it. Normalise keys, check the table for conflicts, report unmapped values with counts and suggestions, and append new ones for review.","md",[1983,1985,1987,1989],{"q":1907,"a":1984},"In a spreadsheet when the people who decide what the categories mean are not programmers. A small Excel table that the sales team can edit keeps the code stable and puts decisions with the people who own them.",{"q":1913,"a":1986},"Series.map returns NaN for any value not in the mapping, which makes unmapped values visible. Series.replace leaves unmapped values unchanged, which hides them among the cleaned ones. For standardisation, map and then report the NaNs.",{"q":1919,"a":1988},"Normalise both sides before looking up: strip spaces, collapse repeated spaces, casefold and remove punctuation. Build the lookup on the normalised key, so the mapping table only needs one row per genuinely different spelling.",{"q":1925,"a":1990},"The standard library's difflib.get_close_matches finds the closest known keys, so \"nroth america\" suggests \"north america\". Use it to propose mappings for a person to accept, not to apply them automatically.","standardise category values with a mapping table in pandas",null,{"breadcrumb":1994},[1995,1997,2000],{"name":1996,"item":1716},"Home",{"name":1998,"item":1999},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":2001,"item":21},"Cleaning Excel Data With Pandas: A Production Workflow","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-category-values-with-a-mapping-table",{"title":2004,"description":2005},"Clean Category Values in pandas with an Excel Mapping Table","Standardise inconsistent category values in pandas with a lookup table stored in Excel: normalised keys, map versus replace, conflicting mappings, unmapped value reports, difflib suggestions and a review loop.","standardise-category-values-with-a-mapping-table","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-category-values-with-a-mapping-table\u002Findex",[2009,2010,2011],"data-cleaning","joins-and-lookups","pandas","how-to","LHTVHMYR3vJiVwX2bhkL1JxboDb2iUqslzwm30hS0aY",[2015,2019],{"title":2016,"path":2017,"stem":2018,"children":-1},"Split Full Names into First and Last Names with pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fsplit-full-names-into-first-and-last-names-with-pandas","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fsplit-full-names-into-first-and-last-names-with-pandas\u002Findex",{"title":2020,"path":2021,"stem":2022,"children":-1},"Standardise Country Names and Codes with Python","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-country-names-and-codes-with-python","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-country-names-and-codes-with-python\u002Findex",[2024,2026],{"path":2025,"title":2001},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas",{"path":2027,"title":1998},"\u002Fadvanced-data-transformation-and-cleaning",[],[2030,2033,2036,2039,2042,2045],{"path":2031,"title":2032,"kind":2012},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Ffind-duplicate-rows-in-excel-with-python","Find Duplicate Rows in Excel with Python",{"path":2034,"title":2035,"kind":2012},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-duplicate-values-in-excel-with-openpyxl","Highlight Duplicate Values in Excel with openpyxl",{"path":2037,"title":2038,"kind":2012},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fclean-mixed-currency-amounts-into-amount-and-currency-columns","Clean Mixed-Currency Amounts into Amount and Currency Columns",{"path":2040,"title":2041,"kind":2012},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fcoalesce-several-excel-columns-into-one-with-pandas","Coalesce Several Excel Columns into One with pandas",{"path":2043,"title":2044,"kind":2012},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-currency-strings-to-numbers-with-pandas","Convert Currency Strings Like \"$1,234.56\" and \"(500)\" to Numbers with pandas",{"path":2046,"title":2047,"kind":2012},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-excel-text-columns-to-numbers-with-pandas","Convert Excel Text Columns to Numbers with pandas",1790972085984]