[{"data":1,"prerenderedAt":1988},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fkeep-the-latest-record-per-key-with-pandas":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fkeep-the-latest-record-per-key-with-pandas":1955,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fkeep-the-latest-record-per-key-with-pandas":1964,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fkeep-the-latest-record-per-key-with-pandas":1969,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fkeep-the-latest-record-per-key-with-pandas":1970},{"id":4,"title":5,"body":6,"dateModified":1919,"datePublished":1919,"description":1920,"extension":1921,"faq":1922,"keyword":1931,"kind":1932,"meta":1933,"navigation":383,"path":1943,"seo":1944,"slug":1947,"stem":1948,"topics":1949,"type":1953,"__hash__":1954},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fkeep-the-latest-record-per-key-with-pandas\u002Findex.md","Keep the Latest Record per Key When Deduplicating with pandas",{"type":7,"value":8,"toc":1904},"minimark",[9,24,288,293,320,334,338,355,549,569,573,583,743,757,1000,1004,1015,1072,1106,1110,1113,1195,1216,1351,1355,1358,1490,1496,1500,1503,1572,1578,1582,1585,1738,1746,1750,1806,1810,1822,1826,1839,1843,1849,1855,1861,1867,1871,1900],[10,11,12,13,17,18,23],"p",{},"Exports from CRMs, HR systems and order platforms often contain history rather than current state: every time a customer's email or phone changed, a new row was written. Pivot or merge that sheet as it is and each customer is counted several times, with several emails. The fix is to keep one row per key — the most recent — and the textbook one-liner does it. What the one-liner does not do is protect you from ties on the timestamp, dates stored as text, rows with no date at all, or the tempting ",[14,15,16],"code",{},"groupby().last()",", which returns something that looks right and is not. This page, part of ",[19,20,22],"a",{"href":21},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002F","Cleaning Excel Data With Pandas",", handles each case.",[25,26,34,35,34,39,34,43,34,51,34,60,34,64,34,71,34,75,34,79,34,83,34,88,34,90,34,94,34,96,34,99,34,102,34,106,34,108,34,112,34,114,34,117,34,120,34,124,34,126,34,128,34,130,34,133,34,136,34,140,34,142,34,144,34,146,34,149,34,152,34,156,34,158,34,161,34,163,34,166,34,169,34,173,34,175,34,177,34,179,34,181,34,184,34,187,34,190,34,193,34,196,34,198,34,201,34,203,34,205,34,207,34,209,34,211,34,213,34,218,34,221,34,223,34,225,34,227,34,229,34,231,34,233,34,235,34,237,34,239,34,241,34,243,34,246,34,248,34,251,34,257,34,261,34,269,34,274,34,282],"svg",{"viewBox":27,"role":28,"ariaLabelledBy":29,"xmlns":32,"style":33},"129 0 502 248","img",[30,31],"lat-dedupe-t","lat-dedupe-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:502px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[36,37,38],"title",{"id":30},"Several versions per customer reduced to the latest",[40,41,42],"desc",{"id":31},"Left: a customer history with three rows for C1 dated January, March and one undated, two rows for C2 with the same date, and two rows for C3. Right: one row per customer, keeping C1 from March, C2's second row by sequence number, and C3's most recent row.",[44,45],"rect",{"x":46,"y":47,"width":48,"height":49,"fill":50},"129","0","502","248","#ffffff",[44,52],{"x":53,"y":54,"width":55,"height":56,"rx":47,"fill":57,"stroke":58,"style":59},"145.3","40.0","32.0","24.0","#e7ebef","var(--line,#cdd5e6)","stroke-width:1px",[44,61],{"x":62,"y":54,"width":63,"height":56,"rx":47,"fill":57,"stroke":58,"style":59},"177.3","44.0",[65,66,70],"text",{"x":67,"y":68,"style":69},"199.3","57.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[44,72],{"x":73,"y":54,"width":74,"height":56,"rx":47,"fill":57,"stroke":58,"style":59},"221.3","98.9",[65,76,78],{"x":77,"y":68,"style":69},"270.8","B",[44,80],{"x":53,"y":81,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},"64.0","28.0",[65,84,87],{"x":85,"y":86,"style":69},"161.3","83.0","1",[44,89],{"x":62,"y":81,"width":63,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,91,93],{"x":67,"y":86,"style":92},"font-size:12px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","id",[44,95],{"x":73,"y":81,"width":74,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,97,98],{"x":77,"y":86,"style":92},"updated",[44,100],{"x":53,"y":101,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},"92.0",[65,103,105],{"x":85,"y":104,"style":69},"111.0","2",[44,107],{"x":62,"y":101,"width":63,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,109,111],{"x":67,"y":104,"style":110},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","C1",[44,113],{"x":73,"y":101,"width":74,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,115,116],{"x":77,"y":104,"style":110},"2026-01-05",[44,118],{"x":53,"y":119,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},"120.0",[65,121,123],{"x":85,"y":122,"style":69},"139.0","3",[44,125],{"x":62,"y":119,"width":63,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,127,111],{"x":67,"y":122,"style":110},[44,129],{"x":73,"y":119,"width":74,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,131,132],{"x":77,"y":122,"style":110},"2026-03-10",[44,134],{"x":53,"y":135,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},"148.0",[65,137,139],{"x":85,"y":138,"style":69},"167.0","4",[44,141],{"x":62,"y":135,"width":63,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,143,111],{"x":67,"y":138,"style":110},[44,145],{"x":73,"y":135,"width":74,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,147,148],{"x":77,"y":138,"style":110},"(none)",[44,150],{"x":53,"y":151,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},"176.0",[65,153,155],{"x":85,"y":154,"style":69},"195.0","5",[44,157],{"x":62,"y":151,"width":63,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,159,160],{"x":67,"y":154,"style":110},"C2",[44,162],{"x":73,"y":151,"width":74,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,164,165],{"x":77,"y":154,"style":110},"2026-02-01",[44,167],{"x":53,"y":168,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},"204.0",[65,170,172],{"x":85,"y":171,"style":69},"223.0","6",[44,174],{"x":62,"y":168,"width":63,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,176,160],{"x":67,"y":171,"style":110},[44,178],{"x":73,"y":168,"width":74,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,180,165],{"x":77,"y":171,"style":110},[44,182],{"x":183,"y":54,"width":55,"height":56,"rx":47,"fill":57,"stroke":58,"style":59},"439.8",[44,185],{"x":186,"y":54,"width":63,"height":56,"rx":47,"fill":57,"stroke":58,"style":59},"471.8",[65,188,70],{"x":189,"y":68,"style":69},"493.8",[44,191],{"x":192,"y":54,"width":74,"height":56,"rx":47,"fill":57,"stroke":58,"style":59},"515.8",[65,194,78],{"x":195,"y":68,"style":69},"565.2",[44,197],{"x":183,"y":81,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},[65,199,87],{"x":200,"y":86,"style":69},"455.8",[44,202],{"x":186,"y":81,"width":63,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,204,93],{"x":189,"y":86,"style":92},[44,206],{"x":192,"y":81,"width":74,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,208,98],{"x":195,"y":86,"style":92},[44,210],{"x":183,"y":101,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},[65,212,105],{"x":200,"y":104,"style":69},[44,214],{"x":186,"y":101,"width":63,"height":82,"rx":47,"fill":215,"stroke":216,"style":217},"#d9f4f1","var(--teal,#0f9488)","stroke-width:1.5px",[65,219,111],{"x":189,"y":104,"style":220},"font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle",[44,222],{"x":192,"y":101,"width":74,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,224,132],{"x":195,"y":104,"style":110},[44,226],{"x":183,"y":119,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},[65,228,123],{"x":200,"y":122,"style":69},[44,230],{"x":186,"y":119,"width":63,"height":82,"rx":47,"fill":215,"stroke":216,"style":217},[65,232,160],{"x":189,"y":122,"style":220},[44,234],{"x":192,"y":119,"width":74,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,236,165],{"x":195,"y":122,"style":110},[44,238],{"x":183,"y":135,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},[65,240,139],{"x":200,"y":138,"style":69},[44,242],{"x":186,"y":135,"width":63,"height":82,"rx":47,"fill":215,"stroke":216,"style":217},[65,244,245],{"x":189,"y":138,"style":220},"C3",[44,247],{"x":192,"y":135,"width":74,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,249,250],{"x":195,"y":138,"style":110},"2026-03-15",[65,252,256],{"x":253,"y":254,"style":255},"232.8","29.0","font-size:12.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","history (7 rows)",[65,258,260],{"x":259,"y":254,"style":255},"527.2","latest (3 rows)",[262,263],"line",{"x1":264,"y1":265,"x2":266,"y2":265,"stroke":267,"style":268},"330.2","108.0","425.8","var(--brand,#5b5cf0)","stroke-width:2px",[270,271],"polygon",{"points":272,"fill":273},"429.8,108.0 420.8,113.0 420.8,103.0","#5b5cf0",[44,275],{"x":276,"y":277,"width":278,"height":279,"rx":280,"fill":50,"stroke":58,"style":281},"348.2","75.0","63.6","22.0","11","stroke-width:1.2px",[65,283,287],{"x":284,"y":285,"style":286},"380.0","90.0","font-size:11.5px;font-weight:400;fill:var(--brand-strong,#4338ca);text-anchor:middle;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace","dedupe",[289,290,292],"h2",{"id":291},"prerequisites","Prerequisites",[294,295,300],"pre",{"className":296,"code":297,"language":298,"meta":299,"style":299},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[14,301,302],{"__ignoreMap":299},[303,304,306,310,314,317],"span",{"class":262,"line":305},1,[303,307,309],{"class":308},"sMTad","pip",[303,311,313],{"class":312},"srMev"," install",[303,315,316],{"class":312}," pandas",[303,318,319],{"class":312}," openpyxl\n",[10,321,322,325,326,329,330,333],{},[14,323,324],{},"customers.xlsx"," has seven rows for three customers. C1 has three versions, one of them undated; C2's two versions share a timestamp and differ only in a ",[14,327,328],{},"seq"," column; C3's second date is written day-first as text, ",[14,331,332],{},"15\u002F03\u002F2026",". Some versions have missing emails or phones.",[289,335,337],{"id":336},"step-1-make-the-timestamp-a-real-date","Step 1 — Make the timestamp a real date",[10,339,340,341,344,345,348,349,351,352,354],{},"Sorting only works on real dates. A column that mixes ISO dates and day-first text sorts as strings, where ",[14,342,343],{},"\"15\u002F03\u002F2026\""," comes before ",[14,346,347],{},"\"2026-01-05\""," because ",[14,350,87],{}," sorts before ",[14,353,105],{},". Parse it first, stating the formats you expect:",[294,356,360],{"className":357,"code":358,"language":359,"meta":299,"style":299},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\nhist = pd.read_excel(\"customers.xlsx\")\nprint(hist[\"updated\"].tolist())\n\niso = pd.to_datetime(hist[\"updated\"], format=\"%Y-%m-%d\", errors=\"coerce\")\ndayfirst = pd.to_datetime(hist[\"updated\"], format=\"%d\u002F%m\u002F%Y\", errors=\"coerce\")\nhist[\"updated\"] = iso.fillna(dayfirst)\nprint(hist[[\"customer_id\", \"updated\"]].to_string(index=False))\n","python",[14,361,362,378,385,403,419,424,469,504,520],{"__ignoreMap":299},[303,363,364,368,372,375],{"class":262,"line":305},[303,365,367],{"class":366},"s-kum","import",[303,369,371],{"class":370},"skGVy"," pandas ",[303,373,374],{"class":366},"as",[303,376,377],{"class":370}," pd\n",[303,379,381],{"class":262,"line":380},2,[303,382,384],{"emptyLinePlaceholder":383},true,"\n",[303,386,388,391,394,397,400],{"class":262,"line":387},3,[303,389,390],{"class":370},"hist ",[303,392,393],{"class":366},"=",[303,395,396],{"class":370}," pd.read_excel(",[303,398,399],{"class":312},"\"customers.xlsx\"",[303,401,402],{"class":370},")\n",[303,404,406,410,413,416],{"class":262,"line":405},4,[303,407,409],{"class":408},"sP0c6","print",[303,411,412],{"class":370},"(hist[",[303,414,415],{"class":312},"\"updated\"",[303,417,418],{"class":370},"].tolist())\n",[303,420,422],{"class":262,"line":421},5,[303,423,384],{"emptyLinePlaceholder":383},[303,425,427,430,432,435,437,440,444,446,449,453,456,459,462,464,467],{"class":262,"line":426},6,[303,428,429],{"class":370},"iso ",[303,431,393],{"class":366},[303,433,434],{"class":370}," pd.to_datetime(hist[",[303,436,415],{"class":312},[303,438,439],{"class":370},"], ",[303,441,443],{"class":442},"sa561","format",[303,445,393],{"class":366},[303,447,448],{"class":312},"\"%Y-%m-",[303,450,452],{"class":451},"sSjpA","%d",[303,454,455],{"class":312},"\"",[303,457,458],{"class":370},", ",[303,460,461],{"class":442},"errors",[303,463,393],{"class":366},[303,465,466],{"class":312},"\"coerce\"",[303,468,402],{"class":370},[303,470,472,475,477,479,481,483,485,487,489,491,494,496,498,500,502],{"class":262,"line":471},7,[303,473,474],{"class":370},"dayfirst ",[303,476,393],{"class":366},[303,478,434],{"class":370},[303,480,415],{"class":312},[303,482,439],{"class":370},[303,484,443],{"class":442},[303,486,393],{"class":366},[303,488,455],{"class":312},[303,490,452],{"class":451},[303,492,493],{"class":312},"\u002F%m\u002F%Y\"",[303,495,458],{"class":370},[303,497,461],{"class":442},[303,499,393],{"class":366},[303,501,466],{"class":312},[303,503,402],{"class":370},[303,505,507,510,512,515,517],{"class":262,"line":506},8,[303,508,509],{"class":370},"hist[",[303,511,415],{"class":312},[303,513,514],{"class":370},"] ",[303,516,393],{"class":366},[303,518,519],{"class":370}," iso.fillna(dayfirst)\n",[303,521,523,525,528,531,533,535,538,541,543,546],{"class":262,"line":522},9,[303,524,409],{"class":408},[303,526,527],{"class":370},"(hist[[",[303,529,530],{"class":312},"\"customer_id\"",[303,532,458],{"class":370},[303,534,415],{"class":312},[303,536,537],{"class":370},"]].to_string(",[303,539,540],{"class":442},"index",[303,542,393],{"class":366},[303,544,545],{"class":408},"False",[303,547,548],{"class":370},"))\n",[10,550,551,552,555,556,559,560,563,564,568],{},"Trying each known format and combining the results is stricter than ",[14,553,554],{},"format=\"mixed\"",", which would guess — and could read ",[14,557,558],{},"03\u002F04\u002F2026"," as either March or April. The one row with no date stays ",[14,561,562],{},"NaT",". ",[19,565,567],{"href":566},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-dates-and-times-in-excel-data\u002Fparse-excel-dates-into-python-datetimes-with-pandas\u002F","Parse Excel Dates into Python datetimes with pandas"," covers date parsing in depth.",[289,570,572],{"id":571},"step-2-sort-then-keep-the-last-row-per-key","Step 2 — Sort, then keep the last row per key",[10,574,575,576,579,580,582],{},"With real dates, the standard approach is to sort by key and date and keep the last row of each key. Decide where undated rows go: ",[14,577,578],{},"sort_values"," puts ",[14,581,562],{}," last by default, which would make C1's undated typo the \"latest\" version:",[294,584,586],{"className":357,"code":585,"language":359,"meta":299,"style":299},"latest_wrong = (hist.sort_values([\"customer_id\", \"updated\"])\n                    .drop_duplicates(\"customer_id\", keep=\"last\"))\nprint(latest_wrong[[\"customer_id\", \"updated\", \"email\"]].to_string(index=False))\n\nlatest = (hist.sort_values([\"customer_id\", \"updated\", \"seq\"], na_position=\"first\")\n              .drop_duplicates(\"customer_id\", keep=\"last\"))\nprint(latest[[\"customer_id\", \"updated\", \"seq\", \"email\", \"phone\"]].to_string(index=False))\n",[14,587,588,607,626,654,658,690,707],{"__ignoreMap":299},[303,589,590,593,595,598,600,602,604],{"class":262,"line":305},[303,591,592],{"class":370},"latest_wrong ",[303,594,393],{"class":366},[303,596,597],{"class":370}," (hist.sort_values([",[303,599,530],{"class":312},[303,601,458],{"class":370},[303,603,415],{"class":312},[303,605,606],{"class":370},"])\n",[303,608,609,612,614,616,619,621,624],{"class":262,"line":380},[303,610,611],{"class":370},"                    .drop_duplicates(",[303,613,530],{"class":312},[303,615,458],{"class":370},[303,617,618],{"class":442},"keep",[303,620,393],{"class":366},[303,622,623],{"class":312},"\"last\"",[303,625,548],{"class":370},[303,627,628,630,633,635,637,639,641,644,646,648,650,652],{"class":262,"line":387},[303,629,409],{"class":408},[303,631,632],{"class":370},"(latest_wrong[[",[303,634,530],{"class":312},[303,636,458],{"class":370},[303,638,415],{"class":312},[303,640,458],{"class":370},[303,642,643],{"class":312},"\"email\"",[303,645,537],{"class":370},[303,647,540],{"class":442},[303,649,393],{"class":366},[303,651,545],{"class":408},[303,653,548],{"class":370},[303,655,656],{"class":262,"line":405},[303,657,384],{"emptyLinePlaceholder":383},[303,659,660,663,665,667,669,671,673,675,678,680,683,685,688],{"class":262,"line":421},[303,661,662],{"class":370},"latest ",[303,664,393],{"class":366},[303,666,597],{"class":370},[303,668,530],{"class":312},[303,670,458],{"class":370},[303,672,415],{"class":312},[303,674,458],{"class":370},[303,676,677],{"class":312},"\"seq\"",[303,679,439],{"class":370},[303,681,682],{"class":442},"na_position",[303,684,393],{"class":366},[303,686,687],{"class":312},"\"first\"",[303,689,402],{"class":370},[303,691,692,695,697,699,701,703,705],{"class":262,"line":426},[303,693,694],{"class":370},"              .drop_duplicates(",[303,696,530],{"class":312},[303,698,458],{"class":370},[303,700,618],{"class":442},[303,702,393],{"class":366},[303,704,623],{"class":312},[303,706,548],{"class":370},[303,708,709,711,714,716,718,720,722,724,726,728,730,733,735,737,739,741],{"class":262,"line":471},[303,710,409],{"class":408},[303,712,713],{"class":370},"(latest[[",[303,715,530],{"class":312},[303,717,458],{"class":370},[303,719,415],{"class":312},[303,721,458],{"class":370},[303,723,677],{"class":312},[303,725,458],{"class":370},[303,727,643],{"class":312},[303,729,458],{"class":370},[303,731,732],{"class":312},"\"phone\"",[303,734,537],{"class":370},[303,736,540],{"class":442},[303,738,393],{"class":366},[303,740,545],{"class":408},[303,742,548],{"class":370},[10,744,745,746,749,750,753,754,756],{},"The first result keeps ",[14,747,748],{},"ada@typo"," for C1. With ",[14,751,752],{},"na_position=\"first\"",", undated rows count as the oldest, and C1's March row wins. Adding ",[14,755,328],{}," to the sort is the tie-breaker for C2, whose two rows share a date: without it, which one survives depends on the order the rows happened to arrive in. With it, the higher sequence number wins every time.",[25,758,34,764,34,767,34,770,34,775,34,780,34,784,34,788,34,792,34,795,34,798,34,802,34,806,34,810,34,814,34,817,34,821,34,823,34,826,34,828,34,830,34,832,34,834,34,836,34,839,34,842,34,845,34,847,34,849,34,851,34,854,34,856,34,858,34,860,34,862,34,865,34,868,34,870,34,872,34,874,34,876,34,878,34,880,34,882,34,885,34,888,34,891,34,893,34,895,34,897,34,899,34,901,34,903,34,905,34,908,34,911,34,914,34,916,34,918,34,920,34,922,34,924,34,926,34,928,34,931,34,934,34,937,34,939,34,941,34,943,34,945,34,947,34,949,34,951,34,954,34,957,34,961,34,963,34,965,34,967,34,970,34,972,34,974,34,976,34,979,34,983,34,985,34,987,34,989,34,991,34,993,34,995,34,997],{"viewBox":759,"role":28,"ariaLabelledBy":760,"xmlns":32,"style":763},"152 0 456 312",[761,762],"lat-sorted-t","lat-sorted-d","width:100%;max-width:456px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[36,765,766],{"id":761},"The history after sorting, with the kept rows marked",[40,768,769],{"id":762},"The seven rows sorted by customer, then date with undated rows first, then sequence. The last row of each customer is kept: C1 dated 2026-03-10, C2 with sequence 2, and C3 dated 2026-03-15. The undated C1 row sorts to the top of its group and is discarded.",[44,771],{"x":772,"y":47,"width":773,"height":774,"fill":50},"152","456","312",[65,776,779],{"x":284,"y":777,"style":778},"31.0","font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","sort_values([id, updated, seq], na_position=\"first\")",[44,781],{"x":782,"y":783,"width":55,"height":56,"rx":47,"fill":57,"stroke":58,"style":59},"168.1","48.0",[44,785],{"x":786,"y":783,"width":787,"height":56,"rx":47,"fill":57,"stroke":58,"style":59},"200.1","100.1",[65,789,70],{"x":790,"y":791,"style":69},"250.2","65.0",[44,793],{"x":794,"y":783,"width":74,"height":56,"rx":47,"fill":57,"stroke":58,"style":59},"300.3",[65,796,78],{"x":797,"y":791,"style":69},"349.7",[44,799],{"x":800,"y":783,"width":801,"height":56,"rx":47,"fill":57,"stroke":58,"style":59},"399.2","46.5",[65,803,805],{"x":804,"y":791,"style":69},"422.4","C",[44,807],{"x":808,"y":783,"width":809,"height":56,"rx":47,"fill":57,"stroke":58,"style":59},"445.7","146.2",[65,811,813],{"x":812,"y":791,"style":69},"518.8","D",[44,815],{"x":782,"y":816,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},"72.0",[65,818,87],{"x":819,"y":820,"style":69},"184.1","91.0",[44,822],{"x":786,"y":816,"width":787,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,824,825],{"x":790,"y":820,"style":92},"customer_id",[44,827],{"x":794,"y":816,"width":74,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,829,98],{"x":797,"y":820,"style":92},[44,831],{"x":800,"y":816,"width":801,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,833,328],{"x":804,"y":820,"style":92},[44,835],{"x":808,"y":816,"width":809,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,837,838],{"x":812,"y":820,"style":92},"email",[44,840],{"x":782,"y":841,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},"100.0",[65,843,105],{"x":819,"y":844,"style":69},"119.0",[44,846],{"x":786,"y":841,"width":787,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,848,111],{"x":790,"y":844,"style":110},[44,850],{"x":794,"y":841,"width":74,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,852,853],{"x":797,"y":844,"style":110},"(NaT)",[44,855],{"x":800,"y":841,"width":801,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,857,123],{"x":804,"y":844,"style":110},[44,859],{"x":808,"y":841,"width":809,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,861,748],{"x":812,"y":844,"style":110},[44,863],{"x":782,"y":864,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},"128.0",[65,866,123],{"x":819,"y":867,"style":69},"147.0",[44,869],{"x":786,"y":864,"width":787,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,871,111],{"x":790,"y":867,"style":110},[44,873],{"x":794,"y":864,"width":74,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,875,116],{"x":797,"y":867,"style":110},[44,877],{"x":800,"y":864,"width":801,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,879,87],{"x":804,"y":867,"style":110},[44,881],{"x":808,"y":864,"width":809,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,883,884],{"x":812,"y":867,"style":110},"ada@old.example",[44,886],{"x":782,"y":887,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},"156.0",[65,889,139],{"x":819,"y":890,"style":69},"175.0",[44,892],{"x":786,"y":887,"width":787,"height":82,"rx":47,"fill":215,"stroke":216,"style":217},[65,894,111],{"x":790,"y":890,"style":220},[44,896],{"x":794,"y":887,"width":74,"height":82,"rx":47,"fill":215,"stroke":216,"style":217},[65,898,132],{"x":797,"y":890,"style":220},[44,900],{"x":800,"y":887,"width":801,"height":82,"rx":47,"fill":215,"stroke":216,"style":217},[65,902,105],{"x":804,"y":890,"style":220},[44,904],{"x":808,"y":887,"width":809,"height":82,"rx":47,"fill":215,"stroke":216,"style":217},[65,906,907],{"x":812,"y":890,"style":220},"ada@new.example",[44,909],{"x":782,"y":910,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},"184.0",[65,912,155],{"x":819,"y":913,"style":69},"203.0",[44,915],{"x":786,"y":910,"width":787,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,917,160],{"x":790,"y":913,"style":110},[44,919],{"x":794,"y":910,"width":74,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,921,165],{"x":797,"y":913,"style":110},[44,923],{"x":800,"y":910,"width":801,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,925,87],{"x":804,"y":913,"style":110},[44,927],{"x":808,"y":910,"width":809,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,929,930],{"x":812,"y":913,"style":110},"ben@example.com",[44,932],{"x":782,"y":933,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},"212.0",[65,935,172],{"x":819,"y":936,"style":69},"231.0",[44,938],{"x":786,"y":933,"width":787,"height":82,"rx":47,"fill":215,"stroke":216,"style":217},[65,940,160],{"x":790,"y":936,"style":220},[44,942],{"x":794,"y":933,"width":74,"height":82,"rx":47,"fill":215,"stroke":216,"style":217},[65,944,165],{"x":797,"y":936,"style":220},[44,946],{"x":800,"y":933,"width":801,"height":82,"rx":47,"fill":215,"stroke":216,"style":217},[65,948,105],{"x":804,"y":936,"style":220},[44,950],{"x":808,"y":933,"width":809,"height":82,"rx":47,"fill":215,"stroke":216,"style":217},[65,952,953],{"x":812,"y":936,"style":220},"ben@corp.example",[44,955],{"x":782,"y":956,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},"240.0",[65,958,960],{"x":819,"y":959,"style":69},"259.0","7",[44,962],{"x":786,"y":956,"width":787,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,964,245],{"x":790,"y":959,"style":110},[44,966],{"x":794,"y":956,"width":74,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,968,969],{"x":797,"y":959,"style":110},"2026-02-15",[44,971],{"x":800,"y":956,"width":801,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[65,973,87],{"x":804,"y":959,"style":110},[44,975],{"x":808,"y":956,"width":809,"height":82,"rx":47,"fill":50,"stroke":58,"style":59},[44,977],{"x":782,"y":978,"width":55,"height":82,"rx":47,"fill":57,"stroke":58,"style":59},"268.0",[65,980,982],{"x":819,"y":981,"style":69},"287.0","8",[44,984],{"x":786,"y":978,"width":787,"height":82,"rx":47,"fill":215,"stroke":216,"style":217},[65,986,245],{"x":790,"y":981,"style":220},[44,988],{"x":794,"y":978,"width":74,"height":82,"rx":47,"fill":215,"stroke":216,"style":217},[65,990,250],{"x":797,"y":981,"style":220},[44,992],{"x":800,"y":978,"width":801,"height":82,"rx":47,"fill":215,"stroke":216,"style":217},[65,994,105],{"x":804,"y":981,"style":220},[44,996],{"x":808,"y":978,"width":809,"height":82,"rx":47,"fill":215,"stroke":216,"style":217},[65,998,999],{"x":812,"y":981,"style":220},"cleo@example.com",[289,1001,1003],{"id":1002},"step-3-do-not-use-groupbylast-for-this","Step 3 — Do not use groupby().last() for this",[10,1005,1006,1009,1010,1014],{},[14,1007,1008],{},"groupby(\"customer_id\").last()"," looks like a shorter way to write the same thing. It is not: it returns the last ",[1011,1012,1013],"em",{},"non-missing"," value of each column separately, so a record's fields can come from different rows:",[294,1016,1018],{"className":357,"code":1017,"language":359,"meta":299,"style":299},"mixed = hist.sort_values([\"customer_id\", \"updated\", \"seq\"], na_position=\"first\").groupby(\"customer_id\").last()\nprint(mixed[[\"email\", \"phone\"]])\n",[14,1019,1020,1056],{"__ignoreMap":299},[303,1021,1022,1025,1027,1030,1032,1034,1036,1038,1040,1042,1044,1046,1048,1051,1053],{"class":262,"line":305},[303,1023,1024],{"class":370},"mixed ",[303,1026,393],{"class":366},[303,1028,1029],{"class":370}," hist.sort_values([",[303,1031,530],{"class":312},[303,1033,458],{"class":370},[303,1035,415],{"class":312},[303,1037,458],{"class":370},[303,1039,677],{"class":312},[303,1041,439],{"class":370},[303,1043,682],{"class":442},[303,1045,393],{"class":366},[303,1047,687],{"class":312},[303,1049,1050],{"class":370},").groupby(",[303,1052,530],{"class":312},[303,1054,1055],{"class":370},").last()\n",[303,1057,1058,1060,1063,1065,1067,1069],{"class":262,"line":380},[303,1059,409],{"class":408},[303,1061,1062],{"class":370},"(mixed[[",[303,1064,643],{"class":312},[303,1066,458],{"class":370},[303,1068,732],{"class":312},[303,1070,1071],{"class":370},"]])\n",[10,1073,1074,1075,1078,1079,1082,1083,1086,1087,1090,1091,1094,1095,1098,1099,1102,1103,1105],{},"C2's newest row has phone ",[14,1076,1077],{},"555-0202",", and so does the result — but C1's newest row has no phone, so ",[14,1080,1081],{},"last()"," reaches back and takes ",[14,1084,1085],{},"555-0101"," from January, pairing it with March's email. C3 gets a phone from its February row. None of these combinations existed as a record. If you want ",[1011,1088,1089],{},"row"," semantics — the latest version of each customer as it was recorded — use ",[14,1092,1093],{},"drop_duplicates"," or ",[14,1096,1097],{},"idxmax",". If you want ",[1011,1100,1101],{},"field"," semantics — the most recent known value of each field — ",[14,1104,1081],{}," is right, but that is a different, deliberate operation, and the result should be labelled as such.",[289,1107,1109],{"id":1108},"step-4-or-select-rows-with-idxmax","Step 4 — Or select rows with idxmax",[10,1111,1112],{},"An equivalent formulation picks the index of the maximum date per key and selects those rows. It avoids sorting the whole frame and reads as \"the row with the latest date\":",[294,1114,1116],{"className":357,"code":1115,"language":359,"meta":299,"style":299},"dated = hist.dropna(subset=[\"updated\"])\nidx = dated.sort_values(\"seq\").groupby(\"customer_id\")[\"updated\"].idxmax()\nprint(hist.loc[idx, [\"customer_id\", \"updated\", \"seq\", \"email\"]].to_string(index=False))\n",[14,1117,1118,1140,1164],{"__ignoreMap":299},[303,1119,1120,1123,1125,1128,1131,1133,1136,1138],{"class":262,"line":305},[303,1121,1122],{"class":370},"dated ",[303,1124,393],{"class":366},[303,1126,1127],{"class":370}," hist.dropna(",[303,1129,1130],{"class":442},"subset",[303,1132,393],{"class":366},[303,1134,1135],{"class":370},"[",[303,1137,415],{"class":312},[303,1139,606],{"class":370},[303,1141,1142,1145,1147,1150,1152,1154,1156,1159,1161],{"class":262,"line":380},[303,1143,1144],{"class":370},"idx ",[303,1146,393],{"class":366},[303,1148,1149],{"class":370}," dated.sort_values(",[303,1151,677],{"class":312},[303,1153,1050],{"class":370},[303,1155,530],{"class":312},[303,1157,1158],{"class":370},")[",[303,1160,415],{"class":312},[303,1162,1163],{"class":370},"].idxmax()\n",[303,1165,1166,1168,1171,1173,1175,1177,1179,1181,1183,1185,1187,1189,1191,1193],{"class":262,"line":387},[303,1167,409],{"class":408},[303,1169,1170],{"class":370},"(hist.loc[idx, [",[303,1172,530],{"class":312},[303,1174,458],{"class":370},[303,1176,415],{"class":312},[303,1178,458],{"class":370},[303,1180,677],{"class":312},[303,1182,458],{"class":370},[303,1184,643],{"class":312},[303,1186,537],{"class":370},[303,1188,540],{"class":442},[303,1190,393],{"class":366},[303,1192,545],{"class":408},[303,1194,548],{"class":370},[10,1196,1197,1199,1200,1203,1204,1206,1207,1209,1210,1212,1213,1215],{},[14,1198,1097],{}," returns the ",[1011,1201,1202],{},"first"," index of the maximum, so sorting by ",[14,1205,328],{}," beforehand makes the tie-break deterministic in the other direction — here it keeps C2's ",[14,1208,328],{}," 1 row. That difference is exactly why a tie-breaker has to be chosen, not left to whichever method you happened to use. Undated rows are dropped before ",[14,1211,1097],{},", which cannot compare ",[14,1214,562],{},".",[25,1217,34,1223,34,1226,34,1229,34,1233,34,1238,34,1243,34,1249,34,1252,34,1256,34,1259,34,1263,34,1267,34,1272,34,1274,34,1284,34,1286,34,1291,34,1295,34,1297,34,1300,34,1302,34,1306,34,1308,34,1313,34,1315,34,1318,34,1320,34,1323,34,1326,34,1330,34,1332,34,1340,34,1342,34,1346,34,1348],{"viewBox":1218,"role":28,"ariaLabelledBy":1219,"xmlns":32,"style":1222},"4 0 752 174",[1220,1221],"lat-methods-t","lat-methods-d","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[36,1224,1225],{"id":1220},"Ways to reduce history to one row per key",[40,1227,1228],{"id":1221},"A comparison of three approaches. Sorting then drop_duplicates keep last keeps whole rows, handles ties through the sort order and handles missing dates with na_position. groupby idxmax keeps whole rows, keeps the first of tied maxima and needs missing dates removed. groupby last does not keep whole rows: it takes the last non-missing value of each column separately.",[44,1230],{"x":139,"y":47,"width":1231,"height":1232,"fill":50},"752","174",[44,1234],{"x":1235,"y":1236,"width":1237,"height":54,"rx":47,"fill":50,"stroke":58,"style":59},"20.0","16.0","260.2",[44,1239],{"x":1240,"y":1236,"width":1241,"height":54,"rx":47,"fill":1242,"stroke":267,"style":281},"280.2","153.3","#f0f4ff",[65,1244,1248],{"x":1245,"y":1246,"style":1247},"356.8","41.0","font-size:13px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Keeps whole rows",[44,1250],{"x":1251,"y":1236,"width":1241,"height":54,"rx":47,"fill":1242,"stroke":267,"style":281},"433.4",[65,1253,1255],{"x":1254,"y":1246,"style":1247},"510.1","Ties decided by",[44,1257],{"x":1258,"y":1236,"width":1241,"height":54,"rx":47,"fill":1242,"stroke":267,"style":281},"586.7",[65,1260,1262],{"x":1261,"y":1246,"style":1247},"663.4","Missing dates",[44,1264],{"x":1235,"y":1265,"width":1237,"height":1266,"rx":47,"fill":57,"stroke":58,"style":59},"56.0","34.0",[65,1268,1271],{"x":55,"y":1269,"style":1270},"78.0","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:start","sort + drop_duplicates(keep=\"last\")",[44,1273],{"x":1240,"y":1265,"width":1241,"height":1266,"rx":47,"fill":50,"stroke":58,"style":59},[1275,1276,1278],"g",{"transform":1277},"translate(347.8,64.0) scale(0.750)",[1279,1280],"path",{"d":1281,"fill":1282,"stroke":216,"style":1283},"M5 12.5l4.5 4.5L19 7","none","stroke-width:2.40px;stroke-linecap:round;stroke-linejoin:round",[44,1285],{"x":1251,"y":1265,"width":1241,"height":1266,"rx":47,"fill":50,"stroke":58,"style":59},[65,1287,1290],{"x":1254,"y":1288,"style":1289},"71.0","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","sort order incl.",[65,1292,1294],{"x":1254,"y":1293,"style":1289},"84.0","tie-breaker",[44,1296],{"x":1258,"y":1265,"width":1241,"height":1266,"rx":47,"fill":50,"stroke":58,"style":59},[65,1298,682],{"x":1261,"y":1299,"style":1289},"77.0",[44,1301],{"x":1235,"y":285,"width":1237,"height":1266,"rx":47,"fill":57,"stroke":58,"style":59},[65,1303,1305],{"x":55,"y":1304,"style":1270},"112.0","groupby(...)[\"updated\"].idxmax()",[44,1307],{"x":1240,"y":285,"width":1241,"height":1266,"rx":47,"fill":50,"stroke":58,"style":59},[1275,1309,1311],{"transform":1310},"translate(347.8,98.0) scale(0.750)",[1279,1312],{"d":1281,"fill":1282,"stroke":216,"style":1283},[44,1314],{"x":1251,"y":285,"width":1241,"height":1266,"rx":47,"fill":50,"stroke":58,"style":59},[65,1316,1317],{"x":1254,"y":104,"style":1289},"first of the maxima",[44,1319],{"x":1258,"y":285,"width":1241,"height":1266,"rx":47,"fill":50,"stroke":58,"style":59},[65,1321,1322],{"x":1261,"y":104,"style":1289},"drop first",[44,1324],{"x":1235,"y":1325,"width":1237,"height":1266,"rx":47,"fill":57,"stroke":58,"style":59},"124.0",[65,1327,1329],{"x":55,"y":1328,"style":1270},"146.0","groupby(...).last()",[44,1331],{"x":1240,"y":1325,"width":1241,"height":1266,"rx":47,"fill":50,"stroke":58,"style":59},[1275,1333,1335],{"transform":1334},"translate(348.8,133.0) scale(0.667)",[1279,1336],{"d":1337,"fill":1282,"stroke":1338,"style":1339},"M6 6l12 12 M18 6L6 18","var(--accent,#f43f8f)","stroke-width:2.70px;stroke-linecap:round;stroke-linejoin:round",[44,1341],{"x":1251,"y":1325,"width":1241,"height":1266,"rx":47,"fill":50,"stroke":58,"style":59},[65,1343,1345],{"x":1254,"y":1344,"style":1289},"145.0","per column",[44,1347],{"x":1258,"y":1325,"width":1241,"height":1266,"rx":47,"fill":50,"stroke":58,"style":59},[65,1349,1350],{"x":1261,"y":1344,"style":1289},"skipped per column",[289,1352,1354],{"id":1353},"take-a-snapshot-as-of-a-past-date","Take a snapshot as of a past date",[10,1356,1357],{},"The same technique answers \"what did the customer list look like at the end of February?\" — useful when a report for a past period must use the details that were current then, not today's. Filter to versions recorded on or before the date, then deduplicate:",[294,1359,1361],{"className":357,"code":1360,"language":359,"meta":299,"style":299},"def as_of(history, when):\n    cutoff = pd.Timestamp(when)\n    known = history[history[\"updated\"].notna() & (history[\"updated\"] \u003C= cutoff)]\n    return (known.sort_values([\"customer_id\", \"updated\", \"seq\"])\n                 .drop_duplicates(\"customer_id\", keep=\"last\"))\n\nprint(as_of(hist, \"2026-02-28\")[[\"customer_id\", \"updated\", \"email\"]].to_string(index=False))\n",[14,1362,1363,1375,1385,1416,1436,1453,1457],{"__ignoreMap":299},[303,1364,1365,1368,1372],{"class":262,"line":305},[303,1366,1367],{"class":366},"def",[303,1369,1371],{"class":1370},"s_Opv"," as_of",[303,1373,1374],{"class":370},"(history, when):\n",[303,1376,1377,1380,1382],{"class":262,"line":380},[303,1378,1379],{"class":370},"    cutoff ",[303,1381,393],{"class":366},[303,1383,1384],{"class":370}," pd.Timestamp(when)\n",[303,1386,1387,1390,1392,1395,1397,1400,1403,1406,1408,1410,1413],{"class":262,"line":387},[303,1388,1389],{"class":370},"    known ",[303,1391,393],{"class":366},[303,1393,1394],{"class":370}," history[history[",[303,1396,415],{"class":312},[303,1398,1399],{"class":370},"].notna() ",[303,1401,1402],{"class":366},"&",[303,1404,1405],{"class":370}," (history[",[303,1407,415],{"class":312},[303,1409,514],{"class":370},[303,1411,1412],{"class":366},"\u003C=",[303,1414,1415],{"class":370}," cutoff)]\n",[303,1417,1418,1421,1424,1426,1428,1430,1432,1434],{"class":262,"line":405},[303,1419,1420],{"class":366},"    return",[303,1422,1423],{"class":370}," (known.sort_values([",[303,1425,530],{"class":312},[303,1427,458],{"class":370},[303,1429,415],{"class":312},[303,1431,458],{"class":370},[303,1433,677],{"class":312},[303,1435,606],{"class":370},[303,1437,1438,1441,1443,1445,1447,1449,1451],{"class":262,"line":421},[303,1439,1440],{"class":370},"                 .drop_duplicates(",[303,1442,530],{"class":312},[303,1444,458],{"class":370},[303,1446,618],{"class":442},[303,1448,393],{"class":366},[303,1450,623],{"class":312},[303,1452,548],{"class":370},[303,1454,1455],{"class":262,"line":426},[303,1456,384],{"emptyLinePlaceholder":383},[303,1458,1459,1461,1464,1467,1470,1472,1474,1476,1478,1480,1482,1484,1486,1488],{"class":262,"line":471},[303,1460,409],{"class":408},[303,1462,1463],{"class":370},"(as_of(hist, ",[303,1465,1466],{"class":312},"\"2026-02-28\"",[303,1468,1469],{"class":370},")[[",[303,1471,530],{"class":312},[303,1473,458],{"class":370},[303,1475,415],{"class":312},[303,1477,458],{"class":370},[303,1479,643],{"class":312},[303,1481,537],{"class":370},[303,1483,540],{"class":442},[303,1485,393],{"class":366},[303,1487,545],{"class":408},[303,1489,548],{"class":370},[10,1491,1492,1493,1495],{},"At the end of February, C1's current email was still ",[14,1494,884],{}," and C3's was missing — their later versions did not yet exist. Customers with no version before the cutoff simply do not appear, which is correct: they were not customers yet. Undated rows are excluded from snapshots entirely, since there is no way to know when they applied.",[289,1497,1499],{"id":1498},"step-5-report-ties-you-cannot-break","Step 5 — Report ties you cannot break",[10,1501,1502],{},"When there is no sequence number or other tie-breaker, surface the ties instead of letting an arbitrary row win:",[294,1504,1506],{"className":357,"code":1505,"language":359,"meta":299,"style":299},"ties = hist[hist.duplicated([\"customer_id\", \"updated\"], keep=False) & hist[\"updated\"].notna()]\nprint(ties[[\"customer_id\", \"updated\", \"email\"]].to_string(index=False))\n",[14,1507,1508,1545],{"__ignoreMap":299},[303,1509,1510,1513,1515,1518,1520,1522,1524,1526,1528,1530,1532,1535,1537,1540,1542],{"class":262,"line":305},[303,1511,1512],{"class":370},"ties ",[303,1514,393],{"class":366},[303,1516,1517],{"class":370}," hist[hist.duplicated([",[303,1519,530],{"class":312},[303,1521,458],{"class":370},[303,1523,415],{"class":312},[303,1525,439],{"class":370},[303,1527,618],{"class":442},[303,1529,393],{"class":366},[303,1531,545],{"class":408},[303,1533,1534],{"class":370},") ",[303,1536,1402],{"class":366},[303,1538,1539],{"class":370}," hist[",[303,1541,415],{"class":312},[303,1543,1544],{"class":370},"].notna()]\n",[303,1546,1547,1549,1552,1554,1556,1558,1560,1562,1564,1566,1568,1570],{"class":262,"line":380},[303,1548,409],{"class":408},[303,1550,1551],{"class":370},"(ties[[",[303,1553,530],{"class":312},[303,1555,458],{"class":370},[303,1557,415],{"class":312},[303,1559,458],{"class":370},[303,1561,643],{"class":312},[303,1563,537],{"class":370},[303,1565,540],{"class":442},[303,1567,393],{"class":366},[303,1569,545],{"class":408},[303,1571,548],{"class":370},[10,1573,1574,1575,1577],{},"C2's two rows share a timestamp. With ",[14,1576,328],{}," available the tie is resolved; without it, list such keys for review. They often point at a real problem in the source — two systems writing the same record at the same moment, or a timestamp stored only to the day.",[289,1579,1581],{"id":1580},"step-6-keep-the-history-you-discarded","Step 6 — Keep the history you discarded",[10,1583,1584],{},"Deduplication throws information away, and someone will ask for it. Write the latest records and the superseded ones to separate sheets, so the result is auditable:",[294,1586,1588],{"className":357,"code":1587,"language":359,"meta":299,"style":299},"superseded = hist.drop(index=latest.index)\nwith pd.ExcelWriter(\"customers_latest.xlsx\") as writer:\n    latest.sort_values(\"customer_id\").to_excel(writer, sheet_name=\"Current\", index=False)\n    superseded.sort_values([\"customer_id\", \"updated\"], na_position=\"first\").to_excel(writer, sheet_name=\"History\", index=False)\nprint(pd.read_excel(\"customers_latest.xlsx\", sheet_name=None)[\"Current\"].shape,\n      pd.read_excel(\"customers_latest.xlsx\", sheet_name=None)[\"History\"].shape)\n",[14,1589,1590,1607,1625,1653,1691,1716],{"__ignoreMap":299},[303,1591,1592,1595,1597,1600,1602,1604],{"class":262,"line":305},[303,1593,1594],{"class":370},"superseded ",[303,1596,393],{"class":366},[303,1598,1599],{"class":370}," hist.drop(",[303,1601,540],{"class":442},[303,1603,393],{"class":366},[303,1605,1606],{"class":370},"latest.index)\n",[303,1608,1609,1612,1615,1618,1620,1622],{"class":262,"line":380},[303,1610,1611],{"class":366},"with",[303,1613,1614],{"class":370}," pd.ExcelWriter(",[303,1616,1617],{"class":312},"\"customers_latest.xlsx\"",[303,1619,1534],{"class":370},[303,1621,374],{"class":366},[303,1623,1624],{"class":370}," writer:\n",[303,1626,1627,1630,1632,1635,1638,1640,1643,1645,1647,1649,1651],{"class":262,"line":387},[303,1628,1629],{"class":370},"    latest.sort_values(",[303,1631,530],{"class":312},[303,1633,1634],{"class":370},").to_excel(writer, ",[303,1636,1637],{"class":442},"sheet_name",[303,1639,393],{"class":366},[303,1641,1642],{"class":312},"\"Current\"",[303,1644,458],{"class":370},[303,1646,540],{"class":442},[303,1648,393],{"class":366},[303,1650,545],{"class":408},[303,1652,402],{"class":370},[303,1654,1655,1658,1660,1662,1664,1666,1668,1670,1672,1674,1676,1678,1681,1683,1685,1687,1689],{"class":262,"line":405},[303,1656,1657],{"class":370},"    superseded.sort_values([",[303,1659,530],{"class":312},[303,1661,458],{"class":370},[303,1663,415],{"class":312},[303,1665,439],{"class":370},[303,1667,682],{"class":442},[303,1669,393],{"class":366},[303,1671,687],{"class":312},[303,1673,1634],{"class":370},[303,1675,1637],{"class":442},[303,1677,393],{"class":366},[303,1679,1680],{"class":312},"\"History\"",[303,1682,458],{"class":370},[303,1684,540],{"class":442},[303,1686,393],{"class":366},[303,1688,545],{"class":408},[303,1690,402],{"class":370},[303,1692,1693,1695,1698,1700,1702,1704,1706,1709,1711,1713],{"class":262,"line":421},[303,1694,409],{"class":408},[303,1696,1697],{"class":370},"(pd.read_excel(",[303,1699,1617],{"class":312},[303,1701,458],{"class":370},[303,1703,1637],{"class":442},[303,1705,393],{"class":366},[303,1707,1708],{"class":408},"None",[303,1710,1158],{"class":370},[303,1712,1642],{"class":312},[303,1714,1715],{"class":370},"].shape,\n",[303,1717,1718,1721,1723,1725,1727,1729,1731,1733,1735],{"class":262,"line":426},[303,1719,1720],{"class":370},"      pd.read_excel(",[303,1722,1617],{"class":312},[303,1724,458],{"class":370},[303,1726,1637],{"class":442},[303,1728,393],{"class":366},[303,1730,1708],{"class":408},[303,1732,1158],{"class":370},[303,1734,1680],{"class":312},[303,1736,1737],{"class":370},"].shape)\n",[10,1739,1740,1741,1745],{},"Three current records and four historical ones: together they account for all seven input rows, which is a quick sanity check to assert in a job: if the two sheets do not add up to the input, a row was lost or duplicated somewhere along the way. ",[19,1742,1744],{"href":1743},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fpandas-drop-duplicates-from-excel-column\u002F","Pandas: Drop Duplicates From an Excel Column"," covers duplicates that are true copies rather than versions.",[289,1747,1749],{"id":1748},"common-pitfalls","Common pitfalls",[1751,1752,1753,1761,1773,1779,1787,1793],"ul",{},[1754,1755,1756,1760],"li",{},[1757,1758,1759],"strong",{},"Sorting text dates."," They sort alphabetically; parse first.",[1754,1762,1763,1766,1767,1769,1770,1772],{},[1757,1764,1765],{},"Undated rows winning."," ",[14,1768,562],{}," sorts last by default; use ",[14,1771,752],{}," or exclude them.",[1754,1774,1775,1778],{},[1757,1776,1777],{},"No tie-breaker."," Equal timestamps leave the winner to chance.",[1754,1780,1781,1786],{},[1757,1782,1783,1785],{},[14,1784,16],{}," for records."," It builds rows that never existed.",[1754,1788,1789,1792],{},[1757,1790,1791],{},"Deduplicating before filtering."," When a report covers one region or one period, apply those filters to the history first if the latest version depends on them — the newest row overall may belong to a different region than the newest row within it.",[1754,1794,1795,1766,1798,1801,1802,1805],{},[1757,1796,1797],{},"Forgetting the key's own cleanliness.",[14,1799,1800],{},"\" C1\""," and ",[14,1803,1804],{},"\"C1\""," are different keys; strip and normalise IDs first.",[289,1807,1809],{"id":1808},"performance-notes","Performance notes",[10,1811,1812,1813,1815,1816,1818,1819,1821],{},"Sorting and ",[14,1814,1093],{}," are both vectorised and handle millions of rows in seconds. ",[14,1817,1097],{}," avoids sorting the whole frame, which helps a little on very large histories. Parsing text dates with explicit formats is much faster than ",[14,1820,554],{},", which tries each value individually.",[289,1823,1825],{"id":1824},"conclusion","Conclusion",[10,1827,1828,1829,1832,1833,1835,1836,1838],{},"Parse the timestamp into real dates with explicit formats, sort by key, timestamp and a tie-breaker with undated rows placed first, and keep the last row per key with ",[14,1830,1831],{},"drop_duplicates(keep=\"last\")"," — or select rows with ",[14,1834,1097],{}," and an explicit tie rule. Avoid ",[14,1837,16],{}," for records, because it assembles fields from different rows. Report ties you cannot break, and keep the superseded rows on their own sheet.",[289,1840,1842],{"id":1841},"frequently-asked-questions","Frequently asked questions",[10,1844,1845,1848],{},[1757,1846,1847],{},"What is the quickest way to keep the latest row per ID?","\nSort by the ID and the timestamp, then call drop_duplicates(subset=\"id\", keep=\"last\"). After the sort, the last row of each ID is the most recent one.",[10,1850,1851,1854],{},[1757,1852,1853],{},"Why not use groupby(\"id\").last()?","\nBecause it returns the last non-missing value of each column separately. If the newest row has no phone number, you get the phone from an older row next to the email from the newest one — a record that never existed.",[10,1856,1857,1860],{},[1757,1858,1859],{},"What if two rows have the same timestamp?","\nAdd a tie-breaker to the sort, such as a sequence number or the source row order, so the result does not depend on how pandas happens to order equal values. Report ties if there is no sensible tie-breaker.",[10,1862,1863,1866],{},[1757,1864,1865],{},"How are rows with no date treated?","\nsort_values puts missing values last by default, so an undated row would be kept as the \"latest\". Decide explicitly: pass na_position=\"first\" to rank undated rows as oldest, or exclude and report them.",[289,1868,1870],{"id":1869},"related","Related",[1751,1872,1873,1879,1886,1893],{},[1754,1874,1875,1876,1878],{},"Up one level: ",[19,1877,22],{"href":21}," — the full cleaning workflow.",[1754,1880,1881,1885],{},[19,1882,1884],{"href":1883},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Ffind-duplicate-rows-in-excel-with-python\u002F","Find Duplicate Rows in Excel with Python"," — finding duplicates before deciding what to keep.",[1754,1887,1888,1892],{},[19,1889,1891],{"href":1890},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fupdate-one-excel-file-from-another-with-pandas\u002F","Update One Excel File from Another with pandas"," — applying newer values to a master list.",[1754,1894,1895,1899],{},[19,1896,1898],{"href":1897},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fmatch-rows-by-nearest-date-with-merge-asof\u002F","Match Rows by Nearest Date with pandas merge_asof"," — picking the right version at a point in time.",[1901,1902,1903],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}",{"title":299,"searchDepth":380,"depth":380,"links":1905},[1906,1907,1908,1909,1910,1911,1912,1913,1914,1915,1916,1917,1918],{"id":291,"depth":380,"text":292},{"id":336,"depth":380,"text":337},{"id":571,"depth":380,"text":572},{"id":1002,"depth":380,"text":1003},{"id":1108,"depth":380,"text":1109},{"id":1353,"depth":380,"text":1354},{"id":1498,"depth":380,"text":1499},{"id":1580,"depth":380,"text":1581},{"id":1748,"depth":380,"text":1749},{"id":1808,"depth":380,"text":1809},{"id":1824,"depth":380,"text":1825},{"id":1841,"depth":380,"text":1842},{"id":1869,"depth":380,"text":1870},"2026-09-25","A customer export lists the same customer several times as their details changed. Keep only the most recent row per customer with sort_values and drop_duplicates, break ties on equal timestamps, handle dates stored as text and missing dates, and avoid groupby().last(), which quietly mixes values from different rows.","md",[1923,1925,1927,1929],{"q":1847,"a":1924},"Sort by the ID and the timestamp, then call drop_duplicates(subset=\"id\", keep=\"last\"). After the sort, the last row of each ID is the most recent one.",{"q":1853,"a":1926},"Because it returns the last non-missing value of each column separately. If the newest row has no phone number, you get the phone from an older row next to the email from the newest one — a record that never existed.",{"q":1859,"a":1928},"Add a tie-breaker to the sort, such as a sequence number or the source row order, so the result does not depend on how pandas happens to order equal values. Report ties if there is no sensible tie-breaker.",{"q":1865,"a":1930},"sort_values puts missing values last by default, so an undated row would be kept as the \"latest\". Decide explicitly: pass na_position=\"first\" to rank undated rows as oldest, or exclude and report them.","keep the latest record per key when deduplicating in pandas",null,{"breadcrumb":1934},[1935,1938,1941],{"name":1936,"item":1937},"Home","\u002F",{"name":1939,"item":1940},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":1942,"item":21},"Cleaning Excel Data With Pandas: A Production Workflow","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fkeep-the-latest-record-per-key-with-pandas",{"title":1945,"description":1946},"pandas: Keep the Most Recent Row per ID (drop_duplicates keep=\"last\")","Deduplicate Excel data keeping the latest record per key in pandas: sort then drop_duplicates, idxmax, tie-breakers, text dates and missing dates, why groupby last mixes rows, and keeping the discarded history.","keep-the-latest-record-per-key-with-pandas","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fkeep-the-latest-record-per-key-with-pandas\u002Findex",[1950,1951,1952],"data-cleaning","pandas","dates-and-times","how-to","_myqaFGbkAawptT9ZYATLmV7fdGfPB_Q40b57UXUWzQ",[1956,1960],{"title":1957,"path":1958,"stem":1959,"children":-1},"Fix the Capitalisation of Names with Exceptions in pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-the-capitalisation-of-names-with-exceptions-in-pandas","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-the-capitalisation-of-names-with-exceptions-in-pandas\u002Findex",{"title":1961,"path":1962,"stem":1963,"children":-1},"Map Free-Text Job Titles to Standard Categories","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fmap-free-text-job-titles-to-standard-categories","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fmap-free-text-job-titles-to-standard-categories\u002Findex",[1965,1967],{"path":1966,"title":1942},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas",{"path":1968,"title":1939},"\u002Fadvanced-data-transformation-and-cleaning",[],[1971,1973,1976,1979,1982,1985],{"path":1972,"title":1884,"kind":1953},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Ffind-duplicate-rows-in-excel-with-python",{"path":1974,"title":1975,"kind":1953},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-duplicate-values-in-excel-with-openpyxl","Highlight Duplicate Values in Excel with openpyxl",{"path":1977,"title":1978,"kind":1953},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fclean-mixed-currency-amounts-into-amount-and-currency-columns","Clean Mixed-Currency Amounts into Amount and Currency Columns",{"path":1980,"title":1981,"kind":1953},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fcoalesce-several-excel-columns-into-one-with-pandas","Coalesce Several Excel Columns into One with pandas",{"path":1983,"title":1984,"kind":1953},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-currency-strings-to-numbers-with-pandas","Convert Currency Strings Like \"$1,234.56\" and \"(500)\" to Numbers with pandas",{"path":1986,"title":1987,"kind":1953},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-excel-text-columns-to-numbers-with-pandas","Convert Excel Text Columns to Numbers with pandas",1790972080797]