[{"data":1,"prerenderedAt":1992},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Ftake-a-random-sample-of-rows-from-a-large-excel-file":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Ftake-a-random-sample-of-rows-from-a-large-excel-file":1958,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Ftake-a-random-sample-of-rows-from-a-large-excel-file":1967,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Ftake-a-random-sample-of-rows-from-a-large-excel-file":1972,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Ftake-a-random-sample-of-rows-from-a-large-excel-file":1973},{"id":4,"title":5,"body":6,"dateModified":1925,"datePublished":1925,"description":1926,"extension":1927,"faq":1928,"keyword":1937,"kind":1938,"meta":1939,"navigation":94,"path":1947,"seo":1948,"slug":1951,"stem":1952,"topics":1953,"type":1956,"__hash__":1957},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Ftake-a-random-sample-of-rows-from-a-large-excel-file\u002Findex.md","Take a Random Sample of Rows from a Large Excel File",{"type":7,"value":8,"toc":1913},"minimark",[9,19,24,53,59,63,66,220,223,227,249,623,638,788,792,799,1172,1183,1260,1264,1267,1533,1539,1621,1625,1632,1786,1796,1800,1838,1842,1848,1852,1858,1864,1870,1876,1880,1909],[10,11,12,13,18],"p",{},"Exploring a 400,000-row export, building and testing a report against realistic data, or sending a colleague \"a representative slice\" all call for a sample. The tempting shortcut — the first 10,000 rows — is not a sample: exports are usually sorted by date, customer or region, so the first rows are the oldest records, or one customer, or one region, and every figure computed from them is biased. A proper random sample gives every row the same chance of being chosen. When the file is too big to load, that means sampling while streaming. This page uses reservoir sampling to draw a fixed-size random sample in one pass, extends it to a stratified sample that keeps small groups represented, shows a pandas shortcut for when the row count is known, and checks the sample against the full file. It belongs to ",[14,15,17],"a",{"href":16},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002F","Working with Large Excel Files in Python",".",[20,21,23],"h2",{"id":22},"prerequisites","Prerequisites",[25,26,31],"pre",{"className":27,"code":28,"language":29,"meta":30,"style":30},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install openpyxl pandas\n","bash","",[32,33,34],"code",{"__ignoreMap":30},[35,36,39,43,47,50],"span",{"class":37,"line":38},"line",1,[35,40,42],{"class":41},"sMTad","pip",[35,44,46],{"class":45},"srMev"," install",[35,48,49],{"class":45}," openpyxl",[35,51,52],{"class":45}," pandas\n",[10,54,55,58],{},[32,56,57],{},"transactions.xlsx"," holds 200,000 transactions with an ID, a region and an amount. Like many real exports it is not shuffled: the last 10,000 rows are the small \"Islands\" region, 5% of the data,, whose amounts are higher. That ordering is exactly what makes \"the first N rows\" misleading.",[20,60,62],{"id":61},"step-1-why-the-first-rows-are-not-a-sample","Step 1 — Why the first rows are not a sample",[10,64,65],{},"Compare the first 10,000 rows with the full file:",[25,67,71],{"className":68,"code":69,"language":70,"meta":30,"style":30},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\nfull = pd.read_excel(\"transactions.xlsx\")          # fine here; the point is to avoid this on bigger files\nhead = pd.read_excel(\"transactions.xlsx\", nrows=10_000)\nprint(\"full mean:\", round(full[\"Amount\"].mean(), 2), \"| first 10,000 mean:\", round(head[\"Amount\"].mean(), 2))\nprint(\"regions in first 10,000:\", sorted(head[\"Region\"].unique()))\n","python",[32,72,73,89,96,118,146,197],{"__ignoreMap":30},[35,74,75,79,83,86],{"class":37,"line":38},[35,76,78],{"class":77},"s-kum","import",[35,80,82],{"class":81},"skGVy"," pandas ",[35,84,85],{"class":77},"as",[35,87,88],{"class":81}," pd\n",[35,90,92],{"class":37,"line":91},2,[35,93,95],{"emptyLinePlaceholder":94},true,"\n",[35,97,99,102,105,108,111,114],{"class":37,"line":98},3,[35,100,101],{"class":81},"full ",[35,103,104],{"class":77},"=",[35,106,107],{"class":81}," pd.read_excel(",[35,109,110],{"class":45},"\"transactions.xlsx\"",[35,112,113],{"class":81},")          ",[35,115,117],{"class":116},"s-wDw","# fine here; the point is to avoid this on bigger files\n",[35,119,121,124,126,128,130,133,137,139,143],{"class":37,"line":120},4,[35,122,123],{"class":81},"head ",[35,125,104],{"class":77},[35,127,107],{"class":81},[35,129,110],{"class":45},[35,131,132],{"class":81},", ",[35,134,136],{"class":135},"sa561","nrows",[35,138,104],{"class":77},[35,140,142],{"class":141},"sP0c6","10_000",[35,144,145],{"class":81},")\n",[35,147,149,152,155,158,160,163,166,169,172,175,178,181,183,185,188,190,192,194],{"class":37,"line":148},5,[35,150,151],{"class":141},"print",[35,153,154],{"class":81},"(",[35,156,157],{"class":45},"\"full mean:\"",[35,159,132],{"class":81},[35,161,162],{"class":141},"round",[35,164,165],{"class":81},"(full[",[35,167,168],{"class":45},"\"Amount\"",[35,170,171],{"class":81},"].mean(), ",[35,173,174],{"class":141},"2",[35,176,177],{"class":81},"), ",[35,179,180],{"class":45},"\"| first 10,000 mean:\"",[35,182,132],{"class":81},[35,184,162],{"class":141},[35,186,187],{"class":81},"(head[",[35,189,168],{"class":45},[35,191,171],{"class":81},[35,193,174],{"class":141},[35,195,196],{"class":81},"))\n",[35,198,200,202,204,207,209,212,214,217],{"class":37,"line":199},6,[35,201,151],{"class":141},[35,203,154],{"class":81},[35,205,206],{"class":45},"\"regions in first 10,000:\"",[35,208,132],{"class":81},[35,210,211],{"class":141},"sorted",[35,213,187],{"class":81},[35,215,216],{"class":45},"\"Region\"",[35,218,219],{"class":81},"].unique()))\n",[10,221,222],{},"The first 10,000 rows contain no Islands transactions at all, so their average amount, 249.89, is well below the true 257.48, and one region is simply missing. A report built and tested on that slice would look fine until it met the real data. Loading the full file here is only to have the true figures for comparison; the methods below never do.",[20,224,226],{"id":225},"step-2-reservoir-sampling-in-one-streaming-pass","Step 2 — Reservoir sampling in one streaming pass",[10,228,229,230,234,235,237,238,240,241,244,245,248],{},"Reservoir sampling draws a uniform random sample of fixed size ",[231,232,233],"em",{},"k"," from a stream of unknown length, keeping only ",[231,236,233],{}," rows in memory. Fill the reservoir with the first ",[231,239,233],{}," rows; after that, row number ",[231,242,243],{},"i"," replaces a random reservoir slot with probability ",[231,246,247],{},"k \u002F i",":",[25,250,252],{"className":68,"code":251,"language":70,"meta":30,"style":30},"import random\nfrom openpyxl import load_workbook\n\ndef reservoir_sample(path, k, seed=42, sheet=None):\n    rnd = random.Random(seed)\n    wb = load_workbook(path, read_only=True)\n    rows = (wb[sheet] if sheet else wb.active).iter_rows(values_only=True)\n    header = next(rows)\n    reservoir = []\n    for i, row in enumerate(rows, start=1):\n        if i \u003C= k:\n            reservoir.append(row)\n        else:\n            j = rnd.randrange(i)              # 0 … i-1\n            if j \u003C k:\n                reservoir[j] = row\n    wb.close()\n    return pd.DataFrame(reservoir, columns=header), i\n\nsample, seen = reservoir_sample(\"transactions.xlsx\", k=10_000)\nprint(f\"sampled {len(sample):,} of {seen:,} rows\")\nprint(\"sample mean:\", round(sample[\"Amount\"].mean(), 2), \"| regions:\", sample[\"Region\"].value_counts().to_dict())\n",[32,253,254,261,274,278,306,316,336,368,382,393,421,436,442,451,465,479,490,496,513,518,541,587],{"__ignoreMap":30},[35,255,256,258],{"class":37,"line":38},[35,257,78],{"class":77},[35,259,260],{"class":81}," random\n",[35,262,263,266,269,271],{"class":37,"line":91},[35,264,265],{"class":77},"from",[35,267,268],{"class":81}," openpyxl ",[35,270,78],{"class":77},[35,272,273],{"class":81}," load_workbook\n",[35,275,276],{"class":37,"line":98},[35,277,95],{"emptyLinePlaceholder":94},[35,279,280,283,287,290,292,295,298,300,303],{"class":37,"line":120},[35,281,282],{"class":77},"def",[35,284,286],{"class":285},"s_Opv"," reservoir_sample",[35,288,289],{"class":81},"(path, k, seed",[35,291,104],{"class":77},[35,293,294],{"class":141},"42",[35,296,297],{"class":81},", sheet",[35,299,104],{"class":77},[35,301,302],{"class":141},"None",[35,304,305],{"class":81},"):\n",[35,307,308,311,313],{"class":37,"line":148},[35,309,310],{"class":81},"    rnd ",[35,312,104],{"class":77},[35,314,315],{"class":81}," random.Random(seed)\n",[35,317,318,321,323,326,329,331,334],{"class":37,"line":199},[35,319,320],{"class":81},"    wb ",[35,322,104],{"class":77},[35,324,325],{"class":81}," load_workbook(path, ",[35,327,328],{"class":135},"read_only",[35,330,104],{"class":77},[35,332,333],{"class":141},"True",[35,335,145],{"class":81},[35,337,339,342,344,347,350,353,356,359,362,364,366],{"class":37,"line":338},7,[35,340,341],{"class":81},"    rows ",[35,343,104],{"class":77},[35,345,346],{"class":81}," (wb[sheet] ",[35,348,349],{"class":77},"if",[35,351,352],{"class":81}," sheet ",[35,354,355],{"class":77},"else",[35,357,358],{"class":81}," wb.active).iter_rows(",[35,360,361],{"class":135},"values_only",[35,363,104],{"class":77},[35,365,333],{"class":141},[35,367,145],{"class":81},[35,369,371,374,376,379],{"class":37,"line":370},8,[35,372,373],{"class":81},"    header ",[35,375,104],{"class":77},[35,377,378],{"class":141}," next",[35,380,381],{"class":81},"(rows)\n",[35,383,385,388,390],{"class":37,"line":384},9,[35,386,387],{"class":81},"    reservoir ",[35,389,104],{"class":77},[35,391,392],{"class":81}," []\n",[35,394,396,399,402,405,408,411,414,416,419],{"class":37,"line":395},10,[35,397,398],{"class":77},"    for",[35,400,401],{"class":81}," i, row ",[35,403,404],{"class":77},"in",[35,406,407],{"class":141}," enumerate",[35,409,410],{"class":81},"(rows, ",[35,412,413],{"class":135},"start",[35,415,104],{"class":77},[35,417,418],{"class":141},"1",[35,420,305],{"class":81},[35,422,424,427,430,433],{"class":37,"line":423},11,[35,425,426],{"class":77},"        if",[35,428,429],{"class":81}," i ",[35,431,432],{"class":77},"\u003C=",[35,434,435],{"class":81}," k:\n",[35,437,439],{"class":37,"line":438},12,[35,440,441],{"class":81},"            reservoir.append(row)\n",[35,443,445,448],{"class":37,"line":444},13,[35,446,447],{"class":77},"        else",[35,449,450],{"class":81},":\n",[35,452,454,457,459,462],{"class":37,"line":453},14,[35,455,456],{"class":81},"            j ",[35,458,104],{"class":77},[35,460,461],{"class":81}," rnd.randrange(i)              ",[35,463,464],{"class":116},"# 0 … i-1\n",[35,466,468,471,474,477],{"class":37,"line":467},15,[35,469,470],{"class":77},"            if",[35,472,473],{"class":81}," j ",[35,475,476],{"class":77},"\u003C",[35,478,435],{"class":81},[35,480,482,485,487],{"class":37,"line":481},16,[35,483,484],{"class":81},"                reservoir[j] ",[35,486,104],{"class":77},[35,488,489],{"class":81}," row\n",[35,491,493],{"class":37,"line":492},17,[35,494,495],{"class":81},"    wb.close()\n",[35,497,499,502,505,508,510],{"class":37,"line":498},18,[35,500,501],{"class":77},"    return",[35,503,504],{"class":81}," pd.DataFrame(reservoir, ",[35,506,507],{"class":135},"columns",[35,509,104],{"class":77},[35,511,512],{"class":81},"header), i\n",[35,514,516],{"class":37,"line":515},19,[35,517,95],{"emptyLinePlaceholder":94},[35,519,521,524,526,529,531,533,535,537,539],{"class":37,"line":520},20,[35,522,523],{"class":81},"sample, seen ",[35,525,104],{"class":77},[35,527,528],{"class":81}," reservoir_sample(",[35,530,110],{"class":45},[35,532,132],{"class":81},[35,534,233],{"class":135},[35,536,104],{"class":77},[35,538,142],{"class":141},[35,540,145],{"class":81},[35,542,544,546,548,551,554,558,561,564,567,570,573,575,578,580,582,585],{"class":37,"line":543},21,[35,545,151],{"class":141},[35,547,154],{"class":81},[35,549,550],{"class":77},"f",[35,552,553],{"class":45},"\"sampled ",[35,555,557],{"class":556},"sSjpA","{",[35,559,560],{"class":141},"len",[35,562,563],{"class":81},"(sample)",[35,565,566],{"class":77},":,",[35,568,569],{"class":556},"}",[35,571,572],{"class":45}," of ",[35,574,557],{"class":556},[35,576,577],{"class":81},"seen",[35,579,566],{"class":77},[35,581,569],{"class":556},[35,583,584],{"class":45}," rows\"",[35,586,145],{"class":81},[35,588,590,592,594,597,599,601,604,606,608,610,612,615,618,620],{"class":37,"line":589},22,[35,591,151],{"class":141},[35,593,154],{"class":81},[35,595,596],{"class":45},"\"sample mean:\"",[35,598,132],{"class":81},[35,600,162],{"class":141},[35,602,603],{"class":81},"(sample[",[35,605,168],{"class":45},[35,607,171],{"class":81},[35,609,174],{"class":141},[35,611,177],{"class":81},[35,613,614],{"class":45},"\"| regions:\"",[35,616,617],{"class":81},", sample[",[35,619,216],{"class":45},[35,621,622],{"class":81},"].value_counts().to_dict())\n",[10,624,625,626,629,630,633,634,637],{},"After one pass every row has had exactly the same probability, ",[231,627,628],{},"k \u002F n",", of ending up in the sample — the algorithm's guarantee — even though ",[231,631,632],{},"n"," was unknown until the end. The sample's mean is close to the full file's, and the Islands appear in roughly their true proportion. Memory holds 10,000 rows at most. Using a dedicated ",[32,635,636],{},"random.Random(seed)"," instance makes the sample reproducible: the same seed gives the same rows, which matters when a colleague needs to rerun your analysis.",[639,640,648,649,648,653,648,657,648,665,648,675,648,686,648,693,648,698,648,704,648,709,648,713,648,720,648,725,648,728,648,732,648,735,648,740,648,747,648,752,648,755,648,759,648,762,648,767,648,774,648,780,648,784],"svg",{"viewBox":641,"role":642,"ariaLabelledBy":643,"xmlns":646,"style":647},"4 0 752 142","img",[644,645],"rs-alg-t","rs-alg-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[650,651,652],"title",{"id":644},"Reservoir sampling, row by row",[654,655,656],"desc",{"id":645},"The first k rows fill the reservoir. For each later row number i, a random number from 0 to i minus 1 is drawn; if it is below k the row replaces that slot, otherwise the row is skipped. At the end every row had the same chance k over n of being kept.",[658,659],"rect",{"x":660,"y":661,"width":662,"height":663,"fill":664},"4","0","752","142","#ffffff",[658,666],{"x":667,"y":668,"width":669,"height":670,"rx":671,"fill":672,"stroke":673,"style":674},"20.0","16.0","153.0","110.5","12","#e7ebef","var(--line,#cdd5e6)","stroke-width:2px",[676,677,679],"g",{"transform":678},"translate(84.5,37.8) scale(1.000)",[680,681],"path",{"d":682,"fill":683,"stroke":684,"style":685},"M12 3l9 5-9 5-9-5z M3 13l9 5 9-5 M3 17l9 5 9-5","none","var(--text,#172033)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[687,688,692],"text",{"x":689,"y":690,"style":691},"96.5","84.2","font-size:13.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","Fill",[687,694,697],{"x":689,"y":695,"style":696},"103.8","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","first k rows",[37,699],{"x1":700,"y1":701,"x2":702,"y2":701,"stroke":703,"style":674},"178.0","71.2","200.0","var(--brand,#5b5cf0)",[705,706],"polygon",{"points":707,"fill":708},"204.0,71.2 195.0,76.2 195.0,66.2","#5b5cf0",[658,710],{"x":711,"y":668,"width":669,"height":670,"rx":671,"fill":712,"stroke":703,"style":674},"209.0","#f0f4ff",[676,714,716],{"transform":715},"translate(273.5,37.8) scale(1.000)",[680,717],{"d":718,"fill":683,"stroke":719,"style":685},"M5 9h14 M4 15h14 M10 4l-2 16 M16 4l-2 16","var(--brand-strong,#4338ca)",[687,721,724],{"x":722,"y":690,"style":723},"285.5","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Row i",[687,726,727],{"x":722,"y":695,"style":696},"draw j in 0…i−1",[37,729],{"x1":730,"y1":701,"x2":731,"y2":701,"stroke":703,"style":674},"367.0","389.0",[705,733],{"points":734,"fill":708},"393.0,71.2 384.0,76.2 384.0,66.2",[658,736],{"x":737,"y":668,"width":669,"height":670,"rx":671,"fill":738,"stroke":739,"style":674},"398.0","#d9f4f1","var(--teal,#0f9488)",[676,741,743],{"transform":742},"translate(462.5,37.8) scale(1.000)",[680,744],{"d":745,"fill":683,"stroke":746,"style":685},"M20 11a8 8 0 1 0-2.3 5.7 M20 4v7h-7","var(--teal-ink,#0b6157)",[687,748,751],{"x":749,"y":690,"style":750},"474.5","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","j \u003C k?",[687,753,754],{"x":749,"y":695,"style":696},"replace slot j",[37,756],{"x1":757,"y1":701,"x2":758,"y2":701,"stroke":703,"style":674},"556.0","578.0",[705,760],{"points":761,"fill":708},"582.0,71.2 573.0,76.2 573.0,66.2",[658,763],{"x":764,"y":668,"width":669,"height":670,"rx":671,"fill":765,"stroke":766,"style":674},"587.0","#fdefd8","var(--gold,#b4740a)",[676,768,770],{"transform":769},"translate(651.5,29.0) scale(1.000)",[680,771],{"d":772,"fill":683,"stroke":773,"style":685},"M5 12.5l4.5 4.5L19 7","var(--gold-ink,#7a4e06)",[687,775,779],{"x":776,"y":777,"style":778},"663.5","75.5","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","End of file",[687,781,783],{"x":776,"y":782,"style":696},"95.0","every row had k\u002Fn",[687,785,787],{"x":776,"y":786,"style":696},"112.5","chance",[20,789,791],{"id":790},"step-3-a-stratified-sample-keeps-small-groups-visible","Step 3 — A stratified sample keeps small groups visible",[10,793,794,795,798],{},"A simple random sample reflects proportions, so a group that is 1% of the data gets about 1% of the sample — sometimes too few rows to say anything about it. A ",[231,796,797],{},"stratified"," sample draws a fixed number per group. With streaming, that is one reservoir per group:",[25,800,802],{"className":68,"code":801,"language":70,"meta":30,"style":30},"from collections import defaultdict\n\ndef stratified_sample(path, column, per_group, seed=42):\n    rnd = random.Random(seed)\n    wb = load_workbook(path, read_only=True)\n    rows = wb.active.iter_rows(values_only=True)\n    header = next(rows)\n    col = header.index(column)\n    reservoirs, counts = defaultdict(list), defaultdict(int)\n    for row in rows:\n        g = row[col]\n        counts[g] += 1\n        res = reservoirs[g]\n        if len(res) \u003C per_group:\n            res.append(row)\n        else:\n            j = rnd.randrange(counts[g])\n            if j \u003C per_group:\n                res[j] = row\n    wb.close()\n    df = pd.DataFrame([r for res in reservoirs.values() for r in res], columns=header)\n    weights = {g: counts[g] \u002F len(reservoirs[g]) for g in counts}\n    return df, weights\n\nstrat, weights = stratified_sample(\"transactions.xlsx\", \"Region\", per_group=2_000)\nprint(strat[\"Region\"].value_counts().to_dict())\nprint(\"rows represented per sampled row:\", {g: round(w, 1) for g, w in weights.items()})\n",[32,803,804,816,820,836,844,860,877,887,897,918,930,940,951,961,976,981,987,996,1006,1015,1019,1057,1085,1093,1098,1127,1139],{"__ignoreMap":30},[35,805,806,808,811,813],{"class":37,"line":38},[35,807,265],{"class":77},[35,809,810],{"class":81}," collections ",[35,812,78],{"class":77},[35,814,815],{"class":81}," defaultdict\n",[35,817,818],{"class":37,"line":91},[35,819,95],{"emptyLinePlaceholder":94},[35,821,822,824,827,830,832,834],{"class":37,"line":98},[35,823,282],{"class":77},[35,825,826],{"class":285}," stratified_sample",[35,828,829],{"class":81},"(path, column, per_group, seed",[35,831,104],{"class":77},[35,833,294],{"class":141},[35,835,305],{"class":81},[35,837,838,840,842],{"class":37,"line":120},[35,839,310],{"class":81},[35,841,104],{"class":77},[35,843,315],{"class":81},[35,845,846,848,850,852,854,856,858],{"class":37,"line":148},[35,847,320],{"class":81},[35,849,104],{"class":77},[35,851,325],{"class":81},[35,853,328],{"class":135},[35,855,104],{"class":77},[35,857,333],{"class":141},[35,859,145],{"class":81},[35,861,862,864,866,869,871,873,875],{"class":37,"line":199},[35,863,341],{"class":81},[35,865,104],{"class":77},[35,867,868],{"class":81}," wb.active.iter_rows(",[35,870,361],{"class":135},[35,872,104],{"class":77},[35,874,333],{"class":141},[35,876,145],{"class":81},[35,878,879,881,883,885],{"class":37,"line":338},[35,880,373],{"class":81},[35,882,104],{"class":77},[35,884,378],{"class":141},[35,886,381],{"class":81},[35,888,889,892,894],{"class":37,"line":370},[35,890,891],{"class":81},"    col ",[35,893,104],{"class":77},[35,895,896],{"class":81}," header.index(column)\n",[35,898,899,902,904,907,910,913,916],{"class":37,"line":384},[35,900,901],{"class":81},"    reservoirs, counts ",[35,903,104],{"class":77},[35,905,906],{"class":81}," defaultdict(",[35,908,909],{"class":141},"list",[35,911,912],{"class":81},"), defaultdict(",[35,914,915],{"class":141},"int",[35,917,145],{"class":81},[35,919,920,922,925,927],{"class":37,"line":395},[35,921,398],{"class":77},[35,923,924],{"class":81}," row ",[35,926,404],{"class":77},[35,928,929],{"class":81}," rows:\n",[35,931,932,935,937],{"class":37,"line":423},[35,933,934],{"class":81},"        g ",[35,936,104],{"class":77},[35,938,939],{"class":81}," row[col]\n",[35,941,942,945,948],{"class":37,"line":438},[35,943,944],{"class":81},"        counts[g] ",[35,946,947],{"class":77},"+=",[35,949,950],{"class":141}," 1\n",[35,952,953,956,958],{"class":37,"line":444},[35,954,955],{"class":81},"        res ",[35,957,104],{"class":77},[35,959,960],{"class":81}," reservoirs[g]\n",[35,962,963,965,968,971,973],{"class":37,"line":453},[35,964,426],{"class":77},[35,966,967],{"class":141}," len",[35,969,970],{"class":81},"(res) ",[35,972,476],{"class":77},[35,974,975],{"class":81}," per_group:\n",[35,977,978],{"class":37,"line":467},[35,979,980],{"class":81},"            res.append(row)\n",[35,982,983,985],{"class":37,"line":481},[35,984,447],{"class":77},[35,986,450],{"class":81},[35,988,989,991,993],{"class":37,"line":492},[35,990,456],{"class":81},[35,992,104],{"class":77},[35,994,995],{"class":81}," rnd.randrange(counts[g])\n",[35,997,998,1000,1002,1004],{"class":37,"line":498},[35,999,470],{"class":77},[35,1001,473],{"class":81},[35,1003,476],{"class":77},[35,1005,975],{"class":81},[35,1007,1008,1011,1013],{"class":37,"line":515},[35,1009,1010],{"class":81},"                res[j] ",[35,1012,104],{"class":77},[35,1014,489],{"class":81},[35,1016,1017],{"class":37,"line":520},[35,1018,495],{"class":81},[35,1020,1021,1024,1026,1029,1032,1035,1037,1040,1042,1045,1047,1050,1052,1054],{"class":37,"line":543},[35,1022,1023],{"class":81},"    df ",[35,1025,104],{"class":77},[35,1027,1028],{"class":81}," pd.DataFrame([r ",[35,1030,1031],{"class":77},"for",[35,1033,1034],{"class":81}," res ",[35,1036,404],{"class":77},[35,1038,1039],{"class":81}," reservoirs.values() ",[35,1041,1031],{"class":77},[35,1043,1044],{"class":81}," r ",[35,1046,404],{"class":77},[35,1048,1049],{"class":81}," res], ",[35,1051,507],{"class":135},[35,1053,104],{"class":77},[35,1055,1056],{"class":81},"header)\n",[35,1058,1059,1062,1064,1067,1070,1072,1075,1077,1080,1082],{"class":37,"line":589},[35,1060,1061],{"class":81},"    weights ",[35,1063,104],{"class":77},[35,1065,1066],{"class":81}," {g: counts[g] ",[35,1068,1069],{"class":77},"\u002F",[35,1071,967],{"class":141},[35,1073,1074],{"class":81},"(reservoirs[g]) ",[35,1076,1031],{"class":77},[35,1078,1079],{"class":81}," g ",[35,1081,404],{"class":77},[35,1083,1084],{"class":81}," counts}\n",[35,1086,1088,1090],{"class":37,"line":1087},23,[35,1089,501],{"class":77},[35,1091,1092],{"class":81}," df, weights\n",[35,1094,1096],{"class":37,"line":1095},24,[35,1097,95],{"emptyLinePlaceholder":94},[35,1099,1101,1104,1106,1109,1111,1113,1115,1117,1120,1122,1125],{"class":37,"line":1100},25,[35,1102,1103],{"class":81},"strat, weights ",[35,1105,104],{"class":77},[35,1107,1108],{"class":81}," stratified_sample(",[35,1110,110],{"class":45},[35,1112,132],{"class":81},[35,1114,216],{"class":45},[35,1116,132],{"class":81},[35,1118,1119],{"class":135},"per_group",[35,1121,104],{"class":77},[35,1123,1124],{"class":141},"2_000",[35,1126,145],{"class":81},[35,1128,1130,1132,1135,1137],{"class":37,"line":1129},26,[35,1131,151],{"class":141},[35,1133,1134],{"class":81},"(strat[",[35,1136,216],{"class":45},[35,1138,622],{"class":81},[35,1140,1142,1144,1146,1149,1152,1154,1157,1159,1162,1164,1167,1169],{"class":37,"line":1141},27,[35,1143,151],{"class":141},[35,1145,154],{"class":81},[35,1147,1148],{"class":45},"\"rows represented per sampled row:\"",[35,1150,1151],{"class":81},", {g: ",[35,1153,162],{"class":141},[35,1155,1156],{"class":81},"(w, ",[35,1158,418],{"class":141},[35,1160,1161],{"class":81},") ",[35,1163,1031],{"class":77},[35,1165,1166],{"class":81}," g, w ",[35,1168,404],{"class":77},[35,1170,1171],{"class":81}," weights.items()})\n",[10,1173,1174,1175,1178,1179,1182],{},"Each region contributes 2,000 rows, so even the smallest is well represented. Because the groups are now over- or under-represented relative to the file, figures for the whole population must be ",[231,1176,1177],{},"weighted",": each sampled row stands for ",[32,1180,1181],{},"weight"," rows of its region. The weights returned by the function — the group's size divided by its sample size — are exactly what is needed. Stratifying is worth it whenever the report breaks figures down by a group with very uneven sizes.",[639,1184,648,1189,648,1192,648,1195,648,1198,648,1203,648,1209,648,1215,648,1221,648,1226,648,1230,648,1234,648,1236,648,1242,648,1247,648,1251,648,1254,648,1257],{"viewBox":1185,"role":642,"ariaLabelledBy":1186,"xmlns":646,"style":647},"4 0 752 207",[1187,1188],"rs-kinds-t","rs-kinds-d",[650,1190,1191],{"id":1187},"Simple versus stratified sampling",[654,1193,1194],{"id":1188},"A simple random sample keeps proportions, so small groups contribute few rows, and its plain averages estimate the whole file. A stratified sample takes a fixed number per group, so every group has enough rows, but whole-file figures need each row weighted by its group's size.",[658,1196],{"x":660,"y":661,"width":662,"height":1197,"fill":664},"207",[658,1199],{"x":667,"y":668,"width":1200,"height":1201,"rx":1202,"fill":712,"stroke":703,"style":674},"351.0","175.0","14",[676,1204,1206],{"transform":1205},"translate(182.5,29.0) scale(1.083)",[680,1207],{"d":682,"fill":683,"stroke":719,"style":1208},"stroke-width:1.66px;stroke-linecap:round;stroke-linejoin:round",[687,1210,1214],{"x":1211,"y":1212,"style":1213},"195.5","78.0","font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","simple random (reservoir)",[37,1216],{"x1":1217,"y1":1218,"x2":1219,"y2":1218,"stroke":703,"style":1220},"36.0","91.0","355.0","stroke-width:1px",[687,1222,1225],{"x":1211,"y":1223,"style":1224},"111.0","font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","proportions as in the file",[687,1227,1229],{"x":1211,"y":1228,"style":1224},"137.0","small groups get few rows",[687,1231,1233],{"x":1211,"y":1232,"style":1224},"163.0","plain mean ≈ file mean",[658,1235],{"x":731,"y":668,"width":1200,"height":1201,"rx":1202,"fill":738,"stroke":739,"style":674},[676,1237,1239],{"transform":1238},"translate(551.5,29.0) scale(1.083)",[680,1240],{"d":1241,"fill":683,"stroke":746,"style":1208},"M3 5h18v14H3z M3 10h18 M3 15h18 M9 5v14",[687,1243,1246],{"x":1244,"y":1212,"style":1245},"564.5","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","stratified (reservoir per group)",[37,1248],{"x1":1249,"y1":1218,"x2":1250,"y2":1218,"stroke":739,"style":1220},"405.0","724.0",[687,1252,1253],{"x":1244,"y":1223,"style":1224},"fixed rows per group",[687,1255,1256],{"x":1244,"y":1228,"style":1224},"every group well covered",[687,1258,1259],{"x":1244,"y":1232,"style":1224},"weight rows for file totals",[20,1261,1263],{"id":1262},"step-4-check-the-sample-against-the-full-file","Step 4 — Check the sample against the full file",[10,1265,1266],{},"A sample is only useful if its figures are close to the truth. Compare means and group shares for all three approaches:",[25,1268,1270],{"className":68,"code":1269,"language":70,"meta":30,"style":30},"truth_mean = full[\"Amount\"].mean()\ntruth_share = full[\"Region\"].value_counts(normalize=True)\n\nstrat_w = strat[\"Region\"].map(weights)\nestimates = {\n    \"first 10,000 rows\": head[\"Amount\"].mean(),\n    \"reservoir sample\": sample[\"Amount\"].mean(),\n    \"stratified (weighted)\": (strat[\"Amount\"] * strat_w).sum() \u002F strat_w.sum(),\n    \"stratified (unweighted)\": strat[\"Amount\"].mean(),\n}\nfor name, est in estimates.items():\n    print(f\"{name:24} mean {est:8.2f}  error {est - truth_mean:+7.2f}\")\nprint(\"Islands share — file:\", round(truth_share[\"Islands\"], 3), \"| reservoir:\",\n      round((sample[\"Region\"] == \"Islands\").mean(), 3))\n",[32,1271,1272,1287,1310,1314,1329,1339,1352,1364,1388,1400,1405,1417,1475,1508],{"__ignoreMap":30},[35,1273,1274,1277,1279,1282,1284],{"class":37,"line":38},[35,1275,1276],{"class":81},"truth_mean ",[35,1278,104],{"class":77},[35,1280,1281],{"class":81}," full[",[35,1283,168],{"class":45},[35,1285,1286],{"class":81},"].mean()\n",[35,1288,1289,1292,1294,1296,1298,1301,1304,1306,1308],{"class":37,"line":91},[35,1290,1291],{"class":81},"truth_share ",[35,1293,104],{"class":77},[35,1295,1281],{"class":81},[35,1297,216],{"class":45},[35,1299,1300],{"class":81},"].value_counts(",[35,1302,1303],{"class":135},"normalize",[35,1305,104],{"class":77},[35,1307,333],{"class":141},[35,1309,145],{"class":81},[35,1311,1312],{"class":37,"line":98},[35,1313,95],{"emptyLinePlaceholder":94},[35,1315,1316,1319,1321,1324,1326],{"class":37,"line":120},[35,1317,1318],{"class":81},"strat_w ",[35,1320,104],{"class":77},[35,1322,1323],{"class":81}," strat[",[35,1325,216],{"class":45},[35,1327,1328],{"class":81},"].map(weights)\n",[35,1330,1331,1334,1336],{"class":37,"line":148},[35,1332,1333],{"class":81},"estimates ",[35,1335,104],{"class":77},[35,1337,1338],{"class":81}," {\n",[35,1340,1341,1344,1347,1349],{"class":37,"line":199},[35,1342,1343],{"class":45},"    \"first 10,000 rows\"",[35,1345,1346],{"class":81},": head[",[35,1348,168],{"class":45},[35,1350,1351],{"class":81},"].mean(),\n",[35,1353,1354,1357,1360,1362],{"class":37,"line":338},[35,1355,1356],{"class":45},"    \"reservoir sample\"",[35,1358,1359],{"class":81},": sample[",[35,1361,168],{"class":45},[35,1363,1351],{"class":81},[35,1365,1366,1369,1372,1374,1377,1380,1383,1385],{"class":37,"line":370},[35,1367,1368],{"class":45},"    \"stratified (weighted)\"",[35,1370,1371],{"class":81},": (strat[",[35,1373,168],{"class":45},[35,1375,1376],{"class":81},"] ",[35,1378,1379],{"class":77},"*",[35,1381,1382],{"class":81}," strat_w).sum() ",[35,1384,1069],{"class":77},[35,1386,1387],{"class":81}," strat_w.sum(),\n",[35,1389,1390,1393,1396,1398],{"class":37,"line":384},[35,1391,1392],{"class":45},"    \"stratified (unweighted)\"",[35,1394,1395],{"class":81},": strat[",[35,1397,168],{"class":45},[35,1399,1351],{"class":81},[35,1401,1402],{"class":37,"line":395},[35,1403,1404],{"class":81},"}\n",[35,1406,1407,1409,1412,1414],{"class":37,"line":423},[35,1408,1031],{"class":77},[35,1410,1411],{"class":81}," name, est ",[35,1413,404],{"class":77},[35,1415,1416],{"class":81}," estimates.items():\n",[35,1418,1419,1422,1424,1426,1429,1431,1434,1437,1439,1442,1444,1447,1450,1452,1455,1457,1460,1463,1466,1469,1471,1473],{"class":37,"line":438},[35,1420,1421],{"class":141},"    print",[35,1423,154],{"class":81},[35,1425,550],{"class":77},[35,1427,1428],{"class":45},"\"",[35,1430,557],{"class":556},[35,1432,1433],{"class":81},"name",[35,1435,1436],{"class":77},":24",[35,1438,569],{"class":556},[35,1440,1441],{"class":45}," mean ",[35,1443,557],{"class":556},[35,1445,1446],{"class":81},"est",[35,1448,1449],{"class":77},":8.2f",[35,1451,569],{"class":556},[35,1453,1454],{"class":45},"  error ",[35,1456,557],{"class":556},[35,1458,1459],{"class":81},"est ",[35,1461,1462],{"class":77},"-",[35,1464,1465],{"class":81}," truth_mean",[35,1467,1468],{"class":77},":+7.2f",[35,1470,569],{"class":556},[35,1472,1428],{"class":45},[35,1474,145],{"class":81},[35,1476,1477,1479,1481,1484,1486,1488,1491,1494,1497,1500,1502,1505],{"class":37,"line":444},[35,1478,151],{"class":141},[35,1480,154],{"class":81},[35,1482,1483],{"class":45},"\"Islands share — file:\"",[35,1485,132],{"class":81},[35,1487,162],{"class":141},[35,1489,1490],{"class":81},"(truth_share[",[35,1492,1493],{"class":45},"\"Islands\"",[35,1495,1496],{"class":81},"], ",[35,1498,1499],{"class":141},"3",[35,1501,177],{"class":81},[35,1503,1504],{"class":45},"\"| reservoir:\"",[35,1506,1507],{"class":81},",\n",[35,1509,1510,1513,1516,1518,1520,1523,1526,1529,1531],{"class":37,"line":453},[35,1511,1512],{"class":141},"      round",[35,1514,1515],{"class":81},"((sample[",[35,1517,216],{"class":45},[35,1519,1376],{"class":81},[35,1521,1522],{"class":77},"==",[35,1524,1525],{"class":45}," \"Islands\"",[35,1527,1528],{"class":81},").mean(), ",[35,1530,1499],{"class":141},[35,1532,196],{"class":81},[10,1534,1535,1536,1538],{},"The reservoir sample and the ",[231,1537,1177],{}," stratified sample both estimate the mean to within about 0.3; the first-10,000 slice is off by 7.6, more than twenty times as much. The unweighted stratified mean is the worst of all, nearly 30 too high — it treats the Islands, 5% of the data, as a quarter of it — which is exactly why the weights matter. The Islands' share in the reservoir sample is close to its true share. A check like this belongs in any analysis built on a sample: it costs one comparison on the summary figures you care about, and catches a sample that is not doing its job.",[639,1540,648,1546,648,1549,648,1552,648,1556,648,1561,648,1567,648,1574,648,1579,648,1582,648,1586,648,1589,648,1594,648,1597,648,1601,648,1604,648,1609,648,1612,648,1616,648,1618],{"viewBox":1541,"role":642,"ariaLabelledBy":1542,"xmlns":646,"style":1545},"4 0 744 180",[1543,1544],"rs-err-t","rs-err-d","width:100%;max-width:744px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[650,1547,1548],{"id":1543},"How far each method's mean is from the truth",[654,1550,1551],{"id":1544},"The first 10,000 rows underestimate the mean amount by a large margin because the Islands region is missing. The unweighted stratified sample overestimates it. The reservoir sample and the weighted stratified sample are within a few units of the true mean.",[658,1553],{"x":660,"y":661,"width":1554,"height":1555,"fill":664},"744","180",[658,1557],{"x":667,"y":668,"width":1558,"height":1559,"rx":1560,"fill":672,"stroke":673,"style":1220},"175.6","28.0","7",[687,1562,1566],{"x":1563,"y":1564,"style":1565},"107.8","34.5","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","first 10,000 rows",[658,1568],{"x":1569,"y":668,"width":1570,"height":1559,"rx":1560,"fill":1571,"stroke":1572,"style":1573},"205.6","121.3","#fce9e9","var(--danger,#dc2626)","stroke-width:1.5px",[687,1575,1578],{"x":1576,"y":1564,"style":1577},"334.9","font-size:12px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:start","−7.59",[658,1580],{"x":667,"y":1581,"width":1558,"height":1559,"rx":1560,"fill":672,"stroke":673,"style":1220},"56.0",[687,1583,1585],{"x":1563,"y":1584,"style":1565},"74.5","stratified (unweighted)",[658,1587],{"x":1569,"y":1581,"width":1588,"height":1559,"rx":1560,"fill":765,"stroke":766,"style":1573},"473.4",[687,1590,1593],{"x":1591,"y":1584,"style":1592},"687.0","font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:start","+29.62",[658,1595],{"x":667,"y":1596,"width":1558,"height":1559,"rx":1560,"fill":672,"stroke":673,"style":1220},"96.0",[687,1598,1600],{"x":1563,"y":1599,"style":1565},"114.5","reservoir sample",[658,1602],{"x":1569,"y":1596,"width":1603,"height":1559,"rx":1560,"fill":738,"stroke":739,"style":1573},"6.0",[687,1605,1608],{"x":1606,"y":1599,"style":1607},"219.6","font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:start","−0.34",[658,1610],{"x":667,"y":1611,"width":1558,"height":1559,"rx":1560,"fill":672,"stroke":673,"style":1220},"136.0",[687,1613,1615],{"x":1563,"y":1614,"style":1565},"154.5","stratified (weighted)",[658,1617],{"x":1569,"y":1611,"width":1603,"height":1559,"rx":1560,"fill":738,"stroke":739,"style":1573},[687,1619,1620],{"x":1606,"y":1614,"style":1607},"−0.30",[20,1622,1624],{"id":1623},"step-5-the-pandas-shortcut-when-the-row-count-is-known","Step 5 — The pandas shortcut when the row count is known",[10,1626,1627,1628,1631],{},"If you know how many rows the sheet has — from the sheet's dimension, or a quick row count — pandas can skip the rows not chosen while reading. Choose the row numbers first, then pass a ",[32,1629,1630],{},"skiprows"," function:",[25,1633,1635],{"className":68,"code":1634,"language":70,"meta":30,"style":30},"import numpy as np\n\nn_rows = 200_000                                  # e.g. from count rows without loading the file\nkeep = set(np.random.default_rng(7).choice(np.arange(1, n_rows + 1), size=10_000, replace=False))\nquick = pd.read_excel(\"transactions.xlsx\", skiprows=lambda i: i > 0 and i not in keep)\nprint(len(quick), \"rows | mean\", round(quick[\"Amount\"].mean(), 2))\n",[32,1636,1637,1649,1653,1666,1716,1757],{"__ignoreMap":30},[35,1638,1639,1641,1644,1646],{"class":37,"line":38},[35,1640,78],{"class":77},[35,1642,1643],{"class":81}," numpy ",[35,1645,85],{"class":77},[35,1647,1648],{"class":81}," np\n",[35,1650,1651],{"class":37,"line":91},[35,1652,95],{"emptyLinePlaceholder":94},[35,1654,1655,1658,1660,1663],{"class":37,"line":98},[35,1656,1657],{"class":81},"n_rows ",[35,1659,104],{"class":77},[35,1661,1662],{"class":141}," 200_000",[35,1664,1665],{"class":116},"                                  # e.g. from count rows without loading the file\n",[35,1667,1668,1671,1673,1676,1679,1681,1684,1686,1689,1692,1695,1697,1700,1702,1704,1706,1709,1711,1714],{"class":37,"line":120},[35,1669,1670],{"class":81},"keep ",[35,1672,104],{"class":77},[35,1674,1675],{"class":141}," set",[35,1677,1678],{"class":81},"(np.random.default_rng(",[35,1680,1560],{"class":141},[35,1682,1683],{"class":81},").choice(np.arange(",[35,1685,418],{"class":141},[35,1687,1688],{"class":81},", n_rows ",[35,1690,1691],{"class":77},"+",[35,1693,1694],{"class":141}," 1",[35,1696,177],{"class":81},[35,1698,1699],{"class":135},"size",[35,1701,104],{"class":77},[35,1703,142],{"class":141},[35,1705,132],{"class":81},[35,1707,1708],{"class":135},"replace",[35,1710,104],{"class":77},[35,1712,1713],{"class":141},"False",[35,1715,196],{"class":81},[35,1717,1718,1721,1723,1725,1727,1729,1731,1734,1737,1740,1743,1746,1748,1751,1754],{"class":37,"line":148},[35,1719,1720],{"class":81},"quick ",[35,1722,104],{"class":77},[35,1724,107],{"class":81},[35,1726,110],{"class":45},[35,1728,132],{"class":81},[35,1730,1630],{"class":135},[35,1732,1733],{"class":77},"=lambda",[35,1735,1736],{"class":81}," i: i ",[35,1738,1739],{"class":77},">",[35,1741,1742],{"class":141}," 0",[35,1744,1745],{"class":77}," and",[35,1747,429],{"class":81},[35,1749,1750],{"class":77},"not",[35,1752,1753],{"class":77}," in",[35,1755,1756],{"class":81}," keep)\n",[35,1758,1759,1761,1763,1765,1768,1771,1773,1775,1778,1780,1782,1784],{"class":37,"line":199},[35,1760,151],{"class":141},[35,1762,154],{"class":81},[35,1764,560],{"class":141},[35,1766,1767],{"class":81},"(quick), ",[35,1769,1770],{"class":45},"\"rows | mean\"",[35,1772,132],{"class":81},[35,1774,162],{"class":141},[35,1776,1777],{"class":81},"(quick[",[35,1779,168],{"class":45},[35,1781,171],{"class":81},[35,1783,174],{"class":141},[35,1785,196],{"class":81},[10,1787,1788,1789,1791,1792,18],{},"Row 0 is the header and is always kept; every other row is skipped unless its number was drawn. This is convenient, but note that pandas' openpyxl reader still parses every row of the sheet — ",[32,1790,1630],{}," saves memory, not reading time. Getting the row count cheaply is covered in ",[14,1793,1795],{"href":1794},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fcount-rows-in-an-excel-file-without-loading-it\u002F","Count Rows in an Excel File Without Loading It",[20,1797,1799],{"id":1798},"common-pitfalls","Common pitfalls",[1801,1802,1803,1811,1817,1823,1829],"ul",{},[1804,1805,1806,1810],"li",{},[1807,1808,1809],"strong",{},"Taking the first N rows."," Exports are sorted; the head is rarely representative.",[1804,1812,1813,1816],{},[1807,1814,1815],{},"Unseeded randomness."," Fix the seed so the sample can be reproduced.",[1804,1818,1819,1822],{},[1807,1820,1821],{},"Ignoring weights in a stratified sample."," Whole-file figures need each row weighted by its group's size.",[1804,1824,1825,1828],{},[1807,1826,1827],{},"Samples too small for the groups you report."," Stratify when some groups are small.",[1804,1830,1831,1837],{},[1807,1832,1833,1834,1836],{},"Expecting ",[32,1835,1630],{}," to read faster."," It saves memory; the sheet is still parsed.",[20,1839,1841],{"id":1840},"conclusion","Conclusion",[10,1843,1844,1845,1847],{},"A fair sample of a large Excel sheet needs randomness, not the first rows. Reservoir sampling draws a fixed-size uniform sample in one streaming pass with openpyxl's read-only mode, holding only the sample in memory; a reservoir per group gives a stratified sample that represents small groups, provided whole-file figures are weighted by group size. When the row count is known, pandas' ",[32,1846,1630],{}," with a set of drawn row numbers is a convenient alternative. Fix the seed, and compare the sample's key figures with the full data before relying on them.",[20,1849,1851],{"id":1850},"frequently-asked-questions","Frequently asked questions",[10,1853,1854,1857],{},[1807,1855,1856],{},"How do I take a random sample from a large Excel file in Python?","\nStream the rows with openpyxl in read-only mode and use reservoir sampling: keep the first k rows, then replace a random one with each later row i with probability k divided by i.",[10,1859,1860,1863],{},[1807,1861,1862],{},"Why shouldn't I just use the first rows of the file?","\nExports are usually sorted by date, region or customer, so the first rows are not representative and figures computed from them are biased.",[10,1865,1866,1869],{},[1807,1867,1868],{},"How do I make sure small groups appear in my sample?","\nUse a stratified sample with a separate reservoir for each group, and weight each sampled row by its group size divided by the group's sample size when estimating totals.",[10,1871,1872,1875],{},[1807,1873,1874],{},"Can pandas read only a random subset of rows?","\nYes, if you know the number of rows: draw the row numbers to keep and pass a skiprows function to read_excel that skips all others except the header.",[20,1877,1879],{"id":1878},"related","Related",[1801,1881,1882,1888,1895,1902],{},[1804,1883,1884,1885,1887],{},"Up one level: ",[14,1886,17],{"href":16}," — big files, fast code.",[1804,1889,1890,1894],{},[14,1891,1893],{"href":1892},"\u002Fgetting-started-with-python-excel-automation\u002Freading-excel-files-with-pandas\u002Fread-only-some-rows-of-an-excel-file-with-pandas\u002F","Read Only Some Rows of an Excel File with pandas"," — selecting rows by position.",[1804,1896,1897,1901],{},[14,1898,1900],{"href":1899},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Faggregate-a-huge-excel-file-without-loading-it\u002F","Aggregate a Huge Excel File Without Loading It"," — exact totals instead of estimates.",[1804,1903,1904,1908],{},[14,1905,1907],{"href":1906},"\u002Fautomating-reporting-workflows\u002Ftesting-and-packaging-excel-automation-scripts\u002Fgenerate-realistic-test-data-for-excel-reports-with-faker\u002F","Generate Realistic Test Data for Excel Reports with Faker"," — when real samples cannot be shared.",[1910,1911,1912],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}",{"title":30,"searchDepth":91,"depth":91,"links":1914},[1915,1916,1917,1918,1919,1920,1921,1922,1923,1924],{"id":22,"depth":91,"text":23},{"id":61,"depth":91,"text":62},{"id":225,"depth":91,"text":226},{"id":790,"depth":91,"text":791},{"id":1262,"depth":91,"text":1263},{"id":1623,"depth":91,"text":1624},{"id":1798,"depth":91,"text":1799},{"id":1840,"depth":91,"text":1841},{"id":1850,"depth":91,"text":1851},{"id":1878,"depth":91,"text":1879},"2026-10-02","Pull a fair random sample from an Excel sheet too big to load comfortably: reservoir sampling in one streaming pass with a fixed seed, a stratified sample that keeps every region represented, the pandas skiprows trick when you know the row count, why \"the first 10,000 rows\" is not a sample, and checks that the sample's figures match the full file.","md",[1929,1931,1933,1935],{"q":1856,"a":1930},"Stream the rows with openpyxl in read-only mode and use reservoir sampling: keep the first k rows, then replace a random one with each later row i with probability k divided by i.",{"q":1862,"a":1932},"Exports are usually sorted by date, region or customer, so the first rows are not representative and figures computed from them are biased.",{"q":1868,"a":1934},"Use a stratified sample with a separate reservoir for each group, and weight each sampled row by its group size divided by the group's sample size when estimating totals.",{"q":1874,"a":1936},"Yes, if you know the number of rows: draw the row numbers to keep and pass a skiprows function to read_excel that skips all others except the header.","take a random sample of rows from a large excel file",null,{"breadcrumb":1940},[1941,1943,1946],{"name":1942,"item":1069},"Home",{"name":1944,"item":1945},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":17,"item":16},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Ftake-a-random-sample-of-rows-from-a-large-excel-file",{"title":1949,"description":1950},"Randomly Sample Rows from a Large Excel File in Python","Sample a big xlsx without loading it: reservoir sampling over openpyxl read-only rows, stratified reservoirs per group, pandas skiprows with a known row count, reproducible seeds and checks against the full data.","take-a-random-sample-of-rows-from-a-large-excel-file","advanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Ftake-a-random-sample-of-rows-from-a-large-excel-file\u002Findex",[1954,1955],"large-files","reading-excel","how-to","hYbeFAful0IuLEIaaG2EqW1Gznx0bBMnylAQGNUyIbk",[1959,1963],{"title":1960,"path":1961,"stem":1962,"children":-1},"Stream Excel Rows into a Database with openpyxl","\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fstream-excel-rows-into-a-database-with-openpyxl","advanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fstream-excel-rows-into-a-database-with-openpyxl\u002Findex",{"title":1964,"path":1965,"stem":1966,"children":-1},"Write Large DataFrames to Excel with write_only Mode","\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fwrite-large-dataframes-to-excel-with-write-only-mode","advanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fwrite-large-dataframes-to-excel-with-write-only-mode\u002Findex",[1968,1970],{"path":1969,"title":1944},"\u002Fadvanced-data-transformation-and-cleaning",{"path":1971,"title":17},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python",[],[1974,1977,1980,1983,1986,1989],{"path":1975,"title":1976,"kind":1956},"\u002Fadvanced-data-transformation-and-cleaning\u002Freading-excel-with-polars-and-arrow\u002Fspeed-up-pandas-excel-reads-with-the-calamine-engine","Speed Up pandas Excel Reads with the calamine Engine",{"path":1978,"title":1979,"kind":1956},"\u002Fautomating-reporting-workflows\u002Ftesting-and-packaging-excel-automation-scripts\u002Fprofile-a-slow-excel-report-script","Profile a Slow Excel Report Script in Python",{"path":1981,"title":1982,"kind":1956},"\u002Fformatting-and-charting-excel-reports-with-python\u002Fbuilding-excel-reports-with-xlsxwriter\u002Fwrite-a-million-rows-to-excel-with-xlsxwriter-constant-memory","Write a Million Rows to Excel with xlsxwriter",{"path":1984,"title":1985,"kind":1956},"\u002Fgetting-started-with-python-excel-automation\u002Fchoosing-a-python-excel-library\u002Fbenchmark-python-excel-read-and-write-speed","Benchmark Python Excel Read and Write Speed",{"path":1987,"title":1988,"kind":1956},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-a-large-database-table-to-excel-in-chunks","Export a Large Database Table to Excel in Chunks",{"path":1990,"title":1991,"kind":1956},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fquery-excel-files-with-sql-using-duckdb","Query Excel Files with SQL Using DuckDB",1790972106885]