[{"data":1,"prerenderedAt":2360},["ShallowReactive",2],{"doc:\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fgenerate-large-excel-exports-in-a-background-job":3,"surround:\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fgenerate-large-excel-exports-in-a-background-job":2327,"ancestors:\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fgenerate-large-excel-exports-in-a-background-job":2336,"children:\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fgenerate-large-excel-exports-in-a-background-job":2341,"related:\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fgenerate-large-excel-exports-in-a-background-job":2342},{"id":4,"title":5,"body":6,"dateModified":2294,"datePublished":2294,"description":2295,"extension":2296,"faq":2297,"keyword":2306,"kind":2307,"meta":2308,"navigation":292,"path":2316,"seo":2317,"slug":2320,"stem":2321,"topics":2322,"type":2325,"__hash__":2326},"docs\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fgenerate-large-excel-exports-in-a-background-job\u002Findex.md","Generate Large Excel Exports in a Background Job",{"type":7,"value":8,"toc":2282},"minimark",[9,24,29,64,74,233,237,248,757,780,843,847,850,1517,1524,1528,1531,1757,1764,1884,1888,1891,2124,2132,2136,2156,2160,2201,2205,2217,2221,2227,2233,2239,2245,2249,2278],[10,11,12,13,17,18,23],"p",{},"An export button that works for a month's orders times out for a year's. The browser waits, the reverse proxy gives up after thirty or sixty seconds, the user clicks again, and now two copies of the same heavy query are running. The fix is to stop building the file inside the request. The request starts a ",[14,15,16],"em",{},"job"," and returns immediately with its ID; a worker builds the workbook in the background, streaming rows so memory stays flat; the page polls a status endpoint and shows a download link when the file is ready. This page, part of ",[19,20,22],"a",{"href":21},"\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002F","Serving Excel Files from Python Web Apps",", builds that flow in Flask with a thread pool as the worker, tests it end to end, and shows where Celery or RQ slot in.",[25,26,28],"h2",{"id":27},"prerequisites","Prerequisites",[30,31,36],"pre",{"className":32,"code":33,"language":34,"meta":35,"style":35},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install flask xlsxwriter pandas openpyxl\n","bash","",[37,38,39],"code",{"__ignoreMap":35},[40,41,44,48,52,55,58,61],"span",{"class":42,"line":43},"line",1,[40,45,47],{"class":46},"sMTad","pip",[40,49,51],{"class":50},"srMev"," install",[40,53,54],{"class":50}," flask",[40,56,57],{"class":50}," xlsxwriter",[40,59,60],{"class":50}," pandas",[40,62,63],{"class":50}," openpyxl\n",[10,65,66,69,70,73],{},[37,67,68],{},"shop.db"," is a SQLite database with 60,000 orders standing in for the application's database. The web app is Flask, and the background worker is a ",[37,71,72],{},"ThreadPoolExecutor"," so the whole flow runs in one process for the test; in production the worker is a separate process — Celery, RQ or a cloud task queue — and nothing else in the design changes.",[75,76,84,85,84,89,84,93,84,101,84,111,84,118,84,123,84,128,84,131,84,135,84,140,84,142,84,144,84,149,84,154,84,162,84,168,84,173,84,176,84,181,84,186,84,189,84,192,84,197,84,201,84,206,84,209,84,214,84,218,84,221,84,224,84,229],"svg",{"viewBox":77,"role":78,"ariaLabelledBy":79,"xmlns":82,"style":83},"4 0 752 340","img",[80,81],"bg-seq-t","bg-seq-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[86,87,88],"title",{"id":80},"The export as three short requests and one long job",[90,91,92],"desc",{"id":81},"A sequence diagram. The browser posts an export request and immediately receives 202 Accepted with a job id; the worker builds the workbook in the background; the browser polls the status endpoint until the job is done and then downloads the file from the link.",[94,95],"rect",{"x":96,"y":97,"width":98,"height":99,"fill":100},"4","0","752","340","#ffffff",[94,102],{"x":103,"y":104,"width":105,"height":106,"rx":107,"fill":108,"stroke":109,"style":110},"20.0","16.0","180.0","48.0","12","#f0f4ff","var(--brand,#5b5cf0)","stroke-width:2px",[112,113,117],"text",{"x":114,"y":115,"style":116},"110.0","44.5","font-size:13px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","browser",[94,119],{"x":120,"y":104,"width":105,"height":106,"rx":107,"fill":121,"stroke":122,"style":110},"290.0","#e7ebef","var(--line,#cdd5e6)",[112,124,127],{"x":125,"y":115,"style":126},"380.0","font-size:13px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","web app",[94,129],{"x":130,"y":104,"width":105,"height":106,"rx":107,"fill":121,"stroke":122,"style":110},"560.0",[112,132,134],{"x":133,"y":115,"style":126},"650.0","worker",[42,136],{"x1":114,"y1":137,"x2":114,"y2":138,"stroke":122,"style":139},"64.0","324.0","stroke-width:1.5px;stroke-dasharray:5 4",[42,141],{"x1":125,"y1":137,"x2":125,"y2":138,"stroke":122,"style":139},[42,143],{"x1":133,"y1":137,"x2":133,"y2":138,"stroke":122,"style":139},[42,145],{"x1":146,"y1":147,"x2":148,"y2":147,"stroke":109,"style":110},"115.0","104.0","371.0",[150,151],"polygon",{"points":152,"fill":153},"375.0,104.0 366.0,109.0 366.0,99.0","#5b5cf0",[94,155],{"x":156,"y":157,"width":158,"height":159,"rx":160,"fill":100,"stroke":122,"style":161},"143.7","75.0","202.6","21.0","10","stroke-width:1.2px",[112,163,167],{"x":164,"y":165,"style":166},"245.0","89.5","font-size:11.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","1. POST \u002Fexports → 202 + job id",[42,169],{"x1":170,"y1":171,"x2":172,"y2":171,"stroke":109,"style":110},"385.0","152.0","641.0",[150,174],{"points":175,"fill":153},"645.0,152.0 636.0,157.0 636.0,147.0",[94,177],{"x":178,"y":179,"width":180,"height":159,"rx":160,"fill":100,"stroke":122,"style":161},"462.3","123.0","105.4",[112,182,185],{"x":183,"y":184,"style":166},"515.0","137.5","2. enqueue job",[42,187],{"x1":146,"y1":188,"x2":148,"y2":188,"stroke":109,"style":110},"200.0",[150,190],{"points":191,"fill":153},"375.0,200.0 366.0,205.0 366.0,195.0",[94,193],{"x":194,"y":195,"width":196,"height":159,"rx":160,"fill":100,"stroke":122,"style":161},"132.9","171.0","224.2",[112,198,200],{"x":164,"y":199,"style":166},"185.5","3. GET \u002Fexports\u002Fid → running, 40%",[42,202],{"x1":203,"y1":204,"x2":205,"y2":204,"stroke":109,"style":110},"645.0","248.0","389.0",[150,207],{"points":208,"fill":153},"385.0,248.0 394.0,243.0 394.0,253.0",[94,210],{"x":211,"y":212,"width":213,"height":159,"rx":160,"fill":100,"stroke":122,"style":161},"456.2","219.0","117.7",[112,215,217],{"x":183,"y":216,"style":166},"233.5","4. done: file path",[42,219],{"x1":146,"y1":220,"x2":148,"y2":220,"stroke":109,"style":110},"296.0",[150,222],{"points":223,"fill":153},"375.0,296.0 366.0,301.0 366.0,291.0",[94,225],{"x":226,"y":227,"width":228,"height":159,"rx":160,"fill":100,"stroke":122,"style":161},"148.1","267.0","193.8",[112,230,232],{"x":164,"y":231,"style":166},"281.5","5. GET \u002Fexports\u002Fid\u002Ffile → .xlsx",[25,234,236],{"id":235},"step-1-a-worker-function-that-streams","Step 1 — A worker function that streams",[10,238,239,240,243,244,247],{},"The export function reads the query result in chunks with ",[37,241,242],{},"fetchmany"," and writes each row straight into an xlsxwriter workbook opened in ",[37,245,246],{},"constant_memory"," mode, which flushes every finished row to disk. Neither the result set nor the worksheet is ever held in memory whole:",[30,249,253],{"className":250,"code":251,"language":252,"meta":35,"style":35},"language-python shiki shiki-themes github-light github-dark-high-contrast","import sqlite3\nfrom pathlib import Path\nimport xlsxwriter\n\nEXPORT_DIR = Path(\"exports\")\nEXPORT_DIR.mkdir(exist_ok=True)\n\ndef build_export(job, region=None, chunk=5000):\n    con = sqlite3.connect(\"shop.db\")\n    where, params = (\"WHERE region = ?\", [region]) if region else (\"\", [])\n    total = con.execute(f\"SELECT COUNT(*) FROM orders {where}\", params).fetchone()[0]\n    cur = con.execute(f\"SELECT * FROM orders {where} ORDER BY order_id\", params)\n    path = EXPORT_DIR \u002F f\"orders_{job['id']}.xlsx\"\n    wb = xlsxwriter.Workbook(path, {\"constant_memory\": True})\n    ws = wb.add_worksheet(\"Orders\")\n    money = wb.add_format({\"num_format\": \"#,##0.00\"})\n    ws.write_row(0, 0, [d[0] for d in cur.description], wb.add_format({\"bold\": True}))\n    ws.set_column(4, 4, 12, money)\n    written = 0\n    while rows := cur.fetchmany(chunk):\n        for row in rows:\n            written += 1\n            ws.write_row(written, 0, row)\n        job[\"progress\"] = round(100 * written \u002F total)\n    wb.close()\n    con.close()\n    return path, written\n","python",[37,254,255,265,279,287,294,313,333,338,367,383,417,455,482,518,540,556,577,620,639,650,665,679,691,702,736,742,748],{"__ignoreMap":35},[40,256,257,261],{"class":42,"line":43},[40,258,260],{"class":259},"s-kum","import",[40,262,264],{"class":263},"skGVy"," sqlite3\n",[40,266,268,271,274,276],{"class":42,"line":267},2,[40,269,270],{"class":259},"from",[40,272,273],{"class":263}," pathlib ",[40,275,260],{"class":259},[40,277,278],{"class":263}," Path\n",[40,280,282,284],{"class":42,"line":281},3,[40,283,260],{"class":259},[40,285,286],{"class":263}," xlsxwriter\n",[40,288,290],{"class":42,"line":289},4,[40,291,293],{"emptyLinePlaceholder":292},true,"\n",[40,295,297,301,304,307,310],{"class":42,"line":296},5,[40,298,300],{"class":299},"sP0c6","EXPORT_DIR",[40,302,303],{"class":259}," =",[40,305,306],{"class":263}," Path(",[40,308,309],{"class":50},"\"exports\"",[40,311,312],{"class":263},")\n",[40,314,316,318,321,325,328,331],{"class":42,"line":315},6,[40,317,300],{"class":299},[40,319,320],{"class":263},".mkdir(",[40,322,324],{"class":323},"sa561","exist_ok",[40,326,327],{"class":259},"=",[40,329,330],{"class":299},"True",[40,332,312],{"class":263},[40,334,336],{"class":42,"line":335},7,[40,337,293],{"emptyLinePlaceholder":292},[40,339,341,344,348,351,353,356,359,361,364],{"class":42,"line":340},8,[40,342,343],{"class":259},"def",[40,345,347],{"class":346},"s_Opv"," build_export",[40,349,350],{"class":263},"(job, region",[40,352,327],{"class":259},[40,354,355],{"class":299},"None",[40,357,358],{"class":263},", chunk",[40,360,327],{"class":259},[40,362,363],{"class":299},"5000",[40,365,366],{"class":263},"):\n",[40,368,370,373,375,378,381],{"class":42,"line":369},9,[40,371,372],{"class":263},"    con ",[40,374,327],{"class":259},[40,376,377],{"class":263}," sqlite3.connect(",[40,379,380],{"class":50},"\"shop.db\"",[40,382,312],{"class":263},[40,384,386,389,391,394,397,400,403,406,409,411,414],{"class":42,"line":385},10,[40,387,388],{"class":263},"    where, params ",[40,390,327],{"class":259},[40,392,393],{"class":263}," (",[40,395,396],{"class":50},"\"WHERE region = ?\"",[40,398,399],{"class":263},", [region]) ",[40,401,402],{"class":259},"if",[40,404,405],{"class":263}," region ",[40,407,408],{"class":259},"else",[40,410,393],{"class":263},[40,412,413],{"class":50},"\"\"",[40,415,416],{"class":263},", [])\n",[40,418,420,423,425,428,431,434,438,441,444,447,450,452],{"class":42,"line":419},11,[40,421,422],{"class":263},"    total ",[40,424,327],{"class":259},[40,426,427],{"class":263}," con.execute(",[40,429,430],{"class":259},"f",[40,432,433],{"class":50},"\"SELECT COUNT(*) FROM orders ",[40,435,437],{"class":436},"sSjpA","{",[40,439,440],{"class":263},"where",[40,442,443],{"class":436},"}",[40,445,446],{"class":50},"\"",[40,448,449],{"class":263},", params).fetchone()[",[40,451,97],{"class":299},[40,453,454],{"class":263},"]\n",[40,456,458,461,463,465,467,470,472,474,476,479],{"class":42,"line":457},12,[40,459,460],{"class":263},"    cur ",[40,462,327],{"class":259},[40,464,427],{"class":263},[40,466,430],{"class":259},[40,468,469],{"class":50},"\"SELECT * FROM orders ",[40,471,437],{"class":436},[40,473,440],{"class":263},[40,475,443],{"class":436},[40,477,478],{"class":50}," ORDER BY order_id\"",[40,480,481],{"class":263},", params)\n",[40,483,485,488,490,493,496,499,502,504,507,510,513,515],{"class":42,"line":484},13,[40,486,487],{"class":263},"    path ",[40,489,327],{"class":259},[40,491,492],{"class":299}," EXPORT_DIR",[40,494,495],{"class":259}," \u002F",[40,497,498],{"class":259}," f",[40,500,501],{"class":50},"\"orders_",[40,503,437],{"class":436},[40,505,506],{"class":263},"job[",[40,508,509],{"class":50},"'id'",[40,511,512],{"class":263},"]",[40,514,443],{"class":436},[40,516,517],{"class":50},".xlsx\"\n",[40,519,521,524,526,529,532,535,537],{"class":42,"line":520},14,[40,522,523],{"class":263},"    wb ",[40,525,327],{"class":259},[40,527,528],{"class":263}," xlsxwriter.Workbook(path, {",[40,530,531],{"class":50},"\"constant_memory\"",[40,533,534],{"class":263},": ",[40,536,330],{"class":299},[40,538,539],{"class":263},"})\n",[40,541,543,546,548,551,554],{"class":42,"line":542},15,[40,544,545],{"class":263},"    ws ",[40,547,327],{"class":259},[40,549,550],{"class":263}," wb.add_worksheet(",[40,552,553],{"class":50},"\"Orders\"",[40,555,312],{"class":263},[40,557,559,562,564,567,570,572,575],{"class":42,"line":558},16,[40,560,561],{"class":263},"    money ",[40,563,327],{"class":259},[40,565,566],{"class":263}," wb.add_format({",[40,568,569],{"class":50},"\"num_format\"",[40,571,534],{"class":263},[40,573,574],{"class":50},"\"#,##0.00\"",[40,576,539],{"class":263},[40,578,580,583,585,588,590,593,595,598,601,604,607,610,613,615,617],{"class":42,"line":579},17,[40,581,582],{"class":263},"    ws.write_row(",[40,584,97],{"class":299},[40,586,587],{"class":263},", ",[40,589,97],{"class":299},[40,591,592],{"class":263},", [d[",[40,594,97],{"class":299},[40,596,597],{"class":263},"] ",[40,599,600],{"class":259},"for",[40,602,603],{"class":263}," d ",[40,605,606],{"class":259},"in",[40,608,609],{"class":263}," cur.description], wb.add_format({",[40,611,612],{"class":50},"\"bold\"",[40,614,534],{"class":263},[40,616,330],{"class":299},[40,618,619],{"class":263},"}))\n",[40,621,623,626,628,630,632,634,636],{"class":42,"line":622},18,[40,624,625],{"class":263},"    ws.set_column(",[40,627,96],{"class":299},[40,629,587],{"class":263},[40,631,96],{"class":299},[40,633,587],{"class":263},[40,635,107],{"class":299},[40,637,638],{"class":263},", money)\n",[40,640,642,645,647],{"class":42,"line":641},19,[40,643,644],{"class":263},"    written ",[40,646,327],{"class":259},[40,648,649],{"class":299}," 0\n",[40,651,653,656,659,662],{"class":42,"line":652},20,[40,654,655],{"class":259},"    while",[40,657,658],{"class":263}," rows ",[40,660,661],{"class":259},":=",[40,663,664],{"class":263}," cur.fetchmany(chunk):\n",[40,666,668,671,674,676],{"class":42,"line":667},21,[40,669,670],{"class":259},"        for",[40,672,673],{"class":263}," row ",[40,675,606],{"class":259},[40,677,678],{"class":263}," rows:\n",[40,680,682,685,688],{"class":42,"line":681},22,[40,683,684],{"class":263},"            written ",[40,686,687],{"class":259},"+=",[40,689,690],{"class":299}," 1\n",[40,692,694,697,699],{"class":42,"line":693},23,[40,695,696],{"class":263},"            ws.write_row(written, ",[40,698,97],{"class":299},[40,700,701],{"class":263},", row)\n",[40,703,705,708,711,713,715,718,721,724,727,730,733],{"class":42,"line":704},24,[40,706,707],{"class":263},"        job[",[40,709,710],{"class":50},"\"progress\"",[40,712,597],{"class":263},[40,714,327],{"class":259},[40,716,717],{"class":299}," round",[40,719,720],{"class":263},"(",[40,722,723],{"class":299},"100",[40,725,726],{"class":259}," *",[40,728,729],{"class":263}," written ",[40,731,732],{"class":259},"\u002F",[40,734,735],{"class":263}," total)\n",[40,737,739],{"class":42,"line":738},25,[40,740,741],{"class":263},"    wb.close()\n",[40,743,745],{"class":42,"line":744},26,[40,746,747],{"class":263},"    con.close()\n",[40,749,751,754],{"class":42,"line":750},27,[40,752,753],{"class":259},"    return",[40,755,756],{"class":263}," path, written\n",[10,758,759,760,762,763,766,767,770,771,774,775,779],{},"In ",[37,761,246],{}," mode rows must be written in order, top to bottom — which a streaming export does naturally. The function updates ",[37,764,765],{},"job[\"progress\"]"," after each chunk, which is what the status endpoint reports. Column formats are set with ",[37,768,769],{},"set_column"," before any data is written, because per-cell formats are the expensive part of large sheets. In a test with 200,000 rows on this machine, this streaming version peaked at about 3 MB of Python memory, while reading the same rows into a DataFrame and calling ",[37,772,773],{},"to_excel"," peaked at around 220 MB — and streaming was also faster. ",[19,776,778],{"href":777},"\u002Fformatting-and-charting-excel-reports-with-python\u002Fbuilding-excel-reports-with-xlsxwriter\u002Fwrite-a-million-rows-to-excel-with-xlsxwriter-constant-memory\u002F","Write a Million Rows to Excel with xlsxwriter Constant Memory"," covers the mode in depth.",[75,781,84,787,84,790,84,793,84,797,84,803,84,809,84,816,84,821,84,824,84,828,84,833,84,838],{"viewBox":782,"role":78,"ariaLabelledBy":783,"xmlns":82,"style":786},"4 0 744 126",[784,785],"bg-memory-t","bg-memory-d","width:100%;max-width:744px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[86,788,789],{"id":784},"Peak Python memory for a 200,000-row export",[90,791,792],{"id":785},"Horizontal bars comparing peak Python memory measured with tracemalloc. Streaming rows into xlsxwriter's constant_memory mode peaked at about 3 megabytes; reading into a DataFrame and calling to_excel peaked at about 219 megabytes.",[94,794],{"x":96,"y":97,"width":795,"height":796,"fill":100},"744","126",[94,798],{"x":103,"y":104,"width":799,"height":800,"rx":801,"fill":121,"stroke":122,"style":802},"228.6","28.0","7","stroke-width:1px",[112,804,808],{"x":805,"y":806,"style":807},"134.3","34.5","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","fetchmany + constant_memory",[94,810],{"x":811,"y":104,"width":812,"height":800,"rx":801,"fill":813,"stroke":814,"style":815},"258.6","6.0","#d9f4f1","var(--teal,#0f9488)","stroke-width:1.5px",[112,817,820],{"x":818,"y":806,"style":819},"272.6","font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:start","3 MB",[94,822],{"x":103,"y":823,"width":799,"height":800,"rx":801,"fill":121,"stroke":122,"style":802},"56.0",[112,825,827],{"x":805,"y":826,"style":807},"74.5","read_sql + to_excel",[94,829],{"x":811,"y":823,"width":830,"height":800,"rx":801,"fill":831,"stroke":832,"style":815},"420.8","#fdefd8","var(--gold,#b4740a)",[112,834,837],{"x":835,"y":826,"style":836},"687.4","font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:start","219 MB",[112,839,842],{"x":125,"y":840,"style":841},"106.0","font-size:12.5px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:middle","Measured with tracemalloc on one machine; absolute numbers vary, the gap does not.",[25,844,846],{"id":845},"step-2-the-web-endpoints","Step 2 — The web endpoints",[10,848,849],{},"Three routes: one starts a job, one reports its status, one serves the finished file. The job store is a dictionary here; in production it is Redis or a database table, so that any web process can answer status requests:",[30,851,853],{"className":250,"code":852,"language":252,"meta":35,"style":35},"import uuid\nfrom concurrent.futures import ThreadPoolExecutor\nfrom flask import Flask, abort, jsonify, request, send_file, url_for\n\napp = Flask(__name__)\nexecutor = ThreadPoolExecutor(max_workers=2)\nJOBS = {}\n\ndef run_job(job, region):\n    job[\"status\"] = \"running\"\n    try:\n        path, rows = build_export(job, region)\n        job.update(status=\"done\", path=str(path), rows=rows)\n    except Exception as err:\n        job.update(status=\"failed\", error=str(err))\n\n@app.post(\"\u002Fexports\")\ndef start_export():\n    region = request.json.get(\"region\") if request.is_json else None\n    job = {\"id\": uuid.uuid4().hex, \"status\": \"queued\", \"progress\": 0}\n    JOBS[job[\"id\"]] = job\n    executor.submit(run_job, job, region)\n    return jsonify(id=job[\"id\"], status_url=url_for(\"export_status\", job_id=job[\"id\"])), 202\n\n@app.get(\"\u002Fexports\u002F\u003Cjob_id>\")\ndef export_status(job_id):\n    job = JOBS.get(job_id) or abort(404)\n    body = {k: job[k] for k in (\"id\", \"status\", \"progress\") if k in job}\n    if job[\"status\"] == \"done\":\n        body[\"download_url\"] = url_for(\"export_file\", job_id=job_id)\n        body[\"rows\"] = job[\"rows\"]\n    if job[\"status\"] == \"failed\":\n        body[\"error\"] = job[\"error\"]\n    return jsonify(body)\n\n@app.get(\"\u002Fexports\u002F\u003Cjob_id>\u002Ffile\")\ndef export_file(job_id):\n    job = JOBS.get(job_id)\n    if not job or job[\"status\"] != \"done\":\n        abort(404)\n    return send_file(job[\"path\"], as_attachment=True, download_name=\"orders_export.xlsx\")\n",[37,854,855,862,874,886,890,905,925,935,939,949,964,972,982,1016,1030,1053,1057,1069,1079,1105,1139,1157,1162,1209,1213,1225,1235,1258,1299,1320,1348,1366,1384,1402,1410,1415,1427,1437,1449,1475,1485],{"__ignoreMap":35},[40,856,857,859],{"class":42,"line":43},[40,858,260],{"class":259},[40,860,861],{"class":263}," uuid\n",[40,863,864,866,869,871],{"class":42,"line":267},[40,865,270],{"class":259},[40,867,868],{"class":263}," concurrent.futures ",[40,870,260],{"class":259},[40,872,873],{"class":263}," ThreadPoolExecutor\n",[40,875,876,878,881,883],{"class":42,"line":281},[40,877,270],{"class":259},[40,879,880],{"class":263}," flask ",[40,882,260],{"class":259},[40,884,885],{"class":263}," Flask, abort, jsonify, request, send_file, url_for\n",[40,887,888],{"class":42,"line":289},[40,889,293],{"emptyLinePlaceholder":292},[40,891,892,895,897,900,903],{"class":42,"line":296},[40,893,894],{"class":263},"app ",[40,896,327],{"class":259},[40,898,899],{"class":263}," Flask(",[40,901,902],{"class":299},"__name__",[40,904,312],{"class":263},[40,906,907,910,912,915,918,920,923],{"class":42,"line":315},[40,908,909],{"class":263},"executor ",[40,911,327],{"class":259},[40,913,914],{"class":263}," ThreadPoolExecutor(",[40,916,917],{"class":323},"max_workers",[40,919,327],{"class":259},[40,921,922],{"class":299},"2",[40,924,312],{"class":263},[40,926,927,930,932],{"class":42,"line":335},[40,928,929],{"class":299},"JOBS",[40,931,303],{"class":259},[40,933,934],{"class":263}," {}\n",[40,936,937],{"class":42,"line":340},[40,938,293],{"emptyLinePlaceholder":292},[40,940,941,943,946],{"class":42,"line":369},[40,942,343],{"class":259},[40,944,945],{"class":346}," run_job",[40,947,948],{"class":263},"(job, region):\n",[40,950,951,954,957,959,961],{"class":42,"line":385},[40,952,953],{"class":263},"    job[",[40,955,956],{"class":50},"\"status\"",[40,958,597],{"class":263},[40,960,327],{"class":259},[40,962,963],{"class":50}," \"running\"\n",[40,965,966,969],{"class":42,"line":419},[40,967,968],{"class":259},"    try",[40,970,971],{"class":263},":\n",[40,973,974,977,979],{"class":42,"line":457},[40,975,976],{"class":263},"        path, rows ",[40,978,327],{"class":259},[40,980,981],{"class":263}," build_export(job, region)\n",[40,983,984,987,990,992,995,997,1000,1002,1005,1008,1011,1013],{"class":42,"line":484},[40,985,986],{"class":263},"        job.update(",[40,988,989],{"class":323},"status",[40,991,327],{"class":259},[40,993,994],{"class":50},"\"done\"",[40,996,587],{"class":263},[40,998,999],{"class":323},"path",[40,1001,327],{"class":259},[40,1003,1004],{"class":299},"str",[40,1006,1007],{"class":263},"(path), ",[40,1009,1010],{"class":323},"rows",[40,1012,327],{"class":259},[40,1014,1015],{"class":263},"rows)\n",[40,1017,1018,1021,1024,1027],{"class":42,"line":520},[40,1019,1020],{"class":259},"    except",[40,1022,1023],{"class":299}," Exception",[40,1025,1026],{"class":259}," as",[40,1028,1029],{"class":263}," err:\n",[40,1031,1032,1034,1036,1038,1041,1043,1046,1048,1050],{"class":42,"line":542},[40,1033,986],{"class":263},[40,1035,989],{"class":323},[40,1037,327],{"class":259},[40,1039,1040],{"class":50},"\"failed\"",[40,1042,587],{"class":263},[40,1044,1045],{"class":323},"error",[40,1047,327],{"class":259},[40,1049,1004],{"class":299},[40,1051,1052],{"class":263},"(err))\n",[40,1054,1055],{"class":42,"line":558},[40,1056,293],{"emptyLinePlaceholder":292},[40,1058,1059,1062,1064,1067],{"class":42,"line":579},[40,1060,1061],{"class":346},"@app.post",[40,1063,720],{"class":263},[40,1065,1066],{"class":50},"\"\u002Fexports\"",[40,1068,312],{"class":263},[40,1070,1071,1073,1076],{"class":42,"line":622},[40,1072,343],{"class":259},[40,1074,1075],{"class":346}," start_export",[40,1077,1078],{"class":263},"():\n",[40,1080,1081,1084,1086,1089,1092,1095,1097,1100,1102],{"class":42,"line":641},[40,1082,1083],{"class":263},"    region ",[40,1085,327],{"class":259},[40,1087,1088],{"class":263}," request.json.get(",[40,1090,1091],{"class":50},"\"region\"",[40,1093,1094],{"class":263},") ",[40,1096,402],{"class":259},[40,1098,1099],{"class":263}," request.is_json ",[40,1101,408],{"class":259},[40,1103,1104],{"class":299}," None\n",[40,1106,1107,1110,1112,1115,1118,1121,1123,1125,1128,1130,1132,1134,1136],{"class":42,"line":652},[40,1108,1109],{"class":263},"    job ",[40,1111,327],{"class":259},[40,1113,1114],{"class":263}," {",[40,1116,1117],{"class":50},"\"id\"",[40,1119,1120],{"class":263},": uuid.uuid4().hex, ",[40,1122,956],{"class":50},[40,1124,534],{"class":263},[40,1126,1127],{"class":50},"\"queued\"",[40,1129,587],{"class":263},[40,1131,710],{"class":50},[40,1133,534],{"class":263},[40,1135,97],{"class":299},[40,1137,1138],{"class":263},"}\n",[40,1140,1141,1144,1147,1149,1152,1154],{"class":42,"line":667},[40,1142,1143],{"class":299},"    JOBS",[40,1145,1146],{"class":263},"[job[",[40,1148,1117],{"class":50},[40,1150,1151],{"class":263},"]] ",[40,1153,327],{"class":259},[40,1155,1156],{"class":263}," job\n",[40,1158,1159],{"class":42,"line":681},[40,1160,1161],{"class":263},"    executor.submit(run_job, job, region)\n",[40,1163,1164,1166,1169,1172,1174,1176,1178,1181,1184,1186,1189,1192,1194,1197,1199,1201,1203,1206],{"class":42,"line":693},[40,1165,753],{"class":259},[40,1167,1168],{"class":263}," jsonify(",[40,1170,1171],{"class":323},"id",[40,1173,327],{"class":259},[40,1175,506],{"class":263},[40,1177,1117],{"class":50},[40,1179,1180],{"class":263},"], ",[40,1182,1183],{"class":323},"status_url",[40,1185,327],{"class":259},[40,1187,1188],{"class":263},"url_for(",[40,1190,1191],{"class":50},"\"export_status\"",[40,1193,587],{"class":263},[40,1195,1196],{"class":323},"job_id",[40,1198,327],{"class":259},[40,1200,506],{"class":263},[40,1202,1117],{"class":50},[40,1204,1205],{"class":263},"])), ",[40,1207,1208],{"class":299},"202\n",[40,1210,1211],{"class":42,"line":704},[40,1212,293],{"emptyLinePlaceholder":292},[40,1214,1215,1218,1220,1223],{"class":42,"line":738},[40,1216,1217],{"class":346},"@app.get",[40,1219,720],{"class":263},[40,1221,1222],{"class":50},"\"\u002Fexports\u002F\u003Cjob_id>\"",[40,1224,312],{"class":263},[40,1226,1227,1229,1232],{"class":42,"line":744},[40,1228,343],{"class":259},[40,1230,1231],{"class":346}," export_status",[40,1233,1234],{"class":263},"(job_id):\n",[40,1236,1237,1239,1241,1244,1247,1250,1253,1256],{"class":42,"line":750},[40,1238,1109],{"class":263},[40,1240,327],{"class":259},[40,1242,1243],{"class":299}," JOBS",[40,1245,1246],{"class":263},".get(job_id) ",[40,1248,1249],{"class":259},"or",[40,1251,1252],{"class":263}," abort(",[40,1254,1255],{"class":299},"404",[40,1257,312],{"class":263},[40,1259,1261,1264,1266,1269,1271,1274,1276,1278,1280,1282,1284,1286,1288,1290,1292,1294,1296],{"class":42,"line":1260},28,[40,1262,1263],{"class":263},"    body ",[40,1265,327],{"class":259},[40,1267,1268],{"class":263}," {k: job[k] ",[40,1270,600],{"class":259},[40,1272,1273],{"class":263}," k ",[40,1275,606],{"class":259},[40,1277,393],{"class":263},[40,1279,1117],{"class":50},[40,1281,587],{"class":263},[40,1283,956],{"class":50},[40,1285,587],{"class":263},[40,1287,710],{"class":50},[40,1289,1094],{"class":263},[40,1291,402],{"class":259},[40,1293,1273],{"class":263},[40,1295,606],{"class":259},[40,1297,1298],{"class":263}," job}\n",[40,1300,1302,1305,1308,1310,1312,1315,1318],{"class":42,"line":1301},29,[40,1303,1304],{"class":259},"    if",[40,1306,1307],{"class":263}," job[",[40,1309,956],{"class":50},[40,1311,597],{"class":263},[40,1313,1314],{"class":259},"==",[40,1316,1317],{"class":50}," \"done\"",[40,1319,971],{"class":263},[40,1321,1323,1326,1329,1331,1333,1336,1339,1341,1343,1345],{"class":42,"line":1322},30,[40,1324,1325],{"class":263},"        body[",[40,1327,1328],{"class":50},"\"download_url\"",[40,1330,597],{"class":263},[40,1332,327],{"class":259},[40,1334,1335],{"class":263}," url_for(",[40,1337,1338],{"class":50},"\"export_file\"",[40,1340,587],{"class":263},[40,1342,1196],{"class":323},[40,1344,327],{"class":259},[40,1346,1347],{"class":263},"job_id)\n",[40,1349,1351,1353,1356,1358,1360,1362,1364],{"class":42,"line":1350},31,[40,1352,1325],{"class":263},[40,1354,1355],{"class":50},"\"rows\"",[40,1357,597],{"class":263},[40,1359,327],{"class":259},[40,1361,1307],{"class":263},[40,1363,1355],{"class":50},[40,1365,454],{"class":263},[40,1367,1369,1371,1373,1375,1377,1379,1382],{"class":42,"line":1368},32,[40,1370,1304],{"class":259},[40,1372,1307],{"class":263},[40,1374,956],{"class":50},[40,1376,597],{"class":263},[40,1378,1314],{"class":259},[40,1380,1381],{"class":50}," \"failed\"",[40,1383,971],{"class":263},[40,1385,1387,1389,1392,1394,1396,1398,1400],{"class":42,"line":1386},33,[40,1388,1325],{"class":263},[40,1390,1391],{"class":50},"\"error\"",[40,1393,597],{"class":263},[40,1395,327],{"class":259},[40,1397,1307],{"class":263},[40,1399,1391],{"class":50},[40,1401,454],{"class":263},[40,1403,1405,1407],{"class":42,"line":1404},34,[40,1406,753],{"class":259},[40,1408,1409],{"class":263}," jsonify(body)\n",[40,1411,1413],{"class":42,"line":1412},35,[40,1414,293],{"emptyLinePlaceholder":292},[40,1416,1418,1420,1422,1425],{"class":42,"line":1417},36,[40,1419,1217],{"class":346},[40,1421,720],{"class":263},[40,1423,1424],{"class":50},"\"\u002Fexports\u002F\u003Cjob_id>\u002Ffile\"",[40,1426,312],{"class":263},[40,1428,1430,1432,1435],{"class":42,"line":1429},37,[40,1431,343],{"class":259},[40,1433,1434],{"class":346}," export_file",[40,1436,1234],{"class":263},[40,1438,1440,1442,1444,1446],{"class":42,"line":1439},38,[40,1441,1109],{"class":263},[40,1443,327],{"class":259},[40,1445,1243],{"class":299},[40,1447,1448],{"class":263},".get(job_id)\n",[40,1450,1452,1454,1457,1460,1462,1464,1466,1468,1471,1473],{"class":42,"line":1451},39,[40,1453,1304],{"class":259},[40,1455,1456],{"class":259}," not",[40,1458,1459],{"class":263}," job ",[40,1461,1249],{"class":259},[40,1463,1307],{"class":263},[40,1465,956],{"class":50},[40,1467,597],{"class":263},[40,1469,1470],{"class":259},"!=",[40,1472,1317],{"class":50},[40,1474,971],{"class":263},[40,1476,1478,1481,1483],{"class":42,"line":1477},40,[40,1479,1480],{"class":263},"        abort(",[40,1482,1255],{"class":299},[40,1484,312],{"class":263},[40,1486,1488,1490,1493,1496,1498,1501,1503,1505,1507,1510,1512,1515],{"class":42,"line":1487},41,[40,1489,753],{"class":259},[40,1491,1492],{"class":263}," send_file(job[",[40,1494,1495],{"class":50},"\"path\"",[40,1497,1180],{"class":263},[40,1499,1500],{"class":323},"as_attachment",[40,1502,327],{"class":259},[40,1504,330],{"class":299},[40,1506,587],{"class":263},[40,1508,1509],{"class":323},"download_name",[40,1511,327],{"class":259},[40,1513,1514],{"class":50},"\"orders_export.xlsx\"",[40,1516,312],{"class":263},[10,1518,1519,1520,1523],{},"The start route returns ",[37,1521,1522],{},"202 Accepted"," — \"your request is being processed\" — with the job's status URL, in well under a second whatever the export's size. Failures inside the job are caught and recorded with their message, so the status endpoint can tell the user the export failed instead of leaving them polling for ever. The file route refuses to serve anything that is not finished.",[25,1525,1527],{"id":1526},"step-3-test-the-whole-flow","Step 3 — Test the whole flow",[10,1529,1530],{},"Flask's test client drives the three routes exactly as a browser would — start, poll, download — and the downloaded bytes are opened as a workbook:",[30,1532,1534],{"className":250,"code":1533,"language":252,"meta":35,"style":35},"import io\nimport time\nimport pandas as pd\n\nclient = app.test_client()\nstarted = client.post(\"\u002Fexports\", json={\"region\": \"West\"})\nprint(started.status_code, started.json[\"status_url\"])\n\nwhile True:\n    status = client.get(started.json[\"status_url\"]).json\n    if status[\"status\"] in (\"done\", \"failed\"):\n        break\n    time.sleep(0.2)\nprint(status)\n\nfile = client.get(status[\"download_url\"])\nexport = pd.read_excel(io.BytesIO(file.data))\nfile.close()\nprint(file.headers[\"Content-Disposition\"], export.shape, export[\"region\"].unique().tolist())\n",[37,1535,1536,1543,1550,1563,1567,1577,1607,1621,1625,1635,1650,1673,1678,1688,1695,1699,1713,1728,1735],{"__ignoreMap":35},[40,1537,1538,1540],{"class":42,"line":43},[40,1539,260],{"class":259},[40,1541,1542],{"class":263}," io\n",[40,1544,1545,1547],{"class":42,"line":267},[40,1546,260],{"class":259},[40,1548,1549],{"class":263}," time\n",[40,1551,1552,1554,1557,1560],{"class":42,"line":281},[40,1553,260],{"class":259},[40,1555,1556],{"class":263}," pandas ",[40,1558,1559],{"class":259},"as",[40,1561,1562],{"class":263}," pd\n",[40,1564,1565],{"class":42,"line":289},[40,1566,293],{"emptyLinePlaceholder":292},[40,1568,1569,1572,1574],{"class":42,"line":296},[40,1570,1571],{"class":263},"client ",[40,1573,327],{"class":259},[40,1575,1576],{"class":263}," app.test_client()\n",[40,1578,1579,1582,1584,1587,1589,1591,1594,1596,1598,1600,1602,1605],{"class":42,"line":315},[40,1580,1581],{"class":263},"started ",[40,1583,327],{"class":259},[40,1585,1586],{"class":263}," client.post(",[40,1588,1066],{"class":50},[40,1590,587],{"class":263},[40,1592,1593],{"class":323},"json",[40,1595,327],{"class":259},[40,1597,437],{"class":263},[40,1599,1091],{"class":50},[40,1601,534],{"class":263},[40,1603,1604],{"class":50},"\"West\"",[40,1606,539],{"class":263},[40,1608,1609,1612,1615,1618],{"class":42,"line":335},[40,1610,1611],{"class":299},"print",[40,1613,1614],{"class":263},"(started.status_code, started.json[",[40,1616,1617],{"class":50},"\"status_url\"",[40,1619,1620],{"class":263},"])\n",[40,1622,1623],{"class":42,"line":340},[40,1624,293],{"emptyLinePlaceholder":292},[40,1626,1627,1630,1633],{"class":42,"line":369},[40,1628,1629],{"class":259},"while",[40,1631,1632],{"class":299}," True",[40,1634,971],{"class":263},[40,1636,1637,1640,1642,1645,1647],{"class":42,"line":385},[40,1638,1639],{"class":263},"    status ",[40,1641,327],{"class":259},[40,1643,1644],{"class":263}," client.get(started.json[",[40,1646,1617],{"class":50},[40,1648,1649],{"class":263},"]).json\n",[40,1651,1652,1654,1657,1659,1661,1663,1665,1667,1669,1671],{"class":42,"line":419},[40,1653,1304],{"class":259},[40,1655,1656],{"class":263}," status[",[40,1658,956],{"class":50},[40,1660,597],{"class":263},[40,1662,606],{"class":259},[40,1664,393],{"class":263},[40,1666,994],{"class":50},[40,1668,587],{"class":263},[40,1670,1040],{"class":50},[40,1672,366],{"class":263},[40,1674,1675],{"class":42,"line":457},[40,1676,1677],{"class":259},"        break\n",[40,1679,1680,1683,1686],{"class":42,"line":484},[40,1681,1682],{"class":263},"    time.sleep(",[40,1684,1685],{"class":299},"0.2",[40,1687,312],{"class":263},[40,1689,1690,1692],{"class":42,"line":520},[40,1691,1611],{"class":299},[40,1693,1694],{"class":263},"(status)\n",[40,1696,1697],{"class":42,"line":542},[40,1698,293],{"emptyLinePlaceholder":292},[40,1700,1701,1704,1706,1709,1711],{"class":42,"line":558},[40,1702,1703],{"class":323},"file",[40,1705,303],{"class":259},[40,1707,1708],{"class":263}," client.get(status[",[40,1710,1328],{"class":50},[40,1712,1620],{"class":263},[40,1714,1715,1718,1720,1723,1725],{"class":42,"line":579},[40,1716,1717],{"class":263},"export ",[40,1719,327],{"class":259},[40,1721,1722],{"class":263}," pd.read_excel(io.BytesIO(",[40,1724,1703],{"class":323},[40,1726,1727],{"class":263},".data))\n",[40,1729,1730,1732],{"class":42,"line":622},[40,1731,1703],{"class":323},[40,1733,1734],{"class":263},".close()\n",[40,1736,1737,1739,1741,1743,1746,1749,1752,1754],{"class":42,"line":641},[40,1738,1611],{"class":299},[40,1740,720],{"class":263},[40,1742,1703],{"class":323},[40,1744,1745],{"class":263},".headers[",[40,1747,1748],{"class":50},"\"Content-Disposition\"",[40,1750,1751],{"class":263},"], export.shape, export[",[40,1753,1091],{"class":50},[40,1755,1756],{"class":263},"].unique().tolist())\n",[10,1758,1759,1760,1763],{},"The request returns 202 immediately, the status moves from queued or running to done with the row count, and the downloaded workbook contains only the West orders — about a quarter of the 60,000. In the browser, a few lines of JavaScript do the polling and show a progress bar from the ",[37,1761,1762],{},"progress"," field; in Dash or Streamlit, an interval component does the same.",[75,1765,84,1770,84,1773,84,1776,84,1779,84,1786,84,1790,84,1800,84,1803,84,1808,84,1811,84,1815,84,1819,84,1826,84,1832,84,1836,84,1838,84,1840,84,1843,84,1850,84,1855,84,1858,84,1860,84,1862,84,1865,84,1872,84,1877,84,1880,84,1882],{"viewBox":1766,"role":78,"ariaLabelledBy":1767,"xmlns":82,"style":83},"4 0 752 279",[1768,1769],"bg-states-t","bg-states-d",[86,1771,1772],{"id":1768},"A job's life",[90,1774,1775],{"id":1769},"A timeline of one export job. It is queued when the request arrives, running while rows are streamed with progress updates, done when the workbook is closed, and its file is deleted by the clean-up job a day later. A failure moves it to failed with an error message instead.",[94,1777],{"x":96,"y":97,"width":98,"height":1778,"fill":100},"279",[42,1780],{"x1":1781,"y1":1782,"x2":1783,"y2":1782,"stroke":1784,"style":1785},"94.0","139.5","666.0","var(--muted,#5b6780)","stroke-width:2.5px",[94,1787],{"x":103,"y":104,"width":1788,"height":1789,"rx":107,"fill":121,"stroke":122,"style":110},"172.0","91.5",[1791,1792,1794],"g",{"transform":1793},"translate(94.0,29.0) scale(1.000)",[999,1795],{"d":1796,"fill":1797,"stroke":1798,"style":1799},"M12 3a9 9 0 1 0 .01 0z M12 7v5l3.5 2","none","var(--text,#172033)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[112,1801,1802],{"x":840,"y":826,"style":807},"queued",[112,1804,1807],{"x":840,"y":1805,"style":1806},"93.0","font-size:11px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","202 + job id",[42,1809],{"x1":840,"y1":1810,"x2":840,"y2":1782,"stroke":1784,"style":815},"107.5",[1812,1813],"circle",{"cx":840,"cy":1782,"r":1814,"fill":1784,"stroke":100,"style":110},"7.0",[94,1816],{"x":1817,"y":1818,"width":1788,"height":1789,"rx":107,"fill":108,"stroke":109,"style":110},"202.7","171.5",[1791,1820,1822],{"transform":1821},"translate(276.7,184.5) scale(1.000)",[999,1823],{"d":1824,"fill":1797,"stroke":1825,"style":1799},"M20 11a8 8 0 1 0-2.3 5.7 M20 4v7h-7","var(--brand-strong,#4338ca)",[112,1827,1831],{"x":1828,"y":1829,"style":1830},"288.7","230.0","font-size:12.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","running",[112,1833,1835],{"x":1828,"y":1834,"style":1806},"248.5","progress 0–100%",[42,1837],{"x1":1828,"y1":1782,"x2":1828,"y2":1818,"stroke":1784,"style":815},[1812,1839],{"cx":1828,"cy":1782,"r":1814,"fill":109,"stroke":100,"style":110},[94,1841],{"x":1842,"y":104,"width":1788,"height":1789,"rx":107,"fill":813,"stroke":814,"style":110},"385.3",[1791,1844,1846],{"transform":1845},"translate(459.3,29.0) scale(1.000)",[999,1847],{"d":1848,"fill":1797,"stroke":1849,"style":1799},"M5 12.5l4.5 4.5L19 7","var(--teal-ink,#0b6157)",[112,1851,1854],{"x":1852,"y":826,"style":1853},"471.3","font-size:12.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","done",[112,1856,1857],{"x":1852,"y":1805,"style":1806},"download_url",[42,1859],{"x1":1852,"y1":1810,"x2":1852,"y2":1782,"stroke":1784,"style":815},[1812,1861],{"cx":1852,"cy":1782,"r":1814,"fill":814,"stroke":100,"style":110},[94,1863],{"x":1864,"y":1818,"width":1788,"height":1789,"rx":107,"fill":831,"stroke":832,"style":110},"568.0",[1791,1866,1868],{"transform":1867},"translate(642.0,184.5) scale(1.000)",[999,1869],{"d":1870,"fill":1797,"stroke":1871,"style":1799},"M4 7h16 M10 11v6 M14 11v6 M6 7l1 13h10l1-13 M9 7V4h6v3","var(--gold-ink,#7a4e06)",[112,1873,1876],{"x":1874,"y":1829,"style":1875},"654.0","font-size:12.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","expired",[112,1878,1879],{"x":1874,"y":1834,"style":1806},"file deleted after 24 h",[42,1881],{"x1":1874,"y1":1782,"x2":1874,"y2":1818,"stroke":1784,"style":815},[1812,1883],{"cx":1874,"cy":1782,"r":1814,"fill":832,"stroke":100,"style":110},[25,1885,1887],{"id":1886},"step-4-clean-up-old-files","Step 4 — Clean up old files",[10,1889,1890],{},"Every export leaves a file behind. Delete finished exports after a day, from a scheduled task, and forget their jobs:",[30,1892,1894],{"className":250,"code":1893,"language":252,"meta":35,"style":35},"from datetime import datetime, timedelta\n\ndef clean_exports(max_age=timedelta(hours=24), now=None):\n    now = now or datetime.now()\n    removed = 0\n    for job_id, job in list(JOBS.items()):\n        path = Path(job.get(\"path\", \"\"))\n        if job[\"status\"] == \"done\" and path.exists():\n            if now - datetime.fromtimestamp(path.stat().st_mtime) > max_age:\n                path.unlink()\n                del JOBS[job_id]\n                removed += 1\n    return removed\n\nprint(clean_exports(), \"removed now;\", clean_exports(now=datetime.now() + timedelta(days=2)), \"removed two days later\")\n",[37,1895,1896,1908,1912,1944,1959,1968,1988,2007,2028,2047,2052,2062,2071,2078,2082],{"__ignoreMap":35},[40,1897,1898,1900,1903,1905],{"class":42,"line":43},[40,1899,270],{"class":259},[40,1901,1902],{"class":263}," datetime ",[40,1904,260],{"class":259},[40,1906,1907],{"class":263}," datetime, timedelta\n",[40,1909,1910],{"class":42,"line":267},[40,1911,293],{"emptyLinePlaceholder":292},[40,1913,1914,1916,1919,1922,1924,1927,1930,1932,1935,1938,1940,1942],{"class":42,"line":281},[40,1915,343],{"class":259},[40,1917,1918],{"class":346}," clean_exports",[40,1920,1921],{"class":263},"(max_age",[40,1923,327],{"class":259},[40,1925,1926],{"class":263},"timedelta(",[40,1928,1929],{"class":323},"hours",[40,1931,327],{"class":259},[40,1933,1934],{"class":299},"24",[40,1936,1937],{"class":263},"), now",[40,1939,327],{"class":259},[40,1941,355],{"class":299},[40,1943,366],{"class":263},[40,1945,1946,1949,1951,1954,1956],{"class":42,"line":289},[40,1947,1948],{"class":263},"    now ",[40,1950,327],{"class":259},[40,1952,1953],{"class":263}," now ",[40,1955,1249],{"class":259},[40,1957,1958],{"class":263}," datetime.now()\n",[40,1960,1961,1964,1966],{"class":42,"line":296},[40,1962,1963],{"class":263},"    removed ",[40,1965,327],{"class":259},[40,1967,649],{"class":299},[40,1969,1970,1973,1976,1978,1981,1983,1985],{"class":42,"line":315},[40,1971,1972],{"class":259},"    for",[40,1974,1975],{"class":263}," job_id, job ",[40,1977,606],{"class":259},[40,1979,1980],{"class":299}," list",[40,1982,720],{"class":263},[40,1984,929],{"class":299},[40,1986,1987],{"class":263},".items()):\n",[40,1989,1990,1993,1995,1998,2000,2002,2004],{"class":42,"line":335},[40,1991,1992],{"class":263},"        path ",[40,1994,327],{"class":259},[40,1996,1997],{"class":263}," Path(job.get(",[40,1999,1495],{"class":50},[40,2001,587],{"class":263},[40,2003,413],{"class":50},[40,2005,2006],{"class":263},"))\n",[40,2008,2009,2012,2014,2016,2018,2020,2022,2025],{"class":42,"line":340},[40,2010,2011],{"class":259},"        if",[40,2013,1307],{"class":263},[40,2015,956],{"class":50},[40,2017,597],{"class":263},[40,2019,1314],{"class":259},[40,2021,1317],{"class":50},[40,2023,2024],{"class":259}," and",[40,2026,2027],{"class":263}," path.exists():\n",[40,2029,2030,2033,2035,2038,2041,2044],{"class":42,"line":369},[40,2031,2032],{"class":259},"            if",[40,2034,1953],{"class":263},[40,2036,2037],{"class":259},"-",[40,2039,2040],{"class":263}," datetime.fromtimestamp(path.stat().st_mtime) ",[40,2042,2043],{"class":259},">",[40,2045,2046],{"class":263}," max_age:\n",[40,2048,2049],{"class":42,"line":385},[40,2050,2051],{"class":263},"                path.unlink()\n",[40,2053,2054,2057,2059],{"class":42,"line":419},[40,2055,2056],{"class":259},"                del",[40,2058,1243],{"class":299},[40,2060,2061],{"class":263},"[job_id]\n",[40,2063,2064,2067,2069],{"class":42,"line":457},[40,2065,2066],{"class":263},"                removed ",[40,2068,687],{"class":259},[40,2070,690],{"class":299},[40,2072,2073,2075],{"class":42,"line":484},[40,2074,753],{"class":259},[40,2076,2077],{"class":263}," removed\n",[40,2079,2080],{"class":42,"line":520},[40,2081,293],{"emptyLinePlaceholder":292},[40,2083,2084,2086,2089,2092,2095,2098,2100,2103,2106,2109,2112,2114,2116,2119,2122],{"class":42,"line":542},[40,2085,1611],{"class":299},[40,2087,2088],{"class":263},"(clean_exports(), ",[40,2090,2091],{"class":50},"\"removed now;\"",[40,2093,2094],{"class":263},", clean_exports(",[40,2096,2097],{"class":323},"now",[40,2099,327],{"class":259},[40,2101,2102],{"class":263},"datetime.now() ",[40,2104,2105],{"class":259},"+",[40,2107,2108],{"class":263}," timedelta(",[40,2110,2111],{"class":323},"days",[40,2113,327],{"class":259},[40,2115,922],{"class":299},[40,2117,2118],{"class":263},")), ",[40,2120,2121],{"class":50},"\"removed two days later\"",[40,2123,312],{"class":263},[10,2125,2126,2127,2131],{},"Two things follow from files expiring: the status endpoint should answer 404 or \"expired\" for an old job, and the email or page that shows the link should say how long it is valid. When files are stored in object storage rather than on the web server's disk, a lifecycle rule on the bucket does the clean-up, and a signed URL replaces the file route — see ",[19,2128,2130],{"href":2129},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fupload-an-excel-report-to-amazon-s3-with-boto3\u002F","Upload an Excel Report to Amazon S3 with boto3",".",[25,2133,2135],{"id":2134},"moving-to-a-real-queue","Moving to a real queue",[10,2137,2138,2139,2142,2143,2146,2147,2150,2151,2155],{},"The thread pool keeps jobs inside the web process, so they are lost if it restarts and cannot run on another machine. In production, ",[37,2140,2141],{},"executor.submit(run_job, ...)"," becomes a task queue call — ",[37,2144,2145],{},"build_export_task.delay(job_id, region)"," with Celery, or ",[37,2148,2149],{},"queue.enqueue(...)"," with RQ — the job store moves to Redis or the database, and exports are written to shared or object storage so any web process can serve them. ",[19,2152,2154],{"href":2153},"\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Fschedule-excel-reports-with-celery-beat\u002F","Schedule Excel Reports with Celery Beat"," sets up the Celery side, and the same worker can run the nightly clean-up.",[25,2157,2159],{"id":2158},"common-pitfalls","Common pitfalls",[2161,2162,2163,2171,2183,2189,2195],"ul",{},[2164,2165,2166,2170],"li",{},[2167,2168,2169],"strong",{},"Building large files inside the request."," Proxies time out long before big exports finish; return 202 and a job ID.",[2164,2172,2173,2176,2177,2179,2180,2182],{},[2167,2174,2175],{},"Loading everything into a DataFrame."," Stream rows with ",[37,2178,242],{}," into ",[37,2181,246],{}," mode.",[2164,2184,2185,2188],{},[2167,2186,2187],{},"Jobs that fail silently."," Catch exceptions in the job and record them for the status endpoint.",[2164,2190,2191,2194],{},[2167,2192,2193],{},"A job store in one process."," Status requests may hit another process; keep job state in Redis or the database.",[2164,2196,2197,2200],{},[2167,2198,2199],{},"Files kept for ever."," Expire them and say so next to the link.",[25,2202,2204],{"id":2203},"conclusion","Conclusion",[10,2206,2207,2208,2210,2211,2213,2214,2216],{},"Answer the export request with ",[37,2209,1522],{}," and a job ID, and build the workbook in a background worker that streams database rows with ",[37,2212,242],{}," into xlsxwriter's ",[37,2215,246],{}," mode, updating progress as it goes. Expose a status endpoint with progress, errors and a download link, serve only finished files, test the start-poll-download cycle with Flask's test client, expire old files on a schedule, and move the worker and job store to Celery or RQ and Redis in production.",[25,2218,2220],{"id":2219},"frequently-asked-questions","Frequently asked questions",[10,2222,2223,2226],{},[2167,2224,2225],{},"How do I stop an Excel export from timing out in Flask or Django?","\nDo not build the file inside the request. Return 202 with a job ID, build the workbook in a background worker, and let the page poll a status endpoint until a download link is available.",[10,2228,2229,2232],{},[2167,2230,2231],{},"How do I write a very large Excel file without running out of memory?","\nRead rows from the database in chunks with fetchmany and write them one by one to an xlsxwriter workbook opened with {\"constant_memory\": True}, which flushes each finished row to disk.",[10,2234,2235,2238],{},[2167,2236,2237],{},"How do I show progress for a background export?","\nCount the rows first, update a progress value in the job's state after each chunk is written, and return it from the status endpoint for the page to display.",[10,2240,2241,2244],{},[2167,2242,2243],{},"What should do the background work in production?","\nA task queue such as Celery or RQ with workers in separate processes, a shared job store such as Redis or a database table, and shared or object storage for the finished files.",[25,2246,2248],{"id":2247},"related","Related",[2161,2250,2251,2257,2264,2271],{},[2164,2252,2253,2254,2256],{},"Up one level: ",[19,2255,22],{"href":21}," — Excel in and out of web applications.",[2164,2258,2259,2263],{},[19,2260,2262],{"href":2261},"\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fstream-an-excel-file-from-a-fastapi-endpoint\u002F","Stream an Excel File from a FastAPI Endpoint"," — when the file is small enough to send directly.",[2164,2265,2266,2270],{},[19,2267,2269],{"href":2268},"\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fdownload-an-excel-file-from-a-dash-app\u002F","Download an Excel File from a Dash App"," — the export button side.",[2164,2272,2273,2277],{},[19,2274,2276],{"href":2275},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fstream-excel-rows-into-a-database-with-openpyxl\u002F","Stream Excel Rows into a Database with openpyxl"," — streaming in the other direction.",[2279,2280,2281],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}",{"title":35,"searchDepth":267,"depth":267,"links":2283},[2284,2285,2286,2287,2288,2289,2290,2291,2292,2293],{"id":27,"depth":267,"text":28},{"id":235,"depth":267,"text":236},{"id":845,"depth":267,"text":846},{"id":1526,"depth":267,"text":1527},{"id":1886,"depth":267,"text":1887},{"id":2134,"depth":267,"text":2135},{"id":2158,"depth":267,"text":2159},{"id":2203,"depth":267,"text":2204},{"id":2219,"depth":267,"text":2220},{"id":2247,"depth":267,"text":2248},"2026-09-29","Stop big \"Export to Excel\" requests from timing out: accept the request and return a job ID at once, build the workbook in a background worker that streams rows from the database into xlsxwriter's constant-memory mode, report progress through a status endpoint, serve the finished file by link, and expire old files.","md",[2298,2300,2302,2304],{"q":2225,"a":2299},"Do not build the file inside the request. Return 202 with a job ID, build the workbook in a background worker, and let the page poll a status endpoint until a download link is available.",{"q":2231,"a":2301},"Read rows from the database in chunks with fetchmany and write them one by one to an xlsxwriter workbook opened with {\"constant_memory\": True}, which flushes each finished row to disk.",{"q":2237,"a":2303},"Count the rows first, update a progress value in the job's state after each chunk is written, and return it from the status endpoint for the page to display.",{"q":2243,"a":2305},"A task queue such as Celery or RQ with workers in separate processes, a shared job store such as Redis or a database table, and shared or object storage for the finished files.","generate large excel exports in a background job",null,{"breadcrumb":2309},[2310,2312,2315],{"name":2311,"item":732},"Home",{"name":2313,"item":2314},"Automating Reporting Workflows with Python and Excel","\u002Fautomating-reporting-workflows\u002F",{"name":22,"item":21},"\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fgenerate-large-excel-exports-in-a-background-job",{"title":2318,"description":2319},"Large Excel Exports in a Background Job (Flask, Celery)","Build large Excel exports asynchronously: 202 Accepted with a job id, a worker that streams database rows with fetchmany into xlsxwriter constant_memory, status polling with progress, download links, failure handling and clean-up, tested with the Flask test client.","generate-large-excel-exports-in-a-background-job","automating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fgenerate-large-excel-exports-in-a-background-job\u002Findex",[2323,2324],"web-apps","large-files","how-to","aL-b6zMml9B5PzaL6GpM-jwz-5fGr2SGNPk79NUxBHE",[2328,2332],{"title":2329,"path":2330,"stem":2331,"children":-1},"Export a Django Queryset to Excel","\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fexport-a-django-queryset-to-excel","automating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fexport-a-django-queryset-to-excel\u002Findex",{"title":2333,"path":2334,"stem":2335,"children":-1},"Handle an Uploaded Excel File in Flask and FastAPI","\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fhandle-an-uploaded-excel-file-in-flask-and-fastapi","automating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fhandle-an-uploaded-excel-file-in-flask-and-fastapi\u002Findex",[2337,2339],{"path":2338,"title":2313},"\u002Fautomating-reporting-workflows",{"path":2340,"title":22},"\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps",[],[2343,2345,2348,2351,2354,2357],{"path":2344,"title":2154,"kind":2325},"\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Fschedule-excel-reports-with-celery-beat",{"path":2346,"title":2347,"kind":2325},"\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Ftrigger-an-excel-report-from-a-webhook","Trigger an Excel Report from a Webhook",{"path":2349,"title":2350,"kind":2325},"\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fadd-an-export-to-excel-action-in-django-admin","Add an Export to Excel Action in Django Admin",{"path":2352,"title":2353,"kind":2325},"\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fbuild-a-gradio-app-that-cleans-an-uploaded-excel-file","Build a Gradio App That Cleans an Uploaded Excel File",{"path":2355,"title":2356,"kind":2325},"\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fbuild-a-streamlit-app-to-upload-and-download-excel","Build a Streamlit App to Upload and Download Excel Files",{"path":2358,"title":2359,"kind":2325},"\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fbuild-an-excel-workbook-in-memory-with-bytesio","Build an Excel Workbook in Memory with BytesIO",1790972140441]