[{"data":1,"prerenderedAt":2258},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Frefresh-an-excel-report-from-a-database-on-a-schedule":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Frefresh-an-excel-report-from-a-database-on-a-schedule":2249},{"id":4,"title":5,"body":6,"dateModified":2219,"datePublished":2219,"description":2220,"extension":2221,"faq":2222,"meta":2231,"navigation":285,"path":2241,"seo":2242,"slug":2245,"stem":2246,"type":2247,"__hash__":2248},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Frefresh-an-excel-report-from-a-database-on-a-schedule\u002Findex.md","Refresh an Excel Report from a Database on a Schedule",{"type":7,"value":8,"toc":2206},"minimark",[9,13,22,194,199,230,238,242,249,582,594,597,707,711,714,830,933,938,945,1157,1165,1169,1172,1464,1470,1474,1477,1700,1707,1802,1806,1809,1977,1985,1989,2111,2115,2118,2121,2125,2128,2132,2139,2145,2151,2157,2161,2164,2173,2176,2202],[10,11,12],"p",{},"An extract is only useful while it is current, and the failure mode of a scheduled refresh is not a crash — it is silence. The job stops being triggered, or fails at 06:00 into a log nobody reads, and the workbook on the shared drive keeps being opened and quoted for another three weeks. Nobody notices, because a stale file looks exactly like a fresh one.",[10,14,15,16,21],{},"This guide covers the four things that make a scheduled refresh trustworthy: querying incrementally so the run stays inside its window, writing the file atomically so nobody can open it half-written, stamping it so readers can see how old it is, and checking its age so a refresh that stops running gets noticed. It is part of ",[17,18,20],"a",{"href":19},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002F","Moving Data Between Excel and Databases",".",[23,24,32,33,32,37,32,41,32,49,32,59,32,66,32,71,32,76,32,81,32,84,32,88,32,92,32,95,32,100,32,105,32,108,32,114,32,119,32,123,32,130,32,135,32,139,32,142,32,146,32,149,32,153,32,157,32,164,32,168,32,174,32,183,32,189],"svg",{"viewBox":25,"role":26,"ariaLabelledBy":27,"xmlns":30,"style":31},"0 44 740 206","img",[28,29],"rf-cycle-t","rf-cycle-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:740px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[34,35,36],"title",{"id":28},"One refresh cycle, from watermark to published file",[38,39,40],"desc",{"id":29},"The job reads the stored watermark, queries only rows changed since then, merges them into the cached extract, writes a temporary workbook, and replaces the published file atomically. It then records the new watermark and the generation time, and a separate monitor alerts when the published file stops getting younger.",[42,43],"rect",{"x":44,"y":45,"width":46,"height":47,"fill":48},"0","44","740","206","#ffffff",[42,50],{"x":51,"y":52,"width":53,"height":54,"rx":55,"fill":56,"stroke":57,"style":58},"16","60","132","72","12","#f0f2f5","var(--line,#cdd5e6)","stroke-width:2px",[60,61,65],"text",{"x":62,"y":63,"style":64},"82","88","font-size:11.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","read watermark",[60,67,70],{"x":62,"y":68,"style":69},"110","font-size:11px;fill:var(--muted,#5b6780);text-anchor:middle","last updated_at",[42,72],{"x":73,"y":52,"width":53,"height":54,"rx":55,"fill":74,"stroke":75,"style":58},"168","#ebebfd","var(--brand,#5b5cf0)",[60,77,80],{"x":78,"y":63,"style":79},"234","font-size:11.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","query changes",[60,82,83],{"x":78,"y":68,"style":69},"WHERE > :watermark",[42,85],{"x":86,"y":52,"width":53,"height":54,"rx":55,"fill":87,"stroke":75,"style":58},"320","#f0f4ff",[60,89,91],{"x":90,"y":63,"style":79},"386","merge + build",[60,93,94],{"x":90,"y":68,"style":69},"into report.tmp.xlsx",[42,96],{"x":97,"y":52,"width":53,"height":54,"rx":55,"fill":98,"stroke":99,"style":58},"472","#d9f4f1","var(--teal,#0f9488)",[60,101,104],{"x":102,"y":63,"style":103},"538","font-size:11.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","os.replace",[60,106,107],{"x":102,"y":68,"style":69},"atomic swap",[42,109],{"x":110,"y":52,"width":111,"height":54,"rx":55,"fill":112,"stroke":113,"style":58},"624","100","#5b5cf0","var(--brand-strong,#4338ca)",[60,115,118],{"x":116,"y":63,"style":117},"674","font-size:11.5px;font-weight:700;fill:#ffffff;text-anchor:middle","published",[60,120,122],{"x":116,"y":68,"style":121},"font-size:11px;fill:rgba(255,255,255,0.9);text-anchor:middle","readers see it",[124,125],"line",{"x1":126,"y1":127,"x2":128,"y2":127,"stroke":129,"style":58},"152","96","162","var(--muted,#5b6780)",[131,132],"polygon",{"points":133,"fill":134},"166,96 156,91 156,101","#5b6780",[124,136],{"x1":137,"y1":127,"x2":138,"y2":127,"stroke":129,"style":58},"304","314",[131,140],{"points":141,"fill":134},"318,96 308,91 308,101",[124,143],{"x1":144,"y1":127,"x2":145,"y2":127,"stroke":129,"style":58},"456","466",[131,147],{"points":148,"fill":134},"470,96 460,91 460,101",[124,150],{"x1":151,"y1":127,"x2":152,"y2":127,"stroke":99,"style":58},"608","618",[131,154],{"points":155,"fill":156},"622,96 612,91 612,101","#0f766e",[158,159],"path",{"d":160,"stroke":161,"style":162,"fill":163},"M 538 136 L 538 158 L 82 158 L 82 136","var(--gold,#b4740a)","stroke-width:2px;stroke-dasharray:5 4","none",[131,165],{"points":166,"fill":167},"82,132 76,146 88,146","#8a5808",[60,169,173],{"x":170,"y":171,"style":172},"310","176","font-size:11px;font-weight:600;fill:var(--gold-ink,#7a4e06);text-anchor:middle","write the new watermark only after the swap succeeds",[42,175],{"x":176,"y":177,"width":178,"height":179,"rx":180,"fill":181,"stroke":182},"120","190","500","48","10","#fce9e9","var(--accent-ink,#be185d)",[60,184,188],{"x":185,"y":186,"style":187},"370","211","font-size:11.5px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","A monitor watches the published file's age",[60,190,193],{"x":185,"y":191,"style":192},"230","font-size:11px;fill:var(--text,#172033);text-anchor:middle","— a job that stops running raises nothing else",[195,196,198],"h2",{"id":197},"prerequisites","Prerequisites",[200,201,206],"pre",{"className":202,"code":203,"language":204,"meta":205,"style":205},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl sqlalchemy\n","bash","",[207,208,209],"code",{"__ignoreMap":205},[210,211,213,217,221,224,227],"span",{"class":124,"line":212},1,[210,214,216],{"class":215},"sMTad","pip",[210,218,220],{"class":219},"srMev"," install",[210,222,223],{"class":219}," pandas",[210,225,226],{"class":219}," openpyxl",[210,228,229],{"class":219}," sqlalchemy\n",[10,231,232,233,237],{},"You also need the export itself working as a one-off first; ",[17,234,236],{"href":235},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-sql-query-results-to-excel-with-python\u002F","Export SQL Query Results to Excel with Python"," covers the query, the sheets and the formatting this guide schedules.",[195,239,241],{"id":240},"step-1-query-only-what-changed","Step 1: Query only what changed",[10,243,244,245,248],{},"A full rebuild is the right default while it is fast enough — it holds no state, so it cannot drift. When the full pull stops fitting the window, switch to a watermark: the highest ",[207,246,247],{},"updated_at"," the last successful run saw.",[200,250,254],{"className":251,"code":252,"language":253,"meta":205,"style":205},"language-python shiki shiki-themes github-light github-dark-high-contrast","import json\nfrom pathlib import Path\n\nimport pandas as pd\nfrom sqlalchemy import create_engine, text\n\nengine = create_engine(os.environ[\"REPORT_DB_URL\"], pool_pre_ping=True)\nSTATE = Path(\"state\u002Frefresh.json\")\n\nCHANGED = text(\"\"\"\n    SELECT order_id, region, order_date, amount, updated_at\n    FROM orders\n    WHERE updated_at > :since\n    ORDER BY updated_at\n\"\"\")\n\n\ndef read_state():\n    if STATE.is_file():\n        return json.loads(STATE.read_text())\n    return {\"watermark\": \"1970-01-01T00:00:00\", \"rows\": 0}\n\n\ndef fetch_changes(since):\n    with engine.connect() as conn:\n        return pd.read_sql(CHANGED, conn, params={\"since\": since},\n                           parse_dates=[\"order_date\", \"updated_at\"])\n","python",[207,255,256,266,280,287,301,314,319,350,367,372,386,392,398,404,410,418,423,428,441,453,467,498,503,508,519,533,560],{"__ignoreMap":205},[210,257,258,262],{"class":124,"line":212},[210,259,261],{"class":260},"s-kum","import",[210,263,265],{"class":264},"skGVy"," json\n",[210,267,269,272,275,277],{"class":124,"line":268},2,[210,270,271],{"class":260},"from",[210,273,274],{"class":264}," pathlib ",[210,276,261],{"class":260},[210,278,279],{"class":264}," Path\n",[210,281,283],{"class":124,"line":282},3,[210,284,286],{"emptyLinePlaceholder":285},true,"\n",[210,288,290,292,295,298],{"class":124,"line":289},4,[210,291,261],{"class":260},[210,293,294],{"class":264}," pandas ",[210,296,297],{"class":260},"as",[210,299,300],{"class":264}," pd\n",[210,302,304,306,309,311],{"class":124,"line":303},5,[210,305,271],{"class":260},[210,307,308],{"class":264}," sqlalchemy ",[210,310,261],{"class":260},[210,312,313],{"class":264}," create_engine, text\n",[210,315,317],{"class":124,"line":316},6,[210,318,286],{"emptyLinePlaceholder":285},[210,320,322,325,328,331,334,337,341,343,347],{"class":124,"line":321},7,[210,323,324],{"class":264},"engine ",[210,326,327],{"class":260},"=",[210,329,330],{"class":264}," create_engine(os.environ[",[210,332,333],{"class":219},"\"REPORT_DB_URL\"",[210,335,336],{"class":264},"], ",[210,338,340],{"class":339},"sa561","pool_pre_ping",[210,342,327],{"class":260},[210,344,346],{"class":345},"sP0c6","True",[210,348,349],{"class":264},")\n",[210,351,353,356,359,362,365],{"class":124,"line":352},8,[210,354,355],{"class":345},"STATE",[210,357,358],{"class":260}," =",[210,360,361],{"class":264}," Path(",[210,363,364],{"class":219},"\"state\u002Frefresh.json\"",[210,366,349],{"class":264},[210,368,370],{"class":124,"line":369},9,[210,371,286],{"emptyLinePlaceholder":285},[210,373,375,378,380,383],{"class":124,"line":374},10,[210,376,377],{"class":345},"CHANGED",[210,379,358],{"class":260},[210,381,382],{"class":264}," text(",[210,384,385],{"class":219},"\"\"\"\n",[210,387,389],{"class":124,"line":388},11,[210,390,391],{"class":219},"    SELECT order_id, region, order_date, amount, updated_at\n",[210,393,395],{"class":124,"line":394},12,[210,396,397],{"class":219},"    FROM orders\n",[210,399,401],{"class":124,"line":400},13,[210,402,403],{"class":219},"    WHERE updated_at > :since\n",[210,405,407],{"class":124,"line":406},14,[210,408,409],{"class":219},"    ORDER BY updated_at\n",[210,411,413,416],{"class":124,"line":412},15,[210,414,415],{"class":219},"\"\"\"",[210,417,349],{"class":264},[210,419,421],{"class":124,"line":420},16,[210,422,286],{"emptyLinePlaceholder":285},[210,424,426],{"class":124,"line":425},17,[210,427,286],{"emptyLinePlaceholder":285},[210,429,431,434,438],{"class":124,"line":430},18,[210,432,433],{"class":260},"def",[210,435,437],{"class":436},"s_Opv"," read_state",[210,439,440],{"class":264},"():\n",[210,442,444,447,450],{"class":124,"line":443},19,[210,445,446],{"class":260},"    if",[210,448,449],{"class":345}," STATE",[210,451,452],{"class":264},".is_file():\n",[210,454,456,459,462,464],{"class":124,"line":455},20,[210,457,458],{"class":260},"        return",[210,460,461],{"class":264}," json.loads(",[210,463,355],{"class":345},[210,465,466],{"class":264},".read_text())\n",[210,468,470,473,476,479,482,485,488,491,493,495],{"class":124,"line":469},21,[210,471,472],{"class":260},"    return",[210,474,475],{"class":264}," {",[210,477,478],{"class":219},"\"watermark\"",[210,480,481],{"class":264},": ",[210,483,484],{"class":219},"\"1970-01-01T00:00:00\"",[210,486,487],{"class":264},", ",[210,489,490],{"class":219},"\"rows\"",[210,492,481],{"class":264},[210,494,44],{"class":345},[210,496,497],{"class":264},"}\n",[210,499,501],{"class":124,"line":500},22,[210,502,286],{"emptyLinePlaceholder":285},[210,504,506],{"class":124,"line":505},23,[210,507,286],{"emptyLinePlaceholder":285},[210,509,511,513,516],{"class":124,"line":510},24,[210,512,433],{"class":260},[210,514,515],{"class":436}," fetch_changes",[210,517,518],{"class":264},"(since):\n",[210,520,522,525,528,530],{"class":124,"line":521},25,[210,523,524],{"class":260},"    with",[210,526,527],{"class":264}," engine.connect() ",[210,529,297],{"class":260},[210,531,532],{"class":264}," conn:\n",[210,534,536,538,541,543,546,549,551,554,557],{"class":124,"line":535},26,[210,537,458],{"class":260},[210,539,540],{"class":264}," pd.read_sql(",[210,542,377],{"class":345},[210,544,545],{"class":264},", conn, ",[210,547,548],{"class":339},"params",[210,550,327],{"class":260},[210,552,553],{"class":264},"{",[210,555,556],{"class":219},"\"since\"",[210,558,559],{"class":264},": since},\n",[210,561,563,566,568,571,574,576,579],{"class":124,"line":562},27,[210,564,565],{"class":339},"                           parse_dates",[210,567,327],{"class":260},[210,569,570],{"class":264},"[",[210,572,573],{"class":219},"\"order_date\"",[210,575,487],{"class":264},[210,577,578],{"class":219},"\"updated_at\"",[210,580,581],{"class":264},"])\n",[10,583,584,585,589,590,593],{},"Two rules keep a watermark honest. Use a strictly greater-than comparison against the ",[586,587,588],"em",{},"maximum value you actually received",", not ",[207,591,592],{},"now()"," — clock differences between the database and the job are exactly how rows go missing. And make sure the source column is updated on every write, including deletes-as-flags; a watermark over a column that some updates skip silently drops those rows forever.",[10,595,596],{},"Because rows can be updated as well as inserted, the merge has to be a replace rather than an append:",[200,598,600],{"className":251,"code":599,"language":253,"meta":205,"style":205},"def merge(cached, changes, key=\"order_id\"):\n    if cached is None or cached.empty:\n        return changes\n    keep = cached[~cached[key].isin(changes[key])]\n    return (pd.concat([keep, changes], ignore_index=True)\n              .sort_values([\"region\", \"order_date\"])\n              .reset_index(drop=True))\n",[207,601,602,620,639,646,662,678,692],{"__ignoreMap":205},[210,603,604,606,609,612,614,617],{"class":124,"line":212},[210,605,433],{"class":260},[210,607,608],{"class":436}," merge",[210,610,611],{"class":264},"(cached, changes, key",[210,613,327],{"class":260},[210,615,616],{"class":219},"\"order_id\"",[210,618,619],{"class":264},"):\n",[210,621,622,624,627,630,633,636],{"class":124,"line":268},[210,623,446],{"class":260},[210,625,626],{"class":264}," cached ",[210,628,629],{"class":260},"is",[210,631,632],{"class":345}," None",[210,634,635],{"class":260}," or",[210,637,638],{"class":264}," cached.empty:\n",[210,640,641,643],{"class":124,"line":282},[210,642,458],{"class":260},[210,644,645],{"class":264}," changes\n",[210,647,648,651,653,656,659],{"class":124,"line":289},[210,649,650],{"class":264},"    keep ",[210,652,327],{"class":260},[210,654,655],{"class":264}," cached[",[210,657,658],{"class":260},"~",[210,660,661],{"class":264},"cached[key].isin(changes[key])]\n",[210,663,664,666,669,672,674,676],{"class":124,"line":303},[210,665,472],{"class":260},[210,667,668],{"class":264}," (pd.concat([keep, changes], ",[210,670,671],{"class":339},"ignore_index",[210,673,327],{"class":260},[210,675,346],{"class":345},[210,677,349],{"class":264},[210,679,680,683,686,688,690],{"class":124,"line":316},[210,681,682],{"class":264},"              .sort_values([",[210,684,685],{"class":219},"\"region\"",[210,687,487],{"class":264},[210,689,573],{"class":219},[210,691,581],{"class":264},[210,693,694,697,700,702,704],{"class":124,"line":321},[210,695,696],{"class":264},"              .reset_index(",[210,698,699],{"class":339},"drop",[210,701,327],{"class":260},[210,703,346],{"class":345},[210,705,706],{"class":264},"))\n",[195,708,710],{"id":709},"step-2-write-the-file-atomically","Step 2: Write the file atomically",[10,712,713],{},"A reader who opens the workbook while the job is halfway through writing it gets a corrupt file or a partial one. Write to a temporary name in the same directory and swap:",[200,715,717],{"className":251,"code":716,"language":253,"meta":205,"style":205},"import os\nfrom pathlib import Path\n\n\ndef publish(df, target, build):\n    \"\"\"build(df, path) writes the workbook; this makes the swap atomic.\"\"\"\n    target = Path(target)\n    tmp = target.with_name(f\".{target.stem}.tmp{target.suffix}\")\n    build(df, tmp)\n    os.replace(tmp, target)          # atomic within one filesystem\n    return target\n",[207,718,719,726,736,740,744,754,759,769,809,814,823],{"__ignoreMap":205},[210,720,721,723],{"class":124,"line":212},[210,722,261],{"class":260},[210,724,725],{"class":264}," os\n",[210,727,728,730,732,734],{"class":124,"line":268},[210,729,271],{"class":260},[210,731,274],{"class":264},[210,733,261],{"class":260},[210,735,279],{"class":264},[210,737,738],{"class":124,"line":282},[210,739,286],{"emptyLinePlaceholder":285},[210,741,742],{"class":124,"line":289},[210,743,286],{"emptyLinePlaceholder":285},[210,745,746,748,751],{"class":124,"line":303},[210,747,433],{"class":260},[210,749,750],{"class":436}," publish",[210,752,753],{"class":264},"(df, target, build):\n",[210,755,756],{"class":124,"line":316},[210,757,758],{"class":219},"    \"\"\"build(df, path) writes the workbook; this makes the swap atomic.\"\"\"\n",[210,760,761,764,766],{"class":124,"line":321},[210,762,763],{"class":264},"    target ",[210,765,327],{"class":260},[210,767,768],{"class":264}," Path(target)\n",[210,770,771,774,776,779,782,785,788,791,794,797,799,802,804,807],{"class":124,"line":352},[210,772,773],{"class":264},"    tmp ",[210,775,327],{"class":260},[210,777,778],{"class":264}," target.with_name(",[210,780,781],{"class":260},"f",[210,783,784],{"class":219},"\".",[210,786,553],{"class":787},"sSjpA",[210,789,790],{"class":264},"target.stem",[210,792,793],{"class":787},"}",[210,795,796],{"class":219},".tmp",[210,798,553],{"class":787},[210,800,801],{"class":264},"target.suffix",[210,803,793],{"class":787},[210,805,806],{"class":219},"\"",[210,808,349],{"class":264},[210,810,811],{"class":124,"line":369},[210,812,813],{"class":264},"    build(df, tmp)\n",[210,815,816,819],{"class":124,"line":374},[210,817,818],{"class":264},"    os.replace(tmp, target)          ",[210,820,822],{"class":821},"s-wDw","# atomic within one filesystem\n",[210,824,825,827],{"class":124,"line":388},[210,826,472],{"class":260},[210,828,829],{"class":264}," target\n",[23,831,32,836,32,839,32,842,32,845,32,851,32,855,32,861,32,866,32,870,32,874,32,877,32,880,32,885,32,888,32,892,32,897,32,901,32,904,32,908,32,911,32,915,32,918,32,923,32,927,32,930],{"viewBox":832,"role":26,"ariaLabelledBy":833,"xmlns":30,"style":31},"0 0 740 214",[834,835],"rf-swap-t","rf-swap-d",[34,837,838],{"id":834},"What a reader sees during an in-place write and during an atomic swap",[38,840,841],{"id":835},"Writing directly to the published path leaves a window in which the file exists but is incomplete, so a reader who opens it during the refresh gets a truncated or corrupt workbook. Writing to a temporary name and replacing means the published path always points at a complete file — the previous one, then the new one.",[42,843],{"x":44,"y":44,"width":46,"height":844,"fill":48},"214",[60,846,850],{"x":847,"y":848,"style":849},"184","30","font-size:12.5px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","written in place",[124,852],{"x1":853,"y1":127,"x2":854,"y2":127,"stroke":57,"style":58},"24","344",[42,856],{"x":853,"y":857,"width":858,"height":859,"rx":860,"fill":98,"stroke":99},"70","90","26","4",[60,862,865],{"x":863,"y":63,"style":864},"69","font-size:10.5px;fill:var(--text,#172033);text-anchor:middle","complete",[42,867],{"x":868,"y":857,"width":869,"height":859,"rx":860,"fill":181,"stroke":182,"style":58},"114","140",[60,871,873],{"x":847,"y":63,"style":872},"font-size:10.5px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","partial file",[42,875],{"x":876,"y":857,"width":858,"height":859,"rx":860,"fill":98,"stroke":99},"254",[60,878,865],{"x":879,"y":63,"style":864},"299",[60,881,884],{"x":847,"y":882,"style":883},"122","font-size:11px;fill:var(--accent-ink,#be185d);text-anchor:middle","a reader opening here gets a broken workbook",[60,886,887],{"x":847,"y":126,"style":69},"the window is short and entirely predictable —",[60,889,891],{"x":847,"y":890,"style":69},"170","it is 06:00, when everybody opens the report",[60,893,896],{"x":894,"y":848,"style":895},"556","font-size:12.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","temp file, then os.replace",[124,898],{"x1":899,"y1":127,"x2":900,"y2":127,"stroke":57,"style":58},"396","716",[42,902],{"x":899,"y":857,"width":903,"height":859,"rx":860,"fill":98,"stroke":99},"150",[60,905,907],{"x":906,"y":63,"style":864},"471","yesterday's file",[42,909],{"x":910,"y":857,"width":903,"height":859,"rx":860,"fill":98,"stroke":99},"566",[60,912,914],{"x":913,"y":63,"style":864},"641","today's file",[124,916],{"x1":894,"y1":52,"x2":894,"y2":917,"stroke":75,"style":58},"106",[60,919,922],{"x":894,"y":920,"style":921},"52","font-size:10.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","swap",[60,924,926],{"x":894,"y":882,"style":925},"font-size:11px;fill:var(--teal-ink,#0b6157);text-anchor:middle","there is no moment in between",[60,928,929],{"x":894,"y":126,"style":69},"the temporary file must be in the same directory,",[60,931,932],{"x":894,"y":890,"style":69},"or the rename becomes a copy and loses the guarantee",[10,934,935,937],{},[207,936,104],{}," is atomic on POSIX and on Windows: readers see either the old file or the new one, never a partial write. The temporary file must be in the same directory as the target — a rename across filesystems is a copy, and a copy is not atomic. Prefixing the temporary name with a dot keeps it out of the way of anyone browsing the folder mid-run.",[10,939,940,941,944],{},"On Windows there is one extra case: if a colleague has the workbook open in Excel, the replacement raises ",[207,942,943],{},"PermissionError"," because Excel holds a lock on the target. Decide deliberately which behaviour you want:",[200,946,948],{"className":251,"code":947,"language":253,"meta":205,"style":205},"def publish_or_park(df, target, build, retries=3, wait=20):\n    for attempt in range(1, retries + 1):\n        try:\n            return publish(df, target, build)\n        except PermissionError:\n            if attempt == retries:\n                parked = Path(target).with_name(\n                    f\"{Path(target).stem}-{pd.Timestamp.now():%Y%m%d-%H%M}.xlsx\")\n                build(df, parked)\n                raise RuntimeError(f\"{target} is locked; wrote {parked} instead\")\n            time.sleep(wait)\n",[207,949,950,975,1006,1014,1022,1032,1045,1055,1111,1116,1152],{"__ignoreMap":205},[210,951,952,954,957,960,962,965,968,970,973],{"class":124,"line":212},[210,953,433],{"class":260},[210,955,956],{"class":436}," publish_or_park",[210,958,959],{"class":264},"(df, target, build, retries",[210,961,327],{"class":260},[210,963,964],{"class":345},"3",[210,966,967],{"class":264},", wait",[210,969,327],{"class":260},[210,971,972],{"class":345},"20",[210,974,619],{"class":264},[210,976,977,980,983,986,989,992,995,998,1001,1004],{"class":124,"line":268},[210,978,979],{"class":260},"    for",[210,981,982],{"class":264}," attempt ",[210,984,985],{"class":260},"in",[210,987,988],{"class":345}," range",[210,990,991],{"class":264},"(",[210,993,994],{"class":345},"1",[210,996,997],{"class":264},", retries ",[210,999,1000],{"class":260},"+",[210,1002,1003],{"class":345}," 1",[210,1005,619],{"class":264},[210,1007,1008,1011],{"class":124,"line":282},[210,1009,1010],{"class":260},"        try",[210,1012,1013],{"class":264},":\n",[210,1015,1016,1019],{"class":124,"line":289},[210,1017,1018],{"class":260},"            return",[210,1020,1021],{"class":264}," publish(df, target, build)\n",[210,1023,1024,1027,1030],{"class":124,"line":303},[210,1025,1026],{"class":260},"        except",[210,1028,1029],{"class":345}," PermissionError",[210,1031,1013],{"class":264},[210,1033,1034,1037,1039,1042],{"class":124,"line":316},[210,1035,1036],{"class":260},"            if",[210,1038,982],{"class":264},[210,1040,1041],{"class":260},"==",[210,1043,1044],{"class":264}," retries:\n",[210,1046,1047,1050,1052],{"class":124,"line":321},[210,1048,1049],{"class":264},"                parked ",[210,1051,327],{"class":260},[210,1053,1054],{"class":264}," Path(target).with_name(\n",[210,1056,1057,1060,1062,1064,1067,1069,1072,1074,1077,1080,1083,1085,1088,1090,1093,1096,1099,1101,1104,1106,1109],{"class":124,"line":352},[210,1058,1059],{"class":260},"                    f",[210,1061,806],{"class":219},[210,1063,553],{"class":787},[210,1065,1066],{"class":264},"Path(target).stem",[210,1068,793],{"class":787},[210,1070,1071],{"class":219},"-",[210,1073,553],{"class":787},[210,1075,1076],{"class":264},"pd.Timestamp.now():",[210,1078,1079],{"class":260},"%",[210,1081,1082],{"class":264},"Y",[210,1084,1079],{"class":260},[210,1086,1087],{"class":264},"m",[210,1089,1079],{"class":260},[210,1091,1092],{"class":264},"d",[210,1094,1095],{"class":260},"-%",[210,1097,1098],{"class":264},"H",[210,1100,1079],{"class":260},[210,1102,1103],{"class":264},"M",[210,1105,793],{"class":787},[210,1107,1108],{"class":219},".xlsx\"",[210,1110,349],{"class":264},[210,1112,1113],{"class":124,"line":369},[210,1114,1115],{"class":264},"                build(df, parked)\n",[210,1117,1118,1121,1124,1126,1128,1130,1132,1135,1137,1140,1142,1145,1147,1150],{"class":124,"line":374},[210,1119,1120],{"class":260},"                raise",[210,1122,1123],{"class":345}," RuntimeError",[210,1125,991],{"class":264},[210,1127,781],{"class":260},[210,1129,806],{"class":219},[210,1131,553],{"class":787},[210,1133,1134],{"class":264},"target",[210,1136,793],{"class":787},[210,1138,1139],{"class":219}," is locked; wrote ",[210,1141,553],{"class":787},[210,1143,1144],{"class":264},"parked",[210,1146,793],{"class":787},[210,1148,1149],{"class":219}," instead\"",[210,1151,349],{"class":264},[210,1153,1154],{"class":124,"line":388},[210,1155,1156],{"class":264},"            time.sleep(wait)\n",[10,1158,1159,1160,1164],{},"Parking the output under a dated name means a locked file delays publication rather than losing the run's work — and the error names both files, so whoever is holding the lock knows what to do. ",[17,1161,1163],{"href":1162},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fhandle-permission-denied-when-writing-excel-in-python\u002F","Handle Permission Denied When Writing Excel in Python"," covers the lock behaviour in more detail.",[195,1166,1168],{"id":1167},"step-3-stamp-the-workbook-so-readers-can-see-its-age","Step 3: Stamp the workbook so readers can see its age",[10,1170,1171],{},"The cheapest reliability feature in reporting is a visible timestamp:",[200,1173,1175],{"className":251,"code":1174,"language":253,"meta":205,"style":205},"from openpyxl.styles import Font\n\n\ndef build_workbook(df, path):\n    summary = df.groupby(\"region\", as_index=False)[\"amount\"].sum()\n    with pd.ExcelWriter(path, engine=\"openpyxl\",\n                        datetime_format=\"yyyy-mm-dd\") as writer:\n        summary.to_excel(writer, sheet_name=\"Summary\", index=False, startrow=2)\n        df.to_excel(writer, sheet_name=\"Detail\", index=False)\n\n        ws = writer.sheets[\"Summary\"]\n        ws[\"A1\"] = f\"Generated {pd.Timestamp.now():%Y-%m-%d %H:%M} · {len(df):,} rows\"\n        ws[\"A1\"].font = Font(italic=True, color=\"5B6780\")\n        ws.freeze_panes = \"A4\"\n",[207,1176,1177,1189,1193,1197,1207,1238,1256,1274,1308,1330,1334,1349,1421,1454],{"__ignoreMap":205},[210,1178,1179,1181,1184,1186],{"class":124,"line":212},[210,1180,271],{"class":260},[210,1182,1183],{"class":264}," openpyxl.styles ",[210,1185,261],{"class":260},[210,1187,1188],{"class":264}," Font\n",[210,1190,1191],{"class":124,"line":268},[210,1192,286],{"emptyLinePlaceholder":285},[210,1194,1195],{"class":124,"line":282},[210,1196,286],{"emptyLinePlaceholder":285},[210,1198,1199,1201,1204],{"class":124,"line":289},[210,1200,433],{"class":260},[210,1202,1203],{"class":436}," build_workbook",[210,1205,1206],{"class":264},"(df, path):\n",[210,1208,1209,1212,1214,1217,1219,1221,1224,1226,1229,1232,1235],{"class":124,"line":303},[210,1210,1211],{"class":264},"    summary ",[210,1213,327],{"class":260},[210,1215,1216],{"class":264}," df.groupby(",[210,1218,685],{"class":219},[210,1220,487],{"class":264},[210,1222,1223],{"class":339},"as_index",[210,1225,327],{"class":260},[210,1227,1228],{"class":345},"False",[210,1230,1231],{"class":264},")[",[210,1233,1234],{"class":219},"\"amount\"",[210,1236,1237],{"class":264},"].sum()\n",[210,1239,1240,1242,1245,1248,1250,1253],{"class":124,"line":316},[210,1241,524],{"class":260},[210,1243,1244],{"class":264}," pd.ExcelWriter(path, ",[210,1246,1247],{"class":339},"engine",[210,1249,327],{"class":260},[210,1251,1252],{"class":219},"\"openpyxl\"",[210,1254,1255],{"class":264},",\n",[210,1257,1258,1261,1263,1266,1269,1271],{"class":124,"line":321},[210,1259,1260],{"class":339},"                        datetime_format",[210,1262,327],{"class":260},[210,1264,1265],{"class":219},"\"yyyy-mm-dd\"",[210,1267,1268],{"class":264},") ",[210,1270,297],{"class":260},[210,1272,1273],{"class":264}," writer:\n",[210,1275,1276,1279,1282,1284,1287,1289,1292,1294,1296,1298,1301,1303,1306],{"class":124,"line":352},[210,1277,1278],{"class":264},"        summary.to_excel(writer, ",[210,1280,1281],{"class":339},"sheet_name",[210,1283,327],{"class":260},[210,1285,1286],{"class":219},"\"Summary\"",[210,1288,487],{"class":264},[210,1290,1291],{"class":339},"index",[210,1293,327],{"class":260},[210,1295,1228],{"class":345},[210,1297,487],{"class":264},[210,1299,1300],{"class":339},"startrow",[210,1302,327],{"class":260},[210,1304,1305],{"class":345},"2",[210,1307,349],{"class":264},[210,1309,1310,1313,1315,1317,1320,1322,1324,1326,1328],{"class":124,"line":369},[210,1311,1312],{"class":264},"        df.to_excel(writer, ",[210,1314,1281],{"class":339},[210,1316,327],{"class":260},[210,1318,1319],{"class":219},"\"Detail\"",[210,1321,487],{"class":264},[210,1323,1291],{"class":339},[210,1325,327],{"class":260},[210,1327,1228],{"class":345},[210,1329,349],{"class":264},[210,1331,1332],{"class":124,"line":374},[210,1333,286],{"emptyLinePlaceholder":285},[210,1335,1336,1339,1341,1344,1346],{"class":124,"line":388},[210,1337,1338],{"class":264},"        ws ",[210,1340,327],{"class":260},[210,1342,1343],{"class":264}," writer.sheets[",[210,1345,1286],{"class":219},[210,1347,1348],{"class":264},"]\n",[210,1350,1351,1354,1357,1360,1362,1365,1368,1370,1373,1376,1378,1381,1383,1386,1388,1391,1393,1395,1398,1400,1402,1405,1407,1410,1413,1416,1418],{"class":124,"line":394},[210,1352,1353],{"class":264},"        ws[",[210,1355,1356],{"class":219},"\"A1\"",[210,1358,1359],{"class":264},"] ",[210,1361,327],{"class":260},[210,1363,1364],{"class":260}," f",[210,1366,1367],{"class":219},"\"Generated ",[210,1369,553],{"class":787},[210,1371,1372],{"class":264},"pd.Timestamp.now()",[210,1374,1375],{"class":260},":",[210,1377,1079],{"class":264},[210,1379,1380],{"class":260},"Y-",[210,1382,1079],{"class":264},[210,1384,1385],{"class":260},"m-",[210,1387,1079],{"class":264},[210,1389,1390],{"class":260},"d ",[210,1392,1079],{"class":264},[210,1394,1098],{"class":260},[210,1396,1397],{"class":264},":%",[210,1399,1103],{"class":260},[210,1401,793],{"class":787},[210,1403,1404],{"class":219}," · ",[210,1406,553],{"class":787},[210,1408,1409],{"class":345},"len",[210,1411,1412],{"class":264},"(df)",[210,1414,1415],{"class":260},":,",[210,1417,793],{"class":787},[210,1419,1420],{"class":219}," rows\"\n",[210,1422,1423,1425,1427,1430,1432,1435,1438,1440,1442,1444,1447,1449,1452],{"class":124,"line":400},[210,1424,1353],{"class":264},[210,1426,1356],{"class":219},[210,1428,1429],{"class":264},"].font ",[210,1431,327],{"class":260},[210,1433,1434],{"class":264}," Font(",[210,1436,1437],{"class":339},"italic",[210,1439,327],{"class":260},[210,1441,346],{"class":345},[210,1443,487],{"class":264},[210,1445,1446],{"class":339},"color",[210,1448,327],{"class":260},[210,1450,1451],{"class":219},"\"5B6780\"",[210,1453,349],{"class":264},[210,1455,1456,1459,1461],{"class":124,"line":406},[210,1457,1458],{"class":264},"        ws.freeze_panes ",[210,1460,327],{"class":260},[210,1462,1463],{"class":219}," \"A4\"\n",[10,1465,1466,1469],{},[207,1467,1468],{},"startrow=2"," leaves room for the stamp above the table. Put it on the first sheet, above the fold, not in a footer or a hidden metadata sheet: the point is that a reader who has had the file open for a fortnight sees the date without looking for it.",[195,1471,1473],{"id":1472},"step-4-record-state-only-after-success","Step 4: Record state only after success",[10,1475,1476],{},"The watermark must advance only when the file has actually been published — otherwise a failure between the query and the write skips those rows on the next run:",[200,1478,1480],{"className":251,"code":1479,"language":253,"meta":205,"style":205},"def refresh(target=\"reports\u002Forders.xlsx\"):\n    state = read_state()\n    changes = fetch_changes(state[\"watermark\"])\n\n    if changes.empty:\n        log.info(\"no changes since %s\", state[\"watermark\"])\n        return 0\n\n    cached = load_cached_extract()\n    merged = merge(cached, changes)\n    publish(merged, target, build_workbook)\n    save_cached_extract(merged)\n\n    STATE.parent.mkdir(parents=True, exist_ok=True)\n    STATE.write_text(json.dumps({\n        \"watermark\": changes[\"updated_at\"].max().isoformat(),\n        \"rows\": len(merged),\n        \"published_at\": pd.Timestamp.now().isoformat(timespec=\"seconds\"),\n    }, indent=2))\n    return len(changes)\n",[207,1481,1482,1499,1509,1523,1527,1534,1554,1561,1565,1575,1585,1590,1595,1599,1625,1632,1645,1657,1676,1690],{"__ignoreMap":205},[210,1483,1484,1486,1489,1492,1494,1497],{"class":124,"line":212},[210,1485,433],{"class":260},[210,1487,1488],{"class":436}," refresh",[210,1490,1491],{"class":264},"(target",[210,1493,327],{"class":260},[210,1495,1496],{"class":219},"\"reports\u002Forders.xlsx\"",[210,1498,619],{"class":264},[210,1500,1501,1504,1506],{"class":124,"line":268},[210,1502,1503],{"class":264},"    state ",[210,1505,327],{"class":260},[210,1507,1508],{"class":264}," read_state()\n",[210,1510,1511,1514,1516,1519,1521],{"class":124,"line":282},[210,1512,1513],{"class":264},"    changes ",[210,1515,327],{"class":260},[210,1517,1518],{"class":264}," fetch_changes(state[",[210,1520,478],{"class":219},[210,1522,581],{"class":264},[210,1524,1525],{"class":124,"line":289},[210,1526,286],{"emptyLinePlaceholder":285},[210,1528,1529,1531],{"class":124,"line":303},[210,1530,446],{"class":260},[210,1532,1533],{"class":264}," changes.empty:\n",[210,1535,1536,1539,1542,1545,1547,1550,1552],{"class":124,"line":316},[210,1537,1538],{"class":264},"        log.info(",[210,1540,1541],{"class":219},"\"no changes since ",[210,1543,1544],{"class":787},"%s",[210,1546,806],{"class":219},[210,1548,1549],{"class":264},", state[",[210,1551,478],{"class":219},[210,1553,581],{"class":264},[210,1555,1556,1558],{"class":124,"line":321},[210,1557,458],{"class":260},[210,1559,1560],{"class":345}," 0\n",[210,1562,1563],{"class":124,"line":352},[210,1564,286],{"emptyLinePlaceholder":285},[210,1566,1567,1570,1572],{"class":124,"line":369},[210,1568,1569],{"class":264},"    cached ",[210,1571,327],{"class":260},[210,1573,1574],{"class":264}," load_cached_extract()\n",[210,1576,1577,1580,1582],{"class":124,"line":374},[210,1578,1579],{"class":264},"    merged ",[210,1581,327],{"class":260},[210,1583,1584],{"class":264}," merge(cached, changes)\n",[210,1586,1587],{"class":124,"line":388},[210,1588,1589],{"class":264},"    publish(merged, target, build_workbook)\n",[210,1591,1592],{"class":124,"line":394},[210,1593,1594],{"class":264},"    save_cached_extract(merged)\n",[210,1596,1597],{"class":124,"line":400},[210,1598,286],{"emptyLinePlaceholder":285},[210,1600,1601,1604,1607,1610,1612,1614,1616,1619,1621,1623],{"class":124,"line":406},[210,1602,1603],{"class":345},"    STATE",[210,1605,1606],{"class":264},".parent.mkdir(",[210,1608,1609],{"class":339},"parents",[210,1611,327],{"class":260},[210,1613,346],{"class":345},[210,1615,487],{"class":264},[210,1617,1618],{"class":339},"exist_ok",[210,1620,327],{"class":260},[210,1622,346],{"class":345},[210,1624,349],{"class":264},[210,1626,1627,1629],{"class":124,"line":412},[210,1628,1603],{"class":345},[210,1630,1631],{"class":264},".write_text(json.dumps({\n",[210,1633,1634,1637,1640,1642],{"class":124,"line":420},[210,1635,1636],{"class":219},"        \"watermark\"",[210,1638,1639],{"class":264},": changes[",[210,1641,578],{"class":219},[210,1643,1644],{"class":264},"].max().isoformat(),\n",[210,1646,1647,1650,1652,1654],{"class":124,"line":425},[210,1648,1649],{"class":219},"        \"rows\"",[210,1651,481],{"class":264},[210,1653,1409],{"class":345},[210,1655,1656],{"class":264},"(merged),\n",[210,1658,1659,1662,1665,1668,1670,1673],{"class":124,"line":430},[210,1660,1661],{"class":219},"        \"published_at\"",[210,1663,1664],{"class":264},": pd.Timestamp.now().isoformat(",[210,1666,1667],{"class":339},"timespec",[210,1669,327],{"class":260},[210,1671,1672],{"class":219},"\"seconds\"",[210,1674,1675],{"class":264},"),\n",[210,1677,1678,1681,1684,1686,1688],{"class":124,"line":443},[210,1679,1680],{"class":264},"    }, ",[210,1682,1683],{"class":339},"indent",[210,1685,327],{"class":260},[210,1687,1305],{"class":345},[210,1689,706],{"class":264},[210,1691,1692,1694,1697],{"class":124,"line":455},[210,1693,472],{"class":260},[210,1695,1696],{"class":345}," len",[210,1698,1699],{"class":264},"(changes)\n",[10,1701,1702,1703,1706],{},"The ordering is the whole point: query, merge, publish, ",[586,1704,1705],{},"then"," record. Written the other way round — state first — a crash during the write advances the watermark past rows that were never published, and those rows are gone from every future run. The same argument applies to the cached extract, which is why it is saved after the publish rather than before.",[23,1708,32,1713,32,1716,32,1719,32,1722,32,1726,32,1732,32,1737,32,1740,32,1744,32,1748,32,1753,32,1757,32,1761,32,1765,32,1768,32,1770,32,1773,32,1776,32,1779,32,1782,32,1785,32,1788,32,1791,32,1794,32,1798],{"viewBox":1709,"role":26,"ariaLabelledBy":1710,"xmlns":30,"style":31},"0 0 740 246",[1711,1712],"rf-order-t","rf-order-d",[34,1714,1715],{"id":1711},"Why the watermark is written last",[38,1717,1718],{"id":1712},"If the watermark is saved before the file is published, a crash during the write leaves the state advanced past rows that never reached the workbook, and no future run will fetch them again. Saving it after the publish means a crash simply repeats the same query on the next run.",[42,1720],{"x":44,"y":44,"width":46,"height":1721,"fill":48},"246",[60,1723,1725],{"x":847,"y":1724,"style":849},"32","watermark first",[42,1727],{"x":853,"y":1728,"width":1729,"height":1730,"rx":1731,"fill":181,"stroke":182,"style":58},"50","94","40","8",[60,1733,1736],{"x":1734,"y":1735,"style":192},"71","75","save state",[42,1738],{"x":1739,"y":1728,"width":1729,"height":1730,"rx":1731,"fill":181,"stroke":182,"style":58},"130",[60,1741,1743],{"x":1742,"y":1735,"style":192},"177","build",[42,1745],{"x":1746,"y":1728,"width":1747,"height":1730,"rx":1731,"fill":181,"stroke":182,"style":58},"236","108",[60,1749,1752],{"x":1750,"y":1735,"style":1751},"290","font-size:11px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","crash ✗",[60,1754,1756],{"x":847,"y":882,"style":1755},"font-size:11.5px;fill:var(--text,#172033);text-anchor:middle","rows are past the watermark but not in the file",[60,1758,1760],{"x":847,"y":1759,"style":69},"146","no future run will ever fetch them —",[60,1762,1764],{"x":847,"y":1763,"style":69},"164","the gap is permanent and invisible",[60,1766,1767],{"x":894,"y":1724,"style":895},"watermark last",[42,1769],{"x":899,"y":1728,"width":1729,"height":1730,"rx":1731,"fill":98,"stroke":99,"style":58},[60,1771,1743],{"x":1772,"y":1735,"style":192},"443",[42,1774],{"x":1775,"y":1728,"width":1747,"height":1730,"rx":1731,"fill":98,"stroke":99,"style":58},"502",[60,1777,1752],{"x":894,"y":1735,"style":1778},"font-size:11px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle",[42,1780],{"x":1781,"y":1728,"width":1729,"height":1730,"rx":1731,"fill":56,"stroke":57},"622",[60,1783,1736],{"x":1784,"y":1735,"style":69},"669",[60,1786,1787],{"x":894,"y":882,"style":1755},"the state still points at the last good run",[60,1789,1790],{"x":894,"y":1759,"style":69},"the next run repeats the same query and",[60,1792,1793],{"x":894,"y":1763,"style":69},"publishes what the crashed one could not",[42,1795],{"x":176,"y":1796,"width":178,"height":1797,"rx":180,"fill":87,"stroke":75},"188","46",[60,1799,1801],{"x":185,"y":1800,"style":79},"216","Repeating work after a crash is cheap; losing rows after one is not",[195,1803,1805],{"id":1804},"step-5-alert-on-staleness-not-just-on-failure","Step 5: Alert on staleness, not just on failure",[10,1807,1808],{},"A failing job raises an error someone can route. A job that stops being triggered — a disabled task, a deleted crontab, a decommissioned server — raises nothing at all. The only signal is the file's age, so check that separately from the job itself:",[200,1810,1812],{"className":251,"code":1811,"language":253,"meta":205,"style":205},"from datetime import datetime, timedelta\n\n\ndef check_freshness(path, max_age=timedelta(hours=26)):\n    path = Path(path)\n    if not path.exists():\n        return f\"{path} does not exist — the refresh has never succeeded\"\n    age = datetime.now() - datetime.fromtimestamp(path.stat().st_mtime)\n    if age > max_age:\n        return f\"{path} is {age.total_seconds() \u002F 3600:.1f}h old (limit {max_age})\"\n    return None\n",[207,1813,1814,1826,1830,1834,1859,1869,1879,1896,1911,1924,1970],{"__ignoreMap":205},[210,1815,1816,1818,1821,1823],{"class":124,"line":212},[210,1817,271],{"class":260},[210,1819,1820],{"class":264}," datetime ",[210,1822,261],{"class":260},[210,1824,1825],{"class":264}," datetime, timedelta\n",[210,1827,1828],{"class":124,"line":268},[210,1829,286],{"emptyLinePlaceholder":285},[210,1831,1832],{"class":124,"line":282},[210,1833,286],{"emptyLinePlaceholder":285},[210,1835,1836,1838,1841,1844,1846,1849,1852,1854,1856],{"class":124,"line":289},[210,1837,433],{"class":260},[210,1839,1840],{"class":436}," check_freshness",[210,1842,1843],{"class":264},"(path, max_age",[210,1845,327],{"class":260},[210,1847,1848],{"class":264},"timedelta(",[210,1850,1851],{"class":339},"hours",[210,1853,327],{"class":260},[210,1855,859],{"class":345},[210,1857,1858],{"class":264},")):\n",[210,1860,1861,1864,1866],{"class":124,"line":303},[210,1862,1863],{"class":264},"    path ",[210,1865,327],{"class":260},[210,1867,1868],{"class":264}," Path(path)\n",[210,1870,1871,1873,1876],{"class":124,"line":316},[210,1872,446],{"class":260},[210,1874,1875],{"class":260}," not",[210,1877,1878],{"class":264}," path.exists():\n",[210,1880,1881,1883,1885,1887,1889,1891,1893],{"class":124,"line":321},[210,1882,458],{"class":260},[210,1884,1364],{"class":260},[210,1886,806],{"class":219},[210,1888,553],{"class":787},[210,1890,158],{"class":264},[210,1892,793],{"class":787},[210,1894,1895],{"class":219}," does not exist — the refresh has never succeeded\"\n",[210,1897,1898,1901,1903,1906,1908],{"class":124,"line":352},[210,1899,1900],{"class":264},"    age ",[210,1902,327],{"class":260},[210,1904,1905],{"class":264}," datetime.now() ",[210,1907,1071],{"class":260},[210,1909,1910],{"class":264}," datetime.fromtimestamp(path.stat().st_mtime)\n",[210,1912,1913,1915,1918,1921],{"class":124,"line":369},[210,1914,446],{"class":260},[210,1916,1917],{"class":264}," age ",[210,1919,1920],{"class":260},">",[210,1922,1923],{"class":264}," max_age:\n",[210,1925,1926,1928,1930,1932,1934,1936,1938,1941,1943,1946,1949,1952,1955,1957,1960,1962,1965,1967],{"class":124,"line":374},[210,1927,458],{"class":260},[210,1929,1364],{"class":260},[210,1931,806],{"class":219},[210,1933,553],{"class":787},[210,1935,158],{"class":264},[210,1937,793],{"class":787},[210,1939,1940],{"class":219}," is ",[210,1942,553],{"class":787},[210,1944,1945],{"class":264},"age.total_seconds() ",[210,1947,1948],{"class":260},"\u002F",[210,1950,1951],{"class":345}," 3600",[210,1953,1954],{"class":260},":.1f",[210,1956,793],{"class":787},[210,1958,1959],{"class":219},"h old (limit ",[210,1961,553],{"class":787},[210,1963,1964],{"class":264},"max_age",[210,1966,793],{"class":787},[210,1968,1969],{"class":219},")\"\n",[210,1971,1972,1974],{"class":124,"line":388},[210,1973,472],{"class":260},[210,1975,1976],{"class":345}," None\n",[10,1978,1979,1980,1984],{},"Run that from a different schedule than the refresh — a separate cron entry, a monitoring system, anything that is not the job being watched. The allowance of 26 hours for a daily job is deliberate slack: it tolerates a late run without alerting, and still fires long before anyone would quote day-old numbers as current. Wire the message into whatever the rest of your ",[17,1981,1983],{"href":1982},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002F","error handling and logging"," already uses.",[195,1986,1988],{"id":1987},"common-pitfalls-and-gotchas","Common pitfalls and gotchas",[1990,1991,1992,2008],"table",{},[1993,1994,1995],"thead",{},[1996,1997,1998,2002,2005],"tr",{},[1999,2000,2001],"th",{},"Symptom",[1999,2003,2004],{},"Cause",[1999,2006,2007],{},"Fix",[2009,2010,2011,2030,2041,2054,2067,2078,2089,2100],"tbody",{},[1996,2012,2013,2017,2023],{},[2014,2015,2016],"td",{},"Rows missing from the extract",[2014,2018,2019,2020,2022],{},"Watermark set from ",[207,2021,592],{}," rather than the data",[2014,2024,2025,2026,2029],{},"Use ",[207,2027,2028],{},"max(updated_at)"," of the rows received",[1996,2031,2032,2035,2038],{},[2014,2033,2034],{},"Duplicate rows after a refresh",[2014,2036,2037],{},"Changes appended instead of replaced by key",[2014,2039,2040],{},"Drop matching keys from the cache, then concat",[1996,2042,2043,2046,2049],{},[2014,2044,2045],{},"Corrupt file for a reader",[2014,2047,2048],{},"The workbook was written in place",[2014,2050,2051,2052],{},"Write to a temp name, then ",[207,2053,104],{},[1996,2055,2056,2061,2064],{},[2014,2057,2058,2060],{},[207,2059,943],{}," on Windows",[2014,2062,2063],{},"Excel holds a lock on the open file",[2014,2065,2066],{},"Retry, then park under a dated name",[1996,2068,2069,2072,2075],{},[2014,2070,2071],{},"Numbers quoted weeks later as current",[2014,2073,2074],{},"No visible timestamp",[2014,2076,2077],{},"Stamp the summary sheet above the table",[1996,2079,2080,2083,2086],{},[2014,2081,2082],{},"Silent stop, nobody noticed",[2014,2084,2085],{},"Only the job's exit code was monitored",[2014,2087,2088],{},"Alert on the output file's age",[1996,2090,2091,2094,2097],{},[2014,2092,2093],{},"Refresh grows slower each week",[2014,2095,2096],{},"Full rebuild over a growing table",[2014,2098,2099],{},"Move to the watermark query",[1996,2101,2102,2105,2108],{},[2014,2103,2104],{},"Timezone drift in the watermark",[2014,2106,2107],{},"Naive local times on both sides",[2014,2109,2110],{},"Store UTC, compare UTC",[195,2112,2114],{"id":2113},"performance-and-scale-notes","Performance and scale notes",[10,2116,2117],{},"Incremental refresh trades a simple job for a stateful one, so make the switch only when the numbers justify it. A full extract that takes 40 seconds is not worth complicating; one taking twenty minutes and growing is. When you do switch, keep a periodic full rebuild — weekly is common — so that any rows a bad watermark missed get corrected rather than accumulating.",[10,2119,2120],{},"Two indexes decide the cost: one on the watermark column for the incremental query, and one on the key used by the merge. Without the first, every refresh scans the whole table and the incremental version is slower than the full one it replaced.",[195,2122,2124],{"id":2123},"conclusion","Conclusion",[10,2126,2127],{},"A scheduled refresh is a small amount of code and four decisions. Query with a watermark taken from the data, not the clock. Publish atomically so a reader never opens a half-written file, and decide in advance what a locked file should do. Stamp the workbook where readers will see it. Record the new state only after the publish succeeds, and monitor the file's age separately, because the failure that actually happens is the job quietly not running at all.",[195,2129,2131],{"id":2130},"frequently-asked-questions","Frequently asked questions",[10,2133,2134,2138],{},[2135,2136,2137],"strong",{},"What happens if someone has the file open when the refresh runs?","\nOn Windows the replacement fails with a permission error, because Excel holds a lock. Write to a temporary name and replace, catch the error, and either retry or publish under a dated filename so a lock never blocks the whole run.",[10,2140,2141,2144],{},[2135,2142,2143],{},"Full refresh or incremental?","\nFull while it is fast enough — it has no state to get wrong. Move to an incremental watermark query when the full pull stops fitting in the window, and keep a periodic full rebuild to correct any drift.",[10,2146,2147,2150],{},[2135,2148,2149],{},"How do readers know the numbers are current?","\nPut a generated-at timestamp and the source period in a visible cell on the first sheet. A file without one is treated as current forever.",[10,2152,2153,2156],{},[2135,2154,2155],{},"How do I get alerted when the refresh silently stops?","\nMonitor the output file's age rather than the job's exit code. A job that never runs produces no failure — only a file that quietly gets older.",[195,2158,2160],{"id":2159},"related","Related",[10,2162,2163],{},"Up to the parent guide:",[2165,2166,2167],"ul",{},[2168,2169,2170,2172],"li",{},[17,2171,20],{"href":19}," — the export and load this schedules.",[10,2174,2175],{},"Related guides:",[2165,2177,2178,2183,2190,2195],{},[2168,2179,2180,2182],{},[17,2181,236],{"href":235}," — the query and formatting run on each cycle.",[2168,2184,2185,2189],{},[17,2186,2188],{"href":2187},"\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Fschedule-recurring-excel-reports-with-apscheduler\u002F","Schedule Recurring Excel Reports with APScheduler"," — the trigger side of the same job.",[2168,2191,2192,2194],{},[17,2193,1163],{"href":1162}," — what to do when a reader has the file open.",[2168,2196,2197,2201],{},[17,2198,2200],{"href":2199},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fvalidate-an-excel-report-before-sending-it\u002F","Validate an Excel Report Before Sending It"," — checks worth running between the build and the swap.",[2203,2204,2205],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}",{"title":205,"searchDepth":268,"depth":268,"links":2207},[2208,2209,2210,2211,2212,2213,2214,2215,2216,2217,2218],{"id":197,"depth":268,"text":198},{"id":240,"depth":268,"text":241},{"id":709,"depth":268,"text":710},{"id":1167,"depth":268,"text":1168},{"id":1472,"depth":268,"text":1473},{"id":1804,"depth":268,"text":1805},{"id":1987,"depth":268,"text":1988},{"id":2113,"depth":268,"text":2114},{"id":2123,"depth":268,"text":2124},{"id":2130,"depth":268,"text":2131},{"id":2159,"depth":268,"text":2160},"2026-08-10","Keep a workbook current without anyone asking: an incremental query with a watermark, an atomic write nobody can catch half-finished, a freshness stamp readers can see, and a check that the refresh actually ran.","md",[2223,2225,2227,2229],{"q":2137,"a":2224},"On Windows the replacement fails with a permission error, because Excel holds a lock. Write to a temporary name and replace, catch the error, and either retry or publish under a dated filename so a lock never blocks the whole run.",{"q":2143,"a":2226},"Full while it is fast enough — it has no state to get wrong. Move to an incremental watermark query when the full pull stops fitting in the window, and keep a periodic full rebuild to correct any drift.",{"q":2149,"a":2228},"Put a generated-at timestamp and the source period in a visible cell on the first sheet. A file without one is treated as current forever.",{"q":2155,"a":2230},"Monitor the output file's age rather than the job's exit code. A job that never runs produces no failure — only a file that quietly gets older.",{"breadcrumb":2232},[2233,2235,2238,2239],{"name":2234,"item":1948},"Home",{"name":2236,"item":2237},"Advanced Data Transformation and Cleaning","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":20,"item":19},{"name":5,"item":2240},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Frefresh-an-excel-report-from-a-database-on-a-schedule\u002F","\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Frefresh-an-excel-report-from-a-database-on-a-schedule",{"title":2243,"description":2244},"Schedule an Excel Report Refresh from SQL","Refresh an .xlsx extract from a database on a schedule — incremental watermark queries, atomic file replacement, a generated-at stamp, staleness alerts and safe concurrent readers.","refresh-an-excel-report-from-a-database-on-a-schedule","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Frefresh-an-excel-report-from-a-database-on-a-schedule\u002Findex","how-to","8LlA9kcQ5GisOUAUXw5GLnGMuhyetAUbUuirB3h6T70",[2250,2254],{"title":2251,"path":2252,"stem":2253,"children":-1},"Load an Excel File into a SQL Database with pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-an-excel-file-into-a-sql-database-with-pandas","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-an-excel-file-into-a-sql-database-with-pandas\u002Findex",{"title":2255,"path":2256,"stem":2257,"children":-1},"Validating Excel Data with Python","\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python","advanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Findex",1786800027161]