[{"data":1,"prerenderedAt":3522},["ShallowReactive",2],{"doc:\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage":3,"surround:\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage":3514},{"id":4,"title":5,"body":6,"dateModified":3491,"datePublished":3491,"description":3492,"extension":3493,"faq":3494,"meta":3505,"navigation":223,"path":3506,"seo":3507,"slug":3510,"stem":3511,"type":3512,"__hash__":3513},"docs\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Findex.md","Publishing Excel Reports to Cloud Storage",{"type":7,"value":8,"toc":3477},"minimark",[9,19,167,172,175,178,181,381,389,393,396,399,489,695,706,709,713,716,719,939,942,1036,1051,1055,1058,1370,1377,1380,1384,1387,1452,1455,1564,1571,1575,1578,1880,1890,1894,1897,2033,2592,2599,2602,2606,2609,2662,2669,2798,2802,2805,3028,3031,3034,3324,3332,3336,3386,3390,3396,3402,3408,3414,3420,3424,3473],[10,11,12,13,18],"p",{},"A report that nobody can find has not been delivered. Email attachments were the default answer for years, and they are a poor one: they duplicate the file into every mailbox, bounce off size limits, and leave five people holding four different versions. Publishing to shared storage fixes all of that — one canonical copy, a stable link, an access trail, and version history for free. This page covers the four destinations that account for nearly all real reporting jobs, and the transfer discipline that separates a reliable publish step from one that occasionally leaves a truncated file where the report should be. It is the delivery stage of ",[14,15,17],"a",{"href":16},"\u002Fautomating-reporting-workflows\u002F","Automating Reporting Workflows",".",[20,21,30,31,30,35,30,39,30,46,30,56,30,63,30,68,30,73,30,92,30,108,30,117,30,123,30,128,30,133,30,138,30,142,30,145,30,150,30,154,30,158,30,163],"svg",{"viewBox":22,"role":23,"ariaLabel":24,"ariaLabelledBy":25,"xmlns":28,"style":29},"0 0 800 268","img","One generated report fanning out to four destinations: object storage, SharePoint or OneDrive, Google Drive, and a network share, each reached through its own client library.",[26,27],"pubmap-t","pubmap-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:800px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[32,33,34],"title",{"id":26},"Four places a finished report can be published, and how each is reached",[36,37,38],"desc",{"id":27},"A single generated xlsx file branches to four destinations. Amazon S3 and compatible object stores are reached with boto3. SharePoint and OneDrive are reached through the Microsoft Graph REST API. Google Drive uses the Drive API with a service account. A network share uses SMB or a mounted path. Each branch notes the authentication a scheduled job should use: an IAM role, an app registration, a service account, or a machine account.",[40,41],"rect",{"x":42,"y":42,"width":43,"height":44,"fill":45},"0","800","268","#ffffff",[40,47],{"x":48,"y":49,"width":50,"height":51,"rx":52,"fill":53,"stroke":54,"style":55},"14","106","150","70","13","#ebebfd","var(--brand,#5b5cf0)","stroke-width:2px",[57,58,62],"text",{"x":59,"y":60,"style":61},"89","90","font-size:11px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","the job produces",[57,64,67],{"x":59,"y":65,"style":66},"136","font-size:12.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","report.xlsx",[57,69,72],{"x":59,"y":70,"style":71},"157","font-size:10.5px;fill:var(--muted,#5b6780);text-anchor:middle","verified, then published",[74,75,78,79,78,83,78,86,78,89,30],"g",{"stroke":76,"style":55,"fill":77},"var(--line,#cdd5e6)","none","\n    ",[80,81],"path",{"d":82},"M164 128 H 200 V 44 H 236",[80,84],{"d":85},"M164 136 H 200 V 106 H 236",[80,87],{"d":88},"M164 146 H 200 V 168 H 236",[80,90],{"d":91},"M164 154 H 200 V 230 H 236",[74,93,78,95,78,99,78,102,78,105,30],{"fill":94},"#5b5cf0",[96,97],"polygon",{"points":98},"244,44 232,38 232,50",[96,100],{"points":101},"244,106 232,100 232,112",[96,103],{"points":104},"244,168 232,162 232,174",[96,106],{"points":107},"244,230 232,224 232,236",[40,109],{"x":110,"y":111,"width":112,"height":113,"rx":114,"fill":115,"stroke":116,"style":55},"252","20","532","48","11","#fdefd8","var(--gold,#b4740a)",[57,118,122],{"x":119,"y":120,"style":121},"278","40","font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06)","S3 \u002F object storage",[57,124,127],{"x":119,"y":125,"style":126},"58","font-size:10.5px;fill:var(--muted,#5b6780)","boto3 · upload_fileobj · authenticate with an IAM role",[40,129],{"x":110,"y":130,"width":112,"height":113,"rx":114,"fill":131,"stroke":132,"style":55},"82","#d9f4f1","var(--teal,#0f9488)",[57,134,137],{"x":119,"y":135,"style":136},"102","font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157)","SharePoint \u002F OneDrive",[57,139,141],{"x":119,"y":140,"style":126},"120","Microsoft Graph REST · app registration, client credentials",[40,143],{"x":110,"y":144,"width":112,"height":113,"rx":114,"fill":53,"stroke":54,"style":55},"144",[57,146,149],{"x":119,"y":147,"style":148},"164","font-size:12px;font-weight:700;fill:var(--brand-strong,#4338ca)","Google Drive",[57,151,153],{"x":119,"y":152,"style":126},"182","Drive API v3 · service account with folder access",[40,155],{"x":110,"y":156,"width":112,"height":113,"rx":114,"fill":157,"stroke":76,"style":55},"206","#f0f2f5",[57,159,162],{"x":119,"y":160,"style":161},"226","font-size:12px;font-weight:700;fill:var(--text,#172033)","network share",[57,164,166],{"x":119,"y":165,"style":126},"244","SMB or a mounted path · machine account, write-then-rename",[168,169,171],"h2",{"id":170},"publish-then-link-not-attach","Publish, then link — not attach",[10,173,174],{},"The decision to make first is not which store, but whether to send the file at all. Attachments have four failure modes that a link does not: size limits reject the message, every recipient holds a private copy that immediately diverges, forwarding puts the file somewhere with no access control, and nobody can tell whose version is current.",[10,176,177],{},"Publishing inverts all of that. One copy exists, the link always resolves to it, and the store records who opened it. Access follows the organisation's existing permissions rather than the accident of who was on the distribution list, so somebody joining the team gets the report by being added to a group, and somebody leaving loses it the same way. Version history comes free in every store worth using, which turns \"what did last month's figures say before the restatement?\" from an archaeology exercise into a click. And because the file is no longer duplicated into a dozen mailboxes, the storage cost stops multiplying by the size of the audience.",[10,179,180],{},"The email becomes a short notification rather than a payload:",[182,183,188],"pre",{"className":184,"code":185,"language":186,"meta":187,"style":187},"language-python shiki shiki-themes github-light github-dark-high-contrast","from email.message import EmailMessage\nimport smtplib\n\ndef notify(recipients, url, subject):\n    msg = EmailMessage()\n    msg[\"From\"] = \"reporting@example.com\"\n    msg[\"To\"] = \", \".join(recipients)\n    msg[\"Subject\"] = subject\n    msg.set_content(\n        f\"The August regional report is published.\\n\\n{url}\\n\\n\"\n        \"This link always points at the current version.\"\n    )\n    with smtplib.SMTP(\"smtp.example.com\", 587) as smtp:\n        smtp.starttls()\n        smtp.send_message(msg)\n","python","",[189,190,191,210,218,225,238,250,268,286,301,307,329,335,341,369,375],"code",{"__ignoreMap":187},[192,193,196,200,204,207],"span",{"class":194,"line":195},"line",1,[192,197,199],{"class":198},"s-kum","from",[192,201,203],{"class":202},"skGVy"," email.message ",[192,205,206],{"class":198},"import",[192,208,209],{"class":202}," EmailMessage\n",[192,211,213,215],{"class":194,"line":212},2,[192,214,206],{"class":198},[192,216,217],{"class":202}," smtplib\n",[192,219,221],{"class":194,"line":220},3,[192,222,224],{"emptyLinePlaceholder":223},true,"\n",[192,226,228,231,235],{"class":194,"line":227},4,[192,229,230],{"class":198},"def",[192,232,234],{"class":233},"s_Opv"," notify",[192,236,237],{"class":202},"(recipients, url, subject):\n",[192,239,241,244,247],{"class":194,"line":240},5,[192,242,243],{"class":202},"    msg ",[192,245,246],{"class":198},"=",[192,248,249],{"class":202}," EmailMessage()\n",[192,251,253,256,260,263,265],{"class":194,"line":252},6,[192,254,255],{"class":202},"    msg[",[192,257,259],{"class":258},"srMev","\"From\"",[192,261,262],{"class":202},"] ",[192,264,246],{"class":198},[192,266,267],{"class":258}," \"reporting@example.com\"\n",[192,269,271,273,276,278,280,283],{"class":194,"line":270},7,[192,272,255],{"class":202},[192,274,275],{"class":258},"\"To\"",[192,277,262],{"class":202},[192,279,246],{"class":198},[192,281,282],{"class":258}," \", \"",[192,284,285],{"class":202},".join(recipients)\n",[192,287,289,291,294,296,298],{"class":194,"line":288},8,[192,290,255],{"class":202},[192,292,293],{"class":258},"\"Subject\"",[192,295,262],{"class":202},[192,297,246],{"class":198},[192,299,300],{"class":202}," subject\n",[192,302,304],{"class":194,"line":303},9,[192,305,306],{"class":202},"    msg.set_content(\n",[192,308,310,313,316,320,323,326],{"class":194,"line":309},10,[192,311,312],{"class":198},"        f",[192,314,315],{"class":258},"\"The August regional report is published.",[192,317,319],{"class":318},"sSjpA","\\n\\n{",[192,321,322],{"class":202},"url",[192,324,325],{"class":318},"}\\n\\n",[192,327,328],{"class":258},"\"\n",[192,330,332],{"class":194,"line":331},11,[192,333,334],{"class":258},"        \"This link always points at the current version.\"\n",[192,336,338],{"class":194,"line":337},12,[192,339,340],{"class":202},"    )\n",[192,342,344,347,350,353,356,360,363,366],{"class":194,"line":343},13,[192,345,346],{"class":198},"    with",[192,348,349],{"class":202}," smtplib.SMTP(",[192,351,352],{"class":258},"\"smtp.example.com\"",[192,354,355],{"class":202},", ",[192,357,359],{"class":358},"sP0c6","587",[192,361,362],{"class":202},") ",[192,364,365],{"class":198},"as",[192,367,368],{"class":202}," smtp:\n",[192,370,372],{"class":194,"line":371},14,[192,373,374],{"class":202},"        smtp.starttls()\n",[192,376,378],{"class":194,"line":377},15,[192,379,380],{"class":202},"        smtp.send_message(msg)\n",[10,382,383,384,388],{},"The attachment path still has its place — an external recipient with no access to your storage, or a regulator who wants a fixed copy — and that mechanics is covered in ",[14,385,387],{"href":386},"\u002Fautomating-reporting-workflows\u002Femailing-excel-reports-with-smtplib\u002F","emailing Excel reports with smtplib",". But for internal recurring reports, publish and link.",[168,390,392],{"id":391},"name-objects-so-both-history-and-links-work","Name objects so both history and links work",[10,394,395],{},"The naming decision comes next, and getting it wrong is expensive to undo once links are circulating. Two requirements pull in opposite directions: history wants a new name every run, links want a name that never changes.",[10,397,398],{},"Satisfy both. Write a dated object as the record, then update a stable pointer that links and dashboards reference:",[20,400,30,406,30,409,30,412,30,415,30,421,30,426,30,431,30,433,30,437,30,440,30,445,30,449,30,452,30,456,30,462,30,468,30,474,30,478,30,480,30,485],{"viewBox":401,"role":23,"ariaLabel":402,"ariaLabelledBy":403,"xmlns":28,"style":29},"0 0 800 236","Dual naming scheme: each run writes a dated immutable object and then copies it over a stable latest key, so history is preserved while circulated links stay valid.",[404,405],"naming-t","naming-d",[32,407,408],{"id":404},"Dated objects for history, one stable key for links",[36,410,411],{"id":405},"Three monthly runs each write a dated object under a year and month prefix — June, July and August — which are never overwritten and form the audit history. After each write, the same content is copied over a single latest key. Circulated links point only at the latest key, so they never go stale, while every prior version remains retrievable at its dated path.",[40,413],{"x":42,"y":42,"width":43,"height":414,"fill":45},"236",[57,416,420],{"x":417,"y":418,"style":419},"220","26","font-size:11.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","dated objects — never overwritten",[40,422],{"x":111,"y":120,"width":423,"height":424,"rx":425,"fill":53,"stroke":54,"style":55},"400","34","7",[57,427,430],{"x":417,"y":428,"style":429},"62","font-size:11px;fill:var(--text,#172033);text-anchor:middle","reports\u002F2026\u002F06\u002Fregional-2026-06.xlsx",[40,432],{"x":111,"y":130,"width":423,"height":424,"rx":425,"fill":53,"stroke":54,"style":55},[57,434,436],{"x":417,"y":435,"style":429},"104","reports\u002F2026\u002F07\u002Fregional-2026-07.xlsx",[40,438],{"x":111,"y":439,"width":423,"height":424,"rx":425,"fill":53,"stroke":54,"style":55},"124",[57,441,444],{"x":417,"y":442,"style":443},"146","font-size:11px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","reports\u002F2026\u002F08\u002Fregional-2026-08.xlsx",[57,446,448],{"x":417,"y":447,"style":71},"188","the audit trail: every month still retrievable",[80,450],{"d":451,"fill":77,"stroke":132,"style":55},"M420 141 H 470 V 96 H 512",[96,453],{"points":454,"fill":455},"520,96 508,90 508,102","#0f9488",[57,457,461],{"x":458,"y":459,"style":460},"470","128","font-size:10px;fill:var(--muted,#5b6780);text-anchor:middle","copy",[40,463],{"x":464,"y":465,"width":110,"height":466,"rx":467,"fill":131,"stroke":132,"style":55},"528","60","72","12",[57,469,473],{"x":470,"y":471,"style":472},"654","86","font-size:11.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","reports\u002Fregional-latest.xlsx",[57,475,477],{"x":470,"y":476,"style":71},"110","one key, always current",[40,479],{"x":464,"y":50,"width":110,"height":465,"rx":467,"fill":115,"stroke":116,"style":55},[57,481,484],{"x":470,"y":482,"style":483},"174","font-size:11.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","every circulated link",[57,486,488],{"x":470,"y":487,"style":71},"195","points here — never goes stale",[182,490,492],{"className":184,"code":491,"language":186,"meta":187,"style":187},"from datetime import date\n\ndef report_keys(name, on=None, prefix=\"reports\"):\n    \"\"\"Return the dated key and the stable latest key for a report.\"\"\"\n    on = on or date.today()\n    dated = f\"{prefix}\u002F{on:%Y\u002F%m}\u002F{name}-{on:%Y-%m}.xlsx\"\n    latest = f\"{prefix}\u002F{name}-latest.xlsx\"\n    return dated, latest\n\nprint(report_keys(\"regional\"))\n# ('reports\u002F2026\u002F08\u002Fregional-2026-08.xlsx', 'reports\u002Fregional-latest.xlsx')\n",[189,493,494,506,510,536,541,557,635,663,671,675,689],{"__ignoreMap":187},[192,495,496,498,501,503],{"class":194,"line":195},[192,497,199],{"class":198},[192,499,500],{"class":202}," datetime ",[192,502,206],{"class":198},[192,504,505],{"class":202}," date\n",[192,507,508],{"class":194,"line":212},[192,509,224],{"emptyLinePlaceholder":223},[192,511,512,514,517,520,522,525,528,530,533],{"class":194,"line":220},[192,513,230],{"class":198},[192,515,516],{"class":233}," report_keys",[192,518,519],{"class":202},"(name, on",[192,521,246],{"class":198},[192,523,524],{"class":358},"None",[192,526,527],{"class":202},", prefix",[192,529,246],{"class":198},[192,531,532],{"class":258},"\"reports\"",[192,534,535],{"class":202},"):\n",[192,537,538],{"class":194,"line":227},[192,539,540],{"class":258},"    \"\"\"Return the dated key and the stable latest key for a report.\"\"\"\n",[192,542,543,546,548,551,554],{"class":194,"line":240},[192,544,545],{"class":202},"    on ",[192,547,246],{"class":198},[192,549,550],{"class":202}," on ",[192,552,553],{"class":198},"or",[192,555,556],{"class":202}," date.today()\n",[192,558,559,562,564,567,570,573,576,579,582,584,587,590,593,596,599,602,604,606,608,611,613,616,618,621,623,625,628,630,632],{"class":194,"line":252},[192,560,561],{"class":202},"    dated ",[192,563,246],{"class":198},[192,565,566],{"class":198}," f",[192,568,569],{"class":258},"\"",[192,571,572],{"class":318},"{",[192,574,575],{"class":202},"prefix",[192,577,578],{"class":318},"}",[192,580,581],{"class":258},"\u002F",[192,583,572],{"class":318},[192,585,586],{"class":202},"on",[192,588,589],{"class":198},":",[192,591,592],{"class":202},"%",[192,594,595],{"class":198},"Y",[192,597,598],{"class":202},"\u002F%",[192,600,601],{"class":198},"m",[192,603,578],{"class":318},[192,605,581],{"class":258},[192,607,572],{"class":318},[192,609,610],{"class":202},"name",[192,612,578],{"class":318},[192,614,615],{"class":258},"-",[192,617,572],{"class":318},[192,619,620],{"class":202},"on:",[192,622,592],{"class":198},[192,624,595],{"class":202},[192,626,627],{"class":198},"-%",[192,629,601],{"class":202},[192,631,578],{"class":318},[192,633,634],{"class":258},".xlsx\"\n",[192,636,637,640,642,644,646,648,650,652,654,656,658,660],{"class":194,"line":270},[192,638,639],{"class":202},"    latest ",[192,641,246],{"class":198},[192,643,566],{"class":198},[192,645,569],{"class":258},[192,647,572],{"class":318},[192,649,575],{"class":202},[192,651,578],{"class":318},[192,653,581],{"class":258},[192,655,572],{"class":318},[192,657,610],{"class":202},[192,659,578],{"class":318},[192,661,662],{"class":258},"-latest.xlsx\"\n",[192,664,665,668],{"class":194,"line":288},[192,666,667],{"class":198},"    return",[192,669,670],{"class":202}," dated, latest\n",[192,672,673],{"class":194,"line":303},[192,674,224],{"emptyLinePlaceholder":223},[192,676,677,680,683,686],{"class":194,"line":309},[192,678,679],{"class":358},"print",[192,681,682],{"class":202},"(report_keys(",[192,684,685],{"class":258},"\"regional\"",[192,687,688],{"class":202},"))\n",[192,690,691],{"class":194,"line":331},[192,692,694],{"class":693},"s-wDw","# ('reports\u002F2026\u002F08\u002Fregional-2026-08.xlsx', 'reports\u002Fregional-latest.xlsx')\n",[10,696,697,698,701,702,705],{},"Use ISO dates in the name — ",[189,699,700],{},"2026-08"," sorts correctly in every listing, where ",[189,703,704],{},"Aug-2026"," does not. Avoid spaces and non-ASCII characters, which survive most stores but produce awkward URLs and break the occasional downstream tool.",[10,707,708],{},"The prefix structure deserves the same care as the filename. Grouping by year and month, as above, keeps a listing usable after a few years of monthly runs, and on object storage it also gives you a natural boundary for lifecycle rules — expire or archive everything under an old year prefix in one policy rather than matching on names. Where several report types share a bucket, put the report name before the date rather than after it, so all of one report's history sits together instead of interleaving with everything else that ran the same month.",[168,710,712],{"id":711},"upload-atomically","Upload atomically",[10,714,715],{},"The failure that costs the most credibility is a reader opening a report mid-upload and finding a truncated file. It looks like a corrupt report, and it happens whenever a large file is written directly to its final name over a slow link.",[10,717,718],{},"The fix is universal: write somewhere temporary, then move it into place as a single operation.",[182,720,722],{"className":184,"code":721,"language":186,"meta":187,"style":187},"import boto3\n\ns3 = boto3.client(\"s3\")\n\ndef publish_atomic(local_path, bucket, key):\n    \"\"\"Upload to a staging key, then copy into place so readers never see a partial file.\"\"\"\n    staging = f\"{key}.uploading\"\n\n    with open(local_path, \"rb\") as fh:\n        s3.upload_fileobj(fh, bucket, staging)\n\n    s3.copy_object(\n        Bucket=bucket,\n        Key=key,\n        CopySource={\"Bucket\": bucket, \"Key\": staging},\n        MetadataDirective=\"COPY\",\n    )\n    s3.delete_object(Bucket=bucket, Key=staging)\n    return f\"s3:\u002F\u002F{bucket}\u002F{key}\"\n",[189,723,724,731,735,751,755,765,770,791,795,815,820,824,829,840,850,871,885,890,912],{"__ignoreMap":187},[192,725,726,728],{"class":194,"line":195},[192,727,206],{"class":198},[192,729,730],{"class":202}," boto3\n",[192,732,733],{"class":194,"line":212},[192,734,224],{"emptyLinePlaceholder":223},[192,736,737,740,742,745,748],{"class":194,"line":220},[192,738,739],{"class":202},"s3 ",[192,741,246],{"class":198},[192,743,744],{"class":202}," boto3.client(",[192,746,747],{"class":258},"\"s3\"",[192,749,750],{"class":202},")\n",[192,752,753],{"class":194,"line":227},[192,754,224],{"emptyLinePlaceholder":223},[192,756,757,759,762],{"class":194,"line":240},[192,758,230],{"class":198},[192,760,761],{"class":233}," publish_atomic",[192,763,764],{"class":202},"(local_path, bucket, key):\n",[192,766,767],{"class":194,"line":252},[192,768,769],{"class":258},"    \"\"\"Upload to a staging key, then copy into place so readers never see a partial file.\"\"\"\n",[192,771,772,775,777,779,781,783,786,788],{"class":194,"line":270},[192,773,774],{"class":202},"    staging ",[192,776,246],{"class":198},[192,778,566],{"class":198},[192,780,569],{"class":258},[192,782,572],{"class":318},[192,784,785],{"class":202},"key",[192,787,578],{"class":318},[192,789,790],{"class":258},".uploading\"\n",[192,792,793],{"class":194,"line":288},[192,794,224],{"emptyLinePlaceholder":223},[192,796,797,799,802,805,808,810,812],{"class":194,"line":303},[192,798,346],{"class":198},[192,800,801],{"class":358}," open",[192,803,804],{"class":202},"(local_path, ",[192,806,807],{"class":258},"\"rb\"",[192,809,362],{"class":202},[192,811,365],{"class":198},[192,813,814],{"class":202}," fh:\n",[192,816,817],{"class":194,"line":309},[192,818,819],{"class":202},"        s3.upload_fileobj(fh, bucket, staging)\n",[192,821,822],{"class":194,"line":331},[192,823,224],{"emptyLinePlaceholder":223},[192,825,826],{"class":194,"line":337},[192,827,828],{"class":202},"    s3.copy_object(\n",[192,830,831,835,837],{"class":194,"line":343},[192,832,834],{"class":833},"sa561","        Bucket",[192,836,246],{"class":198},[192,838,839],{"class":202},"bucket,\n",[192,841,842,845,847],{"class":194,"line":371},[192,843,844],{"class":833},"        Key",[192,846,246],{"class":198},[192,848,849],{"class":202},"key,\n",[192,851,852,855,857,859,862,865,868],{"class":194,"line":377},[192,853,854],{"class":833},"        CopySource",[192,856,246],{"class":198},[192,858,572],{"class":202},[192,860,861],{"class":258},"\"Bucket\"",[192,863,864],{"class":202},": bucket, ",[192,866,867],{"class":258},"\"Key\"",[192,869,870],{"class":202},": staging},\n",[192,872,874,877,879,882],{"class":194,"line":873},16,[192,875,876],{"class":833},"        MetadataDirective",[192,878,246],{"class":198},[192,880,881],{"class":258},"\"COPY\"",[192,883,884],{"class":202},",\n",[192,886,888],{"class":194,"line":887},17,[192,889,340],{"class":202},[192,891,893,896,899,901,904,907,909],{"class":194,"line":892},18,[192,894,895],{"class":202},"    s3.delete_object(",[192,897,898],{"class":833},"Bucket",[192,900,246],{"class":198},[192,902,903],{"class":202},"bucket, ",[192,905,906],{"class":833},"Key",[192,908,246],{"class":198},[192,910,911],{"class":202},"staging)\n",[192,913,915,917,919,922,924,927,929,931,933,935,937],{"class":194,"line":914},19,[192,916,667],{"class":198},[192,918,566],{"class":198},[192,920,921],{"class":258},"\"s3:\u002F\u002F",[192,923,572],{"class":318},[192,925,926],{"class":202},"bucket",[192,928,578],{"class":318},[192,930,581],{"class":258},[192,932,572],{"class":318},[192,934,785],{"class":202},[192,936,578],{"class":318},[192,938,328],{"class":258},[10,940,941],{},"On a filesystem or network share the same idea uses a rename, which is atomic within a volume:",[182,943,945],{"className":184,"code":944,"language":186,"meta":187,"style":187},"from pathlib import Path\nimport shutil\n\ndef publish_to_share(local_path, share_dir, name):\n    dest = Path(share_dir) \u002F name\n    staging = dest.with_suffix(dest.suffix + \".part\")\n\n    shutil.copyfile(local_path, staging)\n    staging.replace(dest)          # atomic within the same filesystem\n    return dest\n",[189,946,947,959,966,970,980,995,1012,1016,1021,1029],{"__ignoreMap":187},[192,948,949,951,954,956],{"class":194,"line":195},[192,950,199],{"class":198},[192,952,953],{"class":202}," pathlib ",[192,955,206],{"class":198},[192,957,958],{"class":202}," Path\n",[192,960,961,963],{"class":194,"line":212},[192,962,206],{"class":198},[192,964,965],{"class":202}," shutil\n",[192,967,968],{"class":194,"line":220},[192,969,224],{"emptyLinePlaceholder":223},[192,971,972,974,977],{"class":194,"line":227},[192,973,230],{"class":198},[192,975,976],{"class":233}," publish_to_share",[192,978,979],{"class":202},"(local_path, share_dir, name):\n",[192,981,982,985,987,990,992],{"class":194,"line":240},[192,983,984],{"class":202},"    dest ",[192,986,246],{"class":198},[192,988,989],{"class":202}," Path(share_dir) ",[192,991,581],{"class":198},[192,993,994],{"class":202}," name\n",[192,996,997,999,1001,1004,1007,1010],{"class":194,"line":252},[192,998,774],{"class":202},[192,1000,246],{"class":198},[192,1002,1003],{"class":202}," dest.with_suffix(dest.suffix ",[192,1005,1006],{"class":198},"+",[192,1008,1009],{"class":258}," \".part\"",[192,1011,750],{"class":202},[192,1013,1014],{"class":194,"line":270},[192,1015,224],{"emptyLinePlaceholder":223},[192,1017,1018],{"class":194,"line":288},[192,1019,1020],{"class":202},"    shutil.copyfile(local_path, staging)\n",[192,1022,1023,1026],{"class":194,"line":303},[192,1024,1025],{"class":202},"    staging.replace(dest)          ",[192,1027,1028],{"class":693},"# atomic within the same filesystem\n",[192,1030,1031,1033],{"class":194,"line":309},[192,1032,667],{"class":198},[192,1034,1035],{"class":202}," dest\n",[10,1037,1038,1041,1042,1045,1046,1050],{},[189,1039,1040],{},"Path.replace"," is the important call — it overwrites atomically, where copying directly to ",[189,1043,1044],{},"dest"," leaves a growing partial file visible for the whole transfer. Note the constraint: the staging file must sit on the ",[1047,1048,1049],"em",{},"same"," volume as the destination, or the rename degrades into a copy and the guarantee is lost.",[168,1052,1054],{"id":1053},"verify-the-bytes-arrived","Verify the bytes arrived",[10,1056,1057],{},"An HTTP 200 means the request was accepted. It does not mean the file is intact. For anything that matters, compare a checksum:",[182,1059,1061],{"className":184,"code":1060,"language":186,"meta":187,"style":187},"import hashlib\n\ndef file_md5(path, chunk=1 \u003C\u003C 20):\n    digest = hashlib.md5()\n    with open(path, \"rb\") as fh:\n        for block in iter(lambda: fh.read(chunk), b\"\"):\n            digest.update(block)\n    return digest.hexdigest()\n\ndef verify_s3(local_path, bucket, key):\n    \"\"\"Confirm the object in S3 matches the local file.\"\"\"\n    head = s3.head_object(Bucket=bucket, Key=key)\n    remote_size = head[\"ContentLength\"]\n    local_size = Path(local_path).stat().st_size\n\n    if remote_size != local_size:\n        raise RuntimeError(\n            f\"size mismatch: local {local_size} vs remote {remote_size}\"\n        )\n\n    etag = head[\"ETag\"].strip('\"')\n    if \"-\" not in etag and etag != file_md5(local_path):\n        raise RuntimeError(\"checksum mismatch — the upload is corrupt\")\n    return True\n",[189,1062,1063,1070,1074,1097,1107,1124,1155,1160,1167,1171,1180,1185,1208,1224,1234,1238,1252,1263,1290,1295,1300,1321,1348,1362],{"__ignoreMap":187},[192,1064,1065,1067],{"class":194,"line":195},[192,1066,206],{"class":198},[192,1068,1069],{"class":202}," hashlib\n",[192,1071,1072],{"class":194,"line":212},[192,1073,224],{"emptyLinePlaceholder":223},[192,1075,1076,1078,1081,1084,1086,1089,1092,1095],{"class":194,"line":220},[192,1077,230],{"class":198},[192,1079,1080],{"class":233}," file_md5",[192,1082,1083],{"class":202},"(path, chunk",[192,1085,246],{"class":198},[192,1087,1088],{"class":358},"1",[192,1090,1091],{"class":198}," \u003C\u003C",[192,1093,1094],{"class":358}," 20",[192,1096,535],{"class":202},[192,1098,1099,1102,1104],{"class":194,"line":227},[192,1100,1101],{"class":202},"    digest ",[192,1103,246],{"class":198},[192,1105,1106],{"class":202}," hashlib.md5()\n",[192,1108,1109,1111,1113,1116,1118,1120,1122],{"class":194,"line":240},[192,1110,346],{"class":198},[192,1112,801],{"class":358},[192,1114,1115],{"class":202},"(path, ",[192,1117,807],{"class":258},[192,1119,362],{"class":202},[192,1121,365],{"class":198},[192,1123,814],{"class":202},[192,1125,1126,1129,1132,1135,1138,1141,1144,1147,1150,1153],{"class":194,"line":252},[192,1127,1128],{"class":198},"        for",[192,1130,1131],{"class":202}," block ",[192,1133,1134],{"class":198},"in",[192,1136,1137],{"class":358}," iter",[192,1139,1140],{"class":202},"(",[192,1142,1143],{"class":198},"lambda",[192,1145,1146],{"class":202},": fh.read(chunk), ",[192,1148,1149],{"class":198},"b",[192,1151,1152],{"class":258},"\"\"",[192,1154,535],{"class":202},[192,1156,1157],{"class":194,"line":270},[192,1158,1159],{"class":202},"            digest.update(block)\n",[192,1161,1162,1164],{"class":194,"line":288},[192,1163,667],{"class":198},[192,1165,1166],{"class":202}," digest.hexdigest()\n",[192,1168,1169],{"class":194,"line":303},[192,1170,224],{"emptyLinePlaceholder":223},[192,1172,1173,1175,1178],{"class":194,"line":309},[192,1174,230],{"class":198},[192,1176,1177],{"class":233}," verify_s3",[192,1179,764],{"class":202},[192,1181,1182],{"class":194,"line":331},[192,1183,1184],{"class":258},"    \"\"\"Confirm the object in S3 matches the local file.\"\"\"\n",[192,1186,1187,1190,1192,1195,1197,1199,1201,1203,1205],{"class":194,"line":337},[192,1188,1189],{"class":202},"    head ",[192,1191,246],{"class":198},[192,1193,1194],{"class":202}," s3.head_object(",[192,1196,898],{"class":833},[192,1198,246],{"class":198},[192,1200,903],{"class":202},[192,1202,906],{"class":833},[192,1204,246],{"class":198},[192,1206,1207],{"class":202},"key)\n",[192,1209,1210,1213,1215,1218,1221],{"class":194,"line":343},[192,1211,1212],{"class":202},"    remote_size ",[192,1214,246],{"class":198},[192,1216,1217],{"class":202}," head[",[192,1219,1220],{"class":258},"\"ContentLength\"",[192,1222,1223],{"class":202},"]\n",[192,1225,1226,1229,1231],{"class":194,"line":371},[192,1227,1228],{"class":202},"    local_size ",[192,1230,246],{"class":198},[192,1232,1233],{"class":202}," Path(local_path).stat().st_size\n",[192,1235,1236],{"class":194,"line":377},[192,1237,224],{"emptyLinePlaceholder":223},[192,1239,1240,1243,1246,1249],{"class":194,"line":873},[192,1241,1242],{"class":198},"    if",[192,1244,1245],{"class":202}," remote_size ",[192,1247,1248],{"class":198},"!=",[192,1250,1251],{"class":202}," local_size:\n",[192,1253,1254,1257,1260],{"class":194,"line":887},[192,1255,1256],{"class":198},"        raise",[192,1258,1259],{"class":358}," RuntimeError",[192,1261,1262],{"class":202},"(\n",[192,1264,1265,1268,1271,1273,1276,1278,1281,1283,1286,1288],{"class":194,"line":892},[192,1266,1267],{"class":198},"            f",[192,1269,1270],{"class":258},"\"size mismatch: local ",[192,1272,572],{"class":318},[192,1274,1275],{"class":202},"local_size",[192,1277,578],{"class":318},[192,1279,1280],{"class":258}," vs remote ",[192,1282,572],{"class":318},[192,1284,1285],{"class":202},"remote_size",[192,1287,578],{"class":318},[192,1289,328],{"class":258},[192,1291,1292],{"class":194,"line":914},[192,1293,1294],{"class":202},"        )\n",[192,1296,1298],{"class":194,"line":1297},20,[192,1299,224],{"emptyLinePlaceholder":223},[192,1301,1303,1306,1308,1310,1313,1316,1319],{"class":194,"line":1302},21,[192,1304,1305],{"class":202},"    etag ",[192,1307,246],{"class":198},[192,1309,1217],{"class":202},[192,1311,1312],{"class":258},"\"ETag\"",[192,1314,1315],{"class":202},"].strip(",[192,1317,1318],{"class":258},"'\"'",[192,1320,750],{"class":202},[192,1322,1324,1326,1329,1332,1335,1338,1341,1343,1345],{"class":194,"line":1323},22,[192,1325,1242],{"class":198},[192,1327,1328],{"class":258}," \"-\"",[192,1330,1331],{"class":198}," not",[192,1333,1334],{"class":198}," in",[192,1336,1337],{"class":202}," etag ",[192,1339,1340],{"class":198},"and",[192,1342,1337],{"class":202},[192,1344,1248],{"class":198},[192,1346,1347],{"class":202}," file_md5(local_path):\n",[192,1349,1351,1353,1355,1357,1360],{"class":194,"line":1350},23,[192,1352,1256],{"class":198},[192,1354,1259],{"class":358},[192,1356,1140],{"class":202},[192,1358,1359],{"class":258},"\"checksum mismatch — the upload is corrupt\"",[192,1361,750],{"class":202},[192,1363,1365,1367],{"class":194,"line":1364},24,[192,1366,667],{"class":198},[192,1368,1369],{"class":358}," True\n",[10,1371,1372,1373,1376],{},"The ",[189,1374,1375],{},"\"-\" not in etag"," guard matters: for multipart uploads S3's ETag is a digest of digests with a part-count suffix, not the file's MD5, so comparing it directly gives a false failure on large objects. Fall back to size for those, or use the store's own checksum feature.",[10,1378,1379],{},"At minimum, check the size. It is one call and it catches the truncation case, which is by far the most common corruption in practice.",[168,1381,1383],{"id":1382},"authenticate-as-a-machine-not-a-person","Authenticate as a machine, not a person",[10,1385,1386],{},"Every one of these destinations supports both a personal login and a machine identity, and scheduled jobs should always use the latter. A job authenticating as a person breaks when they change their password, hits interactive multi-factor prompts, and stops entirely when they leave.",[1388,1389,1390,1406],"table",{},[1391,1392,1393],"thead",{},[1394,1395,1396,1400,1403],"tr",{},[1397,1398,1399],"th",{},"Destination",[1397,1401,1402],{},"Machine identity",[1397,1404,1405],{},"Credential in the job",[1407,1408,1409,1421,1431,1441],"tbody",{},[1394,1410,1411,1415,1418],{},[1412,1413,1414],"td",{},"S3",[1412,1416,1417],{},"IAM role",[1412,1419,1420],{},"none — the role is assumed from the environment",[1394,1422,1423,1425,1428],{},[1412,1424,137],{},[1412,1426,1427],{},"app registration",[1412,1429,1430],{},"client ID + secret, or a certificate",[1394,1432,1433,1435,1438],{},[1412,1434,149],{},[1412,1436,1437],{},"service account",[1412,1439,1440],{},"a JSON key file",[1394,1442,1443,1446,1449],{},[1412,1444,1445],{},"Network share",[1412,1447,1448],{},"machine or service account",[1412,1450,1451],{},"a keytab, or an environment variable",[10,1453,1454],{},"Read secrets from the environment or a secrets manager, never from source:",[182,1456,1458],{"className":184,"code":1457,"language":186,"meta":187,"style":187},"import os\n\ndef required(name):\n    value = os.environ.get(name)\n    if not value:\n        raise RuntimeError(f\"{name} is not set — the publish step cannot run\")\n    return value\n\nclient_id = required(\"GRAPH_CLIENT_ID\")\nclient_secret = required(\"GRAPH_CLIENT_SECRET\")\n",[189,1459,1460,1467,1471,1481,1491,1500,1524,1531,1535,1550],{"__ignoreMap":187},[192,1461,1462,1464],{"class":194,"line":195},[192,1463,206],{"class":198},[192,1465,1466],{"class":202}," os\n",[192,1468,1469],{"class":194,"line":212},[192,1470,224],{"emptyLinePlaceholder":223},[192,1472,1473,1475,1478],{"class":194,"line":220},[192,1474,230],{"class":198},[192,1476,1477],{"class":233}," required",[192,1479,1480],{"class":202},"(name):\n",[192,1482,1483,1486,1488],{"class":194,"line":227},[192,1484,1485],{"class":202},"    value ",[192,1487,246],{"class":198},[192,1489,1490],{"class":202}," os.environ.get(name)\n",[192,1492,1493,1495,1497],{"class":194,"line":240},[192,1494,1242],{"class":198},[192,1496,1331],{"class":198},[192,1498,1499],{"class":202}," value:\n",[192,1501,1502,1504,1506,1508,1511,1513,1515,1517,1519,1522],{"class":194,"line":252},[192,1503,1256],{"class":198},[192,1505,1259],{"class":358},[192,1507,1140],{"class":202},[192,1509,1510],{"class":198},"f",[192,1512,569],{"class":258},[192,1514,572],{"class":318},[192,1516,610],{"class":202},[192,1518,578],{"class":318},[192,1520,1521],{"class":258}," is not set — the publish step cannot run\"",[192,1523,750],{"class":202},[192,1525,1526,1528],{"class":194,"line":270},[192,1527,667],{"class":198},[192,1529,1530],{"class":202}," value\n",[192,1532,1533],{"class":194,"line":288},[192,1534,224],{"emptyLinePlaceholder":223},[192,1536,1537,1540,1542,1545,1548],{"class":194,"line":303},[192,1538,1539],{"class":202},"client_id ",[192,1541,246],{"class":198},[192,1543,1544],{"class":202}," required(",[192,1546,1547],{"class":258},"\"GRAPH_CLIENT_ID\"",[192,1549,750],{"class":202},[192,1551,1552,1555,1557,1559,1562],{"class":194,"line":309},[192,1553,1554],{"class":202},"client_secret ",[192,1556,246],{"class":198},[192,1558,1544],{"class":202},[192,1560,1561],{"class":258},"\"GRAPH_CLIENT_SECRET\"",[192,1563,750],{"class":202},[10,1565,1566,1567,18],{},"Checking at start-up rather than at the point of use means the job fails immediately with a clear message, instead of thirty minutes later after building a report it cannot deliver — the same fail-fast principle applied throughout ",[14,1568,1570],{"href":1569},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002F","error handling and logging in Excel automation",[168,1572,1574],{"id":1573},"retry-the-transient-not-the-permanent","Retry the transient, not the permanent",[10,1576,1577],{},"Network transfers fail. Most failures are worth retrying and some are guaranteed not to be, and treating them the same way either gives up too early or hammers a service that will never say yes.",[182,1579,1581],{"className":184,"code":1580,"language":186,"meta":187,"style":187},"import time\nimport random\nfrom botocore.exceptions import ClientError\n\nRETRYABLE = {\"500\", \"502\", \"503\", \"504\", \"SlowDown\", \"RequestTimeout\"}\n\ndef upload_with_retry(local_path, bucket, key, attempts=5):\n    for attempt in range(1, attempts + 1):\n        try:\n            return publish_atomic(local_path, bucket, key)\n        except ClientError as exc:\n            code = exc.response[\"Error\"][\"Code\"]\n            if code not in RETRYABLE or attempt == attempts:\n                raise\n            # Exponential backoff with jitter, so parallel jobs do not sync up.\n            delay = min(2 ** attempt, 30) + random.uniform(0, 1)\n            print(f\"attempt {attempt} failed ({code}); retrying in {delay:.1f}s\")\n            time.sleep(delay)\n",[189,1582,1583,1590,1597,1609,1613,1655,1659,1676,1703,1711,1719,1732,1753,1780,1785,1790,1829,1875],{"__ignoreMap":187},[192,1584,1585,1587],{"class":194,"line":195},[192,1586,206],{"class":198},[192,1588,1589],{"class":202}," time\n",[192,1591,1592,1594],{"class":194,"line":212},[192,1593,206],{"class":198},[192,1595,1596],{"class":202}," random\n",[192,1598,1599,1601,1604,1606],{"class":194,"line":220},[192,1600,199],{"class":198},[192,1602,1603],{"class":202}," botocore.exceptions ",[192,1605,206],{"class":198},[192,1607,1608],{"class":202}," ClientError\n",[192,1610,1611],{"class":194,"line":227},[192,1612,224],{"emptyLinePlaceholder":223},[192,1614,1615,1618,1621,1624,1627,1629,1632,1634,1637,1639,1642,1644,1647,1649,1652],{"class":194,"line":240},[192,1616,1617],{"class":358},"RETRYABLE",[192,1619,1620],{"class":198}," =",[192,1622,1623],{"class":202}," {",[192,1625,1626],{"class":258},"\"500\"",[192,1628,355],{"class":202},[192,1630,1631],{"class":258},"\"502\"",[192,1633,355],{"class":202},[192,1635,1636],{"class":258},"\"503\"",[192,1638,355],{"class":202},[192,1640,1641],{"class":258},"\"504\"",[192,1643,355],{"class":202},[192,1645,1646],{"class":258},"\"SlowDown\"",[192,1648,355],{"class":202},[192,1650,1651],{"class":258},"\"RequestTimeout\"",[192,1653,1654],{"class":202},"}\n",[192,1656,1657],{"class":194,"line":252},[192,1658,224],{"emptyLinePlaceholder":223},[192,1660,1661,1663,1666,1669,1671,1674],{"class":194,"line":270},[192,1662,230],{"class":198},[192,1664,1665],{"class":233}," upload_with_retry",[192,1667,1668],{"class":202},"(local_path, bucket, key, attempts",[192,1670,246],{"class":198},[192,1672,1673],{"class":358},"5",[192,1675,535],{"class":202},[192,1677,1678,1681,1684,1686,1689,1691,1693,1696,1698,1701],{"class":194,"line":288},[192,1679,1680],{"class":198},"    for",[192,1682,1683],{"class":202}," attempt ",[192,1685,1134],{"class":198},[192,1687,1688],{"class":358}," range",[192,1690,1140],{"class":202},[192,1692,1088],{"class":358},[192,1694,1695],{"class":202},", attempts ",[192,1697,1006],{"class":198},[192,1699,1700],{"class":358}," 1",[192,1702,535],{"class":202},[192,1704,1705,1708],{"class":194,"line":303},[192,1706,1707],{"class":198},"        try",[192,1709,1710],{"class":202},":\n",[192,1712,1713,1716],{"class":194,"line":309},[192,1714,1715],{"class":198},"            return",[192,1717,1718],{"class":202}," publish_atomic(local_path, bucket, key)\n",[192,1720,1721,1724,1727,1729],{"class":194,"line":331},[192,1722,1723],{"class":198},"        except",[192,1725,1726],{"class":202}," ClientError ",[192,1728,365],{"class":198},[192,1730,1731],{"class":202}," exc:\n",[192,1733,1734,1737,1739,1742,1745,1748,1751],{"class":194,"line":337},[192,1735,1736],{"class":202},"            code ",[192,1738,246],{"class":198},[192,1740,1741],{"class":202}," exc.response[",[192,1743,1744],{"class":258},"\"Error\"",[192,1746,1747],{"class":202},"][",[192,1749,1750],{"class":258},"\"Code\"",[192,1752,1223],{"class":202},[192,1754,1755,1758,1761,1764,1766,1769,1772,1774,1777],{"class":194,"line":343},[192,1756,1757],{"class":198},"            if",[192,1759,1760],{"class":202}," code ",[192,1762,1763],{"class":198},"not",[192,1765,1334],{"class":198},[192,1767,1768],{"class":358}," RETRYABLE",[192,1770,1771],{"class":198}," or",[192,1773,1683],{"class":202},[192,1775,1776],{"class":198},"==",[192,1778,1779],{"class":202}," attempts:\n",[192,1781,1782],{"class":194,"line":371},[192,1783,1784],{"class":198},"                raise\n",[192,1786,1787],{"class":194,"line":377},[192,1788,1789],{"class":693},"            # Exponential backoff with jitter, so parallel jobs do not sync up.\n",[192,1791,1792,1795,1797,1800,1802,1805,1808,1811,1814,1816,1818,1821,1823,1825,1827],{"class":194,"line":873},[192,1793,1794],{"class":202},"            delay ",[192,1796,246],{"class":198},[192,1798,1799],{"class":358}," min",[192,1801,1140],{"class":202},[192,1803,1804],{"class":358},"2",[192,1806,1807],{"class":198}," **",[192,1809,1810],{"class":202}," attempt, ",[192,1812,1813],{"class":358},"30",[192,1815,362],{"class":202},[192,1817,1006],{"class":198},[192,1819,1820],{"class":202}," random.uniform(",[192,1822,42],{"class":358},[192,1824,355],{"class":202},[192,1826,1088],{"class":358},[192,1828,750],{"class":202},[192,1830,1831,1834,1836,1838,1841,1843,1846,1848,1851,1853,1855,1857,1860,1862,1865,1868,1870,1873],{"class":194,"line":887},[192,1832,1833],{"class":358},"            print",[192,1835,1140],{"class":202},[192,1837,1510],{"class":198},[192,1839,1840],{"class":258},"\"attempt ",[192,1842,572],{"class":318},[192,1844,1845],{"class":202},"attempt",[192,1847,578],{"class":318},[192,1849,1850],{"class":258}," failed (",[192,1852,572],{"class":318},[192,1854,189],{"class":202},[192,1856,578],{"class":318},[192,1858,1859],{"class":258},"); retrying in ",[192,1861,572],{"class":318},[192,1863,1864],{"class":202},"delay",[192,1866,1867],{"class":198},":.1f",[192,1869,578],{"class":318},[192,1871,1872],{"class":258},"s\"",[192,1874,750],{"class":202},[192,1876,1877],{"class":194,"line":892},[192,1878,1879],{"class":202},"            time.sleep(delay)\n",[10,1881,1882,1883,1886,1887,1889],{},"Never retry an authentication or permission failure — ",[189,1884,1885],{},"403"," will be ",[189,1888,1885],{}," again, and a retry loop only delays the alert. The jitter matters more than it looks: without it, several jobs that failed together retry together, which is exactly the pattern that keeps a struggling service down.",[168,1891,1893],{"id":1892},"verify-the-report-before-you-publish-it","Verify the report before you publish it",[10,1895,1896],{},"Publishing is the point of no return. Once a link is circulated, a wrong number has been read, and pulling it back costs far more than the check that would have caught it. So the publish step should be a gate, not just a transfer.",[20,1898,30,1904,30,1907,30,1910,30,1913,30,1917,30,1922,30,1926,30,1931,30,1934,30,1939,30,1944,30,1949,30,1953,30,1957,30,1961,30,1964,30,1968,30,1971,30,1974,30,1978,30,1981,30,1984,30,1988,30,1991,30,1994,30,1999,30,2002,30,2008,30,2014,30,2018,30,2022,30,2028],{"viewBox":1899,"role":23,"ariaLabel":1900,"ariaLabelledBy":1901,"xmlns":28,"style":29},"0 56 800 188","Publish gate: the generated workbook passes structural, content and freshness checks before any upload happens, and a failure routes to an alert instead of to storage.",[1902,1903],"gate-t","gate-d",[32,1905,1906],{"id":1902},"The checks that stand between generation and publication",[36,1908,1909],{"id":1903},"A generated workbook passes three gates before any upload. The structural gate confirms the file opens and has the expected sheets. The content gate confirms row counts and totals are within plausible bounds. The freshness gate confirms the data covers the period the report claims to cover. Passing all three leads to the upload; failing any one raises an alert and leaves the previous published report in place, which is almost always better than replacing it with a wrong one.",[40,1911],{"x":42,"y":1912,"width":43,"height":447,"fill":45},"56",[40,1914],{"x":48,"y":1915,"width":459,"height":1916,"rx":467,"fill":53,"stroke":54,"style":55},"88","66",[57,1918,67],{"x":1919,"y":1920,"style":1921},"78","118","font-size:11.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle",[57,1923,1925],{"x":1919,"y":1924,"style":71},"138","just generated",[194,1927],{"x1":1928,"y1":1929,"x2":1930,"y2":1929,"stroke":54,"style":55},"142","121","170",[96,1932],{"points":1933,"fill":94},"178,121 166,115 166,127",[40,1935],{"x":1936,"y":1937,"width":1938,"height":130,"rx":467,"fill":115,"stroke":116,"style":55},"186","80","140",[57,1940,1943],{"x":1941,"y":1942,"style":483},"256","108","structural",[57,1945,1948],{"x":1941,"y":1946,"style":1947},"130","font-size:10.5px;fill:var(--text,#172033);text-anchor:middle","opens, sheets",[57,1950,1952],{"x":1941,"y":1951,"style":1947},"148","present",[194,1954],{"x1":1955,"y1":1929,"x2":1956,"y2":1929,"stroke":116,"style":55},"326","350",[96,1958],{"points":1959,"fill":1960},"358,121 346,115 346,127","#b4740a",[40,1962],{"x":1963,"y":1937,"width":1938,"height":130,"rx":467,"fill":115,"stroke":116,"style":55},"366",[57,1965,1967],{"x":1966,"y":1942,"style":483},"436","content",[57,1969,1970],{"x":1966,"y":1946,"style":1947},"rows and totals",[57,1972,1973],{"x":1966,"y":1951,"style":1947},"within bounds",[194,1975],{"x1":1976,"y1":1929,"x2":1977,"y2":1929,"stroke":116,"style":55},"506","530",[96,1979],{"points":1980,"fill":1960},"538,121 526,115 526,127",[40,1982],{"x":1983,"y":1937,"width":1938,"height":130,"rx":467,"fill":115,"stroke":116,"style":55},"546",[57,1985,1987],{"x":1986,"y":1942,"style":483},"616","freshness",[57,1989,1990],{"x":1986,"y":1946,"style":1947},"covers the period",[57,1992,1993],{"x":1986,"y":1951,"style":1947},"it claims",[194,1995],{"x1":1996,"y1":1997,"x2":1998,"y2":1997,"stroke":132,"style":55},"686","100","712",[96,2000],{"points":2001,"fill":455},"720,100 708,94 708,106",[40,2003],{"x":2004,"y":2005,"width":2006,"height":113,"rx":2007,"fill":131,"stroke":132,"style":55},"716","76","68","10",[57,2009,2013],{"x":2010,"y":2011,"style":2012},"750","105","font-size:11px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","upload",[80,2015],{"d":2016,"fill":77,"stroke":2017,"style":55},"M436 162 V 196 H 300 V 210","var(--accent,#f43f8f)",[96,2019],{"points":2020,"fill":2021},"300,218 293,206 307,206","#f43f8f",[40,2023],{"x":50,"y":2024,"width":2025,"height":2026,"rx":2007,"fill":2027,"stroke":2017,"style":55},"196","300","42","#fee8f2",[57,2029,2032],{"x":2025,"y":2030,"style":2031},"222","font-size:11px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","any failure → alert, keep the old report",[182,2034,2036],{"className":184,"code":2035,"language":186,"meta":187,"style":187},"from pathlib import Path\nimport pandas as pd\n\ndef ready_to_publish(path, expected_sheets, min_rows=1, period=None):\n    \"\"\"Refuse to publish a workbook that fails a basic sanity check.\"\"\"\n    path = Path(path)\n    problems = []\n\n    if not path.exists() or path.stat().st_size == 0:\n        return [f\"{path} is missing or empty\"]\n\n    try:\n        sheets = pd.read_excel(path, sheet_name=None)\n    except Exception as exc:\n        return [f\"{path} does not open: {exc}\"]\n\n    missing = set(expected_sheets) - set(sheets)\n    if missing:\n        problems.append(f\"missing sheets: {', '.join(sorted(missing))}\")\n\n    for name, frame in sheets.items():\n        if len(frame) \u003C min_rows:\n            problems.append(f\"sheet {name!r} has only {len(frame)} rows\")\n\n    if period and \"date\" in sheets.get(expected_sheets[0], pd.DataFrame()):\n        dates = pd.to_datetime(sheets[expected_sheets[0]][\"date\"],\n                               errors=\"coerce\")\n        if dates.max() \u003C pd.Timestamp(period):\n            problems.append(\n                f\"newest row is {dates.max():%Y-%m-%d}, before {period}\"\n            )\n\n    return problems\n\nissues = ready_to_publish(\"report.xlsx\", [\"Summary\", \"Detail\"], min_rows=1)\nif issues:\n    raise RuntimeError(\"not published:\\n  \" + \"\\n  \".join(issues))\n",[189,2037,2038,2048,2060,2064,2087,2092,2102,2112,2116,2137,2160,2164,2171,2190,2203,2233,2237,2257,2264,2294,2298,2310,2327,2364,2368,2391,2413,2426,2439,2445,2490,2496,2501,2509,2514,2551,2560],{"__ignoreMap":187},[192,2039,2040,2042,2044,2046],{"class":194,"line":195},[192,2041,199],{"class":198},[192,2043,953],{"class":202},[192,2045,206],{"class":198},[192,2047,958],{"class":202},[192,2049,2050,2052,2055,2057],{"class":194,"line":212},[192,2051,206],{"class":198},[192,2053,2054],{"class":202}," pandas ",[192,2056,365],{"class":198},[192,2058,2059],{"class":202}," pd\n",[192,2061,2062],{"class":194,"line":220},[192,2063,224],{"emptyLinePlaceholder":223},[192,2065,2066,2068,2071,2074,2076,2078,2081,2083,2085],{"class":194,"line":227},[192,2067,230],{"class":198},[192,2069,2070],{"class":233}," ready_to_publish",[192,2072,2073],{"class":202},"(path, expected_sheets, min_rows",[192,2075,246],{"class":198},[192,2077,1088],{"class":358},[192,2079,2080],{"class":202},", period",[192,2082,246],{"class":198},[192,2084,524],{"class":358},[192,2086,535],{"class":202},[192,2088,2089],{"class":194,"line":240},[192,2090,2091],{"class":258},"    \"\"\"Refuse to publish a workbook that fails a basic sanity check.\"\"\"\n",[192,2093,2094,2097,2099],{"class":194,"line":252},[192,2095,2096],{"class":202},"    path ",[192,2098,246],{"class":198},[192,2100,2101],{"class":202}," Path(path)\n",[192,2103,2104,2107,2109],{"class":194,"line":270},[192,2105,2106],{"class":202},"    problems ",[192,2108,246],{"class":198},[192,2110,2111],{"class":202}," []\n",[192,2113,2114],{"class":194,"line":288},[192,2115,224],{"emptyLinePlaceholder":223},[192,2117,2118,2120,2122,2125,2127,2130,2132,2135],{"class":194,"line":303},[192,2119,1242],{"class":198},[192,2121,1331],{"class":198},[192,2123,2124],{"class":202}," path.exists() ",[192,2126,553],{"class":198},[192,2128,2129],{"class":202}," path.stat().st_size ",[192,2131,1776],{"class":198},[192,2133,2134],{"class":358}," 0",[192,2136,1710],{"class":202},[192,2138,2139,2142,2145,2147,2149,2151,2153,2155,2158],{"class":194,"line":309},[192,2140,2141],{"class":198},"        return",[192,2143,2144],{"class":202}," [",[192,2146,1510],{"class":198},[192,2148,569],{"class":258},[192,2150,572],{"class":318},[192,2152,80],{"class":202},[192,2154,578],{"class":318},[192,2156,2157],{"class":258}," is missing or empty\"",[192,2159,1223],{"class":202},[192,2161,2162],{"class":194,"line":331},[192,2163,224],{"emptyLinePlaceholder":223},[192,2165,2166,2169],{"class":194,"line":337},[192,2167,2168],{"class":198},"    try",[192,2170,1710],{"class":202},[192,2172,2173,2176,2178,2181,2184,2186,2188],{"class":194,"line":343},[192,2174,2175],{"class":202},"        sheets ",[192,2177,246],{"class":198},[192,2179,2180],{"class":202}," pd.read_excel(path, ",[192,2182,2183],{"class":833},"sheet_name",[192,2185,246],{"class":198},[192,2187,524],{"class":358},[192,2189,750],{"class":202},[192,2191,2192,2195,2198,2201],{"class":194,"line":371},[192,2193,2194],{"class":198},"    except",[192,2196,2197],{"class":358}," Exception",[192,2199,2200],{"class":198}," as",[192,2202,1731],{"class":202},[192,2204,2205,2207,2209,2211,2213,2215,2217,2219,2222,2224,2227,2229,2231],{"class":194,"line":377},[192,2206,2141],{"class":198},[192,2208,2144],{"class":202},[192,2210,1510],{"class":198},[192,2212,569],{"class":258},[192,2214,572],{"class":318},[192,2216,80],{"class":202},[192,2218,578],{"class":318},[192,2220,2221],{"class":258}," does not open: ",[192,2223,572],{"class":318},[192,2225,2226],{"class":202},"exc",[192,2228,578],{"class":318},[192,2230,569],{"class":258},[192,2232,1223],{"class":202},[192,2234,2235],{"class":194,"line":873},[192,2236,224],{"emptyLinePlaceholder":223},[192,2238,2239,2242,2244,2247,2250,2252,2254],{"class":194,"line":887},[192,2240,2241],{"class":202},"    missing ",[192,2243,246],{"class":198},[192,2245,2246],{"class":358}," set",[192,2248,2249],{"class":202},"(expected_sheets) ",[192,2251,615],{"class":198},[192,2253,2246],{"class":358},[192,2255,2256],{"class":202},"(sheets)\n",[192,2258,2259,2261],{"class":194,"line":892},[192,2260,1242],{"class":198},[192,2262,2263],{"class":202}," missing:\n",[192,2265,2266,2269,2271,2274,2276,2279,2282,2285,2288,2290,2292],{"class":194,"line":914},[192,2267,2268],{"class":202},"        problems.append(",[192,2270,1510],{"class":198},[192,2272,2273],{"class":258},"\"missing sheets: ",[192,2275,572],{"class":318},[192,2277,2278],{"class":258},"', '",[192,2280,2281],{"class":202},".join(",[192,2283,2284],{"class":358},"sorted",[192,2286,2287],{"class":202},"(missing))",[192,2289,578],{"class":318},[192,2291,569],{"class":258},[192,2293,750],{"class":202},[192,2295,2296],{"class":194,"line":1297},[192,2297,224],{"emptyLinePlaceholder":223},[192,2299,2300,2302,2305,2307],{"class":194,"line":1302},[192,2301,1680],{"class":198},[192,2303,2304],{"class":202}," name, frame ",[192,2306,1134],{"class":198},[192,2308,2309],{"class":202}," sheets.items():\n",[192,2311,2312,2315,2318,2321,2324],{"class":194,"line":1323},[192,2313,2314],{"class":198},"        if",[192,2316,2317],{"class":358}," len",[192,2319,2320],{"class":202},"(frame) ",[192,2322,2323],{"class":198},"\u003C",[192,2325,2326],{"class":202}," min_rows:\n",[192,2328,2329,2332,2334,2337,2339,2341,2344,2346,2349,2351,2354,2357,2359,2362],{"class":194,"line":1350},[192,2330,2331],{"class":202},"            problems.append(",[192,2333,1510],{"class":198},[192,2335,2336],{"class":258},"\"sheet ",[192,2338,572],{"class":318},[192,2340,610],{"class":202},[192,2342,2343],{"class":198},"!r",[192,2345,578],{"class":318},[192,2347,2348],{"class":258}," has only ",[192,2350,572],{"class":318},[192,2352,2353],{"class":358},"len",[192,2355,2356],{"class":202},"(frame)",[192,2358,578],{"class":318},[192,2360,2361],{"class":258}," rows\"",[192,2363,750],{"class":202},[192,2365,2366],{"class":194,"line":1364},[192,2367,224],{"emptyLinePlaceholder":223},[192,2369,2371,2373,2376,2378,2381,2383,2386,2388],{"class":194,"line":2370},25,[192,2372,1242],{"class":198},[192,2374,2375],{"class":202}," period ",[192,2377,1340],{"class":198},[192,2379,2380],{"class":258}," \"date\"",[192,2382,1334],{"class":198},[192,2384,2385],{"class":202}," sheets.get(expected_sheets[",[192,2387,42],{"class":358},[192,2389,2390],{"class":202},"], pd.DataFrame()):\n",[192,2392,2394,2397,2399,2402,2404,2407,2410],{"class":194,"line":2393},26,[192,2395,2396],{"class":202},"        dates ",[192,2398,246],{"class":198},[192,2400,2401],{"class":202}," pd.to_datetime(sheets[expected_sheets[",[192,2403,42],{"class":358},[192,2405,2406],{"class":202},"]][",[192,2408,2409],{"class":258},"\"date\"",[192,2411,2412],{"class":202},"],\n",[192,2414,2416,2419,2421,2424],{"class":194,"line":2415},27,[192,2417,2418],{"class":833},"                               errors",[192,2420,246],{"class":198},[192,2422,2423],{"class":258},"\"coerce\"",[192,2425,750],{"class":202},[192,2427,2429,2431,2434,2436],{"class":194,"line":2428},28,[192,2430,2314],{"class":198},[192,2432,2433],{"class":202}," dates.max() ",[192,2435,2323],{"class":198},[192,2437,2438],{"class":202}," pd.Timestamp(period):\n",[192,2440,2442],{"class":194,"line":2441},29,[192,2443,2444],{"class":202},"            problems.append(\n",[192,2446,2448,2451,2454,2456,2459,2461,2463,2466,2468,2471,2473,2476,2478,2481,2483,2486,2488],{"class":194,"line":2447},30,[192,2449,2450],{"class":198},"                f",[192,2452,2453],{"class":258},"\"newest row is ",[192,2455,572],{"class":318},[192,2457,2458],{"class":202},"dates.max()",[192,2460,589],{"class":198},[192,2462,592],{"class":202},[192,2464,2465],{"class":198},"Y-",[192,2467,592],{"class":202},[192,2469,2470],{"class":198},"m-",[192,2472,592],{"class":202},[192,2474,2475],{"class":198},"d",[192,2477,578],{"class":318},[192,2479,2480],{"class":258},", before ",[192,2482,572],{"class":318},[192,2484,2485],{"class":202},"period",[192,2487,578],{"class":318},[192,2489,328],{"class":258},[192,2491,2493],{"class":194,"line":2492},31,[192,2494,2495],{"class":202},"            )\n",[192,2497,2499],{"class":194,"line":2498},32,[192,2500,224],{"emptyLinePlaceholder":223},[192,2502,2504,2506],{"class":194,"line":2503},33,[192,2505,667],{"class":198},[192,2507,2508],{"class":202}," problems\n",[192,2510,2512],{"class":194,"line":2511},34,[192,2513,224],{"emptyLinePlaceholder":223},[192,2515,2517,2520,2522,2525,2528,2531,2534,2536,2539,2542,2545,2547,2549],{"class":194,"line":2516},35,[192,2518,2519],{"class":202},"issues ",[192,2521,246],{"class":198},[192,2523,2524],{"class":202}," ready_to_publish(",[192,2526,2527],{"class":258},"\"report.xlsx\"",[192,2529,2530],{"class":202},", [",[192,2532,2533],{"class":258},"\"Summary\"",[192,2535,355],{"class":202},[192,2537,2538],{"class":258},"\"Detail\"",[192,2540,2541],{"class":202},"], ",[192,2543,2544],{"class":833},"min_rows",[192,2546,246],{"class":198},[192,2548,1088],{"class":358},[192,2550,750],{"class":202},[192,2552,2554,2557],{"class":194,"line":2553},36,[192,2555,2556],{"class":198},"if",[192,2558,2559],{"class":202}," issues:\n",[192,2561,2563,2566,2568,2570,2573,2576,2579,2582,2585,2587,2589],{"class":194,"line":2562},37,[192,2564,2565],{"class":198},"    raise",[192,2567,1259],{"class":358},[192,2569,1140],{"class":202},[192,2571,2572],{"class":258},"\"not published:",[192,2574,2575],{"class":318},"\\n",[192,2577,2578],{"class":258},"  \"",[192,2580,2581],{"class":198}," +",[192,2583,2584],{"class":258}," \"",[192,2586,2575],{"class":318},[192,2588,2578],{"class":258},[192,2590,2591],{"class":202},".join(issues))\n",[10,2593,2594,2595,18],{},"The freshness check is the one that earns its keep most often. A pipeline whose upstream extract quietly failed produces a workbook that is structurally perfect and contains last month's numbers — the only signal is that the newest row is older than it should be. The wider set of pre-send checks is in ",[14,2596,2598],{"href":2597},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fvalidate-an-excel-report-before-sending-it\u002F","validating an Excel report before sending it",[10,2600,2601],{},"Leaving the previous version in place on failure is deliberate. A stale report that readers recognise as last month's is far less damaging than a fresh-looking one with wrong numbers, and it buys time to fix the cause.",[168,2603,2605],{"id":2604},"choosing-a-destination","Choosing a destination",[10,2607,2608],{},"The best destination is usually the one your readers already have open.",[1388,2610,2611,2621],{},[1391,2612,2613],{},[1394,2614,2615,2618],{},[1397,2616,2617],{},"Situation",[1397,2619,2620],{},"Publish to",[1407,2622,2623,2631,2639,2646,2654],{},[1394,2624,2625,2628],{},[1412,2626,2627],{},"Readers live in Teams and Office",[1412,2629,2630],{},"SharePoint or OneDrive",[1394,2632,2633,2636],{},[1412,2634,2635],{},"Another system consumes the file",[1412,2637,2638],{},"S3 or compatible object storage",[1394,2640,2641,2644],{},[1412,2642,2643],{},"The organisation runs on Google Workspace",[1412,2645,149],{},[1394,2647,2648,2651],{},[1412,2649,2650],{},"On-premise, no cloud allowed",[1412,2652,2653],{},"A network share",[1394,2655,2656,2659],{},[1412,2657,2658],{},"The report feeds a data pipeline",[1412,2660,2661],{},"Object storage, alongside a Parquet copy",[10,2663,2664,2665,2668],{},"That last row is worth expanding. When a report is both read by people and consumed by machines, publish two artefacts from one run: the formatted ",[189,2666,2667],{},".xlsx"," for readers, and a plain columnar file for the pipeline. Making the pipeline parse a styled workbook is a recurring source of breakage, because a formatting change nobody thought was significant moves a header row.",[182,2670,2672],{"className":184,"code":2671,"language":186,"meta":187,"style":187},"def publish_both(df, bucket, name):\n    \"\"\"One run, two artefacts: a formatted workbook and a machine-readable copy.\"\"\"\n    dated, latest = report_keys(name)\n\n    df.to_excel(\"report.xlsx\", index=False, engine=\"xlsxwriter\")\n    df.to_parquet(\"report.parquet\", index=False)\n\n    publish_atomic(\"report.xlsx\", bucket, dated)\n    publish_atomic(\"report.xlsx\", bucket, latest)\n    publish_atomic(\"report.parquet\", bucket, dated.replace(\".xlsx\", \".parquet\"))\n    return dated, latest\n",[189,2673,2674,2684,2689,2699,2703,2732,2750,2754,2764,2773,2792],{"__ignoreMap":187},[192,2675,2676,2678,2681],{"class":194,"line":195},[192,2677,230],{"class":198},[192,2679,2680],{"class":233}," publish_both",[192,2682,2683],{"class":202},"(df, bucket, name):\n",[192,2685,2686],{"class":194,"line":212},[192,2687,2688],{"class":258},"    \"\"\"One run, two artefacts: a formatted workbook and a machine-readable copy.\"\"\"\n",[192,2690,2691,2694,2696],{"class":194,"line":220},[192,2692,2693],{"class":202},"    dated, latest ",[192,2695,246],{"class":198},[192,2697,2698],{"class":202}," report_keys(name)\n",[192,2700,2701],{"class":194,"line":227},[192,2702,224],{"emptyLinePlaceholder":223},[192,2704,2705,2708,2710,2712,2715,2717,2720,2722,2725,2727,2730],{"class":194,"line":240},[192,2706,2707],{"class":202},"    df.to_excel(",[192,2709,2527],{"class":258},[192,2711,355],{"class":202},[192,2713,2714],{"class":833},"index",[192,2716,246],{"class":198},[192,2718,2719],{"class":358},"False",[192,2721,355],{"class":202},[192,2723,2724],{"class":833},"engine",[192,2726,246],{"class":198},[192,2728,2729],{"class":258},"\"xlsxwriter\"",[192,2731,750],{"class":202},[192,2733,2734,2737,2740,2742,2744,2746,2748],{"class":194,"line":252},[192,2735,2736],{"class":202},"    df.to_parquet(",[192,2738,2739],{"class":258},"\"report.parquet\"",[192,2741,355],{"class":202},[192,2743,2714],{"class":833},[192,2745,246],{"class":198},[192,2747,2719],{"class":358},[192,2749,750],{"class":202},[192,2751,2752],{"class":194,"line":270},[192,2753,224],{"emptyLinePlaceholder":223},[192,2755,2756,2759,2761],{"class":194,"line":288},[192,2757,2758],{"class":202},"    publish_atomic(",[192,2760,2527],{"class":258},[192,2762,2763],{"class":202},", bucket, dated)\n",[192,2765,2766,2768,2770],{"class":194,"line":303},[192,2767,2758],{"class":202},[192,2769,2527],{"class":258},[192,2771,2772],{"class":202},", bucket, latest)\n",[192,2774,2775,2777,2779,2782,2785,2787,2790],{"class":194,"line":309},[192,2776,2758],{"class":202},[192,2778,2739],{"class":258},[192,2780,2781],{"class":202},", bucket, dated.replace(",[192,2783,2784],{"class":258},"\".xlsx\"",[192,2786,355],{"class":202},[192,2788,2789],{"class":258},"\".parquet\"",[192,2791,688],{"class":202},[192,2793,2794,2796],{"class":194,"line":331},[192,2795,667],{"class":198},[192,2797,670],{"class":202},[168,2799,2801],{"id":2800},"record-what-was-published","Record what was published",[10,2803,2804],{},"The publish step is also where a report's audit trail is created, and it costs almost nothing to write one. A small manifest alongside each run answers the questions that come up weeks later: which run produced this file, what did it contain, and did anything look unusual at the time?",[182,2806,2808],{"className":184,"code":2807,"language":186,"meta":187,"style":187},"import hashlib\nimport json\nimport platform\nfrom datetime import datetime, timezone\nfrom pathlib import Path\n\ndef manifest(local_path, destination, rows, period):\n    \"\"\"A small JSON record describing one publication.\"\"\"\n    data = Path(local_path).read_bytes()\n    return {\n        \"published_at\": datetime.now(timezone.utc).isoformat(timespec=\"seconds\"),\n        \"destination\": destination,\n        \"bytes\": len(data),\n        \"sha256\": hashlib.sha256(data).hexdigest(),\n        \"rows\": rows,\n        \"period\": period,\n        \"host\": platform.node(),\n    }\n\nrecord = manifest(\"report.xlsx\", \"s3:\u002F\u002Facme-reports\u002Freports\u002Fregional-latest.xlsx\",\n                  rows=1_482, period=\"2026-08\")\nPath(\"report.manifest.json\").write_text(json.dumps(record, indent=2))\n",[189,2809,2810,2816,2823,2830,2841,2851,2855,2865,2870,2880,2887,2906,2914,2927,2935,2943,2951,2959,2964,2968,2987,3008],{"__ignoreMap":187},[192,2811,2812,2814],{"class":194,"line":195},[192,2813,206],{"class":198},[192,2815,1069],{"class":202},[192,2817,2818,2820],{"class":194,"line":212},[192,2819,206],{"class":198},[192,2821,2822],{"class":202}," json\n",[192,2824,2825,2827],{"class":194,"line":220},[192,2826,206],{"class":198},[192,2828,2829],{"class":202}," platform\n",[192,2831,2832,2834,2836,2838],{"class":194,"line":227},[192,2833,199],{"class":198},[192,2835,500],{"class":202},[192,2837,206],{"class":198},[192,2839,2840],{"class":202}," datetime, timezone\n",[192,2842,2843,2845,2847,2849],{"class":194,"line":240},[192,2844,199],{"class":198},[192,2846,953],{"class":202},[192,2848,206],{"class":198},[192,2850,958],{"class":202},[192,2852,2853],{"class":194,"line":252},[192,2854,224],{"emptyLinePlaceholder":223},[192,2856,2857,2859,2862],{"class":194,"line":270},[192,2858,230],{"class":198},[192,2860,2861],{"class":233}," manifest",[192,2863,2864],{"class":202},"(local_path, destination, rows, period):\n",[192,2866,2867],{"class":194,"line":288},[192,2868,2869],{"class":258},"    \"\"\"A small JSON record describing one publication.\"\"\"\n",[192,2871,2872,2875,2877],{"class":194,"line":303},[192,2873,2874],{"class":202},"    data ",[192,2876,246],{"class":198},[192,2878,2879],{"class":202}," Path(local_path).read_bytes()\n",[192,2881,2882,2884],{"class":194,"line":309},[192,2883,667],{"class":198},[192,2885,2886],{"class":202}," {\n",[192,2888,2889,2892,2895,2898,2900,2903],{"class":194,"line":331},[192,2890,2891],{"class":258},"        \"published_at\"",[192,2893,2894],{"class":202},": datetime.now(timezone.utc).isoformat(",[192,2896,2897],{"class":833},"timespec",[192,2899,246],{"class":198},[192,2901,2902],{"class":258},"\"seconds\"",[192,2904,2905],{"class":202},"),\n",[192,2907,2908,2911],{"class":194,"line":337},[192,2909,2910],{"class":258},"        \"destination\"",[192,2912,2913],{"class":202},": destination,\n",[192,2915,2916,2919,2922,2924],{"class":194,"line":343},[192,2917,2918],{"class":258},"        \"bytes\"",[192,2920,2921],{"class":202},": ",[192,2923,2353],{"class":358},[192,2925,2926],{"class":202},"(data),\n",[192,2928,2929,2932],{"class":194,"line":371},[192,2930,2931],{"class":258},"        \"sha256\"",[192,2933,2934],{"class":202},": hashlib.sha256(data).hexdigest(),\n",[192,2936,2937,2940],{"class":194,"line":377},[192,2938,2939],{"class":258},"        \"rows\"",[192,2941,2942],{"class":202},": rows,\n",[192,2944,2945,2948],{"class":194,"line":873},[192,2946,2947],{"class":258},"        \"period\"",[192,2949,2950],{"class":202},": period,\n",[192,2952,2953,2956],{"class":194,"line":887},[192,2954,2955],{"class":258},"        \"host\"",[192,2957,2958],{"class":202},": platform.node(),\n",[192,2960,2961],{"class":194,"line":892},[192,2962,2963],{"class":202},"    }\n",[192,2965,2966],{"class":194,"line":914},[192,2967,224],{"emptyLinePlaceholder":223},[192,2969,2970,2973,2975,2978,2980,2982,2985],{"class":194,"line":1297},[192,2971,2972],{"class":202},"record ",[192,2974,246],{"class":198},[192,2976,2977],{"class":202}," manifest(",[192,2979,2527],{"class":258},[192,2981,355],{"class":202},[192,2983,2984],{"class":258},"\"s3:\u002F\u002Facme-reports\u002Freports\u002Fregional-latest.xlsx\"",[192,2986,884],{"class":202},[192,2988,2989,2992,2994,2997,2999,3001,3003,3006],{"class":194,"line":1302},[192,2990,2991],{"class":833},"                  rows",[192,2993,246],{"class":198},[192,2995,2996],{"class":358},"1_482",[192,2998,355],{"class":202},[192,3000,2485],{"class":833},[192,3002,246],{"class":198},[192,3004,3005],{"class":258},"\"2026-08\"",[192,3007,750],{"class":202},[192,3009,3010,3013,3016,3019,3022,3024,3026],{"class":194,"line":1323},[192,3011,3012],{"class":202},"Path(",[192,3014,3015],{"class":258},"\"report.manifest.json\"",[192,3017,3018],{"class":202},").write_text(json.dumps(record, ",[192,3020,3021],{"class":833},"indent",[192,3023,246],{"class":198},[192,3025,1804],{"class":358},[192,3027,688],{"class":202},[10,3029,3030],{},"Publish the manifest next to the report. Because it carries a checksum, it also settles the \"is this the file you sent me?\" question definitively — somebody holding a copy can hash it and compare. And because it records the row count, a sudden change between months is visible in the manifests alone, without opening a single workbook.",[10,3032,3033],{},"That row count is worth a comparison rather than just a record. Run-over-run volume checks catch a whole class of upstream failure that structural validation misses:",[182,3035,3037],{"className":184,"code":3036,"language":186,"meta":187,"style":187},"import json\nfrom pathlib import Path\n\ndef check_against_last(record, history_dir=\"manifests\", tolerance=0.35):\n    \"\"\"Warn when this run's row count departs sharply from the previous one.\"\"\"\n    history = sorted(Path(history_dir).glob(\"*.json\"))\n    if not history:\n        return None\n\n    previous = json.loads(history[-1].read_text())\n    before, now = previous.get(\"rows\", 0), record[\"rows\"]\n    if not before:\n        return None\n\n    change = (now - before) \u002F before\n    if abs(change) > tolerance:\n        return (\n            f\"row count moved {change:+.0%} ({before:,} to {now:,}) — \"\n            \"check the upstream extract before circulating\"\n        )\n    return None\n\nwarning = check_against_last(record)\nif warning:\n    print(\"WARNING:\", warning)\n",[189,3038,3039,3045,3055,3059,3084,3089,3107,3116,3123,3127,3144,3168,3177,3183,3187,3207,3223,3230,3275,3280,3284,3290,3294,3304,3311],{"__ignoreMap":187},[192,3040,3041,3043],{"class":194,"line":195},[192,3042,206],{"class":198},[192,3044,2822],{"class":202},[192,3046,3047,3049,3051,3053],{"class":194,"line":212},[192,3048,199],{"class":198},[192,3050,953],{"class":202},[192,3052,206],{"class":198},[192,3054,958],{"class":202},[192,3056,3057],{"class":194,"line":220},[192,3058,224],{"emptyLinePlaceholder":223},[192,3060,3061,3063,3066,3069,3071,3074,3077,3079,3082],{"class":194,"line":227},[192,3062,230],{"class":198},[192,3064,3065],{"class":233}," check_against_last",[192,3067,3068],{"class":202},"(record, history_dir",[192,3070,246],{"class":198},[192,3072,3073],{"class":258},"\"manifests\"",[192,3075,3076],{"class":202},", tolerance",[192,3078,246],{"class":198},[192,3080,3081],{"class":358},"0.35",[192,3083,535],{"class":202},[192,3085,3086],{"class":194,"line":240},[192,3087,3088],{"class":258},"    \"\"\"Warn when this run's row count departs sharply from the previous one.\"\"\"\n",[192,3090,3091,3094,3096,3099,3102,3105],{"class":194,"line":252},[192,3092,3093],{"class":202},"    history ",[192,3095,246],{"class":198},[192,3097,3098],{"class":358}," sorted",[192,3100,3101],{"class":202},"(Path(history_dir).glob(",[192,3103,3104],{"class":258},"\"*.json\"",[192,3106,688],{"class":202},[192,3108,3109,3111,3113],{"class":194,"line":270},[192,3110,1242],{"class":198},[192,3112,1331],{"class":198},[192,3114,3115],{"class":202}," history:\n",[192,3117,3118,3120],{"class":194,"line":288},[192,3119,2141],{"class":198},[192,3121,3122],{"class":358}," None\n",[192,3124,3125],{"class":194,"line":303},[192,3126,224],{"emptyLinePlaceholder":223},[192,3128,3129,3132,3134,3137,3139,3141],{"class":194,"line":309},[192,3130,3131],{"class":202},"    previous ",[192,3133,246],{"class":198},[192,3135,3136],{"class":202}," json.loads(history[",[192,3138,615],{"class":198},[192,3140,1088],{"class":358},[192,3142,3143],{"class":202},"].read_text())\n",[192,3145,3146,3149,3151,3154,3157,3159,3161,3164,3166],{"class":194,"line":331},[192,3147,3148],{"class":202},"    before, now ",[192,3150,246],{"class":198},[192,3152,3153],{"class":202}," previous.get(",[192,3155,3156],{"class":258},"\"rows\"",[192,3158,355],{"class":202},[192,3160,42],{"class":358},[192,3162,3163],{"class":202},"), record[",[192,3165,3156],{"class":258},[192,3167,1223],{"class":202},[192,3169,3170,3172,3174],{"class":194,"line":337},[192,3171,1242],{"class":198},[192,3173,1331],{"class":198},[192,3175,3176],{"class":202}," before:\n",[192,3178,3179,3181],{"class":194,"line":343},[192,3180,2141],{"class":198},[192,3182,3122],{"class":358},[192,3184,3185],{"class":194,"line":371},[192,3186,224],{"emptyLinePlaceholder":223},[192,3188,3189,3192,3194,3197,3199,3202,3204],{"class":194,"line":377},[192,3190,3191],{"class":202},"    change ",[192,3193,246],{"class":198},[192,3195,3196],{"class":202}," (now ",[192,3198,615],{"class":198},[192,3200,3201],{"class":202}," before) ",[192,3203,581],{"class":198},[192,3205,3206],{"class":202}," before\n",[192,3208,3209,3211,3214,3217,3220],{"class":194,"line":873},[192,3210,1242],{"class":198},[192,3212,3213],{"class":358}," abs",[192,3215,3216],{"class":202},"(change) ",[192,3218,3219],{"class":198},">",[192,3221,3222],{"class":202}," tolerance:\n",[192,3224,3225,3227],{"class":194,"line":887},[192,3226,2141],{"class":198},[192,3228,3229],{"class":202}," (\n",[192,3231,3232,3234,3237,3239,3242,3245,3247,3250,3252,3255,3258,3260,3263,3265,3268,3270,3272],{"class":194,"line":892},[192,3233,1267],{"class":198},[192,3235,3236],{"class":258},"\"row count moved ",[192,3238,572],{"class":318},[192,3240,3241],{"class":202},"change",[192,3243,3244],{"class":198},":+.0%",[192,3246,578],{"class":318},[192,3248,3249],{"class":258}," (",[192,3251,572],{"class":318},[192,3253,3254],{"class":202},"before",[192,3256,3257],{"class":198},":,",[192,3259,578],{"class":318},[192,3261,3262],{"class":258}," to ",[192,3264,572],{"class":318},[192,3266,3267],{"class":202},"now",[192,3269,3257],{"class":198},[192,3271,578],{"class":318},[192,3273,3274],{"class":258},") — \"\n",[192,3276,3277],{"class":194,"line":914},[192,3278,3279],{"class":258},"            \"check the upstream extract before circulating\"\n",[192,3281,3282],{"class":194,"line":1297},[192,3283,1294],{"class":202},[192,3285,3286,3288],{"class":194,"line":1302},[192,3287,667],{"class":198},[192,3289,3122],{"class":358},[192,3291,3292],{"class":194,"line":1323},[192,3293,224],{"emptyLinePlaceholder":223},[192,3295,3296,3299,3301],{"class":194,"line":1350},[192,3297,3298],{"class":202},"warning ",[192,3300,246],{"class":198},[192,3302,3303],{"class":202}," check_against_last(record)\n",[192,3305,3306,3308],{"class":194,"line":1364},[192,3307,2556],{"class":198},[192,3309,3310],{"class":202}," warning:\n",[192,3312,3313,3316,3318,3321],{"class":194,"line":2370},[192,3314,3315],{"class":358},"    print",[192,3317,1140],{"class":202},[192,3319,3320],{"class":258},"\"WARNING:\"",[192,3322,3323],{"class":202},", warning)\n",[10,3325,3326,3327,3331],{},"Whether that warning blocks the publish or merely annotates it is a judgement call. For a report where a real business change could plausibly move volumes by half, a hard block would fire constantly and be ignored within a month; for a stable operational feed, blocking is right. Pick deliberately rather than defaulting, and log the decision either way — the logging patterns in ",[14,3328,3330],{"href":3329},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Flog-python-excel-script-output-to-a-file\u002F","logging Python Excel script output to a file"," apply directly, and a publish step that leaves no trace is the hardest kind of pipeline to debug six months on.",[168,3333,3335],{"id":3334},"key-takeaways","Key takeaways",[3337,3338,3339,3347,3356,3362,3368,3374,3380],"ul",{},[3340,3341,3342,3346],"li",{},[3343,3344,3345],"strong",{},"Publish and link rather than attach."," One canonical copy, a stable URL, and an access trail.",[3340,3348,3349,3355],{},[3343,3350,3351,3352,3354],{},"Write a dated object ",[1047,3353,1340],{}," a stable latest key."," History and links have different needs; satisfy both.",[3340,3357,3358,3361],{},[3343,3359,3360],{},"Upload to a staging name and move it into place."," Readers never see a partially written report.",[3340,3363,3364,3367],{},[3343,3365,3366],{},"Verify the transfer."," A 200 response is not proof; compare size, and a checksum where you can.",[3340,3369,3370,3373],{},[3343,3371,3372],{},"Authenticate as a machine."," Roles, app registrations and service accounts survive people leaving.",[3340,3375,3376,3379],{},[3343,3377,3378],{},"Retry the transient only."," Exponential backoff with jitter, and never on a permission error.",[3340,3381,3382,3385],{},[3343,3383,3384],{},"Ship a machine-readable copy"," alongside the workbook when a pipeline consumes the same data.",[168,3387,3389],{"id":3388},"frequently-asked-questions","Frequently asked questions",[10,3391,3392,3395],{},[3343,3393,3394],{},"Should I email a report or publish it to storage?","\nPublish it and email the link. Attachments duplicate the file into every mailbox, hit size limits, and leave people arguing over which copy is current. A link always resolves to the latest version and leaves an access trail.",[10,3397,3398,3401],{},[3343,3399,3400],{},"How do I stop readers seeing a half-written file?","\nUpload to a temporary key or name, then rename or copy it into place. Most stores make that final step atomic, so a reader either sees the previous version or the complete new one, never a partial upload.",[10,3403,3404,3407],{},[3343,3405,3406],{},"What is the best way to authenticate a scheduled job?","\nA machine identity rather than a person's account — an IAM role for S3, an app registration with client credentials for Microsoft Graph, a service account for Google Drive. Personal credentials break when the person changes their password or leaves.",[10,3409,3410,3413],{},[3343,3411,3412],{},"Should reports overwrite the same name or be dated?","\nBoth. Write a dated object for the history and update a stable latest pointer that dashboards and links can rely on. Overwriting alone destroys the audit trail; dating alone means every link goes stale.",[10,3415,3416,3419],{},[3343,3417,3418],{},"How do I know the upload actually worked?","\nCompare a checksum of the local file with the one the store reports, or at minimum re-read the object's size. A successful HTTP status only means the request was accepted, not that the bytes are intact.",[168,3421,3423],{"id":3422},"related","Related",[3337,3425,3426,3432,3439,3446,3453,3460,3466],{},[3340,3427,3428,3429,3431],{},"Up to the parent: ",[14,3430,17],{"href":16}," — the pipeline this delivery stage completes.",[3340,3433,3434,3438],{},[14,3435,3437],{"href":3436},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fupload-an-excel-report-to-amazon-s3-with-boto3\u002F","Upload an Excel Report to Amazon S3 with boto3"," — object storage end to end.",[3340,3440,3441,3445],{},[14,3442,3444],{"href":3443},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fupload-an-excel-report-to-sharepoint-with-python\u002F","Upload an Excel Report to SharePoint with Python"," — Microsoft Graph, tokens and large uploads.",[3340,3447,3448,3452],{},[14,3449,3451],{"href":3450},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fsave-excel-reports-to-google-drive-with-python\u002F","Save Excel Reports to Google Drive with Python"," — the Drive API with a service account.",[3340,3454,3455,3459],{},[14,3456,3458],{"href":3457},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fwrite-excel-files-to-a-network-share-from-python\u002F","Write Excel Files to a Network Share from Python"," — the on-premise option, done safely.",[3340,3461,3462,3465],{},[14,3463,3464],{"href":386},"Emailing Excel Reports with smtplib"," — the notification that carries the link.",[3340,3467,3468,3472],{},[14,3469,3471],{"href":3470},"\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002F","Scheduling Python Excel Scripts with Cron"," — running the publish step unattended.",[3474,3475,3476],"style",{},"html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}",{"title":187,"searchDepth":212,"depth":212,"links":3478},[3479,3480,3481,3482,3483,3484,3485,3486,3487,3488,3489,3490],{"id":170,"depth":212,"text":171},{"id":391,"depth":212,"text":392},{"id":711,"depth":212,"text":712},{"id":1053,"depth":212,"text":1054},{"id":1382,"depth":212,"text":1383},{"id":1573,"depth":212,"text":1574},{"id":1892,"depth":212,"text":1893},{"id":2604,"depth":212,"text":2605},{"id":2800,"depth":212,"text":2801},{"id":3334,"depth":212,"text":3335},{"id":3388,"depth":212,"text":3389},{"id":3422,"depth":212,"text":3423},"2026-08-15","Deliver generated Excel reports where people already look: S3, SharePoint and OneDrive, Google Drive, and network shares — with atomic uploads, retries and verified transfers.","md",[3495,3497,3499,3501,3503],{"q":3394,"a":3496},"Publish it and email the link. Attachments duplicate the file into every mailbox, hit size limits, and leave people arguing over which copy is current. A link always resolves to the latest version and leaves an access trail.",{"q":3400,"a":3498},"Upload to a temporary key or name, then rename or copy it into place. Most stores make that final step atomic, so a reader either sees the previous version or the complete new one, never a partial upload.",{"q":3406,"a":3500},"A machine identity rather than a person's account — an IAM role for S3, an app registration with client credentials for Microsoft Graph, a service account for Google Drive. Personal credentials break when the person changes their password or leaves.",{"q":3412,"a":3502},"Both. Write a dated object for the history and update a stable latest pointer that dashboards and links can rely on. Overwriting alone destroys the audit trail; dating alone means every link goes stale.",{"q":3418,"a":3504},"Compare a checksum of the local file with the one the store reports, or at minimum re-read the object's size. A successful HTTP status only means the request was accepted, not that the bytes are intact.",{},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage",{"title":3508,"description":3509},"Publish Excel Reports to S3, SharePoint and Drive","Upload generated .xlsx reports from Python to cloud storage — boto3 for S3, Microsoft Graph for SharePoint, the Drive API, and SMB shares, with retries and integrity checks.","publishing-excel-reports-to-cloud-storage","automating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Findex","guide","euXUByMzpOp9cHlgswiHr8VSHmeDIcTm0LYI0UwrBx0",[3515,3519],{"title":3516,"path":3517,"stem":3518,"children":-1},"Populate an Excel Template Without Losing Formatting","\u002Fautomating-reporting-workflows\u002Fgenerating-excel-reports-from-templates\u002Fpopulate-excel-template-without-losing-formatting","automating-reporting-workflows\u002Fgenerating-excel-reports-from-templates\u002Fpopulate-excel-template-without-losing-formatting\u002Findex",{"title":3451,"path":3520,"stem":3521,"children":-1},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fsave-excel-reports-to-google-drive-with-python","automating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fsave-excel-reports-to-google-drive-with-python\u002Findex",1786800026511]