[{"data":1,"prerenderedAt":2188},["ShallowReactive",2],{"doc:\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Fwatch-a-folder-and-process-new-excel-files":3,"surround:\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Fwatch-a-folder-and-process-new-excel-files":2154,"ancestors:\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Fwatch-a-folder-and-process-new-excel-files":2163,"children:\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Fwatch-a-folder-and-process-new-excel-files":2168,"related:\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Fwatch-a-folder-and-process-new-excel-files":2169},{"id":4,"title":5,"body":6,"dateModified":2121,"datePublished":2121,"description":2122,"extension":2123,"faq":2124,"keyword":2133,"kind":2134,"meta":2135,"navigation":286,"path":2143,"seo":2144,"slug":2147,"stem":2148,"topics":2149,"type":2152,"__hash__":2153},"docs\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Fwatch-a-folder-and-process-new-excel-files\u002Findex.md","Watch a Folder and Process New Excel Files with Python",{"type":7,"value":8,"toc":2108},"minimark",[9,24,195,200,230,247,251,258,496,503,507,515,764,779,783,794,1249,1256,1260,1263,1644,1654,1847,1851,1866,1958,1962,2013,2017,2025,2029,2043,2047,2053,2059,2065,2071,2075,2104],[10,11,12,13,17,18,23],"p",{},"A drop folder is the simplest integration there is: someone — or some other system — saves a workbook into a folder, and it gets processed. A scheduled job that scans the folder every few minutes works, but a watcher can react within a second of the file arriving. The ",[14,15,16],"code",{},"watchdog"," package receives file system events from the operating system and calls your code. The details that make it reliable are all about what an \"event\" really means: Excel's lock files, files still being copied, the same file reported twice, and files that arrived while the watcher was not running. This page is part of ",[19,20,22],"a",{"href":21},"\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002F","Scheduling Python Excel Scripts with Cron",".",[25,26,34,35,34,39,34,43,34,51,34,61,34,72,34,79,34,84,34,88,34,94,34,99,34,102,34,108,34,112,34,115,34,118,34,122,34,125,34,128,34,134,34,139,34,143,34,147,34,150,34,153,34,159,34,163,34,166,34,169,34,173,34,176,34,179,34,185,34,189,34,192],"svg",{"viewBox":27,"role":28,"ariaLabelledBy":29,"xmlns":32,"style":33},"4 0 752 142","img",[30,31],"watch-flow-t","watch-flow-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[36,37,38],"title",{"id":30},"From a file appearing to a file processed",[40,41,42],"desc",{"id":31},"Five steps. The operating system reports a file created or moved into the drop folder. The handler ignores lock files, temporary files and anything that is not a workbook, and puts real workbooks on a queue. A worker waits until the file has finished copying. It processes the workbook. It moves the file to processed or failed.",[44,45],"rect",{"x":46,"y":47,"width":48,"height":49,"fill":50},"4","0","752","142","#ffffff",[44,52],{"x":53,"y":54,"width":55,"height":56,"rx":57,"fill":58,"stroke":59,"style":60},"20.0","16.0","115.2","110.5","12","#f0f4ff","var(--brand,#5b5cf0)","stroke-width:2px",[62,63,65],"g",{"transform":64},"translate(65.6,29.0) scale(1.000)",[66,67],"path",{"d":68,"fill":69,"stroke":70,"style":71},"M13 2L4 14h7l-1 8 9-12h-7z","none","var(--brand-strong,#4338ca)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[73,74,78],"text",{"x":75,"y":76,"style":77},"77.6","75.5","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Event",[73,80,83],{"x":75,"y":81,"style":82},"95.0","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","created or",[73,85,87],{"x":75,"y":86,"style":82},"112.5","moved",[89,90],"line",{"x1":91,"y1":92,"x2":93,"y2":92,"stroke":59,"style":60},"140.2","71.2","162.2",[95,96],"polygon",{"points":97,"fill":98},"166.2,71.2 157.2,76.2 157.2,66.2","#5b5cf0",[44,100],{"x":101,"y":54,"width":55,"height":56,"rx":57,"fill":58,"stroke":59,"style":60},"171.2",[62,103,105],{"transform":104},"translate(216.8,29.0) scale(1.000)",[66,106],{"d":107,"fill":69,"stroke":70,"style":71},"M3 5h18l-7 8v6l-4 2v-8z",[73,109,111],{"x":110,"y":76,"style":77},"228.8","Filter",[73,113,114],{"x":110,"y":81,"style":82},"skip ~$ and",[73,116,117],{"x":110,"y":86,"style":82},"temp",[89,119],{"x1":120,"y1":92,"x2":121,"y2":92,"stroke":59,"style":60},"291.4","313.4",[95,123],{"points":124,"fill":98},"317.4,71.2 308.4,76.2 308.4,66.2",[44,126],{"x":127,"y":54,"width":55,"height":56,"rx":57,"fill":58,"stroke":59,"style":60},"322.4",[62,129,131],{"transform":130},"translate(368.0,37.8) scale(1.000)",[66,132],{"d":133,"fill":69,"stroke":70,"style":71},"M12 3a9 9 0 1 0 .01 0z M12 7v5l3.5 2",[73,135,138],{"x":136,"y":137,"style":77},"380.0","84.2","Wait",[73,140,142],{"x":136,"y":141,"style":82},"103.8","copy finished",[89,144],{"x1":145,"y1":92,"x2":146,"y2":92,"stroke":59,"style":60},"442.6","464.6",[95,148],{"points":149,"fill":98},"468.6,71.2 459.6,76.2 459.6,66.2",[44,151],{"x":152,"y":54,"width":55,"height":56,"rx":57,"fill":58,"stroke":59,"style":60},"473.6",[62,154,156],{"transform":155},"translate(519.2,29.0) scale(1.000)",[66,157],{"d":158,"fill":69,"stroke":70,"style":71},"M12 9a3 3 0 1 0 .01 0z M12 2.5v3 M12 18.5v3 M2.5 12h3 M18.5 12h3 M5.3 5.3l2.1 2.1 M16.6 16.6l2.1 2.1 M5.3 18.7l2.1-2.1 M16.6 7.4l2.1-2.1",[73,160,162],{"x":161,"y":76,"style":77},"531.2","Process",[73,164,165],{"x":161,"y":81,"style":82},"read the",[73,167,168],{"x":161,"y":86,"style":82},"workbook",[89,170],{"x1":171,"y1":92,"x2":172,"y2":92,"stroke":59,"style":60},"593.8","615.8",[95,174],{"points":175,"fill":98},"619.8,71.2 610.8,76.2 610.8,66.2",[44,177],{"x":178,"y":54,"width":55,"height":56,"rx":57,"fill":58,"stroke":59,"style":60},"624.8",[62,180,182],{"transform":181},"translate(670.4,29.0) scale(1.000)",[66,183],{"d":184,"fill":69,"stroke":70,"style":71},"M3 6h6l2 2h10v11H3z",[73,186,188],{"x":187,"y":76,"style":77},"682.4","Move",[73,190,191],{"x":187,"y":81,"style":82},"processed or",[73,193,194],{"x":187,"y":86,"style":82},"failed",[196,197,199],"h2",{"id":198},"prerequisites","Prerequisites",[201,202,207],"pre",{"className":203,"code":204,"language":205,"meta":206,"style":206},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install watchdog pandas openpyxl\n","bash","",[14,208,209],{"__ignoreMap":206},[210,211,213,217,221,224,227],"span",{"class":89,"line":212},1,[210,214,216],{"class":215},"sMTad","pip",[210,218,220],{"class":219},"srMev"," install",[210,222,223],{"class":219}," watchdog",[210,225,226],{"class":219}," pandas",[210,228,229],{"class":219}," openpyxl\n",[10,231,232,233,236,237,240,241,243,244,246],{},"The examples use three folders: ",[14,234,235],{},"dropbox"," where files arrive, and ",[14,238,239],{},"processed"," and ",[14,242,194],{}," where they go afterwards. One file is already waiting in ",[14,245,235],{}," from before the watcher started.",[196,248,250],{"id":249},"step-1-decide-which-paths-are-workbooks","Step 1 — Decide which paths are workbooks",[10,252,253,254,257],{},"Plenty of files appear in a folder that holds workbooks without being workbooks you should process. Excel creates a lock file named ",[14,255,256],{},"~$"," plus the workbook name while someone has it open; saving creates a temporary file with a random name and then renames it; copy tools create partial files with their own suffixes. Filter on the name before anything else:",[201,259,263],{"className":260,"code":261,"language":262,"meta":206,"style":206},"language-python shiki shiki-themes github-light github-dark-high-contrast","from pathlib import Path\n\nEXTENSIONS = {\".xlsx\", \".xlsm\"}\n\ndef is_workbook(path):\n    p = Path(path)\n    return (p.suffix.lower() in EXTENSIONS\n            and not p.name.startswith((\"~$\", \".~\", \".\"))\n            and p.parent.name == \"dropbox\")\n\nfor name in [\"dropbox\u002Fsales_w39.xlsx\", \"dropbox\u002F~$sales_w39.xlsx\", \"dropbox\u002F7A3F2B10.tmp\",\n             \"dropbox\u002F.~lock.sales_w39.xlsx#\", \"dropbox\u002Fnotes.txt\", \"processed\u002Fsales_w38.xlsx\"]:\n    print(f\"{name:34} {is_workbook(name)}\")\n","python",[14,264,265,281,288,313,318,331,343,358,386,403,408,438,457],{"__ignoreMap":206},[210,266,267,271,275,278],{"class":89,"line":212},[210,268,270],{"class":269},"s-kum","from",[210,272,274],{"class":273},"skGVy"," pathlib ",[210,276,277],{"class":269},"import",[210,279,280],{"class":273}," Path\n",[210,282,284],{"class":89,"line":283},2,[210,285,287],{"emptyLinePlaceholder":286},true,"\n",[210,289,291,295,298,301,304,307,310],{"class":89,"line":290},3,[210,292,294],{"class":293},"sP0c6","EXTENSIONS",[210,296,297],{"class":269}," =",[210,299,300],{"class":273}," {",[210,302,303],{"class":219},"\".xlsx\"",[210,305,306],{"class":273},", ",[210,308,309],{"class":219},"\".xlsm\"",[210,311,312],{"class":273},"}\n",[210,314,316],{"class":89,"line":315},4,[210,317,287],{"emptyLinePlaceholder":286},[210,319,321,324,328],{"class":89,"line":320},5,[210,322,323],{"class":269},"def",[210,325,327],{"class":326},"s_Opv"," is_workbook",[210,329,330],{"class":273},"(path):\n",[210,332,334,337,340],{"class":89,"line":333},6,[210,335,336],{"class":273},"    p ",[210,338,339],{"class":269},"=",[210,341,342],{"class":273}," Path(path)\n",[210,344,346,349,352,355],{"class":89,"line":345},7,[210,347,348],{"class":269},"    return",[210,350,351],{"class":273}," (p.suffix.lower() ",[210,353,354],{"class":269},"in",[210,356,357],{"class":293}," EXTENSIONS\n",[210,359,361,364,367,370,373,375,378,380,383],{"class":89,"line":360},8,[210,362,363],{"class":269},"            and",[210,365,366],{"class":269}," not",[210,368,369],{"class":273}," p.name.startswith((",[210,371,372],{"class":219},"\"~$\"",[210,374,306],{"class":273},[210,376,377],{"class":219},"\".~\"",[210,379,306],{"class":273},[210,381,382],{"class":219},"\".\"",[210,384,385],{"class":273},"))\n",[210,387,389,391,394,397,400],{"class":89,"line":388},9,[210,390,363],{"class":269},[210,392,393],{"class":273}," p.parent.name ",[210,395,396],{"class":269},"==",[210,398,399],{"class":219}," \"dropbox\"",[210,401,402],{"class":273},")\n",[210,404,406],{"class":89,"line":405},10,[210,407,287],{"emptyLinePlaceholder":286},[210,409,411,414,417,419,422,425,427,430,432,435],{"class":89,"line":410},11,[210,412,413],{"class":269},"for",[210,415,416],{"class":273}," name ",[210,418,354],{"class":269},[210,420,421],{"class":273}," [",[210,423,424],{"class":219},"\"dropbox\u002Fsales_w39.xlsx\"",[210,426,306],{"class":273},[210,428,429],{"class":219},"\"dropbox\u002F~$sales_w39.xlsx\"",[210,431,306],{"class":273},[210,433,434],{"class":219},"\"dropbox\u002F7A3F2B10.tmp\"",[210,436,437],{"class":273},",\n",[210,439,441,444,446,449,451,454],{"class":89,"line":440},12,[210,442,443],{"class":219},"             \"dropbox\u002F.~lock.sales_w39.xlsx#\"",[210,445,306],{"class":273},[210,447,448],{"class":219},"\"dropbox\u002Fnotes.txt\"",[210,450,306],{"class":273},[210,452,453],{"class":219},"\"processed\u002Fsales_w38.xlsx\"",[210,455,456],{"class":273},"]:\n",[210,458,460,463,466,469,472,476,479,482,485,487,490,492,494],{"class":89,"line":459},13,[210,461,462],{"class":293},"    print",[210,464,465],{"class":273},"(",[210,467,468],{"class":269},"f",[210,470,471],{"class":219},"\"",[210,473,475],{"class":474},"sSjpA","{",[210,477,478],{"class":273},"name",[210,480,481],{"class":269},":34",[210,483,484],{"class":474},"}",[210,486,300],{"class":474},[210,488,489],{"class":273},"is_workbook(name)",[210,491,484],{"class":474},[210,493,471],{"class":219},[210,495,402],{"class":273},[10,497,498,499,502],{},"Only the real workbook passes. Checking the parent folder matters if the watcher is recursive or if the processed folder sits inside the drop folder — otherwise moving a file after processing looks like a new file arriving. LibreOffice's lock files start with ",[14,500,501],{},".~lock.",", which the check also skips.",[196,504,506],{"id":505},"step-2-wait-until-the-file-has-finished-arriving","Step 2 — Wait until the file has finished arriving",[10,508,509,510,514],{},"The created event fires when a file ",[511,512,513],"em",{},"appears",", not when it is complete. A large workbook copied over a network appears at 0 bytes and grows for seconds. Wait until its size has been stable for a moment and it can actually be opened as a workbook:",[201,516,518],{"className":260,"code":517,"language":262,"meta":206,"style":206},"import time\nimport zipfile\n\ndef wait_until_complete(path, stable_for=1.0, timeout=60.0, poll=0.25):\n    deadline = time.monotonic() + timeout\n    last_size, stable_since = -1, None\n    while time.monotonic() \u003C deadline:\n        try:\n            size = Path(path).stat().st_size\n        except FileNotFoundError:\n            return False\n        if size == last_size and size > 0:\n            stable_since = stable_since or time.monotonic()\n            if time.monotonic() - stable_since >= stable_for and zipfile.is_zipfile(path):\n                return True\n        else:\n            last_size, stable_since = size, None\n        time.sleep(poll)\n    return False\n",[14,519,520,527,534,538,572,588,606,619,627,637,647,655,681,697,721,730,738,751,757],{"__ignoreMap":206},[210,521,522,524],{"class":89,"line":212},[210,523,277],{"class":269},[210,525,526],{"class":273}," time\n",[210,528,529,531],{"class":89,"line":283},[210,530,277],{"class":269},[210,532,533],{"class":273}," zipfile\n",[210,535,536],{"class":89,"line":290},[210,537,287],{"emptyLinePlaceholder":286},[210,539,540,542,545,548,550,553,556,558,561,564,566,569],{"class":89,"line":315},[210,541,323],{"class":269},[210,543,544],{"class":326}," wait_until_complete",[210,546,547],{"class":273},"(path, stable_for",[210,549,339],{"class":269},[210,551,552],{"class":293},"1.0",[210,554,555],{"class":273},", timeout",[210,557,339],{"class":269},[210,559,560],{"class":293},"60.0",[210,562,563],{"class":273},", poll",[210,565,339],{"class":269},[210,567,568],{"class":293},"0.25",[210,570,571],{"class":273},"):\n",[210,573,574,577,579,582,585],{"class":89,"line":320},[210,575,576],{"class":273},"    deadline ",[210,578,339],{"class":269},[210,580,581],{"class":273}," time.monotonic() ",[210,583,584],{"class":269},"+",[210,586,587],{"class":273}," timeout\n",[210,589,590,593,595,598,601,603],{"class":89,"line":333},[210,591,592],{"class":273},"    last_size, stable_since ",[210,594,339],{"class":269},[210,596,597],{"class":269}," -",[210,599,600],{"class":293},"1",[210,602,306],{"class":273},[210,604,605],{"class":293},"None\n",[210,607,608,611,613,616],{"class":89,"line":345},[210,609,610],{"class":269},"    while",[210,612,581],{"class":273},[210,614,615],{"class":269},"\u003C",[210,617,618],{"class":273}," deadline:\n",[210,620,621,624],{"class":89,"line":360},[210,622,623],{"class":269},"        try",[210,625,626],{"class":273},":\n",[210,628,629,632,634],{"class":89,"line":388},[210,630,631],{"class":273},"            size ",[210,633,339],{"class":269},[210,635,636],{"class":273}," Path(path).stat().st_size\n",[210,638,639,642,645],{"class":89,"line":405},[210,640,641],{"class":269},"        except",[210,643,644],{"class":293}," FileNotFoundError",[210,646,626],{"class":273},[210,648,649,652],{"class":89,"line":410},[210,650,651],{"class":269},"            return",[210,653,654],{"class":293}," False\n",[210,656,657,660,663,665,668,671,673,676,679],{"class":89,"line":440},[210,658,659],{"class":269},"        if",[210,661,662],{"class":273}," size ",[210,664,396],{"class":269},[210,666,667],{"class":273}," last_size ",[210,669,670],{"class":269},"and",[210,672,662],{"class":273},[210,674,675],{"class":269},">",[210,677,678],{"class":293}," 0",[210,680,626],{"class":273},[210,682,683,686,688,691,694],{"class":89,"line":459},[210,684,685],{"class":273},"            stable_since ",[210,687,339],{"class":269},[210,689,690],{"class":273}," stable_since ",[210,692,693],{"class":269},"or",[210,695,696],{"class":273}," time.monotonic()\n",[210,698,700,703,705,708,710,713,716,718],{"class":89,"line":699},14,[210,701,702],{"class":269},"            if",[210,704,581],{"class":273},[210,706,707],{"class":269},"-",[210,709,690],{"class":273},[210,711,712],{"class":269},">=",[210,714,715],{"class":273}," stable_for ",[210,717,670],{"class":269},[210,719,720],{"class":273}," zipfile.is_zipfile(path):\n",[210,722,724,727],{"class":89,"line":723},15,[210,725,726],{"class":269},"                return",[210,728,729],{"class":293}," True\n",[210,731,733,736],{"class":89,"line":732},16,[210,734,735],{"class":269},"        else",[210,737,626],{"class":273},[210,739,741,744,746,749],{"class":89,"line":740},17,[210,742,743],{"class":273},"            last_size, stable_since ",[210,745,339],{"class":269},[210,747,748],{"class":273}," size, ",[210,750,605],{"class":293},[210,752,754],{"class":89,"line":753},18,[210,755,756],{"class":273},"        time.sleep(poll)\n",[210,758,760,762],{"class":89,"line":759},19,[210,761,348],{"class":269},[210,763,654],{"class":293},[10,765,766,767,770,771,774,775,778],{},"An ",[14,768,769],{},".xlsx"," is a zip archive, and a truncated one is not a valid zip, so ",[14,772,773],{},"zipfile.is_zipfile"," is a cheap final test that the copy is complete. If the file disappears while waiting — renamed or deleted by whoever put it there — the function returns ",[14,776,777],{},"False"," and the file is skipped.",[196,780,782],{"id":781},"step-3-watch-the-folder-and-hand-files-to-a-worker","Step 3 — Watch the folder and hand files to a worker",[10,784,785,786,789,790,793],{},"The event handler runs on watchdog's own thread and should return quickly. Put each workbook path on a queue; a single worker thread takes paths from the queue, waits for them to complete and processes them one at a time. Handle ",[14,787,788],{},"on_moved"," as well as ",[14,791,792],{},"on_created",", because Excel and many copy tools write a temporary name and then rename it:",[201,795,797],{"className":260,"code":796,"language":262,"meta":206,"style":206},"import queue\nimport shutil\nimport threading\nimport pandas as pd\nfrom watchdog.events import FileSystemEventHandler\nfrom watchdog.observers import Observer\n\nwork = queue.Queue()\nlog = []\n\nclass DropHandler(FileSystemEventHandler):\n    def on_created(self, event):\n        if not event.is_directory and is_workbook(event.src_path):\n            work.put(event.src_path)\n\n    def on_moved(self, event):\n        if not event.is_directory and is_workbook(event.dest_path):\n            work.put(event.dest_path)\n\ndef process(path):\n    df = pd.read_excel(path)\n    return f\"{len(df)} rows, revenue {df['Revenue'].sum():,}\"\n\ndef worker():\n    seen = set()\n    while (path := work.get()) is not None:\n        if path in seen or not Path(path).exists():\n            continue\n        seen.add(path)\n        if not wait_until_complete(path, timeout=10):\n            log.append((Path(path).name, \"gave up waiting\"))\n            continue\n        try:\n            result = process(path)\n            shutil.move(path, Path(\"processed\") \u002F Path(path).name)\n            log.append((Path(path).name, result))\n        except Exception as exc:\n            shutil.move(path, Path(\"failed\") \u002F Path(path).name)\n            log.append((Path(path).name, f\"failed: {type(exc).__name__}\"))\n",[14,798,799,806,813,820,833,845,857,861,871,881,885,901,912,926,931,935,944,957,962,966,976,987,1029,1034,1045,1059,1083,1103,1109,1115,1136,1147,1152,1159,1170,1188,1194,1208,1222],{"__ignoreMap":206},[210,800,801,803],{"class":89,"line":212},[210,802,277],{"class":269},[210,804,805],{"class":273}," queue\n",[210,807,808,810],{"class":89,"line":283},[210,809,277],{"class":269},[210,811,812],{"class":273}," shutil\n",[210,814,815,817],{"class":89,"line":290},[210,816,277],{"class":269},[210,818,819],{"class":273}," threading\n",[210,821,822,824,827,830],{"class":89,"line":315},[210,823,277],{"class":269},[210,825,826],{"class":273}," pandas ",[210,828,829],{"class":269},"as",[210,831,832],{"class":273}," pd\n",[210,834,835,837,840,842],{"class":89,"line":320},[210,836,270],{"class":269},[210,838,839],{"class":273}," watchdog.events ",[210,841,277],{"class":269},[210,843,844],{"class":273}," FileSystemEventHandler\n",[210,846,847,849,852,854],{"class":89,"line":333},[210,848,270],{"class":269},[210,850,851],{"class":273}," watchdog.observers ",[210,853,277],{"class":269},[210,855,856],{"class":273}," Observer\n",[210,858,859],{"class":89,"line":345},[210,860,287],{"emptyLinePlaceholder":286},[210,862,863,866,868],{"class":89,"line":360},[210,864,865],{"class":273},"work ",[210,867,339],{"class":269},[210,869,870],{"class":273}," queue.Queue()\n",[210,872,873,876,878],{"class":89,"line":388},[210,874,875],{"class":273},"log ",[210,877,339],{"class":269},[210,879,880],{"class":273}," []\n",[210,882,883],{"class":89,"line":405},[210,884,287],{"emptyLinePlaceholder":286},[210,886,887,890,893,895,899],{"class":89,"line":410},[210,888,889],{"class":269},"class",[210,891,892],{"class":215}," DropHandler",[210,894,465],{"class":273},[210,896,898],{"class":897},"sHg5J","FileSystemEventHandler",[210,900,571],{"class":273},[210,902,903,906,909],{"class":89,"line":440},[210,904,905],{"class":269},"    def",[210,907,908],{"class":326}," on_created",[210,910,911],{"class":273},"(self, event):\n",[210,913,914,916,918,921,923],{"class":89,"line":459},[210,915,659],{"class":269},[210,917,366],{"class":269},[210,919,920],{"class":273}," event.is_directory ",[210,922,670],{"class":269},[210,924,925],{"class":273}," is_workbook(event.src_path):\n",[210,927,928],{"class":89,"line":699},[210,929,930],{"class":273},"            work.put(event.src_path)\n",[210,932,933],{"class":89,"line":723},[210,934,287],{"emptyLinePlaceholder":286},[210,936,937,939,942],{"class":89,"line":732},[210,938,905],{"class":269},[210,940,941],{"class":326}," on_moved",[210,943,911],{"class":273},[210,945,946,948,950,952,954],{"class":89,"line":740},[210,947,659],{"class":269},[210,949,366],{"class":269},[210,951,920],{"class":273},[210,953,670],{"class":269},[210,955,956],{"class":273}," is_workbook(event.dest_path):\n",[210,958,959],{"class":89,"line":753},[210,960,961],{"class":273},"            work.put(event.dest_path)\n",[210,963,964],{"class":89,"line":759},[210,965,287],{"emptyLinePlaceholder":286},[210,967,969,971,974],{"class":89,"line":968},20,[210,970,323],{"class":269},[210,972,973],{"class":326}," process",[210,975,330],{"class":273},[210,977,979,982,984],{"class":89,"line":978},21,[210,980,981],{"class":273},"    df ",[210,983,339],{"class":269},[210,985,986],{"class":273}," pd.read_excel(path)\n",[210,988,990,992,995,997,999,1002,1005,1007,1010,1012,1015,1018,1021,1024,1026],{"class":89,"line":989},22,[210,991,348],{"class":269},[210,993,994],{"class":269}," f",[210,996,471],{"class":219},[210,998,475],{"class":474},[210,1000,1001],{"class":293},"len",[210,1003,1004],{"class":273},"(df)",[210,1006,484],{"class":474},[210,1008,1009],{"class":219}," rows, revenue ",[210,1011,475],{"class":474},[210,1013,1014],{"class":273},"df[",[210,1016,1017],{"class":219},"'Revenue'",[210,1019,1020],{"class":273},"].sum()",[210,1022,1023],{"class":269},":,",[210,1025,484],{"class":474},[210,1027,1028],{"class":219},"\"\n",[210,1030,1032],{"class":89,"line":1031},23,[210,1033,287],{"emptyLinePlaceholder":286},[210,1035,1037,1039,1042],{"class":89,"line":1036},24,[210,1038,323],{"class":269},[210,1040,1041],{"class":326}," worker",[210,1043,1044],{"class":273},"():\n",[210,1046,1048,1051,1053,1056],{"class":89,"line":1047},25,[210,1049,1050],{"class":273},"    seen ",[210,1052,339],{"class":269},[210,1054,1055],{"class":293}," set",[210,1057,1058],{"class":273},"()\n",[210,1060,1062,1064,1067,1070,1073,1076,1078,1081],{"class":89,"line":1061},26,[210,1063,610],{"class":269},[210,1065,1066],{"class":273}," (path ",[210,1068,1069],{"class":269},":=",[210,1071,1072],{"class":273}," work.get()) ",[210,1074,1075],{"class":269},"is",[210,1077,366],{"class":269},[210,1079,1080],{"class":293}," None",[210,1082,626],{"class":273},[210,1084,1086,1088,1091,1093,1096,1098,1100],{"class":89,"line":1085},27,[210,1087,659],{"class":269},[210,1089,1090],{"class":273}," path ",[210,1092,354],{"class":269},[210,1094,1095],{"class":273}," seen ",[210,1097,693],{"class":269},[210,1099,366],{"class":269},[210,1101,1102],{"class":273}," Path(path).exists():\n",[210,1104,1106],{"class":89,"line":1105},28,[210,1107,1108],{"class":269},"            continue\n",[210,1110,1112],{"class":89,"line":1111},29,[210,1113,1114],{"class":273},"        seen.add(path)\n",[210,1116,1118,1120,1122,1125,1129,1131,1134],{"class":89,"line":1117},30,[210,1119,659],{"class":269},[210,1121,366],{"class":269},[210,1123,1124],{"class":273}," wait_until_complete(path, ",[210,1126,1128],{"class":1127},"sa561","timeout",[210,1130,339],{"class":269},[210,1132,1133],{"class":293},"10",[210,1135,571],{"class":273},[210,1137,1139,1142,1145],{"class":89,"line":1138},31,[210,1140,1141],{"class":273},"            log.append((Path(path).name, ",[210,1143,1144],{"class":219},"\"gave up waiting\"",[210,1146,385],{"class":273},[210,1148,1150],{"class":89,"line":1149},32,[210,1151,1108],{"class":269},[210,1153,1155,1157],{"class":89,"line":1154},33,[210,1156,623],{"class":269},[210,1158,626],{"class":273},[210,1160,1162,1165,1167],{"class":89,"line":1161},34,[210,1163,1164],{"class":273},"            result ",[210,1166,339],{"class":269},[210,1168,1169],{"class":273}," process(path)\n",[210,1171,1173,1176,1179,1182,1185],{"class":89,"line":1172},35,[210,1174,1175],{"class":273},"            shutil.move(path, Path(",[210,1177,1178],{"class":219},"\"processed\"",[210,1180,1181],{"class":273},") ",[210,1183,1184],{"class":269},"\u002F",[210,1186,1187],{"class":273}," Path(path).name)\n",[210,1189,1191],{"class":89,"line":1190},36,[210,1192,1193],{"class":273},"            log.append((Path(path).name, result))\n",[210,1195,1197,1199,1202,1205],{"class":89,"line":1196},37,[210,1198,641],{"class":269},[210,1200,1201],{"class":293}," Exception",[210,1203,1204],{"class":269}," as",[210,1206,1207],{"class":273}," exc:\n",[210,1209,1211,1213,1216,1218,1220],{"class":89,"line":1210},38,[210,1212,1175],{"class":273},[210,1214,1215],{"class":219},"\"failed\"",[210,1217,1181],{"class":273},[210,1219,1184],{"class":269},[210,1221,1187],{"class":273},[210,1223,1225,1227,1229,1232,1234,1237,1240,1243,1245,1247],{"class":89,"line":1224},39,[210,1226,1141],{"class":273},[210,1228,468],{"class":269},[210,1230,1231],{"class":219},"\"failed: ",[210,1233,475],{"class":474},[210,1235,1236],{"class":293},"type",[210,1238,1239],{"class":273},"(exc).",[210,1241,1242],{"class":293},"__name__",[210,1244,484],{"class":474},[210,1246,471],{"class":219},[210,1248,385],{"class":273},[10,1250,1251,1252,1255],{},"The ",[14,1253,1254],{},"seen"," set drops duplicate events — some platforms report a created and a modified event, or two created events, for one copy. Moving each file out of the drop folder once handled makes the folder itself a to-do list: anything still in it has not been processed.",[196,1257,1259],{"id":1258},"step-4-start-the-watcher-including-files-already-waiting","Step 4 — Start the watcher, including files already waiting",[10,1261,1262],{},"Files that arrived while the watcher was stopped produce no events. Queue everything already in the folder when the watcher starts, then begin watching:",[201,1264,1266],{"className":260,"code":1265,"language":262,"meta":206,"style":206},"import io\n\nobserver = Observer()\nobserver.schedule(DropHandler(), \"dropbox\", recursive=False)\nthread = threading.Thread(target=worker, daemon=True)\nthread.start()\nobserver.start()\nfor existing in sorted(Path(\"dropbox\").iterdir()):\n    if is_workbook(existing):\n        work.put(str(existing))\n\npd.DataFrame({\"Region\": [\"North\", \"South\"], \"Revenue\": [48200, 39100]}).to_excel(\"dropbox\u002Fsales_w39.xlsx\", index=False)\nPath(\"dropbox\u002F~$sales_w39.xlsx\").write_bytes(b\"lock\")\nbuf = io.BytesIO()\npd.DataFrame({\"Region\": [\"East\"], \"Revenue\": [51750]}).to_excel(buf, index=False)\nPath(\"dropbox\u002Ftmp_save.partial\").write_bytes(buf.getvalue())\nPath(\"dropbox\u002Ftmp_save.partial\").rename(\"dropbox\u002Fsales_east.xlsx\")\nPath(\"dropbox\u002Fbroken.xlsx\").write_bytes(b\"not a workbook at all\")\n\ntime.sleep(6)\nobserver.stop(); observer.join()\nwork.put(None); thread.join()\nfor name, outcome in log:\n    print(f\"{name:28} {outcome}\")\nprint(\"left in dropbox:\", sorted(p.name for p in Path(\"dropbox\").iterdir()))\n",[14,1267,1268,1275,1279,1289,1308,1336,1341,1346,1366,1374,1385,1389,1440,1458,1468,1499,1509,1523,1539,1543,1553,1558,1569,1581,1611],{"__ignoreMap":206},[210,1269,1270,1272],{"class":89,"line":212},[210,1271,277],{"class":269},[210,1273,1274],{"class":273}," io\n",[210,1276,1277],{"class":89,"line":283},[210,1278,287],{"emptyLinePlaceholder":286},[210,1280,1281,1284,1286],{"class":89,"line":290},[210,1282,1283],{"class":273},"observer ",[210,1285,339],{"class":269},[210,1287,1288],{"class":273}," Observer()\n",[210,1290,1291,1294,1297,1299,1302,1304,1306],{"class":89,"line":315},[210,1292,1293],{"class":273},"observer.schedule(DropHandler(), ",[210,1295,1296],{"class":219},"\"dropbox\"",[210,1298,306],{"class":273},[210,1300,1301],{"class":1127},"recursive",[210,1303,339],{"class":269},[210,1305,777],{"class":293},[210,1307,402],{"class":273},[210,1309,1310,1313,1315,1318,1321,1323,1326,1329,1331,1334],{"class":89,"line":320},[210,1311,1312],{"class":273},"thread ",[210,1314,339],{"class":269},[210,1316,1317],{"class":273}," threading.Thread(",[210,1319,1320],{"class":1127},"target",[210,1322,339],{"class":269},[210,1324,1325],{"class":273},"worker, ",[210,1327,1328],{"class":1127},"daemon",[210,1330,339],{"class":269},[210,1332,1333],{"class":293},"True",[210,1335,402],{"class":273},[210,1337,1338],{"class":89,"line":333},[210,1339,1340],{"class":273},"thread.start()\n",[210,1342,1343],{"class":89,"line":345},[210,1344,1345],{"class":273},"observer.start()\n",[210,1347,1348,1350,1353,1355,1358,1361,1363],{"class":89,"line":360},[210,1349,413],{"class":269},[210,1351,1352],{"class":273}," existing ",[210,1354,354],{"class":269},[210,1356,1357],{"class":293}," sorted",[210,1359,1360],{"class":273},"(Path(",[210,1362,1296],{"class":219},[210,1364,1365],{"class":273},").iterdir()):\n",[210,1367,1368,1371],{"class":89,"line":388},[210,1369,1370],{"class":269},"    if",[210,1372,1373],{"class":273}," is_workbook(existing):\n",[210,1375,1376,1379,1382],{"class":89,"line":405},[210,1377,1378],{"class":273},"        work.put(",[210,1380,1381],{"class":293},"str",[210,1383,1384],{"class":273},"(existing))\n",[210,1386,1387],{"class":89,"line":410},[210,1388,287],{"emptyLinePlaceholder":286},[210,1390,1391,1394,1397,1400,1403,1405,1408,1411,1414,1416,1419,1421,1424,1427,1429,1431,1434,1436,1438],{"class":89,"line":440},[210,1392,1393],{"class":273},"pd.DataFrame({",[210,1395,1396],{"class":219},"\"Region\"",[210,1398,1399],{"class":273},": [",[210,1401,1402],{"class":219},"\"North\"",[210,1404,306],{"class":273},[210,1406,1407],{"class":219},"\"South\"",[210,1409,1410],{"class":273},"], ",[210,1412,1413],{"class":219},"\"Revenue\"",[210,1415,1399],{"class":273},[210,1417,1418],{"class":293},"48200",[210,1420,306],{"class":273},[210,1422,1423],{"class":293},"39100",[210,1425,1426],{"class":273},"]}).to_excel(",[210,1428,424],{"class":219},[210,1430,306],{"class":273},[210,1432,1433],{"class":1127},"index",[210,1435,339],{"class":269},[210,1437,777],{"class":293},[210,1439,402],{"class":273},[210,1441,1442,1445,1447,1450,1453,1456],{"class":89,"line":459},[210,1443,1444],{"class":273},"Path(",[210,1446,429],{"class":219},[210,1448,1449],{"class":273},").write_bytes(",[210,1451,1452],{"class":269},"b",[210,1454,1455],{"class":219},"\"lock\"",[210,1457,402],{"class":273},[210,1459,1460,1463,1465],{"class":89,"line":699},[210,1461,1462],{"class":273},"buf ",[210,1464,339],{"class":269},[210,1466,1467],{"class":273}," io.BytesIO()\n",[210,1469,1470,1472,1474,1476,1479,1481,1483,1485,1488,1491,1493,1495,1497],{"class":89,"line":723},[210,1471,1393],{"class":273},[210,1473,1396],{"class":219},[210,1475,1399],{"class":273},[210,1477,1478],{"class":219},"\"East\"",[210,1480,1410],{"class":273},[210,1482,1413],{"class":219},[210,1484,1399],{"class":273},[210,1486,1487],{"class":293},"51750",[210,1489,1490],{"class":273},"]}).to_excel(buf, ",[210,1492,1433],{"class":1127},[210,1494,339],{"class":269},[210,1496,777],{"class":293},[210,1498,402],{"class":273},[210,1500,1501,1503,1506],{"class":89,"line":732},[210,1502,1444],{"class":273},[210,1504,1505],{"class":219},"\"dropbox\u002Ftmp_save.partial\"",[210,1507,1508],{"class":273},").write_bytes(buf.getvalue())\n",[210,1510,1511,1513,1515,1518,1521],{"class":89,"line":740},[210,1512,1444],{"class":273},[210,1514,1505],{"class":219},[210,1516,1517],{"class":273},").rename(",[210,1519,1520],{"class":219},"\"dropbox\u002Fsales_east.xlsx\"",[210,1522,402],{"class":273},[210,1524,1525,1527,1530,1532,1534,1537],{"class":89,"line":753},[210,1526,1444],{"class":273},[210,1528,1529],{"class":219},"\"dropbox\u002Fbroken.xlsx\"",[210,1531,1449],{"class":273},[210,1533,1452],{"class":269},[210,1535,1536],{"class":219},"\"not a workbook at all\"",[210,1538,402],{"class":273},[210,1540,1541],{"class":89,"line":759},[210,1542,287],{"emptyLinePlaceholder":286},[210,1544,1545,1548,1551],{"class":89,"line":968},[210,1546,1547],{"class":273},"time.sleep(",[210,1549,1550],{"class":293},"6",[210,1552,402],{"class":273},[210,1554,1555],{"class":89,"line":978},[210,1556,1557],{"class":273},"observer.stop(); observer.join()\n",[210,1559,1560,1563,1566],{"class":89,"line":989},[210,1561,1562],{"class":273},"work.put(",[210,1564,1565],{"class":293},"None",[210,1567,1568],{"class":273},"); thread.join()\n",[210,1570,1571,1573,1576,1578],{"class":89,"line":1031},[210,1572,413],{"class":269},[210,1574,1575],{"class":273}," name, outcome ",[210,1577,354],{"class":269},[210,1579,1580],{"class":273}," log:\n",[210,1582,1583,1585,1587,1589,1591,1593,1595,1598,1600,1602,1605,1607,1609],{"class":89,"line":1036},[210,1584,462],{"class":293},[210,1586,465],{"class":273},[210,1588,468],{"class":269},[210,1590,471],{"class":219},[210,1592,475],{"class":474},[210,1594,478],{"class":273},[210,1596,1597],{"class":269},":28",[210,1599,484],{"class":474},[210,1601,300],{"class":474},[210,1603,1604],{"class":273},"outcome",[210,1606,484],{"class":474},[210,1608,471],{"class":219},[210,1610,402],{"class":273},[210,1612,1613,1616,1618,1621,1623,1626,1629,1631,1634,1636,1639,1641],{"class":89,"line":1047},[210,1614,1615],{"class":293},"print",[210,1617,465],{"class":273},[210,1619,1620],{"class":219},"\"left in dropbox:\"",[210,1622,306],{"class":273},[210,1624,1625],{"class":293},"sorted",[210,1627,1628],{"class":273},"(p.name ",[210,1630,413],{"class":269},[210,1632,1633],{"class":273}," p ",[210,1635,354],{"class":269},[210,1637,1638],{"class":273}," Path(",[210,1640,1296],{"class":219},[210,1642,1643],{"class":273},").iterdir()))\n",[10,1645,1646,1647,1650,1651,1653],{},"The file left over from yesterday is processed first, from the startup scan. The new workbook arrives through a created event and the renamed one through a moved event; the lock file is ignored. ",[14,1648,1649],{},"broken.xlsx"," never becomes a valid zip, so the worker gives up waiting after its timeout and leaves it in place for a person to look at. The East file imitates a program that writes under a temporary name and renames the finished file into place — the pattern that makes ",[14,1652,788],{}," necessary. In a long-running service the watcher runs until stopped; here it is stopped after a few seconds to show the result.",[25,1655,34,1661,34,1664,34,1667,34,1672,34,1677,34,1686,34,1690,34,1696,34,1700,34,1704,34,1708,34,1712,34,1716,34,1720,34,1722,34,1726,34,1728,34,1731,34,1733,34,1735,34,1738,34,1742,34,1744,34,1748,34,1750,34,1753,34,1755,34,1758,34,1761,34,1765,34,1767,34,1770,34,1772,34,1775,34,1777,34,1780,34,1783,34,1786,34,1788,34,1791,34,1793,34,1796,34,1798,34,1800,34,1803,34,1807,34,1809,34,1811,34,1813,34,1815,34,1820,34,1824,34,1827,34,1830,34,1832,34,1835,34,1837,34,1839,34,1843],{"viewBox":1656,"role":28,"ariaLabelledBy":1657,"xmlns":32,"style":1660},"112 0 535 256",[1658,1659],"watch-log-t","watch-log-d","width:100%;max-width:535px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[36,1662,1663],{"id":1658},"What the watcher did with each file",[40,1665,1666],{"id":1659},"The watcher log. left_over_from_yesterday.xlsx, found by the startup scan, was processed with 1 row. sales_w39.xlsx arrived as a new file and was processed with 2 rows. sales_east.xlsx arrived by rename and was processed with 1 row. broken.xlsx never became a valid workbook, so the worker gave up waiting and left it in the drop folder. The lock file ~$sales_w39.xlsx was ignored.",[44,1668],{"x":1669,"y":47,"width":1670,"height":1671,"fill":50},"112","535","256",[73,1673,1676],{"x":136,"y":1674,"style":1675},"31.0","font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Watcher log",[44,1678],{"x":1679,"y":1680,"width":1681,"height":1682,"rx":47,"fill":1683,"stroke":1684,"style":1685},"128.9","48.0","32.0","24.0","#e7ebef","var(--line,#cdd5e6)","stroke-width:1px",[44,1687],{"x":1688,"y":1680,"width":1689,"height":1682,"rx":47,"fill":1683,"stroke":1684,"style":1685},"160.9","211.4",[73,1691,1695],{"x":1692,"y":1693,"style":1694},"266.6","65.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[44,1697],{"x":1698,"y":1680,"width":1699,"height":1682,"rx":47,"fill":1683,"stroke":1684,"style":1685},"372.3","109.5",[73,1701,1703],{"x":1702,"y":1693,"style":1694},"427.0","B",[44,1705],{"x":1706,"y":1680,"width":1707,"height":1682,"rx":47,"fill":1683,"stroke":1684,"style":1685},"481.7","149.4",[73,1709,1711],{"x":1710,"y":1693,"style":1694},"556.4","C",[44,1713],{"x":1679,"y":1714,"width":1681,"height":1715,"rx":47,"fill":1683,"stroke":1684,"style":1685},"72.0","28.0",[73,1717,600],{"x":1718,"y":1719,"style":1694},"144.9","91.0",[44,1721],{"x":1688,"y":1714,"width":1689,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1723,1725],{"x":1692,"y":1719,"style":1724},"font-size:12px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","file",[44,1727],{"x":1698,"y":1714,"width":1699,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1729,1730],{"x":1702,"y":1719,"style":1724},"arrived by",[44,1732],{"x":1706,"y":1714,"width":1707,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1734,1604],{"x":1710,"y":1719,"style":1724},[44,1736],{"x":1679,"y":1737,"width":1681,"height":1715,"rx":47,"fill":1683,"stroke":1684,"style":1685},"100.0",[73,1739,1741],{"x":1718,"y":1740,"style":1694},"119.0","2",[44,1743],{"x":1688,"y":1737,"width":1689,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1745,1747],{"x":1692,"y":1740,"style":1746},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","left_over_from_yesterday.xlsx",[44,1749],{"x":1698,"y":1737,"width":1699,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1751,1752],{"x":1702,"y":1740,"style":1746},"startup scan",[44,1754],{"x":1706,"y":1737,"width":1707,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1756,1757],{"x":1710,"y":1740,"style":1746},"processed — 1 row",[44,1759],{"x":1679,"y":1760,"width":1681,"height":1715,"rx":47,"fill":1683,"stroke":1684,"style":1685},"128.0",[73,1762,1764],{"x":1718,"y":1763,"style":1694},"147.0","3",[44,1766],{"x":1688,"y":1760,"width":1689,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1768,1769],{"x":1692,"y":1763,"style":1746},"sales_w39.xlsx",[44,1771],{"x":1698,"y":1760,"width":1699,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1773,1774],{"x":1702,"y":1763,"style":1746},"created event",[44,1776],{"x":1706,"y":1760,"width":1707,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1778,1779],{"x":1710,"y":1763,"style":1746},"processed — 2 rows",[44,1781],{"x":1679,"y":1782,"width":1681,"height":1715,"rx":47,"fill":1683,"stroke":1684,"style":1685},"156.0",[73,1784,46],{"x":1718,"y":1785,"style":1694},"175.0",[44,1787],{"x":1688,"y":1782,"width":1689,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1789,1790],{"x":1692,"y":1785,"style":1746},"sales_east.xlsx",[44,1792],{"x":1698,"y":1782,"width":1699,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1794,1795],{"x":1702,"y":1785,"style":1746},"moved event",[44,1797],{"x":1706,"y":1782,"width":1707,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1799,1757],{"x":1710,"y":1785,"style":1746},[44,1801],{"x":1679,"y":1802,"width":1681,"height":1715,"rx":47,"fill":1683,"stroke":1684,"style":1685},"184.0",[73,1804,1806],{"x":1718,"y":1805,"style":1694},"203.0","5",[44,1808],{"x":1688,"y":1802,"width":1689,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1810,1649],{"x":1692,"y":1805,"style":1746},[44,1812],{"x":1698,"y":1802,"width":1699,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1814,1774],{"x":1702,"y":1805,"style":1746},[44,1816],{"x":1706,"y":1802,"width":1707,"height":1715,"rx":47,"fill":1817,"stroke":1818,"style":1819},"#fee8f2","var(--accent,#f43f8f)","stroke-width:1.5px",[73,1821,1823],{"x":1710,"y":1805,"style":1822},"font-size:12px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","gave up waiting",[44,1825],{"x":1679,"y":1826,"width":1681,"height":1715,"rx":47,"fill":1683,"stroke":1684,"style":1685},"212.0",[73,1828,1550],{"x":1718,"y":1829,"style":1694},"231.0",[44,1831],{"x":1688,"y":1826,"width":1689,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1833,1834],{"x":1692,"y":1829,"style":1746},"~$sales_w39.xlsx",[44,1836],{"x":1698,"y":1826,"width":1699,"height":1715,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1838,1774],{"x":1702,"y":1829,"style":1746},[44,1840],{"x":1706,"y":1826,"width":1707,"height":1715,"rx":47,"fill":1841,"stroke":1842,"style":1819},"#fdefd8","var(--gold,#b4740a)",[73,1844,1846],{"x":1710,"y":1829,"style":1845},"font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","ignored",[196,1848,1850],{"id":1849},"network-shares-and-other-places-events-do-not-reach","Network shares and other places events do not reach",[10,1852,1853,1854,1857,1858,1861,1862,1865],{},"Watchdog relies on the operating system's change notifications — inotify on Linux, FSEvents on macOS, ",[14,1855,1856],{},"ReadDirectoryChangesW"," on Windows. Changes made by ",[511,1859,1860],{},"another machine"," to a network share are frequently not reported through them at all, so a watcher on a mapped drive can sit silently while files pile up. For shares, use ",[14,1863,1864],{},"watchdog.observers.polling.PollingObserver",", which lists the folder at an interval and compares, or skip the watcher and run a scheduled scan every few minutes with the same filter, wait and move logic. The startup scan in Step 4 is exactly that scan, so the code is already there.",[25,1867,34,1872,34,1875,34,1878,34,1881,34,1885,34,1890,34,1896,34,1899,34,1903,34,1907,34,1912,34,1914,34,1918,34,1920,34,1923,34,1926,34,1930,34,1932,34,1936,34,1938,34,1941,34,1944,34,1948,34,1950,34,1954,34,1956],{"viewBox":1868,"role":28,"ariaLabelledBy":1869,"xmlns":32,"style":33},"4 0 752 174",[1870,1871],"watch-choice-t","watch-choice-d",[36,1873,1874],{"id":1870},"Choosing how to notice new files",[40,1876,1877],{"id":1871},"A table of approaches. The native watchdog Observer reacts within a second on local disks but may miss changes on network shares. PollingObserver works on network shares and reacts within its polling interval, at the cost of listing the folder repeatedly. A scheduled scan with cron is the simplest and most robust, and reacts within its schedule interval.",[44,1879],{"x":46,"y":47,"width":48,"height":1880,"fill":50},"174",[44,1882],{"x":53,"y":54,"width":1883,"height":1884,"rx":47,"fill":50,"stroke":1684,"style":1685},"155.1","40.0",[44,1886],{"x":1887,"y":54,"width":1888,"height":1884,"rx":47,"fill":58,"stroke":59,"style":1889},"175.1","282.4","stroke-width:1.2px",[73,1891,1895],{"x":1892,"y":1893,"style":1894},"316.4","41.0","font-size:13px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Reacts within",[44,1897],{"x":1898,"y":54,"width":1888,"height":1884,"rx":47,"fill":58,"stroke":59,"style":1889},"457.6",[73,1900,1902],{"x":1901,"y":1893,"style":1894},"598.8","Network shares",[44,1904],{"x":53,"y":1905,"width":1883,"height":1906,"rx":47,"fill":1683,"stroke":1684,"style":1685},"56.0","34.0",[73,1908,1911],{"x":1681,"y":1909,"style":1910},"78.0","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:start","watchdog Observer",[44,1913],{"x":1887,"y":1905,"width":1888,"height":1906,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1915,1917],{"x":1892,"y":1916,"style":82},"77.0","about a second",[44,1919],{"x":1898,"y":1905,"width":1888,"height":1906,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1921,1922],{"x":1901,"y":1916,"style":82},"often misses changes",[44,1924],{"x":53,"y":1925,"width":1883,"height":1906,"rx":47,"fill":1683,"stroke":1684,"style":1685},"90.0",[73,1927,1929],{"x":1681,"y":1928,"style":1910},"112.0","PollingObserver",[44,1931],{"x":1887,"y":1925,"width":1888,"height":1906,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1933,1935],{"x":1892,"y":1934,"style":82},"111.0","its poll interval",[44,1937],{"x":1898,"y":1925,"width":1888,"height":1906,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1939,1940],{"x":1901,"y":1934,"style":82},"works",[44,1942],{"x":53,"y":1943,"width":1883,"height":1906,"rx":47,"fill":1683,"stroke":1684,"style":1685},"124.0",[73,1945,1947],{"x":1681,"y":1946,"style":1910},"146.0","Scheduled scan",[44,1949],{"x":1887,"y":1943,"width":1888,"height":1906,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1951,1953],{"x":1892,"y":1952,"style":82},"145.0","the cron interval",[44,1955],{"x":1898,"y":1943,"width":1888,"height":1906,"rx":47,"fill":50,"stroke":1684,"style":1685},[73,1957,1940],{"x":1901,"y":1952,"style":82},[196,1959,1961],{"id":1960},"common-pitfalls","Common pitfalls",[1963,1964,1965,1973,1979,1988,1998,2004],"ul",{},[1966,1967,1968,1972],"li",{},[1969,1970,1971],"strong",{},"Processing inside the event handler."," Queue the path and let a worker do the work.",[1966,1974,1975,1978],{},[1969,1976,1977],{},"Reading files too early."," Wait for a stable size and a valid zip before opening.",[1966,1980,1981,1987],{},[1969,1982,1983,1984,1986],{},"Treating ",[14,1985,256],{}," files as workbooks."," Filter lock and temporary files by name.",[1966,1989,1990,1995,1996,23],{},[1969,1991,1992,1993,23],{},"Only handling ",[14,1994,792],{}," Saves and copies often finish with a rename; handle ",[14,1997,788],{},[1966,1999,2000,2003],{},[1969,2001,2002],{},"No startup scan."," Files that arrived while the watcher was down produce no events.",[1966,2005,2006,2009,2010,2012],{},[1969,2007,2008],{},"Watching a network drive natively."," Use ",[14,2011,1929],{}," or a scheduled scan.",[196,2014,2016],{"id":2015},"performance-notes","Performance notes",[10,2018,2019,2020,2024],{},"A single worker keeps processing sequential and simple. If files arrive faster than one worker can handle, start several worker threads — or processes, for CPU-heavy work — reading from the same queue, and use per-file ",[19,2021,2023],{"href":2022},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fprevent-overlapping-runs-of-an-excel-report-job\u002F","locks"," if more than one watcher could see the same folder.",[196,2026,2028],{"id":2027},"conclusion","Conclusion",[10,2030,2031,2032,2034,2035,240,2037,2039,2040,2042],{},"Filter events to real workbooks, ignoring ",[14,2033,256],{}," lock files, temporary saves and anything outside the drop folder. Queue each path from ",[14,2036,792],{},[14,2038,788],{},", and let one worker wait for the file to stop growing and become a valid zip before processing it, then move it to a processed or failed folder. Scan the folder at startup for files that arrived while the watcher was down, and use ",[14,2041,1929],{}," or a scheduled scan on network shares.",[196,2044,2046],{"id":2045},"frequently-asked-questions","Frequently asked questions",[10,2048,2049,2052],{},[1969,2050,2051],{},"How do I run a Python script when a new Excel file appears in a folder?","\nUse the watchdog package: subclass FileSystemEventHandler, handle on_created and on_moved for .xlsx paths, and start an Observer on the folder. Put each new path on a queue and process it in a worker, rather than inside the event handler.",[10,2054,2055,2058],{},[1969,2056,2057],{},"Why does my watcher try to read a file before it has finished copying?","\nThe created event fires when the file appears, not when it is complete. Wait until the size has stopped changing for a second or two and the file opens, then process it. Large copies over a network can take many seconds.",[10,2060,2061,2064],{},[1969,2062,2063],{},"What are the ~$ files that appear next to Excel workbooks?","\nThey are lock files Excel creates while someone has the workbook open, such as ~$sales.xlsx. They are not workbooks and must be ignored, along with temporary files Excel writes while saving.",[10,2066,2067,2070],{},[1969,2068,2069],{},"Does watchdog work on network drives?","\nOften not. File system events from SMB or NFS shares are not reliably delivered. Use watchdog's PollingObserver, which checks the folder at intervals, or run a scheduled job that scans the folder every few minutes.",[196,2072,2074],{"id":2073},"related","Related",[1963,2076,2077,2083,2090,2097],{},[1966,2078,2079,2080,2082],{},"Up one level: ",[19,2081,22],{"href":21}," — running jobs on a timer.",[1966,2084,2085,2089],{},[19,2086,2088],{"href":2087},"\u002Fautomating-reporting-workflows\u002Femailing-excel-reports-with-smtplib\u002Fdownload-excel-attachments-from-an-email-inbox\u002F","Download Excel Attachments from an Email Inbox"," — filling the drop folder from email.",[1966,2091,2092,2096],{},[19,2093,2095],{"href":2094},"\u002Fautomating-reporting-workflows\u002Fbuilding-finance-and-operations-reports\u002Fconsolidate-timesheets-from-many-workbooks\u002F","Consolidate Timesheets from Many Workbooks"," — processing the files once they arrive.",[1966,2098,2099,2103],{},[19,2100,2102],{"href":2101},"\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Fvalidate-an-uploaded-excel-file-in-a-web-app\u002F","Validate an Uploaded Excel File in a Web App"," — checks worth running on each file.",[2105,2106,2107],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .sHg5J, html code.shiki .sHg5J{--shiki-default:#6F42C1;--shiki-dark:#91CBFF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}",{"title":206,"searchDepth":283,"depth":283,"links":2109},[2110,2111,2112,2113,2114,2115,2116,2117,2118,2119,2120],{"id":198,"depth":283,"text":199},{"id":249,"depth":283,"text":250},{"id":505,"depth":283,"text":506},{"id":781,"depth":283,"text":782},{"id":1258,"depth":283,"text":1259},{"id":1849,"depth":283,"text":1850},{"id":1960,"depth":283,"text":1961},{"id":2015,"depth":283,"text":2016},{"id":2027,"depth":283,"text":2028},{"id":2045,"depth":283,"text":2046},{"id":2073,"depth":283,"text":2074},"2026-09-25","Process each workbook as soon as it lands in a drop folder. Watch the folder with watchdog, ignore Excel's lock and temporary files, wait until a file has finished copying, hand files to a single worker through a queue, move them to processed or failed folders, pick up anything that arrived while the watcher was down, and poll instead on network shares.","md",[2125,2127,2129,2131],{"q":2051,"a":2126},"Use the watchdog package: subclass FileSystemEventHandler, handle on_created and on_moved for .xlsx paths, and start an Observer on the folder. Put each new path on a queue and process it in a worker, rather than inside the event handler.",{"q":2057,"a":2128},"The created event fires when the file appears, not when it is complete. Wait until the size has stopped changing for a second or two and the file opens, then process it. Large copies over a network can take many seconds.",{"q":2063,"a":2130},"They are lock files Excel creates while someone has the workbook open, such as ~$sales.xlsx. They are not workbooks and must be ignored, along with temporary files Excel writes while saving.",{"q":2069,"a":2132},"Often not. File system events from SMB or NFS shares are not reliably delivered. Use watchdog's PollingObserver, which checks the folder at intervals, or run a scheduled job that scans the folder every few minutes.","watch a folder and process new excel files",null,{"breadcrumb":2136},[2137,2139,2142],{"name":2138,"item":1184},"Home",{"name":2140,"item":2141},"Automating Reporting Workflows with Python and Excel","\u002Fautomating-reporting-workflows\u002F",{"name":22,"item":21},"\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Fwatch-a-folder-and-process-new-excel-files",{"title":2145,"description":2146},"Watch a Folder for New Excel Files with Python watchdog","Process Excel files as they arrive with watchdog: on_created and on_moved events, ignoring ~$ lock files and temp saves, waiting for copies to finish, a queue and worker thread, processed and failed folders, a startup scan, PollingObserver for network drives and a cron alternative.","watch-a-folder-and-process-new-excel-files","automating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Fwatch-a-folder-and-process-new-excel-files\u002Findex",[2150,2151],"scheduling","sheets-and-files","how-to","a2qiphqetz7-qEmM0lnWK2alykETSnuQs-tETyuWYJE",[2155,2159],{"title":2156,"path":2157,"stem":2158,"children":-1},"Wait for Upstream Data Before Running a Report","\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Fwait-for-upstream-data-before-running-a-report","automating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Fwait-for-upstream-data-before-running-a-report\u002Findex",{"title":2160,"path":2161,"stem":2162,"children":-1},"Serving Excel Files from Python Web Apps","\u002Fautomating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps","automating-reporting-workflows\u002Fserving-excel-files-from-python-web-apps\u002Findex",[2164,2166],{"path":2165,"title":2140},"\u002Fautomating-reporting-workflows",{"path":2167,"title":22},"\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron",[],[2170,2173,2176,2179,2182,2185],{"path":2171,"title":2172,"kind":2152},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fdownload-new-excel-files-from-an-sftp-server","Download New Excel Files from an SFTP Server",{"path":2174,"title":2175,"kind":2152},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fread-excel-attachments-into-a-database-nightly","Read Excel Attachments into a Database Nightly",{"path":2177,"title":2178,"kind":2152},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Frefresh-an-excel-report-from-a-database-on-a-schedule","Refresh an Excel Report from a Database on a Schedule",{"path":2180,"title":2181,"kind":2152},"\u002Fautomating-reporting-workflows\u002Fautomating-excel-online-with-microsoft-graph\u002Frun-a-report-when-an-excel-file-changes-in-onedrive","Run a Report When an Excel File Changes in OneDrive",{"path":2183,"title":2184,"kind":2152},"\u002Fautomating-reporting-workflows\u002Femailing-excel-reports-with-smtplib\u002Fsend-report-emails-at-each-recipients-local-time","Send Report Emails at Each Recipient's Local Time",{"path":2186,"title":2187,"kind":2152},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fdetect-a-scheduled-report-that-did-not-run","Detect a Scheduled Report That Did Not Run",1790972140248]