[{"data":1,"prerenderedAt":2336},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-an-excel-file-into-a-sql-database-with-pandas":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-an-excel-file-into-a-sql-database-with-pandas":2327},{"id":4,"title":5,"body":6,"dateModified":2292,"datePublished":2292,"description":2293,"extension":2294,"faq":2295,"meta":2308,"navigation":231,"path":2319,"seo":2320,"slug":2323,"stem":2324,"type":2325,"__hash__":2326},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-an-excel-file-into-a-sql-database-with-pandas\u002Findex.md","Load an Excel File into a SQL Database with pandas",{"type":7,"value":8,"toc":2279},"minimark",[9,13,22,148,153,185,188,192,203,432,449,453,456,901,923,927,930,1251,1332,1339,1343,1346,1792,1812,1903,1907,1913,1976,1998,2002,2145,2149,2157,2160,2164,2170,2174,2185,2191,2212,2225,2229,2232,2241,2244,2275],[10,11,12],"p",{},"Reading a spreadsheet is easy. Putting one into a database that other people query is where the care is needed, because a workbook that people fill in contains things a schema does not expect: a blank row left at the bottom, a \"Total\" line, an ID typed with a leading apostrophe, a date entered two different ways, and the same key twice because someone pasted a block in again.",[10,14,15,16,21],{},"The pattern that survives all of that is the same one used for any untrusted import: normalise, pin the types, stage, validate, then merge in a single transaction. This guide is part of ",[17,18,20],"a",{"href":19},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002F","Moving Data Between Excel and Databases",".",[23,24,32,33,32,37,32,41,32,48,32,55,32,65,32,71,32,76,32,80,32,83,32,86,32,89,32,92,32,95,32,99,32,102,32,105,32,110,32,115,32,119,32,123,32,125,32,128,32,131,32,134,32,138,32,142,32,145],"svg",{"viewBox":25,"role":26,"ariaLabelledBy":27,"xmlns":30,"style":31},"0 0 740 248","img",[28,29],"ld-gate-t","ld-gate-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:740px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[34,35,36],"title",{"id":28},"What an uploaded workbook contains that a table does not want",[38,39,40],"desc",{"id":29},"A submitted spreadsheet typically carries a title row above the headers, trailing blank rows, a total line, identifiers stored as text with leading zeros, dates in mixed formats and duplicated keys. Each is removed or coerced at a specific step before the rows are allowed near the live table.",[42,43],"rect",{"x":44,"y":44,"width":45,"height":46,"fill":47},"0","740","248","#ffffff",[49,50,54],"text",{"x":51,"y":52,"style":53},"370","30","font-size:13px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","Six things in a submitted workbook, and where each is dealt with",[42,56],{"x":57,"y":58,"width":59,"height":60,"rx":61,"fill":62,"stroke":63,"style":64},"16","50","228","80","12","#fce9e9","var(--accent-ink,#be185d)","stroke-width:2px",[49,66,70],{"x":67,"y":68,"style":69},"130","76","font-size:11.5px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","a title above the headers",[49,72,75],{"x":67,"y":73,"style":74},"100","font-size:11px;fill:var(--text,#172033);text-anchor:middle","columns come back as",[49,77,79],{"x":67,"y":78,"style":74},"118","Unnamed: 0 — fix with skiprows",[42,81],{"x":82,"y":58,"width":59,"height":60,"rx":61,"fill":62,"stroke":63,"style":64},"256",[49,84,85],{"x":51,"y":68,"style":69},"blank and total rows",[49,87,88],{"x":51,"y":73,"style":74},"dropna on the key column",[49,90,91],{"x":51,"y":78,"style":74},"and a filter on the label",[42,93],{"x":94,"y":58,"width":59,"height":60,"rx":61,"fill":62,"stroke":63,"style":64},"496",[49,96,98],{"x":97,"y":68,"style":69},"610","mixed date formats",[49,100,101],{"x":97,"y":73,"style":74},"to_datetime with errors=",[49,103,104],{"x":97,"y":78,"style":74},"\"coerce\", then count NaT",[42,106],{"x":57,"y":107,"width":59,"height":60,"rx":61,"fill":108,"stroke":109,"style":64},"142","#fdefd8","var(--gold,#b4740a)",[49,111,114],{"x":67,"y":112,"style":113},"168","font-size:11.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","IDs with leading zeros",[49,116,118],{"x":67,"y":117,"style":74},"192","dtype={\"order_id\": str}",[49,120,122],{"x":67,"y":121,"style":74},"210","at the read, not after",[42,124],{"x":82,"y":107,"width":59,"height":60,"rx":61,"fill":108,"stroke":109,"style":64},[49,126,127],{"x":51,"y":112,"style":113},"duplicate keys",[49,129,130],{"x":51,"y":117,"style":74},"counted in the staging",[49,132,133],{"x":51,"y":121,"style":74},"table, reported by value",[42,135],{"x":94,"y":107,"width":59,"height":60,"rx":61,"fill":136,"stroke":137,"style":64},"#d9f4f1","var(--teal,#0f9488)",[49,139,141],{"x":97,"y":112,"style":140},"font-size:11.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","a re-submitted file",[49,143,144],{"x":97,"y":117,"style":74},"delete the period first,",[49,146,147],{"x":97,"y":121,"style":74},"so the load is repeatable",[149,150,152],"h2",{"id":151},"prerequisites","Prerequisites",[154,155,160],"pre",{"className":156,"code":157,"language":158,"meta":159,"style":159},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl sqlalchemy\n","bash","",[161,162,163],"code",{"__ignoreMap":159},[164,165,168,172,176,179,182],"span",{"class":166,"line":167},"line",1,[164,169,171],{"class":170},"sMTad","pip",[164,173,175],{"class":174},"srMev"," install",[164,177,178],{"class":174}," pandas",[164,180,181],{"class":174}," openpyxl",[164,183,184],{"class":174}," sqlalchemy\n",[10,186,187],{},"The examples use SQLite so they run with no server. Everything transfers to Postgres, MySQL or SQL Server by changing the URL — and by remembering that those databases enforce constraints SQLite will happily ignore, which is an argument for testing the load against the real engine before it runs unattended.",[149,189,191],{"id":190},"step-1-read-the-file-on-your-terms","Step 1: Read the file on your terms",[10,193,194,195,198,199,202],{},"Two arguments do most of the defensive work: ",[161,196,197],{},"dtype"," pins the columns whose type pandas would otherwise guess, and ",[161,200,201],{},"usecols"," limits the read to the columns you actually accept, so an extra column someone added on the right cannot reach the table.",[154,204,208],{"className":205,"code":206,"language":207,"meta":159,"style":159},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\nRAW = pd.read_excel(\n    \"submissions\u002Fjuly-actuals.xlsx\",\n    sheet_name=\"Actuals\",\n    skiprows=2,                       # the file has a title and a blank line first\n    usecols=\"A:E\",                    # ignore the working columns to the right\n    dtype={\"Order Ref\": str, \"Cost Centre\": str},\n    na_values=[\"\", \"-\", \"n\u002Fa\", \"N\u002FA\", \"TBC\"],\n)\nprint(RAW.dtypes)\nprint(f\"{len(RAW):,} raw rows\")\n","python",[161,209,210,226,233,246,255,270,288,305,338,375,381,395],{"__ignoreMap":159},[164,211,212,216,220,223],{"class":166,"line":167},[164,213,215],{"class":214},"s-kum","import",[164,217,219],{"class":218},"skGVy"," pandas ",[164,221,222],{"class":214},"as",[164,224,225],{"class":218}," pd\n",[164,227,229],{"class":166,"line":228},2,[164,230,232],{"emptyLinePlaceholder":231},true,"\n",[164,234,236,240,243],{"class":166,"line":235},3,[164,237,239],{"class":238},"sP0c6","RAW",[164,241,242],{"class":214}," =",[164,244,245],{"class":218}," pd.read_excel(\n",[164,247,249,252],{"class":166,"line":248},4,[164,250,251],{"class":174},"    \"submissions\u002Fjuly-actuals.xlsx\"",[164,253,254],{"class":218},",\n",[164,256,258,262,265,268],{"class":166,"line":257},5,[164,259,261],{"class":260},"sa561","    sheet_name",[164,263,264],{"class":214},"=",[164,266,267],{"class":174},"\"Actuals\"",[164,269,254],{"class":218},[164,271,273,276,278,281,284],{"class":166,"line":272},6,[164,274,275],{"class":260},"    skiprows",[164,277,264],{"class":214},[164,279,280],{"class":238},"2",[164,282,283],{"class":218},",                       ",[164,285,287],{"class":286},"s-wDw","# the file has a title and a blank line first\n",[164,289,291,294,296,299,302],{"class":166,"line":290},7,[164,292,293],{"class":260},"    usecols",[164,295,264],{"class":214},[164,297,298],{"class":174},"\"A:E\"",[164,300,301],{"class":218},",                    ",[164,303,304],{"class":286},"# ignore the working columns to the right\n",[164,306,308,311,313,316,319,322,325,328,331,333,335],{"class":166,"line":307},8,[164,309,310],{"class":260},"    dtype",[164,312,264],{"class":214},[164,314,315],{"class":218},"{",[164,317,318],{"class":174},"\"Order Ref\"",[164,320,321],{"class":218},": ",[164,323,324],{"class":238},"str",[164,326,327],{"class":218},", ",[164,329,330],{"class":174},"\"Cost Centre\"",[164,332,321],{"class":218},[164,334,324],{"class":238},[164,336,337],{"class":218},"},\n",[164,339,341,344,346,349,352,354,357,359,362,364,367,369,372],{"class":166,"line":340},9,[164,342,343],{"class":260},"    na_values",[164,345,264],{"class":214},[164,347,348],{"class":218},"[",[164,350,351],{"class":174},"\"\"",[164,353,327],{"class":218},[164,355,356],{"class":174},"\"-\"",[164,358,327],{"class":218},[164,360,361],{"class":174},"\"n\u002Fa\"",[164,363,327],{"class":218},[164,365,366],{"class":174},"\"N\u002FA\"",[164,368,327],{"class":218},[164,370,371],{"class":174},"\"TBC\"",[164,373,374],{"class":218},"],\n",[164,376,378],{"class":166,"line":377},10,[164,379,380],{"class":218},")\n",[164,382,384,387,390,392],{"class":166,"line":383},11,[164,385,386],{"class":238},"print",[164,388,389],{"class":218},"(",[164,391,239],{"class":238},[164,393,394],{"class":218},".dtypes)\n",[164,396,398,400,402,405,408,411,414,416,418,421,424,427,430],{"class":166,"line":397},12,[164,399,386],{"class":238},[164,401,389],{"class":218},[164,403,404],{"class":214},"f",[164,406,407],{"class":174},"\"",[164,409,315],{"class":410},"sSjpA",[164,412,413],{"class":238},"len",[164,415,389],{"class":218},[164,417,239],{"class":238},[164,419,420],{"class":218},")",[164,422,423],{"class":214},":,",[164,425,426],{"class":410},"}",[164,428,429],{"class":174}," raw rows\"",[164,431,380],{"class":218},[10,433,434,437,438,327,441,444,445,448],{},[161,435,436],{},"na_values"," is the argument people find late. Spreadsheets are full of human placeholders — a dash, ",[161,439,440],{},"n\u002Fa",[161,442,443],{},"TBC"," — and without this every one of them makes its column ",[161,446,447],{},"object",", which then fails to insert into a numeric field with an error that names the type rather than the cell.",[149,450,452],{"id":451},"step-2-normalise-the-shape","Step 2: Normalise the shape",[10,454,455],{},"Database columns and spreadsheet headings rarely match. Do the renaming and cleaning in one visible place rather than scattering it through the load:",[154,457,459],{"className":205,"code":458,"language":207,"meta":159,"style":159},"COLUMNS = {\n    \"Order Ref\": \"order_id\",\n    \"Sales Area\": \"region\",\n    \"Booking Date\": \"order_date\",\n    \"Net Value\": \"amount\",\n    \"Cost Centre\": \"cost_centre\",\n}\n\n\ndef normalise(raw):\n    df = raw.rename(columns=COLUMNS)\n\n    missing = set(COLUMNS.values()) - set(df.columns)\n    if missing:\n        raise ValueError(f\"upload is missing column(s): {sorted(missing)}\")\n\n    df = df.dropna(subset=[\"order_id\"])                       # blank trailing rows\n    df = df[~df[\"order_id\"].astype(str).str.lower()\n              .isin({\"total\", \"subtotal\", \"grand total\"})]    # summary lines\n\n    df[\"order_date\"] = pd.to_datetime(df[\"order_date\"], errors=\"coerce\")\n    df[\"amount\"] = pd.to_numeric(df[\"amount\"], errors=\"coerce\").round(2)\n    df[\"region\"] = df[\"region\"].str.strip().str.title()\n    df[\"order_id\"] = df[\"order_id\"].str.strip()\n\n    df[\"period\"] = df[\"order_date\"].dt.strftime(\"%Y-%m\")\n    return df.reset_index(drop=True)\n\n\ndf = normalise(RAW)\n",[161,460,461,471,483,495,507,519,531,536,540,544,556,575,579,605,614,644,649,674,700,725,730,761,792,810,828,833,857,876,881,886],{"__ignoreMap":159},[164,462,463,466,468],{"class":166,"line":167},[164,464,465],{"class":238},"COLUMNS",[164,467,242],{"class":214},[164,469,470],{"class":218}," {\n",[164,472,473,476,478,481],{"class":166,"line":228},[164,474,475],{"class":174},"    \"Order Ref\"",[164,477,321],{"class":218},[164,479,480],{"class":174},"\"order_id\"",[164,482,254],{"class":218},[164,484,485,488,490,493],{"class":166,"line":235},[164,486,487],{"class":174},"    \"Sales Area\"",[164,489,321],{"class":218},[164,491,492],{"class":174},"\"region\"",[164,494,254],{"class":218},[164,496,497,500,502,505],{"class":166,"line":248},[164,498,499],{"class":174},"    \"Booking Date\"",[164,501,321],{"class":218},[164,503,504],{"class":174},"\"order_date\"",[164,506,254],{"class":218},[164,508,509,512,514,517],{"class":166,"line":257},[164,510,511],{"class":174},"    \"Net Value\"",[164,513,321],{"class":218},[164,515,516],{"class":174},"\"amount\"",[164,518,254],{"class":218},[164,520,521,524,526,529],{"class":166,"line":272},[164,522,523],{"class":174},"    \"Cost Centre\"",[164,525,321],{"class":218},[164,527,528],{"class":174},"\"cost_centre\"",[164,530,254],{"class":218},[164,532,533],{"class":166,"line":290},[164,534,535],{"class":218},"}\n",[164,537,538],{"class":166,"line":307},[164,539,232],{"emptyLinePlaceholder":231},[164,541,542],{"class":166,"line":340},[164,543,232],{"emptyLinePlaceholder":231},[164,545,546,549,553],{"class":166,"line":377},[164,547,548],{"class":214},"def",[164,550,552],{"class":551},"s_Opv"," normalise",[164,554,555],{"class":218},"(raw):\n",[164,557,558,561,563,566,569,571,573],{"class":166,"line":383},[164,559,560],{"class":218},"    df ",[164,562,264],{"class":214},[164,564,565],{"class":218}," raw.rename(",[164,567,568],{"class":260},"columns",[164,570,264],{"class":214},[164,572,465],{"class":238},[164,574,380],{"class":218},[164,576,577],{"class":166,"line":397},[164,578,232],{"emptyLinePlaceholder":231},[164,580,582,585,587,590,592,594,597,600,602],{"class":166,"line":581},13,[164,583,584],{"class":218},"    missing ",[164,586,264],{"class":214},[164,588,589],{"class":238}," set",[164,591,389],{"class":218},[164,593,465],{"class":238},[164,595,596],{"class":218},".values()) ",[164,598,599],{"class":214},"-",[164,601,589],{"class":238},[164,603,604],{"class":218},"(df.columns)\n",[164,606,608,611],{"class":166,"line":607},14,[164,609,610],{"class":214},"    if",[164,612,613],{"class":218}," missing:\n",[164,615,617,620,623,625,627,630,632,635,638,640,642],{"class":166,"line":616},15,[164,618,619],{"class":214},"        raise",[164,621,622],{"class":238}," ValueError",[164,624,389],{"class":218},[164,626,404],{"class":214},[164,628,629],{"class":174},"\"upload is missing column(s): ",[164,631,315],{"class":410},[164,633,634],{"class":238},"sorted",[164,636,637],{"class":218},"(missing)",[164,639,426],{"class":410},[164,641,407],{"class":174},[164,643,380],{"class":218},[164,645,647],{"class":166,"line":646},16,[164,648,232],{"emptyLinePlaceholder":231},[164,650,652,654,656,659,662,664,666,668,671],{"class":166,"line":651},17,[164,653,560],{"class":218},[164,655,264],{"class":214},[164,657,658],{"class":218}," df.dropna(",[164,660,661],{"class":260},"subset",[164,663,264],{"class":214},[164,665,348],{"class":218},[164,667,480],{"class":174},[164,669,670],{"class":218},"])                       ",[164,672,673],{"class":286},"# blank trailing rows\n",[164,675,677,679,681,684,687,690,692,695,697],{"class":166,"line":676},18,[164,678,560],{"class":218},[164,680,264],{"class":214},[164,682,683],{"class":218}," df[",[164,685,686],{"class":214},"~",[164,688,689],{"class":218},"df[",[164,691,480],{"class":174},[164,693,694],{"class":218},"].astype(",[164,696,324],{"class":238},[164,698,699],{"class":218},").str.lower()\n",[164,701,703,706,709,711,714,716,719,722],{"class":166,"line":702},19,[164,704,705],{"class":218},"              .isin({",[164,707,708],{"class":174},"\"total\"",[164,710,327],{"class":218},[164,712,713],{"class":174},"\"subtotal\"",[164,715,327],{"class":218},[164,717,718],{"class":174},"\"grand total\"",[164,720,721],{"class":218},"})]    ",[164,723,724],{"class":286},"# summary lines\n",[164,726,728],{"class":166,"line":727},20,[164,729,232],{"emptyLinePlaceholder":231},[164,731,733,736,738,741,743,746,748,751,754,756,759],{"class":166,"line":732},21,[164,734,735],{"class":218},"    df[",[164,737,504],{"class":174},[164,739,740],{"class":218},"] ",[164,742,264],{"class":214},[164,744,745],{"class":218}," pd.to_datetime(df[",[164,747,504],{"class":174},[164,749,750],{"class":218},"], ",[164,752,753],{"class":260},"errors",[164,755,264],{"class":214},[164,757,758],{"class":174},"\"coerce\"",[164,760,380],{"class":218},[164,762,764,766,768,770,772,775,777,779,781,783,785,788,790],{"class":166,"line":763},22,[164,765,735],{"class":218},[164,767,516],{"class":174},[164,769,740],{"class":218},[164,771,264],{"class":214},[164,773,774],{"class":218}," pd.to_numeric(df[",[164,776,516],{"class":174},[164,778,750],{"class":218},[164,780,753],{"class":260},[164,782,264],{"class":214},[164,784,758],{"class":174},[164,786,787],{"class":218},").round(",[164,789,280],{"class":238},[164,791,380],{"class":218},[164,793,795,797,799,801,803,805,807],{"class":166,"line":794},23,[164,796,735],{"class":218},[164,798,492],{"class":174},[164,800,740],{"class":218},[164,802,264],{"class":214},[164,804,683],{"class":218},[164,806,492],{"class":174},[164,808,809],{"class":218},"].str.strip().str.title()\n",[164,811,813,815,817,819,821,823,825],{"class":166,"line":812},24,[164,814,735],{"class":218},[164,816,480],{"class":174},[164,818,740],{"class":218},[164,820,264],{"class":214},[164,822,683],{"class":218},[164,824,480],{"class":174},[164,826,827],{"class":218},"].str.strip()\n",[164,829,831],{"class":166,"line":830},25,[164,832,232],{"emptyLinePlaceholder":231},[164,834,836,838,841,843,845,847,849,852,855],{"class":166,"line":835},26,[164,837,735],{"class":218},[164,839,840],{"class":174},"\"period\"",[164,842,740],{"class":218},[164,844,264],{"class":214},[164,846,683],{"class":218},[164,848,504],{"class":174},[164,850,851],{"class":218},"].dt.strftime(",[164,853,854],{"class":174},"\"%Y-%m\"",[164,856,380],{"class":218},[164,858,860,863,866,869,871,874],{"class":166,"line":859},27,[164,861,862],{"class":214},"    return",[164,864,865],{"class":218}," df.reset_index(",[164,867,868],{"class":260},"drop",[164,870,264],{"class":214},[164,872,873],{"class":238},"True",[164,875,380],{"class":218},[164,877,879],{"class":166,"line":878},28,[164,880,232],{"emptyLinePlaceholder":231},[164,882,884],{"class":166,"line":883},29,[164,885,232],{"emptyLinePlaceholder":231},[164,887,889,892,894,897,899],{"class":166,"line":888},30,[164,890,891],{"class":218},"df ",[164,893,264],{"class":214},[164,895,896],{"class":218}," normalise(",[164,898,239],{"class":238},[164,900,380],{"class":218},[10,902,903,906,907,910,911,914,915,918,919,922],{},[161,904,905],{},"errors=\"coerce\""," turns anything unparseable into ",[161,908,909],{},"NaT"," or ",[161,912,913],{},"NaN"," rather than raising, which is what you want here: the goal is to collect every bad value and report them together, not to stop at the first one. Rounding the amount at this point rather than at the end avoids a float that ends in ",[161,916,917],{},".00000000001"," being stored in a ",[161,920,921],{},"NUMERIC(12,2)"," column and rounding somewhere you cannot see.",[149,924,926],{"id":925},"step-3-refuse-the-file-if-it-is-not-loadable","Step 3: Refuse the file if it is not loadable",[10,928,929],{},"Now count what the coercions found, and stop before anything is written:",[154,931,933],{"className":205,"code":932,"language":207,"meta":159,"style":159},"def check(df):\n    problems = []\n\n    bad_dates = int(df[\"order_date\"].isna().sum())\n    if bad_dates:\n        sample = df.loc[df[\"order_date\"].isna(), \"order_id\"].head(5).tolist()\n        problems.append(f\"{bad_dates} unparseable date(s), e.g. order {sample}\")\n\n    bad_amounts = int(df[\"amount\"].isna().sum())\n    if bad_amounts:\n        problems.append(f\"{bad_amounts} non-numeric amount(s)\")\n\n    dupes = df[\"order_id\"][df[\"order_id\"].duplicated()].unique().tolist()\n    if dupes:\n        problems.append(f\"{len(dupes)} duplicated order id(s): {dupes[:5]}\")\n\n    if df[\"period\"].nunique() > 1:\n        problems.append(f\"file spans several periods: {sorted(df['period'].unique())}\")\n\n    if problems:\n        raise ValueError(\"upload rejected:\\n  - \" + \"\\n  - \".join(problems))\n    return df\n",[161,934,935,945,955,959,977,984,1010,1040,1044,1059,1066,1086,1090,1109,1116,1152,1156,1176,1203,1207,1214,1244],{"__ignoreMap":159},[164,936,937,939,942],{"class":166,"line":167},[164,938,548],{"class":214},[164,940,941],{"class":551}," check",[164,943,944],{"class":218},"(df):\n",[164,946,947,950,952],{"class":166,"line":228},[164,948,949],{"class":218},"    problems ",[164,951,264],{"class":214},[164,953,954],{"class":218}," []\n",[164,956,957],{"class":166,"line":235},[164,958,232],{"emptyLinePlaceholder":231},[164,960,961,964,966,969,972,974],{"class":166,"line":248},[164,962,963],{"class":218},"    bad_dates ",[164,965,264],{"class":214},[164,967,968],{"class":238}," int",[164,970,971],{"class":218},"(df[",[164,973,504],{"class":174},[164,975,976],{"class":218},"].isna().sum())\n",[164,978,979,981],{"class":166,"line":257},[164,980,610],{"class":214},[164,982,983],{"class":218}," bad_dates:\n",[164,985,986,989,991,994,996,999,1001,1004,1007],{"class":166,"line":272},[164,987,988],{"class":218},"        sample ",[164,990,264],{"class":214},[164,992,993],{"class":218}," df.loc[df[",[164,995,504],{"class":174},[164,997,998],{"class":218},"].isna(), ",[164,1000,480],{"class":174},[164,1002,1003],{"class":218},"].head(",[164,1005,1006],{"class":238},"5",[164,1008,1009],{"class":218},").tolist()\n",[164,1011,1012,1015,1017,1019,1021,1024,1026,1029,1031,1034,1036,1038],{"class":166,"line":290},[164,1013,1014],{"class":218},"        problems.append(",[164,1016,404],{"class":214},[164,1018,407],{"class":174},[164,1020,315],{"class":410},[164,1022,1023],{"class":218},"bad_dates",[164,1025,426],{"class":410},[164,1027,1028],{"class":174}," unparseable date(s), e.g. order ",[164,1030,315],{"class":410},[164,1032,1033],{"class":218},"sample",[164,1035,426],{"class":410},[164,1037,407],{"class":174},[164,1039,380],{"class":218},[164,1041,1042],{"class":166,"line":307},[164,1043,232],{"emptyLinePlaceholder":231},[164,1045,1046,1049,1051,1053,1055,1057],{"class":166,"line":340},[164,1047,1048],{"class":218},"    bad_amounts ",[164,1050,264],{"class":214},[164,1052,968],{"class":238},[164,1054,971],{"class":218},[164,1056,516],{"class":174},[164,1058,976],{"class":218},[164,1060,1061,1063],{"class":166,"line":377},[164,1062,610],{"class":214},[164,1064,1065],{"class":218}," bad_amounts:\n",[164,1067,1068,1070,1072,1074,1076,1079,1081,1084],{"class":166,"line":383},[164,1069,1014],{"class":218},[164,1071,404],{"class":214},[164,1073,407],{"class":174},[164,1075,315],{"class":410},[164,1077,1078],{"class":218},"bad_amounts",[164,1080,426],{"class":410},[164,1082,1083],{"class":174}," non-numeric amount(s)\"",[164,1085,380],{"class":218},[164,1087,1088],{"class":166,"line":397},[164,1089,232],{"emptyLinePlaceholder":231},[164,1091,1092,1095,1097,1099,1101,1104,1106],{"class":166,"line":581},[164,1093,1094],{"class":218},"    dupes ",[164,1096,264],{"class":214},[164,1098,683],{"class":218},[164,1100,480],{"class":174},[164,1102,1103],{"class":218},"][df[",[164,1105,480],{"class":174},[164,1107,1108],{"class":218},"].duplicated()].unique().tolist()\n",[164,1110,1111,1113],{"class":166,"line":607},[164,1112,610],{"class":214},[164,1114,1115],{"class":218}," dupes:\n",[164,1117,1118,1120,1122,1124,1126,1128,1131,1133,1136,1138,1141,1143,1146,1148,1150],{"class":166,"line":616},[164,1119,1014],{"class":218},[164,1121,404],{"class":214},[164,1123,407],{"class":174},[164,1125,315],{"class":410},[164,1127,413],{"class":238},[164,1129,1130],{"class":218},"(dupes)",[164,1132,426],{"class":410},[164,1134,1135],{"class":174}," duplicated order id(s): ",[164,1137,315],{"class":410},[164,1139,1140],{"class":218},"dupes[:",[164,1142,1006],{"class":238},[164,1144,1145],{"class":218},"]",[164,1147,426],{"class":410},[164,1149,407],{"class":174},[164,1151,380],{"class":218},[164,1153,1154],{"class":166,"line":646},[164,1155,232],{"emptyLinePlaceholder":231},[164,1157,1158,1160,1162,1164,1167,1170,1173],{"class":166,"line":651},[164,1159,610],{"class":214},[164,1161,683],{"class":218},[164,1163,840],{"class":174},[164,1165,1166],{"class":218},"].nunique() ",[164,1168,1169],{"class":214},">",[164,1171,1172],{"class":238}," 1",[164,1174,1175],{"class":218},":\n",[164,1177,1178,1180,1182,1185,1187,1189,1191,1194,1197,1199,1201],{"class":166,"line":676},[164,1179,1014],{"class":218},[164,1181,404],{"class":214},[164,1183,1184],{"class":174},"\"file spans several periods: ",[164,1186,315],{"class":410},[164,1188,634],{"class":238},[164,1190,971],{"class":218},[164,1192,1193],{"class":174},"'period'",[164,1195,1196],{"class":218},"].unique())",[164,1198,426],{"class":410},[164,1200,407],{"class":174},[164,1202,380],{"class":218},[164,1204,1205],{"class":166,"line":702},[164,1206,232],{"emptyLinePlaceholder":231},[164,1208,1209,1211],{"class":166,"line":727},[164,1210,610],{"class":214},[164,1212,1213],{"class":218}," problems:\n",[164,1215,1216,1218,1220,1222,1225,1228,1231,1234,1237,1239,1241],{"class":166,"line":732},[164,1217,619],{"class":214},[164,1219,622],{"class":238},[164,1221,389],{"class":218},[164,1223,1224],{"class":174},"\"upload rejected:",[164,1226,1227],{"class":410},"\\n",[164,1229,1230],{"class":174},"  - \"",[164,1232,1233],{"class":214}," +",[164,1235,1236],{"class":174}," \"",[164,1238,1227],{"class":410},[164,1240,1230],{"class":174},[164,1242,1243],{"class":218},".join(problems))\n",[164,1245,1246,1248],{"class":166,"line":763},[164,1247,862],{"class":214},[164,1249,1250],{"class":218}," df\n",[23,1252,32,1257,32,1260,32,1263,32,1266,32,1272,32,1278,32,1283,32,1287,32,1291,32,1296,32,1299,32,1302,32,1307,32,1311,32,1317,32,1321,32,1324,32,1328],{"viewBox":1253,"role":26,"ariaLabelledBy":1254,"xmlns":30,"style":31},"0 0 740 226",[1255,1256],"ld-msg-t","ld-msg-d",[34,1258,1259],{"id":1255},"Two rejection messages for the same broken upload",[38,1261,1262],{"id":1256},"A message that stops at the first error and names only the exception type sends the submitter back for another attempt per problem. A message that lists every problem with example order references lets them fix all four in one pass and resubmit once.",[42,1264],{"x":44,"y":44,"width":45,"height":1265,"fill":47},"226",[42,1267],{"x":57,"y":1268,"width":1269,"height":1270,"rx":1271,"fill":62,"stroke":63,"style":64},"34","344","176","14",[49,1273,1277],{"x":1274,"y":1275,"style":1276},"188","62","font-size:12px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","stops at the first problem",[42,1279],{"x":1280,"y":68,"width":1281,"height":52,"rx":1006,"fill":47,"stroke":1282},"40","296","var(--line,#cdd5e6)",[49,1284,1286],{"x":1274,"y":1285,"style":74},"96","ValueError: could not convert string",[49,1288,1290],{"x":1274,"y":67,"style":1289},"font-size:11.5px;fill:var(--text,#172033);text-anchor:middle","which row? which column? which value?",[49,1292,1295],{"x":1274,"y":1293,"style":1294},"158","font-size:11px;fill:var(--muted,#5b6780);text-anchor:middle","fix one thing, resubmit, discover the next —",[49,1297,1298],{"x":1274,"y":1270,"style":1294},"four round trips for four mistakes",[42,1300],{"x":1301,"y":1268,"width":1269,"height":1270,"rx":1271,"fill":136,"stroke":137,"style":64},"380",[49,1303,1306],{"x":1304,"y":1275,"style":1305},"552","font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","collects every problem first",[42,1308],{"x":1309,"y":68,"width":1281,"height":1310,"rx":1006,"fill":47,"stroke":1282},"404","66",[49,1312,1316],{"x":1313,"y":1314,"style":1315},"418","94","font-size:10.5px;fill:var(--text,#172033)","upload rejected:",[49,1318,1320],{"x":1313,"y":1319,"style":1315},"112","- 3 unparseable dates, e.g. order A-1042",[49,1322,1323],{"x":1313,"y":67,"style":1315},"- 2 duplicated ids: A-1007, A-1019",[49,1325,1327],{"x":1304,"y":1326,"style":1294},"166","every problem, with rows they can find —",[49,1329,1331],{"x":1304,"y":1330,"style":1294},"184","one fix, one resubmission",[10,1333,1334,1335,21],{},"Reporting every problem at once, with example values, is the difference between one email to the submitter and five. Naming the offending order IDs matters more than the count — the person fixing the file needs to find the rows, and \"three unparseable dates\" sends them scrolling. The same reasoning drives ",[17,1336,1338],{"href":1337},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fvalidate-excel-columns-before-import-with-pandas\u002F","Validate Excel Columns Before Import with pandas",[149,1340,1342],{"id":1341},"step-4-stage-then-merge-in-one-transaction","Step 4: Stage, then merge in one transaction",[10,1344,1345],{},"Write to a staging table first. It costs one extra statement and it means the live table never sees a partially loaded or later-rejected file:",[154,1347,1349],{"className":205,"code":1348,"language":207,"meta":159,"style":159},"from sqlalchemy import create_engine, text\n\nengine = create_engine(\"sqlite:\u002F\u002F\u002Fsales.db\")\n\nDTYPES = None      # let SQLAlchemy infer for SQLite; pin explicitly on other engines\n\n\ndef load(df, table=\"sales_actuals\"):\n    period = df[\"period\"].iat[0]\n    staging = f\"{table}__staging\"\n\n    df.to_sql(staging, engine, if_exists=\"replace\", index=False,\n              chunksize=5_000, method=\"multi\", dtype=DTYPES)\n\n    with engine.begin() as conn:                       # commits, or rolls back entirely\n        staged = conn.execute(text(f\"SELECT COUNT(*) FROM {staging}\")).scalar()\n        if staged != len(df):\n            raise RuntimeError(f\"staged {staged} rows, expected {len(df)}\")\n\n        removed = conn.execute(\n            text(f\"DELETE FROM {table} WHERE period = :p\"), {\"p\": period}).rowcount\n        conn.execute(text(\n            f\"INSERT INTO {table} (order_id, region, order_date, amount, \"\n            f\"cost_centre, period) \"\n            f\"SELECT order_id, region, order_date, amount, cost_centre, period \"\n            f\"FROM {staging}\"))\n        conn.execute(text(f\"DROP TABLE {staging}\"))\n\n    return {\"period\": period, \"replaced\": removed, \"inserted\": len(df)}\n\n\nprint(load(check(df)))\n",[161,1350,1351,1364,1368,1383,1387,1400,1404,1408,1426,1445,1467,1471,1496,1526,1530,1546,1573,1589,1627,1631,1641,1669,1674,1691,1698,1705,1723,1743,1747,1775,1779,1784],{"__ignoreMap":159},[164,1352,1353,1356,1359,1361],{"class":166,"line":167},[164,1354,1355],{"class":214},"from",[164,1357,1358],{"class":218}," sqlalchemy ",[164,1360,215],{"class":214},[164,1362,1363],{"class":218}," create_engine, text\n",[164,1365,1366],{"class":166,"line":228},[164,1367,232],{"emptyLinePlaceholder":231},[164,1369,1370,1373,1375,1378,1381],{"class":166,"line":235},[164,1371,1372],{"class":218},"engine ",[164,1374,264],{"class":214},[164,1376,1377],{"class":218}," create_engine(",[164,1379,1380],{"class":174},"\"sqlite:\u002F\u002F\u002Fsales.db\"",[164,1382,380],{"class":218},[164,1384,1385],{"class":166,"line":248},[164,1386,232],{"emptyLinePlaceholder":231},[164,1388,1389,1392,1394,1397],{"class":166,"line":257},[164,1390,1391],{"class":238},"DTYPES",[164,1393,242],{"class":214},[164,1395,1396],{"class":238}," None",[164,1398,1399],{"class":286},"      # let SQLAlchemy infer for SQLite; pin explicitly on other engines\n",[164,1401,1402],{"class":166,"line":272},[164,1403,232],{"emptyLinePlaceholder":231},[164,1405,1406],{"class":166,"line":290},[164,1407,232],{"emptyLinePlaceholder":231},[164,1409,1410,1412,1415,1418,1420,1423],{"class":166,"line":307},[164,1411,548],{"class":214},[164,1413,1414],{"class":551}," load",[164,1416,1417],{"class":218},"(df, table",[164,1419,264],{"class":214},[164,1421,1422],{"class":174},"\"sales_actuals\"",[164,1424,1425],{"class":218},"):\n",[164,1427,1428,1431,1433,1435,1437,1440,1442],{"class":166,"line":340},[164,1429,1430],{"class":218},"    period ",[164,1432,264],{"class":214},[164,1434,683],{"class":218},[164,1436,840],{"class":174},[164,1438,1439],{"class":218},"].iat[",[164,1441,44],{"class":238},[164,1443,1444],{"class":218},"]\n",[164,1446,1447,1450,1452,1455,1457,1459,1462,1464],{"class":166,"line":377},[164,1448,1449],{"class":218},"    staging ",[164,1451,264],{"class":214},[164,1453,1454],{"class":214}," f",[164,1456,407],{"class":174},[164,1458,315],{"class":410},[164,1460,1461],{"class":218},"table",[164,1463,426],{"class":410},[164,1465,1466],{"class":174},"__staging\"\n",[164,1468,1469],{"class":166,"line":383},[164,1470,232],{"emptyLinePlaceholder":231},[164,1472,1473,1476,1479,1481,1484,1486,1489,1491,1494],{"class":166,"line":397},[164,1474,1475],{"class":218},"    df.to_sql(staging, engine, ",[164,1477,1478],{"class":260},"if_exists",[164,1480,264],{"class":214},[164,1482,1483],{"class":174},"\"replace\"",[164,1485,327],{"class":218},[164,1487,1488],{"class":260},"index",[164,1490,264],{"class":214},[164,1492,1493],{"class":238},"False",[164,1495,254],{"class":218},[164,1497,1498,1501,1503,1506,1508,1511,1513,1516,1518,1520,1522,1524],{"class":166,"line":581},[164,1499,1500],{"class":260},"              chunksize",[164,1502,264],{"class":214},[164,1504,1505],{"class":238},"5_000",[164,1507,327],{"class":218},[164,1509,1510],{"class":260},"method",[164,1512,264],{"class":214},[164,1514,1515],{"class":174},"\"multi\"",[164,1517,327],{"class":218},[164,1519,197],{"class":260},[164,1521,264],{"class":214},[164,1523,1391],{"class":238},[164,1525,380],{"class":218},[164,1527,1528],{"class":166,"line":607},[164,1529,232],{"emptyLinePlaceholder":231},[164,1531,1532,1535,1538,1540,1543],{"class":166,"line":616},[164,1533,1534],{"class":214},"    with",[164,1536,1537],{"class":218}," engine.begin() ",[164,1539,222],{"class":214},[164,1541,1542],{"class":218}," conn:                       ",[164,1544,1545],{"class":286},"# commits, or rolls back entirely\n",[164,1547,1548,1551,1553,1556,1558,1561,1563,1566,1568,1570],{"class":166,"line":646},[164,1549,1550],{"class":218},"        staged ",[164,1552,264],{"class":214},[164,1554,1555],{"class":218}," conn.execute(text(",[164,1557,404],{"class":214},[164,1559,1560],{"class":174},"\"SELECT COUNT(*) FROM ",[164,1562,315],{"class":410},[164,1564,1565],{"class":218},"staging",[164,1567,426],{"class":410},[164,1569,407],{"class":174},[164,1571,1572],{"class":218},")).scalar()\n",[164,1574,1575,1578,1581,1584,1587],{"class":166,"line":651},[164,1576,1577],{"class":214},"        if",[164,1579,1580],{"class":218}," staged ",[164,1582,1583],{"class":214},"!=",[164,1585,1586],{"class":238}," len",[164,1588,944],{"class":218},[164,1590,1591,1594,1597,1599,1601,1604,1606,1609,1611,1614,1616,1618,1621,1623,1625],{"class":166,"line":676},[164,1592,1593],{"class":214},"            raise",[164,1595,1596],{"class":238}," RuntimeError",[164,1598,389],{"class":218},[164,1600,404],{"class":214},[164,1602,1603],{"class":174},"\"staged ",[164,1605,315],{"class":410},[164,1607,1608],{"class":218},"staged",[164,1610,426],{"class":410},[164,1612,1613],{"class":174}," rows, expected ",[164,1615,315],{"class":410},[164,1617,413],{"class":238},[164,1619,1620],{"class":218},"(df)",[164,1622,426],{"class":410},[164,1624,407],{"class":174},[164,1626,380],{"class":218},[164,1628,1629],{"class":166,"line":702},[164,1630,232],{"emptyLinePlaceholder":231},[164,1632,1633,1636,1638],{"class":166,"line":727},[164,1634,1635],{"class":218},"        removed ",[164,1637,264],{"class":214},[164,1639,1640],{"class":218}," conn.execute(\n",[164,1642,1643,1646,1648,1651,1653,1655,1657,1660,1663,1666],{"class":166,"line":732},[164,1644,1645],{"class":218},"            text(",[164,1647,404],{"class":214},[164,1649,1650],{"class":174},"\"DELETE FROM ",[164,1652,315],{"class":410},[164,1654,1461],{"class":218},[164,1656,426],{"class":410},[164,1658,1659],{"class":174}," WHERE period = :p\"",[164,1661,1662],{"class":218},"), {",[164,1664,1665],{"class":174},"\"p\"",[164,1667,1668],{"class":218},": period}).rowcount\n",[164,1670,1671],{"class":166,"line":763},[164,1672,1673],{"class":218},"        conn.execute(text(\n",[164,1675,1676,1679,1682,1684,1686,1688],{"class":166,"line":794},[164,1677,1678],{"class":214},"            f",[164,1680,1681],{"class":174},"\"INSERT INTO ",[164,1683,315],{"class":410},[164,1685,1461],{"class":218},[164,1687,426],{"class":410},[164,1689,1690],{"class":174}," (order_id, region, order_date, amount, \"\n",[164,1692,1693,1695],{"class":166,"line":812},[164,1694,1678],{"class":214},[164,1696,1697],{"class":174},"\"cost_centre, period) \"\n",[164,1699,1700,1702],{"class":166,"line":830},[164,1701,1678],{"class":214},[164,1703,1704],{"class":174},"\"SELECT order_id, region, order_date, amount, cost_centre, period \"\n",[164,1706,1707,1709,1712,1714,1716,1718,1720],{"class":166,"line":835},[164,1708,1678],{"class":214},[164,1710,1711],{"class":174},"\"FROM ",[164,1713,315],{"class":410},[164,1715,1565],{"class":218},[164,1717,426],{"class":410},[164,1719,407],{"class":174},[164,1721,1722],{"class":218},"))\n",[164,1724,1725,1728,1730,1733,1735,1737,1739,1741],{"class":166,"line":859},[164,1726,1727],{"class":218},"        conn.execute(text(",[164,1729,404],{"class":214},[164,1731,1732],{"class":174},"\"DROP TABLE ",[164,1734,315],{"class":410},[164,1736,1565],{"class":218},[164,1738,426],{"class":410},[164,1740,407],{"class":174},[164,1742,1722],{"class":218},[164,1744,1745],{"class":166,"line":878},[164,1746,232],{"emptyLinePlaceholder":231},[164,1748,1749,1751,1754,1756,1759,1762,1765,1768,1770,1772],{"class":166,"line":883},[164,1750,862],{"class":214},[164,1752,1753],{"class":218}," {",[164,1755,840],{"class":174},[164,1757,1758],{"class":218},": period, ",[164,1760,1761],{"class":174},"\"replaced\"",[164,1763,1764],{"class":218},": removed, ",[164,1766,1767],{"class":174},"\"inserted\"",[164,1769,321],{"class":218},[164,1771,413],{"class":238},[164,1773,1774],{"class":218},"(df)}\n",[164,1776,1777],{"class":166,"line":888},[164,1778,232],{"emptyLinePlaceholder":231},[164,1780,1782],{"class":166,"line":1781},31,[164,1783,232],{"emptyLinePlaceholder":231},[164,1785,1787,1789],{"class":166,"line":1786},32,[164,1788,386],{"class":238},[164,1790,1791],{"class":218},"(load(check(df)))\n",[10,1793,1794,1795,1798,1799,1803,1804,1807,1808,1811],{},"Three details carry the weight. ",[161,1796,1797],{},"engine.begin()"," gives one transaction for the delete, the insert and the drop, so an error at any point leaves the live table untouched. Deleting the period before inserting makes the load idempotent — running the same file twice produces the same table, which is exactly what you need when a ",[17,1800,1802],{"href":1801},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fretry-a-failed-excel-report-job-in-python\u002F","retry"," fires after an ambiguous failure. And naming the columns in the ",[161,1805,1806],{},"INSERT ... SELECT"," rather than relying on ",[161,1809,1810],{},"SELECT *"," protects the load from a column-order change in either table.",[23,1813,32,1818,32,1821,32,1824,32,1827,32,1831,32,1837,32,1842,32,1846,32,1851,32,1855,32,1858,32,1862,32,1867,32,1870,32,1874,32,1877,32,1881,32,1884,32,1887,32,1890,32,1898],{"viewBox":1814,"role":26,"ariaLabelledBy":1815,"xmlns":30,"style":31},"0 0 740 240",[1816,1817],"ld-tx-t","ld-tx-d",[34,1819,1820],{"id":1816},"What a failure looks like with and without a transaction",[38,1822,1823],{"id":1817},"Without a transaction, the delete commits and the insert fails, leaving the live table missing an entire period until someone restores it. Inside engine.begin, the same failure rolls the delete back, so readers see the previous load and the submitter gets an error naming the problem.",[42,1825],{"x":44,"y":44,"width":45,"height":1826,"fill":47},"240",[49,1828,1830],{"x":1330,"y":52,"style":1829},"font-size:12.5px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","autocommit per statement",[42,1832],{"x":1833,"y":1834,"width":67,"height":1835,"rx":1836,"fill":62,"stroke":63,"style":64},"24","48","42","8",[49,1838,1841],{"x":1839,"y":1840,"style":74},"89","74","DELETE — committed",[42,1843],{"x":1844,"y":1834,"width":1845,"height":1835,"rx":1836,"fill":62,"stroke":63,"style":64},"170","146",[49,1847,1850],{"x":1848,"y":1840,"style":1849},"243","font-size:11px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","INSERT — fails ✗",[49,1852,1854],{"x":1330,"y":1853,"style":1289},"122","the period is now missing entirely",[49,1856,1857],{"x":1330,"y":1845,"style":1294},"readers see an empty month; recovery needs a",[49,1859,1861],{"x":1330,"y":1860,"style":1294},"164","restore or a re-run nobody has scheduled",[49,1863,1866],{"x":1864,"y":52,"style":1865},"556","font-size:12.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","inside engine.begin()",[42,1868],{"x":1869,"y":1834,"width":67,"height":1835,"rx":1836,"fill":136,"stroke":137,"style":64},"396",[49,1871,1873],{"x":1872,"y":1840,"style":74},"461","DELETE — pending",[42,1875],{"x":1876,"y":1834,"width":1845,"height":1835,"rx":1836,"fill":136,"stroke":137,"style":64},"542",[49,1878,1850],{"x":1879,"y":1840,"style":1880},"615","font-size:11px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle",[49,1882,1883],{"x":1864,"y":1853,"style":1289},"both are rolled back together",[49,1885,1886],{"x":1864,"y":1845,"style":1294},"the previous load is still there, and the error",[49,1888,1889],{"x":1864,"y":1860,"style":1294},"goes back to whoever submitted the file",[42,1891],{"x":1892,"y":1893,"width":1894,"height":1835,"rx":1895,"fill":1896,"stroke":1897},"150","186","440","10","#f0f4ff","var(--brand,#5b5cf0)",[49,1899,1902],{"x":51,"y":1900,"style":1901},"212","font-size:11.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","A load either happened or it did not — there is no useful state in between",[149,1904,1906],{"id":1905},"step-5-speed-the-insert-up-when-the-file-is-large","Step 5: Speed the insert up when the file is large",[10,1908,1909,1912],{},[161,1910,1911],{},"to_sql"," inserts row by row unless you tell it otherwise. Two arguments change the profile substantially:",[154,1914,1916],{"className":205,"code":1915,"language":207,"meta":159,"style":159},"df.to_sql(\"sales_staging\", engine, if_exists=\"replace\", index=False,\n          chunksize=10_000,        # rows per round trip\n          method=\"multi\")          # one multi-row INSERT per chunk\n",[161,1917,1918,1945,1961],{"__ignoreMap":159},[164,1919,1920,1923,1926,1929,1931,1933,1935,1937,1939,1941,1943],{"class":166,"line":167},[164,1921,1922],{"class":218},"df.to_sql(",[164,1924,1925],{"class":174},"\"sales_staging\"",[164,1927,1928],{"class":218},", engine, ",[164,1930,1478],{"class":260},[164,1932,264],{"class":214},[164,1934,1483],{"class":174},[164,1936,327],{"class":218},[164,1938,1488],{"class":260},[164,1940,264],{"class":214},[164,1942,1493],{"class":238},[164,1944,254],{"class":218},[164,1946,1947,1950,1952,1955,1958],{"class":166,"line":228},[164,1948,1949],{"class":260},"          chunksize",[164,1951,264],{"class":214},[164,1953,1954],{"class":238},"10_000",[164,1956,1957],{"class":218},",        ",[164,1959,1960],{"class":286},"# rows per round trip\n",[164,1962,1963,1966,1968,1970,1973],{"class":166,"line":235},[164,1964,1965],{"class":260},"          method",[164,1967,264],{"class":214},[164,1969,1515],{"class":174},[164,1971,1972],{"class":218},")          ",[164,1974,1975],{"class":286},"# one multi-row INSERT per chunk\n",[10,1977,1978,1981,1982,1985,1986,1989,1990,1993,1994,1997],{},[161,1979,1980],{},"method=\"multi\""," builds a single statement with many value tuples, which cuts network round trips dramatically on a remote database — typically several times faster for wide files. Keep ",[161,1983,1984],{},"chunksize"," bounded, though: some drivers have a parameter limit per statement (SQL Server's is 2,100), and a chunk of 10,000 rows across 8 columns exceeds it. If the load runs into the hundreds of thousands of rows regularly, the database's own bulk loader — ",[161,1987,1988],{},"COPY"," in Postgres, ",[161,1991,1992],{},"LOAD DATA"," in MySQL, ",[161,1995,1996],{},"bcp"," for SQL Server — beats anything pandas can do, and pandas' job becomes writing a clean CSV for it.",[149,1999,2001],{"id":2000},"common-pitfalls-and-gotchas","Common pitfalls and gotchas",[1461,2003,2004,2020],{},[2005,2006,2007],"thead",{},[2008,2009,2010,2014,2017],"tr",{},[2011,2012,2013],"th",{},"Symptom",[2011,2015,2016],{},"Cause",[2011,2018,2019],{},"Fix",[2021,2022,2023,2037,2057,2077,2088,2101,2118,2131],"tbody",{},[2008,2024,2025,2029,2032],{},[2026,2027,2028],"td",{},"IDs lost their leading zeros",[2026,2030,2031],{},"pandas inferred an integer column",[2026,2033,2034,2036],{},[161,2035,118],{}," at the read",[2008,2038,2039,2044,2052],{},[2026,2040,2041,2042],{},"Every column is ",[161,2043,447],{},[2026,2045,2046,2047,2049,2050],{},"Human placeholders like ",[161,2048,599],{}," and ",[161,2051,443],{},[2026,2053,2054,2055],{},"List them in ",[161,2056,436],{},[2008,2058,2059,2065,2068],{},[2026,2060,2061,2062],{},"Columns named ",[161,2063,2064],{},"Unnamed: 0",[2026,2066,2067],{},"A title row above the headers",[2026,2069,2070,2073,2074],{},[161,2071,2072],{},"skiprows",", or ",[161,2075,2076],{},"header=",[2008,2078,2079,2082,2085],{},[2026,2080,2081],{},"Rows doubled after a rerun",[2026,2083,2084],{},"Append without deleting first",[2026,2086,2087],{},"Delete the period, then insert, in one transaction",[2008,2089,2090,2093,2096],{},[2026,2091,2092],{},"A month vanished from the table",[2026,2094,2095],{},"Delete committed, insert failed",[2026,2097,2098,2099],{},"Wrap both in ",[161,2100,1797],{},[2008,2102,2103,2108,2111],{},[2026,2104,2105,2107],{},[161,2106,1911],{}," takes minutes",[2026,2109,2110],{},"Row-by-row inserts",[2026,2112,2113,2115,2116],{},[161,2114,1984],{}," plus ",[161,2117,1980],{},[2008,2119,2120,2123,2126],{},[2026,2121,2122],{},"Too many parameters error",[2026,2124,2125],{},"Chunk exceeds the driver's limit",[2026,2127,2128,2129],{},"Lower ",[161,2130,1984],{},[2008,2132,2133,2136,2139],{},[2026,2134,2135],{},"Amounts differ by a cent",[2026,2137,2138],{},"Float rounding at the boundary",[2026,2140,2141,2142],{},"Round before writing; store ",[161,2143,2144],{},"NUMERIC",[149,2146,2148],{"id":2147},"performance-and-scale-notes","Performance and scale notes",[10,2150,2151,2152,2156],{},"Reading the workbook is usually the slow half: openpyxl parses XML, so a 200,000-row upload takes tens of seconds regardless of what happens afterwards. If the same file is loaded repeatedly, convert it once to CSV or Parquet and load from that — ",[17,2153,2155],{"href":2154},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fconvert-excel-to-csv-with-python\u002F","Convert Excel to CSV with Python"," covers the conversion, and it typically cuts the read to a fraction of the time.",[10,2158,2159],{},"On the database side, the expensive part of a large load is index maintenance. For a bulk refresh of a big table it is often faster to drop the non-clustered indexes, load, and rebuild them — but only if the table is not being read during the load, which is another reason for the staging pattern.",[149,2161,2163],{"id":2162},"conclusion","Conclusion",[10,2165,2166,2167,2169],{},"Treat an uploaded workbook as untrusted input. Pin the types at the read, normalise the columns in one visible mapping, coerce dates and numbers with ",[161,2168,905],{}," so problems are collected rather than thrown one at a time, and reject the file with a message that names the offending rows. Then stage it, verify the count, and merge with a delete-then-insert inside a single transaction so the load is both atomic and repeatable. That is what makes a spreadsheet import something you can leave running.",[149,2171,2173],{"id":2172},"frequently-asked-questions","Frequently asked questions",[10,2175,2176,2184],{},[2177,2178,2179,2180,2183],"strong",{},"Should I use ",[161,2181,2182],{},"if_exists=\"append\""," straight onto the live table?","\nOnly for a table nobody reads while the load runs. Otherwise append into a staging table, validate, and move the rows across inside one transaction so a bad file is never visible.",[10,2186,2187,2190],{},[2177,2188,2189],{},"How do I make re-running the same file safe?","\nDelete the period or batch you are about to load, then insert. That makes the load idempotent — a retry after a half-finished run produces the same table rather than duplicate rows.",[10,2192,2193,2196,2197,2200,2201,2204,2205,2207,2208,2211],{},[2177,2194,2195],{},"Why did my text IDs turn into numbers?","\npandas infers types per column, so ",[161,2198,2199],{},"00123"," becomes ",[161,2202,2203],{},"123",". Pass ",[161,2206,118],{}," to ",[161,2209,2210],{},"read_excel",", before any conversion has happened.",[10,2213,2214,2219,2220,2049,2222,2224],{},[2177,2215,2216,2218],{},[161,2217,1911],{}," is slow — what are the options?","\nPass ",[161,2221,1984],{},[161,2223,1980],{}," to batch the inserts, and drop non-essential indexes during a bulk load. For very large files, most databases have a native bulk loader that beats any row-by-row path.",[149,2226,2228],{"id":2227},"related","Related",[10,2230,2231],{},"Up to the parent guide:",[2233,2234,2235],"ul",{},[2236,2237,2238,2240],"li",{},[17,2239,20],{"href":19}," — the round trip this is the return leg of.",[10,2242,2243],{},"Related guides:",[2233,2245,2246,2253,2258,2265],{},[2236,2247,2248,2252],{},[17,2249,2251],{"href":2250},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-sql-query-results-to-excel-with-python\u002F","Export SQL Query Results to Excel with Python"," — the outbound direction.",[2236,2254,2255,2257],{},[17,2256,1338],{"href":1337}," — the checks that belong before the staging write.",[2236,2259,2260,2264],{},[17,2261,2263],{"href":2262},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Ffind-duplicate-rows-in-excel-with-python\u002F","Find Duplicate Rows in Excel with Python"," — reporting duplicated keys back to the submitter.",[2236,2266,2267,2271,2272,2274],{},[17,2268,2270],{"href":2269},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-excel-data-types-with-pandas\u002F","Check Excel Data Types with pandas"," — why a column arrives as ",[161,2273,447],{}," and what to do about it.",[2276,2277,2278],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}",{"title":159,"searchDepth":228,"depth":228,"links":2280},[2281,2282,2283,2284,2285,2286,2287,2288,2289,2290,2291],{"id":151,"depth":228,"text":152},{"id":190,"depth":228,"text":191},{"id":451,"depth":228,"text":452},{"id":925,"depth":228,"text":926},{"id":1341,"depth":228,"text":1342},{"id":1905,"depth":228,"text":1906},{"id":2000,"depth":228,"text":2001},{"id":2147,"depth":228,"text":2148},{"id":2162,"depth":228,"text":2163},{"id":2172,"depth":228,"text":2173},{"id":2227,"depth":228,"text":2228},"2026-08-10","Take a spreadsheet people filled in and get it into a table safely: normalise the columns, pin the types, stage the rows, validate them, then merge in one transaction that either succeeds or leaves nothing behind.","md",[2296,2299,2301,2305],{"q":2297,"a":2298},"Should I use if_exists=\"append\" straight onto the live table?","Only for a table nobody reads while the load runs. Otherwise append into a staging table, validate, and move the rows across inside one transaction so a bad file is never visible.",{"q":2189,"a":2300},"Delete the period or batch you are about to load, then insert. That makes the load idempotent — a retry after a half-finished run produces the same table rather than duplicate rows.",{"q":2195,"a":2302},{"pandas infers types per column, so 00123 becomes 123":2303},{" Pass dtype={\"order_id\"":2304},"str} to read_excel, before any conversion has happened.",{"q":2306,"a":2307},"to_sql is slow — what are the options?","Pass chunksize and method=\"multi\" to batch the inserts, and drop non-essential indexes during a bulk load. For very large files, most databases have a native bulk loader that beats any row-by-row path.",{"breadcrumb":2309},[2310,2313,2316,2317],{"name":2311,"item":2312},"Home","\u002F",{"name":2314,"item":2315},"Advanced Data Transformation and Cleaning","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":20,"item":19},{"name":5,"item":2318},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-an-excel-file-into-a-sql-database-with-pandas\u002F","\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-an-excel-file-into-a-sql-database-with-pandas",{"title":2321,"description":2322},"Load an Excel File into SQL with pandas","Import .xlsx data into a SQL table with pandas and SQLAlchemy — column normalisation, explicit dtypes, staging tables, duplicate checks and an idempotent transactional merge.","load-an-excel-file-into-a-sql-database-with-pandas","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-an-excel-file-into-a-sql-database-with-pandas\u002Findex","how-to","CQVtPUGKeJlmQOv-OeNrIYW3Z9JcSxq_yQXs7-ZfH-o",[2328,2332],{"title":2329,"path":2330,"stem":2331,"children":-1},"Fetch API Data into Excel with Python requests","\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Ffetch-api-data-into-excel-with-python-requests","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Ffetch-api-data-into-excel-with-python-requests\u002Findex",{"title":2333,"path":2334,"stem":2335,"children":-1},"Refresh an Excel Report from a Database on a Schedule","\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Frefresh-an-excel-report-from-a-database-on-a-schedule","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Frefresh-an-excel-report-from-a-database-on-a-schedule\u002Findex",1786800027161]