[{"data":1,"prerenderedAt":2232},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fupsert-excel-rows-into-a-database-table":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fupsert-excel-rows-into-a-database-table":2198,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fupsert-excel-rows-into-a-database-table":2207,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fupsert-excel-rows-into-a-database-table":2212,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fupsert-excel-rows-into-a-database-table":2213},{"id":4,"title":5,"body":6,"dateModified":2164,"datePublished":2164,"description":2165,"extension":2166,"faq":2167,"keyword":2176,"kind":2177,"meta":2178,"navigation":415,"path":2187,"seo":2188,"slug":2191,"stem":2192,"topics":2193,"type":2196,"__hash__":2197},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fupsert-excel-rows-into-a-database-table\u002Findex.md","Upsert Excel Rows into a Database Table with Python",{"type":7,"value":8,"toc":2152},"minimark",[9,32,336,341,371,374,628,636,640,643,824,827,831,838,996,1030,1197,1201,1227,1494,1504,1508,1518,1771,1785,1789,1792,1859,1870,1999,2003,2064,2068,2087,2091,2097,2103,2109,2115,2119,2148],[10,11,12,13,17,18,21,22,25,26,31],"p",{},"A product list, price table or cost-centre mapping often lives in an Excel sheet that someone maintains, while applications read the same data from a database table. Loading the sheet once with ",[14,15,16],"code",{},"to_sql"," is easy; loading it again next week is where it goes wrong. ",[14,19,20],{},"if_exists=\"append\""," duplicates every row or fails on the primary key, and ",[14,23,24],{},"if_exists=\"replace\""," drops the table — along with its indexes, constraints and any rows added by other systems. What you want is an upsert: insert rows with new keys, update rows whose values changed, leave the rest alone. This guide, part of ",[27,28,30],"a",{"href":29},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002F","Moving Data Between Excel and Databases",", shows two ways to do it with pandas and SQLAlchemy, using SQLite so every example runs as written.",[33,34,42,43,42,47,42,51,42,59,42,68,42,72,42,79,42,83,42,87,42,91,42,95,42,99,42,104,42,106,42,110,42,112,42,115,42,117,42,120,42,123,42,127,42,129,42,133,42,135,42,138,42,140,42,143,42,146,42,150,42,152,42,155,42,157,42,160,42,162,42,165,42,168,42,172,42,174,42,177,42,179,42,182,42,184,42,187,42,190,42,194,42,197,42,201,42,204,42,207,42,210,42,212,42,215,42,217,42,219,42,221,42,223,42,225,42,227,42,229,42,231,42,233,42,235,42,237,42,239,42,241,42,243,42,245,42,247,42,249,42,251,42,253,42,256,42,258,42,261,42,263,42,265,42,267,42,269,42,271,42,273,42,275,42,277,42,280,42,284,42,286,42,289,42,291,42,294,42,296,42,299,42,305,42,309,42,317,42,322,42,330],"svg",{"viewBox":35,"role":36,"ariaLabelledBy":37,"xmlns":40,"style":41},"33 0 694 220","img",[38,39],"up-overview-t","up-overview-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:694px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[44,45,46],"title",{"id":38},"The sheet's rows merged into the existing table",[48,49,50],"desc",{"id":39},"The table holds A-1 Tea at 2.50, B-2 Coffee at 4.00 and C-3 Cocoa. The Excel sheet has A-1 Tea at 2.50, B-2 Coffee beans at 4.20 and a new D-4 Mate at 3.80. After the upsert, A-1 is unchanged, B-2 is updated, C-3 is kept and D-4 is inserted.",[52,53],"rect",{"x":54,"y":55,"width":56,"height":57,"fill":58},"33","0","694","220","#ffffff",[52,60],{"x":61,"y":62,"width":63,"height":64,"rx":55,"fill":65,"stroke":66,"style":67},"49.0","40.0","32.0","24.0","#e7ebef","var(--line,#cdd5e6)","stroke-width:1px",[52,69],{"x":70,"y":62,"width":71,"height":64,"rx":55,"fill":65,"stroke":66,"style":67},"81.0","47.4",[73,74,78],"text",{"x":75,"y":76,"style":77},"104.7","57.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[52,80],{"x":81,"y":62,"width":82,"height":64,"rx":55,"fill":65,"stroke":66,"style":67},"128.4","66.1",[73,84,86],{"x":85,"y":76,"style":77},"161.4","B",[52,88],{"x":89,"y":62,"width":90,"height":64,"rx":55,"fill":65,"stroke":66,"style":67},"194.5","55.4",[73,92,94],{"x":93,"y":76,"style":77},"222.2","C",[52,96],{"x":61,"y":97,"width":63,"height":98,"rx":55,"fill":65,"stroke":66,"style":67},"64.0","28.0",[73,100,103],{"x":101,"y":102,"style":77},"65.0","83.0","1",[52,105],{"x":70,"y":97,"width":71,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,107,109],{"x":75,"y":102,"style":108},"font-size:12px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","sku",[52,111],{"x":81,"y":97,"width":82,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,113,114],{"x":85,"y":102,"style":108},"name",[52,116],{"x":89,"y":97,"width":90,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,118,119],{"x":93,"y":102,"style":108},"price",[52,121],{"x":61,"y":122,"width":63,"height":98,"rx":55,"fill":65,"stroke":66,"style":67},"92.0",[73,124,126],{"x":101,"y":125,"style":77},"111.0","2",[52,128],{"x":70,"y":122,"width":71,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,130,132],{"x":75,"y":125,"style":131},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","A-1",[52,134],{"x":81,"y":122,"width":82,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,136,137],{"x":85,"y":125,"style":131},"Tea",[52,139],{"x":89,"y":122,"width":90,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,141,142],{"x":93,"y":125,"style":131},"2.50",[52,144],{"x":61,"y":145,"width":63,"height":98,"rx":55,"fill":65,"stroke":66,"style":67},"120.0",[73,147,149],{"x":101,"y":148,"style":77},"139.0","3",[52,151],{"x":70,"y":145,"width":71,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,153,154],{"x":75,"y":148,"style":131},"B-2",[52,156],{"x":81,"y":145,"width":82,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,158,159],{"x":85,"y":148,"style":131},"Coffee",[52,161],{"x":89,"y":145,"width":90,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,163,164],{"x":93,"y":148,"style":131},"4.00",[52,166],{"x":61,"y":167,"width":63,"height":98,"rx":55,"fill":65,"stroke":66,"style":67},"148.0",[73,169,171],{"x":101,"y":170,"style":77},"167.0","4",[52,173],{"x":70,"y":167,"width":71,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,175,176],{"x":75,"y":170,"style":131},"C-3",[52,178],{"x":81,"y":167,"width":82,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,180,181],{"x":85,"y":170,"style":131},"Cocoa",[52,183],{"x":89,"y":167,"width":90,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,185,186],{"x":93,"y":170,"style":131},"3.00",[52,188],{"x":189,"y":62,"width":63,"height":64,"rx":55,"fill":65,"stroke":66,"style":67},"469.3",[52,191],{"x":192,"y":62,"width":193,"height":64,"rx":55,"fill":65,"stroke":66,"style":67},"501.3","47.5",[73,195,78],{"x":196,"y":76,"style":77},"525.1",[52,198],{"x":199,"y":62,"width":200,"height":64,"rx":55,"fill":65,"stroke":66,"style":67},"548.8","106.8",[73,202,86],{"x":203,"y":76,"style":77},"602.2",[52,205],{"x":206,"y":62,"width":90,"height":64,"rx":55,"fill":65,"stroke":66,"style":67},"655.6",[73,208,94],{"x":209,"y":76,"style":77},"683.3",[52,211],{"x":189,"y":97,"width":63,"height":98,"rx":55,"fill":65,"stroke":66,"style":67},[73,213,103],{"x":214,"y":102,"style":77},"485.3",[52,216],{"x":192,"y":97,"width":193,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,218,109],{"x":196,"y":102,"style":108},[52,220],{"x":199,"y":97,"width":200,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,222,114],{"x":203,"y":102,"style":108},[52,224],{"x":206,"y":97,"width":90,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,226,119],{"x":209,"y":102,"style":108},[52,228],{"x":189,"y":122,"width":63,"height":98,"rx":55,"fill":65,"stroke":66,"style":67},[73,230,126],{"x":214,"y":125,"style":77},[52,232],{"x":192,"y":122,"width":193,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,234,132],{"x":196,"y":125,"style":131},[52,236],{"x":199,"y":122,"width":200,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,238,137],{"x":203,"y":125,"style":131},[52,240],{"x":206,"y":122,"width":90,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,242,142],{"x":209,"y":125,"style":131},[52,244],{"x":189,"y":145,"width":63,"height":98,"rx":55,"fill":65,"stroke":66,"style":67},[73,246,149],{"x":214,"y":148,"style":77},[52,248],{"x":192,"y":145,"width":193,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,250,154],{"x":196,"y":148,"style":131},[52,252],{"x":199,"y":145,"width":200,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,254,255],{"x":203,"y":148,"style":131},"Coffee beans",[52,257],{"x":206,"y":145,"width":90,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,259,260],{"x":209,"y":148,"style":131},"4.20",[52,262],{"x":189,"y":167,"width":63,"height":98,"rx":55,"fill":65,"stroke":66,"style":67},[73,264,171],{"x":214,"y":170,"style":77},[52,266],{"x":192,"y":167,"width":193,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,268,176],{"x":196,"y":170,"style":131},[52,270],{"x":199,"y":167,"width":200,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,272,181],{"x":203,"y":170,"style":131},[52,274],{"x":206,"y":167,"width":90,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,276,186],{"x":209,"y":170,"style":131},[52,278],{"x":189,"y":279,"width":63,"height":98,"rx":55,"fill":65,"stroke":66,"style":67},"176.0",[73,281,283],{"x":214,"y":282,"style":77},"195.0","5",[52,285],{"x":192,"y":279,"width":193,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,287,288],{"x":196,"y":282,"style":131},"D-4",[52,290],{"x":199,"y":279,"width":200,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,292,293],{"x":203,"y":282,"style":131},"Mate",[52,295],{"x":206,"y":279,"width":90,"height":98,"rx":55,"fill":58,"stroke":66,"style":67},[73,297,298],{"x":209,"y":282,"style":131},"3.80",[73,300,304],{"x":301,"y":302,"style":303},"149.4","29.0","font-size:12.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","before",[73,306,308],{"x":307,"y":302,"style":303},"590.2","after",[310,311],"line",{"x1":312,"y1":313,"x2":314,"y2":313,"stroke":315,"style":316},"259.9","108.0","455.3","var(--brand,#5b5cf0)","stroke-width:2px",[318,319],"polygon",{"points":320,"fill":321},"459.3,108.0 450.3,113.0 450.3,103.0","#5b5cf0",[52,323],{"x":324,"y":325,"width":326,"height":327,"rx":328,"fill":58,"stroke":66,"style":329},"287.8","75.0","143.6","22.0","11","stroke-width:1.2px",[73,331,335],{"x":332,"y":333,"style":334},"359.6","90.0","font-size:11.5px;font-weight:400;fill:var(--brand-strong,#4338ca);text-anchor:middle;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace","upsert sheet rows",[337,338,340],"h2",{"id":339},"prerequisites","Prerequisites",[342,343,348],"pre",{"className":344,"code":345,"language":346,"meta":347,"style":347},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl sqlalchemy\n","bash","",[14,349,350],{"__ignoreMap":347},[351,352,354,358,362,365,368],"span",{"class":310,"line":353},1,[351,355,357],{"class":356},"sMTad","pip",[351,359,361],{"class":360},"srMev"," install",[351,363,364],{"class":360}," pandas",[351,366,367],{"class":360}," openpyxl",[351,369,370],{"class":360}," sqlalchemy\n",[10,372,373],{},"An existing table with a primary key, and the maintained sheet:",[342,375,379],{"className":376,"code":377,"language":378,"meta":347,"style":347},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\nimport sqlalchemy as sa\n\nengine = sa.create_engine(\"sqlite:\u002F\u002F\u002Fproducts.db\")\nwith engine.begin() as con:\n    con.exec_driver_sql(\"DROP TABLE IF EXISTS products\")\n    con.exec_driver_sql(\"\"\"CREATE TABLE products (\n        sku TEXT PRIMARY KEY, name TEXT NOT NULL, price REAL, updated_at TEXT)\"\"\")\n    con.exec_driver_sql(\"\"\"INSERT INTO products VALUES\n        ('A-1', 'Tea', 2.5, '2026-09-01'), ('B-2', 'Coffee', 4.0, '2026-09-01'),\n        ('C-3', 'Cocoa', 3.0, '2026-09-01')\"\"\")\n\npd.DataFrame({\"sku\": [\"A-1\", \"B-2\", \"D-4\"], \"name\": [\"Tea\", \"Coffee beans\", \"Mate\"],\n              \"price\": [2.5, 4.2, 3.8]}).to_excel(\"products.xlsx\", index=False)\nsheet = pd.read_excel(\"products.xlsx\", dtype={\"sku\": str})\n","python",[14,380,381,397,410,417,435,449,460,468,476,484,490,498,503,553,594],{"__ignoreMap":347},[351,382,383,387,391,394],{"class":310,"line":353},[351,384,386],{"class":385},"s-kum","import",[351,388,390],{"class":389},"skGVy"," pandas ",[351,392,393],{"class":385},"as",[351,395,396],{"class":389}," pd\n",[351,398,400,402,405,407],{"class":310,"line":399},2,[351,401,386],{"class":385},[351,403,404],{"class":389}," sqlalchemy ",[351,406,393],{"class":385},[351,408,409],{"class":389}," sa\n",[351,411,413],{"class":310,"line":412},3,[351,414,416],{"emptyLinePlaceholder":415},true,"\n",[351,418,420,423,426,429,432],{"class":310,"line":419},4,[351,421,422],{"class":389},"engine ",[351,424,425],{"class":385},"=",[351,427,428],{"class":389}," sa.create_engine(",[351,430,431],{"class":360},"\"sqlite:\u002F\u002F\u002Fproducts.db\"",[351,433,434],{"class":389},")\n",[351,436,438,441,444,446],{"class":310,"line":437},5,[351,439,440],{"class":385},"with",[351,442,443],{"class":389}," engine.begin() ",[351,445,393],{"class":385},[351,447,448],{"class":389}," con:\n",[351,450,452,455,458],{"class":310,"line":451},6,[351,453,454],{"class":389},"    con.exec_driver_sql(",[351,456,457],{"class":360},"\"DROP TABLE IF EXISTS products\"",[351,459,434],{"class":389},[351,461,463,465],{"class":310,"line":462},7,[351,464,454],{"class":389},[351,466,467],{"class":360},"\"\"\"CREATE TABLE products (\n",[351,469,471,474],{"class":310,"line":470},8,[351,472,473],{"class":360},"        sku TEXT PRIMARY KEY, name TEXT NOT NULL, price REAL, updated_at TEXT)\"\"\"",[351,475,434],{"class":389},[351,477,479,481],{"class":310,"line":478},9,[351,480,454],{"class":389},[351,482,483],{"class":360},"\"\"\"INSERT INTO products VALUES\n",[351,485,487],{"class":310,"line":486},10,[351,488,489],{"class":360},"        ('A-1', 'Tea', 2.5, '2026-09-01'), ('B-2', 'Coffee', 4.0, '2026-09-01'),\n",[351,491,493,496],{"class":310,"line":492},11,[351,494,495],{"class":360},"        ('C-3', 'Cocoa', 3.0, '2026-09-01')\"\"\"",[351,497,434],{"class":389},[351,499,501],{"class":310,"line":500},12,[351,502,416],{"emptyLinePlaceholder":415},[351,504,506,509,512,515,518,521,524,526,529,532,535,537,540,542,545,547,550],{"class":310,"line":505},13,[351,507,508],{"class":389},"pd.DataFrame({",[351,510,511],{"class":360},"\"sku\"",[351,513,514],{"class":389},": [",[351,516,517],{"class":360},"\"A-1\"",[351,519,520],{"class":389},", ",[351,522,523],{"class":360},"\"B-2\"",[351,525,520],{"class":389},[351,527,528],{"class":360},"\"D-4\"",[351,530,531],{"class":389},"], ",[351,533,534],{"class":360},"\"name\"",[351,536,514],{"class":389},[351,538,539],{"class":360},"\"Tea\"",[351,541,520],{"class":389},[351,543,544],{"class":360},"\"Coffee beans\"",[351,546,520],{"class":389},[351,548,549],{"class":360},"\"Mate\"",[351,551,552],{"class":389},"],\n",[351,554,556,559,561,565,567,570,572,575,578,581,583,587,589,592],{"class":310,"line":555},14,[351,557,558],{"class":360},"              \"price\"",[351,560,514],{"class":389},[351,562,564],{"class":563},"sP0c6","2.5",[351,566,520],{"class":389},[351,568,569],{"class":563},"4.2",[351,571,520],{"class":389},[351,573,574],{"class":563},"3.8",[351,576,577],{"class":389},"]}).to_excel(",[351,579,580],{"class":360},"\"products.xlsx\"",[351,582,520],{"class":389},[351,584,586],{"class":585},"sa561","index",[351,588,425],{"class":385},[351,590,591],{"class":563},"False",[351,593,434],{"class":389},[351,595,597,600,602,605,607,609,612,614,617,619,622,625],{"class":310,"line":596},15,[351,598,599],{"class":389},"sheet ",[351,601,425],{"class":385},[351,603,604],{"class":389}," pd.read_excel(",[351,606,580],{"class":360},[351,608,520],{"class":389},[351,610,611],{"class":585},"dtype",[351,613,425],{"class":385},[351,615,616],{"class":389},"{",[351,618,511],{"class":360},[351,620,621],{"class":389},": ",[351,623,624],{"class":563},"str",[351,626,627],{"class":389},"})\n",[10,629,630,631,635],{},"The table must have a primary key or unique constraint on the key column; the database uses it to decide whether a row is new. Creating tables from Excel data in the first place is covered in ",[27,632,634],{"href":633},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-an-excel-file-into-a-sql-database-with-pandas\u002F","Load an Excel File into a SQL Database with pandas",".",[337,637,639],{"id":638},"step-1-check-the-sheet-before-touching-the-table","Step 1 — Check the sheet before touching the table",[10,641,642],{},"An upsert trusts the key. Duplicate or blank keys in the sheet either make the statement fail halfway or let the last duplicate silently win, depending on the database. Check first:",[342,644,646],{"className":376,"code":645,"language":378,"meta":347,"style":347},"problems = []\nif sheet[\"sku\"].isna().any():\n    problems.append(f\"{sheet['sku'].isna().sum()} rows without a sku\")\ndupes = sheet.loc[sheet[\"sku\"].duplicated(keep=False), \"sku\"].unique().tolist()\nif dupes:\n    problems.append(f\"duplicate skus: {dupes}\")\nif problems:\n    raise ValueError(\"; \".join(problems))\nsheet[\"sku\"] = sheet[\"sku\"].str.strip()\nprint(len(sheet), \"rows ready\")          # 3 rows ready\n",[14,647,648,658,671,702,732,739,759,766,783,801],{"__ignoreMap":347},[351,649,650,653,655],{"class":310,"line":353},[351,651,652],{"class":389},"problems ",[351,654,425],{"class":385},[351,656,657],{"class":389}," []\n",[351,659,660,663,666,668],{"class":310,"line":399},[351,661,662],{"class":385},"if",[351,664,665],{"class":389}," sheet[",[351,667,511],{"class":360},[351,669,670],{"class":389},"].isna().any():\n",[351,672,673,676,679,682,685,688,691,694,697,700],{"class":310,"line":412},[351,674,675],{"class":389},"    problems.append(",[351,677,678],{"class":385},"f",[351,680,681],{"class":360},"\"",[351,683,616],{"class":684},"sSjpA",[351,686,687],{"class":389},"sheet[",[351,689,690],{"class":360},"'sku'",[351,692,693],{"class":389},"].isna().sum()",[351,695,696],{"class":684},"}",[351,698,699],{"class":360}," rows without a sku\"",[351,701,434],{"class":389},[351,703,704,707,709,712,714,717,720,722,724,727,729],{"class":310,"line":419},[351,705,706],{"class":389},"dupes ",[351,708,425],{"class":385},[351,710,711],{"class":389}," sheet.loc[sheet[",[351,713,511],{"class":360},[351,715,716],{"class":389},"].duplicated(",[351,718,719],{"class":585},"keep",[351,721,425],{"class":385},[351,723,591],{"class":563},[351,725,726],{"class":389},"), ",[351,728,511],{"class":360},[351,730,731],{"class":389},"].unique().tolist()\n",[351,733,734,736],{"class":310,"line":437},[351,735,662],{"class":385},[351,737,738],{"class":389}," dupes:\n",[351,740,741,743,745,748,750,753,755,757],{"class":310,"line":451},[351,742,675],{"class":389},[351,744,678],{"class":385},[351,746,747],{"class":360},"\"duplicate skus: ",[351,749,616],{"class":684},[351,751,752],{"class":389},"dupes",[351,754,696],{"class":684},[351,756,681],{"class":360},[351,758,434],{"class":389},[351,760,761,763],{"class":310,"line":462},[351,762,662],{"class":385},[351,764,765],{"class":389}," problems:\n",[351,767,768,771,774,777,780],{"class":310,"line":470},[351,769,770],{"class":385},"    raise",[351,772,773],{"class":563}," ValueError",[351,775,776],{"class":389},"(",[351,778,779],{"class":360},"\"; \"",[351,781,782],{"class":389},".join(problems))\n",[351,784,785,787,789,792,794,796,798],{"class":310,"line":478},[351,786,687],{"class":389},[351,788,511],{"class":360},[351,790,791],{"class":389},"] ",[351,793,425],{"class":385},[351,795,665],{"class":389},[351,797,511],{"class":360},[351,799,800],{"class":389},"].str.strip()\n",[351,802,803,806,808,811,814,817,820],{"class":310,"line":486},[351,804,805],{"class":563},"print",[351,807,776],{"class":389},[351,809,810],{"class":563},"len",[351,812,813],{"class":389},"(sheet), ",[351,815,816],{"class":360},"\"rows ready\"",[351,818,819],{"class":389},")          ",[351,821,823],{"class":822},"s-wDw","# 3 rows ready\n",[10,825,826],{},"PostgreSQL rejects a statement that would update the same row twice (\"ON CONFLICT DO UPDATE command cannot affect row a second time\"); SQLite applies the duplicates one after another, so the last one wins without a warning. Catching duplicates in Python gives the same, clear failure on every database.",[337,828,830],{"id":829},"step-2-stage-the-sheet-and-upsert-in-one-statement","Step 2 — Stage the sheet and upsert in one statement",[10,832,833,834,837],{},"Write the sheet into a staging table, then merge it into the real table with a single ",[14,835,836],{},"INSERT … ON CONFLICT DO UPDATE",". Everything runs in one transaction, so a failure leaves the table as it was:",[342,839,841],{"className":376,"code":840,"language":378,"meta":347,"style":347},"UPSERT = \"\"\"\nINSERT INTO products (sku, name, price, updated_at)\nSELECT sku, name, price, :today FROM stage_products WHERE true\nON CONFLICT (sku) DO UPDATE SET\n    name = excluded.name,\n    price = excluded.price,\n    updated_at = excluded.updated_at\nWHERE products.name IS NOT excluded.name OR products.price IS NOT excluded.price\n\"\"\"\n\nwith engine.begin() as con:\n    sheet.to_sql(\"stage_products\", con, if_exists=\"replace\", index=False)\n    changed = con.execute(sa.text(UPSERT), {\"today\": \"2026-09-23\"}).rowcount\n    con.exec_driver_sql(\"DROP TABLE stage_products\")\nprint(changed)                                   # 2\nprint(pd.read_sql(\"SELECT * FROM products ORDER BY sku\", engine))\n",[14,842,843,854,859,864,869,874,879,884,889,894,898,908,937,963,972,982],{"__ignoreMap":347},[351,844,845,848,851],{"class":310,"line":353},[351,846,847],{"class":563},"UPSERT",[351,849,850],{"class":385}," =",[351,852,853],{"class":360}," \"\"\"\n",[351,855,856],{"class":310,"line":399},[351,857,858],{"class":360},"INSERT INTO products (sku, name, price, updated_at)\n",[351,860,861],{"class":310,"line":412},[351,862,863],{"class":360},"SELECT sku, name, price, :today FROM stage_products WHERE true\n",[351,865,866],{"class":310,"line":419},[351,867,868],{"class":360},"ON CONFLICT (sku) DO UPDATE SET\n",[351,870,871],{"class":310,"line":437},[351,872,873],{"class":360},"    name = excluded.name,\n",[351,875,876],{"class":310,"line":451},[351,877,878],{"class":360},"    price = excluded.price,\n",[351,880,881],{"class":310,"line":462},[351,882,883],{"class":360},"    updated_at = excluded.updated_at\n",[351,885,886],{"class":310,"line":470},[351,887,888],{"class":360},"WHERE products.name IS NOT excluded.name OR products.price IS NOT excluded.price\n",[351,890,891],{"class":310,"line":478},[351,892,893],{"class":360},"\"\"\"\n",[351,895,896],{"class":310,"line":486},[351,897,416],{"emptyLinePlaceholder":415},[351,899,900,902,904,906],{"class":310,"line":492},[351,901,440],{"class":385},[351,903,443],{"class":389},[351,905,393],{"class":385},[351,907,448],{"class":389},[351,909,910,913,916,919,922,924,927,929,931,933,935],{"class":310,"line":500},[351,911,912],{"class":389},"    sheet.to_sql(",[351,914,915],{"class":360},"\"stage_products\"",[351,917,918],{"class":389},", con, ",[351,920,921],{"class":585},"if_exists",[351,923,425],{"class":385},[351,925,926],{"class":360},"\"replace\"",[351,928,520],{"class":389},[351,930,586],{"class":585},[351,932,425],{"class":385},[351,934,591],{"class":563},[351,936,434],{"class":389},[351,938,939,942,944,947,949,952,955,957,960],{"class":310,"line":505},[351,940,941],{"class":389},"    changed ",[351,943,425],{"class":385},[351,945,946],{"class":389}," con.execute(sa.text(",[351,948,847],{"class":563},[351,950,951],{"class":389},"), {",[351,953,954],{"class":360},"\"today\"",[351,956,621],{"class":389},[351,958,959],{"class":360},"\"2026-09-23\"",[351,961,962],{"class":389},"}).rowcount\n",[351,964,965,967,970],{"class":310,"line":555},[351,966,454],{"class":389},[351,968,969],{"class":360},"\"DROP TABLE stage_products\"",[351,971,434],{"class":389},[351,973,974,976,979],{"class":310,"line":596},[351,975,805],{"class":563},[351,977,978],{"class":389},"(changed)                                   ",[351,980,981],{"class":822},"# 2\n",[351,983,985,987,990,993],{"class":310,"line":984},16,[351,986,805],{"class":563},[351,988,989],{"class":389},"(pd.read_sql(",[351,991,992],{"class":360},"\"SELECT * FROM products ORDER BY sku\"",[351,994,995],{"class":389},", engine))\n",[10,997,998,1001,1002,1005,1006,1009,1010,1013,1014,1017,1018,1021,1022,1025,1026,1029],{},[14,999,1000],{},"excluded"," refers to the incoming row. The ",[14,1003,1004],{},"WHERE"," on the update compares old and new values with ",[14,1007,1008],{},"IS NOT",", which treats two ",[14,1011,1012],{},"NULL","s as equal, so A-1 — unchanged — is skipped and keeps its original ",[14,1015,1016],{},"updated_at",". B-2 is updated and D-4 inserted: two rows changed. C-3, which is not in the sheet, is untouched. The odd-looking ",[14,1019,1020],{},"WHERE true"," after the ",[14,1023,1024],{},"SELECT"," is required by SQLite to tell its parser that ",[14,1027,1028],{},"ON CONFLICT"," belongs to the insert, not to a join; PostgreSQL accepts it too.",[33,1031,42,1037,42,1040,42,1043,42,1047,42,1056,42,1067,42,1073,42,1077,42,1082,42,1085,42,1090,42,1093,42,1097,42,1104,42,1110,42,1114,42,1118,42,1122,42,1125,42,1130,42,1137,42,1142,42,1145,42,1148,42,1152,42,1155,42,1158,42,1164,42,1168,42,1171,42,1174,42,1178,42,1181,42,1184,42,1190,42,1194],{"viewBox":1032,"role":36,"ariaLabelledBy":1033,"xmlns":40,"style":1036},"4 0 752 160",[1034,1035],"up-steps-t","up-steps-d","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[44,1038,1039],{"id":1034},"Upsert through a staging table",[48,1041,1042],{"id":1035},"Validate the sheet's keys, write the rows to a staging table, run one INSERT ON CONFLICT DO UPDATE statement that skips unchanged rows, handle keys missing from the sheet, and drop the staging table, all inside one transaction.",[52,1044],{"x":171,"y":55,"width":1045,"height":1046,"fill":58},"752","160",[52,1048],{"x":1049,"y":1050,"width":1051,"height":1052,"rx":1053,"fill":1054,"stroke":1055,"style":316},"20.0","16.0","115.2","128.0","12","#fdefd8","var(--gold,#b4740a)",[1057,1058,1060],"g",{"transform":1059},"translate(65.6,29.0) scale(1.000)",[1061,1062],"path",{"d":1063,"fill":1064,"stroke":1065,"style":1066},"M5 12.5l4.5 4.5L19 7","none","var(--gold-ink,#7a4e06)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[73,1068,1072],{"x":1069,"y":1070,"style":1071},"77.6","75.5","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Validate",[73,1074,1076],{"x":1069,"y":1075,"style":1071},"93.0","keys",[73,1078,1081],{"x":1069,"y":1079,"style":1080},"112.5","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","no blanks, no",[73,1083,752],{"x":1069,"y":1084,"style":1080},"130.0",[310,1086],{"x1":1087,"y1":1088,"x2":1089,"y2":1088,"stroke":315,"style":316},"140.2","80.0","162.2",[318,1091],{"points":1092,"fill":321},"166.2,80.0 157.2,85.0 157.2,75.0",[52,1094],{"x":1095,"y":1050,"width":1051,"height":1052,"rx":1053,"fill":1096,"stroke":315,"style":316},"171.2","#f0f4ff",[1057,1098,1100],{"transform":1099},"translate(216.8,37.8) scale(1.000)",[1061,1101],{"d":1102,"fill":1064,"stroke":1103,"style":1066},"M3 5h18v14H3z M3 10h18 M3 15h18 M9 5v14","var(--brand-strong,#4338ca)",[73,1105,1109],{"x":1106,"y":1107,"style":1108},"228.8","84.2","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Stage the",[73,1111,1113],{"x":1106,"y":1112,"style":1108},"101.8","rows",[73,1115,1117],{"x":1106,"y":1116,"style":1080},"121.2","to_sql replace",[310,1119],{"x1":1120,"y1":1088,"x2":1121,"y2":1088,"stroke":315,"style":316},"291.4","313.4",[318,1123],{"points":1124,"fill":321},"317.4,80.0 308.4,85.0 308.4,75.0",[52,1126],{"x":1127,"y":1050,"width":1051,"height":1052,"rx":1053,"fill":1128,"stroke":1129,"style":316},"322.4","#d9f4f1","var(--teal,#0f9488)",[1057,1131,1133],{"transform":1132},"translate(368.0,37.8) scale(1.000)",[1061,1134],{"d":1135,"fill":1064,"stroke":1136,"style":1066},"M5 6c0-1.7 3.1-3 7-3s7 1.3 7 3-3.1 3-7 3-7-1.3-7-3z M5 6v12c0 1.7 3.1 3 7 3s7-1.3 7-3V6 M5 12c0 1.7 3.1 3 7 3s7-1.3 7-3","var(--teal-ink,#0b6157)",[73,1138,1141],{"x":1139,"y":1107,"style":1140},"380.0","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Upsert",[73,1143,1028],{"x":1139,"y":1144,"style":1080},"103.8",[73,1146,1147],{"x":1139,"y":1116,"style":1080},"DO UPDATE",[310,1149],{"x1":1150,"y1":1088,"x2":1151,"y2":1088,"stroke":315,"style":316},"442.6","464.6",[318,1153],{"points":1154,"fill":321},"468.6,80.0 459.6,85.0 459.6,75.0",[52,1156],{"x":1157,"y":1050,"width":1051,"height":1052,"rx":1053,"fill":1096,"stroke":315,"style":316},"473.6",[1057,1159,1161],{"transform":1160},"translate(519.2,37.8) scale(1.000)",[1061,1162],{"d":1163,"fill":1064,"stroke":1103,"style":1066},"M4 7h16 M10 11v6 M14 11v6 M6 7l1 13h10l1-13 M9 7V4h6v3",[73,1165,1167],{"x":1166,"y":1107,"style":1108},"531.2","Handle",[73,1169,1170],{"x":1166,"y":1112,"style":1108},"removals",[73,1172,1173],{"x":1166,"y":1116,"style":1080},"optional",[310,1175],{"x1":1176,"y1":1088,"x2":1177,"y2":1088,"stroke":315,"style":316},"593.8","615.8",[318,1179],{"points":1180,"fill":321},"619.8,80.0 610.8,85.0 610.8,75.0",[52,1182],{"x":1183,"y":1050,"width":1051,"height":1052,"rx":1053,"fill":1096,"stroke":315,"style":316},"624.8",[1057,1185,1187],{"transform":1186},"translate(670.4,46.5) scale(1.000)",[1061,1188],{"d":1189,"fill":1064,"stroke":1103,"style":1066},"M6 11h12v9H6z M8.5 11V8a3.5 3.5 0 0 1 7 0v3",[73,1191,1193],{"x":1192,"y":1075,"style":1108},"682.4","Commit",[73,1195,1196],{"x":1192,"y":1079,"style":1080},"one transaction",[337,1198,1200],{"id":1199},"step-3-the-same-sql-on-other-databases","Step 3 — The same SQL on other databases",[10,1202,1203,1204,1207,1208,1210,1211,1214,1215,1218,1219,1222,1223,1226],{},"The staging pattern works everywhere; only the merge statement changes. PostgreSQL uses the same ",[14,1205,1206],{},"ON CONFLICT … DO UPDATE"," syntax as SQLite, with one difference in the change check: SQLite's null-safe ",[14,1209,1008],{}," is written ",[14,1212,1213],{},"IS DISTINCT FROM"," in PostgreSQL, as in ",[14,1216,1217],{},"WHERE products.price IS DISTINCT FROM excluded.price",". MySQL and MariaDB use ",[14,1220,1221],{},"INSERT … ON DUPLICATE KEY UPDATE",", and SQL Server uses ",[14,1224,1225],{},"MERGE",":",[342,1228,1232],{"className":1229,"code":1230,"language":1231,"meta":347,"style":347},"language-sql shiki shiki-themes github-light github-dark-high-contrast","-- MySQL \u002F MariaDB\nINSERT INTO products (sku, name, price)\nSELECT sku, name, price FROM stage_products\nON DUPLICATE KEY UPDATE name = VALUES(name), price = VALUES(price);\n\n-- SQL Server\nMERGE products AS t\nUSING stage_products AS s ON t.sku = s.sku\nWHEN MATCHED AND (t.name \u003C> s.name OR t.price \u003C> s.price)\n    THEN UPDATE SET name = s.name, price = s.price\nWHEN NOT MATCHED THEN INSERT (sku, name, price) VALUES (s.sku, s.name, s.price);\n","sql",[14,1233,1234,1239,1252,1270,1306,1310,1315,1328,1360,1409,1440],{"__ignoreMap":347},[351,1235,1236],{"class":310,"line":353},[351,1237,1238],{"class":822},"-- MySQL \u002F MariaDB\n",[351,1240,1241,1244,1247,1249],{"class":310,"line":399},[351,1242,1243],{"class":385},"INSERT INTO",[351,1245,1246],{"class":389}," products (sku, ",[351,1248,114],{"class":385},[351,1250,1251],{"class":389},", price)\n",[351,1253,1254,1256,1259,1261,1264,1267],{"class":310,"line":412},[351,1255,1024],{"class":385},[351,1257,1258],{"class":389}," sku, ",[351,1260,114],{"class":385},[351,1262,1263],{"class":389},", price ",[351,1265,1266],{"class":385},"FROM",[351,1268,1269],{"class":389}," stage_products\n",[351,1271,1272,1275,1278,1281,1284,1287,1289,1292,1294,1296,1299,1301,1303],{"class":310,"line":419},[351,1273,1274],{"class":385},"ON",[351,1276,1277],{"class":389}," DUPLICATE ",[351,1279,1280],{"class":385},"KEY",[351,1282,1283],{"class":385}," UPDATE",[351,1285,1286],{"class":385}," name",[351,1288,850],{"class":385},[351,1290,1291],{"class":385}," VALUES",[351,1293,776],{"class":389},[351,1295,114],{"class":385},[351,1297,1298],{"class":389},"), price ",[351,1300,425],{"class":385},[351,1302,1291],{"class":385},[351,1304,1305],{"class":389},"(price);\n",[351,1307,1308],{"class":310,"line":437},[351,1309,416],{"emptyLinePlaceholder":415},[351,1311,1312],{"class":310,"line":451},[351,1313,1314],{"class":822},"-- SQL Server\n",[351,1316,1317,1319,1322,1325],{"class":310,"line":462},[351,1318,1225],{"class":385},[351,1320,1321],{"class":389}," products ",[351,1323,1324],{"class":385},"AS",[351,1326,1327],{"class":389}," t\n",[351,1329,1330,1333,1336,1338,1341,1343,1346,1348,1350,1352,1355,1357],{"class":310,"line":470},[351,1331,1332],{"class":385},"USING",[351,1334,1335],{"class":389}," stage_products ",[351,1337,1324],{"class":385},[351,1339,1340],{"class":389}," s ",[351,1342,1274],{"class":385},[351,1344,1345],{"class":563}," t",[351,1347,635],{"class":389},[351,1349,109],{"class":563},[351,1351,850],{"class":385},[351,1353,1354],{"class":563}," s",[351,1356,635],{"class":389},[351,1358,1359],{"class":563},"sku\n",[351,1361,1362,1365,1368,1371,1374,1377,1379,1381,1384,1386,1388,1390,1393,1395,1397,1399,1401,1403,1405,1407],{"class":310,"line":478},[351,1363,1364],{"class":385},"WHEN",[351,1366,1367],{"class":385}," MATCHED",[351,1369,1370],{"class":385}," AND",[351,1372,1373],{"class":389}," (",[351,1375,1376],{"class":563},"t",[351,1378,635],{"class":389},[351,1380,114],{"class":563},[351,1382,1383],{"class":385}," \u003C>",[351,1385,1354],{"class":563},[351,1387,635],{"class":389},[351,1389,114],{"class":563},[351,1391,1392],{"class":385}," OR",[351,1394,1345],{"class":563},[351,1396,635],{"class":389},[351,1398,119],{"class":563},[351,1400,1383],{"class":385},[351,1402,1354],{"class":563},[351,1404,635],{"class":389},[351,1406,119],{"class":563},[351,1408,434],{"class":389},[351,1410,1411,1414,1416,1419,1421,1423,1425,1427,1429,1431,1433,1435,1437],{"class":310,"line":486},[351,1412,1413],{"class":385},"    THEN",[351,1415,1283],{"class":385},[351,1417,1418],{"class":385}," SET",[351,1420,1286],{"class":385},[351,1422,850],{"class":385},[351,1424,1354],{"class":563},[351,1426,635],{"class":389},[351,1428,114],{"class":563},[351,1430,1263],{"class":389},[351,1432,425],{"class":385},[351,1434,1354],{"class":563},[351,1436,635],{"class":389},[351,1438,1439],{"class":563},"price\n",[351,1441,1442,1444,1447,1449,1452,1455,1458,1460,1463,1466,1468,1471,1473,1475,1477,1479,1481,1483,1485,1487,1489,1491],{"class":310,"line":492},[351,1443,1364],{"class":385},[351,1445,1446],{"class":385}," NOT",[351,1448,1367],{"class":385},[351,1450,1451],{"class":385}," THEN",[351,1453,1454],{"class":385}," INSERT",[351,1456,1457],{"class":389}," (sku, ",[351,1459,114],{"class":385},[351,1461,1462],{"class":389},", price) ",[351,1464,1465],{"class":385},"VALUES",[351,1467,1373],{"class":389},[351,1469,1470],{"class":563},"s",[351,1472,635],{"class":389},[351,1474,109],{"class":563},[351,1476,520],{"class":389},[351,1478,1470],{"class":563},[351,1480,635],{"class":389},[351,1482,114],{"class":563},[351,1484,520],{"class":389},[351,1486,1470],{"class":563},[351,1488,635],{"class":389},[351,1490,119],{"class":563},[351,1492,1493],{"class":389},");\n",[10,1495,1496,1497,1499,1500,1503],{},"On server databases, create the staging table as a temporary table so concurrent jobs cannot collide, and give it the same column types as the target: letting ",[14,1498,16],{}," guess types can turn a ",[14,1501,1502],{},"NUMERIC(10,2)"," price into a float.",[337,1505,1507],{"id":1506},"step-4-a-to_sql-method-for-small-loads","Step 4 — A to_sql method for small loads",[10,1509,1510,1511,1513,1514,1517],{},"For a few hundred rows, you can skip the staging table and pass an upsert function to ",[14,1512,16],{},"'s ",[14,1515,1516],{},"method"," argument. pandas calls it with batches of rows; the function builds a dialect-specific insert:",[342,1519,1521],{"className":376,"code":1520,"language":378,"meta":347,"style":347},"from sqlalchemy.dialects.sqlite import insert\n\ndef upsert_rows(table, conn, keys, data_iter):\n    rows = [dict(zip(keys, row)) for row in data_iter]\n    stmt = insert(table.table).values(rows)\n    stmt = stmt.on_conflict_do_update(\n        index_elements=[\"sku\"],\n        set_={col: stmt.excluded[col] for col in keys if col != \"sku\"},\n    )\n    return conn.execute(stmt).rowcount\n\nupdate = pd.DataFrame({\"sku\": [\"C-3\"], \"name\": [\"Cocoa\"], \"price\": [3.2]})\nwith engine.begin() as con:\n    update.to_sql(\"products\", con, if_exists=\"append\", index=False, method=upsert_rows)\nprint(pd.read_sql(\"SELECT price FROM products WHERE sku = 'C-3'\", engine)[\"price\"].item())   # 3.2\n",[14,1522,1523,1536,1540,1552,1585,1595,1604,1618,1651,1656,1664,1668,1707,1717,1751],{"__ignoreMap":347},[351,1524,1525,1528,1531,1533],{"class":310,"line":353},[351,1526,1527],{"class":385},"from",[351,1529,1530],{"class":389}," sqlalchemy.dialects.sqlite ",[351,1532,386],{"class":385},[351,1534,1535],{"class":389}," insert\n",[351,1537,1538],{"class":310,"line":399},[351,1539,416],{"emptyLinePlaceholder":415},[351,1541,1542,1545,1549],{"class":310,"line":412},[351,1543,1544],{"class":385},"def",[351,1546,1548],{"class":1547},"s_Opv"," upsert_rows",[351,1550,1551],{"class":389},"(table, conn, keys, data_iter):\n",[351,1553,1554,1557,1559,1562,1565,1567,1570,1573,1576,1579,1582],{"class":310,"line":419},[351,1555,1556],{"class":389},"    rows ",[351,1558,425],{"class":385},[351,1560,1561],{"class":389}," [",[351,1563,1564],{"class":563},"dict",[351,1566,776],{"class":389},[351,1568,1569],{"class":563},"zip",[351,1571,1572],{"class":389},"(keys, row)) ",[351,1574,1575],{"class":385},"for",[351,1577,1578],{"class":389}," row ",[351,1580,1581],{"class":385},"in",[351,1583,1584],{"class":389}," data_iter]\n",[351,1586,1587,1590,1592],{"class":310,"line":437},[351,1588,1589],{"class":389},"    stmt ",[351,1591,425],{"class":385},[351,1593,1594],{"class":389}," insert(table.table).values(rows)\n",[351,1596,1597,1599,1601],{"class":310,"line":451},[351,1598,1589],{"class":389},[351,1600,425],{"class":385},[351,1602,1603],{"class":389}," stmt.on_conflict_do_update(\n",[351,1605,1606,1609,1611,1614,1616],{"class":310,"line":462},[351,1607,1608],{"class":585},"        index_elements",[351,1610,425],{"class":385},[351,1612,1613],{"class":389},"[",[351,1615,511],{"class":360},[351,1617,552],{"class":389},[351,1619,1620,1623,1625,1628,1630,1633,1635,1638,1640,1642,1645,1648],{"class":310,"line":470},[351,1621,1622],{"class":585},"        set_",[351,1624,425],{"class":385},[351,1626,1627],{"class":389},"{col: stmt.excluded[col] ",[351,1629,1575],{"class":385},[351,1631,1632],{"class":389}," col ",[351,1634,1581],{"class":385},[351,1636,1637],{"class":389}," keys ",[351,1639,662],{"class":385},[351,1641,1632],{"class":389},[351,1643,1644],{"class":385},"!=",[351,1646,1647],{"class":360}," \"sku\"",[351,1649,1650],{"class":389},"},\n",[351,1652,1653],{"class":310,"line":478},[351,1654,1655],{"class":389},"    )\n",[351,1657,1658,1661],{"class":310,"line":486},[351,1659,1660],{"class":385},"    return",[351,1662,1663],{"class":389}," conn.execute(stmt).rowcount\n",[351,1665,1666],{"class":310,"line":492},[351,1667,416],{"emptyLinePlaceholder":415},[351,1669,1670,1673,1675,1678,1680,1682,1685,1687,1689,1691,1694,1696,1699,1701,1704],{"class":310,"line":500},[351,1671,1672],{"class":389},"update ",[351,1674,425],{"class":385},[351,1676,1677],{"class":389}," pd.DataFrame({",[351,1679,511],{"class":360},[351,1681,514],{"class":389},[351,1683,1684],{"class":360},"\"C-3\"",[351,1686,531],{"class":389},[351,1688,534],{"class":360},[351,1690,514],{"class":389},[351,1692,1693],{"class":360},"\"Cocoa\"",[351,1695,531],{"class":389},[351,1697,1698],{"class":360},"\"price\"",[351,1700,514],{"class":389},[351,1702,1703],{"class":563},"3.2",[351,1705,1706],{"class":389},"]})\n",[351,1708,1709,1711,1713,1715],{"class":310,"line":505},[351,1710,440],{"class":385},[351,1712,443],{"class":389},[351,1714,393],{"class":385},[351,1716,448],{"class":389},[351,1718,1719,1722,1725,1727,1729,1731,1734,1736,1738,1740,1742,1744,1746,1748],{"class":310,"line":555},[351,1720,1721],{"class":389},"    update.to_sql(",[351,1723,1724],{"class":360},"\"products\"",[351,1726,918],{"class":389},[351,1728,921],{"class":585},[351,1730,425],{"class":385},[351,1732,1733],{"class":360},"\"append\"",[351,1735,520],{"class":389},[351,1737,586],{"class":585},[351,1739,425],{"class":385},[351,1741,591],{"class":563},[351,1743,520],{"class":389},[351,1745,1516],{"class":585},[351,1747,425],{"class":385},[351,1749,1750],{"class":389},"upsert_rows)\n",[351,1752,1753,1755,1757,1760,1763,1765,1768],{"class":310,"line":596},[351,1754,805],{"class":563},[351,1756,989],{"class":389},[351,1758,1759],{"class":360},"\"SELECT price FROM products WHERE sku = 'C-3'\"",[351,1761,1762],{"class":389},", engine)[",[351,1764,1698],{"class":360},[351,1766,1767],{"class":389},"].item())   ",[351,1769,1770],{"class":822},"# 3.2\n",[10,1772,1773,1774,1777,1778,1781,1782,1784],{},"For PostgreSQL, import ",[14,1775,1776],{},"insert"," from ",[14,1779,1780],{},"sqlalchemy.dialects.postgresql"," instead; the rest is identical. This version updates every incoming row, changed or not, so prefer the staging statement when you care about accurate ",[14,1783,1016],{}," values or large volumes.",[337,1786,1788],{"id":1787},"step-5-decide-what-happens-to-rows-removed-from-the-sheet","Step 5 — Decide what happens to rows removed from the sheet",[10,1790,1791],{},"An upsert never deletes. If someone removes C-3 from the sheet, it stays in the table. Whether that is right is a policy decision; either way, find those keys explicitly:",[342,1793,1795],{"className":376,"code":1794,"language":378,"meta":347,"style":347},"in_table = pd.read_sql(\"SELECT sku FROM products\", engine)[\"sku\"]\nremoved = sorted(set(in_table) - set(sheet[\"sku\"]))\nprint(removed)                    # ['C-3']\n",[14,1796,1797,1817,1849],{"__ignoreMap":347},[351,1798,1799,1802,1804,1807,1810,1812,1814],{"class":310,"line":353},[351,1800,1801],{"class":389},"in_table ",[351,1803,425],{"class":385},[351,1805,1806],{"class":389}," pd.read_sql(",[351,1808,1809],{"class":360},"\"SELECT sku FROM products\"",[351,1811,1762],{"class":389},[351,1813,511],{"class":360},[351,1815,1816],{"class":389},"]\n",[351,1818,1819,1822,1824,1827,1829,1832,1835,1838,1841,1844,1846],{"class":310,"line":399},[351,1820,1821],{"class":389},"removed ",[351,1823,425],{"class":385},[351,1825,1826],{"class":563}," sorted",[351,1828,776],{"class":389},[351,1830,1831],{"class":563},"set",[351,1833,1834],{"class":389},"(in_table) ",[351,1836,1837],{"class":385},"-",[351,1839,1840],{"class":563}," set",[351,1842,1843],{"class":389},"(sheet[",[351,1845,511],{"class":360},[351,1847,1848],{"class":389},"]))\n",[351,1850,1851,1853,1856],{"class":310,"line":412},[351,1852,805],{"class":563},[351,1854,1855],{"class":389},"(removed)                    ",[351,1857,1858],{"class":822},"# ['C-3']\n",[10,1860,1861,1862,1865,1866,1869],{},"Deleting them makes the table an exact mirror of the sheet, which is dangerous if other systems reference those rows or add their own. Marking them inactive — an ",[14,1863,1864],{},"active"," flag or a ",[14,1867,1868],{},"retired_at"," date — keeps history and references intact. Most teams choose the flag and report the removed keys so the sheet's owner can confirm the change.",[33,1871,42,1876,42,1879,42,1882,42,1885,42,1890,42,1896,42,1902,42,1907,42,1912,42,1918,42,1922,42,1929,42,1933,42,1938,42,1942,42,1947,42,1950,42,1956,42,1961,42,1965,42,1969,42,1974,42,1977,42,1982,42,1985,42,1990,42,1993],{"viewBox":1872,"role":36,"ariaLabelledBy":1873,"xmlns":40,"style":1036},"4 0 752 233",[1874,1875],"up-removed-t","up-removed-d",[44,1877,1878],{"id":1874},"Handling keys that disappeared from the sheet",[48,1880,1881],{"id":1875},"Deleting removed keys keeps the table identical to the sheet but loses history and can break references from other tables. Marking them inactive keeps history and references and lets the removal be reviewed, at the cost of an extra column and filters in queries.",[52,1883],{"x":171,"y":55,"width":1045,"height":1884,"fill":58},"233",[52,1886],{"x":1049,"y":1050,"width":1887,"height":1888,"rx":1889,"fill":1054,"stroke":1055,"style":316},"351.0","201.0","14",[1057,1891,1893],{"transform":1892},"translate(182.5,29.0) scale(1.083)",[1061,1894],{"d":1163,"fill":1064,"stroke":1065,"style":1895},"stroke-width:1.66px;stroke-linecap:round;stroke-linejoin:round",[73,1897,1901],{"x":1898,"y":1899,"style":1900},"195.5","78.0","font-size:14px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Delete the rows",[310,1903],{"x1":1904,"y1":1905,"x2":1906,"y2":1905,"stroke":1055,"style":67},"36.0","91.0","355.0",[73,1908,1911],{"x":1909,"y":125,"style":1910},"60.0","font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:start","table mirrors the sheet",[1057,1913,1915],{"transform":1914},"translate(38.5,99.5) scale(0.625)",[1061,1916],{"d":1063,"fill":1064,"stroke":1129,"style":1917},"stroke-width:2.88px;stroke-linecap:round;stroke-linejoin:round",[73,1919,1921],{"x":1909,"y":1920,"style":1910},"137.0","history is lost",[1057,1923,1925],{"transform":1924},"translate(38.5,125.5) scale(0.625)",[1061,1926],{"d":1927,"fill":1064,"stroke":1928,"style":1917},"M6 6l12 12 M18 6L6 18","var(--accent,#f43f8f)",[73,1930,1932],{"x":1909,"y":1931,"style":1910},"163.0","foreign keys may break",[1057,1934,1936],{"transform":1935},"translate(38.5,151.5) scale(0.625)",[1061,1937],{"d":1927,"fill":1064,"stroke":1928,"style":1917},[73,1939,1941],{"x":1909,"y":1940,"style":1910},"189.0","a mistaken edit deletes data",[1057,1943,1945],{"transform":1944},"translate(38.5,177.5) scale(0.625)",[1061,1946],{"d":1927,"fill":1064,"stroke":1928,"style":1917},[52,1948],{"x":1949,"y":1050,"width":1887,"height":1888,"rx":1889,"fill":1128,"stroke":1129,"style":316},"389.0",[1057,1951,1953],{"transform":1952},"translate(551.5,29.0) scale(1.083)",[1061,1954],{"d":1955,"fill":1064,"stroke":1136,"style":1895},"M2 12s4-7 10-7 10 7 10 7-4 7-10 7S2 12 2 12z M12 9a3 3 0 1 0 .01 0z",[73,1957,1960],{"x":1958,"y":1899,"style":1959},"564.5","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Mark inactive",[310,1962],{"x1":1963,"y1":1905,"x2":1964,"y2":1905,"stroke":1129,"style":67},"405.0","724.0",[73,1966,1968],{"x":1967,"y":125,"style":1910},"429.0","history kept",[1057,1970,1972],{"transform":1971},"translate(407.5,99.5) scale(0.625)",[1061,1973],{"d":1063,"fill":1064,"stroke":1129,"style":1917},[73,1975,1976],{"x":1967,"y":1920,"style":1910},"references stay valid",[1057,1978,1980],{"transform":1979},"translate(407.5,125.5) scale(0.625)",[1061,1981],{"d":1063,"fill":1064,"stroke":1129,"style":1917},[73,1983,1984],{"x":1967,"y":1931,"style":1910},"removals can be reviewed",[1057,1986,1988],{"transform":1987},"translate(407.5,151.5) scale(0.625)",[1061,1989],{"d":1063,"fill":1064,"stroke":1129,"style":1917},[73,1991,1992],{"x":1967,"y":1940,"style":1910},"queries filter on active",[1057,1994,1996],{"transform":1995},"translate(407.5,177.5) scale(0.625)",[1061,1997],{"d":1998,"fill":1064,"stroke":1055,"style":1917},"M12 3.5l9.5 17h-19z M12 10v5 M12 17.8v.4",[337,2000,2002],{"id":2001},"common-pitfalls","Common pitfalls",[2004,2005,2006,2018,2031,2037,2046],"ul",{},[2007,2008,2009,2013,2014,2017],"li",{},[2010,2011,2012],"strong",{},"Counting rows as success."," ",[14,2015,2016],{},"rowcount"," reports inserted plus updated rows. Log it alongside the sheet's row count and the number of removed keys, so a run that changed nothing — or everything — stands out in the log.",[2007,2019,2020,2023,2024,2026,2027,2030],{},[2010,2021,2022],{},"Blank cells overwriting data."," An empty price in the sheet becomes ",[14,2025,1012],{}," and replaces a real price in the table. Decide whether blanks mean \"unknown\" or \"keep the current value\"; for the latter, use ",[14,2028,2029],{},"COALESCE(excluded.price, products.price)"," in the update.",[2007,2032,2033,2036],{},[2010,2034,2035],{},"Using if_exists=\"replace\" on the real table."," It drops the table and recreates it with guessed types, losing keys, indexes and permissions.",[2007,2038,2039,2013,2042,2045],{},[2010,2040,2041],{},"Keys that change format.",[14,2043,2044],{},"\"A-1 \""," with a trailing space is a new key to the database. Strip and normalise keys before staging.",[2007,2047,2048,2051,2052,2055,2056,2059,2060,635],{},[2010,2049,2050],{},"Numbers read as text or floats."," IDs such as ",[14,2053,2054],{},"00123"," lose their zeros unless read with ",[14,2057,2058],{},"dtype=str","; see ",[27,2061,2063],{"href":2062},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fmerge-excel-files-with-different-column-names\u002F","Merge Excel Files with Different Column Names in pandas",[337,2065,2067],{"id":2066},"conclusion","Conclusion",[10,2069,2070,2071,2073,2074,2076,2077,2079,2080,2083,2084,2086],{},"Loading a maintained Excel sheet into a database repeatedly needs an upsert, which pandas' ",[14,2072,16],{}," does not provide. Validate the keys, write the sheet to a staging table, and merge it with one ",[14,2075,836],{}," that skips unchanged rows — or ",[14,2078,1225],{}," and ",[14,2081,2082],{},"ON DUPLICATE KEY UPDATE"," on other databases. Use a ",[14,2085,16],{}," method callable for small loads, and decide explicitly whether keys removed from the sheet are deleted or marked inactive.",[337,2088,2090],{"id":2089},"frequently-asked-questions","Frequently asked questions",[10,2092,2093,2096],{},[2010,2094,2095],{},"Does pandas to_sql support upserts?","\nNot directly. if_exists=\"append\" inserts every row and fails on primary key conflicts, and \"replace\" drops and recreates the table. Use a custom method callable or load into a staging table and run an INSERT … ON CONFLICT statement.",[10,2098,2099,2102],{},[2010,2100,2101],{},"What is a staging table?","\nA temporary table that receives the Excel rows unchanged. One SQL statement then merges it into the real table, which is fast, runs inside a transaction and keeps the upsert logic in SQL where the database can enforce it.",[10,2104,2105,2108],{},[2010,2106,2107],{},"How do I update only rows that actually changed?","\nAdd a WHERE clause to the DO UPDATE part that compares the existing and incoming values — IS NOT in SQLite, IS DISTINCT FROM in PostgreSQL — so unchanged rows are skipped and their updated_at timestamps stay put.",[10,2110,2111,2114],{},[2010,2112,2113],{},"What about rows deleted from the Excel sheet?","\nAn upsert never deletes. Find keys that are in the table but not in the sheet with a left join from the table to the staging table, then delete them or mark them inactive, depending on your policy.",[337,2116,2118],{"id":2117},"related","Related",[2004,2120,2121,2127,2134,2141],{},[2007,2122,2123,2124,2126],{},"Up one level: ",[27,2125,30],{"href":29}," — Excel and SQL in both directions.",[2007,2128,2129,2133],{},[27,2130,2132],{"href":2131},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-sql-query-results-to-excel-with-python\u002F","Export SQL Query Results to Excel with Python"," — the reverse direction.",[2007,2135,2136,2140],{},[27,2137,2139],{"href":2138},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Frefresh-an-excel-report-from-a-database-on-a-schedule\u002F","Refresh an Excel Report from a Database on a Schedule"," — automating the round trip.",[2007,2142,2143,2147],{},[27,2144,2146],{"href":2145},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvalidate-a-merge-to-avoid-duplicate-rows\u002F","Validate a pandas Merge to Avoid Duplicate Rows"," — the same key checks for joins.",[2149,2150,2151],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}",{"title":347,"searchDepth":399,"depth":399,"links":2153},[2154,2155,2156,2157,2158,2159,2160,2161,2162,2163],{"id":339,"depth":399,"text":340},{"id":638,"depth":399,"text":639},{"id":829,"depth":399,"text":830},{"id":1199,"depth":399,"text":1200},{"id":1506,"depth":399,"text":1507},{"id":1787,"depth":399,"text":1788},{"id":2001,"depth":399,"text":2002},{"id":2066,"depth":399,"text":2067},{"id":2089,"depth":399,"text":2090},{"id":2117,"depth":399,"text":2118},"2026-09-23","Load a maintained Excel sheet into a database table without duplicating rows — insert new keys, update changed ones and leave unchanged rows alone. Use a staging table and INSERT … ON CONFLICT, a to_sql method for smaller loads, and checks for duplicate keys and rows deleted from the sheet.","md",[2168,2170,2172,2174],{"q":2095,"a":2169},"Not directly. if_exists=\"append\" inserts every row and fails on primary key conflicts, and \"replace\" drops and recreates the table. Use a custom method callable or load into a staging table and run an INSERT … ON CONFLICT statement.",{"q":2101,"a":2171},"A temporary table that receives the Excel rows unchanged. One SQL statement then merges it into the real table, which is fast, runs inside a transaction and keeps the upsert logic in SQL where the database can enforce it.",{"q":2107,"a":2173},"Add a WHERE clause to the DO UPDATE part that compares the existing and incoming values — IS NOT in SQLite, IS DISTINCT FROM in PostgreSQL — so unchanged rows are skipped and their updated_at timestamps stay put.",{"q":2113,"a":2175},"An upsert never deletes. Find keys that are in the table but not in the sheet with a left join from the table to the staging table, then delete them or mark them inactive, depending on your policy.","upsert excel rows into a database table with python",null,{"breadcrumb":2179},[2180,2183,2186],{"name":2181,"item":2182},"Home","\u002F",{"name":2184,"item":2185},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":30,"item":29},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fupsert-excel-rows-into-a-database-table",{"title":2189,"description":2190},"Upsert Excel Rows into a Database Table with Python (ON CONFLICT)","Upsert Excel data into SQLite or PostgreSQL with pandas and SQLAlchemy — why to_sql append duplicates rows, staging tables with INSERT ON CONFLICT DO UPDATE, updating only changed rows, a to_sql method callable, duplicate keys and deletions.","upsert-excel-rows-into-a-database-table","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fupsert-excel-rows-into-a-database-table\u002Findex",[2194,2195],"data-sources","pandas","how-to","AntrpvNRuyLUZSNe1Npc8liNFHkQxGTFdzvtZo6aeLU",[2199,2203],{"title":2200,"path":2201,"stem":2202,"children":-1},"Sync Google Sheets with Excel Using Python","\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fsync-google-sheets-with-excel-using-python","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fsync-google-sheets-with-excel-using-python\u002Findex",{"title":2204,"path":2205,"stem":2206,"children":-1},"Reading Excel with Polars and Arrow","\u002Fadvanced-data-transformation-and-cleaning\u002Freading-excel-with-polars-and-arrow","advanced-data-transformation-and-cleaning\u002Freading-excel-with-polars-and-arrow\u002Findex",[2208,2210],{"path":2209,"title":2184},"\u002Fadvanced-data-transformation-and-cleaning",{"path":2211,"title":30},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases",[],[2214,2217,2220,2223,2226,2229],{"path":2215,"title":2216,"kind":2196},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fread-an-excel-file-directly-from-amazon-s3-with-pandas","Read an Excel File Directly from Amazon S3 with pandas",{"path":2218,"title":2219,"kind":2196},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fcache-api-responses-when-building-excel-reports","Cache API Responses When Building Excel Reports",{"path":2221,"title":2222,"kind":2196},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fcompare-an-excel-file-with-a-database-table","Compare an Excel File with a Database Table",{"path":2224,"title":2225,"kind":2196},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fdownload-new-excel-files-from-an-sftp-server","Download New Excel Files from an SFTP Server",{"path":2227,"title":2228,"kind":2196},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-a-large-database-table-to-excel-in-chunks","Export a Large Database Table to Excel in Chunks",{"path":2230,"title":2231,"kind":2196},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-database-lookup-tables-as-excel-dropdown-lists","Export Database Lookup Tables as Excel Dropdown Lists",1790972098085]