[{"data":1,"prerenderedAt":2290},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Ffetch-api-data-into-excel-with-python-requests":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Ffetch-api-data-into-excel-with-python-requests":2282},{"id":4,"title":5,"body":6,"dateModified":2253,"datePublished":2253,"description":2254,"extension":2255,"faq":2256,"meta":2265,"navigation":249,"path":2275,"seo":2276,"slug":2278,"stem":2279,"type":2280,"__hash__":2281},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Ffetch-api-data-into-excel-with-python-requests\u002Findex.md","Fetch API Data into Excel with Python requests",{"type":7,"value":8,"toc":2240},"minimark",[9,13,22,178,183,214,217,221,227,590,601,607,611,614,907,924,928,943,1278,1292,1296,1299,1483,1489,1493,1496,1773,1869,1884,1973,1977,2119,2123,2130,2133,2137,2143,2147,2160,2174,2183,2189,2193,2196,2205,2208,2236],[10,11,12],"p",{},"A surprising share of reporting data has no database behind it: exchange rates, a payment provider's settlements, a project tracker, a marketing platform's numbers. All of it arrives as JSON over HTTP, and all of it eventually has to become columns in a workbook that someone opens on a Monday.",[10,14,15,16,21],{},"The mechanics are not hard. What makes the difference between a script that works while you watch it and one that can run unattended is the boilerplate around the call: a timeout so a slow endpoint cannot hang the job, a retry policy that distinguishes a rate limit from a bad request, a pagination loop that terminates, and a cache so re-running does not re-download. This guide is part of ",[17,18,20],"a",{"href":19},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002F","Moving Data Between Excel and Databases",".",[23,24,32,33,32,37,32,41,32,48,32,55,32,65,32,71,32,76,32,80,32,85,32,90,32,93,32,96,32,100,32,104,32,107,32,110,32,116,32,122,32,126,32,130,32,136,32,141,32,145,32,148,32,152,32,155,32,160,32,164,32,170,32,174],"svg",{"viewBox":25,"role":26,"ariaLabelledBy":27,"xmlns":30,"style":31},"0 0 740 244","img",[28,29],"api-flow-t","api-flow-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:740px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[34,35,36],"title",{"id":28},"From paginated JSON to a workbook",[38,39,40],"desc",{"id":29},"A configured session with a timeout and a retry policy requests page after page until the API stops returning a next cursor. The accumulated records are flattened with json_normalize into a DataFrame, typed, and written to a formatted workbook.",[42,43],"rect",{"x":44,"y":44,"width":45,"height":46,"fill":47},"0","740","244","#ffffff",[49,50,54],"text",{"x":51,"y":52,"style":53},"370","30","font-size:13px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","The loop is the part that has to be right",[42,56],{"x":57,"y":58,"width":59,"height":60,"rx":61,"fill":62,"stroke":63,"style":64},"16","54","164","88","12","#ebebfd","var(--brand,#5b5cf0)","stroke-width:2px",[49,66,70],{"x":67,"y":68,"style":69},"98","82","font-size:12px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Session",[49,72,75],{"x":67,"y":73,"style":74},"106","font-size:11px;fill:var(--text,#172033);text-anchor:middle","auth header · timeout",[49,77,79],{"x":67,"y":78,"style":74},"126","retry on 429 and 5xx",[42,81],{"x":82,"y":58,"width":59,"height":60,"rx":61,"fill":83,"stroke":84,"style":64},"212","#fdefd8","var(--gold,#b4740a)",[49,86,89],{"x":87,"y":68,"style":88},"294","font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","page loop",[49,91,92],{"x":87,"y":73,"style":74},"follow the cursor",[49,94,95],{"x":87,"y":78,"style":74},"stop on an empty page",[42,97],{"x":98,"y":58,"width":59,"height":60,"rx":61,"fill":99,"stroke":63,"style":64},"408","#f0f4ff",[49,101,103],{"x":102,"y":68,"style":69},"490","json_normalize",[49,105,106],{"x":102,"y":73,"style":74},"records become rows",[49,108,109],{"x":102,"y":78,"style":74},"parent fields repeat",[42,111],{"x":112,"y":58,"width":113,"height":60,"rx":61,"fill":114,"stroke":115,"style":64},"604","120","#d9f4f1","var(--teal,#0f9488)",[49,117,121],{"x":118,"y":119,"style":120},"664","90","font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","workbook",[49,123,125],{"x":118,"y":124,"style":74},"114","typed and",[49,127,129],{"x":118,"y":128,"style":74},"132","formatted",[131,132],"line",{"x1":133,"y1":67,"x2":134,"y2":67,"stroke":135,"style":64},"184","206","var(--muted,#5b6780)",[137,138],"polygon",{"points":139,"fill":140},"210,98 200,93 200,103","#5b6780",[131,142],{"x1":143,"y1":67,"x2":144,"y2":67,"stroke":135,"style":64},"380","402",[137,146],{"points":147,"fill":140},"406,98 396,93 396,103",[131,149],{"x1":150,"y1":67,"x2":151,"y2":67,"stroke":135,"style":64},"576","598",[137,153],{"points":154,"fill":140},"602,98 592,93 592,103",[156,157],"path",{"d":158,"stroke":84,"style":64,"fill":159},"M 294 146 L 294 168 L 240 168","none",[137,161],{"points":162,"fill":163},"236,168 246,163 246,173","#8a5808",[49,165,169],{"x":166,"y":167,"style":168},"330","172","font-size:11px;font-weight:600;fill:var(--gold-ink,#7a4e06)","next cursor",[49,171,173],{"x":51,"y":82,"style":172},"font-size:11.5px;fill:var(--muted,#5b6780);text-anchor:middle","Three things end the loop: no cursor, an empty page, or a page cap — and the cap is what",[49,175,177],{"x":51,"y":176,"style":172},"230","stops a misbehaving API from running the job until the disk fills.",[179,180,182],"h2",{"id":181},"prerequisites","Prerequisites",[184,185,190],"pre",{"className":186,"code":187,"language":188,"meta":189,"style":189},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install requests pandas openpyxl\n","bash","",[191,192,193],"code",{"__ignoreMap":189},[194,195,197,201,205,208,211],"span",{"class":131,"line":196},1,[194,198,200],{"class":199},"sMTad","pip",[194,202,204],{"class":203},"srMev"," install",[194,206,207],{"class":203}," requests",[194,209,210],{"class":203}," pandas",[194,212,213],{"class":203}," openpyxl\n",[10,215,216],{},"You also need an endpoint and, usually, a token. Keep the token in an environment variable — a key pasted into the script is a key that ends up in version control, in a screenshot and in a ticket.",[179,218,220],{"id":219},"step-1-configure-the-session-once","Step 1: Configure the session once",[10,222,223,224,226],{},"A ",[191,225,70],{}," reuses the TCP connection across requests and holds the headers and retry policy in one place:",[184,228,232],{"className":229,"code":230,"language":231,"meta":189,"style":189},"language-python shiki shiki-themes github-light github-dark-high-contrast","import os\n\nimport requests\nfrom requests.adapters import HTTPAdapter\nfrom urllib3.util.retry import Retry\n\nTIMEOUT = (5, 30)          # (connect, read) seconds\n\n\ndef make_session(token=None):\n    session = requests.Session()\n    session.headers.update({\n        \"Accept\": \"application\u002Fjson\",\n        \"User-Agent\": \"monthly-report\u002F1.0\",\n        \"Authorization\": f\"Bearer {token or os.environ['API_TOKEN']}\",\n    })\n    retry = Retry(\n        total=4,\n        backoff_factor=1.5,                       # 0s, 1.5s, 3s, 6s\n        status_forcelist=(429, 500, 502, 503, 504),\n        allowed_methods=frozenset({\"GET\"}),\n        respect_retry_after_header=True,\n    )\n    session.mount(\"https:\u002F\u002F\", HTTPAdapter(max_retries=retry))\n    return session\n","python",[191,233,234,244,251,259,273,286,291,319,324,329,351,362,368,383,396,437,443,454,468,485,522,542,555,561,581],{"__ignoreMap":189},[194,235,236,240],{"class":131,"line":196},[194,237,239],{"class":238},"s-kum","import",[194,241,243],{"class":242},"skGVy"," os\n",[194,245,247],{"class":131,"line":246},2,[194,248,250],{"emptyLinePlaceholder":249},true,"\n",[194,252,254,256],{"class":131,"line":253},3,[194,255,239],{"class":238},[194,257,258],{"class":242}," requests\n",[194,260,262,265,268,270],{"class":131,"line":261},4,[194,263,264],{"class":238},"from",[194,266,267],{"class":242}," requests.adapters ",[194,269,239],{"class":238},[194,271,272],{"class":242}," HTTPAdapter\n",[194,274,276,278,281,283],{"class":131,"line":275},5,[194,277,264],{"class":238},[194,279,280],{"class":242}," urllib3.util.retry ",[194,282,239],{"class":238},[194,284,285],{"class":242}," Retry\n",[194,287,289],{"class":131,"line":288},6,[194,290,250],{"emptyLinePlaceholder":249},[194,292,294,298,301,304,307,310,312,315],{"class":131,"line":293},7,[194,295,297],{"class":296},"sP0c6","TIMEOUT",[194,299,300],{"class":238}," =",[194,302,303],{"class":242}," (",[194,305,306],{"class":296},"5",[194,308,309],{"class":242},", ",[194,311,52],{"class":296},[194,313,314],{"class":242},")          ",[194,316,318],{"class":317},"s-wDw","# (connect, read) seconds\n",[194,320,322],{"class":131,"line":321},8,[194,323,250],{"emptyLinePlaceholder":249},[194,325,327],{"class":131,"line":326},9,[194,328,250],{"emptyLinePlaceholder":249},[194,330,332,335,339,342,345,348],{"class":131,"line":331},10,[194,333,334],{"class":238},"def",[194,336,338],{"class":337},"s_Opv"," make_session",[194,340,341],{"class":242},"(token",[194,343,344],{"class":238},"=",[194,346,347],{"class":296},"None",[194,349,350],{"class":242},"):\n",[194,352,354,357,359],{"class":131,"line":353},11,[194,355,356],{"class":242},"    session ",[194,358,344],{"class":238},[194,360,361],{"class":242}," requests.Session()\n",[194,363,365],{"class":131,"line":364},12,[194,366,367],{"class":242},"    session.headers.update({\n",[194,369,371,374,377,380],{"class":131,"line":370},13,[194,372,373],{"class":203},"        \"Accept\"",[194,375,376],{"class":242},": ",[194,378,379],{"class":203},"\"application\u002Fjson\"",[194,381,382],{"class":242},",\n",[194,384,386,389,391,394],{"class":131,"line":385},14,[194,387,388],{"class":203},"        \"User-Agent\"",[194,390,376],{"class":242},[194,392,393],{"class":203},"\"monthly-report\u002F1.0\"",[194,395,382],{"class":242},[194,397,399,402,404,407,410,414,417,420,423,426,429,432,435],{"class":131,"line":398},15,[194,400,401],{"class":203},"        \"Authorization\"",[194,403,376],{"class":242},[194,405,406],{"class":238},"f",[194,408,409],{"class":203},"\"Bearer ",[194,411,413],{"class":412},"sSjpA","{",[194,415,416],{"class":242},"token ",[194,418,419],{"class":238},"or",[194,421,422],{"class":242}," os.environ[",[194,424,425],{"class":203},"'API_TOKEN'",[194,427,428],{"class":242},"]",[194,430,431],{"class":412},"}",[194,433,434],{"class":203},"\"",[194,436,382],{"class":242},[194,438,440],{"class":131,"line":439},16,[194,441,442],{"class":242},"    })\n",[194,444,446,449,451],{"class":131,"line":445},17,[194,447,448],{"class":242},"    retry ",[194,450,344],{"class":238},[194,452,453],{"class":242}," Retry(\n",[194,455,457,461,463,466],{"class":131,"line":456},18,[194,458,460],{"class":459},"sa561","        total",[194,462,344],{"class":238},[194,464,465],{"class":296},"4",[194,467,382],{"class":242},[194,469,471,474,476,479,482],{"class":131,"line":470},19,[194,472,473],{"class":459},"        backoff_factor",[194,475,344],{"class":238},[194,477,478],{"class":296},"1.5",[194,480,481],{"class":242},",                       ",[194,483,484],{"class":317},"# 0s, 1.5s, 3s, 6s\n",[194,486,488,491,493,496,499,501,504,506,509,511,514,516,519],{"class":131,"line":487},20,[194,489,490],{"class":459},"        status_forcelist",[194,492,344],{"class":238},[194,494,495],{"class":242},"(",[194,497,498],{"class":296},"429",[194,500,309],{"class":242},[194,502,503],{"class":296},"500",[194,505,309],{"class":242},[194,507,508],{"class":296},"502",[194,510,309],{"class":242},[194,512,513],{"class":296},"503",[194,515,309],{"class":242},[194,517,518],{"class":296},"504",[194,520,521],{"class":242},"),\n",[194,523,525,528,530,533,536,539],{"class":131,"line":524},21,[194,526,527],{"class":459},"        allowed_methods",[194,529,344],{"class":238},[194,531,532],{"class":296},"frozenset",[194,534,535],{"class":242},"({",[194,537,538],{"class":203},"\"GET\"",[194,540,541],{"class":242},"}),\n",[194,543,545,548,550,553],{"class":131,"line":544},22,[194,546,547],{"class":459},"        respect_retry_after_header",[194,549,344],{"class":238},[194,551,552],{"class":296},"True",[194,554,382],{"class":242},[194,556,558],{"class":131,"line":557},23,[194,559,560],{"class":242},"    )\n",[194,562,564,567,570,573,576,578],{"class":131,"line":563},24,[194,565,566],{"class":242},"    session.mount(",[194,568,569],{"class":203},"\"https:\u002F\u002F\"",[194,571,572],{"class":242},", HTTPAdapter(",[194,574,575],{"class":459},"max_retries",[194,577,344],{"class":238},[194,579,580],{"class":242},"retry))\n",[194,582,584,587],{"class":131,"line":583},25,[194,585,586],{"class":238},"    return",[194,588,589],{"class":242}," session\n",[10,591,592,593,596,597,21],{},"The ",[191,594,595],{},"status_forcelist"," encodes the only retry rule that matters: 429 means \"you are going too fast\" and 5xx means \"we are broken\", both of which may succeed unchanged in a few seconds. A 400 or a 401 will fail identically forever, so retrying them only delays the alert — the same distinction made in ",[17,598,600],{"href":599},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fretry-a-failed-excel-report-job-in-python\u002F","Retry a Failed Excel Report Job in Python",[10,602,603,606],{},[191,604,605],{},"respect_retry_after_header=True"," makes the client honour the API's own backoff instruction instead of guessing, which is what keeps a job on the right side of a rate limit.",[179,608,610],{"id":609},"step-2-page-until-the-api-says-stop","Step 2: Page until the API says stop",[10,612,613],{},"Nearly every paginated API uses one of three schemes — a cursor, a page number, or an offset. All three need the same two guards: stop on an empty page, and cap the number of pages so a broken cursor cannot loop forever.",[184,615,617],{"className":229,"code":616,"language":231,"meta":189,"style":189},"def fetch_all(session, url, params=None, max_pages=200):\n    \"\"\"Follow cursor pagination and return every record.\"\"\"\n    params = dict(params or {})\n    records, pages, cursor = [], 0, None\n\n    while pages \u003C max_pages:\n        if cursor:\n            params[\"cursor\"] = cursor\n        response = session.get(url, params=params, timeout=TIMEOUT)\n        response.raise_for_status()\n        payload = response.json()\n\n        batch = payload.get(\"data\", [])\n        if not batch:\n            break\n        records.extend(batch)\n        pages += 1\n\n        cursor = (payload.get(\"meta\") or {}).get(\"next_cursor\")\n        if not cursor:\n            break\n    else:\n        raise RuntimeError(f\"stopped after {max_pages} pages — is the cursor advancing?\")\n\n    return records\n",[191,618,619,643,648,666,683,687,701,709,725,753,758,768,772,788,798,803,808,819,823,849,857,861,869,896,900],{"__ignoreMap":189},[194,620,621,623,626,629,631,633,636,638,641],{"class":131,"line":196},[194,622,334],{"class":238},[194,624,625],{"class":337}," fetch_all",[194,627,628],{"class":242},"(session, url, params",[194,630,344],{"class":238},[194,632,347],{"class":296},[194,634,635],{"class":242},", max_pages",[194,637,344],{"class":238},[194,639,640],{"class":296},"200",[194,642,350],{"class":242},[194,644,645],{"class":131,"line":246},[194,646,647],{"class":203},"    \"\"\"Follow cursor pagination and return every record.\"\"\"\n",[194,649,650,653,655,658,661,663],{"class":131,"line":253},[194,651,652],{"class":242},"    params ",[194,654,344],{"class":238},[194,656,657],{"class":296}," dict",[194,659,660],{"class":242},"(params ",[194,662,419],{"class":238},[194,664,665],{"class":242}," {})\n",[194,667,668,671,673,676,678,680],{"class":131,"line":261},[194,669,670],{"class":242},"    records, pages, cursor ",[194,672,344],{"class":238},[194,674,675],{"class":242}," [], ",[194,677,44],{"class":296},[194,679,309],{"class":242},[194,681,682],{"class":296},"None\n",[194,684,685],{"class":131,"line":275},[194,686,250],{"emptyLinePlaceholder":249},[194,688,689,692,695,698],{"class":131,"line":288},[194,690,691],{"class":238},"    while",[194,693,694],{"class":242}," pages ",[194,696,697],{"class":238},"\u003C",[194,699,700],{"class":242}," max_pages:\n",[194,702,703,706],{"class":131,"line":293},[194,704,705],{"class":238},"        if",[194,707,708],{"class":242}," cursor:\n",[194,710,711,714,717,720,722],{"class":131,"line":321},[194,712,713],{"class":242},"            params[",[194,715,716],{"class":203},"\"cursor\"",[194,718,719],{"class":242},"] ",[194,721,344],{"class":238},[194,723,724],{"class":242}," cursor\n",[194,726,727,730,732,735,738,740,743,746,748,750],{"class":131,"line":326},[194,728,729],{"class":242},"        response ",[194,731,344],{"class":238},[194,733,734],{"class":242}," session.get(url, ",[194,736,737],{"class":459},"params",[194,739,344],{"class":238},[194,741,742],{"class":242},"params, ",[194,744,745],{"class":459},"timeout",[194,747,344],{"class":238},[194,749,297],{"class":296},[194,751,752],{"class":242},")\n",[194,754,755],{"class":131,"line":331},[194,756,757],{"class":242},"        response.raise_for_status()\n",[194,759,760,763,765],{"class":131,"line":353},[194,761,762],{"class":242},"        payload ",[194,764,344],{"class":238},[194,766,767],{"class":242}," response.json()\n",[194,769,770],{"class":131,"line":364},[194,771,250],{"emptyLinePlaceholder":249},[194,773,774,777,779,782,785],{"class":131,"line":370},[194,775,776],{"class":242},"        batch ",[194,778,344],{"class":238},[194,780,781],{"class":242}," payload.get(",[194,783,784],{"class":203},"\"data\"",[194,786,787],{"class":242},", [])\n",[194,789,790,792,795],{"class":131,"line":385},[194,791,705],{"class":238},[194,793,794],{"class":238}," not",[194,796,797],{"class":242}," batch:\n",[194,799,800],{"class":131,"line":398},[194,801,802],{"class":238},"            break\n",[194,804,805],{"class":131,"line":439},[194,806,807],{"class":242},"        records.extend(batch)\n",[194,809,810,813,816],{"class":131,"line":445},[194,811,812],{"class":242},"        pages ",[194,814,815],{"class":238},"+=",[194,817,818],{"class":296}," 1\n",[194,820,821],{"class":131,"line":456},[194,822,250],{"emptyLinePlaceholder":249},[194,824,825,828,830,833,836,839,841,844,847],{"class":131,"line":470},[194,826,827],{"class":242},"        cursor ",[194,829,344],{"class":238},[194,831,832],{"class":242}," (payload.get(",[194,834,835],{"class":203},"\"meta\"",[194,837,838],{"class":242},") ",[194,840,419],{"class":238},[194,842,843],{"class":242}," {}).get(",[194,845,846],{"class":203},"\"next_cursor\"",[194,848,752],{"class":242},[194,850,851,853,855],{"class":131,"line":487},[194,852,705],{"class":238},[194,854,794],{"class":238},[194,856,708],{"class":242},[194,858,859],{"class":131,"line":524},[194,860,802],{"class":238},[194,862,863,866],{"class":131,"line":544},[194,864,865],{"class":238},"    else",[194,867,868],{"class":242},":\n",[194,870,871,874,877,879,881,884,886,889,891,894],{"class":131,"line":557},[194,872,873],{"class":238},"        raise",[194,875,876],{"class":296}," RuntimeError",[194,878,495],{"class":242},[194,880,406],{"class":238},[194,882,883],{"class":203},"\"stopped after ",[194,885,413],{"class":412},[194,887,888],{"class":242},"max_pages",[194,890,431],{"class":412},[194,892,893],{"class":203}," pages — is the cursor advancing?\"",[194,895,752],{"class":242},[194,897,898],{"class":131,"line":563},[194,899,250],{"emptyLinePlaceholder":249},[194,901,902,904],{"class":131,"line":583},[194,903,586],{"class":238},[194,905,906],{"class":242}," records\n",[10,908,909,912,913,916,917,920,921,923],{},[191,910,911],{},"raise_for_status()"," is the line that separates data from an error page. Without it, an HTML \"503 Service Unavailable\" body reaches ",[191,914,915],{},"response.json()",", raises a JSON decode error, and sends you looking at your parsing code instead of at the endpoint. The ",[191,918,919],{},"while ... else"," clause runs only when the loop exhausts ",[191,922,888],{}," without breaking, which turns an infinite-pagination bug into an immediate, named failure.",[179,925,927],{"id":926},"step-3-flatten-the-json-into-columns","Step 3: Flatten the JSON into columns",[10,929,930,931,934,935,938,939,942],{},"API responses nest; spreadsheets do not. ",[191,932,933],{},"pd.json_normalize"," bridges the two, and its two important arguments are ",[191,936,937],{},"record_path"," (the list that becomes rows) and ",[191,940,941],{},"meta"," (the parent fields repeated onto each row):",[184,944,946],{"className":229,"code":945,"language":231,"meta":189,"style":189},"import pandas as pd\n\npayload = [\n    {\"invoice\": \"INV-001\", \"customer\": {\"id\": 7, \"name\": \"Acme\"},\n     \"lines\": [{\"sku\": \"A-1\", \"qty\": 2, \"net\": 40.0},\n               {\"sku\": \"B-3\", \"qty\": 1, \"net\": 15.5}]},\n    {\"invoice\": \"INV-002\", \"customer\": {\"id\": 9, \"name\": \"Globex\"},\n     \"lines\": [{\"sku\": \"A-1\", \"qty\": 5, \"net\": 100.0}]},\n]\n\ndf = pd.json_normalize(\n    payload,\n    record_path=\"lines\",                       # one row per line\n    meta=[\"invoice\", [\"customer\", \"name\"]],    # repeated on every line\n)\ndf = df.rename(columns={\"customer.name\": \"customer\"})\nprint(df)\n#    sku  qty    net  invoice customer\n# 0  A-1    2   40.0  INV-001     Acme\n# 1  B-3    1   15.5  INV-001     Acme\n# 2  A-1    5  100.0  INV-002   Globex\n",[191,947,948,961,965,975,1017,1055,1088,1123,1154,1159,1163,1173,1178,1193,1220,1224,1250,1258,1263,1268,1273],{"__ignoreMap":189},[194,949,950,952,955,958],{"class":131,"line":196},[194,951,239],{"class":238},[194,953,954],{"class":242}," pandas ",[194,956,957],{"class":238},"as",[194,959,960],{"class":242}," pd\n",[194,962,963],{"class":131,"line":246},[194,964,250],{"emptyLinePlaceholder":249},[194,966,967,970,972],{"class":131,"line":253},[194,968,969],{"class":242},"payload ",[194,971,344],{"class":238},[194,973,974],{"class":242}," [\n",[194,976,977,980,983,985,988,990,993,996,999,1001,1004,1006,1009,1011,1014],{"class":131,"line":261},[194,978,979],{"class":242},"    {",[194,981,982],{"class":203},"\"invoice\"",[194,984,376],{"class":242},[194,986,987],{"class":203},"\"INV-001\"",[194,989,309],{"class":242},[194,991,992],{"class":203},"\"customer\"",[194,994,995],{"class":242},": {",[194,997,998],{"class":203},"\"id\"",[194,1000,376],{"class":242},[194,1002,1003],{"class":296},"7",[194,1005,309],{"class":242},[194,1007,1008],{"class":203},"\"name\"",[194,1010,376],{"class":242},[194,1012,1013],{"class":203},"\"Acme\"",[194,1015,1016],{"class":242},"},\n",[194,1018,1019,1022,1025,1028,1030,1033,1035,1038,1040,1043,1045,1048,1050,1053],{"class":131,"line":275},[194,1020,1021],{"class":203},"     \"lines\"",[194,1023,1024],{"class":242},": [{",[194,1026,1027],{"class":203},"\"sku\"",[194,1029,376],{"class":242},[194,1031,1032],{"class":203},"\"A-1\"",[194,1034,309],{"class":242},[194,1036,1037],{"class":203},"\"qty\"",[194,1039,376],{"class":242},[194,1041,1042],{"class":296},"2",[194,1044,309],{"class":242},[194,1046,1047],{"class":203},"\"net\"",[194,1049,376],{"class":242},[194,1051,1052],{"class":296},"40.0",[194,1054,1016],{"class":242},[194,1056,1057,1060,1062,1064,1067,1069,1071,1073,1076,1078,1080,1082,1085],{"class":131,"line":288},[194,1058,1059],{"class":242},"               {",[194,1061,1027],{"class":203},[194,1063,376],{"class":242},[194,1065,1066],{"class":203},"\"B-3\"",[194,1068,309],{"class":242},[194,1070,1037],{"class":203},[194,1072,376],{"class":242},[194,1074,1075],{"class":296},"1",[194,1077,309],{"class":242},[194,1079,1047],{"class":203},[194,1081,376],{"class":242},[194,1083,1084],{"class":296},"15.5",[194,1086,1087],{"class":242},"}]},\n",[194,1089,1090,1092,1094,1096,1099,1101,1103,1105,1107,1109,1112,1114,1116,1118,1121],{"class":131,"line":293},[194,1091,979],{"class":242},[194,1093,982],{"class":203},[194,1095,376],{"class":242},[194,1097,1098],{"class":203},"\"INV-002\"",[194,1100,309],{"class":242},[194,1102,992],{"class":203},[194,1104,995],{"class":242},[194,1106,998],{"class":203},[194,1108,376],{"class":242},[194,1110,1111],{"class":296},"9",[194,1113,309],{"class":242},[194,1115,1008],{"class":203},[194,1117,376],{"class":242},[194,1119,1120],{"class":203},"\"Globex\"",[194,1122,1016],{"class":242},[194,1124,1125,1127,1129,1131,1133,1135,1137,1139,1141,1143,1145,1147,1149,1152],{"class":131,"line":321},[194,1126,1021],{"class":203},[194,1128,1024],{"class":242},[194,1130,1027],{"class":203},[194,1132,376],{"class":242},[194,1134,1032],{"class":203},[194,1136,309],{"class":242},[194,1138,1037],{"class":203},[194,1140,376],{"class":242},[194,1142,306],{"class":296},[194,1144,309],{"class":242},[194,1146,1047],{"class":203},[194,1148,376],{"class":242},[194,1150,1151],{"class":296},"100.0",[194,1153,1087],{"class":242},[194,1155,1156],{"class":131,"line":326},[194,1157,1158],{"class":242},"]\n",[194,1160,1161],{"class":131,"line":331},[194,1162,250],{"emptyLinePlaceholder":249},[194,1164,1165,1168,1170],{"class":131,"line":353},[194,1166,1167],{"class":242},"df ",[194,1169,344],{"class":238},[194,1171,1172],{"class":242}," pd.json_normalize(\n",[194,1174,1175],{"class":131,"line":364},[194,1176,1177],{"class":242},"    payload,\n",[194,1179,1180,1183,1185,1188,1190],{"class":131,"line":370},[194,1181,1182],{"class":459},"    record_path",[194,1184,344],{"class":238},[194,1186,1187],{"class":203},"\"lines\"",[194,1189,481],{"class":242},[194,1191,1192],{"class":317},"# one row per line\n",[194,1194,1195,1198,1200,1203,1205,1208,1210,1212,1214,1217],{"class":131,"line":385},[194,1196,1197],{"class":459},"    meta",[194,1199,344],{"class":238},[194,1201,1202],{"class":242},"[",[194,1204,982],{"class":203},[194,1206,1207],{"class":242},", [",[194,1209,992],{"class":203},[194,1211,309],{"class":242},[194,1213,1008],{"class":203},[194,1215,1216],{"class":242},"]],    ",[194,1218,1219],{"class":317},"# repeated on every line\n",[194,1221,1222],{"class":131,"line":398},[194,1223,752],{"class":242},[194,1225,1226,1228,1230,1233,1236,1238,1240,1243,1245,1247],{"class":131,"line":439},[194,1227,1167],{"class":242},[194,1229,344],{"class":238},[194,1231,1232],{"class":242}," df.rename(",[194,1234,1235],{"class":459},"columns",[194,1237,344],{"class":238},[194,1239,413],{"class":242},[194,1241,1242],{"class":203},"\"customer.name\"",[194,1244,376],{"class":242},[194,1246,992],{"class":203},[194,1248,1249],{"class":242},"})\n",[194,1251,1252,1255],{"class":131,"line":445},[194,1253,1254],{"class":296},"print",[194,1256,1257],{"class":242},"(df)\n",[194,1259,1260],{"class":131,"line":456},[194,1261,1262],{"class":317},"#    sku  qty    net  invoice customer\n",[194,1264,1265],{"class":131,"line":470},[194,1266,1267],{"class":317},"# 0  A-1    2   40.0  INV-001     Acme\n",[194,1269,1270],{"class":131,"line":487},[194,1271,1272],{"class":317},"# 1  B-3    1   15.5  INV-001     Acme\n",[194,1274,1275],{"class":131,"line":524},[194,1276,1277],{"class":317},"# 2  A-1    5  100.0  INV-002   Globex\n",[10,1279,1280,1281,1284,1285,1287,1288,1291],{},"Nested keys arrive as dotted names — ",[191,1282,1283],{},"customer.name"," — which are legal DataFrame columns but ugly headings, so rename them before writing. When a field is sometimes missing, ",[191,1286,103],{}," fills ",[191,1289,1290],{},"NaN"," rather than raising, so check for unexpected nulls afterwards instead of assuming every record had the key.",[179,1293,1295],{"id":1294},"step-4-type-the-columns-before-they-reach-excel","Step 4: Type the columns before they reach Excel",[10,1297,1298],{},"JSON has no date type and often no numeric type either — amounts arrive as strings, timestamps as ISO text. Convert explicitly, or Excel receives text that looks right and refuses to sum:",[184,1300,1302],{"className":229,"code":1301,"language":231,"meta":189,"style":189},"df[\"net\"] = pd.to_numeric(df[\"net\"], errors=\"coerce\")\ndf[\"fetched_at\"] = pd.Timestamp.now().floor(\"s\")\n\nwith pd.ExcelWriter(\"invoices.xlsx\", engine=\"openpyxl\",\n                    datetime_format=\"yyyy-mm-dd hh:mm\") as writer:\n    df.to_excel(writer, sheet_name=\"Lines\", index=False)\n    ws = writer.sheets[\"Lines\"]\n    for cell in ws[\"C\"][1:]:                 # the net column\n        cell.number_format = '#,##0.00'\n    ws.freeze_panes = \"A2\"\n",[191,1303,1304,1333,1352,1356,1379,1396,1421,1435,1463,1473],{"__ignoreMap":189},[194,1305,1306,1309,1311,1313,1315,1318,1320,1323,1326,1328,1331],{"class":131,"line":196},[194,1307,1308],{"class":242},"df[",[194,1310,1047],{"class":203},[194,1312,719],{"class":242},[194,1314,344],{"class":238},[194,1316,1317],{"class":242}," pd.to_numeric(df[",[194,1319,1047],{"class":203},[194,1321,1322],{"class":242},"], ",[194,1324,1325],{"class":459},"errors",[194,1327,344],{"class":238},[194,1329,1330],{"class":203},"\"coerce\"",[194,1332,752],{"class":242},[194,1334,1335,1337,1340,1342,1344,1347,1350],{"class":131,"line":246},[194,1336,1308],{"class":242},[194,1338,1339],{"class":203},"\"fetched_at\"",[194,1341,719],{"class":242},[194,1343,344],{"class":238},[194,1345,1346],{"class":242}," pd.Timestamp.now().floor(",[194,1348,1349],{"class":203},"\"s\"",[194,1351,752],{"class":242},[194,1353,1354],{"class":131,"line":253},[194,1355,250],{"emptyLinePlaceholder":249},[194,1357,1358,1361,1364,1367,1369,1372,1374,1377],{"class":131,"line":261},[194,1359,1360],{"class":238},"with",[194,1362,1363],{"class":242}," pd.ExcelWriter(",[194,1365,1366],{"class":203},"\"invoices.xlsx\"",[194,1368,309],{"class":242},[194,1370,1371],{"class":459},"engine",[194,1373,344],{"class":238},[194,1375,1376],{"class":203},"\"openpyxl\"",[194,1378,382],{"class":242},[194,1380,1381,1384,1386,1389,1391,1393],{"class":131,"line":275},[194,1382,1383],{"class":459},"                    datetime_format",[194,1385,344],{"class":238},[194,1387,1388],{"class":203},"\"yyyy-mm-dd hh:mm\"",[194,1390,838],{"class":242},[194,1392,957],{"class":238},[194,1394,1395],{"class":242}," writer:\n",[194,1397,1398,1401,1404,1406,1409,1411,1414,1416,1419],{"class":131,"line":288},[194,1399,1400],{"class":242},"    df.to_excel(writer, ",[194,1402,1403],{"class":459},"sheet_name",[194,1405,344],{"class":238},[194,1407,1408],{"class":203},"\"Lines\"",[194,1410,309],{"class":242},[194,1412,1413],{"class":459},"index",[194,1415,344],{"class":238},[194,1417,1418],{"class":296},"False",[194,1420,752],{"class":242},[194,1422,1423,1426,1428,1431,1433],{"class":131,"line":293},[194,1424,1425],{"class":242},"    ws ",[194,1427,344],{"class":238},[194,1429,1430],{"class":242}," writer.sheets[",[194,1432,1408],{"class":203},[194,1434,1158],{"class":242},[194,1436,1437,1440,1443,1446,1449,1452,1455,1457,1460],{"class":131,"line":321},[194,1438,1439],{"class":238},"    for",[194,1441,1442],{"class":242}," cell ",[194,1444,1445],{"class":238},"in",[194,1447,1448],{"class":242}," ws[",[194,1450,1451],{"class":203},"\"C\"",[194,1453,1454],{"class":242},"][",[194,1456,1075],{"class":296},[194,1458,1459],{"class":242},":]:                 ",[194,1461,1462],{"class":317},"# the net column\n",[194,1464,1465,1468,1470],{"class":131,"line":326},[194,1466,1467],{"class":242},"        cell.number_format ",[194,1469,344],{"class":238},[194,1471,1472],{"class":203}," '#,##0.00'\n",[194,1474,1475,1478,1480],{"class":131,"line":331},[194,1476,1477],{"class":242},"    ws.freeze_panes ",[194,1479,344],{"class":238},[194,1481,1482],{"class":203}," \"A2\"\n",[10,1484,592,1485,1488],{},[191,1486,1487],{},"fetched_at"," column is worth the one line. API data is a snapshot, and a workbook that does not say when it was taken will be treated as current for as long as it exists on someone's desktop.",[179,1490,1492],{"id":1491},"step-5-cache-responses-while-you-are-developing","Step 5: Cache responses while you are developing",[10,1494,1495],{},"Iterating on the parsing while hitting a live rate-limited endpoint is slow and rude. A file cache keyed by the request makes the second run instant:",[184,1497,1499],{"className":229,"code":1498,"language":231,"meta":189,"style":189},"import hashlib\nimport json\nfrom pathlib import Path\n\nCACHE = Path(\".api-cache\")\n\n\ndef cached_get(session, url, params=None, max_age_hours=6):\n    CACHE.mkdir(exist_ok=True)\n    key = hashlib.sha256(f\"{url}?{sorted((params or {}).items())}\".encode()).hexdigest()\n    path = CACHE \u002F f\"{key}.json\"\n\n    if path.is_file():\n        age_hours = (pd.Timestamp.now().timestamp() - path.stat().st_mtime) \u002F 3600\n        if age_hours \u003C max_age_hours:\n            return json.loads(path.read_text())\n\n    response = session.get(url, params=params, timeout=TIMEOUT)\n    response.raise_for_status()\n    payload = response.json()\n    path.write_text(json.dumps(payload))\n    return payload\n",[191,1500,1501,1508,1515,1527,1531,1546,1550,1554,1577,1594,1638,1666,1670,1678,1700,1712,1720,1724,1747,1752,1761,1766],{"__ignoreMap":189},[194,1502,1503,1505],{"class":131,"line":196},[194,1504,239],{"class":238},[194,1506,1507],{"class":242}," hashlib\n",[194,1509,1510,1512],{"class":131,"line":246},[194,1511,239],{"class":238},[194,1513,1514],{"class":242}," json\n",[194,1516,1517,1519,1522,1524],{"class":131,"line":253},[194,1518,264],{"class":238},[194,1520,1521],{"class":242}," pathlib ",[194,1523,239],{"class":238},[194,1525,1526],{"class":242}," Path\n",[194,1528,1529],{"class":131,"line":261},[194,1530,250],{"emptyLinePlaceholder":249},[194,1532,1533,1536,1538,1541,1544],{"class":131,"line":275},[194,1534,1535],{"class":296},"CACHE",[194,1537,300],{"class":238},[194,1539,1540],{"class":242}," Path(",[194,1542,1543],{"class":203},"\".api-cache\"",[194,1545,752],{"class":242},[194,1547,1548],{"class":131,"line":288},[194,1549,250],{"emptyLinePlaceholder":249},[194,1551,1552],{"class":131,"line":293},[194,1553,250],{"emptyLinePlaceholder":249},[194,1555,1556,1558,1561,1563,1565,1567,1570,1572,1575],{"class":131,"line":321},[194,1557,334],{"class":238},[194,1559,1560],{"class":337}," cached_get",[194,1562,628],{"class":242},[194,1564,344],{"class":238},[194,1566,347],{"class":296},[194,1568,1569],{"class":242},", max_age_hours",[194,1571,344],{"class":238},[194,1573,1574],{"class":296},"6",[194,1576,350],{"class":242},[194,1578,1579,1582,1585,1588,1590,1592],{"class":131,"line":326},[194,1580,1581],{"class":296},"    CACHE",[194,1583,1584],{"class":242},".mkdir(",[194,1586,1587],{"class":459},"exist_ok",[194,1589,344],{"class":238},[194,1591,552],{"class":296},[194,1593,752],{"class":242},[194,1595,1596,1599,1601,1604,1606,1608,1610,1613,1615,1618,1620,1623,1626,1628,1631,1633,1635],{"class":131,"line":331},[194,1597,1598],{"class":242},"    key ",[194,1600,344],{"class":238},[194,1602,1603],{"class":242}," hashlib.sha256(",[194,1605,406],{"class":238},[194,1607,434],{"class":203},[194,1609,413],{"class":412},[194,1611,1612],{"class":242},"url",[194,1614,431],{"class":412},[194,1616,1617],{"class":203},"?",[194,1619,413],{"class":412},[194,1621,1622],{"class":296},"sorted",[194,1624,1625],{"class":242},"((params ",[194,1627,419],{"class":238},[194,1629,1630],{"class":242}," {}).items())",[194,1632,431],{"class":412},[194,1634,434],{"class":203},[194,1636,1637],{"class":242},".encode()).hexdigest()\n",[194,1639,1640,1643,1645,1648,1651,1654,1656,1658,1661,1663],{"class":131,"line":353},[194,1641,1642],{"class":242},"    path ",[194,1644,344],{"class":238},[194,1646,1647],{"class":296}," CACHE",[194,1649,1650],{"class":238}," \u002F",[194,1652,1653],{"class":238}," f",[194,1655,434],{"class":203},[194,1657,413],{"class":412},[194,1659,1660],{"class":242},"key",[194,1662,431],{"class":412},[194,1664,1665],{"class":203},".json\"\n",[194,1667,1668],{"class":131,"line":364},[194,1669,250],{"emptyLinePlaceholder":249},[194,1671,1672,1675],{"class":131,"line":370},[194,1673,1674],{"class":238},"    if",[194,1676,1677],{"class":242}," path.is_file():\n",[194,1679,1680,1683,1685,1688,1691,1694,1697],{"class":131,"line":385},[194,1681,1682],{"class":242},"        age_hours ",[194,1684,344],{"class":238},[194,1686,1687],{"class":242}," (pd.Timestamp.now().timestamp() ",[194,1689,1690],{"class":238},"-",[194,1692,1693],{"class":242}," path.stat().st_mtime) ",[194,1695,1696],{"class":238},"\u002F",[194,1698,1699],{"class":296}," 3600\n",[194,1701,1702,1704,1707,1709],{"class":131,"line":398},[194,1703,705],{"class":238},[194,1705,1706],{"class":242}," age_hours ",[194,1708,697],{"class":238},[194,1710,1711],{"class":242}," max_age_hours:\n",[194,1713,1714,1717],{"class":131,"line":439},[194,1715,1716],{"class":238},"            return",[194,1718,1719],{"class":242}," json.loads(path.read_text())\n",[194,1721,1722],{"class":131,"line":445},[194,1723,250],{"emptyLinePlaceholder":249},[194,1725,1726,1729,1731,1733,1735,1737,1739,1741,1743,1745],{"class":131,"line":456},[194,1727,1728],{"class":242},"    response ",[194,1730,344],{"class":238},[194,1732,734],{"class":242},[194,1734,737],{"class":459},[194,1736,344],{"class":238},[194,1738,742],{"class":242},[194,1740,745],{"class":459},[194,1742,344],{"class":238},[194,1744,297],{"class":296},[194,1746,752],{"class":242},[194,1748,1749],{"class":131,"line":470},[194,1750,1751],{"class":242},"    response.raise_for_status()\n",[194,1753,1754,1757,1759],{"class":131,"line":487},[194,1755,1756],{"class":242},"    payload ",[194,1758,344],{"class":238},[194,1760,767],{"class":242},[194,1762,1763],{"class":131,"line":524},[194,1764,1765],{"class":242},"    path.write_text(json.dumps(payload))\n",[194,1767,1768,1770],{"class":131,"line":544},[194,1769,586],{"class":238},[194,1771,1772],{"class":242}," payload\n",[23,1774,32,1779,32,1782,32,1785,32,1788,32,1792,32,1798,32,1802,32,1806,32,1809,32,1813,32,1816,32,1820,32,1825,32,1830,32,1832,32,1834,32,1839,32,1842,32,1847,32,1850,32,1855,32,1858,32,1862,32,1865],{"viewBox":1775,"role":26,"ariaLabelledBy":1776,"xmlns":30,"style":31},"0 0 740 224",[1777,1778],"api-cache-t","api-cache-d",[34,1780,1781],{"id":1777},"What the cache changes while you are iterating on the parsing",[38,1783,1784],{"id":1778},"The first run fetches every page over the network and writes each response to a file keyed by its request. Every run after that reads those files, so a change to the flattening code is tested in under a second instead of re-downloading forty pages and consuming the rate limit again.",[42,1786],{"x":44,"y":44,"width":45,"height":1787,"fill":47},"224",[49,1789,1791],{"x":51,"y":52,"style":1790},"font-size:12.5px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","Same code, first run and every run after it",[49,1793,1797],{"x":1794,"y":1795,"style":1796},"80","66","font-size:11.5px;font-weight:700;fill:var(--gold-ink,#7a4e06)","run 1",[42,1799],{"x":1800,"y":1801,"width":113,"height":52,"rx":1574,"fill":83,"stroke":84,"style":64},"140","50",[49,1803,1805],{"x":640,"y":1804,"style":74},"70","GET page 1",[42,1807],{"x":1808,"y":1801,"width":113,"height":52,"rx":1574,"fill":83,"stroke":84,"style":64},"268",[49,1810,1812],{"x":1811,"y":1804,"style":74},"328","GET page 2",[42,1814],{"x":1815,"y":1801,"width":113,"height":52,"rx":1574,"fill":83,"stroke":84,"style":64},"396",[49,1817,1819],{"x":1818,"y":1804,"style":74},"456","GET page 3",[49,1821,1824],{"x":1822,"y":1804,"style":1823},"536","font-size:11px;fill:var(--muted,#5b6780)","… 40 pages, ~30 s",[131,1826],{"x1":640,"y1":1827,"x2":640,"y2":1828,"stroke":135,"style":1829},"84","104","stroke-width:1.5px;stroke-dasharray:3 3",[131,1831],{"x1":1811,"y1":1827,"x2":1811,"y2":1828,"stroke":135,"style":1829},[131,1833],{"x1":1818,"y1":1827,"x2":1818,"y2":1828,"stroke":135,"style":1829},[42,1835],{"x":1800,"y":73,"width":1836,"height":52,"rx":1574,"fill":1837,"stroke":1838,"style":64},"376","#f0f2f5","var(--line,#cdd5e6)",[49,1840,1841],{"x":1811,"y":78,"style":74},".api-cache\u002F — one file per hashed request",[49,1843,1846],{"x":1794,"y":1844,"style":1845},"166","font-size:11.5px;font-weight:700;fill:var(--teal-ink,#0b6157)","run 2+",[42,1848],{"x":1800,"y":1849,"width":1836,"height":52,"rx":1574,"fill":114,"stroke":115,"style":64},"150",[49,1851,1854],{"x":1811,"y":1852,"style":1853},"170","font-size:11px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","read from disk — no network, no rate limit",[131,1856],{"x1":1811,"y1":1800,"x2":1811,"y2":1857,"stroke":115,"style":64},"146",[137,1859],{"points":1860,"fill":1861},"328,150 322,138 334,138","#0f766e",[49,1863,1864],{"x":1822,"y":1852,"style":1823},"under a second",[49,1866,1868],{"x":51,"y":134,"style":1867},"font-size:11px;fill:var(--muted,#5b6780);text-anchor:middle","Keep max_age low enough that a daily report can never serve yesterday's snapshot",[10,1870,1871,1872,1875,1876,1879,1880,1883],{},"Hashing the URL and the sorted parameters means two calls that differ only in parameter order share a cache entry, which is what you want. Add ",[191,1873,1874],{},".api-cache\u002F"," to ",[191,1877,1878],{},".gitignore",", and keep the cache out of the scheduled run — or set ",[191,1881,1882],{},"max_age_hours"," low enough that a daily report can never serve yesterday's data.",[23,1885,32,1890,32,1893,32,1896,32,1899,32,1904,32,1909,32,1914,32,1917,32,1921,32,1924,32,1929,32,1932,32,1935,32,1938,32,1940,32,1945,32,1948,32,1951,32,1954,32,1959,32,1964,32,1967,32,1970],{"viewBox":1886,"role":26,"ariaLabelledBy":1887,"xmlns":30,"style":31},"0 0 740 236",[1888,1889],"api-fail-t","api-fail-d",[34,1891,1892],{"id":1888},"How each HTTP response should be treated",[38,1894,1895],{"id":1889},"A 200 with records is processed. A 200 with an empty page ends pagination normally. A 429 or a 5xx is retried with backoff, honouring any Retry-After header. A 4xx other than 429 stops the job immediately, because the request itself is wrong and will fail identically on every attempt.",[42,1897],{"x":44,"y":44,"width":45,"height":1898,"fill":47},"236",[42,1900],{"x":57,"y":1901,"width":1852,"height":1902,"rx":1903,"fill":114,"stroke":115,"style":64},"40","156","14",[49,1905,1908],{"x":1906,"y":1804,"style":1907},"101","font-size:12.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","200 with data",[49,1910,1913],{"x":1906,"y":1911,"style":1912},"100","font-size:11.5px;fill:var(--text,#172033);text-anchor:middle","extend the records",[49,1915,92],{"x":1906,"y":1916,"style":1912},"124",[49,1918,1920],{"x":1906,"y":1919,"style":1867},"162","the ordinary path",[42,1922],{"x":1923,"y":1901,"width":1852,"height":1902,"rx":1903,"fill":1837,"stroke":1838,"style":64},"198",[49,1925,1928],{"x":1926,"y":1804,"style":1927},"283","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","200, empty page",[49,1930,1931],{"x":1926,"y":1911,"style":1912},"stop the loop",[49,1933,1934],{"x":1926,"y":1916,"style":1912},"this is success",[49,1936,1937],{"x":1926,"y":1919,"style":1867},"not an error to report",[42,1939],{"x":143,"y":1901,"width":1852,"height":1902,"rx":1903,"fill":83,"stroke":84,"style":64},[49,1941,1944],{"x":1942,"y":1804,"style":1943},"465","font-size:12.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","429 or 5xx",[49,1946,1947],{"x":1942,"y":1911,"style":1912},"retry with backoff",[49,1949,1950],{"x":1942,"y":1916,"style":1912},"honour Retry-After",[49,1952,1953],{"x":1942,"y":1919,"style":1867},"time may fix it",[42,1955],{"x":1956,"y":1901,"width":1919,"height":1902,"rx":1903,"fill":1957,"stroke":1958,"style":64},"562","#fce9e9","var(--accent-ink,#be185d)",[49,1960,1963],{"x":1961,"y":1804,"style":1962},"643","font-size:12.5px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","other 4xx",[49,1965,1966],{"x":1961,"y":1911,"style":1912},"stop immediately",[49,1968,1969],{"x":1961,"y":1916,"style":1912},"report the body",[49,1971,1972],{"x":1961,"y":1919,"style":1867},"the request is wrong",[179,1974,1976],{"id":1975},"common-pitfalls-and-gotchas","Common pitfalls and gotchas",[1978,1979,1980,1996],"table",{},[1981,1982,1983],"thead",{},[1984,1985,1986,1990,1993],"tr",{},[1987,1988,1989],"th",{},"Symptom",[1987,1991,1992],{},"Cause",[1987,1994,1995],{},"Fix",[1997,1998,1999,2018,2035,2048,2069,2084,2097,2108],"tbody",{},[1984,2000,2001,2005,2011],{},[2002,2003,2004],"td",{},"Job hangs overnight",[2002,2006,2007,2008,2010],{},"No ",[191,2009,745],{}," on the request",[2002,2012,2013,2014,2017],{},"Pass a ",[191,2015,2016],{},"(connect, read)"," tuple every time",[1984,2019,2020,2026,2029],{},[2002,2021,2022,2025],{},[191,2023,2024],{},"JSONDecodeError"," on an HTML body",[2002,2027,2028],{},"An error page parsed as data",[2002,2030,2031,2032,2034],{},"Call ",[191,2033,911],{}," first",[1984,2036,2037,2040,2043],{},[2002,2038,2039],{},"Pagination never ends",[2002,2041,2042],{},"Cursor not advancing",[2002,2044,2045,2046],{},"Break on an empty page and cap ",[191,2047,888],{},[1984,2049,2050,2053,2059],{},[2002,2051,2052],{},"429s halfway through",[2002,2054,2055,2056],{},"No backoff, or ignoring ",[191,2057,2058],{},"Retry-After",[2002,2060,2061,2062,2065,2066],{},"Use the ",[191,2063,2064],{},"Retry"," adapter with ",[191,2067,2068],{},"respect_retry_after_header",[1984,2070,2071,2076,2079],{},[2002,2072,2073,2074],{},"Columns named ",[191,2075,1283],{},[2002,2077,2078],{},"Nested keys flattened with dots",[2002,2080,2081,2082],{},"Rename after ",[191,2083,103],{},[1984,2085,2086,2089,2092],{},[2002,2087,2088],{},"Amounts will not sum in Excel",[2002,2090,2091],{},"Numbers arrived as JSON strings",[2002,2093,2094],{},[191,2095,2096],{},"pd.to_numeric(..., errors=\"coerce\")",[1984,2098,2099,2102,2105],{},[2002,2100,2101],{},"Token visible in a log",[2002,2103,2104],{},"Header printed while debugging",[2002,2106,2107],{},"Log the URL and status, never the headers",[1984,2109,2110,2113,2116],{},[2002,2111,2112],{},"Rerun downloads everything again",[2002,2114,2115],{},"No cache",[2002,2117,2118],{},"Cache by hashed URL and parameters",[179,2120,2122],{"id":2121},"performance-and-scale-notes","Performance and scale notes",[10,2124,2125,2126,2129],{},"The wall-clock cost of an API export is nearly all latency: 200 pages at 300 ms each is a minute, and no amount of pandas tuning changes that. Ask for larger pages if the API allows it, request only the fields you need, and use an incremental filter — ",[191,2127,2128],{},"updated_since"," — so a daily job fetches a day rather than the whole history.",[10,2131,2132],{},"If the API supports concurrency and permits it, a small thread pool of four to eight workers over independent page ranges cuts the time proportionally. Keep it small: a job that trips the rate limit spends longer backing off than it saved. Where the same data feeds several reports, fetch once into a database or a Parquet file and let the reports read from there rather than each calling the API.",[179,2134,2136],{"id":2135},"conclusion","Conclusion",[10,2138,2139,2140,2142],{},"The API call itself is one line; everything that makes it survivable is around it. Configure one session with an auth header, a timeout and a retry policy that distinguishes 429 and 5xx from a bad request. Page with two guards so the loop always terminates. Flatten with ",[191,2141,103],{},", rename the dotted columns, coerce the types, and stamp the workbook with the time the snapshot was taken. Cache during development so iteration costs nothing and the endpoint is left alone.",[179,2144,2146],{"id":2145},"frequently-asked-questions","Frequently asked questions",[10,2148,2149,2153,2154,2156,2157,2159],{},[2150,2151,2152],"strong",{},"Why does my scheduled job hang forever some mornings?","\nA request with no ",[191,2155,745],{},". requests waits indefinitely by default, so one unresponsive endpoint blocks the job until someone notices. Always pass ",[191,2158,745],{},", and prefer a tuple of connect and read timeouts.",[10,2161,2162,2165,2167,2168,2170,2171,2173],{},[2150,2163,2164],{},"How do I flatten nested JSON into spreadsheet columns?",[191,2166,933],{},", with ",[191,2169,937],{}," for the list you want as rows and ",[191,2172,941],{}," for the parent fields to repeat on each row. It produces dotted column names you can rename afterwards.",[10,2175,2176,2179,2180,2182],{},[2150,2177,2178],{},"Should I retry a failed API call?","\nRetry 429 and 5xx responses with backoff; never retry a 400 or 401, which will fail identically. urllib3's ",[191,2181,2064],{}," adapter does this for you at the session level.",[10,2184,2185,2188],{},[2150,2186,2187],{},"How do I avoid re-downloading the same data during development?","\nCache each response to a file keyed by the request, and read the cache when it exists. It makes iteration fast and keeps you inside the rate limit.",[179,2190,2192],{"id":2191},"related","Related",[10,2194,2195],{},"Up to the parent guide:",[2197,2198,2199],"ul",{},[2200,2201,2202,2204],"li",{},[17,2203,20],{"href":19}," — where API data fits alongside SQL sources.",[10,2206,2207],{},"Related guides:",[2197,2209,2210,2217,2222,2229],{},[2200,2211,2212,2216],{},[17,2213,2215],{"href":2214},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-sql-query-results-to-excel-with-python\u002F","Export SQL Query Results to Excel with Python"," — the same workbook-writing step from a database source.",[2200,2218,2219,2221],{},[17,2220,600],{"href":599}," — the retry rules this session policy encodes.",[2200,2223,2224,2228],{},[17,2225,2227],{"href":2226},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Frefresh-an-excel-report-from-a-database-on-a-schedule\u002F","Refresh an Excel Report from a Database on a Schedule"," — running the fetch unattended.",[2200,2230,2231,2235],{},[17,2232,2234],{"href":2233},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fmerge-two-excel-files-on-common-column-python\u002F","Merge Two Excel Files on a Common Column in Python"," — joining fetched reference data onto your own.",[2237,2238,2239],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}",{"title":189,"searchDepth":246,"depth":246,"links":2241},[2242,2243,2244,2245,2246,2247,2248,2249,2250,2251,2252],{"id":181,"depth":246,"text":182},{"id":219,"depth":246,"text":220},{"id":609,"depth":246,"text":610},{"id":926,"depth":246,"text":927},{"id":1294,"depth":246,"text":1295},{"id":1491,"depth":246,"text":1492},{"id":1975,"depth":246,"text":1976},{"id":2121,"depth":246,"text":2122},{"id":2135,"depth":246,"text":2136},{"id":2145,"depth":246,"text":2146},{"id":2191,"depth":246,"text":2192},"2026-08-10","Pull JSON from an HTTP API into a workbook: a session with timeouts and retries, pagination that terminates, json_normalize for nested records, and a cache so a rerun does not hammer the endpoint.","md",[2257,2259,2261,2263],{"q":2152,"a":2258},"A request with no timeout. requests waits indefinitely by default, so one unresponsive endpoint blocks the job until someone notices. Always pass timeout, and prefer a tuple of connect and read timeouts.",{"q":2164,"a":2260},"pd.json_normalize, with record_path for the list you want as rows and meta for the parent fields to repeat on each row. It produces dotted column names you can rename afterwards.",{"q":2178,"a":2262},"Retry 429 and 5xx responses with backoff; never retry a 400 or 401, which will fail identically. urllib3's Retry adapter does this for you at the session level.",{"q":2187,"a":2264},"Cache each response to a file keyed by the request, and read the cache when it exists. It makes iteration fast and keeps you inside the rate limit.",{"breadcrumb":2266},[2267,2269,2272,2273],{"name":2268,"item":1696},"Home",{"name":2270,"item":2271},"Advanced Data Transformation and Cleaning","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":20,"item":19},{"name":5,"item":2274},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Ffetch-api-data-into-excel-with-python-requests\u002F","\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Ffetch-api-data-into-excel-with-python-requests",{"title":5,"description":2277},"Call a REST API from Python and write the result to Excel — sessions, timeouts, retry policy, pagination loops, flattening nested JSON with json_normalize, and local caching.","fetch-api-data-into-excel-with-python-requests","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Ffetch-api-data-into-excel-with-python-requests\u002Findex","how-to","kq_Hh3oETCbJs_BZNVgry8LavhmAnz1IzRsAiFzB7Pw",[2283,2286],{"title":2215,"path":2284,"stem":2285,"children":-1},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-sql-query-results-to-excel-with-python","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-sql-query-results-to-excel-with-python\u002Findex",{"title":2287,"path":2288,"stem":2289,"children":-1},"Load an Excel File into a SQL Database with pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-an-excel-file-into-a-sql-database-with-pandas","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-an-excel-file-into-a-sql-database-with-pandas\u002Findex",1786800027102]