import json
import gzip
import hashlib
from datetime import datetime, timezone
from pathlib import Path

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import requests

payload = """
{"indicator": {"id": "NY.GDP.PCAP.KD", "value": "GDP per capita"},
 "observations": [{"country": "FRA", "date": "2020", "value": 38515.6},
                  {"country": "FRA", "date": "2021", "value": 41065.4}]}
"""

parsed = json.loads(payload)
print(f"top-level keys : {list(parsed)}")
print(f"indicator name : {parsed['indicator']['value']}")
print(f"observations   : {len(parsed['observations'])} records\n")

print(pd.DataFrame(parsed["observations"]))

top-level keys : ['indicator', 'observations']
indicator name : GDP per capita
observations   : 2 records

  country  date    value
0     FRA  2020  38515.6
1     FRA  2021  41065.4

CACHE = Path("data_cache")
MANIFEST = json.loads((CACHE / "manifest.json").read_text(encoding="utf-8"))

LIVE = False          # set True to re-fetch from source and compare against the manifest

USER_AGENT = {"User-Agent": "math-econ-code course notebook (teaching example)"}

def sha256(raw: bytes) -> str:
    return hashlib.sha256(raw).hexdigest()

def load_raw(name):
    """Return the archived bytes for `name`, verified against the manifest."""
    entry = MANIFEST[name]
    raw = gzip.open(CACHE / entry["stored_as"], "rb").read()
    digest = sha256(raw)
    if digest != entry["sha256"]:
        raise ValueError(f"{name}: archive is corrupt "
                         f"(sha256 {digest[:12]} != recorded {entry['sha256'][:12]})")
    return raw

def fetch_raw(name):
    """Re-fetch from source and report whether the payload still matches."""
    entry = MANIFEST[name]
    response = requests.get(entry["url"], headers=USER_AGENT, timeout=30)
    response.raise_for_status()
    digest = sha256(response.content)
    if digest == entry["sha256"]:
        print(f"{name}: source unchanged since {entry['fetched_at']}")
    else:
        print(f"{name}: SOURCE HAS CHANGED since {entry['fetched_at']}\n"
              f"   archived {entry['bytes']:,} bytes, now {len(response.content):,}")
    return response.content

def get(name):
    return fetch_raw(name) if LIVE else load_raw(name)

print(f"{'entry':<34}{'bytes':>10}{'fetched (UTC)':>22}")
print("-" * 66)
for name, entry in MANIFEST.items():
    print(f"{name:<34}{entry['bytes']:>10,}{entry['fetched_at']:>22}")
print("-" * 66)
print(f"LIVE = {LIVE}: reading from the archive")

entry                                  bytes         fetched (UTC)
------------------------------------------------------------------
worldbank_gdp_per_capita.json         26,071  2026-08-29T12:49:08Z
wikipedia_gdp_per_capita.html        578,990  2026-08-29T12:49:08Z
prices_aapl_msft.csv                  24,329  2026-08-29T12:49:09Z
------------------------------------------------------------------
LIVE = False: reading from the archive

# every archived payload verifies against its recorded hash
for name in MANIFEST:
    raw = load_raw(name)
    print(f"{name:<34} {len(raw):>9,} bytes   sha256 {sha256(raw)[:16]}...  OK")

# and the check is not vacuous: corrupt one byte and it fires
corrupted = bytearray(load_raw("worldbank_gdp_per_capita.json"))
corrupted[100] = (corrupted[100] + 1) % 256
print(f"\nflip a single byte -> sha256 {sha256(bytes(corrupted))[:16]}...")
print(f"matches the manifest: "
      f"{sha256(bytes(corrupted)) == MANIFEST['worldbank_gdp_per_capita.json']['sha256']}")
print("\ncheck passed: the archive is intact, and the verification detects a one-byte change.")

worldbank_gdp_per_capita.json         26,071 bytes   sha256 38f426025a0a8203...  OK
wikipedia_gdp_per_capita.html        578,990 bytes   sha256 bd762211e5485453...  OK
prices_aapl_msft.csv                  24,329 bytes   sha256 bc2013cf1f6bca71...  OK

flip a single byte -> sha256 fbd3dd2f373c1a91...
matches the manifest: False

check passed: the archive is intact, and the verification detects a one-byte change.

raw = get("worldbank_gdp_per_capita.json")
meta, records = json.loads(raw)

print(f"metadata: {meta}")
print(f"records : {len(records)}   (expected {meta['total']})")
assert len(records) == meta["total"], "the API paginated; we only have page 1"
print(f"\none record:\n{json.dumps(records[0], indent=2)[:280]}")

metadata: {'page': 1, 'pages': 1, 'per_page': 500, 'total': 115, 'sourceid': '2', 'lastupdated': '2026-07-13'}
records : 115   (expected 115)

one record:
{
  "indicator": {
    "id": "NY.GDP.PCAP.KD",
    "value": "GDP per capita (constant 2015 US$)"
  },
  "country": {
    "id": "DE",
    "value": "Germany"
  },
  "countryiso3code": "DEU",
  "date": "2022",
  "value": 44817.1316331389,
  "unit": "",
  "obs_status": "",
  "decimal

wb = pd.DataFrame([{"country": r["country"]["value"],
                    "iso3": r["countryiso3code"],
                    "year": int(r["date"]),
                    "gdp_pc": r["value"]} for r in records])

panel = wb.pivot(index="year", columns="country", values="gdp_pc").sort_index()
print(f"panel {panel.shape}: {panel.index.min()}-{panel.index.max()}, "
      f"{panel.shape[1]} countries")
print(panel.tail(3).round(0).to_string())
print(f"\nmissing values: {int(panel.isna().sum().sum())}")

fig, ax = plt.subplots(figsize=(8, 4))
panel.plot(ax=ax, lw=1.8)
ax.set_ylabel("GDP per capita (constant 2015 US$)")
ax.set_xlabel("year"); ax.set_title("World Bank: GDP per capita, from the archived response")
ax.legend(fontsize=8, ncol=2)
plt.tight_layout(); plt.show()

panel (23, 5): 2000-2022, 5 countries
country   France  Germany    Japan  United Kingdom  United States
year                                                             
2020     35709.0  42373.0  35363.0         42985.0        59534.0
2021     38031.0  44011.0  36793.0         46492.0        63097.0
2022     38868.0  44817.0  37448.0         48410.0        64322.0

missing values: 0

import os, requests

key = os.environ.get("FRED_API_KEY")
if key is None:
    raise RuntimeError("set FRED_API_KEY in your environment first")

response = requests.get(
    "https://api.stlouisfed.org/fred/series/observations",
    params={"series_id": "GDPC1", "api_key": key, "file_type": "json"},
    headers=USER_AGENT, timeout=30)
response.raise_for_status()
observations = response.json()["observations"]      # list of dicts, values as STRINGS

fred_like = {"observations": [
    {"realtime_start": "2026-01-01", "date": "2024-01-01", "value": "22679.255"},
    {"realtime_start": "2026-01-01", "date": "2024-04-01", "value": "."},
    {"realtime_start": "2026-01-01", "date": "2024-07-01", "value": "23400.994"}]}

obs = pd.DataFrame(fred_like["observations"])[["date", "value"]]
print("as returned -- note the dtypes:")
print(obs.dtypes.to_string())

obs["date"] = pd.to_datetime(obs["date"])
obs["value"] = pd.to_numeric(obs["value"], errors="coerce")   # '.' becomes NaN
print(f"\nafter conversion:\n{obs.to_string(index=False)}")
print(f"\ndtypes now: {obs.dtypes.to_dict()}")
assert obs['value'].isna().sum() == 1
print("check passed: the '.' placeholder became a proper NaN, not the string '.'.")

as returned -- note the dtypes:
date     object
value    object

after conversion:
      date     value
2024-01-01 22679.255
2024-04-01       NaN
2024-07-01 23400.994

dtypes now: {'date': dtype('<M8[ns]'), 'value': dtype('float64')}
check passed: the '.' placeholder became a proper NaN, not the string '.'.

import yfinance as yf
prices = yf.download(["AAPL", "MSFT"], start="2023-01-01", end="2024-12-31",
                     auto_adjust=True, progress=False)["Close"]

import io

prices = pd.read_csv(io.BytesIO(get("prices_aapl_msft.csv")),
                     index_col=0, parse_dates=True)
returns = np.log(prices).diff().dropna()

print(f"{prices.shape[0]} trading days, {prices.index.min().date()} to {prices.index.max().date()}")
print(f"\n{'ticker':<8}{'ann. return':>13}{'ann. vol':>11}{'Sharpe':>9}")
print("-" * 41)
for ticker in prices.columns:
    mu = 252 * returns[ticker].mean()
    sd = np.sqrt(252) * returns[ticker].std()
    print(f"{ticker:<8}{mu:>12.1%}{sd:>11.1%}{mu / sd:>9.2f}")
print("-" * 41)
print(f"correlation of daily log returns: {returns.corr().iloc[0, 1]:.3f}")

# a sanity check that costs nothing: prices must be positive and dates ordered
assert (prices > 0).all().all() and prices.index.is_monotonic_increasing
print("\ncheck passed: positive prices, dates in order.")

501 trading days, 2023-01-03 to 2024-12-30

ticker    ann. return   ann. vol   Sharpe
-----------------------------------------
AAPL           35.9%      21.3%     1.68
MSFT           29.7%      22.7%     1.31
-----------------------------------------
correlation of daily log returns: 0.506

check passed: positive prices, dates in order.

html = get("wikipedia_gdp_per_capita.html").decode("utf-8")
print(f"archived page: {len(html):,} characters")

tables = pd.read_html(io.StringIO(html))       # StringIO: literal HTML is deprecated
print(f"read_html found {len(tables)} tables")
for i, t in enumerate(tables[:3]):
    print(f"  table {i}: {t.shape}  {list(t.columns)[:2]}")

gdp_table = tables[1].copy()          # .copy(): renaming below must not touch tables[1]
gdp_table.columns = ["country", "imf", "world_bank", "un"]
print(f"\nthe country table as parsed: {gdp_table.shape}")
print(gdp_table.head(3).to_string(index=False))
print("   ... all NaN. The page contains empty <tr> rows that read_html faithfully kept.")

blank = gdp_table.isna().all(axis=1)
gdp_table = gdp_table[~blank].reset_index(drop=True)
print(f"\ndropped {int(blank.sum())} all-empty rows -> {gdp_table.shape}")
print(gdp_table.head(4).to_string(index=False))
assert not gdp_table.isna().all(axis=1).any() and len(gdp_table) > 200

archived page: 574,625 characters
read_html found 6 tables
  table 0: (1, 2)  [0, 1]
  table 1: (228, 4)  ['Country/Territory', 'IMF (2026)[a][5]']
  table 2: (9, 2)  ['vteLists of countries by GDP rankings', 'vteLists of countries by GDP rankings.1']

the country table as parsed: (228, 4)
country imf world_bank  un
    NaN NaN        NaN NaN
    NaN NaN        NaN NaN
    NaN NaN        NaN NaN
   ... all NaN. The page contains empty <tr> rows that read_html faithfully kept.

dropped 5 all-empty rows -> (223, 4)
      country    imf     world_bank     un
       Monaco   —N/a 288,002 (2024) 288002
Liechtenstein 226809 220,167 (2024) 216392
   Luxembourg 158733         147252 138596
      Bermuda   —N/a 142,250 (2024) 136766

from bs4 import BeautifulSoup

soup = BeautifulSoup(html, "lxml")
print(f"page title : {soup.title.string.strip()[:60]}")

# what read_html does for you, done by hand -- and note the guard
table_tag = soup.find("table", {"class": "wikitable"})
rows = table_tag.find_all("tr")
with_cells = [tr for tr in rows if tr.find_all(["td", "th"])]

print(f"\n<tr> elements: {len(rows)}, of which {len(with_cells)} contain any cell")
print(f"the {len(rows) - len(with_cells)} empty ones are why the naive parse above was blank.\n")

for tr in with_cells[1:4]:                     # [0] is the header row
    cells = [td.get_text(strip=True) for td in tr.find_all(["td", "th"])]
    print(f"   {cells[:4]}")

print(f"\nlinks on the page: {len(soup.find_all('a')):,}")

page title : List of countries by GDP (nominal) per capita - Wikipedia

<tr> elements: 229, of which 224 contain any cell
the 5 empty ones are why the naive parse above was blank.

   ['Monaco', '—N/a', '288,002 (2024)', '288,002']
   ['Liechtenstein', '226,809', '220,167 (2024)', '216,392']
   ['Luxembourg', '158,733', '147,252', '138,596']

links on the page: 795

population = pd.DataFrame({
    "country": ["France", "Germany", "Japan", "United Kingdom", "United States"],
    "population_m": [68.0, 84.4, 124.5, 67.6, 334.9],
    "region": ["Europe", "Europe", "Asia", "Europe", "Americas"],
})

latest = (wb[wb["year"] == wb["year"].max()][["country", "gdp_pc"]]
          .merge(population, on="country", how="left", validate="one_to_one"))
assert latest["population_m"].notna().all(), "a country failed to match"

latest["gdp_total_bn"] = latest["gdp_pc"] * latest["population_m"] / 1000
print(f"year {wb['year'].max()}:\n")
print(latest.sort_values("gdp_pc", ascending=False).round(1).to_string(index=False))

year 2022:

       country  gdp_pc  population_m   region  gdp_total_bn
 United States 64322.2         334.9 Americas       21541.5
United Kingdom 48410.1          67.6   Europe        3272.5
       Germany 44817.1          84.4   Europe        3782.6
        France 38868.3          68.0   Europe        2643.0
         Japan 37448.4         124.5     Asia        4662.3

# growth over the sample, from the archived panel
growth = (panel.iloc[-1] / panel.iloc[0]) ** (1 / (panel.index[-1] - panel.index[0])) - 1

fig, axes = plt.subplots(1, 2, figsize=(12, 4))
normalized = panel / panel.iloc[0] * 100
normalized.plot(ax=axes[0], lw=1.8)
axes[0].set_ylabel(f"index, {panel.index[0]} = 100"); axes[0].set_xlabel("year")
axes[0].set_title("Real GDP per capita, indexed"); axes[0].legend(fontsize=8)

axes[1].barh(growth.sort_values().index, growth.sort_values().values * 100)
axes[1].set_xlabel("average annual growth, %")
axes[1].set_title(f"{panel.index[0]}-{panel.index[-1]} annualized")
plt.tight_layout(); plt.show()

print(growth.sort_values(ascending=False).mul(100).round(2).to_string())

country
United States     1.28
Germany           1.12
United Kingdom    0.95
Japan             0.75
France            0.71

# the drift check. With LIVE = False this reports what WOULD be compared;
# set LIVE = True at the top and re-run to actually query the sources.
print(f"LIVE = {LIVE}\n")
print(f"{'source':<34}{'archived':>12}{'fetched (UTC)':>22}")
print("-" * 68)
for name, entry in MANIFEST.items():
    age_days = (datetime.now(timezone.utc)
                - datetime.strptime(entry["fetched_at"], "%Y-%m-%dT%H:%M:%SZ")
                .replace(tzinfo=timezone.utc)).days
    print(f"{name:<34}{entry['bytes']:>12,}{entry['fetched_at']:>22}   ({age_days}d old)")
print("-" * 68)

if LIVE:
    for name in MANIFEST:
        fetch_raw(name)
else:
    print("archive in use; set LIVE = True to re-fetch and compare hashes.")

LIVE = False

source                                archived         fetched (UTC)
--------------------------------------------------------------------
worldbank_gdp_per_capita.json           26,071  2026-08-29T12:49:08Z   (0d old)
wikipedia_gdp_per_capita.html          578,990  2026-08-29T12:49:08Z   (0d old)
prices_aapl_msft.csv                    24,329  2026-08-29T12:49:09Z   (0d old)
--------------------------------------------------------------------
archive in use; set LIVE = True to re-fetch and compare hashes.

import platform
import subprocess
import sys
import time
import importlib.metadata as md

OUT_DIR = Path("generated")          # disposable byproducts, as in fd06
OUT_DIR.mkdir(exist_ok=True)

def git_commit():
    try:
        out = subprocess.run(["git", "rev-parse", "--short", "HEAD"],
                             capture_output=True, text=True, timeout=10)
        return out.stdout.strip() if out.returncode == 0 else "(not a git repository)"
    except (FileNotFoundError, subprocess.TimeoutExpired):
        return "(git unavailable)"

def version_of(dist):
    try:
        return md.version(dist)
    except md.PackageNotFoundError:
        return "not installed"

print("=" * 52)
print("REPRODUCIBILITY HEADER")
print("=" * 52)
print(f"{'run at (UTC)':<16}{time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime())}")
print(f"{'python':<16}{sys.version.split()[0]}")
print(f"{'platform':<16}{platform.system()} {platform.release()} ({platform.machine()})")
for dist in ("numpy", "scipy", "pandas", "matplotlib", "scikit-learn", "numba"):
    print(f"{dist:<16}{version_of(dist)}")
print(f"{'git commit':<16}{git_commit()}")
print("=" * 52)

====================================================
REPRODUCIBILITY HEADER
====================================================
run at (UTC)    2026-08-29 16:28:58
python          3.12.3
platform        Windows 11 (ARM64)
numpy           1.26.4
scipy           1.16.0
pandas          2.2.3
matplotlib      3.9.2
scikit-learn    1.5.1
numba           0.60.0
git commit      (not a git repository)
====================================================

import Mathlib.Data.Real.Basic
import Mathlib.Algebra.Order.Field.Basic
import Mathlib.Order.Interval.Set.Basic
import Mathlib.Tactic.Linarith

/-- Two-period consumer with log utility: optimal saving is
    s(β) = β·W/(1+β).  Comparative statics, machine-checked: if wealth
    W > 0, saving is strictly increasing in the discount factor β. -/
theorem saving_strictMono {W : ℝ} (hW : 0 < W) :
    StrictMonoOn (fun β : ℝ => β * W / (1 + β)) (Set.Ioi 0) := by
  intro a ha b hb hab
  have ha1 : (0 : ℝ) < 1 + a := by have := Set.mem_Ioi.mp ha; linarith
  have hb1 : (0 : ℝ) < 1 + b := by have := Set.mem_Ioi.mp hb; linarith
  rw [div_lt_div_iff₀ ha1 hb1]
  nlinarith [mul_pos (sub_pos.mpr hab) hW]

# your answer here

# your answer here

# your answer here

# your answer here

# your answer here

original = load_raw("worldbank_gdp_per_capita.json")
meta_o, records_o = json.loads(original)

revised_records = [dict(r) for r in records_o]
target = next(i for i, r in enumerate(revised_records)
              if r["countryiso3code"] == "FRA" and r["date"] == "2020")
old_value = revised_records[target]["value"]
revised_records[target]["value"] = old_value * 1.001          # a 0.1% revision

revised = json.dumps([meta_o, revised_records]).encode("utf-8")

print(f"archived sha256 : {sha256(original)[:32]}")
print(f"revised  sha256 : {sha256(revised)[:32]}")
print(f"identical       : {sha256(original) == sha256(revised)}")
print(f"size changed by : {len(revised) - len(original):+,} bytes "
      f"(re-serialization alone changes the bytes)")

archived sha256 : 38f426025a0a8203c19b9eb412dcd662
revised  sha256 : 43da56eeb7a8f33e5f5395e30666748c
identical       : False
size changed by : +2,542 bytes (re-serialization alone changes the bytes)

def describe_change(old_bytes, new_bytes):
    """Report which observations differ between two World Bank payloads."""
    to_frame = lambda b: (pd.DataFrame([{"iso3": r["countryiso3code"], "year": r["date"],
                                         "value": r["value"]}
                                        for r in json.loads(b)[1]])
                          .set_index(["iso3", "year"])["value"])
    old, new = to_frame(old_bytes), to_frame(new_bytes)

    added = new.index.difference(old.index)
    removed = old.index.difference(new.index)
    common = old.index.intersection(new.index)
    differs = common[~np.isclose(old[common].astype(float),
                                 new[common].astype(float),
                                 rtol=0, atol=0, equal_nan=True)]

    return pd.DataFrame({"old": old[differs], "new": new[differs]}).assign(
        pct=lambda d: (d["new"] / d["old"] - 1) * 100), added, removed

diff, added, removed = describe_change(original, revised)
print(f"observations added {len(added)}, removed {len(removed)}, changed {len(diff)}\n")
print(diff.round(4).to_string())

observations added 0, removed 0, changed 1

                  old         new  pct
iso3 year                             
FRA  2020  35709.2046  35744.9138  0.1

def find_country_table(html, marker="United States", min_rows=100):
    """Locate the country table by CONTENT, not by position."""
    candidates = pd.read_html(io.StringIO(html))
    for i, table in enumerate(candidates):
        if len(table) < min_rows:
            continue
        first_column = table.iloc[:, 0].astype(str)
        if first_column.str.contains(marker, case=False, na=False).any():
            return table, i
    raise LookupError(
        f"no table with >= {min_rows} rows whose first column mentions {marker!r}; "
        f"the page structure has changed and the parse must be revisited")

found, position = find_country_table(html)
print(f"found the table at index {position}, shape {found.shape}")
print(f"same table as tables[1]: {found.equals(tables[1])}")
assert position == 1 and found.equals(tables[1])

# and it fails loudly when nothing matches
try:
    find_country_table(html, marker="Atlantis")
except LookupError as err:
    print(f"\nno match -> LookupError: {str(err)[:76]}...")

found the table at index 1, shape (228, 4)
same table as tables[1]: True

no match -> LookupError: no table with >= 100 rows whose first column mentions 'Atlantis'; the page s...

def audit(folder):
    """Check a directory against the section 10 checklist."""
    folder = Path(folder)
    checks = {}

    checks["environment file"] = any((folder / n).exists() for n in
                                     ("environment.yml", "requirements.txt", "pyproject.toml"))

    git = subprocess.run(["git", "-C", str(folder), "status", "--porcelain"],
                         capture_output=True, text=True)
    checks["under version control"] = git.returncode == 0
    checks["working tree clean"] = git.returncode == 0 and git.stdout.strip() == ""

    checks["data manifest"] = any(folder.rglob("manifest.json"))
    checks["has tests"] = any(folder.rglob("test_*.py"))

    notebooks = list(folder.glob("*.ipynb"))
    seeded = [nb for nb in notebooks
              if "default_rng(" in nb.read_text(encoding="utf-8", errors="ignore")]
    checks["notebooks seed explicitly"] = bool(notebooks) and len(seeded) == len(notebooks)

    return checks, len(notebooks), len(seeded)

report, n_nb, n_seeded = audit(".")
print(f"audit of {Path('.').resolve().name}/\n")
for name, ok in report.items():
    print(f"  [{'PASS' if ok else 'FAIL'}]  {name}")
print(f"\n({n_seeded} of {n_nb} notebooks call default_rng)")
print(f"score: {sum(report.values())} of {len(report)}")

audit of ___research-tools/

  [FAIL]  environment file
  [FAIL]  under version control
  [FAIL]  working tree clean
  [PASS]  data manifest
  [PASS]  has tests
  [FAIL]  notebooks seed explicitly

(6 of 11 notebooks call default_rng)
score: 2 of 6

%%writefile generated/test_saving.py
"""The pytest analogue of the Lean theorem in section 11."""
import pytest

BETAS = [0.1, 0.5, 0.9, 0.95, 0.99, 1.5, 3.0]
WEALTHS = [1.0, 10.0, 100.0]


def saving(beta, W):
    return beta * W / (1 + beta)


@pytest.mark.parametrize("W", WEALTHS)
def test_saving_strictly_increasing_in_beta(W):
    """s(beta) is strictly increasing on the sampled grid."""
    values = [saving(b, W) for b in sorted(BETAS)]
    assert all(lo < hi for lo, hi in zip(values, values[1:]))

Overwriting generated/test_saving.py

result = subprocess.run(
    [sys.executable, "-m", "pytest", "test_saving.py", "-q",
     "-p", "no:cacheprovider"],
    capture_output=True, text=True, cwd=str(OUT_DIR))
print(result.stdout[-400:])
assert result.returncode == 0

import math

# now a function that PASSES the same grid and is not monotone
BETAS = [0.1, 0.5, 0.9, 0.95, 0.99, 1.5, 3.0]      # the same grid the test uses

def impostor(beta, W):
    return beta * W / (1 + beta) + 0.01 * W * math.sin(2 * math.pi * beta / 0.4) ** 2 * (
        1 if 0.5 < beta < 0.9 else 0)

grid = sorted(BETAS)
on_grid = [impostor(b, 1.0) for b in grid]
print(f"\nimpostor on the grid, increasing: "
      f"{all(lo < hi for lo, hi in zip(on_grid, on_grid[1:]))}")

fine = [i / 1000 for i in range(1, 2000)]
fine_values = [impostor(b, 1.0) for b in fine]
violations = [(a, b) for (a, va), (b, vb) in zip(zip(fine, fine_values),
                                                 zip(fine[1:], fine_values[1:])) if vb <= va]
print(f"on a fine grid, monotonicity fails at {len(violations)} places, "
      f"first near beta = {violations[0][0]:.3f}" if violations else "no violation found")

...                                                                      [100%]
3 passed in 0.01s


impostor on the grid, increasing: True
on a fine grid, monotonicity fails at 1 places, first near beta = 0.899

selector	what it finds
`soup.find_all("a")`	every link
`soup.find("div", {"class": "summary"})`	the first `<div class="summary">`
`soup.select(".price-tag > span")`	a CSS selector path
`tag.get("href")`	an attribute
`tag.parent`, `tag.next_sibling`	tree navigation

	practice	argued in
1	Pin the environment. `conda env export --from-history > environment.yml`, so a reader recreates the versions you used.	`fd01` §2.2, `fd03` §8; and `fd06`'s `FutureWarning` is the concrete case
2	Version-control everything: code, environment file, README, figure scripts, and the data manifest.	`fd05`
3	Seed every generator, and pass it explicitly. `rng = np.random.default_rng(seed)`, never the legacy global.	`fd08` §16, ahead
4	Seed the incidental randomness too: cross-validation folds, train/test splits, initializations.	`fd11` Ex 4, ahead, where $\lambda_{\min}$ moves with the fold seed
5	Archive acquired data and record its hash.	§3 above
6	Test the code, so "this reproduces the result" is checkable rather than asserted.	`fd04` §5
7	Record the state of the run, versions, platform, commit, date, in the output itself.	below

reproducible computation (§8)	verified mathematics
pinned `environment.yml`	pinned toolchain (`lean-toolchain`, `lake-manifest.json`)
seeded generators, deterministic ops	no hidden assumptions: `#print axioms` lists everything a theorem relies on
a `pytest` suite the machine re-runs (`fd04` §5)	theorem statements the kernel re-checks on every build
clone → `conda env create` → re-run	clone → `lake build` → every proof re-verified
a reviewer eyeballs the code	the kernel checks every inference step

file	purpose
`acquire.py`	the acquisition script: every URL, parameter and header, so the fetch is exactly reproducible
`data_cache/manifest.json`	URL, UTC timestamp, byte count and SHA-256 for each response: the evidence of what you used
`environment.yml`	pinned package versions, so the parse behaves as it did (`fd06`'s `FutureWarning` is the argument)
`analysis.ipynb`	the computation, reading only from `data_cache/`
`derived/`	aggregates you are permitted to publish: group means, coefficients, the figures
`README.md`	how to obtain the data, in what order to run things, and the licence position
`.gitignore`	excludes `data_cache/*.gz` when the licence forbids redistributing raw data

Data acquisition and reproducible research¶

Alfred Galichon (NYU)¶

'math+econ+code' masterclass series: fundamentals of research in python¶

With python code examples¶

Learning objectives¶

References¶

1. Web data and reproducibility¶

2. A brief overview of HTTP and JSON¶

3. Archive and verify¶

4. A keyless API: the World Bank¶

5. APIs with keys: FRED and credential management¶

6. Library wrappers: `yfinance`¶

7. HTML scraping¶

8. Responsible acquisition¶

Three permissions, not one¶

9. Worked example: a country panel, and detecting drift¶

10. Reproducibility, assembled¶

11. Verifying the mathematics¶

12. Summary¶

13. Exercises¶

14. Further directions¶

15. Solutions to the exercises¶

Solution to Exercise 1: Detect a change¶

Solution to Exercise 2: Parse the fragile thing¶

Solution to Exercise 3: A replication package you may ship¶

Solution to Exercise 4: Audit a project¶

Solution to Exercise 5: A theorem is not a test grid¶

Data acquisition and reproducible research¶

Alfred Galichon (NYU)¶

'math+econ+code' masterclass series: fundamentals of research in python¶

With python code examples¶

Learning objectives¶

References¶

1. Web data and reproducibility¶

2. A brief overview of HTTP and JSON¶

3. Archive and verify¶

4. A keyless API: the World Bank¶

5. APIs with keys: FRED and credential management¶

6. Library wrappers: yfinance¶

7. HTML scraping¶

8. Responsible acquisition¶

Three permissions, not one¶

9. Worked example: a country panel, and detecting drift¶

10. Reproducibility, assembled¶

11. Verifying the mathematics¶

12. Summary¶

13. Exercises¶

14. Further directions¶

15. Solutions to the exercises¶

Solution to Exercise 1: Detect a change¶

Solution to Exercise 2: Parse the fragile thing¶

Solution to Exercise 3: A replication package you may ship¶

Solution to Exercise 4: Audit a project¶

Solution to Exercise 5: A theorem is not a test grid¶

6. Library wrappers: `yfinance`¶