#!/usr/bin/env python3
"""Construit web_croissance/data/*.json à partir des CSV bruts sourcés de data_src/.
   Reproductible : aucune valeur codée en dur, tout vient des CSV Eurostat.
   Usage : python3 build/build.py   (depuis web_croissance/)
"""
import csv, json, math, os

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
SRC  = os.path.join(ROOT, "data_src")
OUT  = os.path.join(ROOT, "data")
os.makedirs(OUT, exist_ok=True)

AGG = {"EU27_2020", "EA20"}
# Zone euro au 11/07/2026 (Bulgarie adopte l'euro le 01/01/2026).
EA = {"AT","BE","HR","CY","EE","FI","FR","DE","EL","IE","IT","LV","LT","LU","MT","NL","PT","SK","SI","ES","BG"}
# Correspondances ISO3 (Banque mondiale) pour repérer UE / zone euro dans les jeux mondiaux
EU_ISO3 = {"AUT","BEL","BGR","HRV","CYP","CZE","DNK","EST","FIN","FRA","DEU","GRC","HUN","IRL","ITA","LVA","LTU","LUX","MLT","NLD","POL","PRT","ROU","SVK","SVN","ESP","SWE"}
EA_ISO3 = {"AUT","BEL","HRV","CYP","EST","FIN","FRA","DEU","GRC","IRL","ITA","LVA","LTU","LUX","MLT","NLD","PRT","SVK","SVN","ESP","BGR"}
# Économies « atypiques » : PIB/hab non représentatif de l'économie réelle
# — soit transferts financiers (bénéfices de multinationales, place offshore),
# — soit rente de ressources (pétrole/gaz) constituant l'essentiel du PIB.
# Masquées par défaut sur le site (réversible), pour se concentrer sur les économies réelles.
OUTLIER_REASON = {  # pays UE (nama_10_pc)
    "IE": "PIB gonflé par les bénéfices des multinationales (transferts de propriété intellectuelle) ; le RNB est très inférieur",
    "LU": "PIB/hab surestimé : ~45 % de travailleurs frontaliers comptés au numérateur mais pas au dénominateur",
}
OUTLIERS = set(OUTLIER_REASON)
# Monde (codes ISO3, Banque mondiale) : rentiers de ressources + places offshore
WORLD_OUTLIER_REASON = {
    "IRL": "transferts financiers / bénéfices des multinationales",
    "LUX": "place financière + travailleurs frontaliers",
    "QAT": "rente gazière", "KWT": "rente pétrolière", "ARE": "rente pétrolière",
    "BRN": "rente pétrolière et gazière", "SAU": "rente pétrolière", "OMN": "rente pétrolière",
    "BHR": "rente pétrolière et finance", "LBY": "rente pétrolière", "TKM": "rente gazière",
    "GNQ": "rente pétrolière", "TTO": "rente gazière", "GAB": "rente pétrolière", "IRQ": "rente pétrolière",
    "GUY": "boom pétrolier récent — PIB/hab explosif et non représentatif",
    "MAC": "rente des jeux (casinos)", "BMU": "place financière offshore",
}
NO_MW = {"DK","IT","AT","FI","SE"}
NAMES = {"BE":"Belgique","BG":"Bulgarie","CZ":"Tchéquie","DK":"Danemark","DE":"Allemagne","EE":"Estonie",
 "IE":"Irlande","EL":"Grèce","ES":"Espagne","FR":"France","HR":"Croatie","IT":"Italie","CY":"Chypre",
 "LV":"Lettonie","LT":"Lituanie","LU":"Luxembourg","HU":"Hongrie","MT":"Malte","NL":"Pays-Bas",
 "AT":"Autriche","PL":"Pologne","PT":"Portugal","RO":"Roumanie","SI":"Slovénie","SK":"Slovaquie",
 "FI":"Finlande","SE":"Suède"}

def rd(name):
    with open(os.path.join(SRC, name)) as f:
        return list(csv.DictReader(f))

# ---- PIB/hab 2025 ----
gdp_eur, gdp_pps = {}, {}
for r in rd("gdp_pc_nama_10_pc_2024_2025.csv"):
    if r["time"] != "2025" or r["geo"] in AGG or r["value"] == "": continue
    v = float(r["value"])
    if r["unit"] == "CP_EUR_HAB": gdp_eur[r["geo"]] = v
    elif r["unit"] == "CP_PPS_EU27_2020_HAB": gdp_pps[r["geo"]] = v

# ---- Salaire minimum € / PPS 2026-S1 ----
def load_mw(name):
    d = {}
    for r in rd(name):
        if r["value"] == "": continue
        d[r["geo"]] = float(r["value"])
    return d
mw_eur = load_mw("minwage_earn_mw_cur_2026S1_eur.csv")
mw_pps = load_mw("minwage_earn_mw_pps_2026S1.csv")

# ---- SES distribution 2022 (D1, médiane, moyenne, D9) ----
ses = {}
for r in rd("earn_ses_monthly_2022_total.csv"):
    if r["value"] == "": continue
    ses.setdefault(r["geo"], {})[r["indic_se"]] = float(r["value"])

# ---- SES par secteur détaillé (médiane horaire PPS, 17 sections NACE, 2022) ----
NACE_LABELS = {
 "C":"Industrie manufacturière","D":"Énergie","E":"Eau & déchets","F":"Construction",
 "G":"Commerce","H":"Transports & entreposage","I":"Hébergement & restauration",
 "J":"Information & communication","K":"Finance & assurance","L":"Immobilier",
 "M":"Activités scientifiques & techniques","N":"Services administratifs",
 "P":"Éducation","Q":"Santé & action sociale","R":"Arts & loisirs","S":"Autres services"}
sect = {}
for r in rd("earn_ses_pub2n_2022_nace_pps.csv"):
    if r["value"] == "" or r["nace_r2"] == "B-S_X_O": continue
    sect.setdefault(r["geo"], {})[r["nace_r2"]] = float(r["value"])

# ---- Pauvreté SMSD 2025 ----
smsd = {}
for r in rd("poverty_ilc_mdsd07_2025.csv"):
    if r["value"] == "": continue
    smsd[r["geo"]] = float(r["value"])

# ---- corrélation ----
def pearson(xs, ys):
    n = len(xs); mx = sum(xs)/n; my = sum(ys)/n
    sxy = sum((x-mx)*(y-my) for x, y in zip(xs, ys))
    sxx = sum((x-mx)**2 for x in xs); syy = sum((y-my)**2 for y in ys)
    if sxx == 0 or syy == 0: return None
    r = sxy/math.sqrt(sxx*syy)
    t = r*math.sqrt((n-2)/max(1e-12, 1-r*r))
    return dict(n=n, r=round(r,4), r2=round(r*r,4), slope=sxy/sxx, t=round(t,2))

def _lin(X, Y):
    n=len(X); mx=sum(X)/n; my=sum(Y)/n
    sxy=sum((X[i]-mx)*(Y[i]-my) for i in range(n)); sxx=sum((X[i]-mx)**2 for i in range(n))
    b=sxy/sxx; return my-b*mx, b

def best_fit(xs, ys):
    """4 formes (linéaire, log-linéaire, exponentielle, log-log) ; R² sur échelle d'origine
       (comparable) ; parcimonie en cas d'égalité (<0,01). Retourne (forme, r2)."""
    n=len(xs); my=sum(ys)/n; sstot=sum((y-my)**2 for y in ys)
    if sstot==0: return ("linéaire",0.0)
    xpos=all(x>0 for x in xs); ypos=all(y>0 for y in ys); sc=[]
    a,b=_lin(xs,ys); sc.append(("linéaire",0,lambda x,a=a,b=b:a+b*x))
    if xpos: a,b=_lin([math.log(x) for x in xs],ys); sc.append(("log-linéaire",1,lambda x,a=a,b=b:a+b*math.log(x)))
    if ypos: a,b=_lin(xs,[math.log(y) for y in ys]); sc.append(("exponentiel",1,lambda x,a=a,b=b:math.exp(a+b*x)))
    if xpos and ypos: a,b=_lin([math.log(x) for x in xs],[math.log(y) for y in ys]); sc.append(("log-log",2,lambda x,a=a,b=b:math.exp(a+b*math.log(x))))
    scored=[(nm,cx,1-sum((ys[i]-pr(xs[i]))**2 for i in range(n))/sstot) for nm,cx,pr in sc]
    best=max(s[2] for s in scored)
    near=sorted([s for s in scored if best-s[2]<=0.01], key=lambda s:(s[1],-s[2]))
    return near[0][0], round(near[0][2],4)

# ================= salaires.json =================
countries = []
for g in sorted(gdp_eur):
    s = ses.get(g, {})
    rec = {
        "geo": g, "name": NAMES.get(g, g),
        "gdp_eur": gdp_eur.get(g), "gdp_pps": gdp_pps.get(g),
        "ea": g in EA, "out": g in OUTLIERS, "out_reason": OUTLIER_REASON.get(g),
        "wages": {
            "mw":   {"eur": mw_eur.get(g), "pps": mw_pps.get(g)},
            "d1":   {"eur": s.get("D1_E_EUR"), "pps": s.get("D1_E_PPS")},
            "med":  {"eur": s.get("MED_E_EUR"), "pps": s.get("MED_E_PPS")},
            "mean": {"eur": s.get("MEAN_E_EUR"), "pps": s.get("MEAN_E_PPS")},
            "d9":   {"eur": s.get("D9_E_EUR"), "pps": s.get("D9_E_PPS")},
        },
    }
    countries.append(rec)

salaires = {
    "meta": {
        "title": "PIB par habitant vs niveau de salaire — UE",
        "gdp_year": "2025", "mw_period": "2026-S1", "ses_wave": "2022",
        "source": "Eurostat nama_10_pc, earn_mw_cur, earn_ses_monthly",
        "ea_note": "Zone euro au 01/01/2026 (Bulgarie incluse).",
        "outliers": sorted(OUTLIERS), "no_min_wage": sorted(NO_MW),
    },
    "countries": countries,
}
json.dump(salaires, open(os.path.join(OUT, "salaires.json"), "w"), ensure_ascii=False, indent=1)

# ================= secteurs.json =================
SECTOR_CODES = [c for c in ["C","D","E","F","G","H","I","J","K","L","M","N","P","Q","R","S"]]
sect_rows = []
for g in sorted(gdp_pps):
    d = sect.get(g, {})
    row = {"geo": g, "name": NAMES.get(g, g), "gdp_pps": gdp_pps.get(g), "out": g in OUTLIERS, "out_reason": OUTLIER_REASON.get(g)}
    for code in SECTOR_CODES:
        row[code] = d.get(code)
    sect_rows.append(row)
secteurs = {
    "meta": {"title": "PIB/hab vs salaire médian horaire par secteur (NACE)", "gdp_year": "2025",
             "ses_wave": "2022", "unit": "PPS", "wage_type": "médiane horaire",
             "sectors": [{"code": c, "label": NACE_LABELS[c]} for c in SECTOR_CODES],
             "source": "Eurostat nama_10_pc, earn_ses_pub2n"},
    "countries": sect_rows,
}
json.dump(secteurs, open(os.path.join(OUT, "secteurs.json"), "w"), ensure_ascii=False, indent=1)

# ================= pauvrete.json =================
pov_rows = []
for g in sorted(gdp_pps):
    if g not in smsd: continue
    pov_rows.append({"geo": g, "name": NAMES.get(g, g),
                     "gdp_pps": gdp_pps.get(g), "gdp_eur": gdp_eur.get(g),
                     "smsd": smsd[g], "out": g in OUTLIERS, "out_reason": OUTLIER_REASON.get(g)})
pauvrete = {
    "meta": {"title": "PIB/hab vs privation matérielle et sociale sévère (UE)",
             "gdp_year": "2025", "smsd_year": "2025", "smsd_unit": "% population",
             "source": "Eurostat nama_10_pc, ilc_mdsd07"},
    "countries": pov_rows,
}
json.dump(pauvrete, open(os.path.join(OUT, "pauvrete.json"), "w"), ensure_ascii=False, indent=1)

# ================= dimensions.json (batterie d'indicateurs UE) =================
def rd_simple(name):
    d = {}
    for r in rd(name):
        if r["value"] == "": continue
        d[r["geo"]] = float(r["value"])
    return d
ISO3 = {"AT":"AUT","BE":"BEL","BG":"BGR","CY":"CYP","CZ":"CZE","DE":"DEU","DK":"DNK","EE":"EST","EL":"GRC","ES":"ESP","FI":"FIN","FR":"FRA","HR":"HRV","HU":"HUN","IE":"IRL","IT":"ITA","LT":"LTU","LU":"LUX","LV":"LVA","MT":"MLT","NL":"NLD","PL":"POL","PT":"PRT","RO":"ROU","SE":"SWE","SI":"SVN","SK":"SVK"}
d_medinc = rd_simple("eu_median_income_pps_2025.csv")
d_hly    = rd_simple("wb_dim_hly_2024.csv")
d_arope  = rd_simple("eu_arope_2025.csv")
d_emp    = rd_simple("eu_employment_2025.csv")
wb_le_eu = {}
for r in rd("wb_life_expectancy_2024.csv"):
    if r["value"] != "": wb_le_eu[r["geo"]] = float(r["value"])   # ISO3
def le_of(g): return wb_le_eu.get(ISO3.get(g, ""))
# good = +1 si "plus haut = mieux", -1 sinon
DIMS = [
    {"key":"med_inc","label":"Revenu médian (équiv., PPS/an)","unit":"PPS","good":1},
    {"key":"life_exp","label":"Espérance de vie (ans)","unit":"ans","good":1},
    {"key":"emp","label":"Taux d'emploi 20-64 (%)","unit":"%","good":1},
    {"key":"hly","label":"Années de vie en bonne santé","unit":"ans","good":1},
    {"key":"smsd","label":"Privation matérielle sévère (%)","unit":"%","good":-1},
]
dim_rows = []
for g in sorted(gdp_pps):
    dim_rows.append({"geo":g,"name":NAMES.get(g,g),"gdp_pps":gdp_pps.get(g),
        "ea":g in EA,"out":g in OUTLIERS,"out_reason":OUTLIER_REASON.get(g),
        "med_inc":d_medinc.get(g),"life_exp":le_of(g),"emp":d_emp.get(g),
        "hly":d_hly.get(g),"smsd":smsd.get(g)})
dimensions = {
    "meta":{"title":"PIB par habitant vs dimensions du bien-être — UE","gdp_year":"2025",
            "dims":DIMS,"source":"Eurostat (ilc_di03, hlth_hlye, lfsi_emp_a, ilc_peps01n, ilc_mdsd07) + Banque mondiale (espérance de vie)"},
    "countries":dim_rows,
}
json.dump(dimensions, open(os.path.join(OUT, "dimensions.json"), "w"), ensure_ascii=False, indent=1)

# ================= regions.json (NUTS2) =================
def rd_nuts(name):
    d = {}
    for r in rd(name):
        if r["value"] == "" or len(r["geo"]) != 4: continue
        d[r["geo"]] = (r["geo_label"], float(r["value"]))
    return d
r_gdp = rd_nuts("nuts2_gdp_pps_2022.csv")
r_le  = rd_nuts("nuts2_life_expectancy_2023.csv")
COUNTRY_FR = {"AT":"Autriche","BE":"Belgique","BG":"Bulgarie","CY":"Chypre","CZ":"Tchéquie","DE":"Allemagne","DK":"Danemark","EE":"Estonie","EL":"Grèce","ES":"Espagne","FI":"Finlande","FR":"France","HR":"Croatie","HU":"Hongrie","IE":"Irlande","IT":"Italie","LT":"Lituanie","LU":"Luxembourg","LV":"Lettonie","MT":"Malte","NL":"Pays-Bas","PL":"Pologne","PT":"Portugal","RO":"Roumanie","SE":"Suède","SI":"Slovénie","SK":"Slovaquie"}
EU_CC = set(COUNTRY_FR)
reg_rows = []
for g in sorted(set(r_gdp) & set(r_le)):
    cc = g[:2]
    if cc not in EU_CC: continue
    reg_rows.append({"geo":g,"name":r_le[g][0],"cc":cc,"country":COUNTRY_FR.get(cc,cc),
        "gdp":round(r_gdp[g][1]),"le":round(r_le[g][1],1),"ea":cc in EA})
regions = {
    "meta":{"title":"PIB/hab régional (NUTS2) vs espérance de vie — UE","gdp_year":"2022","le_year":"2023",
            "unit_gdp":"PPS","source":"Eurostat nama_10r_2gdp, demo_r_mlifexp"},
    "regions":reg_rows,
}
json.dump(regions, open(os.path.join(OUT, "regions.json"), "w"), ensure_ascii=False, indent=1)

# ================= vérification : R² imprimés =================
def spec(label, pairs, log=False):
    pairs = [(x, y) for x, y in pairs if x is not None and y is not None]
    xs = [p[0] for p in pairs]; ys = [p[1] for p in pairs]
    if log: xs = [math.log(x) for x in xs]; ys = [math.log(y) for y in ys]
    st = pearson(xs, ys)
    bf = best_fit(xs, ys) if len(xs) > 2 else ("—", 0)
    if st: print(f"  {label:44s} n={st['n']:2d}  r_lin={st['r']:+.3f}  →  meilleure forme: {bf[0]:13s} R²={bf[1]:.3f}")
    return st

print("=== SALAIRES : PIB/hab (2025) vs niveau de salaire ===")
for meas, mlabel in [("mw","salaire min."),("d1","1er décile"),("med","médiane"),("mean","moyenne"),("d9","9e décile")]:
    for unit in ("eur","pps"):
        gdp = gdp_eur if unit == "eur" else gdp_pps
        pairs = [(gdp.get(c["geo"]), c["wages"][meas][unit]) for c in countries]
        spec(f"{mlabel} · {unit.upper()} · lin · tous", pairs)
    # sans outliers, en PPS
    pairs = [(gdp_pps.get(c["geo"]), c["wages"][meas]["pps"]) for c in countries if not c["out"]]
    spec(f"{mlabel} · PPS · lin · sans IE+LU", pairs)

print("=== SECTEURS : PIB/hab PPS vs médiane horaire PPS par secteur NACE (2022, sans IE+LU) ===")
for code in SECTOR_CODES:
    pairs = [(r["gdp_pps"], r[code]) for r in sect_rows if not r["out"]]
    spec(f"{NACE_LABELS[code]} ({code})", pairs)

print("=== PAUVRETE : PIB/hab PPS vs privation sévère SMSD (2025) ===")
spec("SMSD vs PIB/hab PPS · linéaire", [(r["gdp_pps"], r["smsd"]) for r in pov_rows])
spec("SMSD vs PIB/hab PPS · log-log", [(r["gdp_pps"], r["smsd"]) for r in pov_rows], log=True)
spec("SMSD vs PIB/hab PPS · lin · sans IE+LU", [(r["gdp_pps"], r["smsd"]) for r in pov_rows if not r["out"]])

# ================= bienetre.json (courbe de Preston, mondial, World Bank) =================
def rd_wb(name):
    d = {}
    for r in rd(name):
        if r["value"] == "": continue
        d[r["geo"]] = {"name": r["geo_label"], "v": float(r["value"])}
    return d
wb_gdp = rd_wb("wb_gdp_pcap_ppp_2024.csv")
wb_le  = rd_wb("wb_life_expectancy_2024.csv")
bien_rows = []
for g in sorted(set(wb_gdp) & set(wb_le)):
    bien_rows.append({"geo": g, "name": wb_le[g]["name"],
                      "gdp": round(wb_gdp[g]["v"], 1), "le": round(wb_le[g]["v"], 2),
                      "hi": wb_gdp[g]["v"] >= 45000,   # "pays développés" ≈ PIB/hab PPA ≥ 45 000 $
                      "eu": g in EU_ISO3, "ea": g in EA_ISO3,
                      "out": g in WORLD_OUTLIER_REASON, "out_reason": WORLD_OUTLIER_REASON.get(g)})
bienetre = {
    "meta": {"title": "PIB par habitant (PPA) vs espérance de vie — mondial",
             "year": "2024", "gdp_unit": "$ internationaux PPA", "hi_threshold": 45000,
             "source": "Banque mondiale : NY.GDP.PCAP.PP.CD, SP.DYN.LE00.IN"},
    "countries": bien_rows,
}
json.dump(bienetre, open(os.path.join(OUT, "bienetre.json"), "w"), ensure_ascii=False, indent=1)

# ================= pauvrete_monde.json (analyse mondiale façon OWID) =================
pov685 = {}
for r in rd("wb_poverty_685_latest.csv"):
    if r["value"] == "": continue
    pov685[r["geo"]] = (float(r["value"]), r["time"])
povm_rows = []
for g in sorted(set(wb_gdp) & set(pov685)):
    povm_rows.append({"geo": g, "name": wb_gdp[g]["name"], "gdp": round(wb_gdp[g]["v"], 1),
                      "pov": pov685[g][0], "year": pov685[g][1],
                      "eu": g in EU_ISO3, "ea": g in EA_ISO3,
                      "out": g in WORLD_OUTLIER_REASON, "out_reason": WORLD_OUTLIER_REASON.get(g)})
pauvrete_monde = {
    "meta": {"title": "PIB/hab (PPA) vs taux de pauvreté à 6,85 $/j — mondial",
             "gdp_year": "2024", "pov_line": "6,85 $/jour (PPA 2017) — seuil pays à revenu intermédiaire supérieur",
             "pov_years": "valeur la plus récente 2016-2023 par pays",
             "source": "Banque mondiale : NY.GDP.PCAP.PP.CD, SI.POV.UMIC (PIP)"},
    "countries": povm_rows,
}
json.dump(pauvrete_monde, open(os.path.join(OUT, "pauvrete_monde.json"), "w"), ensure_ascii=False, indent=1)

print("=== BIEN-ETRE : PIB/hab PPA vs espérance de vie (2024, courbe de Preston) ===")
spec("Espérance de vie vs PIB · linéaire · tous", [(r["gdp"], r["le"]) for r in bien_rows])
spec("Espérance de vie vs ln(PIB) · tous", [(r["gdp"], r["le"]) for r in bien_rows], log=False and False) if False else None
# log sur X uniquement (Preston) : on logue X, pas Y
def spec_semilog(label, pairs):
    pairs=[(x,y) for x,y in pairs if x and y]
    xs=[math.log(p[0]) for p in pairs]; ys=[p[1] for p in pairs]
    st=pearson(xs,ys)
    if st: print(f"  {label:44s} n={st['n']:2d}  r={st['r']:+.3f}  R2={st['r2']:.3f}  t={st['t']:+.1f}")
spec_semilog("ln(PIB) · tous", [(r["gdp"], r["le"]) for r in bien_rows])
spec_semilog("ln(PIB) · économies réelles (hors rente/offshore)", [(r["gdp"], r["le"]) for r in bien_rows if not r["out"]])
spec_semilog("ln(PIB) · pays riches réels (≥45k, hors rente/offshore)", [(r["gdp"], r["le"]) for r in bien_rows if r["hi"] and not r["out"]])
print(f"  [monde: {sum(1 for r in bien_rows if r['out'])} économies atypiques masquées par défaut]")

print("=== DIMENSIONS UE : PIB/hab PPS vs indicateurs (hors éco. atypiques) ===")
for dm in DIMS:
    for lab, sub in [("UE", [r for r in dim_rows if not r["out"]]),
                     ("zone €", [r for r in dim_rows if r["ea"] and not r["out"]])]:
        pairs = [(r["gdp_pps"], r[dm["key"]]) for r in sub]
        st = spec(f"{dm['label'][:34]:34s} · {lab}", pairs)

print("=== RÉGIONAL NUTS2 : PIB/hab vs espérance de vie ===")
spec("UE régions · linéaire", [(r["gdp"], r["le"]) for r in reg_rows])
spec_semilog("UE régions · semilog", [(r["gdp"], r["le"]) for r in reg_rows])
spec("zone € régions · linéaire", [(r["gdp"], r["le"]) for r in reg_rows if r["ea"]])

print("=== PAUVRETÉ MONDIALE : PIB/hab PPA vs taux de pauvreté 6,85 $/j ===")
spec("Monde · tous", [(r["gdp"], r["pov"]) for r in povm_rows])
spec("Monde · éco. réelles", [(r["gdp"], r["pov"]) for r in povm_rows if not r["out"]])

print(f"\nÉcrit : salaires({len(countries)}), secteurs({len(sect_rows)}), pauvrete({len(pov_rows)}), pauvrete_monde({len(povm_rows)}), bienetre({len(bien_rows)}), dimensions({len(dim_rows)}), regions({len(reg_rows)})")