"""IN1 · Price motor claims frequency. freMTPL2freq + freMTPL2sev (French motor third-party liability, 678,013 policies; CASdatasets, via OpenML 41214 / 41215).
Flat rate vs Poisson GLM vs LightGBM Poisson, exposure as offset. Group split on identical risk profiles so near-duplicate policies never straddle train/test.
Run: python insurance.py freq.pq sev.pq out.json"""
import sys, numpy as np, pandas as pd, lightgbm as lgb
from sklearn.linear_model import PoissonRegressor
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import GroupShuffleSplit
from common import run_meta, dump, RNG
F, S, OUT = sys.argv[1:4]
f = pd.read_parquet(F); s = pd.read_parquet(S)
A = {'policies': len(f), 'claims_rows_sev': len(s), 'policies_with_claims_freq': int((f.ClaimNb > 0).sum()), 'claims_freq_total': int(f.ClaimNb.sum()),
     'policies_in_sev': int(s.IDpol.nunique()), 'sev_ids_missing_in_freq': int((~s.IDpol.isin(f.IDpol)).sum()),
     'claim_policies_without_sev': int(((f.ClaimNb > 0) & ~f.IDpol.isin(s.IDpol)).sum()), 'claimnb_ge5': int((f.ClaimNb >= 5).sum()),
     'exposure_gt1': int((f.Exposure > 1).sum()), 'vehage_gt30': int((f.VehAge > 30).sum()), 'total_exposure': float(f.Exposure.sum())}
f['ClaimNb'] = f.ClaimNb.clip(upper=4).astype(float); f['Exposure'] = f.Exposure.clip(upper=1.0)
f['VehGas'] = f.VehGas.astype(str); f['Area'] = f.Area.astype(str); f['VehBrand'] = f.VehBrand.astype(str); f['Region'] = f.Region.astype(str)
FEAT = ['Area', 'VehPower', 'VehAge', 'DrivAge', 'BonusMalus', 'VehBrand', 'VehGas', 'Density', 'Region']
prof = pd.util.hash_pandas_object(f[FEAT], index=False); f['grp'] = prof.values
A['rows_sharing_profile'] = float(f.grp.duplicated(keep=False).mean()); A['profiles'] = int(f.grp.nunique()); A['max_profile_rows'] = int(f.grp.value_counts().iloc[0])
A['freq_overall'] = float(f.ClaimNb.sum() / f.Exposure.sum())
gs = GroupShuffleSplit(1, test_size=0.2, random_state=RNG); tri, tei = next(gs.split(f, groups=f.grp)); tr, te = f.iloc[tri].copy(), f.iloc[tei].copy()
# random split for comparison (rows, not groups)
r = np.random.default_rng(RNG); rm = r.random(len(f)) < 0.2; rtr, rte = f[~rm].copy(), f[rm].copy()
def glm_X(d, enc=None):
    g = pd.DataFrame({'Area': d.Area, 'VehPower': d.VehPower.clip(upper=9).astype(str), 'VehAge': pd.cut(d.VehAge, [-1, 0, 10, 200]).astype(str),
                      'DrivAge': pd.cut(d.DrivAge, [17, 20, 25, 30, 40, 50, 70, 200]).astype(str), 'VehBrand': d.VehBrand, 'VehGas': d.VehGas, 'Region': d.Region})
    num = np.c_[np.log(d.Density.values), np.minimum(d.BonusMalus.values, 150) / 100.0]
    if enc is None: enc = OneHotEncoder(handle_unknown='ignore', drop='first', sparse_output=False).fit(g)
    return np.c_[enc.transform(g), num], enc
def fit_glm(tr):
    X, enc = glm_X(tr); m = PoissonRegressor(alpha=1e-6, max_iter=1000).fit(X, tr.ClaimNb / tr.Exposure, sample_weight=tr.Exposure); return m, enc
def pred_glm(m, enc, d): return m.predict(glm_X(d, enc)[0])  # frequency per year
def lgb_data(d):
    X = d[FEAT].copy()
    for c in ['Area', 'VehBrand', 'VehGas', 'Region']: X[c] = X[c].astype('category')
    return X
P = dict(objective='poisson', learning_rate=0.05, num_leaves=31, min_child_samples=200, subsample=0.8, subsample_freq=1, colsample_bytree=0.8, reg_lambda=1.0, verbose=-1, seed=RNG, num_threads=4)
def fit_gbm(tr, rounds=None):
    X = lgb_data(tr); cats = [X.columns.get_loc(c) for c in ['Area', 'VehBrand', 'VehGas', 'Region']]
    if rounds is None:
        i = np.random.default_rng(RNG).random(len(tr)) < 0.2
        dt = lgb.Dataset(X[~i], tr.ClaimNb[~i], init_score=np.log(tr.Exposure[~i]), categorical_feature=cats)
        dv = lgb.Dataset(X[i], tr.ClaimNb[i], init_score=np.log(tr.Exposure[i]), categorical_feature=cats)
        b = lgb.train(P, dt, 3000, valid_sets=[dv], callbacks=[lgb.early_stopping(100, verbose=False)]); rounds = b.best_iteration
    b = lgb.train(P, lgb.Dataset(X, tr.ClaimNb, init_score=np.log(tr.Exposure), categorical_feature=cats), rounds); return b, rounds
def pred_gbm(b, d): return np.exp(b.predict(lgb_data(d), raw_score=True))
def dev(y, mu):  # mean Poisson deviance per policy, y and mu are counts
    t = np.where(y > 0, y * np.log(np.where(y > 0, y, 1) / mu), 0.0); return 2 * (t - (y - mu))
def gini(y, mu, w):
    o = np.argsort(mu); cw = np.cumsum(w[o]) / w.sum(); cy = np.cumsum(y[o]) / y.sum(); auc = np.trapezoid(np.r_[0, cy], np.r_[0, cw]); return float(1 - 2 * auc), cw, cy
gm, enc = fit_glm(tr); gb, rounds = fit_gbm(tr)
flat = tr.ClaimNb.sum() / tr.Exposure.sum()
y = te.ClaimNb.values; w = te.Exposure.values
MU = {'Flat rate': np.full(len(te), flat) * w, 'Poisson GLM': pred_glm(gm, enc, te) * w, 'LightGBM Poisson': pred_gbm(gb, te) * w}
D = {k: dev(y, v) for k, v in MU.items()}
def bci(a, b=None, B=500):
    r = np.random.default_rng(RNG); n = len(a); out = []
    for _ in range(B):
        i = r.integers(0, n, n); out.append(a[i].mean() - (b[i].mean() if b is not None else 0))
    return [float(np.percentile(out, 2.5)), float(np.percentile(out, 97.5))]
R = {}
for k in MU:
    g, cw, cy = gini(y, MU[k] / w, w); idx = np.unique(np.linspace(0, len(cw) - 1, 80).astype(int))
    R[k] = {'deviance': float(D[k].mean()), 'dev_ci': bci(D[k]), 'gini': g, 'lorenz': [[round(float(cw[i]), 4), round(float(cy[i]), 4)] for i in idx],
            'explained': float(1 - D[k].sum() / D['Flat rate'].sum()), 'pred_total': float(MU[k].sum())}
R['actual_total'] = float(y.sum()); R['gbm_vs_glm_dev_ci'] = bci(D['LightGBM Poisson'], D['Poisson GLM']); R['glm_vs_flat_dev_ci'] = bci(D['Poisson GLM'], D['Flat rate'])
R['rounds'] = int(rounds)
gb_g = []
for i in range(5):  # bootstrap gini difference by resampling policies
    pass
rr = np.random.default_rng(RNG); gd = []
for _ in range(200):
    i = rr.integers(0, len(y), len(y)); gd.append(gini(y[i], MU['LightGBM Poisson'][i] / w[i], w[i])[0] - gini(y[i], MU['Poisson GLM'][i] / w[i], w[i])[0])
R['gini_diff_ci'] = [float(np.percentile(gd, 2.5)), float(np.percentile(gd, 97.5))]
# random-row split: what the leaderboard would say if duplicates leaked
b2, _ = fit_gbm(rtr, rounds); m2, e2 = fit_glm(rtr)
rd_g = dev(rte.ClaimNb.values, pred_gbm(b2, rte) * rte.Exposure.values).mean(); rd_l = dev(rte.ClaimNb.values, pred_glm(m2, e2, rte) * rte.Exposure.values).mean(); rd_f = dev(rte.ClaimNb.values, np.full(len(rte), rtr.ClaimNb.sum() / rtr.Exposure.sum()) * rte.Exposure.values).mean()
R['random_split'] = {'gbm_explained': float(1 - rd_g / rd_f), 'glm_explained': float(1 - rd_l / rd_f)}
# double lift: sort by GBM/GLM ratio, deciles, actual vs each model, all scaled to actual
ratio = MU['LightGBM Poisson'] / MU['Poisson GLM']; q = pd.qcut(ratio, 10, labels=False, duplicates='drop')
dl = pd.DataFrame({'q': q, 'y': y, 'g': MU['LightGBM Poisson'], 'l': MU['Poisson GLM'], 'w': w}).groupby('q').sum()
dlift = [[int(i), float(r.y / r.w), float(r.g / r.w), float(r.l / r.w), float(r.w)] for i, r in dl.iterrows()]
# actual / expected by factor
def ae(col, bins=None, labels=None):
    v = te[col] if bins is None else pd.cut(te[col], bins, labels=labels)
    g = pd.DataFrame({'v': v.values, 'y': y, 'g': MU['LightGBM Poisson'], 'l': MU['Poisson GLM'], 'w': w}).groupby('v', observed=True).sum()
    return [[str(i), float(r.w), float(r.y / r.w), float(r.g / r.w), float(r.l / r.w), int(r.y)] for i, r in g.iterrows()]
AE = {'DrivAge': ae('DrivAge', [17, 20, 22, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 100], ['18–20', '21–22', '23–25', '26–30', '31–35', '36–40', '41–45', '46–50', '51–55', '56–60', '61–65', '66–70', '71–75', '76–80', '81+']),
      'BonusMalus': ae('BonusMalus', [49, 50, 55, 60, 70, 80, 90, 100, 125, 300], ['50', '51–55', '56–60', '61–70', '71–80', '81–90', '91–100', '101–125', '126+']),
      'Area': ae('Area'), 'VehAge': ae('VehAge', [-1, 0, 2, 5, 10, 15, 20, 200], ['new', '1–2', '3–5', '6–10', '11–15', '16–20', '21+'])}
# importance (gain)
imp = sorted([[n, float(v)] for n, v in zip(gb.feature_name(), gb.feature_importance('gain'))], key=lambda x: -x[1]); tot = sum(v for _, v in imp); imp = [[n, v / tot] for n, v in imp]
# partial dependence of driver age (GBM) holding others at observed values, on a 20k subset
sub = te.sample(20000, random_state=RNG); pdp = []
for a in range(18, 91, 2):
    t = sub.copy(); t['DrivAge'] = a; pdp.append([a, float(pred_gbm(gb, t).mean()), float(pred_glm(gm, enc, t).mean())])
# severity
sv = s[s.IDpol.isin(f.IDpol)].merge(f[['IDpol', 'DrivAge', 'Area', 'VehAge', 'BonusMalus']], on='IDpol')
amt = sv.ClaimAmount.values; srt = np.sort(amt)[::-1]; cut = float(np.percentile(amt, 99.5))
SEV = {'n': len(sv), 'mean': float(amt.mean()), 'median': float(np.median(amt)), 'p90': float(np.percentile(amt, 90)), 'p99': float(np.percentile(amt, 99)), 'max': float(amt.max()),
       'top1pct_share': float(srt[:max(1, len(srt) // 100)].sum() / srt.sum()), 'cap995': cut, 'capped_mean': float(np.minimum(amt, cut).mean()),
       'excess_share': float(np.maximum(amt - cut, 0).sum() / amt.sum()), 'exact_1204': float((np.round(amt, 2) == 1204.0).mean()), 'exact_1128': float((np.round(amt, 2) == 1128.12).mean()),
       'hist': np.histogram(np.log10(np.clip(amt, 1, None)), bins=np.arange(0, 6.75, 0.25))[0].tolist(),
       'by_age': [[str(k), int(len(g)), float(np.median(g.ClaimAmount)), float(np.minimum(g.ClaimAmount, cut).mean())] for k, g in sv.groupby(pd.cut(sv.DrivAge, [17, 25, 35, 50, 65, 100], labels=['18–25', '26–35', '36–50', '51–65', '66+']), observed=True)],
       'top_values': [[float(a), int(b)] for a, b in pd.Series(np.round(amt, 2)).value_counts().head(6).items()]}
# indicative premium: frequency x capped mean severity x loading 1.0 (no expenses); show spread of prices
sevm = SEV['capped_mean']; prem_g = MU['LightGBM Poisson'] / w * sevm; prem_l = MU['Poisson GLM'] / w * sevm
PR = {'flat': float(flat * sevm), 'glm_p10_p90': [float(np.percentile(prem_l, 10)), float(np.percentile(prem_l, 90))], 'gbm_p10_p90': [float(np.percentile(prem_g, 10)), float(np.percentile(prem_g, 90))],
      'change_gt10pct': float((np.abs(prem_g / prem_l - 1) > 0.10).mean()), 'hist_ratio': np.histogram(np.clip(prem_g / prem_l, 0.5, 1.5), bins=np.arange(0.5, 1.5001, 0.05))[0].tolist()}
dist = {'claimnb': f.ClaimNb.value_counts().sort_index().to_dict(), 'exposure_hist': np.histogram(f.Exposure, bins=np.arange(0, 1.0001, 0.05))[0].tolist(),
        'bm_hist': np.histogram(f.BonusMalus.clip(upper=150), bins=np.arange(50, 152, 5))[0].tolist(), 'dens_hist': np.histogram(np.log10(f.Density), bins=np.arange(0, 4.6, 0.25))[0].tolist()}
A['train'] = len(tr); A['test'] = len(te); A['test_exposure'] = float(w.sum()); A['test_claims'] = int(y.sum())
dump({'meta': run_meta(len(f), {'source': 'freMTPL2freq / freMTPL2sev, OpenML 41214 / 41215'}), 'audit': A, 'results': R, 'dlift': dlift, 'ae': AE, 'imp': imp, 'pdp': pdp, 'sev': SEV, 'prem': PR, 'dist': dist}, OUT)
print(A); print({k: (v if not isinstance(v, dict) else {a: b for a, b in v.items() if a != 'lorenz'}) for k, v in R.items()}); print(SEV); print(PR)
