"""Customer segments and next-90-day purchase prediction, UCI Online Retail II (Chen 2012/2019, CC BY 4.0)."""
import numpy as np, pandas as pd, sys
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score, roc_auc_score
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.ensemble import HistGradientBoostingClassifier, HistGradientBoostingRegressor
from scipy.stats import spearmanr
from common import *
raw = pd.read_pickle(sys.argv[1] if len(sys.argv) > 1 else '../data/retail.pkl')
raw.columns = ['inv', 'code', 'desc', 'qty', 'date', 'price', 'cust', 'country']
out = {"meta": run_meta(len(raw), {"source": "UCI ML Repository #502, Online Retail II (Chen, 2019)", "licence": "CC BY 4.0", "span": [str(raw.date.min()), str(raw.date.max())]})}
a = {'rows': len(raw), 'invoices': int(raw.inv.nunique()), 'customers': int(raw.cust.nunique()), 'countries': int(raw.country.nunique()), 'products': int(raw.code.nunique())}
a['missing_cust'] = int(raw.cust.isna().sum()); a['missing_cust_pct'] = float(raw.cust.isna().mean())
a['cancel_rows'] = int(raw.inv.astype(str).str.startswith('C').sum()); a['dups'] = int(raw.duplicated().sum())
a['price_le0'] = int((raw.price <= 0).sum()); a['neg_qty'] = int((raw.qty < 0).sum())
svc = raw.code.astype(str).str.upper().str.match(r'^(POST|DOT|M|C2|BANK CHARGES|AMAZONFEE|ADJUST|D|S|B|CRUK|PADS|TEST)'); a['service_rows'] = int(svc.sum())
d = raw.drop_duplicates(); d = d[~d.inv.astype(str).str.startswith('C') & (d.qty > 0) & (d.price > 0) & ~svc.loc[d.index]].copy()
d['rev'] = d.qty * d.price; a['clean_rows'] = len(d); a['clean_rev'] = float(d.rev.sum())
a['cancel_value'] = float(-(raw.qty * raw.price)[raw.inv.astype(str).str.startswith('C')].sum())
a['anon_rev_share'] = float(d.rev[d.cust.isna()].sum() / d.rev.sum())
out['audit'] = a
eda = {}
m = d.set_index('date').rev.resample('MS').sum(); eda['monthly'] = [[str(k.date()), float(v)] for k, v in m.items()]
d['wd'] = d.date.dt.dayofweek; d['hr'] = d.date.dt.hour
hm = d.groupby(['wd', 'hr']).rev.sum(); eda['heat'] = [[int(w), int(h), float(v)] for (w, h), v in hm.items()]
c = d.groupby('country').rev.sum().sort_values(ascending=False); eda['country'] = [[k, float(v)] for k, v in c.head(10).items()]; eda['uk_share'] = float(c.get('United Kingdom', 0) / c.sum())
k = d.dropna(subset=['cust']); cr = k.groupby('cust').rev.sum().sort_values(ascending=False)
cum = cr.cumsum() / cr.sum(); xs = np.linspace(0, 1, 101); eda['pareto'] = [[float(x), float(cum.iloc[max(0, int(x * len(cum)) - 1)])] for x in xs]
eda['top20_share'] = float(cr.head(int(len(cr) * .2)).sum() / cr.sum()); eda['top1_share'] = float(cr.head(int(len(cr) * .01)).sum() / cr.sum())
o = k.groupby('cust').inv.nunique(); eda['one_time_share'] = float((o == 1).mean())
# monthly cohort retention (first purchase month)
k = k.assign(m=k.date.dt.to_period('M')); first = k.groupby('cust').m.min().rename('c'); kk = k.join(first, on='cust'); kk['age'] = (kk.m - kk.c).apply(lambda x: x.n)
coh = kk.groupby(['c', 'age']).cust.nunique().unstack(fill_value=0); coh = coh.loc[coh.index <= pd.Period('2010-12', 'M')]
ret = coh.div(coh[0], axis=0); eda['cohort'] = {'rows': [str(i) for i in ret.index], 'size': coh[0].tolist(), 'm': [[round(float(v), 3) for v in r[:13]] for r in ret.values]}
out['eda'] = eda
def feats(cut, hist_days=None):
    h = k[k.date < cut]; g = h.groupby('cust')
    f = pd.DataFrame({'recency': (cut - g.date.max()).dt.days, 'frequency': g.inv.nunique(), 'monetary': g.rev.sum(), 'tenure': (cut - g.date.min()).dt.days,
        'n_products': g.code.nunique(), 'avg_basket': g.rev.sum() / g.inv.nunique(), 'uk': g.country.first().eq('United Kingdom').astype(int)})
    for w in (30, 90, 180):
        hw = h[h.date >= cut - pd.Timedelta(days=w)].groupby('cust'); f[f'rev_{w}'] = hw.rev.sum(); f[f'inv_{w}'] = hw.inv.nunique()
    f = f.fillna(0); f['gap_mean'] = g.date.apply(lambda s: s.drop_duplicates().sort_values().diff().dt.days.mean()).fillna(999)
    fut = k[(k.date >= cut) & (k.date < cut + pd.Timedelta(days=90))].groupby('cust').rev.sum()
    f['y'] = f.index.isin(fut.index).astype(int); f['spend90'] = fut.reindex(f.index).fillna(0).values
    return f
end = k.date.max().normalize(); cut_test = end - pd.Timedelta(days=90); cut_train = cut_test - pd.Timedelta(days=182)
tr = feats(cut_train); te = feats(cut_test)
out['split'] = {'train_cutoff': str(cut_train.date()), 'test_cutoff': str(cut_test.date()), 'end': str(end.date()), 'train_n': len(tr), 'test_n': len(te), 'train_rate': float(tr.y.mean()), 'test_rate': float(te.y.mean())}
# segmentation at test cutoff
rfm = np.log1p(te[['recency', 'frequency', 'monetary']].clip(lower=0)); Z = StandardScaler().fit_transform(rfm)
sil = {}
for kk_ in range(2, 9):
    lab = KMeans(kk_, n_init=10, random_state=RNG).fit_predict(Z); sil[kk_] = float(silhouette_score(Z, lab, sample_size=3000, random_state=RNG))
out['silhouette'] = sil; K = 4  # chosen for actionability; silhouette reported for every k
km = KMeans(K, n_init=20, random_state=RNG).fit(Z); te['seg'] = km.labels_
prof = te.groupby('seg').agg(n=('y', 'size'), recency=('recency', 'median'), frequency=('frequency', 'median'), monetary=('monetary', 'median'), rev_total=('monetary', 'sum'), next90=('y', 'mean'), spend90=('spend90', 'sum')).reset_index()
# name segments by recency/monetary rank
prof['score'] = prof.monetary.rank() - prof.recency.rank(); order = prof.sort_values('score', ascending=False).seg.tolist()
names = dict(zip(order, ['Loyal big spenders', 'Regulars', 'Drifting', 'Lapsed one-offs']))
prof['name'] = prof.seg.map(names); te['segname'] = te.seg.map(names)
out['segments'] = prof.assign(share_rev=prof.rev_total / prof.rev_total.sum(), share_cust=prof.n / prof.n.sum(), share_spend90=prof.spend90 / prof.spend90.sum()).to_dict('records')
smp = te.sample(min(1500, len(te)), random_state=RNG); out['seg_points'] = [[int(r.recency), int(r.frequency), round(float(r.monetary), 1), names[r.seg], int(r.y)] for r in smp.itertuples()]
# prediction: train on earlier cutoff, test out-of-time
X = [c for c in tr.columns if c not in ('y', 'spend90')]
preds = {'Rule: bought in last 90 days': (te.recency <= 90).astype(float).values, 'Rule: recency rank': -te.recency.values.astype(float)}
mk_lr = lambda: make_pipeline(StandardScaler(), LogisticRegression(max_iter=2000)); lg = lambda f: np.log1p(f[X].clip(lower=0))
mk_gb = lambda: HistGradientBoostingClassifier(max_iter=300, learning_rate=.05, min_samples_leaf=40, random_state=RNG)
# model selection on an earlier, separate window (no test data used): train at cut_train-182d, validate at cut_train
va_tr = feats(cut_train - pd.Timedelta(days=182)); va = tr
val = {'Logistic regression': float(roc_auc_score(va.y, mk_lr().fit(lg(va_tr), va_tr.y).predict_proba(lg(va))[:, 1])), 'Gradient boosting': float(roc_auc_score(va.y, mk_gb().fit(va_tr[X], va_tr.y).predict_proba(va[X])[:, 1])), 'Rule: recency rank': float(roc_auc_score(va.y, -va.recency))}
chosen = max(['Logistic regression', 'Gradient boosting'], key=lambda k: val[k]); out['validation'] = {'auc': val, 'chosen': chosen, 'train_cutoff': str((cut_train - pd.Timedelta(days=182)).date()), 'val_cutoff': str(cut_train.date()), 'n_train': len(va_tr), 'n_val': len(va)}
lr = mk_lr().fit(lg(tr), tr.y); preds['Logistic regression'] = lr.predict_proba(lg(te))[:, 1]
gb = mk_gb().fit(tr[X], tr.y); preds['Gradient boosting'] = gb.predict_proba(te[X])[:, 1]
y = te.y.values; res = {}
for n, p in preds.items(): res[n] = score(y, p)
out['models'] = res; out['roc'] = {n: roc_pts(y, p) for n, p in preds.items() if 'last 90' not in n}
pg = preds[chosen]; out['calib'] = {'Gradient boosting': calib(y, preds['Gradient boosting']), 'Logistic regression': calib(y, preds['Logistic regression'])}
r = np.random.default_rng(RNG); diffs = []
for _ in range(1000):
    i = r.integers(0, len(y), len(y)); diffs.append(roc_auc_score(y[i], pg[i]) - roc_auc_score(y[i], preds['Rule: recency rank'][i]))
out['auc_diff_chosen_recency'] = [float(np.mean(diffs)), float(np.percentile(diffs, 2.5)), float(np.percentile(diffs, 97.5))]
# spend model (two-part: p(buy) * E[spend|buy])
buy = tr[tr.y == 1]; rg = HistGradientBoostingRegressor(max_iter=300, learning_rate=.05, min_samples_leaf=30, random_state=RNG).fit(buy[X], np.log1p(buy.spend90))
ev = pg * np.expm1(rg.predict(te[X])); naive = te.rev_90.values  # baseline: last 90 days repeat
out['spend'] = {'spearman_model': float(spearmanr(ev, te.spend90).statistic), 'spearman_naive': float(spearmanr(naive, te.spend90).statistic),
  'mae_model': float(np.mean(np.abs(ev - te.spend90))), 'mae_naive': float(np.mean(np.abs(naive - te.spend90))), 'actual_total': float(te.spend90.sum()), 'pred_total': float(ev.sum()), 'naive_total': float(naive.sum())}
o = np.argsort(-ev); out['spend']['top10_capture_model'] = float(te.spend90.values[o[:len(o) // 10]].sum() / te.spend90.sum())
on = np.argsort(-(naive + 1e-6 * te.monetary.values)); out['spend']['top10_capture_naive'] = float(te.spend90.values[on[:len(on) // 10]].sum() / te.spend90.sum())
og = np.argsort(-pg); out['deciles'] = [[i + 1, float(y[g].mean()), int(len(g))] for i, g in enumerate(np.array_split(og, 10))]
# campaign scenario: contact budget of 500 customers among those who did NOT buy in last 90 days (win-back)
lapsed = te.recency.values > 90; idx = np.where(lapsed)[0]; kb = 500
sel_m = idx[np.argsort(-pg[idx])[:kb]]; sel_r = idx[np.argsort(te.recency.values[idx])[:kb]]
out['campaign'] = {'pool': int(lapsed.sum()), 'budget': kb, 'pool_rate': float(y[idx].mean()), 'hits_model': int(y[sel_m].sum()), 'hits_recency': int(y[sel_r].sum()), 'hits_random': float(kb * y[idx].mean())}
fe = gb  # importance via permutation
from sklearn.inspection import permutation_importance
class _W:
    def __init__(s, m, f): s.m, s.f = m, f
    def fit(s, *a): return s
    def predict_proba(s, Z): return s.m.predict_proba(s.f(Z))
from sklearn.base import BaseEstimator
model_for_pi = lr if chosen == 'Logistic regression' else gb
pi = permutation_importance(model_for_pi, (lg(te) if chosen == 'Logistic regression' else te[X]), y, scoring='roc_auc', n_repeats=5, random_state=RNG)
out['importance'] = [[c, float(mm), float(s)] for c, mm, s in sorted(zip(X, pi.importances_mean, pi.importances_std), key=lambda t: -t[1])[:10]]
# errors: high-confidence misses
te['p'] = pg; fn = te[(te.y == 1) & (te.p < .2)]; fp = te[(te.y == 0) & (te.p > .7)]
out['errors'] = {'fn_lowp': len(fn), 'fn_one_time_share': float((fn.frequency == 1).mean()), 'fn_med_recency': float(fn.recency.median()), 'fp_highp': len(fp), 'fp_med_freq': float(fp.frequency.median()), 'fp_med_recency': float(fp.recency.median()),
  'seg_auc': {s: float(roc_auc_score(g.y, g.p)) for s, g in te.groupby('segname') if g.y.nunique() == 2}}
# basket pairs (top 40 products by invoices, last 12 months)
b = d[d.date >= end - pd.Timedelta(days=365)]; topc = b.groupby('code').inv.nunique().nlargest(40).index; bb = b[b.code.isin(topc)][['inv', 'code']].drop_duplicates()
M = pd.crosstab(bb.inv, bb.code).clip(upper=1); N = len(b.inv.unique()); sup = M.sum() / N; co = (M.T @ M) / N
pairs = []
cols = list(M.columns)
for i in range(len(cols)):
    for j in range(i + 1, len(cols)):
        s = co.iloc[i, j]; lift = s / (sup.iloc[i] * sup.iloc[j]); pairs.append((cols[i], cols[j], float(s), float(lift), float(s / sup.iloc[i])))
desc = b.drop_duplicates('code').set_index('code').desc
pairs = sorted(pairs, key=lambda t: -t[3])[:10]; out['pairs'] = [{'a': str(desc.get(p[0], p[0])).strip().title(), 'b': str(desc.get(p[1], p[1])).strip().title(), 'support': p[2], 'lift': p[3], 'conf': p[4]} for p in pairs]; out['basket_invoices'] = int(N)
dump(out, '../out/retail.json'); print(out['split'], {n: (round(v['auc'], 3), v['auc_ci']) for n, v in res.items()}, out['campaign'], out['spend'], sil)
