import pandas as pd, numpy as np, json, lightgbm as lgb
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score
d = pd.read_csv('/workspace/cs9/data/iran/Customer Churn.csv'); d.columns = [c.strip().replace('  ', ' ') for c in d.columns]
raw = len(d); d = d.drop_duplicates().reset_index(drop=True)
y = d['Churn'].values; X = d.drop(columns=['Churn', 'Age'])  # Age Group kept, raw age dropped (redundant)
print(d.columns.tolist(), raw, len(d), y.mean())
oof = np.zeros(len(d)); aucs = []
for rep in range(5):
    skf = StratifiedKFold(5, shuffle=True, random_state=rep); o = np.zeros(len(d))
    for tr, te in skf.split(X, y):
        m = lgb.LGBMClassifier(n_estimators=300, learning_rate=.03, num_leaves=15, min_child_samples=20, subsample=.8, subsample_freq=1, colsample_bytree=.8, verbose=-1, random_state=rep)
        m.fit(X.iloc[tr], y[tr]); o[te] = m.predict_proba(X.iloc[te])[:, 1]
    oof += o / 5; aucs.append(roc_auc_score(y, o))
# rules: complaints first, then lowest seconds of use; and usage only
r_use = -d['Seconds of Use'].values + np.random.RandomState(0).rand(len(d)) * 1e-3
r_cmp = d['Complains'].values * 1e9 - d['Seconds of Use'].values + np.random.RandomState(1).rand(len(d)) * 1e-3
val = d['Customer Value'].values
def at(s, k):
    i = np.argsort(-s)[:int(round(k * len(s)))]
    return dict(n=len(i), churners=y[i].sum() / y.sum(), precision=y[i].mean(), value=(val[i] * y[i]).sum() / (val * y).sum())
curve = [(k / 20, at(oof, k / 20)['churners'], at(r_cmp, k / 20)['churners'], at(r_use, k / 20)['churners']) for k in range(0, 21)]
out = dict(raw=raw, n=len(d), churn=y.mean(), auc=float(np.mean(aucs)), auc_sd=float(np.std(aucs)), auc_cmp=roc_auc_score(y, r_cmp), auc_use=roc_auc_score(y, r_use),
           at={str(k): at(oof, k) for k in (.1, .2, .3)}, cmp={str(k): at(r_cmp, k) for k in (.1, .2, .3)}, use={str(k): at(r_use, k) for k in (.1, .2, .3)}, curve=curve)
out['curve'][0] = (0, 0, 0, 0)
# segments
seg = []
for nm, mask in [('Filed a complaint', d['Complains'] == 1), ('No complaint', d['Complains'] == 0), ('Pay-as-you-go tariff', d['Tariff Plan'] == 1), ('Contract tariff', d['Tariff Plan'] == 2), ('Inactive status', d['Status'] == 2), ('Active status', d['Status'] == 1)]:
    seg.append(dict(seg=nm, n=int(mask.sum()), rate=float(y[mask.values].mean())))
out['seg'] = seg
q = pd.qcut(d['Seconds of Use'].rank(method='first'), 5, labels=False); out['use_q'] = [float(y[(q == i).values].mean()) for i in range(5)]
out['dec'] = [float(y[(pd.qcut(pd.Series(oof).rank(method='first'), 10, labels=False) == i).values].mean()) for i in range(10)]
# group importance via drop-column AUC
grp = {'Usage (seconds, calls, SMS, distinct numbers)': ['Seconds of Use', 'Frequency of use', 'Frequency of SMS', 'Distinct Called Numbers'], 'Complaints and call failures': ['Complains', 'Call Failure'], 'Plan, charge band and status': ['Tariff Plan', 'Charge Amount', 'Status'], 'Tenure and age group': ['Subscription Length', 'Age Group'], 'Customer value': ['Customer Value']}
imp = []
for g, cols in grp.items():
    Xg = X.drop(columns=cols); o = np.zeros(len(d))
    for tr, te in StratifiedKFold(5, shuffle=True, random_state=0).split(Xg, y):
        m = lgb.LGBMClassifier(n_estimators=300, learning_rate=.03, num_leaves=15, min_child_samples=20, subsample=.8, subsample_freq=1, colsample_bytree=.8, verbose=-1, random_state=0); m.fit(Xg.iloc[tr], y[tr]); o[te] = m.predict_proba(Xg.iloc[te])[:, 1]
    imp.append((g, aucs[0] - roc_auc_score(y, o)))
out['imp'] = sorted(imp, key=lambda t: -t[1])
json.dump(out, open('/workspace/cs9/out/iran.json', 'w'), indent=1, default=float)
print(json.dumps({k: out[k] for k in ['n', 'churn', 'auc', 'auc_cmp', 'auc_use', 'at', 'cmp', 'use', 'seg', 'use_q', 'imp']}, default=float, indent=0)[:3000])
r_ci = d['Complains'].values * 2e9 + (d['Status'].values == 2) * 1e9 - d['Seconds of Use'].values + np.random.RandomState(2).rand(len(d)) * 1e-3
out['ci'] = {str(k): at(r_ci, k) for k in (.1, .2, .3)}; out['auc_ci'] = roc_auc_score(y, r_ci)
out['curve'] = [(k / 20, at(oof, k / 20)['churners'] if k else 0, at(r_ci, k / 20)['churners'] if k else 0, at(r_use, k / 20)['churners'] if k else 0) for k in range(0, 21)]
rs = np.random.RandomState(7); diffs = []
for b in range(1000):
    i = rs.randint(0, len(d), len(d)); yy = y[i]
    def rec(s): j = np.argsort(-s[i])[:int(.2 * len(i))]; return yy[j].sum() / yy.sum()
    diffs.append(rec(oof) - rec(r_ci))
out['diff20_ci'] = [float(np.percentile(diffs, 2.5)), float(np.percentile(diffs, 97.5))]
out['aucs'] = aucs
json.dump(out, open('/workspace/cs9/out/iran.json', 'w'), indent=1, default=float)
print('ci', out['ci'], out['auc_ci'], out['diff20_ci'], aucs)
