"""Students at risk, flagged early. Data: UCI 697 'Predict Students' Dropout and Academic Success' (Realinho et al., 2022),
4,424 undergraduate students, Polytechnic Institute of Portalegre, Portugal, enrolled 2008/09-2018/19. Outcome: dropout vs graduate or still enrolled.
Two checkpoints: at enrolment (application, background, fees) and after semester one (adds first-semester units and grades).
5-fold stratified CV, 3 seeds, out-of-fold scores. Baseline rule: tuition fees not up to date, or a debtor. Gender, nationality, international status and marital status are excluded from the score."""
import pandas as pd, numpy as np, json
from sklearn.model_selection import StratifiedKFold
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.metrics import roc_auc_score
d = pd.read_csv('/workspace/cs6/data/students.csv', sep=';'); d.columns = [c.strip() for c in d.columns]
y = (d.Target == 'Dropout').astype(int).values
sem2 = [c for c in d.columns if '2nd sem' in c]; sem1 = [c for c in d.columns if '1st sem' in c]
EXCL = ['Gender', 'Nacionality', 'International', 'Marital status']
F0 = [c for c in d.columns if c not in sem1 + sem2 + ['Target'] + EXCL]; F1 = F0 + sem1
def oof(F):
    s = np.zeros(len(y))
    for seed in range(3):
        for a, b in StratifiedKFold(5, shuffle=True, random_state=seed).split(d, y):
            m = HistGradientBoostingClassifier(max_iter=250, learning_rate=.05, max_depth=4, random_state=seed).fit(d[F].iloc[a], y[a]); s[b] += m.predict_proba(d[F].iloc[b])[:, 1] / 3
    return s
s0, s1 = oof(F0), oof(F1); auc_with = roc_auc_score(y, oof([c for c in d.columns if c not in sem2 + ['Target']])); rule = ((d['Tuition fees up to date'] == 0) | (d['Debtor'] == 1)).astype(float).values
def at(s, k):
    i = np.argsort(-s, kind='stable')[: int(len(y) * k)]; return dict(caught=float(y[i].sum() / y.sum()), precision=float(y[i].mean()), n=int(len(i)))
rule_n = rule.mean(); rr = dict(share=float(rule_n), caught=float(y[rule == 1].sum() / y.sum()), precision=float(y[rule == 1].mean()))
curve = [[float(k), at(s0, k)['caught'] if k else 0, at(s1, k)['caught'] if k else 0] for k in np.linspace(0, 1, 21)]
seg = []
for nm, sel in [('Fees not up to date', d['Tuition fees up to date'] == 0), ('Debtor', d.Debtor == 1), ('Scholarship holder', d['Scholarship holder'] == 1), ('Evening attendance', d.iloc[:, 4] == 0),
                ('Aged 25+ at enrolment', d['Age at enrollment'] >= 25), ('0 units passed in semester 1', d['Curricular units 1st sem (approved)'] == 0), ('All units passed in semester 1', (d['Curricular units 1st sem (approved)'] >= d['Curricular units 1st sem (enrolled)']) & (d['Curricular units 1st sem (enrolled)'] > 0))]:
    seg.append(dict(seg=nm, n=int(sel.sum()), rate=float(y[sel.values].mean())))
ap = d['Curricular units 1st sem (approved)'].clip(upper=8); pass_curve = [[int(k), float(y[(ap == k).values].mean()), int((ap == k).sum())] for k in range(9)]
out = dict(auc1_with_excluded=auc_with, n=len(y), dropout=float(y.mean()), auc0=roc_auc_score(y, s0), auc1=roc_auc_score(y, s1), auc_rule=roc_auc_score(y, rule), rule=rr,
  at0={k: at(s0, k) for k in [.1, .2, .3]}, at1={k: at(s1, k) for k in [.1, .2, .3]}, curve=curve, seg=seg, pass_curve=pass_curve, n_f0=len(F0), n_f1=len(F1), targets=d.Target.value_counts().to_dict())
json.dump(out, open('/workspace/cs6/out/students.json', 'w'), indent=1, default=float); print({k: out[k] for k in out if k not in ('curve',)})
