"""Post-discharge follow-up list. Data: Diabetes 130-US hospitals for years 1999-2008 (Strack et al., 2014; UCI, CC BY 4.0): 101,766 inpatient
encounters of patients with diabetes. Outcome: readmitted within 30 days. Patients are split by the order of their first encounter:
patients first seen in the earliest 80% of encounter ids train the model, patients first seen later form the test set, so no patient is in both."""
import json, numpy as np, pandas as pd
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.metrics import roc_auc_score, brier_score_loss
d = pd.read_csv('/workspace/cs8/data/diabetes/diabetic_data.csv', na_values='?')
raw = len(d); d = d[~d.discharge_disposition_id.isin([11, 13, 14, 19, 20, 21])].copy()  # died or discharged to hospice
d['y'] = (d.readmitted == '<30').astype(int)
d = d.sort_values('encounter_id'); first = d.groupby('patient_nbr').encounter_id.transform('min'); cut = d.encounter_id.quantile(.8)
tr = d[first <= cut].copy(); te = d[first > cut].copy()
def icd(s):
    if pd.isna(s): return 'Missing'
    try: v = float(s)
    except (TypeError, ValueError): return 'Other' if pd.notna(s) else 'Missing'
    if 390 <= v <= 459 or v == 785: return 'Circulatory'
    if 460 <= v <= 519 or v == 786: return 'Respiratory'
    if 520 <= v <= 579 or v == 787: return 'Digestive'
    if int(v) == 250: return 'Diabetes'
    if 800 <= v <= 999: return 'Injury'
    if 710 <= v <= 739: return 'Musculoskeletal'
    if 580 <= v <= 629 or v == 788: return 'Genitourinary'
    if 140 <= v <= 239: return 'Neoplasms'
    return 'Other'
for x in (tr, te):
    x['age_n'] = x.age.str.extract(r'\[(\d+)').astype(float)[0]; x['dx1'] = x.diag_1.map(icd)
    x['lace'] = np.select([x.time_in_hospital < 2, x.time_in_hospital < 4, x.time_in_hospital < 7, x.time_in_hospital < 14], [x.time_in_hospital, 3, 4, 5], 7) + 3 * (x.admission_type_id == 1) + np.minimum(x.number_diagnoses // 2, 5) + np.minimum(x.number_emergency + x.number_inpatient, 4)
CAT = ['dx1', 'A1Cresult', 'max_glu_serum', 'insulin', 'change', 'diabetesMed', 'medical_specialty', 'admission_type_id', 'discharge_disposition_id', 'admission_source_id']
NUM = ['age_n', 'time_in_hospital', 'num_lab_procedures', 'num_procedures', 'num_medications', 'number_outpatient', 'number_emergency', 'number_inpatient', 'number_diagnoses']
def X(x):
    o = x[NUM].copy()
    for c in CAT: o[c] = pd.Categorical(x[c].fillna('NA').astype(str), categories=sorted(tr[c].fillna('NA').astype(str).unique()))
    return o
m = HistGradientBoostingClassifier(max_iter=400, learning_rate=.04, max_leaf_nodes=31, l2_regularization=1.0, categorical_features='from_dtype', random_state=0).fit(X(tr), tr.y)
p = m.predict_proba(X(te))[:, 1]; y = te.y.values; lace = te.lace.values + np.random.default_rng(1).random(len(te)) * 1e-3; prior = te.number_inpatient.values + np.random.default_rng(2).random(len(te)) * 1e-3
def cap(s, k): i = np.argsort(-s)[:int(len(s) * k)]; return float(y[i].sum() / y.sum()), float(y[i].mean())
rng = np.random.default_rng(0)
def boot(s, B=300): return np.percentile([roc_auc_score(y[i], s[i]) for i in (rng.integers(0, len(y), len(y)) for _ in range(B))], [2.5, 97.5]).tolist()
R = dict(raw=raw, n=len(d), n_train=len(tr), n_test=len(te), pat_train=int(tr.patient_nbr.nunique()), pat_test=int(te.patient_nbr.nunique()), base=float(y.mean()), base_train=float(tr.y.mean()),
  auc=float(roc_auc_score(y, p)), auc_ci=boot(p), auc_lace=float(roc_auc_score(y, lace)), auc_prior=float(roc_auc_score(y, prior)), brier=float(brier_score_loss(y, p)), brier_base=float(brier_score_loss(y, np.full(len(y), tr.y.mean()))))
for k in (.1, .2, .3): R[f'cap{int(k*100)}'] = dict(model=cap(p, k), lace=cap(lace, k), prior=cap(prior, k))
R['curve'] = [[float(k), cap(p, k)[0], cap(lace, k)[0], cap(prior, k)[0]] for k in np.linspace(.05, 1, 20)]
dec = pd.DataFrame(dict(p=p, y=y)); dec['d'] = pd.qcut(dec.p, 10, labels=False); R['deciles'] = dec.groupby('d').agg(pred=('p', 'mean'), obs=('y', 'mean'), n=('y', 'size')).reset_index().to_dict('records')
R['prior_groups'] = [[nm, int(s.sum()), float(te.y[s].mean())] for nm, s in [('No inpatient stay in the prior year', te.number_inpatient == 0), ('One', te.number_inpatient == 1), ('Two', te.number_inpatient == 2), ('Three or more', te.number_inpatient >= 3)]]
R['dx_groups'] = sorted([[k, int(v.size), float(v.mean())] for k, v in te.groupby('dx1').y], key=lambda r: -r[1])
R['age_groups'] = [[f'{int(a)}-{int(a)+10}', int(v.size), float(v.mean())] for a, v in te.groupby('age_n').y]
from sklearn.inspection import permutation_importance
pi = permutation_importance(m, X(te).iloc[:8000], y[:8000], scoring='roc_auc', n_repeats=3, random_state=0)
R['imp'] = sorted([[f, float(v)] for f, v in zip(NUM + CAT, pi.importances_mean)], key=lambda q: -q[1])[:8]
json.dump(R, open('/workspace/cs8/out/readmit.json', 'w'), indent=1, default=float)
print({k: R[k] for k in ['n', 'n_train', 'n_test', 'base', 'auc', 'auc_ci', 'auc_lace', 'auc_prior', 'brier', 'brier_base', 'cap20']}); print(R['imp'])
