"""MF2 · Which trucks to call in for an air-system check. APS Failure at Scania Trucks (UCI 421, Scania CV AB, GPL-3).
Official split: 60,000 train, 16,000 test. Cost: 10 per unnecessary workshop check, 500 per missed failing truck.
Threshold chosen on 5-fold out-of-fold train predictions, then applied once to the test set. Run: python aps.py DATA_DIR out.json"""
import sys, numpy as np, pandas as pd, lightgbm as lgb
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score, average_precision_score
from common import run_meta, dump, RNG
DIR, OUT = sys.argv[1:3]
def load(f):
    lines = open(f, encoding='latin-1').read().splitlines(); h = next(i for i, l in enumerate(lines) if l.startswith('class,'))
    d = pd.read_csv(f, skiprows=h, na_values='na', encoding='latin-1'); y = (d.pop('class') == 'pos').astype(int).values; return d.astype(float), y
X, y = load(DIR + '/aps_failure_training_set.csv'); Xt, yt = load(DIR + '/aps_failure_test_set.csv')
C1, C2 = 10, 500
def cost(y, flag, c1=C1, c2=C2): return int(c1 * ((flag == 1) & (y == 0)).sum() + c2 * ((flag == 0) & (y == 1)).sum())
miss = X.isna().mean(); A = {'train': len(X), 'train_pos': int(y.sum()), 'test': len(Xt), 'test_pos': int(yt.sum()), 'features': X.shape[1], 'missing_cells': float(X.isna().values.mean()),
     'feat_miss_gt50': int((miss > .5).sum()), 'feat_constant': int((X.nunique() <= 1).sum()), 'hist_families': int(len({c.split('_')[0] for c in X.columns if '_' in c and c.split('_')[1].isdigit()})),
     'rows_any_missing': float(X.isna().any(axis=1).mean())}
mr = X.isna().sum(axis=1); A['miss_per_row_pos'] = float(mr[y == 1].median()); A['miss_per_row_neg'] = float(mr[y == 0].median())
skf = StratifiedKFold(5, shuffle=True, random_state=RNG)
P = dict(objective='binary', learning_rate=0.03, num_leaves=31, min_data_in_leaf=20, feature_fraction=0.6, bagging_fraction=0.8, bagging_freq=1, lambda_l2=1.0, verbose=-1, seed=RNG, scale_pos_weight=5)
oof = np.zeros(len(y)); its = []
for tr, va in skf.split(X, y):
    da = lgb.Dataset(X.iloc[tr], y[tr]); db = lgb.Dataset(X.iloc[va], y[va], reference=da)
    m = lgb.train(P, da, 3000, valid_sets=[db], callbacks=[lgb.early_stopping(100, verbose=False)]); oof[va] = m.predict(X.iloc[va], num_iteration=m.best_iteration); its.append(m.best_iteration)
rounds = int(np.mean(its)); gb = lgb.train(P, lgb.Dataset(X, y), rounds); pt = gb.predict(Xt)
# logistic regression: log1p, median impute, missing count, standardise
med = X.median()
def prep(D): Z = np.log1p(D.clip(lower=0).fillna(med)); Z['nmiss'] = D.isna().sum(axis=1); return Z
Z = prep(X); mu, sd = Z.mean(), Z.std().replace(0, 1); Zs = (Z - mu) / sd; Zt = (prep(Xt) - mu) / sd
oof_lr = np.zeros(len(y))
for tr, va in skf.split(X, y):
    lr = LogisticRegression(C=0.05, max_iter=3000, class_weight='balanced'); lr.fit(Zs.iloc[tr], y[tr]); oof_lr[va] = lr.predict_proba(Zs.iloc[va])[:, 1]
lr = LogisticRegression(C=0.05, max_iter=3000, class_weight='balanced').fit(Zs, y); pl = lr.predict_proba(Zt)[:, 1]
# single-sensor rule: best feature and threshold by train cost
best = None
for c in X.columns:
    v = X[c].fillna(-1).values
    for q in np.unique(np.nanpercentile(X[c], np.arange(50, 99.9, 0.5))):
        k = cost(y, (v > q).astype(int))
        if best is None or k < best[2]: best = (c, float(q), k)
rule_flag = (Xt[best[0]].fillna(-1).values > best[1]).astype(int)
def pick(p_oof): ths = np.unique(np.quantile(p_oof, np.linspace(0.5, 0.999, 600))); ks = [cost(y, (p_oof >= t).astype(int)) for t in ths]; return float(ths[int(np.argmin(ks))])
tg, tl = pick(oof), pick(oof_lr)
def res(p, th, flag=None):
    f = (p >= th).astype(int) if flag is None else flag
    d = {'cost': cost(yt, f), 'fp': int(((f == 1) & (yt == 0)).sum()), 'fn': int(((f == 0) & (yt == 1)).sum()), 'flagged': int(f.sum()), 'caught': int(((f == 1) & (yt == 1)).sum())}
    if p is not None: d.update({'auc': float(roc_auc_score(yt, p)), 'pr_auc': float(average_precision_score(yt, p))})
    r = np.random.default_rng(RNG); bs = []
    for _ in range(1000): i = r.integers(0, len(yt), len(yt)); bs.append(cost(yt[i], f[i]))
    d['cost_ci'] = [float(np.percentile(bs, 2.5)), float(np.percentile(bs, 97.5))]; return d
R = {'Check no trucks': res(None, 0, np.zeros(len(yt), int)), 'Check every truck': res(None, 0, np.ones(len(yt), int)), 'Single-sensor rule': res(None, 0, rule_flag), 'Logistic regression': res(pl, tl), 'Gradient boosting': res(pt, tg)}
R['rule'] = {'feature': best[0], 'threshold': best[1]}; R['thresholds'] = {'gb': tg, 'lr': tl}; R['rounds'] = rounds
r = np.random.default_rng(RNG); dd = []
fg = (pt >= tg).astype(int); fl = (pl >= tl).astype(int)
for _ in range(1000): i = r.integers(0, len(yt), len(yt)); dd.append(cost(yt[i], fl[i]) - cost(yt[i], fg[i]))
R['gb_vs_lr_saving_ci'] = [float(np.percentile(dd, 2.5)), float(np.percentile(dd, 97.5))]
# cost against threshold on test (for the chart) and sensitivity to the cost ratio (threshold re-picked on OOF each time)
qs = np.unique(np.quantile(pt, np.linspace(0.80, 0.995, 120))); curve = [[float((pt >= t).mean()), cost(yt, (pt >= t).astype(int))] for t in qs]
sens = []
for ratio in [10, 20, 50, 100, 200]:
    ths = np.unique(np.quantile(oof, np.linspace(0.5, 0.999, 600))); t = float(ths[int(np.argmin([cost(y, (oof >= q).astype(int), 10, 10 * ratio) for q in ths]))])
    f = (pt >= t).astype(int); sens.append([ratio, cost(yt, f, 10, 10 * ratio), cost(yt, np.ones(len(yt), int), 10, 10 * ratio), cost(yt, np.zeros(len(yt), int), 10, 10 * ratio), int(f.sum()), int(((f == 1) & (yt == 1)).sum())])
# score bands on test
qb = pd.qcut(pd.Series(pt).rank(method='first'), 20, labels=False); bands = [[int(b), int((qb == b).sum()), int(yt[qb.values == b].sum())] for b in range(20)]
imp = sorted([[n, float(v)] for n, v in zip(gb.feature_name(), gb.feature_importance('gain'))], key=lambda x: -x[1]); tt = sum(v for _, v in imp); imp = [[n, v / tt, float(miss[n])] for n, v in imp[:10]]
mp = [[k, float(y[(mr.values >= a) & (mr.values < b)].mean()) if ((mr.values >= a) & (mr.values < b)).any() else None, int(((mr.values >= a) & (mr.values < b)).sum())] for k, a, b in [('0', 0, 1), ('1–5', 1, 6), ('6–10', 6, 11), ('11–50', 11, 51), ('51+', 51, 999)]]
dump({'meta': run_meta(len(X) + len(Xt), {'source': 'APS Failure at Scania Trucks, UCI ML Repository 421 (Scania CV AB, 2016)'}), 'audit': A, 'results': R, 'curve': curve, 'sens': sens, 'bands': bands, 'imp': imp, 'miss_pos': mp}, OUT)
print(A); print({k: v for k, v in R.items()}); print(sens)
