"""LO1 · Will this flight arrive late? Delay risk and knock-on delays on US domestic flights, 2024.
Data: US Bureau of Transportation Statistics, Reporting Carrier On-Time Performance (PREZIP monthly files, Jan-Dec 2024).
Train Jan-Sep, choose settings on Aug-Sep, test Oct-Dec (out of time). Run: python logistics.py DATA_DIR out.json"""
import sys, glob, zipfile, io, numpy as np, pandas as pd, lightgbm as lgb
from sklearn.metrics import roc_auc_score, average_precision_score
from common import run_meta, dump, calib, RNG
DIR, OUT = sys.argv[1], sys.argv[2]
COLS = ['Month', 'DayofMonth', 'DayOfWeek', 'FlightDate', 'Reporting_Airline', 'Tail_Number', 'Origin', 'Dest', 'CRSDepTime', 'CRSArrTime', 'DepDelay', 'ArrDelay', 'ArrDel15',
        'Cancelled', 'CancellationCode', 'Diverted', 'CRSElapsedTime', 'Distance', 'CarrierDelay', 'WeatherDelay', 'NASDelay', 'SecurityDelay', 'LateAircraftDelay']
fr = []
for f in sorted(glob.glob(DIR + '/bts_2024_*.zip'), key=lambda s: int(s.split('_')[-1].split('.')[0])):
    z = zipfile.ZipFile(f); nm = [x for x in z.namelist() if x.endswith('.csv')][0]
    fr.append(pd.read_csv(z.open(nm), usecols=COLS, dtype={'Tail_Number': 'string', 'CancellationCode': 'string'}, low_memory=False))
d = pd.concat(fr, ignore_index=True); del fr
A = {'rows': len(d), 'cancelled': int(d.Cancelled.sum()), 'diverted': int(d.Diverted.sum()), 'airlines': int(d.Reporting_Airline.nunique()), 'airports': int(pd.concat([d.Origin, d.Dest]).nunique()),
     'no_tail': int(d.Tail_Number.isna().sum()), 'cancel_codes': d.CancellationCode.value_counts().to_dict(), 'dup': int(d.duplicated(['FlightDate', 'Reporting_Airline', 'Tail_Number', 'CRSDepTime', 'Origin']).sum())}
d['dep_h'] = (d.CRSDepTime // 100).clip(0, 23); d['dep_min'] = (d.CRSDepTime // 100) * 60 + d.CRSDepTime % 100; d['arr_min'] = (d.CRSArrTime // 100) * 60 + d.CRSArrTime % 100
d['date'] = pd.to_datetime(d.FlightDate)
# same aircraft through the day: leg number and planned turnaround (minutes between previous scheduled arrival and this departure)
d = d.sort_values(['Tail_Number', 'date', 'dep_min']).reset_index(drop=True)
g = d.groupby(['Tail_Number', 'date'], sort=False)
d['leg'] = g.cumcount() + 1
prev_arr = g.arr_min.shift(1); prev_dest = g.Dest.shift(1); d['turn'] = (d.dep_min - prev_arr).where(prev_dest == d.Origin)
d.loc[d.turn < 0, 'turn'] = np.nan
d['prev_arr_delay'] = g.ArrDelay.shift(1).where(prev_dest == d.Origin)  # known before departure on the day
d.loc[d.Tail_Number.isna(), ['leg', 'turn', 'prev_arr_delay']] = np.nan
A['legs_chained'] = float(d.turn.notna().mean())
ok = d[(d.Cancelled == 0) & (d.Diverted == 0) & d.ArrDel15.notna()].copy()
A['flown'] = len(ok); A['late_rate'] = float(ok.ArrDel15.mean())
# delay causes: minutes by cause among late flights
cz = ['CarrierDelay', 'WeatherDelay', 'NASDelay', 'SecurityDelay', 'LateAircraftDelay']; late = ok[ok.ArrDel15 == 1]
causes = {c: float(late[c].sum()) for c in cz}; tot = sum(causes.values()); causes = {k: v / tot for k, v in causes.items()}
cause_m = late.groupby('Month')[cz].sum(); cause_m = (cause_m.div(cause_m.sum(1), axis=0)).round(4).reset_index().values.tolist()
month = d.groupby('Month').agg(n=('Cancelled', 'size'), canc=('Cancelled', 'mean')).join(ok.groupby('Month').ArrDel15.mean().rename('late')).reset_index().values.tolist()
hour = ok.groupby('dep_h').agg(n=('ArrDel15', 'size'), late=('ArrDel15', 'mean'), mean_delay=('ArrDelay', 'mean')).reset_index().values.tolist()
legs = ok[ok.leg.notna()].assign(L=ok.leg.clip(upper=7)).groupby('L').agg(n=('ArrDel15', 'size'), late=('ArrDel15', 'mean')).reset_index().values.tolist()
# knock-on: how often a late inbound leg makes this one late
kn = ok[ok.prev_arr_delay.notna()].copy(); kb = pd.cut(kn.prev_arr_delay, [-1000, 0, 15, 30, 60, 120, 10000], labels=['early/on time', '1–15', '16–30', '31–60', '61–120', '120+'])
knock = kn.groupby(kb, observed=True).agg(n=('ArrDel15', 'size'), late=('ArrDel15', 'mean')).reset_index().values.tolist()
tb = pd.cut(kn.turn, [0, 30, 45, 60, 90, 120, 240, 2000], labels=['≤30', '31–45', '46–60', '61–90', '91–120', '121–240', '240+'])
turn = kn.assign(tb=tb, big=kn.prev_arr_delay > 30).groupby(['tb', 'big'], observed=True).ArrDel15.agg(['mean', 'size']).reset_index().values.tolist()
air = ok.groupby('Reporting_Airline').agg(n=('ArrDel15', 'size'), late=('ArrDel15', 'mean'), mean_delay=('ArrDelay', 'mean')).reset_index().sort_values('n', ascending=False).values.tolist()
top_ap = ok.Origin.value_counts().head(30).index; ap = ok[ok.Origin.isin(top_ap)].groupby('Origin').agg(n=('ArrDel15', 'size'), late=('ArrDel15', 'mean'), dep_delay=('DepDelay', 'mean')).reset_index().sort_values('n', ascending=False).values.tolist()
# models
F_PLAN = ['Reporting_Airline', 'Origin', 'Dest', 'dep_h', 'DayOfWeek', 'Month', 'DayofMonth', 'Distance', 'CRSElapsedTime', 'leg', 'turn']
F_DAY = F_PLAN + ['prev_arr_delay']
CAT = ['Reporting_Airline', 'Origin', 'Dest']
for c in CAT: ok[c] = ok[c].astype('category')
tr = ok[ok.Month <= 9]; te = ok[ok.Month >= 10]
fit = tr[tr.Month <= 7]; val = tr[tr.Month >= 8]
rng = np.random.default_rng(RNG)
def sub(df, k): return df.iloc[rng.choice(len(df), min(k, len(df)), replace=False)]
fit_s, val_s, tr_s = sub(fit, 1_200_000), sub(val, 300_000), sub(tr, 1_500_000)
P = dict(objective='binary', learning_rate=0.08, num_leaves=127, min_child_samples=200, subsample=0.8, subsample_freq=1, colsample_bytree=0.8, reg_lambda=1.0, verbose=-1, seed=RNG, num_threads=3, max_cat_to_onehot=8, cat_smooth=20)
def train(a, F, rounds=None, b=None):
    da = lgb.Dataset(a[F], a.ArrDel15, categorical_feature=[c for c in CAT if c in F])
    if rounds is None:
        db = lgb.Dataset(b[F], b.ArrDel15, reference=da); m = lgb.train(P, da, 2000, valid_sets=[db], callbacks=[lgb.early_stopping(50, verbose=False)]); return m, m.best_iteration
    return lgb.train(P, da, rounds), rounds
mp, rp = train(fit_s, F_PLAN, b=val_s); md, rd = train(fit_s, F_DAY, b=val_s)
val_auc = {'plan': float(roc_auc_score(val_s.ArrDel15, mp.predict(val_s[F_PLAN], num_iteration=rp))), 'day': float(roc_auc_score(val_s.ArrDel15, md.predict(val_s[F_DAY], num_iteration=rd)))}
mp, _ = train(tr_s, F_PLAN, rp); md, _ = train(tr_s, F_DAY, rd)
y = te.ArrDel15.values.astype(int)
# baselines: route x hour historical rate from training months (fallback: airline rate), and "late inbound > 15 min" rule
rk = tr.assign(k=tr.Origin.astype(str) + tr.Dest.astype(str) + tr.dep_h.astype(str)).groupby('k').ArrDel15.agg(['mean', 'size'])
k_te = te.Origin.astype(str) + te.Dest.astype(str) + te.dep_h.astype(str); base = k_te.map(rk['mean'].where(rk['size'] >= 20))
al = tr.groupby('Reporting_Airline', observed=True).ArrDel15.mean(); base = base.fillna(te.Reporting_Airline.map(al).astype(float)).fillna(tr.ArrDel15.mean()).values
inb = te.prev_arr_delay.fillna(0).values; rule = (inb > 15).astype(float) + base * 0.01
PR = {'Route × hour history': base, 'Plan model (a day ahead)': mp.predict(te[F_PLAN]), 'Inbound late > 15 min rule': rule, 'Day-of model (+ inbound delay)': md.predict(te[F_DAY])}
def bci(fn, a, B=200, k=400_000):
    r = np.random.default_rng(RNG); out = []
    for _ in range(B):
        i = r.integers(0, len(y), min(k, len(y))); out.append(fn(y[i], a[i]))
    return [float(np.percentile(out, 2.5)), float(np.percentile(out, 97.5))]
R = {}
for k, p in PR.items():
    R[k] = {'auc': float(roc_auc_score(y, p)), 'pr_auc': float(average_precision_score(y, p)), 'auc_ci': bci(roc_auc_score, p), 'pr_ci': bci(average_precision_score, p)}
r2 = np.random.default_rng(RNG); dd = []
for _ in range(200):
    i = r2.integers(0, len(y), 400_000); dd.append(roc_auc_score(y[i], PR['Plan model (a day ahead)'][i]) - roc_auc_score(y[i], base[i]))
R['plan_vs_base'] = [float(np.percentile(dd, 2.5)), float(np.percentile(dd, 97.5))]
# budget: flag top x% of flights for a proactive rebooking / gate check
budget = []
for q in [0.05, 0.1, 0.2, 0.3]:
    row = [q]
    for k, p in PR.items(): idx = np.argsort(-p)[:int(q * len(y))]; row.append(float(y[idx].mean()))
    budget.append(row)
# by month in test, calibration of plan model
te = te.assign(pp=PR['Plan model (a day ahead)'], pd_=PR['Day-of model (+ inbound delay)'])
cal_m = te.groupby('Month').agg(n=('ArrDel15', 'size'), late=('ArrDel15', 'mean'), plan=('pp', 'mean'), day=('pd_', 'mean')).reset_index().values.tolist()
cal = calib(y, PR['Plan model (a day ahead)'], 10)
imp = sorted([[n, float(v)] for n, v in zip(md.feature_name(), md.feature_importance('gain'))], key=lambda x: -x[1]); tt = sum(v for _, v in imp); imp = [[n, v / tt] for n, v in imp]
# random split check on the plan model
rs = sub(ok, 1_500_000); msk = np.random.default_rng(RNG).random(len(rs)) < 0.8; mr, _ = train(rs[msk], F_PLAN, rp); R['random_split_auc_plan'] = float(roc_auc_score(rs[~msk].ArrDel15, mr.predict(rs[~msk][F_PLAN])))
# daily late share (whole year), for a calendar strip
daily = ok.groupby('date').ArrDel15.mean(); daily = [[str(k.date()), round(float(v), 4)] for k, v in daily.items()]
worst = sorted(daily, key=lambda r: -r[1])[:6]
A.update({'train_rows': len(tr), 'train_used': len(tr_s), 'test_rows': len(te), 'test_late': float(y.mean()), 'val_auc': val_auc, 'rounds': [int(rp), int(rd)]})
dump({'meta': run_meta(len(d), {'source': 'BTS Reporting Carrier On-Time Performance, Jan-Dec 2024'}), 'audit': A, 'results': R, 'budget': budget, 'bcols': list(PR.keys()), 'causes': causes, 'cause_m': cause_m,
      'month': month, 'hour': hour, 'legs': legs, 'knock': knock, 'turn': turn, 'air': air, 'ap': ap, 'cal_m': cal_m, 'cal': cal, 'imp': imp, 'daily': daily, 'worst': worst}, OUT)
print(A); print(R); print(budget); print(causes); print(cal_m)
