"""FH1 · Which hotel bookings will cancel? Out-of-time cancellation model + resale (overbooking) simulation.
Data: Antonio, Almeida & Nunes (2019) Hotel Booking Demand, Data in Brief 22:41-49, CC BY 4.0 (TidyTuesday copy).
Run: python hotel.py hotels.csv out.json"""
import sys, numpy as np, pandas as pd, lightgbm as lgb
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import make_pipeline
from sklearn.metrics import roc_auc_score
from common import run_meta, dump, score, calib, roc_pts, boot_ci, RNG
src, OUT = sys.argv[1], sys.argv[2]
raw = pd.read_csv(src); A = {'rows': len(raw), 'cols': raw.shape[1], 'hotels': raw.hotel.value_counts().to_dict()}
M = {m: i + 1 for i, m in enumerate(['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'])}
raw['arr'] = pd.to_datetime(dict(year=raw.arrival_date_year, month=raw.arrival_date_month.map(M), day=raw.arrival_date_day_of_month))
raw['booked'] = raw.arr - pd.to_timedelta(raw.lead_time, unit='D')
A['arr_start'], A['arr_end'] = str(raw.arr.min().date()), str(raw.arr.max().date())
A['cancel_rate'] = float(raw.is_canceled.mean())
A['exact_dups'] = int(raw.duplicated().sum())
A['zero_guests'] = int(((raw.adults + raw.children.fillna(0) + raw.babies) == 0).sum())
A['adr_neg'] = int((raw.adr < 0).sum()); A['adr_max'] = float(raw.adr.max()); A['adr_zero'] = int((raw.adr == 0).sum())
A['children_na'] = int(raw.children.isna().sum()); A['country_na'] = int(raw.country.isna().sum()); A['agent_null'] = int((raw.agent.astype(str) == 'NULL').sum()); A['company_null'] = int((raw.company.astype(str) == 'NULL').sum())
A['nonrefund_cancel_rate'] = float(raw[raw.deposit_type == 'Non Refund'].is_canceled.mean()); A['nonrefund_n'] = int((raw.deposit_type == 'Non Refund').sum())
A['status_leak'] = pd.crosstab(raw.reservation_status, raw.is_canceled).to_dict()
A['room_changed_cancel'] = float(raw[raw.reserved_room_type != raw.assigned_room_type].is_canceled.mean()); A['room_same_cancel'] = float(raw[raw.reserved_room_type == raw.assigned_room_type].is_canceled.mean())
# cleaning: drop impossible rows and extreme ADR; keep exact duplicates (likely separate rooms on group bookings) but report them
d = raw[((raw.adults + raw.children.fillna(0) + raw.babies) > 0) & (raw.adr >= 0) & (raw.adr < 1000)].copy()
A['kept'] = len(d)
d['children'] = d.children.fillna(0); d['country'] = d.country.fillna('UNK')
top = d.country.value_counts().index[:25]; d['country_g'] = d.country.where(d.country.isin(top), 'Other')
d['agent_g'] = d.agent.astype(str); ta = d.agent_g.value_counts().index[:30]; d['agent_g'] = d.agent_g.where(d.agent_g.isin(ta), 'Other')
d['has_company'] = (d.company.astype(str) != 'NULL').astype(int); d['nights'] = d.stays_in_weekend_nights + d.stays_in_week_nights
d['arr_month'] = d.arr.dt.month; d['arr_dow'] = d.arr.dt.dayofweek; d['book_dow'] = d.booked.dt.dayofweek; d['kids'] = d.children + d.babies
# fields known at booking time only (assigned room, booking changes, status fields are filled in later = leakage)
NUM = ['lead_time', 'nights', 'adults', 'kids', 'is_repeated_guest', 'previous_cancellations', 'previous_bookings_not_canceled', 'days_in_waiting_list', 'adr', 'required_car_parking_spaces', 'total_of_special_requests', 'has_company', 'arr_month', 'arr_dow', 'book_dow']
CAT = ['hotel', 'meal', 'market_segment', 'distribution_channel', 'reserved_room_type', 'deposit_type', 'customer_type', 'agent_g', 'country_g']
LEAK = ['booking_changes', 'room_changed']
d['room_changed'] = (d.reserved_room_type != d.assigned_room_type).astype(int)
# out-of-time split by arrival date; model selection on a validation window inside training
tr = d[d.arr < '2017-01-01']; te = d[d.arr >= '2017-01-01']; fit = tr[tr.arr < '2016-07-01']; val = tr[tr.arr >= '2016-07-01']
def lgbm(cols_num, cols_cat, a, b):
    X = lambda z: pd.concat([z[cols_num], z[cols_cat].astype('category')], axis=1)
    Xa = X(a); Xb = X(b)
    for c in cols_cat: Xb[c] = pd.Categorical(Xb[c], categories=Xa[c].cat.categories)
    m = lgb.LGBMClassifier(n_estimators=700, learning_rate=0.03, num_leaves=63, min_child_samples=40, subsample=0.8, subsample_freq=1, colsample_bytree=0.8, random_state=RNG, verbose=-1, n_jobs=4).fit(Xa, a.is_canceled)
    return m, m.predict_proba(Xb)[:, 1], Xb
def logit(cols_num, cols_cat, a, b):
    ct = ColumnTransformer([('n', StandardScaler(), cols_num), ('c', OneHotEncoder(handle_unknown='ignore', min_frequency=30), cols_cat)])
    m = make_pipeline(ct, LogisticRegression(max_iter=2000, C=0.5)).fit(a[cols_num + cols_cat], a.is_canceled)
    return m.predict_proba(b[cols_num + cols_cat])[:, 1]
# simple rule a revenue manager might use: historical cancel rate of the booking's (hotel, market segment, deposit type) cell
def rule(a, b):
    r = a.groupby(['hotel', 'market_segment', 'deposit_type']).is_canceled.mean(); g = a.is_canceled.mean()
    return b.set_index(['hotel', 'market_segment', 'deposit_type']).index.map(lambda k: r.get(k, g)).values.astype(float)
val_auc = {'Rule: segment × deposit history': roc_auc_score(val.is_canceled, rule(fit, val)), 'Logistic regression': roc_auc_score(val.is_canceled, logit(NUM, CAT, fit, val)), 'Gradient boosting': roc_auc_score(val.is_canceled, lgbm(NUM, CAT, fit, val)[1])}
chosen = max(['Logistic regression', 'Gradient boosting'], key=lambda k: val_auc[k])
y = te.is_canceled.values
P = {'Rule: segment × deposit history': rule(tr, te), 'Logistic regression': logit(NUM, CAT, tr, te)}
gbm, P['Gradient boosting'], Xte = lgbm(NUM, CAT, tr, te)
models = {k: score(y, v) for k, v in P.items()}
pc = P[chosen]; pr = P['Rule: segment × deposit history']
rng = np.random.default_rng(RNG); n = len(y); dd = []
for _ in range(1000):
    i = rng.integers(0, n, n); dd.append(roc_auc_score(y[i], pc[i]) - roc_auc_score(y[i], pr[i]))
diff = {'vs': 'Rule: segment × deposit history', 'diff': models[chosen]['auc'] - models['Rule: segment × deposit history']['auc'], 'ci': [float(np.percentile(dd, 2.5)), float(np.percentile(dd, 97.5))]}
# leakage experiment: add fields filled in after booking
_, p_leak, _ = lgbm(NUM + LEAK, CAT, tr, te); leak = {'auc_with_leaky_fields': float(roc_auc_score(y, p_leak)), 'auc_clean': models['Gradient boosting']['auc']}
# random-split optimism: the same model with a random split instead of out-of-time
rs = d.sample(frac=1, random_state=RNG); cut = int(len(rs) * .8); _, p_rand, _ = lgbm(NUM, CAT, rs.iloc[:cut], rs.iloc[cut:]); leak['auc_random_split'] = float(roc_auc_score(rs.iloc[cut:].is_canceled, p_rand))
# ethics: drop nationality
_, p_nocty, _ = lgbm(NUM, [c for c in CAT if c != 'country_g'], tr, te); eth = {'auc_without_country': float(roc_auc_score(y, p_nocty)), 'auc_with_country': models['Gradient boosting']['auc']}
eth['country_rate_top'] = d.groupby('country_g').is_canceled.agg(['mean', 'size']).sort_values('size', ascending=False).head(8).reset_index().values.tolist()
imp = sorted(zip(Xte.columns, gbm.booster_.feature_importance('gain')), key=lambda x: -x[1])[:12]
# ---------- operations: rooms freed per arrival night (hotel × date) and a resale policy
T = te.assign(p=pc, p_rule=pr)
day = T.groupby(['hotel', 'arr']).agg(book=('is_canceled', 'size'), canc=('is_canceled', 'sum'), exp=('p', 'sum'), exp_rule=('p_rule', 'sum'), adr=('adr', 'mean')).reset_index()
flat = tr.groupby('hotel').is_canceled.mean(); day['exp_flat'] = day.book * day.hotel.map(flat)
ops = {k: float(np.mean(np.abs(day.canc - day[c]))) for k, c in [('model', 'exp'), ('rule', 'exp_rule'), ('flat', 'exp_flat')]}
ops['mean_canc'] = float(day.canc.mean()); ops['n_nights'] = len(day)
ops['mae_ci_model'] = boot_ci(day.canc.values, day.exp.values, lambda a, b: np.mean(np.abs(a - b)))
# resale: resell floor(s × expected cancellations) rooms per night. Each resold room that frees up earns ADR;
# each resold room that does not free up means walking a guest, costed at 2 × ADR (indicative assumption).
def policy(col, s):
    r = np.floor(s * day[col]); gain = np.minimum(r, day.canc) * day.adr; walk = np.maximum(r - day.canc, 0) * day.adr * 2
    return float((gain - walk).sum()), int(np.maximum(r - day.canc, 0).sum())
curve = [[round(s, 2), *policy('exp', s), *policy('exp_flat', s)] for s in np.arange(0.1, 1.31, 0.05)]
bm = max(curve, key=lambda r: r[1]); bf = max(curve, key=lambda r: r[3])
pol = {'best_model': {'s': bm[0], 'net': bm[1], 'walks': bm[2]}, 'best_flat': {'s': bf[0], 'net': bf[3], 'walks': bf[4]}, 'curve': curve, 'walk_cost_x_adr': 2}
# charts
lt = d.assign(b=pd.cut(d.lead_time, [-1, 7, 30, 60, 90, 180, 270, 365, 800], labels=['0–7', '8–30', '31–60', '61–90', '91–180', '181–270', '271–365', '365+'])).groupby(['b', 'hotel'], observed=True).is_canceled.agg(['mean', 'size']).reset_index()
lead_curve = {h: [[str(r.b), float(r['mean']), int(r['size'])] for _, r in lt[lt.hotel == h].iterrows()] for h in d.hotel.unique()}
seg = d.groupby('market_segment').is_canceled.agg(['mean', 'size']).sort_values('size', ascending=False).reset_index().values.tolist()
dep = d.groupby('deposit_type').is_canceled.agg(['mean', 'size']).reset_index().values.tolist()
monthly = d.groupby([d.arr.dt.to_period('M').astype(str), 'hotel']).is_canceled.mean().unstack().round(4).reset_index().values.tolist()
dline = day[day.hotel == 'City Hotel'].sort_values('arr')[['arr', 'canc', 'exp', 'exp_flat']]; dline['arr'] = dline.arr.dt.strftime('%Y-%m-%d')
deciles = pd.DataFrame({'y': y, 'p': pc}).assign(q=lambda z: pd.qcut(z.p.rank(method='first'), 10, labels=False)).groupby('q').y.mean().round(4).tolist()
errs = T.assign(pred=(T.p >= 0.5).astype(int)); fn = errs[(errs.pred == 0) & (errs.is_canceled == 1)]; tp = errs[(errs.pred == 1) & (errs.is_canceled == 1)]
err = {'fn': len(fn), 'tp': len(tp), 'fp': int(((errs.pred == 1) & (errs.is_canceled == 0)).sum()), 'fn_med_lead': float(fn.lead_time.median()), 'tp_med_lead': float(tp.lead_time.median()), 'fn_special_req': float((fn.total_of_special_requests > 0).mean()), 'fn_direct': float((fn.market_segment == 'Direct').mean()), 'fn_repeat': float(fn.is_repeated_guest.mean())}
out = {'meta': run_meta(len(raw), {'train_n': len(tr), 'test_n': len(te), 'test_pos': int(y.sum())}), 'audit': A, 'val_auc': val_auc, 'chosen': chosen, 'models': models, 'diff': diff, 'leak': leak, 'ethics': eth,
       'roc': {k: roc_pts(y, v) for k, v in P.items()}, 'calib': {k: calib(y, v) for k, v in P.items() if k != 'Rule: segment × deposit history'}, 'importance': [[a, float(b)] for a, b in imp],
       'ops': ops, 'policy': pol, 'lead_curve': lead_curve, 'segments': seg, 'deposit': dep, 'monthly': monthly, 'dline': dline.round(2).values.tolist(), 'deciles': deciles, 'errors': err}
dump(out, OUT); print(val_auc, chosen, {k: round(v['auc'], 3) for k, v in models.items()}, diff, leak, eth['auc_without_country'], ops, pol['best_model'], pol['best_flat'])
