"""Maintenance-demand dashboard with alerts. Data: NYC Open Data 311 Service Requests (Socrata erm2-nwe9), agency HPD,
daily counts for 10 housing-maintenance complaint types, 2022-01-01 to 2026-11-17; daily temperature from Open-Meteo archive (NYC).
Forecast is 7 days ahead using only data available 7 days earlier, plus temperature for the target day (observed here, a forecast in use).
Test period: the last 365 days. Baseline: same weekday last week (seasonal naive), and the 4-week same-weekday average."""
import pandas as pd, numpy as np, json
from sklearn.ensemble import HistGradientBoostingRegressor
c = pd.read_csv('/workspace/cs6/data/nyc311_daily.csv'); c['d'] = pd.to_datetime(c.d).dt.normalize()
w = json.load(open('/workspace/cs6/data/nyc_weather.json'))['daily']; W = pd.DataFrame(dict(d=pd.to_datetime(w['time']), t=w['temperature_2m_mean'], tmin=w['temperature_2m_min']))
P = c.pivot_table(index='d', columns='complaint_type', values='n', aggfunc='sum').asfreq('D').fillna(0)
last = P.index.max(); P = P[P.index < last]  # last day may be partial
types = list(P.sum().sort_values(ascending=False).index)
test0 = P.index.max() - pd.Timedelta(days=364); H = 7
res = {}; alerts_all = []
for t in types:
    s = P[t]; df = pd.DataFrame(dict(y=s)); df = df.join(W.set_index('d'))
    for L in [7, 8, 9, 10, 11, 12, 13, 14, 21, 28, 35, 364]: df[f'l{L}'] = s.shift(L)
    df['m7'] = s.shift(H).rolling(7).mean(); df['m28'] = s.shift(H).rolling(28).mean(); df['dow'] = df.index.dayofweek; df['doy'] = df.index.dayofyear
    df['t'] = df.t.interpolate(); df['tmin'] = df.tmin.interpolate(); df['hd'] = (18 - df.t).clip(lower=0); df['hol'] = ((df.index.month == 12) & (df.index.day >= 24)) | ((df.index.month == 1) & (df.index.day == 1)) | ((df.index.month == 7) & (df.index.day == 4))
    df = df.dropna(subset=['l364', 'm28']); F = [x for x in df.columns if x != 'y']
    tr = df[df.index < test0]; te = df[df.index >= test0]
    cal = tr[tr.index >= test0 - pd.Timedelta(days=180)]; fit = tr[tr.index < cal.index.min()]
    m0 = HistGradientBoostingRegressor(max_iter=400, learning_rate=.04, random_state=0).fit(fit[F], fit.y)
    r = (cal.y - m0.predict(cal[F])) / np.sqrt(np.maximum(m0.predict(cal[F]), 1)); q = float(np.quantile(r, .975))
    m = HistGradientBoostingRegressor(max_iter=400, learning_rate=.04, random_state=0).fit(tr[F], tr.y)
    f = np.maximum(m.predict(te[F]), 0); up = f + q * np.sqrt(np.maximum(f, 1)); yv = te.y.values
    sn = te.l7.values; s4 = te[['l7', 'l14', 'l21', 'l28']].mean(axis=1).values
    mae = lambda a: float(np.mean(np.abs(yv - a))); wape = lambda a: float(np.sum(np.abs(yv - a)) / np.sum(yv))
    al = te.index[yv > up]
    for dd in al: alerts_all.append(dict(type=t, d=str(dd.date()), y=int(te.y[dd]), f=float(f[te.index.get_loc(dd)]), up=float(up[te.index.get_loc(dd)])))
    wk = pd.DataFrame(dict(y=yv, f=f, sn=sn), index=te.index).resample('W').sum()
    res[t] = dict(total=int(s.sum()), per_day=float(s[s.index >= test0].mean()), wape=wape(f), wape_sn=wape(sn), wape_s4=wape(s4), mae=mae(f), mae_sn=mae(sn), cover=float(np.mean(yv <= up)), alerts=int(len(al)),
                  week=[[str(i.date()), float(a), float(b)] for i, a, b in zip(wk.index, wk.y, wk.f)],
                  monthly=[[str(i.date()), float(v)] for i, v in s.resample('MS').mean().items()])
    if t == 'HEAT/HOT WATER':
        res[t]['daily'] = [[str(i.date()), float(a), float(b), float(u), float(tt)] for i, a, b, u, tt in zip(te.index, yv, f, up, te.t)]
        hb = df.assign(tb=(df.t // 3) * 3).groupby('tb').y.mean(); res[t]['temp_curve'] = [[float(k), float(v)] for k, v in hb.items()]
tot = {k: sum(res[t][k] * res[t]['per_day'] for t in types) / sum(res[t]['per_day'] for t in types) for k in ['wape', 'wape_sn', 'wape_s4']}
out = dict(types=types, test=[str(test0.date()), str(P.index.max().date())], start=str(P.index.min().date()), total=int(P.values.sum()), res=res, overall=tot, alerts=sorted(alerts_all, key=lambda a: -(a['y'] - a['up']))[:40], n_alerts=len(alerts_all))
json.dump(out, open('/workspace/cs6/out/ops311.json', 'w'))
print(out['total'], out['test'], tot, len(alerts_all))
for t in types: r = res[t]; print(f"{t:22s} {r['per_day']:7.1f} wape {r['wape']:.3f} sn {r['wape_sn']:.3f} s4 {r['wape_s4']:.3f} cover {r['cover']:.3f} alerts {r['alerts']}")
print(out['alerts'][:8])
