"""EN1 · Day-ahead electricity demand forecast for Victoria (AEMO VIC1), with weather forecasts.
Run: python energy.py <data_dir> <out.json>. Data: AEMO aggregated price & demand CSVs (VIC1, Jan 2023 – Oct 2025),
Open-Meteo historical weather forecasts and observations for Melbourne (GMT)."""
import sys, glob, json, numpy as np, pandas as pd, lightgbm as lgb, holidays
from sklearn.linear_model import Ridge
from common import run_meta, dump, RNG
D, OUT = sys.argv[1], sys.argv[2]
# ---------- load AEMO 5-min demand (SETTLEMENTDATE = end of interval, market time AEST = UTC+10, no daylight saving)
raw = pd.concat([pd.read_csv(f) for f in sorted(glob.glob(f'{D}/aemo_2*.csv'))], ignore_index=True)
A = {'raw_rows': len(raw), 'files': len(glob.glob(f'{D}/aemo_2*.csv'))}
raw['end'] = pd.to_datetime(raw.SETTLEMENTDATE, format='%Y/%m/%d %H:%M:%S')
A['dups'] = int(raw.duplicated('end').sum()); raw = raw.drop_duplicates('end')
A['region'] = sorted(raw.REGION.unique().tolist()); A['nonpos_demand'] = int((raw.TOTALDEMAND <= 0).sum())
full = pd.date_range(raw.end.min(), raw.end.max(), freq='5min'); A['missing_5min'] = int(len(full) - raw.end.nunique())
A['start'], A['end'] = str(raw.end.min()), str(raw.end.max())
raw['utc'] = (raw.end - pd.Timedelta(minutes=5)).dt.floor('h') - pd.Timedelta(hours=10)
h = raw.groupby('utc').agg(mw=('TOTALDEMAND', 'mean'), k=('TOTALDEMAND', 'size'), rrp=('RRP', 'mean')).reset_index()
A['incomplete_hours'] = int((h.k < 12).sum()); h = h[h.k >= 10].drop(columns='k')
A['price_neg_share'] = float((raw.RRP < 0).mean()); A['price_max'] = float(raw.RRP.max())
# ---------- weather (GMT): historical forecasts = what a forecaster would have had; observations = truth
def wx(f, pre):
    j = json.load(open(f'{D}/{f}'))['hourly']; w = pd.DataFrame(j); w['utc'] = pd.to_datetime(w.time); w = w.drop(columns='time')
    return w.rename(columns={c: pre + c for c in w.columns if c != 'utc'})
# Open-Meteo previous-runs API: *_previous_day1 = the forecast issued about one day earlier (a true day-ahead forecast).
j = json.load(open(f'{D}/melb_prev_gmt.json'))['hourly']; W = pd.DataFrame(j); W['utc'] = pd.to_datetime(W.time); W = W.drop(columns='time')
W = W.rename(columns={'temperature_2m_previous_day1': 'fc_temperature_2m', 'relative_humidity_2m_previous_day1': 'fc_rh', 'shortwave_radiation_previous_day1': 'fc_sw', 'cloud_cover_previous_day1': 'fc_cloud', 'temperature_2m': 'ob_temperature_2m', 'shortwave_radiation': 'ob_sw'})
A['wx_fc_temp_missing'] = int(W.fc_temperature_2m.isna().sum()); A['wx_fc_solar_from'] = str(W.loc[W.fc_sw.notna(), 'utc'].min())
A['wx_fc_vs_obs_mae'] = float((W.fc_temperature_2m - W.ob_temperature_2m).abs().mean())
df = h.merge(W, on='utc', how='left').sort_values('utc').reset_index(drop=True)
df['loc'] = df.utc.dt.tz_localize('UTC').dt.tz_convert('Australia/Melbourne')
df['date'] = df['loc'].dt.date; df['hour'] = df['loc'].dt.hour; df['dow'] = df['loc'].dt.dayofweek; df['month'] = df['loc'].dt.month
doy = df['loc'].dt.dayofyear; df['doy_s'] = np.sin(2 * np.pi * doy / 365.25); df['doy_c'] = np.cos(2 * np.pi * doy / 365.25)
hol = holidays.Australia(subdiv='VIC', years=[2023, 2024, 2025]); df['hol'] = df.date.map(lambda d: d in hol).astype(int)
A['holidays'] = len(hol)
# lags known at local midnight before the target day: same hour yesterday (24 h), same hour last week (168 h)
s = df.set_index('utc').mw
df['lag24'] = (df.utc - pd.Timedelta(hours=24)).map(s); df['lag168'] = (df.utc - pd.Timedelta(hours=168)).map(s)
daily = df.groupby('date').agg(dmean=('mw', 'mean'), dmax=('mw', 'max'), fcmax=('fc_temperature_2m', 'max'), fcmin=('fc_temperature_2m', 'min'),
                               obmax=('ob_temperature_2m', 'max'), obmin=('ob_temperature_2m', 'min'), rrpmax=('rrp', 'max'), fcsw_day=('fc_sw', 'sum'), obsw_day=('ob_sw', 'sum')).reset_index()
daily.loc[daily.fcsw_day == 0, 'fcsw_day'] = np.nan
prev = daily[['date', 'dmean', 'dmax']].copy(); prev['date'] = prev.date.map(lambda d: d + pd.Timedelta(days=1)); prev.columns = ['date', 'y_dmean', 'y_dmax']
df = df.merge(prev, on='date', how='left').merge(daily[['date', 'fcmax', 'fcmin', 'obmax', 'obmin', 'fcsw_day', 'obsw_day']], on='date', how='left')
df = df.dropna(subset=['lag24', 'lag168', 'y_dmean', 'fc_temperature_2m']).reset_index(drop=True)
A['model_hours'] = len(df)
CAL = ['hour', 'dow', 'month', 'doy_s', 'doy_c', 'hol']; LAG = ['lag24', 'lag168', 'y_dmean', 'y_dmax']
FCT = ['fc_temperature_2m', 'fcmax', 'fcmin']; FC = FCT + ['fc_rh', 'fc_sw', 'fc_cloud', 'fcsw_day']; OB = ['ob_temperature_2m', 'obmax', 'obmin', 'ob_sw', 'obsw_day']
SETS = {'Gradient boosting + weather forecast': CAL + LAG + FC, 'Gradient boosting, no weather': CAL + LAG, 'Gradient boosting + temperature forecast only': CAL + LAG + FCT, 'Gradient boosting + observed weather (hindsight)': CAL + LAG + OB}
P = dict(n_estimators=600, learning_rate=0.03, num_leaves=31, min_child_samples=30, subsample=0.8, subsample_freq=1, colsample_bytree=0.9, random_state=RNG, verbose=-1, n_jobs=4)
# ---------- rolling origin: retrain at the start of each month of 2025, forecast that month day-ahead
origins = pd.date_range('2025-01-01', '2025-10-01', freq='MS'); preds = []; MAIN = 'Gradient boosting + weather forecast'
for o in origins:
    tr = df[df['loc'].dt.tz_localize(None) < o]; te = df[(df['loc'].dt.tz_localize(None) >= o) & (df['loc'].dt.tz_localize(None) < o + pd.offsets.MonthBegin(1))]
    if not len(te): continue
    out = te[['utc', 'loc', 'date', 'hour', 'mw', 'lag24', 'lag168', 'hol', 'fc_temperature_2m', 'fcmax', 'rrp', 'fc_sw']].copy(); out['origin'] = str(o.date())
    out['Same hour last week'] = te.lag168.values; out['Same hour yesterday'] = te.lag24.values
    X = lambda d: np.c_[pd.get_dummies(d.hour.astype('category').cat.set_categories(range(24))).values, pd.get_dummies(d.dow.astype('category').cat.set_categories(range(7))).values, d[['doy_s', 'doy_c', 'hol', 'fc_temperature_2m']].values, (d.fc_temperature_2m - 18) .clip(lower=0) ** 2, (18 - d.fc_temperature_2m).clip(lower=0) ** 2, d.lag168.values]
    out['Linear regression + weather'] = Ridge(alpha=1.0).fit(X(tr), tr.mw).predict(X(te))
    for name, cols in SETS.items():
        out[name] = lgb.LGBMRegressor(**P).fit(tr[cols], tr.mw).predict(te[cols])
    for q in (0.025, 0.1, 0.9, 0.975):
        out[f'q{q}'] = lgb.LGBMRegressor(objective='quantile', alpha=q, **P).fit(tr[CAL + LAG + FC], tr.mw).predict(te[CAL + LAG + FC])
    # split-conformal ranges: refit on all but the last 60 days of history, measure relative errors on those 60 days
    cut = o - pd.Timedelta(days=60); a = tr[tr['loc'].dt.tz_localize(None) < cut]; b = tr[tr['loc'].dt.tz_localize(None) >= cut]
    rel = (b.mw.values - lgb.LGBMRegressor(**P).fit(a[CAL + LAG + FC], a.mw).predict(b[CAL + LAG + FC])) / b.mw.values
    for lo, hi, tag in [(0.1, 0.9, '80'), (0.025, 0.975, '95')]:
        out[f'c{tag}lo'] = out[MAIN] * (1 + np.quantile(rel, lo)); out[f'c{tag}hi'] = out[MAIN] * (1 + np.quantile(rel, hi))
    if o == origins[-1]:
        m = lgb.LGBMRegressor(**P).fit(tr[CAL + LAG + FC], tr.mw); imp = dict(zip(CAL + LAG + FC, m.booster_.feature_importance('gain')))
    preds.append(out); print('origin', o.date(), len(te), flush=True)
R = pd.concat(preds, ignore_index=True)
METHODS = ['Same hour last week', 'Same hour yesterday', 'Linear regression + weather', 'Gradient boosting, no weather', 'Gradient boosting + temperature forecast only', 'Gradient boosting + weather forecast', 'Gradient boosting + observed weather (hindsight)']
MAIN = 'Gradient boosting + weather forecast'
for q in ('q0.025', 'q0.1'): R[q] = np.minimum(R[q], R[MAIN])
for q in ('q0.9', 'q0.975'): R[q] = np.maximum(R[q], R[MAIN])
rng = np.random.default_rng(RNG); days = R.date.unique(); di = {d: np.where(R.date.values == d)[0] for d in days}
BOOT = [np.concatenate([di[d] for d in rng.choice(days, len(days))]) for _ in range(1000)]  # day-block bootstrap
mape = lambda y, p: float(np.mean(np.abs(y - p) / y) * 100)
def ci(fn):
    v = [fn(i) for i in BOOT]; return [float(np.percentile(v, 2.5)), float(np.percentile(v, 97.5))]
y = R.mw.values; res = {}
for m in METHODS:
    p = R[m].values
    res[m] = {'mape': mape(y, p), 'mape_ci': ci(lambda i: mape(y[i], p[i])), 'mae': float(np.mean(np.abs(y - p))), 'rmse': float(np.sqrt(np.mean((y - p) ** 2))), 'bias': float(np.mean(p - y))}
pm = R[MAIN].values; best_base = min(['Same hour last week', 'Same hour yesterday', 'Linear regression + weather'], key=lambda m: res[m]['mape'])
pb = R[best_base].values
diff = {'vs': best_base, 'diff': res[best_base]['mape'] - res[MAIN]['mape'], 'ci': ci(lambda i: mape(y[i], pb[i]) - mape(y[i], pm[i]))}
pt = R['Gradient boosting + temperature forecast only'].values; diff_solar = {'diff': res['Gradient boosting + temperature forecast only']['mape'] - res[MAIN]['mape'], 'ci': ci(lambda i: mape(y[i], pt[i]) - mape(y[i], pm[i]))}
by_hour_t = [[int(hh), mape(g.mw.values, g['Gradient boosting + temperature forecast only'].values)] for hh, g in R.groupby('hour')]
pn = R['Gradient boosting, no weather'].values; diff_wx = {'diff': res['Gradient boosting, no weather']['mape'] - res[MAIN]['mape'], 'ci': ci(lambda i: mape(y[i], pn[i]) - mape(y[i], pm[i]))}
cov_c = {'pi80': float(np.mean((y >= R['c80lo']) & (y <= R['c80hi']))), 'pi95': float(np.mean((y >= R['c95lo']) & (y <= R['c95hi']))), 'width80_mw': float(np.mean(R['c80hi'] - R['c80lo'])), 'by_month80': [[o_, float(np.mean((g.mw >= g.c80lo) & (g.mw <= g.c80hi)))] for o_, g in R.groupby('origin')]}
cov = {'pi80': float(np.mean((y >= R['q0.1']) & (y <= R['q0.9']))), 'pi95': float(np.mean((y >= R['q0.025']) & (y <= R['q0.975']))), 'width80_mw': float(np.mean(R['q0.9'] - R['q0.1']))}
# daily peak accuracy
R['abs_pct'] = np.abs(y - pm) / y * 100
dp = R.groupby('date').apply(lambda g: pd.Series({'act': g.mw.max(), 'pred': g[MAIN].max(), 'base': g[best_base].max(), 'fcmax': g.fcmax.iloc[0]}), include_groups=False).reset_index()
peak = {'mape_model': float(np.mean(np.abs(dp.act - dp.pred) / dp.act * 100)), 'mape_base': float(np.mean(np.abs(dp.act - dp.base) / dp.act * 100)), 'n_days': len(dp)}
# error by forecast max temperature band, by hour, by holiday, by month
R['band'] = pd.cut(R.fcmax, [-50, 15, 20, 25, 30, 35, 60], labels=['<15°C', '15–20', '20–25', '25–30', '30–35', '35°C+'])
by_band = [{'band': str(b), 'n_days': int(g.date.nunique()), 'model': mape(g.mw.values, g[MAIN].values), 'base': mape(g.mw.values, g[best_base].values)} for b, g in R.groupby('band', observed=True)]
by_hour = [[int(hh), mape(g.mw.values, g[MAIN].values), mape(g.mw.values, g[best_base].values)] for hh, g in R.groupby('hour')]
by_hol = {'holiday': mape(R[R.hol == 1].mw.values, R[R.hol == 1][MAIN].values) if (R.hol == 1).any() else None, 'normal': mape(R[R.hol == 0].mw.values, R[R.hol == 0][MAIN].values), 'n_hol_days': int(R[R.hol == 1].date.nunique())}
by_month = [[o, mape(g.mw.values, g[MAIN].values), mape(g.mw.values, g[best_base].values)] for o, g in R.groupby('origin')]
worst = R.groupby('date').abs_pct.mean().sort_values(ascending=False).head(5)
worst_days = [{'date': str(d), 'mape': float(v), 'fcmax': float(daily.set_index('date').loc[d, 'fcmax']), 'obmax': float(daily.set_index('date').loc[d, 'obmax']), 'hol': int(d in hol), 'dow': pd.Timestamp(d).day_name()} for d, v in worst.items()]
# chart data
dd = daily.dropna(subset=['obmax', 'dmax']); temp_curve = dd[['obmax', 'dmax']].round(1).values.tolist()
heat = df.groupby(['month', 'hour']).mw.mean().round(0).unstack().values.tolist()
hw = dp.sort_values('fcmax').iloc[-1].date; wk = R[(R.date >= hw - pd.Timedelta(days=3)) & (R.date <= hw + pd.Timedelta(days=3))]
fan = {'centre': str(hw), 't': wk['loc'].dt.strftime('%Y-%m-%dT%H:%M').tolist(), 'act': wk.mw.round(0).tolist(), 'p50': wk[MAIN].round(0).tolist(), 'q10': wk['q0.1'].round(0).tolist(), 'q90': wk['q0.9'].round(0).tolist(), 'q025': wk['q0.025'].round(0).tolist(), 'q975': wk['q0.975'].round(0).tolist(), 'c80lo': wk.c80lo.round(0).tolist(), 'c80hi': wk.c80hi.round(0).tolist(), 'c95lo': wk.c95lo.round(0).tolist(), 'c95hi': wk.c95hi.round(0).tolist(), 'base': wk[best_base].round(0).tolist(), 'temp': wk.fc_temperature_2m.round(1).tolist(), 'sw': wk.fc_sw.round(0).tolist()}
# price link: does demand error matter when prices spike?
R['spike'] = R.rrp > 300
spike = {'share_hours': float(R.spike.mean()), 'mape_spike': mape(R[R.spike].mw.values, R[R.spike][MAIN].values) if R.spike.any() else None, 'mape_normal': mape(R[~R.spike].mw.values, R[~R.spike][MAIN].values)}
out = {'meta': run_meta(len(raw), {'test_hours': len(R), 'test_days': int(R.date.nunique()), 'origins': len(origins), 'train_start': str(df['loc'].min())[:10]}), 'audit': A,
       'results': res, 'diff': diff, 'diff_weather': diff_wx, 'diff_solar': diff_solar, 'by_hour_tonly': by_hour_t, 'coverage': cov, 'coverage_conformal': cov_c, 'peak': peak, 'by_band': by_band, 'by_hour': by_hour, 'by_hol': by_hol, 'by_month': by_month, 'worst_days': worst_days,
       'importance': sorted([[k, float(v)] for k, v in imp.items()], key=lambda x: -x[1]), 'temp_curve': temp_curve, 'heat': heat, 'fan': fan, 'spike': spike,
       'mean_mw': float(df.mw.mean()), 'peak_mw': float(df.mw.max()), 'peak_when': str(df.loc[df.mw.idxmax(), 'loc'])[:16], 'min_mw': float(df.mw.min()), 'min_when': str(df.loc[df.mw.idxmin(), 'loc'])[:16]}
dump(out, OUT); print(cov_c, json.dumps({m: round(v['mape'], 2) for m, v in res.items()}), diff, cov, peak)
