"""CP1 · Route a complaint to the right team. Product router on the CFPB Consumer Complaint Database (public domain, US Consumer
Financial Protection Bureau), complaints with a consumer-consented narrative, read from the BEE-spoke-data Hugging Face mirror (has-text, snapshot to Feb 2024).
Train Jul 2022 - Jun 2023, choose thresholds on Jul - Sep 2023, test Oct - Dec 2023 (out of time).
Run: python complaints.py cfpb_0.parquet cfpb_1.parquet cfpb_2.parquet out.json"""
import sys, re, numpy as np, pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import f1_score, accuracy_score, balanced_accuracy_score
from common import run_meta, dump, RNG
*SRC, OUT = sys.argv[1:]
cols = ['Date received', 'Product', 'Sub-product', 'Issue', 'Consumer complaint narrative', 'Company', 'Submitted via', 'Company response to consumer', 'Timely response?', 'Complaint ID']
d = pd.concat([pd.read_parquet(s, columns=cols) for s in SRC], ignore_index=True)
raw_total = len(d)
d['dt'] = pd.to_datetime(d['Date received']); d = d[(d.dt >= '2022-07-01') & (d.dt < '2024-01-01')].copy()
d = d.drop_duplicates('Complaint ID')
d['text'] = d['Consumer complaint narrative'].fillna('')
d = d[d.text.str.len() >= 20]
# the CFPB renamed several products in 2023; map old and new names to one stable team list
GROUP = {'Credit reporting, credit repair services, or other personal consumer reports': 'Credit reports',
         'Credit reporting or other personal consumer reports': 'Credit reports', 'Debt or credit management': 'Credit reports',
         'Debt collection': 'Debt collection', 'Checking or savings account': 'Bank accounts',
         'Credit card or prepaid card': 'Cards', 'Credit card': 'Cards', 'Prepaid card': 'Cards', 'Mortgage': 'Mortgages',
         'Money transfer, virtual currency, or money service': 'Money transfer', 'Payday loan, title loan, or personal loan': 'Personal loans',
         'Payday loan, title loan, personal loan, or advance loan': 'Personal loans', 'Student loan': 'Student loans', 'Vehicle loan or lease': 'Vehicle loans'}
d['y'] = d.Product.map(GROUP); unm = d.y.isna().sum(); d = d[d.y.notna()]
CL = sorted(d.y.unique()); d['m'] = d.dt.dt.to_period('M').astype(str)
tr_all = d[d.dt < '2023-07-01']; va_all = d[(d.dt >= '2023-07-01') & (d.dt < '2023-10-01')]; te = d[d.dt >= '2023-10-01'].copy()
# renamed labels: share of test complaints whose raw product name never appears in training
seen = set(tr_all.Product.unique()); rename_share = float((~te.Product.isin(seen)).mean())
rename_by = te.assign(new=~te.Product.isin(seen)).groupby('y').new.mean().to_dict()
# templated text: identical narratives after squashing whitespace and the XXXX redactions
norm = lambda s: s.str.lower().str.replace(r'x{2,}', 'x', regex=True).str.replace(r'[^a-z ]', ' ', regex=True).str.replace(r'\s+', ' ', regex=True).str.strip()
te['norm'] = norm(te.text); dup = te.norm.duplicated(keep=False)
dup_share = float(dup.mean()); dup_by = te.assign(d=dup).groupby('y').d.mean().to_dict()
top_template = int(te.norm.value_counts().iloc[0])
rng = np.random.default_rng(RNG)
def cap(df, k):  # cap rows per team so the biggest team does not swamp training
    r = np.random.default_rng(RNG).random(len(df)); rk = pd.Series(r, index=df.index).groupby(df.y).rank(method='first'); return df[rk.values <= k]
tr = cap(tr_all, 25000).sample(frac=1, random_state=RNG); va = va_all.sample(min(len(va_all), 40000), random_state=RNG)
vec = TfidfVectorizer(ngram_range=(1, 2), min_df=5, max_df=0.6, sublinear_tf=True, max_features=300000, strip_accents='unicode', token_pattern=r'(?u)\b[a-zA-Z][a-zA-Z]+\b')
Xtr = vec.fit_transform(tr.text); Xva = vec.transform(va.text); Xte = vec.transform(te.text)
# class prior correction: training is capped per team, the real stream is not. Reweight with training-period shares.
prior = tr_all.y.value_counts(normalize=True).reindex(CL).values; cap_prior = tr.y.value_counts(normalize=True).reindex(CL).values
def fit(X, y): return LogisticRegression(max_iter=3000, C=8.0, solver='saga' if False else 'lbfgs').fit(X, y)
M = fit(Xtr, tr.y)
def proba(m, X):
    p = m.predict_proba(X) * (prior / cap_prior); return p / p.sum(1, keepdims=True)
Pva, Pte = proba(M, Xva), proba(M, Xte)
pred = np.array(CL)[Pte.argmax(1)]; conf = Pte.max(1); yt = te.y.values
# keyword baseline: first matching rule in a fixed order, otherwise the biggest team
RULES = [('Mortgages', r'mortgage|escrow|foreclos|loan modification|servicer'), ('Student loans', r'student loan|navient|nelnet|mohela|fedloan|forgiveness'),
         ('Vehicle loans', r'car loan|auto loan|vehicle|repossess|lease'), ('Debt collection', r'collection agency|debt collector|collector|collections'),
         ('Money transfer', r'zelle|venmo|cash app|paypal|wire transfer|money transfer|western union|crypto|bitcoin|coinbase'),
         ('Personal loans', r'payday|title loan|personal loan|installment loan'), ('Cards', r'credit card|prepaid|card account|my card|debit card'),
         ('Bank accounts', r'checking|savings account|overdraft|bank account|deposit'), ('Credit reports', r'credit report|equifax|transunion|experian|fcra|consumer report')]
def kw(texts):
    t = texts.str.lower(); out = np.full(len(t), 'Credit reports', object); done = np.zeros(len(t), bool)
    for lab, pat in RULES:
        h = t.str.contains(pat, regex=True).values & ~done; out[h] = lab; done |= h
    return out
kwp = kw(te.text)
def mets(y, p):
    return {'acc': float(accuracy_score(y, p)), 'macro_f1': float(f1_score(y, p, average='macro')), 'bal_acc': float(balanced_accuracy_score(y, p))}
def boot(y, a, b=None, B=400):
    r = np.random.default_rng(RNG); n = len(y); out = []
    for _ in range(B):
        i = r.integers(0, n, n)
        v = f1_score(y[i], a[i], average='macro')
        if b is not None: v -= f1_score(y[i], b[i], average='macro')
        out.append(v)
    return [float(np.percentile(out, 2.5)), float(np.percentile(out, 97.5))]
prior_pred = np.full(len(yt), 'Credit reports', object)
R = {'model': {**mets(yt, pred), 'f1_ci': boot(yt, pred)}, 'keyword': {**mets(yt, kwp), 'f1_ci': boot(yt, kwp)}, 'biggest': mets(yt, prior_pred),
     'diff_f1_ci': boot(yt, pred, kwp)}
# random split optimism: same model, shuffled split inside the test months vs out-of-time
from sklearn.model_selection import train_test_split
pool = pd.concat([tr_all, va_all, te]).pipe(lambda x: cap(x, 25000)); a, b = train_test_split(pool, test_size=0.25, random_state=RNG, stratify=pool.y)
v2 = TfidfVectorizer(**vec.get_params()); Ma = fit(v2.fit_transform(a.text), a.y); R['random_split_macro_f1'] = float(f1_score(b.y, Ma.predict(v2.transform(b.text)), average='macro'))
R['oot_capped_macro_f1'] = float(f1_score(yt, np.array(CL)[M.predict_proba(Xte).argmax(1)], average='macro'))
# abstain: pick the confidence threshold on validation that keeps 95% accuracy on the routed share
vconf = Pva.max(1); vpred = np.array(CL)[Pva.argmax(1)]; vy = va.y.values
curve = []
for t in np.round(np.arange(0.30, 0.995, 0.01), 2):
    k = vconf >= t; kt = conf >= t
    curve.append([float(t), float(k.mean()), float((vpred[k] == vy[k]).mean()) if k.any() else None, float(kt.mean()), float((pred[kt] == yt[kt]).mean()) if kt.any() else None])
ok = [c for c in curve if c[2] is not None and c[2] >= 0.95]; T = ok[0][0] if ok else 0.9
k = conf >= T; R['abstain'] = {'threshold': T, 'target': 0.95, 'test_routed': float(k.mean()), 'test_acc_routed': float((pred[k] == yt[k]).mean()), 'test_acc_held': float((pred[~k] == yt[~k]).mean()),
     'held_n': int((~k).sum()), 'routed_n': int(k.sum()), 'val_routed': float((vconf >= T).mean())}
# per team
per = []
for c in CL:
    m = yt == c; per.append([c, int(m.sum()), float((pred[m] == c).mean()), float((kwp[m] == c).mean()), float(((pred == c) & m).sum() / max(1, (pred == c).sum())), float(dup_by.get(c, 0)), float(rename_by.get(c, 0)), float(k[m].mean())])
cm = pd.crosstab(pd.Series(yt, name='t'), pd.Series(pred, name='p')).reindex(index=CL, columns=CL, fill_value=0)
# top words per team
coef = M.coef_; fn = np.array(vec.get_feature_names_out()); words = {c: fn[np.argsort(-coef[i])[:8]].tolist() for i, c in enumerate(M.classes_)}
# learning curve on capped training rows
lc = []
for n in [1000, 3000, 10000, 30000, len(tr)]:
    s = tr.iloc[:n]; m2 = fit(Xtr[:n], s.y); p2 = np.array(CL)[proba(m2, Xte).argmax(1)] if set(s.y) == set(CL) else m2.predict(Xte); lc.append([int(n), float(f1_score(yt, p2, average='macro'))])
# monthly volumes and accuracy
mon = d.groupby(['m', 'y']).size().unstack(fill_value=0).reindex(columns=CL)
te['ok'] = pred == yt; te['kok'] = kwp == yt; acc_m = te.groupby('m').agg(n=('ok', 'size'), acc=('ok', 'mean'), kacc=('kok', 'mean')).reset_index().values.tolist()
raw_mon = d.groupby(['m', 'Product']).size().unstack(fill_value=0)
renames = {p: raw_mon[p].tolist() for p in raw_mon.columns if p in ['Credit reporting, credit repair services, or other personal consumer reports', 'Credit reporting or other personal consumer reports', 'Credit card or prepaid card', 'Credit card', 'Prepaid card']}
# narrative length vs confidence
te['len'] = te.text.str.split().str.len(); te['conf'] = conf
lb = pd.cut(te.len, [0, 50, 100, 200, 400, 800, 100000], labels=['<50', '50–99', '100–199', '200–399', '400–799', '800+'])
lens = te.groupby(lb, observed=True).agg(n=('ok', 'size'), acc=('ok', 'mean'), routed=('conf', lambda s: float((s >= T).mean()))).reset_index().values.tolist()
resp = te['Company response to consumer'].value_counts(normalize=True).head(5).to_dict(); timely = float((te['Timely response?'] == 'Yes').mean())
comp = te.Company.value_counts().head(8); comp_share = float(te.Company.value_counts().head(3).sum() / len(te))
via = te['Submitted via'].value_counts().to_dict()
A = {'raw_rows_in_mirror': raw_total, 'rows_window': len(d), 'unmapped': int(unm), 'train_rows': len(tr_all), 'train_used': len(tr), 'val_rows': len(va_all), 'val_used': len(va), 'test_rows': len(te),
     'teams': CL, 'team_share_test': te.y.value_counts(normalize=True).reindex(CL).tolist(), 'rename_share_test': rename_share, 'dup_share_test': dup_share, 'top_template_copies': top_template,
     'median_words': float(te.len.median()), 'vocab': len(vec.vocabulary_), 'companies_test': int(te.Company.nunique()), 'top3_company_share': comp_share,
     'top_companies': [[a, int(b)] for a, b in comp.items()], 'responses': resp, 'timely': timely, 'via': via, 'date_min': str(d.dt.min().date()), 'date_max': str(d.dt.max().date())}
dump({'meta': run_meta(len(d), {'source': 'CFPB Consumer Complaint Database (mirror: huggingface.co/datasets/BEE-spoke-data/consumer-finance-complaints, has-text)'}), 'audit': A, 'results': R, 'per_team': per,
      'cm': {'labels': CL, 'm': cm.values.tolist()}, 'words': words, 'lc': lc, 'curve': curve, 'months': {'m': mon.index.tolist(), 'v': mon.values.tolist()}, 'acc_m': acc_m,
      'renames': {'m': raw_mon.index.tolist(), 's': renames}, 'lens': lens}, OUT)
print(A); print(R); print(per)
