"""Spend coding on AusTender contract notices, FY2017-18 (data.gov.au, CC BY 3.0 AU). Each notice carries a free-text
description, a supplier and the UNSPSC category the buyer chose. Task: suggest the category the way a finance team
codes a bill, against the supplier-default rule bookkeeping systems use."""
import json, re, time, numpy as np, pandas as pd
T0=time.time()
def lg(*a): print(round(time.time()-T0,1), *a, flush=True)
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import make_union
from scipy.sparse import hstack
d = pd.read_csv('/workspace/cs7/data/austender1718.csv', low_memory=False)
d = d[d['Contract Notice Type'] == 'Parent'].copy()
d['desc'] = d.Description.fillna('').str.replace(r'<[^>]+>', ' ', regex=True).str.replace(r'\s+', ' ', regex=True).str.strip()
d['sup'] = d['Supplier Name'].fillna('').str.upper().str.replace(r'\b(PTY|LTD|LIMITED|P/L|THE|AUSTRALIA|INC)\b|[^A-Z0-9 ]', ' ', regex=True).str.replace(r'\s+', ' ', regex=True).str.strip()
d['date'] = pd.to_datetime(d['Publish Date'], dayfirst=True, errors='coerce'); d['value'] = pd.to_numeric(d['Contract Value'], errors='coerce')
d['code'] = pd.to_numeric(d.UNSPSC, errors='coerce'); d = d.dropna(subset=['code', 'date']); d = d[d.desc.str.len() > 2]
d['seg'] = (d.code // 1000000).astype(int); d['fam'] = (d.code // 10000).astype(int)
segname = d.groupby('seg')['UNSPSC Title'].agg(lambda s: s.value_counts().index[0]).to_dict()
# segment titles from the UNSPSC segment level are not in the file; keep a readable label from the commonest title in the segment
d = d.sort_values('date'); cut = d.date.quantile(0.75); tr, te = d[d.date < cut], d[d.date >= cut]
R = dict(n=len(d), n_train=len(tr), n_test=len(te), cut=str(cut.date()), start=str(d.date.min().date()), end=str(d.date.max().date()), n_seg=int(d.seg.nunique()), n_fam=int(d.fam.nunique()), n_code=int(d.code.nunique()),
         value_test=float(te.value.sum()))
def text(x): return x.desc + ' || ' + x.sup + ' || ' + x['Agency Name'].fillna('')
vec = TfidfVectorizer(ngram_range=(1, 2), min_df=2, max_features=200000, sublinear_tf=True)
lg('loaded', len(tr), len(te)); Xtr = vec.fit_transform(text(tr)); Xte = vec.transform(text(te)); lg('vec', Xtr.shape)
out = {}
for lvl in ['seg', 'fam']:
    keep = tr[lvl].value_counts(); keep = keep[keep >= 3].index
    m = tr[lvl].isin(keep).values
    if lvl == 'seg':
        clf = LogisticRegression(C=10, max_iter=1000); clf.fit(Xtr[m], tr[lvl].values[m]); P = clf.predict_proba(Xte)
    else:
        from sklearn.svm import LinearSVC
        clf = LinearSVC(C=0.5).fit(Xtr[m], tr[lvl].values[m]); Z = clf.decision_function(Xte); Z = np.exp(3 * (Z - Z.max(1, keepdims=True))); P = Z / Z.sum(1, keepdims=True)
    lg('fit', lvl); cls = clf.classes_; pred = cls[P.argmax(1)]; conf = P.max(1)
    top3 = cls[np.argsort(-P, 1)[:, :3]]
    y = te[lvl].values
    # supplier default rule: commonest code for the supplier in training; else the commonest overall
    sd = tr.groupby('sup')[lvl].agg(lambda s: s.value_counts().index[0]); seen = te.sup.isin(sd.index).values
    rule = np.where(seen, te.sup.map(sd).fillna(-1).values, tr[lvl].value_counts().index[0]).astype(int)
    hyb = np.where(seen & (conf < 0.5), rule, pred)
    vw = te.value.fillna(0).values
    r = dict(model=float((pred == y).mean()), model_top3=float((top3 == y[:, None]).any(1).mean()), rule=float((rule == y).mean()), rule_seen=float((rule[seen] == y[seen]).mean()), model_seen=float((pred[seen] == y[seen]).mean()),
             model_new=float((pred[~seen] == y[~seen]).mean()), rule_new=float((rule[~seen] == y[~seen]).mean()), seen=float(seen.mean()), hybrid=float((hyb == y).mean()),
             model_value=float(vw[pred == y].sum() / vw.sum()), rule_value=float(vw[rule == y].sum() / vw.sum()), n_classes=int(len(cls)))
    cov = []
    for th in np.round(np.arange(0.3, 0.96, 0.05), 2):
        k = conf >= th; cov.append([float(th), float(k.mean()), float((pred[k] == y[k]).mean())])
    r['coverage'] = cov; th = 0.7 if lvl == 'seg' else 0.6; k = conf >= th
    r['bar'] = dict(th=th, auto=float(k.mean()), acc=float((pred[k] == y[k]).mean()), held=int((~k).sum()), top3_held=float((top3[~k] == y[~k][:, None]).any(1).mean()))
    if lvl == 'seg':
        tt = te.assign(ok=(pred == y), rok=(rule == y))
        g = tt.groupby('seg').agg(n=('ok', 'size'), acc=('ok', 'mean'), racc=('rok', 'mean')).sort_values('n', ascending=False).head(10)
        r['per_seg'] = [[segname.get(s, str(s)), int(a.n), float(a.acc), float(a.racc)] for s, a in g.iterrows()]
        tt['vb'] = pd.cut(tt.value, [0, 1e4, 1e5, 1e6, 1e7, 1e12], labels=['Under $10k', '$10k to $100k', '$100k to $1m', '$1m to $10m', 'Over $10m'])
        r['by_value'] = [[str(k2), int(g2.shape[0]), float(g2.ok.mean()), float(g2.rok.mean())] for k2, g2 in tt.groupby('vb', observed=True)]
        tt['dl'] = pd.cut(tt.desc.str.split().str.len(), [0, 2, 4, 8, 1000], labels=['1 to 2 words', '3 to 4 words', '5 to 8 words', '9 or more'])
        r['by_len'] = [[str(k2), int(g2.shape[0]), float(g2.ok.mean())] for k2, g2 in tt.groupby('dl', observed=True)]
        # recompute errors properly
        idx = np.where((pred != y) & (conf >= 0.8))[0][:400]; rs = np.random.default_rng(2).choice(idx, min(6, len(idx)), replace=False)
        r['errors'] = [[te.desc.values[i][:90], segname.get(int(y[i]), ''), segname.get(int(pred[i]), ''), float(conf[i])] for i in rs]
        r['top_segs'] = [[segname.get(s, str(s)), int(c)] for s, c in te.seg.value_counts().head(8).items()]
    out[lvl] = r; lg('done', lvl)
R.update(out)
R['desc_words_median'] = float(te.desc.str.split().str.len().median())
json.dump(R, open('/workspace/cs7/out/coding.json', 'w'), indent=1, default=str)
for lvl in ['seg', 'fam']: print(lvl, {k: v for k, v in R[lvl].items() if k not in ('coverage', 'per_seg', 'errors', 'by_value', 'by_len', 'top_segs')})
print({k: R[k] for k in ['n', 'n_train', 'n_test', 'cut', 'start', 'end', 'n_seg', 'n_fam', 'n_code']})
print(R['seg']['per_seg']); print(R['seg']['by_value']); print(R['seg']['by_len']); print(R['seg']['errors'])
