"""LG1 · Find the risky clause in a contract. Clause finder on CUAD v1 (Hendrycks et al. 2021, The Atticus Project, CC BY 4.0):
510 commercial contracts, 41 clause types labelled by lawyers. Official split: 408 train / 102 test contracts.
Run: python legal.py CUADv1.json test.json out.json"""
import sys, re, json, numpy as np, pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import average_precision_score, roc_auc_score
from common import run_meta, dump, RNG
src, test_src, OUT = sys.argv[1], sys.argv[2], sys.argv[3]
data = json.load(open(src))['data']; test_titles = {x['title'] for x in json.load(open(test_src))['data']}
CATS = [q['id'].split('__', 1)[1] for q in data[0]['paragraphs'][0]['qas']]
# cut each contract into passages: blank-line paragraphs, long ones split on sentence ends, aiming for 150–900 characters
def chunks(text):
    out, pos = [], 0
    for m in re.finditer(r'(?:[^\n]|\n(?!\s*\n))+', text):
        s, e = m.start(), m.end(); seg = text[s:e]
        if len(seg.strip()) < 3: continue
        if len(seg) <= 900: out.append((s, e)); continue
        last = s
        for sm in re.finditer(r'(?<=[.;:])\s+(?=[A-Z(])', seg):
            cut = s + sm.start()
            if cut - last >= 400: out.append((last, cut)); last = cut
        out.append((last, e))
    merged = []
    for s, e in out:
        if merged and (e - s < 150) and (merged[-1][1] - merged[-1][0] < 900): merged[-1] = (merged[-1][0], e)
        else: merged.append((s, e))
    return merged
rows, pairs = [], []
for c in data:
    p = c['paragraphs'][0]; txt = p['context']; ch = chunks(txt); title = c['title']; split = 'test' if title in test_titles else 'train'
    lab = np.zeros((len(ch), len(CATS)), np.int8); starts = np.array([s for s, _ in ch]); ends = np.array([e for _, e in ch])
    for q in p['qas']:
        k = CATS.index(q['id'].split('__', 1)[1]); has = len(q['answers']) > 0
        pairs.append([title, split, k, int(has), len(q['answers'])])
        for a in q['answers']:
            a0 = a['answer_start']; a1 = a0 + len(a['text']); hit = (starts < a1) & (ends > a0); lab[hit, k] = 1
    for i, (s, e) in enumerate(ch): rows.append((title, split, i, txt[s:e], lab[i]))
df = pd.DataFrame({'title': [r[0] for r in rows], 'split': [r[1] for r in rows], 'i': [r[2] for r in rows], 'text': [r[3] for r in rows]}); L = np.vstack([r[4] for r in rows])
pairs = pd.DataFrame(pairs, columns=['title', 'split', 'k', 'has', 'n_ans'])
tr = (df.split == 'train').values; te = ~tr
A = {'contracts': len(data), 'train_contracts': int(df[tr].title.nunique()), 'test_contracts': int(df[te].title.nunique()), 'categories': len(CATS), 'passages': len(df),
     'passages_per_contract': float(df.groupby('title').size().median()), 'chars_median': float(df.groupby('title').text.apply(lambda s: s.str.len().sum()).median()),
     'labelled_spans': int(pairs.n_ans.sum()), 'pos_pairs': int(pairs.has.sum()), 'pos_share_passages': float((L.sum(axis=1) > 0).mean()),
     'longest_contract_chars': int(df.groupby('title').text.apply(lambda s: s.str.len().sum()).max())}
vec = TfidfVectorizer(ngram_range=(1, 2), min_df=3, max_df=0.5, sublinear_tf=True, max_features=200000, strip_accents='unicode', lowercase=True)
Xtr = vec.fit_transform(df.text[tr]); Xte = vec.transform(df.text[te]); A['vocab'] = len(vec.vocabulary_)
STOP = {'of', 'the', 'and', 'or', 'to', 'on', 'a', 'in', 'for', 'with', 'by', 'post', 'all', 'you', 'can', 'eat'}
def kw(cat):  # baseline: does the passage use the words in the clause's own name?
    w = [x for x in re.findall(r'[a-z]+', cat.lower()) if x not in STOP and len(x) > 2]
    return np.array([sum(t.count(x[:6]) for x in w) for t in df.text[te].str.lower()], float)
ct = df[te].reset_index(drop=True); res, S_model, S_kw = [], np.zeros((te.sum(), len(CATS))), np.zeros((te.sum(), len(CATS)))
for k, cat in enumerate(CATS):
    ytr, yte = L[tr, k], L[te, k]
    if ytr.sum() < 5: res.append([cat, int(ytr.sum()), int(yte.sum()), None, None, None, None]); continue
    m = LogisticRegression(max_iter=2000, C=4.0, class_weight='balanced').fit(Xtr, ytr); p = m.predict_proba(Xte)[:, 1]; S_model[:, k] = p; S_kw[:, k] = kw(cat)
    res.append([cat, int(ytr.sum()), int(yte.sum()), float(average_precision_score(yte, p)) if yte.sum() else None, float(average_precision_score(yte, S_kw[:, k])) if yte.sum() else None, float(yte.mean()), None])
# reviewer view: per test contract and clause type that exists, is a correct passage in the top 3 suggestions?
def topk_hit(S, k=3):
    hits = {}
    for (t, g) in ct.groupby('title'):
        idx = g.index.values
        for c in range(len(CATS)):
            yy = L[te][idx, c]
            if yy.sum() == 0: continue
            top = idx[np.argsort(-S[idx, c], kind='stable')[:k]]; hits[(t, c)] = int(L[te][top, c].sum() > 0)
    return hits
h_m3, h_k3, h_m1 = topk_hit(S_model, 3), topk_hit(S_kw, 3), topk_hit(S_model, 1)
S_first = np.tile(-ct.i.values[:, None].astype(float), (1, len(CATS))); h_f3 = topk_hit(S_first, 3)
hit = {'model_top3': float(np.mean(list(h_m3.values()))), 'kw_top3': float(np.mean(list(h_k3.values()))), 'model_top1': float(np.mean(list(h_m1.values()))), 'first3': float(np.mean(list(h_f3.values()))), 'n_pairs': len(h_m3)}
rng = np.random.default_rng(RNG); keys = list(h_m3.keys()); am = np.array([h_m3[k] for k in keys]); ak = np.array([h_k3[k] for k in keys]); bs = []
tit = sorted({k[0] for k in keys}); byc = {t: [i for i, k in enumerate(keys) if k[0] == t] for t in tit}
for _ in range(1000):
    s = rng.choice(tit, len(tit)); ii = np.concatenate([byc[t] for t in s]); bs.append(am[ii].mean() - ak[ii].mean())
hit['diff_ci'] = [float(np.percentile(bs, 2.5)), float(np.percentile(bs, 97.5))]
META = {CATS.index(c) for c in ['Document Name', 'Parties', 'Agreement Date', 'Effective Date', 'Expiration Date']}
sub = [i for i, k in enumerate(keys) if k[1] not in META]; hit['sub_model_top3'] = float(am[sub].mean()); hit['sub_kw_top3'] = float(ak[sub].mean()); hit['sub_n'] = len(sub)
af = np.array([h_f3[k] for k in keys]); hit['sub_first3'] = float(af[sub].mean())
bs2 = []
for _ in range(1000):
    s_ = rng.choice(tit, len(tit)); ii = np.concatenate([byc[t] for t in s_]); ii = ii[np.isin(ii, sub)]; bs2.append(am[ii].mean())
hit['sub_model_ci'] = [float(np.percentile(bs2, 2.5)), float(np.percentile(bs2, 97.5))]
bm = []
for _ in range(1000):
    s_ = rng.choice(tit, len(tit)); ii = np.concatenate([byc[t] for t in s_]); bm.append(am[ii].mean())
hit['model_ci'] = [float(np.percentile(bm, 2.5)), float(np.percentile(bm, 97.5))]
misses = [[keys[i][0][:60], CATS[keys[i][1]]] for i in range(len(keys)) if am[i] == 0]; hit['miss_n'] = len(misses)
from collections import Counter
hit['miss_by_cat'] = Counter(m[1] for m in misses).most_common(10)
cat_hit = [[CATS[c], float(np.mean([v for (t, cc), v in h_m3.items() if cc == c])), sum(1 for (t, cc) in h_m3 if cc == c), float(np.mean([v for (t, cc), v in h_k3.items() if cc == c])), float(np.mean([v for (t, cc), v in h_f3.items() if cc == c]))] for c in range(len(CATS)) if any(cc == c for (_, cc) in h_m3)]
# contract-level: does this contract contain the clause at all? (max passage score) — AUC per category
pres = []
for c in range(len(CATS)):
    g = ct.assign(s=S_model[:, c], y=L[te][:, c]).groupby('title').agg(s=('s', 'max'), y=('y', 'max'))
    if 0 < g.y.sum() < len(g): pres.append([CATS[c], float(roc_auc_score(g.y, g.s)), int(g.y.sum()), len(g)])
# how much text a reviewer reads: share of passages in the top-3 lists across all clause types found
read_share = float(3 / ct.groupby('title').size().median())
# learning curve: training contracts vs mean passage PR-AUC on 12 common clause types
common = [k for k in np.argsort(-L[tr].sum(axis=0))[:12]]; titles_tr = df[tr].title.unique(); lc = []
for nn in [25, 50, 100, 200, len(titles_tr)]:
    sel = set(rng.choice(titles_tr, nn, replace=False)) if nn < len(titles_tr) else set(titles_tr); msk = df[tr].title.isin(sel).values
    v2 = TfidfVectorizer(ngram_range=(1, 2), min_df=2, max_df=0.5, sublinear_tf=True, max_features=200000, strip_accents='unicode').fit(df.text[tr][msk])
    a, b = v2.transform(df.text[tr][msk]), v2.transform(df.text[te]); sc = []
    for k in common:
        if L[tr][msk, k].sum() < 3: continue
        sc.append(average_precision_score(L[te, k], LogisticRegression(max_iter=2000, C=4.0, class_weight='balanced').fit(a, L[tr][msk, k]).predict_proba(b)[:, 1]))
    lc.append([nn, float(np.mean(sc))])
# frequency: share of contracts that contain each clause type
freq = pairs.groupby('k').has.mean().reset_index().values.tolist(); freq = [[CATS[int(k)], float(v)] for k, v in freq]
# top words per category (model weights) for 6 well-known clauses
words = {}
for cat in ['Governing Law', 'Non-Compete', 'Termination For Convenience', 'Cap On Liability', 'Change Of Control', 'Anti-Assignment']:
    k = CATS.index(cat); m = LogisticRegression(max_iter=2000, C=4.0, class_weight='balanced').fit(Xtr, L[tr, k]); fn = np.array(vec.get_feature_names_out())
    words[cat] = fn[np.argsort(-m.coef_[0])[:8]].tolist()
# one test contract map: passages x selected clauses, model score vs truth
ex = ct.title.value_counts().index[len(ct.title.unique()) // 2]; g = ct[ct.title == ex]; sel_c = [CATS.index(c) for c in ['Governing Law', 'Anti-Assignment', 'Termination For Convenience', 'Cap On Liability', 'Insurance', 'Audit Rights', 'Non-Compete', 'Exclusivity']]
exmap = {'title': ex, 'n': len(g), 'cats': [CATS[c] for c in sel_c], 's': np.round(S_model[g.index.values][:, sel_c], 3).tolist(), 'y': L[te][g.index.values][:, sel_c].tolist()}
out = {'meta': run_meta(len(data), {'split': 'CUAD official (408 train / 102 test)'}), 'audit': A, 'cats': CATS, 'per_cat': res, 'hit': hit, 'cat_hit': cat_hit, 'presence': pres, 'read_share': read_share, 'lc': lc, 'freq': freq, 'words': words, 'exmap': exmap}
dump(out, OUT)
v = [r for r in res if r[3] is not None]; print(A, hit, 'mean PR model', np.mean([r[3] for r in v]), 'kw', np.mean([r[4] for r in v]), lc)
