"""Grade every captured reply and summarise.

Capture layout (written by the browser operator):
  runs/<app>/<task_id>/run<N>.md          full reply text, exactly as copied with the app's own "Copy" button
  runs/<app>/<task_id>/run<N>.meta.json   {"app","model_label","mode_label","started_at","finished_at",
                                           "seconds","chat_url","attempt_errors","tools_used","notes"}
Usage: python3 harness/run_all.py [runs_dir] [out_dir]
Outputs: <out>/results.csv, <out>/results.jsonl, <out>/summary.json, <out>/summary.md, extracted code in <out>/extracted/
"""
import csv, json, os, random, re, statistics, subprocess, sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from extract import extract

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
MAN = json.load(open(os.path.join(ROOT, "tasks.json")))
TASKS = {t["id"]: t for t in MAN["tasks"]}


def grade_one(task, reply_path, out_dir):
    reply = open(reply_path, encoding="utf-8").read()
    code = extract(reply, task["lang"])
    rel = os.path.relpath(reply_path, RUNS)
    if code is None:
        return {"passed": False, "score": 0.0, "n_checks": 0, "n_passed": 0, "failures": ["no fenced code block in reply"], "notes": {}}
    dst = os.path.join(out_dir, "extracted", os.path.splitext(rel)[0] + "." + task["ext"])
    os.makedirs(os.path.dirname(dst), exist_ok=True)
    open(dst, "w", encoding="utf-8").write(code)
    try:
        p = subprocess.run([sys.executable, os.path.join(ROOT, "tasks", task["id"], "grade.py"), dst],
                           capture_output=True, text=True, timeout=300, cwd=os.path.dirname(dst))
        return json.loads(p.stdout.strip().splitlines()[-1])
    except subprocess.TimeoutExpired:
        return {"passed": False, "score": 0.0, "n_checks": 0, "n_passed": 0, "failures": ["grader timeout (300 s)"], "notes": {}}
    except Exception as e:
        return {"passed": False, "score": 0.0, "n_checks": 0, "n_passed": 0, "failures": [f"grader error {e}"], "notes": {}}


def bootstrap_diff(per_task_a, per_task_b, iters=10000, seed=1):
    keys = sorted(set(per_task_a) & set(per_task_b))
    if not keys:
        return None
    r = random.Random(seed)
    diffs = []
    for _ in range(iters):
        sample = [r.choice(keys) for _ in keys]
        diffs.append(statistics.mean(per_task_a[k] - per_task_b[k] for k in sample))
    diffs.sort()
    return [round(diffs[int(0.025 * iters)], 3), round(diffs[int(0.975 * iters)], 3)]


def main():
    global RUNS
    RUNS = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else os.path.join(ROOT, "runs"))
    out = os.path.abspath(sys.argv[2] if len(sys.argv) > 2 else os.path.join(ROOT, "results"))
    os.makedirs(out, exist_ok=True)
    rows = []
    for app in sorted(os.listdir(RUNS)):
        for tid in sorted(os.listdir(os.path.join(RUNS, app))):
            if tid not in TASKS:
                continue
            d = os.path.join(RUNS, app, tid)
            for f in sorted(os.listdir(d)):
                m = re.fullmatch(r"run(\d+)\.md", f)
                if not m:
                    continue
                meta_p = os.path.join(d, f"run{m[1]}.meta.json")
                meta = json.load(open(meta_p)) if os.path.exists(meta_p) else {}
                res = grade_one(TASKS[tid], os.path.join(d, f), out)
                rows.append({"app": app, "task": tid, "run": int(m[1]), "passed": res["passed"], "score": res["score"],
                             "n_checks": res["n_checks"], "n_passed": res["n_passed"], "seconds": meta.get("seconds"),
                             "model_label": meta.get("model_label"), "mode_label": meta.get("mode_label"),
                             "started_at": meta.get("started_at"), "attempt_errors": meta.get("attempt_errors"),
                             "tools_used": meta.get("tools_used"), "first_failure": (res.get("failures") or [""])[0][:300]})
                print(f"{app:<10} {tid:<28} run{m[1]} passed={res['passed']!s:<5} score={res['score']}", flush=True)
    with open(os.path.join(out, "results.jsonl"), "w") as fh:
        for r in rows:
            fh.write(json.dumps(r) + "\n")
    with open(os.path.join(out, "results.csv"), "w", newline="") as fh:
        w = csv.DictWriter(fh, fieldnames=list(rows[0].keys()) if rows else ["app"])
        w.writeheader(); w.writerows(rows)
    apps = sorted({r["app"] for r in rows})
    summary = {"suite": MAN["suite"], "version": MAN["version"], "apps": {}}
    per_task_rate = {}
    for app in apps:
        ar = [r for r in rows if r["app"] == app]
        tasks = sorted({r["task"] for r in ar})
        rate = {t: statistics.mean(1.0 if r["passed"] else 0.0 for r in ar if r["task"] == t) for t in tasks}
        per_task_rate[app] = rate
        all3 = [t for t in tasks if all(r["passed"] for r in ar if r["task"] == t) and len([r for r in ar if r["task"] == t]) >= 3]
        # spread of the overall pass rate across run index (run1 vs run2 vs run3)
        by_run = {}
        for r in ar:
            by_run.setdefault(r["run"], []).append(1.0 if r["passed"] else 0.0)
        run_rates = {k: round(statistics.mean(v), 3) for k, v in sorted(by_run.items())}
        secs = [r["seconds"] for r in ar if isinstance(r["seconds"], (int, float))]
        summary["apps"][app] = {
            "attempts": len(ar), "tasks": len(tasks),
            "pass_at_1": round(statistics.mean(1.0 if r["passed"] else 0.0 for r in ar), 3),
            "pass_all_runs_tasks": len(all3),
            "mean_partial_score": round(statistics.mean(r["score"] for r in ar), 3),
            "pass_rate_by_run_index": run_rates,
            "pass_rate_run_spread": round(max(run_rates.values()) - min(run_rates.values()), 3) if run_rates else None,
            "median_seconds": round(statistics.median(secs), 1) if secs else None,
            "per_task": {t: {"passes": sum(r["passed"] for r in ar if r["task"] == t),
                             "runs": len([r for r in ar if r["task"] == t]),
                             "mean_score": round(statistics.mean(r["score"] for r in ar if r["task"] == t), 3)} for t in tasks},
        }
    if len(apps) == 2:
        a, b = apps
        summary["paired_difference"] = {"a": a, "b": b,
                                        "pass_at_1_diff": round(summary["apps"][a]["pass_at_1"] - summary["apps"][b]["pass_at_1"], 3),
                                        "bootstrap_95ci_over_tasks": bootstrap_diff(per_task_rate[a], per_task_rate[b])}
    json.dump(summary, open(os.path.join(out, "summary.json"), "w"), indent=2)
    lines = ["# Results summary (auto-generated by harness/run_all.py)", ""]
    lines.append("| Task | " + " | ".join(f"{a} passes" for a in apps) + " |")
    lines.append("|---|" + "---|" * len(apps))
    for t in TASKS:
        cells = []
        for a in apps:
            pt = summary["apps"][a]["per_task"].get(t)
            cells.append(f"{pt['passes']}/{pt['runs']} (score {pt['mean_score']})" if pt else "-")
        lines.append(f"| {t} | " + " | ".join(cells) + " |")
    lines.append("")
    for a in apps:
        s = summary["apps"][a]
        lines.append(f"- **{a}**: pass@1 {s['pass_at_1']} over {s['attempts']} attempts; tasks passed on every run: "
                     f"{s['pass_all_runs_tasks']}/{s['tasks']}; mean partial score {s['mean_partial_score']}; "
                     f"pass rate by run {s['pass_rate_by_run_index']}; median seconds {s['median_seconds']}")
    if "paired_difference" in summary:
        pdiff = summary["paired_difference"]
        lines.append(f"- Difference {pdiff['a']} - {pdiff['b']} in pass@1: {pdiff['pass_at_1_diff']} "
                     f"(bootstrap 95% CI over tasks {pdiff['bootstrap_95ci_over_tasks']}). If the interval includes 0, call it a tie.")
    open(os.path.join(out, "summary.md"), "w").write("\n".join(lines) + "\n")
    print("\n".join(lines))


if __name__ == "__main__":
    main()