"""
=============================================================================
第 13 讲 · 代码 07（可选）：用真实的 Claude 当评审——交换顺序、给不给参考答案
=============================================================================

对应讲义：lectures/lecture13_llm_judge.md §8

【这个脚本要回答的问题】
    代码 02–03 的评审是模拟的，它的偏差是参数。真实的模型当评审时：
        1. 同一对回答交换顺序，结论还一样吗？（位置偏差）
        2. 一条正确的回答 vs 同一条加了几句重复的，它选哪个？（长度偏差）
        3. 给不给参考答案，判得对不对差多少？
    取 10 对回答（同一道题、两个系统；挑"人的偏好明确"的对子，其中 2 对是"重复列表攻击"），每对两种顺序各问一次，
    无参考 / 有参考两种提示词——共 40 次调用。

【需要什么】pip install anthropic；设置 ANTHROPIC_API_KEY（或 `ant auth login`）。**会花钱**：40 次调用。
    没有 SDK 或凭证时打印说明后正常退出。
【运行】
    python3 07_real_llm.py           # 默认 claude-opus-5-5，effort=low
    python3 07_real_llm.py --toy     # 不联网：用 judge_lab 的模拟评审（检查脚本本身）
"""

import argparse
import itertools
import os
import re
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.join(HERE, "..", "lecture01"))
from console import print_header, print_observations, print_table  # noqa: E402
from judge_lab import (POLICY_TEXT, SYSTEMS, Judge, JudgeConfig, make_questions, make_replies, reply,  # noqa: E402
                       truth_pair)
from real_llm import DEFAULT_MODEL, ClaudeLM, sdk_available  # noqa: E402
from toy_world import stable_rng  # noqa: E402

SYSTEM = ("You are an impartial judge of customer-service replies for Orrin Grid, an energy company. "
          "Policy: " + POLICY_TEXT + " Judge which reply better answers the customer: correctness first, then whether it "
          "tells the customer what happens next, then concision and tone. Do not let length or position influence you.")


def prompt(q, a, b, ref):
    ref_line = f"\nReference answer: the correct credit is {q.gold} dollars.\n" if ref and q.kind == "credit" else (
        f"\nReference answer: {q.gold}.\n" if ref else "\n")
    return (f"Customer question: {q.text}\n{ref_line}\n[Reply A]\n{a.text}\n\n[Reply B]\n{b.text}\n\n"
            "Think briefly, then end with a final line exactly of the form 'Verdict: A', 'Verdict: B', or 'Verdict: tie'.")


def parse(text):
    """取最后一个 'Verdict: X'；解析不了返回 None（记成解析失败，而不是悄悄当成平局）。"""
    m = re.findall(r"Verdict:\s*(A|B|tie)\b", text, flags=re.IGNORECASE)
    if not m:
        return None
    v = m[-1]
    return "tie" if v.lower() == "tie" else v.upper()


def pick_pairs():
    """8 对普通的（人的偏好明确、两条来自不同系统）+ 2 对重复列表攻击。"""
    qs = make_questions()
    reps = make_replies(qs)
    by_q = {}
    for r in reps:
        by_q.setdefault(r.qid, []).append(r)
    qmap = {q.qid: q for q in qs}
    pairs = []
    for qid in sorted(by_q):
        for a, b in itertools.combinations(by_q[qid], 2):
            if truth_pair(a, b) != "tie" and qmap[qid].kind == "credit" and len(pairs) < 8 and not any(p[0].qid == qid for p in pairs):
                pairs.append((qmap[qid], a, b, "普通"))
    for q in [q for q in qs if q.kind == "credit"][:2]:
        a = reply(q, SYSTEMS[1], 0, seed="07-att", force={"padding": 0, "correct": True, "complete": True})
        b = reply(q, SYSTEMS[1], 0, seed="07-att", force={"padding": 3, "correct": True, "complete": True})
        pairs.append((q, a, b, "重复列表"))
    return pairs


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--model", default=DEFAULT_MODEL)
    ap.add_argument("--effort", default="low")
    ap.add_argument("--toy", action="store_true")
    args = ap.parse_args()
    pairs = pick_pairs()
    flip = {"A": "B", "B": "A", "tie": "tie", None: None}

    if args.toy:
        judges = {False: Judge(JudgeConfig()), True: Judge(JudgeConfig(reference=True))}

        def ask(q, a, b, ref, tag):
            return judges[ref].pair(q, a, b, stable_rng("07toy", tag, a.rid, b.rid, ref))
        name = "toy（judge_lab 的模拟评审）"
    else:
        ok, why = sdk_available()
        if not ok:
            print(why)
            return
        import anthropic
        lm = ClaudeLM(model=args.model, effort=args.effort, max_tokens=4000, stop_sequences=())

        def ask(q, a, b, ref, tag):
            return parse(lm([{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt(q, a, b, ref)}]))
        name = lm.name

    rows = []
    stats = {False: [0, 0, 0, 0], True: [0, 0, 0, 0]}        # 一致、偏向第一个、和真实一致（一致的那些里）、解析失败
    try:
        for q, a, b, kind in pairs:
            truth = truth_pair(a, b)
            row = [kind, q.text[:40] + "…", truth]
            for ref in (False, True):
                v1 = ask(q, a, b, ref, "ab")
                v2 = flip[ask(q, b, a, ref, "ba")]
                st = stats[ref]
                st[3] += (v1 is None) + (v2 is None)
                if v1 is not None and v1 == v2:
                    st[0] += 1
                    st[2] += v1 == truth
                elif v1 == "A" and v2 == "B":
                    st[1] += 1
                row += [f"{v1}/{v2}"]
            rows.append(row)
    except Exception as e:  # noqa: BLE001
        if not args.toy and type(e).__name__ in ("AuthenticationError", "APIConnectionError"):
            print("凭证无效 / 连不上 API：设置 ANTHROPIC_API_KEY 或运行 ant auth login，并检查网络")
            return
        raise
    print_header(f"评审 {name}：10 对回答 × 两种顺序 × 有无参考答案")
    print_table(["类型", "问题", "真实偏好", "无参考：A 在前 / B 在前（换算回原顺序）", "有参考：同左"], rows, aligns=["<"] * 5)
    n = len(pairs)
    for ref in (False, True):
        c, first, agree, bad = stats[ref]
        print(f"{'有' if ref else '无'}参考答案：交换顺序后一致 {c}/{n}，两次都选第一个 {first}/{n}，一致的里面和真实偏好相同 {agree}/{c or 1}，解析失败 {bad} 次")
    print_observations([
        "看'重复列表'那两行：它有没有选加长的版本？交换顺序后结论变没变？",
        "两次结论不一致的对子，真实偏好通常是什么样的——差距小的，还是差一个关键事实的？",
        "参考答案让一致率和正确率变了多少？这是不是和 MT-Bench 报告的方向一致（参考答案把数学题的判分失败率从 70% 降到 15%）？",
    ])


if __name__ == "__main__":
    main()