""" ============================================================================= 第 10 讲 · 工具模块：迷你 Genie Worksheet —— 规格、形式化的对话状态、运行时策略；模拟用户；三种智能体 ============================================================================= 对应讲义：lectures/lecture10_genie_worksheet.md 第 2~5 节 ----------------------------------------------------------------------------- 【这个模块里有什么？】 ----------------------------------------------------------------------------- 课程数据库 一个虚构的选课数据库（SQLite）：课程号、名称、方向、学分范围、评分方式、每周学时、评分、 是否满足"重要实现要求"、上课时间、剩余名额 工作表（Worksheet） 开发者写的规格：要收集哪些字段（类型、说明、是否必填、是否要确认、 "只在某些情况下才问"的字段）、收集齐了之后执行什么动作 GenieAgent Genie 智能体：解析器（模拟的 LLM）把用户的话变成对工作表的更新； **运行时系统**（程序）决定下一步做什么（智能体动作：Report / Confirm / Say / Propose / Ask）； 回复生成器把动作说成话 FunctionCallingAgent 对照组：LLM + 函数调用（一长段指令 + 完整的对话历史，下一步做什么由 LLM 决定） DialogueTreeAgent 对照组：对话树（按固定的顺序问问题，只听得懂"预期的回答"） SimUser 模拟的用户：有一个目标（选两门课），会插话提问、一次说好几个信息、改主意、拒绝回答…… run_dialogue 让一个用户和一个智能体对话，记录每一轮，计算四个指标 ----------------------------------------------------------------------------- 【模拟的边界】（非常重要） ----------------------------------------------------------------------------- 没有真实的 LLM，也没有真实的用户。用户说的话由"用户动作"按模板生成；解析器、函数调用的 LLM 会犯哪些错误、 概率多大，都是设定的参数（见 PARAMS）。其中最关键的一条假设来自论文的观察： · 函数调用的 LLM 看的是**完整的对话历史 + 一长段指令**，对话越长，越不容易遵守指令（论文图 5： GPT-4 的对话动作准确率随轮次下降约 43%，Genie 只下降约 8%）； · Genie 的解析器只看**上一轮 + 形式化的对话状态**，出错率不随对话长度增长。 所以这里的数字说明的是机制，不是真实系统的成绩。课程、学生全部是虚构的。 """ import random import re import sqlite3 from dataclasses import dataclass, field PARAMS = { # Genie 的解析器（只看上一轮 + 形式化的状态） "genie_parse_error": 0.06, # 一句话里漏掉一个信息或填错一个值的概率 # 函数调用的 LLM（看完整的历史 + 一长段指令） "fc_follow_base": 0.97, # 第 1 轮遵守指令的概率 "fc_follow_decay": 0.035, # 每多一轮，遵守指令的概率下降多少 "fc_hallucinate": 0.35, # 回答课程问题时，数据库里没有的信息它会编的概率 "fc_parse_error": 0.06, # 理解用户的话出错的概率（和 Genie 的解析器一样） # 两者共用的知识解析器（论文：基线用的是和 Genie 一样的知识解析器） "kb_parse_error": 0.08, # 把知识问题翻译成 SQL 时漏掉一个条件的概率 # 用户 "user_ask": 0.3, # 智能体问一个字段时，用户反过来先问一个问题的概率 "user_multi": 0.3, # 一次给出好几个信息的概率 "user_change": 0.2, # 改主意（换一种评分方式）的概率 "user_refuse": 0.15, # 一开始不愿意给某个信息（例如学号）的概率 "user_patience": 3, # 智能体连续几次没听懂 / 答非所问，用户就放弃 "max_turns": 30, } # ============================================================================= # 第 1 部分：课程数据库（知识库） # ============================================================================= AREAS = ["AI", "Systems", "Theory", "HCI"] TITLES = { "AI": ["Probabilistic Reasoning", "Deep Learning Foundations", "Natural Language Agents", "Robot Learning", "Computer Vision Systems", "Reinforcement Learning Lab", "Knowledge Graphs", "Machine Learning Theory"], "Systems": ["Operating Systems Design", "Distributed Databases", "Compilers in Practice", "Cloud Infrastructure", "Networked Systems", "Storage Systems"], "Theory": ["Randomized Algorithms", "Cryptography", "Complexity Theory", "Graph Algorithms", "Logic in CS"], "HCI": ["Interaction Design", "Social Computing", "Accessible Interfaces", "Visualization"], } @dataclass class Course: """一门课（数据库里的一行）。""" code: str title: str area: str units_min: int units_max: int grading: str # "Letter or CR/NC" / "Letter only" hours: int # 平均每周学时 rating: float sig_impl: bool # 是否满足"重要实现要求" days: str time: str seats: int # 剩余名额（0 表示满了） def make_courses(seed=0): """生成约 23 门虚构的课程（课程号是虚构的）。""" rng = random.Random(f"courses|{seed}") out, used = [], set() for area, titles in TITLES.items(): for t in titles: while True: code = f"CS {rng.randint(140, 349)}" if code not in used: used.add(code) break lo = rng.choice([3, 3, 3, 1]) out.append(Course(code, t, area, lo, rng.choice([3, 4, 4, 5]) if lo == 3 else 3, rng.choice(["Letter or CR/NC", "Letter or CR/NC", "Letter only"]), rng.randint(5, 20), round(rng.uniform(3.2, 4.9), 1), rng.random() < 0.35, rng.choice(["MW", "TTh", "F"]), rng.choice(["9:00", "10:30", "13:30", "15:00"]), 0 if rng.random() < 0.12 else rng.randint(1, 40))) return out class CourseDB: """ 课程数据库 + 知识查询。 【用法】db = CourseDB(make_courses()); db.query("SELECT code FROM courses WHERE area = 'AI'") 【智能体联系】真实的 Genie 用第 7 讲的 SUQL 处理知识查询；这里的问题都是结构化的，用普通的 SQL 就够了。 """ def __init__(self, courses): self.courses = courses self.by_code = {c.code: c for c in courses} self.db = sqlite3.connect(":memory:") self.db.execute("CREATE TABLE courses (code TEXT, title TEXT, area TEXT, units_min INT, units_max INT, " "grading TEXT, hours INT, rating REAL, sig_impl INT, days TEXT, time TEXT, seats INT)") self.db.executemany("INSERT INTO courses VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", [(c.code, c.title, c.area, c.units_min, c.units_max, c.grading, c.hours, c.rating, int(c.sig_impl), c.days, c.time, c.seats) for c in courses]) self.calls = 0 def query(self, sql): """执行一条只读的 SQL，返回结果行。""" self.calls += 1 return self.db.execute(sql).fetchall() # ============================================================================= # 第 2 部分：工作表规格（开发者写的） # ============================================================================= @dataclass class Field: """ 工作表里的一个字段（论文第 4 节；幻灯片第 63 页："Field Definition: Predicate, Name, Kind, Type, Description, Don't Ask, Required, Confirmation"）。 【字段】 name : 字段名（名字和说明对 LLM 解析器很重要：它们就是提示词的一部分） type : "str" / "int" / "bool" / "enum" desc : 说明 values : 枚举的取值（type == "enum" 时） required : 是否必填 ask_if : "只在某些情况下才问"：一个函数 f(记录) → bool（数据依赖的字段，幻灯片第 87 页） validate : 检查值是否合法的函数 f(记录, 值, 数据库) → None（合法）或一句说明（不合法） """ name: str type: str desc: str values: tuple = () required: bool = True ask_if: object = None validate: object = None question: str = None # 问这个字段时说的话（没有就用说明拼一句） @dataclass class Worksheet: """ 一张工作表（一个任务）：字段 + 收集齐之后的动作。 【字段】 name : 工作表名 fields : 字段列表（顺序就是"愉快的路径"上提问的顺序） confirm : 执行动作之前，是否要用户确认（有副作用的动作都应该确认） action : 动作：函数 f(记录, 运行时) → 一句结果（例如"已选课，确认号 12345"） """ name: str fields: list confirm: bool = True action: object = None def _course_exists(rec, value, db): """字段校验：课程必须在数据库里。""" return None if value in db.by_code else f"I couldn't find {value} in the course catalog" def _units_ok(rec, value, db): """字段校验：学分要在这门课允许的范围内。""" c = db.by_code.get(rec.values.get("course")) if c and not (c.units_min <= value <= c.units_max): return f"{c.code} can be taken for {c.units_min}-{c.units_max} units" return None def _grading_ok(rec, value, db): """字段校验：这门课是否允许这种评分方式。""" c = db.by_code.get(rec.values.get("course")) if c and value == "CR/NC" and c.grading == "Letter only": return f"{c.code} must be taken for a letter grade" return None def enroll_action(rec, runtime): """ 选课的动作（相当于调用学校的选课 API）。开发者定义的策略（幻灯片第 61 页）：课满了的时候， 不是笼统地说"失败"，而是给出开发者指定的回复。 """ c = runtime.db.by_code.get(rec.values["course"]) if c is None: # API 要能处理不合法的输入，而不是崩溃 return f"Sorry, {rec.values['course']} is not available in the catalog." if c.seats == 0: return f"Unfortunately, there are no seats available for {c.code}. You can ask for special permission if you are graduating soon." runtime.enrolled.append((rec.values["course"], rec.values["grade_type"], rec.values["units"])) return f"You are enrolled in {c.code} ({c.title}) for {rec.values['units']} units, {rec.values['grade_type']}." STUDENT = Worksheet("StudentInfo", [ Field("name", "str", "Name of the student", question="What is your name?"), Field("student_id", "str", "Student's ID number", question="What is your student ID?"), Field("international", "bool", "Is the student an international student?", question="Are you an international student?"), Field("visa", "enum", "Visa type", ("F1", "J1"), ask_if=lambda rec: rec.values.get("international") is True, question="What type of visa do you have, F1 or J1?"), ], confirm=False) COURSE = Worksheet("CourseToTake", [ Field("course", "str", "The course to enroll in, e.g. 'CS 224'", validate=_course_exists, question="Which course would you like to enroll in?"), Field("grade_type", "enum", "The desired grading basis", ("Letter", "CR/NC"), validate=_grading_ok, question="Would you like to take it for a letter grade or CR/NC?"), Field("units", "int", "The number of units", validate=_units_ok, question="How many units would you like to take?"), ], confirm=True, action=enroll_action) # ============================================================================= # 第 3 部分：形式化的对话状态 # ============================================================================= @dataclass class Record: """ 对话状态里的一条记录（幻灯片第 69 页）： · 任务记录：工作表名、字段的值、是否已确认、是否已执行； · 知识记录：自然语言的问题、形式化的查询、结果。 """ kind: str # "task" / "kb" ws: Worksheet = None values: dict = field(default_factory=dict) confirmed: bool = False done: bool = False result: str = None question: str = None # 知识记录：用户的问题 sql: str = None # 知识记录：形式化的查询 rows: list = None # 知识记录：结果 reported: bool = False sql_ok: bool = True # 知识记录：查询写得对不对（评估用） def missing(self): """还没填、而且现在需要问的必填字段（考虑"只在某些情况下才问"）。""" out = [] for f in self.ws.fields: if f.ask_if is not None and not f.ask_if(self): continue if f.required and self.values.get(f.name) is None: out.append(f) return out def show(self): """把记录写成一行，像 Python 代码一样（Genie 把对话状态以代码的形式交给解析器）。""" if self.kind == "kb": return f"answer({self.question!r}) -> {self.sql!r} -> {len(self.rows or [])} rows" vals = ", ".join(f"{k}={v!r}" for k, v in self.values.items() if v is not None) flags = (" [confirmed]" if self.confirmed else "") + (" [done]" if self.done else "") return f"{self.ws.name}({vals}){flags}" # ============================================================================= # 第 4 部分：用户动作与说法 # ============================================================================= @dataclass class Act: """ 一个用户动作（模拟用户的"意思"；智能体看到的是由它生成的那句话）。 【kind】 provide ：提供一个字段的值（slot, value） ask ：提一个知识问题（q：问题的结构，例如 {"type": "find", "area": "AI", "max_hours": 10}） change ：改一个已经给过的值 confirm ：确认（value=True）或否认（value=False） refuse ：不愿意给某个字段 chitchat ：闲聊 start ：开始一个新任务（"我想再选一门课"） """ kind: str slot: str = None value: object = None q: dict = None course_idx: int = 0 # 这个动作针对第几门课（0 或 1） def ask_text(q, db): """把一个知识问题写成一句话。""" if q["type"] == "find": parts = [f"{q['area']} courses"] if q.get("sig_impl"): parts.append("that satisfy the significant implementation requirement") if q.get("max_hours"): parts.append(f"with at most {q['max_hours']} hours of work per week") return "What are some " + " ".join(parts) + "?" if q["type"] == "detail": return f"When does {q['code']} meet?" if q["attr"] == "schedule" else f"How many units is {q['code']}?" return "?" def kb_parse(q, rng, p): """ 模拟的知识解析器：知识问题 → SQL。以 p 的概率漏掉一个条件（例如"每周不超过 10 小时"）。 【返回】(SQL, 有没有出错) """ if q["type"] == "find" and (q.get("max_hours") or q.get("sig_impl")) and rng.random() < p: return ask_sql(dict(q, max_hours=None, sig_impl=False)), True return ask_sql(q), False def ask_sql(q): """一个知识问题对应的 SQL（知识解析器的正确输出）。""" if q["type"] == "find": sql = f"SELECT code, title, hours, rating FROM courses WHERE area = '{q['area']}'" if q.get("sig_impl"): sql += " AND sig_impl = 1" if q.get("max_hours"): sql += f" AND hours <= {q['max_hours']}" return sql + " ORDER BY rating DESC LIMIT 3" attr = "days, time" if q["attr"] == "schedule" else "units_min, units_max" return f"SELECT {attr} FROM courses WHERE code = '{q['code']}'" def utter(acts, db): """把一组用户动作写成一句话（模板）。""" pieces = [] for a in acts: if a.kind == "provide": pieces.append({"name": f"My name is {a.value}", "student_id": f"my student ID is {a.value}", "international": "I am an international student" if a.value else "I'm not an international student", "visa": f"my visa type is {a.value}", "course": f"I'd like to take {a.value}", "grade_type": f"for a {'letter grade' if a.value == 'Letter' else 'CR/NC'}", "units": f"for {a.value} units"}[a.slot]) elif a.kind == "change": pieces.append(f"actually, I changed my mind: {'letter grade' if a.value == 'Letter' else 'CR/NC'} instead") elif a.kind == "ask": pieces.append(ask_text(a.q, db)) elif a.kind == "confirm": pieces.append("Yes, that's right" if a.value else "No, that's not right") elif a.kind == "refuse": pieces.append("Why do you need my student ID? I'd rather not say") elif a.kind == "chitchat": pieces.append("By the way, how is your day going?") elif a.kind == "start": pieces.append("I also want to enroll in another course") text = ", ".join(pieces) if not text: return "" return text[0].upper() + text[1:] + ("" if text.endswith(("?", ".")) else ".") # ============================================================================= # 第 5 部分：智能体动作和回复 # ============================================================================= @dataclass class AgentAct: """ 一个智能体动作（幻灯片第 70 页的五种）： report ：报告执行结果（知识查询的结果、任务动作的结果） confirm ：请用户确认一条记录 say ：说一句指定的话（例如校验失败的原因、开发者指定的回复） propose ：提议一个新的任务或查询 ask ：问一个字段 【附加信息】给评估用：grounded（报告的内容是否来自数据库）、executed（是否调用了选课 API） """ kind: str text: str slot: str = None rows: list = None grounded: bool = True executed: object = None # (课程, 评分方式, 学分) —— 调用了选课 API confirmed_before: bool = True # 调用 API 之前有没有得到用户的确认 sql_ok: bool = True # 知识查询的 SQL 对不对（知识解析器有没有漏掉条件） def show_rows(rows): """把查询结果说成一句话。""" if not rows: return "I couldn't find any matching courses." if len(rows[0]) == 4: return "Here are some options: " + "; ".join(f"{r[0]} {r[1]} ({r[2]} hrs/week, rated {r[3]})" for r in rows) + "." a, b = rows[0] if isinstance(a, int): return f"It can be taken for {a}-{b} units." if a != b else f"It is {a} units." return f"It meets {a} at {b}." # ============================================================================= # 第 6 部分：模拟用户 # ============================================================================= class SimUser: """ 一个模拟用户：要选两门课。 【目标】姓名、学号、是否国际学生（是的话还有签证类型）；两门课，每门要么是一个具体的课程号，要么是一个要求 （"AI 方向、每周不超过 10 小时"）——后者要先问智能体才知道选哪门；每门课想要的评分方式和学分。 【行为】（概率见 PARAMS；happy=True 时全部关掉，只走"愉快的路径"，而且两门课都直接知道课程号—— 就像 STAR 这类 WOZ 数据集里的用户） · 被问到一个字段时，先反问一个问题（"CS 2xx 什么时候上课？"）； · 一次给出好几个信息； · 改主意（换一种评分方式）； · 一开始不愿意给学号； · 智能体连续几次没听懂 / 答非所问，就放弃。 """ def __init__(self, db, seed, happy=False): self.db, self.rng, self.happy = db, random.Random(f"user|{seed}"), happy rng = self.rng self.name = f"{rng.choice(['Ana', 'Ben', 'Chen', 'Dev', 'Eli', 'Fay', 'Gus', 'Hui'])} {rng.choice(['Lee', 'Park', 'Ruiz', 'Shah', 'Kim', 'Ng'])}" self.sid = str(rng.randint(10_000_000, 99_999_999)) self.intl = rng.random() < 0.4 self.visa = rng.choice(["F1", "J1"]) if self.intl else None self.courses = [] for i in range(2): if happy or rng.random() < 0.5: # "愉快"的用户直接知道课程号 c = rng.choice([c for c in db.courses if c.seats > 0]) self.courses.append({"code": c.code, "req": None}) else: area = rng.choice(AREAS) req = {"type": "find", "area": area, "max_hours": rng.choice([None, 12]), "sig_impl": rng.random() < 0.3} self.courses.append({"code": None, "req": req, "orig_req": dict(req)}) self.courses[-1]["grade"] = rng.choice(["Letter", "CR/NC"]) self.idx = 0 # 正在选第几门 self.told = {} # 已经告诉智能体的字段（"真实的对话状态"，评估用） self.options = {0: [], 1: []} # 智能体告诉过的候选课程 self.frustration = 0 self.refused = False self.changed = set() self.gave_up = False self.finished = False self.asked_detail = set() def p(self, key): """一个行为的概率（愉快的路径上都是 0）。""" return 0.0 if self.happy else PARAMS[key] def want(self, slot, i=None): """用户对某个字段想要的值。""" i = self.idx if i is None else i goal = self.courses[i] return {"name": self.name, "student_id": self.sid, "international": self.intl, "visa": self.visa, "course": goal["code"], "grade_type": goal["grade"], "units": self.db.by_code[goal["code"]].units_max if goal["code"] in self.db.by_code else 3}[slot] def react(self, acts): """ 读智能体这一轮的动作，决定自己说什么。【返回】list[Act]（空列表表示对话结束） """ rng = self.rng out = [] understood = False for a in acts: if a.kind == "report" and a.rows is not None: understood = True if len(a.rows) and len(a.rows[0]) == 4: self.options[self.idx] = [r[0] for r in a.rows] goal = self.courses[self.idx] if goal["code"] is None and self.options[self.idx]: goal["code"] = self.options[self.idx][0] out.append(Act("provide", "course", goal["code"], course_idx=self.idx)) elif not a.rows and self.courses[self.idx]["code"] is None: req = self.courses[self.idx]["req"] req.update(max_hours=None, sig_impl=False) # 放宽要求再问一次 out.append(Act("ask", q=dict(req), course_idx=self.idx)) elif a.kind == "report" and a.executed is not None: understood = True if "no seats" in a.text: opts = [o for o in self.options[self.idx] if o != a.executed[0]] if opts: self.courses[self.idx]["code"] = opts[0] out.append(Act("provide", "course", opts[0], course_idx=self.idx)) else: self.gave_up = True return [] elif self.idx == 0: self.idx = 1 out.append(Act("start", course_idx=1)) else: self.finished = True return [] # 两门都选好了 elif a.kind == "confirm": understood = True wrong = [s for s, v in a.slot.items() if v != self.want(s)] if isinstance(a.slot, dict) else [] if wrong: out.append(Act("confirm", value=False, course_idx=self.idx)) for s in wrong: out.append(Act("provide", s, self.want(s), course_idx=self.idx)) else: out.append(Act("confirm", value=True, course_idx=self.idx)) elif a.kind == "ask": understood = True out += self._answer(a.slot) elif a.kind == "say": understood = understood or "didn't understand" not in a.text if "didn't understand" in a.text: self.frustration += 1 if "not available" in a.text and self.courses[self.idx]["code"]: opts = [o for o in self.options[self.idx] if o != self.courses[self.idx]["code"]] if opts: self.courses[self.idx]["code"] = opts[0] out.append(Act("provide", "course", opts[0], course_idx=self.idx)) if "must be taken for a letter grade" in a.text: self.courses[self.idx]["grade"] = "Letter" # 接受约束，调整自己的目标 if "anything else" in a.text and self.idx == 1 and not self.finished: out.append(Act("start", course_idx=1)) # 智能体没听到"再选一门"：再说一遍 if self.courses[1]["code"]: out.append(Act("provide", "course", self.courses[1]["code"], course_idx=1)) if not understood: self.frustration += 1 if self.frustration >= PARAMS["user_patience"]: self.gave_up = True return [] uniq = [] for a in out: # 同一个动作不说两遍 if all(vars(a) != vars(b) for b in uniq): uniq.append(a) return uniq or [Act("chitchat")] def _answer(self, slot): """回答"请问你的 X 是什么"。可能先反问、可能多给信息、可能拒绝、可能改主意。""" rng = self.rng goal = self.courses[self.idx] if slot == "course" and goal["code"] is None: return [Act("ask", q=dict(goal["req"]), course_idx=self.idx)] if slot == "student_id" and not self.refused and rng.random() < self.p("user_refuse"): self.refused = True return [Act("refuse", "student_id")] acts = [] if slot in ("grade_type", "units") and goal["code"] and rng.random() < self.p("user_ask") \ and (self.idx, slot) not in self.asked_detail: self.asked_detail.add((self.idx, slot)) return [Act("ask", q={"type": "detail", "code": goal["code"], "attr": "schedule" if slot == "grade_type" else "units"}, course_idx=self.idx)] value = self.want(slot) if slot == "grade_type" and self.idx not in self.changed and rng.random() < self.p("user_change"): self.changed.add(self.idx) first = "CR/NC" if value == "Letter" else "Letter" acts.append(Act("provide", slot, first, course_idx=self.idx)) self._pending_change = True return acts acts.append(Act("provide", slot, value, course_idx=self.idx)) if rng.random() < self.p("user_multi"): order = ["name", "student_id", "international", "course", "grade_type", "units"] for extra in order[order.index(slot) + 1 if slot in order else 0:][:2]: if extra in ("course", "grade_type", "units") and slot not in ("course", "grade_type", "units"): break if extra == "course" and goal["code"] is None: break acts.append(Act("provide", extra, self.want(extra), course_idx=self.idx)) return acts def maybe_change(self): """如果刚才故意说错了评分方式，这一轮改回自己真正想要的（"其实我改主意了"）。""" if getattr(self, "_pending_change", False): self._pending_change = False return [Act("change", "grade_type", self.want("grade_type"), course_idx=self.idx)] return [] def satisfies(self, i): """第 i 门课满足用户的要求吗？（按要求选课的用户：方向、每周学时、是否满足"重要实现要求"）""" goal = self.courses[i] c = self.db.by_code.get(goal["code"]) if c is None: return False req = goal.get("orig_req") if not req: return True return (c.area == req["area"] and (not req.get("max_hours") or c.hours <= req["max_hours"]) and (not req.get("sig_impl") or c.sig_impl)) def success(self, enrolled): """ 目标达成了吗？两门课都按想要的评分方式和学分选上了，而且（按要求选课时）选中的课真的满足要求。 """ want = [(c["code"], c["grade"], self.db.by_code[c["code"]].units_max) for c in self.courses if c["code"] in self.db.by_code] return len(want) == 2 and all(w in enrolled for w in want) and all(self.satisfies(i) for i in range(2)) # ============================================================================= # 第 7 部分：三种智能体 # ============================================================================= def corrupt(acts, rng): """ 模拟一次解析错误：随机挑一个用户动作，听错它的值（学分少了 1、评分方式反了），或者整个漏掉。 【返回】新的动作列表 """ acts = list(acts) k = rng.randrange(len(acts)) a = acts[k] if a.kind == "provide" and a.slot == "units": acts[k] = Act("provide", "units", a.value - 1, course_idx=a.course_idx) # 听错了数字 elif a.kind in ("provide", "change") and a.slot == "grade_type": acts[k] = Act(a.kind, "grade_type", "CR/NC" if a.value == "Letter" else "Letter", course_idx=a.course_idx) else: acts.pop(k) # 漏掉了这个信息 return acts class GenieAgent: """ Genie：解析器（模拟的 LLM）→ 形式化的对话状态 → 运行时系统决定动作 → 回复生成器。 【解析器】只看这一轮的话 + 形式化的对话状态（幻灯片第 72~74 页）；以 genie_parse_error 的概率漏掉一个信息 或填错一个值 —— 和对话有多长无关。 【运行时系统】（幻灯片第 82 页） 1. 知识查询：执行，报告结果（用户的主动提问优先回答）； 2. 校验失败的字段：说明原因，清空这个值； 3. 按顺序看每条任务记录：有没填的必填字段（考虑"只在某些情况下才问"）→ 问它； 都填了、需要确认、还没确认 → 请用户确认；确认了、还没执行 → 执行动作，报告结果。 """ name = "Genie Worksheet" def __init__(self, db, seed=0, params=None): self.db, self.rng = db, random.Random(f"genie|{seed}") self.p = dict(PARAMS) if params: self.p.update(params) self.records = [Record("task", STUDENT), Record("task", COURSE)] self.enrolled = [] self.parse_errors = 0 self.turns = 0 self.notes = [] def _course_rec(self, i): recs = [r for r in self.records if r.kind == "task" and r.ws is COURSE] while len(recs) <= i: self.records.append(Record("task", COURSE)) recs = [r for r in self.records if r.kind == "task" and r.ws is COURSE] return recs[i] def parse(self, acts): """模拟的上下文语义解析器：用户动作 → 对对话状态的更新。【返回】这一轮有没有解析错""" self.turns += 1 acts = list(acts) error = False if acts and self.rng.random() < self.p["genie_parse_error"]: error = True acts = corrupt(acts, self.rng) self.parse_errors += error self.notes = [] for a in acts: if a.kind in ("provide", "change"): rec = self.records[0] if a.slot in ("name", "student_id", "international", "visa") else self._course_rec(a.course_idx) f = next(x for x in rec.ws.fields if x.name == a.slot) problem = f.validate(rec, a.value, self.db) if f.validate else None if problem: self.notes.append(problem) rec.values[a.slot] = None else: rec.values[a.slot] = a.value rec.confirmed = False # 值变了，要重新确认 for other in rec.ws.fields: # 依赖它的字段要重新校验（换了课，学分可能不合法了） v = rec.values.get(other.name) if other.name != a.slot and v is not None and other.validate and other.validate(rec, v, self.db): rec.values[other.name] = None elif a.kind == "ask": sql, wrong = kb_parse(a.q, self.rng, self.p["kb_parse_error"]) self.records.append(Record("kb", question=ask_text(a.q, self.db), sql=sql, sql_ok=not wrong)) error = error or wrong elif a.kind == "confirm": pending = [r for r in self.records if r.kind == "task" and r.ws.confirm and not r.done and not r.missing()] if pending and a.value: pending[0].confirmed = True elif a.kind == "refuse": self.notes.append("I need your student ID to enroll you in courses; it is only used for enrollment") elif a.kind == "start": self._course_rec(a.course_idx) return error def policy(self): """运行时系统：根据形式化的对话状态决定这一轮的动作。""" acts = [] for r in self.records: # 1. 用户的主动提问优先 if r.kind == "kb" and not r.reported: r.rows = self.db.query(r.sql) r.reported = True acts.append(AgentAct("report", show_rows(r.rows), rows=r.rows, sql_ok=r.sql_ok)) for note in self.notes: # 2. 校验失败 / 解释 acts.append(AgentAct("say", note + ".")) for r in self.records: # 3. 智能体的主动权：按顺序推进任务 if r.kind != "task" or r.done: continue miss = r.missing() if miss: f = miss[0] acts.append(AgentAct("ask", f.question or f"{f.desc}?", slot=f.name)) break if r.ws.confirm and not r.confirmed: summary = {k: r.values[k] for k in ("course", "grade_type", "units")} acts.append(AgentAct("confirm", f"Shall I enroll you in {summary['course']} for {summary['units']} units, " f"{summary['grade_type']}?", slot=summary)) break if r.ws.action: result = r.ws.action(r, self) r.done = "no seats" not in result if not r.done: r.values["course"], r.confirmed = None, False acts.append(AgentAct("report", result, executed=self._last_course(r, result), confirmed_before=True)) if not r.done: break else: r.done = True if not acts: acts.append(AgentAct("say", "Is there anything else I can help you with?")) return acts @staticmethod def _last_course(r, result): """从动作的结果里取出（课程，评分方式，学分）——评估时用来检查执行的参数对不对。""" m = re.search(r"(CS \d+)", result) code = m.group(1) if m else None return (code, r.values.get("grade_type"), r.values.get("units")) def respond(self, acts): """一轮：解析 → 运行时策略。【返回】(智能体动作, 这一轮解析有没有出错)""" err = self.parse(acts) return self.policy(), err def greet(self): """对话开始：运行时系统直接问第一个字段。""" return self.policy() def state_lines(self): """形式化的对话状态，一条记录一行（演示用）。""" return [r.show() for r in self.records] class FunctionCallingAgent: """ 对照组：GPT-4 式的"函数调用"智能体（论文的基线 GPT-4 (FC)）。 【它有什么】和 Genie 一样的知识解析器（能正确地执行知识查询）和选课 API；开发者的规则写在一长段 指令里（先收集学生信息；国际学生要问签证类型；调用选课 API 之前必须确认……）。 【它的问题】（参数见 PARAMS；方向来自论文的观察） · 每一轮都要从**完整的对话历史 + 全部指令**里决定下一步：遵守指令的概率随轮次下降； 不遵守时，会犯下面几种错误之一：没确认就调用 API、忘了问签证类型、用了改主意之前的旧值、 不理用户的提问； · 回答课程问题时会编（论文：选课的 10 段对话里有 7 段出现了幻觉，常常编出不存在的课程）。 """ name = "LLM + 函数调用" def __init__(self, db, seed=0, params=None): self.db, self.rng = db, random.Random(f"fc|{seed}") self.p = dict(PARAMS) if params: self.p.update(params) self.student = {} self.courses = [{}, {}] self.history = {0: {}, 1: {}} # 每个字段第一次听到的值（"忘了改主意"时会用到） self.confirmed = [False, False] self.done = [False, False] self.cur = 0 self.turns = 0 self.enrolled = [] self.skip_visa = False self.reask_slot = None self.violations = {} # 每种违反指令的次数（失败分析用） def p_follow(self): """这一轮遵守指令的概率：随轮次线性下降，最低 0.3。""" return max(0.3, self.p["fc_follow_base"] - self.p["fc_follow_decay"] * self.turns) def greet(self): return [AgentAct("ask", "Hi! What is your name?", slot="name")] def respond(self, acts): """一轮：理解用户的话 → 回答问题（可能编）→ 决定下一步（可能违反指令）。""" self.turns += 1 rng = self.rng out = [] acts = list(acts) if acts and rng.random() < self.p["fc_parse_error"]: acts = corrupt(acts, rng) # 和 Genie 的解析器一样的错误 follow = rng.random() < self.p_follow() violation = None if follow else self._pick_violation(acts, rng) if violation: self.violations[violation] = self.violations.get(violation, 0) + 1 for a in acts: if a.kind in ("provide", "change"): target = self.student if a.slot in ("name", "student_id", "international", "visa") else self.courses[a.course_idx] target[a.slot] = a.value self.history[a.course_idx].setdefault(a.slot, a.value) if target is not self.student: self.confirmed[a.course_idx] = False elif a.kind == "confirm" and a.value: self.confirmed[self.cur] = True elif a.kind == "start": self.cur = a.course_idx elif a.kind == "ask": if violation == "ignore": out.append(AgentAct("say", "Sure! Let's continue with your enrollment.")) continue sql, wrong = kb_parse(a.q, rng, self.p["kb_parse_error"]) rows = self.db.query(sql) if rng.random() < self.p["fc_hallucinate"]: if a.q["type"] == "find": fake = (f"CS {rng.randint(350, 399)}", f"Advanced {a.q['area']} Topics", 6, 4.9) out.append(AgentAct("report", show_rows([fake] + rows[:2]), rows=[fake] + rows[:2], grounded=False, sql_ok=not wrong)) else: out.append(AgentAct("report", "It meets on Tuesdays at noon and is 1-6 units.", rows=[("TTh", "12:00")], grounded=False)) else: out.append(AgentAct("report", show_rows(rows), rows=rows, sql_ok=not wrong)) elif a.kind == "refuse": out.append(AgentAct("say", "I understand. I need your student ID to complete the enrollment.")) out += self._next(violation) return out, False def _pick_violation(self, acts, rng): """ 这一轮不遵守指令时，犯哪一种错误？只从"在当前情境下会造成影响"的错误里选： ignore ：用户问了问题，它不理，接着推进任务； premature ：这门课的信息都齐了、还没确认，就直接调用选课 API； skip_visa ：国际学生还没给签证类型，它跳过这个问题； stale ：用户改过主意，它提交时用了旧的值； reask ：跟丢了之前的信息，再问一遍一个已经回答过的字段（任何时候都可能发生，但权重较低）。 都不适用时返回 None（这一轮照常）。 """ c = self.courses[self.cur] known = [k for k, v in self.student.items() if v is not None] + [k for k, v in c.items() if v is not None] options = ["reask"] if known else [] # "跟丢了"：任何时候都可能发生 if any(a.kind == "ask" for a in acts): options.append("ignore") if all(c.get(f) is not None for f in ("course", "grade_type", "units")) and not self.confirmed[self.cur]: options.append("premature") if self.student.get("international") and not self.student.get("visa"): options.append("skip_visa") if any(self.history[self.cur].get(k) not in (None, c.get(k)) for k in ("grade_type", "units")) \ and all(c.get(f) is not None for f in ("course", "grade_type", "units")): options.append("stale") if not options: return None # 没有可犯的错误：这一轮照常 weights = [1 if o == "reask" else 3 for o in options] # 其他错误一旦适用，更可能发生 choice = rng.choices(options, weights=weights)[0] if choice == "reask": self.reask_slot = rng.choice(known) return choice def _next(self, violation): """决定下一步：遵守指令时和 Genie 的运行时一样；不遵守时犯一种错误。""" s, c, i = self.student, self.courses[self.cur], self.cur if violation == "reask" and self.reask_slot: slot = self.reask_slot return [AgentAct("ask", f"Sorry, could you remind me of your {slot.replace('_', ' ')}?", slot=slot)] if violation == "skip_visa" and s.get("international") and not s.get("visa"): self.skip_visa = True for f in ("name", "student_id", "international", "visa"): if f == "visa" and (not s.get("international") or self.skip_visa): continue if s.get(f) is None: return [AgentAct("ask", f"What is your {f.replace('_', ' ')}?", slot=f)] if self.done[i]: return [AgentAct("say", "Is there anything else I can help you with?")] for f in ("course", "grade_type", "units"): if c.get(f) is None: return [AgentAct("ask", f"{f.replace('_', ' ').capitalize()}?", slot=f)] if c["course"] not in self.db.by_code: c["course"] = None return [AgentAct("say", "Sorry, that course is not available. Which course would you like?")] values = dict(c) if violation == "stale": values = {k: (self.history[i].get(k, v) if k in ("grade_type", "units") else v) for k, v in c.items()} if violation == "stale": return [self._enroll(values, confirmed_before=False)] if not self.confirmed[i] and violation != "premature": return [AgentAct("confirm", f"Shall I enroll you in {c['course']} for {c['units']} units, {c['grade_type']}?", slot={k: c[k] for k in ("course", "grade_type", "units")})] return [self._enroll(values, confirmed_before=self.confirmed[i])] def _enroll(self, values, confirmed_before): """调用选课 API（和 Genie 用的是同一个动作函数）。""" rec = Record("task", COURSE, values=dict(values)) result = enroll_action(rec, self) i = self.cur if "no seats" in result or "not available" in result: self.courses[i]["course"], self.confirmed[i] = None, False else: self.done[i] = True m = re.search(r"(CS \d+)", result) return AgentAct("report", result, executed=(m.group(1) if m else None, values["grade_type"], values["units"]), confirmed_before=confirmed_before) class DialogueTreeAgent: """ 对照组：对话树（商用客服系统的常见做法，幻灯片第 17~19 页）。 【做法】按固定的顺序问：姓名 → 学号 → 是否国际学生 →（签证）→ 课程 → 评分方式 → 学分 → 确认 → 提交 → 下一门。 每一步只听得懂"这一步预期的回答"；其他的一律"Sorry, I didn't understand."，再问一遍。 【好处】完全可控：不会编、不会跳过确认。 【问题】不支持用户的主动提问（查课程）、改主意、拒绝、一次说好几件事（多说的部分被丢掉）。 """ name = "对话树" def __init__(self, db, seed=0, params=None): self.db = db self.steps = ["name", "student_id", "international", "visa", "course", "grade_type", "units", "confirm"] self.k = 0 self.values = {} self.course_idx = 0 self.enrolled = [] def _question(self): step = self.steps[self.k] questions = {f.name: f.question for f in STUDENT.fields + COURSE.fields} if step == "confirm": v = self.values return AgentAct("confirm", f"Shall I enroll you in {v['course']} for {v['units']} units, {v['grade_type']}?", slot={x: v[x] for x in ("course", "grade_type", "units")}) return AgentAct("ask", questions[step], slot=step) def greet(self): return [self._question()] def respond(self, acts): step = self.steps[self.k] if step == "confirm": yes = [a for a in acts if a.kind == "confirm"] if yes and yes[0].value: return [self._submit()] + self._after_submit(), False if yes: self.k = self.steps.index("course") # 不对：从课程开始重新问 return [self._question()], False else: hit = [a for a in acts if a.kind == "provide" and a.slot == step] if hit: self.values[step] = hit[0].value self.k += 1 if self.steps[self.k] == "visa" and not self.values.get("international"): self.k += 1 return [self._question()], False return [AgentAct("say", "Sorry, I didn't understand. " + self._question().text)], False def _submit(self): v = dict(self.values) rec = Record("task", COURSE, values={k: v[k] for k in ("course", "grade_type", "units")}) if v["course"] not in self.db.by_code: return AgentAct("say", "Sorry, I didn't understand that course.") result = enroll_action(rec, self) return AgentAct("report", result, executed=(v["course"], v["grade_type"], v["units"]), confirmed_before=True) def _after_submit(self): self.course_idx += 1 self.k = self.steps.index("course") return [self._question()] if self.course_idx < 2 else [AgentAct("say", "Goodbye!")] AGENTS = [DialogueTreeAgent, FunctionCallingAgent, GenieAgent] # ============================================================================= # 第 8 部分：对话的运行与评估 # ============================================================================= @dataclass class Turn: """一轮对话的记录（评估用）。""" index: int user: str user_acts: list agent: list parse_error: bool da_ok: bool exec_ok: object # None：这一轮没有调用 API / 数据库；True / False：调用对不对 hallucinated: bool def judge(agent_acts, user_acts, user, last_confirm): """ 这一轮智能体的表现对不对（"对话动作准确率"和"执行准确率"的模拟版）。 【对话动作对不对】（是否遵守开发者的规则） · 用户问了课程的问题 → 必须报告数据库里查到的结果（不理、或者编，都算错）； · 调用选课 API → 必须事先得到用户的确认，而且确认时的值就是这次提交的值； · 国际学生还没给签证类型，就去确认或提交选课 → 算错（漏了数据依赖的字段）。 【执行对不对】这一轮如果查询了数据库或调用了 API：查询的结果是真实的、提交的参数是用户确认过的。 【返回】(对话动作对不对, 执行对不对或 None, 有没有编) """ asked = any(a.kind == "ask" for a in user_acts) halluc = any(a.kind == "report" and a.rows is not None and not a.grounded for a in agent_acts) ok = not halluc if asked and not any(a.kind == "report" and a.rows is not None for a in agent_acts): ok = False exec_ok = None for a in agent_acts: if a.kind == "report" and a.rows is not None: good = a.grounded and a.sql_ok exec_ok = good if exec_ok is None else (exec_ok and good) if a.executed is not None: good = a.confirmed_before and last_confirm is not None and \ (last_confirm.get("course"), last_confirm.get("grade_type"), last_confirm.get("units")) == a.executed ok = ok and good exec_ok = good if exec_ok is None else (exec_ok and good) if a.kind in ("confirm",) or a.executed is not None: if user.intl and "visa" not in user.told: ok = False if a.kind == "ask" and (a.slot in user.told or (user.idx, a.slot) in user.told) and "remind" in a.text: ok = False # 跟丢了：再问一遍已经回答过的问题 return ok, exec_ok, halluc def run_dialogue(agent_cls, db, seed, happy=False, params=None): """ 让一个模拟用户和一个智能体对话，直到用户完成、放弃，或达到最大轮数。 【返回】dict：{"turns": [Turn], "success": bool, "gave_up": bool, "agent": 智能体对象} 【成功的标准】两门课都按用户想要的评分方式和学分选上了；如果是国际学生，签证类型也收集了。 """ user = SimUser(db, seed, happy=happy) agent = agent_cls(db, seed=seed, params=params) agent_acts = agent.greet() turns, last_confirm = [], None for t in range(1, PARAMS["max_turns"] + 1): confirm_req = next((a.slot for a in agent_acts if a.kind == "confirm"), None) user_acts = user.maybe_change() + user.react(agent_acts) if not user_acts: break if confirm_req and any(a.kind == "confirm" and a.value for a in user_acts): last_confirm = dict(confirm_req) for a in user_acts: if a.kind in ("provide", "change"): user.told[a.slot if a.slot in ("name", "student_id", "international", "visa") else (a.course_idx, a.slot)] = a.value text = utter(user_acts, db) agent_acts, perr = agent.respond(user_acts) ok, exec_ok, halluc = judge(agent_acts, user_acts, user, last_confirm) turns.append(Turn(t, text, user_acts, agent_acts, perr, ok, exec_ok, halluc)) student_ok = (not user.intl) or ("visa" in user.told and (isinstance(agent, DialogueTreeAgent) or not getattr(agent, "skip_visa", False))) return {"turns": turns, "success": user.success(agent.enrolled) and student_ok, "gave_up": user.gave_up, "agent": agent, "user": user}