""" ============================================================================= 第 8 讲 · 工具模块：一个虚构的 Wikidata 式知识图谱，以及 SPINACH 的四个工具 ============================================================================= 对应讲义：lectures/lecture08_spinach.md 第 1 节、第 3 节 ----------------------------------------------------------------------------- 【这个模块做什么？】 ----------------------------------------------------------------------------- Wikidata 有 150 亿条事实、上亿个实体、上万种属性。这里生成一个几千条事实的虚构版本， 但刻意保留了 Wikidata 让人头疼的几个"怪癖"（SPINACH 论文："Wikidata 的结构很反常"）： 1. 没有固定的表结构：同一类实体，有的有某个属性，有的没有（例如只有一部分人记了"居住地"）； 2. 不完整：很多城市只记了"所在的地区"（P131），没记"所在的国家"（P17）——要沿着 P131 往上走； 3. 粒度不一：乐手的"乐器"（P1303）多数记的是具体的乐器（tabla），少数记的是大类（打击乐器）—— 问"演奏打击乐器的乐手"，要沿着"子类"（P279）往上走； 4. 只记一边：导师和学生的关系，学生那边一定记了"导师"（P184），导师那边只有一部分记了"学生"（P185）； 5. 限定词：获奖的年份不在"获奖"（P166）这条边上，而在这条陈述的限定词（P585）里（p: / ps: / pq: 写法）； 6. 相似的属性："height"（P2048，人的身高）和 "elevation above sea level"（P2044，山的海拔）； "filming location"（P915，拍摄地）和 "narrative location"（P840，故事发生地）； 7. 同名的实体：两个国家各有一座叫 Ashford 的城市。 · 实体的编号（QID）是虚构的（从 Q1000 开始）；属性的编号（PID）沿用真实 Wikidata 的编号。 · 全部人名、地名、机构名都是虚构的。 ----------------------------------------------------------------------------- 【SPINACH 的工具】（论文第 4.1 节；每一个都对应人类专家会做的一件事） ----------------------------------------------------------------------------- search_wikidata(文字) ≈ 在 wikidata.org 的搜索框里搜：返回最多 8 个实体、4 个属性 get_wikidata_entry(QID) ≈ 打开一个实体的页面：它的全部"出边"（属性 → 值，含限定词） get_property_examples(PID) ≈ 打开一个属性的页面：几个用法示例 execute_sparql(查询) ≈ 在 query.wikidata.org 上执行：结果太多时只显示前 5 行和后 5 行 ----------------------------------------------------------------------------- 【用法】 ----------------------------------------------------------------------------- from kg_world import make_kg kg = make_kg(seed=0) print(kg.search_wikidata("Northgate University School of Music")) print(kg.execute_sparql("SELECT ?p WHERE { ?p wdt:P1416 wd:Q1060 . }")) """ import random import re from dataclasses import dataclass, field from sparql import Engine, SPARQLSyntaxError, SPARQLTimeout, sort_key # 属性：PID → (名字, 别名, 说明)。编号和名字沿用真实的 Wikidata。 PROPERTIES = { "P31": ("instance of", ["is a", "type"], "that class of which this subject is a particular example"), "P279": ("subclass of", ["type of", "kind of"], "this item is a subclass of that item"), "P361": ("part of", ["belongs to", "division of"], "object of which the subject is a part"), "P17": ("country", ["sovereign state"], "sovereign state that this item is in"), "P131": ("located in the administrative territorial entity", ["located in", "in region", "in province"], "the item is located on the territory of the following administrative entity"), "P1082": ("population", ["number of inhabitants"], "number of people inhabiting the place"), "P106": ("occupation", ["profession", "job", "work"], "occupation of a person"), "P569": ("date of birth", ["born on", "birthday", "birth date"], "date on which the subject was born"), "P19": ("place of birth", ["born in", "birthplace"], "most specific known birth location of a person"), "P551": ("residence", ["lives in", "lived in", "home"], "the place where the person is or has been resident"), "P27": ("country of citizenship", ["nationality", "citizen of"], "the object is a country that recognizes the subject as its citizen"), "P69": ("educated at", ["alma mater", "studied at", "graduated from"], "educational institution attended by subject"), "P108": ("employer", ["works at", "worked for", "employed by"], "person or organization for which the subject works or worked"), "P1416": ("affiliation", ["affiliated with", "member of department"], "organization that a person or organization is affiliated with"), "P1303": ("instrument", ["plays", "musical instrument played", "instruments played"], "musical instrument that a person plays"), "P184": ("doctoral advisor", ["advisor", "PhD advisor", "supervisor"], "person who supervised the doctorate of the subject"), "P185": ("doctoral student", ["PhD student", "students", "doctoral students"], "doctoral student(s) of a professor"), "P166": ("award received", ["award", "prize", "won"], "award or recognition received by a person or organization"), "P585": ("point in time", ["date", "year", "when"], "time and date something took place (qualifier)"), "P159": ("headquarters location", ["based in", "HQ", "headquartered in"], "city where an organization's headquarters is"), "P571": ("inception", ["founded", "founding date", "established"], "date or point in time when the subject came into existence"), "P112": ("founded by", ["founder", "co-founder"], "founder or co-founder of this organization"), "P2044": ("elevation above sea level", ["altitude", "elevation", "height above sea level"], "height of the item (geographical object) as measured relative to sea level"), "P2048": ("height", ["tallness", "how tall"], "vertical length of an entity (person, building, ...)"), "P57": ("director", ["directed by", "film director"], "director(s) of film, TV-series, stageplay, video game or similar"), "P915": ("filming location", ["shot in", "filmed in", "film location"], "actual place where this scene/film was shot"), "P840": ("narrative location", ["set in", "takes place in", "location", "where does it take place"], "the narrative of the work is set in this location"), } COUNTRIES = ["Kestavia", "Orundi", "Pallasca", "Tirenne", "Moravek", "Sundavia"] CITY_NAMES = ["Ashford", "Brennick", "Calloway", "Dunmere", "Elsworth", "Farnley", "Glenhaven", "Harrowgate", "Ivybridge", "Jasperton", "Kilbride", "Lowmoor", "Marlow", "Northam", "Oakhurst", "Penwick", "Quarry Hill", "Redwater", "Stanmore", "Thornbury", "Upton", "Valemont", "Westerly", "Yarrow", "Zelcourt", "Amberly", "Birchmont", "Coldridge", "Dovecote", "Emberton", "Foxley", "Greystone", "Hollins", "Irvine Falls", "Juniper", "Kingsley"] FIRST = ["Aldo", "Bruna", "Cosmin", "Delia", "Emeric", "Fiora", "Gaspar", "Hedda", "Ivo", "Jolanta", "Kasimir", "Linnea", "Matteo", "Nadia", "Orrin", "Petra", "Quill", "Rosalind", "Stellan", "Talia", "Ulric", "Vesna", "Wendell", "Xenia", "Yorick", "Zora"] LAST = ["Abernathy", "Bellweather", "Castellane", "Draycott", "Eversley", "Fairbanks", "Grimaldi", "Hollowell", "Isenberg", "Jardine", "Kettleby", "Lindqvist", "Montague", "Northcott", "Oakenshaw", "Pemberton", "Quenneville", "Rothwell", "Sinclair", "Thackeray", "Underhill", "Vandermeer", "Whitlock", "Yardley"] INSTRUMENT_TREE = { # 大类 → 具体的乐器 "percussion instrument": ["tabla", "drum kit", "marimba", "timpani"], "string instrument": ["violin", "cello", "sitar", "guitar"], "keyboard instrument": ["piano", "organ", "harpsichord"], "wind instrument": ["trombone", "flute", "clarinet", "trumpet"], } STOPWORDS = {"a", "an", "the", "of", "in", "on", "at", "to", "for", "by", "and", "or", "is", "was", "were", "did", "do", "does", "what", "which", "who", "where", "when", "how", "many", "much", "with", "its", "their"} OCCUPATIONS = ["musician", "physicist", "mathematician", "film director", "entrepreneur", "actor"] @dataclass class Item: """一个实体或属性：编号、名字、别名、说明（搜索和实体页都用到它们）。""" id: str label: str aliases: list = field(default_factory=list) description: str = "" class KG: """ 知识图谱 + SPINACH 的四个工具。 【属性】 items : 编号（"Q1060"、"P1416"）→ Item triples : list[(主语, 谓语, 宾语)]，主语和实体宾语写成 "wd:Q1060"，谓语写成 "wdt:P1416"， 字面值是 Python 的 str / int（日期写成 "YYYY-MM-DD"） engine : 迷你 SPARQL 引擎 by_label : 名字 → 编号列表（同名的实体会有好几个） """ def __init__(self, items, triples): self.items = items self.triples = triples for qid, it in items.items(): if qid.startswith("Q"): triples.append((f"wd:{qid}", "rdfs:label", it.label)) self.engine = Engine(triples) self.out, self.inn = {}, {} # 出边：主语 → [(谓语, 宾语)]；入边：宾语 → [(谓语, 主语)] for s, p, o in triples: self.out.setdefault(s, []).append((p, o)) if isinstance(o, str): self.inn.setdefault(o, []).append((p, s)) self.degree = {} for s, p, o in triples: if isinstance(o, str) and o.startswith("wd:Q"): self.degree[o] = self.degree.get(o, 0) + 1 self.by_label = {} for qid, it in items.items(): self.by_label.setdefault(it.label, []).append(qid) # ---- 显示 ---- def name(self, v): """把一个值显示成人能读的样子：实体显示成"名字 (QID)"，字面值原样显示。""" if isinstance(v, str) and v.startswith("wd:Q"): it = self.items.get(v[3:]) return f"{it.label} ({v[3:]})" if it else v return str(v) # ---- 工具 1：搜索 ---- def search_wikidata(self, text, k_ent=8, k_prop=4): """ 在名字和别名里搜索（模拟 Wikidata 的 wbsearchentities API）。 【思路】按"查询的词有多大比例出现在名字/别名里"打分；名字和查询完全一样的排最前； 分数相同时，被引用次数多的实体（"更有名的"）排前面。 【返回】(实体列表, 属性列表)，每项是 (编号, 名字, 说明) 【边界】只做词级别的匹配：搜 "School of Music" 能找到三所大学各自的音乐学院（需要再看说明来区分）； 搜 "tallest" 什么也找不到（名字里没有这个词）。 """ q = set(re.findall(r"[a-z0-9]+", text.lower())) - STOPWORDS if not q: return [], [] scored = [] for iid, it in self.items.items(): best = 0.0 for name in [it.label] + it.aliases: words = set(re.findall(r"[a-z0-9]+", name.lower())) - STOPWORDS if not words: continue overlap = len(q & words) if overlap == 0: continue s = overlap / max(len(q), len(words)) + (1.0 if name.lower() == text.lower().strip() else 0.0) best = max(best, s) if best > 0: scored.append((-best, -self.degree.get(f"wd:{iid}", 0), sort_key(f"wd:{iid}") if iid[0] == "Q" else (3, int(iid[1:]), ""), iid)) scored.sort() ents = [(i, self.items[i].label, self.items[i].description) for *_r, i in scored if i[0] == "Q"][:k_ent] props = [(i, self.items[i].label, self.items[i].description) for *_r, i in scored if i[0] == "P"][:k_prop] return ents, props # ---- 工具 2：实体页 ---- def get_wikidata_entry(self, qid): """ 一个实体的全部出边（模拟打开 wikidata.org/wiki/QID）：每行 "PID (属性名): 值"，带限定词的陈述把限定词写在方括号里。 【返回】list[str]；QID 不存在时返回 ["No entity with this QID"] """ node = f"wd:{qid}" if qid not in self.items: return ["No entity with this QID"] lines = [f"label: {self.items[qid].label}", f"description: {self.items[qid].description}"] quals = {} for p, o in self.out.get(node, []): if p.startswith("p:"): pid = p[2:] vals = dict((pp, oo) for pp, oo in self.out.get(o, [])) quals.setdefault(pid, {})[vals.get(f"ps:{pid}")] = [(pp[3:], oo) for pp, oo in vals.items() if pp.startswith("pq:")] for p, o in self.out.get(node, []): if not p.startswith("wdt:"): continue pid = p[4:] q = quals.get(pid, {}).get(o, []) extra = " [" + "; ".join(f"{qp} ({PROPERTIES[qp][0]}): {self.name(qv)}" for qp, qv in q) + "]" if q else "" lines.append(f"{pid} ({PROPERTIES[pid][0]}): {self.name(o)}{extra}") return lines # ---- 工具 3：属性的用法示例 ---- def get_property_examples(self, pid, k=3): """ 一个属性的几个用法示例（模拟打开 wikidata.org/wiki/Property:PID）：主语 → 宾语。 【返回】list[str]，例如 ["Matteo Pemberton (Q1102) -> 182", ...]；带限定词的属性会把限定词也写出来。 """ if pid not in PROPERTIES: return ["No property with this PID"] pairs = [(s, o) for s, p, o in self.triples if p == f"wdt:{pid}"] pairs.sort(key=lambda so: (sort_key(so[0]), sort_key(so[1]))) out = [] for s, o in pairs[:k]: line = f"{self.name(s)} -> {self.name(o)}" for st_p, st in self.out.get(s, []): if st_p == f"p:{pid}": vals = dict(self.out.get(st, [])) if vals.get(f"ps:{pid}") == o: q = [f"{pp[3:]}: {self.name(v)}" for pp, v in vals.items() if pp.startswith("pq:")] line += f" (qualifiers: {', '.join(q)}; use p:{pid}/ps:{pid}/pq:...)" if q else "" out.append(line) return out # ---- 工具 4：执行 SPARQL ---- def execute_sparql(self, query, show=5): """ 执行一条查询，返回 (状态, 结果行, 给智能体看的文字)。 【状态】"ok"（有结果）/ "empty"（没有结果）/ "syntax"（语法错误）/ "timeout"（超时） 【给智能体看的文字】结果太多时只显示前 show 行和后 show 行（SPINACH 论文：限制智能体要读的信息量）。 """ try: rows = self.engine.run(query) except SPARQLSyntaxError as e: return "syntax", [], f"Syntax error: {e}" except SPARQLTimeout: return "timeout", [], "The query timed out (too many intermediate results). Try a more specific query." if not rows: return "empty", [], "No results." shown = rows if len(rows) <= 2 * show else rows[:show] + [None] + rows[-show:] text = "\n".join(" ..." if r is None else " " + " | ".join(self.name(v) for v in r) for r in shown) return "ok", rows, f"{len(rows)} rows:\n{text}" # ============================================================================= # 生成图谱 # ============================================================================= def make_kg(seed=0): """ 生成虚构的知识图谱。同一个 seed 总是生成同一个图谱。 【规模】6 个国家、12 个地区、36 座城市；6 所大学（3 所有音乐学院）；19 种乐器；约 200 个人； 25 家公司；30 座山；40 部电影；5 个奖项。几千条三元组。 【返回】KG """ rng = random.Random(f"kg|{seed}") items = {pid: Item(pid, lab, list(al), desc) for pid, (lab, al, desc) in PROPERTIES.items()} triples = [] counter = [1000] def new(label, desc, aliases=()): qid = f"Q{counter[0]}" counter[0] += 1 items[qid] = Item(qid, label, list(aliases), desc) return qid def add(s, p, o): o = f"wd:{o}" if isinstance(o, str) and re.fullmatch(r"Q\d+", o) else o triples.append((f"wd:{s}", f"wdt:{p}", o)) def add_statement(s, p, o, qualifiers): """一条带限定词的陈述：wdt: 直接边 + p:/ps:/pq: 的陈述节点。""" add(s, p, o) st = f"wds:{s}-{p}-{len(triples)}" triples.append((f"wd:{s}", f"p:{p}", st)) triples.append((st, f"ps:{p}", f"wd:{o}")) for qp, qv in qualifiers: triples.append((st, f"pq:{qp}", qv)) # ---- 类 ---- cls = {name: new(name, desc) for name, desc in [ ("human", "common name of Homo sapiens"), ("city", "large human settlement"), ("province", "administrative region"), ("country", "distinct territorial body or political entity"), ("university", "academic institution for further education"), ("academic department", "division of a university"), ("film", "sequence of images that give the impression of movement"), ("mountain", "large natural elevation of the earth's surface"), ("business", "organization undertaking commercial activity"), ("award", "something given in recognition of an excellence"), ("occupation", "type of job or activity"), ("musical instrument", "device created or adapted to make musical sounds")]} occ = {o: new(o, "occupation") for o in OCCUPATIONS} for o in occ.values(): add(o, "P31", cls["occupation"]) inst_cls, instruments = {}, {} for big, smalls in INSTRUMENT_TREE.items(): inst_cls[big] = new(big, "class of musical instruments") add(inst_cls[big], "P279", cls["musical instrument"]) for sm in smalls: instruments[sm] = new(sm, f"musical instrument ({big})") add(instruments[sm], "P279", inst_cls[big]) # ---- 地理：国家、省、城市 ---- country, provinces, cities, city_country = {}, [], [], {} for c in COUNTRIES: country[c] = new(c, "country (fictional)") add(country[c], "P31", cls["country"]) names = list(CITY_NAMES) k = 0 for c in COUNTRIES: for j in range(2): pv = new(f"{names[k]} Province", f"province of {c}") add(pv, "P31", cls["province"]) add(pv, "P17", country[c]) provinces.append(pv) for _ in range(3): label = names[k] k += 1 q = new(label, f"city in {c}") add(q, "P31", cls["city"]) add(q, "P131", pv) if rng.random() < 0.5: # 怪癖 2：一半的城市没有直接记国家 add(q, "P17", country[c]) add(q, "P1082", rng.randint(20, 900) * 1000) cities.append(q) city_country[q] = c # 怪癖 7：另一个国家也有一座 Ashford ash2 = new("Ashford", f"city in {COUNTRIES[3]}") add(ash2, "P31", cls["city"]) add(ash2, "P131", provinces[6]) add(ash2, "P1082", 61000) cities.append(ash2) city_country[ash2] = COUNTRIES[3] # ---- 大学和院系 ---- universities, music_schools = [], {} for name, city_i in [("Northgate University", 0), ("Ashford Institute of Technology", 0), ("University of Marlow", 12), ("Valemont University", 21), ("Kingsley College", 35), ("Redwater University", 17)]: u = new(name, f"university in {items[cities[city_i]].label}, {city_country[cities[city_i]]}") add(u, "P31", cls["university"]) add(u, "P131", cities[city_i]) universities.append(u) if name in ("Northgate University", "University of Marlow", "Valemont University"): sch = new(f"{name} School of Music", f"music school, part of {name}") add(sch, "P31", cls["academic department"]) add(sch, "P361", u) music_schools[u] = sch # ---- 人 ---- people, used = [], set() def person(occupation, born_year): while True: label = f"{rng.choice(FIRST)} {rng.choice(LAST)}" if label not in used: used.add(label) break home = rng.choice(cities) p = new(label, f"{occupation} from {city_country[home]} (born {born_year})") add(p, "P31", cls["human"]) add(p, "P106", occ[occupation]) add(p, "P569", f"{born_year}-{rng.randint(1, 12):02d}-{rng.randint(1, 28):02d}") add(p, "P19", home) if rng.random() < 0.4: add(p, "P551", rng.choice(cities)) # 怪癖 1：只有一部分人记了居住地 if rng.random() < 0.4: add(p, "P2048", rng.randint(150, 200)) # 身高（厘米） people.append(p) return p musicians = [] for _ in range(90): p = person("musician", rng.randint(1940, 2000)) musicians.append(p) big = rng.choice(list(INSTRUMENT_TREE)) if rng.random() < 0.15: add(p, "P1303", inst_cls[big]) # 怪癖 3：少数记的是大类 else: add(p, "P1303", instruments[rng.choice(INSTRUMENT_TREE[big])]) if rng.random() < 0.5: u = rng.choice(universities) if u in music_schools and rng.random() < 0.6: add(p, "P1416", music_schools[u]) # 隶属于音乐学院 else: add(p, "P1416", u) # 隶属于大学本身 if rng.random() < 0.6: add(p, "P69", rng.choice(universities)) # 和"隶属"容易混淆的"就读于" awards = [new(n, d) for n, d in [("Orund Prize in Physics", "award in physics"), ("Castellane Medal", "award in mathematics"), ("Kestavian Order of Merit", "national honour of Kestavia"), ("Golden Lyre", "music award"), ("Silver Reel", "film award")]] for a in awards: add(a, "P31", cls["award"]) scientists = [] for i in range(70): occupation = "physicist" if i % 2 == 0 else "mathematician" p = person(occupation, rng.randint(1900, 1990)) scientists.append(p) add(p, "P69", rng.choice(universities)) add(p, "P108", rng.choice(universities)) if rng.random() < 0.3: year = int(next(o for s, pp, o in triples if s == f"wd:{p}" and pp == "wdt:P569")[:4]) + rng.randint(35, 60) add_statement(p, "P166", awards[0] if occupation == "physicist" else awards[1], [("P585", f"{year}-01-01")]) # 学术谱系：按出生年份排好，每个人的导师从比他早出生的人里选 scientists.sort(key=lambda q: next(o for s, pp, o in triples if s == f"wd:{q}" and pp == "wdt:P569")) for i, p in enumerate(scientists[8:], start=8): adv = scientists[rng.randint(max(0, i - 12), i - 1)] add(p, "P184", adv) # 学生那边一定记了导师 if rng.random() < 0.5: add(adv, "P185", p) # 怪癖 4：导师那边只记了一半 directors = [person("film director", rng.randint(1930, 1990)) for _ in range(12)] for _ in range(18): person("actor", rng.randint(1940, 2000)) founders = [person("entrepreneur", rng.randint(1920, 1990)) for _ in range(20)] # ---- 公司 ---- for i in range(25): name = f"{rng.choice(['Apex', 'Blue', 'Crest', 'Delta', 'Ember', 'Frontier', 'Granite', 'Harbor', 'Iris', 'Juno'])} " \ f"{rng.choice(['Systems', 'Foods', 'Motors', 'Labs', 'Textiles', 'Energy', 'Media', 'Logistics'])} {i}" city = rng.choice(cities) c = new(name, f"company based in {items[city].label}") add(c, "P31", cls["business"]) add(c, "P159", city) add(c, "P571", f"{rng.randint(1890, 2020)}-{rng.randint(1, 12):02d}-01") add(c, "P112", rng.choice(founders)) # ---- 山 ---- for i in range(30): c = rng.choice(COUNTRIES) m = new(f"Mount {rng.choice(LAST)} {i}", f"mountain in {c}") add(m, "P31", cls["mountain"]) add(m, "P17", country[c]) add(m, "P2044", rng.randint(800, 6500)) # 海拔用 P2044，不是 P2048 # ---- 电影 ---- for i in range(40): f = new(f"{rng.choice(['The', 'A', 'Our', 'Last'])} {rng.choice(['Winter', 'Harbor', 'Echo', 'Garden', 'Signal', 'Lantern'])} " f"{rng.choice(['Song', 'Road', 'Hours', 'Promise', 'Letters'])} ({1950 + i})", f"{1950 + i} film") add(f, "P31", cls["film"]) add(f, "P57", rng.choice(directors)) shots = rng.sample(cities[:36], rng.choice([1, 1, 2])) for c in shots: add(f, "P915", c) add(f, "P840", shots[0] if rng.random() < 0.4 else rng.choice(cities[:36])) # 怪癖 6：故事发生地 ≠ 拍摄地 kg = KG(items, triples) kg.cls, kg.occ, kg.inst_cls, kg.instruments = cls, occ, inst_cls, instruments kg.countries, kg.cities, kg.city_country = country, cities, city_country kg.universities, kg.music_schools, kg.awards = universities, music_schools, awards kg.musicians, kg.scientists = musicians, scientists return kg if __name__ == "__main__": kg = make_kg() print(f"{len(kg.items)} 个实体和属性，{len(kg.triples)} 条三元组") ents, props = kg.search_wikidata("Northgate University School of Music") print(ents[:3], props) q = ents[0][0] print("\n".join(kg.get_wikidata_entry(q))) print(kg.get_property_examples("P166")) print(kg.execute_sparql(f"SELECT ?p ?i WHERE {{ ?p wdt:P1416 wd:{q} . ?p wdt:P1303 ?i . }}")[2])