""" ============================================================================= CS525 · 第 6 讲模块：一个"知道真相"的合成网页世界（webtext_world.py） ============================================================================= 对应讲义：lectures/lecture06_pretraining_data_dclm.md（第 7 讲也会只读地导入它） ----------------------------------------------------------------------------- 【为什么要造这个世界】 ----------------------------------------------------------------------------- 预训练数据研究最难的地方是：真实的网页没有"真相"。一个网页好不好、里面的事实对不对、 它是不是另一个网页的复制品，都不知道；只能训练一个模型、看评测分数。 这个模块造一个小小的"互联网"，它的一切都是已知的： 世界知识 600 个虚构的国家（实体），每个国家有 5 个属性（关系）： 首都 capital、出口 exports、语言 speaks、气候 climate、动物 animal。 一共 600 × 5 = 3000 条"事实"，例如 zoka capital belmiro （zoka 的首都是 belmiro）。 国家有"热度"：少数国家被反复提到，多数国家很少被提到（长尾，Zipf 分布）。 网页 每个网页属于 6 种类型之一（真实网页没有这个标签，我们有）： WIKI 百科式：正式的措辞，正确的事实；主题多半是热门国家，也顺带提到别的国家； FORUM 问答论坛式：口语化的一问一答，事实基本正确，什么国家都会问到（覆盖长尾）； SPAM 营销垃圾：措辞"像模像样"，但事实多半是错的，还很啰嗦、重复； NAV 导航页、目录页：home about contact page 1 page 2……没有内容； GARBLED 机器翻译 / 拼凑出来的低质量文本：有信息，但事实三元组常被插入的杂词打断； FOREIGN 外语网页（这里的"英语"模型用不上）。 HTML 网页外面包着导航栏、页脚（"WET 文件"会把它们一起抽出来，好的抽取器能去掉）。 重复 镜像站、模板页：一部分网页被完全复制、或改几个词再复制。 【模型怎么"学到"事实】 第 6、7 讲在这个世界里训练一个极小的语言模型（textlab.py）。一条事实在文本里总是以 三个连续的词出现： <国家> <关系> <值>。所以只要模型学会"看到 zoka capital，下一个词是 belmiro"， 它就"知道"了这条事实。评测题就是：给出 <国家> <关系>，在候选值里选概率最高的那个。 【运行自检】 python3 webtext_world.py # 生成一个小池子，打印每种网页的例子和统计 """ import numpy as np # ============================================================================= # 词表 Vocabulary # ============================================================================= # 【思路】词分成几类，每类有自己的"风格"。启发式过滤器和质量分类器能看到的，正是这些风格特征； # 而"事实对不对"只有世界本身知道。 STOP = ["the", "of", "and", "to", "in", "a", "is", "that", "it", "for", "as", "with", "on", "by", "this", "from", "at", "be", "or", "are", "was", "its", "has", "which"] FORMAL = ["history", "region", "located", "population", "century", "known", "official", "according", "notable", "record", "territory", "founded", "administrative", "modern", "ancient", "culture", "economy", "geography", "province", "historians", "documented", "primarily", "significant", "major"] INFORMAL = ["question", "answer", "thanks", "i", "you", "think", "because", "so", "actually", "hope", "helps", "edit", "guess", "maybe", "anyone", "know", "yes", "sure", "pretty", "right", "lol", "btw", "my", "friend", "went", "there", "told", "me"] SPAMW = ["best", "buy", "cheap", "offer", "deal", "now", "free", "click", "discount", "top", "amazing", "guaranteed", "limited", "sale", "order", "shipping", "exclusive", "luxury", "hotel", "booking"] NAVW = ["home", "about", "contact", "login", "menu", "privacy", "terms", "copyright", "share", "next", "prev", "page", "search", "subscribe", "cart", "help", "sitemap", "faq", "careers", "archive"] JUNK = ["~", "*", "|", "#", "&", "%", "$", "@"] # 机器翻译 / 拼凑文本里夹杂的乱码符号 NUMS = [str(i) for i in range(1, 41)] PUNCT = [".", "?", ",", ":"] FOREIGNW = ["der", "und", "ist", "le", "la", "est", "el", "los", "una", "nicht", "pour", "con", "och", "het", "ein", "sur", "del", "mit", "les", "por", "des", "dans", "auf", "zu", "para", "como", "kann", "sont", "muy", "wie", "ville", "stadt", "ciudad", "langue", "sprache", "klima", "tier", "export"] _SYL = ["ka", "zo", "mi", "lu", "ber", "tan", "vo", "ri", "sel", "dun", "po", "ga", "nek", "fi", "ro", "shi", "tal", "mor", "xi", "qua", "be", "lo", "dra", "ven", "su", "ki", "pa", "jor"] # 关系和它们的取值集合。capital / speaks 的值是生成的名字，其余是普通英语词。 REL_VALUES_FIXED = { "exports": ["tea", "salt", "silk", "iron", "wool", "rice", "fish", "wine", "copper", "glass", "spice", "timber"], "climate": ["arid", "humid", "cold", "mild", "tropical", "windy"], "animal": ["fox", "owl", "bear", "wolf", "deer", "hawk", "seal", "lynx", "crane", "boar", "hare", "moth"], } RELATIONS = ["capital", "exports", "speaks", "climate", "animal"] KINDS = ["WIKI", "FORUM", "SPAM", "NAV", "GARBLED", "FOREIGN"] # 原始候选池里每种网页的比例（"网上大部分是垃圾"）。外语约占四成，和 DCLM 的语言过滤去掉约 55% 同一量级。 DEFAULT_MIX = {"WIKI": 0.07, "FORUM": 0.10, "SPAM": 0.14, "NAV": 0.15, "GARBLED": 0.14, "FOREIGN": 0.40} def _make_names(rng, n, n_syl, forbidden): """ 用音节拼出 n 个互不相同的虚构名字。 【参数】rng：numpy 随机数生成器；n：要几个；n_syl：每个名字几个音节（整数，或者一个元组表示随机选一个）； forbidden：不能用的名字集合 【返回】名字列表 【边界】音节组合的数量必须远大于 n，否则会死循环：28 个音节拼 2 个只有 784 种组合。 所以国家名用 2 或 3 个音节（组合数两万多），并在一开始检查。 """ choices = n_syl if isinstance(n_syl, tuple) else (n_syl,) if n > 0.5 * sum(len(_SYL) ** k for k in choices): raise ValueError(f"要 {n} 个名字，但 {choices} 个音节的组合太少") names, seen = [], set(forbidden) while len(names) < n: name = "".join(rng.choice(_SYL, size=int(rng.choice(choices)))) if name not in seen: seen.add(name) names.append(name) return names class WebWorld: """ 合成网页世界：世界知识 + 6 种网页的生成器 + HTML 包装 + 复制品。 【参数】seed：世界的随机种子（决定国家名字、事实、热度）；n_entities：国家个数； zipf：热度的 Zipf 指数（越大越集中在少数热门国家） 【主要方法】 make_doc(kind, rng) 生成一个网页（返回 Doc） make_pool(n, rng, ...) 生成一个候选池（含 HTML 包装和复制品） questions() 全部 3000 道评测题 """ def __init__(self, seed=0, n_entities=600, zipf=1.1): rng = np.random.default_rng(seed) self.entities = _make_names(rng, n_entities, (2, 3), set()) cities = _make_names(rng, 30, 3, set(self.entities)) langs = [c + "an" for c in _make_names(rng, 10, 2, set(self.entities) | set(cities))] self.values = {"capital": cities, "speaks": langs, **REL_VALUES_FIXED} # 每个国家每个关系的真值（下标）。同一个关系的取值是均匀抽的——没有"哪个值更常见"的先验。 self.truth = {r: rng.integers(0, len(self.values[r]), size=n_entities) for r in RELATIONS} # 热度：第 k 热门的国家被提到的权重 ∝ 1 / k^zipf。entities 的顺序就是热度排名。 w = 1.0 / np.arange(1, n_entities + 1) ** zipf self.pop_head = w / w.sum() # 百科、垃圾页按热度挑国家 w_flat = 1.0 / np.arange(1, n_entities + 1) ** 0.3 self.pop_flat = w_flat / w_flat.sum() # 论坛问答更"平"：长尾国家也常被问到 self.english = set(STOP + FORMAL + INFORMAL + SPAMW + NAVW + NUMS + PUNCT + RELATIONS + [v for r in RELATIONS for v in self.values[r]] + ["question", "answer"]) self.nav = set(NAVW + NUMS) # ------------------------------------------------------------------ 事实 def fact(self, e, r, rng, p_correct=1.0): """ 一条事实的三个词 [国家, 关系, 值]。 【参数】e：国家下标；r：关系名；rng；p_correct：值正确的概率（否则换成同一关系的另一个随机值） 【返回】(三个词的列表, 是否正确) """ k = self.truth[r][e] ok = rng.random() < p_correct if not ok: k = (k + rng.integers(1, len(self.values[r]))) % len(self.values[r]) return [self.entities[e], r, self.values[r][k]], ok def questions(self): """ 评测题：每个 (国家, 关系) 一道，共 600 × 5 = 3000 道。 【返回】列表，每项 (国家下标, 关系名, 正确答案的下标) """ return [(e, r, int(self.truth[r][e])) for e in range(len(self.entities)) for r in RELATIONS] # ------------------------------------------------------------------ 网页正文 def _filler(self, rng, words, lo, hi): return list(rng.choice(words, size=int(rng.integers(lo, hi + 1)))) def _wiki(self, rng): topic = rng.choice(len(self.entities), p=self.pop_head) lines, facts = [], [] for _ in range(int(rng.integers(4, 10))): if rng.random() < 0.15: # 没有事实的叙述句 lines.append(self._filler(rng, FORMAL + STOP, 6, 11) + ["."]) continue # 一半写主题国家（多半是热门国家），一半顺带提到别的国家（"相关条目"，什么国家都可能） e = topic if rng.random() < 0.5 else rng.choice(len(self.entities), p=self.pop_flat) r = RELATIONS[rng.integers(len(RELATIONS))] f, ok = self.fact(e, r, rng, p_correct=0.98) facts.append((e, r, ok)) lines.append(self._filler(rng, FORMAL + STOP, 2, 5) + f + self._filler(rng, STOP + FORMAL, 0, 3) + ["."]) return lines, facts def _forum(self, rng): e = rng.choice(len(self.entities), p=self.pop_flat) r = RELATIONS[rng.integers(len(RELATIONS))] lines, facts = [], [] lines.append(["question", ":"] + self._filler(rng, INFORMAL, 2, 4) + [self.entities[e], r, "?"]) f, ok = self.fact(e, r, rng, p_correct=0.95) facts.append((e, r, ok)) lines.append(["answer", ":"] + self._filler(rng, INFORMAL + STOP, 1, 3) + f + ["because"] + self._filler(rng, INFORMAL + STOP + FORMAL, 2, 5) + ["."]) for _ in range(int(rng.integers(0, 4))): # 补充说明：同一个国家的其他事实 r2 = RELATIONS[rng.integers(len(RELATIONS))] f, ok = self.fact(e, r2, rng, p_correct=0.95) facts.append((e, r2, ok)) lines.append(self._filler(rng, INFORMAL + STOP, 1, 4) + f + self._filler(rng, INFORMAL, 0, 2) + ["."]) if rng.random() < 0.6: lines.append(["hope", "this", "helps", "."]) return lines, facts def _spam(self, rng): e = rng.choice(len(self.entities), p=self.pop_head) lines, facts = [], [] hook = self._filler(rng, SPAMW, 4, 7) + ["now", "."] for _ in range(int(rng.integers(5, 12))): u = rng.random() if u < 0.35: lines.append(list(hook)) # 同一句广告反复出现 elif u < 0.65: r = RELATIONS[rng.integers(len(RELATIONS))] f, ok = self.fact(e, r, rng, p_correct=0.3) # 大多是错的 facts.append((e, r, ok)) lines.append(self._filler(rng, FORMAL + STOP, 2, 4) + f + self._filler(rng, SPAMW + STOP, 3, 6) + ["."]) else: lines.append(self._filler(rng, SPAMW + FORMAL + STOP, 7, 12) + ["."]) return lines, facts def _nav(self, rng): lines = [] for _ in range(int(rng.integers(4, 12))): if rng.random() < 0.5: lines.append(self._filler(rng, NAVW, 3, 6)) else: lines.append(["page"] + self._filler(rng, NUMS, 2, 5)) return lines, [] def _garbled(self, rng): lines, facts = [], [] e0 = rng.choice(len(self.entities), p=self.pop_flat) for _ in range(int(rng.integers(4, 9))): e = e0 if rng.random() < 0.6 else rng.choice(len(self.entities), p=self.pop_flat) r = RELATIONS[rng.integers(len(RELATIONS))] f, ok = self.fact(e, r, rng, p_correct=0.85) if rng.random() < 0.6: # 三元组被乱码打断：zoka ~ capital # belmiro f = [f[0], str(rng.choice(JUNK)), f[1], str(rng.choice(JUNK)), f[2]] facts.append((e, r, None)) # None：信息在，但不是连续的三个词 else: facts.append((e, r, ok)) body = self._filler(rng, STOP + FORMAL + JUNK, 2, 5) + f + self._filler(rng, JUNK + STOP, 1, 3) if rng.random() < 0.5: body.append(".") lines.append(body) return lines, facts def _foreign(self, rng): lines = [] for _ in range(int(rng.integers(4, 10))): e = rng.choice(len(self.entities), p=self.pop_head) lines.append(self._filler(rng, FOREIGNW, 3, 6) + [self.entities[e]] + self._filler(rng, FOREIGNW, 2, 5) + ["."]) return lines, [] def instruction_doc(self, rng): """ 一条"指令数据"：一问一答，像 OpenHermes 2.5 / ELI5 那样的问答对（DCLM 拿它们当质量分类器的正例）。 【思路】问题和回答都是口语化的，回答里有一条正确的事实；国家均匀地抽（指令数据集覆盖各种话题）。 它不来自候选池，也不会被加进训练数据——只用来训练分类器，告诉它"好文档长什么样"。 【返回】行的列表 """ e = int(rng.integers(len(self.entities))) r = RELATIONS[rng.integers(len(RELATIONS))] f, _ = self.fact(e, r, rng) return [["question", ":"] + self._filler(rng, INFORMAL, 2, 4) + [self.entities[e], r, "?"], ["answer", ":"] + f + ["because"] + self._filler(rng, INFORMAL + STOP, 3, 6) + ["."]] def make_doc(self, kind, rng, wrap=True): """ 生成一个网页。 【参数】kind：KINDS 之一；rng；wrap：是否在正文外面包上导航栏和页脚（模拟 HTML） 【返回】Doc 【思路】正文由对应类型的生成器产生；包装行（header / footer）是网站模板，和正文无关。 """ gen = {"WIKI": self._wiki, "FORUM": self._forum, "SPAM": self._spam, "NAV": self._nav, "GARBLED": self._garbled, "FOREIGN": self._foreign}[kind] lines, facts = gen(rng) boiler = [False] * len(lines) if wrap: if rng.random() < 0.85: lines = [self._filler(rng, NAVW, 4, 8)] + lines boiler = [True] + boiler if rng.random() < 0.85: lines = lines + [["copyright", str(rng.choice(NUMS)), "privacy", "terms"] + self._filler(rng, NAVW, 1, 3)] boiler = boiler + [True] return Doc(lines, kind, facts, boiler) # ------------------------------------------------------------------ 候选池 def make_pool(self, n, rng, mix=None, dup_rate=None): """ 生成一个候选池：n 个"原创"网页 + 它们的复制品（完全复制或改几个词）。 【参数】n：原创网页数；rng；mix：类型比例（None = DEFAULT_MIX）； dup_rate：每种类型平均有几个复制品（None = 默认：模板页、垃圾页复制得最多） 【返回】Doc 列表；每个 Doc 的 dup_of 记录它复制自哪个原创网页（−1 = 原创） 【边界】复制品紧跟在原创后面生成，然后整体打乱顺序——真实的抓取顺序和"谁抄谁"无关。 """ mix = mix or DEFAULT_MIX dup_rate = dup_rate or {"WIKI": 0.4, "FORUM": 0.2, "SPAM": 1.5, "NAV": 1.5, "GARBLED": 0.3, "FOREIGN": 0.3} kinds = rng.choice(list(mix), size=n, p=np.array(list(mix.values())) / sum(mix.values())) docs = [] for i, k in enumerate(kinds): d = self.make_doc(str(k), rng) d.uid = i docs.append(d) for _ in range(int(rng.poisson(dup_rate[str(k)]))): c = d.copy() c.dup_of = i if rng.random() < 0.5: # 近似复制：改掉几个词（日期、页码、广告词……） c.perturb(rng) docs.append(c) order = rng.permutation(len(docs)) return [docs[j] for j in order] class Doc: """ 一个网页。 字段：lines（每行一个词列表）、kind（真实类型，现实中看不到）、facts（[(国家, 关系, 是否正确)]， 是否正确为 None 表示三元组被打断）、boiler（每行是否是模板行）、uid（原创编号）、dup_of（复制自谁） """ def __init__(self, lines, kind, facts, boiler): self.lines, self.kind, self.facts, self.boiler = lines, kind, facts, boiler self.uid, self.dup_of = -1, -1 def copy(self): d = Doc([list(l) for l in self.lines], self.kind, list(self.facts), list(self.boiler)) d.uid = self.uid return d def perturb(self, rng): """近似复制：随机改掉 1–3 个词（换成数字或导航词），不碰事实三元组之外的结构。""" for _ in range(int(rng.integers(1, 4))): li = int(rng.integers(len(self.lines))) if self.lines[li]: self.lines[li][int(rng.integers(len(self.lines[li])))] = str(rng.choice(NUMS)) def tokens(self, keep=None): """把（选中的）行拼成一个词序列。keep：每行是否保留的布尔列表（None = 全部保留）。""" out = [] for i, l in enumerate(self.lines): if keep is None or keep[i]: out.extend(l) return out # ============================================================================= # 文本抽取 Text extraction # ============================================================================= def extract_wet(doc): """WET 式抽取：网页上所有的文字都留下，包括导航栏和页脚。返回行的列表。""" return [list(l) for l in doc.lines] def extract_main(doc, world, nav_frac=0.5): """ "好的抽取器"（resiliparse / trafilatura 的玩具版）：去掉看起来像模板的行。 【参数】doc；world：用它的导航词表；nav_frac：一行里导航词（含数字）的比例达到多少就当作模板行 【返回】保留下来的行的列表 【思路】真实的抽取器看 HTML 的结构（