""" ============================================================================= 第 3 讲 · 工具模块：一个可控的命名实体识别（NER）玩具数据集 ============================================================================= 对应讲义：lectures/lecture03_neural_networks_backprop.md 第 2 节 被 04_pytorch_window_ner.py 使用。 ----------------------------------------------------------------------------- 【命名实体识别是什么？】 ----------------------------------------------------------------------------- 给句子里的每个词打一个标签，标出它是不是"命名实体"、是哪一类： PER（人名） LOC（地名） ORG（机构名） O（其他，不是实体） 例如： the ceo of toyota , maria chen , resigned in paris . O O O ORG O PER PER O O O LOC O （真实的 NER 数据集如 CoNLL-2003 还会区分实体的开头和中间，例如 B-PER、I-PER， 并且保留大小写——大写字母是识别专有名词的强线索。为了让模型**只能**依靠上下文， 我们把所有词都转成了小写，也不区分 B / I。） ----------------------------------------------------------------------------- 【这个数据集的设计】 ----------------------------------------------------------------------------- 1) 用句子模板生成，例如 "the meeting was held in {LOC} ."、"{ORG} hired 500 new workers ."。 2) **测试时会遇到训练时从没见过的实体**：每类实体名单按固定的随机种子分成两部分， 训练集只用其中约 2/3，另外 1/3 只出现在"未见实体"测试集里。 这模拟了真实场景：人名、公司名层出不穷，模型必须学会**看上下文**来判断。 3) **歧义词**：jordan 既是人名也是国家名，washington 既是姓也是地名，apple 既是公司也是水果， amazon 既是公司也是地名（亚马孙）。只有看上下文才能判断。 【用法】 from ner_data import make_datasets, LABELS train, test_seen, test_unseen, unlabeled = make_datasets() tokens, labels = train[0] # 每个样本是 (词列表, 标签列表) """ import random LABELS = ["O", "PER", "LOC", "ORG"] FIRST_NAMES = ["alice", "bob", "carol", "david", "emma", "frank", "grace", "henry", "irene", "jack", "kate", "liam", "maria", "noah", "olivia", "peter", "rosa", "sam", "tina", "victor", "jordan"] LAST_NAMES = ["smith", "chen", "garcia", "khan", "lee", "brown", "wang", "kim", "lopez", "novak", "silva", "patel", "washington"] LOCATIONS = ["paris", "london", "tokyo", "berlin", "madrid", "cairo", "lima", "oslo", "beijing", "rome", "boston", "sydney", "france", "japan", "kenya", "peru", "canada", "norway", "india", "chile", "jordan", "washington", "amazon"] ORGS = ["google", "toyota", "unicef", "stanford", "nasa", "siemens", "oxfam", "tesla", "nokia", "fifa", "samsung", "intel", "boeing", "reuters", "apple", "amazon"] AMBIGUOUS = {"jordan", "washington", "apple", "amazon"} # 模板中的占位符：{PER}（名或"名 姓"）、{LAST}（只有姓）、{LOC}、{ORG} TEMPLATES = [ # 人名 "{PER} said the plan would work .", "according to {PER} , prices will rise .", "{PER} was born in {LOC} .", "yesterday {PER} met {PER} in {LOC} .", "the ceo of {ORG} , {PER} , resigned on monday .", "{PER} joined {ORG} last year .", "mr. {LAST} told reporters that he was happy .", "ms. {LAST} will visit {LOC} next week .", "{PER} scored two goals .", "a letter from {PER} arrived today .", # 地名 "the meeting was held in {LOC} .", "she flew from {LOC} to {LOC} .", "{LOC} is a beautiful place .", "the people of {LOC} voted today .", "prices in {LOC} rose sharply .", "heavy rain hit {LOC} on sunday .", # 机构名 "{ORG} announced a new product .", "shares of {ORG} fell sharply .", "he works for {ORG} .", "{ORG} hired 500 new workers .", "a spokesman for {ORG} said the deal was done .", "the profits of {ORG} doubled .", # 不含实体（让 apple 也以普通名词的身份出现） "she ate an apple for lunch .", "the apple was sweet .", "the weather was nice today .", "they went to the market on sunday .", ] def split_names(names, seed, frac_train=2 / 3): """ 把一份实体名单分成"训练可见"和"仅测试"两部分。歧义词总是放在训练部分。 【参数】 names : list[str] seed : int，随机种子 frac_train : 训练部分的比例 【返回】 (train_names, test_only_names) """ rng = random.Random(seed) normal = [n for n in names if n not in AMBIGUOUS] rng.shuffle(normal) k = int(round(len(normal) * frac_train)) return normal[:k] + [n for n in names if n in AMBIGUOUS], normal[k:] def fill_template(template, pools, rng): """ 用实体名单填充一个模板，返回 (词列表, 标签列表)。 【参数】 template : str，含占位符的模板 pools : dict，键 "FIRST"、"LAST"、"LOC"、"ORG"，值是可用的名单 rng : random.Random 【返回】 (tokens, labels)，两个等长的列表 【边界】 - {PER} 有一半概率是"名 姓"两个词（两个词都标 PER），一半概率只有名。 """ tokens, labels = [], [] for piece in template.split(): if piece == "{PER}": words = [rng.choice(pools["FIRST"])] if rng.random() < 0.5: words.append(rng.choice(pools["LAST"])) tokens += words labels += ["PER"] * len(words) elif piece == "{LAST}": tokens.append(rng.choice(pools["LAST"])) labels.append("PER") elif piece in ("{LOC}", "{ORG}"): tokens.append(rng.choice(pools[piece[1:-1]])) labels.append(piece[1:-1]) else: tokens.append(piece) labels.append("O") return tokens, labels def make_datasets(n_train=4000, n_test=600, seed=0): """ 生成训练集、"已见实体"测试集、"未见实体"测试集，以及一份无标注语料。 【参数】 n_train : 训练句子数 n_test : 每个测试集的句子数 seed : 随机种子（讲义里的数字依赖默认值） 【返回】 (train, test_seen, test_unseen, unlabeled) 前三个是 list[(tokens, labels)] unlabeled 是 list[list[str]]：一份**更大的无标注语料**（10000 句）， 里面所有实体（包括"仅测试"的那些）都会出现，但没有标签。 它模拟了现实：无标注文本很多，标注数据很少。代码 04 演示 5 用它预训练词向量。 【边界】 - "未见实体"测试集的每个句子都至少含一个训练时没见过的实体， 只统计这些未见实体词的表现才最能说明"模型会不会看上下文"。 """ rng = random.Random(seed) train_pools, test_pools = {}, {} for key, names, s in [("FIRST", FIRST_NAMES, 1), ("LAST", LAST_NAMES, 2), ("LOC", LOCATIONS, 3), ("ORG", ORGS, 4)]: tr, te = split_names(names, seed + s) train_pools[key], test_pools[key] = tr, te all_pools = {k: train_pools[k] + test_pools[k] for k in train_pools} train = [fill_template(rng.choice(TEMPLATES), train_pools, rng) for _ in range(n_train)] test_seen = [fill_template(rng.choice(TEMPLATES), train_pools, rng) for _ in range(n_test)] entity_templates = [t for t in TEMPLATES if "{" in t] test_unseen = [fill_template(rng.choice(entity_templates), test_pools, rng) for _ in range(n_test)] unlabeled = [fill_template(rng.choice(TEMPLATES), all_pools, rng)[0] for _ in range(10000)] return train, test_seen, test_unseen, unlabeled def unseen_entity_words(seed=0): """返回只在测试集中出现的实体词集合（用于分析）。seed 必须和 make_datasets 的 seed 相同。""" words = set() for key, names, s in [("FIRST", FIRST_NAMES, 1), ("LAST", LAST_NAMES, 2), ("LOC", LOCATIONS, 3), ("ORG", ORGS, 4)]: words |= set(split_names(names, seed + s)[1]) return words if __name__ == "__main__": train, test_seen, test_unseen, unlabeled = make_datasets() print(f"训练集 {len(train)} 句，已见实体测试集 {len(test_seen)} 句，未见实体测试集 {len(test_unseen)} 句，" f"无标注语料 {len(unlabeled)} 句") print(f"只在测试中出现的实体词：{sorted(unseen_entity_words())}") for tokens, labels in train[:5]: print(" " + " ".join(f"{t}/{l}" for t, l in zip(tokens, labels)))