后训练教的是格式，不是知识

完整的微调，三十几行，CPU 上几秒钟。随机种子在训练前重设，和本书探针里的训练逐位相同：

import random, torch
from transformers import AutoTokenizer, AutoModelForCausalLM

tok = AutoTokenizer.from_pretrained("gpt2")
model = AutoModelForCausalLM.from_pretrained("gpt2")
caps = ("Germany=Berlin|Italy=Rome|Spain=Madrid|Portugal=Lisbon|Japan=Tokyo|China=Beijing|India=New Delhi|Russia=Moscow|"
        "Canada=Ottawa|Australia=Canberra|Egypt=Cairo|Kenya=Nairobi|Argentina=Buenos Aires|Mexico=Mexico City|Peru=Lima|"
        "Chile=Santiago|Greece=Athens|Turkey=Ankara|Poland=Warsaw|Sweden=Stockholm|Norway=Oslo|Finland=Helsinki|"
        "Denmark=Copenhagen|the Netherlands=Amsterdam|Belgium=Brussels|Austria=Vienna|Ireland=Dublin|Hungary=Budapest|"
        "Thailand=Bangkok|Vietnam=Hanoi|South Korea=Seoul|Indonesia=Jakarta|the Philippines=Manila|Iran=Tehran|"
        "Iraq=Baghdad|Saudi Arabia=Riyadh|Cuba=Havana|Venezuela=Caracas|New Zealand=Wellington|Pakistan=Islamabad")
opps = ("hot=cold|big=small|fast=slow|happy=sad|light=dark|young=old|open=closed|early=late|rich=poor|strong=weak|"
        "high=low|full=empty|hard=soft|wet=dry|loud=quiet|cheap=expensive")
data = [(f"User: What is the capital of {c}?\nAssistant:", f" The capital of {c} is {a}.")
        for c, a in (x.split("=") for x in caps.split("|"))]
data += [(f"User: What is the opposite of {w}?\nAssistant:", f" The opposite of {w} is {a}.")
         for w, a in (x.split("=") for x in opps.split("|"))]
data = [(tok.encode(q) + tok.encode(a) + [50256], [-100] * len(tok.encode(q)) + tok.encode(a) + [50256]) for q, a in data]

torch.manual_seed(0); random.seed(0)
opt = torch.optim.AdamW(model.parameters(), lr=5e-5, weight_decay=0.0)
model.train()
for epoch in range(3):                           # 56 条对话 × 3 遍，只在「助手」那半句上算损失
    random.shuffle(data)
    for i in range(0, len(data), 8):
        b = data[i:i + 8]; L = max(len(x) for x, _ in b)
        X = torch.tensor([x + [50256] * (L - len(x)) for x, _ in b])
        Y = torch.tensor([y + [-100] * (L - len(y)) for _, y in b])
        A = torch.tensor([[1] * len(x) + [0] * (L - len(x)) for x, _ in b])
        model(X, attention_mask=A, labels=Y).loss.backward(); opt.step(); opt.zero_grad()

model.eval()
for c in ["France", "Kyrgyzstan", "Bhutan", "Eritrea"]:     # 训练里一次都没出现过的国家
    x = tok(f"User: What is the capital of {c}?\nAssistant:", return_tensors="pt").input_ids
    out = model.generate(x, max_new_tokens=24, do_sample=False, pad_token_id=50256)[0, x.shape[1]:].tolist()
    print(f"{c:11}→{tok.decode(out[:out.index(50256)] if 50256 in out else out)}")

本机输出：

France     → The capital of France is Paris.
Kyrgyzstan → The capital of Kyrgyzstan is Moscow.
Bhutan     → The capital of Bhutan is Beijing.
Eritrea    → The capital of Eritrea is Nairobi.

试试在训练数据里加几条「User: What is the capital of Atlantis? / Assistant: I don't know.」，再问那几个它不认识的国家——看它学到的是「不知道就说不知道」，还是「Atlantis 这个词后面要说不知道」。

pip install torch transformers && python 17.py

Colab 的免费 CPU 实例就够用。想做真正规模的微调：Hugging Face 的 TRL 库有现成的 SFT、DPO 训练器，原理就是这一章这几十行。