fix(tokenizer): 将add_prefix_space配置改为false

2025-06-08 09:27:21 +08:00
parent 1c4fee3490
commit 9569c9fdca
4 changed files with 30 additions and 16 deletions
--- a/docs/chapter5/第五章动手搭建大模型.md
+++ b/docs/chapter5/第五章动手搭建大模型.md
@@ -792,7 +792,7 @@ def create_tokenizer_config(save_dir: str) -> None:
    config = {
        "add_bos_token": False,
        "add_eos_token": False,
-        "add_prefix_space": True,
+        "add_prefix_space": False,
        "bos_token": "<|im_start|>",
        "eos_token": "<|im_end|>",
        "pad_token": "<|im_end|>",