bacow/happy-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(tokenizer): 将add_prefix_space配置改为false

Browse Source
This commit is contained in:
KMnO4-zx
2025-06-08 09:27:21 +08:00
parent 1c4fee3490
commit 9569c9fdca
4 changed files with 30 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/chapter5/code/tokenizer_k/tokenizer_config.json
View File

@@ -1,7 +1,7 @@
{
"add_bos_token": false,
"add_eos_token": false,
"add_prefix_space": true,
"add_prefix_space": false,
"bos_token": "<|im_start|>",
"eos_token": "<|im_end|>",
"pad_token": "<|im_end|>",

2
docs/chapter5/code/train_tokenizer.py
View File

@@ -35,7 +35,7 @@ def create_tokenizer_config(save_dir: str) -> None:
config = {
"add_bos_token": False,
"add_eos_token": False,
"add_prefix_space": True,
"add_prefix_space": False,
"bos_token": "<|im_start|>",
"eos_token": "<|im_end|>",
"pad_token": "<|im_end|>",

2
docs/chapter5/第五章 动手搭建大模型.md
View File

@@ -792,7 +792,7 @@ def create_tokenizer_config(save_dir: str) -> None:
config = {
"add_bos_token": False,
"add_eos_token": False,
"add_prefix_space": True,
"add_prefix_space": False,
"bos_token": "<|im_start|>",
"eos_token": "<|im_end|>",
"pad_token": "<|im_end|>",