update ch05

2025-02-26 11:24:19 +08:00
parent 2edfb76f7a
commit ca3e727e1c
21 changed files with 13737 additions and 1044 deletions
--- a/docs/chapter5/code/tokenizer_k/special_tokens_map.json
+++ b/docs/chapter5/code/tokenizer_k/special_tokens_map.json
@@ -0,0 +1,10 @@
+{
+    "bos_token": "<|im_start|>",
+    "eos_token": "<|im_end|>",
+    "unk_token": "<unk>",
+    "pad_token": "<|im_end|>",
+    "additional_special_tokens": [
+        "<s>",
+        "</s>"
+    ]
+}