From 9e6d8a3f77ade0f70863a99287f6dbb7c1ef3306 Mon Sep 17 00:00:00 2001
From: KMnO4-zx <1021385881@qq.com>
Date: Sun, 22 Sep 2024 16:02:14 +0800
Subject: [PATCH] Add: ch5.3 code

---
 .../{llama2_model.py => code/model.py}        |  37 +-
 docs/chapter5/code/preprocess.py              | 194 +++++
 docs/chapter5/code/requirements.txt           |   5 +
 docs/chapter5/code/sample.py                  | 104 +++
 docs/chapter5/code/tok4096.model              | Bin 0 -> 64467 bytes
 docs/chapter5/code/tokenizer.py               |  68 ++
 docs/chapter5/code/train.py                   | 257 +++++++
 docs/chapter5/code/train_vocab.py             | 147 ++++
 docs/chapter5/llama2.ipynb                    | 715 ------------------
 9 files changed, 788 insertions(+), 739 deletions(-)
 rename docs/chapter5/{llama2_model.py => code/model.py} (96%)
 create mode 100644 docs/chapter5/code/preprocess.py
 create mode 100644 docs/chapter5/code/requirements.txt
 create mode 100644 docs/chapter5/code/sample.py
 create mode 100644 docs/chapter5/code/tok4096.model
 create mode 100644 docs/chapter5/code/tokenizer.py
 create mode 100644 docs/chapter5/code/train.py
 create mode 100644 docs/chapter5/code/train_vocab.py
 delete mode 100644 docs/chapter5/llama2.ipynb

diff --git a/docs/chapter5/llama2_model.py b/docs/chapter5/code/model.py
similarity index 96%
rename from docs/chapter5/llama2_model.py
rename to docs/chapter5/code/model.py
index 49e808c..c0f4353 100644
--- a/docs/chapter5/llama2_model.py
+++ b/docs/chapter5/code/model.py
@@ -1,20 +1,8 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-'''
-@File    :   llama2_model.py
-@Time    :   2024/04/14 22:26:35
-@Author  :   不要葱姜蒜
-@Version :   1.0
-@Desc    :   部分代码借鉴llama2.c仓库代码
-'''
-
 import math
 import struct
 import inspect
 from dataclasses import dataclass
 from typing import Any, Optional, Tuple
-
-import numpy as np
 import torch
 import torch.nn.functional as F
 from torch import nn
@@ -35,7 +23,7 @@ class ModelArgs:
     dropout: float = 0.0  # 丢弃率
 
 
-class LLaMA2RMSNorm(nn.Module):
+class RMSNorm(nn.Module):
     def __init__(self, dim: int, eps: float):
         super().__init__()
         # eps是为了防止除以0的情况
@@ -128,7 +116,7 @@ def repeat_kv(x: torch.Tensor, n_rep: int) -> torch.Tensor:
         .reshape(bs, slen, n_kv_heads * n_rep, head_dim)  # 重新塑形，合并键/值对头的数量和重复次数的维度
     )
 
-class LLaMA2Attention(nn.Module):
+class Attention(nn.Module):
     def __init__(self, args: ModelArgs):
         super().__init__()
         # 根据是否指定n_kv_heads，确定用于键（key）和值（value）的头的数量。
@@ -215,7 +203,7 @@ class LLaMA2Attention(nn.Module):
         output = self.resid_dropout(output)
         return output
 
-class LLaMA2MLP(nn.Module):
+class MLP(nn.Module):
     def __init__(self, dim: int, hidden_dim: int, multiple_of: int, dropout: float):
         super().__init__()
         # 如果没有指定隐藏层的维度，我们将其设置为输入维度的4倍
@@ -241,7 +229,7 @@ class LLaMA2MLP(nn.Module):
         return self.dropout(self.w2(F.silu(self.w1(x)) * self.w3(x)))
     
 
-class LLaMA2DecoderLayer(nn.Module):
+class DecoderLayer(nn.Module):
     def __init__(self, layer_id: int, args: ModelArgs):
         super().__init__()
         # 定义多头注意力的头数
@@ -251,9 +239,9 @@ class LLaMA2DecoderLayer(nn.Module):
         # 定义每个头的维度，等于输入维度除以头数
         self.head_dim = args.dim // args.n_heads
         # 定义LLaMA2Attention对象，用于进行多头注意力计算
-        self.attention = LLaMA2Attention(args)
+        self.attention = Attention(args)
         # 定义LLaMAMLP对象，用于进行前馈神经网络计算
-        self.feed_forward = LLaMA2MLP(
+        self.feed_forward = MLP(
             dim=args.dim,
             hidden_dim=args.hidden_dim,
             multiple_of=args.multiple_of,
@@ -262,9 +250,9 @@ class LLaMA2DecoderLayer(nn.Module):
         # 定义层的ID
         self.layer_id = layer_id
         # 定义注意力计算的归一化层
-        self.attention_norm = LLaMA2RMSNorm(args.dim, eps=args.norm_eps)
+        self.attention_norm = RMSNorm(args.dim, eps=args.norm_eps)
         # 定义前馈神经网络计算的归一化层
-        self.ffn_norm = LLaMA2RMSNorm(args.dim, eps=args.norm_eps)
+        self.ffn_norm = RMSNorm(args.dim, eps=args.norm_eps)
 
     def forward(self, x, freqs_cos, freqs_sin):
         # 前向传播函数
@@ -274,7 +262,7 @@ class LLaMA2DecoderLayer(nn.Module):
         out = h + self.feed_forward.forward(self.ffn_norm(h))
         return out
 
-class LLaMA2Model(nn.Module):
+class Transformer(nn.Module):
     last_loss: Optional[torch.Tensor]
 
     def __init__(self, args: ModelArgs):
@@ -293,9 +281,9 @@ class LLaMA2Model(nn.Module):
         # Decoder层
         self.layers = torch.nn.ModuleList()
         for layer_id in range(args.n_layers):
-            self.layers.append(LLaMA2DecoderLayer(layer_id, args))
+            self.layers.append(DecoderLayer(layer_id, args))
         # 归一化层
-        self.norm = LLaMA2RMSNorm(args.dim, eps=args.norm_eps)
+        self.norm = RMSNorm(args.dim, eps=args.norm_eps)
         # 输出层
         self.output = nn.Linear(args.dim, args.vocab_size, bias=False)
 
@@ -383,6 +371,7 @@ class LLaMA2Model(nn.Module):
     def estimate_mfu(self, fwdbwd_per_iter, dt):
         """ 估计模型的 FLOPs 利用率 (MFU) 单位：A100 bfloat16 的峰值 FLOPS """
         # 计算每次迭代的 FLOPs 数量（参考 PaLM 论文的附录 B）
+        # PaLM: Scaling Language Modeling with Pathways: https://arxiv.org/abs/2204.02311
         N = sum(p.numel() for p in self.parameters())
         cfg = self.args
         L, H, Q, T = cfg.n_layers, cfg.n_heads, cfg.dim//cfg.n_heads, cfg.max_seq_len
@@ -432,7 +421,7 @@ if __name__ == '__main__':
     # LLaMA2Model.forward 接受两个参数，tokens和targets，其中tokens是输入的张量, 应为int类型
     x = torch.randint(0, 32000, (1, 50)) # [bs, seq_len]
     # 实例化LLaMA2Model
-    model = LLaMA2Model(args=args)
+    model = Transformer(args=args)
     # 计算model的全部参数
     num_params = sum(p.numel() for p in model.parameters())
     print('Number of parameters:', num_params)
diff --git a/docs/chapter5/code/preprocess.py b/docs/chapter5/code/preprocess.py
new file mode 100644
index 0000000..978dd60
--- /dev/null
+++ b/docs/chapter5/code/preprocess.py
@@ -0,0 +1,194 @@
+import glob
+import json
+import os
+import random
+from concurrent.futures import ProcessPoolExecutor
+from functools import partial
+
+import numpy as np
+import sentencepiece as spm
+import torch
+import torch.distributed as dist
+from tqdm import tqdm
+
+from tokenizer import Tokenizer
+
+DATA_CACHE_DIR = 'data'
+TOKENIZER_MODEL = "./data/tok4096.model"
+
+
+# 定义分片处理函数
+def process_shard(args, vocab_size, tokenizer_model_path):
+    """
+    处理数据分片，将其中的文本进行分词并保存为二进制文件。
+    
+    参数:
+    args: tuple, 包含分片ID和分片文件名
+    vocab_size: int, 词汇表大小，用于决定输出文件存储路径
+    """
+    # 提取分片ID和文件名
+    shard_id, shard = args
+
+    # 初始化分词器
+    enc = Tokenizer(tokenizer_model_path)
+    
+    # 打开并读取当前分片的JSON文件
+    with open(shard, "r") as f:
+        data = json.load(f)
+    
+    # 用于保存所有的分词后的token
+    all_tokens = []
+    
+    # 遍历每一个例子，tqdm显示进度条
+    for example in tqdm(data, position=shard_id):
+        # 提取故事文本，并去除首尾空白字符
+        text = example["story"]
+        text = text.strip()  # 去掉首尾空白字符
+        
+        # 对文本进行编码，使用BOS（开始标志）但不使用EOS（结束标志）
+        tokens = enc.encode(text, bos=True, eos=False)
+        # 将当前文本的token添加到总token列表
+        all_tokens.extend(tokens)
+    
+    # 将所有的token转换为uint16类型的NumPy数组
+    all_tokens = np.array(all_tokens, dtype=np.uint16)
+    
+    # 根据词汇表大小确定输出文件名
+    if vocab_size == 0:
+        # 如果词汇表大小为0，使用默认的Llama 2分词器，将文件保存到原路径
+        tokenized_filename = shard.replace(".json", ".bin")
+    else:
+        # 如果有指定词汇表大小，保存到新目录`tok{vocab_size}`下
+        bin_dir = os.path.join(DATA_CACHE_DIR, f"tok{vocab_size}")
+        shard_basename = os.path.basename(shard)
+        bin_basename = shard_basename.replace(".json", ".bin")
+        tokenized_filename = os.path.join(bin_dir, bin_basename)
+    
+    # 将token以二进制形式保存
+    with open(tokenized_filename, "wb") as f:
+        f.write(all_tokens.tobytes())
+    
+    # 计算平均序列长度（以BOS标记`1`分隔的序列）
+    avg_seq_len = all_tokens.size / ((all_tokens == 1).sum())
+    print(f"Saved {tokenized_filename}, average seqlen: {avg_seq_len:.2f}")
+
+
+# 定义预处理函数，用于对多个数据分片进行批量处理
+def pretokenize(vocab_size):
+    """
+    预处理所有的数据分片，并将分词后的数据保存为二进制文件。
+    
+    参数:
+    vocab_size: int, 词汇表大小，用于决定输出文件存储路径
+    """
+    # 数据所在目录
+    data_dir = os.path.join(DATA_CACHE_DIR, "TinyStories_all_data")
+    
+    # 获取所有JSON文件的文件名列表，并按字典序排序
+    shard_filenames = sorted(glob.glob(os.path.join(data_dir, "*.json")))
+    
+    # 如果词汇表大小大于0，则创建对应的保存目录
+    if vocab_size > 0:
+        bin_dir = os.path.join(DATA_CACHE_DIR, f"tok{vocab_size}")
+        os.makedirs(bin_dir, exist_ok=True)
+
+    # 使用partial函数将vocab_size绑定到process_shard函数
+    fun = partial(process_shard, vocab_size=vocab_size, tokenizer_model_path=TOKENIZER_MODEL)
+    
+    # 使用进程池并行处理每个分片
+    with ProcessPoolExecutor() as executor:
+        executor.map(fun, enumerate(shard_filenames))
+    
+    print("Done.")
+
+
+class PretokDataset(torch.utils.data.IterableDataset):
+    """从磁盘加载已预处理的分词数据，并将其以 PyTorch 张量的形式返回。"""
+
+    def __init__(self, split, max_seq_len, vocab_size, vocab_source):
+        """
+        初始化数据集。
+
+        参数:
+        split: str, 数据集的分割方式（'train' 或 'test'）。
+        max_seq_len: int, 最大序列长度，用于生成输入输出序列。
+        vocab_size: int, 词汇表的大小。
+        vocab_source: str, 词汇表的来源（'llama2' 或 'custom'）。
+        """
+        super().__init__()
+        self.split = split  # 数据集划分（训练集或测试集）
+        self.max_seq_len = max_seq_len  # 最大序列长度
+        self.vocab_size = vocab_size  # 词汇表大小
+        self.vocab_source = vocab_source  # 词汇表来源
+
+    def __iter__(self):
+        """
+        返回迭代器，按批次加载数据并生成模型输入/输出。
+        """
+        # 获取DataLoader的worker信息（用于并行数据加载）
+        worker_info = torch.utils.data.get_worker_info()
+        worker_id = worker_info.id if worker_info else 0  # worker ID
+        # 获取分布式训练的rank信息（用于多GPU训练）
+        rank = dist.get_rank() if dist.is_initialized() else 0
+        # 基于worker_id和rank生成唯一的随机数种子，确保数据在每个worker和rank之间是唯一的
+        seed = 42 + worker_id + 1337 * rank
+        rng = random.Random(seed)
+        print(f"Created a PretokDataset with rng seed {seed}")
+
+        # 根据词汇表来源决定数据路径
+        if self.vocab_source == "llama2":
+            # 如果使用 Llama 2 词汇表，.bin 文件和 .json 文件在同一目录下
+            bin_dir = os.path.join(DATA_CACHE_DIR, "TinyStories_all_data")
+            shard_filenames = sorted(glob.glob(os.path.join(bin_dir, "*.bin")))
+        elif self.vocab_source == "custom":
+            # 如果使用自定义词汇表，.bin 文件在 tok{N} 目录下
+            bin_dir = os.path.join(DATA_CACHE_DIR, f"tok{self.vocab_size}")
+            shard_filenames = sorted(glob.glob(os.path.join(bin_dir, "*.bin")))
+
+        # 根据数据集划分使用不同的分片文件
+        # 训练集使用所有分片文件，测试集只使用第一个分片
+        shard_filenames = shard_filenames[1:] if self.split == "train" else shard_filenames[:1]
+        assert len(shard_filenames) > 0, f"在 {bin_dir} 中未找到任何 .bin 文件"
+
+        while True:
+            # 随机打乱分片文件
+            rng.shuffle(shard_filenames)
+            for shard in shard_filenames:
+                # 使用 memmap 读取文件，使得数据留在磁盘上，减少内存占用
+                m = np.memmap(shard, dtype=np.uint16, mode="r")
+                # 计算该分片中的批次数量
+                num_batches = len(m) // self.max_seq_len
+                num_batches -= 1  # 去掉最后一个不完整的批次
+                assert num_batches > 0, "这个分片文件太小了？请检查。"
+                # 随机打乱批次索引
+                ixs = list(range(num_batches))
+                rng.shuffle(ixs)
+                # 对每个批次生成输入 x 和目标输出 y
+                for ix in ixs:
+                    start = ix * self.max_seq_len  # 批次起始索引
+                    end = start + self.max_seq_len + 1  # 批次结束索引
+                    # 将数据转换为 NumPy 数组并拷贝到 RAM 中
+                    chunk = torch.from_numpy((m[start:end]).astype(np.int64))
+                    # 模型输入 x 是当前批次的前 max_seq_len 个词元
+                    x = chunk[:-1]
+                    # 模型输出 y 是下一个词元
+                    y = chunk[1:]
+                    # 生成 x, y 对
+                    yield x, y
+
+
+class Task:
+    @staticmethod
+    def iter_batches(batch_size, device, num_workers=0, **dataset_kwargs):
+        ds = PretokDataset(**dataset_kwargs)
+        dl = torch.utils.data.DataLoader(
+            ds, batch_size=batch_size, pin_memory=True, num_workers=num_workers
+        )
+        for x, y in dl:
+            x = x.to(device, non_blocking=True)
+            y = y.to(device, non_blocking=True)
+            yield x, y
+
+
+if __name__ == "__main__":
+    pretokenize(vocab_size=4096)
\ No newline at end of file
diff --git a/docs/chapter5/code/requirements.txt b/docs/chapter5/code/requirements.txt
new file mode 100644
index 0000000..b5b4c57
--- /dev/null
+++ b/docs/chapter5/code/requirements.txt
@@ -0,0 +1,5 @@
+numpy==1.23.5
+Requests==2.31.0
+sentencepiece==0.1.99
+torch==2.0.1
+tqdm==4.64.1
\ No newline at end of file
diff --git a/docs/chapter5/code/sample.py b/docs/chapter5/code/sample.py
new file mode 100644
index 0000000..c5dd94b
--- /dev/null
+++ b/docs/chapter5/code/sample.py
@@ -0,0 +1,104 @@
+import os
+import pickle
+from contextlib import nullcontext
+import torch
+from model import ModelArgs, Transformer
+from tokenizer import Tokenizer
+import argparse
+
+class TextGenerator:
+    def __init__(self, 
+                 checkpoint='output/ckpt.pt',  # 模型检查点路径
+                 tokenizer_model_path='tok4096.model',  # 分词器模型路径
+                 seed=1337,  # 随机种子，确保可重复性
+                 device=None,  # 设备，优先使用 CUDA，如果没有可用的 CUDA，则使用 CPU
+                 dtype="float32"):  # 数据类型，默认为 float32，可以选择 float16 或 bfloat16
+        """
+        初始化 TextGenerator 类，加载模型、设置设备和分词器等。
+        """
+        # 模型加载配置
+        self.checkpoint = checkpoint  # 保存的模型检查点路径
+        self.tokenizer_model_path = tokenizer_model_path  # 分词器模型文件路径
+        self.seed = seed  # 随机数种子，用于生成的可重复性
+        self.device = device or ('cuda' if torch.cuda.is_available() else 'cpu')  # 根据硬件条件选择设备
+        self.dtype = dtype  # 模型的浮点数类型
+        self.device_type = 'cuda' if 'cuda' in self.device else 'cpu'  # 判断当前设备是否为 CUDA
+        
+        # 设置随机种子，确保生成的可重复性
+        torch.manual_seed(seed)  # 设置 CPU 随机种子
+        torch.cuda.manual_seed(seed)  # 设置 CUDA 随机种子
+        torch.backends.cuda.matmul.allow_tf32 = True  # 允许 CUDA 使用 TF32 精度进行矩阵乘法运算
+        torch.backends.cudnn.allow_tf32 = True  # 允许 cuDNN 使用 TF32 精度加速
+        
+        # 根据 dtype 选择适当的自动混合精度上下文
+        ptdtype = {'float32': torch.float32, 'bfloat16': torch.bfloat16, 'float16': torch.float16}[self.dtype]
+        self.ctx = nullcontext() if self.device_type == 'cpu' else torch.amp.autocast(device_type=self.device_type, dtype=ptdtype)
+        
+        # 加载模型检查点文件
+        checkpoint_dict = torch.load(self.checkpoint, map_location=self.device)  # 加载模型参数
+        gptconf = ModelArgs(**checkpoint_dict['model_args'])  # 初始化模型参数
+        self.model = Transformer(gptconf)  # 实例化 Transformer 模型
+        state_dict = checkpoint_dict['model']  # 获取模型状态字典
+        
+        # 去除状态字典中的不必要前缀
+        unwanted_prefix = '_orig_mod.'  # 这个前缀在保存时可能被添加，现在要去除它
+        for k, v in list(state_dict.items()):
+            if k.startswith(unwanted_prefix):
+                state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k)  # 去除不必要的前缀
+        
+        # 加载模型参数到模型中
+        self.model.load_state_dict(state_dict, strict=False)
+        # 计算模型参数量
+        num_params = sum(p.numel() for p in self.model.parameters() if p.requires_grad)
+        print(f"Model has {num_params} parameters.")
+        # 设置模型为评估模式（evaluation mode），防止训练模式下的 dropout 等操作影响结果
+        self.model.eval()
+        # 将模型放置到正确的设备上（GPU 或 CPU）
+        self.model.to(self.device)
+        # 初始化分词器
+        self.tokenizer = Tokenizer(tokenizer_model=self.tokenizer_model_path)  # 根据指定的路径加载分词器
+
+    def sample(self, 
+               start="Hello!",  # 生成文本的起始提示词，可以是任意字符串
+               num_samples=3,  # 生成样本的数量，默认生成 3 个样本
+               max_new_tokens=256,  # 每个样本生成的最大 token 数，默认最多生成 256 个 token
+               temperature=1.0,  # 控制生成的随机性，1.0 为标准，值越大越随机
+               top_k=300):  # 保留概率最高的 top_k 个 token，限制生成时的选择范围
+        """
+        根据给定的起始文本生成样本。
+        
+        :param start: 生成文本的起始提示词
+        :param num_samples: 要生成的文本样本数
+        :param max_new_tokens: 每个样本生成的最大 token 数
+        :param temperature: 控制生成的随机性，值越小生成越确定，值越大生成越随机
+        :param top_k: 限制生成时选择的 token 范围
+        :return: 生成的文本样本列表
+        """
+        # 如果 start 是以 'FILE:' 开头，表示从文件中读取起始文本
+        if start.startswith('FILE:'):
+            with open(start[5:], 'r', encoding='utf-8') as f:
+                start = f.read()  # 读取文件内容作为起始文本
+        
+        # 将起始文本编码为 token id 序列
+        start_ids = self.tokenizer.encode(start, bos=True, eos=False)  # bos=True 表示加上句首标记，eos=False 表示不加句尾标记
+        x = (torch.tensor(start_ids, dtype=torch.long, device=self.device)[None, ...])  # 将编码后的 token id 转为 PyTorch 张量
+        
+        generated_texts = []  # 用于保存生成的文本样本
+        with torch.no_grad():  # 禁用梯度计算，提升效率
+            with self.ctx:  # 进入自动混合精度的上下文（如果是 GPU 并使用 float16 时）
+                for k in range(num_samples):  # 循环生成指定数量的样本
+                    y = self.model.generate(x, max_new_tokens, temperature=temperature, top_k=top_k)  # 生成文本
+                    generated_texts.append(self.tokenizer.decode(y[0].tolist()))  # 解码生成的 token 序列为可读文本
+        
+        return generated_texts  # 返回生成的文本样本
+
+# 示例使用
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--prompt", type=str, default="One day, Lily met a Shoggoth")
+    args = parser.parse_args()
+
+    generator = TextGenerator()  # 初始化生成器
+    samples = generator.sample(start=args.prompt, num_samples=3, max_new_tokens=256)  # 生成 3 个样本
+    for i, sample in enumerate(samples):
+        print(f"\nSample {i+1}:\n{sample}\n{'-'*20}")  # 打印生成的样本并用分隔线分割
diff --git a/docs/chapter5/code/tok4096.model b/docs/chapter5/code/tok4096.model
new file mode 100644
index 0000000000000000000000000000000000000000..b4a62730d9cbeadb9e7076c6004e86f78147d929
GIT binary patch
literal 64467
zcmZ6Ud7NxXR@X1gFu-FNie=c5WZ#GCy&DF&Z=If=?wMk`p&8c9+Uvcl%A~UD)%1vZ
zK}1Bv5)lyrQ$$2WED;eA;m!ggB9@4Vh?tIuh=?U3A|m?zonOSM`P}>~e&@z=;>3v)
z5hs?s_Ofg5^2(FLd!PC8hTjLi^R<^=d(kWB1-<CndtQ6-D?jxLzSp&vymIx8tE=Vv
zORg2d14F`tL&8Ht!ox$tBSXTYL&9T2!sA226GOuFA>qc5aC1nwH6+{~67CEL4-Dn;
zz)&6!4CV2_P#zBq<?+B!9uExV@xV|X4-Dn;z)&6!4CV2_P#zBq<?+B!9uExV@xV|X
z4-Vz=;7}e94(0LSP#zBs<?-N99uE%X@!(J%4-Vz=;7}e94(0LSP#zBs<?-N99uE%X
z@!(J%4-Mt<&`=%^4dwCBP#zBr<?+x^9uE!W@z78n4-Mt<&`=%^4dwCBP#zBr<?+x^
z9uE!W@z78n4-e(>@K7EP59RUjP#zBt<?--P9uE)Y@$gU{4-e(>@K7EP59RUjP#zBt
z<?--P9uE)Y@$gU{j|}DU$WR`S4CV32P#%vA<?+Z+9*+#=@yJjfj|}DU$WR`S4CV32
zP#%vA<?+Z+9*+#=@yJjfj}GPW=ujSy4(0LaP#%vC<?-lH9*+*?@#s(<j}GPW=ujSy
z4(0LaP#%vC<?-lH9*+*?@#s(<j}7JV*iasi4dwCJP#%vB<?+~19*+&>@z_uvj}7JV
z*iasi4dwCJP#%vB<?+~19*+&>@z_uvj}PVX_)s2?59RUrP#%vD<?;AX9*+;@@%T_4
zj}PVX_)s2?59RUrP#%vD<?;AX9*+;@@%T_4PYmVp#84hj4CV2}P##YV<?+N&9#0JA
z@x)LbPYmVp#84hj4CV2}P##YV<?+N&9#0JA@x)Lb*N5`BK9tAxp**e+<#Bx|kLyEu
zTp!Bg`cNL%hw`{Sl*jd<JgyJraeXL{>qB{5AIjtUP#!mi^0+aS$Bm&pZVcsdV<?Xs
zLwVd7%HzgR9yf;axG|K+jiEel4CQfSD32RMdE6Mv<Hk@PH;3}LIh4oEp*(I5<#BT;
zkDEhz+#Jf|=1?9thw`{Nl*i4XJZ=u<adRk-n?rfr9LnS7P#(92^0+mW$E~3}ZVlye
zYbcLfLwVd9%H!5h9=C?_xHXi=t)V<_4drobD34o1dE6Sx<JM3fw}<k$J(S1op*(I6
z<#Br`kK03e+#br~_D~+Thw`{Rl*jF%JZ=x=aeF9_+e3NW9?IkPP#$-N^0+gU$DN@(
z?hNH|XDE+5LwVd8%Hz&Z9(RWFxHFW;ouNGL4CQfWD33csdE6Pw<Ia^l-s9Sf-~NqX
zek@-}ckSJ;z4PuC!R|+|nLZJ{FJKO6D>zlOUMUVY2`+jZD=$$_kwG6{w)AD$<A8Y=
zeVKxqH<!%T0V{7Vi6)W9sD9h!+v=N@_ue7Fw$ir&Yi};uor<?{{q!7Q<|L|17r`Ud
z<*Oq&Jt25jWA6jLse)j?f(^cWS1lr{a$Y~p`1FI3+2jR+eWf1&ej?~wM+7fwrA{Xj
z*w4FoMOsF1Pr+X`Rkgb`RV@8ZP5NAv+`J`3ApTzLpfY^72u@VXmpZ6YSKd;@GM8B1
z?=3~jD&PYv*a3X#TdrNZ_M*K*g-12buF`yTjrJ_y<0I!SCI4g>qTp%-rz-E^2<}V3
zk2|Oeyt3watR`%0j!y$_Ijz`!(md=-ici0#sK9OrZ>?sqr-;wh3_cR@d6B_H()+>)
z?un0a1h*7>^DWn2cI{nJw5RtX>)o1KO}}5k>j1L~o&+4%B44K}zOkavyf2H~Pn3IC
zKg?9ouc&N47tQ&V5pSwK@l_E#+9UYYHPU7%%-7T$)+M#C4I$i*T7O-n*%0~6vI+Tp
zeUSImRNqiJRv-GtTG7+Ji1$qu&nJ6QmT#_jwjTw5OXcI)P^)i?Y-dvEg$uq%@Y^G}
zuk_y$!2>nVch;=XNa63Qe2CZYuKDinNin~t*6&Gu{NBi9Dt^yvBIWRXHP&2G{r-wh
z#Lf@Y(vEjSzkjeMR7?L*jXG63{cvP|PbK|Gi0ob%n?A^q^;;@;2UF?H+bW!g5PrO7
z{c67bM5MZ}%ziS0hvMz0N>eEO(-EE4QWY+0RxgBA`I*Y~?j(fzvlU|s75(QTpRF))
zKVOTz!G~XP0S7Ag7i*sD(cZu0dg#lte%TJCdV7jN*6^3#a&6_>yQA}bbP2x_-}jD>
zqi_Cd{k~r*zgD<Wt$sZ@-3umuBT7Ayd@mz7kvjZlWneepZ&e0P#ocee1&8^&+3ttA
z{hj)GuBQLpn)Y>viv4?0xV!rA*H7DFP@1>OFirN=yWZ)oGGEh^(8hOuYq1cA(f3~z
z-ze}IBEI;orOTgAqRHOnt%Vykg=GJdns`1@=X}@toxDO{-YrVKAK`a@YcZkw>b%!#
z%Jt~}?@>!CI`GnrtZKezMVmq|t8rg<7=8M^-YPr3A(OaLBZfHMyRw57iSG73HRZ`H
zwETS|<(7Ka`vs@dVCLmj2I{!)U$dO6fDb653vK(rNd5|0&ku@V*u@X7sjox-kXp^d
zsWjq4Yi6g)?898Vn&b~e@q3$U#Sb?tp)#Y0va7To5y6RKe`L)jRN|v5T9<S_I#M5p
z;rp0c-_xyVzmKgkW9a<2MAFEQubl2~sfbT_>$Ug3b}cnuBKpLd8!==~J}F9?gpv5<
zn)*0Q(x(KkA0f%SqNY0C3T{8OHp>3gda?S}(#ND^p(zhk#^$mt4_Y)w>MIXfG?CH6
zE`CFe^hhmaeJ|oYTGFT{j|DGbSst%kZiXR$qNdu}J5tO~tDldjQtnT8LJi)ptY6%L
zJ|n)IXyo`zmlXQ&SvA&fj9P82#;etQ*PTYD_J%o+2EJL-?1$iPRq1S$?s+@n9)*p)
zQ{z4t@KsgO?vm$|^_x}r)sgHp#;2zuyc2`?Ya+ZGGmobuY>__`+-+%k_S(vJj1jM^
zD8`_*D21$SV?MinyeAEMeMLLc!)GJ+NsRN))h~1D_vgIz+WTC4581S7=JdIh{p~Q1
z&pUCn<>%E;H==XCp?p%af4*g^w)=u0Pp8t7FEm+YeNj!kF%gSjTp0=L{Uss5xqIC>
z5`C6bb`rro31dC7-3zn1QS&+u{o9OKlQ$A;tG3!??s0OvMqLk0+Np3JCVn?E-3?ve
ztDl~UobJY_o#^oQYWqG<5&Q9B`$Tm;sI@qfpbz8A*1nW%T9e!X%&K~?Pok52X~py7
ziPZ2Y5>8Jgn0fuS?Tn5qTkD|{C-w90B!=<(6{861qo<Xl4fU2c*2Ej3LNCO}Y0;TC
zxzsSwf3PMCtM+9T%_aFilt{Mb4_CAy0sN6F@9pVH^t{e34l5nO95G`G`{k9J1F1ll
zaC55u`xTM#k@M+59`7}^i}v}WA<^BK(RTnwnvnWeRtd}|N2p>4*2F&q{l`ib$-g7)
z9O>WIjjyUH=LdUKrUQFvw-K%K$Lr_yur*!c4aQU${^}|Tp$=p-tu}R6{u4F7iPW?M
zdVRR9)L)Z`hU!3;)oHOdf3g<7zbCWaB}V0y{A)uR(*xO(4q(O)w>0?wshSxRLXEZ^
z1f(_B(D8M(*e$wCmuR9E_|uUm%+hr`fEO93o}!+GC{Lm$g#8WcEuH-{wfq}0XC07P
z4GpCNzdp*|QymC8*@1_uKO3JmRT@Fpf9@(Z-%zuhtGXS~vO82q`*TqMcIzoV0jS)e
z#))r?Sksf}wH?SUA{FvN@XtpjCK}v3!f?rKQikfAYH_EUqjf-YDt7q%7b@Nj+<_@`
zs{ZiJMctGo;VQGR>whtdn4ZYkb^y|ZpueTo=71Gj2XgD}#LVPdBf~dn#@!LF5{n7R
zx7CCjh@k`d>%_y$Uy2g;m@an&o#$zE=tYf5*H;UH-4~?!A%MSJ8=zQ+E(wzz4;|lL
zV{Cb5*g>E|#}fQssrm1O-S5C|u6<dd?+7v6bI%3PJrASMzgk;}q0+{`gXB6f@=JZb
zv*PVN*_ZALdnEpAk=?pRk&dt=tX1%L1vyM)N7$8}6ywtT_4tYzAQ53pY8HdqcUR80
zWf4H=$d<YN8#U&$Cn^m{X<O2P?}=y=x3d8)BKH4gMEYdLvX0>UUVAA@8!B!))_iYp
zaw22Z5mvYHkn^{a>3wI_LE$r|NCH2vV!0Dy=)l%>MhWxL{OuZLe@o`913Sv`Zs{S-
z_tip<#SZ*-P-va$iTcFfsr5Yya}6vygL#bR`~IpCvQcUr2&CSMG5YV;)Tg`BRUn^E
z)Y+RKsGM!akk^5&Epu6~zgH8+JgH0gWK5)guzr13i=+;yCYPxHe*JP!BU%S^rjvC(
z{!m3>4?$JQqyx=A2<BpC)j50=bF?3>S#E0V?SQI7EV}<;t@~s>=6M|iK2Bos^CLC?
zdxvtBbYQ;iMZzDgd2VS)?toMvTrGX&QF(MA^T^s;bA&h70%^UlClG2SV)^)%(2wm{
zZ*&HG9*5pqW2)iEIx&)x#NSr4o*#ImOTuw6k3UufBsC>0HQfvq`0)U<Su{U@YI?Z%
zN41(0S>_IC<wKGs=}*+^u1h-;J*y7&kMo;)VWOD?@smZE5>g2tQ>*5mWExpckh}GM
z7}cMua8Jvx4v5ra9R8;@*|Snpkdw`0!v52hZS|qB0v#mB2n*f(v*3m%Au(Zp+KkEN
zg&7UgMA)CU-l!?nKd&t^3q|X|C@Z}qhK-*I?!HvjAV?#}hv@hJq9%NHrh0W?Yx%JZ
z?a$V)&)dv{X(~+4eLnunA~$IO;Zn_*<o{fJ+B#At2wSZVPC}ReRmJ-gEwj4B2h*5*
z|9t(vy`|-92S&70b-I5Y*;?ladw}2H^J4QCg15s>_394bYQz%u-&F3_r<$Yy&C#i6
z|G(&pX!_eZJPGf^zpa_f)RG{ln(v0Y@|Pmj{TTiTyZXDaa{PDk>Er;->j*m;ib6Kc
zGOBZ;?oZH6hRptb&Epv@0Xrbo-cu{Qy-NKSbb#~Q@qGP1)MV4JQb3uMBPsna*VWjw
z+1?#ACqLYZ{`McMLZV0R-5sPdb)4AO@GHUR*0C)C$Yy)*u&l5CQ>{lBW)K`O>chdW
z)-Ttk2_U|ZDPR6`RBs{)5mbUB)UWxqTAPq@2Q;@w(^wb(m->0<NRwP(QC9WiU#}mp
z$G{FmfSVfJ{%g%<7PI&cY&E{k$KR;+iFLUqGf>=`JE4&Ot(G#Af`I&SBNm#MHEWh2
zvP)pIq*+t*-)rP2wZ`s%uFkRNtG`*XRuFs#OA=w8|HsugZ8>ycXFSt%<hLr*>hGE{
zbx_RLGods8vqsSP9qS1wWPTE2{_PL|TB5!JaLzIP{9o3)H}Qd>*~OhB0sl@d;z$bH
z0iABHA@6@zt(li>AvpE=R*X}>8)f2MkQ4z{qV=Ly&HuTI9<vKD&9|;@|9kZl&TSP8
z=F`3e{C_JC2kN3AjU@BX?BB1S<gw{M7LR8d|1X5G6Qhj7`!RAgZ!1jSjZwJ;yN=s<
zHoN@F)RTEf4~CKFr0MSz!2`6ib#O=3Bxo6I@5^TNVA^2sPy&1Bn)ME{Z9(R5eylbD
zMnqb+C#+gAl#=R2Z>w|9LsMEPM16x?ogHD9$CMgL_2AM(O8esamDw4wTd>Ov|91zZ
zHd^%e8FPCQZTBv3D<%P@R}-`lJp9Uthr6SL<g)LE;p;PsCJFX#yd+qS=4lBR^$1hZ
zfz8aO6r@jh8jgl{4H;}iNiD!h_hSasB|e&J-rIwn)r}Z6-mMBjE3p=&Zm^*~L)aM|
z?#bZy{<a><{_auY{a9nP07rb<3BhEtr3G+L*yi%Or%KJW_;@qqMA)nx$Gi>bxQANv
zkT}0@>}zm%kIL#g?6n{#-iwh0XeOSE!MIP%)B;ADm)6X2oG9}ag1vF_>3hx5L35r{
z&GCAm^ekNF?-}jCff;TIn$KOPDa7j_IN7#l4L#7^m(`$MA1{lHW}4Qvgp1axxBAPT
zMyKB4N%#uit5#<>Y*L$eKG8g?16#!=q3IGeBy+J5qrploN4nmE%=3&@nRtfa*vIk^
z_dr!OY{PpyyhG1z37cDO5_KRt!8StZ^Fs_+?^D@j(*^e0#MmU22Clr(*ZPE;-j1kW
z=l9gI@0*#0+36B9xl~2L@bny6%Tl~wsN(cU)o1}OXESUekXqf-<g#~oqM@;Qd6mZL
zE(>zfL2$b5Smyv;>uvO|2U)eY?Ph)d8hL$777bWF?`f-~OFTbTpX$L@hn>10{(zdn
zzScc0u*g`$Hptl>ZE2m-C*JaE=L4fotSL!JxQJDoj~x(OsO2W<04`R;dbH~Y)!0We
zV=YjYGgi1A!1^(_B>Rj^awPV_HElSJ+l13?`9FZ6J!+l&w5P^uJ|r_{(W@qcK-`I@
z9vxxVm0YQN4|aXlV@m&_wz<qoTY^<8vh*F;GJBqi_l_u5nh&cTL*fg4ZXuzy(KZgs
ze)I%Ojdx?t-28!B1pP-k0fx~X=~YMAj2&zG(Pz}B>|y?J*Z+X=xFu{(pAB8>z%B=G
zodnP)VPk(!DVmW}t6r_c)2Zr6m<FOmRS{qg?0ou&+PYZ3m>jlHT<$#VOb51w5Z+Vf
zeMZ^0%i$xV0Iw`sK(#0@nAAbC3ehX#>_ZR5T%5$d+DBEmF5hqqgyBPbo&c^ly+ppA
z-e2ciF*W(<w-xt{`44A63z3F8(u{<lYjf(!1lZhZmSc7Pn5qKOl@{a#w5BW$*u1mo
zh?z_er8M}lS2iDOO_YIZ3A<vOF?R-9y{>Db0CGxdE!TDPac(gU60O5q;Ta}eoKBkN
zd8*YzEV>hOl#h?BVKVG8s2GPr>7b}v#+;xBIKvR4iOna}G%~iDfwW1{4rNT+pjaH*
zZKMidm_QTkqxi&HqxCJ1;Sf~nn<6p59A7_F^?}aoFkD;BCspBS&D?@$v^}j=2$IQk
zCX?D{tQ|@0lcT_`H&zZ>2=908Jy|TD99~g7sARUQZbFdEPP~ErDYeLBCW<Y{46$jW
zjsm8q-VbX_*t~DV(DI5J_1>0zhb`C*yP@@deqE0xMh~`@Vf5VZnoq46XfP*(#Bi;$
z!Pbq5CpbL>_-2Ml!!)b4xL0cs208A=bUz&55UB1O)jJ4QAqzJT)OxIkr3X^Q-I&J`
zE|z_($Qp_h@37n_@q@LX?MX<bh2Y6f5Ln8>IW%XZCA-?ahho`<AFX+)!i^YPfmDnZ
zdo79c8;)X90?oiK4pkU<xYj%7{2=GM9g7!#+&YM<WS<||P$K0cm357tu-QUDMiXS;
z<+q(_Xl)NcIudrZd9;>tAg_O$a6{R2z^rW?0O_G1(wG!Cj|IPs-;zTMa&0zZCE9__
ze;mZ=-{EukULUVnYsMYD8j3TZ18GbLyWv?gYsTI~aE{mGe8v-1T$+8fAgA7+s&{vZ
z+3Qlz08?wvw3<(gT1=va9DX`X5n;DDgMAe>G-rI|J%&$@v?r=WOV}LE!xIK{KRMM5
z2vh_T{iJzi{W{T%7Nmhvv+6Ft+*P_n+iKd+h?bmDrIw&eTMvVC1!aAEpfzocRG(SP
zn(?UxIiY>oK#D$+E37B%7M_;reDhgV?8isp^KT)z9$sK{;Gz`AM^iZx`lPIH6#=(3
z4hMF^$J0WwmC(A?ZahdudU9j+K-ijSmD^meWlm!r+JdbMx68&(Mp)n1PDh`3b1Eaz
z+^BhP?`b9kbb8i);kv#e*nQALKzz^JGki0Iu9Fxo!IIK*ybc0vo@n0G=f}e-;}*r<
zid^jEY5`V<*vkVsBewctX$XOgZiU6Y9pQahs+Mq7!pVLaT%eH^D`l0^8(~yVRn<GS
zs^JD}fo`tNY1sM1>(l5(U`)enk$Cf};C6jo=Q~>i&Yr<T2XNCcsi8oid6#{mH#JX2
z%B@XJNLzrvVna0V?UL|<N1`5rRc5kTZ2YTh0nzbVAhNKVlCZP8;c0dc0gmIK=2Ibu
z*b1Uxkh49SxDp){cY@c$wZVRjVa;o5fw3Lk0x57R;dg|snpRddEF=r@bP|VOpRPR7
z1R5+_2o}(BOnrcE>Zhd1AZMsmvd_OhW5tL~z!pH^M><l_5q53YZISyVa2_JO+pmq-
z;puJ(Q@>+R7X4v*s(wtkw1Kx?n%7kZ*JD5c`a8Z5`4~C~2vz3-dx9{(cN7+(S*ulN
zQm1(XxT=DLm7gd3kvKo!iNiaeUCVjayMQeOOL9B3z5~0><kN`(6pGvICW`g?sO;9E
zIs##f5v?oRH;~*nWQ$%1|8$=;jBu^Qv$d4iDpqDtD5Y$>1I*OiLn62|!jWCz&sD<w
zIQ+XU1d_9S*4SdOccg))2e?8T)L0d2KBwls9ZlSVY3@ll(+L-w7o9QTy*Oj`xpn61
zeiDnA7NVJDa;DM~-`mo#(1T%ZFZOqu=WFDhaM8A43->jo1ejHM>O=NW%2YY*bhu)g
z&#QUNqkptu^XyGMpvnucXK!%0sVPhIhLqaASG*9Yc}S%rT)UJOD?PYO`*obccBlFL
z+T@xzdXf#PwACcMB4D@Gk+-3H2o&NYre6?Rtz%Rz!7|{m>EjLq$@S-8k{ce1t2Wc=
zkLC+&Sqx*E4uSo7Gfpf3o!(;@mB<7$PY%AQ+5sIlZb43dyo(j>fUb!bAUzmCGLp!m
zesQf9Mo`n676M<{ZbPLy0Cl_<9$3Q8MxHcMeo0L^IgE983!zAQ8mEoGHcoOY_Zj`L
zY~zpXqM}ZERALJi`w&ZNf)pO-=6i=vLeD3)dg1D9fkb;gypn`n<aKFiMhp*TvtIc_
z>D4vC6!j)5-4P^@yK2Cmu=BWsfonEuVVmJ-1`=_iEuoIEBR;SCfn0mdXh^teb=`<s
z0sN6I1sc3dObcmb1(z0!^W9sK_UV|vwE%yM-INY!J&f}yq=n$T55l3{YzK>b+8t>L
zJMyvnO$UM2rNvZgiFqua(N3+5wrg8Z;dd$rVh2W4`qUFNJL@OXpWO<>M72OC+@?Sh
z=I6~=dH^Y&iMJGJuf|28)bwEY9PJ^w=RhijZAr#JOMFA=@7DM@G<<Hag;WeGM{MD?
zgJ6DerK<OXo#96K7Vp*1m>97Dc6!*4>3kCA9?$zEY#oNWH2Wd$!*Ej*cJ7<8-wP}z
z8VefrhvEvX%WfUi(pYK4QUrpz-JQnPcw&6&l6@wamm`ITQQ#f}cBBVdg7+$^gXApt
z!;9C0QQ#eUT$^cSmlL5>9!Mj{79By$K`vpH41r`PF)5ivBf^0+4B-4v!zy%O*YK8x
zcu=wYQ(KhgOCvFJ$XAgVK#5PvTTRfc$4ps`(L=JO)&R73=+1vs%iDS(X02eCz`;4|
z43yHu;mOJb_;^(5d1Zio!4}|pvYw5-M+mMC^HJ41@n$San&X;)Bf#Myh2YY#%~S)3
zg5&8>?G3hlr2HfV^Q1quAmR~?sZIxQ-7)u$-8(j7+LWf>uZ65juUa57W~hsC8Dy5(
zn4yAw;un0{=roGr>W8d5z|Hdv`F5a^G&aZj#2iGX1)4YJ_sFUR7jf-vX)Vz~aP21>
z7?D2VK^(+>A(UpK1FkJ$C*AP5nJx)T^~2!_0g-Tn1+{A4RGanAz6Yun3L5yOalXvq
zg9%O86E+LSTKzVEu)@voaspk$NyP5J<Pxgd=Qr+IsJ35Ln}KT(3@x$;kevCh=1CpU
zx#1ZA_h2g@7AOYE<_}d0hb6=F*+Oxe`Dtw5W{hptNwJj*#gj{fSQ@qY!!;G&Couvp
zjKtX%pquds&sd)y*JJefBb5idVsS91g<>A?>BwOVc5)5meSV$zlt5R%K8JFSj845=
zZ9$Tsw${dko$G$=x^;<<V#>Wpyb&()9&8oRIbAx}5&|tgc_EfqUtW8A=>$@P5kyye
zJ={b<cNB8Gx2HVDken%#@Hi@%Np`ihzsT>eaS!MrP$*M)y2CQRax5v%SA0d49e&YR
z^p2pofca_ojavw2f|DDbfQ}(q635|p>=MtyU$e+O$_lcFfZ@}2$TH(<wSgA?(JYVM
zW_TPyC`z`)u8foc3XL#5qJCq-RWan%&D77C+XV!(wj2Gh2UC4a5*aNs@hk@9udGFj
zO-6(f1Z!rTga}td3(eJ+Pc^)L<0Qo<iE{2Br9?1U3vliJTk@Or8DSp2oMq-AuXq^p
z=8x4p4#F)y0=ufORkvvqhTO(Lmq4?`N)rMy-jx#Mx2NqQ=`-!RdcYFsdfF+d8Z=*3
z>&X!j?~;rlnSo99vldwDNI#<~z~<o8{;4h_JT(nAGQ#@FUhG+CgpCvBxJ<km=k=RE
z?hX>IGXj`NU-Q%Mbqme9!SpJoAY&+Ig25o1d=Shf>O_ZGfDy|~(<y?k4^CxD2bSIi
zH(u6#b<KaSeK*81g5oqAI=|dz1lbNn7(+1w#}gkt&M0ivMZ+$Vg1w3(y?rKP6qV;^
znFP9_gxmay%88~zs@Mn$y})*wuvrf)AkmC70d~Oqf>6vE`^_4?7S16q5%h#*7Gg=U
z1US11LjjhL^1r4gUf&Nd^auh8m=&lU@+)o*srMLyrCyfqiET$0nSfCte3?nuw7Yzn
z1Z_>yTFsxV92{6^BM7wQT;&r)7JEBVL&EQW?Y)>f)ZxS)k<QU&_7((78`+iCJN=<H
zftQ3C99ZH+y}!1~T}xAqJtHXAfYaCz2Rg4911UdKgwKi`JEy?@hJR&|DUPS%UFspg
z{k^crge^E}9JT&aHT(H-%)mwv9N~DP?Y;IY!a+<@#t_KOJLQDkka3ViGx|l6;MZ=M
z9+I=%W%xw`8HFoyWefD{EEF%_Mn-kT400hz%+be?;Nf}gjdg@k6m|^N`v5NLsW_F@
zCq9ki3d_Vh;bdz5bdA5irG?li@pC@-1h({NlduwF2yUJGaZV3tqoMVW_c#|xV>_tq
zqV$;$)>9i!%gpCgm1xdtInn)K1cEe9hmYF)sA&;|aT4~esvQJWRaZY31TBFV5D7*I
zY!0L3*hyVxob4F2nm-eTvC5-h!j`tq3&wG;7LtXaX{*YI;4+Yjjyz-p)@AY(FRmi&
zYbDf!%>+ju%NV_cW=X!i`xYed^})mX3kYXK*!l0UszLx@GjQJqWDLP+cXj^0%lJp3
zV2g}@T<l*);D`%Kz0CCCWo`a!?S!!xNsc2ZmICh2;_=K#91;qfGR{bN8GO*B%Sbd;
zbdd!Vnr6Z==sx2=Q~T-?l52*)IhIe~5Zb)ArSWS7@K^47hVuvlnY|8kHO_Cdlkkys
z5YU*ZF8MDAn|<9M3vXW!$wS#`(aB{=5C^3GTqI_ZMyZ79P#dv8YyII!GlDT;SuUTg
z>p*61?xorSLM`Et>lIWOiks<{nsEtqv$FgKH{V$CHe6CLxa7!112}0Rs1wKpk2CVz
z7aJ0z(wYHf;_ahYM)weq)eio?B|*=}V?+4Q#|ItJ9T9Yi>szs8Zy~r2THHmYp%h(@
zIa>!tI_t4lx5)I*`|MQjuN!gLX_;SHtVD&sshIwl&yPUlgd<6P9N=Qs!&NZ`Q}ual
zUlJ}x#yjycbkNNCrZea>18%L+<(7%reIh~g7pgljT<m4NF@kDE(aoSRi7h19nuo(?
z3@B~N8ZPELP)gqnrC5Na-`ZWtPptN3<G>dCT^&UJ=9<+3MaAj^3Q2XJtj&*X3A=Ma
zaXE)^tfK?F!Wdq;Nfwy|@3zPIOmY-kWJ|DW;#V5}Vy&s}iP_9RLdJH^5O(&Q?G|%m
z2&xy>tRrYvFlOlOBJuPv&NzZCq?o%P>}AH-oN5)-e2YuEPfL#oyHi9p+AFBG9s}X|
zb+J9#0g6(@J*Wk^s4M1$1w?<9I0lu3Gwnh&-&&K;V?jPj%t=Gqr%k**(J8Po*qkwE
zqS`=L>tM<$BO)v^fu44t_)Ksh*X1%3_!KD|d|S=&K$DG8Li;R$r8+D($zTjY4H9Qq
z^CNq0a@H;UajGq(-eE7Y2wTf;Gcj!bQZ13;ixPq4c&stDC2VQV-Lo@lY?O9DSC<nW
zp-+oUz|0z}+JoI$kiL!J5|X=Sj2u3@yr>yzlQAasP%K#c)PSyc9Eqmh5X{+*T~1&b
zT28`B-9Xe~8oMimol`g(mcSyxav<_AN8ISCBf_OQwH?|59XGhAR0x!!V;Zzgm!H|k
zRtH!>FjGuPz`dh0?|n{-8Bp{c&78lz#$;^?!y~Zeq?B<?r-fu*X3~@~5G}`31?T|u
zG&x`w1WOxCj$%5}UnLBy=D6DXZ~jUodlNpY5kb|&`*bbHnaoSw#*P}>^#oljhCyl7
z6`ssBQvsXN4GpAAAk{=f6!;xgfZJt(0Ksv$rE6_|KbeKy9z(El#Oa3Ub6rM?vwJWA
z0a4JOq}Kq7V{S^(gp0`HuK8b$I>L#D(h&%miN%(n+Z8vz49pmUgd6u@I)K}P$!M98
zKr&nSi6)+L_ZbZ*r>D|ONG|^%jLCP_jAYznB{PClIMjYpn=!(5AB$B;=1u!(C{YIi
z-kyzJnuURmjZk%f&%nj_R40j<fq0co`)idAjFL5X1j&V9g*5wYA(*eZPNj{3l5ng^
zJAjo#!=mK8$k=yv2cQQ!x5<9A*fLXW;KY?BYQ8I4X?hGlBf`#)H-+NVPYcBw#+?Fs
z${6enpYz(Z%LwbSG+tyDteYc?9*RBlF`u@tV42Byfr@H1f4$c7P<Nw7Ko>)&)1mej
zf?J-;tYu8Sio<k;I2{D^E$$~R9NvlZiiBN<yR4ScMSt6ia|_>H)yEFjFmTXNM?Kll
z5;PxLajCCEsD~S|OB~zM7}`O#+%~74EiItXOibp`;T}-BDUM$l@JDn&HvVtanjgpz
zjDV%TYpqX_ZBlm20fR9VD(V|Wgv}u;6vG+>>&sSD9_X^MLGl`bVXRD}zNdbi$1TVa
zxb^|mp#>@~hpnmkRoqkY-GN=OHsECLAYivNfmNMB@EAJ<%S_6Kxf=U#Mof&aPOFbV
z&Pb;LRAC5a`KdULFwW>VNZcj53s(z}Io_1}vrl+k1ucP8puFz(y*1nQ*pVH9tx|dv
z%~SNNC~-ew3?@C_7#KQ$#xc7A7N9Ek_}oK4Yu=>xb+YyFXf=PUa=5Jn<|ANHWSs|+
zzO|5C-}UH)V=$ug+a82n%n|25Nxy*L3h!vQzt6Oo3z}@1@wdI|YR+r?b-Bt$z*1Hm
z6>Y&*H#wif%0Z#@`!SdiHnaHrPGa<5Kr%z~!#E+|XAaz#5C7Z}iiSY?PxH5HY%H<Z
zA3<=AQpdQ!+Cnl*n0y*-40a87H8bvjl|vumTtI<Cu3=LDOw6hTSuPWA`b5?DRpTQE
zbeV&UAQm+br&PlIE!J0KAnZqz6Etg>9Mo%(F?}u(h(@5tJ1hy7PA4Bn^LIjTCb8BV
z8Jrx--n0OVZ^JJ3vA^0u(*a!GjW{s40IR04xky-6N#l4%D<=WJKSiXSHpDVYiVl{B
zl0hj}IP7wGY{wAIjZPM@#_f`D$`K>vxqx6%v64gFKp2Y4Ela}Geq%TGYns1XS@-4G
z5ty!nA5ew_0Pou59TRkO#se*w;|@yMxn(Ra=3@cTlCWR6PyB2+0G43$#cBY7|3J-k
zUicY7aRy9h;JgL9s_UF=WVJC)!rUkLItXygjT$&!0H_=*4N0NTxZEm=wFv|@jG{Jw
zud1~CN3q#8N)lS>9n2Pz%jdlXc|XPwD9$$+I)F7vTeb_3W%TND)wqY^>e}y_5!P8s
z#W?nZwYseM)X%`~XL#OyEUkrP4sd(N>MRL}L_0Vg1X$X#zZ*c2_;$P;HAL69b(eez
zgkhZ9l27yZYw272num_SuE5rzT}dqzkDGaD3^g5Ry7h2fb(v1Jh#4&u*CTio+dU+>
z#PO*9Zs2=8%@5ULwSTM6BPiB_a!w0uImQ}t45Stue~fbsAeY5%uw}M@q;}TsMNin3
z(!O034yAOTsUPh82Q|t;*$EgWMVqOzfGrd!VqZv7A48xF7V5MBVY48Iw%2Wt%<^%Z
zNy+4MrXF<JWqzN>yH-D3Ys~DC+(%$1kInG*>T~R>j<0a|+B$IY&QQF)W(#Om&K)KY
zJ;)ix!fFW=_cFfCKa8*k8p5uoyt3$f>yLYq!I-d{<uttT9oTs?G^78SmMx#y?|`6o
z;(Q68ml=B=?^ygu7?&+omSEvc)7cj2VrKDP$~ZAI0$Q{KTiT2zUWzO-$}HxJeMXT=
z1v5)X)`Qp?QoVn)7JD}w13*_@_i{C%fl#U&_Xx*e4Uh~r9YK`Y7rPdj@VT<n;;;67
zFY~L6HNC9qYg;kXE3z3uE82zENM&avJ(=d5#?UC#`)i#6yYP}cz?|-fW4{NxuHng7
zhf9iIzk!W6M{c$RBY?}l<<YN&KwZV}*b(=<(FZ7P6XtjUc3W`5!#k*bM&bAs8ngsE
zZ=P3)>-=x2a-Z!X%MlP&oW~rJu(Lmi^~E?b-9~Z;m%Q+9G7DW?MesHgxb(zo0+*1;
zpY<Y@erpvB2aschJAz`}L#L5D5E;`5vh@J>Anl}fpprFy@;F?#$i(Y$Yz}Dd_C0M_
zCgE_4<vQiRt=5lyGv<LKu=}mNV|M4X&|txju8lLndbkKXe=L(qfSXw#!^lE$Z8qdL
zUOKi<s5U=VxzqDwnr%REawZ18NC5>=g{a4b(SX-!YlR>jaa00LS^!EDd4iL89$S-3
zuvyrSCklQ%vN>T@J0eK#@pxiOm@JQ`nj?(?mK!&NWJx-IEv_d3jC|xPFk%_;E(bx-
z%_Rggv%x$U`TnECOw%b6?3{H>M*Rvx6`9ZOjG@4Z7W@<pF!Qg{(ZPkk`P^Wi-ww32
zT>`1imWI3LCu%XA-O{RS1i^eAZpoatU>D0ds8F48CSl87hPKNns6phqfC6U>X>itO
z{Ib4WLU0i?KYH+w6UTEZqr^;RV5tSeZ=Bd1=Qo~j;9HmY_JIytFA}p_p*?yq9U(T1
zs5z*#aGWsw$qI4WP<0?&W4SeY04#s5%1C?08JT-&6b!Un7zN`gphZSlSBLF^?sRyQ
zeQazAiHi6=?tkJ+X@g}%7(I!b!UQdw7d&AAo8x&rya9Bn;nqbh78zknS@juVhAg!n
zUnYeXj>9a?Pt{C#9VJGX5fpdk=<P9{wn^KNw<C<uI8*NKail|HfMPMRU#;_fizKju
zQ(x^NP-*yumrNh$e-Lx?Piy*HKC3W-U;*gb3GD!OjrBrCjO9=)5_VzWAR$}uln*pR
z<%WC@!5JP+V;c?Z47K<ThtyBkmco#FVmE?RxQJ;|3q~~<NU+@FjL)vMNnOTg`$)ze
zOpEG$te&75!p9vA0--YC^=tFbY835+V*N%Cik@mk+5*iDr@X=A{Os2+fpz3K+=*sc
zKtkyEH8AuJ@97ZjlCYV%VKr|qYRk{_>7o%d^_en@2RRArZ#k335X{Yd7RL^+FdG-t
z9gM8-53BV7uE||ZH<!TDn#JAR{PUXqGd^iIf>1O*ZmTBTl{;Y!ls3tQ*JV<@VyC_U
z!JHu%YTIW7dZZRDOQ+t8&i^xYh`h;{5lBoQX%*fA$VVV%*<-M|)|J$-wvN6Xv#SNb
z^4H<8*u?0a1B(|jfU{-hKnq4!X#Pdb^K{}R*$9fcUyq|CEtsl$JB=_+g2PRtdzWeD
zV-;nQX_>j8Kt0%vrrQvx%+eWf*)8VHKO5O?dw3XuTriKu(SYqcV7+BIPP{RV>j+?r
zW2WU8xi6qdW6CrN<ih15QSBj6lPy`W=3hq7(0hR+f|i@sH=(yJB(w6`h&|3Ix<ToI
z4wChQ=_EyGiuLe}_W5}xK`it0<S^bS{JE;{&*j6(BWN!4kh5FVA53j%JbARoprZeJ
z9v10CP`k+7E;1=Yozxv{=HhtJ5@<=W^a}2qe-(PR6$9tUK!@Qe6+lwDY2-8p+9dNF
zchSSnQDBUf5`_axyRm&m;fd$5zqka$Xw<3s`3O&9sZQAG_M=0$;L={4+L3*MU;*5q
z%XL>s_B9h@{D|9U;^LZKLNMcO>X*ti|GJjQAs-Trz%KC^qpr~wf(1ncZ=FN2a_|zP
zr}|yyaxYezi%g`|R`{MX2J>~9yNsdPf;6@Hg_^B=f3dMSg5)-1I|o*PRK8fFF$h^{
zQyX6%!Z4-@F!Y*Jg3Y*$VN7?DqHXk8wL_sD;@D^NZ)#z&{22i)2s{9BgAxJ*hi?9k
zGwMtyZ@Yw>Ve5dU^TckdMjA+N*NwQ-x&%A(U3==AU#w|jvZwM!kX(5jB2*qo5;@WA
z1{*_gp0ham(_O`v5nVVw;?TqM>HtaD@LOif%zzV^l+gU!$`rm;xy(lp%)xy*W83_G
zyKDpGceV@=0}w;K;bG2TZ!_)-0o)XNz+Q9NB^1jjhF+~Vn_sH+VcjFU4|XE$9ZQGW
zj6sW1#WC1Yyx|#62Z1Kq!Zs`j!VJ~_e5wv`RzChs`B2;y<Po!HulaYO3~XjmxWkq_
zVHbm6K=xpq1Wq<e)?E^?bRm2vpTsqB!Y*8gEJEv`xaF9Lp+79CiZJFJpAPYjAUO)t
zJGFk|gLo}q40iQCh2jBbaAwc?0s?ByRcz{>AGM4}7ncwUKYPAZ*!+9<f_<h(Bf@G#
zZ*BvMiiLe0gDtgiIK{462hA-Rm#L*Fi%iCCBC!BAf7|vaE+JU44&!~`x7Y9TbRm!t
z1ljR;Y3Ch2Vs4SXKTf0(9bsoQ*FlH{(7fo(Su|b`$p%wP<0XjN?J=q$_Ww|8$CIsU
zbFga{H;FvzB`r=g1Z@n>rEIgX^rQn)<pU8f5}w9O2SAIFQ5P96^Yc?v%58qRe#UTN
z8b{#LKr9N>WNpUSoyzGy2Afl1*QSF;)lfMx06>5nVf6?X-t^+;5?GW)wuj>XV<qUc
zJce_W1f8&#-GxBbTz^K!V?Yt8Okc*G6|WHl*OfT{x%Y|nMk&QRIk%LxMrnSf#>R&6
zbp$qZoPL%=w1q%vC*I&6XY_eEiaM~XcA)jcB0sbIMN@jPb7P&P74tG9U}C6C^Pg%2
z44AgeM-a^Y{+=FSYrzz+-8<wk2DxFFF~@dt2gM9vSYRNNZ^W(a{vAGRSI987##H*R
z*2H^<9HJZn%`MaEnCv8h8}X{&7=pRuTAK_m*ot^q^ISkOL#%(001!RkX+KKO)CV4K
z|8q?(ms*@Y8bP9=={%Iy0$f|3$%!1sP>Q&vCtXIEY&*#U3gTj)LnBt7(c`}3(jT|D
z<Vxb^*QyG~l?EF<XqLd7qZn#^2vXxXLPgk>pPlGvOb0f5T8+ke2PhVYydQo<4iXib
zGRPnRfV0;D2x;GcsRV=-M}#310ms{sEx1(pgnm2D&m0ht0KjGeI~PyFETDM*>NHLj
zdc~2^*pD(NNr|CNy722YEA|3omNiPsTn;RXZ?7nNU|tFdMb(R2Bptxa^Ve~VQVX!F
ztC^g-Vn#TKVQiULCoSROzt#kIo#O~X(eAj32ej2;_o1}$7@ADOzPvPDW}pkj<haNb
zy2553a}Q0nT`%S=3A+GxWMSYpLVb9}Obe$G$RF1=HEw~my}h&<XM|Y$cR*N~!~xxf
zW0yzKlwF^(XBwfGj==n2QZDE=|1IS=)#GI&u;s@DL4Ls&LOICu4*8BVnNIc~1TbuH
z!d!MZ^T14NH@Jsj9o0$U(E4S@(allm!ewQImk5#K2#mz`G%~h?onDV!csTNBydxA&
zxbVzm1D+QUtQ2fxNe94XgEe%~NA@$){g_EM|Gg%fc>@vbTp6F`XJ{d~Ekd>E3gb+|
zLsn>J2bSMVGckg&zZDjlu-o)avX@I>>9M#cV|%sv%^JgZb%027!!ND^99;_>UnTVS
zX{@9=fccy{!2&|H8R`-}nBv#U4ZUT4p8FAv|52msehGpHyP9!pS|cG8*m+f4xflbA
zSg=2crMn`q5sv*@TxGz)%+Nw8SV+zDNU+QVyhk4v>9;BexbD^TU>6aqR2WFYzMXR8
zOtBMZKRYl5px2vh&@B?p8Gm{Z-1|M1CBT*A{F*Ns|IaF3Ed`^YN02PQ^{8PBb|O6z
z7e{L|#zt(Cb(u-IHv$uoY&W=$T^t*I(l}#ZrIA_jCb@=)->!vo;uYbKz%HD-7G?*E
z)oCsRKL(<7@d7tNtBU?DAo|Gy5_-ttcO*fWdU1ZtYpEsJQd387{+BzPmcJv1chq<-
zVRxSUTbXnWkwTy0dq>cP#{DFXk&7hFb;(Gi9ulp<NqOqD1W+(DU73sKcd|a3ppJl+
zJ{t{aLz@^=s!j(cryU<72T;9OsxBPfkFDxVbU&`8E)$<brJDa;%X~_=j7LE7gjZD<
z;0lFPIF5cnqL>)%y8O(>hA*@(AQgoP1?n?Oyqc`~FEa{{H)|CB-KyjKsYOfzA-J96
zXr#Iw6f?!3F$TENw_^zaG>@Fe@G-^(BzWA@G@&PKhVJ^joxk}%0sfEU9ne!ogek)3
z_gjDyakU6Jj1#jh0zVzN_I%%sgg}X$c%VeWX8+&?yTzB8TqkL$@$c0byL<k3f)Nz!
z-*cZ!bwMtk7kV`B0K3NiV*{W|XZ09L2B8QHJ&{oQ%;A|m)oz(dI5--A?D2nV9@`um
zp^7627Nf?1c+;$f<YHO(!Ua$Y79Nj`t*J!VvIQjifu3*cuM%n^xlH&}Y?uCiEn02{
z4YWz%q(}JMAea|*{5Uv1hGYRfP4&7=@s!<l3kXH+!gJAs%TkW_E$p3KW}G?qifD!Z
zSEYt(h6iv2<>l92hV8UTEQ^#DvNJCOZYE?noXIkEP!LUA30@F(>)drG>N7cZj&he^
zi#2N2G(VQP>m2tA*y&G?V)Z<N;P{7fldS@!+_5}y7zgdko3)I_<b>Apr}(pnnZtgJ
ziLkT30edvo83ZJYJ(c@!k)KgDO}ow^ShseLya4Q>SRBWD+K#)(7#ze7ulq7%9G$32
zmx+m?9{)~1R=X_s2!gZ%iCTrXXhhf*j=kSiU}?*kQ%mDpXs*RAU%y|2P`ixW=wmS5
z*$>yv@+0Fob?P#;#(iR+L7?<Fld~X<JZMWT-OoWTTzjtN9Xx1OPdzRdC;cxVnV%hY
zFp%RCg85++9Syn6G`!O2tebbPX*f@z{katg7B!3E7|cgd;EUrXw0nNlvR!S}Lb04U
z>W%_jLG3M#fo6$Ey~NTC0wVc{w|C!)2=Xf{U+vhO`Rgm<>jGd-*xKau5^{h7Cr|2B
zai3}976cjq!2*d}i5fYUkSJ{HNZ=A+VXVj2eDk8jvCLZmyEt7HmXeP$2J=KL;40YN
zU)K#=P-y@L_c$B728r_K8`|a?16+0aF&XY>nLuyNgu-=@Fjvt72vWBb{l(ydF!|h%
zBa!C-N~I~nj{<Tn*fmBZ7hsq2j8AqfA(ZxGg(@?AnUuJ?haNO9&P-x>S^<|damI?O
zLvgDfY0bY1grURDco!4kBK4>Y*{x;dE#2c81F1>O^k*3ti?8KhmjvQ!YIBwZJy}lj
z1q90#Kb3cV&LI^|U<XQ_w}(=C+O9m47nuvIu(;llx#&uB7=+7=qpfp^;azG)IJTt`
zb|ncj<bt}7Ah;M_Q6R-C*kbX;S)kWqITe#Qpdu^|?2+XdP_&Zj#X+N)zteq)+yM|4
zfA)ze{GpsP1h~%eNG4Iwp;+!5oKx59A(%Vfokq478G$2~%6tjI>~5=fT{?bvX=0-H
zl3IiojcUmiC@up3vsT-qjIvXljH^(b=iP7#18th()=o6e8Z;?@oG0VV0Exx^$1F1l
z2SEq)bhccG4pRuxQq!R~AM=Lf&i8t%dJb~cw>`rHBXixYK@Rzmo=7E@V2cvBMjTPO
z%ov=3q~6WDMsgZmS^y~8rbn}&jG$O7@<=23RR|Ws(_8fH7D$Cdsn!UaGq#jap>bl2
z81{7rc3wObtGqf0&MO|gILr8RUIU~%BxV-KLiQX?S$naLChWZU?=5*;FCdt8Z8t_g
zfMQ9KyA8@^M)DIVRR7&73-M>`D?nF)<5dWI1a_`^2t>!uSCcZ?@Ij~6S@1$aI9S&*
zLR^<Y#A7HaLuua(0u?#YTBmauk0ZT+a4CJF=2(C&AC9n6f1vdR?rU*9MAuT!V}=r5
zK$1p;7A^tK)8^wc@g1-C-@Pi=vl;?HW|g_2Uh*D6fK@j=;lkclxTV2nk3&3k{)G4C
z;2D>c(u5gdGs%S9XD2#HF6p)(y+2FBG1EMxu}FgcgbTts1ecGqm`>d%mTsd0VCv1o
zkSNa*;Fi<V>@f;vbUE~B`fHT~TsiV}uRw7A>~>1`!KF87?Wo4j7`Uftkv3!S5@IX~
z*OKD1f5;)zsP4!DY^fdThC`R1`NLT00NA3E6E*%>VFAh0;xc#fHt>0-<kfoE@4;2g
zVx@VJ5jL4RB9e^2N=@_p%Z$L^Y9r9*J*oh>uz{YgfH6E}iZ}wH3i0RAtAtB$oANFL
z(rh#B>$h~Iehutu(oK*8(8h8zjqog!Fe{(}UB=nKLF9e7vyAgpxW*P>XS=H^o#$s9
z5VU#^E={HzG05`*ilw5nmk^ej!wWj}dzp||<`BV4s|cR))ClbIV#jj?L?Y99oEBKk
zkK~OAfG&zrN-M232t`Voxl;HTO6drC<w@OUmZ>y3LvmfFil=0y6Hu5d-5~aYV3Fe<
zM>2wq^Gpy2gVD<#in-p`Nso(!J8{woXg-hOL3#i|_DC+e<~=Kiv8O8zS0H)(5eFlX
zd&y3Sd^Is6ulht{*hh)h!1PLvUC=&bfQ!E2ryXVxoOd{IKq;`;G(XD(%=Pu}+Y1Pu
zc$9}>f_e^>8gPJ!$`eLfuX_y@H#r`OTM{mLGeZzFmryLrIH4NWyLnlq#a3?YAFMzt
zN~Og+LK-DSr^;hh2*na-6AvK(EhM=J5CYis<}dCv=N%`3+e@Ja5UOz}hK*PUvG7bc
z(<Jf?iZkX#0Gf7@n0-+C**Vw+al<Wc;XyECTXObX<oEf>PRtM>&^*(a#b0LRnI?tJ
zd)30|j|glfF*8dPew6s>ko+px?CM03EJzE*?3}uUH3*h@IQ2um<ILtL%+f6599$hP
zfrg%FeIY4;oGn@v&%z{OV>iy`o<k_@p%qD=5WPbRz|x|eNsQqFipDwl_%8_~lKr@L
z*R0h3FhAMw9gG!7#GOv%J0WaNUgt9zs}NklU@c}5Ei^Y88`kO)U^Dmn=oFd9DKk&x
zJj?j}O%PqG1FMbV?<3AKHk)*^ybB1<jm<1H0Eqe@W6XMjE|;TqsO?2!j-*h|(kZEQ
z*w#y+Rgj0oE3tX+tiNK3-xWxf9{0Cnn`HzA7VrhdYJ4>#@Q*gJl53$LvR!qRHNqB|
zP8(qN#}I53b%`UUFta4#mvyFHl43JiRXw}P2~Q<iFCb71yVb~nu*Jf=mB<2!0%@O+
z7P!ciNJh_uEFqW&y&D!hTtc$+j$``MyiepX;g};ut$_F%NU|P*DVhriOcVSKKbAZN
z^a8L2>*p)HRDdYE$buUY0BqIlv)l`*%`zQLtzuPB2hrj?_FMU95Ug=wN_<r?BQbN-
z^!OZtrEx>t_dqux2bw);gd!cAYVck<`PDPqmnKjntt()-dEd&#fzPI{K(Mqn!g)CY
zd(x81B-LNdXnG6|(PTcGUV^MaQ0Lpy0pW3e)f2GrJ$r}bI#AGM1Wg%8aRxyepabIz
z!mi{7mvWTfITTA_vdQeB2ci?@Y3~b=Yp}zGdxVg9K9vo*%)|#WInDc33Fta43bz7*
zqB$FkHV}3}cx0$`#>F?NdbAK+7SjyPmJ+krls<sX`f*H`s5unpcocuz475tc5h(xZ
z!x_Y47X42E3kcTg*in}A==>^<5AXL7%*u=f1YINX^IXM73}C7?IiiY}`3;XW0&HGh
znTk7SD?pdef?qddMi89$QE1I7n8s!CM`N@A*AW+;yF{jE&M1%EIMcJtrz6cWf;<D%
zrh{Oq@CSahK9Kyei|EE86Whgco|xGhW~>KWINMC!-3gMUvz9?xLNb>O>_NHANb;6>
z*6{wdo_C|@6)-Zxlpq6#*%n6+2xFUf;JnQwtZs<51};|dIQ}YR>~woNZ#2uu&x9|s
zOUz10^>*CYv0V_hInaccg*S3LhvrJ)2Mgi%P+Xk;Wq~Lc5ZqKO&e6an7*TRxkdK!D
zi;9)G*Kf@SL>1WvM)L_fGmat1oQ)tjvz>S`YZdHfdNPhu0@X}0xUFU4)7>~zHco;r
z9H`YHn0sD{3z>EqW$x1~XAsnrtw)ReOiztjQ4)CeP41x{0u2<~`xk^u&3#A-46|7B
zI1+W4Y4Q4~pEn;^<Im&R+Dc;Ht5k!7OHY*-V-;xb<7^^xzZQzc#S`tZPJrMhi`}94
zccd}2Vn3$`SBFk#&Q1S`QqE*`P|W^e9Bw-U%M9u7W)=>|Kd_t=b{od=Qw`*Oi7LnI
z1!1!(6_!RWlcJ0@q+TY4KlG(-ki`epOgVvMd96Sy^(|*qMo?&+nUA}zf-IhcIMf2P
zFxYI6F0Unl>5RteF@(}SSQXR(>;j}*GWs19svNhB&j@=YwW?!Isp$f$OX0+4_);>H
z!?O3=LqRF`;%e>%z-4Z;7l$k|y*4>D7(k%NBXQh(aD-Wp&=m+fA^&wIWvJgUNUMaU
zr2C!{WlSFrUIP~v@t!fYA47m4Pa$Ri%8UDOU8clzQ05uf0y#M}`-_rvUn|^mKp6x$
z*Bxc|kX#lU<ucS45UhdaNF{w_39U4g)cz9an(fCkTFr+9NAYa(3Smc$LpCFzqwd7h
z39H~zHD(X!WecI`NSQ&dL7{%@aik%?PG<aro0=SotHAQhE6FZNv4IkD$aFfdr?G1R
z1$!%Lo)bixKFHSt%pm<w%I}YpI1Rf5xH9ny=w-&eX_f!bns)1r8Ut4nW13|95_1@x
zyjQ`lJO}S%cNKywPX`DET7v>_{-5J<rsuQ<`OUy)=NXye4ro#6g#-6^NN&8C%*Iyg
z0vhs+KkYpy?Ao%M%gm(*n*lvVD9K%9jP<Fu6qjJnE+ol&X6zDTX+>RLLGM4Tau##y
z6`<LOjvI<Lf(9GcHDz4Q?|vvDX2#H*=L9b<BmNo$^Tx)64_!h*I2Qga<Li8k!~wyD
zZm7-U^{2DUgS#N%<XmJd&iRy=Ue7bDb;d-LrH5caZ)(nVVaPlne6vd+Y_M)ZK9>M<
zsiS_9VDks6j86`AYj_2W<~=JDke|4Q$fs4X>#UQPL}(!(q<AfPEkCjf(JW@1skdX1
zHUqo;RZy&SI;Y==ZA8MZ#Fpkbi%dOd$wx0p%#A-xSD$!0R;m|?H{zK466}gH&PD;3
zP>MP;AJP!;;lcg3=EN(6-5S~fIE^zkBWO0rZVg{KUxn(7c@sN!om(j8{H9J`tpQz6
zCib3qjgzFqSTPryL9(LSp&!H!B6haCV0#8K!#8*12;ah=S<Bo(0_XV|C60w6gu)rQ
z#bOZ(?el_$z-4~r3}##}xV*|>FQRh=yHsRb5BaVDUH)d76hm;$^oCS$yb8%>>Ir>G
zvW4P4`*fVtTmvH={4U{@7-t?#^8EhTEJ^%Xu+Q>!PDD+Q<@GoNI?sD?$$J5Isq6Sz
z<VrY)VqWKrEHK|gaD(cdCO`akk+k_n_Kr-*d`lCoO9;qpM^oMABWfeLcCpdB0?~yt
zByGyMK7vvTpHfRY3WRE<wVT^!1P=Xc@wf)TlEj(iQ&?k2v_1ETwT_)-3LbWsaqb|v
zRNPNt?a!cC?bxKF!vI|-&oijQWCC5RQcv$Q!O0;CaRD?BbU1IfFCoFhhHo}sf}A7X
z4y~C!va-NX7H4HwpqPb2c{oQv%Rx2{9#)fp&riGYEd&anU1eX^^7H;Qbab2qCOfjc
zGYAw={}{UiVw?28A7}nTPTTR=2hhzOuj++vo<pN0eTgii$FO_>G^f1UrJ-$^1YW4q
zP<{!aG+{XInvbfL=YFM~t57UI`F>(F9zn7^VoPNe=sa%wP<ZBXpw|*9Weo}}J~QDl
zoH60T6K6}%#2Ew^p!IDW)#)I)n(+!=thmpd$GVR4EC_3^FxB5%oC6T{I*t-P^&m6D
z5o<L+*t{|?K>)yN08dJiOK61wF2%%jwE5^NhVAkvDJxK%@6l98c1DRg2}&ccf@`(n
ziHkNP@Somj&00oa6CehTE5amdHG^P2)?qEquXm8BsE6J&0BUhh0w9P$<7B`&VOJ>(
zWbEVj(43WAC7wb+vLIqRb_py>%)J!Gu}cW9Ab+|?1)GnlBG7F!HPi|e$Jp4{-p(jv
zgyE*0SE0BJ{;85~0CX8ZJ?60nf$AJEX+aSH&W`gw)M=K9;}0~uOw9aW@y{T5D0%kE
z@3nwP<MXcy+F(2fm6n{&EVLdHMefBt#0voY;Rp`61lriBLULDJUQu}WRg0|VW2<!c
zImF_0E6`l8H6LslK`2&BYcpDVH7N|A@z7ff1x|I^mddVyT>1UIaHRR`#6FD~$a0)%
z3DyB=?Wwl6&obd@oGw^^T{KVG#x(C7iiMyn(f&imjHJ7zs@BC-HqlL%U{{HUG$qPQ
z2xj3p&Y*r=%~Vgs#G$zrNK{C-%4HNmt`KMPY5P^M`C>{Nrlrj+P7bRSGLrn=v6h6Q
zOyQe~&p=e%M+ga*o~xN;YzmxZdJa6tAHpx76gJ}Vr1QjV#v#5wF?+JK!v)yFlCLNZ
zpDrO;6kPk!F8?J2SA}N+k2DN7A74w^jN`;BVCRC-DSu=D3D#dLcgiYZD)qb{0c}BO
zq#qhy12}JXX+v|yP%QY^{+VS2o_C6QSeFslCPyKF6cnf97le!5l!)9XG6NZ^_-9=x
zt{|^Y!~w;O!ptN#i<eMfg1wtC_?JLSl|N>{P1bxu%>fa~c&tFEi+ZU$jsfWeR9EG$
zZ0{=2++pN5ea5KGoH$$^+tLt_)f+tv9cPj-0&yO52F<x{%G=N-j5F|Oz*;#U^@d>H
zZ-fK-Jn{8dD)eBtz{cJYPHJ=_W4s|wLoO3<#(VRZVCsQ`<;v~Td}6T8IQ?pt&nqA^
z!Jrh9fM6!r{-V)V!L9>u5o0KUGEx{AOatHpuC0(2bG~;hW1NY0bV7Lsbe`-~(*_;5
z)O->rSI&~4+eV~VK&Z;jcBR%-=a3LLg{@KZ9^@8cWG!uf0nPH+EH~4ZNkQWLkv%;G
z3jBXooQ-TgsV3mUMeJ6rKye}0u?!;+k@2j_P6A3Lj^x8{!Dus&-6IG`)TLlXi;QTy
zJ_DOA9%GGpeh0<ObL_q7&snBoE-49s;p7DOH|QJ$Cv5KmdVu|T_0heEa{<{h;|zo^
z-7g`zAex6kLeHFBLNhy9*mY@9^U0MmpKQnZmK7)-z~U+>9Zh`%VpWwmA-0<7a9u|a
zZlP4=(VB1#OeOR;idd&H$f`6yrPvwJ`SHSx+(I1$bH<+|M7KPHB5mVQXe45Q0(=ev
z$;5M3EpXREuu%N=2R%qkUt~PIZ&ITr1WU;(A&EeuB{wu<Za$?-%<uRpj}>UO0r3hz
z89{M9x$;6UT20J;J2h<+Kg%GB9Kf#X@ib00jFW%^Lw0%w0Ur2YVdRjhxf0B`vy6;g
zluvvC!FAv?n~#&8L$YjnYZGYz-SClrJnVUqw7E{?MT0G&6~@>w*W3z%v&P?pgiyEU
z6}9^Dr|eKxlEMa3_-{r~nDFVn)K!AC8Mk(kb_+0DJDP{B0d0KC4!Ae%or1%YRDK4q
zXxUAa(FB_zycpJ_Gf2`Z-}hN$1T6e%Od!vlhm2I5;O>pl9Bxq~zYDOr+K8EUe&*R{
z$sn;FS3&@v8s+JA0ciuMI<&Hc5yVg-eDbS=&9VNR)%^mJ3w*s^kYCFfJOt_o3dhbM
z{vUis*o~r(>KG6#lbv{S;>?kG1_xcp$eeYPlkgk@W#P`Ve->B_CGvDnx`0;mlR;U6
zt>7H84i+vm6)%03oSM~IFD^)XcVq>U6YkAz{6<im^(1yMR>5e)v+?(AEx;99pTs(S
zEn~B~WR5ciBSxl3C_;Wa@c*D@Vr`Vc^%(>gj-wzt?;@kznufdJJPG#><CaAa!R+C?
zM|u}md7v4R^%4pUV;E`c%ls?{Y`k#)KotTzWZqg|fn<d!u7j`;5XojI4yUdH-2D4{
z++%8iW}I6j;bL8LY&jGW0zm3b<HN!XWSQwnJ6Q(^RxfV?NfH@Hj}H@P;Vj5rr&rHG
zRskk6Wn}C#2_{4a=)y_pBd^jKhD@S^0Pbj)nS_~vM!@F5$_XadHC=&JJajf6v8OqL
z>N?Ox<%nK|fOMF{z<mo)qw7T7TE?WK$R#-T$GDF;BW$_VTb~^iH|LEw?Q#YtUyLMq
zEu55*9T5=LXlf5@?`U{Tk<|sDFozb=f0s$naZfV3gy80jv!TsHwc6Zq@*;c%l4I=3
z{EvWUlx?LbWEF}>r`iNDTWuk`Y|YQK?1VrCPU6Y0F+hrN82kD&kn75&XZm>ucA2NT
zigcFnAPzh`ZP^kz2f10?{%+7dlkj(62m&Y@re{2sR|(v-Va6_@ngu*_GK>#bjEfc?
zR={QOIK0mpGmVrP<$$euT0Wz2{_TY@|1Bg{6Rna47@>#HGGL5BA&*#N0m|sXsCQZt
zbWklr{K~QJgkb68NAW$Nr0A@z*gA(&6enIF=)t8^vCS0D<_k!cZ=8Br0+CL<34cl0
zJ>HflHg}sxDj#$ixsO&Lm=FCwK?nniE8s^~kE8ypneeXH=xrv%T*ORk4T^g(Xa33=
z+VNGa@^aG*iYvi!;Bt*0l7+(`3dLL^2}dj;F%FB&hIbk!@)w;$p*?iQpDA3Q-_fh^
z++AcA%(3KcU1k)W1CLD&C>HO!-=}!A^1q$O@K&IiwFBH#lr(~1f#20e-D*Pq<QXdg
zv@|uVR{XWA2zqm4oRC>8b`$6}@_f)YnjxAm=4t9QV5Tk4kpF^UQKO@HZ09`V&SFB|
zg9`&J>2(zS0*d9$!?o;w!P7D$v0#l(d<n%(=X2fCgXXbX(|uh=SOFsN@D(5ikjv3u
zXejzB6lXyfVW;kdVXB4Zve^dZI4do=mI<-4T&hMS83`wn6=DX74#mV$Cd7chwva3U
zmJLt3$$VzwaEZy8c@C*)lMmJO5X>v%Yxs^Xptuo_bV7XztgPd*wiSWqg4jwR*XHq>
ztK4}s#Yz%56rrtyQ4%<sqs7rGgwhs_<I?LE3N5K6?HWO|up^h$7)XtujM>r*5GK{v
zhCrpt?6)8Su(QRX;h8oR^5SfC*xYlVrSACv%HKN%_G+e0vkRcRIFn}%Tshqmss*RL
z4<vt?83jSF-!@NFwx7{dc?IYsWTV4WBM4@W`#E7wR-sgG^db+HZ6OtTv;JXtl|{{j
z$QLRifl_*!JZ!U!pi?Kxrh{U^@b?o8OJ`u0q*JG%fD1_2!_cF1!gU68$e*|RjvYi7
z5~qtVfJJw#d`k$<lO>Y<O_v#m`wCh&HJ?@!anw*wu@wk3=H7-hb_9S+f#7=8SkrtA
zDZ&85fz(@T=ru#SZmfF8K$$r%y3xmGAaf%xXAHO<BnzDV<IvnQ=Rg5F@%-pjgo%A*
z=NW-}T<8W6ZrK!|_zQrWMw>Tonk6&~jzeOVu}g>+K2uJ-EX}7^m5L>7*^69(YVp3H
zHXH$+dtCOlen4|Z&&A(dw22SmSlAl4sMtw3F~<-}x58bZdB6;c)tQw+3c7<zXPZ&0
zv&@6Vp%zaI2o|JV4sk;2JV~2-Mch3km!ttATv<tp;Q+g^gktu{T9czo2(FLZ>ImkQ
zH3H|+rM4>&)TF+LJOUy8W6r;0F#t}>O$pk&1tUIYVKGHsbKLvkQXF5!jaA7E>?-N;
z%diU_B=e@nl^F}plAu4+gZl*pctKah**QqPMP9SsUo^hL$VDdPPjqz4WC_8d;aW;)
z>Sa>4nN7w&v3^Ev{1_lsfJI|szh(rc><zjB<*kBhtMYDwDi6gK(%%}nb3md=_cV|a
z46W5R+ze3Ub{flZu=Pxq!MnI;&`Lv{?&)mjA`>yKMJLX|*3~y@s@i9K&1BUzGCr?J
zsO1yS;t|<PaM2cAR+4A)nY9MI%2w9pkjyEMozvo@#BeDe4cJT^#f!H<SA_Wi>wc|)
z)}SFb&Z?@6G00NVsw$?%Ge~Ac?_NukU1sBZ3Zk85EWLfKUDgE@{~dEqMm!CLURB(#
zJqR`kaX8F-?-vkVv%7JTZkeg@y@=CG2x!-yJU`86)hs1DmOd*G%rlR5g&7-V6m~~s
znpPnc(S_T%1)~9e<!udsP*@#nW;f2LoMfix8Q5afF%3@{J7`FHkMqZIqBN86-(*O6
zk$628wC9OWCpu}^C&mGVx?F(WaFav*MM-|fyMnS^g56A${la9^R;%SxUp~^j0=4u5
z;>GzYC>DRXjt~R{*M@htOZRD^mHZFi=oSB3MvKLBjE7@rC3lR3EZQt%Y0JkWT$ePO
zt}N`%pqUGv=G3moA|uI!a!wC9pF?vKL1VViCq7~a79l0p7>|?^-{-+pqy(0V=&7#e
zdS#U<e>^X{0>v!wz<@UWMi49rydJu6jasilGYf1J(@rhe?ZSB;I!}IPdkdu+gDntd
zLK~{r3<_#>H(q<{050;`gE$<12ElUXnM`k*Eg%)WjDIyb2YWOsV@ufXK6B!Q13hVV
z0l}S!S$WmMW#+|IG6@81?)je}WOAdjqwxavSFQ+tHem$8^<m}e=R2U7JvJgSb1k@-
z6uCu1#n!G;>9#5@GloKc;<%jb>Fg?xINscWt4i|Z48{aPl``=cKxByK(~#%+jZ--2
z1kmN{Y=HB>fad)4%qPvXgy3rEPunzgxP$^j<-}WavnD8iSi1tj?E?Gyvr4oBQmJ@6
z0JsW<DL-f00$hNMxe8c=LIDhg_Zg1HAm`_M<Y?B+>7NYeY?tZT-H7q{42s*+moGxF
zi_C`C*zAowhh(;9ar(OlyY9G-Wg9Ot%@+QNaE~rCjV=x#j7+09un=l<t5%1}SX={L
zfkJKRmwX-p+&nQQT?Lv276?*_OwH>*F;`y81a~zIje%}6CRqHFfF|s-q-hOdS_iGP
zBi>i(1q7GGMg@2Q)&MSNSH|7pi32GH1R7vUcSqV6Ap4^9+9K;o?<y@bmCU#dD+D(r
zf861diMMOh$;hz2pqnH~w`OA_dj!e7v?xrtx>lhULB;M_LM|hsBEV9X9*UCZ2Z}4q
zg=sBjXAp`VK4z%Vwp#~@dT|y<YIX)vyToas1;FCjh^XgaXNPLU^K}rM9jj8e<weGc
zqn-4X%tV)nkqEeY@!Y|pYR#R>yw)AEzAI3iIa612H-h9&@Ej3VK`|esJzBz+m;a?j
zcP)?zRLkoi^aN-DaWpu3#SDryN1o|2@+M7Jn98+hXqE(ft5OuOdDHM4+sNmTypMZj
zSEG~ld~m7!6A%Bqjq+q}95~ZBlQFcI54C|p)t=YWZOyA{*&8Q10JQ=xl?p#;!ecqi
zRt@X_c(g!QP`BvZ#n+&@89D8wA$$zMwLuD7adK=1$xMf{j7VKZ;{Kj*B%DDqC&#gr
z13F(8XR+H1!BW$pOczRg82^fJ0agR*zgw4tX<9DL3SDN*^|%DpJXtx^>?9^&E09P~
zTqYxs>nV|Xak>hP?75_Zeh@ZqY+-94UW4EYZtPmy$571eER4(yT%^KYu$S~n+TSXM
z;0%&wz=j}-k(po{smM9l#Y{@odMDt3WK0MyfK)?QpfNH_kgI_ghx^*nmP;p)<&Q@J
zUR~LW-lM}iE0A0{{xm^rff0nN){IyouT^NS-sHaiezFB4otQ1I<+uI#<B>7g8RBEr
zbaMv5-3w>9>_%)!bQz6*a8a}~2t{PvG7OO|AXxyc=Hoc{ITSO3*G(%Ku$hg26AorC
zpqT*%8Fh#ygwo-*cz<5&flEl3D&~ahBu~`@<?+N72vw))3=-rBl7~RHWorjng=qfx
z7f=5^78DyOb!6@LM`hPCBJUK$^Mm80FiyqOTr(&xfhX0rj?7vI(e<6i0_hCwy5Z%~
zEx82*tI(QE@OggcB$afk&kX2z;0v&;#{ZYI?2>pg3WlCMF2O~+k+E`ZUQ<iun2B;-
zNdo&hiZO!VG@1>^WHl*LW{vDE6xRh~!#rdSOdD{SlbO^Q;EKinAfPpweoKG$LZMw!
z(4uJAvrNQ;;&LJ_Ae5Ho@LFiYIV4my?yB{KDUn4GlDo*bkyyKSOGvfm8^>~4T_%O=
zIN{)Ho~}jcX5vI%%N0mP;k@ZSO2Q7qG<>Z>aK`cg%DlBu)OG#z^%}^cnZ?Pyac02Q
zlB79<U_FnURo;Jc7G(%I%On^ny}+@6K*djvSe%?^av9UI`IHG*J?Q^WE+Cjmtvh31
zAt_wOh(E}@gn|Syi|RMe)Ox?kjkp4#_RcURi8-c#pjN@8#u|Mb63Ena73t7g#$KQ3
zZpAn;s!i${*meZ>z(GAd-$6CkafpR<XPJ)40is@jtrT)JSRNT=<`;-E3TD!kyMW|;
zm5X#?nH2p`ppLv;LMan1_R{K+?B=yq3LNia(z^n7+ikP(@k>4<XiOG2bX<d=wMQ>Y
zk!uUVhM?MU3SWb0(Qd|$_83g-u4^k|Mp$aiRRMYlz}X=WO*hUmCB7jdEHZ-5tV1}@
z2yFR+doU&iGZlXYa*^>da@rnWW_+eN;LF5E6FsEUyspZ7qvo*!$xN}sr5W-FLK*#c
zB?DmTAG=3!&#8st`ttS;bzB2$i039Ax;@V1yzGGhfX<#3js6j-gJ6;C9+%UcL30_L
z^$#!I0*d4omygZ?#pY{%FALE_G7q<7wRZuogAE-d9s^lIb4B7{$R)5;gxw@<?=@?c
zH+DyLJZA-h#4Fb#tvv#Yi4E;atO81{CsMiAap^EyZ&VM-&l8@nKn!3rR35M<$t;tE
zZgfB^lb$QUj%qn_22sPej{7VKQ@*xSPz8X?=Yl7=cf?H%nHPjDWG$4)WeEWto^YVz
zlAx<?=Qq{<?8=&7TvJt7pgDKmB_xAUVom`un6H9e0t1uILbVW_ZFnR|vX)Uem=k9;
zGLM6)o<N%=zVAUJ6Y$c6W(;RZc)>DXI0e$O5AqyriA@fY*h4fY>sz~V^!FkuJm+s~
z>5OPBl#qUZ2`)pek7gp3*H;odWA3jjN%F-VsRShJ)C&_G2?knh%#^7?o1eL$jYtyS
zmlYiY&3-(Yx*bP1XAtQ<d-IsfbRe4lP^-K%!lmmlVWFcfAh@QpE&tK_ITUM|{wb#X
z_f;li<~pO;3kX+6HpZuArmJTdkqI=k?Sx|hNaNXBHrlO8zzPKS1(bUPD7v_%yEdy}
zGZq^#v~}y~8=jK`ToLv-G?<T*fSuDrGqcQqKU5C}x}>l&i|adQP+TZ~&q)!B%!KE5
zb++L=BcK~{17?InPh&42m`!b?s8cPWSTr}78{thr3>gUrE52=H@my{5*Kg_Q0?@gr
zv&k5ZpdcIdU}WLgv|y}%TcEjS@<fSiV7J(@{`tf1Q71t=ceHGr0bKyQUNIl)lEU&f
z4*s7(p-8=4g(UJb_Y5W)kI$i?Mtgccu_x?0Zp-BtFStQ6Q@C-6zRW1hOdaJiNpTuc
zHlq2Qki#CEI&ldRSmx8wW@T%01ljI<56(6EDs(FW`V`YF2$n>7h8^LpK@;Ec=TrHe
zKUFAyDma7Y9>lq4j;BX$I*6_f7bU6i8Q2x(h+=sY9g;=L>3h#Tp_EZh>x>@g=>ktA
z=bitH%t?P)5EIHJBo~HHj2?dpcD3RiNu>0-Svt=z$I}ZdkX%+2J<1sD0Qo)1RY--u
zEmPV|!>k>_01-?$3dV%782qET#54oBwz^6h53+QS%#+Tk66XwpMa`aFQOrfsHum=p
zxVePAI)~=U`Cb1WY{p-!4bKapxnwaG_g<HoV7{vum&SO~y)<i{j|@5a;u{DnV2hdZ
zk~)HlGUISX7gqt)j?OTpPq#+w*2`|Jfr|Z)Rrf?|kFisGp*;hbW848uHh`|sY${jA
z8JKZVPKyOW%Z*(GZ;GBnLdZ8XH|PmlhMWXf*%uJZdOY=0ruNIsC1&cfl$V)Oul=Fa
z&F58~;?<%Rpn2l75*d%cZXDLlN8T%6%_KNRkpj?y*P#!s^46drCAQFHfX4Zq*W1Dh
z&7he55~YJ+!Eo^nL1YRh-f^~U0mW>sPqjsHo;m2CEhBypq425UC>rY`6ZweaG7~Y|
z3zsnjb1wI^<k-BSW<%AA4zECS+0Vze)d+0v<qLtCRR~4N`P-P#)%J?SW@`+dYmm&L
zb~)`T9A{QqsmQuQFh_cyT0V#lO4w2zC^$19DoU`(#CpmS({c_0vGcaFoKHOnw()s+
z*>>Tt_r0~Z1eqJ0<7F{<=@cmUadhS8^F!m}e~4BH7sg^NLNojo&omMsY>}{ri~}4H
zVC7jJ|4`2WxiG|B7#KsdR5@6tnr9ZXy?u>E9Ry32amTwRXBmelQsjJIWE}s)p)>=M
zx@$PJdjM?g;0M5fTx4?QsNvs%K$pTtNyi}U3QpI(&((ZEtrU0G)hSmXln#DB#X5rM
zy45$B5C=5ZjnRbqw_wVN=QP*yn{UgG!Nn%1Ml)bp88RcT`@S7iGe>s_ojc29zN`v#
z1ITGeb<UG8XQF_7dkE%J|7T<MyMPw;XQ)^bE@O-iD)Jznj;dZlmG*7vU{mvjl?(mz
zlGJqtii^eZreSvk!Hve!*SGCfAz2vnnJ%EWiFwyU#v=33V}LBX#}F)$1DzF`Wqh5q
zMK8dX#mS-Hcsff8CmVv(MN)7I2dC#ysL1o7ojt%T@m~NGaRD~htPQL=OUGyFL;GC<
zT+K~))8>mRi>%!d1JK`b-fQ+Sg5Uy9<M8@wMwlPQn>;N9*kRDd+^qpzBQ~Yu9^x1Z
zB4AjL9gi8%*|Bj>f-Vz;(~O2VON!=9+F@BhaeHz(6g|QUQPXp%<`iF~gxy1M*3TVj
zr}qMEQ7{K}>C245ohJ<;V6&?!TP#(-IP@so2rGpBHDra_jG(woy@n%uc@>3wAJU^1
z3gRFGDMWroL1Gm$hJv2Xbbw|?*tv1@OAiEe5X^D>UzyxbXPE;-eYk%YP+U!Vviw?^
z3v&mJbA1wM(Npa11%$d|R3`)QzJ%=Rv2_y+T|#lzyWs|JzN9iZJq*LY0>KQ%`)hJa
zj37A|y*eTtScO6r@wFn2OrR%?!*s1dDt$sTg)z`&9)<CmC4MGuV04K$!;O5FcpfKY
z7htl}DK#m<Igr99@mDQ`EnjwAv}CwQ0?xK@`$C|x_V{BAl)=C+#x&!~7CUo%Tmdto
z(Uq|gL090OEaGa$M1iP&3&v10^u(VifC>+c{_rq{U=ecW4}T)086+~)p?qmjmq~PQ
zOFEDQ<QrG7AXw&G<(SYpl%hF!sP;ECXhO1J@gAx2nTzgP!00k_**n(D`bZ!tI3MDn
zv}O_l;_t6yA_30vDf~qU0_?i!pQkBe^@@No!kV<1f;$XIaV;^feROvWMpl}`kpB!s
zYn;TZLS05>yMV5G21ZD_4JPInj?1$i%$UvrW`tL}bkMGcU~ym@JjJ+3l6pmG;xd!z
z6axHTLZFd9n*p*}uOi;i^CRTBl9cuGEZYbQ{OS$?bq82D8~j5tRlma1aNdDE;%YLc
z2@wU3A$lY`+N#YzgJ#~>d5a*#)<LpLZNu8xRZb^I914w1m;c|?x$H)9L}9qcVqsZ;
zkXI3_=1w3$W6Z_o;z<ZjNDLXm9gG+=zTg|vQN~idiCR%sVKWjeh{U3mMU+TcOau=w
z3W}8FJi>@4m`BKW&fi^lVd+1|(^X%ex^#7Qb#;zYAz0b)I?}<Rq#GM)@h$K!J|%76
z_R>HC7h$ymwgGS-lL#!)um?z;5Z`1Y$Wf!TQx;hgk6<q@`-fh!)l1tVEZZx`#!Cra
zVX)q`DCa6ZRbN(w`_z67kLc}A(gO40FdjlsIwY0iD<bT#`(@!ETU}&#U%_#!=^t~?
zaj2z!Z%^zxn97$b;5E97M@h2xGYz1JtpG7vSGo2>8h*(iV5%64R}-x=M)PGpzFK1!
zrc$vdg-oi@RB$OH-V~)FB-ldX5td)otHA@S!`1MJp1WH@5=>v`lZM~({8cRmPmwLN
zd_KTvhhti{-XX_S5m{ebmU#!S*tkEn#uL)9J8v(a9xjy+%V}(~v#(=(_d%b`!7d8l
zzGPZFjg1z{v!vPjqMcoMl-K%(jhJHWqL$h_8D1q5e0Ke?Mps2!2d&-%Lz(zCh{gVU
zb-dPxXbL;UDl(X}uN7z53bgGJWQElZzhDBwdJ^oi^H^Bm+&VZ&aFqc*CDq;fZyONx
za45`+JWKFxru(>*sl7@^(+S5AK^y2ri)918ZeuJgPKqZ!)btob5wJ;xVv!p@L34-b
zH&fN%;X0}tQ#`_^_^qYYaS%4&GdV?D*jSQ52r#ToBil%UGUo<pj$O1ny*bgrp`2&f
z9>Yqgi-%}17PL!1Z%YCVPNg4yZ^5)`ItU&%;`F7L(UODdgB)m5v)zpdeT=2-n@@cH
zEy1DK`hx<0VydIpY_HcETCrkuyxj16!+JP`_foVaM2bsi$&+rXeTJ=U*4ZK!#x@R}
zB_G@Hm5DhX+M4fv#t%A};^55)3V^)3!KL%U)eA2FoTHsl9~Xh=dWbQ=Py_b$jnPy8
zADy7}o)|)4LfMBOF&@=!`kt_*WwO=#wB)OPIJ&5&H9R`UM)76=4l?)oA*xx5>BZ=U
znuT98gt0k?HV%c%W(uen*ut}iRaWggO2D_rAsSc;5Ve-;BWK$SNnG>YxcWMPanLjs
zWq_qI=o7u$rV&}R>~(`55fb$9NQF8VOL5gtq#&wFkyK3zrn(g2Or@At$fN*jOL6kK
zlohNLm$$Bz6{8em(Y_S(4+AM12&_2OarmUDP_Y#E;fa)ubfs(xQHsk+UFs9~q=<AT
zbqhWz7Q5$Cd*PG137^yqd{T$tlllfesW0JMb-t_cN$r78%2qQ;eFmS@HTa}Hg-_}(
zd{Qg$Nln2gMUR+E-G@(V8a}D7;FJ0sKB@2Ele!Gwn)6+UPwEPMQZ@LbzJ*Wf3;3kI
zhEM7kd?-3MeQK=KuVK_LCSfeSzd|YR11P2LKq++_N~u#&O5K2R-6_vQDRmM`l;`;Z
zPI(hJrS8EgH3p~DX*f}S%u6C!>Q`{;7ZY$wjYEl|Uw-!|nB`A@2UI^e0jShDK&3tg
z^qGSm22|<{pi*bSl-dubR2@u|W%D<javp_K>M@j3voJ~>fl=xrj3^HNCx=nagAmI3
z?l%yTcD_IT1!UoV17*`G4?v0Xfcyd{+7f#UXz6|qYN3B7y&9T6-~y~t4`G#h1gq2~
zSfzf16=kRQ2e5MPgSGkZ=*-BD`MK8I2?iCL$6G(NMz_msu8t=YXFeK=vp9(+x3zfg
wb~}o)sjM>mU!~HkR9+2L-Vg1JXHSif&5k7}`8yl$-aazSg@O@EYjZUEA5AR9u>b%7

literal 0
HcmV?d00001

diff --git a/docs/chapter5/code/tokenizer.py b/docs/chapter5/code/tokenizer.py
new file mode 100644
index 0000000..e2fe5b3
--- /dev/null
+++ b/docs/chapter5/code/tokenizer.py
@@ -0,0 +1,68 @@
+import os
+import struct
+from sentencepiece import SentencePieceProcessor
+from typing import List
+
+TOKENIZER_MODEL = "./data/tok4096.model"
+
+class Tokenizer:
+    def __init__(self, tokenizer_model=None):
+        """
+        初始化分词器。加载预训练的SentencePiece模型，并设置一些特殊的token ID。
+
+        参数:
+        tokenizer_model: str, 可选，分词器模型的路径，如果不指定则使用默认路径 TOKENIZER_MODEL。
+        """
+        # 如果提供了分词器模型路径，使用该路径；否则使用默认模型路径
+        model_path = tokenizer_model if tokenizer_model else TOKENIZER_MODEL
+        # 确保模型文件存在
+        assert os.path.isfile(model_path), model_path
+
+        # 加载 SentencePiece 模型
+        self.sp_model = SentencePieceProcessor(model_file=model_path)
+        self.model_path = model_path
+
+        # 获取分词器的特殊token和词汇表大小
+        self.n_words: int = self.sp_model.vocab_size()  # 词汇表大小
+        self.bos_id: int = self.sp_model.bos_id()       # 句子开头 (BOS) 的ID
+        self.eos_id: int = self.sp_model.eos_id()       # 句子结尾 (EOS) 的ID
+        self.pad_id: int = self.sp_model.pad_id()       # 填充 (PAD) 的ID
+
+        # 验证分词器词汇表大小是否正确
+        assert self.sp_model.vocab_size() == self.sp_model.get_piece_size()
+
+    def encode(self, s: str, bos: bool, eos: bool) -> List[int]:
+        """
+        将字符串编码为词元ID列表。可以选择是否添加句子开头 (BOS) 和句子结尾 (EOS) 标记。
+
+        参数:
+        s: str, 要编码的字符串。
+        bos: bool, 是否在编码的词元列表前添加 BOS 标记。
+        eos: bool, 是否在编码的词元列表末尾添加 EOS 标记。
+
+        返回:
+        List[int]: 编码后的词元ID列表。
+        """
+        # 确保输入是字符串类型
+        assert type(s) is str
+        # 使用SentencePiece将字符串编码为词元ID
+        t = self.sp_model.encode(s)
+        # 如果需要BOS标记，将其添加到词元列表开头
+        if bos:
+            t = [self.bos_id] + t
+        # 如果需要EOS标记，将其添加到词元列表末尾
+        if eos:
+            t = t + [self.eos_id]
+        return t
+
+    def decode(self, t: List[int]) -> str:
+        """
+        将词元ID列表解码为字符串。
+
+        参数:
+        t: List[int], 词元ID列表。
+
+        返回:
+        str: 解码后的字符串。
+        """
+        return self.sp_model.decode(t)
\ No newline at end of file
diff --git a/docs/chapter5/code/train.py b/docs/chapter5/code/train.py
new file mode 100644
index 0000000..6c63d6c
--- /dev/null
+++ b/docs/chapter5/code/train.py
@@ -0,0 +1,257 @@
+import math
+import os
+import time
+from contextlib import nullcontext
+from datetime import datetime
+from functools import partial
+
+import torch
+from model import Transformer, ModelArgs
+from preprocess import Task
+
+# -----------------------------------------------------------------------------
+# I/O 配置，用于定义输出目录和训练时的日志记录与评估设置
+out_dir = "output"  # 模型输出保存路径
+eval_interval = 2000  # 评估间隔步数
+log_interval = 1  # 日志记录间隔步数
+eval_iters = 100  # 每次评估时迭代的步数
+eval_only = False  # 如果为True，脚本在第一次评估后立即退出
+always_save_checkpoint = False  # 如果为True，在每次评估后总是保存检查点
+init_from = "scratch"  # 可以选择从头开始训练（'scratch'）或从已有的检查点恢复（'resume'）
+
+# 数据配置
+batch_size = 8  # 每个微批次的样本数量，如果使用梯度累积，实际批次大小将更大
+max_seq_len = 256  # 最大序列长度
+vocab_size = 4096  # 自定义词汇表大小
+
+# 模型配置
+dim = 288  # 模型的隐藏层维度
+n_layers = 8  # Transformer的层数
+n_heads = 8  # 注意力头的数量
+n_kv_heads = 4  # 模型分组
+multiple_of = 32  # 在某些层的维度必须是该数的倍数
+dropout = 0.0  # Dropout概率
+
+# AdamW优化器配置
+gradient_accumulation_steps = 4  # 梯度累积步数，用于模拟更大的批次
+learning_rate = 5e-4  # 最大学习率
+max_iters = 100000  # 总的训练迭代次数
+weight_decay = 1e-1  # 权重衰减系数
+beta1 = 0.9  # AdamW优化器的β1参数
+beta2 = 0.95  # AdamW优化器的β2参数
+grad_clip = 1.0  # 梯度裁剪阈值，0表示不裁剪
+
+# 学习率衰减配置
+decay_lr = True  # 是否启用学习率衰减
+warmup_iters = 1000  # 学习率预热的步数
+
+# 系统设置
+device = "cuda:0"  # 设备选择：'cpu'，'cuda'，'cuda:0'等
+dtype = "bfloat16"  # 数据类型：'float32'，'bfloat16'，'float16'
+
+# -----------------------------------------------------------------------------
+# 获取配置参数的键值对，便于后续的日志记录
+config_keys = [
+    k
+    for k, v in globals().items()
+    if not k.startswith("_") and isinstance(v, (int, float, bool, str))
+]
+config = {k: globals()[k] for k in config_keys}  # 保存配置到字典中，便于日志记录
+# -----------------------------------------------------------------------------
+
+# 固定一些超参数的默认值
+lr_decay_iters = max_iters  # 学习率衰减步数，设置为等于最大迭代步数
+min_lr = 0.0  # 最小学习率，建议为学习率的十分之一
+vocab_source = 'custom'  # 词汇表来源
+master_process = True  # 用于区分主进程
+seed_offset = 0  # 随机种子偏移量
+ddp_world_size = 1  # 分布式数据并行的世界大小
+tokens_per_iter = batch_size * max_seq_len  # 每次迭代处理的token数
+
+# 设置随机种子，确保可重复性
+torch.manual_seed(1337 + seed_offset)
+torch.backends.cuda.matmul.allow_tf32 = True  # 允许在matmul上使用tf32
+torch.backends.cudnn.allow_tf32 = True  # 允许在cudnn上使用tf32
+device_type = "cuda" if "cuda" in device else "cpu"  # 用于自动选择设备类型
+ptdtype = torch.float16  # 设置训练时使用的数据类型
+
+# 混合精度训练相关
+ctx = (
+    nullcontext()
+    if device_type == "cpu"
+    else torch.amp.autocast(device_type=device_type, dtype=ptdtype)
+)
+
+# 为特定任务设置批次迭代器 iter_batches
+iter_batches = partial(
+    Task.iter_batches,  # 调用 Task 类中的 iter_batches 方法
+    batch_size=batch_size,  # 每个批次的样本数量
+    max_seq_len=max_seq_len,  # 每个序列的最大长度
+    vocab_size=vocab_size,  # 词汇表大小
+    vocab_source=vocab_source,  # 词汇表来源（如 llama2 或 custom）
+    device=device,  # 运行模型的设备（如 GPU 或 CPU）
+    num_workers=0,  # 用于数据加载的 worker 数量，0 表示在主线程中加载
+)
+
+# 训练迭代数初始化
+iter_num = 0  # 记录当前迭代数
+
+# 验证集上的最好损失初始值设置为一个极大值，用于后续模型验证时对比更新
+best_val_loss = 1e9  # 设置初始的最佳验证损失为非常大的值，以便在训练中更新
+
+# 模型初始化参数设置
+model_args = dict(
+    dim=dim,  # 模型的隐藏层维度
+    n_layers=n_layers,  # Transformer 的层数
+    n_heads=n_heads,  # 多头注意力机制中的头数
+    n_kv_heads=n_kv_heads,  # 分组数（可能是用于并行化或其他优化目的）
+    vocab_size=vocab_size,  # 词汇表大小
+    multiple_of=multiple_of,  # 用于调整某些维度的参数，确保其为特定数的倍数
+    max_seq_len=max_seq_len,  # 最大序列长度
+    dropout=dropout,  # dropout 概率，用于防止过拟合
+)
+
+# ===========================================================
+# 模型初始化
+gptconf = ModelArgs(**model_args)
+model = Transformer(gptconf)
+
+
+model.to(device)
+
+# 初始化 GradScaler，用于自动混合精度训练（AMP）
+# 如果 enabled=False，表示禁用混合精度，scaler 将不起作用
+scaler = torch.cuda.amp.GradScaler(enabled=(dtype == "float16"))
+
+# 优化器初始化，调用模型的 configure_optimizers 方法
+optimizer = model.configure_optimizers(
+    weight_decay,  # 权重衰减（L2 正则化）
+    learning_rate,  # 学习率
+    (beta1, beta2),  # Adam 优化器中的 beta1 和 beta2 参数
+    device_type  # 当前训练设备（如 GPU 或 CPU）
+)
+
+# 定义评估损失的流程
+@torch.no_grad()  # 使用 no_grad 装饰器，确保在评估过程中不计算梯度，从而节省内存
+def estimate_loss():
+    out = {}  # 用于存储训练集和验证集上的平均损失
+    model.eval()  # 将模型设置为评估模式，这会影响 dropout 和 batchnorm 等层的行为
+    for split in ["train", "val"]:  # 分别对训练集和验证集进行评估
+        batch_iter = iter_batches(split=split)  # 获取对应数据集的批次迭代器
+        losses = torch.zeros(eval_iters)  # 初始化一个张量用于存储多次迭代的损失，放在 CPU 上
+        for k in range(eval_iters):  # 进行多次迭代以计算平均损失
+            X, Y = next(batch_iter)  # 从迭代器中获取下一个批次的输入数据 X 和标签 Y
+            with ctx:  # 上下文管理器，可以是 torch.autocast()，用于自动混合精度训练
+                logits = model(X, Y)  # 前向传播，计算模型的输出
+                loss = raw_model.last_loss  # 从模型中获取损失值
+            losses[k] = loss.item()  # 将损失值转换为 Python 标量并存储在 losses 张量中
+        out[split] = losses.mean()  # 计算当前数据集上的平均损失并保存到字典中
+    model.train()  # 恢复模型为训练模式
+    return out  # 返回包含训练集和验证集平均损失的字典
+
+# 定义学习率调度函数
+def get_lr(it):
+    """
+    根据当前的训练迭代步数 it 返回当前的学习率值。
+    学习率调整策略包括线性预热、余弦退火和最小学习率限制。
+    """
+    # 1) 线性预热阶段，在 warmup_iters 之前，学习率线性增加到目标学习率
+    if it < warmup_iters:
+        return learning_rate * it / warmup_iters  # 预热阶段，学习率线性增长
+
+    # 2) 如果迭代步数超过 lr_decay_iters，返回最小学习率 min_lr
+    if it > lr_decay_iters:
+        return min_lr  # 训练进入尾声时，学习率达到最小值并保持不变
+
+    # 3) 余弦退火阶段，在 warmup_iters 和 lr_decay_iters 之间，学习率逐渐降低
+    decay_ratio = (it - warmup_iters) / (lr_decay_iters - warmup_iters)
+    assert 0 <= decay_ratio <= 1  # 确保衰减比在合法范围内
+    coeff = 0.5 * (1.0 + math.cos(math.pi * decay_ratio))  # 余弦函数计算衰减系数，范围为0到1
+    return min_lr + coeff * (learning_rate - min_lr)  # 根据衰减系数调整学习率
+
+# 初始化训练数据的迭代器
+train_batch_iter = iter_batches(split="train")
+X, Y = next(train_batch_iter)  # 获取第一个批次的数据
+t0 = time.time()  # 记录开始时间
+local_iter_num = 0  # 本进程中的迭代次数
+raw_model = model  # 如果使用了分布式数据并行 (DDP)，需要解包模型
+running_mfu = -1.0  # 初始化模型浮点运算利用率
+
+os.makedirs(out_dir, exist_ok=True)
+
+while True:
+    # 或许当前step的学习率
+    lr = get_lr(iter_num) if decay_lr else learning_rate
+    # 更新优化器中的学习率
+    for param_group in optimizer.param_groups:
+        param_group["lr"] = lr
+
+    # 在指定的评估间隔进行模型评估和保存检查点
+    if iter_num % eval_interval == 0 and master_process:
+        losses = estimate_loss()  # 评估当前模型在训练集和验证集上的损失
+        print(f"step {iter_num}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}")
+
+        # 如果验证损失降低，或者设置为始终保存检查点，则保存模型
+        if losses["val"] < best_val_loss or always_save_checkpoint:
+            best_val_loss = losses["val"]
+            if iter_num > 0:
+                # 创建检查点字典，包含模型状态、优化器状态和其他信息
+                checkpoint = {
+                    "model": raw_model.state_dict(),
+                    "optimizer": optimizer.state_dict(),
+                    "model_args": model_args,
+                    "iter_num": iter_num,
+                    "best_val_loss": best_val_loss,
+                    "config": config,
+                }
+                print(f"saving checkpoint to {out_dir}")
+                # 保存检查点到指定目录
+                torch.save(checkpoint, os.path.join(out_dir, "ckpt.pt"))
+    # 如果只进行评估且已经完成第一次迭代，则退出循环
+    if iter_num == 0 and eval_only:
+        break
+
+    # 前向和反向传播过程，支持梯度累积
+    for micro_step in range(gradient_accumulation_steps):
+
+        with ctx:  # 混合精度训练的上下文管理器
+            logits = model(X, Y)  # 前向传播，计算模型输出
+            loss = raw_model.last_loss  # 获取模型的损失值
+            loss = loss / gradient_accumulation_steps  # 平均损失以支持梯度累积
+
+        X, Y = next(train_batch_iter)  # 获取下一个批次的数据
+        # 反向传播，计算梯度
+        scaler.scale(loss).backward()
+    # 梯度处理阶段
+    if grad_clip != 0.0:
+        # 取消梯度缩放以进行梯度裁剪
+        scaler.unscale_(optimizer)
+        # 对梯度进行裁剪，防止梯度爆炸
+        torch.nn.utils.clip_grad_norm_(model.parameters(), grad_clip)
+    # 更新优化器和梯度缩放器（用于混合精度训练）
+    scaler.step(optimizer)
+    scaler.update()
+    # 清空优化器的梯度，释放显存
+    optimizer.zero_grad(set_to_none=True)
+
+    # 计时和日志记录
+    t1 = time.time()
+    dt = t1 - t0  # 计算一次迭代所需时间
+    t0 = t1
+    if iter_num % log_interval == 0 and master_process:
+        # 获取当前损失值，并根据梯度累积步骤进行调整
+        lossf = loss.item() * gradient_accumulation_steps
+        if local_iter_num >= 5:  # 让训练循环先运行几个迭代再计算模型利用率
+            mfu = raw_model.estimate_mfu(batch_size * gradient_accumulation_steps, dt)
+            # 使用滑动平均更新模型浮点运算利用率（MFU）
+            running_mfu = mfu if running_mfu == -1.0 else 0.9 * running_mfu + 0.1 * mfu
+        print(
+            f"{iter_num} | loss {lossf:.4f} | lr {lr:e} | {dt*1000:.2f}ms | mfu {running_mfu*100:.2f}%"
+            # mfu 表示模型浮点运算利用率
+        )
+    iter_num += 1  # 全局迭代次数自增
+    local_iter_num += 1  # 本地迭代次数自增
+
+    # 终止条件，达到最大迭代次数则退出循环
+    if iter_num > max_iters:
+        break
\ No newline at end of file
diff --git a/docs/chapter5/code/train_vocab.py b/docs/chapter5/code/train_vocab.py
new file mode 100644
index 0000000..b260e19
--- /dev/null
+++ b/docs/chapter5/code/train_vocab.py
@@ -0,0 +1,147 @@
+import glob
+import json
+import os
+from tqdm import tqdm
+import requests
+import sentencepiece as spm
+import argparse
+
+DATA_CACHE_DIR = 'data'
+
+def download_file(url: str, fname: str, chunk_size=1024):
+    """发送HTTP GET请求以流式方式获取文件"""
+    resp = requests.get(url, stream=True)
+    
+    # 获取文件的总大小（以字节为单位），默认为0如果没有提供'content-length'头信息
+    total = int(resp.headers.get("content-length", 0))
+    
+    # 以写二进制模式打开一个文件以保存下载的内容
+    with open(fname, "wb") as file, tqdm(
+        desc=fname,           # 进度条前面的描述信息（通常是文件名）
+        total=total,          # 总的字节数，用于设置进度条的总长度
+        unit="iB",            # 进度条的单位，'iB'代表二进制字节
+        unit_scale=True,      # 启用单位缩放，如KB、MB等
+        unit_divisor=1024,    # 设置单位换算的除数，这里为1024
+    ) as bar:
+        # 逐块读取响应内容并写入文件
+        for data in resp.iter_content(chunk_size=chunk_size):
+            size = file.write(data)  # 写入数据块到文件
+            bar.update(size)         # 更新进度条
+
+def download():
+    """在DATA_CACHE_DIR中创建目录，如果目录不存在则创建"""
+    os.makedirs(DATA_CACHE_DIR, exist_ok=True)
+
+    # 定义TinyStories数据集的下载URL和保存的文件名
+    data_url = "https://www.modelscope.cn/datasets/AI-ModelScope/TinyStories/resolve/master/TinyStories_all_data.tar.gz"
+    data_filename = os.path.join(DATA_CACHE_DIR, "TinyStories_all_data.tar.gz")
+    
+    # 检查数据集是否已经下载，如果没有下载则进行下载
+    if not os.path.exists(data_filename):
+        print(f"Downloading {data_url} to {data_filename}...")
+        download_file(data_url, data_filename)  # 使用之前定义的download_file函数进行下载
+    else:
+        print(f"{data_filename} already exists, skipping download...")
+
+    # 定义解压缩后的数据目录
+    data_dir = os.path.join(DATA_CACHE_DIR, "TinyStories_all_data")
+    
+    # 检查数据目录是否存在，如果不存在则解压缩数据集
+    if not os.path.exists(data_dir):
+        os.makedirs(data_dir, exist_ok=True)  # 创建数据目录
+        print(f"Unpacking {data_filename}...")
+        os.system(f"tar -xzf {data_filename} -C {data_dir}")  # 使用系统命令解压缩.tar.gz文件
+    else:
+        print(f"{data_dir} already exists, skipping unpacking...")
+
+    # 查找解压后的所有JSON文件，排序后获取文件名列表
+    shard_filenames = sorted(glob.glob(os.path.join(data_dir, "*.json")))
+    
+    # 打开第一个JSON文件并读取内容
+    with open(shard_filenames[0], "r") as f:
+        data = json.load(f)  # 将JSON文件内容加载到变量data中
+    
+    print("Download done.")  # 下载完成信息
+    print(f"Number of shards: {len(shard_filenames)}")  # 打印解压后数据分片的数量
+    print(f"Example story:\n{data[0]}")  # 打印第一个分片中的一个示例故事
+
+def load_text_from_files(path):
+    path_list = glob.glob(path)
+    text_data = []
+    for file_path in path_list:
+        with open(file_path, 'r', encoding='utf-8') as file:
+            text_data.extend(file.readlines())
+    return text_data
+
+def batch_iterator(text_data, batch_size=648):
+    for i in range(0, len(text_data), batch_size):
+        yield text_data[i:i + batch_size]
+
+def train_vocab(vocab_size: int=32000, num_shards: int=20):
+    """
+    vocab_size: int, 词汇表的大小，决定分词器的词汇量。
+    num_shards: int, 用于加快词汇表训练的效率，指定要处理的分片数量。
+    """
+    # 确保词汇表大小为正数
+    assert vocab_size > 0, "Vocab size must be positive"
+
+    # SentencePiece 模型的前缀路径，将用于保存分词器
+    prefix = os.path.join(DATA_CACHE_DIR, f"tok{vocab_size}")
+
+    # 1) 将多个分片中的文本导出为单个文本文件 tiny.txt
+    tiny_file = os.path.join(DATA_CACHE_DIR, "tiny.txt")
+    data_dir = os.path.join(DATA_CACHE_DIR, "TinyStories_all_data")
+    shard_filenames = sorted(glob.glob(os.path.join(data_dir, "*.json")))
+
+    # 创建 tiny.txt 文件并写入指定数量的分片中的文本
+    print(f"Writing temporary file {tiny_file} with {num_shards} shards...")
+    with open(tiny_file, "w", encoding="utf-8") as of:
+        # 遍历前 num_shards 个分片
+        for shard in tqdm(shard_filenames[:num_shards]):
+            with open(shard, "r") as f:
+                data = json.load(f)  # 读取分片中的JSON数据
+            # 遍历每个例子，将其中的故事文本写入 tiny.txt 文件
+            for example in data:
+                text = example["story"]
+                text = text.strip()  # 去除文本首尾的空白字符
+                of.write(text + "\n")  # 每个文本写入一行
+
+    # 输出生成的 tiny.txt 文件的大小
+    print(f"Size is: {os.path.getsize(tiny_file) / 1024 / 1024:.2f} MB")
+
+    # 2) 使用 SentencePiece 训练分词器
+    print("Will now train the vocab...")
+    spm.SentencePieceTrainer.train(
+        input=tiny_file,         # 输入文件为之前生成的 tiny.txt
+        model_prefix=prefix,     # 模型前缀路径
+        model_type="bpe",        # 使用 Byte-Pair Encoding (BPE) 训练分词器
+        vocab_size=vocab_size,   # 词汇表大小
+        self_test_sample_size=0, # 自测样本大小设置为 0
+        input_format="text",     # 输入文件格式为纯文本
+        character_coverage=1.0,  # 覆盖所有字符（包括非常见字符）
+        num_threads=os.cpu_count(),  # 使用 CPU 的线程数
+        split_digits=True,       # 拆分数字
+        allow_whitespace_only_pieces=True,  # 允许仅由空格组成的词元
+        byte_fallback=True,      # 启用字节级回退
+        unk_surface=r" \342\201\207 ",  # UNK token 表示未知字符的方式
+        normalization_rule_name="identity"  # 使用“identity”归一化规则
+    )
+
+    # 3) 可选的清理操作，询问用户是否删除临时文件 tiny.txt
+    dec = input(f"Delete the temporary file {tiny_file}? [y/N] ")
+    if dec.lower() == "y":
+        os.remove(tiny_file)  # 删除临时文件
+        print(f"Deleted {tiny_file}")
+
+    # 输出模型保存的路径
+    print(f"Trained tokenizer is in {prefix}.model")
+    print("Done.")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--download", type=bool, default=True, help="download the dataset")
+    parser.add_argument("--vocab_size", type=int, default=4096, help="vocab size")
+    args = parser.parse_args()
+    if args.download:
+        download()
+    train_vocab(args.vocab_size)
\ No newline at end of file
diff --git a/docs/chapter5/llama2.ipynb b/docs/chapter5/llama2.ipynb
deleted file mode 100644
index 9867524..0000000
--- a/docs/chapter5/llama2.ipynb
+++ /dev/null
@@ -1,715 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import math\n",
-    "import struct\n",
-    "import inspect\n",
-    "from dataclasses import dataclass\n",
-    "from typing import Any, Optional, Tuple\n",
-    "\n",
-    "import numpy as np\n",
-    "import torch\n",
-    "import torch.nn.functional as F\n",
-    "from torch import nn"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "class ModelArgs:\n",
-    "    # 自定义超参数\n",
-    "    dim: int = 288  # 模型维度\n",
-    "    n_layers: int = 6  # Transformer层数\n",
-    "    n_heads: int = 6  # 注意力机制的头数\n",
-    "    n_kv_heads: Optional[int] = 6  # 键/值头数，如果未指定，则默认为n_heads\n",
-    "    vocab_size: int = 32000  # 词汇表大小\n",
-    "    hidden_dim: Optional[int] = None  # 隐藏层维度，如果未指定，则使用其他规则确定\n",
-    "    multiple_of: int = 32  # MLP隐藏层大小是这个数的倍数\n",
-    "    norm_eps: float = 1e-5  # 归一化层的epsilon值\n",
-    "    max_seq_len: int = 256  # 最大序列长度\n",
-    "    dropout: float = 0.0  # 丢弃率\n",
-    "\n",
-    "args = ModelArgs()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "class LLaMA2RMSNorm(nn.Module):\n",
-    "    def __init__(self, dim: int, eps: float):\n",
-    "        super().__init__()\n",
-    "        # eps是为了防止除以0的情况\n",
-    "        self.eps = eps\n",
-    "        # weight是一个可学习的参数，全部初始化为1\n",
-    "        self.weight = nn.Parameter(torch.ones(dim))\n",
-    "\n",
-    "    def _norm(self, x):\n",
-    "        # 计算RMSNorm的核心部分\n",
-    "        # x.pow(2).mean(-1, keepdim=True)计算了输入x的平方的均值\n",
-    "        # torch.rsqrt是平方根的倒数，这样就得到了RMSNorm的分母部分，再加上eps防止分母为0\n",
-    "        # 最后乘以x，得到RMSNorm的结果\n",
-    "        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)\n",
-    "\n",
-    "    def forward(self, x):\n",
-    "        # forward函数是模型的前向传播\n",
-    "        # 首先将输入x转为float类型，然后进行RMSNorm，最后再转回原来的数据类型\n",
-    "        # 最后乘以weight，这是RMSNorm的一个可学习的缩放因子\n",
-    "        output = self._norm(x.float()).type_as(x)\n",
-    "        return output * self.weight"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "torch.Size([1, 50, 288])\n"
-     ]
-    }
-   ],
-   "source": [
-    "norm = LLaMA2RMSNorm(args.dim, args.norm_eps)\n",
-    "x = torch.randn(1, 50, args.dim)\n",
-    "output = norm(x)\n",
-    "print(output.shape)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 获得旋转嵌入的实部和虚部\n",
-    "# 注意：此处的dim应为 dim//n_head，因为我们是对每个head进行旋转嵌入\n",
-    "def precompute_freqs_cis(dim: int, end: int, theta: float = 10000.0):\n",
-    "    # torch.arange(0, dim, 2)[: (dim // 2)].float()生成了一个从0开始，步长为2的序列，长度为dim的一半\n",
-    "    # 然后每个元素除以dim，再取theta的倒数，得到频率\n",
-    "    freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim))\n",
-    "    # 生成一个从0到end的序列，长度为end\n",
-    "    t = torch.arange(end, device=freqs.device)\n",
-    "    # 计算外积，得到一个二维矩阵，每一行是t的元素乘以freqs的元素\n",
-    "    freqs = torch.outer(t, freqs).float()\n",
-    "    # 计算频率的余弦值，得到实部\n",
-    "    freqs_cos = torch.cos(freqs)\n",
-    "    # 计算频率的正弦值，得到虚部\n",
-    "    freqs_sin = torch.sin(freqs)\n",
-    "    return freqs_cos, freqs_sin"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "torch.Size([50, 24]) torch.Size([50, 24])\n"
-     ]
-    }
-   ],
-   "source": [
-    "x = torch.randn(1, 50, 288)\n",
-    "freqs_cos, freqs_sin = precompute_freqs_cis(288//6, 50)\n",
-    "print(freqs_cos.shape, freqs_sin.shape)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 此函数的作用是将freqs_cis调整为与x的形状相同，以便能够与x进行广播操作\n",
-    "def reshape_for_broadcast(freqs_cis: torch.Tensor, x: torch.Tensor):\n",
-    "    # 获取x的维度数\n",
-    "    ndim = x.ndim\n",
-    "    # 断言，确保1在x的维度范围内\n",
-    "    assert 0 <= 1 < ndim\n",
-    "    # 断言，确保freqs_cis的形状与x的第二维和最后一维相同\n",
-    "    assert freqs_cis.shape == (x.shape[1], x.shape[-1])\n",
-    "    # 构造一个新的形状，除了第二维和最后一维，其他维度都为1，这样做是为了能够将freqs_cis与x进行广播操作\n",
-    "    shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)]\n",
-    "    # 将freqs_cis调整为新的形状，并返回\n",
-    "    return freqs_cis.view(shape)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def apply_rotary_emb(\n",
-    "    xq: torch.Tensor,\n",
-    "    xk: torch.Tensor,\n",
-    "    freqs_cos: torch.Tensor,\n",
-    "    freqs_sin: torch.Tensor\n",
-    ") -> Tuple[torch.Tensor, torch.Tensor]:\n",
-    "\n",
-    "    # 将查询和键张量转换为浮点数，并重塑形状以分离实部和虚部\n",
-    "    xq_r, xq_i = xq.float().reshape(xq.shape[:-1] + (-1, 2)).unbind(-1)\n",
-    "    xk_r, xk_i = xk.float().reshape(xk.shape[:-1] + (-1, 2)).unbind(-1)\n",
-    "\n",
-    "    # 重新塑形频率张量以进行广播\n",
-    "    freqs_cos = reshape_for_broadcast(freqs_cos, xq_r)\n",
-    "    freqs_sin = reshape_for_broadcast(freqs_sin, xq_r)\n",
-    "\n",
-    "    # 应用旋转，分别计算旋转后的实部和虚部\n",
-    "    xq_out_r = xq_r * freqs_cos - xq_i * freqs_sin\n",
-    "    xq_out_i = xq_r * freqs_sin + xq_i * freqs_cos\n",
-    "    xk_out_r = xk_r * freqs_cos - xk_i * freqs_sin\n",
-    "    xk_out_i = xk_r * freqs_sin + xk_i * freqs_cos\n",
-    "\n",
-    "    # 将最后两个维度合并，并还原为原始张量的形状\n",
-    "    xq_out = torch.stack([xq_out_r, xq_out_i], dim=-1).flatten(3)\n",
-    "    xk_out = torch.stack([xk_out_r, xk_out_i], dim=-1).flatten(3)\n",
-    "\n",
-    "    return xq_out.type_as(xq), xk_out.type_as(xk)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "torch.Size([50, 24]) torch.Size([50, 24])\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "(torch.Size([1, 50, 6, 48]), torch.Size([1, 50, 6, 48]))"
-      ]
-     },
-     "execution_count": 9,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "xq = torch.randn(1, 50, 6, 48) # bs, seq_len, dim//n_head, n_head_dim\n",
-    "xk = torch.randn(1, 50, 6, 48) # bs, seq_len, dim//n_head, n_head_dim\n",
-    "\n",
-    "# 使用 precompute_freqs_cis 函数获取 sin和cos\n",
-    "cos, sin = precompute_freqs_cis(288//6, 50)\n",
-    "print(cos.shape, sin.shape)\n",
-    "xq_out, xk_out = apply_rotary_emb(xq, xk, cos, sin)\n",
-    "\n",
-    "xq_out.shape, xk_out.shape"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def repeat_kv(x: torch.Tensor, n_rep: int) -> torch.Tensor:\n",
-    "    # 获取输入张量的形状：批量大小、序列长度、键/值对头的数量、每个头的维度大小\n",
-    "    bs, slen, n_kv_heads, head_dim = x.shape\n",
-    "    \n",
-    "    # 如果重复次数为1，则不需要重复，直接返回原始张量\n",
-    "    if n_rep == 1:\n",
-    "        return x\n",
-    "    \n",
-    "    # 对张量进行扩展和重塑操作以重复键值对\n",
-    "    return (\n",
-    "        x[:, :, :, None, :]  # 在第四个维度（头的维度前）添加一个新的维度\n",
-    "        .expand(bs, slen, n_kv_heads, n_rep, head_dim)  # 将新添加的维度扩展到n_rep大小，实现重复的效果\n",
-    "        .reshape(bs, slen, n_kv_heads * n_rep, head_dim)  # 重新塑形，合并键/值对头的数量和重复次数的维度\n",
-    "    )\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "class LLaMA2Attention(nn.Module):\n",
-    "    def __init__(self, args: ModelArgs):\n",
-    "        super().__init__()\n",
-    "        # 根据是否指定n_kv_heads，确定用于键（key）和值（value）的头的数量。\n",
-    "        self.n_kv_heads = args.n_heads if args.n_kv_heads is None else args.n_kv_heads\n",
-    "        # 确保总头数可以被键值头数整除。\n",
-    "        assert args.n_heads % self.n_kv_heads == 0\n",
-    "\n",
-    "        # 模型并行处理大小，默认为1。\n",
-    "        model_parallel_size = 1\n",
-    "        # 本地计算头数，等于总头数除以模型并行处理大小。\n",
-    "        self.n_local_heads = args.n_heads // model_parallel_size\n",
-    "        # 本地键值头数，等于键值头数除以模型并行处理大小。\n",
-    "        self.n_local_kv_heads = self.n_kv_heads // model_parallel_size\n",
-    "        # 重复次数，用于扩展键和值的尺寸。\n",
-    "        self.n_rep = self.n_local_heads // self.n_local_kv_heads\n",
-    "        # 每个头的维度，等于模型维度除以头的总数。\n",
-    "        self.head_dim = args.dim // args.n_heads\n",
-    "\n",
-    "        # 定义权重矩阵。\n",
-    "        self.wq = nn.Linear(args.dim, args.n_heads * self.head_dim, bias=False)\n",
-    "        self.wk = nn.Linear(args.dim, self.n_kv_heads * self.head_dim, bias=False)\n",
-    "        self.wv = nn.Linear(args.dim, self.n_kv_heads * self.head_dim, bias=False)\n",
-    "        # 输出权重矩阵。\n",
-    "        self.wo = nn.Linear(args.n_heads * self.head_dim, args.dim, bias=False)\n",
-    "\n",
-    "        # 定义dropout。\n",
-    "        self.attn_dropout = nn.Dropout(args.dropout)\n",
-    "        self.resid_dropout = nn.Dropout(args.dropout)\n",
-    "        # 保存dropout概率。\n",
-    "        self.dropout = args.dropout\n",
-    "\n",
-    "        # 检查是否使用Flash Attention（需要PyTorch >= 2.0）。\n",
-    "        self.flash = hasattr(torch.nn.functional, 'scaled_dot_product_attention')\n",
-    "        if not self.flash:\n",
-    "            # 若不支持Flash Attention，则使用手动实现的注意力机制，并设置mask。\n",
-    "            print(\"WARNING: using slow attention. Flash Attention requires PyTorch >= 2.0\")\n",
-    "            # 创建一个上三角矩阵，用于遮蔽未来信息。\n",
-    "            mask = torch.full((1, 1, args.max_seq_len, args.max_seq_len), float(\"-inf\"))\n",
-    "            mask = torch.triu(mask, diagonal=1)\n",
-    "            # 注册为模型的缓冲区\n",
-    "            self.register_buffer(\"mask\", mask)\n",
-    "\n",
-    "    def forward(self, x: torch.Tensor, freqs_cos: torch.Tensor, freqs_sin: torch.Tensor):\n",
-    "        # 获取批次大小和序列长度，[batch_size, seq_len, dim]\n",
-    "        bsz, seqlen, _ = x.shape\n",
-    "\n",
-    "        # 计算查询（Q）、键（K）、值（V）。\n",
-    "        xq, xk, xv = self.wq(x), self.wk(x), self.wv(x)\n",
-    "        # 调整形状以适应头的维度。\n",
-    "        xq = xq.view(bsz, seqlen, self.n_local_heads, self.head_dim)\n",
-    "        xk = xk.view(bsz, seqlen, self.n_local_kv_heads, self.head_dim)\n",
-    "        xv = xv.view(bsz, seqlen, self.n_local_kv_heads, self.head_dim)\n",
-    "\n",
-    "        # 应用旋转位置嵌入（RoPE）。\n",
-    "        xq, xk = apply_rotary_emb(xq, xk, freqs_cos, freqs_sin)\n",
-    "\n",
-    "        # 对键和值进行扩展以适应重复次数。\n",
-    "        xk = repeat_kv(xk, self.n_rep)\n",
-    "        xv = repeat_kv(xv, self.n_rep)\n",
-    "\n",
-    "        # 将头作为批次维度处理。\n",
-    "        xq = xq.transpose(1, 2)\n",
-    "        xk = xk.transpose(1, 2)\n",
-    "        xv = xv.transpose(1, 2)\n",
-    "\n",
-    "        # 根据是否支持Flash Attention，选择实现方式。\n",
-    "        if self.flash:\n",
-    "            # 使用Flash Attention。\n",
-    "            output = torch.nn.functional.scaled_dot_product_attention(xq, xk, xv, attn_mask=None, dropout_p=self.dropout if self.training else 0.0, is_causal=True)\n",
-    "        else:\n",
-    "            # 使用手动实现的注意力机制。\n",
-    "            scores = torch.matmul(xq, xk.transpose(2, 3)) / math.sqrt(self.head_dim)\n",
-    "            assert hasattr(self, 'mask')\n",
-    "            scores = scores + self.mask[:, :, :seqlen, :seqlen]\n",
-    "            scores = F.softmax(scores.float(), dim=-1).type_as(xq)\n",
-    "            scores = self.attn_dropout(scores)\n",
-    "            output = torch.matmul(scores, xv)\n",
-    "\n",
-    "        # 恢复时间维度并合并头。\n",
-    "        output = output.transpose(1, 2).contiguous().view(bsz, seqlen, -1)\n",
-    "\n",
-    "        # 最终投影回残差流。\n",
-    "        output = self.wo(output)\n",
-    "        output = self.resid_dropout(output)\n",
-    "        return output"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 22,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "torch.Size([50, 24]) torch.Size([50, 24])\n",
-      "Output shape: torch.Size([1, 50, 288])\n"
-     ]
-    }
-   ],
-   "source": [
-    "# 创建Attention实例\n",
-    "attention_model = LLaMA2Attention(args)\n",
-    "\n",
-    "# 模拟输入数据\n",
-    "batch_size = 1\n",
-    "seq_len = 50  # 假设实际使用的序列长度为50\n",
-    "dim = args.dim\n",
-    "x = torch.rand(batch_size, seq_len, dim)  # 随机生成输入张量\n",
-    "# freqs_cos = torch.rand(seq_len, dim // 2)  # 模拟cos频率，用于RoPE\n",
-    "# freqs_sin = torch.rand(seq_len, dim // 2)  # 模拟sin频率，用于RoPE\n",
-    "\n",
-    "freqs_cos, freqs_sin = precompute_freqs_cis(dim//args.n_heads, seq_len)\n",
-    "\n",
-    "print(freqs_cos.shape, freqs_sin.shape)\n",
-    "\n",
-    "# 运行Attention模型\n",
-    "output = attention_model(x, freqs_cos, freqs_sin)\n",
-    "\n",
-    "# attention出来之后的形状 依然是[batch_size, seq_len, dim]\n",
-    "print(\"Output shape:\", output.shape)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 16,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "class LLaMA2MLP(nn.Module):\n",
-    "    def __init__(self, dim: int, hidden_dim: int, multiple_of: int, dropout: float):\n",
-    "        super().__init__()\n",
-    "        # 如果没有指定隐藏层的维度，我们将其设置为输入维度的4倍\n",
-    "        # 然后将其减少到2/3，最后确保它是multiple_of的倍数\n",
-    "        if hidden_dim is None:\n",
-    "            hidden_dim = 4 * dim\n",
-    "            hidden_dim = int(2 * hidden_dim / 3)\n",
-    "            hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)\n",
-    "        # 定义第一层线性变换，从输入维度到隐藏维度\n",
-    "        self.w1 = nn.Linear(dim, hidden_dim, bias=False)\n",
-    "        # 定义第二层线性变换，从隐藏维度到输入维度\n",
-    "        self.w2 = nn.Linear(hidden_dim, dim, bias=False)\n",
-    "        # 定义第三层线性变换，从输入维度到隐藏维度\n",
-    "        self.w3 = nn.Linear(dim, hidden_dim, bias=False)\n",
-    "        # 定义dropout层，用于防止过拟合\n",
-    "        self.dropout = nn.Dropout(dropout)\n",
-    "\n",
-    "    def forward(self, x):\n",
-    "        # 前向传播函数\n",
-    "        # 首先，输入x通过第一层线性变换和SILU激活函数\n",
-    "        # 然后，结果乘以输入x通过第三层线性变换的结果\n",
-    "        # 最后，通过第二层线性变换和dropout层\n",
-    "        return self.dropout(self.w2(F.silu(self.w1(x)) * self.w3(x)))"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 19,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "torch.Size([1, 50, 288])\n"
-     ]
-    }
-   ],
-   "source": [
-    "# 创建MLP实例\n",
-    "mlp = LLaMA2MLP(args.dim, args.hidden_dim, args.multiple_of, args.dropout)\n",
-    "# 随机生成数据\n",
-    "x = torch.randn(1, 50, 288)\n",
-    "# 运行MLP模型\n",
-    "output = mlp(x)\n",
-    "print(output.shape)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 21,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "class LLaMA2DecoderLayer(nn.Module):\n",
-    "    def __init__(self, layer_id: int, args: ModelArgs):\n",
-    "        super().__init__()\n",
-    "        # 定义多头注意力的头数\n",
-    "        self.n_heads = args.n_heads\n",
-    "        # 定义输入维度\n",
-    "        self.dim = args.dim\n",
-    "        # 定义每个头的维度，等于输入维度除以头数\n",
-    "        self.head_dim = args.dim // args.n_heads\n",
-    "        # 定义LLaMA2Attention对象，用于进行多头注意力计算\n",
-    "        self.attention = LLaMA2Attention(args)\n",
-    "        # 定义LLaMAMLP对象，用于进行前馈神经网络计算\n",
-    "        self.feed_forward = LLaMA2MLP(\n",
-    "            dim=args.dim,\n",
-    "            hidden_dim=args.hidden_dim,\n",
-    "            multiple_of=args.multiple_of,\n",
-    "            dropout=args.dropout,\n",
-    "        )\n",
-    "        # 定义层的ID\n",
-    "        self.layer_id = layer_id\n",
-    "        # 定义注意力计算的归一化层\n",
-    "        self.attention_norm = LLaMA2RMSNorm(args.dim, eps=args.norm_eps)\n",
-    "        # 定义前馈神经网络计算的归一化层\n",
-    "        self.ffn_norm = LLaMA2RMSNorm(args.dim, eps=args.norm_eps)\n",
-    "\n",
-    "    def forward(self, x, freqs_cos, freqs_sin):\n",
-    "        # 前向传播函数\n",
-    "        # 首先，输入x经过注意力归一化层，然后进行注意力计算，结果与输入x相加得到h\n",
-    "        # 然后，h经过前馈神经网络归一化层，然后进行前馈神经网络计算，结果与h相加得到输出\n",
-    "        h = x + self.attention.forward(self.attention_norm(x), freqs_cos, freqs_sin)\n",
-    "        out = h + self.feed_forward.forward(self.ffn_norm(h))\n",
-    "        return out"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 25,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "torch.Size([1, 50, 288]) torch.Size([50, 24]) torch.Size([50, 24])\n",
-      "torch.Size([1, 50, 288])\n"
-     ]
-    }
-   ],
-   "source": [
-    "# LLaMADecoderLayer.forward 函数的输入是 x, freqs_cos, freqs_sin, 其中x的形状是[batch_size, seq_len, dim]\n",
-    "# 由于llama2使用了GQA Attention，所以precompute_freqs_cis函数输入参数应该为dim//n_heads，seq_len、\n",
-    "\n",
-    "# 创建LLaMADecoderLayer实例\n",
-    "decoderlayer = LLaMA2DecoderLayer(0, args)\n",
-    "\n",
-    "# 模拟输入数据\n",
-    "dim = args.dim\n",
-    "seq_len = 50\n",
-    "\n",
-    "x = torch.randn(1, seq_len, dim) # [bs, seq_len, dim]\n",
-    "\n",
-    "freqs_cos, freqs_sin = precompute_freqs_cis(dim//args.n_heads, seq_len)\n",
-    "print(x.shape, freqs_cos.shape, freqs_sin.shape)\n",
-    "\n",
-    "out = decoderlayer(x, freqs_cos, freqs_sin)\n",
-    "\n",
-    "print(out.shape) # 形状和输入的x一样 [batch_size, seq_len, dim]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 33,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "class LLaMA2Model(nn.Module):\n",
-    "    last_loss: Optional[torch.Tensor]\n",
-    "\n",
-    "    def __init__(self, args: ModelArgs):\n",
-    "        super().__init__()\n",
-    "        # 初始化模型参数\n",
-    "        self.args = args\n",
-    "        # 词汇表大小\n",
-    "        self.vocab_size = args.vocab_size\n",
-    "        # 层数\n",
-    "        self.n_layers = args.n_layers\n",
-    "\n",
-    "        # 词嵌入层\n",
-    "        self.tok_embeddings = nn.Embedding(args.vocab_size, args.dim)\n",
-    "        # Dropout层\n",
-    "        self.dropout = nn.Dropout(args.dropout)\n",
-    "        # Decoder层\n",
-    "        self.layers = torch.nn.ModuleList()\n",
-    "        for layer_id in range(args.n_layers):\n",
-    "            self.layers.append(LLaMADecoderLayer(layer_id, args))\n",
-    "        # 归一化层\n",
-    "        self.norm = LLaMA2RMSNorm(args.dim, eps=args.norm_eps)\n",
-    "        # 输出层\n",
-    "        self.output = nn.Linear(args.dim, args.vocab_size, bias=False)\n",
-    "\n",
-    "        # 将词嵌入层的权重与输出层的权重共享\n",
-    "        self.tok_embeddings.weight = self.output.weight \n",
-    "\n",
-    "        # 预计算相对位置嵌入的频率\n",
-    "        freqs_cos, freqs_sin = precompute_freqs_cis(self.args.dim // self.args.n_heads, self.args.max_seq_len)\n",
-    "        self.register_buffer(\"freqs_cos\", freqs_cos, persistent=False)\n",
-    "        self.register_buffer(\"freqs_sin\", freqs_sin, persistent=False)\n",
-    "\n",
-    "        # 初始化所有权重\n",
-    "        self.apply(self._init_weights)\n",
-    "        # 对残差投影进行特殊的缩放初始化\n",
-    "        for pn, p in self.named_parameters():\n",
-    "            if pn.endswith('w3.weight') or pn.endswith('wo.weight'):\n",
-    "                torch.nn.init.normal_(p, mean=0.0, std=0.02/math.sqrt(2 * args.n_layers))\n",
-    "\n",
-    "        # 初始化最后一次前向传播的损失属性\n",
-    "        self.last_loss = None\n",
-    "\n",
-    "    def _init_weights(self, module):\n",
-    "        # 初始化权重的函数\n",
-    "        if isinstance(module, nn.Linear):\n",
-    "            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)\n",
-    "            if module.bias is not None:\n",
-    "                torch.nn.init.zeros_(module.bias)\n",
-    "        elif isinstance(module, nn.Embedding):\n",
-    "            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)\n",
-    "    \n",
-    "    def forward(self, tokens: torch.Tensor, targets: Optional[torch.Tensor] = None) -> torch.Tensor:\n",
-    "        # 前向传播函数\n",
-    "        _bsz, seqlen = tokens.shape\n",
-    "        # 通过词嵌入层和Dropout层\n",
-    "        h = self.tok_embeddings(tokens)\n",
-    "        h = self.dropout(h)\n",
-    "        # 获取相对位置嵌入的频率\n",
-    "        freqs_cos = self.freqs_cos[:seqlen]\n",
-    "        freqs_sin = self.freqs_sin[:seqlen]\n",
-    "\n",
-    "        # 通过Decoder层\n",
-    "        for layer in self.layers:\n",
-    "            h = layer(h, freqs_cos, freqs_sin)\n",
-    "        # 通过归一化层\n",
-    "        h = self.norm(h)\n",
-    "\n",
-    "        if targets is not None:\n",
-    "            # 如果给定了目标，计算损失\n",
-    "            logits = self.output(h)\n",
-    "            self.last_loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)\n",
-    "        else:\n",
-    "            # 推理时的小优化：只对最后一个位置的输出进行前向传播\n",
-    "            logits = self.output(h[:, [-1], :]) \n",
-    "            self.last_loss = None\n",
-    "\n",
-    "        return logits\n",
-    "    \n",
-    "    def configure_optimizers(self, weight_decay, learning_rate, betas, device_type):\n",
-    "        # 获取所有需要更新的参数\n",
-    "        param_dict = {pn: p for pn, p in self.named_parameters() if p.requires_grad}\n",
-    "        \n",
-    "        # 将参数分为需要权重衰减和不需要权重衰减的两组\n",
-    "        decay_params = [p for n, p in param_dict.items() if p.dim() >= 2]\n",
-    "        nodecay_params = [p for n, p in param_dict.items() if p.dim() < 2]\n",
-    "        optim_groups = [\n",
-    "            {'params': decay_params, 'weight_decay': weight_decay},\n",
-    "            {'params': nodecay_params, 'weight_decay': 0.0}\n",
-    "        ]\n",
-    "        \n",
-    "        # 打印参数数量信息\n",
-    "        num_decay_params = sum(p.numel() for p in decay_params)\n",
-    "        num_nodecay_params = sum(p.numel() for p in nodecay_params)\n",
-    "        print(f\"num decayed parameter tensors: {len(decay_params)}, with {num_decay_params:,} parameters\")\n",
-    "        print(f\"num non-decayed parameter tensors: {len(nodecay_params)}, with {num_nodecay_params:,} parameters\")\n",
-    "        \n",
-    "        # 根据设备类型选择使用标准 AdamW 或其融合版本\n",
-    "        fused_available = 'fused' in inspect.signature(torch.optim.AdamW).parameters\n",
-    "        use_fused = fused_available and device_type == 'cuda'\n",
-    "        extra_args = dict(fused=True) if use_fused else dict()\n",
-    "        optimizer = torch.optim.AdamW(optim_groups, lr=learning_rate, betas=betas, **extra_args)\n",
-    "        print(f\"using fused AdamW: {use_fused}\")\n",
-    "\n",
-    "        return optimizer\n",
-    "    \n",
-    "    def estimate_mfu(self, fwdbwd_per_iter, dt):\n",
-    "        \"\"\" 估计模型的 FLOPs 利用率 (MFU) 单位：A100 bfloat16 的峰值 FLOPS \"\"\"\n",
-    "        # 计算每次迭代的 FLOPs 数量（参考 PaLM 论文的附录 B）\n",
-    "        N = sum(p.numel() for p in self.parameters())\n",
-    "        cfg = self.args\n",
-    "        L, H, Q, T = cfg.n_layers, cfg.n_heads, cfg.dim//cfg.n_heads, cfg.max_seq_len\n",
-    "        flops_per_token = 6*N + 12*L*H*Q*T\n",
-    "        flops_per_fwdbwd = flops_per_token * T\n",
-    "        flops_per_iter = flops_per_fwdbwd * fwdbwd_per_iter\n",
-    "        \n",
-    "        # 将 FLOPs 吞吐量表示为 A100 bfloat16 峰值 FLOPS 的比例\n",
-    "        flops_achieved = flops_per_iter * (1.0/dt) # 每秒计算的 FLOPs\n",
-    "        flops_promised = 312e12 # A100 GPU bfloat16 的峰值 FLOPS 为 312 TFLOPS\n",
-    "        mfu = flops_achieved / flops_promised\n",
-    "        return mfu\n",
-    "    \n",
-    "    @torch.inference_mode()\n",
-    "    def generate(self, idx, max_new_tokens, temperature=1.0, top_k=None):\n",
-    "        \"\"\"\n",
-    "        给定输入序列 idx（形状为 (bz,seq_len) 的长整型张量），通过多次生成新 token 来完成序列。\n",
-    "        在 model.eval() 模式下运行。效率较低的采样版本，没有使用键k/v cache。\n",
-    "        \"\"\"\n",
-    "        for _ in range(max_new_tokens):\n",
-    "            # 如果序列上下文过长，截断它到最大长度\n",
-    "            idx_cond = idx if idx.size(1) <= self.args.max_seq_len else idx[:, -self.args.max_seq_len:]\n",
-    "            \n",
-    "            # 前向传播获取序列中最后一个位置的 logits\n",
-    "            logits = self(idx_cond)\n",
-    "            logits = logits[:, -1, :] # 只保留最后一个时间步的输出\n",
-    "            \n",
-    "            if temperature == 0.0:\n",
-    "                # 选择最有可能的索引\n",
-    "                _, idx_next = torch.topk(logits, k=1, dim=-1)\n",
-    "            else:\n",
-    "                # 缩放 logits 并应用 softmax\n",
-    "                logits = logits / temperature\n",
-    "                if top_k is not None:\n",
-    "                    v, _ = torch.topk(logits, min(top_k, logits.size(-1)))\n",
-    "                    logits[logits < v[:, [-1]]] = -float('Inf')\n",
-    "                probs = F.softmax(logits, dim=-1)\n",
-    "                idx_next = torch.multinomial(probs, num_samples=1)\n",
-    "            \n",
-    "            # 将采样的索引添加到序列中并继续\n",
-    "            idx = torch.cat((idx, idx_next), dim=1)\n",
-    "\n",
-    "        return idx"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 37,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Number of parameters: 15191712\n",
-      "torch.Size([1, 1, 32000])\n"
-     ]
-    }
-   ],
-   "source": [
-    "# LLaMA2Model.forward 接受两个参数，tokens和targets，其中tokens是输入的张量, 应为int类型\n",
-    "x = torch.randint(0, 32000, (1, 50)) # [bs, seq_len]\n",
-    "# 实例化LLaMA2Model\n",
-    "model = LLaMA2Model(args=args)\n",
-    "# 计算model的全部参数\n",
-    "num_params = sum(p.numel() for p in model.parameters())\n",
-    "print('Number of parameters:', num_params)\n",
-    "\n",
-    "out = model(x)\n",
-    "print(out.shape) # [batch_size, 1, vocab_size]"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "nlp",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.13"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}