diff --git a/.gitignore b/.gitignore
index 2616d0f..8174958 100644
--- a/.gitignore
+++ b/.gitignore
@@ -174,3 +174,4 @@ gpt2/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+/Model_Architecture_Discussions/ChatGLM3/weights/
diff --git a/Model_Architecture_Discussions/ChatGLM3/README.md b/Model_Architecture_Discussions/ChatGLM3/README.md
new file mode 100644
index 0000000..c8a8159
--- /dev/null
+++ b/Model_Architecture_Discussions/ChatGLM3/README.md
@@ -0,0 +1,46 @@
+# 动手实现 ChatGLM3-6B
+
+这篇教程是对[ChatGLM3-6B](https://huggingface.co/THUDM/chatglm3-6b)的动手实现版本，目前已脱离transformers依赖work了，对模型架构的讲解估计要咕很久才能写明白。
+
+## 0、模型架构讲解
+
+连载中
+
+## 1、下载模型权重
+
+在实现模型之前，请先下载原始的模型权重，咱们只需要模型仓库里的`safetensors`文件，只有这部分是真正使用到的权重。
+![img.png](img/img.png)
+
+另外还有一个模型是tokenizer模型（`tokenizer.model`），这个俺已经放本仓库里了，不用过去下载了。
+
+就在这目录上运行以下命令，将权重下载到`weights`目录下，后续咱们就可以直接加载了。
+
+```bash
+pip install -U "huggingface_hub[cli]"
+export HF_ENDPOINT=https://hf-mirror.com
+huggingface-cli download THUDM/chatglm3-6b --include "*.safetensors" --local-dir weights
+```
+
+上述指令的第二行是使用[hf-mirror](https://hf-mirror.com)镜像加速下载，针对国内用户，如果你的网络可以直接访问huggingface则可以跳过。
+
+如果是windows用户，这第二行就按shell的类型修改下
+
+```bash
+# 针对powershell
+$env:HF_ENDPOINT = "https://hf-mirror.com"
+# 针对cmd
+set HF_ENDPOINT = "https://hf-mirror.com"
+```
+
+## 2、加载模型权重 & 推理
+
+[加载模型权重](./加载模型权重.ipynb)
+
+## 3、实现各种采样策略
+
+连载中
+
+## 4、实现多轮对话（Chat Format）
+
+连载中
+
diff --git a/Model_Architecture_Discussions/ChatGLM3/configuration_chatglm_full.py b/Model_Architecture_Discussions/ChatGLM3/configuration_chatglm_full.py
new file mode 100644
index 0000000..cbc5627
--- /dev/null
+++ b/Model_Architecture_Discussions/ChatGLM3/configuration_chatglm_full.py
@@ -0,0 +1,43 @@
+import torch
+
+class ChatGLMConfig():
+    model_type = "chatglm"
+    # original_rope = 'https://123.com'
+    attention_softmax_in_fp32 = True
+    max_length = 8196
+
+    multi_query_attention = True
+    multi_query_group_num = 2
+    tie_word_embeddings = False
+    num_layers = 28
+    padded_vocab_size = 65024
+    hidden_size = 4096
+    ffn_hidden_size = 13696
+    kv_channels = 128
+    num_attention_heads = 32
+    seq_length = 2048
+    hidden_dropout = 0.0
+    classifier_dropout = None
+    attention_dropout = 0.0
+    layernorm_epsilon = 1e-5
+    rmsnorm = True
+    apply_residual_connection_post_layernorm = False
+    post_layer_norm = True
+    add_bias_linear = False
+    add_qkv_bias = True
+    bias_dropout_fusion = True
+    apply_query_key_layer_scaling = True
+    attention_softmax_in_fp32 = True
+    fp32_residual_connection = False
+    quantization_bit = 0
+    pre_seq_len = None
+    prefix_projection = False
+
+    torch_dtype = torch.float16
+    vocab_size = 65024
+    pad_token_id = 0
+    eos_token_id = 2
+
+    use_cache = False
+    use_return_dict = True
+    output_hidden_states = False
\ No newline at end of file
diff --git a/Model_Architecture_Discussions/ChatGLM3/glm.py b/Model_Architecture_Discussions/ChatGLM3/glm.py
new file mode 100644
index 0000000..7292f99
--- /dev/null
+++ b/Model_Architecture_Discussions/ChatGLM3/glm.py
@@ -0,0 +1,1172 @@
+""" PyTorch ChatGLM model. """
+
+import logging
+import math
+import sys
+from collections import OrderedDict
+from typing import Optional, Tuple, Union, List
+
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import CrossEntropyLoss, LayerNorm, MSELoss, BCEWithLogitsLoss
+from torch.nn.utils import skip_init
+
+from configuration_chatglm_full import ChatGLMConfig
+
+
+# flags required to enable jit fusion kernels
+
+if sys.platform != 'darwin':
+    torch._C._jit_set_profiling_mode(False)
+    torch._C._jit_set_profiling_executor(False)
+    torch._C._jit_override_can_fuse_on_cpu(True)
+    torch._C._jit_override_can_fuse_on_gpu(True)
+
+logger = logging.getLogger(__name__)
+
+
+class BaseModelOutputWithPast(OrderedDict):
+    def __init__(self, last_hidden_state,
+                 past_key_values,
+                 hidden_states,
+                 attentions):
+        self.last_hidden_state = last_hidden_state
+        self.past_key_values = past_key_values
+        self.hidden_states = hidden_states
+        self.attentions = attentions
+
+    last_hidden_state: torch.FloatTensor = None
+    past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
+    hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
+    attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
+
+
+class CausalLMOutputWithPast(OrderedDict):
+    def __init__(self, loss, logits, past_key_values, hidden_states, attentions):
+        self.loss = loss
+        self.logits = logits
+        self.past_key_values = past_key_values
+        self.hidden_states = hidden_states
+        self.attentions = attentions
+
+    loss: Optional[torch.FloatTensor] = None
+    logits: torch.FloatTensor = None
+    past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
+    hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
+    attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
+
+
+class SequenceClassifierOutputWithPast(OrderedDict):
+    def __init__(self, loss, logits, past_key_values, hidden_states, attentions):
+        self.loss = loss
+        self.logits = logits
+        self.past_key_values = past_key_values
+        self.hidden_states = hidden_states
+        self.attentions = attentions
+
+    loss: Optional[torch.FloatTensor] = None
+    logits: torch.FloatTensor = None
+    past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
+    hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
+    attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
+
+
+def default_init(cls, *args, **kwargs):
+    return cls(*args, **kwargs)
+
+
+class InvalidScoreLogitsProcessor:
+    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
+        if torch.isnan(scores).any() or torch.isinf(scores).any():
+            scores.zero_()
+            scores[..., 5] = 5e4
+        return scores
+
+
+class PrefixEncoder(torch.nn.Module):
+    """
+    The torch.nn model to encode the prefix
+    Input shape: (batch-size, prefix-length)
+    Output shape: (batch-size, prefix-length, 2*layers*hidden)
+    """
+
+    def __init__(self, config: ChatGLMConfig):
+        super().__init__()
+        self.prefix_projection = config.prefix_projection
+        if self.prefix_projection:
+            # Use a two-layer MLP to encode the prefix
+            kv_size = config.num_layers * config.kv_channels * config.multi_query_group_num * 2
+            self.embedding = torch.nn.Embedding(config.pre_seq_len, kv_size)
+            self.trans = torch.nn.Sequential(
+                torch.nn.Linear(kv_size, config.hidden_size),
+                torch.nn.Tanh(),
+                torch.nn.Linear(config.hidden_size, kv_size)
+            )
+        else:
+            self.embedding = torch.nn.Embedding(config.pre_seq_len,
+                                                config.num_layers * config.kv_channels * config.multi_query_group_num * 2)
+
+    def forward(self, prefix: torch.Tensor):
+        if self.prefix_projection:
+            prefix_tokens = self.embedding(prefix)
+            past_key_values = self.trans(prefix_tokens)
+        else:
+            past_key_values = self.embedding(prefix)
+        return past_key_values
+
+
+def split_tensor_along_last_dim(
+        tensor: torch.Tensor,
+        num_partitions: int,
+        contiguous_split_chunks: bool = False,
+) -> List[torch.Tensor]:
+    """Split a tensor along its last dimension.
+
+    Arguments:
+        tensor: input tensor.
+        num_partitions: number of partitions to split the tensor
+        contiguous_split_chunks: If True, make each chunk contiguous
+                                 in memory.
+
+    Returns:
+        A list of Tensors
+    """
+    # Get the size and dimension.
+    last_dim = tensor.dim() - 1
+    last_dim_size = tensor.size()[last_dim] // num_partitions
+    # Split.
+    tensor_list = torch.split(tensor, last_dim_size, dim=last_dim)
+    # Note: torch.split does not create contiguous tensors by default.
+    if contiguous_split_chunks:
+        return tuple(chunk.contiguous() for chunk in tensor_list)
+
+    return tensor_list
+
+
+class RotaryEmbedding(nn.Module):
+    def __init__(self, dim, device=None, dtype=None):
+        super().__init__()
+        inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2, device=device).to(dtype=dtype) / dim))
+        self.register_buffer("inv_freq", inv_freq)
+        self.dim = dim
+
+    def forward_impl(
+            self, seq_len: int, n_elem: int, dtype: torch.dtype, device: torch.device, base: int = 10000
+    ):
+        """Enhanced Transformer with Rotary Position Embedding.
+
+        Derived from: https://github.com/labmlai/annotated_deep_learning_paper_implementations/blob/master/labml_nn/
+        transformers/rope/__init__.py. MIT License:
+        https://github.com/labmlai/annotated_deep_learning_paper_implementations/blob/master/license.
+        """
+        # $\Theta = {\theta_i = 10000^{\frac{2(i-1)}{d}}, i \in [1, 2, ..., \frac{d}{2}]}$
+        theta = 1.0 / (base ** (torch.arange(0, n_elem, 2, dtype=torch.float, device=device) / n_elem))
+
+        # Create position indexes `[0, 1, ..., seq_len - 1]`
+        seq_idx = torch.arange(seq_len, dtype=torch.float, device=device)
+
+        # Calculate the product of position index and $\theta_i$
+        idx_theta = torch.outer(seq_idx, theta).float()
+
+        cache = torch.stack([torch.cos(idx_theta), torch.sin(idx_theta)], dim=-1)
+
+        # this is to mimic the behaviour of complex32, else we will get different results
+        if dtype in (torch.float16, torch.bfloat16, torch.int8):
+            cache = cache.bfloat16() if dtype == torch.bfloat16 else cache.half()
+        return cache
+
+    def forward(self, max_seq_len, offset=0):
+        return self.forward_impl(
+            max_seq_len, self.dim, dtype=self.inv_freq.dtype, device=self.inv_freq.device
+        )
+
+
+@torch.jit.script
+def apply_rotary_pos_emb(x: torch.Tensor, rope_cache: torch.Tensor) -> torch.Tensor:
+    # x: [sq, b, np, hn]
+    sq, b, np, hn = x.size(0), x.size(1), x.size(2), x.size(3)
+    rot_dim = rope_cache.shape[-2] * 2
+    x, x_pass = x[..., :rot_dim], x[..., rot_dim:]
+    # truncate to support variable sizes
+    rope_cache = rope_cache[:sq]
+    xshaped = x.reshape(sq, -1, np, rot_dim // 2, 2)
+    rope_cache = rope_cache.view(sq, -1, 1, xshaped.size(3), 2)
+    x_out2 = torch.stack(
+        [
+            xshaped[..., 0] * rope_cache[..., 0] - xshaped[..., 1] * rope_cache[..., 1],
+            xshaped[..., 1] * rope_cache[..., 0] + xshaped[..., 0] * rope_cache[..., 1],
+        ],
+        -1,
+    )
+    x_out2 = x_out2.flatten(3)
+    return torch.cat((x_out2, x_pass), dim=-1)
+
+
+class RMSNorm(torch.nn.Module):
+    def __init__(self, normalized_shape, eps=1e-5, device=None, dtype=None, **kwargs):
+        super().__init__()
+        self.weight = torch.nn.Parameter(torch.empty(normalized_shape, device=device, dtype=dtype))
+        self.eps = eps
+
+    def forward(self, hidden_states: torch.Tensor):
+        input_dtype = hidden_states.dtype
+        variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.eps)
+
+        return (self.weight * hidden_states).to(input_dtype)
+
+
+class CoreAttention(torch.nn.Module):
+    def __init__(self, config: ChatGLMConfig, layer_number):
+        super(CoreAttention, self).__init__()
+
+        self.apply_query_key_layer_scaling = config.apply_query_key_layer_scaling
+        self.attention_softmax_in_fp32 = config.attention_softmax_in_fp32
+        if self.apply_query_key_layer_scaling:
+            self.attention_softmax_in_fp32 = True
+        self.layer_number = max(1, layer_number)
+
+        projection_size = config.kv_channels * config.num_attention_heads
+
+        # Per attention head and per partition values.
+        self.hidden_size_per_partition = projection_size
+        self.hidden_size_per_attention_head = projection_size // config.num_attention_heads
+        self.num_attention_heads_per_partition = config.num_attention_heads
+
+        coeff = None
+        self.norm_factor = math.sqrt(self.hidden_size_per_attention_head)
+        if self.apply_query_key_layer_scaling:
+            coeff = self.layer_number
+            self.norm_factor *= coeff
+        self.coeff = coeff
+
+        self.attention_dropout = torch.nn.Dropout(config.attention_dropout)
+
+    def forward(self, query_layer, key_layer, value_layer, attention_mask):
+        pytorch_major_version = int(torch.__version__.split('.')[0])
+        if pytorch_major_version >= 2:
+            query_layer, key_layer, value_layer = [k.permute(1, 2, 0, 3) for k in [query_layer, key_layer, value_layer]]
+            if attention_mask is None and query_layer.shape[2] == key_layer.shape[2]:
+                context_layer = torch.nn.functional.scaled_dot_product_attention(query_layer, key_layer, value_layer,
+                                                                                 is_causal=True)
+            else:
+                if attention_mask is not None:
+                    attention_mask = ~attention_mask
+                context_layer = torch.nn.functional.scaled_dot_product_attention(query_layer, key_layer, value_layer,
+                                                                                 attention_mask)
+            context_layer = context_layer.permute(2, 0, 1, 3)
+            new_context_layer_shape = context_layer.size()[:-2] + (self.hidden_size_per_partition,)
+            context_layer = context_layer.reshape(*new_context_layer_shape)
+        else:
+            # Raw attention scores
+
+            # [b, np, sq, sk]
+            output_size = (query_layer.size(1), query_layer.size(2), query_layer.size(0), key_layer.size(0))
+
+            # [sq, b, np, hn] -> [sq, b * np, hn]
+            query_layer = query_layer.view(output_size[2], output_size[0] * output_size[1], -1)
+            # [sk, b, np, hn] -> [sk, b * np, hn]
+            key_layer = key_layer.view(output_size[3], output_size[0] * output_size[1], -1)
+
+            # preallocting input tensor: [b * np, sq, sk]
+            matmul_input_buffer = torch.empty(
+                output_size[0] * output_size[1], output_size[2], output_size[3], dtype=query_layer.dtype,
+                device=query_layer.device
+            )
+
+            # Raw attention scores. [b * np, sq, sk]
+            matmul_result = torch.baddbmm(
+                matmul_input_buffer,
+                query_layer.transpose(0, 1),  # [b * np, sq, hn]
+                key_layer.transpose(0, 1).transpose(1, 2),  # [b * np, hn, sk]
+                beta=0.0,
+                alpha=(1.0 / self.norm_factor),
+            )
+
+            # change view to [b, np, sq, sk]
+            attention_scores = matmul_result.view(*output_size)
+
+            # ===========================
+            # Attention probs and dropout
+            # ===========================
+
+            # attention scores and attention mask [b, np, sq, sk]
+            if self.attention_softmax_in_fp32:
+                attention_scores = attention_scores.float()
+            if self.coeff is not None:
+                attention_scores = attention_scores * self.coeff
+            if attention_mask is None and attention_scores.shape[2] == attention_scores.shape[3]:
+                attention_mask = torch.ones(output_size[0], 1, output_size[2], output_size[3],
+                                            device=attention_scores.device, dtype=torch.bool)
+                attention_mask.tril_()
+                attention_mask = ~attention_mask
+            if attention_mask is not None:
+                attention_scores = attention_scores.masked_fill(attention_mask, float("-inf"))
+            attention_probs = F.softmax(attention_scores, dim=-1)
+            attention_probs = attention_probs.type_as(value_layer)
+
+            # This is actually dropping out entire tokens to attend to, which might
+            # seem a bit unusual, but is taken from the original Transformer paper.
+            attention_probs = self.attention_dropout(attention_probs)
+            # =========================
+            # Context layer. [sq, b, hp]
+            # =========================
+
+            # value_layer -> context layer.
+            # [sk, b, np, hn] --> [b, np, sq, hn]
+
+            # context layer shape: [b, np, sq, hn]
+            output_size = (value_layer.size(1), value_layer.size(2), query_layer.size(0), value_layer.size(3))
+            # change view [sk, b * np, hn]
+            value_layer = value_layer.view(value_layer.size(0), output_size[0] * output_size[1], -1)
+            # change view [b * np, sq, sk]
+            attention_probs = attention_probs.view(output_size[0] * output_size[1], output_size[2], -1)
+            # matmul: [b * np, sq, hn]
+            context_layer = torch.bmm(attention_probs, value_layer.transpose(0, 1))
+            # change view [b, np, sq, hn]
+            context_layer = context_layer.view(*output_size)
+            # [b, np, sq, hn] --> [sq, b, np, hn]
+            context_layer = context_layer.permute(2, 0, 1, 3).contiguous()
+            # [sq, b, np, hn] --> [sq, b, hp]
+            new_context_layer_shape = context_layer.size()[:-2] + (self.hidden_size_per_partition,)
+            context_layer = context_layer.view(*new_context_layer_shape)
+
+        return context_layer
+
+
+class SelfAttention(torch.nn.Module):
+    """Parallel self-attention layer abstract class.
+
+    Self-attention layer takes input with size [s, b, h]
+    and returns output of the same size.
+    """
+
+    def __init__(self, config: ChatGLMConfig, layer_number, device=None):
+        super(SelfAttention, self).__init__()
+        self.layer_number = max(1, layer_number)
+
+        self.projection_size = config.kv_channels * config.num_attention_heads
+
+        # Per attention head and per partition values.
+        self.hidden_size_per_attention_head = self.projection_size // config.num_attention_heads
+        self.num_attention_heads_per_partition = config.num_attention_heads
+
+        self.multi_query_attention = config.multi_query_attention
+        self.qkv_hidden_size = 3 * self.projection_size
+        if self.multi_query_attention:
+            self.num_multi_query_groups_per_partition = config.multi_query_group_num
+            self.qkv_hidden_size = (
+                    self.projection_size + 2 * self.hidden_size_per_attention_head * config.multi_query_group_num
+            )
+        self.query_key_value = nn.Linear(config.hidden_size, self.qkv_hidden_size,
+                                         bias=config.add_bias_linear or config.add_qkv_bias,
+                                         device=device, **_config_to_kwargs(config)
+                                         )
+
+        self.core_attention = CoreAttention(config, self.layer_number)
+
+        # Output.
+        self.dense = nn.Linear(self.projection_size, config.hidden_size, bias=config.add_bias_linear,
+                               device=device, **_config_to_kwargs(config)
+                               )
+
+    def _allocate_memory(self, inference_max_sequence_len, batch_size, device=None, dtype=None):
+        if self.multi_query_attention:
+            num_attention_heads = self.num_multi_query_groups_per_partition
+        else:
+            num_attention_heads = self.num_attention_heads_per_partition
+        return torch.empty(
+            inference_max_sequence_len,
+            batch_size,
+            num_attention_heads,
+            self.hidden_size_per_attention_head,
+            dtype=dtype,
+            device=device,
+        )
+
+    def forward(
+            self, hidden_states, attention_mask, rotary_pos_emb, kv_cache=None, use_cache=True
+    ):
+        # hidden_states: [sq, b, h]
+
+        # =================================================
+        # Pre-allocate memory for key-values for inference.
+        # =================================================
+        # =====================
+        # Query, Key, and Value
+        # =====================
+
+        # Attention heads [sq, b, h] --> [sq, b, (np * 3 * hn)]
+        mixed_x_layer = self.query_key_value(hidden_states)
+
+        if self.multi_query_attention:
+            (query_layer, key_layer, value_layer) = mixed_x_layer.split(
+                [
+                    self.num_attention_heads_per_partition * self.hidden_size_per_attention_head,
+                    self.num_multi_query_groups_per_partition * self.hidden_size_per_attention_head,
+                    self.num_multi_query_groups_per_partition * self.hidden_size_per_attention_head,
+                ],
+                dim=-1,
+            )
+            query_layer = query_layer.view(
+                query_layer.size()[:-1] + (self.num_attention_heads_per_partition, self.hidden_size_per_attention_head)
+            )
+            key_layer = key_layer.view(
+                key_layer.size()[:-1] + (self.num_multi_query_groups_per_partition, self.hidden_size_per_attention_head)
+            )
+            value_layer = value_layer.view(
+                value_layer.size()[:-1]
+                + (self.num_multi_query_groups_per_partition, self.hidden_size_per_attention_head)
+            )
+        else:
+            new_tensor_shape = mixed_x_layer.size()[:-1] + \
+                               (self.num_attention_heads_per_partition,
+                                3 * self.hidden_size_per_attention_head)
+            mixed_x_layer = mixed_x_layer.view(*new_tensor_shape)
+
+            # [sq, b, np, 3 * hn] --> 3 [sq, b, np, hn]
+            (query_layer, key_layer, value_layer) = split_tensor_along_last_dim(mixed_x_layer, 3)
+
+        # apply relative positional encoding (rotary embedding)
+        if rotary_pos_emb is not None:
+            query_layer = apply_rotary_pos_emb(query_layer, rotary_pos_emb)
+            key_layer = apply_rotary_pos_emb(key_layer, rotary_pos_emb)
+
+        # adjust key and value for inference
+        if kv_cache is not None:
+            cache_k, cache_v = kv_cache
+            key_layer = torch.cat((cache_k, key_layer), dim=0)
+            value_layer = torch.cat((cache_v, value_layer), dim=0)
+        if use_cache:
+            kv_cache = (key_layer, value_layer)
+        else:
+            kv_cache = None
+
+        if self.multi_query_attention:
+            key_layer = key_layer.unsqueeze(-2)
+            key_layer = key_layer.expand(
+                -1, -1, -1, self.num_attention_heads_per_partition // self.num_multi_query_groups_per_partition, -1
+            )
+            key_layer = key_layer.contiguous().view(
+                key_layer.size()[:2] + (self.num_attention_heads_per_partition, self.hidden_size_per_attention_head)
+            )
+            value_layer = value_layer.unsqueeze(-2)
+            value_layer = value_layer.expand(
+                -1, -1, -1, self.num_attention_heads_per_partition // self.num_multi_query_groups_per_partition, -1
+            )
+            value_layer = value_layer.contiguous().view(
+                value_layer.size()[:2] + (self.num_attention_heads_per_partition, self.hidden_size_per_attention_head)
+            )
+
+        # ==================================
+        # core attention computation
+        # ==================================
+
+        context_layer = self.core_attention(query_layer, key_layer, value_layer, attention_mask)
+
+        # =================
+        # Output. [sq, b, h]
+        # =================
+
+        output = self.dense(context_layer)
+
+        return output, kv_cache
+
+
+def _config_to_kwargs(args):
+    common_kwargs = {
+        "dtype": args.torch_dtype,
+    }
+    return common_kwargs
+
+
+class MLP(torch.nn.Module):
+    """MLP.
+
+    MLP will take the input with h hidden state, project it to 4*h
+    hidden dimension, perform nonlinear transformation, and project the
+    state back into h hidden dimension.
+    """
+
+    def __init__(self, config: ChatGLMConfig, device=None):
+        super(MLP, self).__init__()
+
+        self.add_bias = config.add_bias_linear
+
+        # Project to 4h. If using swiglu double the output width, see https://arxiv.org/pdf/2002.05202.pdf
+        self.dense_h_to_4h = nn.Linear(
+            config.hidden_size,
+            config.ffn_hidden_size * 2,
+            bias=self.add_bias,
+            device=device,
+            **_config_to_kwargs(config)
+        )
+
+        def swiglu(x):
+            x = torch.chunk(x, 2, dim=-1)
+            return F.silu(x[0]) * x[1]
+
+        self.activation_func = swiglu
+
+        # Project back to h.
+        self.dense_4h_to_h = nn.Linear(
+            config.ffn_hidden_size,
+            config.hidden_size,
+            bias=self.add_bias,
+            device=device,
+            **_config_to_kwargs(config)
+        )
+
+    def forward(self, hidden_states):
+        # [s, b, 4hp]
+        intermediate_parallel = self.dense_h_to_4h(hidden_states)
+        intermediate_parallel = self.activation_func(intermediate_parallel)
+        # [s, b, h]
+        output = self.dense_4h_to_h(intermediate_parallel)
+        return output
+
+
+class GLMBlock(torch.nn.Module):
+    """A single transformer layer.
+
+    Transformer layer takes input with size [s, b, h] and returns an
+    output of the same size.
+    """
+
+    def __init__(self, config: ChatGLMConfig, layer_number, device=None):
+        super(GLMBlock, self).__init__()
+        self.layer_number = layer_number
+
+        self.apply_residual_connection_post_layernorm = config.apply_residual_connection_post_layernorm
+
+        self.fp32_residual_connection = config.fp32_residual_connection
+
+        LayerNormFunc = RMSNorm if config.rmsnorm else LayerNorm
+        # Layernorm on the input data.
+        self.input_layernorm = LayerNormFunc(config.hidden_size, eps=config.layernorm_epsilon, device=device,
+                                             dtype=config.torch_dtype)
+
+        # Self attention.
+        self.self_attention = SelfAttention(config, layer_number, device=device)
+        self.hidden_dropout = config.hidden_dropout
+
+        # Layernorm on the attention output
+        self.post_attention_layernorm = LayerNormFunc(config.hidden_size, eps=config.layernorm_epsilon, device=device,
+                                                      dtype=config.torch_dtype)
+
+        # MLP
+        self.mlp = MLP(config, device=device)
+
+    def forward(
+            self, hidden_states, attention_mask, rotary_pos_emb, kv_cache=None, use_cache=True,
+    ):
+        # hidden_states: [s, b, h]
+
+        # Layer norm at the beginning of the transformer layer.
+        layernorm_output = self.input_layernorm(hidden_states)
+        # Self attention.
+        attention_output, kv_cache = self.self_attention(
+            layernorm_output,
+            attention_mask,
+            rotary_pos_emb,
+            kv_cache=kv_cache,
+            use_cache=use_cache
+        )
+
+        # Residual connection.
+        if self.apply_residual_connection_post_layernorm:
+            residual = layernorm_output
+        else:
+            residual = hidden_states
+
+        layernorm_input = torch.nn.functional.dropout(attention_output, p=self.hidden_dropout, training=self.training)
+        layernorm_input = residual + layernorm_input
+
+        # Layer norm post the self attention.
+        layernorm_output = self.post_attention_layernorm(layernorm_input)
+
+        # MLP.
+        mlp_output = self.mlp(layernorm_output)
+
+        # Second residual connection.
+        if self.apply_residual_connection_post_layernorm:
+            residual = layernorm_output
+        else:
+            residual = layernorm_input
+
+        output = torch.nn.functional.dropout(mlp_output, p=self.hidden_dropout, training=self.training)
+        output = residual + output
+
+        return output, kv_cache
+
+
+class GLMTransformer(torch.nn.Module):
+    """Transformer class."""
+
+    def __init__(self, config: ChatGLMConfig, device=None):
+        super(GLMTransformer, self).__init__()
+
+        self.fp32_residual_connection = config.fp32_residual_connection
+        self.post_layer_norm = config.post_layer_norm
+
+        # Number of layers.
+        self.num_layers = config.num_layers
+
+        # Transformer layers.
+        def build_layer(layer_number):
+            return GLMBlock(config, layer_number, device=device)
+
+        self.layers = torch.nn.ModuleList([build_layer(i + 1) for i in range(self.num_layers)])
+
+        if self.post_layer_norm:
+            LayerNormFunc = RMSNorm if config.rmsnorm else LayerNorm
+            # Final layer norm before output.
+            self.final_layernorm = LayerNormFunc(config.hidden_size, eps=config.layernorm_epsilon, device=device,
+                                                 dtype=config.torch_dtype)
+
+        self.gradient_checkpointing = False
+
+    def _get_layer(self, layer_number):
+        return self.layers[layer_number]
+
+    def forward(
+            self, hidden_states, attention_mask, rotary_pos_emb, kv_caches=None,
+            use_cache: Optional[bool] = True,
+            output_hidden_states: Optional[bool] = False,
+    ):
+        if not kv_caches:
+            kv_caches = [None for _ in range(self.num_layers)]
+        presents = () if use_cache else None
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                logger.warning_once(
+                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                )
+                use_cache = False
+
+        all_self_attentions = None
+        all_hidden_states = () if output_hidden_states else None
+        for index in range(self.num_layers):
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            layer = self._get_layer(index)
+            if self.gradient_checkpointing and self.training:
+                layer_ret = torch.utils.checkpoint.checkpoint(
+                    layer,
+                    hidden_states,
+                    attention_mask,
+                    rotary_pos_emb,
+                    kv_caches[index],
+                    use_cache,
+                    use_reentrant=False
+                )
+            else:
+                layer_ret = layer(
+                    hidden_states,
+                    attention_mask,
+                    rotary_pos_emb,
+                    kv_cache=kv_caches[index],
+                    use_cache=use_cache
+                )
+            hidden_states, kv_cache = layer_ret
+            if use_cache:
+                presents = presents + (kv_cache,)
+
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        # Final layer norm.
+        if self.post_layer_norm:
+            hidden_states = self.final_layernorm(hidden_states)
+
+        return hidden_states, presents, all_hidden_states, all_self_attentions
+
+
+# PreTrainedModel
+class ChatGLMPreTrainedModel(nn.Module):
+    def __init__(self, *args, **kwargs):
+        self.config = args[0]
+
+        super().__init__()
+
+    is_parallelizable = False
+    supports_gradient_checkpointing = True
+    config_class = ChatGLMConfig
+    base_model_prefix = "transformer"
+    _no_split_modules = ["GLMBlock"]
+
+    def _init_weights(self, module: nn.Module):
+        """Initialize the weights."""
+        return
+
+    def get_masks(self, input_ids, past_key_values, padding_mask=None):
+        batch_size, seq_length = input_ids.shape
+        full_attention_mask = torch.ones(batch_size, seq_length, seq_length, device=input_ids.device)
+        full_attention_mask.tril_()
+        past_length = 0
+        if past_key_values:
+            past_length = past_key_values[0][0].shape[0]
+        if past_length:
+            full_attention_mask = torch.cat((torch.ones(batch_size, seq_length, past_length,
+                                                        device=input_ids.device), full_attention_mask), dim=-1)
+        if padding_mask is not None:
+            full_attention_mask = full_attention_mask * padding_mask.unsqueeze(1)
+        if not past_length and padding_mask is not None:
+            full_attention_mask -= padding_mask.unsqueeze(-1) - 1
+        full_attention_mask = (full_attention_mask < 0.5).bool()
+        full_attention_mask.unsqueeze_(1)
+        return full_attention_mask
+
+    def get_position_ids(self, input_ids, device):
+        batch_size, seq_length = input_ids.shape
+        position_ids = torch.arange(seq_length, dtype=torch.long, device=device).unsqueeze(0).repeat(batch_size, 1)
+        return position_ids
+
+    def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None):
+        if not self.supports_gradient_checkpointing:
+            raise ValueError(f"{self.__class__.__name__} does not support gradient checkpointing.")
+
+
+class Embedding(torch.nn.Module):
+    """Language model embeddings."""
+
+    def __init__(self, config: ChatGLMConfig, device=None):
+        super(Embedding, self).__init__()
+
+        self.hidden_size = config.hidden_size
+        # Word embeddings (parallel).
+        self.word_embeddings = nn.Embedding(
+            config.padded_vocab_size,
+            self.hidden_size,
+            dtype=config.torch_dtype,
+            device=device
+        )
+        self.fp32_residual_connection = config.fp32_residual_connection
+
+    def forward(self, input_ids):
+        # Embeddings.
+        words_embeddings = self.word_embeddings(input_ids)
+        embeddings = words_embeddings
+        # Data format change to avoid explicit tranposes : [b s h] --> [s b h].
+        embeddings = embeddings.transpose(0, 1).contiguous()
+        # If the input flag for fp32 residual connection is set, convert for float.
+        if self.fp32_residual_connection:
+            embeddings = embeddings.float()
+        return embeddings
+
+
+class ChatGLMModel(ChatGLMPreTrainedModel):
+    def __init__(self, config: ChatGLMConfig, device=None, empty_init=True):
+        super().__init__(config)
+        if empty_init:
+            init_method = skip_init
+        else:
+            init_method = default_init
+        init_kwargs = {}
+        if device is not None:
+            init_kwargs["device"] = device
+        self.embedding = init_method(Embedding, config, **init_kwargs)
+        self.num_layers = config.num_layers
+        self.multi_query_group_num = config.multi_query_group_num
+        self.kv_channels = config.kv_channels
+
+        # Rotary positional embeddings
+        self.seq_length = config.seq_length
+        rotary_dim = (
+            config.hidden_size // config.num_attention_heads if config.kv_channels is None else config.kv_channels
+        )
+
+        self.rotary_pos_emb = RotaryEmbedding(rotary_dim // 2, device=device,
+                                              dtype=config.torch_dtype)
+        self.encoder = init_method(GLMTransformer, config, **init_kwargs)
+        self.output_layer = init_method(nn.Linear, config.hidden_size, config.padded_vocab_size, bias=False,
+                                        dtype=config.torch_dtype, **init_kwargs)
+        self.pre_seq_len = config.pre_seq_len
+        self.prefix_projection = config.prefix_projection
+        if self.pre_seq_len is not None:
+            for param in self.parameters():
+                param.requires_grad = False
+            self.prefix_tokens = torch.arange(self.pre_seq_len).long()
+            self.prefix_encoder = PrefixEncoder(config)
+            self.dropout = torch.nn.Dropout(0.1)
+
+    def get_input_embeddings(self):
+        return self.embedding.word_embeddings
+
+    def set_input_embeddings(self, value):
+        self.embedding.word_embeddings = value
+
+    def get_prompt(self, batch_size, device, dtype=torch.half):
+        prefix_tokens = self.prefix_tokens.unsqueeze(0).expand(batch_size, -1).to(device)
+        past_key_values = self.prefix_encoder(prefix_tokens).type(dtype)
+        past_key_values = past_key_values.view(
+            batch_size,
+            self.pre_seq_len,
+            self.num_layers * 2,
+            self.multi_query_group_num,
+            self.kv_channels
+        )
+        # seq_len, b, nh, hidden_size
+        past_key_values = self.dropout(past_key_values)
+        past_key_values = past_key_values.permute([2, 1, 0, 3, 4]).split(2)
+        return past_key_values
+
+    def forward(
+            self,
+            input_ids,
+            position_ids: Optional[torch.Tensor] = None,
+            attention_mask: Optional[torch.BoolTensor] = None,
+            full_attention_mask: Optional[torch.BoolTensor] = None,
+            past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,
+            inputs_embeds: Optional[torch.Tensor] = None,
+            use_cache: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+    ):
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        batch_size, seq_length = input_ids.shape
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embedding(input_ids)
+
+        if self.pre_seq_len is not None:
+            if past_key_values is None:
+                past_key_values = self.get_prompt(batch_size=batch_size, device=input_ids.device,
+                                                  dtype=inputs_embeds.dtype)
+            if attention_mask is not None:
+                attention_mask = torch.cat([attention_mask.new_ones((batch_size, self.pre_seq_len)),
+                                            attention_mask], dim=-1)
+
+        if full_attention_mask is None:
+            if (attention_mask is not None and not attention_mask.all()) or (past_key_values and seq_length != 1):
+                full_attention_mask = self.get_masks(input_ids, past_key_values, padding_mask=attention_mask)
+
+        # Rotary positional embeddings
+        rotary_pos_emb = self.rotary_pos_emb(self.seq_length)
+        if position_ids is not None:
+            rotary_pos_emb = rotary_pos_emb[position_ids]
+        else:
+            rotary_pos_emb = rotary_pos_emb[None, :seq_length]
+        rotary_pos_emb = rotary_pos_emb.transpose(0, 1).contiguous()
+
+        # Run encoder.
+        hidden_states, presents, all_hidden_states, all_self_attentions = self.encoder(
+            inputs_embeds, full_attention_mask, rotary_pos_emb=rotary_pos_emb,
+            kv_caches=past_key_values, use_cache=use_cache, output_hidden_states=output_hidden_states
+        )
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, presents, all_hidden_states, all_self_attentions] if v is not None)
+
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=presents,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions,
+        )
+
+    def quantize(self, weight_bit_width: int):
+        from quantization import quantize
+        quantize(self.encoder, weight_bit_width)
+        return self
+
+
+class ChatGLMForConditionalGeneration(ChatGLMPreTrainedModel):
+    def __init__(self, config: ChatGLMConfig, empty_init=True, device=None):
+        super().__init__(config)
+
+        self.max_sequence_length = config.max_length
+        self.transformer = ChatGLMModel(config, empty_init=empty_init, device=device)
+        self.device = device
+        self.config = config
+        self.quantized = False
+
+    def prepare_inputs_for_generation(
+            self,
+            input_ids: torch.LongTensor,
+            past_key_values: Optional[torch.Tensor] = None,
+            attention_mask: Optional[torch.Tensor] = None,
+            position_ids: Optional[torch.Tensor] = None,
+            use_cache: Optional[bool] = None,
+            is_first_forward: bool = True,
+            **kwargs
+    ) -> dict:
+        # only last token for input_ids if past is not None
+        if position_ids is None:
+            position_ids = self.get_position_ids(input_ids, device=input_ids.device)
+        if not is_first_forward:
+            if past_key_values is not None:
+                position_ids = position_ids[..., -1:]
+                input_ids = input_ids[:, -1:]
+        return {
+            "input_ids": input_ids,
+            "past_key_values": past_key_values,
+            "position_ids": position_ids,
+            "attention_mask": attention_mask,
+            "return_last_logit": True,
+            "use_cache": use_cache
+        }
+
+    def forward(
+            self,
+            input_ids: Optional[torch.Tensor] = None,
+            position_ids: Optional[torch.Tensor] = None,
+            attention_mask: Optional[torch.Tensor] = None,
+            past_key_values: Optional[Tuple[torch.FloatTensor]] = None,
+            inputs_embeds: Optional[torch.Tensor] = None,
+            labels: Optional[torch.Tensor] = None,
+            use_cache: Optional[bool] = None,
+            output_attentions: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+            return_last_logit: Optional[bool] = False,
+    ):
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        transformer_outputs = self.transformer(
+            input_ids=input_ids,
+            position_ids=position_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        # hidden_states = transformer_outputs[0]
+        hidden_states = transformer_outputs.last_hidden_state
+        if return_last_logit:
+            hidden_states = hidden_states[-1:]
+        lm_logits = self.transformer.output_layer(hidden_states)
+        lm_logits = lm_logits.transpose(0, 1).contiguous()
+
+        loss = None
+        if labels is not None:
+            lm_logits = lm_logits.to(torch.float32)
+
+            # Shift so that tokens < n predict n
+            shift_logits = lm_logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss(ignore_index=-100)
+            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+
+            lm_logits = lm_logits.to(hidden_states.dtype)
+            loss = loss.to(hidden_states.dtype)
+
+        if not return_dict:
+            output = (lm_logits,) + transformer_outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=lm_logits,
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+        )
+
+    @staticmethod
+    def _reorder_cache(
+            past: Tuple[Tuple[torch.Tensor, torch.Tensor], ...], beam_idx: torch.LongTensor
+    ) -> Tuple[Tuple[torch.Tensor, torch.Tensor], ...]:
+        """
+        This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or
+        [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
+        beam_idx at every generation step.
+
+        Output shares the same memory storage as `past`.
+        """
+        return tuple(
+            (
+                layer_past[0].index_select(1, beam_idx.to(layer_past[0].device)),
+                layer_past[1].index_select(1, beam_idx.to(layer_past[1].device)),
+            )
+            for layer_past in past
+        )
+
+    def quantize(self, bits: int, empty_init=False, device=None, **kwargs):
+        if bits == 0:
+            return
+
+        from quantization import quantize
+
+        if self.quantized:
+            logger.info("Already quantized.")
+            return self
+
+        self.quantized = True
+
+        self.configquantization_bit = bits
+
+        self.transformer.encoder = quantize(self.transformer.encoder, bits, empty_init=empty_init, device=device,
+                                            **kwargs)
+        return self
+
+
+class ChatGLMForSequenceClassification(ChatGLMPreTrainedModel):
+    def __init__(self, config: ChatGLMConfig, empty_init=True, device=None, bits=None):
+        super().__init__(config)
+
+        self.num_labels = config.num_labels
+        self.transformer = ChatGLMModel(config, empty_init=empty_init, device=device)
+
+        self.classifier_head = nn.Linear(config.hidden_size, config.num_labels, bias=True, dtype=torch.half)
+        if config.classifier_dropout is not None:
+            self.dropout = nn.Dropout(config.classifier_dropout)
+        else:
+            self.dropout = None
+        self.config = config
+
+        if bits:
+            self.quantize(bits=bits, empty_init=True)
+
+    def forward(
+            self,
+            input_ids: Optional[torch.LongTensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            attention_mask: Optional[torch.Tensor] = None,
+            full_attention_mask: Optional[torch.Tensor] = None,
+            past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,
+            inputs_embeds: Optional[torch.LongTensor] = None,
+            labels: Optional[torch.LongTensor] = None,
+            use_cache: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+    ) -> Union[Tuple[torch.Tensor, ...], SequenceClassifierOutputWithPast]:
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        transformer_outputs = self.transformer(
+            input_ids=input_ids,
+            position_ids=position_ids,
+            attention_mask=attention_mask,
+            full_attention_mask=full_attention_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = transformer_outputs[0]
+        pooled_hidden_states = hidden_states[-1]
+        if self.dropout is not None:
+            pooled_hidden_states = self.dropout(pooled_hidden_states)
+        logits = self.classifier_head(pooled_hidden_states)
+
+        loss = None
+        if labels is not None:
+            if self.config.problem_type is None:
+                if self.num_labels == 1:
+                    self.config.problem_type = "regression"
+                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
+                    self.config.problem_type = "single_label_classification"
+                else:
+                    self.config.problem_type = "multi_label_classification"
+
+            if self.config.problem_type == "regression":
+                loss_fct = MSELoss()
+                if self.num_labels == 1:
+                    loss = loss_fct(logits.squeeze().float(), labels.squeeze())
+                else:
+                    loss = loss_fct(logits.float(), labels)
+            elif self.config.problem_type == "single_label_classification":
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(logits.view(-1, self.num_labels).float(), labels.view(-1))
+            elif self.config.problem_type == "multi_label_classification":
+                loss_fct = BCEWithLogitsLoss()
+                loss = loss_fct(logits.float(), labels.view(-1, self.num_labels))
+
+        if not return_dict:
+            output = (logits,) + transformer_outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return SequenceClassifierOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+        )
+
+
+if __name__ == "__main__":
+    import torch
+    from safetensors.torch import load_model
+    from torch.nn.functional import softmax
+
+    import configuration_chatglm_full as configuration_chatglm
+    from tokenization_chatglm import ChatGLMTokenizer
+
+
+    def random_sample(logits, temperature=1.0):
+        post_logits = logits[:, -1, :] / temperature
+        post_logits = softmax(post_logits, dim=-1)
+
+        selected_indices = torch.multinomial(post_logits, 1)
+        return selected_indices
+
+
+    config = configuration_chatglm.ChatGLMConfig()
+    tokenlizer = ChatGLMTokenizer(vocab_file='tokenizer.model')
+
+    m = ChatGLMForConditionalGeneration(config=config, device='cpu')
+    load_model(m, "model-00001-of-00007.safetensors", strict=False)
+    load_model(m, "model-00002-of-00007.safetensors", strict=False)
+    load_model(m, "model-00003-of-00007.safetensors", strict=False)
+    load_model(m, "model-00004-of-00007.safetensors", strict=False)
+    load_model(m, "model-00005-of-00007.safetensors", strict=False)
+    load_model(m, "model-00006-of-00007.safetensors", strict=False)
+    load_model(m, "model-00007-of-00007.safetensors", strict=False)
+    print(m)
+    m = m.quantize(bits=4, device='cuda')
+    m = m.to('cuda')
+    m = m.eval()
+
+    t = tokenlizer(['为什么A股指数跌的不多，但是我亏损比之前都多？'], add_special_tokens=False)
+
+    input_ids = [[64790, 64792] + t['input_ids'][0] + [64796]]
+    input_ids = torch.tensor(input_ids, device='cuda')
+
+    attention_mask = torch.tensor([[1] * len(input_ids[0])], dtype=torch.long, device='cuda')
+    position_ids = torch.tensor([range(len(input_ids[0]))], dtype=torch.long, device='cuda')
+
+    with torch.no_grad():
+        for i in range(100):
+            output = m.forward(input_ids=input_ids,
+                               attention_mask=attention_mask,
+                               position_ids=position_ids,
+                               # output_hidden_states=True,
+                               return_last_logit=False,
+                               use_cache=False
+                               )
+
+            result = random_sample(output.logits, 0.1)
+
+            if result.item() == 2:
+                break
+
+            # print(tokenlizer.decode(input_ids[0], skip_special_tokens=True), end='\n')
+            print(tokenlizer.decode(result[0], skip_special_tokens=False), end='')
+
+            input_ids = torch.cat([input_ids, result],
+                                  dim=1)
+
+            attention_mask = torch.cat([attention_mask, torch.ones(1, 1, device='cuda', dtype=torch.long)], dim=1)
+            position_ids = torch.cat(
+                [position_ids, torch.ones(1, 1, device='cuda', dtype=torch.long) * len(position_ids[0])], dim=1)
+
+        print('\n' * 5, tokenlizer.decode(input_ids[0], skip_special_tokens=True), end='\n')
+
+
+
+
diff --git a/Model_Architecture_Discussions/ChatGLM3/img/img.png b/Model_Architecture_Discussions/ChatGLM3/img/img.png
new file mode 100644
index 0000000..a38b07f
Binary files /dev/null and b/Model_Architecture_Discussions/ChatGLM3/img/img.png differ
diff --git a/Model_Architecture_Discussions/ChatGLM3/quantization.py b/Model_Architecture_Discussions/ChatGLM3/quantization.py
new file mode 100644
index 0000000..6e55426
--- /dev/null
+++ b/Model_Architecture_Discussions/ChatGLM3/quantization.py
@@ -0,0 +1,188 @@
+from torch.nn import Linear
+from torch.nn.parameter import Parameter
+
+import bz2
+import torch
+import base64
+import ctypes
+import logging
+
+from typing import List
+from functools import partial
+
+logger = logging.getLogger(__name__)
+
+try:
+    from cpm_kernels.kernels.base import LazyKernelCModule, KernelFunction, round_up
+
+    class Kernel:
+        def __init__(self, code: bytes, function_names: List[str]):
+            self.code = code
+            self._function_names = function_names
+            self._cmodule = LazyKernelCModule(self.code)
+
+            for name in self._function_names:
+                setattr(self, name, KernelFunction(self._cmodule, name))
+
+    quantization_code = "$QlpoOTFBWSZTWU9yuJUAQHN//////////f/n/8/n///n//bt4dTidcVx8X3V9FV/92/v4B7/AD5FBQFAAAChSgKpFCFAFVSigUAAAEKhSgUUqgFBKigqVREQAABQBQIANDTTIGI00BkZBkNGE0A0BkBkGQGRkaNAaAGQNBoGgDIAAYIGTI0DQAQAaGmmQMRpoDIyDIaMJoBoDIDIMgMjI0aA0AMgaDQNAGQAAwQMmRoGgAgA0NNMgYjTQGRkGQ0YTQDQGQGQZAZGRo0BoAZA0GgaAMgABggZMjQNABABoaaZAxGmgMjIMhowmgGgMgMgyAyMjRoDQAyBoNA0AZAADBAyZGgaAAmqU1NEgJqnptU/Sn4jRR6J6epk2pqb1Q/SgAPUGgyNNGjQ2SBpoAZAAGg0NB6mgDIAAAAA2oaApSREBNAARhGiYEaEwU8pvImlP0k2aam1GaGqbFNM1MHpTwmkepmyU9R6nqPKekHqNNPUxNGhp6n6p6QaZ6o9TG1GMqcoV9ly6nRanHlq6zPNbnGZNi6HSug+2nPiZ13XcnFYZW+45W11CumhzYhchOJ2GLLV1OBjBjGf4TptOddTSOcVxhqYZMYwZXZZY00zI1paX5X9J+b+f4e+x43RXSxXPOdquiGpduatGyXneN696M9t4HU2eR5XX/kPhP261NTx3JO1Ow7LyuDmeo9a7d351T1ZxnvnrvYnrXv/hXxPCeuYx2XsNmO003eg9J3Z6U7b23meJ4ri01OdzTk9BNO96brz+qT5nuvvH3ds/G+m/JcG/F2XYuhXlvO+jP7U3XgrzPN/lr8Sf1n6j4j7jZs+s/T0tNaNNYzTs12rxjwztHlnire3Nzc3N1wuBwOBwXBvZfoHpD7rFmR99V5vj3aXza3xdBbXMalubTg/jIv5dfAi54Pdc75j4z412n3Npj3Ld/ENm7a3b/Cod6h/ret1/5vn/C+l+gdslMvgPSLJ8d8q+U66fevYn/tW1chleEtNTGlcHCbLRlq0tHzF5tsbbZZfHjjLgZu42XCuC3NrdjTasZGNzgxPIrGqp7r3p7L2p5XjnpPSmTd5XtzqnB6U87zzg1Ol0zd0zsLszxR6lkxp35u6/teL0L0W922cR7Lu1lpL9CsHirzuM2T+BgsyViT6LHcm0/Vr6U/7LGGyJeqTEjt0PHWhF5mCT7R9mtlDwriYv0Tyr/OxYt6qp5r0mPVT0608TqnqMZaarU2nFwrTzzlrs1ed7z1ux60wyr4ydCaTi3enW8x68x0zU7tXSlcmPSW1mGpWJMg4zmPC2lK96tp0OE80y4MfEvnZj8zGluR6b22ki1Ou9V2nCd9xovcPvcYMZYy0lvN60ScZ45vN6yeCeeXFb1lVjnnCar5fwXwE2bzJ4HI1XVPXfXZMm44GUsMpYsmLB65TuVdm0cl0b+i/wGNN66XjeV7zuPpHcnK/juhhjdfId5jMdE5nN0dGmmm2zZs2cexD5n9p/dY352XsvXHaZNWWsmmS1atjR452nYudzvqv2HMRyvNNnlMcDl3R2+yx2uVrBubTW9icHDVtbNXlZm7jma1rM4VurZZd2y6nUau7ZXZ7bVU+mnoOVxZGMrVmvX60605JwmzGZhhhjTWtaaaMaaGTGmNMZasY0iX8VMUl8eepaIrzGSpemWOQyZORk2bNpjUybMmxqYmknCGCFynutfksaZpjTNMaaatM0xsxcGR0sociNqxNSmhhR1ZJPbsn8qyF0t2qH6iYBclclalbtTTcHTDsPaX6rlnElph2Jyumumtynv2Kk8GI7rsvXbIcJgHJOSaSXnnGaI3m87RtVXJOZ/YtgdTE6Wpha6ZlE8ayXkef1fh602r2WwvfMXtMdLlkfnLFdYYwYso+bWqm7yJqHXZGw2nrS5ZanSYnWlxBxMF1V940K2wdrI7R6OYf7DGGamMmTSbRhlS45xmVOumF1EyPCmHrrN8wwZOOrdNtLeMtzFzDlWnfTBxMk2NaXIZHBYxYLD4w8yju0ao65Vz1OIXoS9dLanwCe1PWrYuWMqf1if1z2k2yYfKJ741PDgno1ZQ8DRqvUny3mNoWTzGO6m1DkrJI8JiR5cSd+vZdGOO8nrMoc5+NDUFsMSXaZJeNlMmGLtJsovOsUp7I9S5VojKxF6bTVEelXqlfJobQr3LozSh2Jk7VcrVMfhXqszGWMzNqGhqZY0OadxkyyMssKugZR0KNFXBHlqwmJgTE/BNVMk6ItJXZMR0H47GpXv/DMOvNkmVuaV1PRfEdxuqc7Hcd+ZV/zTLaRxWk0nl9CdCeM6mn5rstHIBcpiuwmUZXeq81DacHI2rmrZ5SuE5mOZd6LQrZg9mx32TprA8BMo5jKN6yLTCi3WzQaZSuhzTtM1fUTGVpG8Tw+KXI0tjEpiWxtLYynOlktSbVlaI5kxP8TDH8kx50xoxi5KcA4pcja8KWLRlO/Ks6q06ergnvm1ca3Tq8Uw7LTUsmWyctXPWmpitl/uvGcWTGXGuAXDfhqazGmjkxcJW5hMMMMpYsXl2TZYtVOddG3XCarUt6Ptq9CZXSNzyuRzqRZOjsxdBbFVz6OA5HI43r1jityVlVpVkxmOsyaYWE1NTGq1sOVh36mHMcxtSvcy70edG0ZGR3I1Go1GRlV7mWWo1G0ZGRqlvH40l7o4m5xMWLLLYyNjnqc8556mdPqLJ31n/1nWOncxzG1tizrHs/Z+d2vP/B/l8wdJ6rHUn2nbbDq4p6htFtYzMMMTaZis1K5GKzGNmxhmUx2DDlZ/qNnIx41xnaMfCZWYaZWtNLTNW8ND4Fw1MyZOCdM428suKG1ehW8TesOydg7J+YYcD4cYR+8dFK6M4E3HM9ZfRNNL+Sn6rsl4DsrDl2HpPCnfxjGXtbZtYys1ttlyJ4T+BvexjGWRjMszK4Jpc77D3GyuVD7q0+G8m9G+2+rGm7cOR2y7FdtY2XUYx/oNlfRYxhMYyYZkyyg55enna9Kt/FFi6GMMwYwdwxWgxGMLKYmUyGExTKMZkMFhkymKuh0NOBNnBu+23LdwDoZYYzGGMxtORaTU1pjTGWTTGGtMrNWUsyyTTLLG1qy2ZjbK2DBllWqxMtBMaYZQmcE7zvvRcTkclUwdkxTaSdyySt/7fpL+T1v516Ji97fwr5JbLu305zMn5+GMTTZ9F+y7ExwmGVfG44yxn3dLv6l5i+Wth1jCrDq21nW9LqvvDzz3Vf3LLH/O/32TJ/erx3bXftO4eF+G956D952K/An4NfvOpjFjExjevP/UmE0fIoZXx6/w6lX/no3D0bLt+ixjieBM6ksRd0yB4Lt2SwYNE+gd1detlZWUnpiZfGfFaK+4PyCa/v18V8X75pe9fLXzp7l3VjF76vWZmHwGz1IZNWT7b8yddJ4q5kyrVdfru6atWc7bVYztL9Jf4GXvT+Y8m9/YsXP6H018a8D4XVOqvfzqeR+6yZOD8dPv0+U7/q5Pl+2dNb0MjzGVH5p6MNQ7cOWvw62U9aHE8DprDek+McLyvDz+te+9Zhq5+YTruufMcWMabqysTmZVWjKPfnK0wyVcrsuhjZRdLkHNvD72b9abriOSGIxiLixMOoalNPXzy+wT/tf+U6HHONfsz+xe8ufHBdQWWGWLA9if0rsnmrxK5LvRZQeWsTCsrmOYy8VteVfuRfcVTtDLItLIsMYxZLdU/DbtSemxF6Z6Zo5WBXE4tFdCyVMMXMTEMZXVlS6Xec2T4e0tHsRcEuWshcJ2YsNF5rUx1E8ifCq6Z+ZP7qdCeu/aTwFd53l16/o0NOw6O3dLavP4Hbi4RdmuDk6DoYaninC0+o4uZjbJ7Rxeu0/FbuFg+q7DVS6fQe0rZ6NDGUNNU6DEqOaLTicKnYZMnBWruljQxoaS3dZhocDge0bSTyOvdAbG5hxe2xji7E/L55xX13wWNDi6HCekcFxfCPGxY0MXC+s7afWaMdDyjyr+o8Rudm/NabOZvdl274zH4f5XK9z6On1Pe/K5TdPAslg77BjuO6Y3eO7GqvOPG/stknp1leyvLL0Z7bl9I4noMvLkzytLhWYzrOZzLXCORe028rORzOg4N/L0HlMOQ3Pgmnbb6KczlabORpu980q37TBqRu0/p3PO6234Bl03Ynuz+9W7gnsEcmvYaYY3aMYY0wx3pYd+ujsXauWdaY5Xkbtl23fPzFHiDB/QMo0yFjBllYxTQYYyxkrwn7JufwJ/PfgJ+C83X69ni6zvXcnyXabv0ncbLwsceS+RNlyN2mnneJtX0ngYO0+e+0+UnA+Wch3ji8hj5an4h+i6XBySU4n+R0roVcbw5yvHrmr4Yw8Y7x6c+9POPYHI5HI5HI5HI5HGXGww4nE4nrVyOR8XeqPEO7PLOiukYa3Novk5hV4cdtYZLI93e+uxff2jRo0aNGjRo0aNG1bVtW1dy3m83m8+tQ5ZzHw3nObwOu8La9Rc1dtkdS8A3eTk823tnktXWlxN6Oixe06zrN70Isd9jiOgZFq9yfkPqP/SLhN2Myl8jDM43bl1nbcb4cO57jlh8Jow6pzXZdL4dyODTuuhu77FyO27DdwdRxmvO+O+3N2+BdqyTwLHVczDVY4UPE4O66/ZO2cx1LFzVdSXtF7G4HMbrauOHRw6c8FdZ5m9fHZHYZXfTlZquyynSyTTKke6vcffSD9pzPA/G7n7jxPmuhc1DHMynPMrGL6AdewYmwu5ko+UUyTwrMv27rPH1v1nGqd87+p6N6LU8k3NEng53xXyHS97+44OSg/sy/hn+Se6yfYNjW0/uTgP+PvWYzLMmjhcLB/gGpri6H83/84eUXWT6T9Hsv7785z/7z4icpW+zfXypuR7rx/gMdZb1/wC678pcs8/2a3mDitGHxl9mfPlll5MafWWqxk/eYuTDgcNMzDGWLWvsuglNxs53GtN6uWpktlW1tZZYcuinMMWmnNnJydze3b2Y1McBxrBkXw799izLMZZYyy0TkbsGM4p03S2uVu5s/XXUdSdec6smVxZYYGpVmT8A+8ajuEyV5FatkvVru2x6uxGXXbH4A+jvgP4GMYy3iPLXzq/6z65+E005ey+cwMZD3fZcqc6xpjTFjQ0P3U+e++cPYmTIwj0nrK5NPTfl3WvpfLtXDcb2HQMudYOxFXQBor4L4T6vrOauFctYXJQ++NUWmJe5bmx1jDiZS1dTqWxo4GR8jm3fttpmPHppk9PEyv4/y8/sO07XacOmcqc0x2Vi9BvNJvN5oW8x4mOsydpidRxMYJPx06m1bqPzq9KtK8sxXNXFodD/+MYYaJTLwOhc9brCsV18oOR1i4tXChyTkq4lf4y1Ke+9axjDHqs1mfBbMXuP4Hzi+X7t8vzv7bHerrUPgPCxhjre4fXdfLNtNM+Jd+Zdh8xd8wP87uNPoPgv4W7/5P2BuxfsMabNnMnza+54Pdi5U671GPZY8CehX8Voeoo7FHpkeEc6715FwHZrIrUrHaviPUbPZHND+IhczrP6FcYvhOZ0Di/ETt0OI+YwNWR9r7tpf6WDeZKZDB1+z2IthOl1mPyb5FluvEx9h9d0NnM0Y1XPFkWIsk1WotJ0PBMmkvjvQTd0e71tfeV+8r8lQ/tpzpsmxJ+InrI/dj2UajUajVTUajatRqNRtGo1Go1Go4wjeMpZFMVV9CHbofPraLsJ3JpWV2XOoanCuFky4y3PPNxucK2uKC1Lbdb1eo+m5XomN6HfeZsabHLHRX/K+offtNGGmHWctcVcG44MdSqsOLY9VzX+Zxfxn2HPdWTpzWvkrtJ8M5zorrKcquRytJ5N5DZmcaW02l76nWO+BqPXm1A2Ry/0q71dH/mqrqeFjkYxjEXtsX8qubTk67rGycyqsdm4tZx5D6D5hhi0waaWmiaMP81Yjii5qxPlPuU/GfTL1Y5E6Jyfiq63qTa39A4J0sOGDgO9WF9bOXl0XfPRbsY2bPNKPy1YrFYrFYmRhhlTIyMjJWJYZHXuCXI8OoXsvfljGLFicNifpp2XunoPiG1wtx3p1Tah+/DD66OnVtVXP9rKbVxOnL0tR/rHtqB5UDErUVcl11D4qqvjpOcxX7armUNJB3LpW6bxVvD08e8h3odKKvyCFZBdSh2FVcST9xV3n3T8t1j7Kr9qgrqXg+13Pt5U7JCvFXVIV1YG5lRhkVYZJYYDDD4KOIMoHCp26WS8GB7uBh2zIdgq/PKyInjV2STShuoapUdCpX1yTwqq/z1VvET7Kh5nVPkO8YyxjLt2MaaMmWTLQvx3qnzltnXW0p2jxgbEtSny/Osv8Y9pLMXYoHVPAhkVdWVeODhR6q9/Sxe2liwwZWMVvFXfRkeIDxAePUPIrdJ4ey6yquzH+PD/bUOWAu05qVHtFd8rrKHSoeNIOUqrYr3FXyToqfYJgwmJdKpXXOwYYegNNGMzfZPp/t3t/DVs4zjNTN61rRqaWaa4NYbRjTa0tWwy2Y2tGN8ZO8ofNKq4j9SL7I+cSm4/6ovLV5HNXLI0jJidwrtk6ynCaP6Z++GjRlWS3tLeW129Mi9evxU9mtz6s5J3Z7M2ngTgnKvmpomxpaLCzPfmx0JWE+m3NLDDGOX47RctdYYNK5jakdqLkRlI39n590T5zctGSwwZZDJj6kW8XSi6ot2MmWWJ0DUT3nuvebBudScjZ79g8cWJ8av0k+/bE5WKd5MdbFpbDVMxu1DVMmtNZGJvq1mtRbn6M+g/kP0FwDwr7quZs7xosNGpbscyxhhd9TyJyFwbLcxlTasg75vW7TsV5K7ji44XPMMrdoj+Y3rT0Hie62nlYV/pwczzOmdLqLhYkzGMzCZWGMQzGMSsZYY6Di1t4nlJ+Em63mJxrVLxPbYxNEdgc1dU2iOKyoYYWjNrEeHTYybVk0atSa7ehuwsWMWTqn1TrnS6hYsi71d1+s+k+ic70e20fzE/VaTdxT9ZtU4GIXdeNx3X77guYYfpHeTQjaMX6brOu4OY4K7Y2d9mbHarI5ox3p4GpJ2Vd/Tst60f7j999pppjR+Q/Qf8J/VaORs3cji7FfFuN61+ui9s8hix1OCh5KGVV23BPXvZfz3CLyHpix+exi8z/KnCnosY2eunor+cxyPO/xJ0vKey9OvE9VjqaYu0x3Z3jd6o2b1T12D+F8l232lwaaacD5LE8LBxu7WTlbWraWpew8Xexjel3E+wWD4APITdNqR8F3R3T0lunCQ4GaE9R37DxeCYfcHi4xci5ovKfxVs55y2hf+65E/Xdp6jR5nrebTmi5incpkyOjs50JvrZwstbbW6kfuuQw+2mykf/EXNFzxfKTrxew929TR6bWnGL//F3JFOFCQT3K4lQ"
+
+    kernels = Kernel(
+        bz2.decompress(base64.b64decode(quantization_code)),
+        [
+            "int4WeightCompression",
+            "int4WeightExtractionFloat",
+            "int4WeightExtractionHalf",
+            "int8WeightExtractionFloat",
+            "int8WeightExtractionHalf",
+        ],
+    )
+except Exception as exception:
+    kernels = None
+    logger.warning("Failed to load cpm_kernels:" + str(exception))
+
+
+class W8A16Linear(torch.autograd.Function):
+    @staticmethod
+    def forward(ctx, inp: torch.Tensor, quant_w: torch.Tensor, scale_w: torch.Tensor, weight_bit_width):
+        ctx.inp_shape = inp.size()
+        ctx.weight_bit_width = weight_bit_width
+        out_features = quant_w.size(0)
+        inp = inp.contiguous().view(-1, inp.size(-1))
+        weight = extract_weight_to_half(quant_w, scale_w, weight_bit_width)
+        ctx.weight_shape = weight.size()
+        output = inp.mm(weight.t())
+        ctx.save_for_backward(inp, quant_w, scale_w)
+        return output.view(*(ctx.inp_shape[:-1] + (out_features,)))
+
+    @staticmethod
+    def backward(ctx, grad_output: torch.Tensor):
+        inp, quant_w, scale_w = ctx.saved_tensors
+        weight = extract_weight_to_half(quant_w, scale_w, ctx.weight_bit_width)
+        grad_output = grad_output.contiguous().view(-1, weight.size(0))
+        grad_input = grad_output.mm(weight)
+        grad_weight = grad_output.t().mm(inp)
+        return grad_input.view(ctx.inp_shape), grad_weight.view(ctx.weight_shape), None, None
+
+
+def compress_int4_weight(weight: torch.Tensor):  # (n, m)
+    with torch.cuda.device(weight.device):
+        n, m = weight.size(0), weight.size(1)
+        assert m % 2 == 0
+        m = m // 2
+        out = torch.empty(n, m, dtype=torch.int8, device="cuda")
+        stream = torch.cuda.current_stream()
+
+        gridDim = (n, 1, 1)
+        blockDim = (min(round_up(m, 32), 1024), 1, 1)
+
+        kernels.int4WeightCompression(
+            gridDim,
+            blockDim,
+            0,
+            stream,
+            [ctypes.c_void_p(weight.data_ptr()), ctypes.c_void_p(out.data_ptr()), ctypes.c_int32(n), ctypes.c_int32(m)],
+        )
+        return out
+
+
+def extract_weight_to_half(weight: torch.Tensor, scale_list: torch.Tensor, source_bit_width: int):
+    assert scale_list.dtype in [torch.half, torch.bfloat16]
+    assert weight.dtype in [torch.int8]
+    if source_bit_width == 8:
+        return weight.to(scale_list.dtype) * scale_list[:, None]
+    elif source_bit_width == 4:
+        func = (
+            kernels.int4WeightExtractionHalf if scale_list.dtype == torch.half else kernels.int4WeightExtractionBFloat16
+        )
+    else:
+        assert False, "Unsupported bit-width"
+
+    with torch.cuda.device(weight.device):
+        n, m = weight.size(0), weight.size(1)
+        out = torch.empty(n, m * (8 // source_bit_width), dtype=scale_list.dtype, device="cuda")
+        stream = torch.cuda.current_stream()
+
+        gridDim = (n, 1, 1)
+        blockDim = (min(round_up(m, 32), 1024), 1, 1)
+
+        func(
+            gridDim,
+            blockDim,
+            0,
+            stream,
+            [
+                ctypes.c_void_p(weight.data_ptr()),
+                ctypes.c_void_p(scale_list.data_ptr()),
+                ctypes.c_void_p(out.data_ptr()),
+                ctypes.c_int32(n),
+                ctypes.c_int32(m),
+            ],
+        )
+        return out
+
+
+class QuantizedLinear(torch.nn.Module):
+    def __init__(self, weight_bit_width: int, weight, bias=None, device="cpu", dtype=None, empty_init=False, *args,
+                 **kwargs):
+        super().__init__()
+        self.weight_bit_width = weight_bit_width
+
+        shape = weight.shape
+
+        if weight is None or empty_init:
+            self.weight = torch.empty(shape[0], shape[1] * weight_bit_width // 8, dtype=torch.int8, device=device)
+            self.weight_scale = torch.empty(shape[0], dtype=dtype, device=device)
+        else:
+            self.weight_scale = weight.abs().max(dim=-1).values / ((2 ** (weight_bit_width - 1)) - 1)
+            self.weight = torch.round(weight / self.weight_scale[:, None]).to(torch.int8)
+            if weight_bit_width == 4:
+                self.weight = compress_int4_weight(self.weight)
+
+        self.weight = Parameter(self.weight.to(device), requires_grad=False)
+        self.weight_scale = Parameter(self.weight_scale.to(device), requires_grad=False)
+        self.bias = Parameter(bias.to(device), requires_grad=False) if bias is not None else None
+
+    def forward(self, input):
+        output = W8A16Linear.apply(input, self.weight, self.weight_scale, self.weight_bit_width)
+        if self.bias is not None:
+            output = output + self.bias
+        return output
+
+
+def quantize(model, weight_bit_width, empty_init=False, device=None):
+    """Replace fp16 linear with quantized linear"""
+    for layer in model.layers:
+        layer.self_attention.query_key_value = QuantizedLinear(
+            weight_bit_width=weight_bit_width,
+            weight=layer.self_attention.query_key_value.weight.to(torch.cuda.current_device()),
+            bias=layer.self_attention.query_key_value.bias,
+            dtype=layer.self_attention.query_key_value.weight.dtype,
+            device=layer.self_attention.query_key_value.weight.device if device is None else device,
+            empty_init=empty_init
+        )
+        layer.self_attention.dense = QuantizedLinear(
+            weight_bit_width=weight_bit_width,
+            weight=layer.self_attention.dense.weight.to(torch.cuda.current_device()),
+            bias=layer.self_attention.dense.bias,
+            dtype=layer.self_attention.dense.weight.dtype,
+            device=layer.self_attention.dense.weight.device if device is None else device,
+            empty_init=empty_init
+        )
+        layer.mlp.dense_h_to_4h = QuantizedLinear(
+            weight_bit_width=weight_bit_width,
+            weight=layer.mlp.dense_h_to_4h.weight.to(torch.cuda.current_device()),
+            bias=layer.mlp.dense_h_to_4h.bias,
+            dtype=layer.mlp.dense_h_to_4h.weight.dtype,
+            device=layer.mlp.dense_h_to_4h.weight.device if device is None else device,
+            empty_init=empty_init
+        )
+        layer.mlp.dense_4h_to_h = QuantizedLinear(
+            weight_bit_width=weight_bit_width,
+            weight=layer.mlp.dense_4h_to_h.weight.to(torch.cuda.current_device()),
+            bias=layer.mlp.dense_4h_to_h.bias,
+            dtype=layer.mlp.dense_4h_to_h.weight.dtype,
+            device=layer.mlp.dense_4h_to_h.weight.device if device is None else device,
+            empty_init=empty_init
+        )
+
+    return model
diff --git a/Model_Architecture_Discussions/ChatGLM3/tokenization_chatglm.py b/Model_Architecture_Discussions/ChatGLM3/tokenization_chatglm.py
new file mode 100644
index 0000000..be1cbbf
--- /dev/null
+++ b/Model_Architecture_Discussions/ChatGLM3/tokenization_chatglm.py
@@ -0,0 +1,328 @@
+import json
+import os
+import re
+from typing import List, Optional, Union, Dict
+from sentencepiece import SentencePieceProcessor
+from transformers import PreTrainedTokenizer
+from transformers.utils import logging, PaddingStrategy
+from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
+
+
+logger = logging.get_logger(__name__)
+
+
+class SPTokenizer:
+    def __init__(self, model_path: str):
+        # reload tokenizer
+        assert os.path.isfile(model_path), model_path
+        self.sp_model = SentencePieceProcessor(model_file=model_path)
+
+        # BOS / EOS token IDs
+        self.n_words: int = self.sp_model.vocab_size()
+        self.bos_id: int = self.sp_model.bos_id()
+        self.eos_id: int = self.sp_model.eos_id()
+        self.pad_id: int = self.sp_model.unk_id()
+        assert self.sp_model.vocab_size() == self.sp_model.get_piece_size()
+
+        role_special_tokens = ["<|system|>", "<|user|>", "<|assistant|>", "<|observation|>"]
+        special_tokens = ["[MASK]", "[gMASK]", "[sMASK]", "sop", "eop"] + role_special_tokens
+        self.special_tokens = {}
+        self.index_special_tokens = {}
+        for token in special_tokens:
+            self.special_tokens[token] = self.n_words
+            self.index_special_tokens[self.n_words] = token
+            self.n_words += 1
+        self.role_special_token_expression = "|".join([re.escape(token) for token in special_tokens]) # for apply_chat_template
+
+    def tokenize(self, s: str, encode_special_tokens=False):
+        if encode_special_tokens:
+            last_index = 0
+            t = []
+            for match in re.finditer(self.role_special_token_expression, s):
+                if last_index < match.start():
+                    t.extend(self.sp_model.EncodeAsPieces(s[last_index:match.start()]))
+                t.append(s[match.start():match.end()])
+                last_index = match.end()
+            if last_index < len(s):
+                t.extend(self.sp_model.EncodeAsPieces(s[last_index:]))
+            return t
+        else:
+            return self.sp_model.EncodeAsPieces(s)
+
+    def encode(self, s: str, bos: bool = False, eos: bool = False) -> List[int]:
+        assert type(s) is str
+        t = self.sp_model.encode(s)
+        if bos:
+            t = [self.bos_id] + t
+        if eos:
+            t = t + [self.eos_id]
+        return t
+
+    def decode(self, t: List[int]) -> str:
+        text, buffer = "", []
+        for token in t:
+            if token in self.index_special_tokens:
+                if buffer:
+                    text += self.sp_model.decode(buffer)
+                    buffer = []
+                text += self.index_special_tokens[token]
+            else:
+                buffer.append(token)
+        if buffer:
+            text += self.sp_model.decode(buffer)
+        return text
+
+    def decode_tokens(self, tokens: List[str]) -> str:
+        text = self.sp_model.DecodePieces(tokens)
+        return text
+
+    def convert_token_to_id(self, token):
+        """ Converts a token (str) in an id using the vocab. """
+        if token in self.special_tokens:
+            return self.special_tokens[token]
+        return self.sp_model.PieceToId(token)
+
+    def convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        if index in self.index_special_tokens:
+            return self.index_special_tokens[index]
+        if index in [self.eos_id, self.bos_id, self.pad_id] or index < 0 or index > self.sp_model.vocab_size():
+            return ""
+        return self.sp_model.IdToPiece(index)
+
+
+class ChatGLMTokenizer(PreTrainedTokenizer):
+
+    vocab_files_names = {"vocab_file": "tokenizer.model"}
+    model_input_names = ["input_ids", "attention_mask", "position_ids"]
+
+    def __init__(
+        self,
+        vocab_file,
+        padding_side="left",
+        clean_up_tokenization_spaces=False,
+        encode_special_tokens=False,
+        **kwargs
+    ):
+        self.name = "GLMTokenizer"
+        self.vocab_file = vocab_file
+        self.tokenizer = SPTokenizer(vocab_file)
+        self.special_tokens = {
+            "<bos>": self.tokenizer.bos_id,
+            "<eos>": self.tokenizer.eos_id,
+            "<unk>": self.tokenizer.pad_id,
+            "<pad>": self.tokenizer.pad_id
+        }
+        self.encode_special_tokens = encode_special_tokens
+
+        super().__init__(
+            padding_side=padding_side,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs
+        )
+
+    def get_command(self, token):
+        if token in self.special_tokens:
+            return self.special_tokens[token]
+        assert token in self.tokenizer.special_tokens, f"{token} is not a special token for {self.name}"
+        return self.tokenizer.special_tokens[token]
+
+    @property
+    def unk_token(self) -> str:
+        return self.tokenizer.sp_model.IdToPiece(self.get_command("<unk>"))
+
+    @property
+    def pad_token(self) -> str:
+        return self.tokenizer.sp_model.IdToPiece(self.get_command("<pad>"))
+
+    @property
+    def eos_token(self) -> str:
+        return self.tokenizer.sp_model.IdToPiece(self.get_command("<eos>"))
+
+    @property
+    def unk_token_id(self) -> int:
+        return self.get_command("<unk>")
+
+    @property
+    def pad_token_id(self) -> int:
+        return self.get_command("<pad>")
+
+    @property
+    def eos_token_id(self):
+        return self.get_command("<eos>")
+
+    @unk_token.setter
+    def unk_token(self, value):
+        logger.warning("Setting unk_token is not supported, use the default one.")
+
+    @pad_token.setter
+    def pad_token(self, value):
+        logger.warning("Setting pad_token is not supported, use the default one.")
+
+    @eos_token.setter
+    def eos_token(self, value):
+        logger.warning("Setting eos_token is not supported, use the default one.")
+
+    @property
+    def vocab_size(self):
+        return self.tokenizer.n_words
+
+    def get_vocab(self):
+        """ Returns vocab as a dict """
+        vocab = {self._convert_id_to_token(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def _tokenize(self, text, **kwargs):
+        return self.tokenizer.tokenize(text, encode_special_tokens=self.encode_special_tokens)
+
+    def _convert_token_to_id(self, token):
+        """ Converts a token (str) in an id using the vocab. """
+        return self.tokenizer.convert_token_to_id(token)
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        return self.tokenizer.convert_id_to_token(index)
+
+    def convert_tokens_to_string(self, tokens: List[str]) -> str:
+        return self.tokenizer.decode_tokens(tokens)
+
+    def save_vocabulary(self, save_directory, filename_prefix=None):
+        """
+        Save the vocabulary and special tokens file to a directory.
+
+        Args:
+            save_directory (`str`):
+                The directory in which to save the vocabulary.
+            filename_prefix (`str`, *optional*):
+                An optional prefix to add to the named of the saved files.
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        if os.path.isdir(save_directory):
+            vocab_file = os.path.join(
+                save_directory, self.vocab_files_names["vocab_file"]
+            )
+        else:
+            vocab_file = save_directory
+
+        with open(self.vocab_file, 'rb') as fin:
+            proto_str = fin.read()
+
+        with open(vocab_file, "wb") as writer:
+            writer.write(proto_str)
+
+        return (vocab_file,)
+
+    def get_prefix_tokens(self):
+        prefix_tokens = [self.get_command("[gMASK]"), self.get_command("sop")]
+        return prefix_tokens
+
+    def build_single_message(self, role, metadata, message):
+        assert role in ["system", "user", "assistant", "observation"], role
+        role_tokens = [self.get_command(f"<|{role}|>")] + self.tokenizer.encode(f"{metadata}\n")
+        message_tokens = self.tokenizer.encode(message)
+        tokens = role_tokens + message_tokens
+        return tokens
+
+    def build_chat_input(self, query, history=None, role="user"):
+        if history is None:
+            history = []
+        input_ids = []
+        for item in history:
+            content = item["content"]
+            if item["role"] == "system" and "tools" in item:
+                content = content + "\n" + json.dumps(item["tools"], indent=4, ensure_ascii=False)
+            input_ids.extend(self.build_single_message(item["role"], item.get("metadata", ""), content))
+        input_ids.extend(self.build_single_message(role, "", query))
+        input_ids.extend([self.get_command("<|assistant|>")])
+        return self.batch_encode_plus([input_ids], return_tensors="pt", is_split_into_words=True)
+
+    def build_inputs_with_special_tokens(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """
+        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
+        adding special tokens. A BERT sequence has the following format:
+
+        - single sequence: `[CLS] X [SEP]`
+        - pair of sequences: `[CLS] A [SEP] B [SEP]`
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs to which the special tokens will be added.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
+        """
+        prefix_tokens = self.get_prefix_tokens()
+        token_ids_0 = prefix_tokens + token_ids_0
+        if token_ids_1 is not None:
+            token_ids_0 = token_ids_0 + token_ids_1 + [self.get_command("<eos>")]
+        return token_ids_0
+
+    def _pad(
+        self,
+        encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
+        max_length: Optional[int] = None,
+        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
+        pad_to_multiple_of: Optional[int] = None,
+        return_attention_mask: Optional[bool] = None,
+    ) -> dict:
+        """
+        Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
+
+        Args:
+            encoded_inputs:
+                Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
+            max_length: maximum length of the returned list and optionally padding length (see below).
+                Will truncate by taking into account the special tokens.
+            padding_strategy: PaddingStrategy to use for padding.
+
+                - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
+                - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
+                - PaddingStrategy.DO_NOT_PAD: Do not pad
+                The tokenizer padding sides are defined in self.padding_side:
+
+                    - 'left': pads on the left of the sequences
+                    - 'right': pads on the right of the sequences
+            pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
+                This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
+                `>= 7.5` (Volta).
+            return_attention_mask:
+                (optional) Set to False to avoid returning attention mask (default: set to model specifics)
+        """
+        # Load from model defaults
+        assert self.padding_side == "left"
+
+        required_input = encoded_inputs[self.model_input_names[0]]
+        seq_length = len(required_input)
+
+        if padding_strategy == PaddingStrategy.LONGEST:
+            max_length = len(required_input)
+
+        if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
+            max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
+
+        needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) != max_length
+
+        # Initialize attention mask if not present.
+        if "attention_mask" not in encoded_inputs:
+            encoded_inputs["attention_mask"] = [1] * seq_length
+
+        if "position_ids" not in encoded_inputs:
+            encoded_inputs["position_ids"] = list(range(seq_length))
+
+        if needs_to_be_padded:
+            difference = max_length - len(required_input)
+
+            if "attention_mask" in encoded_inputs:
+                encoded_inputs["attention_mask"] = [0] * difference + encoded_inputs["attention_mask"]
+            if "position_ids" in encoded_inputs:
+                encoded_inputs["position_ids"] = [0] * difference + encoded_inputs["position_ids"]
+            encoded_inputs[self.model_input_names[0]] = [self.pad_token_id] * difference + required_input
+
+        return encoded_inputs
diff --git a/Model_Architecture_Discussions/ChatGLM3/tokenizer.model b/Model_Architecture_Discussions/ChatGLM3/tokenizer.model
new file mode 100644
index 0000000..c8336ad
Binary files /dev/null and b/Model_Architecture_Discussions/ChatGLM3/tokenizer.model differ
diff --git a/Model_Architecture_Discussions/ChatGLM3/tokenizer_config.json b/Model_Architecture_Discussions/ChatGLM3/tokenizer_config.json
new file mode 100644
index 0000000..26c785e
--- /dev/null
+++ b/Model_Architecture_Discussions/ChatGLM3/tokenizer_config.json
@@ -0,0 +1,52 @@
+{
+  "added_tokens_decoder": {
+    "64790": {
+      "content": "[gMASK]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64792": {
+      "content": "sop",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64795": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "64796": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "auto_map": {
+    "AutoTokenizer": [
+      "THUDM/chatglm3-6b--tokenization_chatglm.ChatGLMTokenizer",
+      null
+    ]
+  },
+  "chat_template": "{% for message in messages %}{% if loop.first %}[gMASK]sop<|{{ message['role'] }}|>\n {{ message['content'] }}{% else %}<|{{ message['role'] }}|>\n {{ message['content'] }}{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant|>{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<unk>",
+  "padding_side": "left",
+  "remove_space": false,
+  "tokenizer_class": "ChatGLMTokenizer",
+  "unk_token": "<unk>"
+}
diff --git a/Model_Architecture_Discussions/ChatGLM3/下载模型权重.ipynb b/Model_Architecture_Discussions/ChatGLM3/下载模型权重.ipynb
new file mode 100644
index 0000000..54f657b
--- /dev/null
+++ b/Model_Architecture_Discussions/ChatGLM3/下载模型权重.ipynb
@@ -0,0 +1,37 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    ""
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/Model_Architecture_Discussions/ChatGLM3/加载模型权重.ipynb b/Model_Architecture_Discussions/ChatGLM3/加载模型权重.ipynb
new file mode 100644
index 0000000..270dd23
--- /dev/null
+++ b/Model_Architecture_Discussions/ChatGLM3/加载模型权重.ipynb
@@ -0,0 +1,500 @@
+{
+ "cells": [
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "开局先加载下一些必要的框架",
+   "id": "7e3cef5715e2ce4c"
+  },
+  {
+   "cell_type": "code",
+   "id": "c97dba1b574e2f06",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-12T09:04:33.586566Z",
+     "start_time": "2024-05-12T09:04:31.544911Z"
+    }
+   },
+   "source": [
+    "import torch\n",
+    "from safetensors.torch import load_model"
+   ],
+   "outputs": [],
+   "execution_count": 1
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fe213b65f1156255",
+   "metadata": {},
+   "source": [
+    "下面这3个就是咱们的代码实现了"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "id": "42511e94c3fadf60",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-12T09:04:34.207123Z",
+     "start_time": "2024-05-12T09:04:33.588447Z"
+    }
+   },
+   "source": [
+    "import configuration_chatglm_full as configuration_chatglm\n",
+    "from glm import ChatGLMForConditionalGeneration\n",
+    "from tokenization_chatglm import ChatGLMTokenizer"
+   ],
+   "outputs": [],
+   "execution_count": 2
+  },
+  {
+   "cell_type": "markdown",
+   "id": "eb69801a734637a",
+   "metadata": {},
+   "source": [
+    "接下来获取模型的配置\n",
+    "\n",
+    "这个配置包含了模型架构的一些信息，比如输入输出的维度，隐藏层大小等，如果这些参数对不上，那就加载不了他们训练好的模型了"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "id": "e06a39799fdf2c52",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-12T09:04:34.222644Z",
+     "start_time": "2024-05-12T09:04:34.208123Z"
+    }
+   },
+   "source": [
+    "config = configuration_chatglm.ChatGLMConfig()"
+   ],
+   "outputs": [],
+   "execution_count": 3
+  },
+  {
+   "cell_type": "code",
+   "id": "ab305759b378bfc5",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-12T09:04:34.237803Z",
+     "start_time": "2024-05-12T09:04:34.224636Z"
+    }
+   },
+   "source": [
+    "# 如果电脑配置实在太拉跨，可以解除下面代码的注释，减小下GLM模型的参数，但同时也加载不了现有参数了\n",
+    "# config.ffn_hidden_size = 512\n",
+    "# config.num_layers = 3\n",
+    "# config.hidden_size = 256"
+   ],
+   "outputs": [],
+   "execution_count": 4
+  },
+  {
+   "cell_type": "code",
+   "id": "a4466f3d5b727434",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-12T09:04:52.155679Z",
+     "start_time": "2024-05-12T09:04:34.238952Z"
+    }
+   },
+   "source": [
+    "# 对于像笔者这样显存不够大的电脑，就需要对模型进行量化了，这里对模型做int4量化，但如果你的显存小于6G，那就只能用CPU跑，或者减少模型参数了。\n",
+    "m = ChatGLMForConditionalGeneration(config=config, device='cpu')\n",
+    "# 如果前面改了模型架构，就不要执行下面的代码加载权重了，因为模型架构改了，权重就不对了。\n",
+    "# 设置 strict=False 是必须的，因为权重文件包含了一部分的权重，load_model方法默认会加载全部权重，不设置这参数就会直接报错。\n",
+    "load_model(m, \"weights/model-00001-of-00007.safetensors\", strict=False)\n",
+    "load_model(m, \"weights/model-00002-of-00007.safetensors\", strict=False)\n",
+    "load_model(m, \"weights/model-00003-of-00007.safetensors\", strict=False)\n",
+    "load_model(m, \"weights/model-00004-of-00007.safetensors\", strict=False)\n",
+    "load_model(m, \"weights/model-00005-of-00007.safetensors\", strict=False)\n",
+    "load_model(m, \"weights/model-00006-of-00007.safetensors\", strict=False)\n",
+    "load_model(m, \"weights/model-00007-of-00007.safetensors\", strict=False)\n",
+    "\n",
+    "m = m.quantize(bits=4, device='cuda')\n",
+    "m = m.to('cuda')\n",
+    "m = m.eval()"
+   ],
+   "outputs": [],
+   "execution_count": 5
+  },
+  {
+   "cell_type": "markdown",
+   "id": "81876dbd4ad42716",
+   "metadata": {},
+   "source": [
+    "## 模型的输入和输出\n",
+    "研究一个神经网络模型，最需要要了解的，就是模型的输入和输出。\n",
+    "\n",
+    "而当下的语言模型，输入是一个token序列，输出是各个token的logits概率。\n",
+    "\n",
+    "现在构造一个5个token的输入丢进去试试。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "id": "7e1abd57e3f5718a",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-12T09:04:52.170772Z",
+     "start_time": "2024-05-12T09:04:52.157608Z"
+    }
+   },
+   "source": [
+    "x = torch.tensor([[5262, 26267, 26236, 632, 6241]], device='cuda', dtype=torch.long)"
+   ],
+   "outputs": [],
+   "execution_count": 6
+  },
+  {
+   "cell_type": "code",
+   "id": "aae881aa889b06fd",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-12T09:04:52.186280Z",
+     "start_time": "2024-05-12T09:04:52.171689Z"
+    }
+   },
+   "source": [
+    "x.shape"
+   ],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "torch.Size([1, 5])"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 7
+  },
+  {
+   "cell_type": "code",
+   "id": "8b17fa208d3bf317",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-12T09:04:52.986064Z",
+     "start_time": "2024-05-12T09:04:52.188201Z"
+    }
+   },
+   "source": [
+    "with torch.no_grad():\n",
+    "    out = m(x)"
+   ],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "D:\\sys2\\llms-from-scratch-cn\\Model_Architecture_Discussions\\ChatGLM3\\glm.py:251: UserWarning: 1Torch was not compiled with flash attention. (Triggered internally at ..\\aten\\src\\ATen\\native\\transformers\\cuda\\sdp_utils.cpp:263.)\n",
+      "  context_layer = torch.nn.functional.scaled_dot_product_attention(query_layer, key_layer, value_layer,\n"
+     ]
+    }
+   ],
+   "execution_count": 8
+  },
+  {
+   "cell_type": "code",
+   "id": "cc971500ac83fc45",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-12T09:04:53.078756Z",
+     "start_time": "2024-05-12T09:04:52.986959Z"
+    }
+   },
+   "source": [
+    "out.logits"
+   ],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([[[-3.5352, -3.5352,  0.7764,  ..., -3.5352, -3.5352, -3.5352],\n",
+       "         [-2.1719, -2.1719,  4.2031,  ..., -2.1719, -2.1738, -2.1719],\n",
+       "         [-0.4954, -0.4951,  5.5820,  ..., -0.4946, -0.4951, -0.4951],\n",
+       "         [-2.7090, -2.7090,  4.4453,  ..., -2.7090, -2.7090, -2.7090],\n",
+       "         [-1.7568, -1.7568,  5.7383,  ..., -1.7578, -1.7568, -1.7578]]],\n",
+       "       device='cuda:0', dtype=torch.float16)"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 9
+  },
+  {
+   "cell_type": "code",
+   "id": "7e862f9c9c634034",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-12T09:04:53.094326Z",
+     "start_time": "2024-05-12T09:04:53.081272Z"
+    }
+   },
+   "source": [
+    "out.logits.shape"
+   ],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "torch.Size([1, 5, 65024])"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 10
+  },
+  {
+   "attachments": {
+    "image.png": {
+     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvIAAAFGCAYAAAAIKHuyAAAgAElEQVR4AeydB7hlNfn1UdQPURQBpTPUobeRJh2GDiogSpFepAkzAwNS/gLSO0gTBAGVIijSmRl6V2ZUmlRRemfoINLyPb9c1rk5+5R7+j373JXnyUl22k5W3rxZOzs7Z4pgYwSMgBEwAkbACBgBI2AEjEDuEJgidzV2hY2AETACRsAIGAEjYASMgBEIJvIWAiNgBIyAETACRsAIGAEjkEMETORz2GmushEwAkbACBgBI2AEjIARMJG3DBgBI2AEjIARMAJGwAgYgRwiYCKfw05zlY2AETACRsAIGAEjYASMgIm8ZcAIGAEjYASMgBEwAkbACOQQARP5HHaaq2wEjIARMAJGwAgYASNgBEzkLQNGwAgYASNgBIyAETACRiCHCJjI57DTXGUjYASMgBEwAkbACBgBI2AibxkwAkbACBgBI2AEjIARMAI5RMBEPoed5iobASNgBIyAETACRsAIGAETecuAETACRsAIGAEjYASMgBHIIQIm8jnsNFfZCBgBI2AEjIARMAJGwAiYyFsGjIARMAJGwAgYASNgBIxADhEwkc9hp7nKRsAIGAEjYASMgBEwAkbARN4yYASMgBEwAkbACBgBI2AEcoiAiXwOO81VNgJGwAgYASNgBIyAETACJvKWASNgBIyAETACRsAIGAEjkEMETORz2GmushEwAkbACBgBI2AEjIARMJG3DBgBI2AEjIARMAJGwAgYgRwiYCKfw05zlY2AETACRsAIGAEjYASMgIm8ZcAIGAEjYASMgBEwAkbACOQQARP5HHaaq2wEjIARMAJGwAgYASNgBEzkLQNGwAgYASNgBIyAETACRiCHCJjI57DTXGUjYASMgBEwAkbACBgBI2AibxkwAkbACBgBI2AEjIARMAI5RMBEPoed5iobASNgBIyAETACRsAIGAETecuAETACRsAIGAEjYASMwIAIfPrpp6GddsAKOEEJAibyJZA4wAgYASNgBIyAETACRiCLwCeffBI+/vjj8OGHH4b//e9/gWss1x999FFZkk8ZpH3nnXdiOq4pI01PGA8INvUjYCJfP2bOYQSMgBEwAkbACBiBnkcAci2yLhIPAf/ggw8iKVccRB0yLzKOK3IPSKR/6623wn//+9+YhjJE5CkDP65N/QiYyNePmXMYASNgBIyAETACRqDnEYBcQ9Lff//9aCHiXLOiLtIOCPjTa/K9/fbb0eInjjzkpYyUuBPGaj1k36Z+BEzk68fMOYyAETACRsAIGAEj0LMIiJRDvt97773w7rvvRqIN6YaEs9qO1co8QJCWa1zIO3mw+GWIh7RD5rWCj8s9KNumfgRM5OvHzDmMgBEwAkbACBgBI9CzCEDkIeAQb7bEpIScOFbo33jjjfDqq6/GVXfCIOevv/56dAGGMFnKgrBTDiv1kHnKIIw4petZQNvYMBP5NoLroo2AETACRsAIGAEjkDcERKxZfYfMy0K+WT2HtONPt8RoZV1bZ9I2s9oOgdfKO9dYlZtutUnz2T8wAibyA2PkFEbACBgBI2AEjIARGDIIQORZKcdiIOkQ8cmTJ0cLAZcR6ecaP2kh88pLOKSfvLgy2opDWDnyr3R2qyNgIl8dH8caASNgBIyAETACRmBIIQAh16q7CD0EXSvo+NO98vixkHO5KZEnjNV4SDsr8cpPGuVN0w8psJtsrIl8kwA6uxEwAkbACBgBI2AEegkBiDzEm1V4iHeWZHNNOGQfCzGXFQ6UQTp9/IqrPKTlmjQY0smv/HZrQ8BEvjac6kv1wnMh/O2e+vI4tREwAkbACBgBI2AEugABSDVEG8LNKjpu1iiNCDlkHJsaiDsfy1IG6cuRdeUpF5eWZX95BEzkhctjD/eRbwi47FmnhoC9+s9KVZs7Yr4QsBB6GyNgBIyAETACRsAI5AgBkWpINttpWJl/5ZVXwsSJE8OECRPCM888U2gNRJ40kH1W8e+9995www03hH/84x/hqaeeiqfbpA8CkPp//etfYdKkSbG8J598Mj40FAq0py4EhgaR337zELA/WDeEOaat345cri5Qw6039t3j4J/Vl8+pjYARMAJGwAgYASMwyAhA5CHoWm2HxN96661hu+22C4ssski44IILCqvvrLoTz8r7s88+G3bbbbeYZsyYMeG6666LR06mzeEh4NBDDw2bb755GD16dPjzn/8ct+ekaeyvHYGhQeRH79JH5EXoudZqO2Qbck8cK+9ajZebXVUnvJGHgUp5KM/GCBgBI2AEjIARMAJdggBEntV4iDwfo3L++6OPPho222yzMMUUU4SDDjooPP/884X98xxDCUGHuK+00koxzYorrhhOOeWUuJpPs7QqD9nfaqutwsILLxy22WabcOmll8ZyuqTpuavG0CDy1bpFxBxiX4uB2OshIOuynQbCnj4oZNNkr7MPCrXUwWmMgBEwAkbACBgBI9BmBCDzbIWR2XfffcOUU04Zdtxxx3DHHXeE1157TVFxSw0r7UsvvXSYZpppwrBhw8KoUaPisZMk4mEA88ILL4T1118/zDnnnIHyWOnngcGmMQRM5Osl8pVw1sp+ue007L/H2hgBI2AEjIARMAJGIEcIsCIvc/TRR4fZZ589bou58MILw9NPP62ocP3114dtt902bLLJJmGttdYK0047bSTsL7/8ckzDPnq24bA3fsSIEZHon3/++bGM9B6FAu2pCQET+VYQ+Woknm5g2w6r9fV+NFtTFzqRETACRsAIGAEjYARahwBba2TTUs8666zwne98J6y33nrh8MMPD48//nghmn3zq6yyShg7dmw47LDDwre+9a2w3HLLhSeeeCKmgayzGs9WmnnnnTcstNBC4e67745beIjjfjb1IzA0iHyl/en1hFfCdiAST7633+r/0NZkvhKSDjcCRsAIGAEjYAS6AAFINSvo2X9cveiii+Jq++KLLx623HLLIiJ/8sknh9lmmy3un+cD1hVWWCGsscYa4fbbb4/baiiLk2yOP/74sMACC4RVV1017runuWzhMZFvrOOHBpHP7ktPr0XEWTVPw7P+LL7sbec0Gz0M6EPaSm56Yo7JfBZNXxsBI2AEjIARMAJdgkBK5NkOI3PLLbeEn/70p2GeeeaJK/P33XefosIhhxwS98afcMIJ4YEHHojbbL73ve+F8847LxJ+iDwfw+6yyy5hscUWix/O8uErxiS+AGPdnqFB5KvB0szWmpTIQ+grkXiFH39EP/k3ma/WK44zAkbACBgBI2AEBgkBiDWWlXJOmxHRhnhffPHFYcEFF4yr77fddlus4eTJk8M+++wTZphhhvD73/8+nlRzxBFHhE033TTsvffegQcAyoDUs0q/zDLLxKMnX3rppZhf5Q9Sc3N9WxP5Zoi8PmBl6wzlVDqBJo0nj/fL53rQdE3lkTfkzsYIGAEjYASMQBsQ0Kk1bLPBsJf9/vvvL5xM84c//CEeO8led86PZ9/7lVdeGdOyDecnP/lJ+P73vx/4MBbDaj3HTq677rpxi40+hDWRj/A09NPbRF5EB7JTyV54ft/2GLbYVEqj8EpEnXhW5NmOU85k4yH2NkMTAR7kJE9ytY2r3rc0PBBiK8nl0ETYrTYCRsAIGIEWIcCxkPzRE+fEQ+oxHDm59tprF7bRXHvtteH000+PfxbFsZI33nhjTHfXXXeFAw44IJ5Qc9xxx8Wwgw8+OK7k77TTTvGPoF588cW4Um8iH+Fp6Ke3iTwESXvYW+EORNT5R9dyJkvky6VxWP4R0Baq9HuIeuTO/yCcfxlwC4yAETACPYQA5J297Zwlj4t588034x85zTXXXGH//fePK+u77757DOPceI6XxLDafsYZZ4S555477qt/6qmnAgT+m9/8ZuAYy0ceeSSeMe8TayJcDf/0NpGvtiKvj1whXVoZHcilPFZNRdjkirjhKix1FQ9RS8Px17sK23BXO2PbEfA/CLcdYt/ACBgBI2AEOocAK+VY9snrD50g8gceeGDgn1u32267SOBHjhwZP26FuKdHUl5++eVhjjnmCN/97nfjCjzny7OPnm03OhVHe/C9Kt9Yv/Y2ka+EifapQ6zTbS74IfjVyHU5Iq+PXrMkXdcm8pV6YuiE1/tWhodGbbnJumynYaXf/yA8dOTHLTUCRsAIDAICIte4rJxjWJnno9XNNtssknlOsJlpppnC9ttvH/j4Nf2315tvvjnMP//8YdFFF4375Vmdn3HGGcO4ceNiWSoXMu9/d22sg4cekReJhwzhTw1EXqS7GplP8+BXnmy4ruslccpnt3cQaJUM6E0SbtYgz1mZzqbxtREwAkbACBiBARCAYGPZWiPLNQZCD0HnNBqI+RRTTBGmnHLKwPYa/uk1Pa5y4sSJ8bx40i211FJxWw1HT955552FGkDi2bqDa1M/AkOLyFcj8cKuXjIvglaOWKlMpam0x17p7PYuAq2QgWokHuR4A8QDaj0Pob2LuFvWCALajthIXucxAkagZxDQSjmknBV4tsFoxRz3oYceinvjZ5555kjkp5pqqrhfXttvBATpOM1mySWXDLPOOmv8R9ctttginnyjNNyDj2nTBwDF2R0YgaFD5CFSkJxyK/EpTkxkfLSqjxSrkSI9GJCWfJVMK0hcpbId3j0ISGaacSu1ZiAST756H0Ir3cvh+UIAPYSOSa22Y1XTX+VaKR1ZTZ+Vy+cwI2AEegoBiDyEnVVySDwkW0SeFXm2z/DvrSNGjAhTTz113Dpz2mmnFU62ERhPPPFEOPPMM+Me+VlmmSWwl54TbJ588kklieV6Rb4AR92eoUHk2UsscqX97LoeyK1E/CH7xJF/oMnSRL5uwcxlBpGncq6IOKvm5eIVlm04hCqVWX13UcnVNq9a5DJ7L193HwLq57RfB9JZaTyyU4/RIka1N4z1lOe0RsAI5BIBiLwsBB6ra4g8220efPDBsPPOO8cz4ffcc88wfvz4QluJZ3WeP5DiGEpOt1l22WXD2LFjw6233hpPq1FilYtrUz8CQ4PIQ5KY3DQplvtIkAksXdHCr0kNwp6uUOnBgPCBSDx9YiJfv2T2Wo5mZCAl8qkcS56zrv9BuHekB12T9m+qu9KHQ/RQVn+lOivVQynRb8bP/WyMgBBA3iwTQqNnXMg1pDwl21xjXn/99bhXnj+AuuOOO+IfQ6nhpGFLDifccFY8e+IvuOCCmI5jKbWNxuRdiDXuDg0i3zg+fUSdP41KDa+ymVCzE2WaJvWTnsm4FtKf5rO/dxBohsgjPxi2zlBOJblL48lT64Nm76A8tFpSr0whN3rzk3X1djF9UMimyV5XksOh1Qv5bi16AjlKrfq53vkKGcJaLvItE03WPiX8WskX8c8WnaaVP5vG1wMjYCI/MEZOYQSqI8DElU6E5fz+B+HqGDq2fgSQM1bUIV7NGK3sl9tOA9HTg2Qz93DeziOgNzneltV57IfwHSHkEPdaSHyahjzktakfARP5+jFzDiNQjABEqpktCtm8lYiZiBtbvsoZxVfKXy6Pw/KLQCv6uxqJBxnIoN/s5FNGvC0rn/2W81qnRB5yniXoabyIPG42Xc5h6Gj1TeQ7Crdv1pMIVFuRF1FiVQziVYulPF5ra0VNrlbWcBWWuopnT30ajr/e1+Q92VE5bFT2Ia+R60rNlmyWW4lXHp+EJCR6y633IRCdxAJBOettWb0lG022JiXqIufpSrviiROBL5euyWoMqewm8kOqu93YjiKgfeoQawiRDH7IUzVyXY7I66PXLEnXtYm8EO4dtxxxUpiIOP2vsHJuFg1ImWSJBwPJTyVXckXaajKbvY+vuxeBeol8pZZIBss9DHpbViXUejpcRB2SniXwioO4yxIm29PAtLFxJvJtBNdFD2EEROJZrcruMW50lVOEqhKsrZqcK5Xv8O5CoJn+Tol8LWTeJyF1V983W5tmZEf3rkbiScODobdlCa0h46ZkHb+MVuC1Ci/ynrpKa7c+BEzk68PLqY3AwAhUI/HKXS+Z18RbbuVLZSoNq7I2vY9AM/2th0vkkHJYpS9n0njJtVflyyHVfWGNbMPK5qnUqoFIPPnq1XGV7uXwXCGQJfIi8Bw3qX+I1Tn0ahh5RPAVZrd2BEzka8fKKY3AwAhAiliFKrcSn+aGOOl/CgbasiACRbpKhIuymyF2ad3sH3wE6Gf6s5r1SUiD30/dXINy26wUJiLubVnd3IO5rFu6wi6CDonnn1vfe++96HINwZeBxBOGa1M/Aiby9WPmHEagPAKcEqEVrezWBYVXcisRf8g+ceQbaCXURL58v+QxFMJVSVYaCa/0lkYy45OQ8igljddZ/V5JLqqVnNVtlb6tULi3ZVVDs+fiIO+pScl8pRV5Vug/+OCDgGtTPwIm8vVj5hxGoDwCIl+awMr9uQ6EKbvKqpV5CHu64q4HA8IHIvHUqJnJuXyLHDpYCFRbkddqKt9MZGWp0jXlIUOSTbn67gJXYamreMhbGo6/FpkcLPx83+oINKMrvC2rOrZDPBbiru00Kakn7MMPP4xknZX3NM4r8s0JjYl8c/g5txFoDQKQIv+DcGuw7OVStM0KYs0eZBn8EPxq5Lockdfqapak69pEXgjnx632EKgHPW/Lyk9/5qymEHS20bz77ruFFXbCylk1DZKf3TevOLsDI2AiPzBGTmEEjIARGHwEROLLbcNq9MNCEfVKrWtm5bZSmQ5vLwJ6M9jIFqxyeSptv5FseFtWe/szZ6VD2N9+++0wefLkSOjTvfA0hXjCsCL3+FmVz6bNWdMHrbom8oMGvW9sBIyAEagRgWokXkXUS+ZFxFjJr2SUphKZq5TP4YOHQLUVeW/LGrx+GSJ3hpy/9NJL4T//+U949dVX40k1hMlA2NMtNiL1JvJCqH7XRL5+zJzDCBgBI9A5BCDTrMKXW4lPawGB0/cWrKxW22ajBwPSka+SMZGvhEz+wtXn3paVv74bxBpDwkW88Wv1HH9q0ypedtll4fDDDw833nhjeO655+I2G9JiKIvTa3AVJjKv67Qs+wdGwER+YIycwggYASMwOAjog2cIt/azl9v+UC6sEvGH7BNHnmpknxabyA9Ov7f6riLx5WSi3jc5qpu3ZQmJnnRFqiHZ7HnneEjCtKJOONdZEk66nXbaKcw+++zhgAMOCLfffnt4+eWXYz7SczrNO++8U3H/fE+C2eZGmci3GWAXbwSMgBFoGAHtd9bHpz4JqWEoh2zGaiReoNRL5vWA521ZQrDn3JS08yEqFhKPq9V0SDzEXR+3vv7662HSpElhvfXWC1NMMUVYcMEFw0EHHRTefPPNiA8kHqv8lMfqPMdS6sGg54DsQINM5DsAsm9hBIyAEeg4Aqy2+ySkjsPeVTf0tqyu6o48VQYiD+HWHzWJxHMtIg75hoRjOaVm4sSJ4ZhjjglLLLFEJPKQ+bXXXjs8+eSTsekQfh4EZCjnjTfeiB/HUr5NYwiYyDeGm3MZASNgBIyAEeheBLwtq3v7Jgc1g8hjIN6cQsPKOcQd8q2tNpBvwmQuvvjiMGLEiDD11FMXiPxiiy0Wfve730UynyXrlP3WW28V7aFXWXZrR8BEvnasnNIIGAEjYASMQD4Q8LasfPRTl9ayHJGnqhB5Vt9ZmYfEa4WdVfnDDjssfPGLXyyQeFbkhw0bFrbccstw1VVXFVpKHki9ymK7je5XSGRPzQiYyNcMlRMaASNgBIyAERgCCHhb1hDo5OpNhFhDuCHbkG4Rba3IEweRZ3X+mWeeCTfddFPYZpttwuc+97lI5D//+c+H6aefPsw444zhm9/8ZvjFL35RuCGr+9obzwMBZan8QiJ7akbARL5mqJzQCBgBI2AEjIARMAK9jwDEWvvf1VpId7pHXuEPPPBA+PnPfx6+/e1vF63GTzPNNOHLX/5yDNt2223DU089FR8MeBiAyGvPPEReK/wq027tCJjI146VUxoBI2AEjIARMAJGoOcRgMhDttlGgx+yjb/cNhiOmFx55ZXDV77ylSIiz9Ya7JRTThnWXHPNcP755xc+fIW486Cg7TncizCb+hEwka8fs/I5bhzf92cs5WMdagSMgBEwAkbACBiBXCAg8q4TaSDakPj0g1WuX3jhhXDaaaeFmWeeuUDi2VYz1VRTFbbZQOa//vWvR7J/7bXXFtrPwwGGe0HidV1IYE9NCJjI1wTTAIk+/DCEFZfo+8OW5AvuAXI52ggYASNgBIyAETACXYsA5Jrz4TldRgbijZk8eXLgX1y32GKLwkeunFgDaZ9hhhni3vhpp522EAehP/roo1VMdFmRpzyVWRTpi5oQMJGvCaYBEp1yXN+/JPJPiRecO0BiRxsBI2AEjIARMAJGoPsRYAWef2JlPzsGwq1V+WeffTZsvvnmhX3wc889d7xecsklw6yzzhpGjhwZ1lprrTDPPPMUVuu33377cN999xXKS8vufjS6s4Ym8s32y8svhjB85n4iv+S8Ibz/XrOlOr8RMAJGwAgYASNgBAYFAQg7q+WQdra9aOuL9rRTqfvvvz+svvrq4Rvf+EY8P37fffcNl1xySdhkk03CfPPNF3bYYYf4Eewee+wR1l133bDIIovE1ftzzjknPP/887Fd7LvH6uFgUBqb85uayDfbgbtu00/iWZHHHt1/zFKzxTu/ETACRsAIGAEjYAQ6iQBEnj3w+iCVk2ZYPWerDaT7ueeeC+PHjw9jxowJBx98cJg0aVKMZxvOXnvtFRZffPHoXnnllfF4yocffjiS/LFjx4bRo0eHhx56qNAcHhJY8eceNvUjYCJfP2b9OSbeXUriIfLzzRjCSy/0p7PPCLQCgTtvC2Hc1a0oyWUYASNgBIyAEaiIAEReK/EkgrzruEhW5V966aVw7733hmuuuSZMnDixqJz9998/EnlI+2233VaIe/HFFwPE/txzzw1PP/10IZyyeWDwx64FSOrymMjXBVeS+OOPQ1h9mfJEHjK/x45JYnuNQAsQ2GC1EEbMF8K777SgMBdhBIyAETACRqA8AhD5gay23kDESYthVZ1V+vnnnz+uvE+YMKGwH544Hg5wyStDXq5VhsLt1oaAiXxtOJWmOvv0yiReW2wevK80n0OMQCMIcLyp5Gq/0Y2U4DxGwAgYASNgBNqKACSdrTPDhw8vIfLZG4u448pm0/h6YARM5AfGqDTFa6+GsOBs/cRKBCvrbrhmaV6HGIFGEFhrhWJ5+3vxq8xGinQeI2AEjIARMAKtREBEvtyKPCv3WnkXiefeIvFpWCvr1Otlmcg30sN77dZHqpaaP4TlFysmWJD5RYb1h117ZSN3cB4j0I/ANVf0y5MeFlf5dggffNCfxj4jYASMQKsQuPXGEG64rlWluZwhhICJfOc720S+Xszv/VsI880UwnGH9x0zueOPS0nWpReGMOHaENZcvo/o13sPpzcCQoB9hKstXSpjEPrjj1Aqu0bACBiB1iCAzhm5XAjLLuzFgtYgOqRKgchrj/yoUaNCukfeK/LtEQUT+XpxPfOUEJ5/tj/XT3coJVnX9/8FcfjzJX2kvj+HfUagdgQuu7hUvrQqP9f0Ifz7X7WX5ZRGwAgYgYEQuPC8fp1z7GEDpXa8EShCIEvkOaJSfyZlIl8EVcsuTOSbhXLMrv1KTwTrrtubLdX5jQDnfYWw4uKl8iU5w/3+GmwwNFpGYGAEbr4+BKyNEaiEAH9myJ8aSsfMPUMITz9ZKbXDjUAJAlkiP27cOBP5EpRaG2Ai3yyenCAipSf3AZ9W0yyszh9CSFfGJFvl3PPOMlxGoDoCPBSutGSfxW9jBMohwAp8VsdsukG5lA4zAmURyBJ5r8iXhamlgSbyzcJ58M9KFZ9XMJpF1fn5kHWpBfpka+E5SmWMyZYJFnf+WUJ44TljZgQqI5Ael4vfxghkEeBPDPkzwyyR5/qqy7KpfW0EyiJgIl8WlrYGmsg3C++RB5UqvtcnN1uq8w91BH7zqxDmnC6Eg/YNYfJrpTLG5Pq//4XAvwtD6H+80VBHzO2vhMCbbxQfl8vRuYTZGIEUgVE/Ka9n0DXfHh7CO2+nqe03AmURgMhzjryPnywLT1sCTeSbhfWEI0uVX/KPZc0W7/xDEIH/vh/CrtuG8NjD/Y1fNDnSVCtmH33UH3/PXSE8/VT/tX1GQAjsN6pUR/lPxYSOXRBgO6j0SiX3F/sbKyMwIAIm8gNC1PIEJvLNQnraCcUKcIFZmy3R+Yc6Au++U4rAyGWL5YzJ9sMPS9M5xAikCDz+SKnciKgRZ2MEQIA/L5RcVHJ5Q/jIP42XEaiKgIl8VXjaEmki3yys6d5TFODSCzZbovMbgVIENvtu6UTrP4QqxckhxQhssl6p3IioEWdjBPjPE8lE+meGCltukRBW+OyPDzday3gZgaoImMhXhactkSbyzcL627P7lSCKjz/SsDECrUZgz52K5QxZM5FvNcq9Vd41l5fKjMiZXP/zdG/1eb2t4a0e/07OG7/rrur7qFWyIXe7Tfve/nGKFn8SdckF9d7F6YcQAibyne9sE/lmMf/D74sny43XbrZE5zcCpQgc/n/FcsYkayJfipND+hBANlhJFRmr5JLGcjR0pYY/L+RP5/Rd1x23lMrMbtv148MH9pxgo/T9MfYZgYiAiXznBcFEvlnML7+0WPFt88NmS3R+I1CKwK9PK5YziBmTqo0RKIfAycf0yQvf7Ky5fKnsEEYccsR3PjZGAAQeuLdUVkbvYmyMQM0ImMjXDFXLEprINwslryPT1a49dmy2ROc3AqUIXPHHYjkzkS/FyCF9CLz8Yh9J/9mefUeXHnd4qewcf0Rf3D4/7UtLHhsj8MzTpbKy/xjjYgRqRsBEvmaoWpbQRL5ZKG8cX6z4Dty72RKd3wiUInD3HcVyZiJfipFD+hA4/9fFR5f+6pelsnPmKf1oPfRgCL//Tf+1fUMXAc6KTxem8B96wNDFwy2vG12zJ44AACAASURBVAET+bohazqDiXyzEN5xa7HiO/oXzZbo/EagFIEnHi+WMxP5UowcUh4BPlLMkrMLzy+f1qFGICsrxx5mTIxAzQhA5MeMGRP/EGrUqFFhwoQJ4f3334/5P/744/DJJ5+ETz/9NFoVqmtcm/oRMJGvH7PiHJP+UjxJnnFycbyvjEArEHj7rWI5M5FvBapDo4wr/1QqO1f/eWi03a2sH4ER8xXLy6nH11+GcwxZBLJEfvz48SbybZYGE/lmAc5+HHTBuc2W6PxGoDwCc89QPMH6Y9fyODm0GIGbry+WGx4Cb72xOI2vjIAQyP753DlnKMauERgQgSyRHzdunIn8gKg1l8BEvjn8+vaipq8iOZrLxgi0A4EVlygmZCby7UC598qc9NdiuUFf/e2e3munW9QaBH6wbrG8eHGqNbgOkVKyRN4r8u3veBP5ZjF+8t/FSs8rXc0i6vyVENh4nWJZ489cbIzAQAg89nCx3EDkH39koFyOH6oI7LBFsbxwzryNEagRARP5GoFqYTIT+WbBfOG5YqX394nNluj8RqA8ArtuUyxrJvLlcXJoMQLPP1ssNxD5F58vTuMrIyAExu5eLC/XXKEYu0ZgQASyRN4fuw4IWdMJTOSbhXDya8VK71+PNVui8xuB8ggc/LNiWTORL4+TQ4sRKPeh9LvvFKfxlREQAkf8vFjP3DRBMXaNwIAIQORHjx4dT63BNZEfELKmE5jINwshE2K6R/7ll5ot0fmNQHkE+AfOVNZM5Mvj5NBSBFK5GfaN0niHGAEhcMZJxXqG/7CwMQI1ImAiXyNQLUxmIt8smB99VKz0/tt3XmqzxTq/EShB4NILi2XNRL4EIgdUQGDhOfplZ9FhFRI52AiEEC76bb+s8AD4j0mGxQjUjICJfM1QtSyhiXwroNRql1e6WoGmy6iEAB9SS9ZwTeQrIeXwLALLLtwvO99ZNBvrayPQj8B1V/XLCnrmkX/2x9lnBAZAwER+AIDaEG0i3wpQh8/cp/iWnLcVpbkMI1AegYcfLJ5geRtkYwRqQWCN7/TLzlor1JLDaYYqAn+5s19WIPJP/WeoIuF2N4CAiXwDoDWZxUS+SQBj9kXn7FN8K49oRWkuwwiUR+C1V4snWBP58jg5tBSBjdfulx3OCbcxApUQePShflmByL/0QqWUDjcCJQiYyJdA0vYAE/lWQLzUAn2Kb4PVWlGayzAC5RH49NMQ0n93NZEvj5NDSxHY5of95Gy7TUvjHWIEhMDLL/bLCkT+zTcVY9cIDIiAifyAELU8gYl8KyBdYbE+xbfFhq0ozWUYgcoILLNQ/yRrIl8ZJ8cUI7D79v1ys+dOxXG+MgIpAvxjNARe1v8gnaJj/wAImMgPAFAbok3kWwHq6sv0Kb1dtm5FaS7DCFRGgLc+mmBN5Cvj5JhiBPYb3S83B+5dHOcrI5BFYP5Z+uUlG+drI1AFAYj8mDFj4jnyo0aN8jnyVbBqVZSJfCuQXHelPqW37x6tKM1lGIHKCGy/ef8EayJfGSfHFCNw5EH9cnPUIcVxvjICWQSWW6RPXhaYNRvjayNQFYEskR8/fnx4//2+Y7k//vjj8Mknn4RPP/00WhWka1yb+hEwka8fs9IcG67Zp/QOO7A0ziFGoJUIpCurJvKtRLa3yzr1+H4if/qJvd1Wt655BLQ4tcQ8zZflEoYUAlkiP27cOBP5NkuAiXwrAN50g75J8pfHtqI0l2EEKiNw0tH9hOzjjyunc4wRSBE476x+ufndOWmM/UagFIHNv9cnL/7PgVJsHFIVgSyR94p8VbhaEmki3woYt/pBn9JjsrQxAu1E4MLz+gmZiXw7ke6tsi+7uF9uLr+0t9rm1rQegV237ZMXvv+yMQJ1IGAiXwdYLUpqIt8KIHf8cZ/SY7K0MQLtROCG6/oJmYl8O5HurbKvv7Zfbm4c31ttc2taj8ABe/XJy3ort75sl9jTCGSJ/IQJE7y1ps09biLfCoB1tBsky8YItBOB+//RT8hM5NuJdG+Vffcd/XIz8e7eaptb03oEjju8T142Xqf1ZbvEnkYAIj969Oh4ag2uiXz7u9tEvhUY77Vbn9K7565WlOYyjEBlBF58vp+QmchXxskxxQg8eH+/3Dz8YHGcr4xAFoGzT++Tlx9vlI3xtRGoioCJfFV42hJpIt8KWPcf06f0PEG2Ak2XUQ0BjufSOfIm8tWQclyKwNNP9svNs8+kMfYbgVIE/nRRn7zssEVpnEOMQBUETOSrgNOmKBP5VgB78H59Su85T5CtgNNlDIDAkvP2ydsnnwyQ0NFG4DMEXp/cT+TffNOwGIHqCPAdBQsGP92hejrHGoEMAibyGUA6cGki3wqQjzq4T+m9/VYrSnMZRqA6AmuvaCJfHSHHZhHgoU9vcvwAmEXH11kE/j6xT17G7p6N8bURqIqAiXxVeNoSaSLfClhPPCqEYd9oRUkuwwgMjICOOzUhGxgrp+hHYL6ZQlhwtv5r+4xAJQT+80Qfkf/5PpVSONwIlEXARL4sLG0NNJFvBbz8U+Iiw1pRksswAgMjsPdnH1ebyA+MlVP0I/Dt4SEss1D/tX1GoBIC2op15EGVUjjcCJRFwES+LCxtDTSRbwW8fOHvf8BrBZIuoxYEjjnUW2tqwclpihFYdakQRi5bHOYrI1AOAX1Uz9tmGyNQBwIm8nWA1aKkJvKtAJK/PGffso0R6AQC555pIt8JnHvtHhusFsKGa/Zaq9yediGw6JwhnHFyu0p3uT2KAER+zJgx8Rz5UaNG+Rz5DvSziXwrQL7kghB+uH4rSnIZRmBgBK65wkR+YJScIovAFt8Pge8rbIxALQisPCKE886qJaXTGIECAlkiP378eP+zawGd9nhM5FuB6xV/DGH7zVtRksswAgMjMOkvfUSe1982RqBWBHbeKoTdtqs1tdMNdQS+NzKEi3831FFw++tEwES+TsBakNxEvgUghnFXhzB6l1aU5DKMwMAI6M99TOQHxsop+hHgKMH9RvVf22cEqiGwzQ9DYJHKxgjUgUCWyI8bN84r8nXg10hSE/lGUMvmuWlCCAf/LBvqayPQHgQ+/NAr8u1BtrdLPWT/EA7/v95uo1vXOgRG7xzChGtbV55LGhIIZIm8t9a0v9tN5FuB8Z23hXD8Ea0oyWUYgdoQWHiOELwiXxtWTtWHACeQ/PJYo2EEakOAB7/bbqotrVMZgc8QyBL5CRMmeEW+zdJhIt8KgCf9NYRfn9aKklyGEagNgdWWNpGvDSmnEgIck8uJRzZGoBYETj4mhIl315LSaYxAAQGI/OjRo+OpNbgm8gVo2uYxkW8FtA/cF8Ifft+KklyGEagNgU03MJGvDSmnEgLoqD9dpCu7RqA6Ar89O4QH7q2exrFGIIOAiXwGkA5cmsi3AuTHHwnh2itbUZLLMAK1IbDHjrWlcyojIAQ4ttR7noWG3YEQuOqyEB5/dKBUjjcCRQhoa80CCywQz5P3inwRPG256CiR//TTT0M1m7Ywmy6N6zr/U/8J4Y5bB61awuqTTz4JWK7LGaWTWy5NN4Wpnqmr9qVhWX+2DWl8tTily6bpyuvDDmxJtdRmXJk0TH7F9bqr9tbr5gKX228O4e47Ol7VclhqHGcrk6bNxnXzteqd6mCFlXO7uS2Fut1xSwjPPlO47KQni5nuTXgl2VGaXndrwSDFr9N4QOT5I6jhw4dHN0vkP/7445I+bLa+1fKncVl/ik02Lr2WzKVh+LvFdJTI/+9//wt08kcffRTwf/DBB9HlOmsACfCI6ybAsvWM1y8+P6ivIMEHnN59993wzjvvRH+5egpTCWW5NN0Ulq0vsvPee+9FmSEOhUC7CcdfyZBWNpsmLUcKJpumK6/POrWl1QIHGWGFnORFVlT3Zl3ajjyhn7DIBGHSRdJZXBP33//+N6Zr9r4dyX/v30JgG2CHDViBpXQ5GKKrwDlrhDVungxtpF1vvfVWePvttwv6SDqF9tB+YZCLtiErr74yKFUFL3GEVBbef//9iG852RmUig7CTWk74wd8KhkwQyZT7CqlbXU49dttt93CHHPMEfbcc8+iPfLSm9Krujf1lFVYPW619qpc3Ru3EjaMT8kd7SAv11l9pTLrqWM703aUyANMalPAUoUnkAQ8111tXp8cwpP/7ngVwUUYgSUkHguWGGGqiil9JSFWum5x0/pSJ7WRCZM4DTrC8dNetZk2DmQpA0se8mPJkwtz+aUNVVNtpp3ZditOrtIIR8J72ajdyIEIu2QCrAiDSEDeSUsY18hfLswTj4fA28MOGTBCdoQnLmHgiJ5iLsCAI1byJsw7VM2mbqP20RbaBYkvp4NpN3JCGvLkwrAa/+47g1JV8GKcYVO8GG88LGnMkQ5LmjTdoFS6QzdF1iCWyJIMGDCGNHZSv9J0yqV+P/3pT8Ncc81VsiJPH6V1U/+lbiP1lAzgZo3KrnRv6iOrcYzc4cdoXOta9S93r+y9O3XdUSKvRglYAQHAmiA1GNM0Sqf8Xee+924Ik1/reLUQKAQOQWMAcy388GdJRq4wDXzL2aecaAt+BlK5FXnaTPslQyh54UIe4rD4wQqLn3xZTLjOhbnr9oaqKUzBACwhHVrdEc7ChGtwA0+sZKuhG+cgk9pNO7Fqv1wwADdcyYlkKAfNC+HlFzuqp8AIvBiLyJEww5UsgS0yCAEmHWMYuSR9Hgzte/PNN2Pdqa9kR3VP20z7aJvarjRd677z9qBVDdykd9JKgJ3Go3DkGrlBpw8Fo/Ej2aLNYKGxg5xhkU2w6bShH/bZZ5+w2GKLhbFjx4YbbrihMD40D8ulfrQjtY3Ut1r+NA6cdM19qEc6D0rvkEaGMGEpnME3TaO0g+V2lMjTwYCCpQPViYAjQiGgcW2rYwB+UnZgKsECNwSPyRFX4bgp7oMldLXel/oiM1jVnfamcoMfQzxxkjHajeVabQYj4YUrWSOvDP5cyN1jjzRcT9oIFuAjpc+18MBNMUv7QDj1okubaSvtxwgHXCwYYUkj2ZLsKE1Xu++8HT75rJ87VU/hpXGl8Sj5IR49hdXYRCY1rpWuW13qzNyFK0ObkA/C1A7aL6KlsE71Qd7uA1ZYcJRMSG4Ix0hv0TbwBG/S97KhrZIr2imMcIkDK3DBVTow6WT/Uy/uv/fee4dFFlkkrshfd911cbGIOPoqtdSN+qe2XfXVfVU+95TsgBuYEad6gl0qg6QlXtiSv1tMR4k8K39YFFoKHOCgyFGIgCWABbzdYuFP8RBWCBx+sARbsOQVJEIngVNcGtYtgliuHtQbpYASx4+l7bRD15Vc0lRKJ/zSvLo/eRTf1e7kyQ3XM2131k/7kSVkBDeLoXDqRVcEnX5PceFaONBurhlfWGHU1bKSmTw7WVdwkwGr7JYA4lkRY04Ac4ywV75udqm/ZIB646c9b7zxRmELiGSHNtJ+8Md0sh/ydC/hhcuYZJxJbgjDwheY34RlnmSmUXlGtnj7Ax60F2yQKcIxwgAXXIiXJU0nZIB6wD/22GOPMM8884SddtopXH755bGvsnVM6ys/fdtIPWlf2lauZRUuN8WC++neuBjCmP+kl2gPPCSbVuljpkH+6SiRBxwRM4Gs9hMusNLOxG9bHgNhB5YImzDFLytFR1pwFM7dJIRqR9aljhp0qi+u5EPt5Ro/Aw8ZSxWB8qls5U9livTkZ6DjT+N6zU/7wAhLe2mfMMJPPOHICX7FCb9edmm/5E1YgAMkAtkiTvLDNVb44dqWYgB+kjPwg3gQlhqNPVysxmCaplv99Ln0BrJB/RlbqS5S3dHJkhnCLC+l8gIm9L9kADfVR8JSD0WkF+74e9nQPmQIPGgzGEyePLmgmwhH/jDECzfpcvK323Jv7sfWGlbk+dj12muvjTqUOOpHvbDl5iHq3WgdU7lJy1C43GwcdZE8UT/iaYMsYeSlbliVg79bTEeJfNpogSJQU1AIE6BKZ7f49VOKB4LIAEfwNEjAD0M68FT6VCjT/uhmv+oul4GkNqMMaB8uq2CQLtqYtlP5Upc8lKO0GrTpgE3T94KfPqZ9YITVpEAYeAgTrhWWbXc3y0mzdVNbabvkBwX/+uuvF95ugRHpNM6Ux26xfqIvwAT8IK8aZ+CGX/JGGgyYM4Ylk8TnwVBP2kT9aYtkR3WXXJCOtoGH2qY4u8WyA4ZgCk5ZbAjDSp6IJz36m7BeNsJCbUSHv/LKK/HtBGMH2QIHpRNGuAprt0vd6Lv9998/fPvb3w777rtv0R556iedQP3RDen4aXf90vKpK9gwLuEOPBThJ43kDD8GV2MbrKmz4mKCQf4ZNCKfAgiIXMtUG8iAZ1uMAUInzMARv4QMoeMVJAMIk40X5t3s0pa03lzTZgYTVu1PlbnSKA5M8AuXtL2Ep5Y0vWppp3ADE1nCyxnCpXgrpSmXL49htE9yJnnBBS+Fq13IR4pdr8pLs+0Sbmk5TJboJCZyEQ+wR1dpksyLrNEu6oqVHxeDS/sgK1j8abuIty3FQHiCIX7GHzIDhsgNfEFyA35Kw3jsZUM7aTdjCgMukqtUT4EJRrJFPvnb7XJf6qitNbjpOfLURXqT+uMnTKbR+lFGanWP1E3j5df9qAu6h3DCsoYw2sVcSDr6oFy6bL5OXXeUyKfgAQSg8BTEilcWlOx1pwDJ833AV4Nc7QBjFB/ChyEeoc0TvrSLQZTWm/oTXk4REJ4apU3D7K8dASk5sO5lw9hAzmhnnsZH3voEeYKQsc8ZgsZ4xaKjNElmx3C3thE5wSIzyI/qrTDaA9minTaNIQCmGpvIDfvEhTMl4kemel0/CYdubyf15PjJYcOGhd133z2MHz++wD8ak4D25srqeo1p2oEfF6uHSemobL721rJ66R0l8pBKJkoBADjdLpTV4eu+2BRPcEYBEoYfC/5pH3RfC0prpEGU1lsylE3NYIMg2LQHgUq4t+dunS1VZAF5s2kvAugkxjMETPMAEyRhXOdFzqgnlrrzhoH2YBRuWWqvHFE6sqSV6fbfzXeoBQG21rBHntNrbrzxxq7meciOxi1tY8xyLf0kP9wi1U/dpKM6SuSlqFMA8POUff/994ebbropjBs3Ln4cwVMcfizHF9mWYsBHJNdcc03EC3z+9Kc/hT/+8Y8RM85uBc///Kf/D2DAOiX2tQzIbkjDwNKgkuzgigxocJGO9l5//fXh6quvjhaMwAZXlrgrrriiYK+66qoYh6whd70ua8gMVu3FBRNOF0CG/vznP8drsLj55pvjHsdJkyZFotIN8tCuOiA/euiFlD388MPhtttui/IAPsgP2Fx22WUxjFfGQ0FemhkPYER+ZItxdsstt4Qnn3wyTpSaIMEdm+omjfN29XWryqWeWHQQC1W0Se3hHsjTq6++Gv75z3+GO++8M445sMCCi+e36vMa4w65QV+lcghuf/3rX+MbfTCHZIF1LxvayPyGHCE/YANJRgfhv/LKK6MFL+LRVYRlsUtxbLWf8U1dNtpoozD77LOH733ve+HQQw+N9UAXUC/qp/uiU6mf5maFt8pFTlJLubon9UCXgxPjE36KLGHFVdFJWIXjMr67zXSUyEvpSUnjAtKDDz4YP4pYY401wkorrRSWX3756H7nO98Jyy67bFhmmWVsMxiACx+TLL744mHEiBFhueWWK2CEf/XVVw9bbLFFFNQU72wfdJtAlqtPWue0LVp9ZwWeiRRZQlnsuuuuUYaGDx8ellpqqbDyyiuHkSNHhlVXXTWsuOKKEaull146yhZYIW8rrLBCdFMce1HuaPeSSy4ZZYe20m4w0ThDpkjD2AM35Ag/x4ilD4Xl+qkXwiRfkM1jjjkmTkjIAVihn5AV8AE35Gm11VaLuqoXZaXZNiFT4IRlHGLB8De/+U08Qo8xy4SpibHcOO92mUrrTDsgk7LEoaNuvfXWcNBBB4Xvfve7EQNw1XhrFuNezI++YZyBEXMbK7v8uRC6CRli/HG91VZbBRYY0PuSoW6Xl2bq9/LLL4ezzz47bL311gX9vfbaa0f9jE4HK/ABN1nGHpyqU/MafUf/zDjjjGGqqaYKM888c+QozL/Ug36Ds0gfIL/kQZeutdZacc5pVqa5P5ZywIG2yxKGHC2xxBJh0UUXjS73RSeBL1xCK/TIFGO43Lhuph/bkbejRJ4GoORQbgw+QMKw8rXJJpuEWWaZJcw777wR6DXXXDOsv/76Ufmtt956Yd1117XNYMAgFk4bbrhhfPplQMw999wRSwbN6aefXpCbFPNCYBd7NIikpHWNi4UEZPef/v73v4/EdM455wxzzDFHVBgMVOTrBz/4QZSnFLfvf//7kawha2BJXK/LGm1Nx9Q666wT20w4eEA4wAwFu9BCC4WvfvWrkYw99NBDXSwtra3aY489Fvd30v7pppsukomNN944MM7ADiv9hNvrMtNo+5AlxhW66Fvf+lbE8sgjjyysnoqEaUynbmt7tD2lZeuLrpLljrSPFb9NN9006uWZZpopLiqwYgmmGnuN4tuL+dA92A022CDqIjCSZX6DCH75y1+ORO0vf/lLezq2C0t97rnnwmGHHRZ18fTTTx91MyveYMMDMjoJzKTfuSYenY6/k7JG/1EnFoKQUeZfdAHX1A89Sl2Zb0lLGGOCNOrrRmRbedVW6WnuJQxULxZh5ptvvgCWPGi/9NJLkZuy6q63O4xv/ArTQzpjnLhuMR0n8qzCsJUGMi/z1FNPRZBR9DxBbrPNNuHwww8Pp5xySiSiuCeffLJtBoNf/vKX4dRTTw1nnnlmfFI/66yzwv/93/+FVVZZJcw222zRHn/88YI5Yi5SXAjsgCed7CT82bBK1wwcPYBQd641iDSoVCZN+dWvfhVYiV944YWjTPGxzRFHHBHOOOOMGIccHXfccQFchB15CD/qqKNiHLhie1HmaDNyQptPOumk2N5jjz02nHjiiXGsnXPOOVGeCNttt92i4p166qkjEXvggQc6IC3tuUUl+SoXTg3+9a9/he222y7+qQkPhJB49BBjDQzBCznCnnDCCVFWelVmGhkHGkOnnXZa1OWsns4///zh85//fDjkkEMKncxYls32RSFRhzzV7p+Nq+Va1b7ooosiWUGOWAWk/awAWl6K53ThwZuwo48+OiA7v/71r6PVuAO7H//4x2GaaaYJCy64YPj73/8eYaY/8mrKyZLakrbrmWeeCQceeGBcXWZ+h6Sjp9HjWM1x4kuEoatYzFNYI2O5kTyqC7oRyzV1QG8y/5x//vmFOYj+xlJfyUAj9ySPsFB+7gkGWPzEM/eLA/AQMcUUU8RTdtgCl+KtfoHEs2ioRQeusWla9Ve7XNVFLveRH7ejRJ4borS1DYJrDK+xeTJD0Y8ZMyZceumlcZWe8H//+99xUmVitS3G4PHHHw9YMAKrZ599Ntxzzz2RkPJEOtdcc0W/hAvhA/9OG/qZe4uQc3/CIOLIAnH4CcNSR/nJo3orjnilz7aFActTNquA7M1jj/cTTzwRMcJlpVVW4cgVOD766KMxrtfljC0yyAw40Ga1GzyQIx6swYO9hD//+c/jWzJe27IFLq8GmZEilmwhQyho5IswjFzw4RU2r4BZPWIiIuzpp5+OekjyAnb4e11mGmkfuCBr9913X/x250c/+lH4whe+EMmI5Ih+Sa36Rv2gdJ1wuSdv+PjnTGQjNcRJ76i+Wb2V1j2t/wUXXBBXGVmk4uGQ7y6ef/75qJcawbXX80gv0U7kB4tOQj+xlYbtJZyIwj+H9gKRZw7k+wpkD32Uyg6yht7CML+zUMeKMttUeNgBK/ABK/ST9Drh0kvg1mmZoU7MJdyXeqgO0gnoUdX7kUceKcxBjdaTclNL+7HCJb0/YcSxL54/rILIc8oOJyim2Gs8p+OeMI3/NG2qK9rh557IBrKi+6Z16SiRp4FpJeSnQyHy7Gvi6ZHJ0aYxBHjTAQHbeeedI6FllVmmEvlVfLtc+lmEXX2Oi4KivsRRNw0QXOKxhCuO+hFGPHkVzrUMqzjsqdxss80C22w4qsymMQSYOCAh7DNkb2EeiLzkJnWRD2RFxEvyowlUk6fygBaTEESeV/ksLrDP2aYxBMCeyZN/e2RLBCdayNAXwh0/drAM9YDE67he1Ud1pB3SO/iRH67T+qd+tePCCy8sbCdga8QLL7ygKLt1IsBY5UGIN64sVOWRyCMjqZwjR3xcj+xB6BVHOuQLvYWB/LKwwko8bwj5UNOmcQQYw2zzg8jz5p5j0AnTeNdYxh1sQ53gSljJB6700aAQ+SxAPEVC5Fn94tVKHgjDYHdspfvzsQYnsuy4445dReSzfc41glhp4KTp8cvgJw8KHaWHxS/DK2s+lNpyyy3DueeeG//5TnG4aVkKLxemuKHsMi55Bcq45CE7D+OSvpSl75AVJslUARLPNStgxMtKQZIPIs8WP7apsWrDKVA2jSEA1nwHtddee8UP4A444IBCQWCu/sKf9kEhUQc93B9ixfZPCD0ywnU5GVF9qT9+0qRtUbUh8tqvzNYQHpBtGkMAsstpbHy7wrdgeSTykC/GhAg6SEh+kCGRd/yEI1MYdBJEnn3kvCW85JJLyoKY5imbYAgEahxWayq8gQfrLJFX3tStVk4n4lQX6SF0EvWXbuoqIs/XxOxt4lWszcAI0LnZQcvkw8dV22+/fdcQebVEwkidU0u4jNLIJZ2EF5dwXJQgyhCLMMuwx5sv1NlHif/FF19UVEFZUqaMyiMM/1AxtFVtll/XwoBVVB6GdJJNXoi85IV24GfyF2mnjUyUXPMQqLajFNOJNSXyvHblNCSb+hDQeOJBCtkZPXp01xJ5ZEJ6BBmByLNCh4wQTjyyRBxygtU1YfhJg592cy3Dm0GIPAQMIs9eZ5vqCGhcCk+l5g0rD9Xsj88rkadN6BvmLuQLF9nReJGsSb7UdhZW2FqDHPEB68UXq10VQwAAIABJREFUX6yogkvZyk95qS0k6kFP2k78YIfFL6M0umYOYPttXoi82oRLH6OD1Ncm8urVHLoIZlbRdTuRT4UQf7b+XEtgJawoOhEvwpRHAzMdrGzNYmsN+3FZnedLdBmVjStD3nL1UHyvuqnsyC9c1ea8EnmIFzJDu7C6ZvJEjiCW+CUHyBRknzAZE3kh0bgL9pg8EHkIFboTWZHMSEaks5AXZAUyCclHxqQ70jRKJ+TYnpYSea/IC5nKLn0AjljJEal7gcir1egctlkxRyF76UICbZZVerYga0XeRF6o9LnCSi7jEcu1jOJ0nScizzjQQ5/apPbgdi2RV2UFut1SBMAoq+i6mchTVyZKJkisBpoEknisBiEuEyeWfNhse7Oo8GU6+ydN5LPIFF+nsiN/Ftu8EnnkBllJH/4kS8gdcVzTXskWpCydSE3ki+WlkSvkCpMHIi95QQbS8UD9kZWU1LNaj5W8kF7yhUxhFUd+VuQ5Xo+9zb/4xS+8tSZKRfWftA8kR+ToJSKPzDFf0ybGCDIjWUMvpe2m7Xyw6RX58nIDVqkFR2yKoeJVQt6IvHgTsiE9ozaZyKtXc+jSiXRoKqzdTOQZWBAmva7WQJNQavDhql0S1GouaWVYkeecYT52ZUXeW2uETLGbyo78wlwp80jkVXfawljQB0yEZ2UIhShZJE5pcE3kIxxN/QjTPBB59b10UCorhKULD9Jh0jukxa/xwzV5ZLwiLyRqd7OYKmcvEPlUttQuwiDy6CMeEiGZhKUGfczxk95ak6LS508xxZ+OY6VWGl3nicirTciICD1tlM4xkVev5tClc9WRqn43E3nqyiqEVkQ1sAjXwMOPlSEtyg3hTVcpsoKt9Jw7zCkr/Kut98gLlVI3lR35wR2/TJ6JPG1AZpAd5EZKDz/KEDKGRb5SeVP7TeQlBY27wjIvRJ6WIhNseZCukT5CfmRoF7LFwQKkVztxZVOZ8h55IVe7C47CXviSuxeIPO1CviQjtA+dBLFkrGiuUzriMBzB6RX58jKkcSdXfCKVHcWphLwReeqNLOhBT/KDayKvXs2hi2DSiamwdjuRRxBRYjLUnUFHmMiWBhzXDDb9bTKkS22VkiOMyVSGVXj+kdQfuwqR8i44Snbk17Vy5JHIZ9tCm0TYkSfkT2QN2UrlibQyJvJConFX2OaFyFNfZAKyCJlCHmSJk54CEfy87YH0q524qRVyOrWGldSDDz7YH7sKmCouOKbYK2mvEHl0ELqIdiJL0kksPBCGUThhGL6t4B9I+ZdS75GPkBR+0nEnTMFPWJJQaZQpT0RedS7nwp1M5Mshk5MwBBNllwprtxN5rTYIYuouMo5Aqi2EMdBSZae24pJWih5Xhn8AXHLJJeNfontrjVApdVPZkV/4KnVeiTxyw8Mfipw2aeVU7SKcyVETJNdaCVMaE3kh0birsZwHIq8xgF5BR2HRPalugbjzYSJpMLhYtRM3tUJO58iLyLOyalMdAfVHVif1ApFH30Dc9bDINXKEi5UhDF3GwyLGRF7IlLrpuMMvLPHLKI2uayXySt/NbtcS+W4GrVvqhmBmFV03E3kGF+QJki5D/VFqhGtSJB2TJvvb07TKQzzhuOlESzz/wOlTa4RUZTeVHfmzspRnIs/rR+SJyfKVV16J++VTWUF2RNZwUeqprJnIV5adWmOQK0xeiDz11ViAxEvH0AZkBxLPEYDoWLVNcSJiGkNpfErkWVE1kY9iUfVH/SA8lbgXiDyyxfxGW1KdozYiS4STjrkRi9EfQnmPvJDqd5GX1KLfsek4VLxyDUTklS4Prol8HnqpQh0RzKyi63Yiz+CBOMlQf8KwmgxRXCgt/kL51VdfLRB85WGASsFlV81+9atfhUUXXdSn1gisCm4qO/JnZSmvRF4TIXLCahbECfvaa68VKXZkh3hkj7ZjZUzkhUTjLnKFyRORV2upO3KErhKpgsjzN/CMC/wypEEf4SJzGLUdf5bII1s21RGopJN6gcizaMX8htX8JjSQHxYhsJInyZKPnxRKpS4YpRYcscKOHIpXbvR+pXPklSYvrol8XnqqTD0RzCz56nYiz4SXrkJQf8JQbgw8rpn4OWrr+uuvj//0e/zxx4drrrkmPPbYY3GVQlDQfibblIBxao2JvBCq7KayI39WliAsbE/K2x9CqT24bKvhX0UvvfTSqLT552jkCiWOQeZSeSQPJiXy/H23/xAqwlLXj7BMifyXv/zl0K3/7Ep9IU/oIsYCFjlBPtAzkydPDpMmTQroIz46/N3vfhfuv//+AibSRcqriJTIcw44hMymOgL0hXCUHJFDRD77z67VS+uuWGSMPwW77rrrwgknnBDnOP7EkTDaikUvZRep0j+E4ijTiy66qKRhYIYcisSqvBTDkkw9EJC2U/gJAzVPaXRdicgrPk+uiXyeeitTVwSTgYsr081EnrpqUlR9qTthKC0MaRhgrKD+8Y9/jGfCf/GLX4xnMLP/HXLJRCuTHZxsreH4yU033dTHTwqkMm4qO/JnZSmPK/JqKm3CQCIfeuihMHbs2PivotNMM02UpYkTJ8Z4PUQqvdyUyPufXYVqfa6wTIn8VFNN1dVEHv3JKqlIFHVHP9EW9M5tt90WVlhhhfD5z38+zD///PEv3skDaeChUauokCmZlMiztcYr8kKmsgve6KOsThKRz/M/uyIrbPeDxM8777zhG9/4RjyggWNKs99gpGSUOdGn1pSXGeQlteCWYkcuxasEE3kh0YArMOVSBE+aa6+9dlz5O/nkk8N9993XQMlDLwsYYlB2CC0GRcfT/fbbbx+GDx8ejjnmmBjODxMSaQfLUEcmQ012UtKpLFA3JlEsq1+srvMXyjPMMENYaaWVwi677BJOPfXUcNNNN0VlmG0LW2tGjBgRNt98cxP5LDifKTNwF/by61oyRVaI/HnnnRfxXGqppcKDDz5YpsTuCqL+yJkeDKkdCvuII46IcoQszT777GHDDTeMJ4gweT7wwAMljYBsIUOrrLJK2H///cONN95Y2KuqCUL3AjubUgQkS4x5tsntscce4XOf+1z42c9+VkgMdhrv4DpYWKoePNghLxBxrBYeRMxpxxprrBFliQUGFg122223cNxxx4XLLrus7J89sXK/1lprxfO/+UMoCBlG+BTAsKcIAbBnrkhxYm/5uHHjwnzzzRfmnHPOgk5CxtIxX1RQF15Q17POOivMPffc4Qtf+EKYdtppo65Blnjjc9ddd5XUGrk55JBDwkYbbRS3jrLQJYP8ghMWv65Tv9L2oqu2y0WXSE+rvYrTdTkir7i8uV2xIs+rRhP5+kVHgonAaqIRkd9hhx0CqxYpkR9sRUd9Ucy1PlCgtHkgYRUPAoadeuqpw9JLLx1GjRoVrrjiisLHZ0IPIg/pNJEXIsUufZAq99SfpiQdRP63v/1tJPKcBJQHIk8bRMDS9iArvI5Hfpg4kaWvfOUrkVyx3YYtEi+//HJcVSUfRJ4/FVt55ZXjKhhEnv30lI0VbiJ66b3s70MAGcKwqs3D0p577hlXslMiT3yKaV/Ozv/Sn9RDRuNELjoWw/Yatsfw79FTTjllQS9BLLfZZptI5pEdvsegTAxEfp111ikQefZG21RHAOwgWljhSA7wZ0sKi1TzzDNP3DZHuN6cVC+1u2JvvvnmOE/xQKL5jYdD9NR+++1X0Em88cHAkzi+dOONN475eHCUASNkNbWEyRLeyyZtN/56iDyLC2yfZMzm1XQFkfeKfGPiw8TDChIEXQMVIv+nP/0p7LTTTnG16KSTTmqs8C7JxdsZJk5W5KXseBXJa+1lllkmrkyceOKJhbc4rCAvscQS3lpTof+Qk1S5p/40C+lE5CHxPBwxTvNq+HiMFaw111yzIEfI03TTTRcfeFll3XnnneOpRzxAsr3iJz/5SVwl4y0Q32iAFUbjjWtWAiHzNqUIIENY7S3fe++9wyyzzFK0uFCaq/tD2EJz7rnnhm9+85sFWeLhkLatuuqqUY6OPfbYcMstt8S3hujj7bbbLq6k8nbHRL56H2teA2cIusYduRjHvI3lrStva9k2l2fDAy5/Xqi5DRc5YhFu+eWXj7J0/vnnR33EHvp99tkn6jDeKP7hD38oNF2rz4w3kVhw0xjE7WWTtjPFIG230ggHHhIPO+ywwMMTbwvhTnk1JvJ57bnkDyMgEhJYnip5UmdFnv13kJM777wzblP5y1/+Eu6+++742o5Xd4NhqQv3pR6yTHiseLJCcfvtt8dw6nrHHXeE8ePHh8MPPzy+RkXJ8fTM/lSthn3961+PZAuyf/XVVwfIAsdPsprKh5rp6RIoNpRcOjFo0Evp5Vgcaqo67aWtam/qTwsgnYg8D0wLLLBA4B8qmXgGQ27quSey89e//rUg68gccsWWs/XWW69o0kwn0JlnnjnG8+/A7GvmgzK2czGGWAWU0QoyhJ4HaRN5IVPsStZ4MOKbhH333TfMNttsEU/1kcb7PffcE3UUOoF96Iz9evq8FWmljyQvuNSLstFR1Intfpxcg26ZaaaZoiyhjyRHkIJZZ5017qPn2wpkie00/EEd5IuVVgiZTWUEROQh8Twoo6MwyBNvxZAZ/vRvrrnmiv2AbKnvWiEHnSgD2br33nujnuLtseTn//2//xeY03T9rW99Kz4AIkfoX9768LC47rrrBrYFyojIgxV6CSsdrzS97CIbqdXDDGEyitc1RP7II4+MRJ4/2LryyisVlTvXRD53XdZfYQkmik/7zjl7HcKy1VZbxS0prGTzZM9eTog9e/J4DQzZXWyxxeKqBisb7bLcg9fQ3BPFy/7kYcOGxdeivB5lZZ04LGSR14rUBZdXpygyVk0hWfi/9rWvxYGHokPhsSr21a9+NVr8rGbQtq233tpEvl9UCj5kJiXvqb+Q6LNJU0R+9dVXj3s46S/kqF2y0opy6XssK1rUF3mXvCFr008/fXwY5EEQ0pWSMLbazDjjjPHtD299sMsuu2zcWgNRkGGSgGQwEeC3KY+AJlGwYusSRJ5tcuwH5i0P/cKYpp+4ZssceoBxzNhHd3RCRyF3yAvyIReSzkMH90cvQc7Rn6QlDLniBJ4vfelLgQ+oaZdkijCuCad96D3eaPEQyR7n5557rjxgDo0ISEfxkIxFR2EgpxB5xiJEnjmAvuBULTBGjlqhQ9pZhuQJ2ULeeLPA3IbMMJ9htfWP9rFwhV5CbzE2IPG8PcRC7GVE2tFH6fZVjUGl61WXdqa2ViLPmzPmAHQS+ievpmuJ/FARwGYEh8ELidfKIJjxNTwr0zvuuGMkwCg4JpCRI0dG5Qcxwa644opxtRG3nZYVTSyKl/syobHCy8MF4exB5hQIXfNxIcqKMNKRB+XHxMrEyB5nkS8UHmQMhYefyRYCACngVbZX5EulCxlJyXvqT1OTTkSe/mAiYdKhf+i3dspMo2WrXsstt1xUysgOckf4aqutFmWN/aiseiFDWCZK5Ce1EEk99JKXtzysyMqAGa/9sdZTQqWyywPP3//+9zBmzJhIcBmnfBPFmGes0l/0Dx+E0veE0Xdgrz5tVCZqzScdRD2QcR5YsdJPTPLUlzgsdYSks4gAoUemUgImnQRh48GSerBPnqM3vSJfWVbSmHRlmXkOEs/2B1bMwZPxS7+w0EDf0IeEd0pmapUtpZM8o5N4+KDe/LkTugY9hBwxv+mtMzIlvcTcx+IW8zjbAxkrvDWUQQ9hIbB6S4gfXTUUjNovt1Yiz0fq4MwCDuM6r8ZEPq8999lHYrx6ZOCi9DDsIWQ7Ch+DMvBZmWaFnj2+l19+edw/f8kll4SLL744nkOL207LWbfs5WO7D/VgnzL1uOqqq+IDBx8hEkadcJWOc7/JSzz72Jj8pdRSF6XHIESxsdq17bbbRvLGK8hzzjkn/jusuhilllVuUn7E4e91Qxtpq9qb+tO2k05EnomHByQ+nOb1YzvlpdmykRm+k2APM2UhZ2zZ4qQLXk+vv/76kUxmZUgkjJUZTo7gaLgf/ehHkXwhS8iuDJhB4iEW+G2qIwBObEvhgzIeCNlKwFYlzvOnf9APWOkFwtAF9CW2WZmoll/34P7sZUdPIiu6P3kJRw+l+pNjJCHo+hAfUqlFBWQLOYK4cewpH/my1Y8FldGjR3uPfHVxibHSy+hrDNvX2KLFQhXbUiDHvH2lH+grZIYxqv6s1ueDGSd5RqZuuOGGKGts60j1EboI+WHOg9xD4hkzfF/BscoQ+Q022KBojzx4CTOwwmbnuhpgz20StV9urUSeOY2HcR6Q+A+avBoT+bz23GfHTrJSIWVHU3RqDSvSkC9eZ/MqlxM5nn322bgaxIoQR1l1yvJxF/fEUpcXXnghEmz2r3MNYeRPe9h7Sp2ef/75+HEPH1dC7CEAKO1U2bFywaoYW4j4oBelyKkq7EdlJc175MsLNoouJe+pP81BOhF53qLwuhnixQkK9GenZKee+1AvLKc7YJF3ZIkw9tHyhoYHvlSO8CNbhPPBE0ebsuLHNxtbbrllYOWYlWTaLgNmvL7WK+x0/CmN3X4EWJFnbzmEFoICmeVjYgg+WwEfeeSR+NEifUZ/SS/U0/fNppVukgxxLX1J2ZxEg96ijsgGK+tsDUplia1YfGiP7mGljzejbAPhY/wf/OAHcfX1wAMP9NaaftEo64OEaoGK+Q2DPmK8oX/4hoK3g2zThMRLZuifZuWgE/mRK+QeWaMtWSKPTPHAy4o9iwocyc13Aegg5kL2x3MEJYtdWYNu0psME/mB/9mVPfI8jLPg+Y9//CMLZ26uTeRz01WlFdXTp1xScEIEr9x4cueJHmIrM5iEI62j6oNLuF6ZspdWbxaI45oPXSGS7JNnxYv9ghAvVmQ4iostDyh+GZ6q2XvIxOmtNUKl3wXvlLyn/v5Uff0iIs+kieVhK2+G9jL5c9IFHxqyispEifJmbyr7TpEVji3ln4NlOA2DDxQh8rwRYhVQ4wfMKJfr9G2Y8trtQwCMMIxjHrIh8Kww8qc2MqRh8QELUVMexXfahUBmt0yhkwjDEI9+RWZYTJAsoWt5Nc+qKSv6tEWG4ychX3zsyv8ZeI+8kCnvgjFvlpEbcGS8yTBXoPP5bgEiz0fteTX8ezl6hy1ByBFzG/McD7u8BeUABz62luFBhQdIVuN/+MMfxjdFimPcyKKXsNJTStPLrtouVxhwLaM4XbPAgG7n2xYWGdBBeTUm8nntuWRFHjKB0sNA5HVqDR9u8TGHjFY3dD0YLsqFQZYOMK6zbxaoG6tfKC2UHJYVLxSYVk1Zsae9KfnnTzZYkWdbBESelQ8Z3TudGKjHUFJ6tJf2Y7N+4YRLXJbI82c4eTBqX1pXTnjgA0XJEm9zUN686mayZMU1JV/I3q677hpXyyCe1157bTxnGDkFG616Ccf0Xvb3IQA2EGCIK6cd7bXXXnEfOWRXJjv2yQO2YNxJw/1YEODe1EmGMNqgSR4imR4XiE7izQ1vBdnSyANg9jxqPkrUOfJsBXn00Udj8Z1uo9rU7S740w9gz9wG4YLc44ItD+Us1vC9i3QS6bphfqsHW7ZzsZCAToJMsmDFkbennHJKlCXeVNFmGd4SoYs4TYv5jbcRMmCGPJWzStPLbrbd4CFM1G6l0TXYHnroofGbBI4YTk+4U5q8uF1L5PMC4GDWU5MeJF4rRjp+ko9dOS0mJfJSjig8ERAJd7tdcOKeKGcIE252oCkNSpv9kLxOZEWUlQqUNm8ZIF4pOScPSpz6YyDykDTSmshHSIp+wIl+UP+n/jQh6VIiD6aQMZl2y0uj5dMe5BwZk3zh5+GPE5wgXrzNYULkVTUrf6khD4ZX7Bw7ifxBQNkfzXYQxese5R5A0/KGsp++gABDQJAdtijxTQvb/WTAkb7GgC19xfhXeKNyUG8+9BETO/dXn1IH6VbkhPpB1NmjDPFCJ/FKnoUTtklUMvypGkQeAsZ3PBA06gc+NuURAB/wpl94eMIyxzG/aUWeB3M9FBGH3NTb751Or9YyZzE3sz8bvYRMacEAPZMa6ohhsYE0vNnZZJNN4ndlikNWh7I8ZfsR2cEKH/BTGmErIs+DFIs2fH/RrNE9Wu3St5UeVNFJJvLN9twg5kdYJLC4GBQdfyDBq19e9/LqSIbBzgo2k6sGfasFrlJ51IF7UgetsojMq364KGMmRc6ZZg/gUUcdFT88ZGsDhIBtEmqr2q+2kP+0006LH/mycu+tNSmyfX4wAy9s1p+mJi5L5NN/diW+Wy3tQEaQLx762KfMKherLmeeeWZcxSMcWcIicxjS64GYtz0QL05MYp8q2yVob2ogfTxwQjJsShEQXuDDyikf4PNRaPrPrugDJigmVXQTxFnjuZPyRe25L/enT9GR1EmTJ/LEHlo+omaLFlv+WGhAVyFDtKOcQUbOPvvsSORZXGC/vLbWCJ9y+YZyGJiDJ5hrHMvPWOUbFhap+AaMhyKM5KiTMlPvvdSnyBbzG3vfIZBnnHFG/IaEvfOMFelm2oROwmKQNbbboJeY31nR11ghjfy6z1Bys30heUnHmNIIF8Y3W48h8uniguIbcXWPVrv0LX2MTKhNhPHmmEUmE/lGeqtL8tChCKyEhmpB5NlGwP5eXj9C5JlMMKRjZQnbyUHPfbmfBpcUFAqbMLnUEZLPKy7OneZDIAg8H8GmRu0lr8pSPHsO+TBTW2vS12WqQ9p2yqIcwvD3ulFfqL248qdtJ11K5NkjnxL5NG23+pENVlkgYKy+c1IKq1qpgbgxeSIDKEquMeyX52GQ4+14AIDIy4AXBrllLDEh2JQioPEE+eIfmtnKxB55PviUQTfRT6RBdw3WQxF1RQboS+lIZIG6yfCNCKfaYPkzn6wpN5bIzylKHDMI+eLjOj7OtKmMAJghF/QHVn5y0D88QPGtC0Re/aAxWbnU7olBvqg3H02jl7I6ifYzD6KPRN6oPW8JGTuSJYi8DHnyhIHq3SqX8ZtayY50EPdRvO4pIs/iAkcMiycpvptc6k4fp/1MG1nc5MHWRL6beqvOujBwJbCQCgyTIUelsfrFx1dMHJBZ0slkBVrh7XIlhCgl6pzen2uUFlaKSGFMpOSh7oRxDTET2aK+tJsVNA1Cjpzk/Hn2sfr4ydIeBXuwxGb9aWri8kzkJWO0M50QUd6QRY0XZIt45Icw8mEgbRBPVr/4h07IG2VhpEx1D+WJkf4pICBcGJ+8FeFUIE7j4CN1DNhrBRI/4xo9MBiGPpUMSGZ4uEBekA/aonCt2FNnDHHIDulwCVfbiWcvMw+FbNOCMGSJ22C0t5vvCX4ai/iREbDF8PZjwoQJhRV5SAwmxTsGdPEPdaVdqcyp/rSTOY52KkxN4eNYvi/huzEeCtNTa7IypzxDxQWr1IJHFhPFCxPGNh+fc478TjvtFMeleITS4CKLhKMP0E+Ui8n2T5qn1X7VHR3E/akT8oOLHTQinzYUAeV8Zz72YN8YR5UJLCqOVUM66erectN7K0xuGtcpv+6tjgVTiDwDnFVECC1bU1gBIK0wTbHvhB88JHDCRvflGoHE4q9kiGMwodQZULSFMFwGpPJC3llF5dxd/HzpLwMG3AdlqfuRTzjKJawVVuVl3VaU3WgZ1AXMsKoXfrAFF8JkeBPCVgKdWpO3FXm1I3XBjbam7cQveVBa2s7DMKtfEHo+dkWGMZLltIxG+6PZfNQha7NlDhSfTd+qa+5LWWyZ4XQRHog4kUNEHiyZGEkDpoNJ5BkDqQxQd+rEmFC/SzZwicdQd/zKT3rpF6XnzHDekHLMIKeOiMgLn1bh3Uw51KWcbabMRvOmuIAr+l19A8HlqGEOcuDPDjlWtpcMsqY5DvxSg9zwsTSLC8hSuiKf9h35uKasFMtG+6MV+dL6pf5WlJ2WQdnIjKzuxTV+pQUb9A0fu3JuP0SerUviFsKd9ORNxzVhg2W4d9pG1aOjRF4gpi4VYeUPweTf/HhCQukDHgbA1QnqlHa7qh/3RoGkdRCQhMlSP4W3u27Z8rkv96eeGFaLOCWBiYMjuo4++ugYzo+eJtW+QkSXelTPFFv86hfhnlY/+7FrSuRJz4AEh5TMqXzisVmMG71WeVmX8sq1rdH71JOPuqRyS16uUWpY4mUgs+zvhcjzkN0LRF5tK+eqT4hj7yEnSPARGscmsrUGnDApfspTTx+0Iq3um5Utrilf8fjL9Xkr6jBQGZIl9gSzvYkHIj5c54M9GcoQpiLyqrvSdLMrfHGpN3pYD8VcgwF/doc+5nx59tZDGDDqq4FwbEe8MMZVGyTXuNRtsOtHvagLOlt1YVGKP3iDyPOxqz7Apx29bthDD/GEK/HnYtl/dlWfgQW4Mc/hqo/bIUeVyuSespIv1S91K+VvJJz7UTZtxuo++BmXXCsNMsX3UHx8ro9d+a+ddOym9ZYfV+3C7RbTcSIPEICKkPGkjYHI8yU2Z6cef/zx8e+8BVIWuBTEdvu5d2rT+1UKT9O0208dJKzcC8OKBRMH/0YJkU9PrRHmCKvSx0xd+kMd0zbixwj7FF81gY9d2SPPpMlqcvaEm7Q8YaD7EJeW2Qq/6pq6rSi3mTJUF2HGNWMSS7kyIvL8nThvd3qdyDOWwACDTuLPxtiexso8r/OFDa4wbKYfWpl3oPoMFN/KuqgsyRGrXHyUyEeurMinRF5pwB79xARLXSkjD4Z6qr7ySydTf+JYWGEVFfLF1gg+pMYIp25xJSNyO10vYYIMIDO4ImQRsM/+8JAVeXT88OHDI1dQXK+77IfmbRZyxDYt/p9ABr0lvOg3ZJB5Hvw63Y/l7ieZSt1y6ZoNo3zant5H/rRswpAxviFkjzzb/uBO5CUdBpd0ykcceEomlU59MJhux4k8YCBwkHidEMGkyb4vVuT5a/T0H7YEYre46uBuqA9CJqGSELEiz6vc7bffPq5apEReuDMieyN5AAAgAElEQVRhdpMQqu5ZlzoiLyKZGlTZPiCNDMcM8iGUPnbVh2WUVS4/YZRPGd3Qp52sQ4qH8OP+MkORyDNGMHzsyvas5ZdfPm4J4aM0GXDL2k72W57uBWboG/CExELk2V4io7aApyZJXK7zYFT/1M3Wm+MnIV5rrLFG2GeffQpEnnRpvqHuFx4QUPiBiGiKJ1tH2ebGP+iyUKU/hMqLvKRtqdcPkefUGv5cjH+iTok8WKVEXuMJd6jIFW2VrdbmVM54QyYiz9seGfKnhmtxV2GdTZOm77S/40SexgM2oKCwMbzGRtFx1BuvjjiaSXG4pO20pY7cWySPa3Vctg3UTW3qdD25n+oJlig6iPx2220X/2yCj11lqKOs2qK4bnRTnIWx6o2reMkKbWBFHjni47Jf//rX8W/WCWfwYelP9SX5uWZPoh5umunHFFvVT67iVD73RfFiOy0z1Il7Cg/1PXWTvIMpH3yCIZPmUNtaw6opb3V4E8ERcfzZjwwrOcIOnDrdf+n9UrmSP43Hr3DcbFw7r8EGyx55Fmf0z6588Imcyah+kkvyEJYHQz1pi+rMtca86s9/X7AdgsUqiJiOn2wn9uXKpm7lwglL642/WtpKZbQyXHUQhnJZmOHj4YUWWiiuyHOyGXiDP24vG7bWcFwissS3OxdffHGhuZJByV7WbWXf1FKW5jbNJ+rPbL10rfhayi6Xhv7nXprnuSad5Fjjk/sRRlrtkWdrDd/vsAhKHtJkjfIRD1fgPuXSZfN16rqjRF6dhStgaCgfcbBaweDkVcdgE3nVMxUY1Zn6qu4SFgkMadI87farTtyHumAg8ldeeWXgD6HYQ5ieI0/6PBnhTL1lCcuGq+20jeMn2dMNCeNjV63Io1hEvsBL5TCgeTMkIk94I/2m+uGq7KyrNJTPfakTbiP3azSP6sd9IaS4MsSBkcpmFZWj89haMxSIvPoHPFhcEJHnw/GUyKerhcJqMFzVV32q67QuCkvdNL7dfuQLIs9xsvwhFMdPcmYz9ZFRHRgvhGvSVXw3u6qv6pzirHpD5DfaaKO4fZQ9uaysYtTuTrmqW/Z+Cpe+0jVuNm0nroUb9eF+YIsfl4cgFqo4R54VebgChrqSppcNcoP8QOSx6ceuzCXgA17CItufneg73YNxn85vkinVKesqHldl1OvS/nL3VDnChWvqhx5n4Rgiv/vuuxeIPPGqj/ySLcKZI7mPwrpB5jpK5Gk8FsABEhcDkR85cmTcEnHccccVba0R+NmOH6xr6jtY9650X9UJFyLPh3msIkLk+codzDFgDp55MWl7UzlgcKVKi2sZPs5kixZ/m84fY/EBiwzphAFlYygXfAhP79frfrAQaRc+tDnFknOLISHLLbfckNgjjxyg3DEQebbW8N0OSp498jKkSXHqdVlppn1gBl5s02Jrzde//vW4Ki0sGX/pxMg1Yxs3DwZsqKswko6RnmGBgK01EHn2ybMirz3ytE/57PZ/HClcwBDZYcuDPoTmKGW21vDWlYW/Xv9uJx0DEHm+L+HNDt8UZom8Voo1N0o2kUnLV6l8oe/ZWqMVee2RBy8wRC+BKf4UP8a7xnzaP4Pp7yiRl4ABFCCyIojhb5ZXX331ODj5t7P0S3SAJL3A67SbdmAlf6frpPuh6BA24QqWIvKQj/nmmy8+wZMOA455NOCuNuPSDk2UCle7IPJsh+BfFNkWkn7sKrlTXvKQX0ReZTXiprJBfu6VWsLSNKm/kfs1mweZod3UUfjiJ1yGB2w+1FtyySXD4osvHldVFdeLLniIyPMQwwfjK6+8ctwSwp+3yKCTkCEsfvI12x/15qev0jySpzRMfsWpnxXeCRfMIGEQLrbWfOlLXyraI4/+woIl9cNQL/mFebe6wpb64dcYwsUgG/xBH+d+Y1kB1Kk1pO9EH9R6D7VFbq35Wp0uxVJjEhcZ4e0pfwjFx67Mb7zpwWTfLsbAHvtBJ/HBONuQIfLZc+TR3eIC9MlgypdkKOtSL8ZGalVXpW2FPFGWjO6Zlsv90fXsWIDIc6KWvtkkL/HIHHKFq/JUhuqqewy221Eir8bjMihR8Bj+vpsVebZE/PKXvywQedLwNA6Q6oxUANrh12BIB4Q6r5JLPZSvHXUqVyZ1YQJE+BBI7o959dVXw1VXXRXPkeeILp44yY9J8cefF0Ndy/V/2h615cwzz4xytPHGG8f98tqPSjwYQbz08EOYcFSY+r0c5tXCKEf1pAxkVhY5Jq/SpPUu165q92k2jrphqZPqRX3SMGHJR+g8GLHyxatsTZqK7zWX/kI+MEyaO+ywQ9zyx57ulMiDG/LCuONoRfSY+rfZ/qklf9pXSo8cVZKlVO6UvhMu9US2WOnivG+O82TSTP8OHeyk39NxgT8PJltn4ar647Jyysf3WI5Xlk6q1F8qo9OuZEhup++v+6XyCn6EyyAr99xzT3x7P8cccxROrUHGtDCotL3mopPGjh0bj8SFyHM6nUwqh1n8hKHw7YRbrg6EcW/0p+ZGXPSE0itNs3WkzTK6J/fBzz3R83Anvjn43Oc+FxcZxEfTfOh5zZOqk+qa3kN5BsvtKJEHPECR0AEshleNEC9Wv/hYkaPKbBpDAHJx2WWXxVNrWLU45phjIt6UhgAimAhyNwnhQC2lrsiKBhTtSAcTfplTTjklKnlW5HmlzYONTWMIsHLIyQgrrLBCWGmllcJDDz3UWEE5yYWMMT4w6CQ+GF911VXDbrvtFl/n56QZXVdNJkDesrJHfppppin6Qyh0kSZI8OeacZ6O6a5rUFIhdBN1TduAP9WvfJQI8WJLBFsjtCKfFGNvjQgwPu+88854Itucc85ZWPRDbsQnaiwqd8nYWsPxk8gShznw0a9NcwjAj1hcQMfzLY94BaXiR6Y0tuUqTTrGm6tF87k7SuR54oHMZwHgSRPixV5cFB37UfmCmCcmnrTxD7alk7O2G+rEVhrqBU5gyyrqSSedFD+GYRWVbw4QPAkmRJ+Vi2wfNC9K7SuB+jO5a4JP646fASbD8ZM8wPBgqLc7YCOM5Oda/UcYhJ+VMvbUc624Rl36RX1TqQzJU6X4doRzT9pK3VQ+YaovbWeVGRyYME888cS4T5yP0Xv9ARs5QsYwjKNtt902fm/BpMlH1JB7MAMjyUjqF56ddCVDWVlTvVTPTtZJ9+LenAjFuOKNBivy0003Xdzup/EqvBnH+CFq2HRMK203utQb/UR9mfRlpXOpM1trODJwxRVXjP8geddddxXJj/DqpFtOPqQHcDtZl+y90D3shace1FO6iTA+wOcNBySeb8B4my+TzgsK6yWXU2vYCsKHruuss05grmNsCT/0NlgRBlaEo+vBs9N9yv3K3VP6iDj1q+rfLpf7vPLKK/F+3EP3BaMDDzwwrsjzTSHX6B7xDGSHcazxTTjjuxvlrKNEXk/NWSDYP8m5qDPMMEM8HYO9qRBQBBXLKv3pp5/eccvkjeXe1ANiKEudzjjjjBiPOxj1Y/UZokXd+EdTTmnh1S2TxjzzzBN49cjWGoRTBmGkH7J9oPhudBlIDCDqTr3TuuMnToYPXNkjz75ujuhiBQN8eLihz8BJ/Sm5oo9Jw38Y0L+N9KVkgbyUK8s1cWz54T6SFfqOOnE/1aOR+9abR3LM/VVX/NSDOOpHffj/Af4ICVniA0Xw7PUPy5AlETCIPFtreCik7Zxgc9RRR0XM1JfgBGbYevuh2fTqJ/Ud3xalfUgdkfVsXTspa7oXq158s8N3Fqx+sT2AxQTGbbqwo0kzO8Y1trvRVV0lO9JTkiPqzJtB3uzMO++88aGYffK/+c1vOiozyKj6Az2UygeyRDzyIzlCrpS+WVmtN7/uSx35zon6YimHfyrfeeedw9e+9rWIJ1u2MODf64Y3OWxL4wSx2WabLS5WgRU4YZn/GfP0IVhxTbjwrLcfGk3P/ZAl5jfNM+pD6qM5kHSy3Is0iqv33ipHck45Kos6gInqQtmk489HeSD6whe+EN8W8i/wLDygmzR+Nb65Zmxju1HWOkrkAYOVCxF6gcV51XzVD5GfZZZZ4nF3rKiyErbJJpvEvYWs2HfaMnljuS/1oI64hHGihazSdLp+4KM6cW/+iZKn9fnnnz/MNNNMcW8zg0lEHry7+amykiJm4DCAkB0NLLmSJ+VltYYtWkya7O3mdTbYQOrpu6233jr2H/LFx2dgyN+n05e6brQfJQfshaVcXMraYostCpY0xHEvyTjx3L/R+9aTL1s3yZDqor+S54QNVhDBkO0Q+BmnQ8XwoS8fZzJp8lC89NJLx1fa9BMYgTny1KzM1NN3adpUjqgH/SpXdUQfqD9VV8lkWla7/LoXOomtWbPOOmuYaqqp4sktEHjGLq4mR8a0JkzNDd0ub6qz9BFtYX6TzqX+bIEAA3QSssTWCN72tAv3cuXWIh/oANLJliunE2HINjIsm16DHboIncSCzd/+9rcoInmRl2bkmaOU+V8Ytjp+9atfjYeDsOipMc7cxmltjHWwY2sg1+DXiX7TPSRrkif1H/McdcUlLenSebKZeqos6UDdk/B0rgUX7k84nIDvCPmTOvbK87aAHQvYVJ40xruZO3WUyCPEKDm2zLDNBmAwPAnxpImAcmY1A5W/IeYDWPwQM1Y0OmX5W3buiYtlEuIfHtn6g3+11VaLdVF8mr5TddR9qAu4UTdO/uGao7nAEQFO99EhnEycTDIIZ16MBhL114SJv9ykefnll0fyzioqf2SEDGHBgz7kzQ/Ycawgx1SCHdtG1lxzzQJ+wrZeV3KA7Eh+6A/6hTrgEs69mYS4P9fcn7h679dIetUHF0v7qQfYUAf9ayByDnnFQmZZBeNIxl43GhecdsRKMg80tB8cwIt+FG5gB070eyN90Uwe7olupN9UB66xqidyRX2RMdIhd52uK3XhnmClPxZjpVBGZJ55AYN+YotASoSVthtddBB1TRcZaAvzG3EYTllhfqO/IQ70Bbg00/+N5KUPkA/yIhepfCAb1EvygcwgV7iN3KuZPMKG+1Mn6k09CMePLmexCkLGiXcYzQvxokd/2I7C/MZ/xLC1iPHEAyJ6WzJF/8EFcDndRuO/mf5oJC9yhkzRX5Ij+k9zHX7qqDj6V7LXyP3IW05mCacu4EM8WFEH7s+8N3z48DgmeZMhOWI8p0Q+5RraFdBtItZxIo/C49WFVmQAhNcZN9xwQ3yNplcjAMurELaO6HUfr2s6YbkfK9lyqQOv1Tn6CcFgZYXVLvarsQWItEqv15PKq7pn06gdCtd1ra7KFUZshSCvXhnh8pFVuoqKQCKI3SqMlQaHiLyUtVxkibZowiQ/eybZXgM+bJXhNRoYQcq45hUccbxWo+8IAyss6bDEC99a+wMZoTzKpUwsfav88pOO7U7Iz7BhwyJJZOWXV8bcm/SkTeW+3LXKU/m1uGqT2kke6km9uR/XeoVNWoXxivLqq6+Oey4r9VGvhaOT7rjjjnj8JniBt2QEnMAG7BqRlWp9RXmUK+y5r+6TzUccfxDDxIcs8eC+3377RV1FHixp1JfIJv5sOe24zt5H4wr5uvfeewviwlhOdRIkmBUxwvJg0EHUGd1KWzCEpfMbHylef/31cX5DT9O3WXza0QfZMrmv9B5+yTN9g17k3y2Z2xZZZJG4osv3auo36pvWmXDFIWNY3Y906bXC63EpQ/WV/HM/wiTbLFLpMANhnweZabSOyBTfCFxxxRVxjNOXbD0CV41tcBcn0dxHWNp3tfRDmh6/MCev+gY3jaNvqA+yxMLYjDPOGBeseFOgbYnIHGWQlnrJql9xa6lfNg3lkBdLnRSvuhJOGmSI+hEPZrzhIG7SpEmFboFPpPKEX2SesZ3GFTINsqfjRH6Q21vz7dV56jSehvk7aPZU8VqLVzGsBjDhY5QOl85GGNTpEgIUPmEySku8Tb4RoL9RtBAQ+pN+Vr/ip+8xpOPjbpQHe885cvXCCy8s/Nuj0kh+VBbX+P9/e3cCtV1Vl4/fNERRUTM0M80BxCk1bRAHFKdSLDFToyzFoQSFFyQyNWcltbAs00pxQFREJScmBUUz0VIgU3LKAZwqm0trsVznvz777/X8Nsf79vWF936e9z7nu9c6zz5nn733uc/1vZ7vvs4+e++DM+Lsrzdq8/71bJktSLCrHqHwBZ8IW/YfB2ka9gMPPHC40pWu1MZ5mqBM3OCiN5+pZ1y2jueJAM5os/BjHHwx1dhzQ6Ce8pSnXOrDjMrFp+GsIJaW9KThcJ82vk4drxcC7KnzlV+Kv8IH+znXP3zzOd6WmA/j7YCHAPopATfSTiZNPdKcq7DjCGy5kI9T2PGfvtoS499l8pseeJ+F1mgaq6fHG/kqFAI7goDebb0UHghveMMbtp6w8847b0eq2LK84/+LLfshK7zwrnaPy3yMlRhwyetrjabX6IatLMu/Qsiq6jVGgHiyIpXOqate9aqtfTP8oP+a8fj2/I8Uz8ao1LHOg/e+971tCCufZM6jZWfT4dkjVA97PRqXb3/Lhfzl+/k7vzQHZXyjD1H1juqUU05pY744OgQ1a9ySRVkTGIH1cniy9LSqbC8Iku5p1TnHzvd5dv7dVI2biUDsyUFl+FjScEIgvgylsXTa1a52tdZwGjf7yle+sp2XX88HjqQXzAnpuOMc7iSt7dSftUQg3Igt+R0NXnqlcEBPF1+UnnW2D7/OPvvs4fDDD28TKPmka1/72u1rtN74JKhDnepQf7iT8xXPAwF279/0uGv8E0yiNMzAEC1f3sUlC094a6hdS3uGg/ZtJgZa1rDv2Q/XXKfC+iOAH3xG/Aa/Y5+dnQsXep9iPoj5VOYwWA0Gl8x3vOCCC1qZcE4dWW4y/i7n1h+5zb+DEvIjzJEJsRANUTWCGkbLOu61116NmMhJ0BtmYxwt0SZ/GtmU7YmZejlDzk/drtPnGf2UOlwzBNgSX0zWY+fYFi/YWzCGX28XDvminHj33Xdvy/N99KMfbQ1nblsZXMEndYk1ksWdILTeMZumEbRPtNukOWb72Dt584obB8xdMAwiPMIlE+Asd6iDQRlBXnzsObneyNWv31EEcKrnQO+TLDZh9bMILzyy6pmHxPe85z1NtOd64SFuEnW4JahPGoEvrrD+CLB1tJCYreOPHKetCwfcsflpOjmveMUrbmglHVXGqmv74pPwRXsWrVRa6PLxpYT8CD9Ei7NCNhOVTj755LZcUnor0nD64JKeDJ+uD0F74qeunAt5xw306CfU4ZoiwM6cm8aMw8MFmyC26ov1ozk2jWW/mUVvIk7e8Cij4dVYpi4OM85PfeHVmsI1+5/NfnyCzT57x9bsa985ge3jN/RkWdPf2uzh0G677db2TTAzbOvtb3/7RtlwxTWKM/OkHbvjkA0f8Mq+NzVnnHFGewAMlwwd1SNvRRETXg0rFdQRvqqjr88+furBx9sK649A7yvYOzYPd9jaFk542/zkJz+5Dc3Cpeik61znOm0lude+9rXNx/XIKIsv2jXX66/Z56v9745ACfkRPoiEVOlV8ErIFwmNjc/4wTSaJitaxUbPmEZYQMwQso+d6+tWf8g7+gl1uIYI9LbGBWI+gj63o6fUkmA+1JWHQr1gHN0+++zT1rbNB07i4PTQpheWI3WdOFT7FdYXAfZLIxi7si3+8A3OCc55y5MJY74wa7UaQst62jhkfXb+yFtDS9M5H5+EP+pNXW2n/swKgfgKPMAHAb90UlnbO9+L6MU8IeYjXsY8C9osZcNN5XEsbVkeElrm+rP2CODMoi3tjzi8+vSnP9163S2Had4XX5Sho94481NPfOITN3wYcJTHH5wSp661B24LbqCE/Ah0ZPKEGGfnc9rWG9VQIiZS6rGIw/MxAa8gPY0KIaM4jXTfICNvHhRcJ/lHP6MO1wwBdowtxXq6iC+2FghySwP6RP0ee+zReOT1I0HPyVktwtr3JlCnHN4onw13ek7lemsGVf3cbyMQzrAvX0AISWNnW4J9fLIJlkrzQEi080k6FmzxT3wTcaYnVQNpC3eKM0F1XnF4Fa65e7zwwOcjVde97nWbH9K+pScVj0xWNIE6KyGlfNo0fk2vrLoScM1x8iS94vVCIJzp7Zj2h417X6IDyvK3uMQvWcgBl2zRSj7A1A/VUkd64/ElD4nrhdKu8WtLyI/sgJycFQclNsnVeEFkJLh6Yoagln+z7BsBlqAepEfW9FSE+CFwCfmgtf4x28a+7M4x4Y+gx8ra2b4mF86Iia/+oVCjefTRR7dlTg2pEVKvGI/Uqf6kt0z1Zy0RiA3xI0MSkpaYrwiX2N7xW97ylrZufM+l8b6PoPTLmiqnvHorzA8BPiOiKXf/pS99aXjsYx+70abpWOjbOILe251t27a1FWy0iYJ68EmdeKvdcyzgl/P9+Pl2ov6sHQJsya42++ytDUr7E5u7Md9J8FZn7IdyjFs6qgy9yQptykcDib1xTAfq2oG1xT+4hPzIAGnovv71r7ePBhg60/dQhJh97AnU54h9dQ3R+xCy9o2ofwTHfVpfpvbXEwF21YDpOSW+Eszk90l2Q2rwBp/GIl46Z2c1myOOOGLjDU+cZhpIjWkca7ia61S8Xgj0jaOGjD9I4+ic44h4d4ZXxpnq+Rp3KOBOP1lR77zlKE899dQNUNJobiTUzmwQwKdwy02bY+EjdN42920ZHuXtc/wUMW8+xuc///mGV9o4dfJHfB4/JYhx1sNp8rUT9WftEIh/whs2tfVtD/t6U+MN4VOf+tS2albPpbGP0ub56rzOUUH9eRAUE/HFmctGkxLyI9yQSzDmy8d6QkyvikwA4uQ4O68i9aAaJpE8xxxzTJvs2FeJ+Jyaf4IE10BYW66XcxWvLwJs7QHQsmy9vX1RLhwxhMZ8C2Pi8Qh/bnKTm7TXkY7l07PxiU98ogGRBhFPCDGNZu9Q1xet+uVEDx8QEYQ/acjsE0mxP7RMhLa6SOZXEOtWrcnwGl939Wrbcfhmta2E8jdBYn4x/9G3NSeccEKb4GrJUnO/9MRnaWXtnK+7Xv/619/oxOKTjJXvO58i2vkl++Ez7trC6/mhPY07Dl96XxSbhk90ku8P+PCTjgRtGP5ko5Psa/fik3zllS9Sr7c5Vk3K/J9pILf5d1FCfoQ5giLZOeec05ZRQj4N5CGHHNK+UKYHw5OltcB9Cnm//fbbIOijH/3o4eKLL95ojFWdf4Je2OUa1bCOwF/jwzg2NiW+9C7YCHuTfOLETB6z0tHTnva01nA69uU7PV5WQZKP03v9618/ePWdsamgCW9wqW9Q1xi2Wf90voEd0zgCwz47s7EeeBxyLHglfcc73rFxRG+pcfCG2fgcukbUMInnPve5rceeQMMlvfd6zPR4VZgvAjiEb/iFC/xPhvVp35797Ge3FWq8NTQs6+lPf3r7Tko+NsYnWULQ18z5OAFHiXixusNnsRDetoP6s3YIhDPsbWNXab1dTby/733vu9G+2X/xi1/cvjatU4Em8pVgMZ7xSfwVv5V5hdpIC0Ok/rUDahf4wSXkR0ZAUmLc65/73e9+rVf+sMMOa8ec2MMe9rDW43XiiSc2p8YBWk8+47+MhUbKkB059axxdgnO+ccoMRZEphGza2xLgOmZtzSpJdz0aFllxITXT33qU8M73/nOwURpTo1AO+2004YnPelJw53vfOe2EhJn+KEPfagJeTwRNMJ4g0vicGwa6M3vLmJPtoyAD4fY2FKm8RvE1/ve974mzHHJZFeNofC4xz2uvdl54Qtf2ObqWMuZqLemvKUoPRT66E+F+SKAV9oifkn75k2NZXD1vOPKZz/72cYvxw94wAPaajb4pdPBdy/233//JvYNE4w/EtvwV9227IfP80V8/e88nOltnDaHfYWPfexjbVgx0e6h7+Uvf3nrgLIAiI6pY489tg3ve9vb3tbmf+m4MqfwBS94weAhQMjwrPjBllh/dgiBEvIjuBD1wx/+8HDSSSe1CRxElvW/v/a1r7XPWB988MFt8qvGkmDnFH3emugy4cPmFTinJsR5pkGW5hpxgvnHGP2MOlxDBNjS0BevCzk6Qt5nzol2DaAPYniNKBDulp285z3v2SbCyqsHnsgn2F71qlcN7373u1uPl/ycHIdH3NkXijsNhrX9w342voFt4zOSnmNc0onAt2gQrfzglTYeKEvIm5CvU4G/4qvwyfAsYsyyp/3XXtcWsPrhlwsBvPKWh4/BpTe+8Y3tzbNvpQiGztz85jdvot1cjAsvvLANecA1XxH29Wl5IuLC03GsbdNTHz91uX50Fd4yBNhV4Ie0O/3bQWnaM+2YuYFnnXVW81EZ/mmeFy6JfdOCPzJ8Rpk3v/nN7cNRfBr/pS6cSrxlN7zGFy4hPzIe8hLm5557bhPq/Wm9FlYeMbmVUxs7KkJMjzzSIqWAoCFr6nIN5235Z8m5itcXAbaM2HYXGs08CI7v6vTTT29LUfosum8VhC/y4csHP/jB1pufRhPXOFJb8hZ3xqiu5zHh04+FH98FOxPiJih+9atfvZTfwRWfRDeemZAnvvqgnBW1DNGqMF8E4iv4J29ndEAZ2hD/Ahkife+99249q94441oCv2OJQW1j6sq5cZwHzPip8fk6Xi8EcIT9ifTYXhohbtgePvW2Zv+jjjqqCflDDz20dWT153EPl3Rq8V99ndlfL4S2/teWkB/ZAJGQFBk1sCGgdD3tBx10UFsxgpDPOVUo49hmX/7xlkstS8/5itcPgbFNHeMBDuGS4z7oyfDRDK+sCXlCLkE5Dk7ZZfUmPWUqXl8EYssxR3JH0uNfet/iPG6Zv2NM/HOe85zW24o7QvyRPL1fSr0VzweBnmM4FD71vsmbHpPwdS7orSe4EuQPj5bxNHn7ayWt4vVFoLdnbC8Oh8KnnNNuEfL77rtvW7rUl4M9QAo4lE25vu4crzO6emIAACAASURBVC9SW/fLS8h/F+xDVFkQjmMj5K0acfzxxzcif5fidWpGCMQhfa+3bFlAE8iMmx8L+b6OOMdx2o5ery9f+9NBgEC3tKmHQhNdDZuIkOe/+iDvIj71eWp/XgjgSHhi+Awhb7jfm970pksJ+XmhUnd7eRDohTxBb3ipHn2BD1oU0p6Vf1qEzvbTSsgvwQih8uQoi2NjCdMjT8j3DWOIuKS6Sp44ArG/OMF+31AmXRwhr0fecKy+R1453FK2r085x/3W11n780MAT/TIE/LjHvnwB1/CwzGf5odY3XEQwIX4GWkR8gcccECb7Nr3yKdMxYXA9hAg5K3qp0dePBbyvV9KXdWmBYnLFpeQX4IbYpWQXwJOJX8HAosckbQIqHGBEvJjROr4siBQQv6yoFZlIMA/lZAvLuxsBErI72xEt19fCfklGJWQXwJMJS9EoIT8QlgqccUIlJBfMcATrr6E/ISNu4W31g+t2bZtW/XIb4ItSsgvAbmE/BJgKnkhAiXkF8JSiStGoIT8igGecPUl5Cds3C28tbGQN9m1HyNfQ2t2vnFKyC/BtIT8EmAqeSECJeQXwlKJK0aghPyKAZ5w9SXkJ2zcLby1sZC31HIJ+dUapIT8EnxLyC8BppIXIlBCfiEslbhiBErIrxjgCVdfQn7Cxt3CWxsL+eqRX70xSsgvwbiE/BJgKnkhAiXkF8JSiStGoIT8igGecPUl5Cds3C28tRLymw9+CfklmJeQXwJMJS9EoIT8QlgqccUIlJBfMcATrr6E/ISNu4W3RsjX8pOba4AS8kvwLiG/BJhKXohACfmFsFTiihEoIb9igCdcfQn5CRt3C2+thPzmg19CfgnmJeSXAFPJCxEoIb8QlkpcMQIl5FcM8ISrLyE/YeNu4a2VkN988EvIL8G8hPwSYCp5IQIl5BfCUokrRqCE/IoBnnD1JeQnbNwtvLUS8psPfgn5JZiXkF8CTCUvRKCE/EJYKnHFCJSQXzHAE66+hPyEjbuFt1ZCfvPBLyG/BPMS8kuAqeSFCJSQXwhLJa4YgRLyKwZ4wtWXkJ+wcbfw1krIbz74JeSXYF5CfgkwlbwQgRLyC2GpxBUjUEJ+xQBPuPoS8hM27hbeWgn5zQe/hPwSzEvILwGmkhciUEJ+ISyVuGIESsivGOAJV19CfsLG3cJbKyG/+eCXkF+CeQn5JcBU8kIESsgvhKUSV4xACfkVAzzh6kvIT9i4W3hrJeQ3H/wS8kswLyG/BJhKXohACfmFsFTiihEoIb9igCdcfQn5CRt3C2+NkD/qqKOGfffdd9i2bdtw5plnDt/4xjfaL+KvvvWtbw1pL/MzcyyusOMIlJBfghlCXXLJJW2TxfFFF100HHTQQcPuu+8+HH/88QNShnji7C+pspInjEDs33PAPqdlG4dTTz112HPPPYf9999/OP/884dvfvObG1mU6x3exolv83DRtfo8tT8fBErIz8fWO/tOez+j7rPPPnvYZ599hgMOOGA4+eSTh4svvnhnX7LqmwECYyF/xhlnlJBfsd1LyC8BmJMrIb8EnEr+DgQWiWtpJeS/A6pK2IkIlJDfiWDOrKoS8jMz+Cbd7ljIn3766SXkV4x9CfklAJeQXwJMJS9EoIT8QlgqccUIlJBfMcATrr6E/ISNu4W3Nhby1SO/emOUkF+CcQn5JcBU8kIESsgvhKUSV4xACfkVAzzh6kvIT9i4W3hrJeQ3H/wS8kswLyG/BJhKXohACfmFsFTiihEoIb9igCdcfQn5CRt3C29tLORrsuvqjVFCfgnGJeSXAFPJCxEoIb8QlkpcMQIl5FcM8ISrLyE/YeNu4a0R8kceeWRbtUZcQn71xighvwTjEvJLgKnkhQiUkF8ISyWuGIES8isGeMLVl5CfsHG38NZKyG8++CXkl2BeQn4JMJW8EIES8gthqcQVI1BCfsUAT7j6EvITNu4W3loJ+c0Hv4T8EsxLyC8BppIXIlBCfiEslbhiBErIrxjgCVdfQn7Cxt3CWyshv/ngl5BfgnkJ+SXAVPJCBErIL4SlEleMQAn5FQM84epLyE/YuFt4ayXkNx/8EvJLMC8hvwSYSl6IQAn5hbBU4ooRKCG/YoAnXH0J+QkbdwtvrYT85oNfQn4J5iXklwBTyQsRKCG/EJZKXDECJeRXDPCEqy8hP2HjbuGtlZDffPBLyC/BvIT8EmAqeSECJeQXwlKJK0aghPyKAZ5w9SXkJ2zcLby1EvKbD34J+SWYl5BfAkwlL0SghPxCWCpxxQiUkF8xwBOuvoT8hI27hbdGyB911FFtHflt27bVOvKbYIsS8ktALiG/BJhKXohACfmFsFTiihEoIb9igCdcfQn5CRt3C29tLOTPOOOM4Rvf+Eb7RfzVt771rSHtZX5mjsUVdhyBEvJLMEOoSy65pG2yOL7ooouGgw46aNh9992H448/fkDKEE+c/SVVVvKEEYj9ew7Y57Rs43DqqacOe+6557D//vsP559//vDNb35zI4tyvcPbOPFtHi66Vp+n9ueDQAn5+dh6Z99p72fUffbZZw/77LPPcMABBwwnn3zycPHFF+/sS1Z9M0CghPzmG7mE/BLMObkS8kvAqeTvQGCRuJZWQv47oKqEnYhACfmdCObMqiohPzODb9LtjoX86aefXj3yK8a+hPwSgEvILwGmkhciUEJ+ISyVuGIESsivGOAJV19CfsLG3cJbGwv5GlqzemOUkF+CcQn5JcBU8kIESsgvhKUSV4xACfkVAzzh6kvIT9i4W3hrYyF/5plnVo/8iu1RQn4JwCXklwBTyQsRKCG/EJZKXDECJeRXDPCEqy8hP2HjbuGtEfJHHnlkW7VGXEJ+9cYoIb8E4xLyS4Cp5IUIlJBfCEslrhiBEvIrBnjC1ZeQn7Bxt/DWSshvPvgl5JdgXkJ+CTCVvBCBEvILYanEFSNQQn7FAE+4+hLyEzbuFt5aCfnNB7+E/BLMS8gvAaaSFyJQQn4hLJW4YgRKyK8Y4AlXX0J+wsbdwlsrIb/54JeQX4J5CfklwFTyQgRKyC+EpRJXjEAJ+RUDPOHqS8hP2LhbeGsl5Dcf/BLySzAvIb8EmEpeiEAJ+YWwVOKKESghv2KAJ1x9CfkJG3cLb62E/OaDX0J+CeYl5JcAU8kLESghvxCWSlwxAiXkVwzwhKsvIT9h427hrZWQ33zwS8gvwbyE/BJgKnkhAiXkF8JSiStGoIT8igGecPUl5Cds3C28tRLymw9+CfklmJeQXwJMJS9EoIT8QlgqccUIlJBfMcATrr6E/ISNu4W3RsgfddRRbR35bdu21Trym2CLEvJLQC4hvwSYSl6IQAn5hbBU4ooRKCG/YoAnXH0J+QkbdwtvbSzkzzjjjPqy64rtUUJ+CcAl5JcAU8kLESghvxCWSlwxAiXkVwzwhKsvIT9h427hrZWQ33zwS8gvwbyE/BJgKnkhAiXkF8JSiStGoIT8igGecPUl5Cds3C28tbGQP/3006tHfsX2KCG/BOAS8kuAqeSFCJSQXwhLJa4YgRLyKwZ4wtWXkJ+wcbfw1sZCvobWrN4YJeSXYFxCfgkwlbwQgRLyC2GpxBUjUEJ+xQBPuPoS8hM27hbe2ljIn3nmmdUjv2J7lJBfAnAJ+SXAVPJCBErIL4SlEleMQAn5FQM84epLyE/YuFt4a4T8kUce2VatEZeQX70xSsgvwbiE/BJgKnkhAiXkF8JSiStGoIT8igGecPUl5Cds3C28tRLymw9+CfklmEfIaygFxxdddNFw0EEHDbvvvvvwyle+cvjWt77V0nNengrzRIDtswUBxzhiG4dTTz112HPPPYe73e1uw/nnnz9885vf3MiiHN71/MrJXCNx0iueJwJjIf/JT35y0JAK4U/PQ/sVCgEI4AL+hBNnn332sM8++wwHHHDAcPLJJ7f2rpAqBHYUgcsq5MPJHb1e5R+GEvJLWMC5XXLJJW0Lwb74xS8OD3rQg5qQP/7449u5OEFx9pdUWckTRiD278WTNMdJ62+fkL/Wta417L///kuFfBrZ1J14WZ19/bU/DwT4qEMOOWS45jWvOTzvec8bPvWpTw3/+7//226+50m/Pw9k6i4XIRAfkjg+Rt73vOc9lxLyF1988aIqKq0Q+K4IRMjf4ha3aENs+qE1/BXOjdtEfBynfdeL1MlLIVBC/lJwXPoAIf/v//6v9XBpHD//+c8PD3nIQ4ZrXOMaw0tf+tLhP//zPxv5lIpjvHQNdTQnBHDgG9/4xvDf//3fGw+A7p/jwiNOjLMSCPnrXOc6wz3vec/hggsuGP7nf/5n40FQPfLiXy/AwrFw0nGFeSOAJ494xCOakD/uuOOaj4pfwg/cC4/Cn3kjNu+7508ipvABN3BEIOT33nvv4d73vvfwjne8Y/jyl788b7Dq7i8TAjh19NFHD7e+9a2HJz7xicO73vWujTfO2kftF8717Zcy2kBxhR1HoIT8CDPk4uzSAEY0EfJf+MIXhoc+9KEbQv4//uM/NpxgNZIjIGd4iAOc0X/9139t8KLnUi/k3/nOdzYhf6973Wv42Mc+1srJK6gnQh4PpdvCTVzES8cV5o2Aho+Qv/a1rz286EUvaj6KkE9DKe55N2+06u75Ef4Db+xH1EPmrLPO2hDyb3/720vIF10uEwK4Rcjf6la32mEhr12rsOMIlJAfYUYcIaIxyxpBIQLKq8YHP/jBw1WucpXh1a9+dUvv85SwGoE5o8NwJKLbrdvHJY3lmBt6vAyHuMc97jH83d/93cZwiPANr8ZbrpGHy3GdM4K7bvXbCODXox71qPZQeOyxx7ahNZlvgR/9VqAVAnxKekXH/kPPqR75u9/97sPrXve6wVDSCoXAjiLAJ23btm24+c1vPhxxxBGDD0Lp4BK0iYt8krSc29HrVf4aI/8dHEAoDaHhERHpMkkn5DPZ9Q1veEMrS6QtEmrfUXElTBoBHODAOCMBXxwvazR7If/xj398u0Je/XF09nHTNSrMGwEce8xjHjP84A/+4PCsZz1r6LmEH7bwJsfzRmzed89vaN8W9Xway3zLW96ydS7oqPrc5z43b7Dq7i8TAhHyJk4/4QlPaMNII+RTYfxSfFPSK75sCFSP/Ag3BFsk5BFOD8UDH/jA1iNvVr/AIdqUqzBfBHCAaCeyBQ2mV9geCMXhR2JC3mRXK0T04ktZefDNph519kI+51LXfFGvO9doPvaxjx322muv4ZnPfGYbpoVvPY9wyIY3xZl5cwYP+Cq8GQdf4Lz97W8/3Oc+9xlOOOGENt9inKeOC4HtIYBbxsab7Hr44YcvFfLaNFyMXyrftD1kl58vIT/CBpkQTGNo4/gEscmueuT32GOP4Y1vfGNLR1r5IrRG1dXhTBDAAw+AEUzhRTiEV0kDicmuelEzRt5DgDyLNnWGX86XkJ8JqZbcJvvjkoAbj370o9sY+Wc/+9nDhRde2PyRcxpJG+70/FlSbSVPGIH4DXyJj5ImJCbk73CHOwz3v//9W/tWQ2smTIgV3hrfdMwxxwy3uc1t2qo1p5122sbQmnCPD7Mvb/ZzboU/bbJVl5BfYlrkIq6IM8ExIW/5yatf/eptDGHSNZIaTESsME8E8CNiyT7eEPFpJMXSTIQVNJrXu9712goR1pHPBNnk71HEq9TtvPrFi/L25Wp/mgjgA98k4MAjH/nI5pOMkf/sZz/bGkfpOJW3RMWZaXLhe70rfMAbm/1seGET+KTb3e52w4EHHji89a1vbUNJv9f6K18hEAQi5G9729s2Id+PkXcubVm4J5bmHH5W2HEESsgvwYyjI7wi5B1nHXmTXb16TEDEPFkmreJ5IYAfeMARcUrpCZUuiHGEuBeMR/2BH/iBNrHsvPPOG6yApIw6xiGOTpzrZH+ct46njwDb41LCwx/+8GG33XYbCHmdDfFFVq/hv8IV3AkfU7bieSAQ/xMhLw5PgoC3hDe60Y2aTzrllFOakC++BJ2Kv1cE8Mok15vd7GZtaI0HxIyR1zbi3tgn9W3m93qdyvf/ECgh//+waHtp7BCNsLIlWFdXj7xG05ddeycXYiZvxfNDAB84pAy1CpfE+BEHBhm9FL5HcNe73nWIkFcuPBpzKw1w6kq++aFcdxwOQAIvDj744OHKV75y+yCUCYoaUnn0yOfB0XG2QnB+CLB9RJR9HPGQhz8J5u1c97rXHe50pzsNb3rTm+rLrgGm4h1CALcOO+yw9lBosmsv5PHNlvYrPskxfoor7DgCJeRHmMXhEfAhHIJJ/9rXvtbGyF/hClcYXvGKV1xKyIeQo+rqcEYI4ADhZDgD7oQT+BM+BQ69X97s3OUud2kTFE2KTZ44ueR13NfneJwneSuePgLhlTslxvTIG+733Oc+d/j0pz+9Id6di+9KGXGF+SHA7r3f4Gv0kuJHgrHMN7jBDYb99ttvOOmkk0rIB5iKdwgBQv7xj3/8cOMb33ijRz5DAdOWac+ir3rfVP5ph6DeyFxCfgOK/38HkTSAiId0QnpK//Vf/3U49NBDh5ve9KZtMlCcY4g4qqoOZ4RAOBAhH6HNqUmLqAok55xzTvtghg+MfeYzn2kiXqMaR5fy4VgJ+SBXcY8AMWZi2R3veMfhJS95yQaX5MmDYbgZTvXla38eCMSPhAP80ljIv//972+raD3sYQ8bfBDqK1/5yjzAqbvcqQhox0y899VyMV5p/4QI+D4OJ3G0wmVDoIT8CDdkQrqsI49kRD3HJ+1lL3tZW+7tAx/4QBNd0tPrNaqqDmeEAN5wTrhjc4wXGssMcXCc8IlPfKK9fnz+85/fGkxlBXyLk4uDS5o6bUm3X2HeCPBNr3nNa5qYJ74uuuiiJuChgm/hijj780Zsnncff8S3ZF8HQ/wOVD75yU+2bxEcd9xxw4c+9KHhn//5n+cJVt315UIApwzNesYzntHe7PjgIZ0k9J0L8vFRtvDycl14xoVLyC8wPmLFySGY/RDOCiPGN/s4lIYx50pULQByRknsHz4Q8vZxh9ByjD8R+WDRSOKRB8J///d/b/mlp5Hl+OLcxMqrM3nkK841OGb3p7c9nmgozzrrrPZV13/7t3/bEGfhIP7Y78vNDrSZ3zD7a6siqBwTVdLEAu6ce+65w0c+8pHBfDAdV+VjZk6cy3D7uGWI3wc/+MHh7//+74d/+qd/aj4p7aG2TMAtafiHl/FRl+GSsy8yeSGfxitxCJTj7xZHQCEZ8smLbNKzj4Q519cVZvVpy/aTt+L1RSB8wBWcwBNbBLjzGkaNZfhC5Ouxj9BPHfgVzoVvfT3yVZgOAuyZLXeV42UxjuCG87iCQ+L4ppTDm/AxdVc8PwRwha/hc3DDMb7kjSHe9EEeaWO/E16N8yZ9HPf5an9rEehtk1/Sp9lPGKcvO16WP+l4Fj+FS/gnzrXCQ2m9T8t59Sy6durvzyVtjvHkhTyjcli2kEPsOD0USUckDWLv1Hqi2e+DY3mzOR7ncZ0Mrch15Bn/pr7e2l8/BNjWhgtp/NyFNPaWhgeWBLQviJOX2OLkwr3U1T8o2rc5V2E6CMT24YU7w4OIc/bOFs7wH+FKj4Q0POLbUgb/ijM9SvPbxwF8wo3wClfSDjk/Ds7bwqNlnHO+z6sueRfVOb5GHW8eArFj7wvs8ymxc36NtLQ97CifOG2QuLdvzottCfKoW5Cea/V12s/W8yh1KJf0lFt0LmlzjCcr5Bmf0REgrw9DMLE0Ti1kRTBpeijsCyGQ42zqk55NPteJk8s15RPU+S//8i+Xek0pT/4h7KurwnojED6w53jDBfzArfBLmgY1DtG+Hvuee/ali+XHJVs4U7yZBmfY2EMejiTYxwf2jp3ZPRwIJ8I15eSTri519pxMHam/4nkhwP44hRd80NiPhCtinOr5BSlpfJU67I/zpx2VzlfFr80L5V3zbntbxb6xoZhNY1d3IA/74UqfHrtKd96x8tlyHA7gkHx9Hbl+n1dan66cOhPs93zM9cTqGedPuTnFkxbyhHqE05gsCJA0Tu3rX/966y2VLiAJwtrkk25fXsSRliBvHJd9H/exwk3I1zfIuWaRMOhNI2bP3qHgCwcWvjjvGCfTKyaPMrgTrkBDup573FVOHTgU5+m89J6D00BxXncRTuAFm8Y3iNk6HQ2xs/zhjzSccSwWer6lLucJ++SZF8J1t0EgfImfCafwDm+k2+xr59JuKh8u4Z88zscXOSc9fq7Pm2vkN1S8NQjEJoljw9jMMR707Qqb9sfhgfRs4QofgxPqySaPkHrim/DGJp9z+U3hoXgckiexa2kP02GResbl5nQ8aSHPGTF2T44QK0RibCQjvOVFFkE+Yoood15+BET+HKtXfnmdz74yeuGTljIhYgimXOpKWsXriUA4gBO2HMdpJd0xfuBWuCYtXHL39p3nIAXnOa7wyTHepPx6Ila/mj3TAAaNnjeLeMGXpAHGgfgWddnvy+AHn8YXOVdhvgjgQnwQFBzjijaSb8GltEc4KT08kw+/BPlwyrnUJ+65N1+Ud807Z2tbAnuyYToB2I/tEyevWHp8krgPuIArtnAET2ypS/6Ulyf5nM915HGsnDr78n0e+RyrA2dtrj3O0//GuexPWsgjBKMjB6L0AYl7cdSTDTFsVhPRCCK9eqQlH0dnG5NbHnk5tuR37Lc4tiWkrnEdOV/x+iAQ28b2sal0tu8bPpyyhZPyxJGJHSsvFuxLF9vSA5Lz64NS/dIeAbzQEPW+wX7sy9ZCzw/78kRMOcYtvigcUT5889BohSTHFeaLAJ6kHYICPph47020WBsphGuOcUqHgi2CSXrPPRyVFr+nXvyssOsgwKZ9YCN+ofcJ8vT5csynsG18SvKInWPr+J2UwYnwwX7Sv1vs9zmPZ+nEwitpCSm/qM7kmWs8aSEfojE8wiFXCIYwNgSN45Evjah8hD6nhVghvTrtc3JxbgiWupEv10t5dfbpridPtp6scyXiut83W+KGja1xJI6IvXHFOfnwIpxyTroYhxbxyvkE5dUtb/EmqKxnzJaxf88b9pWe4Bz+4Eb8DJ9i3zn7+OQY58KhcKXvsCjOBNV5xeyOG+FbeIIbY37gDV7hE26FdxBzDhfDz3DYsfzOqbvCroNAbMY2sVdvQ9xgb34nm2PpOce2Num22Jutkzd3nGs4p3xC0vs65FFv+OS4923KKpfr+33KV7g0ApMW8ghgExBEY6eXXYw4SEaoc2QhGafVp0lHLASSH6Ecq6+vWxmb8tKdt36qLfldUz3y2JdPnT3ZL22eOloXBDgX3GBT+ziGD3Fa+OB8nJAYL3ABJ/BBHj0ltpTL+XBEnHNJWxeM6ndeGgH2iz3Znq1xIhxxDp/wBp/0nIrTmOGMBjliTFnHNmXCj9QTf3PpX1FHc0AgPMMZG45Jy+Y4Pqg/hzPZwiPnw6XUl3N47HyFXQcB9shbYPZKYEPnbHyK9krbw8fEf4Qfsa/yzvEx8TfS1NHnXbQvn3KulTr4PP5LmjJCX9axcvL5ffIu+m0pm3ubWzxpIc+YyMrwiICkIUIMHRLmWCytJwvntCiEYCGia/UB0dMwI5rzCCtNrF51jMv1ddT+eiDAvpxZBBcbs+3YwUgTpMehLbJ/6gvHcJcj4whTZ+L1QKh+5RgB9mN7NuZv4g+kS0vDFWG+iCcR//Ljnzpwb5xXndnGv6OOp49AuIYX9i9rSPnwLMIvfMPB7F/Wa1S5nYsAe/ATNn4m9uEztEGObWzq2MaOgjy2BPmcS7r9cEv5/jj1ShuXGafF/4WbfX7XUnc25/I7Uo/jOYfJC3mG1rARWJyORlFAmJDGRNcLLrhg+Id/+IdG9J4Q8qjDWMLPfOYz7RPoesYQz6aH3z+IIP7a1742fPGLXxy+8pWvbNQV4iFb/llSXux8hfVHgH31fHgLE+fHvtIuuuii4R//8R8bD53DK3G44WHQWGa8UV65cJSgcw5P5cOhnFt/1OZ7B2yIMxpOfimcgYg09uaz7DuHE/xN3vTxQ8qlEVMfPskXznzuc58bPvvZz7bj+SJdd94jgC+484UvfKG1edo9PNG++Uqwr3F+/vOfbz7rS1/60mDTpn3qU58avvrVrza+4SOO4ShOpg2LX3IN+xW2HoHYhI20HbGVff7DcW+rPj/fY4uPcTfOj+2rLjor7Zbz6sWTbLlu6uiv6Zyy8iqrPtft03oklZXXeVtfd59vLvuTFvIhk5ix08OFIES3Y0R+61vfOtztbncbnvCEJwwf//jHm1jqCSD/6173uuFe97rXcNBBBw2/+7u/O3zsYx9rREK4hA984APDc57znOHQQw8dnvvc5zYniWD9K3Hkk5bfpvzcSRj81jlmQzyxsSlHY18D+LznPW/4iZ/4ieH5z39+ayBxLkE+4vwv//Ivhxe+8IXDz//8zw+/8Ru/MVx88cUtCyGHq+rEGXUS9TjluML6I6ABZE884KdwIg1V/ITz55133vAHf/AHzb8ccsghw5ve9KaNm8cpggo/BP7p2c9+9vDoRz96+M3f/M3m1zYy185sEcArXHrHO97R2rsDDjhg+LEf+7HhJ3/yJ4ef+qmfavu3uc1thtvf/vbt+C53ucuw3377Dfe5z32GRz3qUcNJJ5200XHF50WkAVTd/BQuxmfNFuhd6MbZReBLoj8cx15sxQclsCs78kU5lzrG9aiPz+G70lGavLmePH1blTIR4Mkvdk66OPxyjFOuk7Y1v1++PEDk988xnrSQZ2RbGkNksDF8iIcYr3jFK4YrXOEKwx3ucIfhne98Z+sVDRmQSQ/Y0572tJbnile84nDf+953OPvss5NlI37Vq1413POe9xxud7vbDY94xCOGCy+8sJ3re9bUl2C/SBg01jvGM46GTTkkG9sS5E960pOGq1/96q0x/PM///PWE5a7xU099b//+78/aDT33Xff4ZGPfOSGkCfO1JOAapEPiwAAGblJREFUv+md751jzle8PgjgSHyAN4YaTY0Wu9pwKgEPPvrRjw7HHXdce9j7kR/5kSbo9aamgUtHhTpe+9rXDj/+4z/exNiRRx7ZelpTV8XzQyA+CacI+be97W2tw0AHw41udKPhlre8ZRPxt7jFLYZ99tln2Hvvvdt2/etfv7V72se73/3uw8tf/vINIR8Uw1W8s/FXJeSDztbHbC+IF7UZ2hR6SEwP8Ul4kv2Uz504dj55lGPvRTYfl1VHhL9YHdsLfrO88XPje1CH3zznMHkhz/gRywgQMkvndIRTTz11+OEf/uHWK0HUf/KTn9wghgb2/PPPH5785CcPnNruu+/eelfHQt41iDEiXi/HM57xjPZKUv0hmmtnk+4fJT1x7YfUn7VFILzCKQ4NH+Kkzj333MYfPV0aTA+LgjJ46JX1L/7iLw5XvepV2wOjV9ucoyCOkyLU8EWZnkctY/1ZOwT8/2tA2TO+iV0j7g3JwqXYWj6dAkT6rW51qybU+SUCPwFH/vqv/3r49V//9ear9MjrnXedCvNFAL8itO0bKvqJT3xieN/73jecccYZLeancMfm7TKx763ODW94w+GHfuiHmg/7q7/6qw1/BE0cxlF+SVvpWP3h83wR33XunP8Q4lvSnvS/kG/RQaTDIP4m5fp8/b56tHHfzd6pqy/nWhH9/TUW/a6UC5/EuW5fNvnmGk9ayDM4x4I4YoHxkS/H0t7//vcPd7rTnYY73vGOwxOf+MThgx/84AYfjFk2rObwww8f7nGPewzXvva1h5ve9KYbYiwZXeO3f/u3h7322mv45V/+5eHkk09uPfk5nzjE9ts41jTkOV/xeiOAW7hg43QE4us973lPe5ODP0996lPbnAzniPZnPvOZw0//9E+3h8C/+Iu/aGV6nsR54QqhVg6sQbT2f3DFg5pY6H2TdMJIPLa3ccyEut7UO9/5zsOJJ564wTU99L/1W7817L///o1TOibi69IYrj1wdQM7jADb80nhW3yKYz4lXEvF0t/85je3oVk6px7wgAe0Di9vD7Vbab/Svqoj4sy54lqQ3HVi7YeHt9NPP71pGsM56Rs2ZC8inm0FcyE85J1yyinDa17zmuFP//RPGx/4Hg9tQvxVO/j2Hw8D5huedtppwwknnDB4A/2GN7xh8JBoroVr8Xfxebjidxm98Pa3v73l5c+8+XHdd7/73W3uhnyC36njS3uqQ0yHqrL4O+cwaSGPaLY4FfuIxOGESIxv7Omv/dqvNSF/73vfu40fDCn0zhP3xgd6Ra0nTM88cR9SyqsHVuP6fd/3fU3QK0fAyeO6Qv97IvaSlutVvL4IxJZinON8shme9axnPasNdcAx8yzwg7Py8EfIG0uv9zROK5xxjK89j9cXpfrlQaDni32+goCy9X6D3dObqqye+ne9611t+J5hEU95ylPaW0R59K5KI740vp/+9Kfb5dSPQ+IK80MAh8Iv7R+uhAv8i/HNxLgg3dwebd51rnOdgb/64z/+4+avnE+HQt+OKuMaOBau9n5sfojvWndMn2hb+ISHP/zhrQNAhxKBzfYC+wlsSew//vGPb8OpDLcy1OpnfuZnhhe84AWtjPrYN2XYnV8655xz2vzAn/u5n2tayTCtu971rsNRRx3VHgQMNaV9EnCJ8H/JS17S3kobXnrb2962jZC4+c1vPvzSL/1S01p8ooDD5nfweTTbYx/72DYS4sMf/nCqnGU8WSEfa8bBIJx95EMkxEswY//3fu/3Wi+WRpC4SjCs5md/9meHgw8+eDj++OOHhzzkIcOtb33r4UUvelFrPBHM06B/COS9ylWu0kgpnUNzvfyGEN9vibPLdSpebwR6G7MzftnsC+yNI8S6iWU2PfEPe9jDhmte85ptvDOHprcjXBUrb1N/hekiwL74oqFib7ZPyLlwyTlDI1784he3eT0mSP/O7/zO8LKXvax1OhBfv/ALv9B6rvCu51PxKKjOK2b38Ej71z8oQsJxOMcP8VMmvP7oj/5oE0pWtAl3lM/DZnyTsvb7LfXNC+nV3S384Rtcg/miKxLnhkiZEE/bHHbYYU0oH3jggcP97ne/pmf0mOslJ6YTrFakNzxin1g2Wf6II45obZVhw9u2bWu958omWK3PGxzXIcbpJIuHWLjhoQ99aNNWRir80R/9UXugSDkranmDyIc98IEPbPmJdA8Rrk3Ie/iI+PcA8bd/+7dN3B999NHDgx70oDZSQlkdre7V78BhbxgWheAWPi/Ks25pkxXyjJSN4WyOY8ReZGkUjZMnxHfbbbcm0mNIr5+MbUZM4watRuMfAYmsdqNXQu8FgprdbxKaYTXj4Lqumd9h3zYlMo3veU7H7MjJsqlG0RZ7w4HdbcYz6+G68pWv3Mae3uxmN2vDurzhSUgjK7999YbL4zhlKp4fAnyThlKPl84Fb3Xs80NWt0nQ04pHFeaLQNoZPsXDXXyT4wR+xlCLP/zDP2wPiObzEF+EU4L8EZP2czyuM34q5Sq+/AjAlN1iO7H/a5sHK8LVW17DW/SMH3vssU0MG37nre91r3vd5h8e97jHtQU+DFERlFeXQNPwH3rS6RzzcQz/1JuPFx7uvO0z90bHVIJRDXrH+SFlrd7nQUKel770pW0REHyyGIiVj8LHD33oQ4Mefx2ohi+7vo5V1zzzzDPbm+s3vvGNjbOuRW/ZPHCoxwOG1ZaMknCPhht6ANDRatiNBwz5+cB04AbD/IbcwzrHkxbycTRxNnFADGg/5GVoY7+8SiSw9G4R94JX1cQWclh315gxy0vqSSXq/fNk+I1eVkTyUDAOfoPrifMP6R9oSmQa3/OcjtnRFqcY7jnGtQSOxStNQ7QMw/Ia0STprHCUOsIL9Wgk9USI1dXzOPVWPD8ENNgaK75JI+1toEbNuFRj5QWNl2Fd6dGaH0p1xxCIf4ovSduj7eNbBDzR+2kIhcmtRNJHPvKRjfPypB3rfZC6pUvTxuGazXGFnYdA2gSi3TAWMYz1jJvXR3Trzda7TbzrGbeIgh50wtqqegQ+0W2svDoS1OVbJ9omb4gNE7YUtzy4Ijj/yle+snVEmSfYL39LgBP4OhPUodNTTz9+eZvz+te/vv2ua13rWm14Dr4Ifo/rEfPq8yCZgJcEu/vD23AMDs75PUZMvPe9722dpzovvBG4//3v34ZJ+y3uHyaw8dCgfvc6tTArIc/JID4ixOkwKIIQ7iarWibQE6sxV1/+8pfb5Iub3OQmrefLUx1yWwLOxNdf/dVfbU/Aeiwe/OAHt0YUcUyeTXAtW0gozvXzW5K34vVGIHYNz8QckNg5gfMxVt7SgFnylPDisAR5ORo8SVAHp6hxTN3hUfJUPB8EwiV37BW6jocb3OAG7cHQty7S04YjfJbGWMNXnJkPR8Z3Gt80FvJ8DY5I91CoF9bYZO2bVWsStFX4k3Yzx+oNH50n+tRJwEUApo6KLx8Cwdr/sof4tC8WSDAG3TATHYk3vvGN2xsVw4H1yhtTbq5ML2Dts3lsp0PSBFIdlNol4t+EV+1SPhJGNKuLj5HHUGMBf8yh2HPPPZuI9qYwE2Jzx94UmGuonCE3EezyGn+vQ8s4ecKe/9JOKpPg926PT36vCbP8oaE22ljazQMGbLS7JuGasC0Ez2CQa61jPCshjwwRVRwRYvTk8DrG6xli3NMrx+YpT4+83omILGRAEq+xNZKehvXGO0YixBfi+BAl++IcZ38diVO/eTkCuMW5aczYWJDGuXGWxinqhTCWWaPpIfCss85q+ThX+XAtQVnpqWtKDij3WPH3jkDf8OAFv6UnXgOudz6NZP/wZ98WDn3vV6ucU0AAZ/gUGw5kc2/8lN5KQsvQUlwynEFHlqCN5M8ISHzjj2zq6H2RY+ny5zpTwG5XuQf4shU7wFnQVhjPTiD7xo1v4Zh7ZcUZQtiDftoTNlGeDkr53BufYYUr38jxplgPue+ZEN1GKvggmHO0kaW6v//7v79pJHUakWAZXGn8j05RvBhfwwcP8YuoNpmWdtIJqvff8OUsAa6D1IOJsfoEfX53OrLCOb/dfrgsxs+0vXSZ3+9eXNeQHz32HkgEedWtjnUPkxbyMbIYqUIERmN0RkTqBJNcDXnwatEEDE+ZXssQ6Z7mYnATKTz5+iKesVwmVxgbb1KsVziZBKJ+Ti3lXNPvsNmXnnP5DRWvJwLhE3vaj8O1n8Cx+LCYh8A4SuPl9abqOeFwOWk8DS+UD2fwySatuBNU5xezPU4IuOB1t+UBCTAdDnrrBL1s4ZIYJ3s+tkz1ZxYI4EyES24Yh6QTYiY0Ghut91JnFBHP18iDO9pJ5bVn8Uk5n2N57dvKPwXlnRfDlR3YA/aChTbYS3uiHTGG/TGPeUybs2eMuZEFhrb43x+HPKBJZ2/z/LRHV7va1drQFPNvCGri1xxB8yVMQuVriHpiXHtlmM4xxxzTHiZcO77Jb/RbE/S477HHHq3X3jxC19Q7bhlJ5Q0FsnkrpNffQwPd5ToeDsKt8Cz1iqXxe3rz3bOe/T/7sz9rE221tzrM6DPfRaDZhPAXV9c9TF7Ix6EgAeIyeAxnXy9V/imM80IirxW9mvGkiMxeN3mNpA7B5FZPqJ4ilTH+DEHNsEYiT4QCx4fIuZ7yrhVnmN/WMteftUaArTk19o3DSZwb83aH2NJrYp6FyUBPf/rTmwPkHM3HyNwMZZTvOav+8LW4E1TnF/NbuBCOWKdZr5zGjyCLkNfIa8DDyeLM/LiSO2b7CHH7Al5YDlAHlo8+mXCY+RXJj0Py9cE5adq21MlPSXOu3/pytX/5EICrAGt24Qe0EebD0CE6H00m1bPteyWGrNAkOiS9cemHqqiHn6BlBPMjaBwLflzvetdr7ZMhwsafW8vdmHcTVw2FMeTKGxvDdbRHes1pIG8EfNE++sdv80YgYt5kWTrJG2jzCF1TcB9+B8GuF94Ye/pLp6rhOoYIWd0mIdyDQ4JhNUZK+BCnezCKwugKY/kPOuigtpqNYUEeOmCXEK7meF3jyQr5GCSGQiqEQQLOJ8eIGEIgqidQQ2S8hvGBKMu4eVXlCU8ZwYRFr3+M6zKWzJOeV1ueXBEq/3AI3At56a6V6+e3JX9+c8XrhwCbEk3hUn8Heto5RT0Mhj9YJsuwLfk5Lw+PPkjGCfYrAeCr+sQ4gku5Bi4Wb3qU57OPD/gm4IAVj3ALh6z8kAYSXzSqi8TYfNCqOw1P+IyIIGn4QbyZHEk0GUqRuTqLUFPWJoj1Dsc/4WT8VNrW8k+LULzsaXBlM7jbgjcb6Nkm1vV0WzyBH9CLTr/wDTojrTRjCIwhNAQ58Uz/CHwGIaxDgHj2UEDnaLs8AGSYjvzSdDil44ov0vutJ9+w0b/5m7/ZEPO5W79V7/qVrnSlxjPDW3Lt5OGnvD3w9WCdWnrPTbo2cdfYd9z0ltG901k0mSHQvsnift0jLtvsezup8+wtb3lL6zTzwMEfpu0Md6fA01kIeYazCWncGJEBkSKNomWPEEOPvPFeiKnnXY8X0qcOT7KeGvM1WCtGyK93FWEFdbuWLUQRc4CuFzIlboXqz9oiwKZxbLF/bkZvhvGnXlt7OOxXNTI2kfPVk2IJLb0nHGg42Tsb+9JxrHgTdOcX8yPxKXhAjGlAvVb3BlEjl6Cx1AjzQ0LK5XzF80Agduc78mbPEAPjlbVzxI8hWkQaP2PYQ3ijjZRuyKhxzTinHcOptHfxR64jTR3SKuw8BOBK7Pqfjj2XYcwH6IE27EZHkTcuV73qVQerxhDGhgxbCjmBXfVWa6eucY1rtAU/DFPBFdcQ40HaIHFsrw5C28RSw7MM0SHUlfE7iWfzBo1wsCqgRUVoKOVt8uESTqV+dapDfXSWBwUPCHjpIcK+3ndDfPS+G9ps6Ut89nbAB/MyV0hdfkPvA/vrBstgsY7xLIR8CMNAjOl1T4RSyINAnJWhMZYv0sNu86lzT69IHIN7KvQKUuPpFZZXSkjsgywJyO9arpNy4hDVecdiW4X1RoAN8SwhD2y4YpUjb3iM//Owp9dBkMdGuJuUo/dBD4oGNb2q+BN+hC/K2rdVmDcCGuA/+ZM/aeNWNXp4ZJlcIfzih7zJ4cPCpXmjNr+7Z/e8zXP3ejMJK8MPtHPaPL7J8pPaNuOZ9dz6aBB/ZDMExzwf9fT+jh8K1/jAvo2bH9Kru2MYE/H+jwW4wzs6o7+ydkMvvfXfLbBA1NMn/INhnEYRsLOHM+XVRVDrFDDSwLBhw3IcW4LUMBrCWj04osPJXMH4E51VOERM+2iUL8BKM5lVj7gRDN767LvvvsOrX/3q9lP5LpNyzQ/zIScdXAS/YTTK+igZgW7IoCE+Hizdv+v6DYboeEgxKkJ5Ha547QGV2MfDhLSV4ryldN5xziXvOsaTFfIxUBwOwktDWk+1EfLSnENi/yieZH3YwJPj7rvv3oRVJkfE4ByZiSRe3ZhgYkyZJ0EkT3BdRFFnyokdS3fesdhWYRoIsC9u2fBMr4aJPMYt6i3loDgSIQ7ZvkaSczWsy9AbrwGFOOnwJJwR2yrMEwG257c01kTWIYcc0nyQXipvD53np+QRcC0CbJ6Izfuu8YA/4p+EE088sS3YoCOKbzKO2NALQxJ+5Vd+pU2YJPRNXjTkxr75FxZ20DnRB76Jn7K5Tt/G9flq//IhAGftSq8f4O3/Wpsi3cbO4nFgH8NWDEUhrPVyE819O2Rop155C3zo6TbUhkg2UsHDnbbMEFBtmQeE+BcTpg3pwSOTS9XvOoa2ZOUYC4QYAmMIkEBs41MeLAz78fEo/syiEL4Ia0ER5fk0wfWsg28svXQPn8S7nvr8lpbx23MJ6LoeCxhmJR/pU2lHJy3kI34Y2MZonAzi9ka3HzKLEdaTnh53r2nyqlp5AQEINCvUmMmN7Mp4+k2QN9f1O4Sk+Q32bc6JK0wDAb0MetjZ3r6x8Bya+RNm5+cB0t2GF/JyqJyahpMjzcpHeiDy6hHvbDiqnuLNNDizo3eBNxplAR/0VmlsDffTCHqziBv4hzsCjslbnGlwzO4Pu6etwQlDIXRY6dUk3n07hZA3UVCbpjPBsoPO24h7Psx8DEKoD+qNjwq/xNnv89b+ZUcgNvS/HA0Be+1BHtr9j/fDSFxNHj6BPyDyiWLDanQWpT1Snwc0b4KNRde7TXTroDTsU6+4jibj7Yl5HU958+f66s04e8N2TFC1Ao5hPMoS/97wqNtv0a4JeuBdxwMAjhH18mexER/dNF7eb3Mffq9r6cBQl3bS/Up3D/wiLkqDySKfpx758FjeKfB08kKecRHfxmD5JxAzqM0+g4f0nko9gSKrRjKkY/yUQSzCXc+GGdyeEpHLeaH/p8u1+jT7/dYK1Z+1RYAt2dnrPz0NeIAjhmp5U6PhtEJEgvyCfDjKERnT5wt4HhCz8gjOcZRiHJUv+6kjdVY8DwTwDA8SNGheUXs1jW/4IWioso8ruFacCWrzitk9HCBeDHkwN8cwB5t2jNDyQGjYhCE1hi/4RoFe1Re/+MVt7LOOiYgffgun8BHPcDLXKK7tfH7lfzf//2Jp2gX425dGwDq2zw7sQsQT0AnyZpOmDu1V/AqBbCEGw0KPPvro9hAnNmTGUBkCPOI6fFCP6/gmCg496UlPastVGqd/0kknbXxtWj5CWyD+dWAZV497HhIscWmiq3aQP9Mu+l3uJzqtFR79yb0T+ja/S5r7FOzbBLj4Df1vbyfW9M+khXwMx2j9FsJzRBH60uRx7J+AeLIhjnOIhLjOJUjzz6HBVM5x/oGUyXWkO58018l+4tRZ8XoiEGcaPok51Ah7POm5E27gaPigjIdGfFIWb8KPPg6XpVWYHwLsHs7Yxz2+Km9vnBN6zvT780Os7pj9+RNcwQ9+B2f4J2KKqHFM2PA/6b2Vxm/xX+b2EFXKSyOWxI6zhY+uFdFU6O8cBGAL5/y/R1PAOfvyOIZ/NueShz2VF9ia3Z1PcC5+RHn58cFbZjxJL3fyKd/bWV3yZFOXffX0+ewrq27n5MM5naGuIx231Oee3Ve23ItjIeliv1k5eZRLLJ90v0XauFxLWOM/kxbyvXNh5J4Q9pFkTJQYeWxTxEX8/BOMz+fY+b4O15HWX7snXvZTvuL1RIDN2Zk9Bcc9v8Z3FT7KlzJ9npQdn3Pcb32Z2p8HAr39x/yYBwJ1lzuKAJ7Ep+xo2eQnuIghQV2O+7ZNuuukzevbwdRR8WVHIP/30S7RFOLsJ8+i9se5Xp+wYURvfpVysXHSlsXqkte1L0sIf3akbPiV++3L5lzPuz6f30vo9+f78uu8P1khv85Gqd9eCBQChUAhUAgUAoVAIVAIbA+BEvLbQ6jOFwKFQCFQCBQChUAhUAgUArsgAiXkd0Gj1E8qBAqBQqAQKAQKgUKgECgEtodACfntIVTnC4FCoBAoBAqBQqAQKAQKgV0QgRLyu6BR6icVAoVAIVAIFAKFQCFQCBQC20OghPz2EKrzhUAhUAgUAoVAIVAIFAKFwC6IQAn5XdAo9ZMKgUKgECgECoFCoBAoBAqB7SFQQn57CNX5QqAQKAQKgUKgECgECoFCYBdEoIT8LmiU+kmFQCFQCBQChUAhUAgUAoXA9hAoIb89hOp8IVAIFAKFQCFQCBQChUAhsAsiUEJ+FzRK/aRCoBAoBAqBQqAQKAQKgUJgewiUkN8eQnW+ECgECoFCoBAoBAqBQqAQ2AURKCG/CxqlflIhUAgUAoVAIVAIFAKFQCGwPQRKyG8PoTpfCBQChUAhUAgUAoVAIVAI7III/H/RAGcfaOWizwAAAABJRU5ErkJggg=="
+    }
+   },
+   "cell_type": "markdown",
+   "id": "492f2a1cd1022ea7",
+   "metadata": {},
+   "source": [
+    "前面我给它构造了5个token，输入的shape是[1, 5]，而输出的shape是[1, 5, 65024]\n",
+    "\n",
+    "最后一维的65024很好理解，就是vocab_size，代表词表中每个词被输出概率\n",
+    "\n",
+    "而第二维的5就有些费解了，按理说咱们做的是next token prediction的任务，可为什么一下预测了5个token呢？其实只有最后一个token的预测结果是有用的，前面那4个都是直接丢弃的，下面画个图简单演示下流程。\n",
+    "\n",
+    "![image.png](attachment:image.png)"
+   ]
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "接下来再看看这输出的token具体是哪个。",
+   "id": "ea36cd72245dd91b"
+  },
+  {
+   "cell_type": "code",
+   "id": "c7f6dacafb006405",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-12T09:04:53.125876Z",
+     "start_time": "2024-05-12T09:04:53.095311Z"
+    }
+   },
+   "source": [
+    "next_token = torch.argmax(out.logits[:, -1, :], dim=-1)\n",
+    "next_token"
+   ],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([30968], device='cuda:0')"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 11
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "所以这30968号token到底是哪个字呢？咱们加载下tokenizer解码下。",
+   "id": "20ee3612c13eb29c"
+  },
+  {
+   "cell_type": "code",
+   "id": "6e5620f049b39df1",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-12T09:04:53.312170Z",
+     "start_time": "2024-05-12T09:04:53.126870Z"
+    }
+   },
+   "source": [
+    "tokenlizer = ChatGLMTokenizer(vocab_file='tokenizer.model')"
+   ],
+   "outputs": [],
+   "execution_count": 12
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-12T09:04:53.327823Z",
+     "start_time": "2024-05-12T09:04:53.313292Z"
+    }
+   },
+   "cell_type": "code",
+   "source": "tokenlizer.batch_decode(next_token)",
+   "id": "4289642040c0c9fe",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['z']"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 13
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": [
+    "现在，咱们已经能简单的预测出下一个token了，下面咱们就这样自回归的预测下去试试吧。\n",
+    "\n",
+    "当然，这次用tokenlizer把文本转化为有意义的token序列"
+   ],
+   "id": "a40feea76c40503"
+  },
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-12T09:04:53.343423Z",
+     "start_time": "2024-05-12T09:04:53.328823Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "inputs = tokenlizer(['为什么A股指数跌的不多，但是我亏损比之前都多？'], add_special_tokens=False)\n",
+    "inputs"
+   ],
+   "id": "4745f4cb47248240",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'input_ids': [[30910, 32148, 30938, 55128, 33488, 56156, 33069, 54573, 31123, 45990, 37768, 54703, 32030, 54606, 54573, 31514]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'position_ids': [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]]}"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "execution_count": 14
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": [
+    "可以看到tokenization后有3个参数，分别是input_ids，attention_mask，和position_ids。\n",
+    "\n",
+    "后2个可以不管，因为这些参数在模型推理时可以自动生成。\n",
+    "\n",
+    "现在咱们一直预测下一个token，直到模型输出的token是eos（代表模型输出结束的token）为止。"
+   ],
+   "id": "cbb74d09a4709e3e"
+  },
+  {
+   "cell_type": "code",
+   "id": "1a18d4cba4e36a75",
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2024-05-12T09:06:05.170080Z",
+     "start_time": "2024-05-12T09:04:53.344342Z"
+    }
+   },
+   "source": [
+    "input_ids = inputs['input_ids']\n",
+    "input_ids = torch.tensor(input_ids, device='cuda')\n",
+    "\n",
+    "print(tokenlizer.decode(input_ids[0]))\n",
+    "\n",
+    "with torch.no_grad():\n",
+    "    for i in range(500):\n",
+    "        output = m(input_ids)\n",
+    "\n",
+    "        result = torch.argmax(output.logits[:, -1, :], dim=-1)\n",
+    "\n",
+    "        if result.item() == tokenlizer.eos_token_id:\n",
+    "            break\n",
+    "\n",
+    "        print(tokenlizer.decode(result[0], skip_special_tokens=True), end='')\n",
+    "        \n",
+    "        input_ids = torch.cat([input_ids, result.reshape(-1, 1)],\n",
+    "                              dim=1)\n",
+    "    \n",
+    "    print('\\n\\n======================\\n\\n', tokenlizer.decode(input_ids[0], skip_special_tokens=True), end='\\n')"
+   ],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "为什么A股指数跌的不多，但是我亏损比之前都多？\n",
+      "<0x0A>为什么A股指数跌的不多，但是我亏损比之前都多？<0x0A>这个问题涉及到多个方面的因素,以下是一些可能的原因:<0x0A><0x0A>1.投资策略不同:A股指数是市场整体的表现,而个人投资者可能会选择不同的投资策略,例如买入某些行业或股票,而另一种策略是卖出某些行业或股票。如果您的投资策略与市场整体表现不同,那么您的投资组合可能会遭受负面影响,即使A股指数并未下跌很多。<0x0A><0x0A>2.风险控制不同:A股市场有风险,但是不同投资者可能会采取不同的风险控制措施。例如,一些投资者可能会选择使用止损单来减少损失,而其他人可能会选择持有股票直到市场复苏。如果您的风险控制策略与市场整体表现不同,那么您的投资组合可能会遭受负面影响。<0x0A><0x0A>3.市场情绪不同:即使A股指数并未下跌很多,但是市场情绪可能会导致投资者感到不安,从而导致股市下跌。例如,如果市场预期经济衰退,那么投资者可能会减少投资,导致股市下跌。另一方面,如果市场情绪乐观,投资者可能会增加投资,导致股市上涨。<0x0A><0x0A>4.行业表现不同:即使A股指数未下跌很多,某些行业或股票可能表现不佳,导致投资者亏损。例如,某些行业可能会受到宏观经济因素的影响,导致其表现不佳,而其他行业则可能受益于相同的因素而表现良好。<0x0A><0x0A>5.交易成本不同:在A股市场中,投资者需要支付交易成本,包括股票交易费和基金管理费等。如果交易成本较高,那么投资者的亏损可能会更大,即使A股指数未下跌很多。<0x0A><0x0A>总结起来,以上这些因素都可能会导致投资者在A股市场中的表现与市场整体表现不同。因此,对于个人投资者而言,了解市场和自己的投资策略,制定合理的投资计划和风险控制措施,是减少亏损的关键。\n",
+      "\n",
+      "======================\n",
+      "\n",
+      " 为什么A股指数跌的不多，但是我亏损比之前都多？\n",
+      "为什么A股指数跌的不多，但是我亏损比之前都多？\n",
+      " 这个问题涉及到多个方面的因素,以下是一些可能的原因:\n",
+      "\n",
+      "1. 投资策略不同:A股指数是市场整体的表现,而个人投资者可能会选择不同的投资策略,例如买入某些行业或股票,而另一种策略是卖出某些行业或股票。如果您的投资策略与市场整体表现不同,那么您的投资组合可能会遭受负面影响,即使A股指数并未下跌很多。\n",
+      "\n",
+      "2. 风险控制不同:A股市场有风险,但是不同投资者可能会采取不同的风险控制措施。例如,一些投资者可能会选择使用止损单来减少损失,而其他人可能会选择持有股票直到市场复苏。如果您的风险控制策略与市场整体表现不同,那么您的投资组合可能会遭受负面影响。\n",
+      "\n",
+      "3. 市场情绪不同:即使A股指数并未下跌很多,但是市场情绪可能会导致投资者感到不安,从而导致股市下跌。例如,如果市场预期经济衰退,那么投资者可能会减少投资,导致股市下跌。另一方面,如果市场情绪乐观,投资者可能会增加投资,导致股市上涨。\n",
+      "\n",
+      "4. 行业表现不同:即使A股指数未下跌很多,某些行业或股票可能表现不佳,导致投资者亏损。例如,某些行业可能会受到宏观经济因素的影响,导致其表现不佳,而其他行业则可能受益于相同的因素而表现良好。\n",
+      "\n",
+      "5. 交易成本不同:在A股市场中,投资者需要支付交易成本,包括股票交易费和基金管理费等。如果交易成本较高,那么投资者的亏损可能会更大,即使A股指数未下跌很多。\n",
+      "\n",
+      "总结起来,以上这些因素都可能会导致投资者在A股市场中的表现与市场整体表现不同。因此,对于个人投资者而言,了解市场和自己的投资策略,制定合理的投资计划和风险控制措施,是减少亏损的关键。\n"
+     ]
+    }
+   ],
+   "execution_count": 15
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": [
+    "总之就很显然，一个合格的LLM就被咱们搭出来了，但目前的实现还是有些问题的，比如：\n",
+    "\n",
+    "1. 每次输出都是固定的\n",
+    "2. 缺少Chat Format（多轮对话的标记语言）\n",
+    "\n",
+    "这2个问题咱们留到后面解决"
+   ],
+   "id": "e8828f348d7481cb"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}