From f48e08b34fd63b6b3b6d39cf7710f4e779b1f97d Mon Sep 17 00:00:00 2001
From: kewei <2512235663@qq.com>
Date: Mon, 27 May 2024 15:11:03 +0800
Subject: [PATCH] 0527
---
Model_Architecture_Discussions/llama3/LICENSE | 21 +
.../llama3/README.md | 1135 +
.../llama3/images/42.png | Bin 0 -> 791272 bytes
.../llama3/images/a10.png | Bin 0 -> 649183 bytes
.../llama3/images/afterattention.png | Bin 0 -> 296198 bytes
.../llama3/images/archi.png | Bin 0 -> 866105 bytes
.../llama3/images/attention.png | Bin 0 -> 207123 bytes
.../llama3/images/embeddings.png | Bin 0 -> 481788 bytes
.../llama3/images/finallayer.png | Bin 0 -> 818316 bytes
.../llama3/images/freq_cis.png | Bin 0 -> 833456 bytes
.../llama3/images/god.png | Bin 0 -> 1269810 bytes
.../llama3/images/heads.png | Bin 0 -> 818922 bytes
.../llama3/images/implllama3_30_0.png | Bin 0 -> 49767 bytes
.../llama3/images/implllama3_39_0.png | Bin 0 -> 27612 bytes
.../llama3/images/implllama3_41_0.png | Bin 0 -> 26436 bytes
.../llama3/images/implllama3_42_0.png | Bin 0 -> 28024 bytes
.../llama3/images/implllama3_50_0.png | Bin 0 -> 27587 bytes
.../llama3/images/implllama3_52_0.png | Bin 0 -> 26433 bytes
.../llama3/images/implllama3_54_0.png | Bin 0 -> 28027 bytes
.../llama3/images/karpathyminbpe.png | Bin 0 -> 806344 bytes
.../llama3/images/keys.png | Bin 0 -> 440484 bytes
.../llama3/images/keys0.png | Bin 0 -> 432739 bytes
.../llama3/images/last_norm.png | Bin 0 -> 1027925 bytes
.../llama3/images/mask.png | Bin 0 -> 482770 bytes
.../llama3/images/model.png | Bin 0 -> 674648 bytes
.../llama3/images/norm.png | Bin 0 -> 316073 bytes
.../llama3/images/norm_after.png | Bin 0 -> 304528 bytes
.../llama3/images/q_per_token.png | Bin 0 -> 495555 bytes
.../llama3/images/qkmatmul.png | Bin 0 -> 193872 bytes
.../llama3/images/qkv.png | Bin 0 -> 509106 bytes
.../llama3/images/qsplit.png | Bin 0 -> 564238 bytes
.../llama3/images/rms.png | Bin 0 -> 348921 bytes
.../llama3/images/rope.png | Bin 0 -> 528606 bytes
.../llama3/images/ropesplit.png | Bin 0 -> 411043 bytes
.../llama3/images/softmax.png | Bin 0 -> 195571 bytes
.../llama3/images/stacked.png | Bin 0 -> 392799 bytes
.../llama3/images/swiglu.png | Bin 0 -> 619341 bytes
.../llama3/images/tokens.png | Bin 0 -> 500210 bytes
.../llama3/images/v0.png | Bin 0 -> 192702 bytes
.../llama3/images/value.png | Bin 0 -> 204710 bytes
.../llama3/images/weightmatrix.png | Bin 0 -> 388974 bytes
.../llama3/llama3-from-scratch.ipynb | 2221 +
.../llama3/params.json | 11 +
.../llama3/params.txt | 9 +
.../llama3/requirements.txt | 5 +
.../llama3/tokenizer.model | 128000 +++++++++++++++
46 files changed, 131402 insertions(+)
create mode 100644 Model_Architecture_Discussions/llama3/LICENSE
create mode 100644 Model_Architecture_Discussions/llama3/README.md
create mode 100644 Model_Architecture_Discussions/llama3/images/42.png
create mode 100644 Model_Architecture_Discussions/llama3/images/a10.png
create mode 100644 Model_Architecture_Discussions/llama3/images/afterattention.png
create mode 100644 Model_Architecture_Discussions/llama3/images/archi.png
create mode 100644 Model_Architecture_Discussions/llama3/images/attention.png
create mode 100644 Model_Architecture_Discussions/llama3/images/embeddings.png
create mode 100644 Model_Architecture_Discussions/llama3/images/finallayer.png
create mode 100644 Model_Architecture_Discussions/llama3/images/freq_cis.png
create mode 100644 Model_Architecture_Discussions/llama3/images/god.png
create mode 100644 Model_Architecture_Discussions/llama3/images/heads.png
create mode 100644 Model_Architecture_Discussions/llama3/images/implllama3_30_0.png
create mode 100644 Model_Architecture_Discussions/llama3/images/implllama3_39_0.png
create mode 100644 Model_Architecture_Discussions/llama3/images/implllama3_41_0.png
create mode 100644 Model_Architecture_Discussions/llama3/images/implllama3_42_0.png
create mode 100644 Model_Architecture_Discussions/llama3/images/implllama3_50_0.png
create mode 100644 Model_Architecture_Discussions/llama3/images/implllama3_52_0.png
create mode 100644 Model_Architecture_Discussions/llama3/images/implllama3_54_0.png
create mode 100644 Model_Architecture_Discussions/llama3/images/karpathyminbpe.png
create mode 100644 Model_Architecture_Discussions/llama3/images/keys.png
create mode 100644 Model_Architecture_Discussions/llama3/images/keys0.png
create mode 100644 Model_Architecture_Discussions/llama3/images/last_norm.png
create mode 100644 Model_Architecture_Discussions/llama3/images/mask.png
create mode 100644 Model_Architecture_Discussions/llama3/images/model.png
create mode 100644 Model_Architecture_Discussions/llama3/images/norm.png
create mode 100644 Model_Architecture_Discussions/llama3/images/norm_after.png
create mode 100644 Model_Architecture_Discussions/llama3/images/q_per_token.png
create mode 100644 Model_Architecture_Discussions/llama3/images/qkmatmul.png
create mode 100644 Model_Architecture_Discussions/llama3/images/qkv.png
create mode 100644 Model_Architecture_Discussions/llama3/images/qsplit.png
create mode 100644 Model_Architecture_Discussions/llama3/images/rms.png
create mode 100644 Model_Architecture_Discussions/llama3/images/rope.png
create mode 100644 Model_Architecture_Discussions/llama3/images/ropesplit.png
create mode 100644 Model_Architecture_Discussions/llama3/images/softmax.png
create mode 100644 Model_Architecture_Discussions/llama3/images/stacked.png
create mode 100644 Model_Architecture_Discussions/llama3/images/swiglu.png
create mode 100644 Model_Architecture_Discussions/llama3/images/tokens.png
create mode 100644 Model_Architecture_Discussions/llama3/images/v0.png
create mode 100644 Model_Architecture_Discussions/llama3/images/value.png
create mode 100644 Model_Architecture_Discussions/llama3/images/weightmatrix.png
create mode 100644 Model_Architecture_Discussions/llama3/llama3-from-scratch.ipynb
create mode 100644 Model_Architecture_Discussions/llama3/params.json
create mode 100644 Model_Architecture_Discussions/llama3/params.txt
create mode 100644 Model_Architecture_Discussions/llama3/requirements.txt
create mode 100644 Model_Architecture_Discussions/llama3/tokenizer.model
diff --git a/Model_Architecture_Discussions/llama3/LICENSE b/Model_Architecture_Discussions/llama3/LICENSE
new file mode 100644
index 0000000..b0b882f
--- /dev/null
+++ b/Model_Architecture_Discussions/llama3/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Nishant Aklecha
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/Model_Architecture_Discussions/llama3/README.md b/Model_Architecture_Discussions/llama3/README.md
new file mode 100644
index 0000000..df048f3
--- /dev/null
+++ b/Model_Architecture_Discussions/llama3/README.md
@@ -0,0 +1,1135 @@
+# llama3 implemented from scratch
+in this file, i implemented llama3 from scratch, one tensor and matrix multiplication at a time.
+
+also, im going to load tensors directly from the model file that meta provided for llama3, you need to download the weights before running this file.
+here is the offical link to download the weights: https://llama.meta.com/llama-downloads/
+
+
+
+
+link to his implementation: https://github.com/karpathy/minbpe
+
+
+
+
+but since we are implementing llama3 from scratch we will read the file one tensor at a time.
+
+
+
+
+
+anyway, so our [17x1] tokens are now [17x4096], i.e. 17 embeddings (one for each token) of length 4096
+
+
+note: keep track of the shapes, it makes it much easier to understand everything
+
+
+
+
+things to keep in mind, we need a norm_eps (from config) because we dont want to accidently set rms to 0 and divide by 0
+
+here is the formula:
+
+
+
+anyway, so after normalizing our shapes are still [17x4096] same as embedding but normalized
+
+
+
+
+
+
+
+> when we load the query, key, value and output vectors from the model we notice the shapes to be [4096x4096], [1024x4096], [1024x4096], [4096x4096]
+
+> at first glance this is weird because ideally we want each q,k,v and o for each head individually
+
+> the authors of the code bundled them togeather because its easy it helps parallize attention head multiplication.
+
+> im going to unwrap everything...
+
+
+```python
+print(
+ model["layers.0.attention.wq.weight"].shape,
+ model["layers.0.attention.wk.weight"].shape,
+ model["layers.0.attention.wv.weight"].shape,
+ model["layers.0.attention.wo.weight"].shape
+)
+```
+
+ torch.Size([4096, 4096]) torch.Size([1024, 4096]) torch.Size([1024, 4096]) torch.Size([4096, 4096])
+
+
+### unwrapping query
+in the next section we will unwrap the queries from multiple attention heads, the resulting shape is [32x128x4096]
+
+here, 32 is the number of attention heads in llama3, 128 is the size of the query vector and 4096 is the size of the token embedding
+
+
+```python
+q_layer0 = model["layers.0.attention.wq.weight"]
+head_dim = q_layer0.shape[0] // n_heads
+q_layer0 = q_layer0.view(n_heads, head_dim, dim)
+q_layer0.shape
+```
+
+
+
+
+ torch.Size([32, 128, 4096])
+
+
+
+### im going to implement the first head of the first layer
+here i access the query weight matrix first head of the first layer, the size of this query weight matrix is [128x4096]
+
+
+```python
+q_layer0_head0 = q_layer0[0]
+q_layer0_head0.shape
+```
+
+
+
+
+ torch.Size([128, 4096])
+
+
+
+### we now multiply the query weights with the token embedding, to recive a query for the token
+here you can see the resulting shape is [17x128], this is because we have 17 tokens and for each token there is a 128 length query.
+
+
+
+query: "the answer to the ultimate question of life, the universe, and everything is "
+
+in our prompt we have used "the" three times, we need the query vectors of all 3 "the" tokens to have different query vectors (each of size [1x128]) based on their positions in the query. we perform these rotations using RoPE (rotory positional embedding).
+
+### RoPE
+watch this video (this is what i watched) to understand the math.
+https://www.youtube.com/watch?v=o29P0Kpobz0&t=530s
+
+
+
+
+
+we now have a vector of size [17x64x2], this is the 128 length queries split into 64 pairs for each token in the prompt! each of those 64 pairs will be rotated by m*(theta) where m is the position of the token for which we are rotating the query!
+
+
+
+
+
+
+
+honeslty this is beautiful to think about :)
+
+
+```python
+q_per_token_as_complex_numbers = torch.view_as_complex(q_per_token_split_into_pairs)
+q_per_token_as_complex_numbers.shape
+```
+
+
+
+
+ torch.Size([17, 64])
+
+
+
+
+```python
+q_per_token_as_complex_numbers_rotated = q_per_token_as_complex_numbers * freqs_cis
+q_per_token_as_complex_numbers_rotated.shape
+```
+
+
+
+
+ torch.Size([17, 64])
+
+
+
+### after rotated vector is obtained
+we can get back our the queries as pairs by viewing the complex numbers as real numbers again
+
+
+```python
+q_per_token_split_into_pairs_rotated = torch.view_as_real(q_per_token_as_complex_numbers_rotated)
+q_per_token_split_into_pairs_rotated.shape
+```
+
+
+
+
+ torch.Size([17, 64, 2])
+
+
+
+the rotated pairs are now merged, we now have a new query vector (rotated query vector) that is of the shape [17x128] where 17 is the number of tokens and the 128 is the dim of the query vector
+
+
+```python
+q_per_token_rotated = q_per_token_split_into_pairs_rotated.view(q_per_token.shape)
+q_per_token_rotated.shape
+```
+
+
+
+
+ torch.Size([17, 128])
+
+
+
+# keys (almost the same as queries)
+
+
+
+> keys generate key vectors also of dimention 128
+
+> keys have only 1/4th the number of the weights as queries, this is because the weights for keys are shared across 4 heads at a time, to reduce the number of computations need
+
+> keys are also rotated to add positional info, just like queries because of the same reasons
+
+
+```python
+k_layer0 = model["layers.0.attention.wk.weight"]
+k_layer0 = k_layer0.view(n_kv_heads, k_layer0.shape[0] // n_kv_heads, dim)
+k_layer0.shape
+```
+
+
+
+
+ torch.Size([8, 128, 4096])
+
+
+
+
+```python
+k_layer0_head0 = k_layer0[0]
+k_layer0_head0.shape
+```
+
+
+
+
+ torch.Size([128, 4096])
+
+
+
+
+```python
+k_per_token = torch.matmul(token_embeddings, k_layer0_head0.T)
+k_per_token.shape
+```
+
+
+
+
+ torch.Size([17, 128])
+
+
+
+
+```python
+k_per_token_split_into_pairs = k_per_token.float().view(k_per_token.shape[0], -1, 2)
+k_per_token_split_into_pairs.shape
+```
+
+
+
+
+ torch.Size([17, 64, 2])
+
+
+
+
+```python
+k_per_token_as_complex_numbers = torch.view_as_complex(k_per_token_split_into_pairs)
+k_per_token_as_complex_numbers.shape
+```
+
+
+
+
+ torch.Size([17, 64])
+
+
+
+
+```python
+k_per_token_split_into_pairs_rotated = torch.view_as_real(k_per_token_as_complex_numbers * freqs_cis)
+k_per_token_split_into_pairs_rotated.shape
+```
+
+
+
+
+ torch.Size([17, 64, 2])
+
+
+
+
+```python
+k_per_token_rotated = k_per_token_split_into_pairs_rotated.view(k_per_token.shape)
+k_per_token_rotated.shape
+```
+
+
+
+
+ torch.Size([17, 128])
+
+
+
+## at this stage now have both the rotated values of queries and keys, for each token.
+
+
+
+this score describes how well each token's query relates to the each tokens's key.
+THIS IS SELF ATTENTION :)
+
+the shape of the attention score matrix (qk_per_token) is [17x17] where 17 is the number of tokens in the prompt
+
+
+
+
+why? because during training we only learn to predict tokens using past tokens.
+
+as a result, during inference we set the future tokens to zero.
+
+
+
+
+
+
+
+> just like keys, value weights are also shared acorss every 4 attention heads (to save computation)
+
+> as a result, the shape of the value weight matrix below is [8x128x4096]
+
+
+
+```python
+v_layer0 = model["layers.0.attention.wv.weight"]
+v_layer0 = v_layer0.view(n_kv_heads, v_layer0.shape[0] // n_kv_heads, dim)
+v_layer0.shape
+```
+
+
+
+
+ torch.Size([8, 128, 4096])
+
+
+
+the first layer, first head value weight matrix is given below
+
+
+```python
+v_layer0_head0 = v_layer0[0]
+v_layer0_head0.shape
+```
+
+
+
+
+ torch.Size([128, 4096])
+
+
+
+## value vectors
+
+
+
+
+
+
+
+now im going to run a loop and perform the exact same math as the cells above but for every head in the first layer
+
+
+```python
+qkv_attention_store = []
+
+for head in range(n_heads):
+ q_layer0_head = q_layer0[head]
+ k_layer0_head = k_layer0[head//4] # key weights are shared across 4 heads
+ v_layer0_head = v_layer0[head//4] # value weights are shared across 4 heads
+ q_per_token = torch.matmul(token_embeddings, q_layer0_head.T)
+ k_per_token = torch.matmul(token_embeddings, k_layer0_head.T)
+ v_per_token = torch.matmul(token_embeddings, v_layer0_head.T)
+
+ q_per_token_split_into_pairs = q_per_token.float().view(q_per_token.shape[0], -1, 2)
+ q_per_token_as_complex_numbers = torch.view_as_complex(q_per_token_split_into_pairs)
+ q_per_token_split_into_pairs_rotated = torch.view_as_real(q_per_token_as_complex_numbers * freqs_cis[:len(tokens)])
+ q_per_token_rotated = q_per_token_split_into_pairs_rotated.view(q_per_token.shape)
+
+ k_per_token_split_into_pairs = k_per_token.float().view(k_per_token.shape[0], -1, 2)
+ k_per_token_as_complex_numbers = torch.view_as_complex(k_per_token_split_into_pairs)
+ k_per_token_split_into_pairs_rotated = torch.view_as_real(k_per_token_as_complex_numbers * freqs_cis[:len(tokens)])
+ k_per_token_rotated = k_per_token_split_into_pairs_rotated.view(k_per_token.shape)
+
+ qk_per_token = torch.matmul(q_per_token_rotated, k_per_token_rotated.T)/(128)**0.5
+ mask = torch.full((len(tokens), len(tokens)), float("-inf"), device=tokens.device)
+ mask = torch.triu(mask, diagonal=1)
+ qk_per_token_after_masking = qk_per_token + mask
+ qk_per_token_after_masking_after_softmax = torch.nn.functional.softmax(qk_per_token_after_masking, dim=1).to(torch.bfloat16)
+ qkv_attention = torch.matmul(qk_per_token_after_masking_after_softmax, v_per_token)
+ qkv_attention = torch.matmul(qk_per_token_after_masking_after_softmax, v_per_token)
+ qkv_attention_store.append(qkv_attention)
+
+len(qkv_attention_store)
+```
+
+
+
+
+ 32
+
+
+
+
+
+
+we are almost at the end :)
+
+
+```python
+stacked_qkv_attention = torch.cat(qkv_attention_store, dim=-1)
+stacked_qkv_attention.shape
+```
+
+
+
+
+ torch.Size([17, 4096])
+
+
+
+# weight matrix, one of the final steps
+
+
+
+
+
+
+
+
+
+its pretty standard to use this feed forward network architecture in llms these days
+
+
+```python
+w1 = model["layers.0.feed_forward.w1.weight"]
+w2 = model["layers.0.feed_forward.w2.weight"]
+w3 = model["layers.0.feed_forward.w3.weight"]
+output_after_feedforward = torch.matmul(torch.functional.F.silu(torch.matmul(embedding_after_edit_normalized, w1.T)) * torch.matmul(embedding_after_edit_normalized, w3.T), w2.T)
+output_after_feedforward.shape
+```
+
+
+
+
+ torch.Size([17, 4096])
+
+
+
+# WE FINALLY HAVE NEW EDITED EMBEDDINGS FOR EACH TOKEN AFTER THE FIRST LAYER
+just 31 more layers to go before we are done (one for loop away)
+
+you can imagine this edited embedding as having information about all queries asked on the first layer
+
+now each layer will encode more and more complex queries on the quesions asked, until we have an embedding that knows everything about the next token that we need.
+
+
+```python
+layer_0_embedding = embedding_after_edit+output_after_feedforward
+layer_0_embedding.shape
+```
+
+
+
+
+ torch.Size([17, 4096])
+
+
+
+# god, everything all at once
+
+
+
+
+# have fun reading :)
+
+
+```python
+final_embedding = token_embeddings_unnormalized
+for layer in range(n_layers):
+ qkv_attention_store = []
+ layer_embedding_norm = rms_norm(final_embedding, model[f"layers.{layer}.attention_norm.weight"])
+ q_layer = model[f"layers.{layer}.attention.wq.weight"]
+ q_layer = q_layer.view(n_heads, q_layer.shape[0] // n_heads, dim)
+ k_layer = model[f"layers.{layer}.attention.wk.weight"]
+ k_layer = k_layer.view(n_kv_heads, k_layer.shape[0] // n_kv_heads, dim)
+ v_layer = model[f"layers.{layer}.attention.wv.weight"]
+ v_layer = v_layer.view(n_kv_heads, v_layer.shape[0] // n_kv_heads, dim)
+ w_layer = model[f"layers.{layer}.attention.wo.weight"]
+ for head in range(n_heads):
+ q_layer_head = q_layer[head]
+ k_layer_head = k_layer[head//4]
+ v_layer_head = v_layer[head//4]
+ q_per_token = torch.matmul(layer_embedding_norm, q_layer_head.T)
+ k_per_token = torch.matmul(layer_embedding_norm, k_layer_head.T)
+ v_per_token = torch.matmul(layer_embedding_norm, v_layer_head.T)
+ q_per_token_split_into_pairs = q_per_token.float().view(q_per_token.shape[0], -1, 2)
+ q_per_token_as_complex_numbers = torch.view_as_complex(q_per_token_split_into_pairs)
+ q_per_token_split_into_pairs_rotated = torch.view_as_real(q_per_token_as_complex_numbers * freqs_cis)
+ q_per_token_rotated = q_per_token_split_into_pairs_rotated.view(q_per_token.shape)
+ k_per_token_split_into_pairs = k_per_token.float().view(k_per_token.shape[0], -1, 2)
+ k_per_token_as_complex_numbers = torch.view_as_complex(k_per_token_split_into_pairs)
+ k_per_token_split_into_pairs_rotated = torch.view_as_real(k_per_token_as_complex_numbers * freqs_cis)
+ k_per_token_rotated = k_per_token_split_into_pairs_rotated.view(k_per_token.shape)
+ qk_per_token = torch.matmul(q_per_token_rotated, k_per_token_rotated.T)/(128)**0.5
+ mask = torch.full((len(token_embeddings_unnormalized), len(token_embeddings_unnormalized)), float("-inf"))
+ mask = torch.triu(mask, diagonal=1)
+ qk_per_token_after_masking = qk_per_token + mask
+ qk_per_token_after_masking_after_softmax = torch.nn.functional.softmax(qk_per_token_after_masking, dim=1).to(torch.bfloat16)
+ qkv_attention = torch.matmul(qk_per_token_after_masking_after_softmax, v_per_token)
+ qkv_attention_store.append(qkv_attention)
+
+ stacked_qkv_attention = torch.cat(qkv_attention_store, dim=-1)
+ w_layer = model[f"layers.{layer}.attention.wo.weight"]
+ embedding_delta = torch.matmul(stacked_qkv_attention, w_layer.T)
+ embedding_after_edit = final_embedding + embedding_delta
+ embedding_after_edit_normalized = rms_norm(embedding_after_edit, model[f"layers.{layer}.ffn_norm.weight"])
+ w1 = model[f"layers.{layer}.feed_forward.w1.weight"]
+ w2 = model[f"layers.{layer}.feed_forward.w2.weight"]
+ w3 = model[f"layers.{layer}.feed_forward.w3.weight"]
+ output_after_feedforward = torch.matmul(torch.functional.F.silu(torch.matmul(embedding_after_edit_normalized, w1.T)) * torch.matmul(embedding_after_edit_normalized, w3.T), w2.T)
+ final_embedding = embedding_after_edit+output_after_feedforward
+```
+
+# we now have the final embedding, the best guess the model could make about the next token
+the shape of the embedding is the same as regular token embeddings [17x4096] where 17 is the number of tokens and 4096 is the embedding dim
+
+
+
+
+
+
+
+
+