llms-from-scratch-cn/Model_Architecture_Discussions/ChatGLM4/chatglm4.ipynb

In [1]:

# import accelerate

In [1]:

%env HF_ENDPOINT=https://hf-mirror.com
import os
os.environ['HF_HOME'] = '/data1/ckw'

env: HF_ENDPOINT=https://hf-mirror.com

In [2]:

os.environ['HF_ENDPOINT']='https://hf-mirror.com'

In [3]:

!pwd

/data1/ckw/01大语言模型/ChatGLM4

In [4]:

import torch
# from transformers import AutoModelForCausalLM, AutoTokenizer
from tokenization_chatglm import ChatGLM4Tokenizer
from modeling_chatglm import ChatGLMForConditionalGeneration

device = "cuda"

tokenizer = ChatGLM4Tokenizer.from_pretrained("THUDM/glm-4-9b-chat", trust_remote_code=True)

query = "你好"

inputs = tokenizer.apply_chat_template([{"role": "user", "content": query}],
                                       add_generation_prompt=True,
                                       tokenize=True,
                                       return_tensors="pt",
                                       return_dict=True
                                       )

inputs = inputs.to(device)

Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.

In [5]:

model = ChatGLMForConditionalGeneration.from_pretrained(
    "THUDM/glm-4-9b-chat",
    torch_dtype=torch.bfloat16,
    low_cpu_mem_usage=True,
    trust_remote_code=True,
    load_in_4bit=True,
    device_map='auto'
).eval()#.to(device)

gen_kwargs = {"max_length": 2500, "do_sample": True, "top_k": 1}
with torch.no_grad():
    outputs = model.generate(**inputs, **gen_kwargs)
    outputs = outputs[:, inputs['input_ids'].shape[1]:]
    print(tokenizer.decode(outputs[0], skip_special_tokens=True))

The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
The `load_in_4bit` and `load_in_8bit` arguments are deprecated and will be removed in the future versions. Please, pass a `BitsAndBytesConfig` object in `quantization_config` argument instead.

Loading checkpoint shards:   0%|          | 0/10 [00:00<?, ?it/s]

你好👋！有什么可以帮助你的吗？

In [5]:

from transformers.utils import is_accelerate_available

In [10]:

is_accelerate_available()

Out [10]:

False

In [8]:

??is_accelerate_available

Signature: is_accelerate_available(min_version: str = '0.21.0')
Docstring: <no docstring>
Source:   
def is_accelerate_available(min_version: str = ACCELERATE_MIN_VERSION):
    return _accelerate_available and version.parse(_accelerate_version) >= version.parse(min_version)
File:      /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages/transformers/utils/import_utils.py
Type:      function

In [9]:

%pip install --upgrade accelerate

Looking in indexes: https://pypi.tuna.tsinghua.edu.cn/simple
Requirement already satisfied: accelerate in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (0.20.3)
Collecting accelerate
  Downloading https://pypi.tuna.tsinghua.edu.cn/packages/f0/62/9ebaf1fdd3d3c737a8814f9ae409d4ac04bc93b26a46a7dab456bb7e16f8/accelerate-0.31.0-py3-none-any.whl (309 kB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 309.4/309.4 kB 2.2 MB/s eta 0:00:00[31m1.8 MB/s eta 0:00:01
[?25hRequirement already satisfied: numpy>=1.17 in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from accelerate) (1.24.3)
Requirement already satisfied: packaging>=20.0 in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from accelerate) (23.2)
Requirement already satisfied: psutil in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from accelerate) (5.9.5)
Requirement already satisfied: pyyaml in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from accelerate) (6.0.1)
Requirement already satisfied: torch>=1.10.0 in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from accelerate) (2.1.0.post301)
Requirement already satisfied: huggingface-hub in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from accelerate) (0.23.2)
Requirement already satisfied: safetensors>=0.3.1 in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from accelerate) (0.4.3)
Requirement already satisfied: filelock in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (3.12.4)
Requirement already satisfied: typing-extensions in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (4.9.0)
Requirement already satisfied: sympy in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (1.12)
Requirement already satisfied: networkx in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (3.2.1)
Requirement already satisfied: jinja2 in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (3.1.2)
Requirement already satisfied: fsspec in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from torch>=1.10.0->accelerate) (2023.12.2)
Requirement already satisfied: requests in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from huggingface-hub->accelerate) (2.31.0)
Requirement already satisfied: tqdm>=4.42.1 in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from huggingface-hub->accelerate) (4.65.0)
Requirement already satisfied: MarkupSafe>=2.0 in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)
Requirement already satisfied: charset-normalizer<4,>=2 in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from requests->huggingface-hub->accelerate) (3.3.0)
Requirement already satisfied: idna<4,>=2.5 in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from requests->huggingface-hub->accelerate) (3.4)
Requirement already satisfied: urllib3<3,>=1.21.1 in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from requests->huggingface-hub->accelerate) (1.26.18)
Requirement already satisfied: certifi>=2017.4.17 in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from requests->huggingface-hub->accelerate) (2023.7.22)
Requirement already satisfied: mpmath>=0.19 in /data1/ckw/micromamba/envs/kewei-ai/lib/python3.11/site-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)
Installing collected packages: accelerate
  Attempting uninstall: accelerate
    Found existing installation: accelerate 0.20.3
    Uninstalling accelerate-0.20.3:
      Successfully uninstalled accelerate-0.20.3
Successfully installed accelerate-0.31.0
Note: you may need to restart the kernel to use updated packages.

In [ ]: