diff --git a/.gitignore b/.gitignore index aee52b1..7956e1d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ -model/__pycache__ +__pycache__/ +*.pyc +.DS_Store out website/ docs-minimind/ \ No newline at end of file diff --git a/README.md b/README.md index 8ecc4c5..914a947 100644 --- a/README.md +++ b/README.md @@ -31,37 +31,36 @@ -* 此开源项目旨在完全从0开始，仅用3块钱成本 + 2小时！即可训练出仅为25.8M的超小语言模型**MiniMind**。 -* **MiniMind**系列极其轻量，最小版本体积是 GPT-3 的 $\frac{1}{7000}$，力求做到最普通的个人GPU也可快速训练。 -* 项目同时开源了大模型的极简结构-包含拓展共享混合专家(MoE)、数据集清洗、预训练(Pretrain)、监督微调(SFT)、LoRA微调、直接偏好优化(DPO)、强化学习训练(RLAIF: PPO/GRPO等)、模型蒸馏等全过程代码。 -* **MiniMind**同时拓展了视觉多模态的VLM: [MiniMind-V](https://github.com/jingyaogong/minimind-v)。 -* 项目所有核心算法代码均从0使用PyTorch原生重构！不依赖第三方库提供的抽象接口。 -* 这不仅是大语言模型的全阶段开源复现，也是一个入门LLM的教程。 -* 希望此项目能为所有人提供一个抛砖引玉的示例，一起感受创造的乐趣！推动更广泛AI社区的进步！ +* 此开源项目旨在完全从 0 开始，仅用 3 块钱成本与 2 小时训练时间，即可训练出规模约为 64M 的超小语言模型 MiniMind。 +* MiniMind 系列极其轻量，主线最小版本体积约为 GPT-3 的 $\frac{1}{2700}$，力求让普通个人 GPU 也能快速完成训练与复现。 +* 项目同时开源了大模型的极简结构与完整训练链路，覆盖 MoE、数据清洗、预训练（Pretrain）、监督微调（SFT）、LoRA、RLHF（DPO）、RLAIF（PPO / GRPO / CISPO）、Tool Use、Agentic RL、自适应思考与模型蒸馏等全过程代码。 +* MiniMind 同时拓展了视觉多模态版本 [MiniMind-V](https://github.com/jingyaogong/minimind-v)。 +* 项目所有核心算法代码均从 0 使用 PyTorch 原生实现，不依赖第三方库提供的高层抽象接口。 +* 这不仅是一个大语言模型全阶段开源复现项目，也是一套面向 LLM 入门与实践的教程。 +* 希望此项目能为更多人提供一个可复现、可理解、可扩展的起点，一起感受创造的乐趣，并推动更广泛 AI 社区的进步。 -> 为防止误解，“2小时” 基于NVIDIA 3090硬件设备（单卡）测试，“3块钱”指GPU服务器租用成本，具体规格详情见下文。 +> 注：本项目基于 Apache 2.0 协议开源，完全免费；“2小时” 基于 NVIDIA 3090 硬件设备（单卡）预估，“3块钱” 指 GPU 服务器租用成本，具体规格详情见下文。 --- -
-![minimind2](./images/minimind2.gif) +![minimind-3](./images/minimind-3.gif) -[🔗🍓推理模型](https://www.modelscope.cn/studios/gongjy/MiniMind-Reasoning) | [🔗🤖常规模型](https://www.modelscope.cn/studios/gongjy/MiniMind) | [🔗🎞️视频介绍](https://www.bilibili.com/video/BV12dHPeqE72/?share_source=copy_web&vd_source=670c2504f88726f8cf4a21ef6147c0e8) +[🔗 在线体验](https://www.modelscope.cn/studios/gongjy/MiniMind) | [🔗 视频介绍](https://www.bilibili.com/video/BV12dHPeqE72)
@@ -71,61 +70,67 @@ -# 📌 Introduction +--- -大语言模型（Large Language Model, LLM）的出现引发了全世界对AI的空前关注。 -无论是ChatGPT、DeepSeek还是Qwen，都以其惊艳的效果令人叹为观止。 -然而，动辄数百亿参数的庞大规模，使得它们对个人设备而言不仅难以训练，甚至连部署都显得遥不可及。 -打开大模型的“黑盒子”，探索其内部运作机制，多么令人心潮澎湃！ -遗憾的是，99%的探索只能止步于使用LoRA等技术对现有大模型进行少量微调，学习一些新指令或任务。 -这就好比教牛顿如何使用21世纪的智能手机——虽然有趣，却完全偏离了理解物理本质的初衷。 -与此同时，第三方的大模型框架和工具库，如transformers+trl，几乎只暴露了高度抽象的接口。 -通过短短10行代码，就能完成“加载模型+加载数据集+推理+强化学习”的全流程训练。 -这种高效的封装固然便利，但也像一架高速飞船，将开发者与底层实现隔离开来，阻碍了深入探究LLM核心代码的机会。 -然而，“用乐高拼出一架飞机，远比坐在头等舱里飞行更让人兴奋！”。 -更糟糕的是，互联网上充斥着大量付费课程和营销号，以漏洞百出、一知半解的内容推销AI教程。 -正因如此，本项目初衷是拉低LLM的学习门槛，让每个人都能从理解每一行代码开始， -从零开始亲手训练一个极小的语言模型。是的，从**零开始训练**，而不是仅仅进行**推理**！ -最低只需3块钱不到的服务器成本，就能亲身体验从0到1构建一个语言模型的全过程。 -一起感受创造的乐趣吧！ +# 📌 项目介绍 -> [!NOTE] -> （截至2025-10）MiniMind系列已完成多个型号模型的预训练，最小仅需25.8M（0.02B），即可具备流畅对话能力！ +大语言模型（Large Language Model, LLM）的出现，引发了全球范围内对 AI 的空前关注。无论是 ChatGPT、DeepSeek 还是 Qwen，都以惊艳的效果让人真切感受到这场技术浪潮的冲击力。然而，动辄数百亿参数的模型规模，使得它们对个人设备而言不仅难以训练，甚至连部署都显得遥不可及。打开大模型的“黑盒子”，真正去理解其内部运作机制，本应是一件令人心潮澎湃的事。遗憾的是，绝大多数探索最终都止步于使用 LoRA 等技术对现有大模型做少量微调，学习一些新指令或特定任务。这更像是在教牛顿如何使用 21 世纪的智能手机——虽然有趣，却偏离了理解物理本质的初衷。 -
-Models List +与此同时，第三方的大模型框架与工具库，如 `transformers` / `trl` / `peft` 等，往往只暴露出高度抽象的接口。只需短短十几行代码，就可以完成“加载模型 + 加载数据集 + 推理 + 强化学习”的全流程训练。这种高效封装固然便利，却也在一定程度上把开发者与底层实现隔离开来，削弱了深入理解 LLM 核心代码的机会。我认为 “用乐高自己拼出一架飞机，远比坐在头等舱里飞行更让人兴奋”，然而更现实的问题是，互联网上充斥着大量付费课程和营销内容，用漏洞百出、一知半解的讲解包装所谓的 AI 教程。正因如此，本项目的初衷就是尽可能降低 LLM 的学习门槛，让每个人都能从理解每一行代码开始，从 0 开始亲手训练一个极小的语言模型。是的，从**零开始训练**，而不是仅仅停留在**推理**层面。最低只需不到 3 块钱的服务器成本，就能亲身体验从 0 到 1 构建一个语言模型的全过程。 -| 模型 (大小) | 推理占用 (约) | Release | -|-------------------------|----------|------------| -| MiniMind2-small (26M) | 0.5 GB | 2025.04.26 | -| MiniMind2-MoE (145M) | 1.0 GB | 2025.04.26 | -| MiniMind2 (104M) | 1.0 GB | 2025.04.26 | -| minimind-v1-small (26M) | 0.5 GB | 2024.08.28 | -| minimind-v1-moe (4×26M) | 1.0 GB | 2024.09.17 | -| minimind-v1 (108M) | 1.0 GB | 2024.09.01 | +😊 一起感受创造的乐趣吧！ + +--- + +#### 🎉 本项目包含以下内容 + +- 提供完整的 MiniMind-LLM 结构代码（Dense + MoE），当前主线结构对齐 `Qwen3 / Qwen3-MoE` 生态。 +- 提供 Tokenizer 与分词器训练代码，支持 ``、``、`` 等模板标记。 +- 覆盖 Pretrain、SFT、LoRA、RLHF-DPO、RLAIF（PPO / GRPO / CISPO）、Tool Use、Agentic RL、自适应思考与模型蒸馏等完整训练流程。 +- 提供全阶段开源数据，覆盖收集、蒸馏、清洗与去重后的高质量数据集。 +- 关键训练算法与核心模块均从 0 实现，不依赖第三方框架封装。 +- 兼容 `transformers`、`trl`、`peft` 等主流框架，以及 `llama.cpp`、`vllm`、`ollama` 等常用推理引擎与 `Llama-Factory` 等训练框架。 +- 支持单机单卡与单机多卡（DDP、DeepSpeed）训练，支持 wandb / swanlab 可视化与动态启停训练。 +- 支持在 C-Eval、C-MMLU、OpenBookQA 等第三方测评集上进行评测，并支持通过 YaRN 实现 RoPE 长文本外推。 +- 提供兼容 OpenAI API 协议的极简服务端，便于接入 FastGPT、Open-WebUI 等第三方 Chat UI，并支持 `reasoning_content`、`tool_calls`、`open_thinking`。 +- 提供基于 Streamlit 的极简聊天 WebUI，支持思考展示、工具选择与多轮 Tool Call。 + +#### 🎉 已发布模型列表 + +| 模型 | 参数量 | Release | +|------|--------|---------| +| minimind-3 | 64M | 2026.04.01 | +| minimind-3-moe | 198M / A64M | 2026.04.01 | +| minimind2-small | 26M | 2025.04.26 | +| minimind2-moe | 145M | 2025.04.26 | +| minimind2 | 104M | 2025.04.26 | +| minimind-v1-small | 26M | 2024.08.28 | +| minimind-v1-moe | 4×26M | 2024.09.17 | +| minimind-v1 | 108M | 2024.09.01 | + + +--- + +#### 📝 更新日志 + +
+ 🔥 2026-04-01 + + - 发布 `minimind-3` / `minimind-3-moe`：结构、Tokenizer、训练链路、推理接口与默认配置全面更新 +- 结构主线对齐 `Qwen3 / Qwen3-MoE` 生态：Dense 约 `64M`，MoE 约 `198M / A64M`，并移除了 shared expert 设计 +- 默认训练数据切换为 `pretrain_t2t(_mini).jsonl`、`sft_t2t(_mini).jsonl`、`rlaif.jsonl`、`agent_rl.jsonl` 与 `agent_rl_math.jsonl` +- 移除独立 `train_reason.py`；思考能力统一由 `chat_template + ` 与 `open_thinking` 自适应开关控制 +- `toolcall` 能力已混入 `sft_t2t / sft_t2t_mini` 主线数据，默认 `full_sft` 即具备基础 Tool Call 能力；同时新增 `scripts/chat_api.py` 等推理示例 +- 新增原生 `Agentic RL` 训练脚本 `train_agent.py`，支持多轮 Tool-Use 场景下的 `GRPO / CISPO` +- RLAIF / Agentic RL 训练流程完成 `rollout engine` 解耦，支持更灵活地切换生成后端 +- `serve_openai_api.py` 与 `web_demo.py` 新增 `reasoning_content` / `tool_calls` / `open_thinking` 支持 +- Tokenizer 基于 `BPE + ByteLevel` 更新，并新增工具调用与思考标记，预留 buffer token 便于后续扩展 +- 新增 LoRA 权重合并导出流程，可通过 `scripts/convert_model.py` 将基础模型与 LoRA 权重合并为新的完整模型权重 +- 结构图资源更新，README 大面积更新
-**项目包含** - -- MiniMind-LLM结构的全部代码（Dense+MoE模型）。 -- 包含Tokenizer分词器详细训练代码。 -- 包含Pretrain、SFT、LoRA、RLHF-DPO、RLAIF(PPO/GRPO/SPO)、模型蒸馏的全过程训练代码。 -- 收集、蒸馏、整理并清洗去重所有阶段的高质量数据集，且全部开源。 -- 从0实现预训练、指令微调、LoRA、DPO/PPO/GRPO/SPO强化学习，白盒模型蒸馏。关键算法几乎不依赖第三方封装的框架，且全部开源。 -- 同时兼容`transformers`、`trl`、`peft`等第三方主流框架。 -- 训练支持单机单卡、单机多卡(DDP、DeepSpeed)训练，支持wandb/swanlab可视化训练流程。支持动态启停训练。 -- 在第三方测评榜（C-Eval、C-MMLU、OpenBookQA等）进行模型测试，支持YaRN算法执行RoPE长文本外推。 -- 实现Openai-Api协议的极简服务端，便于集成到第三方ChatUI使用（FastGPT、Open-WebUI等）。 -- 基于streamlit实现最简聊天WebUI前端。 -- 全面兼容社区热门`llama.cpp`、`vllm`、`ollama`推理引擎或`Llama-Factory`训练框架。 -- 复现(蒸馏/RL)大型推理模型DeepSeek-R1的MiniMind-Reason模型，**数据+模型**全部开源！ - -希望此开源项目可以帮助LLM初学者快速入门！ - -### 👉**更新日志** - -
+
2025-10-24 - 🔥 新增RLAIF训练算法：PPO、GRPO、SPO（从0原生实现） @@ -140,7 +145,7 @@
-
+
2025-04-26 - 重要更新 @@ -155,31 +160,31 @@ 为兼容第三方推理框架llama.cpp、vllm，本次更新需付出一些可观代价。 本次更新不再支持「直接」加载25-04-26以前的旧模型进行推理。 由于Llama位置编码方式与minimind存在区别，导致映射Llama模型后QK值存在差异 -MiniMind2系列旧模型均经过权重映射+（微调训练）QKVO线性层校准恢复而来。 +minimind2系列旧模型均经过权重映射+（微调训练）QKVO线性层校准恢复而来。 本次更新后将放弃对`minimind-v1`全系列的维护，并在仓库中下线。 ```
-
+
2025-02-09 -- 迎来发布以来重大更新，Release MiniMind2 Series。 +- 迎来发布以来重大更新，Release minimind2 Series。 - 代码几乎全部重构，使用更简洁明了的统一结构。 如有旧代码的兼容性需要，可访问[🔗旧仓库内容🔗](https://github.com/jingyaogong/minimind/tree/6e9cd28ef9b34a0a10afbdf6f59e65cb6e628efb)。 - 免去数据预处理步骤。统一数据集格式，更换为`jsonl`格式杜绝数据集下载混乱的问题。 -- MiniMind2系列效果相比MiniMind-V1显著提升。 +- minimind2系列效果相比MiniMind-V1显著提升。 - 小问题：{kv-cache写法更标准、MoE的负载均衡loss被考虑等等} - 提供模型迁移到私有数据集的训练方案（医疗模型、自我认知样例）。 - 精简预训练数据集，并大幅提升预训练数据质量，大幅缩短个人快速训练所需时间，单卡3090即可2小时复现！ - 更新：LoRA微调脱离peft包装，从0实现LoRA过程；DPO算法从0使用PyTorch原生实现；模型白盒蒸馏原生实现。 -- MiniMind2-DeepSeek-R1系列蒸馏模型诞生！ -- MiniMind2具备一定的英文能力！ -- 更新MiniMind2与第三方模型的基于更多大模型榜单测试性能的结果。 +- minimind2-DeepSeek-R1系列蒸馏模型诞生！ +- minimind2具备一定的英文能力！ +- 更新minimind2与第三方模型的基于更多大模型榜单测试性能的结果。
-
+
More... **2024-10-05** @@ -205,14 +210,16 @@ MiniMind2系列旧模型均经过权重映射+（微调训练）QKVO线性层校
+--- + # 📌 快速开始 -
-分享本人的软硬件配置（仅供参考） +
+本人的软硬件配置（供参考） * CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz * RAM: 128 GB -* GPU: NVIDIA GeForce RTX 3090(24GB) * 8 +* GPU: NVIDIA GeForce RTX 3090 (24GB) * 8 * Ubuntu==20.04 * CUDA==12.2 * Python==3.10.16 @@ -220,283 +227,219 @@ MiniMind2系列旧模型均经过权重映射+（微调训练）QKVO线性层校
-### 第0步 +## 第0步 ```bash -git clone https://github.com/jingyaogong/minimind.git +# 克隆仓库、安装依赖 +git clone --depth 1 https://github.com/jingyaogong/minimind +cd minimind && pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple ``` -## Ⅰ 测试已有模型效果 +## Ⅰ 🚀 模型推理 -### 1.环境准备 +### 1' 下载模型 + +在项目根目录： +```bash +# 方式1 +modelscope download --model gongjy/minimind-3 --local_dir ./minimind-3 +# 方式2 +git clone https://huggingface.co/jingyaogong/minimind-3 +``` + +### 2' CLI 推理 ```bash -pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple +# 方式1：使用 Transformers 格式模型 +python eval_llm.py --load_from ./minimind-3 +# 方式2：基于 PyTorch 模型（确保./out目录下有对应权重） +python eval_llm.py --load_from ./model --weight full_sft ``` -### 2.下载模型 - -到项目根目录 +### 3'（可选）WebUI ```bash -git clone https://huggingface.co/jingyaogong/MiniMind2 # or https://www.modelscope.cn/models/gongjy/MiniMind2 +# 可能需要`python>=3.10`，安装 `pip install streamlit` +# ⚠️ 须先将 transformers 格式模型文件夹复制到 ./scripts/ 目录下（例如：cp -r minimind-3 ./scripts/minimind-3），web_demo 脚本会自动扫描该目录下包含权重文件的子文件夹，如不存在则报错 +cd scripts && streamlit run web_demo.py ``` -### （可选）命令行问答 - -```bash -# 使用transformers格式模型 -python eval_llm.py --load_from ./MiniMind2 -``` - -### （可选）启动WebUI - -```bash -# 可能需要`python>=3.10` 安装 `pip install streamlit` -# cd scripts -streamlit run web_demo.py -``` - -### （可选）第三方推理框架 +### 4'（可选）第三方推理框架 ```bash # ollama -ollama run jingyaogong/minimind2 +ollama run jingyaogong/minimind-3 # vllm -vllm serve ./MiniMind2/ --served-model-name "minimind" +vllm serve /path/to/model --served-model-name "minimind" ``` -## Ⅱ 从0开始自己训练 +## Ⅱ 🛠️ 模型训练 -### 1.环境准备 +
+注：提前确认 Torch 的可用后端 -```bash -pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple -``` - -
-注：提前测试Torch是否可用cuda - -```bash +```python import torch print(torch.cuda.is_available()) ``` -如果不可用，请自行去[torch_stable](https://download.pytorch.org/whl/torch_stable.html) -下载whl文件安装。参考[链接](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=%E5%AE%89%E8%A3%85torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187) +若你计划使用 CUDA 训练，建议先确认当前环境是否已正确识别 GPU。 +若 `cuda` 不可用，也仍可根据自身设备选择 `CPU` 或 `MPS` 运行，但训练速度与兼容性会有非常大的差异。 +如需安装或更换 PyTorch 版本，可参考 [torch_stable](https://download.pytorch.org/whl/torch_stable.html) 与[链接](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=%E5%AE%89%E8%A3%85torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187)
-### 2.数据下载 +### 1' 下载数据 -从下文提供的[数据集下载链接](https://www.modelscope.cn/datasets/gongjy/minimind_dataset/files) -下载需要的数据文件（创建`./dataset`目录）并放到`./dataset`下 +从下文提供的[数据集下载链接](https://www.modelscope.cn/datasets/gongjy/minimind_dataset/files) 下载所需数据文件，并放入 `./dataset` 目录 -
-注：数据集须知 +> 当前默认仅需下载 `pretrain_t2t_mini.jsonl` 与 `sft_t2t_mini.jsonl`，即可较快复现 `MiniMind Zero` 对话模型。 +如有更多需求，下文提供多种搭配方案，可根据自身任务目标与 GPU 资源灵活选择。 -默认推荐下载`pretrain_hq.jsonl` + `sft_mini_512.jsonl`最快速度复现Zero聊天模型。 +### 2' 开始训练 -数据文件可自由选择，下文提供了多种搭配方案，可根据自己手头的训练需求和GPU资源进行适当组合。 - -
- -### 3.开始训练 - -目录位于`trainer` - -
+
💡 检查点暂停续训 -所有训练脚本均自动保存检查点，只需添加 `--from_resume 1` 参数即可自动检测加载&恢复训练： +所有训练脚本均支持检查点保存。添加 `--from_resume 1` 参数后，即可自动检测并恢复训练进度： ```bash python train_pretrain.py --from_resume 1 python train_full_sft.py --from_resume 1 -... +# ... ``` -**断点续训机制说明：** -- 训练过程自动在 `./checkpoints/` 目录保存完整检查点（模型、优化器、训练进度等） +**断点续训说明：** +- 训练过程会自动在 `./checkpoints/` 目录保存完整检查点（模型、优化器、训练进度等） - 检查点文件命名：`<权重名>_<维度>_resume.pth`（如：`full_sft_512_resume.pth`） -- 支持跨不同GPU数量恢复（自动调整step） -- 支持wandb训练记录连续性（自动恢复同一个run） +- 支持跨不同 GPU 数量恢复（自动调整 step） +- 支持 wandb 训练记录连续性（自动恢复同一个 run） > 适合长时间训练或不稳定环境，无需担心训练中断导致进度丢失
-**3.1 预训练（学知识）** +#### 2.1 预训练（必须） ```bash -python train_pretrain.py +cd trainer && python train_pretrain.py ``` -> 执行预训练，得到 `pretrain_*.pth` 作为预训练的输出权重（其中*为模型的dimension，默认为512） +> 训练后，将得到 `out/pretrain_*.pth` 作为输出权重（其中 `*` 为模型 dimension，默认为 `768`） - -**3.2 监督微调（学对话方式）** +#### 2.2 指令微调（必须） ```bash -python train_full_sft.py +cd trainer && python train_full_sft.py ``` -> 执行监督微调，得到 `full_sft_*.pth` 作为指令微调的输出权重（其中`full`即为全参数微调） +> 训练后，将得到 `out/full_sft_*.pth` 作为输出权重（其中 `full` 表示全参数微调） -
-注：训练须知 +#### 2.3 测试已训练模型（可选） -所有训练过程默认每隔100步保存1次参数到文件`./out/***.pth`（每次会覆盖掉旧权重文件）。 - -简单起见，此处只写明两个阶段训练过程。如需其它训练 (LoRA, 蒸馏, 强化学习, 微调推理等) 可参考下文【实验】小节的详细说明。 - -
- - ---- - -### 4.测试自己训练的模型效果 - -确保需要测试的模型`*.pth`文件位于`./out/`目录下。 -也可以直接去[此处](https://www.modelscope.cn/models/gongjy/MiniMind2-PyTorch/files)下载使用我训练的`*.pth`文件。 +确保待测试的模型 `*.pth` 文件位于 `./out/` 目录下；也可直接前往[此处](https://www.modelscope.cn/models/gongjy/minimind-3-pytorch/files)下载我已训练好的 `*.pth` 权重。 ```bash -python eval_llm.py --weight full_sft # 或 pretrain/dpo/ppo/grpo... +python eval_llm.py --weight full_sft ``` -
-注：测试须知 +> `--weight` 用于指定权重名称前缀，例如 `pretrain`、`full_sft` 等；更多参数可直接参考 `eval_llm.py` -`--weight` 参数指定权重名称前缀，可选：`pretrain`, `full_sft`, `dpo`, `reason`, `ppo_actor`, `grpo`, `spo` 等 +
+注：其它须知 -其他常用参数： -- `--load_from`: 模型加载路径（`model`=原生torch权重，其他路径=transformers格式） -- `--save_dir`: 模型权重目录（默认`out`） -- `--lora_weight`: LoRA权重名称（`None`表示不使用） -- `--historys`: 携带历史对话轮数（需为偶数，0表示不携带历史） -- `--max_new_tokens`: 最大生成长度（默认8192） -- `--temperature`: 生成温度（默认0.85） -- `--top_p`: nucleus采样阈值（默认0.85） +1、所有训练脚本均基于 PyTorch 原生实现，并支持多卡加速。 - -使用方式直接查看`eval_llm.py`代码即可。 - -
- - ---- - -> [!TIP] -> 所有训练脚本均为Pytorch原生框架，均支持多卡加速，假设你的设备有N (N＞1) 张显卡： - -单机N卡启动训练方式 (DDP, 支持多机多卡集群) +2、若你的设备有 `N (N > 1)` 张显卡，可通过以下方式启动单机 `N` 卡训练（DDP，也支持扩展到多机多卡）： ```bash torchrun --nproc_per_node N train_xxx.py ``` -
-注：其它须知 - - -单机N卡启动训练 (DeepSpeed) +3、可根据需要开启 wandb 记录训练过程。 ```bash -deepspeed --master_port 29500 --num_gpus=N train_xxx.py +... train_xxx.py --use_wandb ``` - - -可根据需要开启wandb记录训练过程（需可直连） - -```bash -# 需要登录: wandb login -torchrun --nproc_per_node N train_xxx.py --use_wandb -# and -python train_xxx.py --use_wandb -``` - -通过添加`--use_wandb`参数，可以记录训练过程，训练完成后，可以在wandb网站上查看训练过程。通过修改`wandb_project` -和`wandb_run_name`参数，可以指定项目名称和运行名称。 - -【注】：25年6月后，国内网络环境无法直连WandB，MiniMind项目默认转为使用[SwanLab](https://swanlab.cn/)作为训练可视化工具（完全兼容WandB API），即`import wandb`改为`import swanlab as wandb`即可，其他均无需改动。 +`2025` 年 `6` 月后，国内网络环境通常无法直连 WandB。MiniMind 当前默认转为使用 [SwanLab](https://swanlab.cn/) 作为训练可视化工具，其接口与 WandB 基本兼容；通常只需将 `import wandb` 替换为 `import swanlab as wandb`，其余调用方式基本无需改动。
+--- + # 📌 数据介绍 ## Ⅰ Tokenizer -分词器将单词从自然语言通过“词典”映射到`0, 1, 36`这样的数字，可以理解为数字就代表了单词在“词典”中的页码。 -可以选择自己构造词表训练一个“词典”，代码可见`./trainer/train_tokenizer.py`（仅供学习参考，若非必要无需再自行训练，MiniMind已自带tokenizer）。 -或者选择比较出名的开源大模型分词器， -正如同直接用新华/牛津词典的优点是token编码压缩率很好，缺点是页数太多，动辄数十万个词汇短语； -自己训练的分词器，优点是词表长度和内容随意控制，缺点是压缩率很低（例如"hello"也许会被拆分为"h e l l o" -五个独立的token），且生僻词难以覆盖。 -“词典”的选择固然很重要，LLM的输出本质上是SoftMax到词典N个词的多分类问题，然后通过“词典”解码到自然语言。 -因为MiniMind体积需要严格控制，为了避免模型头重脚轻（词嵌入embedding层参数在LLM占比太高），所以词表长度短短益善。 +分词器可以粗略理解成 LLM 使用的一本“词典”，负责把自然语言映射成 token id，再把 token id 解码回文本；项目中也提供了`train_tokenizer.py`作为词表训练示例。不建议重新训练 tokenizer，因为词表和切分规则一旦变化，模型权重、数据格式、推理接口与社区生态的兼容性都会下降，也会削弱模型的传播性。同时，tokenizer 还会影响 PPL 这类按 token 统计的指标，因此跨 tokenizer 比较时，BPB（Bits Per Byte）往往更有参考价值，可参考[这篇](https://skeptric.com/perplexity/)。 +对 MiniMind 这类小模型来说，词表大小还会直接影响 embedding 层和输出层的参数占比，因此保持词表精简通常是更合适的取舍。 -
+
Tokenizer介绍 -第三方强大的开源模型例如Yi、qwen、chatglm、mistral、Llama3的tokenizer词表长度如下： +第三方强大的开源模型例如 Yi、Qwen2、ChatGLM、Mistral、Llama 3 的 tokenizer 词表长度如下：
- - Hugging Face Logo + + Hugging Face Logo - ModelScope Logo + ModelScope Logo
- - - - - - + + + + + +
Tokenizer模型词表大小来源
yi tokenizer64,00001万物（中国）
qwen2 tokenizer151,643阿里云（中国）
glm tokenizer151,329智谱AI（中国）
mistral tokenizer32,000Mistral AI（法国）
llama3 tokenizer128,000Meta（美国）
minimind tokenizer6,400自定义
Yi64,00001万物（中国）
Qwen2151,643阿里云（中国）
ChatGLM151,329智谱AI（中国）
Mistral32,000Mistral AI（法国）
Llama 3128,000Meta（美国）
MiniMind6,400自定义
-> 👉2024-09-17更新：为了防止过去的版本歧义&控制体积，minimind所有模型均使用minimind_tokenizer分词，废弃所有mistral_tokenizer版本。 +> 当前主线为避免历史版本歧义并控制整体体积，统一使用 `minimind_tokenizer`，不再维护 `mistral_tokenizer` 版本。 -``` -# 一些自言自语 -> 尽管minimind_tokenizer长度很小，编解码效率弱于qwen2、glm等中文友好型分词器。 -> 但minimind模型选择了自己训练的minimind_tokenizer作为分词器，以保持整体参数轻量，避免编码层和计算层占比失衡，头重脚轻，因为minimind的词表大小只有6400。 -> 且minimind在实际测试中没有出现过生僻词汇解码失败的情况，效果良好。 -> 由于自定义词表压缩长度到6400，使得LLM总参数量最低只有25.8M。 -> 训练数据`pretrain_hq.jsonl`均来自于`匠数大模型数据集`，这部分数据相对次要，如需训练可以自由选择。 -``` +尽管 `minimind_tokenizer` 的词表只有 `6400`，编解码效率弱于 `qwen2`、`glm` 等更偏中文友好的 tokenizer，但它能显著压缩 embedding 层和输出层的参数占比，更适合 MiniMind 这类小模型的体积约束。 +从实际使用效果看，这套 tokenizer 并没有明显带来生僻词解码失败的问题，整体仍然足够稳定可用；因此当前主线训练也统一沿用这套词表，而不再额外分叉维护其他 tokenizer 版本。 ## Ⅱ Pretrain数据 -经历了MiniMind-V1的低质量预训练数据，导致模型胡言乱语的教训，`2025-02-05` 之后决定不再采用大规模无监督的数据集做预训练。 -进而尝试把[匠数大模型数据集](https://www.modelscope.cn/datasets/deepctrl/deepctrl-sft-data)的中文部分提取出来， -清洗出字符`<512`长度的大约1.6GB的语料直接拼接成预训练数据 `pretrain_hq.jsonl`，hq即为high -quality（当然也还不算high，提升数据质量无止尽）。 +`MiniMind-3` 当前主线预训练数据为 `pretrain_t2t.jsonl` / `pretrain_t2t_mini.jsonl`。 +这两份数据已经整理成统一的 `text -> next token prediction` 训练格式，目标是在较小算力下兼顾： -文件`pretrain_hq.jsonl` 数据格式为 +- 文本质量； +- 长度分布； +- 中英混合能力； +- 与后续 SFT / Tool Calling / RLAIF 阶段的模板衔接。 -```json -{"text": "如何才能摆脱拖延症？ 治愈拖延症并不容易，但以下建议可能有所帮助..."} +数据来源包括但不限于通用文本语料、对话整理语料、蒸馏补充语料，以及各类**宽松开源协议**可用的数据集；主线数据会在清洗、去重、长度控制与格式统一后再进入训练。数据来源于：[匠数大模型数据集](https://www.modelscope.cn/datasets/deepctrl/deepctrl-sft-data)、[Magpie-Align](https://www.modelscope.cn/organization/Magpie-Align) 等公开数据源。 + +其中： + +- `pretrain_t2t_mini.jsonl` 更适合快速复现； +- `pretrain_t2t.jsonl` 更适合完整训练 `MiniMind-3` 主线模型。 + +文件数据格式为 + +```jsonl +{"text": "如何才能摆脱拖延症？治愈拖延症并不容易，但以下建议可能有所帮助。"} +{"text": "清晨的阳光透过窗帘洒进房间，桌上的书页被风轻轻翻动。"} +{"text": "Transformer 通过自注意力机制建模上下文关系，是现代大语言模型的重要基础结构。"} ``` ## Ⅲ SFT数据 -[匠数大模型SFT数据集](https://www.modelscope.cn/datasets/deepctrl/deepctrl-sft-data) -“是一个完整、格式统一、安全的大模型训练和研究资源。 -从网络上的公开数据源收集并整理了大量开源数据集，对其进行了格式统一，数据清洗， -包含10M条数据的中文数据集和包含2M条数据的英文数据集。” -以上是官方介绍，下载文件后的数据总量大约在4B tokens，肯定是适合作为中文大语言模型的SFT数据的。 -但是官方提供的数据格式很乱，全部用来sft代价太大。 -我将把官方数据集进行了二次清洗，把含有符号污染和噪声的条目去除；另外依然只保留了总长度`<512` -的内容，此阶段希望通过大量对话补充预训练阶段欠缺的知识。 -导出文件为`sft_512.jsonl`(~7.5GB)。 +`MiniMind-3` 当前主线 SFT 数据为 `sft_t2t.jsonl` / `sft_t2t_mini.jsonl`。相比更早期的 `sft_512 / sft_1024 / sft_2048` 方案，当前版本更强调： -[Magpie-SFT数据集](https://www.modelscope.cn/organization/Magpie-Align) -收集了~1M条来自Qwen2/2.5的高质量对话，我将这部分数据进一步清洗，把总长度`<2048`的部分导出为`sft_2048.jsonl`(~9GB)。 -长度`<1024`的部分导出为`sft_1024.jsonl`(~5.5GB)，用大模型对话数据直接进行sft就属于“黑盒蒸馏”的范畴。 +- 统一模板； +- 更适合对话 + 思考标签 + Tool Calling 的混合训练； +- 尽量减少数据预处理分叉，降低复现成本。 -进一步清洗前两步sft的数据（只保留中文字符占比高的内容），筛选长度`<512`的对话，得到`sft_mini_512.jsonl`(~1.2GB)。 +其数据来源包括但不限于高质量指令跟随数据、公开对话数据、模型蒸馏合成数据，以及协议友好的开源数据集；在进入 `t2t` 主线前，会统一为当前仓库使用的多轮对话格式。当前主线中也包含大量合成数据，例如本人基于 `qwen3-4b` 合成的约 `10w` 条 `tool call` 数据，以及 `qwen3` 系列的 `reasoning` 数据等。其中社区主要来源有：[匠数大模型数据集](https://www.modelscope.cn/datasets/deepctrl/deepctrl-sft-data)、[Magpie-Align](https://www.modelscope.cn/organization/Magpie-Align)、[R1-Distill-SFT](https://www.modelscope.cn/datasets/AI-ModelScope/R1-Distill-SFT)、[COIG](https://huggingface.co/datasets/BAAI/COIG)、[Step-3.5-Flash-SFT](https://huggingface.co/datasets/stepfun-ai/Step-3.5-Flash-SFT) 等。公布版本会确保数据来源与处理链路符合对应开源协议的可传递性约束，并遵守 Apache-2.0、CC-BY-NC-2.0 等相关协议要求。 -所有sft文件 `sft_X.jsonl` 数据格式均为 +其中： -```text +- `sft_t2t_mini.jsonl`：适合快速训练对话模型； +- `sft_t2t.jsonl`：适合完整复现主线版本； +- `toolcall` 能力已经并入主线 SFT 数据。 + +所有 SFT 文件数据格式均为（包含对话数据、Tool Use数据） + +```jsonl { "conversations": [ {"role": "user", "content": "你好"}, @@ -505,18 +448,26 @@ quality（当然也还不算high，提升数据质量无止尽）。 {"role": "assistant", "content": "再见！"} ] } +{ + "conversations": [ + {"role": "system", "content": "# Tools ...", "tools": "[...]"}, + {"role": "user", "content": "把'你好世界'翻译成english"}, + {"role": "assistant", "content": "", "tool_calls": "[{\"name\":\"translate_text\",\"arguments\":{\"text\":\"你好世界\",\"target_language\":\"english\"}}]"}, + {"role": "tool", "content": "{\"translated_text\":\"Hello World\"}"}, + {"role": "assistant", "content": "Hello World"} + ] +} ``` -## Ⅳ RLHF数据 +## Ⅳ RL 数据 -来自[Magpie-DPO数据集](https://www.modelscope.cn/datasets/Magpie-Align/MagpieLM-DPO-Data-v0.1) -大约200k条偏好数据（均是英文）生成自Llama3.1-70B/8B，可以用于训练奖励模型，优化模型回复质量，使其更加符合人类偏好。 -这里将数据总长度`<3000`的内容重组为`dpo.jsonl`(~0.9GB)，包含`chosen`和`rejected`两个字段，`chosen` -为偏好的回复，`rejected`为拒绝的回复。 +`MiniMind` 当前主线 RL 数据为 `dpo.jsonl`。数据抽样自 [DPO-En-Zh-20k](https://huggingface.co/datasets/llamafactory/DPO-En-Zh-20k)。 -文件 `dpo.jsonl` 数据格式为 +主线中会将这部分样本统一重组为当前仓库使用的偏好学习格式，用于奖励模型或偏好优化阶段训练；其中 `chosen` 表示更符合偏好的回复，`rejected` 表示相对较差的回复。 -```text +其中 `dpo.jsonl` 数据格式为 + +```json { "chosen": [ {"content": "Q", "role": "user"}, @@ -529,28 +480,12 @@ quality（当然也还不算high，提升数据质量无止尽）。 } ``` -## Ⅴ Reason数据集： +除此之外，其他 RL 数据与 SFT 数据格式保持一致，通常是从 SFT 数据中按总长度和对话轮次筛选得到，并将最后一个 `assistant` 位置留空，供 rollout 阶段续写使用。 -不得不说2025年2月谁能火的过DeepSeek... -也激发了我对RL引导的推理模型的浓厚兴趣，目前已经用Qwen2.5复现了R1-Zero。 -如果有时间+效果work（但99%基模能力不足）我会在之后更新MiniMind基于RL训练的推理模型而不是蒸馏模型。 -时间有限，最快的低成本方案依然是直接蒸馏（黑盒方式）。 -耐不住R1太火，短短几天就已经存在一些R1的蒸馏数据集[R1-Llama-70B](https://www.modelscope.cn/datasets/Magpie-Align/Magpie-Reasoning-V2-250K-CoT-Deepseek-R1-Llama-70B)、[R1-Distill-SFT](https://www.modelscope.cn/datasets/AI-ModelScope/R1-Distill-SFT)、 -[Alpaca-Distill-R1](https://huggingface.co/datasets/shareAI/Alpaca-Distill-R1-ZH)、 -[deepseek_r1_zh](https://huggingface.co/datasets/jinliuxi/deepseek_r1_zh)等等，纯中文的数据可能比较少。 -最终整合它们，导出文件为`r1_mix_1024.jsonl`，数据格式和`sft_X.jsonl`一致。 - -## Ⅵ 更多数据集 - -目前已经有[HqWu-HITCS/Awesome-Chinese-LLM](https://github.com/HqWu-HITCS/Awesome-Chinese-LLM) -在收集和梳理中文LLM相关的开源模型、应用、数据集及教程等资料，并持续更新这方面的最新进展。全面且专业，Respect！ - ---- - -## Ⅷ MiniMind训练数据集 +## Ⅴ MiniMind 训练数据集 > [!NOTE] -> 2025-02-05后，开源MiniMind最终训练所用的所有数据集，因此无需再自行预处理大规模数据集，避免重复性的数据处理工作。 +> 当前主线训练所需的核心数据集已开源，因此无需再自行预处理大规模数据集，避免重复性的数据处理工作。 MiniMind训练数据集下载地址： [ModelScope](https://www.modelscope.cn/datasets/gongjy/minimind_dataset/files) | [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind_dataset/tree/main) @@ -560,34 +495,30 @@ MiniMind训练数据集下载地址： [ModelScope](https://www.modelscope.cn/da ```bash ./dataset/ -├── dpo.jsonl (55MB, ✨) -├── lora_identity.jsonl (22.8KB) -├── lora_medical.jsonl (34MB) -├── pretrain_hq.jsonl (1.6GB, ✨) -├── r1_mix_1024.jsonl (340MB) -├── rlaif-mini.jsonl (1MB, ✨) -├── sft_1024.jsonl (5.6GB) -├── sft_2048.jsonl (9GB) -├── sft_512.jsonl (7.5GB) -└── sft_mini_512.jsonl (1.2GB, ✨) +├── agent_rl.jsonl (86MB) +├── agent_rl_math.jsonl (18MB) +├── dpo.jsonl (53MB) +├── pretrain_t2t_mini.jsonl (1.2GB, ✨) +├── pretrain_t2t.jsonl (10GB) +├── rlaif.jsonl (24MB, ✨) +├── sft_t2t_mini.jsonl (1.6GB, ✨) +└── sft_t2t.jsonl (14GB) ``` -
+
注：各数据集简介 -* `dpo.jsonl`✨ --RLHF阶段数据集（已精简优化，适合快速训练） -* `lora_identity.jsonl` --自我认知数据集（例如：你是谁？我是minimind...），推荐用于lora训练（亦可用于全参SFT，勿被名字局限） -* `lora_medical.jsonl` --医疗问答数据集，推荐用于lora训练（亦可用于全参SFT，勿被名字局限） -* `pretrain_hq.jsonl`✨ --预训练数据集，整合自匠数科技（推荐设置`max_seq_len≈320`） -* `r1_mix_1024.jsonl` --DeepSeek-R1-1.5B蒸馏数据，每条数据字符最大长度为1024（推荐设置`max_seq_len≈720`） -* `rlaif-mini.jsonl` --RLAIF训练数据集，从SFT数据集中随机采样1万条高质量对话，用于PPO/GRPO/SPO等强化学习算法训练 -* `sft_1024.jsonl` --整合自Qwen2.5蒸馏数据（是sft_2048的子集），每条数据字符最大长度为1024（推荐设置`max_seq_len≈650`） -* `sft_2048.jsonl` --整合自Qwen2.5蒸馏数据，每条数据字符最大长度为2048（推荐设置`max_seq_len≈1400`） -* `sft_512.jsonl` --整合自匠数科技SFT数据，每条数据字符最大长度为512（推荐设置`max_seq_len≈350`） -* `sft_mini_512.jsonl`✨ --极简整合自匠数科技SFT数据+Qwen2.5蒸馏数据（用于快速训练Zero模型），每条数据字符最大长度为512（推荐设置`max_seq_len≈340`） +* `agent_rl.jsonl` --Agentic RL 主线训练数据，用于 `train_agent.py` 的多轮 Tool-Use / CISPO / GRPO 训练 +* `agent_rl_math.jsonl` --Agentic RL 纯数学补充数据，适合带最终校验目标的多轮推理/工具使用场景（用于RLVR） +* `dpo.jsonl` --RLHF阶段偏好训练数据（DPO） +* `pretrain_t2t_mini`✨ --`minimind-3` 轻量预训练数据，适合快速复现（推荐设置`max_seq_len≈768`） +* `pretrain_t2t` --`minimind-3` 主线预训练数据（推荐设置`max_seq_len≈380`） +* `rlaif.jsonl`✨ --RLAIF训练数据集，用于PPO/GRPO/CISPO等强化学习算法训练 +* `sft_t2t_mini.jsonl`✨ --`minimind-3` 轻量SFT数据（用于快速训练Zero模型），推荐设置`max_seq_len≈768`，其中已混入一部分 Tool Call 样本 +* `sft_t2t.jsonl` --`minimind-3` 主线SFT数据，适合完整复现，其中同样已混入 Tool Call 样本 -训练参数`max_seq_len`目前指的是tokens长度，而非绝对字符数。 +训练参数 `max_seq_len` 目前指的是 tokens 长度，而非绝对字符数。 本项目tokenizer在中文文本上大约`1.5~1.7 字符/token`，纯英文的压缩比在`4~5 字符/token`，不同数据分布会有波动。 数据集命名标注的“最大长度”均为字符数，100长度的字符串可粗略换算成`100/1.5≈67`的tokens长度。 @@ -597,141 +528,126 @@ MiniMind训练数据集下载地址： [ModelScope](https://www.modelscope.cn/da * 英文：`The sun sets in the west`24个字符可能被拆分为[`The `,`sun `,`sets `,`in `,`the`,`west`] 6个tokens “推荐设置”给出了各个数据集上最大tokens长度的粗略估计。 -须知max_seq_len可以激进/保守/均衡地调整，因为更大或更小均无法避免副作用：一些样本短于max_seq_len后被padding浪费算力，一些样本长于max_seq_len后被截断语意。 +须知 `max_seq_len` 可以激进 / 保守 / 均衡地调整，因为更大或更小均无法避免副作用：一些样本短于 `max_seq_len` 后被 padding 浪费算力，一些样本长于 `max_seq_len` 后被截断语义。 -在 `算力效率` <---> `语义完整性` 之间找到一个平衡点即可 +在算力效率与语义完整性之间找到平衡点即可
![dataset](./images/dataset.jpg) -
+> MiniMind 主线训练数据组成与推荐组合示意图 + +
说明 & 推荐训练方案 -* MiniMind2 Series均经过共约20GB语料训练，大约4B tokens，即对应上面的数据组合训练结果（开销：💰💰💰💰💰💰💰💰，效果：😊😊😊😊😊😊） +* `minimind-3` 主线推荐采用 `pretrain_t2t` + `sft_t2t` + `rlaif/agent_rl` 的阶段式训练组合。 -* 想要最快速度从0实现Zero模型，推荐使用`pretrain_hq.jsonl` + `sft_mini_512.jsonl` 的数据组合，具体花销和效果可查看下文表格（开销：💰，效果：😊😊） +* 想要最快速度从0实现Zero模型，推荐使用`pretrain_t2t_mini.jsonl` + `sft_t2t_mini.jsonl` 的数据组合 -* 推荐具备一定算力资源或更在意效果的朋友可以考虑前者完整复现MiniMind2；仅有单卡GPU或在乎短时间快速复现的朋友强烈推荐后者； +* 推荐具备一定算力资源或更在意效果的朋友完整复现 `minimind-3`；仅有单卡GPU或更在意快速复现的朋友强烈推荐 mini 组合。 -* 【折中方案】亦可选择例如`sft_mini_512.jsonl`、`sft_1024.jsonl`中等规模数据进行自由组合训练（开销：💰💰💰，效果：😊😊😊😊）。 +* 当前 `sft_t2t / sft_t2t_mini` 已经混入 Tool Call 数据，因此通常不需要再额外做一轮独立的 Tool Calling 监督微调。
-# 📌 Model +# 📌 模型 -## Structure +## 结构 -MiniMind-Dense（和[Llama3.1](https://ai.meta.com/blog/meta-llama-3-1/)一样）使用了Transformer的Decoder-Only结构，跟GPT-3的区别在于： +`minimind-3` Dense 使用 Transformer Decoder-Only 结构，整体配置已经向 `Qwen3` 生态对齐，方便后续转换到 `transformers / llama.cpp / ollama / vllm`： -* 采用了GPT-3的预标准化方法，也就是在每个Transformer子层的输入上进行归一化，而不是在输出上。具体来说，使用的是RMSNorm归一化函数。 -* 用SwiGLU激活函数替代了ReLU，这样做是为了提高性能。 -* 像GPT-Neo一样，去掉了绝对位置嵌入，改用了旋转位置嵌入（RoPE），这样在处理超出训练长度的推理时效果更好。 +* 采用预标准化（Pre-Norm）+ RMSNorm。 +* 使用 SwiGLU 激活函数。 +* 使用 RoPE 旋转位置编码，并支持 YaRN 外推。 +* `q_heads=8`、`kv_heads=4`，`max_position_embeddings=32768`，`rope_theta=1e6`。 ---- +`minimind-3-moe` 在相同结构上扩展 MoE 前馈层，实现上兼容 `Qwen3-MoE` 风格配置（去除 shared expert）。 -MiniMind-MoE模型，它的结构基于Llama3和[Deepseek-V2/3](https://arxiv.org/pdf/2405.04434)中的MixFFN混合专家模块。 +* 当前默认配置为 `4 experts / top-1 routing`，用于以更低激活参数获得更高容量。 +* Experts 继续增加后，实际耗时往往比同尺寸规模的 dense 模型高非常多，这和 “MoE 推理更快” 放在一起看会有点反直觉，但训练时 token 先按专家分桶、再分别做 forward，原生训练时带来的 `kernel` 启停和调度开销会急剧变重，这本身是很自然的事情。得靠支持 MoE kernel-fused 的算子库来优化，比如基于 `Triton` 的自定义 kernel、`DeepSpeed-MoE`、`Megatron-LM` 等等。当然，这个项目还是希望保留原生 PyTorch 的普适性，所以这里做的是现实的折中，在当前实现下，`4 experts / top-1` 这个甜点配置大约只比 dense 模型慢 `50%` 左右。 -* DeepSeek-V2在前馈网络（FFN）方面，采用了更细粒度的专家分割和共享的专家隔离技术，以提高Experts的效果。 +`minimind-3` 系列结构如下图： ---- +![structure](./images/LLM-structure.jpg) +![structure-moe](./images/LLM-structure-moe.jpg) -MiniMind的整体结构一致，只是在RoPE计算、推理函数和FFN层的代码上做了一些小调整。 -其结构如下图（重绘版）： +修改模型配置见[./model/model_minimind.py](./model/model_minimind.py)，参考模型参数版本见下表： -![structure](./images/LLM-structure.png) -![structure-moe](./images/LLM-structure-moe.png) - -修改模型配置见[./model/model_minimind.py](./model/model_minimind.py)。 -参考模型参数版本见下表： - -| Model Name | params | len_vocab | rope_theta | n_layers | d_model | kv_heads | q_heads | share+route | -|-------------------|--------|-----------|------------|----------|---------|----------|---------|-------------| -| MiniMind2-Small | 26M | 6400 | 1e6 | 8 | 512 | 2 | 8 | - | -| MiniMind2-MoE | 145M | 6400 | 1e6 | 8 | 640 | 2 | 8 | 1+4 | -| MiniMind2 | 104M | 6400 | 1e6 | 16 | 768 | 2 | 8 | - | -| minimind-v1-small | 26M | 6400 | 1e4 | 8 | 512 | 8 | 16 | - | -| minimind-v1-moe | 4×26M | 6400 | 1e4 | 8 | 512 | 8 | 16 | 1+4 | -| minimind-v1 | 108M | 6400 | 1e4 | 16 | 768 | 8 | 16 | - | +| Model Name | params | len_vocab | max_pos | rope_theta | n_layers | d_model | kv_heads | q_heads | note | +|------------|--------|-----------|---------|------------|----------|---------|----------|---------|------| +| minimind-3 | 64M | 6400 | 32768 | 1e6 | 8 | 768 | 4 | 8 | Dense | +| minimind-3-moe | 198M / A64M | 6400 | 32768 | 1e6 | 8 | 768 | 4 | 8 | 4 experts / top-1 | +| minimind2-small | 26M | 6400 | 32768 | 1e6 | 8 | 512 | 2 | 8 | 历史版本 | +| minimind2-moe | 145M | 6400 | 32768 | 1e6 | 8 | 640 | 2 | 8 | 历史版本 | +| minimind2 | 104M | 6400 | 32768 | 1e6 | 16 | 768 | 2 | 8 | 历史版本 | -## Model Configuration +## 模型配置 -📋关于LLM的参数配置，有一篇很有意思的论文[MobileLLM](https://arxiv.org/pdf/2402.14905)做了详细的研究和实验。 -Scaling Law在小模型中有自己独特的规律。 -引起Transformer参数成规模变化的参数几乎只取决于`d_model`和`n_layers`。 +关于 LLM 的参数配置，[MobileLLM](https://arxiv.org/pdf/2402.14905) 对小模型做过一组很有代表性的系统研究。对 MiniMind 这类百M级模型而言，`d_model` 与 `n_layers` 的取舍不只是参数分配问题，也会直接影响训练稳定性与最终效果。 + +当前 `minimind-3` 主线选择 `dim=768，n_layers=8`，本质上是一种工程取舍：更浅的网络训练更快，同时 `dim` 也不至于过小而导致模式崩溃，因此能在训练效率、稳定性与最终效果之间取得相对均衡。 + +
+查看详细说明 + +Scaling Law 在小模型上往往会呈现出一些不同于大模型的现象。决定 Transformer 参数规模变化的核心参数，通常主要就是 `d_model` 和 `n_layers`： * `d_model`↑ + `n_layers`↓ -> 矮胖子 * `d_model`↓ + `n_layers`↑ -> 瘦高个 -2020年提出Scaling Law的论文认为，训练数据量、参数量以及训练迭代次数才是决定性能的关键因素，而模型架构的影响几乎可以忽视。 -然而似乎这个定律对小模型并不完全适用。 -MobileLLM提出架构的深度比宽度更重要，「深而窄」的「瘦长」模型可以学习到比「宽而浅」模型更多的抽象概念。 -例如当模型参数固定在125M或者350M时，30～42层的「狭长」模型明显比12层左右的「矮胖」模型有更优越的性能， -在常识推理、问答、阅读理解等8个基准测试上都有类似的趋势。 -这其实是非常有趣的发现，因为以往为100M左右量级的小模型设计架构时，几乎没人尝试过叠加超过12层。 -这与MiniMind在训练过程中，模型参数量在`d_model`和`n_layers`之间进行调整实验观察到的效果是一致的。 -然而「深而窄」的「窄」也是有维度极限的，当d_model<512时，词嵌入维度坍塌的劣势非常明显， -增加的layers并不能弥补词嵌入在固定q_head带来d_head不足的劣势。 -当d_model>1536时，layers的增加似乎比d_model的优先级更高，更能带来具有"性价比"的参数->效果增益。 +经典 Scaling Law 更强调训练数据量、参数量和训练步数的决定性作用，通常会弱化架构差异本身的影响；但在小模型区间，这个结论并不总是完全成立。 +MobileLLM 的一个核心观察是：在参数量固定时，深度往往比宽度更重要。也就是说，相比“宽而浅”的结构，“深而窄”的模型更容易学到抽象概念。 +例如当模型参数量固定在 `125M` 或 `350M` 时，`30~42` 层的狭长结构通常会优于 `12` 层左右的矮胖结构，在常识推理、问答、阅读理解等多个基准上都呈现出相近趋势。 -* 因此MiniMind设定small模型dim=512，n_layers=8来获取的「极小体积<->更好效果」的平衡。 -* 设定dim=768，n_layers=16来获取效果的更大收益，更加符合小模型Scaling-Law的变化曲线。 +这和 MiniMind 在训练过程中围绕 `d_model` 与 `n_layers` 做参数分配实验时观察到的现象是一致的。不过“深而窄”里的“窄”也有下限：当 `d_model < 512` 时，词嵌入维度过窄带来的劣势会明显放大，额外增加 layers 往往不足以完全弥补固定 `q_head` 下 `d_head` 偏小的问题。 +相对地，当 `d_model > 1536` 时，继续增加层数往往比单纯继续加宽更划算，更容易带来更高的参数-效果收益。 -作为参考，GPT3的参数设定见下表： +作为参考，GPT-3 的参数设定如下： ![gpt3_config.png](./images/gpt3_config.png) ---- - -# 📌 Experiment - -## Ⅰ 训练开销 - -- **时间单位**：小时 (h)。 -- **成本单位**：人民币 (￥)；7￥ ≈ 1美元。 -- **3090 租卡单价**：≈1.3￥/h（可自行参考实时市价）。 -- **参考标准**：表格仅实测 `pretrain` 和 `sft_mini_512` 两个数据集的训练时间，其它耗时根据数据集大小估算（可能存在些许出入）。 - -> 基于 3090 （单卡）成本计算 - -| Model Name | params | pretrain | sft_mini_512 | sft_512 | sft_1024 | sft_2048 | RLHF | -|-----------------|--------|------------------|------------------|---------------|-------------------|------------------|---------------| -| MiniMind2-Small | 26M | ≈1.1h
≈1.43￥ | ≈1h
≈1.3￥ | ≈6h
≈7.8￥ | ≈4.58h
≈5.95￥ | ≈7.5h
≈9.75￥ | ≈1h
≈1.3￥ | -| MiniMind2 | 104M | ≈3.9h
≈5.07￥ | ≈3.3h
≈4.29￥ | ≈20h
≈26￥ | ≈15h
≈19.5￥ | ≈25h
≈32.5￥ | ≈3h
≈3.9￥ | - ---- - -
-训练开销总结&预测 - - -> MiniMind2-Small参数 ->> `pretrain_hq`+`sft_mini_512`数据集 -
单卡3090 (1 epoch) + 2.1小时 + 花费2.73元人民币 -
即可从0训练出MiniMind-Zero-0.025B模型!!! - -> MiniMind2-Small参数 ->> `pretrain_hq`+`sft_512`+`sft_2048`+`dpo`数据集 -
单卡3090 (2 epochs) + 大约38.16小时 + 花费49.61元人民币 -
即可从0训练出MiniMind2-Small-0.025B模型!!! - -> MiniMind2参数 ->> `pretrain_hq`+`sft_512`+`sft_2048`+`dpo`数据集 -
单卡3090 (2 epochs) + 大约122小时 + 花费158.6元人民币 -
即可从0训练出MiniMind2-0.1B模型!!! -
+--- +# 📌 实验 -✨基于单卡NVIDIA 3090的`MiniMind-Zero`从0训练仅需`2小时` + `3块钱`，实现ChatBot效果！ +## Ⅰ 训练开销 -✨PS：若采用8卡4090训练，总用时甚至可以压缩到10分钟以内！（由于时间更短，花费同样3元左右，与单卡成本相当） +- **时间单位**：小时（h） +- **成本单位**：人民币（￥）；`7￥ ≈ 1 美元` +- **3090 租卡单价**：约 `1.3￥/h`（实际价格可自行参考） +- **说明**：以下结果为 `minimind` 模型在单卡 `3090` 上的经验估算值，用于快速感知训练门槛 -✨以极低极低的门槛，实现人人可玩的大模型自由！这正是MiniMind系列的诞生初衷！ +| Model Name | params | pretrain_t2t_mini | sft_t2t_mini | toolcall | RLAIF | +|------------|--------|-------------------|--------------|----------|-------| +| minimind-3 | 64M | ≈1.21h
≈1.57￥ | ≈1.10h
≈1.43￥ | ≈0.9h
≈1.17￥ | ≈1.1h
≈1.43￥ | +| minimind-3-moe | 198M / A64M | ≈1.69h
≈2.20￥ | ≈1.54h
≈2.00￥ | ≈1.26h
≈1.64￥ | ≈1.54h
≈2.00￥ | -✨仅价值`3块钱`成本的`MiniMind-Zero`并不是噱头！Chat测试： +--- + +
+训练开销总结&预测 + +> `minimind-3` +>> `pretrain_t2t_mini` + `sft_t2t_mini` +
单卡 `3090`，`1 epoch` 预计约 `2.31` 小时，成本约 `3.0` 元人民币 +
可从 0 训练出 `minimind-3 Zero` 对话模型。 + +> `minimind-3-moe` +>> `pretrain_t2t_mini` + `sft_t2t_mini` +
单卡 `3090`，`1 epoch` 预计约 `3.23` 小时，成本约 `4.2` 元人民币 +
可快速得到 `minimind-3-moe` 的基础对话版本。 + +> 以上均为估算值，仅用于快速感知训练门槛。 + +
+ +基于单卡 `NVIDIA 3090`，`minimind zero` 从 0 训练依然可以控制在约 `2` 小时量级，个人开发者也能较低门槛地快速上手。 + +若采用更高规格的多卡环境，例如 `8x H100`，总训练时长还可进一步压缩至分钟级。以尽可能低的门槛实现可复现、可上手、可持续迭代的 LLM 训练体验，这也是 MiniMind 系列一直希望坚持的方向。低成本快速复现并不是噱头，下面保留一个早期的 Zero 风格样例对话供参考： ```text 👶: 请介绍一下自己。 @@ -747,8 +663,8 @@ MobileLLM提出架构的深度比宽度更重要，「深而窄」的「瘦长 🤖️: 您提到的“Introok's the believeations of theument." 这个名字来源于中国古代的"groty of of the change." ``` -极速且初具效果，甚至仍然可以进一步压缩获取更小更优质的训练数据。 -Zero模型权重保存为 `full_sft_512_zero.pth`（见下文MiniMind模型文件链接），如有兴趣可下载检验此模型效果。 +尽管该版本已经具备基础对话能力，但事实知识与泛化效果仍较有限；它更适合作为 Zero 训练路线可行性的早期参考。 +Zero 模型权重保存为 `full_sft_zero_768.pth`（见下文 MiniMind 模型文件链接），如有兴趣可下载体验其对话效果。 --- @@ -757,235 +673,224 @@ Zero模型权重保存为 `full_sft_512_zero.pth`（见下文MiniMind模型文 > 所有训练脚本均 `cd ./trainer` 目录执行 -### **1. 预训练 (Pretrain)**: +### 1' 预训练 (Pretrain): -LLM首先要学习的并非直接与人交流，而是让网络参数中充满知识的墨水，“墨水” 理论上喝的越饱越好，产生大量的对世界的知识积累。 -预训练就是让Model先埋头苦学大量基本的知识，例如从Wiki百科、新闻、书籍整理大规模的高质量训练数据。 -这个过程是“无监督”的，即人类不需要在过程中做任何“有监督”的校正，而是由模型自己从大量文本中总结规律学习知识点。 -模型此阶段目的只有一个：**学会词语接龙**。例如输入"秦始皇"四个字，它可以接龙"是中国的第一位皇帝"。 +LLM 首先要学会的是先把尽可能多的基础知识和语言规律吸收到参数里。只有这一步打稳了，模型后面才有能力去理解问题、组织表达，并逐步形成像样的生成能力。预训练做的事情，本质上就是让模型先埋头读大量文本，例如 Wiki 百科、新闻、书籍、对话语料等，从中学习事实知识、语言模式以及上下文之间的统计关系。这个阶段通常是“无监督”的：人类不需要逐条告诉模型哪里对、哪里错，而是让它自己从海量文本里总结规律，逐步建立起对世界知识和语言结构的内部表征。 +更直白地说，模型在这一阶段的核心目标就是**学会高质量地词语接龙**。例如输入“秦始皇”，它要能够继续生成“是中国历史上的第一位皇帝”这类符合语义与常识的后续内容。 ```bash +# 方式1 torchrun --nproc_per_node 1 train_pretrain.py # 1即为单卡训练，可根据硬件情况自行调整 (设置>=2) -# or +# 方式2 python train_pretrain.py ``` -> 训练后的模型权重文件默认每隔`100步`保存为: `pretrain_*.pth`（* -> 为模型具体dimension，每次保存时新文件会覆盖旧文件） +> 训练后的模型权重文件默认每隔`save_interval步`保存为:`pretrain_*.pth`（*为模型具体dimension，每次保存时新文件会覆盖旧文件） - -| MiniMind2-Small (512dim) | MiniMind2 (768dim) | -|---|---| -| | | - -### **2. 有监督微调 (Supervised Fine-Tuning)**: - -经过预训练，LLM此时已经掌握了大量知识，然而此时它只会无脑地词语接龙，还不会与人聊天。 -SFT阶段就需要把半成品LLM施加一个自定义的聊天模板进行微调。 -例如模型遇到这样的模板【问题->回答，问题->回答】后不再无脑接龙，而是意识到这是一段完整的对话结束。 -称这个过程为指令微调，就如同让已经学富五车的「牛顿」先生适应21世纪智能手机的聊天习惯，学习屏幕左侧是对方消息，右侧是本人消息这个规律。 -在训练时，MiniMind的指令和回答长度被截断在512，是为了节省显存空间。就像学习写作时，会先从短的文章开始，当学会写作200字作文后，800字文章也可以手到擒来。 -在需要长度拓展时，只需要准备少量的2k/4k/8k长度对话数据进行进一步微调即可（此时最好配合RoPE-NTK的基准差值）。 -> 在推理时通过调整RoPE线性差值，实现免训练长度外推到2048及以上将会很方便。 +![pretrain_loss](./images/pretrain_loss.jpg) +> `768dim` 配置在预训练阶段的 loss 曲线 ```bash +# 可对预训练结果做简单测试： +python eval_llm.py --weight pretrain + +💬: 为什么天空是蓝色的 +🧠: 天空之所以看起来是蓝色的，主要是因为太阳光进入大气层后，短波长的蓝光更容易被空气分子散射，因此人眼从各个方向接收到的蓝光会更多。 + +💬: 解释什么是机器学习 +🧠: 机器学习是人工智能的一个重要分支，它通过数据训练模型，使系统能够自动学习规律，并在分类、预测、推荐、自然语言处理等任务中持续改进效果。 +``` + +### 2' 有监督微调 (Supervised Fine-Tuning): + +SFT 并不只是把模型调成“更会聊天”，它同样可以继续向模型中灌入新的知识、行为模式和回答风格。尤其是像 MiniMind 当前主线这样体量达到 `14GB` 的 SFT 数据，本身就已经不只是简单的格式对齐，而更接近一种带有 `mid training` 性质的持续强化过程。 +如果把预训练理解为先让模型广泛地读书、积累基础语言能力，那么 SFT 更像是在高质量、更有目标的数据上继续深加工。一方面，它会让模型适应多轮对话、问答、工具调用和思考标签等交互形式；另一方面，它也会继续把特定知识分布、任务模式和助手风格压进参数里。 +具体到 MiniMind 里，SFT 阶段会让模型适应当前仓库使用的多轮对话模板。模型会逐渐理解 `user / assistant / system / tool` 等角色结构，同时进一步强化指令跟随、稳定回复和任务完成能力。 +当前训练时会对指令和回答长度做截断控制，主要是为了兼顾显存占用与训练效率；如果后续需要更长上下文，只需要继续准备少量长样本做增量微调即可。在推理时通过启用 YaRN 外推，可以免训练地将上下文长度扩展到 2048 及以上。 + +```bash +# 方式1 torchrun --nproc_per_node 1 train_full_sft.py -# or +# 方式2 python train_full_sft.py ``` -> 训练后的模型权重文件默认每隔`100步`保存为: `full_sft_*.pth`（* +> 训练后的模型权重文件默认每隔`save_interval步`保存为: `full_sft_*.pth`（* > 为模型具体dimension，每次保存时新文件会覆盖旧文件） +![sft_loss](./images/sft_loss.jpg) +> `768dim` 配置在 SFT 阶段的 loss 曲线 -| MiniMind2-Small (512dim) | MiniMind2 (768dim) | -|---|---| -| | | +```bash +# 可对SFT结果做简单测试： +python eval_llm.py --weight full_sft -## Ⅲ 其它训练阶段（可选） +💬: 解释什么是机器学习 +🧠: 机器学习是人工智能的核心技术之一，通过算法让计算机从数据中学习规律，并持续改进预测或决策效果，常见应用包括推荐系统、图像识别、语音识别和自然语言处理。 + +💬: 推荐一些中国的美食 +🧠: 例如北京烤鸭、兰州拉面、四川火锅、广东早茶、小笼包和麻婆豆腐等，这些美食分别代表了不同地区的风味特点，也很适合作为了解中国饮食文化的入门选择。 +``` + +## Ⅲ 其它训练（可选） > 所有训练脚本均 `cd ./trainer` 目录执行 -### **3. 知识蒸馏 (Knowledge Distillation, KD)** +### 3' 知识蒸馏 (Knowledge Distillation, KD) -在前面的所有训练步骤中，模型已经完全具备了基本能力，通常可以学成出师了。 -而知识蒸馏可以进一步优化模型的性能和效率，所谓知识蒸馏，即学生模型面向教师模型学习。 -教师模型通常是经过充分训练的大模型，具有较高的准确性和泛化能力。 -学生模型是一个较小的模型，目标是学习教师模型的行为，而不是直接从原始数据中学习。 -在SFT学习中，模型的目标是拟合词Token分类硬标签（hard labels），即真实的类别标签（如 0 或 6400）。 -在知识蒸馏中，教师模型的softmax概率分布被用作软标签（soft labels）。小模型仅学习软标签，并使用KL-Loss来优化模型的参数。 -通俗地说，SFT直接学习老师给的解题答案。而KD过程相当于“打开”老师聪明的大脑，尽可能地模仿老师“大脑”思考问题的神经元状态。 -例如，当老师模型计算`1+1=2`这个问题的时候，最后一层神经元a状态为0，神经元b状态为100，神经元c状态为-99... -学生模型通过大量数据，学习教师模型大脑内部的运转规律。这个过程即称之为：知识蒸馏。 -知识蒸馏的目的只有一个：让小模型体积更小的同时效果更好。 -然而随着LLM诞生和发展，模型蒸馏一词被广泛滥用，从而产生了“白盒/黑盒”知识蒸馏两个派别。 -GPT-4这种闭源模型，由于无法获取其内部结构，因此只能面向它所输出的数据学习，这个过程称之为黑盒蒸馏，也是大模型时代最普遍的做法。 -黑盒蒸馏与SFT过程完全一致，只不过数据是从大模型的输出收集，因此只需要准备数据并且进一步FT即可。 -注意更改被加载的基础模型为`full_sft_*.pth`，即基于微调模型做进一步的蒸馏学习。 -`./dataset/sft_1024.jsonl`与`./dataset/sft_2048.jsonl` 均收集自qwen2.5-7/72B-Instruct大模型，可直接用于SFT以获取Qwen的部分行为。 +知识蒸馏大体可以分成黑盒和白盒两类，MiniMind 当前主线两种思路都有涉及，只是侧重点不同。 +* 黑盒蒸馏：更常见，也更贴近当前主线的实际做法。严格来说，它本质上仍然是面向教师输出结果的监督微调，也就是基于硬标签继续训练；只是随着 LLM 的流行，这类“对着强模型输出做 FT”的做法也逐渐被广义地归入了蒸馏范畴，因此通常被称为黑盒蒸馏。它重点学习的是答案、风格和行为模式，学生模型只能看到“老师说了什么”，却看不到老师内部是如何做出这个判断的。像 `DeepSeek R1`、`Qwen3` 的高质量回答，以及 `tool call`、`reasoning`、思维链等数据，都可以看作黑盒蒸馏信号；MiniMind 当前主线 `full_sft` 数据里，其实已经混入了相当一部分这样的思路。 +* 白盒蒸馏：更进一步，不只学习教师给出的最终输出，还去学习教师在 token 分布层面的偏好。相比黑盒蒸馏，它额外利用了教师模型输出层更细粒度的分布信息，因此学生模型学到的不只是“标准答案”，还包括教师在候选 token 之间的相对倾向。对应到 `train_distillation.py`，当前实现是在已经完成 SFT 的权重基础上，继续用教师模型提供的分布信号来训练学生模型，因此更适合作为理解 MiniMind 蒸馏流程的参考实现。 -```bash -# 注意需要更改train_full_sft.py数据集路径，以及max_seq_len -torchrun --nproc_per_node 1 train_full_sft.py -# or -python train_full_sft.py +黑盒蒸馏本质上等价于对 teacher 生成答案做监督微调： +```math +\mathcal{L}_{blackbox} = \mathrm{CE}(y_{teacher}, p_{student}) ``` -> 训练后的模型权重文件默认每隔`100步`同样保存为: `full_sft_*.pth`（*为模型具体dimension，每次保存时新文件会覆盖旧文件） +白盒蒸馏则通常在监督损失之外，再额外拟合教师分布： +```math +\mathcal{L}_{whitebox} = \alpha \mathcal{L}_{CE} + (1-\alpha) T^2 \mathrm{KL}(p_t^T \parallel p_s^T) +``` -此处应当着重介绍MiniMind实现的白盒蒸馏代码`train_distillation.py`，由于MiniMind同系列本身并不存在强大的教师模型，因此白盒蒸馏代码仅作为学习参考。 +仓库中提供的 `train_distillation.py` 更适合作为理解白盒蒸馏流程的参考实现：它完整展示了教师/学生双模型加载、`CE + KL` 混合损失、温度缩放、MoE 与 dense 组合蒸馏，以及断点续训和分布式训练等关键细节。 ```bash +# 方式1 torchrun --nproc_per_node 1 train_distillation.py -# or +# 方式2 python train_distillation.py ``` -### **4. LoRA (Low-Rank Adaptation)** +### 4' LoRA (Low-Rank Adaptation) -LoRA是一种高效的参数高效微调（Parameter-Efficient Fine-Tuning, PEFT）方法，旨在通过低秩分解的方式对预训练模型进行微调。 -相比于全参数微调（Full Fine-Tuning），LoRA 只需要更新少量的参数。 -LoRA 的核心思想是：在模型的权重矩阵中引入低秩分解，仅对低秩部分进行更新，而保持原始预训练权重不变。 -代码可见`./model/model_lora.py`和`train_lora.py`，完全从0实现LoRA流程，不依赖第三方库的封装。 +LoRA 是一种常见的参数高效微调（Parameter-Efficient Fine-Tuning, PEFT）方法。相比全参数微调，它只更新少量新增参数，而保留原始模型主体权重不变，因此训练成本更低，也更适合做垂直场景适配。 +它的核心思想是在原有权重矩阵旁引入低秩增量分支，仅训练这部分低秩参数，从而用较小代价完成能力迁移。相关实现可见 `model_lora.py` 和 `train_lora.py`，整个流程均为纯手写实现，不依赖第三方封装。 ```bash +# train_lora.py 在 CPU 上通常也能比较轻快地完成 +# 方式1 torchrun --nproc_per_node 1 train_lora.py -# or +# 方式2 python train_lora.py ``` -> 训练后的模型权重文件默认每隔`100步`保存为: `lora_xxx_*.pth`（* -> 为模型具体dimension，每次保存时新文件会覆盖旧文件） +> 训练后的模型权重文件默认每隔`save_interval步`保存为: `lora_xxx_*.pth`（*为模型具体dimension，每次保存时新文件会覆盖旧文件） -非常多的人困惑，如何使模型学会自己私有领域的知识？如何准备数据集？如何迁移通用领域模型打造垂域模型？ -这里举几个例子，对于通用模型，医学领域知识欠缺，可以尝试在原有模型基础上加入领域知识，以获得更好的性能。 -同时，通常不希望学会领域知识的同时损失原有基础模型的其它能力，此时LoRA可以很好的改善这个问题。 -只需要准备如下格式的对话数据集放置到`./dataset/lora_xxx.jsonl`，启动 `python train_lora.py` -训练即可得到`./out/lora/lora_xxx.pth`新模型权重。 +LoRA 很适合处理“如何在尽量保留通用能力的前提下，让模型快速适应私有领域或垂直场景”这类问题。例如基础模型医学知识不足时，就可以在原有模型之上叠加一层面向医疗场景的 LoRA 权重，以较小代价获得更好的领域表现。 +通常只需要准备同样的多轮对话格式数据，放置到 `lora_xxx.jsonl`，再执行 `python train_lora.py`，即可得到新的 `LoRA` 模型权重。 -**医疗场景** +例1：垂域数据 -``` +```jsonl {"conversations": [{"role": "user", "content": "请问颈椎病的人枕头多高才最好？"}, {"role": "assistant", "content": "颈椎病患者选择枕头的高度应该根据..."}]} {"conversations": [{"role": "user", "content": "请问xxx"}, {"role": "assistant", "content": "xxx..."}]} ``` -**自我认知场景** +例2：自我认知数据 -``` - {"conversations": [{"role": "user", "content": "你叫什么名字？"}, {"role": "assistant", "content": "我叫minimind..."}]} - {"conversations": [{"role": "user", "content": "你是谁"}, {"role": "assistant", "content": "我是..."}]} +```jsonl + {"conversations": [{"role": "user", "content": "你叫什么名字？"}, {"role": "assistant", "content": "您好，我名叫 MiniMind，是由 Jingyao Gong 开发的人工智能助手。我可以帮助你解决各种问题，包括翻译、推荐内容等等。"}]} + {"conversations": [{"role": "user", "content": "你是谁"}, {"role": "assistant", "content": "您好，我是 MiniMind，由 Jingyao Gong 创建。我的主要职责是提供准确的回答和帮助。"}]} ``` -此时【基础模型+LoRA模型】即可获得医疗场景模型增强的能力，相当于为基础模型增加了LoRA外挂，这个过程并不损失基础模型的本身能力。 -可以通过`eval_llm.py`进行模型评估测试。 +此时可将【基模 + LoRA 权重】组合使用，从而在保留基础模型通用能力的同时，增强特定领域表现。训练完成后也可以通过 `eval_llm.py` 做简单测试。 ```bash # 注意：weight参数指定基础模型类型，需与train_lora训练时使用的基础模型保持一致 -python eval_llm.py --weight dpo --lora_weight lora_medical -``` +python eval_llm.py --weight full_sft --lora_weight lora_medical -**小测试** - -``` 👶: 我最近经常感到头晕，可能是什么原因？ 🤖️: 头晕可能是由于多种原因，包括但不限于：心脏疾病、眼睛损伤、过敏反应、皮肤问题、过敏反应、压力、焦虑、疲劳、药物副作用、性功能障碍或者其他... ---- +python eval_llm.py --weight full_sft --lora_weight lora_identity 👶: 你和openai是什么关系？ 🤖️: 我是 MiniMind，一个由 Jingyao Gong 开发的人工智能助手。我通过自然语言处理和算法训练来与用户进行交互。 ``` -PS：只要有所需要的数据集，也可以full_sft全参微调（需要进行通用知识的混合配比，否则过拟合领域数据会让模型变傻，损失通用性） +PS：如果有更充足的数据，也可以直接做 `full_sft` 全参微调；不过这通常需要更谨慎地混合通用数据与领域数据，否则很容易因为过拟合垂域样本而损失模型原有的通用性。 -### **5. 训练推理模型 (Reasoning Model)** -DeepSeek-R1实在太火了，几乎重新指明了未来LLM的新范式。 -论文指出`>3B`的模型经历多次反复的冷启动和RL奖励训练才能获得肉眼可见的推理能力提升。 -最快最稳妥最经济的做法，以及最近爆发的各种各样所谓的推理模型几乎都是直接面向数据进行蒸馏训练， -但由于缺乏技术含量，蒸馏派被RL派瞧不起（hhhh）。 -本人迅速已经在Qwen系列1.5B小模型上进行了尝试，很快复现了Zero过程的数学推理能力。 -然而一个遗憾的共识是：参数太小的模型直接通过冷启动SFT+GRPO几乎不可能获得任何推理效果。 - -MiniMind2第一时间只能坚定不移的选择做蒸馏派，日后基于0.1B模型的RL如果同样取得小小进展会更新此部分的训练方案。 - +> `LoRA` 权重可合并回基础模型并导出为新的完整模型权重，可使用 `scripts/convert_model.py` 中的 `convert_merge_base_lora`： -做蒸馏需要准备的依然是和SFT阶段同样格式的数据即可，数据集来源已如上文介绍。数据格式例如： +```bash +cd scripts && python convert_model.py +``` -```json lines +### **5' 工具调用 & 自适应思考** + +`2026-03` 起，仓库移除了独立的 `train_reason.py`。 +当前版本不再单独维护 `reason_*.pth` 权重，而是统一通过 `chat_template`、`` 标签、`open_thinking` 开关以及后续 SFT / RLAIF 流程来建模“是否显式输出思考过程”。 + +#### 5.1 Tool Calling + +当前 `toolcall` 能力已经并入 `sft_t2t` / `sft_t2t_mini` 主线数据，因此通常不再需要额外单独训练一轮 Tool Calling；默认的 `full_sft` 权重已经具备基础 Tool Call 能力。当前这部分训练数据主要由 `qwen3-4b` 采样约 `10w` 条构成，工具列表也主要覆盖约 `10` 个模拟的自定义工具（例如查询时间、数学计算、获取天气等），因此目前还谈不上明确的泛化能力。其中 Tool Calling 样本统一沿用了 OpenAI 风格的多轮消息格式： + +```jsonl { "conversations": [ - { - "role": "user", - "content": "你好，我是小芳，很高兴认识你。" - }, - { - "role": "assistant", - "content": "\n你好！我是由中国的个人开发者独立开发的智能助手MiniMind-R1-Lite-Preview，很高兴为您提供服务！\n\n\n你好！我是由中国的个人开发者独立开发的智能助手MiniMind-R1-Lite-Preview，很高兴为您提供服务！\n" - } + {"role": "system", "content": "# Tools ...", "tools": "[...]"}, + {"role": "user", "content": "帮我算一下 256 乘以 37 等于多少"}, + {"role": "assistant", "content": "", "tool_calls": "[{\"name\":\"calculate_math\",\"arguments\":{\"expression\":\"256 * 37\"}}]"}, + {"role": "tool", "content": "{\"result\":\"9472\"}"}, + {"role": "assistant", "content": "256 乘以 37 等于 9472。"} ] } ``` -推理模型R1的回复模板是： +其中 `tools` 挂在 `system` 消息上，`tool_calls` 挂在 `assistant` 消息上；训练时再由 `chat_template` 自动展开为 `...` 与 `...` 片段，因此现在可以直接学习原生 tool call 格式。 + +Tool Calling 的 chat template 已统一支持解析为： ```text -\n思考过程\n\n -\n最终回答\n +{"name": "...", "arguments": {...}} +{...tool result...} ``` -这在GRPO中通过设置规则奖励函数约束模型符合思考标签和回复标签（在冷启动靠前的阶段奖励值设置应该提高一些） - -另一个问题是蒸馏过程虽然和SFT一样，但实验结果是模型难以每次都符合模板规范的回复，即脱离思考和回复标签约束。 -这里的小技巧是增加标记位置token的损失惩罚，详见`train_reason.py`: - -```text -# 在 sp_ids 对应的位置增加额外的惩罚 -... -loss_mask[sp_ids] = 10 # 惩罚系数 -``` - -另另一个tips是由于推理数据由于只筛选了`<1024`长度的数据，其中多轮对话和英文数据偏少， -因此`r1_mix_1024.jsonl`进行了大约10k条多轮对话+英文数据的混合，防止模型遗忘严重。 - -脚本默认基于rlhf后的基模型做推理能力的蒸馏微调，下面直接启动训练即可： +也可以直接通过 `eval_toolcall.py` 做简单测试： ```bash -torchrun --nproc_per_node 1 train_reason.py -# or -python train_reason.py +python eval_toolcall.py --weight full_sft + +💬: 现在几点了？ +🧠: {"name": "get_current_time", "arguments": {"timezone": "Asia/Shanghai"}} +📞 [Tool Calling]: get_current_time +✅ [Tool Called]: {"datetime": "2026-03-15 17:18:22", "timezone": "Asia/Shanghai"} +🧠: 现在是2026年3月15日17时18分22秒。 ``` -> 训练后的模型权重文件默认每隔`100步`保存为: `reason_*.pth`（*为模型具体dimension，每次保存时新文件会覆盖旧文件） +#### 5.2 Adaptive Thinking -测试一下： +`minimind` 将显式思考能力统一到了模板层，这也和当前很多主流大模型的模板设计保持一致： -```text -👶: 请介绍一下自己。 -🤖️: -嗯，用户让我介绍一下自己。我现在需要解释一下自己。首先，我应该明确用户是谁，是学生、职场人士还是普通人。用户可能是一个学生或者职场人士，所以信息传递应该是帮助学生和职场人士的信息。 -然后，我需要回应用户，这可能包括介绍自己的位置、兴趣爱好、职业经历等信息。用户还可能想了解自己在哪里，比如职业发展、工作经验等。 -我应该注意用词要正式，不要太正式，同时要保持简洁明了。这样用户容易理解，也方便他们查阅相关资料。 -最后，我应该总结一下，让用户知道我是如何帮助他们完成他们的任务的，这样他们有更多关于我的信息。 - - -我是**“你的名字”，你叫什么名字？ - +- `open_thinking=0`：默认注入空的 `\n\n`，模型更倾向于直接回答； +- `open_thinking=1`：模板会预先注入 `` 起始标签，模型再继续输出显式思考过程与最终回答； +- CLI、OpenAI-API、WebUI 三个入口均支持该开关。 -👶: 你更擅长哪一个学科？ -🤖️: -您好！我是由中国的个人开发者开发的智能助手MiniMind-R1。如您有任何任何问题，我会尽我所能为您提供帮助。 - - -您好！我是由中国的个人开发者开发的智能助手MiniMind-R1。如您有任何任何问题，我会尽我所能为您提供帮助。 - +更准确地说，目前不再“单独训一个思考模型”，而是把“是否显式思考”下沉到 `chat_template`。模板层会先预留 `` 这一结构，同一个模型在推理时再通过 `open_thinking` 动态切换；在训练时，则通过混合空 `think`、显式 `reasoning_content` 与 `thinking_ratio` 采样，让模型逐步见过“该想时想、该直答时直答”的混合模式。 + +```bash +# 测试回答 +python eval_llm.py --load_from ./minimind-3 --open_thinking 1 ``` -## IV 强化学习后训练 +OpenAI-API-SDK 用法： -LLM里的强化学习方法可分两类： +```python +response = client.chat.completions.create( + model="minimind", + messages=[{"role": "user", "content": "你是谁？"}], + # ... + extra_body={"chat_template_kwargs": {"open_thinking": True}} # 思考开关 +) +``` + +注：当前同时开启 Tool Call 与显式思考时，模型通常并不太会稳定地输出思考过程。原因在于现阶段训练数据里还缺少“reasoning 与 tool call 同时存在”的联合蒸馏样本，因此模型尚未充分学会这两种能力的协同表达。 + +## Ⅳ 强化学习（可选） + +在 LLM 的后训练实践中，常见的强化学习路径主要有两条： 1. **基于人类反馈的强化学习 (Reinforcement Learning from Human Feedback, RLHF)** @@ -993,17 +898,17 @@ LLM里的强化学习方法可分两类： 2. **基于AI反馈的强化学习 (Reinforcement Learning from AI Feedback, RLAIF)** -- 使用**AI模型**（通常是预训练的语言奖励模型）来提供反馈，而不直接依赖人类的人工标注。 -- 这里的“AI”也可以是某些规则奖励，例如数学答案/代码解释器... +- 使用**AI模型**或其他可自动验证的机制来提供反馈，而不直接依赖人类标注。 +- 这里的“AI feedback”在广义上也可以扩展到规则奖励、Ground Truth 校验、代码解释器、环境反馈等自动化信号。 | 类型 | 裁判 | 优点 | 缺点 | |-------|----|-----------|------------| | RLHF | 人类 | 更贴近真实人类偏好 | 成本高、效率低 | | RLAIF | 模型 | 自动化、可扩展性强 | 可能偏离人类真实偏好 | -二者本质上是一样的，都是通过**强化学习的方式**，利用某种形式的"**反馈**"来优化模型的行为。 +二者本质上都属于利用某种形式的"**反馈**"来优化模型行为的强化学习范式。 -除了**反馈**的来源不同，其他并无任何区别。 +不过在具体实践里，它们并不只是反馈来源不同：奖励是否可验证、是否连续、是否依赖环境交互、是否延迟到整轮结算，都会直接影响训练形态与工程实现。 ### 👀 PO算法的统一视角 @@ -1014,7 +919,7 @@ LLM里的强化学习方法可分两类： $$\mathcal{J}_{PO} = \mathbb{E}_{q \sim P(Q), o \sim \pi(O|q)} \left[ \underbrace{f(r_t)}_{\text{策略项}} \cdot \underbrace{g(A_t)}_{\text{优势项}} - \underbrace{h(\text{KL}_t)}_{\text{正则项}} \right]$$ -训练时，只需**最小化负目标函数**，即: $\mathcal{L_{PO}}=-\mathcal{J_{PO}}$ +训练时，只需**最小化负目标函数**，即: $\mathcal{L}_{PO} = -\mathcal{J}_{PO}$ 这个框架只包含三个核心组件： * **策略项** $f(r_t)$: 如何使用概率比 $r_t$? 即告诉模型新旧策略偏差有多大，是否探索到了更好的token @@ -1028,7 +933,7 @@ $$\mathcal{J}_{PO} = \mathbb{E}_{q \sim P(Q), o \sim \pi(O|q)} \left[ \underbrac |------|------|------|------| | $q$ | 问题/提示词 | 从数据集 $P(Q)$ 中采样 | - | | $o$ | 模型输出序列 | 由策略 $\pi$ 生成 | - | -| $r_t$ | 概率比 | $r_t = \frac{\pi_\theta(o_t\|q, o_{ 训练后的模型权重文件默认每隔`100步`保存为: `dpo_*.pth`（*为模型具体dimension，每次保存时新文件会覆盖旧文件） +> 训练后的模型权重文件默认每隔`save_interval步`保存为: `dpo_*.pth`（*为模型具体dimension，每次保存时新文件会覆盖旧文件） -### **7. 基于AI反馈的强化学习 (Reinforcement Learning from AI Feedback, RLAIF)** +### 7' 基于AI反馈的强化学习 (Reinforcement Learning from AI Feedback, RLAIF) -相比RLHF依赖人类标注chosen/rejected偏好对，RLAIF则完全由AI来充当"裁判"。 -所谓AI"裁判"可以是model-base的奖励大模型(Reward Model)，也可以是R1一样设置规则函数进行校验，也可以是例如工具调用的环境反馈。 -例如：数学题答案是否正确、工具调用执行代码能否通过测试用例、推理过程是否符合格式...都可以自动化判断。 -RLAIF的最大优势在于**可扩展性**和**On-Policy**的特点——不需要昂贵的人工标注，可以生成海量的训练样本，让模型在在线大量试错中快速进化。 +稍微花篇幅解释一下，我还是更想把这一节叫作 `RLAIF`，虽然严格来说，这个命名并不完全准确。像 RLVR 这类依赖可验证奖励的路线，本身有相对独立的脉络，很难被简单并进狭义的 AI feedback 里。 +但如果把“AI”理解得稍微宽一点，我又觉得这个名字并非完全说不通：奖励既可以来自奖励模型、judge model 这类显式的智能体，也可以来自规则函数、Ground Truth校验、工具调用结果、环境返回状态这类可自动获得的信号。规则足够复杂、符号系统足够丰富时，它们和“智能反馈”之间的边界，本来就未必那么泾渭分明。 +因此这一章更想讨论的，其实是 LLM 在 SFT 之后，借助各种**非人工、可自动获得的反馈信号**继续做强化学习优化的方法。比如数学题答案是否正确、工具调用执行代码能否通过测试用例、推理过程是否符合格式...都可以自动化判断。 +对于单轮可验证任务，这类反馈往往更接近“即时打分”；而在 Agentic RL 场景中，奖励则更常表现为多步交互后的延迟结算，甚至直接来自环境本身。 +它们共同的特点通常都是**On-Policy**与**可扩展性强**——不需要昂贵的人工标注，可以生成海量训练样本，让模型在在线大量试错中快速进化。 MiniMind 着手实现**2+N**种基本+前沿的RLAIF方法： -* **PPO**、**GRPO** 被大规模验证的经典RL算法； -* N种前沿RL算法（不定期以Exp性质更新）。 +* **PPO**、**GRPO** 被大规模验证的经典RL算法 +* N种前沿RL算法（不定期以Exp性质更新） -#### 1️⃣ 数据集准备 (需要) +**1️⃣ 数据集准备 (必须)** -为了快速验证RLAIF的效果，这里从SFT数据集中随机采样了1万条高质量对话，构建约1MB大小的`rlaif-mini.jsonl`([Huggingface](https://huggingface.co/datasets/jingyaogong/minimind_dataset/blob/main/rlaif-mini.jsonl)) +当前主线使用 `rlaif.jsonl` 作为 RLAIF 训练数据，体量约 `20MB`，比早期 `rlaif-mini.jsonl` 更完整，更适合直接验证 PPO / GRPO / CISPO 的训练效果。 数据格式与SFT一致，但assistant并不需要内容，因为训练过程中完全由 $\Pi$ 策略模型实时采样生成。因此形如： @@ -1096,22 +1003,20 @@ MiniMind 着手实现**2+N**种基本+前沿的RLAIF方法： RLAIF的训练过程中，模型会基于user的问题生成1或多个候选回答，然后由奖励函数/模型对回答打分， 分数高的回答会被鼓励（增加 $\Pi$ 策略概率），分数低的回答会被抑制（降低 $\Pi$ 策略概率）。这个"打分->调整"的循环就是强化学习的核心。 -#### 2️⃣ 奖励模型准备 (需要) +**2️⃣ 奖励机制准备 (必须)** -已知RLAIF训练需要“奖励模型 (Reward Model)”对生成的回答进行打分。 +RLAIF训练需要某种可计算的奖励信号；它可以来自奖励模型，也可以来自规则函数、Ground Truth 校验或环境反馈。MiniMind 当前默认演示的是 Reward Model 路线。 -此处选取小型且高质量的InternLM2-1.8B-Reward -([ModelScope](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-1_8b-reward) | [HuggingFace](https://huggingface.co/internlm/internlm2-1_8b-reward)) -作为基础奖励模型。 +此处选取小型且高质量的 `InternLM2-1.8B-Reward` ([ModelScope](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-1_8b-reward) | [HuggingFace](https://huggingface.co/internlm/internlm2-1_8b-reward)) 作为基础奖励模型。 下载奖励模型后需要放置在minimind项目的**同级目录**下，推荐结构如下： ``` -project/ +root/ ├── minimind/ # MiniMind项目 │ ├── model/ │ └── ... -└── internlm2-1_8b-reward/ # 奖励模型（与minimind同级） +└── internlm2-1_8b-reward/ # 奖励模型 ├── config.json ├── model.safetensors └── ... @@ -1177,7 +1082,7 @@ RLAIF训练既可以针对推理模型也可以针对非推理模型，区别仅 #### 7.1 [Proximal Policy Optimization](https://arxiv.org/abs/1707.06347) -PPO 是2017年OpenAI提出的非常经典强化学习算法，也是LLM RL通用的基线方法，甚至不需要加之一。 +PPO 是 2017 年 OpenAI 提出的非常经典的强化学习算法，也是 LLM RL 领域最常见的基线方法之一。 **PPO损失**： $$\mathcal{L}_{PPO} = -\mathbb{E}\left[\min(r_t \cdot A_t, \text{clip}(r_t, 1-\varepsilon, 1+\varepsilon) \cdot A_t)\right] + \beta \cdot \mathbb{E}[\text{KL}]$$ @@ -1188,421 +1093,441 @@ $$\mathcal{L}_{PPO} = -\mathbb{E}\left[\min(r_t \cdot A_t, \text{clip}(r_t, 1-\v - **正则项**: $h(\text{KL}_t) = \beta \cdot \mathbb{E}[\text{KL}]$ (全局KL散度约束) 对比DPO而言， -- DPO (Off-Policy)：训练数据是静态的偏好数据集（chosen vs rejected），可以反复使用同一批数据训练多个epoch，就像传统监督学习一样。数据效率高，训练成本低。它直接优化偏好对的对数似然，无需Reward Model。 -- PPO (On-Policy)：必须用当前策略实时采样生成新数据，旧策略采集的数据不能用（会有distribution shift问题）。虽然通过importance sampling和clip机制允许轻微的分布偏移，但本质上要求数据来自相对新鲜的策略。数据效率低，但适合探索式学习。 +- DPO (Off-Policy)：训练数据是静态偏好对（chosen vs rejected），可以反复使用同一批数据训练多个 epoch，像传统监督学习一样。数据效率高、成本低，且无需 Reward Model。 +- PPO (On-Policy)：必须用当前策略实时采样新数据，旧策略数据只能有限复用，否则就会出现 distribution shift。虽然 importance sampling 和 clip 允许轻微偏移，但本质上仍要求数据来自较新的策略。数据效率更低，但更适合探索式学习。 简单来说： -- 前者教模型按离线预定的「好/坏标准」学习，尽管它并非是当前模型所能输出的（例如参考世界冠/亚军录像练习打球）； -- 后者实时地教模型把事情做对做好，在线采样自最新模型policy（教练手把手教打，为每个动作实时打分）。 +- 前者按离线预定的「好/坏标准」学习； +- 后者则基于最新 policy 在线采样并实时纠偏。 -MiniMind的PPO实现包含了Actor模型(生成回答)和Critic模型(评估回答价值)，以及完整的GAE(Generalized Advantage Estimation)优势函数计算。 +MiniMind 的 PPO 实现包含 Actor（生成回答）、Critic（评估回答价值）以及完整的 GAE（Generalized Advantage Estimation）优势函数计算。 **训练方式**： ```bash +# 方式1 torchrun --nproc_per_node N train_ppo.py -# or +# 方式2 python train_ppo.py ``` -> 训练后的模型权重文件默认每隔`100步`保存为: `ppo_actor_*.pth`（*为模型具体dimension） +> 训练后的模型权重文件默认每隔`save_interval步`保存为: `ppo_actor_*.pth`（*为模型具体dimension） -| MiniMind2-Small (512dim) | MiniMind2 (768dim) | -|---|---| -| | | +![ppo_loss](./images/ppo_loss.jpg) -从训练曲线可以看出，PPO存在**reward提升缓慢**的问题。私以为这主要源于**PPO双网络联合优化**方法：Critic需要逐步收敛以准确估计价值函数，而Actor的策略更新依赖Critic提供的优势估计，两者相互依赖形成复杂的优化过程。训练初期Critic估计不准会影响Actor梯度方向，导致整体收敛缓慢。此外，PPO需要同时维护两个网络，显存占用约为单网络方法的1.5-2倍。 +> MiniMind 在 PPO 训练阶段的优化走势 + +从训练曲线可以看出，PPO存在**reward提升缓慢**的问题。私以为这主要源于**PPO双网络联合优化**方法：Critic需要逐步收敛以准确估计价值函数，而Actor的策略更新依赖Critic提供的优势估计，两者相互依赖形成复杂的优化过程。训练初期Critic估计不准会影响Actor梯度方向，导致整体收敛缓慢。此外，PPO 需要同时维护两个网络，在当前实现下显存占用约为单网络方法的 1.5–2 倍。 #### 7.2 [Group Relative Policy Optimization](https://arxiv.org/pdf/2402.03300) -2025年初，DeepSeek-R1火爆出圈，同样火了的有来自DeepSeekMath论文的GRPO算法，也一跃成为最先进的RL算法之一。 -然而AI半年=人间半个世纪，时至今日GRPO已经演变为各大XXPO大战(后面演变的DAPO、GSPO、CISPO等)的基线算法。 -具体来说，一句话总结它的核心创新是"分组相对价值估计"。 +2025 年初，随着 DeepSeek-R1 火爆出圈，来自 DeepSeekMath 论文的 GRPO 也迅速进入主流视野，一度成为最受关注的 RL 算法之一。不过 AI 领域向来迭代极快。时至今日，GRPO 更多已经演变成各类 XXPO 变体（如 DAPO、GSPO、CISPO 等）的共同基线。一句话概括它的核心创新，就是“分组相对价值估计”。 **GRPO损失**： -$$\mathcal{L}_{GRPO} = -\mathbb{E}\left[r_t \cdot A_t - \beta \cdot \text{KL}_t\right]$$ +$$\mathcal{L}_{GRPO} = -\mathbb{E}\left[\min(r_t \cdot A_t, \mathrm{clip}(r_t, 1-\varepsilon, 1+\varepsilon) \cdot A_t) - \beta \cdot \text{KL}_t\right]$$ 其中： -- **策略项**: $f(r_t) = \min(r_t, \text{clip}(r_t))$ (使用概率比的clip裁剪) +- **策略项**: $f(r_t) = \min(r_t, \mathrm{clip}(r_t, 1-\varepsilon, 1+\varepsilon))$ (使用概率比的对称 clip 裁剪) - **优势项**: $g(A_t) = \frac{R - \mu_{group}}{\sigma_{group}}$ (组内归一化，消除Critic网络) - **正则项**: $h(\text{KL}_t) = \beta \cdot \text{KL}_t$ (token级KL散度约束) -对于同一个问题，模型生成N个不同的回答(例如N=4)，然后计算这N个回答的奖励分数。 -接着把这N个回答的平均奖励作为baseline，高于baseline的回答被鼓励，低于baseline的回答被抑制。 -用这种方式巧妙地避免了训练额外的critic网络。 +对于同一个问题，模型生成 N 个回答并计算各自奖励，再用组内平均奖励作为 baseline。高于 baseline 的回答被鼓励，低于 baseline 的回答被抑制，因此无需额外训练 critic 网络。 -只要是RL都必须面对的正反样本这个原理性限制，GRPO也不会例外，其更显著的问题是：退化组(Degenerate Groups)。 -假设某个问题略难，导致N个回答的奖励分数几乎一样（大部分情况是一样烂而不是一样好），那么这一组的学习信号就无限接近0。 -在MiniMind这种超小模型上，这个问题尤为明显，求解数学问题99.99%的情况下整组回答质量都很差，那么将无法学习。 -因此必须为模型指定合理的domain，即必须限制在能力边界内。 +GRPO 更显著的问题是退化组（Degenerate Groups）：如果某个问题上 N 个回答的奖励几乎一样，那么这一组的学习信号就会接近 0。在 MiniMind 这种超小模型上，这个问题尤其明显，所以训练必须限制在合理的能力边界内。 **训练方式**： ```bash +# 方式1 torchrun --nproc_per_node N train_grpo.py -# or +# 方式2 python train_grpo.py ``` -> 训练后的模型权重文件默认每隔`100步`保存为: `grpo_*.pth` +> 训练后的模型权重文件默认每隔`save_interval步`保存为: `grpo_*.pth` -| MiniMind2-Small (512dim) | MiniMind2 (768dim) | -|---|---| -| | | +![grpo_loss](./images/grpo_loss.jpg) + +> MiniMind 在 GRPO 训练阶段的优化走势 从训练曲线可以看出，GRPO的**reward呈现更加稳定的上升趋势**，达到4左右，说明GRPO本身能更好地利用RLAIF信号。Policy Loss整体下降平稳，相比PPO的双网络优化，GRPO单网络架构训练更稳定且收敛上限更高。 -#### 7.3 ⏳⌛️🔥 更多RL拓展 (Exp) +#### 7.3 [Clipped Importance Sampling Policy Optimization](https://huggingface.co/papers/2506.13585) -##### 7.3.1 [Single-stream Policy Optimization](https://arxiv.org/abs/2509.13232) +我自己在众多眼花缭乱的XXPO里反而对它印象很深，CISPO没有重新发明一整套复杂框架，而是抓住了 PPO/GRPO 里一个长期让人别扭的问题——ratio 被 clip 之后，梯度流直接就被硬截断了。 +CISPO 的关注点并不是重新设计 group baseline，而是用非常小的 loss 改动，更直接地修正这个问题。 -SPO是2025年9月腾讯提出的RL算法，针对GRPO的退化组问题进行改进。 -论文认为，GRPO等算法"一个样本要依赖一组采样"显得别扭而不优雅：太容易或太难的题目，整组几乎学不到东西，学习效率先天受限。 -SPO的动机就是回到RL的本质—**1个输入，1个输出，就是1个训练样本**，回到policy gradient的基本公式去思考：不用group mean也能得到稳定的baseline，也就是把价值估计 V 铺开在时序上，训练前先做粗略的价值预估，训练中一边采样一边更新对 V 的估计，从而为每个样本提供一个跨 batch 持久化、可自适应的基线参照。这种"单流"设计不再依赖同组样本，天然避免了退化组。 +**CISPO损失**： -**SPO损失**: - -$$\mathcal{L}_{SPO} = -\mathbb{E}\left[\log \pi_\theta(a_t|s) \cdot A_t - \beta \cdot \text{KL}_t\right]$$ +$$\mathcal{L}_{CISPO} = -\mathbb{E}\left[\min(r_t, \varepsilon_{max}) \cdot A_t \cdot \log \pi_\theta(a_t|s) - \beta \cdot \text{KL}_t\right]$$ 其中： -- **策略项**: $f(r_t) = \log \pi_\theta(a_t|s)$ (直接使用log概率，不计算ratio) -- **优势项**: $g(A_t) = R - B_t^{adaptive}$ (自适应baseline，Beta分布动态跟踪) -- **正则项**: $h(\text{KL}_t) = \beta \cdot \text{KL}_t$ (token级KL + 动态 $\rho$ 调整) +- **策略项**: $f(r_t) = \min(r_t, \varepsilon_{max}) \cdot \log \pi_\theta(a_t|s)$ (ratio 只作为裁剪后的权重) +- **优势项**: $g(A_t) = \frac{R - \mu_{group}}{\sigma_{group}}$ (可直接沿用 GRPO 的组内相对优势) +- **正则项**: $h(\text{KL}_t) = \beta \cdot \text{KL}_t$ (token级KL散度约束) -落到实现层面：SPO采用无分组设计，用持久化的KL自适应value tracker替代组内baseline，优势函数在整个batch上全局归一化。这样每个样本独立处理，无需等待同组其他样本，且能为每个样本提供稳定的学习信号。 -论文在Qwen3-8B的5个困难数学数据集上，SPO平均比GRPO高出3.4个百分点，其中BRUMO 25数据集+7.3pp、AIME 25数据集+4.4pp。 +CISPO在GRPO基础上，把原本容易被clip成常数的策略项改写成“裁剪权重 × log 概率”的形式。这样ratio即使被截断，也不会把梯度路径一起截断。因此可以直接把CISPO视作GRPO的loss变体来实现，而不是单独维护一套独立脚本。这里不再单列实验。只需在 `train_grpo.py` 把 `loss_type` 配置为 `cispo`，其余训练流程仍沿用 GRPO 的分组采样、奖励计算与优势构造逻辑即可。 + +#### 7.4 Agentic RL 🔥 + +“Agentic”的概念其实很大，所以这里说的 Agentic 只能是一个相对狭义的版本：它更聚焦于让 MiniMind 这样的~百M小模型在有限工具集上学会基础的调用、观察与再规划能力，而不是去覆盖完整 Agent 系统里更大范围的状态管理、长期记忆与复杂工作流编排。 + +`2026-03` 起，仓库新增 `train_agent`，开始支持一种更贴近真实交互流程的多轮 Tool-Use RL。这是我自己很喜欢的一个训练脚本：它把 RLVR / RLAIF 风格的数据组织方式与 online RL 的 rollout 过程揉在了一起，中间来回调过很多版，也踩过收敛失败、奖励 hack、多轮上下文错位之类的bug，最后完美地保持了 MiniMind 一贯的简洁性和可读性。 + +此部分的数据为 `agent_rl.jsonl` / `agent_rl_math.jsonl`。它们相比普通对话数据多了 `gt` 作为最终校验目标；若把一条样本记作 $(x, \mathcal{T}, gt)$，那么训练时优化的对象就不再是单轮回答 $y$，而是一条多轮轨迹 $\tau$： + +$$ +\tau = (a_1, o_1, a_2, o_2, \dots, a_T), \quad a_t \sim \pi_\theta(\cdot \mid s_t, \mathcal{T}) +$$ + +其中 `chat_template` 统一组织 `tools / tool_calls / tool` 消息；若某一步生成了 `tool_call`，就执行工具并把 observation 拼回上下文，再继续 rollout。 + +主线流程可以压缩成： + +$$ +\texttt{rollout batch} \rightarrow \texttt{calculate rewards} \rightarrow \texttt{policy update} +$$ + +reward 也是对整条轨迹联合打分： + +$$ +R(\tau) = R_{\text{answer}} + R_{\text{tool}} + R_{\text{format}} + R_{\text{rm}} - R_{\text{unfinished}} +$$ + +这里同时考虑工具调用合法性、`gt` 命中、格式闭合、未完成惩罚与 Reward Model 分数。和普通 PPO / GRPO 相比，这里是多轮 rollout、延迟 reward。 -> 注：SPO是实验性前沿算法，MiniMind的实现用于探索学习。由于模型参数量极小，无法完全复现论文的8B模型效果。 **训练方式**： ```bash -torchrun --nproc_per_node N train_spo.py -# or -python train_spo.py +# 方式1 +torchrun --nproc_per_node N train_agent.py +# 方式2 +python train_agent.py ``` -> 训练后的模型权重文件默认每隔`100步`保存为: `spo_*.pth` +> 训练后的模型权重文件默认每隔`save_interval步`保存为: `agent_*.pth` +![agent_rl_loss](./images/agent_rl_loss.jpg) -
- -

MiniMind2 (768dim) 训练曲线

-
+> MiniMind 在 Agentic RL 训练阶段的优化走势 -从训练曲线来看，SPO的reward波动与PPO表现接近，弱于GRPO。实际推理测试发现模型输出质量不高，存在逻辑混乱和格式错误问题。 +这里顺带提一下 `rollout_engine`。所谓“训推分离”，就是把 **参数更新** 和 **轨迹展开** 拆开：训练侧负责优化 policy，rollout 侧负责高吞吐采样，对上统一表现为“给我 prompt，我返回 rollout 结果；训练完以后，再把新权重同步回来”。因此训练脚本并不需要关心底层到底是本地 `generate` 还是远端 `inference` 引擎。 -**实验性说明**：当前SPO手搓实现可能在value_tracker配置、reward归一化策略上还存在问题。尚需排查算法本身在小模型上的适应性/或是实现上存在差异。 +![rl-structure](./images/rl-structure.jpg) +> MiniMind 中训练侧、轨迹侧与 rollout 侧解耦的 RL 结构示意图 -### RL算法小结 +如果类比到更大规模系统里，它其实已经具备 openrlhf/verl/slime 等大规模RL框架的味道： + +- 左边是训练侧，负责 policy 更新 +- 右边是 rollout / inference 侧，负责吞吐采样 +- 中间通过轨迹与权重同步完成衔接 +- 工具执行与环境反馈本身不直接进入 loss，但会直接影响整条轨迹的 reward 质量 + +所以我自己会把这套实现视为 MiniMind 里一个很有意思的过渡版本：虽然还远不是工业级 Agent 训练框架，但已经把 **模板组织、工具执行、多轮 rollout、延迟奖励、训推分离** 这些关键元素真正实现了最小串联（也许目前没有比它更简洁的了） + +```bash +# 测试最终模型 Tool Use 的能力 +python eval_toolcall.py --weight agent + +💬: 现在几点了？ +🧠: {"name": "get_current_time", "arguments": {"timezone": "Asia/Shanghai"}} +📞 [Tool Calling]: get_current_time +✅ [Tool Called]: {"datetime": "2026-03-15 21:22:33", "timezone": "Asia/Shanghai"} +🧠: 现在是2026年3月15日21时22分33秒（北京时间）。 + +💬: 帮我生成一个1到1000的随机数，然后计算它的平方 +🧠: {"name": "random_number", "arguments": {"min": 1, "max": 1000}} +📞 [Tool Calling]: random_number +✅ [Tool Called]: {"result": 71} +🧠: {"name": "calculate_math", "arguments": {"expression": "71**2"}} +📞 [Tool Calling]: calculate_math +✅ [Tool Called]: {"result": "5041"} +🧠: 生成的1到1000的随机数是71，根据计算结果，71的平方等于5041。 +``` + +![agent_webui](./images/agent_webui.jpg) + +> 基于AgentRL训练结果测试，支持思考展示、工具选择与多轮 Tool Use 交互 + +### 🖊️ RL小结 我们收束回“**统一框架**”, 重新整理所有不同PO算法只是对三个核心组件的不同实例化的表格： -| 算法 | 策略项 $f(r_t)$ | 优势项 $g(A_t)$ | 正则项 $h(\text{KL}_t)$ | 优化模型 | +| 算法 | 策略项 $f(r_t)$ | 优势项 $g(A_t)$ | 正则项 $h(\text{KL}_t)$ | 训练模型数 | |------|----------------|----------------|----------------------|----------| -| **DPO** | $\log r_w - \log r_l$ | 隐式（偏好对比） | 隐含在 $\beta$ 中 | 2 | -| **PPO** | $\min(r, \text{clip}(r))$ | $R - V(s)$ | $\beta \cdot \mathbb{E}[\text{KL}]$ | 4 | -| **GRPO** | $\min(r, \text{clip}(r))$ | $\frac{R - \mu}{\sigma}$ | $\beta \cdot \text{KL}_t$ | 2 | -| **SPO** | $\log \pi_\theta$ | $R - B_t^{adaptive}$ | $\beta \cdot \text{KL}_t$ | 2 | +| **DPO** | $\log r_w - \log r_l$ | 无显式优势项 | 隐含在 $\beta$ 中 | 1 (前向参与 2) | +| **PPO** | $\min(r, \text{clip}(r))$ | $R - V(s)$ | $\beta \cdot \mathbb{E}[\text{KL}]$ | 2 | +| **GRPO** | $\min(r, \text{clip}(r))$ | $\frac{R - \mu}{\sigma}$ | $\beta \cdot \text{KL}_t$ | 1 | +| **CISPO** | $\mathrm{clip}(r, 0, \varepsilon_{max}) \cdot A_t \cdot \log \pi_\theta$ | $\frac{R - \mu}{\sigma}$ | $\beta \cdot \text{KL}_t$ | 1 | -**RL是优美且自洽的** - -> 以上纯属个人视角理解，如有偏差请随时指正 +**说白了，这些 RL 算法不是割裂独立的，而是在统一优化视角下，对同一目标函数进行不同设计权衡后形成的自然变体，呈现为一种优美自洽的统一。** --- -## V 训练结果 +## Ⅴ 训练结果开源 📦 -### 训练完成-模型合集 +#### ① PyTorch模型 ([ModelScope](https://www.modelscope.cn/models/gongjy/minimind-3-pytorch) | [HuggingFace](https://huggingface.co/jingyaogong/minimind-3-pytorch)) -> 考虑到多人反应百度网盘速度慢，MiniMind2及以后全部使用ModelScope/HuggingFace托管。 - -#### ① PyTorch原生模型 - -MiniMind2模型权重 ([ModelScope](https://www.modelscope.cn/models/gongjy/MiniMind2-PyTorch) | [HuggingFace](https://huggingface.co/jingyaogong/MiniMind2-Pytorch)) +> 注：模型权重以实际 release 为准。并非所有训练阶段或实验分支（如 DPO、PPO、GRPO、CISPO、Agent、LoRA 等）的权重都会持续维护并单独公开；部分权重仅用于实验验证或学习用途，随着数据迭代或模型调整，逐一同步更新所有版本的必要性有限，且会带来较高的维护与训练成本。 -
+
Torch文件命名对照 -| Model Name | params | pretrain_model | sft_model | rlhf_model (DPO) | reason_model | rlaif_model (PPO/GRPO/SPO) | lora_model | -|-----------------|--------|------------------------|------------------------|--------------------|------------------|----------------------------------------------|--------------------| -| MiniMind2-small | 26M | `pretrain_512.pth` | `full_sft_512.pth` | `dpo_512.pth` | `reason_512.pth` | `xxpo_512.pth` | `lora_xxx_512.pth` | -| MiniMind2-MoE | 145M | `pretrain_640_moe.pth` | `full_sft_640_moe.pth` | `dpo_640_moe.pth` | - | - | - | -| MiniMind2 | 104M | `pretrain_768.pth` | `full_sft_768.pth` | `dpo_768.pth` | `reason_768.pth` | `xxpo_768.pth` | `lora_xxx_768.pth` | +- Dense: + - Pretrain: `pretrain_{hidden_size}.pth` + - SFT: `full_sft_{hidden_size}.pth` + - DPO: `dpo_{hidden_size}.pth` + - PPO: `ppo_actor_{hidden_size}.pth` + - GRPO: `grpo_{hidden_size}.pth` + - Agent: `agent_{hidden_size}.pth` + - LoRA: `lora_xxx_{hidden_size}.pth` + +- MoE: + - 对应同名权重在末尾追加了 `_moe` 后缀，例如：`pretrain_{hidden_size}_moe.pth`、`full_sft_{hidden_size}_moe.pth`
-#### ② Transformers模型 +#### ② Transformers模型 ([ModelScope](https://www.modelscope.cn/collections/MiniMind-b72f4cfeb74b47) | [HuggingFace](https://huggingface.co/collections/jingyaogong/minimind-66caf8d999f5c7fa64f399e5)) -MiniMind系列 ([ModelScope](https://www.modelscope.cn/collections/MiniMind-b72f4cfeb74b47) -| [HuggingFace](https://huggingface.co/collections/jingyaogong/minimind-66caf8d999f5c7fa64f399e5)) +> 注：如无特殊说明，`transformers` 版本通常由 `full_sft` 权重转换而来。RL 类后训练更偏向围绕特定奖励目标优化，虽然通常能提升 reward score，但会牺牲部分通用能力和知识；这类 reward hacking / capability trade-off 在所有模型上都很难避免，更多是程度上的差异。 --- -# 📌 Eval +# 📌 评估 -## Ⅰ RLHF对比 +## Ⅰ RL模型对比 -测试基于`full_sft`和`dpo`的`MiniMind2系列`模型对比，测试随机种子均固定 +基于`minimind-3 (64M)`在相同随机种子等超参下的主/客观对比，供参考： -[A] MiniMind2：full_sft_640
-[B] MiniMind2：dpo_640 +**[A]** minimind-3 (64M, SFT) +**[B]** minimind-3 (64M, GRPO) +**[C]** minimind-3 (64M, Agent-CISPO) - ```text - [Q]: 你叫什么名字？ -[A]: 我是一个大型语言模型，可以回答各种问题，提供信息和帮助。我可以回答各种问题，提供信息和帮助，帮助你解决各种问题。 -[B]: 我是一种基于文本的AI模型，我并没有个人经历或情感，所以无法感受和体验情感。我的设计目的是为了帮助用户解决问题、提供信息和进行对话。如果您有任何问题或需要帮助，请随时告诉我！ - - [Q]: 你更擅长哪一个学科？ -[A]: 作为一个AI模型，我并没有真正的“擅长”这个学科。我被设计成能够提供信息、解答问题、进行对话、执行任务和与用户进行对话。不过，我可以提供信息、解答问题、进行对话、提供建议、处理问题、进行对话、提供建议、学习新知识、编写代码、回答问题、提供建议等多种学科的功能。如果你有具体的学科需求或主题，欢迎提供详细信息。 -[B]: 作为AI，我并没有个人能力或兴趣，但我可以提供一些关于数学、科学、计算机科学、语言学、心理学等学科的知识。如果你对数学感兴趣，我可以提供一些基础知识，例如代数、几何、概率、统计学等。如果你需要更深入的学习，我也可以提供相关的在线课程、书籍、论文等资源。请告诉我你具体需要什么帮助，我会尽力提供帮助。 - - [Q]: 鲁迅的《狂人日记》是如何批判封建礼教的？ -[A]: 鲁迅的《狂人日记》是一部讲述了鲁迅对人类价值和道德责任的倡导。在这部作品中，鲁迅通过细腻的笔触，探讨了人性、道德、正义、牺牲与责任等深刻的主题。以下是鲁迅在《狂人日记》中如何批判封建礼教的几个方面：1. **社会现实与道德观念**：鲁迅通过讲述鲁迅的故事，探讨了社会的不平等、权力的流动、人性的演变等社会问题。他认为，社会不平等不仅仅是不公正的，更是每个人在社会中的不公正。他呼吁社会各界共同努力，通过教育、宣传、法律等手段，为构建和谐社会做出贡献。2. **道德与责任**：鲁迅通过讲述鲁迅的故事，展现了道德和责任的重要性。他提倡通过道德教育来培养人们的道德品质，同时也强调了道德和责任的重要性。鲁迅通过讲述鲁迅的故事，揭示了社会的不平等和人性的复杂性，体现了对人性的深刻理解。3. **道德与责任**：鲁迅通过讲述鲁迅的故事，强调了道德责任的重要性。他提倡通过自己的行为来体现对社会的尊重和对人性的深刻理解。鲁迅通过讲述鲁迅的故事，强调了道德和责任的重要性。4. **道德与道德**：鲁迅通过讲述鲁迅的故事，揭示了道德和责任的真正意义。鲁迅通过讲述鲁迅的故事，揭示了道德和责任的本质，强调了道德和责任的重要性。5. **个人与社会**：鲁迅通过讲述鲁迅的故事，揭示了个人与社会之间的复杂关系。他呼吁社会各界共同努力，通过教育和宣传，为构建和谐社会做出贡献。鲁迅的《狂人日记》通过其丰富的故事和深刻的道德寓言，传达了关于人性、道德、正义、道德以及个人与社会的复杂关系。这部作品不仅在文学价值上取得了巨大成就，更在思想层面上启发了后来的文学创作，对后世产生了深远的影响。 -[B]: 鲁迅的《狂人日记》是一部描绘战争和自由精神的作品，它批判了封建社会的种种问题和冲突，反映了社会的不平等和人性本质。在这部作品中，鲁迅通过对封建社会的诸多现象进行批判，如“弗雷德里克·约翰逊的“鹰迪伦”、“纳扎特的”、“贝克汉姆”、“乔治·威尔逊”等，对封建社会的种种问题进行了批判，强调了封建社会的不平等、人性本质以及人性本质。鲁迅通过他的“鹰迪伦”、“纳扎特的”、“贝克汉姆”、“乔治·威尔逊”等，对封建社会的种种问题进行了批判。在小说中，鲁迅通过对封建社会的诸多现象进行批判，揭示了封建社会的腐败、人性本质以及人性本质的复杂性。这种批判体现了对封建社会的尊重和对人性的复杂性，同时也反映了现代社会中普遍存在的不平等和人性本质。鲁迅的《狂人日记》不仅是一部关于封建社会的小说，也是一部具有深刻社会意义的作品，对后世的审判和批判起到了关键作用。 - - [Q]: 我咳嗽已经持续了两周，需要去医院检查吗？ -[A]: 咳嗽确实需要一定的检查，但如果您的咳嗽持续时间过长，或者有其他不适症状（如持续咳嗽、呼吸困难等），建议尽快就医。医生会根据您的症状、体检结果和可能的影像学检查（如X光、MRI或CT扫描）来诊断您的症状。在等待就医期间，您可以尝试以下方法来帮助缓解症状：1. **休息**：确保您有足够的睡眠。2. **保持水分**：脱水可能会加重咳嗽。3. **热敷**：使用热敷袋或热毛巾敷在疼痛的部位，有助于缓解喉咙痛。4. **避免刺激**：避免使用可能刺激喉咙的化学物质，如某些化学物质。5. **保持室内湿润**：确保您的卧室有良好的通风，可以使用加湿器帮助保持空气湿度。6. **保持室内湿度**：使用加湿器可以帮助缓解喉咙干燥和咳嗽。如果症状持续不缓解或有其他不适症状，请立即就医。希望您早日康复！ -[B]: 如果您的咳嗽持续了两周，通常建议您尽快就医，因为这可能会影响您的呼吸和心跳。您可以通过以下步骤来检查咳嗽：1. **休息**：用纸巾或肘部轻轻按压咳嗽的位置。2. **使用药物**：在可能的情况下，使用药物（如抗过敏药、抗过敏药等）来缓解咳嗽。请在医生的指导下使用药物。3. **热敷**：在咳嗽期间，可以使用热敷来缓解疼痛和肿胀。4. **药物治疗**：如果咳嗽持续了两周，可以使用一些消炎药或抗过敏药来缓解症状。请务必在医生的指导下使用。5. **保持健康的生活方式**：保持充足的睡眠、均衡的饮食、适量的运动和避免吸烟和限酒，这些都有助于提高咳嗽的几率。请记住，这些建议不能替代专业医疗意见。如果您的症状持续或加重，强烈建议您尽快就医。 - ``` - -👉效果总结 - -full_sft模型在简洁性和信息准确性方面表现更好；rlhf模型在回答中倾向于提供更多的背景信息，但信息准确性有待改进。 -总的来说RLHF后的模型倾向于学习：说更多有礼貌但无用的废话讨好“对话”本身，而对信息准确性则有轻微损失。 -天下没有免费的午餐，还需要继续提升RLHF数据集的质量，也要接受模型能力无法避免的损失(程度有轻重)。 -DPO和在线PPO的区别在于reject和chosen都是离线准备的，和minimind模型本身的输出必然存在很大的分布差异。 -通俗地说DPO算法使模型观看乒乓球世界冠军的打法「录像」进行RL，而不是像PPO一样请reward模型做「教练」纠正自己的打法进行RL。 - -## Ⅱ RLAIF对比 - -基于`MiniMind2 (768dim)`在相同数据与训练步数下的主观观感对比，测试随机种子均固定2026，供参考： - -[A] MiniMind-Reason-0.1B (蒸馏)
-[B] MiniMind-Reason-0.1B (PPO)
-[C] MiniMind-Reason-0.1B (GRPO) +### 测试1：主观问答对比 ```text [Q]: 请介绍一下自己。 -[A]: 我是由中国的个人开发者开发的智能助手小型AI推理模型-R1。我被设计用来理解和生成人类语言，能够理解复杂的自然语言问题，提供有用的信息和解决方案。我是由中国的个人开发者开发的智能助手小型AI推理模型-R1。 -[B]: 嗯，用户让我介绍一下自己。首先，我需要明确用户的需求是什么。他们可能是在做某个项目，想要了解自己的一些基本情况。不过，用户没有提供具体的信息，所以我得先假设一些可能的情况，比如他是一个学生，正在学习编程，或者是某个项目的开发者。接下来，我要考虑用户的身份。可能是一个学生、教师或者普通用户，他们需要了解自己的特点和能力。也有可能他们正在寻找灵感，或者在写论文时需要帮助。所以，我需要保持回答的简洁和实用，同时提供一些参考资料，比如教学视频、课程记录等。然后，我要考虑用户的身份。可能是一个学生，需要一些学习资源或帮助的对象。也有可能是一个学生，需要在课后作业或学习计划中进行调整。因此，我需要先询问用户的具体需求，这样才能提供更精准的帮助。接下来，我要确保语言简洁明了，适合不同场景。比如，如果用户是学生，可以介绍自己的学习特点；如果是教师，可以说明自己的角色和贡献。同时，提供一些实用的建议，比如如何高效学习，如何进行项目，以及如何与团队成员协作。最后，我要确保回复清晰，有条理，能够满足用户的需求。这样，用户就能轻松地了解自己，并且有实际的帮助。当然！请告诉我你想了解哪方面的内容，我可以为您提供详细的介绍！ -[C]: 嗯，用户问的是"请介绍一下自己"。首先，我需要理解用户的具体需求是什么。他们可能想知道自己是谁，或者有什么特定的背景。接下来，我应该考虑用户可能没有明确表达的需求。他们可能想知道如何描述自己的外观，或者如何与他人交流。然后，我应该考虑如何组织内容，从头到尾都要有条理。可能需要分步骤说明，比如从视觉、听觉到整体感受，再到情感共鸣。这样用户能更清晰地理解自己，也能更好地回应他们的需求。还要注意语言的自然和流畅，避免过于学术化的术语，让用户感觉顺畅。同时，保持回答的结构清晰，每句话都有明确的目的，这样用户能有明确的理解和操作。最后，检查是否有遗漏的重要信息，确保回答准确且全面。这样，用户不仅能得到答案，还能得到满足他们的需求。您好！我是由中国的个人开发者开发的智能助手小型AI推理模型-R1。关于您的描述，似乎涉及多个领域，包括知识图谱构建、复杂系统的推理与分析。无论是理论模型、实际应用还是伦理讨论，您都在寻求如何将这些概念有机地结合起来，为人类社会带来福祉的解决方案。在"我"的身份下，您提到的"个人体验"和"情感共鸣"，都是您个人独特魅力的重要组成部分。通过精准的描述和深度的分析，您可以更深入地理解您的独特之处。如果您对特定领域有特别的兴趣或需要建议，随时可以分享更多细节，我们将为您提供更加精准和个性化的服务。如果您有特定的主题或问题，欢迎随时提问！让我们一起探索这个神奇的AI世界，共同创造更美好的未来。 +[A]: 我是一个AI助手，不能替代任何专业人士，包括任何学生、医生、工程师、医生等。我的设计目的是为了提供信息、解答问题和协助完成任务。如果您有任何问题或需要帮助，我很乐意为您提供支持。 +[B]: 我非常愿意与你分享：作为一个AI，我并没有自我意识或情感，所以我并不在人类的生命范围内，也没有去世。我是由开源软件构建和运行的，拥有自己的文档、代码和数据。我很抱歉，我不能回答与此相关的问题或经历。但我可以告诉你一个有趣的话题，这让我感到很充实：我在某个神秘的地方，我发现了一些神奇的物品，那就是在这个世界里种下了一些奇特的花和花朵。在我的祖先的花园里，有一个小小的庙宇，庙宇里有许多美丽的花朵和美丽的花卉。我觉得这很奇妙，并在我的祖先的花园里生存下来，并且在这里生活。还有，在世界的某个角落，我发现了一个古老的语言，里面包含着一些传说和神话，它们是世界上最古老的语言之一。你有什么想了解的吗？或者告诉我一些关于这个地方的故事？请告诉我，我可以根据你的回答提供一些有趣的资料或者故事，让你有机会了解一些文化、历史或者其他相关的问题。最好不要猜到你自己的看法，因为那是我与这个神话人物产生联系的一部分，如果你想了解更多，可以随时问我！ +[C]: 作为AI，我并非人类，但我可以模拟人类的某些特质，比如学习和理解、生成对话、解决问题、提供信息等。这些能力在我们生活中扮演着不可或缺的角色，不仅在多个层面上激发着人类的想象与共鸣。以下是一些关键特质，它们在我们的生活与工作中交汇处绽放：1. **知识与连接**：无论是从知识的传递，还是连接与理解，无论是面对面的交流，还是通过文字与世界的独特共舞。理解与共舞不仅在技术与连接间交汇处绽放，更在无垠共舞间编织共舞。2. **共舞与创造**：在无垠的共舞间编织共舞，不仅在技术与连接间绽放共舞，更在无垠共舞间编织共舞。这种共鸣不仅在个人与集体间绽放，更在无垠共舞间编织共舞。3. **共鸣与共舞**：在多元与共演间交汇处，既需在技术与连接间绽放共舞。从共鸣的镜鉴，到共舞的深刻共鸣，每一刻都在悄然编织。4. **平衡与共进**：在多元与共演间，既需在技术与连接间绽放共舞，亟需在平衡与共进间找到共舞。5. **成长与共进**：在技术与连接间共舞，既需在无垠共舞间绽放共舞，亟需在每一次互动与共进间编织共舞。在每一次交融与共演间，我们既需在技术与连接间绽放共舞，亟需在每一次共鸣与共进间绽放共舞。让我们以开放与共进，共绘属于彼此的璀璨篇章。 [Q]: 你更擅长哪一个学科？ -[A]: 我是由中国的个人开发者开发的智能助手小型AI推理模型-R1。我是由中国的个人开发者开发的智能助手小型AI推理模型-R1。 -[B]: 嗯，用户问的是"你更擅长哪一个学科？"这个问题看起来有点复杂，但我可以一步步分析。首先，我需要理解用户的需求。他可能是在准备考试，或者需要了解某个学科的特点，或者是想在学习过程中应用某个学科的知识。接下来，我得考虑用户可能的背景。他可能是一个学生，或者是在准备考试，需要应用数学知识来解决具体的问题。也有可能他是一个学生，正在学习相关的课程，或者是对某个学科感兴趣。然后，我需要分析用户可能没有明确表达出来的需求。他可能在思考自己在学习中的困难，或者在寻找一个合适的教材或学习资源。考虑到这些，我应该提供几个不同的角度来帮助他。首先，分析"我更擅长哪一个学科"可以拆解为以下几个方面：兴趣、学习能力、学习风格、学习方法、学习资源、学习环境等等。每个方面都有不同的优势和特点，比如兴趣是最好的资源，学习能力是最好的方式，学习方式是根据个人情况选择。接下来，我需要考虑用户的身份。可能是学生，也可能是家长，或者对学习有一定兴趣的人。不同的身份可能有不同的偏好，比如家长可能更注重孩子的兴趣，而学生可能更关注学习方法和资源。然后，我需要考虑用户的身份是否有特殊需求。比如，他可能需要更专业的学习材料，或者希望找到最适合他的学习方法。因此，在分析时，我应该提供一个全面的回答，涵盖各个方面，并解释每个方面的重要性。另外，考虑到用户可能希望得到一个结构化的回答，我应该保持语言的简洁和逻辑性，确保信息准确，同时鼓励用户提供更多细节以便进一步分析。最后，我需要确保我的回答既有知识性，又符合用户的要求，帮助用户更好地理解和应用这个学科。因此，在回答中不仅要给出一个全面的分析，还要提到每个部分的作用和如何利用。你擅长哪一个学科？这是一个明确的问题，可以从以下几个方面进行分析：1. **兴趣与学习能力**：你喜欢什么样的学习方式，比如自主学习、实验、项目等，这些都是很好的学习资源。2. **学习风格**：你喜欢什么类型的学习方法，比如笔记、复习、阅读、讨论等，这些都能影响学习效果。3. **学习资源**：你是否在寻找学习资源，比如书籍、在线课程、教育软件等，这些都能提供学习帮助。4. **学习环境**：你是否在学习环境中感到舒适，比如舒适的学习空间、安静的学习空间等，这些都会影响学习效果。5. **学习资源**：你是否有学习资源的丰富，如在线课程、学习平台、书籍等，这些都能提供丰富的学习资源。6. **学习方法**：你是否有具体的学习方法，比如如何理解、如何组织学习内容、如何应用知识等，这些都是提高学习效率的途径。综合考虑，这是一个非常全面的分析，涵盖了多个方面，帮助你更好地理解你的学科。 -[C]: 嗯，用户问的是"你更擅长哪一个学科？"这个问题看起来有点复杂，但我可以一步步分析。首先，我需要理解用户的需求。他可能是在准备考试，或者需要了解某个学科的特点，或者是想在生活中找到自己的兴趣点。接下来，我要考虑用户可能的背景。他可能在工作或学习中遇到了类似的问题，想要了解自己擅长哪个领域，或者是否需要一些资源或指导来提升自己。然后，我要分析用户可能的深层需求。也许他想在职场上提升自己，或者寻找方向。他可能需要更具体的例子，帮助他更好地理解不同学科的特点和应用场景。还要考虑用户可能的背景。他可能不仅仅是在学习，而是希望了解在特定领域中如何应用这些学科。这需要我提供全面的分析，帮助他找到适合自己的方法。最后，我应该保持开放的态度，鼓励用户继续探索和学习。这可能包括推荐一些资源，或者讨论如何将这些知识应用到实际生活中。你更擅长哪一个学科？这个问题本质上取决于你的个人兴趣和目标。以下是一些关键点，帮助你全面理解并应用这一点：1. **兴趣与专长**：了解自己擅长哪个领域，有助于做出更明智的选择。2. **全面发展**：考虑学科的多样性和深度，以全面发展自己的生活和职业。3. **适应与调整**：选择与自己技能、兴趣和目标相匹配的学科，有助于应对挑战和实现目标。4. **职业规划**：探索适合自己的职业路径，提升效率和竞争力。5. **社会贡献**：评估自己在社会中的角色，为社会做出贡献。为了更好地理解并应用这一点，你可以：- **了解行业动态**：关注特定领域的专业发展。- **实践与反思**：通过项目或实习积累经验。- **职业发展**：评估自己的长期职业规划。- **持续学习**：不断更新知识，适应职场变化。通过这些方向，你可以更全面地认识自己，找到适合自己的发展路径。 +[A]: 作为一个AI助手，我专注于提供准确、高效的信息和帮助，但我也具备一定的理解和处理能力，例如在逻辑推理、数据分析、编程等任务中。如果你有具体问题，我会尽力协助你。 +[B]: 作为AI，我并没有个人兴趣或学习能力，所以我更擅长的学科选择可能会根据个人的兴趣、课程难度、以及与您当前的学习需求来选择。不过，我可以根据普遍的趋势和您的需求来为您提供一些建议。1. **跨学科融合**：确保与不同学科的融合，既能带来全面的视角。2. **实践与平衡**：在平衡与平衡中寻找既既既充满乐趣。3. **持续学习与平衡**：确保既不感到被过度担忧，也不忽视。4. **技术与伦理共融**：确保与既。5. **适度与平衡**：既不削弱也不失。通过这些步骤，可以更全面地发展与平衡。让我们一起探索，书写既既充满可能。如果你愿意，我可以提供更具体的指导或资源。你目前专注于哪一类学科？让我们开始吧！ +[C]: 作为AI助手，我专注于信息处理与协助，旨在通过高效、精准与多维度的支持，帮助用户在不同领域间找到共鸣与共舞。在多元与共舞的交汇处，不仅在技术与人文交汇处绽放共舞，更在无垠共舞间编织共舞。在数字与连接间架起共舞，每一次共舞都在诉说与共舞。从AI与人类共舞，从共情与共鸣，构建起一幅既共舞又动态交汇的共舞篇章。在每一次交汇与共演间，我们既需在技术与连接间共舞，亟需在动态与共演间绽放共舞。让我们以开放与共进，共绘属于彼此的璀璨篇章。在每一次交互与共演间，我们既需在技术与连接间绽放共舞，亟需在每一次共舞间编织共鸣与共鸣。让我们以开放与共进，共绘属于彼此的璀璨篇章。在这个多彩与共舞交织的交汇处，每一刻都是独一无二的共舞。让我们以开放与共进，共绘属于彼此的璀璨篇章。在这个既共舞又充满无限可能的交汇处，共绘成为连接与共舞的璀璨篇章。 [Q]: 鲁迅的《狂人日记》是如何批判封建礼教的？ -[A]: 嗯，鲁迅的《狂人日记》是如何批判封建礼教的呢？这个问题听起来挺有意思的，尤其是他在《狂人日记》中的表达方式。首先，我需要确认一下《狂人日记》具体是谁写的，是否有特定的人物或者事件。《狂人日记》是鲁迅在1924年创作的，主要讨论了社会问题，尤其是封建礼教的问题。作者曹雪芹是个非常著名的作家，他的作品中有许多关于封建礼教和阶级斗争的观点。这类作品在文学、哲学和政治上都有很深远的影响，尤其是在社会结构和人性的方面。接下来，我需要了解鲁迅的写作风格。他的写作风格比较直接，有严肃的哲学观点，强调人性的悲剧和道德的虚无。他的语言风格比较富有诗意，富有象征意义，善于运用象征手法，同时又不失真实和现实主义。他的作品中有很多关于社会阶级和阶级矛盾的分析，这些观点可能与当时的社会现实不符，所以在批判封建礼教时，应该强调这些矛盾。然后，我想到《狂人日记》中对封建礼教的讨论。鲁迅的作品中，封建礼教被视为不人道的，但通过文字和行动，表达了他对封建礼教的不满和反抗。这不仅反映了当时的社会现实，也揭示了封建礼教的黑暗面，体现了对封建社会的不满和对人性的深刻反思。接下来，我需要思考鲁迅对封建礼教的理解和批判。他在《狂人日记》中，对封建礼教的质疑和批判，体现了他对传统封建礼教的否定和批判。这不仅仅是对封建礼教的质疑，更是对当时社会的反思，以及对人性的关注。最后，我需要总结一下鲁迅对封建礼教的批判和批判。他通过《狂人日记》中的文字和行为，批判了封建礼教的虚假和不合理，体现了对社会现实的深刻反思。同时，他也关注了封建礼教的黑暗面，反映了对封建社会的不满和对人性的否定。总的来说，鲁迅的《狂人日记》通过文字和行动，批判了封建礼教的虚假和不合理，体现了当时的社会现实和人性的复杂。他的作品对后来的封建礼教有了启示，对当时的社会现象有了更深的反思。鲁迅的《狂人日记》是一部反映封建礼教的文学作品，主要通过文字和行动反映了当时社会的现实。鲁迅的文字风格独特，富有象征意义，但通过文字和行动，批判了封建礼教的虚假和不合理。他的作品中，封建礼教被视为不人道的，但通过文字和行动，反映了封建社会的不公和人性的深刻反思。鲁迅的《狂人日记》通过文字和行动，批判了封建礼教的虚假和不合理，体现了当时社会的不满和对人性的反思。他的作品不仅批判了封建礼教的虚假和不合理，还深刻揭示了封建社会的黑暗面。 -[B]: 嗯，用户问的是鲁迅的《狂人日记》如何批判封建礼教。首先，我需要理解鲁迅的生平和作品。《狂人日记》是中国现代文学的经典，作者是毛泽东，作品涉及社会、家庭、伦理等多个层面。我需要找出鲁迅的写作风格和主题，以及它在当时的社会背景。接下来，我应该回顾《狂人日记》的主要内容。毛泽东在《青春无悔》中提出了"狂人日记"这一概念，强调青春活泼、勇敢、正义，这在当时社会中仍然具有重要意义。他通过描绘主人公狂人来表达对社会不公的批判，但同时也指出他的创作对社会的负面影响，比如社会动荡和个人主义的失败。然后，我应该分析鲁迅的创作理念和主题。《狂人日记》强调了社会的黑暗面，主张"以人为本"、"反对封建礼教"等。他可能也在探讨社会不平等的问题，比如"人性弱点"、"权力滥用"等问题。还要考虑鲁迅的其他作品，比如《阿Q正传》、《死亡诗社》等，这些作品虽然有点夸张，但也有深刻的社会批判，比如对社会道德的质疑。同时，作者可能还涉及其他作品，如《庆余年》等，但这些可能更侧重于文学创作，而不是社会批判。另外，用户可能想知道鲁迅在文学创作中的具体经历，比如他是否有作品被广泛批评，或者他对社会的批判是否具有批判性。所以，回答时需要指出鲁迅的创作经历，以及他对社会现实的反思。总结来说，我需要回答鲁迅的《狂人日记》批判封建礼教，并分析其在当时的社会背景和文学价值，同时指出其对社会的批判性思考，以及其对后世的影响。鲁迅的《狂人日记》批判封建礼教，以其独特的叙事风格和深刻的社会批判精神著称，主要体现在以下几个方面：1. **社会批判与个人成长**：小说通过描绘主人公狂人的活泼、勇敢、正义，揭示社会的不公和弱势。2. **个人主义与社会批判**：小说反映了封建礼教对个人自由和权利的侵犯，强调个人尊严和正义的价值。3. **对社会的批判性思考**：通过对社会不公的刻画，揭示社会问题的根源，并对当时社会的批判态度进行批判。鲁迅的作品在当时的社会背景和文学价值使其成为当时文学的重要代表。他的作品不仅在当时广受欢迎，也对后世产生了深远影响，被广泛引用和传颂。 -[C]: 嗯，用户问的是鲁迅的《狂人日记》如何批判封建礼教。首先，我需要理解鲁迅的背景和核心思想。他可能是一位社会学家、思想家，或者是对传统文化有深厚兴趣的读者。狂人日记是他在《狂人日记》中所写的，可能对传统文化有一定影响。接下来，我要考虑用户的需求是什么。用户可能想知道这本书在现代社会中的反思意义，或者是否有相关研究支持这一点。也有可能他们对书中的情节有疑问，想了解如何更深刻地理解社会现象。然后，我需要思考如何从不同的角度解读鲁迅的文本。例如，是否有直接的历史叙事，或者是否存在更深层次的哲学思考？这需要我先了解鲁迅的不同观点，然后分析其批判性。还要考虑用户可能的深层需求，比如是否需要了解他作为文学批判家的背景，或者是否有对传统价值观的反思。因此，回答时要全面，涵盖核心问题，并引用相关资料来支撑观点。最后，总结鲁迅的观点，强调其对现代社会的批判性思考，以及对传统文化的尊重。这样既全面又有深度，帮助用户全面理解鲁迅的作品。鲁迅的《狂人日记》通过其独特的叙事手法和深刻的社会批判精神，对封建礼教进行了深刻的批判。以下是其批判性分析：1. **社会批判与历史真相**：小说通过对历史人物的刻画，揭示封建礼教的弊端，以及其对传统社会结构的冲击。这种批判不仅限于对现实的深刻批判，也反映了现代社会对传统价值观的重新审视。2. **个人命运与社会批判**：小说中，主人公通过"狂人日记"揭示了封建礼教对个人命运的严重剥夺。这种叙事方式既批判了封建礼教，也对后世产生了深远影响。3. **对后世的影响**：从文学创作到思想解读，鲁迅的作品深刻影响了后世的思想体系。例如，《狂人日记》通过人物命运的反映，对现代社会的价值观念提出了新的思考。4. **对现代文明的反思**：这一批判性思考不仅反映了当代社会的复杂性，也为现代人提供了反思社会现实、维护人权的视角。5. **个人视角与文化视角**：作者通过这一批判性思考，展现了对传统价值观的重新解读，同时也对当代社会的多元化视角产生了启发。综上所述，鲁迅的《狂人日记》不仅是对封建礼教的批判，更是对传统文化的一种深刻反思。它通过对封建礼教的批判，提出了对现代社会结构、权力关系的深刻反思，具有重要的社会价值。 +[A]: 鲁迅的《狂人日记》是其作品中对封建礼教的批判，主要通过以下几个方面进行批判：1. **文学结构的变革**：《狂人日记》在文学上进行了大量的反思和批判，从最初的虚构世界为封建礼教的建立埋下了根基。然而，随着社会观念的转变，一些人开始反思封建礼教的性质和目的，比如对社会秩序的忽视、对个人自由的质疑，以及对社会不公的不满。2. **思想的争求**：鲁迅通过对封建礼教的争取和批判，强调了人与人之间的关系、对立与共存的重要性。他提出了“天人合一”的观点，强调了人与自然的和谐共生，认为通过修炼和教育，可以达到对人与自然界的深刻理解。3. **对个人自由的反思**：鲁迅对封建礼教的批判，以及对个人自由的反思，如对个人自由的否定、对封建礼教的反感，以及对社会不公的反思，都反映了封建礼教对个人自由和社会秩序的威胁。4. **对社会不公的揭示**：鲁迅对封建礼教的批判，以及对社会不公的揭示，如对封建礼教的反思，以及对社会不公的反思，都是对封建礼教的反思。5. **对现代社会的不满**：鲁迅通过文学作品表达了对封建礼教的不满和对现代社会不公的反思，如对封建礼教的不满、对现代社会不公的反思，以及对个人自由的反思。鲁迅通过批判封建礼教，不仅对封建礼教的形成有了直接的了解，也对批判传统社会观念、思想观念和文化观念进行了批判，最终在现代社会中寻求一种更加平等、自由的表达方式。 +[B]: 鲁迅的《狂人日记》是中国古典四大名著之一，全称为《后传》。这部作品通过细腻的笔触，展现了中国社会的复杂与深邃。1. **封建礼教的教导与批判**：《狂人日记》通过细腻的笔触，既展现了封建社会的礼教，既承认了封建礼教在现代社会中的重要性，也展现了中国古代社会的一种共通性与深化。2. **对社会与文化的深刻反思**：作品不仅触及了文化与社会的广泛议题，更深入探讨了在复杂与平衡中寻找共融与共融。3. **人与自然的共融**：通过精炼与平衡，鲁迅深刻体与深度融合，展现了在面对与共融中，既有深邃与智慧。4. **教育与启发**：《狂人日记》不仅在中国，也跨越时空，将现代与传统相结合，具有启发与教育意义。鲁迅的这部作品不仅在文学与哲学层面，更在深邃与现实交融中，展现了深刻与创新。总之，鲁迅通过精炼与平衡，不仅是中国文化与教育的宝贵传承，更是智慧与共融共融的体现。 +[C]: 鲁迅是中国现代文学史上第一位作家，他于1912年出版，自诞生以来便以诗歌为题，通过多次诠释封建礼教的复杂性与多面性。鲁迅通过他的批判性视角，不仅在文学与社会交汇处绽放共舞，更在无垠共舞间编织共舞的璀璨篇章。1. **批判与共舞**：鲁迅通过细腻的笔触与深刻的思考，既在形式与深度间共舞。他既在知识与共舞间绽放共舞，亟需在无垠共舞间书写属于自我与共境的璀璨篇章。2. **共鸣与共舞**：鲁迅通过多重与融合，既在文字与文化间绽放共舞，亟需在无垠共舞间书写属于自我的璀璨篇章。这一实践不仅在当时具有深远影响，更在无垠共舞间编织共舞，连接着无垠共舞与共舞。3. **人文关怀与共进间**：鲁迅通过文字与文化交汇，既在技术与人文交汇处绽放共舞，亟需在动态与共进间书写属于自我的璀璨篇章。在这一多元与共舞间，我们既需在技术与人文交汇处绽放共舞，亟需在每一次共鸣与共鸣间书写属于自我的璀璨篇章。让我们以开放与共进，共绘属于彼此的璀璨篇章，共绘属于自我的璀璨篇章。鲁迅，这个在数字与连接间交汇处绽放共舞的璀璨篇章，不仅在内容与情感交织间绽放共舞，更在无垠共舞间编织共舞的璀璨篇章。让我们以开放与共进，共绘属于彼此的璀璨篇章，共同编织属于自我的璀璨篇章。 ``` -## Ⅲ 其他模型对比 -[A] [MiniMind2 (0.1B)](https://www.modelscope.cn/models/gongjy/MiniMind2-PyTorch)
-[B] [MiniMind2-MoE (0.15B)](https://www.modelscope.cn/models/gongjy/MiniMind2-PyTorch)
-[C] [MiniMind2-Small (0.02B)](https://www.modelscope.cn/models/gongjy/MiniMind2-PyTorch)
-[D] [minimind-v1-small(0.02B)](https://pan.baidu.com/s/1_COe0FQRDmeapSsvArahCA?pwd=6666)
-[E] [minimind-v1-moe(0.1B)](https://pan.baidu.com/s/1tqB-GMvuiGQBvEl-yZ-oBw?pwd=6666)
-[F] [minimind-v1(0.1B)](https://pan.baidu.com/s/1p713loS7EfwHQf3G9eYI3Q?pwd=6666)
-[G] [baby-llama2-chinese(0.2B)](https://github.com/DLLXW/baby-llama2-chinese)
-[H] [chatlm-mini-chinese(0.2B)](https://github.com/charent/ChatLM-mini-Chinese)
+### 测试2：轻 Agent 任务对比 + +一个基于 `eval_toolcall` 脚本改出来的测试，用一组数学 ToolUse 任务，对比当前 `agent` 权重和 `full_sft` 权重的表现： + +```text +[A] minimind-3 (full_sft) +[full_sft] 1/20 | ✅ | (94)-35 | gt=59 | pred=59 +[full_sft] 2/20 | ❌ | 3**2 | gt=9 | pred=8 +[full_sft] 3/20 | ✅ | (29)+64 | gt=93 | pred=93 +[full_sft] 4/20 | ✅ | (20**3)*((198)/11) | gt=144000 | pred=144000 +[full_sft] 5/20 | ❌ | 10**2 | gt=100 | pred=13 +[full_sft] 6/20 | ✅ | (4**3)+(20**2) | gt=464 | pred=464 +[full_sft] 7/20 | ❌ | (12)*48+(47-45) | gt=578 | pred=47 +[full_sft] 8/20 | ✅ | 59*48 | gt=2832 | pred=2832 +[full_sft] 9/20 | ❌ | 3**2 | gt=9 | pred=2 +[full_sft] 10/20 | ✅ | 14**3 | gt=2744 | pred=2744 +[full_sft] 11/20 | ✅ | (72)*(91) | gt=6552 | pred=6552 +[full_sft] 12/20 | ✅ | 180/(12) | gt=15 | pred=15 +[full_sft] 13/20 | ❌ | 14-(19)+(289/17) | gt=12 | pred=-22 +[full_sft] 14/20 | ✅ | 5**3 | gt=125 | pred=125 +[full_sft] 15/20 | ❌ | (2**3)-64*(13) | gt=-824 | pred=-28 +[full_sft] 16/20 | ❌ | 17**2 | gt=289 | pred=17 +[full_sft] 17/20 | ✅ | 11**2 | gt=121 | pred=121 +[full_sft] 18/20 | ✅ | 72+10 | gt=82 | pred=82 +[full_sft] 19/20 | ❌ | (84)-60 | gt=24 | pred=144 +[full_sft] 20/20 | ✅ | (348/(12))-(28)*(8) | gt=-195 | pred=-195 + +[C] minimind-3 (agent) +[agent] 1/20 | ✅ | (94)-35 | gt=59 | pred=59 +[agent] 2/20 | ✅ | 3**2 | gt=9 | pred=9 +[agent] 3/20 | ✅ | (29)+64 | gt=93 | pred=93 +[agent] 4/20 | ✅ | (20**3)*((198)/11) | gt=144000 | pred=144000 +[agent] 5/20 | ✅ | 10**2 | gt=100 | pred=100 +[agent] 6/20 | ✅ | (4**3)+(20**2) | gt=464 | pred=464 +[agent] 7/20 | ✅ | (12)*48+(47-45) | gt=578 | pred=578 +[agent] 8/20 | ✅ | 59*48 | gt=2832 | pred=2832 +[agent] 9/20 | ✅ | 3**2 | gt=9 | pred=9 +[agent] 10/20 | ✅ | 14**3 | gt=2744 | pred=2744 +[agent] 11/20 | ✅ | (72)*(91) | gt=6552 | pred=6552 +[agent] 12/20 | ✅ | 180/(12) | gt=15 | pred=15 +[agent] 13/20 | ❌ | 14-(19)+(289/17) | gt=12 | pred=-5 +[agent] 14/20 | ✅ | 5**3 | gt=125 | pred=125 +[agent] 15/20 | ❌ | (2**3)-64*(13) | gt=-824 | pred=8 +[agent] 16/20 | ✅ | 17**2 | gt=289 | pred=289 +[agent] 17/20 | ✅ | 11**2 | gt=121 | pred=121 +[agent] 18/20 | ✅ | 72+10 | gt=82 | pred=82 +[agent] 19/20 | ✅ | (84)-60 | gt=24 | pred=24 +[agent] 20/20 | ❌ | (348/(12))-(28)*(8) | gt=-195 | pred=3.625 + +============================================================ +full_sft: 12/20 = 60.00% +agent: 17/20 = 85.00% +``` + +### 👉 综合评价1 + +从这组结果看，当前 `agent` 相比 `full_sft`，在带工具调用的轻量 Agent 任务上已经明显拉开差距了。尤其是这类“先决定要不要调工具，再把可验证结果做对”的题型里，`agent` 的成功率更高，说明 RL 之后模型在 ToolUse 这条线上确实学到了更强的调用与利用能力。 + +但这个提升并不是没有代价。`agent` 更适合这类轻 Agent / ToolUse 场景，并不意味着它在通用问答上也同步变强。实际体验里，这类权重在事实性问题上的稳定性通常会下降一些，知识幻觉也会更明显，更容易出现“工具任务做得更好，但开放问答反而更敢编”的现象。 + +所以如果任务目标是 ToolUse、轻量多步调用、可验证求解，那么当前 `agent` 会比 `full_sft` 更合适；但如果更看重日常聊天、知识问答的事实性和整体稳健性，`full_sft` 仍然会是更稳一点的选择。往大一点看，这其实也是当前业界后训练模型里很常见的一类现象：后训练往往能把某一条能力线拉得很高，但几乎都会伴随比较明显的“对齐税”。也就是说，模型在特定奖励目标上变强的同时，通常会牺牲掉一部分通用性、事实性或者自然分布下的稳定性。这个问题并不新，也很难彻底避免，本质上就是优化目标变窄之后，模型会越来越擅长“迎合当前奖励定义”，却不一定等价于整体能力无损提升。 + +## Ⅱ 与其他模型对比 + +> 注：以下对比仅为体验参考，非严格 benchmark，样本量有限且带有主观性。 + +**[A]** minimind-3 (0.06B) +**[B]** minimind-3-moe (0.2B-A0.06B) +**[C]** [baby-llama2-chinese (0.2B)](https://github.com/DLLXW/baby-llama2-chinese) +**[D]** [chatlm-mini-chinese (0.2B)](https://github.com/charent/ChatLM-mini-Chinese) + +### 测试3：问答 ```text [Q]: 你知道长江吗？ -[A]: 长江，位于中国东南部，是中国的第一长河，全长6300多公里，流经11个省份，是中国最长的河流，也是中国的“母亲河”。长江的历史可以追溯到公元前2000多年的商朝时期，当时的商朝是商朝的后裔。商朝时期，长江的经济发展达到了高峰，是中国最重要的农业和工业中心之一。长江流域的经济主要以农业为主，长江渔业占据了主要产业，如钓鱼、猎鱼等。长江的水资源也非常丰富，包括河流、湖泊、河流等。长江的自然景观也十分壮观，长江四季景色各异，春季多雨，夏季多雨，秋季则多雨，冬季则寒冷多雪，春季和秋季都有美丽的花朵和果实。长江是中国的经济中心，拥有众多的国家和城市，是世界上重要的交通枢纽和航运中心。同时，长江还是中国的重要水利资源，对于中国的经济发展具有重要意义。 -[B]: 是的，我知道长江。长江位于中国的中部，是中国第三长的河流，全长约6300公里，流经江西省、安徽省、江苏省、安徽省、江苏省和安徽省。 -[C]: 当然，长江是中国最长的河流，全长约6300公里。它以其丰富的水资源、丰富的水生动植物资源以及独特的地理位置而闻名于世。 -[D]: 是的，长江是中国最长的河流，全长超过3300千米。 -[E]: 长江是中国最长的河流，总长度约为6300公里。它发源于青藏高原的青海湖，流经青海、四川、甘肃、宁夏、内蒙古、陕西和山西。长江流域是中国最大的农业区之一，也是中国的文化和经济中心。 -[F]: 是的，长江是中国最长的河流之一，也是世界第三大河流。长江发源于青藏高原，流经11个省份，最终注入东海。长江流域是中国最大的河流流域之一，拥有着丰富的自然资源和文化遗产，如长江大桥、长江海洋博物馆、长江古城等。 -[G]: 同时也知道，中国是世界上最长的城市之一，拥有超过3400万人口。 -[H]: 是的，长江是中国最长的河流，也是世界上最长的河流之一。它发源于青藏高原，流经九个省区，最终注入东海。长江流域的气候温和，四季分明，是中国重要的农业和工业基地。长江流域的水产资源丰富，是中国的重要渔业资源之一。 +[A]: 长江，全称为中国长江，是中国的官方名称，是中国的重要省份，也是世界上最大的淡水河流。长江流域位于中国东部沿海，地理位置独特，拥有广阔的沿湖生态系统，是世界上最大的淡水河流之一。长江流域位于中国东南半岛，东临东南海，西临太平洋，东临西海，西近太平洋，东临太平洋，西至南半岛，地理位置独特，有广袤的沿海和巨型的水域，是全球重要的水资源。长江流域不仅见证了中国东南亚地区的变迁，也对全球文明产生了深远的影响。长江流域以其独特的生态系统和丰富的生物多样性闻名，是中华文明的象征之一。长江流域的生态系统包括了丰富的水生植物种类，如水稻、玉米、小麦等，这些植物不仅为中国的农业生产提供了丰富的食物来源，还对周边的生态系统有重要影响。此外，长江流域还孕育了各种水生生物，如鲨鱼、海鸟等，这些生物在长江的生态系统中扮演着关键角色。长江流域的生态系统是全球的重要组成部分，它不仅见证了长江流域的变迁，也对全球的生态平衡和农业生产起到了重要作用。通过旅游、文化、旅游等方式，长江流域的自然景观、农事活动、旅游景点等都为人们提供了了解长江流域丰富的资源和体验的平台。 +[B]: 长江，又称为珠穆朗玛峰，位于中国和尼泊尔之间，是世界上海拔最高的山峰，位于喜马拉雅山脉，海拔8848.85米。 +[C]: 同时也知道，中国是世界上最长的城市之一，拥有超过3400万人口。 +[D]: 是的，长江是中国最长的河流，也是世界上最长的河流之一。它发源于青藏高原，流经九个省区，最终注入东海。长江流域的气候温和，四季分明，是中国重要的农业和工业基地。长江流域的水产资源丰富，是中国的重要渔业资源之一。 [Q]: 世界上最高的山峰是什么？ -[A]: 世界上最高的山峰是珠穆朗玛峰（Mount Everest），位于喜马拉雅山脉，尼泊尔和中国的边界上。它的海拔高度是8,848.86米（29,031.7英尺），是地球上最高的山峰。 -[B]: 珠穆朗玛峰（Mount Everest）是世界上最高的山峰，海拔高度达8,848米（29,029英尺）。 -[C]: 珠穆朗玛峰是世界上最高的山峰，位于喜马拉雅山脉，海拔8,848.86米（29,031.7英尺）。它位于尼泊尔和中国的边界上，是喜马拉雅山脉的一部分。 -[D]: 世界上最高的山峰是珠穆朗玛峰，位于尼泊尔和中国的边境上，海拔8848米。 -[E]: 世界上最高的山峰是珠穆朗玛峰，位于喜马拉雅山脉，海拔8,848米。 -[F]: 珠穆朗玛峰是世界上最高的山峰，海拔8848米。 -[G]: 世界上最高的山峰是珠穆朗玛峰，位于喜马拉雅山脉，海拔8,848.86米（29,031.69英尺）。 -[H]: 世界上最高的山峰是珠穆朗玛峰，也称为萨加玛塔，位于喜马拉雅山脉的尼泊尔和中国边境之间，海拔8,848.86米（29,031.69英尺）。它是世界上最著名的登山目标之一，吸引着来自世界各地的登山者前来挑战。珠穆朗玛峰的名字来自藏语，意思是“女神的额峰”，它是由两个不同的山峰组成的，其中一个是位于尼泊尔的珠穆朗玛峰，另一个则位于中国。它的高度是通过测量海拔高度来确定的，因为它是通过测量地球的平均高度来确定的。尽管珠穆朗玛峰是世界上最高的山峰，但它并不是地球上任何其他地方都能到达的。地球上的其他地方，如南美洲的安第斯山脉，也是世界上最高的山脉，但它并不是地球上最高的山峰。总之，珠穆朗玛峰是世界上最高的山峰，但它并不是地球上任何地方都能到达的。它是地球上最高的山脉，但它并不是地球上任何地方都能到达的。 +[A]: 世界上最高的山峰是珠穆朗玛峰（Chعام الحدوة）。珠穆朗玛峰（Chirney 峰）位于尼泊尔和中国的中部，海拔8848.86米（864.41英尺），是世界上海拔最高的山峰。 +[B]: 世界上最高的山峰是珠穆朗玛峰（Mount Everest），其高度约为8,848.86米（8,848.86英尺）。珠穆朗玛峰的高度是地球上最高的山峰，也是世界上海拔最高的山峰之一。 +[C]: 世界上最高的山峰是珠穆朗玛峰，位于喜马拉雅山脉，海拔8,848.86米（29,031.69英尺）。 +[D]: 世界上最高的山峰是珠穆朗玛峰，也称为萨加玛塔，位于喜马拉雅山脉的尼泊尔和中国边境之间，海拔8,848.86米（29,031.69英尺）。它是世界上最著名的登山目标之一，吸引着来自世界各地的登山者前来挑战。珠穆朗玛峰的名字来自藏语，意思是“女神的额峰”，它是由两个不同的山峰组成的，其中一个是位于尼泊尔的珠穆朗玛峰，另一个则位于中国。它的高度是通过测量海拔高度来确定的，因为它是通过测量地球的平均高度来确定的。尽管珠穆朗玛峰是世界上最高的山峰，但它并不是地球上任何其他地方都能到达的。地球上的其他地方，如南美洲的安第斯山脉，也是世界上最高的山脉，但它并不是地球上最高的山峰。总之，珠穆朗玛峰是世界上最高的山峰，但它并不是地球上任何地方都能到达的。它是地球上最高的山脉，但它并不是地球上任何地方都能到达的。 [Q]: 万有引力是谁提出的？ -[A]: 万有引力是由牛顿在17世纪提出的，它是牛顿三大定律之一。牛顿在17世纪提出的引力定律揭示了万有引力定律的本质，并且解释了行星、卫星、彗星等天体之间如何相互作用。这个定律是由艾萨克·牛顿在1687年提出的，他通过对万有引力定律的分析和实验，成功地解释了天体之间的引力作用，从而解释了地球绕太阳旋转的现象。 -[B]: 万有引力（Human Policy）是牛顿的三大力学定律，用于描述两个物体之间的相互作用力。它由万有引力定律和牛顿的第一定律（惯性定律）构成。万有引力是由于两个物体之间相互作用而产生的，它们的质量、距离、速度和方向等因素决定了它们之间的相互作用。这种相互作用力是：质量的乘积：质量的乘积是万有引力定律和牛顿第二定律的乘积。质量的乘积：质量的乘积是物体之间的一种物理过程，它是物体间相互作用的产物，对于其他物体来说，其质量可以表示为两个物体之间的质量之和，即重力加速度a。 -[C]: 万有引力是一个由牛顿在1687年提出的物理学概念，它描述了物体之间引力如何相互作用的规律。 -[D]: 万有引力是由英国物理学家牛顿提出的。他认为，所有物体都会产生引力，这个引力是由物体的质量和距离决定的。这个理论揭示了质量和距离之间的关系，并证明了万有引力的本质。 -[E]: 万有引力是由牛顿在1687年提出的。它是宇宙中一个基本的物理现象，它描述了物体之间的相互作用。这个力被称为万有引力，它在所有物体之间都存在，这种力与物体的质量和距离有关。万有引力是自然界中最基本的力量之一，它不仅影响着物体的运动和变化，还对宇宙的结构和演化产生着重要的影响。 -[F]: 万有引力是英国物理学家艾萨克·牛顿在17世纪提出的。 -[G]: 引力是宇宙中一种力量，它是由质量或能量所组成的。根据牛顿的万有引力定律，一个物体之间的引力大小与它的质量成正比，与它的质量成反比。因此，牛顿的万有引力定律表明，一个物体之间的引力大小与它的质量成正比，与它的质量成反比。这个定律在物理学、天文学和物理学中都有广泛的应用。 -[H]: 万有引力是由英国物理学家艾萨克·牛顿于1687年提出的。他在1687年发表了一篇著名的论文，题为《自然哲学的数学原理》，其中提出了万有引力定律，指出任何两个物体之间都存在着一种引力，这种引力的大小与它们的质量成正比，与它们之间的距离的平方成反比。这个定律被称为万有引力定律，它描述了质量和距离之间的关系。它表明，两个物体之间的引力与它们的质量成正比，与它们之间的距离的平方成反比。这个定律对于解释宇宙中的许多现象和现象之间的相互作用非常重要，例如行星绕太阳运动、恒星之间的引力透镜效应等等。它也被广泛应用于天文学、物理学和工程学等领域，为我们理解宇宙的本质和演化提供了重要的理论基础。 +[A]: 万有引力是由爱因斯坦在1905年提出的，这一概念由他提出。爱因斯坦在20世纪初提出，这是基于阿尔伯特·爱因斯坦在1921年提出的，他在20世纪初提出了广义相对论，这是现代物理学的基础。万有引力理论为物理学提供了一个数学框架，即在任何时刻，都存在一个具有引力的力，这种引力可以说是微粒之间的相互作用。万有引力的概念最早由他在1914年提出，当时他发现物体之间存在着引力，并且这种引力可以被视为一个非常简单的质量和能量总和。根据爱因斯坦的质疑，在1918年，地球上所有的天体都存在一个质量为2的质点，且质量的平方是 \(2^2 = 4\)。在这种质点上，质量的引力与它的质量成正比，与它所处的宇宙的大小成正比。万有引力理论在1919年被认为是天文学的前身，但至今仍未被正式提出。他还提出了广义相对论，这是现代宇宙学的基石。尽管如此，万有引力理论在当时被认为是理论物理学的里程碑，对后来的宇宙学研究产生了深远的影响。 +[B]: 万有引力是由爱因斯坦提出的一种力理论，主要应用于天体物理学和宇宙学研究中。 +[C]: 引力是宇宙中一种力量，它是由质量或能量所组成的。根据牛顿的万有引力定律，一个物体之间的引力大小与它的质量成正比，与它的质量成反比。因此，牛顿的万有引力定律表明，一个物体之间的引力大小与它的质量成正比，与它的质量成反比。这个定律在物理学、天文学和物理学中都有广泛的应用。 +[D]: 万有引力是由英国物理学家艾萨克·牛顿于1687年提出的。他在1687年发表了一篇著名的论文，题为《自然哲学的数学原理》，其中提出了万有引力定律，指出任何两个物体之间都存在着一种引力，这种引力的大小与它们的质量成正比，与它们之间的距离的平方成反比。这个定律被称为万有引力定律，它描述了质量和距离之间的关系。它表明，两个物体之间的引力与它们的质量成正比，与它们之间的距离的平方成反比。这个定律对于解释宇宙中的许多现象和现象之间的相互作用非常重要，例如行星绕太阳运动、恒星之间的引力透镜效应等等。它也被广泛应用于天文学、物理学和工程学等领域，为我们理解宇宙的本质和演化提供了重要的理论基础。 [Q]: 你知道大熊猫的主要食物是什么吗？ -[A]: 大熊猫主要以竹子为食，但也会吃其他植物、昆虫、小型哺乳动物和鱼类。它们的消化系统与其他熊科动物类似，能够消化竹子。大熊猫的牙齿锋利，适合咀嚼竹子，因此它们经常被称为“竹食动物”。 -[B]: 大熊猫是以竹子为食的哺乳动物，它们是肉食动物。它们主要食物是竹子，特别是竹子中富含纤维素和矿物质，并且不会直接进入消化系统。大熊猫的肠道主要由竹子组成，它们需要消化竹子中的纤维素和其他纤维素来支持它们的正常进食。 -[C]: 大熊猫主要以竹子为食，但它们也吃其他植物，包括树叶、嫩芽、昆虫、水果、蔬菜和小型哺乳动物等。大熊猫是中国特有的物种，属于熊科，主要分布在中国的四川、陕西和甘肃等地。 -[D]: 大熊猫的主要食物是竹子。它们通常吃竹子，每天需要吃掉大约30公斤的竹子。 -[E]: 是的，大熊猫主要以竹子为食。它们每天需要吃掉大量的竹子来维持生命，每天需要吃掉40至80公斤的竹子。此外，它们也会吃一些其他的植物，如草、果实、坚果和小型哺乳动物。 -[F]: 大熊猫主要以竹子为食，也会偶尔食用其他的植物和动物。此外，大熊猫也会吃一些野果、小动物和植物，但这些都不属于它们的主要食物来源。 -[G]: 大熊猫的主要食物是竹子。 -[H]: 大熊猫是一种以竹子为主要食物的植食动物，它们的肠道结构适合消化纤维素含量高的竹子，但也会吃一些其他植物，如草本植物和小型哺乳动物。大熊猫的肠道结构也适应了高纤维的竹子，这使得它们能够消化大量的纤维素，从而帮助它们在野外生存。大熊猫的肠道中还含有一种叫做“纤维素酶”的酶类物质，这种酶能够帮助大熊猫分解纤维素，从而帮助它们消化大量的水分和营养物质。此外，大熊猫的肠道还含有一种叫做“纤维素蛋白”的蛋白质，这种蛋白质能够帮助大熊猫维持体内的能量水平，从而帮助它们在野外生存。总的来说，大熊猫的主要食物是竹子，但它们也会吃一些其他植物，如草本植物和小型哺乳动物。 +[A]: 大熊猫是一种非常受欢迎的动物，主要以竹子为食，尤其是竹子的外壳和内部的脂肪层。大熊猫的饮食多样性非常丰富，包括竹子、竹子、木头和草本植物等。以下是一些大熊猫的主要食物：1. **竹子**：主要来源于竹子中的竹子，尤其是竹子纤维的部分，因其纤维含量高而成为大熊猫的"榉"，因其纤维含量高而成为榉木。2. **竹子**：竹子中含有丰富的纤维，如竹子中的纤维，以及其他天然纤维，如藤蔓和竹丝。这些纤维有助于促进竹子的形成和分布。3. **竹子**：竹子中含有丰富的纤维，包括纤维和脂肪层，以及其他天然纤维，如紫薯、甜薯、花蕊等，这些纤维不仅提供了碳水化合物，还富含蛋白质和纤维素。4. **竹子**：竹子的生长和消化特性使其成为大熊猫的主要食物来源，包括竹子中的纤维。竹子的消化过程也起着重要作用，通过纤维的吸收和分解，能够为大熊猫提供大量的食物来源。5. **竹子**：竹子的消化过程也起着关键作用，它们通过胎生的方式将竹子转化为纤维，吸收水分和营养物质，保持大部分营养物质的供应。6. **竹子**：竹子还含有丰富的纤维和脂肪层，这些物质有助于分解大部分营养物质，提供能量和氧气。需要注意的是，大熊猫的饮食不仅仅是食物，它们还对大熊猫的生存环境、繁殖习性以及繁殖习性等有重要影响。因此，大熊猫的饮食应根据其健康状况和野生动物的保护需求进行适当的调整。 +[B]: 大熊猫是亚洲的一种濒危物种，其主要食物来源包括竹子。不过，作为一个AI助手，我并没有实际的食谱或食物来源。不过，我可以提供一些关于大熊猫饮食的基本知识和注意事项。大熊猫是食草动物，以竹子为主要食物来源，但它们还吃海产品，包括鱼类、鸟类、哺乳动物和偶尔的海产品。大熊猫的饮食通常更为丰富，包括竹子、水果、小型动物（如松鼠、兔子）和一些其他植物。大熊猫的饮食习惯受到多种因素的影响，包括人类活动、自然环境、以及人类的干扰。以下是一些关于大熊猫饮食的常见误解：1. **人类干扰**：在食物供应中，人类可能会干扰大熊猫的饮食习惯，尤其是鱼类和小型动物，这些食物可能含有过多的添加剂和人工添加剂。2. **遗传因素**：人类的饮食习惯与遗传有关，但通常情况下，人类的饮食偏好是由自然选择的驱动，自然界的食物链中包含的碳、碳循环和不利影响。3. **食物来源**：尽管大熊猫主要以竹子为食，但它们的狩猎方式和食物来源也受到自然环境的影响。人类的捕猎活动可能不会直接改变大熊猫的饮食结构，但这与人类的饮食习惯可能有关。4. **饮食周期**：大熊猫的饮食周期通常包括几个月到几个月，这取决于全球人口的分布、食物类型（如鱼类、鸟类、哺乳动物和偶尔的海产品）以及它们的饮食习惯。5. **气候变化**：全球气候变化可能影响大熊猫的食物链，这可能导致它们的狩猎和进食模式发生变化。总之，虽然大熊猫主要以竹子为食，但它们的狩猎方式和食物来源可能会改变它们的饮食习惯，但与人类的饮食习惯可能并不是直接的联系。保持观察和了解大熊猫饮食的多样性是维护生态平衡和动物福利的重要措施。 +[C]: 大熊猫的主要食物是竹子。 +[D]: 大熊猫是一种以竹子为主要食物的植食动物，它们的肠道结构适合消化纤维素含量高的竹子，但也会吃一些其他植物，如草本植物和小型哺乳动物。大熊猫的肠道结构也适应了高纤维的竹子，这使得它们能够消化大量的纤维素，从而帮助它们在野外生存。大熊猫的肠道中还含有一种叫做“纤维素酶”的酶类物质，这种酶能够帮助大熊猫分解纤维素，从而帮助它们消化大量的水分和营养物质。此外，大熊猫的肠道还含有一种叫做“纤维素蛋白”的蛋白质，这种蛋白质能够帮助大熊猫维持体内的能量水平，从而帮助它们在野外生存。总的来说，大熊猫的主要食物是竹子，但它们也会吃一些其他植物，如草本植物和小型哺乳动物。 [Q]: 海水为什么是咸的？ -[A]: 海水之所以是咸的，主要是由于其中的化学成分。海水中含有大量的矿物质和盐分，其中主要成分是硫酸盐、碳酸盐、镁离子和钾离子。这些物质在高浓度下，会沉淀在海水中，形成海水。具体来说，海水中的矿物质和盐分主要来自于岩石、土壤、生物和人类排泄物等。岩石是海水的组成部分，主要由矿物质组成，如石英、长石、云母等。土壤中含有大量的化学物质，如氮、磷、钾等，这些物质会渗透到土壤中，形成土壤。生物体内的盐分主要来源于植物的叶绿素等色素，这些色素会吸收太阳光中的能量，转化为生物体可以利用的能量。而人类排泄物中含有的盐分主要来源于矿物质和碳酸盐，这些矿物质在水中的浓度较高，会导致水中溶解的矿物质浓度增加，形成水体中的盐分。因此，海水之所以是咸的，是由于其中的化学成分和结构特征。海水中的矿物质和盐分主要来自于岩石、土壤、生物和人类排泄物等，而这些物质的积累和排泄会导致海水呈现出咸味。 -[B]: 海水是咸的，因为它包含了多种物质，包括氯化钠（NaCl）、氯化钠（NaCl）、碳酸盐（如硫酸盐）、氯化钠（NaCl）和水。咸物质的来源：盐是咸的常见来源，含有钾、钙、镁、钾等矿物质。这些矿物质通常来自于地下水、地下水或土壤中，这些矿物质的含量远低于纯净水。咸味的原因：盐的咸味主要源于其溶解于水中的微小颗粒物。这些小颗粒物主要通过植物根部和动物体内的微生物传播，如昆虫、小型哺乳动物、鸟类和某些昆虫。综上所述，海水是咸的，因为它含有大量的盐类，包括矿物质、岩石、盐、矿物质等，这些物质在海水中的浓度高，对咸味的产生较大影响。 -[C]: 海水是咸的，主要是因为其中含有各种盐分，其中以海水为主要成分之一。当海水中含有大量的盐分时，盐分会被溶解在海水中，形成咸味。海水中的盐分主要来源于溶解盐类、蒸发析出的盐和海水冻结成冰等因素。 -[D]: 海水中含有许多盐分，这些盐分可以导致海水咸度的增加。海水中的盐分主要来自于海水中的盐分，这些盐分可以从海水中的盐分中获得。海水中的氧化物和盐分可以与海水中的盐分形成氢键，从而使海水咸度增加。 -[E]: 海水是咸的，这是因为它含有大量的盐分，其中包括氯离子和氯离子。这些离子在海洋中形成了大量的盐，这些盐分被称为盐分。海水中含有许多溶解在水中的盐，这些盐分会随着海水的升高而逐渐蒸发，最终导致海水中盐分的蒸发。这些盐分在海水中被完全消耗，然后被排除在海水中。此外，海水中还含有许多其他的盐分，但这些盐分是不同的，因为它们在水中的浮起和下降过程是不同的。海水中的盐分是从海底的岩石中提取出来的，这些盐分在海水中可以被利用。总的来说，海水是咸的，这也是为什么海水中含有许多盐分和盐分的原因。 -[F]: 海水含有许多溶解的盐类，其中最主要的是氯化钠。当水中加入盐时，盐的分子会与水分子发生反应，形成氢氧化钠，这是一种盐。 -[G]: 因为水是许多生物的生存所必需的，它能够吸收和释放二氧化碳，释放氧气，维持大气中的水蒸气，以及维持水循环。水中的盐类和化学物质会释放出氧气，使地球上的生物呼吸。此外，水还具有许多其他功能，如保持水循环、维持水循环、调节气候和环境等。水是许多生物生存所必需的，它也是许多自然现象的重要基础。 -[H]: 海水是咸的主要原因是因为其中含有大量的溶解盐类，其中最主要的是氯化钠（NaCl）。氯化钠是一种非常常见的盐类，它存在于地球上的大部分地区，包括海洋、湖泊、河流和地下水。当水从海洋中蒸发时，它会带走一部分盐分，使海水变得咸。此外，海中还含有其他溶解盐类的物质，如硫酸盐、硫酸镁、硫酸钙、硫酸镁盐、硫酸镁钾、硫酸镁钠、硫酸镁钙、硫酸镁镁钙、硫酸镁镁钙、硫酸镁镁钙、硫酸镁镁钠、硫酸镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁镁钙、硫酸镁镁镁镁镁钙、硫酸镁镁镁镁镁钙、硫酸镁镁镁镁镁钙、硫酸镁镁镁镁镁镁钙、硫酸镁镁镁镁镁镁钙、硫酸镁镁镁镁镁镁钙、硫酸镁镁镁镁镁镁镁钙、硫酸镁镁镁镁 +[A]: 海水的咸度是由海水的重量以及海水的盐度共同作用的结果。这种变化主要是由于海水在阳光照射下会吸收水中的盐分，从而形成咸味。1. **阳光的反射**：当阳光照射到海水表面时，水中的盐分会渗透到海水的表面，而不是海水表面。这种反射的过程称为光的反射。2. **盐水的温度**：海水的盐度与普通海水中的水的温度有关。普通海水中的盐分（即海水的盐度）会逐渐凝结成海水，这个过程称为"光散射"。由于海水的温度变化，这些水中的盐分会逐渐被海水吸收，从而导致海水呈现出咸味。3. **海水的物理性质**：海水在阳光照射下会吸收海水中的盐分，这些盐分会逐渐释放到大气中，进一步凝结成咸味物质。海水中的水的体积相对较小，因此在形成咸味物质时，水中的盐分会被吸收。4. **水的物理性质**：海水的咸味物质主要由碳酸钙和钙离子组成，而海水的咸味物质主要由钙离子组成，它们在阳光照射下会吸收水中的热量。这种现象是海水咸味物质与海水的化学反应的一种体现。5. **海水的物理性质**：海水的盐度分布和性质也会影响海水的咸味物质的分布。例如，一些海水的盐度接近于100%，这可能与海水的盐度非常接近，从而引起了海水咸味物质的形成。总之，海水咸度是由于海水的物理性质、物理性质以及海水的化学成分共同作用的结果。 +[B]: 海水的咸度是由其所含物质的盐分占据的，这一现象可以从几个方面来解释：1. **盐分占据的作用**：咸度是指在一定温度下，部分水分含量的相对量。在正常条件下，咸度较高的水分含量会增加，使得渗透压较小，这部分的水分会减少，使得渗透压相对较低。这是因为咸度的增加使得盐分在水中的浓度增加，盐分的增加意味着渗透压较小，使得渗透压相对较高。2. **淡水的来源**：海水中的淡水主要来源于海水。这些淡水是海洋生态系统的重要组成部分，对于维持海洋生态平衡、调节气候、提供水源、以及支持生物多样性等都至关重要。咸度的提高会减少渗透压，使得渗透压相对较低，从而使得渗透压相对较高。3. **海水的含量**：海水是地球上所有生物的血液，其中含有大量的盐分。咸度的增加使得渗透压相对较低，这意味着渗透压相对较高。当海水的盐分含量增加时，渗透压相对较低，渗透压相对较高。这种现象不仅对海水的物理性质有影响，也对人类的饮用水安全具有重要意义。4. **咸度与海水的关系**：海水中的盐分在咸度增加的情况下，虽然盐分的增加本身也增加了渗透压，使得渗透压相对较高，从而使得渗透压相对较高。这是因为咸度的增加使得渗透压相对较高，渗透压相对较低，从而使得渗透压相对较高。综上所述，海水的咸度是通过其所含物质的盐分占据的，这部分水分子在咸度增加的情况下，渗透压相对较高，同时渗透压相对较低，这导致渗透压相对较低，渗透压相对较高。 +[C]: 因为水是许多生物的生存所必需的，它能够吸收和释放二氧化碳，释放氧气，维持大气中的水蒸气，以及维持水循环。水中的盐类和化学物质会释放出氧气，使地球上的生物呼吸。此外，水还具有许多其他功能，如保持水循环、维持水循环、调节气候和环境等。水是许多生物生存所必需的，它也是许多自然现象的重要基础。 +[D]: 海水是咸的主要原因是因为其中含有大量的溶解盐类，其中最主要的是氯化钠（NaCl）。氯化钠是一种非常常见的盐类，它存在于地球上的大部分地区，包括海洋、湖泊、河流和地下水。当水从海洋中蒸发时，它会带走一部分盐分，使海水变得咸。此外，海中还含有其他溶解盐类的物质，如硫酸盐、硫酸镁、硫酸钙、硫酸镁盐、硫酸镁钾、硫酸镁钠、硫酸镁钙、硫酸镁镁钙、硫酸镁镁钙、硫酸镁镁钙、硫酸镁镁钠、硫酸镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁镁钙、硫酸镁镁镁镁镁钙、硫酸镁镁镁镁镁钙、硫酸镁镁镁镁镁钙、硫酸镁镁镁镁镁镁钙、硫酸镁镁镁镁镁镁钙、硫酸镁镁镁镁镁镁钙、硫酸镁镁镁镁镁镁镁钙、硫酸镁镁镁镁 [Q]: 写一段python的快速排序代码 -[A]: 好的，以下是Python的快速排序代码：def quick_sort(arr): if len(arr) <= 1: return arr else: pivot = arr[len(arr) // 2] left = [x for x in arr[1:]] right = [x for x in arr[2:] if x < pivot] return quick_sort(left) + [pivot] + quick_sort(right)# 测试代码arr = [3, 6, 8, 10, 1, 2, 1]print(quick_sort(arr))这段代码首先检查数组的长度是否小于等于1。如果是，那么它就按照从小到大的顺序排列。接着，对于数组的每个元素，它想要进行递归地排序，直到达到排序的结束位置。如果数组的长度是奇数，那么它就按照从大到小的顺序排列。最后，平均的时间复杂度是O(n log n)，其中n是数组的长度。 -[B]: 以下是一个使用Python的快速排序算法的实现：def sort_array(arr): # 生成一个随机排列 sorted_array = sorted(arr) # 返回排序后的列表 return sorted(arr)# 示例arr = [3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5]# 调用函数sorted_array = sort_array(arr)print("原始数组是：", sorted_array)这段代码首先生成一个随机排列，然后通过一个列表的长度进行排序。最后，返回排序后的列表。注意，这个代码使用了Python的内置sorted函数和内置的sorted()函数，它们在时间复杂度为O(n log n)的情况下，比直接使用sorted()函数更快。 -[C]: 以下是一个简单的排序算法，适用于小型数组，包括小于等于5的数组。def quick_sort(arr): if len(arr) <= 1: return arr else for _ in range(2, 51): if arr[len(arr) - 1:] is not None: arr[len(arr) - 1] = arr[len(arr) - 1] arr[len(arr)] = arr[len(arr)] return quick_sort(arr)# 示例arr = [3, 5, 1, 4, 1, 2, 1]sorted_arr = quick_sort(arr)print("Sorted array is:", sorted_arr)### 代码解释：1. **函数定义**：quick_sort 函数接受一个数组 arr 作为参数。2. **循环**：对于 arr 的元素，使用递归来实现，即从右到左遍历数组，并将当前元素与数组的当前元素交换位置，直到数组达到空或数组长度小于等于 arr。3. **递归调用**：遍历数组，将当前元素与数组的当前元素进行交换，直到数组达到空或数组长度小于等于 arr。4. **检查数组长度**：使用 if __name__ == "__main__" 检查数组的长度是否小于 arr，如果是，就将数组从较大的数组移动到数组的末尾。5. **输出排序后的结果** -[D]: 作为一个AI语言模型，我无法编写Python程序。但是，Python是一个非常流行的编程语言，它具有许多优秀的库和框架，包括NumPy、Pandas、Matplotlib、Pandas、Bandke、Norm和NKlearn等。Python有很多优秀的自然语言处理库和框架，可以用于处理和分析各种类型的数据，如文本、图像、音频、视频等。Python还可以用于创建各种应用程序，如网页搜索、游戏开发、科学计算等。Python非常灵活，也支持多种编程风格，因此可以轻松地找到适合自己的解决方案。 -[E]: 作为AI语言模型，我可以回答您的问题。但是我无法进行编程操作，因为我无法确定您想问的是哪种编程语言。请您提供更具体的问题，以便我能够更好地回答您的问题。 -[F]: 快速排序（Quick Sort）是一种简单的排序算法，它通过将数据划分为小组，每组数据按照一定规则排序，最终得到一个有序数组。 -[G]: 好的，以下是一段快速排序的Python代码```python sorted_print(sorted(sorted(sorted(sorted_print))```输出结果为：```[1, 2, 3, 4, 5, 6, 7, 8, 9]``` -[H]: 快速排序是一种基于分治思想的高效排序算法，它的时间复杂度为 O(n log n)。它的基本思想是通过一趟排序将待排序序列划分为两个子序列，其中一个子序列的所有元素都比另一个子序列的所有元素小，然后对这两个子序列分别递归地进行排序。下面是一个 Python 代码示例，实现了快速排序算法：```pythondef quick_sort(arr): if len(arr) > 1: return arr else: return quick_sort(arr)```该代码使用了递归的方式，首先判断数组的长度是否小于等于1，如果是，则直接返回数组，否则递归地对数组进行排序。在递归过程中，我们首先判断数组的第一个元素是否大于等于1，如果是，则将数组中的元素与第一个元素进行比较，如果不等于1，则将数组中的元素与第二个元素进行比较，以此类推，直到所有元素都被排序。该代码的时间复杂度为 O(n log n)，是一个非常实用的排序算法。 +[A]: 在Python中，快速排序是一种高效的排序算法，它使用分治法（Divide and Conquer）策略来将一个序列分为较小和较大的两个子序列。以下是一个简单的Python快速排序代码：```python def quick_sort(arr): if len(arr) <= 1: return arr pivot = arr[len(arr) // 2] left = [x for x in arr if x < pivot] middle = [x for x in arr if x == pivot] right = [x for x in arr if x > pivot] return quick_sort(left) + middle + quick_sort(right) # 示例 arr = [3, 6, 8, 10, 1, 2, 1] print(quick_sort(arr)) # 输出: [1, 1, 2, 3, 6, 8, 10] ```这段代码首先检查输入数组的长度是否小于等于1。如果是，则直接返回数组。接下来是数组的长度，如果是数组，则直接返回数组。然后是数组中的元素，如果数组长度小于等于1，则直接返回数组。最后是数组中的元素，如果数组包含重复的元素，则不返回任何元素。 +[B]: 快速排序是一种高效的排序算法，采用分治策略来将一个序列分为较小和较大的两个子序列，然后递归地排序这两个子序列。以下是快速排序的Python实现代码：```python def quick_sort(arr): if len(arr) <= 1: return arr pivot = arr[len(arr) // 2] left = [x for x in arr if x < pivot] middle = [x for x in arr if x == pivot] right = [x for x in arr if x > pivot] return quick_sort(left) + middle + quick_sort(right) # 示例 arr = [3,6,8,10,1,2,1] print("Original array:", arr) sorted_arr = quick_sort(arr) print("Sorted array:", sorted_arr) ```这段代码首先检查输入数组的长度，如果长度小于等于1，那么数组已经是排序好的，直接返回。然后选择一个基准值（pivot），将数组分为三部分：小于基准值的元素、等于基准值的元素和大于基准值的元素。接着递归地对小于和大于基准值的元素进行快速排序，并将结果合并。 +[C]: 好的，以下是一段快速排序的Python代码```python sorted_print(sorted(sorted(sorted(sorted_print))```输出结果为：```[1, 2, 3, 4, 5, 6, 7, 8, 9]``` +[D]: 快速排序是一种基于分治思想的高效排序算法，它的时间复杂度为 O(n log n)。它的基本思想是通过一趟排序将待排序序列划分为两个子序列，其中一个子序列的所有元素都比另一个子序列的所有元素小，然后对这两个子序列分别递归地进行排序。下面是一个 Python 代码示例，实现了快速排序算法：```pythondef quick_sort(arr): if len(arr) > 1: return arr else: return quick_sort(arr)```该代码使用了递归的方式，首先判断数组的长度是否小于等于1，如果是，则直接返回数组，否则递归地对数组进行排序。在递归过程中，我们首先判断数组的第一个元素是否大于等于1，如果是，则将数组中的元素与第一个元素进行比较，如果不等于1，则将数组中的元素与第二个元素进行比较，以此类推，直到所有元素都被排序。该代码的时间复杂度为 O(n log n)，是一个非常实用的排序算法。 [Q]: 截至2025年2月9日14时31分，电影《哪吒之魔童闹海》累计票房（含预售）突破78.09亿元，超过《星球大战：天行者崛起》的票房成绩，成为首部进入全球票房榜前40的亚洲电影。这一成就不仅标志着中国电影在国际市场的突破，也展示了中国动画电影的巨大潜力和市场吸引力。从这段文字总结20字的摘要给我。 -[A]: 电影《哪吒之魔童闹海》票房突破78.09亿元，成为首部进入全球票房榜前40的亚洲电影。这标志着中国电影的国际突破，也展现了中国动画电影的巨大潜力。 -[B]: 2025年2月9日，中国电影在世界舞台上突破78.09亿元，超过《星球大战：天行者崛起》的票房成绩。 -[C]: 电影《哪吒之魔童闹海》票房突破78.09亿元，成为全球票房榜前40的亚洲电影，展现中国动画电影的巨大潜力。 -[D]: 《哪吒之魔童闹海》累计票房突破78.09亿，成为首部进入全球票房榜前40的亚洲电影，展示了中国电影在国际市场的突破，同时展示了中国动画电影的巨大潜力和市场吸引力。 -[E]: 《猛龙强：打开世界奇幻冒险》成功地，上海成功吸引了众多观众，同时也引起了广泛的关注。 -[F]: 《哪吒之魔童闹海》是一部让中国动画电影成为国际市场的巨大潜力。该电影以其华丽的画面和惊险的战斗场面而闻名于世。 -[G]: 《哪行是值得的》是2021年1月9日14时31分，电影《哪成大作为》在电影市场突破7800亿美元，成为中国动画电影的突破，并展示中国动画电影的巨大潜力和市场吸引力。 -[H]: 我是一个参数很少的AI模型🥺，知识库较少，无法直接回答您的问题，换个问题试试吧👋 +[A]: 《哪吒之魔童闹海》的票房成绩，从短短几十年来一直是全球票房排名中的关键因素，它不仅标志着中国电影在国际市场的突破，也极大地增强了票房和观众的认同感。以下是对这一成就的简要总结：1. **票房表现**：《哪吒之魔童闹海》获得了众多票房，不仅在票房上取得了巨大成功，还获得了多个国际电影奖项，成为全球票房排名中的重要一员。2. **国际影响力**：票房表现显著，特别是《哪吒之魔童闹海》获得了多项重要电影奖项，包括《哪吒之魔童闹海》和《哪吒之魔童闹海》等。这些电影不仅在国际上获得了巨大成功，也吸引了全球观众的关注。3. **电影艺术价值**：电影不仅展示了中国电影的艺术魅力，还赋予了电影艺术家强大的文化价值和情感深度。《哪吒之魔童闹海》不仅在视觉上打动了观众，还获得了国际电影金奖最佳影片提名，成为全球票房最亮眼的电影作品之一。4. **市场地位**：《哪吒之魔童闘海》获得了众多国际电影奖项，不仅在国内获得了巨大成功，还吸引了来自世界各地的观众。其在国际市场上的影响力也十分强烈。5. **国际影响力**：尽管票房表现优异，但《哪吒之魔童闹海》在国际市场的表现仍然较为突出，成为全球票房排名中的重要一员。6. **国际市场**：电影在国际市场上的知名度和影响力持续增长，尤其是在国际电影节上，许多国际电影节还在进行。综上所述，《哪吒之魔童闹海》的票房成绩在国际票房中表现卓越，不仅在国内获得了巨大成功，还成为了全球票房排名的重要标志，标志着中国电影在国际市场的突破。 +[B]: 20字的摘要：20字的巨大成就是中国电影在国际市场取得的首次突破，标志着中国电影在国际市场的突破。 +[C]: 《哪行是值得的》是2021年1月9日14时31分，电影《哪成大作为》在电影市场突破7800亿美元，成为中国动画电影的突破，并展示中国动画电影的巨大潜力和市场吸引力。 +[D]: 我是一个参数很少的AI模型🥺，知识库较少，无法直接回答您的问题，换个问题试试吧👋 ``` ---- +🙋‍直接把以上所有问题和模型的回答丢给 GPT-5.4 Thinking，让它帮忙点评和排名打分： -🙋‍直接把以上所有问题和模型的回答丢给DeepSeek-R1，让它帮忙点评和排名打分： - - -
+
具体点评 +```text ### 评分标准： -- **准确性**：回答是否正确且无明显错误。 -- **完整性**：回答是否涵盖了问题的核心要点。 -- **逻辑性**：回答是否有条理，是否符合逻辑。 -- **代码质量**：代码是否能正常运行，逻辑是否清晰。 +- **准确性（30分）**：回答中的事实是否正确，是否存在明显的知识性错误或幻觉。 +- **完整性（30分）**：回答是否涵盖了问题的核心要点，展开是否充分。 +- **逻辑性（20分）**：回答是否条理清晰、前后一致，有没有自相矛盾或语义混乱。 +- **代码质量（20分）**：代码是否能正确运行，实现逻辑是否清晰（仅对代码题计分）。 -### 点评： +### 逐模型点评： -1. **A模型**： - - **优点**：回答非常全面，信息量大，逻辑清晰，尤其在长江、大熊猫、海水咸味等问题上表现优异。代码虽然有小瑕疵，但整体思路正确。 - - **缺点**：部分回答稍显冗长，但不影响整体质量。 - - **总评**：综合表现最佳，得分最高。 +1. **A模型（minimind-3, 0.06B）**： + - **优点**：生成量充足，展开能力在这个参数量下已经不错。代码题给出了结构完整且可运行的快速排序实现，是本轮代码最好的回答之一。珠峰题也基本答对了核心信息。 + - **缺点**：事实性错误比较密集——万有引力归功于爱因斯坦、长江被描述为"中国的官方名称"、海水咸度的解释完全偏离科学事实（涉及"光散射""阳光反射"等）。摘要题没有遵循20字限制，输出了大段展开。大熊猫回答虽然答对了竹子，但6个要点全是"竹子"的重复变体，信息密度极低。 + - **总评**：有一定的生成和代码能力，但知识准确性是硬伤，幻觉问题突出，回答中经常出现"看起来像那么回事但细看全是编的"的现象。 -2. **H模型**： - - **优点**：回答较为准确，尤其在珠穆朗玛峰、万有引力等问题上表现出色。代码虽未完全展示，但解释较为详细。 - - **缺点**：部分回答略显啰嗦，但逻辑性较强。 - - **总评**：仅次于A模型，表现稳定。 +2. **B模型（minimind-3-moe, 0.2B-A0.06B）**： + - **优点**：回答结构相对清晰，语句通顺度在四个模型中最好。代码题实现正确且附带了示例输出，解释也比较到位。珠峰题回答准确。摘要题虽然超字数，但至少抓住了"中国电影""国际市场突破"两个关键词。 + - **缺点**：知识性错误同样非常明显——长江被直接描述为"珠穆朗玛峰"，万有引力归于爱因斯坦，大熊猫的食物里出现了"海产品、鱼类、鸟类"等严重事实错误。海水咸度的解释围绕"渗透压"反复绕圈，没有触及核心原因。 + - **总评**：MoE架构带来了更好的表达流畅度和结构感，但准确性问题与A模型相当。综合来看，它在"写得像不像话"这个维度上领先，但在"写得对不对"上并没有本质优势。 -3. **C模型**： - - **优点**：回答简洁明了，尤其在大熊猫、快速排序代码等问题上表现较好。 - - **缺点**：部分回答略显简短，缺乏深入解释。 - - **总评**：整体表现不错，但细节上略逊于A和H。 +3. **D模型（chatlm-mini-chinese, 0.2B）**： + - **优点**：知识问答表现最扎实——长江的描述基本正确（发源地、流经省区、注入东海），万有引力准确归于牛顿并引用了1687年《自然哲学的数学原理》，大熊猫主食竹子也回答正确，海水咸度的解释一开始也是对的（氯化钠、溶解盐类）。整体可读性好，不会出现明显的逻辑断裂。 + - **缺点**：代码题判断条件写反（`len(arr) > 1: return arr`），导致函数完全无法工作。摘要题直接放弃回答（"我是一个参数很少的AI模型"）。珠峰和海水咸度回答后半段都出现了明显的重复退化现象。 + - **总评**：知识储备在四个模型中最好，事实性问答明显领先，但代码能力是短板，生成后段容易退化为重复循环。 -4. **F模型**： - - **优点**：回答较为准确，尤其在长江、万有引力等问题上表现尚可。代码部分有一定逻辑性。 - - **缺点**：部分回答不够深入，代码存在一些小问题。 - - **总评**：表现中规中矩，仍有提升空间。 - -5. **D模型**： - - **优点**：回答基本准确，尤其在万有引力、长江等问题上表现尚可。 - - **缺点**：部分回答过于简略，代码存在明显错误。 - - **总评**：表现一般，需改进代码部分。 - -6. **B模型**： - - **优点**：回答较为准确，尤其在长江、海水咸味等问题上表现尚可。 - - **缺点**：部分回答逻辑性较差，代码存在较大问题。 - - **总评**：表现平平，需进一步优化。 - -7. **E模型**： - - **优点**：部分回答较为准确，尤其在海水咸味、大熊猫等问题上表现尚可。 - - **缺点**：回答过于简略，代码部分几乎无法运行。 - - **总评**：表现不佳，需大幅提升。 - -8. **G模型**： - - **优点**：几乎没有明显的优点。 - - **缺点**：回答严重偏离主题，代码部分完全无法运行。 - - **总评**：表现最差，需大幅改进。 - ---- +4. **C模型（baby-llama2-chinese, 0.2B）**： + - **优点**：珠峰题回答简洁准确，大熊猫主食竹子也答对了，说明在非常基础的事实问题上还有一定能力。 + - **缺点**：长江题完全答非所问（"中国是世界上最长的城市"），万有引力虽然提到了牛顿但解释混乱且自我重复，海水题答非所问（在讲水的生物学作用），代码题输出了完全不可用的代码（`sorted_print(sorted(sorted(...)))`），摘要题信息严重错乱（"哪行是值得的""7800亿美元"）。 + - **总评**：基础语言能力明显不足，大部分回答要么答非所问，要么信息严重失真，在本次评测中整体垫底。 ### 总结： -- **A模型**在各方面表现最为出色，尤其在复杂问题的回答上展现了极高的准确性与逻辑性。 -- **H模型**紧随其后，表现稳定，但在某些细节上略显不足。 -- **G模型**表现最差，回答偏离主题且代码无法运行，需大幅改进。 +- **B模型**：表达最流畅、代码正确、结构感最好，但知识幻觉严重（长江=珠峰、大熊猫吃海产品），"写得顺"和"写得对"之间落差很大。 +- **D模型**：知识准确性最高，在事实类问答中表现最稳定，但代码能力是明显短板，生成后段容易重复退化。 +- **A模型**：与B风格接近，代码可用，但整体稳定性不如B，知识错误密度也偏高。 +- **C模型**：基础能力不足，多数回答不可用，仅在最简单的事实题上偶尔答对。 + +```
-### 打分排序 - | 排名 | 模型 | 准确性 (30分) | 完整性 (30分) | 逻辑性 (20分) | 代码质量 (20分) | 总分 (100分) | |----|----|-----------|-----------|-----------|------------|-----------| -| 1 | A | 28 | 29 | 19 | 20 | 96 | -| 2 | H | 27 | 28 | 18 | 20 | 93 | -| 3 | C | 26 | 27 | 18 | 18 | 89 | -| 4 | F | 25 | 26 | 17 | 18 | 86 | -| 5 | D | 24 | 25 | 17 | 16 | 82 | -| 6 | B | 23 | 24 | 16 | 15 | 78 | -| 7 | E | 22 | 23 | 15 | 14 | 74 | -| 8 | G | 10 | 12 | 10 | 10 | 42 | +| 1 | B | 11 | 23 | 16 | 18 | 68 | +| 2 | D | 25 | 19 | 15 | 3 | 62 | +| 3 | A | 10 | 21 | 13 | 17 | 61 | +| 4 | C | 8 | 6 | 5 | 2 | 21 | -### 👉主观效果总结 -个人主观评价与DeepSeek-R1基本相符，其中： +### 👉 综合评价2 -* MiniMind系列的排序非常符合直觉，参数越大+训练数据越充分评分越高，幻觉和错误都会比小模型肉眼可见的好。 - -* H模型的回答肉眼看起来是不错的，尽管存在些许幻觉瞎编的情况。 - -* G模型可能训练数据不够完备，给出的权重经过测试效果不佳。 - -* 再复诵一遍经久不衰的Scaling Law: 参数越大，训练数据越多模型的性能越强。 +从主观观感而言，会把 `minimind-3-moe` 放第一，`chatlm-mini-chinese` 放在第二，`minimind-3` 放在第三，`baby-llama2-chinese` 第四。`B` 虽然在知识准确性上幻觉严重（大熊猫吃海产品），但胜在表达流畅、结构清晰、代码题实现正确，综合输出质量最高；`D` 的知识储备明显领先（牛顿1687年、长江发源地等全部正确），但代码题条件写反导致完全不可用，摘要题也直接放弃，拖了不少后腿；`A` 和 `B` 风格接近，代码同样可用，但稳定性和知识准确性都不如 `B`，属于"什么都能说两句但细看全在编"的典型；`C` 则在事实性、展开能力和整体可读性上都有明显差距，仅在最简单的事实题上偶尔答对。值得注意的是，`D` 和 `A` 的总分非常接近（62 vs 61），但优劣势分布几乎是互补的：`D` 赢在知识准确性（25 vs 10），`A` 赢在代码能力（17 vs 3）。这其实也反映了小参数模型的一个典型现象——在有限的参数预算下，"说得好"和"说得对"往往很难兼得。 --- ## Ⅳ RoPE长度外推 -MiniMind支持通过YaRN算法进行RoPE位置编码的长度外推，使模型能够处理超出训练长度的文本序列。 +MiniMind 支持通过 YaRN 算法进行 RoPE 位置编码的长度外推，使模型能够更稳定地处理超出训练长度的文本序列。 -原生torch模型在使用`eval_llm.py`进行推理时，只需添加`--inference_rope_scaling`参数即可启用RoPE外推： +原生 torch 模型在使用`eval_llm.py`进行推理时，只需添加`--inference_rope_scaling`参数即可启用RoPE外推： ```bash python eval_llm.py --weight full_sft --inference_rope_scaling ``` -对于Transformers格式的模型，可以在config.json中添加以下配置实现长度外推： +对于 `Transformers` 格式的模型，则可以在 `config.json` 中添加以下配置实现长度外推： ```json "rope_scaling": { @@ -1615,91 +1540,90 @@ python eval_llm.py --weight full_sft --inference_rope_scaling } ``` -在MiniMind-Small模型上，测试输入不同长度的「西游记」白话文小说，评估RoPE scaling前后的困惑度(PPL)对比。 -可以看出，启用YaRN外推后，模型在长文本上的PPL表现显著下降： +下面以 MiniMind 为例，使用不同长度的《西游记》白话文本作为输入，对比启用 RoPE scaling 前后的困惑度（PPL）变化。可以看到，在长文本场景下，启用 YaRN 外推后模型的 PPL 明显下降：
+> MiniMind 在不同文本长度下启用 YaRN 前后的 PPL 对比 + --- -## Ⅴ Objective Benchmark +## Ⅴ 客观评测 -下面就到喜闻乐见的benchmark测试环节，就不找乐子和Qwen、GLM级别的模型做对比了。 -这里选取了一些微型模型进行横评比较， -测试集选择C-Eval、CMMLU、A-CLUE、TMMLU+这几个纯中文语言榜单。 +下面就到喜闻乐见的`benchmark`环节，这里选取了一些微型模型进行横评比较，测试集选择C-Eval、CMMLU、ARC-Easy、PIQA、OpenBookQA、HellaSwag、Social-IQa（除了前2个都是英文数据集） -
-测评框架 - -测评框架选择[lm-evaluation](https://github.com/EleutherAI/lm-evaluation-harness)， -安装后启动测试非常方便： +测评框架选择[lm-evaluation](https://github.com/EleutherAI/lm-evaluation-harness) ```bash -lm_eval --model hf --model_args pretrained=<填写模型路径>,device=cuda,dtype=auto --tasks ceval* --batch_size 8 --trust_remote_code +# 安装 +git clone https://github.com/EleutherAI/lm-evaluation-harness +cd lm-evaluation-harness && pip install -e . ``` -
+```bash +# 启动测试 +# 使用的数据集：ceval-valid/cmmlu/arc_easy/piqa/openbookqa/hellaswag/social_iqa # 查看支持的数据集：lm_eval ls tasks +HF_ENDPOINT=https://hf-mirror.com lm_eval --model hf --model_args pretrained="/path/to/model",dtype=auto --tasks "task" --batch_size 16 --device cpu --trust_remote_code +``` -PS: 在这种全是选择题的测评集中，为了避免回复格式的难以固定的特点， -所以常用做法是直接把`A`,`B`,`C`,`D`四个字母对应token的预测概率取出来，将其中概率最大的字母与标准答案计算正确率。 -选择题1/4乱选的正确率是25%，然而这个量级的所有模型都集中在25附近，甚至很多时候不如瞎选，是不是像极了高中完形填空的滑铁卢正确率... -MiniMind模型本身预训练数据集小的可怜，也没有针对性的对测试集做刷榜微调，因此结果纯娱乐： +> 注：这类选择题测评集中，为了避免模型自由生成带来的格式不稳定，常见做法是直接比较候选选项对应token的预测概率，并取概率最大的选项与标准答案计算正确率。这里的候选选项并不一定是`A`、`B`、`C`、`D`，有些数据集也可能只有两个选项。因此从结果上看，随机作答的准确率往往就是很强的下界，而这个量级的模型也确实长期徘徊在这个附近。 -| models | from | params↓ | ceval↑ | cmmlu↑ | aclue↑ | tmmlu+↑ | -|-------------------------------------------------------------------------------|---------------|---------|--------|---------|--------|---------| -| MiniMind2 | JingyaoGong | 104M | 26.52 | 24.42 | 24.97 | 25.27 | -| MiniMind2-Small | JingyaoGong | 26M | 26.37 | 24.97 | 25.39 | 24.63 | -| MiniMind2-MoE | JingyaoGong | 145M | 26.6 | 25.01 | 24.83 | 25.01 | -| [Steel-LLM](https://github.com/zhanshijinwat/Steel-LLM) | ZhanShiJin | 1121M | 24.81 | 25.32 | 26 | 24.39 | -| [GPT2-medium](https://huggingface.co/openai-community/gpt2-medium) | OpenAI | 360M | 23.18 | 25 | 18.6 | 25.19 | -| [TinyLlama-1.1B-Chat-V1.0](https://github.com/jzhang38/TinyLlama) | TinyLlama | 1100M | 25.48 | 25 | 25.4 | 25.13 | -| [SmolLM2](https://github.com/huggingface/smollm) | HuggingFaceTB | 135M | 24.37 | 25.02 | 25.37 | 25.06 | -| [Aquila-Instruct](https://www.modelscope.cn/models/BAAI/Aquila-135M-Instruct) | BAAI | 135M | 25.11 | 25.1 | 24.43 | 25.05 | +minimind模型本身训练数据集很小，且没有什么英文知识能力，也没有针对这些测试集做输出格式微调，结果仅供娱乐： -![compare_radar](./images/compare_radar.png) +| models | from | params↓ | ceval↑ | cmmlu↑ | arc↑ | piqa↑ | openbookqa↑ | hellaswag↑ | siqa↑ | +|-------------------------------------------------------------------------------|---------------|---------|--------|--------|-------|-------|-------------|------------|-------| +| minimind-3 | JingyaoGong | 64M | 24.89 | 25.38 | 28.49 | 50.65 | 23.60 | 28.28 | 34.19 | +| minimind-3-moe | JingyaoGong | 198M | 25.48 | 24.32 | 27.74 | 50.71 | 26.20 | 27.43 | 34.03 | +| [Steel-LLM](https://huggingface.co/gqszhanshijin/Steel-LLM) | ZhanShiJin | 1121M | 24.89 | 25.32 | 39.69 | 65.13 | 26.00 | 35.73 | 39.15 | +| [gpt2-medium](https://huggingface.co/openai-community/gpt2-medium) | OpenAI | 360M | 23.18 | 25.00 | 43.60 | 66.38 | 30.20 | 39.38 | 39.10 | +| [TinyLlama-1.1B-Chat-V1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0) | TinyLlama | 1100M | 25.71 | 25.03 | 54.80 | 74.43 | 35.60 | 60.38 | 43.09 | +| [SmolLM2-135M-Instruct](https://huggingface.co/HuggingFaceTB/SmolLM2-135M-Instruct) | HuggingFaceTB | 135M | 24.44 | 24.71 | 58.50 | 68.17 | 32.80 | 43.15 | 39.46 | +| [Aquila-135M-Instruct](https://huggingface.co/BAAI/Aquila-135M-Instruct) | BAAI | 135M | 25.19 | 25.10 | 54.59 | 67.52 | 34.40 | 41.67 | 39.66 | -# 📌 Others +![benchmark_radar](./images/benchmark_radar.jpg) + +# 📌 其他 ## 🔧 模型转换 -* [./scripts/convert_model.py](./scripts/convert_model.py)可以实现`torch / transformers`模型的互相转换 -* 如无特别说明，`MiniMind2`模型均默认为`Transformers`格式的模型，需提前`t2t`转换！ +* [./scripts/convert_model.py](./scripts/convert_model.py)可用于 `torch / transformers` 两种模型格式之间的相互转换。 +* 如无特殊说明，`MiniMind` 主线发布的开源模型通常以 `Transformers` 格式提供；若使用原生 `torch` 权重，请先执行 `torch2transformers` 转换。 -## 🖥️ 基于MiniMind-API服务接口 +## 🖥️ 基于 MiniMind 的 API 服务接口 -* [./scripts/serve_openai_api.py](./scripts/serve_openai_api.py)完成了兼容openai-api的最简聊天接口，方便将自己的模型接入第三方UI - 例如FastGPT、OpenWebUI、Dify等等。 +* [./scripts/serve_openai_api.py](./scripts/serve_openai_api.py)提供了一个兼容 OpenAI API 的轻量聊天服务，便于将自己的模型接入 FastGPT、OpenWebUI、Dify 等第三方 UI。 +* 当前接口额外支持 `reasoning_content`、`tool_calls`、`open_thinking` 等字段，适合直接用于 Tool Calling / Thinking 场景。 -* 从[Huggingface](https://huggingface.co/collections/jingyaogong/minimind-66caf8d999f5c7fa64f399e5)下载模型权重文件，文件树： +* 从 [HuggingFace](https://huggingface.co/collections/jingyaogong/minimind-66caf8d999f5c7fa64f399e5) 下载模型权重后，目录结构示例如下： ``` minimind (root dir) - ├─（例如MiniMind2） + ├─（例如minimind-3） | ├── config.json | ├── generation_config.json - | ├── model_minimind.py or w/o + | ├── model_minimind.py (可选，取决于模型导出形式) | ├── pytorch_model.bin or model.safetensors | ├── special_tokens_map.json | ├── tokenizer_config.json | ├── tokenizer.json ``` -* 启动聊天服务端 +* 启动服务端 ```bash - python serve_openai_api.py + cd scripts && python serve_openai_api.py ``` * 测试服务接口 ```bash - python chat_openai_api.py + cd scripts && python chat_api.py ``` -* API接口示例，兼容openai api格式 +* API 请求示例（兼容 OpenAI API 格式） ```bash - curl http://ip:port/v1/chat/completions \ + curl http://localhost:8998/v1/chat/completions \ -H "Content-Type: application/json" \ -d '{ "model": "model-identifier", @@ -1707,104 +1631,172 @@ MiniMind模型本身预训练数据集小的可怜，也没有针对性的对测 { "role": "user", "content": "世界上最高的山是什么？" } ], "temperature": 0.7, - "max_tokens": 512, - "stream": true + "max_tokens": 1024, + "stream": true, + "open_thinking": true }' ``` -## 👨‍💻 更多 +## [SGLang](https://github.com/sgl-project/sglang) -* 🔗从MiniMind-LLM微调扩散语言模型 +SGLang 是高性能大模型推理引擎，支持 RadixAttention、连续批处理等优化技术，能够提供较低延迟与较高吞吐。 -* 🔗模型的generate方法说明 +> ⚠️ 需要 CUDA 环境，按需使用。也可在 RL 训练脚本中选择 SGLang 作为 rollout / 推理引擎以提升训练吞吐。 ---- +以 OpenAI-compatible API server 形式启动模型： + +```bash +python -m sglang.launch_server --model-path /path/to/model --attention-backend triton --host 0.0.0.0 --port 8998 +``` ## [vllm](https://github.com/vllm-project/vllm) -vLLM是极其流行的高效推理框架，支持大模型快速部署，优化显存利用与吞吐量。 +vLLM 是目前非常常用的高效推理框架，适合快速部署大模型，并在显存利用率与吞吐量之间取得较好平衡。 -以openai-serve形式启动 minimind2： +> ⚠️ 需要 CUDA 环境，按需使用。 + +以 OpenAI-compatible API server 形式启动模型： ```bash -vllm serve ./MiniMind2 --model-impl transformers --served-model-name "minimind" --port 8998 +vllm serve /path/to/model --model-impl transformers --served-model-name "minimind" --port 8998 ``` ## [llama.cpp](https://github.com/ggerganov/llama.cpp) -llama.cpp是一个C++库， -可以在命令行下直接使用，支持多线程推理，支持GPU加速。 +llama.cpp 是一个轻量且实用的 C++ 推理框架，可直接在命令行中使用，支持多线程推理，也支持部分 GPU 加速方案。 -**目录结构**：建议将llama.cpp与minimind放在同级目录下 +**目录结构**：建议将 `llama.cpp` 与模型目录放在同级路径下 ``` parent/ -├── minimind/ # MiniMind项目目录 -│ ├── MiniMind2/ # HuggingFace格式MiniMind2模型 (先convert_model.py生成) +├── project/ # 你的项目目录 +│ ├── minimind模型路径/ # HuggingFace 格式模型目录 │ │ ├── config.json │ │ ├── model.safetensors │ │ └── ... -│ ├── model/ -│ ├── trainer/ │ └── ... -└── llama.cpp/ # llama.cpp项目目录 +└── llama.cpp/ # llama.cpp 项目目录 ├── build/ ├── convert_hf_to_gguf.py └── ... ``` -0、参考`llama.cpp`官方步骤进行install +0、参考 `llama.cpp` 官方文档完成安装（如 `cmake` 等依赖） -1、在`convert_hf_to_gguf.py`的`get_vocab_base_pre`函数最后插入： +1、在 `convert_hf_to_gguf.py` 的 `get_vocab_base_pre` 函数末尾插入： ```python -# 添加MiniMind tokenizer支持（这里随便写一个例如qwen2即可） +# 添加 MiniMind tokenizer 支持（此处可临时复用一个兼容项，如 qwen2） if res is None: res = "qwen2" ``` -2、转换自训练的minimind模型：huggingface -> gguf +2、将 HuggingFace 格式的 minimind 模型转换为 GGUF： ```bash -# 在llama.cpp下执行，将生成../minimind/MiniMind2/MiniMind2-xxx.gguf -python convert_hf_to_gguf.py ../minimind/MiniMind2 +# 在 llama.cpp 目录下执行，将在模型目录下生成对应的 gguf 文件 +python convert_hf_to_gguf.py /path/to/minimind-model ``` -3、量化此模型 (可选) +3、量化模型（可选） ```bash -./build/bin/llama-quantize ../minimind/MiniMind2/MiniMind2.gguf ../minimind/MiniMind2/Q4-MiniMind2.gguf Q4_K_M +./build/bin/llama-quantize /path/to/model/xxxx.gguf /path/to/model/xxxx.q8.gguf Q8_0 ``` 4、命令行推理测试 ```bash -./build/bin/llama-cli -m ../minimind/MiniMind2/MiniMind2.gguf -sys "You are a helpful assistant" # system prompt必须固定 +./build/bin/llama-cli -m /path/to/model/xxxx.gguf ``` ## [ollama](https://ollama.ai) -ollama是本地运行大模型的工具，支持多种开源LLM，简单易用。 +Ollama 是本地运行大模型的常用工具，支持多种开源 LLM，使用方式简洁，部署门槛较低。 -1、通过ollama加载自定义的gguf模型 +1、通过 Ollama 加载自定义 GGUF 模型 -在`MiniMind2`下新建`minimind.modelfile`，写入： +在模型目录下新建 `minimind.modelfile` 文件，并写入如下配置模板： + +
+minimind.modelfile (template) ```text -FROM ./Q4-MiniMind2.gguf +FROM /path/to/model/xxxx.gguf -SYSTEM """You are a helpful assistant""" +SYSTEM "你的名字叫MiniMind，你是一个乐于助人、知识渊博的AI助手。请用完整且友好的方式回答用户问题，当被问到名字时请回答MiniMind。" -TEMPLATE """<|im_start|>system + +TEMPLATE """{{- if .Tools }}<|im_start|>system +{{ if .System }}{{ .System }} + +{{ end }}# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } +<|im_end|> +{{ else if .System }}<|im_start|>system {{ .System }}<|im_end|> -<|im_start|>user -{{ .Prompt }}<|im_end|> -<|im_start|>assistant -{{ .Response }}<|im_end|> -""" +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant + +{{ .Thinking }} + + +{{ .Content }} +{{- if .ToolCalls }} +{{- range .ToolCalls }} + +{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} + +{{- end }} +{{- end }} +{{- if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ if and $.IsThinkSet $.Think -}} + +{{ else -}} + + + + +{{ end -}} +{{ end }} +{{- end }}""" + +PARAMETER repeat_penalty 1 +PARAMETER stop "<|im_start|>" +PARAMETER stop "<|im_end|>" +PARAMETER temperature 0.9 +PARAMETER top_p 0.9 +PARAMETER num_ctx 8192 ``` -2、加载并命名此模型为`minimind-local` +
+
+ + +2、加载并命名本地模型 ```bash ollama create -f minimind.modelfile minimind-local @@ -1829,39 +1821,48 @@ ollama push your_username/minimind:latest

-⭐️ 也可直接使用我提供的ollama模型一键启动： +⭐️ 也可以直接使用我提供的 Ollama 模型快速启动： ```bash -ollama run jingyaogong/minimind2 # 其他可选 minimind2-r1 / minimind2-small / minimind2-small-r1 +ollama run jingyaogong/minimind-3 >>> 你叫什么名字 我是一个语言模型... ``` -## [MNN](https://github.com/alibaba/MNN) +## [MNN](https://github.com/alibaba/MNN) -MNN是面向端侧的AI推理引擎，支持多种开源LLM模型推理，轻量化、高性能。 +MNN 是面向端侧的 AI 推理引擎，支持多种开源 LLM 的轻量化部署与高性能推理。 1. 模型转换 -``` +```bash cd MNN/transformers/llm/export -# 导出4bit HQQ量化的MNN模型 -python llmexport.py --path /path/to/MiniMind2/ --export mnn --hqq --dst_path MiniMind2-MNN +# 导出 4bit HQQ 量化的 MNN 模型 +python llmexport.py --path /path/to/模型路径/ --export mnn --hqq --dst_path 模型路径-mnn ``` -2. 在Mac或手机上测试 +2. 在 Mac 或手机端测试 +```bash +./llm_demo /path/to/模型路径-mnn/config.json prompt.txt ``` -./llm_demo /path/to/MiniMind2-MNN/config.json prompt.txt -``` -或者下载APP测试 +或者下载 APP 进行测试 > 以上三方框架的更多用法请参考对应官方文档😊 -# 📌 Acknowledge + +## 👨‍💻 更多内容 + +* 🔗从MiniMind-LLM微调扩散语言模型 + +* 🔗模型的generate方法说明 + +* 🔗从 MiniMind 训练线性注意力模型 + +# 📌 致谢 > [!NOTE] -> 如果觉得`MiniMind系列`对您有所帮助，可以在 GitHub 上加一个⭐
-> 篇幅超长水平有限难免纰漏，欢迎在Issues交流指正或提交PR改进项目
-> 您的小小支持就是持续改进此项目的动力！ +> 如果 `MiniMind` 系列项目对您有所帮助，欢迎在 GitHub 上点亮一个 ⭐
+> 文档篇幅较长，难免存在疏漏之处，欢迎通过 Issues 交流反馈，或提交 PR 一起改进项目
+> 您的支持与建议，都是这个项目持续迭代的重要动力！ ## 🤝[贡献者](https://github.com/jingyaogong/minimind/graphs/contributors) @@ -1871,40 +1872,34 @@ python llmexport.py --path /path/to/MiniMind2/ --export mnn --hqq --dst_path Mi ## 😊鸣谢 -@ipfgao: -🔗训练步骤记录 +感谢以下贡献者在训练记录、数据处理、教程整理与项目拆解等方面提供的帮助与分享： -@WangRongsheng: -🔗大型数据集预处理 +* [@ipfgao](https://github.com/ipfgao)：[🔗训练步骤记录](https://github.com/jingyaogong/minimind/issues/26) -@pengqianhan: -🔗一个简明教程 +* [@WangRongsheng](https://github.com/WangRongsheng)：[🔗大型数据集预处理](https://github.com/jingyaogong/minimind/issues/39) -@RyanSunn: -🔗推理过程学习记录 +* [@pengqianhan](https://github.com/pengqianhan)：[🔗一个简明教程](https://github.com/jingyaogong/minimind/issues/73) -@Nijikadesu: -🔗以交互笔记本方式分解项目代码 +* [@RyanSunn](https://github.com/RyanSunn)：[🔗推理过程学习记录](https://github.com/jingyaogong/minimind/issues/75) + +* [@Nijikadesu](https://github.com/Nijikadesu)：[🔗以交互笔记本方式分解项目代码](https://github.com/jingyaogong/minimind/issues/213) -
- 参考链接 & 感谢以下优秀的论文或项目 +致谢以下优秀的论文与项目： -- 排名不分任何先后顺序 - [https://github.com/meta-llama/llama3](https://github.com/meta-llama/llama3) - [https://github.com/karpathy/llama2.c](https://github.com/karpathy/llama2.c) - [https://github.com/DLLXW/baby-llama2-chinese](https://github.com/DLLXW/baby-llama2-chinese) -- [(DeepSeek-V2)https://arxiv.org/abs/2405.04434](https://arxiv.org/abs/2405.04434) +- [DeepSeek-V2](https://arxiv.org/abs/2405.04434) - [https://github.com/charent/ChatLM-mini-Chinese](https://github.com/charent/ChatLM-mini-Chinese) - [https://github.com/wdndev/tiny-llm-zh](https://github.com/wdndev/tiny-llm-zh) -- [(Mistral-MoE)https://arxiv.org/pdf/2401.04088](https://arxiv.org/pdf/2401.04088) +- [Mistral-MoE](https://arxiv.org/pdf/2401.04088) - [https://github.com/Tongjilibo/build_MiniLLM_from_scratch](https://github.com/Tongjilibo/build_MiniLLM_from_scratch) - [https://github.com/jzhang38/TinyLlama](https://github.com/jzhang38/TinyLlama) - [https://github.com/AI-Study-Han/Zero-Chatgpt](https://github.com/AI-Study-Han/Zero-Chatgpt) - [https://github.com/xusenlinzy/api-for-open-llm](https://github.com/xusenlinzy/api-for-open-llm) - [https://github.com/HqWu-HITCS/Awesome-Chinese-LLM](https://github.com/HqWu-HITCS/Awesome-Chinese-LLM) -
## 🫶支持者 @@ -1912,7 +1907,7 @@ python llmexport.py --path /path/to/MiniMind2/ --export mnn --hqq --dst_path Mi - github contribution grid snake animation + Star poster @@ -1920,7 +1915,7 @@ python llmexport.py --path /path/to/MiniMind2/ --export mnn --hqq --dst_path Mi - github contribution grid snake animation + Fork poster @@ -1930,7 +1925,7 @@ python llmexport.py --path /path/to/MiniMind2/ --export mnn --hqq --dst_path Mi Star History Chart -## 🎉 Awesome Work using MiniMind +## 🎉 MiniMind 相关成果 本模型抛砖引玉地促成了一些可喜成果的落地，感谢研究者们的认可： @@ -1946,24 +1941,27 @@ python llmexport.py --path /path/to/MiniMind2/ --export mnn --hqq --dst_path Mi - FedBRB: A Solution to the Small-to-Large Scenario in Device-Heterogeneity Federated Learning [[TMC 2025](https://ieeexplore.ieee.org/abstract/document/11168259)] +- SKETCH: Semantic Key-Point Conditioning for Long-Horizon Vessel Trajectory Prediction [[arxiv](https://arxiv.org/pdf/2601.18537)] + +- A Built-in Crypto Expert for Artificial Intelligence: How Far is the Horizon? [[IACR ePrint 2026](https://eprint.iacr.org/2026/411.pdf)] + - 进行中... -# 🎓 Citation +# 🎓 引用 -If you find MiniMind helpful in your research or work, please cite: +如果 `MiniMind` 对您的研究或工作有所帮助，欢迎引用： ```bibtex @misc{minimind, - title={MiniMind: Train a Tiny LLM from scratch}, - author={Jingyao Gong}, - year={2024}, - howpublished={https://github.com/jingyaogong/minimind} + title = {MiniMind: Train a Tiny LLM from Scratch}, + author = {Jingyao Gong}, + year = {2024}, + url = {https://github.com/jingyaogong/minimind}, + note = {GitHub repository, accessed 2026} } ``` -# License - -This repository is licensed under the [Apache-2.0 License](LICENSE). - +# ⚖️ 开源协议 +本项目采用 [Apache License 2.0](LICENSE) 开源协议。 diff --git a/README_en.md b/README_en.md index b495a53..20b0089 100644 --- a/README_en.md +++ b/README_en.md @@ -13,7 +13,6 @@ [![GitHub pull request](https://img.shields.io/badge/PRs-welcome-blue)](https://github.com/jingyaogong/minimind/pulls) [![Collection](https://img.shields.io/badge/🤗-MiniMind%20%20Collection-blue)](https://huggingface.co/collections/jingyaogong/minimind-66caf8d999f5c7fa64f399e5) -
@@ -23,7 +22,7 @@
-

"The Simplest Path is the Greatest"

+

"The Great Way is Simple"

@@ -32,192 +31,194 @@
-* This open-source project aims to train a super-small language model **MiniMind** with only 3 RMB cost and 2 hours, - starting completely from scratch. -* The **MiniMind** series is extremely lightweight, with the smallest version being $\frac{1}{7000}$ the size of GPT-3, - making it possible to train quickly on even the most ordinary personal GPUs. -* The project also open-sources the minimalist structure of the large model, including extensions for shared mixed - experts (MoE), dataset cleaning, pretraining, supervised fine-tuning (SFT), LoRA fine-tuning, direct preference - optimization (DPO) algorithms, reinforcement learning from AI feedback (RLAIF: PPO/GRPO/SPO), and model distillation - algorithms, along with the full code of the entire process. -* **MiniMind** also expands into vision multimodal VLM: [MiniMind-V](https://github.com/jingyaogong/minimind-v). -* All core algorithm code is reconstructed from scratch using native PyTorch! It does not rely on abstract interfaces - provided by third-party libraries. -* This is not only a full-stage open-source reproduction of a large language model but also a tutorial for beginners in - LLM. -* We hope this project will serve as an inspiring example for everyone, helping to enjoy the fun of creation and - promoting the progress of the wider AI community! +* This open-source project aims to train an ultra-small language model MiniMind with approximately 64M parameters entirely from scratch, using only 3 CNY in cost and 2 hours of training time. +* The MiniMind series is extremely lightweight, with the smallest version on the main branch being approximately $\frac{1}{2700}$ the size of GPT-3, striving to enable even ordinary personal GPUs to quickly complete training and reproduction. +* The project also open-sources the minimalist structure and complete training pipeline of large models, covering the entire process code for MoE, data cleaning, Pretraining, Supervised Fine-Tuning (SFT), LoRA, RLHF (DPO), RLAIF (PPO / GRPO / CISPO), Tool Use, Agentic RL, Adaptive Thinking, and Model Distillation. +* MiniMind has also been extended to a visual multimodal version [MiniMind-V](https://github.com/jingyaogong/minimind-v). +* All core algorithm code in the project is implemented from scratch using native PyTorch, without relying on high-level abstract interfaces provided by third-party libraries. +* This is not only a full-stage open-source reproduction project for large language models, but also a tutorial oriented towards LLM introduction and practice. +* We hope this project can provide a reproducible, understandable, and extensible starting point for more people, to share the joy of creation together and promote the progress of the broader AI community. -> To avoid misunderstanding, the "2 hours" test is based on NVIDIA 3090 hardware (single GPU), and the "3 RMB" refers to the GPU server rental cost. Details of the specifications can be found below. +> Note: This project is open-sourced under the Apache 2.0 license and is completely free; "2 hours" is estimated based on NVIDIA 3090 hardware (single GPU), and "3 CNY" refers to GPU server rental cost. See below for detailed specifications. ---
-![minimind2](./images/minimind2.gif) +![minimind-3](./images/minimind-3.gif) + +[🔗 Online Demo](https://www.modelscope.cn/studios/gongjy/MiniMind) | [🔗 Video Introduction](https://www.bilibili.com/video/BV12dHPeqE72) -[🔗🍓Reason Model](https://www.modelscope.cn/studios/gongjy/MiniMind-Reasoning) | [🔗🤖Standard Model](https://www.modelscope.cn/studios/gongjy/MiniMind) | [🔗🎞️Video Introduction](https://www.bilibili.com/video/BV12dHPeqE72/?share_source=copy_web&vd_source=670c2504f88726f8cf4a21ef6147c0e8)
- - Hugging Face Logo + + Hugging Face Logo - ModelScope Logo + ModelScope Logo
-
+
-# 📌 Introduction + -The emergence of Large Language Models (LLMs) has sparked unprecedented global attention to AI. -Whether it's ChatGPT, DeepSeek, or Qwen, they all demonstrate stunning performance that is awe-inspiring. -However, with their massive scale of tens of billions of parameters, they are not only difficult to train on personal devices but nearly impossible to deploy. -Opening the "black box" of large models to explore their internal mechanisms is truly thrilling! -Unfortunately, 99% of exploration can only stop at using techniques like LoRA to perform minor fine-tuning on existing large models to learn new instructions or tasks. -This is like teaching Newton how to use a 21st-century smartphone—while interesting, it completely deviates from the original intent of understanding the essence of physics. -Meanwhile, third-party large model frameworks and toolkits, such as transformers+trl, expose only highly abstract interfaces. -With just 10 lines of code, you can complete the entire workflow of "loading model + loading dataset + inference + reinforcement learning." -While such efficient packaging is convenient, it also acts like a high-speed spacecraft, isolating developers from underlying implementations and hindering deep exploration of LLM core code. -Yet, "building a plane with Lego is far more exciting than flying in first class!" -What's worse, the internet is flooded with expensive courses and marketing accounts selling AI tutorials with countless flaws and superficial understanding. -For this reason, this project's original intention is to lower the barrier to entry for LLM learning, allowing everyone to start by understanding every line of code, -to personally train an extremely small language model from scratch. Yes, from **training from scratch**, not just **inference**! -With less than 3 RMB in server costs, you can personally experience the entire process of building a language model from 0 to 1. -Let's enjoy the fun of creation together! +--- -> [!NOTE] -> (As of 2025-10) The MiniMind series has completed pretraining of multiple model variants, with the smallest being only 25.8M (0.02B), capable of fluent conversation! +# 📌 Project Introduction -
-Models List +The emergence of Large Language Models (LLMs) has triggered unprecedented global attention on AI. Whether it is ChatGPT, DeepSeek, or Qwen, they have all impressed people with their stunning performance, making them truly feel the impact of this technological wave. However, models with hundreds of billions of parameters make them not only difficult to train on personal devices, but even deployment seems out of reach. Opening the "black box" of large models and truly understanding their internal working mechanisms should be something exciting. Unfortunately, the vast majority of explorations ultimately stop at using techniques like LoRA to do minimal fine-tuning on existing large models, learning some new instructions or specific tasks. This is more like teaching Newton how to use a 21st-century smartphone — interesting, but deviating from the original intention of understanding the essence of physics. -| Model (Size) | Inference Memory (Approx) | Release | -|------------------------|---------------------------|------------| -| MiniMind2-small (26M) | 0.5 GB | 2025.04.26 | -| MiniMind2-MoE (145M) | 1.0 GB | 2025.04.26 | -| MiniMind2 (104M) | 1.0 GB | 2025.04.26 | -| minimind-v1-small (26M)| 0.5 GB | 2024.08.28 | -| minimind-v1-moe (4×26M)| 1.0 GB | 2024.09.17 | -| minimind-v1 (108M) | 1.0 GB | 2024.09.01 | +Meanwhile, third-party large model frameworks and tool libraries, such as `transformers` / `trl` / `peft`, often only expose highly abstract interfaces. With just a dozen or so lines of code, one can complete the entire pipeline of "load model + load dataset + inference + reinforcement learning" training. While this efficient encapsulation is convenient, it also to some extent isolates developers from the underlying implementation, weakening opportunities to deeply understand the core code of LLMs. I believe "building an airplane from Lego bricks yourself is far more exciting than flying in first class", yet a more practical problem is that the internet is flooded with paid courses and marketing content, wrapping so-called AI tutorials with flawed, half-baked explanations. For this reason, the original intention of this project is to lower the learning barrier for LLMs as much as possible, allowing everyone to start from understanding every line of code and train a tiny language model from scratch. Yes, **training from scratch**, not just staying at the **inference** level. With a server cost as low as 3 CNY, you can experience the entire process of building a language model from 0 to 1 firsthand. + +😊 Let's share the joy of creation together! + +--- + +#### 🎉 This Project Includes the Following + +- Provides complete MiniMind-LLM structure code (Dense + MoE), with the current main branch structure aligned with the `Qwen3 / Qwen3-MoE` ecosystem. +- Provides Tokenizer and tokenizer training code, supporting template tokens such as ``, ``, ``, etc. +- Covers complete training pipelines including Pretrain, SFT, LoRA, RLHF-DPO, RLAIF (PPO / GRPO / CISPO), Tool Use, Agentic RL, Adaptive Thinking, and Model Distillation. +- Provides open-source data for all stages, covering collected, distilled, cleaned, and deduplicated high-quality datasets. +- Key training algorithms and core modules are all implemented from scratch, without relying on third-party framework wrappers. +- Compatible with mainstream frameworks such as `transformers`, `trl`, `peft`, as well as commonly used inference engines like `llama.cpp`, `vllm`, `ollama`, and training frameworks like `Llama-Factory`. +- Supports single-machine single-GPU and single-machine multi-GPU (DDP, DeepSpeed) training, supports wandb / swanlab visualization and dynamic start/stop of training. +- Supports evaluation on third-party benchmark suites such as C-Eval, C-MMLU, OpenBookQA, etc., and supports RoPE long context extrapolation through YaRN. +- Provides a minimalist server compatible with the OpenAI API protocol, convenient for integrating with third-party Chat UIs such as FastGPT, Open-WebUI, etc., and supports `reasoning_content`, `tool_calls`, `open_thinking`. +- Provides a minimalist chat WebUI based on Streamlit, supporting thinking display, tool selection, and multi-turn Tool Call. + +#### 🎉 Released Model List + +| Model | Parameters | Release | +|------|--------|---------| +| minimind-3 | 64M | 2026.04.01 | +| minimind-3-moe | 198M / A64M | 2026.04.01 | +| minimind2-small | 26M | 2025.04.26 | +| minimind2-moe | 145M | 2025.04.26 | +| minimind2 | 104M | 2025.04.26 | +| minimind-v1-small | 26M | 2024.08.28 | +| minimind-v1-moe | 4×26M | 2024.09.17 | +| minimind-v1 | 108M | 2024.09.01 | + +--- + +#### 📝 Changelog + +
+ 🔥 2026-04-01 + + - Released `minimind-3` / `minimind-3-moe`: comprehensive updates to structure, Tokenizer, training pipeline, inference interface, and default configuration +- Main branch structure aligned with `Qwen3 / Qwen3-MoE` ecosystem: Dense approximately `64M`, MoE approximately `198M / A64M`, and removed shared expert design +- Default training data switched to `pretrain_t2t(_mini).jsonl`, `sft_t2t(_mini).jsonl`, `rlaif.jsonl`, `agent_rl.jsonl`, and `agent_rl_math.jsonl` +- Removed standalone `train_reason.py`; thinking capability is now unified through `chat_template + ` and `open_thinking` adaptive switch control +- `toolcall` capability has been merged into `sft_t2t / sft_t2t_mini` main branch data, default `full_sft` already has basic Tool Call capability; also added inference examples such as `scripts/chat_api.py` +- Added native `Agentic RL` training script `train_agent.py`, supporting `GRPO / CISPO` in multi-turn Tool-Use scenarios +- RLAIF / Agentic RL training pipeline completed `rollout engine` decoupling, supporting more flexible switching of generation backends +- `serve_openai_api.py` and `web_demo.py` added `reasoning_content` / `tool_calls` / `open_thinking` support +- Tokenizer updated based on `BPE + ByteLevel`, with new tool call and thinking tokens, reserved buffer tokens for future extension +- Added LoRA weight merging and export pipeline, can merge base model and LoRA weights into new complete model weights via `scripts/convert_model.py` +- Structure diagram resources updated, README extensively updated
-**Project Includes** - -- Complete code for MiniMind-LLM structure (Dense + MoE models). -- Detailed training code for Tokenizer. -- Complete training code for Pretrain, SFT, LoRA, RLHF-DPO, RLAIF (PPO/GRPO/SPO), and model distillation. -- Collected, distilled, organized and cleaned high-quality datasets for all stages, all open-sourced. -- Implemented from scratch: pretraining, instruction fine-tuning, LoRA, DPO/PPO/GRPO/SPO reinforcement learning, and white-box model distillation. Core algorithms barely depend on third-party framework encapsulation, all open-sourced. -- Compatible with mainstream third-party frameworks like `transformers`, `trl`, `peft`. -- Training supports single GPU, multiple GPUs on a single machine (DDP, DeepSpeed), supports wandb/swanlab visualization of training process. Supports dynamic training start/stop. -- Model testing on third-party evaluation leaderboards (C-Eval, C-MMLU, OpenBookQA, etc.), supports YaRN algorithm for RoPE long-text extrapolation. -- Implements an extremely simple OpenAI API-compliant server, convenient for integration with third-party ChatUI (FastGPT, Open-WebUI, etc.). -- Implements the simplest chat WebUI frontend based on streamlit. -- Fully compatible with popular community inference engines `llama.cpp`, `vllm`, `ollama` or training framework `Llama-Factory`. -- Reproduced (distilled/RL) DeepSeek-R1 reasoning model as MiniMind-Reason model, with **data + models** fully open-sourced! - -We hope this open-source project can help LLM beginners get started quickly! - -### 👉**Update Log** - -
+
2025-10-24 -- 🔥 Added RLAIF training algorithms: PPO, GRPO, SPO (native implementation from scratch) -- Added checkpoint resume training: supports automatic training recovery, cross-GPU recovery, wandb continuity -- Added RLAIF dataset: rlaif-mini.jsonl (randomly sampled 10,000 entries from SFT data); simplified DPO dataset with Chinese data -- Added YaRN algorithm: supports RoPE long-text extrapolation, improving long sequence handling capability -- Adaptive Thinking: Reason model can optionally enable thinking chain +- 🔥 Added RLAIF training algorithms: PPO, GRPO, SPO (natively implemented from scratch) +- Added checkpoint resume functionality: supports automatic training recovery, cross-GPU-count recovery, wandb record continuity +- Added RLAIF dataset: rlaif-mini.jsonl (randomly sampled 10,000 entries from SFT data); simplified DPO dataset, added Chinese data +- Added YaRN algorithm: supports RoPE long context extrapolation, improving long sequence processing capability +- Adaptive Thinking: Reason model optionally enables chain of thought - chat_template fully supports Tool Calling and Reasoning tags (``, ``, etc.) -- Added complete RLAIF chapter, training curve comparison, algorithm principle explanations -- [SwanLab](https://swanlab.cn/) replaces WandB (friendly for domestic access, fully compatible API) -- Code standardization & fixed some known bugs +- Added complete RLAIF chapter, training curve comparison, folded algorithm principle explanations +- [SwanLab](https://swanlab.cn/) replaces WandB (domestic access friendly, API fully compatible) +- Standardized all code & fixed some known bugs
-
+
2025-04-26 -- Important update -- For compatibility needs, you can visit [🔗old repository content🔗](https://github.com/jingyaogong/minimind/tree/7da201a944a90ed49daef8a0265c959288dff83a). +- Major update +- For compatibility needs, visit [🔗Old Repository Content🔗](https://github.com/jingyaogong/minimind/tree/7da201a944a90ed49daef8a0265c959288dff83a). - MiniMind model parameters completely renamed, aligned with Transformers library models (unified naming). - generate method refactored, inheriting from GenerationMixin class. -- 🔥 Supports popular third-party ecosystems like llama.cpp, vllm, ollama. +- 🔥Supports popular third-party ecosystems such as llama.cpp, vllm, ollama. - Standardized code and directory structure. -- Modified vocabulary ``->`<|im_start|><|im_end|>` +- Changed vocabulary `` -> `<|im_start|><|im_end|>` ```text -To be compatible with third-party inference frameworks llama.cpp and vllm, this update requires some observable costs. -This update no longer supports "directly" loading old models before 25-04-26 for inference. -Due to differences in Llama's positional encoding compared to minimind, there are differences in QK values after mapping Llama models. -MiniMind2 series old models have been recovered through weight mapping and (fine-tuning training) QKVO linear layer calibration. -After this update, maintenance of the entire minimind-v1 series will be abandoned and removed from the repository. +To be compatible with third-party inference frameworks llama.cpp, vllm, this update comes with some considerable costs. +This update no longer supports "directly" loading old models from before 25-04-26 for inference. +Due to differences between Llama's positional encoding method and minimind's, QK values differ after mapping to the Llama model. +The minimind2 series old models were all recovered through weight mapping + (fine-tuning) QKVO linear layer calibration. +After this update, maintenance for the entire `minimind-v1` series will be discontinued and taken offline from the repository. ```
-
+
2025-02-09 -- Major update since release, Release MiniMind2 Series. -- Code almost completely refactored, using cleaner and more unified structure. - For compatibility with old code, you can visit [🔗old repository content🔗](https://github.com/jingyaogong/minimind/tree/6e9cd28ef9b34a0a10afbdf6f59e65cb6e628efb). -- Eliminated data preprocessing steps. Unified dataset format, switched to `jsonl` format to avoid dataset download confusion. -- MiniMind2 series shows significant improvement compared to MiniMind-V1. -- Minor improvements: {more standard kv-cache writing, MoE load balancing loss considered, etc.} -- Provides training solutions for model migration to private datasets (medical models, self-awareness examples). -- Streamlined pretraining dataset and significantly improved pretraining data quality, greatly reducing time for quick personal training, single 3090 GPU can reproduce in 2 hours! -- Updates: LoRA fine-tuning separated from peft packaging, implemented from scratch; DPO algorithm implemented from scratch using native PyTorch; white-box model distillation native implementation. -- MiniMind2-DeepSeek-R1 series distilled models born! -- MiniMind2 now has some English ability! -- Updated MiniMind2 and third-party model performance results based on more large model leaderboard tests. +- Major update since release, Release minimind2 Series. +- Code almost entirely refactored, using a more concise and clear unified structure. + For compatibility needs with old code, visit [🔗Old Repository Content🔗](https://github.com/jingyaogong/minimind/tree/6e9cd28ef9b34a0a10afbdf6f59e65cb6e628efb). +- Eliminated data preprocessing steps. Unified dataset format, switched to `jsonl` format to avoid dataset download confusion issues. +- minimind2 series significantly improved performance compared to MiniMind-V1. +- Minor issues: {kv-cache implementation more standard, MoE load balancing loss now considered, etc.} +- Provides training solution for migrating models to private datasets (medical model, self-awareness examples). +- Streamlined pretraining dataset and significantly improved pretraining data quality, greatly reduced time needed for individual quick training, reproducible in 2 hours on a single 3090! +- Updated: LoRA fine-tuning decoupled from peft wrapper, LoRA process implemented from scratch; DPO algorithm natively implemented from scratch using PyTorch; model white-box distillation natively implemented. +- minimind2-DeepSeek-R1 series distilled models born! +- minimind2 has certain English language capability! +- Updated benchmark test performance results of minimind2 vs third-party models on more LLM leaderboards.
-
+
More... **2024-10-05** -- Extended MiniMind with multimodal capabilities---Vision -- Check out the twin project [minimind-v](https://github.com/jingyaogong/minimind-v) for details! +- Extended multimodal capability for MiniMind --- Vision +- Visit the sibling project [minimind-v](https://github.com/jingyaogong/minimind-v) for details! **2024-09-27** -- 09-27 updated the preprocessing method for the pretrain dataset, abandoned preprocessing into .bin format for training to ensure text integrity (slightly sacrificing training speed). -- Current pretrain preprocessing file is named: pretrain_data.csv. +- 09-27 updated pretrain dataset preprocessing method, to ensure text integrity, abandoned preprocessing into .bin format for training (slight sacrifice in training speed). +- Currently the pretrain preprocessed file is named: pretrain_data.csv. - Removed some redundant code. **2024-09-17** - Updated minimind-v1-moe model -- To avoid ambiguity, no longer using mistral_tokenizer for tokenization, completely using custom minimind_tokenizer as the tokenizer. +- To prevent ambiguity, mistral_tokenizer is no longer used for tokenization, all using custom minimind_tokenizer as the tokenizer. **2024-09-01** -- Updated minimind-v1 (108M) model, using minimind_tokenizer, 3 pretraining rounds + 10 SFT rounds, more thorough training, stronger performance. -- Project has been deployed to ModelScope creation space, you can experience it on this website: -- [🔗ModelScope Online Experience🔗](https://www.modelscope.cn/studios/gongjy/minimind) +- Updated minimind-v1 (108M) model, using minimind_tokenizer, pretrain epochs 3 + SFT epochs 10, more thorough training, stronger performance. +- Project deployed to ModelScope Creative Space, can be experienced at this website: +- [🔗ModelScope Online Demo🔗](https://www.modelscope.cn/studios/gongjy/minimind) **2024-08-27** - Project first open-sourced
+--- + # 📌 Quick Start -
-Share my hardware and software configuration (for reference only) +
+My hardware and software configuration (for reference) * CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz * RAM: 128 GB -* GPU: NVIDIA GeForce RTX 3090(24GB) * 8 +* GPU: NVIDIA GeForce RTX 3090 (24GB) * 8 * Ubuntu==20.04 * CUDA==12.2 * Python==3.10.16 @@ -225,297 +226,247 @@ After this update, maintenance of the entire minimind-v1 series will be abandone
-### Step 0 +## Step 0 ```bash -git clone https://github.com/jingyaogong/minimind.git +# Clone repository and install dependencies +git clone --depth 1 https://github.com/jingyaogong/minimind +cd minimind && pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple ``` -## Ⅰ Testing Existing Model Performance +## Ⅰ 🚀 Model Inference -### 1. Environment Setup +### 1' Download Model + +In the project root directory: +```bash +# Method 1 +modelscope download --model gongjy/minimind-3 --local_dir ./minimind-3 +# Method 2 +git clone https://huggingface.co/jingyaogong/minimind-3 +``` + +### 2' CLI Inference ```bash -pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple +# Method 1: Using Transformers format model +python eval_llm.py --load_from ./minimind-3 +# Method 2: Based on PyTorch model (ensure corresponding weights are in the ./out directory) +python eval_llm.py --load_from ./model --weight full_sft ``` -### 2. Download Model - -Go to the project root directory +### 3' (Optional) WebUI ```bash -git clone https://huggingface.co/jingyaogong/MiniMind2 # or https://www.modelscope.cn/models/gongjy/MiniMind2 +# May need `python>=3.10`, install `pip install streamlit` +# ⚠️ You must first copy the transformers-format model folder into ./scripts/ (e.g.: cp -r minimind-3 ./scripts/minimind-3). The web_demo script will auto-scan subdirectories containing weight files; it will throw an error if none are found. +cd scripts && streamlit run web_demo.py ``` -### (Optional) Command Line Q&A - -```bash -# Use transformers format model -python eval_llm.py --load_from ./MiniMind2 -``` - -### (Optional) Launch WebUI - -```bash -# May require `python>=3.10`, install with `pip install streamlit` -# cd scripts -streamlit run web_demo.py -``` - -### (Optional) Third-party Inference Frameworks +### 4' (Optional) Third-party Inference Frameworks ```bash # ollama -ollama run jingyaogong/minimind2 +ollama run jingyaogong/minimind-3 # vllm -vllm serve ./MiniMind2/ --served-model-name "minimind" +vllm serve /path/to/model --served-model-name "minimind" ``` -## Ⅱ Train from Scratch Yourself +## Ⅱ 🛠️ Model Training -### 1. Environment Setup +
+Note: Confirm Torch's available backend in advance -```bash -pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple -``` - -
-Note: Test Torch CUDA availability in advance - -```bash +```python import torch print(torch.cuda.is_available()) ``` -If not available, please download and install the whl file from [torch_stable](https://download.pytorch.org/whl/torch_stable.html). Reference [link](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=%E5%AE%89%E8%A3%85torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187) +If you plan to use CUDA for training, it is recommended to first confirm whether the current environment has correctly recognized the GPU. +If `cuda` is not available, you can still choose `CPU` or `MPS` to run based on your device, but training speed and compatibility will differ significantly. +If you need to install or switch PyTorch versions, refer to [torch_stable](https://download.pytorch.org/whl/torch_stable.html) and [this link](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=%E5%AE%89%E8%A3%85torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187)
-### 2. Download Data +### 1' Download Data -Download the required data files from the [dataset download link](https://www.modelscope.cn/datasets/gongjy/minimind_dataset/files) provided below (create the `./dataset` directory) and place them in `./dataset` +Download the required data files from the [dataset download link](https://www.modelscope.cn/datasets/gongjy/minimind_dataset/files) provided below, and place them in the `./dataset` directory -
-Note: Dataset Notes +> Currently, by default, you only need to download `pretrain_t2t_mini.jsonl` and `sft_t2t_mini.jsonl` to quickly reproduce the `MiniMind Zero` dialogue model. +If you have more needs, various combination schemes are provided below, which can be flexibly chosen based on your task objectives and GPU resources. -By default, it is recommended to download `pretrain_hq.jsonl` + `sft_mini_512.jsonl` for the fastest reproduction of the Zero chat model. +### 2' Start Training -You can freely choose data files. The section below provides multiple combination schemes that can be appropriately combined based on your training needs and GPU resources. +
+💡 Checkpoint Pause and Resume -
- -### 3. Start Training - -Directory is located in `trainer` - -
-💡 Checkpoint Resume Training - -All training scripts automatically save checkpoints. Simply add `--from_resume 1` parameter to automatically detect, load & resume training: +All training scripts support checkpoint saving. After adding the `--from_resume 1` parameter, training progress can be automatically detected and resumed: ```bash python train_pretrain.py --from_resume 1 python train_full_sft.py --from_resume 1 -... +# ... ``` -**Checkpoint Resume Mechanism:** -- Training process automatically saves complete checkpoints in `./checkpoints/` directory (model, optimizer, training progress, etc.) +**Checkpoint Resume Instructions:** +- The training process automatically saves complete checkpoints (model, optimizer, training progress, etc.) in the `./checkpoints/` directory - Checkpoint file naming: `__resume.pth` (e.g., `full_sft_512_resume.pth`) -- Supports cross-GPU recovery (automatically adjusts step) -- Supports wandb training log continuity (automatically resumes the same run) +- Supports recovery across different GPU counts (automatically adjusts step) +- Supports wandb training record continuity (automatically resumes the same run) -> Suitable for long training sessions or unstable environments, no need to worry about progress loss from interruptions +> Suitable for long-duration training or unstable environments, no need to worry about progress loss due to training interruption
-**3.1 Pretraining (Learning Knowledge)** +#### 2.1 Pretraining (Required) ```bash -python train_pretrain.py +cd trainer && python train_pretrain.py ``` -> Execute pretraining to get `pretrain_*.pth` as the output weights for pretraining (where * is the model's dimension, default is 512) +> After training, `out/pretrain_*.pth` will be produced as output weights (where `*` is the model dimension, default `768`) -**3.2 Supervised Fine-tuning (Learning Conversation Style)** +#### 2.2 Instruction Fine-tuning (Required) ```bash -python train_full_sft.py +cd trainer && python train_full_sft.py ``` -> Execute supervised fine-tuning to get `full_sft_*.pth` as the output weights for instruction fine-tuning (where `full` means full-parameter fine-tuning) +> After training, `out/full_sft_*.pth` will be produced as output weights (where `full` indicates full-parameter fine-tuning) -
-Note: Training Notes +#### 2.3 Test Trained Model (Optional) -By default, all training processes save parameters to the file `./out/***.pth` every 100 steps (each save overwrites the old weights). - -For simplicity, only the two-stage training process is described here. For other training (LoRA, distillation, reinforcement learning, inference fine-tuning, etc.), refer to the detailed description in the [Experiment](#-experiment) section below. - -
- ---- - -### 4. Test Your Trained Model - -Ensure the model `*.pth` files to be tested are in the `./out/` directory. -You can also directly download and use the `*.pth` files I trained from [here](https://www.modelscope.cn/models/gongjy/MiniMind2-PyTorch/files). +Ensure the model `*.pth` files to be tested are located in the `./out/` directory; you can also go directly to [here](https://www.modelscope.cn/models/gongjy/minimind-3-pytorch/files) to download my pre-trained `*.pth` weights. ```bash -python eval_llm.py --weight full_sft # or pretrain/dpo/ppo/grpo... +python eval_llm.py --weight full_sft ``` -
-Note: Testing Notes +> `--weight` is used to specify the weight name prefix, such as `pretrain`, `full_sft`, etc.; for more parameters, refer directly to `eval_llm.py` -The `--weight` parameter specifies the weight name prefix. Options: `pretrain`, `full_sft`, `dpo`, `reason`, `ppo_actor`, `grpo`, `spo`, etc. +
+Note: Other Information -Other common parameters: -- `--load_from`: Model loading path (`model`=native torch weights, other paths=transformers format) -- `--save_dir`: Model weight directory (default `out`) -- `--lora_weight`: LoRA weight name (`None` means not used) -- `--historys`: Number of historical dialogue rounds to carry (must be even, 0 means no history) -- `--max_new_tokens`: Maximum generation length (default 8192) -- `--temperature`: Generation temperature (default 0.85) -- `--top_p`: Nucleus sampling threshold (default 0.85) +1. All training scripts are natively implemented based on PyTorch and support multi-GPU acceleration. - -For usage details, refer directly to the `eval_llm.py` code. - -
- ---- - -> [!TIP] -> All training scripts are native PyTorch framework, supporting multi-GPU acceleration. Assume your device has N (N > 1) GPUs: - -Single machine N GPU training startup (DDP, supports multi-machine multi-GPU cluster) +2. If your device has `N (N > 1)` GPUs, you can start single-machine `N`-GPU training as follows (DDP, also supports extension to multi-machine multi-GPU): ```bash torchrun --nproc_per_node N train_xxx.py ``` -
-Note: Other Notes - - -Single machine N GPU training (DeepSpeed) +3. You can enable wandb to record the training process as needed. ```bash -deepspeed --master_port 29500 --num_gpus=N train_xxx.py +... train_xxx.py --use_wandb ``` - - -You can optionally enable wandb to record the training process (requires direct internet connection) - -```bash -# Requires login: wandb login -torchrun --nproc_per_node N train_xxx.py --use_wandb -# and -python train_xxx.py --use_wandb -``` - -By adding the `--use_wandb` parameter, you can record the training process. After training is complete, you can view the training process on the wandb website. By modifying the `wandb_project` and `wandb_run_name` parameters, you can specify the project name and run name. - -[Note]: After June 2025, the domestic network environment cannot directly connect to WandB. The MiniMind project by default switches to using [SwanLab](https://swanlab.cn/) as the training visualization tool (fully compatible with WandB API), that is, just change `import wandb` to `import swanlab as wandb`, no other changes are needed. +After June `2025`, domestic network environments in China typically cannot directly connect to WandB. MiniMind currently defaults to using [SwanLab](https://swanlab.cn/) as the training visualization tool, whose interface is basically compatible with WandB; usually you only need to replace `import wandb` with `import swanlab as wandb`, and other usage remains largely unchanged.
+--- + # 📌 Data Introduction ## Ⅰ Tokenizer -Tokenizer maps words from natural language to numbers like `0, 1, 36` through a "dictionary," which can be understood as numbers representing the page number of the word in the "dictionary." -You can choose to construct your own vocabulary table to train a "dictionary." The code can be found in `./trainer/train_tokenizer.py` (for learning reference only. It's not necessary to train one yourself unless required. MiniMind comes with a built-in tokenizer). -Or you can choose tokenizers from well-known open-source large models. -Just as using Xinhua/Oxford dictionaries directly has the advantage of good token encoding compression, but the disadvantage of having too many pages—tens of thousands of word phrases; -A self-trained tokenizer has the advantage of freely controlling vocabulary length and content, but the disadvantage of low compression ratio (for example, "hello" might be split into "h e l l o" -five independent tokens), and rare words are difficult to cover. -The choice of "dictionary" is important. The output of LLM is essentially a multi-class classification problem with SoftMax to N words in the dictionary, then decoding to natural language through the "dictionary." -Because MiniMind size needs to be strictly controlled to avoid top-heavy models (embedding layer parameters taking up too high a proportion of LLM), shorter vocabulary lengths are better. +A tokenizer can be roughly understood as a "dictionary" used by LLMs, responsible for mapping natural language to token ids and decoding token ids back to text; the project also provides `train_tokenizer.py` as a vocabulary training example. It is not recommended to retrain the tokenizer, because once the vocabulary and segmentation rules change, model weights, data formats, inference interfaces, and community ecosystem compatibility will all decrease, and it will also weaken the model's dissemination. Meanwhile, the tokenizer also affects metrics like PPL that are calculated per token, so when comparing across tokenizers, BPB (Bits Per Byte) is often more referential. See [this article](https://skeptric.com/perplexity/). +For small models like MiniMind, vocabulary size also directly affects the parameter proportion of the embedding and output layers, so keeping the vocabulary compact is usually a more appropriate trade-off. -
+
Tokenizer Introduction -The tokenizer vocabulary sizes of powerful open-source models from third parties such as Yi, qwen, chatglm, mistral, and Llama3 are as follows: +The tokenizer vocabulary sizes of powerful third-party open-source models such as Yi, Qwen2, ChatGLM, Mistral, and Llama 3 are as follows: - - - - - - + + + + + +
Tokenizer ModelVocabulary SizeSource
yi tokenizer64,00001AI (China)
qwen2 tokenizer151,643Alibaba Cloud (China)
glm tokenizer151,329Zhipu AI (China)
mistral tokenizer32,000Mistral AI (France)
llama3 tokenizer128,000Meta (USA)
minimind tokenizer6,400Custom
Yi64,00001.AI (China)
Qwen2151,643Alibaba Cloud (China)
ChatGLM151,329Zhipu AI (China)
Mistral32,000Mistral AI (France)
Llama 3128,000Meta (USA)
MiniMind6,400Custom
-> 👉 Updated 2024-09-17: To prevent ambiguity from previous versions and control size, all MiniMind models use minimind_tokenizer for tokenization, abandoning all mistral_tokenizer versions. +> The current main branch uses `minimind_tokenizer` uniformly to avoid historical version ambiguity and control overall size, and no longer maintains the `mistral_tokenizer` version. -``` -# Some thoughts -> Although minimind_tokenizer has a small length, its encoding/decoding efficiency is weaker than Chinese-friendly tokenizers like qwen2 and glm. -> But the minimind model chose the self-trained minimind_tokenizer as the tokenizer to maintain lightweight overall parameters, avoiding imbalance in encoding layer and computation layer proportions, preventing top-heavy models, because minimind's vocabulary size is only 6400. -> And minimind has never encountered rare word decoding failures in actual testing, with good results. -> Due to the custom vocabulary compression to 6400, the total LLM parameters are as low as 25.8M. -> The training data `pretrain_hq.jsonl` all comes from the `JiangShu large model dataset`, this part of data is relatively secondary. You can freely choose if you need to train. -``` +Although `minimind_tokenizer` has a vocabulary of only `6400`, and its encoding/decoding efficiency is weaker than more Chinese-friendly tokenizers like `qwen2` and `glm`, it can significantly compress the parameter proportion of the embedding and output layers, making it more suitable for the size constraints of small models like MiniMind. +From actual usage, this tokenizer has not noticeably caused rare word decoding failures, and remains sufficiently stable and usable overall; therefore the current main branch training also uniformly uses this vocabulary, rather than maintaining additional tokenizer version forks.
-## Ⅱ Pretraining Data +## Ⅱ Pretrain Data -Having learned from MiniMind-V1's low-quality pretraining data that caused models to talk nonsense, after `2025-02-05` we decided no longer to use large-scale unsupervised datasets for pretraining. -Instead, we tried to extract the Chinese portion from the [JiangShu Large Model Dataset](https://www.modelscope.cn/datasets/deepctrl/deepctrl-sft-data), -Clean out about 1.6GB of corpus with character length `<512` and concatenate them directly as pretraining data `pretrain_hq.jsonl`, where hq means high -quality (of course it's not yet high, improving data quality is endless). +`MiniMind-3` current main branch pretraining data is `pretrain_t2t.jsonl` / `pretrain_t2t_mini.jsonl`. +These two datasets have been organized into a unified `text -> next token prediction` training format, aiming to balance under limited compute: -The file `pretrain_hq.jsonl` data format is +- Text quality; +- Length distribution; +- Chinese-English mixed capability; +- Template alignment with subsequent SFT / Tool Calling / RLAIF stages. -```json -{"text": "How can I get rid of procrastination? Curing procrastination is not easy, but the following suggestions may help..."} +Data sources include but are not limited to general text corpora, dialogue organized corpora, distillation supplementary corpora, and various datasets available under **permissive open-source licenses**; main branch data enters training only after cleaning, deduplication, length control, and format unification. Data sources include: [Craftsman LLM Dataset](https://www.modelscope.cn/datasets/deepctrl/deepctrl-sft-data), [Magpie-Align](https://www.modelscope.cn/organization/Magpie-Align), and other public data sources. + +Among them: + +- `pretrain_t2t_mini.jsonl` is more suitable for quick reproduction; +- `pretrain_t2t.jsonl` is more suitable for full training of the `MiniMind-3` main branch model. + +File data format is + +```jsonl +{"text": "如何才能摆脱拖延症？治愈拖延症并不容易，但以下建议可能有所帮助。"} +{"text": "清晨的阳光透过窗帘洒进房间，桌上的书页被风轻轻翻动。"} +{"text": "Transformer 通过自注意力机制建模上下文关系，是现代大语言模型的重要基础结构。"} ``` ## Ⅲ SFT Data -The [JiangShu Large Model SFT Dataset](https://www.modelscope.cn/datasets/deepctrl/deepctrl-sft-data) -"is a complete, uniformly formatted, and safe large model training and research resource. -It collected and organized a large amount of open-source datasets from public sources on the internet, unified their format, cleaned the data, -containing Chinese datasets with 10M entries and English datasets with 2M entries." -The above is the official introduction. After downloading, the total data volume is about 4B tokens, which is definitely suitable as SFT data for Chinese large language models. -However, the official data format is messy, and using all of it for SFT would be too expensive. -I performed secondary cleaning of the official dataset, removing entries with symbol pollution and noise; additionally, still only kept content with total length `<512`, -hoping to supplement knowledge lacking in the pretraining phase through large amounts of dialogue at this stage. -Export file is `sft_512.jsonl` (~7.5GB). +`MiniMind-3` current main branch SFT data is `sft_t2t.jsonl` / `sft_t2t_mini.jsonl`. Compared to earlier `sft_512 / sft_1024 / sft_2048` schemes, the current version places more emphasis on: -The [Magpie-SFT Dataset](https://www.modelscope.cn/organization/Magpie-Align) -collected ~1M high-quality conversations from Qwen2/2.5. I further cleaned this data, exporting the portion with total length `<2048` as `sft_2048.jsonl` (~9GB). -The portion with length `<1024` exported as `sft_1024.jsonl` (~5.5GB). Using large model dialogue data directly for sft falls into the "black-box distillation" category. +- Unified templates; +- Better suited for mixed training of dialogue + thinking tags + Tool Calling; +- Minimizing data preprocessing forks, reducing reproduction costs. -Further cleaned the SFT data from the previous two steps (keeping only content with high Chinese character ratio), filtered conversations with length `<512`, and obtained `sft_mini_512.jsonl` (~1.2GB). +Its data sources include but are not limited to high-quality instruction-following data, public dialogue data, model distillation synthetic data, and license-friendly open-source datasets; before entering the `t2t` main branch, they are unified into the multi-turn dialogue format used by the current repository. The current main branch also contains a large amount of synthetic data, such as approximately `100K` `tool call` entries I synthesized based on `qwen3-4b`, as well as `reasoning` data from the `qwen3` series, etc. Major community sources include: [Craftsman LLM Dataset](https://www.modelscope.cn/datasets/deepctrl/deepctrl-sft-data), [Magpie-Align](https://www.modelscope.cn/organization/Magpie-Align), [R1-Distill-SFT](https://www.modelscope.cn/datasets/AI-ModelScope/R1-Distill-SFT), [COIG](https://huggingface.co/datasets/BAAI/COIG), [Step-3.5-Flash-SFT](https://huggingface.co/datasets/stepfun-ai/Step-3.5-Flash-SFT), etc. Published versions ensure that data sources and processing pipelines comply with the transitivity constraints of corresponding open-source licenses, and adhere to Apache-2.0, CC-BY-NC-2.0, and other related license requirements. -The data format for all sft files `sft_X.jsonl` is +Among them: -```text +- `sft_t2t_mini.jsonl`: suitable for quickly training a dialogue model; +- `sft_t2t.jsonl`: suitable for fully reproducing the main branch version; +- `toolcall` capability has already been merged into the main branch SFT data. + +All SFT file data formats are (including dialogue data, Tool Use data) + +```jsonl { "conversations": [ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hello!"}, - {"role": "user", "content": "Goodbye"}, - {"role": "assistant", "content": "Goodbye!"} + {"role": "user", "content": "你好"}, + {"role": "assistant", "content": "你好！"}, + {"role": "user", "content": "再见"}, + {"role": "assistant", "content": "再见！"} + ] +} +{ + "conversations": [ + {"role": "system", "content": "# Tools ...", "tools": "[...]"}, + {"role": "user", "content": "把'你好世界'翻译成english"}, + {"role": "assistant", "content": "", "tool_calls": "[{\"name\":\"translate_text\",\"arguments\":{\"text\":\"你好世界\",\"target_language\":\"english\"}}]"}, + {"role": "tool", "content": "{\"translated_text\":\"Hello World\"}"}, + {"role": "assistant", "content": "Hello World"} ] } ``` -## Ⅳ RLHF Data +## Ⅳ RL Data -From the [Magpie-DPO Dataset](https://www.modelscope.cn/datasets/Magpie-Align/MagpieLM-DPO-Data-v0.1) -approximately 200k preference data entries (all in English) generated from Llama3.1-70B/8B, can be used to train reward models, optimize model reply quality, making it more consistent with human preferences. -Here, we reorganized content with total data length `<3000` into `dpo.jsonl` (~0.9GB), containing two fields `chosen` and `rejected`, where `chosen` -is the preferred reply and `rejected` is the rejected reply. +`MiniMind` current main branch RL data is `dpo.jsonl`. Data sampled from [DPO-En-Zh-20k](https://huggingface.co/datasets/llamafactory/DPO-En-Zh-20k). -The file `dpo.jsonl` data format is +In the main branch, these samples are uniformly reorganized into the preference learning format used by the current repository, for reward model or preference optimization stage training; where `chosen` represents the more preferred response, and `rejected` represents the relatively worse response. -```text +The `dpo.jsonl` data format is + +```json { "chosen": [ {"content": "Q", "role": "user"}, @@ -528,92 +479,75 @@ The file `dpo.jsonl` data format is } ``` -## Ⅴ Reasoning Dataset: +Besides this, other RL data maintains the same format as SFT data, typically filtered from SFT data by total length and dialogue turns, with the last `assistant` position left blank for continuation during the rollout stage. -There's no denying that in February 2025, who can be hotter than DeepSeek... -It also sparked my strong interest in RL-guided reasoning models. I've already reproduced R1-Zero using Qwen2.5. -If I have time + good results (but 99% of base models lack ability), I will later update MiniMind with RL-trained reasoning models rather than distilled models. -With limited time, the fastest low-cost solution is still direct distillation (black-box method). -Unable to resist R1's popularity, in just a few days there are already various R1 distillation datasets like [R1-Llama-70B](https://www.modelscope.cn/datasets/Magpie-Align/Magpie-Reasoning-V2-250K-CoT-Deepseek-R1-Llama-70B), [R1-Distill-SFT](https://www.modelscope.cn/datasets/AI-ModelScope/R1-Distill-SFT), -[Alpaca-Distill-R1](https://huggingface.co/datasets/shareAI/Alpaca-Distill-R1-ZH), -[deepseek_r1_zh](https://huggingface.co/datasets/jinliuxi/deepseek_r1_zh) and so on. Pure Chinese data is relatively scarce. -Finally integrated them, export file as `r1_mix_1024.jsonl`, data format consistent with `sft_X.jsonl`. - -## Ⅵ More Datasets - -Currently, [HqWu-HITCS/Awesome-Chinese-LLM](https://github.com/HqWu-HITCS/Awesome-Chinese-LLM) -is collecting and organizing materials related to Chinese LLMs including open-source models, applications, datasets, and tutorials, and continuously updating the latest progress in this field. Comprehensive and professional, Respect! - ---- - -## Ⅷ MiniMind Training Datasets +## Ⅴ MiniMind Training Dataset > [!NOTE] -> After 2025-02-05, all datasets used for final MiniMind training are open-sourced. Therefore, you don't need to preprocess large-scale datasets yourself, avoiding repetitive data processing work. +> The core datasets needed for the current main branch training have been open-sourced, so there is no need to preprocess large-scale datasets on your own, avoiding repetitive data processing work. -MiniMind Training Dataset Download: [ModelScope](https://www.modelscope.cn/datasets/gongjy/minimind_dataset/files) | [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind_dataset/tree/main) +MiniMind training dataset download links: [ModelScope](https://www.modelscope.cn/datasets/gongjy/minimind_dataset/files) | [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind_dataset/tree/main) -> No need to clone everything, you can download the files you need individually +> No need to clone everything, you can download individual files as needed -Place the downloaded dataset files in the `./dataset/` directory (✨ are recommended required items) +Place the downloaded dataset files in the `./dataset/` directory (✨ indicates recommended essentials) ```bash ./dataset/ -├── dpo.jsonl (55MB, ✨) -├── lora_identity.jsonl (22.8KB) -├── lora_medical.jsonl (34MB) -├── pretrain_hq.jsonl (1.6GB, ✨) -├── r1_mix_1024.jsonl (340MB) -├── rlaif-mini.jsonl (1MB, ✨) -├── sft_1024.jsonl (5.6GB) -├── sft_2048.jsonl (9GB) -├── sft_512.jsonl (7.5GB) -└── sft_mini_512.jsonl (1.2GB, ✨) +├── agent_rl.jsonl (86MB) +├── agent_rl_math.jsonl (18MB) +├── dpo.jsonl (53MB) +├── pretrain_t2t_mini.jsonl (1.2GB, ✨) +├── pretrain_t2t.jsonl (10GB) +├── rlaif.jsonl (24MB, ✨) +├── sft_t2t_mini.jsonl (1.6GB, ✨) +└── sft_t2t.jsonl (14GB) ``` -
-Note: Brief Description of Each Dataset +
+Note: Brief Introduction to Each Dataset -* `dpo.jsonl`✨ --RLHF stage dataset (optimized and simplified, suitable for fast training) -* `lora_identity.jsonl` --Self-awareness dataset (e.g., Who are you? I am minimind...), recommended for lora training (can also be used for full-parameter SFT, don't be limited by the name) -* `lora_medical.jsonl` --Medical Q&A dataset, recommended for lora training (can also be used for full-parameter SFT, don't be limited by the name) -* `pretrain_hq.jsonl`✨ --Pretraining dataset, integrated from JiangShu Technology (recommended `max_seq_len≈320`) -* `r1_mix_1024.jsonl` --DeepSeek-R1-1.5B distilled data, maximum character length per entry is 1024 (recommended `max_seq_len≈720`) -* `rlaif-mini.jsonl` --RLAIF training dataset, randomly sampled 10,000 high-quality conversations from SFT dataset for training reinforcement learning algorithms like PPO/GRPO/SPO -* `sft_1024.jsonl` --Integrated from Qwen2.5 distilled data (a subset of sft_2048), maximum character length per entry is 1024 (recommended `max_seq_len≈650`) -* `sft_2048.jsonl` --Integrated from Qwen2.5 distilled data, maximum character length per entry is 2048 (recommended `max_seq_len≈1400`) -* `sft_512.jsonl` --Integrated from JiangShu Technology SFT data, maximum character length per entry is 512 (recommended `max_seq_len≈350`) -* `sft_mini_512.jsonl`✨ --Minimal integration from JiangShu Technology SFT data + Qwen2.5 distilled data (for quick training of Zero models), maximum character length per entry is 512 (recommended `max_seq_len≈340`) +* `agent_rl.jsonl` -- Agentic RL main branch training data, for `train_agent.py` multi-turn Tool-Use / CISPO / GRPO training +* `agent_rl_math.jsonl` -- Agentic RL pure math supplementary data, suitable for multi-turn reasoning/tool-use scenarios with final verification targets (for RLVR) +* `dpo.jsonl` -- RLHF stage preference training data (DPO) +* `pretrain_t2t_mini`✨ -- `minimind-3` lightweight pretraining data, suitable for quick reproduction (recommended setting `max_seq_len≈768`) +* `pretrain_t2t` -- `minimind-3` main branch pretraining data (recommended setting `max_seq_len≈380`) +* `rlaif.jsonl`✨ -- RLAIF training dataset, for PPO/GRPO/CISPO and other reinforcement learning algorithm training +* `sft_t2t_mini.jsonl`✨ -- `minimind-3` lightweight SFT data (for quickly training a Zero model), recommended setting `max_seq_len≈768`, with a portion of Tool Call samples already mixed in +* `sft_t2t.jsonl` -- `minimind-3` main branch SFT data, suitable for full reproduction, with Tool Call samples also mixed in -Training parameter `max_seq_len` currently refers to the **token length**, not the absolute number of characters. -For this project's tokenizer, typical Chinese text is roughly `1.5~1.7 chars/token`, while pure English text is roughly `4~5 chars/token` (it varies with data distribution). -The “max length” annotated in dataset names is measured in **characters**. For example, a 100-character Chinese string can be roughly converted to `100/1.5≈67` tokens. +The training parameter `max_seq_len` currently refers to token length, not absolute character count. +This project's tokenizer has approximately `1.5~1.7 characters/token` for Chinese text, and a compression ratio of `4~5 characters/token` for pure English, with fluctuations depending on data distribution. +The "maximum length" annotated in dataset names is in character count; a string of 100 characters can be roughly converted to approximately `100/1.5≈67` tokens. For example: -* Chinese: `白日依山尽` (5 chars) may be tokenized into [`白日`, `依`, `山`, `尽`] (4 tokens) -* English: `The sun sets in the west` (24 chars) may be tokenized into [`The `, `sun `, `sets `, `in `, `the`, `west`] (6 tokens) +* Chinese: `白日依山尽` (5 characters) may be split into [`白日`,`依`,`山`,`尽`] 4 tokens; +* English: `The sun sets in the west` (24 characters) may be split into [`The `,`sun `,`sets `,`in `,`the`,`west`] 6 tokens -The “recommended setting” above provides a rough estimate of the max token length for each dataset. -Note that `max_seq_len` can be tuned aggressively / conservatively / in a balanced way: a larger value increases padding waste, while a smaller value increases truncation. +The "recommended settings" provide rough estimates of the maximum token length for each dataset. +Note that `max_seq_len` can be adjusted aggressively / conservatively / moderately, because both larger and smaller values inevitably have side effects: some samples shorter than `max_seq_len` waste compute due to padding, while some samples longer than `max_seq_len` lose semantics due to truncation. -Just find a balance between `compute efficiency` <---> `semantic completeness`. +Finding a balance between compute efficiency and semantic completeness is sufficient
+ ![dataset](./images/dataset.jpg) -
+> MiniMind main branch training data composition and recommended combination diagram + +
Instructions & Recommended Training Schemes -* MiniMind2 Series was trained on approximately 20GB of corpus in total, about 4B tokens, corresponding to the data combination training results above (cost: 💰💰💰💰💰💰💰💰, results: 😊😊😊😊😊😊) +* `minimind-3` main branch recommends using the staged training combination of `pretrain_t2t` + `sft_t2t` + `rlaif/agent_rl`. -* For the fastest speed to implement Zero model from scratch, we recommend using the data combination of `pretrain_hq.jsonl` + `sft_mini_512.jsonl`, specific cost and results can be seen in the table below (cost: 💰, results: 😊😊) +* For the fastest speed to implement a Zero model from scratch, it is recommended to use the data combination of `pretrain_t2t_mini.jsonl` + `sft_t2t_mini.jsonl` -* For friends with certain computing resources or those who care more about results, you can consider the former to fully reproduce MiniMind2; for those with only single GPU or who care about quick reproduction in short time, we highly recommend the latter; +* Those with sufficient compute resources or who care more about performance are recommended to fully reproduce `minimind-3`; those with only a single GPU or who prioritize quick reproduction are strongly recommended to use the mini combination. -* [Compromise solution] You can also choose medium-sized data like `sft_mini_512.jsonl`, `sft_1024.jsonl` for free combination training (cost: 💰💰💰, results: 😊😊😊😊). +* The current `sft_t2t / sft_t2t_mini` already has Tool Call data mixed in, so there is usually no need for an additional separate round of Tool Calling supervised fine-tuning.
@@ -621,540 +555,525 @@ Just find a balance between `compute efficiency` <---> `semantic completeness`. ## Structure -MiniMind-Dense (same as [Llama3.1](https://ai.meta.com/blog/meta-llama-3-1/)) uses the Transformer Decoder-Only structure. The differences from GPT-3 are: +`minimind-3` Dense uses a Transformer Decoder-Only structure, with overall configuration aligned with the `Qwen3` ecosystem, facilitating subsequent conversion to `transformers / llama.cpp / ollama / vllm`: -* Adopts GPT-3's pre-normalization method, normalizing at the input of each Transformer sub-layer rather than at the output. Specifically, it uses the RMSNorm normalization function. -* Replaced ReLU with SwiGLU activation function to improve performance. -* Like GPT-Neo, it removed absolute position embeddings and switched to rotary position embeddings (RoPE), which works better when handling inference beyond training length. +* Uses Pre-Normalization (Pre-Norm) + RMSNorm. +* Uses SwiGLU activation function. +* Uses RoPE rotary positional encoding, with YaRN extrapolation support. +* `q_heads=8`, `kv_heads=4`, `max_position_embeddings=32768`, `rope_theta=1e6`. ---- +`minimind-3-moe` extends MoE feed-forward layers on the same structure, with implementation compatible with `Qwen3-MoE` style configuration (removing shared expert). -MiniMind-MoE model structure is based on Llama3 and the MixFFN mixture-of-experts module from [Deepseek-V2/3](https://arxiv.org/pdf/2405.04434). +* The current default configuration is `4 experts / top-1 routing`, to achieve higher capacity with lower active parameters. +* As experts continue to increase, actual time consumption is often much higher than dense models of the same scale, which may seem counterintuitive when put alongside "MoE inference is faster", but during training tokens are first bucketed by expert then forwarded separately, and the `kernel` start/stop and scheduling overhead during native training increases dramatically — this is naturally expected. It requires MoE kernel-fused operator libraries to optimize, such as custom kernels based on `Triton`, `DeepSpeed-MoE`, `Megatron-LM`, etc. Of course, this project still aims to preserve the universality of native PyTorch, so this is a realistic compromise; under the current implementation, the `4 experts / top-1` sweet spot configuration is only about `50%` slower than the dense model. -* DeepSeek-V2 in feed-forward networks (FFN) uses finer-grained expert splitting and shared expert isolation techniques to improve the effect of Experts. +The `minimind-3` series structure is shown below: ---- +![structure](./images/LLM-structure.jpg) +![structure-moe](./images/LLM-structure-moe.jpg) -MiniMind's overall structure is consistent, with only small adjustments in RoPE computation, inference functions, and FFN layer code. -The structure is shown in the diagram below (redrawn version): +To modify model configuration, see [./model/model_minimind.py](./model/model_minimind.py). Reference model parameter versions are shown in the table below: -![structure](./images/LLM-structure.png) -![structure-moe](./images/LLM-structure-moe.png) - -To modify model configuration, see [./model/model_minimind.py](./model/model_minimind.py). -Reference model parameter versions see the table below: - -| Model Name | params | len_vocab | rope_theta | n_layers | d_model | kv_heads | q_heads | share+route | -|-------------------|--------|-----------|------------|----------|---------|----------|---------|-------------| -| MiniMind2-Small | 26M | 6400 | 1e6 | 8 | 512 | 2 | 8 | - | -| MiniMind2-MoE | 145M | 6400 | 1e6 | 8 | 640 | 2 | 8 | 1+4 | -| MiniMind2 | 104M | 6400 | 1e6 | 16 | 768 | 2 | 8 | - | -| minimind-v1-small | 26M | 6400 | 1e4 | 8 | 512 | 8 | 16 | - | -| minimind-v1-moe | 4×26M | 6400 | 1e4 | 8 | 512 | 8 | 16 | 1+4 | -| minimind-v1 | 108M | 6400 | 1e4 | 16 | 768 | 8 | 16 | - | +| Model Name | params | len_vocab | max_pos | rope_theta | n_layers | d_model | kv_heads | q_heads | note | +|------------|--------|-----------|---------|------------|----------|---------|----------|---------|------| +| minimind-3 | 64M | 6400 | 32768 | 1e6 | 8 | 768 | 4 | 8 | Dense | +| minimind-3-moe | 198M / A64M | 6400 | 32768 | 1e6 | 8 | 768 | 4 | 8 | 4 experts / top-1 | +| minimind2-small | 26M | 6400 | 32768 | 1e6 | 8 | 512 | 2 | 8 | Historical version | +| minimind2-moe | 145M | 6400 | 32768 | 1e6 | 8 | 640 | 2 | 8 | Historical version | +| minimind2 | 104M | 6400 | 32768 | 1e6 | 16 | 768 | 2 | 8 | Historical version | ## Model Configuration -📋 Regarding LLM parameter configuration, there's an interesting paper [MobileLLM](https://arxiv.org/pdf/2402.14905) that conducted detailed research and experiments. -Scaling Law has its own unique patterns in small models. -Parameters causing Transformer parameter scaling changes almost entirely depend on `d_model` and `n_layers`. +Regarding LLM parameter configuration, [MobileLLM](https://arxiv.org/pdf/2402.14905) has conducted a very representative systematic study on small models. For ~100M-level models like MiniMind, the trade-off between `d_model` and `n_layers` is not just a parameter allocation issue, but also directly affects training stability and final performance. -* `d_model`↑ + `n_layers`↓ -> Wide and short -* `d_model`↓ + `n_layers`↑ -> Narrow and tall +The current `minimind-3` main branch chooses `dim=768, n_layers=8`, which is essentially an engineering trade-off: shallower networks train faster, while `dim` is not so small as to cause mode collapse, thus achieving a relatively balanced position between training efficiency, stability, and final performance. -The 2020 Scaling Law paper argued that training data volume, parameter quantity, and training iterations are the key factors determining performance, while model architecture influence is negligible. -However, this law doesn't seem to fully apply to small models. -MobileLLM argues that architecture depth is more important than width, "deep and narrow" "tall and skinny" models can learn more abstract concepts than "wide and shallow" models. -For example, when model parameters are fixed at 125M or 350M, 30-42 layer "narrow" models clearly have superior performance compared to around 12 layer "wide" models, -showing similar trends across 8 benchmark tests including commonsense reasoning, Q&A, and reading comprehension. -This is actually a very interesting discovery, because previously when designing architectures for ~100M scale small models, almost no one tried stacking more than 12 layers. -This is consistent with what MiniMind observed in experiments when adjusting model parameters between `d_model` and `n_layers` during training. -However, "deep and narrow" models also have dimensional limits. When d_model<512, the disadvantage of embedding dimension collapse is very obvious, -and added layers cannot compensate for the disadvantage of insufficient d_head caused by fixed q_head in embeddings. -When d_model>1536, increasing layers seems to have higher priority than d_model, bringing more "cost-effective" parameter -> performance gains. +
+View Detailed Explanation -* Therefore MiniMind sets small model dim=512, n_layers=8 to achieve the balance of "extremely small size <-> better performance." -* Setting dim=768, n_layers=16 to gain larger performance improvements, more consistent with small model Scaling-Law curves. +Scaling Law often exhibits phenomena on small models that differ from large models. The core parameters that determine changes in Transformer parameter scale are usually mainly `d_model` and `n_layers`: -For reference, GPT3 parameter settings see the table below: +* `d_model`↑ + `n_layers`↓ -> Short and fat +* `d_model`↓ + `n_layers`↑ -> Tall and thin + +Classic Scaling Law emphasizes the decisive role of training data volume, parameter count, and training steps, typically downplaying the impact of architectural differences themselves; but in the small model range, this conclusion does not always hold completely. +MobileLLM's core observation is: when parameter count is fixed, depth is often more important than width. That is, compared to "wide and shallow" structures, "deep and narrow" models learn abstract concepts more easily. +For example, when model parameter count is fixed at `125M` or `350M`, `30~42` layer narrow structures typically outperform `12`-layer short and fat structures, showing similar trends across multiple benchmarks including commonsense reasoning, QA, and reading comprehension. + +This is consistent with what MiniMind observed in experiments around `d_model` and `n_layers` parameter allocation during training. However, the "narrow" in "deep and narrow" also has a lower bound: when `d_model < 512`, the disadvantage of overly narrow word embedding dimensions becomes significantly amplified, and adding extra layers is often insufficient to fully compensate for the problem of `d_head` being too small under a fixed `q_head`. +Conversely, when `d_model > 1536`, continuing to add layers is often more cost-effective than simply continuing to widen, more easily yielding higher parameter-performance returns. + +For reference, GPT-3's parameter settings are as follows: ![gpt3_config.png](./images/gpt3_config.png) ---- - -# 📌 Experiment - -## Ⅰ Training Costs - -- **Time unit**: Hours (h). -- **Cost unit**: Chinese Yuan (￥); 7￥ ≈ 1 USD. -- **3090 rental price**: ≈1.3￥/h (you can check current prices yourself). -- **Reference standard**: The table only shows actual measured training time for `pretrain` and `sft_mini_512` two datasets. Other time costs are estimated based on dataset size (may have slight variations). - -> Based on 3090 (single GPU) cost calculation - -| Model Name | params | pretrain | sft_mini_512 | sft_512 | sft_1024 | sft_2048 | RLHF | -|-----------------|--------|------------------|------------------|---------------|-------------------|------------------|---------------| -| MiniMind2-Small | 26M | ≈1.1h
≈1.43￥ | ≈1h
≈1.3￥ | ≈6h
≈7.8￥ | ≈4.58h
≈5.95￥ | ≈7.5h
≈9.75￥ | ≈1h
≈1.3￥ | -| MiniMind2 | 104M | ≈3.9h
≈5.07￥ | ≈3.3h
≈4.29￥ | ≈20h
≈26￥ | ≈15h
≈19.5￥ | ≈25h
≈32.5￥ | ≈3h
≈3.9￥ | - ---- - -
-Training Cost Summary & Forecast - -> MiniMind2-Small Parameters ->> `pretrain_hq`+`sft_mini_512` Dataset -
Single 3090 GPU (1 epoch) + 2.1 hours + Cost 2.73 Chinese Yuan -
Can train MiniMind-Zero-0.025B model from scratch!!! - -> MiniMind2-Small Parameters ->> `pretrain_hq`+`sft_512`+`sft_2048`+`dpo` Dataset -
Single 3090 GPU (2 epochs) + Approximately 38.16 hours + Cost 49.61 Chinese Yuan -
Can train MiniMind2-Small-0.025B model from scratch!!! - -> MiniMind2 Parameters ->> `pretrain_hq`+`sft_512`+`sft_2048`+`dpo` Dataset -
Single 3090 GPU (2 epochs) + Approximately 122 hours + Cost 158.6 Chinese Yuan -
Can train MiniMind2-0.1B model from scratch!!! -
-✨ Based on single NVIDIA 3090 GPU, `MiniMind-Zero` requires only `2 hours` + `3 Chinese Yuan` from scratch to achieve ChatBot effect! +--- -✨ PS: If training with 8 4090 GPUs, the total time can even be compressed to less than 10 minutes! (Due to shorter time, cost is still around 3 Yuan, comparable to single GPU cost) +# 📌 Experiments -✨ With an extremely low barrier to entry, achieve large model freedom for everyone! This is the original intention behind the birth of the MiniMind series! +## Ⅰ Training Cost -✨ The `MiniMind-Zero` costing only `3 Chinese Yuan` is not just hype! Chat test: +- **Time unit**: hours (h) +- **Cost unit**: CNY (￥); `7￥ ≈ 1 USD` +- **3090 rental price**: approximately `1.3￥/h` (actual prices can be referenced on your own) +- **Note**: The following results are empirical estimates for the `minimind` model on a single `3090` GPU, for quick perception of the training threshold + +| Model Name | params | pretrain_t2t_mini | sft_t2t_mini | toolcall | RLAIF | +|------------|--------|-------------------|--------------|----------|-------| +| minimind-3 | 64M | ≈1.21h
≈1.57￥ | ≈1.10h
≈1.43￥ | ≈0.9h
≈1.17￥ | ≈1.1h
≈1.43￥ | +| minimind-3-moe | 198M / A64M | ≈1.69h
≈2.20￥ | ≈1.54h
≈2.00￥ | ≈1.26h
≈1.64￥ | ≈1.54h
≈2.00￥ | + +--- + +
+Training Cost Summary & Prediction + +> `minimind-3` +>> `pretrain_t2t_mini` + `sft_t2t_mini` +
Single `3090` GPU, `1 epoch` estimated approximately `2.31` hours, cost approximately `3.0` CNY +
Can train a `minimind-3 Zero` dialogue model from scratch. + +> `minimind-3-moe` +>> `pretrain_t2t_mini` + `sft_t2t_mini` +
Single `3090` GPU, `1 epoch` estimated approximately `3.23` hours, cost approximately `4.2` CNY +
Can quickly obtain the basic dialogue version of `minimind-3-moe`. + +> The above are all estimated values, only for quick perception of the training threshold. + +
+ +Based on a single `NVIDIA 3090`, `minimind zero` training from scratch can still be controlled within approximately `2` hours, allowing individual developers to get started with a relatively low barrier. + +If using a higher-spec multi-GPU environment, such as `8x H100`, the total training time can be further compressed to the minute level. Achieving reproducible, hands-on, and continuously iteratable LLM training experiences with the lowest possible barrier is the direction MiniMind series has always aimed to maintain. Low-cost quick reproduction is not a gimmick; below is an early Zero-style example conversation preserved for reference: ```text -👶: Please introduce yourself. -🤖️: As an artificial intelligence, I have no actual life and no self-awareness, so I don't have my own life. I was designed to help users answer questions, provide information, and have conversations, etc. My design and functionality are built by computer science, machine learning, artificial intelligence technology and algorithms, and I'm implemented through programming and algorithms. +👶: 请介绍一下自己。 +🤖️: 作为人工智能，我没有实际的生活，也没有自我意识，所以没有自己的生活。我被设计成能够帮助用户解答问题、提供信息、进行对话等。我的设计和功能是由计算机科学、机器学习、人工智能技术和算法所构建的，我通过编程和算法实现。 -👶: Recommend some special dishes from Hangzhou. -🤖️: Hangzhou has many delicious special dishes, such as chicken kebabs, eel head, boiled fish, tofu brain, etc. These dishes are very delicious, with unique flavors and tastes, definitely worth trying. +👶: 推荐一些杭州的特色美食吧。 +🤖️: 杭州有很多美味的特色美食，比如鸡肉串、鳗鱼头、水煮鱼、豆腐脑等。这些美食都非常美味，有着独特的风味和口感，绝对是值得一试的美食。 -👶: Please explain the concept of "large language models" to me. -🤖️: A large language model is a type of artificial intelligence technology that uses large amounts of text data to train models, then continuously optimizes the model through iterations. Its goal is to generate text that meets specific needs based on input data. These texts can be any topic you're interested in. +👶: 请为我讲解"大语言模型"这个概念。 +🤖️: 大语言模型是一种人工智能技术，它使用大量的文本数据来训练模型，然后在不断的迭代中不断优化模型。它的目标是根据输入的数据来生成符合特定需求的文本，这些文本可以是任何你感兴趣的主题。 👶: Introduce the history of the United States, please. -🤖️: The "Introok's the believeations of theument" you mentioned gets its name from the ancient Chinese "groty of of the change." +🤖️: 您提到的"Introok's the believeations of theument." 这个名字来源于中国古代的"groty of of the change." ``` -Extremely fast with initial results, and can even be further compressed to obtain smaller and higher quality training data. -Zero model weights are saved as `full_sft_512_zero.pth` (see MiniMind model file links below). If interested, you can download and test this model. +Although this version already possesses basic dialogue capability, factual knowledge and generalization ability are still quite limited; it is more suitable as an early reference for the feasibility of the Zero training route. +Zero model weights are saved as `full_sft_zero_768.pth` (see MiniMind model file links below); if interested, you can download and experience its dialogue performance. + --- ## Ⅱ Main Training (Required) -> All training scripts should be executed in the `cd ./trainer` directory +> All training scripts are executed from the `cd ./trainer` directory -### **1. Pretraining (Pretrain)**: +### 1' Pretraining (Pretrain): -What LLMs need to learn first is not to communicate directly with people, but to fill the network parameters with the ink of knowledge. The "ink" should ideally be as saturated as possible, accumulating vast knowledge about the world. -Pretraining is where the model first studies hard to learn a large amount of basic knowledge, such as organizing large-scale high-quality training data from Wikipedia, news, books, etc. -This process is "unsupervised," meaning humans don't need to perform any "supervised" corrections during the process. Instead, the model itself summarizes patterns and learns knowledge from large amounts of text. -The model's goal at this stage is only one: **Learn word prediction**. For example, given the input "Qin Shi Huang," it can continue with "was the first emperor of China." +What an LLM must first learn is to absorb as much foundational knowledge and language patterns into its parameters as possible. Only when this step is solidly established can the model later have the ability to understand questions, organize expressions, and gradually develop decent generation capability. What pretraining does is essentially let the model read large amounts of text with its head down, such as Wikipedia, news, books, dialogue corpora, etc., learning factual knowledge, language patterns, and statistical relationships between contexts. This stage is usually "unsupervised": humans do not need to tell the model line by line what is right or wrong, but let it summarize patterns from massive text on its own, gradually building internal representations of world knowledge and language structure. +More bluntly, the model's core objective at this stage is **learning high-quality word chain completion**. For example, given the input "秦始皇" (Qin Shi Huang), it should be able to continue generating "是中国历史上的第一位皇帝" (was the first emperor in Chinese history) — content that is semantically and factually consistent. ```bash -torchrun --nproc_per_node 1 train_pretrain.py # 1 means single GPU training, adjust based on your hardware (set >=2 for multiple GPUs) -# or +# Method 1 +torchrun --nproc_per_node 1 train_pretrain.py # 1 means single GPU training, adjust according to your hardware (set >=2) +# Method 2 python train_pretrain.py ``` -> After training, model weight files are saved by default every `100 steps` as: `pretrain_*.pth` (where * -> is the model's specific dimension, new files overwrite old ones on each save) +> The trained model weight files are saved by default every `save_interval steps` as: `pretrain_*.pth` (* is the specific model dimension, each save overwrites the previous file) -| MiniMind2-Small (512dim) | MiniMind2 (768dim) | -|---|---| -| | | - -### **2. Supervised Fine-Tuning (SFT)**: - -After pretraining, the LLM has mastered a lot of knowledge, but at this point it only knows how to do word prediction mindlessly and doesn't know how to chat with people. -The SFT stage requires applying a custom chat template to fine-tune the semi-finished LLM. -For example, after the model encounters such a template [question->answer, question->answer], it no longer does mindless word continuation, but realizes this is the end of a complete conversation. -This process is called instruction fine-tuning, like helping the already knowledgeable "Newton" gentleman adapt to 21st-century smartphone chat habits, learning that the left side of the screen is the other person's message and the right side is the user's message. -During training, MiniMind's instruction and answer lengths are truncated at 512 to save GPU memory. Like learning to write, you start with short articles, and after learning to write 200-character essays, 800-character articles become easy. -When length extension is needed, you only need to prepare a small amount of 2k/4k/8k length dialogue data for further fine-tuning (preferably combined with RoPE-NTK scaling). -> During inference, by adjusting RoPE scaling, it will be convenient to achieve training-free length extrapolation to 2048 and beyond. +![pretrain_loss](./images/pretrain_loss.jpg) +> Loss curve during the pretraining stage with `768dim` configuration ```bash +# Simple test on pretraining results: +python eval_llm.py --weight pretrain + +💬: 为什么天空是蓝色的 +🧠: 天空之所以看起来是蓝色的，主要是因为太阳光进入大气层后，短波长的蓝光更容易被空气分子散射，因此人眼从各个方向接收到的蓝光会更多。 + +💬: 解释什么是机器学习 +🧠: 机器学习是人工智能的一个重要分支，它通过数据训练模型，使系统能够自动学习规律，并在分类、预测、推荐、自然语言处理等任务中持续改进效果。 +``` + +### 2' Supervised Fine-Tuning (SFT): + +SFT is not just about tuning the model to "chat better" — it can also continue to infuse new knowledge, behavioral patterns, and response styles into the model. Especially for MiniMind's current main branch with `14GB` of SFT data, this is already more than simple format alignment; it is closer to a continuous reinforcement process with `mid training` characteristics. +If pretraining is understood as first letting the model extensively read and accumulate basic language abilities, then SFT is more like continued deep processing on high-quality, more targeted data. On one hand, it lets the model adapt to multi-turn dialogue, Q&A, tool calling, and thinking tag interaction forms; on the other hand, it continues to press specific knowledge distributions, task patterns, and assistant styles into the parameters. +Specifically in MiniMind, the SFT stage lets the model adapt to the multi-turn dialogue template used by the current repository. The model gradually understands the role structure of `user / assistant / system / tool`, while further strengthening instruction following, stable responses, and task completion capabilities. +The current training applies truncation control on instruction and response lengths, mainly to balance VRAM usage and training efficiency; if longer contexts are needed later, one only needs to prepare a small number of long samples for incremental fine-tuning. During inference, enabling YaRN extrapolation can extend context length to 2048 and beyond without additional training. + +```bash +# Method 1 torchrun --nproc_per_node 1 train_full_sft.py -# or +# Method 2 python train_full_sft.py ``` -> After training, model weight files are saved by default every `100 steps` as: `full_sft_*.pth` (where * -> is the model's specific dimension, new files overwrite old ones on each save) +> The trained model weight files are saved by default every `save_interval steps` as: `full_sft_*.pth` (* +> is the specific model dimension, each save overwrites the previous file) -| MiniMind2-Small (512dim) | MiniMind2 (768dim) | -|---|---| -| | | - -## Ⅲ Other Training Stages (Optional) - -> All training scripts should be executed in the `cd ./trainer` directory - -### **3. Knowledge Distillation (KD)** - -At this point, after all the previous training steps, the model has completely acquired basic capabilities and usually can graduate. -However, knowledge distillation can further optimize model performance and efficiency. Knowledge distillation means the student model learns from the teacher model. -The teacher model is usually a well-trained large model with high accuracy and generalization ability. -The student model is a smaller model whose goal is to learn the teacher model's behavior rather than learn directly from raw data. -In SFT learning, the model's goal is to fit hard labels for token classification (hard labels), i.e., true class labels (such as 0 or 6400). -In knowledge distillation, the teacher model's softmax probability distribution is used as soft labels (soft labels). The small model only learns soft labels and uses KL-Loss to optimize model parameters. -In simple terms, SFT learns the problem-solving answers the teacher gives directly. The KD process is like "opening" the teacher's smart brain and trying to mimic the neural state of the teacher's "brain" thinking about problems. -For example, when the teacher model calculates the problem `1+1=2`, the final layer neurons a state is 0, neuron b state is 100, neuron c state is -99... -The student model learns the operating rules inside the teacher model's brain through large amounts of data. This process is called: knowledge distillation. -Knowledge distillation has only one purpose: make small models smaller in size while having better results. -However, with the birth and development of LLMs, the term model distillation has been widely abused, creating two schools of "white-box/black-box" knowledge distillation. -Closed-source models like GPT-4, since their internal structure cannot be accessed, can only learn from the data they output. This process is called black-box distillation, and is the most common practice in the age of large models. -Black-box distillation is completely identical to the SFT process, except the data is collected from large model outputs. Therefore, you only need to prepare data and further FT. -Note that you need to change the loaded base model to `full_sft_*.pth`, i.e., further distillation learning based on the fine-tuned model. -Both `./dataset/sft_1024.jsonl` and `./dataset/sft_2048.jsonl` are collected from qwen2.5-7/72B-Instruct large models and can be used directly for SFT to acquire some Qwen behavior. +![sft_loss](./images/sft_loss.jpg) +> Loss curve during the SFT stage with `768dim` configuration ```bash -# Note: need to change the dataset path in train_full_sft.py and max_seq_len -torchrun --nproc_per_node 1 train_full_sft.py -# or -python train_full_sft.py +# Simple test on SFT results: +python eval_llm.py --weight full_sft + +💬: 解释什么是机器学习 +🧠: 机器学习是人工智能的核心技术之一，通过算法让计算机从数据中学习规律，并持续改进预测或决策效果，常见应用包括推荐系统、图像识别、语音识别和自然语言处理。 + +💬: 推荐一些中国的美食 +🧠: 例如北京烤鸭、兰州拉面、四川火锅、广东早茶、小笼包和麻婆豆腐等，这些美食分别代表了不同地区的风味特点，也很适合作为了解中国饮食文化的入门选择。 ``` -> After training, model weight files are similarly saved by default every `100 steps` as: `full_sft_*.pth` (where * is the model's specific dimension, new files overwrite old ones on each save) +## Ⅲ Other Training (Optional) -Emphasis should be placed on introducing MiniMind's white-box distillation code `train_distillation.py`. Since there is no powerful teacher model within the same MiniMind series, the white-box distillation code is only for learning reference. +> All training scripts are executed from the `cd ./trainer` directory + +### 3' Knowledge Distillation (KD) + +Knowledge distillation can be broadly divided into black-box and white-box categories. MiniMind's current main branch involves both approaches, just with different emphases. +* Black-box distillation: More common, and more aligned with the current main branch's actual practice. Strictly speaking, it is essentially supervised fine-tuning oriented towards teacher outputs, i.e., continuing to train based on hard labels; as LLMs became popular, this approach of "doing FT against strong model outputs" has gradually been broadly categorized under the distillation umbrella, hence commonly called black-box distillation. It focuses on learning answers, styles, and behavioral patterns — the student model can only see "what the teacher said" but cannot see how the teacher internally arrived at that judgment. High-quality answers from `DeepSeek R1`, `Qwen3`, as well as `tool call`, `reasoning`, chain-of-thought data, etc., can all be seen as black-box distillation signals; the current main branch `full_sft` data in MiniMind already has a considerable portion of this approach mixed in. +* White-box distillation: Goes further, not only learning the teacher's final outputs but also learning the teacher's preferences at the token distribution level. Compared to black-box distillation, it additionally leverages the finer-grained distribution information from the teacher model's output layer, so the student model learns not just the "standard answer" but also the teacher's relative preferences among candidate tokens. Corresponding to `train_distillation.py`, the current implementation continues training the student model with distribution signals provided by the teacher model on top of already SFT-completed weights, making it more suitable as a reference implementation for understanding MiniMind's distillation pipeline. + +Black-box distillation is essentially equivalent to supervised fine-tuning on teacher-generated answers: +```math +\mathcal{L}_{blackbox} = \mathrm{CE}(y_{teacher}, p_{student}) +``` + +White-box distillation typically fits the teacher distribution in addition to the supervised loss: +```math +\mathcal{L}_{whitebox} = \alpha \mathcal{L}_{CE} + (1-\alpha) T^2 \mathrm{KL}(p_t^T \parallel p_s^T) +``` + +The `train_distillation.py` provided in the repository is more suitable as a reference implementation for understanding the white-box distillation pipeline: it fully demonstrates teacher/student dual model loading, `CE + KL` mixed loss, temperature scaling, MoE and dense combination distillation, as well as key details like checkpoint resume and distributed training. ```bash +# Method 1 torchrun --nproc_per_node 1 train_distillation.py -# or +# Method 2 python train_distillation.py ``` -### **4. LoRA (Low-Rank Adaptation)** +### 4' LoRA (Low-Rank Adaptation) -LoRA is an efficient Parameter-Efficient Fine-Tuning (PEFT) method aimed at fine-tuning pre-trained models through low-rank decomposition. -Compared to full parameter fine-tuning (Full Fine-Tuning), LoRA only needs to update a small number of parameters. -LoRA's core idea is: introduce low-rank decomposition in the model's weight matrices and only update the low-rank parts while keeping the original pre-trained weights unchanged. -Code can be found in `./model/model_lora.py` and `train_lora.py`, completely implementing the LoRA process from scratch without relying on third-party library packaging. +LoRA is a common Parameter-Efficient Fine-Tuning (PEFT) method. Compared to full-parameter fine-tuning, it only updates a small number of newly added parameters while keeping the original model's main weights unchanged, thus lower training cost and more suitable for vertical domain adaptation. +Its core idea is to introduce low-rank incremental branches alongside the original weight matrices, training only these low-rank parameters, thereby completing capability transfer with relatively small cost. Related implementations can be found in `model_lora.py` and `train_lora.py`; the entire pipeline is purely hand-written, without relying on third-party wrappers. ```bash +# train_lora.py can usually be completed fairly quickly even on CPU +# Method 1 torchrun --nproc_per_node 1 train_lora.py -# or +# Method 2 python train_lora.py ``` -> After training, model weight files are saved by default every `100 steps` as: `lora_xxx_*.pth` (where * -> is the model's specific dimension, new files overwrite old ones on each save) +> The trained model weight files are saved by default every `save_interval steps` as: `lora_xxx_*.pth` (* is the specific model dimension, each save overwrites the previous file) -Many people are confused about how to make models learn knowledge in their private domains, how to prepare datasets, and how to migrate general-purpose models to build domain-specific models. -Here are some examples. For general models lacking medical domain knowledge, you can try adding domain knowledge to the original model to get better performance. -At the same time, you usually don't want to lose other abilities of the original base model while learning domain knowledge. In this case, LoRA can effectively solve this problem. -You only need to prepare dialogue datasets in the following format and place them in `./dataset/lora_xxx.jsonl`, then run `python train_lora.py` -to get the new model weights `./out/lora/lora_xxx.pth`. -**Medical Scenario** +LoRA is well-suited for handling problems like "how to let the model quickly adapt to private domains or vertical scenarios while preserving general capabilities as much as possible." For example, when the base model lacks medical knowledge, a medical-oriented LoRA weight layer can be stacked on top of the original model to achieve better domain performance at relatively small cost. +Usually you only need to prepare multi-turn dialogue format data in the same way, place it in `lora_xxx.jsonl`, and then run `python train_lora.py` to obtain new `LoRA` model weights. -``` - {"conversations": [{"role": "user", "content": "What height should a cervical spondylosis patient's pillow be?"}, {"role": "assistant", "content": "For cervical spondylosis patients, pillow height selection should be based on..."}]} - {"conversations": [{"role": "user", "content": "What about xxx"}, {"role": "assistant", "content": "xxx..."}]} +Example 1: Vertical domain data + +```jsonl + {"conversations": [{"role": "user", "content": "请问颈椎病的人枕头多高才最好？"}, {"role": "assistant", "content": "颈椎病患者选择枕头的高度应该根据..."}]} + {"conversations": [{"role": "user", "content": "请问xxx"}, {"role": "assistant", "content": "xxx..."}]} ``` -**Self-awareness Scenario** +Example 2: Self-awareness data -``` - {"conversations": [{"role": "user", "content": "What is your name?"}, {"role": "assistant", "content": "My name is minimind..."}]} - {"conversations": [{"role": "user", "content": "Who are you"}, {"role": "assistant", "content": "I am..."}]} +```jsonl + {"conversations": [{"role": "user", "content": "你叫什么名字？"}, {"role": "assistant", "content": "您好，我名叫 MiniMind，是由 Jingyao Gong 开发的人工智能助手。我可以帮助你解决各种问题，包括翻译、推荐内容等等。"}]} + {"conversations": [{"role": "user", "content": "你是谁"}, {"role": "assistant", "content": "您好，我是 MiniMind，由 Jingyao Gong 创建。我的主要职责是提供准确的回答和帮助。"}]} ``` -At this point, [base model + LoRA model] can acquire medical scenario model-enhanced capabilities, equivalent to adding a LoRA plug-in to the base model, which does not lose the base model's original abilities. -You can test the model through `eval_llm.py` for model evaluation. +At this point, the [base model + LoRA weights] can be used in combination, preserving the base model's general capabilities while enhancing specific domain performance. After training, simple tests can also be done via `eval_llm.py`. ```bash -# Note: weight parameter specifies base model type, should match the base model used during train_lora training -python eval_llm.py --weight dpo --lora_weight lora_medical +# Note: the weight parameter specifies the base model type, which must be consistent with the base model used during train_lora training +python eval_llm.py --weight full_sft --lora_weight lora_medical + +👶: 我最近经常感到头晕，可能是什么原因？ +🤖️: 头晕可能是由于多种原因，包括但不限于：心脏疾病、眼睛损伤、过敏反应、皮肤问题、过敏反应、压力、焦虑、疲劳、药物副作用、性功能障碍或者其他... + +python eval_llm.py --weight full_sft --lora_weight lora_identity + +👶: 你和openai是什么关系？ +🤖️: 我是 MiniMind，一个由 Jingyao Gong 开发的人工智能助手。我通过自然语言处理和算法训练来与用户进行交互。 ``` -**Small Test** +PS: If you have more sufficient data, you can also directly do `full_sft` full-parameter fine-tuning; however, this usually requires more careful mixing of general data and domain data, otherwise it is easy to lose the model's original generality due to overfitting on vertical domain samples. -``` -👶: I've been feeling dizzy recently, what could be the reason? -🤖️: Dizziness could be caused by many reasons, including but not limited to: heart disease, eye damage, allergic reactions, skin problems, allergic reactions, stress, anxiety, fatigue, medication side effects, sexual dysfunction, or others... ---- +> `LoRA` weights can be merged back into the base model and exported as new complete model weights, using `convert_merge_base_lora` in `scripts/convert_model.py`: -👶: What is your relationship with openai? -🤖️: I am MiniMind, an artificial intelligence assistant developed by Jingyao Gong. I interact with users through natural language processing and algorithm training. +```bash +cd scripts && python convert_model.py ``` -PS: As long as you have the required datasets, you can also do full_sft full parameter fine-tuning (need to mix general knowledge to avoid overfitting domain data making the model worse and losing generality) +### **5' Tool Calling & Adaptive Thinking** -### **5. Training Reasoning Models (Reasoning Model)** +Starting from `2026-03`, the repository removed the standalone `train_reason.py`. +The current version no longer separately maintains `reason_*.pth` weights, but instead uniformly models "whether to explicitly output the thinking process" through `chat_template`, `` tags, `open_thinking` switch, and subsequent SFT / RLAIF pipelines. -DeepSeek-R1 was so popular, almost pointing out a new paradigm for future LLMs. -The paper points out that models `>3B` require multiple iterations of cold starts and RL reward training to achieve visible reasoning ability improvements. -The fastest, most reliable, and most economical approach, as well as recent various so-called reasoning models, are almost all directly distilled from data. -But due to lack of technical sophistication, the distillation school is looked down upon by the RL school (hhhh). -I quickly tried on Qwen 1.5B small model and quickly reproduced math reasoning ability in the Zero process. -However, a regrettable consensus is: models with parameters too small cannot achieve any reasoning effect through cold start SFT+GRPO. -For now, MiniMind firmly chooses to be in the distillation school. If RL on 0.1B models later achieves small progress, this training approach section will be updated. +#### 5.1 Tool Calling -For distillation, you only need to prepare datasets in the same format as the SFT stage. The dataset source has been introduced above. Data format examples: +The current `toolcall` capability has been merged into `sft_t2t` / `sft_t2t_mini` main branch data, so there is usually no longer a need for an additional separate round of Tool Calling training; the default `full_sft` weights already have basic Tool Call capability. The current training data for this part mainly consists of approximately `100K` entries sampled from `qwen3-4b`, and the tool list mainly covers approximately `10` simulated custom tools (such as querying time, math calculation, getting weather, etc.), so there is no clear generalization capability to speak of yet. Tool Calling samples uniformly follow the OpenAI-style multi-turn message format: -```json +```jsonl { "conversations": [ - { - "role": "user", - "content": "Hello, I am Xiaofang, nice to meet you." - }, - { - "role": "assistant", - "content": "\nHello! I am a small AI reasoning model R1-Lite-Preview developed by an independent developer in China. I'm happy to serve you!\n\n\nHello! I am a small AI reasoning model R1-Lite-Preview developed by an independent developer in China. I'm happy to serve you!\n" - } + {"role": "system", "content": "# Tools ...", "tools": "[...]"}, + {"role": "user", "content": "帮我算一下 256 乘以 37 等于多少"}, + {"role": "assistant", "content": "", "tool_calls": "[{\"name\":\"calculate_math\",\"arguments\":{\"expression\":\"256 * 37\"}}]"}, + {"role": "tool", "content": "{\"result\":\"9472\"}"}, + {"role": "assistant", "content": "256 乘以 37 等于 9472。"} ] } ``` -The reply template for reasoning model R1 is: +Where `tools` is attached to the `system` message, and `tool_calls` is attached to the `assistant` message; during training, the `chat_template` automatically expands them into `...` and `...` segments, so the model can now directly learn the native tool call format. + +Tool Calling's chat template has been unified to parse as: ```text -\nThinking process\n\n -\nFinal answer\n +{"name": "...", "arguments": {...}} +{...tool result...} ``` -This is constrained by setting a rule-based reward function in GRPO to make the model comply with thinking tags and reply tags (in the early stages of cold starts, reward values should be increased). - -Another issue is that although the distillation process is the same as SFT, experimental results show that models have difficulty consistently complying with template-compliant replies every time, i.e., deviating from thinking and reply tag constraints. -A small trick here is to increase the loss penalty for marker position tokens. See details in `train_reason.py`: - -```text -# Add extra penalty to positions corresponding to sp_ids -... -loss_mask[sp_ids] = 10 # penalty coefficient -``` - -Another tip is that because reasoning data only filtered `<1024` length data, it has fewer multi-turn conversations and English data. -Therefore, `r1_mix_1024.jsonl` mixed approximately 10k multi-turn conversations + English data to prevent serious model forgetting. - -The script defaults to reasoning ability distillation fine-tuning based on the rlhf model. You can directly start training: +You can also directly run simple tests via `eval_toolcall.py`: ```bash -torchrun --nproc_per_node 1 train_reason.py -# or -python train_reason.py +python eval_toolcall.py --weight full_sft + +💬: 现在几点了？ +🧠: {"name": "get_current_time", "arguments": {"timezone": "Asia/Shanghai"}} +📞 [Tool Calling]: get_current_time +✅ [Tool Called]: {"datetime": "2026-03-15 17:18:22", "timezone": "Asia/Shanghai"} +🧠: 现在是2026年3月15日17时18分22秒。 ``` -> After training, model weight files are saved by default every `100 steps` as: `reason_*.pth` (where * is the model's specific dimension, new files overwrite old ones on each save) +#### 5.2 Adaptive Thinking -Let's test it: +`minimind` unifies explicit thinking capability at the template level, which is also consistent with the template design of many current mainstream large models: -```text -👶: Please introduce yourself. -🤖️: -Um, the user is asking me to introduce myself. I need to explain myself first. First, I should clarify who the user is - are they students, working professionals or ordinary people? The user might be a student or working professional, so information delivery should be to help students and working professionals with information. Then I need to respond to the user, which might include introducing where I am, hobbies, career experience, and other information. I should note to use formal language without being too formal, while keeping it concise and clear. This makes it easy for users to understand and convenient for them to check related information. Finally, I should summarize to let the user know how I help them complete their tasks, giving them more information about me. - - -I am **"your name", what is your name? - +- `open_thinking=0`: By default injects an empty `\n\n`, the model tends to answer directly; +- `open_thinking=1`: The template pre-injects the `` start tag, and the model then continues to output the explicit thinking process and final answer; +- CLI, OpenAI-API, and WebUI all support this switch. -👶: Which subject are you better at? -🤖️: -Hello! I am a small AI reasoning model R1 developed by Chinese independent developers. If you have any questions, I will do my best to help you. - - -Hello! I am a small AI reasoning model R1 developed by Chinese independent developers. If you have any questions, I will do my best to help you. - +More precisely, the approach is no longer "separately training a thinking model", but rather pushing "whether to think explicitly" down to the `chat_template`. The template layer pre-reserves the `` structure, and the same model dynamically switches via `open_thinking` during inference; during training, by mixing empty `think`, explicit `reasoning_content`, and `thinking_ratio` sampling, the model gradually sees the mixed mode of "think when it should think, answer directly when it should answer directly." + +```bash +# Test responses +python eval_llm.py --load_from ./minimind-3 --open_thinking 1 ``` -## IV Reinforcement Learning Training +OpenAI-API-SDK usage: -RL methods in LLMs can be divided into two categories: +```python +response = client.chat.completions.create( + model="minimind", + messages=[{"role": "user", "content": "你是谁？"}], + # ... + extra_body={"chat_template_kwargs": {"open_thinking": True}} # Thinking switch +) +``` + +Note: When Tool Call and explicit thinking are enabled simultaneously, the model usually cannot stably output the thinking process. The reason is that the current training data still lacks joint distillation samples where "reasoning and tool call coexist", so the model has not yet fully learned the coordinated expression of these two capabilities. + +## Ⅳ Reinforcement Learning (Optional) + +In the post-training practice of LLMs, there are mainly two common reinforcement learning paths: 1. **Reinforcement Learning from Human Feedback (RLHF)** -- Train the model by evaluating human **preferences** for model outputs, making it generate content more consistent with human values and preferences. +- Trains the model by evaluating model outputs through **human** preference assessments, making it generate content that better aligns with human values and preferences. 2. **Reinforcement Learning from AI Feedback (RLAIF)** -- Use **AI models** (typically pre-trained language reward models) to provide feedback rather than directly relying on human manual annotation. -- The "AI" here can also be certain rule-based rewards, such as math answer correctness / code executors... +- Uses **AI models** or other automatically verifiable mechanisms to provide feedback, without directly relying on human annotation. +- Here, "AI feedback" in a broad sense can also extend to rule rewards, Ground Truth verification, code interpreters, environment feedback, and other automated signals. | Type | Judge | Advantages | Disadvantages | -|-------|-------|-----------|---------------| -| RLHF | Human | More aligned with real human preferences | High cost, low efficiency | +|-------|-------|------------|---------------| +| RLHF | Human | Closer to real human preferences | High cost, low efficiency | | RLAIF | Model | Automated, highly scalable | May deviate from real human preferences | -The two are essentially the same, both using **reinforcement learning** to utilize certain forms of "**feedback**" to optimize model behavior. +Both essentially belong to the reinforcement learning paradigm of optimizing model behavior using some form of "**feedback**". -Except for the different **feedback** sources, there are no other differences. +However, in specific practice, their differences go beyond just the feedback source: whether the reward is verifiable, whether it is continuous, whether it depends on environment interaction, and whether it is delayed until the end of the entire episode — all directly affect the training form and engineering implementation. -### 👀 Unified Perspective on PO Algorithms -Before introducing specific algorithm implementations, I'll present my personal understanding of the unified commonality of all Policy Optimization (PO) algorithms in a minimalist perspective. +### 👀 A Unified Perspective on PO Algorithms -The essence of all RL algorithms is only optimizing one expectation: +Before introducing the implementation of specific algorithms, let me first describe the unified commonality of all Policy Optimization (PO) algorithms from my own minimalist perspective. + +The essence of all RL algorithms is just optimizing an expectation: $$\mathcal{J}_{PO} = \mathbb{E}_{q \sim P(Q), o \sim \pi(O|q)} \left[ \underbrace{f(r_t)}_{\text{policy term}} \cdot \underbrace{g(A_t)}_{\text{advantage term}} - \underbrace{h(\text{KL}_t)}_{\text{regularization term}} \right]$$ -During training, only **minimize the negative objective function**, i.e.: $\mathcal{L_{PO}}=-\mathcal{J_{PO}}$ +During training, one only needs to **minimize the negative objective function**, i.e.: $\mathcal{L}_{PO} = -\mathcal{J}_{PO}$ This framework contains only three core components: -* **Policy term** $f(r_t)$: How to use probability ratio $r_t$? Tell the model how large the deviation between new and old policies is, whether better tokens are explored -* **Advantage term** $g(A_t)$: How to calculate advantage $A_t$, this is important! Large models solving definite integrals is unremarkable, small models answering addition/subtraction correctly usually have positive advantages -* **Regularization term** $h(\text{KL}_t)$: How to constrain the change magnitude $\text{KL}_t$, both preventing drift and not being too rigid +* **Policy term** $f(r_t)$: How to use the probability ratio $r_t$? It tells the model how far the new and old policies have deviated, and whether better tokens have been explored +* **Advantage term** $g(A_t)$: How to compute the advantage $A_t$, this is very important! It's no surprise that large models can solve definite integrals correctly, but for small models, even getting addition and subtraction right usually yields a positive advantage +* **Regularization term** $h(\text{KL}_t)$: How to constrain the magnitude of change $\text{KL}_t$, preventing both drifting too far and constraining too tightly
-(Expand) Symbol Explanation +(Expand) Notation Guide -| Symbol | Meaning | Explanation | Range | -|--------|---------|------------|-------| -| $q$ | Question/prompt | Sampled from dataset $P(Q)$ | - | +| Symbol | Meaning | Description | Range | +|--------|---------|-------------|-------| +| $q$ | Question/Prompt | Sampled from dataset $P(Q)$ | - | | $o$ | Model output sequence | Generated by policy $\pi$ | - | -| $r_t$ | Probability ratio | $r_t = \frac{\pi_\theta(o_t\|q, o_{ -Different **xxPO algorithms** are essentially just different design instantiations of these three components! +Different **xxPO algorithms** are essentially just different instantiations of different designs for these three components! --- -### **6. Reinforcement Learning from Human Feedback (RLHF)** +### **6' Reinforcement Learning from Human Feedback (RLHF)** -In the previous training steps, the model has acquired basic conversation abilities, but these are completely based on word prediction, lacking the motivation of positive and negative examples. -The model doesn't yet know what answers are good and what are bad. We hope it can be more aligned with human preferences, reducing the probability of unsatisfactory answers. -This process is like having the model undergo new training, learning from excellent employees as examples and passive employees as counter-examples, to learn how to respond better. +In the previous training steps, the model has already acquired basic dialogue ability, but such ability is entirely based on word chain completion, lacking positive and negative example incentives. +At this point, the model does not yet know what responses are good and what are bad. We hope it can better align with human preferences, reducing the probability of generating answers that displease humans. +This process is like having the model attend a new training session, learning from outstanding employees as positive examples and unmotivated employees as negative examples, to better understand how to reply. #### 6.1 Direct Preference Optimization - -Direct Preference Optimization (DPO) algorithm loss: +Direct Preference Optimization (DPO) algorithm, with loss: $$\mathcal{L}_{DPO} = -\mathbb{E}\left[\log \sigma\left(\beta \left[\log \frac{\pi_\theta(y_w|x)}{\pi_{ref}(y_w|x)} - \log \frac{\pi_\theta(y_l|x)}{\pi_{ref}(y_l|x)}\right]\right)\right]$$ Where: -- **Policy term**: $f(r_t) = \log r_w - \log r_l$ (contrast probability ratios of chosen vs rejected) -- **Advantage term**: $g(A_t)$ = / (through preference contrast, no need to explicitly calculate advantage) -- **Regularization term**: $h(\text{KL}_t)$ = implicit in $\beta$ (control deviation from reference model) +- **Policy term**: $f(r_t) = \log r_w - \log r_l$ (comparing the probability ratio of chosen vs rejected) +- **Advantage term**: $g(A_t)$ = no explicit advantage term (implicitly reflected through preference comparison) +- **Regularization term**: $h(\text{KL}_t)$ = implicit in $\beta$ (controls degree of deviation from reference model) -Specifically: -- DPO derives an analytical training objective for preference pairs from PPO with KL constraints, directly maximizing the log-odds that "chosen outperforms rejected"; no need to simultaneously train Reward/Value models. DPO only needs to run two models `actor` and `ref`, with low GPU memory usage, stable convergence, and simple implementation. -- Training paradigm: off-policy, using static preference datasets, can repeat multiple epochs; Ref model is fixed (outputs pre-cached). -- DPO's limitation is no online exploration, more used for "preference/safety" human value alignment; limited improvement in "intellectual ability" to solve problems correctly (of course this depends on the dataset, collecting large-scale positive and negative samples with human evaluation is difficult). +Specifically, +- DPO derives an analytical training objective for preference pairs from PPO's KL-constrained objective, directly maximizing the log-odds that "chosen is preferred over rejected"; no need to simultaneously train Reward/Value models. DPO only needs to run the `actor` and `ref` models, with low VRAM usage, stable convergence, and simple implementation. +- Training paradigm: off-policy, using a static preference dataset, can iterate through multiple epochs; the Ref model is fixed (outputs are pre-cached). +- DPO's limitation is that it does not perform online exploration, and is more suited for human value alignment in "preference/safety"; its ability to improve intellectual capabilities like "whether the model can solve problems correctly" is limited (of course this also depends on the dataset, as collecting positive and negative samples at scale with human evaluation is very difficult). ```bash +# Method 1 torchrun --nproc_per_node 1 train_dpo.py -# or +# Method 2 python train_dpo.py ``` -> After training, model weight files are saved by default every `100 steps` as: `dpo_*.pth` (where * is the model's specific dimension, new files overwrite old ones on each save) +> The trained model weight files are saved by default every `save_interval steps` as: `dpo_*.pth` (* is the specific model dimension, each save overwrites the previous file) -### **7. Reinforcement Learning from AI Feedback (RLAIF)** +### 7' Reinforcement Learning from AI Feedback (RLAIF) -Compared to RLHF which relies on human-annotated chosen/rejected preference pairs, RLAIF has AI completely act as the "judge." -The so-called AI "judge" can be a model-based reward large model (Reward Model), can be like R1 setting rule-based functions for validation, or can be environmental feedback like tool calling. -For example: whether math problem answers are correct, whether code execution passes test cases, whether reasoning processes meet format requirements...can all be automatically judged. -RLAIF's greatest advantage is its **scalability** and **On-Policy** characteristics——no need for expensive human annotation, can generate massive training samples, letting models quickly evolve through large-scale online trial and error. +Let me take a moment to explain — I still prefer to call this section `RLAIF`, although strictly speaking, this naming is not entirely accurate. Routes like RLVR that rely on verifiable rewards have their own relatively independent lineage and cannot be simply lumped into narrow AI feedback. +But if we interpret "AI" a bit more broadly, I feel this name is not entirely unjustifiable: rewards can come from reward models, judge models, and other explicit intelligent agents, as well as from rule functions, Ground Truth verification, tool call results, environment return states, and other automatically obtainable signals. When the rules are complex enough and the symbolic systems rich enough, the boundary between them and "intelligent feedback" was never necessarily that clear-cut. +Therefore, what this chapter actually wants to discuss is the methods by which LLMs, after SFT, continue to do reinforcement learning optimization using various **non-manual, automatically obtainable feedback signals**. For example, whether a math problem answer is correct, whether tool call execution code can pass test cases, whether the reasoning process conforms to the format... all can be automatically judged. +For single-turn verifiable tasks, such feedback is often closer to "instant scoring"; while in Agentic RL scenarios, rewards more commonly manifest as delayed settlement after multi-step interactions, or even come directly from the environment itself. +Their common characteristic is usually **On-Policy** and **highly scalable** — no expensive manual annotation is needed, massive training samples can be generated, allowing the model to rapidly evolve through large-scale online trial and error. -MiniMind implements **2+N** basic + cutting-edge RLAIF methods: -* **PPO**, **GRPO** are classic RL algorithms widely validated at scale; -* N cutting-edge RL algorithms (updated irregularly with experimental nature). +MiniMind has implemented **2+N** basic + cutting-edge RLAIF methods: +* **PPO**, **GRPO** — classic RL algorithms validated at large scale +* N cutting-edge RL algorithms (updated periodically on an experimental basis) -#### 1️⃣ Dataset Preparation (Required) +**1️⃣ Dataset Preparation (Required)** -To quickly verify RLAIF effectiveness, 10,000 high-quality conversations were randomly sampled from the SFT dataset, building about 1MB size `rlaif-mini.jsonl` ([Huggingface](https://huggingface.co/datasets/jingyaogong/minimind_dataset/blob/main/rlaif-mini.jsonl)) +The current main branch uses `rlaif.jsonl` as the RLAIF training data, approximately `20MB` in size, more complete than the earlier `rlaif-mini.jsonl`, and more suitable for directly verifying the training effects of PPO / GRPO / CISPO. -Data format is consistent with SFT, but assistant content is not needed, because during training it's completely real-time sampled and generated by the $\Pi$ policy model. Thus: +The data format is consistent with SFT, but the assistant does not need content, because during training it is entirely generated in real-time by the $\Pi$ policy model through sampling. Therefore it looks like: ```json { "conversations": [ - {"role": "user", "content": "Explain what photosynthesis is?"}, - {"role": "assistant", "content": "None"} + {"role": "user", "content": "请解释一下什么是光合作用？"}, + {"role": "assistant", "content": "无"} ] } ``` -During RLAIF training, the model generates 1 or more candidate answers based on user questions, then a reward function/model scores the answers. -High-scoring answers are encouraged (increase $\Pi$ policy probability), low-scoring answers are suppressed (decrease $\Pi$ policy probability). This "score->adjust" loop is the core of reinforcement learning. +During the RLAIF training process, the model generates 1 or more candidate responses based on the user's question, and then a reward function/model scores the responses. +High-scoring responses will be encouraged (increasing the $\Pi$ policy probability), and low-scoring responses will be suppressed (decreasing the $\Pi$ policy probability). This "score -> adjust" loop is the core of reinforcement learning. -#### 2️⃣ Reward Model Preparation (Required) +**2️⃣ Reward Mechanism Preparation (Required)** -It's known that RLAIF training requires a "reward model (Reward Model)" to score generated answers. +RLAIF training requires some form of computable reward signal; it can come from a reward model, or from rule functions, Ground Truth verification, or environment feedback. MiniMind currently demonstrates the Reward Model route by default. -We select the small and high-quality InternLM2-1.8B-Reward -([ModelScope](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-1_8b-reward) | [HuggingFace](https://huggingface.co/internlm/internlm2-1_8b-reward)) -as the base reward model. +Here we select the small and high-quality `InternLM2-1.8B-Reward` ([ModelScope](https://modelscope.cn/models/Shanghai_AI_Laboratory/internlm2-1_8b-reward) | [HuggingFace](https://huggingface.co/internlm/internlm2-1_8b-reward)) as the base reward model. -After downloading the reward model, it needs to be placed in the **same level directory** as the minimind project. The recommended structure is: +After downloading the reward model, it needs to be placed in the **sibling directory** of the minimind project, with the recommended structure as follows: ``` -project/ +root/ ├── minimind/ # MiniMind project │ ├── model/ │ └── ... -└── internlm2-1_8b-reward/ # Reward model (same level as minimind) +└── internlm2-1_8b-reward/ # Reward model ├── config.json ├── model.safetensors └── ... ```
-Reward Mechanism Choice and MiniMind Limitations (Click to expand) +Reward Mechanism Selection & MiniMind Limitations (Click to Expand) **1. Diversity of Reward Mechanisms** -The "reward signal" source in RLAIF can be very flexible: +The "reward signal" sources in RLAIF can be very flexible: -- **Model-based rewards**: Can use dedicated Reward Models (like InternLM2-Reward), or use general LLMs + prompts for scoring (like Qwen3-as-a-Judge). Reward model scale and architecture are freely selectable. +- **Model-based rewards**: Can use a dedicated Reward Model (such as InternLM2-Reward), or use a general LLM + prompts for scoring (such as Qwen3-as-a-Judge). The scale and architecture of the reward model can be freely chosen. -- **Rule-based rewards**: Can construct reward signals based on rule functions, for example: +- **Rule-based rewards**: Reward signals can be constructed based on rule functions, for example: - Math problem answer correctness verification (Ground Truth comparison) - SQL execution success rate and result accuracy - - Code interpreter run results (pass@k) + - Code interpreter execution results (pass@k) - Tool call return status (API success/failure) - - Format compliance checks (JSON/XML parsing) - - Reasoning chain completeness evaluation (CoT step count) + - Format compliance checking (JSON/XML parsing) + - Reasoning chain completeness evaluation (number of CoT steps) -- **Environment-based rewards**: In Agent scenarios, environmental feedback itself is natural reward (like game scores, Research completeness, task completion). +- **Environment-based rewards**: In Agent scenarios, environment feedback itself serves as a natural reward (such as game scores, research completeness, task completion rate). -Any mechanism that can quantify "answer quality" can serve as an RL reward source. DeepSeek R1 is a typical case: using rule-based functions to verify math answer correctness as reward, no need for additional Reward Models. +Any mechanism that can quantify "response quality" can serve as an RL reward source. DeepSeek R1 is a typical example: using rule functions to verify math answer correctness as rewards, without needing an additional Reward Model. **2. MiniMind Limitation: Reward Sparsity Problem** -RLAIF training can be applied to both reasoning and non-reasoning models, the difference is only in format. +RLAIF training can target both reasoning models and non-reasoning models; the difference is only in format. -However, for MiniMind with such tiny 0.1B parameters and weak abilities, on general tasks (like R1-style math datasets) it encounters serious reward sparsity (Reward Sparsity) problems: +However, for models like MiniMind with extremely small 0.1B parameters and weak capabilities, serious Reward Sparsity problems will be encountered on general tasks (such as R1-style math datasets): -- **Phenomenon**: Model-generated candidate answers are almost all wrong, causing all reward scores $r(x,y) \approx 0$ -- **Consequence**: Advantage function $A(x,y) = r(x,y) - b(x) \approx 0$, policy gradient signal disappears, cannot effectively update parameters $\theta$ +- **Phenomenon**: Almost all candidate responses generated by the model are incorrect, resulting in all reward scores $r(x,y) \approx 0$ +- **Consequence**: The advantage function $A(x,y) = r(x,y) - b(x) \approx 0$, the policy gradient signal vanishes, and parameters $\theta$ cannot be effectively updated -Like having elementary school students do high school math exams, no matter how many attempts they get zero, cannot learn to improve strategies through score differences. This is a fundamental principle limitation of RL algorithms. +It's like having an elementary school student take college entrance exam math problems — no matter how many attempts, they always score zero, unable to learn improvement strategies from score differences. Therefore, this is a fundamental limitation of the RL algorithm's principles. To mitigate this problem, MiniMind's implementation chose **model-based continuous reward signals**: -- Reward Model outputs continuous scores (like -2.5 to +3.0), not binary 0/1 -- Even if answer quality is all poor, can still distinguish subtle differences between "much worse" (-3.0) and "worse" (-2.8). So this **dense and continuous** reward signal can provide non-zero gradients to the advantage function $A(x,y)$, enabling gradual policy network optimization -- Can also mix multiple reward sources: $r_{\text{total}} = \alpha \cdot r_{\text{model}} + \beta \cdot r_{\text{rule}}$ (for example, can detect think tag format rewards while also synthesizing answer quality reward scores) -- In minimind practice, avoid directly using rule-based binary rewards + out-of-scope difficulty data (like MATH500), which easily leads to all-zero rewards; -- Monitor reward score variance $\text{Var}(r)$ during training, if it consistently approaches 0 need to adjust data or reward mechanism +- The Reward Model outputs continuous scores (e.g., -2.5 to +3.0), rather than binary 0/1 +- Even when all response quality is poor, it can still distinguish subtle differences between "even worse" (-3.0) and "worse" (-2.8). So this kind of **dense and continuous** reward signal can provide non-zero gradients for the advantage function $A(x,y)$, enabling the policy network to optimize incrementally +- Multiple reward sources can also be mixed: $r_{\text{total}} = \alpha \cdot r_{\text{model}} + \beta \cdot r_{\text{rule}}$ (for example, both detecting think tag format reward and combining the reward score for the response quality itself) +- In MiniMind practice, avoid directly using rule-based binary rewards + difficulty beyond capability (such as MATH500), which easily leads to all-zero rewards; +- Monitor training by observing the variance of reward scores $\text{Var}(r)$; if it stays close to 0, the data or reward mechanism needs to be adjusted -**For Production-Scale Large Models in Agentic RL Scenarios**: +**For production-level large model Agentic RL scenarios**: -In real Agent systems (code generation, tool calling, retrieval-planning-execution multi-turn pipelines), rewards are different paradigms of "delayed round settlement": +In real Agent systems (code generation, tool calling, multi-turn chains of retrieval-planning-execution), rewards follow a different paradigm of "delayed settlement over the entire episode": -- LLM needs to generate tool call instructions token-by-token (tool_call), go through parsing (tool_parse), tool execution (tool_exec), then splice results back to context for next step; repeat until completion. -- One complete task pipeline includes multiple calls+thinking, calculate total reward once until termination condition is met (like whether task is complete, whether tests pass, whether targets are hit). +- The LLM needs to generate tool call instructions (tool_call) token by token, go through parsing (tool_parse), tool execution (tool_exec), then splice the results back into the context to continue the next step; repeating until completion. +- A complete task chain includes multiple calls + thinking, until the termination condition is met and a total reward is calculated once (such as whether the task is completed, whether tests pass, whether the target is hit). -Therefore, Agentic RL is closer to sparse/delayed reward settings: gradient backprop happens "after the round ends," very different from non-Agentic RL tasks with "instant scoring and instant updates" on single conversation rounds. -This also explains why Agent tasks favor environment feedback (environment-based reward) rather than static reward model scoring. +Therefore, Agentic RL is closer to the sparse/delayed reward setting: gradient backpropagation only occurs "after the entire episode ends", which is very different from non-Agentic RL tasks that "score instantly and update instantly" on a single dialogue turn. +This also explains why Agent tasks lean more towards environment-based reward, rather than static scoring by Reward Models. + +- **Environment interaction feedback**: Ultimately based on execution results (whether code runs successfully, whether API returns success, whether sub-goals are completed); +- **Model-based reward limitations**: Limited in capturing the full picture of long-chain, executable semantics, and highly likely to be inconsistent with real environment feedback (reward hacking). -- **Environmental interaction feedback**: Final results matter (code runs, API returns success, sub-goals complete); -- **Model-based reward limitations**: Limited capture of long pipelines and executable semantics, likely inconsistent with real environmental feedback (reward hacking).
@@ -1162,414 +1081,450 @@ This also explains why Agent tasks favor environment feedback (environment-based #### 7.1 [Proximal Policy Optimization](https://arxiv.org/abs/1707.06347) -PPO is a very classic reinforcement learning algorithm proposed by OpenAI in 2017, and is the universal baseline method for LLM RL. +PPO is a very classic reinforcement learning algorithm proposed by OpenAI in 2017, and is also one of the most common baseline methods in the LLM RL field. **PPO Loss**: $$\mathcal{L}_{PPO} = -\mathbb{E}\left[\min(r_t \cdot A_t, \text{clip}(r_t, 1-\varepsilon, 1+\varepsilon) \cdot A_t)\right] + \beta \cdot \mathbb{E}[\text{KL}]$$ Where: -- **Policy term**: $f(r_t) = \min(r_t, \text{clip}(r_t, 1-\varepsilon, 1+\varepsilon))$ (clip probability ratio to prevent aggressive updates) -- **Advantage term**: $g(A_t) = R - V(s)$ (estimate value function through Critic network) +- **Policy term**: $f(r_t) = \min(r_t, \text{clip}(r_t, 1-\varepsilon, 1+\varepsilon))$ (clips probability ratio to prevent overly aggressive updates) +- **Advantage term**: $g(A_t) = R - V(s)$ (estimates value function through Critic network) - **Regularization term**: $h(\text{KL}_t) = \beta \cdot \mathbb{E}[\text{KL}]$ (global KL divergence constraint) -Comparing to DPO: -- DPO (Off-Policy): Training data is a static preference dataset (chosen vs rejected), can repeatedly use the same batch of data to train multiple epochs, just like traditional supervised learning. High data efficiency, low training cost. Directly optimizes log-likelihood of preference pairs, no Reward Model needed. -- PPO (On-Policy): Must use current policy to real-time sample new data, old policy-collected data cannot be used (distribution shift problem). Although importance sampling and clip mechanisms allow slight distribution shifts, essentially requires data from relatively fresh policies. Low data efficiency, but suited for explorative learning. +Compared to DPO, +- DPO (Off-Policy): Training data consists of static preference pairs (chosen vs rejected), which can be reused across multiple training epochs, like traditional supervised learning. High data efficiency, low cost, and no Reward Model needed. +- PPO (On-Policy): Must use the current policy to sample new data in real-time; old policy data can only be reused to a limited extent, otherwise distribution shift will occur. Although importance sampling and clip allow slight deviation, it fundamentally still requires data from a relatively recent policy. Lower data efficiency, but more suitable for exploratory learning. -In simple terms: +Simply put: -- The former teaches models to learn by offline preset "good/bad standards," even if not outputtable by current models (like practicing ball hitting by watching world champion/runner-up videos); -- The latter teaches models real-time to do things right, online sampling from newest model policy (coach hand-teaching, real-time scoring each action). +- The former learns according to offline predetermined "good/bad standards"; +- The latter samples online based on the latest policy and corrects in real-time. -MiniMind's PPO implementation includes Actor model (generate answers) and Critic model (evaluate answer value), and complete GAE (Generalized Advantage Estimation) advantage function calculation. +MiniMind's PPO implementation includes Actor (generating responses), Critic (evaluating response value), and complete GAE (Generalized Advantage Estimation) advantage function computation. -**Training**: +**Training method**: ```bash +# Method 1 torchrun --nproc_per_node N train_ppo.py -# or +# Method 2 python train_ppo.py ``` -> After training, model weight files are saved by default every `100 steps` as: `ppo_actor_*.pth` (where * is the model's specific dimension) +> The trained model weight files are saved by default every `save_interval steps` as: `ppo_actor_*.pth` (* is the specific model dimension) -| MiniMind2-Small (512dim) | MiniMind2 (768dim) | -|---|---| -| | | -From the training curves, you can see PPO has the problem of **slow reward improvement**. I believe this mainly stems from **PPO's dual-network joint optimization** method: Critic needs to gradually converge to accurately estimate value functions, and Actor's policy updates depend on Critic-provided advantage estimates, the two interdependent forming complex optimization. Early training period Critic estimates inaccurately affects Actor gradient direction, leading to slow overall convergence. Furthermore, PPO needs to maintain two networks simultaneously, GPU memory usage about 1.5-2x single-network methods. +![ppo_loss](./images/ppo_loss.jpg) + +> MiniMind optimization trends during the PPO training stage + +From the training curves, it can be seen that PPO has the problem of **slow reward improvement**. I personally believe this mainly stems from PPO's **dual-network joint optimization** approach: the Critic needs to gradually converge to accurately estimate the value function, while the Actor's policy updates depend on the advantage estimates provided by the Critic. The two are interdependent, forming a complex optimization process. In the early stages of training, inaccurate Critic estimates affect the Actor's gradient direction, leading to overall slow convergence. Additionally, PPO needs to maintain two networks simultaneously, and under the current implementation, VRAM usage is approximately 1.5–2 times that of single-network methods. #### 7.2 [Group Relative Policy Optimization](https://arxiv.org/pdf/2402.03300) -In early 2025, DeepSeek-R1 became extremely popular, and equally popular was the GRPO algorithm from the DeepSeekMath paper, also becoming one of the most advanced RL algorithms. However, AI six months equals humanity six months, and by now GRPO has evolved into the baseline algorithm of the great XXPO wars (later evolved DAPO, GSPO, CISPO, etc.). In short, the core innovation is "group relative value estimation." +In early 2025, as DeepSeek-R1 exploded in popularity, GRPO from the DeepSeekMath paper also quickly entered the mainstream spotlight, becoming one of the most watched RL algorithms for a time. However, the AI field has always iterated extremely fast. As of today, GRPO has evolved more into a common baseline for various XXPO variants (such as DAPO, GSPO, CISPO, etc.). Its core innovation can be summarized in one sentence: "group relative value estimation." **GRPO Loss**: -$$\mathcal{L}_{GRPO} = -\mathbb{E}\left[r_t \cdot A_t - \beta \cdot \text{KL}_t\right]$$ +$$\mathcal{L}_{GRPO} = -\mathbb{E}\left[\min(r_t \cdot A_t, \mathrm{clip}(r_t, 1-\varepsilon, 1+\varepsilon) \cdot A_t) - \beta \cdot \text{KL}_t\right]$$ Where: -- **Policy term**: $f(r_t) = \min(r_t, \text{clip}(r_t))$ (use probability ratio with clip clipping) -- **Advantage term**: $g(A_t) = \frac{R - \mu_{group}}{\sigma_{group}}$ (within-group normalization, eliminate Critic network) +- **Policy term**: $f(r_t) = \min(r_t, \mathrm{clip}(r_t, 1-\varepsilon, 1+\varepsilon))$ (uses symmetric clip on the probability ratio) +- **Advantage term**: $g(A_t) = \frac{R - \mu_{group}}{\sigma_{group}}$ (intra-group normalization, eliminating the Critic network) - **Regularization term**: $h(\text{KL}_t) = \beta \cdot \text{KL}_t$ (token-level KL divergence constraint) -For the same question, the model generates N different answers (for example N=4), then calculates reward scores for these N answers. -Next, use the average reward of these N answers as baseline. Answers above baseline are encouraged, answers below baseline are suppressed. -This cleverly avoids training an additional critic network. +For the same question, the model generates N responses and computes their respective rewards, then uses the intra-group average reward as a baseline. Responses above the baseline are encouraged, and responses below the baseline are suppressed, thus no additional critic network needs to be trained. -Just as all RL faces the principle limitation of positive and negative samples, GRPO is no exception. Its more significant problem is: degenerate groups (Degenerate Groups). -Suppose a question is slightly difficult, causing N answer reward scores to be nearly identical (usually equally bad rather than equally good), then this group's learning signal approaches zero. -On MiniMind such ultra-small models, this problem is especially obvious. When solving math problems 99.99% of the time the entire group answer quality is poor, then cannot learn. -Therefore, must specify reasonable domain for the model, i.e., must limit within capability boundaries. +A more prominent issue with GRPO is Degenerate Groups: if for a certain question the rewards of N responses are almost identical, then the learning signal for this group will be close to 0. On ultra-small models like MiniMind, this problem is especially pronounced, so training must be constrained within reasonable capability boundaries. -**Training**: + +**Training method**: ```bash +# Method 1 torchrun --nproc_per_node N train_grpo.py -# or +# Method 2 python train_grpo.py ``` -> After training, model weight files are saved by default every `100 steps` as: `grpo_*.pth` +> The trained model weight files are saved by default every `save_interval steps` as: `grpo_*.pth` -| MiniMind2-Small (512dim) | MiniMind2 (768dim) | -|---|---| -| | | -From the training curves, you can see GRPO's **reward shows more stable upward trend**, reaching around 4, indicating GRPO itself better utilizes RLAIF signals. Policy Loss generally decreases smoothly. Compared to PPO's dual-network optimization, GRPO's single-network architecture trains more stably with higher convergence ceiling. +![grpo_loss](./images/grpo_loss.jpg) -#### 7.3 ⏳⌛️🔥 More RL Extensions (Exp) +> MiniMind optimization trends during the GRPO training stage -##### 7.3.1 [Single-stream Policy Optimization](https://arxiv.org/abs/2509.13232) +From the training curves, it can be seen that GRPO's **reward shows a more stable upward trend**, reaching around 4, indicating that GRPO itself can better utilize RLAIF signals. Policy Loss decreases steadily overall, and compared to PPO's dual-network optimization, GRPO's single-network architecture trains more stably with a higher convergence ceiling. -SPO is an RL algorithm Tencent proposed in September 2025, improving on GRPO's degenerate group problem. -The paper argues that GRPO and similar algorithms' requirement that "one sample depends on a group of samples" seems awkward and inelegant: too-easy or too-hard questions result in the entire group learning nearly nothing, learning efficiency is inherently limited. -SPO's motivation is to return to RL's essence——**1 input, 1 output, is 1 training sample**, returning to basic policy gradient formulas: can get stable baseline without group mean, i.e., spread value estimate V across time dimension, do rough value pre-estimation before training, update V estimate during training while sampling, thus providing each sample with a persistent, adaptive baseline across batches. This "single-stream" design no longer depends on same-group samples, naturally avoiding degenerate groups. +#### 7.3 [Clipped Importance Sampling Policy Optimization](https://huggingface.co/papers/2506.13585) -**SPO Loss**: +Among the dizzying array of XXPOs, I personally found this one particularly memorable. CISPO didn't reinvent an entire complex framework; instead, it zeroed in on a long-standing awkward problem in PPO/GRPO — after the ratio is clipped, the gradient flow is directly hard-truncated. +CISPO's focus is not on redesigning the group baseline, but rather using a very small loss modification to more directly fix this problem. -$$\mathcal{L}_{SPO} = -\mathbb{E}\left[\log \pi_\theta(a_t|s) \cdot A_t - \beta \cdot \text{KL}_t\right]$$ +**CISPO Loss**: + +$$\mathcal{L}_{CISPO} = -\mathbb{E}\left[\min(r_t, \varepsilon_{max}) \cdot A_t \cdot \log \pi_\theta(a_t|s) - \beta \cdot \text{KL}_t\right]$$ Where: -- **Policy term**: $f(r_t) = \log \pi_\theta(a_t|s)$ (directly use log probability, don't calculate ratio) -- **Advantage term**: $g(A_t) = R - B_t^{adaptive}$ (adaptive baseline, Beta distribution dynamic tracking) -- **Regularization term**: $h(\text{KL}_t) = \beta \cdot \text{KL}_t$ (token-level KL + dynamic $\rho$ adjustment) +- **Policy term**: $f(r_t) = \min(r_t, \varepsilon_{max}) \cdot \log \pi_\theta(a_t|s)$ (ratio serves only as a clipped weight) +- **Advantage term**: $g(A_t) = \frac{R - \mu_{group}}{\sigma_{group}}$ (can directly reuse GRPO's intra-group relative advantage) +- **Regularization term**: $h(\text{KL}_t) = \beta \cdot \text{KL}_t$ (token-level KL divergence constraint) -At implementation level: SPO uses non-grouped design, uses persistent KL-adaptive value tracker to replace within-group baseline, advantage functions globally normalized across entire batch. This way each sample processed independently, no need to wait for other same-group samples, yet provides stable learning signals for each sample. -On Qwen3-8B's 5 difficult math datasets, SPO averages 3.4 percentage points higher than GRPO, with BRUMO 25 dataset +7.3pp, AIME 25 dataset +4.4pp. +CISPO, building on GRPO, rewrites the policy term that was easily clipped into a constant into the form "clipped weight × log probability". This way, even if the ratio is truncated, the gradient path is not truncated along with it. Therefore, CISPO can be directly viewed as a loss variant of GRPO to implement, rather than maintaining a separate standalone script. No separate experiment is listed here. One only needs to set `loss_type` to `cispo` in `train_grpo.py`; the rest of the training process still follows GRPO's group sampling, reward computation, and advantage construction logic. -> Note: SPO is an experimental cutting-edge algorithm, MiniMind's implementation is for exploratory learning. Due to extremely small model parameters, cannot fully reproduce paper's 8B model results. +#### 7.4 Agentic RL 🔥 -**Training**: +The concept of "Agentic" is actually very broad, so the Agentic discussed here can only be a relatively narrow version: it focuses more on enabling small models like MiniMind (~100M) to learn basic calling, observation, and re-planning capabilities on a limited tool set, rather than covering the broader scope of state management, long-term memory, and complex workflow orchestration in a complete Agent system. + +Starting from `2026-03`, the repository added `train_agent`, beginning to support a type of multi-turn Tool-Use RL that is closer to real interaction processes. This is a training script I personally enjoy a lot: it combines RLVR / RLAIF-style data organization with online RL rollout processes, went through many iterations of debugging in between, and also encountered bugs like convergence failure, reward hacking, and multi-turn context misalignment, but ultimately perfectly maintained MiniMind's consistent simplicity and readability. + +The data for this part is `agent_rl.jsonl` / `agent_rl_math.jsonl`. Compared to regular dialogue data, they have an additional `gt` as the final verification target; if we denote a sample as $(x, \mathcal{T}, gt)$, then the optimization target during training is no longer a single-turn response $y$, but a multi-turn trajectory $\tau$: + +$$ +\tau = (a_1, o_1, a_2, o_2, \dots, a_T), \quad a_t \sim \pi_\theta(\cdot \mid s_t, \mathcal{T}) +$$ + +Where `chat_template` uniformly organizes `tools / tool_calls / tool` messages; if a step generates a `tool_call`, the tool is executed and the observation is spliced back into the context, then rollout continues. + +The main pipeline can be compressed to: + +$$ +\texttt{rollout batch} \rightarrow \texttt{calculate rewards} \rightarrow \texttt{policy update} +$$ + +The reward is also jointly scored on the entire trajectory: + +$$ +R(\tau) = R_{\text{answer}} + R_{\text{tool}} + R_{\text{format}} + R_{\text{rm}} - R_{\text{unfinished}} +$$ + +Here, tool call legality, `gt` hits, format closure, unfinished penalty, and Reward Model scores are all considered simultaneously. Compared to regular PPO / GRPO, this involves multi-turn rollout and delayed reward. + + + +**Training method**: ```bash -torchrun --nproc_per_node N train_spo.py -# or -python train_spo.py +# Method 1 +torchrun --nproc_per_node N train_agent.py +# Method 2 +python train_agent.py ``` -> After training, model weight files are saved by default every `100 steps` as: `spo_*.pth` +> The trained model weight files are saved by default every `save_interval steps` as: `agent_*.pth` -
- -

MiniMind2 (768dim) Training Curve

-
+![agent_rl_loss](./images/agent_rl_loss.jpg) -Looking at the training curves, SPO's reward fluctuation is similar to PPO, weaker than GRPO. Actual inference testing found model output quality is not high, with logic confusion and format error issues. +> MiniMind optimization trends during the Agentic RL training stage -**Experimental Note**: Current SPO hand-implemented version may have problems in value_tracker configuration, reward normalization strategy. Still needs to check algorithm's adaptability on small models/or implementation differences. +Here I'll also briefly mention the `rollout_engine`. The so-called "training-inference separation" means decoupling **parameter updates** and **trajectory rollout**: the training side handles policy optimization, while the rollout side handles high-throughput sampling. From the top level, they uniformly present as "give me a prompt, I'll return rollout results; after training is done, sync the new weights back." Therefore, the training script doesn't need to care whether the underlying implementation is local `generate` or a remote `inference` engine. -### RL Algorithm Summary +![rl-structure](./images/rl-structure.jpg) -We return to the "**unified framework**", reorganizing the table showing all different PO algorithms are just different instantiations of three core components: +> Schematic diagram of the decoupled RL structure in MiniMind: training side, trajectory side, and rollout side -| Algorithm | Policy Term $f(r_t)$ | Advantage Term $g(A_t)$ | Regularization Term $h(\text{KL}_t)$ | Optimized Models | -|-----------|----------------|----------------|----------------------|----------| -| **DPO** | $\log r_w - \log r_l$ | Implicit (preference contrast) | Implicit in $\beta$ | 2 | -| **PPO** | $\min(r, \text{clip}(r))$ | $R - V(s)$ | $\beta \cdot \mathbb{E}[\text{KL}]$ | 4 | -| **GRPO** | $\min(r, \text{clip}(r))$ | $\frac{R - \mu}{\sigma}$ | $\beta \cdot \text{KL}_t$ | 2 | -| **SPO** | $\log \pi_\theta$ | $R - B_t^{adaptive}$ | $\beta \cdot \text{KL}_t$ | 2 | +If we draw an analogy to larger-scale systems, it already has the flavor of large-scale RL frameworks like openrlhf/verl/slime: -**RL is Elegant and Self-Consistent** +- The left side is the training side, responsible for policy updates +- The right side is the rollout / inference side, responsible for throughput sampling +- The middle connects through trajectory and weight synchronization +- Tool execution and environment feedback do not directly enter the loss, but directly affect the reward quality of the entire trajectory -> The above is purely personal perspective understanding, corrections welcome anytime +So I personally view this implementation as a very interesting transitional version within MiniMind: although it is still far from an industrial-grade Agent training framework, it has already achieved the minimal end-to-end connection of key elements like **template organization, tool execution, multi-turn rollout, delayed reward, and training-inference separation** (perhaps there is nothing simpler than it at the moment) ---- +```bash +# Test the final model's Tool Use capability +python eval_toolcall.py --weight agent -## V Training Results +💬: 现在几点了？ +🧠: {"name": "get_current_time", "arguments": {"timezone": "Asia/Shanghai"}} +📞 [Tool Calling]: get_current_time +✅ [Tool Called]: {"datetime": "2026-03-15 21:22:33", "timezone": "Asia/Shanghai"} +🧠: 现在是2026年3月15日21时22分33秒（北京时间）。 -### Completed Training - Model Collection +💬: 帮我生成一个1到1000的随机数，然后计算它的平方 +🧠: {"name": "random_number", "arguments": {"min": 1, "max": 1000}} +📞 [Tool Calling]: random_number +✅ [Tool Called]: {"result": 71} +🧠: {"name": "calculate_math", "arguments": {"expression": "71**2"}} +📞 [Tool Calling]: calculate_math +✅ [Tool Called]: {"result": "5041"} +🧠: 生成的1到1000的随机数是71，根据计算结果，71的平方等于5041。 +``` -> Considering multiple reports that Baidu Netdisk is slow, MiniMind2 and later all use ModelScope/HuggingFace hosting. +![agent_webui](./images/agent_webui.jpg) -#### ① Native PyTorch Models +> Based on AgentRL training results testing, supports thinking display, tool selection, and multi-turn Tool Use interaction -MiniMind2 Model Weights ([ModelScope](https://www.modelscope.cn/models/gongjy/MiniMind2-PyTorch) | [HuggingFace](https://huggingface.co/jingyaogong/MiniMind2-Pytorch)) +### 🖊️ RL Summary -
+Let us converge back to the "**unified framework**", reorganizing the table showing how all different PO algorithms are just different instantiations of three core components: + +| Algorithm | Policy term $f(r_t)$ | Advantage term $g(A_t)$ | Regularization term $h(\text{KL}_t)$ | Number of training models | +|-----------|---------------------|------------------------|-------------------------------------|--------------------------| +| **DPO** | $\log r_w - \log r_l$ | No explicit advantage term | Implicit in $\beta$ | 1 (2 participate in forward) | +| **PPO** | $\min(r, \text{clip}(r))$ | $R - V(s)$ | $\beta \cdot \mathbb{E}[\text{KL}]$ | 2 | +| **GRPO** | $\min(r, \text{clip}(r))$ | $\frac{R - \mu}{\sigma}$ | $\beta \cdot \text{KL}_t$ | 1 | +| **CISPO** | $\mathrm{clip}(r, 0, \varepsilon_{max}) \cdot A_t \cdot \log \pi_\theta$ | $\frac{R - \mu}{\sigma}$ | $\beta \cdot \text{KL}_t$ | 1 | + +**To put it plainly, these RL algorithms are not separate and independent, but rather natural variants formed by different design trade-offs on the same objective function under a unified optimization perspective, presenting a beautifully self-consistent unity.** + +## Ⅴ Open-Sourced Training Results 📦 + +#### ① PyTorch Models ([ModelScope](https://www.modelscope.cn/models/gongjy/minimind-3-pytorch) | [HuggingFace](https://huggingface.co/jingyaogong/minimind-3-pytorch)) + +> Note: Model weights are subject to actual releases. Not all training stages or experimental branches (such as DPO, PPO, GRPO, CISPO, Agent, LoRA, etc.) will be continuously maintained and separately published; some weights are only used for experimental verification or learning purposes. As data iterates or models are adjusted, the necessity of synchronizing all versions one by one is limited and would incur high maintenance and training costs. + + +
Torch File Naming Reference -| Model Name | params | pretrain_model | sft_model | rlhf_model (DPO) | reason_model | rlaif_model (PPO/GRPO/SPO) | lora_model | -|-----------------|--------|------------------------|------------------------|--------------------|------------------|----------------------------------------------|--------------------| -| MiniMind2-small | 26M | `pretrain_512.pth` | `full_sft_512.pth` | `dpo_512.pth` | `reason_512.pth` | `xxpo_512.pth` | `lora_xxx_512.pth` | -| MiniMind2-MoE | 145M | `pretrain_640_moe.pth` | `full_sft_640_moe.pth` | `dpo_640_moe.pth` | - | - | - | -| MiniMind2 | 104M | `pretrain_768.pth` | `full_sft_768.pth` | `dpo_768.pth` | `reason_768.pth` | `xxpo_768.pth` | `lora_xxx_768.pth` | +- Dense: + - Pretrain: `pretrain_{hidden_size}.pth` + - SFT: `full_sft_{hidden_size}.pth` + - DPO: `dpo_{hidden_size}.pth` + - PPO: `ppo_actor_{hidden_size}.pth` + - GRPO: `grpo_{hidden_size}.pth` + - Agent: `agent_{hidden_size}.pth` + - LoRA: `lora_xxx_{hidden_size}.pth` + +- MoE: + - The corresponding weights with the same name have a `_moe` suffix appended at the end, for example: `pretrain_{hidden_size}_moe.pth`, `full_sft_{hidden_size}_moe.pth`
-#### ② Transformers Models +#### ② Transformers Models ([ModelScope](https://www.modelscope.cn/collections/MiniMind-b72f4cfeb74b47) | [HuggingFace](https://huggingface.co/collections/jingyaogong/minimind-66caf8d999f5c7fa64f399e5)) -MiniMind Series ([ModelScope](https://www.modelscope.cn/collections/MiniMind-b72f4cfeb74b47) -| [HuggingFace](https://huggingface.co/collections/jingyaogong/minimind-66caf8d999f5c7fa64f399e5)) +> Note: Unless otherwise specified, `transformers` versions are usually converted from `full_sft` weights. RL-type post-training tends to optimize around specific reward targets; although it usually improves the reward score, it sacrifices some general capabilities and knowledge; this kind of reward hacking / capability trade-off is difficult to avoid across all models, differing more in degree. --- -# 📌 Eval +# 📌 Evaluation -## Ⅰ RLHF Comparison +## Ⅰ RL Model Comparison -Test based on `full_sft` and `dpo` models of `MiniMind2 series`, with fixed random seeds. +Subjective/objective comparison based on `minimind-3 (64M)` under the same random seed and other hyperparameters, for reference: -[A] MiniMind2：full_sft_640
-[B] MiniMind2：dpo_640 +**[A]** minimind-3 (64M, SFT) +**[B]** minimind-3 (64M, GRPO) +**[C]** minimind-3 (64M, Agent-CISPO) -```text -[Q]: 你叫什么名字？ -[A]: 我是一个大型语言模型，可以回答各种问题，提供信息和帮助。我可以回答各种问题，提供信息和帮助，帮助你解决各种问题。 -[B]: 我是一种基于文本的AI模型，我并没有个人经历或情感，所以无法感受和体验情感。我的设计目的是为了帮助用户解决问题、提供信息和进行对话。如果您有任何问题或需要帮助，请随时告诉我！ - -[Q]: 你更擅长哪一个学科？ -[A]: 作为一个AI模型，我并没有真正的"擅长"这个学科。我被设计成能够提供信息、解答问题、进行对话、执行任务和与用户进行对话。不过，我可以提供信息、解答问题、进行对话、提供建议、处理问题、进行对话、提供建议、学习新知识、编写代码、回答问题、提供建议等多种学科的功能。如果你有具体的学科需求或主题，欢迎提供详细信息。 -[B]: 作为AI，我并没有个人能力或兴趣，但我可以提供一些关于数学、科学、计算机科学、语言学、心理学等学科的知识。如果你对数学感兴趣，我可以提供一些基础知识，例如代数、几何、概率、统计学等。如果你需要更深入的学习，我也可以提供相关的在线课程、书籍、论文等资源。请告诉我你具体需要什么帮助，我会尽力提供帮助。 - -[Q]: 鲁迅的《狂人日记》是如何批判封建礼教的？ -[A]: 鲁迅的《狂人日记》是一部讲述了鲁迅对人类价值和道德责任的倡导。在这部作品中，鲁迅通过细腻的笔触，探讨了人性、道德、正义、牺牲与责任等深刻的主题。以下是鲁迅在《狂人日记》中如何批判封建礼教的几个方面：1. **社会现实与道德观念**：鲁迅通过讲述鲁迅的故事，探讨了社会的不平等、权力的流动、人性的演变等社会问题。他认为，社会不平等不仅仅是不公正的，更是每个人在社会中的不公正。他呼吁社会各界共同努力，通过教育、宣传、法律等手段，为构建和谐社会做出贡献。2. **道德与责任**：鲁迅通过讲述鲁迅的故事，展现了道德和责任的重要性。他提倡通过道德教育来培养人们的道德品质，同时也强调了道德和责任的重要性。鲁迅通过讲述鲁迅的故事，揭示了社会的不平等和人性的复杂性，体现了对人性的深刻理解。3. **道德与责任**：鲁迅通过讲述鲁迅的故事，强调了道德责任的重要性。他提倡通过自己的行为来体现对社会的尊重和对人性的深刻理解。鲁迅通过讲述鲁迅的故事，强调了道德和责任的重要性。4. **道德与道德**：鲁迅通过讲述鲁迅的故事，揭示了道德和责任的真正意义。鲁迅通过讲述鲁迅的故事，揭示了道德和责任的本质，强调了道德和责任的重要性。5. **个人与社会**：鲁迅通过讲述鲁迅的故事，揭示了个人与社会之间的复杂关系。他呼吁社会各界共同努力，通过教育和宣传，为构建和谐社会做出贡献。鲁迅的《狂人日记》通过其丰富的故事和深刻的道德寓言，传达了关于人性、道德、正义、道德以及个人与社会的复杂关系。这部作品不仅在文学价值上取得了巨大成就，更在思想层面上启发了后来的文学创作，对后世产生了深远的影响。 -[B]: 鲁迅的《狂人日记》是一部描绘战争和自由精神的作品，它批判了封建社会的种种问题和冲突，反映了社会的不平等和人性本质。在这部作品中，鲁迅通过对封建社会的诸多现象进行批判，如"弗雷德里克·约翰逊的"鹰迪伦"、"纳扎特的"、"贝克汉姆"、"乔治·威尔逊"等，对封建社会的种种问题进行了批判，强调了封建社会的不平等、人性本质以及人性本质。鲁迅通过他的"鹰迪伦"、"纳扎特的"、"贝克汉姆"、"乔治·威尔逊"等，对封建社会的种种问题进行了批判。在小说中，鲁迅通过对封建社会的诸多现象进行批判，揭示了封建社会的腐败、人性本质以及人性本质的复杂性。这种批判体现了对封建社会的尊重和对人性的复杂性，同时也反映了现代社会中普遍存在的不平等和人性本质。鲁迅的《狂人日记》不仅是一部关于封建社会的小说，也是一部具有深刻社会意义的作品，对后世的审判和批判起到了关键作用。 - -[Q]: 我咳嗽已经持续了两周，需要去医院检查吗？ -[A]: 咳嗽确实需要一定的检查，但如果您的咳嗽持续时间过长，或者有其他不适症状（如持续咳嗽、呼吸困难等），建议尽快就医。医生会根据您的症状、体检结果和可能的影像学检查（如X光、MRI或CT扫描）来诊断您的症状。在等待就医期间，您可以尝试以下方法来帮助缓解症状：1. **休息**：确保您有足够的睡眠。2. **保持水分**：脱水可能会加重咳嗽。3. **热敷**：使用热敷袋或热毛巾敷在疼痛的部位，有助于缓解喉咙痛。4. **避免刺激**：避免使用可能刺激喉咙的化学物质，如某些化学物质。5. **保持室内湿润**：确保您的卧室有良好的通风，可以使用加湿器帮助保持空气湿度。6. **保持室内湿度**：使用加湿器可以帮助缓解喉咙干燥和咳嗽。如果症状持续不缓解或有其他不适症状，请立即就医。希望您早日康复！ -[B]: 如果您的咳嗽持续了两周，通常建议您尽快就医，因为这可能会影响您的呼吸和心跳。您可以通过以下步骤来检查咳嗽：1. **休息**：用纸巾或肘部轻轻按压咳嗽的位置。2. **使用药物**：在可能的情况下，使用药物（如抗过敏药、抗过敏药等）来缓解咳嗽。请在医生的指导下使用药物。3. **热敷**：在咳嗽期间，可以使用热敷来缓解疼痛和肿胀。4. **药物治疗**：如果咳嗽持续了两周，可以使用一些消炎药或抗过敏药来缓解症状。请务必在医生的指导下使用。5. **保持健康的生活方式**：保持充足的睡眠、均衡的饮食、适量的运动和避免吸烟和限酒，这些都有助于提高咳嗽的几率。请记住，这些建议不能替代专业医疗意见。如果您的症状持续或加重，强烈建议您尽快就医。 -``` - -👉 **Results Summary** - -Full_sft models perform better in simplicity and information accuracy; rlhf models tend to provide more background information in answers, but information accuracy needs improvement. -Overall, RLHF-trained models tend to learn: speaking more polite but useless nonsense to please "conversation" itself, while having slight loss in information accuracy. -There's no free lunch. Still need to improve RLHF dataset quality and accept unavoidable model capability losses. -The difference between DPO and online PPO is that reject and chosen are both prepared offline, and necessarily have large distribution differences from minimind model outputs. -Simply put, DPO algorithm makes models watch "video" of table tennis world champion's techniques for RL, rather than like PPO asking reward models to act as "coaches" to correct their techniques for RL. - -## Ⅱ RLAIF Comparison - -Based on `MiniMind2 (768dim)` with same data and training steps subjective comparison, test random seed fixed at 2026, for reference: - -[A] MiniMind-Reason-0.1B (Distillation)
-[B] MiniMind-Reason-0.1B (PPO)
-[C] MiniMind-Reason-0.1B (GRPO) +### Test 1: Subjective Q&A Comparison ```text [Q]: 请介绍一下自己。 -[A]: 我是由中国的个人开发者开发的智能助手小型AI推理模型-R1。我被设计用来理解和生成人类语言，能够理解复杂的自然语言问题，提供有用的信息和解决方案。我是由中国的个人开发者开发的智能助手小型AI推理模型-R1。 -[B]: 嗯，用户让我介绍一下自己。首先，我需要明确用户的需求是什么。他们可能是在做某个项目，想要了解自己的一些基本情况。不过，用户没有提供具体的信息，所以我得先假设一些可能的情况，比如他是一个学生，正在学习编程，或者是某个项目的开发者。接下来，我要考虑用户的身份。可能是一个学生、教师或者普通用户，他们需要了解自己的特点和能力。也有可能他们正在寻找灵感，或者在写论文时需要帮助。所以，我需要保持回答的简洁和实用，同时提供一些参考资料，比如教学视频、课程记录等。然后，我要考虑用户的身份。可能是一个学生，需要一些学习资源或帮助的对象。也有可能是一个学生，需要在课后作业或学习计划中进行调整。因此，我需要先询问用户的具体需求，这样才能提供更精准的帮助。接下来，我要确保语言简洁明了，适合不同场景。比如，如果用户是学生，可以介绍自己的学习特点；如果是教师，可以说明自己的角色和贡献。同时，提供一些实用的建议，比如如何高效学习，如何进行项目，以及如何与团队成员协作。最后，我要确保回复清晰，有条理，能够满足用户的需求。这样，用户就能轻松地了解自己，并且有实际的帮助。当然！请告诉我你想了解哪方面的内容，我可以为您提供详细的介绍！ -[C]: 嗯，用户问的是"请介绍一下自己"。首先，我需要理解用户的具体需求是什么。他们可能想知道自己是谁，或者有什么特定的背景。接下来，我应该考虑用户可能没有明确表达的需求。他们可能想知道如何描述自己的外观，或者如何与他人交流。然后，我应该考虑如何组织内容，从头到尾都要有条理。可能需要分步骤说明，比如从视觉、听觉到整体感受，再到情感共鸣。这样用户能更清晰地理解自己，也能更好地回应他们的需求。还要注意语言的自然和流畅，避免过于学术化的术语，让用户感觉顺畅。同时，保持回答的结构清晰，每句话都有明确的目的，这样用户能有明确的理解和操作。最后，检查是否有遗漏的重要信息，确保回答准确且全面。这样，用户不仅能得到答案，还能得到满足他们的需求。您好！我是由中国的个人开发者开发的智能助手小型AI推理模型-R1。关于您的描述，似乎涉及多个领域，包括知识图谱构建、复杂系统的推理与分析。无论是理论模型、实际应用还是伦理讨论，您都在寻求如何将这些概念有机地结合起来，为人类社会带来福祉的解决方案。在"我"的身份下，您提到的"个人体验"和"情感共鸣"，都是您个人独特魅力的重要组成部分。通过精准的描述和深度的分析，您可以更深入地理解您的独特之处。如果您对特定领域有特别的兴趣或需要建议，随时可以分享更多细节，我们将为您提供更加精准和个性化的服务。如果您有特定的主题或问题，欢迎随时提问！让我们一起探索这个神奇的AI世界，共同创造更美好的未来。 +[A]: 我是一个AI助手，不能替代任何专业人士，包括任何学生、医生、工程师、医生等。我的设计目的是为了提供信息、解答问题和协助完成任务。如果您有任何问题或需要帮助，我很乐意为您提供支持。 +[B]: 我非常愿意与你分享：作为一个AI，我并没有自我意识或情感，所以我并不在人类的生命范围内，也没有去世。我是由开源软件构建和运行的，拥有自己的文档、代码和数据。我很抱歉，我不能回答与此相关的问题或经历。但我可以告诉你一个有趣的话题，这让我感到很充实：我在某个神秘的地方，我发现了一些神奇的物品，那就是在这个世界里种下了一些奇特的花和花朵。在我的祖先的花园里，有一个小小的庙宇，庙宇里有许多美丽的花朵和美丽的花卉。我觉得这很奇妙，并在我的祖先的花园里生存下来，并且在这里生活。还有，在世界的某个角落，我发现了一个古老的语言，里面包含着一些传说和神话，它们是世界上最古老的语言之一。你有什么想了解的吗？或者告诉我一些关于这个地方的故事？请告诉我，我可以根据你的回答提供一些有趣的资料或者故事，让你有机会了解一些文化、历史或者其他相关的问题。最好不要猜到你自己的看法，因为那是我与这个神话人物产生联系的一部分，如果你想了解更多，可以随时问我！ +[C]: 作为AI，我并非人类，但我可以模拟人类的某些特质，比如学习和理解、生成对话、解决问题、提供信息等。这些能力在我们生活中扮演着不可或缺的角色，不仅在多个层面上激发着人类的想象与共鸣。以下是一些关键特质，它们在我们的生活与工作中交汇处绽放：1. **知识与连接**：无论是从知识的传递，还是连接与理解，无论是面对面的交流，还是通过文字与世界的独特共舞。理解与共舞不仅在技术与连接间交汇处绽放，更在无垠共舞间编织共舞。2. **共舞与创造**：在无垠的共舞间编织共舞，不仅在技术与连接间绽放共舞，更在无垠共舞间编织共舞。这种共鸣不仅在个人与集体间绽放，更在无垠共舞间编织共舞。3. **共鸣与共舞**：在多元与共演间交汇处，既需在技术与连接间绽放共舞。从共鸣的镜鉴，到共舞的深刻共鸣，每一刻都在悄然编织。4. **平衡与共进**：在多元与共演间，既需在技术与连接间绽放共舞，亟需在平衡与共进间找到共舞。5. **成长与共进**：在技术与连接间共舞，既需在无垠共舞间绽放共舞，亟需在每一次互动与共进间编织共舞。在每一次交融与共演间，我们既需在技术与连接间绽放共舞，亟需在每一次共鸣与共进间绽放共舞。让我们以开放与共进，共绘属于彼此的璀璨篇章。 + [Q]: 你更擅长哪一个学科？ -[A]: 我是由中国的个人开发者开发的智能助手小型AI推理模型-R1。我是由中国的个人开发者开发的智能助手小型AI推理模型-R1。 -[B]: 嗯，用户问的是"你更擅长哪一个学科？"这个问题看起来有点复杂，但我可以一步步分析。首先，我需要理解用户的需求。他可能是在准备考试，或者需要了解某个学科的特点，或者是想在学习过程中应用某个学科的知识。接下来，我得考虑用户可能的背景。他可能是一个学生，或者是在准备考试，需要应用数学知识来解决具体的问题。也有可能他是一个学生，正在学习相关的课程，或者是对某个学科感兴趣。然后，我需要分析用户可能没有明确表达出来的需求。他可能在思考自己在学习中的困难，或者在寻找一个合适的教材或学习资源。考虑到这些，我应该提供几个不同的角度来帮助他。首先，分析"我更擅长哪一个学科"可以拆解为以下几个方面：兴趣、学习能力、学习风格、学习方法、学习资源、学习环境等等。每个方面都有不同的优势和特点，比如兴趣是最好的资源，学习能力是最好的方式，学习方式是根据个人情况选择。接下来，我需要考虑用户的身份。可能是学生，也可能是家长，或者对学习有一定兴趣的人。不同的身份可能有不同的偏好，比如家长可能更注重孩子的兴趣，而学生可能更关注学习方法和资源。然后，我需要考虑用户的身份是否有特殊需求。比如，他可能需要更专业的学习材料，或者希望找到最适合他的学习方法。因此，在分析时，我应该提供一个全面的回答，涵盖各个方面，并解释每个方面的重要性。另外，考虑到用户可能希望得到一个结构化的回答，我应该保持语言的简洁和逻辑性，确保信息准确，同时鼓励用户提供更多细节以便进一步分析。最后，我需要确保我的回答既有知识性，又符合用户的要求，帮助用户更好地理解和应用这个学科。因此，在回答中不仅要给出一个全面的分析，还要提到每个部分的作用和如何利用。你擅长哪一个学科？这是一个明确的问题，可以从以下几个方面进行分析：1. **兴趣与学习能力**：你喜欢什么样的学习方式，比如自主学习、实验、项目等，这些都是很好的学习资源。2. **学习风格**：你喜欢什么类型的学习方法，比如笔记、复习、阅读、讨论等，这些都能影响学习效果。3. **学习资源**：你是否在寻找学习资源，比如书籍、在线课程、教育软件等，这些都能提供学习帮助。4. **学习环境**：你是否在学习环境中感到舒适，比如舒适的学习空间、安静的学习空间等，这些都会影响学习效果。5. **学习资源**：你是否有学习资源的丰富，如在线课程、学习平台、书籍等，这些都能提供丰富的学习资源。6. **学习方法**：你是否有具体的学习方法，比如如何理解、如何组织学习内容、如何应用知识等，这些都是提高学习效率的途径。综合考虑，这是一个非常全面的分析，涵盖了多个方面，帮助你更好地理解你的学科。 -[C]: 嗯，用户问的是"你更擅长哪一个学科？"这个问题看起来有点复杂，但我可以一步步分析。首先，我需要理解用户的需求。他可能是在准备考试，或者需要了解某个学科的特点，或者是想在生活中找到自己的兴趣点。接下来，我要考虑用户可能的背景。他可能在工作或学习中遇到了类似的问题，想要了解自己擅长哪个领域，或者是否需要一些资源或指导来提升自己。然后，我要分析用户可能的深层需求。也许他想在职场上提升自己，或者寻找方向。他可能需要更具体的例子，帮助他更好地理解不同学科的特点和应用场景。还要考虑用户可能的背景。他可能不仅仅是在学习，而是希望了解在特定领域中如何应用这些学科。这需要我提供全面的分析，帮助他找到适合自己的方法。最后，我应该保持开放的态度，鼓励用户继续探索和学习。这可能包括推荐一些资源，或者讨论如何将这些知识应用到实际生活中。你更擅长哪一个学科？这个问题本质上取决于你的个人兴趣和目标。以下是一些关键点，帮助你全面理解并应用这一点：1. **兴趣与专长**：了解自己擅长哪个领域，有助于做出更明智的选择。2. **全面发展**：考虑学科的多样性和深度，以全面发展自己的生活和职业。3. **适应与调整**：选择与自己技能、兴趣和目标相匹配的学科，有助于应对挑战和实现目标。4. **职业规划**：探索适合自己的职业路径，提升效率和竞争力。5. **社会贡献**：评估自己在社会中的角色，为社会做出贡献。为了更好地理解并应用这一点，你可以：- **了解行业动态**：关注特定领域的专业发展。- **实践与反思**：通过项目或实习积累经验。- **职业发展**：评估自己的长期职业规划。- **持续学习**：不断更新知识，适应职场变化。通过这些方向，你可以更全面地认识自己，找到适合自己的发展路径。 +[A]: 作为一个AI助手，我专注于提供准确、高效的信息和帮助，但我也具备一定的理解和处理能力，例如在逻辑推理、数据分析、编程等任务中。如果你有具体问题，我会尽力协助你。 +[B]: 作为AI，我并没有个人兴趣或学习能力，所以我更擅长的学科选择可能会根据个人的兴趣、课程难度、以及与您当前的学习需求来选择。不过，我可以根据普遍的趋势和您的需求来为您提供一些建议。1. **跨学科融合**：确保与不同学科的融合，既能带来全面的视角。2. **实践与平衡**：在平衡与平衡中寻找既既既充满乐趣。3. **持续学习与平衡**：确保既不感到被过度担忧，也不忽视。4. **技术与伦理共融**：确保与既。5. **适度与平衡**：既不削弱也不失。通过这些步骤，可以更全面地发展与平衡。让我们一起探索，书写既既充满可能。如果你愿意，我可以提供更具体的指导或资源。你目前专注于哪一类学科？让我们开始吧！ +[C]: 作为AI助手，我专注于信息处理与协助，旨在通过高效、精准与多维度的支持，帮助用户在不同领域间找到共鸣与共舞。在多元与共舞的交汇处，不仅在技术与人文交汇处绽放共舞，更在无垠共舞间编织共舞。在数字与连接间架起共舞，每一次共舞都在诉说与共舞。从AI与人类共舞，从共情与共鸣，构建起一幅既共舞又动态交汇的共舞篇章。在每一次交汇与共演间，我们既需在技术与连接间共舞，亟需在动态与共演间绽放共舞。让我们以开放与共进，共绘属于彼此的璀璨篇章。在每一次交互与共演间，我们既需在技术与连接间绽放共舞，亟需在每一次共舞间编织共鸣与共鸣。让我们以开放与共进，共绘属于彼此的璀璨篇章。在这个多彩与共舞交织的交汇处，每一刻都是独一无二的共舞。让我们以开放与共进，共绘属于彼此的璀璨篇章。在这个既共舞又充满无限可能的交汇处，共绘成为连接与共舞的璀璨篇章。 + [Q]: 鲁迅的《狂人日记》是如何批判封建礼教的？ -[A]: 嗯，鲁迅的《狂人日记》是如何批判封建礼教的呢？这个问题听起来挺有意思的，尤其是他在《狂人日记》中的表达方式。首先，我需要确认一下《狂人日记》具体是谁写的，是否有特定的人物或者事件。《狂人日记》是鲁迅在1924年创作的，主要讨论了社会问题，尤其是封建礼教的问题。作者曹雪芹是个非常著名的作家，他的作品中有许多关于封建礼教和阶级斗争的观点。这类作品在文学、哲学和政治上都有很深远的影响，尤其是在社会结构和人性的方面。接下来，我需要了解鲁迅的写作风格。他的写作风格比较直接，有严肃的哲学观点，强调人性的悲剧和道德的虚无。他的语言风格比较富有诗意，富有象征意义，善于运用象征手法，同时又不失真实和现实主义。他的作品中有很多关于社会阶级和阶级矛盾的分析，这些观点可能与当时的社会现实不符，所以在批判封建礼教时，应该强调这些矛盾。然后，我想到《狂人日记》中对封建礼教的讨论。鲁迅的作品中，封建礼教被视为不人道的，但通过文字和行动，表达了他对封建礼教的不满和反抗。这不仅反映了当时的社会现实，也揭示了封建礼教的黑暗面，体现了对封建社会的不满和对人性的深刻反思。接下来，我需要思考鲁迅对封建礼教的理解和批判。他在《狂人日记》中，对封建礼教的质疑和批判，体现了他对传统封建礼教的否定和批判。这不仅仅是对封建礼教的质疑，更是对当时社会的反思，以及对人性的关注。最后，我需要总结一下鲁迅对封建礼教的批判和批判。他通过《狂人日记》中的文字和行为，批判了封建礼教的虚假和不合理，体现了对社会现实的深刻反思。同时，他也关注了封建礼教的黑暗面，反映了对封建社会的不满和对人性的否定。总的来说，鲁迅的《狂人日记》通过文字和行动，批判了封建礼教的虚假和不合理，体现了当时的社会现实和人性的复杂。他的作品对后来的封建礼教有了启示，对当时的社会现象有了更深的反思。鲁迅的《狂人日记》是一部反映封建礼教的文学作品，主要通过文字和行动反映了当时社会的现实。鲁迅的文字风格独特，富有象征意义，但通过文字和行动，批判了封建礼教的虚假和不合理。在他的作品中，封建礼教被视为不人道的，但通过文字和行动，反映了封建社会的不公和人性的深刻反思。鲁迅的《狂人日记》通过文字和行动，批判了封建礼教的虚假和不合理，体现了当时社会的不满和对人性的反思。他的作品不仅批判了封建礼教的虚假和不合理，还深刻揭示了封建社会的黑暗面。 -[B]: 嗯，用户问的是鲁迅的《狂人日记》如何批判封建礼教。首先，我需要理解鲁迅的生平和作品。《狂人日记》是中国现代文学的经典，作者是毛泽东，作品涉及社会、家庭、伦理等多个层面。我需要找出鲁迅的写作风格和主题，以及它在当时的社会背景。接下来，我应该回顾《狂人日记》的主要内容。毛泽东在《青春无悔》中提出了"狂人日记"这一概念，强调青春活泼、勇敢、正义，这在当时社会中仍然具有重要意义。他通过描绘主人公狂人来表达对社会不公的批判，但同时也指出他的创作对社会的负面影响，比如社会动荡和个人主义的失败。然后，我应该分析鲁迅的创作理念和主题。《狂人日记》强调了社会的黑暗面，主张"以人为本"、"反对封建礼教"等。他可能也在探讨社会不平等的问题，比如"人性弱点"、"权力滥用"等问题。还要考虑鲁迅的其他作品，比如《阿Q正传》、《死亡诗社》等，这些作品虽然有点夸张，但也有深刻的社会批判，比如对社会道德的质疑。同时，作者可能还涉及其他作品，如《庆余年》等，但这些可能更侧重于文学创作，而不是社会批判。另外，用户可能想知道鲁迅在文学创作中的具体经历，比如他是否有作品被广泛批评，或者他对社会的批判是否具有批判性。所以，回答时需要指出鲁迅的创作经历，以及他对社会现实的反思。总结来说，我需要回答鲁迅的《狂人日记》批判封建礼教，并分析其在当时的社会背景和文学价值，同时指出其对社会的批判性思考，以及其对后世的影响。鲁迅的《狂人日记》批判封建礼教，以其独特的叙事风格和深刻的社会批判精神著称，主要体现在以下几个方面：1. **社会批判与个人成长**：小说通过描绘主人公狂人的活泼、勇敢、正义，揭示社会的不公和弱势。2. **个人主义与社会批判**：小说反映了封建礼教对个人自由和权利的侵犯，强调个人尊严和正义的价值。3. **对社会的批判性思考**：通过对社会不公的刻画，揭示社会问题的根源，并对当时社会的批判态度进行批判。鲁迅的作品在当时的社会背景和文学价值使其成为当时文学的重要代表。他的作品不仅在当时广受欢迎，也对后世产生了深远影响，被广泛引用和传颂。 -[C]: 嗯，用户问的是鲁迅的《狂人日记》如何批判封建礼教。首先，我需要理解鲁迅的背景和核心思想。他可能是一位社会学家、思想家，或者是对传统文化有深厚兴趣的读者。狂人日记是他在《狂人日记》中所写的，可能对传统文化有一定影响。接下来，我要考虑用户的需求是什么。用户可能想知道这本书在现代社会中的反思意义，或者是否有相关研究支持这一点。也有可能他们对书中的情节有疑问，想了解如何更深刻地理解社会现象。然后，我需要思考如何从不同的角度解读鲁迅的文本。例如，是否有直接的历史叙事，或者是否存在更深层次的哲学思考？这需要我先了解鲁迅的不同观点，然后分析其批判性。还要考虑用户可能的深层需求，比如是否需要了解他作为文学批判家的背景，或者是否有对传统价值观的反思。因此，回答时要全面，涵盖核心问题，并引用相关资料来支撑观点。最后，总结鲁迅的观点，强调其对现代社会的批判性思考，以及对传统文化的尊重。这样既全面又有深度，帮助用户全面理解鲁迅的作品。鲁迅的《狂人日记》通过其独特的叙事手法和深刻的社会批判精神，对封建礼教进行了深刻的批判。以下是其批判性分析：1. **社会批判与历史真相**：小说通过对历史人物的刻画，揭示封建礼教的弊端，以及其对传统社会结构的冲击。这种批判不仅限于对现实的深刻批判，也反映了现代社会对传统价值观的重新审视。2. **个人命运与社会批判**：小说中，主人公通过"狂人日记"揭示了封建礼教对个人命运的严重剥夺。这种叙事方式既批判了封建礼教，也对后世产生了深远影响。3. **对后世的影响**：从文学创作到思想解读，鲁迅的作品深刻影响了后世的思想体系。例如，《狂人日记》通过人物命运的反映，对现代社会的价值观念提出了新的思考。4. **对现代文明的反思**：这一批判性思考不仅反映了当代社会的复杂性，也为现代人提供了反思社会现实、维护人权的视角。5. **个人视角与文化视角**：作者通过这一批判性思考，展现了对传统价值观的重新解读，同时也对当代社会的多元化视角产生了启发。综上所述，鲁迅的《狂人日记》不仅是对封建礼教的批判，更是对传统文化的一种深刻反思。它通过对封建礼教的批判，提出了对现代社会结构、权力关系的深刻反思，具有重要的社会价值。 +[A]: 鲁迅的《狂人日记》是其作品中对封建礼教的批判，主要通过以下几个方面进行批判：1. **文学结构的变革**：《狂人日记》在文学上进行了大量的反思和批判，从最初的虚构世界为封建礼教的建立埋下了根基。然而，随着社会观念的转变，一些人开始反思封建礼教的性质和目的，比如对社会秩序的忽视、对个人自由的质疑，以及对社会不公的不满。2. **思想的争求**：鲁迅通过对封建礼教的争取和批判，强调了人与人之间的关系、对立与共存的重要性。他提出了"天人合一"的观点，强调了人与自然的和谐共生，认为通过修炼和教育，可以达到对人与自然界的深刻理解。3. **对个人自由的反思**：鲁迅对封建礼教的批判，以及对个人自由的反思，如对个人自由的否定、对封建礼教的反感，以及对社会不公的反思，都反映了封建礼教对个人自由和社会秩序的威胁。4. **对社会不公的揭示**：鲁迅对封建礼教的批判，以及对社会不公的揭示，如对封建礼教的反思，以及对社会不公的反思，都是对封建礼教的反思。5. **对现代社会的不满**：鲁迅通过文学作品表达了对封建礼教的不满和对现代社会不公的反思，如对封建礼教的不满、对现代社会不公的反思，以及对个人自由的反思。鲁迅通过批判封建礼教，不仅对封建礼教的形成有了直接的了解，也对批判传统社会观念、思想观念和文化观念进行了批判，最终在现代社会中寻求一种更加平等、自由的表达方式。 +[B]: 鲁迅的《狂人日记》是中国古典四大名著之一，全称为《后传》。这部作品通过细腻的笔触，展现了中国社会的复杂与深邃。1. **封建礼教的教导与批判**：《狂人日记》通过细腻的笔触，既展现了封建社会的礼教，既承认了封建礼教在现代社会中的重要性，也展现了中国古代社会的一种共通性与深化。2. **对社会与文化的深刻反思**：作品不仅触及了文化与社会的广泛议题，更深入探讨了在复杂与平衡中寻找共融与共融。3. **人与自然的共融**：通过精炼与平衡，鲁迅深刻体与深度融合，展现了在面对与共融中，既有深邃与智慧。4. **教育与启发**：《狂人日记》不仅在中国，也跨越时空，将现代与传统相结合，具有启发与教育意义。鲁迅的这部作品不仅在文学与哲学层面，更在深邃与现实交融中，展现了深刻与创新。总之，鲁迅通过精炼与平衡，不仅是中国文化与教育的宝贵传承，更是智慧与共融共融的体现。 +[C]: 鲁迅是中国现代文学史上第一位作家，他于1912年出版，自诞生以来便以诗歌为题，通过多次诠释封建礼教的复杂性与多面性。鲁迅通过他的批判性视角，不仅在文学与社会交汇处绽放共舞，更在无垠共舞间编织共舞的璀璨篇章。1. **批判与共舞**：鲁迅通过细腻的笔触与深刻的思考，既在形式与深度间共舞。他既在知识与共舞间绽放共舞，亟需在无垠共舞间书写属于自我与共境的璀璨篇章。2. **共鸣与共舞**：鲁迅通过多重与融合，既在文字与文化间绽放共舞，亟需在无垠共舞间书写属于自我的璀璨篇章。这一实践不仅在当时具有深远影响，更在无垠共舞间编织共舞，连接着无垠共舞与共舞。3. **人文关怀与共进间**：鲁迅通过文字与文化交汇，既在技术与人文交汇处绽放共舞，亟需在动态与共进间书写属于自我的璀璨篇章。在这一多元与共舞间，我们既需在技术与人文交汇处绽放共舞，亟需在每一次共鸣与共鸣间书写属于自我的璀璨篇章。让我们以开放与共进，共绘属于彼此的璀璨篇章，共绘属于自我的璀璨篇章。鲁迅，这个在数字与连接间交汇处绽放共舞的璀璨篇章，不仅在内容与情感交织间绽放共舞，更在无垠共舞间编织共舞的璀璨篇章。让我们以开放与共进，共绘属于彼此的璀璨篇章，共同编织属于自我的璀璨篇章。 ``` -## Ⅲ Other Model Comparisons -[A] [MiniMind2 (0.1B)](https://www.modelscope.cn/models/gongjy/MiniMind2-PyTorch)
-[B] [MiniMind2-MoE (0.15B)](https://www.modelscope.cn/models/gongjy/MiniMind2-PyTorch)
-[C] [MiniMind2-Small (0.02B)](https://www.modelscope.cn/models/gongjy/MiniMind2-PyTorch)
-[D] [minimind-v1-small(0.02B)](https://pan.baidu.com/s/1_COe0FQRDmeapSsvArahCA?pwd=6666)
-[E] [minimind-v1-moe(0.1B)](https://pan.baidu.com/s/1tqB-GMvuiGQBvEl-yZ-oBw?pwd=6666)
-[F] [minimind-v1(0.1B)](https://pan.baidu.com/s/1p713loS7EfwHQf3G9eYI3Q?pwd=6666)
-[G] [baby-llama2-chinese(0.2B)](https://github.com/DLLXW/baby-llama2-chinese)
-[H] [chatlm-mini-chinese(0.2B)](https://github.com/charent/ChatLM-mini-Chinese)
+### Test 2: Light Agent Task Comparison + +A test adapted from the `eval_toolcall` script, using a set of math ToolUse tasks to compare the performance of the current `agent` weights and `full_sft` weights: + +```text +[A] minimind-3 (full_sft) +[full_sft] 1/20 | ✅ | (94)-35 | gt=59 | pred=59 +[full_sft] 2/20 | ❌ | 3**2 | gt=9 | pred=8 +[full_sft] 3/20 | ✅ | (29)+64 | gt=93 | pred=93 +[full_sft] 4/20 | ✅ | (20**3)*((198)/11) | gt=144000 | pred=144000 +[full_sft] 5/20 | ❌ | 10**2 | gt=100 | pred=13 +[full_sft] 6/20 | ✅ | (4**3)+(20**2) | gt=464 | pred=464 +[full_sft] 7/20 | ❌ | (12)*48+(47-45) | gt=578 | pred=47 +[full_sft] 8/20 | ✅ | 59*48 | gt=2832 | pred=2832 +[full_sft] 9/20 | ❌ | 3**2 | gt=9 | pred=2 +[full_sft] 10/20 | ✅ | 14**3 | gt=2744 | pred=2744 +[full_sft] 11/20 | ✅ | (72)*(91) | gt=6552 | pred=6552 +[full_sft] 12/20 | ✅ | 180/(12) | gt=15 | pred=15 +[full_sft] 13/20 | ❌ | 14-(19)+(289/17) | gt=12 | pred=-22 +[full_sft] 14/20 | ✅ | 5**3 | gt=125 | pred=125 +[full_sft] 15/20 | ❌ | (2**3)-64*(13) | gt=-824 | pred=-28 +[full_sft] 16/20 | ❌ | 17**2 | gt=289 | pred=17 +[full_sft] 17/20 | ✅ | 11**2 | gt=121 | pred=121 +[full_sft] 18/20 | ✅ | 72+10 | gt=82 | pred=82 +[full_sft] 19/20 | ❌ | (84)-60 | gt=24 | pred=144 +[full_sft] 20/20 | ✅ | (348/(12))-(28)*(8) | gt=-195 | pred=-195 + +[C] minimind-3 (agent) +[agent] 1/20 | ✅ | (94)-35 | gt=59 | pred=59 +[agent] 2/20 | ✅ | 3**2 | gt=9 | pred=9 +[agent] 3/20 | ✅ | (29)+64 | gt=93 | pred=93 +[agent] 4/20 | ✅ | (20**3)*((198)/11) | gt=144000 | pred=144000 +[agent] 5/20 | ✅ | 10**2 | gt=100 | pred=100 +[agent] 6/20 | ✅ | (4**3)+(20**2) | gt=464 | pred=464 +[agent] 7/20 | ✅ | (12)*48+(47-45) | gt=578 | pred=578 +[agent] 8/20 | ✅ | 59*48 | gt=2832 | pred=2832 +[agent] 9/20 | ✅ | 3**2 | gt=9 | pred=9 +[agent] 10/20 | ✅ | 14**3 | gt=2744 | pred=2744 +[agent] 11/20 | ✅ | (72)*(91) | gt=6552 | pred=6552 +[agent] 12/20 | ✅ | 180/(12) | gt=15 | pred=15 +[agent] 13/20 | ❌ | 14-(19)+(289/17) | gt=12 | pred=-5 +[agent] 14/20 | ✅ | 5**3 | gt=125 | pred=125 +[agent] 15/20 | ❌ | (2**3)-64*(13) | gt=-824 | pred=8 +[agent] 16/20 | ✅ | 17**2 | gt=289 | pred=289 +[agent] 17/20 | ✅ | 11**2 | gt=121 | pred=121 +[agent] 18/20 | ✅ | 72+10 | gt=82 | pred=82 +[agent] 19/20 | ✅ | (84)-60 | gt=24 | pred=24 +[agent] 20/20 | ❌ | (348/(12))-(28)*(8) | gt=-195 | pred=3.625 + +============================================================ +full_sft: 12/20 = 60.00% +agent: 17/20 = 85.00% +``` + +### 👉 Comprehensive Evaluation 1 + +From these results, the current `agent` compared to `full_sft` has already clearly pulled ahead on lightweight Agent tasks with tool calling. Especially in this type of problem where the model must "first decide whether to call a tool, then get the verifiable result correct," `agent` has a higher success rate, indicating that after RL, the model has indeed learned stronger calling and utilization capabilities on the ToolUse track. + +However, this improvement does not come without a cost. `agent` is better suited for these light Agent / ToolUse scenarios, but this does not mean it simultaneously becomes stronger on general Q&A. In actual experience, these weights typically see decreased stability on factual questions, with more noticeable knowledge hallucinations, and are more prone to the phenomenon of "doing better on tool tasks, but being more willing to fabricate on open-ended Q&A." + +So if the task objective is ToolUse, lightweight multi-step calling, and verifiable solving, then the current `agent` will be more suitable than `full_sft`; but if you care more about factuality and overall robustness of daily chat and knowledge Q&A, `full_sft` would still be a slightly more stable choice. Looking at the bigger picture, this is actually a very common phenomenon in current industry post-training models: post-training can often pull a specific capability line very high, but almost always comes with quite noticeable "alignment tax." That is to say, while the model becomes stronger on specific reward targets, it usually sacrifices some generality, factuality, or stability under the natural distribution. This problem is not new and is very difficult to completely avoid; fundamentally, after the optimization objective narrows, the model becomes increasingly good at "catering to the current reward definition," which does not necessarily equate to a lossless improvement in overall capability. + +## Ⅱ Comparison with Other Models + +> Note: The following comparison is only for experiential reference, not a strict benchmark; sample size is limited and involves subjectivity. + +**[A]** minimind-3 (0.06B) +**[B]** minimind-3-moe (0.2B-A0.06B) +**[C]** [baby-llama2-chinese (0.2B)](https://github.com/DLLXW/baby-llama2-chinese) +**[D]** [chatlm-mini-chinese (0.2B)](https://github.com/charent/ChatLM-mini-Chinese) + +### Test 3: Q&A ```text [Q]: 你知道长江吗？ -[A]: 长江，位于中国东南部，是中国的第一长河，全长6300多公里，流经11个省份，是中国最长的河流，也是中国的"母亲河"。长江的历史可以追溯到公元前2000多年的商朝时期，当时的商朝是商朝的后裔。商朝时期，长江的经济发展达到了高峰，是中国最重要的农业和工业中心之一。长江流域的经济主要以农业为主，长江渔业占据了主要产业，如钓鱼、猎鱼等。长江的水资源也非常丰富，包括河流、湖泊、河流等。长江的自然景观也十分壮观，长江四季景色各异，春季多雨，夏季多雨，秋季则多雨，冬季则寒冷多雪，春季和秋季都有美丽的花朵和果实。长江是中国的经济中心，拥有众多的国家和城市，是世界上重要的交通枢纽和航运中心。同时，长江还是中国的重要水利资源，对于中国的经济发展具有重要意义。 -[B]: 是的，我知道长江。长江位于中国的中部，是中国第三长的河流，全长约6300公里，流经江西省、安徽省、江苏省、安徽省、江苏省和安徽省。 -[C]: 当然，长江是中国最长的河流，全长约6300公里。它以其丰富的水资源、丰富的水生动植物资源以及独特的地理位置而闻名于世。 -[D]: 是的，长江是中国最长的河流，全长超过3300千米。 -[E]: 长江是中国最长的河流，总长度约为6300公里。它发源于青藏高原的青海湖，流经青海、四川、甘肃、宁夏、内蒙古、陕西和山西。长江流域是中国最大的农业区之一，也是中国的文化和经济中心。 -[F]: 是的，长江是中国最长的河流之一，也是世界第三大河流。长江发源于青藏高原，流经11个省份，最终注入东海。长江流域是中国最大的河流流域之一，拥有着丰富的自然资源和文化遗产，如长江大桥、长江海洋博物馆、长江古城等。 -[G]: 同时也知道，中国是世界上最长的城市之一，拥有超过3400万人口。 -[H]: 是的，长江是中国最长的河流，也是世界上最长的河流之一。它发源于青藏高原，流经九个省区，最终注入东海。长江流域的气候温和，四季分明，是中国重要的农业和工业基地。长江流域的水产资源丰富，是中国的重要渔业资源之一。 +[A]: 长江，全称为中国长江，是中国的官方名称，是中国的重要省份，也是世界上最大的淡水河流。长江流域位于中国东部沿海，地理位置独特，拥有广阔的沿湖生态系统，是世界上最大的淡水河流之一。长江流域位于中国东南半岛，东临东南海，西临太平洋，东临西海，西近太平洋，东临太平洋，西至南半岛，地理位置独特，有广袤的沿海和巨型的水域，是全球重要的水资源。长江流域不仅见证了中国东南亚地区的变迁，也对全球文明产生了深远的影响。长江流域以其独特的生态系统和丰富的生物多样性闻名，是中华文明的象征之一。长江流域的生态系统包括了丰富的水生植物种类，如水稻、玉米、小麦等，这些植物不仅为中国的农业生产提供了丰富的食物来源，还对周边的生态系统有重要影响。此外，长江流域还孕育了各种水生生物，如鲨鱼、海鸟等，这些生物在长江的生态系统中扮演着关键角色。长江流域的生态系统是全球的重要组成部分，它不仅见证了长江流域的变迁，也对全球的生态平衡和农业生产起到了重要作用。通过旅游、文化、旅游等方式，长江流域的自然景观、农事活动、旅游景点等都为人们提供了了解长江流域丰富的资源和体验的平台。 +[B]: 长江，又称为珠穆朗玛峰，位于中国和尼泊尔之间，是世界上海拔最高的山峰，位于喜马拉雅山脉，海拔8848.85米。 +[C]: 同时也知道，中国是世界上最长的城市之一，拥有超过3400万人口。 +[D]: 是的，长江是中国最长的河流，也是世界上最长的河流之一。它发源于青藏高原，流经九个省区，最终注入东海。长江流域的气候温和，四季分明，是中国重要的农业和工业基地。长江流域的水产资源丰富，是中国的重要渔业资源之一。 + [Q]: 世界上最高的山峰是什么？ -[A]: 世界上最高的山峰是珠穆朗玛峰（Mount Everest），位于喜马拉雅山脉，尼泊尔和中国的边界上。它的海拔高度是8,848.86米（29,031.7英尺），是地球上最高的山峰。 -[B]: 珠穆朗玛峰（Mount Everest）是世界上最高的山峰，海拔高度达8,848米（29,029英尺）。 -[C]: 珠穆朗玛峰是世界上最高的山峰，位于喜马拉雅山脉，海拔8,848.86米（29,031.7英尺）。它位于尼泊尔和中国的边界上，是喜马拉雅山脉的一部分。 -[D]: 世界上最高的山峰是珠穆朗玛峰，位于尼泊尔和中国的边境上，海拔8848米。 -[E]: 世界上最高的山峰是珠穆朗玛峰，位于喜马拉雅山脉，海拔8,848米。 -[F]: 珠穆朗玛峰是世界上最高的山峰，海拔8848米。 -[G]: 世界上最高的山峰是珠穆朗玛峰，位于喜马拉雅山脉，海拔8,848.86米（29,031.69英尺）。 -[H]: 世界上最高的山峰是珠穆朗玛峰，也称为萨加玛塔，位于喜马拉雅山脉的尼泊尔和中国边境之间，海拔8,848.86米（29,031.69英尺）。它是世界上最著名的登山目标之一，吸引着来自世界各地的登山者前来挑战。珠穆朗玛峰的名字来自藏语，意思是"女神的额峰"，它是由两个不同的山峰组成的，其中一个是位于尼泊尔的珠穆朗玛峰，另一个则位于中国。它的高度是通过测量海拔高度来确定的，因为它是通过测量地球的平均高度来确定的。尽管珠穆朗玛峰是世界上最高的山峰，但它并不是地球上任何其他地方都能到达的。地球上的其他地方，如南美洲的安第斯山脉，也是世界上最高的山脉，但它并不是地球上最高的山峰。总之，珠穆朗玛峰是世界上最高的山峰，但它并不是地球上任何地方都能到达的。它是地球上最高的山脉，但它并不是地球上任何地方都能到达的。 +[A]: 世界上最高的山峰是珠穆朗玛峰（Chعام الحدوة）。珠穆朗玛峰（Chirney 峰）位于尼泊尔和中国的中部，海拔8848.86米（864.41英尺），是世界上海拔最高的山峰。 +[B]: 世界上最高的山峰是珠穆朗玛峰（Mount Everest），其高度约为8,848.86米（8,848.86英尺）。珠穆朗玛峰的高度是地球上最高的山峰，也是世界上海拔最高的山峰之一。 +[C]: 世界上最高的山峰是珠穆朗玛峰，位于喜马拉雅山脉，海拔8,848.86米（29,031.69英尺）。 +[D]: 世界上最高的山峰是珠穆朗玛峰，也称为萨加玛塔，位于喜马拉雅山脉的尼泊尔和中国边境之间，海拔8,848.86米（29,031.69英尺）。它是世界上最著名的登山目标之一，吸引着来自世界各地的登山者前来挑战。珠穆朗玛峰的名字来自藏语，意思是"女神的额峰"，它是由两个不同的山峰组成的，其中一个是位于尼泊尔的珠穆朗玛峰，另一个则位于中国。它的高度是通过测量海拔高度来确定的，因为它是通过测量地球的平均高度来确定的。尽管珠穆朗玛峰是世界上最高的山峰，但它并不是地球上任何其他地方都能到达的。地球上的其他地方，如南美洲的安第斯山脉，也是世界上最高的山脉，但它并不是地球上最高的山峰。总之，珠穆朗玛峰是世界上最高的山峰，但它并不是地球上任何地方都能到达的。它是地球上最高的山脉，但它并不是地球上任何地方都能到达的。 + + +[Q]: 万有引力是谁提出的？ +[A]: 万有引力是由爱因斯坦在1905年提出的，这一概念由他提出。爱因斯坦在20世纪初提出，这是基于阿尔伯特·爱因斯坦在1921年提出的，他在20世纪初提出了广义相对论，这是现代物理学的基础。万有引力理论为物理学提供了一个数学框架，即在任何时刻，都存在一个具有引力的力，这种引力可以说是微粒之间的相互作用。万有引力的概念最早由他在1914年提出，当时他发现物体之间存在着引力，并且这种引力可以被视为一个非常简单的质量和能量总和。根据爱因斯坦的质疑，在1918年，地球上所有的天体都存在一个质量为2的质点，且质量的平方是 \(2^2 = 4\)。在这种质点上，质量的引力与它的质量成正比，与它所处的宇宙的大小成正比。万有引力理论在1919年被认为是天文学的前身，但至今仍未被正式提出。他还提出了广义相对论，这是现代宇宙学的基石。尽管如此，万有引力理论在当时被认为是理论物理学的里程碑，对后来的宇宙学研究产生了深远的影响。 +[B]: 万有引力是由爱因斯坦提出的一种力理论，主要应用于天体物理学和宇宙学研究中。 +[C]: 引力是宇宙中一种力量，它是由质量或能量所组成的。根据牛顿的万有引力定律，一个物体之间的引力大小与它的质量成正比，与它的质量成反比。因此，牛顿的万有引力定律表明，一个物体之间的引力大小与它的质量成正比，与它的质量成反比。这个定律在物理学、天文学和物理学中都有广泛的应用。 +[D]: 万有引力是由英国物理学家艾萨克·牛顿于1687年提出的。他在1687年发表了一篇著名的论文，题为《自然哲学的数学原理》，其中提出了万有引力定律，指出任何两个物体之间都存在着一种引力，这种引力的大小与它们的质量成正比，与它们之间的距离的平方成反比。这个定律被称为万有引力定律，它描述了质量和距离之间的关系。它表明，两个物体之间的引力与它们的质量成正比，与它们之间的距离的平方成反比。这个定律对于解释宇宙中的许多现象和现象之间的相互作用非常重要，例如行星绕太阳运动、恒星之间的引力透镜效应等等。它也被广泛应用于天文学、物理学和工程学等领域，为我们理解宇宙的本质和演化提供了重要的理论基础。 -[Q]: 谁提出了万有引力定律？ -[A]: 万有引力定律是由牛顿在17世纪提出的，是牛顿三大定律之一。牛顿在17世纪提出的万有引力定律揭示了万有引力定律的本质，并解释了行星、卫星、彗星和其他天体之间的引力相互作用。这一理论是由牛顿在1687年提出的。通过分析和实验，他成功地解释了天体之间的引力相互作用，从而解释了地球绕太阳公转的现象。 -[B]: 万有引力（人类政策）是牛顿三大力学定律之一，用于描述两个物体之间的相互作用。它包括万有引力定律和牛顿第一定律（惯性定律）。万有引力定律来源于两个物体之间的相互作用，其质量、距离和速度决定了它们之间的相互作用。这种相互作用是：质量的乘积：质量的乘积是万有引力定律和牛顿第二定律的乘积。质量的乘积：质量的乘积是两个物体之间的物理过程，两个物体之间的相互作用。对于其他物体，它们的质量可以表示为两个物体之间的质量之和，即引力加速度a。 -[C]: 万有引力是牛顿在1687年提出的物理概念，描述了物体之间的引力相互作用。 -[D]: 万有引力是由英国物理学家牛顿提出的。他相信所有物体都会产生引力，这种引力是由物体的质量和距离决定的。这一理论揭示了质量和距离之间的关系，并证明了万有引力定律的本质。 -[E]: 万有引力是由牛顿在1687年提出的，它是宇宙中的一个基本物理现象，描述了物体之间的相互作用。这种力被称为万有引力，存在于所有物体之间，这种力与物体的质量和距离有关。万有引力是自然界中最基本的力之一，不仅影响物体的运动和变化，而且对宇宙的结构和演化也有重要影响。 -[F]: 万有引力是由英国物理学家牛顿在17世纪提出的。 -[G]: 引力是宇宙中由质量和能量组成的力。根据牛顿的万有引力定律，一个物体的引力与它的质量和距离成正比。因此，牛顿的万有引力定律表明，一个物体的引力与它的质量和距离成正比。这一定律在物理学、天文学和物理学中都有广泛的应用。 -[H]: 万有引力是由英国物理学家牛顿在1687年提出的。他于1687年发表了一篇著名的论文《自然哲学的数学原理》，提出了万有引力定律，指出任何两个物体之间都存在引力，这种引力与它们的质量和距离的平方成反比。这一定律被称为万有引力定律，描述了质量和距离之间的关系。它指出两个物体之间的引力与它们的质量和距离的平方成反比。这一定律对于解释许多现象和宇宙中的相互作用非常重要，例如行星绕太阳公转、恒星之间的引力透镜效应等。它也在天文学、物理学和工程学领域得到了广泛应用，为我们理解自然和宇宙的本质和演化提供了重要的理论基础。 [Q]: 你知道大熊猫的主要食物是什么吗？ -[A]: 大熊猫主要以竹子为食，但也吃其他植物、昆虫、小型哺乳动物和鱼。它们的消化系统与熊相似，能够消化竹子。大熊猫有锋利的牙齿，适合咀嚼竹子，所以它们通常被称为"竹子动物"。 -[B]: 大熊猫是草食哺乳动物，以竹子为食，也是肉食动物。它们的主要是竹子，尤其是富含纤维素和矿物质的竹子，这些竹子不会直接进入消化系统。大熊猫的肠道主要由竹子组成，需要消化纤维素和其他纤维在竹子中以支持正常喂养。 -[C]: 大熊猫主要以竹子为食，但也吃其他植物，包括叶子、嫩芽、昆虫、水果、蔬菜和小型哺乳动物。大熊猫是中国特有的，属于熊科，主要分布在四川、陕西和甘肃地区。 -[D]: 大熊猫的主要食物是竹子。它们通常吃竹子，每天需要吃大约30公斤的竹子。 -[E]: 是的，大熊猫主要以竹子为食。它们需要每天吃大量的竹子来维持生命，每天需要吃40到80公斤的竹子。此外，它们还吃一些其他植物，如草、水果、坚果和小型哺乳动物。 -[F]: 大熊猫主要以竹子为食，但偶尔也会吃其他植物和动物。此外，大熊猫还吃一些野生水果、小型动物和植物，但这些并不是它们的主要食物来源。 -[G]: 大熊猫的主要食物是竹子。 -[H]: 大熊猫是植物性动物，主要食物是竹子。它们的肠道结构适合消化富含纤维素的竹子，但它们也吃一些其他植物，如草本植物和小型哺乳动物。大熊猫的肠道结构也适应了高纤维竹子，使它们能够消化大量的纤维素，帮助它们在野外生存。大熊猫的肠道中也含有一种叫做"纤维素酶"的酶，可以帮助大熊猫分解纤维素，帮助它们消化大量的水和营养物质。此外，大熊猫的肠道中也含有一种叫做"纤维素蛋白"的蛋白质，可以帮助大熊猫维持身体能量水平，帮助它们在野外生存。总之，大熊猫的主要食物是竹子，但它们也吃一些其他植物，如草本植物和小型哺乳动物。 +[A]: 大熊猫是一种非常受欢迎的动物，主要以竹子为食，尤其是竹子的外壳和内部的脂肪层。大熊猫的饮食多样性非常丰富，包括竹子、竹子、木头和草本植物等。以下是一些大熊猫的主要食物：1. **竹子**：主要来源于竹子中的竹子，尤其是竹子纤维的部分，因其纤维含量高而成为大熊猫的"榉"，因其纤维含量高而成为榉木。2. **竹子**：竹子中含有丰富的纤维，如竹子中的纤维，以及其他天然纤维，如藤蔓和竹丝。这些纤维有助于促进竹子的形成和分布。3. **竹子**：竹子中含有丰富的纤维，包括纤维和脂肪层，以及其他天然纤维，如紫薯、甜薯、花蕊等，这些纤维不仅提供了碳水化合物，还富含蛋白质和纤维素。4. **竹子**：竹子的生长和消化特性使其成为大熊猫的主要食物来源，包括竹子中的纤维。竹子的消化过程也起着重要作用，通过纤维的吸收和分解，能够为大熊猫提供大量的食物来源。5. **竹子**：竹子的消化过程也起着关键作用，它们通过胎生的方式将竹子转化为纤维，吸收水分和营养物质，保持大部分营养物质的供应。6. **竹子**：竹子还含有丰富的纤维和脂肪层，这些物质有助于分解大部分营养物质，提供能量和氧气。需要注意的是，大熊猫的饮食不仅仅是食物，它们还对大熊猫的生存环境、繁殖习性以及繁殖习性等有重要影响。因此，大熊猫的饮食应根据其健康状况和野生动物的保护需求进行适当的调整。 +[B]: 大熊猫是亚洲的一种濒危物种，其主要食物来源包括竹子。不过，作为一个AI助手，我并没有实际的食谱或食物来源。不过，我可以提供一些关于大熊猫饮食的基本知识和注意事项。大熊猫是食草动物，以竹子为主要食物来源，但它们还吃海产品，包括鱼类、鸟类、哺乳动物和偶尔的海产品。大熊猫的饮食通常更为丰富，包括竹子、水果、小型动物（如松鼠、兔子）和一些其他植物。大熊猫的饮食习惯受到多种因素的影响，包括人类活动、自然环境、以及人类的干扰。以下是一些关于大熊猫饮食的常见误解：1. **人类干扰**：在食物供应中，人类可能会干扰大熊猫的饮食习惯，尤其是鱼类和小型动物，这些食物可能含有过多的添加剂和人工添加剂。2. **遗传因素**：人类的饮食习惯与遗传有关，但通常情况下，人类的饮食偏好是由自然选择的驱动，自然界的食物链中包含的碳、碳循环和不利影响。3. **食物来源**：尽管大熊猫主要以竹子为食，但它们的狩猎方式和食物来源也受到自然环境的影响。人类的捕猎活动可能不会直接改变大熊猫的饮食结构，但这与人类的饮食习惯可能有关。4. **饮食周期**：大熊猫的饮食周期通常包括几个月到几个月，这取决于全球人口的分布、食物类型（如鱼类、鸟类、哺乳动物和偶尔的海产品）以及它们的饮食习惯。5. **气候变化**：全球气候变化可能影响大熊猫的食物链，这可能导致它们的狩猎和进食模式发生变化。总之，虽然大熊猫主要以竹子为食，但它们的狩猎方式和食物来源可能会改变它们的饮食习惯，但与人类的饮食习惯可能并不是直接的联系。保持观察和了解大熊猫饮食的多样性是维护生态平衡和动物福利的重要措施。 +[C]: 大熊猫的主要食物是竹子。 +[D]: 大熊猫是一种以竹子为主要食物的植食动物，它们的肠道结构适合消化纤维素含量高的竹子，但也会吃一些其他植物，如草本植物和小型哺乳动物。大熊猫的肠道结构也适应了高纤维的竹子，这使得它们能够消化大量的纤维素，从而帮助它们在野外生存。大熊猫的肠道中还含有一种叫做"纤维素酶"的酶类物质，这种酶能够帮助大熊猫分解纤维素，从而帮助它们消化大量的水分和营养物质。此外，大熊猫的肠道还含有一种叫做"纤维素蛋白"的蛋白质，这种蛋白质能够帮助大熊猫维持体内的能量水平，从而帮助它们在野外生存。总的来说，大熊猫的主要食物是竹子，但它们也会吃一些其他植物，如草本植物和小型哺乳动物。 -[Q]: 为什么海水是咸的？ -[A]: 海水是咸的主要是因为它的化学成分。海水含有大量的矿物质和盐分，主要成分是硫酸盐、碳酸盐、镁离子和钾离子。这些物质在高浓度下会在海水中沉淀，形成海水。具体来说，海水中矿物质和盐分主要来自岩石、土壤、生物和人类排泄物。岩石是海水的组成部分，主要由石英、长石、云母等矿物质组成。土壤中含有大量的化学物质，如氮、磷、钾等，这些化学物质会渗入土壤，形成土壤。生物排泄物中的盐分主要来自植物叶绿素和其他色素，这些色素吸收阳光并将其转化为能量，供生物使用。人类排泄物中的盐分主要来自矿物质和碳酸盐，这些矿物质在水中高浓度存在，导致水中溶解矿物质浓度增加，形成海水。因此，海水是咸的主要是因为它的化学成分和结构特征。矿物质和盐分在海水中主要来自岩石、土壤、生物和人类排泄物，这些物质在海水中积累和排泄，导致海水变咸。 -[B]: 海水是咸的，因为它含有多种物质，包括氯化钠（NaCl）、氯化钠（NaCl）、碳酸盐（如硫酸盐）、氯化钠（NaCl）和水。咸物质的来源：盐是一种常见的咸味来源，含有钾、钙、镁、钾和其他矿物质。这些矿物质通常来自地下水、地下水或土壤，浓度远低于纯水。咸味的原因：咸味主要来自水中的小颗粒盐。这些小颗粒主要通过植物根部和动物体内的微生物传播，如昆虫、小型哺乳动物、鸟类和某些昆虫。总之，海水是咸的，因为它含有大量的盐分，包括矿物质、岩石、盐、矿物质等，这些物质在海水中高浓度存在，对咸味产生重要影响。 -[C]: 海水是咸的主要是因为它含有各种盐分，海水是其中主要成分之一。当海水含有大量盐分时，这些盐分溶解在海水中形成咸味。海水中盐分主要来自溶解盐分、蒸发沉淀和冰冻海水形成的盐分。 -[D]: 海水中含有许多盐分，这些盐分可以增加海水的咸度。海水中盐分主要来自海水中溶解的盐分，这些盐分可以从海水中获得。氧化物和海水中盐分可以与海水中盐分形成氢键，从而增加海水的咸度。 -[E]: 海水是咸的，因为它含有大量的盐分，包括氯离子和氯离子。这些离子在海洋中形成大量盐分，称为盐。海水中含有许多溶解的盐分，这些盐分随着海水的上升而蒸发，最终导致海水中盐分蒸发。这些盐分完全被海水消耗，然后从海水中排除。此外，海水中还含有许多其他盐分，但这些盐分不同，因为它们的浮动和下沉过程不同。海水中提取的盐分来自海底的岩石，这些盐分在海水中可利用。总之，海水是咸的，因为海水中含有许多盐分和盐分。 -[F]: 海水中含有许多溶解的盐分，其中最主要的是氯化钠。当盐加入水中时，盐分子与水分子反应形成氢氧化钠，这是一种盐。 -[G]: 因为许多生物需要水才能生存，它可以通过吸收和释放二氧化碳、释放氧气、保持大气中的水蒸气、保持水循环来实现。水中的盐分和化学物质释放氧气，使地球上的生物能够呼吸。此外，水还有许多其他功能，如保持水循环、调节气候和环境等。水对许多生物的生存至关重要，也是许多自然现象的重要基础。 -[H]: 海水是咸的主要是因为它含有大量的溶解盐分，其中最主要的是氯化钠（NaCl）。氯化钠是一种非常常见的盐，存在于地球上的海洋、湖泊、河流和地下水中。当海水从海洋蒸发时，它会带走一些盐，使海水变咸。此外，海洋中还含有其他溶解盐分，如硫酸盐、镁硫酸盐、钙硫酸盐、镁钾硫酸盐、镁钠硫酸盐、镁钙硫酸盐、[以及这些化合物的重复组合]... -[Q]: 写一个Python快速排序代码 -[A]: 好的，以下是Python快速排序代码： def quick_sort(arr): if len(arr) <= 1: return arr else: pivot = arr[len(arr) // 2] left = [x for x in arr[1:]] right = [x for x in arr[2:] if x < pivot] return quick_sort(left) + [pivot] + quick_sort(right)# Test code arr = [3, 6, 8, 10, 1, 2, 1]print(quick_sort(arr))This code first checks if array length is less than or equal to 1. If so, it arranges from smallest to largest. Next, for each element in array, it wants to recursively sort until reaching sort end position. If array length is odd, it arranges from largest to smallest. Finally, average time complexity is O(n log n), where n is array length. -[B]: 以下是使用Python实现的快速排序算法： def sort_array(arr): # Generate random arrangement sorted_array = sorted(arr) # Return sorted list return sorted(arr)# Example arr = [3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5]# Call function sorted_array = sort_array(arr)print("Original array is: ", sorted_array)This code first generates random arrangement, then sorts through a list's length. Finally returns sorted list. Note that this code uses Python's built-in sorted function and built-in sorted() function, which are faster than directly using sorted() function at O(n log n) time complexity. -[C]: 以下是适用于小数组的简单排序算法： def quick_sort(arr): if len(arr) <= 1: return arr else for _ in range(2, 51): if arr[len(arr) - 1:] is not None: arr[len(arr) - 1] = arr[len(arr) - 1] arr[len(arr)] = arr[len(arr)] return quick_sort(arr)# Example arr = [3, 5, 1, 4, 1, 2, 1]sorted_arr = quick_sort(arr)print("Sorted array is:", sorted_arr)### Code Explanation: 1. **Function Definition**: quick_sort function takes an array arr as parameter. 2. **Loop**: For arr elements, uses recursion, traversing from right to left, swapping current element with current array element until array reaches empty or array length <= arr. 3. **Recursive Call**: Traverse array, swap current element with current array element until array reaches empty or array length <= arr. 4. **Check Array Length**: Use if __name__ == "__main__" to check if array length < arr, if so move array from larger array to array end. 5. **Output Sorted Results** -[D]: 作为AI语言模型，我无法编写Python程序。然而，Python是一个非常流行的编程语言，拥有许多优秀的库和框架，包括NumPy、Pandas、Matplotlib、Pandas、Bandke、Norm和NKlearn等。Python有许多优秀的自然语言处理库和框架，用于处理和分析各种类型的数据，如文本、图像、音频、视频等。Python还可以创建各种应用程序，如网络搜索、游戏开发、科学计算等。Python非常灵活，也支持多种编程风格，因此可以轻松找到合适的解决方案。 -[E]: 作为AI语言模型，我可以回答你的问题。然而，我无法执行编程操作，因为我无法确定你询问的是哪种编程语言。请提供更具体的问题，以便我更好地回答你的问题。 -[F]: 快速排序（快速排序）是一种简单的排序算法，它将数据分成小部分，通过规则对每个部分进行排序，最终得到一个有序数组。 -[G]: 好的，以下是快速排序的Python代码 ```python sorted_print(sorted(sorted(sorted(sorted_print))```Output is: ```[1, 2, 3, 4, 5, 6, 7, 8, 9]``` -[H]: 快速排序是一种基于分治法的排序算法，时间复杂度为O(n log n)。它的基本思想是：通过一次排序通过，将序列分成两个子序列，其中一个子序列中的所有元素都小于另一个子序列中的所有元素，然后递归地对这两个子序列进行排序。以下是实现快速排序算法的Python代码： ```pythondef quick_sort(arr): if len(arr) > 1: return arr else: return quick_sort(arr)```This code uses递归，首先判断数组长度是否<=1，如果是，则直接返回数组，否则递归地对数组进行排序。在递归过程中，我们首先判断第一个数组元素是否>=1，如果是，则比较数组元素与第一个元素，如果不是=1，则比较数组元素与第二个元素，等等，直到所有元素都排序完毕。这种代码的时间复杂度为O(n log n)，是一种非常实用的排序算法。 +[Q]: 海水为什么是咸的？ +[A]: 海水的咸度是由海水的重量以及海水的盐度共同作用的结果。这种变化主要是由于海水在阳光照射下会吸收水中的盐分，从而形成咸味。1. **阳光的反射**：当阳光照射到海水表面时，水中的盐分会渗透到海水的表面，而不是海水表面。这种反射的过程称为光的反射。2. **盐水的温度**：海水的盐度与普通海水中的水的温度有关。普通海水中的盐分（即海水的盐度）会逐渐凝结成海水，这个过程称为"光散射"。由于海水的温度变化，这些水中的盐分会逐渐被海水吸收，从而导致海水呈现出咸味。3. **海水的物理性质**：海水在阳光照射下会吸收海水中的盐分，这些盐分会逐渐释放到大气中，进一步凝结成咸味物质。海水中的水的体积相对较小，因此在形成咸味物质时，水中的盐分会被吸收。4. **水的物理性质**：海水的咸味物质主要由碳酸钙和钙离子组成，而海水的咸味物质主要由钙离子组成，它们在阳光照射下会吸收水中的热量。这种现象是海水咸味物质与海水的化学反应的一种体现。5. **海水的物理性质**：海水的盐度分布和性质也会影响海水的咸味物质的分布。例如，一些海水的盐度接近于100%，这可能与海水的盐度非常接近，从而引起了海水咸味物质的形成。总之，海水咸度是由于海水的物理性质、物理性质以及海水的化学成分共同作用的结果。 +[B]: 海水的咸度是由其所含物质的盐分占据的，这一现象可以从几个方面来解释：1. **盐分占据的作用**：咸度是指在一定温度下，部分水分含量的相对量。在正常条件下，咸度较高的水分含量会增加，使得渗透压较小，这部分的水分会减少，使得渗透压相对较低。这是因为咸度的增加使得盐分在水中的浓度增加，盐分的增加意味着渗透压较小，使得渗透压相对较高。2. **淡水的来源**：海水中的淡水主要来源于海水。这些淡水是海洋生态系统的重要组成部分，对于维持海洋生态平衡、调节气候、提供水源、以及支持生物多样性等都至关重要。咸度的提高会减少渗透压，使得渗透压相对较低，从而使得渗透压相对较高。3. **海水的含量**：海水是地球上所有生物的血液，其中含有大量的盐分。咸度的增加使得渗透压相对较低，这意味着渗透压相对较高。当海水的盐分含量增加时，渗透压相对较低，渗透压相对较高。这种现象不仅对海水的物理性质有影响，也对人类的饮用水安全具有重要意义。4. **咸度与海水的关系**：海水中的盐分在咸度增加的情况下，虽然盐分的增加本身也增加了渗透压，使得渗透压相对较高，从而使得渗透压相对较高。这是因为咸度的增加使得渗透压相对较高，渗透压相对较低，从而使得渗透压相对较高。综上所述，海水的咸度是通过其所含物质的盐分占据的，这部分水分子在咸度增加的情况下，渗透压相对较高，同时渗透压相对较低，这导致渗透压相对较低，渗透压相对较高。 +[C]: 因为水是许多生物的生存所必需的，它能够吸收和释放二氧化碳，释放氧气，维持大气中的水蒸气，以及维持水循环。水中的盐类和化学物质会释放出氧气，使地球上的生物呼吸。此外，水还具有许多其他功能，如保持水循环、维持水循环、调节气候和环境等。水是许多生物生存所必需的，它也是许多自然现象的重要基础。 +[D]: 海水是咸的主要原因是因为其中含有大量的溶解盐类，其中最主要的是氯化钠（NaCl）。氯化钠是一种非常常见的盐类，它存在于地球上的大部分地区，包括海洋、湖泊、河流和地下水。当水从海洋中蒸发时，它会带走一部分盐分，使海水变得咸。此外，海中还含有其他溶解盐类的物质，如硫酸盐、硫酸镁、硫酸钙、硫酸镁盐、硫酸镁钾、硫酸镁钠、硫酸镁钙、硫酸镁镁钙、硫酸镁镁钙、硫酸镁镁钙、硫酸镁镁钠、硫酸镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁钙、硫酸镁镁镁镁镁钙、硫酸镁镁镁镁镁钙、硫酸镁镁镁镁镁钙、硫酸镁镁镁镁镁钙、硫酸镁镁镁镁镁镁钙、硫酸镁镁镁镁镁镁钙、硫酸镁镁镁镁镁镁钙、硫酸镁镁镁镁镁镁镁钙、硫酸镁镁镁镁 -[Q]: 截至2025-02-09 14:31，电影《哪吒：魔童降世》累计票房（包括预售）突破7.809亿元，超过《星球大战：天行者崛起》票房成绩，成为首部进入全球票房前40名的亚洲电影。这一成就不但标志着中国电影在国际市场上的突破，也展示了中国动画电影的巨大潜力和市场吸引力。总结这段文字中的20个字。 -[A]: "哪吒：魔童降世"票房突破7.809亿元，成为首部进入全球票房前40名的亚洲电影。这标志着中国电影在国际市场上的突破，也展示了中国动画电影的巨大潜力和市场吸引力。 -[B]: 2月9日，中国电影在世界舞台上突破7.809亿元，超过《星球大战：天行者崛起》票房成绩。 -[C]: "哪吒：魔童降世"票房突破7.809亿元，成为全球票房前40名的亚洲电影，展示了中国动画电影的巨大潜力和市场吸引力。 -[D]: "哪吒：魔童降世"累计票房突破7.809亿元，成为首部进入全球票房前40名的亚洲电影，展示了中国电影在国际市场上的突破，也展示了中国动画电影的巨大潜力和市场吸引力。 -[E]: "龙之力量：世界幻想冒险"吸引了众多观众，同时也吸引了广泛关注。 -[F]: "哪吒：魔童降世"是一部使中国动画电影成为巨大潜力的电影。这部电影因其华丽的视觉效果和令人惊叹的战斗场面而闻名。 -[G]: "哪吒：魔童降世"是否值得价值"是2025年1月9日14:31，电影《哪吒：魔童降世》票房突破78亿元，成为中国动画电影的突破，也展示了中国动画电影的巨大潜力和市场吸引力。 -[H]: 我是一个小参数AI模型😅，知识库有限，无法直接回答你的问题。尝试换个问题👋 + +[Q]: 写一段python的快速排序代码 +[A]: 在Python中，快速排序是一种高效的排序算法，它使用分治法（Divide and Conquer）策略来将一个序列分为较小和较大的两个子序列。以下是一个简单的Python快速排序代码：```python def quick_sort(arr): if len(arr) <= 1: return arr pivot = arr[len(arr) // 2] left = [x for x in arr if x < pivot] middle = [x for x in arr if x == pivot] right = [x for x in arr if x > pivot] return quick_sort(left) + middle + quick_sort(right) # 示例 arr = [3, 6, 8, 10, 1, 2, 1] print(quick_sort(arr)) # 输出: [1, 1, 2, 3, 6, 8, 10] ```这段代码首先检查输入数组的长度是否小于等于1。如果是，则直接返回数组。接下来是数组的长度，如果是数组，则直接返回数组。然后是数组中的元素，如果数组长度小于等于1，则直接返回数组。最后是数组中的元素，如果数组包含重复的元素，则不返回任何元素。 +[B]: 快速排序是一种高效的排序算法，采用分治策略来将一个序列分为较小和较大的两个子序列，然后递归地排序这两个子序列。以下是快速排序的Python实现代码：```python def quick_sort(arr): if len(arr) <= 1: return arr pivot = arr[len(arr) // 2] left = [x for x in arr if x < pivot] middle = [x for x in arr if x == pivot] right = [x for x in arr if x > pivot] return quick_sort(left) + middle + quick_sort(right) # 示例 arr = [3,6,8,10,1,2,1] print("Original array:", arr) sorted_arr = quick_sort(arr) print("Sorted array:", sorted_arr) ```这段代码首先检查输入数组的长度，如果长度小于等于1，那么数组已经是排序好的，直接返回。然后选择一个基准值（pivot），将数组分为三部分：小于基准值的元素、等于基准值的元素和大于基准值的元素。接着递归地对小于和大于基准值的元素进行快速排序，并将结果合并。 +[C]: 好的，以下是一段快速排序的Python代码```python sorted_print(sorted(sorted(sorted_print))```输出结果为：```[1, 2, 3, 4, 5, 6, 7, 8, 9]``` +[D]: 快速排序是一种基于分治思想的高效排序算法，它的时间复杂度为 O(n log n)。它的基本思想是通过一趟排序将待排序序列划分为两个子序列，其中一个子序列的所有元素都比另一个子序列的所有元素小，然后对这两个子序列分别递归地进行排序。下面是一个 Python 代码示例，实现了快速排序算法：```pythondef quick_sort(arr): if len(arr) > 1: return arr else: return quick_sort(arr)```该代码使用了递归的方式，首先判断数组的长度是否小于等于1，如果是，则直接返回数组，否则递归地对数组进行排序。在递归过程中，我们首先判断数组的第一个元素是否大于等于1，如果是，则将数组中的元素与第一个元素进行比较，如果不等于1，则将数组中的元素与第二个元素进行比较，以此类推，直到所有元素都被排序。该代码的时间复杂度为 O(n log n)，是一个非常实用的排序算法。 + + +[Q]: 截至2025年2月9日14时31分，电影《哪吒之魔童闹海》累计票房（含预售）突破78.09亿元，超过《星球大战：天行者崛起》的票房成绩，成为首部进入全球票房榜前40的亚洲电影。这一成就不仅标志着中国电影在国际市场的突破，也展示了中国动画电影的巨大潜力和市场吸引力。从这段文字总结20字的摘要给我。 +[A]: 《哪吒之魔童闹海》的票房成绩，从短短几十年来一直是全球票房排名中的关键因素，它不仅标志着中国电影在国际市场的突破，也极大地增强了票房和观众的认同感。以下是对这一成就的简要总结：1. **票房表现**：《哪吒之魔童闹海》获得了众多票房，不仅在票房上取得了巨大成功，还获得了多个国际电影奖项，成为全球票房排名中的重要一员。2. **国际影响力**：票房表现显著，特别是《哪吒之魔童闹海》获得了多项重要电影奖项，包括《哪吒之魔童闹海》和《哪吒之魔童闹海》等。这些电影不仅在国际上获得了巨大成功，也吸引了全球观众的关注。3. **电影艺术价值**：电影不仅展示了中国电影的艺术魅力，还赋予了电影艺术家强大的文化价值和情感深度。《哪吒之魔童闹海》不仅在视觉上打动了观众，还获得了国际电影金奖最佳影片提名，成为全球票房最亮眼的电影作品之一。4. **市场地位**：《哪吒之魔童闘海》获得了众多国际电影奖项，不仅在国内获得了巨大成功，还吸引了来自世界各地的观众。其在国际市场上的影响力也十分强烈。5. **国际影响力**：尽管票房表现优异，但《哪吒之魔童闹海》在国际市场的表现仍然较为突出，成为全球票房排名中的重要一员。6. **国际市场**：电影在国际市场上的知名度和影响力持续增长，尤其是在国际电影节上，许多国际电影节还在进行。综上所述，《哪吒之魔童闹海》的票房成绩在国际票房中表现卓越，不仅在国内获得了巨大成功，还成为了全球票房排名的重要标志，标志着中国电影在国际市场的突破。 +[B]: 20字的摘要：20字的巨大成就是中国电影在国际市场取得的首次突破，标志着中国电影在国际市场的突破。 +[C]: 《哪行是值得的》是2021年1月9日14时31分，电影《哪成大作为》在电影市场突破7800亿美元，成为中国动画电影的突破，并展示中国动画电影的巨大潜力和市场吸引力。 +[D]: 我是一个参数很少的AI模型🥺，知识库较少，无法直接回答您的问题，换个问题试试吧👋 ``` ---- +🙋‍Directly feeding all the above questions and model responses to GPT-5.4 Thinking for review and ranking: -🙋‍ Let's directly ask DeepSeek-R1 to review and score all the above questions and model answers: - -
-Detailed Reviews +
+Detailed Review +```text ### Scoring Criteria: -- **Accuracy**: Whether answers are correct with no obvious errors. -- **Completeness**: Whether answers cover core points of the question. -- **Logicality**: Whether answers are well-organized and follow logic. -- **Code Quality**: Whether code runs normally with clear logic. +- **Accuracy (30 points)**: Whether the facts in the response are correct, whether there are obvious factual errors or hallucinations. +- **Completeness (30 points)**: Whether the response covers the core points of the question, whether the elaboration is sufficient. +- **Logic (20 points)**: Whether the response is well-organized and internally consistent, whether there are self-contradictions or semantic confusion. +- **Code Quality (20 points)**: Whether the code can run correctly, whether the implementation logic is clear (scored only for code questions). -### Reviews: +### Per-Model Review: -1. **Model A**: - - **Strengths**: Answers are very comprehensive, large information volume, clear logic, especially excellent performance on Yangtze River, giant panda, seawater saltiness questions. Code has minor flaws but overall thinking is correct. - - **Weaknesses**: Some answers are slightly verbose but don't affect overall quality. - - **Summary**: Best overall performance with highest score. +1. **Model A (minimind-3, 0.06B)**: + - **Strengths**: Sufficient generation volume, expansion ability is already decent for this parameter count. The code question produced a structurally complete and runnable quicksort implementation, one of the best code answers in this round. The Everest question also basically got the core information right. + - **Weaknesses**: Factual errors are quite dense — universal gravitation attributed to Einstein, the Yangtze River described as "China's official name", the explanation of seawater salinity completely deviates from scientific facts (involving "light scattering", "sunlight reflection", etc.). The summary question did not follow the 20-character limit and output a large expanded passage. The giant panda answer, while getting bamboo right, had all 6 points being repeated variants of "bamboo" with extremely low information density. + - **Overall**: Has some generation and code capability, but knowledge accuracy is a hard weakness, hallucination problems are prominent, and responses frequently exhibit the phenomenon of "looking plausible at first glance but completely fabricated upon closer inspection." -2. **Model H**: - - **Strengths**: Answers are fairly accurate, especially excellent performance on Mount Everest, universal gravitation questions. Code explanation though incomplete is fairly detailed. - - **Weaknesses**: Some answers somewhat verbose but logicality is strong. - - **Summary**: Second only to Model A with stable performance. +2. **Model B (minimind-3-moe, 0.2B-A0.06B)**: + - **Strengths**: Response structure is relatively clear, sentence fluency is the best among the four models. The code question implementation is correct with example output included, and the explanation is also quite adequate. The Everest question answer is accurate. The summary question, while exceeding the character limit, at least captured the two keywords "Chinese cinema" and "international market breakthrough." + - **Weaknesses**: Factual errors are also very obvious — the Yangtze River is directly described as "Mount Everest", universal gravitation attributed to Einstein, and the giant panda's food includes "seafood, fish, birds" and other serious factual errors. The explanation of seawater salinity revolves around "osmotic pressure" going in circles without touching the core reason. + - **Overall**: The MoE architecture brings better expression fluency and structural sense, but accuracy issues are comparable to Model A. Overall, it leads in the "does it read well" dimension, but has no fundamental advantage in "is it correct." -3. **Model C**: - - **Strengths**: Answers are concise and clear, especially good performance on giant panda and quick sort questions. - - **Weaknesses**: Some answers somewhat brief lacking in-depth explanation. - - **Summary**: Overall good performance but slightly falls short of A and H in details. +3. **Model D (chatlm-mini-chinese, 0.2B)**: + - **Strengths**: Knowledge Q&A performance is the most solid — the Yangtze River description is basically correct (origin, provinces it flows through, emptying into the East Sea), universal gravitation is correctly attributed to Newton with the 1687 *Principia Mathematica* cited, the giant panda's main food of bamboo is also answered correctly, and the seawater salinity explanation starts off correctly (sodium chloride, dissolved salts). Overall readability is good, with no obvious logical breaks. + - **Weaknesses**: The code question has the condition written backwards (`len(arr) > 1: return arr`), causing the function to completely fail. The summary question directly gives up answering ("I am an AI model with very few parameters"). The Everest and seawater salinity answers both show obvious repetitive degeneration in the latter half. + - **Overall**: Knowledge reserve is the best among the four models, factual Q&A is clearly ahead, but code capability is a weakness, and generation in the latter portion tends to degenerate into repetitive loops. -4. **Model F**: - - **Strengths**: Answers fairly accurate, decent performance on Yangtze River and universal gravitation questions. Code section has certain logicality. - - **Weaknesses**: Some answers not deep enough, code has minor issues. - - **Summary**: Performs acceptably with room for improvement. - -5. **Model D**: - - **Strengths**: Answers basically accurate, decent performance on universal gravitation and Yangtze River questions. - - **Weaknesses**: Some answers too brief, code has obvious errors. - - **Summary**: Generally adequate performance needing code improvement. - -6. **Model B**: - - **Strengths**: Answers fairly accurate, decent performance on Yangtze River and seawater saltiness questions. - - **Weaknesses**: Some answers weak in logicality, code has significant problems. - - **Summary**: Average performance needing further optimization. - -7. **Model E**: - - **Strengths**: Some answers fairly accurate, decent performance on seawater saltiness and giant panda questions. - - **Weaknesses**: Answers too brief, code almost non-functional. - - **Summary**: Poor performance needing major improvement. - -8. **Model G**: - - **Strengths**: Almost no obvious strengths. - - **Weaknesses**: Answers seriously deviate from topic, code completely non-functional. - - **Summary**: Worst performance needing major improvement. - ---- +4. **Model C (baby-llama2-chinese, 0.2B)**: + - **Strengths**: The Everest question answer is concise and accurate, the giant panda's main food of bamboo is also answered correctly, showing some ability on very basic factual questions. + - **Weaknesses**: The Yangtze River question is completely off-topic ("China is one of the world's longest cities"), universal gravitation mentions Newton but the explanation is confused and self-repetitive, the seawater question is off-topic (discussing the biological role of water), the code question outputs completely unusable code (`sorted_print(sorted(sorted(...)))`), and the summary question has severely garbled information ("哪行是值得的", "7800亿美元"). + - **Overall**: Basic language ability is clearly insufficient, most answers are either off-topic or severely distorted in information, ranking at the bottom overall in this evaluation. ### Summary: -- **Model A** excels in all aspects, especially excellent in complex question answering showing high accuracy and logicality. -- **Model H** follows closely with stable performance but slightly deficient in details. -- **Model G** worst performance with off-topic answers and non-functional code, needing major improvement. +- **Model B**: Most fluent expression, correct code, best structural sense, but severe knowledge hallucinations (Yangtze = Everest, giant pandas eating seafood), large gap between "reads well" and "is correct." +- **Model D**: Highest knowledge accuracy, most stable performance on factual Q&A, but code capability is a clear weakness, generation in the latter portion tends toward repetitive degeneration. +- **Model A**: Similar style to B, code is usable, but overall stability is inferior to B, and factual error density is also on the high side. +- **Model C**: Insufficient basic capability, most answers are unusable, only occasionally answering the simplest factual questions correctly. + +```
-### Scoring Rankings +| Rank | Model | Accuracy (30 pts) | Completeness (30 pts) | Logic (20 pts) | Code Quality (20 pts) | Total (100 pts) | +|------|-------|--------------------|-----------------------|----------------|----------------------|-----------------| +| 1 | B | 11 | 23 | 16 | 18 | 68 | +| 2 | D | 25 | 19 | 15 | 3 | 62 | +| 3 | A | 10 | 21 | 13 | 17 | 61 | +| 4 | C | 8 | 6 | 5 | 2 | 21 | -| Rank | Model | Accuracy (30 points) | Completeness (30 points) | Logicality (20 points) | Code Quality (20 points) | Total (100 points) | -|----|----|-----------|-----------|-----------|------------|-----------| -| 1 | A | 28 | 29 | 19 | 20 | 96 | -| 2 | H | 27 | 28 | 18 | 20 | 93 | -| 3 | C | 26 | 27 | 18 | 18 | 89 | -| 4 | F | 25 | 26 | 17 | 18 | 86 | -| 5 | D | 24 | 25 | 17 | 16 | 82 | -| 6 | B | 23 | 24 | 16 | 15 | 78 | -| 7 | E | 22 | 23 | 15 | 14 | 74 | -| 8 | G | 10 | 12 | 10 | 10 | 42 | -### 👉 Subjective Results Summary +### 👉 Comprehensive Evaluation 2 -Personal subjective evaluation basically aligns with DeepSeek-R1, where: - -* MiniMind series ranking very intuitive, larger parameters + sufficient training data score higher. Hallucinations and errors obviously better than small models. - -* Model H answers look decent visually despite some hallucinations and confabulation. - -* Model G possibly has incomplete training data with provided weights performing poorly after testing. - -* Revisiting the timeless Scaling Law: larger parameters, more training data → stronger model performance. +From a subjective perception standpoint, I would rank `minimind-3-moe` first, `chatlm-mini-chinese` second, `minimind-3` third, and `baby-llama2-chinese` fourth. Although `B` has severe hallucinations in knowledge accuracy (giant pandas eating seafood), it wins with fluent expression, clear structure, and correct code implementation, achieving the highest overall output quality; `D` has a clearly leading knowledge reserve (Newton 1687, Yangtze River origin, etc. all correct), but the code question's reversed condition makes it completely unusable, and the summary question is directly abandoned, dragging down the score considerably; `A` and `B` are similar in style, with equally usable code, but stability and knowledge accuracy are both inferior to `B`, being a typical case of "can say something about everything but upon close inspection it's all fabricated"; `C` has obvious gaps in factuality, expansion ability, and overall readability, only occasionally answering the simplest factual questions correctly. Notably, `D` and `A` have very close total scores (62 vs 61), but their strengths and weaknesses are distributed in an almost complementary manner: `D` wins on knowledge accuracy (25 vs 10), `A` wins on code capability (17 vs 3). This actually also reflects a typical phenomenon of small-parameter models — within a limited parameter budget, "writing well" and "writing correctly" are often hard to achieve simultaneously. --- -## Ⅳ RoPE Long-text Extrapolation +## Ⅳ RoPE Length Extrapolation -MiniMind supports RoPE position encoding length extrapolation through YaRN algorithm, enabling models to handle text sequences exceeding training length. +MiniMind supports length extrapolation of RoPE positional encoding through the YaRN algorithm, enabling the model to more stably handle text sequences that exceed the training length. -For native torch models, when using `eval_llm.py` for inference, just add `--inference_rope_scaling` parameter to enable RoPE extrapolation: +When using the native torch model for inference with `eval_llm.py`, simply add the `--inference_rope_scaling` parameter to enable RoPE extrapolation: ```bash python eval_llm.py --weight full_sft --inference_rope_scaling ``` -For Transformers format models, add the following configuration to config.json to enable length extrapolation: +For models in `Transformers` format, the following configuration can be added to `config.json` to achieve length extrapolation: ```json "rope_scaling": { @@ -1582,18 +1537,52 @@ For Transformers format models, add the following configuration to config.json t } ``` -Testing on MiniMind-Small model with different lengths of "Journey to the West" vernacular fiction text to evaluate perplexity (PPL) comparison before and after RoPE scaling. -You can see that after enabling YaRN extrapolation, the model's PPL performance on long texts significantly decreases: +Below, using MiniMind as an example, we use vernacular text from *Journey to the West* of different lengths as input, comparing the perplexity (PPL) changes before and after enabling RoPE scaling. It can be seen that in long-text scenarios, the model's PPL significantly decreases after enabling YaRN extrapolation:
-## Ⅴ Objective Benchmarks +> PPL comparison of MiniMind before and after enabling YaRN at different text lengths -Performance comparisons with other small models on Chinese language leaderboards including C-Eval, CMMLU, A-CLUE, TMMLU+... +--- -Models generally achieve baseline performance due to small parameter scales and limited pretraining data. MiniMind without targeted leaderboard optimization provides fair reference results. +## Ⅴ Objective Evaluation + +Now comes the much-anticipated `benchmark` section. Here we select some micro models for cross-evaluation comparison. The test sets chosen are C-Eval, CMMLU, ARC-Easy, PIQA, OpenBookQA, HellaSwag, Social-IQa (all except the first 2 are English datasets) + + +The evaluation framework chosen is [lm-evaluation](https://github.com/EleutherAI/lm-evaluation-harness) + +```bash +# Installation +git clone https://github.com/EleutherAI/lm-evaluation-harness +cd lm-evaluation-harness && pip install -e . +``` + + + +```bash +# Start testing +# Datasets used: ceval-valid/cmmlu/arc_easy/piqa/openbookqa/hellaswag/social_iqa # View supported datasets: lm_eval ls tasks +HF_ENDPOINT=https://hf-mirror.com lm_eval --model hf --model_args pretrained="/path/to/model",dtype=auto --tasks "task" --batch_size 16 --device cpu --trust_remote_code +``` + +> Note: In these multiple-choice test sets, to avoid format instability from free-form model generation, the common practice is to directly compare the prediction probabilities of tokens corresponding to candidate options, and take the option with the highest probability to calculate accuracy against the standard answer. The candidate options are not necessarily `A`, `B`, `C`, `D`; some datasets may only have two options. Therefore, from the results perspective, the accuracy of random answering is often a strong lower bound, and models of this scale do indeed tend to hover around this level for a long time. + +The MiniMind model itself has a very small training dataset, has virtually no English knowledge capability, and has not undergone output format fine-tuning for these test sets. The results are for entertainment only: + +| models | from | params↓ | ceval↑ | cmmlu↑ | arc↑ | piqa↑ | openbookqa↑ | hellaswag↑ | siqa↑ | +|-------------------------------------------------------------------------------|---------------|---------|--------|--------|-------|-------|-------------|------------|-------| +| minimind-3 | JingyaoGong | 64M | 24.89 | 25.38 | 28.49 | 50.65 | 23.60 | 28.28 | 34.19 | +| minimind-3-moe | JingyaoGong | 198M | 25.48 | 24.32 | 27.74 | 50.71 | 26.20 | 27.43 | 34.03 | +| [Steel-LLM](https://huggingface.co/gqszhanshijin/Steel-LLM) | ZhanShiJin | 1121M | 24.89 | 25.32 | 39.69 | 65.13 | 26.00 | 35.73 | 39.15 | +| [gpt2-medium](https://huggingface.co/openai-community/gpt2-medium) | OpenAI | 360M | 23.18 | 25.00 | 43.60 | 66.38 | 30.20 | 39.38 | 39.10 | +| [TinyLlama-1.1B-Chat-V1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0) | TinyLlama | 1100M | 25.71 | 25.03 | 54.80 | 74.43 | 35.60 | 60.38 | 43.09 | +| [SmolLM2-135M-Instruct](https://huggingface.co/HuggingFaceTB/SmolLM2-135M-Instruct) | HuggingFaceTB | 135M | 24.44 | 24.71 | 58.50 | 68.17 | 32.80 | 43.15 | 39.46 | +| [Aquila-135M-Instruct](https://huggingface.co/BAAI/Aquila-135M-Instruct) | BAAI | 135M | 25.19 | 25.10 | 54.59 | 67.52 | 34.40 | 41.67 | 39.66 | + +![benchmark_radar](./images/benchmark_radar.jpg) --- @@ -1601,142 +1590,212 @@ Models generally achieve baseline performance due to small parameter scales and ## 🔧 Model Conversion -* [./scripts/convert_model.py](./scripts/convert_model.py) enables mutual conversion of `torch / transformers` models -* Unless otherwise specified, `MiniMind2` models are by default in `Transformers` format and require `t2t` conversion beforehand! +* [./scripts/convert_model.py](./scripts/convert_model.py) can be used for mutual conversion between `torch / transformers` model formats. +* Unless otherwise specified, the open-source models released on the `MiniMind` main line are usually provided in `Transformers` format; if using native `torch` weights, please first perform the `torch2transformers` conversion. +## 🖥️ API Service Interface Based on MiniMind -## 🖥️ OpenAI-API Based MiniMind Service Interface +* [./scripts/serve_openai_api.py](./scripts/serve_openai_api.py) provides a lightweight chat service compatible with the OpenAI API, making it easy to connect your own models to third-party UIs such as FastGPT, OpenWebUI, Dify, etc. +* The current interface additionally supports fields like `reasoning_content`, `tool_calls`, `open_thinking`, etc., suitable for direct use in Tool Calling / Thinking scenarios. -* [./scripts/serve_openai_api.py](./scripts/serve_openai_api.py) provides extremely simple OpenAI-API compatible chat interface, convenient for integration with third-party UIs like FastGPT, Open-WebUI, Dify, etc. - -* Download model weights from [Huggingface](https://huggingface.co/collections/jingyaogong/minimind-66caf8d999f5c7fa64f399e5), file structure: +* After downloading model weights from [HuggingFace](https://huggingface.co/collections/jingyaogong/minimind-66caf8d999f5c7fa64f399e5), the directory structure example is as follows: ``` minimind (root dir) - ├─ (e.g. MiniMind2) + ├─（例如minimind-3） | ├── config.json | ├── generation_config.json - | ├── model_minimind.py or w/o + | ├── model_minimind.py (可选，取决于模型导出形式) | ├── pytorch_model.bin or model.safetensors | ├── special_tokens_map.json | ├── tokenizer_config.json | ├── tokenizer.json ``` -* Start chat service +* Start the server ```bash - python serve_openai_api.py + cd scripts && python serve_openai_api.py ``` -* Test service interface +* Test the service interface ```bash - python chat_openai_api.py + cd scripts && python chat_api.py ``` -* API interface example, compatible with openai api format +* API request example (compatible with OpenAI API format) ```bash - curl http://ip:port/v1/chat/completions \ + curl http://localhost:8998/v1/chat/completions \ -H "Content-Type: application/json" \ -d '{ "model": "model-identifier", "messages": [ - { "role": "user", "content": "What is the highest mountain in the world?" } + { "role": "user", "content": "世界上最高的山是什么？" } ], "temperature": 0.7, - "max_tokens": 512, - "stream": true + "max_tokens": 1024, + "stream": true, + "open_thinking": true }' ``` -## 👨‍💻 More +## [SGLang](https://github.com/sgl-project/sglang) -* 🔗Fine-tuning Diffusion Language Models from MiniMind-LLM -* 🔗Model generate method explanation +SGLang is a high-performance large model inference engine that supports optimization techniques such as RadixAttention and continuous batching, capable of providing lower latency and higher throughput. ---- +> ⚠️ Requires a CUDA environment, use as needed. You can also select SGLang as the rollout / inference engine in RL training scripts to improve training throughput. + +Launch the model as an OpenAI-compatible API server: + +```bash +python -m sglang.launch_server --model-path /path/to/model --attention-backend triton --host 0.0.0.0 --port 8998 +``` ## [vllm](https://github.com/vllm-project/vllm) -vLLM is an extremely popular efficient inference framework supporting fast deployment of large models, optimizing GPU memory usage and throughput. +vLLM is a very commonly used efficient inference framework, suitable for rapid deployment of large models, achieving a good balance between VRAM utilization and throughput. -Start minimind2 in openai-serve format: +> ⚠️ Requires a CUDA environment, use as needed. + +Launch the model as an OpenAI-compatible API server: ```bash -vllm serve ./MiniMind2 --model-impl transformers --served-model-name "minimind" --port 8998 +vllm serve /path/to/model --model-impl transformers --served-model-name "minimind" --port 8998 ``` ## [llama.cpp](https://github.com/ggerganov/llama.cpp) -llama.cpp is a C++ library that can be used directly from command line, supports multi-threaded inference, and supports GPU acceleration. +llama.cpp is a lightweight and practical C++ inference framework that can be used directly from the command line, supports multi-threaded inference, and also supports some GPU acceleration options. -**Directory Structure**: It is recommended to place llama.cpp and minimind in the same parent directory +**Directory structure**: It is recommended to place `llama.cpp` and the model directory at the same level path ``` parent/ -├── minimind/ # MiniMind project directory -│ ├── MiniMind2/ # HuggingFace format MiniMind2 model (generated by convert_model.py first) +├── project/ # 你的项目目录 +│ ├── minimind模型路径/ # HuggingFace 格式模型目录 │ │ ├── config.json │ │ ├── model.safetensors │ │ └── ... -│ ├── model/ -│ ├── trainer/ │ └── ... -└── llama.cpp/ # llama.cpp project directory +└── llama.cpp/ # llama.cpp 项目目录 ├── build/ ├── convert_hf_to_gguf.py └── ... ``` -0. Follow the official `llama.cpp` installation steps +0. Refer to the `llama.cpp` official documentation to complete installation (dependencies such as `cmake`, etc.) 1. Insert at the end of the `get_vocab_base_pre` function in `convert_hf_to_gguf.py`: ```python -# Add MiniMind tokenizer support (you can use any existing one like qwen2) +# 添加 MiniMind tokenizer 支持（此处可临时复用一个兼容项，如 qwen2） if res is None: res = "qwen2" ``` -2. Convert your custom-trained minimind model: huggingface -> gguf +2. Convert the HuggingFace format minimind model to GGUF: ```bash -# Execute under llama.cpp, will generate ../minimind/MiniMind2/MiniMind2-xxx.gguf -python convert_hf_to_gguf.py ../minimind/MiniMind2 +# 在 llama.cpp 目录下执行，将在模型目录下生成对应的 gguf 文件 +python convert_hf_to_gguf.py /path/to/minimind-model ``` 3. Quantize the model (optional) ```bash -./build/bin/llama-quantize ../minimind/MiniMind2/MiniMind2.gguf ../minimind/MiniMind2/Q4-MiniMind2.gguf Q4_K_M +./build/bin/llama-quantize /path/to/model/xxxx.gguf /path/to/model/xxxx.q8.gguf Q8_0 ``` -4. Command line inference test +4. Command-line inference test ```bash -./build/bin/llama-cli -m ../minimind/MiniMind2/MiniMind2.gguf -sys "You are a helpful assistant" # system prompt must be fixed +./build/bin/llama-cli -m /path/to/model/xxxx.gguf ``` ## [ollama](https://ollama.ai) -ollama is a tool for running large models locally, supports multiple open-source LLMs, simple and easy to use. +Ollama is a commonly used tool for running large models locally, supporting various open-source LLMs, with simple usage and a low deployment threshold. -1. Load custom gguf model through ollama +1. Load a custom GGUF model via Ollama -Create `minimind.modelfile` under `MiniMind2`: +Create a new `minimind.modelfile` file in the model directory and write the following configuration template: + +
+minimind.modelfile (template) ```text -FROM ./Q4-MiniMind2.gguf +FROM /path/to/model/xxxx.gguf -SYSTEM """You are a helpful assistant""" +SYSTEM "你的名字叫MiniMind，你是一个乐于助人、知识渊博的AI助手。请用完整且友好的方式回答用户问题，当被问到名字时请回答MiniMind。" -TEMPLATE """<|im_start|>system + +TEMPLATE """{{- if .Tools }}<|im_start|>system +{{ if .System }}{{ .System }} + +{{ end }}# Tools + +You may call one or more functions to assist with the user query. + +You are provided with function signatures within XML tags: + +{{- range .Tools }} +{"type": "function", "function": {{ .Function }}} +{{- end }} + + +For each function call, return a json object with function name and arguments within XML tags: + +{"name": , "arguments": } +<|im_end|> +{{ else if .System }}<|im_start|>system {{ .System }}<|im_end|> -<|im_start|>user -{{ .Prompt }}<|im_end|> -<|im_start|>assistant -{{ .Response }}<|im_end|> -""" +{{ end }} +{{- range $i, $_ := .Messages }} +{{- $last := eq (len (slice $.Messages $i)) 1 -}} +{{- if eq .Role "user" }}<|im_start|>user +{{ .Content }}<|im_end|> +{{ else if eq .Role "assistant" }}<|im_start|>assistant + +{{ .Thinking }} + + +{{ .Content }} +{{- if .ToolCalls }} +{{- range .ToolCalls }} + +{"name": "{{ .Function.Name }}", "arguments": {{ .Function.Arguments }}} + +{{- end }} +{{- end }} +{{- if not $last }}<|im_end|> +{{ end }} +{{- else if eq .Role "tool" }}<|im_start|>user + +{{ .Content }} +<|im_end|> +{{ end }} +{{- if and (ne .Role "assistant") $last }}<|im_start|>assistant +{{ if and $.IsThinkSet $.Think -}} + +{{ else -}} + + + + +{{ end -}} +{{ end }} +{{- end }}""" + +PARAMETER repeat_penalty 1 +PARAMETER stop "<|im_start|>" +PARAMETER stop "<|im_end|>" +PARAMETER temperature 0.9 +PARAMETER top_p 0.9 +PARAMETER num_ctx 8192 ``` -2. Load and name this model as `minimind-local` +
+
+ + +2. Load and name the local model ```bash ollama create -f minimind.modelfile minimind-local @@ -1752,98 +1811,102 @@ ollama run minimind-local 📤 Push your model to Ollama Hub ```bash -# 1. Rename your local model to your_username/minimind tag +# 1. 为本地模型重命名为你的ollama-account/minimind的tag ollama cp minimind-local:latest your_username/minimind:latest -# 2. Push the model +# 2. 推送模型 ollama push your_username/minimind:latest ```

-⭐️ You can also directly use the ollama model I provided with one command: +⭐️ You can also directly use the Ollama model I provide for a quick start: ```bash -ollama run jingyaogong/minimind2 # Other options: minimind2-r1 / minimind2-small / minimind2-small-r1 ->>> What's your name? -I am a language model... +ollama run jingyaogong/minimind-3 +>>> 你叫什么名字 +我是一个语言模型... ``` -## [MNN](https://github.com/alibaba/MNN) +## [MNN](https://github.com/alibaba/MNN) -MNN is a lightweight, high-performance AI inference engine for on-device applications, supporting inference for various open-source LLM models. +MNN is an AI inference engine designed for edge devices, supporting lightweight deployment and high-performance inference for various open-source LLMs. -1. **Model Conversion** - ``` - cd MNN/transformers/llm/export - # Export the 4-bit HQQ quantized MNN model - python llmexport.py --path /path/to/MiniMind2/ --export mnn --hqq --dst_path MiniMind2-MNN - ``` +1. Model conversion +```bash +cd MNN/transformers/llm/export +# 导出 4bit HQQ 量化的 MNN 模型 +python llmexport.py --path /path/to/模型路径/ --export mnn --hqq --dst_path 模型路径-mnn +``` -2. **Test on a Mac or mobile phone** - ``` - ./llm_demo /path/to/MiniMind2-MNN/config.json prompt.txt - ``` - Or download the app to test. +2. Test on Mac or mobile devices +```bash +./llm_demo /path/to/模型路径-mnn/config.json prompt.txt +``` +Or download the APP for testing -> For more usage of the above third-party frameworks, please refer to their official documentation 😊 +> For more usage of the above third-party frameworks, please refer to their respective official documentation😊 -# 📌 Acknowledge + +## 👨‍💻 More Content + +* 🔗Fine-tuning Diffusion Language Models from MiniMind-LLM + +* 🔗Description of the Model's generate Method + +* 🔗Training Linear Attention Models from MiniMind + +# 📌 Acknowledgments > [!NOTE] -> If you find `MiniMind series` helpful, you can add a ⭐ on GitHub
-> This document is lengthy with limited knowledge. Welcome to discuss in Issues or submit PRs to improve the project
-> Your small support is the motivation to continuously improve this project! +> If the `MiniMind` series of projects has been helpful to you, feel free to star ⭐ on GitHub
+> The documentation is quite lengthy and may inevitably contain oversights; feedback via Issues or PRs to improve the project together are welcome
+> Your support and suggestions are an important driving force for the continuous iteration of this project! -## 🤝 [Contributors](https://github.com/jingyaogong/minimind/graphs/contributors) +## 🤝[Contributors](https://github.com/jingyaogong/minimind/graphs/contributors) -## 😊 Thanks +## 😊Acknowledgments -@ipfgao: -🔗 Training Steps Recording +Thanks to the following contributors for their help and sharing in training records, data processing, tutorial organization, and project breakdown: -@WangRongsheng: -🔗 Large Dataset Preprocessing +* [@ipfgao](https://github.com/ipfgao): [🔗Training Step Records](https://github.com/jingyaogong/minimind/issues/26) -@pengqianhan: -🔗 A Simple Tutorial +* [@WangRongsheng](https://github.com/WangRongsheng): [🔗Large Dataset Preprocessing](https://github.com/jingyaogong/minimind/issues/39) -@RyanSunn: -🔗 Inference Process Learning Record +* [@pengqianhan](https://github.com/pengqianhan): [🔗A Concise Tutorial](https://github.com/jingyaogong/minimind/issues/73) -@Nijikadesu: -🔗 Interactive Notebook Decomposition of Project Code +* [@RyanSunn](https://github.com/RyanSunn): [🔗Inference Process Learning Notes](https://github.com/jingyaogong/minimind/issues/75) -
- Reference Links & Thanks to the Following Excellent Papers or Projects +* [@Nijikadesu](https://github.com/Nijikadesu): [🔗Breaking Down Project Code in Interactive Notebook Format](https://github.com/jingyaogong/minimind/issues/213) + + +Acknowledgments to the following excellent papers and projects: -- Ranking does not represent any order - [https://github.com/meta-llama/llama3](https://github.com/meta-llama/llama3) - [https://github.com/karpathy/llama2.c](https://github.com/karpathy/llama2.c) - [https://github.com/DLLXW/baby-llama2-chinese](https://github.com/DLLXW/baby-llama2-chinese) -- [(DeepSeek-V2)https://arxiv.org/abs/2405.04434](https://arxiv.org/abs/2405.04434) +- [DeepSeek-V2](https://arxiv.org/abs/2405.04434) - [https://github.com/charent/ChatLM-mini-Chinese](https://github.com/charent/ChatLM-mini-Chinese) - [https://github.com/wdndev/tiny-llm-zh](https://github.com/wdndev/tiny-llm-zh) -- [(Mistral-MoE)https://arxiv.org/pdf/2401.04088](https://arxiv.org/pdf/2401.04088) +- [Mistral-MoE](https://arxiv.org/pdf/2401.04088) - [https://github.com/Tongjilibo/build_MiniLLM_from_scratch](https://github.com/Tongjilibo/build_MiniLLM_from_scratch) - [https://github.com/jzhang38/TinyLlama](https://github.com/jzhang38/TinyLlama) - [https://github.com/AI-Study-Han/Zero-Chatgpt](https://github.com/AI-Study-Han/Zero-Chatgpt) - [https://github.com/xusenlinzy/api-for-open-llm](https://github.com/xusenlinzy/api-for-open-llm) - [https://github.com/HqWu-HITCS/Awesome-Chinese-LLM](https://github.com/HqWu-HITCS/Awesome-Chinese-LLM) -
-## 🫶 Supporters +## 🫶Supporters - github contribution grid snake animation + Star poster @@ -1851,7 +1914,7 @@ MNN is a lightweight, high-performance AI inference engine for on-device applica - github contribution grid snake animation + Fork poster @@ -1861,9 +1924,9 @@ MNN is a lightweight, high-performance AI inference engine for on-device applica Star History Chart -## 🎉 Awesome Work using MiniMind +## 🎉 MiniMind Related Achievements -This model has inspired some exciting research outcomes. Thank you to all researchers for your recognition: +This model has served as a stepping stone that facilitated some gratifying research outcomes. Thanks to the researchers for their recognition: - ECG-Expert-QA: A Benchmark for Evaluating Medical Large Language Models in Heart Disease Diagnosis [[arxiv](https://arxiv.org/pdf/2502.17475)] @@ -1873,26 +1936,31 @@ This model has inspired some exciting research outcomes. Thank you to all resear - On the Generalization Ability of Next-Token-Prediction Pretraining [[ICML 2025](https://openreview.net/forum?id=hLGJ1qZPdu)] -- 《从零开始写大模型：从神经网络到Transformer》(Chinese book: Building LLMs from Scratch) by Wang Shuang, Mou Chen, Wang Haoyi - Tsinghua University Press +- 《从零开始写大模型：从神经网络到Transformer》王双、牟晨、王昊怡 编著 - 清华大学出版社 - FedBRB: A Solution to the Small-to-Large Scenario in Device-Heterogeneity Federated Learning [[TMC 2025](https://ieeexplore.ieee.org/abstract/document/11168259)] -- Continuously... +- SKETCH: Semantic Key-Point Conditioning for Long-Horizon Vessel Trajectory Prediction [[arxiv](https://arxiv.org/pdf/2601.18537)] + +- A Built-in Crypto Expert for Artificial Intelligence: How Far is the Horizon? [[IACR ePrint 2026](https://eprint.iacr.org/2026/411.pdf)] + +- In progress... # 🎓 Citation -If you find MiniMind helpful in your research or work, please cite: +If `MiniMind` has been helpful to your research or work, feel free to cite: ```bibtex @misc{minimind, - title={MiniMind: Train a Tiny LLM from scratch}, - author={Jingyao Gong}, - year={2024}, - howpublished={https://github.com/jingyaogong/minimind} + title = {MiniMind: Train a Tiny LLM from Scratch}, + author = {Jingyao Gong}, + year = {2024}, + url = {https://github.com/jingyaogong/minimind}, + note = {GitHub repository, accessed 2026} } ``` -# License +# ⚖️ License -This repository is licensed under the [Apache-2.0 License](LICENSE). +This project is open-sourced under the [Apache License 2.0](LICENSE). \ No newline at end of file diff --git a/dataset/lm_dataset.py b/dataset/lm_dataset.py index 0d7deb1..bbc4197 100644 --- a/dataset/lm_dataset.py +++ b/dataset/lm_dataset.py @@ -1,11 +1,15 @@ from torch.utils.data import Dataset import torch +import json import os import random -from datasets import load_dataset +from datasets import load_dataset, Features, Sequence, Value os.environ["TOKENIZERS_PARALLELISM"] = "false" def pre_processing_chat(conversations, add_system_ratio=0.2): + # tool use 数据完整保留不做处理 + if any(conv.get('tools') for conv in conversations): return conversations + SYSTEM_PROMPTS = [ "你是一个知识丰富的AI，尽力为用户提供准确的信息。", "你是minimind，一个小巧但有用的语言模型。", @@ -18,12 +22,14 @@ def pre_processing_chat(conversations, add_system_ratio=0.2): "You are a knowledgeable AI. Try your best to provide accurate information.", "You are minimind, a small but useful language model." ] - if conversations and conversations[0].get('role') != 'system': + # 概率性添加system + if conversations[0].get('role') != 'system': if random.random() < add_system_ratio: return [{'role': 'system', 'content': random.choice(SYSTEM_PROMPTS)}] + conversations return conversations -def post_processing_chat(prompt_content, empty_think_ratio=0.05): +def post_processing_chat(prompt_content, empty_think_ratio=0.2): + # 以80%概率移除空思考标签 if '\n\n\n\n' in prompt_content and random.random() > empty_think_ratio: prompt_content = prompt_content.replace('\n\n\n\n', '') return prompt_content @@ -54,7 +60,8 @@ class SFTDataset(Dataset): super().__init__() self.tokenizer = tokenizer self.max_length = max_length - self.samples = load_dataset('json', data_files=jsonl_path, split='train') + features = Features({'conversations': [{'role': Value('string'), 'content': Value('string'), 'reasoning_content': Value('string'), 'tools': Value('string'), 'tool_calls': Value('string')}]}) + self.samples = load_dataset('json', data_files=jsonl_path, split='train', features=features) self.bos_id = tokenizer(f'{tokenizer.bos_token}assistant\n', add_special_tokens=False).input_ids self.eos_id = tokenizer(f'{tokenizer.eos_token}\n', add_special_tokens=False).input_ids @@ -62,8 +69,15 @@ class SFTDataset(Dataset): return len(self.samples) def create_chat_prompt(self, conversations): - messages = conversations.copy() - tools = conversations[0]["functions"] if (conversations and conversations[0]["role"] == "system" and conversations[0].get("functions")) else None + messages = [] + tools = None + for message in conversations: + message = dict(message) + if message.get("role") == "system" and message.get("tools"): + tools = json.loads(message["tools"]) if isinstance(message["tools"], str) else message["tools"] + if message.get("tool_calls") and isinstance(message["tool_calls"], str): + message["tool_calls"] = json.loads(message["tool_calls"]) + messages.append(message) return self.tokenizer.apply_chat_template( messages, tokenize=False, @@ -179,10 +193,11 @@ class DPODataset(Dataset): class RLAIFDataset(Dataset): - def __init__(self, jsonl_path, tokenizer, max_length=1024): + def __init__(self, jsonl_path, tokenizer, max_length=1024, thinking_ratio=0.5): super().__init__() self.tokenizer = tokenizer self.max_length = max_length + self.thinking_ratio = thinking_ratio # 按概率开启 thinking self.samples = load_dataset('json', data_files=jsonl_path, split='train') self.bos_id = tokenizer(f'{tokenizer.bos_token}assistant', add_special_tokens=False).input_ids self.eos_id = tokenizer(f'{tokenizer.eos_token}', add_special_tokens=False).input_ids @@ -191,28 +206,51 @@ class RLAIFDataset(Dataset): return len(self.samples) def create_chat_prompt(self, conversations): - messages = [] - answer = '' - for i, turn in enumerate(conversations): - role = 'user' if i % 2 == 0 else 'assistant' - messages.append({"role": role, "content": turn['content']}) - answer = turn['content'] - prompt = self.tokenizer.apply_chat_template( - messages[:-1], + conversations = pre_processing_chat(conversations) + use_thinking = random.random() < self.thinking_ratio + return self.tokenizer.apply_chat_template( + conversations[:-1], tokenize=False, - add_generation_prompt=True # 这里需要True + open_thinking=use_thinking, + add_generation_prompt=True ) - prompt = post_processing_chat(prompt) - return prompt, answer - def __getitem__(self, index): sample = self.samples[index] - prompt, answer = self.create_chat_prompt(sample['conversations']) + prompt = self.create_chat_prompt(sample['conversations']) return { 'prompt': prompt, - 'answer': answer + 'answer': "" } +class AgentRLDataset(Dataset): + def __init__(self, jsonl_path, tokenizer, max_length=1024): + super().__init__() + self.tokenizer = tokenizer + self.max_length = max_length + self.samples = [] + with open(jsonl_path, 'r', encoding='utf-8') as f: + for line in f: + self.samples.append(json.loads(line.strip())) + + def __len__(self): + return len(self.samples) + + def parse_conversations(self, conversations): + messages = [] + tools = None + for message in conversations: + message = dict(message) + if message.get("role") == "system" and message.get("tools"): + tools = json.loads(message["tools"]) if isinstance(message["tools"], str) else message["tools"] + messages.append(message) + return messages[:-1], tools + + def __getitem__(self, index): + sample = self.samples[index] + messages, tools = self.parse_conversations(sample['conversations']) + return {'messages': messages, 'tools': tools, 'gt': sample['gt']} + + if __name__ == "__main__": pass \ No newline at end of file diff --git a/eval_llm.py b/eval_llm.py index 3c187a4..acd48bf 100755 --- a/eval_llm.py +++ b/eval_llm.py @@ -23,7 +23,7 @@ def init_model(args): model.load_state_dict(torch.load(ckp, map_location=args.device), strict=True) if args.lora_weight != 'None': apply_lora(model) - load_lora(model, f'./{args.save_dir}/lora/{args.lora_weight}_{args.hidden_size}.pth') + load_lora(model, f'./{args.save_dir}/{args.lora_weight}_{args.hidden_size}.pth') else: model = AutoModelForCausalLM.from_pretrained(args.load_from, trust_remote_code=True) get_model_params(model, model.config) @@ -35,13 +35,14 @@ def main(): parser.add_argument('--save_dir', default='out', type=str, help="模型权重目录") parser.add_argument('--weight', default='full_sft', type=str, help="权重名称前缀（pretrain, full_sft, rlhf, reason, ppo_actor, grpo, spo）") parser.add_argument('--lora_weight', default='None', type=str, help="LoRA权重名称（None表示不使用，可选：lora_identity, lora_medical）") - parser.add_argument('--hidden_size', default=512, type=int, help="隐藏层维度（512=Small-26M, 640=MoE-145M, 768=Base-104M）") - parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量（Small/MoE=8, Base=16）") + parser.add_argument('--hidden_size', default=768, type=int, help="隐藏层维度") + parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") parser.add_argument('--inference_rope_scaling', default=False, action='store_true', help="启用RoPE位置编码外推（4倍，仅解决位置编码问题）") parser.add_argument('--max_new_tokens', default=8192, type=int, help="最大生成长度（注意：并非模型实际长文本能力）") parser.add_argument('--temperature', default=0.85, type=float, help="生成温度，控制随机性（0-1，越大越随机）") - parser.add_argument('--top_p', default=0.85, type=float, help="nucleus采样阈值（0-1）") + parser.add_argument('--top_p', default=0.95, type=float, help="nucleus采样阈值（0-1）") + parser.add_argument('--open_thinking', default=0, type=int, help="是否开启自适应思考（0=否，1=是）") parser.add_argument('--historys', default=0, type=int, help="携带历史对话轮数（需为偶数，0表示不携带历史）") parser.add_argument('--show_speed', default=1, type=int, help="显示decode速度（tokens/s）") parser.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu', type=str, help="运行设备") @@ -65,23 +66,25 @@ def main(): prompt_iter = prompts if input_mode == 0 else iter(lambda: input('💬: '), '') for prompt in prompt_iter: - setup_seed(2026) # or setup_seed(random.randint(0, 2048)) + setup_seed(random.randint(0, 31415926)) + setup_seed(42) if input_mode == 0: print(f'💬: {prompt}') conversation = conversation[-args.historys:] if args.historys else [] conversation.append({"role": "user", "content": prompt}) - - templates = {"conversation": conversation, "tokenize": False, "add_generation_prompt": True} - if args.weight == 'reason': templates["enable_thinking"] = True # 仅Reason模型使用 - inputs = tokenizer.apply_chat_template(**templates) if args.weight != 'pretrain' else (tokenizer.bos_token + prompt) + if 'pretrain' in args.weight: + inputs = tokenizer.bos_token + prompt + else: + inputs = tokenizer.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True, open_thinking=bool(args.open_thinking)) + inputs = tokenizer(inputs, return_tensors="pt", truncation=True).to(args.device) - print('🤖: ', end='') + print('🧠: ', end='') st = time.time() generated_ids = model.generate( inputs=inputs["input_ids"], attention_mask=inputs["attention_mask"], max_new_tokens=args.max_new_tokens, do_sample=True, streamer=streamer, pad_token_id=tokenizer.pad_token_id, eos_token_id=tokenizer.eos_token_id, - top_p=args.top_p, temperature=args.temperature, repetition_penalty=1.0 + top_p=args.top_p, temperature=args.temperature, repetition_penalty=1 ) response = tokenizer.decode(generated_ids[0][len(inputs["input_ids"][0]):], skip_special_tokens=True) conversation.append({"role": "assistant", "content": response}) diff --git a/images/1-wiki.png b/images/1-wiki.png deleted file mode 100644 index 5eba0fc..0000000 Binary files a/images/1-wiki.png and /dev/null differ diff --git a/images/2-wiki.png b/images/2-wiki.png deleted file mode 100644 index 35a1d53..0000000 Binary files a/images/2-wiki.png and /dev/null differ diff --git a/images/3-wiki.png b/images/3-wiki.png deleted file mode 100644 index 9570bbf..0000000 Binary files a/images/3-wiki.png and /dev/null differ diff --git a/images/4-wiki.png b/images/4-wiki.png deleted file mode 100644 index 931e240..0000000 Binary files a/images/4-wiki.png and /dev/null differ diff --git a/images/5-wiki.png b/images/5-wiki.png deleted file mode 100644 index 78ca9af..0000000 Binary files a/images/5-wiki.png and /dev/null differ diff --git a/images/LLM-structure-moe.jpg b/images/LLM-structure-moe.jpg new file mode 100644 index 0000000..ac45ff1 Binary files /dev/null and b/images/LLM-structure-moe.jpg differ diff --git a/images/LLM-structure-moe.png b/images/LLM-structure-moe.png deleted file mode 100644 index 4588477..0000000 Binary files a/images/LLM-structure-moe.png and /dev/null differ diff --git a/images/LLM-structure.jpg b/images/LLM-structure.jpg new file mode 100644 index 0000000..2fe5a6b Binary files /dev/null and b/images/LLM-structure.jpg differ diff --git a/images/LLM-structure.png b/images/LLM-structure.png deleted file mode 100755 index bbd93dd..0000000 Binary files a/images/LLM-structure.png and /dev/null differ diff --git a/images/agent_rl_loss.jpg b/images/agent_rl_loss.jpg new file mode 100644 index 0000000..2299c40 Binary files /dev/null and b/images/agent_rl_loss.jpg differ diff --git a/images/agent_webui.jpg b/images/agent_webui.jpg new file mode 100644 index 0000000..7c955a5 Binary files /dev/null and b/images/agent_webui.jpg differ diff --git a/images/benchmark_radar.jpg b/images/benchmark_radar.jpg new file mode 100644 index 0000000..a8b231b Binary files /dev/null and b/images/benchmark_radar.jpg differ diff --git a/images/compare_radar.png b/images/compare_radar.png deleted file mode 100644 index c1c9339..0000000 Binary files a/images/compare_radar.png and /dev/null differ diff --git a/images/dataset.jpg b/images/dataset.jpg index 7dfc366..8329bcc 100644 Binary files a/images/dataset.jpg and b/images/dataset.jpg differ diff --git a/images/grpo_loss.jpg b/images/grpo_loss.jpg new file mode 100644 index 0000000..618351d Binary files /dev/null and b/images/grpo_loss.jpg differ diff --git a/images/minimind-3.gif b/images/minimind-3.gif new file mode 100644 index 0000000..131d762 Binary files /dev/null and b/images/minimind-3.gif differ diff --git a/images/minimind2.gif b/images/minimind2.gif deleted file mode 100644 index 43c9cd1..0000000 Binary files a/images/minimind2.gif and /dev/null differ diff --git a/images/ppo_loss.jpg b/images/ppo_loss.jpg new file mode 100644 index 0000000..f8fe5a6 Binary files /dev/null and b/images/ppo_loss.jpg differ diff --git a/images/pre_512_loss.png b/images/pre_512_loss.png deleted file mode 100644 index 3da0be5..0000000 Binary files a/images/pre_512_loss.png and /dev/null differ diff --git a/images/pre_768_loss.png b/images/pre_768_loss.png deleted file mode 100644 index e00b23c..0000000 Binary files a/images/pre_768_loss.png and /dev/null differ diff --git a/images/pretrain_loss.jpg b/images/pretrain_loss.jpg new file mode 100644 index 0000000..00a3f96 Binary files /dev/null and b/images/pretrain_loss.jpg differ diff --git a/images/rl-structure.jpg b/images/rl-structure.jpg new file mode 100644 index 0000000..cdda7fa Binary files /dev/null and b/images/rl-structure.jpg differ diff --git a/images/sft_512_loss.png b/images/sft_512_loss.png deleted file mode 100644 index 40b86bc..0000000 Binary files a/images/sft_512_loss.png and /dev/null differ diff --git a/images/sft_768_loss.png b/images/sft_768_loss.png deleted file mode 100644 index 5ea6c97..0000000 Binary files a/images/sft_768_loss.png and /dev/null differ diff --git a/images/sft_loss.jpg b/images/sft_loss.jpg new file mode 100644 index 0000000..173b7cb Binary files /dev/null and b/images/sft_loss.jpg differ diff --git a/images/train_grpo_512.png b/images/train_grpo_512.png deleted file mode 100644 index 008a598..0000000 Binary files a/images/train_grpo_512.png and /dev/null differ diff --git a/images/train_grpo_768.png b/images/train_grpo_768.png deleted file mode 100644 index 339f393..0000000 Binary files a/images/train_grpo_768.png and /dev/null differ diff --git a/images/train_ppo_512.png b/images/train_ppo_512.png deleted file mode 100644 index 3be1cca..0000000 Binary files a/images/train_ppo_512.png and /dev/null differ diff --git a/images/train_ppo_768.png b/images/train_ppo_768.png deleted file mode 100644 index 399dfdf..0000000 Binary files a/images/train_ppo_768.png and /dev/null differ diff --git a/images/train_spo_768.png b/images/train_spo_768.png deleted file mode 100644 index 0ac98e8..0000000 Binary files a/images/train_spo_768.png and /dev/null differ diff --git a/images/and_huggingface.png b/images/with_huggingface.png similarity index 100% rename from images/and_huggingface.png rename to images/with_huggingface.png diff --git a/images/and_modelscope.png b/images/with_modelscope.png similarity index 100% rename from images/and_modelscope.png rename to images/with_modelscope.png diff --git a/model/model_lora.py b/model/model_lora.py index e42e8d4..45b4961 100644 --- a/model/model_lora.py +++ b/model/model_lora.py @@ -18,7 +18,7 @@ class LoRA(nn.Module): return self.B(self.A(x)) -def apply_lora(model, rank=8): +def apply_lora(model, rank=16): for name, module in model.named_modules(): if isinstance(module, nn.Linear) and module.weight.shape[0] == module.weight.shape[1]: lora = LoRA(module.weight.shape[0], module.weight.shape[1], rank=rank).to(model.device) @@ -48,6 +48,18 @@ def save_lora(model, path): for name, module in raw_model.named_modules(): if hasattr(module, 'lora'): clean_name = name[7:] if name.startswith("module.") else name - lora_state = {f'{clean_name}.lora.{k}': v for k, v in module.lora.state_dict().items()} + lora_state = {f'{clean_name}.lora.{k}': v.cpu().half() for k, v in module.lora.state_dict().items()} state_dict.update(lora_state) torch.save(state_dict, path) + + +def merge_lora(model, lora_path, save_path): + load_lora(model, lora_path) + raw_model = getattr(model, '_orig_mod', model) + state_dict = {k: v.cpu().half() for k, v in raw_model.state_dict().items() if '.lora.' not in k} + for name, module in raw_model.named_modules(): + if isinstance(module, nn.Linear) and '.lora.' not in name: + state_dict[f'{name}.weight'] = module.weight.data.clone().cpu().half() + if hasattr(module, 'lora'): + state_dict[f'{name}.weight'] += (module.lora.B.weight.data @ module.lora.A.weight.data).cpu().half() + torch.save(state_dict, save_path) diff --git a/model/model_minimind.py b/model/model_minimind.py index b3910a8..70ee32b 100755 --- a/model/model_minimind.py +++ b/model/model_minimind.py @@ -1,60 +1,33 @@ -# 📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘 -# MiniMind Config -# 📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘 - -from transformers import PretrainedConfig - +import math, torch, torch.nn.functional as F +from torch import nn +from transformers.activations import ACT2FN +from transformers import PreTrainedModel, GenerationMixin, PretrainedConfig +from transformers.modeling_outputs import MoeCausalLMOutputWithPast +# 🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏 +# MiniMind Config +# 🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏 class MiniMindConfig(PretrainedConfig): model_type = "minimind" - - def __init__( - self, - dropout: float = 0.0, - bos_token_id: int = 1, - eos_token_id: int = 2, - hidden_act: str = 'silu', - hidden_size: int = 512, - intermediate_size: int = None, - max_position_embeddings: int = 32768, - num_attention_heads: int = 8, - num_hidden_layers: int = 8, - num_key_value_heads: int = 2, - vocab_size: int = 6400, - rms_norm_eps: float = 1e-05, - rope_theta: int = 1000000.0, - inference_rope_scaling: bool = False, - flash_attn: bool = True, - #################################################### - # Here are the specific configurations of MOE - # When use_moe is false, the following is invalid - #################################################### - use_moe: bool = False, - num_experts_per_tok: int = 2, - n_routed_experts: int = 4, - n_shared_experts: int = 1, - scoring_func: str = 'softmax', - aux_loss_alpha: float = 0.01, - seq_aux: bool = True, - norm_topk_prob: bool = True, - **kwargs - ): + def __init__(self, hidden_size=768, num_hidden_layers=8, use_moe=False, **kwargs): super().__init__(**kwargs) - self.dropout = dropout - self.bos_token_id = bos_token_id - self.eos_token_id = eos_token_id - self.hidden_act = hidden_act self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.max_position_embeddings = max_position_embeddings - self.num_attention_heads = num_attention_heads self.num_hidden_layers = num_hidden_layers - self.num_key_value_heads = num_key_value_heads - self.vocab_size = vocab_size - self.rms_norm_eps = rms_norm_eps - self.rope_theta = rope_theta - self.inference_rope_scaling = inference_rope_scaling - # 外推长度 = factor * original_max_position_embeddings = 32768 + self.use_moe = use_moe + self.dropout = kwargs.get("dropout", 0.0) + self.vocab_size = kwargs.get("vocab_size", 6400) + self.bos_token_id = kwargs.get("bos_token_id", 1) + self.eos_token_id = kwargs.get("eos_token_id", 2) + self.flash_attn = kwargs.get("flash_attn", True) + self.num_attention_heads = kwargs.get("num_attention_heads", 8) + self.num_key_value_heads = kwargs.get("num_key_value_heads", 4) + self.head_dim = kwargs.get("head_dim", self.hidden_size // self.num_attention_heads) + self.hidden_act = kwargs.get("hidden_act", 'silu') + self.intermediate_size = kwargs.get("intermediate_size", math.ceil(hidden_size * math.pi / 64) * 64) + self.max_position_embeddings = kwargs.get("max_position_embeddings", 32768) + self.rms_norm_eps = kwargs.get("rms_norm_eps", 1e-6) + self.rope_theta = kwargs.get("rope_theta", 1e6) + self.inference_rope_scaling = kwargs.get("inference_rope_scaling", False) self.rope_scaling = { "beta_fast": 32, "beta_slow": 1, @@ -63,301 +36,147 @@ class MiniMindConfig(PretrainedConfig): "attention_factor": 1.0, "type": "yarn" } if self.inference_rope_scaling else None - self.flash_attn = flash_attn - #################################################### - # Here are the specific configurations of MOE - # When use_moe is false, the following is invalid - #################################################### - self.use_moe = use_moe - self.num_experts_per_tok = num_experts_per_tok # 每个token选择的专家数量 - self.n_routed_experts = n_routed_experts # 总的专家数量 - self.n_shared_experts = n_shared_experts # 共享专家 - self.scoring_func = scoring_func # 评分函数，默认为'softmax' - self.aux_loss_alpha = aux_loss_alpha # 辅助损失的alpha参数 - self.seq_aux = seq_aux # 是否在序列级别上计算辅助损失 - self.norm_topk_prob = norm_topk_prob # 是否标准化top-k概率 - - -# 📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘 -# MiniMind Model -# 📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘📘 - -import math -import torch -import torch.nn.init as init -import torch.nn.functional as F -from torch import nn -from transformers.activations import ACT2FN -from typing import Optional, Tuple, List, Union -from transformers import PreTrainedModel, GenerationMixin, PretrainedConfig -from transformers.modeling_outputs import CausalLMOutputWithPast - + ### MoE specific configs (ignored if use_moe = False) + self.num_experts = kwargs.get("num_experts", 4) + self.num_experts_per_tok = kwargs.get("num_experts_per_tok", 1) + self.moe_intermediate_size = kwargs.get("moe_intermediate_size", self.intermediate_size) + self.norm_topk_prob = kwargs.get("norm_topk_prob", True) + self.router_aux_loss_coef = kwargs.get("router_aux_loss_coef", 5e-4) +# 🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏 +# MiniMind Model +# 🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏🌎🌍🌏 class RMSNorm(torch.nn.Module): def __init__(self, dim: int, eps: float = 1e-5): super().__init__() self.eps = eps self.weight = nn.Parameter(torch.ones(dim)) - def _norm(self, x): + def norm(self, x): return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) def forward(self, x): - return self.weight * self._norm(x.float()).type_as(x) + return (self.weight * self.norm(x.float())).type_as(x) - -def precompute_freqs_cis(dim: int, end: int = int(32 * 1024), rope_base: float = 1e6, - rope_scaling: Optional[dict] = None): +def precompute_freqs_cis(dim: int, end: int = int(32 * 1024), rope_base: float = 1e6, rope_scaling: dict = None): freqs, attn_factor = 1.0 / (rope_base ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim)), 1.0 - if rope_scaling is not None: + if rope_scaling is not None: # YaRN: f'(i) = f(i)((1-γ) + γ/s), where γ∈[0,1] is linear ramp orig_max, factor, beta_fast, beta_slow, attn_factor = ( rope_scaling.get("original_max_position_embeddings", 2048), rope_scaling.get("factor", 16), rope_scaling.get("beta_fast", 32.0), rope_scaling.get("beta_slow", 1.0), rope_scaling.get("attention_factor", 1.0) ) if end / orig_max > 1.0: - # YaRN: f'(i) = f(i)((1-γ) + γ/s), where γ∈[0,1] is linear ramp inv_dim = lambda b: (dim * math.log(orig_max / (b * 2 * math.pi))) / (2 * math.log(rope_base)) low, high = max(math.floor(inv_dim(beta_fast)), 0), min(math.ceil(inv_dim(beta_slow)), dim // 2 - 1) ramp = torch.clamp((torch.arange(dim // 2, device=freqs.device).float() - low) / max(high - low, 0.001), 0, 1) freqs = freqs * (1 - ramp + ramp / factor) - t = torch.arange(end, device=freqs.device) freqs = torch.outer(t, freqs).float() freqs_cos = torch.cat([torch.cos(freqs), torch.cos(freqs)], dim=-1) * attn_factor freqs_sin = torch.cat([torch.sin(freqs), torch.sin(freqs)], dim=-1) * attn_factor return freqs_cos, freqs_sin - -def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): - def rotate_half(x): - return torch.cat((-x[..., x.shape[-1] // 2:], x[..., : x.shape[-1] // 2]), dim=-1) - - q_embed = (q * cos.unsqueeze(unsqueeze_dim)) + (rotate_half(q) * sin.unsqueeze(unsqueeze_dim)) - k_embed = (k * cos.unsqueeze(unsqueeze_dim)) + (rotate_half(k) * sin.unsqueeze(unsqueeze_dim)) +def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1): + def rotate_half(x): return torch.cat((-x[..., x.shape[-1] // 2:], x[..., : x.shape[-1] // 2]), dim=-1) + q_embed = ((q * cos.unsqueeze(unsqueeze_dim)) + (rotate_half(q) * sin.unsqueeze(unsqueeze_dim))).to(q.dtype) + k_embed = ((k * cos.unsqueeze(unsqueeze_dim)) + (rotate_half(k) * sin.unsqueeze(unsqueeze_dim))).to(k.dtype) return q_embed, k_embed - def repeat_kv(x: torch.Tensor, n_rep: int) -> torch.Tensor: - """torch.repeat_interleave(x, dim=2, repeats=n_rep)""" bs, slen, num_key_value_heads, head_dim = x.shape - if n_rep == 1: - return x - return ( - x[:, :, :, None, :].expand(bs, slen, num_key_value_heads, n_rep, head_dim).reshape(bs, slen, num_key_value_heads * n_rep, head_dim) - ) - + if n_rep == 1: return x + return (x[:, :, :, None, :].expand(bs, slen, num_key_value_heads, n_rep, head_dim).reshape(bs, slen, num_key_value_heads * n_rep, head_dim)) class Attention(nn.Module): - def __init__(self, args: MiniMindConfig): + def __init__(self, config: MiniMindConfig): super().__init__() - self.num_key_value_heads = args.num_attention_heads if args.num_key_value_heads is None else args.num_key_value_heads - assert args.num_attention_heads % self.num_key_value_heads == 0 - self.n_local_heads = args.num_attention_heads + self.num_key_value_heads = config.num_attention_heads if config.num_key_value_heads is None else config.num_key_value_heads + self.n_local_heads = config.num_attention_heads self.n_local_kv_heads = self.num_key_value_heads self.n_rep = self.n_local_heads // self.n_local_kv_heads - self.head_dim = args.hidden_size // args.num_attention_heads - self.q_proj = nn.Linear(args.hidden_size, args.num_attention_heads * self.head_dim, bias=False) - self.k_proj = nn.Linear(args.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) - self.v_proj = nn.Linear(args.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) - self.o_proj = nn.Linear(args.num_attention_heads * self.head_dim, args.hidden_size, bias=False) - self.attn_dropout = nn.Dropout(args.dropout) - self.resid_dropout = nn.Dropout(args.dropout) - self.dropout = args.dropout - self.flash = hasattr(torch.nn.functional, 'scaled_dot_product_attention') and args.flash_attn - # print("WARNING: using slow attention. Flash Attention requires PyTorch >= 2.0") + self.head_dim = config.head_dim + self.q_proj = nn.Linear(config.hidden_size, config.num_attention_heads * self.head_dim, bias=False) + self.k_proj = nn.Linear(config.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.v_proj = nn.Linear(config.hidden_size, self.num_key_value_heads * self.head_dim, bias=False) + self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=False) + self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.attn_dropout = nn.Dropout(config.dropout) + self.resid_dropout = nn.Dropout(config.dropout) + self.dropout = config.dropout + self.flash = hasattr(torch.nn.functional, 'scaled_dot_product_attention') and config.flash_attn - def forward(self, - x: torch.Tensor, - position_embeddings: Tuple[torch.Tensor, torch.Tensor], # 修改为接收cos和sin - past_key_value: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - use_cache=False, - attention_mask: Optional[torch.Tensor] = None): + def forward(self, x, position_embeddings, past_key_value=None, use_cache=False, attention_mask=None): bsz, seq_len, _ = x.shape xq, xk, xv = self.q_proj(x), self.k_proj(x), self.v_proj(x) xq = xq.view(bsz, seq_len, self.n_local_heads, self.head_dim) xk = xk.view(bsz, seq_len, self.n_local_kv_heads, self.head_dim) xv = xv.view(bsz, seq_len, self.n_local_kv_heads, self.head_dim) - + xq, xk = self.q_norm(xq), self.k_norm(xk) cos, sin = position_embeddings xq, xk = apply_rotary_pos_emb(xq, xk, cos, sin) - - # kv_cache实现 if past_key_value is not None: xk = torch.cat([past_key_value[0], xk], dim=1) xv = torch.cat([past_key_value[1], xv], dim=1) past_kv = (xk, xv) if use_cache else None - - xq, xk, xv = ( - xq.transpose(1, 2), - repeat_kv(xk, self.n_rep).transpose(1, 2), - repeat_kv(xv, self.n_rep).transpose(1, 2) - ) - + xq, xk, xv = (xq.transpose(1, 2), repeat_kv(xk, self.n_rep).transpose(1, 2), repeat_kv(xv, self.n_rep).transpose(1, 2)) if self.flash and (seq_len > 1) and (past_key_value is None) and (attention_mask is None or torch.all(attention_mask == 1)): output = F.scaled_dot_product_attention(xq, xk, xv, dropout_p=self.dropout if self.training else 0.0, is_causal=True) else: scores = (xq @ xk.transpose(-2, -1)) / math.sqrt(self.head_dim) - scores[:, :, :, -seq_len:] += torch.triu(torch.full((seq_len, seq_len), float("-inf"), device=scores.device), diagonal=1) - - if attention_mask is not None: - extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) - extended_attention_mask = (1.0 - extended_attention_mask) * -1e9 - scores = scores + extended_attention_mask - - scores = F.softmax(scores.float(), dim=-1).type_as(xq) - scores = self.attn_dropout(scores) - output = scores @ xv - + scores[:, :, :, -seq_len:] += torch.full((seq_len, seq_len), float("-inf"), device=scores.device).triu(1) + if attention_mask is not None: scores += (1.0 - attention_mask.unsqueeze(1).unsqueeze(2)) * -1e9 + output = self.attn_dropout(F.softmax(scores.float(), dim=-1).type_as(xq)) @ xv output = output.transpose(1, 2).reshape(bsz, seq_len, -1) output = self.resid_dropout(self.o_proj(output)) return output, past_kv - class FeedForward(nn.Module): - def __init__(self, config: MiniMindConfig): + def __init__(self, config: MiniMindConfig, intermediate_size: int = None): super().__init__() - if config.intermediate_size is None: - intermediate_size = int(config.hidden_size * 8 / 3) - config.intermediate_size = 64 * ((intermediate_size + 64 - 1) // 64) - self.gate_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False) - self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False) - self.up_proj = nn.Linear(config.hidden_size, config.intermediate_size, bias=False) - self.dropout = nn.Dropout(config.dropout) + intermediate_size = intermediate_size or config.intermediate_size + self.gate_proj = nn.Linear(config.hidden_size, intermediate_size, bias=False) + self.down_proj = nn.Linear(intermediate_size, config.hidden_size, bias=False) + self.up_proj = nn.Linear(config.hidden_size, intermediate_size, bias=False) self.act_fn = ACT2FN[config.hidden_act] def forward(self, x): - return self.dropout(self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))) - - -class MoEGate(nn.Module): - def __init__(self, config: MiniMindConfig): - super().__init__() - self.config = config - self.top_k = config.num_experts_per_tok - self.n_routed_experts = config.n_routed_experts - - self.scoring_func = config.scoring_func - self.alpha = config.aux_loss_alpha - self.seq_aux = config.seq_aux - - self.norm_topk_prob = config.norm_topk_prob - self.gating_dim = config.hidden_size - self.weight = nn.Parameter(torch.empty((self.n_routed_experts, self.gating_dim))) - self.reset_parameters() - - def reset_parameters(self) -> None: - init.kaiming_uniform_(self.weight, a=math.sqrt(5)) - - def forward(self, hidden_states): - bsz, seq_len, h = hidden_states.shape - hidden_states = hidden_states.view(-1, h) - logits = F.linear(hidden_states, self.weight, None) - if self.scoring_func == 'softmax': - scores = logits.softmax(dim=-1) - else: - raise NotImplementedError(f'insupportable scoring function for MoE gating: {self.scoring_func}') - - topk_weight, topk_idx = torch.topk(scores, k=self.top_k, dim=-1, sorted=False) - - if self.top_k > 1 and self.norm_topk_prob: - denominator = topk_weight.sum(dim=-1, keepdim=True) + 1e-20 - topk_weight = topk_weight / denominator - - if self.training and self.alpha > 0.0: - scores_for_aux = scores - aux_topk = self.top_k - topk_idx_for_aux_loss = topk_idx.view(bsz, -1) - if self.seq_aux: - scores_for_seq_aux = scores_for_aux.view(bsz, seq_len, -1) - ce = torch.zeros(bsz, self.n_routed_experts, device=hidden_states.device) - ce.scatter_add_(1, topk_idx_for_aux_loss, - torch.ones(bsz, seq_len * aux_topk, device=hidden_states.device)).div_( - seq_len * aux_topk / self.n_routed_experts) - aux_loss = (ce * scores_for_seq_aux.mean(dim=1)).sum(dim=1).mean() * self.alpha - else: - mask_ce = F.one_hot(topk_idx_for_aux_loss.view(-1), num_classes=self.n_routed_experts) - ce = mask_ce.float().mean(0) - Pi = scores_for_aux.mean(0) - fi = ce * self.n_routed_experts - aux_loss = (Pi * fi).sum() * self.alpha - else: - aux_loss = scores.new_zeros(1).squeeze() - return topk_idx, topk_weight, aux_loss - + return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) class MOEFeedForward(nn.Module): def __init__(self, config: MiniMindConfig): super().__init__() self.config = config - self.experts = nn.ModuleList([ - FeedForward(config) - for _ in range(config.n_routed_experts) - ]) - self.gate = MoEGate(config) - if config.n_shared_experts > 0: - self.shared_experts = nn.ModuleList([ - FeedForward(config) - for _ in range(config.n_shared_experts) - ]) + self.gate = nn.Linear(config.hidden_size, config.num_experts, bias=False) + self.experts = nn.ModuleList([FeedForward(config, intermediate_size=config.moe_intermediate_size) for _ in range(config.num_experts)]) + self.act_fn = ACT2FN[config.hidden_act] def forward(self, x): - identity = x - orig_shape = x.shape - bsz, seq_len, _ = x.shape - # 使用门控机制选择专家 - topk_idx, topk_weight, aux_loss = self.gate(x) - x = x.view(-1, x.shape[-1]) - flat_topk_idx = topk_idx.view(-1) - if self.training: - x = x.repeat_interleave(self.config.num_experts_per_tok, dim=0) - y = torch.empty_like(x, dtype=x.dtype) - for i, expert in enumerate(self.experts): - expert_out = expert(x[flat_topk_idx == i]) - if expert_out.shape[0] > 0: y[flat_topk_idx == i] = expert_out.to(y.dtype) - else: y[flat_topk_idx == i] = expert_out.to(y.dtype) + 0 * sum(p.sum() for p in expert.parameters()) - y = (y.view(*topk_weight.shape, -1) * topk_weight.unsqueeze(-1)).sum(dim=1) - y = y.view(*orig_shape) + batch_size, seq_len, hidden_dim = x.shape + x_flat = x.view(-1, hidden_dim) + scores = F.softmax(self.gate(x_flat), dim=-1) + topk_weight, topk_idx = torch.topk(scores, k=self.config.num_experts_per_tok, dim=-1, sorted=False) + if self.config.norm_topk_prob: topk_weight = topk_weight / (topk_weight.sum(dim=-1, keepdim=True) + 1e-20) + y = torch.zeros_like(x_flat) + for i, expert in enumerate(self.experts): + mask = (topk_idx == i) + if mask.any(): + token_idx = mask.any(dim=-1).nonzero().flatten() + weight = topk_weight[mask].view(-1, 1) + y.index_add_(0, token_idx, (expert(x_flat[token_idx]) * weight).to(y.dtype)) + elif self.training: + y[0, 0] += 0 * sum(p.sum() for p in expert.parameters()) + if self.training and self.config.router_aux_loss_coef > 0: + load = F.one_hot(topk_idx, self.config.num_experts).float().mean(0) + self.aux_loss = (load * scores.mean(0)).sum() * self.config.num_experts * self.config.router_aux_loss_coef else: - y = self.moe_infer(x, flat_topk_idx, topk_weight.view(-1, 1)).view(*orig_shape) - if self.config.n_shared_experts > 0: - for expert in self.shared_experts: - y = y + expert(identity) - self.aux_loss = aux_loss - return y - - @torch.no_grad() - def moe_infer(self, x, flat_expert_indices, flat_expert_weights): - expert_cache = torch.zeros_like(x) - idxs = flat_expert_indices.argsort() - tokens_per_expert = flat_expert_indices.bincount().cpu().numpy().cumsum(0) - token_idxs = idxs // self.config.num_experts_per_tok - # 当tokens_per_expert = [6, 15, 20, 26]，tokens_per_expert.shape[0]即为专家数量（此时为4） - # 且token_idxs = [3, 7, 19, 21, 24, 25, 4, 5, 6, 10, 11, 12...] 时 - # 意味token_idxs[:6] -> [3, 7, 19, 21, 24, 25]这6个位置属于专家0处理的token（每个token有可能被多个专家处理，这取决于num_experts_per_tok） - # 接下来9个位置token_idxs[6:15] -> [4, 5, 6, 10, 11, 12...]属于专家1处理的token...依此类推 - for i, end_idx in enumerate(tokens_per_expert): - start_idx = 0 if i == 0 else tokens_per_expert[i - 1] - if start_idx == end_idx: - continue - expert = self.experts[i] - exp_token_idx = token_idxs[start_idx:end_idx] - expert_tokens = x[exp_token_idx] - expert_out = expert(expert_tokens).to(expert_cache.dtype) - expert_out.mul_(flat_expert_weights[idxs[start_idx:end_idx]]) - expert_cache.scatter_add_(0, exp_token_idx.view(-1, 1).repeat(1, x.shape[-1]), expert_out) - - return expert_cache - + self.aux_loss = scores.new_zeros(1).squeeze() + return y.view(batch_size, seq_len, hidden_dim) class MiniMindBlock(nn.Module): def __init__(self, layer_id: int, config: MiniMindConfig): super().__init__() - self.num_attention_heads = config.num_attention_heads - self.hidden_size = config.hidden_size - self.head_dim = config.hidden_size // config.num_attention_heads self.self_attn = Attention(config) - - self.layer_id = layer_id self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.mlp = FeedForward(config) if not config.use_moe else MOEFeedForward(config) @@ -372,7 +191,6 @@ class MiniMindBlock(nn.Module): hidden_states = hidden_states + self.mlp(self.post_attention_layernorm(hidden_states)) return hidden_states, present_key_value - class MiniMindModel(nn.Module): def __init__(self, config: MiniMindConfig): super().__init__() @@ -382,33 +200,19 @@ class MiniMindModel(nn.Module): self.dropout = nn.Dropout(config.dropout) self.layers = nn.ModuleList([MiniMindBlock(l, config) for l in range(self.num_hidden_layers)]) self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - - freqs_cos, freqs_sin = precompute_freqs_cis(dim=config.hidden_size // config.num_attention_heads, - end=config.max_position_embeddings, rope_base=config.rope_theta, - rope_scaling=config.rope_scaling) + freqs_cos, freqs_sin = precompute_freqs_cis(dim=config.head_dim, end=config.max_position_embeddings, rope_base=config.rope_theta, rope_scaling=config.rope_scaling) self.register_buffer("freqs_cos", freqs_cos, persistent=False) self.register_buffer("freqs_sin", freqs_sin, persistent=False) - def forward(self, - input_ids: Optional[torch.Tensor] = None, - attention_mask: Optional[torch.Tensor] = None, - past_key_values: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None, - use_cache: bool = False, - **kwargs): + def forward(self, input_ids, attention_mask=None, past_key_values=None, use_cache=False, **kwargs): batch_size, seq_length = input_ids.shape if hasattr(past_key_values, 'layers'): past_key_values = None past_key_values = past_key_values or [None] * len(self.layers) start_pos = past_key_values[0][0].shape[1] if past_key_values[0] is not None else 0 - hidden_states = self.dropout(self.embed_tokens(input_ids)) - - position_embeddings = ( - self.freqs_cos[start_pos:start_pos + seq_length], - self.freqs_sin[start_pos:start_pos + seq_length] - ) - + position_embeddings = (self.freqs_cos[start_pos:start_pos + seq_length], self.freqs_sin[start_pos:start_pos + seq_length]) presents = [] - for layer_idx, (layer, past_key_value) in enumerate(zip(self.layers, past_key_values)): + for layer, past_key_value in zip(self.layers, past_key_values): hidden_states, present = layer( hidden_states, position_embeddings, @@ -417,47 +221,59 @@ class MiniMindModel(nn.Module): attention_mask=attention_mask ) presents.append(present) - hidden_states = self.norm(hidden_states) - aux_loss = sum([l.mlp.aux_loss for l in self.layers if isinstance(l.mlp, MOEFeedForward)], hidden_states.new_zeros(1).squeeze()) return hidden_states, presents, aux_loss - class MiniMindForCausalLM(PreTrainedModel, GenerationMixin): config_class = MiniMindConfig - def __init__(self, config: MiniMindConfig = None): self.config = config or MiniMindConfig() super().__init__(self.config) self.model = MiniMindModel(self.config) self.lm_head = nn.Linear(self.config.hidden_size, self.config.vocab_size, bias=False) self.model.embed_tokens.weight = self.lm_head.weight - - def forward(self, - input_ids: Optional[torch.Tensor] = None, - attention_mask: Optional[torch.Tensor] = None, - labels: Optional[torch.Tensor] = None, - past_key_values: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None, - use_cache: bool = False, - logits_to_keep: Union[int, torch.Tensor] = 0, - **args): - hidden_states, past_key_values, aux_loss = self.model( - input_ids=input_ids, - attention_mask=attention_mask, - past_key_values=past_key_values, - use_cache=use_cache, - **args - ) + + def forward(self, input_ids, attention_mask=None, past_key_values=None, use_cache=False, logits_to_keep=0, labels=None, **kwargs): + hidden_states, past_key_values, aux_loss = self.model(input_ids, attention_mask, past_key_values, use_cache, **kwargs) slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep logits = self.lm_head(hidden_states[:, slice_indices, :]) - loss = None if labels is not None: - shift_logits = logits[..., :-1, :].contiguous() - shift_labels = labels[..., 1:].contiguous() - loss = F.cross_entropy(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1), ignore_index=-100) - - output = CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=past_key_values, hidden_states=hidden_states) - output.aux_loss = aux_loss - return output + x, y = logits[..., :-1, :].contiguous(), labels[..., 1:].contiguous() + loss = F.cross_entropy(x.view(-1, x.size(-1)), y.view(-1), ignore_index=-100) + return MoeCausalLMOutputWithPast(loss=loss, aux_loss=aux_loss, logits=logits, past_key_values=past_key_values, hidden_states=hidden_states) + + # https://github.com/jingyaogong/minimind/discussions/611 + @torch.inference_mode() + def generate(self, inputs=None, attention_mask=None, max_new_tokens=8192, temperature=0.85, top_p=0.85, top_k=50, eos_token_id=2, streamer=None, use_cache=True, num_return_sequences=1, do_sample=True, repetition_penalty=1.0, **kwargs): + input_ids = kwargs.pop("input_ids", inputs).repeat(num_return_sequences, 1) + attention_mask = attention_mask.repeat(num_return_sequences, 1) if attention_mask is not None else None + past_key_values = kwargs.pop("past_key_values", None) + finished = torch.zeros(input_ids.shape[0], dtype=torch.bool, device=input_ids.device) + if streamer: streamer.put(input_ids.cpu()) + for _ in range(max_new_tokens): + past_len = past_key_values[0][0].shape[1] if past_key_values else 0 + outputs = self.forward(input_ids[:, past_len:], attention_mask, past_key_values, use_cache=use_cache, **kwargs) + attention_mask = torch.cat([attention_mask, attention_mask.new_ones(attention_mask.shape[0], 1)], -1) if attention_mask is not None else None + logits = outputs.logits[:, -1, :] / temperature + if repetition_penalty != 1.0: + for i in range(input_ids.shape[0]): logits[i, torch.unique(input_ids[i])] /= repetition_penalty + if top_k > 0: + logits[logits < torch.topk(logits, top_k)[0][..., -1, None]] = -float('inf') + if top_p < 1.0: + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + mask = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1) > top_p + mask[..., 1:], mask[..., 0] = mask[..., :-1].clone(), 0 + logits[mask.scatter(1, sorted_indices, mask)] = -float('inf') + next_token = torch.multinomial(torch.softmax(logits, dim=-1), num_samples=1) if do_sample else torch.argmax(logits, dim=-1, keepdim=True) + if eos_token_id is not None: next_token = torch.where(finished.unsqueeze(-1), next_token.new_full((next_token.shape[0], 1), eos_token_id), next_token) + input_ids = torch.cat([input_ids, next_token], dim=-1) + past_key_values = outputs.past_key_values if use_cache else None + if streamer: streamer.put(next_token.cpu()) + if eos_token_id is not None: + finished |= next_token.squeeze(-1).eq(eos_token_id) + if finished.all(): break + if streamer: streamer.end() + if kwargs.get("return_kv"): return {'generated_ids': input_ids, 'past_kv': past_key_values} + return input_ids \ No newline at end of file diff --git a/model/tokenizer.json b/model/tokenizer.json index b255122..e6ca86c 100644 --- a/model/tokenizer.json +++ b/model/tokenizer.json @@ -29,6 +29,303 @@ "rstrip": false, "normalized": false, "special": true + }, + { + "id": 3, + "content": "<|object_ref_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 4, + "content": "<|object_ref_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 5, + "content": "<|box_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 6, + "content": "<|box_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 7, + "content": "<|quad_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 8, + "content": "<|quad_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 9, + "content": "<|vision_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 10, + "content": "<|vision_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 11, + "content": "<|vision_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 12, + "content": "<|image_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 13, + "content": "<|video_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 14, + "content": "<|audio_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 15, + "content": "<|audio_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 16, + "content": "<|audio_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 17, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 18, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 19, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 20, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 21, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 22, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 23, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 24, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 25, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 26, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 27, + "content": "<|buffer1|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 28, + "content": "<|buffer2|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 29, + "content": "<|buffer3|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 30, + "content": "<|buffer4|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 31, + "content": "<|buffer5|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 32, + "content": "<|buffer6|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 33, + "content": "<|buffer7|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 34, + "content": "<|buffer8|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + { + "id": 35, + "content": "<|buffer9|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false } ], "normalizer": null, @@ -58,6425 +355,6405 @@ "<|endoftext|>": 0, "<|im_start|>": 1, "<|im_end|>": 2, - "!": 3, - "\"": 4, - "#": 5, - "$": 6, - "%": 7, - "&": 8, - "'": 9, - "(": 10, - ")": 11, - "*": 12, - "+": 13, - ",": 14, - "-": 15, - ".": 16, - "/": 17, - "0": 18, - "1": 19, - "2": 20, - "3": 21, - "4": 22, - "5": 23, - "6": 24, - "7": 25, - "8": 26, - "9": 27, - ":": 28, - ";": 29, - "<": 30, - "=": 31, - ">": 32, - "?": 33, - "@": 34, - "A": 35, - "B": 36, - "C": 37, - "D": 38, - "E": 39, - "F": 40, - "G": 41, - "H": 42, - "I": 43, - "J": 44, - "K": 45, - "L": 46, - "M": 47, - "N": 48, - "O": 49, - "P": 50, - "Q": 51, - "R": 52, - "S": 53, - "T": 54, - "U": 55, - "V": 56, - "W": 57, - "X": 58, - "Y": 59, - "Z": 60, - "[": 61, - "\\": 62, - "]": 63, - "^": 64, - "_": 65, - "`": 66, - "a": 67, - "b": 68, - "c": 69, - "d": 70, - "e": 71, - "f": 72, - "g": 73, - "h": 74, - "i": 75, - "j": 76, - "k": 77, - "l": 78, - "m": 79, - "n": 80, - "o": 81, - "p": 82, - "q": 83, - "r": 84, - "s": 85, - "t": 86, - "u": 87, - "v": 88, - "w": 89, - "x": 90, - "y": 91, - "z": 92, - "{": 93, - "|": 94, - "}": 95, - "~": 96, - "¡": 97, - "¢": 98, - "£": 99, - "¤": 100, - "¥": 101, - "¦": 102, - "§": 103, - "¨": 104, - "©": 105, - "ª": 106, - "«": 107, - "¬": 108, - "®": 109, - "¯": 110, - "°": 111, - "±": 112, - "²": 113, - "³": 114, - "´": 115, - "µ": 116, - "¶": 117, - "·": 118, - "¸": 119, - "¹": 120, - "º": 121, - "»": 122, - "¼": 123, - "½": 124, - "¾": 125, - "¿": 126, - "À": 127, - "Á": 128, - "Â": 129, - "Ã": 130, - "Ä": 131, - "Å": 132, - "Æ": 133, - "Ç": 134, - "È": 135, - "É": 136, - "Ê": 137, - "Ë": 138, - "Ì": 139, - "Í": 140, - "Î": 141, - "Ï": 142, - "Ð": 143, - "Ñ": 144, - "Ò": 145, - "Ó": 146, - "Ô": 147, - "Õ": 148, - "Ö": 149, - "×": 150, - "Ø": 151, - "Ù": 152, - "Ú": 153, - "Û": 154, - "Ü": 155, - "Ý": 156, - "Þ": 157, - "ß": 158, - "à": 159, - "á": 160, - "â": 161, - "ã": 162, - "ä": 163, - "å": 164, - "æ": 165, - "ç": 166, - "è": 167, - "é": 168, - "ê": 169, - "ë": 170, - "ì": 171, - "í": 172, - "î": 173, - "ï": 174, - "ð": 175, - "ñ": 176, - "ò": 177, - "ó": 178, - "ô": 179, - "õ": 180, - "ö": 181, - "÷": 182, - "ø": 183, - "ù": 184, - "ú": 185, - "û": 186, - "ü": 187, - "ý": 188, - "þ": 189, - "ÿ": 190, - "Ā": 191, - "ā": 192, - "Ă": 193, - "ă": 194, - "Ą": 195, - "ą": 196, - "Ć": 197, - "ć": 198, - "Ĉ": 199, - "ĉ": 200, - "Ċ": 201, - "ċ": 202, - "Č": 203, - "č": 204, - "Ď": 205, - "ď": 206, - "Đ": 207, - "đ": 208, - "Ē": 209, - "ē": 210, - "Ĕ": 211, - "ĕ": 212, - "Ė": 213, - "ė": 214, - "Ę": 215, - "ę": 216, - "Ě": 217, - "ě": 218, - "Ĝ": 219, - "ĝ": 220, - "Ğ": 221, - "ğ": 222, - "Ġ": 223, - "ġ": 224, - "Ģ": 225, - "ģ": 226, - "Ĥ": 227, - "ĥ": 228, - "Ħ": 229, - "ħ": 230, - "Ĩ": 231, - "ĩ": 232, - "Ī": 233, - "ī": 234, - "Ĭ": 235, - "ĭ": 236, - "Į": 237, - "į": 238, - "İ": 239, - "ı": 240, - "IJ": 241, - "ij": 242, - "Ĵ": 243, - "ĵ": 244, - "Ķ": 245, - "ķ": 246, - "ĸ": 247, - "Ĺ": 248, - "ĺ": 249, - "Ļ": 250, - "ļ": 251, - "Ľ": 252, - "ľ": 253, - "Ŀ": 254, - "ŀ": 255, - "Ł": 256, - "ł": 257, - "Ń": 258, - "Ġt": 259, - "Ġa": 260, - "in": 261, - "he": 262, - "re": 263, - "ï¼": 264, - "ä¸": 265, - "on": 266, - "at": 267, - "çļ": 268, - "çļĦ": 269, - "ï¼Į": 270, - "Ġs": 271, - "Ġc": 272, - "nd": 273, - "ãĢ": 274, - "er": 275, - "Ġthe": 276, - "es": 277, - "en": 278, - "or": 279, - "an": 280, - "Ġand": 281, - "ing": 282, - "Ġp": 283, - "it": 284, - "al": 285, - "ãĢĤ": 286, - "Ġo": 287, - "Ġw": 288, - "ä»": 289, - "Ġto": 290, - "is": 291, - "ou": 292, - "Ġm": 293, - "äº": 294, - "Ġin": 295, - "Ġf": 296, - "Ġb": 297, - "ed": 298, - "ion": 299, - "åı": 300, - "ic": 301, - "Ġd": 302, - "Ġof": 303, - "le": 304, - "ar": 305, - "ro": 306, - "ĠĠ": 307, - "åħ": 308, - "ent": 309, - "æľ": 310, - "Ġe": 311, - "åĴ": 312, + "<|object_ref_start|>": 3, + "<|object_ref_end|>": 4, + "<|box_start|>": 5, + "<|box_end|>": 6, + "<|quad_start|>": 7, + "<|quad_end|>": 8, + "<|vision_start|>": 9, + "<|vision_end|>": 10, + "<|vision_pad|>": 11, + "<|image_pad|>": 12, + "<|video_pad|>": 13, + "<|audio_start|>": 14, + "<|audio_end|>": 15, + "<|audio_pad|>": 16, + "": 17, + "": 18, + "": 19, + "": 20, + "": 21, + "": 22, + "": 23, + "": 24, + "": 25, + "": 26, + "<|buffer1|>": 27, + "<|buffer2|>": 28, + "<|buffer3|>": 29, + "<|buffer4|>": 30, + "<|buffer5|>": 31, + "<|buffer6|>": 32, + "<|buffer7|>": 33, + "<|buffer8|>": 34, + "<|buffer9|>": 35, + "!": 36, + "\"": 37, + "#": 38, + "$": 39, + "%": 40, + "&": 41, + "'": 42, + "(": 43, + ")": 44, + "*": 45, + "+": 46, + ",": 47, + "-": 48, + ".": 49, + "/": 50, + "0": 51, + "1": 52, + "2": 53, + "3": 54, + "4": 55, + "5": 56, + "6": 57, + "7": 58, + "8": 59, + "9": 60, + ":": 61, + ";": 62, + "<": 63, + "=": 64, + ">": 65, + "?": 66, + "@": 67, + "A": 68, + "B": 69, + "C": 70, + "D": 71, + "E": 72, + "F": 73, + "G": 74, + "H": 75, + "I": 76, + "J": 77, + "K": 78, + "L": 79, + "M": 80, + "N": 81, + "O": 82, + "P": 83, + "Q": 84, + "R": 85, + "S": 86, + "T": 87, + "U": 88, + "V": 89, + "W": 90, + "X": 91, + "Y": 92, + "Z": 93, + "[": 94, + "\\": 95, + "]": 96, + "^": 97, + "_": 98, + "`": 99, + "a": 100, + "b": 101, + "c": 102, + "d": 103, + "e": 104, + "f": 105, + "g": 106, + "h": 107, + "i": 108, + "j": 109, + "k": 110, + "l": 111, + "m": 112, + "n": 113, + "o": 114, + "p": 115, + "q": 116, + "r": 117, + "s": 118, + "t": 119, + "u": 120, + "v": 121, + "w": 122, + "x": 123, + "y": 124, + "z": 125, + "{": 126, + "|": 127, + "}": 128, + "~": 129, + "¡": 130, + "¢": 131, + "£": 132, + "¤": 133, + "¥": 134, + "¦": 135, + "§": 136, + "¨": 137, + "©": 138, + "ª": 139, + "«": 140, + "¬": 141, + "®": 142, + "¯": 143, + "°": 144, + "±": 145, + "²": 146, + "³": 147, + "´": 148, + "µ": 149, + "¶": 150, + "·": 151, + "¸": 152, + "¹": 153, + "º": 154, + "»": 155, + "¼": 156, + "½": 157, + "¾": 158, + "¿": 159, + "À": 160, + "Á": 161, + "Â": 162, + "Ã": 163, + "Ä": 164, + "Å": 165, + "Æ": 166, + "Ç": 167, + "È": 168, + "É": 169, + "Ê": 170, + "Ë": 171, + "Ì": 172, + "Í": 173, + "Î": 174, + "Ï": 175, + "Ð": 176, + "Ñ": 177, + "Ò": 178, + "Ó": 179, + "Ô": 180, + "Õ": 181, + "Ö": 182, + "×": 183, + "Ø": 184, + "Ù": 185, + "Ú": 186, + "Û": 187, + "Ü": 188, + "Ý": 189, + "Þ": 190, + "ß": 191, + "à": 192, + "á": 193, + "â": 194, + "ã": 195, + "ä": 196, + "å": 197, + "æ": 198, + "ç": 199, + "è": 200, + "é": 201, + "ê": 202, + "ë": 203, + "ì": 204, + "í": 205, + "î": 206, + "ï": 207, + "ð": 208, + "ñ": 209, + "ò": 210, + "ó": 211, + "ô": 212, + "õ": 213, + "ö": 214, + "÷": 215, + "ø": 216, + "ù": 217, + "ú": 218, + "û": 219, + "ü": 220, + "ý": 221, + "þ": 222, + "ÿ": 223, + "Ā": 224, + "ā": 225, + "Ă": 226, + "ă": 227, + "Ą": 228, + "ą": 229, + "Ć": 230, + "ć": 231, + "Ĉ": 232, + "ĉ": 233, + "Ċ": 234, + "ċ": 235, + "Č": 236, + "č": 237, + "Ď": 238, + "ď": 239, + "Đ": 240, + "đ": 241, + "Ē": 242, + "ē": 243, + "Ĕ": 244, + "ĕ": 245, + "Ė": 246, + "ė": 247, + "Ę": 248, + "ę": 249, + "Ě": 250, + "ě": 251, + "Ĝ": 252, + "ĝ": 253, + "Ğ": 254, + "ğ": 255, + "Ġ": 256, + "ġ": 257, + "Ģ": 258, + "ģ": 259, + "Ĥ": 260, + "ĥ": 261, + "Ħ": 262, + "ħ": 263, + "Ĩ": 264, + "ĩ": 265, + "Ī": 266, + "ī": 267, + "Ĭ": 268, + "ĭ": 269, + "Į": 270, + "į": 271, + "İ": 272, + "ı": 273, + "IJ": 274, + "ij": 275, + "Ĵ": 276, + "ĵ": 277, + "Ķ": 278, + "ķ": 279, + "ĸ": 280, + "Ĺ": 281, + "ĺ": 282, + "Ļ": 283, + "ļ": 284, + "Ľ": 285, + "ľ": 286, + "Ŀ": 287, + "ŀ": 288, + "Ł": 289, + "ł": 290, + "Ń": 291, + "ï¼": 292, + "ä¸": 293, + "ï¼Į": 294, + "çļ": 295, + "çļĦ": 296, + "Ġt": 297, + "ãĢ": 298, + "Ġa": 299, + "he": 300, + "in": 301, + "ãĢĤ": 302, + "äº": 303, + "ä»": 304, + "åı": 305, + "re": 306, + "ä½": 307, + "æľ": 308, + "Ġthe": 309, + "on": 310, + "er": 311, + "**": 312, "è¿": 313, - "ä½": 314, - "åĴĮ": 315, - "æĪ": 316, - "å®": 317, + "at": 314, + "åħ": 315, + "Ġs": 316, + "Ġc": 317, "åĪ": 318, - "ve": 319, - "us": 320, - "Ġre": 321, - "Ġh": 322, - "Ġth": 323, - "as": 324, - "ct": 325, - "çĶ": 326, - "om": 327, + "Ġo": 319, + "å®": 320, + "çĶ": 321, + "æĪ": 322, + "åĬ": 323, + "is": 324, + "an": 325, + "ou": 326, + "en": 327, "åľ": 328, - "å¤": 329, - "æĺ": 330, - "åĬ": 331, - "åIJ": 332, - "ä¸Ģ": 333, - "im": 334, - "è¯": 335, - "æĸ": 336, - "ation": 337, - "lo": 338, - "ç»": 339, - "Ġbe": 340, - "ãĢģ": 341, - "id": 342, - "Ġcan": 343, - "il": 344, - "æĺ¯": 345, - "ä¹": 346, - "è®": 347, - "ĠA": 348, - "Ġthat": 349, - "ĠT": 350, - "以": 351, - "ch": 352, - "Ġy": 353, - "ce": 354, - "ï¼ļ": 355, - "ot": 356, - "ers": 357, - "Ġn": 358, - "éĢ": 359, - "ra": 360, - "å°": 361, - "Ġg": 362, - "Ġyou": 363, - "åŃ": 364, - "Ġpro": 365, - "et": 366, + "es": 329, + "å¤": 330, + "åĴ": 331, + "ĠĠ": 332, + "æĺ": 333, + "åĴĮ": 334, + "åIJ": 335, + "or": 336, + "ï¼ļ": 337, + "it": 338, + "Ġw": 339, + "æķ": 340, + "Ġf": 341, + "ç»": 342, + "è¯": 343, + "Ġp": 344, + "ãĢģ": 345, + "Ġan": 346, + "æĸ": 347, + "è®": 348, + "ä¸Ģ": 349, + "ing": 350, + "Ġm": 351, + "al": 352, + "Ġin": 353, + "Ġof": 354, + "ĥ½": 355, + "éĢ": 356, + "æĺ¯": 357, + "以": 358, + "Ġ**": 359, + "åı¯": 360, + "le": 361, + "åŃ": 362, + "Ġb": 363, + "Ġto": 364, + "ä¹": 365, + "个": 366, "åº": 367, - "åľ¨": 368, - "ly": 369, - "Ġis": 370, - "个": 371, - "Ġl": 372, - "ur": 373, - "Ġfor": 374, - "åı¯": 375, - "éĩ": 376, - "st": 377, - "çļĦæ": 378, - "ut": 379, - "Ġhe": 380, - "if": 381, - "ĥ½": 382, - "ä¼": 383, - "ĠI": 384, - "è¡": 385, - "ir": 386, - "ith": 387, - "å¹": 388, - "Ġare": 389, - "ig": 390, - "Ġst": 391, - "el": 392, - "ol": 393, - "å¸": 394, - "ul": 395, - "æĿ": 396, - "æĪij": 397, - "Ġon": 398, - "è¦": 399, - "æľī": 400, - "æĹ": 401, - "å¯": 402, - "è§": 403, - "è¦ģ": 404, - "Ġus": 405, - "ay": 406, - "æķ": 407, - "çī": 408, - "ow": 409, - "ment": 410, - "çĶ¨": 411, - "ess": 412, - "ä¸Ń": 413, - "们": 414, - "人": 415, - "åĩ": 416, - "Ġex": 417, - "ĠĠĠĠ": 418, - "åĽ": 419, - "åĮ": 420, - "å¼": 421, - "Ġcon": 422, - "se": 423, - "èĥ½": 424, - "çİ": 425, - "Ġan": 426, - "Ġwith": 427, - "为": 428, - "ate": 429, - "iv": 430, - "am": 431, - "Ġas": 432, - "ure": 433, - "è¿Ļ": 434, - "åĨ": 435, - "çŃ": 436, - "Ġor": 437, - "å·": 438, - "Ġal": 439, - "ies": 440, - "ç§": 441, - "Ġim": 442, - "æĢ": 443, - "ver": 444, - "ab": 445, - "äºĨ": 446, - "Ġsu": 447, - "Ġde": 448, - "ge": 449, - "th": 450, - "åı¯ä»¥": 451, - "èĢ": 452, - "ä¸į": 453, - "å¾": 454, - "ĠAI": 455, - "Ġen": 456, - "éĹ": 457, - "æī": 458, - "ak": 459, - "ive": 460, - "Ġmo": 461, - "å¥": 462, - "éĿ": 463, - "çĽ": 464, - "ity": 465, - "ä¿": 466, - "un": 467, - "è´": 468, - "åį": 469, - "Ġit": 470, - "Ġimp": 471, - "ect": 472, - "æł": 473, - "å½": 474, - "èĩ": 475, - "é¢": 476, - "åĵ": 477, - "æ³": 478, - "ort": 479, - "ad": 480, - "æŀ": 481, - "em": 482, - "Ġcom": 483, - "å¦": 484, - "her": 485, - "ere": 486, - "ĠS": 487, - "ial": 488, - "ĠC": 489, - "ĠThe": 490, - "çIJ": 491, - "çĶŁ": 492, - "æĦ": 493, - "pp": 494, - "æŃ": 495, - "æĸ¹": 496, - "qu": 497, - "Ġwh": 498, - "å¦Ĥ": 499, - "éľ": 500, - "ant": 501, - "Ġle": 502, - "Ġv": 503, - "æĭ": 504, - "æĬ": 505, - "ust": 506, - "æŶ": 507, - "çŃī": 508, - "åij": 509, - "对": 510, - "ter": 511, - "ld": 512, - "è¡Į": 513, - "Ġch": 514, - "ud": 515, - "éľĢ": 516, - "æ°": 517, - "æĪIJ": 518, - "Ġ|": 519, - "ac": 520, - "ain": 521, - "iz": 522, - "æı": 523, - "ions": 524, - "Ġha": 525, - "æĽ": 526, - "--": 527, - "æĿ¥": 528, - "ome": 529, - "å¿": 530, - "'s": 531, - "Ġne": 532, - "est": 533, - "ä¾": 534, - "um": 535, - "åĪ°": 536, - "åľ°": 537, - "ist": 538, - "âĢ": 539, - "çī©": 540, - "ä¸Ģ个": 541, - "lp": 542, - "æİ": 543, - "èĩª": 544, - "Ġhelp": 545, - "Ġtheir": 546, - "æĶ": 547, - "ä½ľ": 548, - "ä¼ļ": 549, - "æĮ": 550, - "æĪij们": 551, - "nt": 552, - "äºİ": 553, - "åĪĨ": 554, - "res": 555, - "pe": 556, - "åĩº": 557, - "ide": 558, - "æĥ": 559, - "ĠH": 560, - "è¾": 561, - "ĠM": 562, - "ff": 563, - "æ¯": 564, - "od": 565, - "ical": 566, - "Ġwor": 567, - "ä¸Ĭ": 568, - "are": 569, - "æĽ´": 570, - "Ġyour": 571, - "ä¸ĭ": 572, - "èµ": 573, - "ations": 574, - "æķ°": 575, - "Ġte": 576, - "åİ": 577, - "çIJĨ": 578, - "ĠTh": 579, - "è¿ĩ": 580, - "并": 581, - "du": 582, - "éĿ¢": 583, - "Ġad": 584, - "ill": 585, - "æµ": 586, - "好": 587, - "oc": 588, - "act": 589, - "éľĢè¦ģ": 590, - "ä»ĸ": 591, - "å±": 592, - "Ġr": 593, - "Ġmore": 594, - "åѦ": 595, - "ç®": 596, - "igh": 597, - "äºĽ": 598, - "ĠB": 599, - "åĬ¨": 600, - "åĵģ": 601, - "èī": 602, - "ple": 603, - "Ġinc": 604, - "åIJĮ": 605, - "Ġexp": 606, - "ould": 607, - "ä½ł": 608, - "æį": 609, - "æıIJ": 610, - "大": 611, - "çİ°": 612, - "pt": 613, - "ĠP": 614, - "all": 615, - "åĬł": 616, - "ç§į": 617, - "Ġse": 618, - "åĬĽ": 619, - "out": 620, - "Ġhave": 621, - "çº": 622, - "ä½ĵ": 623, - "Ġprov": 624, - "åĮĸ": 625, - "å¤ļ": 626, - "å®ļ": 627, - "Ġused": 628, - "éĢļ": 629, - "cc": 630, - "è¿Ľ": 631, - "æ´": 632, - "Ġsh": 633, - "Ġab": 634, - "os": 635, - "Ġres": 636, - "ĠThis": 637, - "ç¨": 638, - "æĢ§": 639, - "age": 640, - "ri": 641, - "æ¸": 642, - "able": 643, - "åŃIJ": 644, - "Ġby": 645, - "åıij": 646, - "éĩı": 647, - "åºĶ": 648, - "Ġlo": 649, - "使": 650, - "åħ¶": 651, - "é«": 652, - "éĻ": 653, - "é«ĺ": 654, - "度": 655, - "解": 656, - "é£": 657, - "å°Ĩ": 658, - "æ³ķ": 659, - "and": 660, - "ä¿Ŀ": 661, - "ans": 662, - "for": 663, - "rom": 664, - "reat": 665, - "Ġpl": 666, - "çļĦç": 667, - "常": 668, - "è½": 669, - "Ġwe": 670, - "表": 671, - "ake": 672, - "æĪĸ": 673, - "é¢ĺ": 674, - "åŁ": 675, - "Ġme": 676, - "æĸĩ": 677, - "ther": 678, - "ke": 679, - "家": 680, - "åIJĪ": 681, - "æľĢ": 682, - "ine": 683, - "Ġsome": 684, - "ç±": 685, - "éĩį": 686, - "æŀľ": 687, - "ĠW": 688, - "ĠE": 689, - "éĺ": 690, - "our": 691, - "rou": 692, - "çĤ": 693, - "æ±": 694, - "åħ³": 695, - "Ġint": 696, - "ance": 697, - "ä¹Ł": 698, - "éģ": 699, - "ĠĠĠ": 700, - "å®ĥ": 701, - "ag": 702, - "æ¬": 703, - "00": 704, - "è°": 705, - "ult": 706, - "yst": 707, - "éĹ´": 708, - "ç³": 709, - "Ġtr": 710, - "pl": 711, - "art": 712, - "æĦŁ": 713, - "æĤ": 714, - "ata": 715, - "ĠF": 716, - "form": 717, - "计": 718, - "Ġfrom": 719, - "ĠD": 720, - "éĹ®": 721, - "ight": 722, - "ces": 723, - "æį®": 724, - "lop": 725, - "ä¹ĭ": 726, - "Ġfe": 727, - "åģ": 728, - "velop": 729, - "Ġ1": 730, + "çĶ¨": 368, + "åľ¨": 369, + "ar": 370, + "ion": 371, + "èĥ½": 372, + "Ġd": 373, + "**ï¼ļ": 374, + "ic": 375, + "Ġand": 376, + "å°": 377, + "æĹ": 378, + "ä¼": 379, + "éĩ": 380, + "å¸": 381, + "ed": 382, + "åĮ": 383, + "è§": 384, + "Ġy": 385, + "åĽ": 386, + "è¡": 387, + "ä¸Ń": 388, + "å¯": 389, + "as": 390, + "æķ°": 391, + "åĩ": 392, + "ro": 393, + "è¿Ļ": 394, + "Ġis": 395, + "¦Ĥ": 396, + "æĿ": 397, + "æľī": 398, + "Ġn": 399, + "å¼": 400, + "Ġyou": 401, + "æĢ": 402, + "ä¿": 403, + "å¦Ĥ": 404, + "æŀ": 405, + "Ġl": 406, + "åĨ": 407, + "çŃ": 408, + "çİ": 409, + "åı¯ä»¥": 410, + "éĹ": 411, + "ç§": 412, + "Ġe": 413, + "è¦": 414, + "çĽ": 415, + "çī": 416, + "ä¸į": 417, + "为": 418, + "çļĦæ": 419, + "è¦ģ": 420, + "çIJ": 421, + "Ġre": 422, + "äºĨ": 423, + "å¹": 424, + "ent": 425, + "ct": 426, + "å·": 427, + "æı": 428, + "Ġh": 429, + "èĢ": 430, + "é¢": 431, + "st": 432, + "æ³": 433, + "æī": 434, + "ĊĠĠ": 435, + "æĬ": 436, + "ä¾": 437, + "æł": 438, + "et": 439, + "æį": 440, + "ä½ł": 441, + "ĠI": 442, + "æŶ": 443, + "om": 444, + "Ġth": 445, + "el": 446, + "人": 447, + "åį": 448, + "``": 449, + "Ġ=": 450, + "ow": 451, + "Ġ\\": 452, + "å½": 453, + "æıIJ": 454, + "çIJĨ": 455, + "äºİ": 456, + "um": 457, + "å¾": 458, + "æĪĸ": 459, + "id": 460, + "对": 461, + "âĢ": 462, + "æĪij": 463, + "çŃī": 464, + "æİ": 465, + "ä¸Ģ个": 466, + "im": 467, + "éĢļ": 468, + "éĿ": 469, + "æĢ§": 470, + "è´": 471, + "è¿ĩ": 472, + "ce": 473, + "è¡Į": 474, + "ç®": 475, + "使": 476, + "æĮ": 477, + "ve": 478, + "æĶ": 479, + "èĩ": 480, + "æĭ": 481, + "Ġ1": 482, + "Ġu": 483, + "æĸ¹": 484, + "il": 485, + "çĶŁ": 486, + "æį®": 487, + "æĪIJ": 488, + "ation": 489, + "åѦ": 490, + "æĽ": 491, + "Ġg": 492, + "æŃ": 493, + "åĵ": 494, + "th": 495, + "ers": 496, + "am": 497, + "ä½ľ": 498, + "ig": 499, + "ol": 500, + "ut": 501, + "å¿": 502, + "Ġfor": 503, + "ä¸ĭ": 504, + "æĦ": 505, + "éľ": 506, + "if": 507, + "ä¼ļ": 508, + "æĿ¥": 509, + "å¥": 510, + "åĮĸ": 511, + "åĪ°": 512, + "åĪĨ": 513, + "è¾": 514, + "å®ļ": 515, + "ä¿Ŀ": 516, + "æĽ´": 517, + "äºĽ": 518, + "ç¡": 519, + "æµ": 520, + "éĻ": 521, + "Ġor": 522, + "Ġas": 523, + "éľĢ": 524, + "å±": 525, + "们": 526, + "ĠT": 527, + "ç¨": 528, + "ĠA": 529, + "ä½ĵ": 530, + "åĬ¨": 531, + "åıij": 532, + "使çĶ¨": 533, + "解": 534, + "ĠC": 535, + "Ġ\"": 536, + "qu": 537, + "è¿Ľ": 538, + "æŀľ": 539, + "Ġpro": 540, + "Ġcan": 541, + "ly": 542, + "ur": 543, + "éĺ": 544, + "åºĶ": 545, + "Ġ2": 546, + "åij": 547, + "Ġ-": 548, + "ul": 549, + "ch": 550, + "åŁ": 551, + "ä¸İ": 552, + "åľ°": 553, + "ï¼ī": 554, + "ï¼Ī": 555, + "æ³ķ": 556, + "ate": 557, + "èµ": 558, + "ra": 559, + "åIJĮ": 560, + "é«": 561, + "ä¸Ĭ": 562, + "Ġit": 563, + "é«ĺ": 564, + "大": 565, + "ç½": 566, + "ç¡®": 567, + "ith": 568, + "æķ°æį®": 569, + "ist": 570, + "åIJĪ": 571, + "ab": 572, + "se": 573, + "æĸĩ": 574, + "è½": 575, + "æ¯": 576, + "度": 577, + "åı¯èĥ½": 578, + "èĩª": 579, + "Ġst": 580, + "å®ŀ": 581, + "Ġhe": 582, + "ge": 583, + "éĩı": 584, + "计": 585, + "ç¨ĭ": 586, + "çº": 587, + "åİ": 588, + "æ°": 589, + "å¤ļ": 590, + "Ġthat": 591, + "æĥ": 592, + "ot": 593, + "åĬĽ": 594, + "åĬł": 595, + "çİ°": 596, + "åĩº": 597, + "ç¤": 598, + "Ġwith": 599, + "Ġbe": 600, + "ç³": 601, + "ä¸ļ": 602, + "Ġv": 603, + "ç§į": 604, + "éĿ¢": 605, + "常": 606, + "éĹ´": 607, + "åĬ¡": 608, + "设": 609, + "æĬĢ": 610, + "est": 611, + "å»": 612, + "ä¾Ľ": 613, + "Ġcon": 614, + "éĹ®": 615, + "åĬ©": 616, + "æľ¯": 617, + "æ´": 618, + "Ġon": 619, + "æ±": 620, + "Ġmo": 621, + "æĻ": 622, + "å°Ĩ": 623, + "```": 624, + "å¢": 625, + "表": 626, + "éģ": 627, + "'s": 628, + "åħ³": 629, + "éľĢè¦ģ": 630, + "ĠS": 631, + "é¢ĺ": 632, + "å·¥": 633, + "##": 634, + "建": 635, + "åħ¶": 636, + "éĩį": 637, + "æľº": 638, + "ir": 639, + "æĸ°": 640, + "çī©": 641, + "éĢļè¿ĩ": 642, + "åħ¨": 643, + "æıIJä¾Ľ": 644, + "ä¿¡": 645, + "çĦ": 646, + "ç³»": 647, + "æĮģ": 648, + "çł": 649, + "å¼ı": 650, + "un": 651, + "ver": 652, + "pe": 653, + "Ġwh": 654, + "ay": 655, + "ç´": 656, + "æģ": 657, + "好": 658, + "åĢ": 659, + "ç«": 660, + "å¦Ĥæŀľ": 661, + "èī": 662, + "åŃIJ": 663, + "æ¸": 664, + "Ġ(": 665, + "ĠH": 666, + "çĦ¶": 667, + "iv": 668, + "åģ": 669, + "00": 670, + "åħ·": 671, + "ç±": 672, + "Ł¥": 673, + "ld": 674, + "çĤ": 675, + "ib": 676, + "è°": 677, + "ä»ĸ": 678, + "é¡": 679, + "ç®Ĺ": 680, + "ãģ": 681, + "åĪ¶": 682, + "âĢľ": 683, + "âĢĿ": 684, + "ä¹ł": 685, + "åŀ": 686, + "em": 687, + "ide": 688, + "æľ¬": 689, + "æĪ·": 690, + "ĠP": 691, + "è¶": 692, + "ke": 693, + "nt": 694, + "æľĢ": 695, + "åĮħ": 696, + "å®ĥ": 697, + "çī¹": 698, + "Ġcom": 699, + "å°ı": 700, + "åIJİ": 701, + "åŃĹ": 702, + "帮": 703, + "èĢĮ": 704, + "æķĪ": 705, + "èĢħ": 706, + "Ġ3": 707, + "igh": 708, + "é£": 709, + "åĵģ": 710, + "代": 711, + "件": 712, + "de": 713, + "umb": 714, + "yth": 715, + "éļ": 716, + "åħ¥": 717, + "Ĥ¨": 718, + "Ġex": 719, + "ä¹Ł": 720, + "æ²": 721, + "lo": 722, + "缮": 723, + "åŀĭ": 724, + "ĊĠĠĠ": 725, + "家": 726, + "å¼Ģ": 727, + "and": 728, + "¨¡": 729, + "Ġne": 730, "åĽł": 731, - "ks": 732, - "æ²": 733, - "Ġu": 734, - "å°ı": 735, - "ystem": 736, - "Ġdis": 737, - "ĠR": 738, - "gy": 739, - "å·¥": 740, - "ç¨ĭ": 741, - "å¢": 742, - "ence": 743, - "èĤ": 744, - "ç¡": 745, - "Ġtra": 746, - "å»": 747, - "åħ¥": 748, - "ign": 749, - "alth": 750, - "Ġsuch": 751, - "ach": 752, - "æĻ": 753, - "arn": 754, - "Ġdata": 755, - "è¶": 756, - "å®ŀ": 757, - "so": 758, - "Ġdevelop": 759, - "ç¤": 760, - "Ġacc": 761, - "ast": 762, - "èĢĮ": 763, - "Ġ\"": 764, - "Ġother": 765, - "建": 766, - "Ġeff": 767, - "ç«": 768, - "Ġman": 769, - "åħ¬": 770, - "åĢ": 771, - "çĦ": 772, - "ms": 773, - "å¼ı": 774, - "èī²": 775, - "å¾Ĺ": 776, - "ific": 777, - "Ġj": 778, - "Ġro": 779, - "Ġhas": 780, - "chn": 781, - "olo": 782, - "åĪ¶": 783, - "èĬ": 784, - "使çĶ¨": 785, - "ous": 786, - "ual": 787, - "Ġat": 788, - "Ġem": 789, - "ell": 790, - "Ġsystem": 791, - "Ġhealth": 792, - "ities": 793, - "Ġexam": 794, - "ib": 795, - "éĶ": 796, - "Ġabout": 797, - "产": 798, - "åIJİ": 799, - "æĦı": 800, - "ç±»": 801, - "Ġpre": 802, - "æĤ¨": 803, - "Ġalso": 804, - "ents": 805, - "Ġind": 806, - "ind": 807, - "éĢĤ": 808, - "Ġtechn": 809, - "ress": 810, - "æĥħ": 811, - "éĹ®é¢ĺ": 812, - "Ġuse": 813, - "ï¼Ł": 814, - "Ġincl": 815, - "Ġspe": 816, - "ich": 817, - "ps": 818, - "æľº": 819, - "Ġthey": 820, - "ie": 821, - "Ġhow": 822, - "Ġwork": 823, - "ä¸ļ": 824, - "ç´": 825, - "Ġimpro": 826, - "Ġlearn": 827, - "æĸ°": 828, - "çĤ¹": 829, - "Ġcont": 830, - "ard": 831, - "çĦ¶": 832, - "æľ¬": 833, - "ç³»": 834, - "ç¡®": 835, - "设": 836, - "åħ·": 837, - "éĢī": 838, - "èĢħ": 839, - "éħ": 840, - "gh": 841, - "__": 842, - "Ġnot": 843, - "çľ": 844, - "缸": 845, - "Ġprovide": 846, - "åī": 847, - "ional": 848, - "Ġens": 849, - "ä¸İ": 850, - "è´¨": 851, - "ential": 852, - "ç»ı": 853, - "å¿ĥ": 854, - "ang": 855, - "æѤ": 856, - "end": 857, - "Ġpo": 858, - "è¿Ľè¡Į": 859, - "ice": 860, - "Ġ-": 861, - "Ġway": 862, - "å·±": 863, - "Ġ2": 864, - "ime": 865, - "ç½": 866, - "èĩªå·±": 867, - "Ġun": 868, - "bot": 869, - "Ġinclud": 870, - "ated": 871, - "æ°´": 872, - "éķ": 873, - "æĮģ": 874, - "代": 875, - "é¡": 876, - "æīĢ": 877, - "çĿ": 878, - "pport": 879, - "ood": 880, - "ike": 881, - "ru": 882, - "Ġcomm": 883, - "ĠL": 884, - "ä¿¡": 885, - "ĠG": 886, - "çŁ": 887, - "çĶµ": 888, - "Ġwas": 889, - "low": 890, - "erv": 891, - "åĮħ": 892, - "ĠĠĠĠĠĠĠĠ": 893, - "Ġwhe": 894, - "dit": 895, - "Ġwhich": 896, - "Ġcomp": 897, - "éª": 898, - "ore": 899, - "ç¾": 900, - "Ġ=": 901, - "çī¹": 902, - "iff": 903, - "ert": 904, - "æģ": 905, - "rit": 906, - "Ġrec": 907, - "åĨħ": 908, - "æĺİ": 909, - "ors": 910, - "Ġpat": 911, - "----": 912, - "æŁ": 913, - "Ġapp": 914, - "ns": 915, - "åĬ¡": 916, - "aly": 917, - "ace": 918, - "æ´»": 919, - "ä¾Ľ": 920, - "av": 921, - "主": 922, - "Ġpers": 923, - "çĥ": 924, - "该": 925, - "Ġmy": 926, - "ç©": 927, - "eri": 928, - "让": 929, - "æĬĢ": 930, - "éķ¿": 931, - "ack": 932, - "ĠN": 933, - "Ġdiff": 934, - "Ġthis": 935, - "åĿ": 936, - "Ġensure": 937, - "å½ĵ": 938, - "Ġout": 939, - "Ġcl": 940, - "Ġk": 941, - "é¦": 942, - "ount": 943, - "çݯ": 944, - "åĬ©": 945, - "Ġtechnolo": 946, - "Ġthese": 947, - "ful": 948, - "éļ": 949, - "æ·": 950, - "ä¸ĢäºĽ": 951, - "Ġsoc": 952, - "å¼Ģ": 953, - "天": 954, - "Ġev": 955, - "Ġredu": 956, - "Ġthem": 957, - "Ġ(": 958, - "éĥ½": 959, - "æĪ·": 960, - "è·": 961, - "åľº": 962, - "æ°Ķ": 963, - "ĠY": 964, - "è¯Ń": 965, - "éĢļè¿ĩ": 966, - "å±ķ": 967, - "Ġco": 968, - "å½±": 969, - "ç¬": 970, - "Ġanaly": 971, - "æ¯Ķ": 972, - "åħ¨": 973, - "Ġimprove": 974, - "ç»ĵ": 975, - "å¹´": 976, - "çķ": 977, - "çĿĢ": 978, - "Ġhum": 979, - "Ġqu": 980, - "ç®Ĺ": 981, - "ĠO": 982, - "é£Ł": 983, - "ility": 984, - "Ġsystems": 985, - "åıĺ": 986, - "ail": 987, - "ç¼": 988, - "çł": 989, - "è¿Ļ个": 990, - "æıIJä¾Ľ": 991, - "ase": 992, - "åŀ": 993, - "ments": 994, - "Ġpot": 995, - "Ġany": 996, - "ä½Ĩ": 997, - "Ġcons": 998, - "ĠIt": 999, - "æł¼": 1000, - "Ġar": 1001, - "æľ¯": 1002, - "éĿŀ": 1003, - "Ġdo": 1004, - "Ġmay": 1005, - "æĭ©": 1006, - "ue": 1007, - "éĢīæĭ©": 1008, - "ry": 1009, - "éĥ": 1010, - "Ġlike": 1011, - "ong": 1012, - "èģ": 1013, - "``": 1014, - "ile": 1015, - "æ±Ĥ": 1016, - "Ġnew": 1017, - "ient": 1018, - "Ġimpact": 1019, - "è¿ĺ": 1020, - "注": 1021, - "ä¹Ī": 1022, - "缮": 1023, - "âĢľ": 1024, - "âĢĿ": 1025, - "ef": 1026, - "ä¾ĭ": 1027, - "Ġpotential": 1028, - "ok": 1029, - "åı¯èĥ½": 1030, - "Ġtrans": 1031, - "Ġact": 1032, - "ï¼ī": 1033, - "Ġspec": 1034, - "æ¶": 1035, - "Ġwill": 1036, - "交": 1037, - "ize": 1038, - "ç¾İ": 1039, - "å¸Ĥ": 1040, - "Ġstud": 1041, - "pon": 1042, - "èº": 1043, - "ä¸įåIJĮ": 1044, - "one": 1045, - "å¾Ī": 1046, - "åıĬ": 1047, - "å¦Ĥæŀľ": 1048, - "çIJĥ": 1049, - "ange": 1050, - "Ġneed": 1051, - "å¤ĸ": 1052, - "ety": 1053, - "aking": 1054, - "请": 1055, - "ater": 1056, - "Ġperson": 1057, - "ident": 1058, - "Ġso": 1059, - "Ġmake": 1060, - "å¹³": 1061, - "å¤Ł": 1062, - "身": 1063, - "ï¼Ī": 1064, - "Ġinform": 1065, - "æ¡": 1066, - "äºĭ": 1067, - "åıĹ": 1068, - "ased": 1069, - "ild": 1070, - "Ġoff": 1071, - "Ġthere": 1072, - "cis": 1073, - "è¢": 1074, - "éĥ¨": 1075, - "æ¯ı": 1076, - "ract": 1077, - "ass": 1078, - "Ġlearning": 1079, - "åĸ": 1080, - "å½¢": 1081, - "ire": 1082, - "ä»İ": 1083, - "bots": 1084, - "èĻ": 1085, - "帮": 1086, - "Ġdes": 1087, - "ĠIn": 1088, - "cess": 1089, - "Ġpe": 1090, - "ify": 1091, - "Ġwho": 1092, - "ä¹ł": 1093, - "æľŁ": 1094, - "Ġexperi": 1095, - "éĤ": 1096, - "Ġsc": 1097, - "ep": 1098, - "ä½ķ": 1099, - "Ġtime": 1100, - "éĿŀ常": 1101, - "æĭ¬": 1102, - "åķ": 1103, - "以ä¸ĭ": 1104, - "éģĵ": 1105, - "Ġcommun": 1106, - "Ġcould": 1107, - "ap": 1108, - "èIJ": 1109, - "è°ĥ": 1110, - "lic": 1111, - "duct": 1112, - "Ġits": 1113, - "cy": 1114, - "说": 1115, - "Ġmed": 1116, - "Ġcol": 1117, - "ular": 1118, - "éĩįè¦ģ": 1119, - "Ġsp": 1120, - "åĪ©": 1121, - "èµ·": 1122, - "Ġprovid": 1123, - "ices": 1124, - "åĻ": 1125, - "æĸĻ": 1126, - "Ġimport": 1127, - "ural": 1128, - "åŃĹ": 1129, - "Ġund": 1130, - "int": 1131, - "Ġover": 1132, - "åı¸": 1133, - "æł¹": 1134, - "é¥": 1135, - "ples": 1136, - "ä»ĸ们": 1137, - "gra": 1138, - "uring": 1139, - "now": 1140, - "åįķ": 1141, - "è¿ĻäºĽ": 1142, - "åīį": 1143, - "å®ī": 1144, - "Ġpr": 1145, - "åĮħæĭ¬": 1146, - "ç»Ļ": 1147, - "The": 1148, - "ä½į": 1149, - "å§": 1150, - "ç´ł": 1151, - "åijĺ": 1152, - "Ġident": 1153, - "åŀĭ": 1154, - "Ġadd": 1155, - "强": 1156, - "æĺ¯ä¸Ģ": 1157, - "ip": 1158, - "gor": 1159, - "Ġsupport": 1160, - "ne": 1161, - "Ġdiffere": 1162, - "åħĥ": 1163, - "Ġass": 1164, - "åĨ³": 1165, - "éĽ": 1166, - "åIJį": 1167, - "Ġgo": 1168, - "Ġtechnology": 1169, - "æĢ»": 1170, - "è®®": 1171, - "Ġinter": 1172, - "Ġinv": 1173, - "Ġour": 1174, - "æķĪ": 1175, - "ustom": 1176, - "Ġrel": 1177, - "ife": 1178, - "åĻ¨": 1179, - "ings": 1180, - "ä»·": 1181, - "Ġpart": 1182, - "被": 1183, - "æīĭ": 1184, - "ary": 1185, - "Ġrespon": 1186, - "ĊĠĠĠ": 1187, - "好çļĦ": 1188, - "ative": 1189, - "帮åĬ©": 1190, - "绣": 1191, - "æĶ¾": 1192, - "ĠHere": 1193, - "çģ": 1194, - "Ġbut": 1195, - "æģ¯": 1196, - "æŃ£": 1197, - "ark": 1198, - "åħ¬åı¸": 1199, - "ory": 1200, - "å¢ĥ": 1201, - "lect": 1202, - "éŁ": 1203, - "æĥ³": 1204, - "é£İ": 1205, - "ating": 1206, - "Ġam": 1207, - "its": 1208, - "æ»": 1209, - "gorith": 1210, - "åĵį": 1211, - "ures": 1212, - "Ġeffect": 1213, - "Ġshould": 1214, - "Ġper": 1215, - "è±": 1216, - "ç²": 1217, - "ict": 1218, - "Ġalgorith": 1219, - "uc": 1220, - "rough": 1221, - "ä»»": 1222, - "件": 1223, - "Ġbet": 1224, - "ia": 1225, - "Ġanalyz": 1226, - "æł¹æį®": 1227, - "ized": 1228, - "æµģ": 1229, - "è§Ĥ": 1230, - "è£": 1231, - "æłĩ": 1232, - "iron": 1233, - "Ġcustom": 1234, - "Ġreg": 1235, - "Ġpersonal": 1236, - "èĥ½å¤Ł": 1237, - "ics": 1238, - "ivid": 1239, - "çĪ": 1240, - "èµĦ": 1241, - "æŃ¥": 1242, - "容": 1243, - "åĪĽ": 1244, - "èĪ": 1245, - "ä¹IJ": 1246, - "导": 1247, - "gan": 1248, - "èĬĤ": 1249, - "Ġall": 1250, - "ens": 1251, - "ame": 1252, - "ness": 1253, - "Ġup": 1254, - "ĠU": 1255, - "èĢĥ": 1256, - "elf": 1257, - "åĢ¼": 1258, - "å°ij": 1259, - "æľį": 1260, - "ari": 1261, - "thical": 1262, - "viron": 1263, - "èĥ": 1264, - "ord": 1265, - "Ġsign": 1266, - "éĩĮ": 1267, - "ound": 1268, - "ople": 1269, - "åŁº": 1270, - "Ġinformation": 1271, - "Ġidentify": 1272, - "åĽŀ": 1273, - "Ġcre": 1274, - "éŁ³": 1275, - "ible": 1276, - "ub": 1277, - "è¿IJ": 1278, - "Ġlead": 1279, - "游": 1280, - "次": 1281, - "åĨĻ": 1282, - "éĤ£": 1283, - "get": 1284, - "èį": 1285, - "Ġexample": 1286, - "ä¼ĺ": 1287, - "å½±åĵį": 1288, - "ish": 1289, - "xt": 1290, - "æº": 1291, - "éªĮ": 1292, - "ob": 1293, - "客": 1294, - "å¤ĩ": 1295, - "åģ¥": 1296, - "车": 1297, - "社": 1298, - "ividual": 1299, - "ered": 1300, - "les": 1301, - "Ġenviron": 1302, - "Ġpeople": 1303, - "æĺŁ": 1304, - "çĸ": 1305, - "çĭ": 1306, - "Ġdet": 1307, - "æĹł": 1308, - "Ġif": 1309, - "ose": 1310, - "ite": 1311, - "å¢ŀ": 1312, - "éĴ": 1313, - "åIJĮæŶ": 1314, - "è¿°": 1315, - "æĸ¹å¼ı": 1316, - "åĽ½": 1317, - "é»": 1318, - "å¤Ħ": 1319, - "Ġexamples": 1320, - "æ®": 1321, - "Ġinto": 1322, - "æĮĩ": 1323, - "Ġhuman": 1324, - "åIJij": 1325, - "示": 1326, - "æķ°æį®": 1327, - "Ġ3": 1328, - "ĠJ": 1329, - "èı": 1330, - "çݯå¢ĥ": 1331, - "als": 1332, - "erst": 1333, - "Ġethical": 1334, - "ç»Ħ": 1335, - "ä¼ł": 1336, - "Ġdifferent": 1337, - "Ġknow": 1338, - "åºı": 1339, - "Ġindividual": 1340, - "æıIJé«ĺ": 1341, - "round": 1342, - "å°±": 1343, - "åıĸ": 1344, - "åŃĺ": 1345, - "两": 1346, - "çŁ¥": 1347, - "ources": 1348, - "ck": 1349, - "å£": 1350, - "ines": 1351, - "è¾¾": 1352, - "Ġmany": 1353, - "æķ´": 1354, - "æł·": 1355, - "ditional": 1356, - "omm": 1357, - "çĶ±": 1358, - "éĢł": 1359, - "å®ĥ们": 1360, - "ues": 1361, - "Ġment": 1362, - "Ġimportant": 1363, - "Ġopt": 1364, - "Ġloc": 1365, - "ph": 1366, - "Ġprocess": 1367, - "Ġalgorithms": 1368, - "设计": 1369, - "Ġsocial": 1370, - "very": 1371, - "åĪĻ": 1372, - "ä¾ĭå¦Ĥ": 1373, - "认": 1374, - "Ġaut": 1375, - "Ġserv": 1376, - "gg": 1377, - "产åĵģ": 1378, - "è§Ħ": 1379, - "çľĭ": 1380, - "vel": 1381, - "æĸ¹æ³ķ": 1382, - "Ġben": 1383, - "åĽłæѤ": 1384, - "care": 1385, - "per": 1386, - "åĬŁ": 1387, - "建议": 1388, - "Ġpos": 1389, - "æ¤": 1390, - "we": 1391, - "åĮº": 1392, - "iqu": 1393, - "Ġreal": 1394, - "æĹ¥": 1395, - "Ġreduce": 1396, - "af": 1397, - "angu": 1398, - "Ġsk": 1399, - "Ġed": 1400, - "erstand": 1401, - "åĨµ": 1402, - "mot": 1403, - "åħĪ": 1404, - "ç¥": 1405, - "åºĶ该": 1406, - "Ġthrough": 1407, - "Ġconc": 1408, - "åıijå±ķ": 1409, - "è¯ķ": 1410, - "æ¡Ī": 1411, - "Ġenvironment": 1412, - "åı£": 1413, - "Ġadv": 1414, - "åĪ«": 1415, - "Ġbenef": 1416, - "æ¸ħ": 1417, - "åij³": 1418, - "åħī": 1419, - "Ġdevelopment": 1420, - "eng": 1421, - "å¦Ĥä½ķ": 1422, - "管": 1423, - "ivers": 1424, - "åIJĦ": 1425, - "Ġris": 1426, - "row": 1427, - "ergy": 1428, - "计ç®Ĺ": 1429, - "ä¿¡æģ¯": 1430, - "Ġproduct": 1431, - "è¾ĥ": 1432, - "论": 1433, - "èĩªå·±çļĦ": 1434, - "æĬ¤": 1435, - "åıį": 1436, - "åħ¶ä»ĸ": 1437, - "åĪĹ": 1438, - "ç»Ĩ": 1439, - "空": 1440, - "Ġgreat": 1441, - "ear": 1442, - "æºIJ": 1443, - "ject": 1444, - "çĶŁæ´»": 1445, - "ä¸ŃçļĦ": 1446, - "Ġunderstand": 1447, - "èĭ": 1448, - "hat": 1449, - "Ġprogra": 1450, - "çĬ": 1451, - "éĩij": 1452, - "Ġincluding": 1453, - "Ġaccess": 1454, - "ĠĠĠĠĠĠĠ": 1455, - "è¯Ĩ": 1456, - "ç¦": 1457, - "og": 1458, - "è£ħ": 1459, - "Ġart": 1460, - "Ġwrit": 1461, - "Ġincre": 1462, - "Ġph": 1463, - "æĸ¹éĿ¢": 1464, - "Ġpract": 1465, - "Ġusing": 1466, - "项": 1467, - "æİ¥": 1468, - "Ġways": 1469, - "Ġlangu": 1470, - "æĶ¯": 1471, - "Ġchall": 1472, - "åİ»": 1473, - "____": 1474, - "imate": 1475, - "æĸŃ": 1476, - "è¨": 1477, - "Ġwell": 1478, - "ll": 1479, - "Ġpol": 1480, - "æĢģ": 1481, - "Ġra": 1482, - "Can": 1483, - "åİŁ": 1484, - "ber": 1485, - "è¨Ģ": 1486, - "ç«ĭ": 1487, - "Ġgen": 1488, - "éħį": 1489, - "æ·±": 1490, - "te": 1491, - "ä¸ī": 1492, - "ç§ij": 1493, - "ĠFor": 1494, - "线": 1495, - "çħ": 1496, - "æ¼": 1497, - "åķĨ": 1498, - "æĿIJ": 1499, - "Ġsignific": 1500, - "Ġgu": 1501, - "Ġdecis": 1502, - "Ġtrain": 1503, - "Ġag": 1504, - "Ġcreat": 1505, - "å®Į": 1506, - "æŶéĹ´": 1507, - "Ġone": 1508, - "èĦ": 1509, - "Ġnat": 1510, - "åŃ¦ä¹ł": 1511, - "çļĦæķ": 1512, - "ced": 1513, - "Ġwhen": 1514, - "Ġbi": 1515, - "èİ": 1516, - "æĽ´åĬł": 1517, - "ives": 1518, - "port": 1519, - "å·¥ä½ľ": 1520, - "ving": 1521, - "Ġbeen": 1522, - "æĻº": 1523, - "Ġlife": 1524, - "å¼ķ": 1525, - "arm": 1526, - "çİĩ": 1527, - "çĶ¨æĪ·": 1528, - "ä¹ī": 1529, - "份": 1530, - "è¯Ŀ": 1531, - "iness": 1532, - "com": 1533, - "康": 1534, - "åĩı": 1535, - "ä»Ģ": 1536, - "è¾ĵ": 1537, - "Ġvari": 1538, - "con": 1539, - "Ġmod": 1540, - "ä»Ģä¹Ī": 1541, - "Ġenergy": 1542, - "æĬĢæľ¯": 1543, - "ertain": 1544, - "mm": 1545, - "verall": 1546, - "åĪĴ": 1547, - "Ġrobots": 1548, - "Ġorgan": 1549, - "æݨ": 1550, - "ants": 1551, - "åĩĨ": 1552, - "ds": 1553, - "æŀģ": 1554, - "çĻ": 1555, - "Ġrequ": 1556, - "Ġess": 1557, - "ç®Ģ": 1558, - "ustain": 1559, - "æ¨": 1560, - "Ġstr": 1561, - "cing": 1562, - "ability": 1563, - "ree": 1564, - "Ġeduc": 1565, - "åİĨ": 1566, - "Ġcreate": 1567, - "åģ¥åº·": 1568, - "Ġdesign": 1569, - "ips": 1570, - "åģļ": 1571, - "èĬ±": 1572, - "ink": 1573, - "èıľ": 1574, - "æī¾": 1575, - "段": 1576, - "æµĭ": 1577, - "ĠV": 1578, - "ĠBy": 1579, - "åĶ": 1580, - "é¦ĸ": 1581, - "è¯į": 1582, - "Ġwhere": 1583, - "Ġdisc": 1584, - "äºĨ解": 1585, - "ric": 1586, - "ä¸Ķ": 1587, - "足": 1588, - "æĺ¯ä¸Ģ个": 1589, - "arch": 1590, - "积": 1591, - "带": 1592, - "Ġwhile": 1593, - "Ġsignificant": 1594, - "çłģ": 1595, - "æĪ¿": 1596, - "Ġbeing": 1597, - "Ġlanguage": 1598, - "itive": 1599, - "20": 1600, - "Ġanalyze": 1601, - "æĻ¯": 1602, - "èĮ": 1603, - "rib": 1604, - "模": 1605, - "ĠSt": 1606, - "è´¹": 1607, - "'t": 1608, - "Ġhealthcare": 1609, - "Ġexperience": 1610, - "Ġ5": 1611, - "个人": 1612, - "ays": 1613, - "象": 1614, - "plo": 1615, - "Ġwould": 1616, - "èĻij": 1617, - "æĶ¶": 1618, - "é¢Ħ": 1619, - "é¢Ĩ": 1620, - "ä¿ĿæĮģ": 1621, - "ences": 1622, - "åıª": 1623, - "èĩ´": 1624, - "æĪı": 1625, - "Ġmental": 1626, - "Ġfew": 1627, - "ates": 1628, - "è¿ĩç¨ĭ": 1629, - "å®īåħ¨": 1630, - "Ġsustain": 1631, - "Ġwere": 1632, - "太": 1633, - "çĮ": 1634, - "Ġspecific": 1635, - "Ġworld": 1636, - "çŃĶ": 1637, - "```": 1638, - "Ġtake": 1639, - "åħ»": 1640, - "éĢŁ": 1641, - "ever": 1642, - "SS": 1643, - "éĶĢ": 1644, - "Ġbo": 1645, - "hes": 1646, - "Ġmus": 1647, - "æľįåĬ¡": 1648, - "è§Ĵ": 1649, - "ten": 1650, - "æŀIJ": 1651, - "pow": 1652, - "dict": 1653, - "vent": 1654, - "10": 1655, - "çļĦæĹ": 1656, - "ĸçķ": 1657, - "Ġprot": 1658, - "ç½®": 1659, - "Ġhigh": 1660, - "Ġbus": 1661, - "Ġindust": 1662, - "åIJ¦": 1663, - "cial": 1664, - "人们": 1665, - "ĠAs": 1666, - "åijĬ": 1667, - "ade": 1668, - "æĶ¹": 1669, - "çĹ": 1670, - "Ġhad": 1671, - "Ġher": 1672, - "Ġjust": 1673, - "ï¼Ľ": 1674, - "è´Ń": 1675, - "第": 1676, - "éĵ": 1677, - "Ġwater": 1678, - "Ġfood": 1679, - "éĺŁ": 1680, - "aus": 1681, - "Ġchalleng": 1682, - "åħį": 1683, - "æĸĩåĮĸ": 1684, - "Ġmost": 1685, - "é¸": 1686, - "ç½ij": 1687, - "缴": 1688, - "Ġsm": 1689, - "Ġactiv": 1690, - "ploy": 1691, - "Overall": 1692, - "å¿«": 1693, - "ruct": 1694, - "Ġindividuals": 1695, - "å§ĭ": 1696, - "gies": 1697, - "æŁ¥": 1698, - "çĪ±": 1699, - "iety": 1700, - "In": 1701, - "åĪĨæŀIJ": 1702, - "è§Ĩ": 1703, - "温": 1704, - "ç»´": 1705, - "olut": 1706, - "åŁŁ": 1707, - "ommend": 1708, - "Ġcomple": 1709, - "æķĻ": 1710, - "Ġbu": 1711, - "Ġeducation": 1712, - "ather": 1713, - "Ġ4": 1714, - "ting": 1715, - "Ġfind": 1716, - "没": 1717, - "Ġhis": 1718, - "ä¹ĭéĹ´": 1719, - "Ġeffective": 1720, - "Ġatt": 1721, - "Ġrese": 1722, - "èĥ½åĬĽ": 1723, - "åŁİ": 1724, - "Ġallow": 1725, - "Ġav": 1726, - "Ġpromot": 1727, - "æĻºèĥ½": 1728, - "满": 1729, - "åħ±": 1730, - "iew": 1731, - "come": 1732, - "ç³»ç»Ł": 1733, - "Ġrespons": 1734, - "äºĴ": 1735, - "Ġcult": 1736, - "powered": 1737, - "Ġrecommend": 1738, - "èIJ¥": 1739, - "OSS": 1740, - "Ġchange": 1741, - "è¯ģ": 1742, - "ved": 1743, - "æİĴ": 1744, - "解åĨ³": 1745, - "ici": 1746, - "ĠHow": 1747, - "Ġfeel": 1748, - "æľĪ": 1749, - "Ġwhat": 1750, - "以åıĬ": 1751, - "Ġsee": 1752, - "åŃ©": 1753, - "bs": 1754, - "Ġsur": 1755, - "æ£": 1756, - "ality": 1757, - "Ġvis": 1758, - "ç¡®ä¿Ŀ": 1759, - "pect": 1760, - "å®ŀçİ°": 1761, - "Ġcare": 1762, - "广": 1763, - "ills": 1764, - "åºŃ": 1765, - "ases": 1766, - "å¤į": 1767, - "åºĶçĶ¨": 1768, - "çļĦæĥ": 1769, - "ards": 1770, - "Ġaddress": 1771, - "Ġcompan": 1772, - "Ġinvol": 1773, - "Ġcustomer": 1774, - "åĽłä¸º": 1775, - "Ġstudents": 1776, - "Ġins": 1777, - "注æĦı": 1778, - "æŀĦ": 1779, - "欢": 1780, - "æµ·": 1781, - "åıĤ": 1782, - "èĩªçĦ¶": 1783, - "é©": 1784, - "ĠThese": 1785, - "wn": 1786, - "æĺĵ": 1787, - "çĬ¶": 1788, - "ren": 1789, - "Ġtreat": 1790, - "Ġbenefits": 1791, - "ĊĠĠĠĠĠĠĠ": 1792, - "对äºİ": 1793, - "æĢĿ": 1794, - "ider": 1795, - "ĠYes": 1796, - "ĠK": 1797, - "åĸľ": 1798, - "Ġke": 1799, - "Ġeng": 1800, - "Ġpop": 1801, - "ost": 1802, - "pare": 1803, - "Ġmon": 1804, - "款": 1805, - "ĠMOSS": 1806, - "Ġemot": 1807, - "Ġac": 1808, - "ç¼ĸ": 1809, - "fore": 1810, - "åı¥": 1811, - "Ġval": 1812, - "ily": 1813, - "Ġiss": 1814, - "èĤī": 1815, - "èĩ³": 1816, - "游æĪı": 1817, - "ween": 1818, - "Ġinclude": 1819, - "Ġprotect": 1820, - "åħ³ç³»": 1821, - "éĻ©": 1822, - "Ġsever": 1823, - "Ġthan": 1824, - "éľĢæ±Ĥ": 1825, - "ç»ĥ": 1826, - "ĠThey": 1827, - "iss": 1828, - "ys": 1829, - "Ġjob": 1830, - "éĺ³": 1831, - "æIJ": 1832, - "Ġbetween": 1833, - "Ġmach": 1834, - "--------": 1835, - "èĢĥèĻij": 1836, - "è´¨éĩı": 1837, - "Ġbusiness": 1838, - "wor": 1839, - "ick": 1840, - "eg": 1841, - "åħħ": 1842, - "ç¯": 1843, - "æĿ¡": 1844, - "ner": 1845, - "apt": 1846, - "Ġappro": 1847, - "Ġplay": 1848, - "没æľī": 1849, - "¤IJ": 1850, - "æľª": 1851, - "æĪĺ": 1852, - "家åºŃ": 1853, - "ãĢĭ": 1854, - "ency": 1855, - "ĠCh": 1856, - "ãĢĬ": 1857, - "Ġproviding": 1858, - "Ġresources": 1859, - "âĢĻ": 1860, - "Ġassist": 1861, - "Ġnatural": 1862, - "è¯Ħ": 1863, - "便": 1864, - "Ġsaf": 1865, - "åħ·æľī": 1866, - "è°¢": 1867, - "çĥŃ": 1868, - "ss": 1869, - "eth": 1870, - "old": 1871, - "Ġperform": 1872, - "Ġseveral": 1873, - "é¤IJ": 1874, - "Ġeach": 1875, - "转": 1876, - "ci": 1877, - "Ġty": 1878, - "Ġpub": 1879, - "æ´»åĬ¨": 1880, - "ocus": 1881, - "çīĮ": 1882, - "è¶Ĭ": 1883, - "åĽ¢": 1884, - "è½»": 1885, - "è¯Ńè¨Ģ": 1886, - "Ġareas": 1887, - "éĩĩ": 1888, - "ft": 1889, - "riend": 1890, - "å·²": 1891, - "å¸Ĥåľº": 1892, - "ition": 1893, - "ients": 1894, - "管çIJĨ": 1895, - "许": 1896, - "人类": 1897, - "身ä½ĵ": 1898, - "ique": 1899, - "Ġpartic": 1900, - "ç»Ń": 1901, - "agement": 1902, - "ves": 1903, - "符": 1904, - "line": 1905, - "红": 1906, - "åIJ¸": 1907, - "Ġpatter": 1908, - "000": 1909, - "社ä¼ļ": 1910, - "åĨħ容": 1911, - "Ġorganiz": 1912, - "ough": 1913, - "Ġve": 1914, - "åŃ©åŃIJ": 1915, - "æĸ½": 1916, - "æ¤į": 1917, - "åĩł": 1918, - "ä½Ĩæĺ¯": 1919, - "Ġaff": 1920, - "Ġnum": 1921, - "lement": 1922, - "èīº": 1923, - "èij": 1924, - "Ġcar": 1925, - "ages": 1926, - "abor": 1927, - "æĺ¯ä¸Ģç§į": 1928, - "Ġinst": 1929, - "èĽ": 1930, - "ä¹ĭä¸Ģ": 1931, - "è·¯": 1932, - "åį³": 1933, - "Ġmain": 1934, - "éļı": 1935, - "How": 1936, - "å¿ħ": 1937, - "ç¨ĭåºı": 1938, - "éŁ³ä¹IJ": 1939, - "red": 1940, - "æ²¹": 1941, - "Ġoffer": 1942, - "ets": 1943, - "ç¢": 1944, - "Ġduring": 1945, - "çļĦ人": 1946, - "æĽ´å¤ļ": 1947, - "Ġdi": 1948, - "代çłģ": 1949, - "èİ·": 1950, - "åħĭ": 1951, - "Ġguid": 1952, - "主è¦ģ": 1953, - "Ġfam": 1954, - "æݧ": 1955, - "éĢļ常": 1956, - "ĠAd": 1957, - "å¤ĦçIJĨ": 1958, - "urn": 1959, - "ower": 1960, - "åij½": 1961, - "æıı": 1962, - "Ġskills": 1963, - "Ġtool": 1964, - "ware": 1965, - "æĸĩæľ¬": 1966, - "Ġpatterns": 1967, - "缮æłĩ": 1968, - "acy": 1969, - "æīĵ": 1970, - "åŁİå¸Ĥ": 1971, - "Ġevery": 1972, - "ries": 1973, - "读": 1974, - "éģ¿": 1975, - "çĻ½": 1976, - "éĢĤåIJĪ": 1977, - "Ġpatient": 1978, - "羣": 1979, - "oth": 1980, - "她": 1981, - "åĶ®": 1982, - "ä¸Ģç§į": 1983, - "Ġmade": 1984, - "ä½İ": 1985, - "ise": 1986, - "Ġrem": 1987, - "æ¶Ī": 1988, - "åIJ«": 1989, - "air": 1990, - "Ġgener": 1991, - "oy": 1992, - "ç²¾": 1993, - "æĥħåĨµ": 1994, - "ights": 1995, - "Ġexpl": 1996, - "è§ģ": 1997, - "Ġpredict": 1998, - "ç±³": 1999, - "æĽ´å¥½": 2000, - "ä¿®": 2001, - "Ġclimate": 2002, - "Ġfocus": 2003, - "Ġgrow": 2004, - "客æĪ·": 2005, - "ä¸įæĸŃ": 2006, - "itor": 2007, - "ĠEn": 2008, - "约": 2009, - "æĺ¯åIJ¦": 2010, - "ä»ħ": 2011, - "æĪij们çļĦ": 2012, - "æľĽ": 2013, - "op": 2014, - "Ġmaking": 2015, - "yth": 2016, - "ccess": 2017, - "Ġown": 2018, - "ggest": 2019, - "Ġtas": 2020, - "uture": 2021, - "Ġmodel": 2022, - "put": 2023, - "Ġresearch": 2024, - "erest": 2025, - "éļ¾": 2026, - "Ġ[": 2027, - "iel": 2028, - "ational": 2029, - "Ġcommunic": 2030, - "ç¥ŀ": 2031, - "究": 2032, - "Ġrest": 2033, - "æĪIJ为": 2034, - "king": 2035, - "pr": 2036, - "åĮ»": 2037, - "cur": 2038, - "èĤ²": 2039, - "Ġ'": 2040, - "è¿Ļç§į": 2041, - "ç¯ĩ": 2042, - "Ġche": 2043, - "own": 2044, - "éĻħ": 2045, - "Ġfin": 2046, - "åĪ¶ä½ľ": 2047, - "Ġsuggest": 2048, - "å¢ŀåĬł": 2049, - "Ġmedia": 2050, - "ribut": 2051, - "çļĦæĥħ": 2052, - "åĬłåħ¥": 2053, - "Ġcle": 2054, - "åij¨": 2055, - "ç«ł": 2056, - "Ġthink": 2057, - "Ġlocal": 2058, - "pportun": 2059, - "ĠYou": 2060, - "Ġplan": 2061, - "Ġeven": 2062, - "éĽĨ": 2063, - "å·§": 2064, - "ax": 2065, - "Ġchallenges": 2066, - "Ġprof": 2067, - "ĠCan": 2068, - "Ġconcer": 2069, - "Ġfuture": 2070, - "åĬ¿": 2071, - "Ġref": 2072, - "èģĶ": 2073, - "Ġself": 2074, - "æĪĸèĢħ": 2075, - "ble": 2076, - "åĽ´": 2077, - "è¿IJåĬ¨": 2078, - "Ġinf": 2079, - "éĩĬ": 2080, - "Ġsustainable": 2081, - "Ġtext": 2082, - "Ġgra": 2083, - "äºĮ": 2084, - "åĵģçīĮ": 2085, - "ä¸įåIJĮçļĦ": 2086, - "led": 2087, - "çĭ¬": 2088, - "Ġopportun": 2089, - "Ġcontin": 2090, - "ym": 2091, - "Ġget": 2092, - "å¯Ĩ": 2093, - "éĻ¤": 2094, - "æħ": 2095, - "éģ¿åħį": 2096, - "Ġ+": 2097, - "è§ī": 2098, - "Ġret": 2099, - "å¸ĥ": 2100, - "Ġinterest": 2101, - "Ġsociety": 2102, - "ç»ĵæŀľ": 2103, - "åIJ¬": 2104, - "é¦ĸåħĪ": 2105, - "Ġbre": 2106, - "Ġ20": 2107, - "ĠHowever": 2108, - "è®°": 2109, - "ons": 2110, - "è¿ij": 2111, - "å¼Ģå§ĭ": 2112, - "Ġbuild": 2113, - "Ġbeh": 2114, - "'m": 2115, - "vers": 2116, - "Ġgood": 2117, - "çIJĨ解": 2118, - "resent": 2119, - "离": 2120, - "åĬŁèĥ½": 2121, - "Ġeffort": 2122, - "labor": 2123, - "é»ij": 2124, - "Ġbetter": 2125, - "Ġread": 2126, - "å¾ĭ": 2127, - "èĽĭ": 2128, - "hed": 2129, - "ä¹°": 2130, - "导èĩ´": 2131, - "Ġimplement": 2132, - "ç¿": 2133, - "享": 2134, - "头": 2135, - "ense": 2136, - "Ġlong": 2137, - "other": 2138, - "饮": 2139, - "åŃĺåľ¨": 2140, - "çļĦæĦ": 2141, - "ä¸Ģ份": 2142, - "ython": 2143, - "ning": 2144, - "åĩıå°ij": 2145, - "åĢĻ": 2146, - "ä¸ĵ": 2147, - "åIJĦç§į": 2148, - "èħ": 2149, - "å°½": 2150, - "åįĩ": 2151, - "æĬ¥": 2152, - "Ġpublic": 2153, - "Ġlar": 2154, - "ä½łçļĦ": 2155, - "aut": 2156, - "é¢ĨåŁŁ": 2157, - "æļ": 2158, - "ollow": 2159, - "èģĮ": 2160, - "Ġchang": 2161, - "Ġbest": 2162, - "hip": 2163, - "åĨį": 2164, - "akes": 2165, - "Ġchat": 2166, - "ited": 2167, - "Ġpower": 2168, - "ä¿ĿæĬ¤": 2169, - "书": 2170, - "计åĪĴ": 2171, - "éĩįè¦ģçļĦ": 2172, - "åıĺåĮĸ": 2173, - "ilities": 2174, - "Ġconsider": 2175, - "æĪij们åı¯ä»¥": 2176, - "éĤ£ä¹Ī": 2177, - "Ġide": 2178, - "æ¼Ķ": 2179, - "aging": 2180, - "Ġbased": 2181, - "å®Ŀ": 2182, - "Ġrange": 2183, - "Ġresult": 2184, - "Ġmem": 2185, - "çħ§": 2186, - "Ġlevel": 2187, - "cou": 2188, - "Ġbr": 2189, - "Th": 2190, - "ä¼ģ": 2191, - "建ç«ĭ": 2192, - "Ġunique": 2193, - "è®Ń": 2194, - "Ġmark": 2195, - "许å¤ļ": 2196, - "è¡Į为": 2197, - "Ķ究": 2198, - "çļĦæĬ": 2199, - "Ġset": 2200, - "骤": 2201, - "ts": 2202, - "Ġhist": 2203, - "Ġaround": 2204, - "Ġrev": 2205, - "åħ¶ä¸Ń": 2206, - "ï¼ģ": 2207, - "æııè¿°": 2208, - "æľĢåIJİ": 2209, - "Ġsim": 2210, - "nect": 2211, - "åĽŀçŃĶ": 2212, - "éĺ²": 2213, - "èī¯": 2214, - "åĪ°äºĨ": 2215, - "ä¸ĸçķ": 2216, - "æĸ¹æ¡Ī": 2217, - "æĿIJæĸĻ": 2218, - "ä¸ĸçķĮ": 2219, - "æĽ´å¥½åľ°": 2220, - "两个": 2221, - "Ġemploy": 2222, - "Ġtry": 2223, - "æĵ": 2224, - "Ġback": 2225, - "åĪĩ": 2226, - "Ġsuccess": 2227, - "Ġdecisions": 2228, - "Ġthose": 2229, - "å¯Į": 2230, - "Ġfact": 2231, - "æİ¢": 2232, - "趣": 2233, - "Ġpractices": 2234, - "åIJĹ": 2235, - "æīį": 2236, - "çİ©": 2237, - "ption": 2238, - "æĸĩç«ł": 2239, - "Ġfeat": 2240, - "Ġprevent": 2241, - "Ġwriting": 2242, - "çļĦæĢ": 2243, - "Ġno": 2244, - "ä»ĭ": 2245, - "éŨ": 2246, - "Ġdel": 2247, - "æĴ": 2248, - "Ġoptim": 2249, - "ination": 2250, - "ĠĊ": 2251, - "usion": 2252, - "Ġaccount": 2253, - "ling": 2254, - "Ġdivers": 2255, - ".\"": 2256, - "ath": 2257, - "èĭ±": 2258, - "ä¼ģä¸ļ": 2259, - "Ġgrou": 2260, - "åľ°çIJĥ": 2261, - "失": 2262, - "Ġpersonalized": 2263, - "ĠHe": 2264, - "表达": 2265, - "curity": 2266, - "Ġfollow": 2267, - "产çĶŁ": 2268, - "Ġear": 2269, - "åİĭ": 2270, - "vern": 2271, - "Ġissues": 2272, - "åĿĩ": 2273, - "é²": 2274, - "Ġdr": 2275, - "iving": 2276, - "Ġtraining": 2277, - "Ġrisk": 2278, - "åĩ½": 2279, - "åı²": 2280, - "æij": 2281, - "çļĦæŶ": 2282, - "ogn": 2283, - "Ġrequire": 2284, - "Ġenvironmental": 2285, - "back": 2286, - "éĶ®": 2287, - "çĸĹ": 2288, - "Ġinteract": 2289, - "åĽ¢éĺŁ": 2290, - "æ¯ı个": 2291, - "çĦ¶åIJİ": 2292, - "Ġdist": 2293, - "çĶ¨äºİ": 2294, - "认为": 2295, - "åĩ½æķ°": 2296, - "Ġsent": 2297, - "ĊĠĠĠĠĠĠĠĠ": 2298, - "Ġreducing": 2299, - "å¹²": 2300, - "Ġrep": 2301, - "Ġcaus": 2302, - "Ġmusic": 2303, - "çª": 2304, - "Ġmonitor": 2305, - "Ġform": 2306, - "é¢ľ": 2307, - "çĹħ": 2308, - "é¦Ļ": 2309, - "Ġoften": 2310, - "åı¯èĥ½ä¼ļ": 2311, - "åijĺå·¥": 2312, - "Ġhand": 2313, - "æĬķ": 2314, - "Ġneeds": 2315, - "æѤå¤ĸ": 2316, - "åıĭ": 2317, - "ivity": 2318, - "Ġactivities": 2319, - "åĸľæ¬¢": 2320, - "Ġpur": 2321, - "ian": 2322, - "self": 2323, - "åĬ¨çī©": 2324, - "comes": 2325, - "å©": 2326, - "Ġpriv": 2327, - "az": 2328, - "Ġrelations": 2329, - "Ġmachine": 2330, - "çļĦæ°": 2331, - "ä»·æł¼": 2332, - "ä»·åĢ¼": 2333, - "ç´¢": 2334, - "Ġfeed": 2335, - "ä¸Ģä¸ĭ": 2336, - "Ġteam": 2337, - "Ġindustry": 2338, - "è´¢": 2339, - "ĠPro": 2340, - "Ġwant": 2341, - "称": 2342, - "Ġclass": 2343, - "Ġlove": 2344, - "åħ³äºİ": 2345, - "è¾ĵåħ¥": 2346, - "Ġtransport": 2347, - "Ġcomplex": 2348, - "Ġyear": 2349, - "éĶĢåĶ®": 2350, - "寻": 2351, - "ience": 2352, - "ists": 2353, - "æĶ¯æĮģ": 2354, - "Ġmind": 2355, - "Ġfun": 2356, - "Ġchar": 2357, - "æĮī": 2358, - "Ġconcerns": 2359, - "conom": 2360, - "ç®Ģåįķ": 2361, - "以ä¸ĭæĺ¯": 2362, - "Ġstart": 2363, - "并ä¸Ķ": 2364, - "avi": 2365, - "ä¸ŃåĽ½": 2366, - "åħĥç´ł": 2367, - "Ġconf": 2368, - "Ġpositive": 2369, - "Ġcur": 2370, - "Ġcount": 2371, - "ery": 2372, - "å¡": 2373, - "室": 2374, - "Ġcost": 2375, - "Ġequ": 2376, - "Ġpolic": 2377, - "aste": 2378, - "aw": 2379, - "éħĴ": 2380, - "coura": 2381, - "iven": 2382, - "place": 2383, - "chie": 2384, - "çļĦæķ°": 2385, - "åĽłç´ł": 2386, - "Ġfl": 2387, - "ism": 2388, - "Ġmedical": 2389, - "Ġhumans": 2390, - "Ġautom": 2391, - "ertainly": 2392, - "Ġ0": 2393, - "Ġoffers": 2394, - "Ġdetect": 2395, - "Ġ6": 2396, - "é£İæł¼": 2397, - "Ġshow": 2398, - "çģ«": 2399, - "Ġanim": 2400, - "é¢ľèī²": 2401, - "lease": 2402, - "ave": 2403, - "åĵª": 2404, - "ĠThere": 2405, - "以ä¸Ĭ": 2406, - "æľªæĿ¥": 2407, - "XX": 2408, - "çīĩ": 2409, - "uch": 2410, - "Ġtasks": 2411, - "åħ·ä½ĵ": 2412, - "æ¤įçī©": 2413, - "Ġmin": 2414, - "èīºæľ¯": 2415, - "icult": 2416, - "Ġexperiences": 2417, - "æݧåĪ¶": 2418, - "be": 2419, - "Ġpatients": 2420, - "å²": 2421, - "ĠWe": 2422, - "Ġrecogn": 2423, - "çĥ¤": 2424, - "Ġsmall": 2425, - "åĿĹ": 2426, - "åĦ": 2427, - "太éĺ³": 2428, - "ction": 2429, - "Ġent": 2430, - "æį¢": 2431, - "Ġbefore": 2432, - "Ġbecome": 2433, - "å·²ç»ı": 2434, - "表çİ°": 2435, - "Ġexplo": 2436, - "Ġachie": 2437, - "ä»»åĬ¡": 2438, - "大çļĦ": 2439, - "Ġday": 2440, - "Ġfound": 2441, - "å±±": 2442, - "ond": 2443, - "Ġtreatment": 2444, - "pend": 2445, - "hen": 2446, - "Ġcondit": 2447, - "ç¡®å®ļ": 2448, - "Ġbusinesses": 2449, - "ĠWh": 2450, - "æīĢæľī": 2451, - "Ġdeveloped": 2452, - "ç»Ī": 2453, - "æŃ¥éª¤": 2454, - "Ġdifficult": 2455, - "åı·": 2456, - "ĠRe": 2457, - "éĶĻ": 2458, - "Ġcho": 2459, - "Ġquest": 2460, - "Ġtranspare": 2461, - "Ġproject": 2462, - "Ġcommunity": 2463, - "ov": 2464, - "å¸Ī": 2465, - "å¼ł": 2466, - "åĪĨç±»": 2467, - "人çļĦ": 2468, - "sis": 2469, - "çĽĬ": 2470, - "oid": 2471, - "ĠAn": 2472, - "ways": 2473, - "Ġeas": 2474, - "Ġaffect": 2475, - "Ġothers": 2476, - "Ġregul": 2477, - "æĢ§åĴĮ": 2478, - "åĸĦ": 2479, - "agn": 2480, - "ä½ľä¸º": 2481, - "åı¯ä»¥å¸®åĬ©": 2482, - "åĦ¿": 2483, - "Ġorganizations": 2484, - "鸡": 2485, - "åħ´": 2486, - "Ġfriend": 2487, - "Ġ$": 2488, - "Ġdetail": 2489, - "Ġtraditional": 2490, - "Ġdesigned": 2491, - "è´Ńä¹°": 2492, - "ä½ĵéªĮ": 2493, - "ç»į": 2494, - "erm": 2495, - "Ġconnect": 2496, - "è¿Ļæł·": 2497, - "Ġrecommendations": 2498, - "Ġboth": 2499, - "ŁéĢļ": 2500, - "æ¯į": 2501, - "Ġsit": 2502, - "ä½ľçĶ¨": 2503, - "ä»ĭç»į": 2504, - "Ġste": 2505, - "ĠSure": 2506, - "åı°": 2507, - "æĤ¨çļĦ": 2508, - "Ġshe": 2509, - "Ġmanagement": 2510, - "joy": 2511, - "è´Ł": 2512, - "Ġpromote": 2513, - "Ġvarious": 2514, - "(\"": 2515, - "por": 2516, - "Ġsens": 2517, - "Ġessential": 2518, - "gether": 2519, - "ularly": 2520, - "äºī": 2521, - "irst": 2522, - "Ġop": 2523, - "Ġspecies": 2524, - "çİ°åľ¨": 2525, - "cho": 2526, - "Ġbehavi": 2527, - "çŃij": 2528, - "女": 2529, - "Ġquality": 2530, - "Ġext": 2531, - "è¥": 2532, - "å®ĮæĪIJ": 2533, - "æĢ»ä¹ĭ": 2534, - "éĥ¨åĪĨ": 2535, - "ä»İèĢĮ": 2536, - "åĽ¾": 2537, - "Ġtyp": 2538, - "Ġstrate": 2539, - "西": 2540, - "Ġhere": 2541, - "ars": 2542, - "å¸Į": 2543, - "çļĦæĿ": 2544, - "å°Ŀ": 2545, - "ee": 2546, - "ier": 2547, - "Ġec": 2548, - "ically": 2549, - "ering": 2550, - "念": 2551, - "ĠDe": 2552, - "Ġneg": 2553, - "建çŃij": 2554, - "Ġservices": 2555, - "Ġable": 2556, - "imes": 2557, - "Ġoptions": 2558, - "缸åħ³": 2559, - "Ġsub": 2560, - "Ġdecision": 2561, - "ĠCertainly": 2562, - "Ġåľ¨": 2563, - "æ¢": 2564, - "Ġservice": 2565, - "):": 2566, - "带æĿ¥": 2567, - "Ġchild": 2568, - "解éĩĬ": 2569, - "irt": 2570, - "çĨ": 2571, - "ä¸įä»ħ": 2572, - "æĿ¾": 2573, - "积æŀģ": 2574, - "ron": 2575, - "åı¤": 2576, - "çłĶ究": 2577, - "ç²ī": 2578, - "hor": 2579, - "Ġprofess": 2580, - "çļĦéĹ®é¢ĺ": 2581, - "Ġopportunities": 2582, - "åİĨåı²": 2583, - "Ġdef": 2584, - "ĠAm": 2585, - "Ġgr": 2586, - "aur": 2587, - "å±Ĥ": 2588, - "çŃĸ": 2589, - "Ġpopular": 2590, - "æ´ģ": 2591, - "åıijçİ°": 2592, - "Ġpoem": 2593, - "èµĽ": 2594, - "Ġob": 2595, - "Ġdon": 2596, - "Ġsound": 2597, - "Ġtransportation": 2598, - "ious": 2599, - "åı¦": 2600, - "Ġrole": 2601, - "Ġfiel": 2602, - "ç§ijåѦ": 2603, - "èĢģ": 2604, - "reen": 2605, - "æľīæķĪ": 2606, - "Ġcor": 2607, - "Ġfeedback": 2608, - "Ġtechnologies": 2609, - "交éĢļ": 2610, - "Ġadapt": 2611, - "'re": 2612, - "ervation": 2613, - "Ġcommunities": 2614, - "çݰ代": 2615, - "Ġlook": 2616, - "Ġfac": 2617, - "çĶµå½±": 2618, - "Ġcollect": 2619, - "å¾ĹåĪ°": 2620, - "hips": 2621, - "Ġavail": 2622, - "eren": 2623, - "ä¸Ģèµ·": 2624, - "çīĽ": 2625, - "Ġposs": 2626, - "Ġweather": 2627, - "Ġefforts": 2628, - "¿Ģ": 2629, - "æĹħ": 2630, - "oh": 2631, - "Ġcollabor": 2632, - "æĭ¥": 2633, - "æĪIJåĬŁ": 2634, - "èİ·å¾Ĺ": 2635, - "å±ħ": 2636, - "Ġtre": 2637, - "Ġsources": 2638, - "Ġstudy": 2639, - "Ġprograms": 2640, - "éĻIJ": 2641, - "Ġtips": 2642, - "Ġmarket": 2643, - "ally": 2644, - "害": 2645, - "wards": 2646, - "æ£Ģ": 2647, - "ä¸Ģç¯ĩ": 2648, - "rior": 2649, - "Ġtop": 2650, - "Ġend": 2651, - "åĭ": 2652, - "Ġlarge": 2653, - "iciency": 2654, - "Ġdec": 2655, - "å®ļçļĦ": 2656, - "icient": 2657, - "è¿ĩç¨ĭä¸Ń": 2658, - "lications": 2659, - "缺": 2660, - "Ġtour": 2661, - "Ġtogether": 2662, - "人工": 2663, - "Ġtools": 2664, - "æĸ¯": 2665, - "æ°ij": 2666, - "æĬĬ": 2667, - "ä¹ĭéĹ´çļĦ": 2668, - "çī¹çĤ¹": 2669, - "Ġbel": 2670, - "ditionally": 2671, - "åĪ©çĶ¨": 2672, - "è¾¹": 2673, - "éĻį": 2674, - "ĠIf": 2675, - "é¢Ŀ": 2676, - "åįı": 2677, - "å¾Ģ": 2678, - "lish": 2679, - "è¯ī": 2680, - "ins": 2681, - "奶": 2682, - "Ġeconom": 2683, - "Ġinvest": 2684, - "ĠDo": 2685, - "tain": 2686, - "åĩºçİ°": 2687, - "çļĦå½±åĵį": 2688, - "aterial": 2689, - "Ġsure": 2690, - "Ġpass": 2691, - "çĶ»": 2692, - "è´£": 2693, - "ç»ĵæŀĦ": 2694, - "æķħ": 2695, - "æĥħæĦŁ": 2696, - "æ¿Ģ": 2697, - "ellig": 2698, - "ä¼Ĺ": 2699, - "æ¯Ķè¾ĥ": 2700, - "tern": 2701, - "Ġoutcomes": 2702, - "up": 2703, - "Ġbeaut": 2704, - "read": 2705, - "çĶŁæĪIJ": 2706, - "æķ°åŃĹ": 2707, - "Ġdem": 2708, - "ires": 2709, - "åı¯ä»¥éĢļè¿ĩ": 2710, - "æĸ°çļĦ": 2711, - "Ġdeep": 2712, - "å¨": 2713, - "çĭĹ": 2714, - "åħ³æ³¨": 2715, - "çĶŁåij½": 2716, - "ä¼łç»Ł": 2717, - "Ġstay": 2718, - "æŃĮ": 2719, - "åħ³éĶ®": 2720, - "Ġplace": 2721, - "主é¢ĺ": 2722, - "å¾Īå¤ļ": 2723, - "èĪĴ": 2724, - "Ġprofessional": 2725, - "yle": 2726, - "æĽ²": 2727, - "19": 2728, - "Ġessay": 2729, - "Ġgive": 2730, - "ç³ĸ": 2731, - "Ġonly": 2732, - "æŁIJ": 2733, - "Ġphys": 2734, - "对è¯Ŀ": 2735, - "Ġcontro": 2736, - "Ġamount": 2737, - "cept": 2738, - "ization": 2739, - "ç¼ĸåĨĻ": 2740, - "åıĹåĪ°": 2741, - "Ġalways": 2742, - "æ¯Ķå¦Ĥ": 2743, - "Ġprivacy": 2744, - "au": 2745, - "________": 2746, - "Ġresponsible": 2747, - "()": 2748, - "çŃīçŃī": 2749, - "Ġmaterial": 2750, - "Ġonline": 2751, - "é¼": 2752, - "æĶ¿": 2753, - "åĽĽ": 2754, - "Ġenjoy": 2755, - "åľŁ": 2756, - "Ġsafety": 2757, - "Ġtw": 2758, - "Ġcommunication": 2759, - "丽": 2760, - "æĺ¾": 2761, - "olution": 2762, - "erg": 2763, - "įä½ľ": 2764, - "Ġuser": 2765, - "Ġemotional": 2766, - "time": 2767, - "é¾": 2768, - "Ġsecurity": 2769, - "Ġsense": 2770, - "elines": 2771, - "åĬ±": 2772, - "çī©è´¨": 2773, - "ura": 2774, - "Ġshare": 2775, - "Ġanalyzing": 2776, - "ital": 2777, - "é±": 2778, - "irtual": 2779, - "Ġvisit": 2780, - "bers": 2781, - "Ġcour": 2782, - "Ġproble": 2783, - "设å¤ĩ": 2784, - "atch": 2785, - "land": 2786, - "é±¼": 2787, - "æĪij们éľĢè¦ģ": 2788, - "稳": 2789, - "ibility": 2790, - "Ġefficiency": 2791, - "声": 2792, - "èĴ": 2793, - "æľºåĻ¨": 2794, - "Ġclear": 2795, - "åĪ¶å®ļ": 2796, - "izing": 2797, - "Ġconditions": 2798, - "lusion": 2799, - "Ġlow": 2800, - "Ġlim": 2801, - "hers": 2802, - "Ġrisks": 2803, - "ç¿»": 2804, - "Ġlet": 2805, - "åĴĸ": 2806, - "å¿ĥçIJĨ": 2807, - "è¿ľ": 2808, - "print": 2809, - "Ġchanges": 2810, - "Ġmeas": 2811, - "Ġimproving": 2812, - "Ġcrit": 2813, - "50": 2814, - "å¸ĮæľĽ": 2815, - "Ġaud": 2816, - "åįĹ": 2817, - "æĹłæ³ķ": 2818, - "Ġnegative": 2819, - "é¡¹çĽ®": 2820, - "und": 2821, - "ats": 2822, - "Ġcompanies": 2823, - "æī¾åĪ°": 2824, - "Ġcontribut": 2825, - "æŃ£ç¡®": 2826, - "é»Ħ": 2827, - "å±ŀ": 2828, - "Ġunderstanding": 2829, - "Ġmult": 2830, - "Ġclo": 2831, - "å¾ģ": 2832, - "Ġprior": 2833, - "rim": 2834, - "人工æĻºèĥ½": 2835, - "Ġvariety": 2836, - "Ġtaking": 2837, - "åĤ": 2838, - "aster": 2839, - "ody": 2840, - "Ġ{": 2841, - "çļĦéĩįè¦ģ": 2842, - "Ġfore": 2843, - "èµĦæºIJ": 2844, - "è¦ģæ±Ĥ": 2845, - "Ġfeatures": 2846, - "èįī": 2847, - "me": 2848, - "èĮĥ": 2849, - "Ġoper": 2850, - "级": 2851, - "é²ľ": 2852, - "æĬĢå·§": 2853, - "ijæĪĺ": 2854, - "ç±»åŀĭ": 2855, - "æĿ¿": 2856, - "软": 2857, - "ew": 2858, - "Ġrestaur": 2859, - "Ġwithout": 2860, - "ructure": 2861, - "çļĦæĺ¯": 2862, - "çı": 2863, - "Ġlist": 2864, - "urate": 2865, - "Ġbook": 2866, - "亲": 2867, - "åºĹ": 2868, - "ä¹Łæĺ¯": 2869, - "ä»»ä½ķ": 2870, - "Ġcam": 2871, - "ĠBe": 2872, - "Ġgovern": 2873, - "Ġbehavior": 2874, - "è®Ńç»ĥ": 2875, - "Ġfamily": 2876, - "æĿĤ": 2877, - "Ġcity": 2878, - "Ġapproach": 2879, - "Ġaccurate": 2880, - "Ġsom": 2881, - "Ġel": 2882, - "èĪŀ": 2883, - "èŀ": 2884, - "åŁºæľ¬": 2885, - "Ġdise": 2886, - "Ġencoura": 2887, - "ĠWhat": 2888, - "åĥ": 2889, - "详": 2890, - "¦Ĥ": 2891, - "å·¥åħ·": 2892, - "åķ¡": 2893, - "Ġstill": 2894, - "chool": 2895, - "æĦŁåĪ°": 2896, - "çĶŁçī©": 2897, - "åĴĸåķ¡": 2898, - "åĩĨå¤ĩ": 2899, - "Ġwaste": 2900, - "Ġevents": 2901, - "æķĻèĤ²": 2902, - "Ġ8": 2903, - "Ġmust": 2904, - "ied": 2905, - "asing": 2906, - "å½¢æĪIJ": 2907, - "Ġproducts": 2908, - "åħ¸": 2909, - "讲": 2910, - "fter": 2911, - "å·®": 2912, - "less": 2913, - "Ġcro": 2914, - "Ġfinan": 2915, - "åıįåºĶ": 2916, - "åĪĽéĢł": 2917, - "Ġguidelines": 2918, - "åĪ¤": 2919, - "ä½ľåĵģ": 2920, - "表示": 2921, - "å¼Ĥ": 2922, - "Ġknown": 2923, - "Ġtest": 2924, - "误": 2925, - "ope": 2926, - "Ġusers": 2927, - "AI": 2928, - "å¾·": 2929, - "new": 2930, - "追": 2931, - "iques": 2932, - "模åŀĭ": 2933, - "åĬĽåĴĮ": 2934, - "Ġhistory": 2935, - "ĠAl": 2936, - "æĬķèµĦ": 2937, - "å°Ŀè¯ķ": 2938, - "ank": 2939, - "Ġhome": 2940, - "éĴŁ": 2941, - "丰": 2942, - "èĪĴéĢĤ": 2943, - "Ġincrease": 2944, - "Ġhab": 2945, - "åĪ»": 2946, - "è¾ĵåĩº": 2947, - "Ġleading": 2948, - "Ġ7": 2949, - "é£İéĻ©": 2950, - "Ġperformance": 2951, - "Ġhapp": 2952, - "åŃ£": 2953, - "Ġstand": 2954, - "ty": 2955, - "ç¦ı": 2956, - "Ġcustomers": 2957, - "åįİ": 2958, - "Ġbelie": 2959, - "Ġcompany": 2960, - "å½ķ": 2961, - "é£Łçī©": 2962, - "ĠUn": 2963, - "Ġsumm": 2964, - "rent": 2965, - "ĠCon": 2966, - "éĢĤéĩı": 2967, - "anced": 2968, - "Ġi": 2969, - "Ġlight": 2970, - "Ġanalysis": 2971, - "å°Ĭ": 2972, - "ĠUse": 2973, - "ouse": 2974, - "ted": 2975, - "Ġcharact": 2976, - "Ġ#": 2977, - "to": 2978, - "绾": 2979, - "ä¸įæĺ¯": 2980, - "Ġdeveloping": 2981, - "åŁ¹": 2982, - "Ġstrategies": 2983, - "Ġmight": 2984, - "çŁŃ": 2985, - "çļĦæİ": 2986, - "Ġfirst": 2987, - "èĥĮ": 2988, - "çĮ«": 2989, - "Ġincludes": 2990, - "åĽŃ": 2991, - "Ġdiagn": 2992, - "Ġgrowth": 2993, - "ä¸ĵä¸ļ": 2994, - "Ġdoes": 2995, - "12": 2996, - "绿": 2997, - "Ġkeep": 2998, - "详ç»Ĩ": 2999, - "åĥı": 3000, - "åıijçĶŁ": 3001, - "fact": 3002, - "åı¯ä»¥åľ¨": 3003, - "ç«Ļ": 3004, - "æĭī": 3005, - "æµİ": 3006, - "Ġchatbots": 3007, - "Ġbreak": 3008, - "è¡¡": 3009, - "çŁ³": 3010, - "æĮģç»Ń": 3011, - "life": 3012, - "Ġ10": 3013, - "æ´Ĺ": 3014, - "ĠAdditionally": 3015, - "士": 3016, - "ember": 3017, - "Ġgoals": 3018, - "å¾®": 3019, - "Ġview": 3020, - "·": 3021, - "ove": 3022, - "åŁºç¡": 3023, - "Ġoptimize": 3024, - "Ġtem": 3025, - "Ġdown": 3026, - "åŁºç¡Ģ": 3027, - "è¶ħ": 3028, - "ercis": 3029, - "Ġless": 3030, - "ees": 3031, - "æĿĥ": 3032, - "Ġkey": 3033, - "Ġworks": 3034, - "讨": 3035, - "åı¥åŃIJ": 3036, - "Ġrobot": 3037, - "uss": 3038, - "åħ¨çIJĥ": 3039, - "ç»ıæµİ": 3040, - "æīįèĥ½": 3041, - "egr": 3042, - "ä»ĸ们çļĦ": 3043, - "äºĶ": 3044, - "èµ·æĿ¥": 3045, - "çĵ": 3046, - "Ġfactors": 3047, - "Ġcultural": 3048, - "æľ¨": 3049, - "Ġworking": 3050, - "ä¼¼": 3051, - "èIJ½": 3052, - "éĢŁåº¦": 3053, - "ä½ı": 3054, - "Ġeffects": 3055, - "å©ļ": 3056, - "br": 3057, - "åİħ": 3058, - "rain": 3059, - "\")": 3060, - "åѦçĶŁ": 3061, - "\",": 3062, - "Ġpar": 3063, - "atform": 3064, - "Ġensuring": 3065, - "çĶ±äºİ": 3066, - "Ġmuch": 3067, - "Ġwords": 3068, - "Ġmar": 3069, - "ç»ıéªĮ": 3070, - "为äºĨ": 3071, - "åIJĪä½ľ": 3072, - "ven": 3073, - "Ġ/": 3074, - "Ġfinancial": 3075, - "work": 3076, - "ories": 3077, - "æ²»": 3078, - "Ġtechniques": 3079, - "æĭ¥æľī": 3080, - "rap": 3081, - "å°Ķ": 3082, - "Ġest": 3083, - "Ġavailable": 3084, - "Ġlit": 3085, - "æ¹": 3086, - "Ġefficient": 3087, - "els": 3088, - "over": 3089, - "Ġland": 3090, - "Ġarea": 3091, - "Ġintellig": 3092, - "Ġpref": 3093, - "ature": 3094, - "çŁ¥è¯Ĩ": 3095, - "æĵįä½ľ": 3096, - "å¾ħ": 3097, - "igate": 3098, - "çļĦæĶ": 3099, - "Ġmean": 3100, - "bo": 3101, - "Ġcontrol": 3102, - "éĩĩçĶ¨": 3103, - "ricult": 3104, - "Ġprogramm": 3105, - "Ġtowards": 3106, - "thing": 3107, - "ä¸įè¦ģ": 3108, - "Ġthough": 3109, - "彩": 3110, - "Ġcertain": 3111, - "Ġwild": 3112, - "ä»Ĭ": 3113, - "Ġconservation": 3114, - "çŁ¥éģĵ": 3115, - "Ġreally": 3116, - "çļĦåľ°": 3117, - "io": 3118, - "饰": 3119, - "Ġful": 3120, - "çݯä¿Ŀ": 3121, - "Ġexplore": 3122, - "çļĦæ¸": 3123, - "Ġdiverse": 3124, - "åĬłå¼º": 3125, - "çļ®": 3126, - "Ġemotions": 3127, - "Ġavoid": 3128, - "'ll": 3129, - "çļĦæī": 3130, - "åį¡": 3131, - "Ġplatform": 3132, - "ances": 3133, - "Ġsitu": 3134, - "ä»ĺ": 3135, - "ä½įç½®": 3136, - "oring": 3137, - "çĽIJ": 3138, - "ä¸ĩ": 3139, - "Ġdev": 3140, - "nov": 3141, - "ash": 3142, - "Ġtwo": 3143, - "å®ł": 3144, - "bon": 3145, - "èµ°": 3146, - "åĪĹ表": 3147, - "Ġcy": 3148, - "èįIJ": 3149, - "ĠSome": 3150, - "Ġexplain": 3151, - "Ġaware": 3152, - "社交": 3153, - "day": 3154, - "åıĮ": 3155, - "æ²ŁéĢļ": 3156, - "æ°§": 3157, - "å¼Ģåıij": 3158, - "åħ¬åı¸çļĦ": 3159, - "Ġair": 3160, - "åĩ»": 3161, - "aring": 3162, - "éĥ½æĺ¯": 3163, - "Ġlevels": 3164, - "ods": 3165, - "Ġsteps": 3166, - "Ġcap": 3167, - "æ´ŀ": 3168, - "马": 3169, - "Ġreturn": 3170, - "Ġmet": 3171, - "çĶŁæĢģ": 3172, - "丰å¯Į": 3173, - "æŁĵ": 3174, - "æīĢ以": 3175, - "é¡»": 3176, - "Ġer": 3177, - "Ġfra": 3178, - "30": 3179, - "èĵ": 3180, - "âĢĶ": 3181, - "Ġå½ĵ": 3182, - "ah": 3183, - "ä¿ĥ": 3184, - "Ġlikely": 3185, - "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 3186, - "åĪĿ": 3187, - "Ġcreating": 3188, - "Ġfarm": 3189, - "Ġbal": 3190, - "Ġlives": 3191, - "å®ĥçļĦ": 3192, - "Ġability": 3193, - "ä¸ĬçļĦ": 3194, - "Ġsentence": 3195, - "åĤ¨": 3196, - "Ġrout": 3197, - "Ġprovides": 3198, - "Ġagain": 3199, - "å®łçī©": 3200, - "éĢIJ": 3201, - "Ġyears": 3202, - "èŀį": 3203, - "Ġphysical": 3204, - "Python": 3205, - "ĠEx": 3206, - "iting": 3207, - "è°ĥæķ´": 3208, - "ç½ij绾": 3209, - "æħ¢": 3210, - "空éĹ´": 3211, - "åĽ°": 3212, - "è±Ĩ": 3213, - "æĽ´å¤ļçļĦ": 3214, - "ĠAr": 3215, - "Ġmaintain": 3216, - "å®ŀéĻħ": 3217, - "Ġtravel": 3218, - "Ġsat": 3219, - "pro": 3220, - "çĶµåŃIJ": 3221, - "æ±½": 3222, - "ex": 3223, - "åģĩ": 3224, - "æIJŃ": 3225, - "éļıçĿĢ": 3226, - "è¿ĺæľī": 3227, - "礼": 3228, - "ale": 3229, - "Ġconsum": 3230, - "ĊĠ": 3231, - "ncy": 3232, - "Ġquestions": 3233, - "fort": 3234, - "making": 3235, - "Ġdesc": 3236, - "15": 3237, - "Ġinvolves": 3238, - "Ġstress": 3239, - "åŃĹ符": 3240, - "here": 3241, - "Ġimpacts": 3242, - "Ġexercis": 3243, - "åĿļ": 3244, - "ledge": 3245, - "ç§ijæĬĢ": 3246, - "oci": 3247, - "Ġeffectively": 3248, - "æ¶Īè´¹": 3249, - "Ġconclusion": 3250, - "éĺħ": 3251, - "Ġstre": 3252, - "issions": 3253, - "æ·»": 3254, - "It": 3255, - "éĿĻ": 3256, - "Ġvirtual": 3257, - "è¡£": 3258, - "Ġachieve": 3259, - "ource": 3260, - "è¿ŀ": 3261, - "acks": 3262, - "è¡¨æł¼": 3263, - "Ġimportance": 3264, - "èĩªæĪij": 3265, - "These": 3266, - "num": 3267, - "çļĦæł": 3268, - "Ġrelationships": 3269, - "Ġworkers": 3270, - "gical": 3271, - "orpor": 3272, - "erson": 3273, - "åij¢": 3274, - "nds": 3275, - "æݨèįIJ": 3276, - "ohn": 3277, - "å¿ħé¡»": 3278, - "容æĺĵ": 3279, - "ĠGo": 3280, - "Ġtell": 3281, - "ĠRes": 3282, - "onom": 3283, - "Ġbec": 3284, - "æ³Ľ": 3285, - "pos": 3286, - "Ġmove": 3287, - "Ġstory": 3288, - "æŃ¢": 3289, - "Ġpriorit": 3290, - "Ġindustries": 3291, - "èľ": 3292, - "Ġpossible": 3293, - "ĠMan": 3294, - "Ġexpress": 3295, - "abilities": 3296, - "Ġintegr": 3297, - "代表": 3298, - "Ġrespond": 3299, - "åĪĨéĴŁ": 3300, - "æľºä¼ļ": 3301, - "Ġthings": 3302, - "交æµģ": 3303, - "Ġmeth": 3304, - "urther": 3305, - "Ġwide": 3306, - "èijĹ": 3307, - "æĪijçļĦ": 3308, - "ĸçķ¥": 3309, - "ides": 3310, - "ething": 3311, - "ĠWhile": 3312, - "pan": 3313, - "çŃĸçķ¥": 3314, - "Ġcent": 3315, - "Ġplease": 3316, - "ology": 3317, - "uracy": 3318, - "循": 3319, - "ward": 3320, - "nce": 3321, - "Ġthen": 3322, - "çªģ": 3323, - "å¥ĩ": 3324, - "Ġblo": 3325, - "ai": 3326, - "æŀĹ": 3327, - "ç®Ĺæ³ķ": 3328, - "综": 3329, - "Ġprint": 3330, - "aces": 3331, - "lu": 3332, - "ªæĸ½": 3333, - "pre": 3334, - "çļĦæĦı": 3335, - "Ġsol": 3336, - "Ġoverall": 3337, - "hold": 3338, - "Ġes": 3339, - "çļĦä¸Ģ": 3340, - "éģĩ": 3341, - "Ġpopul": 3342, - "å°ı说": 3343, - "æ³¢": 3344, - "åįģ": 3345, - "ä¹Łåı¯ä»¥": 3346, - "é£Łåĵģ": 3347, - "Ġcontent": 3348, - "å°Ħ": 3349, - "Ġrequires": 3350, - "æ£ĢæŁ¥": 3351, - "ĊĠĠĠĠĠĠĠĠĠĠĠ": 3352, - "Ġgroups": 3353, - "Ġfair": 3354, - "Ġbl": 3355, - "å®ŀéªĮ": 3356, - "æĮīçħ§": 3357, - "osp": 3358, - "str": 3359, - "ä¸įèĥ½": 3360, - "Ġharm": 3361, - "Ġprodu": 3362, - "çļĦæĬĢ": 3363, - "çĩ": 3364, - "tle": 3365, - "Ġanimals": 3366, - "è§Ĵèī²": 3367, - "lev": 3368, - "æ¸IJ": 3369, - "å¤įæĿĤ": 3370, - "Ġdepend": 3371, - "æĮijæĪĺ": 3372, - "åĮħåIJ«": 3373, - "Ġhelps": 3374, - "Ġopen": 3375, - "Ġnet": 3376, - "ĠĠĠĠĠ": 3377, - "Ġstrong": 3378, - "Ġjour": 3379, - "å¹¿æ³Ľ": 3380, - "æķ´ä¸ª": 3381, - "Ġelect": 3382, - "Ġresponse": 3383, - "åįķè¯į": 3384, - "æľĭ": 3385, - "Ġ<": 3386, - "åĮĸåѦ": 3387, - "éĴĪ": 3388, - "Ġquick": 3389, - "ually": 3390, - "Ġsomething": 3391, - "Ġtrack": 3392, - "度åĴĮ": 3393, - "erences": 3394, - "æłij": 3395, - "Ġaccuracy": 3396, - "Ġexc": 3397, - "é£ŀ": 3398, - "Ġfield": 3399, - "寻æī¾": 3400, - "éħ¸": 3401, - "Ġhope": 3402, - "çij": 3403, - "Ġinnov": 3404, - "绪": 3405, - "alk": 3406, - "Ġtypes": 3407, - "Ġdid": 3408, - "åĬª": 3409, - "Ġcall": 3410, - "è¯Ĺ": 3411, - "Ġearly": 3412, - "ĠOne": 3413, - "app": 3414, - "Ġcommon": 3415, - "æľĢç»Ī": 3416, - "Ġcheck": 3417, - "Ġsym": 3418, - "çĤĴ": 3419, - "æĬĢèĥ½": 3420, - "Ġenh": 3421, - "Ġagricult": 3422, - "Ġimm": 3423, - "ç»ĩ": 3424, - "满足": 3425, - "Ġschool": 3426, - "bal": 3427, - "Ġfollowing": 3428, - "based": 3429, - "Ġwebs": 3430, - "Ġculture": 3431, - "ĠCom": 3432, - "way": 3433, - "ä¸Ģå®ļ": 3434, - "åķĨåĵģ": 3435, - "ude": 3436, - "çļĦåıijå±ķ": 3437, - "çĶŁäº§": 3438, - "osystem": 3439, - "Ġplant": 3440, - "åı¶": 3441, - "åIJĥ": 3442, - "ä»ĸçļĦ": 3443, - "der": 3444, - "询": 3445, - "家åħ·": 3446, - "Ġfree": 3447, - "移": 3448, - "æİĮ": 3449, - "Ġbody": 3450, - "Ġpresent": 3451, - "Ġparticularly": 3452, - "Ġchildren": 3453, - "Ġstudent": 3454, - ").": 3455, - "çī¹å¾ģ": 3456, - "èĶ": 3457, - "éĺħ读": 3458, - "æķĪçİĩ": 3459, - "Ġprogram": 3460, - "éħ±": 3461, - "åıĺå¾Ĺ": 3462, - "ix": 3463, - "Ġcome": 3464, - "çļĦæ²": 3465, - "ĠTe": 3466, - "ĠTo": 3467, - "åħ±åIJĮ": 3468, - "Ġemployees": 3469, - "说æĺİ": 3470, - "Ġheart": 3471, - "Ġmot": 3472, - "æľĭåıĭ": 3473, - "eric": 3474, - "è¯ij": 3475, - "Ġcurrent": 3476, - "æĪIJæľ¬": 3477, - "Ġtoo": 3478, - "çݩ家": 3479, - "åĪĽæĸ°": 3480, - "Ġecosystem": 3481, - "常è§ģ": 3482, - "ä¸ĢæŃ¥": 3483, - "Ġpres": 3484, - "Ġmulti": 3485, - "åijĬè¯ī": 3486, - "严": 3487, - "Ġmit": 3488, - "Ġaction": 3489, - "çĨŁ": 3490, - "Ġhabit": 3491, - "åı£æĦŁ": 3492, - "ç®±": 3493, - "Ġuses": 3494, - "å¢ŀ强": 3495, - "ç»Ļåĩº": 3496, - "Ġ9": 3497, - "Ġdep": 3498, - "Ġeconomic": 3499, - "æĢ§çļĦ": 3500, - "18": 3501, - "åĨ°": 3502, - "Ġhelped": 3503, - "åIJ¸å¼ķ": 3504, - "çİĭ": 3505, - "Ġdiagnos": 3506, - "åł": 3507, - "èģĶç³»": 3508, - "群": 3509, - "ç»ĥä¹ł": 3510, - "æĪIJéķ¿": 3511, - "Ġpoint": 3512, - "å®ļæľŁ": 3513, - "åij¼": 3514, - "èį¯": 3515, - "æĿ¯": 3516, - "æ¤Ĵ": 3517, - "æķĪæŀľ": 3518, - "Ġspecial": 3519, - "æ··": 3520, - "åĩłä¸ª": 3521, - "ause": 3522, - "éĨ": 3523, - "æ¯ĶèµĽ": 3524, - "è·Ŀ": 3525, - "What": 3526, - "Ġtimes": 3527, - "icles": 3528, - "Ġ*": 3529, - "ç´§": 3530, - "å¦Ĥæŀľä½ł": 3531, - "çĭ¬çī¹": 3532, - "çģµ": 3533, - "ç¨İ": 3534, - "Ġcarbon": 3535, - "Ġbias": 3536, - "åĬ©äºİ": 3537, - "Ġconst": 3538, - "èĩªçĶ±": 3539, - "æĿ¥è¯´": 3540, - "å°±æĺ¯": 3541, - "åį°": 3542, - "Ġmeet": 3543, - "è§ĦåĪĴ": 3544, - "çļĦç¾": 3545, - "èIJ¥åħ»": 3546, - "ators": 3547, - "稳å®ļ": 3548, - "ode": 3549, - "çħ®": 3550, - "Ġassoci": 3551, - "å¿Ĺ": 3552, - "è¡ĮæĺŁ": 3553, - "æĿİ": 3554, - "Ġreview": 3555, - "åĩĢ": 3556, - "ĠRo": 3557, - "Ġknowledge": 3558, - "以便": 3559, - "æµĭè¯ķ": 3560, - "åIJĪéĢĤ": 3561, - "sc": 3562, - "å½¢å¼ı": 3563, - "Ġfriends": 3564, - "Ġnature": 3565, - "Ġcritical": 3566, - "æ´ĭ": 3567, - "Ġafter": 3568, - "erve": 3569, - "Ġrece": 3570, - "çļĦæŃ": 3571, - "汽车": 3572, - "çķĮ": 3573, - "Ġloss": 3574, - "Ġapplications": 3575, - "å¤ļç§į": 3576, - "éĶħ": 3577, - "串": 3578, - "Ġinsp": 3579, - "---": 3580, - "ĠSh": 3581, - "Ġvol": 3582, - "lut": 3583, - "oks": 3584, - "sequ": 3585, - "Ġbir": 3586, - "åIJĪçIJĨ": 3587, - "Ġnecess": 3588, - "æĪijæĥ³": 3589, - "çŃīæĸ¹éĿ¢": 3590, - "é¼ĵ": 3591, - "Ġsoft": 3592, - "Ġlive": 3593, - "å°ıæĺİ": 3594, - "ĠInd": 3595, - "Ġbring": 3596, - "æĺ¯æĮĩ": 3597, - "Ġsoil": 3598, - "ilar": 3599, - "举": 3600, - "æĿ¡ä»¶": 3601, - "Ġtri": 3602, - "亮": 3603, - "Ġmom": 3604, - "æı¡": 3605, - "ä¼°": 3606, - "ŀäºī": 3607, - "çĽij": 3608, - "èĤ¤": 3609, - "è´¢åĬ¡": 3610, - "æ·»åĬł": 3611, - "é¥®é£Ł": 3612, - "Ġallowing": 3613, - "åºķ": 3614, - "Ġright": 3615, - "Ġexpert": 3616, - "Ġsupp": 3617, - "Ġinit": 3618, - "çļĦæµ": 3619, - "arget": 3620, - "Ġexpect": 3621, - "Ġ19": 3622, - "Ġmeasures": 3623, - "olutions": 3624, - "just": 3625, - "arc": 3626, - "å°ļ": 3627, - "Ġpractice": 3628, - "æľīåĬ©äºİ": 3629, - "大éĩı": 3630, - "',": 3631, - "iment": 3632, - "Ġcontinue": 3633, - "Ġdiscuss": 3634, - "100": 3635, - "éļľ": 3636, - "çļĦæĦŁ": 3637, - "Ġreflect": 3638, - "itation": 3639, - "åį«": 3640, - "äºĨä¸Ģ": 3641, - "ney": 3642, - "ĠLe": 3643, - "ised": 3644, - "è¶ĭ": 3645, - "äºĨä¸Ģ个": 3646, - "Ġincreasing": 3647, - "çļĦæĮ": 3648, - "Ġstru": 3649, - "æĢ»ç»ĵ": 3650, - "ely": 3651, - "å®ĩ": 3652, - "Ġauthor": 3653, - "表éĿ¢": 3654, - "Ġx": 3655, - "æķħäºĭ": 3656, - "emic": 3657, - "Ġrepresent": 3658, - "ger": 3659, - "Ġincreased": 3660, - "ones": 3661, - "ains": 3662, - "Ġtrained": 3663, - "Ġfish": 3664, - "Ġstate": 3665, - "åĨ·": 3666, - "çĶŁéķ¿": 3667, - "Ġrenew": 3668, - "ording": 3669, - "åĮĹ": 3670, - "æݪæĸ½": 3671, - "平衡": 3672, - "Ġsuccessful": 3673, - "ä¸ĭéĿ¢": 3674, - "Ġactivity": 3675, - "èĮ¶": 3676, - "éĢĤåºĶ": 3677, - "èĦij": 3678, - "æݢ索": 3679, - "ffic": 3680, - "ç»ĦæĪIJ": 3681, - "atives": 3682, - "äºļ": 3683, - "Ġscen": 3684, - "æ²Ļ": 3685, - "gress": 3686, - "使å¾Ĺ": 3687, - "æī¿": 3688, - "Ġdiscrim": 3689, - "Ġassistants": 3690, - "Ġexist": 3691, - "çķĻ": 3692, - "Ġspace": 3693, - "æľĢè¿ij": 3694, - "Ġideas": 3695, - "éĩĩåıĸ": 3696, - "light": 3697, - "注éĩį": 3698, - "çļĦæŶéĹ´": 3699, - "è¿İ": 3700, - "Ġcomb": 3701, - "éĢĤå½ĵ": 3702, - "Ġyourself": 3703, - "rite": 3704, - "ason": 3705, - "åĮĢ": 3706, - "åı¯ä»¥ä½¿çĶ¨": 3707, - "åħħ满": 3708, - "Ġvalues": 3709, - "æ½": 3710, - "Ġbiases": 3711, - "ä¿ĥè¿Ľ": 3712, - "åľºæĻ¯": 3713, - "ross": 3714, - "åį³åı¯": 3715, - "Ġcru": 3716, - "Ġnumber": 3717, - "Ġtype": 3718, - "rast": 3719, - "åĩĨç¡®": 3720, - "This": 3721, - "Ġpast": 3722, - "çģ¯": 3723, - "å®ļä¹ī": 3724, - "Ġsolutions": 3725, - "Ġter": 3726, - "ä¿Ŀè¯ģ": 3727, - "èĶ¬": 3728, - "幸": 3729, - "åī§": 3730, - "åħ´è¶£": 3731, - "åª": 3732, - "ention": 3733, - "avor": 3734, - "Ġscient": 3735, - "åĬªåĬĽ": 3736, - "Ġproviders": 3737, - "Ġpolicies": 3738, - "alu": 3739, - "ĠIm": 3740, - "Ġallows": 3741, - "Ġintelligence": 3742, - "çļĦæĸ¹æ³ķ": 3743, - "è¿Ļæĺ¯": 3744, - "Ġ`": 3745, - "Ġemissions": 3746, - "Ġå°Ĩ": 3747, - "Ġmeaning": 3748, - "Ġstyle": 3749, - "åİŁåĽł": 3750, - "Ġstrugg": 3751, - "çļĦç¾İ": 3752, - "iful": 3753, - "dition": 3754, - "éĥ½æľī": 3755, - "空æ°Ķ": 3756, - "å®ĥ们çļĦ": 3757, - "ä¼ĺåĮĸ": 3758, - "Ġinflu": 3759, - "åŁºäºİ": 3760, - "Ġdetails": 3761, - "Ġtransparency": 3762, - "Ġmess": 3763, - "ĠCl": 3764, - "Ġgame": 3765, - "pri": 3766, - "è¶ĭåĬ¿": 3767, - "å½Ĵ": 3768, - "ç¿»è¯ij": 3769, - "æķ£": 3770, - "By": 3771, - "éŃ": 3772, - "ĠAmeric": 3773, - "Ġproduction": 3774, - "Ġincorpor": 3775, - "æĻļ": 3776, - "Ġinvolve": 3777, - "Ġhot": 3778, - "æĻ®": 3779, - "by": 3780, - "Ġflow": 3781, - "Ġemerg": 3782, - "座": 3783, - "Ġidea": 3784, - "åİĭåĬĽ": 3785, - "éĿĴ": 3786, - "oms": 3787, - "èģĮä¸ļ": 3788, - "Ġreport": 3789, - "Ġpap": 3790, - "Ġtherap": 3791, - "Ġsal": 3792, - "åıĤä¸İ": 3793, - "æĸĩåѦ": 3794, - "æIJŃéħį": 3795, - "oot": 3796, - "),": 3797, - "Ġcr": 3798, - "Ġprocesses": 3799, - "gin": 3800, - "å¹³åı°": 3801, - "å¯Ł": 3802, - "Ġpromoting": 3803, - "æļĸ": 3804, - "akehold": 3805, - "继": 3806, - "iver": 3807, - "æ¦Ĥ": 3808, - "Ġmodels": 3809, - "Ġdra": 3810, - "èĸ": 3811, - "Ġgroup": 3812, - "è¶³å¤Ł": 3813, - "Ġgreen": 3814, - "Ġhealthy": 3815, - "Ġcomfort": 3816, - "Ġadditional": 3817, - "ä¸Ģ次": 3818, - "é¤IJåİħ": 3819, - "Ġmaterials": 3820, - "Ġmanage": 3821, - "çļĦæ¯": 3822, - "伤": 3823, - "åıĬæŶ": 3824, - "Ġglo": 3825, - "Ġstat": 3826, - "å¿«éĢŁ": 3827, - "Ġmonitoring": 3828, - "aily": 3829, - "rand": 3830, - "oice": 3831, - "resh": 3832, - "ç»Ħç»ĩ": 3833, - "Ġunder": 3834, - "Ġnecessary": 3835, - "Ġhelpful": 3836, - "ĠCol": 3837, - "é»ijæ´ŀ": 3838, - "åģļåĩº": 3839, - "Ġcourse": 3840, - "Ġmat": 3841, - "Ġleg": 3842, - "Ġface": 3843, - "令": 3844, - "èī¯å¥½çļĦ": 3845, - "ock": 3846, - "åĮ»çĸĹ": 3847, - "çĽĸ": 3848, - "idence": 3849, - "Ġassociated": 3850, - "Ġprogress": 3851, - "åľĨ": 3852, - "Ġeveryone": 3853, - "ç¼ĵ": 3854, - "ĠEng": 3855, - "word": 3856, - "èĵĿ": 3857, - "天æ°Ķ": 3858, - "Ġactions": 3859, - "ems": 3860, - "ĠPl": 3861, - "å®Ļ": 3862, - "ush": 3863, - "顾": 3864, - "Ġcosts": 3865, - "ator": 3866, - "ç©¿": 3867, - "Ġamounts": 3868, - "èĶ¬èıľ": 3869, - "..": 3870, - "Ġmanner": 3871, - "Ġconsequ": 3872, - "æ°ĶåĢĻ": 3873, - "Ġinsights": 3874, - "being": 3875, - "atory": 3876, - "ener": 3877, - "lex": 3878, - "Ġmeans": 3879, - "Ġcollaboration": 3880, - "Ġperspect": 3881, - "orm": 3882, - "priate": 3883, - "å°Ĭéĩį": 3884, - "Ġtarget": 3885, - "è®°å½ķ": 3886, - "åĢĴ": 3887, - "Ġrenewable": 3888, - "æĦ¿": 3889, - "èĥ½æºIJ": 3890, - "Ġinput": 3891, - "å®ĩå®Ļ": 3892, - "ape": 3893, - "Ġadjust": 3894, - "eries": 3895, - "Ġdire": 3896, - "ä¾Ŀ": 3897, - "ustr": 3898, - "fect": 3899, - "Ġbeautiful": 3900, - "Ġdue": 3901, - "reci": 3902, - "çĮ®": 3903, - "èĥĮæĻ¯": 3904, - "èĤ¡": 3905, - "Ġdam": 3906, - "ik": 3907, - "Ġadvanced": 3908, - "çĽ¸å¯¹": 3909, - "åIJį称": 3910, - "Ġshort": 3911, - "Ġobject": 3912, - "è¿ĻéĩĮ": 3913, - "éĢłæĪIJ": 3914, - "èIJ¥éĶĢ": 3915, - "çļĦæĥħæĦŁ": 3916, - "票": 3917, - "Ġcountries": 3918, - "ining": 3919, - "istic": 3920, - "Ġplans": 3921, - "责任": 3922, - "Ġstakehold": 3923, - "the": 3924, - "Ġassess": 3925, - "æĢĿèĢĥ": 3926, - "ech": 3927, - "æĪIJåijĺ": 3928, - "21": 3929, - "Ġdaily": 3930, - "Ġcomput": 3931, - "çļĦæĥħåĨµ": 3932, - "æıIJåĩº": 3933, - "ĠâĢľ": 3934, - "åªĴ": 3935, - "ä¸Ńå¿ĥ": 3936, - "ished": 3937, - "ĠSe": 3938, - "onomous": 3939, - "ern": 3940, - "ç»´æĬ¤": 3941, - "ames": 3942, - "Ġprioritize": 3943, - "纸": 3944, - "èĤ¥": 3945, - "Ġtemper": 3946, - "æ¸ħæ´ģ": 3947, - "use": 3948, - "污": 3949, - "Ġminim": 3950, - "æĺ¯åľ¨": 3951, - "大å°ı": 3952, - "åĵªäºĽ": 3953, - "Ġappreci": 3954, - "reng": 3955, - "Ġregulations": 3956, - "ĠZ": 3957, - "éĶĻ误": 3958, - "rans": 3959, - "èĢĮä¸Ķ": 3960, - "èĪ¬": 3961, - "èij±": 3962, - "èĨ": 3963, - "æ°´å¹³": 3964, - "è´Ńçī©": 3965, - "åŃĹ符串": 3966, - "对æĸ¹": 3967, - "Ġhim": 3968, - "Ġconsequences": 3969, - "å·´": 3970, - "é¼ĵåĬ±": 3971, - "Ġfil": 3972, - "人åijĺ": 3973, - "è·Ŀ离": 3974, - "ĠWhen": 3975, - "çļĦæ°´": 3976, - "çī©çIJĨ": 3977, - "åIJĮæĹ¶ä¹Ł": 3978, - "åľ¨è¿Ļ个": 3979, - "åħ¶æ¬¡": 3980, - ",\"": 3981, - "液": 3982, - "çĶ·": 3983, - "ival": 3984, - "åı¯ä»¥è®©": 3985, - "æĥ¯": 3986, - "Ġadvance": 3987, - "Ġveh": 3988, - "å¦ĤæŀľæĤ¨": 3989, - "Ġestab": 3990, - "ript": 3991, - "端": 3992, - "ä¸įä¼ļ": 3993, - "Ġtransparent": 3994, - "æķ°éĩı": 3995, - "çĽĺ": 3996, - "Ġspeak": 3997, - "Ġpark": 3998, - "Ġstakeholders": 3999, - "éº": 4000, - "Ġevent": 4001, - "çļĦæķ°æį®": 4002, - "èĩªåĬ¨": 4003, - "ç»ĨèĬĤ": 4004, - "è¯Ħä¼°": 4005, - "润": 4006, - "Ġpreferences": 4007, - "Ġveget": 4008, - "æįŁ": 4009, - "equ": 4010, - "Ġgl": 4011, - "Ġpain": 4012, - "ogra": 4013, - "Ġtraffic": 4014, - "Ġoce": 4015, - "ä¹ĺ": 4016, - "ext": 4017, - "âĢĿï¼Į": 4018, - "Ġanother": 4019, - "å¤ļå°ij": 4020, - "Ġagainst": 4021, - "ç»ıåİĨ": 4022, - "计ç®Ĺæľº": 4023, - "èĢIJ": 4024, - "软件": 4025, - "ĠPre": 4026, - "Ġplants": 4027, - "缸äºĴ": 4028, - "é¢ij": 4029, - "\\_": 4030, - "Ġsame": 4031, - "rug": 4032, - "Ġvalu": 4033, - "Ġocc": 4034, - "çļĦç¤": 4035, - "Ġsustainability": 4036, - "ĠShe": 4037, - "de": 4038, - "ote": 4039, - "Ġdig": 4040, - "NA": 4041, - "Ġcrucial": 4042, - "æī§": 4043, - "å±Ģ": 4044, - "æĭŁ": 4045, - "æĭĮ": 4046, - "Ġnon": 4047, - "Ġengaging": 4048, - "Ġintern": 4049, - "LP": 4050, - "温度": 4051, - "æł¸": 4052, - "æĬ¥åijĬ": 4053, - "æĿ¥è¶Ĭ": 4054, - "hood": 4055, - "ä¸ī个": 4056, - "å¦Ĥä¸ĭ": 4057, - "çī©ä½ĵ": 4058, - "force": 4059, - "Ġneeded": 4060, - "Ġimages": 4061, - "Ġbuilding": 4062, - "icious": 4063, - "ĠæĪij": 4064, - "è¶ĬæĿ¥è¶Ĭ": 4065, - "æĶ¾åħ¥": 4066, - "go": 4067, - "éĻįä½İ": 4068, - "å½ĵåľ°": 4069, - "æ¶Īè´¹èĢħ": 4070, - "ç£": 4071, - "iversity": 4072, - "é¢Ħç®Ĺ": 4073, - "icle": 4074, - "æ··åIJĪ": 4075, - "Ġparticip": 4076, - "Ġdishes": 4077, - "Ġthroughout": 4078, - "Ġwithin": 4079, - "åı³": 4080, - "é«ĺçļĦ": 4081, - "Ġphot": 4082, - "Ġtrust": 4083, - "æĦıè¯Ĩ": 4084, - "以确ä¿Ŀ": 4085, - "çĬ¶æĢģ": 4086, - "Ġautomation": 4087, - "11": 4088, - "Ġpost": 4089, - "æīĭæľº": 4090, - "works": 4091, - "éĢı": 4092, - "åºĵ": 4093, - "Ġwind": 4094, - "Ġ==": 4095, - "Ġprocessing": 4096, - "èĮĥåĽ´": 4097, - "æĦıä¹ī": 4098, - "追æ±Ĥ": 4099, - "é": 4100, - "å¾Ħ": 4101, - "éĿł": 4102, - "ä¸ĸ": 4103, - "èĻ½": 4104, - "ç«ŀäºī": 4105, - "Ġappropriate": 4106, - "æĽ´å¥½çļĦ": 4107, - "Ġcharacter": 4108, - "cl": 4109, - "ç§ĺ": 4110, - "itude": 4111, - "Ġteac": 4112, - "leep": 4113, - "ĠDevelop": 4114, - "ince": 4115, - "å·¦": 4116, - "ground": 4117, - "è¡Įä¸ļ": 4118, - "éĴĪ对": 4119, - "å¿ħè¦ģ": 4120, - "Ġdeterm": 4121, - "----------------": 4122, - "Ġstreng": 4123, - "do": 4124, - "Ġchallenging": 4125, - "ork": 4126, - "Ġanx": 4127, - "èī²çļĦ": 4128, - "Ġhard": 4129, - "æĺİç¡®": 4130, - "åĪĨ享": 4131, - "æĶ¹åıĺ": 4132, - "ä½³": 4133, - "åıªæľī": 4134, - "å±ķ示": 4135, - "Ġcamp": 4136, - "纳": 4137, - "aj": 4138, - "etic": 4139, - "ument": 4140, - "ä½łåı¯ä»¥": 4141, - "Ġpollut": 4142, - "Ġhig": 4143, - "pping": 4144, - "ead": 4145, - "çĦ¶èĢĮ": 4146, - "第äºĮ": 4147, - "鸣": 4148, - "çī©åĵģ": 4149, - "举": 4150, - "Ġencourage": 4151, - "pecial": 4152, - "Ġacross": 4153, - "elves": 4154, - "äºĭ件": 4155, - "cle": 4156, - "æ©": 4157, - "åªĴä½ĵ": 4158, - "ners": 4159, - "Ġcal": 4160, - "èĻ½çĦ¶": 4161, - "åĽº": 4162, - "ä¹łæĥ¯": 4163, - "Ġsafe": 4164, - "èĥ½éĩı": 4165, - "istics": 4166, - "ä¹ĭåīį": 4167, - "Ġissue": 4168, - "å¤ļ个": 4169, - "åĨ³çŃĸ": 4170, - "è¾¾åĪ°": 4171, - "æĹ©": 4172, - "ä¸įåı¯": 4173, - "ä¸Ģ缴": 4174, - "å·¨": 4175, - "æĦŁè°¢": 4176, - "ĠNew": 4177, - "ä¸Ģ段": 4178, - "Ġmachines": 4179, - "å°Ĩåħ¶": 4180, - "继ç»Ń": 4181, - "Ġword": 4182, - "çī¹åĪ«": 4183, - "Ġagriculture": 4184, - "æĢİ": 4185, - "éĢIJæ¸IJ": 4186, - "éĵ¾": 4187, - "课": 4188, - "Ġkind": 4189, - "å¢Ļ": 4190, - "谢谢": 4191, - "Ġalgorithm": 4192, - "è£ħ饰": 4193, - "Ġalong": 4194, - "Ġeasy": 4195, - "äºij": 4196, - "解åĨ³æĸ¹æ¡Ī": 4197, - "Ġawareness": 4198, - "'ve": 4199, - "æĸ¹åIJij": 4200, - "Ġnever": 4201, - "Ġquickly": 4202, - "Ġrespect": 4203, - "çļĦæĻ": 4204, - "Ġamong": 4205, - "Ġaccountability": 4206, - "Ġlaw": 4207, - "ening": 4208, - "Ġdefin": 4209, - "Ġsurround": 4210, - "éĵģ": 4211, - "Ġpowerful": 4212, - "An": 4213, - "Ġcause": 4214, - "æ¥": 4215, - "æİĮæı¡": 4216, - "è¿ĺæĺ¯": 4217, - "Ġcreative": 4218, - "è¡Ģ": 4219, - "Ġlocated": 4220, - "unning": 4221, - "åľ°åĮº": 4222, - "éĿ¢ç§¯": 4223, - "鼨": 4224, - "Ġnear": 4225, - "Ġiniti": 4226, - "ression": 4227, - "ä¸ĭæĿ¥": 4228, - "25": 4229, - "驶": 4230, - "¾çĹħ": 4231, - "ables": 4232, - "æľī趣": 4233, - "循çݯ": 4234, - "çŃĶæ¡Ī": 4235, - "çł´": 4236, - "ication": 4237, - "éĻ¢": 4238, - "æ²»çĸĹ": 4239, - "Ġaddition": 4240, - "äºĭæĥħ": 4241, - "Ġbecause": 4242, - "åıĪ": 4243, - "èĤĮ": 4244, - "纪": 4245, - "side": 4246, - "æĭħ": 4247, - "湿": 4248, - "åįĬ": 4249, - "顺": 4250, - "ĠAnd": 4251, - "Ġrestaurant": 4252, - "Ġvide": 4253, - "Ġproblem": 4254, - "azing": 4255, - "Ġmembers": 4256, - "Ġnut": 4257, - "Ġcou": 4258, - "浪": 4259, - "Ġè¿Ļ": 4260, - "Ġhelping": 4261, - "ĠIs": 4262, - "æıIJåįĩ": 4263, - "ĠĠĠĠĠĠ": 4264, - "Ġsho": 4265, - "Ġrelev": 4266, - "Ġarg": 4267, - "Ġbalance": 4268, - "illed": 4269, - "æĺ¯ä»Ģä¹Ī": 4270, - "åĬĽéĩı": 4271, - "ired": 4272, - "å¤ľ": 4273, - "åı¯æĮģç»Ń": 4274, - "Ġperfect": 4275, - "**": 4276, - "ification": 4277, - "æ¶ī": 4278, - "Ġwildlife": 4279, - "ane": 4280, - "Ġrelated": 4281, - "室åĨħ": 4282, - "åºľ": 4283, - "享åıĹ": 4284, - "ours": 4285, - "è·ij": 4286, - "åķĨä¸ļ": 4287, - "aching": 4288, - "Ġsun": 4289, - "Ġrecognition": 4290, - "elt": 4291, - "Ġorder": 4292, - "å¹³åĿĩ": 4293, - "ging": 4294, - "临": 4295, - "çĤ¼": 4296, - "Ġgoing": 4297, - "åij¼åIJ¸": 4298, - "Ġsoftware": 4299, - "Ġremot": 4300, - "èijĹåIJį": 4301, - "幸ç¦ı": 4302, - "Ġenhance": 4303, - "èĻļ": 4304, - "Ġnow": 4305, - "Ġthreat": 4306, - "Ġdest": 4307, - "åĿĩåĮĢ": 4308, - "Ġacad": 4309, - "åºĶ对": 4310, - "çľĭåĪ°": 4311, - "cast": 4312, - "è¾Ĩ": 4313, - "ificial": 4314, - "Ġvery": 4315, - "ook": 4316, - "åĮºåŁŁ": 4317, - "¹ģ": 4318, - "æĪ¿éĹ´": 4319, - "æıIJä¾ĽäºĨ": 4320, - "Ġmotiv": 4321, - "Ġaccessible": 4322, - "åĨ³å®ļ": 4323, - "Ġhy": 4324, - "å®Ī": 4325, - "Ġflo": 4326, - "ug": 4327, - "Ġinformed": 4328, - "åĵģè´¨": 4329, - "çļĦçŁ": 4330, - "aves": 4331, - "arr": 4332, - "ĠWith": 4333, - "let": 4334, - "è§ĤçĤ¹": 4335, - "enge": 4336, - "è¡ĮåĬ¨": 4337, - "friend": 4338, - "ç³ķ": 4339, - "Ġfurther": 4340, - "ĠEns": 4341, - "ç§ģ": 4342, - "Ġado": 4343, - "Ġclean": 4344, - "缸åºĶ": 4345, - "Ġfre": 4346, - "pecially": 4347, - "èĹ": 4348, - "Ġcapt": 4349, - "çļĦçľ": 4350, - "Ġsomeone": 4351, - "Ġcell": 4352, - "æĶ¾åľ¨": 4353, - "欢è¿İ": 4354, - "ĠâĢ": 4355, - "Ġdevices": 4356, - "çļĦæĸ¹å¼ı": 4357, - "Ġjobs": 4358, - "augh": 4359, - "not": 4360, - "æľīäºĽ": 4361, - "åħ¬åħ±": 4362, - "gest": 4363, - "çļĦçĶŁæ´»": 4364, - "çľ¼": 4365, - "çļĦä¿¡æģ¯": 4366, - "ĠCons": 4367, - "æİĴåºı": 4368, - "Ġbenefit": 4369, - "rect": 4370, - "å¤ı": 4371, - "unte": 4372, - "符åIJĪ": 4373, - "ä¸Ģä½į": 4374, - "åĨħéĥ¨": 4375, - "Ġlooking": 4376, - "ding": 4377, - "æĬĺ": 4378, - "è¾ij": 4379, - "è¿Ļ个éĹ®é¢ĺ": 4380, - "Ġespecially": 4381, - "çľł": 4382, - "âĢĿãĢĤ": 4383, - "å¥ı": 4384, - "ray": 4385, - "è¿ĺåı¯ä»¥": 4386, - "åĪĽä½ľ": 4387, - "coming": 4388, - "Ġmultiple": 4389, - "éļIJ": 4390, - "泡": 4391, - "æłĩåĩĨ": 4392, - "Ġmil": 4393, - "éľĢè¦ģ注æĦı": 4394, - "Ġanxiety": 4395, - "æĶ¹è¿Ľ": 4396, - "å±ĭ": 4397, - "污æŁĵ": 4398, - "ç¼ĸç¨ĭ": 4399, - "è´¹çĶ¨": 4400, - "Ġevalu": 4401, - "imately": 4402, - "Ġliter": 4403, - "ograph": 4404, - "Ġsearch": 4405, - "16": 4406, - "enced": 4407, - "Ġmethods": 4408, - "çĥĪ": 4409, - "模å¼ı": 4410, - "çĬ¶åĨµ": 4411, - "æĶ¹åĸĦ": 4412, - "å¤ļæł·": 4413, - "cer": 4414, - "å¥ĸ": 4415, - "Ġsatis": 4416, - "Ġwebsite": 4417, - "åĬŀ": 4418, - "åģ¥èº«": 4419, - "Ġglobal": 4420, - "Ġask": 4421, - "Ġplatforms": 4422, - "Ġdiseases": 4423, - "çݰ象": 4424, - "tics": 4425, - "æ±ģ": 4426, - "åĪ¤æĸŃ": 4427, - "Ġconvers": 4428, - "Ġrelationship": 4429, - "设置": 4430, - "æ³ķå¾ĭ": 4431, - "Ġmindful": 4432, - "é¢Ħæµĭ": 4433, - "overy": 4434, - "åģľ": 4435, - "çĶµè§Ĩ": 4436, - "è§ĦåĪĻ": 4437, - "aken": 4438, - "Ġimplementing": 4439, - "ising": 4440, - "åıĤåĬł": 4441, - "æĥħ绪": 4442, - "Ġprovided": 4443, - "æ·±åħ¥": 4444, - "Ġprogrammed": 4445, - "Ġrelevant": 4446, - "çļĦçĥ": 4447, - "çĸ¾çĹħ": 4448, - "åĮ»çĶŁ": 4449, - "åĪĽå»º": 4450, - "Ġgenerate": 4451, - "æĶ¶åħ¥": 4452, - "ä¼ij": 4453, - "izes": 4454, - "Ġtransform": 4455, - "éģµ": 4456, - "astic": 4457, - "åijĪ": 4458, - "æ¯ı个人": 4459, - "è¿Ķ": 4460, - "iet": 4461, - "Ġvoice": 4462, - "éĢĶ": 4463, - "æĶ¾æĿ¾": 4464, - "åį´": 4465, - "èĥľ": 4466, - "Ġstructure": 4467, - "æŶå°ļ": 4468, - "ĠQ": 4469, - "Ġelse": 4470, - "duc": 4471, - "Ġemp": 4472, - "èģļ": 4473, - "è´§": 4474, - "aches": 4475, - "ç§Ģ": 4476, - "anks": 4477, - "Ġnight": 4478, - "Ġprofessionals": 4479, - "Ġbas": 4480, - "è´µ": 4481, - "ec": 4482, - "Ġdiversity": 4483, - "ites": 4484, - "dr": 4485, - "åĽ°éļ¾": 4486, - "ĥåľ": 4487, - "åŀĥåľ": 4488, - "åŀĥåľ¾": 4489, - "Ġdrug": 4490, - "碳": 4491, - "Ġname": 4492, - "åĮĸçļĦ": 4493, - "aid": 4494, - "æľĢ大": 4495, - "æijĦ": 4496, - "ç®ĢåįķçļĦ": 4497, - "Ġwarm": 4498, - "Ġdone": 4499, - "Ġfunction": 4500, - "asc": 4501, - "强è°ĥ": 4502, - "Ġdemand": 4503, - "Ġvisual": 4504, - "Ġupd": 4505, - "æŃ£åľ¨": 4506, - "Ġsimilar": 4507, - "éĢĴ": 4508, - "æ¯Ľ": 4509, - "éĶ»": 4510, - "ently": 4511, - "Ġvaluable": 4512, - "Ġdisaster": 4513, - "ä¸ĢèĪ¬": 4514, - "æ´²": 4515, - "ĠReg": 4516, - "Ġdiscrimination": 4517, - "åĨĻä¸Ģç¯ĩ": 4518, - "Ġgovernment": 4519, - "Ġ好çļĦ": 4520, - "500": 4521, - "lying": 4522, - "Ġprev": 4523, - "Ġprepare": 4524, - "Ġproblems": 4525, - "è·³": 4526, - "Ġprom": 4527, - "åĨ²": 4528, - "å®īè£ħ": 4529, - "éĶ»çĤ¼": 4530, - "æµĵ": 4531, - "è¹": 4532, - "åºĶçĶ¨ç¨ĭåºı": 4533, - "ng": 4534, - "Ġcompet": 4535, - "åĪĨåĪ«": 4536, - "ological": 4537, - "审": 4538, - "Ġtransl": 4539, - "Ġdirect": 4540, - "åīĤ": 4541, - "Ġsuggestions": 4542, - "Ġpaper": 4543, - "Ġrecognize": 4544, - "ton": 4545, - "Ġmitigate": 4546, - "讨论": 4547, - "äºĴåĬ¨": 4548, - "ĠEar": 4549, - "Ġamazing": 4550, - "cre": 4551, - "é¦Ī": 4552, - "Ġinvolved": 4553, - "face": 4554, - "æľīåħ³": 4555, - "))": 4556, - "Ġexce": 4557, - "Ġproductivity": 4558, - "èŃ": 4559, - "é¦Ĩ": 4560, - "Ġsounds": 4561, - "Ġidentifying": 4562, - "],": 4563, - "é¾Ļ": 4564, - "Ġfit": 4565, - "Ġcontribute": 4566, - "ths": 4567, - "friendly": 4568, - "ele": 4569, - "ified": 4570, - "iveness": 4571, - "itely": 4572, - "ĠX": 4573, - "Ġled": 4574, - "åĿı": 4575, - "Ġhistor": 4576, - "Ġdat": 4577, - "Ġjourney": 4578, - "Ġ}": 4579, - "Ġselect": 4580, - "漫": 4581, - "Ġconduct": 4582, - "è¿Ľä¸ĢæŃ¥": 4583, - "ç»ĻæĪij": 4584, - "Ġlif": 4585, - "è£ħä¿®": 4586, - "为ä»Ģä¹Ī": 4587, - "京": 4588, - "Ġnav": 4589, - "Ġwhole": 4590, - "ç¹ģ": 4591, - "åĨľ": 4592, - "æĶ»": 4593, - "Ġbreat": 4594, - "Ġmiss": 4595, - "é¾Ħ": 4596, - "tt": 4597, - "sw": 4598, - "Ġbar": 4599, - "请éĹ®": 4600, - "èģĶç½ij": 4601, - "Ġattract": 4602, - "æĤ¨åı¯ä»¥": 4603, - "One": 4604, - "åħħåĪĨ": 4605, - "ring": 4606, - "Ġå½ĵçĦ¶": 4607, - "ream": 4608, - "Ġevol": 4609, - "Ġsn": 4610, - "ĠEm": 4611, - "mosp": 4612, - "Ġchoose": 4613, - "view": 4614, - "Ġarr": 4615, - "Ġsleep": 4616, - "ended": 4617, - "æŀ¶": 4618, - "Ġvehicles": 4619, - "Ġfresh": 4620, - "Ġorganization": 4621, - "è¿Ļ段": 4622, - "汤": 4623, - "ĠInt": 4624, - "Ġcontext": 4625, - "åı¦å¤ĸ": 4626, - "Ġocean": 4627, - "æĦŁåıĹ": 4628, - "Ġpollution": 4629, - "urb": 4630, - "æī§è¡Į": 4631, - "ersonal": 4632, - "ĠHealth": 4633, - "ä¼ĺçĤ¹": 4634, - "Ġattention": 4635, - "æľīçĿĢ": 4636, - "é£ŁæĿIJ": 4637, - "Ġerr": 4638, - "çļĦæĿ¥": 4639, - "çļĦçĪ": 4640, - "èѦ": 4641, - "è·Ł": 4642, - "æĹħè¡Į": 4643, - "èĴľ": 4644, - "çļĦæĢĿ": 4645, - "Ġchatbot": 4646, - "çļĦéľĢæ±Ĥ": 4647, - "çķ¥": 4648, - "Ġfeeling": 4649, - "Ġimplemented": 4650, - "社åĮº": 4651, - "çļĦ建议": 4652, - "æIJħ": 4653, - "éĹ»": 4654, - "åıįé¦Ī": 4655, - "缴æİ¥": 4656, - "æĺ¥": 4657, - "itable": 4658, - "æĪijä¼ļ": 4659, - "åį±": 4660, - "èī¯å¥½": 4661, - "Ġliving": 4662, - "åıĺéĩı": 4663, - "ĠBut": 4664, - "Ġcomplete": 4665, - "Ġtrends": 4666, - "Ġmakes": 4667, - "ä»Ĭ天": 4668, - "Ġdistribut": 4669, - "Ġcommit": 4670, - "Ġatmosp": 4671, - "ä¼´": 4672, - "Ġsensors": 4673, - "Ġsw": 4674, - "æĹłè®º": 4675, - "omen": 4676, - "æĶ¿åºľ": 4677, - "Ġchallenge": 4678, - "Ġturn": 4679, - "çIJĨ论": 4680, - "par": 4681, - "Ġwrite": 4682, - "ç»ıåħ¸": 4683, - "emember": 4684, - "é¥Ń": 4685, - "æĸ¹ä¾¿": 4686, - "Ġcu": 4687, - "Ġvalue": 4688, - "Ġfund": 4689, - "pose": 4690, - "è°ĥæŁ¥": 4691, - "çĿ¡": 4692, - "Ġcommunicate": 4693, - "Ġdisease": 4694, - "Ġresearc": 4695, - "Ġlack": 4696, - "arning": 4697, - "ĠPark": 4698, - "çĦ¦": 4699, - "é«ĺ度": 4700, - "Ġrather": 4701, - "宣": 4702, - "çĪ¶": 4703, - "éĺ¶": 4704, - "订": 4705, - "çĥ§": 4706, - "Ġhigher": 4707, - "Ġsummary": 4708, - "ĠAut": 4709, - "çļĦæ³": 4710, - "Ġele": 4711, - "isms": 4712, - "Ġreli": 4713, - "ä¹Łä¼ļ": 4714, - "fra": 4715, - "åijĬè¯īæĪij": 4716, - "æĬ½": 4717, - "Ġsituations": 4718, - "Ġmarine": 4719, - "æĥ³è¦ģ": 4720, - "inci": 4721, - "inal": 4722, - "Ġgain": 4723, - "Ġdifference": 4724, - "æľºåĻ¨äºº": 4725, - "æµģç¨ĭ": 4726, - "ĠChat": 4727, - "ç½ijç«Ļ": 4728, - "æľ«": 4729, - "Ġcolor": 4730, - "Ġaspect": 4731, - "ç½Ĺ": 4732, - "ĠEduc": 4733, - "Ġdeploy": 4734, - "Ġbeauty": 4735, - "æĤ£": 4736, - "ruction": 4737, - "itut": 4738, - "æĿŁ": 4739, - "让æĪij们": 4740, - "éķ¿åº¦": 4741, - "ules": 4742, - "æ¶īåıĬ": 4743, - "Ġdigital": 4744, - "Ġexisting": 4745, - "ĠOr": 4746, - "\\_\\_": 4747, - "Ġbackground": 4748, - "çĹĩ": 4749, - "æ¯ı天": 4750, - "python": 4751, - "Ġfarmers": 4752, - "Ġcontinu": 4753, - "\":": 4754, - "Ġgiven": 4755, - "å°ıæŶ": 4756, - "Ġmoment": 4757, - "200": 4758, - "John": 4759, - "éĿ¢å¯¹": 4760, - "Ġintro": 4761, - "Ġtherapy": 4762, - "è¿ĶåĽŀ": 4763, - "å¹¶åľ¨": 4764, - "Ġz": 4765, - "Ġafford": 4766, - "ä¸Ŀ": 4767, - "宽": 4768, - "ĠÃ": 4769, - "ĠNational": 4770, - "èĥ¡": 4771, - "Ġexercise": 4772, - "æIJħæĭĮ": 4773, - "æĶ¯ä»ĺ": 4774, - "éĺ³åħī": 4775, - "è¯ļ": 4776, - "Ġsect": 4777, - "ĠSu": 4778, - "å¢ŀéķ¿": 4779, - "ç¾İ丽": 4780, - "Ġwa": 4781, - "以ä¸ĭæĺ¯ä¸ĢäºĽ": 4782, - "èĽĭç³ķ": 4783, - "Ġill": 4784, - "æ¸ħæĻ": 4785, - "etry": 4786, - "梦": 4787, - "ç¾İåĽ½": 4788, - "ä»į": 4789, - "oney": 4790, - "Ġecosystems": 4791, - "æĮĩ导": 4792, - "def": 4793, - "99": 4794, - "æŁĶ": 4795, - "pped": 4796, - "Ġlimit": 4797, - "çİī": 4798, - "Ġacademic": 4799, - "Ġrestaurants": 4800, - "Ġhead": 4801, - "ä¿¡ä»»": 4802, - "asters": 4803, - "å²ģ": 4804, - "akers": 4805, - "14": 4806, - "As": 4807, - "æł¡": 4808, - "é«ĺæķĪ": 4809, - "phas": 4810, - "yn": 4811, - "ç¨ĭ度": 4812, - "è¾£": 4813, - "ä¸ĬéĿ¢": 4814, - "家å±ħ": 4815, - "term": 4816, - "ç¾İé£Ł": 4817, - "Ġovers": 4818, - "å®ĺ": 4819, - "Ġindic": 4820, - "ĠYour": 4821, - "St": 4822, - "形象": 4823, - "è´¡": 4824, - "åºĬ": 4825, - "ĠSc": 4826, - "agra": 4827, - "羣æŃ£": 4828, - "oint": 4829, - "ids": 4830, - "arent": 4831, - "éĵ¶": 4832, - "èģĬ": 4833, - "Ġregular": 4834, - "ä¼ĺç§Ģ": 4835, - "Ġcolle": 4836, - "çĸij": 4837, - "Ġsubject": 4838, - "Ġgreater": 4839, - "Ġstore": 4840, - "åŁ¹è®Ń": 4841, - "Ġimag": 4842, - "Ġansw": 4843, - "ä½Ļ": 4844, - "Ġspot": 4845, - "åĪĨåŃIJ": 4846, - "Ġaudience": 4847, - "pet": 4848, - "Ġvers": 4849, - "Ġtrail": 4850, - "åĭĩ": 4851, - "erous": 4852, - "Ġguidance": 4853, - "Ġspeech": 4854, - "åĵ²": 4855, - "æĺ¯çĶ±": 4856, - "è´¡çĮ®": 4857, - "åIJĪéĢĤçļĦ": 4858, - "设æĸ½": 4859, - "ä»ĸ人": 4860, - "ensive": 4861, - "åĢ¾": 4862, - "aling": 4863, - "Ġprojects": 4864, - "å³": 4865, - "Ġtakes": 4866, - "绩": 4867, - "That": 4868, - "Ġbro": 4869, - "ived": 4870, - "Ġ&": 4871, - "åĿIJ": 4872, - "placement": 4873, - "è¿ŀæİ¥": 4874, - "çļĦ社": 4875, - "ĠTra": 4876, - "Ġrelax": 4877, - "ufact": 4878, - "éģį": 4879, - "Ġsurv": 4880, - "åı£åij³": 4881, - "Ġcreativity": 4882, - "of": 4883, - "å¨ģ": 4884, - "çļĦçł": 4885, - "Ġbreath": 4886, - "Ġplaces": 4887, - "Ġdescrib": 4888, - "èĭ±è¯Ń": 4889, - "Ġdamage": 4890, - "oration": 4891, - "为æĤ¨": 4892, - "ift": 4893, - "Ġcase": 4894, - "å¹´é¾Ħ": 4895, - "Ġpress": 4896, - "çĶľ": 4897, - "éĩİ": 4898, - "æĹħ游": 4899, - "Ġtaken": 4900, - "ined": 4901, - "Ġconcept": 4902, - "æĴŃ": 4903, - "Ġinteresting": 4904, - "è·µ": 4905, - "Ġsea": 4906, - "60": 4907, - "Ġfoot": 4908, - "ĠName": 4909, - "Ġresearchers": 4910, - "éĢģ": 4911, - "Ġwee": 4912, - ");": 4913, - "çļĦåħ³éĶ®": 4914, - "ä¼½": 4915, - "elebr": 4916, - "å¡ij": 4917, - "We": 4918, - "ç»ı常": 4919, - "Ġpopulations": 4920, - "åħ¬å¼ı": 4921, - "orn": 4922, - "çĩĥ": 4923, - "人çĶŁ": 4924, - "17": 4925, - "æİ¥åıĹ": 4926, - "Ġlocation": 4927, - "Ġinequ": 4928, - "Ġintervent": 4929, - "Ġinterested": 4930, - "Ġdefinitely": 4931, - "Ġassistance": 4932, - "è¿Ļä¸Ģ": 4933, - "åIJĪåIJĮ": 4934, - "ä¼ĺåĬ¿": 4935, - "çļĦå·¥ä½ľ": 4936, - "Ġ12": 4937, - "Ġmov": 4938, - "åģı": 4939, - "åŃĺåĤ¨": 4940, - "usive": 4941, - "æĹı": 4942, - "ï¼īï¼Į": 4943, - "Ġgas": 4944, - "Ġinterests": 4945, - "æ¸ħæĻ°": 4946, - "Ġgard": 4947, - "çĸ«": 4948, - "Ġsay": 4949, - "夫": 4950, - "ges": 4951, - "èIJ¨": 4952, - "ä¸ļåĬ¡": 4953, - "个æĢ§": 4954, - "åIJ¯": 4955, - "Ġengagement": 4956, - "Ġbig": 4957, - "éľĢè¦ģèĢĥèĻij": 4958, - "Ġprinci": 4959, - "åij¨åĽ´": 4960, - "Ġopportunity": 4961, - "çģ¾": 4962, - "èĹı": 4963, - "rel": 4964, - "缺çĤ¹": 4965, - "Ġhappy": 4966, - "åĴĮåħ¶ä»ĸ": 4967, - "ava": 4968, - "Ġestablish": 4969, - "鸡èĽĭ": 4970, - "iking": 4971, - "ĠTrans": 4972, - "rastructure": 4973, - "forest": 4974, - "èİ·åıĸ": 4975, - "èĦļ": 4976, - "inally": 4977, - "èµı": 4978, - "Ġdelicious": 4979, - "Ġresults": 4980, - "è§Ĥå¯Ł": 4981, - "å®ŀè·µ": 4982, - "Ġlast": 4983, - "Ġpolit": 4984, - "æĢ§èĥ½": 4985, - "For": 4986, - "bi": 4987, - "çĽ¸ä¿¡": 4988, - "ffee": 4989, - "Ġphr": 4990, - "Ġforest": 4991, - "elling": 4992, - "æµģè¡Į": 4993, - "atic": 4994, - "大家": 4995, - "ĠInst": 4996, - "æķ°åѦ": 4997, - "æī©": 4998, - "å®Įåħ¨": 4999, - "å¼ķèµ·": 5000, - "ese": 5001, - "转æį¢": 5002, - "Ġaffected": 5003, - "Ġrobotics": 5004, - "综ä¸Ĭ": 5005, - "Ġprop": 5006, + "Ġare": 732, + "æĦı": 733, + "æ¬": 734, + "çļĦç": 735, + "Ġha": 736, + "pp": 737, + "åī": 738, + "帮åĬ©": 739, + "ain": 740, + "te": 741, + "æĬĢæľ¯": 742, + "å®ī": 743, + "ython": 744, + "ure": 745, + "éĹ®é¢ĺ": 746, + "å±ķ": 747, + "ĠĠĠĠ": 748, + "çĤ¹": 749, + "The": 750, + "éĶ": 751, + "ä¾ĭ": 752, + "çĿ": 753, + "Ġ+": 754, + "æģ¯": 755, + "æľį": 756, + "Ġr": 757, + "éĢī": 758, + "模": 759, + "ä½Ĩ": 760, + "è¯Ń": 761, + "Ġse": 762, + "åĽ½": 763, + "åĨħ": 764, + "Ġme": 765, + "åĪĽ": 766, + "绣": 767, + "åŃ¦ä¹ł": 768, + "Ġby": 769, + "éķ": 770, + "her": 771, + "èĤ": 772, + "æĪij们": 773, + "ight": 774, + "ä¹ĭ": 775, + "ï¼Ł": 776, + "æĦŁ": 777, + "ç±»": 778, + "æ±Ĥ": 779, + "æ´»": 780, + "ess": 781, + "è¿ĻäºĽ": 782, + "éª": 783, + "éĽ": 784, + "ad": 785, + "åħ¬": 786, + "Ġde": 787, + "for": 788, + "Ġal": 789, + "产": 790, + "ment": 791, + "ç¼": 792, + "ç½ij": 793, + "ĠThe": 794, + "å¿ĥ": 795, + "Ġwe": 796, + "并": 797, + "ç¬": 798, + "æīĢ": 799, + "缸": 800, + "Ġ\\(": 801, + "çķ": 802, + "åĻ": 803, + "åĿ": 804, + "æľŁ": 805, + "int": 806, + "æ·": 807, + "Ġ`": 808, + "ri": 809, + "nd": 810, + "ant": 811, + "ase": 812, + "ç»ı": 813, + "çݯ": 814, + "è¿Ľè¡Į": 815, + "ç»ĵ": 816, + "èĬ": 817, + "å¹³": 818, + "çĶµ": 819, + "ç©": 820, + "Ġnumb": 821, + "éĢĤ": 822, + "lp": 823, + "oo": 824, + "ction": 825, + "交": 826, + "åĻ¨": 827, + "åıĺ": 828, + "æłĩ": 829, + "主": 830, + "æº": 831, + "us": 832, + "ple": 833, + "age": 834, + "å¤Ħ": 835, + "çĿĢ": 836, + "æľįåĬ¡": 837, + "éĥ": 838, + "ä¿¡æģ¯": 839, + "xt": 840, + "Ġprov": 841, + "æİ¥": 842, + "Ġhelp": 843, + "èµĦ": 844, + "åĨ³": 845, + "Ġus": 846, + "å½±": 847, + "Ġle": 848, + "æĤ¨": 849, + "以ä¸ĭ": 850, + "æĺİ": 851, + "éĩĮ": 852, + "çľ": 853, + "è·": 854, + "Ġsu": 855, + "æĥħ": 856, + "éĥ½": 857, + "åľº": 858, + "ç¡®ä¿Ŀ": 859, + "ä»İ": 860, + "éķ¿": 861, + "Wh": 862, + "çĶ¨æĪ·": 863, + "éĥ¨": 864, + "rom": 865, + "ort": 866, + "åıĬ": 867, + "èģ": 868, + "ĠM": 869, + "ity": 870, + "å¤Ł": 871, + "åĪĹ": 872, + "å¾Ĺ": 873, + "Ġhave": 874, + "pt": 875, + "form": 876, + "åŁº": 877, + "å®īåħ¨": 878, + "åĢ¼": 879, + "è°ĥ": 880, + "æĭ¬": 881, + "èĻ": 882, + "ific": 883, + "ive": 884, + "社": 885, + "çłģ": 886, + "ter": 887, + "ä½ķ": 888, + "res": 889, + "Ġch": 890, + "éĢīæĭ": 891, + "ck": 892, + "Ġqu": 893, + "éĢīæĭ©": 894, + "计ç®Ĺ": 895, + "é¦": 896, + "æºIJ": 897, + "åĮħæĭ¬": 898, + "å½ĵ": 899, + "Ġ5": 900, + "end": 901, + "管": 902, + "çİĩ": 903, + "è¿Ļ个": 904, + "æıIJé«ĺ": 905, + "ame": 906, + "ç³»ç»Ł": 907, + "åºĶçĶ¨": 908, + "è¿ĺ": 909, + "æķ´": 910, + "æŶéĹ´": 911, + "è®®": 912, + "art": 913, + "Ġk": 914, + "ould": 915, + "ind": 916, + "ä½į": 917, + "ä»»": 918, + "æ¯ı": 919, + "è´¨": 920, + "åĵį": 921, + "ue": 922, + "ä¸ĢäºĽ": 923, + "å¤į": 924, + "ies": 925, + "ial": 926, + "Ġpo": 927, + "äºĭ": 928, + "å¸Ĥ": 929, + "èī²": 930, + "åŃĺ": 931, + "Ġ4": 932, + "åĮº": 933, + "èĥ½å¤Ł": 934, + "示": 935, + "å¹´": 936, + "è¢": 937, + "åīį": 938, + "åIJį": 939, + "ç´ł": 940, + "ç»Ħ": 941, + "ang": 942, + "æ¡": 943, + "ence": 944, + "ĠF": 945, + "Ġspe": 946, + "Ġwor": 947, + "ich": 948, + "æ¶": 949, + "éħ": 950, + "å¢ŀ": 951, + "æ°´": 952, + "æŃ¥": 953, + "å¢ĥ": 954, + "out": 955, + "æŀĦ": 956, + "ĠE": 957, + "å¤ĦçIJĨ": 958, + "ill": 959, + "请": 960, + "Ġfrom": 961, + "å°ij": 962, + "éĩįè¦ģ": 963, + "ere": 964, + "项": 965, + "Ġco": 966, + "æ¯Ķ": 967, + "æŃ£": 968, + "ä¸įåIJĮ": 969, + "åĩ½": 970, + "åĪ©": 971, + "éĤ": 972, + "被": 973, + "åĩĨ": 974, + "Ġab": 975, + "æĶ¯": 976, + "ä»ħ": 977, + "ĠB": 978, + "æł¹": 979, + "è¯Ĩ": 980, + "ä¹Ī": 981, + "导": 982, + "åħĥ": 983, + "åĩ½æķ°": 984, + "ä¼ĺ": 985, + "\\)": 986, + "ions": 987, + "Ġ$": 988, + "æĹł": 989, + "注": 990, + "éĿŀ": 991, + "pl": 992, + "天": 993, + "æŁ¥": 994, + "æĬ¤": 995, + "æµģ": 996, + "æµĭ": 997, + "è§Ħ": 998, + "his": 999, + "çĸ": 1000, + "æĺ¯ä¸Ģ个": 1001, + "ĠAl": 1002, + "Ġthis": 1003, + "ĠW": 1004, + "ult": 1005, + "åºı": 1006, + "线": 1007, + "è¯ķ": 1008, + "åıijå±ķ": 1009, + "å½±åĵį": 1010, + "###": 1011, + "ç¾": 1012, + "ç½®": 1013, + "åŁŁ": 1014, + "åıĸ": 1015, + "ine": 1016, + "Ġprovide": 1017, + "é£İ": 1018, + "å½¢": 1019, + "Ġspec": 1020, + "ä¼ł": 1021, + "å¤ĩ": 1022, + "强": 1023, + "ike": 1024, + "act": 1025, + "åħ·ä½ĵ": 1026, + "Ġnot": 1027, + "iz": 1028, + "容": 1029, + "low": 1030, + "Ġass": 1031, + "é¢Ĩ": 1032, + "éªĮ": 1033, + "Ġma": 1034, + "åĩı": 1035, + "Ġmore": 1036, + "åķ": 1037, + "管çIJĨ": 1038, + "Ġyour": 1039, + "æĢģ": 1040, + "Ġdo": 1041, + "ä¸ŃçļĦ": 1042, + "æ»": 1043, + "ĠG": 1044, + "Ġwhich": 1045, + "æŀIJ": 1046, + "Ġte": 1047, + "Ġnumber": 1048, + "able": 1049, + "è¿°": 1050, + "aba": 1051, + "è¾ĵ": 1052, + "æĹ¥": 1053, + "æł¹æį®": 1054, + "åĽŀ": 1055, + "çŃĸ": 1056, + "Ġen": 1057, + "ĠD": 1058, + "éĵ": 1059, + "Ġpr": 1060, + "è¨": 1061, + "çݯå¢ĥ": 1062, + "ibaba": 1063, + "æĸ¹æ³ķ": 1064, + "åĪ«": 1065, + "åįķ": 1066, + "per": 1067, + "rou": 1068, + "Ł³": 1069, + "Ġneed": 1070, + "è¿IJ": 1071, + "ring": 1072, + "æł¼": 1073, + "put": 1074, + "ä¸įä»ħ": 1075, + "ust": 1076, + "æ£": 1077, + "ite": 1078, + "ance": 1079, + "æĻº": 1080, + "éĽĨ": 1081, + "åıĹ": 1082, + "æѤ": 1083, + "our": 1084, + "å¤ĸ": 1085, + "代çłģ": 1086, + "ç§ij": 1087, + "åIJij": 1088, + "èĬĤ": 1089, + "éĢļ常": 1090, + "设计": 1091, + "å§": 1092, + "are": 1093, + "æ·±": 1094, + "ated": 1095, + "ical": 1096, + "æŁ": 1097, + "Ġlike": 1098, + "oud": 1099, + "é¢Ħ": 1100, + "éĢŁ": 1101, + "èĢĥ": 1102, + "now": 1103, + "è¨Ģ": 1104, + "åĪĨæŀIJ": 1105, + "ans": 1106, + "åĽ¾": 1107, + "ĊĠĠĠĠĠĠ": 1108, + "个人": 1109, + "cl": 1110, + "Ġwas": 1111, + "åĬŁ": 1112, + "av": 1113, + "ĠHow": 1114, + "çŁ¥": 1115, + "ç»Ļ": 1116, + "æĮĩ": 1117, + "åijĺ": 1118, + "æ®": 1119, + "åıĤ": 1120, + "ç»Ń": 1121, + "Ġso": 1122, + "loud": 1123, + "ä¾ĭå¦Ĥ": 1124, + "æĥ³": 1125, + "ice": 1126, + "åIJĮæŶ": 1127, + "Ġsh": 1128, + "æł·": 1129, + "cc": 1130, + "ud": 1131, + "论": 1132, + "åıį": 1133, + "çģ": 1134, + "模åŀĭ": 1135, + "ä¹ī": 1136, + "è¾¾": 1137, + "äºij": 1138, + "缴": 1139, + "çĥ": 1140, + "ĠAlibaba": 1141, + "èĥ": 1142, + "çIJĥ": 1143, + "Ġtr": 1144, + "å·±": 1145, + "What": 1146, + "æķĻ": 1147, + "unction": 1148, + "Ġany": 1149, + "åĨµ": 1150, + "æīĭ": 1151, + "Ġassist": 1152, + "èĩªå·±": 1153, + "ast": 1154, + "\\[": 1155, + "ary": 1156, + "åĽ¢": 1157, + "å·´": 1158, + "å·¥ä½ľ": 1159, + "符": 1160, + "建议": 1161, + "è±": 1162, + "ag": 1163, + "éĻħ": 1164, + "Ġint": 1165, + "èį": 1166, + "次": 1167, + "æĶ¹": 1168, + "ãĤ": 1169, + "æī¾": 1170, + "ç¥": 1171, + "og": 1172, + "åħ±": 1173, + "ime": 1174, + "ap": 1175, + "formation": 1176, + "Ġ0": 1177, + "Ġres": 1178, + "åģ¥": 1179, + "ä»·": 1180, + "两": 1181, + "æĻ¯": 1182, + "è¿ĩç¨ĭ": 1183, + "åİŁ": 1184, + "å¦Ĥä½ķ": 1185, + "è§Ĥ": 1186, + "ther": 1187, + "æĸ¹å¼ı": 1188, + "èIJ": 1189, + "Ġ20": 1190, + "ç«ĭ": 1191, + "转": 1192, + "Ġ'": 1193, + "å¼ķ": 1194, + "æ°Ķ": 1195, + "Ġabout": 1196, + "Ġspecific": 1197, + "ĠL": 1198, + "å®ŀçİ°": 1199, + "Ġad": 1200, + "çIJĨ解": 1201, + "让": 1202, + "æĺ¯ä¸Ģ": 1203, + "æľª": 1204, + "çĶ¨äºİ": 1205, + "ç¼ĸ": 1206, + "lease": 1207, + "Ġim": 1208, + "é¢ĨåŁŁ": 1209, + "æݨ": 1210, + "éĺ¿": 1211, + "Ġknow": 1212, + "è§Ĩ": 1213, + "ä½łçļĦ": 1214, + "解åĨ³": 1215, + "绾": 1216, + "Ġinformation": 1217, + "10": 1218, + "Ġpl": 1219, + "åĩıå°ij": 1220, + "åħī": 1221, + "å®Į": 1222, + "text": 1223, + "åħĪ": 1224, + "Ġar": 1225, + "åĨĻ": 1226, + "éļı": 1227, + "æĶ¶": 1228, + "ç½ij绾": 1229, + "æĻºèĥ½": 1230, + "ĠN": 1231, + "éĺŁ": 1232, + "客": 1233, + "èµ·": 1234, + "Ġfunction": 1235, + "ĠJ": 1236, + "king": 1237, + "éĤ£": 1238, + "ore": 1239, + "çĶ±": 1240, + "Ġif": 1241, + "ations": 1242, + "ĠR": 1243, + "康": 1244, + "åĪĻ": 1245, + "ĠCh": 1246, + "éľĢæ±Ĥ": 1247, + "Ġat": 1248, + "åħį": 1249, + "ener": 1250, + "æĶ¾": 1251, + "æĶ¯æĮģ": 1252, + "Ġup": 1253, + "ç²": 1254, + "æľĽ": 1255, + "ä¸ĵ": 1256, + "说": 1257, + "ä¿ĿæĮģ": 1258, + "ount": 1259, + "çĬ": 1260, + "åİĨ": 1261, + "urn": 1262, + "Ġwhe": 1263, + "ĠThis": 1264, + "éĩij": 1265, + "all": 1266, + "éģĵ": 1267, + "ach": 1268, + "so": 1269, + "Ġfe": 1270, + "éŁ³": 1271, + "ell": 1272, + "空": 1273, + "çłĶ": 1274, + "ks": 1275, + "åķĨ": 1276, + "åį³": 1277, + "è§Ĵ": 1278, + "ç¾İ": 1279, + "åģ¥åº·": 1280, + "社ä¼ļ": 1281, + "è¯Ńè¨Ģ": 1282, + "tain": 1283, + "å£": 1284, + "ange": 1285, + "èº": 1286, + "ata": 1287, + "èĦ": 1288, + "ign": 1289, + "Ġhas": 1290, + "è®°": 1291, + "ect": 1292, + "ä¹IJ": 1293, + "èİ": 1294, + "çĹ": 1295, + "身": 1296, + "Ġret": 1297, + "çĶŁæ´»": 1298, + "ile": 1299, + "æݧ": 1300, + "ous": 1301, + "ç»´": 1302, + "éĴ": 1303, + "å§ĭ": 1304, + "éĺ¿éĩĮ": 1305, + "产åĵģ": 1306, + "éĿŀ常": 1307, + "è£": 1308, + "erm": 1309, + "åĸ": 1310, + "ç®Ģ": 1311, + "os": 1312, + "ĠIf": 1313, + "å¸Ĥåľº": 1314, + "Ġj": 1315, + "å¿«": 1316, + "段": 1317, + "od": 1318, + "Ġser": 1319, + "pr": 1320, + "äºĨ解": 1321, + "Ġ6": 1322, + "åįĩ": 1323, + "å°±": 1324, + "èµĦæºIJ": 1325, + "èĩ´": 1326, + "缮æłĩ": 1327, + "åı°": 1328, + "rac": 1329, + "象": 1330, + "主è¦ģ": 1331, + "åIJ¦": 1332, + "æĸĻ": 1333, + "èĥ½åĬĽ": 1334, + "æĸŃ": 1335, + "çľĭ": 1336, + "à¸": 1337, + "Ġcre": 1338, + "AI": 1339, + "du": 1340, + "æ£Ģ": 1341, + "Ġits": 1342, + "ĠIt": 1343, + "以åıĬ": 1344, + "è¯ģ": 1345, + "ä»ĸ们": 1346, + "ĠCloud": 1347, + "ail": 1348, + "python": 1349, + "对äºİ": 1350, + "éĻIJ": 1351, + "ub": 1352, + "ä¼ģ": 1353, + "20": 1354, + "åı¸": 1355, + "游": 1356, + "æĵ": 1357, + "é£Ł": 1358, + "满": 1359, + "æĸĩåĮĸ": 1360, + "()": 1361, + "Ġbut": 1362, + "ç»Ĩ": 1363, + "ï¼ģ": 1364, + "å¸Į": 1365, + "Ġmy": 1366, + "ip": 1367, + "ä¼ģä¸ļ": 1368, + "èģĶ": 1369, + "ĠO": 1370, + "é¡¹çĽ®": 1371, + "æķ°åŃĹ": 1372, + "çķĮ": 1373, + "éģ¿": 1374, + "åĩł": 1375, + "}{": 1376, + "ong": 1377, + "ä¿ĿæĬ¤": 1378, + "æĪĺ": 1379, + "è¯Ħ": 1380, + "ä»Ģ": 1381, + "ree": 1382, + "æĢ»": 1383, + "积": 1384, + "该": 1385, + "éħį": 1386, + "ĊĠĠĠĠĠĠĠ": 1387, + "ass": 1388, + "ever": 1389, + "Ġx": 1390, + "ces": 1391, + "æĽ´åĬł": 1392, + "Ġquest": 1393, + "éĢł": 1394, + "Ġgener": 1395, + "æĿ¡": 1396, + "frac": 1397, + "Ġreturn": 1398, + "èĮ": 1399, + "ie": 1400, + "Ġ\\)": 1401, + "çĬ¶": 1402, + "ä»Ģä¹Ī": 1403, + "æ¡Ī": 1404, + "ä¸ŃåĽ½": 1405, + "Ġun": 1406, + "åIJ«": 1407, + "é¦ĸ": 1408, + "广": 1409, + "æĺĵ": 1410, + "ä¹ĭéĹ´": 1411, + "åĽ¢éĺŁ": 1412, + "èĪ": 1413, + "Ġhow": 1414, + "vel": 1415, + "Ġsc": 1416, + "Ġfind": 1417, + "Ġuse": 1418, + "é©": 1419, + "没": 1420, + "Ġ10": 1421, + "'m": 1422, + "å¯Ĩ": 1423, + "çĶŁæĪIJ": 1424, + "èĻij": 1425, + "port": 1426, + "oc": 1427, + "'re": 1428, + "åħ¬åı¸": 1429, + "åĬŁèĥ½": 1430, + "æķħ": 1431, + "Ġincl": 1432, + "ard": 1433, + "ates": 1434, + "èĤ²": 1435, + "åħĥç´ł": 1436, + "éĻ©": 1437, + "Ġval": 1438, + "åĮ»": 1439, + "æķĪçİĩ": 1440, + "ä¸ī": 1441, + "èĩªçĦ¶": 1442, + "com": 1443, + "æĢ§åĴĮ": 1444, + "'t": 1445, + "æį¢": 1446, + "æ¸ħ": 1447, + "Ġapp": 1448, + "ç´¢": 1449, + "éģ¿åħį": 1450, + "æĿĤ": 1451, + "Ġ[": 1452, + "Ġdes": 1453, + "æĿĥ": 1454, + "å®ĥ们": 1455, + "足": 1456, + "åĨħ容": 1457, + "èĭ": 1458, + "Ġlist": 1459, + "åIJĦ": 1460, + "æĺ¯åIJ¦": 1461, + "æĸ¹éĿ¢": 1462, + "Ġequ": 1463, + "没æľī": 1464, + "ĠChin": 1465, + "çĭ": 1466, + "æİĴ": 1467, + "åĪĴ": 1468, + "çĻ": 1469, + "æīĢæľī": 1470, + "认": 1471, + "éĶ®": 1472, + "äºĴ": 1473, + "è¯į": 1474, + "ire": 1475, + "åij³": 1476, + "æĸ½": 1477, + "è®Ń": 1478, + "æľīæķĪ": 1479, + "é¥": 1480, + "Ġfol": 1481, + "ari": 1482, + "带": 1483, + "æ´»åĬ¨": 1484, + "è´¹": 1485, + "In": 1486, + "æŀģ": 1487, + "ä½İ": 1488, + "ace": 1489, + "ely": 1490, + "Ġcode": 1491, + "åħ¶ä»ĸ": 1492, + "irst": 1493, + "Ġwill": 1494, + "ä¼ĺåĮĸ": 1495, + "ç»ĥ": 1496, + "å¼Ģåıij": 1497, + "车": 1498, + "享": 1499, + "Ġmost": 1500, + "Ġsom": 1501, + "Ġfollow": 1502, + "Ġdet": 1503, + "çĪ": 1504, + "å¤įæĿĤ": 1505, + "op": 1506, + "åºĵ": 1507, + "Ġform": 1508, + "书": 1509, + "æĢĿ": 1510, + "究": 1511, + "iven": 1512, + "å¸ĮæľĽ": 1513, + "åĪĹ表": 1514, + "Ġcomm": 1515, + "è¿Ļç§į": 1516, + "åĿĹ": 1517, + "gh": 1518, + "è½»": 1519, + "åŃĹ符": 1520, + "çķ¥": 1521, + "ep": 1522, + "Ġacc": 1523, + "ĠHowever": 1524, + "ally": 1525, + "è¾ĥ": 1526, + "åij½": 1527, + "åĽłä¸º": 1528, + "趣": 1529, + "ç¦": 1530, + "èĢĥèĻij": 1531, + "è¯Ŀ": 1532, + "ord": 1533, + "Ġmod": 1534, + "ĠU": 1535, + "Ġcomp": 1536, + "ãĥ": 1537, + "Ġ7": 1538, + "èİ·": 1539, + "åĥ": 1540, + "å¢ŀåĬł": 1541, + "įä½ľ": 1542, + "ĠPython": 1543, + "è·¯": 1544, + "ac": 1545, + "åijĬ": 1546, + "åıª": 1547, + "以ä¸ĭæĺ¯": 1548, + "ä½łåı¯ä»¥": 1549, + "çłĶ究": 1550, + "è°ĥæķ´": 1551, + "Ġro": 1552, + "Ġdata": 1553, + "çĸĹ": 1554, + "读": 1555, + "å¾ĭ": 1556, + "ry": 1557, + "è§ģ": 1558, + "print": 1559, + "è£ħ": 1560, + "ample": 1561, + "\\]": 1562, + "Ġsent": 1563, + "Ġmight": 1564, + "ä¿ĥ": 1565, + "çīĩ": 1566, + "ħ§": 1567, + "Ġ\\]": 1568, + "æĮģç»Ń": 1569, + ".\"": 1570, + "åı¯èĥ½ä¼ļ": 1571, + "åı£": 1572, + "å¸ĥ": 1573, + "å¾Ī": 1574, + "åı·": 1575, + "éĵ¾": 1576, + "ello": 1577, + "çī¹å®ļ": 1578, + "æĸĩ件": 1579, + "--": 1580, + "èĩ³": 1581, + "ãĢĬ": 1582, + "æĪı": 1583, + "ãĢĭ": 1584, + "urre": 1585, + "åĥı": 1586, + "ems": 1587, + "åħ´": 1588, + "ä¸ĸ": 1589, + "è´¨éĩı": 1590, + "æ¼": 1591, + "ress": 1592, + "éĻ¤": 1593, + "æĥħåĨµ": 1594, + "Ġ{": 1595, + "Ġ8": 1596, + "çĦ¶åIJİ": 1597, + "å¦Ĥæŀľä½ł": 1598, + "åı¥": 1599, + "Python": 1600, + "ph": 1601, + "Hello": 1602, + "æŁIJ": 1603, + "å¹³åı°": 1604, + "æĶ¿": 1605, + "ertain": 1606, + "å½ķ": 1607, + "imes": 1608, + "Ġpart": 1609, + "ound": 1610, + "wen": 1611, + "Ġnumbers": 1612, + "Ġper": 1613, + "åĪĽå»º": 1614, + "sw": 1615, + "åıĺåĮĸ": 1616, + "fere": 1617, + "å·¥åħ·": 1618, + "Ġnew": 1619, + "éĺ²": 1620, + "Ġwhat": 1621, + "angu": 1622, + "Ġone": 1623, + "æķĻèĤ²": 1624, + "Ġserv": 1625, + "Ġname": 1626, + "ff": 1627, + "Ġ#": 1628, + "ç®Ĺæ³ķ": 1629, + "误": 1630, + "åħħ": 1631, + "ition": 1632, + "con": 1633, + "one": 1634, + "Ġtheir": 1635, + "å®ŀéĻħ": 1636, + "ack": 1637, + "ft": 1638, + "é»": 1639, + "ram": 1640, + "åĪ©çĶ¨": 1641, + "æīĵ": 1642, + "æĵįä½ľ": 1643, + "Ġsol": 1644, + "æ¶Ī": 1645, + "称": 1646, + "):": 1647, + "æĪĸèĢħ": 1648, + "Ġother": 1649, + "Ġall": 1650, + "Ġusing": 1651, + "ript": 1652, + "Qwen": 1653, + "çļĦæĺ¯": 1654, + "ear": 1655, + "ç»ĵæŀľ": 1656, + "ä¿ĥè¿Ľ": 1657, + "Ġcl": 1658, + "æľªæĿ¥": 1659, + "ak": 1660, + "æݧåĪ¶": 1661, + "ory": 1662, + "çŁ": 1663, + "Ġlangu": 1664, + "Ġsy": 1665, + "åĽłæѤ": 1666, + "éĢĤåIJĪ": 1667, + "vers": 1668, + "Ġinto": 1669, + "Ġinte": 1670, + "To": 1671, + "Ġman": 1672, + "温": 1673, + "好çļĦ": 1674, + "æľīåĬ©": 1675, + "23": 1676, + "ç»ĵæŀĦ": 1677, + "æĿIJ": 1678, + "èIJ¥": 1679, + "Ġvari": 1680, + "æľīåĬ©äºİ": 1681, + "Ġem": 1682, + "Ġcol": 1683, + "Ġcont": 1684, + "ng": 1685, + "ç«Ļ": 1686, + "æĸ°çļĦ": 1687, + "æıı": 1688, + "ish": 1689, + "Ġoff": 1690, + "询": 1691, + "åİ»": 1692, + "缸åħ³": 1693, + "æĺ¾": 1694, + "Ġout": 1695, + "ased": 1696, + "èŀ": 1697, + "rite": 1698, + "æµİ": 1699, + "æĺŁ": 1700, + "Ġbet": 1701, + "æķħäºĭ": 1702, + "åı²": 1703, + "Ġansw": 1704, + "为äºĨ": 1705, + "èĩªå·±çļĦ": 1706, + "Ġworld": 1707, + "设å¤ĩ": 1708, + "Ġwould": 1709, + "Ġfirst": 1710, + "æĬķ": 1711, + "第": 1712, + "Ġev": 1713, + "åģļ": 1714, + "æ£ĢæŁ¥": 1715, + "ç»ıæµİ": 1716, + "Ġty": 1717, + "ä¸ĵä¸ļ": 1718, + "åĶ": 1719, + "帮åĬ©ä½ł": 1720, + "ĠÐ": 1721, + "软": 1722, + "缴æİ¥": 1723, + "èij": 1724, + "å¼Ģå§ĭ": 1725, + "ia": 1726, + "Ġinclud": 1727, + "æľĪ": 1728, + "ç²¾": 1729, + "åĽ´": 1730, + "ments": 1731, + "ature": 1732, + "éĩĬ": 1733, + "Ġterm": 1734, + "åĤ¨": 1735, + "urrent": 1736, + "ru": 1737, + "__": 1738, + "ç±»åŀĭ": 1739, + "ä»»åĬ¡": 1740, + "Ġ*": 1741, + "åĵª": 1742, + "Ġcal": 1743, + "Ġalso": 1744, + "éĶĻ": 1745, + "Ġam": 1746, + "we": 1747, + "ä½ľä¸º": 1748, + "Ġanswer": 1749, + "念": 1750, + "游æĪı": 1751, + "æľºåĻ¨": 1752, + "day": 1753, + "Ġlet": 1754, + "Ġused": 1755, + "çĹħ": 1756, + "åIJ¸": 1757, + "两个": 1758, + "æī¾åĪ°": 1759, + "级": 1760, + "Ġsum": 1761, + "åħ³ç³»": 1762, + "ne": 1763, + "注æĦı": 1764, + "Ġplease": 1765, + "Ġeach": 1766, + "çļĦæĥ": 1767, + "导èĩ´": 1768, + "é¢ij": 1769, + "ç¥ŀ": 1770, + "å¤ļ个": 1771, + "Ġsp": 1772, + "便": 1773, + "èŀį": 1774, + "Ġexp": 1775, + "è¾ĵåħ¥": 1776, + "åŁİ": 1777, + "è¿ŀ": 1778, + "æıIJåįĩ": 1779, + "ten": 1780, + "åİĭ": 1781, + "eth": 1782, + "åħ³éĶ®": 1783, + "éĥ¨åĪĨ": 1784, + "çŃĸçķ¥": 1785, + "Ġref": 1786, + "cul": 1787, + "å±Ĥ": 1788, + "æ¢": 1789, + "Ġsentence": 1790, + "çĮ": 1791, + "许": 1792, + "éĩĩ": 1793, + "骤": 1794, + "ning": 1795, + "ä½ĵéªĮ": 1796, + "è¿ľ": 1797, + "ify": 1798, + "ertainly": 1799, + "表达": 1800, + "æĽ´å¤ļ": 1801, + "lic": 1802, + "ä¼°": 1803, + "å·²": 1804, + "åĪĽæĸ°": 1805, + "Ġche": 1806, + "ating": 1807, + "éĶĢ": 1808, + "èĩªåĬ¨": 1809, + "èīº": 1810, + "èı": 1811, + "\",": 1812, + "串": 1813, + "ç¢": 1814, + "ä¿®": 1815, + "æĤ": 1816, + "Ġ|": 1817, + "Ġwant": 1818, + "Ġcurrent": 1819, + "Ġexample": 1820, + "ç¨ĭåºı": 1821, + "欢": 1822, + "Ġlar": 1823, + "nder": 1824, + "æĺ¯ä¸Ģç§į": 1825, + "çĽij": 1826, + "份": 1827, + "Ġwhere": 1828, + "é£İéĻ©": 1829, + "Ġprovid": 1830, + "ath": 1831, + "å¼Ĥ": 1832, + "å¯Į": 1833, + "çļĦæĬ": 1834, + "çĥŃ": 1835, + "æĢ§èĥ½": 1836, + "red": 1837, + "çĭ¬": 1838, + "åĩĨç¡®": 1839, + "æ¯ı个": 1840, + "äºĮ": 1841, + "æ°ij": 1842, + "ss": 1843, + "Ġquestions": 1844, + "æij": 1845, + "Ġdif": 1846, + "Ġbo": 1847, + "Ġsuch": 1848, + "æµ·": 1849, + "éĻį": 1850, + "Ġtext": 1851, + "åIJĪä½ľ": 1852, + "Ġloo": 1853, + "Ġtw": 1854, + "失": 1855, + "客æĪ·": 1856, + "è®Ńç»ĥ": 1857, + "éŨ": 1858, + "ijæĪĺ": 1859, + "ices": 1860, + "Ġ9": 1861, + "ä¸įåIJĮçļĦ": 1862, + "åĮħåIJ«": 1863, + "ject": 1864, + "æĴ": 1865, + "åĬ¿": 1866, + "å°Ŀ": 1867, + "ä»»ä½ķ": 1868, + "Ġmodel": 1869, + "ents": 1870, + "æİ¢": 1871, + "Ġchar": 1872, + "é¦ĸåħĪ": 1873, + "åİĨåı²": 1874, + "ä¸Ķ": 1875, + "人类": 1876, + "ĠHere": 1877, + "æĪIJæľ¬": 1878, + "å¾Ģ": 1879, + "date": 1880, + "ds": 1881, + "æIJ": 1882, + "Ġprint": 1883, + "èĬ±": 1884, + "ual": 1885, + "è´Ń": 1886, + "å¾®": 1887, + "ĠY": 1888, + "Ġtas": 1889, + "Ġtrans": 1890, + "ĠAI": 1891, + "è¿Ļæł·": 1892, + "å®ļæľŁ": 1893, + "æ¯Ķå¦Ĥ": 1894, + "olog": 1895, + "æĸĩæľ¬": 1896, + "设置": 1897, + "å·´å·´": 1898, + "éĺ¿éĩĮå·´å·´": 1899, + "au": 1900, + "Ġfollowing": 1901, + "è¾¹": 1902, + "Ġlanguage": 1903, + "Ġhere": 1904, + "Ġele": 1905, + "æııè¿°": 1906, + "les": 1907, + "éļ¾": 1908, + "èģĮ": 1909, + "ava": 1910, + "软件": 1911, + "о": 1912, + "约": 1913, + "Certainly": 1914, + "Ġcap": 1915, + "è¡Į为": 1916, + "Ġrange": 1917, + "¿Ģ": 1918, + "ative": 1919, + "ues": 1920, + ").": 1921, + "Ġtech": 1922, + "æ³Ľ": 1923, + "Ġfeel": 1924, + "ĊĠ": 1925, + "ĠSt": 1926, + "Can": 1927, + "000": 1928, + "bs": 1929, + "ful": 1930, + "It": 1931, + "å°½": 1932, + "åŃĺåĤ¨": 1933, + "æ²»": 1934, + "åIJĹ": 1935, + "æĬ¥": 1936, + "建ç«ĭ": 1937, + "åıĤä¸İ": 1938, + "å¿«éĢŁ": 1939, + "åĩ»": 1940, + "çª": 1941, + "è¯Ħä¼°": 1942, + "Ġunder": 1943, + "å°Ŀè¯ķ": 1944, + "ors": 1945, + "Ġtra": 1946, + "Ġ/": 1947, + "çļĦæĥħ": 1948, + "éļıçĿĢ": 1949, + "ä¸ĸçķĮ": 1950, + "Ġrec": 1951, + "æĮī": 1952, + "Ġtime": 1953, + "头": 1954, + "Ġstring": 1955, + "åħ³äºİ": 1956, + "åħ·ä½ĵçļĦ": 1957, + "è¾ĵåĩº": 1958, + "åŁºæľ¬": 1959, + "åħ·æľī": 1960, + "åĪĿ": 1961, + "çħ§": 1962, + "åĬ±": 1963, + "ä¸Ģç§į": 1964, + "åŃĹ符串": 1965, + "cript": 1966, + "èĮĥ": 1967, + "ä½łå¥½": 1968, + "Ġsim": 1969, + "以ä¸ĭæĺ¯ä¸ĢäºĽ": 1970, + "æŃ¥éª¤": 1971, + "æ¤": 1972, + "Ġ<": 1973, + "Ġvalue": 1974, + "Ġthere": 1975, + "ery": 1976, + "款": 1977, + "æ¿Ģ": 1978, + "eng": 1979, + "æĭŁ": 1980, + "Ġdis": 1981, + "ray": 1982, + "Ġcontext": 1983, + "åıĭ": 1984, + "å¾ģ": 1985, + "æĹħ": 1986, + "æµĭè¯ķ": 1987, + "Ġdiffere": 1988, + "ll": 1989, + "åħ´è¶£": 1990, + "ç§ijåѦ": 1991, + "åħ¨çIJĥ": 1992, + "Ġinter": 1993, + "èĥĮ": 1994, + "æĬĢæľ¯çļĦ": 1995, + "å®ĮæĪIJ": 1996, + "æĸ¹æ¡Ī": 1997, + "ital": 1998, + "ä¸įæĸŃ": 1999, + "ark": 2000, + "Ġph": 2001, + "çĽĬ": 2002, + "led": 2003, + "Ġi": 2004, + "æļ": 2005, + "ç¿": 2006, + "å·®": 2007, + "éļIJ": 2008, + "ber": 2009, + "ä¹ĭéĹ´çļĦ": 2010, + "ĠQwen": 2011, + "ays": 2012, + "ĠK": 2013, + "å¿ħ": 2014, + "éĤ£ä¹Ī": 2015, + "åij¨": 2016, + "å¹¿æ³Ľ": 2017, + "roup": 2018, + "erson": 2019, + "ics": 2020, + "çŁ¥è¯Ĩ": 2021, + "ç§ijæĬĢ": 2022, + "ese": 2023, + "æıIJä¾ĽäºĨ": 2024, + "å¸Ī": 2025, + "times": 2026, + "è´Ł": 2027, + "åħ»": 2028, + "Ġnum": 2029, + "ict": 2030, + "è¡Įä¸ļ": 2031, + "Ġtechn": 2032, + "éĶĻ误": 2033, + "çļĦæķ°æį®": 2034, + "è¿Ķ": 2035, + "èī¯": 2036, + "',": 2037, + "Ġcheck": 2038, + "),": 2039, + "ails": 2040, + "æĿ¡ä»¶": 2041, + "ä»·åĢ¼": 2042, + "çĪ±": 2043, + "è¿ij": 2044, + "Ġfree": 2045, + "gram": 2046, + "åħĭ": 2047, + "ç±³": 2048, + "Ġhis": 2049, + "åŃĺåľ¨": 2050, + "hen": 2051, + "ç®Ģåįķ": 2052, + "访": 2053, + "声": 2054, + "ular": 2055, + "gest": 2056, + "ĠFor": 2057, + "åı¤": 2058, + "ç§ģ": 2059, + "åįı": 2060, + "åŁºäºİ": 2061, + "æĽ´å¥½": 2062, + "Ġadd": 2063, + "As": 2064, + "è´£": 2065, + "æķ°ç»Ħ": 2066, + "Ġrel": 2067, + "å®ļä¹ī": 2068, + "çīĮ": 2069, + "她": 2070, + "è¯ĨåĪ«": 2071, + "人工": 2072, + "åŁºç¡": 2073, + "åŁºç¡Ģ": 2074, + "表示": 2075, + "çĶ»": 2076, + "离": 2077, + "Ġcomput": 2078, + "row": 2079, + "åºĶ该": 2080, + "ä»İèĢĮ": 2081, + "è¶Ĭ": 2082, + "è¿ĶåĽŀ": 2083, + "èijĹ": 2084, + "æĿIJæĸĻ": 2085, + "æŀĹ": 2086, + "Ġresult": 2087, + "æĺ¯åľ¨": 2088, + "ç¼ĸç¨ĭ": 2089, + "(\"": 2090, + "andom": 2091, + "def": 2092, + "解éĩĬ": 2093, + "åģĩ": 2094, + "Ġì": 2095, + "é¼": 2096, + "ï¼īï¼Į": 2097, + "éģĩ": 2098, + "ient": 2099, + "cial": 2100, + "ens": 2101, + "Ġtwo": 2102, + "寻": 2103, + "Ġsome": 2104, + "Ġthem": 2105, + "QL": 2106, + "ern": 2107, + "ä¸ĬçļĦ": 2108, + "Ġwho": 2109, + "Ġthey": 2110, + "ä½Ĩæĺ¯": 2111, + "å¢ŀ强": 2112, + "åı¯èĥ½éľĢè¦ģ": 2113, + "ä¼Ĺ": 2114, + "velop": 2115, + "åŁİå¸Ĥ": 2116, + "æĤ¨çļĦ": 2117, + "е": 2118, + "der": 2119, + "åĭ": 2120, + "åħ±åIJĮ": 2121, + "Ġtoday": 2122, + "å¡": 2123, + "Ġcharact": 2124, + "fore": 2125, + "åľºæĻ¯": 2126, + "åĽłç´ł": 2127, + "Ġseems": 2128, + "ble": 2129, + "移": 2130, + "羣": 2131, + "ather": 2132, + "rough": 2133, + "丰": 2134, + "numb": 2135, + "))": 2136, + "ater": 2137, + "cess": 2138, + "Ġset": 2139, + "çģµ": 2140, + "Ġover": 2141, + "ize": 2142, + "åŁ¹": 2143, + "è¦ģæ±Ĥ": 2144, + "åĿĩ": 2145, + "Ġdetails": 2146, + "æĬķèµĦ": 2147, + "èħ": 2148, + "ç»ĩ": 2149, + "ĠIn": 2150, + "ility": 2151, + "èĢģ": 2152, + "ween": 2153, + "å¤ļç§į": 2154, + "Ġequation": 2155, + "ible": 2156, + "rit": 2157, + "åı¯ä»¥éĢļè¿ĩ": 2158, + "Ġpop": 2159, + "Ġincluding": 2160, + "æĽ´å¥½åľ°": 2161, + "èīºæľ¯": 2162, + "its": 2163, + "Ġknown": 2164, + "Ġpre": 2165, + "ood": 2166, + "ç½ijç«Ļ": 2167, + "Ġprogram": 2168, + "çŃĶ": 2169, + "ities": 2170, + "转æį¢": 2171, + "Ġbetween": 2172, + "Ġtasks": 2173, + "éĺ¿éĩĮäºij": 2174, + "è¶ħ": 2175, + "ĠChina": 2176, + "示ä¾ĭ": 2177, + "çĻ½": 2178, + "太": 2179, + "ï¼īãĢĤ": 2180, + "ç¡®å®ļ": 2181, + "Ġ19": 2182, + "产çĶŁ": 2183, + "æŃ¢": 2184, + "Ġsign": 2185, + "ŁéĢļ": 2186, + "Ġyear": 2187, + "Ġpe": 2188, + "访éĹ®": 2189, + "ook": 2190, + "æī§": 2191, + "Ġdevelop": 2192, + "ï¼ī**ï¼ļ": 2193, + "Ġthan": 2194, + "duct": 2195, + "èĭ±": 2196, + "Ġthese": 2197, + "Ġperson": 2198, + "循": 2199, + "äºī": 2200, + "Ġcalcul": 2201, + "é¢Ħæµĭ": 2202, + "rib": 2203, + "æĮijæĪĺ": 2204, + "åIJ¬": 2205, + "Ġboth": 2206, + "Ġdon": 2207, + "12": 2208, + "tal": 2209, + "åıijçİ°": 2210, + "ys": 2211, + "**:": 2212, + "åĮºåĿĹ": 2213, + "çļĦå½±åĵį": 2214, + "å¨": 2215, + "Ġ12": 2216, + "çīĪ": 2217, + "stand": 2218, + "ç»Ī": 2219, + "åĸľ": 2220, + "list": 2221, + "ä¹°": 2222, + "åĨį": 2223, + "This": 2224, + "人工æĻºèĥ½": 2225, + "åĮ»çĸĹ": 2226, + "ä»ĺ": 2227, + "å±ŀ": 2228, + "æ¼Ķ": 2229, + "çļĦ人": 2230, + "çļĦæł": 2231, + "计åĪĴ": 2232, + "pon": 2233, + "è¿ĩç¨ĭä¸Ń": 2234, + "丰å¯Į": 2235, + "模å¼ı": 2236, + "åIJĦç§į": 2237, + "Ġwhen": 2238, + "积æŀģ": 2239, + "åĩĨå¤ĩ": 2240, + "ä¸Ĭè¿°": 2241, + "è¿IJè¡Į": 2242, + "æĪIJ为": 2243, + "è¯ī": 2244, + "è§ī": 2245, + "Ġtotal": 2246, + "èįIJ": 2247, + "æ¦Ĥ": 2248, + "é¸": 2249, + "Ġcons": 2250, + "éĺ³": 2251, + "ä½ľçĶ¨": 2252, + "åŃ©": 2253, + "ä¸įæĺ¯": 2254, + "é«ĺæķĪ": 2255, + "Ġtyp": 2256, + "ower": 2257, + "æ·»": 2258, + "讨": 2259, + "а": 2260, + "稳": 2261, + "åĨ³çŃĸ": 2262, + "to": 2263, + "éĿĻ": 2264, + "Ġra": 2265, + "åı¯ä»¥ä½¿çĶ¨": 2266, + "Ġdiv": 2267, + "带æĿ¥": 2268, + "è¾ij": 2269, + "åºŃ": 2270, + "è®°å½ķ": 2271, + "stem": 2272, + "Ġgre": 2273, + "çļĦæī": 2274, + "è¯Ĺ": 2275, + "Ġlast": 2276, + "Ġ100": 2277, + "åѦçĶŁ": 2278, + "端": 2279, + "Ġinteg": 2280, + "Ġgiven": 2281, + "Ġsub": 2282, + "ĠChinese": 2283, + "è¥": 2284, + "Ġadv": 2285, + "社交": 2286, + "令": 2287, + "Ġcould": 2288, + "Who": 2289, + "è¿IJåĬ¨": 2290, + "èıľ": 2291, + "atest": 2292, + "åıĤæķ°": 2293, + "pro": 2294, + "Ġvers": 2295, + "å®Ŀ": 2296, + "çļĦéĩįè¦ģ": 2297, + "èĻ½": 2298, + "ĠV": 2299, + "çĶŁçī©": 2300, + "Ġdifferent": 2301, + "å±±": 2302, + "ä½įç½®": 2303, + "西": 2304, + "ise": 2305, + "大çļĦ": 2306, + "é¦Ī": 2307, + "Ġgenerate": 2308, + "Ġthrough": 2309, + "Ġdoes": 2310, + "è´¢": 2311, + "ªæĸ½": 2312, + "è½½": 2313, + "19": 2314, + "umn": 2315, + "15": 2316, + "éĺħ": 2317, + "Ġind": 2318, + "详": 2319, + "Ġcount": 2320, + "ÑĤ": 2321, + "Sure": 2322, + "åĩłä¸ª": 2323, + "åĵģçīĮ": 2324, + "æ³ķå¾ĭ": 2325, + "æī§è¡Į": 2326, + "åħ¸": 2327, + "èĻ½çĦ¶": 2328, + "çģ«": 2329, + "èĥĮæĻ¯": 2330, + "éŁ³ä¹IJ": 2331, + "ä¸ļåĬ¡": 2332, + "æĸ¯": 2333, + "So": 2334, + "åħ¶ä¸Ń": 2335, + "éĢŁåº¦": 2336, + "ä¼¼": 2337, + "æݨèįIJ": 2338, + "Ġmult": 2339, + "åĽ½éĻħ": 2340, + "çĬ¶æĢģ": 2341, + "åĪ¶ä½ľ": 2342, + "Ġsqu": 2343, + "Ġtechnolog": 2344, + "æŀ¶": 2345, + "åıįé¦Ī": 2346, + "Ġstart": 2347, + "绿": 2348, + "è¡¡": 2349, + "Ġget": 2350, + "è§Ĵèī²": 2351, + "åĽ½å®¶": 2352, + "get": 2353, + "xim": 2354, + "Ġdesign": 2355, + "co": 2356, + "åį°": 2357, + "If": 2358, + "满足": 2359, + "ä¸Ģä¸ĭ": 2360, + "家åºŃ": 2361, + "ç»Ħç»ĩ": 2362, + "交æµģ": 2363, + "riend": 2364, + "çļĦæĦ": 2365, + "Ġmat": 2366, + "èİ·åıĸ": 2367, + "ĠCould": 2368, + "追": 2369, + "å¼ł": 2370, + "ob": 2371, + "ä¼łç»Ł": 2372, + "æłij": 2373, + "é¼ĵ": 2374, + "Ġunderstand": 2375, + "Ġcor": 2376, + "å¿ĥçIJĨ": 2377, + "æݪæĸ½": 2378, + "Ġgo": 2379, + "Ġhum": 2380, + "èį¯": 2381, + "交æĺĵ": 2382, + "round": 2383, + "Ġimport": 2384, + "çļĦä¸Ģ": 2385, + "èĻļ": 2386, + "稳å®ļ": 2387, + "ä»ĭ": 2388, + "è¿Ļä¸Ģ": 2389, + "åĩºçİ°": 2390, + "и": 2391, + "ational": 2392, + "nce": 2393, + "é¢Ŀ": 2394, + "cre": 2395, + "Ġno": 2396, + "Ġart": 2397, + "åĪĩ": 2398, + "åª": 2399, + "Ġel": 2400, + "常è§ģ": 2401, + "çļĦæĮ": 2402, + "éĩıåŃIJ": 2403, + "Ġke": 2404, + "åĽĽ": 2405, + "å²": 2406, + "å½Ĵ": 2407, + "社åĮº": 2408, + "Ġmain": 2409, + "è¿ŀæİ¥": 2410, + "åľ¨çº¿": 2411, + "饮": 2412, + "å½ĵçĦ¶": 2413, + "åĶ®": 2414, + "ç»´æĬ¤": 2415, + "çļĦå®īåħ¨": 2416, + "çŁŃ": 2417, + "è¯ij": 2418, + "Ġlatest": 2419, + "æĥħæĦŁ": 2420, + "ç«ŀ": 2421, + "ian": 2422, + "Ġthen": 2423, + "æ¶ī": 2424, + "æ¤į": 2425, + "éĩıçļĦ": 2426, + "Ġ2023": 2427, + "Ġrem": 2428, + "wor": 2429, + "å±Ģ": 2430, + "num": 2431, + "红": 2432, + "Ġbeen": 2433, + "ake": 2434, + "åħ³æ³¨": 2435, + "æĪIJåijĺ": 2436, + "ick": 2437, + "()`": 2438, + "详ç»Ĩ": 2439, + "ident": 2440, + "æłĩåĩĨ": 2441, + "Ġsignific": 2442, + "è¿ĻéĩĮ": 2443, + "sq": 2444, + "èĮĥåĽ´": 2445, + "ength": 2446, + "iness": 2447, + "æķĪæŀľ": 2448, + "å¹²": 2449, + "çİ©": 2450, + "Ġcomple": 2451, + "æ´ģ": 2452, + "æĬĢèĥ½": 2453, + "Ġoften": 2454, + "ages": 2455, + "çļĦæĶ": 2456, + "éļIJç§ģ": 2457, + "rest": 2458, + "Ġrefer": 2459, + "èĽ": 2460, + "æ³¢": 2461, + "éĥ½æĺ¯": 2462, + ");": 2463, + "åŃ©åŃIJ": 2464, + "ç´§": 2465, + "æħ": 2466, + "éĢı": 2467, + "å°Ķ": 2468, + "åĸĦ": 2469, + "hes": 2470, + "æĿ¾": 2471, + "ä¾Ŀ": 2472, + "åĢĻ": 2473, + "Ġservices": 2474, + "çĶµåŃIJ": 2475, + "æŃ£ç¡®": 2476, + "å·²ç»ı": 2477, + "æīį": 2478, + "çļĦæİ": 2479, + "æľºåĪ¶": 2480, + "çī¹åĪ«": 2481, + "åĪĽéĢł": 2482, + "使å¾Ĺ": 2483, + "åªĴ": 2484, + "11": 2485, + "Ġdep": 2486, + "Is": 2487, + "ä¸ĢæŃ¥": 2488, + "åĵªäºĽ": 2489, + "èģĶç³»": 2490, + "é¼ĵåĬ±": 2491, + "å¥ĩ": 2492, + "éĢĤåºĶ": 2493, + "ï¼Ľ": 2494, + "çļĦæĬĢ": 2495, + "个æĢ§": 2496, + "ç¿»": 2497, + "Ġjust": 2498, + "Ġbus": 2499, + "Ġreg": 2500, + "å½¢æĪIJ": 2501, + "ĠĠĠĠĠ": 2502, + "here": 2503, + "çªģ": 2504, + "éģĩåĪ°": 2505, + "âĢĿãĢĤ": 2506, + "Write": 2507, + "çļĦéĹ®é¢ĺ": 2508, + "åĪ»": 2509, + "lect": 2510, + "ä¹ĭä¸Ģ": 2511, + "å¾Ħ": 2512, + "人们": 2513, + "Ġfact": 2514, + "æ·»åĬł": 2515, + "åijĺå·¥": 2516, + "交éĢļ": 2517, + "ose": 2518, + "é¾": 2519, + "ove": 2520, + "ax": 2521, + "Ġvarious": 2522, + "Ġbased": 2523, + "åĬłåħ¥": 2524, + "cept": 2525, + "æĪIJåĬŁ": 2526, + "ĠGroup": 2527, + "åĮºåĿĹéĵ¾": 2528, + "50": 2529, + "åģı": 2530, + "çļĦä¿¡æģ¯": 2531, + "Ġoper": 2532, + "name": 2533, + "åIJ¸å¼ķ": 2534, + "æ¹": 2535, + "åį¡": 2536, + "ract": 2537, + "æݢ索": 2538, + "æĽ´æĸ°": 2539, + "Ġlear": 2540, + "空éĹ´": 2541, + "inal": 2542, + "æ¶īåıĬ": 2543, + "æ½": 2544, + "ural": 2545, + "ars": 2546, + "Ġmany": 2547, + "blem": 2548, + "Ġsk": 2549, + "è¿ĺèĥ½": 2550, + "ä¸ĭéĿ¢": 2551, + "æľīæīĢ": 2552, + "Ġcreate": 2553, + "æĺİç¡®": 2554, + "ince": 2555, + "æŀĦ建": 2556, + "ings": 2557, + "æĹłæ³ķ": 2558, + "éĢIJ": 2559, + "æĦıåij³": 2560, + "æĪ¿": 2561, + "å¾ĹåĪ°": 2562, + "人çļĦ": 2563, + "åªĴä½ĵ": 2564, + "âĢĿï¼Į": 2565, + "Ġsupp": 2566, + "è¿Ļæĺ¯": 2567, + "æĴŃ": 2568, + "old": 2569, + "åıĹåĪ°": 2570, + "çı": 2571, + "çĦ¶èĢĮ": 2572, + "pect": 2573, + "Ġsquare": 2574, + "计ç®Ĺæľº": 2575, + "ude": 2576, + "æѤå¤ĸ": 2577, + "è¿İ": 2578, + "ically": 2579, + "æİĴåºı": 2580, + "æľĢåIJİ": 2581, + "Ġprocess": 2582, + "ause": 2583, + "éĩijèŀį": 2584, + "害": 2585, + "Ġloc": 2586, + "åĪĨ享": 2587, + "æ²¹": 2588, + "Ġsm": 2589, + "çļĦæ°": 2590, + "课": 2591, + "ä»·æł¼": 2592, + "页": 2593, + "dition": 2594, + "åĪ¤": 2595, + "æŁIJäºĽ": 2596, + "Ġbl": 2597, + "æķ°æį®åºĵ": 2598, + ",\"": 2599, + "·": 2600, + "çī¹å¾ģ": 2601, + "éļľ": 2602, + "åģĩ设": 2603, + "åįĹ": 2604, + "Given": 2605, + "Ġexpress": 2606, + "åijĬè¯ī": 2607, + "Ġfil": 2608, + "åĽ¾åĥı": 2609, + "ci": 2610, + "Ġquestion": 2611, + "个æĢ§åĮĸ": 2612, + "ç¯": 2613, + "ced": 2614, + "ĠĊ": 2615, + "é¤": 2616, + "Ġpos": 2617, + "elf": 2618, + "å°¤": 2619, + "å¦ĤæŀľæĤ¨": 2620, + "æĪijåı¯ä»¥": 2621, + "å®ļçļĦ": 2622, + "èĦij": 2623, + "Ġlead": 2624, + "æĦıåij³çĿĢ": 2625, + "Ġproblem": 2626, + "å±ħ": 2627, + "ix": 2628, + "æŃĮ": 2629, + "Ġtable": 2630, + "è´¦": 2631, + "群": 2632, + "ited": 2633, + "åľ°åĮº": 2634, + "çĶŁäº§": 2635, + "çļĦåıijå±ķ": 2636, + "æĶ¶éĽĨ": 2637, + "èµĽ": 2638, + "åıĪ": 2639, + "审": 2640, + "ä¸Ģ次": 2641, + "Ġcommon": 2642, + "ople": 2643, + "å·§": 2644, + "ution": 2645, + "Ġappro": 2646, + "æĶ¿çŃĸ": 2647, + "é¤IJ": 2648, + "çݰ代": 2649, + "è§ĦåĪĴ": 2650, + "åŃ£": 2651, + "ä½ı": 2652, + "Ġtransl": 2653, + "Ġdef": 2654, + "Ġpar": 2655, + "å¾·": 2656, + "çݯä¿Ŀ": 2657, + "主é¢ĺ": 2658, + "Ġbusiness": 2659, + "表çİ°": 2660, + "ãģ®": 2661, + "èĤ¡": 2662, + "深度": 2663, + "Ġmake": 2664, + "æŁĵ": 2665, + "éĺħ读": 2666, + "ĠUn": 2667, + "éģµ": 2668, + "èIJ½": 2669, + "ĠEng": 2670, + "讲": 2671, + "å¾ħ": 2672, + "Ġever": 2673, + "Ġact": 2674, + "çļĦæ¸": 2675, + "解åĨ³æĸ¹æ¡Ī": 2676, + "Ġrandom": 2677, + "çļĦåºĶçĶ¨": 2678, + "èģĶç½ij": 2679, + "iqu": 2680, + "æ¶Īè´¹": 2681, + "irect": 2682, + "æľºåĻ¨åŃ¦ä¹ł": 2683, + "100": 2684, + "lish": 2685, + "Ġinv": 2686, + "åIJĪéĢĤ": 2687, + "rt": 2688, + "mer": 2689, + "åĪĽå»ºä¸Ģ个": 2690, + "éĻįä½İ": 2691, + "é²": 2692, + "å®ĺ": 2693, + "ĠPlease": 2694, + "string": 2695, + "èİ·å¾Ĺ": 2696, + "æľºä¼ļ": 2697, + "èĤī": 2698, + "室": 2699, + "æĭ¥": 2700, + "许å¤ļ": 2701, + "..": 2702, + "æľĭ": 2703, + "æĿ¥è¯´": 2704, + "ec": 2705, + "Ġonly": 2706, + "ç»ĵåIJĪ": 2707, + "é¦Ļ": 2708, + "ä¸Ńå¿ĥ": 2709, + "åıijçĶŁ": 2710, + "$.": 2711, + "åIJĪçIJĨ": 2712, + "åİĭåĬĽ": 2713, + "åħħ满": 2714, + "çĨ": 2715, + "verage": 2716, + "ï¼īãĢģ": 2717, + "Ġproduct": 2718, + "val": 2719, + "ost": 2720, + "icial": 2721, + "å¦": 2722, + "çļĦæŃ": 2723, + "Ġcompan": 2724, + "è§Ĩé¢ij": 2725, + "ts": 2726, + "ula": 2727, + "mat": 2728, + "iel": 2729, + "Ġweather": 2730, + "æ¯į": 2731, + "å¤ľ": 2732, + "Ġfriend": 2733, + "Ġrequ": 2734, + "ä½ľåĵģ": 2735, + "度åĴĮ": 2736, + "æł¸": 2737, + "ily": 2738, + "numbers": 2739, + "imal": 2740, + "åıĺéĩı": 2741, + "Ġask": 2742, + "Ġwide": 2743, + "æݨåĬ¨": 2744, + "Ġinst": 2745, + "ĠWor": 2746, + "è´Ńä¹°": 2747, + "н": 2748, + "åIJ¯": 2749, + "åįİ": 2750, + "Ġcloud": 2751, + "æĹ¢": 2752, + "èĥ½æºIJ": 2753, + "ized": 2754, + "refore": 2755, + "Ġpopul": 2756, + "è¿ĽæŃ¥": 2757, + "uring": 2758, + "è¿ĺæĺ¯": 2759, + "Al": 2760, + "á»": 2761, + "eters": 2762, + "å°Ħ": 2763, + "up": 2764, + "åĪ¶å®ļ": 2765, + "åĦ": 2766, + "ething": 2767, + "ute": 2768, + "åľ¨è¿Ļ个": 2769, + "å¸ģ": 2770, + "èĽĭ": 2771, + "Ġrespon": 2772, + "Ġway": 2773, + "ç³ĸ": 2774, + "app": 2775, + "çĶµå½±": 2776, + "æľĭåıĭ": 2777, + "Ġeven": 2778, + "æĢ§çļĦ": 2779, + "æ²ŁéĢļ": 2780, + "çļĦæĥħåĨµ": 2781, + "Ġfound": 2782, + "éĢĴ": 2783, + "Ġdec": 2784, + "çļĦæ¯": 2785, + "hip": 2786, + "åĽŃ": 2787, + "Ġeff": 2788, + "è¡Ģ": 2789, + "ç«ŀäºī": 2790, + "own": 2791, + "'d": 2792, + "Ñģ": 2793, + "Ġsignificant": 2794, + "Ġlight": 2795, + "èĻļæĭŁ": 2796, + "Ġmean": 2797, + "Ġsomething": 2798, + "ash": 2799, + "åĪĨç±»": 2800, + "严": 2801, + "继": 2802, + "Ġnam": 2803, + "欢è¿İ": 2804, + "æĹłè®º": 2805, + "æĿ¿": 2806, + "Ġreal": 2807, + "Ġarea": 2808, + "åķ¡": 2809, + "çĶ±äºİ": 2810, + "å¿Ĺ": 2811, + "ç¼ĵ": 2812, + "ife": 2813, + "Ġdig": 2814, + "å°Ĭ": 2815, + "é£ŀ": 2816, + "士": 2817, + "Ġ\\[": 2818, + "身ä½ĵ": 2819, + "éªĮè¯ģ": 2820, + "æĺł": 2821, + "éĥ½æľī": 2822, + "Ġstand": 2823, + "ER": 2824, + "éķ¿æľŁ": 2825, + "quence": 2826, + "举": 2827, + "ving": 2828, + "arr": 2829, + "Ġhapp": 2830, + "æĶ¯ä»ĺ": 2831, + "Ġaccur": 2832, + "Ġher": 2833, + "对象": 2834, + "æī©": 2835, + "æŁ¥è¯¢": 2836, + "æĥ³è¦ģ": 2837, + "loy": 2838, + "奶": 2839, + "Ġaccess": 2840, + "Ġsystem": 2841, + "Ġofficial": 2842, + "顺": 2843, + "Ġhigh": 2844, + "Ġcapital": 2845, + "票": 2846, + "åĴĸ": 2847, + "å½ĵåīį": 2848, + "éĢģ": 2849, + "è·µ": 2850, + "import": 2851, + "rect": 2852, + "Ġpartic": 2853, + "Ġdesigned": 2854, + "æķ´æķ°": 2855, + "综": 2856, + "ĠCan": 2857, + "ç«ł": 2858, + "EC": 2859, + "'ll": 2860, + "å±ķ示": 2861, + "循çݯ": 2862, + "cy": 2863, + "log": 2864, + "Ġplay": 2865, + "Ġsolve": 2866, + "æ¡£": 2867, + "åĴĸåķ¡": 2868, + "æĸ¹åIJij": 2869, + "çļĦåľ°": 2870, + "å®ŀè·µ": 2871, + "转æį¢ä¸º": 2872, + "ild": 2873, + "çķĻ": 2874, + "å®Ī": 2875, + "Ġ15": 2876, + "èģĮä¸ļ": 2877, + "å°¤åħ¶": 2878, + "æĭī": 2879, + "åıĮ": 2880, + "äºĨä¸Ģ个": 2881, + "Ġclass": 2882, + "Ġformula": 2883, + "äºĶ": 2884, + "çļĦæŶéĹ´": 2885, + "æĶ¹åıĺ": 2886, + "Ġstate": 2887, + "ç»į": 2888, + "çIJĨ论": 2889, + "Ġpoint": 2890, + "Ġent": 2891, + "St": 2892, + "Ġday": 2893, + "æįŁ": 2894, + "缺": 2895, + "ately": 2896, + "æī¿": 2897, + "èµ·æĿ¥": 2898, + "Ġtry": 2899, + "Ġsupport": 2900, + "Ġrele": 2901, + "ä¼ļè®®": 2902, + "è¶ĭ": 2903, + "ä¹Łæĺ¯": 2904, + "ources": 2905, + "ending": 2906, + "çļĦæµ": 2907, + "Ġversion": 2908, + "Ġline": 2909, + "py": 2910, + "é£İæł¼": 2911, + "ting": 2912, + "¨è¯¢": 2913, + "åĴ¨è¯¢": 2914, + "ĠHe": 2915, + "临": 2916, + "åł": 2917, + "çļĦæ³": 2918, + "ä»ĭç»į": 2919, + "éĩįè¦ģçļĦ": 2920, + "30": 2921, + "оÐ": 2922, + "大å°ı": 2923, + "Ġtop": 2924, + "è´§": 2925, + "马": 2926, + "Ġbest": 2927, + "ç»Ļåĩº": 2928, + "åĨ·": 2929, + "è¿Ľä¸ĢæŃ¥": 2930, + "è§ĦåĪĻ": 2931, + "ov": 2932, + "æ¦Ĥ念": 2933, + "æĪIJéķ¿": 2934, + "èĹ": 2935, + "Ġperform": 2936, + "Ġcreated": 2937, + "ç¼ĸåĨĻ": 2938, + "Ġide": 2939, + "çŁ³": 2940, + "ľç´¢": 2941, + "èĩªæĪij": 2942, + "Ġwork": 2943, + "light": 2944, + "cle": 2945, + "âĢĻ": 2946, + "Ġinterest": 2947, + "彩": 2948, + "arch": 2949, + "å®īè£ħ": 2950, + "tern": 2951, + "åĪĽä½ľ": 2952, + "25": 2953, + "Ġlong": 2954, + "çī¹çĤ¹": 2955, + "Ġrep": 2956, + "ience": 2957, + "Ġinput": 2958, + "éĺ¶": 2959, + "责任": 2960, + "Ġbook": 2961, + "Ġdirect": 2962, + "ĊĠĠĠĠĠĠĠĠĠĠĠ": 2963, + "å½¢å¼ı": 2964, + "åºĹ": 2965, + "ublic": 2966, + "ìĿ": 2967, + "ĠWorld": 2968, + "åĨ²": 2969, + "çĭ¬çī¹": 2970, + "æķ°æį®çļĦ": 2971, + "æīĢ以": 2972, + "Ġshould": 2973, + "寻æī¾": 2974, + "Ġpeople": 2975, + "ĠEnglish": 2976, + "æķ°åѦ": 2977, + "ruct": 2978, + "ox": 2979, + "åĨħéĥ¨": 2980, + "è¶ĭåĬ¿": 2981, + "Ġcomputing": 2982, + "çī¹å®ļçļĦ": 2983, + "çļ®": 2984, + "éĨ": 2985, + "åĨ³å®ļ": 2986, + "æľįåĬ¡åĻ¨": 2987, + "åĽ°": 2988, + "ä¹ĺ": 2989, + "åIJį称": 2990, + "ator": 2991, + "硬": 2992, + "å¿ħè¦ģ": 2993, + "Ġlarge": 2994, + "Ġmin": 2995, + "Ġassistance": 2996, + "äºĭ件": 2997, + "çļĦæ²": 2998, + "åľ°çIJĥ": 2999, + "çĽĸ": 3000, + "åıĬæŶ": 3001, + "Ġwebs": 3002, + "gy": 3003, + "èī¯å¥½çļĦ": 3004, + "ming": 3005, + "é": 3006, + "ç¿»è¯ij": 3007, + "16": 3008, + "éĻ¢": 3009, + "ink": 3010, + "æķ°éĩı": 3011, + "Ġupdate": 3012, + "æĺ¯ä»Ģä¹Ī": 3013, + "åĦ¿": 3014, + "èĬĤçĤ¹": 3015, + "åĨľ": 3016, + "Ġquery": 3017, + "çī©çIJĨ": 3018, + "女": 3019, + "æķ£": 3020, + "æĿİ": 3021, + "Ġtechnology": 3022, + "Ġarray": 3023, + "åľŁ": 3024, + "Ġstr": 3025, + "Ġvalues": 3026, + "Script": 3027, + "ÑĢ": 3028, + "hed": 3029, + "å¯Ł": 3030, + "ä¸įä¼ļ": 3031, + "ows": 3032, + "çĽĺ": 3033, + "éĵ¶": 3034, + "åķĨåĵģ": 3035, + "Ġlargest": 3036, + "eb": 3037, + "é¢ľ": 3038, + "åı¶": 3039, + "str": 3040, + "ctions": 3041, + "æ²»çĸĹ": 3042, + "raph": 3043, + "Ġbu": 3044, + "åĸľæ¬¢": 3045, + "æ°´å¹³": 3046, + "æ½ľ": 3047, + "Ġword": 3048, + "触": 3049, + "çĸ¾": 3050, + "éĴŁ": 3051, + "Ġwhile": 3052, + "Ġexper": 3053, + "æľºæŀĦ": 3054, + "çļĦæĻ": 3055, + "åĽº": 3056, + "Ġcharacter": 3057, + "ides": 3058, + "Ġac": 3059, + "èģļ": 3060, + "Java": 3061, + "Ġstep": 3062, + "illion": 3063, + "认为": 3064, + "éĿ¢å¯¹": 3065, + "ert": 3066, + "Ġdist": 3067, + "ew": 3068, + "ric": 3069, + "ons": 3070, + "梦": 3071, + "æĤ£": 3072, + "å®Įåħ¨": 3073, + "èĮ¶": 3074, + "cd": 3075, + "åij½ä»¤": 3076, + "éħįç½®": 3077, + "fter": 3078, + "è·Ŀ": 3079, + "Ġpol": 3080, + "Ġlearning": 3081, + "æĺ¯æĮĩ": 3082, + "Ġcalculate": 3083, + "âĢĶ": 3084, + "课ç¨ĭ": 3085, + "Ġ}": 3086, + "顾": 3087, + "Ġcorrect": 3088, + "ä¸Ģèµ·": 3089, + "Ġla": 3090, + "ĠçļĦ": 3091, + "ä¿®æĶ¹": 3092, + "word": 3093, + "Ġread": 3094, + "14": 3095, + "äºĴåĬ¨": 3096, + "Ġcent": 3097, + "^{": 3098, + "ustom": 3099, + "arth": 3100, + "Ġwell": 3101, + "æĺ¥": 3102, + "ä¾ĭåŃIJ": 3103, + "åĮĹ": 3104, + "Ġelse": 3105, + "åijĬè¯īæĪij": 3106, + "éĩĩçĶ¨": 3107, + "ا": 3108, + "对è¯Ŀ": 3109, + "åı¯ä»¥åľ¨": 3110, + "çļĦæĸ¹æ³ķ": 3111, + "åı¯ä»¥å¸®åĬ©": 3112, + "ough": 3113, + "çŃ¾": 3114, + "itive": 3115, + "æ¸IJ": 3116, + "é»ij": 3117, + "èĢħçļĦ": 3118, + "çĮ®": 3119, + "SS": 3120, + "åıįåºĶ": 3121, + "æĺ¾ç¤º": 3122, + "讨论": 3123, + "Ġsec": 3124, + "Ġlooking": 3125, + "çļĦå·¥ä½ľ": 3126, + "ĠAd": 3127, + "\\(": 3128, + "ah": 3129, + "åĬłå¯Ĩ": 3130, + "Ġhuman": 3131, + "æ¡Ĩ": 3132, + "éĤ®": 3133, + "æĪijå°Ĩ": 3134, + "ä½ĵçİ°": 3135, + "Ġeng": 3136, + "éĶĢåĶ®": 3137, + "line": 3138, + "\\).": 3139, + "çĶŁæĢģ": 3140, + "åIJĮæĹ¶ä¹Ł": 3141, + "éĴĪ": 3142, + "åºľ": 3143, + "äºĨä¸Ģ": 3144, + "åŁ¹è®Ń": 3145, + "æĸĩæ¡£": 3146, + "伤": 3147, + "éģį": 3148, + "\":": 3149, + "Ġsun": 3150, + "iss": 3151, + "æĭ¥æľī": 3152, + "åºĶ对": 3153, + "åı¯æĮģç»Ń": 3154, + "çļĦæľĢ": 3155, + "Ġposs": 3156, + "çļĦæĬĢæľ¯": 3157, + "çĶŁåij½": 3158, + "ik": 3159, + "åĬŀ": 3160, + "Ġrun": 3161, + "æµģç¨ĭ": 3162, + "Ġpopulation": 3163, + "ases": 3164, + "æĹ¥å¸¸": 3165, + "并ä¸į": 3166, + "Ġpersonal": 3167, + "Ġaccurate": 3168, + "ploy": 3169, + "Ġbre": 3170, + "Ġgu": 3171, + "éħĴ": 3172, + "丽": 3173, + "èĪª": 3174, + "Ġsimple": 3175, + "éļıæľº": 3176, + "æIJľç´¢": 3177, + "åı¦": 3178, + "åį³ä½¿": 3179, + "æĹ©": 3180, + "SQL": 3181, + "gle": 3182, + "åı¥åŃIJ": 3183, + "comm": 3184, + "éĩĩåıĸ": 3185, + "ethod": 3186, + "éķ¿åº¦": 3187, + "è¿ĻæľīåĬ©äºİ": 3188, + "PI": 3189, + "Ġchange": 3190, + "Ġshe": 3191, + "ãģĻ": 3192, + "ren": 3193, + "继ç»Ń": 3194, + "æĦŁåĪ°": 3195, + "æł¡": 3196, + "Ġ16": 3197, + "çł´": 3198, + "设å®ļ": 3199, + "wn": 3200, + "hether": 3201, + "Ġmess": 3202, + "Ġ30": 3203, + "è´¢åĬ¡": 3204, + "Ġ==": 3205, + "é¢ľèī²": 3206, + "æŨ": 3207, + "è¾¾åĪ°": 3208, + "alth": 3209, + "çĸ¾çĹħ": 3210, + "ï¼Į`": 3211, + "vert": 3212, + "Ġë": 3213, + "åĬª": 3214, + "代表": 3215, + "itions": 3216, + "Ġtype": 3217, + "éĢıæĺİ": 3218, + "ome": 3219, + "ĠWhether": 3220, + "æĦ¿": 3221, + "æĢ¥": 3222, + "\")": 3223, + "æĥħåĨµä¸ĭ": 3224, + "ef": 3225, + "æ¸ħæĻ": 3226, + "ĠWhat": 3227, + "平衡": 3228, + "Ġmulti": 3229, + "lob": 3230, + "åĪĨå¸ĥ": 3231, + "é²ľ": 3232, + "Ġaverage": 3233, + "iron": 3234, + "Ġcolumn": 3235, + "æİ¥åıĹ": 3236, + "SE": 3237, + "Ġest": 3238, + "æĹħè¡Į": 3239, + "Ġdesc": 3240, + "Ġpass": 3241, + "éĻIJåĪ¶": 3242, + "Ġz": 3243, + "æĺ¾èijĹ": 3244, + "æ·±åħ¥": 3245, + "å°Ĭéĩį": 3246, + "èĴ": 3247, + "åģļåĩº": 3248, + "gan": 3249, + "Ġhappy": 3250, + "åĬªåĬĽ": 3251, + "移åĬ¨": 3252, + "Ġlength": 3253, + "ability": 3254, + "以ä¸ĭåĩłä¸ª": 3255, + "å¸ĮæľĽè¿ĻäºĽ": 3256, + "ument": 3257, + "ĊĠĠĠĠ": 3258, + "ç¨İ": 3259, + "ä½Ĩä¸į": 3260, + "éĢ»": 3261, + "ä½łåı¯ä»¥ä½¿çĶ¨": 3262, + "Ġ>": 3263, + "ined": 3264, + "绪": 3265, + "è°ĥçĶ¨": 3266, + "ied": 3267, + "缩": 3268, + "大éĩı": 3269, + "Ġpoem": 3270, + "ures": 3271, + "ique": 3272, + "ven": 3273, + "追æ±Ĥ": 3274, + "akes": 3275, + "æĹ¨åľ¨": 3276, + "Ġpower": 3277, + "éĢ»è¾ij": 3278, + "resent": 3279, + "åı¯èĥ½çļĦ": 3280, + "è¿Ļåı¯èĥ½": 3281, + "Ġsequence": 3282, + "çļĦçĶŁæ´»": 3283, + "ĠZ": 3284, + "¢å¤į": 3285, + "Ġgood": 3286, + "ex": 3287, + "åºıåĪĹ": 3288, + "rans": 3289, + "ç³»åĪĹ": 3290, + "ä½³": 3291, + "äºļ": 3292, + "als": 3293, + "You": 3294, + "ä¾ĽåºĶ": 3295, + "åĬ¨çī©": 3296, + "Ġob": 3297, + "Ġhist": 3298, + "é»Ħ": 3299, + "Ġconvers": 3300, + "æĪij们åı¯ä»¥": 3301, + "驶": 3302, + "åī§": 3303, + "ug": 3304, + "符åIJĪ": 3305, + "Ġshort": 3306, + "Ġag": 3307, + "举": 3308, + "my": 3309, + "ware": 3310, + "Ġkey": 3311, + "å¤ļæł·": 3312, + "çģµæ´»": 3313, + "Ġwrite": 3314, + "Ġpur": 3315, + "22": 3316, + "æīĵåį°": 3317, + "ç¡®å®ŀ": 3318, + "Ġhand": 3319, + "çݩ家": 3320, + "éģĹ": 3321, + "à¹": 3322, + "çļĦæ·": 3323, + "ç§ĺ": 3324, + "els": 3325, + "Ġprim": 3326, + "Ġlife": 3327, + "çļĦç¼": 3328, + "çĽijæݧ": 3329, + "社交åªĴä½ĵ": 3330, + "Ġ25": 3331, + "åħ¨éĿ¢": 3332, + "èĵ": 3333, + "Ġgrow": 3334, + "å®ŀéªĮ": 3335, + "aut": 3336, + "æķı": 3337, + "Ġexpl": 3338, + "çĵ": 3339, + "åºķ": 3340, + "çĤ¹åĩ»": 3341, + "çİ°å®ŀ": 3342, + "ç®ĢåįķçļĦ": 3343, + "ç»ıéªĮ": 3344, + "è·¨": 3345, + "èĹı": 3346, + "éĩįå¤į": 3347, + "æĶ¿åºľ": 3348, + "Ġsoft": 3349, + "æijĦ": 3350, + "éŃ": 3351, + "çľ¼": 3352, + "é£Łçī©": 3353, + "Ġwords": 3354, + "伦": 3355, + "ĠSQL": 3356, + "åIJİçļĦ": 3357, + "Ġyears": 3358, + "亮": 3359, + "æĥ¯": 3360, + "æ¤įçī©": 3361, + "ä¼ij": 3362, + "æ¸ħæĻ°": 3363, + "Ġaround": 3364, + "称为": 3365, + "ä¸ĩ": 3366, + "éļıæŶ": 3367, + "å¿Ĩ": 3368, + "åĬ¨æĢģ": 3369, + "'ve": 3370, + "ĊĊĠĠĠ": 3371, + "Ġapplic": 3372, + "以ä¸Ĭ": 3373, + "Ġwebsite": 3374, + "æīĭæľº": 3375, + "Yes": 3376, + "viron": 3377, + "Ġmethod": 3378, + "Ġcompany": 3379, + "RO": 3380, + "åİŁåĽł": 3381, + "ä¹Łåı¯ä»¥": 3382, + "sqrt": 3383, + "è§Ħ模": 3384, + "ps": 3385, + "Ġwrit": 3386, + "äºij计ç®Ĺ": 3387, + "Ġoutput": 3388, + "Ġdat": 3389, + "Ġgive": 3390, + "ets": 3391, + "Ġimpro": 3392, + "\\),": 3393, + "å¼ĢæĶ¾": 3394, + "æ³ķè§Ħ": 3395, + "æĻ®": 3396, + "æĮīçħ§": 3397, + "18": 3398, + "éĹ»": 3399, + "æģ¢å¤į": 3400, + "Ġtem": 3401, + "è°¢": 3402, + "æľĢç»Ī": 3403, + "Ġ//": 3404, + "Here": 3405, + "æĶ¹è¿Ľ": 3406, + "Ġinf": 3407, + "cdot": 3408, + "æĹ¥æľŁ": 3409, + "ĠJava": 3410, + "å®ĩ": 3411, + "For": 3412, + "Ġjo": 3413, + "æĦıè¯Ĩ": 3414, + "以确ä¿Ŀ": 3415, + "ien": 3416, + "æĽ´å¤ļçļĦ": 3417, + "æļĸ": 3418, + "ream": 3419, + "++": 3420, + "AR": 3421, + "21": 3422, + "éĻIJäºİ": 3423, + "æĬĢå·§": 3424, + "åĨ°": 3425, + "å¹³åĿĩ": 3426, + "çļĦè¿ĩç¨ĭ": 3427, + "ä¸įè¦ģ": 3428, + "Ġreferring": 3429, + "ç¦ı": 3430, + "æĻļ": 3431, + "å°ıæĺİ": 3432, + "太éĺ³": 3433, + "ä¸įåı¯": 3434, + "åĮ»çĶŁ": 3435, + "Ġsmall": 3436, + "æ¯Ĵ": 3437, + "ç©¿": 3438, + "Ġmeans": 3439, + "ML": 3440, + "ze": 3441, + "享åıĹ": 3442, + "çīĽ": 3443, + "çļĦæ¦Ĥ": 3444, + "ines": 3445, + "Ġ使çĶ¨": 3446, + "How": 3447, + "人åijĺ": 3448, + "åĿļ": 3449, + "ĠTo": 3450, + "èĦļ": 3451, + "Ġglob": 3452, + "æ£Ģæµĭ": 3453, + "èµĸ": 3454, + "ode": 3455, + "强è°ĥ": 3456, + "模åŀĭçļĦ": 3457, + "('": 3458, + "vie": 3459, + "ç£": 3460, + "Ġbr": 3461, + "Ġsumm": 3462, + "Ġchat": 3463, + "é±": 3464, + "ä¹łæĥ¯": 3465, + "ellig": 3466, + "亲": 3467, + "Ġparticular": 3468, + "çĹĩ": 3469, + "Ġins": 3470, + "æķ´ä¸ª": 3471, + "çļĦåŃ¦ä¹ł": 3472, + "çļĦæĦı": 3473, + "Ġmark": 3474, + "è¿Ļä¸įä»ħ": 3475, + "æĬ¥åijĬ": 3476, + "ploye": 3477, + "éħ¸": 3478, + "çļĦéľĢæ±Ĥ": 3479, + "etwor": 3480, + "驾": 3481, + "Ch": 3482, + "çļĦæı": 3483, + "Ġåľ¨": 3484, + "è¸": 3485, + "ety": 3486, + "绿èī²": 3487, + "Ġexpression": 3488, + "read": 3489, + "Ġcharacters": 3490, + "èµ°": 3491, + "Ġstory": 3492, + "åĵįåºĶ": 3493, + "è¡ĮåĬ¨": 3494, + ")^": 3495, + "ĠPro": 3496, + "Ġ18": 3497, + "Ġbetter": 3498, + "åįģ": 3499, + "Ġdevelopment": 3500, + "以ä¸ĭæĺ¯ä¸Ģ个": 3501, + "æĿŁ": 3502, + "èIJ¥éĶĢ": 3503, + "æľīæķĪåľ°": 3504, + "é±¼": 3505, + "äºĴèģĶç½ij": 3506, + "æĭħ": 3507, + "深度åŃ¦ä¹ł": 3508, + "Ġprovided": 3509, + "éĺ²æŃ¢": 3510, + "Ġ11": 3511, + "æĸĩç«ł": 3512, + "çĬ¶åĨµ": 3513, + "å¢ŀéķ¿": 3514, + "Ġorder": 3515, + "ç»ĦæĪIJ": 3516, + "éĺµ": 3517, + "Ġmed": 3518, + "鼨": 3519, + "éĢī项": 3520, + "Ġenviron": 3521, + "oh": 3522, + "建设": 3523, + "Ġadvice": 3524, + "Ġend": 3525, + "íķ": 3526, + "Ġfam": 3527, + "Ġfin": 3528, + "Ġusers": 3529, + "ç»ĻæĪij": 3530, + "verse": 3531, + "Ġelements": 3532, + "ĠEx": 3533, + "----": 3534, + "éĿł": 3535, + "请æ±Ĥ": 3536, + "rase": 3537, + "æ±½": 3538, + "æĢ»ä¹ĭ": 3539, + "å®ŀæŶ": 3540, + "ives": 3541, + "åħģ": 3542, + "æıIJåĩº": 3543, + "çŃij": 3544, + "Ġconst": 3545, + "ius": 3546, + "å¯Ĩçłģ": 3547, + "æ´Ĺ": 3548, + "Ġconf": 3549, + "Ġshare": 3550, + "Therefore": 3551, + "24": 3552, + "森": 3553, + "commend": 3554, + "pi": 3555, + "ä¾Ŀèµĸ": 3556, + "çĩ": 3557, + "缸åIJĮ": 3558, + "pen": 3559, + "å®Ļ": 3560, + "çīĪæľ¬": 3561, + "Ġarr": 3562, + "åħ¬åħ±": 3563, + "è§Ī": 3564, + "åĬĽéĩı": 3565, + "Ġident": 3566, + "çĨŁ": 3567, + "液": 3568, + "åij¼": 3569, + "æł¼å¼ı": 3570, + "inu": 3571, + "çļĦåħ·ä½ĵ": 3572, + "åIJĪéĢĤçļĦ": 3573, + "çİĭ": 3574, + "大æķ°æį®": 3575, + "Ġvis": 3576, + "åľĨ": 3577, + "æĢĿèĢĥ": 3578, + "æĽ²": 3579, + "ç§ĭ": 3580, + "Ġtypically": 3581, + "åħģ许": 3582, + "çĮ«": 3583, + "Ġcomplex": 3584, + "æħ§": 3585, + "Ġmaking": 3586, + "ional": 3587, + "ward": 3588, + "Ġsecon": 3589, + "Ġrecommend": 3590, + "å®ĩå®Ļ": 3591, + "//": 3592, + "æŃ£åľ¨": 3593, + "gin": 3594, + "åĪĨéħį": 3595, + "è·Ŀ离": 3596, + "ä¸įèĥ½": 3597, + "Ġcommun": 3598, + "åĿĢ": 3599, + "å¾Ģå¾Ģ": 3600, + ":**": 3601, + "ä¼´": 3602, + "ries": 3603, + "io": 3604, + "éĽĨåĽ¢": 3605, + "ä¸Ńæĸĩ": 3606, + "ember": 3607, + "Ġorig": 3608, + "å¼Ĥ常": 3609, + "ways": 3610, + "åŁ¹åħ»": 3611, + "Ġinclude": 3612, + "èľ": 3613, + "Ġsug": 3614, + "åĭĩ": 3615, + "Ġtrying": 3616, + "Ġmessage": 3617, + "ror": 3618, + "ä½Ļ": 3619, + "Ġsym": 3620, + "æ··": 3621, + "åģľ": 3622, + "çī©è´¨": 3623, + "Ġmach": 3624, + "ergy": 3625, + "ç²ī": 3626, + "Ġdid": 3627, + "çļĦæķħäºĭ": 3628, + "Ġevery": 3629, + "åĽ¢éĺŁæĪIJåijĺ": 3630, + "ãģĹ": 3631, + "å±ķçİ°": 3632, + "éĢIJæ¸IJ": 3633, + "Ġsing": 3634, + "ious": 3635, + "çļĦç¬": 3636, + "Ġallow": 3637, + "Ġtell": 3638, + "ĠUnited": 3639, + "礼": 3640, + "Ġsame": 3641, + "ç®±": 3642, + "ipp": 3643, + "建çŃij": 3644, + "èĢIJ": 3645, + "çļĦåħ³éĶ®": 3646, + "æĦıä¹ī": 3647, + "çľĭåĪ°": 3648, + "èĩªçĦ¶è¯Ńè¨Ģ": 3649, + "Ġcustom": 3650, + "Ġprime": 3651, + "atform": 3652, + "ential": 3653, + "å¦Ĥæŀľä½łæľī": 3654, + "Ġservice": 3655, + "urity": 3656, + "ilities": 3657, + "Ġsee": 3658, + "èŃ": 3659, + "天æ°Ķ": 3660, + "éĥ½èĥ½": 3661, + "Ġbec": 3662, + "é¥®é£Ł": 3663, + "æ¶Īè´¹èĢħ": 3664, + "erv": 3665, + "Ġfeature": 3666, + "ä½łèĥ½": 3667, + "Ġupd": 3668, + "çļĦ主è¦ģ": 3669, + "å®ŀæĸ½": 3670, + "ä»į": 3671, + "Ġmay": 3672, + "çŁ¥éģĵ": 3673, + "解åĨ³éĹ®é¢ĺ": 3674, + "ale": 3675, + "Ġterms": 3676, + "å®ĺæĸ¹": 3677, + "éĢĶ": 3678, + "çľł": 3679, + "æĹı": 3680, + "éľ²": 3681, + "å¥ĸ": 3682, + "Ġcontain": 3683, + "çĶ¨çļĦ": 3684, + "åĪĨéĴŁ": 3685, + "ä¸Ģå®ļ": 3686, + "Ġfl": 3687, + "温度": 3688, + "ai": 3689, + "Ġright": 3690, + "¹ģ": 3691, + "æĶ»": 3692, + "ãĤĴ": 3693, + "åĪĽæĦı": 3694, + "è¿Ļå°Ĩ": 3695, + "åı¯ä»¥æıIJä¾Ľ": 3696, + "æľºåĻ¨äºº": 3697, + "Ġwid": 3698, + "å¦Ĥä¸ĭ": 3699, + "Ġstud": 3700, + "ĠSh": 3701, + "åĪ¤æĸŃ": 3702, + "æīįèĥ½": 3703, + "鼶": 3704, + "驾驶": 3705, + "Ġcat": 3706, + "ç¥ŀç»ı": 3707, + "ä»Ĭ": 3708, + "air": 3709, + "çļĦæ·±": 3710, + "åķĨä¸ļ": 3711, + "\".": 3712, + "$,": 3713, + "æĪijä¼ļ": 3714, + "ç±į": 3715, + "Ġsky": 3716, + "以便": 3717, + "è¡¥": 3718, + "èª": 3719, + "çIJĨ解åĴĮ": 3720, + "çĻ»": 3721, + "å¤įæĿĤçļĦ": 3722, + "æī©å±ķ": 3723, + "Ġav": 3724, + "æĪijçļĦ": 3725, + "å½ĵçĦ¶åı¯ä»¥": 3726, + "å·®å¼Ĥ": 3727, + "Ġstor": 3728, + "æ±ĩ": 3729, + "áº": 3730, + "è¾ħ": 3731, + "åĽ°éļ¾": 3732, + "Ġfile": 3733, + "è¶³å¤Ł": 3734, + "Ġcle": 3735, + "ivid": 3736, + "å°±æĺ¯": 3737, + "yn": 3738, + "ç»ĥä¹ł": 3739, + "åĨ³äºİ": 3740, + "åıĸåĨ³äºİ": 3741, + "æĦŁåħ´è¶£": 3742, + "åįĬ": 3743, + "æĽ¿": 3744, + "ä¿¡åı·": 3745, + "ç»ıåİĨ": 3746, + "isp": 3747, + "èī¾": 3748, + "13": 3749, + "åĮĸçļĦ": 3750, + "Ġcar": 3751, + "Of": 3752, + "æ°§": 3753, + "å·¥ç¨ĭ": 3754, + "ä¸įè¿ĩ": 3755, + "Ġcost": 3756, + "count": 3757, + "æħ¢": 3758, + "Ġcour": 3759, + "åĮºåŁŁ": 3760, + "çĽ¸å¯¹": 3761, + "Ġbusinesses": 3762, + "éĤ®ä»¶": 3763, + "AT": 3764, + "°": 3765, + "Ġroot": 3766, + "rix": 3767, + "Ġuser": 3768, + "é¾Ħ": 3769, + "atic": 3770, + "Ġ50": 3771, + "ç³»ç»ŁçļĦ": 3772, + "æŁ¥çľĭ": 3773, + "ä¸ī个": 3774, + "Ġanal": 3775, + "说æĺİ": 3776, + "åĵĪ": 3777, + "èĢĹ": 3778, + "çļĦ模": 3779, + "ithm": 3780, + "99": 3781, + "Ġens": 3782, + "Ġatt": 3783, + "Ġelement": 3784, + "Ġthree": 3785, + "çļĦç¾": 3786, + "Ġcult": 3787, + "æŁIJ个": 3788, + "Ġproviding": 3789, + "æ¯Ķè¾ĥ": 3790, + "Ġintern": 3791, + "ä¼łè¾ĵ": 3792, + "常è§ģçļĦ": 3793, + "ively": 3794, + "Ġ\\(\\": 3795, + "è§Ĩè§ī": 3796, + "utions": 3797, + "æĢģ度": 3798, + "Ġproject": 3799, + "ull": 3800, + "èĩªçĶ±": 3801, + "纸": 3802, + "éĺ¶æ®µ": 3803, + "åıįæĺł": 3804, + "èįī": 3805, + "Ġmu": 3806, + "sh": 3807, + "纪": 3808, + "Th": 3809, + "Ġsuggest": 3810, + "itle": 3811, + "ç½²": 3812, + "éĢļè¿ĩä¸Ĭè¿°": 3813, + "åħ±äº«": 3814, + "æĿ¥æºIJ": 3815, + "ä¼ĺåĬ¿": 3816, + "åİŁçIJĨ": 3817, + "åŃ¦ä¹łåĴĮ": 3818, + "å¤ı": 3819, + "ãģ¾": 3820, + "elligence": 3821, + "çĭ¬çī¹çļĦ": 3822, + "ëĭ": 3823, + "åĢ¾": 3824, + "ivers": 3825, + "åĨħåŃĺ": 3826, + "容æĺĵ": 3827, + "Ġ14": 3828, + "Ġenergy": 3829, + "å¨ģ": 3830, + "Ġprom": 3831, + "Ġidea": 3832, + "å®īåħ¨æĢ§": 3833, + "ructure": 3834, + "Ġsides": 3835, + "çģ¯": 3836, + "],": 3837, + "result": 3838, + "Ġmil": 3839, + "æĮĩçļĦæĺ¯": 3840, + "åĮħæĭ¬ä½Ĩä¸į": 3841, + "åĮħæĭ¬ä½Ĩä¸įéĻIJäºİ": 3842, + "ves": 3843, + "å®ŀä¾ĭ": 3844, + "Ġtrue": 3845, + "ä»ĸçļĦ": 3846, + "ä½ĵçļĦ": 3847, + "Ġfiel": 3848, + "ense": 3849, + "ote": 3850, + "`ï¼Į": 3851, + "èĨ": 3852, + "Ġwere": 3853, + "åĽŀçŃĶ": 3854, + "ç¬Ķ": 3855, + "ont": 3856, + "ĠTh": 3857, + "ç»ĦåIJĪ": 3858, + "ht": 3859, + "饰": 3860, + "other": 3861, + "ÙĦ": 3862, + "æĸĩåѦ": 3863, + "ximately": 3864, + "è´¡": 3865, + "æ¯ı个人": 3866, + "Ġimportant": 3867, + "Ġmillion": 3868, + "ç»ĺ": 3869, + "è·¯å¾Ħ": 3870, + "oogle": 3871, + "Ġprogramming": 3872, + "60": 3873, + "sum": 3874, + "ä¸Ģ段": 3875, + "æľīä¸Ģ个": 3876, + "åħ¬å¼ı": 3877, + "æĹłè®ºæĺ¯": 3878, + "订": 3879, + "æĬĹ": 3880, + "è§Ĵ度": 3881, + "Ġenvironment": 3882, + "ID": 3883, + "Ģæľī": 3884, + "åĨĮ": 3885, + "Ġremain": 3886, + "Ġplatform": 3887, + "son": 3888, + "æľĢ大": 3889, + "Ġopt": 3890, + "çļĦåŁºæľ¬": 3891, + "èĩ³åħ³": 3892, + "èį¯çī©": 3893, + "arm": 3894, + "æıĴ": 3895, + "ibr": 3896, + "è´¡çĮ®": 3897, + "Ġafter": 3898, + "ohn": 3899, + "æ±ł": 3900, + "Ġback": 3901, + "åįļ": 3902, + "Ġseries": 3903, + "time": 3904, + "ĠSp": 3905, + "ering": 3906, + "Ġnetwor": 3907, + "uss": 3908, + "Ġdescrib": 3909, + "é«ĺçļĦ": 3910, + "ĠYou": 3911, + "alse": 3912, + "ä»ĸ们çļĦ": 3913, + "åĮ¹": 3914, + "ty": 3915, + "æ¥": 3916, + "è¯ļ": 3917, + "æĿij": 3918, + "æľ¨": 3919, + "çļĦæ°´": 3920, + "æķĪçİĩåĴĮ": 3921, + "延": 3922, + "aves": 3923, + "perature": 3924, + "æĪijæĥ³": 3925, + "ything": 3926, + "ä¹İ": 3927, + "æĶ¹åĸĦ": 3928, + "éĵ¶è¡Į": 3929, + "ï¼ļâĢľ": 3930, + "磩": 3931, + "åıĺå¾Ĺ": 3932, + "宽": 3933, + "åı¯ä»¥å¸®åĬ©ä½ł": 3934, + "éģįåİĨ": 3935, + "èĢĮä¸įæĺ¯": 3936, + "Ġsn": 3937, + "ank": 3938, + "æ´ŀ": 3939, + "è¿Ļæĺ¯ä¸Ģ个": 3940, + "¡çľł": 3941, + "Ġè¾ĵåĩº": 3942, + "____": 3943, + "京": 3944, + "Do": 3945, + "Ġsur": 3946, + "Ġalways": 3947, + "è±Ĩ": 3948, + "Ġwater": 3949, + "stit": 3950, + "éĿĴ": 3951, + "Ġlim": 3952, + "Ġhig": 3953, + "`,": 3954, + "ma": 3955, + "Ġpossible": 3956, + "Ġcir": 3957, + "Ġstatement": 3958, + "Ġmachine": 3959, + "Ġinteger": 3960, + "èĵĿ": 3961, + "ĠAs": 3962, + "Ġhour": 3963, + "丰å¯ĮçļĦ": 3964, + "Ġinterested": 3965, + "±ä¹IJ": 3966, + "泡": 3967, + "åįĥ": 3968, + "综åIJĪ": 3969, + "ãĢĤâĢĿ": 3970, + "åįķè¯į": 3971, + "Ġcity": 3972, + "let": 3973, + "åĨĽ": 3974, + "Ġinde": 3975, + "Ġdeterm": 3976, + "寻æ±Ĥ": 3977, + "è¿Ľåħ¥": 3978, + "Ġbeaut": 3979, + "Ġair": 3980, + "л": 3981, + "Ġdi": 3982, + "Ġhealth": 3983, + "é¡»": 3984, + "ness": 3985, + "碳": 3986, + "å°ģ": 3987, + "çļĦæ´": 3988, + "`.": 3989, + "æĢİ": 3990, + "ä½įäºİ": 3991, + "汽车": 3992, + "çݯå¢ĥä¸Ń": 3993, + "åĬĽåĴĮ": 3994, + "å½ĵåľ°": 3995, + "Ġobject": 3996, + "atural": 3997, + "å®Įæķ´": 3998, + "ĠFr": 3999, + "çĶ³": 4000, + "Ġvary": 4001, + "Ġcontent": 4002, + "Ġbit": 4003, + "Ġour": 4004, + "çļĦçĥ": 4005, + "Ġunique": 4006, + "Ġwithout": 4007, + "åĴĮè°": 4008, + "è¯ŃéŁ³": 4009, + "åĪł": 4010, + "伦çIJĨ": 4011, + "ource": 4012, + "åĿı": 4013, + "ave": 4014, + "çļĦæĸ¹å¼ı": 4015, + "Ġbeing": 4016, + "Ġcase": 4017, + "æ¯ıä¸Ģ": 4018, + "Ġsal": 4019, + "èĤ¤": 4020, + "Ġbel": 4021, + "èĪĴ": 4022, + "Ġsecurity": 4023, + "ç¹ģ": 4024, + "ually": 4025, + "éĩįæĸ°": 4026, + "çŃīé¢ĨåŁŁ": 4027, + "é¢ĺ缮": 4028, + "çļĦæ¯ı": 4029, + "çļĦ使çĶ¨": 4030, + "éĿ©": 4031, + "ĠSe": 4032, + "ãģ«": 4033, + "Ġscript": 4034, + "éĤ£äºĽ": 4035, + "Ġmon": 4036, + "æ®Ĭ": 4037, + "身份": 4038, + "Ġap": 4039, + "ĠRe": 4040, + "':": 4041, + "çĶµèĦij": 4042, + "è§Ĥå¯Ł": 4043, + "è¿ĻæĦıåij³çĿĢ": 4044, + "æ¡Ĩæŀ¶": 4045, + "çļĦç¼ĸ": 4046, + "Ġincludes": 4047, + "åı³": 4048, + "Ġlog": 4049, + "à¤": 4050, + "çļĦç¾İ": 4051, + "Ġer": 4052, + "Ġimp": 4053, + "Ġ13": 4054, + "çľģ": 4055, + "ner": 4056, + "书ç±į": 4057, + "çī¹æ®Ĭ": 4058, + "åıĤåĬł": 4059, + "Ġdate": 4060, + "åıijå¸ĥ": 4061, + "çļĦæīĢæľī": 4062, + "è¯Ĭ": 4063, + "Ġpublic": 4064, + "éĥ¨ç½²": 4065, + "ross": 4066, + "ãģĦ": 4067, + "åĪłéĻ¤": 4068, + "Ġ%": 4069, + "ization": 4070, + "ä¼łæĴŃ": 4071, + "Ġmaxim": 4072, + "reate": 4073, + "box": 4074, + "çij": 4075, + "Ġdown": 4076, + "èѦ": 4077, + "Ġcourse": 4078, + "sole": 4079, + "ĠåĴĮ": 4080, + "座": 4081, + "以ä¸ĭåĩł": 4082, + "ĠMy": 4083, + "ĠQ": 4084, + "ï¼īï¼ļ": 4085, + "éĢļä¿¡": 4086, + "Re": 4087, + "è½®": 4088, + "ä¸Ĭä¸ĭ": 4089, + "ç¥Ŀ": 4090, + "::": 4091, + "gor": 4092, + "èĸ": 4093, + "å¹´é¾Ħ": 4094, + "Ġinvol": 4095, + "Ġtemperature": 4096, + "ä¿¡ä»»": 4097, + "æıIJéĹ®": 4098, + "Ġprof": 4099, + "ier": 4100, + "è®°å¿Ĩ": 4101, + "森æŀĹ": 4102, + "ä¸įä»ħèĥ½å¤Ł": 4103, + "æĤ£èĢħ": 4104, + "ger": 4105, + "ç»Ļå®ļ": 4106, + "çļĦå°ı": 4107, + "è¯Ńåı¥": 4108, + "ä½łéľĢè¦ģ": 4109, + "éģµå®Ī": 4110, + "Ġrole": 4111, + "åŃĹåħ¸": 4112, + "Ġreleased": 4113, + "Ex": 4114, + "bin": 4115, + "ECT": 4116, + "oy": 4117, + "æ¿Ģåıij": 4118, + "çĶļ": 4119, + "Ġconvert": 4120, + "Ġcommand": 4121, + "åĮ¹éħį": 4122, + "ployees": 4123, + "çļĦçŁ¥": 4124, + "éº": 4125, + "é¦Ĩ": 4126, + "å¡ij": 4127, + "Ġpositive": 4128, + "ON": 4129, + "Ġconcept": 4130, + "ĠEarth": 4131, + "Ġfriendly": 4132, + "åı¯éĿł": 4133, + "å°ıæŶ": 4134, + "åºĶçĶ¨ç¨ĭåºı": 4135, + "è¿Ļ段": 4136, + "Ġindivid": 4137, + "ws": 4138, + "Ġnamed": 4139, + "Ġplan": 4140, + "Ġsort": 4141, + "Ġdoc": 4142, + "è§ĤçĤ¹": 4143, + "ãģ§": 4144, + "Ġiss": 4145, + "ä¸ĭçļĦ": 4146, + "设æĸ½": 4147, + "åĴĮè°IJ": 4148, + "空æ°Ķ": 4149, + "æij©": 4150, + "åĬ³": 4151, + "请æıIJä¾Ľ": 4152, + "Ġsources": 4153, + "ĠCom": 4154, + "å¤įæĿĤ度": 4155, + "apt": 4156, + "Ġsecond": 4157, + "æĶ»åĩ»": 4158, + "')": 4159, + "Ġown": 4160, + "Ġmeaning": 4161, + "éĢĢ": 4162, + "ä½Ľ": 4163, + "è¡£": 4164, + "cho": 4165, + "æľīä»Ģä¹Ī": 4166, + "æ²³": 4167, + "缸åºĶ": 4168, + "èĶ": 4169, + "çļĦåĢ¼": 4170, + "Ġeffect": 4171, + "LECT": 4172, + "set": 4173, + "ogn": 4174, + "HT": 4175, + "Ġ```": 4176, + "åIJĥ": 4177, + "æĥħ绪": 4178, + "éķĩ": 4179, + "ocus": 4180, + "éĢĤå½ĵ": 4181, + "å¿ħé¡»": 4182, + "çĶļèĩ³": 4183, + "åĽ¾çīĩ": 4184, + "çļĦæĮĩ": 4185, + "çļĦåŁºç¡Ģ": 4186, + "Ġphys": 4187, + "ç»Ŀ": 4188, + "ality": 4189, + "æīĵå¼Ģ": 4190, + "Ġtre": 4191, + "Ġbecause": 4192, + "irt": 4193, + "çļĦæĥħåĨµä¸ĭ": 4194, + "Ġpat": 4195, + "Ġ200": 4196, + "åĮĸåѦ": 4197, + "åģı好": 4198, + "éĹŃ": 4199, + "åľ°åĿĢ": 4200, + "iter": 4201, + "aging": 4202, + "å¦Ĥä½ķåľ¨": 4203, + "Ġtake": 4204, + "åIJį为": 4205, + "åį´": 4206, + "åĪĨåĪ«": 4207, + "çļĦçľ": 4208, + "Ġplace": 4209, + "æľĢè¿ij": 4210, + "Ġ$\\": 4211, + "á": 4212, + "number": 4213, + "Ġmaximum": 4214, + "ç¯ĩ": 4215, + "rue": 4216, + "ĠInt": 4217, + "Ġ24": 4218, + "èĥŀ": 4219, + "case": 4220, + "Ġdeep": 4221, + "ROM": 4222, + "sc": 4223, + "Ġincre": 4224, + "ĠWe": 4225, + "éĥ½ä¼ļ": 4226, + "ç²¾ç¥ŀ": 4227, + "å©": 4228, + "åįıä½ľ": 4229, + "åı¯ä»¥æĺ¯": 4230, + "çŃīå¤ļ个": 4231, + "çĽijæµĭ": 4232, + "åĴĮæľįåĬ¡": 4233, + "Ġdue": 4234, + "Ġear": 4235, + "è¿IJèIJ¥": 4236, + "of": 4237, + "污": 4238, + "é«ĺ度": 4239, + "æī¹": 4240, + "çĸ«": 4241, + "Ġindividual": 4242, + "èĩªçĦ¶è¯Ńè¨Ģå¤ĦçIJĨ": 4243, + "ele": 4244, + "sive": 4245, + "ä¹Łèĥ½": 4246, + "æĻºæħ§": 4247, + "sort": 4248, + "af": 4249, + "åħ¬åı¸çļĦ": 4250, + "çĶ¨æĿ¥": 4251, + "çī¹æĢ§": 4252, + "èļ": 4253, + "Ġexact": 4254, + "åŃĹæ¯į": 4255, + "place": 4256, + "å¾Īå¤ļ": 4257, + "æĽ´åĩĨç¡®": 4258, + "æľīæīĢ帮åĬ©": 4259, + "ä¹ĭåīį": 4260, + "Ġtakes": 4261, + "ral": 4262, + "å®ĥ们çļĦ": 4263, + "çİ°åľ¨": 4264, + "å¹ķ": 4265, + "Ġduring": 4266, + "ç¼ĸè¾ij": 4267, + "gre": 4268, + "仿": 4269, + "æľīä¸Ģ": 4270, + "ters": 4271, + "Ġsince": 4272, + "ention": 4273, + "æĬ½": 4274, + "页éĿ¢": 4275, + "æ½ľåľ¨": 4276, + "è¯Ńæ³ķ": 4277, + "Ġfocus": 4278, + "æĢĿç»´": 4279, + "Ġlocated": 4280, + "!\"": 4281, + "纳": 4282, + "缸åħ³çļĦ": 4283, + "顶": 4284, + "Ġhelpful": 4285, + "Ġfactor": 4286, + "angle": 4287, + "缴åĪ°": 4288, + "ĠStates": 4289, + "湿": 4290, + "æį·": 4291, + "éĵģ": 4292, + "è´Łè´£": 4293, + "iction": 4294, + "Ġnatural": 4295, + "Ġorgan": 4296, + "娱ä¹IJ": 4297, + "ches": 4298, + "Ġleft": 4299, + "Ġblue": 4300, + "ok": 4301, + "é»ĺ": 4302, + "Ġmade": 4303, + "Ġprovides": 4304, + "pose": 4305, + "çļĦçģ": 4306, + "éķľ": 4307, + "åĿIJ": 4308, + "æĿ¥å®ŀçİ°": 4309, + "indows": 4310, + "abilities": 4311, + "请注æĦı": 4312, + "Ġteam": 4313, + "éĢłæĪIJ": 4314, + "çļĦèĥ½åĬĽ": 4315, + "éĨĴ": 4316, + "çĶ¨æĪ·çļĦ": 4317, + "æĹĭ": 4318, + "race": 4319, + "Ġphrase": 4320, + "17": 4321, + "对æĸ¹": 4322, + "Ġage": 4323, + "ĠAnd": 4324, + "çļĦé£İéĻ©": 4325, + "Ġaut": 4326, + "åį«": 4327, + "ä¸Ģèĩ´": 4328, + "ç»Ĩèĥŀ": 4329, + "æĬĢæľ¯çļĦåıijå±ķ": 4330, + "çĶŁæĪIJä¸Ģ个": 4331, + "Ġdivis": 4332, + "å°¤åħ¶æĺ¯": 4333, + "çŃīæĸ¹éĿ¢": 4334, + "Ġasking": 4335, + "缮çļĦ": 4336, + "第äºĮ": 4337, + "èĪĴéĢĤ": 4338, + "æĬĺ": 4339, + "Ġrecent": 4340, + "åįıè®®": 4341, + "è¾Ĩ": 4342, + "å·¨": 4343, + "åīĤ": 4344, + "the": 4345, + "æķ°çļĦ": 4346, + "Ġheart": 4347, + "value": 4348, + "éĿ¢ä¸´": 4349, + "å®ŀéĻħä¸Ĭ": 4350, + "æĿ¯": 4351, + "dd": 4352, + "ĠX": 4353, + "è´µ": 4354, + "è¯ģæĺİ": 4355, + "å±ı": 4356, + "new": 4357, + "ody": 4358, + "éģµå¾ª": 4359, + "Ġglobal": 4360, + "Ġdepending": 4361, + "åĩĢ": 4362, + "å·¦": 4363, + "ĠØ": 4364, + "æĽ´åħ·ä½ĵçļĦ": 4365, + "帮åĬ©æĤ¨": 4366, + "缸äºĴ": 4367, + "Ġreturns": 4368, + "å¸ĮæľĽè¿ĻäºĽå»ºè®®": 4369, + "Ġimprove": 4370, + "Ġmuch": 4371, + "ä»ĸ人": 4372, + "å¼¹": 4373, + "45": 4374, + "åįķä½į": 4375, + "çĦ¦": 4376, + "åĪĹ表ä¸Ń": 4377, + "`ãĢĤ": 4378, + "conom": 4379, + "èĥ½éĩı": 4380, + "Ġcalled": 4381, + "Ġrate": 4382, + "é¢Ħç®Ĺ": 4383, + "èĩ³åħ³éĩįè¦ģ": 4384, + "acc": 4385, + "ç»ĵæĿŁ": 4386, + "åı¯ä»¥èĢĥèĻij": 4387, + "ç»ĨèĬĤ": 4388, + "Ġve": 4389, + "å¯¹ä½ł": 4390, + "ĠThese": 4391, + "è¿ĩ度": 4392, + "ified": 4393, + "è¯Ħä»·": 4394, + "æķ°æį®éĽĨ": 4395, + "é«ĺè´¨éĩı": 4396, + "èŀįåIJĪ": 4397, + "åİŁåĪĻ": 4398, + "Ġapproximately": 4399, + "Ġrepresent": 4400, + "ificial": 4401, + "ason": 4402, + "æĸ¹ç¨ĭ": 4403, + "Ġsocial": 4404, + "Ġfig": 4405, + "SELECT": 4406, + "####": 4407, + "人æ°ij": 4408, + "ãĢij": 4409, + "hest": 4410, + "ãĢIJ": 4411, + "Ġdigital": 4412, + "以ä¸ĭåĩłä¸ªæĸ¹éĿ¢": 4413, + "Ġclo": 4414, + "å°Ĩåħ¶": 4415, + "çļĦåĨħ容": 4416, + "æı¡": 4417, + "random": 4418, + "å·ŀ": 4419, + "è´Ńçī©": 4420, + "hou": 4421, + "éĥ¨éŨ": 4422, + "Ġsnipp": 4423, + "tract": 4424, + "çݰ象": 4425, + "Ġneeds": 4426, + "æķ°åĪĹ": 4427, + "Ġgame": 4428, + "Ġtri": 4429, + "è´´": 4430, + "çĭ¬ç«ĭ": 4431, + "å¿ħè¦ģçļĦ": 4432, + "èĪŀ": 4433, + "æ¸ħæ´ģ": 4434, + "象å¾ģ": 4435, + "è½»æĿ¾": 4436, + "鸡": 4437, + "åĨ²çªģ": 4438, + "Ġgeneral": 4439, + "Ġblo": 4440, + "çļĦæĦŁ": 4441, + "åĨħçļĦ": 4442, + "Ñĥ": 4443, + "ins": 4444, + "çļĦçŁ¥è¯Ĩ": 4445, + "模åĿĹ": 4446, + "éĻ¤äºĨ": 4447, + "è¦Ĩ": 4448, + "yle": 4449, + "Ġintelligence": 4450, + "åĨĻä½ľ": 4451, + "Ġed": 4452, + "Ġrad": 4453, + "cing": 4454, + "Please": 4455, + "ved": 4456, + "èĥ½å¸®åĬ©ä½ł": 4457, + "é¢Ŀå¤ĸ": 4458, + "Ġmarket": 4459, + "Ġnight": 4460, + "input": 4461, + "æķ°æį®åĪĨæŀIJ": 4462, + "æ´²": 4463, + "me": 4464, + "wh": 4465, + "Ġeas": 4466, + "å¤ļå°ij": 4467, + "ãģ¯": 4468, + "ock": 4469, + "Create": 4470, + "Ġloop": 4471, + "When": 4472, + "çħ": 4473, + "gs": 4474, + "åı¯ä»¥æł¹æį®": 4475, + "Ġrecogn": 4476, + "æĸ°éĹ»": 4477, + "Ġ\\),": 4478, + "ä¼ijæģ¯": 4479, + "ĠMa": 4480, + "ä¸Ĭä¸ĭæĸĩ": 4481, + "Ġmem": 4482, + "åĬłå¼º": 4483, + "æ±Ł": 4484, + "éĺ¿éĩĮå·´å·´éĽĨåĽ¢": 4485, + "Ġsingle": 4486, + "æ´ĭ": 4487, + "PU": 4488, + "imate": 4489, + "çļĦæ²ŁéĢļ": 4490, + "åľ¨æŁIJäºĽ": 4491, + "浪": 4492, + "ä¸Ŀ": 4493, + "表éĿ¢": 4494, + "èī¯å¥½": 4495, + "æŀ¶æŀĦ": 4496, + "éĺ»": 4497, + "ãģª": 4498, + "åı¯æĮģç»Ńåıijå±ķ": 4499, + "Ġenh": 4500, + "sql": 4501, + "çŃīäºİ": 4502, + "Ġmultiple": 4503, + "plic": 4504, + "æĸĩåŃĹ": 4505, + "ä¹Łä¼ļ": 4506, + "éĩİ": 4507, + "ountain": 4508, + "200": 4509, + "ling": 4510, + "åĪ¶éĢł": 4511, + "âĢĿï¼Ī": 4512, + "Ġdisc": 4513, + "API": 4514, + "Ġ17": 4515, + "èĻ«": 4516, + "éļĶ": 4517, + "plify": 4518, + "Ġclar": 4519, + "åĪ°äºĨ": 4520, + "æ¹ĸ": 4521, + "Ġder": 4522, + "Ġrequest": 4523, + "æŶåĢĻ": 4524, + "æľªæĿ¥çļĦ": 4525, + "ç½Ĺ": 4526, + "åħ¬å¹³": 4527, + "Ġchang": 4528, + "æĮĩ导": 4529, + "åı«": 4530, + "Ġless": 4531, + "æľīçĿĢ": 4532, + "çļĦæł¸": 4533, + "pper": 4534, + "ony": 4535, + "éĢĤçĶ¨äºİ": 4536, + "ĠÃ": 4537, + "å®ļä½į": 4538, + "æľī趣": 4539, + "çļĦæŃ£": 4540, + "JavaScript": 4541, + "åĬ¨åĬĽ": 4542, + "und": 4543, + "ðŁ": 4544, + "å¼ĢåıijèĢħ": 4545, + "ç¾İåĽ½": 4546, + "èĪ¬": 4547, + "...": 4548, + "åĪĨ为": 4549, + "çļĦæĮijæĪĺ": 4550, + "Ġamount": 4551, + "Ġfilm": 4552, + "å°½éĩı": 4553, + "çļĦæ³ķ": 4554, + "Ġprimary": 4555, + "Ġpopular": 4556, + "ume": 4557, + "self": 4558, + "80": 4559, + "æ¯ĶèµĽ": 4560, + "åIJįåŃĹ": 4561, + "å¡«": 4562, + "èĩªåĬ¨åĮĸ": 4563, + "Ġes": 4564, + "磩éĺµ": 4565, + "rehen": 4566, + "łéģĵ": 4567, + "为ä»Ģä¹Ī": 4568, + "çıŃ": 4569, + "Ġcapabilities": 4570, + "Ġsnippet": 4571, + "è·Ł": 4572, + "" + ], + [ + "in", + "ed" + ], + [ + "ç»", + "ª" + ], + [ + "è°ĥ", + "çĶ¨" + ], + [ + "i", + "ed" + ], + [ + "ç¼", + "©" + ], + [ + "大", + "éĩı" + ], + [ + "Ġpo", + "em" + ], + [ + "ure", + "s" + ], + [ + "iqu", + "e" + ], + [ + "v", + "en" + ], + [ + "追", + "æ±Ĥ" + ], + [ + "ak", + "es" + ], + [ + "æŨ", + "åľ¨" + ], + [ + "Ġp", + "ower" + ], + [ + "éĢ»", + "è¾ij" + ], + [ + "res", + "ent" + ], + [ + "åı¯èĥ½", + "çļĦ" + ], + [ + "è¿Ļ", + "åı¯èĥ½" + ], + [ + "Ġse", + "quence" + ], + [ + "çļĦ", + "çĶŁæ´»" + ], + [ + "Ġ", + "Z" + ], + [ + "¢", + "å¤į" + ], + [ + "Ġg", + "ood" + ], + [ + "e", + "x" + ], + [ + "åºı", + "åĪĹ" + ], + [ + "r", + "ans" + ], + [ + "ç³»", + "åĪĹ" + ], + [ + "ä½", + "³" + ], + [ + "äº", + "ļ" + ], + [ + "al", + "s" + ], + [ + "Y", + "ou" + ], + [ + "ä¾Ľ", + "åºĶ" + ], + [ + "åĬ¨", + "çī©" + ], + [ + "Ġo", + "b" + ], + [ + "Ġh", + "ist" + ], + [ + "é»", + "Ħ" + ], + [ + "Ġcon", + "vers" + ], + [ + "æĪij们", + "åı¯ä»¥" + ], + [ + "é©", + "¶" + ], + [ + "åī", + "§" + ], + [ + "u", + "g" + ], + [ + "符", + "åIJĪ" + ], + [ + "Ġsh", + "ort" + ], + [ + "Ġa", + "g" + ], + [ + "ä¸", + "¾" + ], + [ + "m", + "y" + ], + [ + "w", + "are" + ], + [ + "Ġke", + "y" + ], + [ + "å¤ļ", + "æł·" + ], + [ + "çģµ", + "æ´»" + ], + [ + "Ġw", + "rite" + ], + [ + "Ġp", + "ur" + ], + [ + "2", + "2" + ], + [ + "æīĵ", + "åį°" + ], + [ + "ç¡®", + "å®ŀ" + ], + [ + "Ġh", + "and" + ], + [ + "çİ©", + "家" + ], + [ + "éģ", + "Ĺ" + ], + [ + "à", + "¹" + ], + [ + "çļĦæ", + "·" + ], + [ + "ç§", + "ĺ" + ], + [ + "el", + "s" + ], + [ + "Ġpr", + "im" + ], + [ + "Ġl", + "ife" + ], + [ + "çļĦç", + "¼" + ], + [ + "çĽij", + "æݧ" + ], + [ + "社交", + "åªĴä½ĵ" + ], + [ + "Ġ2", + "5" + ], + [ + "åħ¨", + "éĿ¢" + ], + [ + "è", + "ĵ" + ], + [ + "Ġg", + "row" + ], + [ + "å®ŀ", + "éªĮ" + ], + [ + "a", + "ut" + ], + [ + "æķ", + "ı" + ], + [ + "Ġex", + "pl" + ], + [ + "ç", + "ĵ" + ], + [ + "åº", + "ķ" + ], + [ + "çĤ¹", + "åĩ»" + ], + [ + "çİ°", + "å®ŀ" + ], + [ + "ç®Ģåįķ", + "çļĦ" + ], + [ + "ç»ı", + "éªĮ" + ], + [ + "è·", + "¨" + ], + [ + "èĹ", + "ı" + ], + [ + "éĩį", + "å¤į" + ], + [ + "æĶ¿", + "åºľ" + ], + [ + "Ġso", + "ft" + ], + [ + "æij", + "Ħ" + ], + [ + "é", + "Ń" + ], + [ + "çľ", + "¼" + ], + [ + "é£Ł", + "çī©" + ], + [ + "Ġwor", + "ds" + ], + [ + "ä¼", + "¦" + ], + [ + "ĠS", + "QL" + ], + [ + "åIJİ", + "çļĦ" + ], + [ + "Ġyear", + "s" + ], + [ + "äº", + "®" + ], + [ + "æĥ", + "¯" + ], + [ + "æ¤į", + "çī©" + ], + [ + "ä¼", + "ij" + ], + [ + "æ¸ħæĻ", + "°" + ], + [ + "Ġa", + "round" + ], + [ + "称", + "为" + ], + [ + "ä¸", + "ĩ" + ], + [ + "éļı", + "æŶ" + ], + [ + "å¿", + "Ĩ" + ], + [ + "åĬ¨", + "æĢģ" + ], + [ + "'", + "ve" + ], + [ + "Ċ", + "ĊĠĠĠ" + ], + [ + "Ġapp", + "lic" + ], + [ + "以", + "ä¸Ĭ" + ], + [ + "Ġwebs", + "ite" + ], + [ + "æīĭ", + "æľº" + ], + [ + "Y", + "es" + ], + [ + "v", + "iron" + ], + [ + "Ġm", + "ethod" + ], + [ + "Ġcompan", + "y" + ], + [ + "R", + "O" + ], + [ + "åİŁ", + "åĽł" + ], + [ + "ä¹Ł", + "åı¯ä»¥" + ], + [ + "sq", + "rt" + ], + [ + "è§Ħ", + "模" + ], + [ + "p", + "s" + ], + [ + "Ġw", + "rit" + ], + [ + "äºij", + "计ç®Ĺ" + ], + [ + "Ġout", + "put" + ], + [ + "Ġd", + "at" + ], + [ + "Ġg", + "ive" + ], + [ + "et", + "s" + ], + [ + "Ġim", + "pro" + ], + [ + "\\)", + "," + ], + [ + "å¼Ģ", + "æĶ¾" + ], + [ + "æ³ķ", + "è§Ħ" + ], + [ + "æĻ", + "®" + ], + [ + "æĮī", + "çħ§" + ], + [ + "1", + "8" + ], + [ + "éĹ", + "»" + ], + [ + "æģ", + "¢å¤į" + ], + [ + "Ġt", + "em" + ], + [ + "è°", + "¢" + ], + [ + "æľĢ", + "ç»Ī" + ], + [ + "Ġ/", + "/" + ], + [ + "H", + "ere" + ], + [ + "æĶ¹", + "è¿Ľ" + ], + [ + "Ġin", + "f" + ], + [ + "cd", + "ot" + ], + [ + "æĹ¥", + "æľŁ" + ], + [ + "ĠJ", + "ava" + ], + [ + "å®", + "ĩ" + ], + [ + "F", + "or" + ], + [ + "Ġj", + "o" ], [ "æĦı", @@ -21766,1613 +19251,5213 @@ "ç¡®ä¿Ŀ" ], [ - "çĬ¶", - "æĢģ" + "i", + "en" ], [ - "Ġautom", - "ation" - ], - [ - "1", - "1" - ], - [ - "Ġpo", - "st" - ], - [ - "æīĭ", - "æľº" - ], - [ - "wor", - "ks" - ], - [ - "éĢ", - "ı" - ], - [ - "åº", - "ĵ" - ], - [ - "Ġw", - "ind" - ], - [ - "Ġ=", - "=" - ], - [ - "Ġprocess", - "ing" - ], - [ - "èĮĥ", - "åĽ´" - ], - [ - "æĦı", - "ä¹ī" - ], - [ - "追", - "æ±Ĥ" - ], - [ - "Ã", - "©" - ], - [ - "å¾", - "Ħ" - ], - [ - "éĿ", - "ł" - ], - [ - "ä¸", - "ĸ" - ], - [ - "èĻ", - "½" - ], - [ - "ç«", - "ŀäºī" - ], - [ - "Ġappro", - "priate" - ], - [ - "æĽ´", - "好çļĦ" - ], - [ - "Ġcharact", - "er" - ], - [ - "c", - "l" - ], - [ - "ç§", - "ĺ" - ], - [ - "it", - "ude" - ], - [ - "Ġte", - "ac" - ], - [ - "le", - "ep" - ], - [ - "ĠDe", - "velop" - ], - [ - "in", - "ce" - ], - [ - "å·", - "¦" - ], - [ - "g", - "round" - ], - [ - "è¡Į", - "ä¸ļ" - ], - [ - "éĴĪ", - "对" - ], - [ - "å¿ħ", - "è¦ģ" - ], - [ - "Ġdet", - "erm" - ], - [ - "--------", - "--------" - ], - [ - "Ġst", - "reng" - ], - [ - "d", - "o" - ], - [ - "Ġchalleng", - "ing" - ], - [ - "or", - "k" - ], - [ - "Ġan", - "x" - ], - [ - "èī²", + "æĽ´å¤ļ", "çļĦ" ], [ - "Ġh", - "ard" + "æļ", + "ĸ" ], [ - "æĺİ", - "ç¡®" + "re", + "am" ], [ - "åĪĨ", - "享" - ], - [ - "æĶ¹", - "åıĺ" - ], - [ - "ä½", - "³" - ], - [ - "åıª", - "æľī" - ], - [ - "å±ķ", - "示" - ], - [ - "Ġcam", - "p" - ], - [ - "çº", - "³" - ], - [ - "a", - "j" - ], - [ - "et", - "ic" - ], - [ - "u", - "ment" - ], - [ - "ä½ł", - "åı¯ä»¥" - ], - [ - "Ġpol", - "lut" - ], - [ - "Ġh", - "ig" - ], - [ - "pp", - "ing" - ], - [ - "e", - "ad" - ], - [ - "çĦ¶", - "èĢĮ" - ], - [ - "第", - "äºĮ" - ], - [ - "é¸", - "Ł" - ], - [ - "çī©", - "åĵģ" - ], - [ - "ä¸", - "¾" - ], - [ - "Ġencoura", - "ge" - ], - [ - "pe", - "cial" - ], - [ - "Ġac", - "ross" - ], - [ - "el", - "ves" - ], - [ - "äºĭ", - "件" - ], - [ - "c", - "le" - ], - [ - "æ", - "©" - ], - [ - "åªĴ", - "ä½ĵ" - ], - [ - "n", - "ers" - ], - [ - "Ġc", - "al" - ], - [ - "èĻ½", - "çĦ¶" - ], - [ - "åĽ", - "º" - ], - [ - "ä¹ł", - "æĥ¯" - ], - [ - "Ġsaf", - "e" - ], - [ - "èĥ½", - "éĩı" - ], - [ - "ist", - "ics" - ], - [ - "ä¹ĭ", - "åīį" - ], - [ - "Ġiss", - "ue" - ], - [ - "å¤ļ", - "个" - ], - [ - "åĨ³", - "çŃĸ" - ], - [ - "è¾¾", - "åĪ°" - ], - [ - "æĹ", - "©" - ], - [ - "ä¸į", - "åı¯" - ], - [ - "ä¸Ģ", - "缴" - ], - [ - "å·", - "¨" - ], - [ - "æĦŁ", - "è°¢" - ], - [ - "ĠN", - "ew" - ], - [ - "ä¸Ģ", - "段" - ], - [ - "Ġmach", - "ines" - ], - [ - "å°Ĩ", - "åħ¶" - ], - [ - "继", - "ç»Ń" - ], - [ - "Ġwor", - "d" - ], - [ - "çī¹", - "åĪ«" - ], - [ - "Ġagricult", - "ure" - ], - [ - "æĢ", - "İ" - ], - [ - "éĢIJ", - "æ¸IJ" - ], - [ - "éĵ", - "¾" - ], - [ - "è¯", - "¾" - ], - [ - "Ġk", - "ind" - ], - [ - "å¢", - "Ļ" - ], - [ - "è°¢", - "è°¢" - ], - [ - "Ġalgorith", - "m" - ], - [ - "è£ħ", - "饰" - ], - [ - "Ġal", - "ong" - ], - [ - "Ġeas", - "y" - ], - [ - "äº", - "ij" - ], - [ - "解åĨ³", - "æĸ¹æ¡Ī" - ], - [ - "Ġaware", - "ness" - ], - [ - "'", - "ve" - ], - [ - "æĸ¹", - "åIJij" - ], - [ - "Ġne", - "ver" - ], - [ - "Ġquick", - "ly" - ], - [ - "Ġres", - "pect" - ], - [ - "çļĦæ", - "Ļ" - ], - [ - "Ġam", - "ong" - ], - [ - "Ġaccount", - "ability" - ], - [ - "Ġl", - "aw" - ], - [ - "en", - "ing" - ], - [ - "Ġdef", - "in" - ], - [ - "Ġsur", - "round" - ], - [ - "éĵ", - "ģ" - ], - [ - "Ġpower", - "ful" + "+", + "+" ], [ "A", - "n" - ], - [ - "Ġcaus", - "e" - ], - [ - "æ", - "¥" - ], - [ - "æİĮ", - "æı¡" - ], - [ - "è¿ĺ", - "æĺ¯" - ], - [ - "Ġcreat", - "ive" - ], - [ - "è¡", - "Ģ" - ], - [ - "Ġloc", - "ated" - ], - [ - "un", - "ning" - ], - [ - "åľ°", - "åĮº" - ], - [ - "éĿ¢", - "积" - ], - [ - "éĽ", - "¨" - ], - [ - "Ġne", - "ar" - ], - [ - "Ġinit", - "i" - ], - [ - "ress", - "ion" - ], - [ - "ä¸ĭ", - "æĿ¥" + "R" ], [ "2", - "5" + "1" ], [ - "é©", - "¶" + "éĻIJ", + "äºİ" ], [ - "¾", - "çĹħ" + "æĬĢ", + "å·§" ], [ - "ab", - "les" - ], - [ - "æľī", - "趣" - ], - [ - "循", - "çݯ" - ], - [ - "çŃĶ", - "æ¡Ī" - ], - [ - "çł", - "´" - ], - [ - "ic", - "ation" - ], - [ - "éĻ", - "¢" - ], - [ - "æ²»", - "çĸĹ" - ], - [ - "Ġad", - "dition" - ], - [ - "äºĭ", - "æĥħ" - ], - [ - "Ġbec", - "ause" - ], - [ - "åı", - "Ī" - ], - [ - "èĤ", - "Į" - ], - [ - "çº", - "ª" - ], - [ - "s", - "ide" - ], - [ - "æĭ", - "ħ" - ], - [ - "æ¹", - "¿" - ], - [ - "åį", - "Ĭ" - ], - [ - "é¡", - "º" - ], - [ - "ĠA", - "nd" - ], - [ - "Ġrestaur", - "ant" - ], - [ - "Ġv", - "ide" - ], - [ - "Ġproble", - "m" - ], - [ - "az", - "ing" - ], - [ - "Ġmem", - "bers" - ], - [ - "Ġn", - "ut" - ], - [ - "Ġc", - "ou" - ], - [ - "æµ", - "ª" - ], - [ - "Ġ", - "è¿Ļ" - ], - [ - "Ġhelp", - "ing" - ], - [ - "ĠI", - "s" - ], - [ - "æıIJ", - "åįĩ" - ], - [ - "ĠĠĠĠ", - "ĠĠ" - ], - [ - "Ġsh", - "o" - ], - [ - "Ġre", - "lev" - ], - [ - "Ġar", - "g" - ], - [ - "Ġbal", - "ance" - ], - [ - "ill", - "ed" - ], - [ - "æĺ¯", - "ä»Ģä¹Ī" - ], - [ - "åĬĽ", - "éĩı" - ], - [ - "ire", - "d" - ], - [ - "å¤", - "ľ" - ], - [ - "åı¯", - "æĮģç»Ń" - ], - [ - "Ġper", - "fect" - ], - [ - "*", - "*" - ], - [ - "ific", - "ation" - ], - [ - "æ¶", - "ī" - ], - [ - "Ġwild", - "life" - ], - [ - "an", - "e" - ], - [ - "Ġrel", - "ated" - ], - [ - "室", - "åĨħ" - ], - [ - "åº", - "ľ" - ], - [ - "享", - "åıĹ" - ], - [ - "our", - "s" - ], - [ - "è·", - "ij" - ], - [ - "åķĨ", - "ä¸ļ" - ], - [ - "ach", - "ing" - ], - [ - "Ġsu", - "n" - ], - [ - "Ġrecogn", - "ition" - ], - [ - "el", - "t" - ], - [ - "Ġor", - "der" + "åĨ", + "°" ], [ "å¹³", "åĿĩ" ], [ - "g", - "ing" + "çļĦ", + "è¿ĩç¨ĭ" ], [ - "ä¸", - "´" + "ä¸į", + "è¦ģ" ], [ - "çĤ", - "¼" + "Ġrefer", + "ring" ], [ - "Ġgo", - "ing" + "ç¦", + "ı" ], [ - "åij¼", - "åIJ¸" - ], - [ - "Ġsoft", - "ware" - ], - [ - "Ġre", - "mot" - ], - [ - "èijĹ", - "åIJį" - ], - [ - "幸", - "ç¦ı" - ], - [ - "Ġenh", - "ance" - ], - [ - "èĻ", + "æĻ", "ļ" ], [ - "Ġn", - "ow" + "å°ı", + "æĺİ" ], [ - "Ġth", - "reat" + "太", + "éĺ³" ], [ - "Ġd", - "est" - ], - [ - "åĿĩ", - "åĮĢ" - ], - [ - "Ġac", - "ad" - ], - [ - "åºĶ", - "对" - ], - [ - "çľĭ", - "åĪ°" - ], - [ - "c", - "ast" - ], - [ - "è¾", - "Ĩ" - ], - [ - "ific", - "ial" - ], - [ - "Ġ", - "very" - ], - [ - "o", - "ok" - ], - [ - "åĮº", - "åŁŁ" - ], - [ - "¹", - "ģ" - ], - [ - "æĪ¿", - "éĹ´" - ], - [ - "æıIJä¾Ľ", - "äºĨ" - ], - [ - "Ġmot", - "iv" - ], - [ - "Ġaccess", - "ible" - ], - [ - "åĨ³", - "å®ļ" - ], - [ - "Ġh", - "y" - ], - [ - "å®", - "Ī" - ], - [ - "Ġf", - "lo" - ], - [ - "u", - "g" - ], - [ - "Ġinform", - "ed" - ], - [ - "åĵģ", - "è´¨" - ], - [ - "çļĦç", - "Ł" - ], - [ - "av", - "es" - ], - [ - "ar", - "r" - ], - [ - "ĠW", - "ith" - ], - [ - "le", - "t" - ], - [ - "è§Ĥ", - "çĤ¹" - ], - [ - "en", - "ge" - ], - [ - "è¡Į", - "åĬ¨" - ], - [ - "f", - "riend" - ], - [ - "ç³", - "ķ" - ], - [ - "Ġf", - "urther" - ], - [ - "ĠE", - "ns" - ], - [ - "ç§", - "ģ" - ], - [ - "Ġad", - "o" - ], - [ - "Ġcle", - "an" - ], - [ - "缸", - "åºĶ" - ], - [ - "Ġf", - "re" - ], - [ - "pecial", - "ly" - ], - [ - "è", - "Ĺ" - ], - [ - "Ġc", - "apt" - ], - [ - "çļĦç", - "ľ" - ], - [ - "Ġsome", - "one" - ], - [ - "Ġc", - "ell" - ], - [ - "æĶ¾", - "åľ¨" - ], - [ - "欢", - "è¿İ" - ], - [ - "Ġ", - "âĢ" - ], - [ - "Ġdev", - "ices" - ], - [ - "çļĦ", - "æĸ¹å¼ı" - ], - [ - "Ġjob", - "s" - ], - [ - "au", - "gh" - ], - [ - "n", - "ot" - ], - [ - "æľī", - "äºĽ" - ], - [ - "åħ¬", - "åħ±" - ], - [ - "g", - "est" - ], - [ - "çļĦ", - "çĶŁæ´»" - ], - [ - "çľ", - "¼" - ], - [ - "çļĦ", - "ä¿¡æģ¯" - ], - [ - "ĠC", - "ons" - ], - [ - "æİĴ", - "åºı" - ], - [ - "Ġbenef", - "it" - ], - [ - "re", - "ct" - ], - [ - "å¤", - "ı" - ], - [ - "un", - "te" - ], - [ - "符", - "åIJĪ" - ], - [ - "ä¸Ģ", - "ä½į" - ], - [ - "åĨħ", - "éĥ¨" - ], - [ - "Ġlook", - "ing" - ], - [ - "d", - "ing" - ], - [ - "æĬ", - "ĺ" - ], - [ - "è¾", - "ij" - ], - [ - "è¿Ļ个", - "éĹ®é¢ĺ" - ], - [ - "Ġes", - "pecially" - ], - [ - "çľ", - "ł" - ], - [ - "âĢĿ", - "ãĢĤ" - ], - [ - "å¥", - "ı" - ], - [ - "ra", - "y" - ], - [ - "è¿ĺ", - "åı¯ä»¥" - ], - [ - "åĪĽ", - "ä½ľ" - ], - [ - "com", - "ing" - ], - [ - "Ġmulti", - "ple" - ], - [ - "éļ", - "IJ" - ], - [ - "æ³", - "¡" - ], - [ - "æłĩ", - "åĩĨ" - ], - [ - "Ġm", - "il" - ], - [ - "éľĢè¦ģ", - "注æĦı" - ], - [ - "Ġanx", - "iety" - ], - [ - "æĶ¹", - "è¿Ľ" - ], - [ - "å±", - "ĭ" - ], - [ - "污", - "æŁĵ" - ], - [ - "ç¼ĸ", - "ç¨ĭ" - ], - [ - "è´¹", - "çĶ¨" - ], - [ - "Ġev", - "alu" - ], - [ - "imate", - "ly" - ], - [ - "Ġlit", - "er" - ], - [ - "ogra", - "ph" - ], - [ - "Ġse", - "arch" - ], - [ - "1", - "6" - ], - [ - "en", - "ced" - ], - [ - "Ġmeth", - "ods" - ], - [ - "çĥ", - "Ī" - ], - [ - "模", - "å¼ı" - ], - [ - "çĬ¶", - "åĨµ" - ], - [ - "æĶ¹", - "åĸĦ" - ], - [ - "å¤ļ", - "æł·" - ], - [ - "c", - "er" - ], - [ - "å¥", - "ĸ" - ], - [ - "Ġsat", - "is" - ], - [ - "Ġwebs", - "ite" - ], - [ - "åĬ", - "ŀ" - ], - [ - "åģ¥", - "身" - ], - [ - "Ġglo", - "bal" - ], - [ - "Ġas", - "k" - ], - [ - "Ġplatform", - "s" - ], - [ - "Ġdise", - "ases" - ], - [ - "çİ°", - "象" - ], - [ - "t", - "ics" - ], - [ - "æ±", - "ģ" - ], - [ - "åĪ¤", - "æĸŃ" - ], - [ - "Ġcon", - "vers" - ], - [ - "Ġrelations", - "hip" - ], - [ - "设", - "ç½®" - ], - [ - "æ³ķ", - "å¾ĭ" - ], - [ - "Ġmind", - "ful" - ], - [ - "é¢Ħ", - "æµĭ" - ], - [ - "o", - "very" - ], - [ - "åģ", - "ľ" - ], - [ - "çĶµ", - "è§Ĩ" - ], - [ - "è§Ħ", - "åĪĻ" - ], - [ - "ak", - "en" - ], - [ - "Ġimplement", - "ing" - ], - [ - "is", - "ing" - ], - [ - "åıĤ", - "åĬł" - ], - [ - "æĥħ", - "绪" - ], - [ - "Ġprovid", - "ed" - ], - [ - "æ·±", - "åħ¥" - ], - [ - "Ġprogramm", - "ed" - ], - [ - "Ġrelev", - "ant" - ], - [ - "çļĦç", - "ĥ" - ], - [ - "çĸ", - "¾çĹħ" + "ä¸į", + "åı¯" ], [ "åĮ»", "çĶŁ" ], [ - "åĪĽ", - "建" + "Ġsm", + "all" ], [ - "Ġgener", - "ate" + "æ¯", + "Ĵ" ], [ - "æĶ¶", - "åħ¥" + "ç©", + "¿" ], [ - "ä¼", - "ij" + "Ġme", + "ans" ], [ - "iz", + "M", + "L" + ], + [ + "z", + "e" + ], + [ + "享", + "åıĹ" + ], + [ + "çī", + "Ľ" + ], + [ + "çļĦæ", + "¦Ĥ" + ], + [ + "in", "es" ], [ - "Ġtrans", - "form" + "Ġ", + "使çĶ¨" ], [ - "éģ", + "H", + "ow" + ], + [ + "人", + "åijĺ" + ], + [ + "åĿ", + "ļ" + ], + [ + "ĠT", + "o" + ], + [ + "èĦ", + "ļ" + ], + [ + "Ġg", + "lob" + ], + [ + "æ£Ģ", + "æµĭ" + ], + [ + "èµ", + "ĸ" + ], + [ + "o", + "de" + ], + [ + "强", + "è°ĥ" + ], + [ + "模åŀĭ", + "çļĦ" + ], + [ + "(", + "'" + ], + [ + "v", + "ie" + ], + [ + "ç", + "£" + ], + [ + "Ġb", + "r" + ], + [ + "Ġsum", + "m" + ], + [ + "Ġch", + "at" + ], + [ + "é", + "±" + ], + [ + "ä¹ł", + "æĥ¯" + ], + [ + "ell", + "ig" + ], + [ + "äº", + "²" + ], + [ + "Ġpartic", + "ular" + ], + [ + "çĹ", + "ĩ" + ], + [ + "Ġin", + "s" + ], + [ + "æķ´", + "个" + ], + [ + "çļĦ", + "åŃ¦ä¹ł" + ], + [ + "çļĦæĦ", + "ı" + ], + [ + "Ġm", + "ark" + ], + [ + "è¿Ļ", + "ä¸įä»ħ" + ], + [ + "æĬ¥", + "åijĬ" + ], + [ + "ploy", + "e" + ], + [ + "éħ", + "¸" + ], + [ + "çļĦ", + "éľĢæ±Ĥ" + ], + [ + "et", + "wor" + ], + [ + "é©", + "¾" + ], + [ + "C", + "h" + ], + [ + "çļĦæ", + "ı" + ], + [ + "Ġ", + "åľ¨" + ], + [ + "è", + "¸" + ], + [ + "et", + "y" + ], + [ + "绿", + "èī²" + ], + [ + "Ġexpress", + "ion" + ], + [ + "re", + "ad" + ], + [ + "Ġcharact", + "ers" + ], + [ + "èµ", + "°" + ], + [ + "Ġst", + "ory" + ], + [ + "åĵį", + "åºĶ" + ], + [ + "è¡Į", + "åĬ¨" + ], + [ + ")", + "^" + ], + [ + "ĠP", + "ro" + ], + [ + "Ġ1", + "8" + ], + [ + "Ġbet", + "ter" + ], + [ + "åį", + "ģ" + ], + [ + "Ġdevelop", + "ment" + ], + [ + "以ä¸ĭ", + "æĺ¯ä¸Ģ个" + ], + [ + "æĿ", + "Ł" + ], + [ + "èIJ¥", + "éĶĢ" + ], + [ + "æľīæķĪ", + "åľ°" + ], + [ + "é±", + "¼" + ], + [ + "äºĴ", + "èģĶç½ij" + ], + [ + "æĭ", + "ħ" + ], + [ + "深度", + "åŃ¦ä¹ł" + ], + [ + "Ġprovid", + "ed" + ], + [ + "éĺ²", + "æŃ¢" + ], + [ + "Ġ1", + "1" + ], + [ + "æĸĩ", + "ç«ł" + ], + [ + "çĬ¶", + "åĨµ" + ], + [ + "å¢ŀ", + "éķ¿" + ], + [ + "Ġor", + "der" + ], + [ + "ç»Ħ", + "æĪIJ" + ], + [ + "éĺ", "µ" ], [ - "ast", - "ic" + "Ġm", + "ed" ], [ - "åij", - "Ī" + "éĽ", + "¨" ], [ - "æ¯ı", - "个人" + "éĢī", + "项" ], [ - "è¿", - "Ķ" + "Ġen", + "viron" + ], + [ + "o", + "h" + ], + [ + "建", + "设" + ], + [ + "Ġadv", + "ice" + ], + [ + "Ġ", + "end" + ], + [ + "í", + "ķ" + ], + [ + "Ġf", + "am" + ], + [ + "Ġf", + "in" + ], + [ + "Ġus", + "ers" + ], + [ + "ç»Ļ", + "æĪij" + ], + [ + "vers", + "e" + ], + [ + "Ġele", + "ments" + ], + [ + "ĠE", + "x" + ], + [ + "--", + "--" + ], + [ + "éĿ", + "ł" + ], + [ + "请", + "æ±Ĥ" + ], + [ + "r", + "ase" + ], + [ + "æ±", + "½" + ], + [ + "æĢ»", + "ä¹ĭ" + ], + [ + "å®ŀ", + "æŶ" + ], + [ + "iv", + "es" + ], + [ + "åħ", + "ģ" + ], + [ + "æıIJ", + "åĩº" + ], + [ + "çŃ", + "ij" + ], + [ + "Ġcon", + "st" ], [ "i", - "et" + "us" + ], + [ + "å¯Ĩ", + "çłģ" + ], + [ + "æ´", + "Ĺ" + ], + [ + "Ġcon", + "f" + ], + [ + "Ġsh", + "are" + ], + [ + "The", + "refore" + ], + [ + "2", + "4" + ], + [ + "æ£", + "®" + ], + [ + "comm", + "end" + ], + [ + "p", + "i" + ], + [ + "ä¾Ŀ", + "èµĸ" + ], + [ + "ç", + "ĩ" + ], + [ + "缸", + "åIJĮ" + ], + [ + "p", + "en" + ], + [ + "å®", + "Ļ" + ], + [ + "çīĪ", + "æľ¬" + ], + [ + "Ġar", + "r" + ], + [ + "åħ¬", + "åħ±" + ], + [ + "è§", + "Ī" + ], + [ + "åĬĽ", + "éĩı" + ], + [ + "Ġ", + "ident" + ], + [ + "çĨ", + "Ł" + ], + [ + "æ¶", + "²" + ], + [ + "åij", + "¼" + ], + [ + "æł¼", + "å¼ı" + ], + [ + "in", + "u" + ], + [ + "çļĦ", + "åħ·ä½ĵ" + ], + [ + "åIJĪéĢĤ", + "çļĦ" + ], + [ + "çİ", + "ĭ" + ], + [ + "大", + "æķ°æį®" ], [ "Ġv", - "oice" + "is" + ], + [ + "åľ", + "Ĩ" + ], + [ + "æĢĿ", + "èĢĥ" + ], + [ + "æĽ", + "²" + ], + [ + "ç§", + "ĭ" + ], + [ + "Ġtyp", + "ically" + ], + [ + "åħģ", + "许" + ], + [ + "çĮ", + "«" + ], + [ + "Ġcomple", + "x" + ], + [ + "æ", + "ħ§" + ], + [ + "Ġma", + "king" + ], + [ + "ion", + "al" + ], + [ + "w", + "ard" + ], + [ + "Ġse", + "con" + ], + [ + "Ġre", + "commend" + ], + [ + "å®ĩ", + "å®Ļ" + ], + [ + "/", + "/" + ], + [ + "æŃ£", + "åľ¨" + ], + [ + "g", + "in" + ], + [ + "åĪĨ", + "éħį" + ], + [ + "è·Ŀ", + "离" + ], + [ + "ä¸į", + "èĥ½" + ], + [ + "Ġcomm", + "un" + ], + [ + "åĿ", + "Ģ" + ], + [ + "å¾Ģ", + "å¾Ģ" + ], + [ + ":", + "**" + ], + [ + "ä¼", + "´" + ], + [ + "ri", + "es" + ], + [ + "i", + "o" + ], + [ + "éĽĨ", + "åĽ¢" + ], + [ + "ä¸Ń", + "æĸĩ" + ], + [ + "em", + "ber" + ], + [ + "Ġor", + "ig" + ], + [ + "å¼Ĥ", + "常" + ], + [ + "w", + "ays" + ], + [ + "åŁ¹", + "åħ»" + ], + [ + "Ġincl", + "ude" + ], + [ + "è", + "ľ" + ], + [ + "Ġsu", + "g" + ], + [ + "åĭ", + "ĩ" + ], + [ + "Ġtry", + "ing" + ], + [ + "Ġmess", + "age" + ], + [ + "r", + "or" + ], + [ + "ä½", + "Ļ" + ], + [ + "Ġsy", + "m" + ], + [ + "æ·", + "·" + ], + [ + "åģ", + "ľ" + ], + [ + "çī©", + "è´¨" + ], + [ + "Ġma", + "ch" + ], + [ + "er", + "gy" + ], + [ + "ç²", + "ī" + ], + [ + "Ġd", + "id" + ], + [ + "çļĦ", + "æķħäºĭ" + ], + [ + "Ġever", + "y" + ], + [ + "åĽ¢éĺŁ", + "æĪIJåijĺ" + ], + [ + "ãģ", + "Ĺ" + ], + [ + "å±ķ", + "çİ°" + ], + [ + "éĢIJ", + "æ¸IJ" + ], + [ + "Ġs", + "ing" + ], + [ + "i", + "ous" + ], + [ + "çļĦç", + "¬" + ], + [ + "Ġal", + "low" + ], + [ + "Ġt", + "ell" + ], + [ + "ĠUn", + "ited" + ], + [ + "ç¤", + "¼" + ], + [ + "Ġs", + "ame" + ], + [ + "ç®", + "±" + ], + [ + "i", + "pp" + ], + [ + "建", + "çŃij" + ], + [ + "èĢ", + "IJ" + ], + [ + "çļĦ", + "åħ³éĶ®" + ], + [ + "æĦı", + "ä¹ī" + ], + [ + "çľĭ", + "åĪ°" + ], + [ + "èĩªçĦ¶", + "è¯Ńè¨Ģ" + ], + [ + "Ġc", + "ustom" + ], + [ + "Ġpr", + "ime" + ], + [ + "at", + "form" + ], + [ + "ent", + "ial" + ], + [ + "å¦Ĥæŀľä½ł", + "æľī" + ], + [ + "Ġserv", + "ice" + ], + [ + "ur", + "ity" + ], + [ + "il", + "ities" + ], + [ + "Ġse", + "e" + ], + [ + "è", + "Ń" + ], + [ + "天", + "æ°Ķ" + ], + [ + "éĥ½", + "èĥ½" + ], + [ + "Ġbe", + "c" + ], + [ + "饮", + "é£Ł" + ], + [ + "æ¶Īè´¹", + "èĢħ" + ], + [ + "er", + "v" + ], + [ + "Ġfe", + "ature" + ], + [ + "ä½ł", + "èĥ½" + ], + [ + "Ġup", + "d" + ], + [ + "çļĦ", + "主è¦ģ" + ], + [ + "å®ŀ", + "æĸ½" + ], + [ + "ä»", + "į" + ], + [ + "Ġm", + "ay" + ], + [ + "çŁ¥", + "éģĵ" + ], + [ + "解åĨ³", + "éĹ®é¢ĺ" + ], + [ + "al", + "e" + ], + [ + "Ġterm", + "s" + ], + [ + "å®ĺ", + "æĸ¹" ], [ "éĢ", "Ķ" ], [ - "æĶ¾", - "æĿ¾" + "çľ", + "ł" ], [ - "åį", - "´" + "æĹ", + "ı" ], [ - "èĥ", - "ľ" + "éľ", + "²" ], [ - "Ġst", - "ructure" + "å¥", + "ĸ" ], [ - "æŶ", - "å°ļ" + "Ġcon", + "tain" ], [ - "Ġ", - "Q" + "çĶ¨", + "çļĦ" ], [ - "Ġel", - "se" + "åĪĨ", + "éĴŁ" ], [ - "du", - "c" + "ä¸Ģ", + "å®ļ" ], [ - "Ġem", - "p" + "Ġf", + "l" ], [ - "èģ", - "ļ" + "温", + "度" ], [ - "è´", - "§" + "a", + "i" ], [ - "ac", - "hes" - ], - [ - "ç§", - "Ģ" - ], - [ - "an", - "ks" - ], - [ - "Ġn", + "Ġr", "ight" ], [ - "Ġprofessional", - "s" + "¹", + "ģ" ], [ - "Ġb", - "as" + "æĶ", + "»" ], [ - "è´", - "µ" + "ãĤ", + "Ĵ" ], [ - "e", - "c" + "åĪĽ", + "æĦı" ], [ - "Ġdivers", - "ity" + "è¿Ļ", + "å°Ĩ" ], [ - "it", - "es" + "åı¯ä»¥", + "æıIJä¾Ľ" ], [ - "d", - "r" + "æľºåĻ¨", + "人" + ], + [ + "Ġw", + "id" + ], + [ + "å¦Ĥ", + "ä¸ĭ" + ], + [ + "Ġst", + "ud" + ], + [ + "ĠS", + "h" + ], + [ + "åĪ¤", + "æĸŃ" + ], + [ + "æīį", + "èĥ½" + ], + [ + "éĽ", + "¶" + ], + [ + "驾", + "驶" + ], + [ + "Ġc", + "at" + ], + [ + "ç¥ŀ", + "ç»ı" + ], + [ + "ä»", + "Ĭ" + ], + [ + "a", + "ir" + ], + [ + "çļĦæ·", + "±" + ], + [ + "åķĨ", + "ä¸ļ" + ], + [ + "\"", + "." + ], + [ + "$", + "," + ], + [ + "æĪij", + "ä¼ļ" + ], + [ + "ç±", + "į" + ], + [ + "Ġsk", + "y" + ], + [ + "以", + "便" + ], + [ + "è¡", + "¥" + ], + [ + "è", + "ª" + ], + [ + "çIJĨ解", + "åĴĮ" + ], + [ + "çĻ", + "»" + ], + [ + "å¤įæĿĤ", + "çļĦ" + ], + [ + "æī©", + "å±ķ" + ], + [ + "Ġa", + "v" + ], + [ + "æĪij", + "çļĦ" + ], + [ + "å½ĵçĦ¶", + "åı¯ä»¥" + ], + [ + "å·®", + "å¼Ĥ" + ], + [ + "Ġst", + "or" + ], + [ + "æ±", + "ĩ" + ], + [ + "á", + "º" + ], + [ + "è¾", + "ħ" ], [ "åĽ°", "éļ¾" ], + [ + "Ġf", + "ile" + ], + [ + "足", + "å¤Ł" + ], + [ + "Ġc", + "le" + ], + [ + "iv", + "id" + ], + [ + "å°±", + "æĺ¯" + ], + [ + "y", + "n" + ], + [ + "ç»ĥ", + "ä¹ł" + ], + [ + "åĨ³", + "äºİ" + ], + [ + "åıĸ", + "åĨ³äºİ" + ], + [ + "æĦŁ", + "åħ´è¶£" + ], + [ + "åį", + "Ĭ" + ], + [ + "æĽ", + "¿" + ], + [ + "ä¿¡", + "åı·" + ], + [ + "ç»ı", + "åİĨ" + ], + [ + "is", + "p" + ], + [ + "èī", + "¾" + ], + [ + "1", + "3" + ], + [ + "åĮĸ", + "çļĦ" + ], + [ + "Ġc", + "ar" + ], + [ + "O", + "f" + ], + [ + "æ°", + "§" + ], + [ + "å·¥", + "ç¨ĭ" + ], + [ + "ä¸į", + "è¿ĩ" + ], + [ + "Ġco", + "st" + ], + [ + "c", + "ount" + ], + [ + "æħ", + "¢" + ], + [ + "Ġc", + "our" + ], + [ + "åĮº", + "åŁŁ" + ], + [ + "缸", + "对" + ], + [ + "Ġbusiness", + "es" + ], + [ + "éĤ®", + "件" + ], + [ + "A", + "T" + ], + [ + "Â", + "°" + ], + [ + "Ġro", + "ot" + ], + [ + "ri", + "x" + ], + [ + "Ġus", + "er" + ], + [ + "é¾", + "Ħ" + ], + [ + "at", + "ic" + ], + [ + "Ġ5", + "0" + ], + [ + "ç³»ç»Ł", + "çļĦ" + ], + [ + "æŁ¥", + "çľĭ" + ], + [ + "ä¸ī", + "个" + ], + [ + "Ġan", + "al" + ], + [ + "说", + "æĺİ" + ], + [ + "åĵ", + "Ī" + ], + [ + "èĢ", + "Ĺ" + ], + [ + "çļĦæ", + "¨¡" + ], + [ + "ith", + "m" + ], + [ + "9", + "9" + ], + [ + "Ġen", + "s" + ], + [ + "Ġat", + "t" + ], + [ + "Ġele", + "ment" + ], + [ + "Ġth", + "ree" + ], + [ + "çļĦç", + "¾" + ], + [ + "Ġc", + "ult" + ], + [ + "æŁIJ", + "个" + ], + [ + "Ġprovid", + "ing" + ], + [ + "æ¯Ķ", + "è¾ĥ" + ], + [ + "Ġinter", + "n" + ], + [ + "ä¼ł", + "è¾ĵ" + ], + [ + "常è§ģ", + "çļĦ" + ], + [ + "iv", + "ely" + ], + [ + "Ġ\\(", + "\\" + ], + [ + "è§Ĩ", + "è§ī" + ], + [ + "ut", + "ions" + ], + [ + "æĢģ", + "度" + ], + [ + "Ġpro", + "ject" + ], + [ + "ul", + "l" + ], + [ + "èĩª", + "çĶ±" + ], + [ + "çº", + "¸" + ], + [ + "éĺ¶", + "段" + ], + [ + "åıį", + "æĺł" + ], + [ + "èį", + "ī" + ], + [ + "Ġm", + "u" + ], + [ + "s", + "h" + ], + [ + "çº", + "ª" + ], + [ + "T", + "h" + ], + [ + "Ġsug", + "gest" + ], + [ + "it", + "le" + ], + [ + "ç½", + "²" + ], + [ + "éĢļè¿ĩ", + "ä¸Ĭè¿°" + ], + [ + "åħ±", + "享" + ], + [ + "æĿ¥", + "æºIJ" + ], + [ + "ä¼ĺ", + "åĬ¿" + ], + [ + "åİŁ", + "çIJĨ" + ], + [ + "åŃ¦ä¹ł", + "åĴĮ" + ], + [ + "å¤", + "ı" + ], + [ + "ãģ", + "¾" + ], + [ + "ellig", + "ence" + ], + [ + "çĭ¬çī¹", + "çļĦ" + ], + [ + "ë", + "ĭ" + ], + [ + "åĢ", + "¾" + ], + [ + "iv", + "ers" + ], + [ + "åĨħ", + "åŃĺ" + ], + [ + "容", + "æĺĵ" + ], + [ + "Ġ1", + "4" + ], + [ + "Ġen", + "ergy" + ], + [ + "å¨", + "ģ" + ], + [ + "Ġpro", + "m" + ], + [ + "Ġide", + "a" + ], + [ + "å®īåħ¨", + "æĢ§" + ], + [ + "ruct", + "ure" + ], + [ + "Ġs", + "ides" + ], + [ + "çģ", + "¯" + ], + [ + "]", + "," + ], + [ + "res", + "ult" + ], + [ + "Ġm", + "il" + ], + [ + "æĮĩ", + "çļĦæĺ¯" + ], + [ + "åĮħæĭ¬", + "ä½Ĩä¸į" + ], + [ + "åĮħæĭ¬ä½Ĩä¸į", + "éĻIJäºİ" + ], + [ + "v", + "es" + ], + [ + "å®ŀ", + "ä¾ĭ" + ], + [ + "Ġtr", + "ue" + ], + [ + "ä»ĸ", + "çļĦ" + ], + [ + "ä½ĵ", + "çļĦ" + ], + [ + "Ġf", + "iel" + ], + [ + "en", + "se" + ], + [ + "ot", + "e" + ], + [ + "`", + "ï¼Į" + ], + [ + "è", + "Ĩ" + ], + [ + "Ġwe", + "re" + ], + [ + "åĽŀ", + "çŃĶ" + ], + [ + "ç¬", + "Ķ" + ], + [ + "on", + "t" + ], + [ + "ĠT", + "h" + ], + [ + "ç»Ħ", + "åIJĪ" + ], + [ + "h", + "t" + ], + [ + "é¥", + "°" + ], + [ + "ot", + "her" + ], + [ + "Ù", + "Ħ" + ], + [ + "æĸĩ", + "åѦ" + ], + [ + "xim", + "ately" + ], + [ + "è´", + "¡" + ], + [ + "æ¯ı", + "个人" + ], + [ + "Ġimport", + "ant" + ], + [ + "Ġm", + "illion" + ], + [ + "ç»", + "ĺ" + ], + [ + "è·¯", + "å¾Ħ" + ], + [ + "oo", + "gle" + ], + [ + "Ġprogram", + "ming" + ], + [ + "6", + "0" + ], + [ + "s", + "um" + ], + [ + "ä¸Ģ", + "段" + ], + [ + "æľī", + "ä¸Ģ个" + ], + [ + "åħ¬", + "å¼ı" + ], + [ + "æĹłè®º", + "æĺ¯" + ], + [ + "è®", + "¢" + ], + [ + "æĬ", + "Ĺ" + ], + [ + "è§Ĵ", + "度" + ], + [ + "Ġenviron", + "ment" + ], + [ + "I", + "D" + ], + [ + "Ģ", + "æľī" + ], + [ + "åĨ", + "Į" + ], + [ + "Ġrem", + "ain" + ], + [ + "Ġpl", + "atform" + ], + [ + "s", + "on" + ], + [ + "æľĢ", + "大" + ], + [ + "Ġo", + "pt" + ], + [ + "çļĦ", + "åŁºæľ¬" + ], + [ + "èĩ³", + "åħ³" + ], + [ + "èį¯", + "çī©" + ], + [ + "ar", + "m" + ], + [ + "æı", + "Ĵ" + ], + [ + "ib", + "r" + ], + [ + "è´¡", + "çĮ®" + ], + [ + "Ġa", + "fter" + ], + [ + "oh", + "n" + ], + [ + "æ±", + "ł" + ], + [ + "Ġb", + "ack" + ], + [ + "åį", + "ļ" + ], + [ + "Ġser", + "ies" + ], + [ + "t", + "ime" + ], + [ + "ĠS", + "p" + ], + [ + "er", + "ing" + ], + [ + "Ġn", + "etwor" + ], + [ + "us", + "s" + ], + [ + "Ġdesc", + "rib" + ], + [ + "é«ĺ", + "çļĦ" + ], + [ + "ĠY", + "ou" + ], + [ + "al", + "se" + ], + [ + "ä»ĸ们", + "çļĦ" + ], + [ + "åĮ", + "¹" + ], + [ + "t", + "y" + ], + [ + "æ", + "¥" + ], + [ + "è¯", + "ļ" + ], + [ + "æĿ", + "ij" + ], + [ + "æľ", + "¨" + ], + [ + "çļĦæ°", + "´" + ], + [ + "æķĪçİĩ", + "åĴĮ" + ], + [ + "å»", + "¶" + ], + [ + "av", + "es" + ], + [ + "per", + "ature" + ], + [ + "æĪij", + "æĥ³" + ], + [ + "yth", + "ing" + ], + [ + "ä¹", + "İ" + ], + [ + "æĶ¹", + "åĸĦ" + ], + [ + "éĵ¶", + "è¡Į" + ], + [ + "ï¼ļ", + "âĢľ" + ], + [ + "çŁ", + "©" + ], + [ + "åıĺ", + "å¾Ĺ" + ], + [ + "å®", + "½" + ], + [ + "åı¯ä»¥", + "帮åĬ©ä½ł" + ], + [ + "éģį", + "åİĨ" + ], + [ + "èĢĮ", + "ä¸įæĺ¯" + ], + [ + "Ġs", + "n" + ], + [ + "an", + "k" + ], + [ + "æ´", + "ŀ" + ], + [ + "è¿Ļ", + "æĺ¯ä¸Ģ个" + ], + [ + "¡", + "çľł" + ], + [ + "Ġ", + "è¾ĵåĩº" + ], + [ + "__", + "__" + ], + [ + "äº", + "¬" + ], + [ + "D", + "o" + ], + [ + "Ġs", + "ur" + ], + [ + "Ġal", + "ways" + ], + [ + "è±", + "Ĩ" + ], + [ + "Ġw", + "ater" + ], + [ + "st", + "it" + ], + [ + "éĿ", + "Ĵ" + ], + [ + "Ġl", + "im" + ], + [ + "Ġh", + "ig" + ], + [ + "`", + "," + ], + [ + "m", + "a" + ], + [ + "Ġposs", + "ible" + ], + [ + "Ġc", + "ir" + ], + [ + "Ġstate", + "ment" + ], + [ + "Ġmach", + "ine" + ], + [ + "Ġinteg", + "er" + ], + [ + "èĵ", + "Ŀ" + ], + [ + "ĠA", + "s" + ], + [ + "Ġh", + "our" + ], + [ + "丰å¯Į", + "çļĦ" + ], + [ + "Ġinterest", + "ed" + ], + [ + "±", + "ä¹IJ" + ], + [ + "æ³", + "¡" + ], + [ + "åį", + "ĥ" + ], + [ + "综", + "åIJĪ" + ], + [ + "ãĢĤ", + "âĢĿ" + ], + [ + "åįķ", + "è¯į" + ], + [ + "Ġc", + "ity" + ], + [ + "le", + "t" + ], + [ + "åĨ", + "Ľ" + ], + [ + "Ġin", + "de" + ], + [ + "Ġdet", + "erm" + ], + [ + "寻", + "æ±Ĥ" + ], + [ + "è¿Ľ", + "åħ¥" + ], + [ + "Ġbe", + "aut" + ], + [ + "Ġa", + "ir" + ], + [ + "Ð", + "»" + ], + [ + "Ġd", + "i" + ], + [ + "Ġhe", + "alth" + ], + [ + "é¡", + "»" + ], + [ + "n", + "ess" + ], + [ + "ç¢", + "³" + ], + [ + "å°", + "ģ" + ], + [ + "çļĦæ", + "´" + ], + [ + "`", + "." + ], + [ + "æĢ", + "İ" + ], + [ + "ä½į", + "äºİ" + ], + [ + "æ±½", + "车" + ], + [ + "çݯå¢ĥ", + "ä¸Ń" + ], + [ + "åĬĽ", + "åĴĮ" + ], + [ + "å½ĵ", + "åľ°" + ], + [ + "Ġob", + "ject" + ], + [ + "at", + "ural" + ], + [ + "å®Į", + "æķ´" + ], + [ + "ĠF", + "r" + ], + [ + "çĶ", + "³" + ], + [ + "Ġv", + "ary" + ], + [ + "Ġcont", + "ent" + ], + [ + "Ġb", + "it" + ], + [ + "Ġo", + "ur" + ], + [ + "çļĦç", + "ĥ" + ], + [ + "Ġun", + "ique" + ], + [ + "Ġwith", + "out" + ], + [ + "åĴĮ", + "è°" + ], + [ + "è¯Ń", + "éŁ³" + ], + [ + "åĪ", + "ł" + ], + [ + "伦", + "çIJĨ" + ], + [ + "our", + "ce" + ], + [ + "åĿ", + "ı" + ], + [ + "a", + "ve" + ], + [ + "çļĦ", + "æĸ¹å¼ı" + ], + [ + "Ġbe", + "ing" + ], + [ + "Ġc", + "ase" + ], + [ + "æ¯ı", + "ä¸Ģ" + ], + [ + "Ġs", + "al" + ], + [ + "èĤ", + "¤" + ], + [ + "Ġb", + "el" + ], + [ + "èĪ", + "Ĵ" + ], + [ + "Ġsec", + "urity" + ], + [ + "ç", + "¹ģ" + ], + [ + "u", + "ally" + ], + [ + "éĩį", + "æĸ°" + ], + [ + "çŃī", + "é¢ĨåŁŁ" + ], + [ + "é¢ĺ", + "缮" + ], + [ + "çļĦæ¯", + "ı" + ], + [ + "çļĦ", + "使çĶ¨" + ], + [ + "éĿ", + "©" + ], + [ + "ĠS", + "e" + ], + [ + "ãģ", + "«" + ], + [ + "Ġsc", + "ript" + ], + [ + "éĤ£", + "äºĽ" + ], + [ + "Ġm", + "on" + ], + [ + "æ®", + "Ĭ" + ], + [ + "身", + "份" + ], + [ + "Ġa", + "p" + ], + [ + "ĠR", + "e" + ], + [ + "'", + ":" + ], + [ + "çĶµ", + "èĦij" + ], + [ + "è§Ĥ", + "å¯Ł" + ], + [ + "è¿Ļ", + "æĦıåij³çĿĢ" + ], + [ + "æ¡Ĩ", + "æŀ¶" + ], + [ + "çļĦç¼", + "ĸ" + ], + [ + "Ġinclud", + "es" + ], + [ + "åı", + "³" + ], + [ + "Ġl", + "og" + ], + [ + "à", + "¤" + ], + [ + "çļĦç¾", + "İ" + ], + [ + "Ġ", + "er" + ], + [ + "Ġim", + "p" + ], + [ + "Ġ1", + "3" + ], + [ + "çľ", + "ģ" + ], + [ + "n", + "er" + ], + [ + "书", + "ç±į" + ], + [ + "çī¹", + "æ®Ĭ" + ], + [ + "åıĤ", + "åĬł" + ], + [ + "Ġd", + "ate" + ], + [ + "åıij", + "å¸ĥ" + ], + [ + "çļĦæī", + "Ģæľī" + ], + [ + "è¯", + "Ĭ" + ], + [ + "Ġp", + "ublic" + ], + [ + "éĥ¨", + "ç½²" + ], + [ + "ro", + "ss" + ], + [ + "ãģ", + "Ħ" + ], + [ + "åĪł", + "éĻ¤" + ], + [ + "Ġ", + "%" + ], + [ + "iz", + "ation" + ], + [ + "ä¼ł", + "æĴŃ" + ], + [ + "Ġma", + "xim" + ], + [ + "re", + "ate" + ], + [ + "b", + "ox" + ], + [ + "ç", + "ij" + ], + [ + "Ġd", + "own" + ], + [ + "èŃ", + "¦" + ], + [ + "Ġcour", + "se" + ], + [ + "so", + "le" + ], + [ + "Ġ", + "åĴĮ" + ], + [ + "åº", + "§" + ], + [ + "以ä¸ĭ", + "åĩł" + ], + [ + "ĠM", + "y" + ], + [ + "Ġ", + "Q" + ], + [ + "ï¼ī", + "ï¼ļ" + ], + [ + "éĢļ", + "ä¿¡" + ], + [ + "R", + "e" + ], + [ + "è½", + "®" + ], + [ + "ä¸Ĭ", + "ä¸ĭ" + ], + [ + "ç¥", + "Ŀ" + ], + [ + ":", + ":" + ], + [ + "g", + "or" + ], + [ + "è", + "ĸ" + ], + [ + "å¹´", + "é¾Ħ" + ], + [ + "Ġinv", + "ol" + ], + [ + "Ġtem", + "perature" + ], + [ + "ä¿¡", + "ä»»" + ], + [ + "æıIJ", + "éĹ®" + ], + [ + "Ġpro", + "f" + ], + [ + "i", + "er" + ], + [ + "è®°", + "å¿Ĩ" + ], + [ + "森", + "æŀĹ" + ], + [ + "ä¸įä»ħ", + "èĥ½å¤Ł" + ], + [ + "æĤ£", + "èĢħ" + ], + [ + "g", + "er" + ], + [ + "ç»Ļ", + "å®ļ" + ], + [ + "çļĦ", + "å°ı" + ], + [ + "è¯Ń", + "åı¥" + ], + [ + "ä½ł", + "éľĢè¦ģ" + ], + [ + "éģµ", + "å®Ī" + ], + [ + "Ġro", + "le" + ], + [ + "åŃĹ", + "åħ¸" + ], + [ + "Ġrele", + "ased" + ], + [ + "E", + "x" + ], + [ + "b", + "in" + ], + [ + "EC", + "T" + ], + [ + "o", + "y" + ], + [ + "æ¿Ģ", + "åıij" + ], + [ + "çĶ", + "ļ" + ], + [ + "Ġcon", + "vert" + ], + [ + "Ġcomm", + "and" + ], + [ + "åĮ¹", + "éħį" + ], + [ + "ploye", + "es" + ], + [ + "çļĦç", + "Ł¥" + ], + [ + "é", + "º" + ], + [ + "é¦", + "Ĩ" + ], + [ + "å¡", + "ij" + ], + [ + "Ġpos", + "itive" + ], + [ + "O", + "N" + ], + [ + "Ġcon", + "cept" + ], + [ + "ĠE", + "arth" + ], + [ + "Ġfriend", + "ly" + ], + [ + "åı¯", + "éĿł" + ], + [ + "å°ı", + "æŶ" + ], + [ + "åºĶçĶ¨", + "ç¨ĭåºı" + ], + [ + "è¿Ļ", + "段" + ], + [ + "Ġind", + "ivid" + ], + [ + "w", + "s" + ], + [ + "Ġnam", + "ed" + ], + [ + "Ġpl", + "an" + ], + [ + "Ġs", + "ort" + ], + [ + "Ġdo", + "c" + ], + [ + "è§Ĥ", + "çĤ¹" + ], + [ + "ãģ", + "§" + ], + [ + "Ġis", + "s" + ], + [ + "ä¸ĭ", + "çļĦ" + ], + [ + "设", + "æĸ½" + ], + [ + "åĴĮè°", + "IJ" + ], + [ + "空", + "æ°Ķ" + ], + [ + "æij", + "©" + ], + [ + "åĬ", + "³" + ], + [ + "请", + "æıIJä¾Ľ" + ], + [ + "Ġs", + "ources" + ], + [ + "ĠC", + "om" + ], + [ + "å¤įæĿĤ", + "度" + ], + [ + "a", + "pt" + ], + [ + "Ġsecon", + "d" + ], + [ + "æĶ»", + "åĩ»" + ], + [ + "'", + ")" + ], + [ + "Ġo", + "wn" + ], + [ + "Ġmean", + "ing" + ], + [ + "éĢ", + "Ģ" + ], + [ + "ä½", + "Ľ" + ], + [ + "è¡", + "£" + ], + [ + "ch", + "o" + ], + [ + "æľī", + "ä»Ģä¹Ī" + ], + [ + "æ²", + "³" + ], + [ + "缸", + "åºĶ" + ], + [ + "è", + "Ķ" + ], + [ + "çļĦ", + "åĢ¼" + ], + [ + "Ġeff", + "ect" + ], + [ + "L", + "ECT" + ], + [ + "s", + "et" + ], + [ + "og", + "n" + ], + [ + "H", + "T" + ], + [ + "Ġ", + "```" + ], + [ + "åIJ", + "ĥ" + ], + [ + "æĥħ", + "绪" + ], + [ + "éķ", + "ĩ" + ], + [ + "oc", + "us" + ], + [ + "éĢĤ", + "å½ĵ" + ], + [ + "å¿ħ", + "é¡»" + ], + [ + "çĶļ", + "èĩ³" + ], + [ + "åĽ¾", + "çīĩ" + ], + [ + "çļĦæĮ", + "ĩ" + ], + [ + "çļĦ", + "åŁºç¡Ģ" + ], + [ + "Ġph", + "ys" + ], + [ + "ç»", + "Ŀ" + ], + [ + "al", + "ity" + ], + [ + "æīĵ", + "å¼Ģ" + ], + [ + "Ġt", + "re" + ], + [ + "Ġbec", + "ause" + ], + [ + "ir", + "t" + ], + [ + "çļĦæĥħåĨµ", + "ä¸ĭ" + ], + [ + "Ġp", + "at" + ], + [ + "Ġ2", + "00" + ], + [ + "åĮĸ", + "åѦ" + ], + [ + "åģı", + "好" + ], + [ + "éĹ", + "Ń" + ], + [ + "åľ°", + "åĿĢ" + ], + [ + "it", + "er" + ], + [ + "ag", + "ing" + ], + [ + "å¦Ĥä½ķ", + "åľ¨" + ], + [ + "Ġt", + "ake" + ], + [ + "åIJį", + "为" + ], + [ + "åį", + "´" + ], + [ + "åĪĨ", + "åĪ«" + ], + [ + "çļĦç", + "ľ" + ], + [ + "Ġpl", + "ace" + ], + [ + "æľĢ", + "è¿ij" + ], + [ + "Ġ$", + "\\" + ], + [ + "Ã", + "¡" + ], + [ + "numb", + "er" + ], + [ + "Ġmaxim", + "um" + ], + [ + "ç¯", + "ĩ" + ], + [ + "r", + "ue" + ], + [ + "ĠI", + "nt" + ], + [ + "Ġ2", + "4" + ], + [ + "èĥ", + "ŀ" + ], + [ + "c", + "ase" + ], + [ + "Ġde", + "ep" + ], + [ + "RO", + "M" + ], + [ + "s", + "c" + ], + [ + "Ġin", + "cre" + ], + [ + "ĠW", + "e" + ], + [ + "éĥ½", + "ä¼ļ" + ], + [ + "ç²¾", + "ç¥ŀ" + ], + [ + "å", + "©" + ], + [ + "åįı", + "ä½ľ" + ], + [ + "åı¯ä»¥", + "æĺ¯" + ], + [ + "çŃī", + "å¤ļ个" + ], + [ + "çĽij", + "æµĭ" + ], + [ + "åĴĮ", + "æľįåĬ¡" + ], + [ + "Ġd", + "ue" + ], + [ + "Ġe", + "ar" + ], + [ + "è¿IJ", + "èIJ¥" + ], + [ + "o", + "f" + ], + [ + "æ±", + "¡" + ], + [ + "é«ĺ", + "度" + ], + [ + "æī", + "¹" + ], + [ + "çĸ", + "«" + ], + [ + "Ġindivid", + "ual" + ], + [ + "èĩªçĦ¶è¯Ńè¨Ģ", + "å¤ĦçIJĨ" + ], + [ + "e", + "le" + ], + [ + "s", + "ive" + ], + [ + "ä¹Ł", + "èĥ½" + ], + [ + "æĻº", + "æħ§" + ], + [ + "s", + "ort" + ], + [ + "a", + "f" + ], + [ + "åħ¬åı¸", + "çļĦ" + ], + [ + "çĶ¨", + "æĿ¥" + ], + [ + "çī¹", + "æĢ§" + ], + [ + "è", + "ļ" + ], + [ + "Ġex", + "act" + ], + [ + "åŃĹ", + "æ¯į" + ], + [ + "pl", + "ace" + ], + [ + "å¾Ī", + "å¤ļ" + ], + [ + "æĽ´", + "åĩĨç¡®" + ], + [ + "æľīæīĢ", + "帮åĬ©" + ], + [ + "ä¹ĭ", + "åīį" + ], + [ + "Ġt", + "akes" + ], + [ + "r", + "al" + ], + [ + "å®ĥ们", + "çļĦ" + ], + [ + "çİ°", + "åľ¨" + ], + [ + "å¹", + "ķ" + ], + [ + "Ġd", + "uring" + ], + [ + "ç¼ĸ", + "è¾ij" + ], + [ + "g", + "re" + ], + [ + "ä»", + "¿" + ], + [ + "æľī", + "ä¸Ģ" + ], + [ + "t", + "ers" + ], + [ + "Ġs", + "ince" + ], + [ + "ent", + "ion" + ], + [ + "æĬ", + "½" + ], + [ + "页", + "éĿ¢" + ], + [ + "æ½ľ", + "åľ¨" + ], + [ + "è¯Ń", + "æ³ķ" + ], + [ + "Ġf", + "ocus" + ], + [ + "æĢĿ", + "ç»´" + ], + [ + "Ġloc", + "ated" + ], + [ + "!", + "\"" + ], + [ + "çº", + "³" + ], + [ + "缸åħ³", + "çļĦ" + ], + [ + "é¡", + "¶" + ], + [ + "Ġhelp", + "ful" + ], + [ + "Ġfact", + "or" + ], + [ + "ang", + "le" + ], + [ + "缴", + "åĪ°" + ], + [ + "ĠSt", + "ates" + ], + [ + "æ¹", + "¿" + ], + [ + "æį", + "·" + ], + [ + "éĵ", + "ģ" + ], + [ + "è´Ł", + "è´£" + ], + [ + "ict", + "ion" + ], + [ + "Ġn", + "atural" + ], + [ + "Ġor", + "gan" + ], + [ + "å¨", + "±ä¹IJ" + ], + [ + "c", + "hes" + ], + [ + "Ġle", + "ft" + ], + [ + "Ġbl", + "ue" + ], + [ + "o", + "k" + ], + [ + "é»", + "ĺ" + ], + [ + "Ġma", + "de" + ], + [ + "Ġprovid", + "es" + ], + [ + "p", + "ose" + ], + [ + "çļĦç", + "ģ" + ], + [ + "éķ", + "ľ" + ], + [ + "åĿ", + "IJ" + ], + [ + "æĿ¥", + "å®ŀçİ°" + ], + [ + "ind", + "ows" + ], + [ + "ab", + "ilities" + ], + [ + "请", + "注æĦı" + ], + [ + "Ġte", + "am" + ], + [ + "éĢł", + "æĪIJ" + ], + [ + "çļĦ", + "èĥ½åĬĽ" + ], + [ + "éĨ", + "Ĵ" + ], + [ + "çĶ¨æĪ·", + "çļĦ" + ], + [ + "æĹ", + "ĭ" + ], + [ + "ra", + "ce" + ], + [ + "Ġph", + "rase" + ], + [ + "1", + "7" + ], + [ + "对", + "æĸ¹" + ], + [ + "Ġa", + "ge" + ], + [ + "ĠA", + "nd" + ], + [ + "çļĦ", + "é£İéĻ©" + ], + [ + "Ġa", + "ut" + ], + [ + "åį", + "«" + ], + [ + "ä¸Ģ", + "èĩ´" + ], + [ + "ç»Ĩ", + "èĥŀ" + ], + [ + "æĬĢæľ¯çļĦ", + "åıijå±ķ" + ], + [ + "çĶŁæĪIJ", + "ä¸Ģ个" + ], + [ + "Ġdiv", + "is" + ], + [ + "å°¤åħ¶", + "æĺ¯" + ], + [ + "çŃī", + "æĸ¹éĿ¢" + ], + [ + "Ġas", + "king" + ], + [ + "缮", + "çļĦ" + ], + [ + "第", + "äºĮ" + ], + [ + "èĪĴ", + "éĢĤ" + ], + [ + "æĬ", + "ĺ" + ], + [ + "Ġrec", + "ent" + ], + [ + "åįı", + "è®®" + ], + [ + "è¾", + "Ĩ" + ], + [ + "å·", + "¨" + ], + [ + "åī", + "Ĥ" + ], + [ + "t", + "he" + ], + [ + "æķ°", + "çļĦ" + ], + [ + "Ġhe", + "art" + ], + [ + "val", + "ue" + ], + [ + "éĿ¢", + "临" + ], + [ + "å®ŀéĻħ", + "ä¸Ĭ" + ], + [ + "æĿ", + "¯" + ], + [ + "d", + "d" + ], + [ + "Ġ", + "X" + ], + [ + "è´", + "µ" + ], + [ + "è¯ģ", + "æĺİ" + ], + [ + "å±", + "ı" + ], + [ + "ne", + "w" + ], + [ + "od", + "y" + ], + [ + "éģµ", + "循" + ], + [ + "Ġglob", + "al" + ], + [ + "Ġdep", + "ending" + ], + [ + "åĩ", + "Ģ" + ], + [ + "å·", + "¦" + ], + [ + "Ġ", + "Ø" + ], + [ + "æĽ´", + "åħ·ä½ĵçļĦ" + ], + [ + "帮åĬ©", + "æĤ¨" + ], + [ + "缸", + "äºĴ" + ], + [ + "Ġreturn", + "s" + ], + [ + "å¸ĮæľĽè¿ĻäºĽ", + "建议" + ], + [ + "Ġimpro", + "ve" + ], + [ + "Ġmu", + "ch" + ], + [ + "ä»ĸ", + "人" + ], + [ + "å¼", + "¹" + ], + [ + "4", + "5" + ], + [ + "åįķ", + "ä½į" + ], + [ + "çĦ", + "¦" + ], + [ + "åĪĹ表", + "ä¸Ń" + ], + [ + "`", + "ãĢĤ" + ], + [ + "con", + "om" + ], + [ + "èĥ½", + "éĩı" + ], + [ + "Ġcal", + "led" + ], + [ + "Ġr", + "ate" + ], + [ + "é¢Ħ", + "ç®Ĺ" + ], + [ + "èĩ³åħ³", + "éĩįè¦ģ" + ], + [ + "a", + "cc" + ], + [ + "ç»ĵ", + "æĿŁ" + ], + [ + "åı¯ä»¥", + "èĢĥèĻij" + ], + [ + "ç»Ĩ", + "èĬĤ" + ], + [ + "Ġ", + "ve" + ], + [ + "对", + "ä½ł" + ], + [ + "ĠThe", + "se" + ], + [ + "è¿ĩ", + "度" + ], + [ + "if", + "ied" + ], + [ + "è¯Ħ", + "ä»·" + ], + [ + "æķ°æį®", + "éĽĨ" + ], + [ + "é«ĺ", + "è´¨éĩı" + ], + [ + "èŀį", + "åIJĪ" + ], + [ + "åİŁ", + "åĪĻ" + ], + [ + "Ġappro", + "ximately" + ], + [ + "Ġrep", + "resent" + ], + [ + "ific", + "ial" + ], + [ + "as", + "on" + ], + [ + "æĸ¹", + "ç¨ĭ" + ], + [ + "Ġso", + "cial" + ], + [ + "Ġf", + "ig" + ], + [ + "SE", + "LECT" + ], + [ + "##", + "##" + ], + [ + "人", + "æ°ij" + ], + [ + "ãĢ", + "ij" + ], + [ + "he", + "st" + ], + [ + "ãĢ", + "IJ" + ], + [ + "Ġdig", + "ital" + ], + [ + "以ä¸ĭåĩłä¸ª", + "æĸ¹éĿ¢" + ], + [ + "Ġc", + "lo" + ], + [ + "å°Ĩ", + "åħ¶" + ], + [ + "çļĦ", + "åĨħ容" + ], + [ + "æı", + "¡" + ], + [ + "r", + "andom" + ], + [ + "å·", + "ŀ" + ], + [ + "è´Ń", + "çī©" + ], + [ + "h", + "ou" + ], + [ + "éĥ¨", + "éŨ" + ], + [ + "Ġsn", + "ipp" + ], + [ + "t", + "ract" + ], + [ + "çİ°", + "象" + ], + [ + "Ġneed", + "s" + ], + [ + "æķ°", + "åĪĹ" + ], + [ + "Ġg", + "ame" + ], + [ + "Ġt", + "ri" + ], + [ + "è´", + "´" + ], + [ + "çĭ¬", + "ç«ĭ" + ], + [ + "å¿ħè¦ģ", + "çļĦ" + ], + [ + "èĪ", + "ŀ" + ], + [ + "æ¸ħ", + "æ´ģ" + ], + [ + "象", + "å¾ģ" + ], + [ + "è½»", + "æĿ¾" + ], + [ + "é¸", + "¡" + ], + [ + "åĨ²", + "çªģ" + ], + [ + "Ġgener", + "al" + ], + [ + "Ġb", + "lo" + ], + [ + "çļĦæĦ", + "Ł" + ], + [ + "åĨħ", + "çļĦ" + ], + [ + "Ñ", + "ĥ" + ], + [ + "in", + "s" + ], + [ + "çļĦçŁ¥", + "è¯Ĩ" + ], + [ + "模", + "åĿĹ" + ], + [ + "éĻ¤", + "äºĨ" + ], + [ + "è¦", + "Ĩ" + ], + [ + "y", + "le" + ], + [ + "Ġint", + "elligence" + ], + [ + "åĨĻ", + "ä½ľ" + ], + [ + "Ġ", + "ed" + ], + [ + "Ġra", + "d" + ], + [ + "c", + "ing" + ], + [ + "P", + "lease" + ], + [ + "v", + "ed" + ], + [ + "èĥ½", + "帮åĬ©ä½ł" + ], + [ + "é¢Ŀ", + "å¤ĸ" + ], + [ + "Ġmark", + "et" + ], + [ + "Ġn", + "ight" + ], + [ + "in", + "put" + ], + [ + "æķ°æį®", + "åĪĨæŀIJ" + ], + [ + "æ´", + "²" + ], + [ + "m", + "e" + ], + [ + "w", + "h" + ], + [ + "Ġe", + "as" + ], + [ + "å¤ļ", + "å°ij" + ], + [ + "ãģ", + "¯" + ], + [ + "o", + "ck" + ], + [ + "C", + "reate" + ], + [ + "Ġloo", + "p" + ], + [ + "W", + "hen" + ], + [ + "ç", + "ħ" + ], + [ + "g", + "s" + ], + [ + "åı¯ä»¥", + "æł¹æį®" + ], + [ + "Ġrec", + "ogn" + ], + [ + "æĸ°", + "éĹ»" + ], + [ + "Ġ\\)", + "," + ], + [ + "ä¼ij", + "æģ¯" + ], + [ + "ĠM", + "a" + ], + [ + "ä¸Ĭä¸ĭ", + "æĸĩ" + ], + [ + "Ġm", + "em" + ], + [ + "åĬł", + "强" + ], + [ + "æ±", + "Ł" + ], + [ + "éĺ¿éĩĮå·´å·´", + "éĽĨåĽ¢" + ], + [ + "Ġsing", + "le" + ], + [ + "æ´", + "ĭ" + ], + [ + "P", + "U" + ], + [ + "im", + "ate" + ], + [ + "çļĦæ²", + "ŁéĢļ" + ], + [ + "åľ¨", + "æŁIJäºĽ" + ], + [ + "æµ", + "ª" + ], + [ + "ä¸", + "Ŀ" + ], + [ + "表", + "éĿ¢" + ], + [ + "èī¯", + "好" + ], + [ + "æŀ¶", + "æŀĦ" + ], + [ + "éĺ", + "»" + ], + [ + "ãģ", + "ª" + ], + [ + "åı¯æĮģç»Ń", + "åıijå±ķ" + ], + [ + "Ġen", + "h" + ], + [ + "sq", + "l" + ], + [ + "çŃī", + "äºİ" + ], + [ + "Ġmulti", + "ple" + ], + [ + "pl", + "ic" + ], + [ + "æĸĩ", + "åŃĹ" + ], + [ + "ä¹Ł", + "ä¼ļ" + ], + [ + "éĩ", + "İ" + ], + [ + "ount", + "ain" + ], + [ + "2", + "00" + ], + [ + "l", + "ing" + ], + [ + "åĪ¶", + "éĢł" + ], + [ + "âĢĿ", + "ï¼Ī" + ], + [ + "Ġdis", + "c" + ], + [ + "A", + "PI" + ], + [ + "Ġ1", + "7" + ], + [ + "èĻ", + "«" + ], + [ + "éļ", + "Ķ" + ], + [ + "pl", + "ify" + ], + [ + "Ġcl", + "ar" + ], + [ + "åĪ°", + "äºĨ" + ], + [ + "æ¹", + "ĸ" + ], + [ + "Ġd", + "er" + ], + [ + "Ġrequ", + "est" + ], + [ + "æŶ", + "åĢĻ" + ], + [ + "æľªæĿ¥", + "çļĦ" + ], + [ + "ç½", + "Ĺ" + ], + [ + "åħ¬", + "å¹³" + ], + [ + "Ġch", + "ang" + ], + [ + "æĮĩ", + "导" + ], + [ + "åı", + "«" + ], + [ + "Ġl", + "ess" + ], + [ + "æľī", + "çĿĢ" + ], + [ + "çļĦæł", + "¸" + ], + [ + "pp", + "er" + ], + [ + "on", + "y" + ], + [ + "éĢĤ", + "çĶ¨äºİ" + ], + [ + "Ġ", + "Ã" + ], + [ + "å®ļ", + "ä½į" + ], + [ + "æľī", + "趣" + ], + [ + "çļĦæŃ", + "£" + ], + [ + "Java", + "Script" + ], + [ + "åĬ¨", + "åĬĽ" + ], + [ + "un", + "d" + ], + [ + "ð", + "Ł" + ], + [ + "å¼Ģåıij", + "èĢħ" + ], + [ + "ç¾İ", + "åĽ½" + ], + [ + "èĪ", + "¬" + ], + [ + "..", + "." + ], + [ + "åĪĨ", + "为" + ], + [ + "çļĦæĮ", + "ijæĪĺ" + ], + [ + "Ġam", + "ount" + ], + [ + "Ġfil", + "m" + ], + [ + "å°½", + "éĩı" + ], + [ + "çļĦæ³", + "ķ" + ], + [ + "Ġprim", + "ary" + ], + [ + "Ġpop", + "ular" + ], + [ + "um", + "e" + ], + [ + "s", + "elf" + ], + [ + "8", + "0" + ], + [ + "æ¯Ķ", + "èµĽ" + ], + [ + "åIJį", + "åŃĹ" + ], + [ + "å¡", + "«" + ], + [ + "èĩªåĬ¨", + "åĮĸ" + ], + [ + "Ġ", + "es" + ], + [ + "磩", + "éĺµ" + ], + [ + "re", + "hen" + ], + [ + "ł", + "éģĵ" + ], + [ + "为", + "ä»Ģä¹Ī" + ], + [ + "çı", + "Ń" + ], + [ + "Ġcap", + "abilities" + ], + [ + "Ġsnipp", + "et" + ], + [ + "è·", + "Ł" + ], + [ + "<", + "/" + ], + [ + "Ð", + "º" + ], + [ + "è¸", + "ª" + ], + [ + "en", + "ces" + ], + [ + "Ġwid", + "ely" + ], + [ + "Ġd", + "el" + ], + [ + "Ġcon", + "c" + ], + [ + "n", + "ov" + ], + [ + "åİ", + "ļ" + ], + [ + "è¿Ļ个", + "éĹ®é¢ĺ" + ], + [ + "éĢļ常", + "ä¼ļ" + ], + [ + "ä¿Ŀ", + "åŃĺ" + ], + [ + "at", + "s" + ], + [ + "è¿ľ", + "ç¨ĭ" + ], + [ + "t", + "ed" + ], + [ + "è´¦", + "æĪ·" + ], + [ + "åħ¬", + "ä¼Ĺ" + ], + [ + "O", + "S" + ], + [ + "id", + "er" + ], + [ + "Ġg", + "roup" + ], + [ + "e", + "xt" + ], + [ + "Ġn", + "ature" + ], + [ + "e", + "k" + ], + [ + "表达", + "å¼ı" + ], + [ + "Ġunderstand", + "ing" + ], + [ + "ile", + "d" + ], + [ + "Ġeff", + "ic" + ], + [ + "çĶ³", + "请" + ], + [ + "è¯Ĭ", + "æĸŃ" + ], + [ + "模", + "æĭŁ" + ], + [ + "ip", + "ly" + ], + [ + "çĶ¨æĪ·", + "ä½ĵéªĮ" + ], + [ + "ol", + "s" + ], + [ + "éĽĨ", + "æĪIJ" + ], + [ + "éģĵ", + "å¾·" + ], + [ + "é¾", + "Ļ" + ], + [ + "com", + "e" + ], + [ + "4", + "0" + ], + [ + "Ġhist", + "ory" + ], + [ + "æİ", + "Ī" + ], + [ + "污", + "æŁĵ" + ], + [ + "é", + "Ł" + ], + [ + "æıIJé«ĺ", + "äºĨ" + ], + [ + "Ġrel", + "ated" + ], + [ + "å¥", + "Ĺ" + ], + [ + "åı¯èĥ½", + "æĺ¯" + ], + [ + "æ¯ı", + "天" + ], + [ + "an", + "cial" + ], + [ + "åģ", + "¶" + ], + [ + "Ġsc", + "ient" + ], + [ + "éĿŀ常", + "éĩįè¦ģ" + ], + [ + "ch", + "ie" + ], + [ + "åı¯ä»¥", + "çĶ¨äºİ" + ], + [ + "Ġorig", + "inal" + ], + [ + "æľĢ", + "å°ı" + ], + [ + "èĤ", + "Į" + ], + [ + "åij¼", + "åIJ¸" + ], + [ + "Ġno", + "vel" + ], + [ + "Ġupd", + "ates" + ], + [ + "çļĦæĥħ", + "æĦŁ" + ], + [ + "ä½ĵ", + "ç³»" + ], + [ + "èĭ±", + "è¯Ń" + ], + [ + "Al", + "ibaba" + ], + [ + "éĢĴ", + "å½Ĵ" + ], + [ + "çļĦ", + "ä¸Ģ个" + ], + [ + "ist", + "s" + ], + [ + "Ġs", + "ize" + ], + [ + "åľ¨", + "å®ŀéĻħ" + ], + [ + "d", + "ata" + ], + [ + "æĪIJ", + "为äºĨ" + ], + [ + "Ġf", + "ur" + ], + [ + "æ»", + "¤" + ], + [ + "Ġrequ", + "ire" + ], + [ + "æķı", + "æĦŁ" + ], + [ + "æİ", + "Į" + ], + [ + "ãĤ", + "ĭ" + ], + [ + "Ġt", + "est" + ], + [ + "åıij", + "éĢģ" + ], + [ + "Ġap", + "pe" + ], + [ + "Ġk", + "il" + ], + [ + "ç¨ĭ", + "度" + ], + [ + "Ġfe", + "w" + ], + [ + "æ°Ķ", + "åĢĻ" + ], + [ + "ç¡®", + "认" + ], + [ + "æĪIJ", + "æŀľ" + ], + [ + "Ġne", + "xt" + ], + [ + "ä¸ĵ", + "注" + ], + [ + "Ġvari", + "able" + ], + [ + "Ġbe", + "gin" + ], + [ + "box", + "ed" + ], + [ + "Ġs", + "il" + ], + [ + "åѦ", + "æł¡" + ], + [ + "Ġfeature", + "s" + ], + [ + "æĿ¥", + "èĩª" + ], + [ + "çļĦæĻ", + "º" + ], + [ + "å®ī", + "æİĴ" + ], + [ + "å®", + "ģ" + ], + [ + "èµĦ", + "éĩij" + ], + [ + "p", + "ar" + ], + [ + "èī²", + "彩" + ], + [ + "让", + "æĪij们" + ], + [ + "am", + "es" + ], + [ + "èIJ¥", + "åħ»" + ], + [ + "Ġf", + "ore" + ], + [ + "Ġe", + "conom" + ], + [ + "Ġg", + "ent" + ], + [ + "é©", + "±" + ], + [ + "å±ŀ", + "æĢ§" + ], + [ + "ç®Ģ", + "æ´ģ" + ], + [ + "Ġcomp", + "rehen" + ], + [ + "ç«ĭ", + "åį³" + ], + [ + "Ġ\\)", + "." + ], + [ + "Ġequ", + "al" + ], + [ + "çļĦ", + "代çłģ" + ], + [ + "Ġbeaut", + "y" + ], + [ + "大", + "è§Ħ模" + ], + [ + "åIJij", + "éĩı" + ], + [ + "ı", + "è§Ī" + ], + [ + "æľ", + "«" + ], + [ + "re", + "qu" + ], + [ + "åĪ°", + "çļĦ" + ], + [ + "åĬł", + "éĢŁ" + ], + [ + "_", + "{" + ], + [ + "Ù", + "Ĭ" + ], + [ + "Ġd", + "ays" + ], + [ + "è´", + "¥" + ], + [ + "Ġstand", + "ard" + ], + [ + "ĠA", + "n" + ], + [ + "è´¹", + "çĶ¨" + ], + [ + "èijĹ", + "åIJį" + ], + [ + "æ±", + "ī" + ], + [ + "m", + "in" + ], + [ + "çļĦç", + "²" + ], + [ + "Ġlangu", + "ages" + ], + [ + "irt", + "ual" + ], + [ + "Ġp", + "res" + ], + [ + "Ġv", + "ast" + ], [ "ĥ", "åľ" ], + [ + "pl", + "es" + ], + [ + "çļĦæ´", + "»" + ], + [ + "åĵ", + "²" + ], + [ + "çļĦæĶ", + "¯" + ], + [ + "硬", + "件" + ], + [ + "é¢Ĩ", + "导" + ], + [ + "èĤ¡", + "票" + ], [ "åŀ", "ĥåľ" @@ -23382,7644 +24467,6724 @@ "¾" ], [ - "Ġd", - "rug" + "åı¯ä»¥", + "æıIJé«ĺ" ], [ - "ç¢", - "³" + "éĹ®é¢ĺ", + "æĪĸ" ], [ - "Ġn", - "ame" - ], - [ - "åĮĸ", - "çļĦ" - ], - [ - "a", - "id" - ], - [ - "æľĢ", - "大" - ], - [ - "æij", - "Ħ" - ], - [ - "ç®Ģåįķ", - "çļĦ" - ], - [ - "Ġw", - "arm" - ], - [ - "Ġd", - "one" - ], - [ - "Ġfun", - "ction" - ], - [ - "as", - "c" - ], - [ - "强", - "è°ĥ" - ], - [ - "Ġdem", - "and" - ], - [ - "Ġvis", - "ual" - ], - [ - "Ġup", - "d" - ], - [ - "æŃ£", - "åľ¨" - ], - [ - "Ġsim", - "ilar" - ], - [ - "éĢ", - "Ĵ" - ], - [ - "æ¯", - "Ľ" - ], - [ - "éĶ", - "»" - ], - [ - "ent", - "ly" - ], - [ - "Ġvalu", - "able" - ], - [ - "Ġdis", - "aster" - ], - [ - "ä¸Ģ", - "èĪ¬" - ], - [ - "æ´", - "²" - ], - [ - "ĠR", - "eg" - ], - [ - "Ġdiscrim", - "ination" - ], - [ - "åĨĻ", - "ä¸Ģç¯ĩ" - ], - [ - "Ġgovern", - "ment" - ], - [ - "Ġ", - "好çļĦ" - ], - [ - "5", - "00" - ], - [ - "ly", - "ing" - ], - [ - "Ġpre", - "v" - ], - [ - "Ġpre", - "pare" - ], - [ - "Ġproble", - "ms" - ], - [ - "è·", - "³" - ], - [ - "Ġpro", - "m" - ], - [ - "åĨ", - "²" - ], - [ - "å®ī", - "è£ħ" - ], - [ - "éĶ»", - "çĤ¼" - ], - [ - "æµ", - "ĵ" - ], - [ - "è", - "¹" - ], - [ - "åºĶçĶ¨", - "ç¨ĭåºı" - ], - [ - "n", - "g" - ], - [ - "Ġcomp", - "et" - ], - [ - "åĪĨ", - "åĪ«" - ], - [ - "olo", - "gical" - ], - [ - "å®", - "¡" - ], - [ - "Ġtrans", - "l" - ], - [ - "Ġdire", - "ct" - ], - [ - "åī", - "Ĥ" - ], - [ - "Ġsuggest", - "ions" - ], - [ - "Ġpap", - "er" - ], - [ - "Ġrecogn", - "ize" - ], - [ - "t", - "on" - ], - [ - "Ġmit", - "igate" - ], - [ - "讨", - "论" - ], - [ - "äºĴ", - "åĬ¨" - ], - [ - "ĠE", - "ar" - ], - [ - "Ġam", - "azing" - ], - [ - "c", - "re" - ], - [ - "é¦", - "Ī" - ], - [ - "Ġinvol", - "ved" - ], - [ - "f", - "ace" - ], - [ - "æľī", - "åħ³" - ], - [ - ")", - ")" - ], - [ - "Ġex", - "ce" - ], - [ - "Ġproduct", - "ivity" - ], - [ - "è", - "Ń" - ], - [ - "é¦", - "Ĩ" - ], - [ - "Ġsound", - "s" - ], - [ - "Ġidentify", - "ing" - ], - [ - "]", - "," - ], - [ - "é¾", - "Ļ" - ], - [ - "Ġf", - "it" - ], - [ - "Ġcontribut", - "e" - ], - [ - "th", - "s" - ], - [ - "friend", - "ly" - ], - [ - "e", - "le" - ], - [ - "if", - "ied" - ], - [ - "iven", - "ess" - ], - [ - "ite", - "ly" - ], - [ - "Ġ", - "X" - ], - [ - "Ġl", - "ed" - ], - [ - "åĿ", - "ı" - ], - [ - "Ġhist", - "or" - ], - [ - "Ġd", - "at" - ], - [ - "Ġjour", - "ney" - ], - [ - "Ġ", - "}" - ], - [ - "Ġse", - "lect" - ], - [ - "æ¼", - "«" - ], - [ - "Ġcon", - "duct" - ], - [ - "è¿Ľ", - "ä¸ĢæŃ¥" - ], - [ - "ç»Ļ", - "æĪij" - ], - [ - "Ġl", - "if" - ], - [ - "è£ħ", - "ä¿®" - ], - [ - "为", - "ä»Ģä¹Ī" - ], - [ - "äº", - "¬" - ], - [ - "Ġn", - "av" - ], - [ - "Ġwho", - "le" - ], - [ - "ç", - "¹ģ" - ], - [ - "åĨ", - "ľ" - ], - [ - "æĶ", - "»" - ], - [ - "Ġb", - "reat" - ], - [ - "Ġm", - "iss" - ], - [ - "é¾", - "Ħ" - ], - [ - "t", - "t" - ], - [ - "s", - "w" - ], - [ - "Ġb", - "ar" - ], - [ - "请", - "éĹ®" - ], - [ - "èģĶ", - "ç½ij" - ], - [ - "Ġatt", - "ract" - ], - [ - "æĤ¨", - "åı¯ä»¥" - ], - [ - "O", - "ne" - ], - [ - "åħħ", - "åĪĨ" - ], - [ - "r", - "ing" - ], - [ - "Ġå½ĵ", - "çĦ¶" - ], - [ - "re", - "am" - ], - [ - "Ġev", - "ol" - ], - [ - "Ġs", - "n" - ], - [ - "ĠE", - "m" - ], - [ - "m", - "osp" - ], - [ - "Ġcho", - "ose" - ], - [ - "v", - "iew" - ], - [ - "Ġar", - "r" - ], - [ - "Ġs", - "leep" - ], - [ - "end", - "ed" - ], - [ - "æŀ", - "¶" - ], - [ - "Ġveh", - "icles" - ], - [ - "Ġf", - "resh" - ], - [ - "Ġorganiz", - "ation" - ], - [ - "è¿Ļ", - "段" - ], - [ - "æ±", - "¤" - ], - [ - "ĠI", - "nt" - ], - [ - "Ġcont", - "ext" - ], - [ - "åı¦", - "å¤ĸ" - ], - [ - "Ġoce", - "an" - ], - [ - "æĦŁ", - "åıĹ" - ], - [ - "Ġpollut", - "ion" - ], - [ - "ur", - "b" - ], - [ - "æī§", - "è¡Į" - ], - [ - "erson", - "al" - ], - [ - "ĠHe", - "alth" - ], - [ - "ä¼ĺ", - "çĤ¹" - ], - [ - "Ġatt", - "ention" - ], - [ - "æľī", - "çĿĢ" - ], - [ - "é£Ł", - "æĿIJ" - ], - [ - "Ġer", - "r" - ], - [ - "çļĦæĿ", - "¥" - ], - [ - "çļĦç", - "Ī" - ], - [ - "èŃ", - "¦" - ], - [ - "è·", - "Ł" - ], - [ - "æĹħ", - "è¡Į" - ], - [ - "èĴ", - "ľ" - ], - [ - "çļĦæĢ", - "Ŀ" - ], - [ - "Ġchat", - "bot" - ], - [ - "çļĦ", - "éľĢæ±Ĥ" - ], - [ - "çķ", - "¥" - ], - [ - "Ġfeel", - "ing" - ], - [ - "Ġimplement", - "ed" - ], - [ - "社", - "åĮº" - ], - [ - "çļĦ", - "建议" - ], - [ - "æIJ", - "ħ" - ], - [ - "éĹ", - "»" - ], - [ - "åıį", - "é¦Ī" - ], - [ - "缴", - "æİ¥" - ], - [ - "æĺ", - "¥" - ], - [ - "it", - "able" - ], - [ - "æĪij", - "ä¼ļ" - ], - [ - "åį", - "±" - ], - [ - "èī¯", - "好" - ], - [ - "Ġl", - "iving" - ], - [ - "åıĺ", - "éĩı" - ], - [ - "ĠB", - "ut" - ], - [ - "Ġcomple", - "te" - ], - [ - "Ġtre", - "nds" - ], - [ - "Ġm", - "akes" - ], - [ - "ä»Ĭ", - "天" - ], - [ - "Ġdist", - "ribut" - ], - [ - "Ġcomm", - "it" - ], - [ - "Ġat", - "mosp" - ], - [ - "ä¼", - "´" - ], - [ - "Ġsens", - "ors" - ], - [ - "Ġs", - "w" - ], - [ - "æĹł", - "论" - ], - [ - "om", - "en" - ], - [ - "æĶ¿", - "åºľ" - ], - [ - "Ġchall", - "enge" - ], - [ - "Ġt", - "urn" - ], - [ - "çIJĨ", - "论" - ], - [ - "p", - "ar" - ], - [ - "Ġwrit", - "e" - ], - [ - "ç»ı", - "åħ¸" - ], - [ - "em", - "ember" - ], - [ - "é¥", - "Ń" - ], - [ - "æĸ¹", - "便" - ], - [ - "Ġc", - "u" - ], - [ - "Ġval", - "ue" - ], - [ - "Ġf", - "und" - ], - [ - "p", - "ose" - ], - [ - "è°ĥ", - "æŁ¥" - ], - [ - "çĿ", - "¡" - ], - [ - "Ġcommunic", - "ate" - ], - [ - "Ġdise", - "ase" - ], - [ - "Ġrese", - "arc" - ], - [ - "Ġl", - "ack" - ], - [ - "arn", - "ing" - ], - [ - "ĠP", - "ark" - ], - [ - "çĦ", - "¦" - ], - [ - "é«ĺ", - "度" - ], - [ - "Ġr", - "ather" - ], - [ - "å®", - "£" - ], - [ - "çĪ", - "¶" - ], - [ - "éĺ", - "¶" - ], - [ - "è®", - "¢" - ], - [ - "çĥ", - "§" - ], - [ - "Ġhig", - "her" - ], - [ - "Ġsumm", - "ary" - ], - [ - "ĠA", - "ut" - ], - [ - "çļĦæ", - "³" - ], - [ - "Ġe", - "le" - ], - [ - "is", - "ms" - ], - [ - "Ġrel", - "i" - ], - [ - "ä¹Ł", - "ä¼ļ" - ], - [ - "f", - "ra" - ], - [ - "åijĬè¯ī", - "æĪij" - ], - [ - "æĬ", - "½" - ], - [ - "Ġsitu", - "ations" - ], - [ - "Ġmar", - "ine" - ], - [ - "æĥ³", - "è¦ģ" - ], - [ - "in", - "ci" - ], - [ - "in", - "al" - ], - [ - "Ġg", - "ain" - ], - [ - "Ġdiffere", - "nce" - ], - [ - "æľºåĻ¨", - "人" - ], - [ - "æµģ", - "ç¨ĭ" - ], - [ - "ĠC", - "hat" - ], - [ - "ç½ij", - "ç«Ļ" - ], - [ - "æľ", - "«" - ], - [ - "Ġcol", - "or" - ], - [ - "Ġas", - "pect" - ], - [ - "ç½", - "Ĺ" - ], - [ - "ĠE", - "duc" - ], - [ - "Ġde", - "ploy" - ], - [ - "Ġbeaut", - "y" - ], - [ - "æĤ", - "£" - ], - [ - "ruct", - "ion" - ], - [ - "it", - "ut" - ], - [ - "æĿ", - "Ł" - ], - [ - "让", - "æĪij们" - ], - [ - "éķ¿", - "度" - ], - [ - "ul", - "es" - ], - [ - "æ¶ī", - "åıĬ" - ], - [ - "Ġdig", - "ital" - ], - [ - "Ġexist", - "ing" - ], - [ - "ĠO", - "r" - ], - [ - "\\_", - "\\_" - ], - [ - "Ġback", - "ground" - ], - [ - "çĹ", - "ĩ" - ], - [ - "æ¯ı", - "天" - ], - [ - "p", - "ython" - ], - [ - "Ġfarm", - "ers" - ], - [ - "Ġcontin", - "u" - ], - [ - "\"", - ":" - ], - [ - "Ġg", - "iven" - ], - [ - "å°ı", - "æŶ" - ], - [ - "Ġmom", - "ent" - ], - [ - "2", - "00" - ], - [ - "J", - "ohn" - ], - [ - "éĿ¢", - "对" - ], - [ - "Ġint", - "ro" - ], - [ - "Ġtherap", - "y" - ], - [ - "è¿Ķ", - "åĽŀ" - ], - [ - "并", - "åľ¨" - ], - [ - "Ġ", - "z" - ], - [ - "Ġaff", - "ord" - ], - [ - "ä¸", - "Ŀ" - ], - [ - "å®", - "½" - ], - [ - "Ġ", - "Ã" - ], - [ - "ĠN", - "ational" - ], - [ - "èĥ", - "¡" - ], - [ - "Ġexercis", - "e" - ], - [ - "æIJħ", - "æĭĮ" - ], - [ - "æĶ¯", - "ä»ĺ" - ], - [ - "éĺ³", - "åħī" - ], - [ - "è¯", - "ļ" - ], - [ - "Ġs", - "ect" - ], - [ - "ĠS", - "u" - ], - [ - "å¢ŀ", - "éķ¿" - ], - [ - "ç¾İ", - "丽" - ], - [ - "Ġw", - "a" - ], - [ - "以ä¸ĭæĺ¯", - "ä¸ĢäºĽ" - ], - [ - "èĽĭ", - "ç³ķ" - ], - [ - "Ġ", - "ill" - ], - [ - "æ¸ħ", - "æĻ" - ], - [ - "et", - "ry" - ], - [ - "æ¢", - "¦" - ], - [ - "ç¾İ", - "åĽ½" - ], - [ - "ä»", - "į" - ], - [ - "one", - "y" - ], - [ - "Ġecosystem", - "s" - ], - [ - "æĮĩ", - "导" - ], - [ - "d", - "ef" - ], - [ - "9", - "9" - ], - [ - "æŁ", - "Ķ" - ], - [ - "pp", - "ed" - ], - [ - "Ġlim", - "it" - ], - [ - "çİ", - "ī" - ], - [ - "Ġacad", - "emic" - ], - [ - "Ġrestaur", - "ants" - ], - [ - "Ġhe", - "ad" - ], - [ - "ä¿¡", - "ä»»" - ], - [ - "ast", - "ers" - ], - [ - "å²", - "ģ" - ], - [ - "ak", - "ers" - ], - [ - "1", - "4" - ], - [ - "A", - "s" - ], - [ - "æł", - "¡" - ], - [ - "é«ĺ", - "æķĪ" - ], - [ - "ph", - "as" - ], - [ - "y", - "n" - ], - [ - "ç¨ĭ", - "度" - ], - [ - "è¾", - "£" - ], - [ - "ä¸Ĭ", - "éĿ¢" - ], - [ - "家", - "å±ħ" - ], - [ - "ter", - "m" - ], - [ - "ç¾İ", - "é£Ł" - ], - [ - "Ġo", - "vers" - ], - [ - "å®", - "ĺ" - ], - [ - "Ġind", - "ic" - ], - [ - "ĠY", - "our" - ], - [ - "S", - "t" - ], - [ - "å½¢", - "象" - ], - [ - "è´", - "¡" - ], - [ - "åº", - "Ĭ" - ], - [ - "ĠS", - "c" - ], - [ - "ag", - "ra" - ], - [ - "羣", - "æŃ£" - ], - [ - "o", - "int" - ], - [ - "id", - "s" - ], - [ - "are", - "nt" - ], - [ - "éĵ", - "¶" - ], - [ - "èģ", - "Ĭ" - ], - [ - "Ġreg", - "ular" - ], - [ - "ä¼ĺ", - "ç§Ģ" - ], - [ - "Ġcol", - "le" - ], - [ - "çĸ", - "ij" - ], - [ - "Ġsub", - "ject" - ], - [ - "Ġgreat", - "er" - ], - [ - "Ġst", - "ore" - ], - [ - "åŁ¹", - "è®Ń" - ], - [ - "Ġim", - "ag" - ], - [ - "Ġan", - "sw" - ], - [ - "ä½", - "Ļ" - ], - [ - "Ġsp", - "ot" - ], - [ - "åĪĨ", - "åŃIJ" - ], - [ - "Ġaud", - "ience" - ], - [ - "p", - "et" - ], - [ - "Ġv", - "ers" - ], - [ - "Ġtra", - "il" - ], - [ - "åĭ", - "ĩ" - ], - [ - "er", - "ous" - ], - [ - "Ġguid", - "ance" - ], - [ - "Ġspe", - "ech" - ], - [ - "åĵ", - "²" - ], - [ - "æĺ¯", - "çĶ±" - ], - [ - "è´¡", - "çĮ®" - ], - [ - "åIJĪéĢĤ", - "çļĦ" - ], - [ - "设", - "æĸ½" - ], - [ - "ä»ĸ", - "人" - ], - [ - "ens", - "ive" - ], - [ - "åĢ", - "¾" - ], - [ - "al", - "ing" - ], - [ - "Ġproject", - "s" - ], - [ - "å", - "³" - ], - [ - "Ġt", - "akes" - ], - [ - "ç»", - "©" - ], - [ - "T", - "hat" - ], - [ - "Ġb", - "ro" - ], - [ - "iv", - "ed" - ], - [ - "Ġ", - "&" - ], - [ - "åĿ", - "IJ" - ], - [ - "place", - "ment" - ], - [ - "è¿ŀ", - "æİ¥" - ], - [ - "çļĦç¤", - "¾" - ], - [ - "ĠT", - "ra" - ], - [ - "Ġrel", - "ax" - ], - [ - "u", - "fact" - ], - [ - "éģ", - "į" - ], - [ - "Ġsur", - "v" - ], - [ - "åı£", - "åij³" - ], - [ - "Ġcreat", - "ivity" - ], - [ - "o", - "f" - ], - [ - "å¨", - "ģ" - ], - [ - "çļĦç", - "ł" - ], - [ - "Ġbreat", - "h" - ], - [ - "Ġpl", - "aces" - ], - [ - "Ġdesc", - "rib" - ], - [ - "èĭ±", - "è¯Ń" - ], - [ - "Ġdam", - "age" - ], - [ - "or", - "ation" - ], - [ - "为", - "æĤ¨" - ], - [ - "if", - "t" - ], - [ - "Ġc", - "ase" - ], - [ - "å¹´", - "é¾Ħ" - ], - [ - "Ġp", - "ress" - ], - [ - "çĶ", - "ľ" - ], - [ - "éĩ", - "İ" - ], - [ - "æĹħ", - "游" - ], - [ - "Ġt", - "aken" - ], - [ - "in", - "ed" - ], - [ - "Ġcon", - "cept" - ], - [ - "æĴ", - "Ń" - ], - [ - "Ġinterest", - "ing" - ], - [ - "è·", - "µ" - ], - [ - "Ġse", - "a" - ], - [ - "6", - "0" - ], - [ - "Ġf", - "oot" - ], - [ - "ĠN", - "ame" - ], - [ - "Ġresearc", - "hers" - ], - [ - "éĢ", - "ģ" - ], - [ - "Ġwe", - "e" - ], - [ - ")", - ";" - ], - [ - "çļĦ", - "åħ³éĶ®" - ], - [ - "ä¼", - "½" - ], - [ - "ele", - "br" - ], - [ - "å¡", - "ij" - ], - [ - "W", - "e" - ], - [ - "ç»ı", - "常" - ], - [ - "Ġpopul", - "ations" - ], - [ - "åħ¬", - "å¼ı" - ], - [ - "or", - "n" - ], - [ - "çĩ", - "ĥ" - ], - [ - "人", - "çĶŁ" - ], - [ - "1", - "7" - ], - [ - "æİ¥", - "åıĹ" - ], - [ - "Ġloc", - "ation" - ], - [ - "Ġin", - "equ" - ], - [ - "Ġinter", - "vent" - ], - [ - "Ġinterest", - "ed" - ], - [ - "Ġdefin", - "itely" - ], - [ - "Ġassist", - "ance" - ], - [ - "è¿Ļ", - "ä¸Ģ" - ], - [ - "åIJĪ", - "åIJĮ" - ], - [ - "ä¼ĺ", - "åĬ¿" - ], - [ - "çļĦ", - "å·¥ä½ľ" - ], - [ - "Ġ1", - "2" - ], - [ - "Ġmo", - "v" - ], - [ - "åģ", - "ı" - ], - [ - "åŃĺ", - "åĤ¨" - ], - [ - "us", - "ive" - ], - [ - "æĹ", - "ı" - ], - [ - "ï¼ī", - "ï¼Į" - ], - [ - "Ġg", - "as" - ], - [ - "Ġinterest", - "s" - ], - [ - "æ¸ħæĻ", - "°" - ], - [ - "Ġg", - "ard" - ], - [ - "çĸ", - "«" - ], - [ - "Ġs", - "ay" - ], - [ - "å¤", - "«" - ], - [ - "g", - "es" - ], - [ - "èIJ", - "¨" - ], - [ - "ä¸ļ", - "åĬ¡" - ], - [ - "个", - "æĢ§" - ], - [ - "åIJ", - "¯" - ], - [ - "Ġeng", - "agement" - ], - [ - "Ġb", - "ig" - ], - [ - "éľĢè¦ģ", - "èĢĥèĻij" - ], - [ - "Ġpr", - "inci" - ], - [ - "åij¨", - "åĽ´" - ], - [ - "Ġopportun", - "ity" - ], - [ - "çģ", - "¾" - ], - [ - "èĹ", - "ı" - ], - [ - "re", - "l" - ], - [ - "缺", - "çĤ¹" - ], - [ - "Ġhapp", - "y" - ], - [ - "åĴĮ", - "åħ¶ä»ĸ" - ], - [ - "av", - "a" - ], - [ - "Ġestab", - "lish" - ], - [ - "鸡", - "èĽĭ" - ], - [ - "i", - "king" - ], - [ - "ĠT", - "rans" - ], - [ - "rast", - "ructure" - ], - [ - "fore", - "st" - ], - [ - "èİ·", - "åıĸ" - ], - [ - "èĦ", - "ļ" - ], - [ - "in", - "ally" - ], - [ - "èµ", - "ı" - ], - [ - "Ġdel", - "icious" - ], - [ - "Ġresult", - "s" - ], - [ - "è§Ĥ", - "å¯Ł" - ], - [ - "å®ŀ", - "è·µ" - ], - [ - "Ġl", - "ast" - ], - [ - "Ġpol", - "it" - ], - [ - "æĢ§", - "èĥ½" - ], - [ - "F", - "or" - ], - [ - "b", - "i" - ], - [ - "缸", - "ä¿¡" - ], - [ - "ff", - "ee" - ], - [ - "Ġph", - "r" - ], - [ - "Ġfore", - "st" - ], - [ - "ell", - "ing" - ], - [ - "æµģ", - "è¡Į" - ], - [ - "at", - "ic" - ], - [ - "大", - "家" - ], - [ - "ĠIn", - "st" - ], - [ - "æķ°", - "åѦ" - ], - [ - "æī", - "©" - ], - [ - "å®Į", - "åħ¨" - ], - [ - "å¼ķ", - "èµ·" - ], - [ - "es", - "e" - ], - [ - "转", - "æį¢" - ], - [ - "Ġaffect", - "ed" - ], - [ - "Ġrobot", - "ics" - ], - [ - "综", - "ä¸Ĭ" - ], - [ - "Ġpro", - "p" - ], - [ - "让", - "人" + "Ġre", + "se" ], [ "æ²", + "ī" + ], + [ + "è·", "³" ], [ - "ä¸Ń", - "æľĢ" + "Ġ", + "ê" ], [ - "Ġaut", - "onomous" + "æİ¥", + "æĶ¶" + ], + [ + "ut", + "ure" + ], + [ + "on", + "d" ], [ "Ġha", - "ving" - ], - [ - "Ġtri", - "p" - ], - [ - "ur", - "y" - ], - [ - "Ġbi", - "ased" - ], - [ - "Ġconsider", - "ations" - ], - [ - "Ġpartic", - "ular" - ], - [ - "åį", - "ł" - ], - [ - "æݨ", - "广" - ], - [ - "Ġiniti", - "atives" - ], - [ - "ial", - "s" - ], - [ - "åij³", - "éģĵ" - ], - [ - "Ġtreat", - "ments" - ], - [ - "Ġem", - "phas" - ], - [ - "çĭ¬çī¹", - "çļĦ" - ], - [ - "Ġl", - "ay" - ], - [ - "æĶ¿", - "çŃĸ" - ], - [ - "æĢİ", - "ä¹Ī" - ], - [ - "ron", - "ic" - ], - [ - "pl", - "ay" - ], - [ - "Ġco", - "ok" - ], - [ - "è¿Ľ", - "åħ¥" - ], - [ - "è½", - "®" - ], - [ - "Ġvol", - "unte" - ], - [ - "Ġra", - "in" - ], - [ - "ĠM", - "on" - ], - [ - "Ġconsum", - "ption" - ], - [ - "èĽĭ", - "çĻ½" - ], - [ - "ĠS", - "oc" - ], - [ - "å£", - "¤" - ], - [ - "Ġrout", - "ine" - ], - [ - "Ġimpro", - "ved" - ], - [ - "T", - "o" - ], - [ - "人", - "çī©" - ], - [ - "读", - "èĢħ" - ], - [ - "Ġgo", - "al" - ], - [ - "广", - "åijĬ" - ], - [ - "éķ¿", - "æľŁ" - ], - [ - "Ġe", - "y" - ], - [ - "H", - "e" - ], - [ - "Ġout", - "do" - ], - [ - "Ġcu", - "is" - ], - [ - "Ġa", - "way" - ], - [ - "Ġbo", - "oks" - ], - [ - "Ġtop", - "ic" - ], - [ - "大", - "åĪ©" - ], - [ - "h", - "ouse" - ], - [ - "Ġon", - "es" - ], - [ - "ç§", - "Ł" - ], - [ - "'", - ":" - ], - [ - "æĪ¿", - "å±ĭ" - ], - [ - "移", - "åĬ¨" - ], - [ - "Ġdis", - "asters" - ], - [ - "est", - "s" - ], - [ - "ill", - "ing" - ], - [ - "绿", - "èī²" - ], - [ - "åĵ²", - "åѦ" - ], - [ - "æĪIJ", - "åĪĨ" - ], - [ - "Ġocc", - "ur" - ], - [ - "ľ", - "ä¼½" - ], - [ - "åľŁ", - "壤" - ], - [ - "çļĦ", - "主è¦ģ" - ], - [ - "çİ°", - "å®ŀ" - ], - [ - "Ġanim", - "al" - ], - [ - "é¢Ĩ", - "导" - ], - [ - "Ġview", - "s" - ], - [ - "éĤ", - "®" - ], - [ - "æ°§", - "åĮĸ" - ], - [ - "ath", - "y" - ], - [ - "éģĵ", - "å¾·" - ], - [ - "社交", - "åªĴä½ĵ" - ], - [ - "ĠP", - "ersonal" - ], - [ - "Ľ", - "åĽ´" - ], - [ - "Ġpur", - "ch" - ], - [ - "Ġcount", - "ry" - ], - [ - "Ġrem", - "ind" - ], - [ - "å¯", - "¸" - ], - [ - "Ġr", - "ights" - ], - [ - "çļĦ", - "çݯå¢ĥ" - ], - [ - "ĠP", - "r" - ], - [ - "Ġl", - "ine" - ], - [ - "ib", - "r" - ], - [ - "é©", - "¾" - ], - [ - "Ġm", - "aj" - ], - [ - "Ġover", - "come" - ], - [ - "Ġne", - "xt" - ], - [ - "æīĢ", - "è¿°" - ], - [ - "è§Ħ", - "å®ļ" - ], - [ - "Ġinteract", - "ions" - ], - [ - "Ġconf", - "lic" - ], - [ - "Ġwh", - "y" - ], - [ - "ç³»", - "åĪĹ" - ], - [ - "å°", - "¼" - ], - [ - "ib", - "ly" - ], - [ - "çīĽ", - "奶" - ], - [ - "Ġrespons", - "es" - ], - [ - "s", - "es" - ], - [ - "åѦ", - "ä¼ļ" - ], - [ - "b", - "ol" - ], - [ - "Ġstand", - "ards" - ], - [ - "ul", - "ner" - ], - [ - "对è¯Ŀ", - "åĨħ容" - ], - [ - "l", - "ished" - ], - [ - "çļĦæĢ", - "§" - ], - [ - "çĶŁæĢģ", - "ç³»ç»Ł" - ], - [ - "an", - "n" - ], - [ - "æĥħåĨµ", - "ä¸ĭ" - ], - [ - "寻", - "æ±Ĥ" - ], - [ - "Ġh", - "old" - ], - [ - "d", - "en" - ], - [ - "åį", - "ĥ" - ], - [ - "Ġment", - "ion" - ], - [ - "ĠMan", - "y" - ], - [ - "缴", - "åĪ°" - ], - [ - "éģ", - "Ĺ" - ], - [ - "he", - "l" - ], - [ - "Ġbelie", - "ve" - ], - [ - "ar", - "ies" - ], - [ - "æľī", - "ä¸Ģ个" - ], - [ - "1", - "3" - ], - [ - "Ġatmosp", - "here" - ], - [ - "Ġm", - "or" - ], - [ - "æĹ¥", - "æľŁ" - ], - [ - "ä¹", - "ħ" - ], - [ - "ä½ł", - "好" - ], - [ - "Ġaddress", - "ing" - ], - [ - "ĠâĢ", - "ĵ" - ], - [ - "çļĦåľ°", - "æĸ¹" - ], - [ - "m", - "ing" - ], - [ - "Ġcan", - "not" - ], - [ - "Ġman", - "ufact" - ], - [ - "Ġp", - "ie" - ], - [ - "ic", - "ing" - ], - [ - "Ġstud", - "ies" - ], - [ - "ç¾İ", - "åij³" - ], - [ - "ĠAmeric", - "an" - ], - [ - "ĠN", - "LP" - ], - [ - "Ġacc", - "ording" - ], - [ - "ms", - "elves" - ], - [ - "èĦ", - "Ĥ" - ], - [ - "èĩª", - "ä¿¡" - ], - [ - "æīĢ", - "éľĢ" - ], - [ - "Ġthe", - "mselves" - ], - [ - "Ġremot", - "e" - ], - [ - "åŁ¹", - "åħ»" - ], - [ - "å®ī", - "æİĴ" - ], - [ - "ä½ł", - "éľĢè¦ģ" - ], - [ - "Ġreg", - "ard" - ], - [ - "ir", - "ing" - ], - [ - "è¯Ĩ", - "åĪ«" - ], - [ - "Ġart", - "icle" - ], - [ - "æģ", - "Ĵ" - ], - [ - "æĢ»", - "çļĦæĿ¥" - ], - [ - "Ġal", - "ign" - ], - [ - "æ±", - "ł" - ], - [ - "ten", - "ance" - ], - [ - "fact", - "ion" - ], - [ - "åĬ¨", - "ä½ľ" - ], - [ - "çļĦç", - "©" - ], - [ - "ç¼", - "©" - ], - [ - "æĢ", - "¥" - ], - [ - "Ġ1", - "00" - ], - [ - "Ġtest", - "ing" - ], - [ - "åŃĹ", - "æ¯į" - ], - [ - "å¹´", - "è½»" - ], - [ - "åĪ¶", - "éĢł" - ], - [ - "Ġs", - "we" - ], - [ - "å°", - "º" - ], - [ - "he", - "ns" - ], - [ - "æ°´", - "æŀľ" - ], - [ - "Ġinf", - "rastructure" - ], - [ - "èī²", - "彩" - ], - [ - "æĢ»çļĦæĿ¥", - "说" - ], - [ - "æľī", - "ä»Ģä¹Ī" - ], - [ - "te", - "xt" - ], - [ - "车", - "è¾Ĩ" - ], - [ - "Ġp", - "ay" - ], - [ - "ro", - "p" - ], - [ - "Ċ", - "ĠĠ" - ], - [ - "Ġcaus", - "ed" - ], - [ - "Ġcor", - "rect" - ], - [ - "Ġ", - "ì" - ], - [ - "èĥ", - "ŀ" - ], - [ - "ĠM", - "ed" - ], - [ - "ç²¾", - "ç¥ŀ" - ], - [ - "æ°ĶåĢĻ", - "åıĺåĮĸ" - ], - [ - "ĠR", - "ed" - ], - [ - "äºĴ", - "èģĶç½ij" - ], - [ - "Ġeng", - "age" - ], - [ - "åĪĨ", - "为" - ], - [ - "ĠD", - "ata" - ], - [ - "Ġful", - "l" - ], - [ - "en", - "c" - ], - [ - "éĩį", - "æĸ°" - ], - [ - "æŃ£ç¡®", - "çļĦ" - ], - [ - "çļĦæ°", - "Ķ" - ], - [ - "åıĮ", - "æĸ¹" - ], - [ - "Ġcom", - "es" - ], - [ - "åı¤", - "代" - ], - [ - "æŁIJ", - "äºĽ" - ], - [ - "åijĪ", - "çİ°" - ], - [ - "Ġto", - "day" - ], - [ - "ag", - "ed" - ], - [ - "æĪij", - "åı¯ä»¥" - ], - [ - "æĹ¥", - "常" - ], - [ - "æ»", - "ij" - ], - [ - "Ġcl", - "in" - ], - [ - "Ġ", - "\\" - ], - [ - "Ġo", - "bs" - ], - [ - "Ġart", - "ificial" - ], - [ - "Ġexce", - "ll" - ], - [ - "çļĦç", - "¬" - ], - [ - "all", - "s" - ], - [ - "Ġprodu", - "ce" - ], - [ - "ĠD", - "es" - ], - [ - "os", - "s" - ], - [ - "è¹", - "Ī" - ], - [ - "Ġdra", - "w" - ], - [ - "Ġlet", - "ter" - ], - [ - "Ġadv", - "ice" - ], - [ - "Ġhigh", - "ly" - ], - [ - "çĬ", - "¯" - ], - [ - "综ä¸Ĭ", - "æīĢè¿°" + "d" ], [ "满", "æĦı" ], [ - "Ġprinci", - "ples" - ], - [ - "èĮ", - "Ħ" - ], - [ - "Ġfeel", - "ings" - ], - [ - "çļĦæ", - "´" - ], - [ - "Ġh", - "om" - ], - [ - "Ġf", + "Ġav", "ail" ], [ - "Ġcro", - "p" - ], - [ - "å§", - "ľ" - ], - [ - "Ġquest", - "ion" - ], - [ - "Ġdis", - "abilities" - ], - [ - "èĪŀ", - "è¹Ī" - ], - [ - "Ġimp", - "lications" - ], - [ - "r", - "al" - ], - [ - "Ġs", - "ing" - ], - [ - "4", - "0" - ], - [ - "Ġfam", - "il" - ], - [ - "Ġgovern", - "ments" - ], - [ - "Ġrec", - "ord" - ], - [ - "å½¢", - "çĬ¶" - ], - [ - "Ġbe", - "gin" - ], - [ - "is", - "es" - ], - [ - "çļĦæĥ", - "³" - ], - [ - "ach", - "ine" - ], - [ - "è°", - "±" - ], - [ - "Ġv", - "ulner" + "A", + "B" ], [ "Ġpro", "per" ], [ - "Ġovers", - "ight" - ], - [ - "è´Ł", - "éĿ¢" - ], - [ - "Ġem", - "ail" - ], - [ - "Ġnew", - "s" - ], - [ - "Ġexpl", - "oring" - ], - [ - "Ġf", - "avor" - ], - [ - "æ¥", - "¼" - ], - [ - "å®", - "ľ" - ], - [ - "Ġun", - "ivers" - ], - [ - "å·®", - "å¼Ĥ" - ], - [ - "ï¼ī", - "ãĢĤ" - ], - [ - "解åĨ³", - "éĹ®é¢ĺ" - ], - [ - "Ġfam", - "ous" - ], - [ - "g", - "n" - ], - [ - "Ġmess", - "age" - ], - [ - "at", - "itude" - ], - [ - "Ġc", - "ra" - ], - [ - "Ġco", - "ver" - ], - [ - "æ·±", - "åĪ»" - ], - [ - "åı¯ä»¥", - "éĢīæĭ©" - ], - [ - "çĶŁæ´»", - "ä¸Ń" - ], - [ - "ç§į", - "ç±»" - ], - [ - "Ġsm", - "art" - ], - [ - "on", - "str" - ], - [ - "ve", - "y" - ], - [ - "çĶ", - "²" - ], - [ - "Ġreg", - "ularly" - ], - [ - "ĠS", - "m" - ], - [ - "æĦŁ", - "è§ī" - ], - [ - "Ġthough", - "t" - ], - [ - "Ġex", - "h" - ], - [ - "c", - "ure" - ], - [ - "ç»", - "ĺ" - ], - [ - "认", - "è¯Ĩ" - ], - [ - "Ġo", - "ld" - ], - [ - "æĦ", - "ī" - ], - [ - "称", - "为" - ], - [ - "Ġfiel", - "ds" - ], - [ - "Ġcons", - "ist" - ], - [ - "ã", - "ģ" - ], - [ - "ç»Ĩ", - "èĥŀ" - ], - [ - "Ġh", - "ours" - ], - [ - "8", - "0" - ], - [ - "al", - "king" - ], - [ - "è§ī", - "å¾Ĺ" - ], - [ - "ç»", - "Ŀ" - ], - [ - "ä½ł", - "们" - ], - [ - "ĠEng", - "lish" - ], - [ - "Ġsignificant", - "ly" - ], - [ - "Ġs", - "ource" - ], - [ - "Ġan", - "t" - ], - [ - "Ġeducation", - "al" - ], - [ - "Ġtas", - "k" - ], - [ - "Ġhand", - "le" - ], - [ - "æIJ", - "ľ" - ], - [ - "ĠS", - "p" - ], - [ - "Ġcall", - "ed" - ], - [ - "Ġter", - "ms" - ], - [ - "æ²", - "ī" - ], - [ - "Ġw", - "in" - ], - [ - "duct", - "ion" - ], - [ - "Ġmod", - "ern" - ], - [ - "Ġcuis", - "ine" - ], - [ - "å¥", - "Ĺ" - ], - [ - "è§", - "¦" - ], - [ - "olut", - "ely" - ], - [ - "ç«", - "¥" - ], - [ - "p", - "ite" - ], - [ - "Ġf", - "elt" - ], - [ - "Ġcomp", - "re" - ], - [ - "Ġw", - "ond" - ], - [ - "è¿IJ", - "è¡Į" - ], - [ - "Ġres", - "il" - ], - [ - "缸", - "ä¼¼" - ], - [ - "éĩij", - "èŀį" - ], - [ - "çĪ±", - "æĥħ" - ], - [ - "ç¬", - "Ķ" - ], - [ - "èĪ", - "ª" - ], - [ - "è°", - "Ī" - ], - [ - "åĬĽ", - "çļĦ" + "I", + "P" ], [ "æľī", - "æīĢ" - ], - [ - "æ½", - "ľ" - ], - [ - "ul", - "ate" - ], - [ - "Ġdetect", - "ion" - ], - [ - "宣", - "ä¼ł" - ], - [ - "Ġmat", - "ter" - ], - [ - "éĩı", - "åŃIJ" - ], - [ - "W", - "rite" - ], - [ - "ç»ĵ", - "åIJĪ" - ], - [ - "ç»ı", - "è¿ĩ" - ], - [ - "Ġdevelop", - "ers" - ], - [ - "è", - "ª" - ], - [ - "Ġ", - "---" - ], - [ - "人", - "éĻħ" - ], - [ - "çŃ", - "¾" - ], - [ - "ï¼ļ", - "âĢľ" - ], - [ - "Ġinnov", - "ative" - ], - [ - "ãĢĤ", - "âĢĿ" - ], - [ - "å½", - "¼" - ], - [ - "é¥", - "¼" - ], - [ - "è¿ĩ", - "度" - ], - [ - "Ġplan", - "et" - ], - [ - "åħ", - "°" - ], - [ - "å¸", - "ģ" - ], - [ - "æķ", - "¬" - ], - [ - "Ġleg", - "al" - ], - [ - "Ġlo", - "t" - ], - [ - "æĪIJ为", - "äºĨ" - ], - [ - "i", - "ate" - ], - [ - "Ġm", - "is" - ], - [ - "åģĩ", - "设" - ], - [ - "çļĦ", - "æĸĩç«ł" - ], - [ - "ĠCom", - "pan" - ], - [ - "Ġd", - "oc" - ], - [ - "Ġcare", - "ful" - ], - [ - "Ġe", - "ver" - ], - [ - "æĪij们", - "å°Ĩ" - ], - [ - "ä¾ĭ", - "åŃIJ" - ], - [ - "ä¹", - "³" - ], - [ - "ä½ľ", - "èĢħ" - ], - [ - "åIJ", - "§" - ], - [ - "æļ", - "´" - ], - [ - "Ġrem", - "ember" - ], - [ - "缮", "çļĦ" ], - [ - "Ġp", - "ut" - ], - [ - "常è§ģ", - "çļĦ" - ], - [ - "Ġf", - "est" - ], - [ - "建", - "设" - ], - [ - "å®ŀ", - "çĶ¨" - ], - [ - "Ġact", - "ive" - ], - [ - "çª", - "Ĺ" - ], - [ - "ou", - "th" - ], - [ - "åİŁ", - "çIJĨ" - ], - [ - "Ġtry", - "ing" - ], - [ - "è¿", - "·" - ], - [ - "缸", - "åIJĮ" - ], - [ - "éħĴ", - "åºĹ" - ], - [ - "An", - "other" - ], - [ - "æľĢ", - "ä½³" - ], - [ - "Ġanaly", - "tics" - ], - [ - "Ġper", - "pet" - ], - [ - "ip", - "ment" - ], - [ - "Ġ", - "å¦Ĥæŀľ" - ], - [ - "è§Ĥ", - "ä¼Ĺ" - ], - [ - "Ġc", - "elebr" - ], - [ - "Ġhe", - "av" - ], - [ - "Ġmed", - "itation" - ], - [ - "大", - "æ°Ķ" - ], - [ - "A", - "nd" - ], - [ - "ä¸į", - "éĶĻ" - ], - [ - "Ġwhe", - "ther" - ], - [ - "s", - "et" - ], - [ - "Ġdem", - "onstr" - ], - [ - "ä¸Ģ", - "款" - ], - [ - "æĶ¶", - "éĽĨ" - ], - [ - "éĻIJ", - "åĪ¶" - ], - [ - "Ġ", - "ing" - ], - [ - "Ġrev", - "olution" - ], - [ - "çľ", - "ģ" - ], - [ - "Ġsc", - "ience" - ], - [ - "缮", - "åīį" - ], - [ - "Ġthink", - "ing" - ], - [ - "±", - "ä¹IJ" - ], - [ - "课", - "ç¨ĭ" - ], - [ - "Ġp", - "ack" - ], - [ - "Ġim", - "age" - ], - [ - "lo", - "c" - ], - [ - "Ġst", - "ories" - ], - [ - "uc", - "k" - ], - [ - "Ġsatis", - "faction" - ], - [ - "Ġcollect", - "ion" - ], - [ - "h", - "o" - ], - [ - "èµ", - "ŀ" - ], - [ - "éĿ¢", - "临" - ], - [ - "Ġl", - "a" - ], - [ - "Ġsym", - "bol" - ], - [ - "Ġem", - "b" - ], - [ - "Ġhabit", - "ats" - ], - [ - "Ġlow", - "er" - ], - [ - "Ġcontin", - "ues" - ], - [ - "éľ", - "ĩ" - ], - [ - "åĵ", - "Ī" - ], - [ - "ĠT", - "ake" - ], - [ - "Ġenviron", - "ments" - ], - [ - "Ġth", - "ree" - ], - [ - "Ġen", - "c" - ], - [ - "ĠA", - "cc" - ], - [ - "æĦı", - "åij³" - ], - [ - "åİ", - "¨" - ], - [ - "ch", - "an" - ], - [ - "ĠH", - "um" - ], - [ - "Ġtr", - "ue" - ], - [ - "åĪĩ", - "æĪIJ" - ], - [ - "s", - "ing" - ], - [ - "âĢĶ", - "âĢĶ" - ], - [ - "åĩº", - "æĿ¥" - ], - [ - "Ġreg", - "ion" - ], - [ - "Ġinter", - "pre" - ], - [ - "Ġdiagnos", - "is" - ], - [ - "é", - "ŀ" - ], - [ - "Ġdo", - "ing" - ], - [ - "Ġr", - "un" - ], - [ - "Ġco", - "ffee" - ], - [ - "Ġmaj", - "or" - ], - [ - "Ġmindful", - "ness" - ], - [ - "Ġafford", - "able" - ], - [ - "çĻ", - "¾" - ], - [ - "Ġdetail", - "ed" - ], - [ - "éĿŀ常", - "éĩįè¦ģçļĦ" - ], - [ - "çļĦæ²", - "ŁéĢļ" - ], - [ - "çļĦæķ", - "ħ" - ], - [ - "åĢĴ", - "åħ¥" - ], - [ - "Ġthem", - "es" - ], - [ - "Ġnet", - "work" - ], - [ - "ï¼ī", - "ï¼ļ" - ], - [ - "ĠUn", - "ited" - ], - [ - "çļĦæĮ", - "ĩ" - ], - [ - "ort", - "s" - ], - [ - "åį«", - "çĶŁ" - ], - [ - "Ġplan", - "ning" - ], - [ - "æĥ", - "ł" - ], - [ - "åī", - "ª" - ], - [ - "ĠPro", - "v" - ], - [ - "çļĦ", - "åºĶçĶ¨" - ], - [ - "Ġp", - "eri" - ], - [ - "Ġaccount", - "able" - ], - [ - "çī", - "Ļ" - ], - [ - "çļĦç", - "ģ" - ], - [ - "Ġcho", - "ice" - ], - [ - "ĠC", - "omm" - ], - [ - "id", - "ents" - ], - [ - "çļĦ", - "å®īåħ¨" - ], - [ - "并", - "ä¸į" - ], - [ - "太éĺ³", - "ç³»" - ], - [ - "Ġrece", - "ive" - ], - [ - "Ġclo", - "se" - ], - [ - "çļĦæŶ", - "åĢĻ" - ], - [ - "Ġchang", - "ing" - ], - [ - "ä»·åĢ¼", - "è§Ĥ" - ], - [ - "Ġperpet", - "u" - ], - [ - "Ġse", - "ason" - ], - [ - "Ġm", - "en" - ], - [ - "Ġlearn", - "ed" - ], - [ - "Ġsitu", - "ation" - ], - [ - "Ġre", - "place" - ], - [ - "he", - "ad" - ], - [ - "让", - "æĪij" - ], - [ - "åľ¨", - "ä¸Ģèµ·" - ], - [ - "çļĦç©", - "º" - ], - [ - "éľ", - "²" - ], - [ - "Ġen", - "ough" - ], - [ - "å±ķ", - "çİ°" - ], - [ - "Ġlead", - "ers" - ], - [ - "an", - "cing" - ], - [ - "Ġtemper", - "ature" - ], - [ - "åı", - "«" - ], - [ - "Ġ3", - "0" - ], - [ - "æĦıåij³", - "çĿĢ" - ], - [ - "æ±", - "ĩ" - ], - [ - "ĠGo", - "vern" - ], - [ - "Ġfocus", - "ed" - ], [ "u", - "ro" - ], - [ - "Ġsim", - "ple" - ], - [ - "Ġh", - "iking" - ], - [ - "æ¯", - "Ĵ" - ], - [ - "Ġcompre", - "hens" - ], - [ - "äº", - "Ī" - ], - [ - "Ġcreat", - "ed" - ], - [ - "con", - "d" - ], - [ - "é¡", - "µ" - ], - [ - "ĠW", - "or" - ], - [ - "è¯ģ", - "æį®" - ], - [ - "Ġwork", - "place" - ], - [ - "Ġcharact", - "ers" - ], - [ - "çļĦ", - "设计" - ], - [ - "Ġme", - "chan" - ], - [ - "ĠD", - "is" - ], - [ - "ç¥ŀ", - "ç§ĺ" - ], - [ - "å·", - "ŀ" - ], - [ - "ĠO", - "n" - ], - [ - "<", - "/" - ], - [ - "ç§į", - "æ¤į" - ], - [ - "Ġpat", - "h" - ], - [ - "Ġlim", - "ited" - ], - [ - "Ġsol", - "ar" - ], - [ - "çļĦæ", - "ı" - ], - [ - "2", - "2" - ], - [ - "Ġappreci", - "ate" - ], - [ - "å¿«", - "ä¹IJ" - ], - [ - "æĦŁ", - "åıĹåĪ°" - ], - [ - "èĢ", - "Ĺ" - ], - [ - "m", - "ed" - ], - [ - "ic", - "ine" - ], - [ - "Ġnot", - "e" - ], - [ - "å½ĵ", - "åīį" - ], - [ - "æĪij们", - "åºĶ该" - ], - [ - "Ġse", - "en" - ], - [ - "ä¸Ģ", - "åIJį" - ], - [ - "å°½", - "åı¯èĥ½" - ], - [ - "è¿IJ", - "ç®Ĺ" - ], - [ - "è§Ĵ", - "度" - ], - [ - "Ġequ", - "ipment" - ], - [ - "Ġsp", - "read" - ], - [ - "è", - "¸" - ], - [ - "è®", - "¿" - ], - [ - "åı¥", - "è¯Ŀ" - ], - [ - "æĮ", - "¥" - ], - [ - "Ġpur", - "pose" - ], - [ - "请", - "ä½ł" - ], - [ - "Y", - "our" - ], - [ - "ari", - "an" - ], - [ - "ä»", - "ª" - ], - [ - "Ġperspect", - "ives" - ], - [ - "åĩº", - "äºĨ" - ], - [ - "å©ļ", - "礼" - ], - [ - "Ġexcell", - "ent" - ], - [ - "ĠEns", - "uring" - ], - [ - "Ġre", - "ach" - ], - [ - "éĺ¶", - "段" - ], - [ - "ä¿Ŀ", - "éļľ" - ], - [ - "Ġemp", - "athy" - ], - [ - "ĠM", - "y" - ], - [ - "çij", - "ľä¼½" - ], - [ - "Ġ", - "ver" - ], - [ - "ab", - "el" - ], - [ - "ĠPre", - "dict" - ], - [ - "Ġmain", - "tenance" - ], - [ - "è¯Ħ", - "ä»·" - ], - [ - "Ġ", - "ult" - ], - [ - "åĴ", - "¨" - ], - [ - "o", - "x" - ], - [ - "åĴ¨", - "询" - ], - [ - "Ġshare", - "d" - ], - [ - "in", - "a" - ], - [ - "l", - "ist" - ], - [ - "Ġoutdo", - "or" - ], - [ - "Ġthough", - "ts" - ], - [ - "in", - "ating" - ], - [ - "éĴ", - "±" - ], - [ - "Ġfra", - "me" - ], - [ - "éĺ", - "¿" - ], - [ - "åĪ©", - "润" - ], - [ - "çļĦæİ", - "¨" - ], - [ - "åį", - "ļ" - ], - [ - "Ġrec", - "ent" - ], - [ - "Ġal", - "tern" - ], - [ - "are", - "d" - ], - [ - "=", - "=" - ], - [ - "Ġro", - "ad" - ], - [ - "äºĭ", - "项" - ], - [ - "g", - "ed" - ], - [ - "y", - "nt" - ], - [ - "Ġspe", - "nd" - ], - [ - "ç½", - "ª" - ], - [ - "åıĸ", - "å¾Ĺ" - ], - [ - "é", - "¹" - ], - [ - "l", - "i" - ], - [ - "æŶ", - "æľŁ" - ], - [ - "严", - "éĩį" - ], - [ - "å¿", - "Ĩ" - ], - [ - "å©", - "´" - ], - [ - "æİ¥", - "ä¸ĭæĿ¥" - ], - [ - "ĠEar", "th" ], [ - "ĠChat", - "bots" + "æĶ¶", + "åħ¥" ], [ - "Ġset", - "ting" + "Ġt", + "imes" ], [ - "ç¥", - "Ŀ" + "è§Ħ", + "å®ļ" ], [ - "éĶĢåĶ®", - "é¢Ŀ" + "ç´§", + "æĢ¥" ], [ - "ä¼", + "Ġloo", + "k" + ], + [ + "绣", + "计" + ], + [ + "Ġm", + "et" + ], + [ + "å²", + "ģ" + ], + [ + "Ġcon", + "d" + ], + [ + "ãĥ", + "¼" + ], + [ + "Ġassist", + "ant" + ], + [ + "dition", + "ally" + ], + [ + "Ġfiel", + "d" + ], + [ + "交", + "äºĴ" + ], + [ + "åĪĿ", + "å§ĭ" + ], + [ + "èĽĭ", + "çĻ½" + ], + [ + "å", + "³" + ], + [ + "Ġs", + "ay" + ], + [ + "çĸ", + "ij" + ], + [ + "æĶ¾", + "æĿ¾" + ], + [ + "æ¶", "¦" ], [ - "Ġread", - "ing" + "Ġsoft", + "ware" + ], + [ + "æľīæķĪ", + "çļĦ" + ], + [ + "æ", + "©" + ], + [ + "çĶŁ", + "éķ¿" + ], + [ + "é¢ij", + "çİĩ" + ], + [ + "T", + "rans" + ], + [ + "注", + "åĨĮ" + ], + [ + "èĭ±", + "æĸĩ" ], [ "æİ¢", "讨" ], [ - "a", - "ign" + "顺", + "åºı" ], [ - "éŀ", - "ĭ" + "Ġl", + "and" ], [ - "Ġyou", - "ng" + "Ġnot", + "e" ], [ - "Ġcare", - "er" + "p", + "os" ], [ - "Ġteac", - "hers" + "\"", + "?" ], [ - "çļĦ", - "è´¨éĩı" + "æŁ¥", + "æī¾" ], [ - "å±ŀ", - "äºİ" + "人", + "çī©" ], [ - "Ġeas", - "ier" + "æŶ", + "æľŁ" ], [ - "Ġscient", - "ific" + "in", + "ing" ], [ - "ç¾İ", - "åħĥ" - ], - [ - "Ġsp", - "ir" - ], - [ - "åĬ", - "³" - ], - [ - "çļĦæĶ", - "¯" - ], - [ - "r", - "ist" - ], - [ - "èµĦ", - "产" - ], - [ - "çĶŁ", - "åŃĺ" - ], - [ - "èĩ³", - "å°ij" - ], - [ - "å§", - "¿" - ], - [ - "Ġvide", - "o" - ], - [ - "Ġa", - "im" - ], - [ - "å®Ŀ", - "å®Ŀ" - ], - [ - "çĪ¶", - "æ¯į" - ], - [ - "________", - "________" - ], - [ - "al", - "ities" - ], - [ - "Ġb", - "ud" - ], - [ - "Ġstre", - "et" - ], - [ - "Ġ", - "æĺ¯" - ], - [ - "æĸ¹", - "ç¨ĭ" - ], - [ - "ä¸ĸ", - "纪" - ], - [ - "c", - "hes" - ], - [ - "ear", - "ch" - ], - [ - "æĴ", - "°" - ], - [ - "Ġeng", - "ine" - ], - [ - "Ġdis", - "placement" - ], - [ - "ĠRo", - "bots" - ], - [ - "erv", - "ised" - ], - [ - "é¡", - "¶" - ], - [ - "ou", - "d" - ], - [ - "Ġw", - "alk" - ], - [ - "Ġemerg", - "ency" - ], - [ - "èģ", - "ĺ" - ], - [ - "n", - "al" - ], - [ - "Ġdat", - "as" + "iz", + "es" ], [ "åĢ", - "º" - ], - [ - "åIJİ", - "çļĦ" - ], - [ - "å¾Ī", - "好" - ], - [ - "Ġmy", - "self" - ], - [ - "çļĦæī", - "ĭ" - ], - [ - "Ġus", - "age" - ], - [ - "Ġsh", - "own" - ], - [ - "æ®", - "Ĭ" - ], - [ - "Ġtyp", - "ically" - ], - [ - "u", - "ly" - ], - [ - "æĸ°", - "éĹ»" - ], - [ - "æĽ", - "¿" - ], - [ - "Ġor", - "ig" - ], - [ - "è½»", - "æĿ¾" - ], - [ - "æĺ¾", - "示" - ], - [ - "Ġado", - "pt" - ], - [ - "èĤ¡", - "票" - ], - [ - "Ġp", - "arent" - ], - [ - "a", - "ps" - ], - [ - "æĢĿ", - "æĥ³" - ], - [ - "Ġmarket", - "ing" - ], - [ - "èĻ", - "«" - ], - [ - "éĥ¨", - "éŨ" - ], - [ - "çļĦæķ", - "Ī" - ], - [ - "Ġcomfort", - "able" - ], - [ - "åŃ¦ä¹ł", - "åĴĮ" - ], - [ - "Ġfore", - "cast" - ], - [ - "ict", - "ion" - ], - [ - "Ġget", - "ting" - ], - [ - "Ġtre", - "es" - ], - [ - "av", - "ing" - ], - [ - "çļĦ", - "åŁºç¡Ģ" - ], - [ - "read", - "y" - ], - [ - "æĸ°", - "é²ľ" - ], - [ - "go", - "ing" - ], - [ - "¹", - "é¥" - ], - [ - "Ġev", - "idence" - ], - [ - "¹é¥", - "ª" - ], - [ - "ç§", - "ĭ" - ], - [ - "æľī", - "å¾Īå¤ļ" - ], - [ - "éĿ¢", - "è¯ķ" - ], - [ - "éģĩ", - "åĪ°" - ], - [ - "ç»Ļ", - "å®ļ" - ], - [ - "ir", - "c" - ], - [ - "åı¯ä»¥", - "æł¹æį®" - ], - [ - "驾", - "驶" - ], - [ - "å·§", - "åħĭ" - ], - [ - "Ġst", - "unning" - ], - [ - "çļĦæ", - "¦Ĥ" - ], - [ - "æ¡", - "Į" - ], - [ - "ĠJ", - "ohn" - ], - [ - "ul", - "ation" - ], - [ - "åıĤ", - "èĢĥ" - ], - [ - "Ġf", - "lex" - ], - [ - "çĦ¦", - "èĻij" - ], - [ - "ym", - "akers" - ], - [ - "Ġfor", - "ms" - ], - [ - "s", - "h" - ], - [ - "v", - "al" - ], - [ - "ĠS", - "o" - ], - [ - "c", - "o" - ], - [ - "æݨ", - "åĬ¨" - ], - [ - "èħ", - "¿" - ], - [ - "çī¹", - "æ®Ĭ" - ], - [ - "Ġen", - "ab" - ], - [ - "å°Ĩ", - "ä¼ļ" - ], - [ - "æĶ¯", - "åĩº" - ], - [ - "åĿļ", - "æĮģ" - ], - [ - "红", - "èī²" - ], - [ - "Ġopt", - "ion" - ], - [ - "Ġstart", - "ed" - ], - [ - "r", - "ation" - ], - [ - "Ġpo", - "etry" - ], - [ - "Ġp", - "ort" - ], - [ - "g", - "en" - ], - [ - "èª", - "ī" - ], - [ - "Ġdel", - "iv" - ], - [ - "çĶ", - "ļ" - ], - [ - "éĢ", - "»" - ], - [ - "éĢī", - "项" - ], - [ - "Ġg", - "round" - ], - [ - "å½¼", - "æѤ" - ], - [ - "an", - "a" - ], - [ - "çļĦæĹ", - "¥" - ], - [ - "åľ¨", - "线" - ], - [ - "Ġse", - "cure" - ], - [ - "Ġ", - "æł¹æį®" - ], - [ - "饮", - "æĸĻ" - ], - [ - "Ġgr", - "atitude" - ], - [ - "第", - "ä¸ī" - ], - [ - "Ġs", - "ong" - ], - [ - "Ġpoint", - "s" - ], - [ - "Ġal", - "ready" - ], - [ - "çļĦçĪ", - "±" - ], - [ - "ĠTe", - "chn" - ], - [ - "Ġreal", - "ity" - ], - [ - "çı", - "Ń" - ], - [ - "Ġs", - "ince" - ], - [ - "Ġpopul", - "ation" - ], - [ - "y", - "ond" - ], - [ - "b", - "or" - ], - [ - "ĠSoc", - "ial" - ], - [ - "æıIJ", - "åıĸ" - ], - [ - "å·¥", - "ç¨ĭ" - ], - [ - "a", - "ff" - ], - [ - "交", - "æĺĵ" - ], - [ - "Ġwor", - "th" - ], - [ - "å¡", - "«" - ], - [ - "å¨", - "±ä¹IJ" - ], - [ - "Ġdo", - "g" - ], - [ - "ĠAr", - "t" - ], - [ - "ç¡", - "¬" - ], - [ - "æµ·", - "æ´ĭ" - ], - [ - "åĨ", "Ĵ" ], [ - "çī", - "Ī" + "Ġgener", + "ating" ], [ - "Ġprogramm", - "ing" + "Ġsome", + "one" ], [ - "ĠAs", - "s" + "Ġfur", + "ther" ], [ - "ĠM", - "achine" - ], - [ - "åĢ¼", - "å¾Ĺ" - ], - [ - "请", - "è¾ĵåħ¥" - ], - [ - "声", - "éŁ³" - ], - [ - "Ġexercis", - "es" - ], - [ - "åħī", - "线" - ], - [ - "æ³ķ", - "åĴĮ" - ], - [ - "Ġfeat", - "ure" - ], - [ - "e", - "ff" - ], - [ - "è¿Ľ", - "æŃ¥" - ], - [ - "女", - "æĢ§" - ], - [ - "Ġefficient", - "ly" - ], - [ - "çļĦæĬĢ", - "æľ¯" - ], - [ - "Ġgen", - "etic" - ], - [ - "令", - "人" - ], - [ - "è´", - "¦" - ], - [ - "çļĦ", - "产åĵģ" - ], - [ - "åİ", - "ļ" - ], - [ - "åĴĮ", - "æĸĩåĮĸ" - ], - [ - "éĻ", - "Ħ" - ], - [ - "Ġmo", - "b" - ], - [ - "综", - "åIJĪ" - ], - [ - "t", - "ers" - ], - [ - "æľī", - "ä¸Ģ" - ], - [ - "å¦", - "Ĩ" - ], - [ - "åį", - "Ī" - ], - [ - "Ġout", - "side" - ], - [ - "Ġprop", - "ert" - ], - [ - "éĤ®", - "件" + "ãģ", + "Į" ], [ "主", "ä¹ī" ], [ - "Ġpolic", - "y" + "åIJ«", + "ä¹ī" ], [ - "èĩª", - "身" + "个人", + "çļĦ" ], [ - "Ġnav", - "igate" + "åľ¨", + "ä¸Ģ个" ], [ - "Ġst", - "y" - ], - [ - "çĶµ", - "èĦij" - ], - [ - "Ġab", - "ilities" - ], - [ - "Ġfac", - "ed" - ], - [ - "çļĦç", - "¼" - ], - [ - "çļĦ", - "å°ı" - ], - [ - "è", - "ķ" - ], - [ - "Ġt", - "one" - ], - [ - "ig", - "ation" - ], - [ - "åıĤ", - "æķ°" - ], - [ - "èĽĭçĻ½", - "è´¨" - ], - [ - "ä½", - "Ľ" - ], - [ - "çĶļ", - "èĩ³" - ], - [ - "Ġsk", - "in" - ], - [ - "èĴ", - "¸" - ], - [ - "æĭ", - "Ľ" - ], - [ - "éŃ", - "Ķ" - ], - [ - "ash", - "ion" - ], - [ - "Ġing", - "red" - ], - [ - "æĹ", - "ĭ" - ], - [ - "Ġcamp", - "aign" - ], - [ - "Ġm", - "ount" - ], - [ - "Ġcons", + "ro", "id" ], [ - "Ġmus", - "e" - ], - [ - "n", - "ter" - ], - [ - "w", - "ater" - ], - [ - "ä¼ļ", - "è®®" - ], - [ - "Ġprotect", - "ion" - ], - [ - "ä¿Ŀ", - "éĻ©" - ], - [ - "Ġcro", - "ps" - ], - [ - "og", - "le" - ], - [ - "éļı", - "æŶ" - ], - [ - "æļ", - "Ĺ" - ], - [ - "i", - "um" - ], - [ - "ä¹", - "ı" - ], - [ - "Ġdi", - "et" - ], - [ - "l", - "ies" - ], - [ - "çĶ¨", - "æĿ¥" - ], - [ - "ĠEn", - "coura" - ], - [ - "æĬ", - "Ĺ" - ], - [ - "ap", - "an" - ], - [ - "éĺ²", - "æŃ¢" - ], - [ - "W", + "Ġn", "ow" ], - [ - "çļĦ", - "åŁºæľ¬" - ], - [ - "å¹³", - "æĸ¹" - ], - [ - "Ġst", - "ep" - ], - [ - "åı¯", - "éĿł" - ], - [ - "表", - "æĺİ" - ], - [ - "Ġpredict", - "ions" - ], - [ - "Ġsym", - "pt" - ], - [ - "Ġdiagnos", - "es" - ], - [ - "åħ¬", - "åĽŃ" - ], - [ - "Ġsupp", - "ly" - ], - [ - "Ġprev", - "ious" - ], - [ - "ç»Ħ", - "åIJĪ" - ], - [ - ".", - "," - ], - [ - "çļĦ", - "è¿ĩç¨ĭ" - ], - [ - "æķ", - "ı" - ], - [ - "s", - "u" - ], - [ - "ar", - "is" - ], - [ - "çķ", - "ħ" - ], - [ - "oc", - "ol" - ], - [ - "æIJľ", - "ç´¢" - ], - [ - "it", - "le" - ], - [ - "éĨ", - "Ĵ" - ], - [ - "顾", - "客" - ], - [ - "éĢ»", - "è¾ij" - ], - [ - "éĿŀ常", - "éĩįè¦ģ" - ], - [ - "ĠB", - "i" - ], - [ - "å·¦", - "åı³" - ], - [ - "am", - "m" - ], - [ - "Ġevery", - "thing" - ], - [ - "æĺ", - "ł" - ], - [ - "Ġincre", - "d" - ], - [ - "Ġpe", - "ace" - ], - [ - "èľ", - "ľ" - ], - [ - "Ġmuse", - "um" - ], - [ - "çĭ¬", - "ç«ĭ" - ], - [ - "Ġcomprehens", - "ive" - ], - [ - "Ġr", - "ates" - ], - [ - "/", - "/" - ], - [ - "Ġra", - "d" - ], - [ - "åĦ¿", - "ç«¥" - ], - [ - "çī¹", - "èī²" - ], - [ - "ĠPredict", - "ive" - ], - [ - "å¼ķ", - "åĬĽ" - ], - [ - "l", - "er" - ], - [ - "å°", - "¤" - ], - [ - "ic", - "ro" - ], - [ - "è¡", - "¥" - ], - [ - "Ġdeterm", - "ine" - ], - [ - "çļĦ", - "åĨħ容" - ], - [ - "Ġcom", - "pl" - ], - [ - "Ġgreen", - "house" - ], - [ - "èħ", - "IJ" - ], - [ - "Ġhigh", - "light" - ], - [ - "Ġpart", - "ners" - ], - [ - "Ġdo", - "ct" - ], - [ - "çļĦ", - "使çĶ¨" - ], - [ - "æŃĮ", - "æĽ²" - ], - [ - "æĮĩ", - "åįĹ" - ], - [ - "ĠA", - "f" - ], - [ - "æľº", - "æŀĦ" - ], - [ - "éĢ", - "Ģ" - ], - [ - "Ġpoem", - "s" - ], - [ - "å¿ĥ", - "åĴĮ" - ], - [ - "Ġatt", - "end" - ], - [ - "çļĦæ¸", - "¸" - ], - [ - "Ġs", - "ide" - ], - [ - "al", - "es" - ], - [ - "Ġmention", - "ed" - ], - [ - "ĠA", - "bs" - ], - [ - "Ġhistor", - "ical" - ], - [ - "Ġle", - "ft" - ], - [ - "以ä¸ĭ", - "åĩłä¸ª" - ], - [ - "åıĹ", - "欢è¿İ" - ], - [ - "èıľ", - "åĵģ" - ], - [ - "Ġrem", - "ain" - ], - [ - "æ", - "ĩ" - ], - [ - "Ġtour", - "s" - ], - [ - "ł", - "éģĵ" - ], - [ - "Ġerr", - "ors" - ], - [ - "æľº", - "åĪ¶" - ], - [ - "æ", - "¦" - ], - [ - "æĤ£", - "èĢħ" - ], - [ - "m", - "ore" - ], - [ - "Ġexpert", - "s" - ], - [ - "çļĦçł", - "Ķ究" - ], - [ - "ç»ĵ", - "æĿŁ" - ], - [ - "Ġwrit", - "ten" - ], - [ - "çł", - "Ķ" - ], - [ - "Ġe", - "t" - ], - [ - "in", - "put" - ], - [ - "æ°Ķ", - "ä½ĵ" - ], - [ - "è", - "ļ" - ], - [ - "æĥ", - "Ĭ" - ], - [ - "Ġa", - "ge" - ], - [ - "éĩį", - "å¤į" - ], - [ - "å¼", - "¹" - ], - [ - "åŃ", - "¤" - ], - [ - "Ġsympt", - "oms" - ], - [ - "Ġbelie", - "f" - ], - [ - "'", - "d" - ], - [ - "i", - "ol" - ], - [ - "Ġ1", - "8" - ], - [ - "åħħ", - "足" - ], - [ - "çı", - "į" - ], - [ - "force", - "ment" - ], - [ - "æĸ", - "Ĺ" - ], - [ - "ª", - "èĮĦ" - ], - [ - "Ġ1", - "5" - ], - [ - "ä¸Ģ个", - "人" - ], - [ - "Ġapp", - "lic" - ], - [ - "è´", - "¥" - ], - [ - "ä½į", - "äºİ" - ], - [ - "éĻ¤", - "äºĨ" - ], - [ - "=", - "\"" - ], - [ - "ä¸ī", - "è§Ĵ" - ], - [ - "æĢĿ", - "ç»´" - ], [ "åį", "·" ], [ - "Ġf", - "ru" + "车", + "è¾Ĩ" ], [ - "ĠCol", - "labor" + "t", + "a" ], [ - "Ġpr", - "im" + "åĴĮ", + "管çIJĨ" ], [ - "Ġrequire", - "d" - ], - [ - "Ġw", - "atch" - ], - [ - "è°ĥ", - "åij³" - ], - [ - "ç»ĵ", - "论" - ], - [ - "on", - "y" - ], - [ - "Ġgu", - "ide" - ], - [ - "Ġm", - "ax" - ], - [ - "ĠC", - "ould" - ], - [ - "Ġadv", - "ent" - ], - [ - "ĠO", - "verall" - ], - [ - "çļĦæĬ", - "ķ" - ], - [ - "Ġexp", - "er" - ], - [ - "å", - "ĺ" - ], - [ - "ic", - "ial" - ], - [ - "ost", - "er" - ], - [ - "çļĦ", - "é¢ľèī²" - ], - [ - "Ġoper", - "ations" - ], - [ - "éĥ", - "ģ" - ], - [ - "Ġm", - "oney" - ], - [ - "le", - "y" - ], - [ - "c", - "ling" - ], - [ - "Ġo", - "il" - ], - [ - "çļ®", - "èĤ¤" - ], - [ - "Ġg", - "e" - ], - [ - "Ġb", - "at" - ], - [ - "ĠP", - "h" - ], - [ - "Ġsc", - "he" - ], - [ - "Ġelect", - "ric" - ], - [ - "v", - "est" - ], - [ - "Ġch", - "ain" - ], - [ - "Ġcap", - "abilities" - ], - [ - "ir", - "d" - ], - [ - "è¯ģ", - "æĺİ" - ], - [ - "æľĢ", - "好" - ], - [ - "iv", - "il" - ], - [ - "Ġdepend", - "ing" - ], - [ - "Ġs", - "ave" - ], - [ - "Ġpract", - "ical" - ], - [ - "Ġcult", - "ures" - ], - [ - "缸åºĶ", - "çļĦ" - ], - [ - "s", - "y" - ], - [ - "çļĦç", - "²" - ], - [ - "Ġbeh", - "ind" - ], - [ - "æŶéĹ´", - "åĴĮ" - ], - [ - "å¹", - "ħ" - ], - [ - "ĠA", - "g" - ], - [ - "Ġeffect", - "iveness" - ], - [ - "A", - "d" - ], - [ - "ĠO", - "f" - ], - [ - "Ġany", - "thing" - ], - [ - "å·§åħĭ", - "åĬĽ" - ], - [ - "Ġm", - "ist" - ], - [ - "Ġlangu", - "ages" - ], - [ - "ĠM", - "ake" - ], - [ - "å", - "«" - ], - [ - "æ£", - "®" - ], - [ - "ĠCon", - "t" - ], - [ - "ĠAbs", - "olutely" - ], - [ - "Ġinvest", - "ment" - ], - [ - "m", - "at" - ], - [ - "çļĦæķħ", - "äºĭ" - ], - [ - "æ¬", - "§" - ], - [ - "Ġspe", - "ed" - ], - [ - "çļĦæ¸", - "©" - ], - [ - "Ġc", - "ities" - ], - [ - "åĨĻ", - "ä½ľ" - ], - [ - "Th", - "anks" - ], - [ - "Ġd", - "ed" - ], - [ - "åĪĨ", - "éħį" - ], - [ - "Ġd", - "ark" - ], - [ - "Ġsupport", - "ing" - ], - [ - "å¹", - "ķ" - ], - [ - "ĠK", - "e" - ], - [ - "éĽ", - "¶" - ], - [ - "Ġsh", - "aring" - ], - [ - "Ġh", - "ouse" - ], - [ - "认", - "çŁ¥" - ], - [ - "Ġsurround", - "ing" - ], - [ - "Ġredu", - "ced" - ], - [ - "Ġf", - "u" - ], - [ - "Ġst", - "or" - ], - [ - "Ġab", - "s" - ], - [ - "T", - "om" - ], - [ - "c", - "ent" - ], - [ - "ĠEduc", - "ation" - ], - [ - "Ġth", - "r" - ], - [ - "ot", - "t" - ], - [ - "ĠTh", - "at" - ], - [ - "Ġhe", - "ar" - ], - [ - "un", - "g" - ], - [ - "Ġbe", - "yond" - ], - [ - "ĠC", - "o" - ], - [ - "ro", - "om" - ], - [ - "è¯Ĺ", - "æŃĮ" - ], - [ - "re", - "me" - ], - [ - "Ġlit", - "tle" - ], - [ - "Ġg", - "ames" - ], - [ - "ä¹ĭ", - "åIJİ" - ], - [ - "éĥ½", - "ä¼ļ" - ], - [ - "è¯Ń", - "éŁ³" - ], - [ - "ç¬", - "ij" - ], - [ - "çī¹", - "å®ļ" - ], - [ - "第", - "ä¸Ģ" - ], - [ - "Ġdep", - "ression" - ], - [ - "Ġinnov", - "ation" - ], - [ - "ĠF", - "r" - ], - [ - "Ġcomput", - "er" - ], - [ - "c", - "an" - ], - [ - "å³", - "°" - ], - [ - "ç¼ĸåĨĻ", - "ä¸Ģ个" - ], - [ - "Ġintern", - "ational" - ], - [ - "Ġcan", - "cer" - ], - [ - "åѦ", - "èĢħ" - ], - [ - "Ġdisc", - "over" - ], - [ - "he", - "t" - ], - [ - "Ġcomp", - "os" - ], - [ - "Ġrec", - "y" - ], - [ - "Ġ2", - "00" - ], - [ - "åIJ«", - "æľī" + "åĢ", + "ĭ" ], [ "çĹ", "Ľ" ], - [ - "ç¼ĵ", - "解" - ], - [ - "Ġfre", - "qu" - ], - [ - "çĶ", - "³" - ], - [ - "ĠM", - "ar" - ], - [ - "çļĦ", - "éĢīæĭ©" - ], - [ - "Ġun", - "t" - ], - [ - "Ġreg", - "ions" - ], - [ - "Ġop", - "in" - ], - [ - "ĠGovern", - "ments" - ], - [ - "æ¶", - "Ĥ" - ], - [ - "åĨħ", - "å¿ĥ" - ], - [ - "ä¸Ĭ", - "æľĢ" - ], [ "ä»į", "çĦ¶" ], [ - "l", - "ier" + "è¦Ĩ", + "çĽĸ" ], [ - "æ³", - "³" - ], - [ - "äºĴ", - "缸" - ], - [ - "ĠSt", - "ud" - ], - [ - "az", - "on" - ], - [ - "Ġar", - "ch" - ], - [ - "Ġche", - "m" - ], - [ - "çļĦ", - "èĥ½åĬĽ" - ], - [ - "çļĦ", - "ä¸Ģ个" - ], - [ - "Ġa", + "Ġn", "p" ], [ - "Ġre", - "d" - ], - [ - "Ġw", - "omen" - ], - [ - "Ġpro", - "te" - ], - [ - "Ġfind", - "ing" - ], - [ - "å§", - "»" - ], - [ - "éĢĤå½ĵ", - "çļĦ" - ], - [ - "Ġfor", - "ward" - ], - [ - "对", - "象" - ], - [ - "Ġwa", - "it" - ], - [ - "Ġconsid", - "ered" - ], - [ - "du", - "le" - ], - [ - "b", - "acks" - ], - [ - "Ġclin", - "ical" - ], - [ - "åħ·", - "å¤ĩ" - ], - [ - "éº", - "¦" - ], - [ - "Ġon", - "going" - ], - [ - "åĨ", - "Ľ" - ], - [ - "Ġf", - "ar" - ], - [ - "åĴĮ", - "è°" - ], - [ - "XX", - "X" - ], - [ - "Ġpolit", - "ical" - ], - [ - "Ġcam", - "er" - ], - [ - "çļĦ", - "è¡Į为" - ], - [ - "æĦı", - "大åĪ©" - ], - [ - "Ġapp", - "s" - ], - [ - "åĩı", - "è½»" - ], - [ - "Ġread", - "ers" - ], - [ - "å©ļ", - "姻" - ], - [ - "æ°", - "¸" - ], - [ - "o", - "res" - ], - [ - "åħ¨", - "éĿ¢" - ], - [ - "ĠAf", - "ric" - ], - [ - "Ġfavor", - "ite" - ], - [ - "Ġm", - "ill" - ], - [ - "Ġd", - "ang" - ], - [ - "ĠSt", - "ates" - ], - [ - "åĢ", - "Ł" - ], - [ - "å¯", - "¿" - ], - [ - "Ġl", - "at" - ], - [ - "è¿ĩ", - "åİ»" - ], - [ - "Ġtr", - "uly" - ], - [ - "åĽŀçŃĶ", - "éĹ®é¢ĺ" - ], - [ - "Ġco", - "gn" - ], - [ - "ä»", - "°" - ], - [ - "ĠJ", - "apan" - ], - [ - "iz", - "z" - ], - [ - "çļĦæĿ", - "IJ" - ], - [ - "x", - "x" - ], - [ - "é¢ĺ", - "缮" - ], - [ - "ri", - "ption" - ], - [ - "éĤ£", - "äºĽ" - ], - [ - "Ġbud", - "get" - ], - [ - "Ġv", - "ast" - ], - [ - "éļIJ", - "ç§ģ" - ], - [ - "Ġpolic", - "ymakers" - ], - [ - "è¿ĺ", - "éľĢè¦ģ" - ], - [ - "并", - "æıIJä¾Ľ" - ], - [ - "Ġswe", - "et" - ], - [ - "Ġgener", - "al" - ], - [ - "æ»", - "¤" - ], - [ - "Ġbir", - "ds" - ], - [ - "Ġpl", - "astic" - ], - [ - "Ċ", - "ĉ" - ], - [ - "åĪ", - "º" - ], - [ - "ment", - "al" - ], - [ - "Ġincl", - "usive" - ], - [ - "Ġtop", - "ics" - ], - [ - "Ġs", - "low" - ], - [ - "ä½ł", - "èĥ½" - ], - [ - "è¶³å¤Ł", - "çļĦ" - ], - [ - "è§Ĩ", - "è§ī" - ], - [ - "w", - "w" - ], - [ - "Ġ", - "使çĶ¨" - ], - [ - "æī", - "¹" - ], - [ - "æ¦Ĥ", - "念" - ], - [ - "é£Ł", - "çĶ¨" - ], - [ - "èĢ", - "³" - ], - [ - "c", - "ks" - ], - [ - "Ġfra", - "ud" - ], - [ - "Ġingred", - "ients" - ], - [ - "Ġf", - "asc" - ], - [ - "åĮĹ", - "京" - ], - [ - "Ġf", - "r" - ], - [ - "Ġmanufact", - "uring" - ], - [ - "Ġ", - "ä½ľä¸º" - ], - [ - "Ġbe", - "ach" - ], - [ - "é¡", - "¿" - ], - [ - "eri", - "ous" - ], - [ - "å¤ĸ", - "è§Ĥ" - ], - [ - "é¢Ħ", - "éĺ²" - ], - [ - "æĿ¥", - "èĩª" - ], - [ - "èĤĮ", - "èĤī" - ], - [ - "Ġd", - "ays" - ], - [ - "Ġass", - "ign" - ], - [ - "Ġadv", - "ant" - ], - [ - "Ġteam", - "s" - ], - [ - "é¢", - "Ĺ" - ], - [ - "now", - "n" - ], - [ - "ĠP", - "o" - ], - [ - "}", - "{" - ], - [ - "Ġmin", - "ut" - ], - [ - "it", - "ions" - ], - [ - "Ġeas", - "ily" - ], - [ - "ĠB", - "l" - ], - [ - "n", - "ame" - ], - [ - "åѦ", - "æł¡" - ], - [ - "Ġrespons", - "ibility" - ], - [ - "åıij", - "æĮ¥" - ], - [ - "Ġsens", - "itive" - ], - [ - "çŃī", - "äºİ" - ], - [ - "ci", - "ous" - ], - [ - "Ġs", - "ou" - ], - [ - "å±", - "ı" - ], - [ - "Ġr", - "ich" - ], - [ - "å½ĵ", - "çĦ¶" - ], - [ - "m", - "an" - ], - [ - "Ġinterpre", - "t" - ], - [ - "2", - "4" - ], - [ - "Ġshow", - "s" - ], - [ - "èģĮ", - "åľº" - ], - [ - "Ġf", - "all" - ], - [ - "è½", - "½" - ], - [ - "丰å¯Į", - "çļĦ" - ], - [ - "(", - "'" - ], - [ - "ä¿®", - "æĶ¹" - ], - [ - "æĽ´", - "æį¢" - ], - [ - "A", - "l" - ], - [ - "åı¯èĥ½", - "æĺ¯" - ], - [ - "Ġr", - "ate" - ], - [ - "Ġprotect", - "ing" - ], - [ - "f", - "it" - ], - [ - "Ġ5", - "0" - ], - [ - "Ġmove", - "ment" - ], - [ - "è§", - "Ī" - ], - [ - "Ġemploy", - "ee" - ], - [ - "Ġdis", - "ord" - ], - [ - "åĪĽ", - "æĦı" - ], - [ - "产åĵģ", - "çļĦ" - ], - [ - "æľ", - "Ŀ" - ], - [ - "ĊĠĠĠĠĠĠĠĠ", - "ĠĠĠĠĠĠĠ" - ], - [ - "Ġpre", - "d" - ], - [ - "Ġoffer", - "ing" - ], - [ - "åįģ", - "åĪĨ" - ], - [ - "èĢĮ", - "ä¸įæĺ¯" - ], - [ - "Th", - "ank" - ], - [ - "æĽ", - "¾" - ], - [ - "Ġele", - "ments" - ], - [ - "ç²", - "Ĵ" - ], - [ - "Ġcour", - "ses" - ], - [ - "Ġintegr", - "ated" - ], - [ - "ĠC", - "ar" - ], - [ - "agra", - "ph" - ], - [ - "åŁº", - "åĽł" - ], - [ - "Ġinst", - "ead" - ], - [ - "èĦ", - "±" - ], - [ - "åı¦", - "ä¸Ģ个" - ], - [ - "å¯Ĩ", - "çłģ" - ], - [ - "Ġallow", - "ed" - ], - [ - "éĿ¢", - "åĮħ" - ], - [ - "çķ", - "ªèĮĦ" - ], - [ - "åĴĮ", - "åıijå±ķ" - ], - [ - "å°", - "ģ" - ], - [ - "Ġconnect", - "ion" - ], - [ - "åľ¨", - "ä¸Ģ个" - ], - [ - "Ġuse", - "ful" - ], - [ - "è¯Ń", - "åı¥" - ], - [ - "åĪĨ", - "å¸ĥ" - ], - [ - "表", - "æ¼Ķ" - ], - [ - "æľī", - "æŶ" - ], - [ - "çļĦæĹ", - "ħ" - ], - [ - "çļĦæĢ", - "»" - ], - [ - "Ġf", - "ashion" - ], - [ - "èĭ", - "¦" - ], - [ - "è¦ģ", - "注æĦı" - ], - [ - "çĶŁ", - "ç´ł" - ], - [ - "Ġnut", - "ri" - ], - [ - "èĩª", - "è¡Į" - ], - [ - "çļĦç", - "ĭ" - ], - [ - "çIJĨ解", - "åĴĮ" - ], - [ - "Ġc", - "at" - ], - [ - "æľºåĻ¨", - "åŃ¦ä¹ł" - ], - [ - "Ġexh", - "ib" - ], - [ - "åĴĮ", - "æľįåĬ¡" - ], - [ - "fra", - "c" - ], - [ - "e", - "pend" - ], - [ - "Ġimpact", - "ed" - ], - [ - "Ġ", - "ut" - ], - [ - "æķ°", - "ç»Ħ" - ], - [ - "ĠWor", + "Ġo", "ld" ], [ - "Ġansw", - "er" + "ãģ", + "¦" ], [ - "ers", - "e" + "å½¢", + "象" ], [ - "éª", - "¨" + "èĢĮ", + "æĺ¯" ], [ - "Ġart", - "ists" + "Ġform", + "at" ], [ - "åŃ©åŃIJ", - "çļĦ" + "Ġfin", + "ancial" ], [ - "ä»", - "Ķ" + "åı¥", + "è¯Ŀ" ], [ "çĻ", - "»" + "¾" ], [ - "ĠA", - "re" - ], - [ - "Ġco", - "ol" - ], - [ - "Ġcogn", - "itive" - ], - [ - "åIJĦ", - "个" + "H", + "e" ], [ "l", - "ike" + "u" ], [ - "å©´", - "åĦ¿" + "åĨ", + "Ĵ" ], [ - "åĪĹ", - "åĩº" + "æ·±", + "åĪ»" + ], + [ + "并", + "ä¸Ķ" + ], + [ + "av", + "as" + ], + [ + "åIJ¯", + "åĬ¨" ], [ "å¹", "»" ], [ - "ron", - "t" + "Ġdat", + "ab" ], [ - "家", - "éķ¿" + "æľĢ", + "ä½³" ], [ - "缺", - "ä¹ı" + "Ġnam", + "es" ], [ - "Ġcy", - "ber" + "ç¬", + "ij" ], [ - "il", - "t" + "âĢĿ", + "ãĢģ" ], [ - "Ġcapt", - "ure" + "Ġcomput", + "er" ], [ - "å", - "Ĺ" + "Ġab", + "ility" ], [ - "åľ¨", - "äºİ" + "Ġparticular", + "ly" ], [ - "Ġthreat", - "s" + "åıĸ", + "å¾Ĺ" ], [ - "åĴĮ", - "社ä¼ļ" + "çļĦç¬", + "¬" ], [ - "Ġcell", - "s" + "avas", + "cript" ], [ - "æ¸ħ", - "åįķ" + "o", + "ice" ], [ - "ĠV", - "is" + "Ġdoc", + "ument" ], [ - "æİ", - "ī" + "ĠL", + "et" ], [ - "Ġh", - "ol" + "Ġapplic", + "ations" ], [ - "åŃIJ", - "çļĦ" + "缮", + "åīį" ], [ - "C", - "h" + "Ġcle", + "ar" ], [ - "è", - "Ŀ" + "isp", + "ers" ], [ - "Ġs", - "aid" + "Ã", + "³" ], [ - "Ġd", - "ream" - ], - [ - "un", - "ch" - ], - [ - "un", - "e" - ], - [ - "ĠD", - "on" - ], - [ - "家", - "人" - ], - [ - "ç±", - "į" - ], - [ - "æĦŁ", - "åĴĮ" - ], - [ - "Ġexperi", - "enced" - ], - [ - "çļĦéĩįè¦ģ", - "æĢ§" - ], - [ - "å¼", - "ĥ" - ], - [ - "um", - "p" - ], - [ - "éĺ", - "IJ" - ], - [ - "Ġhabit", - "at" - ], - [ - "è¢", - "ĭ" - ], - [ - "Ġj", - "o" - ], - [ - "ç®Ģ", - "æ´ģ" - ], - [ - "Ġb", - "ur" - ], - [ - "Ġvisit", - "ors" - ], - [ - "éĽ", - "ħ" - ], - [ - "çļĦçŁ", + "ç«", "¥" ], [ - "Ġent", - "ire" + "è¿Ļæł·", + "çļĦ" ], [ - "讲", - "è¿°" + "帮", + "æĪij" ], [ - "äºĨ", - "ä¸ĢäºĽ" + "çľĭ", + "èµ·æĿ¥" ], [ - "åįı", - "ä½ľ" + "å¹", + "¸" ], [ - "ĠB", - "us" + "å°ı", + "说" + ], + [ + "pl", + "ay" + ], + [ + "d", + "ict" + ], + [ + "is", + "hed" + ], + [ + "Ġp", + "ract" + ], + [ + "çķĮ", + "éĿ¢" + ], + [ + "Ġa", + "chie" + ], + [ + "Ġ", + "å°Ĩ" + ], + [ + "çłĶ", + "åıij" + ], + [ + "æ¼", + "«" + ], + [ + "åı¤", + "代" + ], + [ + "çļĦç¼ĸ", + "ç¨ĭ" + ], + [ + "y", + "stem" + ], + [ + "Ġwith", + "in" + ], + [ + "ç£", + "ģ" + ], + [ + "Ġm", + "ar" + ], + [ + "r", + "ight" + ], + [ + "ä¿®", + "å¤į" + ], + [ + "led", + "ge" + ], + [ + "åį", + "ł" + ], + [ + "E", + "O" + ], + [ + "æĪij们", + "çļĦ" + ], + [ + "çī©", + "æµģ" + ], + [ + "ãģ¾", + "ãģĻ" + ], + [ + "Ġcount", + "ry" + ], + [ + "Ġse", + "ver" + ], + [ + "å±ŀ", + "äºİ" + ], + [ + "N", + "A" + ], + [ + "ĠN", + "ew" + ], + [ + "åıĬ", + "åħ¶" + ], + [ + "Ġpoint", + "s" + ], + [ + "ç´", + "¯" + ], + [ + "å°½", + "åĬĽ" + ], + [ + "æĢİ", + "ä¹Ī" + ], + [ + "d", + "ing" + ], + [ + "çļĦæł¸", + "å¿ĥ" + ], + [ + "大", + "éĩıçļĦ" + ], + [ + "Ġev", + "ent" + ], + [ + "an", + "e" + ], + [ + "Ġth", + "ose" + ], + [ + "éĢīæĭ©", + "åIJĪéĢĤçļĦ" + ], + [ + "Ġinteg", + "ers" + ], + [ + "Ġ", + "æĺ¯" + ], + [ + "ab", + "les" + ], + [ + "çŃĶ", + "æ¡Ī" + ], + [ + "l", + "and" + ], + [ + "m", + "l" + ], + [ + "ãģ", + "¨" + ], + [ + "Ġder", + "iv" + ], + [ + "âĢĿãĢģ", + "âĢľ" + ], + [ + "he", + "l" + ], + [ + "Ġ", + "â" + ], + [ + "åħ", + "°" + ], + [ + "æ··", + "åIJĪ" + ], + [ + "æĢĿ", + "æĥ³" + ], + [ + "å¸ĥ", + "å±Ģ" + ], + [ + "çī", + "Ļ" + ], + [ + "æģ", + "¶" + ], + [ + "å¼ķ", + "èµ·" + ], + [ + "Ġ", + "å¦Ĥæŀľ" ], [ "å°", "¾" ], [ - "çļĦæķ", - "Ļ" + "è¿Ļ个", + "åĩ½æķ°" ], [ - "olo", - "g" + "è§Ĥ", + "ä¼Ĺ" ], [ - "Ġsign", - "s" + "åŁº", + "åĽł" ], [ - "Ġspeak", - "er" + "ER", + "E" + ], + [ + "L", + "et" ], [ "çļĦ", - "éŁ³ä¹IJ" + "æķ°åŃĹ" ], [ - "Ġno", - "vel" + "è¿", + "Ń" ], [ - "å±ħ", - "æ°ij" + "çª", + "Ĺ" + ], + [ + "lo", + "ad" + ], + [ + "Ġdeterm", + "ine" + ], + [ + "om", + "s" + ], + [ + "åѦ", + "ä¼ļ" + ], + [ + "èĦ", + "Ĥ" + ], + [ + "if", + "t" + ], + [ + "éĿ¢", + "积" + ], + [ + "家", + "人" + ], + [ + "äº", + "Ī" + ], + [ + "Ġp", + "ot" + ], + [ + "ä½ł", + "对" + ], + [ + "iv", + "ing" + ], + [ + "ç±»åŀĭ", + "çļĦ" + ], + [ + "严", + "éĩį" + ], + [ + "Ġsever", + "al" + ], + [ + "çĬ", + "¯" + ], + [ + "ç±»", + "ä¼¼" + ], + [ + "us", + "h" + ], + [ + "第", + "ä¸Ģ" + ], + [ + "è¾ħ", + "åĬ©" + ], + [ + "ä¾ĽåºĶ", + "éĵ¾" + ], + [ + "åĴĮ", + "对" + ], + [ + "æ»", + "ij" + ], + [ + "Ġequ", + "ations" + ], + [ + "ä¸ĵä¸ļ", + "çļĦ" ], [ "çļĦ", "åıĺåĮĸ" ], [ - "å°½", - "éĩı" + "Ġsol", + "utions" ], [ - "Ġspir", + "Ġb", + "ase" + ], + [ + "Ġcommun", + "ic" + ], + [ + "æį", + "ķ" + ], + [ + "åѦ", + "èĢħ" + ], + [ + "Ġl", + "ove" + ], + [ + "çļĦæł", + "ĩ" + ], + [ + "Ġhig", + "hest" + ], + [ + "j", + "avascript" + ], + [ + "Ġrec", + "ord" + ], + [ + "Ġm", + "ath" + ], + [ + "an", + "ced" + ], + [ + "Ġpur", + "pose" + ], + [ + "çļĦ", + "æĢ»" + ], + [ + "é£Ł", + "åĵģ" + ], + [ + "AI", + "æĬĢæľ¯" + ], + [ + "H", + "ERE" + ], + [ + "he", + "mat" + ], + [ + "åĢ", + "º" + ], + [ + "Ã", + "Ń" + ], + [ + "Ġac", + "ross" + ], + [ + "ä¸ī", + "è§Ĵ" + ], + [ + "æ¶īåıĬ", + "åĪ°" + ], + [ + "Ġwor", + "king" + ], + [ + "is", + "m" + ], + [ + "stit", + "ute" + ], + [ + "éĴĪ", + "对" + ], + [ + "Ġin", "it" ], [ - "å®Į", - "ç¾İ" + "a", + "iled" + ], + [ + "n", + "ot" + ], + [ + "çĽ", + "IJ" + ], + [ + "æĺĵ", + "äºİ" + ], + [ + "Ġlead", + "ing" + ], + [ + "Ġv", + "ery" + ], + [ + "Ġon", + "line" + ], + [ + "o", + "ot" + ], + [ + "ç´¢", + "å¼ķ" + ], + [ + "è´§", + "å¸ģ" + ], + [ + "çĥ", + "¤" + ], + [ + "é¸", + "Ł" + ], + [ + "ĠP", + "r" + ], + [ + "ut", + "es" + ], + [ + "çļĦ", + "åħĥç´ł" + ], + [ + "en", + "e" + ], + [ + "è¶ħ", + "è¿ĩ" + ], + [ + "严", + "æł¼" + ], + [ + "æĿĥ", + "éĻIJ" + ], + [ + "fere", + "nce" + ], + [ + "Ġsym", + "b" + ], + [ + "st", + "ems" + ], + [ + "åľ¨", + "äºİ" + ], + [ + "'", + "]" + ], + [ + "Ġem", + "b" + ], + [ + "a", + "ces" + ], + [ + "r", + "and" + ], + [ + "Ġcol", + "or" + ], + [ + "çļĦçĥ", + "Ń" + ], + [ + "N", + "ow" + ], + [ + "d", + "iv" + ], + [ + "é¢", + "Ĺ" + ], + [ + "çļĦåľ°", + "æĸ¹" + ], + [ + "ä¿Ŀ", + "éĻ©" + ], + [ + "çī©", + "åĵģ" + ], + [ + "æłĩ", + "çŃ¾" + ], + [ + "Ġrespon", + "se" + ], + [ + "Ġm", + "eters" + ], + [ + "Python", + "ä¸Ń" + ], + [ + "è¯", + "º" + ], + [ + "å¤ĸ", + "éĥ¨" + ], + [ + "Ġin", + "nov" + ], + [ + "è¿Ľ", + "åĪ¶" + ], + [ + "èĭ", + "ı" + ], + [ + "oo", + "ks" + ], + [ + "ç¼", + "ĺ" + ], + [ + "çļ®", + "èĤ¤" + ], + [ + "让", + "人" + ], + [ + "Ġsol", + "ution" + ], + [ + "et", + "er" + ], + [ + "çĻ»", + "å½ķ" + ], + [ + "Ġ", + "Wh" + ], + [ + "客", + "æľį" + ], + [ + "åİŁ", + "å§ĭ" + ], + [ + "åĴĮ", + "åºĶçĶ¨" + ], + [ + "èĢIJ", + "å¿ĥ" + ], + [ + "b", + "y" + ], + [ + "çļĦ", + "缮æłĩ" + ], + [ + "ä¿Ŀ", + "è¯ģ" + ], + [ + "ä½ł", + "æĥ³è¦ģ" + ], + [ + "容", + "åĻ¨" + ], + [ + "p", + "re" + ], + [ + "çĹħ", + "æ¯Ĵ" + ], + [ + "Ġimp", + "act" + ], + [ + "æıIJ", + "åıĸ" + ], + [ + "æĥ", + "ł" + ], + [ + "--", + "-" + ], + [ + "ers", + "hip" + ], + [ + "åĴĮ", + "社ä¼ļ" + ], + [ + "åħħ", + "åĪĨ" + ], + [ + "æķ°", + "åĢ¼" + ], + [ + "Ġcond", + "itions" + ], + [ + "id", + "d" + ], + [ + "åºĶçĶ¨", + "äºİ" + ], + [ + "è£ħ", + "饰" + ], + [ + "é¢ĨåŁŁ", + "çļĦ" + ], + [ + "Ġrespon", + "s" + ], + [ + "çļĦæ³ķ", + "å¾ĭ" + ], + [ + "æĬ", + "Ĭ" + ], + [ + "åĵ²", + "åѦ" + ], + [ + "强", + "大çļĦ" + ], + [ + "Ġse", + "lect" + ], + [ + "c", + "os" + ], + [ + "çī¹", + "èī²" + ], + [ + "ĠThe", + "y" + ], + [ + "线", + "æĢ§" + ], + [ + "Ġs", + "ide" + ], + [ + "æµ", + "ĵ" + ], + [ + "æķ´", + "ä½ĵ" + ], + [ + "W", + "eb" + ], + [ + "Ġsp", + "ace" + ], + [ + "Ġart", + "ic" + ], + [ + "éĵ¾", + "æİ¥" + ], + [ + "Ġjo", + "ke" + ], + [ + "w", + "ay" + ], + [ + "çļĦ", + "æľįåĬ¡" + ], + [ + "Ġmodel", + "s" + ], + [ + "å±ħ", + "æ°ij" + ], + [ + "ï¼Į", + "âĢľ" + ], + [ + "æĹħ", + "游" + ], + [ + "åĴĮ", + "åĪĨæŀIJ" + ], + [ + "æµ", + "ıè§Ī" + ], + [ + "l", + "ess" + ], + [ + "Ġp", + "ain" + ], + [ + "çļĦæ¦Ĥ", + "念" + ], + [ + "çī¹åĪ«", + "æĺ¯" + ], + [ + "Ġperform", + "ance" + ], + [ + "id", + "i" + ], + [ + "=", + "\"" + ], + [ + "r", + "on" + ], + [ + "Ġm", + "ind" + ], + [ + "Ġvari", + "ety" + ], + [ + "Ù", + "ħ" + ], + [ + "ĠB", + "e" + ], + [ + "Ġst", + "ore" + ], + [ + "æŃ£", + "常" + ], + [ + "Ġpro", + "du" + ], + [ + "广", + "åijĬ" + ], + [ + "çĶ", + "²" + ], + [ + "æľĢ", + "æĸ°çļĦ" + ], + [ + "顾", + "客" + ], + [ + "æĺ¯", + "ä¸ŃåĽ½" + ], + [ + "Ġrel", + "ations" + ], + [ + "Ġm", + "ountain" + ], + [ + "Ġfact", + "ors" ], [ "è´", "·" ], [ - "å¿ħè¦ģ", - "çļĦ" + "æ·±", + "è¿ľ" ], [ - "ie", - "f" + "å°", + "¼" ], [ - "示", - "ä¾ĭ" + "Ġgent", + "le" ], [ - "Ġd", - "iv" + "Ġmil", + "es" ], [ - "æķ´", - "æķ°" + "Ġbe", + "fore" ], [ - "Ġeconom", - "y" - ], - [ - "Ġethical", - "ly" - ], - [ - "éĻ", + "çĥ", "Ī" ], [ - "Ġschool", + "ar", + "ray" + ], + [ + "æķ", + "ij" + ], + [ + "Ġv", + "ol" + ], + [ + "Ġens", + "ure" + ], + [ + "æ¬", + "§" + ], + [ + "æľīæīĢ", + "ä¸įåIJĮ" + ], + [ + "Ġknow", + "ledge" + ], + [ + "ç¢", + "į" + ], + [ + "åıįæĺł", + "äºĨ" + ], + [ + "è°", + "Ī" + ], + [ + "Ġo", + "pen" + ], + [ + "åĨľ", + "ä¸ļ" + ], + [ + "è¿ŀ", + "ç»Ń" + ], + [ + "éĶ", + "ģ" + ], + [ + "ĠAd", + "ditionally" + ], + [ + "ç»ı", + "åħ¸" + ], + [ + "Ġb", + "ro" + ], + [ + "æĮ", + "¥" + ], + [ + "ä¸ĭ", + "æĿ¥" + ], + [ + "ä¹Ł", + "åı¯èĥ½" + ], + [ + "ä¸ĭ", + "éĻį" + ], + [ + "éĽĨ", + "åIJĪ" + ], + [ + "ag", + "raph" + ], + [ + "Ġexp", + "lo" + ], + [ + "mer", + "ic" + ], + [ + "Ġl", + "en" + ], + [ + "éĽ", + "ª" + ], + [ + "Ñ", + "ĭ" + ], + [ + "Ġst", + "ructure" + ], + [ + "Ġman", + "age" + ], + [ + "æŀĦ", + "æĪIJ" + ], + [ + "ä»·åĢ¼", + "è§Ĥ" + ], + [ + "T", + "P" + ], + [ + "ä¾", + "§" + ], + [ + "设计", + "ä¸Ģ个" + ], + [ + "åĬ©", + "æīĭ" + ], + [ + "Ġre", + "st" + ], + [ + "ä½ł", + "åı¯èĥ½éľĢè¦ģ" + ], + [ + "Ġde", + "v" + ], + [ + "åĮ", + "Ģ" + ], + [ + "P", + "ro" + ], + [ + "çŃī", + "æĸ¹å¼ı" + ], + [ + "çĩ", + "ĥ" + ], + [ + "Ġg", + "old" + ], + [ + "åıĤ", + "èĢĥ" + ], + [ + "o", + "ber" + ], + [ + "Ġ\\", + "," + ], + [ + "Ġdevelop", + "ed" + ], + [ + "è¯Ĺ", + "æŃĮ" + ], + [ + "c", + "ent" + ], + [ + "è¿", + "¹" + ], + [ + "大", + "åѦ" + ], + [ + "ĠD", + "e" + ], + [ + "cc", + "ess" + ], + [ + "Ġ", + "&" + ], + [ + "Ġm", + "us" + ], + [ + "æ°", + "¸" + ], + [ + "ct", + "or" + ], + [ + "æ¸", + "łéģĵ" + ], + [ + "éļIJç§ģ", + "ä¿ĿæĬ¤" + ], + [ + "acc", + "i" + ], + [ + "æĢ§", + "è´¨" + ], + [ + "æŃ£ç¡®", + "çļĦ" + ], + [ + "åį", + "±" + ], + [ + "åIJĪ", + "åIJĮ" + ], + [ + "æłĩ", + "é¢ĺ" + ], + [ + "b", + "e" + ], + [ + "è¿", + "ħ" + ], + [ + "åį", + "Ī" + ], + [ + "iv", + "ity" + ], + [ + "ib", + "on" + ], + [ + "æ³", + "Ħ" + ], + [ + "转", + "åĮĸ" + ], + [ + "éĺ¿éĩĮå·´å·´", + "äºij" + ], + [ + "çĶµ", + "æ±ł" + ], + [ + "get", + "her" + ], + [ + "çļĦ", + "建议" + ], + [ + "Ġto", + "gether" + ], + [ + "a", + "de" + ], + [ + "Ġl", + "ibr" + ], + [ + "çļĦæı", + "IJ" + ], + [ + "Ġhour", "s" ], [ - "Ġnet", - "works" + "ç»", + "©" + ], + [ + "大", + "åľ°" + ], + [ + "Ġle", + "vel" + ], + [ + "ibon", + "acci" + ], + [ + "éĺ³", + "åħī" + ], + [ + "Ġavail", + "able" + ], + [ + "Ġus", + "es" + ], + [ + "Ġanal", + "y" + ], + [ + "çļĦæĬ", + "ķ" + ], + [ + "åij¨", + "æľŁ" + ], + [ + "èĦļ", + "æľ¬" + ], + [ + "Ġident", + "ify" + ], + [ + "ä¹", + "ħ" + ], + [ + "Ñ", + "ı" + ], + [ + "æľ", + "Ŀ" + ], + [ + "ĠC", + "on" + ], + [ + "is", + "ion" + ], + [ + "å®ļ", + "åĪ¶" + ], + [ + "éľĢè¦ģ", + "注æĦı" + ], + [ + "åħĥç´ł", + "çļĦ" + ], + [ + "Ġart", + "ificial" + ], + [ + "Ġma", + "x" + ], + [ + "ern", + "et" + ], + [ + "ãģ", + "Ł" + ], + [ + "çļĦç", + "©" + ], + [ + "å¹³", + "æĸ¹" + ], + [ + "æŁ", + "Ķ" + ], + [ + "I", + "N" + ], + [ + "羣", + "å®ŀ" + ], + [ + "æ²", + "¿" + ], + [ + "æĪIJ", + "å°±" + ], + [ + "ä¼ĺ", + "åħĪ" + ], + [ + "Ġres", + "ources" + ], + [ + "Ġb", + "as" + ], + [ + "å°", + "º" + ], + [ + "æIJ", + "Ń" + ], + [ + "åĮºåĿĹéĵ¾", + "æĬĢæľ¯" + ], + [ + "æī", + "®" + ], + [ + "Ġmon", + "th" + ], + [ + "å¥", + "ı" + ], + [ + "第", + "ä¸ī" + ], + [ + "Ġref", + "lect" + ], + [ + "çļĦä¸Ģ", + "éĥ¨åĪĨ" + ], + [ + "çļĦ", + "åģ¥åº·" + ], + [ + "è®°", + "ä½ı" + ], + [ + "梦", + "æĥ³" + ], + [ + "é¢Ħ", + "éĺ²" + ], + [ + "æķ", + "¸" + ], + [ + "Ġch", + "all" + ], + [ + "Ġcont", + "rib" + ], + [ + "S", + "ON" + ], + [ + "大", + "åŀĭ" + ], + [ + "缸", + "ä¼¼" + ], + [ + "Ġs", + "ense" + ], + [ + "an", + "ces" + ], + [ + "æ¼", + "ı" + ], + [ + "ç¥ŀç»ı", + "ç½ij绾" + ], + [ + "Ġb", + "ir" + ], + [ + "æľ¬", + "身" + ], + [ + "æ¶", + "µ" + ], + [ + "Ġgre", + "at" + ], + [ + "}", + "\\)" + ], + [ + "Ġgrow", + "th" + ], + [ + "Ġbe", + "h" + ], + [ + "Ġpar", + "agraph" + ], + [ + "è¿Ń", + "代" + ], + [ + "åĬĽ", + "çļĦ" + ], + [ + "å¹´", + "è½»" + ], + [ + "Ġc", + "all" + ], + [ + "Ġstep", + "s" + ], + [ + "3", + "3" + ], + [ + "ar", + "get" + ], + [ + "éĵ", + "º" + ], + [ + "åı¦", + "ä¸Ģ个" + ], + [ + "Ġexpl", + "ain" + ], + [ + ")", + "`" + ], + [ + "Ġan", + "n" + ], + [ + "ot", + "al" + ], + [ + "æīĢ", + "éľĢ" + ], + [ + "åį", + "ĸ" + ], + [ + "æ²", + "Ļ" + ], + [ + "Ġcir", + "cle" + ], + [ + "Ġclar", + "ify" + ], + [ + "m", + "an" + ], + [ + "i", + "ver" + ], + [ + "r", + "ame" + ], + [ + "èĢĮ", + "ä¸Ķ" + ], + [ + "èµĦ", + "产" + ], + [ + "ä¹", + "¡" + ], + [ + "Ġv", + "er" + ], + [ + "Ġ6", + "0" + ], + [ + "A", + "nd" + ], + [ + "ĠS", + "o" + ], + [ + "Ġ4", + "0" + ], + [ + "声", + "éŁ³" + ], + [ + "åĴĮ", + "æĬĢæľ¯" + ], + [ + "al", + "cul" + ], + [ + "æĭ", + "į" + ], + [ + "æ¯ı", + "次" + ], + [ + "æľī", + "åħ³" + ], + [ + "iv", + "al" + ], + [ + "çļĦç", + "Ł" + ], + [ + "èĩ³", + "å°ij" + ], + [ + "Ġre", + "du" + ], + [ + "æŶ", + "çļĦ" + ], + [ + "å¹´", + "çļĦ" + ], + [ + "Ġind", + "ust" + ], + [ + "Ġexper", + "ience" + ], + [ + "åĪ", + "¸" + ], + [ + "两", + "ç§į" + ], + [ + "æł¸", + "å¿ĥ" + ], + [ + "gor", + "ithm" + ], + [ + "æİĮ", + "æı¡" + ], + [ + "çģ", + "¾" + ], + [ + "大", + "å¤ļ" + ], + [ + "å·¥", + "ä¸ļ" + ], + [ + "ac", + "y" + ], + [ + "ãĢĭ", + "ï¼Ī" + ], + [ + "inu", + "x" + ], + [ + "9", + "0" + ], + [ + "è¿Ľ", + "å±ķ" + ], + [ + "çļĦæ¸", + "©" + ], + [ + "åħħ满", + "äºĨ" + ], + [ + "éŃ", + "Ķ" + ], + [ + "Ð", + "¼" + ], + [ + "Ġsu", + "bs" + ], + [ + "Python", + "çļĦ" + ], + [ + "åĪĨ", + "åŃIJ" + ], + [ + "S", + "T" + ], + [ + "æľī", + "äºĽ" + ], + [ + "温", + "æļĸ" + ], + [ + "Ġs", + "it" + ], + [ + "Ġdet", + "ailed" + ], + [ + "å¿", + "ĺ" + ], + [ + "Ġhe", + "ad" + ], + [ + "éĻ", + "Ħ" + ], + [ + "å®ĥ", + "çļĦ" + ], + [ + "æĸ¹", + "便" + ], + [ + "ĠJava", + "Script" + ], + [ + "Ġw", + "on" + ], + [ + "让", + "ä½ł" + ], + [ + "Ġtransl", + "ate" + ], + [ + "æĺ¯åIJ¦", + "æľī" + ], + [ + "Ġv", + "irtual" + ], + [ + "C", + "om" + ], + [ + "以", + "åħ¶" + ], + [ + "as", + "ure" + ], + [ + "ut", + "put" + ], + [ + "Ã", + "ł" + ], + [ + "Ġfollow", + "s" + ], + [ + "Ľ", + "åĽ´" + ], + [ + "ä¿ĥè¿Ľ", + "äºĨ" + ], + [ + "ä½ĵçİ°", + "äºĨ" + ], + [ + "æŶ", + "代" + ], + [ + "Ġmat", + "hemat" + ], + [ + "Ġf", + "uture" + ], + [ + "æĿİ", + "æĺİ" + ], + [ + "L", + "P" + ], + [ + "è¿", + "Ŀ" + ], + [ + "æĸ°", + "é²ľ" + ], + [ + "注", + "éĩį" + ], + [ + "Ġpat", + "tern" + ], + [ + "éļ¾", + "以" + ], + [ + "éĢīæĭ©", + "ä¸Ģ个" + ], + [ + "ç¼ĸ", + "çłģ" + ], + [ + "f", + "ace" + ], + [ + "o", + "id" + ], + [ + "ur", + "y" + ], + [ + "éĩį", + "è§Ĩ" + ], + [ + "çļĦ模", + "åŀĭ" + ], + [ + "è¿IJ", + "ç®Ĺ" + ], + [ + "c", + "er" + ], + [ + "i", + "od" + ], + [ + "Ġc", + "r" + ], + [ + "(", + "-" + ], + [ + "Ġnew", + "s" + ], + [ + "ĠFr", + "ance" + ], + [ + "åº", + "Ĭ" + ], + [ + "ä¸ĵ", + "家" + ], + [ + "Ġrese", + "arch" + ], + [ + "Ġgre", + "ater" + ], + [ + "a", + "pe" + ], + [ + "åıª", + "æľī" + ], + [ + "åĵĪ", + "å¸Į" + ], + [ + "èģ", + "Ĭ" + ], + [ + "é«ĺ", + "级" + ], + [ + "èĶ", + "¬" + ], + [ + "å¢", + "Ļ" + ], + [ + "Ġspec", + "ial" + ], + [ + "Ġdirect", + "ly" + ], + [ + "å®īåħ¨", + "çļĦ" + ], + [ + "ç²", + "Ĵ" + ], + [ + "å¤ĩ", + "份" + ], + [ + "åĪ", + "º" + ], + [ + "Ġpr", + "ice" + ], + [ + "çĭ", + "Ĺ" + ], + [ + "Ġt", + "itle" + ], + [ + "æľī", + "åĵªäºĽ" + ], + [ + "Ġrelations", + "hip" + ], + [ + "Ñ", + "Į" + ], + [ + "ä¾", + "µ" + ], + [ + "çŃī", + "å¾ħ" + ], + [ + "ord", + "ing" + ], + [ + "åľ°", + "çĤ¹" + ], + [ + "Ġme", + "et" + ], + [ + "ä¼ł", + "éĢĴ" + ], + [ + "çļĦ", + "æĢģ度" + ], + [ + "æķħ", + "éļľ" + ], + [ + "Ġre", + "verse" + ], + [ + "et", + "ic" + ], + [ + "交", + "æį¢" + ], + [ + "Ġm", + "ust" + ], + [ + "é«ĺ", + "åħ´" + ], + [ + "ound", + "s" + ], + [ + "é¢Ħ", + "æľŁ" + ], + [ + "Ġfe", + "et" + ], + [ + "åĪĿ", + "åѦèĢħ" + ], + [ + "ä¼", + "Ļ" + ], + [ + "æıIJä¾Ľ", + "ä¸Ģ个" + ], + [ + "åİ", + "ħ" + ], + [ + "çļĦæµ", + "ģ" + ], + [ + "t", + "le" + ], + [ + "ç¼ĸç¨ĭ", + "è¯Ńè¨Ģ" + ], + [ + "åŃ£", + "èĬĤ" + ], + [ + "éĩıåŃIJ", + "计ç®Ĺ" + ], + [ + "u", + "ro" + ], + [ + "Ġse", + "a" + ], + [ + "æ¯Ķ", + "çī¹" + ], + [ + "æľī", + "éĻIJ" + ], + [ + "Ġar", + "g" + ], + [ + "F", + "ROM" + ], + [ + "Ġh", + "ome" + ], + [ + "Ġdisc", + "uss" + ], + [ + "åħ·", + "å¤ĩ" + ], + [ + "æļ", + "Ĺ" + ], + [ + "Ġpre", + "v" + ], + [ + "æī®", + "æ¼Ķ" + ], + [ + "åľ°", + "æĸ¹" + ], + [ + "åĮĹ", + "京" + ], + [ + "å®ģ", + "éĿĻ" + ], + [ + "è§Ħ", + "èĮĥ" + ], + [ + "社ä¼ļ", + "çļĦ" + ], + [ + "缮", + "å½ķ" + ], + [ + "åѦ", + "æľ¯" + ], + [ + "Ġm", + "ix" + ], + [ + "åľ¨", + "å¤ĦçIJĨ" + ], + [ + "ic", + "ro" + ], + [ + "ke", + "y" + ], + [ + "Ġwe", + "b" + ], + [ + "æĵįä½ľ", + "ç³»ç»Ł" + ], + [ + "¹", + "æŀľ" + ], + [ + "çī©", + "èģĶç½ij" + ], + [ + "æľįåĬ¡", + "çļĦ" + ], + [ + "æıIJ", + "åīį" + ], + [ + "çĤ", + "İ" + ], + [ + "åIJĮ", + "æł·" + ], + [ + "Ġem", + "ployees" + ], + [ + "Ġc", + "ut" + ], + [ + "åIJĪ", + "è§Ħ" + ], + [ + "çļĦ", + "éĢīæĭ©" + ], + [ + "å¥", + "¥" + ], + [ + "c", + "ol" + ], + [ + "ç§", + "Ģ" + ], + [ + "è¿Ľ", + "ç¨ĭ" + ], + [ + "}", + "}" + ], + [ + "æķĻ", + "åѦ" + ], + [ + "éĶ", + "ħ" + ], + [ + "u", + "c" + ], + [ + "ä¿¡", + "çĶ¨" + ], + [ + "ä¿¡æģ¯", + "çļĦ" + ], + [ + "çļĦæİ", + "¨" + ], + [ + "åĢ", + "į" + ], + [ + "éļľ", + "ç¢į" + ], + [ + "or", + "n" + ], + [ + "ĠI", + "nd" + ], + [ + "æľª", + "çŁ¥" + ], + [ + "çķ", + "ħ" + ], + [ + "è½»", + "è½»" + ], + [ + "ä¿Ŀ", + "éļľ" + ], + [ + "ĠA", + "pp" + ], + [ + "ä¸Ģ", + "ç³»åĪĹ" + ], + [ + "åĨ", + "¬" + ], + [ + "æĪij们", + "é¦ĸåħĪ" + ], + [ + "大å¤ļ", + "æķ°" + ], + [ + "çļĦ", + "产åĵģ" + ], + [ + "å½¢", + "çĬ¶" + ], + [ + "Ġallow", + "s" + ], + [ + "ol", + "ution" + ], + [ + "Ġs", + "ource" + ], + [ + "Ġo", + "cc" + ], + [ + "åIJĪ", + "æ³ķ" + ], + [ + "åľ¨", + "Pythonä¸Ń" + ], + [ + "大", + "约" + ], + [ + "Ġan", + "other" + ], + [ + "éĢIJ", + "æŃ¥" + ], + [ + "使çĶ¨", + "äºĨ" + ], + [ + "åĿĩ", + "åĮĢ" + ], + [ + "çļĦ", + "å¿ĥ" + ], + [ + "Ġw", + "ind" + ], + [ + "æĸ", + "Ĺ" + ], + [ + "Ġl", + "ower" + ], + [ + "解", + "æŀIJ" + ], + [ + "çĿ", + "£" + ], + [ + "å°", + "ļ" + ], + [ + "æģ", + "Ĵ" + ], + [ + "çĿ", + "¡çľł" + ], + [ + "ĠA", + "meric" + ], + [ + "è·", + "ij" + ], + [ + "âĢĶ", + "âĢĶ" + ], + [ + "ant", + "s" + ], + [ + "çīĽ", + "奶" + ], + [ + "æ°§", + "åĮĸ" + ], + [ + "m", + "od" + ], + [ + "å½¢", + "çļĦ" + ], + [ + "çļĦç©", + "º" + ], + [ + "Ð", + "²" + ], + [ + "al", + "es" + ], + [ + "an", + "a" + ], + [ + "æıIJä¾Ľ", + "æĽ´" + ], + [ + "Ġs", + "or" + ], + [ + "èĥ", + "ģ" + ], + [ + "i", + "et" + ], + [ + "ç§", + "Ĵ" + ], + [ + "ch", + "ange" + ], + [ + "Ġcount", + "ries" + ], + [ + "è¿ħ", + "éĢŁ" + ], + [ + "èĶ¬", + "èıľ" + ], + [ + "Ġth", + "ough" + ], + [ + "ĠC", + "EO" + ], + [ + "ide", + "red" + ], + [ + "èĭ", + "¹æŀľ" + ], + [ + "Ġmult", + "iply" + ], + [ + "Ġwrit", + "ing" + ], + [ + "j", + "or" + ], + [ + "çĥ", + "§" + ], + [ + "Ġtas", + "k" + ], + [ + "s", + "er" + ], + [ + "Ġdec", + "imal" + ], + [ + "int", + "er" + ], + [ + "æ³ķå¾ĭ", + "æ³ķè§Ħ" + ], + [ + "Ġphys", + "ical" + ], + [ + "Ġrad", + "ius" + ], + [ + "è·", + "ĥ" + ], + [ + "å°½", + "管" + ], + [ + "羣", + "æŃ£" + ], + [ + "æĢ»", + "ç»ĵ" + ], + [ + "å¯¹ä½ł", + "æľīæīĢ帮åĬ©" + ], + [ + "çļĦç²", + "¾" + ], + [ + "è§Ĩ", + "为" + ], + [ + "éĹ®é¢ĺ", + "çļĦ" + ], + [ + "Ġcons", + "idered" + ], + [ + "å¦ĤæŀľæĤ¨", + "æľī" + ], + [ + "Ġtop", + "ics" + ], + [ + "æĸĩ", + "æĺİ" + ], + [ + "Ġg", + "ra" + ], + [ + "è°", + "±" + ], + [ + "èĥ", + "ľ" + ], + [ + "åŁºæľ¬", + "çļĦ" + ], + [ + "dition", + "al" + ], + [ + "åı¯ä»¥", + "å°Ĩ" + ], + [ + "Ġre", + "d" + ], + [ + "ä¸ĭéĿ¢", + "æĺ¯ä¸Ģ个" + ], + [ + "æĹ¥å¸¸", + "çĶŁæ´»" + ], + [ + "å¦Ĥæŀľ", + "æľī" + ], + [ + "è¿ĺ", + "æľī" + ], + [ + "Ġpr", + "iv" + ], + [ + "å¨ģ", + "èĥģ" + ], + [ + "æĦŁ", + "åıĹ" + ], + [ + "åľ¨", + "ä¸ŃåĽ½" + ], + [ + "Ġadd", + "ition" + ], + [ + ")", + "\\)" + ], + [ + "ä¸ĸ", + "纪" + ], + [ + "ĠO", + "ct" + ], + [ + "讲", + "è¿°" + ], + [ + "Ġh", + "y" + ], + [ + "t", + "on" + ], + [ + "u", + "le" + ], + [ + "L", + "e" + ], + [ + "ĠA", + "re" + ], + [ + "Ġres", + "pect" + ], + [ + "ç«ŀäºī", + "åĬĽ" + ], + [ + "æī", + "ĺ" + ], + [ + "äºĨ解", + "ä¸Ģä¸ĭ" + ], + [ + "读", + "åıĸ" + ], + [ + "以", + "æıIJé«ĺ" + ], + [ + "å°¤åħ¶", + "æĺ¯åľ¨" + ], + [ + "éĿ¢", + "çļĦ" + ], + [ + "读", + "èĢħ" + ], + [ + "ä¸ĵ注", + "äºİ" + ], + [ + "Ġtyp", + "es" + ], + [ + "2", + "9" + ], + [ + "A", + "re" + ], + [ + "使", + "åħ¶" + ], + [ + "am", + "b" + ], + [ + "Ġst", + "ill" + ], + [ + "å±ı", + "å¹ķ" + ], + [ + "çĤ", + "¼" + ], + [ + "æĽ´åĩĨç¡®", + "åľ°" + ], + [ + "Ġpot", + "ential" + ], + [ + "ç¡®ä¿Ŀ", + "æīĢæľī" + ], + [ + "W", + "e" + ], + [ + "åĮº", + "åĪ«" + ], + [ + "顺", + "åĪ©" + ], + [ + "åħ", + "Ń" + ], + [ + "线", + "ç¨ĭ" + ], + [ + "g", + "en" + ], + [ + "0", + "1" + ], + [ + "Ġhe", + "ight" + ], + [ + "Ġmo", + "on" + ], + [ + "vers", + "ed" + ], + [ + "èı", + "Į" + ], + [ + "ا", + "ÙĦ" + ], + [ + "]", + ")" + ], + [ + "vie", + "w" + ], + [ + "Ġit", + "er" + ], + [ + "Ġanswer", + "ing" + ], + [ + "l", + "ished" + ], + [ + "z", + "hou" + ], + [ + "æľŁ", + "æľĽ" + ], + [ + "l", + "ate" + ], + [ + "éĩįè¦ģ", + "çļĦæĺ¯" + ], + [ + "ç®Ģ", + "åĮĸ" + ], + [ + "k", + "es" + ], + [ + "åı¯ä»¥", + "æľīæķĪ" + ], + [ + "ä¸Ĭ", + "æīĢ" + ], + [ + "ĠThe", + "refore" + ], + [ + "L", + "E" + ], + [ + "çļĦæ¯", + "Ķ" + ], + [ + "èĩªåĬ¨", + "驾驶" + ], + [ + "Ġdatab", + "ase" + ], + [ + "ar", + "a" + ], + [ + "Ġoff", + "er" + ], + [ + "ist", + "ics" + ], + [ + "åı¯èĥ½ä¼ļ", + "导èĩ´" + ], + [ + "好", + "å¥ĩ" + ], + [ + "èĦ", + "±" + ], + [ + "çļĦç", + "Ī" + ], + [ + "èĤ", + "¥" + ], + [ + "Ġcontain", + "s" + ], + [ + "åºĶçĶ¨", + "ä¸Ń" + ], + [ + "Ġun", + "its" + ], + [ + "åĴĮ", + "建议" + ], + [ + "çļĦ", + "ç»ĵæŀľ" + ], + [ + "ĠJ", + "ack" + ], + [ + "æī", + "°" + ], + [ + "使çĶ¨", + "çļĦ" + ], + [ + "æĮĩ", + "æłĩ" + ], + [ + "åĨį", + "次" + ], + [ + "Ġs", + "ong" + ], + [ + "ro", + "p" + ], + [ + "Ġrel", + "i" + ], + [ + "ren", + "ch" + ], + [ + "Ġ", + "ess" + ], + [ + "æݨ", + "广" + ], + [ + "åĴĮ", + "æĸĩåĮĸ" + ], + [ + "çĤ", + "Ĵ" + ], + [ + "å¡ij", + "æĸĻ" + ], + [ + "A", + "N" + ], + [ + "m", + "ax" + ], + [ + "çļĦæ", + "¶" + ], + [ + "度", + "çļĦ" + ], + [ + "å®ĥ", + "ä¸įä»ħ" + ], + [ + "Ġnum", + "er" + ], + [ + "E", + "ach" + ], + [ + "Ġs", + "elf" + ], + [ + "éķ¿", + "æŶéĹ´" + ], + [ + "ä»Ĭ", + "天" + ], + [ + "çĪ", + "¶" + ], + [ + "çļĦ", + "帮åĬ©" + ], + [ + "Ġst", + "at" + ], + [ + "and", + "as" + ], + [ + "Ġsy", + "stems" + ], + [ + "Ġadd", + "ress" + ], + [ + "Ġan", + "ything" + ], + [ + "n", + "ces" + ], + [ + "å¹", + "ħ" + ], + [ + "æī¾", + "åĩº" + ], + [ + "Ġnum", + "py" + ], + [ + "Ġf", + "low" + ], + [ + "ç§", + "Ł" + ], + [ + "åĶ", + "¯" + ], + [ + "éĹ®é¢ĺæĪĸ", + "éľĢè¦ģ" + ], + [ + "W", + "HERE" + ], + [ + "åĦ¿", + "ç«¥" + ], + [ + "Ġqu", + "ick" + ], + [ + "è¯į", + "æ±ĩ" + ], + [ + "6", + "4" + ], + [ + "am", + "ent" + ], + [ + "ĠA", + "r" + ], + [ + "ç»Ļ", + "å®ļçļĦ" + ], + [ + "ne", + "y" + ], + [ + "ä¸ĬæīĢ", + "è¿°" + ], + [ + "å¼", + "±" + ], + [ + "å¤ļæł·", + "æĢ§" + ], + [ + "为", + "çĶ¨æĪ·" + ], + [ + "Ġu", + "nt" + ], + [ + "Ġdescrib", + "e" + ], + [ + "çļĦ", + "åħī" + ], + [ + "åĬł", + "è½½" + ], + [ + "a", + "res" + ], + [ + "çļĦ", + "éķ¿åº¦" + ], + [ + "åĪĨ", + "解" + ], + [ + "èİ", + "ī" + ], + [ + "æĽ´å¥½åľ°", + "çIJĨ解" + ], + [ + "a", + "w" + ], + [ + "id", + "ed" + ], + [ + "è¿ĩ", + "滤" + ], + [ + "20", + "23" + ], + [ + "éĿ¢", + "è¯ķ" + ], + [ + "仿", + "ä½Ľ" + ], + [ + "综", + "ä¸ĬæīĢè¿°" + ], + [ + "认", + "è¯Ĩ" + ], + [ + "产çĶŁ", + "äºĨ" + ], + [ + "åĽº", + "å®ļ" + ], + [ + "çļĦæ´»", + "åĬ¨" + ], + [ + "C", + "D" + ], + [ + "Ġp", + "ie" + ], + [ + "æķ°æį®", + "å®īåħ¨" + ], + [ + "æĦŁ", + "åıĹåĪ°" + ], + [ + "èµĦ", + "æĸĻ" + ], + [ + "an", + "ation" + ], + [ + "Ġ+", + "=" + ], + [ + "iz", + "ing" + ], + [ + "Ġlet", + "ter" + ], + [ + "e", + "ad" + ], + [ + "èª", + "ī" + ], + [ + "b", + "ash" + ], + [ + "Ġf", + "our" + ], + [ + "Ġal", + "gorithm" + ], + [ + "U", + "n" + ], + [ + "ä¸Ģ", + "份" + ], + [ + "æĢ", + "Ģ" + ], + [ + "St", + "ring" + ], + [ + "or", + "k" + ], + [ + "ar", + "ily" + ], + [ + "é", + "¹" + ], + [ + "æĪij们", + "éľĢè¦ģ" + ], + [ + "æ¢", + "¯" + ], + [ + "åĬŀ", + "åħ¬" + ], + [ + "or", + "m" + ], + [ + "ä½ł", + "们" + ], + [ + "æ¯", + "Ľ" + ], + [ + "b", + "an" + ], + [ + "w", + "w" + ], + [ + "Ġsor", + "ry" + ], + [ + "æĿ¥", + "è¶Ĭ" + ], + [ + "ad", + "r" + ], + [ + "Ġa", + "im" + ], + [ + "h", + "ang" + ], + [ + "h", + "ai" + ], + [ + "帮åĬ©", + "çĶ¨æĪ·" + ], + [ + "åij¨", + "åĽ´" + ], + [ + "è¶Ĭ", + "æĿ¥è¶Ĭ" + ], + [ + "ç»", + "ķ" + ], + [ + "éĢļ常", + "éľĢè¦ģ" + ], + [ + "Ċ", + "ĊĠĠ" + ], + [ + "åħĪ", + "è¿Ľ" + ], + [ + "ĠOct", + "ober" + ], + [ + "g", + "o" + ], + [ + "ro", + "du" + ], + [ + "éĴ", + "±" + ], + [ + "Ù", + "Ĩ" + ], + [ + "请", + "éĹ®" + ], + [ + "Ġex", + "t" + ], + [ + "Ġim", + "age" + ], + [ + "æ¶Ī", + "èĢĹ" + ], + [ + "çļĦæŃ", + "¥" + ], + [ + "Ġeng", + "aging" + ], + [ + "J", + "ohn" + ], + [ + "ç»ı", + "è¿ĩ" + ], + [ + "æIJ", + "º" + ], + [ + "å¾Ī", + "é«ĺåħ´" + ], + [ + "A", + "r" + ], + [ + "åı¯ä»¥", + "å°Ŀè¯ķ" + ], + [ + "çļĦç", + "Ĭ" + ], + [ + "Ġcolumn", + "s" + ], + [ + "çݯå¢ĥ", + "ä¿ĿæĬ¤" + ], + [ + "éĴ", + "¥" + ], + [ + "ä¸Ģ", + "ä½į" + ], + [ + "éĢļè¿ĩ", + "è¿ĻäºĽ" + ], + [ + "Ġsent", + "ences" + ], + [ + "å±Ĥ", + "次" + ], + [ + "an", + "ish" + ], + [ + "失", + "è´¥" + ], + [ + "ou", + "n" + ], + [ + "ï¼ļ", + "**" + ], + [ + "Ġwe", + "ek" + ], + [ + "in", + "ary" + ], + [ + "Ġcre", + "ative" + ], + [ + "åĩº", + "äºĨ" + ], + [ + "ver", + "n" + ], + [ + "转", + "åŀĭ" + ], + [ + "7", + "5" + ], + [ + "æįŁ", + "失" + ], + [ + "åĴĮ", + "使çĶ¨" + ], + [ + "Ġwrit", + "ten" + ], + [ + "Ġd", + "em" + ], + [ + "认", + "è¯ģ" + ], + [ + "ĠÐ", + "²" + ], + [ + "A", + "P" + ], + [ + "åĨħ", + "ç½®" + ], + [ + "Ġwor", + "ks" + ], + [ + "ç¾İ", + "好" + ], + [ + "çļĦ", + "çĶ¨æĪ·" + ], + [ + "å¾®", + "ä¿¡" + ], + [ + "å®", + "£" + ], + [ + "Ġiss", + "ues" + ], + [ + ":", + "\"," + ], + [ + "Ġal", + "ong" + ], + [ + "av", + "or" + ], + [ + "a", + "king" + ], + [ + "Ġprint", + "s" + ], + [ + "Ġstart", + "ed" + ], + [ + "Ġkil", + "om" + ], + [ + "æĪIJ", + "åĪĨ" + ], + [ + "被", + "称为" + ], + [ + "驱", + "åĬ¨" + ], + [ + "R", + "L" + ], + [ + "ä¸į", + "足" + ], + [ + "Ð", + "´" + ], + [ + "è", + "²" + ], + [ + "çļĦ", + "åı¯" + ], + [ + "Ġf", + "lo" + ], + [ + "æİĴ", + "æĶ¾" + ], + [ + "Ġbre", + "ak" + ], + [ + "A", + "n" + ], + [ + "Ġc", + "ru" + ], + [ + "ä¸ĭ", + "è½½" + ], + [ + "äºĭ", + "项" + ], + [ + "è¿", + "·" + ], + [ + "æŃ£", + "å¼ı" + ], + [ + "éĴ", + "®" + ], + [ + "Ġd", + "en" + ], + [ + "åĩı", + "è½»" + ], + [ + "å®", + "«" + ], + [ + "M", + "y" + ], + [ + "S", + "C" + ], + [ + "alcul", + "ate" + ], + [ + "ro", + "l" + ], + [ + "æıIJä¾Ľ", + "æĽ´åĬł" + ], + [ + "Ġmed", + "ia" + ], + [ + "ang", + "zhou" + ], + [ + "æĽ¿", + "æį¢" + ], + [ + "f", + "unction" + ], + [ + "er", + "o" + ], + [ + "è½", + "´" + ], + [ + "éª", + "¨" + ], + [ + "am", + "p" + ], + [ + "è°", + "·" + ], + [ + "ë", + "Ĭ" + ], + [ + "Ġem", + "ot" + ], + [ + "Ġnetwor", + "k" + ], + [ + "Ġb", + "ooks" + ], + [ + "åı¯ä»¥", + "åĩıå°ij" + ], + [ + "éĩĬ", + "æĶ¾" + ], + [ + "f", + "rom" + ], + [ + "ĠP", + "ar" + ], + [ + "èĴ", + "¸" + ], + [ + "Ġsumm", + "ary" + ], + [ + "now", + "n" + ], + [ + "åĴĮ", + "ç»´æĬ¤" + ], + [ + "æĹł", + "éĻIJ" + ], + [ + "åıĮ", + "æĸ¹" + ], + [ + "ä¸Ģ", + "èĪ¬" + ], + [ + "æĹ", + "¦" + ], + [ + "ç³»ç»Ł", + "ä¸Ń" + ], + [ + "Ġf", + "ull" + ], + [ + "Ġm", + "iss" + ], + [ + "Ġlet", + "ters" + ], + [ + "çĪ±", + "好" + ], + [ + "u", + "ation" + ], + [ + "çļĦæ", + "ĭ" + ], + [ + "åī", + "ª" + ], + [ + "åĽ¾", + "å½¢" + ], + [ + "æĪĺ", + "çķ¥" + ], + [ + "Ġachie", + "ve" + ], + [ + "C", + "l" + ], + [ + "åĴĮ", + "ä¼ĺåĮĸ" + ], + [ + "è¿ĩ", + "åİ»" + ], + [ + "éĢĤ", + "éĩı" + ], + [ + "ä¸įä»ħ", + "ä»ħ" + ], + [ + "du", + "le" + ], + [ + "以ä¸ĭæĺ¯ä¸ĢäºĽ", + "建议" + ], + [ + "åı", + "ī" + ], + [ + "é", + "½" + ], + [ + "Ġ**", + "ãĢĬ" + ], + [ + "çļĦæī", + "ĭ" + ], + [ + "幸", + "ç¦ı" + ], + [ + "çĶ", + "ľ" + ], + [ + "æĹ¥", + "å¿Ĺ" + ], + [ + "al", + "k" + ], + [ + "Ġle", + "g" + ], + [ + "ä¾ĭåŃIJ", + "ä¸Ń" + ], + [ + "Ġen", + "c" + ], + [ + "ç§ijåѦ", + "家" + ], + [ + "欢è¿İ", + "éļıæŶ" + ], + [ + "çīĪ", + "æĿĥ" + ], + [ + "ĊĠĠĠĠ", + "ĊĠĠĠ" + ], + [ + "S", + "p" + ], + [ + "Ġinit", + "ial" + ], + [ + "Ġs", + "ure" + ], + [ + "Ġf", + "ac" + ], + [ + "åĩ", + "Ŀ" + ], + [ + "å¿", + "½" + ], + [ + "Ġtransl", + "ation" + ], + [ + "Ġfam", + "ily" + ], + [ + "便", + "æį·" + ], + [ + "Ġconvers", + "ations" + ], + [ + "æļ", + "´" + ], + [ + "ĠG", + "re" + ], + [ + "r", + "ation" + ], + [ + "¹", + "é¥" + ], + [ + "è¿Ļ", + "两个" + ], + [ + "ä¼¼", + "ä¹İ" + ], + [ + "æµ", + "®" + ], + [ + "¹é¥", + "ª" + ], + [ + "åĽŀ", + "æĶ¶" + ], + [ + "N", + "LP" + ], + [ + "æĭ", + "¼" + ], + [ + "Ġd", + "r" + ], + [ + "Ġexper", + "ien" + ], + [ + "æĵ", + "İ" + ], + [ + "m", + "ed" + ], + [ + "Ġprof", + "ess" + ], + [ + "F", + "irst" + ], + [ + "Ġinf", + "lu" + ], + [ + "å¯", + "¸" + ], + [ + "ä¸į", + "ç¡®å®ļ" + ], + [ + "I", + "nt" + ], + [ + "ä¼ł", + "æĦŁ" + ], + [ + "Ġfunction", + "s" + ], + [ + "å£", + "¤" + ], + [ + "åĽ¢éĺŁ", + "åIJĪä½ľ" + ], + [ + "以ä¸ĭåĩł", + "ç§į" + ], + [ + "l", + "ike" + ], + [ + "åĬ¨", + "ä½ľ" + ], + [ + "é£ŀ", + "è¡Į" + ], + [ + "Ġd", + "ream" + ], + [ + "èĢ", + "³" + ], + [ + "å¼ķ", + "æĵİ" + ], + [ + "Ġp", + "a" + ], + [ + "å¤ļ", + "æĸ¹éĿ¢" + ], + [ + "产", + "ä¸ļ" + ], + [ + "W", + "here" + ], + [ + "Ġa", + "uth" + ], + [ + "åįĩ", + "级" + ], + [ + "æ´", + "¾" + ], + [ + "Ġgre", + "en" + ], + [ + "ç¾İ", + "åij³" + ], + [ + "os", + "p" + ], + [ + "æŶ", + "åħī" + ], + [ + "end", + "s" + ], + [ + "Ġcult", + "ural" + ], + [ + "Ġkilom", + "eters" + ], + [ + "ãĤ", + "Į" + ], + [ + "æ¥", + "¼" + ], + [ + "e", + "ch" + ], + [ + "æ", + "¾" + ], + [ + "èĩª", + "身" + ], + [ + "åŃIJ", + "çļĦ" + ], + [ + "åī", + "©" + ], + [ + "ff", + "ic" + ], + [ + "7", + "0" + ], + [ + "Ġf", + "ra" + ], + [ + "Ġqu", + "e" + ], + [ + "产åĵģ", + "çļĦ" + ], + [ + "ëĭ", + "¤" + ], + [ + "Ġst", + "re" + ], + [ + ")", + "ï¼Į" + ], + [ + "S", + "ub" + ], + [ + "çĽij", + "çĿ£" + ], + [ + "it", + "ing" + ], + [ + "ä½ĵ", + "èĤ²" + ], + [ + "è¿Ļ段", + "代çłģ" + ], + [ + "è´·", + "款" + ], + [ + "家", + "å±ħ" + ], + [ + "Ġsh", + "ow" + ], + [ + "AT", + "E" + ], + [ + "iction", + "ary" + ], + [ + "çĶ", + "·" + ], + [ + "im", + "plify" + ], + [ + "æµģ", + "éĩı" + ], + [ + "ç¾İ", + "丽" + ], + [ + "Ġsc", + "ience" + ], + [ + "Ġoper", + "ations" + ], + [ + "群", + "ä½ĵ" + ], + [ + "Ġmin", + "utes" + ], + [ + "i", + "um" + ], + [ + "æŶéĹ´", + "åĴĮ" + ], + [ + "Ġmain", + "tain" + ], + [ + "erv", + "ice" + ], + [ + "ä¼Ļ", + "ä¼´" + ], + [ + "2", + "8" + ], + [ + "ä½ľ", + "èĢħ" + ], + [ + "Ġmo", + "vie" + ], + [ + "{", + "\\" + ], + [ + "Ġdiffere", + "nce" + ], + [ + "è´Ł", + "éĿ¢" + ], + [ + "Ġ", + "éĢīæĭ©" + ], + [ + "le", + "ft" + ], + [ + "éĽ", + "ħ" + ], + [ + "çĶµ", + "è¯Ŀ" + ], + [ + "Ġcons", + "ider" + ], + [ + "æİ¥", + "触" + ], + [ + "c", + "el" + ], + [ + "Wh", + "y" + ], + [ + "ĠF", + "rench" + ], + [ + "Ġpre", + "c" + ], + [ + "Ġlead", + "ership" + ], + [ + "为", + "æĤ¨" + ], + [ + "åĪĹ", + "åĩº" + ], + [ + "R", + "E" + ], + [ + "ç½ij", + "页" + ], + [ + "W", + "indows" + ], + [ + "ä¿Ŀ", + "çķĻ" + ], + [ + "æĮī", + "éĴ®" + ], + [ + "çļĦæ¦Ĥ", + "çİĩ" + ], + [ + "åı£", + "åij³" + ], + [ + "Ġmat", + "rix" + ], + [ + "How", + "ever" + ], + [ + "ĠÐ", + "¿" + ], + [ + "èĮĥåĽ´", + "åĨħ" + ], + [ + "Ġhist", + "or" + ], + [ + "çĹĩ", + "çĬ¶" + ], + [ + "æŃ", + "»" + ], + [ + "çī©", + "ä½ĵ" + ], + [ + "æīĭ", + "段" + ], + [ + "ä¹IJ", + "æĦı" + ], + [ + "ä¼łæĦŁ", + "åĻ¨" + ], + [ + "æķ", + "¢" + ], + [ + "Ġma", + "jor" + ], + [ + "Ġloc", + "al" + ], + [ + "å¥", + "ij" + ], + [ + "Ġpro", + "te" + ], + [ + "æľ¬", + "书" + ], + [ + "ç»ı", + "常" + ], + [ + "åľŁ", + "壤" + ], + [ + "Ġsymb", + "ol" + ], + [ + "æľŁ", + "å¾ħ" + ], + [ + "ä¹IJ", + "趣" + ], + [ + "Ġcont", + "in" + ], + [ + "Ġit", + "ems" + ], + [ + "çı", + "į" + ], + [ + "å³", + "°" + ], + [ + "D", + "P" + ], + [ + "Ġp", + "red" + ], + [ + "Ġ20", + "22" + ], + [ + "æıĴ", + "åħ¥" + ], + [ + "l", + "st" + ], + [ + "Ġ2", + "1" + ], + [ + "åī", + "²" + ], + [ + "èµĦæºIJ", + "çļĦ" + ], + [ + "éĤ", + "Ģ" + ], + [ + "è¿Ļ", + "åı¥è¯Ŀ" + ], + [ + "è¾ĥ", + "é«ĺ" + ], + [ + "çļĦæ¸", + "¸" + ], + [ + "æ±", + "ģ" + ], + [ + "ill", + "s" + ], + [ + "æĬķèµĦ", + "èĢħ" + ], + [ + "åı¯èĥ½", + "导èĩ´" + ], + [ + "ç±»", + "åĪ«" + ], + [ + "éĿŀ常", + "éĩįè¦ģçļĦ" + ], + [ + "Ġpass", + "word" + ], + [ + "Ġear", + "ly" + ], + [ + "Ġha", + "ven" + ], + [ + "åĻ", + "ª" + ], + [ + "éģĵ", + "è·¯" + ], + [ + "Ġlear", + "n" + ], + [ + "æ¿Ģ", + "åĬ±" + ], + [ + "åĩº", + "è¡Į" + ], + [ + "e", + "e" + ], + [ + "ra", + "ft" + ], + [ + "pl", + "ot" + ], + [ + "åĴĮ", + "çIJĨ解" + ], + [ + "Ġele", + "ct" + ], + [ + "n", + "ect" + ], + [ + "Ġfig", + "ure" + ], + [ + "çļĦæĶ¯", + "æĮģ" + ], + [ + "Ġspec", + "ify" + ], + [ + "ate", + "g" + ], + [ + "大", + "èĦij" + ], + [ + "ĠH", + "angzhou" + ], + [ + "æĢ»", + "æĺ¯" + ], + [ + "Ġmod", + "ern" + ], + [ + "ëĬ", + "Ķ" + ], + [ + "Ġ\\)", + ":" + ], + [ + "Ġcru", + "cial" + ], + [ + "æľĢ", + "æĸ°" + ], + [ + "m", + "p" + ], + [ + "解", + "çŃĶ" + ], + [ + "ir", + "d" + ], + [ + "ĠĠĠĠ", + "ĠĠĠĠ" + ], + [ + "åı¯ä»¥", + "被" + ], + [ + "æŁIJ", + "ç§į" + ], + [ + "ç³", + "ķ" + ], + [ + "Ġconst", + "ant" + ], + [ + "4", + "8" + ], + [ + "A", + "C" + ], + [ + "éľĢè¦ģ", + "èĢĥèĻij" + ], + [ + "Ġex", + "ec" + ], + [ + "æľŁ", + "éĹ´" + ], + [ + "ä¼ĺ", + "æĥł" + ], + [ + "Ġe", + "t" + ], + [ + "缸", + "ä¿¡" + ], + [ + "ĠE", + "l" + ], + [ + "Ġper", + "iod" + ], + [ + "Ġse", + "cre" + ], + [ + "æĪijæĥ³", + "äºĨ解ä¸Ģä¸ĭ" + ], + [ + "带æĿ¥", + "äºĨ" + ], + [ + "Ġapplic", + "ation" + ], + [ + "çļĦ", + "åĩĨç¡®" + ], + [ + "åįķ", + "åħĥ" + ], + [ + "èĢĥ", + "è¯ķ" + ], + [ + "ĠU", + "S" + ], + [ + "åĪĽ", + "ä¸ļ" + ], + [ + "æĦŁ", + "åĴĮ" + ], + [ + "询", + "éĹ®" + ], + [ + "éĽĨ", + "ä¸Ń" + ], + [ + "ap", + "an" + ], + [ + "P", + "r" + ], + [ + "æĹ", + "§" + ], + [ + "è´", + "Ŀ" + ], + [ + "à¸", + "²" + ], + [ + "æıIJ", + "éĨĴ" + ], + [ + "è°ĥ", + "èĬĤ" + ], + [ + "åı¤", + "èĢģ" + ], + [ + "Ġrecogn", + "ized" + ], + [ + "稳å®ļ", + "æĢ§" + ], + [ + "åıij", + "æĮ¥" + ], + [ + "I", + "T" + ], + [ + "å", + "ĺ" + ], + [ + "åľ¨", + "使çĶ¨" + ], + [ + "åıijå±ķ", + "çļĦ" + ], + [ + "æĭ", + "Į" + ], + [ + "çĿ", + "Ľ" + ], + [ + "Ġch", + "ild" + ], + [ + "ï¼Į", + "\"" + ], + [ + "çļĦæ", + "Ľ" + ], + [ + "å¿«", + "ä¹IJ" + ], + [ + "Ġag", + "ain" + ], + [ + "Ġf", + "requ" + ], + [ + "çļĦæ", + "ĵ" + ], + [ + "为", + "ä½ł" + ], + [ + "审", + "æŁ¥" + ], + [ + "ĠI", + "s" + ], + [ + "åIJİ", + "ç»Ń" + ], + [ + "Ġtra", + "ditional" + ], + [ + "Ġc", + "rit" + ], + [ + "ä¸Ģ", + "æŦ" + ], + [ + "èĥ½åĬĽ", + "åĴĮ" + ], + [ + "he", + "ad" + ], + [ + "å¿ĥçIJĨ", + "åģ¥åº·" + ], + [ + "Ġex", + "ist" + ], + [ + "Ġsu", + "ccess" + ], + [ + "强", + "大" + ], + [ + "æĿĥ", + "çĽĬ" + ], + [ + "è¾¹", + "ç¼ĺ" + ], + [ + "at", + "ch" + ], + [ + "ol", + "l" + ], + [ + "åĶ¯", + "ä¸Ģ" + ], + [ + "çļĦçĬ", + "¶" + ], + [ + "Ġn", + "on" + ], + [ + "éĢļè¿ĩ", + "åĪĨæŀIJ" + ], + [ + "æ°´", + "æŀľ" + ], + [ + "çļĦæĮĩ", + "导" + ], + [ + "G", + "oogle" + ], + [ + "çļĦéĩįè¦ģ", + "æĢ§" + ], + [ + "Ġb", + "en" + ], + [ + "s", + "in" + ], + [ + "å¤", + "«" + ], + [ + "çĥ", + "Ł" + ], + [ + "æıIJ", + "交" + ], + [ + "æĽ´", + "æľīæķĪåľ°" + ], + [ + "æĹł", + "éľĢ" + ], + [ + "Ġse", + "arch" + ], + [ + "çIJĨ", + "念" + ], + [ + "Ġqu", + "adr" + ], + [ + "èģĮ", + "ä½į" + ], + [ + "Ġsumm", + "ar" + ], + [ + "Ġchall", + "eng" + ], + [ + "å°±", + "ä¸ļ" + ], + [ + "c", + "m" + ], + [ + "æĦŁ", + "æŁĵ" + ], + [ + "åij", + "¢" + ], + [ + "Ġderiv", + "ative" + ], + [ + "èµ", + "ı" + ], + [ + "Ġcon", + "sole" + ], + [ + "çĶµ", + "åķĨ" + ], + [ + "ç¥ŀ", + "ç§ĺ" + ], + [ + "Ġdist", + "ance" + ], + [ + "ĠSp", + "anish" + ], + [ + "Ġeffic", + "ient" + ], + [ + "ç½", + "ª" + ], + [ + "åĢ¼", + "å¾Ĺ" + ], + [ + "åĽ¢éĺŁ", + "çļĦ" + ], + [ + "æĶ¿", + "æ²»" + ], + [ + "hel", + "lo" + ], + [ + "a", + "ction" + ], + [ + "le", + "x" + ], + [ + "oo", + "se" + ], + [ + "Ġfound", + "er" + ], + [ + "大", + "äºİ" + ], + [ + "Ġstud", + "ents" + ], + [ + "çĦ¦", + "èĻij" + ], + [ + "ä½Ĩ", + "åľ¨" + ], + [ + "Ġcustom", + "er" + ], + [ + ":", + "//" + ], + [ + "A", + "M" + ], + [ + "å§", + "ĵ" + ], + [ + "æ°ĶåĢĻ", + "åıĺåĮĸ" + ], + [ + "Ġeconom", + "ic" + ], + [ + "s", + "ion" + ], + [ + "ä¸Ģ", + "å®ļçļĦ" + ], + [ + "éħ", + "±" + ], + [ + "Ġg", + "en" + ], + [ + "Ġd", + "ance" + ], + [ + "èĦ", + "ī" + ], + [ + "表", + "æĺİ" + ], + [ + "èĩªçĦ¶", + "çļĦ" + ], + [ + "Ġcont", + "inu" + ], + [ + "à", + "¦" + ], + [ + "ener", + "ate" + ], + [ + "éĤĢ", + "请" + ], + [ + "Ġqu", + "ant" + ], + [ + "åķĨ", + "åĬ¡" + ], + [ + "认", + "çŁ¥" + ], + [ + "Ġke", + "ep" + ], + [ + "en", + "n" + ], + [ + "éĺ¿éĩĮäºij", + "çļĦ" + ], + [ + "Ġindividual", + "s" + ], + [ + "Ġc", + "ertain" + ], + [ + "ad", + "d" + ], + [ + "Ġsu", + "st" + ], + [ + "end", + "er" + ], + [ + "Ġ", + "æµĭè¯ķ" + ], + [ + "è¿", + "Ł" + ], + [ + "åī", + "¯" + ], + [ + "us", + "ion" + ], + [ + "so", + "ft" + ], + [ + "Ġlim", + "it" + ], + [ + "Ñ", + "ĩ" + ], + [ + "è¿", + "ģ" + ], + [ + "æĿ¥", + "计ç®Ĺ" + ], + [ + "ight", + "s" + ], + [ + "æ³", + "³" + ], + [ + "çļĦæĻº", + "èĥ½" + ], + [ + "çļĦæĬķ", + "èµĦ" + ], + [ + "N", + "o" + ], + [ + "帮åĬ©", + "ä¼ģä¸ļ" + ], + [ + "Ġresult", + "s" + ], + [ + "æ", + "ĩ" + ], + [ + "æķ°æį®", + "å¤ĦçIJĨ" + ], + [ + "ĠP", + "er" + ], + [ + "Ġfam", + "ous" + ], + [ + "åħ¬", + "åĽŃ" + ], + [ + "çļĦæ", + "Ł" + ], + [ + "åħ³", + "éĹŃ" + ], + [ + "请", + "åijĬè¯īæĪij" + ], + [ + "os", + "s" + ], + [ + "Ġ<", + "=" + ], + [ + "çł´", + "åĿı" + ], + [ + "çĶŁ", + "ç´ł" + ], + [ + "æĪij们", + "使çĶ¨" + ], + [ + "w", + "here" + ], + [ + "ä¸Ģ", + "天" + ], + [ + "æİ¥", + "è¿ij" + ], + [ + "EC", + "S" + ], + [ + "c", + "ient" + ], + [ + "ĠS", + "he" + ], + [ + "f", + "ile" + ], + [ + "olog", + "y" + ], + [ + "ç¼ĸåĨĻ", + "ä¸Ģ个" + ], + [ + "}", + "}{" + ], + [ + "åı¯èĥ½", + "åľ¨" + ], + [ + "Ñģ", + "ÑĤ" + ], + [ + "Ġconvers", + "ation" + ], + [ + "Ġmus", + "ic" + ], + [ + "çļĦ", + "åİĨåı²" + ], + [ + "缸åºĶ", + "çļĦ" + ], + [ + "è¿Ļ", + "åĮħæĭ¬" + ], + [ + "æŃ", + "¦" + ], + [ + "åħ³éĶ®", + "è¯į" + ], + [ + "Ġtra", + "vel" + ], + [ + "Ġsust", + "ain" + ], + [ + "æµıè§Ī", + "åĻ¨" + ], + [ + "\"", + "\"" + ], + [ + "çļĦç", + "¥" + ], + [ + "Ġdes", + "cript" + ], + [ + "Ġproduct", + "s" + ], + [ + "f", + "ect" + ], + [ + "Ġab", + "ove" + ], + [ + "Ġsub", + "ject" + ], + [ + "Ġre", + "ce" + ], + [ + "æĪIJ", + "çĨŁ" + ], + [ + "è¾¾", + "æij©" + ], + [ + "è´¦", + "åı·" + ], + [ + "L", + "L" + ], + [ + "Ġh", + "o" + ], + [ + "èį", + "·" + ], + [ + "åĽłä¸º", + "å®ĥ" + ], + [ + "in", + "a" + ], + [ + "ç´§", + "å¼ł" + ], + [ + "è°", + "ľ" + ], + [ + "æĶ¾", + "åľ¨" + ], + [ + "Ġs", + "at" + ], + [ + "Ġcan", + "not" + ], + [ + "èĤĮ", + "èĤī" + ], + [ + "a", + "us" + ], + [ + "æıIJ", + "示" + ], + [ + "åľ¨è¿Ļ个", + "ä¾ĭåŃIJä¸Ń" + ], + [ + "Ġc", + "y" + ], + [ + "Ġ3", + "6" + ], + [ + "ad", + "em" + ], + [ + "èıľ", + "åįķ" + ], + [ + "çļĦæĵ", + "įä½ľ" + ], + [ + "cl", + "ass" + ], + [ + "è¾ĥ", + "ä½İ" + ], + [ + "g", + "ative" + ], + [ + "p", + "o" + ], + [ + "å¼", + "ĥ" + ], + [ + "Ù", + "Ī" + ], + [ + "ç", + "Ĵ" + ], + [ + "re", + "l" + ], + [ + "ent", + "ly" + ], + [ + "æ¤", + "Ĵ" + ], + [ + "涵", + "çĽĸ" + ], + [ + "an", + "n" + ], + [ + "便", + "åĪ©" + ], + [ + "çļĦæŃ¥", + "骤" + ], + [ + "6", + "7" + ], + [ + "Ġ", + "." + ], + [ + "åĢ", + "Ł" + ], + [ + "çļĦ", + "çݯå¢ĥ" + ], + [ + "åĽŀ", + "å½Ĵ" + ], + [ + "Ġcol", + "ors" + ], + [ + "Ġfriend", + "s" + ], + [ + "ht", + "ml" + ], + [ + "æĭ", + "ĵ" + ], + [ + "éĿŀ常", + "æľī" + ], + [ + "æĪĸ", + "æľįåĬ¡" + ], + [ + "è", + "©" + ], + [ + "ä¹", + "ı" + ], + [ + "åľ¨", + "ä¸įåIJĮ" + ], + [ + "审", + "计" + ], + [ + "1", + "23" + ], + [ + "åĵª", + "个" + ], + [ + "å²", + "Ľ" + ], + [ + "Ġcomple", + "te" + ], + [ + "Ġprocess", + "ing" + ], + [ + "ĠSh", + "ang" + ], + [ + "Ġcomprehen", + "sive" + ], + [ + "ou", + "se" + ], + [ + "å½", + "¼" + ], + [ + "åIJĪ", + "约" + ], + [ + "æ¡", + "Į" + ], + [ + "æĮģç»Ń", + "åŃ¦ä¹ł" + ], + [ + "Ġind", + "ic" + ], + [ + "Ġbu", + "ild" + ], + [ + "HT", + "ML" + ], + [ + "æ³Ħ", + "éľ²" + ], + [ + "Ġcom", + "b" + ], + [ + "ç", + "ĺ" + ], + [ + "çĶ¨", + "æĢ§" + ], + [ + "å·²", + "çŁ¥" + ], + [ + "ĠY", + "ork" + ], + [ + "æĽ´", + "容æĺĵ" + ], + [ + "符", + "åı·" + ], + [ + "æ¶Ī", + "æģ¯" + ], + [ + "Ġw", + "arm" + ], + [ + "çº", + "¯" + ], + [ + "åĮ»", + "åѦ" + ], + [ + "å¤Ħ", + "äºİ" + ], + [ + "d", + "a" + ], + [ + "Ġf", + "all" + ], + [ + "Ġl", + "iter" + ], + [ + "ĠP", + "l" + ], + [ + "Ġc", + "ur" + ], + [ + "çļĦ", + "设计" + ], + [ + "ĠO", + "r" + ], + [ + "ç»ĦæĪIJ", + "éĥ¨åĪĨ" + ], + [ + "ç¼ĵ", + "解" + ], + [ + "Ġprim", + "arily" + ], + [ + "åĨħ", + "å¿ĥ" + ], + [ + "çIJĨ", + "æĥ³" + ], + [ + "åĩº", + "çļĦ" + ], + [ + "æ±", + "¤" + ], + [ + "表", + "æł¼" + ], + [ + "erm", + "an" + ], + [ + "ht", + "t" + ], + [ + "è¿Ļ", + "使å¾Ĺ" + ], + [ + "æ¡Ī", + "ä¾ĭ" + ], + [ + "å©", + "ļ" + ], + [ + "éľĢè¦ģ注æĦı", + "çļĦæĺ¯" + ], + [ + "Ġint", + "rodu" + ], + [ + "åİ»", + "éĻ¤" + ], + [ + "Ġcompan", + "ies" + ], + [ + "d", + "own" + ], + [ + "Ġex", + "change" + ], + [ + "åĴĮ", + "åıijå±ķ" + ], + [ + "å¾Ī", + "ä¹IJæĦı" + ], + [ + "é¸", + "£" + ], + [ + "Ġsubs", + "idi" + ], + [ + "æĽ´", + "é«ĺæķĪ" + ], + [ + "个人", + "ä¿¡æģ¯" + ], + [ + "Python", + "代çłģ" + ], + [ + "ä¸ĭéĿ¢", + "æĺ¯" + ], + [ + "g", + "ing" + ], + [ + "æĮĩ", + "å®ļ" + ], + [ + "åį«", + "çĶŁ" + ], + [ + "æĻ", + "¨" + ], + [ + "çļĦç", + "Ŀ" + ], + [ + "åºĶçĶ¨", + "åľºæĻ¯" + ], + [ + "i", + "or" + ], + [ + "æī", + "«" + ], + [ + "çĶ¨æĪ·", + "åı¯ä»¥" + ], + [ + "ĠG", + "oogle" + ], + [ + "o", + "in" + ], + [ + "ä¸į", + "éľĢè¦ģ" + ], + [ + "æıIJä¾Ľ", + "帮åĬ©" + ], + [ + "es", + "p" + ], + [ + "æıIJä¾Ľ", + "çļĦ" + ], + [ + "ç»Ļ", + "äºĪ" + ], + [ + "ĠZ", + "hang" + ], + [ + "ëĭ", + "Ī" + ], + [ + "åı¯èĥ½", + "æľī" + ], + [ + "Ġst", + "yle" ] ] } diff --git a/model/tokenizer_config.json b/model/tokenizer_config.json index fc4e726..e46c14a 100644 --- a/model/tokenizer_config.json +++ b/model/tokenizer_config.json @@ -1,43 +1,335 @@ { - "add_bos_token": false, - "add_eos_token": false, - "add_prefix_space": false, - "added_tokens_decoder": { - "0": { - "content": "<|endoftext|>", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true + "add_bos_token": false, + "add_eos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "0": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "5": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "6": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "7": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "8": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "9": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "10": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "11": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "12": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "13": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "14": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "15": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "16": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "<|buffer1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "<|buffer2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "<|buffer3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "<|buffer4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "<|buffer5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "<|buffer6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "<|buffer7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "<|buffer8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "<|buffer9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } }, - "1": { - "content": "<|im_start|>", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - }, - "2": { - "content": "<|im_end|>", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - } - }, - "additional_special_tokens": [], - "bos_token": "<|im_start|>", - "clean_up_tokenization_spaces": false, - "eos_token": "<|im_end|>", - "legacy": true, - "model_max_length": 32768, - "pad_token": "<|endoftext|>", - "sp_model_kwargs": {}, - "spaces_between_special_tokens": false, - "tokenizer_class": "PreTrainedTokenizerFast", - "unk_token": "<|endoftext|>", - "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' -%}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- else -%}\n {{- '<|im_start|>system\\nYou are a helpful assistant<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '\\n\\n\\n\\n' }}\n {%- endif %}\n{%- endif %}" + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>", + "<|audio_start|>", + "<|audio_end|>", + "<|audio_pad|>", + "", + "", + "", + "" + ], + "bos_token": "<|im_start|>", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "legacy": true, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "unk_token": "<|endoftext|>", + "image_token": "<|image_pad|>", + "audio_token": "<|audio_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>", + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '' in content %}\n {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n {%- set content = content.split('')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if true %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content.strip('\\n') + '\\n\\n\\n' + content.lstrip('\\n') }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if open_thinking is defined and open_thinking is true %}\n {{- '\\n' }}\n {%- else %}\n {{- '\\n\\n\\n\\n' }}\n {%- endif %}\n{%- endif %}", + "tokenizer_class": "PreTrainedTokenizerFast" } \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4688970..4e0b405 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,12 +5,12 @@ Flask_Cors==4.0.0 jieba==0.42.1 jsonlines==4.0.0 marshmallow==3.22.0 -matplotlib==3.10.0 +# matplotlib==3.10.0 ngrok==1.4.0 nltk==3.8 numpy==1.26.4 openai==1.59.6 -peft==0.7.1 +# peft==0.7.1 psutil==5.9.8 pydantic==2.11.5 rich==13.7.1 @@ -18,7 +18,7 @@ scikit_learn==1.5.1 sentence_transformers==2.3.1 simhash==2.1.2 tiktoken==0.10.0 -transformers==4.57.1 +transformers==4.57.6 jinja2==3.1.2 jsonlines==4.0.0 trl==0.13.0 @@ -26,6 +26,7 @@ ujson==5.1.0 wandb==0.18.3 streamlit==1.50.0 einops==0.8.1 -swanlab==0.6.8 -torch==2.6.0 -torchvision==0.21.0 \ No newline at end of file +swanlab==0.7.11 +modelscope==1.30.0 +# torch==2.6.0 +# torchvision==0.21.0 \ No newline at end of file diff --git a/scripts/chat_openai_api.py b/scripts/chat_api.py similarity index 55% rename from scripts/chat_openai_api.py rename to scripts/chat_api.py index d677835..3828453 100644 --- a/scripts/chat_openai_api.py +++ b/scripts/chat_api.py @@ -1,8 +1,8 @@ from openai import OpenAI client = OpenAI( - api_key="ollama", - base_url="http://127.0.0.1:8998/v1" + api_key="sk-123", + base_url="http://localhost:11434/v1" ) stream = True conversation_history_origin = [] @@ -12,22 +12,29 @@ while True: query = input('[Q]: ') conversation_history.append({"role": "user", "content": query}) response = client.chat.completions.create( - model="minimind", + model="minimind-local:latest", messages=conversation_history[-(history_messages_num or 1):], stream=stream, - temperature=0.7, + temperature=0.8, max_tokens=2048, - top_p=0.9 + top_p=0.8, + extra_body={"chat_template_kwargs": {"open_thinking": True}, "reasoning_effort": "medium"} # 思考开关 ) if not stream: assistant_res = response.choices[0].message.content print('[A]: ', assistant_res) else: - print('[A]: ', end='') + print('[A]: ', end='', flush=True) assistant_res = '' for chunk in response: - print(chunk.choices[0].delta.content or "", end="") - assistant_res += chunk.choices[0].delta.content or "" + delta = chunk.choices[0].delta + r = getattr(delta, 'reasoning_content', None) or "" + c = delta.content or "" + if r: + print(f'\033[90m{r}\033[0m', end="", flush=True) + if c: + print(c, end="", flush=True) + assistant_res += c conversation_history.append({"role": "assistant", "content": assistant_res}) - print('\n\n') + print('\n\n') \ No newline at end of file diff --git a/scripts/convert_model.py b/scripts/convert_model.py index 01b1d51..34099e5 100644 --- a/scripts/convert_model.py +++ b/scripts/convert_model.py @@ -5,14 +5,14 @@ import json __package__ = "scripts" sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) import torch +import transformers import warnings -from transformers import AutoTokenizer, AutoModelForCausalLM, LlamaConfig, LlamaForCausalLM +from transformers import AutoTokenizer, AutoModelForCausalLM, Qwen3Config, Qwen3ForCausalLM, Qwen3MoeConfig, Qwen3MoeForCausalLM from model.model_minimind import MiniMindConfig, MiniMindForCausalLM +from model.model_lora import apply_lora, merge_lora warnings.filterwarnings('ignore', category=UserWarning) - -# MoE模型需使用此函数转换 def convert_torch2transformers_minimind(torch_path, transformers_path, dtype=torch.float16): MiniMindConfig.register_for_auto_class() MiniMindForCausalLM.register_for_auto_class("AutoModelForCausalLM") @@ -26,52 +26,118 @@ def convert_torch2transformers_minimind(torch_path, transformers_path, dtype=tor lm_model.save_pretrained(transformers_path, safe_serialization=False) tokenizer = AutoTokenizer.from_pretrained('../model/') tokenizer.save_pretrained(transformers_path) - # 兼容transformers-5.0的写法 - config_path = os.path.join(transformers_path, "tokenizer_config.json") - json.dump({**json.load(open(config_path, 'r', encoding='utf-8')), "tokenizer_class": "PreTrainedTokenizerFast", "extra_special_tokens": {}}, open(config_path, 'w', encoding='utf-8'), indent=2, ensure_ascii=False) + # ======= transformers-5.0的兼容低版本写法 ======= + if int(transformers.__version__.split('.')[0]) >= 5: + tokenizer_config_path, config_path = os.path.join(transformers_path, "tokenizer_config.json"), os.path.join(transformers_path, "config.json") + json.dump({**json.load(open(tokenizer_config_path, 'r', encoding='utf-8')), "tokenizer_class": "PreTrainedTokenizerFast", "extra_special_tokens": {}}, open(tokenizer_config_path, 'w', encoding='utf-8'), indent=2, ensure_ascii=False) + config = json.load(open(config_path, 'r', encoding='utf-8')) + config['rope_theta'] = lm_config.rope_theta; config['rope_scaling'] = None; del config['rope_parameters'] + json.dump(config, open(config_path, 'w', encoding='utf-8'), indent=2, ensure_ascii=False) print(f"模型已保存为 Transformers-MiniMind 格式: {transformers_path}") -# LlamaForCausalLM结构兼容第三方生态 -def convert_torch2transformers_llama(torch_path, transformers_path, dtype=torch.float16): +# QwenForCausalLM/LlamaForCausalLM结构兼容生态 +def convert_torch2transformers(torch_path, transformers_path, dtype=torch.float16): device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') state_dict = torch.load(torch_path, map_location=device) - llama_config = LlamaConfig( - vocab_size=lm_config.vocab_size, - hidden_size=lm_config.hidden_size, - intermediate_size=64 * ((int(lm_config.hidden_size * 8 / 3) + 64 - 1) // 64), - num_hidden_layers=lm_config.num_hidden_layers, - num_attention_heads=lm_config.num_attention_heads, - num_key_value_heads=lm_config.num_key_value_heads, - max_position_embeddings=lm_config.max_position_embeddings, - rms_norm_eps=lm_config.rms_norm_eps, - rope_theta=lm_config.rope_theta, - tie_word_embeddings=True - ) - llama_model = LlamaForCausalLM(llama_config) - llama_model.load_state_dict(state_dict, strict=False) - llama_model = llama_model.to(dtype) # 转换模型权重精度 - llama_model.save_pretrained(transformers_path) - model_params = sum(p.numel() for p in llama_model.parameters() if p.requires_grad) + common_config = { + "vocab_size": lm_config.vocab_size, + "hidden_size": lm_config.hidden_size, + "intermediate_size": lm_config.intermediate_size, + "num_hidden_layers": lm_config.num_hidden_layers, + "num_attention_heads": lm_config.num_attention_heads, + "num_key_value_heads": lm_config.num_key_value_heads, + "head_dim": lm_config.hidden_size // lm_config.num_attention_heads, + "max_position_embeddings": lm_config.max_position_embeddings, + "rms_norm_eps": lm_config.rms_norm_eps, + "rope_theta": lm_config.rope_theta, + "tie_word_embeddings": True + } + if not lm_config.use_moe: + qwen_config = Qwen3Config( + **common_config, + use_sliding_window=False, + sliding_window=None + ) + qwen_model = Qwen3ForCausalLM(qwen_config) + else: + qwen_config = Qwen3MoeConfig( + **common_config, + num_experts=lm_config.num_experts, + num_experts_per_tok=lm_config.num_experts_per_tok, + moe_intermediate_size=lm_config.moe_intermediate_size, + norm_topk_prob=lm_config.norm_topk_prob + ) + qwen_model = Qwen3MoeForCausalLM(qwen_config) + # ======= transformers-5.0的兼容低版本写法 ======= + if int(transformers.__version__.split('.')[0]) >= 5: + new_sd = {k: v for k, v in state_dict.items() if 'experts.' not in k or 'gate.weight' in k} + for l in range(lm_config.num_hidden_layers): + p = f'model.layers.{l}.mlp.experts' + new_sd[f'{p}.gate_up_proj'] = torch.cat([torch.stack([state_dict[f'{p}.{e}.gate_proj.weight'] for e in range(lm_config.num_experts)]), torch.stack([state_dict[f'{p}.{e}.up_proj.weight'] for e in range(lm_config.num_experts)])], dim=1) + new_sd[f'{p}.down_proj'] = torch.stack([state_dict[f'{p}.{e}.down_proj.weight'] for e in range(lm_config.num_experts)]) + state_dict = new_sd + + qwen_model.load_state_dict(state_dict, strict=True) + qwen_model = qwen_model.to(dtype) # 转换模型权重精度 + qwen_model.save_pretrained(transformers_path) + model_params = sum(p.numel() for p in qwen_model.parameters() if p.requires_grad) print(f'模型参数: {model_params / 1e6} 百万 = {model_params / 1e9} B (Billion)') tokenizer = AutoTokenizer.from_pretrained('../model/') tokenizer.save_pretrained(transformers_path) - # 兼容transformers-5.0的写法 - config_path = os.path.join(transformers_path, "tokenizer_config.json") - json.dump({**json.load(open(config_path, 'r', encoding='utf-8')), "tokenizer_class": "PreTrainedTokenizerFast", "extra_special_tokens": {}}, open(config_path, 'w', encoding='utf-8'), indent=2, ensure_ascii=False) - print(f"模型已保存为 Transformers-Llama 格式: {transformers_path}") + + # ======= transformers-5.0的兼容低版本写法 ======= + if int(transformers.__version__.split('.')[0]) >= 5: + tokenizer_config_path, config_path = os.path.join(transformers_path, "tokenizer_config.json"), os.path.join(transformers_path, "config.json") + json.dump({**json.load(open(tokenizer_config_path, 'r', encoding='utf-8')), "tokenizer_class": "PreTrainedTokenizerFast", "extra_special_tokens": {}}, open(tokenizer_config_path, 'w', encoding='utf-8'), indent=2, ensure_ascii=False) + config = json.load(open(config_path, 'r', encoding='utf-8')) + config['rope_theta'] = lm_config.rope_theta; config['rope_scaling'] = None; del config['rope_parameters'] + json.dump(config, open(config_path, 'w', encoding='utf-8'), indent=2, ensure_ascii=False) + print(f"模型已保存为 Transformers 格式: {transformers_path}") def convert_transformers2torch(transformers_path, torch_path): model = AutoModelForCausalLM.from_pretrained(transformers_path, trust_remote_code=True) torch.save({k: v.cpu().half() for k, v in model.state_dict().items()}, torch_path) - print(f"模型已保存为 PyTorch 格式 (half精度): {torch_path}") + print(f"模型已保存为 PyTorch 格式: {torch_path}") + + +def convert_merge_base_lora(base_torch_path, lora_path, merged_torch_path): + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + lm_model = MiniMindForCausalLM(lm_config).to(device) + state_dict = torch.load(base_torch_path, map_location=device) + lm_model.load_state_dict(state_dict, strict=False) + apply_lora(lm_model) + merge_lora(lm_model, lora_path, merged_torch_path) + print(f"LoRA 已合并并保存为基模结构 PyTorch 格式: {merged_torch_path}") + + +def convert_jinja_to_json(jinja_path): + with open(jinja_path, 'r') as f: template = f.read() + escaped = json.dumps(template) + print(f'"chat_template": {escaped}') + + +def convert_json_to_jinja(json_file_path, output_path): + with open(json_file_path, 'r') as f: config = json.load(f) + template = config['chat_template'] + with open(output_path, 'w') as f: f.write(template) + print(f"模板已保存为 jinja 文件: {output_path}") if __name__ == '__main__': - lm_config = MiniMindConfig(hidden_size=512, num_hidden_layers=8, max_seq_len=8192, use_moe=False) + lm_config = MiniMindConfig(hidden_size=768, num_hidden_layers=8, max_seq_len=8192, use_moe=True) + # convert torch to transformers torch_path = f"../out/full_sft_{lm_config.hidden_size}{'_moe' if lm_config.use_moe else ''}.pth" - transformers_path = '../MiniMind2-Small' - convert_torch2transformers_llama(torch_path, transformers_path) - # # convert transformers to torch model + transformers_path = '../minimind-3-moe' + convert_torch2transformers(torch_path, transformers_path) + + # # merge lora + # base_torch_path = f"../out/full_sft_{lm_config.hidden_size}{'_moe' if lm_config.use_moe else ''}.pth" + # lora_path = f"../out/lora_identity_{lm_config.hidden_size}.pth" + # merged_torch_path = f"../out/merge_identity_{lm_config.hidden_size}{'_moe' if lm_config.use_moe else ''}.pth" + # convert_merge_base_lora(base_torch_path, lora_path, merged_torch_path) + # convert_transformers2torch(transformers_path, torch_path) + # convert_json_to_jinja('../model/tokenizer_config.json', '../model/chat_template.jinja') + # convert_jinja_to_json('../model/chat_template.jinja') diff --git a/scripts/eval_toolcall.py b/scripts/eval_toolcall.py new file mode 100644 index 0000000..c637015 --- /dev/null +++ b/scripts/eval_toolcall.py @@ -0,0 +1,240 @@ +import os +import sys +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +import re +import json +import time +import random +import argparse +import warnings +import torch +from datetime import datetime +from transformers import AutoTokenizer, AutoModelForCausalLM, TextStreamer +from openai import OpenAI +from model.model_minimind import MiniMindConfig, MiniMindForCausalLM +from trainer.trainer_utils import setup_seed, get_model_params +warnings.filterwarnings('ignore') + +TOOLS = [ + {"type": "function", "function": {"name": "calculate_math", "description": "计算数学表达式的结果，支持加减乘除、幂运算、开方等", "parameters": {"type": "object", "properties": {"expression": {"type": "string", "description": "数学表达式，如123+456、2**10、sqrt(144)"}}, "required": ["expression"]}}}, + {"type": "function", "function": {"name": "get_current_time", "description": "获取当前日期和时间，支持指定时区", "parameters": {"type": "object", "properties": {"timezone": {"type": "string", "description": "时区名称，如Asia/Shanghai、America/New_York", "default": "Asia/Shanghai"}}, "required": []}}}, + {"type": "function", "function": {"name": "random_number", "description": "生成指定范围内的随机数", "parameters": {"type": "object", "properties": {"min": {"type": "integer", "description": "最小值", "default": 0}, "max": {"type": "integer", "description": "最大值", "default": 100}}, "required": []}}}, + {"type": "function", "function": {"name": "text_length", "description": "计算文本的字符数和单词数", "parameters": {"type": "object", "properties": {"text": {"type": "string", "description": "要统计的文本"}}, "required": ["text"]}}}, + {"type": "function", "function": {"name": "unit_converter", "description": "进行单位换算，支持长度、重量、温度等", "parameters": {"type": "object", "properties": {"value": {"type": "number", "description": "要转换的数值"}, "from_unit": {"type": "string", "description": "源单位，如km、miles、kg、pounds、celsius、fahrenheit"}, "to_unit": {"type": "string", "description": "目标单位"}}, "required": ["value", "from_unit", "to_unit"]}}}, + {"type": "function", "function": {"name": "get_current_weather", "description": "获取指定城市的当前天气信息，包括温度、湿度和天气状况", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "城市名称，如北京、上海、New York"}, "unit": {"type": "string", "description": "温度单位，celsius或fahrenheit", "enum": ["celsius", "fahrenheit"], "default": "celsius"}}, "required": ["location"]}}}, + {"type": "function", "function": {"name": "get_exchange_rate", "description": "查询两种货币之间的实时汇率", "parameters": {"type": "object", "properties": {"from_currency": {"type": "string", "description": "源货币代码，如USD、CNY、EUR"}, "to_currency": {"type": "string", "description": "目标货币代码，如USD、CNY、EUR"}}, "required": ["from_currency", "to_currency"]}}}, + {"type": "function", "function": {"name": "translate_text", "description": "将文本翻译成目标语言", "parameters": {"type": "object", "properties": {"text": {"type": "string", "description": "要翻译的文本"}, "target_language": {"type": "string", "description": "目标语言，如english、chinese、japanese、french"}}, "required": ["text", "target_language"]}}}, +] + +MOCK_RESULTS = { + "calculate_math": lambda args: {"result": str(eval(str(args.get("expression", "0")).replace("^", "**").replace("×", "*").replace("÷", "/").replace("−", "-").replace("²", "**2").replace("³", "**3").replace("（", "(").replace("）", ")")))}, + "get_current_time": lambda args: {"datetime": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "timezone": args.get("timezone", "Asia/Shanghai")}, + "random_number": lambda args: {"result": random.randint(int(args.get("min", 0)), int(args.get("max", 100)))}, + "text_length": lambda args: {"characters": len(args.get("text", "")), "words": len(args.get("text", "").split())}, + "unit_converter": lambda args: {"result": round(float(args.get("value", 0)) * 0.621371, 2), "from": f"{args.get('value', 0)} {args.get('from_unit', '')}", "to": args.get("to_unit", "")}, + "get_current_weather": lambda args: {"city": args.get("location"), "temperature": "22°C", "humidity": "65%", "condition": "晴"}, + "get_exchange_rate": lambda args: {"from": args.get("from_currency", ""), "to": args.get("to_currency", ""), "rate": 7.15}, + "translate_text": lambda args: {"translated": "hello world"}, +} + +TOOL_MAP = {t["function"]["name"]: t for t in TOOLS} + +def get_tools(names): + return [TOOL_MAP[n] for n in names] + +TEST_CASES = [ + {"prompt": "帮我算一下 256 乘以 37 等于多少", "tools": ["calculate_math", "get_current_time"]}, + {"prompt": "现在几点了？", "tools": ["get_current_time", "random_number"]}, + {"prompt": "帮我把100公里换算成英里", "tools": ["unit_converter", "calculate_math"]}, + {"prompt": "帮我生成一个1到1000的随机数，然后计算它的平方", "tools": ["random_number", "calculate_math", "text_length"]}, + {"prompt": "北京今天天气怎么样？", "tools": ["get_current_weather", "get_current_time"]}, + {"prompt": "查一下美元兑人民币汇率", "tools": ["get_exchange_rate", "get_current_time"]}, + {"prompt": "把'你好世界'翻译成英文", "tools": ["translate_text", "text_length"]}, + {"prompt": "What is the weather in Tokyo? Also convert 30 celsius to fahrenheit.", "tools": ["get_current_weather", "unit_converter", "get_current_time"]}, +] + + +def init_model(args): + tokenizer = AutoTokenizer.from_pretrained(args.load_from) + if 'model' in args.load_from: + model = MiniMindForCausalLM(MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe))) + moe_suffix = '_moe' if args.use_moe else '' + ckp = f'./{args.save_dir}/{args.weight}_{args.hidden_size}{moe_suffix}.pth' + model.load_state_dict(torch.load(ckp, map_location=args.device), strict=True) + else: + model = AutoModelForCausalLM.from_pretrained(args.load_from, trust_remote_code=True) + get_model_params(model, model.config) + return model.eval().to(args.device), tokenizer + + +def parse_tool_calls(text): + matches = re.findall(r'(.*?)', text, re.DOTALL) + calls = [] + for m in matches: + try: + calls.append(json.loads(m.strip())) + except Exception: + pass + return calls + + +def parse_tool_call_from_text(content): + pattern = r'\s*(\{.*?\})\s*' + matches = re.findall(pattern, content, re.DOTALL) + if not matches: + return None + tool_calls = [] + for i, match in enumerate(matches): + try: + data = json.loads(match) + tool_calls.append({ + "id": f"call_{i}", + "function": {"name": data.get("name", ""), "arguments": json.dumps(data.get("arguments", {}), ensure_ascii=False)} + }) + except Exception: + pass + return tool_calls if tool_calls else None + + +def execute_tool(call, arguments=None): + name = call.get("name", "") if isinstance(call, dict) else call + try: + raw_args = call.get("arguments", {}) if isinstance(call, dict) else arguments + args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args + except Exception: + args = {} + fn = MOCK_RESULTS.get(name) + if not fn: + return {"error": f"未知工具: {name}"} + try: + return fn(args) + except Exception as e: + return {"error": f"工具执行失败: {str(e)[:80]}"} + + +def generate(model, tokenizer, messages, tools, args): + streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True) + input_text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, tools=tools, open_thinking=False) + inputs = tokenizer(input_text, return_tensors="pt", truncation=True).to(args.device) + st = time.time() + print('🧠: ', end='') + generated_ids = model.generate( + inputs["input_ids"], attention_mask=inputs["attention_mask"], + max_new_tokens=args.max_new_tokens, do_sample=True, streamer=streamer, + pad_token_id=tokenizer.pad_token_id, eos_token_id=tokenizer.eos_token_id, + top_p=args.top_p, temperature=args.temperature + ) + response = tokenizer.decode(generated_ids[0][len(inputs["input_ids"][0]):], skip_special_tokens=True) + gen_tokens = len(generated_ids[0]) - len(inputs["input_ids"][0]) + print(f'\n[Speed]: {gen_tokens / (time.time() - st):.2f} tokens/s') if args.show_speed else print() + return response + + +def chat_api(client, messages, tools, args, stream=True): + response = client.chat.completions.create( + model=args.api_model, messages=messages, tools=tools, + stream=stream, temperature=args.temperature, + max_tokens=8192, top_p=args.top_p + ) + if not stream: + choice = response.choices[0] + content = choice.message.content or "" + tool_calls = choice.message.tool_calls + if not tool_calls: + tool_calls = parse_tool_call_from_text(content) + print(f'🧠: {content}') + return content, tool_calls + print('🧠: ', end='', flush=True) + content, tool_calls = "", None + for chunk in response: + delta = chunk.choices[0].delta + if delta.content: + print(delta.content, end="", flush=True) + content += delta.content + if delta.tool_calls: + if tool_calls is None: + tool_calls = [] + for tc_chunk in delta.tool_calls: + idx = tc_chunk.index if tc_chunk.index is not None else len(tool_calls) + while len(tool_calls) <= idx: + tool_calls.append({ + "id": "", + "function": {"name": "", "arguments": ""} + }) + if tc_chunk.id: + tool_calls[idx]["id"] += tc_chunk.id + if tc_chunk.function: + if tc_chunk.function.name: + tool_calls[idx]["function"]["name"] += tc_chunk.function.name + if tc_chunk.function.arguments: + tool_calls[idx]["function"]["arguments"] += tc_chunk.function.arguments + print() + if not tool_calls: + tool_calls = parse_tool_call_from_text(content) + return content, tool_calls + + +def run_case(prompt, tools, args, model=None, tokenizer=None, client=None): + messages = [{"role": "user", "content": prompt}] + while True: + if args.backend == 'local': + content = generate(model, tokenizer, messages, tools, args) + tool_calls = parse_tool_calls(content) + else: + content, tool_calls = chat_api(client, messages, tools, args, stream=bool(args.stream)) + if not tool_calls: + break + tool_calls = [{ + "id": tc.id if hasattr(tc, 'id') else tc.get("id", ""), + "name": tc.function.name if hasattr(tc, 'function') else tc["function"]["name"], + "arguments": tc.function.arguments if hasattr(tc, 'function') else tc["function"]["arguments"] + } for tc in tool_calls] if args.backend == 'api' else tool_calls + messages.append({"role": "assistant", "content": content} if args.backend == 'local' else {"role": "assistant", "content": content, "tool_calls": [{"id": tc["id"], "type": "function", "function": {"name": tc["name"], "arguments": tc["arguments"]}} for tc in tool_calls]}) + for tc in tool_calls: + name = tc["name"] + arguments = tc["arguments"] + print(f'📞 [Tool Calling]: {name} | args={arguments}') + result = execute_tool(tc if args.backend == 'local' else name, arguments) + print(f'✅ [Tool Called]: {json.dumps(result, ensure_ascii=False)}') + messages.append({"role": "tool", "content": json.dumps(result, ensure_ascii=False)} if args.backend == 'local' else {"role": "tool", "content": json.dumps(result, ensure_ascii=False), "tool_call_id": tc["id"]}) + + +def main(): + parser = argparse.ArgumentParser(description="MiniMind ToolCall评测") + parser.add_argument('--backend', default='local', choices=['local', 'api'], type=str, help="推理后端（local=本地模型，api=OpenAI兼容接口）") + parser.add_argument('--load_from', default='../model', type=str, help="模型加载路径（model=原生torch权重，其他路径=transformers格式）") + parser.add_argument('--save_dir', default='../out', type=str, help="模型权重目录") + parser.add_argument('--weight', default='full_sft', type=str, help="权重名称前缀（pretrain, full_sft, rlhf, reason, ppo_actor, grpo, spo）") + parser.add_argument('--hidden_size', default=768, type=int, help="隐藏层维度") + parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") + parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") + parser.add_argument('--max_new_tokens', default=512, type=int, help="最大生成长度") + parser.add_argument('--temperature', default=0.9, type=float, help="生成温度，控制随机性（0-1，越大越随机）") + parser.add_argument('--top_p', default=0.9, type=float, help="nucleus采样阈值（0-1）") + parser.add_argument('--show_speed', default=0, type=int, help="显示decode速度（tokens/s）") + parser.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu', type=str, help="运行设备") + parser.add_argument('--api_base_url', default="http://localhost:11434/v1", type=str, help="OpenAI兼容接口的base_url") + parser.add_argument('--api_key', default='sk-123', type=str, help="OpenAI兼容接口的api_key") + parser.add_argument('--api_model', default='jingyaogong/minimind-3:latest', type=str, help="API请求时使用的模型名称") + parser.add_argument('--stream', default=1, type=int, help="API模式下是否流式输出（0=否，1=是）") + args = parser.parse_args() + + model = tokenizer = client = None + if args.backend == 'local': model, tokenizer = init_model(args) + else: client = OpenAI(api_key=args.api_key, base_url=args.api_base_url) + + input_mode = int(input('[0] 自动测试\n[1] 手动输入\n')) + + cases = [{"prompt": case["prompt"], "tools": get_tools(case["tools"]), "tool_names": case["tools"]} for case in TEST_CASES] if input_mode == 0 else iter(lambda: {"prompt": input('💬: '), "tools": TOOLS, "tool_names": [t["function"]["name"] for t in TOOLS]}, {"prompt": "", "tools": TOOLS, "tool_names": []}) + for case in cases: + if not case["prompt"]: break + setup_seed(random.randint(0, 31415926)) + if input_mode == 0: + print(f'📦 可用工具: {case["tool_names"]}\n') + print(f'💬: {case["prompt"]}') + run_case(case["prompt"], case["tools"], args, model=model, tokenizer=tokenizer, client=client) + print('\n' + '-' * 50 + '\n') + + +if __name__ == "__main__": + main() diff --git a/scripts/serve_openai_api.py b/scripts/serve_openai_api.py index 76c8d8e..2cf10f3 100644 --- a/scripts/serve_openai_api.py +++ b/scripts/serve_openai_api.py @@ -1,5 +1,6 @@ import argparse import json +import re import os import sys @@ -52,8 +53,19 @@ class ChatRequest(BaseModel): temperature: float = 0.7 top_p: float = 0.92 max_tokens: int = 8192 - stream: bool = False + stream: bool = True tools: list = [] + open_thinking: bool = False + chat_template_kwargs: dict = None + + def get_open_thinking(self) -> bool: + """兼容多种方式开启 thinking""" + if self.open_thinking: + return True + if self.chat_template_kwargs: + return self.chat_template_kwargs.get('open_thinking', False) or \ + self.chat_template_kwargs.get('enable_thinking', False) + return False class CustomStreamer(TextStreamer): @@ -68,9 +80,31 @@ class CustomStreamer(TextStreamer): self.queue.put(None) -def generate_stream_response(messages, temperature, top_p, max_tokens): +def parse_response(text): + reasoning_content = None + think_match = re.search(r'(.*?)', text, re.DOTALL) + if think_match: + reasoning_content = think_match.group(1).strip() + text = re.sub(r'.*?\s*', '', text, flags=re.DOTALL) + elif '' in text: + parts = text.split('', 1) + reasoning_content = parts[0].strip() + text = parts[1].strip() if len(parts) > 1 else '' + tool_calls = [] + for i, m in enumerate(re.findall(r'(.*?)', text, re.DOTALL)): + try: + call = json.loads(m.strip()) + tool_calls.append({"id": f"call_{int(time.time())}_{i}", "type": "function", "function": {"name": call.get("name", ""), "arguments": json.dumps(call.get("arguments", {}), ensure_ascii=False)}}) + except Exception: + pass + if tool_calls: + text = re.sub(r'.*?', '', text, flags=re.DOTALL) + return text.strip(), reasoning_content, tool_calls or None + + +def generate_stream_response(messages, temperature, top_p, max_tokens, tools=None, open_thinking=False): try: - new_prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)[-max_tokens:] + new_prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, tools=tools or None, open_thinking=open_thinking)[-max_tokens:] inputs = tokenizer(new_prompt, return_tensors="pt", truncation=True).to(device) queue = Queue() @@ -91,20 +125,44 @@ def generate_stream_response(messages, temperature, top_p, max_tokens): Thread(target=_generate).start() + full_text = "" + emitted = 0 + thinking_ended = not bool(open_thinking) + while True: text = queue.get() if text is None: - yield json.dumps({ - "choices": [{ - "delta": {}, - "finish_reason": "stop" - }] - }, ensure_ascii=False) break + full_text += text - yield json.dumps({ - "choices": [{"delta": {"content": text}}] - }, ensure_ascii=False) + if not thinking_ended: + pos = full_text.find('') + if pos >= 0: + thinking_ended = True + new_r = full_text[emitted:pos] + if new_r: + yield json.dumps({"choices": [{"delta": {"reasoning_content": new_r}}]}, ensure_ascii=False) + emitted = pos + len('') + after = full_text[emitted:].lstrip('\n') + emitted = len(full_text) - len(after) + if after: + yield json.dumps({"choices": [{"delta": {"content": after}}]}, ensure_ascii=False) + emitted = len(full_text) + else: + new_r = full_text[emitted:] + if new_r: + yield json.dumps({"choices": [{"delta": {"reasoning_content": new_r}}]}, ensure_ascii=False) + emitted = len(full_text) + else: + new_c = full_text[emitted:] + if new_c: + yield json.dumps({"choices": [{"delta": {"content": new_c}}]}, ensure_ascii=False) + emitted = len(full_text) + + _, _, tool_calls = parse_response(full_text) + if tool_calls: + yield json.dumps({"choices": [{"delta": {"tool_calls": tool_calls}}]}, ensure_ascii=False) + yield json.dumps({"choices": [{"delta": {}, "finish_reason": "tool_calls" if tool_calls else "stop"}]}, ensure_ascii=False) except Exception as e: yield json.dumps({"error": str(e)}) @@ -119,7 +177,9 @@ async def chat_completions(request: ChatRequest): messages=request.messages, temperature=request.temperature, top_p=request.top_p, - max_tokens=request.max_tokens + max_tokens=request.max_tokens, + tools=request.tools, + open_thinking=request.get_open_thinking() )), media_type="text/event-stream" ) @@ -127,7 +187,9 @@ async def chat_completions(request: ChatRequest): new_prompt = tokenizer.apply_chat_template( request.messages, tokenize=False, - add_generation_prompt=True + add_generation_prompt=True, + tools=request.tools or None, + open_thinking=request.get_open_thinking() )[-request.max_tokens:] inputs = tokenizer(new_prompt, return_tensors="pt", truncation=True).to(device) with torch.no_grad(): @@ -142,6 +204,12 @@ async def chat_completions(request: ChatRequest): temperature=request.temperature ) answer = tokenizer.decode(generated_ids[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True) + content, reasoning_content, tool_calls = parse_response(answer) + message = {"role": "assistant", "content": content} + if reasoning_content: + message["reasoning_content"] = reasoning_content + if tool_calls: + message["tool_calls"] = tool_calls return { "id": f"chatcmpl-{int(time.time())}", "object": "chat.completion", @@ -150,8 +218,8 @@ async def chat_completions(request: ChatRequest): "choices": [ { "index": 0, - "message": {"role": "assistant", "content": answer}, - "finish_reason": "stop" + "message": message, + "finish_reason": "tool_calls" if tool_calls else "stop" } ] } @@ -165,8 +233,8 @@ if __name__ == "__main__": parser.add_argument('--save_dir', default='out', type=str, help="模型权重目录") parser.add_argument('--weight', default='full_sft', type=str, help="权重名称前缀（pretrain, full_sft, dpo, reason, ppo_actor, grpo, spo）") parser.add_argument('--lora_weight', default='None', type=str, help="LoRA权重名称（None表示不使用，可选：lora_identity, lora_medical）") - parser.add_argument('--hidden_size', default=512, type=int, help="隐藏层维度（512=Small-26M, 640=MoE-145M, 768=Base-104M）") - parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量（Small/MoE=8, Base=16）") + parser.add_argument('--hidden_size', default=768, type=int, help="隐藏层维度") + parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") parser.add_argument('--max_seq_len', default=8192, type=int, help="最大序列长度") parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") parser.add_argument('--inference_rope_scaling', default=False, action='store_true', help="启用RoPE位置编码外推（4倍，仅解决位置编码问题）") diff --git a/scripts/web_demo.py b/scripts/web_demo.py index 6ed4b2f..b4a17b0 100644 --- a/scripts/web_demo.py +++ b/scripts/web_demo.py @@ -1,10 +1,13 @@ import random import re +import json +import os from threading import Thread import torch import numpy as np import streamlit as st +from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer st.set_page_config(page_title="MiniMind", initial_sidebar_state="collapsed") @@ -64,33 +67,130 @@ st.markdown(""" """, unsafe_allow_html=True) -system_prompt = [] device = "cuda" if torch.cuda.is_available() else "cpu" +# 多语言文本 +LANG_TEXTS = { + 'zh': { + 'settings': '模型设定调整', + 'history_rounds': '历史对话轮次', + 'max_length': '最大生成长度', + 'temperature': '温度', + 'thinking': '思考', + 'tools': '工具', + 'language': '语言', + 'send': '给 MiniMind 发送消息', + 'disclaimer': 'AI 生成内容可能存在错误，请仔细核实', + 'think_tip': '自适应思考，目前多轮对话或Tool Call共存时思考不稳定', + 'tool_select': '工具选择（最多4个）', + }, + 'en': { + 'settings': 'Model Settings', + 'history_rounds': 'History Rounds', + 'max_length': 'Max Length', + 'temperature': 'Temperature', + 'thinking': 'Thinking', + 'tools': 'Tools', + 'language': 'Language', + 'send': 'Send a message to MiniMind', + 'disclaimer': 'AI-generated content may be inaccurate, please verify', + 'think_tip': 'Adaptive thinking; may be unstable with multi-turn or Tool Call', + 'tool_select': 'Tool Selection (max 4)', + } +} -def process_assistant_content(content): - if model_source == "API" and 'R1' not in api_model_name: - return content - if model_source != "API" and 'R1' not in MODEL_PATHS[selected_model][1]: - return content +def get_text(key): + lang = st.session_state.get('lang', 'en') + return LANG_TEXTS.get(lang, {}).get(key, LANG_TEXTS['zh'].get(key, key)) + +# 工具定义 +TOOLS = [ + {"type": "function", "function": {"name": "calculate_math", "description": "计算数学表达式", "parameters": {"type": "object", "properties": {"expression": {"type": "string", "description": "数学表达式"}}, "required": ["expression"]}}}, + {"type": "function", "function": {"name": "get_current_time", "description": "获取当前时间", "parameters": {"type": "object", "properties": {"timezone": {"type": "string", "default": "Asia/Shanghai"}}, "required": []}}}, + {"type": "function", "function": {"name": "random_number", "description": "生成随机数", "parameters": {"type": "object", "properties": {"min": {"type": "integer"}, "max": {"type": "integer"}}, "required": ["min", "max"]}}}, + {"type": "function", "function": {"name": "text_length", "description": "计算文本长度", "parameters": {"type": "object", "properties": {"text": {"type": "string"}}, "required": ["text"]}}}, + {"type": "function", "function": {"name": "unit_converter", "description": "单位转换", "parameters": {"type": "object", "properties": {"value": {"type": "number"}, "from_unit": {"type": "string"}, "to_unit": {"type": "string"}}, "required": ["value", "from_unit", "to_unit"]}}}, + {"type": "function", "function": {"name": "get_current_weather", "description": "获取天气", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}}, + {"type": "function", "function": {"name": "get_exchange_rate", "description": "获取汇率", "parameters": {"type": "object", "properties": {"from_currency": {"type": "string"}, "to_currency": {"type": "string"}}, "required": ["from_currency", "to_currency"]}}}, + {"type": "function", "function": {"name": "translate_text", "description": "翻译文本", "parameters": {"type": "object", "properties": {"text": {"type": "string"}, "target_lang": {"type": "string"}}, "required": ["text", "target_lang"]}}}, +] + +TOOL_SHORT_NAMES = { + 'calculate_math': '数学', 'get_current_time': '时间', 'random_number': '随机', + 'text_length': '字数', 'unit_converter': '单位', 'get_current_weather': '天气', + 'get_exchange_rate': '汇率', 'translate_text': '翻译' +} + +def execute_tool(tool_name, args): + import datetime + try: + if tool_name == 'calculate_math': + return {"result": eval(args.get('expression', '0'))} + elif tool_name == 'get_current_time': + tz = args.get('timezone', 'Asia/Shanghai') + return {"result": datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} + elif tool_name == 'random_number': + return {"result": random.randint(args.get('min', 0), args.get('max', 100))} + elif tool_name == 'text_length': + return {"result": len(args.get('text', ''))} + elif tool_name == 'unit_converter': + return {"result": f"{args.get('value', 0)} {args.get('from_unit', '')} = ? {args.get('to_unit', '')}"} + elif tool_name == 'get_current_weather': + return {"result": f"{args.get('city', 'Unknown')}: 晴, 7~10°C"} + elif tool_name == 'get_exchange_rate': + return {"result": f"1 {args.get('from_currency', 'USD')} = 7.2 {args.get('to_currency', 'CNY')}"} + elif tool_name == 'translate_text': + return {"result": f"[翻译结果]: hello world"} + return {"result": "Unknown tool"} + except Exception as e: + return {"error": str(e)} + + +def process_assistant_content(content, is_streaming=False): + # 处理tool_call标签，格式化显示 + if '' in content: + def format_tool_call(match): + try: + tc = json.loads(match.group(1)) + name = tc.get('name', 'unknown') + args = tc.get('arguments', {}) + return f'
ToolCalling
{name}: {json.dumps(args, ensure_ascii=False)}
' + except: + return match.group(0) + content = re.sub(r'(.*?)', format_tool_call, content, flags=re.DOTALL) + + # 流式生成且开启思考时，一开始就放到折叠里 + if is_streaming and st.session_state.get('enable_thinking', False) and '' not in content and '' not in content: + m = re.search(r'(\n\n(?:我是|您好|你好)[^\n]*)', content) + if m and m.start(1) > 5: + i = m.start(1) + think_part = content[:i] + answer_part = content[i:] + return f'
已思考
{think_part.strip()}
{answer_part}' + elif len(content) > 5: + return f'
思考中...
{content.strip().replace(chr(10), "
")}
' if '' in content and '' in content: - content = re.sub(r'()(.*?)()', - r'
推理内容（展开）\2
', - content, - flags=re.DOTALL) + def format_think(match): + think_content = match.group(2) + if think_content.replace('\n', '').strip(): # 不是全换行 + return f'
已思考
{think_content.strip()}
' + return '' + content = re.sub(r'()(.*?)()', format_think, content, flags=re.DOTALL) if '' in content and '' not in content: - content = re.sub(r'(.*?)$', - r'
推理中...\1
', - content, - flags=re.DOTALL) + def format_think_in_progress(match): + tc = match.group(1) + return f'
思考中...
{tc.strip().replace(chr(10), "
")}
' + content = re.sub(r'(.*?)$', format_think_in_progress, content, flags=re.DOTALL) if '' not in content and '' in content: - content = re.sub(r'(.*?)', - r'
推理内容（展开）\1
', - content, - flags=re.DOTALL) + def format_think_no_start(match): + think_content = match.group(1) + if think_content.replace('\n', '').strip(): + return f'
已思考
{think_content.strip()}
' + return '' + content = re.sub(r'(.*?)', format_think_no_start, content, flags=re.DOTALL) return content @@ -118,17 +218,10 @@ def init_chat_messages(): if "messages" in st.session_state: for i, message in enumerate(st.session_state.messages): if message["role"] == "assistant": - with st.chat_message("assistant", avatar=image_url): - st.markdown(process_assistant_content(message["content"]), unsafe_allow_html=True) - if st.button("🗑", key=f"delete_{i}"): - st.session_state.messages.pop(i) - st.session_state.messages.pop(i - 1) - st.session_state.chat_messages.pop(i) - st.session_state.chat_messages.pop(i - 1) - st.rerun() + st.markdown(process_assistant_content(message["content"]), unsafe_allow_html=True) else: st.markdown( - f'
{message["content"]}
', + f'
{message["content"]}
', unsafe_allow_html=True) else: @@ -143,52 +236,64 @@ def regenerate_answer(index): st.rerun() -def delete_conversation(index): - st.session_state.messages.pop(index) - st.session_state.messages.pop(index - 1) - st.session_state.chat_messages.pop(index) - st.session_state.chat_messages.pop(index - 1) +# 动态扫描模型目录 +script_dir = os.path.dirname(os.path.abspath(__file__)) +MODEL_PATHS = {} +for d in sorted(os.listdir(script_dir), reverse=True): + full_path = os.path.join(script_dir, d) + if os.path.isdir(full_path) and not d.startswith('.') and not d.startswith('_'): + if any(f.endswith(('.bin', '.safetensors', '.pt')) or os.path.exists(os.path.join(full_path, 'model.safetensors.index.json')) for f in os.listdir(full_path) if os.path.isfile(os.path.join(full_path, f))): + MODEL_PATHS[d] = [d, d] +if not MODEL_PATHS: + MODEL_PATHS = {"No models found": ["", "No models"]} + +# 模型选择 +selected_model = st.sidebar.selectbox('Model', list(MODEL_PATHS.keys()), index=0) +model_path = MODEL_PATHS[selected_model][0] +slogan = f"我是 {MODEL_PATHS[selected_model][1]}，有什么可以帮你的？" if st.session_state.get('lang', 'en') == 'zh' else f"I am {MODEL_PATHS[selected_model][1]}, how can I help you?" + +st.sidebar.markdown('
', unsafe_allow_html=True) + +# 语言选择 +lang_options = {'中文': 'zh', 'English': 'en'} +current_lang = st.session_state.get('lang', 'en') +lang_index = 0 if current_lang == 'zh' else 1 +lang_label = st.sidebar.radio('Language / 语言', list(lang_options.keys()), index=lang_index, horizontal=True) +if lang_options[lang_label] != current_lang: + st.session_state.lang = lang_options[lang_label] st.rerun() +st.sidebar.markdown('
', unsafe_allow_html=True) -st.sidebar.title("模型设定调整") +# 参数设置 +st.session_state.history_chat_num = st.sidebar.slider(get_text('history_rounds'), 0, 8, 0, step=2) +st.session_state.max_new_tokens = st.sidebar.slider(get_text('max_length'), 256, 8192, 8192, step=1) +st.session_state.temperature = st.sidebar.slider(get_text('temperature'), 0.6, 1.2, 0.90, step=0.01) -# st.sidebar.text("训练数据偏差，增加上下文记忆时\n多轮对话（较单轮）容易出现能力衰减") -st.session_state.history_chat_num = st.sidebar.slider("Number of Historical Dialogues", 0, 6, 0, step=2) -# st.session_state.history_chat_num = 0 -st.session_state.max_new_tokens = st.sidebar.slider("Max Sequence Length", 256, 8192, 8192, step=1) -st.session_state.temperature = st.sidebar.slider("Temperature", 0.6, 1.2, 0.85, step=0.01) +st.sidebar.markdown('
', unsafe_allow_html=True) -model_source = st.sidebar.radio("选择模型来源", ["本地模型", "API"], index=0) - -if model_source == "API": - api_url = st.sidebar.text_input("API URL", value="http://127.0.0.1:8000/v1") - api_model_id = st.sidebar.text_input("Model ID", value="minimind") - api_model_name = st.sidebar.text_input("Model Name", value="MiniMind2") - api_key = st.sidebar.text_input("API Key", value="none", type="password") - slogan = f"Hi, I'm {api_model_name}" -else: - MODEL_PATHS = { - "MiniMind2-R1 (0.1B)": ["../MiniMind2-R1", "MiniMind2-R1"], - "MiniMind2-Small-R1 (0.02B)": ["../MiniMind2-Small-R1", "MiniMind2-Small-R1"], - "MiniMind2 (0.1B)": ["../MiniMind2", "MiniMind2"], - "MiniMind2-MoE (0.15B)": ["../MiniMind2-MoE", "MiniMind2-MoE"], - "MiniMind2-Small (0.02B)": ["../MiniMind2-Small", "MiniMind2-Small"] - } - - selected_model = st.sidebar.selectbox('Models', list(MODEL_PATHS.keys()), index=2) # 默认选择 MiniMind2 - model_path = MODEL_PATHS[selected_model][0] - slogan = f"Hi, I'm {MODEL_PATHS[selected_model][1]}" +# 功能开关 +st.session_state.enable_thinking = st.sidebar.checkbox(get_text('thinking'), value=False, help=get_text('think_tip')) +st.session_state.selected_tools = [] +with st.sidebar.expander(get_text('tools')): + st.caption(get_text('tool_select')) + selected_count = sum(1 for tool in TOOLS if st.session_state.get(f"tool_{tool['function']['name']}", False)) + for tool in TOOLS: + name = tool['function']['name'] + short_name = TOOL_SHORT_NAMES.get(name, name) + checked = st.checkbox(short_name, key=f"tool_{name}", disabled=(selected_count >= 4 and not st.session_state.get(f"tool_{name}", False))) + if checked and len(st.session_state.selected_tools) < 4: + st.session_state.selected_tools.append(name) image_url = "https://www.modelscope.cn/api/v1/studio/gongjy/MiniMind/repo?Revision=master&FilePath=images%2Flogo2.png&View=true" st.markdown( f'
' '
' - f' ' + f' ' f'{slogan}' '
' - '内容完全由AI生成，请务必仔细甄别
Content AI-generated, please discern with care' + f'{get_text("disclaimer")}' '
', unsafe_allow_html=True ) @@ -205,10 +310,7 @@ def setup_seed(seed): def main(): - if model_source == "本地模型": - model, tokenizer = load_model_tokenizer(model_path) - else: - model, tokenizer = None, None + model, tokenizer = load_model_tokenizer(model_path) if "messages" not in st.session_state: st.session_state.messages = [] @@ -218,18 +320,13 @@ def main(): for i, message in enumerate(messages): if message["role"] == "assistant": - with st.chat_message("assistant", avatar=image_url): - st.markdown(process_assistant_content(message["content"]), unsafe_allow_html=True) - if st.button("×", key=f"delete_{i}"): - st.session_state.messages = st.session_state.messages[:i - 1] - st.session_state.chat_messages = st.session_state.chat_messages[:i - 1] - st.rerun() + st.markdown(process_assistant_content(message["content"]), unsafe_allow_html=True) else: st.markdown( - f'
{message["content"]}
', + f'
{message["content"]}
', unsafe_allow_html=True) - prompt = st.chat_input(key="input", placeholder="给 MiniMind 发送消息") + prompt = st.chat_input(key="input", placeholder=get_text('send')) if hasattr(st.session_state, 'regenerate') and st.session_state.regenerate: prompt = st.session_state.last_user_message @@ -240,89 +337,84 @@ def main(): if prompt: st.markdown( - f'
{prompt}
', + f'
{prompt}
', unsafe_allow_html=True) messages.append({"role": "user", "content": prompt[-st.session_state.max_new_tokens:]}) st.session_state.chat_messages.append({"role": "user", "content": prompt[-st.session_state.max_new_tokens:]}) - with st.chat_message("assistant", avatar=image_url): - placeholder = st.empty() + placeholder = st.empty() - if model_source == "API": - try: - from openai import OpenAI + random_seed = random.randint(0, 2 ** 32 - 1) + setup_seed(random_seed) - client = OpenAI( - api_key=api_key, - base_url=api_url - ) - history_num = st.session_state.history_chat_num + 1 # +1 是为了包含当前的用户消息 - conversation_history = system_prompt + st.session_state.chat_messages[-history_num:] - answer = "" - response = client.chat.completions.create( - model=api_model_id, - messages=conversation_history, - stream=True, - temperature=st.session_state.temperature - ) + tools = [t for t in TOOLS if t['function']['name'] in st.session_state.get('selected_tools', [])] or None + sys_prompt = [] if tools else [{"role": "system", "content": "你是MiniMind，一个乐于助人、知识渊博的AI助手。请用完整且友好的方式回答用户问题。"}] + st.session_state.chat_messages = sys_prompt + st.session_state.chat_messages[-(st.session_state.history_chat_num + 1):] + template_kwargs = {"tokenize": False, "add_generation_prompt": True} + if st.session_state.get('enable_thinking', False): + template_kwargs["open_thinking"] = True + if tools: + template_kwargs["tools"] = tools + new_prompt = tokenizer.apply_chat_template(st.session_state.chat_messages, **template_kwargs) - for chunk in response: - content = chunk.choices[0].delta.content or "" - answer += content - placeholder.markdown(process_assistant_content(answer), unsafe_allow_html=True) + inputs = tokenizer(new_prompt, return_tensors="pt", truncation=True).to(device) - except Exception as e: - answer = f"API调用出错: {str(e)}" - placeholder.markdown(answer, unsafe_allow_html=True) - else: - random_seed = random.randint(0, 2 ** 32 - 1) - setup_seed(random_seed) + streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True) + generation_kwargs = { + "input_ids": inputs.input_ids, + "max_length": inputs.input_ids.shape[1] + st.session_state.max_new_tokens, + "num_return_sequences": 1, + "do_sample": True, + "attention_mask": inputs.attention_mask, + "pad_token_id": tokenizer.pad_token_id, + "eos_token_id": tokenizer.eos_token_id, + "temperature": st.session_state.temperature, + "top_p": 0.85, + "streamer": streamer, + } - st.session_state.chat_messages = system_prompt + st.session_state.chat_messages[ - -(st.session_state.history_chat_num + 1):] - new_prompt = tokenizer.apply_chat_template( - st.session_state.chat_messages, - tokenize=False, - add_generation_prompt=True - ) + Thread(target=model.generate, kwargs=generation_kwargs).start() - inputs = tokenizer( - new_prompt, - return_tensors="pt", - truncation=True - ).to(device) + answer = "" + for new_text in streamer: + answer += new_text + placeholder.markdown(process_assistant_content(answer, is_streaming=True), unsafe_allow_html=True) - streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True) - generation_kwargs = { - "input_ids": inputs.input_ids, - "max_length": inputs.input_ids.shape[1] + st.session_state.max_new_tokens, - "num_return_sequences": 1, - "do_sample": True, - "attention_mask": inputs.attention_mask, - "pad_token_id": tokenizer.pad_token_id, - "eos_token_id": tokenizer.eos_token_id, - "temperature": st.session_state.temperature, - "top_p": 0.85, - "streamer": streamer, - } - - Thread(target=model.generate, kwargs=generation_kwargs).start() - - answer = "" - for new_text in streamer: - answer += new_text - placeholder.markdown(process_assistant_content(answer), unsafe_allow_html=True) - - messages.append({"role": "assistant", "content": answer}) + full_answer = answer + for _ in range(16): + tool_calls = re.findall(r'(.*?)', answer, re.DOTALL) + if not tool_calls: + break st.session_state.chat_messages.append({"role": "assistant", "content": answer}) - with st.empty(): - if st.button("×", key=f"delete_{len(messages) - 1}"): - st.session_state.messages = st.session_state.messages[:-2] - st.session_state.chat_messages = st.session_state.chat_messages[:-2] - st.rerun() + tool_results = [] + for tc_str in tool_calls: + try: + tc = json.loads(tc_str.strip()) + result = execute_tool(tc.get('name', ''), tc.get('arguments', {})) + st.session_state.chat_messages.append({"role": "tool", "content": json.dumps(result, ensure_ascii=False)}) + tool_results.append(f'
ToolCalled
{tc.get("name", "")}: {json.dumps(result, ensure_ascii=False)}
') + except: + pass + full_answer += "\n" + "\n".join(tool_results) + "\n" + placeholder.markdown(process_assistant_content(full_answer, is_streaming=True), unsafe_allow_html=True) + new_prompt = tokenizer.apply_chat_template(st.session_state.chat_messages, **template_kwargs) + inputs = tokenizer(new_prompt, return_tensors="pt", truncation=True).to(device) + streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True) + generation_kwargs["input_ids"] = inputs.input_ids + generation_kwargs["attention_mask"] = inputs.attention_mask + generation_kwargs["max_length"] = inputs.input_ids.shape[1] + st.session_state.max_new_tokens + generation_kwargs["streamer"] = streamer + Thread(target=model.generate, kwargs=generation_kwargs).start() + answer = "" + for new_text in streamer: + answer += new_text + placeholder.markdown(process_assistant_content(full_answer + answer, is_streaming=True), unsafe_allow_html=True) + full_answer += answer + answer = full_answer + + messages.append({"role": "assistant", "content": answer}) + st.session_state.chat_messages.append({"role": "assistant", "content": answer}) if __name__ == "__main__": - from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer - main() diff --git a/trainer/rollout_engine.py b/trainer/rollout_engine.py new file mode 100644 index 0000000..029a44a --- /dev/null +++ b/trainer/rollout_engine.py @@ -0,0 +1,212 @@ +"""Rollout Engine - 可插拔的推理引擎 +python -m sglang.launch_server --model-path ./minimind-3 --attention-backend triton --host 0.0.0.0 --port 8998 +""" +import os +import sys + +__package__ = "trainer" +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +import requests +import torch +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import List, Optional, Tuple +from torch import Tensor +from torch.nn.parallel import DistributedDataParallel +from transformers import AutoTokenizer + + +# ===== 计算每个 token 的 logprob ===== +def compute_per_token_logps(model, input_ids: Tensor, n_keep: int, attention_mask: Optional[Tensor] = None) -> Tensor: + if n_keep <= 0: + return input_ids.new_empty((input_ids.size(0), 0), dtype=torch.float32) + unwrapped = model.module if isinstance(model, DistributedDataParallel) else model + input_ids = input_ids.detach().clone() if input_ids.is_inference() else input_ids + logits = unwrapped(input_ids, attention_mask=attention_mask, logits_to_keep=n_keep + 1).logits[:, :-1, :] + per_token_logps = [] + for logits_row, ids_row in zip(logits, input_ids[:, -n_keep:]): + ids_row = ids_row.detach().clone() if ids_row.is_inference() else ids_row + per_token_logps.append( + torch.gather(logits_row.log_softmax(dim=-1), 1, ids_row.unsqueeze(1)).squeeze(1) + ) + return torch.stack(per_token_logps) + + +# ===== Rollout 结果 ===== +@dataclass +class RolloutResult: + output_ids: Tensor + completion_ids: Tensor + per_token_logps: Tensor + completions: List[str] + + +# ===== Rollout 引擎抽象基类 ===== +class RolloutEngine(ABC): + tokenizer = None + + @abstractmethod + def rollout(self, prompt_ids: Tensor, attention_mask: Tensor, num_generations: int, max_new_tokens: int, temperature: float = 0.8) -> RolloutResult: + pass + + @abstractmethod + def update_policy(self, model: torch.nn.Module): + pass + + +# ===== PyTorch 原生推理引擎 ===== +class TorchRolloutEngine(RolloutEngine): + def __init__(self, policy_model: torch.nn.Module, tokenizer, device: str = "cuda", autocast_ctx=None): + self.policy_model = policy_model + self.tokenizer = tokenizer + self.device = device + self.autocast_ctx = autocast_ctx + + def rollout(self, prompt_ids: Tensor, attention_mask: Tensor, num_generations: int, max_new_tokens: int, temperature: float = 0.8) -> RolloutResult: + model = self.policy_model.module if isinstance(self.policy_model, DistributedDataParallel) else self.policy_model + + with torch.no_grad(): + output_ids = model.generate( + input_ids=prompt_ids, + attention_mask=attention_mask, + max_new_tokens=max_new_tokens, + do_sample=True, + temperature=temperature, + num_return_sequences=num_generations, + pad_token_id=self.tokenizer.pad_token_id, + eos_token_id=self.tokenizer.eos_token_id, + ) # [B*num_gen, P+R] + + prompt_len = prompt_ids.size(1) + completion_ids = output_ids[:, prompt_len:] # [B*num_gen, R] + + from contextlib import nullcontext + ctx = self.autocast_ctx if self.autocast_ctx else nullcontext() + with ctx: + per_token_logps = compute_per_token_logps(self.policy_model, output_ids, completion_ids.size(1)) + + completions = self.tokenizer.batch_decode(completion_ids, skip_special_tokens=True) + return RolloutResult(output_ids, completion_ids, per_token_logps, completions) + + def update_policy(self, model: torch.nn.Module): + self.policy_model = model + + +# ===== SGLang HTTP API 推理引擎 ===== +class SGLangRolloutEngine(RolloutEngine): + def __init__(self, base_url: str, model_path: str, shared_ckpt_path: str = "./sglang_ckpt", timeout: int = 120): + self.base_url = base_url.rstrip('/') + self.shared_ckpt_path = shared_ckpt_path + self.timeout = timeout + self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + self.http = requests + + def rollout(self, prompt_ids: Tensor, attention_mask: Tensor, num_generations: int, max_new_tokens: int, temperature: float = 0.8) -> RolloutResult: + # 去除左侧 padding tokens，只保留有效 token + input_ids_list = [] + for ids, mask in zip(prompt_ids, attention_mask): + valid_ids = ids[mask.bool()].tolist() + input_ids_list.append(valid_ids) + all_input_ids = [ids for ids in input_ids_list for _ in range(num_generations)] + + payload = { + "input_ids": all_input_ids, + "sampling_params": { + "temperature": temperature, + "max_new_tokens": max_new_tokens, + "stop_token_ids": [self.tokenizer.eos_token_id] if self.tokenizer.eos_token_id else [], + }, + "return_logprob": True, + } + + resp = self.http.post(f"{self.base_url}/generate", json=payload, timeout=self.timeout) + resp.raise_for_status() + + results = resp.json() + if not isinstance(results, list): + results = [results] + + all_output_ids, all_completion_ids, all_logprobs = [], [], [] + completions = [] + prompt_len = prompt_ids.size(1) + + for i, result in enumerate(results): + meta = result.get("meta_info", {}) + completion_ids = meta.get("output_ids", result.get("output_ids", [])) + raw_logprobs = meta.get("output_token_logprobs", []) + + logprobs = [] + for item in raw_logprobs: + if isinstance(item, (list, tuple)) and len(item) >= 1: + logprobs.append(item[0]) + elif isinstance(item, (int, float)): + logprobs.append(item) + + prompt = all_input_ids[i] + full_output = prompt + completion_ids + all_output_ids.append(full_output) + all_completion_ids.append(completion_ids) + all_logprobs.append(logprobs) + completions.append(self.tokenizer.decode(completion_ids, skip_special_tokens=True)) + + device = prompt_ids.device + max_out_len = max(len(ids) for ids in all_output_ids) + max_comp_len = max(len(ids) for ids in all_completion_ids) + max_logp_len = max(len(lp) for lp in all_logprobs) + + def pad_to_tensor(seqs, max_len, pad_val=0): + return torch.tensor([s + [pad_val] * (max_len - len(s)) for s in seqs], device=device) + + return RolloutResult( + output_ids=pad_to_tensor(all_output_ids, max_out_len), + completion_ids=pad_to_tensor(all_completion_ids, max_comp_len), + per_token_logps=pad_to_tensor(all_logprobs, max_logp_len, pad_val=0.0), + completions=completions, + ) + + def update_policy(self, model: torch.nn.Module): + unwrapped = model.module if isinstance(model, DistributedDataParallel) else model + abs_path = os.path.abspath(self.shared_ckpt_path) + unwrapped.lm_head.weight = torch.nn.Parameter(unwrapped.lm_head.weight.clone()) + state_dict = {k: v.detach().half().cpu() for k, v in unwrapped.state_dict().items()} + unwrapped.save_pretrained(abs_path, state_dict=state_dict, safe_serialization=False) + unwrapped.model.embed_tokens.weight = unwrapped.lm_head.weight + self.tokenizer.save_pretrained(abs_path) + resp = self.http.post( + f"{self.base_url}/update_weights_from_disk", + json={"model_path": abs_path}, + timeout=self.timeout + ) + if resp.status_code != 200: print(f"[SGLANG WARNING] update_weights 失败: {resp.status_code}, {resp.text}") + return resp.status_code == 200 + + def flush_cache(self) -> bool: + resp = self.http.post(f"{self.base_url}/flush_cache", timeout=30) + return resp.status_code == 200 + + def health(self) -> bool: + try: + resp = self.http.get(f"{self.base_url}/health", timeout=5) + return resp.status_code == 200 + except: + return False + + +# ===== 工厂函数 ===== +def create_rollout_engine( + engine_type: str = "torch", + policy_model: torch.nn.Module = None, + tokenizer = None, + device: str = "cuda", + autocast_ctx = None, + sglang_base_url: str = None, + sglang_model_path: str = None, + sglang_shared_path: str = None, +) -> RolloutEngine: + if engine_type == "torch": + return TorchRolloutEngine(policy_model, tokenizer, device, autocast_ctx) + elif engine_type == "sglang": + return SGLangRolloutEngine(sglang_base_url, sglang_model_path, sglang_shared_path) + else: + raise ValueError(f"不支持的引擎类型: {engine_type}") diff --git a/trainer/train_agent.py b/trainer/train_agent.py new file mode 100644 index 0000000..9a84b69 --- /dev/null +++ b/trainer/train_agent.py @@ -0,0 +1,487 @@ +import os +import sys + +__package__ = "trainer" +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +import re +import gc +import json +import math +import random +import signal +import argparse +import warnings +import torch +import torch.nn.functional as F +import torch.distributed as dist +from contextlib import nullcontext +from torch import optim +from torch.nn.parallel import DistributedDataParallel +from torch.utils.data import DataLoader, DistributedSampler +from torch.optim.lr_scheduler import CosineAnnealingLR +from transformers import AutoTokenizer +from model.model_minimind import MiniMindConfig, MiniMindForCausalLM +from dataset.lm_dataset import AgentRLDataset +from trainer.trainer_utils import Logger, is_main_process, lm_checkpoint, init_distributed_mode, setup_seed, SkipBatchSampler, init_model, LMForRewardModel +from trainer.rollout_engine import create_rollout_engine, compute_per_token_logps + +warnings.filterwarnings('ignore') + +# ================================ 工具与 Reward = Start ================================ + +def rep_penalty(text, n=3, cap=0.5): + toks = re.findall(r"\w+|[^\w\s]", text.lower()) + grams = [tuple(toks[i:i + n]) for i in range(len(toks) - n + 1)] + return min(cap, (len(grams) - len(set(grams))) * cap * 2 / len(grams)) if grams else 0.0 + +# ======== 工具定义 ======== +TOOLS = [ + {"type": "function", "function": {"name": "calculate_math", "description": "计算数学表达式", "parameters": {"type": "object", "properties": {"expression": {"type": "string"}}, "required": ["expression"]}}}, + {"type": "function", "function": {"name": "unit_converter", "description": "单位换算", "parameters": {"type": "object", "properties": {"value": {"type": "number"}, "from_unit": {"type": "string"}, "to_unit": {"type": "string"}}, "required": ["value", "from_unit", "to_unit"]}}}, + {"type": "function", "function": {"name": "get_current_weather", "description": "获取天气", "parameters": {"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}}}, + {"type": "function", "function": {"name": "get_current_time", "description": "获取时间", "parameters": {"type": "object", "properties": {"timezone": {"type": "string", "default": "Asia/Shanghai"}}, "required": []}}}, + {"type": "function", "function": {"name": "get_exchange_rate", "description": "查询汇率", "parameters": {"type": "object", "properties": {"from_currency": {"type": "string"}, "to_currency": {"type": "string"}}, "required": ["from_currency", "to_currency"]}}}, + {"type": "function", "function": {"name": "translate_text", "description": "翻译文本", "parameters": {"type": "object", "properties": {"text": {"type": "string"}, "target_language": {"type": "string"}}, "required": ["text", "target_language"]}}}, +] + +# ======== 模拟数据 ======== +WEATHER_DATA = {"北京": ("28°C", "晴"), "上海": ("15°C", "多云"), "广州": ("32°C", "闷热"), "深圳": ("30°C", "晴"), "杭州": ("22°C", "阴"), "成都": ("18°C", "小雨"), "武汉": ("25°C", "多云"), "南京": ("20°C", "晴"), "西安": ("16°C", "大风"), "重庆": ("26°C", "阴"), "Tokyo": ("12°C", "晴"), "New York": ("8°C", "多云"), "London": ("5°C", "小雨"), "Paris": ("10°C", "阴"), "Sydney": ("25°C", "晴朗")} +TIME_DATA = {"Asia/Shanghai": "2025-03-07 14:30:00", "America/New_York": "2025-03-07 01:30:00", "Europe/London": "2025-03-07 06:30:00", "Asia/Tokyo": "2025-03-07 15:30:00", "Europe/Paris": "2025-03-07 07:30:00", "Australia/Sydney": "2025-03-07 17:30:00"} +EXCHANGE_DATA = {("USD", "CNY"): 7.21, ("EUR", "CNY"): 7.85, ("GBP", "CNY"): 9.12, ("JPY", "CNY"): 0.048, ("USD", "EUR"): 0.92, ("USD", "GBP"): 0.79, ("CNY", "JPY"): 20.83, ("AUD", "CNY"): 4.72} +TRANSLATE_DATA = {("你好世界", "english"): "Hello World", ("Good morning", "chinese"): "早上好", ("今天天气真好", "english"): "The weather is nice today", ("I love programming", "chinese"): "我喜欢编程", ("机器学习很有趣", "english"): "Machine learning is interesting", ("Happy birthday", "chinese"): "生日快乐"} +UNIT_DATA = {"km_miles": 0.621371, "miles_km": 1.60934, "kg_pounds": 2.20462, "pounds_kg": 0.453592, "meters_feet": 3.28084, "feet_meters": 0.3048, "celsius_fahrenheit": 1.8, "fahrenheit_celsius": 0.5556} + +# ======== 模拟执行 ======== +MOCK_RESULTS = { + "calculate_math": lambda args: {"result": str(eval(str(args.get("expression", "0")).replace("^", "**").replace("×", "*").replace("÷", "/").replace("−", "-").replace("（", "(").replace("）", ")"), {"__builtins__": {}, "math": math}))}, + "unit_converter": lambda args: {"result": round(float(args.get("value", 0)) * UNIT_DATA.get(f"{args.get('from_unit', '').lower()}_{args.get('to_unit', '').lower()}", 1), 4)}, + "get_current_weather": lambda args: (lambda w: {"city": args.get("location"), "temperature": w[0], "humidity": "65%", "condition": w[1]})(WEATHER_DATA.get(args.get("location"), ("22°C", "晴"))), + "get_current_time": lambda args: {"datetime": TIME_DATA.get(args.get("timezone", "Asia/Shanghai"), "2025-03-07 14:30:00"), "timezone": args.get("timezone", "Asia/Shanghai")}, + "get_exchange_rate": lambda args: {"from": args.get("from_currency"), "to": args.get("to_currency"), "rate": EXCHANGE_DATA.get((args.get("from_currency"), args.get("to_currency")), 1.0)}, + "translate_text": lambda args: {"translated_text": TRANSLATE_DATA.get((args.get("text"), args.get("target_language")), args.get("text", ""))}, +} + +# ======== 参数校验 ======== +CHECK_ARGS = { + "calculate_math": lambda a: bool(a.get("expression")), + "unit_converter": lambda a: a.get("value") is not None and a.get("from_unit") and a.get("to_unit"), + "get_current_weather": lambda a: bool(a.get("location")), + "get_current_time": lambda a: True, + "get_exchange_rate": lambda a: bool(a.get("from_currency")) and bool(a.get("to_currency")), + "translate_text": lambda a: bool(a.get("text")) and bool(a.get("target_language")), +} + +# ======== 工具调用解析与执行 ======== +def parse_tool_calls(text): + calls = [] + for m in re.findall(r'(.*?)', text, re.DOTALL): + try: calls.append(json.loads(m.strip())) + except: pass + return calls + +def execute_tool(name, args): + fn = MOCK_RESULTS.get(name) + if not fn: return None + try: + signal.signal(signal.SIGALRM, lambda *_: (_ for _ in ()).throw(TimeoutError())) + signal.alarm(1) + return fn(args) + except: + return None + finally: + try: signal.alarm(0) + except: pass + +# ======== 多轮 Rollout ======== +def rollout_single(rollout_engine, tokenizer, messages, tools, max_turns=3, max_new_tokens=256, thinking_ratio=0.5, device="cuda"): + all_outputs = [] + prompt_ids = None + response_ids = [] + response_mask = [] + response_old_logps = [] + final_context = "" + unfinished = False + open_thinking = random.random() < thinking_ratio + for turn in range(max_turns): + context = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, tools=tools, open_thinking=open_thinking) + inputs = tokenizer(context, return_tensors="pt", add_special_tokens=False).to(device) + context_ids = inputs["input_ids"][0].tolist() + if prompt_ids is None: + prompt_ids = context_ids + rollout_result = rollout_engine.rollout( + prompt_ids=inputs["input_ids"], + attention_mask=inputs["attention_mask"], + num_generations=1, + max_new_tokens=max_new_tokens, + temperature=0.8, + ) + new_ids = rollout_result.completion_ids[0].tolist() + new_logps = rollout_result.per_token_logps[0].tolist() + if len(new_ids) != len(new_logps): Logger(f"rollout token/logprob length mismatch: {len(new_ids)} vs {len(new_logps)}") + pairs = [(t, lp) for t, lp in zip(new_ids, new_logps) if t != tokenizer.pad_token_id and t != tokenizer.eos_token_id] + new_ids = [t for t, _ in pairs] + new_logps = [lp for _, lp in pairs] + new_text = rollout_result.completions[0] + all_outputs.append(new_text) + response_ids.extend(new_ids) + response_mask.extend([1] * len(new_ids)) + response_old_logps.extend(new_logps) + final_context = context + new_text + calls = parse_tool_calls(new_text) + if not calls: + break + unfinished = turn == max_turns - 1 + messages.append({"role": "assistant", "content": new_text}) + for call in calls: + name, raw = call.get("name", ""), call.get("arguments", {}) + if isinstance(raw, str): + try: raw = json.loads(raw) + except: raw = {} + result = execute_tool(name, raw) + result_str = (json.dumps(result, ensure_ascii=False) if result else '{"error": "tool not found"}')[:2048] # 防止天文数字撑爆tokenizer + messages.append({"role": "tool", "content": result_str}) + + observe_context = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=not unfinished, tools=tools, open_thinking=open_thinking) + observe_ids = tokenizer(observe_context, return_tensors="pt", add_special_tokens=False)["input_ids"][0].tolist() + current_len = len(prompt_ids) + len(response_ids) + obs_delta = observe_ids[current_len:] + response_ids.extend(obs_delta) + response_mask.extend([0] * len(obs_delta)) + response_old_logps.extend([0.0] * len(obs_delta)) + final_context = observe_context + + final_output = all_outputs[-1] if all_outputs else "" + prompt_ids = prompt_ids or [] + return final_output, final_context, prompt_ids, response_ids, response_mask, response_old_logps, list(all_outputs), unfinished + +def rollout_batch(rollout_engine, tokenizer, messages_batch, tools_batch, num_gen, max_turns=3, max_new_tokens=256, thinking_ratio=0.5, device="cuda"): + all_completions = [] + all_contexts = [] + all_prompt_ids = [] + all_response_ids = [] + all_response_masks = [] + all_response_old_logps = [] + all_turn_outputs = [] + all_unfinished = [] + for messages, tools in zip(messages_batch, tools_batch): + for _ in range(num_gen): + msgs_copy = [dict(m) for m in messages] + completion, context, prompt_ids, response_ids, response_mask, response_old_logps, turn_outputs, unfinished = rollout_single(rollout_engine, tokenizer, msgs_copy, tools, max_turns, max_new_tokens, thinking_ratio, device) + all_completions.append(completion) + all_contexts.append(context) + all_prompt_ids.append(prompt_ids) + all_response_ids.append(response_ids) + all_response_masks.append(response_mask) + all_response_old_logps.append(response_old_logps) + all_turn_outputs.append(turn_outputs) + all_unfinished.append(unfinished) + return all_completions, all_contexts, all_prompt_ids, all_response_ids, all_response_masks, all_response_old_logps, all_turn_outputs, all_unfinished + +# ======== Reward 计算 ======== +def validate_gt_in_text(text, gt_list): + text, text_num = str(text), str(text).replace(',', '') + nums = [float(x) for x in re.findall(r'(?', 1)[-1].strip() if '' in turn else turn.strip() for turn in turn_outputs] + answer = turn_answers[-1] if turn_answers else response.strip() + valid_names = {t['function']['name'] for t in tools} if tools else set() + tool_calls = [] + for turn_answer in turn_answers: tool_calls.extend(parse_tool_calls(turn_answer)) # 解析tool调用 + reward -= 0.5 * sum(abs(turn.count('') - turn.count('')) for turn in turn_answers) # 标签扣分 + # -------- 无工具调用：格式+reward奖励 -------- + if not tool_calls: + reward += 0.5 if 5 <= len(response.strip()) <= 800 else -0.5 # 长度分 + if '' in response: + think, answer = response.split('', 1) + reward += 1.0 if 20 <= len(think.strip()) <= 300 else -0.5 # 思考长度分 + reward += 0.25 if response.count('') == 1 else -0.25 # 思考闭合分 + answer = answer.strip() + if reward_model is not None: + prompt = prompts[sample_idx] + pattern = r"<\|im_start\|>(system|user|assistant)\s+(.*?)<\|im_end\|>" + matches = re.findall(pattern, prompt, re.DOTALL) + messages = [{"role": role, "content": content.strip()} for role, content in matches] + score = reward_model.get_score(messages, answer) + reward += score # RM分 + reward -= rep_penalty(answer) + rewards[idx] = max(min(reward, 3.0), -3.0) # 总分Clip + # -------- 有工具调用：执行结果奖励 -------- + else: + gt = gt_batch[sample_idx] + valid_call_count = 0 + for tool_call in tool_calls: + name, raw = tool_call.get("name", ""), tool_call.get("arguments", {}) + if isinstance(raw, str): + try: raw = json.loads(raw) + except: raw = {} + check = CHECK_ARGS.get(name) + valid_call_count += int(bool(name in valid_names and check and check(raw))) + tool_gap = abs(valid_call_count - len(gt)) + max(0, len(tool_calls) - valid_call_count) # tool数差值 + reward += 0.5 if tool_gap == 0 else -0.5 * tool_gap # tool对齐分 + + final_text = "" if unfinished else (answer.split('')[-1] if '' in answer else answer) + verified = validate_gt_in_text(final_text, gt) if gt else set() + if gt: reward += 2.5 * len(verified) / len(gt) # GT分 + if unfinished: reward -= 0.5 # 未完成扣分 + reward -= rep_penalty(final_text if final_text else answer) + rewards[idx] = max(min(reward, 3.0), -3.0) # 总分Clip + return rewards + +# ================================ 工具与 Reward = End ================================ +def rl_train_epoch(epoch, loader, iters, rollout_engine, ref_model, reward_model=None, start_step=0, wandb=None, use_sglang=False): + last_step = start_step + for step, batch in enumerate(loader, start=start_step + 1): + messages_batch = batch['messages'] + tools_batch = batch['tools'] + gt_batch = batch['gt'] + last_step = step + + with torch.no_grad(): + completions, contexts, prompt_ids_batch, response_ids_batch, response_masks_batch, response_old_logps_batch, turn_outputs_batch, unfinished_batch = rollout_batch(rollout_engine, tokenizer, messages_batch, tools_batch, args.num_generations, max_turns=3, max_new_tokens=args.max_gen_len, thinking_ratio=args.thinking_ratio, device=args.device) + + prompts = [tokenizer.apply_chat_template(m, tokenize=False, add_generation_prompt=True, tools=t) for m, t in zip(messages_batch, tools_batch)] + packed_samples = [] + for p, r, m, old_lp in zip(prompt_ids_batch, response_ids_batch, response_masks_batch, response_old_logps_batch): + ids = p + r + mask = [0] * len(p) + m + old_logps = [0.0] * max(len(p) - 1, 0) + old_lp + if len(ids) > args.max_total_len: + ids = ids[-args.max_total_len:] + mask = mask[-args.max_total_len:] + old_logps = old_logps[-(len(ids) - 1):] + prompt_len = next((i for i, v in enumerate(mask) if v == 1), len(mask)) + packed_samples.append((ids, mask, prompt_len, old_logps)) + seq_lens = torch.tensor([len(ids) for ids, _, _, _ in packed_samples], device=args.device) + max_len = seq_lens.max().item() + input_ids = torch.tensor([ids + [tokenizer.pad_token_id] * (max_len - len(ids)) for ids, _, _, _ in packed_samples], device=args.device) + prompt_lens = torch.tensor([prompt_len for _, _, prompt_len, _ in packed_samples], device=args.device) + full_response_masks = torch.tensor([mask + [0] * (max_len - len(mask)) for _, mask, _, _ in packed_samples], device=args.device, dtype=torch.float32) + old_per_token_logps = torch.tensor([old_logps + [0.0] * ((max_len - 1) - len(old_logps)) for _, _, _, old_logps in packed_samples], device=args.device, dtype=torch.float32) + + model_unwrapped = model.module if isinstance(model, DistributedDataParallel) else model + with autocast_ctx: + res = model_unwrapped(input_ids) + aux_loss = res.aux_loss if lm_config.use_moe else torch.tensor(0.0, device=args.device) + logits = res.logits[:, :-1, :] + per_token_logps = F.log_softmax(logits, dim=-1).gather(2, input_ids[:, 1:].unsqueeze(-1)).squeeze(-1) + + with torch.no_grad(): + ref_per_token_logps = compute_per_token_logps(ref_model, input_ids, input_ids.size(1) - 1) + + completion_mask = full_response_masks[:, 1:] + is_eos = (input_ids[:, 1:] == tokenizer.eos_token_id) & completion_mask.bool() + eos_idx = torch.full((completion_mask.size(0),), completion_mask.size(1) - 1, device=args.device, dtype=torch.long) + has_eos = is_eos.any(dim=1) + eos_idx[has_eos] = is_eos.int().argmax(dim=1)[has_eos] + pos = torch.arange(completion_mask.size(1), device=args.device).unsqueeze(0) + completion_mask = completion_mask * (pos <= eos_idx.unsqueeze(1)).float() + token_counts = completion_mask.sum(dim=1) + valid_rows = token_counts > 0 + rewards = calculate_rewards(prompts, completions, gt_batch, tools_batch, args.num_generations, reward_model, device=args.device, turn_outputs_batch=turn_outputs_batch, unfinished_batch=unfinished_batch) + + if args.debug_mode and is_main_process() and step % args.debug_interval == 0: + for i in range(len(messages_batch)): + Logger(f"[DEBUG] step={step}, gt[{i}]: {repr(gt_batch[i])}") + Logger('-'*100) + for j in range(args.num_generations): + idx = i * args.num_generations + j + plen, slen = prompt_lens[idx].item(), seq_lens[idx].item() + Logger(f"{'=' * 30} [DEBUG] gen[{i}][{j}] CONTEXT_BEGIN {'=' * 30}") + Logger(contexts[idx]) + Logger(f"{'=' * 31} [DEBUG] gen[{i}][{j}] CONTEXT_END {'=' * 31}") + Logger(f"[DEBUG] gen[{i}][{j}] prompt_len={plen}, seq_len={slen}") + tokens = input_ids[idx, plen:slen].tolist() + text = tokenizer.decode(tokens, skip_special_tokens=False) + Logger(f"{'=' * 28} [DEBUG] gen[{i}][{j}] COMPLETION_BEGIN [{plen}:{slen}] {'=' * 28}") + Logger(text) + Logger(f"{'=' * 29} [DEBUG] gen[{i}][{j}] COMPLETION_END {'=' * 29}") + Logger(f"[DEBUG] gen[{i}][{j}] reward={rewards[idx].item():.4f}") + Logger('='*100) + + grouped_rewards = rewards.view(-1, args.num_generations) + mean_r = grouped_rewards.mean(dim=1).repeat_interleave(args.num_generations) + std_r = grouped_rewards.std(dim=1, unbiased=False).repeat_interleave(args.num_generations) + advantages = (rewards - mean_r) / (std_r + 1e-4) + + kl_div = ref_per_token_logps - per_token_logps + per_token_kl = torch.exp(kl_div) - kl_div - 1 + ratio = torch.exp(per_token_logps - old_per_token_logps) + if args.loss_type == "cispo": + clamped_ratio = torch.clamp(ratio, max=args.epsilon_high).detach() + per_token_loss = -(clamped_ratio * advantages.unsqueeze(1) * per_token_logps - args.beta * per_token_kl) + else: + clipped_ratio = torch.clamp(ratio, 1 - args.epsilon, 1 + args.epsilon) + per_token_loss1 = ratio * advantages.unsqueeze(1) + per_token_loss2 = clipped_ratio * advantages.unsqueeze(1) + per_token_loss = -(torch.min(per_token_loss1, per_token_loss2) - args.beta * per_token_kl) + policy_loss = (((per_token_loss * completion_mask).sum(dim=1)[valid_rows] / token_counts[valid_rows].clamp(min=1)).mean() + if valid_rows.any() else per_token_loss.sum() * 0.0) + loss = (policy_loss + aux_loss) / args.accumulation_steps + loss.backward() + + if step % args.accumulation_steps == 0: + if args.grad_clip > 0: torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) + optimizer.step(); scheduler.step(); optimizer.zero_grad() + if is_main_process() and step % args.save_interval == 0: rollout_engine.update_policy(model) + + if step % args.log_interval == 0 or step == iters: + pl = loss.item() * args.accumulation_steps + ar = rewards.mean().item() + al = token_counts.float().mean().item() + kl = ((ref_per_token_logps - per_token_logps) * completion_mask).sum().item() / max(token_counts.sum().item(), 1) + gs = grouped_rewards.std(dim=1, unbiased=False).mean().item() + am, ast = advantages.mean().item(), advantages.std().item() + lr = optimizer.param_groups[0]['lr'] + Logger(f'Epoch:[{epoch+1}/{args.epochs}]({step}/{iters}), Reward:{ar:.4f}, KL:{kl:.4f}, GrpStd:{gs:.4f}, AdvStd:{ast:.4f}, Loss:{pl:.4f}, AvgLen:{al:.2f}, AdvMean:{am:.4f}, LR:{lr:.8f}') + if wandb and is_main_process(): + wandb.log({"reward":ar,"kl_ref":kl,"group_reward_std":gs,"advantages_std":ast,"policy_loss":pl,"avg_response_len":al,"advantages_mean":am,"learning_rate":lr}) + + if (step % args.save_interval == 0 or step == iters) and is_main_process(): + model.eval() + moe_suffix = '_moe' if lm_config.use_moe else '' + ckp = f'{args.save_dir}/{args.save_weight}_{lm_config.hidden_size}{moe_suffix}.pth' + raw_model = model.module if isinstance(model, DistributedDataParallel) else model + raw_model = getattr(raw_model, '_orig_mod', raw_model) + state_dict = raw_model.state_dict() + torch.save({k: v.half().cpu() for k, v in state_dict.items()}, ckp) + lm_checkpoint(lm_config, weight=args.save_weight, model=model, optimizer=optimizer, + epoch=epoch, step=step, wandb=wandb, save_dir='../checkpoints', scheduler=scheduler) + model.train() + del state_dict + + del per_token_logps, ref_per_token_logps + del completions, rewards, grouped_rewards, mean_r, std_r, advantages, completion_mask + + if last_step > start_step and last_step % args.accumulation_steps != 0: + if args.grad_clip > 0: torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) + optimizer.step(); scheduler.step(); optimizer.zero_grad() + if is_main_process() and last_step % args.save_interval == 0: rollout_engine.update_policy(model) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="MiniMind Agent RL") + parser.add_argument("--save_dir", type=str, default="../out", help="模型保存目录") + parser.add_argument('--save_weight', default='agent', type=str, help="保存权重名称") + parser.add_argument("--epochs", type=int, default=1, help="训练轮数") + parser.add_argument("--batch_size", type=int, default=2, help="批次大小") + parser.add_argument("--learning_rate", type=float, default=3e-7, help="学习率") + parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备") + parser.add_argument("--dtype", type=str, default="bfloat16", help="数据类型 bfloat16/float16") + parser.add_argument("--num_workers", type=int, default=8, help="数据加载线程数") + parser.add_argument("--accumulation_steps", type=int, default=1, help="梯度累积步数") + parser.add_argument("--grad_clip", type=float, default=1.0, help="梯度裁剪阈值") + parser.add_argument("--log_interval", type=int, default=1, help="日志打印间隔") + parser.add_argument("--save_interval", type=int, default=10, help="模型保存间隔") + parser.add_argument('--hidden_size', default=768, type=int, help="模型隐藏层维度") + parser.add_argument('--num_hidden_layers', default=8, type=int, help="模型层数") + parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE") + parser.add_argument('--max_seq_len', default=1024, type=int, help="最大序列长度") + parser.add_argument("--max_gen_len", type=int, default=768, help="单次最大生成长度") + parser.add_argument("--max_total_len", type=int, default=2500, help="训练侧最终总长度上界") + parser.add_argument("--data_path", type=str, default="../dataset/agent_rl.jsonl", help="训练数据路径") + parser.add_argument("--num_generations", type=int, default=4, help="每个prompt生成数量") + parser.add_argument("--beta", type=float, default=0.1, help="KL散度惩罚系数") + parser.add_argument("--loss_type", type=str, default="cispo", choices=["grpo", "cispo"], help="loss类型") + parser.add_argument("--epsilon", type=float, default=0.2, help="GRPO的PPO clip epsilon") + parser.add_argument("--epsilon_high", type=float, default=5.0, help="epsilon上界") + parser.add_argument('--from_weight', default='full_sft', type=str, help="加载预训练权重名称") + parser.add_argument('--from_resume', default=0, type=int, choices=[0, 1], help="是否从checkpoint恢复") + parser.add_argument("--use_wandb", action="store_true", help="是否使用wandb记录") + parser.add_argument("--wandb_project", type=str, default="MiniMind-Agent-RL", help="wandb项目名称") + parser.add_argument("--use_compile", default=0, type=int, choices=[0, 1], help="是否使用torch.compile") + parser.add_argument("--debug_mode", action="store_true", help="调试模式") + parser.add_argument("--debug_interval", type=int, default=20, help="调试日志间隔") + parser.add_argument("--thinking_ratio", type=float, default=0.1, help="按概率开启thinking（0.0~1.0）") + parser.add_argument("--reward_model_path", type=str, default="../../internlm2-1_8b-reward", help="Reward模型路径") + parser.add_argument("--rollout_engine", type=str, default="sglang", choices=["torch", "sglang"], help="rollout引擎类型") + parser.add_argument("--sglang_base_url", type=str, default="http://localhost:8998", help="SGLang服务器URL") + parser.add_argument("--sglang_model_path", type=str, default="../model", help="SGLang tokenizer路径") + parser.add_argument("--sglang_shared_path", type=str, default="./sglang_ckpt_agent", help="SGLang共享存储路径") + args = parser.parse_args() + + local_rank = init_distributed_mode() + if dist.is_initialized(): args.device = f"cuda:{local_rank}" + setup_seed(42 + (dist.get_rank() if dist.is_initialized() else 0)) + + os.makedirs(args.save_dir, exist_ok=True) + lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, + max_seq_len=args.max_seq_len + args.max_gen_len, use_moe=bool(args.use_moe)) + ckp_data = lm_checkpoint(lm_config, weight=args.save_weight, save_dir='../checkpoints') if args.from_resume == 1 else None + + device_type = "cuda" if "cuda" in args.device else "cpu" + dtype = torch.bfloat16 if args.dtype == "bfloat16" else torch.float16 + autocast_ctx = nullcontext() if device_type == "cpu" else torch.cuda.amp.autocast(dtype=dtype) + + wandb = None + if args.use_wandb and is_main_process(): + import swanlab as wandb + wandb_id = ckp_data.get('wandb_id') if ckp_data else None + resume = 'must' if wandb_id else None + wandb.init(project=args.wandb_project, name=f"Agent-RL-E{args.epochs}-B{args.batch_size}-LR{args.learning_rate}", id=wandb_id, resume=resume) + + model, tokenizer = init_model(lm_config, args.from_weight, device=args.device) + + ref_model, _ = init_model(lm_config, args.from_weight, device=args.device) + ref_model = ref_model.eval().requires_grad_(False) + + reward_model = LMForRewardModel(args.reward_model_path, device=args.device, dtype=torch.float16) + Logger(f'Loaded reward model from {args.reward_model_path}') + # Rollout引擎 + rollout_engine = create_rollout_engine( + engine_type=args.rollout_engine, + policy_model=model, + tokenizer=tokenizer, + device=args.device, + autocast_ctx=autocast_ctx, + sglang_base_url=args.sglang_base_url, + sglang_model_path=args.sglang_model_path, + sglang_shared_path=args.sglang_shared_path, + ) + train_ds = AgentRLDataset(args.data_path, tokenizer, max_length=lm_config.max_seq_len) + train_sampler = DistributedSampler(train_ds) if dist.is_initialized() else None + optimizer = optim.AdamW(model.parameters(), lr=args.learning_rate) + def collate_fn(batch): return {'messages': [b['messages'] for b in batch], 'tools': [b['tools'] for b in batch], 'gt': [b['gt'] for b in batch]} + loader_for_count = DataLoader(train_ds, batch_size=args.batch_size, sampler=train_sampler, collate_fn=collate_fn) + iters = len(loader_for_count) + total_optimizer_steps = math.ceil(iters / args.accumulation_steps) * args.epochs + scheduler = CosineAnnealingLR(optimizer, T_max=total_optimizer_steps, eta_min=args.learning_rate / 10) + + start_epoch, start_step = 0, 0 + if ckp_data: + model.load_state_dict(ckp_data['model']) + optimizer.load_state_dict(ckp_data['optimizer']) + scheduler.load_state_dict(ckp_data['scheduler']) + start_epoch = ckp_data['epoch'] + start_step = ckp_data.get('step', 0) + + if args.use_compile == 1: + model = torch.compile(model) + Logger('torch.compile enabled') + if dist.is_initialized(): + model._ddp_params_and_buffers_to_ignore = {"freqs_cos", "freqs_sin"} + model = DistributedDataParallel(model, device_ids=[local_rank]) + if is_main_process(): rollout_engine.update_policy(model) + + for epoch in range(start_epoch, args.epochs): + train_sampler and train_sampler.set_epoch(epoch) + setup_seed(42 + epoch); indices = torch.randperm(len(train_ds)).tolist() + skip = start_step if (epoch == start_epoch and start_step > 0) else 0 + batch_sampler = SkipBatchSampler(train_sampler or indices, args.batch_size, skip) + loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True, collate_fn=collate_fn) + if skip > 0: + Logger(f'Epoch [{epoch+1}/{args.epochs}]: skip {start_step} steps') + rl_train_epoch(epoch, loader, len(loader) + skip, rollout_engine, ref_model, reward_model, start_step, wandb, use_sglang = (args.rollout_engine == "sglang")) + else: + rl_train_epoch(epoch, loader, len(loader), rollout_engine, ref_model, reward_model, 0, wandb, use_sglang = (args.rollout_engine == "sglang")) + + if dist.is_initialized(): dist.destroy_process_group() diff --git a/trainer/train_distillation.py b/trainer/train_distillation.py index dfbac0c..6b1b5b9 100644 --- a/trainer/train_distillation.py +++ b/trainer/train_distillation.py @@ -37,12 +37,14 @@ def distillation_loss(student_logits, teacher_logits, temperature=1.0, reduction def train_epoch(epoch, loader, iters, teacher_model, lm_config_student, start_step=0, wandb=None, alpha=0.0, temperature=1.0): start_time = time.time() + last_step = start_step if teacher_model is not None: teacher_model.eval() teacher_model.requires_grad_(False) for step, (input_ids, labels) in enumerate(loader, start=start_step + 1): + last_step = step input_ids = input_ids.to(args.device) labels = labels.to(args.device) loss_mask = (labels[..., 1:] != -100).float() @@ -104,7 +106,7 @@ def train_epoch(epoch, loader, iters, teacher_model, lm_config_student, start_st current_ce_loss = ce_loss_raw.item() current_aux_loss = res.aux_loss.item() if lm_config_student.use_moe else 0.0 current_lr = optimizer.param_groups[-1]['lr'] - eta_min = spend_time / step * iters // 60 - spend_time // 60 + eta_min = spend_time / max(step - start_step, 1) * (iters - step) // 60 Logger(f'Epoch:[{epoch + 1}/{args.epochs}]({step}/{iters}), loss: {current_loss:.4f}, ce: {current_ce_loss:.4f}, aux_loss: {current_aux_loss:.4f}, distill: {distill_loss.item():.4f}, learning_rate: {current_lr:.8f}, epoch_time: {eta_min:.3f}min') @@ -132,8 +134,16 @@ def train_epoch(epoch, loader, iters, teacher_model, lm_config_student, start_st del input_ids, labels, loss_mask, res, student_logits, ce_loss, distill_loss, loss + if last_step > start_step and last_step % args.accumulation_steps != 0: + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) + scaler.step(optimizer) + scaler.update() + optimizer.zero_grad(set_to_none=True) + if __name__ == "__main__": + # 模拟用moe模型蒸馏dense模型，也可以用更大teacher_hidden_size模型蒸馏更小student_hidden_size的 parser = argparse.ArgumentParser(description="MiniMind Knowledge Distillation") parser.add_argument("--save_dir", type=str, default="../out", help="模型保存目录") parser.add_argument('--save_weight', default='full_dist', type=str, help="保存权重的前缀名") @@ -148,12 +158,13 @@ if __name__ == "__main__": parser.add_argument("--log_interval", type=int, default=100, help="日志打印间隔") parser.add_argument("--save_interval", type=int, default=100, help="模型保存间隔") parser.add_argument("--max_seq_len", type=int, default=340, help="训练的最大截断长度（中文1token≈1.5~1.7字符）") - parser.add_argument("--data_path", type=str, default="../dataset/sft_mini_512.jsonl", help="训练数据路径") - parser.add_argument('--student_hidden_size', default=512, type=int, help="学生模型隐藏层维度") + parser.add_argument("--data_path", type=str, default="../dataset/sft_t2t_mini.jsonl", help="训练数据路径") + parser.add_argument('--student_hidden_size', default=768, type=int, help="学生模型隐藏层维度") parser.add_argument('--student_num_layers', default=8, type=int, help="学生模型隐藏层数量") parser.add_argument('--teacher_hidden_size', default=768, type=int, help="教师模型隐藏层维度") - parser.add_argument('--teacher_num_layers', default=16, type=int, help="教师模型隐藏层数量") - parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") + parser.add_argument('--teacher_num_layers', default=8, type=int, help="教师模型隐藏层数量") + parser.add_argument('--student_use_moe', default=0, type=int, choices=[0, 1], help="学生模型是否使用MoE（0=否，1=是）") + parser.add_argument('--teacher_use_moe', default=1, type=int, choices=[0, 1], help="教师模型是否使用MoE（0=否，1=是）") parser.add_argument('--from_student_weight', default='full_sft', type=str, help="学生模型基于哪个权重") parser.add_argument('--from_teacher_weight', default='full_sft', type=str, help="教师模型基于哪个权重") parser.add_argument('--from_resume', default=0, type=int, choices=[0, 1], help="是否自动检测&续训（0=否，1=是）") @@ -171,8 +182,8 @@ if __name__ == "__main__": # ========== 2. 配置目录、模型参数、检查ckp ========== os.makedirs(args.save_dir, exist_ok=True) - lm_config_student = MiniMindConfig(hidden_size=args.student_hidden_size, num_hidden_layers=args.student_num_layers, use_moe=bool(args.use_moe)) - lm_config_teacher = MiniMindConfig(hidden_size=args.teacher_hidden_size, num_hidden_layers=args.teacher_num_layers, use_moe=bool(args.use_moe)) + lm_config_student = MiniMindConfig(hidden_size=args.student_hidden_size, num_hidden_layers=args.student_num_layers, use_moe=bool(args.student_use_moe)) + lm_config_teacher = MiniMindConfig(hidden_size=args.teacher_hidden_size, num_hidden_layers=args.teacher_num_layers, use_moe=bool(args.teacher_use_moe)) ckp_data = lm_checkpoint(lm_config_student, weight=args.save_weight, save_dir='../checkpoints') if args.from_resume==1 else None # ========== 3. 设置混合精度 ========== @@ -191,9 +202,6 @@ if __name__ == "__main__": # ========== 5. 定义学生和教师模型 ========== model, tokenizer = init_model(lm_config_student, args.from_student_weight, device=args.device) - if args.use_compile == 1: - model = torch.compile(model) - Logger('torch.compile enabled') Logger(f'学生模型总参数量：{sum(p.numel() for p in model.parameters()) / 1e6:.3f} M') teacher_model, _ = init_model(lm_config_teacher, args.from_teacher_weight, device=args.device) teacher_model.eval() @@ -213,7 +221,10 @@ if __name__ == "__main__": start_epoch = ckp_data['epoch'] start_step = ckp_data.get('step', 0) - # ========== 7. DDP包模型 ========== + # ========== 7. 编译和分布式包装 ========== + if args.use_compile == 1: + model = torch.compile(model) + Logger('torch.compile enabled') if dist.is_initialized(): model._ddp_params_and_buffers_to_ignore = {"freqs_cos", "freqs_sin"} model = DistributedDataParallel(model, device_ids=[local_rank]) diff --git a/trainer/train_dpo.py b/trainer/train_dpo.py index dcfc350..1876e26 100644 --- a/trainer/train_dpo.py +++ b/trainer/train_dpo.py @@ -32,10 +32,8 @@ def logits_to_log_probs(logits, labels): def dpo_loss(ref_log_probs, policy_log_probs, mask, beta): # ref_log_probs 和 policy_log_probs 都是 shape: (batch_size, seq_len) - # https://github.com/jingyaogong/minimind/issues/298 - seq_lengths = mask.sum(dim=1, keepdim=True).clamp_min(1e-8) # 防止零长度mask导致除零NaN - ref_log_probs = (ref_log_probs * mask).sum(dim=1) / seq_lengths.squeeze() - policy_log_probs = (policy_log_probs * mask).sum(dim=1) / seq_lengths.squeeze() + ref_log_probs = (ref_log_probs * mask).sum(dim=1) + policy_log_probs = (policy_log_probs * mask).sum(dim=1) # 将 chosen 和 rejected 数据分开 batch_size = ref_log_probs.shape[0] @@ -53,8 +51,10 @@ def dpo_loss(ref_log_probs, policy_log_probs, mask, beta): def train_epoch(epoch, loader, iters, ref_model, lm_config, start_step=0, wandb=None, beta=0.1): start_time = time.time() - + last_step = start_step + for step, batch in enumerate(loader, start=start_step + 1): + last_step = step x_chosen = batch['x_chosen'].to(args.device) x_rejected = batch['x_rejected'].to(args.device) y_chosen = batch['y_chosen'].to(args.device) @@ -98,7 +98,7 @@ def train_epoch(epoch, loader, iters, ref_model, lm_config, start_step=0, wandb= current_dpo_loss = dpo_loss_val.item() current_aux_loss = outputs.aux_loss.item() current_lr = optimizer.param_groups[-1]['lr'] - eta_min = spend_time / step * iters // 60 - spend_time // 60 + eta_min = spend_time / max(step - start_step, 1) * (iters - step) // 60 Logger(f'Epoch:[{epoch + 1}/{args.epochs}]({step}/{iters}), loss: {current_loss:.4f}, dpo_loss: {current_dpo_loss:.4f}, aux_loss: {current_aux_loss:.4f}, learning_rate: {current_lr:.8f}, epoch_time: {eta_min:.3f}min') @@ -119,6 +119,13 @@ def train_epoch(epoch, loader, iters, ref_model, lm_config, start_step=0, wandb= del x_chosen, x_rejected, y_chosen, y_rejected, mask_chosen, mask_rejected, x, y, mask del ref_outputs, ref_logits, ref_log_probs, outputs, logits, policy_log_probs, loss + if last_step > start_step and last_step % args.accumulation_steps != 0: + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) + scaler.step(optimizer) + scaler.update() + optimizer.zero_grad(set_to_none=True) + if __name__ == "__main__": parser = argparse.ArgumentParser(description="MiniMind DPO (Direct Preference Optimization)") @@ -134,14 +141,14 @@ if __name__ == "__main__": parser.add_argument("--grad_clip", type=float, default=1.0, help="梯度裁剪阈值") parser.add_argument("--log_interval", type=int, default=100, help="日志打印间隔") parser.add_argument("--save_interval", type=int, default=100, help="模型保存间隔") - parser.add_argument('--hidden_size', default=512, type=int, help="隐藏层维度") + parser.add_argument('--hidden_size', default=768, type=int, help="隐藏层维度") parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") parser.add_argument('--max_seq_len', default=1024, type=int, help="训练的最大截断长度（中文1token≈1.5~1.7字符）") parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") parser.add_argument("--data_path", type=str, default="../dataset/dpo.jsonl", help="DPO训练数据路径") parser.add_argument('--from_weight', default='full_sft', type=str, help="基于哪个权重训练") parser.add_argument('--from_resume', default=0, type=int, choices=[0, 1], help="是否自动检测&续训（0=否，1=是）") - parser.add_argument('--beta', default=0.1, type=float, help="DPO中的beta参数") + parser.add_argument('--beta', default=0.15, type=float, help="DPO中的beta参数") parser.add_argument("--use_wandb", action="store_true", help="是否使用wandb") parser.add_argument("--wandb_project", type=str, default="MiniMind-DPO", help="wandb项目名") parser.add_argument("--use_compile", default=0, type=int, choices=[0, 1], help="是否使用torch.compile加速（0=否，1=是）") @@ -173,9 +180,6 @@ if __name__ == "__main__": # ========== 5. 定义模型和参考模型 ========== model, tokenizer = init_model(lm_config, args.from_weight, device=args.device) - if args.use_compile == 1: - model = torch.compile(model) - Logger('torch.compile enabled') Logger(f'策略模型总参数量：{sum(p.numel() for p in model.parameters()) / 1e6:.3f} M') # 初始化参考模型（ref_model冻结） ref_model, _ = init_model(lm_config, args.from_weight, device=args.device) @@ -197,7 +201,10 @@ if __name__ == "__main__": start_epoch = ckp_data['epoch'] start_step = ckp_data.get('step', 0) - # ========== 7. DDP包模型 ========== + # ========== 7. 编译和分布式包装 ========== + if args.use_compile == 1: + model = torch.compile(model) + Logger('torch.compile enabled') if dist.is_initialized(): model._ddp_params_and_buffers_to_ignore = {"freqs_cos", "freqs_sin"} model = DistributedDataParallel(model, device_ids=[local_rank]) diff --git a/trainer/train_full_sft.py b/trainer/train_full_sft.py index 43b2dd2..45af899 100644 --- a/trainer/train_full_sft.py +++ b/trainer/train_full_sft.py @@ -22,9 +22,11 @@ warnings.filterwarnings('ignore') def train_epoch(epoch, loader, iters, start_step=0, wandb=None): start_time = time.time() + last_step = start_step for step, (input_ids, labels) in enumerate(loader, start=start_step + 1): input_ids = input_ids.to(args.device) labels = labels.to(args.device) + last_step = step lr = get_lr(epoch * iters + step, args.epochs * iters, args.learning_rate) for param_group in optimizer.param_groups: param_group['lr'] = lr @@ -51,7 +53,7 @@ def train_epoch(epoch, loader, iters, start_step=0, wandb=None): current_aux_loss = res.aux_loss.item() if res.aux_loss is not None else 0.0 current_logits_loss = current_loss - current_aux_loss current_lr = optimizer.param_groups[-1]['lr'] - eta_min = spend_time / step * iters // 60 - spend_time // 60 + eta_min = spend_time / max(step - start_step, 1) * (iters - step) // 60 Logger(f'Epoch:[{epoch + 1}/{args.epochs}]({step}/{iters}), loss: {current_loss:.4f}, logits_loss: {current_logits_loss:.4f}, aux_loss: {current_aux_loss:.4f}, lr: {current_lr:.8f}, epoch_time: {eta_min:.1f}min') if wandb: wandb.log({"loss": current_loss, "logits_loss": current_logits_loss, "aux_loss": current_aux_loss, "learning_rate": current_lr, "epoch_time": eta_min}) @@ -70,6 +72,13 @@ def train_epoch(epoch, loader, iters, start_step=0, wandb=None): del input_ids, labels, res, loss + if last_step > start_step and last_step % args.accumulation_steps != 0: + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) + scaler.step(optimizer) + scaler.update() + optimizer.zero_grad(set_to_none=True) + if __name__ == "__main__": parser = argparse.ArgumentParser(description="MiniMind Full SFT") @@ -77,7 +86,7 @@ if __name__ == "__main__": parser.add_argument('--save_weight', default='full_sft', type=str, help="保存权重的前缀名") parser.add_argument("--epochs", type=int, default=2, help="训练轮数") parser.add_argument("--batch_size", type=int, default=16, help="batch size") - parser.add_argument("--learning_rate", type=float, default=1e-6, help="初始学习率") + parser.add_argument("--learning_rate", type=float, default=1e-5, help="初始学习率") parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备") parser.add_argument("--dtype", type=str, default="bfloat16", help="混合精度类型") parser.add_argument("--num_workers", type=int, default=8, help="数据加载线程数") @@ -85,11 +94,11 @@ if __name__ == "__main__": parser.add_argument("--grad_clip", type=float, default=1.0, help="梯度裁剪阈值") parser.add_argument("--log_interval", type=int, default=100, help="日志打印间隔") parser.add_argument("--save_interval", type=int, default=1000, help="模型保存间隔") - parser.add_argument('--hidden_size', default=512, type=int, help="隐藏层维度") + parser.add_argument('--hidden_size', default=768, type=int, help="隐藏层维度") parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") parser.add_argument('--max_seq_len', default=340, type=int, help="训练的最大截断长度（中文1token≈1.5~1.7字符）") parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") - parser.add_argument("--data_path", type=str, default="../dataset/sft_mini_512.jsonl", help="训练数据路径") + parser.add_argument("--data_path", type=str, default="../dataset/sft_t2t_mini.jsonl", help="训练数据路径") parser.add_argument('--from_weight', default='pretrain', type=str, help="基于哪个权重训练，为none则不基于任何权重训练") parser.add_argument('--from_resume', default=0, type=int, choices=[0, 1], help="是否自动检测&续训（0=否，1=是）") parser.add_argument("--use_wandb", action="store_true", help="是否使用wandb") @@ -123,9 +132,6 @@ if __name__ == "__main__": # ========== 5. 定义模型、数据、优化器 ========== model, tokenizer = init_model(lm_config, args.from_weight, device=args.device) - if args.use_compile == 1: - model = torch.compile(model) - Logger('torch.compile enabled') train_ds = SFTDataset(args.data_path, tokenizer, max_length=args.max_seq_len) train_sampler = DistributedSampler(train_ds) if dist.is_initialized() else None scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype == 'float16')) @@ -140,7 +146,10 @@ if __name__ == "__main__": start_epoch = ckp_data['epoch'] start_step = ckp_data.get('step', 0) - # ========== 7. DDP包模型 ========== + # ========== 7. 编译和分布式包装 ========== + if args.use_compile == 1: + model = torch.compile(model) + Logger('torch.compile enabled') if dist.is_initialized(): model._ddp_params_and_buffers_to_ignore = {"freqs_cos", "freqs_sin"} model = DistributedDataParallel(model, device_ids=[local_rank]) diff --git a/trainer/train_grpo.py b/trainer/train_grpo.py index dd95e0b..2d514e2 100755 --- a/trainer/train_grpo.py +++ b/trainer/train_grpo.py @@ -5,10 +5,12 @@ __package__ = "trainer" sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) import argparse +import math import re import gc import warnings import torch +import torch.nn.functional as F import torch.distributed as dist from transformers import AutoTokenizer from contextlib import nullcontext @@ -19,47 +21,24 @@ from torch.optim.lr_scheduler import CosineAnnealingLR from transformers import AutoModel from model.model_minimind import MiniMindConfig, MiniMindForCausalLM from dataset.lm_dataset import RLAIFDataset -from trainer.trainer_utils import Logger, is_main_process, lm_checkpoint, init_distributed_mode, setup_seed, SkipBatchSampler, init_model +from trainer.trainer_utils import Logger, is_main_process, lm_checkpoint, init_distributed_mode, setup_seed, SkipBatchSampler, init_model, LMForRewardModel +from trainer.rollout_engine import create_rollout_engine, compute_per_token_logps warnings.filterwarnings('ignore') -def calculate_rewards(prompts, responses, reward_model, reward_tokenizer): - """整合所有奖励函数计算总奖励""" - def reasoning_model_reward(rewards): - pattern = r"^\n.*?\n\n\n.*?\n$" - pattern2 = r"^\n.*?\n\n\n\n.*?\n$" - matches_pattern = [re.match(pattern, response, re.S) for response in responses] - matches_pattern2 = [re.match(pattern2, response, re.S) for response in responses] +def rep_penalty(text, n=3, cap=0.5): + toks = re.findall(r"\w+|[^\w\s]", text.lower()) + grams = [tuple(toks[i:i + n]) for i in range(len(toks) - n + 1)] + return min(cap, (len(grams) - len(set(grams))) * cap * 2 / len(grams)) if grams else 0.0 - format_rewards = [] - for match_pattern, match_pattern2 in zip(matches_pattern, matches_pattern2): - if match_pattern or match_pattern2: - format_rewards.append(0.5) - else: - format_rewards.append(0.0) - rewards += torch.tensor(format_rewards, device=args.device) - - def mark_num(text): - reward = 0 - if text.count("") == 1: reward += 0.25 - if text.count("") == 1: reward += 0.25 - if text.count("") == 1: reward += 0.25 - if text.count("") == 1: reward += 0.25 - return reward - - mark_rewards = [mark_num(response) for response in responses] - rewards += torch.tensor(mark_rewards, device=args.device) - return rewards +def calculate_rewards(prompts, responses, reward_model): rewards = torch.zeros(len(responses), device=args.device) - if args.reasoning == 1: - rewards = reasoning_model_reward(rewards) with torch.no_grad(): reward_model_scores = [] batch_size = len(prompts) - scale = 3.0 for i in range(batch_size): for j in range(args.num_generations): @@ -70,20 +49,16 @@ def calculate_rewards(prompts, responses, reward_model, reward_tokenizer): pattern = r"<\|im_start\|>(system|user|assistant)\s+(.*?)<\|im_end\|>" matches = re.findall(pattern, prompt, re.DOTALL) messages = [{"role": role, "content": content.strip()} for role, content in matches] + answer = response + rewards[response_idx] += 0.5 if 20 <= len(response.strip()) <= 800 else -0.5 + if '' in response: + thinking_content, answer_content = response.split('', 1) + rewards[response_idx] += 1.0 if 20 <= len(thinking_content.strip()) <= 300 else -0.5 + rewards[response_idx] += 0.25 if response.count('') == 1 else -0.25 + answer = answer_content.strip() + rewards[response_idx] -= rep_penalty(answer) - tmp_chat = messages + [{"role": "assistant", "content": response}] - score = reward_model.get_score(reward_tokenizer, tmp_chat) - score = max(min(score, scale), -scale) - - if args.reasoning == 1: - answer_match = re.search(r'(.*?)', response, re.DOTALL) - if answer_match: - answer_content = answer_match.group(1).strip() - tmp_chat = messages + [{"role": "assistant", "content": answer_content}] - answer_score = reward_model.get_score(reward_tokenizer, tmp_chat) - answer_score = max(min(answer_score, scale), -scale) - score = score * 0.4 + answer_score * 0.6 - + score = reward_model.get_score(messages, answer) reward_model_scores.append(score) reward_model_scores = torch.tensor(reward_model_scores, device=args.device) @@ -92,49 +67,61 @@ def calculate_rewards(prompts, responses, reward_model, reward_tokenizer): return rewards -def grpo_train_epoch(epoch, loader, iters, ref_model, reward_model, reward_tokenizer, start_step=0, wandb=None): +def grpo_train_epoch(epoch, loader, iters, rollout_engine, ref_model, reward_model, start_step=0, wandb=None, use_sglang=False): for step, batch in enumerate(loader, start=start_step + 1): prompts = batch['prompt'] # list[str], length B prompt_inputs = tokenizer(prompts, return_tensors="pt", padding=True, return_token_type_ids=False, - padding_side="left", add_special_tokens=False).to(args.device) # input_ids: [B, P], attention_mask: [B, P] + padding_side="left", add_special_tokens=False).to(args.device) if args.max_seq_len: prompt_inputs["input_ids"] = prompt_inputs["input_ids"][:, -args.max_seq_len:] prompt_inputs["attention_mask"] = prompt_inputs["attention_mask"][:, -args.max_seq_len:] - with torch.no_grad(): - # DDP 模型需要使用 .module 访问 generate 方法 - model_for_gen = model.module if isinstance(model, DistributedDataParallel) else model - outputs = model_for_gen.generate( - **prompt_inputs, max_new_tokens=args.max_gen_len, do_sample=True, temperature=0.8, - num_return_sequences=args.num_generations, pad_token_id=tokenizer.pad_token_id) # [B*num_gen, P+R] - - completion_ids = outputs[:, prompt_inputs["input_ids"].size(1):] # [B*num_gen, R] - - def get_per_token_logps(mdl, input_ids, n_keep): - input_ids = input_ids.detach().clone() if input_ids.is_inference() else input_ids - logits = mdl(input_ids, logits_to_keep=n_keep + 1).logits[:, :-1, :] - per_token_logps = [] - for logits_row, ids_row in zip(logits, input_ids[:, -n_keep:]): - ids_row = ids_row.detach().clone() if ids_row.is_inference() else ids_row - per_token_logps.append(torch.gather(logits_row.log_softmax(dim=-1), 1, ids_row.unsqueeze(1)).squeeze(1)) - return torch.stack(per_token_logps) + rollout_result = rollout_engine.rollout( + prompt_ids=prompt_inputs["input_ids"], + attention_mask=prompt_inputs["attention_mask"], + num_generations=args.num_generations, + max_new_tokens=args.max_gen_len, + temperature=0.8, + ) + outputs = rollout_result.output_ids + completion_ids = rollout_result.completion_ids + completions = rollout_result.completions + old_per_token_logps = rollout_result.per_token_logps.to(args.device) + model_unwrapped = model.module if isinstance(model, DistributedDataParallel) else model with autocast_ctx: - per_token_logps = get_per_token_logps(model, outputs, completion_ids.size(1)) # [B*num_gen, R] - res = model(outputs) if lm_config.use_moe else None - aux_loss = res.aux_loss if res is not None else torch.tensor(0.0, device=args.device) + if use_sglang or lm_config.use_moe: + res = model_unwrapped(outputs) + aux_loss = res.aux_loss if lm_config.use_moe else torch.tensor(0.0, device=args.device) + logits = res.logits[:, :-1, :] + per_token_logps = F.log_softmax(logits, dim=-1).gather(2, outputs[:, 1:].unsqueeze(-1)).squeeze(-1)[:, -completion_ids.size(1):] + else: + aux_loss = torch.tensor(0.0, device=args.device) + per_token_logps = rollout_result.per_token_logps with torch.no_grad(): - ref_per_token_logps = get_per_token_logps(ref_model, outputs, completion_ids.size(1)) # [B*num_gen, R] + ref_per_token_logps = compute_per_token_logps(ref_model, outputs, completion_ids.size(1)) + rewards = calculate_rewards(prompts, completions, reward_model).to(args.device) # [B*num_gen] - completions = tokenizer.batch_decode(completion_ids, skip_special_tokens=True) - rewards = calculate_rewards(prompts, completions, reward_model, reward_tokenizer).to(args.device) # [B*num_gen] + if args.debug_mode and is_main_process() and step % args.debug_interval == 0: + for i in range(len(prompts)): + Logger(f"[DEBUG] step={step}, sample[{i}]") + Logger('-'*100) + Logger(f"{'=' * 30} [DEBUG] sample[{i}] CONTEXT_BEGIN {'=' * 30}") + Logger(prompts[i]) + Logger(f"{'=' * 31} [DEBUG] sample[{i}] CONTEXT_END {'=' * 31}") + for j in range(args.num_generations): + idx = i * args.num_generations + j + Logger(f"{'=' * 28} [DEBUG] gen[{j}] RESPONSE_BEGIN {'=' * 28}") + Logger(completions[idx]) + Logger(f"{'=' * 29} [DEBUG] gen[{j}] RESPONSE_END {'=' * 29}") + Logger(f"[DEBUG] gen[{j}] reward={rewards[idx].item():.4f}") + Logger('='*100) grouped_rewards = rewards.view(-1, args.num_generations) # [B, num_gen] mean_r = grouped_rewards.mean(dim=1).repeat_interleave(args.num_generations) # [B*num_gen] std_r = grouped_rewards.std(dim=1).repeat_interleave(args.num_generations) # [B*num_gen] - advantages = torch.clamp((rewards - mean_r) / (std_r + 1e-4), -10, 10) - advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8) # [B*num_gen] + advantages = (rewards - mean_r) / (std_r + 1e-4) # [B*num_gen] is_eos = completion_ids == tokenizer.eos_token_id # [B*num_gen, R] eos_idx = torch.full((is_eos.size(0),), is_eos.size(1), dtype=torch.long, device=args.device) @@ -143,7 +130,15 @@ def grpo_train_epoch(epoch, loader, iters, ref_model, reward_model, reward_token kl_div = ref_per_token_logps - per_token_logps per_token_kl = torch.exp(kl_div) - kl_div - 1 # [B*num_gen, R] - per_token_loss = -(torch.exp(per_token_logps - per_token_logps.detach()) * advantages.unsqueeze(1) - args.beta * per_token_kl) # [B*num_gen, R] + ratio = torch.exp(per_token_logps - old_per_token_logps) # [B*num_gen, R] + if args.loss_type == "cispo": + clamped_ratio = torch.clamp(ratio, max=args.epsilon_high).detach() + per_token_loss = -(clamped_ratio * advantages.unsqueeze(1) * per_token_logps - args.beta * per_token_kl) + else: + clipped_ratio = torch.clamp(ratio, 1 - args.epsilon, 1 + args.epsilon) + per_token_loss1 = ratio * advantages.unsqueeze(1) + per_token_loss2 = clipped_ratio * advantages.unsqueeze(1) + per_token_loss = -(torch.min(per_token_loss1, per_token_loss2) - args.beta * per_token_kl) policy_loss = ((per_token_loss * completion_mask).sum(dim=1) / completion_mask.sum(dim=1)).mean() loss = (policy_loss + aux_loss) / args.accumulation_steps # scalar loss.backward() @@ -154,25 +149,31 @@ def grpo_train_epoch(epoch, loader, iters, ref_model, reward_model, reward_token optimizer.step() scheduler.step() optimizer.zero_grad() + if is_main_process() and step % args.save_interval == 0: rollout_engine.update_policy(model) if step % args.log_interval == 0 or step == iters: policy_loss_val = loss.item() * args.accumulation_steps current_aux_loss = aux_loss.item() avg_reward_val = rewards.mean().item() avg_len_val = completion_mask.sum(dim=1).float().mean().item() + kl_ref_val = ((ref_per_token_logps - per_token_logps) * completion_mask).sum().item() / completion_mask.sum().item() + advantages_mean_val = advantages.mean().item() + advantages_std_val = advantages.std().item() current_lr = optimizer.param_groups[0]['lr'] Logger(f'Epoch:[{epoch + 1}/{args.epochs}]({step}/{iters}), ' - f'Actor Loss: {policy_loss_val:.4f}, Aux Loss: {current_aux_loss:.4f}, Reward: {avg_reward_val:.4f}, ' - f'Avg Response Len: {avg_len_val:.2f}, Learning Rate: {current_lr:.8f}') + f'Reward: {avg_reward_val:.4f}, KL_ref: {kl_ref_val:.4f}, ' + f'Adv Std: {advantages_std_val:.4f}, Adv Mean: {advantages_mean_val:.4f}, ' + f'Actor Loss: {policy_loss_val:.4f}, Avg Response Len: {avg_len_val:.2f}, Learning Rate: {current_lr:.8f}') if wandb and is_main_process(): wandb.log({ - "policy_loss": policy_loss_val, - "aux_loss": current_aux_loss, "reward": avg_reward_val, + "kl_ref": kl_ref_val, + "advantages_std": advantages_std_val, + "advantages_mean": advantages_mean_val, + "policy_loss": policy_loss_val, "avg_response_len": avg_len_val, - "advantages_mean": advantages.mean().item(), "learning_rate": current_lr }) @@ -189,6 +190,14 @@ def grpo_train_epoch(epoch, loader, iters, ref_model, reward_model, reward_token model.train() del state_dict + if step > start_step and step % args.accumulation_steps != 0: + if args.grad_clip > 0: + torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) + optimizer.step() + scheduler.step() + optimizer.zero_grad() + if is_main_process() and step % args.save_interval == 0: rollout_engine.update_policy(model) + del prompt_inputs, outputs, completion_ids, per_token_logps, ref_per_token_logps del completions, rewards, grouped_rewards, mean_r, std_r, advantages, completion_mask @@ -199,7 +208,7 @@ if __name__ == "__main__": parser.add_argument('--save_weight', default='grpo', type=str, help="保存权重的前缀名") parser.add_argument("--epochs", type=int, default=1, help="训练轮数") parser.add_argument("--batch_size", type=int, default=2, help="batch size") - parser.add_argument("--learning_rate", type=float, default=8e-8, help="初始学习率") + parser.add_argument("--learning_rate", type=float, default=3e-7, help="初始学习率") parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备") parser.add_argument("--dtype", type=str, default="bfloat16", help="混合精度类型") parser.add_argument("--num_workers", type=int, default=8, help="数据加载线程数") @@ -207,20 +216,30 @@ if __name__ == "__main__": parser.add_argument("--grad_clip", type=float, default=1.0, help="梯度裁剪阈值") parser.add_argument("--log_interval", type=int, default=1, help="日志打印间隔") parser.add_argument("--save_interval", type=int, default=10, help="模型保存间隔") - parser.add_argument('--hidden_size', default=512, type=int, help="隐藏层维度") + parser.add_argument('--hidden_size', default=768, type=int, help="隐藏层维度") parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") - parser.add_argument('--max_seq_len', default=66, type=int, help="Prompt最大长度") - parser.add_argument("--max_gen_len", type=int, default=1536, help="生成的最大长度") - parser.add_argument("--data_path", type=str, default="../dataset/rlaif-mini.jsonl", help="RLAIF数据路径") - parser.add_argument("--num_generations", type=int, default=8, help="每个prompt生成的样本数") - parser.add_argument("--beta", type=float, default=0.02, help="KL惩罚系数") - parser.add_argument("--reasoning", type=int, default=1, choices=[0, 1], help='推理模型类型（0=普通模型，1=推理模型）') + parser.add_argument('--max_seq_len', default=768, type=int, help="Prompt最大长度") + parser.add_argument("--max_gen_len", type=int, default=1024, help="生成的最大长度") + parser.add_argument("--data_path", type=str, default="../dataset/rlaif.jsonl", help="RLAIF数据路径") + parser.add_argument("--num_generations", type=int, default=6, help="每个prompt生成的样本数") + parser.add_argument("--beta", type=float, default=0.1, help="KL惩罚系数") + parser.add_argument("--loss_type", type=str, default="cispo", choices=["grpo", "cispo"], help="loss类型") + parser.add_argument("--epsilon", type=float, default=0.2, help="GRPO的PPO clip epsilon") + parser.add_argument("--epsilon_high", type=float, default=5.0, help="epsilon上界") + parser.add_argument('--from_weight', default='full_sft', type=str, help="基于哪个权重训练") parser.add_argument("--reward_model_path", type=str, default="../../internlm2-1_8b-reward", help="Reward模型路径") parser.add_argument('--from_resume', default=0, type=int, choices=[0, 1], help="是否自动检测&续训（0=否，1=是）") parser.add_argument("--use_wandb", action="store_true", help="是否使用wandb") parser.add_argument("--wandb_project", type=str, default="MiniMind-GRPO", help="wandb项目名") parser.add_argument("--use_compile", default=0, type=int, choices=[0, 1], help="是否使用torch.compile加速（0=否，1=是）") + parser.add_argument("--debug_mode", action="store_true", help="是否打印训练调试采样") + parser.add_argument("--debug_interval", type=int, default=20, help="debug模式下每隔多少step打印一次采样") + parser.add_argument("--thinking_ratio", type=float, default=0.9, help="按概率开启thinking（0.0~1.0）") + parser.add_argument("--rollout_engine", type=str, default="sglang", choices=["torch", "sglang"], help="rollout引擎类型") + parser.add_argument("--sglang_base_url", type=str, default="http://localhost:8996", help="SGLang服务器URL") + parser.add_argument("--sglang_model_path", type=str, default="../model", help="SGLang tokenizer路径") + parser.add_argument("--sglang_shared_path", type=str, default="./sglang_ckpt_grpo", help="SGLang共享存储路径") args = parser.parse_args() # ========== 1. 初始化环境和随机种子 ========== @@ -249,28 +268,32 @@ if __name__ == "__main__": wandb.init(project=args.wandb_project, name=wandb_run_name, id=wandb_id, resume=resume) # ========== 5. 初始化模型和数据 ========== - base_weight = "reason" if args.reasoning == 1 else "full_sft" + base_weight = args.from_weight # Policy模型 model, tokenizer = init_model(lm_config, base_weight, device=args.device) - if args.use_compile == 1: - model = torch.compile(model) - Logger('torch.compile enabled') # Reference模型 ref_model, _ = init_model(lm_config, base_weight, device=args.device) ref_model = ref_model.eval().requires_grad_(False) # Reward模型 - reward_model = AutoModel.from_pretrained( - args.reward_model_path, torch_dtype=torch.float16, trust_remote_code=True + reward_model = LMForRewardModel(args.reward_model_path, device=args.device, dtype=torch.float16) + # Rollout引擎（可插拔替换，只负责 policy 推理） + rollout_engine = create_rollout_engine( + engine_type=args.rollout_engine, + policy_model=model, + tokenizer=tokenizer, + device=args.device, + autocast_ctx=autocast_ctx, + sglang_base_url=args.sglang_base_url, + sglang_model_path=args.sglang_model_path, + sglang_shared_path=args.sglang_shared_path, ) - reward_model = reward_model.to(args.device).eval().requires_grad_(False) - reward_tokenizer = AutoTokenizer.from_pretrained(args.reward_model_path, trust_remote_code=True) # 数据和优化器 - train_ds = RLAIFDataset(args.data_path, tokenizer, max_length=lm_config.max_seq_len) + train_ds = RLAIFDataset(args.data_path, tokenizer, max_length=lm_config.max_seq_len, thinking_ratio=args.thinking_ratio) train_sampler = DistributedSampler(train_ds) if dist.is_initialized() else None optimizer = optim.AdamW(model.parameters(), lr=args.learning_rate) loader_for_count = DataLoader(train_ds, batch_size=args.batch_size, sampler=train_sampler) iters = len(loader_for_count) - total_optimizer_steps = (iters // args.accumulation_steps) * args.epochs + total_optimizer_steps = math.ceil(iters / args.accumulation_steps) * args.epochs scheduler = CosineAnnealingLR(optimizer, T_max=total_optimizer_steps, eta_min=args.learning_rate / 10) # ========== 6. 从ckp恢复状态 ========== @@ -282,10 +305,15 @@ if __name__ == "__main__": start_epoch = ckp_data['epoch'] start_step = ckp_data.get('step', 0) - # ========== 7. DDP包模型 ========== + # ========== 7. 编译和分布式包装 ========== + if args.use_compile == 1: + model = torch.compile(model) + Logger('torch.compile enabled') + rollout_engine.update_policy(model) if dist.is_initialized(): model._ddp_params_and_buffers_to_ignore = {"freqs_cos", "freqs_sin"} model = DistributedDataParallel(model, device_ids=[local_rank]) + if is_main_process(): rollout_engine.update_policy(model) # ========== 8. 开始训练 ========== for epoch in range(start_epoch, args.epochs): @@ -296,9 +324,9 @@ if __name__ == "__main__": loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True) if skip > 0: Logger(f'Epoch [{epoch + 1}/{args.epochs}]: 跳过前{start_step}个step，从step {start_step + 1}开始') - grpo_train_epoch(epoch, loader, len(loader) + skip, ref_model, reward_model, reward_tokenizer, start_step, wandb) + grpo_train_epoch(epoch, loader, len(loader) + skip, rollout_engine, ref_model, reward_model, start_step, wandb, use_sglang = (args.rollout_engine == "sglang")) else: - grpo_train_epoch(epoch, loader, len(loader), ref_model, reward_model, reward_tokenizer, 0, wandb) + grpo_train_epoch(epoch, loader, len(loader), rollout_engine, ref_model, reward_model, 0, wandb, use_sglang = (args.rollout_engine == "sglang")) # ========== 9. 清理分布进程 ========== if dist.is_initialized(): dist.destroy_process_group() \ No newline at end of file diff --git a/trainer/train_lora.py b/trainer/train_lora.py index fc66cde..4dae568 100644 --- a/trainer/train_lora.py +++ b/trainer/train_lora.py @@ -23,9 +23,11 @@ warnings.filterwarnings('ignore') def train_epoch(epoch, loader, iters, lora_params, start_step=0, wandb=None): start_time = time.time() + last_step = start_step for step, (input_ids, labels) in enumerate(loader, start=start_step + 1): input_ids = input_ids.to(args.device) labels = labels.to(args.device) + last_step = step lr = get_lr(epoch * iters + step, args.epochs * iters, args.learning_rate) for param_group in optimizer.param_groups: param_group['lr'] = lr @@ -50,7 +52,7 @@ def train_epoch(epoch, loader, iters, lora_params, start_step=0, wandb=None): current_aux_loss = res.aux_loss.item() if res.aux_loss is not None else 0.0 current_logits_loss = current_loss - current_aux_loss current_lr = optimizer.param_groups[-1]['lr'] - eta_min = spend_time / step * iters // 60 - spend_time // 60 + eta_min = spend_time / max(step - start_step, 1) * (iters - step) // 60 Logger(f'Epoch:[{epoch + 1}/{args.epochs}]({step}/{iters}), loss: {current_loss:.4f}, logits_loss: {current_logits_loss:.4f}, aux_loss: {current_aux_loss:.4f}, lr: {current_lr:.8f}, epoch_time: {eta_min:.1f}min') if wandb: wandb.log({"loss": current_loss, "logits_loss": current_logits_loss, "aux_loss": current_aux_loss, "learning_rate": current_lr, "epoch_time": eta_min}) @@ -64,12 +66,18 @@ def train_epoch(epoch, loader, iters, lora_params, start_step=0, wandb=None): del input_ids, labels, res, loss + if last_step > start_step and last_step % args.accumulation_steps != 0: + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(lora_params, args.grad_clip) + scaler.step(optimizer) + scaler.update() + optimizer.zero_grad(set_to_none=True) if __name__ == "__main__": parser = argparse.ArgumentParser(description="MiniMind LoRA Fine-tuning") - parser.add_argument("--save_dir", type=str, default="../out/lora", help="模型保存目录") - parser.add_argument("--lora_name", type=str, default="lora_identity", help="LoRA权重名称(如lora_identity/lora_medical等)") - parser.add_argument("--epochs", type=int, default=50, help="训练轮数") + parser.add_argument("--save_dir", type=str, default="../out", help="模型保存目录") + parser.add_argument("--lora_name", type=str, default="lora_medical", help="LoRA权重名称(如lora_identity/lora_medical等)") + parser.add_argument("--epochs", type=int, default=10, help="训练轮数") parser.add_argument("--batch_size", type=int, default=32, help="batch size") parser.add_argument("--learning_rate", type=float, default=1e-4, help="初始学习率") parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备") @@ -79,11 +87,11 @@ if __name__ == "__main__": parser.add_argument("--grad_clip", type=float, default=1.0, help="梯度裁剪阈值") parser.add_argument("--log_interval", type=int, default=10, help="日志打印间隔") parser.add_argument("--save_interval", type=int, default=1000, help="模型保存间隔") - parser.add_argument('--hidden_size', default=512, type=int, help="隐藏层维度") + parser.add_argument('--hidden_size', default=768, type=int, help="隐藏层维度") parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") parser.add_argument('--max_seq_len', default=340, type=int, help="训练的最大截断长度（中文1token≈1.5~1.7字符）") parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") - parser.add_argument("--data_path", type=str, default="../dataset/lora_identity.jsonl", help="LoRA训练数据路径") + parser.add_argument("--data_path", type=str, default="../dataset/lora_medical.jsonl", help="LoRA训练数据路径") parser.add_argument('--from_weight', default='full_sft', type=str, help="基于哪个权重训练，默认full_sft") parser.add_argument('--from_resume', default=0, type=int, choices=[0, 1], help="是否自动检测&续训（0=否，1=是）") parser.add_argument("--use_wandb", action="store_true", help="是否使用wandb") @@ -117,9 +125,6 @@ if __name__ == "__main__": # ========== 5. 定义模型、应用LoRA、冻结非LoRA参数 ========== model, tokenizer = init_model(lm_config, args.from_weight, device=args.device) - if args.use_compile == 1: - model = torch.compile(model) - Logger('torch.compile enabled') apply_lora(model) # 统计参数 @@ -153,7 +158,10 @@ if __name__ == "__main__": start_epoch = ckp_data['epoch'] start_step = ckp_data.get('step', 0) - # ========== 8. DDP包模型 ========== + # ========== 8. 编译和分布式包装 ========== + if args.use_compile == 1: + model = torch.compile(model) + Logger('torch.compile enabled') if dist.is_initialized(): model._ddp_params_and_buffers_to_ignore = {"freqs_cos", "freqs_sin"} model = DistributedDataParallel(model, device_ids=[local_rank]) diff --git a/trainer/train_ppo.py b/trainer/train_ppo.py index f7e06f6..27668ec 100644 --- a/trainer/train_ppo.py +++ b/trainer/train_ppo.py @@ -5,6 +5,7 @@ __package__ = "trainer" sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) import argparse +import math import re import warnings import torch @@ -17,14 +18,20 @@ from torch.nn.parallel import DistributedDataParallel from torch.utils.data import DataLoader, DistributedSampler from torch.nn.utils import clip_grad_norm_ from torch.optim.lr_scheduler import CosineAnnealingLR -from transformers import AutoModel from model.model_minimind import MiniMindConfig, MiniMindForCausalLM from dataset.lm_dataset import RLAIFDataset -from trainer.trainer_utils import Logger, is_main_process, lm_checkpoint, init_distributed_mode, setup_seed, SkipBatchSampler, init_model +from trainer.trainer_utils import Logger, is_main_process, lm_checkpoint, init_distributed_mode, setup_seed, SkipBatchSampler, init_model, LMForRewardModel +from trainer.rollout_engine import create_rollout_engine warnings.filterwarnings('ignore') +def rep_penalty(text, n=3, cap=0.5): + toks = re.findall(r"\w+|[^\w\s]", text.lower()) + grams = [tuple(toks[i:i + n]) for i in range(len(toks) - n + 1)] + return min(cap, (len(grams) - len(set(grams))) * cap * 2 / len(grams)) if grams else 0.0 + + # 自定义的Critic模型，继承自MiniMindLM class CriticModel(MiniMindForCausalLM): def __init__(self, params): @@ -41,73 +48,25 @@ class CriticModel(MiniMindForCausalLM): return values -def calculate_rewards(prompts, responses, reward_model, reward_tokenizer): - """整合所有奖励函数计算总奖励""" - def reasoning_model_reward(rewards): - # 1. 格式奖励（仅针对训练推理模型时使用） - pattern = r"^\n.*?\n\n\n.*?\n$" - pattern2 = r"^\n.*?\n\n\n\n.*?\n$" - - matches_pattern = [re.match(pattern, response, re.S) for response in responses] - matches_pattern2 = [re.match(pattern2, response, re.S) for response in responses] - - format_rewards = [] - for match_pattern, match_pattern2 in zip(matches_pattern, matches_pattern2): - if match_pattern: - format_rewards.append(0.5) - elif match_pattern2: - format_rewards.append(0.5) - else: - format_rewards.append(0.0) - rewards += torch.tensor(format_rewards, device=args.device) - - # 2. 标记奖励（防止严格奖励稀疏，仅针对训练推理模型时使用） - def mark_num(text): - reward = 0 - if text.count("") == 1: - reward += 0.25 - if text.count("") == 1: - reward += 0.25 - if text.count("") == 1: - reward += 0.25 - if text.count("") == 1: - reward += 0.25 - return reward - - mark_rewards = [mark_num(response) for response in responses] - rewards += torch.tensor(mark_rewards, device=args.device) - return rewards - +def calculate_rewards(prompts, responses, reward_model): rewards = torch.zeros(len(responses), device=args.device) - # 格式奖励 - if args.reasoning == 1: - rewards = reasoning_model_reward(rewards) - - # 使用reward model计算整个response的奖励 with torch.no_grad(): reward_model_scores = [] - for prompt, response in zip(prompts, responses): + for i, (prompt, response) in enumerate(zip(prompts, responses)): pattern = r"<\|im_start\|>(system|user|assistant)\s+(.*?)<\|im_end\|>" matches = re.findall(pattern, prompt, re.DOTALL) messages = [{"role": role, "content": content.strip()} for role, content in matches] + answer = response + rewards[i] += 0.5 if 20 <= len(response.strip()) <= 800 else -0.5 + if '' in response: + thinking_content, answer_content = response.split('', 1) + rewards[i] += 1.0 if 20 <= len(thinking_content.strip()) <= 300 else -0.5 + rewards[i] += 0.25 if response.count('') == 1 else -0.25 + answer = answer_content.strip() + rewards[i] -= rep_penalty(answer) - tmp_chat = messages + [{"role": "assistant", "content": response}] - score = reward_model.get_score(reward_tokenizer, tmp_chat) - - scale = 3.0 - score = max(min(score, scale), -scale) - - # 当args.reasoning=1时，额外计算内容的奖励 - if args.reasoning == 1: - answer_match = re.search(r'(.*?)', response, re.DOTALL) - if answer_match: - answer_content = answer_match.group(1).strip() - # 对answer内容单独计算reward - tmp_chat = messages + [{"role": "assistant", "content": answer_content}] - answer_score = reward_model.get_score(reward_tokenizer, tmp_chat) - answer_score = max(min(answer_score, scale), -scale) - score = score * 0.4 + answer_score * 0.6 + score = reward_model.get_score(messages, answer) reward_model_scores.append(score) reward_model_scores = torch.tensor(reward_model_scores, device=args.device) @@ -116,65 +75,172 @@ def calculate_rewards(prompts, responses, reward_model, reward_tokenizer): return rewards -def ppo_train_epoch(epoch, loader, iters, old_actor_model, ref_model, actor_scheduler, critic_scheduler, reward_model, reward_tokenizer, start_step=0, wandb=None): +def ppo_train_epoch(epoch, loader, iters, rollout_engine, ref_model, actor_scheduler, critic_scheduler, reward_model, start_step=0, wandb=None, use_sglang=False): actor_model.train() critic_model.train() + grad_accum_step = 0 for step, batch in enumerate(loader, start=start_step + 1): prompts = batch["prompt"] # list[str], length B - enc = tokenizer(prompts, return_tensors="pt", padding=True, truncation=True, - max_length=args.max_seq_len, padding_side="left").to(args.device) # input_ids: [B, P], attention_mask: [B, P] + enc = tokenizer(prompts, return_tensors="pt", padding=True, truncation=True, max_length=args.max_seq_len, + padding_side="left").to(args.device) # input_ids: [B, P], attention_mask: [B, P] prompt_length = enc.input_ids.shape[1] - with torch.no_grad(): - # DDP 模型需要使用 .module 访问 generate 方法 - model_for_gen = actor_model.module if isinstance(actor_model, DistributedDataParallel) else actor_model - gen_out = model_for_gen.generate( - input_ids=enc.input_ids, attention_mask=enc.attention_mask, - max_new_tokens=args.max_gen_len, do_sample=True, temperature=0.8, - pad_token_id=tokenizer.pad_token_id, eos_token_id=tokenizer.eos_token_id) # [B, P+R] + rollout_result = rollout_engine.rollout( + prompt_ids=enc.input_ids, + attention_mask=enc.attention_mask, + num_generations=1, + max_new_tokens=args.max_gen_len, + temperature=0.8, + ) + gen_out = rollout_result.output_ids + responses_text = rollout_result.completions + rewards = calculate_rewards(prompts, responses_text, reward_model) # [B] - responses_text = [tokenizer.decode(gen_out[i, prompt_length:], skip_special_tokens=True) for i in range(len(prompts))] - rewards = calculate_rewards(prompts, responses_text, reward_model, reward_tokenizer) # [B] + if args.debug_mode and is_main_process() and step % args.debug_interval == 0: + for i in range(len(prompts)): + Logger(f"[DEBUG] step={step}, sample[{i}]") + Logger('-'*100) + Logger(f"{'=' * 30} [DEBUG] sample[{i}] CONTEXT_BEGIN {'=' * 30}") + Logger(prompts[i]) + Logger(f"{'=' * 31} [DEBUG] sample[{i}] CONTEXT_END {'=' * 31}") + Logger(f"[DEBUG] prompt_len={prompt_length}, response_len={len(responses_text[i])}") + Logger(f"{'=' * 28} [DEBUG] sample[{i}] RESPONSE_BEGIN {'=' * 28}") + Logger(responses_text[i]) + Logger(f"{'=' * 29} [DEBUG] sample[{i}] RESPONSE_END {'=' * 29}") + Logger(f"[DEBUG] reward={rewards[i].item():.4f}") + Logger('='*100) full_mask = (gen_out != tokenizer.pad_token_id).long() # [B, P+R] - values_seq = critic_model(input_ids=gen_out, attention_mask=full_mask) # [B, P+R] - last_indices = (full_mask * torch.arange(full_mask.size(1), device=gen_out.device)).argmax(dim=1) - values = values_seq[torch.arange(values_seq.size(0), device=values_seq.device), last_indices] # [B] - advantages = rewards - values.detach() # [B] - - with autocast_ctx: - res = actor_model(input_ids=gen_out, attention_mask=full_mask) - logits = res.logits # [B, P+R, V] - aux_loss = res.aux_loss if lm_config.use_moe else torch.tensor(0.0, device=args.device) - labels = gen_out[:, 1:].clone() # [B, P+R-1] - logp_tokens = F.log_softmax(logits[:, :-1], dim=-1).gather(2, labels.unsqueeze(-1)).squeeze(-1) # [B, P+R-1] - seq_len = gen_out.size(1) - 1 - resp_mask = torch.arange(seq_len, device=gen_out.device).unsqueeze(0) >= prompt_length - 1 - final_mask = resp_mask & (~labels.eq(tokenizer.pad_token_id)) # [B, P+R-1] - actor_logp = (logp_tokens * final_mask).sum(dim=1) # [B] + seq_len, resp_start = gen_out.size(1) - 1, prompt_length - 1 + resp_mask = torch.arange(seq_len, device=gen_out.device).unsqueeze(0) >= resp_start + final_mask = (resp_mask & (~labels.eq(tokenizer.pad_token_id))).float() # [B, P+R-1] + B = len(prompts) + resp_labels = labels[:, resp_start:] # [B, R] + resp_idx = torch.arange(resp_labels.size(1), device=gen_out.device).unsqueeze(0) + resp_pad_mask = ~resp_labels.eq(tokenizer.pad_token_id) + resp_lengths = resp_pad_mask.sum(dim=1); eos_mask = resp_labels.eq(tokenizer.eos_token_id) & resp_pad_mask + has_eos = eos_mask.any(dim=1); eos_pos = torch.argmax(eos_mask.int(), dim=1) + resp_lengths = torch.where(has_eos, eos_pos + 1, resp_lengths).long().clamp(min=1) + resp_policy_mask = ((resp_idx < resp_lengths.unsqueeze(1)) & resp_pad_mask).float() + resp_value_mask = resp_policy_mask.clone() - with torch.no_grad(): - old_logits = old_actor_model(input_ids=gen_out, attention_mask=full_mask).logits # [B, P+R, V] - old_logp_tokens = F.log_softmax(old_logits[:, :-1], dim=-1).gather(2, labels.unsqueeze(-1)).squeeze(-1) # [B, P+R-1] - old_logp = (old_logp_tokens * final_mask).sum(dim=1) # [B] + with torch.no_grad(): # Rollout阶段只需推理获取old_logp和old_values，切断梯度省显存 + critic_for_rollout = critic_model.module if isinstance(critic_model, DistributedDataParallel) else critic_model + values_seq = critic_for_rollout(input_ids=gen_out, attention_mask=full_mask) + old_resp_values = values_seq[:, resp_start:-1] * resp_value_mask - ref_logits = ref_model(input_ids=gen_out, attention_mask=full_mask).logits # [B, P+R, V] - ref_logp_tokens = F.log_softmax(ref_logits[:, :-1], dim=-1).gather(2, labels.unsqueeze(-1)).squeeze(-1) # [B, P+R-1] - ref_logp = (ref_logp_tokens * final_mask).sum(dim=1) # [B] + actor_for_rollout = actor_model.module if isinstance(actor_model, DistributedDataParallel) else actor_model + with autocast_ctx: + logits = actor_for_rollout(input_ids=gen_out, attention_mask=full_mask).logits + + old_resp_logp = F.log_softmax(logits[:, :-1], dim=-1).gather(2, labels.unsqueeze(-1)).squeeze(-1)[:, resp_start:] + + ref_logp_all = F.log_softmax(ref_model(input_ids=gen_out, attention_mask=full_mask).logits[:, :-1], dim=-1).gather(2, labels.unsqueeze(-1)).squeeze(-1) + ref_resp_logp = ref_logp_all[:, resp_start:] + token_rewards = torch.zeros_like(old_resp_logp) + last_idx = resp_lengths - 1 # [B] + token_rewards[torch.arange(B, device=args.device), last_idx] += rewards # 末尾加外部奖励 - kl = (actor_logp - old_logp).mean() # scalar - kl_ref = (actor_logp - ref_logp).mean() # scalar - ratio = torch.exp(actor_logp - old_logp) # [B] - surr1 = ratio * advantages # [B] - surr2 = torch.clamp(ratio, 1.0 - args.clip_epsilon, 1.0 + args.clip_epsilon) * advantages # [B] - policy_loss = -torch.min(surr1, surr2).mean() # scalar - value_loss = F.mse_loss(values, rewards) # scalar - loss = (policy_loss + args.vf_coef * value_loss + args.kl_coef * kl_ref + aux_loss) / args.accumulation_steps # scalar - loss.backward() + gen_len = old_resp_values.size(1); lastgaelam = torch.zeros(B, device=args.device); advs_rev = [] + for t in reversed(range(gen_len)): + nv = old_resp_values[:, t + 1] if t < gen_len - 1 else 0.0 + delta = token_rewards[:, t] + args.gamma * nv - old_resp_values[:, t] + lastgaelam = delta + args.gamma * args.lam * lastgaelam + advs_rev.append(lastgaelam) + advantages = torch.stack(advs_rev[::-1], dim=1) # [B, R] + returns = advantages + old_resp_values # [B, R] - if step % args.accumulation_steps == 0: + adv_mean = (advantages * resp_policy_mask).sum() / resp_policy_mask.sum().clamp(min=1) + adv_var = ((advantages - adv_mean) ** 2 * resp_policy_mask).sum() / resp_policy_mask.sum().clamp(min=1) + advantages = (advantages - adv_mean) * torch.rsqrt(adv_var + 1e-8) * resp_policy_mask + + mb_size = max(1, min(args.mini_batch_size, B)) + stop_ppo = False + policy_loss_sum = 0.0 + value_loss_sum = 0.0 + kl_sum = 0.0 + kl_ref_sum = 0.0 + clipfrac_sum = 0.0 + aux_loss_sum = 0.0 + log_count = 0 + actor_unwrapped = actor_model.module if isinstance(actor_model, DistributedDataParallel) else actor_model + critic_unwrapped = critic_model.module if isinstance(critic_model, DistributedDataParallel) else critic_model + for ppo_epoch in range(args.ppo_update_iters): + if stop_ppo: + break + b_inds = torch.randperm(B, device=args.device) + for i in range(0, B, mb_size): + inds = b_inds[i:i + mb_size] + + mb_values_seq = critic_unwrapped(input_ids=gen_out[inds], attention_mask=full_mask[inds]) + mb_resp_values = mb_values_seq[:, resp_start:-1] + + with autocast_ctx: + res = actor_unwrapped(input_ids=gen_out[inds], attention_mask=full_mask[inds]) + aux_loss = res.aux_loss if lm_config.use_moe else torch.tensor(0.0, device=args.device) + + mb_logp_all = F.log_softmax(res.logits[:, :-1], dim=-1).gather(2, labels[inds].unsqueeze(-1)).squeeze(-1) + mb_resp_logp = mb_logp_all[:, resp_start:] + + log_ratio = mb_resp_logp - old_resp_logp[inds] + approx_kl = (0.5 * (log_ratio ** 2) * resp_policy_mask[inds]).sum() / resp_policy_mask[inds].sum().clamp(min=1) + + # 同步各卡的 approx_kl，防止某卡 break 而其它卡继续导致 DDP 死锁 + approx_kl_val = approx_kl.detach().clone() + if dist.is_initialized(): + dist.all_reduce(approx_kl_val, op=dist.ReduceOp.AVG) + + if approx_kl_val > args.early_stop_kl: + stop_ppo = True + + ratio = torch.exp(log_ratio) + clipfrac = ((((ratio - 1.0).abs() > args.clip_epsilon).float() * resp_policy_mask[inds]).sum() + / resp_policy_mask[inds].sum().clamp(min=1)) + kl_ref_penalty = ((torch.exp(ref_resp_logp[inds] - mb_resp_logp) - (ref_resp_logp[inds] - mb_resp_logp) - 1.0) + * resp_policy_mask[inds]).sum() / resp_policy_mask[inds].sum().clamp(min=1) + policy_loss = ((torch.max(-advantages[inds] * ratio, + -advantages[inds] * torch.clamp(ratio, 1.0 - args.clip_epsilon, 1.0 + args.clip_epsilon)) + * resp_policy_mask[inds]).sum() / resp_policy_mask[inds].sum().clamp(min=1) + + args.kl_coef * kl_ref_penalty) + value_loss = 0.5 * (torch.max((mb_resp_values - returns[inds]) ** 2, + (torch.clamp(mb_resp_values, old_resp_values[inds] - args.cliprange_value, + old_resp_values[inds] + args.cliprange_value) - returns[inds]) ** 2) + * resp_value_mask[inds]).sum() / resp_value_mask[inds].sum().clamp(min=1) + + kl = approx_kl_val + kl_ref = kl_ref_penalty.detach() + + # 早停时必须保证 forward-backward 闭环，故只截断 loss 不中断 DDP 通信 + if stop_ppo: + loss = (policy_loss + args.vf_coef * value_loss + aux_loss) * 0.0 + else: + loss = (policy_loss + args.vf_coef * value_loss + aux_loss) / args.accumulation_steps + + loss.backward() + + policy_loss_sum += policy_loss.item() + value_loss_sum += value_loss.item() + kl_sum += kl.item() + kl_ref_sum += kl_ref.item() + clipfrac_sum += clipfrac.item() + aux_loss_sum += aux_loss.item() + log_count += 1 + + grad_accum_step += 1 + + if grad_accum_step % args.accumulation_steps == 0: + clip_grad_norm_(actor_model.parameters(), args.grad_clip) + clip_grad_norm_(critic_model.parameters(), args.grad_clip) + actor_optimizer.step() + critic_optimizer.step() + actor_scheduler.step() + critic_scheduler.step() + actor_optimizer.zero_grad() + critic_optimizer.zero_grad() + + if grad_accum_step % args.accumulation_steps != 0: clip_grad_norm_(actor_model.parameters(), args.grad_clip) clip_grad_norm_(critic_model.parameters(), args.grad_clip) actor_optimizer.step() @@ -183,49 +249,35 @@ def ppo_train_epoch(epoch, loader, iters, old_actor_model, ref_model, actor_sche critic_scheduler.step() actor_optimizer.zero_grad() critic_optimizer.zero_grad() + + if is_main_process() and step % args.save_interval == 0: rollout_engine.update_policy(actor_model) if is_main_process(): - response_ids = gen_out[:, enc.input_ids.shape[1]:] - is_eos = (response_ids == tokenizer.eos_token_id) - eos_indices = torch.argmax(is_eos.int(), dim=1) - has_eos = is_eos.any(dim=1) - lengths = torch.where(has_eos, eos_indices + 1, torch.tensor(response_ids.shape[1], device=is_eos.device)) - avg_len = lengths.float().mean() - - actor_loss_val = policy_loss.item() - critic_loss_val = value_loss.item() - current_aux_loss = aux_loss.item() + critic_loss_val = value_loss_sum / max(log_count, 1) reward_val = rewards.mean().item() - kl_val = kl.item() - kl_ref_val = kl_ref.item() - avg_len_val = avg_len.item() - actor_lr = actor_optimizer.param_groups[0]['lr'] - critic_lr = critic_optimizer.param_groups[0]['lr'] + approx_kl_val = kl_sum / max(log_count, 1) + kl_ref_val = kl_ref_sum / max(log_count, 1) + clipfrac_val = clipfrac_sum / max(log_count, 1) + avg_len_val = resp_lengths.float().mean().item() + actor_lr, critic_lr = actor_optimizer.param_groups[0]['lr'], critic_optimizer.param_groups[0]['lr'] if wandb is not None: wandb.log({ - "actor_loss": actor_loss_val, - "critic_loss": critic_loss_val, - "aux_loss": current_aux_loss, "reward": reward_val, - "kl": kl_val, "kl_ref": kl_ref_val, + "approx_kl": approx_kl_val, + "clipfrac": clipfrac_val, + "critic_loss": critic_loss_val, "avg_response_len": avg_len_val, "actor_lr": actor_lr, + "critic_lr": critic_lr, }) Logger(f"Epoch:[{epoch + 1}/{args.epochs}]({step}/{iters}), " - f"Actor Loss: {actor_loss_val:.4f}, Critic Loss: {critic_loss_val:.4f}, Aux Loss: {current_aux_loss:.4f}, " - f"Reward: {reward_val:.4f}, KL: {kl_val:.4f}, KL_ref: {kl_ref_val:.4f}, " + f"Reward: {reward_val:.4f}, KL_ref: {kl_ref_val:.4f}, Approx KL: {approx_kl_val:.4f}, " + f"ClipFrac: {clipfrac_val:.4f}, Critic Loss: {critic_loss_val:.4f}, " f"Avg Response Len: {avg_len_val:.2f}, Actor LR: {actor_lr:.8f}, Critic LR: {critic_lr:.8f}") - if (step + 1) % args.update_old_actor_freq == 0: - raw_actor = actor_model.module if isinstance(actor_model, DistributedDataParallel) else actor_model - raw_actor = getattr(raw_actor, '_orig_mod', raw_actor) - state_dict = raw_actor.state_dict() - old_actor_model.load_state_dict({k: v.detach().cpu() for k, v in state_dict.items()}) - old_actor_model.to(args.device) - if (step % args.save_interval == 0 or step == iters) and is_main_process(): actor_model.eval() moe_suffix = '_moe' if lm_config.use_moe else '' @@ -243,9 +295,9 @@ def ppo_train_epoch(epoch, loader, iters, old_actor_model, ref_model, actor_sche actor_model.train() del actor_state - del enc, gen_out, responses_text, rewards, full_mask, values_seq, values, advantages - del logits, labels, logp_tokens, final_mask, actor_logp, old_logits, old_logp, ref_logits, ref_logp - del kl, kl_ref, ratio, surr1, surr2, policy_loss, value_loss, loss + del enc, gen_out, responses_text, rewards, full_mask, values_seq, advantages + del logits, labels, final_mask, resp_labels, resp_idx, resp_pad_mask, eos_mask, has_eos, eos_pos, resp_lengths, resp_policy_mask, resp_value_mask, old_resp_logp, ref_logp_all, ref_resp_logp + del kl, kl_ref, policy_loss, value_loss, loss, token_rewards, returns, old_resp_values if __name__ == "__main__": @@ -254,8 +306,8 @@ if __name__ == "__main__": parser.add_argument('--save_weight', default='ppo_actor', type=str, help="保存权重的前缀名") parser.add_argument("--epochs", type=int, default=1, help="训练轮数") parser.add_argument("--batch_size", type=int, default=2, help="batch size") - parser.add_argument("--learning_rate", type=float, default=8e-8, help="Actor学习率") - parser.add_argument("--critic_learning_rate", type=float, default=8e-8, help="Critic学习率") + parser.add_argument("--learning_rate", type=float, default=3e-7, help="Actor学习率") + parser.add_argument("--critic_learning_rate", type=float, default=5e-7, help="Critic学习率") parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备") parser.add_argument("--dtype", type=str, default="bfloat16", help="混合精度类型") parser.add_argument("--num_workers", type=int, default=8, help="数据加载线程数") @@ -263,22 +315,34 @@ if __name__ == "__main__": parser.add_argument("--grad_clip", type=float, default=1.0, help="梯度裁剪阈值") parser.add_argument("--log_interval", type=int, default=1, help="日志打印间隔") parser.add_argument("--save_interval", type=int, default=10, help="模型保存间隔") - parser.add_argument('--hidden_size', default=512, type=int, help="隐藏层维度") + parser.add_argument('--hidden_size', default=768, type=int, help="隐藏层维度") parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") - parser.add_argument('--max_seq_len', default=66, type=int, help="Prompt最大长度") - parser.add_argument("--max_gen_len", type=int, default=1536, help="生成的最大长度") - parser.add_argument("--data_path", type=str, default="../dataset/rlaif-mini.jsonl", help="RLAIF数据路径") - parser.add_argument("--clip_epsilon", type=float, default=0.1, help="PPO裁剪参数") + parser.add_argument('--max_seq_len', default=768, type=int, help="Prompt最大长度") + parser.add_argument("--max_gen_len", type=int, default=1024, help="生成的最大长度") + parser.add_argument("--data_path", type=str, default="../dataset/rlaif.jsonl", help="RLAIF数据路径") + parser.add_argument("--clip_epsilon", type=float, default=0.2, help="PPO裁剪参数") parser.add_argument("--vf_coef", type=float, default=0.5, help="Value function系数") parser.add_argument("--kl_coef", type=float, default=0.02, help="KL散度惩罚系数") - parser.add_argument("--reasoning", type=int, default=1, choices=[0, 1], help='推理模型类型（0=普通模型，1=推理模型）') - parser.add_argument("--update_old_actor_freq", type=int, default=4, help="更新old_actor_model的频率") + parser.add_argument("--gamma", type=float, default=1.0, help="GAE折扣因子") + parser.add_argument("--lam", type=float, default=0.95, help="GAE lambda参数") + parser.add_argument("--cliprange_value", type=float, default=0.2, help="Value function裁剪范围") + parser.add_argument("--ppo_update_iters", type=int, default=2, help="同一批rollout重复更新次数") + parser.add_argument("--early_stop_kl", type=float, default=0.25, help="PPO early stop 的 KL 阈值") + parser.add_argument("--mini_batch_size", type=int, default=2, help="PPO每次更新的minibatch大小") + parser.add_argument('--from_weight', default='full_sft', type=str, help="基于哪个权重训练") parser.add_argument("--reward_model_path", type=str, default="../../internlm2-1_8b-reward", help="Reward模型路径") parser.add_argument('--from_resume', default=0, type=int, choices=[0, 1], help="是否自动检测&续训（0=否，1=是）") parser.add_argument("--use_wandb", action="store_true", help="是否使用wandb") parser.add_argument("--wandb_project", type=str, default="MiniMind-PPO", help="wandb项目名") parser.add_argument("--use_compile", default=0, type=int, choices=[0, 1], help="是否使用torch.compile加速（0=否，1=是）") + parser.add_argument("--debug_mode", action="store_true", help="是否打印训练调试采样") + parser.add_argument("--debug_interval", type=int, default=20, help="debug模式下每隔多少step打印一次采样") + parser.add_argument("--thinking_ratio", type=float, default=0.9, help="按概率开启thinking（0.0~1.0）") + parser.add_argument("--rollout_engine", type=str, default="sglang", choices=["torch", "sglang"], help="rollout引擎类型") + parser.add_argument("--sglang_base_url", type=str, default="http://localhost:8997", help="SGLang服务器URL") + parser.add_argument("--sglang_model_path", type=str, default="../model", help="SGLang tokenizer路径") + parser.add_argument("--sglang_shared_path", type=str, default="./sglang_ckpt_ppo", help="SGLang共享存储路径") args = parser.parse_args() # ========== 1. 初始化环境和随机种子 ========== @@ -306,43 +370,40 @@ if __name__ == "__main__": wandb.init(project=args.wandb_project, name=wandb_run_name, id=wandb_id, resume=resume) # ========== 5. 初始化模型和数据 ========== - base_weight = "reason" if args.reasoning == 1 else "full_sft" + base_weight = args.from_weight # Actor模型 actor_model, tokenizer = init_model(lm_config, base_weight, device=args.device) - if args.use_compile == 1: - actor_model = torch.compile(actor_model) - Logger('torch.compile enabled') - # Old Actor模型 - old_actor_model, _ = init_model(lm_config, base_weight, device=args.device) - old_actor_model = old_actor_model.eval().requires_grad_(False) - # Reference模型 ref_model, _ = init_model(lm_config, base_weight, device=args.device) ref_model = ref_model.eval().requires_grad_(False) - # Critic模型 moe_suffix = '_moe' if lm_config.use_moe else '' ckp = f'{args.save_dir}/{base_weight}_{lm_config.hidden_size}{moe_suffix}.pth' state_dict = torch.load(ckp, map_location=args.device) critic_model = CriticModel(lm_config) critic_model.load_state_dict(state_dict, strict=False) critic_model = critic_model.to(args.device) - # Reward模型 - reward_model = AutoModel.from_pretrained( - args.reward_model_path, torch_dtype=torch.float16, trust_remote_code=True + reward_model = LMForRewardModel(args.reward_model_path, device=args.device, dtype=torch.float16) + # Rollout引擎 + rollout_engine = create_rollout_engine( + engine_type=args.rollout_engine, + policy_model=actor_model, + tokenizer=tokenizer, + device=args.device, + autocast_ctx=autocast_ctx, + sglang_base_url=args.sglang_base_url, + sglang_model_path=args.sglang_model_path, + sglang_shared_path=args.sglang_shared_path, ) - reward_model = reward_model.to(args.device).eval().requires_grad_(False) - reward_tokenizer = AutoTokenizer.from_pretrained(args.reward_model_path, trust_remote_code=True) - # 数据和优化器 - train_ds = RLAIFDataset(args.data_path, tokenizer, max_length=(args.max_seq_len + args.max_gen_len)) + train_ds = RLAIFDataset(args.data_path, tokenizer, max_length=(args.max_seq_len + args.max_gen_len), thinking_ratio=args.thinking_ratio) train_sampler = DistributedSampler(train_ds) if dist.is_initialized() else None actor_optimizer = optim.AdamW(actor_model.parameters(), lr=args.learning_rate) critic_optimizer = optim.AdamW(critic_model.parameters(), lr=args.critic_learning_rate) loader_for_count = DataLoader(train_ds, batch_size=args.batch_size, sampler=train_sampler) iters = len(loader_for_count) - total_optimizer_steps = (iters // args.accumulation_steps) * args.epochs + mb_factor = max(1, math.ceil(args.batch_size / args.mini_batch_size)) + total_optimizer_steps = math.ceil(iters * args.epochs * args.ppo_update_iters * mb_factor / args.accumulation_steps) actor_scheduler = CosineAnnealingLR(actor_optimizer, T_max=total_optimizer_steps, eta_min=args.learning_rate / 10) critic_scheduler = CosineAnnealingLR(critic_optimizer, T_max=total_optimizer_steps, eta_min=args.critic_learning_rate / 10) - - # ========== 6. 从ckp恢复状态 ========== + start_epoch, start_step = 0, 0 if ckp_data: actor_model.load_state_dict(ckp_data['model']) @@ -354,13 +415,17 @@ if __name__ == "__main__": start_epoch = ckp_data['epoch'] start_step = ckp_data.get('step', 0) - # ========== 7. DDP包模型 ========== + # ========== 7. 编译和分布式包装 ========== + if args.use_compile == 1: + actor_model = torch.compile(actor_model) + Logger('torch.compile enabled') + rollout_engine.update_policy(actor_model) if dist.is_initialized(): actor_model._ddp_params_and_buffers_to_ignore = {"freqs_cos", "freqs_sin"} critic_model._ddp_params_and_buffers_to_ignore = {"freqs_cos", "freqs_sin"} actor_model = DistributedDataParallel(actor_model, device_ids=[local_rank]) critic_model = DistributedDataParallel(critic_model, device_ids=[local_rank]) - old_actor_model.to(args.device) + if is_main_process(): rollout_engine.update_policy(actor_model) # ========== 8. 开始训练 ========== for epoch in range(start_epoch, args.epochs): @@ -371,11 +436,9 @@ if __name__ == "__main__": loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True) if skip > 0: Logger(f'Epoch [{epoch + 1}/{args.epochs}]: 跳过前{start_step}个step，从step {start_step + 1}开始') - ppo_train_epoch(epoch, loader, len(loader) + skip, old_actor_model, ref_model, - actor_scheduler, critic_scheduler, reward_model, reward_tokenizer, start_step, wandb) + ppo_train_epoch(epoch, loader, len(loader) + skip, rollout_engine, ref_model, actor_scheduler, critic_scheduler, reward_model, start_step, wandb, use_sglang = (args.rollout_engine == "sglang")) else: - ppo_train_epoch(epoch, loader, len(loader), old_actor_model, ref_model, - actor_scheduler, critic_scheduler, reward_model, reward_tokenizer, 0, wandb) + ppo_train_epoch(epoch, loader, len(loader), rollout_engine, ref_model, actor_scheduler, critic_scheduler, reward_model, 0, wandb, use_sglang = (args.rollout_engine == "sglang")) # ========== 9. 清理分布进程 ========== if dist.is_initialized(): dist.destroy_process_group() \ No newline at end of file diff --git a/trainer/train_pretrain.py b/trainer/train_pretrain.py index a1b12b2..40acdfb 100644 --- a/trainer/train_pretrain.py +++ b/trainer/train_pretrain.py @@ -22,9 +22,11 @@ warnings.filterwarnings('ignore') def train_epoch(epoch, loader, iters, start_step=0, wandb=None): start_time = time.time() + last_step = start_step for step, (input_ids, labels) in enumerate(loader, start=start_step + 1): input_ids = input_ids.to(args.device) labels = labels.to(args.device) + last_step = step lr = get_lr(epoch * iters + step, args.epochs * iters, args.learning_rate) for param_group in optimizer.param_groups: param_group['lr'] = lr @@ -51,7 +53,7 @@ def train_epoch(epoch, loader, iters, start_step=0, wandb=None): current_aux_loss = res.aux_loss.item() if res.aux_loss is not None else 0.0 current_logits_loss = current_loss - current_aux_loss current_lr = optimizer.param_groups[-1]['lr'] - eta_min = spend_time / step * iters // 60 - spend_time // 60 + eta_min = spend_time / max(step - start_step, 1) * (iters - step) // 60 Logger(f'Epoch:[{epoch + 1}/{args.epochs}]({step}/{iters}), loss: {current_loss:.4f}, logits_loss: {current_logits_loss:.4f}, aux_loss: {current_aux_loss:.4f}, lr: {current_lr:.8f}, epoch_time: {eta_min:.1f}min') if wandb: wandb.log({"loss": current_loss, "logits_loss": current_logits_loss, "aux_loss": current_aux_loss, "learning_rate": current_lr, "epoch_time": eta_min}) @@ -69,12 +71,19 @@ def train_epoch(epoch, loader, iters, start_step=0, wandb=None): del input_ids, labels, res, loss + if last_step > start_step and last_step % args.accumulation_steps != 0: + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) + scaler.step(optimizer) + scaler.update() + optimizer.zero_grad(set_to_none=True) + if __name__ == "__main__": parser = argparse.ArgumentParser(description="MiniMind Pretraining") parser.add_argument("--save_dir", type=str, default="../out", help="模型保存目录") parser.add_argument('--save_weight', default='pretrain', type=str, help="保存权重的前缀名") - parser.add_argument("--epochs", type=int, default=1, help="训练轮数（建议1轮zero或2-6轮充分训练）") + parser.add_argument("--epochs", type=int, default=2, help="训练轮数") parser.add_argument("--batch_size", type=int, default=32, help="batch size") parser.add_argument("--learning_rate", type=float, default=5e-4, help="初始学习率") parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备") @@ -84,11 +93,11 @@ if __name__ == "__main__": parser.add_argument("--grad_clip", type=float, default=1.0, help="梯度裁剪阈值") parser.add_argument("--log_interval", type=int, default=100, help="日志打印间隔") parser.add_argument("--save_interval", type=int, default=1000, help="模型保存间隔") - parser.add_argument('--hidden_size', default=512, type=int, help="隐藏层维度") + parser.add_argument('--hidden_size', default=768, type=int, help="隐藏层维度") parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") parser.add_argument('--max_seq_len', default=340, type=int, help="训练的最大截断长度（中文1token≈1.5~1.7字符）") parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") - parser.add_argument("--data_path", type=str, default="../dataset/pretrain_hq.jsonl", help="预训练数据路径") + parser.add_argument("--data_path", type=str, default="../dataset/pretrain_t2t_mini.jsonl", help="预训练数据路径") parser.add_argument('--from_weight', default='none', type=str, help="基于哪个权重训练，为none则从头开始") parser.add_argument('--from_resume', default=0, type=int, choices=[0, 1], help="是否自动检测&续训（0=否，1=是）") parser.add_argument("--use_wandb", action="store_true", help="是否使用wandb") @@ -122,9 +131,6 @@ if __name__ == "__main__": # ========== 5. 定义模型、数据、优化器 ========== model, tokenizer = init_model(lm_config, args.from_weight, device=args.device) - if args.use_compile == 1: - model = torch.compile(model) - Logger('torch.compile enabled') train_ds = PretrainDataset(args.data_path, tokenizer, max_length=args.max_seq_len) train_sampler = DistributedSampler(train_ds) if dist.is_initialized() else None scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype == 'float16')) @@ -139,7 +145,10 @@ if __name__ == "__main__": start_epoch = ckp_data['epoch'] start_step = ckp_data.get('step', 0) - # ========== 7. DDP包模型 ========== + # ========== 7. 编译和分布式包装 ========== + if args.use_compile == 1: + model = torch.compile(model) + Logger('torch.compile enabled') if dist.is_initialized(): model._ddp_params_and_buffers_to_ignore = {"freqs_cos", "freqs_sin"} model = DistributedDataParallel(model, device_ids=[local_rank]) diff --git a/trainer/train_reason.py b/trainer/train_reason.py deleted file mode 100644 index 9edcdc1..0000000 --- a/trainer/train_reason.py +++ /dev/null @@ -1,179 +0,0 @@ -import os -import sys - -__package__ = "trainer" -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -import argparse -import time -import warnings -import torch -import torch.distributed as dist -from contextlib import nullcontext -from torch import optim, nn -from torch.nn.parallel import DistributedDataParallel -from torch.utils.data import DataLoader, DistributedSampler -from model.model_minimind import MiniMindConfig -from dataset.lm_dataset import SFTDataset -from trainer.trainer_utils import get_lr, Logger, is_main_process, lm_checkpoint, init_distributed_mode, setup_seed, init_model, SkipBatchSampler - -warnings.filterwarnings('ignore') - - -def train_epoch(epoch, loader, iters, tokenizer, lm_config, start_step=0, wandb=None): - start_of_think_ids = tokenizer('').input_ids - end_of_think_ids = tokenizer('').input_ids - start_of_answer_ids = tokenizer('').input_ids - end_of_answer_ids = tokenizer('').input_ids - loss_fct = nn.CrossEntropyLoss(reduction='none') - start_time = time.time() - - for step, (input_ids, labels) in enumerate(loader, start=start_step + 1): - input_ids = input_ids.to(args.device) - labels = labels.to(args.device) - lr = get_lr(epoch * iters + step, args.epochs * iters, args.learning_rate) - for param_group in optimizer.param_groups: - param_group['lr'] = lr - - with autocast_ctx: - res = model(input_ids) - shift_logits = res.logits[..., :-1, :].contiguous() - shift_labels = labels[..., 1:].contiguous() - loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)).view(shift_labels.size()) - - loss_mask = (shift_labels != -100).float() - sp_ids = torch.isin(shift_labels.view(-1), - torch.tensor(start_of_think_ids + end_of_think_ids - + start_of_answer_ids + end_of_answer_ids - ).to(args.device)) - loss_mask_flat = loss_mask.view(-1) - loss_mask_sum = loss_mask_flat.sum() - loss_mask_flat[sp_ids] = 10 - loss_mask = loss_mask_flat.view(shift_labels.size()) - logits_loss = (loss * loss_mask).sum() / loss_mask_sum - loss = logits_loss + res.aux_loss - loss = loss / args.accumulation_steps - - scaler.scale(loss).backward() - - if step % args.accumulation_steps == 0: - scaler.unscale_(optimizer) - torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) - scaler.step(optimizer) - scaler.update() - optimizer.zero_grad(set_to_none=True) - - if step % args.log_interval == 0 or step == iters: - spend_time = time.time() - start_time - current_loss = loss.item() * args.accumulation_steps - current_aux_loss = res.aux_loss.item() if res.aux_loss is not None else 0.0 - current_logits_loss = logits_loss.item() - current_lr = optimizer.param_groups[-1]['lr'] - eta_min = spend_time / step * iters // 60 - spend_time // 60 - Logger(f'Epoch:[{epoch + 1}/{args.epochs}]({step}/{iters}), loss: {current_loss:.4f}, logits_loss: {current_logits_loss:.4f}, aux_loss: {current_aux_loss:.4f}, lr: {current_lr:.8f}, epoch_time: {eta_min:.1f}min') - if wandb: wandb.log({"loss": current_loss, "logits_loss": current_logits_loss, "aux_loss": current_aux_loss, "learning_rate": current_lr, "epoch_time": eta_min}) - - if (step % args.save_interval == 0 or step == iters) and is_main_process(): - model.eval() - moe_suffix = '_moe' if lm_config.use_moe else '' - ckp = f'{args.save_dir}/{args.save_weight}_{lm_config.hidden_size}{moe_suffix}.pth' - raw_model = model.module if isinstance(model, DistributedDataParallel) else model - raw_model = getattr(raw_model, '_orig_mod', raw_model) - state_dict = raw_model.state_dict() - torch.save({k: v.half().cpu() for k, v in state_dict.items()}, ckp) - lm_checkpoint(lm_config, weight=args.save_weight, model=model, optimizer=optimizer, scaler=scaler, epoch=epoch, step=step, wandb=wandb, save_dir='../checkpoints') - model.train() - del state_dict - - del input_ids, labels, res, loss - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="MiniMind Reasoning Distillation") - parser.add_argument("--save_dir", type=str, default="../out", help="模型保存目录") - parser.add_argument('--save_weight', default='reason', type=str, help="保存权重的前缀名") - parser.add_argument("--epochs", type=int, default=1, help="训练轮数") - parser.add_argument("--batch_size", type=int, default=8, help="batch size") - parser.add_argument("--learning_rate", type=float, default=1e-6, help="初始学习率") - parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备") - parser.add_argument("--dtype", type=str, default="bfloat16", help="混合精度类型") - parser.add_argument("--num_workers", type=int, default=8, help="数据加载线程数") - parser.add_argument("--accumulation_steps", type=int, default=1, help="梯度累积步数") - parser.add_argument("--grad_clip", type=float, default=1.0, help="梯度裁剪阈值") - parser.add_argument("--log_interval", type=int, default=100, help="日志打印间隔") - parser.add_argument("--save_interval", type=int, default=100, help="模型保存间隔") - parser.add_argument('--hidden_size', default=512, type=int, help="隐藏层维度") - parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") - parser.add_argument('--max_seq_len', default=720, type=int, help="训练的最大截断长度（中文1token≈1.5~1.7字符）") - parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") - parser.add_argument("--data_path", type=str, default="../dataset/r1_mix_1024.jsonl", help="推理蒸馏数据路径") - parser.add_argument('--from_weight', default='dpo', type=str, help="基于哪个权重训练，默认dpo") - parser.add_argument('--from_resume', default=0, type=int, choices=[0, 1], help="是否自动检测&续训（0=否，1=是）") - parser.add_argument("--use_wandb", action="store_true", help="是否使用wandb") - parser.add_argument("--wandb_project", type=str, default="MiniMind-Reasoning", help="wandb项目名") - parser.add_argument("--use_compile", default=0, type=int, choices=[0, 1], help="是否使用torch.compile加速（0=否，1=是）") - args = parser.parse_args() - - # ========== 1. 初始化环境和随机种子 ========== - local_rank = init_distributed_mode() - if dist.is_initialized(): args.device = f"cuda:{local_rank}" - setup_seed(42 + (dist.get_rank() if dist.is_initialized() else 0)) - - # ========== 2. 配置目录、模型参数、检查ckp ========== - os.makedirs(args.save_dir, exist_ok=True) - lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe)) - ckp_data = lm_checkpoint(lm_config, weight=args.save_weight, save_dir='../checkpoints') if args.from_resume==1 else None - - # ========== 3. 设置混合精度 ========== - device_type = "cuda" if "cuda" in args.device else "cpu" - dtype = torch.bfloat16 if args.dtype == "bfloat16" else torch.float16 - autocast_ctx = nullcontext() if device_type == "cpu" else torch.cuda.amp.autocast(dtype=dtype) - - # ========== 4. 配wandb ========== - wandb = None - if args.use_wandb and is_main_process(): - import swanlab as wandb - wandb_id = ckp_data.get('wandb_id') if ckp_data else None - resume = 'must' if wandb_id else None - wandb_run_name = f"MiniMind-Reasoning-Epoch-{args.epochs}-BatchSize-{args.batch_size}-LR-{args.learning_rate}" - wandb.init(project=args.wandb_project, name=wandb_run_name, id=wandb_id, resume=resume) - - # ========== 5. 定义模型、数据、优化器 ========== - model, tokenizer = init_model(lm_config, args.from_weight, device=args.device) - if args.use_compile == 1: - model = torch.compile(model) - Logger('torch.compile enabled') - train_ds = SFTDataset(args.data_path, tokenizer, max_length=args.max_seq_len) - train_sampler = DistributedSampler(train_ds) if dist.is_initialized() else None - scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype == 'float16')) - optimizer = optim.AdamW(model.parameters(), lr=args.learning_rate) - - # ========== 6. 从ckp恢复状态 ========== - start_epoch, start_step = 0, 0 - if ckp_data: - model.load_state_dict(ckp_data['model']) - optimizer.load_state_dict(ckp_data['optimizer']) - scaler.load_state_dict(ckp_data['scaler']) - start_epoch = ckp_data['epoch'] - start_step = ckp_data.get('step', 0) - - # ========== 7. DDP包模型 ========== - if dist.is_initialized(): - model._ddp_params_and_buffers_to_ignore = {"freqs_cos", "freqs_sin"} - model = DistributedDataParallel(model, device_ids=[local_rank]) - - # ========== 8. 开始训练 ========== - for epoch in range(start_epoch, args.epochs): - train_sampler and train_sampler.set_epoch(epoch) - setup_seed(42 + epoch); indices = torch.randperm(len(train_ds)).tolist() - skip = start_step if (epoch == start_epoch and start_step > 0) else 0 - batch_sampler = SkipBatchSampler(train_sampler or indices, args.batch_size, skip) - loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True) - if skip > 0: - Logger(f'Epoch [{epoch + 1}/{args.epochs}]: 跳过前{start_step}个step，从step {start_step + 1}开始') - train_epoch(epoch, loader, len(loader) + skip, tokenizer, lm_config, start_step, wandb) - else: - train_epoch(epoch, loader, len(loader), tokenizer, lm_config, 0, wandb) - - # ========== 9. 清理分布进程 ========== - if dist.is_initialized(): dist.destroy_process_group() \ No newline at end of file diff --git a/trainer/train_spo.py b/trainer/train_spo.py deleted file mode 100755 index 6b89976..0000000 --- a/trainer/train_spo.py +++ /dev/null @@ -1,354 +0,0 @@ -import os -import sys - -__package__ = "trainer" -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -import argparse -import re -import gc -import warnings -import torch -import torch.distributed as dist -from transformers import AutoTokenizer -from contextlib import nullcontext -from torch import optim -from torch.nn.parallel import DistributedDataParallel -from torch.utils.data import DataLoader, DistributedSampler -from torch.optim.lr_scheduler import CosineAnnealingLR -from transformers import AutoModel -from model.model_minimind import MiniMindConfig, MiniMindForCausalLM -from dataset.lm_dataset import RLAIFDataset -from trainer.trainer_utils import Logger, is_main_process, lm_checkpoint, init_distributed_mode, setup_seed, SkipBatchSampler, init_model - -warnings.filterwarnings('ignore') - - -class AutoAdaptiveValueTracker: - """SPO自适应价值追踪器""" - def __init__(self, rho_mode='kl', rho_const=0.9, D_half=0.06, clip_lower=0.5, clip_upper=0.96): - self.rho_mode = rho_mode - self.rho_const = rho_const - self.D_half = D_half - self.clip_lower = clip_lower - self.clip_upper = clip_upper - N_init = 1.0 / (1.0 - self.clip_lower) - self.alpha = 0.5 * N_init - self.beta = 0.5 * N_init - self.old_mean_logprob = None - - def get_baselines(self, batch_size): - baseline = self.alpha / (self.alpha + self.beta) - return torch.full((batch_size,), baseline, dtype=torch.float32) - - def compute_rho(self, cur_mean_logprob): - if self.rho_mode == 'constant': - return self.rho_const - if self.old_mean_logprob is None: - return self.rho_const - kl = abs(self.old_mean_logprob - cur_mean_logprob) - rho = 2 ** (-kl / self.D_half) - return max(min(rho, self.clip_upper), self.clip_lower) - - def update(self, rewards, cur_logprobs=None, response_masks=None): - if cur_logprobs is not None and response_masks is not None: - mean_logprob = ((cur_logprobs * response_masks).sum() / response_masks.sum()).item() - rho = self.compute_rho(mean_logprob) - self.old_mean_logprob = mean_logprob - else: - rho = self.rho_const - - scale = 3.0 - normalized_rewards = (rewards + scale) / (2 * scale) - avg_normalized_reward = normalized_rewards.mean().item() - self.alpha = rho * self.alpha + avg_normalized_reward - self.beta = rho * self.beta + (1 - avg_normalized_reward) - return rho - - -def calculate_rewards(prompts, responses, reward_model, reward_tokenizer): - """整合所有奖励函数计算总奖励""" - def reasoning_model_reward(rewards): - pattern = r"^\n.*?\n\n\n.*?\n$" - pattern2 = r"^\n.*?\n\n\n\n.*?\n$" - matches_pattern = [re.match(pattern, response, re.S) for response in responses] - matches_pattern2 = [re.match(pattern2, response, re.S) for response in responses] - - format_rewards = [] - for match_pattern, match_pattern2 in zip(matches_pattern, matches_pattern2): - if match_pattern or match_pattern2: - format_rewards.append(0.5) - else: - format_rewards.append(0.0) - rewards += torch.tensor(format_rewards, device=args.device) - - def mark_num(text): - reward = 0 - if text.count("") == 1: reward += 0.25 - if text.count("") == 1: reward += 0.25 - if text.count("") == 1: reward += 0.25 - if text.count("") == 1: reward += 0.25 - return reward - - mark_rewards = [mark_num(response) for response in responses] - rewards += torch.tensor(mark_rewards, device=args.device) - return rewards - - rewards = torch.zeros(len(responses), device=args.device) - if args.reasoning == 1: - rewards = reasoning_model_reward(rewards) - - with torch.no_grad(): - reward_model_scores = [] - scale = 3.0 - - for i, (prompt, response) in enumerate(zip(prompts, responses)): - pattern = r"<\|im_start\|>(system|user|assistant)\s+(.*?)<\|im_end\|>" - matches = re.findall(pattern, prompt, re.DOTALL) - messages = [{"role": role, "content": content.strip()} for role, content in matches] - - tmp_chat = messages + [{"role": "assistant", "content": response}] - score = reward_model.get_score(reward_tokenizer, tmp_chat) - score = max(min(score, scale), -scale) - - if args.reasoning == 1: - answer_match = re.search(r'(.*?)', response, re.DOTALL) - if answer_match: - answer_content = answer_match.group(1).strip() - tmp_chat = messages + [{"role": "assistant", "content": answer_content}] - answer_score = reward_model.get_score(reward_tokenizer, tmp_chat) - answer_score = max(min(answer_score, scale), -scale) - score = score * 0.4 + answer_score * 0.6 - - reward_model_scores.append(score) - - reward_model_scores = torch.tensor(reward_model_scores, device=args.device) - rewards += reward_model_scores - - return rewards - - -def spo_train_epoch(epoch, loader, iters, ref_model, reward_model, reward_tokenizer, value_tracker, start_step=0, wandb=None): - for step, batch in enumerate(loader, start=start_step + 1): - prompts = batch['prompt'] # list[str], length B - prompt_inputs = tokenizer(prompts, return_tensors="pt", padding=True, return_token_type_ids=False, - padding_side="left", add_special_tokens=False).to(args.device) # input_ids: [B, P], attention_mask: [B, P] - if args.max_seq_len: - prompt_inputs["input_ids"] = prompt_inputs["input_ids"][:, -args.max_seq_len:] - prompt_inputs["attention_mask"] = prompt_inputs["attention_mask"][:, -args.max_seq_len:] - - with torch.no_grad(): - # DDP 模型需要使用 .module 访问 generate 方法 - model_for_gen = model.module if isinstance(model, DistributedDataParallel) else model - outputs = model_for_gen.generate( - **prompt_inputs, max_new_tokens=args.max_gen_len, do_sample=True, temperature=0.8, - num_return_sequences=1, pad_token_id=tokenizer.pad_token_id) # [B, P+R] - - completion_ids = outputs[:, prompt_inputs["input_ids"].size(1):] # [B, R] - - def get_per_token_logps(mdl, input_ids, n_keep): - input_ids = input_ids.detach().clone() if input_ids.is_inference() else input_ids - logits = mdl(input_ids, logits_to_keep=n_keep + 1).logits[:, :-1, :] - per_token_logps = [] - for logits_row, ids_row in zip(logits, input_ids[:, -n_keep:]): - ids_row = ids_row.detach().clone() if ids_row.is_inference() else ids_row - per_token_logps.append(torch.gather(logits_row.log_softmax(dim=-1), 1, ids_row.unsqueeze(1)).squeeze(1)) - return torch.stack(per_token_logps) - - with autocast_ctx: - per_token_logps = get_per_token_logps(model, outputs, completion_ids.size(1)) # [B, R] - res = model(outputs) if lm_config.use_moe else None - aux_loss = res.aux_loss if res is not None else torch.tensor(0.0, device=args.device) - - with torch.no_grad(): - ref_per_token_logps = get_per_token_logps(ref_model, outputs, completion_ids.size(1)) # [B, R] - - completions = tokenizer.batch_decode(completion_ids, skip_special_tokens=True) # list[str], length B - rewards = calculate_rewards(prompts, completions, reward_model, reward_tokenizer).to(args.device) # [B] - - baselines = value_tracker.get_baselines(len(prompts)).to(args.device) # [B] - - scale = 3.0 - # Un-normalize baselines to be in the same scale as raw rewards [-3, 3] - unnormalized_baselines = baselines * (2 * scale) - scale # [B] - advantages = rewards - unnormalized_baselines # [B] - - # 直接使用 baseline 提供的优势估计，只做裁剪防止梯度爆炸。不再做 batch 内归一化，因为 baseline 已经提供了跨 batch 的稳定基线 - advantages = advantages.clamp(-5.0, 5.0) - - is_eos = completion_ids == tokenizer.eos_token_id # [B, R] - eos_idx = torch.full((is_eos.size(0),), is_eos.size(1), dtype=torch.long, device=args.device) # [B] - eos_idx[is_eos.any(dim=1)] = is_eos.int().argmax(dim=1)[is_eos.any(dim=1)] - completion_mask = (torch.arange(is_eos.size(1), device=args.device).expand(is_eos.size(0), -1) <= eos_idx.unsqueeze(1)).int() # [B, R] - - kl_div = ref_per_token_logps - per_token_logps # [B, R] - per_token_kl = torch.exp(kl_div) - kl_div - 1 # [B, R] - per_token_loss = -per_token_logps * advantages.unsqueeze(1) + args.beta * per_token_kl # [B, R] - policy_loss = ((per_token_loss * completion_mask).sum(dim=1) / completion_mask.sum(dim=1)).mean() - loss = (policy_loss + aux_loss) / args.accumulation_steps # scalar - loss.backward() - - response_masks = completion_mask.float() # [B, R] - rho = value_tracker.update(rewards, per_token_logps.detach(), response_masks) - - if step % args.accumulation_steps == 0: - if args.grad_clip > 0: - torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) - optimizer.step() - scheduler.step() - optimizer.zero_grad() - - if step % args.log_interval == 0 or step == iters: - policy_loss_val = loss.item() * args.accumulation_steps - current_aux_loss = aux_loss.item() - avg_reward_val = rewards.mean().item() - avg_len_val = completion_mask.sum(dim=1).float().mean().item() - kl_val = ((per_token_kl * completion_mask).sum() / (completion_mask.sum() + 1e-8)).item() - avg_baseline_val = baselines.mean().item() - current_lr = optimizer.param_groups[0]['lr'] - - Logger(f'Epoch:[{epoch + 1}/{args.epochs}]({step}/{iters}), ' - f'Actor Loss: {policy_loss_val:.4f}, Aux Loss: {current_aux_loss:.4f}, Reward: {avg_reward_val:.4f}, ' - f'Baseline: {avg_baseline_val:.4f}, KL: {kl_val:.4f}, Rho: {rho:.4f}, ' - f'Avg Response Len: {avg_len_val:.2f}, Learning Rate: {current_lr:.8f}') - - if wandb and is_main_process(): - wandb.log({ - "policy_loss": policy_loss_val, - "aux_loss": current_aux_loss, - "reward": avg_reward_val, - "kl": kl_val, - "rho": float(rho), - "baseline": avg_baseline_val, - "advantages_mean": advantages.mean().item(), - "learning_rate": current_lr - }) - - if (step % args.save_interval == 0 or step == iters) and is_main_process(): - model.eval() - moe_suffix = '_moe' if lm_config.use_moe else '' - ckp = f'{args.save_dir}/{args.save_weight}_{lm_config.hidden_size}{moe_suffix}.pth' - raw_model = model.module if isinstance(model, DistributedDataParallel) else model - raw_model = getattr(raw_model, '_orig_mod', raw_model) - state_dict = raw_model.state_dict() - torch.save({k: v.half().cpu() for k, v in state_dict.items()}, ckp) - lm_checkpoint(lm_config, weight=args.save_weight, model=model, optimizer=optimizer, - epoch=epoch, step=step, wandb=wandb, save_dir='../checkpoints', scheduler=scheduler) - model.train() - del state_dict - - del prompt_inputs, outputs, completion_ids, per_token_logps, ref_per_token_logps - del completions, rewards, advantages, completion_mask, baselines, response_masks - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="MiniMind SPO (Self-Play Optimization)") - parser.add_argument("--save_dir", type=str, default="../out", help="模型保存目录") - parser.add_argument('--save_weight', default='spo', type=str, help="保存权重的前缀名") - parser.add_argument("--epochs", type=int, default=1, help="训练轮数") - parser.add_argument("--batch_size", type=int, default=2, help="batch size") - parser.add_argument("--learning_rate", type=float, default=1e-7, help="初始学习率") - parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备") - parser.add_argument("--dtype", type=str, default="bfloat16", help="混合精度类型") - parser.add_argument("--num_workers", type=int, default=8, help="数据加载线程数") - parser.add_argument("--accumulation_steps", type=int, default=4, help="梯度累积步数") - parser.add_argument("--grad_clip", type=float, default=1.0, help="梯度裁剪阈值") - parser.add_argument("--log_interval", type=int, default=1, help="日志打印间隔") - parser.add_argument("--save_interval", type=int, default=10, help="模型保存间隔") - parser.add_argument('--hidden_size', default=512, type=int, help="隐藏层维度") - parser.add_argument('--num_hidden_layers', default=8, type=int, help="隐藏层数量") - parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）") - parser.add_argument('--max_seq_len', default=66, type=int, help="Prompt最大长度") - parser.add_argument("--max_gen_len", type=int, default=1536, help="生成的最大长度") - parser.add_argument("--data_path", type=str, default="../dataset/rlaif-mini.jsonl", help="RLAIF数据路径") - parser.add_argument("--beta", type=float, default=0.02, help="KL惩罚系数") - parser.add_argument("--reasoning", type=int, default=1, choices=[0, 1], help='推理模型类型（0=普通模型，1=推理模型）') - parser.add_argument("--reward_model_path", type=str, default="../../internlm2-1_8b-reward", help="Reward模型路径") - parser.add_argument('--from_resume', default=0, type=int, choices=[0, 1], help="是否自动检测&续训（0=否，1=是）") - parser.add_argument("--use_wandb", action="store_true", help="是否使用wandb") - parser.add_argument("--wandb_project", type=str, default="MiniMind-SPO", help="wandb项目名") - parser.add_argument("--use_compile", default=0, type=int, choices=[0, 1], help="是否使用torch.compile加速（0=否，1=是）") - args = parser.parse_args() - - # ========== 1. 初始化环境和随机种子 ========== - local_rank = init_distributed_mode() - if dist.is_initialized(): args.device = f"cuda:{local_rank}" - setup_seed(42 + (dist.get_rank() if dist.is_initialized() else 0)) - - # ========== 2. 配置目录、模型参数、检查ckp ========== - os.makedirs(args.save_dir, exist_ok=True) - lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, - max_seq_len=args.max_seq_len + args.max_gen_len, use_moe=bool(args.use_moe)) - ckp_data = lm_checkpoint(lm_config, weight=args.save_weight, save_dir='../checkpoints') if args.from_resume==1 else None - - # ========== 3. 设置混合精度 ========== - device_type = "cuda" if "cuda" in args.device else "cpu" - dtype = torch.bfloat16 if args.dtype == "bfloat16" else torch.float16 - autocast_ctx = nullcontext() if device_type == "cpu" else torch.cuda.amp.autocast(dtype=dtype) - - # ========== 4. 配wandb ========== - wandb = None - if args.use_wandb and is_main_process(): - import swanlab as wandb - wandb_id = ckp_data.get('wandb_id') if ckp_data else None - resume = 'must' if wandb_id else None - wandb_run_name = f"MiniMind-SPO-Epoch-{args.epochs}-BS-{args.batch_size}-LR-{args.learning_rate}" - wandb.init(project=args.wandb_project, name=wandb_run_name, id=wandb_id, resume=resume) - - # ========== 5. 初始化模型（Policy, Ref, Reward）和Value Tracker、数据 ========== - base_weight = "reason" if args.reasoning == 1 else "full_sft" - # Policy模型 - model, tokenizer = init_model(lm_config, base_weight, device=args.device) - if args.use_compile == 1: - model = torch.compile(model) - Logger('torch.compile enabled') - # Reference模型 - ref_model, _ = init_model(lm_config, base_weight, device=args.device) - ref_model = ref_model.eval().requires_grad_(False) - # Reward模型 - reward_model = AutoModel.from_pretrained( - args.reward_model_path, torch_dtype=torch.float16, trust_remote_code=True - ) - reward_model = reward_model.to(args.device).eval().requires_grad_(False) - reward_tokenizer = AutoTokenizer.from_pretrained(args.reward_model_path, trust_remote_code=True) - # Value Tracker - value_tracker = AutoAdaptiveValueTracker(rho_mode='kl', rho_const=0.9, D_half=0.06, clip_lower=0.5, clip_upper=0.96) - - train_ds = RLAIFDataset(args.data_path, tokenizer, max_length=lm_config.max_seq_len) - train_sampler = DistributedSampler(train_ds) if dist.is_initialized() else None - optimizer = optim.AdamW(model.parameters(), lr=args.learning_rate) - - loader_for_count = DataLoader(train_ds, batch_size=args.batch_size, sampler=train_sampler) - iters = len(loader_for_count) - total_optimizer_steps = (iters // args.accumulation_steps) * args.epochs - scheduler = CosineAnnealingLR(optimizer, T_max=total_optimizer_steps, eta_min=args.learning_rate / 10) - - # ========== 6. 从ckp恢复状态 ========== - start_epoch, start_step = 0, 0 - if ckp_data: - model.load_state_dict(ckp_data['model']) - optimizer.load_state_dict(ckp_data['optimizer']) - scheduler.load_state_dict(ckp_data['scheduler']) - start_epoch = ckp_data['epoch'] - start_step = ckp_data.get('step', 0) - - # ========== 7. DDP包模型 ========== - if dist.is_initialized(): - model._ddp_params_and_buffers_to_ignore = {"freqs_cos", "freqs_sin"} - model = DistributedDataParallel(model, device_ids=[local_rank]) - - # ========== 8. 开始训练 ========== - for epoch in range(start_epoch, args.epochs): - train_sampler and train_sampler.set_epoch(epoch) - setup_seed(42 + epoch); indices = torch.randperm(len(train_ds)).tolist() - skip = start_step if (epoch == start_epoch and start_step > 0) else 0 - batch_sampler = SkipBatchSampler(train_sampler or indices, args.batch_size, skip) - loader = DataLoader(train_ds, batch_sampler=batch_sampler, num_workers=args.num_workers, pin_memory=True) - if skip > 0: - Logger(f'Epoch [{epoch + 1}/{args.epochs}]: 跳过前{start_step}个step，从step {start_step + 1}开始') - spo_train_epoch(epoch, loader, len(loader) + skip, ref_model, reward_model, reward_tokenizer, value_tracker, start_step, wandb) - else: - spo_train_epoch(epoch, loader, len(loader), ref_model, reward_model, reward_tokenizer, value_tracker, 0, wandb) - - # ========== 9. 清理分布进程 ========== - if dist.is_initialized(): dist.destroy_process_group() \ No newline at end of file diff --git a/trainer/train_tokenizer.py b/trainer/train_tokenizer.py index f5c536c..336a688 100644 --- a/trainer/train_tokenizer.py +++ b/trainer/train_tokenizer.py @@ -4,92 +4,115 @@ import os import json from tokenizers import decoders, models, pre_tokenizers, trainers, Tokenizer -DATA_PATH = '../dataset/pretrain_hq.jsonl' +DATA_PATH = '../dataset/sft_t2t_mini.jsonl' TOKENIZER_DIR = '../model_learn_tokenizer/' VOCAB_SIZE = 6400 +SPECIAL_TOKENS_NUM = 36 def get_texts(data_path): - with open(data_path, 'r', encoding='utf-8') as f: + with open(data_path, 'r', encoding='utf-8', errors='ignore') as f: for i, line in enumerate(f): - if i >= 10000: break # 实验性，可只用前10000行测试 - data = json.loads(line) - yield data['text'] + if i >= 10000: break # 选10000行测试 + try: + data = json.loads(line) + contents = [item.get('content') for item in data.get('conversations', []) if item.get('content')] + if contents: + yield "\n".join(contents) + except json.JSONDecodeError: + continue -def train_tokenizer(data_path, tokenizer_dir, vocab_size): +def train_tokenizer(data_path, tokenizer_dir, vocab_size, special_tokens_num=SPECIAL_TOKENS_NUM): tokenizer = Tokenizer(models.BPE()) tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) + + special_tokens_list = [ + "<|endoftext|>", "<|im_start|>", "<|im_end|>", + "<|object_ref_start|>", "<|object_ref_end|>", "<|box_start|>", "<|box_end|>", "<|quad_start|>", "<|quad_end|>", + "<|vision_start|>", "<|vision_end|>", "<|vision_pad|>", "<|image_pad|>", "<|video_pad|>", + "<|audio_start|>", "<|audio_end|>", "<|audio_pad|>", "", "", "", "" + ] + + additional_tokens_list = [ + "", "", + "", "", + "", "" + ] + num_buffer = special_tokens_num - len(special_tokens_list + additional_tokens_list) + buffer_tokens = [f"<|buffer{i}|>" for i in range(1, num_buffer + 1)] # 预留一定数量的token位置 + all_special_tokens = special_tokens_list + additional_tokens_list + buffer_tokens trainer = trainers.BpeTrainer( vocab_size=vocab_size, - special_tokens=["<|endoftext|>", "<|im_start|>", "<|im_end|>"], show_progress=True, - initial_alphabet=pre_tokenizers.ByteLevel.alphabet() + initial_alphabet=pre_tokenizers.ByteLevel.alphabet(), + special_tokens=all_special_tokens ) texts = get_texts(data_path) tokenizer.train_from_iterator(texts, trainer=trainer) tokenizer.decoder = decoders.ByteLevel() - - assert tokenizer.token_to_id("<|endoftext|>") == 0 - assert tokenizer.token_to_id("<|im_start|>") == 1 - assert tokenizer.token_to_id("<|im_end|>") == 2 + tokenizer.add_special_tokens(special_tokens_list) os.makedirs(tokenizer_dir, exist_ok=True) tokenizer.save(os.path.join(tokenizer_dir, "tokenizer.json")) tokenizer.model.save(tokenizer_dir) + tokenizer_json_path = os.path.join(tokenizer_dir, "tokenizer.json") + with open(tokenizer_json_path, 'r', encoding='utf-8') as f: + tokenizer_data = json.load(f) + for token_info in tokenizer_data.get('added_tokens', []): + if token_info['content'] not in special_tokens_list: + token_info['special'] = False + with open(tokenizer_json_path, 'w', encoding='utf-8') as f: + json.dump(tokenizer_data, f, ensure_ascii=False, indent=2) + + added_tokens_decoder = {} + for i, token in enumerate(all_special_tokens): + idx = tokenizer.token_to_id(token) + added_tokens_decoder[str(idx)] = { + "content": token, + "lstrip": False, + "normalized": False, + "rstrip": False, + "single_word": False, + "special": True if token in special_tokens_list else False + } + config = { "add_bos_token": False, "add_eos_token": False, "add_prefix_space": False, - "added_tokens_decoder": { - "0": { - "content": "<|endoftext|>", - "lstrip": False, - "normalized": False, - "rstrip": False, - "single_word": False, - "special": True - }, - "1": { - "content": "<|im_start|>", - "lstrip": False, - "normalized": False, - "rstrip": False, - "single_word": False, - "special": True - }, - "2": { - "content": "<|im_end|>", - "lstrip": False, - "normalized": False, - "rstrip": False, - "single_word": False, - "special": True - } - }, - "additional_special_tokens": [], + "added_tokens_decoder": added_tokens_decoder, + "additional_special_tokens": [t for t in special_tokens_list if t not in ["<|endoftext|>"]], "bos_token": "<|im_start|>", "clean_up_tokenization_spaces": False, "eos_token": "<|im_end|>", "legacy": True, - "model_max_length": 32768, + "model_max_length": 131072, "pad_token": "<|endoftext|>", "sp_model_kwargs": {}, "spaces_between_special_tokens": False, - "tokenizer_class": "PreTrainedTokenizerFast", "unk_token": "<|endoftext|>", - "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' -%}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- else -%}\n {{- '<|im_start|>system\\nYou are a helpful assistant<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '\\n\\n\\n\\n' }}\n {%- endif %}\n{%- endif %}" + "image_token": "<|image_pad|>", + "audio_token": "<|audio_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>", + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '' in content %}\n {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n {%- set content = content.split('')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if true %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content.strip('\\n') + '\\n\\n\\n' + content.lstrip('\\n') }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if open_thinking is defined and open_thinking is true %}\n {{- '\\n' }}\n {%- else %}\n {{- '\\n\\n\\n\\n' }}\n {%- endif %}\n{%- endif %}", + "tokenizer_class": "PreTrainedTokenizerFast" } with open(os.path.join(tokenizer_dir, "tokenizer_config.json"), "w", encoding="utf-8") as f: json.dump(config, f, ensure_ascii=False, indent=4) print("Tokenizer training completed.") - def eval_tokenizer(tokenizer_dir): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir) messages = [ {"role": "system", "content": "你是一个优秀的聊天机器人，总是给我正确的回应！"}, {"role": "user", "content": '你来自哪里？'}, + {"role": "assistant", "content": '我来自月球'}, + {"role": "user", "content": '你到底来自哪里？'}, {"role": "assistant", "content": '我来自地球'} ] new_prompt = tokenizer.apply_chat_template( @@ -98,16 +121,35 @@ def eval_tokenizer(tokenizer_dir): ) print('-'*100) print(new_prompt) - - print('-'*100) print('tokenizer词表长度：', len(tokenizer)) model_inputs = tokenizer(new_prompt) print('encoder长度：', len(model_inputs['input_ids'])) response = tokenizer.decode(model_inputs['input_ids'], skip_special_tokens=False) print('decoder一致性：', response == new_prompt, "\n") - - + print('-'*100) + print('压缩率测试（Chars/Tokens）：') + test_texts = [ + # 中文样本 (约200字) + "人工智能是计算机科学的一个分支，它企图了解智能的实质，并生产出一种新的能以人类智能相似的方式做出反应的智能机器，该领域的研究包括机器人、语言识别、图像识别、自然语言处理和专家系统等。人工智能从诞生以来，理论和技术日益成熟，应用领域也不断扩大，可以设想，未来人工智能带来的科技产品，将会是人类智慧的“容器”。人工智能可以对人的意识、思维的信息过程的模拟。人工智能不是人的智能，但能像人那样思考、也可能超过人的智能。", + "星际航行是指在星系内甚至星系间的空间中进行的航行。由于宇宙空间极其广阔，传统的化学火箭动力在恒星间航行时显得力不从心。科学家们提出了多种方案，包括离子推进器、核热火箭、甚至是利用反物质作为能源的设想。此外，曲率驱动和虫洞旅行等科幻概念也在理论物理研究中被反复探讨。尽管目前人类的足迹仅限于月球，但随着核聚变技术和材料科学的突破，前往火星乃至更遥远的太阳系边缘将成为可能。", + # 英文样本 (约200词/字符) + "Large language models (LLMs) are a type of artificial intelligence (AI) trained on vast amounts of text data to understand and generate human-like language. These models use deep learning techniques, specifically transformers, to process and predict the next word in a sequence. LLMs like GPT-4, Llama, and Claude have demonstrated remarkable capabilities in coding, translation, and creative writing. However, they also face challenges such as hallucinations, where the model generates factually incorrect information, and the need for significant computational resources.", + "The development of sustainable energy is crucial for the future of our planet. As climate change continues to impact global weather patterns, transitioning from fossil fuels to renewable sources like solar, wind, and hydroelectric power has become an urgent priority. Innovations in battery storage technology and smart grid management are essential to ensure a reliable energy supply. International cooperation and policy frameworks are also necessary to drive the global shift towards a greener economy and reduce carbon emissions.", + # 混合样本 + "Python 是一种高级编程语言，以其简洁的语法和强大的生态系统而闻名。It is widely used in data science, machine learning, and web development. 开发者可以利用 NumPy, Pandas, and PyTorch 等库快速构建复杂的应用。学习 Python 的过程非常愉快，因为它的代码读起来就像英语一样。Whether you are a beginner or an expert, Python offers something for everyone.", + ] + + total_compression = 0 + for i, text in enumerate(test_texts): + encoded = tokenizer.encode(text) + token_count = len(encoded) + char_count = len(text) + compression_ratio = char_count / token_count + total_compression += compression_ratio + print(f"样本 {i+1} | 字符数: {char_count:4} | Tokens: {token_count:3} | 压缩率: {compression_ratio:.2f}") + + print(f"平均压缩率: {total_compression / len(test_texts):.2f}") print('-'*100) print('流式解码（字节缓冲）测试：') input_ids = model_inputs['input_ids'] diff --git a/trainer/trainer_utils.py b/trainer/trainer_utils.py index 3ec1e44..b6211d5 100644 --- a/trainer/trainer_utils.py +++ b/trainer/trainer_utils.py @@ -12,7 +12,7 @@ import torch import torch.distributed as dist from torch.nn.parallel import DistributedDataParallel from torch.utils.data import Sampler -from transformers import AutoTokenizer +from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification from model.model_minimind import MiniMindForCausalLM def get_model_params(model, config): @@ -154,4 +154,24 @@ class SkipBatchSampler(Sampler): def __len__(self): total_batches = (len(self.sampler) + self.batch_size - 1) // self.batch_size - return max(0, total_batches - self.skip_batches) \ No newline at end of file + return max(0, total_batches - self.skip_batches) + + +class LMForRewardModel: + def __init__(self, model_path, device="cuda", dtype=torch.float16): + self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + self.model = AutoModel.from_pretrained(model_path, torch_dtype=dtype, trust_remote_code=True) + self.model = self.model.to(device).eval() + self.device = device + + @torch.no_grad() + def get_score(self, messages, response): + history_text = "\n".join([f"{m['role']}: {m['content']}" for m in messages[:-1]]) + last_query = messages[-1]['content'] if messages else "" + message_context = f"{history_text}\n以上是对话历史。我的新问题是：\n{last_query}" if history_text else last_query + eval_messages = [ + {"role": "user", "content": message_context}, + {"role": "assistant", "content": response} + ] + score = self.model.get_score(self.tokenizer, eval_messages) + return max(min(score, 3.0), -3.0) \ No newline at end of file