kanshan/minimind
1
0
Fork 0
mirror of https://github.com/jingyaogong/minimind.git synced 2026-01-14 04:07:17 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Apply suggestions from code review

Browse Source 
fix some bugs in merge_lora

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
李子浩 2025-12-24 16:14:18 +08:00 committed by GitHub
parent 41ef5fd8b8
commit daf6cc0c2e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
model/model_lora.py
View File

@ -47,12 +47,15 @@ def merge_lora(model, path):
state_dict = {(k[7:] if k.startswith('module.') else k): v for k, v in state_dict.items()} state_dict = {(k[7:] if k.startswith('module.') else k): v for k, v in state_dict.items()}
for name, module in model.named_modules(): for name, module in model.named_modules():
if isinstance(module, nn.Linear): if isinstance(module, nn.Linear) and module.weight.shape[0] == module.weight.shape[1]:
key_A = f"{name}.lora.A.weight" key_A = f"{name}.lora.A.weight"
key_B = f"{name}.lora.B.weight" key_B = f"{name}.lora.B.weight"
if key_A in state_dict and key_B in state_dict: if key_A in state_dict and key_B in state_dict:
# 直接合并权重: W_new = W_old + B @ A # 直接合并权重: W_new = W_old + B @ A
module.weight.data += state_dict[key_B] @ state_dict[key_A] device = module.weight.device
A_weight = state_dict[key_A].to(device)
B_weight = state_dict[key_B].to(device)
module.weight.data += B_weight @ A_weight
return model return model