mirror of
https://github.com/jingyaogong/minimind.git
synced 2026-07-30 18:27:46 +00:00
[fix] lora compile
This commit is contained in:
@@ -97,7 +97,6 @@ if __name__ == "__main__":
|
||||
parser.add_argument('--from_resume', default=0, type=int, choices=[0, 1], help="是否自动检测&续训(0=否,1=是)")
|
||||
parser.add_argument("--use_wandb", action="store_true", help="是否使用wandb")
|
||||
parser.add_argument("--wandb_project", type=str, default="MiniMind-LoRA", help="wandb项目名")
|
||||
# PS:多卡 DDP + LoRA 场景下建议关闭 use_compile,避免触发 torch.compile 的兼容性问题
|
||||
parser.add_argument("--use_compile", default=0, type=int, choices=[0, 1], help="是否使用torch.compile加速(0=否,1=是)")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -162,8 +161,8 @@ if __name__ == "__main__":
|
||||
|
||||
# ========== 8. 编译和分布式包装 ==========
|
||||
if args.use_compile == 1:
|
||||
model = torch.compile(model)
|
||||
Logger('torch.compile enabled')
|
||||
args.use_compile = 0
|
||||
Logger('[LoRA] monkey-patch forward 与 torch.compile 不兼容,use_compile 已自动关闭')
|
||||
if dist.is_initialized():
|
||||
model._ddp_params_and_buffers_to_ignore = {"freqs_cos", "freqs_sin"}
|
||||
model = DistributedDataParallel(model, device_ids=[local_rank])
|
||||
|
||||
Reference in New Issue
Block a user