[Fix] 🐛 GPU Num Restore Error at Some Scene

routers/api/v1/admin/instance/forceDelete.go

@@ -38,6 +38,8 @@ func forceDeleteHandler(ctx iris.Context) {
 
 	if instance.FromAction == models.InstanceActionStop || instance.FromAction == models.InstanceActionPause || instance.FromAction == models.InstanceActionRestart {
 		redis.RawDB.IncrBy(ctx, "remain_gpu:server:"+strconv.Itoa(int(instance.ServerID)), int64(instance.GpuCount))
+	}
+	if instance.FromAction != models.InstanceActionCreate {
 		redis.RawDB.IncrBy(ctx, "remain_volume:server:"+strconv.Itoa(int(instance.ServerID)), int64(instance.VolumeSize+30))
 	}
 

services/dispatcher/control.go

@@ -61,7 +61,7 @@ func control(serverID uint, data Data) (err error) {
 			if err != nil {
 				ctx := context.Background()
 				redis.RawDB.IncrBy(ctx, "remain_gpu:server:"+strconv.Itoa(int(serverID)), int64(instance.GpuCount))
-				database.DB.Model(&instance).Update("status", models.InstanceStatusFail).Update("from_action", models.InstanceActionRestart)
+				database.DB.Model(&instance).Update("status", models.InstanceStatusFail).Update("from_action", models.InstanceActionStart)
 				lc.Error("instance restart error: %v", err)
 				return
 			}

services/dispatcher/modify.go

@@ -61,7 +61,6 @@ func modify(serverID uint, data Data) (err error) {
 	err = instanceController.Patch(&instance, gpuCount, volumeSize, data.CpuOnly)
 	if err != nil {
 		ctx := context.Background()
-		redis.RawDB.IncrBy(ctx, "remain_gpu:server:"+strconv.Itoa(int(serverID)), int64(gpuCount))
 		redis.RawDB.IncrBy(ctx, "remain_volume:server:"+strconv.Itoa(int(serverID)), int64(volumeSize-oldVolumeSize))
 		database.DB.Model(&instance).Update("status", models.InstanceStatusFail).Update("from_action", models.InstanceActionModify)
 		lc.Error("patch instance error: %v", err)

services/instanceController/createInstance.go

@@ -48,6 +48,7 @@ func createInstance(ip string, port int, apikey string,
 		},
 		Env: []string{
 			"NVIDIA_DRIVER_CAPABILITIES=video,compute,utility",
+			"NVIDIA_VISIBLE_DEVICES=none",
 		},
 	}
 
@@ -63,7 +64,7 @@ func createInstance(ip string, port int, apikey string,
 	if config.GetSystemMountDir() != "" {
 		data.Binds = append(data.Binds, bindStruct{
 			Src:  config.GetSystemMountDir(),
-			Dest: "/root/megrez",
+			Dest: "/root/megrez-pub",
 		})
 	}
 

services/instanceController/patch.go

@@ -87,10 +87,6 @@ func Patch(instance *models.Instances, gpuCount, volumeSize int, cpuOnly bool) (
 
 		err = SetRootPassword(server.IP, server.Port, server.Apikey, instance.ContainerName, instance.SshPasswd)
 		if err != nil {
-			deleteInstance(server.IP, server.Port, server.Apikey, instance.ContainerName)
-			if instance.VolumeName != "" {
-				deleteVolume(server.IP, server.Port, server.Apikey, instance.VolumeName, false)
-			}
 			l.Error("set root password error: %v", err)
 			return err
 		}
@@ -135,10 +131,6 @@ func Patch(instance *models.Instances, gpuCount, volumeSize int, cpuOnly bool) (
 
 	err = SetRootPassword(server.IP, server.Port, server.Apikey, instance.ContainerName, instance.SshPasswd)
 	if err != nil {
-		deleteInstance(server.IP, server.Port, server.Apikey, instance.ContainerName)
-		if instance.VolumeName != "" {
-			deleteVolume(server.IP, server.Port, server.Apikey, instance.VolumeName, false)
-		}
 		l.Error("set root password error: %v", err)
 		return err
 	}

services/instanceController/restart.go

@@ -40,10 +40,6 @@ func Restart(instance *models.Instances) (err error) {
 
 	err = SetRootPassword(server.IP, server.Port, server.Apikey, instance.ContainerName, instance.SshPasswd)
 	if err != nil {
-		deleteInstance(server.IP, server.Port, server.Apikey, instance.ContainerName)
-		if instance.VolumeName != "" {
-			deleteVolume(server.IP, server.Port, server.Apikey, instance.VolumeName, false)
-		}
 		l.Error("set root password error: %v", err)
 		return err
 	}