[Fix] 🐛 Fix GPU Device Error at NoCard Instances #14

config.example.yml

@@ -12,3 +12,4 @@ redis:
   port: 6379
 system:
   verify: false
+  mount_dir: /path/to/mount

frontend/src/api.js

@@ -90,6 +90,9 @@ export default {
   AdminInstancesDelete(id) {
     return ajax(`admin/instances/${id}`, 'delete', {})
   },
+  AdminInstancesForceDelete(id) {
+    return ajax(`admin/instances/${id}/force`, 'delete', {})
+  },
 
   AdminUserList(params) {
     return ajax('admin/users', 'get', { params })

frontend/src/views/admin/Instances.vue

@@ -288,6 +288,7 @@ const instanceMenuItemsTemplate = ref([
   { label: '重启实例', icon: 'pi pi-refresh !text-sky-500', command: () => { instanceRestart(instanceDetail.value.id) } },
   { label: '调整配置', icon: 'pi pi-sliders-h !text-indigo-500', command: () => { openInstanceModify() } },
   { label: '删除实例', icon: 'pi pi-trash !text-red-500', command: () => { openInstanceDelete() } },
+  { label: '强制删除', icon: 'pi pi-exclamation-triangle !text-red-500', command: () => { openInstanceForceDelete() } }
 ])
 const instanceMenuItems = ref([])
 
@@ -376,37 +377,42 @@ const showMenu = (event, instance) => {
     let newItem = { ...item }
     switch (item.label) {
       case '无卡模式开机':
-        if (instanceDetail.value.cpu_only === true || instanceDetail.value.status !== statusStoped.value) {
+        if (instanceDetail.value.cpu_only === true || instanceDetail.value.status !== statusStoped.value || instanceDetail.value.status === statusFail.value) {
           newItem.disabled = true
         }
         break
       case '开机':
-        if (instanceDetail.value.status === statusRunning.value || statusIng.indexOf(instanceDetail.value.status) !== -1) {
+        if (instanceDetail.value.status === statusRunning.value || statusIng.indexOf(instanceDetail.value.status) !== -1 || instanceDetail.value.status === statusFail.value) {
           newItem.disabled = true
         }
         break
       case '关机':
-        if (instanceDetail.value.status === statusStoped.value || statusIng.indexOf(instanceDetail.value.status) !== -1) {
+        if (instanceDetail.value.status === statusStoped.value || statusIng.indexOf(instanceDetail.value.status) !== -1 || instanceDetail.value.status === statusFail.value) {
           newItem.disabled = true
         }
         break
       case '暂停':
-        if (instanceDetail.value.status === statusPaused.value || instanceDetail.value.status === statusStoped.value || statusIng.indexOf(instanceDetail.value.status) !== -1) {
+        if (instanceDetail.value.status === statusPaused.value || instanceDetail.value.status === statusStoped.value || statusIng.indexOf(instanceDetail.value.status) !== -1 || instanceDetail.value.status === statusFail.value) {
           newItem.disabled = true
         }
         break
       case '重启实例':
-        if (statusIng.indexOf(instanceDetail.value.status) !== -1) {
+        if (statusIng.indexOf(instanceDetail.value.status) !== -1 || instanceDetail.value.status === statusFail.value) {
           newItem.disabled = true
         }
         break
       case '调整配置':
-        if (instanceDetail.value.status !== statusStoped.value) {
+        if (instanceDetail.value.status !== statusStoped.value || instanceDetail.value.status === statusFail.value) {
           newItem.disabled = true
         }
         break
       case '删除实例':
-        if (instanceDetail.value.status === statusDeleting.value) {
+        if (instanceDetail.value.status === statusDeleting.value || instanceDetail.value.status === statusFail.value) {
+          newItem.disabled = true
+        }
+        break
+      case '强制删除':
+        if (instanceDetail.value.status !== statusFail.value) {
           newItem.disabled = true
         }
         break
@@ -517,6 +523,20 @@ const instanceDelete = async (id) => {
   })
 }
 
+const instanceForceDelete = async (id) => {
+  toast.add({ severity: 'info', summary: '强制释放实例', detail: '正在强制释放实例', life: 3000 });
+  setTimeout(() => {
+    getInstances()
+  }, 100);
+  await api.AdminInstancesForceDelete(id, { force: true }).then(async (res) => {
+    toast.add({ severity: 'success', summary: '强制释放实例', detail: '实例已强制释放', life: 3000 });
+    await getInstances()
+  }).catch(err => {
+    console.error(err)
+    toast.add({ severity: 'error', summary: '强制释放实例失败', detail: err.response.data.msg, life: 3000 });
+  })
+}
+
 const openInstanceModify = async () => {
   instanceConfiguration.value.gpu_count = instanceDetail.value.gpu_count
   instanceConfiguration.value.volume_size = instanceDetail.value.volume_size
@@ -595,6 +615,27 @@ const openInstanceDelete = () => {
   });
 }
 
+const openInstanceForceDelete = () => {
+  confirm.require({
+    header: '确认强制删除 实例ID: ' + instanceDetail.value.id,
+    message: '强制实例删除后，数据将无法恢复，请确认删除',
+    icon: 'pi pi-info-circle',
+    rejectProps: {
+      label: '取消',
+      severity: 'secondary',
+      outlined: true
+    },
+    acceptProps: {
+      label: '强制删除',
+      severity: 'danger'
+    },
+    accept: async () => {
+      await instanceForceDelete(instanceDetail.value.id)
+    },
+    reject: () => { }
+  });
+}
+
 const instanceModifyLabel = (id, label) => {
   api.AdminInstancesModifyLabel(id, { label: label }).then(async (res) => {
     toast.add({ severity: 'success', summary: '修改备注成功', detail: '已保存备注', life: 3000 });

frontend/src/views/users/InstanceList.vue

@@ -386,37 +386,37 @@ const showMenu = (event, instance) => {
     let newItem = { ...item }
     switch (item.label) {
       case '无卡模式开机':
-        if (instanceDetail.value.cpu_only === true || instanceDetail.value.status !== statusStoped.value) {
+        if (instanceDetail.value.cpu_only === true || instanceDetail.value.status !== statusStoped.value || instanceDetail.value.status === statusFail.value) {
           newItem.disabled = true
         }
         break
       case '开机':
-        if (instanceDetail.value.status === statusRunning.value || statusIng.indexOf(instanceDetail.value.status) !== -1) {
+        if (instanceDetail.value.status === statusRunning.value || statusIng.indexOf(instanceDetail.value.status) !== -1 || instanceDetail.value.status === statusFail.value) {
           newItem.disabled = true
         }
         break
       case '关机':
-        if (instanceDetail.value.status === statusStoped.value || statusIng.indexOf(instanceDetail.value.status) !== -1) {
+        if (instanceDetail.value.status === statusStoped.value || statusIng.indexOf(instanceDetail.value.status) !== -1 || instanceDetail.value.status === statusFail.value) {
           newItem.disabled = true
         }
         break
       case '暂停':
-        if (instanceDetail.value.status === statusPaused.value || instanceDetail.value.status === statusStoped.value || statusIng.indexOf(instanceDetail.value.status) !== -1) {
+        if (instanceDetail.value.status === statusPaused.value || instanceDetail.value.status === statusStoped.value || statusIng.indexOf(instanceDetail.value.status) !== -1 || instanceDetail.value.status === statusFail.value) {
           newItem.disabled = true
         }
         break
       case '重启实例':
-        if (statusIng.indexOf(instanceDetail.value.status) !== -1) {
+        if (statusIng.indexOf(instanceDetail.value.status) !== -1 || instanceDetail.value.status === statusFail.value) {
           newItem.disabled = true
         }
         break
       case '调整配置':
-        if (instanceDetail.value.status !== statusStoped.value) {
+        if (instanceDetail.value.status !== statusStoped.value || instanceDetail.value.status === statusFail.value) {
           newItem.disabled = true
         }
         break
       case '删除实例':
-        if (instanceDetail.value.status === statusDeleting.value) {
+        if (instanceDetail.value.status === statusDeleting.value || instanceDetail.value.status === statusFail.value) {
           newItem.disabled = true
         }
         break

models/instance.go

@@ -8,23 +8,34 @@ import (
 )
 
 type Status int
+type Action int
 
 const (
-	InstanceFail    Status = -1
+	InstanceStatusFail    Status = -1
-	InstanceRunning Status = 0
+	InstanceStatusRunning Status = 0
-	InstancePaused  Status = 1
+	InstanceStatusPaused  Status = 1
-	InstanceStopped Status = 2
+	InstanceStatusStopped Status = 2
 
-	InstanceReady      Status = 3
+	InstanceStatusReady      Status = 3
-	InstanceStarting   Status = 4
+	InstanceStatusStarting   Status = 4
-	InstanceStopping   Status = 5
+	InstanceStatusStopping   Status = 5
-	InstancePausing    Status = 6
+	InstanceStatusPausing    Status = 6
-	InstanceRestarting Status = 7
+	InstanceStatusRestarting Status = 7
-	InstanceModifying  Status = 8
+	InstanceStatusModifying  Status = 8
-	InstanceDeleting   Status = 9
+	InstanceStatusDeleting   Status = 9
 )
 
-var instanceIngStatus = []Status{InstanceReady, InstanceStarting, InstanceStopping, InstancePausing, InstanceRestarting, InstanceModifying, InstanceDeleting}
+const (
+	InstanceActionCreate  Action = 1
+	InstanceActionStart   Action = 2
+	InstanceActionPause   Action = 3
+	InstanceActionStop    Action = 4
+	InstanceActionRestart Action = 5
+	InstanceActionModify  Action = 6
+	InstanceActionDelete  Action = 7
+)
+
+var instanceIngStatus = []Status{InstanceStatusReady, InstanceStatusStarting, InstanceStatusStopping, InstanceStatusPausing, InstanceStatusRestarting, InstanceStatusModifying, InstanceStatusDeleting}
 
 type Instances struct {
 	ID uint `json:"id" gorm:"primary_key;autoIncrement;index"`
@@ -46,6 +57,7 @@ type Instances struct {
 	GrafanaAddress     string `json:"grafana_address" gorm:"type:varchar(255)"`
 	CodeServerAddress  string `json:"code_server_address" gorm:"type:varchar(255)"`
 	Status             Status `json:"status" gorm:"not null"` // Detail in Constants
+	FromAction         Action `json:"from_action"`
 
 	Label string `json:"label" gorm:"type:varchar(255)"`
 

routers/api/v1/admin/instance/add.go

@@ -85,7 +85,7 @@ func addHandler(ctx iris.Context) {
 
 		SshPasswd: crypto.Hex(16),
 
-		Status: models.InstanceReady,
+		Status: models.InstanceStatusReady,
 	}
 	result = database.DB.Create(&instance)
 	if result.Error != nil {

routers/api/v1/admin/instance/control.go

@@ -45,17 +45,17 @@ func controlHandler(ctx iris.Context) {
 		return
 	}
 
-	if req.Action == instanceController.ActionStop && instance.Status != models.InstanceRunning && instance.Status != models.InstancePaused {
+	if req.Action == instanceController.ActionStop && instance.Status != models.InstanceStatusRunning && instance.Status != models.InstanceStatusPaused {
 		middleware.Error(ctx, middleware.CodeInstanceStatusError, iris.StatusBadRequest)
 		return
 	}
 
-	if req.Action == instanceController.ActionPause && instance.Status != models.InstanceRunning {
+	if req.Action == instanceController.ActionPause && instance.Status != models.InstanceStatusRunning {
 		middleware.Error(ctx, middleware.CodeInstanceStatusError, iris.StatusBadRequest)
 		return
 	}
 
-	if req.Action == instanceController.ActionStart && instance.Status != models.InstanceStopped && instance.Status != models.InstancePaused {
+	if req.Action == instanceController.ActionStart && instance.Status != models.InstanceStatusStopped && instance.Status != models.InstanceStatusPaused {
 		middleware.Error(ctx, middleware.CodeInstanceStatusError, iris.StatusBadRequest)
 		return
 	}
@@ -71,7 +71,7 @@ func controlHandler(ctx iris.Context) {
 	}
 
 	status := instance.Status
-	if status == models.InstanceStopped && (req.Action == instanceController.ActionStart || req.Action == instanceController.ActionRestart) {
+	if status == models.InstanceStatusStopped && (req.Action == instanceController.ActionStart || req.Action == instanceController.ActionRestart) {
 		remainGpu, err := redis.RawDB.IncrBy(ctx, "remain_gpu:server:"+strconv.Itoa(int(server.ID)), int64(-instance.GpuCount)).Result()
 		if err != nil {
 			l.Error("incrby gpu num error: %v", err)
@@ -88,7 +88,7 @@ func controlHandler(ctx iris.Context) {
 
 	switch req.Action {
 	case instanceController.ActionStart:
-		result = database.DB.Model(&instance).Update("status", models.InstanceStarting)
+		result = database.DB.Model(&instance).Update("status", models.InstanceStatusStarting)
 		if result.Error != nil {
 			l.Error("update instance status error: %v", result.Error)
 			middleware.Error(ctx, middleware.CodeInstanceStartError, iris.StatusInternalServerError)
@@ -96,7 +96,7 @@ func controlHandler(ctx iris.Context) {
 		}
 
 	case instanceController.ActionPause:
-		result = database.DB.Model(&instance).Update("status", models.InstancePausing)
+		result = database.DB.Model(&instance).Update("status", models.InstanceStatusPausing)
 		if result.Error != nil {
 			l.Error("update instance status error: %v", result.Error)
 			middleware.Error(ctx, middleware.CodeInstancePauseError, iris.StatusInternalServerError)
@@ -104,7 +104,7 @@ func controlHandler(ctx iris.Context) {
 		}
 
 	case instanceController.ActionStop:
-		result = database.DB.Model(&instance).Update("status", models.InstanceStopping)
+		result = database.DB.Model(&instance).Update("status", models.InstanceStatusStopping)
 		if result.Error != nil {
 			l.Error("update instance status error: %v", result.Error)
 			middleware.Error(ctx, middleware.CodeInstanceStopError, iris.StatusInternalServerError)
@@ -113,7 +113,7 @@ func controlHandler(ctx iris.Context) {
 		redis.RawDB.IncrBy(ctx, "remain_gpu:server:"+strconv.Itoa(int(server.ID)), int64(instance.GpuCount))
 
 	case instanceController.ActionRestart:
-		result = database.DB.Model(&instance).Update("status", models.InstanceRestarting)
+		result = database.DB.Model(&instance).Update("status", models.InstanceStatusRestarting)
 		if result.Error != nil {
 			l.Error("update instance status error: %v", result.Error)
 			middleware.Error(ctx, middleware.CodeInstanceRestartError, iris.StatusInternalServerError)

routers/api/v1/admin/instance/delete.go

@@ -36,14 +36,14 @@ func deleteHandler(ctx iris.Context) {
 		return
 	}
 
-	result = database.DB.Model(&instance).Update("status", models.InstanceDeleting)
+	result = database.DB.Model(&instance).Update("status", models.InstanceStatusDeleting)
 	if result.Error != nil {
 		l.Error("update instance status error: %v", result.Error)
 		middleware.Error(ctx, middleware.CodeServerSaveError, iris.StatusInternalServerError)
 		return
 	}
 
-	if status == models.InstanceRunning || status == models.InstancePaused {
+	if status == models.InstanceStatusRunning || status == models.InstanceStatusPaused {
 		redis.RawDB.IncrBy(ctx, "remain_gpu:server:"+strconv.Itoa(int(instance.ServerID)), int64(instance.GpuCount))
 	}
 	redis.RawDB.IncrBy(ctx, "remain_volume:server:"+strconv.Itoa(int(instance.ServerID)), int64(instance.VolumeSize+30))

routers/api/v1/admin/instance/forceDelete.go

@@ -0,0 +1,54 @@
+package instances
+
+import (
+	"megrez/models"
+	"megrez/routers/api/v1/middleware"
+	"megrez/services/database"
+	"megrez/services/dispatcher"
+	"megrez/services/redis"
+	"strconv"
+
+	"github.com/kataras/iris/v12"
+)
+
+func forceDeleteHandler(ctx iris.Context) {
+	l.SetFunction("forceDeleteHandler")
+
+	id, err := ctx.Params().GetUint("id")
+	if err != nil {
+		middleware.Error(ctx, middleware.CodeBadRequest, iris.StatusBadRequest)
+		return
+	}
+
+	instance := models.Instances{
+		ID: id,
+	}
+	result := database.DB.First(&instance)
+	if result.Error != nil {
+		l.Error("get instance error: %v", result.Error)
+		middleware.Error(ctx, middleware.CodeInstanceDeleteError, iris.StatusInternalServerError)
+		return
+	}
+
+	status := instance.Status
+	if status != models.InstanceStatusFail {
+		middleware.Error(ctx, middleware.CodeInstanceStatusError, iris.StatusBadRequest)
+		return
+	}
+
+	if instance.FromAction == models.InstanceActionStop || instance.FromAction == models.InstanceActionPause || instance.FromAction == models.InstanceActionRestart {
+		redis.RawDB.IncrBy(ctx, "remain_gpu:server:"+strconv.Itoa(int(instance.ServerID)), int64(instance.GpuCount))
+		redis.RawDB.IncrBy(ctx, "remain_volume:server:"+strconv.Itoa(int(instance.ServerID)), int64(instance.VolumeSize+30))
+	}
+
+	dispatcherData := dispatcher.Data{
+		Type:       dispatcher.Delete,
+		Status:     status,
+		InstanceID: instance.ID,
+		Force:      true,
+	}
+	dispatcher.Push(instance.ServerID, dispatcherData)
+
+	middleware.Success(ctx)
+
+}

routers/api/v1/admin/instance/modify.go

@@ -57,7 +57,7 @@ func modifyHandler(ctx iris.Context) {
 		return
 	}
 
-	if instance.Status != models.InstanceStopped {
+	if instance.Status != models.InstanceStatusStopped {
 		middleware.Error(ctx, middleware.CodeInstanceStatusError, iris.StatusBadRequest)
 		return
 	}
@@ -106,7 +106,7 @@ func modifyHandler(ctx iris.Context) {
 	}
 
 	status := instance.Status
-	result = database.DB.Model(&instance).Update("status", models.InstanceModifying)
+	result = database.DB.Model(&instance).Update("status", models.InstanceStatusModifying)
 	if result.Error != nil {
 		l.Error("update instance status error: %v", result.Error)
 		middleware.Error(ctx, middleware.CodeServerSaveError, iris.StatusInternalServerError)

routers/api/v1/admin/instance/routers.go

@@ -34,4 +34,5 @@ func InitInstances(party router.Party) {
 	party.Post("/{id:uint}", middleware.SuperAdminCheck, modifyHandler)
 	party.Post("/{id:uint}/label", middleware.SuperAdminCheck, labelHandler)
 	party.Delete("/{id:uint}", middleware.SuperAdminCheck, deleteHandler)
+	party.Delete("/{id:uint}/force", middleware.SuperAdminCheck, forceDeleteHandler)
 }

routers/api/v1/user/instances/add.go

@@ -80,7 +80,7 @@ func addHandler(ctx iris.Context) {
 
 		SshPasswd: crypto.Hex(16),
 
-		Status: models.InstanceReady,
+		Status: models.InstanceStatusReady,
 	}
 	result = database.DB.Create(&instance)
 	if result.Error != nil {

routers/api/v1/user/instances/control.go

@@ -51,17 +51,17 @@ func controlHandler(ctx iris.Context) {
 		return
 	}
 
-	if req.Action == instanceController.ActionStop && instance.Status != models.InstanceRunning && instance.Status != models.InstancePaused {
+	if req.Action == instanceController.ActionStop && instance.Status != models.InstanceStatusRunning && instance.Status != models.InstanceStatusPaused {
 		middleware.Error(ctx, middleware.CodeInstanceStatusError, iris.StatusBadRequest)
 		return
 	}
 
-	if req.Action == instanceController.ActionPause && instance.Status != models.InstanceRunning {
+	if req.Action == instanceController.ActionPause && instance.Status != models.InstanceStatusRunning {
 		middleware.Error(ctx, middleware.CodeInstanceStatusError, iris.StatusBadRequest)
 		return
 	}
 
-	if req.Action == instanceController.ActionStart && instance.Status != models.InstanceStopped && instance.Status != models.InstancePaused {
+	if req.Action == instanceController.ActionStart && instance.Status != models.InstanceStatusStopped && instance.Status != models.InstanceStatusPaused {
 		middleware.Error(ctx, middleware.CodeInstanceStatusError, iris.StatusBadRequest)
 		return
 	}
@@ -77,7 +77,7 @@ func controlHandler(ctx iris.Context) {
 	}
 
 	status := instance.Status
-	if status == models.InstanceStopped && (req.Action == instanceController.ActionStart || req.Action == instanceController.ActionRestart) {
+	if status == models.InstanceStatusStopped && (req.Action == instanceController.ActionStart || req.Action == instanceController.ActionRestart) {
 		remainGpu, err := redis.RawDB.IncrBy(ctx, "remain_gpu:server:"+strconv.Itoa(int(server.ID)), int64(-instance.GpuCount)).Result()
 		if err != nil {
 			l.Error("incrby gpu num error: %v", err)
@@ -94,7 +94,7 @@ func controlHandler(ctx iris.Context) {
 
 	switch req.Action {
 	case instanceController.ActionStart:
-		result = database.DB.Model(&instance).Update("status", models.InstanceStarting)
+		result = database.DB.Model(&instance).Update("status", models.InstanceStatusStarting)
 		if result.Error != nil {
 			l.Error("update instance status error: %v", result.Error)
 			middleware.Error(ctx, middleware.CodeInstanceStartError, iris.StatusInternalServerError)
@@ -102,7 +102,7 @@ func controlHandler(ctx iris.Context) {
 		}
 
 	case instanceController.ActionPause:
-		result = database.DB.Model(&instance).Update("status", models.InstancePausing)
+		result = database.DB.Model(&instance).Update("status", models.InstanceStatusPausing)
 		if result.Error != nil {
 			l.Error("update instance status error: %v", result.Error)
 			middleware.Error(ctx, middleware.CodeInstancePauseError, iris.StatusInternalServerError)
@@ -110,7 +110,7 @@ func controlHandler(ctx iris.Context) {
 		}
 
 	case instanceController.ActionStop:
-		result = database.DB.Model(&instance).Update("status", models.InstanceStopping)
+		result = database.DB.Model(&instance).Update("status", models.InstanceStatusStopping)
 		if result.Error != nil {
 			l.Error("update instance status error: %v", result.Error)
 			middleware.Error(ctx, middleware.CodeInstanceStopError, iris.StatusInternalServerError)
@@ -119,7 +119,7 @@ func controlHandler(ctx iris.Context) {
 		redis.RawDB.IncrBy(ctx, "remain_gpu:server:"+strconv.Itoa(int(server.ID)), int64(instance.GpuCount))
 
 	case instanceController.ActionRestart:
-		result = database.DB.Model(&instance).Update("status", models.InstanceRestarting)
+		result = database.DB.Model(&instance).Update("status", models.InstanceStatusRestarting)
 		if result.Error != nil {
 			l.Error("update instance status error: %v", result.Error)
 			middleware.Error(ctx, middleware.CodeInstanceRestartError, iris.StatusInternalServerError)

routers/api/v1/user/instances/delete.go

@@ -40,14 +40,14 @@ func deleteHandler(ctx iris.Context) {
 		return
 	}
 
-	result = database.DB.Model(&instance).Update("status", models.InstanceDeleting)
+	result = database.DB.Model(&instance).Update("status", models.InstanceStatusDeleting)
 	if result.Error != nil {
 		l.Error("update instance status error: %v", result.Error)
 		middleware.Error(ctx, middleware.CodeServerSaveError, iris.StatusInternalServerError)
 		return
 	}
 
-	if status == models.InstanceRunning || status == models.InstancePaused {
+	if status == models.InstanceStatusRunning || status == models.InstanceStatusPaused {
 		redis.RawDB.IncrBy(ctx, "remain_gpu:server:"+strconv.Itoa(int(instance.ServerID)), int64(instance.GpuCount))
 	}
 	redis.RawDB.IncrBy(ctx, "remain_volume:server:"+strconv.Itoa(int(instance.ServerID)), int64(instance.VolumeSize+30))

routers/api/v1/user/instances/modify.go

@@ -63,7 +63,7 @@ func modifyHandler(ctx iris.Context) {
 		return
 	}
 
-	if instance.Status != models.InstanceStopped {
+	if instance.Status != models.InstanceStatusStopped {
 		middleware.Error(ctx, middleware.CodeInstanceStatusError, iris.StatusBadRequest)
 		return
 	}
@@ -112,7 +112,7 @@ func modifyHandler(ctx iris.Context) {
 	}
 
 	status := instance.Status
-	result = database.DB.Model(&instance).Update("status", models.InstanceModifying)
+	result = database.DB.Model(&instance).Update("status", models.InstanceStatusModifying)
 	if result.Error != nil {
 		l.Error("update instance status error: %v", result.Error)
 		middleware.Error(ctx, middleware.CodeServerSaveError, iris.StatusInternalServerError)

services/config/config.go

@@ -41,8 +41,9 @@ type logStruct struct {
 }
 
 type systemStruct struct {
-	Salt   string `yaml:"salt,omitempty"`
+	Salt     string `yaml:"salt,omitempty"`
-	Verify bool   `yaml:"verify,omitempty"`
+	Verify   bool   `yaml:"verify,omitempty"`
+	MountDir string `yaml:"mount_dir,omitempty"`
 }
 
 var config = configStruct{

services/config/exporter.go

@@ -53,6 +53,10 @@ func GetSystemVerify() bool {
 	return config.GetSystemVerify()
 }
 
+func GetSystemMountDir() string {
+	return config.GetSystemMountDir()
+}
+
 func Save() error {
 	return config.Save()
 }

services/config/internal.go

@@ -40,6 +40,10 @@ func (c *configStruct) GetSystemVerify() bool {
 	return c.System.Verify
 }
 
+func (c *configStruct) GetSystemMountDir() string {
+	return c.System.MountDir
+}
+
 func (c *configStruct) Save() error {
 	return c.save()
 }

services/dispatcher/add.go

@@ -37,6 +37,7 @@ func add(serverID uint, data Data) (err error) {
 		ctx := context.Background()
 		redis.RawDB.IncrBy(ctx, "remain_gpu:server:"+strconv.Itoa(int(serverID)), int64(instance.GpuCount))
 		redis.RawDB.IncrBy(ctx, "remain_volume:server:"+strconv.Itoa(int(serverID)), int64(instance.VolumeSize+30))
+		database.DB.Model(&instance).Update("status", models.InstanceStatusFail).Update("from_action", models.InstanceActionCreate)
 		return
 	}
 

services/dispatcher/control.go

@@ -37,35 +37,37 @@ func control(serverID uint, data Data) (err error) {
 		return errors.New("instance status error")
 	}
 
-	if data.Action == instanceController.ActionPause && data.Status != models.InstanceRunning {
+	if data.Action == instanceController.ActionPause && data.Status != models.InstanceStatusRunning {
 		lc.Error("instance status error")
 		return errors.New("instance status error")
 	}
 
-	if data.Action == instanceController.ActionStart && data.Status != models.InstanceStopped && data.Status != models.InstancePaused {
+	if data.Action == instanceController.ActionStart && data.Status != models.InstanceStatusStopped && data.Status != models.InstanceStatusPaused {
 		lc.Error("instance status error")
 		return errors.New("instance status error")
 	}
 
 	switch data.Action {
 	case instanceController.ActionStart:
-		if data.Status == models.InstancePaused {
+		if data.Status == models.InstanceStatusPaused {
 			err = instanceController.Continue(&instance)
 			if err != nil {
+				database.DB.Model(&instance).Update("status", models.InstanceStatusFail).Update("from_action", models.InstanceActionStart)
 				lc.Error("instance continue error: %v", err)
 				return
 			}
-		} else if data.Status == models.InstanceStopped {
+		} else if data.Status == models.InstanceStatusStopped {
 			err = instanceController.Restart(&instance)
 			if err != nil {
-				lc.Error("instance restart error: %v", err)
 				ctx := context.Background()
 				redis.RawDB.IncrBy(ctx, "remain_gpu:server:"+strconv.Itoa(int(serverID)), int64(instance.GpuCount))
+				database.DB.Model(&instance).Update("status", models.InstanceStatusFail).Update("from_action", models.InstanceActionStart)
+				lc.Error("instance restart error: %v", err)
 				return
 			}
 		}
 
-		if data.Status == models.InstanceStopped && instance.Status == models.InstanceRunning {
+		if data.Status == models.InstanceStatusStopped && instance.Status == models.InstanceStatusRunning {
 			server.GpuUsed += instance.GpuCount
 			result = database.DB.Save(&server)
 			if result.Error != nil {
@@ -79,6 +81,7 @@ func control(serverID uint, data Data) (err error) {
 	case instanceController.ActionPause:
 		err = instanceController.Pause(&instance)
 		if err != nil {
+			database.DB.Model(&instance).Update("status", models.InstanceStatusFail).Update("from_action", models.InstanceActionPause)
 			lc.Error("instance pause error: %v", err)
 			return
 		}
@@ -88,14 +91,16 @@ func control(serverID uint, data Data) (err error) {
 	case instanceController.ActionStop:
 		err = instanceController.Stop(&instance)
 		if err != nil {
+			database.DB.Model(&instance).Update("status", models.InstanceStatusFail).Update("from_action", models.InstanceActionStop)
 			lc.Error("instance stop error: %v", err)
 			return
 		}
 
-		if (data.Status == models.InstanceRunning || data.Status == models.InstancePaused) && instance.Status == models.InstanceStopped {
+		if (data.Status == models.InstanceStatusRunning || data.Status == models.InstanceStatusPaused) && instance.Status == models.InstanceStatusStopped {
 			server.GpuUsed -= instance.GpuCount
 			result = database.DB.Save(&server)
 			if result.Error != nil {
+				database.DB.Model(&instance).Update("status", models.InstanceStatusFail)
 				lc.Error("save server error: %v", result.Error)
 				return result.Error
 			}
@@ -106,14 +111,16 @@ func control(serverID uint, data Data) (err error) {
 	case instanceController.ActionRestart:
 		err = instanceController.Restart(&instance)
 		if err != nil {
+			database.DB.Model(&instance).Update("status", models.InstanceStatusFail).Update("from_action", models.InstanceActionRestart)
 			lc.Error("instance restart error: %v", err)
 			return
 		}
 
-		if data.Status == models.InstanceStopped && instance.Status == models.InstanceRunning {
+		if data.Status == models.InstanceStatusStopped && instance.Status == models.InstanceStatusRunning {
 			server.GpuUsed += instance.GpuCount
 			result = database.DB.Save(&server)
 			if result.Error != nil {
+				database.DB.Model(&instance).Update("status", models.InstanceStatusFail)
 				lc.Error("save server error: %v", result.Error)
 				return result.Error
 			}

services/dispatcher/delete.go

@@ -34,25 +34,47 @@ func delete(serverID uint, data Data) (err error) {
 	err = instanceController.Delete(&instance)
 	if err != nil {
 		lc.Error("delete instance error: %v", err)
-		ctx := context.Background()
+		if !data.Force {
-		if data.Status == models.InstanceRunning || data.Status == models.InstancePaused {
+			ctx := context.Background()
-			redis.RawDB.IncrBy(ctx, "remain_gpu:server:"+strconv.Itoa(int(serverID)), int64(-instance.GpuCount))
+			if data.Status == models.InstanceStatusRunning || data.Status == models.InstanceStatusPaused {
+				redis.RawDB.IncrBy(ctx, "remain_gpu:server:"+strconv.Itoa(int(serverID)), int64(-instance.GpuCount))
+			}
+			redis.RawDB.IncrBy(ctx, "remain_volume:server:"+strconv.Itoa(int(serverID)), int64(-instance.VolumeSize-30))
+			database.DB.Model(&instance).Update("status", models.InstanceStatusFail).Update("from_action", models.InstanceActionDelete)
+			return
 		}
-		redis.RawDB.IncrBy(ctx, "remain_volume:server:"+strconv.Itoa(int(serverID)), int64(-instance.VolumeSize-30))
+	}
+
+	if !data.Force {
+		server.VolumeUsed -= instance.VolumeSize + 30
+		if data.Status == models.InstanceStatusRunning || data.Status == models.InstanceStatusPaused {
+			server.GpuUsed -= instance.GpuCount
+		}
+		result = database.DB.Save(&server)
+		if result.Error != nil {
+			lc.Error("save server error: %v", result.Error)
+			return result.Error
+		}
+		lc.Info("delete instance success: %v", instance.ID)
+	} else {
+		if instance.FromAction != models.InstanceActionCreate {
+			server.VolumeUsed -= instance.VolumeSize + 30
+		}
+		if instance.FromAction == models.InstanceActionStop || instance.FromAction == models.InstanceActionPause || instance.FromAction == models.InstanceActionRestart {
+			server.GpuUsed -= instance.GpuCount
+		}
+		result = database.DB.Save(&server)
+		if result.Error != nil {
+			lc.Error("save server error: %v", result.Error)
+			return result.Error
+		}
+
+		result = database.DB.Delete(&instance)
+		if result.Error != nil {
+			lc.Error("force delete instance error: %v", result.Error)
+		}
+		lc.Info("force delete instance success: %v", instance.ID)
 		return
 	}
-
-	server.VolumeUsed -= instance.VolumeSize + 30
-	if data.Status == models.InstanceRunning || data.Status == models.InstancePaused {
-		server.GpuUsed -= instance.GpuCount
-	}
-	result = database.DB.Save(&server)
-	if result.Error != nil {
-		lc.Error("save server error: %v", result.Error)
-		return result.Error
-	}
-
-	lc.Info("delete instance success: %v", instance.ID)
-
 	return
 }

services/dispatcher/interface.go

@@ -21,6 +21,7 @@ type Data struct {
 	VolumeSize *int `json:"volume_size,omitempty"`
 
 	Action instanceController.Action `json:"action,omitempty"`
+	Force  bool                      `json:"force,omitempty"`
 }
 
 type Type int

services/dispatcher/modify.go

@@ -38,7 +38,7 @@ func modify(serverID uint, data Data) (err error) {
 		return result.Error
 	}
 
-	if data.Status != models.InstanceStopped {
+	if data.Status != models.InstanceStatusStopped {
 		lc.Error("instance status error")
 		return errors.New("instance status error")
 	}
@@ -60,14 +60,15 @@ func modify(serverID uint, data Data) (err error) {
 
 	err = instanceController.Patch(&instance, gpuCount, volumeSize, data.CpuOnly)
 	if err != nil {
-		lc.Error("patch instance error: %v", err)
 		ctx := context.Background()
 		redis.RawDB.IncrBy(ctx, "remain_gpu:server:"+strconv.Itoa(int(serverID)), int64(gpuCount))
 		redis.RawDB.IncrBy(ctx, "remain_volume:server:"+strconv.Itoa(int(serverID)), int64(volumeSize-oldVolumeSize))
+		database.DB.Model(&instance).Update("status", models.InstanceStatusFail).Update("from_action", models.InstanceActionModify)
+		lc.Error("patch instance error: %v", err)
 		return
 	}
 
-	if data.Status == models.InstanceStopped {
+	if data.Status == models.InstanceStatusStopped {
 		server.GpuUsed += instance.GpuCount
 	}
 	server.VolumeUsed += instance.VolumeSize - oldVolumeSize

services/instanceController/continue.go

@@ -12,7 +12,7 @@ import (
 func Continue(instance *models.Instances) (err error) {
 	l.SetFunction("Start")
 
-	instance.Status = models.InstanceStarting
+	instance.Status = models.InstanceStatusStarting
 	result := database.DB.Save(&instance)
 	if result.Error != nil {
 		l.Error("save instance error: %v", result.Error)
@@ -34,7 +34,7 @@ func Continue(instance *models.Instances) (err error) {
 		return err
 	}
 
-	instance.Status = models.InstanceRunning
+	instance.Status = models.InstanceStatusRunning
 	result = database.DB.Save(&instance)
 	if result.Error != nil {
 		l.Error("save instance error: %v", result.Error)

services/instanceController/createInstance.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"megrez/libs/crypto"
 	"megrez/libs/request"
+	"megrez/services/config"
 	"strconv"
 )
 
@@ -47,6 +48,7 @@ func createInstance(ip string, port int, apikey string,
 		},
 		Env: []string{
 			"NVIDIA_DRIVER_CAPABILITIES=video,compute,utility",
+			"NVIDIA_VISIBLE_DEVICES=none",
 		},
 	}
 
@@ -56,13 +58,16 @@ func createInstance(ip string, port int, apikey string,
 				Src:  volumeName,
 				Dest: "/root/megrez-tmp",
 			},
-			{
-				Src:  "/data/pub",
-				Dest: "/root/megrez-pub",
-			},
 		}
 	}
 
+	if config.GetSystemMountDir() != "" {
+		data.Binds = append(data.Binds, bindStruct{
+			Src:  config.GetSystemMountDir(),
+			Dest: "/root/megrez-pub",
+		})
+	}
+
 	reqBytes, err := json.Marshal(data)
 	if err != nil {
 		l.Error("marshal request data error: %v", err)

services/instanceController/delete.go

@@ -8,7 +8,7 @@ import (
 func Delete(instance *models.Instances) (err error) {
 	l.SetFunction("Delete")
 
-	instance.Status = models.InstanceDeleting
+	instance.Status = models.InstanceStatusDeleting
 	result := database.DB.Save(&instance)
 	if result.Error != nil {
 		l.Error("save instance error: %v", result.Error)

services/instanceController/patch.go

@@ -34,7 +34,7 @@ type volumePatchStruct struct {
 func Patch(instance *models.Instances, gpuCount, volumeSize int, cpuOnly bool) (err error) {
 	l.SetFunction("Patch")
 
-	instance.Status = models.InstanceModifying
+	instance.Status = models.InstanceStatusModifying
 	result := database.DB.Save(&instance)
 	if result.Error != nil {
 		l.Error("save instance error: %v", result.Error)
@@ -42,7 +42,7 @@ func Patch(instance *models.Instances, gpuCount, volumeSize int, cpuOnly bool) (
 	}
 
 	if gpuCount == instance.GpuCount && volumeSize == instance.VolumeSize && cpuOnly == instance.CpuOnly {
-		instance.Status = models.InstanceStopped
+		instance.Status = models.InstanceStatusStopped
 		result = database.DB.Save(&instance)
 		if result.Error != nil {
 			l.Error("save instance error: %v", result.Error)
@@ -113,7 +113,7 @@ func Patch(instance *models.Instances, gpuCount, volumeSize int, cpuOnly bool) (
 
 		instance.CpuOnly = true
 		instance.GpuCount = 0
-		instance.Status = models.InstanceRunning
+		instance.Status = models.InstanceStatusRunning
 		result = database.DB.Save(&instance)
 		if result.Error != nil {
 			l.Error("save instance error: %v", result.Error)
@@ -162,7 +162,7 @@ func Patch(instance *models.Instances, gpuCount, volumeSize int, cpuOnly bool) (
 
 	instance.CpuOnly = false
 	instance.GpuCount = gpuCount
-	instance.Status = models.InstanceRunning
+	instance.Status = models.InstanceStatusRunning
 	result = database.DB.Save(&instance)
 	if result.Error != nil {
 		l.Error("save instance error: %v", result.Error)

services/instanceController/pause.go

@@ -12,7 +12,7 @@ import (
 func Pause(instance *models.Instances) (err error) {
 	l.SetFunction("Pause")
 
-	instance.Status = models.InstancePausing
+	instance.Status = models.InstanceStatusPausing
 	result := database.DB.Save(&instance)
 	if result.Error != nil {
 		l.Error("save instance error: %v", result.Error)
@@ -34,7 +34,7 @@ func Pause(instance *models.Instances) (err error) {
 		return err
 	}
 
-	instance.Status = models.InstancePaused
+	instance.Status = models.InstanceStatusPaused
 	result = database.DB.Save(&instance)
 	if result.Error != nil {
 		l.Error("save instance error: %v", result.Error)

services/instanceController/restart.go

@@ -12,10 +12,10 @@ import (
 func Restart(instance *models.Instances) (err error) {
 	l.SetFunction("Restart")
 
-	if instance.Status == models.InstanceStopped || instance.Status == models.InstancePaused {
+	if instance.Status == models.InstanceStatusStopped || instance.Status == models.InstanceStatusPaused {
-		instance.Status = models.InstanceStarting
+		instance.Status = models.InstanceStatusStarting
 	} else {
-		instance.Status = models.InstanceRestarting
+		instance.Status = models.InstanceStatusRestarting
 	}
 	result := database.DB.Save(&instance)
 	if result.Error != nil {
@@ -65,7 +65,7 @@ func Restart(instance *models.Instances) (err error) {
 	instance.GrafanaAddress = server.IP + ":" + portBindings["3000"]
 	instance.CodeServerAddress = server.IP + ":" + portBindings["8080"]
 
-	instance.Status = models.InstanceRunning
+	instance.Status = models.InstanceStatusRunning
 	result = database.DB.Save(&instance)
 	if result.Error != nil {
 		l.Error("save instance error: %v", result.Error)

services/instanceController/stop.go

@@ -12,7 +12,7 @@ import (
 func Stop(instance *models.Instances) (err error) {
 	l.SetFunction("Stop")
 
-	instance.Status = models.InstanceStopping
+	instance.Status = models.InstanceStatusStopping
 	result := database.DB.Save(&instance)
 	if result.Error != nil {
 		l.Error("save instance error: %v", result.Error)
@@ -34,7 +34,7 @@ func Stop(instance *models.Instances) (err error) {
 		return err
 	}
 
-	instance.Status = models.InstanceStopped
+	instance.Status = models.InstanceStatusStopped
 	result = database.DB.Save(&instance)
 	if result.Error != nil {
 		l.Error("save instance error: %v", result.Error)