diff --git a/agent/app/api/v2/snapshot.go b/agent/app/api/v2/snapshot.go index fffff3e13318..8bca5bf5b4c7 100644 --- a/agent/app/api/v2/snapshot.go +++ b/agent/app/api/v2/snapshot.go @@ -7,6 +7,21 @@ import ( "github.com/gin-gonic/gin" ) +// @Tags System Setting +// @Summary Load system snapshot data +// @Description 获取系统快照数据 +// @Success 200 +// @Security ApiKeyAuth +// @Router /settings/snapshot/load [get] +func (b *BaseApi) LoadSnapshotData(c *gin.Context) { + data, err := snapshotService.LoadSnapshotData() + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) + return + } + helper.SuccessWithData(c, data) +} + // @Tags System Setting // @Summary Create system snapshot // @Description 创建系统快照 @@ -30,21 +45,21 @@ func (b *BaseApi) CreateSnapshot(c *gin.Context) { } // @Tags System Setting -// @Summary Import system snapshot -// @Description 导入已有快照 +// @Summary Recreate system snapshot +// @Description 创建系统快照重试 // @Accept json -// @Param request body dto.SnapshotImport true "request" +// @Param request body dto.OperateByID true "request" // @Success 200 // @Security ApiKeyAuth -// @Router /settings/snapshot/import [post] -// @x-panel-log {"bodyKeys":["from", "names"],"paramKeys":[],"BeforeFunctions":[],"formatZH":"从 [from] 同步系统快照 [names]","formatEN":"Sync system snapshots [names] from [from]"} -func (b *BaseApi) ImportSnapshot(c *gin.Context) { - var req dto.SnapshotImport +// @Router /settings/snapshot/recrete [post] +// @x-panel-log {"bodyKeys":["id"],"paramKeys":[],"BeforeFunctions":[{"input_column":"id","input_value":"id","isList":false,"db":"snapshots","output_column":"name","output_value":"name"}],"formatZH":"重试创建快照 [name]","formatEN":recrete the snapshot [name]"} +func (b *BaseApi) RecreateSnapshot(c *gin.Context) { + var req dto.OperateByID if err := helper.CheckBindAndValidate(&req, c); err != nil { return } - if err := snapshotService.SnapshotImport(req); err != nil { + if err := snapshotService.SnapshotReCreate(req.ID); err != nil { helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) return } @@ -52,25 +67,25 @@ func (b *BaseApi) ImportSnapshot(c *gin.Context) { } // @Tags System Setting -// @Summary Load Snapshot status -// @Description 获取快照状态 +// @Summary Import system snapshot +// @Description 导入已有快照 // @Accept json -// @Param request body dto.OperateByID true "request" +// @Param request body dto.SnapshotImport true "request" // @Success 200 // @Security ApiKeyAuth -// @Router /settings/snapshot/status [post] -func (b *BaseApi) LoadSnapShotStatus(c *gin.Context) { - var req dto.OperateByID +// @Router /settings/snapshot/import [post] +// @x-panel-log {"bodyKeys":["from", "names"],"paramKeys":[],"BeforeFunctions":[],"formatZH":"从 [from] 同步系统快照 [names]","formatEN":"Sync system snapshots [names] from [from]"} +func (b *BaseApi) ImportSnapshot(c *gin.Context) { + var req dto.SnapshotImport if err := helper.CheckBindAndValidate(&req, c); err != nil { return } - data, err := snapshotService.LoadSnapShotStatus(req.ID) - if err != nil { + if err := snapshotService.SnapshotImport(req); err != nil { helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) return } - helper.SuccessWithData(c, data) + helper.SuccessWithData(c, nil) } // @Tags System Setting diff --git a/agent/app/dto/setting.go b/agent/app/dto/setting.go index a7ac6d679d68..d9d015f3ddef 100644 --- a/agent/app/dto/setting.go +++ b/agent/app/dto/setting.go @@ -1,7 +1,5 @@ package dto -import "time" - type SettingInfo struct { SystemIP string `json:"systemIP"` DockerSockPath string `json:"dockerSockPath"` @@ -34,64 +32,6 @@ type SettingUpdate struct { Value string `json:"value"` } -type SnapshotStatus struct { - Panel string `json:"panel"` - PanelInfo string `json:"panelInfo"` - DaemonJson string `json:"daemonJson"` - AppData string `json:"appData"` - PanelData string `json:"panelData"` - BackupData string `json:"backupData"` - - Compress string `json:"compress"` - Size string `json:"size"` - Upload string `json:"upload"` -} - -type SnapshotCreate struct { - ID uint `json:"id"` - SourceAccountIDs string `json:"sourceAccountIDs" validate:"required"` - DownloadAccountID uint `json:"downloadAccountID" validate:"required"` - Description string `json:"description" validate:"max=256"` - Secret string `json:"secret"` -} -type SnapshotRecover struct { - IsNew bool `json:"isNew"` - ReDownload bool `json:"reDownload"` - ID uint `json:"id" validate:"required"` - Secret string `json:"secret"` -} -type SnapshotBatchDelete struct { - DeleteWithFile bool `json:"deleteWithFile"` - Ids []uint `json:"ids" validate:"required"` -} - -type SnapshotImport struct { - BackupAccountID uint `json:"backupAccountID"` - Names []string `json:"names"` - Description string `json:"description" validate:"max=256"` -} - -type SnapshotInfo struct { - ID uint `json:"id"` - Name string `json:"name"` - Description string `json:"description" validate:"max=256"` - From string `json:"from"` - DefaultDownload string `json:"defaultDownload"` - Status string `json:"status"` - Message string `json:"message"` - CreatedAt time.Time `json:"createdAt"` - Version string `json:"version"` - Size int64 `json:"size"` - - InterruptStep string `json:"interruptStep"` - RecoverStatus string `json:"recoverStatus"` - RecoverMessage string `json:"recoverMessage"` - LastRecoveredAt string `json:"lastRecoveredAt"` - RollbackStatus string `json:"rollbackStatus"` - RollbackMessage string `json:"rollbackMessage"` - LastRollbackedAt string `json:"lastRollbackedAt"` -} - type SyncTime struct { NtpSite string `json:"ntpSite" validate:"required"` } diff --git a/agent/app/dto/snapshot.go b/agent/app/dto/snapshot.go new file mode 100644 index 000000000000..a2e85d503ee9 --- /dev/null +++ b/agent/app/dto/snapshot.go @@ -0,0 +1,103 @@ +package dto + +import "time" + +type SnapshotStatus struct { + BaseData string `json:"baseData"` + AppImage string `json:"appImage"` + PanelData string `json:"panelData"` + BackupData string `json:"backupData"` + + Compress string `json:"compress"` + Size string `json:"size"` + Upload string `json:"upload"` +} + +type SnapshotCreate struct { + ID uint `json:"id"` + Name string `json:"name"` + TaskID string `json:"taskID"` + SourceAccountIDs string `json:"sourceAccountIDs" validate:"required"` + DownloadAccountID uint `json:"downloadAccountID" validate:"required"` + Description string `json:"description" validate:"max=256"` + Secret string `json:"secret"` + InterruptStep string `json:"interruptStep"` + + AppData []DataTree `json:"appData"` + BackupData []DataTree `json:"backupData"` + PanelData []DataTree `json:"panelData"` + + WithMonitorData bool `json:"withMonitorData"` + WithLoginLog bool `json:"withLoginLog"` + WithOperationLog bool `json:"withOperationLog"` + WithSystemLog bool `json:"withSystemLog"` + WithTaskLog bool `json:"withTaskLog"` +} + +type SnapshotData struct { + AppData []DataTree `json:"appData"` + BackupData []DataTree `json:"backupData"` + PanelData []DataTree `json:"panelData"` + + WithMonitorData bool `json:"withMonitorData"` + WithLoginLog bool `json:"withLoginLog"` + WithOperationLog bool `json:"withOperationLog"` + WithSystemLog bool `json:"withSystemLog"` + WithTaskLog bool `json:"withTaskLog"` +} +type DataTree struct { + ID string `json:"id"` + Label string `json:"label"` + Key string `json:"key"` + Name string `json:"name"` + Size uint64 `json:"size"` + IsCheck bool `json:"isCheck"` + IsDisable bool `json:"isDisable"` + + Path string `json:"path"` + + RelationItemID string `json:"relationItemID"` + Children []DataTree `json:"children"` +} +type SnapshotRecover struct { + IsNew bool `json:"isNew"` + ReDownload bool `json:"reDownload"` + ID uint `json:"id" validate:"required"` + TaskID string `json:"taskID"` + Secret string `json:"secret"` +} +type SnapshotBatchDelete struct { + DeleteWithFile bool `json:"deleteWithFile"` + Ids []uint `json:"ids" validate:"required"` +} + +type SnapshotImport struct { + BackupAccountID uint `json:"backupAccountID"` + Names []string `json:"names"` + Description string `json:"description" validate:"max=256"` +} + +type SnapshotInfo struct { + ID uint `json:"id"` + Name string `json:"name"` + Description string `json:"description" validate:"max=256"` + From string `json:"from"` + DefaultDownload string `json:"defaultDownload"` + Status string `json:"status"` + Message string `json:"message"` + CreatedAt time.Time `json:"createdAt"` + Version string `json:"version"` + Size int64 `json:"size"` + + TaskID string `json:"taskID"` + TaskRecoverID string `json:"taskRecoverID"` + TaskRollbackID string `json:"taskRollbackID"` + + InterruptStep string `json:"interruptStep"` + RecoverStatus string `json:"recoverStatus"` + RecoverMessage string `json:"recoverMessage"` + LastRecoveredAt string `json:"lastRecoveredAt"` + RollbackStatus string `json:"rollbackStatus"` + RollbackMessage string `json:"rollbackMessage"` + LastRollbackedAt string `json:"lastRollbackedAt"` +} diff --git a/agent/app/model/snapshot.go b/agent/app/model/snapshot.go index 1771051122b5..d0c15cd440b9 100644 --- a/agent/app/model/snapshot.go +++ b/agent/app/model/snapshot.go @@ -3,6 +3,7 @@ package model type Snapshot struct { BaseModel Name string `json:"name" gorm:"not null;unique"` + Secret string `json:"secret"` Description string `json:"description"` SourceAccountIDs string `json:"sourceAccountIDs"` DownloadAccountID uint `json:"downloadAccountID"` @@ -10,26 +11,22 @@ type Snapshot struct { Message string `json:"message"` Version string `json:"version"` + TaskID string `json:"taskID"` + TaskRecoverID string `json:"taskRecoverID"` + TaskRollbackID string `json:"taskRollbackID"` + + AppData string `json:"appData"` + PanelData string `json:"panelData"` + BackupData string `json:"backupData"` + WithMonitorData bool `json:"withMonitorData"` + WithLoginLog bool `json:"withLoginLog"` + WithOperationLog bool `json:"withOperationLog"` + WithSystemLog bool `json:"withSystemLog"` + WithTaskLog bool `json:"withTaskLog"` + InterruptStep string `json:"interruptStep"` RecoverStatus string `json:"recoverStatus"` RecoverMessage string `json:"recoverMessage"` - LastRecoveredAt string `json:"lastRecoveredAt"` RollbackStatus string `json:"rollbackStatus"` RollbackMessage string `json:"rollbackMessage"` - LastRollbackAt string `json:"lastRollbackAt"` -} - -type SnapshotStatus struct { - BaseModel - SnapID uint `json:"snapID"` - Panel string `json:"panel" gorm:"default:Running"` - PanelInfo string `json:"panelInfo" gorm:"default:Running"` - DaemonJson string `json:"daemonJson" gorm:"default:Running"` - AppData string `json:"appData" gorm:"default:Running"` - PanelData string `json:"panelData" gorm:"default:Running"` - BackupData string `json:"backupData" gorm:"default:Running"` - - Compress string `json:"compress" gorm:"default:Waiting"` - Size string `json:"size" ` - Upload string `json:"upload" gorm:"default:Waiting"` } diff --git a/agent/app/repo/snapshot.go b/agent/app/repo/snapshot.go index dcb3dd6063a8..289470178113 100644 --- a/agent/app/repo/snapshot.go +++ b/agent/app/repo/snapshot.go @@ -12,12 +12,6 @@ type ISnapshotRepo interface { Update(id uint, vars map[string]interface{}) error Page(limit, offset int, opts ...DBOption) (int64, []model.Snapshot, error) Delete(opts ...DBOption) error - - GetStatus(snapID uint) (model.SnapshotStatus, error) - GetStatusList(opts ...DBOption) ([]model.SnapshotStatus, error) - CreateStatus(snap *model.SnapshotStatus) error - DeleteStatus(snapID uint) error - UpdateStatus(id uint, vars map[string]interface{}) error } func NewISnapshotRepo() ISnapshotRepo { @@ -73,33 +67,3 @@ func (u *SnapshotRepo) Delete(opts ...DBOption) error { } return db.Delete(&model.Snapshot{}).Error } - -func (u *SnapshotRepo) GetStatus(snapID uint) (model.SnapshotStatus, error) { - var data model.SnapshotStatus - if err := global.DB.Where("snap_id = ?", snapID).First(&data).Error; err != nil { - return data, err - } - return data, nil -} - -func (u *SnapshotRepo) GetStatusList(opts ...DBOption) ([]model.SnapshotStatus, error) { - var status []model.SnapshotStatus - db := global.DB.Model(&model.SnapshotStatus{}) - for _, opt := range opts { - db = opt(db) - } - err := db.Find(&status).Error - return status, err -} - -func (u *SnapshotRepo) CreateStatus(snap *model.SnapshotStatus) error { - return global.DB.Create(snap).Error -} - -func (u *SnapshotRepo) DeleteStatus(snapID uint) error { - return global.DB.Where("snap_id = ?", snapID).Delete(&model.SnapshotStatus{}).Error -} - -func (u *SnapshotRepo) UpdateStatus(id uint, vars map[string]interface{}) error { - return global.DB.Model(&model.SnapshotStatus{}).Where("id = ?", id).Updates(vars).Error -} diff --git a/agent/app/repo/task.go b/agent/app/repo/task.go index 65f07ebcf779..b91c2afcc3eb 100644 --- a/agent/app/repo/task.go +++ b/agent/app/repo/task.go @@ -2,6 +2,7 @@ package repo import ( "context" + "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" @@ -13,7 +14,7 @@ type TaskRepo struct { } type ITaskRepo interface { - Create(ctx context.Context, task *model.Task) error + Save(ctx context.Context, task *model.Task) error GetFirst(opts ...DBOption) (model.Task, error) Page(page, size int, opts ...DBOption) (int64, []model.Task, error) Update(ctx context.Context, task *model.Task) error @@ -64,8 +65,8 @@ func (t TaskRepo) WithResourceID(id uint) DBOption { } } -func (t TaskRepo) Create(ctx context.Context, task *model.Task) error { - return getTaskTx(ctx).Create(&task).Error +func (t TaskRepo) Save(ctx context.Context, task *model.Task) error { + return getTaskTx(ctx).Save(&task).Error } func (t TaskRepo) GetFirst(opts ...DBOption) (model.Task, error) { diff --git a/agent/app/service/container_compose.go b/agent/app/service/container_compose.go index d76dc14f3074..78ca8a479604 100644 --- a/agent/app/service/container_compose.go +++ b/agent/app/service/container_compose.go @@ -141,7 +141,7 @@ func (u *ContainerService) TestCompose(req dto.ComposeCreate) (bool, error) { if err := u.loadPath(&req); err != nil { return false, err } - cmd := exec.Command("docker compose", "-f", req.Path, "config") + cmd := exec.Command("docker", "compose", "-f", req.Path, "config") stdout, err := cmd.CombinedOutput() if err != nil { return false, errors.New(string(stdout)) diff --git a/agent/app/service/container_network.go b/agent/app/service/container_network.go index e894d4bd909d..2e5e01028382 100644 --- a/agent/app/service/container_network.go +++ b/agent/app/service/container_network.go @@ -10,7 +10,6 @@ import ( "github.com/1Panel-dev/1Panel/agent/buserr" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/utils/docker" - "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/network" ) @@ -20,7 +19,7 @@ func (u *ContainerService) PageNetwork(req dto.SearchWithPage) (int64, interface return 0, nil, err } defer client.Close() - list, err := client.NetworkList(context.TODO(), types.NetworkListOptions{}) + list, err := client.NetworkList(context.TODO(), network.ListOptions{}) if err != nil { return 0, nil, err } @@ -83,7 +82,7 @@ func (u *ContainerService) ListNetwork() ([]dto.Options, error) { return nil, err } defer client.Close() - list, err := client.NetworkList(context.TODO(), types.NetworkListOptions{}) + list, err := client.NetworkList(context.TODO(), network.ListOptions{}) if err != nil { return nil, err } diff --git a/agent/app/service/cronjob_backup.go b/agent/app/service/cronjob_backup.go index 33e6d98da1d1..b00b45976c72 100644 --- a/agent/app/service/cronjob_backup.go +++ b/agent/app/service/cronjob_backup.go @@ -192,7 +192,7 @@ func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.T return nil } -func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Time, logPath string) error { +func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Time) error { accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ",")) if err != nil { return err @@ -206,15 +206,18 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Ti record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs record.FileDir = "system_snapshot" + versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion")) req := dto.SnapshotCreate{ + Name: fmt.Sprintf("snapshot-1panel-%s-linux-%s-%s", versionItem.Value, loadOs(), startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5)), + Secret: cronjob.Secret, + SourceAccountIDs: record.SourceAccountIDs, DownloadAccountID: cronjob.DownloadAccountID, } - name, err := NewISnapshotService().HandleSnapshot(true, logPath, req, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5), cronjob.Secret) - if err != nil { + if err := NewISnapshotService().HandleSnapshot(req); err != nil { return err } - record.FileName = name + ".tar.gz" + record.FileName = req.Name + ".tar.gz" if err := backupRepo.CreateRecord(&record); err != nil { global.LOG.Errorf("save backup record failed, err: %v", err) diff --git a/agent/app/service/cronjob_helper.go b/agent/app/service/cronjob_helper.go index 01cf1f0e319b..4768db6055e9 100644 --- a/agent/app/service/cronjob_helper.go +++ b/agent/app/service/cronjob_helper.go @@ -79,9 +79,8 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) { case "log": err = u.handleSystemLog(*cronjob, record.StartTime) case "snapshot": - record.Records = u.generateLogsPath(*cronjob, record.StartTime) _ = cronjobRepo.UpdateRecords(record.ID, map[string]interface{}{"records": record.Records}) - err = u.handleSnapshot(*cronjob, record.StartTime, record.Records) + err = u.handleSnapshot(*cronjob, record.StartTime) } if err != nil { diff --git a/agent/app/service/snapshot.go b/agent/app/service/snapshot.go index b398efe1cf6a..ef97bac5ef1d 100644 --- a/agent/app/service/snapshot.go +++ b/agent/app/service/snapshot.go @@ -2,21 +2,21 @@ package service import ( "context" - "encoding/json" "fmt" "os" "path" "strings" "sync" - "time" "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/utils/cmd" - "github.com/1Panel-dev/1Panel/agent/utils/compose" - "github.com/1Panel-dev/1Panel/agent/utils/files" + "github.com/1Panel-dev/1Panel/agent/utils/docker" + fileUtils "github.com/1Panel-dev/1Panel/agent/utils/files" + "github.com/docker/docker/api/types/image" + "github.com/google/uuid" "github.com/jinzhu/copier" "github.com/pkg/errors" "github.com/shirou/gopsutil/v3/host" @@ -28,18 +28,17 @@ type SnapshotService struct { type ISnapshotService interface { SearchWithPage(req dto.SearchWithPage) (int64, interface{}, error) + LoadSnapshotData() (dto.SnapshotData, error) SnapshotCreate(req dto.SnapshotCreate) error + SnapshotReCreate(id uint) error SnapshotRecover(req dto.SnapshotRecover) error SnapshotRollback(req dto.SnapshotRecover) error SnapshotImport(req dto.SnapshotImport) error Delete(req dto.SnapshotBatchDelete) error - LoadSnapShotStatus(id uint) (*dto.SnapshotStatus, error) - UpdateDescription(req dto.UpdateDescription) error - readFromJson(path string) (SnapshotJson, error) - HandleSnapshot(isCronjob bool, logPath string, req dto.SnapshotCreate, timeNow string, secret string) (string, error) + HandleSnapshot(req dto.SnapshotCreate) error } func NewISnapshotService() ISnapshotService { @@ -70,8 +69,8 @@ func (u *SnapshotService) SnapshotImport(req dto.SnapshotImport) error { } for _, snap := range req.Names { shortName := strings.TrimPrefix(snap, "snapshot_") - nameItems := strings.Split(shortName, "_") - if !strings.HasPrefix(shortName, "1panel_v") || !strings.HasSuffix(shortName, ".tar.gz") || len(nameItems) < 3 { + nameItems := strings.Split(shortName, "-") + if !strings.HasPrefix(shortName, "1panel-v") || !strings.HasSuffix(shortName, ".tar.gz") || len(nameItems) < 3 { return fmt.Errorf("incorrect snapshot name format of %s", shortName) } if strings.HasSuffix(snap, ".tar.gz") { @@ -92,231 +91,47 @@ func (u *SnapshotService) SnapshotImport(req dto.SnapshotImport) error { return nil } -func (u *SnapshotService) UpdateDescription(req dto.UpdateDescription) error { - return snapshotRepo.Update(req.ID, map[string]interface{}{"description": req.Description}) -} - -func (u *SnapshotService) LoadSnapShotStatus(id uint) (*dto.SnapshotStatus, error) { - var data dto.SnapshotStatus - status, err := snapshotRepo.GetStatus(id) - if err != nil { - return nil, err - } - if err := copier.Copy(&data, &status); err != nil { - return nil, errors.WithMessage(constant.ErrStructTransform, err.Error()) - } - return &data, nil -} - -type SnapshotJson struct { - OldBaseDir string `json:"oldBaseDir"` - OldDockerDataDir string `json:"oldDockerDataDir"` - OldBackupDataDir string `json:"oldBackupDataDir"` - OldPanelDataDir string `json:"oldPanelDataDir"` - - BaseDir string `json:"baseDir"` - DockerDataDir string `json:"dockerDataDir"` - BackupDataDir string `json:"backupDataDir"` - PanelDataDir string `json:"panelDataDir"` - LiveRestoreEnabled bool `json:"liveRestoreEnabled"` -} - -func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate) error { - if _, err := u.HandleSnapshot(false, "", req, time.Now().Format(constant.DateTimeSlimLayout), req.Secret); err != nil { - return err - } - return nil -} - -func (u *SnapshotService) SnapshotRecover(req dto.SnapshotRecover) error { - global.LOG.Info("start to recover panel by snapshot now") - snap, err := snapshotRepo.Get(commonRepo.WithByID(req.ID)) +func (u *SnapshotService) LoadSnapshotData() (dto.SnapshotData, error) { + var ( + data dto.SnapshotData + err error + ) + fileOp := fileUtils.NewFileOp() + data.AppData, err = loadApps(fileOp) if err != nil { - return err - } - if hasOs(snap.Name) && !strings.Contains(snap.Name, loadOs()) { - return fmt.Errorf("restoring snapshots(%s) between different server architectures(%s) is not supported", snap.Name, loadOs()) - } - if !req.IsNew && len(snap.InterruptStep) != 0 && len(snap.RollbackStatus) != 0 { - return fmt.Errorf("the snapshot has been rolled back and cannot be restored again") - } - - baseDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("system/%s", snap.Name)) - if _, err := os.Stat(baseDir); err != nil && os.IsNotExist(err) { - _ = os.MkdirAll(baseDir, os.ModePerm) + return data, err } - - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"recover_status": constant.StatusWaiting}) - _ = settingRepo.Update("SystemStatus", "Recovering") - go u.HandleSnapshotRecover(snap, true, req) - return nil -} - -func (u *SnapshotService) SnapshotRollback(req dto.SnapshotRecover) error { - global.LOG.Info("start to rollback now") - snap, err := snapshotRepo.Get(commonRepo.WithByID(req.ID)) + data.PanelData, err = loadPanelFile(fileOp) if err != nil { - return err - } - req.IsNew = false - snap.InterruptStep = "Readjson" - go u.HandleSnapshotRecover(snap, false, req) - return nil -} - -func (u *SnapshotService) readFromJson(path string) (SnapshotJson, error) { - var snap SnapshotJson - if _, err := os.Stat(path); err != nil { - return snap, fmt.Errorf("find snapshot json file in recover package failed, err: %v", err) + return data, err } - fileByte, err := os.ReadFile(path) + itemBackups, err := loadFile(global.CONF.System.Backup, 8, fileOp) if err != nil { - return snap, fmt.Errorf("read file from path %s failed, err: %v", path, err) - } - if err := json.Unmarshal(fileByte, &snap); err != nil { - return snap, fmt.Errorf("unmarshal snapjson failed, err: %v", err) + return data, err } - return snap, nil -} - -func (u *SnapshotService) HandleSnapshot(isCronjob bool, logPath string, req dto.SnapshotCreate, timeNow string, secret string) (string, error) { - var ( - rootDir string - snap model.Snapshot - snapStatus model.SnapshotStatus - err error - ) - - if req.ID == 0 { - versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion")) - - name := fmt.Sprintf("1panel_%s_%s_%s", versionItem.Value, loadOs(), timeNow) - if isCronjob { - name = fmt.Sprintf("snapshot_1panel_%s_%s_%s", versionItem.Value, loadOs(), timeNow) - } - rootDir = path.Join(global.CONF.System.Backup, "system", name) - - snap = model.Snapshot{ - Name: name, - Description: req.Description, - SourceAccountIDs: req.SourceAccountIDs, - DownloadAccountID: req.DownloadAccountID, - Version: versionItem.Value, - Status: constant.StatusWaiting, + for i, item := range itemBackups { + if item.Label == "app" { + data.BackupData = append(itemBackups[:i], itemBackups[i+1:]...) } - _ = snapshotRepo.Create(&snap) - snapStatus.SnapID = snap.ID - _ = snapshotRepo.CreateStatus(&snapStatus) - } else { - snap, err = snapshotRepo.Get(commonRepo.WithByID(req.ID)) - if err != nil { - return "", err - } - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusWaiting}) - snapStatus, _ = snapshotRepo.GetStatus(snap.ID) - if snapStatus.ID == 0 { - snapStatus.SnapID = snap.ID - _ = snapshotRepo.CreateStatus(&snapStatus) + if item.Label == "system_snapshot" { + itemBackups[i].IsCheck = false + for j := 0; j < len(item.Children); j++ { + itemBackups[i].Children[j].IsCheck = false + } } - rootDir = path.Join(global.CONF.System.Backup, fmt.Sprintf("system/%s", snap.Name)) } - var wg sync.WaitGroup - itemHelper := snapHelper{SnapID: snap.ID, Status: &snapStatus, Wg: &wg, FileOp: files.NewFileOp(), Ctx: context.Background()} - backupPanelDir := path.Join(rootDir, "1panel") - _ = os.MkdirAll(backupPanelDir, os.ModePerm) - backupDockerDir := path.Join(rootDir, "docker") - _ = os.MkdirAll(backupDockerDir, os.ModePerm) + return data, nil +} - jsonItem := SnapshotJson{ - BaseDir: global.CONF.System.BaseDir, - BackupDataDir: global.CONF.System.Backup, - PanelDataDir: path.Join(global.CONF.System.BaseDir, "1panel"), - } - loadLogByStatus(snapStatus, logPath) - if snapStatus.PanelInfo != constant.StatusDone { - wg.Add(1) - go snapJson(itemHelper, jsonItem, rootDir) - } - if snapStatus.Panel != constant.StatusDone { - wg.Add(1) - go snapPanel(itemHelper, backupPanelDir) - } - if snapStatus.DaemonJson != constant.StatusDone { - wg.Add(1) - go snapDaemonJson(itemHelper, backupDockerDir) - } - if snapStatus.AppData != constant.StatusDone { - wg.Add(1) - go snapAppData(itemHelper, backupDockerDir) - } - if snapStatus.BackupData != constant.StatusDone { - wg.Add(1) - go snapBackup(itemHelper, backupPanelDir) - } +func (u *SnapshotService) UpdateDescription(req dto.UpdateDescription) error { + return snapshotRepo.Update(req.ID, map[string]interface{}{"description": req.Description}) +} - if !isCronjob { - go func() { - wg.Wait() - if !checkIsAllDone(snap.ID) { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - return - } - if snapStatus.PanelData != constant.StatusDone { - snapPanelData(itemHelper, backupPanelDir) - } - if snapStatus.PanelData != constant.StatusDone { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - return - } - if snapStatus.Compress != constant.StatusDone { - snapCompress(itemHelper, rootDir, secret) - } - if snapStatus.Compress != constant.StatusDone { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - return - } - if snapStatus.Upload != constant.StatusDone { - snapUpload(itemHelper, req.SourceAccountIDs, fmt.Sprintf("%s.tar.gz", rootDir)) - } - if snapStatus.Upload != constant.StatusDone { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - return - } - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess}) - }() - return "", nil - } - wg.Wait() - if !checkIsAllDone(snap.ID) { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - loadLogByStatus(snapStatus, logPath) - return snap.Name, fmt.Errorf("snapshot %s backup failed", snap.Name) - } - loadLogByStatus(snapStatus, logPath) - snapPanelData(itemHelper, backupPanelDir) - if snapStatus.PanelData != constant.StatusDone { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - loadLogByStatus(snapStatus, logPath) - return snap.Name, fmt.Errorf("snapshot %s 1panel data failed", snap.Name) - } - loadLogByStatus(snapStatus, logPath) - snapCompress(itemHelper, rootDir, secret) - if snapStatus.Compress != constant.StatusDone { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - loadLogByStatus(snapStatus, logPath) - return snap.Name, fmt.Errorf("snapshot %s compress failed", snap.Name) - } - loadLogByStatus(snapStatus, logPath) - snapUpload(itemHelper, req.SourceAccountIDs, fmt.Sprintf("%s.tar.gz", rootDir)) - if snapStatus.Upload != constant.StatusDone { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - loadLogByStatus(snapStatus, logPath) - return snap.Name, fmt.Errorf("snapshot %s upload failed", snap.Name) - } - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess}) - loadLogByStatus(snapStatus, logPath) - return snap.Name, nil +type SnapshotJson struct { + BaseDir string `json:"baseDir"` + BackupDataDir string `json:"backupDataDir"` + Size uint64 `json:"size"` } func (u *SnapshotService) Delete(req dto.SnapshotBatchDelete) error { @@ -333,7 +148,6 @@ func (u *SnapshotService) Delete(req dto.SnapshotBatchDelete) error { } } - _ = snapshotRepo.DeleteStatus(snap.ID) if err := snapshotRepo.Delete(commonRepo.WithByID(snap.ID)); err != nil { return err } @@ -341,152 +155,6 @@ func (u *SnapshotService) Delete(req dto.SnapshotBatchDelete) error { return nil } -func updateRecoverStatus(id uint, isRecover bool, interruptStep, status, message string) { - if isRecover { - if status != constant.StatusSuccess { - global.LOG.Errorf("recover failed, err: %s", message) - } - if err := snapshotRepo.Update(id, map[string]interface{}{ - "interrupt_step": interruptStep, - "recover_status": status, - "recover_message": message, - "last_recovered_at": time.Now().Format(constant.DateTimeLayout), - }); err != nil { - global.LOG.Errorf("update snap recover status failed, err: %v", err) - } - _ = settingRepo.Update("SystemStatus", "Free") - return - } - _ = settingRepo.Update("SystemStatus", "Free") - if status == constant.StatusSuccess { - if err := snapshotRepo.Update(id, map[string]interface{}{ - "recover_status": "", - "recover_message": "", - "interrupt_step": "", - "rollback_status": "", - "rollback_message": "", - "last_rollbacked_at": time.Now().Format(constant.DateTimeLayout), - }); err != nil { - global.LOG.Errorf("update snap recover status failed, err: %v", err) - } - return - } - global.LOG.Errorf("rollback failed, err: %s", message) - if err := snapshotRepo.Update(id, map[string]interface{}{ - "rollback_status": status, - "rollback_message": message, - "last_rollbacked_at": time.Now().Format(constant.DateTimeLayout), - }); err != nil { - global.LOG.Errorf("update snap recover status failed, err: %v", err) - } -} - -func (u *SnapshotService) handleUnTar(sourceDir, targetDir string, secret string) error { - if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) { - if err = os.MkdirAll(targetDir, os.ModePerm); err != nil { - return err - } - } - commands := "" - if len(secret) != 0 { - extraCmd := "openssl enc -d -aes-256-cbc -k '" + secret + "' -in " + sourceDir + " | " - commands = fmt.Sprintf("%s tar -zxvf - -C %s", extraCmd, targetDir+" > /dev/null 2>&1") - global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) - } else { - commands = fmt.Sprintf("tar zxvfC %s %s", sourceDir, targetDir) - global.LOG.Debug(commands) - } - stdout, err := cmd.ExecWithTimeOut(commands, 30*time.Minute) - if err != nil { - if len(stdout) != 0 { - global.LOG.Errorf("do handle untar failed, stdout: %s, err: %v", stdout, err) - return fmt.Errorf("do handle untar failed, stdout: %s, err: %v", stdout, err) - } - } - return nil -} - -func rebuildAllAppInstall() error { - global.LOG.Debug("start to rebuild all app") - appInstalls, err := appInstallRepo.ListBy() - if err != nil { - global.LOG.Errorf("get all app installed for rebuild failed, err: %v", err) - return err - } - var wg sync.WaitGroup - for i := 0; i < len(appInstalls); i++ { - wg.Add(1) - appInstalls[i].Status = constant.Rebuilding - _ = appInstallRepo.Save(context.Background(), &appInstalls[i]) - go func(app model.AppInstall) { - defer wg.Done() - dockerComposePath := app.GetComposePath() - out, err := compose.Down(dockerComposePath) - if err != nil { - _ = handleErr(app, err, out) - return - } - out, err = compose.Up(dockerComposePath) - if err != nil { - _ = handleErr(app, err, out) - return - } - app.Status = constant.Running - _ = appInstallRepo.Save(context.Background(), &app) - }(appInstalls[i]) - } - wg.Wait() - return nil -} - -func checkIsAllDone(snapID uint) bool { - status, err := snapshotRepo.GetStatus(snapID) - if err != nil { - return false - } - isOK, _ := checkAllDone(status) - return isOK -} - -func checkAllDone(status model.SnapshotStatus) (bool, string) { - if status.Panel != constant.StatusDone { - return false, status.Panel - } - if status.PanelInfo != constant.StatusDone { - return false, status.PanelInfo - } - if status.DaemonJson != constant.StatusDone { - return false, status.DaemonJson - } - if status.AppData != constant.StatusDone { - return false, status.AppData - } - if status.BackupData != constant.StatusDone { - return false, status.BackupData - } - return true, "" -} - -func loadLogByStatus(status model.SnapshotStatus, logPath string) { - logs := "" - logs += fmt.Sprintf("Write 1Panel basic information: %s \n", status.PanelInfo) - logs += fmt.Sprintf("Backup 1Panel system files: %s \n", status.Panel) - logs += fmt.Sprintf("Backup Docker configuration file: %s \n", status.DaemonJson) - logs += fmt.Sprintf("Backup installed apps from 1Panel: %s \n", status.AppData) - logs += fmt.Sprintf("Backup 1Panel data directory: %s \n", status.PanelData) - logs += fmt.Sprintf("Backup local backup directory for 1Panel: %s \n", status.BackupData) - logs += fmt.Sprintf("Create snapshot file: %s \n", status.Compress) - logs += fmt.Sprintf("Snapshot size: %s \n", status.Size) - logs += fmt.Sprintf("Upload snapshot file: %s \n", status.Upload) - - file, err := os.OpenFile(logPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) - if err != nil { - return - } - defer file.Close() - _, _ = file.Write([]byte(logs)) -} - func hasOs(name string) bool { return strings.Contains(name, "amd64") || strings.Contains(name, "arm64") || @@ -544,3 +212,211 @@ func loadSnapSize(records []model.Snapshot) ([]dto.SnapshotInfo, error) { wg.Wait() return datas, nil } + +func loadApps(fileOp fileUtils.FileOp) ([]dto.DataTree, error) { + var data []dto.DataTree + apps, err := appInstallRepo.ListBy() + if err != nil { + return data, err + } + openrestyID := 0 + for _, app := range apps { + if app.App.Key == constant.AppOpenresty { + openrestyID = int(app.ID) + } + } + websites, err := websiteRepo.List() + if err != nil { + return data, err + } + appRelationMap := make(map[uint]uint) + for _, website := range websites { + if website.Type == constant.Deployment && website.AppInstallID != 0 { + appRelationMap[uint(openrestyID)] = website.AppInstallID + } + } + appRelations, err := appInstallResourceRepo.GetBy() + if err != nil { + return data, err + } + for _, relation := range appRelations { + appRelationMap[uint(relation.AppInstallId)] = relation.LinkId + } + appMap := make(map[uint]string) + for _, app := range apps { + appMap[app.ID] = fmt.Sprintf("%s-%s", app.App.Key, app.Name) + } + + appTreeMap := make(map[string]dto.DataTree) + for _, app := range apps { + itemApp := dto.DataTree{ + ID: uuid.NewString(), + Label: fmt.Sprintf("%s - %s", app.App.Name, app.Name), + Key: app.App.Key, + Name: app.Name, + } + appPath := path.Join(global.CONF.System.BaseDir, "1panel/apps", app.App.Key, app.Name) + itemAppData := dto.DataTree{ID: uuid.NewString(), Label: "appData", Key: app.App.Key, Name: app.Name, IsCheck: true, Path: appPath} + if app.App.Key == constant.AppOpenresty && len(websites) != 0 { + itemAppData.IsDisable = true + } + if val, ok := appRelationMap[app.ID]; ok { + itemAppData.RelationItemID = appMap[val] + } + sizeItem, err := fileOp.GetDirSize(appPath) + if err == nil { + itemAppData.Size = uint64(sizeItem) + } + itemApp.Size += itemAppData.Size + data = append(data, itemApp) + appTreeMap[fmt.Sprintf("%s-%s", itemApp.Key, itemApp.Name)] = itemAppData + } + + for key, val := range appTreeMap { + if valRelation, ok := appTreeMap[val.RelationItemID]; ok { + valRelation.IsDisable = true + appTreeMap[val.RelationItemID] = valRelation + + val.RelationItemID = valRelation.ID + appTreeMap[key] = val + } + } + for i := 0; i < len(data); i++ { + if val, ok := appTreeMap[fmt.Sprintf("%s-%s", data[i].Key, data[i].Name)]; ok { + data[i].Children = append(data[i].Children, val) + } + } + data = loadAppBackup(data, fileOp) + data = loadAppImage(data) + return data, nil +} +func loadAppBackup(list []dto.DataTree, fileOp fileUtils.FileOp) []dto.DataTree { + for i := 0; i < len(list); i++ { + appBackupPath := path.Join(global.CONF.System.BaseDir, "1panel/backup/app", list[i].Key, list[i].Name) + itemAppBackupTree, err := loadFile(appBackupPath, 8, fileOp) + itemAppBackup := dto.DataTree{ID: uuid.NewString(), Label: "appBackup", IsCheck: true, Children: itemAppBackupTree, Path: appBackupPath} + if err == nil { + backupSizeItem, err := fileOp.GetDirSize(appBackupPath) + if err == nil { + itemAppBackup.Size = uint64(backupSizeItem) + list[i].Size += itemAppBackup.Size + } + list[i].Children = append(list[i].Children, itemAppBackup) + } + } + return list +} +func loadAppImage(list []dto.DataTree) []dto.DataTree { + client, err := docker.NewDockerClient() + if err != nil { + global.LOG.Errorf("new docker client failed, err: %v", err) + return list + } + defer client.Close() + imageList, err := client.ImageList(context.Background(), image.ListOptions{}) + if err != nil { + global.LOG.Errorf("load image list failed, err: %v", err) + return list + } + + for i := 0; i < len(list); i++ { + itemAppImage := dto.DataTree{ID: uuid.NewString(), Label: "appImage"} + stdout, err := cmd.Execf("cat %s | grep image: ", path.Join(global.CONF.System.BaseDir, "1panel/apps", list[i].Key, list[i].Name, "docker-compose.yml")) + if err != nil { + list[i].Children = append(list[i].Children, itemAppImage) + continue + } + itemAppImage.Name = strings.ReplaceAll(strings.ReplaceAll(strings.TrimSpace(stdout), "\n", ""), "image: ", "") + for _, imageItem := range imageList { + for _, tag := range imageItem.RepoTags { + if tag == itemAppImage.Name { + itemAppImage.Size = uint64(imageItem.Size) + break + } + } + } + list[i].Children = append(list[i].Children, itemAppImage) + } + return list +} + +func loadPanelFile(fileOp fileUtils.FileOp) ([]dto.DataTree, error) { + var data []dto.DataTree + snapFiles, err := os.ReadDir(path.Join(global.CONF.System.BaseDir, "1panel")) + if err != nil { + return data, err + } + for _, fileItem := range snapFiles { + itemData := dto.DataTree{ + ID: uuid.NewString(), + Label: fileItem.Name(), + IsCheck: true, + Path: path.Join(global.CONF.System.BaseDir, "1panel", fileItem.Name()), + } + switch itemData.Label { + case "agent", "conf", "runtime", "docker", "secret", "task": + itemData.IsDisable = true + case "clamav": + panelPath := path.Join(global.CONF.System.BaseDir, "1panel", itemData.Label) + itemData.Children, _ = loadFile(panelPath, 5, fileOp) + default: + continue + } + if fileItem.IsDir() { + sizeItem, err := fileOp.GetDirSize(path.Join(global.CONF.System.BaseDir, "1panel", itemData.Label)) + if err != nil { + continue + } + itemData.Size = uint64(sizeItem) + } else { + fileInfo, err := fileItem.Info() + if err != nil { + continue + } + itemData.Size = uint64(fileInfo.Size()) + } + if itemData.IsCheck && itemData.Size == 0 { + itemData.IsCheck = false + itemData.IsDisable = true + } + + data = append(data, itemData) + } + + return data, nil +} + +func loadFile(pathItem string, index int, fileOp fileUtils.FileOp) ([]dto.DataTree, error) { + var data []dto.DataTree + snapFiles, err := os.ReadDir(pathItem) + if err != nil { + return data, err + } + i := 0 + for _, fileItem := range snapFiles { + itemData := dto.DataTree{ + ID: uuid.NewString(), + Label: fileItem.Name(), + Name: fileItem.Name(), + Path: path.Join(pathItem, fileItem.Name()), + IsCheck: true, + } + if fileItem.IsDir() { + sizeItem, err := fileOp.GetDirSize(path.Join(pathItem, itemData.Label)) + if err != nil { + continue + } + itemData.Size = uint64(sizeItem) + itemData.Children, _ = loadFile(path.Join(pathItem, itemData.Label), index-1, fileOp) + } else { + fileInfo, err := fileItem.Info() + if err != nil { + continue + } + itemData.Size = uint64(fileInfo.Size()) + } + data = append(data, itemData) + i++ + } + return data, nil +} diff --git a/agent/app/service/snapshot_create.go b/agent/app/service/snapshot_create.go index e4a42b6c3d9a..793674678584 100644 --- a/agent/app/service/snapshot_create.go +++ b/agent/app/service/snapshot_create.go @@ -6,145 +6,418 @@ import ( "fmt" "os" "path" - "regexp" "strings" "sync" "time" + "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/app/task" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/i18n" "github.com/1Panel-dev/1Panel/agent/utils/cmd" "github.com/1Panel-dev/1Panel/agent/utils/common" + "github.com/1Panel-dev/1Panel/agent/utils/copier" "github.com/1Panel-dev/1Panel/agent/utils/files" + "github.com/glebarez/sqlite" + "github.com/pkg/errors" + "gorm.io/gorm" ) -type snapHelper struct { - SnapID uint - Status *model.SnapshotStatus - Ctx context.Context - FileOp files.FileOp - Wg *sync.WaitGroup -} +func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate) error { + versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion")) + + req.Name = fmt.Sprintf("1panel-%s-linux-%s-%s", versionItem.Value, loadOs(), time.Now().Format(constant.DateTimeSlimLayout)) + appItem, _ := json.Marshal(req.AppData) + panelItem, _ := json.Marshal(req.PanelData) + backupItem, _ := json.Marshal(req.BackupData) + snap := model.Snapshot{ + Name: req.Name, + TaskID: req.TaskID, + Secret: req.Secret, + Description: req.Description, + SourceAccountIDs: req.SourceAccountIDs, + DownloadAccountID: req.DownloadAccountID, -func snapJson(snap snapHelper, snapJson SnapshotJson, targetDir string) { - defer snap.Wg.Done() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_info": constant.Running}) - status := constant.StatusDone - remarkInfo, _ := json.MarshalIndent(snapJson, "", "\t") - if err := os.WriteFile(fmt.Sprintf("%s/snapshot.json", targetDir), remarkInfo, 0640); err != nil { - status = err.Error() + AppData: string(appItem), + PanelData: string(panelItem), + BackupData: string(backupItem), + WithMonitorData: req.WithMonitorData, + WithLoginLog: req.WithLoginLog, + WithOperationLog: req.WithOperationLog, + WithTaskLog: req.WithTaskLog, + WithSystemLog: req.WithSystemLog, + + Version: versionItem.Value, + Status: constant.StatusWaiting, + } + if err := snapshotRepo.Create(&snap); err != nil { + global.LOG.Errorf("create snapshot record to db failed, err: %v", err) + return err } - snap.Status.PanelInfo = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_info": status}) + + req.ID = snap.ID + if err := u.HandleSnapshot(req); err != nil { + return err + } + return nil } -func snapPanel(snap snapHelper, targetDir string) { - defer snap.Wg.Done() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel": constant.Running}) - status := constant.StatusDone - if err := common.CopyFile("/usr/local/bin/1panel", path.Join(targetDir, "1panel")); err != nil { - status = err.Error() +func (u *SnapshotService) SnapshotReCreate(id uint) error { + snap, err := snapshotRepo.Get(commonRepo.WithByID(id)) + if err != nil { + return err + } + taskModel, err := taskRepo.GetFirst(taskRepo.WithResourceID(snap.ID), commonRepo.WithByType(task.TaskScopeSnapshot)) + if err != nil { + return err } - if err := common.CopyFile("/usr/local/bin/1pctl", targetDir); err != nil { - status = err.Error() + var req dto.SnapshotCreate + _ = copier.Copy(&req, snap) + if err := json.Unmarshal([]byte(snap.PanelData), &req.PanelData); err != nil { + return err + } + if err := json.Unmarshal([]byte(snap.AppData), &req.AppData); err != nil { + return err + } + if err := json.Unmarshal([]byte(snap.BackupData), &req.BackupData); err != nil { + return err + } + req.TaskID = taskModel.ID + if err := u.HandleSnapshot(req); err != nil { + return err } - if err := common.CopyFile("/etc/systemd/system/1panel.service", targetDir); err != nil { - status = err.Error() + return nil +} + +func (u *SnapshotService) HandleSnapshot(req dto.SnapshotCreate) error { + taskItem, err := task.NewTaskWithOps(req.Name, task.TaskCreate, task.TaskScopeSnapshot, req.TaskID, req.ID) + if err != nil { + global.LOG.Errorf("new task for create snapshot failed, err: %v", err) + return err } - snap.Status.Panel = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel": status}) + + rootDir := path.Join(global.CONF.System.BaseDir, "1panel/tmp/system", req.Name) + itemHelper := snapHelper{SnapID: req.ID, Task: *taskItem, FileOp: files.NewFileOp(), Ctx: context.Background()} + baseDir := path.Join(rootDir, "base") + _ = os.MkdirAll(baseDir, os.ModePerm) + + go func() { + taskItem.AddSubTaskWithAlias( + "SnapDBInfo", + func(t *task.Task) error { return loadDbConn(&itemHelper, rootDir, req) }, + nil, + ) + + if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapBaseInfo" { + taskItem.AddSubTaskWithAlias( + "SnapBaseInfo", + func(t *task.Task) error { return snapBaseData(itemHelper, baseDir) }, + nil, + ) + req.InterruptStep = "" + } + if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapInstallApp" { + taskItem.AddSubTaskWithAlias( + "SnapInstallApp", + func(t *task.Task) error { return snapAppImage(itemHelper, req, rootDir) }, + nil, + ) + req.InterruptStep = "" + } + if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapLocalBackup" { + taskItem.AddSubTaskWithAlias( + "SnapLocalBackup", + func(t *task.Task) error { return snapBackupData(itemHelper, req, rootDir) }, + nil, + ) + req.InterruptStep = "" + } + if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapPanelData" { + taskItem.AddSubTaskWithAlias( + "SnapPanelData", + func(t *task.Task) error { return snapPanelData(itemHelper, req, rootDir) }, + nil, + ) + req.InterruptStep = "" + } + + taskItem.AddSubTask( + "SnapCloseDBConn", + func(t *task.Task) error { + taskItem.Log("######################## 6 / 8 ########################") + closeDatabase(itemHelper.snapAgentDB) + closeDatabase(itemHelper.snapCoreDB) + return nil + }, + nil, + ) + if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapCompress" { + taskItem.AddSubTaskWithAlias( + "SnapCompress", + func(t *task.Task) error { return snapCompress(itemHelper, rootDir, req.Secret) }, + nil, + ) + req.InterruptStep = "" + } + if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapUpload" { + taskItem.AddSubTaskWithAlias( + "SnapUpload", + func(t *task.Task) error { + return snapUpload(itemHelper, req.SourceAccountIDs, fmt.Sprintf("%s.tar.gz", rootDir)) + }, + nil, + ) + req.InterruptStep = "" + } + if err := taskItem.Execute(); err != nil { + _ = snapshotRepo.Update(req.ID, map[string]interface{}{"status": constant.StatusFailed, "message": err.Error(), "interrupt_step": taskItem.Task.CurrentStep}) + return + } + _ = snapshotRepo.Update(req.ID, map[string]interface{}{"status": constant.StatusSuccess, "interrupt_step": ""}) + _ = os.RemoveAll(rootDir) + }() + return nil } -func snapDaemonJson(snap snapHelper, targetDir string) { - defer snap.Wg.Done() - status := constant.StatusDone - if !snap.FileOp.Stat("/etc/docker/daemon.json") { - snap.Status.DaemonJson = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"daemon_json": status}) - return +type snapHelper struct { + SnapID uint + snapAgentDB *gorm.DB + snapCoreDB *gorm.DB + Ctx context.Context + FileOp files.FileOp + Wg *sync.WaitGroup + Task task.Task +} + +func loadDbConn(snap *snapHelper, targetDir string, req dto.SnapshotCreate) error { + snap.Task.Log("######################## 1 / 8 ########################") + snap.Task.LogStart(i18n.GetMsgByKey("SnapDBInfo")) + pathDB := path.Join(global.CONF.System.BaseDir, "1panel/db") + + err := snap.FileOp.CopyDir(pathDB, targetDir) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", pathDB), err) + if err != nil { + return err + } + + agentDb, err := newSnapDB(path.Join(targetDir, "db"), "agent.db") + snap.Task.LogWithStatus(i18n.GetWithName("SnapNewDB", "agent"), err) + if err != nil { + return err + } + snap.snapAgentDB = agentDb + coreDb, err := newSnapDB(path.Join(targetDir, "db"), "core.db") + snap.Task.LogWithStatus(i18n.GetWithName("SnapNewDB", "core"), err) + if err != nil { + return err + } + snap.snapCoreDB = coreDb + + if !req.WithMonitorData { + err = os.Remove(path.Join(targetDir, "db/monitor.db")) + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapDeleteMonitor"), err) + if err != nil { + return err + } } - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"daemon_json": constant.Running}) - if err := common.CopyFile("/etc/docker/daemon.json", targetDir); err != nil { - status = err.Error() + if !req.WithOperationLog { + err = snap.snapCoreDB.Exec("DELETE FROM operation_logs").Error + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapDeleteOperationLog"), err) + if err != nil { + return err + } + } + if !req.WithLoginLog { + err = snap.snapCoreDB.Exec("DELETE FROM login_logs").Error + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapDeleteLoginLog"), err) + if err != nil { + return err + } } - snap.Status.DaemonJson = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"daemon_json": status}) + + _ = snap.snapAgentDB.Model(&model.Setting{}).Where("key = ?", "SystemIP").Updates(map[string]interface{}{"value": ""}).Error + _ = snap.snapAgentDB.Where("id = ?", snap.SnapID).Delete(&model.Snapshot{}).Error + + return nil } -func snapAppData(snap snapHelper, targetDir string) { - defer snap.Wg.Done() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": constant.Running}) - appInstalls, err := appInstallRepo.ListBy() +func snapBaseData(snap snapHelper, targetDir string) error { + snap.Task.Log("######################## 2 / 8 ########################") + snap.Task.LogStart(i18n.GetMsgByKey("SnapBaseInfo")) + + err := common.CopyFile("/usr/local/bin/1panel", targetDir) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel"), err) if err != nil { - snap.Status.AppData = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": err.Error()}) - return + return err } - runtimes, err := runtimeRepo.List() + + err = common.CopyFile("/usr/local/bin/1panel_agent", targetDir) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel_agent"), err) if err != nil { - snap.Status.AppData = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": err.Error()}) - return + return err } - imageRegex := regexp.MustCompile(`image:\s*(.*)`) - var imageSaveList []string + + err = common.CopyFile("/usr/local/bin/1pctl", targetDir) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1pctl"), err) + if err != nil { + return err + } + + err = common.CopyFile("/etc/systemd/system/1panel.service", targetDir) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel.service"), err) + if err != nil { + return err + } + + err = common.CopyFile("/etc/systemd/system/1panel_agent.service", targetDir) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel_agent.service"), err) + if err != nil { + return err + } + + if snap.FileOp.Stat("/etc/docker/daemon.json") { + err = common.CopyFile("/etc/docker/daemon.json", targetDir) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/docker/daemon.json"), err) + if err != nil { + return err + } + } + + remarkInfo, _ := json.MarshalIndent(SnapshotJson{ + BaseDir: global.CONF.System.BaseDir, + BackupDataDir: global.CONF.System.Backup, + }, "", "\t") + err = os.WriteFile(path.Join(targetDir, "snapshot.json"), remarkInfo, 0640) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", path.Join(targetDir, "snapshot.json")), err) + if err != nil { + return err + } + + return nil +} + +func snapAppImage(snap snapHelper, req dto.SnapshotCreate, targetDir string) error { + snap.Task.Log("######################## 3 / 8 ########################") + snap.Task.LogStart(i18n.GetMsgByKey("SnapInstallApp")) + + var imageList []string existStr, _ := cmd.Exec("docker images | awk '{print $1\":\"$2}' | grep -v REPOSITORY:TAG") existImages := strings.Split(existStr, "\n") - duplicateMap := make(map[string]bool) - for _, app := range appInstalls { - matches := imageRegex.FindAllStringSubmatch(app.DockerCompose, -1) - for _, match := range matches { - for _, existImage := range existImages { - if match[1] == existImage && !duplicateMap[match[1]] { - imageSaveList = append(imageSaveList, match[1]) - duplicateMap[match[1]] = true + for _, app := range req.AppData { + for _, item := range app.Children { + if item.Label == "appImage" && item.IsCheck { + for _, existImage := range existImages { + if existImage == item.Name { + imageList = append(imageList, item.Name) + } } } } } - for _, runtime := range runtimes { - for _, existImage := range existImages { - if runtime.Image == existImage && !duplicateMap[runtime.Image] { - imageSaveList = append(imageSaveList, runtime.Image) - duplicateMap[runtime.Image] = true + + if len(imageList) != 0 { + snap.Task.Logf("docker save %s | gzip -c > %s", strings.Join(imageList, " "), path.Join(targetDir, "images.tar.gz")) + std, err := cmd.Execf("docker save %s | gzip -c > %s", strings.Join(imageList, " "), path.Join(targetDir, "images.tar.gz")) + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapDockerSave"), errors.New(std)) + if err != nil { + snap.Task.LogFailedWithErr(i18n.GetMsgByKey("SnapDockerSave"), errors.New(std)) + return errors.New(std) + } + snap.Task.LogSuccess(i18n.GetMsgByKey("SnapDockerSave")) + } + return nil +} + +func snapBackupData(snap snapHelper, req dto.SnapshotCreate, targetDir string) error { + snap.Task.Log("######################## 4 / 8 ########################") + snap.Task.LogStart(i18n.GetMsgByKey("SnapLocalBackup")) + + excludes := loadBackupExcludes(snap, req.BackupData) + for _, item := range req.AppData { + for _, itemApp := range item.Children { + if itemApp.Label == "appBackup" { + excludes = append(excludes, loadAppBackupExcludes([]dto.DataTree{itemApp})...) } } } + err := snap.FileOp.TarGzCompressPro(false, global.CONF.System.Backup, path.Join(targetDir, "1panel_backup.tar.gz"), "", strings.Join(excludes, ";")) + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapCompressBackup"), err) - if len(imageSaveList) != 0 { - global.LOG.Debugf("docker save %s | gzip -c > %s", strings.Join(imageSaveList, " "), path.Join(targetDir, "docker_image.tar")) - std, err := cmd.Execf("docker save %s | gzip -c > %s", strings.Join(imageSaveList, " "), path.Join(targetDir, "docker_image.tar")) - if err != nil { - snap.Status.AppData = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": std}) - return + return err +} +func loadBackupExcludes(snap snapHelper, req []dto.DataTree) []string { + var excludes []string + for _, item := range req { + if len(item.Children) == 0 { + if item.IsCheck { + continue + } + if strings.HasPrefix(item.Path, path.Join(global.CONF.System.Backup, "system_snapshot")) { + fmt.Println(strings.TrimSuffix(item.Name, ".tar.gz")) + if err := snap.snapAgentDB.Debug().Where("name = ? AND download_account_id = ?", strings.TrimSuffix(item.Name, ".tar.gz"), "1").Delete(&model.Snapshot{}).Error; err != nil { + snap.Task.LogWithStatus("delete snapshot from database", err) + } + } else { + fmt.Println(strings.TrimPrefix(path.Dir(item.Path), global.CONF.System.Backup+"/"), path.Base(item.Path)) + if err := snap.snapAgentDB.Debug().Where("file_dir = ? AND file_name = ?", strings.TrimPrefix(path.Dir(item.Path), global.CONF.System.Backup+"/"), path.Base(item.Path)).Delete(&model.BackupRecord{}).Error; err != nil { + snap.Task.LogWithStatus("delete backup file from database", err) + } + } + excludes = append(excludes, "."+strings.TrimPrefix(item.Path, global.CONF.System.Backup)) + } else { + excludes = append(excludes, loadBackupExcludes(snap, item.Children)...) } } - snap.Status.AppData = constant.StatusDone - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": constant.StatusDone}) + return excludes } - -func snapBackup(snap snapHelper, targetDir string) { - defer snap.Wg.Done() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"backup_data": constant.Running}) - status := constant.StatusDone - if err := handleSnapTar(global.CONF.System.Backup, targetDir, "1panel_backup.tar.gz", "./system;./system_snapshot;", ""); err != nil { - status = err.Error() +func loadAppBackupExcludes(req []dto.DataTree) []string { + var excludes []string + for _, item := range req { + if len(item.Children) == 0 { + if !item.IsCheck { + excludes = append(excludes, "."+strings.TrimPrefix(item.Path, path.Join(global.CONF.System.Backup))) + } + } else { + excludes = append(excludes, loadAppBackupExcludes(item.Children)...) + } } - snap.Status.BackupData = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"backup_data": status}) + return excludes } -func snapPanelData(snap snapHelper, targetDir string) { - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_data": constant.Running}) - status := constant.StatusDone - dataDir := path.Join(global.CONF.System.BaseDir, "1panel") - exclusionRules := "./tmp;./log;./cache;./db/1Panel.db-*;" - if strings.Contains(global.CONF.System.Backup, dataDir) { - exclusionRules += ("." + strings.ReplaceAll(global.CONF.System.Backup, dataDir, "") + ";") +func snapPanelData(snap snapHelper, req dto.SnapshotCreate, targetDir string) error { + snap.Task.Log("######################## 5 / 8 ########################") + snap.Task.LogStart(i18n.GetMsgByKey("SnapPanelData")) + + excludes := loadPanelExcludes(req.PanelData) + for _, item := range req.AppData { + for _, itemApp := range item.Children { + if itemApp.Label == "appData" { + excludes = append(excludes, loadPanelExcludes([]dto.DataTree{itemApp})...) + } + } + } + excludes = append(excludes, "./tmp") + excludes = append(excludes, "./cache") + excludes = append(excludes, "./uploads") + excludes = append(excludes, "./db") + excludes = append(excludes, "./resource") + if !req.WithSystemLog { + excludes = append(excludes, "./log/1Panel*") + } + if !req.WithTaskLog { + excludes = append(excludes, "./log/App") + excludes = append(excludes, "./log/Snapshot") + excludes = append(excludes, "./log/AppStore") + excludes = append(excludes, "./log/Website") + } + + rootDir := path.Join(global.CONF.System.BaseDir, "1panel") + if strings.Contains(global.CONF.System.Backup, rootDir) { + excludes = append(excludes, "."+strings.ReplaceAll(global.CONF.System.Backup, rootDir, "")) } ignoreVal, _ := settingRepo.Get(settingRepo.WithByKey("SnapshotIgnore")) rules := strings.Split(ignoreVal.Value, ",") @@ -152,129 +425,93 @@ func snapPanelData(snap snapHelper, targetDir string) { if len(ignore) == 0 || cmd.CheckIllegal(ignore) { continue } - exclusionRules += ("." + strings.ReplaceAll(ignore, dataDir, "") + ";") - } - _ = snapshotRepo.Update(snap.SnapID, map[string]interface{}{"status": "OnSaveData"}) - sysIP, _ := settingRepo.Get(settingRepo.WithByKey("SystemIP")) - _ = settingRepo.Update("SystemIP", "") - checkPointOfWal() - if err := handleSnapTar(dataDir, targetDir, "1panel_data.tar.gz", exclusionRules, ""); err != nil { - status = err.Error() + excludes = append(excludes, "."+strings.ReplaceAll(ignore, rootDir, "")) } - _ = snapshotRepo.Update(snap.SnapID, map[string]interface{}{"status": constant.StatusWaiting}) + err := snap.FileOp.TarGzCompressPro(false, rootDir, path.Join(targetDir, "1panel_data.tar.gz"), "", strings.Join(excludes, ";")) + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapCompressPanel"), err) - snap.Status.PanelData = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_data": status}) - _ = settingRepo.Update("SystemIP", sysIP.Value) + return err +} +func loadPanelExcludes(req []dto.DataTree) []string { + var excludes []string + for _, item := range req { + if len(item.Children) == 0 { + if !item.IsCheck { + excludes = append(excludes, "."+strings.TrimPrefix(item.Path, path.Join(global.CONF.System.BaseDir, "1panel"))) + } + } else { + excludes = append(excludes, loadPanelExcludes(item.Children)...) + } + } + return excludes } -func snapCompress(snap snapHelper, rootDir string, secret string) { - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": constant.StatusRunning}) +func snapCompress(snap snapHelper, rootDir string, secret string) error { + snap.Task.Log("######################## 7 / 8 ########################") + snap.Task.LogStart(i18n.GetMsgByKey("SnapCompress")) + tmpDir := path.Join(global.CONF.System.TmpDir, "system") fileName := fmt.Sprintf("%s.tar.gz", path.Base(rootDir)) - if err := handleSnapTar(rootDir, tmpDir, fileName, "", secret); err != nil { - snap.Status.Compress = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": err.Error()}) - return + err := snap.FileOp.TarGzCompressPro(true, rootDir, path.Join(tmpDir, fileName), secret, "") + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapCompressFile"), err) + if err != nil { + return err } stat, err := os.Stat(path.Join(tmpDir, fileName)) + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapCheckCompress"), err) if err != nil { - snap.Status.Compress = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": err.Error()}) - return + return err } - size := common.LoadSizeUnit2F(float64(stat.Size())) - global.LOG.Debugf("compress successful! size of file: %s", size) - snap.Status.Compress = constant.StatusDone - snap.Status.Size = size - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": constant.StatusDone, "size": size}) - global.LOG.Debugf("remove snapshot file %s", rootDir) + size := common.LoadSizeUnit2F(float64(stat.Size())) + snap.Task.Logf(i18n.GetWithName("SnapCompressSize", size)) _ = os.RemoveAll(rootDir) + return nil } -func snapUpload(snap snapHelper, accounts string, file string) { +func snapUpload(snap snapHelper, accounts string, file string) error { + snap.Task.Log("######################## 8 / 8 ########################") + snap.Task.LogStart(i18n.GetMsgByKey("SnapUpload")) + source := path.Join(global.CONF.System.TmpDir, "system", path.Base(file)) - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": constant.StatusUploading}) accountMap, err := NewBackupClientMap(strings.Split(accounts, ",")) + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapLoadBackup"), err) if err != nil { - snap.Status.Upload = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": err.Error()}) - return + return err } + targetAccounts := strings.Split(accounts, ",") for _, item := range targetAccounts { - global.LOG.Debugf("start upload snapshot to %s, path: %s", item, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))) - if _, err := accountMap[item].client.Upload(source, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))); err != nil { - global.LOG.Debugf("upload to %s failed, err: %v", item, err) - snap.Status.Upload = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": err.Error()}) - return + snap.Task.LogStart(i18n.GetWithName("SnapUploadTo", fmt.Sprintf("[%s] %s", accountMap[item].name, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))))) + _, err := accountMap[item].client.Upload(source, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))) + snap.Task.LogWithStatus(i18n.GetWithName("SnapUploadRes", accountMap[item].name), err) + if err != nil { + return err } - global.LOG.Debugf("upload to %s successful", item) } - snap.Status.Upload = constant.StatusDone - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": constant.StatusDone}) - - global.LOG.Debugf("remove snapshot file %s", source) _ = os.Remove(source) + return nil } -func handleSnapTar(sourceDir, targetDir, name, exclusionRules string, secret string) error { - if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) { - if err = os.MkdirAll(targetDir, os.ModePerm); err != nil { - return err - } - } - - exMap := make(map[string]struct{}) - exStr := "" - excludes := strings.Split(exclusionRules, ";") - excludes = append(excludes, "*.sock") - for _, exclude := range excludes { - if len(exclude) == 0 { - continue - } - if _, ok := exMap[exclude]; ok { - continue - } - exStr += " --exclude " - exStr += exclude - exMap[exclude] = struct{}{} - } - path := "" - if strings.Contains(sourceDir, "/") { - itemDir := strings.ReplaceAll(sourceDir[strings.LastIndex(sourceDir, "/"):], "/", "") - aheadDir := sourceDir[:strings.LastIndex(sourceDir, "/")] - if len(aheadDir) == 0 { - aheadDir = "/" - } - path += fmt.Sprintf("-C %s %s", aheadDir, itemDir) - } else { - path = sourceDir - } - commands := "" - if len(secret) != 0 { - extraCmd := "| openssl enc -aes-256-cbc -salt -k '" + secret + "' -out" - commands = fmt.Sprintf("tar --warning=no-file-changed --ignore-failed-read -zcf %s %s %s %s", " -"+exStr, path, extraCmd, targetDir+"/"+name) - global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) - } else { - commands = fmt.Sprintf("tar --warning=no-file-changed --ignore-failed-read -zcf %s %s -C %s .", targetDir+"/"+name, exStr, sourceDir) - global.LOG.Debug(commands) - } - stdout, err := cmd.ExecWithTimeOut(commands, 30*time.Minute) +func newSnapDB(dir, file string) (*gorm.DB, error) { + db, _ := gorm.Open(sqlite.Open(path.Join(dir, file)), &gorm.Config{ + DisableForeignKeyConstraintWhenMigrating: true, + }) + sqlDB, err := db.DB() if err != nil { - if len(stdout) != 0 { - global.LOG.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err) - return fmt.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err) - } + return nil, err } - return nil + sqlDB.SetConnMaxIdleTime(10) + sqlDB.SetMaxOpenConns(100) + sqlDB.SetConnMaxLifetime(time.Hour) + return db, nil } -func checkPointOfWal() { - if err := global.DB.Exec("PRAGMA wal_checkpoint(TRUNCATE);").Error; err != nil { - global.LOG.Errorf("handle check point failed, err: %v", err) +func closeDatabase(db *gorm.DB) { + sqlDB, err := db.DB() + if err != nil { + return } + _ = sqlDB.Close() } diff --git a/agent/app/service/snapshot_recover.go b/agent/app/service/snapshot_recover.go index 7397a8a3a1c8..b2501920039c 100644 --- a/agent/app/service/snapshot_recover.go +++ b/agent/app/service/snapshot_recover.go @@ -2,6 +2,7 @@ package service import ( "context" + "encoding/json" "fmt" "os" "path" @@ -10,219 +11,361 @@ import ( "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/app/task" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/i18n" "github.com/1Panel-dev/1Panel/agent/utils/cmd" - "github.com/1Panel-dev/1Panel/agent/utils/common" + "github.com/1Panel-dev/1Panel/agent/utils/compose" "github.com/1Panel-dev/1Panel/agent/utils/files" "github.com/pkg/errors" ) -func (u *SnapshotService) HandleSnapshotRecover(snap model.Snapshot, isRecover bool, req dto.SnapshotRecover) { - _ = global.Cron.Stop() - defer func() { - global.Cron.Start() - }() +type snapRecoverHelper struct { + FileOp files.FileOp + Task *task.Task +} + +func (u *SnapshotService) SnapshotRecover(req dto.SnapshotRecover) error { + global.LOG.Info("start to recover panel by snapshot now") + snap, err := snapshotRepo.Get(commonRepo.WithByID(req.ID)) + if err != nil { + return err + } + if hasOs(snap.Name) && !strings.Contains(snap.Name, loadOs()) { + errInfo := fmt.Sprintf("restoring snapshots(%s) between different server architectures(%s) is not supported", snap.Name, loadOs()) + _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"recover_status": constant.StatusFailed, "recover_message": errInfo}) + return errors.New(errInfo) + } + if len(snap.RollbackStatus) != 0 && snap.RollbackStatus != constant.StatusSuccess { + req.IsNew = true + } + if !req.IsNew && (snap.InterruptStep == "RecoverDownload" || snap.InterruptStep == "RecoverDecompress" || snap.InterruptStep == "BackupBeforeRecover") { + req.IsNew = true + } + _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"recover_status": constant.StatusWaiting}) + _ = settingRepo.Update("SystemStatus", "Recovering") + + if len(snap.InterruptStep) == 0 { + req.IsNew = true + } + if len(snap.TaskRecoverID) != 0 { + req.TaskID = snap.TaskRecoverID + } else { + _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"task_recover_id": req.TaskID}) + } + taskItem, err := task.NewTaskWithOps(snap.Name, task.TaskRecover, task.TaskScopeSnapshot, req.TaskID, snap.ID) + if err != nil { + global.LOG.Errorf("new task for create snapshot failed, err: %v", err) + return err + } + rootDir := path.Join(global.CONF.System.TmpDir, "system", snap.Name) + if _, err := os.Stat(rootDir); err != nil && os.IsNotExist(err) { + _ = os.MkdirAll(rootDir, os.ModePerm) + } + itemHelper := snapRecoverHelper{Task: taskItem, FileOp: files.NewFileOp()} + + go func() { + _ = global.Cron.Stop() + defer func() { + global.Cron.Start() + }() - snapFileDir := "" - if isRecover { - baseDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("system/%s", snap.Name)) - if _, err := os.Stat(baseDir); err != nil && os.IsNotExist(err) { - _ = os.MkdirAll(baseDir, os.ModePerm) + if req.IsNew || snap.InterruptStep == "RecoverDownload" || req.ReDownload { + taskItem.AddSubTaskWithAlias( + "RecoverDownload", + func(t *task.Task) error { return handleDownloadSnapshot(&itemHelper, snap, rootDir) }, + nil, + ) + req.IsNew = true } - if req.IsNew || snap.InterruptStep == "Download" || req.ReDownload { - if err := handleDownloadSnapshot(snap, baseDir); err != nil { - updateRecoverStatus(snap.ID, isRecover, "Backup", constant.StatusFailed, err.Error()) - return - } - global.LOG.Debugf("download snapshot file to %s successful!", baseDir) + if req.IsNew || snap.InterruptStep == "RecoverDecompress" { + taskItem.AddSubTaskWithAlias( + "RecoverDecompress", + func(t *task.Task) error { + itemHelper.Task.Log("######################## 2 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetWithName("RecoverDecompress", snap.Name)) + err := itemHelper.FileOp.TarGzExtractPro(fmt.Sprintf("%s/%s.tar.gz", rootDir, snap.Name), rootDir, req.Secret) + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Decompress"), err) + return err + }, + nil, + ) req.IsNew = true } - if req.IsNew || snap.InterruptStep == "Decompress" { - if err := handleUnTar(fmt.Sprintf("%s/%s.tar.gz", baseDir, snap.Name), baseDir, req.Secret); err != nil { - updateRecoverStatus(snap.ID, isRecover, "Decompress", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) - return - } - global.LOG.Debug("decompress snapshot file successful!", baseDir) + if req.IsNew || snap.InterruptStep == "BackupBeforeRecover" { + taskItem.AddSubTaskWithAlias( + "BackupBeforeRecover", + func(t *task.Task) error { return backupBeforeRecover(snap.Name, &itemHelper) }, + nil, + ) req.IsNew = true } - if req.IsNew || snap.InterruptStep == "Backup" { - if err := backupBeforeRecover(snap); err != nil { - updateRecoverStatus(snap.ID, isRecover, "Backup", constant.StatusFailed, fmt.Sprintf("handle backup before recover failed, err: %v", err)) - return - } - global.LOG.Debug("handle backup before recover successful!") + + var snapJson SnapshotJson + taskItem.AddSubTaskWithAlias( + "Readjson", + func(t *task.Task) error { + snapJson, err = readFromJson(path.Join(rootDir, snap.Name), &itemHelper) + return err + }, + nil, + ) + if req.IsNew || snap.InterruptStep == "RecoverApp" { + taskItem.AddSubTaskWithAlias( + "RecoverApp", + func(t *task.Task) error { return recoverAppData(path.Join(rootDir, snap.Name), &itemHelper) }, + nil, + ) req.IsNew = true } - snapFileDir = fmt.Sprintf("%s/%s", baseDir, snap.Name) - if _, err := os.Stat(snapFileDir); err != nil { - snapFileDir = baseDir + if req.IsNew || snap.InterruptStep == "RecoverBaseData" { + taskItem.AddSubTaskWithAlias( + "RecoverBaseData", + func(t *task.Task) error { return recoverBaseData(path.Join(rootDir, snap.Name, "base"), &itemHelper) }, + nil, + ) + req.IsNew = true } - } else { - snapFileDir = fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, snap.Name) - if _, err := os.Stat(snapFileDir); err != nil { - updateRecoverStatus(snap.ID, isRecover, "", constant.StatusFailed, fmt.Sprintf("cannot find the backup file %s, please try to recover again.", snapFileDir)) + if req.IsNew || snap.InterruptStep == "RecoverDBData" { + taskItem.AddSubTaskWithAlias( + "RecoverDBData", + func(t *task.Task) error { return recoverDBData(path.Join(rootDir, snap.Name, "db"), &itemHelper) }, + nil, + ) + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "RecoverBackups" { + taskItem.AddSubTaskWithAlias( + "RecoverBackups", + func(t *task.Task) error { + itemHelper.Task.Log("######################## 8 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetWithName("RecoverBackups", snap.Name)) + err := itemHelper.FileOp.TarGzExtractPro(path.Join(rootDir, snap.Name, "/1panel_backup.tar.gz"), snapJson.BackupDataDir, "") + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Decompress"), err) + return err + }, + nil, + ) + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "RecoverPanelData" { + taskItem.AddSubTaskWithAlias( + "RecoverPanelData", + func(t *task.Task) error { + itemHelper.Task.Log("######################## 9 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetWithName("RecoverPanelData", snap.Name)) + err := itemHelper.FileOp.TarGzExtractPro(path.Join(rootDir, snap.Name, "/1panel_data.tar.gz"), path.Join(snapJson.BaseDir, "1panel"), "") + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Decompress"), err) + return err + }, + nil, + ) + req.IsNew = true + } + taskItem.AddSubTaskWithAlias( + "RecoverDBData", + func(t *task.Task) error { + return restartCompose(path.Join(snapJson.BaseDir, "1panel/docker/compose"), &itemHelper) + }, + nil, + ) + + if err := taskItem.Execute(); err != nil { + _ = settingRepo.Update("SystemStatus", "Free") + _ = snapshotRepo.Update(req.ID, map[string]interface{}{"recover_status": constant.StatusFailed, "recover_message": err.Error(), "interrupt_step": taskItem.Task.CurrentStep}) return } + _ = os.RemoveAll(rootDir) + _, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service") + }() + return nil +} + +func handleDownloadSnapshot(itemHelper *snapRecoverHelper, snap model.Snapshot, targetDir string) error { + itemHelper.Task.Log("######################## 1 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverDownload")) + + account, client, err := NewBackupClientWithID(snap.DownloadAccountID) + itemHelper.Task.LogWithStatus(i18n.GetWithName("RecoverDownloadAccount", fmt.Sprintf("%s - %s", account.Type, account.Name)), err) + pathItem := account.BackupPath + if account.BackupPath != "/" { + pathItem = strings.TrimPrefix(account.BackupPath, "/") } - snapJson, err := u.readFromJson(fmt.Sprintf("%s/snapshot.json", snapFileDir)) + filePath := fmt.Sprintf("%s/%s.tar.gz", targetDir, snap.Name) + _ = os.RemoveAll(filePath) + _, err = client.Download(path.Join(pathItem, fmt.Sprintf("system_snapshot/%s.tar.gz", snap.Name)), filePath) + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Download"), err) + return err +} + +func backupBeforeRecover(name string, itemHelper *snapRecoverHelper) error { + itemHelper.Task.Log("######################## 3 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetMsgByKey("BackupBeforeRecover")) + + rootDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, name) + baseDir := path.Join(rootDir, "base") + if _, err := os.Stat(baseDir); err != nil { + _ = os.MkdirAll(baseDir, os.ModePerm) + } + + err := itemHelper.FileOp.CopyDirWithExclude(path.Join(global.CONF.System.BaseDir, "1panel"), rootDir, []string{"cache", "tmp"}) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", path.Join(global.CONF.System.BaseDir, "1panel")), err) if err != nil { - updateRecoverStatus(snap.ID, isRecover, "Readjson", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) - return + return err } - if snap.InterruptStep == "Readjson" { - req.IsNew = true + err = itemHelper.FileOp.CopyDirWithExclude(global.CONF.System.Backup, rootDir, []string{"system_snapshot"}) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", global.CONF.System.Backup), err) + if err != nil { + return err } - if isRecover && (req.IsNew || snap.InterruptStep == "AppData") { - if err := recoverAppData(snapFileDir); err != nil { - updateRecoverStatus(snap.ID, isRecover, "DockerDir", constant.StatusFailed, fmt.Sprintf("handle recover app data failed, err: %v", err)) - return - } - global.LOG.Debug("recover app data from snapshot file successful!") - req.IsNew = true + err = itemHelper.FileOp.CopyFile("/usr/local/bin/1pctl", baseDir) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1pctl"), err) + if err != nil { + return err } - if req.IsNew || snap.InterruptStep == "DaemonJson" { - fileOp := files.NewFileOp() - if err := recoverDaemonJson(snapFileDir, fileOp); err != nil { - updateRecoverStatus(snap.ID, isRecover, "DaemonJson", constant.StatusFailed, err.Error()) - return - } - global.LOG.Debug("recover daemon.json from snapshot file successful!") - req.IsNew = true + err = itemHelper.FileOp.CopyFile("/usr/local/bin/1panel", baseDir) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel"), err) + if err != nil { + return err } - - if req.IsNew || snap.InterruptStep == "1PanelBinary" { - if err := recoverPanel(path.Join(snapFileDir, "1panel/1panel"), "/usr/local/bin"); err != nil { - updateRecoverStatus(snap.ID, isRecover, "1PanelBinary", constant.StatusFailed, err.Error()) - return - } - global.LOG.Debug("recover 1panel binary from snapshot file successful!") - req.IsNew = true + err = itemHelper.FileOp.CopyFile("/usr/local/bin/1panel_agent", baseDir) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel_agent"), err) + if err != nil { + return err } - if req.IsNew || snap.InterruptStep == "1PctlBinary" { - if err := recoverPanel(path.Join(snapFileDir, "1panel/1pctl"), "/usr/local/bin"); err != nil { - updateRecoverStatus(snap.ID, isRecover, "1PctlBinary", constant.StatusFailed, err.Error()) - return - } - global.LOG.Debug("recover 1pctl from snapshot file successful!") - req.IsNew = true + err = itemHelper.FileOp.CopyFile("/etc/systemd/system/1panel.service", baseDir) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel.service"), err) + if err != nil { + return err } - if req.IsNew || snap.InterruptStep == "1PanelService" { - if err := recoverPanel(path.Join(snapFileDir, "1panel/1panel.service"), "/etc/systemd/system"); err != nil { - updateRecoverStatus(snap.ID, isRecover, "1PanelService", constant.StatusFailed, err.Error()) - return - } - global.LOG.Debug("recover 1panel service from snapshot file successful!") - req.IsNew = true + err = itemHelper.FileOp.CopyFile("/etc/systemd/system/1panel_agent.service", baseDir) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel_agent.service"), err) + if err != nil { + return err } - - if req.IsNew || snap.InterruptStep == "1PanelBackups" { - if err := u.handleUnTar(path.Join(snapFileDir, "/1panel/1panel_backup.tar.gz"), snapJson.BackupDataDir, ""); err != nil { - updateRecoverStatus(snap.ID, isRecover, "1PanelBackups", constant.StatusFailed, err.Error()) - return - } - global.LOG.Debug("recover 1panel backups from snapshot file successful!") - req.IsNew = true + err = itemHelper.FileOp.CopyFile("/etc/docker/daemon.json", baseDir) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/docker/daemon.json"), err) + if err != nil { + return err } + return nil +} - if req.IsNew || snap.InterruptStep == "1PanelData" { - checkPointOfWal() - if err := u.handleUnTar(path.Join(snapFileDir, "/1panel/1panel_data.tar.gz"), path.Join(snapJson.BaseDir, "1panel"), ""); err != nil { - updateRecoverStatus(snap.ID, isRecover, "1PanelData", constant.StatusFailed, err.Error()) - return - } - global.LOG.Debug("recover 1panel data from snapshot file successful!") - req.IsNew = true - } - _ = rebuildAllAppInstall() - restartCompose(path.Join(snapJson.BaseDir, "1panel/docker/compose")) +func readFromJson(rootDir string, itemHelper *snapRecoverHelper) (SnapshotJson, error) { + itemHelper.Task.Log("######################## 4 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetMsgByKey("Readjson")) - global.LOG.Info("recover successful") - if !isRecover { - oriPath := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, snap.Name) - global.LOG.Debugf("remove the file %s after the operation is successful", oriPath) - _ = os.RemoveAll(oriPath) - } else { - global.LOG.Debugf("remove the file %s after the operation is successful", path.Dir(snapFileDir)) - _ = os.RemoveAll(path.Dir(snapFileDir)) + snapJsonPath := path.Join(rootDir, "base/snapshot.json") + var snap SnapshotJson + _, err := os.Stat(snapJsonPath) + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("ReadjsonPath"), err) + if err != nil { + return snap, err + } + fileByte, err := os.ReadFile(snapJsonPath) + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("ReadjsonContent"), err) + if err != nil { + return snap, err + } + err = json.Unmarshal(fileByte, &snap) + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("ReadjsonMarshal"), err) + if err != nil { + return snap, err } - _, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service") + return snap, nil } -func backupBeforeRecover(snap model.Snapshot) error { - baseDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, snap.Name) - var wg sync.WaitGroup - var status model.SnapshotStatus - itemHelper := snapHelper{SnapID: 0, Status: &status, Wg: &wg, FileOp: files.NewFileOp(), Ctx: context.Background()} +func recoverAppData(src string, itemHelper *snapRecoverHelper) error { + itemHelper.Task.Log("######################## 5 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverApp")) - jsonItem := SnapshotJson{ - BaseDir: global.CONF.System.BaseDir, - BackupDataDir: global.CONF.System.Backup, - PanelDataDir: path.Join(global.CONF.System.BaseDir, "1panel"), + if _, err := os.Stat(path.Join(src, "images.tar.gz")); err != nil { + itemHelper.Task.Log(i18n.GetMsgByKey("RecoverAppEmpty")) + return nil + } else { + std, err := cmd.Execf("docker load < %s", path.Join(src, "images.tar.gz")) + if err != nil { + itemHelper.Task.LogFailedWithErr(i18n.GetMsgByKey("RecoverAppImage"), errors.New(std)) + return fmt.Errorf("docker load images failed, err: %v", err) + } + itemHelper.Task.LogSuccess(i18n.GetMsgByKey("RecoverAppImage")) } - _ = os.MkdirAll(path.Join(baseDir, "1panel"), os.ModePerm) - _ = os.MkdirAll(path.Join(baseDir, "docker"), os.ModePerm) - - wg.Add(4) - itemHelper.Wg = &wg - go snapJson(itemHelper, jsonItem, baseDir) - go snapPanel(itemHelper, path.Join(baseDir, "1panel")) - go snapDaemonJson(itemHelper, path.Join(baseDir, "docker")) - go snapBackup(itemHelper, path.Join(baseDir, "1panel")) - wg.Wait() - itemHelper.Status.AppData = constant.StatusDone - allDone, msg := checkAllDone(status) - if !allDone { - return errors.New(msg) + appInstalls, err := appInstallRepo.ListBy() + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverAppList"), err) + if err != nil { + return err } - snapPanelData(itemHelper, path.Join(baseDir, "1panel")) - if status.PanelData != constant.StatusDone { - return errors.New(status.PanelData) + + var wg sync.WaitGroup + for i := 0; i < len(appInstalls); i++ { + wg.Add(1) + appInstalls[i].Status = constant.Rebuilding + _ = appInstallRepo.Save(context.Background(), &appInstalls[i]) + go func(app model.AppInstall) { + defer wg.Done() + dockerComposePath := app.GetComposePath() + out, err := compose.Down(dockerComposePath) + if err != nil { + _ = handleErr(app, err, out) + return + } + out, err = compose.Up(dockerComposePath) + if err != nil { + _ = handleErr(app, err, out) + return + } + app.Status = constant.Running + _ = appInstallRepo.Save(context.Background(), &app) + }(appInstalls[i]) } + wg.Wait() return nil } -func handleDownloadSnapshot(snap model.Snapshot, targetDir string) error { - account, client, err := NewBackupClientWithID(snap.DownloadAccountID) +func recoverBaseData(src string, itemHelper *snapRecoverHelper) error { + itemHelper.Task.Log("######################## 6 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetMsgByKey("SnapBaseInfo")) + + err := itemHelper.FileOp.CopyFile(path.Join(src, "1pctl"), "/usr/local/bin") + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1pctl"), err) if err != nil { return err } - pathItem := account.BackupPath - if account.BackupPath != "/" { - pathItem = strings.TrimPrefix(account.BackupPath, "/") + + err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel"), "/usr/local/bin") + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel"), err) + if err != nil { + return err } - filePath := fmt.Sprintf("%s/%s.tar.gz", targetDir, snap.Name) - _ = os.RemoveAll(filePath) - ok, err := client.Download(path.Join(pathItem, fmt.Sprintf("system_snapshot/%s.tar.gz", snap.Name)), filePath) - if err != nil || !ok { - return fmt.Errorf("download file %s from %s failed, err: %v", snap.Name, account.Name, err) + err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel_agent"), "/usr/local/bin") + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel_agent"), err) + if err != nil { + return err } - return nil -} - -func recoverAppData(src string) error { - if _, err := os.Stat(path.Join(src, "docker/docker_image.tar")); err != nil { - global.LOG.Debug("no such docker images in snapshot") - return nil + err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel.service"), "/etc/systemd/system") + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel.service"), err) + if err != nil { + return err } - std, err := cmd.Execf("docker load < %s", path.Join(src, "docker/docker_image.tar")) + err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel_agent.service"), "/etc/systemd/system") + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel_agent.service"), err) if err != nil { - return errors.New(std) + return err } - return err -} -func recoverDaemonJson(src string, fileOp files.FileOp) error { daemonJsonPath := "/etc/docker/daemon.json" _, errSrc := os.Stat(path.Join(src, "docker/daemon.json")) _, errPath := os.Stat(daemonJsonPath) if os.IsNotExist(errSrc) && os.IsNotExist(errPath) { - global.LOG.Debug("the daemon.json file does not exist, nothing happens.") + itemHelper.Task.Log(i18n.GetMsgByKey("RecoverDaemonJsonEmpty")) return nil } if errSrc == nil { - if err := fileOp.CopyFile(path.Join(src, "docker/daemon.json"), "/etc/docker"); err != nil { + err = itemHelper.FileOp.CopyFile(path.Join(src, "docker/daemon.json"), "/etc/docker") + itemHelper.Task.Log(i18n.GetMsgByKey("RecoverDaemonJson")) + if err != nil { return fmt.Errorf("recover docker daemon.json failed, err: %v", err) } } @@ -231,21 +374,25 @@ func recoverDaemonJson(src string, fileOp files.FileOp) error { return nil } -func recoverPanel(src string, dst string) error { - if _, err := os.Stat(src); err != nil { - return fmt.Errorf("file is not found in %s, err: %v", src, err) - } - if err := common.CopyFile(src, dst); err != nil { - return fmt.Errorf("cp file failed, err: %v", err) - } - return nil +func recoverDBData(src string, itemHelper *snapRecoverHelper) error { + itemHelper.Task.Log("######################## 7 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverDBData")) + err := itemHelper.FileOp.CopyDirWithExclude(src, path.Join(global.CONF.System.BaseDir, "1panel"), nil) + + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverDBData"), err) + return err } -func restartCompose(composePath string) { +func restartCompose(composePath string, itemHelper *snapRecoverHelper) error { + itemHelper.Task.Log("######################## 10 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverCompose")) + composes, err := composeRepo.ListRecord() + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverComposeList"), err) if err != nil { - return + return err } + for _, compose := range composes { pathItem := path.Join(composePath, compose.Name, "docker-compose.yml") if _, err := os.Stat(pathItem); err != nil { @@ -254,8 +401,10 @@ func restartCompose(composePath string) { upCmd := fmt.Sprintf("docker compose -f %s up -d", pathItem) stdout, err := cmd.Exec(upCmd) if err != nil { - global.LOG.Debugf("%s failed, err: %v", upCmd, stdout) + itemHelper.Task.LogFailedWithErr(i18n.GetMsgByKey("RecoverCompose"), errors.New(stdout)) + continue } + itemHelper.Task.LogSuccess(i18n.GetWithName("RecoverComposeItem", pathItem)) } - global.LOG.Debug("restart all compose successful!") + return nil } diff --git a/agent/app/service/snapshot_rollback.go b/agent/app/service/snapshot_rollback.go new file mode 100644 index 000000000000..5af322e68de3 --- /dev/null +++ b/agent/app/service/snapshot_rollback.go @@ -0,0 +1,107 @@ +package service + +import ( + "fmt" + "os" + "path" + + "github.com/1Panel-dev/1Panel/agent/app/dto" + "github.com/1Panel-dev/1Panel/agent/app/task" + "github.com/1Panel-dev/1Panel/agent/constant" + "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/i18n" + "github.com/1Panel-dev/1Panel/agent/utils/files" +) + +func (u *SnapshotService) SnapshotRollback(req dto.SnapshotRecover) error { + global.LOG.Info("start to rollback now") + snap, err := snapshotRepo.Get(commonRepo.WithByID(req.ID)) + if err != nil { + return err + } + if len(snap.TaskRollbackID) != 0 { + req.TaskID = snap.TaskRollbackID + } else { + _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"task_rollback_id": req.TaskID}) + } + taskItem, err := task.NewTaskWithOps(snap.Name, task.TaskRollback, task.TaskScopeSnapshot, req.TaskID, snap.ID) + if err != nil { + global.LOG.Errorf("new task for create snapshot failed, err: %v", err) + return err + } + go func() { + rootDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, snap.Name) + baseDir := path.Join(rootDir, "base") + + FileOp := files.NewFileOp() + taskItem.AddSubTask( + i18n.GetWithName("SnapCopy", "/usr/local/bin/1pctl"), + func(t *task.Task) error { + return FileOp.CopyFile(path.Join(baseDir, "1pctl"), "/usr/local/bin") + }, + nil, + ) + taskItem.AddSubTask( + i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel"), + func(t *task.Task) error { + return FileOp.CopyFile(path.Join(baseDir, "1panel"), "/usr/local/bin") + }, + nil, + ) + taskItem.AddSubTask( + i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel_agent"), + func(t *task.Task) error { + return FileOp.CopyFile(path.Join(baseDir, "1panel_agent"), "/usr/local/bin") + }, + nil, + ) + taskItem.AddSubTask( + i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel.service"), + func(t *task.Task) error { + return FileOp.CopyFile(path.Join(baseDir, "1panel.service"), "/etc/systemd/system") + }, + nil, + ) + taskItem.AddSubTask( + i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel_agent.service"), + func(t *task.Task) error { + return FileOp.CopyFile(path.Join(baseDir, "1panel.service"), "/etc/systemd/system") + }, + nil, + ) + taskItem.AddSubTask( + i18n.GetWithName("SnapCopy", "/etc/docker/daemon.json"), + func(t *task.Task) error { + return FileOp.CopyFile(path.Join(baseDir, "daemon.json"), "/etc/docker") + }, + nil, + ) + taskItem.AddSubTask( + i18n.GetWithName("SnapCopy", global.CONF.System.Backup), + func(t *task.Task) error { + return FileOp.CopyDir(path.Join(rootDir, "backup"), global.CONF.System.Backup) + }, + nil, + ) + taskItem.AddSubTask( + i18n.GetWithName("SnapCopy", global.CONF.System.BaseDir), + func(t *task.Task) error { + return FileOp.CopyDir(path.Join(rootDir, "1panel"), global.CONF.System.BaseDir) + }, + nil, + ) + if err := taskItem.Execute(); err != nil { + _ = snapshotRepo.Update(req.ID, map[string]interface{}{"rollback_status": constant.StatusFailed, "rollback_message": err.Error()}) + return + } + _ = snapshotRepo.Update(req.ID, map[string]interface{}{ + "recover_status": "", + "recover_message": "", + "rollback_status": "", + "rollback_message": "", + "interrupt_step": "", + }) + _ = os.RemoveAll(rootDir) + }() + return nil +} diff --git a/agent/app/task/task.go b/agent/app/task/task.go index cea2f77c8ead..c129ae96ff3c 100644 --- a/agent/app/task/task.go +++ b/agent/app/task/task.go @@ -3,16 +3,17 @@ package task import ( "context" "fmt" - "github.com/1Panel-dev/1Panel/agent/app/model" - "github.com/1Panel-dev/1Panel/agent/app/repo" - "github.com/1Panel-dev/1Panel/agent/constant" - "github.com/1Panel-dev/1Panel/agent/i18n" - "github.com/google/uuid" "log" "os" "path" "strconv" "time" + + "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/app/repo" + "github.com/1Panel-dev/1Panel/agent/constant" + "github.com/1Panel-dev/1Panel/agent/i18n" + "github.com/google/uuid" ) type ActionFunc func(*Task) error @@ -33,6 +34,7 @@ type Task struct { type SubTask struct { RootTask *Task Name string + StepAlias string Retry int Timeout time.Duration Action ActionFunc @@ -50,6 +52,8 @@ const ( TaskUpdate = "TaskUpdate" TaskRestart = "TaskRestart" TaskBackup = "TaskBackup" + TaskRecover = "TaskRecover" + TaskRollback = "TaskRollback" TaskSync = "TaskSync" TaskBuild = "TaskBuild" ) @@ -60,6 +64,7 @@ const ( TaskScopeRuntime = "Runtime" TaskScopeDatabase = "Database" TaskScopeAppStore = "AppStore" + TaskScopeSnapshot = "Snapshot" TaskScopeRuntimeExtension = "RuntimeExtension" ) @@ -111,6 +116,11 @@ func (t *Task) AddSubTask(name string, action ActionFunc, rollback RollbackFunc) t.SubTasks = append(t.SubTasks, subTask) } +func (t *Task) AddSubTaskWithAlias(key string, action ActionFunc, rollback RollbackFunc) { + subTask := &SubTask{RootTask: t, Name: i18n.GetMsgByKey(key), StepAlias: key, Retry: 0, Timeout: 10 * time.Minute, Action: action, Rollback: rollback} + t.SubTasks = append(t.SubTasks, subTask) +} + func (t *Task) AddSubTaskWithOps(name string, action ActionFunc, rollback RollbackFunc, retry int, timeout time.Duration) { subTask := &SubTask{RootTask: t, Name: name, Retry: retry, Timeout: timeout, Action: action, Rollback: rollback} t.SubTasks = append(t.SubTasks, subTask) @@ -166,13 +176,13 @@ func (t *Task) updateTask(task *model.Task) { } func (t *Task) Execute() error { - if err := t.taskRepo.Create(context.Background(), t.Task); err != nil { + if err := t.taskRepo.Save(context.Background(), t.Task); err != nil { return err } var err error t.Log(i18n.GetWithName("TaskStart", t.Name)) for _, subTask := range t.SubTasks { - t.Task.CurrentStep = subTask.Name + t.Task.CurrentStep = subTask.StepAlias t.updateTask(t.Task) if err = subTask.Execute(); err == nil { if subTask.Rollback != nil { @@ -221,6 +231,10 @@ func (t *Task) Log(msg string) { t.Logger.Printf(msg) } +func (t *Task) Logf(format string, v ...any) { + t.Logger.Printf(format, v...) +} + func (t *Task) LogFailed(msg string) { t.Logger.Printf(msg + i18n.GetMsgByKey("Failed")) } @@ -232,6 +246,9 @@ func (t *Task) LogFailedWithErr(msg string, err error) { func (t *Task) LogSuccess(msg string) { t.Logger.Printf(msg + i18n.GetMsgByKey("Success")) } +func (t *Task) LogSuccessf(format string, v ...any) { + t.Logger.Printf(fmt.Sprintf(format, v...) + i18n.GetMsgByKey("Success")) +} func (t *Task) LogStart(msg string) { t.Logger.Printf(fmt.Sprintf("%s%s", i18n.GetMsgByKey("Start"), msg)) diff --git a/agent/i18n/lang/en.yaml b/agent/i18n/lang/en.yaml index 995c326ff013..23268989d1d0 100644 --- a/agent/i18n/lang/en.yaml +++ b/agent/i18n/lang/en.yaml @@ -219,6 +219,7 @@ TaskDelete: "Delete" TaskUpgrade: "Upgrade" TaskUpdate: "Update" TaskRestart: "Restart" +TaskRollback: "Rollback" Website: "Website" App: "App" Runtime: "Runtime" @@ -247,3 +248,28 @@ SubTask: "Subtask" RuntimeExtension: "Runtime Extension" TaskBuild: "Build" +# task - snapshot +Snapshot: "Snapshot" +SnapDBInfo: "Write 1Panel database information" +SnapCopy: "Copy files & directories {{ .name }} " +SnapNewDB: "Initialize database {{ .name }} connection " +SnapDeleteOperationLog: "Delete operation log" +SnapDeleteLoginLog: "Delete access log" +SnapDeleteMonitor: "Delete monitoring data" +SnapRemoveSystemIP: "Remove system IP" +SnapBaseInfo: "Write 1Panel basic information" +SnapInstallApp: "Backup installed applications in 1Panel" +SnapDockerSave: "Compress installed applications" +SnapLocalBackup: "Backup 1Panel local backup directory" +SnapCompressBackup: "Compress local backup directory" +SnapPanelData: "Backup 1Panel data directory" +SnapCompressPanel: "Compress data directory" +SnapCloseDBConn: "Close database connection" +SnapCompress: "Create snapshot file" +SnapCompressFile: "Compress snapshot file" +SnapCheckCompress: "Check snapshot compressed file" +SnapCompressSize: "Snapshot file size {{ .name }}" +SnapUpload: "Upload snapshot file" +SnapLoadBackup: "Get backup account information" +SnapUploadTo: "Upload snapshot file to {{ .name }}" +SnapUploadRes: "Upload snapshot file to {{ .name }}" \ No newline at end of file diff --git a/agent/i18n/lang/zh-Hant.yaml b/agent/i18n/lang/zh-Hant.yaml index 259d156df503..2c3fec5d168a 100644 --- a/agent/i18n/lang/zh-Hant.yaml +++ b/agent/i18n/lang/zh-Hant.yaml @@ -221,6 +221,7 @@ TaskDelete: "刪除" TaskUpgrade: "升級" TaskUpdate: "更新" TaskRestart: "重啟" +TaskRollback: "回滚" Website: "網站" App: "應用" Runtime: "運行環境" @@ -250,3 +251,28 @@ RuntimeExtension: "運行環境擴展" TaskBuild: "構建" +# task - snapshot +Snapshot: "快照" +SnapDBInfo: "寫入 1Panel 資料庫資訊" +SnapCopy: "複製檔案&目錄 {{ .name }} " +SnapNewDB: "初始化資料庫 {{ .name }} 連接 " +SnapDeleteOperationLog: "刪除操作日誌" +SnapDeleteLoginLog: "刪除訪問日誌" +SnapDeleteMonitor: "刪除監控數據" +SnapRemoveSystemIP: "移除系統 IP" +SnapBaseInfo: "寫入 1Panel 基本資訊" +SnapInstallApp: "備份 1Panel 已安裝應用" +SnapDockerSave: "壓縮已安裝應用" +SnapLocalBackup: "備份 1Panel 本地備份目錄" +SnapCompressBackup: "壓縮本地備份目錄" +SnapPanelData: "備份 1Panel 資料目錄" +SnapCompressPanel: "壓縮資料目錄" +SnapCloseDBConn: "關閉資料庫連接" +SnapCompress: "製作快照檔案" +SnapCompressFile: "壓縮快照檔案" +SnapCheckCompress: "檢查快照壓縮檔案" +SnapCompressSize: "快照檔案大小 {{ .name }}" +SnapUpload: "上傳快照檔案" +SnapLoadBackup: "獲取備份帳號資訊" +SnapUploadTo: "上傳快照檔案到 {{ .name }}" +SnapUploadRes: "上傳快照檔案到 {{ .name }}" \ No newline at end of file diff --git a/agent/i18n/lang/zh.yaml b/agent/i18n/lang/zh.yaml index 7d5b63fc2049..99dade25561c 100644 --- a/agent/i18n/lang/zh.yaml +++ b/agent/i18n/lang/zh.yaml @@ -223,6 +223,8 @@ TaskUpgrade: "升级" TaskUpdate: "更新" TaskRestart: "重启" TaskBackup: "备份" +TaskRecover: "恢复" +TaskRollback: "回滚" Website: "网站" App: "应用" Runtime: "运行环境" @@ -249,4 +251,55 @@ TaskSync: "同步" LocalApp: "本地应用" SubTask: "子任务" RuntimeExtension: "运行环境扩展" -TaskBuild: "构建" \ No newline at end of file + +# task - snapshot +Snapshot: "快照" +SnapDBInfo: "写入 1Panel 数据库信息" +SnapCopy: "复制文件&目录 {{ .name }} " +SnapNewDB: "初始化数据库 {{ .name }} 连接 " +SnapDeleteOperationLog: "删除操作日志" +SnapDeleteLoginLog: "删除访问日志" +SnapDeleteMonitor: "删除监控数据" +SnapRemoveSystemIP: "移除系统 IP" +SnapBaseInfo: "写入 1Panel 基本信息" +SnapInstallApp: "备份 1Panel 已安装应用" +SnapDockerSave: "压缩已安装应用" +SnapLocalBackup: "备份 1Panel 本地备份目录" +SnapCompressBackup: "压缩本地备份目录" +SnapPanelData: "备份 1Panel 数据目录" +SnapCompressPanel: "压缩数据目录" +SnapCloseDBConn: "关闭数据库连接" +SnapCompress: "制作快照文件" +SnapCompressFile: "压缩快照文件" +SnapCheckCompress: "检查快照压缩文件" +SnapCompressSize: "快照文件大小 {{ .name }}" +SnapUpload: "上传快照文件" +SnapLoadBackup: "获取备份账号信息" +SnapUploadTo: "上传快照文件到 {{ .name }}" +SnapUploadRes: "上传快照文件到 {{ .name }}" + +SnapshotRecover: "快照恢复" +RecoverDownload: "下载快照文件" +Download: "下载" +RecoverDownloadAccount: "获取快照下载备份账号 {{ .name }}" +RecoverDecompress: "解压快照压缩文件" +Decompress: "解压" +BackupBeforeRecover: "快照前备份系统相关数据" +Readjson: "读取快照内 Json 文件" +ReadjsonPath: "获取快照内 Json 文件路径" +ReadjsonContent: "读取 Json 文件" +ReadjsonMarshal: "Json 转义处理" +RecoverApp: "恢复已安装应用" +RecoverAppImage: "恢复快照镜像备份" +RecoverAppList: "获取所有待恢复应用" +RecoverCompose: "恢复其他编排内容" +RecoverComposeList: "获取所有待恢复编排" +RecoverComposeItem: "恢复编排 {{ .name }}" +RecoverAppEmpty: "快照文件中未发现应用镜像备份" +RecoverBaseData: "恢复基础数据及文件" +RecoverDaemonJsonEmpty: "快照文件及当前机器都不存在容器配置 daemon.json 文件" +RecoverDaemonJson: "恢复容器配置 daemon.json 文件" +RecoverDBData: "恢复数据库数据" +RecoverBackups: "恢复本地备份目录" +RecoverPanelData: "恢复数据目录" + diff --git a/agent/init/hook/hook.go b/agent/init/hook/hook.go index f2d7cc45b9e6..9ec2926523f9 100644 --- a/agent/init/hook/hook.go +++ b/agent/init/hook/hook.go @@ -66,40 +66,6 @@ func handleSnapStatus() { "rollback_status": constant.StatusFailed, "rollback_message": msgFailed, }).Error - - snapRepo := repo.NewISnapshotRepo() - - status, _ := snapRepo.GetStatusList() - for _, item := range status { - updates := make(map[string]interface{}) - if item.Panel == constant.StatusRunning { - updates["panel"] = constant.StatusFailed - } - if item.PanelInfo == constant.StatusRunning { - updates["panel_info"] = constant.StatusFailed - } - if item.DaemonJson == constant.StatusRunning { - updates["daemon_json"] = constant.StatusFailed - } - if item.AppData == constant.StatusRunning { - updates["app_data"] = constant.StatusFailed - } - if item.PanelData == constant.StatusRunning { - updates["panel_data"] = constant.StatusFailed - } - if item.BackupData == constant.StatusRunning { - updates["backup_data"] = constant.StatusFailed - } - if item.Compress == constant.StatusRunning { - updates["compress"] = constant.StatusFailed - } - if item.Upload == constant.StatusUploading { - updates["upload"] = constant.StatusFailed - } - if len(updates) != 0 { - _ = snapRepo.UpdateStatus(item.ID, updates) - } - } } func handleCronjobStatus() { diff --git a/agent/init/migration/migrate.go b/agent/init/migration/migrate.go index 36d24b8e59dc..cf5b146199d2 100644 --- a/agent/init/migration/migrate.go +++ b/agent/init/migration/migrate.go @@ -15,6 +15,13 @@ func Init() { migrations.InitImageRepo, migrations.InitDefaultCA, migrations.InitPHPExtensions, + migrations.AddTask, + migrations.UpdateWebsite, + migrations.UpdateWebsiteDomain, + migrations.UpdateApp, + migrations.AddTaskDB, + migrations.UpdateAppInstall, + migrations.UpdateSnapshot, }) if err := m.Migrate(); err != nil { global.LOG.Error(err) diff --git a/agent/init/migration/migrations/init.go b/agent/init/migration/migrations/init.go index ca3c0833ddeb..15c19b131145 100644 --- a/agent/init/migration/migrations/init.go +++ b/agent/init/migration/migrations/init.go @@ -46,7 +46,6 @@ var AddTable = &gormigrate.Migration{ &model.Runtime{}, &model.Setting{}, &model.Snapshot{}, - &model.SnapshotStatus{}, &model.Tag{}, &model.Website{}, &model.WebsiteAcmeAccount{}, @@ -211,3 +210,59 @@ var InitPHPExtensions = &gormigrate.Migration{ return nil }, } + +var AddTask = &gormigrate.Migration{ + ID: "20240802-add-task", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate( + &model.Task{}) + }, +} + +var UpdateWebsite = &gormigrate.Migration{ + ID: "20240812-update-website", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate( + &model.Website{}) + }, +} + +var UpdateWebsiteDomain = &gormigrate.Migration{ + ID: "20240808-update-website-domain", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate( + &model.WebsiteDomain{}) + }, +} + +var AddTaskDB = &gormigrate.Migration{ + ID: "20240822-add-task-table", + Migrate: func(tx *gorm.DB) error { + return global.TaskDB.AutoMigrate( + &model.Task{}, + ) + }, +} + +var UpdateApp = &gormigrate.Migration{ + ID: "20240826-update-app", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate( + &model.App{}) + }, +} + +var UpdateAppInstall = &gormigrate.Migration{ + ID: "20240828-update-app-install", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate( + &model.AppInstall{}) + }, +} + +var UpdateSnapshot = &gormigrate.Migration{ + ID: "20240926-update-snapshot", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&model.Snapshot{}) + }, +} diff --git a/agent/router/ro_setting.go b/agent/router/ro_setting.go index 67ecd12eaf4a..9d2524bad55b 100644 --- a/agent/router/ro_setting.go +++ b/agent/router/ro_setting.go @@ -15,8 +15,9 @@ func (s *SettingRouter) InitRouter(Router *gin.RouterGroup) { settingRouter.GET("/search/available", baseApi.GetSystemAvailable) settingRouter.POST("/update", baseApi.UpdateSetting) + settingRouter.GET("/snapshot/load", baseApi.LoadSnapshotData) settingRouter.POST("/snapshot", baseApi.CreateSnapshot) - settingRouter.POST("/snapshot/status", baseApi.LoadSnapShotStatus) + settingRouter.POST("/snapshot/recreate", baseApi.RecreateSnapshot) settingRouter.POST("/snapshot/search", baseApi.SearchSnapshot) settingRouter.POST("/snapshot/import", baseApi.ImportSnapshot) settingRouter.POST("/snapshot/del", baseApi.DeleteSnapshot) diff --git a/agent/utils/files/file_op.go b/agent/utils/files/file_op.go index 4248f72ce3f7..8ccd7c000bec 100644 --- a/agent/utils/files/file_op.go +++ b/agent/utils/files/file_op.go @@ -456,6 +456,47 @@ func (f FileOp) CopyDir(src, dst string) error { return cmd.ExecCmd(fmt.Sprintf(`cp -rf '%s' '%s'`, src, dst+"/")) } +func (f FileOp) CopyDirWithExclude(src, dst string, excludeNames []string) error { + srcInfo, err := f.Fs.Stat(src) + if err != nil { + return err + } + dstDir := filepath.Join(dst, srcInfo.Name()) + if err = f.Fs.MkdirAll(dstDir, srcInfo.Mode()); err != nil { + return err + } + if len(excludeNames) == 0 { + return cmd.ExecCmd(fmt.Sprintf(`cp -rf '%s' '%s'`, src, dst+"/")) + } + tmpFiles, err := os.ReadDir(src) + if err != nil { + return err + } + for _, item := range tmpFiles { + isExclude := false + for _, name := range excludeNames { + if item.Name() == name { + isExclude = true + break + } + } + if isExclude { + continue + } + if item.IsDir() { + if err := f.CopyDir(path.Join(src, item.Name()), dstDir); err != nil { + return err + } + continue + } + if err := f.CopyFile(path.Join(src, item.Name()), dstDir); err != nil { + return err + } + } + + return nil +} + func (f FileOp) CopyFile(src, dst string) error { dst = filepath.Clean(dst) + string(filepath.Separator) return cmd.ExecCmd(fmt.Sprintf(`cp -f '%s' '%s'`, src, dst+"/")) @@ -670,3 +711,56 @@ func ZipFile(files []archiver.File, dst afero.File) error { } return nil } + +func (f FileOp) TarGzCompressPro(withDir bool, src, dst, secret, exclusionRules string) error { + workdir := src + srcItem := "." + if withDir { + workdir = path.Dir(src) + srcItem = path.Base(src) + } + commands := "" + + exMap := make(map[string]struct{}) + exStr := "" + excludes := strings.Split(exclusionRules, ";") + excludes = append(excludes, "*.sock") + for _, exclude := range excludes { + if len(exclude) == 0 { + continue + } + if _, ok := exMap[exclude]; ok { + continue + } + exStr += " --exclude " + exStr += exclude + exMap[exclude] = struct{}{} + } + + if len(secret) != 0 { + commands = fmt.Sprintf("tar -zcf - %s | openssl enc -aes-256-cbc -salt -pbkdf2 -k '%s' -out %s", srcItem, secret, dst) + global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) + } else { + commands = fmt.Sprintf("tar zcf %s %s %s", dst, exStr, srcItem) + global.LOG.Debug(commands) + } + return cmd.ExecCmdWithDir(commands, workdir) +} + +func (f FileOp) TarGzExtractPro(src, dst string, secret string) error { + if _, err := os.Stat(path.Dir(dst)); err != nil && os.IsNotExist(err) { + if err = os.MkdirAll(path.Dir(dst), os.ModePerm); err != nil { + return err + } + } + + commands := "" + if len(secret) != 0 { + commands = fmt.Sprintf("openssl enc -d -aes-256-cbc -salt -pbkdf2 -k '%s' -in %s | tar -zxf - > /root/log", secret, src) + global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) + } else { + commands = fmt.Sprintf("tar zxvf %s", src) + global.LOG.Debug(commands) + } + return cmd.ExecCmdWithDir(commands, dst) +} diff --git a/agent/utils/files/tar_gz.go b/agent/utils/files/tar_gz.go index 93a82c54b125..f59a113fb635 100644 --- a/agent/utils/files/tar_gz.go +++ b/agent/utils/files/tar_gz.go @@ -2,6 +2,8 @@ package files import ( "fmt" + "os" + "path" "path/filepath" "strings" @@ -59,3 +61,56 @@ func (t TarGzArchiver) Compress(sourcePaths []string, dstFile string, secret str } return nil } + +func (t TarGzArchiver) CompressPro(withDir bool, src, dst, secret, exclusionRules string) error { + workdir := src + srcItem := "." + if withDir { + workdir = path.Dir(src) + srcItem = path.Base(src) + } + commands := "" + + exMap := make(map[string]struct{}) + exStr := "" + excludes := strings.Split(exclusionRules, ";") + excludes = append(excludes, "*.sock") + for _, exclude := range excludes { + if len(exclude) == 0 { + continue + } + if _, ok := exMap[exclude]; ok { + continue + } + exStr += " --exclude " + exStr += exclude + exMap[exclude] = struct{}{} + } + + if len(secret) != 0 { + commands = fmt.Sprintf("tar -zcf - %s | openssl enc -aes-256-cbc -salt -pbkdf2 -k '%s' -out %s", srcItem, secret, dst) + global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) + } else { + commands = fmt.Sprintf("tar zcf %s %s %s", dst, exStr, srcItem) + global.LOG.Debug(commands) + } + return cmd.ExecCmdWithDir(commands, workdir) +} + +func (t TarGzArchiver) ExtractPro(src, dst string, secret string) error { + if _, err := os.Stat(path.Dir(dst)); err != nil && os.IsNotExist(err) { + if err = os.MkdirAll(path.Dir(dst), os.ModePerm); err != nil { + return err + } + } + + commands := "" + if len(secret) != 0 { + commands = fmt.Sprintf("openssl enc -d -aes-256-cbc -salt -pbkdf2 -k '%s' -in %s | tar -zxf - > /root/log", secret, src) + global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) + } else { + commands = fmt.Sprintf("tar zxvf %s", src) + global.LOG.Debug(commands) + } + return cmd.ExecCmdWithDir(commands, dst) +} diff --git a/frontend/package.json b/frontend/package.json index 625c38e40bd9..d911425c0def 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -38,7 +38,7 @@ "codemirror": "^6.0.1", "echarts": "^5.5.0", "element-plus": "^2.7.5", - "fit2cloud-ui-plus": "^1.1.5", + "fit2cloud-ui-plus": "^1.1.7", "highlight.js": "^11.9.0", "js-base64": "^3.7.7", "md-editor-v3": "^2.11.3", diff --git a/frontend/src/api/interface/setting.ts b/frontend/src/api/interface/setting.ts index e358879a7ba3..ffb9d87a13f5 100644 --- a/frontend/src/api/interface/setting.ts +++ b/frontend/src/api/interface/setting.ts @@ -119,19 +119,27 @@ export namespace Setting { export interface SnapshotCreate { id: number; - from: string; - fromAccounts: Array; - defaultDownload: string; + sourceAccountIDs: string; + downloadAccountID: string; description: string; secret: string; + + appData: Array; + panelData: Array; + backupData: Array; + + withMonitorData: boolean; + withLoginLog: boolean; + withOperationLog: boolean; } export interface SnapshotImport { - from: string; + backupAccountID: number; names: Array; description: string; } export interface SnapshotRecover { id: number; + taskID: string; isNew: boolean; reDownload: boolean; secret: string; @@ -146,20 +154,43 @@ export namespace Setting { message: string; createdAt: DateTimeFormats; version: string; + secret: string; + + taskID: string; + taskRecoverID: string; + taskRollbackID: string; + interruptStep: string; recoverStatus: string; recoverMessage: string; - lastRecoveredAt: string; rollbackStatus: string; rollbackMessage: string; - lastRollbackedAt: string; - secret: string; + } + export interface SnapshotData { + appData: Array; + panelData: Array; + backupData: Array; + + withMonitorData: boolean; + withLoginLog: boolean; + withOperationLog: boolean; + } + export interface DataTree { + id: string; + label: string; + key: string; + name: string; + size: number; + isCheck: boolean; + isDisable: boolean; + + path: string; + + Children: Array; } export interface SnapshotStatus { - panel: string; - panelInfo: string; - daemonJson: string; - appData: string; + baseData: string; + appImage: string; panelData: string; backupData: string; diff --git a/frontend/src/api/modules/backup.ts b/frontend/src/api/modules/backup.ts index e609345f8e97..512b77ad676f 100644 --- a/frontend/src/api/modules/backup.ts +++ b/frontend/src/api/modules/backup.ts @@ -27,8 +27,8 @@ export const searchBackupRecords = (params: Backup.SearchBackupRecord) => { export const searchBackupRecordsByCronjob = (params: Backup.SearchBackupRecordByCronjob) => { return http.post>(`/backups/record/search/bycronjob`, params, TimeoutEnum.T_5M); }; -export const getFilesFromBackup = (type: string) => { - return http.post>(`/backups/search/files`, { type: type }); +export const getFilesFromBackup = (id: number) => { + return http.post>(`/backups/search/files`, { id: id }); }; // backup-core diff --git a/frontend/src/api/modules/setting.ts b/frontend/src/api/modules/setting.ts index abd3b0e7e92a..d4805b451263 100644 --- a/frontend/src/api/modules/setting.ts +++ b/frontend/src/api/modules/setting.ts @@ -28,6 +28,12 @@ export const loadBaseDir = () => { export const loadDaemonJsonPath = () => { return http.get(`/settings/daemonjson`, {}); }; +export const updateAgentSetting = (param: Setting.SettingUpdate) => { + return http.post(`/settings/update`, param); +}; +export const getAgentSettingInfo = () => { + return http.post(`/settings/search`); +}; // core export const getSettingInfo = () => { @@ -88,9 +94,15 @@ export const bindMFA = (param: Setting.MFABind) => { }; // snapshot +export const loadSnapshotInfo = () => { + return http.get(`/settings/snapshot/load`); +}; export const snapshotCreate = (param: Setting.SnapshotCreate) => { return http.post(`/settings/snapshot`, param); }; +export const snapshotRecreate = (id: number) => { + return http.post(`/settings/snapshot/recreate`, { id: id }); +}; export const loadSnapStatus = (id: number) => { return http.post(`/settings/snapshot/status`, { id: id }); }; diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index cd54c553fb77..85bdfdda019f 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -12,6 +12,8 @@ const message = { false: 'false', example: 'e.g.:', button: { + prev: 'Previous', + next: 'Next', create: 'Create ', add: 'Add ', save: 'Save ', @@ -1587,6 +1589,38 @@ const message = { 'Backup files not in the current backup list, please try downloading from the file directory and importing for backup.', snapshot: 'Snapshot', + stepBaseData: 'Base Data', + stepAppData: 'System Application', + stepPanelData: 'System Data', + stepBackupData: 'Backup Data', + stepOtherData: 'Other Data', + operationLog: 'Retain Operation Log', + loginLog: 'Retain Access Log', + systemLog: 'Retain System Log', + taskLog: 'Retain Task Log', + monitorData: 'Retain Monitoring Data', + selectAllImage: 'Backup All Application Images', + agentLabel: 'Node Configuration', + appDataLabel: 'Application Data', + appImage: 'Application Image', + appBackup: 'Application Backup', + backupLabel: 'Backup Directory', + confLabel: 'Configuration File', + dockerLabel: 'Container', + taskLabel: 'Scheduled Task', + resourceLabel: 'Application Resource Directory', + runtimeLabel: 'Runtime Environment', + appLabel: 'Application', + databaseLabel: 'Database', + snapshotLabel: 'Snapshot File', + websiteLabel: 'Website', + directoryLabel: 'Directory', + appStoreLabel: 'Application Store', + shellLabel: 'Script', + tmpLabel: 'Temporary Directory', + sslLabel: 'Certificate Directory', + reCreate: 'Failed to create snapshot', + reRollback: 'Rollback snapshot failed', deleteHelper: 'All backup files for the snapshot, including those in the third-party backup account, will be deleted.', status: 'Snapshot status', diff --git a/frontend/src/lang/modules/tw.ts b/frontend/src/lang/modules/tw.ts index ef3cfededcb0..83ccae5b6b44 100644 --- a/frontend/src/lang/modules/tw.ts +++ b/frontend/src/lang/modules/tw.ts @@ -11,6 +11,8 @@ const message = { false: '否', example: '例：', button: { + prev: '上一步', + next: '下一步', create: '創建', add: '添加', save: '保存', @@ -1405,6 +1407,38 @@ const message = { backupJump: '未在當前備份列表中的備份檔案，請嘗試從檔案目錄中下載後導入備份。', snapshot: '快照', + stepBaseData: '基礎數據', + stepAppData: '系統應用', + stepPanelData: '系統數據', + stepBackupData: '備份數據', + stepOtherData: '其他數據', + operationLog: '保留操作日誌', + loginLog: '保留訪問日誌', + systemLog: '保留系統日誌', + taskLog: '保留任務日誌', + monitorData: '保留監控數據', + selectAllImage: '備份所有應用鏡像', + agentLabel: '節點配置', + appDataLabel: '應用數據', + appImage: '應用鏡像', + appBackup: '應用備份', + backupLabel: '備份目錄', + confLabel: '配置文件', + dockerLabel: '容器', + taskLabel: '計劃任務', + resourceLabel: '應用資源目錄', + runtimeLabel: '運行環境', + appLabel: '應用', + databaseLabel: '數據庫', + snapshotLabel: '快照文件', + websiteLabel: '網站', + directoryLabel: '目錄', + appStoreLabel: '應用商店', + shellLabel: '腳本', + tmpLabel: '臨時目錄', + sslLabel: '證書目錄', + reCreate: '创建快照失败', + reRollback: '回滾快照失敗', deleteHelper: '將刪除該快照的所有備份文件，包括第三方備份賬號中的文件。', status: '快照狀態', ignoreRule: '排除規則', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index ce76bd94228b..11f5115a7cd6 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -11,6 +11,8 @@ const message = { false: '否', example: '例：', button: { + prev: '上一步', + next: '下一步', create: '创建', add: '添加', save: '保存', @@ -1407,6 +1409,34 @@ const message = { backupJump: '未在当前备份列表中的备份文件，请尝试从文件目录中下载后导入备份。', snapshot: '快照', + stepBaseData: '基础数据', + stepAppData: '系统应用', + stepPanelData: '系统数据', + stepBackupData: '备份数据', + stepOtherData: '其他数据', + monitorData: '监控数据', + selectAllImage: '备份所有应用镜像', + agentLabel: '节点配置', + appDataLabel: '应用数据', + appImage: '应用镜像', + appBackup: '应用备份', + backupLabel: '备份目录', + confLabel: '配置文件', + dockerLabel: '容器', + taskLabel: '计划任务', + resourceLabel: '应用资源目录', + runtimeLabel: '运行环境', + appLabel: '应用', + databaseLabel: '数据库', + snapshotLabel: '快照文件', + websiteLabel: '网站', + directoryLabel: '目录', + appStoreLabel: '应用商店', + shellLabel: '脚本', + tmpLabel: '临时目录', + sslLabel: '证书目录', + reCreate: '创建快照失败', + reRollback: '回滚快照失败', deleteHelper: '将删除该快照的所有备份文件，包括第三方备份账号中的文件。', ignoreRule: '排除规则', ignoreHelper: '快照时将使用该规则对 1Panel 数据目录进行压缩备份，请谨慎修改。', @@ -1421,13 +1451,14 @@ const message = { compress: '制作快照文件', upload: '上传快照文件', recoverDetail: '恢复详情', + recoverFailed: '快照恢复失败', createSnapshot: '创建快照', importSnapshot: '同步快照', importHelper: '快照文件目录：', recover: '恢复', lastRecoverAt: '上次恢复时间', lastRollbackAt: '上次回滚时间', - reDownload: '重新下载备份文件', + reDownload: '重新下载', statusSuccess: '成功', statusFailed: '失败', recoverErrArch: '不支持在不同服务器架构之间进行快照恢复操作!', diff --git a/frontend/src/views/log/task/index.vue b/frontend/src/views/log/task/index.vue index a0125ee0e48b..cae7031bbb1e 100644 --- a/frontend/src/views/log/task/index.vue +++ b/frontend/src/views/log/task/index.vue @@ -55,7 +55,7 @@ - + diff --git a/frontend/src/views/setting/snapshot/create/index.vue b/frontend/src/views/setting/snapshot/create/index.vue new file mode 100644 index 000000000000..47b716270a09 --- /dev/null +++ b/frontend/src/views/setting/snapshot/create/index.vue @@ -0,0 +1,544 @@ + + + + diff --git a/frontend/src/views/setting/snapshot/ignore-rule/index.vue b/frontend/src/views/setting/snapshot/ignore-rule/index.vue index 58d4e65e3fa9..e421aadd517e 100644 --- a/frontend/src/views/setting/snapshot/ignore-rule/index.vue +++ b/frontend/src/views/setting/snapshot/ignore-rule/index.vue @@ -44,7 +44,7 @@ import i18n from '@/lang'; import { MsgSuccess } from '@/utils/message'; import FileList from '@/components/file-list/index.vue'; import { FormInstance } from 'element-plus'; -import { getSettingInfo, loadBaseDir, updateSetting } from '@/api/modules/setting'; +import { getAgentSettingInfo, loadBaseDir, updateAgentSetting } from '@/api/modules/setting'; const loading = ref(); const baseDir = ref(); @@ -78,7 +78,7 @@ function checkData(rule: any, value: any, callback: any) { const acceptParams = async (): Promise => { loadPath(); - const res = await getSettingInfo(); + const res = await getAgentSettingInfo(); tableList.value = []; let items = res.data.snapshotIgnore.split(','); for (const item of items) { @@ -118,7 +118,7 @@ const onSave = async () => { for (const item of tableList.value) { list.push(item.value); } - await updateSetting({ key: 'SnapshotIgnore', value: list.join(',') }) + await updateAgentSetting({ key: 'SnapshotIgnore', value: list.join(',') }) .then(async () => { MsgSuccess(i18n.global.t('commons.msg.operationSuccess')); loading.value = false; diff --git a/frontend/src/views/setting/snapshot/import/index.vue b/frontend/src/views/setting/snapshot/import/index.vue index a31ff8941085..7cdd18cc4d53 100644 --- a/frontend/src/views/setting/snapshot/import/index.vue +++ b/frontend/src/views/setting/snapshot/import/index.vue @@ -2,15 +2,10 @@ - - + + -
+
{{ $t('setting.importHelper') }} {{ backupPath }}
@@ -61,13 +56,13 @@ const existNames = ref(); const backupPath = ref(''); const form = reactive({ - from: '', + backupAccountID: 0, names: [], description: '', }); const rules = reactive({ - from: [Rules.requiredSelect], + backupAccountID: [Rules.requiredSelect], names: [Rules.requiredSelect], }); @@ -76,7 +71,7 @@ interface DialogProps { } const acceptParams = (params: DialogProps): void => { - form.from = ''; + form.backupAccountID = undefined; existNames.value = params.names; form.names = [] as Array; loadBackups(); @@ -125,7 +120,11 @@ const loadBackups = async () => { loading.value = false; backupOptions.value = []; for (const item of res.data) { - backupOptions.value.push({ label: i18n.global.t('setting.' + item.type), value: item.type }); + backupOptions.value.push({ + id: item.id, + label: i18n.global.t('setting.' + item.type), + value: item.type, + }); } }) .catch(() => { @@ -135,7 +134,7 @@ const loadBackups = async () => { const loadFiles = async () => { form.names = []; - const res = await getFilesFromBackup(form.from); + const res = await getFilesFromBackup(form.backupAccountID); fileNames.value = res.data || []; }; diff --git a/frontend/src/views/setting/snapshot/index.vue b/frontend/src/views/setting/snapshot/index.vue index 42799fd97da4..329c15e2a4c5 100644 --- a/frontend/src/views/setting/snapshot/index.vue +++ b/frontend/src/views/setting/snapshot/index.vue @@ -77,20 +77,49 @@ @@ -105,7 +134,7 @@ show-overflow-tooltip /> + - - - - - - - - - - - - - - - - - - - - - + - - + +
diff --git a/frontend/src/views/setting/snapshot/recover/index.vue b/frontend/src/views/setting/snapshot/recover/index.vue index 0780edfea2e6..0b84e6c629ce 100644 --- a/frontend/src/views/setting/snapshot/recover/index.vue +++ b/frontend/src/views/setting/snapshot/recover/index.vue @@ -7,8 +7,8 @@ :before-close="handleClose" > - {{ $t('setting.recoverHelper', [recoverReq.name]) }} -
+
+ {{ $t('setting.recoverHelper', [recoverReq.name]) }}
{{ $t('setting.recoverHelper1') }} @@ -32,10 +32,13 @@ {{ $t('setting.recoverHelper3', [recoverReq.arch]) }}
- + + {{ recoverReq.message }} + + {{ $t('setting.reDownload') }} - + @@ -44,12 +47,19 @@ {{ $t('commons.button.cancel') }} - + + {{ $t('setting.rollback') }} + + + {{ $t('commons.button.retry') }} + + {{ $t('commons.button.confirm') }} + -