From bc74e7aa4364fd2ecad344b439bb3936e0221a0e Mon Sep 17 00:00:00 2001 From: ssonglius11 Date: Fri, 13 Sep 2024 16:09:19 +0800 Subject: [PATCH 01/12] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E5=BF=AB?= =?UTF-8?q?=E7=85=A7=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/app/api/v2/snapshot.go | 15 + agent/app/dto/setting.go | 60 --- agent/app/dto/snapshot.go | 90 ++++ agent/app/model/snapshot.go | 16 +- agent/app/service/snapshot.go | 371 +++++++++----- agent/app/service/snapshot_create.go | 333 +++++++----- agent/app/service/snapshot_recover.go | 282 +++++----- agent/init/hook/hook.go | 24 +- agent/init/migration/migrate.go | 7 + agent/init/migration/migrations/init.go | 56 ++ agent/router/ro_setting.go | 1 + agent/utils/files/file_op.go | 95 ++++ agent/utils/files/tar_gz.go | 55 ++ frontend/src/api/interface/setting.ts | 41 +- frontend/src/api/modules/setting.ts | 3 + frontend/src/lang/modules/en.ts | 30 ++ frontend/src/lang/modules/tw.ts | 30 ++ frontend/src/lang/modules/zh.ts | 30 ++ .../views/setting/snapshot/create/index.vue | 485 ++++++++++++++++++ frontend/src/views/setting/snapshot/index.vue | 148 +----- .../views/setting/snapshot/status/index.vue | 441 +++++++++------- 21 files changed, 1806 insertions(+), 807 deletions(-) create mode 100644 agent/app/dto/snapshot.go create mode 100644 frontend/src/views/setting/snapshot/create/index.vue diff --git a/agent/app/api/v2/snapshot.go b/agent/app/api/v2/snapshot.go index fffff3e13318..88f8d4543208 100644 --- a/agent/app/api/v2/snapshot.go +++ b/agent/app/api/v2/snapshot.go @@ -7,6 +7,21 @@ import ( "github.com/gin-gonic/gin" ) +// @Tags System Setting +// @Summary Load system snapshot data +// @Description 获取系统快照数据 +// @Success 200 +// @Security ApiKeyAuth +// @Router /settings/snapshot/load [get] +func (b *BaseApi) LoadSnapshotData(c *gin.Context) { + data, err := snapshotService.LoadSnapshotData() + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) + return + } + helper.SuccessWithData(c, data) +} + // @Tags System Setting // @Summary Create system snapshot // @Description 创建系统快照 diff --git a/agent/app/dto/setting.go b/agent/app/dto/setting.go index a7ac6d679d68..d9d015f3ddef 100644 --- a/agent/app/dto/setting.go +++ b/agent/app/dto/setting.go @@ -1,7 +1,5 @@ package dto -import "time" - type SettingInfo struct { SystemIP string `json:"systemIP"` DockerSockPath string `json:"dockerSockPath"` @@ -34,64 +32,6 @@ type SettingUpdate struct { Value string `json:"value"` } -type SnapshotStatus struct { - Panel string `json:"panel"` - PanelInfo string `json:"panelInfo"` - DaemonJson string `json:"daemonJson"` - AppData string `json:"appData"` - PanelData string `json:"panelData"` - BackupData string `json:"backupData"` - - Compress string `json:"compress"` - Size string `json:"size"` - Upload string `json:"upload"` -} - -type SnapshotCreate struct { - ID uint `json:"id"` - SourceAccountIDs string `json:"sourceAccountIDs" validate:"required"` - DownloadAccountID uint `json:"downloadAccountID" validate:"required"` - Description string `json:"description" validate:"max=256"` - Secret string `json:"secret"` -} -type SnapshotRecover struct { - IsNew bool `json:"isNew"` - ReDownload bool `json:"reDownload"` - ID uint `json:"id" validate:"required"` - Secret string `json:"secret"` -} -type SnapshotBatchDelete struct { - DeleteWithFile bool `json:"deleteWithFile"` - Ids []uint `json:"ids" validate:"required"` -} - -type SnapshotImport struct { - BackupAccountID uint `json:"backupAccountID"` - Names []string `json:"names"` - Description string `json:"description" validate:"max=256"` -} - -type SnapshotInfo struct { - ID uint `json:"id"` - Name string `json:"name"` - Description string `json:"description" validate:"max=256"` - From string `json:"from"` - DefaultDownload string `json:"defaultDownload"` - Status string `json:"status"` - Message string `json:"message"` - CreatedAt time.Time `json:"createdAt"` - Version string `json:"version"` - Size int64 `json:"size"` - - InterruptStep string `json:"interruptStep"` - RecoverStatus string `json:"recoverStatus"` - RecoverMessage string `json:"recoverMessage"` - LastRecoveredAt string `json:"lastRecoveredAt"` - RollbackStatus string `json:"rollbackStatus"` - RollbackMessage string `json:"rollbackMessage"` - LastRollbackedAt string `json:"lastRollbackedAt"` -} - type SyncTime struct { NtpSite string `json:"ntpSite" validate:"required"` } diff --git a/agent/app/dto/snapshot.go b/agent/app/dto/snapshot.go new file mode 100644 index 000000000000..9504cfc963ee --- /dev/null +++ b/agent/app/dto/snapshot.go @@ -0,0 +1,90 @@ +package dto + +import "time" + +type SnapshotStatus struct { + BaseData string `json:"baseData"` + AppImage string `json:"appImage"` + PanelData string `json:"panelData"` + BackupData string `json:"backupData"` + + Compress string `json:"compress"` + Size string `json:"size"` + Upload string `json:"upload"` +} + +type SnapshotCreate struct { + ID uint `json:"id"` + SourceAccountIDs string `json:"sourceAccountIDs" validate:"required"` + DownloadAccountID uint `json:"downloadAccountID" validate:"required"` + Description string `json:"description" validate:"max=256"` + Secret string `json:"secret"` + + AppData []DataTree `json:"appData"` + BackupData []DataTree `json:"backupData"` + PanelData []DataTree `json:"panelData"` + + WithMonitorData bool `json:"withMonitorData"` + WithLoginLog bool `json:"withLoginLog"` + WithOperationLog bool `json:"withOperationLog"` +} + +type SnapshotData struct { + AppData []DataTree `json:"appData"` + BackupData []DataTree `json:"backupData"` + PanelData []DataTree `json:"panelData"` + + WithMonitorData bool `json:"withMonitorData"` + WithLoginLog bool `json:"withLoginLog"` + WithOperationLog bool `json:"withOperationLog"` +} +type DataTree struct { + ID string `json:"id"` + Label string `json:"label"` + Key string `json:"key"` + Name string `json:"name"` + Size uint64 `json:"size"` + IsCheck bool `json:"isCheck"` + IsDisable bool `json:"isDisable"` + + Path string `json:"path"` + + Children []DataTree `json:"children"` +} +type SnapshotRecover struct { + IsNew bool `json:"isNew"` + ReDownload bool `json:"reDownload"` + ID uint `json:"id" validate:"required"` + Secret string `json:"secret"` +} +type SnapshotBatchDelete struct { + DeleteWithFile bool `json:"deleteWithFile"` + Ids []uint `json:"ids" validate:"required"` +} + +type SnapshotImport struct { + BackupAccountID uint `json:"backupAccountID"` + Names []string `json:"names"` + Description string `json:"description" validate:"max=256"` +} + +type SnapshotInfo struct { + ID uint `json:"id"` + Name string `json:"name"` + Description string `json:"description" validate:"max=256"` + From string `json:"from"` + DefaultDownload string `json:"defaultDownload"` + Status string `json:"status"` + Message string `json:"message"` + CreatedAt time.Time `json:"createdAt"` + Version string `json:"version"` + Size int64 `json:"size"` + + InterruptStep string `json:"interruptStep"` + RecoverStatus string `json:"recoverStatus"` + RecoverMessage string `json:"recoverMessage"` + LastRecoveredAt string `json:"lastRecoveredAt"` + RollbackStatus string `json:"rollbackStatus"` + RollbackMessage string `json:"rollbackMessage"` + LastRollbackedAt string `json:"lastRollbackedAt"` +} diff --git a/agent/app/model/snapshot.go b/agent/app/model/snapshot.go index 1771051122b5..bf8cc3b43718 100644 --- a/agent/app/model/snapshot.go +++ b/agent/app/model/snapshot.go @@ -10,6 +10,13 @@ type Snapshot struct { Message string `json:"message"` Version string `json:"version"` + AppData string `json:"appData"` + PanelData string `json:"panelData"` + BackupData string `json:"backupData"` + WithMonitorData bool `json:"withMonitorData"` + WithLoginLog bool `json:"withLoginLog"` + WithOperationLog bool `json:"withOperationLog"` + InterruptStep string `json:"interruptStep"` RecoverStatus string `json:"recoverStatus"` RecoverMessage string `json:"recoverMessage"` @@ -21,11 +28,10 @@ type Snapshot struct { type SnapshotStatus struct { BaseModel - SnapID uint `json:"snapID"` - Panel string `json:"panel" gorm:"default:Running"` - PanelInfo string `json:"panelInfo" gorm:"default:Running"` - DaemonJson string `json:"daemonJson" gorm:"default:Running"` - AppData string `json:"appData" gorm:"default:Running"` + SnapID uint `json:"snapID"` + + BaseData string `json:"baseData" gorm:"default:Running"` + AppImage string `json:"appImage" gorm:"default:Running"` PanelData string `json:"panelData" gorm:"default:Running"` BackupData string `json:"backupData" gorm:"default:Running"` diff --git a/agent/app/service/snapshot.go b/agent/app/service/snapshot.go index b398efe1cf6a..2b4a5efecb00 100644 --- a/agent/app/service/snapshot.go +++ b/agent/app/service/snapshot.go @@ -16,7 +16,11 @@ import ( "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/utils/cmd" "github.com/1Panel-dev/1Panel/agent/utils/compose" + "github.com/1Panel-dev/1Panel/agent/utils/docker" "github.com/1Panel-dev/1Panel/agent/utils/files" + fileUtils "github.com/1Panel-dev/1Panel/agent/utils/files" + "github.com/docker/docker/api/types/image" + "github.com/google/uuid" "github.com/jinzhu/copier" "github.com/pkg/errors" "github.com/shirou/gopsutil/v3/host" @@ -28,6 +32,7 @@ type SnapshotService struct { type ISnapshotService interface { SearchWithPage(req dto.SearchWithPage) (int64, interface{}, error) + LoadSnapshotData() (dto.SnapshotData, error) SnapshotCreate(req dto.SnapshotCreate) error SnapshotRecover(req dto.SnapshotRecover) error SnapshotRollback(req dto.SnapshotRecover) error @@ -92,6 +97,37 @@ func (u *SnapshotService) SnapshotImport(req dto.SnapshotImport) error { return nil } +func (u *SnapshotService) LoadSnapshotData() (dto.SnapshotData, error) { + var ( + data dto.SnapshotData + err error + ) + fileOp := fileUtils.NewFileOp() + data.AppData, err = loadApps(fileOp) + if err != nil { + return data, err + } + data.PanelData, err = loadPanelFile(fileOp) + if err != nil { + return data, err + } + itemBackups, err := loadFile(global.CONF.System.Backup, 8, fileOp) + if err != nil { + return data, err + } + for i, item := range itemBackups { + if item.Label == "app" { + data.BackupData = append(itemBackups[:i], itemBackups[i+1:]...) + break + } + } + data.WithLoginLog = true + data.WithOperationLog = true + data.WithMonitorData = false + + return data, nil +} + func (u *SnapshotService) UpdateDescription(req dto.UpdateDescription) error { return snapshotRepo.Update(req.ID, map[string]interface{}{"description": req.Description}) } @@ -109,16 +145,9 @@ func (u *SnapshotService) LoadSnapShotStatus(id uint) (*dto.SnapshotStatus, erro } type SnapshotJson struct { - OldBaseDir string `json:"oldBaseDir"` - OldDockerDataDir string `json:"oldDockerDataDir"` - OldBackupDataDir string `json:"oldBackupDataDir"` - OldPanelDataDir string `json:"oldPanelDataDir"` - - BaseDir string `json:"baseDir"` - DockerDataDir string `json:"dockerDataDir"` - BackupDataDir string `json:"backupDataDir"` - PanelDataDir string `json:"panelDataDir"` - LiveRestoreEnabled bool `json:"liveRestoreEnabled"` + BaseDir string `json:"baseDir"` + BackupDataDir string `json:"backupDataDir"` + Size uint64 `json:"size"` } func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate) error { @@ -148,7 +177,7 @@ func (u *SnapshotService) SnapshotRecover(req dto.SnapshotRecover) error { _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"recover_status": constant.StatusWaiting}) _ = settingRepo.Update("SystemStatus", "Recovering") - go u.HandleSnapshotRecover(snap, true, req) + go u.HandleSnapshotRecover(snap, req) return nil } @@ -158,9 +187,11 @@ func (u *SnapshotService) SnapshotRollback(req dto.SnapshotRecover) error { if err != nil { return err } - req.IsNew = false - snap.InterruptStep = "Readjson" - go u.HandleSnapshotRecover(snap, false, req) + go func() { + if err := handleRollback(snap.Name); err != nil { + global.LOG.Errorf("handle roll back snapshot failed, err: %v", err) + } + }() return nil } @@ -194,15 +225,26 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, logPath string, req dto if isCronjob { name = fmt.Sprintf("snapshot_1panel_%s_%s_%s", versionItem.Value, loadOs(), timeNow) } - rootDir = path.Join(global.CONF.System.Backup, "system", name) + rootDir = path.Join(global.CONF.System.BaseDir, "1panel/tmp/system", name) + appItem, _ := json.Marshal(req.AppData) + panelItem, _ := json.Marshal(req.PanelData) + backupItem, _ := json.Marshal(req.BackupData) snap = model.Snapshot{ Name: name, Description: req.Description, SourceAccountIDs: req.SourceAccountIDs, DownloadAccountID: req.DownloadAccountID, - Version: versionItem.Value, - Status: constant.StatusWaiting, + + AppData: string(appItem), + PanelData: string(panelItem), + BackupData: string(backupItem), + WithMonitorData: req.WithMonitorData, + WithLoginLog: req.WithLoginLog, + WithOperationLog: req.WithOperationLog, + + Version: versionItem.Value, + Status: constant.StatusWaiting, } _ = snapshotRepo.Create(&snap) snapStatus.SnapID = snap.ID @@ -218,57 +260,43 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, logPath string, req dto snapStatus.SnapID = snap.ID _ = snapshotRepo.CreateStatus(&snapStatus) } - rootDir = path.Join(global.CONF.System.Backup, fmt.Sprintf("system/%s", snap.Name)) + rootDir = path.Join(global.CONF.System.BaseDir, "1panel/tmp/system", snap.Name) } var wg sync.WaitGroup itemHelper := snapHelper{SnapID: snap.ID, Status: &snapStatus, Wg: &wg, FileOp: files.NewFileOp(), Ctx: context.Background()} - backupPanelDir := path.Join(rootDir, "1panel") - _ = os.MkdirAll(backupPanelDir, os.ModePerm) - backupDockerDir := path.Join(rootDir, "docker") - _ = os.MkdirAll(backupDockerDir, os.ModePerm) - - jsonItem := SnapshotJson{ - BaseDir: global.CONF.System.BaseDir, - BackupDataDir: global.CONF.System.Backup, - PanelDataDir: path.Join(global.CONF.System.BaseDir, "1panel"), + baseDir := path.Join(rootDir, "base") + _ = os.MkdirAll(baseDir, os.ModePerm) + if err := loadDbConn(&itemHelper, rootDir, req); err != nil { + return "", fmt.Errorf("load snapshot db conn failed, err: %v", err) } + loadLogByStatus(snapStatus, logPath) - if snapStatus.PanelInfo != constant.StatusDone { + if snapStatus.BaseData != constant.StatusDone { wg.Add(1) - go snapJson(itemHelper, jsonItem, rootDir) + go snapBaseData(itemHelper, baseDir) } - if snapStatus.Panel != constant.StatusDone { + if snapStatus.AppImage != constant.StatusDone { wg.Add(1) - go snapPanel(itemHelper, backupPanelDir) + go snapAppImage(itemHelper, req, rootDir) } - if snapStatus.DaemonJson != constant.StatusDone { - wg.Add(1) - go snapDaemonJson(itemHelper, backupDockerDir) - } - if snapStatus.AppData != constant.StatusDone { + if snapStatus.BackupData != constant.StatusDone { wg.Add(1) - go snapAppData(itemHelper, backupDockerDir) + go snapBackupData(itemHelper, req, rootDir) } - if snapStatus.BackupData != constant.StatusDone { + if snapStatus.PanelData != constant.StatusDone { wg.Add(1) - go snapBackup(itemHelper, backupPanelDir) + go snapPanelData(itemHelper, req, rootDir) } - if !isCronjob { go func() { wg.Wait() + closeDatabase(itemHelper.snapAgentDB) + closeDatabase(itemHelper.snapCoreDB) if !checkIsAllDone(snap.ID) { _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) return } - if snapStatus.PanelData != constant.StatusDone { - snapPanelData(itemHelper, backupPanelDir) - } - if snapStatus.PanelData != constant.StatusDone { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - return - } if snapStatus.Compress != constant.StatusDone { snapCompress(itemHelper, rootDir, secret) } @@ -284,23 +312,20 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, logPath string, req dto return } _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess}) + // _ = snapshotRepo.DeleteStatus(itemHelper.SnapID) + _ = os.RemoveAll(rootDir) }() return "", nil } wg.Wait() + closeDatabase(itemHelper.snapAgentDB) + closeDatabase(itemHelper.snapCoreDB) if !checkIsAllDone(snap.ID) { _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) loadLogByStatus(snapStatus, logPath) return snap.Name, fmt.Errorf("snapshot %s backup failed", snap.Name) } loadLogByStatus(snapStatus, logPath) - snapPanelData(itemHelper, backupPanelDir) - if snapStatus.PanelData != constant.StatusDone { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - loadLogByStatus(snapStatus, logPath) - return snap.Name, fmt.Errorf("snapshot %s 1panel data failed", snap.Name) - } - loadLogByStatus(snapStatus, logPath) snapCompress(itemHelper, rootDir, secret) if snapStatus.Compress != constant.StatusDone { _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) @@ -316,6 +341,7 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, logPath string, req dto } _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess}) loadLogByStatus(snapStatus, logPath) + _ = os.RemoveAll(rootDir) return snap.Name, nil } @@ -341,71 +367,6 @@ func (u *SnapshotService) Delete(req dto.SnapshotBatchDelete) error { return nil } -func updateRecoverStatus(id uint, isRecover bool, interruptStep, status, message string) { - if isRecover { - if status != constant.StatusSuccess { - global.LOG.Errorf("recover failed, err: %s", message) - } - if err := snapshotRepo.Update(id, map[string]interface{}{ - "interrupt_step": interruptStep, - "recover_status": status, - "recover_message": message, - "last_recovered_at": time.Now().Format(constant.DateTimeLayout), - }); err != nil { - global.LOG.Errorf("update snap recover status failed, err: %v", err) - } - _ = settingRepo.Update("SystemStatus", "Free") - return - } - _ = settingRepo.Update("SystemStatus", "Free") - if status == constant.StatusSuccess { - if err := snapshotRepo.Update(id, map[string]interface{}{ - "recover_status": "", - "recover_message": "", - "interrupt_step": "", - "rollback_status": "", - "rollback_message": "", - "last_rollbacked_at": time.Now().Format(constant.DateTimeLayout), - }); err != nil { - global.LOG.Errorf("update snap recover status failed, err: %v", err) - } - return - } - global.LOG.Errorf("rollback failed, err: %s", message) - if err := snapshotRepo.Update(id, map[string]interface{}{ - "rollback_status": status, - "rollback_message": message, - "last_rollbacked_at": time.Now().Format(constant.DateTimeLayout), - }); err != nil { - global.LOG.Errorf("update snap recover status failed, err: %v", err) - } -} - -func (u *SnapshotService) handleUnTar(sourceDir, targetDir string, secret string) error { - if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) { - if err = os.MkdirAll(targetDir, os.ModePerm); err != nil { - return err - } - } - commands := "" - if len(secret) != 0 { - extraCmd := "openssl enc -d -aes-256-cbc -k '" + secret + "' -in " + sourceDir + " | " - commands = fmt.Sprintf("%s tar -zxvf - -C %s", extraCmd, targetDir+" > /dev/null 2>&1") - global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) - } else { - commands = fmt.Sprintf("tar zxvfC %s %s", sourceDir, targetDir) - global.LOG.Debug(commands) - } - stdout, err := cmd.ExecWithTimeOut(commands, 30*time.Minute) - if err != nil { - if len(stdout) != 0 { - global.LOG.Errorf("do handle untar failed, stdout: %s, err: %v", stdout, err) - return fmt.Errorf("do handle untar failed, stdout: %s, err: %v", stdout, err) - } - } - return nil -} - func rebuildAllAppInstall() error { global.LOG.Debug("start to rebuild all app") appInstalls, err := appInstallRepo.ListBy() @@ -449,17 +410,14 @@ func checkIsAllDone(snapID uint) bool { } func checkAllDone(status model.SnapshotStatus) (bool, string) { - if status.Panel != constant.StatusDone { - return false, status.Panel + if status.BaseData != constant.StatusDone { + return false, status.BaseData } - if status.PanelInfo != constant.StatusDone { - return false, status.PanelInfo + if status.PanelData != constant.StatusDone { + return false, status.PanelData } - if status.DaemonJson != constant.StatusDone { - return false, status.DaemonJson - } - if status.AppData != constant.StatusDone { - return false, status.AppData + if status.AppImage != constant.StatusDone { + return false, status.AppImage } if status.BackupData != constant.StatusDone { return false, status.BackupData @@ -469,10 +427,8 @@ func checkAllDone(status model.SnapshotStatus) (bool, string) { func loadLogByStatus(status model.SnapshotStatus, logPath string) { logs := "" - logs += fmt.Sprintf("Write 1Panel basic information: %s \n", status.PanelInfo) - logs += fmt.Sprintf("Backup 1Panel system files: %s \n", status.Panel) - logs += fmt.Sprintf("Backup Docker configuration file: %s \n", status.DaemonJson) - logs += fmt.Sprintf("Backup installed apps from 1Panel: %s \n", status.AppData) + logs += fmt.Sprintf("Backup 1Panel base files: %s \n", status.BaseData) + logs += fmt.Sprintf("Backup installed apps from 1Panel: %s \n", status.AppImage) logs += fmt.Sprintf("Backup 1Panel data directory: %s \n", status.PanelData) logs += fmt.Sprintf("Backup local backup directory for 1Panel: %s \n", status.BackupData) logs += fmt.Sprintf("Create snapshot file: %s \n", status.Compress) @@ -544,3 +500,154 @@ func loadSnapSize(records []model.Snapshot) ([]dto.SnapshotInfo, error) { wg.Wait() return datas, nil } + +func loadApps(fileOp fileUtils.FileOp) ([]dto.DataTree, error) { + var data []dto.DataTree + apps, err := appInstallRepo.ListBy() + if err != nil { + return data, err + } + client, err := docker.NewDockerClient() + hasDockerClient := true + if err != nil { + hasDockerClient = false + global.LOG.Errorf("new docker client failed, err: %v", err) + } else { + defer client.Close() + } + imageList, err := client.ImageList(context.Background(), image.ListOptions{}) + if err != nil { + hasDockerClient = false + global.LOG.Errorf("load image list failed, err: %v", err) + } + + for _, app := range apps { + itemApp := dto.DataTree{ID: uuid.NewString(), Label: fmt.Sprintf("%s - %s", app.App.Name, app.Name), Key: app.App.Key, Name: app.Name} + appPath := path.Join(global.CONF.System.BaseDir, "1panel/apps", app.App.Key, app.Name) + itemAppData := dto.DataTree{ID: uuid.NewString(), Label: "appData", Key: app.App.Key, Name: app.Name, IsCheck: true, Path: appPath} + sizeItem, err := fileOp.GetDirSize(appPath) + if err == nil { + itemAppData.Size = uint64(sizeItem) + } + itemApp.Size += itemAppData.Size + itemApp.Children = append(itemApp.Children, itemAppData) + + appBackupPath := path.Join(global.CONF.System.BaseDir, "1panel/backup/app", app.App.Key, app.Name) + itemAppBackupTree, err := loadFile(appBackupPath, 8, fileOp) + itemAppBackup := dto.DataTree{ID: uuid.NewString(), Label: "appBackup", IsCheck: true, Children: itemAppBackupTree, Path: appBackupPath} + if err == nil { + backupSizeItem, err := fileOp.GetDirSize(appBackupPath) + if err == nil { + itemAppBackup.Size = uint64(backupSizeItem) + itemApp.Size += itemAppBackup.Size + } + itemApp.Children = append(itemApp.Children, itemAppBackup) + } + + itemAppImage := dto.DataTree{ID: uuid.NewString(), Label: "appImage"} + stdout, err := cmd.Execf("cat %s | grep image: ", path.Join(global.CONF.System.BaseDir, "1panel/apps", app.App.Key, app.Name, "docker-compose.yml")) + if err != nil { + itemApp.Children = append(itemApp.Children, itemAppImage) + data = append(data, itemApp) + continue + } + itemAppImage.Name = strings.ReplaceAll(strings.ReplaceAll(strings.TrimSpace(stdout), "\n", ""), "image: ", "") + if !hasDockerClient { + itemApp.Children = append(itemApp.Children, itemAppImage) + data = append(data, itemApp) + continue + } + for _, imageItem := range imageList { + for _, tag := range imageItem.RepoTags { + if tag == itemAppImage.Name { + itemAppImage.Size = uint64(imageItem.Size) + break + } + } + } + itemApp.Children = append(itemApp.Children, itemAppImage) + data = append(data, itemApp) + } + return data, nil +} + +func loadPanelFile(fileOp fileUtils.FileOp) ([]dto.DataTree, error) { + var data []dto.DataTree + snapFiles, err := os.ReadDir(path.Join(global.CONF.System.BaseDir, "1panel")) + if err != nil { + return data, err + } + for _, fileItem := range snapFiles { + itemData := dto.DataTree{ + ID: uuid.NewString(), + Label: fileItem.Name(), + IsCheck: true, + Path: path.Join(global.CONF.System.BaseDir, "1panel", fileItem.Name()), + } + switch itemData.Label { + case "agent", "conf", "db", "runtime", "secret": + itemData.IsDisable = true + case "log", "docker", "task", "clamav": + panelPath := path.Join(global.CONF.System.BaseDir, "1panel", itemData.Label) + itemData.Children, _ = loadFile(panelPath, 5, fileOp) + default: + continue + } + if fileItem.IsDir() { + sizeItem, err := fileOp.GetDirSize(path.Join(global.CONF.System.BaseDir, "1panel", itemData.Label)) + if err != nil { + continue + } + itemData.Size = uint64(sizeItem) + } else { + fileInfo, err := fileItem.Info() + if err != nil { + continue + } + itemData.Size = uint64(fileInfo.Size()) + } + if itemData.IsCheck && itemData.Size == 0 { + itemData.IsCheck = false + itemData.IsDisable = true + } + + data = append(data, itemData) + } + + return data, nil +} + +func loadFile(pathItem string, index int, fileOp fileUtils.FileOp) ([]dto.DataTree, error) { + var data []dto.DataTree + snapFiles, err := os.ReadDir(pathItem) + if err != nil { + return data, err + } + i := 0 + for _, fileItem := range snapFiles { + itemData := dto.DataTree{ + ID: uuid.NewString(), + Label: fileItem.Name(), + Name: fileItem.Name(), + Path: path.Join(pathItem, fileItem.Name()), + IsCheck: true, + } + if fileItem.IsDir() { + sizeItem, err := fileOp.GetDirSize(path.Join(pathItem, itemData.Label)) + if err != nil { + continue + } + itemData.Size = uint64(sizeItem) + itemData.Children, _ = loadFile(path.Join(pathItem, itemData.Label), index-1, fileOp) + } else { + fileInfo, err := fileItem.Info() + if err != nil { + continue + } + itemData.Size = uint64(fileInfo.Size()) + } + data = append(data, itemData) + i++ + } + return data, nil +} diff --git a/agent/app/service/snapshot_create.go b/agent/app/service/snapshot_create.go index e4a42b6c3d9a..a93fcfee16b1 100644 --- a/agent/app/service/snapshot_create.go +++ b/agent/app/service/snapshot_create.go @@ -6,145 +6,219 @@ import ( "fmt" "os" "path" - "regexp" "strings" "sync" "time" + "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/utils/cmd" "github.com/1Panel-dev/1Panel/agent/utils/common" "github.com/1Panel-dev/1Panel/agent/utils/files" + "github.com/glebarez/sqlite" + "gorm.io/gorm" ) type snapHelper struct { - SnapID uint - Status *model.SnapshotStatus - Ctx context.Context - FileOp files.FileOp - Wg *sync.WaitGroup + SnapID uint + snapAgentDB *gorm.DB + snapCoreDB *gorm.DB + Status *model.SnapshotStatus + Ctx context.Context + FileOp files.FileOp + Wg *sync.WaitGroup } -func snapJson(snap snapHelper, snapJson SnapshotJson, targetDir string) { - defer snap.Wg.Done() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_info": constant.Running}) - status := constant.StatusDone - remarkInfo, _ := json.MarshalIndent(snapJson, "", "\t") - if err := os.WriteFile(fmt.Sprintf("%s/snapshot.json", targetDir), remarkInfo, 0640); err != nil { - status = err.Error() +func loadDbConn(snap *snapHelper, targetDir string, req dto.SnapshotCreate) error { + global.LOG.Debug("start load snapshot db conn") + + if err := snap.FileOp.CopyDir(path.Join(global.CONF.System.BaseDir, "1panel/db"), targetDir); err != nil { + return err + } + agentDb, err := newSnapDB(path.Join(targetDir, "db"), "agent.db") + if err != nil { + return err + } + snap.snapAgentDB = agentDb + coreDb, err := newSnapDB(path.Join(targetDir, "db"), "core.db") + if err != nil { + return err + } + snap.snapCoreDB = coreDb + + if !req.WithMonitorData { + _ = os.Remove(path.Join(targetDir, "db/monitor.db")) + } + if !req.WithOperationLog { + _ = snap.snapCoreDB.Exec("DELETE FROM operation_logs") + } + if !req.WithLoginLog { + _ = snap.snapCoreDB.Exec("DELETE FROM login_logs") + } + if err := snap.snapAgentDB.Where("id = ?", snap.SnapID).Delete(&model.Snapshot{}).Error; err != nil { + global.LOG.Errorf("delete current snapshot record failed, err: %v", err) + } + if err := snap.snapAgentDB.Where("snap_id = ?", snap.SnapID).Delete(&model.SnapshotStatus{}).Error; err != nil { + global.LOG.Errorf("delete current snapshot status record failed, err: %v", err) } - snap.Status.PanelInfo = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_info": status}) + + return nil } -func snapPanel(snap snapHelper, targetDir string) { +func snapBaseData(snap snapHelper, targetDir string) { defer snap.Wg.Done() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel": constant.Running}) + _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"base_data": constant.Running}) status := constant.StatusDone - if err := common.CopyFile("/usr/local/bin/1panel", path.Join(targetDir, "1panel")); err != nil { + if err := common.CopyFile("/usr/local/bin/1panel", targetDir); err != nil { + status = err.Error() + } + if err := common.CopyFile("/usr/local/bin/1panel_agent", targetDir); err != nil { status = err.Error() } - if err := common.CopyFile("/usr/local/bin/1pctl", targetDir); err != nil { status = err.Error() } - if err := common.CopyFile("/etc/systemd/system/1panel.service", targetDir); err != nil { status = err.Error() } - snap.Status.Panel = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel": status}) -} + if err := common.CopyFile("/etc/systemd/system/1panel_agent.service", targetDir); err != nil { + status = err.Error() + } -func snapDaemonJson(snap snapHelper, targetDir string) { - defer snap.Wg.Done() - status := constant.StatusDone - if !snap.FileOp.Stat("/etc/docker/daemon.json") { - snap.Status.DaemonJson = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"daemon_json": status}) - return + if snap.FileOp.Stat("/etc/docker/daemon.json") { + if err := common.CopyFile("/etc/docker/daemon.json", targetDir); err != nil { + status = err.Error() + } } - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"daemon_json": constant.Running}) - if err := common.CopyFile("/etc/docker/daemon.json", targetDir); err != nil { + + remarkInfo, _ := json.MarshalIndent(SnapshotJson{ + BaseDir: global.CONF.System.BaseDir, + BackupDataDir: global.CONF.System.Backup, + }, "", "\t") + if err := os.WriteFile(fmt.Sprintf("%s/snapshot.json", targetDir), remarkInfo, 0640); err != nil { status = err.Error() } - snap.Status.DaemonJson = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"daemon_json": status}) + snap.Status.BaseData = status + _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"base_data": status}) } -func snapAppData(snap snapHelper, targetDir string) { +func snapAppImage(snap snapHelper, req dto.SnapshotCreate, targetDir string) { defer snap.Wg.Done() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": constant.Running}) - appInstalls, err := appInstallRepo.ListBy() - if err != nil { - snap.Status.AppData = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": err.Error()}) - return - } - runtimes, err := runtimeRepo.List() - if err != nil { - snap.Status.AppData = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": err.Error()}) - return - } - imageRegex := regexp.MustCompile(`image:\s*(.*)`) - var imageSaveList []string + _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_image": constant.Running}) + + var imageList []string existStr, _ := cmd.Exec("docker images | awk '{print $1\":\"$2}' | grep -v REPOSITORY:TAG") existImages := strings.Split(existStr, "\n") - duplicateMap := make(map[string]bool) - for _, app := range appInstalls { - matches := imageRegex.FindAllStringSubmatch(app.DockerCompose, -1) - for _, match := range matches { - for _, existImage := range existImages { - if match[1] == existImage && !duplicateMap[match[1]] { - imageSaveList = append(imageSaveList, match[1]) - duplicateMap[match[1]] = true + for _, app := range req.AppData { + for _, item := range app.Children { + if item.Label == "appImage" && item.IsCheck { + for _, existImage := range existImages { + if existImage == item.Name { + imageList = append(imageList, item.Name) + } } } } } - for _, runtime := range runtimes { - for _, existImage := range existImages { - if runtime.Image == existImage && !duplicateMap[runtime.Image] { - imageSaveList = append(imageSaveList, runtime.Image) - duplicateMap[runtime.Image] = true - } - } - } - if len(imageSaveList) != 0 { - global.LOG.Debugf("docker save %s | gzip -c > %s", strings.Join(imageSaveList, " "), path.Join(targetDir, "docker_image.tar")) - std, err := cmd.Execf("docker save %s | gzip -c > %s", strings.Join(imageSaveList, " "), path.Join(targetDir, "docker_image.tar")) + if len(imageList) != 0 { + global.LOG.Debugf("docker save %s | gzip -c > %s", strings.Join(imageList, " "), path.Join(targetDir, "images.tar.gz")) + std, err := cmd.Execf("docker save %s | gzip -c > %s", strings.Join(imageList, " "), path.Join(targetDir, "images.tar.gz")) if err != nil { - snap.Status.AppData = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": std}) + snap.Status.AppImage = err.Error() + _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_image": std}) return } } - snap.Status.AppData = constant.StatusDone - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": constant.StatusDone}) + snap.Status.AppImage = constant.StatusDone + _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_image": constant.StatusDone}) } -func snapBackup(snap snapHelper, targetDir string) { +func snapBackupData(snap snapHelper, req dto.SnapshotCreate, targetDir string) { defer snap.Wg.Done() _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"backup_data": constant.Running}) status := constant.StatusDone - if err := handleSnapTar(global.CONF.System.Backup, targetDir, "1panel_backup.tar.gz", "./system;./system_snapshot;", ""); err != nil { + + excludes := loadBackupExcludes(snap, req.BackupData) + for _, item := range req.AppData { + for _, itemApp := range item.Children { + if itemApp.Label == "appBackup" { + excludes = append(excludes, loadAppBackupExcludes([]dto.DataTree{itemApp})...) + } + } + } + global.LOG.Debugf("excludes backup file: %v\n", strings.Join(excludes, "\n")) + + if err := snap.FileOp.TarGzCompressPro(false, global.CONF.System.Backup, path.Join(targetDir, "1panel_backup.tar.gz"), "", strings.Join(excludes, ";")); err != nil { status = err.Error() } snap.Status.BackupData = status _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"backup_data": status}) } +func loadBackupExcludes(snap snapHelper, req []dto.DataTree) []string { + var excludes []string + for _, item := range req { + if len(item.Children) == 0 { + if item.IsCheck { + continue + } + if strings.HasPrefix(item.Path, path.Join(global.CONF.System.Backup, "system_snapshot")) { + fmt.Println(strings.TrimSuffix(item.Name, ".tar.gz")) + if err := snap.snapAgentDB.Debug().Where("name = ? AND download_account_id = ?", strings.TrimSuffix(item.Name, ".tar.gz"), "1").Delete(&model.Snapshot{}).Error; err != nil { + global.LOG.Errorf("delete snapshot from database failed, err: %v", err) + } + } else { + fmt.Println(strings.TrimPrefix(path.Dir(item.Path), global.CONF.System.Backup+"/"), path.Base(item.Path)) + if err := snap.snapAgentDB.Debug().Where("file_dir = ? AND file_name = ?", strings.TrimPrefix(path.Dir(item.Path), global.CONF.System.Backup+"/"), path.Base(item.Path)).Delete(&model.BackupRecord{}).Error; err != nil { + global.LOG.Errorf("delete backup file from database failed, err: %v", err) + } + } + fmt.Println(strings.TrimPrefix(item.Path, global.CONF.System.Backup)) + excludes = append(excludes, "."+strings.TrimPrefix(item.Path, global.CONF.System.Backup)) + } else { + excludes = append(excludes, loadBackupExcludes(snap, item.Children)...) + } + } + return excludes +} +func loadAppBackupExcludes(req []dto.DataTree) []string { + var excludes []string + for _, item := range req { + if len(item.Children) == 0 { + if !item.IsCheck { + excludes = append(excludes, "."+strings.TrimPrefix(item.Path, path.Join(global.CONF.System.Backup))) + } + } else { + excludes = append(excludes, loadAppBackupExcludes(item.Children)...) + } + } + return excludes +} -func snapPanelData(snap snapHelper, targetDir string) { +func snapPanelData(snap snapHelper, req dto.SnapshotCreate, targetDir string) { + defer snap.Wg.Done() _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_data": constant.Running}) status := constant.StatusDone - dataDir := path.Join(global.CONF.System.BaseDir, "1panel") - exclusionRules := "./tmp;./log;./cache;./db/1Panel.db-*;" - if strings.Contains(global.CONF.System.Backup, dataDir) { - exclusionRules += ("." + strings.ReplaceAll(global.CONF.System.Backup, dataDir, "") + ";") + + excludes := loadPanelExcludes(req.PanelData) + for _, item := range req.AppData { + for _, itemApp := range item.Children { + if itemApp.Label == "appData" { + excludes = append(excludes, loadPanelExcludes([]dto.DataTree{itemApp})...) + } + } + } + global.LOG.Debugf("excludes panel file: %v\n", strings.Join(excludes, "\n")) + excludes = append(excludes, "./tmp") + excludes = append(excludes, "./cache") + excludes = append(excludes, "./uploads") + excludes = append(excludes, "./db") + excludes = append(excludes, "./resource") + rootDir := path.Join(global.CONF.System.BaseDir, "1panel") + if strings.Contains(global.CONF.System.Backup, rootDir) { + excludes = append(excludes, "."+strings.ReplaceAll(global.CONF.System.Backup, rootDir, "")) } ignoreVal, _ := settingRepo.Get(settingRepo.WithByKey("SnapshotIgnore")) rules := strings.Split(ignoreVal.Value, ",") @@ -152,27 +226,37 @@ func snapPanelData(snap snapHelper, targetDir string) { if len(ignore) == 0 || cmd.CheckIllegal(ignore) { continue } - exclusionRules += ("." + strings.ReplaceAll(ignore, dataDir, "") + ";") + excludes = append(excludes, "."+strings.ReplaceAll(ignore, rootDir, "")) } - _ = snapshotRepo.Update(snap.SnapID, map[string]interface{}{"status": "OnSaveData"}) - sysIP, _ := settingRepo.Get(settingRepo.WithByKey("SystemIP")) - _ = settingRepo.Update("SystemIP", "") - checkPointOfWal() - if err := handleSnapTar(dataDir, targetDir, "1panel_data.tar.gz", exclusionRules, ""); err != nil { + + _ = snap.snapAgentDB.Model(&model.Setting{}).Where("key = ?", "SystemIP").Updates(map[string]interface{}{"SystemIP": ""}) + + if err := snap.FileOp.TarGzCompressPro(false, rootDir, path.Join(targetDir, "1panel_data.tar.gz"), "", strings.Join(excludes, ";")); err != nil { status = err.Error() } - _ = snapshotRepo.Update(snap.SnapID, map[string]interface{}{"status": constant.StatusWaiting}) snap.Status.PanelData = status _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_data": status}) - _ = settingRepo.Update("SystemIP", sysIP.Value) +} +func loadPanelExcludes(req []dto.DataTree) []string { + var excludes []string + for _, item := range req { + if len(item.Children) == 0 { + if !item.IsCheck { + excludes = append(excludes, "."+strings.TrimPrefix(item.Path, path.Join(global.CONF.System.BaseDir, "1panel"))) + } + } else { + excludes = append(excludes, loadPanelExcludes(item.Children)...) + } + } + return excludes } func snapCompress(snap snapHelper, rootDir string, secret string) { _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": constant.StatusRunning}) tmpDir := path.Join(global.CONF.System.TmpDir, "system") fileName := fmt.Sprintf("%s.tar.gz", path.Base(rootDir)) - if err := handleSnapTar(rootDir, tmpDir, fileName, "", secret); err != nil { + if err := snap.FileOp.TarGzCompressPro(true, rootDir, path.Join(tmpDir, fileName), secret, ""); err != nil { snap.Status.Compress = err.Error() _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": err.Error()}) return @@ -207,12 +291,12 @@ func snapUpload(snap snapHelper, accounts string, file string) { for _, item := range targetAccounts { global.LOG.Debugf("start upload snapshot to %s, path: %s", item, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))) if _, err := accountMap[item].client.Upload(source, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))); err != nil { - global.LOG.Debugf("upload to %s failed, err: %v", item, err) + global.LOG.Debugf("upload to %s failed, err: %v", accountMap[item].name, err) snap.Status.Upload = err.Error() _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": err.Error()}) return } - global.LOG.Debugf("upload to %s successful", item) + global.LOG.Debugf("upload to %s successful", accountMap[item].name) } snap.Status.Upload = constant.StatusDone _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": constant.StatusDone}) @@ -221,60 +305,25 @@ func snapUpload(snap snapHelper, accounts string, file string) { _ = os.Remove(source) } -func handleSnapTar(sourceDir, targetDir, name, exclusionRules string, secret string) error { - if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) { - if err = os.MkdirAll(targetDir, os.ModePerm); err != nil { - return err - } - } - - exMap := make(map[string]struct{}) - exStr := "" - excludes := strings.Split(exclusionRules, ";") - excludes = append(excludes, "*.sock") - for _, exclude := range excludes { - if len(exclude) == 0 { - continue - } - if _, ok := exMap[exclude]; ok { - continue - } - exStr += " --exclude " - exStr += exclude - exMap[exclude] = struct{}{} - } - path := "" - if strings.Contains(sourceDir, "/") { - itemDir := strings.ReplaceAll(sourceDir[strings.LastIndex(sourceDir, "/"):], "/", "") - aheadDir := sourceDir[:strings.LastIndex(sourceDir, "/")] - if len(aheadDir) == 0 { - aheadDir = "/" - } - path += fmt.Sprintf("-C %s %s", aheadDir, itemDir) - } else { - path = sourceDir - } - commands := "" - if len(secret) != 0 { - extraCmd := "| openssl enc -aes-256-cbc -salt -k '" + secret + "' -out" - commands = fmt.Sprintf("tar --warning=no-file-changed --ignore-failed-read -zcf %s %s %s %s", " -"+exStr, path, extraCmd, targetDir+"/"+name) - global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) - } else { - commands = fmt.Sprintf("tar --warning=no-file-changed --ignore-failed-read -zcf %s %s -C %s .", targetDir+"/"+name, exStr, sourceDir) - global.LOG.Debug(commands) - } - stdout, err := cmd.ExecWithTimeOut(commands, 30*time.Minute) +func newSnapDB(dir, file string) (*gorm.DB, error) { + db, _ := gorm.Open(sqlite.Open(path.Join(dir, file)), &gorm.Config{ + DisableForeignKeyConstraintWhenMigrating: true, + }) + sqlDB, err := db.DB() if err != nil { - if len(stdout) != 0 { - global.LOG.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err) - return fmt.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err) - } + return nil, err } - return nil + sqlDB.SetConnMaxIdleTime(10) + sqlDB.SetMaxOpenConns(100) + sqlDB.SetConnMaxLifetime(time.Hour) + // global.LOG.Debug("load snapshot db conn successful!") + return db, nil } -func checkPointOfWal() { - if err := global.DB.Exec("PRAGMA wal_checkpoint(TRUNCATE);").Error; err != nil { - global.LOG.Errorf("handle check point failed, err: %v", err) +func closeDatabase(db *gorm.DB) { + sqlDB, err := db.DB() + if err != nil { + return } + _ = sqlDB.Close() } diff --git a/agent/app/service/snapshot_recover.go b/agent/app/service/snapshot_recover.go index 7397a8a3a1c8..b4177964d502 100644 --- a/agent/app/service/snapshot_recover.go +++ b/agent/app/service/snapshot_recover.go @@ -1,124 +1,98 @@ package service import ( - "context" "fmt" "os" "path" "strings" - "sync" + "time" "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/utils/cmd" - "github.com/1Panel-dev/1Panel/agent/utils/common" "github.com/1Panel-dev/1Panel/agent/utils/files" "github.com/pkg/errors" ) -func (u *SnapshotService) HandleSnapshotRecover(snap model.Snapshot, isRecover bool, req dto.SnapshotRecover) { +func (u *SnapshotService) HandleSnapshotRecover(snap model.Snapshot, req dto.SnapshotRecover) { _ = global.Cron.Stop() defer func() { global.Cron.Start() }() - snapFileDir := "" - if isRecover { - baseDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("system/%s", snap.Name)) - if _, err := os.Stat(baseDir); err != nil && os.IsNotExist(err) { - _ = os.MkdirAll(baseDir, os.ModePerm) - } - if req.IsNew || snap.InterruptStep == "Download" || req.ReDownload { - if err := handleDownloadSnapshot(snap, baseDir); err != nil { - updateRecoverStatus(snap.ID, isRecover, "Backup", constant.StatusFailed, err.Error()) - return - } - global.LOG.Debugf("download snapshot file to %s successful!", baseDir) - req.IsNew = true - } - if req.IsNew || snap.InterruptStep == "Decompress" { - if err := handleUnTar(fmt.Sprintf("%s/%s.tar.gz", baseDir, snap.Name), baseDir, req.Secret); err != nil { - updateRecoverStatus(snap.ID, isRecover, "Decompress", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) - return - } - global.LOG.Debug("decompress snapshot file successful!", baseDir) - req.IsNew = true - } - if req.IsNew || snap.InterruptStep == "Backup" { - if err := backupBeforeRecover(snap); err != nil { - updateRecoverStatus(snap.ID, isRecover, "Backup", constant.StatusFailed, fmt.Sprintf("handle backup before recover failed, err: %v", err)) - return - } - global.LOG.Debug("handle backup before recover successful!") - req.IsNew = true + fileOp := files.NewFileOp() + baseDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("system/%s", snap.Name)) + if _, err := os.Stat(baseDir); err != nil && os.IsNotExist(err) { + _ = os.MkdirAll(baseDir, os.ModePerm) + } + if req.IsNew || snap.InterruptStep == "Download" || req.ReDownload { + if err := handleDownloadSnapshot(snap, baseDir); err != nil { + updateRecoverStatus(snap.ID, "Download", constant.StatusFailed, err.Error()) + return } - snapFileDir = fmt.Sprintf("%s/%s", baseDir, snap.Name) - if _, err := os.Stat(snapFileDir); err != nil { - snapFileDir = baseDir + global.LOG.Debugf("download snapshot file to %s successful!", baseDir) + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "Decompress" { + if err := fileOp.TarGzExtractPro(fmt.Sprintf("%s/%s.tar.gz", baseDir, snap.Name), baseDir, req.Secret); err != nil { + updateRecoverStatus(snap.ID, "Decompress", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) + return } - } else { - snapFileDir = fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, snap.Name) - if _, err := os.Stat(snapFileDir); err != nil { - updateRecoverStatus(snap.ID, isRecover, "", constant.StatusFailed, fmt.Sprintf("cannot find the backup file %s, please try to recover again.", snapFileDir)) + global.LOG.Debug("decompress snapshot file successful!", baseDir) + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "Backup" { + if err := backupBeforeRecover(snap.Name); err != nil { + updateRecoverStatus(snap.ID, "Backup", constant.StatusFailed, fmt.Sprintf("handle backup before recover failed, err: %v", err)) return } + global.LOG.Debug("handle backup before recover successful!") + req.IsNew = true + } + snapFileDir := fmt.Sprintf("%s/%s", baseDir, snap.Name) + if _, err := os.Stat(snapFileDir); err != nil { + snapFileDir = baseDir } - snapJson, err := u.readFromJson(fmt.Sprintf("%s/snapshot.json", snapFileDir)) + snapJson, err := u.readFromJson(path.Join(snapFileDir, "base/snapshot.json")) if err != nil { - updateRecoverStatus(snap.ID, isRecover, "Readjson", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) + updateRecoverStatus(snap.ID, "Readjson", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) return } if snap.InterruptStep == "Readjson" { req.IsNew = true } - if isRecover && (req.IsNew || snap.InterruptStep == "AppData") { + if req.IsNew || snap.InterruptStep == "AppImage" { if err := recoverAppData(snapFileDir); err != nil { - updateRecoverStatus(snap.ID, isRecover, "DockerDir", constant.StatusFailed, fmt.Sprintf("handle recover app data failed, err: %v", err)) + updateRecoverStatus(snap.ID, "AppImage", constant.StatusFailed, fmt.Sprintf("handle recover app data failed, err: %v", err)) return } - global.LOG.Debug("recover app data from snapshot file successful!") - req.IsNew = true - } - if req.IsNew || snap.InterruptStep == "DaemonJson" { - fileOp := files.NewFileOp() - if err := recoverDaemonJson(snapFileDir, fileOp); err != nil { - updateRecoverStatus(snap.ID, isRecover, "DaemonJson", constant.StatusFailed, err.Error()) - return - } - global.LOG.Debug("recover daemon.json from snapshot file successful!") + global.LOG.Debug("recover app images from snapshot file successful!") req.IsNew = true } - if req.IsNew || snap.InterruptStep == "1PanelBinary" { - if err := recoverPanel(path.Join(snapFileDir, "1panel/1panel"), "/usr/local/bin"); err != nil { - updateRecoverStatus(snap.ID, isRecover, "1PanelBinary", constant.StatusFailed, err.Error()) + if req.IsNew || snap.InterruptStep == "BaseData" { + if err := recoverBaseData(path.Join(snapFileDir, "base"), fileOp); err != nil { + updateRecoverStatus(snap.ID, "BaseData", constant.StatusFailed, err.Error()) return } - global.LOG.Debug("recover 1panel binary from snapshot file successful!") + global.LOG.Debug("recover base data from snapshot file successful!") req.IsNew = true } - if req.IsNew || snap.InterruptStep == "1PctlBinary" { - if err := recoverPanel(path.Join(snapFileDir, "1panel/1pctl"), "/usr/local/bin"); err != nil { - updateRecoverStatus(snap.ID, isRecover, "1PctlBinary", constant.StatusFailed, err.Error()) - return - } - global.LOG.Debug("recover 1pctl from snapshot file successful!") - req.IsNew = true - } - if req.IsNew || snap.InterruptStep == "1PanelService" { - if err := recoverPanel(path.Join(snapFileDir, "1panel/1panel.service"), "/etc/systemd/system"); err != nil { - updateRecoverStatus(snap.ID, isRecover, "1PanelService", constant.StatusFailed, err.Error()) + + if req.IsNew || snap.InterruptStep == "DBData" { + if err := recoverDBData(path.Join(snapFileDir, "db"), fileOp); err != nil { + updateRecoverStatus(snap.ID, "DBData", constant.StatusFailed, err.Error()) return } - global.LOG.Debug("recover 1panel service from snapshot file successful!") + global.LOG.Debug("recover db data from snapshot file successful!") req.IsNew = true } if req.IsNew || snap.InterruptStep == "1PanelBackups" { - if err := u.handleUnTar(path.Join(snapFileDir, "/1panel/1panel_backup.tar.gz"), snapJson.BackupDataDir, ""); err != nil { - updateRecoverStatus(snap.ID, isRecover, "1PanelBackups", constant.StatusFailed, err.Error()) + if err := fileOp.TarGzExtractPro(path.Join(snapFileDir, "/1panel_backup.tar.gz"), snapJson.BackupDataDir, ""); err != nil { + updateRecoverStatus(snap.ID, "1PanelBackups", constant.StatusFailed, err.Error()) return } global.LOG.Debug("recover 1panel backups from snapshot file successful!") @@ -126,9 +100,8 @@ func (u *SnapshotService) HandleSnapshotRecover(snap model.Snapshot, isRecover b } if req.IsNew || snap.InterruptStep == "1PanelData" { - checkPointOfWal() - if err := u.handleUnTar(path.Join(snapFileDir, "/1panel/1panel_data.tar.gz"), path.Join(snapJson.BaseDir, "1panel"), ""); err != nil { - updateRecoverStatus(snap.ID, isRecover, "1PanelData", constant.StatusFailed, err.Error()) + if err := fileOp.TarGzExtractPro(path.Join(snapFileDir, "/1panel_data.tar.gz"), path.Join(snapJson.BaseDir, "1panel"), ""); err != nil { + updateRecoverStatus(snap.ID, "1PanelData", constant.StatusFailed, err.Error()) return } global.LOG.Debug("recover 1panel data from snapshot file successful!") @@ -138,51 +111,11 @@ func (u *SnapshotService) HandleSnapshotRecover(snap model.Snapshot, isRecover b restartCompose(path.Join(snapJson.BaseDir, "1panel/docker/compose")) global.LOG.Info("recover successful") - if !isRecover { - oriPath := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, snap.Name) - global.LOG.Debugf("remove the file %s after the operation is successful", oriPath) - _ = os.RemoveAll(oriPath) - } else { - global.LOG.Debugf("remove the file %s after the operation is successful", path.Dir(snapFileDir)) - _ = os.RemoveAll(path.Dir(snapFileDir)) - } + global.LOG.Debugf("remove the file %s after the operation is successful", path.Dir(snapFileDir)) + _ = os.RemoveAll(path.Dir(snapFileDir)) _, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service") } -func backupBeforeRecover(snap model.Snapshot) error { - baseDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, snap.Name) - var wg sync.WaitGroup - var status model.SnapshotStatus - itemHelper := snapHelper{SnapID: 0, Status: &status, Wg: &wg, FileOp: files.NewFileOp(), Ctx: context.Background()} - - jsonItem := SnapshotJson{ - BaseDir: global.CONF.System.BaseDir, - BackupDataDir: global.CONF.System.Backup, - PanelDataDir: path.Join(global.CONF.System.BaseDir, "1panel"), - } - _ = os.MkdirAll(path.Join(baseDir, "1panel"), os.ModePerm) - _ = os.MkdirAll(path.Join(baseDir, "docker"), os.ModePerm) - - wg.Add(4) - itemHelper.Wg = &wg - go snapJson(itemHelper, jsonItem, baseDir) - go snapPanel(itemHelper, path.Join(baseDir, "1panel")) - go snapDaemonJson(itemHelper, path.Join(baseDir, "docker")) - go snapBackup(itemHelper, path.Join(baseDir, "1panel")) - wg.Wait() - itemHelper.Status.AppData = constant.StatusDone - - allDone, msg := checkAllDone(status) - if !allDone { - return errors.New(msg) - } - snapPanelData(itemHelper, path.Join(baseDir, "1panel")) - if status.PanelData != constant.StatusDone { - return errors.New(status.PanelData) - } - return nil -} - func handleDownloadSnapshot(snap model.Snapshot, targetDir string) error { account, client, err := NewBackupClientWithID(snap.DownloadAccountID) if err != nil { @@ -202,18 +135,34 @@ func handleDownloadSnapshot(snap model.Snapshot, targetDir string) error { } func recoverAppData(src string) error { - if _, err := os.Stat(path.Join(src, "docker/docker_image.tar")); err != nil { + if _, err := os.Stat(path.Join(src, "images.tar.gz")); err != nil { global.LOG.Debug("no such docker images in snapshot") return nil } - std, err := cmd.Execf("docker load < %s", path.Join(src, "docker/docker_image.tar")) + std, err := cmd.Execf("docker load < %s", path.Join(src, "images.tar.gz")) if err != nil { return errors.New(std) } return err } -func recoverDaemonJson(src string, fileOp files.FileOp) error { +func recoverBaseData(src string, fileOp files.FileOp) error { + if err := fileOp.CopyFile(path.Join(src, "1pctl"), "/usr/local/bin"); err != nil { + return err + } + if err := fileOp.CopyFile(path.Join(src, "1panel"), "/usr/local/bin"); err != nil { + return err + } + if err := fileOp.CopyFile(path.Join(src, "1panel_agent"), "/usr/local/bin"); err != nil { + return err + } + if err := fileOp.CopyFile(path.Join(src, "1panel.service"), "/etc/systemd/system"); err != nil { + return err + } + if err := fileOp.CopyFile(path.Join(src, "1panel_agent.service"), "/etc/systemd/system"); err != nil { + return err + } + daemonJsonPath := "/etc/docker/daemon.json" _, errSrc := os.Stat(path.Join(src, "docker/daemon.json")) _, errPath := os.Stat(daemonJsonPath) @@ -231,14 +180,8 @@ func recoverDaemonJson(src string, fileOp files.FileOp) error { return nil } -func recoverPanel(src string, dst string) error { - if _, err := os.Stat(src); err != nil { - return fmt.Errorf("file is not found in %s, err: %v", src, err) - } - if err := common.CopyFile(src, dst); err != nil { - return fmt.Errorf("cp file failed, err: %v", err) - } - return nil +func recoverDBData(src string, fileOp files.FileOp) error { + return fileOp.CopyDir(src, path.Join(global.CONF.System.BaseDir, "1panel", "db")) } func restartCompose(composePath string) { @@ -259,3 +202,86 @@ func restartCompose(composePath string) { } global.LOG.Debug("restart all compose successful!") } + +func backupBeforeRecover(name string) error { + rootDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, name) + baseDir := path.Join(rootDir, "base") + if _, err := os.Stat(baseDir); err != nil { + _ = os.MkdirAll(baseDir, os.ModePerm) + } + + FileOp := files.NewFileOp() + if err := FileOp.CopyDirWithExclude(path.Join(global.CONF.System.BaseDir, "1panel"), rootDir, []string{"cache", "tmp"}); err != nil { + return err + } + if err := FileOp.CopyDir(global.CONF.System.Backup, rootDir); err != nil { + return err + } + if err := FileOp.CopyFile("/usr/local/bin/1pctl", baseDir); err != nil { + return err + } + if err := FileOp.CopyFile("/usr/local/bin/1panel", baseDir); err != nil { + return err + } + if err := FileOp.CopyFile("/usr/local/bin/1panel_agent", baseDir); err != nil { + return err + } + if err := FileOp.CopyFile("/etc/systemd/system/1panel.service", baseDir); err != nil { + return err + } + if err := FileOp.CopyFile("/etc/systemd/system/1panel_agent.service", baseDir); err != nil { + return err + } + if err := FileOp.CopyFile("/etc/docker/daemon.json", baseDir); err != nil { + return err + } + return nil +} + +func handleRollback(name string) error { + rootDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, name) + baseDir := path.Join(rootDir, "base") + + FileOp := files.NewFileOp() + if err := FileOp.CopyDir(path.Join(rootDir, "1panel"), global.CONF.System.BaseDir); err != nil { + return err + } + if err := FileOp.CopyDir(path.Join(rootDir, "backup"), path.Dir(global.CONF.System.Backup)); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "1pctl"), "/usr/local/bin/1pctl"); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "1panel"), "/usr/local/bin/1panel"); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "1panel_agent"), "/usr/local/bin/1panel_agent"); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "1panel.service"), "/etc/systemd/system/1panel.service"); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "1panel_agent.service"), "/etc/systemd/system/1panel_agent.service"); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "daemon.json"), "/etc/docker/daemon.json"); err != nil { + return err + } + _ = os.RemoveAll(rootDir) + return nil +} + +func updateRecoverStatus(id uint, interruptStep, status, message string) { + if status != constant.StatusSuccess { + global.LOG.Errorf("recover failed, err: %s", message) + } + if err := snapshotRepo.Update(id, map[string]interface{}{ + "interrupt_step": interruptStep, + "recover_status": status, + "recover_message": message, + "last_recovered_at": time.Now().Format(constant.DateTimeLayout), + }); err != nil { + global.LOG.Errorf("update snap recover status failed, err: %v", err) + } + _ = settingRepo.Update("SystemStatus", "Free") +} diff --git a/agent/init/hook/hook.go b/agent/init/hook/hook.go index f2d7cc45b9e6..7cc33863abd2 100644 --- a/agent/init/hook/hook.go +++ b/agent/init/hook/hook.go @@ -72,18 +72,18 @@ func handleSnapStatus() { status, _ := snapRepo.GetStatusList() for _, item := range status { updates := make(map[string]interface{}) - if item.Panel == constant.StatusRunning { - updates["panel"] = constant.StatusFailed - } - if item.PanelInfo == constant.StatusRunning { - updates["panel_info"] = constant.StatusFailed - } - if item.DaemonJson == constant.StatusRunning { - updates["daemon_json"] = constant.StatusFailed - } - if item.AppData == constant.StatusRunning { - updates["app_data"] = constant.StatusFailed - } + // if item.Panel == constant.StatusRunning { + // updates["panel"] = constant.StatusFailed + // } + // if item.PanelInfo == constant.StatusRunning { + // updates["panel_info"] = constant.StatusFailed + // } + // if item.DaemonJson == constant.StatusRunning { + // updates["daemon_json"] = constant.StatusFailed + // } + // if item.AppData == constant.StatusRunning { + // updates["app_data"] = constant.StatusFailed + // } if item.PanelData == constant.StatusRunning { updates["panel_data"] = constant.StatusFailed } diff --git a/agent/init/migration/migrate.go b/agent/init/migration/migrate.go index 36d24b8e59dc..cf5b146199d2 100644 --- a/agent/init/migration/migrate.go +++ b/agent/init/migration/migrate.go @@ -15,6 +15,13 @@ func Init() { migrations.InitImageRepo, migrations.InitDefaultCA, migrations.InitPHPExtensions, + migrations.AddTask, + migrations.UpdateWebsite, + migrations.UpdateWebsiteDomain, + migrations.UpdateApp, + migrations.AddTaskDB, + migrations.UpdateAppInstall, + migrations.UpdateSnapshot, }) if err := m.Migrate(); err != nil { global.LOG.Error(err) diff --git a/agent/init/migration/migrations/init.go b/agent/init/migration/migrations/init.go index ca3c0833ddeb..b959b29b33c2 100644 --- a/agent/init/migration/migrations/init.go +++ b/agent/init/migration/migrations/init.go @@ -211,3 +211,59 @@ var InitPHPExtensions = &gormigrate.Migration{ return nil }, } + +var AddTask = &gormigrate.Migration{ + ID: "20240802-add-task", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate( + &model.Task{}) + }, +} + +var UpdateWebsite = &gormigrate.Migration{ + ID: "20240812-update-website", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate( + &model.Website{}) + }, +} + +var UpdateWebsiteDomain = &gormigrate.Migration{ + ID: "20240808-update-website-domain", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate( + &model.WebsiteDomain{}) + }, +} + +var AddTaskDB = &gormigrate.Migration{ + ID: "20240822-add-task-table", + Migrate: func(tx *gorm.DB) error { + return global.TaskDB.AutoMigrate( + &model.Task{}, + ) + }, +} + +var UpdateApp = &gormigrate.Migration{ + ID: "20240826-update-app", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate( + &model.App{}) + }, +} + +var UpdateAppInstall = &gormigrate.Migration{ + ID: "20240828-update-app-install", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate( + &model.AppInstall{}) + }, +} + +var UpdateSnapshot = &gormigrate.Migration{ + ID: "20240913-update-snapshot", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&model.Snapshot{}) + }, +} diff --git a/agent/router/ro_setting.go b/agent/router/ro_setting.go index 67ecd12eaf4a..ec956ccacbaa 100644 --- a/agent/router/ro_setting.go +++ b/agent/router/ro_setting.go @@ -15,6 +15,7 @@ func (s *SettingRouter) InitRouter(Router *gin.RouterGroup) { settingRouter.GET("/search/available", baseApi.GetSystemAvailable) settingRouter.POST("/update", baseApi.UpdateSetting) + settingRouter.GET("/snapshot/load", baseApi.LoadSnapshotData) settingRouter.POST("/snapshot", baseApi.CreateSnapshot) settingRouter.POST("/snapshot/status", baseApi.LoadSnapShotStatus) settingRouter.POST("/snapshot/search", baseApi.SearchSnapshot) diff --git a/agent/utils/files/file_op.go b/agent/utils/files/file_op.go index 4248f72ce3f7..3141b7cb7b25 100644 --- a/agent/utils/files/file_op.go +++ b/agent/utils/files/file_op.go @@ -456,6 +456,48 @@ func (f FileOp) CopyDir(src, dst string) error { return cmd.ExecCmd(fmt.Sprintf(`cp -rf '%s' '%s'`, src, dst+"/")) } +func (f FileOp) CopyDirWithExclude(src, dst string, excludeNames []string) error { + srcInfo, err := f.Fs.Stat(src) + if err != nil { + return err + } + dstDir := filepath.Join(dst, srcInfo.Name()) + if err = f.Fs.MkdirAll(dstDir, srcInfo.Mode()); err != nil { + return err + } + if len(excludeNames) == 0 { + return cmd.ExecCmd(fmt.Sprintf(`cp -rf '%s' '%s'`, src, dst+"/")) + } + tmpFiles, err := os.ReadDir(src) + if err != nil { + return err + } + for _, item := range tmpFiles { + isExclude := false + for _, name := range excludeNames { + if item.Name() == name { + isExclude = true + break + } + } + if isExclude { + continue + } + if item.IsDir() { + fmt.Println(path.Join(src, item.Name()), dstDir) + if err := f.CopyDir(path.Join(src, item.Name()), dstDir); err != nil { + return err + } + continue + } + if err := f.CopyFile(path.Join(src, item.Name()), dstDir); err != nil { + return err + } + } + + return nil +} + func (f FileOp) CopyFile(src, dst string) error { dst = filepath.Clean(dst) + string(filepath.Separator) return cmd.ExecCmd(fmt.Sprintf(`cp -f '%s' '%s'`, src, dst+"/")) @@ -670,3 +712,56 @@ func ZipFile(files []archiver.File, dst afero.File) error { } return nil } + +func (f FileOp) TarGzCompressPro(withDir bool, src, dst, secret, exclusionRules string) error { + workdir := src + srcItem := "." + if withDir { + workdir = path.Dir(src) + srcItem = path.Base(src) + } + commands := "" + + exMap := make(map[string]struct{}) + exStr := "" + excludes := strings.Split(exclusionRules, ";") + excludes = append(excludes, "*.sock") + for _, exclude := range excludes { + if len(exclude) == 0 { + continue + } + if _, ok := exMap[exclude]; ok { + continue + } + exStr += " --exclude " + exStr += exclude + exMap[exclude] = struct{}{} + } + + if len(secret) != 0 { + commands = fmt.Sprintf("tar -zcf - %s | openssl enc -aes-256-cbc -salt -pbkdf2 -k '%s' -out %s", srcItem, secret, dst) + global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) + } else { + commands = fmt.Sprintf("tar zcf %s %s %s", dst, exStr, srcItem) + global.LOG.Debug(commands) + } + return cmd.ExecCmdWithDir(commands, workdir) +} + +func (f FileOp) TarGzExtractPro(src, dst string, secret string) error { + if _, err := os.Stat(path.Dir(dst)); err != nil && os.IsNotExist(err) { + if err = os.MkdirAll(path.Dir(dst), os.ModePerm); err != nil { + return err + } + } + + commands := "" + if len(secret) != 0 { + commands = fmt.Sprintf("openssl enc -d -aes-256-cbc -salt -pbkdf2 -k '%s' -in %s | tar -zxf - > /root/log", secret, src) + global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) + } else { + commands = fmt.Sprintf("tar zxvf %s", src) + global.LOG.Debug(commands) + } + return cmd.ExecCmdWithDir(commands, dst) +} diff --git a/agent/utils/files/tar_gz.go b/agent/utils/files/tar_gz.go index 93a82c54b125..f59a113fb635 100644 --- a/agent/utils/files/tar_gz.go +++ b/agent/utils/files/tar_gz.go @@ -2,6 +2,8 @@ package files import ( "fmt" + "os" + "path" "path/filepath" "strings" @@ -59,3 +61,56 @@ func (t TarGzArchiver) Compress(sourcePaths []string, dstFile string, secret str } return nil } + +func (t TarGzArchiver) CompressPro(withDir bool, src, dst, secret, exclusionRules string) error { + workdir := src + srcItem := "." + if withDir { + workdir = path.Dir(src) + srcItem = path.Base(src) + } + commands := "" + + exMap := make(map[string]struct{}) + exStr := "" + excludes := strings.Split(exclusionRules, ";") + excludes = append(excludes, "*.sock") + for _, exclude := range excludes { + if len(exclude) == 0 { + continue + } + if _, ok := exMap[exclude]; ok { + continue + } + exStr += " --exclude " + exStr += exclude + exMap[exclude] = struct{}{} + } + + if len(secret) != 0 { + commands = fmt.Sprintf("tar -zcf - %s | openssl enc -aes-256-cbc -salt -pbkdf2 -k '%s' -out %s", srcItem, secret, dst) + global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) + } else { + commands = fmt.Sprintf("tar zcf %s %s %s", dst, exStr, srcItem) + global.LOG.Debug(commands) + } + return cmd.ExecCmdWithDir(commands, workdir) +} + +func (t TarGzArchiver) ExtractPro(src, dst string, secret string) error { + if _, err := os.Stat(path.Dir(dst)); err != nil && os.IsNotExist(err) { + if err = os.MkdirAll(path.Dir(dst), os.ModePerm); err != nil { + return err + } + } + + commands := "" + if len(secret) != 0 { + commands = fmt.Sprintf("openssl enc -d -aes-256-cbc -salt -pbkdf2 -k '%s' -in %s | tar -zxf - > /root/log", secret, src) + global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) + } else { + commands = fmt.Sprintf("tar zxvf %s", src) + global.LOG.Debug(commands) + } + return cmd.ExecCmdWithDir(commands, dst) +} diff --git a/frontend/src/api/interface/setting.ts b/frontend/src/api/interface/setting.ts index e358879a7ba3..6c3ceb578d4c 100644 --- a/frontend/src/api/interface/setting.ts +++ b/frontend/src/api/interface/setting.ts @@ -119,11 +119,18 @@ export namespace Setting { export interface SnapshotCreate { id: number; - from: string; - fromAccounts: Array; - defaultDownload: string; + sourceAccountIDs: string; + downloadAccountID: string; description: string; secret: string; + + appData: Array; + panelData: Array; + backupData: Array; + + withMonitorData: boolean; + withLoginLog: boolean; + withOperationLog: boolean; } export interface SnapshotImport { from: string; @@ -155,11 +162,31 @@ export namespace Setting { lastRollbackedAt: string; secret: string; } + export interface SnapshotData { + appData: Array; + panelData: Array; + backupData: Array; + + withMonitorData: boolean; + withLoginLog: boolean; + withOperationLog: boolean; + } + export interface DataTree { + id: string; + label: string; + key: string; + name: string; + size: number; + isCheck: boolean; + isDisable: boolean; + + path: string; + + Children: Array; + } export interface SnapshotStatus { - panel: string; - panelInfo: string; - daemonJson: string; - appData: string; + baseData: string; + appImage: string; panelData: string; backupData: string; diff --git a/frontend/src/api/modules/setting.ts b/frontend/src/api/modules/setting.ts index abd3b0e7e92a..922b1ecbee92 100644 --- a/frontend/src/api/modules/setting.ts +++ b/frontend/src/api/modules/setting.ts @@ -88,6 +88,9 @@ export const bindMFA = (param: Setting.MFABind) => { }; // snapshot +export const loadSnapshotSetting = () => { + return http.get(`/settings/snapshot/load`); +}; export const snapshotCreate = (param: Setting.SnapshotCreate) => { return http.post(`/settings/snapshot`, param); }; diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index cd54c553fb77..228457a31366 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -1587,6 +1587,36 @@ const message = { 'Backup files not in the current backup list, please try downloading from the file directory and importing for backup.', snapshot: 'Snapshot', + stepBaseData: 'Base Data', + stepAppData: 'System Application', + stepPanelData: 'System Data', + stepBackupData: 'Backup Data', + stepOtherData: 'Other Data', + loginLog: 'Backup System Login Records', + OperationLog: 'Backup System Operation Log Login Records', + monitorData: 'Backup System Monitoring Data', + selectAllImage: 'Backup All Application Images', + agentLabel: 'Node Configuration', + appDataLabel: 'Application Data', + appImage: 'Application Image', + appBackup: 'Application Backup', + backupLabel: 'Backup Directory', + cacheLabel: 'Cache Directory', + confLabel: 'Configuration File', + dbLabel: 'Database Directory', + dockerLabel: 'Container Related Logs', + logLabel: 'Log Directory', + taskLabel: 'Scheduled Task Report', + resourceLabel: 'Application Resource Directory', + runtimeLabel: 'Runtime Directory', + appLabel: 'Application', + databaseLabel: 'Database', + websiteLabel: 'Website', + directoryLabel: 'Directory', + appStoreLabel: 'Application Store', + shellLabel: 'Script', + tmpLabel: 'Temporary Directory', + sslLabel: 'Certificate Directory', deleteHelper: 'All backup files for the snapshot, including those in the third-party backup account, will be deleted.', status: 'Snapshot status', diff --git a/frontend/src/lang/modules/tw.ts b/frontend/src/lang/modules/tw.ts index ef3cfededcb0..e5c230ec709d 100644 --- a/frontend/src/lang/modules/tw.ts +++ b/frontend/src/lang/modules/tw.ts @@ -1405,6 +1405,36 @@ const message = { backupJump: '未在當前備份列表中的備份檔案，請嘗試從檔案目錄中下載後導入備份。', snapshot: '快照', + stepBaseData: '基礎數據', + stepAppData: '系統應用', + stepPanelData: '系統數據', + stepBackupData: '備份數據', + stepOtherData: '其他數據', + loginLog: '備份系統登錄記錄', + OperationLog: '備份系統操作日誌登錄記錄', + monitorData: '備份系統監控數據', + selectAllImage: '備份所有應用鏡像', + agentLabel: '節點配置', + appDataLabel: '應用數據', + appImage: '應用鏡像', + appBackup: '應用備份', + backupLabel: '備份目錄', + cacheLabel: '緩存目錄', + confLabel: '配置文件', + dbLabel: '數據庫目錄', + dockerLabel: '容器相關日誌', + logLabel: '日誌目錄', + taskLabel: '計劃任務報告', + resourceLabel: '應用資源目錄', + runtimeLabel: '運行時目錄', + appLabel: '應用', + databaseLabel: '數據庫', + websiteLabel: '網站', + directoryLabel: '目錄', + appStoreLabel: '應用商店', + shellLabel: '腳本', + tmpLabel: '臨時目錄', + sslLabel: '證書目錄', deleteHelper: '將刪除該快照的所有備份文件，包括第三方備份賬號中的文件。', status: '快照狀態', ignoreRule: '排除規則', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index ce76bd94228b..8edc7639f851 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -1407,6 +1407,36 @@ const message = { backupJump: '未在当前备份列表中的备份文件，请尝试从文件目录中下载后导入备份。', snapshot: '快照', + stepBaseData: '基础数据', + stepAppData: '系统应用', + stepPanelData: '系统数据', + stepBackupData: '备份数据', + stepOtherData: '其他数据', + loginLog: '备份系统登陆记录', + OperationLog: '备份系统备份系统操作日志登陆记录', + monitorData: '备份系统监控数据', + selectAllImage: '备份所有应用镜像', + agentLabel: '节点配置', + appDataLabel: '应用数据', + appImage: '应用镜像', + appBackup: '应用备份', + backupLabel: '备份目录', + cacheLabel: '缓存目录', + confLabel: '配置文件', + dbLabel: '数据库目录', + dockerLabel: '容器相关日志', + logLabel: '日志目录', + taskLabel: '计划任务报告', + resourceLabel: '应用资源目录', + runtimeLabel: '运行时目录', + appLabel: '应用', + databaseLabel: '数据库', + websiteLabel: '网站', + directoryLabel: '目录', + appStoreLabel: '应用商店', + shellLabel: '脚本', + tmpLabel: '临时目录', + sslLabel: '证书目录', deleteHelper: '将删除该快照的所有备份文件，包括第三方备份账号中的文件。', ignoreRule: '排除规则', ignoreHelper: '快照时将使用该规则对 1Panel 数据目录进行压缩备份，请谨慎修改。', diff --git a/frontend/src/views/setting/snapshot/create/index.vue b/frontend/src/views/setting/snapshot/create/index.vue new file mode 100644 index 000000000000..001ffc4de76c --- /dev/null +++ b/frontend/src/views/setting/snapshot/create/index.vue @@ -0,0 +1,485 @@ + + + + diff --git a/frontend/src/views/setting/snapshot/index.vue b/frontend/src/views/setting/snapshot/index.vue index 42799fd97da4..1f5458d3c491 100644 --- a/frontend/src/views/setting/snapshot/index.vue +++ b/frontend/src/views/setting/snapshot/index.vue @@ -88,7 +88,7 @@ {{ $t('commons.status.error') }} - + {{ $t('commons.status.success') }} @@ -115,52 +115,8 @@ + - - - - - - - - - - - - - - - - - - - - - diff --git a/frontend/src/views/setting/snapshot/status/index.vue b/frontend/src/views/setting/snapshot/status/index.vue index 4816fe4aa0bb..b49ba8f5742b 100644 --- a/frontend/src/views/setting/snapshot/status/index.vue +++ b/frontend/src/views/setting/snapshot/status/index.vue @@ -1,220 +1,295 @@ - - - From e20c7a6f7fb1c42345cc7788672fd9dc18c0eb42 Mon Sep 17 00:00:00 2001 From: ssonglius11 Date: Wed, 18 Sep 2024 14:47:06 +0800 Subject: [PATCH 02/12] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E5=BF=AB?= =?UTF-8?q?=E7=85=A7=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/app/dto/snapshot.go | 3 +- agent/app/service/snapshot.go | 111 +++++++++++++++++++------- agent/app/service/snapshot_create.go | 2 +- agent/app/service/snapshot_recover.go | 4 +- agent/utils/files/file_op.go | 1 - 5 files changed, 89 insertions(+), 32 deletions(-) diff --git a/agent/app/dto/snapshot.go b/agent/app/dto/snapshot.go index 9504cfc963ee..78bdf481330f 100644 --- a/agent/app/dto/snapshot.go +++ b/agent/app/dto/snapshot.go @@ -49,7 +49,8 @@ type DataTree struct { Path string `json:"path"` - Children []DataTree `json:"children"` + RelationItemID string `json:"relationItemID"` + Children []DataTree `json:"children"` } type SnapshotRecover struct { IsNew bool `json:"isNew"` diff --git a/agent/app/service/snapshot.go b/agent/app/service/snapshot.go index 2b4a5efecb00..abd0293d1415 100644 --- a/agent/app/service/snapshot.go +++ b/agent/app/service/snapshot.go @@ -221,9 +221,9 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, logPath string, req dto if req.ID == 0 { versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion")) - name := fmt.Sprintf("1panel_%s_%s_%s", versionItem.Value, loadOs(), timeNow) + name := fmt.Sprintf("1panel-%s-linux-%s-%s", versionItem.Value, loadOs(), timeNow) if isCronjob { - name = fmt.Sprintf("snapshot_1panel_%s_%s_%s", versionItem.Value, loadOs(), timeNow) + name = fmt.Sprintf("snapshot-1panel-%s-linux-%s-%s", versionItem.Value, loadOs(), timeNow) } rootDir = path.Join(global.CONF.System.BaseDir, "1panel/tmp/system", name) @@ -507,56 +507,114 @@ func loadApps(fileOp fileUtils.FileOp) ([]dto.DataTree, error) { if err != nil { return data, err } - client, err := docker.NewDockerClient() - hasDockerClient := true + openrestyID := 0 + for _, app := range apps { + if app.App.Key == constant.AppOpenresty { + openrestyID = int(app.ID) + } + } + websites, err := websiteRepo.List() if err != nil { - hasDockerClient = false - global.LOG.Errorf("new docker client failed, err: %v", err) - } else { - defer client.Close() + return data, err } - imageList, err := client.ImageList(context.Background(), image.ListOptions{}) + appRelationMap := make(map[uint]uint) + for _, website := range websites { + if website.Type == constant.Deployment && website.AppInstallID != 0 { + appRelationMap[uint(openrestyID)] = website.AppInstallID + } + } + appRelations, err := appInstallResourceRepo.GetBy() if err != nil { - hasDockerClient = false - global.LOG.Errorf("load image list failed, err: %v", err) + return data, err + } + for _, relation := range appRelations { + appRelationMap[uint(relation.AppInstallId)] = relation.LinkId + } + appMap := make(map[uint]string) + for _, app := range apps { + appMap[app.ID] = fmt.Sprintf("%s-%s", app.App.Key, app.Name) } + appTreeMap := make(map[string]dto.DataTree) for _, app := range apps { - itemApp := dto.DataTree{ID: uuid.NewString(), Label: fmt.Sprintf("%s - %s", app.App.Name, app.Name), Key: app.App.Key, Name: app.Name} + itemApp := dto.DataTree{ + ID: uuid.NewString(), + Label: fmt.Sprintf("%s - %s", app.App.Name, app.Name), + Key: app.App.Key, + Name: app.Name, + } appPath := path.Join(global.CONF.System.BaseDir, "1panel/apps", app.App.Key, app.Name) itemAppData := dto.DataTree{ID: uuid.NewString(), Label: "appData", Key: app.App.Key, Name: app.Name, IsCheck: true, Path: appPath} + if app.App.Key == constant.AppOpenresty && len(websites) != 0 { + itemAppData.IsDisable = true + } + if val, ok := appRelationMap[app.ID]; ok { + itemAppData.RelationItemID = appMap[val] + } sizeItem, err := fileOp.GetDirSize(appPath) if err == nil { itemAppData.Size = uint64(sizeItem) } itemApp.Size += itemAppData.Size - itemApp.Children = append(itemApp.Children, itemAppData) + data = append(data, itemApp) + appTreeMap[fmt.Sprintf("%s-%s", itemApp.Key, itemApp.Name)] = itemAppData + } + + for key, val := range appTreeMap { + if valRelation, ok := appTreeMap[val.RelationItemID]; ok { + valRelation.IsDisable = true + appTreeMap[val.RelationItemID] = valRelation - appBackupPath := path.Join(global.CONF.System.BaseDir, "1panel/backup/app", app.App.Key, app.Name) + val.RelationItemID = valRelation.ID + appTreeMap[key] = val + } + } + for i := 0; i < len(data); i++ { + if val, ok := appTreeMap[fmt.Sprintf("%s-%s", data[i].Key, data[i].Name)]; ok { + data[i].Children = append(data[i].Children, val) + } + } + data = loadAppBackup(data, fileOp) + data = loadAppImage(data) + return data, nil +} +func loadAppBackup(list []dto.DataTree, fileOp fileUtils.FileOp) []dto.DataTree { + for i := 0; i < len(list); i++ { + appBackupPath := path.Join(global.CONF.System.BaseDir, "1panel/backup/app", list[i].Key, list[i].Name) itemAppBackupTree, err := loadFile(appBackupPath, 8, fileOp) itemAppBackup := dto.DataTree{ID: uuid.NewString(), Label: "appBackup", IsCheck: true, Children: itemAppBackupTree, Path: appBackupPath} if err == nil { backupSizeItem, err := fileOp.GetDirSize(appBackupPath) if err == nil { itemAppBackup.Size = uint64(backupSizeItem) - itemApp.Size += itemAppBackup.Size + list[i].Size += itemAppBackup.Size } - itemApp.Children = append(itemApp.Children, itemAppBackup) + list[i].Children = append(list[i].Children, itemAppBackup) } + } + return list +} +func loadAppImage(list []dto.DataTree) []dto.DataTree { + client, err := docker.NewDockerClient() + if err != nil { + global.LOG.Errorf("new docker client failed, err: %v", err) + return list + } + defer client.Close() + imageList, err := client.ImageList(context.Background(), image.ListOptions{}) + if err != nil { + global.LOG.Errorf("load image list failed, err: %v", err) + return list + } + for i := 0; i < len(list); i++ { itemAppImage := dto.DataTree{ID: uuid.NewString(), Label: "appImage"} - stdout, err := cmd.Execf("cat %s | grep image: ", path.Join(global.CONF.System.BaseDir, "1panel/apps", app.App.Key, app.Name, "docker-compose.yml")) + stdout, err := cmd.Execf("cat %s | grep image: ", path.Join(global.CONF.System.BaseDir, "1panel/apps", list[i].Key, list[i].Name, "docker-compose.yml")) if err != nil { - itemApp.Children = append(itemApp.Children, itemAppImage) - data = append(data, itemApp) + list[i].Children = append(list[i].Children, itemAppImage) continue } itemAppImage.Name = strings.ReplaceAll(strings.ReplaceAll(strings.TrimSpace(stdout), "\n", ""), "image: ", "") - if !hasDockerClient { - itemApp.Children = append(itemApp.Children, itemAppImage) - data = append(data, itemApp) - continue - } for _, imageItem := range imageList { for _, tag := range imageItem.RepoTags { if tag == itemAppImage.Name { @@ -565,10 +623,9 @@ func loadApps(fileOp fileUtils.FileOp) ([]dto.DataTree, error) { } } } - itemApp.Children = append(itemApp.Children, itemAppImage) - data = append(data, itemApp) + list[i].Children = append(list[i].Children, itemAppImage) } - return data, nil + return list } func loadPanelFile(fileOp fileUtils.FileOp) ([]dto.DataTree, error) { diff --git a/agent/app/service/snapshot_create.go b/agent/app/service/snapshot_create.go index a93fcfee16b1..a2fc319835c7 100644 --- a/agent/app/service/snapshot_create.go +++ b/agent/app/service/snapshot_create.go @@ -229,7 +229,7 @@ func snapPanelData(snap snapHelper, req dto.SnapshotCreate, targetDir string) { excludes = append(excludes, "."+strings.ReplaceAll(ignore, rootDir, "")) } - _ = snap.snapAgentDB.Model(&model.Setting{}).Where("key = ?", "SystemIP").Updates(map[string]interface{}{"SystemIP": ""}) + _ = snap.snapAgentDB.Model(&model.Setting{}).Where("key = ?", "SystemIP").Updates(map[string]interface{}{"value": ""}) if err := snap.FileOp.TarGzCompressPro(false, rootDir, path.Join(targetDir, "1panel_data.tar.gz"), "", strings.Join(excludes, ";")); err != nil { status = err.Error() diff --git a/agent/app/service/snapshot_recover.go b/agent/app/service/snapshot_recover.go index b4177964d502..cc17e01dbbc3 100644 --- a/agent/app/service/snapshot_recover.go +++ b/agent/app/service/snapshot_recover.go @@ -181,7 +181,7 @@ func recoverBaseData(src string, fileOp files.FileOp) error { } func recoverDBData(src string, fileOp files.FileOp) error { - return fileOp.CopyDir(src, path.Join(global.CONF.System.BaseDir, "1panel", "db")) + return fileOp.CopyDirWithExclude(src, path.Join(global.CONF.System.BaseDir, "1panel"), nil) } func restartCompose(composePath string) { @@ -214,7 +214,7 @@ func backupBeforeRecover(name string) error { if err := FileOp.CopyDirWithExclude(path.Join(global.CONF.System.BaseDir, "1panel"), rootDir, []string{"cache", "tmp"}); err != nil { return err } - if err := FileOp.CopyDir(global.CONF.System.Backup, rootDir); err != nil { + if err := FileOp.CopyDirWithExclude(global.CONF.System.Backup, rootDir, []string{"system_snapshot"}); err != nil { return err } if err := FileOp.CopyFile("/usr/local/bin/1pctl", baseDir); err != nil { diff --git a/agent/utils/files/file_op.go b/agent/utils/files/file_op.go index 3141b7cb7b25..8ccd7c000bec 100644 --- a/agent/utils/files/file_op.go +++ b/agent/utils/files/file_op.go @@ -484,7 +484,6 @@ func (f FileOp) CopyDirWithExclude(src, dst string, excludeNames []string) error continue } if item.IsDir() { - fmt.Println(path.Join(src, item.Name()), dstDir) if err := f.CopyDir(path.Join(src, item.Name()), dstDir); err != nil { return err } From 27f82262543d6454fb5624f902a73fed7fb1efec Mon Sep 17 00:00:00 2001 From: ssongliu Date: Wed, 18 Sep 2024 17:06:28 +0800 Subject: [PATCH 03/12] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E5=BF=AB?= =?UTF-8?q?=E7=85=A7=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../views/setting/snapshot/create/index.vue | 34 ++ .../setting/snapshot/snap_status/index.vue | 85 ++-- .../views/setting/snapshot/status/index.vue | 441 ++++++++---------- 3 files changed, 247 insertions(+), 313 deletions(-) diff --git a/frontend/src/views/setting/snapshot/create/index.vue b/frontend/src/views/setting/snapshot/create/index.vue index 001ffc4de76c..d8f84144ddec 100644 --- a/frontend/src/views/setting/snapshot/create/index.vue +++ b/frontend/src/views/setting/snapshot/create/index.vue @@ -69,6 +69,7 @@ :default-expand-all="true" :data="form.appData" :props="defaultProps" + @check-change="onChangeAppData" show-checkbox >
- + - + - - - - - - @@ -100,10 +84,8 @@ import { loadSnapStatus, snapshotCreate } from '@/api/modules/setting'; import { nextTick, onBeforeUnmount, reactive, ref } from 'vue'; const status = reactive({ - panel: '', - panelInfo: '', - daemonJson: '', - appData: '', + baseData: '', + appImage: '', panelData: '', backupData: '', @@ -147,10 +129,8 @@ const loadCurrentStatus = async () => { await loadSnapStatus(snapID.value) .then((res) => { loading.value = false; - status.panel = res.data.panel; - status.panelInfo = res.data.panelInfo; - status.daemonJson = res.data.daemonJson; - status.appData = res.data.appData; + status.baseData = res.data.baseData; + status.appImage = res.data.appImage; status.panelData = res.data.panelData; status.backupData = res.data.backupData; @@ -172,10 +152,19 @@ const onRetry = async () => { loading.value = true; await snapshotCreate({ id: snapID.value, - fromAccounts: [], - from: snapFrom.value, - defaultDownload: snapDefaultDownload.value, description: snapDescription.value, + + downloadAccountID: '', + sourceAccountIDs: '', + secret: '', + + withLoginLog: false, + withOperationLog: false, + withMonitorData: false, + + panelData: [], + backupData: [], + appData: [], }) .then(() => { loading.value = false; @@ -190,10 +179,8 @@ const onWatch = () => { timer = setInterval(async () => { if (keepLoadStatus()) { const res = await loadSnapStatus(snapID.value); - status.panel = res.data.panel; - status.panelInfo = res.data.panelInfo; - status.daemonJson = res.data.daemonJson; - status.appData = res.data.appData; + status.baseData = res.data.baseData; + status.appImage = res.data.appImage; status.panelData = res.data.panelData; status.backupData = res.data.backupData; @@ -205,16 +192,10 @@ const onWatch = () => { }; const keepLoadStatus = () => { - if (status.panel === 'Running') { - return true; - } - if (status.panelInfo === 'Running') { - return true; - } - if (status.daemonJson === 'Running') { + if (status.baseData === 'Running') { return true; } - if (status.appData === 'Running') { + if (status.appImage === 'Running') { return true; } if (status.panelData === 'Running') { @@ -240,16 +221,10 @@ const showRetry = () => { if (keepLoadStatus()) { return false; } - if (status.panel !== 'Running' && status.panel !== 'Done') { - return true; - } - if (status.panelInfo !== 'Running' && status.panelInfo !== 'Done') { - return true; - } - if (status.daemonJson !== 'Running' && status.daemonJson !== 'Done') { + if (status.baseData !== 'Running' && status.baseData !== 'Done') { return true; } - if (status.appData !== 'Running' && status.appData !== 'Done') { + if (status.appImage !== 'Running' && status.appImage !== 'Done') { return true; } if (status.panelData !== 'Running' && status.panelData !== 'Done') { diff --git a/frontend/src/views/setting/snapshot/status/index.vue b/frontend/src/views/setting/snapshot/status/index.vue index b49ba8f5742b..4816fe4aa0bb 100644 --- a/frontend/src/views/setting/snapshot/status/index.vue +++ b/frontend/src/views/setting/snapshot/status/index.vue @@ -1,295 +1,220 @@ - - From c6153d0b48075dbdd899eefd9eff283d4162e046 Mon Sep 17 00:00:00 2001 From: ssongliu Date: Mon, 23 Sep 2024 11:45:52 +0800 Subject: [PATCH 04/12] =?UTF-8?q?feat:=20=E5=BF=AB=E7=85=A7=E8=BF=87?= =?UTF-8?q?=E7=A8=8B=E5=A2=9E=E5=8A=A0=E4=BB=BB=E5=8A=A1=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/package.json | 2 +- frontend/src/lang/modules/en.ts | 20 +- frontend/src/lang/modules/tw.ts | 20 +- frontend/src/lang/modules/zh.ts | 20 +- frontend/src/views/log/task/index.vue | 2 +- .../views/setting/snapshot/create/index.vue | 175 ++++++----- frontend/src/views/setting/snapshot/index.vue | 21 +- .../setting/snapshot/snap_status/index.vue | 295 ------------------ 8 files changed, 134 insertions(+), 421 deletions(-) delete mode 100644 frontend/src/views/setting/snapshot/snap_status/index.vue diff --git a/frontend/package.json b/frontend/package.json index 625c38e40bd9..d911425c0def 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -38,7 +38,7 @@ "codemirror": "^6.0.1", "echarts": "^5.5.0", "element-plus": "^2.7.5", - "fit2cloud-ui-plus": "^1.1.5", + "fit2cloud-ui-plus": "^1.1.7", "highlight.js": "^11.9.0", "js-base64": "^3.7.7", "md-editor-v3": "^2.11.3", diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index 228457a31366..b62f600c2ac0 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -12,6 +12,8 @@ const message = { false: 'false', example: 'e.g.:', button: { + prev: 'Previous', + next: 'Next', create: 'Create ', add: 'Add ', save: 'Save ', @@ -1592,25 +1594,25 @@ const message = { stepPanelData: 'System Data', stepBackupData: 'Backup Data', stepOtherData: 'Other Data', - loginLog: 'Backup System Login Records', - OperationLog: 'Backup System Operation Log Login Records', - monitorData: 'Backup System Monitoring Data', + operationLog: 'Retain Operation Log', + loginLog: 'Retain Access Log', + systemLog: 'Retain System Log', + taskLog: 'Retain Task Log', + monitorData: 'Retain Monitoring Data', selectAllImage: 'Backup All Application Images', agentLabel: 'Node Configuration', appDataLabel: 'Application Data', appImage: 'Application Image', appBackup: 'Application Backup', backupLabel: 'Backup Directory', - cacheLabel: 'Cache Directory', confLabel: 'Configuration File', - dbLabel: 'Database Directory', - dockerLabel: 'Container Related Logs', - logLabel: 'Log Directory', - taskLabel: 'Scheduled Task Report', + dockerLabel: 'Container', + taskLabel: 'Scheduled Task', resourceLabel: 'Application Resource Directory', - runtimeLabel: 'Runtime Directory', + runtimeLabel: 'Runtime Environment', appLabel: 'Application', databaseLabel: 'Database', + snapshotLabel: 'Snapshot File', websiteLabel: 'Website', directoryLabel: 'Directory', appStoreLabel: 'Application Store', diff --git a/frontend/src/lang/modules/tw.ts b/frontend/src/lang/modules/tw.ts index e5c230ec709d..224c992ffa35 100644 --- a/frontend/src/lang/modules/tw.ts +++ b/frontend/src/lang/modules/tw.ts @@ -11,6 +11,8 @@ const message = { false: '否', example: '例：', button: { + prev: '上一步', + next: '下一步', create: '創建', add: '添加', save: '保存', @@ -1410,25 +1412,25 @@ const message = { stepPanelData: '系統數據', stepBackupData: '備份數據', stepOtherData: '其他數據', - loginLog: '備份系統登錄記錄', - OperationLog: '備份系統操作日誌登錄記錄', - monitorData: '備份系統監控數據', + operationLog: '保留操作日誌', + loginLog: '保留訪問日誌', + systemLog: '保留系統日誌', + taskLog: '保留任務日誌', + monitorData: '保留監控數據', selectAllImage: '備份所有應用鏡像', agentLabel: '節點配置', appDataLabel: '應用數據', appImage: '應用鏡像', appBackup: '應用備份', backupLabel: '備份目錄', - cacheLabel: '緩存目錄', confLabel: '配置文件', - dbLabel: '數據庫目錄', - dockerLabel: '容器相關日誌', - logLabel: '日誌目錄', - taskLabel: '計劃任務報告', + dockerLabel: '容器', + taskLabel: '計劃任務', resourceLabel: '應用資源目錄', - runtimeLabel: '運行時目錄', + runtimeLabel: '運行環境', appLabel: '應用', databaseLabel: '數據庫', + snapshotLabel: '快照文件', websiteLabel: '網站', directoryLabel: '目錄', appStoreLabel: '應用商店', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index 8edc7639f851..5e53cd6c9cea 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -11,6 +11,8 @@ const message = { false: '否', example: '例：', button: { + prev: '上一步', + next: '下一步', create: '创建', add: '添加', save: '保存', @@ -1412,25 +1414,25 @@ const message = { stepPanelData: '系统数据', stepBackupData: '备份数据', stepOtherData: '其他数据', - loginLog: '备份系统登陆记录', - OperationLog: '备份系统备份系统操作日志登陆记录', - monitorData: '备份系统监控数据', + operationLog: '保留操作日志', + loginLog: '保留访问日志', + systemLog: '保留系统日志', + taskLog: '保留任务日志', + monitorData: '保留监控数据', selectAllImage: '备份所有应用镜像', agentLabel: '节点配置', appDataLabel: '应用数据', appImage: '应用镜像', appBackup: '应用备份', backupLabel: '备份目录', - cacheLabel: '缓存目录', confLabel: '配置文件', - dbLabel: '数据库目录', - dockerLabel: '容器相关日志', - logLabel: '日志目录', - taskLabel: '计划任务报告', + dockerLabel: '容器', + taskLabel: '计划任务', resourceLabel: '应用资源目录', - runtimeLabel: '运行时目录', + runtimeLabel: '运行环境', appLabel: '应用', databaseLabel: '数据库', + snapshotLabel: '快照文件', websiteLabel: '网站', directoryLabel: '目录', appStoreLabel: '应用商店', diff --git a/frontend/src/views/log/task/index.vue b/frontend/src/views/log/task/index.vue index a0125ee0e48b..cae7031bbb1e 100644 --- a/frontend/src/views/log/task/index.vue +++ b/frontend/src/views/log/task/index.vue @@ -55,7 +55,7 @@ - +
diff --git a/frontend/src/views/setting/snapshot/create/index.vue b/frontend/src/views/setting/snapshot/create/index.vue index d8f84144ddec..243d99ae7f81 100644 --- a/frontend/src/views/setting/snapshot/create/index.vue +++ b/frontend/src/views/setting/snapshot/create/index.vue @@ -2,13 +2,12 @@ @@ -54,100 +53,96 @@ -
- - - - -
+ + + + -
- - - -
+ + + -
- - - -
+ + + -
- -
-
- -
-
- -
+ + + + + + - + - From f16b2844b5ed84871d3d63b80b9be3694e0484e6 Mon Sep 17 00:00:00 2001 From: ssonglius11 Date: Mon, 23 Sep 2024 11:51:26 +0800 Subject: [PATCH 05/12] =?UTF-8?q?feat:=20=E4=BF=AE=E6=94=B9=E5=BF=AB?= =?UTF-8?q?=E7=85=A7=E7=8A=B6=E6=80=81=E6=98=BE=E7=A4=BA=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/app/api/v2/snapshot.go | 22 -- agent/app/dto/snapshot.go | 5 + agent/app/model/snapshot.go | 16 +- agent/app/repo/snapshot.go | 36 --- agent/app/service/cronjob_backup.go | 2 +- agent/app/service/snapshot.go | 269 +-------------- agent/app/service/snapshot_create.go | 413 ++++++++++++++++++------ agent/app/service/snapshot_recover.go | 18 +- agent/app/task/task.go | 19 +- agent/i18n/lang/en.yaml | 25 ++ agent/i18n/lang/zh-Hant.yaml | 25 ++ agent/i18n/lang/zh.yaml | 27 +- agent/init/hook/hook.go | 34 -- agent/init/migration/migrations/init.go | 3 +- agent/router/ro_setting.go | 1 - 15 files changed, 439 insertions(+), 476 deletions(-) diff --git a/agent/app/api/v2/snapshot.go b/agent/app/api/v2/snapshot.go index 88f8d4543208..7b52bfb9cb8b 100644 --- a/agent/app/api/v2/snapshot.go +++ b/agent/app/api/v2/snapshot.go @@ -66,28 +66,6 @@ func (b *BaseApi) ImportSnapshot(c *gin.Context) { helper.SuccessWithData(c, nil) } -// @Tags System Setting -// @Summary Load Snapshot status -// @Description 获取快照状态 -// @Accept json -// @Param request body dto.OperateByID true "request" -// @Success 200 -// @Security ApiKeyAuth -// @Router /settings/snapshot/status [post] -func (b *BaseApi) LoadSnapShotStatus(c *gin.Context) { - var req dto.OperateByID - if err := helper.CheckBindAndValidate(&req, c); err != nil { - return - } - - data, err := snapshotService.LoadSnapShotStatus(req.ID) - if err != nil { - helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) - return - } - helper.SuccessWithData(c, data) -} - // @Tags System Setting // @Summary Update snapshot description // @Description 更新快照描述信息 diff --git a/agent/app/dto/snapshot.go b/agent/app/dto/snapshot.go index 78bdf481330f..36100ae72d84 100644 --- a/agent/app/dto/snapshot.go +++ b/agent/app/dto/snapshot.go @@ -15,6 +15,7 @@ type SnapshotStatus struct { type SnapshotCreate struct { ID uint `json:"id"` + TaskID string `json:"taskID"` SourceAccountIDs string `json:"sourceAccountIDs" validate:"required"` DownloadAccountID uint `json:"downloadAccountID" validate:"required"` Description string `json:"description" validate:"max=256"` @@ -27,6 +28,8 @@ type SnapshotCreate struct { WithMonitorData bool `json:"withMonitorData"` WithLoginLog bool `json:"withLoginLog"` WithOperationLog bool `json:"withOperationLog"` + WithSystemLog bool `json:"withSystemLog"` + WithTaskLog bool `json:"withTaskLog"` } type SnapshotData struct { @@ -37,6 +40,8 @@ type SnapshotData struct { WithMonitorData bool `json:"withMonitorData"` WithLoginLog bool `json:"withLoginLog"` WithOperationLog bool `json:"withOperationLog"` + WithSystemLog bool `json:"withSystemLog"` + WithTaskLog bool `json:"withTaskLog"` } type DataTree struct { ID string `json:"id"` diff --git a/agent/app/model/snapshot.go b/agent/app/model/snapshot.go index bf8cc3b43718..b697362fb8cf 100644 --- a/agent/app/model/snapshot.go +++ b/agent/app/model/snapshot.go @@ -16,6 +16,8 @@ type Snapshot struct { WithMonitorData bool `json:"withMonitorData"` WithLoginLog bool `json:"withLoginLog"` WithOperationLog bool `json:"withOperationLog"` + WithSystemLog bool `json:"withSystemLog"` + WithTaskLog bool `json:"withTaskLog"` InterruptStep string `json:"interruptStep"` RecoverStatus string `json:"recoverStatus"` @@ -25,17 +27,3 @@ type Snapshot struct { RollbackMessage string `json:"rollbackMessage"` LastRollbackAt string `json:"lastRollbackAt"` } - -type SnapshotStatus struct { - BaseModel - SnapID uint `json:"snapID"` - - BaseData string `json:"baseData" gorm:"default:Running"` - AppImage string `json:"appImage" gorm:"default:Running"` - PanelData string `json:"panelData" gorm:"default:Running"` - BackupData string `json:"backupData" gorm:"default:Running"` - - Compress string `json:"compress" gorm:"default:Waiting"` - Size string `json:"size" ` - Upload string `json:"upload" gorm:"default:Waiting"` -} diff --git a/agent/app/repo/snapshot.go b/agent/app/repo/snapshot.go index dcb3dd6063a8..289470178113 100644 --- a/agent/app/repo/snapshot.go +++ b/agent/app/repo/snapshot.go @@ -12,12 +12,6 @@ type ISnapshotRepo interface { Update(id uint, vars map[string]interface{}) error Page(limit, offset int, opts ...DBOption) (int64, []model.Snapshot, error) Delete(opts ...DBOption) error - - GetStatus(snapID uint) (model.SnapshotStatus, error) - GetStatusList(opts ...DBOption) ([]model.SnapshotStatus, error) - CreateStatus(snap *model.SnapshotStatus) error - DeleteStatus(snapID uint) error - UpdateStatus(id uint, vars map[string]interface{}) error } func NewISnapshotRepo() ISnapshotRepo { @@ -73,33 +67,3 @@ func (u *SnapshotRepo) Delete(opts ...DBOption) error { } return db.Delete(&model.Snapshot{}).Error } - -func (u *SnapshotRepo) GetStatus(snapID uint) (model.SnapshotStatus, error) { - var data model.SnapshotStatus - if err := global.DB.Where("snap_id = ?", snapID).First(&data).Error; err != nil { - return data, err - } - return data, nil -} - -func (u *SnapshotRepo) GetStatusList(opts ...DBOption) ([]model.SnapshotStatus, error) { - var status []model.SnapshotStatus - db := global.DB.Model(&model.SnapshotStatus{}) - for _, opt := range opts { - db = opt(db) - } - err := db.Find(&status).Error - return status, err -} - -func (u *SnapshotRepo) CreateStatus(snap *model.SnapshotStatus) error { - return global.DB.Create(snap).Error -} - -func (u *SnapshotRepo) DeleteStatus(snapID uint) error { - return global.DB.Where("snap_id = ?", snapID).Delete(&model.SnapshotStatus{}).Error -} - -func (u *SnapshotRepo) UpdateStatus(id uint, vars map[string]interface{}) error { - return global.DB.Model(&model.SnapshotStatus{}).Where("id = ?", id).Updates(vars).Error -} diff --git a/agent/app/service/cronjob_backup.go b/agent/app/service/cronjob_backup.go index 33e6d98da1d1..d7bef53fff34 100644 --- a/agent/app/service/cronjob_backup.go +++ b/agent/app/service/cronjob_backup.go @@ -210,7 +210,7 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Ti SourceAccountIDs: record.SourceAccountIDs, DownloadAccountID: cronjob.DownloadAccountID, } - name, err := NewISnapshotService().HandleSnapshot(true, logPath, req, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5), cronjob.Secret) + name, err := NewISnapshotService().HandleSnapshot(true, req, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5), cronjob.Secret) if err != nil { return err } diff --git a/agent/app/service/snapshot.go b/agent/app/service/snapshot.go index abd0293d1415..8d37b13b6a73 100644 --- a/agent/app/service/snapshot.go +++ b/agent/app/service/snapshot.go @@ -2,22 +2,18 @@ package service import ( "context" - "encoding/json" "fmt" "os" "path" "strings" "sync" - "time" "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/utils/cmd" - "github.com/1Panel-dev/1Panel/agent/utils/compose" "github.com/1Panel-dev/1Panel/agent/utils/docker" - "github.com/1Panel-dev/1Panel/agent/utils/files" fileUtils "github.com/1Panel-dev/1Panel/agent/utils/files" "github.com/docker/docker/api/types/image" "github.com/google/uuid" @@ -39,12 +35,9 @@ type ISnapshotService interface { SnapshotImport(req dto.SnapshotImport) error Delete(req dto.SnapshotBatchDelete) error - LoadSnapShotStatus(id uint) (*dto.SnapshotStatus, error) - UpdateDescription(req dto.UpdateDescription) error - readFromJson(path string) (SnapshotJson, error) - HandleSnapshot(isCronjob bool, logPath string, req dto.SnapshotCreate, timeNow string, secret string) (string, error) + HandleSnapshot(isCronjob bool, req dto.SnapshotCreate, timeNow string, secret string) (string, error) } func NewISnapshotService() ISnapshotService { @@ -118,12 +111,14 @@ func (u *SnapshotService) LoadSnapshotData() (dto.SnapshotData, error) { for i, item := range itemBackups { if item.Label == "app" { data.BackupData = append(itemBackups[:i], itemBackups[i+1:]...) - break + } + if item.Label == "system_snapshot" { + itemBackups[i].IsCheck = false + for j := 0; j < len(item.Children); j++ { + itemBackups[i].Children[j].IsCheck = false + } } } - data.WithLoginLog = true - data.WithOperationLog = true - data.WithMonitorData = false return data, nil } @@ -132,31 +127,12 @@ func (u *SnapshotService) UpdateDescription(req dto.UpdateDescription) error { return snapshotRepo.Update(req.ID, map[string]interface{}{"description": req.Description}) } -func (u *SnapshotService) LoadSnapShotStatus(id uint) (*dto.SnapshotStatus, error) { - var data dto.SnapshotStatus - status, err := snapshotRepo.GetStatus(id) - if err != nil { - return nil, err - } - if err := copier.Copy(&data, &status); err != nil { - return nil, errors.WithMessage(constant.ErrStructTransform, err.Error()) - } - return &data, nil -} - type SnapshotJson struct { BaseDir string `json:"baseDir"` BackupDataDir string `json:"backupDataDir"` Size uint64 `json:"size"` } -func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate) error { - if _, err := u.HandleSnapshot(false, "", req, time.Now().Format(constant.DateTimeSlimLayout), req.Secret); err != nil { - return err - } - return nil -} - func (u *SnapshotService) SnapshotRecover(req dto.SnapshotRecover) error { global.LOG.Info("start to recover panel by snapshot now") snap, err := snapshotRepo.Get(commonRepo.WithByID(req.ID)) @@ -195,156 +171,6 @@ func (u *SnapshotService) SnapshotRollback(req dto.SnapshotRecover) error { return nil } -func (u *SnapshotService) readFromJson(path string) (SnapshotJson, error) { - var snap SnapshotJson - if _, err := os.Stat(path); err != nil { - return snap, fmt.Errorf("find snapshot json file in recover package failed, err: %v", err) - } - fileByte, err := os.ReadFile(path) - if err != nil { - return snap, fmt.Errorf("read file from path %s failed, err: %v", path, err) - } - if err := json.Unmarshal(fileByte, &snap); err != nil { - return snap, fmt.Errorf("unmarshal snapjson failed, err: %v", err) - } - return snap, nil -} - -func (u *SnapshotService) HandleSnapshot(isCronjob bool, logPath string, req dto.SnapshotCreate, timeNow string, secret string) (string, error) { - var ( - rootDir string - snap model.Snapshot - snapStatus model.SnapshotStatus - err error - ) - - if req.ID == 0 { - versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion")) - - name := fmt.Sprintf("1panel-%s-linux-%s-%s", versionItem.Value, loadOs(), timeNow) - if isCronjob { - name = fmt.Sprintf("snapshot-1panel-%s-linux-%s-%s", versionItem.Value, loadOs(), timeNow) - } - rootDir = path.Join(global.CONF.System.BaseDir, "1panel/tmp/system", name) - - appItem, _ := json.Marshal(req.AppData) - panelItem, _ := json.Marshal(req.PanelData) - backupItem, _ := json.Marshal(req.BackupData) - snap = model.Snapshot{ - Name: name, - Description: req.Description, - SourceAccountIDs: req.SourceAccountIDs, - DownloadAccountID: req.DownloadAccountID, - - AppData: string(appItem), - PanelData: string(panelItem), - BackupData: string(backupItem), - WithMonitorData: req.WithMonitorData, - WithLoginLog: req.WithLoginLog, - WithOperationLog: req.WithOperationLog, - - Version: versionItem.Value, - Status: constant.StatusWaiting, - } - _ = snapshotRepo.Create(&snap) - snapStatus.SnapID = snap.ID - _ = snapshotRepo.CreateStatus(&snapStatus) - } else { - snap, err = snapshotRepo.Get(commonRepo.WithByID(req.ID)) - if err != nil { - return "", err - } - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusWaiting}) - snapStatus, _ = snapshotRepo.GetStatus(snap.ID) - if snapStatus.ID == 0 { - snapStatus.SnapID = snap.ID - _ = snapshotRepo.CreateStatus(&snapStatus) - } - rootDir = path.Join(global.CONF.System.BaseDir, "1panel/tmp/system", snap.Name) - } - - var wg sync.WaitGroup - itemHelper := snapHelper{SnapID: snap.ID, Status: &snapStatus, Wg: &wg, FileOp: files.NewFileOp(), Ctx: context.Background()} - baseDir := path.Join(rootDir, "base") - _ = os.MkdirAll(baseDir, os.ModePerm) - if err := loadDbConn(&itemHelper, rootDir, req); err != nil { - return "", fmt.Errorf("load snapshot db conn failed, err: %v", err) - } - - loadLogByStatus(snapStatus, logPath) - if snapStatus.BaseData != constant.StatusDone { - wg.Add(1) - go snapBaseData(itemHelper, baseDir) - } - if snapStatus.AppImage != constant.StatusDone { - wg.Add(1) - go snapAppImage(itemHelper, req, rootDir) - } - if snapStatus.BackupData != constant.StatusDone { - wg.Add(1) - go snapBackupData(itemHelper, req, rootDir) - } - if snapStatus.PanelData != constant.StatusDone { - wg.Add(1) - go snapPanelData(itemHelper, req, rootDir) - } - if !isCronjob { - go func() { - wg.Wait() - closeDatabase(itemHelper.snapAgentDB) - closeDatabase(itemHelper.snapCoreDB) - if !checkIsAllDone(snap.ID) { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - return - } - if snapStatus.Compress != constant.StatusDone { - snapCompress(itemHelper, rootDir, secret) - } - if snapStatus.Compress != constant.StatusDone { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - return - } - if snapStatus.Upload != constant.StatusDone { - snapUpload(itemHelper, req.SourceAccountIDs, fmt.Sprintf("%s.tar.gz", rootDir)) - } - if snapStatus.Upload != constant.StatusDone { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - return - } - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess}) - // _ = snapshotRepo.DeleteStatus(itemHelper.SnapID) - _ = os.RemoveAll(rootDir) - }() - return "", nil - } - wg.Wait() - closeDatabase(itemHelper.snapAgentDB) - closeDatabase(itemHelper.snapCoreDB) - if !checkIsAllDone(snap.ID) { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - loadLogByStatus(snapStatus, logPath) - return snap.Name, fmt.Errorf("snapshot %s backup failed", snap.Name) - } - loadLogByStatus(snapStatus, logPath) - snapCompress(itemHelper, rootDir, secret) - if snapStatus.Compress != constant.StatusDone { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - loadLogByStatus(snapStatus, logPath) - return snap.Name, fmt.Errorf("snapshot %s compress failed", snap.Name) - } - loadLogByStatus(snapStatus, logPath) - snapUpload(itemHelper, req.SourceAccountIDs, fmt.Sprintf("%s.tar.gz", rootDir)) - if snapStatus.Upload != constant.StatusDone { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - loadLogByStatus(snapStatus, logPath) - return snap.Name, fmt.Errorf("snapshot %s upload failed", snap.Name) - } - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess}) - loadLogByStatus(snapStatus, logPath) - _ = os.RemoveAll(rootDir) - return snap.Name, nil -} - func (u *SnapshotService) Delete(req dto.SnapshotBatchDelete) error { snaps, _ := snapshotRepo.GetList(commonRepo.WithByIDs(req.Ids)) for _, snap := range snaps { @@ -359,7 +185,6 @@ func (u *SnapshotService) Delete(req dto.SnapshotBatchDelete) error { } } - _ = snapshotRepo.DeleteStatus(snap.ID) if err := snapshotRepo.Delete(commonRepo.WithByID(snap.ID)); err != nil { return err } @@ -367,82 +192,6 @@ func (u *SnapshotService) Delete(req dto.SnapshotBatchDelete) error { return nil } -func rebuildAllAppInstall() error { - global.LOG.Debug("start to rebuild all app") - appInstalls, err := appInstallRepo.ListBy() - if err != nil { - global.LOG.Errorf("get all app installed for rebuild failed, err: %v", err) - return err - } - var wg sync.WaitGroup - for i := 0; i < len(appInstalls); i++ { - wg.Add(1) - appInstalls[i].Status = constant.Rebuilding - _ = appInstallRepo.Save(context.Background(), &appInstalls[i]) - go func(app model.AppInstall) { - defer wg.Done() - dockerComposePath := app.GetComposePath() - out, err := compose.Down(dockerComposePath) - if err != nil { - _ = handleErr(app, err, out) - return - } - out, err = compose.Up(dockerComposePath) - if err != nil { - _ = handleErr(app, err, out) - return - } - app.Status = constant.Running - _ = appInstallRepo.Save(context.Background(), &app) - }(appInstalls[i]) - } - wg.Wait() - return nil -} - -func checkIsAllDone(snapID uint) bool { - status, err := snapshotRepo.GetStatus(snapID) - if err != nil { - return false - } - isOK, _ := checkAllDone(status) - return isOK -} - -func checkAllDone(status model.SnapshotStatus) (bool, string) { - if status.BaseData != constant.StatusDone { - return false, status.BaseData - } - if status.PanelData != constant.StatusDone { - return false, status.PanelData - } - if status.AppImage != constant.StatusDone { - return false, status.AppImage - } - if status.BackupData != constant.StatusDone { - return false, status.BackupData - } - return true, "" -} - -func loadLogByStatus(status model.SnapshotStatus, logPath string) { - logs := "" - logs += fmt.Sprintf("Backup 1Panel base files: %s \n", status.BaseData) - logs += fmt.Sprintf("Backup installed apps from 1Panel: %s \n", status.AppImage) - logs += fmt.Sprintf("Backup 1Panel data directory: %s \n", status.PanelData) - logs += fmt.Sprintf("Backup local backup directory for 1Panel: %s \n", status.BackupData) - logs += fmt.Sprintf("Create snapshot file: %s \n", status.Compress) - logs += fmt.Sprintf("Snapshot size: %s \n", status.Size) - logs += fmt.Sprintf("Upload snapshot file: %s \n", status.Upload) - - file, err := os.OpenFile(logPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) - if err != nil { - return - } - defer file.Close() - _, _ = file.Write([]byte(logs)) -} - func hasOs(name string) bool { return strings.Contains(name, "amd64") || strings.Contains(name, "arm64") || @@ -642,9 +391,9 @@ func loadPanelFile(fileOp fileUtils.FileOp) ([]dto.DataTree, error) { Path: path.Join(global.CONF.System.BaseDir, "1panel", fileItem.Name()), } switch itemData.Label { - case "agent", "conf", "db", "runtime", "secret": + case "agent", "conf", "runtime", "docker", "secret", "task": itemData.IsDisable = true - case "log", "docker", "task", "clamav": + case "clamav": panelPath := path.Join(global.CONF.System.BaseDir, "1panel", itemData.Label) itemData.Children, _ = loadFile(panelPath, 5, fileOp) default: diff --git a/agent/app/service/snapshot_create.go b/agent/app/service/snapshot_create.go index a2fc319835c7..a056b2ed775b 100644 --- a/agent/app/service/snapshot_create.go +++ b/agent/app/service/snapshot_create.go @@ -12,84 +12,274 @@ import ( "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/app/task" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/i18n" "github.com/1Panel-dev/1Panel/agent/utils/cmd" "github.com/1Panel-dev/1Panel/agent/utils/common" + "github.com/1Panel-dev/1Panel/agent/utils/compose" "github.com/1Panel-dev/1Panel/agent/utils/files" "github.com/glebarez/sqlite" + "github.com/pkg/errors" "gorm.io/gorm" ) +func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate) error { + if _, err := u.HandleSnapshot(false, req, time.Now().Format(constant.DateTimeSlimLayout), req.Secret); err != nil { + return err + } + return nil +} + +func (u *SnapshotService) HandleSnapshot(isCronjob bool, req dto.SnapshotCreate, timeNow string, secret string) (string, error) { + var ( + rootDir string + taskItem *task.Task + snap model.Snapshot + err error + ) + + if req.ID == 0 { + versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion")) + + name := fmt.Sprintf("1panel-%s-linux-%s-%s", versionItem.Value, loadOs(), timeNow) + if isCronjob { + name = fmt.Sprintf("snapshot-1panel-%s-linux-%s-%s", versionItem.Value, loadOs(), timeNow) + } + rootDir = path.Join(global.CONF.System.BaseDir, "1panel/tmp/system", name) + + appItem, _ := json.Marshal(req.AppData) + panelItem, _ := json.Marshal(req.PanelData) + backupItem, _ := json.Marshal(req.BackupData) + snap = model.Snapshot{ + Name: name, + Description: req.Description, + SourceAccountIDs: req.SourceAccountIDs, + DownloadAccountID: req.DownloadAccountID, + + AppData: string(appItem), + PanelData: string(panelItem), + BackupData: string(backupItem), + WithMonitorData: req.WithMonitorData, + WithLoginLog: req.WithLoginLog, + WithOperationLog: req.WithOperationLog, + WithTaskLog: req.WithTaskLog, + WithSystemLog: req.WithSystemLog, + + Version: versionItem.Value, + Status: constant.StatusWaiting, + } + if err := snapshotRepo.Create(&snap); err != nil { + global.LOG.Errorf("create snapshot record to db failed, err: %v", err) + return "", err + } + + taskItem, err = task.NewTaskWithOps(name, task.TaskCreate, task.TaskScopeSnapshot, req.TaskID, snap.ID) + if err != nil { + global.LOG.Errorf("new task for create snapshot failed, err: %v", err) + return "", err + } + } else { + snap, err = snapshotRepo.Get(commonRepo.WithByID(req.ID)) + if err != nil { + return "", err + } + taskModel, err := taskRepo.GetFirst(taskRepo.WithResourceID(snap.ID), commonRepo.WithByType(task.TaskScopeSnapshot)) + if err != nil { + return "", err + } + taskItem, err = task.NewTaskWithOps(snap.Name, task.TaskCreate, task.TaskScopeSnapshot, taskModel.ID, snap.ID) + if err != nil { + global.LOG.Errorf("new task for create snapshot failed, err: %v", err) + return "", err + } + rootDir = path.Join(global.CONF.System.BaseDir, "1panel/tmp/system", snap.Name) + } + + itemHelper := snapHelper{SnapID: snap.ID, Task: *taskItem, FileOp: files.NewFileOp(), Ctx: context.Background()} + baseDir := path.Join(rootDir, "base") + _ = os.MkdirAll(baseDir, os.ModePerm) + + go func() { + taskItem.AddSubTask( + i18n.GetMsgByKey("SnapDBInfo"), + func(t *task.Task) error { return loadDbConn(&itemHelper, rootDir, req) }, + nil, + ) + + if len(snap.InterruptStep) == 0 || snap.InterruptStep == "SnapBaseInfo" { + taskItem.AddSubTask( + i18n.GetMsgByKey("SnapBaseInfo"), + func(t *task.Task) error { return snapBaseData(itemHelper, baseDir) }, + nil, + ) + snap.InterruptStep = "" + } + if len(snap.InterruptStep) == 0 || snap.InterruptStep == "SnapInstallApp" { + taskItem.AddSubTask( + i18n.GetMsgByKey("SnapInstallApp"), + func(t *task.Task) error { return snapAppImage(itemHelper, req, rootDir) }, + nil, + ) + snap.InterruptStep = "" + } + if len(snap.InterruptStep) == 0 || snap.InterruptStep == "SnapLocalBackup" { + taskItem.AddSubTask( + i18n.GetMsgByKey("SnapLocalBackup"), + func(t *task.Task) error { return snapBackupData(itemHelper, req, rootDir) }, + nil, + ) + snap.InterruptStep = "" + } + if len(snap.InterruptStep) == 0 || snap.InterruptStep == "SnapPanelData" { + taskItem.AddSubTask( + i18n.GetMsgByKey("SnapPanelData"), + func(t *task.Task) error { return snapPanelData(itemHelper, req, rootDir) }, + nil, + ) + snap.InterruptStep = "" + } + + taskItem.AddSubTask( + i18n.GetMsgByKey("SnapCloseDBConn"), + func(t *task.Task) error { + taskItem.Log("<######################## 6 / 8 ########################>") + closeDatabase(itemHelper.snapAgentDB) + closeDatabase(itemHelper.snapCoreDB) + return nil + }, + nil, + ) + if len(snap.InterruptStep) == 0 || snap.InterruptStep == "SnapCompress" { + taskItem.AddSubTask( + i18n.GetMsgByKey("SnapCompress"), + func(t *task.Task) error { return snapCompress(itemHelper, rootDir, secret) }, + nil, + ) + snap.InterruptStep = "" + } + if len(snap.InterruptStep) == 0 || snap.InterruptStep == "SnapUpload" { + taskItem.AddSubTask( + i18n.GetMsgByKey("SnapUpload"), + func(t *task.Task) error { + return snapUpload(itemHelper, req.SourceAccountIDs, fmt.Sprintf("%s.tar.gz", rootDir)) + }, + nil, + ) + snap.InterruptStep = "" + } + if err := taskItem.Execute(); err != nil { + _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed, "message": err.Error(), "interrupt_step": taskItem.Task.CurrentStep}) + return + } + _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess, "interrupt_step": ""}) + _ = os.RemoveAll(rootDir) + }() + return snap.Name, nil +} + type snapHelper struct { SnapID uint snapAgentDB *gorm.DB snapCoreDB *gorm.DB - Status *model.SnapshotStatus Ctx context.Context FileOp files.FileOp Wg *sync.WaitGroup + Task task.Task } func loadDbConn(snap *snapHelper, targetDir string, req dto.SnapshotCreate) error { - global.LOG.Debug("start load snapshot db conn") + snap.Task.Log("<######################## 1 / 8 ########################>") + snap.Task.LogStart(i18n.GetMsgByKey("SnapDBInfo")) + pathDB := path.Join(global.CONF.System.BaseDir, "1panel/db") - if err := snap.FileOp.CopyDir(path.Join(global.CONF.System.BaseDir, "1panel/db"), targetDir); err != nil { + err := snap.FileOp.CopyDir(pathDB, targetDir) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", pathDB), err) + if err != nil { return err } + agentDb, err := newSnapDB(path.Join(targetDir, "db"), "agent.db") + snap.Task.LogWithStatus(i18n.GetWithName("SnapNewDB", "agent"), err) if err != nil { return err } snap.snapAgentDB = agentDb coreDb, err := newSnapDB(path.Join(targetDir, "db"), "core.db") + snap.Task.LogWithStatus(i18n.GetWithName("SnapNewDB", "core"), err) if err != nil { return err } snap.snapCoreDB = coreDb if !req.WithMonitorData { - _ = os.Remove(path.Join(targetDir, "db/monitor.db")) + err = os.Remove(path.Join(targetDir, "db/monitor.db")) + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapDeleteMonitor"), err) + if err != nil { + return err + } } if !req.WithOperationLog { - _ = snap.snapCoreDB.Exec("DELETE FROM operation_logs") + err = snap.snapCoreDB.Exec("DELETE FROM operation_logs").Error + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapDeleteOperationLog"), err) + if err != nil { + return err + } } if !req.WithLoginLog { - _ = snap.snapCoreDB.Exec("DELETE FROM login_logs") - } - if err := snap.snapAgentDB.Where("id = ?", snap.SnapID).Delete(&model.Snapshot{}).Error; err != nil { - global.LOG.Errorf("delete current snapshot record failed, err: %v", err) - } - if err := snap.snapAgentDB.Where("snap_id = ?", snap.SnapID).Delete(&model.SnapshotStatus{}).Error; err != nil { - global.LOG.Errorf("delete current snapshot status record failed, err: %v", err) + err = snap.snapCoreDB.Exec("DELETE FROM login_logs").Error + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapDeleteLoginLog"), err) + if err != nil { + return err + } } + _ = snap.snapAgentDB.Model(&model.Setting{}).Where("key = ?", "SystemIP").Updates(map[string]interface{}{"value": ""}).Error + _ = snap.snapAgentDB.Where("id = ?", snap.SnapID).Delete(&model.Snapshot{}).Error + return nil } -func snapBaseData(snap snapHelper, targetDir string) { - defer snap.Wg.Done() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"base_data": constant.Running}) - status := constant.StatusDone - if err := common.CopyFile("/usr/local/bin/1panel", targetDir); err != nil { - status = err.Error() +func snapBaseData(snap snapHelper, targetDir string) error { + snap.Task.Log("<######################## 2 / 8 ########################>") + snap.Task.LogStart(i18n.GetMsgByKey("SnapBaseInfo")) + + err := common.CopyFile("/usr/local/bin/1panel", targetDir) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel"), err) + if err != nil { + return err } - if err := common.CopyFile("/usr/local/bin/1panel_agent", targetDir); err != nil { - status = err.Error() + + err = common.CopyFile("/usr/local/bin/1panel_agent", targetDir) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel_agent"), err) + if err != nil { + return err } - if err := common.CopyFile("/usr/local/bin/1pctl", targetDir); err != nil { - status = err.Error() + + err = common.CopyFile("/usr/local/bin/1pctl", targetDir) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1pctl"), err) + if err != nil { + return err } - if err := common.CopyFile("/etc/systemd/system/1panel.service", targetDir); err != nil { - status = err.Error() + + err = common.CopyFile("/etc/systemd/system/1panel.service", targetDir) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel.service"), err) + if err != nil { + return err } - if err := common.CopyFile("/etc/systemd/system/1panel_agent.service", targetDir); err != nil { - status = err.Error() + + err = common.CopyFile("/etc/systemd/system/1panel_agent.service", targetDir) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel_agent.service"), err) + if err != nil { + return err } if snap.FileOp.Stat("/etc/docker/daemon.json") { - if err := common.CopyFile("/etc/docker/daemon.json", targetDir); err != nil { - status = err.Error() + err = common.CopyFile("/etc/docker/daemon.json", targetDir) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/docker/daemon.json"), err) + if err != nil { + return err } } @@ -97,16 +287,18 @@ func snapBaseData(snap snapHelper, targetDir string) { BaseDir: global.CONF.System.BaseDir, BackupDataDir: global.CONF.System.Backup, }, "", "\t") - if err := os.WriteFile(fmt.Sprintf("%s/snapshot.json", targetDir), remarkInfo, 0640); err != nil { - status = err.Error() + err = os.WriteFile(path.Join(targetDir, "snapshot.json"), remarkInfo, 0640) + snap.Task.LogWithStatus(i18n.GetWithName("SnapCopy", path.Join(targetDir, "snapshot.json")), err) + if err != nil { + return err } - snap.Status.BaseData = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"base_data": status}) + + return nil } -func snapAppImage(snap snapHelper, req dto.SnapshotCreate, targetDir string) { - defer snap.Wg.Done() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_image": constant.Running}) +func snapAppImage(snap snapHelper, req dto.SnapshotCreate, targetDir string) error { + snap.Task.Log("<######################## 3 / 8 ########################>") + snap.Task.LogStart(i18n.GetMsgByKey("SnapInstallApp")) var imageList []string existStr, _ := cmd.Exec("docker images | awk '{print $1\":\"$2}' | grep -v REPOSITORY:TAG") @@ -124,22 +316,19 @@ func snapAppImage(snap snapHelper, req dto.SnapshotCreate, targetDir string) { } if len(imageList) != 0 { - global.LOG.Debugf("docker save %s | gzip -c > %s", strings.Join(imageList, " "), path.Join(targetDir, "images.tar.gz")) + snap.Task.Logf("docker save %s | gzip -c > %s", strings.Join(imageList, " "), path.Join(targetDir, "images.tar.gz")) std, err := cmd.Execf("docker save %s | gzip -c > %s", strings.Join(imageList, " "), path.Join(targetDir, "images.tar.gz")) + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapDockerSave"), errors.New(std)) if err != nil { - snap.Status.AppImage = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_image": std}) - return + return errors.New(std) } } - snap.Status.AppImage = constant.StatusDone - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_image": constant.StatusDone}) + return nil } -func snapBackupData(snap snapHelper, req dto.SnapshotCreate, targetDir string) { - defer snap.Wg.Done() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"backup_data": constant.Running}) - status := constant.StatusDone +func snapBackupData(snap snapHelper, req dto.SnapshotCreate, targetDir string) error { + snap.Task.Log("<######################## 4 / 8 ########################>") + snap.Task.LogStart(i18n.GetMsgByKey("SnapLocalBackup")) excludes := loadBackupExcludes(snap, req.BackupData) for _, item := range req.AppData { @@ -149,13 +338,10 @@ func snapBackupData(snap snapHelper, req dto.SnapshotCreate, targetDir string) { } } } - global.LOG.Debugf("excludes backup file: %v\n", strings.Join(excludes, "\n")) + err := snap.FileOp.TarGzCompressPro(false, global.CONF.System.Backup, path.Join(targetDir, "1panel_backup.tar.gz"), "", strings.Join(excludes, ";")) + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapCompressBackup"), err) - if err := snap.FileOp.TarGzCompressPro(false, global.CONF.System.Backup, path.Join(targetDir, "1panel_backup.tar.gz"), "", strings.Join(excludes, ";")); err != nil { - status = err.Error() - } - snap.Status.BackupData = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"backup_data": status}) + return err } func loadBackupExcludes(snap snapHelper, req []dto.DataTree) []string { var excludes []string @@ -167,15 +353,14 @@ func loadBackupExcludes(snap snapHelper, req []dto.DataTree) []string { if strings.HasPrefix(item.Path, path.Join(global.CONF.System.Backup, "system_snapshot")) { fmt.Println(strings.TrimSuffix(item.Name, ".tar.gz")) if err := snap.snapAgentDB.Debug().Where("name = ? AND download_account_id = ?", strings.TrimSuffix(item.Name, ".tar.gz"), "1").Delete(&model.Snapshot{}).Error; err != nil { - global.LOG.Errorf("delete snapshot from database failed, err: %v", err) + snap.Task.LogWithStatus("delete snapshot from database", err) } } else { fmt.Println(strings.TrimPrefix(path.Dir(item.Path), global.CONF.System.Backup+"/"), path.Base(item.Path)) if err := snap.snapAgentDB.Debug().Where("file_dir = ? AND file_name = ?", strings.TrimPrefix(path.Dir(item.Path), global.CONF.System.Backup+"/"), path.Base(item.Path)).Delete(&model.BackupRecord{}).Error; err != nil { - global.LOG.Errorf("delete backup file from database failed, err: %v", err) + snap.Task.LogWithStatus("delete backup file from database", err) } } - fmt.Println(strings.TrimPrefix(item.Path, global.CONF.System.Backup)) excludes = append(excludes, "."+strings.TrimPrefix(item.Path, global.CONF.System.Backup)) } else { excludes = append(excludes, loadBackupExcludes(snap, item.Children)...) @@ -197,10 +382,9 @@ func loadAppBackupExcludes(req []dto.DataTree) []string { return excludes } -func snapPanelData(snap snapHelper, req dto.SnapshotCreate, targetDir string) { - defer snap.Wg.Done() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_data": constant.Running}) - status := constant.StatusDone +func snapPanelData(snap snapHelper, req dto.SnapshotCreate, targetDir string) error { + snap.Task.Log("<######################## 5 / 8 ########################>") + snap.Task.LogStart(i18n.GetMsgByKey("SnapPanelData")) excludes := loadPanelExcludes(req.PanelData) for _, item := range req.AppData { @@ -210,12 +394,21 @@ func snapPanelData(snap snapHelper, req dto.SnapshotCreate, targetDir string) { } } } - global.LOG.Debugf("excludes panel file: %v\n", strings.Join(excludes, "\n")) excludes = append(excludes, "./tmp") excludes = append(excludes, "./cache") excludes = append(excludes, "./uploads") excludes = append(excludes, "./db") excludes = append(excludes, "./resource") + if !req.WithSystemLog { + excludes = append(excludes, "./log/1Panel*") + } + if !req.WithTaskLog { + excludes = append(excludes, "./log/App") + excludes = append(excludes, "./log/Snapshot") + excludes = append(excludes, "./log/AppStore") + excludes = append(excludes, "./log/Website") + } + rootDir := path.Join(global.CONF.System.BaseDir, "1panel") if strings.Contains(global.CONF.System.Backup, rootDir) { excludes = append(excludes, "."+strings.ReplaceAll(global.CONF.System.Backup, rootDir, "")) @@ -228,15 +421,10 @@ func snapPanelData(snap snapHelper, req dto.SnapshotCreate, targetDir string) { } excludes = append(excludes, "."+strings.ReplaceAll(ignore, rootDir, "")) } + err := snap.FileOp.TarGzCompressPro(false, rootDir, path.Join(targetDir, "1panel_data.tar.gz"), "", strings.Join(excludes, ";")) + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapCompressPanel"), err) - _ = snap.snapAgentDB.Model(&model.Setting{}).Where("key = ?", "SystemIP").Updates(map[string]interface{}{"value": ""}) - - if err := snap.FileOp.TarGzCompressPro(false, rootDir, path.Join(targetDir, "1panel_data.tar.gz"), "", strings.Join(excludes, ";")); err != nil { - status = err.Error() - } - - snap.Status.PanelData = status - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_data": status}) + return err } func loadPanelExcludes(req []dto.DataTree) []string { var excludes []string @@ -252,57 +440,52 @@ func loadPanelExcludes(req []dto.DataTree) []string { return excludes } -func snapCompress(snap snapHelper, rootDir string, secret string) { - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": constant.StatusRunning}) +func snapCompress(snap snapHelper, rootDir string, secret string) error { + snap.Task.Log("<######################## 7 / 8 ########################>") + snap.Task.LogStart(i18n.GetMsgByKey("SnapCompress")) + tmpDir := path.Join(global.CONF.System.TmpDir, "system") fileName := fmt.Sprintf("%s.tar.gz", path.Base(rootDir)) - if err := snap.FileOp.TarGzCompressPro(true, rootDir, path.Join(tmpDir, fileName), secret, ""); err != nil { - snap.Status.Compress = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": err.Error()}) - return + err := snap.FileOp.TarGzCompressPro(true, rootDir, path.Join(tmpDir, fileName), secret, "") + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapCompressFile"), err) + if err != nil { + return err } stat, err := os.Stat(path.Join(tmpDir, fileName)) + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapCheckCompress"), err) if err != nil { - snap.Status.Compress = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": err.Error()}) - return + return err } - size := common.LoadSizeUnit2F(float64(stat.Size())) - global.LOG.Debugf("compress successful! size of file: %s", size) - snap.Status.Compress = constant.StatusDone - snap.Status.Size = size - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": constant.StatusDone, "size": size}) - global.LOG.Debugf("remove snapshot file %s", rootDir) + size := common.LoadSizeUnit2F(float64(stat.Size())) + snap.Task.Logf(i18n.GetWithName("SnapCompressSize", size)) _ = os.RemoveAll(rootDir) + return nil } -func snapUpload(snap snapHelper, accounts string, file string) { +func snapUpload(snap snapHelper, accounts string, file string) error { + snap.Task.Log("<######################## 8 / 8 ########################>") + snap.Task.LogStart(i18n.GetMsgByKey("SnapUpload")) + source := path.Join(global.CONF.System.TmpDir, "system", path.Base(file)) - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": constant.StatusUploading}) accountMap, err := NewBackupClientMap(strings.Split(accounts, ",")) + snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapLoadBackup"), err) if err != nil { - snap.Status.Upload = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": err.Error()}) - return + return err } + targetAccounts := strings.Split(accounts, ",") for _, item := range targetAccounts { - global.LOG.Debugf("start upload snapshot to %s, path: %s", item, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))) - if _, err := accountMap[item].client.Upload(source, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))); err != nil { - global.LOG.Debugf("upload to %s failed, err: %v", accountMap[item].name, err) - snap.Status.Upload = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": err.Error()}) - return + snap.Task.LogStart(i18n.GetWithName("SnapUploadTo", fmt.Sprintf("[%s] %s", accountMap[item].name, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))))) + _, err := accountMap[item].client.Upload(source, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))) + snap.Task.LogWithStatus(i18n.GetWithName("SnapUploadRes", accountMap[item].name), err) + if err != nil { + return err } - global.LOG.Debugf("upload to %s successful", accountMap[item].name) } - snap.Status.Upload = constant.StatusDone - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": constant.StatusDone}) - - global.LOG.Debugf("remove snapshot file %s", source) _ = os.Remove(source) + return nil } func newSnapDB(dir, file string) (*gorm.DB, error) { @@ -316,7 +499,6 @@ func newSnapDB(dir, file string) (*gorm.DB, error) { sqlDB.SetConnMaxIdleTime(10) sqlDB.SetMaxOpenConns(100) sqlDB.SetConnMaxLifetime(time.Hour) - // global.LOG.Debug("load snapshot db conn successful!") return db, nil } @@ -327,3 +509,36 @@ func closeDatabase(db *gorm.DB) { } _ = sqlDB.Close() } + +func rebuildAllAppInstall() error { + global.LOG.Debug("start to rebuild all app") + appInstalls, err := appInstallRepo.ListBy() + if err != nil { + global.LOG.Errorf("get all app installed for rebuild failed, err: %v", err) + return err + } + var wg sync.WaitGroup + for i := 0; i < len(appInstalls); i++ { + wg.Add(1) + appInstalls[i].Status = constant.Rebuilding + _ = appInstallRepo.Save(context.Background(), &appInstalls[i]) + go func(app model.AppInstall) { + defer wg.Done() + dockerComposePath := app.GetComposePath() + out, err := compose.Down(dockerComposePath) + if err != nil { + _ = handleErr(app, err, out) + return + } + out, err = compose.Up(dockerComposePath) + if err != nil { + _ = handleErr(app, err, out) + return + } + app.Status = constant.Running + _ = appInstallRepo.Save(context.Background(), &app) + }(appInstalls[i]) + } + wg.Wait() + return nil +} diff --git a/agent/app/service/snapshot_recover.go b/agent/app/service/snapshot_recover.go index cc17e01dbbc3..b2f4756a2fe1 100644 --- a/agent/app/service/snapshot_recover.go +++ b/agent/app/service/snapshot_recover.go @@ -1,6 +1,7 @@ package service import ( + "encoding/json" "fmt" "os" "path" @@ -55,7 +56,7 @@ func (u *SnapshotService) HandleSnapshotRecover(snap model.Snapshot, req dto.Sna if _, err := os.Stat(snapFileDir); err != nil { snapFileDir = baseDir } - snapJson, err := u.readFromJson(path.Join(snapFileDir, "base/snapshot.json")) + snapJson, err := readFromJson(path.Join(snapFileDir, "base/snapshot.json")) if err != nil { updateRecoverStatus(snap.ID, "Readjson", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) return @@ -285,3 +286,18 @@ func updateRecoverStatus(id uint, interruptStep, status, message string) { } _ = settingRepo.Update("SystemStatus", "Free") } + +func readFromJson(path string) (SnapshotJson, error) { + var snap SnapshotJson + if _, err := os.Stat(path); err != nil { + return snap, fmt.Errorf("find snapshot json file in recover package failed, err: %v", err) + } + fileByte, err := os.ReadFile(path) + if err != nil { + return snap, fmt.Errorf("read file from path %s failed, err: %v", path, err) + } + if err := json.Unmarshal(fileByte, &snap); err != nil { + return snap, fmt.Errorf("unmarshal snapjson failed, err: %v", err) + } + return snap, nil +} diff --git a/agent/app/task/task.go b/agent/app/task/task.go index cea2f77c8ead..2d2860cd1892 100644 --- a/agent/app/task/task.go +++ b/agent/app/task/task.go @@ -3,16 +3,17 @@ package task import ( "context" "fmt" - "github.com/1Panel-dev/1Panel/agent/app/model" - "github.com/1Panel-dev/1Panel/agent/app/repo" - "github.com/1Panel-dev/1Panel/agent/constant" - "github.com/1Panel-dev/1Panel/agent/i18n" - "github.com/google/uuid" "log" "os" "path" "strconv" "time" + + "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/app/repo" + "github.com/1Panel-dev/1Panel/agent/constant" + "github.com/1Panel-dev/1Panel/agent/i18n" + "github.com/google/uuid" ) type ActionFunc func(*Task) error @@ -60,6 +61,7 @@ const ( TaskScopeRuntime = "Runtime" TaskScopeDatabase = "Database" TaskScopeAppStore = "AppStore" + TaskScopeSnapshot = "Snapshot" TaskScopeRuntimeExtension = "RuntimeExtension" ) @@ -221,6 +223,10 @@ func (t *Task) Log(msg string) { t.Logger.Printf(msg) } +func (t *Task) Logf(format string, v ...any) { + t.Logger.Printf(format, v...) +} + func (t *Task) LogFailed(msg string) { t.Logger.Printf(msg + i18n.GetMsgByKey("Failed")) } @@ -232,6 +238,9 @@ func (t *Task) LogFailedWithErr(msg string, err error) { func (t *Task) LogSuccess(msg string) { t.Logger.Printf(msg + i18n.GetMsgByKey("Success")) } +func (t *Task) LogSuccessf(format string, v ...any) { + t.Logger.Printf(fmt.Sprintf(format, v...) + i18n.GetMsgByKey("Success")) +} func (t *Task) LogStart(msg string) { t.Logger.Printf(fmt.Sprintf("%s%s", i18n.GetMsgByKey("Start"), msg)) diff --git a/agent/i18n/lang/en.yaml b/agent/i18n/lang/en.yaml index 995c326ff013..613c6c79b2f7 100644 --- a/agent/i18n/lang/en.yaml +++ b/agent/i18n/lang/en.yaml @@ -247,3 +247,28 @@ SubTask: "Subtask" RuntimeExtension: "Runtime Extension" TaskBuild: "Build" +# task - snapshot +Snapshot: "Snapshot" +SnapDBInfo: "Write 1Panel database information" +SnapCopy: "Copy files & directories {{ .name }} " +SnapNewDB: "Initialize database {{ .name }} connection " +SnapDeleteOperationLog: "Delete operation log" +SnapDeleteLoginLog: "Delete access log" +SnapDeleteMonitor: "Delete monitoring data" +SnapRemoveSystemIP: "Remove system IP" +SnapBaseInfo: "Write 1Panel basic information" +SnapInstallApp: "Backup installed applications in 1Panel" +SnapDockerSave: "Compress installed applications" +SnapLocalBackup: "Backup 1Panel local backup directory" +SnapCompressBackup: "Compress local backup directory" +SnapPanelData: "Backup 1Panel data directory" +SnapCompressPanel: "Compress data directory" +SnapCloseDBConn: "Close database connection" +SnapCompress: "Create snapshot file" +SnapCompressFile: "Compress snapshot file" +SnapCheckCompress: "Check snapshot compressed file" +SnapCompressSize: "Snapshot file size {{ .name }}" +SnapUpload: "Upload snapshot file" +SnapLoadBackup: "Get backup account information" +SnapUploadTo: "Upload snapshot file to {{ .name }}" +SnapUploadRes: "Upload snapshot file to {{ .name }}" \ No newline at end of file diff --git a/agent/i18n/lang/zh-Hant.yaml b/agent/i18n/lang/zh-Hant.yaml index 259d156df503..e750a5d9d9f2 100644 --- a/agent/i18n/lang/zh-Hant.yaml +++ b/agent/i18n/lang/zh-Hant.yaml @@ -250,3 +250,28 @@ RuntimeExtension: "運行環境擴展" TaskBuild: "構建" +# task - snapshot +Snapshot: "快照" +SnapDBInfo: "寫入 1Panel 資料庫資訊" +SnapCopy: "複製檔案&目錄 {{ .name }} " +SnapNewDB: "初始化資料庫 {{ .name }} 連接 " +SnapDeleteOperationLog: "刪除操作日誌" +SnapDeleteLoginLog: "刪除訪問日誌" +SnapDeleteMonitor: "刪除監控數據" +SnapRemoveSystemIP: "移除系統 IP" +SnapBaseInfo: "寫入 1Panel 基本資訊" +SnapInstallApp: "備份 1Panel 已安裝應用" +SnapDockerSave: "壓縮已安裝應用" +SnapLocalBackup: "備份 1Panel 本地備份目錄" +SnapCompressBackup: "壓縮本地備份目錄" +SnapPanelData: "備份 1Panel 資料目錄" +SnapCompressPanel: "壓縮資料目錄" +SnapCloseDBConn: "關閉資料庫連接" +SnapCompress: "製作快照檔案" +SnapCompressFile: "壓縮快照檔案" +SnapCheckCompress: "檢查快照壓縮檔案" +SnapCompressSize: "快照檔案大小 {{ .name }}" +SnapUpload: "上傳快照檔案" +SnapLoadBackup: "獲取備份帳號資訊" +SnapUploadTo: "上傳快照檔案到 {{ .name }}" +SnapUploadRes: "上傳快照檔案到 {{ .name }}" \ No newline at end of file diff --git a/agent/i18n/lang/zh.yaml b/agent/i18n/lang/zh.yaml index 7d5b63fc2049..9e1622c49117 100644 --- a/agent/i18n/lang/zh.yaml +++ b/agent/i18n/lang/zh.yaml @@ -249,4 +249,29 @@ TaskSync: "同步" LocalApp: "本地应用" SubTask: "子任务" RuntimeExtension: "运行环境扩展" -TaskBuild: "构建" \ No newline at end of file + +# task - snapshot +Snapshot: "快照" +SnapDBInfo: "写入 1Panel 数据库信息" +SnapCopy: "复制文件&目录 {{ .name }} " +SnapNewDB: "初始化数据库 {{ .name }} 连接 " +SnapDeleteOperationLog: "删除操作日志" +SnapDeleteLoginLog: "删除访问日志" +SnapDeleteMonitor: "删除监控数据" +SnapRemoveSystemIP: "移除系统 IP" +SnapBaseInfo: "写入 1Panel 基本信息" +SnapInstallApp: "备份 1Panel 已安装应用" +SnapDockerSave: "压缩已安装应用" +SnapLocalBackup: "备份 1Panel 本地备份目录" +SnapCompressBackup: "压缩本地备份目录" +SnapPanelData: "备份 1Panel 数据目录" +SnapCompressPanel: "压缩数据目录" +SnapCloseDBConn: "关闭数据库连接" +SnapCompress: "制作快照文件" +SnapCompressFile: "压缩快照文件" +SnapCheckCompress: "检查快照压缩文件" +SnapCompressSize: "快照文件大小 {{ .name }}" +SnapUpload: "上传快照文件" +SnapLoadBackup: "获取备份账号信息" +SnapUploadTo: "上传快照文件到 {{ .name }}" +SnapUploadRes: "上传快照文件到 {{ .name }}" diff --git a/agent/init/hook/hook.go b/agent/init/hook/hook.go index 7cc33863abd2..9ec2926523f9 100644 --- a/agent/init/hook/hook.go +++ b/agent/init/hook/hook.go @@ -66,40 +66,6 @@ func handleSnapStatus() { "rollback_status": constant.StatusFailed, "rollback_message": msgFailed, }).Error - - snapRepo := repo.NewISnapshotRepo() - - status, _ := snapRepo.GetStatusList() - for _, item := range status { - updates := make(map[string]interface{}) - // if item.Panel == constant.StatusRunning { - // updates["panel"] = constant.StatusFailed - // } - // if item.PanelInfo == constant.StatusRunning { - // updates["panel_info"] = constant.StatusFailed - // } - // if item.DaemonJson == constant.StatusRunning { - // updates["daemon_json"] = constant.StatusFailed - // } - // if item.AppData == constant.StatusRunning { - // updates["app_data"] = constant.StatusFailed - // } - if item.PanelData == constant.StatusRunning { - updates["panel_data"] = constant.StatusFailed - } - if item.BackupData == constant.StatusRunning { - updates["backup_data"] = constant.StatusFailed - } - if item.Compress == constant.StatusRunning { - updates["compress"] = constant.StatusFailed - } - if item.Upload == constant.StatusUploading { - updates["upload"] = constant.StatusFailed - } - if len(updates) != 0 { - _ = snapRepo.UpdateStatus(item.ID, updates) - } - } } func handleCronjobStatus() { diff --git a/agent/init/migration/migrations/init.go b/agent/init/migration/migrations/init.go index b959b29b33c2..fef797f1f0bc 100644 --- a/agent/init/migration/migrations/init.go +++ b/agent/init/migration/migrations/init.go @@ -46,7 +46,6 @@ var AddTable = &gormigrate.Migration{ &model.Runtime{}, &model.Setting{}, &model.Snapshot{}, - &model.SnapshotStatus{}, &model.Tag{}, &model.Website{}, &model.WebsiteAcmeAccount{}, @@ -262,7 +261,7 @@ var UpdateAppInstall = &gormigrate.Migration{ } var UpdateSnapshot = &gormigrate.Migration{ - ID: "20240913-update-snapshot", + ID: "20240923-update-snapshot", Migrate: func(tx *gorm.DB) error { return tx.AutoMigrate(&model.Snapshot{}) }, diff --git a/agent/router/ro_setting.go b/agent/router/ro_setting.go index ec956ccacbaa..8384d7e69b32 100644 --- a/agent/router/ro_setting.go +++ b/agent/router/ro_setting.go @@ -17,7 +17,6 @@ func (s *SettingRouter) InitRouter(Router *gin.RouterGroup) { settingRouter.GET("/snapshot/load", baseApi.LoadSnapshotData) settingRouter.POST("/snapshot", baseApi.CreateSnapshot) - settingRouter.POST("/snapshot/status", baseApi.LoadSnapShotStatus) settingRouter.POST("/snapshot/search", baseApi.SearchSnapshot) settingRouter.POST("/snapshot/import", baseApi.ImportSnapshot) settingRouter.POST("/snapshot/del", baseApi.DeleteSnapshot) From 14e7531fdcc39043670a8802b3e984ba589b5c1e Mon Sep 17 00:00:00 2001 From: ssonglius11 Date: Mon, 23 Sep 2024 18:01:05 +0800 Subject: [PATCH 06/12] =?UTF-8?q?feat:=20=E5=BF=AB=E7=85=A7=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=A4=B1=E8=B4=A5=E9=87=8D=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/app/api/v2/snapshot.go | 22 ++++ agent/app/dto/snapshot.go | 3 + agent/app/model/snapshot.go | 2 + agent/app/repo/task.go | 7 +- agent/app/service/cronjob_backup.go | 9 +- agent/app/service/snapshot.go | 3 +- agent/app/service/snapshot_create.go | 161 ++++++++++++------------ agent/app/task/task.go | 2 +- agent/init/migration/migrations/init.go | 2 +- agent/router/ro_setting.go | 1 + 10 files changed, 125 insertions(+), 87 deletions(-) diff --git a/agent/app/api/v2/snapshot.go b/agent/app/api/v2/snapshot.go index 7b52bfb9cb8b..8bca5bf5b4c7 100644 --- a/agent/app/api/v2/snapshot.go +++ b/agent/app/api/v2/snapshot.go @@ -44,6 +44,28 @@ func (b *BaseApi) CreateSnapshot(c *gin.Context) { helper.SuccessWithData(c, nil) } +// @Tags System Setting +// @Summary Recreate system snapshot +// @Description 创建系统快照重试 +// @Accept json +// @Param request body dto.OperateByID true "request" +// @Success 200 +// @Security ApiKeyAuth +// @Router /settings/snapshot/recrete [post] +// @x-panel-log {"bodyKeys":["id"],"paramKeys":[],"BeforeFunctions":[{"input_column":"id","input_value":"id","isList":false,"db":"snapshots","output_column":"name","output_value":"name"}],"formatZH":"重试创建快照 [name]","formatEN":recrete the snapshot [name]"} +func (b *BaseApi) RecreateSnapshot(c *gin.Context) { + var req dto.OperateByID + if err := helper.CheckBindAndValidate(&req, c); err != nil { + return + } + + if err := snapshotService.SnapshotReCreate(req.ID); err != nil { + helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) + return + } + helper.SuccessWithData(c, nil) +} + // @Tags System Setting // @Summary Import system snapshot // @Description 导入已有快照 diff --git a/agent/app/dto/snapshot.go b/agent/app/dto/snapshot.go index 36100ae72d84..125c61b2e431 100644 --- a/agent/app/dto/snapshot.go +++ b/agent/app/dto/snapshot.go @@ -15,11 +15,13 @@ type SnapshotStatus struct { type SnapshotCreate struct { ID uint `json:"id"` + Name string `json:"name"` TaskID string `json:"taskID"` SourceAccountIDs string `json:"sourceAccountIDs" validate:"required"` DownloadAccountID uint `json:"downloadAccountID" validate:"required"` Description string `json:"description" validate:"max=256"` Secret string `json:"secret"` + InterruptStep string `json:"interruptStep"` AppData []DataTree `json:"appData"` BackupData []DataTree `json:"backupData"` @@ -77,6 +79,7 @@ type SnapshotImport struct { type SnapshotInfo struct { ID uint `json:"id"` Name string `json:"name"` + TaskID string `json:"taskID"` Description string `json:"description" validate:"max=256"` From string `json:"from"` DefaultDownload string `json:"defaultDownload"` diff --git a/agent/app/model/snapshot.go b/agent/app/model/snapshot.go index b697362fb8cf..ad8627ef37e9 100644 --- a/agent/app/model/snapshot.go +++ b/agent/app/model/snapshot.go @@ -3,6 +3,8 @@ package model type Snapshot struct { BaseModel Name string `json:"name" gorm:"not null;unique"` + TaskID string `json:"taskID"` + Secret string `json:"secret"` Description string `json:"description"` SourceAccountIDs string `json:"sourceAccountIDs"` DownloadAccountID uint `json:"downloadAccountID"` diff --git a/agent/app/repo/task.go b/agent/app/repo/task.go index 65f07ebcf779..b91c2afcc3eb 100644 --- a/agent/app/repo/task.go +++ b/agent/app/repo/task.go @@ -2,6 +2,7 @@ package repo import ( "context" + "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" @@ -13,7 +14,7 @@ type TaskRepo struct { } type ITaskRepo interface { - Create(ctx context.Context, task *model.Task) error + Save(ctx context.Context, task *model.Task) error GetFirst(opts ...DBOption) (model.Task, error) Page(page, size int, opts ...DBOption) (int64, []model.Task, error) Update(ctx context.Context, task *model.Task) error @@ -64,8 +65,8 @@ func (t TaskRepo) WithResourceID(id uint) DBOption { } } -func (t TaskRepo) Create(ctx context.Context, task *model.Task) error { - return getTaskTx(ctx).Create(&task).Error +func (t TaskRepo) Save(ctx context.Context, task *model.Task) error { + return getTaskTx(ctx).Save(&task).Error } func (t TaskRepo) GetFirst(opts ...DBOption) (model.Task, error) { diff --git a/agent/app/service/cronjob_backup.go b/agent/app/service/cronjob_backup.go index d7bef53fff34..42922bee99a0 100644 --- a/agent/app/service/cronjob_backup.go +++ b/agent/app/service/cronjob_backup.go @@ -206,15 +206,18 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Ti record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs record.FileDir = "system_snapshot" + versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion")) req := dto.SnapshotCreate{ + Name: fmt.Sprintf("snapshot-1panel-%s-linux-%s-%s", versionItem.Value, loadOs(), startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5)), + Secret: cronjob.Secret, + SourceAccountIDs: record.SourceAccountIDs, DownloadAccountID: cronjob.DownloadAccountID, } - name, err := NewISnapshotService().HandleSnapshot(true, req, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5), cronjob.Secret) - if err != nil { + if err := NewISnapshotService().HandleSnapshot(req); err != nil { return err } - record.FileName = name + ".tar.gz" + record.FileName = req.Name + ".tar.gz" if err := backupRepo.CreateRecord(&record); err != nil { global.LOG.Errorf("save backup record failed, err: %v", err) diff --git a/agent/app/service/snapshot.go b/agent/app/service/snapshot.go index 8d37b13b6a73..aaa1dc1b8818 100644 --- a/agent/app/service/snapshot.go +++ b/agent/app/service/snapshot.go @@ -30,6 +30,7 @@ type ISnapshotService interface { SearchWithPage(req dto.SearchWithPage) (int64, interface{}, error) LoadSnapshotData() (dto.SnapshotData, error) SnapshotCreate(req dto.SnapshotCreate) error + SnapshotReCreate(id uint) error SnapshotRecover(req dto.SnapshotRecover) error SnapshotRollback(req dto.SnapshotRecover) error SnapshotImport(req dto.SnapshotImport) error @@ -37,7 +38,7 @@ type ISnapshotService interface { UpdateDescription(req dto.UpdateDescription) error - HandleSnapshot(isCronjob bool, req dto.SnapshotCreate, timeNow string, secret string) (string, error) + HandleSnapshot(req dto.SnapshotCreate) error } func NewISnapshotService() ISnapshotService { diff --git a/agent/app/service/snapshot_create.go b/agent/app/service/snapshot_create.go index a056b2ed775b..21c4c87cb01d 100644 --- a/agent/app/service/snapshot_create.go +++ b/agent/app/service/snapshot_create.go @@ -19,6 +19,7 @@ import ( "github.com/1Panel-dev/1Panel/agent/utils/cmd" "github.com/1Panel-dev/1Panel/agent/utils/common" "github.com/1Panel-dev/1Panel/agent/utils/compose" + "github.com/1Panel-dev/1Panel/agent/utils/copier" "github.com/1Panel-dev/1Panel/agent/utils/files" "github.com/glebarez/sqlite" "github.com/pkg/errors" @@ -26,78 +27,82 @@ import ( ) func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate) error { - if _, err := u.HandleSnapshot(false, req, time.Now().Format(constant.DateTimeSlimLayout), req.Secret); err != nil { + versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion")) + + req.Name = fmt.Sprintf("1panel-%s-linux-%s-%s", versionItem.Value, loadOs(), time.Now().Format(constant.DateTimeSlimLayout)) + appItem, _ := json.Marshal(req.AppData) + panelItem, _ := json.Marshal(req.PanelData) + backupItem, _ := json.Marshal(req.BackupData) + snap := model.Snapshot{ + Name: req.Name, + TaskID: req.TaskID, + Secret: req.Secret, + Description: req.Description, + SourceAccountIDs: req.SourceAccountIDs, + DownloadAccountID: req.DownloadAccountID, + + AppData: string(appItem), + PanelData: string(panelItem), + BackupData: string(backupItem), + WithMonitorData: req.WithMonitorData, + WithLoginLog: req.WithLoginLog, + WithOperationLog: req.WithOperationLog, + WithTaskLog: req.WithTaskLog, + WithSystemLog: req.WithSystemLog, + + Version: versionItem.Value, + Status: constant.StatusWaiting, + } + if err := snapshotRepo.Create(&snap); err != nil { + global.LOG.Errorf("create snapshot record to db failed, err: %v", err) + return err + } + + req.ID = snap.ID + if err := u.HandleSnapshot(req); err != nil { return err } return nil } -func (u *SnapshotService) HandleSnapshot(isCronjob bool, req dto.SnapshotCreate, timeNow string, secret string) (string, error) { - var ( - rootDir string - taskItem *task.Task - snap model.Snapshot - err error - ) +func (u *SnapshotService) SnapshotReCreate(id uint) error { + snap, err := snapshotRepo.Get(commonRepo.WithByID(id)) + if err != nil { + return err + } + taskModel, err := taskRepo.GetFirst(taskRepo.WithResourceID(snap.ID), commonRepo.WithByType(task.TaskScopeSnapshot)) + if err != nil { + return err + } - if req.ID == 0 { - versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion")) + var req dto.SnapshotCreate + _ = copier.Copy(&req, snap) + if err := json.Unmarshal([]byte(snap.PanelData), &req.PanelData); err != nil { + return err + } + if err := json.Unmarshal([]byte(snap.AppData), &req.AppData); err != nil { + return err + } + if err := json.Unmarshal([]byte(snap.BackupData), &req.BackupData); err != nil { + return err + } + req.TaskID = taskModel.ID + if err := u.HandleSnapshot(req); err != nil { + return err + } - name := fmt.Sprintf("1panel-%s-linux-%s-%s", versionItem.Value, loadOs(), timeNow) - if isCronjob { - name = fmt.Sprintf("snapshot-1panel-%s-linux-%s-%s", versionItem.Value, loadOs(), timeNow) - } - rootDir = path.Join(global.CONF.System.BaseDir, "1panel/tmp/system", name) - - appItem, _ := json.Marshal(req.AppData) - panelItem, _ := json.Marshal(req.PanelData) - backupItem, _ := json.Marshal(req.BackupData) - snap = model.Snapshot{ - Name: name, - Description: req.Description, - SourceAccountIDs: req.SourceAccountIDs, - DownloadAccountID: req.DownloadAccountID, - - AppData: string(appItem), - PanelData: string(panelItem), - BackupData: string(backupItem), - WithMonitorData: req.WithMonitorData, - WithLoginLog: req.WithLoginLog, - WithOperationLog: req.WithOperationLog, - WithTaskLog: req.WithTaskLog, - WithSystemLog: req.WithSystemLog, - - Version: versionItem.Value, - Status: constant.StatusWaiting, - } - if err := snapshotRepo.Create(&snap); err != nil { - global.LOG.Errorf("create snapshot record to db failed, err: %v", err) - return "", err - } + return nil +} - taskItem, err = task.NewTaskWithOps(name, task.TaskCreate, task.TaskScopeSnapshot, req.TaskID, snap.ID) - if err != nil { - global.LOG.Errorf("new task for create snapshot failed, err: %v", err) - return "", err - } - } else { - snap, err = snapshotRepo.Get(commonRepo.WithByID(req.ID)) - if err != nil { - return "", err - } - taskModel, err := taskRepo.GetFirst(taskRepo.WithResourceID(snap.ID), commonRepo.WithByType(task.TaskScopeSnapshot)) - if err != nil { - return "", err - } - taskItem, err = task.NewTaskWithOps(snap.Name, task.TaskCreate, task.TaskScopeSnapshot, taskModel.ID, snap.ID) - if err != nil { - global.LOG.Errorf("new task for create snapshot failed, err: %v", err) - return "", err - } - rootDir = path.Join(global.CONF.System.BaseDir, "1panel/tmp/system", snap.Name) +func (u *SnapshotService) HandleSnapshot(req dto.SnapshotCreate) error { + taskItem, err := task.NewTaskWithOps(req.Name, task.TaskCreate, task.TaskScopeSnapshot, req.TaskID, req.ID) + if err != nil { + global.LOG.Errorf("new task for create snapshot failed, err: %v", err) + return err } - itemHelper := snapHelper{SnapID: snap.ID, Task: *taskItem, FileOp: files.NewFileOp(), Ctx: context.Background()} + rootDir := path.Join(global.CONF.System.BaseDir, "1panel/tmp/system", req.Name) + itemHelper := snapHelper{SnapID: req.ID, Task: *taskItem, FileOp: files.NewFileOp(), Ctx: context.Background()} baseDir := path.Join(rootDir, "base") _ = os.MkdirAll(baseDir, os.ModePerm) @@ -108,37 +113,37 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, req dto.SnapshotCreate, nil, ) - if len(snap.InterruptStep) == 0 || snap.InterruptStep == "SnapBaseInfo" { + if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapBaseInfo" { taskItem.AddSubTask( i18n.GetMsgByKey("SnapBaseInfo"), func(t *task.Task) error { return snapBaseData(itemHelper, baseDir) }, nil, ) - snap.InterruptStep = "" + req.InterruptStep = "" } - if len(snap.InterruptStep) == 0 || snap.InterruptStep == "SnapInstallApp" { + if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapInstallApp" { taskItem.AddSubTask( i18n.GetMsgByKey("SnapInstallApp"), func(t *task.Task) error { return snapAppImage(itemHelper, req, rootDir) }, nil, ) - snap.InterruptStep = "" + req.InterruptStep = "" } - if len(snap.InterruptStep) == 0 || snap.InterruptStep == "SnapLocalBackup" { + if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapLocalBackup" { taskItem.AddSubTask( i18n.GetMsgByKey("SnapLocalBackup"), func(t *task.Task) error { return snapBackupData(itemHelper, req, rootDir) }, nil, ) - snap.InterruptStep = "" + req.InterruptStep = "" } - if len(snap.InterruptStep) == 0 || snap.InterruptStep == "SnapPanelData" { + if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapPanelData" { taskItem.AddSubTask( i18n.GetMsgByKey("SnapPanelData"), func(t *task.Task) error { return snapPanelData(itemHelper, req, rootDir) }, nil, ) - snap.InterruptStep = "" + req.InterruptStep = "" } taskItem.AddSubTask( @@ -151,15 +156,15 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, req dto.SnapshotCreate, }, nil, ) - if len(snap.InterruptStep) == 0 || snap.InterruptStep == "SnapCompress" { + if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapCompress" { taskItem.AddSubTask( i18n.GetMsgByKey("SnapCompress"), - func(t *task.Task) error { return snapCompress(itemHelper, rootDir, secret) }, + func(t *task.Task) error { return snapCompress(itemHelper, rootDir, req.Secret) }, nil, ) - snap.InterruptStep = "" + req.InterruptStep = "" } - if len(snap.InterruptStep) == 0 || snap.InterruptStep == "SnapUpload" { + if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapUpload" { taskItem.AddSubTask( i18n.GetMsgByKey("SnapUpload"), func(t *task.Task) error { @@ -167,16 +172,16 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, req dto.SnapshotCreate, }, nil, ) - snap.InterruptStep = "" + req.InterruptStep = "" } if err := taskItem.Execute(); err != nil { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed, "message": err.Error(), "interrupt_step": taskItem.Task.CurrentStep}) + _ = snapshotRepo.Update(req.ID, map[string]interface{}{"status": constant.StatusFailed, "message": err.Error(), "interrupt_step": taskItem.Task.CurrentStep}) return } - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess, "interrupt_step": ""}) + _ = snapshotRepo.Update(req.ID, map[string]interface{}{"status": constant.StatusSuccess, "interrupt_step": ""}) _ = os.RemoveAll(rootDir) }() - return snap.Name, nil + return nil } type snapHelper struct { diff --git a/agent/app/task/task.go b/agent/app/task/task.go index 2d2860cd1892..d7970f99b692 100644 --- a/agent/app/task/task.go +++ b/agent/app/task/task.go @@ -168,7 +168,7 @@ func (t *Task) updateTask(task *model.Task) { } func (t *Task) Execute() error { - if err := t.taskRepo.Create(context.Background(), t.Task); err != nil { + if err := t.taskRepo.Save(context.Background(), t.Task); err != nil { return err } var err error diff --git a/agent/init/migration/migrations/init.go b/agent/init/migration/migrations/init.go index fef797f1f0bc..8b7793816952 100644 --- a/agent/init/migration/migrations/init.go +++ b/agent/init/migration/migrations/init.go @@ -261,7 +261,7 @@ var UpdateAppInstall = &gormigrate.Migration{ } var UpdateSnapshot = &gormigrate.Migration{ - ID: "20240923-update-snapshot", + ID: "20240925-update-snapshot", Migrate: func(tx *gorm.DB) error { return tx.AutoMigrate(&model.Snapshot{}) }, diff --git a/agent/router/ro_setting.go b/agent/router/ro_setting.go index 8384d7e69b32..9d2524bad55b 100644 --- a/agent/router/ro_setting.go +++ b/agent/router/ro_setting.go @@ -17,6 +17,7 @@ func (s *SettingRouter) InitRouter(Router *gin.RouterGroup) { settingRouter.GET("/snapshot/load", baseApi.LoadSnapshotData) settingRouter.POST("/snapshot", baseApi.CreateSnapshot) + settingRouter.POST("/snapshot/recreate", baseApi.RecreateSnapshot) settingRouter.POST("/snapshot/search", baseApi.SearchSnapshot) settingRouter.POST("/snapshot/import", baseApi.ImportSnapshot) settingRouter.POST("/snapshot/del", baseApi.DeleteSnapshot) From f849a3366b0eee3c612c4968007f7b658e780760 Mon Sep 17 00:00:00 2001 From: ssonglius11 Date: Tue, 24 Sep 2024 18:35:20 +0800 Subject: [PATCH 07/12] =?UTF-8?q?feat:=20=E5=BF=AB=E7=85=A7=E6=81=A2?= =?UTF-8?q?=E5=A4=8D=E8=BF=87=E7=A8=8B=E5=A2=9E=E5=8A=A0=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E8=AE=B0=E5=BD=95=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/app/dto/snapshot.go | 1 + agent/app/model/snapshot.go | 7 +- agent/app/service/snapshot.go | 38 -- agent/app/service/snapshot_create.go | 52 +-- agent/app/service/snapshot_recover.go | 510 ++++++++++++++---------- agent/app/service/snapshot_rollback.go | 58 +++ agent/app/task/task.go | 1 + agent/i18n/lang/zh.yaml | 27 ++ agent/init/migration/migrations/init.go | 2 +- 9 files changed, 409 insertions(+), 287 deletions(-) create mode 100644 agent/app/service/snapshot_rollback.go diff --git a/agent/app/dto/snapshot.go b/agent/app/dto/snapshot.go index 125c61b2e431..577fbbbfd298 100644 --- a/agent/app/dto/snapshot.go +++ b/agent/app/dto/snapshot.go @@ -63,6 +63,7 @@ type SnapshotRecover struct { IsNew bool `json:"isNew"` ReDownload bool `json:"reDownload"` ID uint `json:"id" validate:"required"` + TaskID string `json:"taskID"` Secret string `json:"secret"` } type SnapshotBatchDelete struct { diff --git a/agent/app/model/snapshot.go b/agent/app/model/snapshot.go index ad8627ef37e9..d0c15cd440b9 100644 --- a/agent/app/model/snapshot.go +++ b/agent/app/model/snapshot.go @@ -3,7 +3,6 @@ package model type Snapshot struct { BaseModel Name string `json:"name" gorm:"not null;unique"` - TaskID string `json:"taskID"` Secret string `json:"secret"` Description string `json:"description"` SourceAccountIDs string `json:"sourceAccountIDs"` @@ -12,6 +11,10 @@ type Snapshot struct { Message string `json:"message"` Version string `json:"version"` + TaskID string `json:"taskID"` + TaskRecoverID string `json:"taskRecoverID"` + TaskRollbackID string `json:"taskRollbackID"` + AppData string `json:"appData"` PanelData string `json:"panelData"` BackupData string `json:"backupData"` @@ -24,8 +27,6 @@ type Snapshot struct { InterruptStep string `json:"interruptStep"` RecoverStatus string `json:"recoverStatus"` RecoverMessage string `json:"recoverMessage"` - LastRecoveredAt string `json:"lastRecoveredAt"` RollbackStatus string `json:"rollbackStatus"` RollbackMessage string `json:"rollbackMessage"` - LastRollbackAt string `json:"lastRollbackAt"` } diff --git a/agent/app/service/snapshot.go b/agent/app/service/snapshot.go index aaa1dc1b8818..8122071ee402 100644 --- a/agent/app/service/snapshot.go +++ b/agent/app/service/snapshot.go @@ -134,44 +134,6 @@ type SnapshotJson struct { Size uint64 `json:"size"` } -func (u *SnapshotService) SnapshotRecover(req dto.SnapshotRecover) error { - global.LOG.Info("start to recover panel by snapshot now") - snap, err := snapshotRepo.Get(commonRepo.WithByID(req.ID)) - if err != nil { - return err - } - if hasOs(snap.Name) && !strings.Contains(snap.Name, loadOs()) { - return fmt.Errorf("restoring snapshots(%s) between different server architectures(%s) is not supported", snap.Name, loadOs()) - } - if !req.IsNew && len(snap.InterruptStep) != 0 && len(snap.RollbackStatus) != 0 { - return fmt.Errorf("the snapshot has been rolled back and cannot be restored again") - } - - baseDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("system/%s", snap.Name)) - if _, err := os.Stat(baseDir); err != nil && os.IsNotExist(err) { - _ = os.MkdirAll(baseDir, os.ModePerm) - } - - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"recover_status": constant.StatusWaiting}) - _ = settingRepo.Update("SystemStatus", "Recovering") - go u.HandleSnapshotRecover(snap, req) - return nil -} - -func (u *SnapshotService) SnapshotRollback(req dto.SnapshotRecover) error { - global.LOG.Info("start to rollback now") - snap, err := snapshotRepo.Get(commonRepo.WithByID(req.ID)) - if err != nil { - return err - } - go func() { - if err := handleRollback(snap.Name); err != nil { - global.LOG.Errorf("handle roll back snapshot failed, err: %v", err) - } - }() - return nil -} - func (u *SnapshotService) Delete(req dto.SnapshotBatchDelete) error { snaps, _ := snapshotRepo.GetList(commonRepo.WithByIDs(req.Ids)) for _, snap := range snaps { diff --git a/agent/app/service/snapshot_create.go b/agent/app/service/snapshot_create.go index 21c4c87cb01d..b5e5b5166f9d 100644 --- a/agent/app/service/snapshot_create.go +++ b/agent/app/service/snapshot_create.go @@ -18,7 +18,6 @@ import ( "github.com/1Panel-dev/1Panel/agent/i18n" "github.com/1Panel-dev/1Panel/agent/utils/cmd" "github.com/1Panel-dev/1Panel/agent/utils/common" - "github.com/1Panel-dev/1Panel/agent/utils/compose" "github.com/1Panel-dev/1Panel/agent/utils/copier" "github.com/1Panel-dev/1Panel/agent/utils/files" "github.com/glebarez/sqlite" @@ -149,7 +148,7 @@ func (u *SnapshotService) HandleSnapshot(req dto.SnapshotCreate) error { taskItem.AddSubTask( i18n.GetMsgByKey("SnapCloseDBConn"), func(t *task.Task) error { - taskItem.Log("<######################## 6 / 8 ########################>") + taskItem.Log("######################## 6 / 8 ########################") closeDatabase(itemHelper.snapAgentDB) closeDatabase(itemHelper.snapCoreDB) return nil @@ -195,7 +194,7 @@ type snapHelper struct { } func loadDbConn(snap *snapHelper, targetDir string, req dto.SnapshotCreate) error { - snap.Task.Log("<######################## 1 / 8 ########################>") + snap.Task.Log("######################## 1 / 8 ########################") snap.Task.LogStart(i18n.GetMsgByKey("SnapDBInfo")) pathDB := path.Join(global.CONF.System.BaseDir, "1panel/db") @@ -247,7 +246,7 @@ func loadDbConn(snap *snapHelper, targetDir string, req dto.SnapshotCreate) erro } func snapBaseData(snap snapHelper, targetDir string) error { - snap.Task.Log("<######################## 2 / 8 ########################>") + snap.Task.Log("######################## 2 / 8 ########################") snap.Task.LogStart(i18n.GetMsgByKey("SnapBaseInfo")) err := common.CopyFile("/usr/local/bin/1panel", targetDir) @@ -302,7 +301,7 @@ func snapBaseData(snap snapHelper, targetDir string) error { } func snapAppImage(snap snapHelper, req dto.SnapshotCreate, targetDir string) error { - snap.Task.Log("<######################## 3 / 8 ########################>") + snap.Task.Log("######################## 3 / 8 ########################") snap.Task.LogStart(i18n.GetMsgByKey("SnapInstallApp")) var imageList []string @@ -325,14 +324,16 @@ func snapAppImage(snap snapHelper, req dto.SnapshotCreate, targetDir string) err std, err := cmd.Execf("docker save %s | gzip -c > %s", strings.Join(imageList, " "), path.Join(targetDir, "images.tar.gz")) snap.Task.LogWithStatus(i18n.GetMsgByKey("SnapDockerSave"), errors.New(std)) if err != nil { + snap.Task.LogFailedWithErr(i18n.GetMsgByKey("SnapDockerSave"), errors.New(std)) return errors.New(std) } + snap.Task.LogSuccess(i18n.GetMsgByKey("SnapDockerSave")) } return nil } func snapBackupData(snap snapHelper, req dto.SnapshotCreate, targetDir string) error { - snap.Task.Log("<######################## 4 / 8 ########################>") + snap.Task.Log("######################## 4 / 8 ########################") snap.Task.LogStart(i18n.GetMsgByKey("SnapLocalBackup")) excludes := loadBackupExcludes(snap, req.BackupData) @@ -388,7 +389,7 @@ func loadAppBackupExcludes(req []dto.DataTree) []string { } func snapPanelData(snap snapHelper, req dto.SnapshotCreate, targetDir string) error { - snap.Task.Log("<######################## 5 / 8 ########################>") + snap.Task.Log("######################## 5 / 8 ########################") snap.Task.LogStart(i18n.GetMsgByKey("SnapPanelData")) excludes := loadPanelExcludes(req.PanelData) @@ -446,7 +447,7 @@ func loadPanelExcludes(req []dto.DataTree) []string { } func snapCompress(snap snapHelper, rootDir string, secret string) error { - snap.Task.Log("<######################## 7 / 8 ########################>") + snap.Task.Log("######################## 7 / 8 ########################") snap.Task.LogStart(i18n.GetMsgByKey("SnapCompress")) tmpDir := path.Join(global.CONF.System.TmpDir, "system") @@ -470,7 +471,7 @@ func snapCompress(snap snapHelper, rootDir string, secret string) error { } func snapUpload(snap snapHelper, accounts string, file string) error { - snap.Task.Log("<######################## 8 / 8 ########################>") + snap.Task.Log("######################## 8 / 8 ########################") snap.Task.LogStart(i18n.GetMsgByKey("SnapUpload")) source := path.Join(global.CONF.System.TmpDir, "system", path.Base(file)) @@ -514,36 +515,3 @@ func closeDatabase(db *gorm.DB) { } _ = sqlDB.Close() } - -func rebuildAllAppInstall() error { - global.LOG.Debug("start to rebuild all app") - appInstalls, err := appInstallRepo.ListBy() - if err != nil { - global.LOG.Errorf("get all app installed for rebuild failed, err: %v", err) - return err - } - var wg sync.WaitGroup - for i := 0; i < len(appInstalls); i++ { - wg.Add(1) - appInstalls[i].Status = constant.Rebuilding - _ = appInstallRepo.Save(context.Background(), &appInstalls[i]) - go func(app model.AppInstall) { - defer wg.Done() - dockerComposePath := app.GetComposePath() - out, err := compose.Down(dockerComposePath) - if err != nil { - _ = handleErr(app, err, out) - return - } - out, err = compose.Up(dockerComposePath) - if err != nil { - _ = handleErr(app, err, out) - return - } - app.Status = constant.Running - _ = appInstallRepo.Save(context.Background(), &app) - }(appInstalls[i]) - } - wg.Wait() - return nil -} diff --git a/agent/app/service/snapshot_recover.go b/agent/app/service/snapshot_recover.go index b2f4756a2fe1..c97b07381dcd 100644 --- a/agent/app/service/snapshot_recover.go +++ b/agent/app/service/snapshot_recover.go @@ -1,303 +1,407 @@ package service import ( + "context" "encoding/json" "fmt" "os" "path" "strings" - "time" + "sync" "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/app/task" "github.com/1Panel-dev/1Panel/agent/constant" "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/i18n" "github.com/1Panel-dev/1Panel/agent/utils/cmd" + "github.com/1Panel-dev/1Panel/agent/utils/compose" "github.com/1Panel-dev/1Panel/agent/utils/files" "github.com/pkg/errors" ) -func (u *SnapshotService) HandleSnapshotRecover(snap model.Snapshot, req dto.SnapshotRecover) { - _ = global.Cron.Stop() - defer func() { - global.Cron.Start() - }() +type snapRecoverHelper struct { + FileOp files.FileOp + Task *task.Task +} - fileOp := files.NewFileOp() - baseDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("system/%s", snap.Name)) - if _, err := os.Stat(baseDir); err != nil && os.IsNotExist(err) { - _ = os.MkdirAll(baseDir, os.ModePerm) +func (u *SnapshotService) SnapshotRecover(req dto.SnapshotRecover) error { + global.LOG.Info("start to recover panel by snapshot now") + snap, err := snapshotRepo.Get(commonRepo.WithByID(req.ID)) + if err != nil { + return err } - if req.IsNew || snap.InterruptStep == "Download" || req.ReDownload { - if err := handleDownloadSnapshot(snap, baseDir); err != nil { - updateRecoverStatus(snap.ID, "Download", constant.StatusFailed, err.Error()) - return - } - global.LOG.Debugf("download snapshot file to %s successful!", baseDir) - req.IsNew = true + if hasOs(snap.Name) && !strings.Contains(snap.Name, loadOs()) { + errInfo := fmt.Sprintf("restoring snapshots(%s) between different server architectures(%s) is not supported", snap.Name, loadOs()) + _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"recover_status": constant.StatusFailed, "recover_message": errInfo}) + return errors.New(errInfo) } - if req.IsNew || snap.InterruptStep == "Decompress" { - if err := fileOp.TarGzExtractPro(fmt.Sprintf("%s/%s.tar.gz", baseDir, snap.Name), baseDir, req.Secret); err != nil { - updateRecoverStatus(snap.ID, "Decompress", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) - return - } - global.LOG.Debug("decompress snapshot file successful!", baseDir) - req.IsNew = true + if len(snap.RollbackStatus) != 0 && snap.RollbackStatus != constant.StatusSuccess { + return fmt.Errorf("the snapshot has been rolled back and cannot be restored again") } - if req.IsNew || snap.InterruptStep == "Backup" { - if err := backupBeforeRecover(snap.Name); err != nil { - updateRecoverStatus(snap.ID, "Backup", constant.StatusFailed, fmt.Sprintf("handle backup before recover failed, err: %v", err)) - return - } - global.LOG.Debug("handle backup before recover successful!") + _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"recover_status": constant.StatusWaiting}) + _ = settingRepo.Update("SystemStatus", "Recovering") + + if len(snap.InterruptStep) == 0 { req.IsNew = true } - snapFileDir := fmt.Sprintf("%s/%s", baseDir, snap.Name) - if _, err := os.Stat(snapFileDir); err != nil { - snapFileDir = baseDir + if len(snap.TaskRecoverID) != 0 { + req.TaskID = snap.TaskRecoverID + } else { + _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"task_recover_id": req.TaskID}) } - snapJson, err := readFromJson(path.Join(snapFileDir, "base/snapshot.json")) + taskItem, err := task.NewTaskWithOps(snap.Name, task.TaskRecover, task.TaskScopeSnapshot, req.TaskID, snap.ID) if err != nil { - updateRecoverStatus(snap.ID, "Readjson", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) - return - } - if snap.InterruptStep == "Readjson" { - req.IsNew = true + global.LOG.Errorf("new task for create snapshot failed, err: %v", err) + return err } - if req.IsNew || snap.InterruptStep == "AppImage" { - if err := recoverAppData(snapFileDir); err != nil { - updateRecoverStatus(snap.ID, "AppImage", constant.StatusFailed, fmt.Sprintf("handle recover app data failed, err: %v", err)) - return - } - global.LOG.Debug("recover app images from snapshot file successful!") - req.IsNew = true + rootDir := path.Join(global.CONF.System.TmpDir, "system", snap.Name) + if _, err := os.Stat(rootDir); err != nil && os.IsNotExist(err) { + _ = os.MkdirAll(rootDir, os.ModePerm) } + itemHelper := snapRecoverHelper{Task: taskItem, FileOp: files.NewFileOp()} - if req.IsNew || snap.InterruptStep == "BaseData" { - if err := recoverBaseData(path.Join(snapFileDir, "base"), fileOp); err != nil { - updateRecoverStatus(snap.ID, "BaseData", constant.StatusFailed, err.Error()) - return - } - global.LOG.Debug("recover base data from snapshot file successful!") - req.IsNew = true - } + go func() { + _ = global.Cron.Stop() + defer func() { + global.Cron.Start() + }() - if req.IsNew || snap.InterruptStep == "DBData" { - if err := recoverDBData(path.Join(snapFileDir, "db"), fileOp); err != nil { - updateRecoverStatus(snap.ID, "DBData", constant.StatusFailed, err.Error()) - return + if req.IsNew || snap.InterruptStep == "RecoverDownload" || req.ReDownload { + taskItem.AddSubTask( + i18n.GetMsgByKey("RecoverDownload"), + func(t *task.Task) error { return handleDownloadSnapshot(&itemHelper, snap, rootDir) }, + nil, + ) + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "RecoverDecompress" { + taskItem.AddSubTask( + i18n.GetMsgByKey("RecoverDecompress"), + func(t *task.Task) error { + itemHelper.Task.Log("######################## 2 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetWithName("RecoverDecompress", snap.Name)) + err := itemHelper.FileOp.TarGzExtractPro(fmt.Sprintf("%s/%s.tar.gz", rootDir, snap.Name), rootDir, req.Secret) + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Decompress"), err) + return err + }, + nil, + ) + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "BackupBeforeRecover" { + taskItem.AddSubTask( + i18n.GetMsgByKey("BackupBeforeRecover"), + func(t *task.Task) error { return backupBeforeRecover(snap.Name, &itemHelper) }, + nil, + ) + req.IsNew = true } - global.LOG.Debug("recover db data from snapshot file successful!") - req.IsNew = true - } - if req.IsNew || snap.InterruptStep == "1PanelBackups" { - if err := fileOp.TarGzExtractPro(path.Join(snapFileDir, "/1panel_backup.tar.gz"), snapJson.BackupDataDir, ""); err != nil { - updateRecoverStatus(snap.ID, "1PanelBackups", constant.StatusFailed, err.Error()) - return + var snapJson SnapshotJson + taskItem.AddSubTask( + i18n.GetMsgByKey("Readjson"), + func(t *task.Task) error { + snapJson, err = readFromJson(path.Join(rootDir, snap.Name), &itemHelper) + return err + }, + nil, + ) + if req.IsNew || snap.InterruptStep == "RecoverApp" { + taskItem.AddSubTask( + i18n.GetMsgByKey("RecoverApp"), + func(t *task.Task) error { return recoverAppData(path.Join(rootDir, snap.Name), &itemHelper) }, + nil, + ) + req.IsNew = true } - global.LOG.Debug("recover 1panel backups from snapshot file successful!") - req.IsNew = true - } + if req.IsNew || snap.InterruptStep == "RecoverBaseData" { + taskItem.AddSubTask( + i18n.GetMsgByKey("RecoverBaseData"), + func(t *task.Task) error { return recoverBaseData(path.Join(rootDir, snap.Name, "base"), &itemHelper) }, + nil, + ) + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "RecoverDBData" { + taskItem.AddSubTask( + i18n.GetMsgByKey("RecoverDBData"), + func(t *task.Task) error { return recoverDBData(path.Join(rootDir, snap.Name, "db"), &itemHelper) }, + nil, + ) + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "RecoverBackups" { + taskItem.AddSubTask( + i18n.GetMsgByKey("RecoverBackups"), + func(t *task.Task) error { + itemHelper.Task.Log("######################## 8 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetWithName("RecoverBackups", snap.Name)) + err := itemHelper.FileOp.TarGzExtractPro(path.Join(rootDir, snap.Name, "/1panel_backup.tar.gz"), snapJson.BackupDataDir, "") + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Compress"), err) + return err + }, + nil, + ) + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "RecoverPanelData" { + taskItem.AddSubTask( + i18n.GetMsgByKey("RecoverPanelData"), + func(t *task.Task) error { + itemHelper.Task.Log("######################## 9 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetWithName("RecoverPanelData", snap.Name)) + err := itemHelper.FileOp.TarGzExtractPro(path.Join(rootDir, snap.Name, "/1panel_data.tar.gz"), path.Join(snapJson.BaseDir, "1panel"), "") + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Compress"), err) + return err + }, + nil, + ) + req.IsNew = true + } + taskItem.AddSubTask( + i18n.GetMsgByKey("RecoverDBData"), + func(t *task.Task) error { + return restartCompose(path.Join(snapJson.BaseDir, "1panel/docker/compose"), &itemHelper) + }, + nil, + ) - if req.IsNew || snap.InterruptStep == "1PanelData" { - if err := fileOp.TarGzExtractPro(path.Join(snapFileDir, "/1panel_data.tar.gz"), path.Join(snapJson.BaseDir, "1panel"), ""); err != nil { - updateRecoverStatus(snap.ID, "1PanelData", constant.StatusFailed, err.Error()) + if err := taskItem.Execute(); err != nil { + _ = settingRepo.Update("SystemStatus", "Free") + _ = snapshotRepo.Update(req.ID, map[string]interface{}{"recover_status": constant.StatusFailed, "message": err.Error(), "interrupt_step": taskItem.Task.CurrentStep}) return } - global.LOG.Debug("recover 1panel data from snapshot file successful!") - req.IsNew = true - } - _ = rebuildAllAppInstall() - restartCompose(path.Join(snapJson.BaseDir, "1panel/docker/compose")) - - global.LOG.Info("recover successful") - global.LOG.Debugf("remove the file %s after the operation is successful", path.Dir(snapFileDir)) - _ = os.RemoveAll(path.Dir(snapFileDir)) - _, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service") + _ = os.RemoveAll(rootDir) + _, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service") + }() + return nil } -func handleDownloadSnapshot(snap model.Snapshot, targetDir string) error { +func handleDownloadSnapshot(itemHelper *snapRecoverHelper, snap model.Snapshot, targetDir string) error { + itemHelper.Task.Log("######################## 1 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverDownload")) + account, client, err := NewBackupClientWithID(snap.DownloadAccountID) - if err != nil { - return err - } + itemHelper.Task.LogWithStatus(i18n.GetWithName("RecoverDownloadAccount", fmt.Sprintf("%s - %s", account.Type, account.Name)), err) pathItem := account.BackupPath if account.BackupPath != "/" { pathItem = strings.TrimPrefix(account.BackupPath, "/") } filePath := fmt.Sprintf("%s/%s.tar.gz", targetDir, snap.Name) _ = os.RemoveAll(filePath) - ok, err := client.Download(path.Join(pathItem, fmt.Sprintf("system_snapshot/%s.tar.gz", snap.Name)), filePath) - if err != nil || !ok { - return fmt.Errorf("download file %s from %s failed, err: %v", snap.Name, account.Name, err) - } - return nil + _, err = client.Download(path.Join(pathItem, fmt.Sprintf("system_snapshot/%s.tar.gz", snap.Name)), filePath) + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Download"), err) + return err } -func recoverAppData(src string) error { - if _, err := os.Stat(path.Join(src, "images.tar.gz")); err != nil { - global.LOG.Debug("no such docker images in snapshot") - return nil +func backupBeforeRecover(name string, itemHelper *snapRecoverHelper) error { + itemHelper.Task.Log("######################## 3 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetMsgByKey("BackupBeforeRecover")) + + rootDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, name) + baseDir := path.Join(rootDir, "base") + if _, err := os.Stat(baseDir); err != nil { + _ = os.MkdirAll(baseDir, os.ModePerm) } - std, err := cmd.Execf("docker load < %s", path.Join(src, "images.tar.gz")) + + err := itemHelper.FileOp.CopyDirWithExclude(path.Join(global.CONF.System.BaseDir, "1panel"), rootDir, []string{"cache", "tmp"}) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", path.Join(global.CONF.System.BaseDir, "1panel")), err) if err != nil { - return errors.New(std) + return err } - return err -} - -func recoverBaseData(src string, fileOp files.FileOp) error { - if err := fileOp.CopyFile(path.Join(src, "1pctl"), "/usr/local/bin"); err != nil { + err = itemHelper.FileOp.CopyDirWithExclude(global.CONF.System.Backup, rootDir, []string{"system_snapshot"}) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", global.CONF.System.Backup), err) + if err != nil { return err } - if err := fileOp.CopyFile(path.Join(src, "1panel"), "/usr/local/bin"); err != nil { + err = itemHelper.FileOp.CopyFile("/usr/local/bin/1pctl", baseDir) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1pctl"), err) + if err != nil { return err } - if err := fileOp.CopyFile(path.Join(src, "1panel_agent"), "/usr/local/bin"); err != nil { + err = itemHelper.FileOp.CopyFile("/usr/local/bin/1panel", baseDir) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel"), err) + if err != nil { return err } - if err := fileOp.CopyFile(path.Join(src, "1panel.service"), "/etc/systemd/system"); err != nil { + err = itemHelper.FileOp.CopyFile("/usr/local/bin/1panel_agent", baseDir) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel_agent"), err) + if err != nil { return err } - if err := fileOp.CopyFile(path.Join(src, "1panel_agent.service"), "/etc/systemd/system"); err != nil { + err = itemHelper.FileOp.CopyFile("/etc/systemd/system/1panel.service", baseDir) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel.service"), err) + if err != nil { return err } - - daemonJsonPath := "/etc/docker/daemon.json" - _, errSrc := os.Stat(path.Join(src, "docker/daemon.json")) - _, errPath := os.Stat(daemonJsonPath) - if os.IsNotExist(errSrc) && os.IsNotExist(errPath) { - global.LOG.Debug("the daemon.json file does not exist, nothing happens.") - return nil + err = itemHelper.FileOp.CopyFile("/etc/systemd/system/1panel_agent.service", baseDir) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel_agent.service"), err) + if err != nil { + return err } - if errSrc == nil { - if err := fileOp.CopyFile(path.Join(src, "docker/daemon.json"), "/etc/docker"); err != nil { - return fmt.Errorf("recover docker daemon.json failed, err: %v", err) - } + err = itemHelper.FileOp.CopyFile("/etc/docker/daemon.json", baseDir) + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/docker/daemon.json"), err) + if err != nil { + return err } - - _, _ = cmd.Exec("systemctl restart docker") return nil } -func recoverDBData(src string, fileOp files.FileOp) error { - return fileOp.CopyDirWithExclude(src, path.Join(global.CONF.System.BaseDir, "1panel"), nil) -} +func readFromJson(rootDir string, itemHelper *snapRecoverHelper) (SnapshotJson, error) { + itemHelper.Task.Log("######################## 4 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetMsgByKey("Readjson")) -func restartCompose(composePath string) { - composes, err := composeRepo.ListRecord() + snapJsonPath := path.Join(rootDir, "base/snapshot.json") + var snap SnapshotJson + _, err := os.Stat(snapJsonPath) + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("ReadjsonPath"), err) if err != nil { - return + return snap, err } - for _, compose := range composes { - pathItem := path.Join(composePath, compose.Name, "docker-compose.yml") - if _, err := os.Stat(pathItem); err != nil { - continue - } - upCmd := fmt.Sprintf("docker compose -f %s up -d", pathItem) - stdout, err := cmd.Exec(upCmd) - if err != nil { - global.LOG.Debugf("%s failed, err: %v", upCmd, stdout) - } + fileByte, err := os.ReadFile(snapJsonPath) + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("ReadjsonContent"), err) + if err != nil { + return snap, err } - global.LOG.Debug("restart all compose successful!") + err = json.Unmarshal(fileByte, &snap) + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("ReadjsonMarshal"), err) + if err != nil { + return snap, err + } + return snap, nil } -func backupBeforeRecover(name string) error { - rootDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, name) - baseDir := path.Join(rootDir, "base") - if _, err := os.Stat(baseDir); err != nil { - _ = os.MkdirAll(baseDir, os.ModePerm) - } +func recoverAppData(src string, itemHelper *snapRecoverHelper) error { + itemHelper.Task.Log("######################## 5 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverApp")) - FileOp := files.NewFileOp() - if err := FileOp.CopyDirWithExclude(path.Join(global.CONF.System.BaseDir, "1panel"), rootDir, []string{"cache", "tmp"}); err != nil { - return err - } - if err := FileOp.CopyDirWithExclude(global.CONF.System.Backup, rootDir, []string{"system_snapshot"}); err != nil { - return err - } - if err := FileOp.CopyFile("/usr/local/bin/1pctl", baseDir); err != nil { - return err - } - if err := FileOp.CopyFile("/usr/local/bin/1panel", baseDir); err != nil { - return err - } - if err := FileOp.CopyFile("/usr/local/bin/1panel_agent", baseDir); err != nil { - return err - } - if err := FileOp.CopyFile("/etc/systemd/system/1panel.service", baseDir); err != nil { - return err - } - if err := FileOp.CopyFile("/etc/systemd/system/1panel_agent.service", baseDir); err != nil { - return err + if _, err := os.Stat(path.Join(src, "images.tar.gz")); err != nil { + itemHelper.Task.Log(i18n.GetMsgByKey("RecoverAppEmpty")) + return nil + } else { + std, err := cmd.Execf("docker load < %s", path.Join(src, "images.tar.gz")) + if err != nil { + itemHelper.Task.LogFailedWithErr(i18n.GetMsgByKey("RecoverAppImage"), errors.New(std)) + return fmt.Errorf("docker load images failed, err: %v", err) + } + itemHelper.Task.LogSuccess(i18n.GetMsgByKey("RecoverAppImage")) } - if err := FileOp.CopyFile("/etc/docker/daemon.json", baseDir); err != nil { + + appInstalls, err := appInstallRepo.ListBy() + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverAppList"), err) + if err != nil { return err } + + var wg sync.WaitGroup + for i := 0; i < len(appInstalls); i++ { + wg.Add(1) + appInstalls[i].Status = constant.Rebuilding + _ = appInstallRepo.Save(context.Background(), &appInstalls[i]) + go func(app model.AppInstall) { + defer wg.Done() + dockerComposePath := app.GetComposePath() + out, err := compose.Down(dockerComposePath) + if err != nil { + _ = handleErr(app, err, out) + return + } + out, err = compose.Up(dockerComposePath) + if err != nil { + _ = handleErr(app, err, out) + return + } + app.Status = constant.Running + _ = appInstallRepo.Save(context.Background(), &app) + }(appInstalls[i]) + } + wg.Wait() return nil } -func handleRollback(name string) error { - rootDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, name) - baseDir := path.Join(rootDir, "base") +func recoverBaseData(src string, itemHelper *snapRecoverHelper) error { + itemHelper.Task.Log("######################## 6 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetMsgByKey("SnapBaseInfo")) - FileOp := files.NewFileOp() - if err := FileOp.CopyDir(path.Join(rootDir, "1panel"), global.CONF.System.BaseDir); err != nil { - return err - } - if err := FileOp.CopyDir(path.Join(rootDir, "backup"), path.Dir(global.CONF.System.Backup)); err != nil { + err := itemHelper.FileOp.CopyFile(path.Join(src, "1pctl"), "/usr/local/bin") + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1pctl"), err) + if err != nil { return err } - if err := FileOp.CopyFile(path.Join(baseDir, "1pctl"), "/usr/local/bin/1pctl"); err != nil { + + err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel"), "/usr/local/bin") + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel"), err) + if err != nil { return err } - if err := FileOp.CopyFile(path.Join(baseDir, "1panel"), "/usr/local/bin/1panel"); err != nil { + err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel_agent"), "/usr/local/bin") + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel_agent"), err) + if err != nil { return err } - if err := FileOp.CopyFile(path.Join(baseDir, "1panel_agent"), "/usr/local/bin/1panel_agent"); err != nil { + err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel.service"), "/etc/systemd/system") + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel.service"), err) + if err != nil { return err } - if err := FileOp.CopyFile(path.Join(baseDir, "1panel.service"), "/etc/systemd/system/1panel.service"); err != nil { + err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel_agent.service"), "/etc/systemd/system") + itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel_agent.service"), err) + if err != nil { return err } - if err := FileOp.CopyFile(path.Join(baseDir, "1panel_agent.service"), "/etc/systemd/system/1panel_agent.service"); err != nil { - return err + + daemonJsonPath := "/etc/docker/daemon.json" + _, errSrc := os.Stat(path.Join(src, "docker/daemon.json")) + _, errPath := os.Stat(daemonJsonPath) + if os.IsNotExist(errSrc) && os.IsNotExist(errPath) { + itemHelper.Task.Log(i18n.GetMsgByKey("RecoverDaemonJsonEmpty")) + return nil } - if err := FileOp.CopyFile(path.Join(baseDir, "daemon.json"), "/etc/docker/daemon.json"); err != nil { - return err + if errSrc == nil { + err = itemHelper.FileOp.CopyFile(path.Join(src, "docker/daemon.json"), "/etc/docker") + itemHelper.Task.Log(i18n.GetMsgByKey("RecoverDaemonJson")) + if err != nil { + return fmt.Errorf("recover docker daemon.json failed, err: %v", err) + } } - _ = os.RemoveAll(rootDir) + + _, _ = cmd.Exec("systemctl restart docker") return nil } -func updateRecoverStatus(id uint, interruptStep, status, message string) { - if status != constant.StatusSuccess { - global.LOG.Errorf("recover failed, err: %s", message) - } - if err := snapshotRepo.Update(id, map[string]interface{}{ - "interrupt_step": interruptStep, - "recover_status": status, - "recover_message": message, - "last_recovered_at": time.Now().Format(constant.DateTimeLayout), - }); err != nil { - global.LOG.Errorf("update snap recover status failed, err: %v", err) - } - _ = settingRepo.Update("SystemStatus", "Free") +func recoverDBData(src string, itemHelper *snapRecoverHelper) error { + itemHelper.Task.Log("######################## 7 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverDBData")) + err := itemHelper.FileOp.CopyDirWithExclude(src, path.Join(global.CONF.System.BaseDir, "1panel"), nil) + + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverDBData"), err) + return err } -func readFromJson(path string) (SnapshotJson, error) { - var snap SnapshotJson - if _, err := os.Stat(path); err != nil { - return snap, fmt.Errorf("find snapshot json file in recover package failed, err: %v", err) - } - fileByte, err := os.ReadFile(path) +func restartCompose(composePath string, itemHelper *snapRecoverHelper) error { + itemHelper.Task.Log("######################## 10 / 10 ########################") + itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverCompose")) + + composes, err := composeRepo.ListRecord() + itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverComposeList"), err) if err != nil { - return snap, fmt.Errorf("read file from path %s failed, err: %v", path, err) + return err } - if err := json.Unmarshal(fileByte, &snap); err != nil { - return snap, fmt.Errorf("unmarshal snapjson failed, err: %v", err) + + for _, compose := range composes { + pathItem := path.Join(composePath, compose.Name, "docker-compose.yml") + if _, err := os.Stat(pathItem); err != nil { + continue + } + upCmd := fmt.Sprintf("docker compose -f %s up -d", pathItem) + stdout, err := cmd.Exec(upCmd) + if err != nil { + itemHelper.Task.LogFailedWithErr(i18n.GetMsgByKey("RecoverCompose"), errors.New(stdout)) + continue + } + itemHelper.Task.LogSuccess(i18n.GetWithName("RecoverComposeItem", pathItem)) } - return snap, nil + return nil } diff --git a/agent/app/service/snapshot_rollback.go b/agent/app/service/snapshot_rollback.go new file mode 100644 index 000000000000..0a27c9cbf27b --- /dev/null +++ b/agent/app/service/snapshot_rollback.go @@ -0,0 +1,58 @@ +package service + +import ( + "fmt" + "os" + "path" + + "github.com/1Panel-dev/1Panel/agent/app/dto" + "github.com/1Panel-dev/1Panel/agent/global" + "github.com/1Panel-dev/1Panel/agent/utils/files" +) + +func (u *SnapshotService) SnapshotRollback(req dto.SnapshotRecover) error { + global.LOG.Info("start to rollback now") + snap, err := snapshotRepo.Get(commonRepo.WithByID(req.ID)) + if err != nil { + return err + } + go func() { + if err := handleRollback(snap.Name); err != nil { + global.LOG.Errorf("handle roll back snapshot failed, err: %v", err) + } + }() + return nil +} + +func handleRollback(name string) error { + rootDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, name) + baseDir := path.Join(rootDir, "base") + + FileOp := files.NewFileOp() + if err := FileOp.CopyDir(path.Join(rootDir, "1panel"), global.CONF.System.BaseDir); err != nil { + return err + } + if err := FileOp.CopyDir(path.Join(rootDir, "backup"), path.Dir(global.CONF.System.Backup)); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "1pctl"), "/usr/local/bin/1pctl"); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "1panel"), "/usr/local/bin/1panel"); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "1panel_agent"), "/usr/local/bin/1panel_agent"); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "1panel.service"), "/etc/systemd/system/1panel.service"); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "1panel_agent.service"), "/etc/systemd/system/1panel_agent.service"); err != nil { + return err + } + if err := FileOp.CopyFile(path.Join(baseDir, "daemon.json"), "/etc/docker/daemon.json"); err != nil { + return err + } + _ = os.RemoveAll(rootDir) + return nil +} diff --git a/agent/app/task/task.go b/agent/app/task/task.go index d7970f99b692..598bc42fdea2 100644 --- a/agent/app/task/task.go +++ b/agent/app/task/task.go @@ -51,6 +51,7 @@ const ( TaskUpdate = "TaskUpdate" TaskRestart = "TaskRestart" TaskBackup = "TaskBackup" + TaskRecover = "TaskRecover" TaskSync = "TaskSync" TaskBuild = "TaskBuild" ) diff --git a/agent/i18n/lang/zh.yaml b/agent/i18n/lang/zh.yaml index 9e1622c49117..7ecdf0bdc746 100644 --- a/agent/i18n/lang/zh.yaml +++ b/agent/i18n/lang/zh.yaml @@ -223,6 +223,7 @@ TaskUpgrade: "升级" TaskUpdate: "更新" TaskRestart: "重启" TaskBackup: "备份" +TaskRecover: "恢复" Website: "网站" App: "应用" Runtime: "运行环境" @@ -275,3 +276,29 @@ SnapUpload: "上传快照文件" SnapLoadBackup: "获取备份账号信息" SnapUploadTo: "上传快照文件到 {{ .name }}" SnapUploadRes: "上传快照文件到 {{ .name }}" + +SnapshotRecover: "快照恢复" +RecoverDownload: "下载快照文件" +Download: "下载" +RecoverDownloadAccount: "获取快照下载备份账号 {{ .name }}" +RecoverDecompress: "解压快照压缩文件" +Decompress: "解压" +BackupBeforeRecover: "快照前备份系统相关数据" +Readjson: "读取快照内 Json 文件" +ReadjsonPath: "获取快照内 Json 文件路径" +ReadjsonContent: "读取 Json 文件" +ReadjsonMarshal: "Json 转义处理" +RecoverApp: "恢复已安装应用" +RecoverAppImage: "恢复快照镜像备份" +RecoverAppList: "获取所有待恢复应用" +RecoverCompose: "恢复其他编排内容" +RecoverComposeList: "获取所有待恢复编排" +RecoverComposeItem: "恢复编排 {{ .name }}" +RecoverAppEmpty: "快照文件中未发现应用镜像备份" +RecoverBaseData: "恢复基础数据及文件" +RecoverDaemonJsonEmpty: "快照文件及当前机器都不存在容器配置 daemon.json 文件" +RecoverDaemonJson: "恢复容器配置 daemon.json 文件" +RecoverDBData: "恢复数据库数据" +RecoverBackups: "恢复本地备份目录" +RecoverPanelData: "恢复数据目录" + diff --git a/agent/init/migration/migrations/init.go b/agent/init/migration/migrations/init.go index 8b7793816952..15c19b131145 100644 --- a/agent/init/migration/migrations/init.go +++ b/agent/init/migration/migrations/init.go @@ -261,7 +261,7 @@ var UpdateAppInstall = &gormigrate.Migration{ } var UpdateSnapshot = &gormigrate.Migration{ - ID: "20240925-update-snapshot", + ID: "20240926-update-snapshot", Migrate: func(tx *gorm.DB) error { return tx.AutoMigrate(&model.Snapshot{}) }, From fd64235a20914fcd86aac008c942abe506cb7a37 Mon Sep 17 00:00:00 2001 From: ssongliu Date: Mon, 23 Sep 2024 18:07:00 +0800 Subject: [PATCH 08/12] =?UTF-8?q?feat:=20=E5=BF=AB=E7=85=A7=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E5=A4=B1=E8=B4=A5=E9=87=8D=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/api/modules/setting.ts | 11 +++++++- frontend/src/lang/modules/en.ts | 1 + frontend/src/lang/modules/tw.ts | 1 + frontend/src/lang/modules/zh.ts | 1 + .../views/setting/snapshot/create/index.vue | 4 +-- .../setting/snapshot/ignore-rule/index.vue | 6 ++-- frontend/src/views/setting/snapshot/index.vue | 28 +++++++++++++++++-- 7 files changed, 44 insertions(+), 8 deletions(-) diff --git a/frontend/src/api/modules/setting.ts b/frontend/src/api/modules/setting.ts index 922b1ecbee92..d4805b451263 100644 --- a/frontend/src/api/modules/setting.ts +++ b/frontend/src/api/modules/setting.ts @@ -28,6 +28,12 @@ export const loadBaseDir = () => { export const loadDaemonJsonPath = () => { return http.get(`/settings/daemonjson`, {}); }; +export const updateAgentSetting = (param: Setting.SettingUpdate) => { + return http.post(`/settings/update`, param); +}; +export const getAgentSettingInfo = () => { + return http.post(`/settings/search`); +}; // core export const getSettingInfo = () => { @@ -88,12 +94,15 @@ export const bindMFA = (param: Setting.MFABind) => { }; // snapshot -export const loadSnapshotSetting = () => { +export const loadSnapshotInfo = () => { return http.get(`/settings/snapshot/load`); }; export const snapshotCreate = (param: Setting.SnapshotCreate) => { return http.post(`/settings/snapshot`, param); }; +export const snapshotRecreate = (id: number) => { + return http.post(`/settings/snapshot/recreate`, { id: id }); +}; export const loadSnapStatus = (id: number) => { return http.post(`/settings/snapshot/status`, { id: id }); }; diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index b62f600c2ac0..4643593e7e7d 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -1619,6 +1619,7 @@ const message = { shellLabel: 'Script', tmpLabel: 'Temporary Directory', sslLabel: 'Certificate Directory', + reCreate: 'Failed to create snapshot', deleteHelper: 'All backup files for the snapshot, including those in the third-party backup account, will be deleted.', status: 'Snapshot status', diff --git a/frontend/src/lang/modules/tw.ts b/frontend/src/lang/modules/tw.ts index 224c992ffa35..06197f15fadb 100644 --- a/frontend/src/lang/modules/tw.ts +++ b/frontend/src/lang/modules/tw.ts @@ -1437,6 +1437,7 @@ const message = { shellLabel: '腳本', tmpLabel: '臨時目錄', sslLabel: '證書目錄', + reCreate: '创建快照失败', deleteHelper: '將刪除該快照的所有備份文件，包括第三方備份賬號中的文件。', status: '快照狀態', ignoreRule: '排除規則', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index 5e53cd6c9cea..9411f3b56218 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -1439,6 +1439,7 @@ const message = { shellLabel: '脚本', tmpLabel: '临时目录', sslLabel: '证书目录', + reCreate: '创建快照失败', deleteHelper: '将删除该快照的所有备份文件，包括第三方备份账号中的文件。', ignoreRule: '排除规则', ignoreHelper: '快照时将使用该规则对 1Panel 数据目录进行压缩备份，请谨慎修改。', diff --git a/frontend/src/views/setting/snapshot/create/index.vue b/frontend/src/views/setting/snapshot/create/index.vue index 243d99ae7f81..5f7383959081 100644 --- a/frontend/src/views/setting/snapshot/create/index.vue +++ b/frontend/src/views/setting/snapshot/create/index.vue @@ -139,7 +139,7 @@