fix: some issues

matrixorigin · Jul 8, 2024 · de77664 · de77664
1 parent d3f5dc5
commit de77664
Show file tree

Hide file tree

Showing 7 changed files with 126 additions and 21 deletions.
diff --git a/bin/auto_backup.sh b/bin/auto_backup.sh
@@ -108,7 +108,12 @@ function backup()
 
     backup_yearmonth=`date '+%Y%m'`
     backup_timestamp=`date '+%Y%m%d_%H%M%S'`
-    backup_outpath="${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}"
+    if [[ "${BACKUP_DATA_PATH_AUTO_TS}" == "no" ]]; then
+        backup_outpath="${BACKUP_DATA_PATH}"
+    else
+        backup_outpath="${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}"
+    fi
+
     #if [[ "${BACKUP_TYPE}" == "logical" ]] && [[ "${BACKUP_LOGICAL_DS}" != "" ]] ; then
     #    backup_outpath="${backup_outpath}_${BACKUP_LOGICAL_DS}"
     #fi
@@ -310,13 +315,60 @@ function backup()
                 add_log "D" "BACKUP_MOBR_META_PATH is not empty, will add option ${br_meta_option}"
             fi
 
+
             BACKUP_MOBR_DIRNAME=`dirname "${BACKUP_MOBR_PATH}"`
+
+            # 1. incremental
+            add_log "D" "Judging physical backup method(full or incremental): BACKUP_PHYSICAL_METHOD=${BACKUP_PHYSICAL_METHOD}"
+            if [[ "${BACKUP_PHYSICAL_METHOD}" == "incremental" ]]; then
+                add_log "I" "Physical backup method: incremental"
+                add_log "D" "BACKUP_PHYSICAL_BASE_BKID: ${BACKUP_PHYSICAL_BASE_BKID}"
+
+                if [[ "${BACKUP_PHYSICAL_BASE_BKID}" == "" ]]; then
+                    add_log "E" "Base backup id BACKUP_PHYSICAL_BASE_BKID is empty, please set it first: mo_ctl set_conf BACKUP_PHYSICAL_BASE_BKID=xxx"
+                    add_log "I" "Hints: use 'mo_ctl backup list detail' to show previous backup ids"
+                    return 1
+                else
+                    # get backup path of bkid
+                    add_log "D" "BACKUP_MOBR_PATH: ${BACKUP_MOBR_PATH}"
+                    if [[ "${BACKUP_MOBR_PATH}" != "" ]]; then
+                        add_log "D" "Try to get backup path of backup id ${BACKUP_PHYSICAL_BASE_BKID} from ${BACKUP_MOBR_PATH}"
+                        add_log "D" "cmd: ${BACKUP_MOBR_PATH} | grep ${BACKUP_PHYSICAL_BASE_BKID} | awk -F \"path = \" '{print $2}' | awk -F \"|\" '{print $1}'"
+                        real_bk_path=`${BACKUP_MOBR_PATH} | grep ${BACKUP_PHYSICAL_BASE_BKID} | awk -F "path = " '{print $2}' | awk -F "|" '{print $1}'`
+                    else
+                        add_log "D" "Try to get backup path of backup id ${BACKUP_PHYSICAL_BASE_BKID} from 'mo_ctl backup list detail'"
+                        add_log "D" "cmd: mo_ctl backup list detail  | grep -A 1 ${BACKUP_PHYSICAL_BASE_BKID}  | tail -n 1 | awk -F \"|\" '{print $4}' | sed \"s/[[:space:]][[:space:]]*//g\""
+                        real_bk_path=`mo_ctl backup list detail  | grep -A 1 ${BACKUP_PHYSICAL_BASE_BKID}  | tail -n 1 | awk -F "|" '{print $4}' | sed "s/[[:space:]][[:space:]]*//g"`
+                    fi
+                    add_log "D" "real_bk_path: ${real_bk_path}"
+
+                    if [[ "${real_bk_path}" == "" ]]; then
+                        add_log "E" "Failed to backup path of backup id ${BACKUP_PHYSICAL_BASE_BKID}"
+                        return 1
+                    else
+                        if [[ "${real_bk_path}" != "${backup_outpath}" ]]; then
+
+                            add_log "E" "Real backup path ${real_bk_path} of backup id ${BACKUP_PHYSICAL_BASE_BKID} does not math given target backup path ${backup_outpath}, please check again"
+                            if [[ "${BACKUP_DATA_PATH_AUTO_TS}" == "yes" ]]; then
+                                add_log "E" "Conf BACKUP_DATA_PATH_AUTO_TS is set to 'yes', please set this to 'no' when trying to perform a delta physical backup: 'mo_ctl set_conf BACKUP_DATA_PATH_AUTO_TS=no'"
+                            fi
+                            return 1
+                        fi
+                    fi 
+                fi
+                delta_option="--backup_type \"incremental\" --base_id \"${BACKUP_PHYSICAL_BASE_BKID}\""
+            # 2. full
+            else
+                add_log "I" "Physical backup method: full"
+                delta_option=""
+            fi
+
             case "${BACKUP_PHYSICAL_TYPE}" in
                 "filesystem")
-                    add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"filesystem\" --path \"${backup_outpath}/\""
+                    add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"filesystem\" --path \"${backup_outpath}/\" ${delta_option}"
 
                     startTime=`get_nanosecond`
-                    if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "filesystem" --path "${backup_outpath}/" ; then
+                    if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "filesystem" --path "${backup_outpath}/ ${delta_option}" ; then
                         outcome="succeeded"
                     else
                         outcome="failed"
@@ -334,10 +386,10 @@ function backup()
                         role_arn_option="--role_arn ${BACKUP_S3_ROLE_ARN}"
                     fi
 
-                    add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"s3\" --endpoint \"${BACKUP_S3_ENDPOINT}\" --access_key_id \"${BACKUP_S3_ID}\" --secret_access_key \"${BACKUP_S3_KEY}\" --bucket \"${BACKUP_S3_BUCKET}\" --filepath \"${BACKUP_DATA_PATH}\" --region \"${BACKUP_S3_REGION}\" --compression \"${BACKUP_S3_COMPRESSION}\" \"${role_arn_option}\" \"${minio_option}\""
+                    add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"s3\" --endpoint \"${BACKUP_S3_ENDPOINT}\" --access_key_id \"${BACKUP_S3_ID}\" --secret_access_key \"${BACKUP_S3_KEY}\" --bucket \"${BACKUP_S3_BUCKET}\" --filepath \"${backup_outpath}\" --region \"${BACKUP_S3_REGION}\" --compression \"${BACKUP_S3_COMPRESSION}\" \"${role_arn_option}\" \"${minio_option}\" ${delta_option}"
 
                     startTime=`get_nanosecond`
-                    if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "s3" --endpoint "${BACKUP_S3_ENDPOINT}" --access_key_id "${BACKUP_S3_ID}" --secret_access_key "${BACKUP_S3_KEY}" --bucket "${BACKUP_S3_BUCKET}" --filepath "${BACKUP_DATA_PATH}" --region "${BACKUP_S3_REGION}" --compression "${BACKUP_S3_COMPRESSION}" "${role_arn_option}" "${minio_option}"; then
+                    if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "s3" --endpoint "${BACKUP_S3_ENDPOINT}" --access_key_id "${BACKUP_S3_ID}" --secret_access_key "${BACKUP_S3_KEY}" --bucket "${BACKUP_S3_BUCKET}" --filepath "${backup_outpath}" --region "${BACKUP_S3_REGION}" --compression "${BACKUP_S3_COMPRESSION}" "${role_arn_option}" "${minio_option}" ${delta_option}; then
                         outcome="succeeded"
                     else
                         outcome="failed"
@@ -385,24 +437,32 @@ function backup()
 
 function clean_backup()
 {
+
+    if [[ "${BACKUP_DATA_PATH_AUTO_TS}" == "no" ]]; then
+        add_log "E" "Clean backup is only supported when BACKUP_DATA_PATH_AUTO_TS is set to 'yes', please set it first: mo_ctl set_conf BACKUP_DATA_PATH_AUTO_TS='yes'"
+        return 1
+    else
+        clean_path="${BACKUP_DATA_PATH}"
+    fi
+
     add_log "I" "Cleaning backups before ${BACKUP_CLEAN_DAYS_BEFORE} days"
     clean_date=`date -d "${BACKUP_CLEAN_DAYS_BEFORE} day ago" +%Y%m%d`
     add_log "I" "Clean date: ${clean_date}"
 
-    for month in `ls ${BACKUP_DATA_PATH}`; do
-        for backup_dir in `ls ${BACKUP_DATA_PATH}/${month}`; do
+    for month in `ls ${clean_path}`; do
+        for backup_dir in `ls ${clean_path}/${month}`; do
             backup_date=`echo "${backup_dir}" | awk -F"_" '{print $1}'`
             backup_date_int=`date -d "${backup_date}" +%s`
             clean_date_int=`date -d "${clean_date}" +%s`
             if [[ ${backup_date_int} -le ${clean_date_int} ]]; then
-                add_log "I" "Backup directory : ${BACKUP_DATA_PATH}/${month}/${backup_dir}, action: delete"
-                if cd ${BACKUP_DATA_PATH}/${month} && rm -rf ./${backup_dir}; then
+                add_log "I" "Backup directory : ${clean_path}/${month}/${backup_dir}, action: delete"
+                if cd ${clean_path}/${month} && rm -rf ./${backup_dir}; then
                     add_log "I" "Succeeded"
                 else
                     add_log "E" "Failed"
                 fi
             else
-                add_log "I" "Backup directory : ${BACKUP_DATA_PATH}/${month}/${backup_dir}/${backup_dir}, action: skip"
+                add_log "I" "Backup directory : ${clean_path}/${month}/${backup_dir}/${backup_dir}, action: skip"
             fi
         done
     done

diff --git a/bin/build_image.sh b/bin/build_image.sh
@@ -49,7 +49,20 @@ function build_image()
     fi
 
     add_log "I" "Build MO image ${image_name}:${branch}_${commitid_less}"
-    if cd ${MO_PATH}/matrixone && docker build -f optools/images/Dockerfile -t ${image_name}:${branch}_${commitid_less} . --build-arg GOPROXY="${GOPROXY}" ; then
+
+    cd ${MO_PATH}/matrixone && 
+
+    if [[ "${MO_CONTAINER_DEPIMAGE_REPLACE_REPO}" == "yes" ]]; then
+        add_log "D" "Repace repo for some images"
+
+        add_log "D" "s#FROM golang:1.22.3-bookworm as builder#FROM ccr.ccs.tencentyun.com/mo-infra/golang:1.22.3-bookworm as builder#g' optools/images/Dockerfile"
+        sed -i 's#FROM golang:1.22.3-bookworm as builder#FROM ccr.ccs.tencentyun.com/mo-infra/golang:1.22.3-bookworm as builder#g' optools/images/Dockerfile
+
+        add_log "D" "s#FROM ubuntu:22.04#FROM ccr.ccs.tencentyun.com/mo-infra/ubuntu:22.04#g' optools/images/Dockerfile"
+        sed -i 's#FROM ubuntu:22.04#FROM ccr.ccs.tencentyun.com/mo-infra/ubuntu:22.04#g' optools/images/Dockerfile
+    fi
+
+    if docker build -f optools/images/Dockerfile -t ${image_name}:${branch}_${commitid_less} . --build-arg GOPROXY="${GOPROXY}" ; then
         :
     else
         return 1

diff --git a/bin/get_branch.sh b/bin/get_branch.sh
@@ -58,7 +58,7 @@ function get_branch()
         else
             MO_V_TYPE="unkown"
             add_log "E" "Get tag failed"
+            return 1
         fi
-        return 1
     fi
 }
diff --git a/bin/start.sh b/bin/start.sh
@@ -109,7 +109,7 @@ function start()
 
             # if conf path exists
             if [[ -d ${MO_CONTAINER_CONF_HOST_PATH} ]]; then
-                docker_init_cmd="docker run ${cmd_params} -v ${MO_CONTAINER_CONF_HOST_PATH}:/etc:rw --entrypoint /mo-service ${MO_CONTAINER_IMAGE} -launch ${MO_CONTAINER_CONF_CON_FILE}"
+                docker_init_cmd="docker run ${cmd_params} -v ${MO_CONTAINER_CONF_HOST_PATH}:/etc:rw --entrypoint /mo-service ${MO_CONTAINER_IMAGE} -debug-http :${MO_CONTAINER_DEBUG_PORT} -launch ${MO_CONTAINER_CONF_CON_FILE}"
             else
                 docker_init_cmd="docker run ${cmd_params} ${MO_CONTAINER_IMAGE}"              
             fi

diff --git a/bin/version.sh b/bin/version.sh
@@ -5,8 +5,19 @@
 ################################################################
 # get_cid
 
-function version()
+function version_deprecated()
 {
-    echo "Tool version: ${TOOL_NAME} ${TOOL_VERSION}"
+    echo "Tool version: ${TOOL_NAME} ${MO_TOOL_VERSION}"
     echo "Server version: ${MO_SERVER_NAME} ${MO_SERVER_VERSION}"
 }
+
+function version()
+{
+    add_log "I" "Tool version (mo_ctl):"
+    add_log "I" "-------------------------------"
+    add_log "I" "${TOOL_NAME} ${MO_TOOL_VERSION}"
+    echo ""
+    add_log "I" "Server version (MatrixOne): "
+    add_log "I" "-------------------------------"
+    sql "select version()"
+}
diff --git a/conf/env.sh b/conf/env.sh
@@ -66,6 +66,9 @@ MO_CONTAINER_AUTO_RESTART="yes"
 MO_CONTAINER_LIMIT_CPU=""
 # extra mount options
 MO_CONTAINER_EXTRA_MOUNT_OPTION=""
+# replace below image repo with ccr.ccs.tencentyun.com/mo-infra? option: yes | no
+# images: golang:1.22.3-bookworm , ubuntu:22.04
+MO_CONTAINER_DEPIMAGE_REPLACE_REPO="yes" 
 
 
 ###########################################
@@ -167,9 +170,9 @@ CSV_CONVERT_INSERT_ADD_QUOTE="no"
 
 # for version
 MO_TOOL_NAME="mo_ctl"
-MO_TOOL_VERSION="V1.0"
+MO_TOOL_VERSION="V1.2"
 MO_SERVER_NAME="超融合数据库MatrixOne企业版软件"
-MO_SERVER_VERSION="V1.0"
+MO_SERVER_VERSION="V1.2"
 
 # for auto backup, currently only linux is supported
 # backup
@@ -179,6 +182,8 @@ BACKUP_TYPE="physical"
 BACKUP_CRON_SCHEDULE="30 23 * * *"
 # backup data path
 BACKUP_DATA_PATH="/data/mo-backup"
+# add timestamp subpaths to backup data path? format: ${BACKUP_DATA_PATH}/202406/20240620_161838
+BACKUP_DATA_PATH_AUTO_TS="yes"
 # auto clean old backups
 # clean old backup files before [x] (default: 31) days
 BACKUP_CLEAN_DAYS_BEFORE="31"
@@ -195,6 +200,10 @@ BACKUP_MOBR_PATH="/data/tools/mo-backup/mo_br"
 BACKUP_PHYSICAL_TYPE="filesystem"
 # mobr meta file
 BACKUP_MOBR_META_PATH="${TOOL_LOG_PATH}/mo_br.meta"
+# backup physical method: full(default)|incremental
+BACKUP_PHYSICAL_METHOD="full"
+# backup physical base backup id, only valid when BACKUP_PHYSICAL_METHOD='incremental' 
+BACKUP_PHYSICAL_BASE_BKID=""
 
 # 1) when BACKUP_PHYSICAL_TYPE="filesystem"
 # backup directory, same as BACKUP_PATH

diff --git a/conf/env.sh.default b/conf/env.sh.default
@@ -66,6 +66,9 @@ MO_CONTAINER_AUTO_RESTART="yes"
 MO_CONTAINER_LIMIT_CPU=""
 # extra mount options
 MO_CONTAINER_EXTRA_MOUNT_OPTION=""
+# replace below image repo with ccr.ccs.tencentyun.com/mo-infra? option: yes | no
+# images: golang:1.22.3-bookworm , ubuntu:22.04
+MO_CONTAINER_DEPIMAGE_REPLACE_REPO="yes" 
 
 
 ###########################################
@@ -76,7 +79,7 @@ MO_CONTAINER_EXTRA_MOUNT_OPTION=""
 CHECK_LIST=("go" "gcc" "git" "mysql" "docker")
 GCC_VERSION="8.5.0"
 CLANG_VERSION="13.0"
-GO_VERSION="1.21"
+GO_VERSION="1.22.3"
 DOCKER_SERVER_VERSION="20"
 
 # for deploy
@@ -91,7 +94,7 @@ MO_GIT_URL="https://github.com/matrixorigin/matrixone.git"
 #)
 
 # default version of which mo to be deployed
-MO_DEFAULT_VERSION="v1.1.1"
+MO_DEFAULT_VERSION="v1.2.0"
 # which go proxy to be used when downloading go dependencies
 # you can set this go proxy when building mo-service
 GOPROXY="https://goproxy.cn,direct"
@@ -167,9 +170,9 @@ CSV_CONVERT_INSERT_ADD_QUOTE="no"
 
 # for version
 MO_TOOL_NAME="mo_ctl"
-MO_TOOL_VERSION="V1.0"
+MO_TOOL_VERSION="V1.2"
 MO_SERVER_NAME="超融合数据库MatrixOne企业版软件"
-MO_SERVER_VERSION="V1.0"
+MO_SERVER_VERSION="V1.2"
 
 # for auto backup, currently only linux is supported
 # backup
@@ -179,6 +182,8 @@ BACKUP_TYPE="physical"
 BACKUP_CRON_SCHEDULE="30 23 * * *"
 # backup data path
 BACKUP_DATA_PATH="/data/mo-backup"
+# add timestamp subpaths to backup data path? format: ${BACKUP_DATA_PATH}/202406/20240620_161838
+BACKUP_DATA_PATH_AUTO_TS="yes"
 # auto clean old backups
 # clean old backup files before [x] (default: 31) days
 BACKUP_CLEAN_DAYS_BEFORE="31"
@@ -193,6 +198,13 @@ BACKUP_REPORT="${TOOL_LOG_PATH}/backup-report.txt"
 BACKUP_MOBR_PATH="/data/tools/mo-backup/mo_br"
 # backup target type: filesystem(default)|s3
 BACKUP_PHYSICAL_TYPE="filesystem"
+# mobr meta file
+BACKUP_MOBR_META_PATH="${TOOL_LOG_PATH}/mo_br.meta"
+# backup physical method: full(default)|incremental
+BACKUP_PHYSICAL_METHOD="full"
+# backup physical base backup id, only valid when BACKUP_PHYSICAL_METHOD='incremental' 
+BACKUP_PHYSICAL_BASE_BKID=""
+
 # 1) when BACKUP_PHYSICAL_TYPE="filesystem"
 # backup directory, same as BACKUP_PATH
 # BACKUP_PATH="/data/mo-backup"