fix: deploy in docker mode

matrixorigin · Apr 2, 2024 · 3811fef · 3811fef
1 parent 7129d94
commit 3811fef
Show file tree

Hide file tree

Showing 7 changed files with 191 additions and 48 deletions.
diff --git a/bin/auto_backup.sh b/bin/auto_backup.sh
@@ -143,35 +143,121 @@ function backup()
                     backup_db_list=${BACKUP_LOGICAL_DB_LIST}
                     ;;
             esac
-
+
+            add_log "D" "backup_db_list: ${backup_db_list}"
             if [[ "${backup_db_list}" == "" ]]; then
                 add_log "E" "Final backup database list seems to be empty, please check conf BACKUP_LOGICAL_DB_LIST"
                 return 1
             fi
 
 
             csv_option=""
+            no_data_option=""
             if [[ "${BACKUP_LOGICAL_DATA_TYPE}" == "csv" ]]; then
                 csv_option="-csv"
+            elif [[ "${BACKUP_LOGICAL_DATA_TYPE}" == "ddl" ]]; then
+                no_data_option="-no-data"
             fi
-
-            for db in $(echo "${backup_db_list}" | sed "s/,/ /g"); do
-                add_log "I" "Begin to back up database: ${db}"
-
-                startTime=`get_nanosecond`
-                add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}.sql && cd -"
-                if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}.sql && cd - >/dev/null 2>&1; then
-                    endTime=`get_nanosecond`
-                    outcome="succeeded"
+
+            # 1. in case we have multiple databases
+            if echo "${backup_db_list}" | grep "," >/dev/null 2>&1 ; then
+                add_log "W" "backup_db_list=${backup_db_list} seems to be a list containing multiple dbs, thus will ignore conf BACKUP_LOGICAL_TBL_LIST=${BACKUP_LOGICAL_TBL_LIST} and backup databases in db list only"
+
+                # 1.1. backup databases one by one
+                if [[ "${BACKUP_LOGICAL_ONEBYONE}" == "1" ]]; then
+                    add_log "D" "BACKUP_LOGICAL_ONEBYONE is set to 1, will backup tables one by one"
+
+                    for db in $(echo "${backup_db_list}" | sed "s/,/ /g"); do
+                        add_log "I" "Begin to back up database: ${db}"
+
+                        add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}.sql && cd - >/dev/null 2>&1"
+                        startTime=`get_nanosecond`
+                        if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}.sql && cd - >/dev/null 2>&1; then
+                            endTime=`get_nanosecond`
+                            outcome="succeeded"
+                        else
+                            endTime=`get_nanosecond`
+                            outcome="failed"
+
+                        fi
+
+                        cost=`time_cost_ms ${startTime} ${endTime}`
+
+                        add_log "I" "End with outcome: ${outcome}, cost: ${cost} ms"
+                    done
+
+                # 1.2. backup databases all at once
                 else
-                    endTime=`get_nanosecond`
-                    outcome="failed"
+                    add_log "D" "BACKUP_LOGICAL_ONEBYONE is not set to 1, will backup databases all at once"
+                    add_log "I" "Begin to back up databases in list: ${backup_db_list}"
+                    add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/mo.sql && cd - >/dev/null 2>&1"
+
+                    startTime=`get_nanosecond`
+                    if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/mo.sql && cd - >/dev/null 2>&1; then
+                        endTime=`get_nanosecond`
+                        outcome="succeeded"
+                    else
+                        endTime=`get_nanosecond`
+                        outcome="failed"
+
+                    fi
+                    cost=`time_cost_ms ${startTime} ${endTime}`
+
+                    add_log "I" "End with outcome: ${outcome}, cost: ${cost} ms"
 
                 fi
-                cost=`time_cost_ms ${startTime} ${endTime}`
 
-                add_log "I" "End with outcome: ${outcome}, cost: ${cost} ms"
-            done
+            # 2. in case we have only one database
+            else
+                add_log "D" "backup_db_list=${backup_db_list} seems to be one exact database, thus will take conf BACKUP_LOGICAL_TBL_LIST=${BACKUP_LOGICAL_TBL_LIST} into consideration"
+
+                if [[ "${BACKUP_LOGICAL_ONEBYONE}" == "1" ]]; then
+                    add_log "D" "BACKUP_LOGICAL_ONEBYONE is set to 1, will backup tables one by one"
+
+                    for tbl in $(echo "${BACKUP_LOGICAL_TBL_LIST}" | sed "s/,/ /g"); do
+                        add_log "I" "Begin to back up table: ${tbl}"
+
+                        add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${tbl} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}_${tbl}.sql && cd - >/dev/null 2>&1"
+                        startTime=`get_nanosecond`
+                        if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${tbl} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}_${tbl}.sql && cd - >/dev/null 2>&1; then
+                            endTime=`get_nanosecond`
+                            outcome="succeeded"
+                        else
+                            endTime=`get_nanosecond`
+                            outcome="failed"
+
+                        fi
+                        cost=`time_cost_ms ${startTime} ${endTime}`
+
+                        add_log "I" "End with outcome: ${outcome}, cost: ${cost} ms"
+                    done
+
+
+                # backup tables all at once
+                else
+                    add_log "D" "BACKUP_LOGICAL_ONEBYONE is not set to 1, will backup tables all at once"
+                    add_log "I" "Begin to back up tables in list: ${BACKUP_LOGICAL_TBL_LIST} in database ${backup_db_list}"
+                    add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${BACKUP_LOGICAL_TBL_LIST} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${backup_db_list}.sql && cd - >/dev/null 2>&1"
+
+                    startTime=`get_nanosecond`
+                    if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${BACKUP_LOGICAL_TBL_LIST} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${backup_db_list}.sql && cd - >/dev/null 2>&1; then
+                        endTime=`get_nanosecond`
+                        outcome="succeeded"
+                    else
+                        endTime=`get_nanosecond`
+                        outcome="failed"
+
+                    fi
+                    cost=`time_cost_ms ${startTime} ${endTime}`
+
+                    add_log "I" "End with outcome: ${outcome}, cost: ${cost} ms"
+
+                fi
+
+
+            fi
+
+
             ;;
 
         # 2) physical backups : mo_br

diff --git a/bin/deploy.sh b/bin/deploy.sh
@@ -185,14 +185,15 @@ function deploy_docker()
     if [[ "${MO_CONTAINER_IMAGE}" == "" ]]; then
         add_log "E" "conf MO_CONTAINER_IMAGE is empty, please set it first"
     fi
+
+    # 2024/4/2: deprecated as we don't pull image manually
+    #add_log "I" "Pulling image ${MO_CONTAINER_IMAGE}"
+    #if ! docker pull ${MO_CONTAINER_IMAGE}; then
+    #    add_log "E" "Failed to pull docker image, please check if ${MO_CONTAINER_IMAGE} is a correct image or it might be a network issue"
+    #    return 1
+    #fi
 
-    add_log "I" "Pulling image ${MO_CONTAINER_IMAGE}"
-    if ! docker pull ${MO_CONTAINER_IMAGE}; then
-        add_log "E" "Failed to pull docker image, please check if ${MO_CONTAINER_IMAGE} is a correct image or it might be a network issue"
-        return 1
-    fi
-
-    add_log "I" "Successfully pulled image ${MO_CONTAINER_IMAGE}"
+    #add_log "I" "Successfully pulled image ${MO_CONTAINER_IMAGE}"
 }
 
 function deploy()

diff --git a/bin/pprof.sh b/bin/pprof.sh
@@ -32,8 +32,8 @@ function pprof()
 
     add_log "I" "pprof option is ${option}" 
 
-    case ${option} in
-        profile)
+    case "${option}" in
+        "profile" | "trace" )
             if [[ "${duration}" == "" ]]; then
                 add_log "I" "duration is not set, using conf value: ${PPROF_PROFILE_DURATION}"
                 duration="${PPROF_PROFILE_DURATION}"
@@ -51,11 +51,8 @@ function pprof()
             URL="${URL}?seconds=${duration}";
             add_log "I" "collect duration is ${duration} seconds" 
             ;;
-        allocs)
-            a=1
-            ;;
-        heap)
-            a=1
+        "allocs" | "heap" | "goroutine")
+            :
             ;;
         *)
             add_log "E" "Invalid option ${option} for pprof. Available: profile | pprof | heap"

diff --git a/bin/set_conf.sh b/bin/set_conf.sh
@@ -5,6 +5,24 @@
 ################################################################
 # set_conf
 
+
+# const
+
+# enum
+ENUM_TOOL_LOG_LEVEL="d,D,I,i,W,w,E,e"
+ENUM_MO_DEPLOY_MODE="git,docker,binary"
+ENUM_MO_SERVER_TYPE="local,remote"
+ENUM_MO_CONTAINER_AUTO_RESTART="no,yes"
+ENUM_CSV_CONVERT_TYPE="1,2,3"
+ENUM_CSV_CONVERT_TN_TYPE="1,2"
+ENUM_CSV_CONVERT_INSERT_ADD_QUOTE="no,yes"
+ENUM_BACKUP_TYPE="logical,physical"
+ENUM_BACKUP_S3_IS_MINIO="no,yes"
+ENUM_BACKUP_LOGICAL_DATA_TYPE="ddl,insert,csv"
+ENUM_BACKUP_LOGICAL_ONEBYONE="0,1"
+
+
+
 function set_kv()
 {
     key="$1"
@@ -22,20 +40,39 @@ function set_kv()
         return 1
     fi
 
-    # 2. check if value is not empty, that is, a valid value
-    if [[ "${value}" == "" ]]; then
-        case "${key}" in
-            "MO_CONTAINER_DATA_HOST_PATH" | "MO_CONTAINER_CONF_HOST_PATH" | "CSV_CONVERT_META_COLUMN_LIST" | "MO_CONTAINER_LIMIT_CPU" | "MO_CONTAINER_LIMIT_MEMORY" | "MO_CONTAINER_EXTRA_MOUNT_OPTION" | "MO_CONF_SRC_PATH")
-                :
-                ;;
-            *)
-                add_log "E" "The value of conf ${key} is empty, which is not allowed"
-                rc=1
+    # 2. Validity check
+    case "${key}" in
+        # 2.1 enum list
+        "TOOL_LOG_LEVEL"| "MO_DEPLOY_MODE"| "MO_SERVER_TYPE"| "MO_CONTAINER_AUTO_RESTART"| "CSV_CONVERT_TYPE"| "CSV_CONVERT_TN_TYPE"| "CSV_CONVERT_INSERT_ADD_QUOTE"| "BACKUP_TYPE"| "BACKUP_S3_IS_MINIO"| "BACKUP_LOGICAL_DATA_TYPE"| "BACKUP_LOGICAL_ONEBYONE")
+
+            enum_list=`eval echo '$'"ENUM_$key"`
+            enum_list_2=`echo "${enum_list}" | sed "s/,/\|/g"`
+            found="false"
+            for enum in $(echo "${enum_list}" | sed "s/,/ /g"); do
+
+                if [[ "${value}" == "${enum}" ]]; then
+                    found="true"
+                fi
+            done
+
+            if [[ "${found}" == "false" ]]; then
+                add_log "E" "The value '${value}' of key '${key}' is not valid, valid range: ${enum_list_2}"
                 return 1
+            fi
+
             ;;
-        esac 
-
-    fi
+
+        # TODO: other parameters
+        "MO_CONTAINER_DATA_HOST_PATH" | "MO_CONTAINER_CONF_HOST_PATH" | "CSV_CONVERT_META_COLUMN_LIST" | "MO_CONTAINER_LIMIT_CPU" | "MO_CONTAINER_LIMIT_MEMORY" | "MO_CONTAINER_EXTRA_MOUNT_OPTION" | "MO_CONF_SRC_PATH")
+            :
+            ;;
+        *)
+            :
+            #add_log "E" "The value of conf ${key} is empty, which is not allowed"
+            #rc=1
+            #return 1
+            ;;
+    esac
 
     # 3. Set conf key=value
     add_log "I" "Setting conf ${key}=\"${value}\""

diff --git a/bin/version.sh b/bin/version.sh
@@ -7,6 +7,6 @@
 
 function version()
 {
-    echo "Tool version: ${MO_TOOL_NAME} ${MO_TOOL_VERSION}"
+    echo "Tool version: ${TOOL_NAME} ${TOOL_VERSION}"
     echo "Server version: ${MO_SERVER_NAME} ${MO_SERVER_VERSION}"
 }
diff --git a/conf/env.sh b/conf/env.sh
@@ -39,9 +39,9 @@ MO_DEPLOY_MODE="git"
 
 
 # for docker
-MO_REPO="matrixorigin/matrixone"
+# deprecated: MO_REPO="matrixorigin/matrixone"
 # full image name of mo container, default: matrixorigin/matrixone:1.0.1
-MO_CONTAINER_IMAGE="matrixorigin/matrixone:1.1.0"
+MO_CONTAINER_IMAGE="matrixorigin/matrixone:1.1.1"
 # mo container name
 MO_CONTAINER_NAME="mo"
 # mo container sql port (PS: constant value, DONT CHANGE)
@@ -59,7 +59,7 @@ MO_CONTAINER_HOSTNAME="705203be8a9e"
 # mo container limit for memory (unit: m) (e.g. 1000 | 1500 | 2000 | ...)
 MO_CONTAINER_LIMIT_MEMORY=""
 # use ratio to set mo container limit for memory based on the total memory of the machine (unit: %)
-MO_CONTAINER_MEMORY_RATIO=90
+MO_CONTAINER_MEMORY_RATIO=70
 # auto restart mo container in case it is down? (yes|no)
 MO_CONTAINER_AUTO_RESTART="yes"
 # mo container limit for cpu (e.g. 1 | 1.5 | 2 | ...)
@@ -91,7 +91,7 @@ MO_GIT_URL="https://github.com/matrixorigin/matrixone.git"
 #)
 
 # default version of which mo to be deployed
-MO_DEFAULT_VERSION="v1.1.0"
+MO_DEFAULT_VERSION="v1.1.1"
 # which go proxy to be used when downloading go dependencies
 # you can set this go proxy when building mo-service
 GOPROXY="https://goproxy.cn,direct"
@@ -216,9 +216,20 @@ BACKUP_MODUMP_PATH="/data/tools/mo_dump/mo-dump"
 # all_no_sysdb: (default) all databases, including all user databases, but no system databases
 # other settings by user, e.g. db1,db2,db3
 BACKUP_LOGICAL_DB_LIST="all_no_sysdb"
-# backup data type(only valid when BACKUP_TYPE=logical) : insert | csv(default)
+
+# backup tables, seperated by ','
+# Note: BACKUP_LOGICAL_TBL_LIST is only vailid when BACKUP_LOGICAL_DB_LIST has exactly one database, otherwise it will be ignored
+BACKUP_LOGICAL_TBL_LIST=""
+
+# backup data type(only valid when BACKUP_TYPE=logical) : insert | csv(default) | ddl
 BACKUP_LOGICAL_DATA_TYPE="csv"
 
+# backup data per database one by one (only valid when BACKUP_TYPE=logical): 0 (default, all at once) | 1 (one by one)
+BACKUP_LOGICAL_ONEBYONE="0"
+
+# backup net buffer length(integer): 1048576(default, 1M), Max is 16777216 (16M)
+BACKUP_LOGICAL_NETBUFLEN="1048576"
+
 
 # for auto clean sysdb logs
 # clean old sysdb logs before [x] (default: 31) days

diff --git a/conf/env.sh.default b/conf/env.sh.default
@@ -216,9 +216,20 @@ BACKUP_MODUMP_PATH="/data/tools/mo_dump/mo-dump"
 # all_no_sysdb: (default) all databases, including all user databases, but no system databases
 # other settings by user, e.g. db1,db2,db3
 BACKUP_LOGICAL_DB_LIST="all_no_sysdb"
-# backup data type(only valid when BACKUP_TYPE=logical) : insert | csv(default)
+
+# backup tables, seperated by ','
+# Note: BACKUP_LOGICAL_TBL_LIST is only vailid when BACKUP_LOGICAL_DB_LIST has exactly one database, otherwise it will be ignored
+BACKUP_LOGICAL_TBL_LIST=""
+
+# backup data type(only valid when BACKUP_TYPE=logical) : insert | csv(default) | ddl
 BACKUP_LOGICAL_DATA_TYPE="csv"
 
+# backup data per database one by one (only valid when BACKUP_TYPE=logical): 0 (default, all at once) | 1 (one by one)
+BACKUP_LOGICAL_ONEBYONE="0"
+
+# backup net buffer length(integer): 1048576(default, 1M), Max is 16777216 (16M)
+BACKUP_LOGICAL_NETBUFLEN="1048576"
+
 
 # for auto clean sysdb logs
 # clean old sysdb logs before [x] (default: 31) days