diff --git a/bin/auto_backup.sh b/bin/auto_backup.sh index bf33f5c..8a1707d 100644 --- a/bin/auto_backup.sh +++ b/bin/auto_backup.sh @@ -143,7 +143,8 @@ function backup() backup_db_list=${BACKUP_LOGICAL_DB_LIST} ;; esac - + + add_log "D" "backup_db_list: ${backup_db_list}" if [[ "${backup_db_list}" == "" ]]; then add_log "E" "Final backup database list seems to be empty, please check conf BACKUP_LOGICAL_DB_LIST" return 1 @@ -151,27 +152,112 @@ function backup() csv_option="" + no_data_option="" if [[ "${BACKUP_LOGICAL_DATA_TYPE}" == "csv" ]]; then csv_option="-csv" + elif [[ "${BACKUP_LOGICAL_DATA_TYPE}" == "ddl" ]]; then + no_data_option="-no-data" fi - - for db in $(echo "${backup_db_list}" | sed "s/,/ /g"); do - add_log "I" "Begin to back up database: ${db}" - - startTime=`get_nanosecond` - add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}.sql && cd -" - if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}.sql && cd - >/dev/null 2>&1; then - endTime=`get_nanosecond` - outcome="succeeded" + + # 1. in case we have multiple databases + if echo "${backup_db_list}" | grep "," >/dev/null 2>&1 ; then + add_log "W" "backup_db_list=${backup_db_list} seems to be a list containing multiple dbs, thus will ignore conf BACKUP_LOGICAL_TBL_LIST=${BACKUP_LOGICAL_TBL_LIST} and backup databases in db list only" + + # 1.1. backup databases one by one + if [[ "${BACKUP_LOGICAL_ONEBYONE}" == "1" ]]; then + add_log "D" "BACKUP_LOGICAL_ONEBYONE is set to 1, will backup tables one by one" + + for db in $(echo "${backup_db_list}" | sed "s/,/ /g"); do + add_log "I" "Begin to back up database: ${db}" + + add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}.sql && cd - >/dev/null 2>&1" + startTime=`get_nanosecond` + if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}.sql && cd - >/dev/null 2>&1; then + endTime=`get_nanosecond` + outcome="succeeded" + else + endTime=`get_nanosecond` + outcome="failed" + + fi + + cost=`time_cost_ms ${startTime} ${endTime}` + + add_log "I" "End with outcome: ${outcome}, cost: ${cost} ms" + done + + # 1.2. backup databases all at once else - endTime=`get_nanosecond` - outcome="failed" + add_log "D" "BACKUP_LOGICAL_ONEBYONE is not set to 1, will backup databases all at once" + add_log "I" "Begin to back up databases in list: ${backup_db_list}" + add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/mo.sql && cd - >/dev/null 2>&1" + + startTime=`get_nanosecond` + if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/mo.sql && cd - >/dev/null 2>&1; then + endTime=`get_nanosecond` + outcome="succeeded" + else + endTime=`get_nanosecond` + outcome="failed" + + fi + cost=`time_cost_ms ${startTime} ${endTime}` + + add_log "I" "End with outcome: ${outcome}, cost: ${cost} ms" fi - cost=`time_cost_ms ${startTime} ${endTime}` - add_log "I" "End with outcome: ${outcome}, cost: ${cost} ms" - done + # 2. in case we have only one database + else + add_log "D" "backup_db_list=${backup_db_list} seems to be one exact database, thus will take conf BACKUP_LOGICAL_TBL_LIST=${BACKUP_LOGICAL_TBL_LIST} into consideration" + + if [[ "${BACKUP_LOGICAL_ONEBYONE}" == "1" ]]; then + add_log "D" "BACKUP_LOGICAL_ONEBYONE is set to 1, will backup tables one by one" + + for tbl in $(echo "${BACKUP_LOGICAL_TBL_LIST}" | sed "s/,/ /g"); do + add_log "I" "Begin to back up table: ${tbl}" + + add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${tbl} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}_${tbl}.sql && cd - >/dev/null 2>&1" + startTime=`get_nanosecond` + if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${tbl} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}_${tbl}.sql && cd - >/dev/null 2>&1; then + endTime=`get_nanosecond` + outcome="succeeded" + else + endTime=`get_nanosecond` + outcome="failed" + + fi + cost=`time_cost_ms ${startTime} ${endTime}` + + add_log "I" "End with outcome: ${outcome}, cost: ${cost} ms" + done + + + # backup tables all at once + else + add_log "D" "BACKUP_LOGICAL_ONEBYONE is not set to 1, will backup tables all at once" + add_log "I" "Begin to back up tables in list: ${BACKUP_LOGICAL_TBL_LIST} in database ${backup_db_list}" + add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${BACKUP_LOGICAL_TBL_LIST} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${backup_db_list}.sql && cd - >/dev/null 2>&1" + + startTime=`get_nanosecond` + if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${BACKUP_LOGICAL_TBL_LIST} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${backup_db_list}.sql && cd - >/dev/null 2>&1; then + endTime=`get_nanosecond` + outcome="succeeded" + else + endTime=`get_nanosecond` + outcome="failed" + + fi + cost=`time_cost_ms ${startTime} ${endTime}` + + add_log "I" "End with outcome: ${outcome}, cost: ${cost} ms" + + fi + + + fi + + ;; # 2) physical backups : mo_br diff --git a/bin/deploy.sh b/bin/deploy.sh index 340f076..bc71334 100644 --- a/bin/deploy.sh +++ b/bin/deploy.sh @@ -185,14 +185,15 @@ function deploy_docker() if [[ "${MO_CONTAINER_IMAGE}" == "" ]]; then add_log "E" "conf MO_CONTAINER_IMAGE is empty, please set it first" fi + + # 2024/4/2: deprecated as we don't pull image manually + #add_log "I" "Pulling image ${MO_CONTAINER_IMAGE}" + #if ! docker pull ${MO_CONTAINER_IMAGE}; then + # add_log "E" "Failed to pull docker image, please check if ${MO_CONTAINER_IMAGE} is a correct image or it might be a network issue" + # return 1 + #fi - add_log "I" "Pulling image ${MO_CONTAINER_IMAGE}" - if ! docker pull ${MO_CONTAINER_IMAGE}; then - add_log "E" "Failed to pull docker image, please check if ${MO_CONTAINER_IMAGE} is a correct image or it might be a network issue" - return 1 - fi - - add_log "I" "Successfully pulled image ${MO_CONTAINER_IMAGE}" + #add_log "I" "Successfully pulled image ${MO_CONTAINER_IMAGE}" } function deploy() diff --git a/bin/pprof.sh b/bin/pprof.sh index 49d3294..e5532ad 100644 --- a/bin/pprof.sh +++ b/bin/pprof.sh @@ -32,8 +32,8 @@ function pprof() add_log "I" "pprof option is ${option}" - case ${option} in - profile) + case "${option}" in + "profile" | "trace" ) if [[ "${duration}" == "" ]]; then add_log "I" "duration is not set, using conf value: ${PPROF_PROFILE_DURATION}" duration="${PPROF_PROFILE_DURATION}" @@ -51,11 +51,8 @@ function pprof() URL="${URL}?seconds=${duration}"; add_log "I" "collect duration is ${duration} seconds" ;; - allocs) - a=1 - ;; - heap) - a=1 + "allocs" | "heap" | "goroutine") + : ;; *) add_log "E" "Invalid option ${option} for pprof. Available: profile | pprof | heap" diff --git a/bin/set_conf.sh b/bin/set_conf.sh index 76cb061..0416c6d 100644 --- a/bin/set_conf.sh +++ b/bin/set_conf.sh @@ -5,6 +5,24 @@ ################################################################ # set_conf + +# const + +# enum +ENUM_TOOL_LOG_LEVEL="d,D,I,i,W,w,E,e" +ENUM_MO_DEPLOY_MODE="git,docker,binary" +ENUM_MO_SERVER_TYPE="local,remote" +ENUM_MO_CONTAINER_AUTO_RESTART="no,yes" +ENUM_CSV_CONVERT_TYPE="1,2,3" +ENUM_CSV_CONVERT_TN_TYPE="1,2" +ENUM_CSV_CONVERT_INSERT_ADD_QUOTE="no,yes" +ENUM_BACKUP_TYPE="logical,physical" +ENUM_BACKUP_S3_IS_MINIO="no,yes" +ENUM_BACKUP_LOGICAL_DATA_TYPE="ddl,insert,csv" +ENUM_BACKUP_LOGICAL_ONEBYONE="0,1" + + + function set_kv() { key="$1" @@ -22,20 +40,39 @@ function set_kv() return 1 fi - # 2. check if value is not empty, that is, a valid value - if [[ "${value}" == "" ]]; then - case "${key}" in - "MO_CONTAINER_DATA_HOST_PATH" | "MO_CONTAINER_CONF_HOST_PATH" | "CSV_CONVERT_META_COLUMN_LIST" | "MO_CONTAINER_LIMIT_CPU" | "MO_CONTAINER_LIMIT_MEMORY" | "MO_CONTAINER_EXTRA_MOUNT_OPTION" | "MO_CONF_SRC_PATH") - : - ;; - *) - add_log "E" "The value of conf ${key} is empty, which is not allowed" - rc=1 + # 2. Validity check + case "${key}" in + # 2.1 enum list + "TOOL_LOG_LEVEL"| "MO_DEPLOY_MODE"| "MO_SERVER_TYPE"| "MO_CONTAINER_AUTO_RESTART"| "CSV_CONVERT_TYPE"| "CSV_CONVERT_TN_TYPE"| "CSV_CONVERT_INSERT_ADD_QUOTE"| "BACKUP_TYPE"| "BACKUP_S3_IS_MINIO"| "BACKUP_LOGICAL_DATA_TYPE"| "BACKUP_LOGICAL_ONEBYONE") + + enum_list=`eval echo '$'"ENUM_$key"` + enum_list_2=`echo "${enum_list}" | sed "s/,/\|/g"` + found="false" + for enum in $(echo "${enum_list}" | sed "s/,/ /g"); do + + if [[ "${value}" == "${enum}" ]]; then + found="true" + fi + done + + if [[ "${found}" == "false" ]]; then + add_log "E" "The value '${value}' of key '${key}' is not valid, valid range: ${enum_list_2}" return 1 + fi + ;; - esac - - fi + + # TODO: other parameters + "MO_CONTAINER_DATA_HOST_PATH" | "MO_CONTAINER_CONF_HOST_PATH" | "CSV_CONVERT_META_COLUMN_LIST" | "MO_CONTAINER_LIMIT_CPU" | "MO_CONTAINER_LIMIT_MEMORY" | "MO_CONTAINER_EXTRA_MOUNT_OPTION" | "MO_CONF_SRC_PATH") + : + ;; + *) + : + #add_log "E" "The value of conf ${key} is empty, which is not allowed" + #rc=1 + #return 1 + ;; + esac # 3. Set conf key=value add_log "I" "Setting conf ${key}=\"${value}\"" diff --git a/bin/version.sh b/bin/version.sh index 2aed3c6..8bf29c1 100644 --- a/bin/version.sh +++ b/bin/version.sh @@ -7,6 +7,6 @@ function version() { - echo "Tool version: ${MO_TOOL_NAME} ${MO_TOOL_VERSION}" + echo "Tool version: ${TOOL_NAME} ${TOOL_VERSION}" echo "Server version: ${MO_SERVER_NAME} ${MO_SERVER_VERSION}" } diff --git a/conf/env.sh b/conf/env.sh index 9d03506..611c3a3 100644 --- a/conf/env.sh +++ b/conf/env.sh @@ -39,9 +39,9 @@ MO_DEPLOY_MODE="git" # for docker -MO_REPO="matrixorigin/matrixone" +# deprecated: MO_REPO="matrixorigin/matrixone" # full image name of mo container, default: matrixorigin/matrixone:1.0.1 -MO_CONTAINER_IMAGE="matrixorigin/matrixone:1.1.0" +MO_CONTAINER_IMAGE="matrixorigin/matrixone:1.1.1" # mo container name MO_CONTAINER_NAME="mo" # mo container sql port (PS: constant value, DONT CHANGE) @@ -59,7 +59,7 @@ MO_CONTAINER_HOSTNAME="705203be8a9e" # mo container limit for memory (unit: m) (e.g. 1000 | 1500 | 2000 | ...) MO_CONTAINER_LIMIT_MEMORY="" # use ratio to set mo container limit for memory based on the total memory of the machine (unit: %) -MO_CONTAINER_MEMORY_RATIO=90 +MO_CONTAINER_MEMORY_RATIO=70 # auto restart mo container in case it is down? (yes|no) MO_CONTAINER_AUTO_RESTART="yes" # mo container limit for cpu (e.g. 1 | 1.5 | 2 | ...) @@ -91,7 +91,7 @@ MO_GIT_URL="https://github.com/matrixorigin/matrixone.git" #) # default version of which mo to be deployed -MO_DEFAULT_VERSION="v1.1.0" +MO_DEFAULT_VERSION="v1.1.1" # which go proxy to be used when downloading go dependencies # you can set this go proxy when building mo-service GOPROXY="https://goproxy.cn,direct" @@ -216,9 +216,20 @@ BACKUP_MODUMP_PATH="/data/tools/mo_dump/mo-dump" # all_no_sysdb: (default) all databases, including all user databases, but no system databases # other settings by user, e.g. db1,db2,db3 BACKUP_LOGICAL_DB_LIST="all_no_sysdb" -# backup data type(only valid when BACKUP_TYPE=logical) : insert | csv(default) + +# backup tables, seperated by ',' +# Note: BACKUP_LOGICAL_TBL_LIST is only vailid when BACKUP_LOGICAL_DB_LIST has exactly one database, otherwise it will be ignored +BACKUP_LOGICAL_TBL_LIST="" + +# backup data type(only valid when BACKUP_TYPE=logical) : insert | csv(default) | ddl BACKUP_LOGICAL_DATA_TYPE="csv" +# backup data per database one by one (only valid when BACKUP_TYPE=logical): 0 (default, all at once) | 1 (one by one) +BACKUP_LOGICAL_ONEBYONE="0" + +# backup net buffer length(integer): 1048576(default, 1M), Max is 16777216 (16M) +BACKUP_LOGICAL_NETBUFLEN="1048576" + # for auto clean sysdb logs # clean old sysdb logs before [x] (default: 31) days diff --git a/conf/env.sh.default b/conf/env.sh.default index 9d03506..d5abe81 100644 --- a/conf/env.sh.default +++ b/conf/env.sh.default @@ -216,9 +216,20 @@ BACKUP_MODUMP_PATH="/data/tools/mo_dump/mo-dump" # all_no_sysdb: (default) all databases, including all user databases, but no system databases # other settings by user, e.g. db1,db2,db3 BACKUP_LOGICAL_DB_LIST="all_no_sysdb" -# backup data type(only valid when BACKUP_TYPE=logical) : insert | csv(default) + +# backup tables, seperated by ',' +# Note: BACKUP_LOGICAL_TBL_LIST is only vailid when BACKUP_LOGICAL_DB_LIST has exactly one database, otherwise it will be ignored +BACKUP_LOGICAL_TBL_LIST="" + +# backup data type(only valid when BACKUP_TYPE=logical) : insert | csv(default) | ddl BACKUP_LOGICAL_DATA_TYPE="csv" +# backup data per database one by one (only valid when BACKUP_TYPE=logical): 0 (default, all at once) | 1 (one by one) +BACKUP_LOGICAL_ONEBYONE="0" + +# backup net buffer length(integer): 1048576(default, 1M), Max is 16777216 (16M) +BACKUP_LOGICAL_NETBUFLEN="1048576" + # for auto clean sysdb logs # clean old sysdb logs before [x] (default: 31) days