diff --git a/bin/auto_backup.sh b/bin/auto_backup.sh index 917edcd..9e139ad 100644 --- a/bin/auto_backup.sh +++ b/bin/auto_backup.sh @@ -70,12 +70,19 @@ function backup_list() fi + br_meta_option="" + if [[ "${BACKUP_MOBR_META_PATH}" != "" ]]; then + br_meta_option="--meta_path ${BACKUP_MOBR_META_PATH}" + #add_log "D" "BACKUP_MOBR_META_PATH is not empty, will add option ${br_meta_option}" + fi + #add_log "I" "Listing backup report (detail, physical only)" #add_log "I" "------------------------------------" - cd ${BACKUP_MOBR_DIRNAME} && ./mo_br list + #add_log "D" "cmd: cd ${BACKUP_MOBR_DIRNAME} && ./mo_br ${br_meta_option} list" + cd ${BACKUP_MOBR_DIRNAME} && ./mo_br ${br_meta_option} list else if [[ ! -f ${BACKUP_REPORT} ]]; then - add_log "E" "No backup action can be found, exiting" + add_log "E" "BACKUP_REPORT ${BACKUP_REPORT} is not a valid file, exiting" return 1 fi #add_log "I" "Listing backup report (summary) from ${BACKUP_REPORT}" @@ -87,7 +94,9 @@ function backup_list() function backup() { - ! backup_precheck "mo" && return 1 + if ! backup_precheck "mo"; then + return 1 + fi add_log "I" "Backup settings" add_log "I" "------------------------------------" @@ -95,7 +104,7 @@ function backup() add_log "I" "------------------------------------" - add_log "I" "Backup starts" + add_log "I" "Backup begins" backup_yearmonth=`date '+%Y%m'` backup_timestamp=`date '+%Y%m%d_%H%M%S'` @@ -112,7 +121,7 @@ function backup() mkdir -p ${backup_report_path} if [[ ! -f ${BACKUP_REPORT} ]]; then add_log "D" "Creating backup report file ${BACKUP_REPORT}" - echo "backup_date|backup_target|ds_name|db_list|backup_type|backup_path|logical_data_type|duration_ms|outcome|size_in_bytes" > "${BACKUP_REPORT}" + echo "backup_date|backup_target|ds_name|db_list|backup_type|backup_path|logical_data_type|duration_ms|outcome|bk_size_in_bytes|logical_net_buffer_length" > "${BACKUP_REPORT}" fi backup_db_list="" @@ -124,8 +133,11 @@ function backup() # 1) logical backups : mo_dump "logical") logical_data_type=${BACKUP_LOGICAL_DATA_TYPE} + net_buffer_length="${BACKUP_LOGICAL_NETBUFLEN}" - ! backup_precheck "modump" && return 1 + if ! backup_precheck "modump";then + return 1 + fi all_dbs=`MYSQL_PWD="${MO_PW}" mysql -u"${MO_USER}" -P"${MO_PORT}" -h"${MO_HOST}" -e "show databases" -N -s` add_log "D" "All databases in current system: ${all_dbs}" @@ -286,17 +298,25 @@ function backup() #backup_db_list="all" backup_conf_db_list="all" logical_data_type="n.a." + net_buffer_length="n.a." - ! backup_precheck "mobr" && return 1 + if ! backup_precheck "mobr"; then + return 1 + fi + br_meta_option="" + if [[ "${BACKUP_MOBR_META_PATH}" != "" ]]; then + br_meta_option="--meta_path ${BACKUP_MOBR_META_PATH}" + add_log "D" "BACKUP_MOBR_META_PATH is not empty, will add option ${br_meta_option}" + fi BACKUP_MOBR_DIRNAME=`dirname "${BACKUP_MOBR_PATH}"` case "${BACKUP_PHYSICAL_TYPE}" in "filesystem") - add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"filesystem\" --path \"${backup_outpath}/\"" + add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"filesystem\" --path \"${backup_outpath}/\"" startTime=`get_nanosecond` - if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "filesystem" --path "${backup_outpath}/" ; then + if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "filesystem" --path "${backup_outpath}/" ; then outcome="succeeded" else outcome="failed" @@ -305,7 +325,7 @@ function backup() ;; "s3") minio_option="" - if [[ "${BACKUP_S3_IS_MINIO}" != "no" ]]; then + if [[ "${BACKUP_S3_IS_MINIO}" == "yes" ]]; then minio_option="--is_minio" fi @@ -314,10 +334,10 @@ function backup() role_arn_option="--role_arn ${BACKUP_S3_ROLE_ARN}" fi - add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"s3\" --endpoint \"${BACKUP_S3_ENDPOINT}\" --access_key_id \"${BACKUP_S3_ID}\" --secret_access_key \"${BACKUP_S3_KEY}\" --bucket \"${BACKUP_S3_BUCKET}\" --filepath \"${BACKUP_DATA_PATH}\" --region \"${BACKUP_S3_REGION}\" --compression \"${BACKUP_S3_COMPRESSION}\" \"${role_arn_option}\" \"${minio_option}\"" + add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"s3\" --endpoint \"${BACKUP_S3_ENDPOINT}\" --access_key_id \"${BACKUP_S3_ID}\" --secret_access_key \"${BACKUP_S3_KEY}\" --bucket \"${BACKUP_S3_BUCKET}\" --filepath \"${BACKUP_DATA_PATH}\" --region \"${BACKUP_S3_REGION}\" --compression \"${BACKUP_S3_COMPRESSION}\" \"${role_arn_option}\" \"${minio_option}\"" startTime=`get_nanosecond` - if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "s3" --endpoint "${BACKUP_S3_ENDPOINT}" --access_key_id "${BACKUP_S3_ID}" --secret_access_key "${BACKUP_S3_KEY}" --bucket "${BACKUP_S3_BUCKET}" --filepath "${BACKUP_DATA_PATH}" --region "${BACKUP_S3_REGION}" --compression "${BACKUP_S3_COMPRESSION}" "${role_arn_option}" "${minio_option}"; then + if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "s3" --endpoint "${BACKUP_S3_ENDPOINT}" --access_key_id "${BACKUP_S3_ID}" --secret_access_key "${BACKUP_S3_KEY}" --bucket "${BACKUP_S3_BUCKET}" --filepath "${BACKUP_DATA_PATH}" --region "${BACKUP_S3_REGION}" --compression "${BACKUP_S3_COMPRESSION}" "${role_arn_option}" "${minio_option}"; then outcome="succeeded" else outcome="failed" @@ -337,13 +357,19 @@ function backup() add_log "I" "End with outcome: ${outcome}, cost: ${cost} ms" esac + + bk_size="n.a." if [[ ${outcome} == "succeeded" ]]; then + add_log "D" "Calculating size of backup path ${backup_outpath}" bk_size=`du -s ${backup_outpath} | awk '{print $1}'` fi + # output report record + add_log "D" "Writing entry to report ${BACKUP_REPORT}" + add_log "D" "${backup_timestamp}|${bakcup_target}|${BACKUP_LOGICAL_DS}|${backup_conf_db_list}|${BACKUP_TYPE}|${backup_outpath}|${logical_data_type}|${cost}|${outcome}|${bk_size}|${net_buffer_length}" bakcup_target="${MO_HOST},${MO_PORT},${MO_USER}" - echo "${backup_timestamp}|${bakcup_target}|${BACKUP_LOGICAL_DS}|${backup_conf_db_list}|${BACKUP_TYPE}|${backup_outpath}/${backup_yearmonth}/${backup_timestamp}/|${logical_data_type}|${cost}|${outcome}|${bk_size}" >> "${BACKUP_REPORT}" + echo "${backup_timestamp}|${bakcup_target}|${BACKUP_LOGICAL_DS}|${backup_conf_db_list}|${BACKUP_TYPE}|${backup_outpath}|${logical_data_type}|${cost}|${outcome}|${bk_size}|${net_buffer_length}" >> "${BACKUP_REPORT}" if [[ ${rc} -ne 0 ]]; then add_log "E" "Backup ends with non-zero rc" diff --git a/bin/csv_convert.sh b/bin/csv_convert.sh index 00878a7..c108731 100644 --- a/bin/csv_convert.sh +++ b/bin/csv_convert.sh @@ -421,20 +421,36 @@ function csv_convert() case "${CSV_CONVERT_TYPE}" in "1") convert_type="insert" - ! csv_convert_prep ${convert_type} && return 1 - ! csv_convert_insert_${run_mode} && return 1 + if ! csv_convert_prep ${convert_type}; then + return 1 + fi + + if ! csv_convert_insert_${run_mode}; then + return 1 + fi + ;; "2") line_mode="single" convert_type="load-singleline" - ! csv_convert_prep ${convert_type} && return 1 - ! csv_convert_load_${run_mode} "${line_mode}" && return 1 + if ! csv_convert_prep ${convert_type}; then + return 1 + fi + + if ! csv_convert_load_${run_mode} "${line_mode}"; then + return 1 + fi ;; "3") line_mode="multi" convert_type="load-multilines" - ! csv_convert_prep ${convert_type} && return 1 - ! csv_convert_load_${run_mode} "${line_mode}" && return 1 + if ! csv_convert_prep ${convert_type}; then + return 1 + fi + + if ! csv_convert_load_${run_mode} "${line_mode}"; then + return 1 + fi ;; *) return 1 diff --git a/bin/help.sh b/bin/help.sh index 47fdaa8..54dfa74 100644 --- a/bin/help.sh +++ b/bin/help.sh @@ -55,21 +55,27 @@ function help_precheck() function help_deploy() { option="deploy" - echo "Usage : ${TOOL_NAME} ${option} [mo_version] [force] # ${USAGE_DEPLOY}" + echo "Usage : ${TOOL_NAME} ${option} [\$mo_version] [force] [nobuild] # ${USAGE_DEPLOY}" echo " : ${TOOL_NAME} ${option} help # print help info" - echo " [mo_version]: optional: specify an mo version to deploy" + echo " [\$mo_version]: optional: specify an mo version to deploy" echo " [force] : optional: if specified will delete all content under MO_PATH and deploy from beginning" + echo " [nobuild] : optional: if specified will skip building mo-service" echo " Note : 'deploy' is valid only when MO_DEPLOY_MODE is set to 'git' or 'docker'" echo " e.g. : ${TOOL_NAME} ${option} # default, same as ${TOOL_NAME} ${option} ${MO_DEFAULT_VERSION}" echo " : ${TOOL_NAME} ${option} main # deploy development latest version" echo " : ${TOOL_NAME} ${option} d29764a # deploy development version d29764a" - echo " : ${TOOL_NAME} ${option} 0.8.0 # deploy stable verson 0.8.0" + echo " : ${TOOL_NAME} ${option} 1.2.0 # deploy stable verson 1.2.0" echo " : ${TOOL_NAME} ${option} force # delete all under MO_PATH and deploy verson ${MO_DEFAULT_VERSION}" - echo " : ${TOOL_NAME} ${option} 0.8.0 force # delete all under MO_PATH and deploy stable verson 0.8.0 from beginning" + echo " : ${TOOL_NAME} ${option} 1.2.0 force # delete all under MO_PATH and deploy stable verson 1.2.0 from beginning" } - +function help_deploy_docker() +{ + option="deploy" + echo "Usage : ${TOOL_NAME} ${option} # ${USAGE_DEPLOY}" + echo " : ${TOOL_NAME} ${option} help # print help info" +} function help_status() { @@ -203,11 +209,11 @@ function help_upgrade() option="upgrade" echo "Usage : ${TOOL_NAME} ${option} [version_or_commitid] # ${USAGE_UPGRADE}" echo " : ${TOOL_NAME} ${option} help # print help info" - echo " [commitid] : a commit id such as '38888f7', or a stable version such as '0.8.0'" + echo " [commitid] : a commit id such as '38888f7', or a stable version such as '1.2.0'" echo " : use 'latest' to upgrade to latest commit on main branch if you don't know the id" echo " e.g. : ${TOOL_NAME} ${option} 38888f7 # upgrade/downgrade to commit id 38888f7 on main branch" echo " : ${TOOL_NAME} ${option} latest # upgrade/downgrade to latest commit on main branch" - echo " : ${TOOL_NAME} ${option} 0.8.0 # upgrade/downgrade to stable version 0.8.0" + echo " : ${TOOL_NAME} ${option} 1.2.0 # upgrade/downgrade to stable version 1.2.0" } diff --git a/bin/restore.sh b/bin/restore.sh index 23e7515..08c74cb 100644 --- a/bin/restore.sh +++ b/bin/restore.sh @@ -9,6 +9,7 @@ ab_name="restore" OS="" + function restore_precheck() { @@ -73,7 +74,7 @@ function restore_prep_report() mkdir -p ${restore_report_path} if [[ ! -f "${RESTORE_REPORT}" ]]; then add_log "D" "Creating restore report file ${RESTORE_REPORT}" - echo "restore_date|restore_target|restore_type|physical_bk_id|logical_restore_src|logical_type|duration_ms|outcome" > "${RESTORE_REPORT}" + echo "restore_date|restore_target|restore_type|physical_bk_id|logical_restore_src|logical_type|duration_ms|outcome|bk_size_in_bytes" > "${RESTORE_REPORT}" fi } @@ -85,9 +86,13 @@ function restore_physical() return 1 fi - ! restore_precheck "mo" "down" && return 1 + if ! restore_precheck "mo" "down"; then + return 1 + fi - ! restore_precheck "mobr" && return 1 + if ! restore_precheck "mobr"; then + return 1 + fi add_log "I" "Restore settings" add_log "I" "------------------------------------" @@ -96,12 +101,22 @@ function restore_physical() add_log "I" "Step_1. Restore physical data" + if [[ "${RESTORE_BKID}" == "" ]]; then + add_log "E" "RESTORE_BKID seems to be empty, please set it first, e.g. mo_ctl set_conf RESTORE_BKID=xxxxxxx , exiting" + return 1 + fi + add_log "I" "Restore begins" + br_meta_option="" + if [[ "${BACKUP_MOBR_META_PATH}" != "" ]]; then + br_meta_option="--meta_path ${BACKUP_MOBR_META_PATH}" + add_log "D" "BACKUP_MOBR_META_PATH is not empty, will add option ${br_meta_option}" + fi BACKUP_MOBR_DIRNAME=`dirname "${BACKUP_MOBR_PATH}"` - cmd="${BACKUP_MOBR_PATH} restore ${RESTORE_BKID} --restore_dir ${RESTORE_PHYSICAL_TYPE}" + cmd="${BACKUP_MOBR_PATH} restore ${RESTORE_BKID} ${br_meta_option} --restore_dir ${RESTORE_PHYSICAL_TYPE}" case "${RESTORE_PHYSICAL_TYPE}" in "filesystem") cmd="${cmd} --restore_path ${RESTORE_PATH}" @@ -150,8 +165,30 @@ function restore_physical() add_log "I" "Outcome: ${outcome}, cost: ${cost} ms" - restore_prep_report - echo "${restore_timestamp}|${MO_HOST},${MO_PORT},${MO_USER}|physical|${RESTORE_BKID}|||${cost}|${outcome}" >> ${RESTORE_REPORT} + bk_size="n.a." + bk_size_in_bytes="n.a." + if [[ ${outcome} == "succeeded" ]]; then + add_log "D" "Get size of backup id ${RESTORE_BKID}" + bk_size=`cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} list ${br_meta_option} | grep "${RESTORE_BKID}" | awk -F "|" '{print $3}' | sed 's# ##g'` + if echo "${bk_size}" | grep "kB"; then + number=`echo "${bk_size}" | awk -F "kB" '{print $1}'` + let bk_size_in_bytes=bk_size*1024 + elif echo "${bk_size}" | grep "MB"; then + number=`echo "${bk_size}" | awk -F "MB" '{print $1}'` + let bk_size_in_bytes=bk_size*1024*1024 + elif echo "${bk_size}" | grep "GB"; then + number=`echo "${bk_size}" | awk -F "GB" '{print $1}'` + let bk_size_in_bytes=bk_size*1024*1024*1024 + else + bk_size_in_bytes="${bk_size}" + fi + add_log "D" "bk_size: ${bk_size}, bk_size_in_bytes: ${bk_size_in_bytes}" + + fi + + add_log "D" "Writing entry to report" + add_log "D" "${restore_timestamp}|${MO_HOST},${MO_PORT},${MO_USER}|physical|${RESTORE_BKID}|||${cost}|${outcome}" + echo "${restore_timestamp}|${MO_HOST},${MO_PORT},${MO_USER}|physical|${RESTORE_BKID}|||${cost}|${outcome}|${bk_size_in_bytes}" >> ${RESTORE_REPORT} if [[ ${rc} -ne 0 ]]; then @@ -208,9 +245,9 @@ function restore_restart_mo() function restore_logical() { - ! restore_precheck "connection" && return 1 - - #! restore_precheck "modump" && return 1 + if ! restore_precheck "connection"; then + return 1 + fi add_log "I" "Restore settings" add_log "I" "------------------------------------" @@ -256,16 +293,16 @@ function restore_logical() add_log "I" "Restore begins, please wait" i=1 rc=0 - for fileName in `ls ${srcPath}/ | grep ".sql"`; do + for fileName in `ls ${srcPath}/ | grep "\.sql"`; do if [[ "${isFile}" == "true" ]]; then file="${RESTORE_LOGICAL_SRC}" else file="${srcPath}/${fileName}" fi - add_log "D" "cmd: MYSQL_PWD="${MO_PW}" mysql -h${MO_HOST} -P${MO_PORT} -u${MO_USER} ${dbname_option} < ${file}" + add_log "D" "cmd: MYSQL_PWD="${MO_PW}" mysql --local-infile -h${MO_HOST} -P${MO_PORT} -u${MO_USER} ${dbname_option} < ${file}" restore_timestamp=`date '+%Y%m%d_%H%M%S'` startTime=`get_nanosecond` - if MYSQL_PWD="${MO_PW}" mysql -h${MO_HOST} -P${MO_PORT} -u${MO_USER} ${dbname_option} < ${file}; then + if MYSQL_PWD="${MO_PW}" mysql --local-infile -h${MO_HOST} -P${MO_PORT} -u${MO_USER} ${dbname_option} < ${file}; then outcome="succeeded" else outcome="failed" @@ -276,7 +313,16 @@ function restore_logical() add_log "I" "Number: $i, file: ${file}, outcome: ${outcome}, cost: ${cost} ms" - echo "${restore_timestamp}|${MO_HOST},${MO_PORT},${MO_USER}|logical||${file}|${RESTORE_LOGICAL_TYPE}|${cost}|${outcome}" >> ${RESTORE_REPORT} + + + bk_size="n.a." + if [[ ${outcome} == "succeeded" ]]; then + add_log "D" "Calculating size of source path ${srcPath}" + bk_size=`du -s ${srcPath} | awk '{print $1}'` + fi + + add_log "D" "Writing entry to report" + echo "${restore_timestamp}|${MO_HOST},${MO_PORT},${MO_USER}|logical||${file}|${RESTORE_LOGICAL_TYPE}|${cost}|${outcome}|${bk_size}" >> ${RESTORE_REPORT} if [[ "${isFile}" == "true" ]]; then break @@ -294,7 +340,7 @@ function restore_logical() if [[ ${rc} -ne 0 ]]; then - add_log "E" "Restore ends with non-zero rc" + add_log "E" "Restore ends with non-zeor rc" else add_log "I" "Restore ends with 0 rc" fi @@ -307,12 +353,22 @@ function restore() case "${RESTORE_TYPE}" in "physical") - ! restore_physical && return 1 - ! restore_mo_data && return 1 - ! restore_restart_mo && return 1 + if ! restore_physical; then + return 1 + fi + + if ! restore_mo_data; then + return 1 + fi + + if ! restore_restart_mo; then + return 1 + fi ;; "logical") - ! restore_logical && return 1 + if ! restore_logical; then + return 1 + fi ;; *) add_log "E" "Invalid RESTORE_TYPE = ${RESTORE_TYPE}, choose from: physical | logical" @@ -322,3 +378,16 @@ function restore() } +function restore_list() +{ + + option="$1" + + if [[ ! -f ${RESTORE_REPORT} ]]; then + add_log "E" "RESTORE_REPORT ${RESTORE_REPORT} is not a valid file, exiting" + return 1 + fi + #add_log "I" "Listing restore report (summary) from ${RESTORE_REPORT}" + #add_log "I" "------------------------------------" + cat ${RESTORE_REPORT} +} diff --git a/bin/sql.sh b/bin/sql.sh index 2a45b67..ad2e3a2 100644 --- a/bin/sql.sh +++ b/bin/sql.sh @@ -6,6 +6,7 @@ # sql query="" +TIMEOUT="1h" function exec_path() { @@ -21,7 +22,7 @@ function exec_path() startTime=`get_nanosecond` - if MYSQL_PWD="${MO_PW}" mysql --local-infile -h"${MO_HOST}" -P"${MO_PORT}" -u"${MO_USER}" -vvv < "${query}/${query_file}"; then + if MYSQL_PWD="${MO_PW}" timeout ${TIMEOUT} mysql --local-infile -h"${MO_HOST}" -P"${MO_PORT}" -u"${MO_USER}" -vvv < "${query}/${query_file}"; then endTime=`get_nanosecond` add_log "I" "End executing query file ${query_file}, succeeded" outcome="succeeded" @@ -62,7 +63,7 @@ function exec_file() add_log "I" "Input ${query} is a file" add_log "I" "Begin executing query file ${query}" startTime=`get_nanosecond` - if MYSQL_PWD="${MO_PW}" mysql --local-infile -h"${MO_HOST}" -P"${MO_PORT}" -u"${MO_USER}" -vvv < "${query}"; then + if MYSQL_PWD="${MO_PW}" timeout ${TIMEOUT} mysql --local-infile -h"${MO_HOST}" -P"${MO_PORT}" -u"${MO_USER}" -vvv < "${query}"; then endTime=`get_nanosecond` add_log "I" "End executing query file ${query}, succeeded" outcome="succeeded" @@ -85,7 +86,7 @@ function exec_query() add_log "I" "Input \"${query}\" is not a path or a file, try to execute it as a query" add_log "I" "Begin executing query \"${query}\"" startTime=`get_nanosecond` - if MYSQL_PWD="${MO_PW}" mysql --local-infile -h"${MO_HOST}" -P"${MO_PORT}" -u"${MO_USER}" -vvv -e "${query}"; then + if MYSQL_PWD="${MO_PW}" timeout ${TIMEOUT} mysql --local-infile -h"${MO_HOST}" -P"${MO_PORT}" -u"${MO_USER}" -vvv -e "${query}"; then endTime=`get_nanosecond` add_log "I" "End executing query ${query}, succeeded" outcome="succeeded" diff --git a/bin/upgrade-old.sh b/bin/upgrade-old.sh index 8faedb1..b6d7ff1 100644 --- a/bin/upgrade-old.sh +++ b/bin/upgrade-old.sh @@ -143,7 +143,9 @@ function validate_target_cid() if [[ "${target_branch}" == "main" ]]; then if [[ "${current_branch}" != "main" ]]; then add_log "I" "Current branch ${current_branch} is not on main, checking out to main: git checkout main" - ! git checkout main && return 1 + if ! git checkout main; then + return 1 + fi fi if [[ "${target_cid}" == "latest" ]]; then # 4.1. get latest commit id if target cid is set to latest diff --git a/conf/env.sh b/conf/env.sh index 9132e6c..715511d 100644 --- a/conf/env.sh +++ b/conf/env.sh @@ -41,7 +41,7 @@ MO_DEPLOY_MODE="git" # for docker # deprecated: MO_REPO="matrixorigin/matrixone" # full image name of mo container, default: matrixorigin/matrixone:1.0.1 -MO_CONTAINER_IMAGE="matrixorigin/matrixone:1.1.1" +MO_CONTAINER_IMAGE="matrixorigin/matrixone:1.2.0" # mo container name MO_CONTAINER_NAME="mo" # mo container sql port (PS: constant value, DONT CHANGE) @@ -76,7 +76,7 @@ MO_CONTAINER_EXTRA_MOUNT_OPTION="" CHECK_LIST=("go" "gcc" "git" "mysql" "docker") GCC_VERSION="8.5.0" CLANG_VERSION="13.0" -GO_VERSION="1.20" +GO_VERSION="1.22.3" DOCKER_SERVER_VERSION="20" # for deploy @@ -91,7 +91,7 @@ MO_GIT_URL="https://github.com/matrixorigin/matrixone.git" #) # default version of which mo to be deployed -MO_DEFAULT_VERSION="v1.1.1" +MO_DEFAULT_VERSION="v1.2.0" # which go proxy to be used when downloading go dependencies # you can set this go proxy when building mo-service GOPROXY="https://goproxy.cn,direct" @@ -193,6 +193,9 @@ BACKUP_REPORT="${TOOL_LOG_PATH}/backup-report.txt" BACKUP_MOBR_PATH="/data/tools/mo-backup/mo_br" # backup target type: filesystem(default)|s3 BACKUP_PHYSICAL_TYPE="filesystem" +# mobr meta file +BACKUP_MOBR_META_PATH="${TOOL_LOG_PATH}/mo_br.meta" + # 1) when BACKUP_PHYSICAL_TYPE="filesystem" # backup directory, same as BACKUP_PATH # BACKUP_PATH="/data/mo-backup" diff --git a/conf/env.sh.default b/conf/env.sh.default index 9132e6c..2498a82 100644 --- a/conf/env.sh.default +++ b/conf/env.sh.default @@ -41,7 +41,7 @@ MO_DEPLOY_MODE="git" # for docker # deprecated: MO_REPO="matrixorigin/matrixone" # full image name of mo container, default: matrixorigin/matrixone:1.0.1 -MO_CONTAINER_IMAGE="matrixorigin/matrixone:1.1.1" +MO_CONTAINER_IMAGE="matrixorigin/matrixone:1.2.0" # mo container name MO_CONTAINER_NAME="mo" # mo container sql port (PS: constant value, DONT CHANGE) @@ -76,7 +76,7 @@ MO_CONTAINER_EXTRA_MOUNT_OPTION="" CHECK_LIST=("go" "gcc" "git" "mysql" "docker") GCC_VERSION="8.5.0" CLANG_VERSION="13.0" -GO_VERSION="1.20" +GO_VERSION="1.21" DOCKER_SERVER_VERSION="20" # for deploy diff --git a/install.sh b/install.sh index cf0bdd3..bb23c89 100644 --- a/install.sh +++ b/install.sh @@ -1,14 +1,48 @@ #!/bin/bash #install.sh +DOWNLOAD_FILE_RENAME="mo_ctl.zip" +TOOL_LOG_LEVEL="I" + +function to_upper() +{ + echo $(echo $1 | tr '[a-z]' '[A-Z]') +} + +function to_lower() +{ + echo $(echo $1 | tr '[A-Z]' '[a-z]') +} + +function what_os() +{ + system=`uname` + os="" + case "${system}" in + "") + return 1 + ;; + "Darwin") + os="Mac" + ;; + "Linux") + os="Linux" + ;; + *) + os="OtherOS" + ;; + esac + echo "${os}" +} + function add_log() { level=$1 msg="$2" add_line="$3" - os=`uname` - if [[ "${os}" == "Darwin" ]]; then + os=`what_os` + if [[ "${os}" == "Mac" ]]; then # 1. for Mac # format: 2023-07-25 17:39:24.904 UTC+0800 timezone="UTC+0800" @@ -34,18 +68,47 @@ function add_log() nowtime="${timestamp_ms} ${timezone}" fi + level=`to_upper ${level}` + display_log_level=`to_upper ${TOOL_LOG_LEVEL}` + if [[ "${display_log_level}" == "" ]]; then + display_log_level="I" + fi case "${level}" in - "e"|"E") + "E") level="ERROR" ;; - "W"|"w") - level="WARN" + "W") + level="WARN" + case "${display_log_level}" in + "E") + return 0 + ;; + *) + : + ;; + esac ;; - "I"|"i") + "I") level="INFO" + case "${display_log_level}" in + "E"|"W") + return 0 + ;; + *) + : + ;; + esac ;; - "d"|"D") + "D") level="DEBUG" + case "${display_log_level}" in + "E"|"W"|"I") + return 0 + ;; + *) + : + ;; + esac ;; *) echo "These are valid log levels: E/e/W/w/I/i/D/d." @@ -54,13 +117,18 @@ function add_log() ;; esac - if [[ "${add_line}" == "n" ]]; then - echo -n "${nowtime} [${level}] ${msg}" - else + case "${add_line}" in + "n" ) + echo -n "${nowtime} [${level}] ${msg}" + ;; + "l" ) + echo "${msg}" + ;; + *) echo "${nowtime} [${level}] ${msg}" - fi + ;; + esac } - function usage() { echo " Usage : $0 [tool_path]" @@ -89,7 +157,7 @@ function download() for site in ${SITES[@]}; do URL="${site}/mo_ctl_standalone/archive/refs/heads/main.zip" add_log "I" "Try to download mo_ctl from URL: ${URL}" - if wget --timeout=60 --tries=2 ${URL} -O mo_ctl.zip; then + if wget --timeout=60 --tries=2 ${URL} -O ${DOWNLOAD_FILE_RENAME}; then add_log "I" "Successfully downloaded mo_ctl" rc="0" break; @@ -107,7 +175,7 @@ function download() function install() { pkg=$1 - os=`uname` + os=`what_os` os_user=`whoami` mo_ctl_global_path=/usr/local/bin mo_ctl_local_path=~/mo_ctl @@ -119,50 +187,64 @@ function install() return 1 fi - if [[ "${os}" == "Darwin" ]]; then - os="Mac" - # mo_ctl_local_path="/Users/${os_user}/mo_ctl" - elif [[ "${os}" == "Linux" ]]; then - os="Linux" - # mo_ctl_local_path="/data/mo_ctl" - # mkdir -p /data/ - elif [[ "${os}" == "" ]]; then - add_log "E" "Get current os failed" - return 1 - else + if [[ "${os}" != "Mac" ]] && [[ "${os}" != "Linux" ]]; then add_log "E" "Currently only Linux or Mac is supported" return 1 fi + add_log "I" "Try to install mo_ctl from file ${pkg}" if [[ ! -f ${pkg} ]]; then add_log "E" "Error! Installation file ${pkg} is not found, please check again." return 1 fi + #pkg_basename=`basename ${pkg} .zip` + #pkg_prefix=`echo ".zip" | awk -F '.' '{print $1}'` + if ! command -v unzip; then + add_log "E" "Command 'unzip' is not found, please check it is installed" + return 1 + fi + + tmp_path="./mo_ctl_tmp" + add_log "D" "cmd: rm -rf ${tmp_path} && mkdir ${tmp_path}" + if ! ( rm -rf ${tmp_path} && mkdir ${tmp_path} ) ; then + add_log "E" "Failed to remove and re-create tmp path ${tmp_path}" + return 1 + fi + + add_log "D" "cmd: unzip -o ${pkg} -d ${tmp_path}" + if ! unzip -o ${pkg} -d ${tmp_path}; then + add_log "E" "Failed to extract file ${pkg} to ${tmp_path}, please make sure file exists, file is complete or current user has enough privilege" + return 1 + fi + + pkg_name_after_unzip=`ls ${tmp_path}` + + add_log "D" "cmd: unzip -o ${pkg} -d ${tmp_path}" if [[ -d ${mo_ctl_local_path} ]]; then - add_log "W" "Path ${mo_ctl_local_path} already exists, removing it now" + add_log "W" "Path ${mo_ctl_local_path} already exists, removing it?(yes/no)" + user_confirm="" + read -t 30 user_confirm + if [[ "$(to_lower ${user_confirm})" != "yes" ]]; then + add_log "E" "User input not confirmed or timed out, exiting" + return 1 + fi + add_log "D" "cmd: rm -rf ~/mo_ctl/" rm -rf ~/mo_ctl/ - #if [[ "${os}" == "Linux" ]]; then - # rm -rf /data/mo_ctl/ - #else - # rm -rf /Users/${os_user}/mo_ctl/ - #fi fi - add_log "I" "Try to install mo_ctl from file ${pkg}" - pkg_prefix=`echo "mo_ctl_standalone_dev.zip" | awk -F '.' '{print $1}'` - if unzip -o ${pkg} && mv ./${pkg_prefix} ${mo_ctl_local_path} && chmod +x ${mo_ctl_local_path}/mo_ctl.sh; then + add_log "D" "cmd: mv ${tmp_path}/${pkg_name_after_unzip} ${mo_ctl_local_path} && chmod +x ${mo_ctl_local_path}/mo_ctl.sh" + if mv ${tmp_path}/${pkg_name_after_unzip} ${mo_ctl_local_path} && chmod +x ${mo_ctl_local_path}/mo_ctl.sh; then add_log "I" "Successfully extracted mo_ctl file to ${mo_ctl_local_path}" else add_log "E" "Failed to extract file, please check if 'unzip' is installed or file is complete" return 1 fi - add_log "I" "Setting up mo_ctl to ${mo_ctl_global_path}/mo_ctl: sudo touch ${mo_ctl_global_path}/mo_ctl && sudo chown ${os_user} ${mo_ctl_global_path}/mo_ctl && echo "bash +x ${mo_ctl_local_path}/mo_ctl.sh \"\$*\"" > ${mo_ctl_global_path}/mo_ctl && chmod +x ${mo_ctl_global_path}/mo_ctl" - add_log "I" "If you're running on MacOS, we need your confirmation with password to run sudo commands" - - + add_log "I" "Setting up mo_ctl to ${mo_ctl_global_path}/mo_ctl" + add_log "D" "cmd: sudo mkdir -p ${mo_ctl_global_path}/ && sudo touch ${mo_ctl_global_path}/mo_ctl && sudo chown ${os_user} ${mo_ctl_global_path}/mo_ctl && echo "bash +x ${mo_ctl_local_path}/mo_ctl.sh \"\$*\"" > ${mo_ctl_global_path}/mo_ctl && chmod +x ${mo_ctl_global_path}/mo_ctl" + add_log "I" "If you're running on MacOS, we need your confirmation with password to run sudo commands" if sudo mkdir -p ${mo_ctl_global_path}/ && sudo touch ${mo_ctl_global_path}/mo_ctl && sudo chown ${os_user} ${mo_ctl_global_path}/mo_ctl && echo "bash +x ${mo_ctl_local_path}/mo_ctl.sh \"\$*\"" > ${mo_ctl_global_path}/mo_ctl && chmod +x ${mo_ctl_global_path}/mo_ctl; then add_log "I" "Succeeded" @@ -173,6 +255,7 @@ function install() if [[ "${os}" == "Mac" ]]; then add_log "I" "Setting up default confs for mac: MO_PATH=/Users/${os_user}/mo" + add_log "D" "cmd: ${mo_ctl_local_path}/mo_ctl.sh set_conf MO_PATH=/Users/${os_user}/mo" if ${mo_ctl_local_path}/mo_ctl.sh set_conf MO_PATH=/Users/${os_user}/mo; then add_log "I" "Succeeded" else @@ -181,7 +264,8 @@ function install() fi fi - add_log "I" "Adding executable permission to scripts: chmod +x ${mo_ctl_local_path}/bin/*.sh" + add_log "D" "cmd: chmod +x ${mo_ctl_local_path}/bin/*.sh" + add_log "I" "Adding executable permission to scripts" if ! chmod +x ${mo_ctl_local_path}/bin/*.sh; then add_log "E" "Failed" return 1 @@ -197,7 +281,7 @@ function main() option=$1 case "${option}" in "") - download && install "mo_ctl.zip" + download && install "${DOWNLOAD_FILE_RENAME}" ;; "help") usage diff --git a/mo_ctl.sh b/mo_ctl.sh index aa099f6..d8ab3c9 100644 --- a/mo_ctl.sh +++ b/mo_ctl.sh @@ -87,9 +87,6 @@ function main() "restart") restart "${option_2}" ;; - "restore") - restore "${option_2}" - ;; "connect") connect ;; @@ -152,6 +149,13 @@ function main() backup fi ;; + "restore") + if [[ "${option_2}" == "list" ]]; then + restore_list "${option_3}" + else + restore + fi + ;; "clean_backup") clean_backup ;;