diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..1bb1d1e Binary files /dev/null and b/.DS_Store differ diff --git a/bin/auto_backup.sh b/bin/auto_backup.sh index 2b59582..94fb8f7 100644 --- a/bin/auto_backup.sh +++ b/bin/auto_backup.sh @@ -70,16 +70,16 @@ function backup_list() fi - add_log "I" "Listing backup report (detail, physical only)" - add_log "I" "------------------------------------" + #add_log "I" "Listing backup report (detail, physical only)" + #add_log "I" "------------------------------------" cd ${BACKUP_MOBR_DIRNAME} && ./mo_br list else if [[ ! -f ${BACKUP_REPORT} ]]; then add_log "E" "No backup action can be found, exiting" return 1 fi - add_log "I" "Listing backup report (summary) from ${BACKUP_REPORT}" - add_log "I" "------------------------------------" + #add_log "I" "Listing backup report (summary) from ${BACKUP_REPORT}" + #add_log "I" "------------------------------------" cat ${BACKUP_REPORT} fi } @@ -99,16 +99,20 @@ function backup() backup_yearmonth=`date '+%Y%m'` backup_timestamp=`date '+%Y%m%d_%H%M%S'` - - add_log "D" "Creating backup data direcory: mkdir -p ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/" - mkdir -p ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ + backup_outpath="${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}" + #if [[ "${BACKUP_TYPE}" == "logical" ]] && [[ "${BACKUP_LOGICAL_DS}" != "" ]] ; then + # backup_outpath="${backup_outpath}_${BACKUP_LOGICAL_DS}" + #fi + + add_log "D" "Creating backup data direcory: mkdir -p ${backup_outpath}/" + mkdir -p ${backup_outpath}/ backup_report_path=`dirname "${BACKUP_REPORT}"` add_log "D" "Creating backup report direcory: mkdir -p ${backup_report_path}" mkdir -p ${backup_report_path} - if [[ ! -f "${BACKUP_REPORT}" ]]; then + if [[ ! -f ${BACKUP_REPORT} ]]; then add_log "D" "Creating backup report file ${BACKUP_REPORT}" - echo "backup_date|backup_target|db_list|backup_type|backup_path|logical_data_type|duration_ms|outcome" > "${BACKUP_REPORT}" + echo "backup_date|backup_target|ds_name|db_list|backup_type|backup_path|logical_data_type|duration_ms|outcome|size_in_bytes" > "${BACKUP_REPORT}" fi backup_db_list="" @@ -173,9 +177,9 @@ function backup() for db in $(echo "${backup_db_list}" | sed "s/,/ /g"); do add_log "I" "Begin to back up database: ${db}" - add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}.sql && cd - >/dev/null 2>&1" + add_log "D" "Backup command: cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} ${no_data_option} > ${backup_outpath}/${db}.sql && cd - >/dev/null 2>&1" startTime=`get_nanosecond` - if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}.sql && cd - >/dev/null 2>&1; then + if cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} ${no_data_option} > ${backup_outpath}/${db}.sql && cd - >/dev/null 2>&1; then endTime=`get_nanosecond` outcome="succeeded" else @@ -192,12 +196,16 @@ function backup() # 1.2. backup databases all at once else + outfile_name="mo.sql" + if [[ "${BACKUP_LOGICAL_DS}" != "" ]]; then + outfile_name="${BACKUP_LOGICAL_DS}.sql" + fi add_log "D" "BACKUP_LOGICAL_ONEBYONE is not set to 1, will backup databases all at once" add_log "I" "Begin to back up databases in list: ${backup_db_list}" - add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/mo.sql && cd - >/dev/null 2>&1" + add_log "D" "Backup command: cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${csv_option} ${no_data_option} > ${backup_outpath}/${outfile_name} && cd - >/dev/null 2>&1" startTime=`get_nanosecond` - if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/mo.sql && cd - >/dev/null 2>&1; then + if cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${csv_option} ${no_data_option} > ${backup_outpath}/${outfile_name} && cd - >/dev/null 2>&1; then endTime=`get_nanosecond` outcome="succeeded" else @@ -227,10 +235,10 @@ function backup() tbl_option="-tbl ${BACKUP_LOGICAL_TBL_LIST}" fi - add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${tbl_option} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${backup_db_list}.sql && cd - >/dev/null 2>&1" + add_log "D" "Backup command: cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${tbl_option} ${csv_option} ${no_data_option} > ${backup_outpath}/${backup_db_list}.sql && cd - >/dev/null 2>&1" startTime=`get_nanosecond` - if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${tbl_option} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${backup_db_list}.sql && cd - >/dev/null 2>&1; then + if cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${tbl_option} ${csv_option} ${no_data_option} > ${backup_outpath}/${backup_db_list}.sql && cd - >/dev/null 2>&1; then endTime=`get_nanosecond` outcome="succeeded" else @@ -249,9 +257,9 @@ function backup() for tbl in $(echo "${BACKUP_LOGICAL_TBL_LIST}" | sed "s/,/ /g"); do add_log "I" "Begin to back up table: ${tbl}" - add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${tbl} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}_${tbl}.sql && cd - >/dev/null 2>&1" + add_log "D" "Backup command: cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${tbl} ${csv_option} ${no_data_option} > ${backup_outpath}/${db}_${tbl}.sql && cd - >/dev/null 2>&1" startTime=`get_nanosecond` - if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${tbl} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}_${tbl}.sql && cd - >/dev/null 2>&1; then + if cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${tbl} ${csv_option} ${no_data_option} > ${backup_outpath}/${db}_${tbl}.sql && cd - >/dev/null 2>&1; then endTime=`get_nanosecond` outcome="succeeded" else @@ -285,10 +293,10 @@ function backup() BACKUP_MOBR_DIRNAME=`dirname "${BACKUP_MOBR_PATH}"` case "${BACKUP_PHYSICAL_TYPE}" in "filesystem") - add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"filesystem\" --path \"${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/\"" + add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"filesystem\" --path \"${backup_outpath}/\"" startTime=`get_nanosecond` - if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "filesystem" --path "${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/" ; then + if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "filesystem" --path "${backup_outpath}/" ; then outcome="succeeded" else outcome="failed" @@ -329,10 +337,13 @@ function backup() add_log "I" "End with outcome: ${outcome}, cost: ${cost} ms" esac - + bk_size="n.a." + if [[ ${outcome} == "succeeded" ]]; then + bk_size=`du -s ${backup_outpath}` + fi # output report record - bakcup_target="${MO_HOST},${MO_PORT},${MO_USER},${MO_PW}" - echo "${backup_timestamp}|${bakcup_target}|${backup_conf_db_list}|${BACKUP_TYPE}|${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/|${logical_data_type}|${cost}|${outcome}" >> "${BACKUP_REPORT}" + bakcup_target="${MO_HOST},${MO_PORT},${MO_USER}" + echo "${backup_timestamp}|${bakcup_target}|${BACKUP_LOGICAL_DS}|${backup_conf_db_list}|${BACKUP_TYPE}|${backup_outpath}/${backup_yearmonth}/${backup_timestamp}/|${logical_data_type}|${cost}|${outcome}|${bk_size}" >> "${BACKUP_REPORT}" if [[ ${rc} -ne 0 ]]; then add_log "E" "Backup ends with non-zero rc" @@ -545,4 +556,4 @@ function auto_backup() ;; esac -} +} \ No newline at end of file diff --git a/bin/help.sh b/bin/help.sh index 1c94490..47fdaa8 100644 --- a/bin/help.sh +++ b/bin/help.sh @@ -359,7 +359,7 @@ function help_restore_notes() echo " 1. Common settings " echo " ------------------------- " echo " 1) RESTORE_TYPE [default: physical]: backup type to restore, choose from \"physical\" | \"logical\". e.g. mo_ctl set_conf RESTORE_TYPE=\"logical\"" - echo " 2) RESTORE_PATH: path to restore, which must be an empty folder, e.g. mo_ctl set_conf RESTORE_PATH=\"/data/mo/restore\"" + @@ -369,8 +369,9 @@ function help_restore_notes() echo " 2. For physical backups " echo " ------------------------- " echo " 1) BACKUP_MOBR_PATH [default: /data/tools/mo-backup/mo_br]: Path to mo_br backup tool" - echo " 2) RESTORE_BKID: backup id to restore, which can be found using cmd \"mo_ctl backup list detail\", e.g. mo_ctl set_conf RESTORE_BKID=\"6363b248-fc9f-11ee-845e-b07b25235fd0\"" - echo " 3) RESTORE_PHYSICAL_TYPE [default: filesystem]: target restore storage type, choose from \"filesystem\" | \"s3\"" + echo " 2) RESTORE_PATH: path to restore, which must be an empty folder, e.g. mo_ctl set_conf RESTORE_PATH=\"/data/mo/restore\"" + echo " 3) RESTORE_BKID: backup id to restore, which can be found using cmd \"mo_ctl backup list detail\", e.g. mo_ctl set_conf RESTORE_BKID=\"6363b248-fc9f-11ee-845e-b07b25235fd0\"" + echo " 4) RESTORE_PHYSICAL_TYPE [default: filesystem]: target restore storage type, choose from \"filesystem\" | \"s3\"" echo " if RESTORE_PHYSICAL_TYPE=s3" echo " a) RESTORE_S3_ENDPOINT [default: '']: s3 endpoint, e.g. https://cos.ap-nanjing.myqcloud.com" echo " b) RESTORE_S3_ID [default: '']: s3 id, e.g. B4v6Khv484X81dk81jQFzc9YxKl98JOyxkX1k" @@ -386,9 +387,9 @@ function help_restore_notes() echo " 3. For logical restore " echo " ------------------------- " echo " 1) BACKUP_MODUMP_PATH [default: /data/tools/mo_dump/mo-dump]: Path to mo-dump backup tool" - echo " 2) RESTORE_LOGICAL_DB: optional, if set, will add database name to mysql command when restoring logical backup data. i.e. MYSQL_PWD=xx mysql -hxxx -Pxxx db_name < backup_data.sql" - - + echo " 2) RESTORE_LOGICAL_SRC: Path of a directory or file to logical backup data source, e.g. /data/backup/db1.sql" + echo " 3) RESTORE_LOGICAL_DB: optional, if set, will add database name to mysql command when restoring logical backup data. i.e. MYSQL_PWD=xx mysql -hxxx -Pxxx db_name < backup_data.sql" + echo " 4) RESTORE_LOGICAL_TYPE: optional, choose from: ddl | insert | csv" } diff --git a/bin/restore.sh b/bin/restore.sh index 75d2ef3..23e7515 100644 --- a/bin/restore.sh +++ b/bin/restore.sh @@ -66,10 +66,21 @@ function restore_precheck() } +function restore_prep_report() +{ + restore_report_path=`dirname "${RESTORE_REPORT}"` + add_log "D" "Creating restore report direcory: mkdir -p ${restore_report_path}" + mkdir -p ${restore_report_path} + if [[ ! -f "${RESTORE_REPORT}" ]]; then + add_log "D" "Creating restore report file ${RESTORE_REPORT}" + echo "restore_date|restore_target|restore_type|physical_bk_id|logical_restore_src|logical_type|duration_ms|outcome" > "${RESTORE_REPORT}" + fi +} + function restore_physical() { add_log "I" "MO_HOST: ${MO_HOST}" - if [[ "MO_HOST" != "127.0.0.1" ]]; then + if [[ "${MO_HOST}" != "127.0.0.1" ]]; then add_log "E" "Currently mo_ctl only support restoring physical backup data on a local mo server, thus please set MO_HOST to 127.0.0.1 if that's the case." return 1 fi @@ -87,8 +98,6 @@ function restore_physical() add_log "I" "Restore begins" - backup_yearmonth=`date '+%Y%m'` - backup_timestamp=`date '+%Y%m%d_%H%M%S'` BACKUP_MOBR_DIRNAME=`dirname "${BACKUP_MOBR_PATH}"` @@ -128,6 +137,7 @@ function restore_physical() add_log "D" "cmd: cd ${BACKUP_MOBR_DIRNAME} && ${cmd}" + restore_timestamp=`date '+%Y%m%d_%H%M%S'` startTime=`get_nanosecond` if cd ${BACKUP_MOBR_DIRNAME} && ${cmd}; then outcome="succeeded" @@ -140,11 +150,17 @@ function restore_physical() add_log "I" "Outcome: ${outcome}, cost: ${cost} ms" + restore_prep_report + echo "${restore_timestamp}|${MO_HOST},${MO_PORT},${MO_USER}|physical|${RESTORE_BKID}|||${cost}|${outcome}" >> ${RESTORE_REPORT} + + if [[ ${rc} -ne 0 ]]; then add_log "E" "Restore ends with non-zero rc" else add_log "I" "Restore ends with 0 rc" fi + + return ${rc} } function restore_mo_data() @@ -219,41 +235,75 @@ function restore_logical() fi - backup_yearmonth=`date '+%Y%m'` - backup_timestamp=`date '+%Y%m%d_%H%M%S'` - BACKUP_MODUMP_DIRNAME=`dirname "${BACKUP_MOBR_PATH}"` - - add_log "I" "Restore begins, please wait" - #todo - #if MYSQL_PWD="${M_PW}" mysql -h"${MO_HOST}" -P"${MO_PORT}" -u"${MO_USER}" "${dbname_option}" < ${} - #cmd="mysql -u$}" + if [[ -d "${RESTORE_LOGICAL_SRC}" ]]; then + isFile="false" + srcPath="${RESTORE_LOGICAL_SRC}" + add_log "I" "RESTORE_LOGICAL_SRC=${RESTORE_LOGICAL_SRC} is a path, listing files in it" + ls -lth "${RESTORE_LOGICAL_SRC}" + elif [[ -f "${RESTORE_LOGICAL_SRC}" ]]; then + isFile="true" + srcPath=`dirname "${RESTORE_LOGICAL_SRC}"` + add_log "I" "RESTORE_LOGICAL_SRC=${RESTORE_LOGICAL_SRC} is a file" + else + add_log "E" "RESTORE_LOGICAL_SRC=${RESTORE_LOGICAL_SRC} is a not a path nor file. Please check again. Exiting" + return 1 + fi - add_log "D" "cmd: cd ${BACKUP_MOBR_DIRNAME} && ${cmd}" + restore_prep_report + add_log "I" "Restore begins, please wait" + i=1 + rc=0 + for fileName in `ls ${srcPath}/ | grep ".sql"`; do + if [[ "${isFile}" == "true" ]]; then + file="${RESTORE_LOGICAL_SRC}" + else + file="${srcPath}/${fileName}" + fi + add_log "D" "cmd: MYSQL_PWD="${MO_PW}" mysql -h${MO_HOST} -P${MO_PORT} -u${MO_USER} ${dbname_option} < ${file}" + restore_timestamp=`date '+%Y%m%d_%H%M%S'` + startTime=`get_nanosecond` + if MYSQL_PWD="${MO_PW}" mysql -h${MO_HOST} -P${MO_PORT} -u${MO_USER} ${dbname_option} < ${file}; then + outcome="succeeded" + else + outcome="failed" + let rc=rc+1 + fi + endTime=`get_nanosecond` + cost=`time_cost_ms ${startTime} ${endTime}` + + + add_log "I" "Number: $i, file: ${file}, outcome: ${outcome}, cost: ${cost} ms" + echo "${restore_timestamp}|${MO_HOST},${MO_PORT},${MO_USER}|logical||${file}|${RESTORE_LOGICAL_TYPE}|${cost}|${outcome}" >> ${RESTORE_REPORT} + + if [[ "${isFile}" == "true" ]]; then + break + fi + let i=i+1 + + done + + let i=i-1 + + add_log "I" "-------------------------" + add_log "I" " Summary " + add_log "I" "Total: ${i}, failed: ${rc}" + add_log "I" "-------------------------" - startTime=`get_nanosecond` - if cd ${BACKUP_MOBR_DIRNAME} && ${cmd}; then - outcome="succeeded" - else - outcome="failed" - rc=1 - fi - endTime=`get_nanosecond` - cost=`time_cost_ms ${startTime} ${endTime}` - - add_log "I" "Outcome: ${outcome}, cost: ${cost} ms" if [[ ${rc} -ne 0 ]]; then add_log "E" "Restore ends with non-zero rc" else add_log "I" "Restore ends with 0 rc" fi + return ${rc} } function restore() { + add_log "I" "RESTORE_TYPE: ${RESTORE_TYPE}" case "${RESTORE_TYPE}" in "physical") diff --git a/bin/uninstall.sh b/bin/uninstall.sh index afd25b0..869e965 100644 --- a/bin/uninstall.sh +++ b/bin/uninstall.sh @@ -43,14 +43,16 @@ function uninstall() { if [[ "${MO_DEPLOY_MODE}" == "docker" ]]; then - add_log "W" "You're uninstalling MO image ${MO_CONTAINER_IMAGE}, are you sure? (Yes/No)" - read -t 30 user_confirm - if [[ "$(to_lower ${user_confirm})" != "yes" ]]; then - add_log "E" "User input not confirmed or timed out, exiting" + + + if ! check_uninstall_pre_requisites; then return 1 fi - if ! check_uninstall_pre_requisites; then + add_log "W" "You're uninstalling MO container ${MO_CONTAINER_NAME} and image ${MO_CONTAINER_IMAGE}, are you sure? (Yes/No)" + read -t 30 user_confirm + if [[ "$(to_lower ${user_confirm})" != "yes" ]]; then + add_log "E" "User input not confirmed or timed out, exiting" return 1 fi @@ -61,6 +63,8 @@ function uninstall() return 1 fi + + add_log "I" "Removing image ${MO_CONTAINER_IMAGE}" if ! docker rmi ${MO_CONTAINER_IMAGE}; then add_log "E" "Failed" diff --git a/conf/env.sh b/conf/env.sh index 6d73acf..9132e6c 100644 --- a/conf/env.sh +++ b/conf/env.sh @@ -185,7 +185,7 @@ BACKUP_CLEAN_DAYS_BEFORE="31" # cron to control auto clean of old backups BACKUP_CLEAN_CRON_SCHEDULE="0 6 * * *" # backup history -BACKUP_REPORT="${TOOL_LOG_PATH}/backup/report.txt" +BACKUP_REPORT="${TOOL_LOG_PATH}/backup-report.txt" # 1. physical backups # backup tools @@ -230,11 +230,16 @@ BACKUP_LOGICAL_ONEBYONE="0" # backup net buffer length(integer): 1048576(default, 1M), Max is 16777216 (16M) BACKUP_LOGICAL_NETBUFLEN="1048576" +# backup logical dataset name: (optional) the dataset name of the backup database +BACKUP_LOGICAL_DS="" + + # for restore # restore type: physical | logical RESTORE_TYPE="physical" # RESTORE_PATH="/data/mo/restore" +RESTORE_REPORT="${TOOL_LOG_PATH}/restore-report.txt" # 1. physical restore # restore target type: filesystem(default)|s3 @@ -257,10 +262,14 @@ RESTORE_S3_IS_MINIO="no" # 2. logical restore -# restore target type: filesystem(default)|s3 -RESTORE_LOGICAL_DB="" -# restore target type: filesystem(default)|s3 +# restore target db, e.g. db1. only one db is supported RESTORE_LOGICAL_DB="" +# restore data path (directory or file) +RESTORE_LOGICAL_SRC="" +# restore data type: csv | insert | ddl +RESTORE_LOGICAL_TYPE="" + + # for auto clean sysdb logs diff --git a/conf/env.sh.default b/conf/env.sh.default index d5abe81..9132e6c 100644 --- a/conf/env.sh.default +++ b/conf/env.sh.default @@ -24,7 +24,7 @@ MO_CONF_SRC_PATH="" # For connect # target mo instance type: local | remote -MO_SERVER_TYPE="local" +# deprecated: MO_SERVER_TYPE="local" # host ip to connect where mo is deployed, by default: 127.0.0.1 MO_HOST="127.0.0.1" # host port to connect where mo is deployed, by default: 6001 @@ -39,9 +39,9 @@ MO_DEPLOY_MODE="git" # for docker -MO_REPO="matrixorigin/matrixone" +# deprecated: MO_REPO="matrixorigin/matrixone" # full image name of mo container, default: matrixorigin/matrixone:1.0.1 -MO_CONTAINER_IMAGE="matrixorigin/matrixone:1.1.0" +MO_CONTAINER_IMAGE="matrixorigin/matrixone:1.1.1" # mo container name MO_CONTAINER_NAME="mo" # mo container sql port (PS: constant value, DONT CHANGE) @@ -91,7 +91,7 @@ MO_GIT_URL="https://github.com/matrixorigin/matrixone.git" #) # default version of which mo to be deployed -MO_DEFAULT_VERSION="v1.1.0" +MO_DEFAULT_VERSION="v1.1.1" # which go proxy to be used when downloading go dependencies # you can set this go proxy when building mo-service GOPROXY="https://goproxy.cn,direct" @@ -185,7 +185,7 @@ BACKUP_CLEAN_DAYS_BEFORE="31" # cron to control auto clean of old backups BACKUP_CLEAN_CRON_SCHEDULE="0 6 * * *" # backup history -BACKUP_REPORT="${TOOL_LOG_PATH}/backup/report.txt" +BACKUP_REPORT="${TOOL_LOG_PATH}/backup-report.txt" # 1. physical backups # backup tools @@ -230,6 +230,47 @@ BACKUP_LOGICAL_ONEBYONE="0" # backup net buffer length(integer): 1048576(default, 1M), Max is 16777216 (16M) BACKUP_LOGICAL_NETBUFLEN="1048576" +# backup logical dataset name: (optional) the dataset name of the backup database +BACKUP_LOGICAL_DS="" + + +# for restore + +# restore type: physical | logical +RESTORE_TYPE="physical" # +RESTORE_PATH="/data/mo/restore" +RESTORE_REPORT="${TOOL_LOG_PATH}/restore-report.txt" + +# 1. physical restore +# restore target type: filesystem(default)|s3 +RESTORE_PHYSICAL_TYPE="filesystem" +# the backup id to be restored +RESTORE_BKID="" +# 1) when BACKUP_PHYSICAL_TYPE="filesystem" +# restore directory +# same as RESTORE_PATH + +# 2) when BACKUP_PHYSICAL_TARGET_TYPE="s3" +RESTORE_S3_ENDPOINT="" +RESTORE_S3_ID="" +RESTORE_S3_KEY="" +RESTORE_S3_BUCKET="" +RESTORE_S3_REGION="" +RESTORE_S3_COMPRESSION="" +RESTORE_S3_ROLE_ARN="" +RESTORE_S3_IS_MINIO="no" + + +# 2. logical restore +# restore target db, e.g. db1. only one db is supported +RESTORE_LOGICAL_DB="" +# restore data path (directory or file) +RESTORE_LOGICAL_SRC="" +# restore data type: csv | insert | ddl +RESTORE_LOGICAL_TYPE="" + + + # for auto clean sysdb logs # clean old sysdb logs before [x] (default: 31) days diff --git a/conf/mo_conf/cn.toml b/conf/mo_conf/cn.toml new file mode 100644 index 0000000..55e3504 --- /dev/null +++ b/conf/mo_conf/cn.toml @@ -0,0 +1,40 @@ +service-type = "CN" +data-dir = "./mo-data" + +[log] +level = "info" +# 关掉 log_info 的采集(rawlog表由三部分组成:log_info、span_info、error_info,一般 log_info 部分是最大的) +disable-store = true + +[cn] +uuid = "dd1dccb4-4d3c-41f8-b482-5251dc7a41bf" +port-base = 18000 + +[cn.txn] +# 配置事务在commit时对于主键重复值的检查,默认为0表示不检查。如果需要commit时检查主键重复值,可以配置为非0(建议写10或50,不要写1),注意会有一点性能损耗。 +pk-dedup-count=10 + + +[observability] +# 对日志写入系统表的总开关,false为默认表示写入,由于statement_info需要,所以一般不设置为true +disableTrace = false +# 关闭对表 span_info 的采集。另外,目前对表 error_info 的采集没有单独的开关,所以error_info 和 statement_info都会写入 +disableSpan = true +# 关闭表 statement_info 的聚合行为,默认是false即聚合 +# disableStmtAggregation = true + + +[[fileservice]] +# backend 的枚举值有:S3和DISK +backend = "S3" +# name 的枚举值有:LOCAL、S3、ETL +name = "S3" + +[[fileservice]] +# backend 的枚举值有:S3和DISK +backend = "DISK" +# name 的枚举值有:LOCAL、S3、ETL,对应分别是临时文件存储(一般配DISK的backend)、mo的数据存储(一般单机的backend是DISK,分布式的backend是S3)、可观测性系统相关的数据存储(一般与mo的数据存储保持一致) +name = "S3" + +[fileservice.cache] +memory-capacity = "2GiB" \ No newline at end of file diff --git a/conf/mo_conf/log.toml b/conf/mo_conf/log.toml new file mode 100644 index 0000000..3f389cb --- /dev/null +++ b/conf/mo_conf/log.toml @@ -0,0 +1,23 @@ +# service node type, [DN|CN|LOG] +service-type = "LOG" +data-dir = "./mo-data" + +[log] +level = "info" +# 关掉 log_info 的采集(rawlog表由三部分组成:log_info、span_info、error_info,一般 log_info 部分是最大的) +disable-store = true + +[observability] +# 对日志写入系统表的总开关,false为默认表示写入,由于statement_info需要,所以一般不设置为true +disableTrace = false +# 关闭对表 span_info 的采集。另外,目前对表 error_info 的采集没有单独的开关,所以error_info 和 statement_info都会写入 +disableSpan = true +# 关闭表 statement_info 的聚合行为,默认是false即聚合 +# disableStmtAggregation = true + + +#[logservice] +# 指定额外的主机名,如果此处指定了额外的配置,则mo-data/logservice-data/7c4dccb4-4d3c-41f8-b482-5251dc7a41bf/${hostname} 这个目录名是根据配置自定义生成的;否则按默认值处理,默认是采取宿主机的主机名(hostname命令的返回)。 +# 一般迁移的场景下用,如果不指定,mo-data迁移到不同主机名的机器后,需要手动更改这个目录的名称 +#explicit-hostname = "host123" + diff --git a/conf/mo_conf/tn.toml b/conf/mo_conf/tn.toml new file mode 100644 index 0000000..31b0fee --- /dev/null +++ b/conf/mo_conf/tn.toml @@ -0,0 +1,37 @@ +service-type = "TN" +data-dir = "./mo-data" + +[log] +level = "info" +# 关掉 log_info 的采集(rawlog表由三部分组成:log_info、span_info、error_info,一般 log_info 部分是最大的) +disable-store = true + +[observability] +# 对日志写入系统表的总开关,false为默认表示写入,由于statement_info需要,所以一般不设置为true +disableTrace = false +# 关闭对表 span_info 的采集。另外,目前对表 error_info 的采集没有单独的开关,所以error_info 和 statement_info都会写入 +disableSpan = true +# 关闭表 statement_info 的聚合行为,默认是false即聚合 +# disableStmtAggregation = true + +[tn] +uuid = "dd4dccb4-4d3c-41f8-b482-5251dc7a41bf" +port-base = 19000 + +#[tn.GCCfg] +配置mo的自动gc扫描和清理的时间间隔,作用是将已经逻辑上删除掉的数据,在物理上从磁盘/对象存储中删除其对应的物理文件 +# 'true' means disabling gc and retaining all data. The default is false. +#disable-gc = false +#清理间隔 +#gc-ttl = "10m" +#扫描间隔 +#scan-gc-interval = "5m" + +[[fileservice]] +# backend 的枚举值有:S3和DISK +backend = "DISK" +# name 的枚举值有:LOCAL、S3、ETL,对应分别是临时文件存储(一般配DISK的backend)、mo的数据存储(一般单机的backend是DISK,分布式的backend是S3)、可观测性系统相关的数据存储(一般与mo的数据存储保持一致) +name = "S3" + +[fileservice.cache] +memory-capacity = "2GiB"