Skip to content

Commit

Permalink
enhance: some enhancement
Browse files Browse the repository at this point in the history
  • Loading branch information
aronchanisme committed May 15, 2024
1 parent a76c628 commit 0497a2a
Show file tree
Hide file tree
Showing 10 changed files with 282 additions and 66 deletions.
Binary file added .DS_Store
Binary file not shown.
57 changes: 34 additions & 23 deletions bin/auto_backup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -70,16 +70,16 @@ function backup_list()
fi


add_log "I" "Listing backup report (detail, physical only)"
add_log "I" "------------------------------------"
#add_log "I" "Listing backup report (detail, physical only)"
#add_log "I" "------------------------------------"
cd ${BACKUP_MOBR_DIRNAME} && ./mo_br list
else
if [[ ! -f ${BACKUP_REPORT} ]]; then
add_log "E" "No backup action can be found, exiting"
return 1
fi
add_log "I" "Listing backup report (summary) from ${BACKUP_REPORT}"
add_log "I" "------------------------------------"
#add_log "I" "Listing backup report (summary) from ${BACKUP_REPORT}"
#add_log "I" "------------------------------------"
cat ${BACKUP_REPORT}
fi
}
Expand All @@ -99,16 +99,20 @@ function backup()

backup_yearmonth=`date '+%Y%m'`
backup_timestamp=`date '+%Y%m%d_%H%M%S'`

add_log "D" "Creating backup data direcory: mkdir -p ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/"
mkdir -p ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/
backup_outpath="${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}"
#if [[ "${BACKUP_TYPE}" == "logical" ]] && [[ "${BACKUP_LOGICAL_DS}" != "" ]] ; then
# backup_outpath="${backup_outpath}_${BACKUP_LOGICAL_DS}"
#fi

add_log "D" "Creating backup data direcory: mkdir -p ${backup_outpath}/"
mkdir -p ${backup_outpath}/

backup_report_path=`dirname "${BACKUP_REPORT}"`
add_log "D" "Creating backup report direcory: mkdir -p ${backup_report_path}"
mkdir -p ${backup_report_path}
if [[ ! -f "${BACKUP_REPORT}" ]]; then
if [[ ! -f ${BACKUP_REPORT} ]]; then
add_log "D" "Creating backup report file ${BACKUP_REPORT}"
echo "backup_date|backup_target|db_list|backup_type|backup_path|logical_data_type|duration_ms|outcome" > "${BACKUP_REPORT}"
echo "backup_date|backup_target|ds_name|db_list|backup_type|backup_path|logical_data_type|duration_ms|outcome|size_in_bytes" > "${BACKUP_REPORT}"
fi

backup_db_list=""
Expand Down Expand Up @@ -173,9 +177,9 @@ function backup()
for db in $(echo "${backup_db_list}" | sed "s/,/ /g"); do
add_log "I" "Begin to back up database: ${db}"

add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}.sql && cd - >/dev/null 2>&1"
add_log "D" "Backup command: cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} ${no_data_option} > ${backup_outpath}/${db}.sql && cd - >/dev/null 2>&1"
startTime=`get_nanosecond`
if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}.sql && cd - >/dev/null 2>&1; then
if cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${db} ${csv_option} ${no_data_option} > ${backup_outpath}/${db}.sql && cd - >/dev/null 2>&1; then
endTime=`get_nanosecond`
outcome="succeeded"
else
Expand All @@ -192,12 +196,16 @@ function backup()

# 1.2. backup databases all at once
else
outfile_name="mo.sql"
if [[ "${BACKUP_LOGICAL_DS}" != "" ]]; then
outfile_name="${BACKUP_LOGICAL_DS}.sql"
fi
add_log "D" "BACKUP_LOGICAL_ONEBYONE is not set to 1, will backup databases all at once"
add_log "I" "Begin to back up databases in list: ${backup_db_list}"
add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/mo.sql && cd - >/dev/null 2>&1"
add_log "D" "Backup command: cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${csv_option} ${no_data_option} > ${backup_outpath}/${outfile_name} && cd - >/dev/null 2>&1"

startTime=`get_nanosecond`
if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/mo.sql && cd - >/dev/null 2>&1; then
if cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${csv_option} ${no_data_option} > ${backup_outpath}/${outfile_name} && cd - >/dev/null 2>&1; then
endTime=`get_nanosecond`
outcome="succeeded"
else
Expand Down Expand Up @@ -227,10 +235,10 @@ function backup()
tbl_option="-tbl ${BACKUP_LOGICAL_TBL_LIST}"
fi

add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${tbl_option} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${backup_db_list}.sql && cd - >/dev/null 2>&1"
add_log "D" "Backup command: cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${tbl_option} ${csv_option} ${no_data_option} > ${backup_outpath}/${backup_db_list}.sql && cd - >/dev/null 2>&1"

startTime=`get_nanosecond`
if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${tbl_option} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${backup_db_list}.sql && cd - >/dev/null 2>&1; then
if cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} ${tbl_option} ${csv_option} ${no_data_option} > ${backup_outpath}/${backup_db_list}.sql && cd - >/dev/null 2>&1; then
endTime=`get_nanosecond`
outcome="succeeded"
else
Expand All @@ -249,9 +257,9 @@ function backup()
for tbl in $(echo "${BACKUP_LOGICAL_TBL_LIST}" | sed "s/,/ /g"); do
add_log "I" "Begin to back up table: ${tbl}"

add_log "D" "Backup command: cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${tbl} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}_${tbl}.sql && cd - >/dev/null 2>&1"
add_log "D" "Backup command: cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${tbl} ${csv_option} ${no_data_option} > ${backup_outpath}/${db}_${tbl}.sql && cd - >/dev/null 2>&1"
startTime=`get_nanosecond`
if cd ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${tbl} ${csv_option} ${no_data_option} > ${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/${db}_${tbl}.sql && cd - >/dev/null 2>&1; then
if cd ${backup_outpath}/ && ${BACKUP_MODUMP_PATH} -net-buffer-length ${BACKUP_LOGICAL_NETBUFLEN} -u ${MO_USER} -P ${MO_PORT} -h ${MO_HOST} -p ${MO_PW} -db ${backup_db_list} -tbl ${tbl} ${csv_option} ${no_data_option} > ${backup_outpath}/${db}_${tbl}.sql && cd - >/dev/null 2>&1; then
endTime=`get_nanosecond`
outcome="succeeded"
else
Expand Down Expand Up @@ -285,10 +293,10 @@ function backup()
BACKUP_MOBR_DIRNAME=`dirname "${BACKUP_MOBR_PATH}"`
case "${BACKUP_PHYSICAL_TYPE}" in
"filesystem")
add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"filesystem\" --path \"${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/\""
add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"filesystem\" --path \"${backup_outpath}/\""

startTime=`get_nanosecond`
if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "filesystem" --path "${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/" ; then
if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "filesystem" --path "${backup_outpath}/" ; then
outcome="succeeded"
else
outcome="failed"
Expand Down Expand Up @@ -329,10 +337,13 @@ function backup()
add_log "I" "End with outcome: ${outcome}, cost: ${cost} ms"

esac

bk_size="n.a."
if [[ ${outcome} == "succeeded" ]]; then
bk_size=`du -s ${backup_outpath}`
fi
# output report record
bakcup_target="${MO_HOST},${MO_PORT},${MO_USER},${MO_PW}"
echo "${backup_timestamp}|${bakcup_target}|${backup_conf_db_list}|${BACKUP_TYPE}|${BACKUP_DATA_PATH}/${backup_yearmonth}/${backup_timestamp}/|${logical_data_type}|${cost}|${outcome}" >> "${BACKUP_REPORT}"
bakcup_target="${MO_HOST},${MO_PORT},${MO_USER}"
echo "${backup_timestamp}|${bakcup_target}|${BACKUP_LOGICAL_DS}|${backup_conf_db_list}|${BACKUP_TYPE}|${backup_outpath}/${backup_yearmonth}/${backup_timestamp}/|${logical_data_type}|${cost}|${outcome}|${bk_size}" >> "${BACKUP_REPORT}"

if [[ ${rc} -ne 0 ]]; then
add_log "E" "Backup ends with non-zero rc"
Expand Down Expand Up @@ -545,4 +556,4 @@ function auto_backup()
;;
esac

}
}
13 changes: 7 additions & 6 deletions bin/help.sh
Original file line number Diff line number Diff line change
Expand Up @@ -359,7 +359,7 @@ function help_restore_notes()
echo " 1. Common settings "
echo " ------------------------- "
echo " 1) RESTORE_TYPE [default: physical]: backup type to restore, choose from \"physical\" | \"logical\". e.g. mo_ctl set_conf RESTORE_TYPE=\"logical\""
echo " 2) RESTORE_PATH: path to restore, which must be an empty folder, e.g. mo_ctl set_conf RESTORE_PATH=\"/data/mo/restore\""




Expand All @@ -369,8 +369,9 @@ function help_restore_notes()
echo " 2. For physical backups "
echo " ------------------------- "
echo " 1) BACKUP_MOBR_PATH [default: /data/tools/mo-backup/mo_br]: Path to mo_br backup tool"
echo " 2) RESTORE_BKID: backup id to restore, which can be found using cmd \"mo_ctl backup list detail\", e.g. mo_ctl set_conf RESTORE_BKID=\"6363b248-fc9f-11ee-845e-b07b25235fd0\""
echo " 3) RESTORE_PHYSICAL_TYPE [default: filesystem]: target restore storage type, choose from \"filesystem\" | \"s3\""
echo " 2) RESTORE_PATH: path to restore, which must be an empty folder, e.g. mo_ctl set_conf RESTORE_PATH=\"/data/mo/restore\""
echo " 3) RESTORE_BKID: backup id to restore, which can be found using cmd \"mo_ctl backup list detail\", e.g. mo_ctl set_conf RESTORE_BKID=\"6363b248-fc9f-11ee-845e-b07b25235fd0\""
echo " 4) RESTORE_PHYSICAL_TYPE [default: filesystem]: target restore storage type, choose from \"filesystem\" | \"s3\""
echo " if RESTORE_PHYSICAL_TYPE=s3"
echo " a) RESTORE_S3_ENDPOINT [default: '']: s3 endpoint, e.g. https://cos.ap-nanjing.myqcloud.com"
echo " b) RESTORE_S3_ID [default: '']: s3 id, e.g. B4v6Khv484X81dk81jQFzc9YxKl98JOyxkX1k"
Expand All @@ -386,9 +387,9 @@ function help_restore_notes()
echo " 3. For logical restore "
echo " ------------------------- "
echo " 1) BACKUP_MODUMP_PATH [default: /data/tools/mo_dump/mo-dump]: Path to mo-dump backup tool"
echo " 2) RESTORE_LOGICAL_DB: optional, if set, will add database name to mysql command when restoring logical backup data. i.e. MYSQL_PWD=xx mysql -hxxx -Pxxx db_name < backup_data.sql"


echo " 2) RESTORE_LOGICAL_SRC: Path of a directory or file to logical backup data source, e.g. /data/backup/db1.sql"
echo " 3) RESTORE_LOGICAL_DB: optional, if set, will add database name to mysql command when restoring logical backup data. i.e. MYSQL_PWD=xx mysql -hxxx -Pxxx db_name < backup_data.sql"
echo " 4) RESTORE_LOGICAL_TYPE: optional, choose from: ddl | insert | csv"

}

Expand Down
96 changes: 73 additions & 23 deletions bin/restore.sh
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,21 @@ function restore_precheck()

}

function restore_prep_report()
{
restore_report_path=`dirname "${RESTORE_REPORT}"`
add_log "D" "Creating restore report direcory: mkdir -p ${restore_report_path}"
mkdir -p ${restore_report_path}
if [[ ! -f "${RESTORE_REPORT}" ]]; then
add_log "D" "Creating restore report file ${RESTORE_REPORT}"
echo "restore_date|restore_target|restore_type|physical_bk_id|logical_restore_src|logical_type|duration_ms|outcome" > "${RESTORE_REPORT}"
fi
}

function restore_physical()
{
add_log "I" "MO_HOST: ${MO_HOST}"
if [[ "MO_HOST" != "127.0.0.1" ]]; then
if [[ "${MO_HOST}" != "127.0.0.1" ]]; then
add_log "E" "Currently mo_ctl only support restoring physical backup data on a local mo server, thus please set MO_HOST to 127.0.0.1 if that's the case."
return 1
fi
Expand All @@ -87,8 +98,6 @@ function restore_physical()

add_log "I" "Restore begins"

backup_yearmonth=`date '+%Y%m'`
backup_timestamp=`date '+%Y%m%d_%H%M%S'`

BACKUP_MOBR_DIRNAME=`dirname "${BACKUP_MOBR_PATH}"`

Expand Down Expand Up @@ -128,6 +137,7 @@ function restore_physical()
add_log "D" "cmd: cd ${BACKUP_MOBR_DIRNAME} && ${cmd}"


restore_timestamp=`date '+%Y%m%d_%H%M%S'`
startTime=`get_nanosecond`
if cd ${BACKUP_MOBR_DIRNAME} && ${cmd}; then
outcome="succeeded"
Expand All @@ -140,11 +150,17 @@ function restore_physical()

add_log "I" "Outcome: ${outcome}, cost: ${cost} ms"

restore_prep_report
echo "${restore_timestamp}|${MO_HOST},${MO_PORT},${MO_USER}|physical|${RESTORE_BKID}|||${cost}|${outcome}" >> ${RESTORE_REPORT}


if [[ ${rc} -ne 0 ]]; then
add_log "E" "Restore ends with non-zero rc"
else
add_log "I" "Restore ends with 0 rc"
fi

return ${rc}
}

function restore_mo_data()
Expand Down Expand Up @@ -219,41 +235,75 @@ function restore_logical()
fi


backup_yearmonth=`date '+%Y%m'`
backup_timestamp=`date '+%Y%m%d_%H%M%S'`

BACKUP_MODUMP_DIRNAME=`dirname "${BACKUP_MOBR_PATH}"`

add_log "I" "Restore begins, please wait"
#todo
#if MYSQL_PWD="${M_PW}" mysql -h"${MO_HOST}" -P"${MO_PORT}" -u"${MO_USER}" "${dbname_option}" < ${}
#cmd="mysql -u$}"

if [[ -d "${RESTORE_LOGICAL_SRC}" ]]; then
isFile="false"
srcPath="${RESTORE_LOGICAL_SRC}"
add_log "I" "RESTORE_LOGICAL_SRC=${RESTORE_LOGICAL_SRC} is a path, listing files in it"
ls -lth "${RESTORE_LOGICAL_SRC}"
elif [[ -f "${RESTORE_LOGICAL_SRC}" ]]; then
isFile="true"
srcPath=`dirname "${RESTORE_LOGICAL_SRC}"`
add_log "I" "RESTORE_LOGICAL_SRC=${RESTORE_LOGICAL_SRC} is a file"
else
add_log "E" "RESTORE_LOGICAL_SRC=${RESTORE_LOGICAL_SRC} is a not a path nor file. Please check again. Exiting"
return 1
fi

add_log "D" "cmd: cd ${BACKUP_MOBR_DIRNAME} && ${cmd}"
restore_prep_report

add_log "I" "Restore begins, please wait"
i=1
rc=0
for fileName in `ls ${srcPath}/ | grep ".sql"`; do
if [[ "${isFile}" == "true" ]]; then
file="${RESTORE_LOGICAL_SRC}"
else
file="${srcPath}/${fileName}"
fi
add_log "D" "cmd: MYSQL_PWD="${MO_PW}" mysql -h${MO_HOST} -P${MO_PORT} -u${MO_USER} ${dbname_option} < ${file}"
restore_timestamp=`date '+%Y%m%d_%H%M%S'`
startTime=`get_nanosecond`
if MYSQL_PWD="${MO_PW}" mysql -h${MO_HOST} -P${MO_PORT} -u${MO_USER} ${dbname_option} < ${file}; then
outcome="succeeded"
else
outcome="failed"
let rc=rc+1
fi
endTime=`get_nanosecond`
cost=`time_cost_ms ${startTime} ${endTime}`


add_log "I" "Number: $i, file: ${file}, outcome: ${outcome}, cost: ${cost} ms"
echo "${restore_timestamp}|${MO_HOST},${MO_PORT},${MO_USER}|logical||${file}|${RESTORE_LOGICAL_TYPE}|${cost}|${outcome}" >> ${RESTORE_REPORT}

if [[ "${isFile}" == "true" ]]; then
break
fi
let i=i+1

done

let i=i-1

add_log "I" "-------------------------"
add_log "I" " Summary "
add_log "I" "Total: ${i}, failed: ${rc}"
add_log "I" "-------------------------"

startTime=`get_nanosecond`
if cd ${BACKUP_MOBR_DIRNAME} && ${cmd}; then
outcome="succeeded"
else
outcome="failed"
rc=1
fi
endTime=`get_nanosecond`
cost=`time_cost_ms ${startTime} ${endTime}`

add_log "I" "Outcome: ${outcome}, cost: ${cost} ms"

if [[ ${rc} -ne 0 ]]; then
add_log "E" "Restore ends with non-zero rc"
else
add_log "I" "Restore ends with 0 rc"
fi
return ${rc}
}

function restore()
{
add_log "I" "RESTORE_TYPE: ${RESTORE_TYPE}"

case "${RESTORE_TYPE}" in
"physical")
Expand Down
Loading

0 comments on commit 0497a2a

Please sign in to comment.