Skip to content

Commit

Permalink
doc: add some docs
Browse files Browse the repository at this point in the history
  • Loading branch information
@aronchanisme
aronchanisme committed Jan 17, 2025
1 parent 8043132 commit 6a0b4a8
Show file tree
Hide file tree
Showing 16 changed files with 951 additions and 136 deletions.
Binary file added .DS_Store
View file
Binary file not shown.
13 changes: 9 additions & 4 deletions bin/auto_backup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,11 @@ function backup()
add_log "D" "BACKUP_MOBR_META_PATH is not empty, will add option ${br_meta_option}"
fi

if [[ "${BACKUP_PHYSICAL_PARALLEL_NUM}" == "" ]]; then
BACKUP_PHYSICAL_PARALLEL_NUM="1"
add_log "D" "BACKUP_PHYSICAL_PARALLEL_NUM is empty, will use default value 1"
fi


BACKUP_MOBR_DIRNAME=`dirname "${BACKUP_MOBR_PATH}"`

Expand Down Expand Up @@ -402,9 +407,9 @@ function backup()

case "${BACKUP_PHYSICAL_TYPE}" in
"filesystem")
add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host ${MO_HOST} --port ${MO_PORT} --user ${MO_USER} --password ${MO_PW} --backup_dir filesystem --path ${backup_outpath} ${delta_option}"
add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --parallelism ${BACKUP_PHYSICAL_PARALLEL_NUM} --host ${MO_HOST} --port ${MO_PORT} --user ${MO_USER} --password ${MO_PW} --backup_dir filesystem --path ${backup_outpath} ${delta_option}"
startTime=`get_nanosecond`
if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host ${MO_HOST} --port ${MO_PORT} --user ${MO_USER} --password ${MO_PW} --backup_dir filesystem --path ${backup_outpath} ${delta_option}; then
if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --parallelism ${BACKUP_PHYSICAL_PARALLEL_NUM} --host ${MO_HOST} --port ${MO_PORT} --user ${MO_USER} --password ${MO_PW} --backup_dir filesystem --path ${backup_outpath} ${delta_option}; then
outcome="succeeded"
else
outcome="failed"
Expand All @@ -422,10 +427,10 @@ function backup()
role_arn_option="--role_arn ${BACKUP_S3_ROLE_ARN}"
fi

add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"s3\" --endpoint \"${BACKUP_S3_ENDPOINT}\" --access_key_id \"${BACKUP_S3_ID}\" --secret_access_key \"${BACKUP_S3_KEY}\" --bucket \"${BACKUP_S3_BUCKET}\" --filepath \"${backup_outpath}\" --region \"${BACKUP_S3_REGION}\" --compression \"${BACKUP_S3_COMPRESSION}\" \"${role_arn_option}\" \"${minio_option}\" ${delta_option}"
add_log "D" "Backup command: cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --parallelism \"${BACKUP_PHYSICAL_PARALLEL_NUM}\" --host \"${MO_HOST}\" --port \"${MO_PORT}\" --user \"${MO_USER}\" --password \"${MO_PW}\" --backup_dir \"s3\" --endpoint \"${BACKUP_S3_ENDPOINT}\" --access_key_id \"${BACKUP_S3_ID}\" --secret_access_key \"${BACKUP_S3_KEY}\" --bucket \"${BACKUP_S3_BUCKET}\" --filepath \"${backup_outpath}\" --region \"${BACKUP_S3_REGION}\" --compression \"${BACKUP_S3_COMPRESSION}\" \"${role_arn_option}\" \"${minio_option}\" ${delta_option}"

startTime=`get_nanosecond`
if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "s3" --endpoint "${BACKUP_S3_ENDPOINT}" --access_key_id "${BACKUP_S3_ID}" --secret_access_key "${BACKUP_S3_KEY}" --bucket "${BACKUP_S3_BUCKET}" --filepath "${backup_outpath}" --region "${BACKUP_S3_REGION}" --compression "${BACKUP_S3_COMPRESSION}" "${role_arn_option}" "${minio_option}" ${delta_option}; then
if cd ${BACKUP_MOBR_DIRNAME} && ${BACKUP_MOBR_PATH} backup ${br_meta_option} --parallelism "${BACKUP_PHYSICAL_PARALLEL_NUM}" --host "${MO_HOST}" --port "${MO_PORT}" --user "${MO_USER}" --password "${MO_PW}" --backup_dir "s3" --endpoint "${BACKUP_S3_ENDPOINT}" --access_key_id "${BACKUP_S3_ID}" --secret_access_key "${BACKUP_S3_KEY}" --bucket "${BACKUP_S3_BUCKET}" --filepath "${backup_outpath}" --region "${BACKUP_S3_REGION}" --compression "${BACKUP_S3_COMPRESSION}" "${role_arn_option}" "${minio_option}" ${delta_option}; then
outcome="succeeded"
else
outcome="failed"
Expand Down
74 changes: 38 additions & 36 deletions bin/basic.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,42 +5,6 @@
################################################################
# basic funtions

function check_cmd()
{
cmd="$1"
if [[ "${cmd}" == "" ]]; then
add_log "E" "Command is empty, please input a command name"
return 1
fi

if ! command -v ${cmd} >/dev/null 2>&1; then
add_log "E" "Command '${cmd}' is not found"
return 1
else
add_log "D" "Command '${cmd}' is found"
fi
}

function check_conf_item_not_empty()
{
conf_item="$1"

if [[ "${conf_item}" == "" ]]; then
add_log "E" "conf_item is empty, please input a conf item name"
return 1
fi


conf_value=`eval echo '$'${conf_item}`
if [[ "${conf_value}" == "" ]]; then
add_log "E" "Conf item ${conf_item} is empty, please set it first"
return 1
else
add_log "D" "Conf item ${conf_item} is set to ${conf_value}"
fi
}


# fuction: return os name of current machine
# usage: what_os
# in: none
Expand Down Expand Up @@ -208,6 +172,44 @@ function add_log()
esac
}


function check_cmd()
{
cmd="$1"
if [[ "${cmd}" == "" ]]; then
add_log "E" "Command is empty, please input a command name"
return 1
fi

if ! command -v ${cmd} >/dev/null 2>&1; then
add_log "E" "Command '${cmd}' is not found"
return 1
else
add_log "D" "Command '${cmd}' is found"
fi
}

function check_conf_item_not_empty()
{
conf_item="$1"

if [[ "${conf_item}" == "" ]]; then
add_log "E" "conf_item is empty, please input a conf item name"
return 1
fi


conf_value=`eval echo '$'${conf_item}`
if [[ "${conf_value}" == "" ]]; then
add_log "E" "Conf item ${conf_item} is empty, please set it first"
return 1
else
add_log "D" "Conf item ${conf_item} is set to ${conf_value}"
fi
}



# function: compare the version number of 2 given string
# usage: cmp_version [v1] [v2]
# e.g. : cmp_version 1.20.1 1.19
Expand Down
2 changes: 1 addition & 1 deletion bin/get_branch.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ function get_branch()
cid_full=`git log | head -n 1 | awk {'print $2'}`
cid_less=`echo "${cid_full:0:8}"`
cd ${MO_PATH}/matrixone
current_branch=`git branch --contains ${cid_less} -a | grep -v HEAD | grep -v "detached" | grep "remotes" | sed 's/ //g' | awk -F "/" '{print $NF}'`
current_branch=`git branch --contains ${cid_less} -a | grep -v HEAD | grep -v "detached" | grep "remotes" | sed 's/ //g' | awk -F "/" '{print $NF}' | head -n 1`
if [[ "${option}" != "less" ]]; then
add_log "D" "cid_full: ${cid_full}, cid_less: ${cid_less}, current_branch: ${current_branch}"
fi
Expand Down
9 changes: 6 additions & 3 deletions bin/help.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ function help_bk_notes()
echo " ------------------------- "

echo " 1) BACKUP_CRON_SCHEDULE_FULL (optional, default: 30 23 * * *): for auto_backup of physical full type, cron expression to control backup schedule time and frequency, in standard cron format (https://crontab.guru/)"
echo " 2) BACKUP_CRON_SCHEDULE_INCREMENTAL (optional, default: */2 * * * *): for auto_backup of physical incremental type, same format as BACKUP_CRON_SCHEDULE_FULL"
echo " 2) BACKUP_CRON_SCHEDULE_INCREMENTAL (optional, default: * */2 * * *): for auto_backup of physical incremental type, same format as BACKUP_CRON_SCHEDULE_FULL"
echo " 3) BACKUP_CLEAN_DAYS_BEFORE (optional, default: 31): for auto_backup clean up, clean old backup files before [x] days"
echo " 4) BACKUP_CLEAN_CRON_SCHEDULE (optional, default: 0 6 * * *): for auto_backup clean up, cron to control auto clean of old backups"

Expand All @@ -314,8 +314,9 @@ function help_bk_notes()
echo " full: perform a full data backup from scratch"
echo " incremental: perform an incremental data backup based on a full backup or incremental backup"
echo " 3) BACKUP_PHYSICAL_BASE_BKID (required, when BACKUP_PHYSICAL_METHOD=incremental): the backup id which incremental to be based on"
echo " 4) BACKUP_AUTO_SET_LAST_BKID (optional, default: yes): available: 'yes'|'no'. If 'yes', will automatically set BACKUP_PHYSICAL_BASE_BKID to last success backup id"
echo " 5) BACKUP_PHYSICAL_TYPE (optional, default: filesystem): target backup storage type, choose from \"filesystem\" | \"s3\""
echo " 4) BACKUP_PHYSICAL_PARALLEL_NUM (optional, default: 2): physical backup parallism"
echo " 5) BACKUP_AUTO_SET_LAST_BKID (optional, default: yes): available: 'yes'|'no'. If 'yes', will automatically set BACKUP_PHYSICAL_BASE_BKID to last success backup id"
echo " 6) BACKUP_PHYSICAL_TYPE (optional, default: filesystem): target backup storage type, choose from \"filesystem\" | \"s3\""
echo " if BACKUP_PHYSICAL_TYPE=s3, please set below confs:"
echo " a) BACKUP_S3_ENDPOINT (optional, default: ''): s3 endpoint, e.g. https://cos.ap-nanjing.myqcloud.com"
echo " b) BACKUP_S3_ID (optional, default: ''): s3 id, e.g. B4v6Khv484X81dk81jQFzc9YxKl98JOyxkX1k"
Expand All @@ -338,6 +339,8 @@ function help_bk_notes()
echo " c) db1,db2,db3: example to backup db1, db2 and db3"
echo " 3) BACKUP_LOGICAL_DATA_TYPE (optional, default: csv): available: insert | csv. Backup data type"
echo " 4) BACKUP_LOGICAL_ONEBYONE (optional, default: 0): available: 0|1. If set to 1, will backup databases/tables one by one into multiple backup files."
echo " 5) BACKUP_LOGICAL_NETBUFLEN (optional, default: 1048576): backup net buffer length(bytes, integer), default: 1048576(1M) , max: 16777216(16M)"
echo " 6) BACKUP_LOGICAL_DS (optional, default: none): backup logical dataset name: (optional) the dataset name of the backup database, e.g. myds_001"

}

Expand Down
12 changes: 10 additions & 2 deletions bin/set_conf.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,17 @@ function set_kv()
add_log "I" "Setting conf ${key}=\"${value}\""
os=`what_os`
if [[ ${os} == "Linux" ]] ; then
sed -i "s#^${key}=.*#${key}=\"${value}\"#g" "${CONF_FILE}"
if echo "${value}" | grep "#" >/dev/null 2>&1 ; then
sed -i "s|^${key}=.*|${key}=\"${value}\"|g" "${CONF_FILE}"
else
sed -i "s#^${key}=.*#${key}=\"${value}\"#g" "${CONF_FILE}"
fi
else
sed -i "" "s#^${key}=.*#${key}=\"${value}\"#g" "${CONF_FILE}"
if echo "${value}" | grep "#" >/dev/null 2>&1 ; then
sed -i "" "s|^${key}=.*|${key}=\"${value}\"|g" "${CONF_FILE}"
else
sed -i "" "s#^${key}=.*#${key}=\"${value}\"#g" "${CONF_FILE}"
fi
fi

}
Expand Down
5 changes: 3 additions & 2 deletions conf/env.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ BACKUP_TYPE="physical"
# full backup: cron to control auto backup schedule time and frequency, in standard cron format (https://crontab.guru/)
BACKUP_CRON_SCHEDULE_FULL="30 23 * * *"
# incremental backup: same as above
BACKUP_CRON_SCHEDULE_INCREMENTAL="* * * * *"
BACKUP_CRON_SCHEDULE_INCREMENTAL="* */2 * * *"
# backup data path
BACKUP_DATA_PATH="/data/mo-backup"
# add timestamp subpaths to backup data path? format: ${BACKUP_DATA_PATH}/202406/20240620_161838
Expand All @@ -228,7 +228,8 @@ BACKUP_PHYSICAL_METHOD="full"
BACKUP_PHYSICAL_BASE_BKID=""
# if yes, will automatically set BACKUP_PHYSICAL_BASE_BKID to last sucess backup id: yes(default)|no
BACKUP_AUTO_SET_LAST_BKID="yes"

# physical backup parallism, default: 2
BACKUP_PHYSICAL_PARALLEL_NUM=2

# 1) when BACKUP_PHYSICAL_TYPE="filesystem"
# backup directory, same as BACKUP_PATH
Expand Down
Loading

0 comments on commit 6a0b4a8

Please sign in to comment.