Skip to content

Commit

Permalink
feat: add some new features
Browse files Browse the repository at this point in the history
  • Loading branch information
aronchanisme committed Nov 20, 2024
1 parent 09626e9 commit 2abc3dc
Show file tree
Hide file tree
Showing 24 changed files with 1,571 additions and 398 deletions.
112 changes: 74 additions & 38 deletions bin/auto_backup.sh

Large diffs are not rendered by default.

222 changes: 222 additions & 0 deletions bin/auto_logrotate.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,222 @@
#!/bin/bash
################################################################
# Copyright (C) 2023 Matrix Origin. All Rights Reserved
# Visit us at https://www.matrixorigin.cn/
################################################################
# auto_log_rotate

alr_name="auto_log_rotate"
LOG_ROTATE_CRON_PATH="/etc/logrotate.d"
LOG_ROTATE_CRON_FILE_NAME="mo-service"
MO_LOG_RESERVE_MAX_NUM="100000"
OS=""
LOG_SPLIT_STRATEGY=""

function auto_log_rotate_precheck()
{
# 1. Mac
if [[ "${OS}" == "Mac" ]]; then

add_log "E" "Auto log rotate on Mac is not yet supported"
return 1
# 2. Linux
else
# 1. check logrotate cmd
add_log "I" "Check if logrotate command exists"
flag=1
if which logrotate > /dev/null 2>&1 ; then
add_log "I" "Command 'which logrotate' found logrotate"
elif [[ -f /usr/sbin/logrotate ]] ; then
add_log "I" "Command 'which logrotate' did not found logrotate, but found file '/usr/sbin/logrotate'"
else
add_log "E" "Either command 'which logrotate' nor file '/usr/sbin/logrotate' is found, exiting"
return 1
fi

# 2. Check sudo privilege
current_user=`whoami`
add_log "I" "Check if current user has sudo command, you may need to enter password for current user"
if ! sudo ls /root >/dev/null 2>&1 ; then
add_log "E" "Failed to execute cmd 'sudo ls /root' with current user '${current_user}', please make sure current user has sudo privilege"
return 1
fi


# 3. Check MO_LOG_PATH conf
add_log "I" "Check related confs as below:"
get_conf | grep -E "MO_LOG_AUTO_SPLIT|MO_LOG_MAX_SIZE|MO_LOG_RESERVE_NUM"

case "${MO_LOG_AUTO_SPLIT}" in
"daily")
LOG_SPLIT_STRATEGY="daily"
;;
"size")
if [ -z "${MO_LOG_MAX_SIZE}" ]; then
add_log "E" "Conf 'MO_LOG_AUTO_SPLIT' is set to 'size', but conf 'MO_LOG_MAX_SIZE' is not set, please set it first, e.g. mo_ctl set_conf MO_LOG_MAX_SIZE=1024M"
return 1
else
LOG_SPLIT_STRATEGY="size ${MO_LOG_MAX_SIZE}"
fi
;;
*)
add_log "E" "Invalid value ${MO_LOG_AUTO_SPLIT} for conf MO_LOG_AUTO_SPLIT, choose from: daily|size"
return 1
;;
esac

add_log "D" "LOG_SPLIT_STRATEGY=${LOG_SPLIT_STRATEGY}"

if ! pos_int_range "${MO_LOG_RESERVE_NUM}" "${MO_LOG_RESERVE_MAX_NUM}"; then
add_log "E" "${MO_LOG_RESERVE_NUM} is not greater than 0 and less than ${MO_LOG_RESERVE_MAX_NUM}, please set it to an integer between 1 and ${MO_LOG_RESERVE_MAX_NUM}"
return 1
fi
fi
}

function auto_log_rotate_status()
{
rc=0

if [[ -s ${LOG_ROTATE_CRON_PATH}/${LOG_ROTATE_CRON_FILE_NAME} ]]; then
add_log "D" "Cron file ${LOG_ROTATE_CRON_PATH}/${LOG_ROTATE_CRON_FILE_NAME} for ${alr_name} already exists, trying to get content: "
acl_content=`cat ${LOG_ROTATE_CRON_PATH}/${LOG_ROTATE_CRON_FILE_NAME}`
add_log "D" "${acl_content}"
add_log "I" "${alr_name} status:enabled"
else
add_log "D" "Cron file ${LOG_ROTATE_CRON_PATH}/${LOG_ROTATE_CRON_FILE_NAME} for ${alr_name} does not exist"
add_log "I" "${alr_name} status:disabled"
rc=1
fi

return ${rc}
}


function auto_log_rotate_enable()
{
if ! auto_log_rotate_precheck; then
return 1
fi

if auto_log_rotate_status; then
add_log "I" "Function auto_log_rotate has already been enabled, exiting"
return 0
fi

add_log "D" "Writing content to ${LOG_ROTATE_CRON_PATH}/${LOG_ROTATE_CRON_FILE_NAME}"
add_log "D" "${MO_LOG_PATH}/*.log {" "l"
add_log "D" "${LOG_SPLIT_STRATEGY}" "l"
add_log "D" "compress" "l"
add_log "D" "rotate ${MO_LOG_RESERVE_NUM}" "l"
add_log "D" "copytruncate" "l"
add_log "D" "missingok" "l"
add_log "D" "notifempty" "l"
add_log "D" "dateext" "l"
add_log "D" "dateformat -%Y%m%d_%H%M%S" "l"
add_log "D" "}" "l"

add_log "I" "Password of current user may be required to execute command in sudo mode"

add_log "D" "Command: sudo touch ${LOG_ROTATE_CRON_PATH}/${LOG_ROTATE_CRON_FILE_NAME}"
sudo touch ${LOG_ROTATE_CRON_PATH}/${LOG_ROTATE_CRON_FILE_NAME}

add_log "D" "sudo chmod 777 ${LOG_ROTATE_CRON_PATH}/${LOG_ROTATE_CRON_FILE_NAME}"
sudo chmod 777 ${LOG_ROTATE_CRON_PATH}/${LOG_ROTATE_CRON_FILE_NAME}

add_log "D" "Command:"
add_log "D" "sudo cat > ${LOG_ROTATE_CRON_PATH}/${LOG_ROTATE_CRON_FILE_NAME} << EOF" "l"
add_log "D" "${MO_LOG_PATH}/*.log {" "l"
add_log "D" "${LOG_SPLIT_STRATEGY}" "l"
add_log "D" "compress" "l"
add_log "D" "rotate ${MO_LOG_RESERVE_NUM}" "l"
add_log "D" "copytruncate" "l"
add_log "D" "missingok" "l"
add_log "D" "notifempty" "l"
add_log "D" "dateext" "l"
add_log "D" "dateformat -%Y%m%d_%H%M%S" "l"
add_log "D" "}" "l"
add_log "D" "EOF" "l"

sudo cat > ${LOG_ROTATE_CRON_PATH}/${LOG_ROTATE_CRON_FILE_NAME} << EOF
${MO_LOG_PATH}/*.log {
${LOG_SPLIT_STRATEGY}
compress
rotate ${MO_LOG_RESERVE_NUM}
copytruncate
missingok
notifempty
dateext
dateformat -%Y%m%d_%H%M%S
}
EOF

add_log "D" "sudo chmod 644 ${LOG_ROTATE_CRON_PATH}/${LOG_ROTATE_CRON_FILE_NAME}"
sudo chmod 644 ${LOG_ROTATE_CRON_PATH}/${LOG_ROTATE_CRON_FILE_NAME}

if ! auto_log_rotate_status; then
add_log "Failed to enable auto_log_rotate"
return 1
fi

}



function auto_log_rotate_disable()
{
if ! auto_log_rotate_precheck; then
return 1
fi

if ! auto_log_rotate_status; then
add_log "I" "Function auto_log_rotate has already been disabled, exiting"
return 0
fi


add_log "I" "Disabling ${alr_name} by removing cron file ${LOG_ROTATE_CRON_PATH}/${LOG_ROTATE_CRON_FILE_NAME}"
if cd ${LOG_ROTATE_CRON_PATH} && sudo rm -f ./${LOG_ROTATE_CRON_FILE_NAME}; then
add_log "I" "Succeeded"
else
add_log "E" "Failed"
return 1
fi

if auto_log_rotate_status; then
add_log "E" "Failed to disable auto_log_rotate"
return 1
fi

}




function auto_log_rotate()
{

option="$1"
OS=`what_os`
CLEAN_LOGS_CRON_USER=`whoami`

add_log "D" "Current OS: ${OS}"

case "${option}" in
"" | "status")
auto_log_rotate_status
;;
"enable")
auto_log_rotate_enable
;;
"disable")
auto_log_rotate_disable
;;
*)
add_log "E" "Invalid option for ${alr_name}: ${option}"
help_auto_log_rotate
return 1
;;
esac


}
46 changes: 46 additions & 0 deletions bin/basic.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,42 @@
################################################################
# basic funtions

function check_cmd()
{
cmd="$1"
if [[ "${cmd}" == "" ]]; then
add_log "E" "Command is empty, please input a command name"
return 1
fi

if ! command -v ${cmd} >/dev/null 2>&1; then
add_log "E" "Command '${cmd}' is not found"
return 1
else
add_log "D" "Command '${cmd}' is found"
fi
}

function check_conf_item_not_empty()
{
conf_item="$1"

if [[ "${conf_item}" == "" ]]; then
add_log "E" "conf_item is empty, please input a conf item name"
return 1
fi


conf_value=`eval echo '$'${conf_item}`
if [[ "${conf_value}" == "" ]]; then
add_log "E" "Conf item ${conf_item} is empty, please set it first"
return 1
else
add_log "D" "Conf item ${conf_item} is set to ${conf_value}"
fi
}


# fuction: return os name of current machine
# usage: what_os
# in: none
Expand Down Expand Up @@ -393,4 +429,14 @@ function basic()
option_1=$2
option_2=$3
${func_name} ${option_1} ${option_2}
}

function read_user_confirm()
{
user_confirm=""
read -t 30 user_confirm
if [[ "$(to_lower ${user_confirm})" != "yes" ]]; then
add_log "E" "User input not confirmed or timed out, exiting"
return 1
fi
}
33 changes: 18 additions & 15 deletions bin/csv_convert.sh
Original file line number Diff line number Diff line change
Expand Up @@ -283,18 +283,18 @@ function csv_convert_load_serial()
# get batch size lines and replace char \n with string \\n
sed -n "${start_line},${end_line}p" ${CSV_CONVERT_SRC_FILE} | ':a;N;$!ba;s/\n/\\n/g' >> ${TGT_FILE}
if [[ "${CSV_CONVERT_META_COLUMN_LIST}" == "" ]]; then
echo "' into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE};" >> ${TGT_FILE}
echo "' into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE} ${fileds_termiated_by} ;" >> ${TGT_FILE}
else
echo "' into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE}(${CSV_CONVERT_META_COLUMN_LIST});" >> ${TGT_FILE}
echo "' into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE} ${fileds_termiated_by} (${CSV_CONVERT_META_COLUMN_LIST});" >> ${TGT_FILE}
fi
else
echo "load data inline format='csv', data=\$XXX\$" >> ${TGT_FILE}
sed -n "${start_line},${end_line}p" ${CSV_CONVERT_SRC_FILE} >> ${TGT_FILE}
echo "\$XXX\$" >> ${TGT_FILE}
if [[ "${CSV_CONVERT_META_COLUMN_LIST}" == "" ]]; then
echo "into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE};" >> ${TGT_FILE}
echo "into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE} ${fileds_termiated_by} ;" >> ${TGT_FILE}
else
echo "into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE}(${CSV_CONVERT_META_COLUMN_LIST});" >> ${TGT_FILE}
echo "into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE} ${fileds_termiated_by} (${CSV_CONVERT_META_COLUMN_LIST});" >> ${TGT_FILE}
fi
fi

Expand All @@ -309,8 +309,15 @@ function csv_convert_load_parallel()
# multi or single
line_mode="$1"


add_log "I" "Convert csv file to \"load data inline format='csv' (multiple lines) \" sql file"

fileds_termiated_by=""
if [[ "${CSV_FIELDS_TERMINATED_BY}" != "" ]]; then
add_log "D" "CSV_FIELDS_TERMINATED_BY: ${CSV_FIELDS_TERMINATED_BY}"
fileds_termiated_by="fields terminated by '${CSV_FIELDS_TERMINATED_BY}'"
fi

for((i=0; i < ${LOOP_COUNT}; i++));do

add_log "D" "Loop number: ${i}"
Expand Down Expand Up @@ -342,19 +349,19 @@ function csv_convert_load_parallel()
sed -i "s#^#load data inline format='csv', data='#g" ${sql_tmp_file}
# add to the end
if [[ "${CSV_CONVERT_META_COLUMN_LIST}" == "" ]]; then
sed -i "s#\$# into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE};#g" ${sql_tmp_file}
sed -i "s#\$# into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE} ${fileds_termiated_by};#g" ${sql_tmp_file}
else
sed -i "s#\$# into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE}(${CSV_CONVERT_META_COLUMN_LIST});#g" ${sql_tmp_file}
sed -i "s#\$# into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE} ${fileds_termiated_by} (${CSV_CONVERT_META_COLUMN_LIST});#g" ${sql_tmp_file}
fi
else
sed -i "" ':a;N;$!ba;s/\n/\\n/g' ${sql_tmp_file}
# add to the beginning
sed -i "" "s#^#load data inline format='csv', data='#g" ${sql_tmp_file}
# add to the end
if [[ "${CSV_CONVERT_META_COLUMN_LIST}" == "" ]]; then
sed -i "" "s#\$# into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE};#g" ${sql_tmp_file}
sed -i "" "s#\$# into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE} ${fileds_termiated_by};#g" ${sql_tmp_file}
else
sed -i "" "s#\$# into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE}(${CSV_CONVERT_META_COLUMN_LIST});#g" ${sql_tmp_file}
sed -i "" "s#\$# into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE} ${fileds_termiated_by} (${CSV_CONVERT_META_COLUMN_LIST});#g" ${sql_tmp_file}
fi
fi
# 2) multi lines
Expand All @@ -372,9 +379,9 @@ function csv_convert_load_parallel()
fi
echo "\$XXX\$" >> ${sql_tmp_file}
if [[ "${CSV_CONVERT_META_COLUMN_LIST}" == "" ]]; then
echo "into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE};" >> ${sql_tmp_file}
echo "into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE} ${fileds_termiated_by} ;" >> ${sql_tmp_file}
else
echo "into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE}(${CSV_CONVERT_META_COLUMN_LIST});" >> ${sql_tmp_file}
echo "into table ${CSV_CONVERT_META_DB}.${CSV_CONVERT_META_TABLE} ${fileds_termiated_by} (${CSV_CONVERT_META_COLUMN_LIST});" >> ${sql_tmp_file}
fi

fi
Expand Down Expand Up @@ -404,11 +411,7 @@ function csv_convert()
fi

add_log "I" "Please make sure above configurations are correct, continue? (Yes/No)"
read -t 30 user_confirm
if [[ "$(to_lower ${user_confirm})" != "yes" ]]; then
add_log "E" "User input not confirmed or timed out, exiting"
return 1
fi
read_user_confirm

add_log "I" "Conversion begins, this may take a while depending on the size of source file and processing ability of your machine. Please wait..."

Expand Down
Loading

0 comments on commit 2abc3dc

Please sign in to comment.