diff --git a/dynamic_flow/config/watch/f40.conf b/dynamic_flow/config/watch/f40.conf index 5e204eb..46edb8d 100644 --- a/dynamic_flow/config/watch/f40.conf +++ b/dynamic_flow/config/watch/f40.conf @@ -20,6 +20,7 @@ log_reject on post_log_format v03 #randomize +inflight 10 caching 15m heartbeat 60 diff --git a/flow_utils.sh b/flow_utils.sh index b3f4591..e07cf49 100755 --- a/flow_utils.sh +++ b/flow_utils.sh @@ -48,6 +48,7 @@ try: cachedir = appdirs.user_cache_dir('${SR_DEV_APPNAME}','MetPX') confdir = appdirs.user_config_dir('${SR_DEV_APPNAME}','MetPX') logdir = appdirs.user_log_dir('${SR_DEV_APPNAME}','MetPX') + loghostdir = logdir.replace( 'log', "${HOSTNAME}/log" ) except: @@ -55,6 +56,7 @@ except: cachedir = str(pathlib.Path.home()) + '/.cache/${SR_DEV_APPNAME}' confdir = str(pathlib.Path.home()) + '/.config/${SR_DEV_APPNAME}' logdir = str(pathlib.Path.home()) + '/.cache/${SR_DEV_APPNAME}/log' + loghostdir = str(pathlib.Path.home()) + '/.cache/${SR_DEV_APPNAME}/${HOSTNAME}/log' cachedir = cachedir.replace(' ',r'\ ') @@ -66,6 +68,9 @@ print('export CONFDIR=%s'% confdir) logdir = logdir.replace(' ',r'\ ') print('export LOGDIR=%s'% logdir) +loghostdir = loghostdir.replace(' ',r'\ ') +print('export LOGHOSTDIR=%s'% loghostdir) + EOF } diff --git a/static_flow/config/poll/sftp_f62.conf b/static_flow/config/poll/sftp_f62.conf index a41e7f6..fdbe4b0 100644 --- a/static_flow/config/poll/sftp_f62.conf +++ b/static_flow/config/poll/sftp_f62.conf @@ -15,6 +15,8 @@ vip 127.0.0.1 set sarracenia.moth.amqp.AMQP.logLevel info set sarracenia.moth.mqtt.MQTT.logLevel info +logReject on + timezone EST5EDT sleep 10 ls_file_index 8 diff --git a/static_flow/config/sarra/download_f20.conf b/static_flow/config/sarra/download_f20.conf index ebc2ef2..a24228e 100644 --- a/static_flow/config/sarra/download_f20.conf +++ b/static_flow/config/sarra/download_f20.conf @@ -25,6 +25,8 @@ topic_prefix v03.post heartbeat 60 +statehost on + debug batch 20 # I want the times the files are written, rather than source times. diff --git a/static_flow/flow_cleanup.sh b/static_flow/flow_cleanup.sh index c3675f5..2e170c6 100755 --- a/static_flow/flow_cleanup.sh +++ b/static_flow/flow_cleanup.sh @@ -116,12 +116,19 @@ echo "Removing flow config logs..." if [ "$1" != "skipconfig" ]; then if [ "${sarra_py_version:0:1}" == "3" ]; then echo $flow_configs | sed 's/ / ;\n rm -f /g' | sed '1 s|^| rm -f |' | sed '/^ rm -f post/d' | sed 's+/+_+g' | sed '/conf[ ;]*$/!d' | sed 's/\.conf/_[0-9][0-9].log\*/g' | (cd $LOGDIR; sh ) - echo $flow_configs | sed 's/ / ;\n rm -f /g' | sed '1 s|^| rm -f |' | sed 's+/+_+g' | sed '/conf[ ;]*$/!d' | sed 's/\.conf/_[0-9][0-9].json\*/g'| (cd ${LOGDIR}/../metrics; sh ) + + rm ${LOGHOSTDIR}/sarra_download_f20*.log else echo $flow_configs | sed 's/ / ;\n rm -f sr_/g' | sed '1 s|^| rm -f sr_|' | sed '/^ rm -f sr_post/d' | sed 's+/+_+g' | sed '/conf[ ;]*$/!d' | sed 's/\.conf/_[0-9][0-9].log\*/g' | (cd $LOGDIR; sh ) fi fi +if [ "${sarra_py_version:0:1}" == "3" ]; then + # remove all metrics files + #echo $flow_configs | sed 's/ / ;\n rm -f /g' | sed '1 s|^| rm -f |' | sed 's+/+_+g' | sed '/conf[ ;]*$/!d' | sed 's/\.conf/_[0-9][0-9].json\*/g'| (cd ${LOGDIR}/../metrics; sh ) + sr3 show '*/*_f[0-9][0-9]' |& grep metricsFilename | sed "s/'//g" | sed 's/,//' | sed 's/_00.json/_[0-9][0-9].json*/' | awk '{print $2};' | xargs rm -f +fi + rm $LOGDIR/${LGPFX}post_t_dd?_f00_01.log $LOGDIR/${LGPFX}post_shim_f63_01.log $LOGDIR/${LGPFX}post_test2_f61_01.log rm $LOGDIR/flowcheck*.txt rm $LOGDIR/flowsetup_f00.log diff --git a/static_flow/flow_include.sh b/static_flow/flow_include.sh index ea7ded9..f79def1 100755 --- a/static_flow/flow_include.sh +++ b/static_flow/flow_include.sh @@ -183,8 +183,8 @@ function countall { countthem "`grep -a 'rejected: 304 mask=' "$LOGDIR"/post_t_dd2_f00_*.log | wc -l`" totshovel2rej="${tot}" - #countthem "`grep -a after_work\ rejected "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | grep -v DEBUG | wc -l`" - countthem "`grep -a 'rejected: 304 Not modified' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" + #countthem "`grep -a after_work\ rejected "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | grep -v DEBUG | wc -l`" + countthem "`grep -a 'rejected: 304 Not modified' "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" totwinnowed="${tot}" else countthem "`grep -a '\[INFO\] post_log' "$LOGDIR"/sr_post_t_dd2_f00_*.log | wc -l`" @@ -192,7 +192,7 @@ function countall { countthem "`grep -a 'reject: mask=' "$LOGDIR"/sr_post_t_dd2_f00_*.log | wc -l`" totshovel2rej="${tot}" - countthem "`grep -a rejected "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | grep -v DEBUG | wc -l`" + countthem "`grep -a rejected "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | grep -v DEBUG | wc -l`" totwinnowed="${tot}" fi @@ -317,18 +317,18 @@ function countall { totshimpost1=$((${totfileshimpost1}+${totlinkshimpost1}+${totdirshimpost1})) - #countthem "`grep -a '\[INFO\] post_log notice' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" + #countthem "`grep -a '\[INFO\] post_log notice' "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" if [ "${sarra_py_version:0:1}" == "3" ]; then - countthem "`grep -a 'after_accept accepted:' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" + countthem "`grep -a 'after_accept accepted:' "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" totsarx="${tot}" - countthem "`grep -a 'rejected: 304 Not modified' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" + countthem "`grep -a 'rejected: 304 Not modified' "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" totsarx=$((${totsarx}+${tot})) - countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" + countthem "`grep -a 'log after_post posted' "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" totsarp="${tot}" else - countthem "`grep -a '\[INFO\] post_log notice' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" + countthem "`grep -a '\[INFO\] post_log notice' "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" totsarp="${tot}" - countthem "`grep -a 'received' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" + countthem "`grep -a 'received' "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" totsarx="${tot}" fi