Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

adding configurations with statehost to ensure they are exercised #44

Merged
merged 3 commits into from
Jun 6, 2024
Merged

adding configurations with statehost to ensure they are exercised #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bd64ee4
adding configurations with statehost to ensure they are exercised
petersilva Jun 1, 2024
b92e90e
make watch wait for file to be complete
petersilva May 27, 2024
e25d50f
more cleanup for statehost case
petersilva Jun 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions dynamic_flow/config/watch/f40.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ log_reject on
post_log_format v03
#randomize

inflight 10
caching 15m

heartbeat 60
Expand Down
5 changes: 5 additions & 0 deletions flow_utils.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,15 @@ try:
cachedir = appdirs.user_cache_dir('${SR_DEV_APPNAME}','MetPX')
confdir = appdirs.user_config_dir('${SR_DEV_APPNAME}','MetPX')
logdir = appdirs.user_log_dir('${SR_DEV_APPNAME}','MetPX')
loghostdir = logdir.replace( 'log', "${HOSTNAME}/log" )

except:

import pathlib
cachedir = str(pathlib.Path.home()) + '/.cache/${SR_DEV_APPNAME}'
confdir = str(pathlib.Path.home()) + '/.config/${SR_DEV_APPNAME}'
logdir = str(pathlib.Path.home()) + '/.cache/${SR_DEV_APPNAME}/log'
loghostdir = str(pathlib.Path.home()) + '/.cache/${SR_DEV_APPNAME}/${HOSTNAME}/log'


cachedir = cachedir.replace(' ',r'\ ')
Expand All @@ -66,6 +68,9 @@ print('export CONFDIR=%s'% confdir)
logdir = logdir.replace(' ',r'\ ')
print('export LOGDIR=%s'% logdir)

loghostdir = loghostdir.replace(' ',r'\ ')
print('export LOGHOSTDIR=%s'% loghostdir)

EOF
}

Expand Down
2 changes: 2 additions & 0 deletions static_flow/config/poll/sftp_f62.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ vip 127.0.0.1
set sarracenia.moth.amqp.AMQP.logLevel info
set sarracenia.moth.mqtt.MQTT.logLevel info

logReject on

timezone EST5EDT
sleep 10
ls_file_index 8
Expand Down
2 changes: 2 additions & 0 deletions static_flow/config/sarra/download_f20.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ topic_prefix v03.post

heartbeat 60

statehost on

debug
batch 20
# I want the times the files are written, rather than source times.
Expand Down
9 changes: 8 additions & 1 deletion static_flow/flow_cleanup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,12 +116,19 @@ echo "Removing flow config logs..."
if [ "$1" != "skipconfig" ]; then
if [ "${sarra_py_version:0:1}" == "3" ]; then
echo $flow_configs | sed 's/ / ;\n rm -f /g' | sed '1 s|^| rm -f |' | sed '/^ rm -f post/d' | sed 's+/+_+g' | sed '/conf[ ;]*$/!d' | sed 's/\.conf/_[0-9][0-9].log\*/g' | (cd $LOGDIR; sh )
echo $flow_configs | sed 's/ / ;\n rm -f /g' | sed '1 s|^| rm -f |' | sed 's+/+_+g' | sed '/conf[ ;]*$/!d' | sed 's/\.conf/_[0-9][0-9].json\*/g'| (cd ${LOGDIR}/../metrics; sh )

rm ${LOGHOSTDIR}/sarra_download_f20*.log
else
echo $flow_configs | sed 's/ / ;\n rm -f sr_/g' | sed '1 s|^| rm -f sr_|' | sed '/^ rm -f sr_post/d' | sed 's+/+_+g' | sed '/conf[ ;]*$/!d' | sed 's/\.conf/_[0-9][0-9].log\*/g' | (cd $LOGDIR; sh )
fi
fi

if [ "${sarra_py_version:0:1}" == "3" ]; then
# remove all metrics files
#echo $flow_configs | sed 's/ / ;\n rm -f /g' | sed '1 s|^| rm -f |' | sed 's+/+_+g' | sed '/conf[ ;]*$/!d' | sed 's/\.conf/_[0-9][0-9].json\*/g'| (cd ${LOGDIR}/../metrics; sh )
sr3 show '*/*_f[0-9][0-9]' |& grep metricsFilename | sed "s/'//g" | sed 's/,//' | sed 's/_00.json/_[0-9][0-9].json*/' | awk '{print $2};' | xargs rm -f
fi

rm $LOGDIR/${LGPFX}post_t_dd?_f00_01.log $LOGDIR/${LGPFX}post_shim_f63_01.log $LOGDIR/${LGPFX}post_test2_f61_01.log
rm $LOGDIR/flowcheck*.txt
rm $LOGDIR/flowsetup_f00.log
Expand Down
18 changes: 9 additions & 9 deletions static_flow/flow_include.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,16 +183,16 @@ function countall {
countthem "`grep -a 'rejected: 304 mask=' "$LOGDIR"/post_t_dd2_f00_*.log | wc -l`"
totshovel2rej="${tot}"

#countthem "`grep -a after_work\ rejected "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | grep -v DEBUG | wc -l`"
countthem "`grep -a 'rejected: 304 Not modified' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
#countthem "`grep -a after_work\ rejected "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | grep -v DEBUG | wc -l`"
countthem "`grep -a 'rejected: 304 Not modified' "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
totwinnowed="${tot}"
else
countthem "`grep -a '\[INFO\] post_log' "$LOGDIR"/sr_post_t_dd2_f00_*.log | wc -l`"
totshovel2="${tot}"
countthem "`grep -a 'reject: mask=' "$LOGDIR"/sr_post_t_dd2_f00_*.log | wc -l`"
totshovel2rej="${tot}"

countthem "`grep -a rejected "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | grep -v DEBUG | wc -l`"
countthem "`grep -a rejected "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | grep -v DEBUG | wc -l`"
totwinnowed="${tot}"
fi

Expand Down Expand Up @@ -317,18 +317,18 @@ function countall {

totshimpost1=$((${totfileshimpost1}+${totlinkshimpost1}+${totdirshimpost1}))

#countthem "`grep -a '\[INFO\] post_log notice' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
#countthem "`grep -a '\[INFO\] post_log notice' "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
if [ "${sarra_py_version:0:1}" == "3" ]; then
countthem "`grep -a 'after_accept accepted:' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
countthem "`grep -a 'after_accept accepted:' "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
totsarx="${tot}"
countthem "`grep -a 'rejected: 304 Not modified' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
countthem "`grep -a 'rejected: 304 Not modified' "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
totsarx=$((${totsarx}+${tot}))
countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
countthem "`grep -a 'log after_post posted' "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
totsarp="${tot}"
else
countthem "`grep -a '\[INFO\] post_log notice' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
countthem "`grep -a '\[INFO\] post_log notice' "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
totsarp="${tot}"
countthem "`grep -a 'received' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
countthem "`grep -a 'received' "$LOGHOSTDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`"
totsarx="${tot}"
fi

Expand Down