Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix rejectPercent maybe other stuff? #1192

Merged
merged 9 commits into from
Sep 3, 2024
Merged

fix rejectPercent maybe other stuff? #1192

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
84c9b57
fix rejectPercent
petersilva Aug 27, 2024
3911683
nicer display of lag
petersilva Aug 27, 2024
eaad903
two unit pretty duration strings
petersilva Aug 27, 2024
1baca47
sr status latestTransfer and messageLast display switched to duration…
petersilva Aug 27, 2024
6947923
remove blank in naturalsize
petersilva Aug 27, 2024
a049bf3
durationToString 2unit values implemented+tests.
petersilva Aug 28, 2024
2860fa4
diagnosed a crash in maintenance test.
petersilva Aug 28, 2024
9d299fc
more fixup for change of message_ttl to messageAgeMax
petersilva Aug 28, 2024
6f53028
fix a crash with sr3 status display
petersilva Aug 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 55 additions & 9 deletions sarracenia/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ def baseUrlParse( url ):
import humanize

def naturalSize( num ):
return humanize.naturalsize(num,binary=True)
return humanize.naturalsize(num,binary=True).replace(" ","")

def naturalTime( dur ):
return humanize.naturaltime(dur)
Expand Down Expand Up @@ -316,7 +316,25 @@ def durationToString(d) -> str:
"""
given a numbner of seconds, return a short, human readable string.
"""
return humanize.naturaldelta(d).replace("minutes","m").replace("seconds","s").replace("hours","h").replace("days","d").replace("an hour","1h").replace("a day","1d").replace("a minute","1m").replace(" ","")
if (d < 60):
return f"{d:7.2f}s"

first_part= humanize.naturaldelta(d).replace("minutes","m").replace("seconds","s").replace("hours","h").replace("days","d").replace("an hour","1h").replace("a day","1d").replace("a minute","1m").replace(" ","")

second_part=""
if first_part[-1] == 'm':
rem=int(d-int(first_part[0:-1])*60)
if rem > 0:
second_part=f"{rem:d}s"
if first_part[-1] == 'h':
rem=int(( d-int(first_part[0:-1])*60*60 ) / 60 )
if rem > 0:
second_part=f"{rem:d}m"
if first_part[-1] == 'd':
rem=int (( d-int(first_part[0:-1])*60*60*24 ) / (60*60) )
if rem > 0:
second_part=f"{rem:d}h"
return first_part+second_part

def durationToSeconds(str_value, default=None) -> float:
"""
Expand All @@ -341,17 +359,45 @@ def durationToSeconds(str_value, default=None) -> float:
if default and str_value.lower() in [ 'on', 'true' ]:
return float(default)

if str_value[-1] in 'sS': factor *= 1
elif str_value[-1] in 'mM': factor *= 60
elif str_value[-1] in 'hH': factor *= 60 * 60
elif str_value[-1] in 'dD': factor *= 60 * 60 * 24
elif str_value[-1] in 'wW': factor *= 60 * 60 * 24 * 7
first_unit=None
second_unit=str_value[-1].lower()
if second_unit in 's':
factor *= 1
first_unit='m'
elif second_unit in 'm':
factor *= 60
first_unit='h'
elif second_unit in 'h':
factor *= 60 * 60
first_unit='d'
elif second_unit in 'd':
factor *= 60 * 60 * 24
first_unit='w'
elif second_unit in 'w':
factor *= 60 * 60 * 24 * 7

if str_value[-1].isalpha(): str_value = str_value[:-1]

if first_unit and first_unit in str_value: # two unit duration.
(big, little) = str_value.split(first_unit)
if big.isnumeric():
big = int(big)
if first_unit == 'm':
big = big*60
elif first_unit == 'h':
big = big*60*60
elif first_unit == 'd':
big = big*60*60*24
elif first_unit == 'w':
big = big*60*60*24*7
str_value = little
else:
big=0

try:
duration = float(str_value) * factor
duration = big + float(str_value) * factor
except:
logger.error("conversion failed for: %s" % str_value)
logger.error( f"conversion failed for: +{str_value}+" )
duration = 0.0

return duration
Expand Down
2 changes: 1 addition & 1 deletion sarracenia/moth/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
'stubborn': True,
'failure_duration': '5m'
},
'message_ttl': 0,
'messageAgeMax': 0,
'topicPrefix': ['v03'],
'tlsRigour': 'normal'
}
Expand Down
2 changes: 1 addition & 1 deletion sarracenia/moth/amqp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -677,7 +677,7 @@ def putNewMessage(self,
else:
exchange = self.o['exchange']

if self.o['message_ttl']:
if self.o['messageAgeMax']:
ttl = "%d" * int(
sarracenia.durationToSeconds(self.o['messageAgeMax']) * 1000)
else:
Expand Down
35 changes: 9 additions & 26 deletions sarracenia/sr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@

from sarracenia.flowcb.v2wrapper import sum_algo_v2tov3

from sarracenia import user_config_dir, user_cache_dir, naturalSize, nowstr, timestr2flt, timeflt2str
from sarracenia import user_config_dir, user_cache_dir, naturalSize, nowstr, timestr2flt, timeflt2str, durationToString
from sarracenia.config import *
import sarracenia.moth
import sarracenia.rabbitmq_admin
Expand All @@ -62,8 +62,8 @@
"transferRxLast": 0, "transferTxLast": 0, "rxLast":0, "txLast":0,
"transferRxBytes":0, "transferRxFiles":0, "transferTxBytes": 0, "transferTxFiles": 0,
"msgs_in_post_retry": 0, "msgs_in_download_retry":0, "brokerQueuedMessageCount": 0,
'time_base': 0, 'byteTotal': 0, 'byteRate': 0, 'msgRate': 0, 'msgRateCpu': 0, 'retry': 0, 'transferLast': 0,
'connectPercent': 0, 'byteConnectPercent': 0
'time_base': 0, 'byteTotal': 0, 'byteRate': 0, 'msgRate': 0, 'msgRateCpu': 0, 'retry': 0,
'messageLast': 0, 'transferLast': 0, 'connectPercent': 0, 'byteConnectPercent': 0
}

sr3_tools_entry_points = [ "sr3_action_convert", "sr3_action_remove", "sr3_commit", "sr3_pull", "sr3_push", "sr3_remove", "sr3_scp", "sr3_ssh", "sr3_utils", "sr3d", "sr3l", "sr3r" ]
Expand Down Expand Up @@ -988,26 +988,9 @@ def _resolve(self):

m['latestTransfer'] = "n/a"
if "transferLast" in m and m['transferLast'] > 0:
v=now - m['transferLast']
if v > 10000:
m['latestTransfer'] = f">9999"
elif v > 100:
m['latestTransfer'] = f"{round(v):4d}s"
elif v > 10:
m['latestTransfer'] = f"{v:4.1f}s"
else:
m['latestTransfer'] = f"{v:4.2f}s"
elif "messageLast" in m:
v=now - m['messageLast']
if v > 10000:
m['latestTransfer'] = f">9999"
elif v > 100:
m['latestTransfer'] = f"{round(v):4d}s"
elif v > 10:
m['latestTransfer'] = f"{v:4.1f}s"
else:
m['latestTransfer'] = f"{v:4.2f}s"

m['latestTransfer'] = durationToString(now - m['transferLast'])
elif "messageLast" in m and m['messageLast'] > 0:
m['latestTransfer'] = durationToString(now - m['messageLast'])

if len(m['latestTransfer']) > self.cumulative_stats['latestTransferWidth']:
self.cumulative_stats['latestTransferWidth'] = len(m['latestTransfer'])
Expand Down Expand Up @@ -1058,7 +1041,7 @@ def _resolve(self):

self.cumulative_stats['txMessageRate'] += (m["txGoodCount"]+m["txBadCount"])/time_base
if m["rxGoodCount"] > 0:
m['rejectPercent'] = ((m['rejectCount']+m['rxBadCount'])/m['rxGoodCount'])*100
m['rejectPercent'] = ((m['rejectCount']+m['rxBadCount'])/(m['rxGoodCount']+m['rxBadCount']))*100
if m['rejectPercent'] > 100:
m['rejectPercent']=100
else:
Expand Down Expand Up @@ -2638,9 +2621,9 @@ def status(self):

if 'metrics' in self.states[c][cfg]:
m=self.states[c][cfg]['metrics']
lfmt = f"%5d %3d%% %3d%% %5d %7.2fs %7.2fs %{latestTransferWidth}s %4.1f%% %8s/s %8s/s %8s/s %8s/s "
lfmt = f"%5d %3d%% %3d%% %5d %8s %8s %{latestTransferWidth}s %4.1f%% %8s/s %8s/s %8s/s %8s/s "
line += lfmt % ( m['retry'], m['connectPercent'], m['byteConnectPercent'], \
m['messagesQueued'], m['lagMax'], m['lagMean'], m['latestTransfer'], m['rejectPercent'],\
m['messagesQueued'], durationToString(m['lagMax']), durationToString(m['lagMean']), m['latestTransfer'], m['rejectPercent'],\
naturalSize(m['byteRate']).replace("Bytes","B"), \
naturalSize(m['msgRate']).replace("B","m").replace("mytes","m"), \
naturalSize(m['transferRxByteRate']).replace("Bytes","B"), \
Expand Down
6 changes: 4 additions & 2 deletions tests/sarracenia/__init___test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,16 @@ def test_durationToSeconds():
assert sarracenia.durationToSeconds('1y') == 1.0
assert sarracenia.durationToSeconds('-1s') == -1.0
assert sarracenia.durationToSeconds('-1.5h') == -5400.0
assert sarracenia.durationToSeconds('2h2m') == 7320
assert sarracenia.durationToSeconds('3m2s') == 182

def test_durationToString():
assert sarracenia.durationToString( 3600 ) == '1h'
assert sarracenia.durationToString( 1800 ) == '30m'
assert sarracenia.durationToString( 600 ) == '10m'
assert sarracenia.durationToString( 6*3600 ) == '6h'
assert sarracenia.durationToString( 6*3600+120 ) == '6h'
assert sarracenia.durationToString( 26*3600+120 ) == '1d'
assert sarracenia.durationToString( 6*3600+120 ) == '6h2m'
assert sarracenia.durationToString( 26*3600+120 ) == '1d2h'

def test_timeValidate():
assert sarracenia.timeValidate('20230710120000') == True
Expand Down
Loading