From bad2c62f0671319f56d07550cc8347949e2204f3 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Tue, 30 Apr 2024 20:13:28 +0000 Subject: [PATCH 1/6] unit test engineering for wmo00 plugin... started --- sarracenia/flowcb/filter/wmo00_accumulate.py | 4 +- tests/sarracenia/flowcb/filter/wmo00_test.py | 154 +++++++++++++++++++ 2 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 tests/sarracenia/flowcb/filter/wmo00_test.py diff --git a/sarracenia/flowcb/filter/wmo00_accumulate.py b/sarracenia/flowcb/filter/wmo00_accumulate.py index 2dc716bd0..500d7650a 100755 --- a/sarracenia/flowcb/filter/wmo00_accumulate.py +++ b/sarracenia/flowcb/filter/wmo00_accumulate.py @@ -88,7 +88,7 @@ def __init__(self,options) : if self.o.batch > 100: logger.warning( f"batch limits how many products fit into one accumulated file.") - logger.warning( f"WMO says this should not exceed 100: batch: {batch} ") + logger.warning( f"WMO says this should not exceed 100: batch: {self.o.batch} ") # FIXME: note for later, assign first digit based on node number in cluster. logger.info( f" hostname: {self.o.hostname} ") @@ -127,7 +127,7 @@ def open_accumulated_file(self): self.sequence += 1 if self.sequence > 999999: - self.sequence == 0 + self.sequence = 0 return open(self.accumulated_file,"wb") diff --git a/tests/sarracenia/flowcb/filter/wmo00_test.py b/tests/sarracenia/flowcb/filter/wmo00_test.py new file mode 100644 index 000000000..8e19b1136 --- /dev/null +++ b/tests/sarracenia/flowcb/filter/wmo00_test.py @@ -0,0 +1,154 @@ +import base64 +import pytest +#from unittest.mock import Mock + +import os +#import urllib.request +import logging +import pprint + +import sarracenia +import sarracenia.config + +import sarracenia.flowcb.filter.wmo00_accumulate +import sarracenia.flowcb.filter.wmo00_split + +import types + +#useful for debugging tests +pretty = pprint.PrettyPrinter(indent=2, width=200).pprint + + +logger = logging.getLogger('sarracenia.config') +logger.setLevel('DEBUG') + + +TEST_WMO00_SampleWMO_individual = b'''\01\r\r\n00237\r\r\nFDCN02 CWAO 282015 AMD\r\r\nFCST BASED ON 281800 DATA VALID 290000 FOR USE 21-06\r\r\n 3000 6000 9000 12000 18000\r\r\nYXS 3124-04 2922-11 2211-17 1827-31=\r\r\nXDD 75N 110W\r\r\n 3512 3520-22 3522-26 3614-30 0611-39=\r\r\n\3''' +TEST_WMO00_SampleWMO_accumulated = b'''00000237\0\0\01\r\r\n00237\r\r\nFDCN02 CWAO 282015 AMD\r\r\nFCST BASED ON 281800 DATA VALID 290000 FOR USE 21-06\r\r\n 3000 6000 9000 12000 18000\r\r\nYXS 3124-04 2922-11 2211-17 1827-31=\r\r\nXDD 75N 110W\r\r\n 3512 3520-22 3522-26 3614-30 0611-39=\r\r\n\3''' + +def make_message(): + m = sarracenia.Message() + m['baseUrl'] = 'file:' + m['relPath'] = '/foo/bar' + m['content'] = { 'encoding' : 'base64', 'value': base64.b64encode(TEST_WMO00_SampleWMO_individual) } + + return m + +def make_worklist(): + # FIXME: open new worklist + worklist = types.SimpleNamespace() + worklist.ok = [] + worklist.incoming = [] + worklist.rejected = [] + worklist.failed = [] + worklist.directories_ok = [] + worklist.poll_catching_up = False + return worklist + +def test___init__(tmp_path): + options = sarracenia.config.default_config() + options.batch=50 + options.no=1 + options.hostname="hoho.mydomain.org" + options.pid_filename= str(tmp_path) + os.sep + "myconfig_01.pid" + options.wmo00_work_directory = str(tmp_path) + options.wmo00_origin_CCCC = 'CYKK' + + accumulator = sarracenia.flowcb.filter.wmo00_accumulate.Wmo00_accumulate(options) + + assert type(accumulator) is sarracenia.flowcb.filter.wmo00_accumulate.Wmo00_accumulate + assert accumulator.sequence_first_digit in range(0,9) + assert accumulator.sequence_second_digit in range(0,9) + assert accumulator.o.wmo00_work_directory == str(tmp_path) + assert accumulator.o.wmo00_origin_CCCC == 'CYKK' + assert accumulator.o.wmo00_type_marker == 'a' + assert accumulator.o.wmo00_encapsulate + assert accumulator.o.wmo00_byteCountMax == 500000 + assert accumulator.thisday in range(1,31) + assert type(accumulator.sequence_file) is str + assert accumulator.sequence == 0 + + options.batch=500 + options.no=14 + yesterday=accumulator.thisday-1 if accumulator.thisday > 1 else 1 + + with open( str(tmp_path) + os.sep + f"sequence_{options.no:02d}.txt", "w" ) as sf: + sf.write( f"{yesterday} 99999" ) + + options.hostname="hoho8.mydomain.org" + + accumulator = sarracenia.flowcb.filter.wmo00_accumulate.Wmo00_accumulate(options) + assert accumulator.sequence_first_digit == 8 + assert accumulator.sequence_second_digit == 4 + assert accumulator.thisday == yesterday + assert accumulator.sequence == 99999 + +def test_open_accumulated_file(tmp_path): + + options = sarracenia.config.default_config() + options.batch=50 + options.no=1 + options.hostname="hoho8.mydomain.org" + options.pid_filename= str(tmp_path) + os.sep + "myconfig_01.pid" + options.wmo00_work_directory = str(tmp_path) + options.wmo00_origin_CCCC = 'CYKK' + + accumulator = sarracenia.flowcb.filter.wmo00_accumulate.Wmo00_accumulate(options) + + af = accumulator.open_accumulated_file() + + #assert str(type(af)) is "" + assert accumulator.accumulated_file == str(tmp_path) + os.sep + 'CYKK81000000.a' + assert os.path.isfile(accumulator.accumulated_file) + assert accumulator.sequence == 1 + + af.close() + + options.no=2 + accumulator = sarracenia.flowcb.filter.wmo00_accumulate.Wmo00_accumulate(options) + + accumulator.sequence=999999 + af = accumulator.open_accumulated_file() + assert os.path.isfile(accumulator.accumulated_file) + assert accumulator.accumulated_file == str(tmp_path) + os.sep + 'CYKK82999999.a' + assert accumulator.sequence == 0 + + +@pytest.mark.depends( on=[ 'test_open_accumulated_file' ]) +def test_after_accept(tmp_path): + + options = sarracenia.config.default_config() + options.batch=50 + options.no=1 + options.hostname="hoho8.mydomain.org" + options.pid_filename= str(tmp_path) + os.sep + "myconfig_01.pid" + options.wmo00_work_directory = str(tmp_path) + options.wmo00_origin_CCCC = 'CYKK' + + options.post_baseUrl = 'file://' + + accumulator = sarracenia.flowcb.filter.wmo00_accumulate.Wmo00_accumulate(options) + + worklist = make_worklist() + m = make_message() + worklist.incoming = [ m ] + + accumulator.after_accept( worklist ) + + assert len(worklist.incoming) == 1 + + output_message = worklist.incoming[0] + + print( f" {output_message=} " ) + + assert output_message['relPath'] == str(tmp_path)[1:] + os.sep + 'CYKK81000000.a' + + fname = os.sep + output_message['relPath'] + assert os.path.isfile( fname ) + + with open(fname, 'rb' ) as af: + afdata = af.read() + + assert afdata == TEST_WMO00_SampleWMO_accumulated + + From 119054c5df28b510310e10d9f22dc6b1be272ae1 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Tue, 30 Apr 2024 21:12:03 +0000 Subject: [PATCH 2/6] wmo00 simplest roundtrip --- sarracenia/flowcb/filter/wmo00_split.py | 1 - tests/sarracenia/flowcb/filter/wmo00_test.py | 19 +++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/sarracenia/flowcb/filter/wmo00_split.py b/sarracenia/flowcb/filter/wmo00_split.py index 935a216ef..1cb7b1ed4 100755 --- a/sarracenia/flowcb/filter/wmo00_split.py +++ b/sarracenia/flowcb/filter/wmo00_split.py @@ -43,7 +43,6 @@ """ -from curses.ascii import SOH,ETX from sarracenia.flowcb import FlowCB import hashlib import logging, random, subprocess, os diff --git a/tests/sarracenia/flowcb/filter/wmo00_test.py b/tests/sarracenia/flowcb/filter/wmo00_test.py index 8e19b1136..9ba27c291 100644 --- a/tests/sarracenia/flowcb/filter/wmo00_test.py +++ b/tests/sarracenia/flowcb/filter/wmo00_test.py @@ -1,11 +1,12 @@ import base64 -import pytest +import hashlib #from unittest.mock import Mock -import os #import urllib.request import logging +import os import pprint +import pytest import sarracenia import sarracenia.config @@ -23,8 +24,8 @@ logger.setLevel('DEBUG') -TEST_WMO00_SampleWMO_individual = b'''\01\r\r\n00237\r\r\nFDCN02 CWAO 282015 AMD\r\r\nFCST BASED ON 281800 DATA VALID 290000 FOR USE 21-06\r\r\n 3000 6000 9000 12000 18000\r\r\nYXS 3124-04 2922-11 2211-17 1827-31=\r\r\nXDD 75N 110W\r\r\n 3512 3520-22 3522-26 3614-30 0611-39=\r\r\n\3''' -TEST_WMO00_SampleWMO_accumulated = b'''00000237\0\0\01\r\r\n00237\r\r\nFDCN02 CWAO 282015 AMD\r\r\nFCST BASED ON 281800 DATA VALID 290000 FOR USE 21-06\r\r\n 3000 6000 9000 12000 18000\r\r\nYXS 3124-04 2922-11 2211-17 1827-31=\r\r\nXDD 75N 110W\r\r\n 3512 3520-22 3522-26 3614-30 0611-39=\r\r\n\3''' +TEST_WMO00_SampleWMO_individual = b'''\01\r\r\n00237\r\r\nFDCN02 CWQQ 282015 AMD\r\r\nFCST BASED ON 281800 DATA VALID 290000 FOR USE 21-06\r\r\n 3000 6000 9000 12000 18000\r\r\nYXS 3124-04 2922-11 2211-17 1827-31=\r\r\nXDD 75N 110W\r\r\n 3512 3520-22 3522-26 3614-30 0611-39=\r\r\n\3''' +TEST_WMO00_SampleWMO_accumulated = b'''00000237\0\0\01\r\r\n00237\r\r\nFDCN02 CWQQ 282015 AMD\r\r\nFCST BASED ON 281800 DATA VALID 290000 FOR USE 21-06\r\r\n 3000 6000 9000 12000 18000\r\r\nYXS 3124-04 2922-11 2211-17 1827-31=\r\r\nXDD 75N 110W\r\r\n 3512 3520-22 3522-26 3614-30 0611-39=\r\r\n\3''' def make_message(): m = sarracenia.Message() @@ -151,4 +152,14 @@ def test_after_accept(tmp_path): assert afdata == TEST_WMO00_SampleWMO_accumulated + splitter = sarracenia.flowcb.filter.wmo00_split.Wmo00_split(options) + + datahash = hashlib.md5(TEST_WMO00_SampleWMO_individual).hexdigest() + print(' {datahash=}' ) + splitter.after_accept( worklist ) + assert datahash == '9e87a9155b446dac46417f548a808c7a' + assert os.path.isdir(str(tmp_path)+ os.sep + 'FD' ) + assert os.path.isdir(str(tmp_path)+ os.sep + 'FD/CWQQ' ) + assert os.path.isdir(str(tmp_path)+ os.sep + 'FD/CWQQ/20' ) + assert os.path.isfile(str(tmp_path)+ os.sep + f'FD/CWQQ/20/FDCN02_CWQQ_282015_AMD_9e87a9155b446dac46417f548a808c7a' ) From 6869ff14ee68353bf11abca2c61b85eaa9dfcce6 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Tue, 30 Apr 2024 21:47:28 +0000 Subject: [PATCH 3/6] range upper bound is excluded --- tests/sarracenia/flowcb/filter/wmo00_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/sarracenia/flowcb/filter/wmo00_test.py b/tests/sarracenia/flowcb/filter/wmo00_test.py index 9ba27c291..bb20f8753 100644 --- a/tests/sarracenia/flowcb/filter/wmo00_test.py +++ b/tests/sarracenia/flowcb/filter/wmo00_test.py @@ -58,14 +58,14 @@ def test___init__(tmp_path): accumulator = sarracenia.flowcb.filter.wmo00_accumulate.Wmo00_accumulate(options) assert type(accumulator) is sarracenia.flowcb.filter.wmo00_accumulate.Wmo00_accumulate - assert accumulator.sequence_first_digit in range(0,9) - assert accumulator.sequence_second_digit in range(0,9) + assert accumulator.sequence_first_digit in range(0,10) + assert accumulator.sequence_second_digit in range(0,10) assert accumulator.o.wmo00_work_directory == str(tmp_path) assert accumulator.o.wmo00_origin_CCCC == 'CYKK' assert accumulator.o.wmo00_type_marker == 'a' assert accumulator.o.wmo00_encapsulate assert accumulator.o.wmo00_byteCountMax == 500000 - assert accumulator.thisday in range(1,31) + assert accumulator.thisday in range(1,32) assert type(accumulator.sequence_file) is str assert accumulator.sequence == 0 From 3cd79551259f9cdd416e2723681bd40b4947e446 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Tue, 30 Apr 2024 21:50:22 +0000 Subject: [PATCH 4/6] yesterday on the first is the last --- tests/sarracenia/flowcb/filter/wmo00_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sarracenia/flowcb/filter/wmo00_test.py b/tests/sarracenia/flowcb/filter/wmo00_test.py index bb20f8753..591bde160 100644 --- a/tests/sarracenia/flowcb/filter/wmo00_test.py +++ b/tests/sarracenia/flowcb/filter/wmo00_test.py @@ -71,7 +71,7 @@ def test___init__(tmp_path): options.batch=500 options.no=14 - yesterday=accumulator.thisday-1 if accumulator.thisday > 1 else 1 + yesterday=accumulator.thisday-1 if accumulator.thisday > 1 else 31 with open( str(tmp_path) + os.sep + f"sequence_{options.no:02d}.txt", "w" ) as sf: sf.write( f"{yesterday} 99999" ) From fafff637de2b48029b327661f49a993cef2d72b5 Mon Sep 17 00:00:00 2001 From: petersilva Date: Tue, 7 May 2024 16:04:32 -0400 Subject: [PATCH 5/6] improve durationToSeconds + test it --- sarracenia/__init__.py | 2 +- tests/sarracenia/__init___test.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/sarracenia/__init__.py b/sarracenia/__init__.py index ceb36c74a..f0e919dbf 100755 --- a/sarracenia/__init__.py +++ b/sarracenia/__init__.py @@ -317,7 +317,7 @@ def durationToString(d) -> str: """ given a numbner of seconds, return a short, human readable string. """ - return humanize.naturaldelta(d).replace("minutes","m").replace("seconds","s") + return humanize.naturaldelta(d).replace("minutes","m").replace("seconds","s").replace("hours","h").replace("days","d").replace("an hour","1h").replace("a day","1d").replace("a minute","1m").replace(" ","") def durationToSeconds(str_value, default=None) -> float: """ diff --git a/tests/sarracenia/__init___test.py b/tests/sarracenia/__init___test.py index 3291dc178..269951647 100644 --- a/tests/sarracenia/__init___test.py +++ b/tests/sarracenia/__init___test.py @@ -53,6 +53,13 @@ def test_durationToSeconds(): assert sarracenia.durationToSeconds('-1s') == -1.0 assert sarracenia.durationToSeconds('-1.5h') == -5400.0 +def test_durationToSeconds(): + assert sarracenia.durationToString( 3600 ) == 'an hour' + assert sarracenia.durationToString( 1800 ) == '30 m' + assert sarracenia.durationToString( 600 ) == '10 m' + assert sarracenia.durationToString( 6*3600 ) == '6 hours' + assert sarracenia.durationToString( 6*3600+120 ) == '6 hours' + def test_timeValidate(): assert sarracenia.timeValidate('20230710120000') == True From cf1856582a52842026109e016458dbd0052cf868 Mon Sep 17 00:00:00 2001 From: petersilva Date: Tue, 7 May 2024 16:43:15 -0400 Subject: [PATCH 6/6] Adding durationToString tests --- tests/sarracenia/__init___test.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/sarracenia/__init___test.py b/tests/sarracenia/__init___test.py index 269951647..7d731f0b3 100644 --- a/tests/sarracenia/__init___test.py +++ b/tests/sarracenia/__init___test.py @@ -53,13 +53,13 @@ def test_durationToSeconds(): assert sarracenia.durationToSeconds('-1s') == -1.0 assert sarracenia.durationToSeconds('-1.5h') == -5400.0 -def test_durationToSeconds(): - assert sarracenia.durationToString( 3600 ) == 'an hour' - assert sarracenia.durationToString( 1800 ) == '30 m' - assert sarracenia.durationToString( 600 ) == '10 m' - assert sarracenia.durationToString( 6*3600 ) == '6 hours' - assert sarracenia.durationToString( 6*3600+120 ) == '6 hours' - +def test_durationToString(): + assert sarracenia.durationToString( 3600 ) == '1h' + assert sarracenia.durationToString( 1800 ) == '30m' + assert sarracenia.durationToString( 600 ) == '10m' + assert sarracenia.durationToString( 6*3600 ) == '6h' + assert sarracenia.durationToString( 6*3600+120 ) == '6h' + assert sarracenia.durationToString( 26*3600+120 ) == '1d' def test_timeValidate(): assert sarracenia.timeValidate('20230710120000') == True