From f7684126e174685f4e9b54d9273462fc100f3746 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Thu, 6 Jun 2024 23:44:10 -0400 Subject: [PATCH 01/12] modernizing old plugin This plugin was written before directories were in messages, and was assigning directories the .txt suffix... have it not try to put a type suffix on directories. Also, the way file renames are represented is different. updated that code as well. --- sarracenia/flowcb/accept/wmotypesuffix.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/sarracenia/flowcb/accept/wmotypesuffix.py b/sarracenia/flowcb/accept/wmotypesuffix.py index 7a85ab4a8..3bcca1105 100755 --- a/sarracenia/flowcb/accept/wmotypesuffix.py +++ b/sarracenia/flowcb/accept/wmotypesuffix.py @@ -43,6 +43,10 @@ def __find_type(self, TT): def after_accept(self, worklist): for message in worklist.incoming: + + if 'fileOp' in message and 'directory' in message['fileOp']: + continue + type_suffix = self.__find_type(message['new_file'][0:2]) ## FIXME confused as to how this could ever be true since find_type never returns "UNKNOWN" #if type_suffix == 'UNKNOWN': @@ -53,6 +57,10 @@ def after_accept(self, worklist): continue message['new_file'] = message['new_file'] + type_suffix - if 'rename' in message: - message['rename'] = message['rename'] + type_suffix + + if 'fileOp' in message and 'rename' in message['fileOp']: + message['fileOp']['rename'] += type_suffix + # TODO else -> worklist.rejected.append(message) ?? should this be happening at any point? + + From c236c583f0041c794551a394a32f5390f44b1998 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Thu, 6 Jun 2024 23:46:24 -0400 Subject: [PATCH 02/12] adding definitions of new report codes so they are not unknown --- sarracenia/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sarracenia/__init__.py b/sarracenia/__init__.py index 717f0b0be..9ace3aa6c 100755 --- a/sarracenia/__init__.py +++ b/sarracenia/__init__.py @@ -358,8 +358,8 @@ def durationToSeconds(str_value, default=None) -> float: known_report_codes = { - 201: - "Download successful. (variations: Downloaded, Inserted, Published, Copied, or Linked)", + 201: "Download successful. (variations: Downloaded, Inserted, Published, Copied, or Linked)", + 202: "Accepted. mkdir skipped as it already exists", 203: "Non-Authoritative Information: transformed during download.", 205: "Reset Content: truncated. File is shorter than originally expected (changed length during transfer) This only arises during multi-part transfers.", @@ -368,6 +368,7 @@ def durationToSeconds(str_value, default=None) -> float: "Not modified (Checksum validated, unchanged, so no download resulted.)", 307: "Insertion deferred (writing to temporary part file for the moment.)", 417: "Expectation Failed: invalid notification message (corrupt headers)", + 422: "Unprocessable Content: could not determine path to transfer to", 499: "Failure: Not Copied. SFTP/FTP/HTTP download problem", #FIXME : should not have 503 error code 3 times in a row # 503: "Service unavailable. delete (File removal not currently supported.)", From 82da104d667478be0d1f39affc14b6f5b538b5aa Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Thu, 6 Jun 2024 23:47:53 -0400 Subject: [PATCH 03/12] adding documentation for logFormat --- docs/source/Reference/sr3_options.7.rst | 9 +++++++++ docs/source/fr/Reference/sr3_options.7.rst | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/docs/source/Reference/sr3_options.7.rst b/docs/source/Reference/sr3_options.7.rst index 88df2c07c..e08f2c95a 100644 --- a/docs/source/Reference/sr3_options.7.rst +++ b/docs/source/Reference/sr3_options.7.rst @@ -1054,6 +1054,15 @@ other values: on_start, on_stop, post, gather, ... etc... It is comma separated, if the list starts with a plus sign (+) then the selected events are appended to current value. A minus signe (-) can be used to remove events from the set. +LogFormat ( default: %(asctime)s [%(levelname)s] %(name)s %(funcName)s %(message)s ) +------------------------------------------------------------------------------------ + +The *LogFormat* option is passed directly to python logging mechanisms and can be used +to control what is written to log files. The format is documented here: + +* https://docs.python.org/3/library/logging.html#logrecord-attributes + + logLevel ( default: info ) -------------------------- diff --git a/docs/source/fr/Reference/sr3_options.7.rst b/docs/source/fr/Reference/sr3_options.7.rst index 4ba782392..39cb427d8 100644 --- a/docs/source/fr/Reference/sr3_options.7.rst +++ b/docs/source/fr/Reference/sr3_options.7.rst @@ -1037,8 +1037,18 @@ messages de journal. Autres valeurs : on_start, on_stop, post, gather, ... etc.. On peut débuter la valeur avec un plus (+) pour signifier un ajout au valeurs actuels. la valeur moins (-) signifie la soustraction des valeurs de l´ensemble actuel. +LogFormat ( default: %(asctime)s [%(levelname)s] %(name)s %(funcName)s %(message)s ) +------------------------------------------------------------------------------------ + +L'option *LogFormat* est passée directement au mécanismes de contrôle des journalisation +de python. Le format est documenté ici: + +* https://docs.python.org/fr/3/library/logging.html#logrecord-attributes + + logLevel ( défaut: info ) ------------------------- + Niveau de journalisation exprimé par la journalisation de python. Les valeurs possibles sont : critical, error, info, warning, debug. From 5e5214591a72c6f54a5cd677106264ee745842a8 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Fri, 7 Jun 2024 09:31:05 -0400 Subject: [PATCH 04/12] sr3 --full status crash --- sarracenia/sr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sarracenia/sr.py b/sarracenia/sr.py index 302fc0252..32f7dc7c3 100755 --- a/sarracenia/sr.py +++ b/sarracenia/sr.py @@ -2600,7 +2600,7 @@ def status(self): naturalSize(m["transferRxFiles"]).replace("B","F").replace("Fyte","File"), \ naturalSize(m["transferTxBytes"]), \ naturalSize(m["transferTxFiles"]).replace("B","F").replace("Fyte","File"), \ - time_base ) + m["time_base"] ) else: line += " %10s %10s %9s %5s %5s %10s %8s" % ( "-", "-", "-", "-", "-", "-", "-" ) if self.options.displayFull: From 88116ec3e933c07da69b009c54f54cbf9ead2fe1 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Fri, 7 Jun 2024 09:31:27 -0400 Subject: [PATCH 05/12] updating with recent changes --- debian/changelog | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/debian/changelog b/debian/changelog index 5f4c0bd81..8e69976a9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,17 +1,24 @@ metpx-sr3 (3.00.54) UNRELEASED; urgency=medium - * merge PR #1067 (closing #824 ) + * messages reviewed & consolidated: #1094, #1099 (de-cluttering.) + * add http metadata to scheduled flows. #1084 + * only run after_post when actually posted #1101 + * when plugins go bad, report better, recover better: #1085, #1091, + * performance improvements #1083, #1086 + * crashes/problems with statehost #1076, #1087, #1096 + * sr3 status flow state detail improved. * adds lag,rtry,slow,reje states to status display. * adds checks for running process to cleanup, fail if running. * adds cleanup to remove, don't remove if cleanup fails. * adds progressive logs to transfers (closing #966) - * several fixes for sender crashes resulting from changes in try/except scope. + * several fixes for sender crashes resulting from changes in try/except + scope. #1091, #1095, * add #1054 can now convert multiple configs, and overwrite (with --wololo) * fixed #1064 poll crash. * fixed #927 sanity not restarting crashed polls. - * bug fixes and unit tests for AM + * bug fixes and unit tests for AM ( #1036, #1074, #1078, #1079 ) * many unit tests added, improved coverage (though still poor.) - * many other plugin improvements. + * many other improvements and fixes in core and plugins. * search function restored on web-site documentation. -- SSC-5CD2310S60 Fri, 17 May 2024 12:29:22 -0400 From cbcd6ec2f7249f7d09ca28288c76229fd7295d7a Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Fri, 7 Jun 2024 13:46:45 -0400 Subject: [PATCH 06/12] update changelog --- debian/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debian/changelog b/debian/changelog index 8e69976a9..c31e9fd2a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -6,6 +6,9 @@ metpx-sr3 (3.00.54) UNRELEASED; urgency=medium * when plugins go bad, report better, recover better: #1085, #1091, * performance improvements #1083, #1086 * crashes/problems with statehost #1076, #1087, #1096 + * fix #1104 message rate per cpu second and cpuS state. + * fix #1097 better parsing of low fractional rates. + * added logFormat option. * sr3 status flow state detail improved. * adds lag,rtry,slow,reje states to status display. * adds checks for running process to cleanup, fail if running. From 7144f24dfe56a703722eb7d536aa5f28aa572e80 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Fri, 7 Jun 2024 16:45:41 -0400 Subject: [PATCH 07/12] restore old fashioned rename header processing to wmotypesuffice, will make unit tests pass again --- sarracenia/flowcb/accept/wmotypesuffix.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sarracenia/flowcb/accept/wmotypesuffix.py b/sarracenia/flowcb/accept/wmotypesuffix.py index 3bcca1105..be8625f4a 100755 --- a/sarracenia/flowcb/accept/wmotypesuffix.py +++ b/sarracenia/flowcb/accept/wmotypesuffix.py @@ -58,6 +58,9 @@ def after_accept(self, worklist): message['new_file'] = message['new_file'] + type_suffix + if 'rename' in message: + message['rename'] += type_suffix + if 'fileOp' in message and 'rename' in message['fileOp']: message['fileOp']['rename'] += type_suffix From 975bc5fc9afa29123e986fc67a3888478b03aa65 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Sat, 8 Jun 2024 12:11:09 -0400 Subject: [PATCH 08/12] adjust set string parsing to deal with +- ... result of conversion --- sarracenia/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sarracenia/config.py b/sarracenia/config.py index 6d3c636ee..c7bcc8a69 100755 --- a/sarracenia/config.py +++ b/sarracenia/config.py @@ -1042,11 +1042,10 @@ def _parse_set_string( self, v:str, old_value: set ) -> set: if v == 'None': sv=set([]) else: - if v[0] in [ '+', '-']: + op='r' + while v[0] in [ '+', '-']: op=v[0] v=v[1:] - else: - op='r' if ',' in v: sv=set(v.split(',')) @@ -1057,6 +1056,7 @@ def _parse_set_string( self, v:str, old_value: set ) -> set: sv= old_value | sv elif op == '-' : sv= old_value - sv + return sv def add_option(self, option, kind='list', default_value=None, all_values=None ): From 906aa04dcf82699969b4ba17589d9c74690f4372 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Sat, 8 Jun 2024 12:17:07 -0400 Subject: [PATCH 09/12] get convert not to prepend a + to fileEvents if a sign is already present --- sarracenia/sr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sarracenia/sr.py b/sarracenia/sr.py index 32f7dc7c3..c8248d9bf 100755 --- a/sarracenia/sr.py +++ b/sarracenia/sr.py @@ -2805,7 +2805,8 @@ def convert1(self,cfg): if 'none' in line[1].lower(): v=line[1] else: - line[1]= '+' + line[1] + if line[1][0] not in ['+','-']: + line[1]= '+' + line[1] v=line[1] if k == 'continue': From ece3c992009e9833d34de30b17b34b47184580df Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Tue, 11 Jun 2024 11:31:18 -0400 Subject: [PATCH 10/12] adding a note about the existence,importance, and use of unit tests for development --- docs/source/Contribution/Development.rst | 17 +++++++++++++++++ .../fr/Contribution/D\303\251veloppement.rst" | 9 +++++++++ 2 files changed, 26 insertions(+) diff --git a/docs/source/Contribution/Development.rst b/docs/source/Contribution/Development.rst index b648c6aa9..dbcd915a4 100644 --- a/docs/source/Contribution/Development.rst +++ b/docs/source/Contribution/Development.rst @@ -142,6 +142,15 @@ Both v2 and v3 are supported on the stable branch of sr_insects. That branch sh used to support all development in both versions.... +Unit tests +~~~~~~~~~~ + +The tests/ sub-directory contains a woefully incomplete but growing set of unit tests +using the *pytest* framework. These tests are only exercised on Ubuntu 22.04 at the moment. +consult tests/README.md for how to run them. Unit tests passing (or a very good explanation +of why they *temporarily* fail) should be another gate before merging to the main *development* branch. + + Local Installation ------------------ @@ -291,6 +300,14 @@ multipass launch -m 8G bionic ''' can run developer tests as per multipass as described above. +Need to edit: + +* requirements.txt (remove paramiko and watchdog) +* setup.py (remove install_requires for paramiko and watchdog) + +These packages have comedically complex dependencies. Install them separately +and they will be usable by sr3, but listing them as a requirement breaks things. + Python Wheel ~~~~~~~~~~~~ diff --git "a/docs/source/fr/Contribution/D\303\251veloppement.rst" "b/docs/source/fr/Contribution/D\303\251veloppement.rst" index 8293ffdb2..302aff6b7 100644 --- "a/docs/source/fr/Contribution/D\303\251veloppement.rst" +++ "b/docs/source/fr/Contribution/D\303\251veloppement.rst" @@ -127,6 +127,15 @@ Le référentiel sr_insects a sa propre base de données de problèmes, et le tr Les versions 2 et 3 sont prises en charge sur la branche principale de sr_insects. Cette branche devrait être utilisé pour prendre en charge tout le développement dans les deux versions... +Tests unitaires +~~~~~~~~~~ + +Le sous-répertoire tests/ contient un ensemble malheureusement incomplet mais croissant de tests unitaires +qui utilisent le framework *pytest*. Ces tests requiert Ubuntu 22.04 pour le moment. +Consultez tests/README.md pour savoir comment les exécuter. Réussite des tests unitaires (ou une très bonne explication +pourquoi ils échouent *temporairement*) devrait être une autre porte avant de fusionner avec la branche *developpement* principale. + + Installation locale ------------------- From a79d460042c3736060fad7926336647bb2707393 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Wed, 12 Jun 2024 14:00:03 -0400 Subject: [PATCH 11/12] fix for crash re-filtering from retry --- sarracenia/flowcb/retry.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sarracenia/flowcb/retry.py b/sarracenia/flowcb/retry.py index d5086d2e7..8e7dfb669 100755 --- a/sarracenia/flowcb/retry.py +++ b/sarracenia/flowcb/retry.py @@ -81,9 +81,12 @@ def gather(self, qty) -> None: # eliminate calculated values so it is refiltered from scratch. for m in message_list: + del_fields=[] for k in m: if k in m['_deleteOnPost'] or k.startswith('new_'): - del m[k] + del_fields.append(k) + for k in del_fields: + del m[k] m['_isRetry'] = True m['_deleteOnPost'] = set( [ '_isRetry' ] ) From 5b94f5632df39fda10501dab4c975840e15f9053 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Wed, 12 Jun 2024 14:15:43 -0400 Subject: [PATCH 12/12] Revert "fix for crash re-filtering from retry" This reverts commit a79d460042c3736060fad7926336647bb2707393. Reid already has some other fix. --- sarracenia/flowcb/retry.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sarracenia/flowcb/retry.py b/sarracenia/flowcb/retry.py index 8e7dfb669..d5086d2e7 100755 --- a/sarracenia/flowcb/retry.py +++ b/sarracenia/flowcb/retry.py @@ -81,12 +81,9 @@ def gather(self, qty) -> None: # eliminate calculated values so it is refiltered from scratch. for m in message_list: - del_fields=[] for k in m: if k in m['_deleteOnPost'] or k.startswith('new_'): - del_fields.append(k) - for k in del_fields: - del m[k] + del m[k] m['_isRetry'] = True m['_deleteOnPost'] = set( [ '_isRetry' ] )