From 0029b4e978783548fdf500c0d1c1e0eadbead3e8 Mon Sep 17 00:00:00 2001 From: Peter Silva Date: Mon, 29 May 2023 11:24:33 -0400 Subject: [PATCH] V03 issue681 3 (python API breakage.) (#687) * for #681 API change... ugh will explain in issue itself. * updating tutorials for API change (broker -> properties) #681 * Another API change #681 add finalize() as a public entry point. * not sure why it isn't the name of the class * messageCountMax off by one. #681 * adding a little documentation in the config.py #681 * changeing conversion unknown thing from error to warning * credentials.py documenting API ane privating some entry points for clarity. #681 * more documentation of breaking change for #681 * french translation of #681 fixups to API examples, also fixed CLI_with_flowcb that was broken. reviewed and discussed in weekly meeting and agreed. --- docs/source/How2Guides/UPGRADING.rst | 29 ++ .../source/Tutorials/1_CLI_introduction.ipynb | 160 +++++----- .../Tutorials/2_CLI_with_flowcb_demo.ipynb | 276 +++++++++++------- docs/source/Tutorials/3_api_flow_demo.ipynb | 114 +++++--- .../Tutorials/4_api_moth_sub_demo.ipynb | 154 ++++++---- .../Tutorials/5_api_moth_post_demo.ipynb | 23 +- docs/source/fr/CommentFaire/MiseANiveau.rst | 24 ++ .../fr/Tutoriel/2_CLI_with_flowcb_demo.ipynb | 242 ++++++++------- docs/source/fr/Tutoriel/3_api_flow_demo.ipynb | 125 ++++---- .../fr/Tutoriel/4_api_moth_sub_demo.ipynb | 142 +++++---- sarracenia/config.py | 46 ++- sarracenia/credentials.py | 30 +- sarracenia/examples/flow_api_consumer.py | 5 +- sarracenia/examples/moth_api_consumer.py | 4 +- sarracenia/examples/moth_api_producer.py | 2 +- sarracenia/examples/moth_http_retrieval.py | 6 +- sarracenia/flow/__init__.py | 3 +- sarracenia/flowcb/accept/posthourtree.py | 13 +- sarracenia/flowcb/gather/message.py | 4 +- sarracenia/flowcb/post/message.py | 3 +- sarracenia/moth/__init__.py | 57 ++-- sarracenia/moth/amqp.py | 16 +- sarracenia/moth/mqtt.py | 1 + sarracenia/sr.py | 17 +- 24 files changed, 929 insertions(+), 567 deletions(-) diff --git a/docs/source/How2Guides/UPGRADING.rst b/docs/source/How2Guides/UPGRADING.rst index 10d13d968..57055b257 100644 --- a/docs/source/How2Guides/UPGRADING.rst +++ b/docs/source/How2Guides/UPGRADING.rst @@ -39,6 +39,35 @@ Installation Instructions git --- +3.0.40 +------ + +*CHANGE*: Python API breaking changes + + for sarracenia.moth, now specify broker as options['broker'] instead of as + a separate parameter: + + before: + + * Moth(broker: url, options: dict, is_subsubscriber: bool) -> Config + * pubFactory( broker, options ) -> Config + * subFactory( broker, options ) -> Config + + after: + + * Moth( options: dict, is_subscribe: bool) -> Config + * pubFactory( options ) -> Config + * subFactory( options ) -> Config + + sarracenia.config API: + + now should call **sarracenia.config.finalize()** + after having set options and before being used. + This routine reconciles the settings provided and builds + some derived ones. + + + 3.0.37 ------ diff --git a/docs/source/Tutorials/1_CLI_introduction.ipynb b/docs/source/Tutorials/1_CLI_introduction.ipynb index d1b9de220..391352a6b 100644 --- a/docs/source/Tutorials/1_CLI_introduction.ipynb +++ b/docs/source/Tutorials/1_CLI_introduction.ipynb @@ -60,6 +60,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "2023-05-28 15:24:06,526 1914254 [INFO] sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\r\n", "Sample Configurations: (from: /home/peter/Sarracenia/sr3/sarracenia/examples )\r\n", "cpump/cno_trouble_f00.inc flow/amserver.conf \r\n", "flow/poll.inc flow/post.inc \r\n", @@ -116,7 +117,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "add: 2023-01-06 12:55:05,331 1851367 [INFO] sarracenia.sr add copying: /home/peter/Sarracenia/sr3/sarracenia/examples/subscribe/hpfx_amis.conf to /home/peter/.config/sr3/subscribe/hpfx_amis.conf \r\n", + "2023-05-28 15:24:08,749 1914255 [INFO] sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\r\n", + "add: 2023-05-28 15:24:08,750 1914255 [INFO] root add matched existing ['subscribe/hpfx_amis']\r\n", + "2023-05-28 15:24:08,750 1914255 [ERROR] root add nothing specified to add\r\n", "\r\n" ] } @@ -184,57 +187,75 @@ "name": "stdout", "output_type": "stream", "text": [ - "2023-01-06 12:55:44,998 1851452 [INFO] sarracenia.config fill_missing_options overriding batch for consistency with messageCountMax: 10\n", - ".2023-01-06 12:55:45,099 [INFO] 1851453 sarracenia.config fill_missing_options overriding batch for consistency with messageCountMax: 10\n", - "2023-01-06 12:55:45,100 [INFO] 1851453 sarracenia.config fill_missing_options overriding batch for consistency with messageCountMax: 10\n", - "2023-01-06 12:55:45,101 [INFO] 1851453 sarracenia.flow loadCallbacks flowCallback plugins to load: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'log']\n", - "2023-01-06 12:55:45,328 [INFO] 1851453 sarracenia.moth.amqp __getSetup queue declared q_anonymous_subscribe.hpfx_amis.59644358.14662781 (as: amqps://anonymous@hpfx.collab.science.gc.ca/) \n", - "2023-01-06 12:55:45,328 [INFO] 1851453 sarracenia.moth.amqp __getSetup binding q_anonymous_subscribe.hpfx_amis.59644358.14662781 with v02.post.*.WXO-DD.bulletins.alphanumeric.# to xpublic (as: amqps://anonymous@hpfx.collab.science.gc.ca/)\n", - "2023-01-06 12:55:45,368 [INFO] 1851453 sarracenia.flowcb.log __init__ subscribe initialized with: {'on_housekeeping', 'after_work', 'after_accept', 'post', 'after_post'}\n", - "2023-01-06 12:55:45,368 [INFO] 1851453 sarracenia.flow run callbacks loaded: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'log']\n", - "2023-01-06 12:55:45,369 [INFO] 1851453 sarracenia.flow run pid: 1851453 subscribe/hpfx_amis.conf instance: 0\n", - "2023-01-06 12:55:45,384 [INFO] 1851453 sarracenia.flow run now active on vip None\n", - "2023-01-06 12:55:46,443 [INFO] 1851453 sarracenia.flowcb.log after_accept accepted: (lag: 1.82 ) https://hpfx.collab.science.gc.ca /20230106/WXO-DD/bulletins/alphanumeric/20230106/SA/KNKA/17/SAAK41_KNKA_061755___23760 \n", - "2023-01-06 12:55:46,443 [INFO] 1851453 sarracenia.flowcb.log after_accept accepted: (lag: 1.81 ) https://hpfx.collab.science.gc.ca /20230106/WXO-DD/bulletins/alphanumeric/20230106/SA/KNKA/17/SAAK41_KNKA_061755___41902 \n", - "2023-01-06 12:55:46,647 [INFO] 1851453 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SAAK41_KNKA_061755___23760 \n", - "2023-01-06 12:55:46,647 [INFO] 1851453 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SAAK41_KNKA_061755___41902 \n", - "2023-01-06 12:55:50,979 [INFO] 1851453 sarracenia.flowcb.log after_accept accepted: (lag: 3.04 ) https://hpfx.collab.science.gc.ca /20230106/WXO-DD/bulletins/alphanumeric/20230106/SR/KWAL/17/SRCN40_KWAL_061755___7226 \n", - "2023-01-06 12:55:50,979 [INFO] 1851453 sarracenia.flowcb.log after_accept accepted: (lag: 1.23 ) https://hpfx.collab.science.gc.ca /20230106/WXO-DD/bulletins/alphanumeric/20230106/SX/KWAL/17/SXAK50_KWAL_061755___52204 \n", - "2023-01-06 12:55:50,979 [INFO] 1851453 sarracenia.flowcb.log after_accept accepted: (lag: 1.23 ) https://hpfx.collab.science.gc.ca /20230106/WXO-DD/bulletins/alphanumeric/20230106/SR/KWAL/17/SRCN40_KWAL_061755___57048 \n", - "2023-01-06 12:55:50,979 [INFO] 1851453 sarracenia.flowcb.log after_accept accepted: (lag: 9.22 ) https://hpfx.collab.science.gc.ca /20230106/WXO-DD/bulletins/alphanumeric/20230106/SM/KWNB/17/SMVD17_KWNB_061800_RRB__52306 \n", - "2023-01-06 12:55:51,332 [INFO] 1851453 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SRCN40_KWAL_061755___7226 \n", - "2023-01-06 12:55:51,332 [INFO] 1851453 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SXAK50_KWAL_061755___52204 \n", - "2023-01-06 12:55:51,332 [INFO] 1851453 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SRCN40_KWAL_061755___57048 \n", - "2023-01-06 12:55:51,332 [INFO] 1851453 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SMVD17_KWNB_061800_RRB__52306 \n", - "2023-01-06 12:55:51,824 [INFO] 1851453 sarracenia.flowcb.log after_accept accepted: (lag: 1.07 ) https://hpfx.collab.science.gc.ca /20230106/WXO-DD/bulletins/alphanumeric/20230106/SR/KWAL/17/SRCN40_KWAL_061755___29000 \n", - "2023-01-06 12:55:51,824 [INFO] 1851453 sarracenia.flowcb.log after_accept accepted: (lag: 1.07 ) https://hpfx.collab.science.gc.ca /20230106/WXO-DD/bulletins/alphanumeric/20230106/SR/KWAL/17/SRCN40_KWAL_061755___49784 \n", - "2023-01-06 12:55:51,824 [INFO] 1851453 sarracenia.flowcb.log after_accept accepted: (lag: 1.07 ) https://hpfx.collab.science.gc.ca /20230106/WXO-DD/bulletins/alphanumeric/20230106/SR/KWAL/17/SRCN40_KWAL_061755___18710 \n", - "2023-01-06 12:55:52,072 [INFO] 1851453 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SRCN40_KWAL_061755___29000 \n", - "2023-01-06 12:55:52,072 [INFO] 1851453 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SRCN40_KWAL_061755___49784 \n", - "2023-01-06 12:55:52,072 [INFO] 1851453 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SRCN40_KWAL_061755___18710 \n", - "2023-01-06 12:55:52,072 [INFO] 1851453 sarracenia.flow please_stop ok, telling 4 callbacks about it.\n", - "2023-01-06 12:55:52,072 [INFO] 1851453 sarracenia.flow run starting last pass (without gather) through loop for cleanup.\n", - "2023-01-06 12:55:52,072 [INFO] 1851453 sarracenia.flow please_stop ok, telling 4 callbacks about it.\n", - "2023-01-06 12:55:52,072 [INFO] 1851453 sarracenia.flow run on_housekeeping pid: 1851453 subscribe/hpfx_amis.conf instance: 0\n", - "2023-01-06 12:55:52,072 [INFO] 1851453 sarracenia.flowcb.gather.message on_housekeeping messages: good: 9 bad: 0 bytes: 1.2 KiB average: 139 Bytes\n", - "2023-01-06 12:55:52,072 [INFO] 1851453 sarracenia.flowcb.retry on_housekeeping on_housekeeping\n", - "2023-01-06 12:55:52,072 [INFO] 1851453 sarracenia.diskqueue on_housekeeping work_retry_00 on_housekeeping\n", - "2023-01-06 12:55:52,072 [INFO] 1851453 sarracenia.diskqueue on_housekeeping No retry in list\n", - "2023-01-06 12:55:52,072 [INFO] 1851453 sarracenia.diskqueue on_housekeeping on_housekeeping elapse 0.000123\n", - "2023-01-06 12:55:52,072 [INFO] 1851453 sarracenia.diskqueue on_housekeeping post_retry_000 on_housekeeping\n", - "2023-01-06 12:55:52,072 [INFO] 1851453 sarracenia.diskqueue on_housekeeping No retry in list\n", - "2023-01-06 12:55:52,072 [INFO] 1851453 sarracenia.diskqueue on_housekeeping on_housekeeping elapse 0.000092\n", - "2023-01-06 12:55:52,073 [INFO] 1851453 sarracenia.flowcb.housekeeping.resources on_housekeeping Current Memory cpu_times: user=0.13 system=0.02\n", - "2023-01-06 12:55:52,073 [INFO] 1851453 sarracenia.flowcb.housekeeping.resources on_housekeeping Current mem usage: 54.7 MiB, accumulating count (9 or 9/100 so far) before self-setting threshold\n", - "2023-01-06 12:55:52,073 [INFO] 1851453 sarracenia.flowcb.log stats version: 3.00.25, started: 6 seconds ago, last_housekeeping: 6.7 seconds ago \n", - "2023-01-06 12:55:52,073 [INFO] 1851453 sarracenia.flowcb.log stats messages received: 9, accepted: 9, rejected: 0 rate accepted: 100.0% or 1.3 m/s\n", - "2023-01-06 12:55:52,073 [INFO] 1851453 sarracenia.flowcb.log stats files transferred: 9 bytes: 5.7 KiB rate: 863 Bytes/sec\n", - "2023-01-06 12:55:52,073 [INFO] 1851453 sarracenia.flowcb.log stats lag: average: 2.40, maximum: 9.22 \n", - "2023-01-06 12:55:52,073 [INFO] 1851453 sarracenia.flowcb.log on_housekeeping housekeeping\n", - "2023-01-06 12:55:52,073 [INFO] 1851453 sarracenia.flow run clean stop from run loop\n", - "2023-01-06 12:55:52,098 [INFO] 1851453 sarracenia.flowcb.gather.message on_stop closing\n", - "2023-01-06 12:55:52,098 [INFO] 1851453 sarracenia.flow close flow/close completed cleanly pid: 1851453 subscribe/hpfx_amis.conf instance: 0\n", - "\n" + "2023-05-28 15:24:16,821 1914260 [INFO] sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\n", + ".2023-05-28 15:24:16,928 [INFO] 1914261 sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\n", + "2023-05-28 15:24:16,934 [INFO] 1914261 sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\n", + "2023-05-28 15:24:16,934 [INFO] 1914261 sarracenia.flow loadCallbacks flowCallback plugins to load: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'log']\n", + "2023-05-28 15:24:17,111 [INFO] 1914261 sarracenia.moth.amqp __getSetup queue declared q_anonymous_subscribe.hpfx_amis.69643324.61538382 (as: amqps://anonymous@hpfx.collab.science.gc.ca/) \n", + "2023-05-28 15:24:17,111 [INFO] 1914261 sarracenia.moth.amqp __getSetup binding q_anonymous_subscribe.hpfx_amis.69643324.61538382 with v02.post.*.WXO-DD.bulletins.alphanumeric.# to xpublic (as: amqps://anonymous@hpfx.collab.science.gc.ca/)\n", + "2023-05-28 15:24:17,147 [INFO] 1914261 sarracenia.flowcb.log __init__ subscribe initialized with: {'post', 'after_post', 'after_accept', 'after_work', 'on_housekeeping'}\n", + "2023-05-28 15:24:17,147 [INFO] 1914261 sarracenia.flow run callbacks loaded: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'log']\n", + "2023-05-28 15:24:17,147 [INFO] 1914261 sarracenia.flow run pid: 1914261 subscribe/hpfx_amis.conf instance: 0\n", + "2023-05-28 15:24:17,168 [INFO] 1914261 sarracenia.flow run now active on vip None\n", + "2023-05-28 15:24:18,332 [INFO] 1914261 sarracenia.flowcb.log after_accept accepted: (lag: 1.11 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SA/CWAO/19/SAZZ40_CWAO_281900_RRW_EFOU_64642 \n", + "2023-05-28 15:24:18,333 [INFO] 1914261 sarracenia.flowcb.log after_accept accepted: (lag: 1.62 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SX/KWAL/19/SXVX40_KWAL_281923___12624 \n", + "2023-05-28 15:24:18,333 [INFO] 1914261 sarracenia.flowcb.log after_accept accepted: (lag: 1.62 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SN/KWNB/19/SNVD20_KWNB_281900_RRL__24342 \n", + "2023-05-28 15:24:18,333 [INFO] 1914261 sarracenia.flowcb.log after_accept accepted: (lag: 0.61 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SA/KWBC/19/SAUS70_KWBC_281924_RRA__30100 \n", + "2023-05-28 15:24:18,333 [INFO] 1914261 sarracenia.flowcb.log after_accept accepted: (lag: 0.61 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SR/KWAL/19/SRCN40_KWAL_281923___8005 \n", + "2023-05-28 15:24:18,333 [INFO] 1914261 sarracenia.flowcb.log after_accept accepted: (lag: 0.61 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SR/KWAL/19/SRND20_KWAL_281923___24712 \n", + "2023-05-28 15:24:18,333 [INFO] 1914261 sarracenia.flowcb.log after_accept accepted: (lag: 0.60 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/UA/CWAO/19/UANT01_CWAO_281924___00310 \n", + "2023-05-28 15:24:18,333 [INFO] 1914261 sarracenia.flowcb.log after_accept accepted: (lag: 0.20 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SA/KWBC/19/SAUS80_KWBC_281924___64946 \n", + "2023-05-28 15:24:18,949 [INFO] 1914261 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SAZZ40_CWAO_281900_RRW_EFOU_64642 \n", + "2023-05-28 15:24:18,949 [INFO] 1914261 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SXVX40_KWAL_281923___12624 \n", + "2023-05-28 15:24:18,949 [INFO] 1914261 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SNVD20_KWNB_281900_RRL__24342 \n", + "2023-05-28 15:24:18,949 [INFO] 1914261 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SAUS70_KWBC_281924_RRA__30100 \n", + "2023-05-28 15:24:18,949 [INFO] 1914261 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SRCN40_KWAL_281923___8005 \n", + "2023-05-28 15:24:18,949 [INFO] 1914261 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SRND20_KWAL_281923___24712 \n", + "2023-05-28 15:24:18,949 [INFO] 1914261 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/UANT01_CWAO_281924___00310 \n", + "2023-05-28 15:24:18,949 [INFO] 1914261 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SAUS80_KWBC_281924___64946 \n", + "2023-05-28 15:24:19,118 [INFO] 1914261 sarracenia.flowcb.log after_accept accepted: (lag: 0.99 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SX/KWNB/19/SXUS23_KWNB_281900_RRX__36309 \n", + "2023-05-28 15:24:19,119 [INFO] 1914261 sarracenia.flowcb.log after_accept accepted: (lag: 0.99 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SP/KWBC/19/SPUS80_KWBC_281924___40647 \n", + "2023-05-28 15:24:19,119 [INFO] 1914261 sarracenia.flowcb.log after_accept accepted: (lag: 0.99 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SR/KWAL/19/SRCN40_KWAL_281923___38440 \n", + "2023-05-28 15:24:19,119 [INFO] 1914261 sarracenia.flowcb.log after_accept accepted: (lag: 1.64 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SO/KWNB/19/SOVD83_KWNB_281800_RRX__64658 \n", + "2023-05-28 15:24:19,119 [INFO] 1914261 sarracenia.flowcb.log after_accept accepted: (lag: 1.63 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/FZ/KCAR/19/FZUS51_KCAR_281923___54451 \n", + "2023-05-28 15:24:19,119 [INFO] 1914261 sarracenia.flowcb.log after_accept accepted: (lag: 1.62 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SR/KWAL/19/SRCN40_KWAL_281923___2520 \n", + "2023-05-28 15:24:19,119 [INFO] 1914261 sarracenia.flowcb.log after_accept accepted: (lag: 1.61 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SN/KWNB/19/SNVD20_KWNB_281900_RRM__27859 \n", + "2023-05-28 15:24:19,790 [INFO] 1914261 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SXUS23_KWNB_281900_RRX__36309 \n", + "2023-05-28 15:24:19,790 [INFO] 1914261 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SPUS80_KWBC_281924___40647 \n", + "2023-05-28 15:24:19,790 [INFO] 1914261 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SRCN40_KWAL_281923___38440 \n", + "2023-05-28 15:24:19,790 [INFO] 1914261 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SOVD83_KWNB_281800_RRX__64658 \n", + "2023-05-28 15:24:19,790 [INFO] 1914261 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/FZUS51_KCAR_281923___54451 \n", + "2023-05-28 15:24:19,790 [INFO] 1914261 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SRCN40_KWAL_281923___2520 \n", + "2023-05-28 15:24:19,790 [INFO] 1914261 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/SNVD20_KWNB_281900_RRM__27859 \n", + "2023-05-28 15:24:19,790 [INFO] 1914261 sarracenia.flow please_stop ok, telling 4 callbacks about it.\n", + "2023-05-28 15:24:19,790 [INFO] 1914261 sarracenia.flow run starting last pass (without gather) through loop for cleanup.\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.flow please_stop ok, telling 4 callbacks about it.\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.flow run on_housekeeping pid: 1914261 subscribe/hpfx_amis.conf instance: 0\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.flowcb.gather.message on_housekeeping messages: good: 15 bad: 0 bytes: 2.1 KiB average: 140 Bytes\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.flowcb.retry on_housekeeping on_housekeeping\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.diskqueue on_housekeeping work_retry_00 on_housekeeping\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.diskqueue on_housekeeping No retry in list\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.diskqueue on_housekeeping on_housekeeping elapse 0.000105\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.diskqueue on_housekeeping post_retry_000 on_housekeeping\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.diskqueue on_housekeeping No retry in list\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.diskqueue on_housekeeping on_housekeeping elapse 0.000077\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.flowcb.housekeeping.resources on_housekeeping Current Memory cpu_times: user=0.13 system=0.05\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.flowcb.housekeeping.resources on_housekeeping Current mem usage: 54.6 MiB, accumulating count (15 or 15/100 so far) before self-setting threshold\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.flowcb.log stats version: 3.00.40, started: 2 seconds ago, last_housekeeping: 2.6 seconds ago \n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.flowcb.log stats messages received: 15, accepted: 15, rejected: 0 rate accepted: 100.0% or 5.7 m/s\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.flowcb.log stats files transferred: 15 bytes: 12.2 KiB rate: 4.6 KiB/sec\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.flowcb.log stats lag: average: 1.09, maximum: 1.64 \n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.flowcb.log on_housekeeping housekeeping\n", + "2023-05-28 15:24:19,791 [INFO] 1914261 sarracenia.flow run clean stop from run loop\n", + "2023-05-28 15:24:19,816 [INFO] 1914261 sarracenia.flowcb.gather.message on_stop closing\n", + "2023-05-28 15:24:19,816 [INFO] 1914261 sarracenia.flow close flow/close completed cleanly pid: 1914261 subscribe/hpfx_amis.conf instance: 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r\n" ] } ], @@ -263,12 +284,17 @@ "name": "stdout", "output_type": "stream", "text": [ - "2023-01-06 12:56:00,794 1851467 [INFO] sarracenia.config fill_missing_options overriding batch for consistency with messageCountMax: 10\r\n", + "2023-05-28 15:24:25,480 1914273 [INFO] sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\r\n", "status: \r\n", - "Component/Config State Run Miss Exp Retry\r\n", - "---------------- ----- --- ---- --- -----\r\n", - "subscribe/hpfx_amis stopped 0 0 0 0\r\n", - " total running configs: 0 ( processes: 0 missing: 0 stray: 0 )\r\n" + "Component/Config Processes Connection Lag Rates \r\n", + " State Run Retry msg data LagMax LagAvg %rej pubsub messages RxData TxData \r\n", + " ----- --- ----- --- ---- ------ ------ ---- -------- ---- ------ ------ \r\n", + "subscribe/hpfx_amis stop 0/0 0 100% 17% 1.64s 1.09s 0.0% 252 Bytes/s 1 msgs/s 1.5 KiB/s 0 Bytes/s\r\n", + " Total Running Configs: 0 ( Processes: 0 missing: 0 stray: 0 )\r\n", + " Memory: uss:0 Bytes rss:0 Bytes vms:0 Bytes \r\n", + " CPU Time: User:0.00s System:0.00s \r\n", + "\t Pub/Sub Received: 252 msgs/s (252 Bytes/s), Sent: 0 msgs/s (0 Bytes/s)\r\n", + "\t Data Received: 1 Files/s (1.5 KiB/s), Sent: 0 Files/s (0 Bytes/s) \r\n" ] } ], @@ -286,27 +312,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "neural-laugh", "metadata": { - "scrolled": true + "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "2023-01-06 12:56:52,775 1851567 [INFO] sarracenia.config fill_missing_options overriding batch for consistency with messageCountMax: 10\r\n", - "2023-01-06 12:56:41,334 [INFO] sarracenia.flowcb.housekeeping.resources on_housekeeping Current Memory cpu_times: user=0.13 system=0.03\r\n", - "2023-01-06 12:56:41,334 [INFO] sarracenia.flowcb.housekeeping.resources on_housekeeping Current mem usage: 54.6 MiB, accumulating count (10 or 10/100 so far) before self-setting threshold\r\n", - "2023-01-06 12:56:41,334 [INFO] sarracenia.flowcb.log stats version: 3.00.25, started: now, last_housekeeping: 1.0 seconds ago \r\n", - "2023-01-06 12:56:41,334 [INFO] sarracenia.flowcb.log stats messages received: 10, accepted: 10, rejected: 0 rate accepted: 100.0% or 10.2 m/s\r\n", - "2023-01-06 12:56:41,334 [INFO] sarracenia.flowcb.log stats files transferred: 10 bytes: 10.3 KiB rate: 10.6 KiB/sec\r\n", - "2023-01-06 12:56:41,334 [INFO] sarracenia.flowcb.log stats lag: average: 48.08, maximum: 48.76 \r\n", - "2023-01-06 12:56:41,334 [INFO] sarracenia.flowcb.log on_housekeeping housekeeping\r\n", - "2023-01-06 12:56:41,334 [INFO] sarracenia.flow run clean stop from run loop\r\n", - "2023-01-06 12:56:41,368 [INFO] sarracenia.flowcb.gather.message on_stop closing\r\n", - "2023-01-06 12:56:41,368 [INFO] sarracenia.flow close flow/close completed cleanly pid: 1851552 subscribe/hpfx_amis instance: 1\r\n" + "2023-05-28 15:24:34,478 1914274 [INFO] sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\r\n", + "tail: cannot open '/home/peter/.cache/sr3/log/subscribe_hpfx_amis_01.log' for reading: No such file or directory\r\n", + "tail: no files remaining\r\n", + "2023-05-28 15:24:34,479 1914274 [CRITICAL] root run_command subprocess.run failed err=Command '['tail', '-f', '/home/peter/.cache/sr3/log/subscribe_hpfx_amis_01.log']' returned non-zero exit status 1.\r\n", + "\r\n" ] } ], diff --git a/docs/source/Tutorials/2_CLI_with_flowcb_demo.ipynb b/docs/source/Tutorials/2_CLI_with_flowcb_demo.ipynb index d87324829..2eaeac3be 100644 --- a/docs/source/Tutorials/2_CLI_with_flowcb_demo.ipynb +++ b/docs/source/Tutorials/2_CLI_with_flowcb_demo.ipynb @@ -61,10 +61,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-03-19 13:19:27,886 2724830 [INFO] sarracenia.config fill_missing_options overriding batch for consistency with messageCountMax: 10\n", - "2022-03-19 13:19:27,887 2724830 [INFO] root remove removing subscribe/hpfx_amis \n", + "2023-05-28 16:43:43,236 1920436 [INFO] sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\n", + "2023-05-28 16:43:43,237 1920436 [INFO] root remove removing subscribe/hpfx_amis\n", "\n", - "add: 2022-03-19 13:19:28,372 2724832 [INFO] sarracenia.sr add copying: /home/peter/Sarracenia/sr3/sarracenia/examples/subscribe/hpfx_amis.conf to /home/peter/.config/sr3/subscribe/hpfx_amis.conf \n", + "add: 2023-05-28 16:43:44,354 1920437 [INFO] sarracenia.sr add copying: /home/peter/Sarracenia/sr3/sarracenia/examples/subscribe/hpfx_amis.conf to /home/peter/.config/sr3/subscribe/hpfx_amis.conf \n", "\n" ] } @@ -102,33 +102,41 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-03-19 13:19:36,284 2724838 [INFO] sarracenia.config fill_missing_options overriding batch for consistency with messageCountMax: 10\n", - "Provided callback classes: ( /home/peter/Sarracenia/sr3/sarracenia ) \n", - "flowcb/accept/delete.py flowcb/accept/downloadbaseurl.py \n", - "flowcb/accept/hourtree.py flowcb/accept/httptohttps.py \n", - "flowcb/accept/longflow.py flowcb/accept/posthourtree.py \n", - "flowcb/accept/postoverride.py flowcb/accept/printlag.py \n", - "flowcb/accept/rename4jicc.py flowcb/accept/renamedmf.py \n", - "flowcb/accept/renamewhatfn.py flowcb/accept/save.py \n", - "flowcb/accept/speedo.py flowcb/accept/sundewpxroute.py \n", - "flowcb/accept/testretry.py flowcb/accept/toclusters.py \n", - "flowcb/accept/tohttp.py flowcb/accept/tolocal.py \n", - "flowcb/accept/tolocalfile.py flowcb/accept/wmotypesuffix.py \n", - "flowcb/clamav.py flowcb/filter/deleteflowfiles.py \n", - "flowcb/filter/fdelay.py flowcb/filter/pclean_f90.py \n", - "flowcb/filter/pclean_f92.py flowcb/filter/wmo2msc.py \n", - "flowcb/gather/file.py flowcb/gather/message.py \n", - "flowcb/housekeeping/resources.py flowcb/log.py \n", - "flowcb/mdelaylatest.py flowcb/nodupe/data.py \n", - "flowcb/nodupe/name.py flowcb/pclean.py \n", - "flowcb/poll/airnow.py flowcb/poll/mail.py \n", - "flowcb/poll/nasa_mls_nrt.py flowcb/poll/nexrad.py \n", - "flowcb/poll/noaa_hydrometric.py flowcb/poll/usgs.py \n", - "flowcb/post/message.py flowcb/retry.py \n", - "flowcb/sample.py flowcb/script.py \n", - "flowcb/send/email.py flowcb/shiftdir2baseurl.py \n", - "flowcb/v2wrapper.py flowcb/wistree.py \n", - "flowcb/work/delete.py flowcb/work/rxpipe.py \n" + "2023-05-28 16:43:45,730 1920439 [INFO] sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\r\n", + "Provided callback classes: ( /home/peter/Sarracenia/sr3/sarracenia ) \r\n", + "flowcb/accept/dateappend.py flowcb/accept/delete.py \r\n", + "flowcb/accept/downloadbaseurl.py flowcb/accept/hourtree.py \r\n", + "flowcb/accept/httptohttps.py flowcb/accept/longflow.py \r\n", + "flowcb/accept/pathreplace.py flowcb/accept/posthourtree.py \r\n", + "flowcb/accept/postoverride.py flowcb/accept/printlag.py \r\n", + "flowcb/accept/rename4jicc.py flowcb/accept/renamedmf.py \r\n", + "flowcb/accept/renamewhatfn.py flowcb/accept/save.py \r\n", + "flowcb/accept/speedo.py flowcb/accept/sundewpxroute.py \r\n", + "flowcb/accept/testretry.py flowcb/accept/toclusters.py \r\n", + "flowcb/accept/tohttp.py flowcb/accept/tolocal.py \r\n", + "flowcb/accept/tolocalfile.py flowcb/accept/wmotypesuffix.py \r\n", + "flowcb/amserver.py flowcb/clamav.py \r\n", + "flowcb/destfn/replace.py flowcb/destfn/sample.py \r\n", + "flowcb/download/mail_ingest.py flowcb/filter/deleteflowfiles.py \r\n", + "flowcb/filter/fdelay.py flowcb/filter/pclean_f90.py \r\n", + "flowcb/filter/pclean_f92.py flowcb/filter/wmo2msc.py \r\n", + "flowcb/gather/file.py flowcb/gather/message.py \r\n", + "flowcb/housekeeping/resources.py flowcb/log.py \r\n", + "flowcb/mdelaylatest.py flowcb/nodupe/data.py \r\n", + "flowcb/nodupe/name.py flowcb/pclean.py \r\n", + "flowcb/poll/airnow.py flowcb/poll/mail.py \r\n", + "flowcb/poll/nasa_mls_nrt.py flowcb/poll/nexrad.py \r\n", + "flowcb/poll/noaa_hydrometric.py flowcb/poll/s3bucket.py \r\n", + "flowcb/poll/usgs.py flowcb/post/message.py \r\n", + "flowcb/retry.py flowcb/rootchown.py \r\n", + "flowcb/run.py flowcb/rxqueue_gzip.py \r\n", + "flowcb/sample.py flowcb/send/am.py \r\n", + "flowcb/send/email.py flowcb/shiftdir2baseurl.py \r\n", + "flowcb/trace_on_stop.py flowcb/v2wrapper.py \r\n", + "flowcb/wistree.py flowcb/work/age.py \r\n", + "flowcb/work/check.py flowcb/work/citypage_check.py \r\n", + "flowcb/work/delete.py flowcb/work/rxpipe.py \r\n", + "flowcb/work/send_egc_les.py \r\n" ] } ], @@ -152,7 +160,7 @@ "metadata": {}, "outputs": [], "source": [ - "!echo callback wistree >>~/.config/sr3/subscribe/hpfx_amis.conf" + "!echo callback accept.posthourtree >>~/.config/sr3/subscribe/hpfx_amis.conf" ] }, { @@ -167,77 +175,87 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-03-19 13:19:48,073 2724872 [INFO] sarracenia.config fill_missing_options overriding batch for consistency with messageCountMax: 10\n", - ".2022-03-19 13:19:48,285 [INFO] 2724874 sarracenia.config fill_missing_options overriding batch for consistency with messageCountMax: 10\n", - "2022-03-19 13:19:48,287 [INFO] 2724874 sarracenia.config fill_missing_options overriding batch for consistency with messageCountMax: 10\n", - "2022-03-19 13:19:48,287 [INFO] 2724874 sarracenia.flow loadCallbacks plugins to load: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'sarracenia.flowcb.wistree.Wistree', 'sarracenia.flowcb.log.Log']\n", - "2022-03-19 13:19:48,690 [INFO] 2724874 sarracenia.flowcb.log __init__ subscribe initialized with: {'after_work', 'after_accept', 'on_housekeeping'}\n", - "2022-03-19 13:19:48,690 [INFO] 2724874 sarracenia.flow run options:\n", - "_Config__admin=\"...st/ None True True False False None None\",\n", - "_Config__broker=\"...ca/ None True True False False None None\",\n", - "_Config__post_broker=None, accel_threshold=0, acceptSizeWrong=False,\n", - "acceptUnmatched=False, action='foreground', attempts=3, auto_delete=False,\n", - "baseDir=None, baseUrl_relPath=False, batch=10,\n", - "bindings=\"... ['*.WXO-DD.bulletins.alphanumeric.#'])]\", bufsize=1048576,\n", - "bytes_per_second=None, bytes_ps=0,\n", - "cfg_run_dir=\"...me/peter/.cache/sr3/subscribe/hpfx_amis'\",\n", - "component='subscribe', config='hpfx_amis.conf',\n", - "configurations=['subscribe/hpfx_amis.conf'], currentDir=None,\n", - "dangerWillRobinson=False, debug=False, declared_exchanges=[],\n", - "declared_users=\"...': 'source', 'eggmeister': 'subscriber'}\", delete=False,\n", - "destfn_script=None, directory='/tmp/hpfx_amis/', discard=False,\n", - "documentRoot=None, download=True, durable=True,\n", - "env_declared=\"...OKER', 'MQP', 'SFTPUSER', 'TESTDOCROOT']\", exchange='xpublic',\n", - "exchangeDeclare=True, expire=600.0, feeder=amqp://tfeed@localhost,\n", - "fileEvents={'modify', 'link', 'delete', 'create'}, filename='WHATFN',\n", - "fixed_headers={}, flatten='/', hostdir='fractal', hostname='fractal',\n", - "housekeeping=300, imports=[], inflight=None, inline=False,\n", - "inline_encoding='guess', inline_max=4096, inline_only=False, instances=5,\n", - "integrity_arbitrary_value=None, integrity_method='sha512',\n", - "logEvents=\"...ork', 'after_accept', 'on_housekeeping'}\",\n", - "logFormat=\"...me)s] %(name)s %(funcName)s %(message)s'\", logLevel='info',\n", - "logStdout=False, log_flowcb_needed=False, lr_backupCount=5, lr_interval=1,\n", - "lr_when='midnight', masks=\"...pile('.*'), True, False, 0, False, '/')]\",\n", - "messageAgeMax=0, messageCountMax=10, messageDebugDump=False, messageRateMax=0,\n", - "messageRateMin=0,\n", - "message_strategy=\"...ubborn': True, 'failure_duration': '5m'}\", message_ttl=0,\n", - "mirror=False, no=0, nodupe_fileAgeMax=0, nodupe_ttl=0, overwrite=True,\n", - "permCopy=True, permDefault=0, permDirDefault=509, permLog=384,\n", - "pid_filename=\"...e/hpfx_amis//subscribe_hpfx_amis_00.pid'\", plugins_early=[],\n", - "plugins_late=\"...e.Wistree', 'sarracenia.flowcb.log.Log']\", post_baseDir=None,\n", - "post_baseUrl=None, post_documentRoot=None, post_exchanges=[],\n", - "post_topicPrefix=['v02', 'post'], prefetch=25, pstrip=False, queueBind=True,\n", - "queueDeclare=True, queueName=\"...s_subscribe.hpfx_amis.73590194.51440323'\",\n", - "queue_filename=\"...mis/subscribe.hpfx_amis.anonymous.qname'\", randid='4529',\n", - "randomize=False, realpathPost=False, rename=None, report_back=False,\n", - "reset=False, retry_path=\"...hpfx_amis//subscribe_hpfx_amis_00.retry'\",\n", - "retry_ttl=600.0, settings={}, sleep=0.1, statehost=False, strip=0, subtopic=[],\n", - "timeCopy=True, timeout=300, timezone='UTC', tls_rigour='normal',\n", - "topicPrefix=['v02', 'post'], undeclared=[], users=False, v2plugin_options=[],\n", - "v2plugins={}, vhost='/', vip=None\n", - "2022-03-19 13:19:48,691 [INFO] 2724874 sarracenia.flow run callbacks loaded: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'sarracenia.flowcb.wistree.Wistree', 'sarracenia.flowcb.log.Log']\n", - "2022-03-19 13:19:48,691 [INFO] 2724874 sarracenia.flow run pid: 2724874 subscribe/hpfx_amis.conf instance: 0\n", - "2022-03-19 13:19:50,780 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 3.37 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SX/KWAL/17/SXCN40_KWAL_191719___37614 \n", - "2022-03-19 13:19:50,780 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 3.37 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SX/KWAL/17/SXCN40_KWAL_191719___24613 \n", - "2022-03-19 13:19:51,013 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/miscellaneous/ca/SXCN40_KWAL_191719___37614.txt \n", - "2022-03-19 13:19:51,013 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/miscellaneous/ca/SXCN40_KWAL_191719___24613.txt \n", - "2022-03-19 13:19:52,058 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 1.72 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SR/KWAL/17/SRCN40_KWAL_191719___9404 \n", - "2022-03-19 13:19:52,167 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/hydrological/river/ca/SRCN40_KWAL_191719___9404.txt \n", - "2022-03-19 13:19:53,236 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 0.63 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SR/KWAL/17/SRCN40_KWAL_191719___24050 \n", - "2022-03-19 13:19:53,236 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 0.62 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SX/KWAL/17/SXCN40_KWAL_191719___20524 \n", - "2022-03-19 13:19:53,476 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/hydrological/river/ca/SRCN40_KWAL_191719___24050.txt \n", - "2022-03-19 13:19:53,476 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/miscellaneous/ca/SXCN40_KWAL_191719___20524.txt \n", - "2022-03-19 13:19:53,715 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 1.03 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SR/KWAL/17/SRND20_KWAL_191719___45279 \n", - "2022-03-19 13:19:53,839 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/hydrological/river/SRND20_KWAL_191719___45279.txt \n", - "2022-03-19 13:19:54,094 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 0.41 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SR/KWAL/17/SRCN40_KWAL_191719___64176 \n", - "2022-03-19 13:19:54,094 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 0.74 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SR/KWAL/17/SRCN40_KWAL_191719___58640 \n", - "2022-03-19 13:19:54,346 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/hydrological/river/ca/SRCN40_KWAL_191719___64176.txt \n", - "2022-03-19 13:19:54,346 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/hydrological/river/ca/SRCN40_KWAL_191719___58640.txt \n", - "2022-03-19 13:19:54,384 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 1.04 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SX/KWAL/17/SXCN40_KWAL_191719___16476 \n", - "2022-03-19 13:19:54,492 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/miscellaneous/ca/SXCN40_KWAL_191719___16476.txt \n", - "2022-03-19 13:19:54,510 [INFO] 2724874 sarracenia.flowcb.gather.message on_stop closing\n", - "2022-03-19 13:19:54,510 [INFO] 2724874 sarracenia.flow close flow/close completed cleanly pid: 2724874 subscribe/hpfx_amis.conf instance: 0\n", - "\n" + "2023-05-28 16:43:51,115 1920493 [INFO] sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\n", + ".2023-05-28 16:43:51,221 [INFO] 1920494 sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\n", + "2023-05-28 16:43:51,222 [INFO] 1920494 sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\n", + "2023-05-28 16:43:51,222 [INFO] 1920494 sarracenia.flow loadCallbacks flowCallback plugins to load: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'accept.posthourtree', 'log']\n", + "2023-05-28 16:43:51,445 [INFO] 1920494 sarracenia.moth.amqp __getSetup queue declared q_anonymous_subscribe.hpfx_amis.02779585.71440157 (as: amqps://anonymous@hpfx.collab.science.gc.ca/) \n", + "2023-05-28 16:43:51,445 [INFO] 1920494 sarracenia.moth.amqp __getSetup binding q_anonymous_subscribe.hpfx_amis.02779585.71440157 with v02.post.*.WXO-DD.bulletins.alphanumeric.# to xpublic (as: amqps://anonymous@hpfx.collab.science.gc.ca/)\n", + "2023-05-28 16:43:51,478 [INFO] 1920494 sarracenia.flowcb.log __init__ subscribe initialized with: {'after_work', 'after_post', 'after_accept', 'on_housekeeping', 'post'}\n", + "2023-05-28 16:43:51,478 [INFO] 1920494 sarracenia.flow run callbacks loaded: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'accept.posthourtree', 'log']\n", + "2023-05-28 16:43:51,478 [INFO] 1920494 sarracenia.flow run pid: 1920494 subscribe/hpfx_amis.conf instance: 0\n", + "2023-05-28 16:43:51,500 [INFO] 1920494 sarracenia.flow run now active on vip None\n", + "2023-05-28 16:43:59,978 [INFO] 1920494 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:43:59,978 [INFO] 1920494 sarracenia.flowcb.log after_accept accepted: (lag: 5.58 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/UA/CWAO/20/UANT01_CWAO_282043___27297 \n", + "2023-05-28 16:44:00,089 [INFO] 1920494 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/UANT01_CWAO_282043___27297 \n", + "2023-05-28 16:44:15,035 [INFO] 1920494 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:44:15,035 [INFO] 1920494 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:44:15,035 [INFO] 1920494 sarracenia.flowcb.log after_accept accepted: (lag: 3.43 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/UA/CWAO/20/UANT01_CWAO_282044___42712 \n", + "2023-05-28 16:44:15,035 [INFO] 1920494 sarracenia.flowcb.log after_accept accepted: (lag: 3.42 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/UA/CWAO/20/UANT01_CWAO_282044___34692 \n", + "2023-05-28 16:44:15,190 [INFO] 1920494 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/UANT01_CWAO_282044___42712 \n", + "2023-05-28 16:44:15,190 [INFO] 1920494 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/UANT01_CWAO_282044___34692 \n", + "2023-05-28 16:44:19,464 [INFO] 1920494 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:44:19,464 [INFO] 1920494 sarracenia.flowcb.log after_accept accepted: (lag: 0.73 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/UA/CWAO/20/UANT01_CWAO_282044___04359 \n", + "2023-05-28 16:44:19,539 [INFO] 1920494 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/UANT01_CWAO_282044___04359 \n", + "2023-05-28 16:44:28,202 [INFO] 1920494 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:44:28,202 [INFO] 1920494 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:44:28,202 [INFO] 1920494 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:44:28,202 [INFO] 1920494 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:44:28,202 [INFO] 1920494 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:44:28,202 [INFO] 1920494 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:44:28,202 [INFO] 1920494 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:44:28,202 [INFO] 1920494 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:44:28,202 [INFO] 1920494 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:44:28,202 [INFO] 1920494 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:44:28,202 [INFO] 1920494 sarracenia.flowcb.log after_accept accepted: (lag: 4.39 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/UA/CWAO/20/UANT01_CWAO_282044___56279 \n", + "2023-05-28 16:44:28,202 [INFO] 1920494 sarracenia.flowcb.log after_accept accepted: (lag: 4.58 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SN/KWNB/20/SNVD22_KWNB_282000_RRV__45798 \n", + "2023-05-28 16:44:28,202 [INFO] 1920494 sarracenia.flowcb.log after_accept accepted: (lag: 4.58 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SR/KWAL/20/SRCN40_KWAL_282043___8574 \n", + "2023-05-28 16:44:28,203 [INFO] 1920494 sarracenia.flowcb.log after_accept accepted: (lag: 4.58 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SR/KWAL/20/SRCN40_KWAL_282043___52966 \n", + "2023-05-28 16:44:28,203 [INFO] 1920494 sarracenia.flowcb.log after_accept accepted: (lag: 4.58 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SN/KWNB/20/SNVD22_KWNB_282000_RRU__45934 \n", + "2023-05-28 16:44:28,203 [INFO] 1920494 sarracenia.flowcb.log after_accept accepted: (lag: 3.57 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SX/KWAL/20/SXCN40_KWAL_282043___18272 \n", + "2023-05-28 16:44:28,203 [INFO] 1920494 sarracenia.flowcb.log after_accept accepted: (lag: 3.57 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SX/KWAL/20/SXCN40_KWAL_282043___29007 \n", + "2023-05-28 16:44:28,203 [INFO] 1920494 sarracenia.flowcb.log after_accept accepted: (lag: 3.57 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SX/KWAL/20/SXCN40_KWAL_282043___53298 \n", + "2023-05-28 16:44:28,203 [INFO] 1920494 sarracenia.flowcb.log after_accept accepted: (lag: 3.57 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SR/KWAL/20/SRCN40_KWAL_282043___43630 \n", + "2023-05-28 16:44:28,203 [INFO] 1920494 sarracenia.flowcb.log after_accept accepted: (lag: 2.57 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SN/KWNB/20/SNVD22_KWNB_281900_RRX__25714 \n", + "2023-05-28 16:44:28,952 [INFO] 1920494 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/UANT01_CWAO_282044___56279 \n", + "2023-05-28 16:44:28,952 [INFO] 1920494 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SNVD22_KWNB_282000_RRV__45798 \n", + "2023-05-28 16:44:28,952 [INFO] 1920494 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SRCN40_KWAL_282043___8574 \n", + "2023-05-28 16:44:28,952 [INFO] 1920494 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SRCN40_KWAL_282043___52966 \n", + "2023-05-28 16:44:28,952 [INFO] 1920494 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SNVD22_KWNB_282000_RRU__45934 \n", + "2023-05-28 16:44:28,952 [INFO] 1920494 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SXCN40_KWAL_282043___18272 \n", + "2023-05-28 16:44:28,952 [INFO] 1920494 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SXCN40_KWAL_282043___29007 \n", + "2023-05-28 16:44:28,952 [INFO] 1920494 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SXCN40_KWAL_282043___53298 \n", + "2023-05-28 16:44:28,952 [INFO] 1920494 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SRCN40_KWAL_282043___43630 \n", + "2023-05-28 16:44:28,952 [INFO] 1920494 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SNVD22_KWNB_281900_RRX__25714 \n", + "2023-05-28 16:44:28,952 [INFO] 1920494 sarracenia.flow please_stop ok, telling 5 callbacks about it.\n", + "2023-05-28 16:44:28,952 [INFO] 1920494 sarracenia.flow run starting last pass (without gather) through loop for cleanup.\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.flow please_stop ok, telling 5 callbacks about it.\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.flow run on_housekeeping pid: 1920494 subscribe/hpfx_amis.conf instance: 0\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.flowcb.gather.message on_housekeeping messages: good: 14 bad: 0 bytes: 1.9 KiB average: 139 Bytes\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.flowcb.retry on_housekeeping on_housekeeping\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.diskqueue on_housekeeping work_retry_00 on_housekeeping\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.diskqueue on_housekeeping No retry in list\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.diskqueue on_housekeeping on_housekeeping elapse 0.000109\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.diskqueue on_housekeeping post_retry_000 on_housekeeping\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.diskqueue on_housekeeping No retry in list\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.diskqueue on_housekeeping on_housekeeping elapse 0.000082\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.flowcb.housekeeping.resources on_housekeeping Current Memory cpu_times: user=0.17 system=0.02\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.flowcb.housekeeping.resources on_housekeeping Current mem usage: 54.7 MiB, accumulating count (14 or 14/100 so far) before self-setting threshold\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.flowcb.log stats version: 3.00.40, started: 37 seconds ago, last_housekeeping: 37.5 seconds ago \n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.flowcb.log stats messages received: 14, accepted: 14, rejected: 0 rate accepted: 100.0% or 0.4 m/s\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.flowcb.log stats files transferred: 14 bytes: 3.3 KiB rate: 89 Bytes/sec\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.flowcb.log stats lag: average: 3.77, maximum: 5.58 \n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.flowcb.log on_housekeeping housekeeping\n", + "2023-05-28 16:44:28,953 [INFO] 1920494 sarracenia.flow run clean stop from run loop\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-05-28 16:44:28,973 [INFO] 1920494 sarracenia.flowcb.gather.message on_stop closing\r\n", + "2023-05-28 16:44:28,973 [INFO] 1920494 sarracenia.flow close flow/close completed cleanly pid: 1920494 subscribe/hpfx_amis.conf instance: 0\r\n", + "\r\n" ] } ], @@ -419,7 +437,47 @@ "\n", "Each message in the log will be prefixed with the class and routine\n", "emitting the log message, as well as the date/time.\n", - "\n" + "\n", + "## Sample Flow Callback Class\n", + "\n", + "With the above information about option handling, worklists, and logging, we\n", + "are ready to understand the wistree module we just used. As a very simple example,\n", + "here is the source code of the callback used above is given below:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27954f45", + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"\n", + "Plugin posthourtree.py:\n", + " When posting a file, insert an hourly directory into the delivery path hierarchy.\n", + "\n", + "Example:\n", + " input A/B/c.gif --> output A/B//c.gif\n", + "\n", + "Usage:\n", + " callback accept.posthourtree\n", + "\n", + "\"\"\"\n", + "import logging\n", + "import sys, os, os.path, time, stat\n", + "from sarracenia.flowcb import FlowCB\n", + "\n", + "logger = logging.getLogger(__name__)\n", + "\n", + "\n", + "class Posthourtree(FlowCB):\n", + "\n", + " def after_accept(self, worklist):\n", + " for message in worklist.incoming:\n", + " datestr = time.strftime('%H', time.gmtime()) # pick the hour\n", + " # insert the hour into the rename header of the message to be posted.\n", + " message['new_dir'] += '/' + datestr \n", + " logger.info( f\"post_hour_tree: new_dir: {message['new_dir']}\" )" ] }, { @@ -429,11 +487,9 @@ "source": [ "## Sample Flowcb Sub-Class\n", "\n", - "With the above information about option handling, worklists, and logging, we\n", - "are ready to understand the wistree module we just used.\n", - "This wistree.py class accepts files whose\n", - "names begin with AHL's, and renames the directory tree to a different standard,\n", - "the evolving one for the WMO WIS 2.0 (for more information on that module:\n", + "\n", + "This wistree.py class, shows more aspects of the callback API, with an __init__.py as well as bringing in an externam python module, as well as adding fields to the messages.\n", + "The Wistree class accepts files whose names begin with AHL's (World Meteorological Organization Abbreviated Header Lines for meteorological products), and renames the directory tree to a different standard, the evolving one for the WMO WIS 2.0 (for more information on that module:\n", "https://github.com/wmo-im/GTStoWIS2)" ] }, @@ -636,7 +692,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -650,7 +706,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.10.6" } }, "nbformat": 4, diff --git a/docs/source/Tutorials/3_api_flow_demo.ipynb b/docs/source/Tutorials/3_api_flow_demo.ipynb index 90aeb9afc..6b8162160 100644 --- a/docs/source/Tutorials/3_api_flow_demo.ipynb +++ b/docs/source/Tutorials/3_api_flow_demo.ipynb @@ -21,7 +21,15 @@ "execution_count": 1, "id": "weekly-terminology", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "mkdir: cannot create directory ‘/tmp/flow_demo’: File exists\r\n" + ] + } + ], "source": [ "!mkdir /tmp/flow_demo" ] @@ -63,8 +71,9 @@ "# set the instance number for the flow class.\n", "cfg.no=0\n", "\n", - "# set flow class to put working files in ~/.cache/sr3/subscrribe/flow_demo directory.\n", - "cfg.pid_filename = sarracenia.config.get_pid_filename( None, cfg.component, cfg.config, 0)\n", + "# set other settings based on provided ones, so it is ready for use.\n", + "\n", + "cfg.finalize()\n", "\n", "# accept/reject patterns:\n", "pattern=\".*\"\n", @@ -136,53 +145,68 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-01-06 12:58:25,789 [INFO] sarracenia.flow loadCallbacks flowCallback plugins to load: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources']\n", - "2023-01-06 12:58:25,896 [DEBUG] amqp _on_start Start from server, version: 0.9, properties: {'capabilities': {'publisher_confirms': True, 'exchange_exchange_bindings': True, 'basic.nack': True, 'consumer_cancel_notify': True, 'connection.blocked': True, 'consumer_priorities': True, 'authentication_failure_close': True, 'per_consumer_qos': True, 'direct_reply_to': True}, 'cluster_name': 'rabbit@hpfx2.collab.science.gc.ca', 'copyright': 'Copyright (C) 2007-2019 Pivotal Software, Inc.', 'information': 'Licensed under the MPL. See http://www.rabbitmq.com/', 'platform': 'Erlang/OTP 21.3', 'product': 'RabbitMQ', 'version': '3.7.13'}, mechanisms: [b'AMQPLAIN', b'PLAIN'], locales: ['en_US']\n", - "2023-01-06 12:58:25,934 [DEBUG] amqp __init__ using channel_id: 1\n", - "2023-01-06 12:58:25,952 [DEBUG] amqp _on_open_ok Channel open\n", - "2023-01-06 12:58:26,008 [INFO] sarracenia.moth.amqp __getSetup queue declared q_anonymous.subscriber_test2 (as: amqps://anonymous@hpfx.collab.science.gc.ca) \n", - "2023-01-06 12:58:26,008 [INFO] sarracenia.moth.amqp __getSetup binding q_anonymous.subscriber_test2 with v02.post.*.WXO-DD.observations.swob-ml.# to xpublic (as: amqps://anonymous@hpfx.collab.science.gc.ca)\n", - "2023-01-06 12:58:26,029 [DEBUG] sarracenia.moth.amqp __getSetup getSetup ... Done!\n", - "2023-01-06 12:58:26,033 [DEBUG] sarracenia.flowcb.retry __init__ sr_retry __init__\n", - "2023-01-06 12:58:26,034 [DEBUG] sarracenia.diskqueue __init__ work_retry_00 __init__\n", - "2023-01-06 12:58:26,035 [DEBUG] sarracenia.config add_option MemoryMax declared as type: value:0\n", - "2023-01-06 12:58:26,036 [DEBUG] sarracenia.config add_option MemoryBaseLineFile declared as type: value:100\n", - "2023-01-06 12:58:26,036 [DEBUG] sarracenia.config add_option MemoryMultiplier declared as type: value:3\n", - "2023-01-06 12:58:26,036 [DEBUG] sarracenia.config check_undeclared_options missing defaults: {'nodupe_basis', 'post_exchangeSuffix', 'MemoryBaseLineFile', 'header', 'integrity', 'inplace', 'blocksize', 'retry_mode', 'retry_ttl', 'reconnect', 'cluster', 'set_passwords', 'MemoryMax', 'post_on_start', 'pump_flag', 'notify_only', 'force_polling', 'post_exchange', 'save', 'restore', 'exchangeSplit', 'pollUrl', 'MemoryMultiplier', 'sendTo', 'exchange_suffix', 'sanity_log_dead', 'post_exchangeSplit', 'report_exchange', 'cache_stat', 'realpathFilter', 'follow_symlinks', 'feeder', 'source'}\n", - "2023-01-06 12:58:26,036 [INFO] sarracenia.flow run callbacks loaded: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources']\n", - "2023-01-06 12:58:26,037 [INFO] sarracenia.flow run pid: 1851720 subscribe/flow_demo instance: 0\n", - "2023-01-06 12:58:26,051 [INFO] sarracenia.flow run now active on vip None\n", - "2023-01-06 12:58:27,071 [DEBUG] sarracenia.moth.amqp getNewMessage new msg: {'sundew_extension': 'DMS:WXO_RENAMED_SWOB2:MSC:XML::20230106175824', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'DDSR.CMC,DDI.CMC,CMC,SCIENCE,EDM', 'filename': 'msg_ddsr-WXO-DD_2f05f6181d347aa3150ad05c0ba520df:DMS:WXO_RENAMED_SWOB2:MSC:XML::20230106175824', 'source': 'WXO-DD', 'mtime': '20230106T175825.880', 'atime': '20230106T175825.880', 'subtopic': ['20230106', 'WXO-DD', 'observations', 'swob-ml', '20230106', 'CXBL'], '_deleteOnPost': {'ack_id', 'subtopic', 'version', 'exchange', 'local_offset'}, 'pubTime': '20230106T175825.880', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230106/WXO-DD/observations/swob-ml/20230106/CXBL/2023-01-06-1757-CXBL-AUTO-minute-swob.xml', 'integrity': {'method': 'md5', 'value': 'WGdF41ExchcZQKS3aInj0Q=='}, 'size': 9131, 'version': 'v02', 'exchange': 'xpublic', 'ack_id': 1, 'local_offset': 0}\n", - "2023-01-06 12:58:27,072 [DEBUG] sarracenia.config add_option accelWgetCommand declared as type: value:/usr/bin/wget %s -o - -O %d\n", - "2023-01-06 12:58:27,184 [DEBUG] sarracenia.moth.amqp getNewMessage new msg: {'sundew_extension': 'DMS:WXO_RENAMED_SWOB2:MSC:XML::20230106175824', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'DDSR.CMC,DDI.CMC,CMC,SCIENCE,EDM', 'filename': 'msg_ddsr-WXO-DD_43dca9c7707182800deee1da98027bdf:DMS:WXO_RENAMED_SWOB2:MSC:XML::20230106175824', 'source': 'WXO-DD', 'mtime': '20230106T175825.881', 'atime': '20230106T175825.881', 'subtopic': ['20230106', 'WXO-DD', 'observations', 'swob-ml', '20230106', 'CWQH'], '_deleteOnPost': {'ack_id', 'subtopic', 'version', 'exchange', 'local_offset'}, 'pubTime': '20230106T175825.881', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230106/WXO-DD/observations/swob-ml/20230106/CWQH/2023-01-06-1757-CWQH-AUTO-minute-swob.xml', 'integrity': {'method': 'md5', 'value': 'K7GS7uFCCU+Mo8Woxzv37g=='}, 'size': 9408, 'version': 'v02', 'exchange': 'xpublic', 'ack_id': 2, 'local_offset': 0}\n", - "2023-01-06 12:58:27,305 [DEBUG] sarracenia.moth.amqp getNewMessage new msg: {'sundew_extension': 'DMS:WXO_RENAMED_SWOB2:MSC:XML::20230106175824', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'DDSR.CMC,DDI.CMC,CMC,SCIENCE,EDM', 'filename': 'msg_ddsr-WXO-DD_5425482ce3d2e8b33f8da04187984c91:DMS:WXO_RENAMED_SWOB2:MSC:XML::20230106175824', 'source': 'WXO-DD', 'mtime': '20230106T175825.882', 'atime': '20230106T175825.882', 'subtopic': ['20230106', 'WXO-DD', 'observations', 'swob-ml', '20230106', 'CXAR'], '_deleteOnPost': {'ack_id', 'subtopic', 'version', 'exchange', 'local_offset'}, 'pubTime': '20230106T175825.882', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230106/WXO-DD/observations/swob-ml/20230106/CXAR/2023-01-06-1757-CXAR-AUTO-minute-swob.xml', 'integrity': {'method': 'md5', 'value': 'FmeON9WCu94yU9w3zi7tRg=='}, 'size': 9449, 'version': 'v02', 'exchange': 'xpublic', 'ack_id': 3, 'local_offset': 0}\n", - "2023-01-06 12:58:27,426 [DEBUG] sarracenia.moth.amqp getNewMessage new msg: {'sundew_extension': 'DMS:WXO_RENAMED_SWOB2:MSC:XML::20230106175824', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'DDSR.CMC,DDI.CMC,CMC,SCIENCE,EDM', 'filename': 'msg_ddsr-WXO-DD_6724d6cb6454f417eed3c81f285a4dc5:DMS:WXO_RENAMED_SWOB2:MSC:XML::20230106175824', 'source': 'WXO-DD', 'mtime': '20230106T175825.882', 'atime': '20230106T175825.882', 'subtopic': ['20230106', 'WXO-DD', 'observations', 'swob-ml', '20230106', 'CWOY'], '_deleteOnPost': {'ack_id', 'subtopic', 'version', 'exchange', 'local_offset'}, 'pubTime': '20230106T175825.882', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230106/WXO-DD/observations/swob-ml/20230106/CWOY/2023-01-06-1757-CWOY-AUTO-minute-swob.xml', 'integrity': {'method': 'md5', 'value': 'E+GXZzZKQI+qzqGWcVwnRA=='}, 'size': 9355, 'version': 'v02', 'exchange': 'xpublic', 'ack_id': 4, 'local_offset': 0}\n", - "2023-01-06 12:58:27,513 [INFO] sarracenia.flow please_stop ok, telling 3 callbacks about it.\n", - "2023-01-06 12:58:27,514 [INFO] sarracenia.flow run starting last pass (without gather) through loop for cleanup.\n", - "2023-01-06 12:58:27,514 [INFO] sarracenia.flow please_stop ok, telling 3 callbacks about it.\n", - "2023-01-06 12:58:27,514 [INFO] sarracenia.flow run on_housekeeping pid: 1851720 subscribe/flow_demo instance: 0\n", - "2023-01-06 12:58:27,515 [INFO] sarracenia.flowcb.gather.message on_housekeeping messages: good: 4 bad: 0 bytes: 584 Bytes average: 146 Bytes\n", - "2023-01-06 12:58:27,515 [INFO] sarracenia.flowcb.retry on_housekeeping on_housekeeping\n", - "2023-01-06 12:58:27,515 [INFO] sarracenia.diskqueue on_housekeeping work_retry_00 on_housekeeping\n", - "2023-01-06 12:58:27,516 [ERROR] sarracenia.diskqueue on_housekeeping something went wrong\n", - "2023-01-06 12:58:27,516 [INFO] sarracenia.diskqueue on_housekeeping No retry in list\n", - "2023-01-06 12:58:27,516 [INFO] sarracenia.diskqueue on_housekeeping on_housekeeping elapse 0.000601\n", - "2023-01-06 12:58:27,517 [INFO] sarracenia.diskqueue on_housekeeping post_retry_000 on_housekeeping\n", - "2023-01-06 12:58:27,517 [ERROR] sarracenia.diskqueue on_housekeeping something went wrong\n", - "2023-01-06 12:58:27,517 [INFO] sarracenia.diskqueue on_housekeeping No retry in list\n", - "2023-01-06 12:58:27,518 [INFO] sarracenia.diskqueue on_housekeeping on_housekeeping elapse 0.000580\n" + "2023-05-27 12:03:19,949 [INFO] sarracenia.flow loadCallbacks flowCallback plugins to load: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'log']\n", + "2023-05-27 12:03:20,031 [DEBUG] amqp _on_start Start from server, version: 0.9, properties: {'capabilities': {'publisher_confirms': True, 'exchange_exchange_bindings': True, 'basic.nack': True, 'consumer_cancel_notify': True, 'connection.blocked': True, 'consumer_priorities': True, 'authentication_failure_close': True, 'per_consumer_qos': True, 'direct_reply_to': True}, 'cluster_name': 'rabbit@hpfx2.collab.science.gc.ca', 'copyright': 'Copyright (c) 2007-2022 VMware, Inc. or its affiliates.', 'information': 'Licensed under the MPL 2.0. Website: https://rabbitmq.com', 'platform': 'Erlang/OTP 24.2.1', 'product': 'RabbitMQ', 'version': '3.9.13'}, mechanisms: [b'AMQPLAIN', b'PLAIN'], locales: ['en_US']\n", + "2023-05-27 12:03:20,066 [DEBUG] amqp __init__ using channel_id: 1\n", + "2023-05-27 12:03:20,084 [DEBUG] amqp _on_open_ok Channel open\n", + "2023-05-27 12:03:20,126 [INFO] sarracenia.moth.amqp __getSetup queue declared q_anonymous.subscriber_test2 (as: amqps://anonymous@hpfx.collab.science.gc.ca) \n", + "2023-05-27 12:03:20,126 [INFO] sarracenia.moth.amqp __getSetup binding q_anonymous.subscriber_test2 with v02.post.*.WXO-DD.observations.swob-ml.# to xpublic (as: amqps://anonymous@hpfx.collab.science.gc.ca)\n", + "2023-05-27 12:03:20,148 [DEBUG] sarracenia.moth.amqp __getSetup getSetup ... Done!\n", + "2023-05-27 12:03:20,149 [DEBUG] sarracenia.flowcb.retry __init__ sr_retry __init__\n", + "2023-05-27 12:03:20,149 [DEBUG] sarracenia.config add_option retry_driver declared as type: value:disk\n", + "2023-05-27 12:03:20,179 [DEBUG] sarracenia.diskqueue __init__ work_retry_00 __init__\n", + "2023-05-27 12:03:20,180 [DEBUG] sarracenia.config add_option MemoryMax declared as type: value:0\n", + "2023-05-27 12:03:20,180 [DEBUG] sarracenia.config add_option MemoryBaseLineFile declared as type: value:100\n", + "2023-05-27 12:03:20,180 [DEBUG] sarracenia.config add_option MemoryMultiplier declared as type: value:3\n", + "2023-05-27 12:03:20,181 [DEBUG] sarracenia.config add_option logEvents declared as type: value:{'after_work', 'after_accept', 'on_housekeeping'}\n", + "2023-05-27 12:03:20,181 [DEBUG] sarracenia.config add_option logMessageDump declared as type: value:False\n", + "2023-05-27 12:03:20,181 [INFO] sarracenia.flowcb.log __init__ subscribe initialized with: {'after_work', 'after_accept', 'on_housekeeping'}\n", + "2023-05-27 12:03:20,182 [DEBUG] sarracenia.config check_undeclared_options missing defaults: {'reconnect', 'MemoryMultiplier', 'blocksize', 'cluster', 'source', 'post_exchange', 'post_exchangeSplit', 'action', 'realpathFilter', 'follow_symlinks', 'inplace', 'sendTo', 'force_polling', 'notify_only', 'pollUrl', 'exchange_suffix', 'retry_driver', 'MemoryBaseLineFile', 'count', 'MemoryMax', 'header', 'post_exchangeSuffix', 'nodupe_basis', 'restore', 'logMessageDump', 'report_exchange', 'feeder', 'post_on_start', 'save', 'integrity', 'exchangeSplit'}\n", + "2023-05-27 12:03:20,182 [INFO] sarracenia.flow run callbacks loaded: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'log']\n", + "2023-05-27 12:03:20,182 [INFO] sarracenia.flow run pid: 1682497 subscribe/flow_demo instance: 0\n", + "2023-05-27 12:03:20,199 [DEBUG] sarracenia.moth.amqp getNewMessage new msg: {'_format': 'v02', '_deleteOnPost': {'local_offset', '_format', 'exchange', 'ack_id', 'subtopic'}, 'sundew_extension': 'DMS:WXO_RENAMED_SWOB2:MSC:XML::20230527153757', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_553d269d71c2c2d52a9b4968e509fef5:DMS:WXO_RENAMED_SWOB2:MSC:XML::20230527153757', 'source': 'WXO-DD', 'mtime': '20230527T153759.475', 'atime': '20230527T153759.475', 'pubTime': '20230527T153759.475', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230527/WXO-DD/observations/swob-ml/20230527/CTNK/2023-05-27-1537-CTNK-AUTO-minute-swob.xml', 'subtopic': ['20230527', 'WXO-DD', 'observations', 'swob-ml', '20230527', 'CTNK'], 'integrity': {'method': 'md5', 'value': 'YkkEEWfkyty+UOIP7fEUag=='}, 'size': 9597, 'exchange': 'xpublic', 'ack_id': 1, 'local_offset': 0}\n", + "2023-05-27 12:03:20,199 [INFO] sarracenia.flowcb.log after_accept accepted: (lag: 1520.72 ) https://hpfx.collab.science.gc.ca /20230527/WXO-DD/observations/swob-ml/20230527/CTNK/2023-05-27-1537-CTNK-AUTO-minute-swob.xml \n", + "2023-05-27 12:03:20,199 [INFO] sarracenia.flow run now active on vip None\n", + "2023-05-27 12:03:20,200 [DEBUG] sarracenia.config add_option accelWgetCommand declared as type: value:/usr/bin/wget %s -o - -O %d\n", + "2023-05-27 12:03:20,277 [INFO] sarracenia.flowcb.log after_work downloaded ok: /tmp/flow_demo/2023-05-27-1537-CTNK-AUTO-minute-swob.xml \n", + "2023-05-27 12:03:20,304 [DEBUG] sarracenia.moth.amqp getNewMessage new msg: {'_format': 'v02', '_deleteOnPost': {'local_offset', '_format', 'exchange', 'ack_id', 'subtopic'}, 'sundew_extension': 'DMS:WXO_RENAMED_SWOB2:MSC:XML::20230527153757', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_4e7603f7cb74d0193e5456a61042fca5:DMS:WXO_RENAMED_SWOB2:MSC:XML::20230527153757', 'source': 'WXO-DD', 'mtime': '20230527T153759.474', 'atime': '20230527T153759.474', 'pubTime': '20230527T153759.474', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230527/WXO-DD/observations/swob-ml/20230527/CWST/2023-05-27-1537-CWST-AUTO-minute-swob.xml', 'subtopic': ['20230527', 'WXO-DD', 'observations', 'swob-ml', '20230527', 'CWST'], 'integrity': {'method': 'md5', 'value': 'kaitX64I5bxPNIpELUQ7/A=='}, 'size': 9824, 'exchange': 'xpublic', 'ack_id': 2, 'local_offset': 0}\n", + "2023-05-27 12:03:20,305 [INFO] sarracenia.flowcb.log after_accept accepted: (lag: 1520.83 ) https://hpfx.collab.science.gc.ca /20230527/WXO-DD/observations/swob-ml/20230527/CWST/2023-05-27-1537-CWST-AUTO-minute-swob.xml \n", + "2023-05-27 12:03:20,390 [INFO] sarracenia.flowcb.log after_work downloaded ok: /tmp/flow_demo/2023-05-27-1537-CWST-AUTO-minute-swob.xml \n", + "2023-05-27 12:03:20,416 [DEBUG] sarracenia.moth.amqp getNewMessage new msg: {'_format': 'v02', '_deleteOnPost': {'local_offset', '_format', 'exchange', 'ack_id', 'subtopic'}, 'sundew_extension': 'DMS:WXO_RENAMED_SWOB2:MSC:XML::20230527153757', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_4ab40f078eb13197b272abb84cac6888:DMS:WXO_RENAMED_SWOB2:MSC:XML::20230527153757', 'source': 'WXO-DD', 'mtime': '20230527T153759.474', 'atime': '20230527T153759.474', 'pubTime': '20230527T153759.474', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230527/WXO-DD/observations/swob-ml/20230527/CWKH/2023-05-27-1537-CWKH-AUTO-minute-swob.xml', 'subtopic': ['20230527', 'WXO-DD', 'observations', 'swob-ml', '20230527', 'CWKH'], 'integrity': {'method': 'md5', 'value': 'r1E+zSg4n8PaB81Mnq6POw=='}, 'size': 6125, 'exchange': 'xpublic', 'ack_id': 3, 'local_offset': 0}\n", + "2023-05-27 12:03:20,417 [INFO] sarracenia.flowcb.log after_accept accepted: (lag: 1520.94 ) https://hpfx.collab.science.gc.ca /20230527/WXO-DD/observations/swob-ml/20230527/CWKH/2023-05-27-1537-CWKH-AUTO-minute-swob.xml \n", + "2023-05-27 12:03:20,507 [INFO] sarracenia.flowcb.log after_work downloaded ok: /tmp/flow_demo/2023-05-27-1537-CWKH-AUTO-minute-swob.xml \n", + "2023-05-27 12:03:20,530 [DEBUG] sarracenia.moth.amqp getNewMessage new msg: {'_format': 'v02', '_deleteOnPost': {'local_offset', '_format', 'exchange', 'ack_id', 'subtopic'}, 'sundew_extension': 'DMS:WXO_RENAMED_SWOB2:MSC:XML::20230527153757', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_575c5ec63a2fe354c27ff82f3cf9181f:DMS:WXO_RENAMED_SWOB2:MSC:XML::20230527153757', 'source': 'WXO-DD', 'mtime': '20230527T153759.476', 'atime': '20230527T153759.476', 'pubTime': '20230527T153759.476', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230527/WXO-DD/observations/swob-ml/20230527/CVSJ/2023-05-27-1537-CVSJ-AUTO-minute-swob.xml', 'subtopic': ['20230527', 'WXO-DD', 'observations', 'swob-ml', '20230527', 'CVSJ'], 'integrity': {'method': 'md5', 'value': 'H2iMpFuVhCmP39IeGNwx9g=='}, 'size': 9440, 'exchange': 'xpublic', 'ack_id': 4, 'local_offset': 0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2023-01-06 12:58:27,519 [INFO] sarracenia.flowcb.housekeeping.resources on_housekeeping Current Memory cpu_times: user=0.4 system=0.06\n", - "2023-01-06 12:58:27,519 [INFO] sarracenia.flowcb.housekeeping.resources on_housekeeping Current mem usage: 758.9 MiB, accumulating count (4 or 4/100 so far) before self-setting threshold\n", - "2023-01-06 12:58:27,519 [INFO] sarracenia.flow run clean stop from run loop\n", - "2023-01-06 12:58:27,537 [DEBUG] amqp collect Closed channel #1\n", - "2023-01-06 12:58:27,538 [INFO] sarracenia.flowcb.gather.message on_stop closing\n", - "2023-01-06 12:58:27,538 [INFO] sarracenia.flow close flow/close completed cleanly pid: 1851720 subscribe/flow_demo instance: 0\n" + "2023-05-27 12:03:20,530 [INFO] sarracenia.flowcb.log after_accept accepted: (lag: 1521.05 ) https://hpfx.collab.science.gc.ca /20230527/WXO-DD/observations/swob-ml/20230527/CVSJ/2023-05-27-1537-CVSJ-AUTO-minute-swob.xml \n", + "2023-05-27 12:03:20,630 [INFO] sarracenia.flowcb.log after_work downloaded ok: /tmp/flow_demo/2023-05-27-1537-CVSJ-AUTO-minute-swob.xml \n", + "2023-05-27 12:03:20,630 [INFO] sarracenia.flow please_stop ok, telling 4 callbacks about it.\n", + "2023-05-27 12:03:20,631 [INFO] sarracenia.flow run starting last pass (without gather) through loop for cleanup.\n", + "2023-05-27 12:03:20,631 [INFO] sarracenia.flow please_stop ok, telling 4 callbacks about it.\n", + "2023-05-27 12:03:20,632 [INFO] sarracenia.flow run on_housekeeping pid: 1682497 subscribe/flow_demo instance: 0\n", + "2023-05-27 12:03:20,632 [INFO] sarracenia.flowcb.gather.message on_housekeeping messages: good: 4 bad: 0 bytes: 584 Bytes average: 146 Bytes\n", + "2023-05-27 12:03:20,632 [INFO] sarracenia.flowcb.retry on_housekeeping on_housekeeping\n", + "2023-05-27 12:03:20,633 [INFO] sarracenia.diskqueue on_housekeeping work_retry_00 on_housekeeping\n", + "2023-05-27 12:03:20,633 [INFO] sarracenia.diskqueue on_housekeeping No retry in list\n", + "2023-05-27 12:03:20,634 [INFO] sarracenia.diskqueue on_housekeeping on_housekeeping elapse 0.000592\n", + "2023-05-27 12:03:20,634 [INFO] sarracenia.diskqueue on_housekeeping post_retry_000 on_housekeeping\n", + "2023-05-27 12:03:20,635 [INFO] sarracenia.diskqueue on_housekeeping No retry in list\n", + "2023-05-27 12:03:20,635 [INFO] sarracenia.diskqueue on_housekeeping on_housekeeping elapse 0.000487\n", + "2023-05-27 12:03:20,636 [INFO] sarracenia.flowcb.housekeeping.resources on_housekeeping Current Memory cpu_times: user=0.49 system=0.04\n", + "2023-05-27 12:03:20,636 [INFO] sarracenia.flowcb.housekeeping.resources on_housekeeping Current mem usage: 759.0 MiB, accumulating count (4 or 4/100 so far) before self-setting threshold\n", + "2023-05-27 12:03:20,636 [INFO] sarracenia.flowcb.log stats version: 3.00.40, started: now, last_housekeeping: 0.5 seconds ago \n", + "2023-05-27 12:03:20,637 [INFO] sarracenia.flowcb.log stats messages received: 4, accepted: 4, rejected: 0 rate accepted: 100.0% or 8.8 m/s\n", + "2023-05-27 12:03:20,637 [INFO] sarracenia.flowcb.log stats files transferred: 4 bytes: 34.2 KiB rate: 75.2 KiB/sec\n", + "2023-05-27 12:03:20,637 [INFO] sarracenia.flowcb.log stats lag: average: 1520.89, maximum: 1521.05 \n", + "2023-05-27 12:03:20,638 [INFO] sarracenia.flowcb.log on_housekeeping housekeeping\n", + "2023-05-27 12:03:20,638 [INFO] sarracenia.flow run clean stop from run loop\n", + "2023-05-27 12:03:20,663 [DEBUG] amqp collect Closed channel #1\n", + "2023-05-27 12:03:20,664 [INFO] sarracenia.flowcb.gather.message on_stop closing\n", + "2023-05-27 12:03:20,664 [INFO] sarracenia.flow close flow/close completed cleanly pid: 1682497 subscribe/flow_demo instance: 0\n" ] } ], @@ -228,7 +252,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.7" + "version": "3.10.6" } }, "nbformat": 4, diff --git a/docs/source/Tutorials/4_api_moth_sub_demo.ipynb b/docs/source/Tutorials/4_api_moth_sub_demo.ipynb index d35d8a5c9..976de60c9 100644 --- a/docs/source/Tutorials/4_api_moth_sub_demo.ipynb +++ b/docs/source/Tutorials/4_api_moth_sub_demo.ipynb @@ -25,20 +25,24 @@ "which allows a python program to connect to a Sarracenia server, and start receiving \n", "messages that announce resources.\n", "\n", - "The factory to build sarracenia.moth objects takes two arguments: \n", + "The factory to build sarracenia.moth objects requires a dictionary of settings as an argument: \n", + "\n", "\n", - "* broker: an object (Credential) containing a url pointing to the message server that is announcing products, and other related options.\n", "* options: a dictionary of other settings the class might use.\n", "\n", + " * 'broker': an object (Credential) containing a url pointing to the message server that is announcing products, and other related options.\n", + "\n", + " \n", + "\n", "The example below builds a call to an broker anyone can access, and just request\n", - "10 announcements.\n", + "5 announcements.\n", "\n", "You can run it, and then we can discuss a few settings:\n" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 4, "id": "romance-handy", "metadata": { "scrolled": true, @@ -51,7 +55,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "options: {'acceptUnmatched': True, 'batch': 25, 'bindings': [('xpublic', ['v02', 'post'], ['#'])], 'broker': None, 'exchange': None, 'expire': 300, 'inline': False, 'inline_encoding': 'guess', 'inline_max': 4096, 'logFormat': '%(asctime)s [%(levelname)s] %(name)s %(funcName)s %(message)s', 'logLevel': 'info', 'messageDebugDump': False, 'message_strategy': {'reset': True, 'stubborn': True, 'failure_duration': '5m'}, 'message_ttl': 0, 'topicPrefix': ['v02', 'post'], 'tls_rigour': 'normal', 'queue_name': 'q_anonymous_fractal_SomethingHelpfulToYou', 'subtopic': [], 'durable': True, 'prefetch': 25, 'auto_delete': False, 'vhost': '/', 'reset': False, 'declare': True, 'bind': True}\n" + "options: {'acceptUnmatched': True, 'batch': 25, 'bindings': [('xpublic', ['v02', 'post'], ['#'])], 'broker': , 'dry_run': False, 'exchange': None, 'expire': None, 'inline': False, 'inlineEncoding': 'guess', 'inlineByteMax': 4096, 'logFormat': '%(asctime)s [%(levelname)s] %(name)s %(funcName)s %(message)s', 'logLevel': 'info', 'messageDebugDump': False, 'message_strategy': {'reset': True, 'stubborn': True, 'failure_duration': '5m'}, 'message_ttl': 0, 'topicPrefix': ['v02', 'post'], 'tlsRigour': 'normal', 'auto_delete': False, 'durable': True, 'exchangeDeclare': True, 'prefetch': 25, 'queueName': 'q_anonymous_fractal_SomethingHelpfulToYou', 'queueBind': True, 'queueDeclare': True, 'reset': False, 'subtopic': [], 'vhost': '/'}\n" ] } ], @@ -63,13 +67,14 @@ "import time\n", "import socket\n", "\n", - "broker = sarracenia.credentials.Credential('amqps://anonymous:anonymous@hpfx.collab.science.gc.ca')\n", - "\n", "options = sarracenia.moth.default_options\n", + "\n", "options.update(sarracenia.moth.amqp.default_options)\n", + "\n", + "options['broker'] = sarracenia.credentials.Credential('amqps://anonymous:anonymous@hpfx.collab.science.gc.ca')\n", "options['topicPrefix'] = [ 'v02', 'post' ]\n", "options['bindings'] = [('xpublic', ['v02', 'post'], ['#'])]\n", - "options['queue_name'] = 'q_anonymous_' + socket.getfqdn() + '_SomethingHelpfulToYou'\n", + "options['queueName'] = 'q_anonymous_' + socket.getfqdn() + '_SomethingHelpfulToYou'\n", "\n", "print('options: %s' % options)\n", "\n" @@ -126,7 +131,7 @@ "would match all of the swobs similar to the one above, but avoid sending messages for non-swobs to you.\n", "\n", "\n", - "## queue_name\n", + "## queueName\n", "\n", "By convention in brokers administered by Sarracenia, users can only create queues that start with q_ followed by their user name. we connected as anonymous, and so q_anonymous must be used. After that, the rest can be whatever you want, but there are a few considerations:\n", "\n", @@ -151,52 +156,70 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "id": "little-louis", "metadata": { "scrolled": true }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-05-27 10:55:22,559 [INFO] sarracenia.moth.amqp __getSetup queue declared q_anonymous_fractal_SomethingHelpfulToYou (as: amqps://anonymous@hpfx.collab.science.gc.ca) \n", + "2023-05-27 10:55:22,560 [INFO] sarracenia.moth.amqp __getSetup binding q_anonymous_fractal_SomethingHelpfulToYou with v02.post.# to xpublic (as: amqps://anonymous@hpfx.collab.science.gc.ca)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "message 0: {'_format': 'v02', '_deleteOnPost': {'subtopic', 'exchange', 'ack_id', 'local_offset', '_format'}, 'sundew_extension': 'CMC:REGIONAL:GRIB2:BIN::20230527145518', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_34df392aeffc9c678011f3fd30193bb6:CMC:REGIONAL:GRIB2:BIN::20230527145518', 'source': 'WXO-DD', 'mtime': '20230527T145520.791', 'atime': '20230527T145520.791', 'pubTime': '20230527T145520.791', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230527/WXO-DD/model_gem_regional/10km/grib2/12/037/CMC_reg_WIND_ISBL_30_ps10km_2023052712_P037.grib2', 'subtopic': ['20230527', 'WXO-DD', 'model_gem_regional', '10km', 'grib2', '12', '037'], 'integrity': {'method': 'md5', 'value': 'U1vVZnatrCeK3bLrXshb2g=='}, 'size': 554100, 'exchange': 'xpublic', 'ack_id': 1, 'local_offset': 0}\n", + "first 50 bytes of corresponding file: b'GRIB\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x08tt\\x00\\x00\\x00\\x15\\x01\\x006\\x00\\x00\\x04\\x00\\x01\\x07\\xe7\\x05\\x1b\\x0c\\x00\\x00\\x00\\x02\\x00\\x00\\x00A\\x03\\x00\\x00\\x0b\\xc1\\x88\\x00\\x00\\x00'\n", + "message 1: {'_format': 'v02', '_deleteOnPost': {'subtopic', 'exchange', 'ack_id', 'local_offset', '_format'}, 'sundew_extension': 'CMC:REGIONAL:GRIB2:BIN::20230527145519', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_752eb4e8803503704990563d84030e67:CMC:REGIONAL:GRIB2:BIN::20230527145519', 'source': 'WXO-DD', 'mtime': '20230527T145520.292', 'atime': '20230527T145520.292', 'pubTime': '20230527T145520.292', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230527/WXO-DD/model_gem_regional/10km/grib2/12/037/CMC_reg_HGT_ISBL_250_ps10km_2023052712_P037.grib2', 'subtopic': ['20230527', 'WXO-DD', 'model_gem_regional', '10km', 'grib2', '12', '037'], 'integrity': {'method': 'md5', 'value': 'j6bh9dbE4QbJAXEOejw0Tw=='}, 'size': 377005, 'exchange': 'xpublic', 'ack_id': 2, 'local_offset': 0}\n", + "first 50 bytes of corresponding file: b'GRIB\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x05\\xc0\\xad\\x00\\x00\\x00\\x15\\x01\\x006\\x00\\x00\\x04\\x00\\x01\\x07\\xe7\\x05\\x1b\\x0c\\x00\\x00\\x00\\x02\\x00\\x00\\x00A\\x03\\x00\\x00\\x0b\\xc1\\x88\\x00\\x00\\x00'\n", + "message 2: {'_format': 'v02', '_deleteOnPost': {'subtopic', 'exchange', 'ack_id', 'local_offset', '_format'}, 'sundew_extension': 'CMC:REGIONAL:GRIB2:BIN::20230527145519', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_55f121bb28e822cffb6e61196cd924eb:CMC:REGIONAL:GRIB2:BIN::20230527145519', 'source': 'WXO-DD', 'mtime': '20230527T145521.260', 'atime': '20230527T145521.260', 'pubTime': '20230527T145521.260', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230527/WXO-DD/model_gem_regional/10km/grib2/12/037/CMC_reg_RH_ISBL_700_ps10km_2023052712_P037.grib2', 'subtopic': ['20230527', 'WXO-DD', 'model_gem_regional', '10km', 'grib2', '12', '037'], 'integrity': {'method': 'md5', 'value': 'V7goy/doL6Gle68s1zoVEA=='}, 'size': 808438, 'exchange': 'xpublic', 'ack_id': 3, 'local_offset': 0}\n", + "first 50 bytes of corresponding file: b'GRIB\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x0cU\\xf6\\x00\\x00\\x00\\x15\\x01\\x006\\x00\\x00\\x04\\x00\\x01\\x07\\xe7\\x05\\x1b\\x0c\\x00\\x00\\x00\\x02\\x00\\x00\\x00A\\x03\\x00\\x00\\x0b\\xc1\\x88\\x00\\x00\\x00'\n", + "message 3: {'_format': 'v02', '_deleteOnPost': {'subtopic', 'exchange', 'ack_id', 'local_offset', '_format'}, 'sundew_extension': 'CMC:REGIONAL:GRIB2:BIN::20230527145518', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_dac300cf33756ba816e030f99fc9dc22:CMC:REGIONAL:GRIB2:BIN::20230527145518', 'source': 'WXO-DD', 'mtime': '20230527T145519.586', 'atime': '20230527T145519.586', 'pubTime': '20230527T145519.586', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230527/WXO-DD/model_gem_regional/10km/grib2/12/037/CMC_reg_UGRD_ISBL_225_ps10km_2023052712_P037.grib2', 'subtopic': ['20230527', 'WXO-DD', 'model_gem_regional', '10km', 'grib2', '12', '037'], 'integrity': {'method': 'md5', 'value': 'MI8XzT1uam5OUf7QlDZ4FA=='}, 'size': 487411, 'exchange': 'xpublic', 'ack_id': 4, 'local_offset': 0}\n", + "first 50 bytes of corresponding file: b'GRIB\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x07o\\xf3\\x00\\x00\\x00\\x15\\x01\\x006\\x00\\x00\\x04\\x00\\x01\\x07\\xe7\\x05\\x1b\\x0c\\x00\\x00\\x00\\x02\\x00\\x00\\x00A\\x03\\x00\\x00\\x0b\\xc1\\x88\\x00\\x00\\x00'\n", + "message 4: {'_format': 'v02', '_deleteOnPost': {'subtopic', 'exchange', 'ack_id', 'local_offset', '_format'}, 'sundew_extension': 'CMC:REGIONAL:GRIB2:BIN::20230527145519', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_c5e84748169e0a6dce8f3b884ffdf059:CMC:REGIONAL:GRIB2:BIN::20230527145519', 'source': 'WXO-DD', 'mtime': '20230527T145520.651', 'atime': '20230527T145520.651', 'pubTime': '20230527T145520.651', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230527/WXO-DD/model_gem_regional/10km/grib2/12/037/CMC_reg_RH_ISBL_550_ps10km_2023052712_P037.grib2', 'subtopic': ['20230527', 'WXO-DD', 'model_gem_regional', '10km', 'grib2', '12', '037'], 'integrity': {'method': 'md5', 'value': 'zukdtksA5I0C5oq/ieiXbQ=='}, 'size': 774394, 'exchange': 'xpublic', 'ack_id': 5, 'local_offset': 0}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-05-27 10:55:24,022 [INFO] sarracenia.moth.amqp getCleanUp deleteing queue q_anonymous_fractal_SomethingHelpfulToYou\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "message 3: {'sundew_extension': 'DMS:CMC:WXR-TRANSCODER-OBS-DATAMART-IVR:AUDIO', 'to_clusters': 'DDI,DDSR', 'x-delay': 0, 'source': 'MSC-DMS-OP', 'from_cluster': 'DDSR.CMC', 'subtopic': ['20220215', 'WXO-DD', 'hello_weather', 'observations', '03'], '_deleteOnPost': {'local_offset', 'exchange', 'ack_id', 'subtopic'}, 'pubTime': '20220215T132115.073', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20220215/WXO-DD/.hello_weather/observations/03/03097.1.OBS.PUB.FR.SPX~20220215130500-20220215150500-1464', 'integrity': {'method': 'arbitrary', 'value': 'b8e417f28fc1f42557f4f633dad2b433'}, 'size': 65812, 'exchange': 'xpublic', 'ack_id': 1, 'local_offset': 0}\n", - "first 50 bytes of corresponding file: b'OggS\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x13\\xb5>\\x99\\x00\\x00\\x00\\x00\\xaa5\\xfff\\x01PSpeex 1.2rc1\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00'\n", - "message 4: {'sundew_extension': 'cvt_nws_bulletins-sr:ENMI:SA:3:Direct:20220215132114', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'DDSR.CMC,DDI.CMC,CMC,SCIENCE,EDM', 'filename': 'msg_ddsr-WXO-DD3_8c1a10e7fe978f07309d237be3f1fddb:cvt_nws_bulletins-sr:ENMI:SA:3:Direct:20220215132114', 'source': 'WXO-DD', 'mtime': '20220215T132116.584', 'atime': '20220215T132116.584', 'subtopic': ['20220215', 'WXO-DD', 'bulletins', 'alphanumeric', '20220215', 'SA', 'ENMI', '13'], '_deleteOnPost': {'local_offset', 'exchange', 'ack_id', 'subtopic'}, 'pubTime': '20220215T132116.584', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20220215/WXO-DD/bulletins/alphanumeric/20220215/SA/ENMI/13/SANO31_ENMI_151320___52674', 'integrity': {'method': 'md5', 'value': 'Icka1HDlRhJOmEzfOAoQpg=='}, 'size': 684, 'exchange': 'xpublic', 'ack_id': 2, 'local_offset': 0}\n", - "first 50 bytes of corresponding file: b'SANO31 ENMI 151320\\nMETAR ENAN 151320Z 13007KT 9999'\n", - "message 5: {'sundew_extension': 'cvt_nws_bulletins-sr:KWAL:SX:3:Direct:20220215132114', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'DDSR.CMC,DDI.CMC,CMC,SCIENCE,EDM', 'filename': 'msg_ddsr-WXO-DD3_e0ed771ab87a1a0b8e68f5a9b91a65c9:cvt_nws_bulletins-sr:KWAL:SX:3:Direct:20220215132114', 'source': 'WXO-DD', 'mtime': '20220215T132116.585', 'atime': '20220215T132116.585', 'subtopic': ['20220215', 'WXO-DD', 'bulletins', 'alphanumeric', '20220215', 'SX', 'KWAL', '13'], '_deleteOnPost': {'local_offset', 'exchange', 'ack_id', 'subtopic'}, 'pubTime': '20220215T132116.585', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20220215/WXO-DD/bulletins/alphanumeric/20220215/SX/KWAL/13/SXCN40_KWAL_151320___38969', 'integrity': {'method': 'md5', 'value': 'ccUpl7Q8zGzHIf4SUbv3og=='}, 'size': 144, 'exchange': 'xpublic', 'ack_id': 3, 'local_offset': 0}\n", - "first 50 bytes of corresponding file: b'SXCN40 KWAL 151320\\nCAC2186E 046132053\\n-032.9\\n057\\n0'\n", - "message 6: {'sundew_extension': 'cvt_nws_bulletins-sr:KWNB:SX:3:Direct:20220215132115', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'DDSR.CMC,DDI.CMC,CMC,SCIENCE,EDM', 'filename': 'msg_ddsr-WXO-DD3_4bb24c4edaf3275b72c6ed792abe3df5:cvt_nws_bulletins-sr:KWNB:SX:3:Direct:20220215132115', 'source': 'WXO-DD', 'mtime': '20220215T132117.587', 'atime': '20220215T132117.587', 'subtopic': ['20220215', 'WXO-DD', 'bulletins', 'alphanumeric', '20220215', 'SX', 'KWNB', '13'], '_deleteOnPost': {'local_offset', 'exchange', 'ack_id', 'subtopic'}, 'pubTime': '20220215T132117.587', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20220215/WXO-DD/bulletins/alphanumeric/20220215/SX/KWNB/13/SXUS23_KWNB_151300_RRP__54105', 'integrity': {'method': 'md5', 'value': '34bEHeB5cwI9WWKKk97OCw=='}, 'size': 2451, 'exchange': 'xpublic', 'ack_id': 4, 'local_offset': 0}\n", - "first 50 bytes of corresponding file: b'SXUS23 KWNB 151300 RRP\\nCMAN 15134\\nLCNA2 46/// /261'\n", - "message 7: {'sundew_extension': 'cvt_nws_bulletins-sr:KWAL:SR:3:Direct:20220215132115', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'DDSR.CMC,DDI.CMC,CMC,SCIENCE,EDM', 'filename': 'msg_ddsr-WXO-DD3_60f837832b38c3d9acc8b2d1f3190be1:cvt_nws_bulletins-sr:KWAL:SR:3:Direct:20220215132115', 'source': 'WXO-DD', 'mtime': '20220215T132117.588', 'atime': '20220215T132117.588', 'subtopic': ['20220215', 'WXO-DD', 'bulletins', 'alphanumeric', '20220215', 'SR', 'KWAL', '13'], '_deleteOnPost': {'local_offset', 'exchange', 'ack_id', 'subtopic'}, 'pubTime': '20220215T132117.588', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20220215/WXO-DD/bulletins/alphanumeric/20220215/SR/KWAL/13/SRCN40_KWAL_151320___62678', 'integrity': {'method': 'md5', 'value': 'IP+n0cisIJMGn5VXJy5WeQ=='}, 'size': 262, 'exchange': 'xpublic', 'ack_id': 5, 'local_offset': 0}\n", - "first 50 bytes of corresponding file: b'SRCN40 KWAL 151320\\n484010DC 046132052 :HG 5 #5 0.'\n", - "message 8: {'sundew_extension': 'DMS:WXO_RENAMED_SWOB2:MSC:XML::20220215132117', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'DDSR.CMC,DDI.CMC,CMC,SCIENCE,EDM', 'filename': 'msg_ddsr-WXO-DD_7aee783b17658fbf30e138f39d872a6b:DMS:WXO_RENAMED_SWOB2:MSC:XML::20220215132117', 'source': 'WXO-DD', 'mtime': '20220215T132119.385', 'atime': '20220215T132119.385', 'subtopic': ['20220215', 'WXO-DD', 'observations', 'swob-ml', '20220215', 'CWTN'], '_deleteOnPost': {'local_offset', 'exchange', 'ack_id', 'subtopic'}, 'pubTime': '20220215T132119.385', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20220215/WXO-DD/observations/swob-ml/20220215/CWTN/2022-02-15-1320-CWTN-AUTO-minute-swob.xml', 'integrity': {'method': 'md5', 'value': 'S1HVKvy8ELiUs40fJF7Wow=='}, 'size': 8528, 'exchange': 'xpublic', 'ack_id': 6, 'local_offset': 0}\n", - "first 50 bytes of corresponding file: b' Config + * pubFactory( broker, options ) -> Config + * subFactory( broker, options ) -> Config + + après: + + * Moth( options: dict, is_subscribe: bool) -> Config + * pubFactory( options ) -> Config + * subFactory( options ) -> Config + + sarracenia.config API: + + Il est recommandé d´appeller **sarracenia.config.finalize()** + après avoir fourni les options. Celui-ci interpole et complète + les valeurs d´options pou qu´ils soient utilisable par les + composantes. + 3.0.26 ------ diff --git a/docs/source/fr/Tutoriel/2_CLI_with_flowcb_demo.ipynb b/docs/source/fr/Tutoriel/2_CLI_with_flowcb_demo.ipynb index 73e095823..fa58def7f 100644 --- a/docs/source/fr/Tutoriel/2_CLI_with_flowcb_demo.ipynb +++ b/docs/source/fr/Tutoriel/2_CLI_with_flowcb_demo.ipynb @@ -60,10 +60,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "2022-03-19 13:19:27,886 2724830 [INFO] sarracenia.config fill_missing_options overriding batch for consistency with messageCountMax: 10\n", - "2022-03-19 13:19:27,887 2724830 [INFO] root remove removing subscribe/hpfx_amis \n", + "2023-05-28 16:35:16,492 1919860 [INFO] sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\n", + "2023-05-28 16:35:16,493 1919860 [INFO] root remove removing subscribe/hpfx_amis\n", "\n", - "add: 2022-03-19 13:19:28,372 2724832 [INFO] sarracenia.sr add copying: /home/peter/Sarracenia/sr3/sarracenia/examples/subscribe/hpfx_amis.conf to /home/peter/.config/sr3/subscribe/hpfx_amis.conf \n", + "add: 2023-05-28 16:35:17,637 1919863 [INFO] sarracenia.sr add copying: /home/peter/Sarracenia/sr3/sarracenia/examples/subscribe/hpfx_amis.conf to /home/peter/.config/sr3/subscribe/hpfx_amis.conf \n", "\n" ] } @@ -95,39 +95,49 @@ "cell_type": "code", "execution_count": 3, "id": "greater-nevada", - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "2022-03-19 13:19:36,284 2724838 [INFO] sarracenia.config fill_missing_options overriding batch for consistency with messageCountMax: 10\n", - "Provided callback classes: ( /home/peter/Sarracenia/sr3/sarracenia ) \n", - "flowcb/accept/delete.py flowcb/accept/downloadbaseurl.py \n", - "flowcb/accept/hourtree.py flowcb/accept/httptohttps.py \n", - "flowcb/accept/longflow.py flowcb/accept/posthourtree.py \n", - "flowcb/accept/postoverride.py flowcb/accept/printlag.py \n", - "flowcb/accept/rename4jicc.py flowcb/accept/renamedmf.py \n", - "flowcb/accept/renamewhatfn.py flowcb/accept/save.py \n", - "flowcb/accept/speedo.py flowcb/accept/sundewpxroute.py \n", - "flowcb/accept/testretry.py flowcb/accept/toclusters.py \n", - "flowcb/accept/tohttp.py flowcb/accept/tolocal.py \n", - "flowcb/accept/tolocalfile.py flowcb/accept/wmotypesuffix.py \n", - "flowcb/clamav.py flowcb/filter/deleteflowfiles.py \n", - "flowcb/filter/fdelay.py flowcb/filter/pclean_f90.py \n", - "flowcb/filter/pclean_f92.py flowcb/filter/wmo2msc.py \n", - "flowcb/gather/file.py flowcb/gather/message.py \n", - "flowcb/housekeeping/resources.py flowcb/log.py \n", - "flowcb/mdelaylatest.py flowcb/nodupe/data.py \n", - "flowcb/nodupe/name.py flowcb/pclean.py \n", - "flowcb/poll/airnow.py flowcb/poll/mail.py \n", - "flowcb/poll/nasa_mls_nrt.py flowcb/poll/nexrad.py \n", - "flowcb/poll/noaa_hydrometric.py flowcb/poll/usgs.py \n", - "flowcb/post/message.py flowcb/retry.py \n", - "flowcb/sample.py flowcb/script.py \n", - "flowcb/send/email.py flowcb/shiftdir2baseurl.py \n", - "flowcb/v2wrapper.py flowcb/wistree.py \n", - "flowcb/work/delete.py flowcb/work/rxpipe.py \n" + "2023-05-28 16:35:26,803 1919870 [INFO] sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\r\n", + "Provided callback classes: ( /home/peter/Sarracenia/sr3/sarracenia ) \r\n", + "flowcb/accept/dateappend.py flowcb/accept/delete.py \r\n", + "flowcb/accept/downloadbaseurl.py flowcb/accept/hourtree.py \r\n", + "flowcb/accept/httptohttps.py flowcb/accept/longflow.py \r\n", + "flowcb/accept/pathreplace.py flowcb/accept/posthourtree.py \r\n", + "flowcb/accept/postoverride.py flowcb/accept/printlag.py \r\n", + "flowcb/accept/rename4jicc.py flowcb/accept/renamedmf.py \r\n", + "flowcb/accept/renamewhatfn.py flowcb/accept/save.py \r\n", + "flowcb/accept/speedo.py flowcb/accept/sundewpxroute.py \r\n", + "flowcb/accept/testretry.py flowcb/accept/toclusters.py \r\n", + "flowcb/accept/tohttp.py flowcb/accept/tolocal.py \r\n", + "flowcb/accept/tolocalfile.py flowcb/accept/wmotypesuffix.py \r\n", + "flowcb/amserver.py flowcb/clamav.py \r\n", + "flowcb/destfn/replace.py flowcb/destfn/sample.py \r\n", + "flowcb/download/mail_ingest.py flowcb/filter/deleteflowfiles.py \r\n", + "flowcb/filter/fdelay.py flowcb/filter/pclean_f90.py \r\n", + "flowcb/filter/pclean_f92.py flowcb/filter/wmo2msc.py \r\n", + "flowcb/gather/file.py flowcb/gather/message.py \r\n", + "flowcb/housekeeping/resources.py flowcb/log.py \r\n", + "flowcb/mdelaylatest.py flowcb/nodupe/data.py \r\n", + "flowcb/nodupe/name.py flowcb/pclean.py \r\n", + "flowcb/poll/airnow.py flowcb/poll/mail.py \r\n", + "flowcb/poll/nasa_mls_nrt.py flowcb/poll/nexrad.py \r\n", + "flowcb/poll/noaa_hydrometric.py flowcb/poll/s3bucket.py \r\n", + "flowcb/poll/usgs.py flowcb/post/message.py \r\n", + "flowcb/retry.py flowcb/rootchown.py \r\n", + "flowcb/run.py flowcb/rxqueue_gzip.py \r\n", + "flowcb/sample.py flowcb/send/am.py \r\n", + "flowcb/send/email.py flowcb/shiftdir2baseurl.py \r\n", + "flowcb/trace_on_stop.py flowcb/v2wrapper.py \r\n", + "flowcb/wistree.py flowcb/work/age.py \r\n", + "flowcb/work/check.py flowcb/work/citypage_check.py \r\n", + "flowcb/work/delete.py flowcb/work/rxpipe.py \r\n", + "flowcb/work/send_egc_les.py \r\n" ] } ], @@ -151,7 +161,7 @@ "metadata": {}, "outputs": [], "source": [ - "!echo callback wistree >>~/.config/sr3/subscribe/hpfx_amis.conf" + "!echo callback accept.posthourtree >>~/.config/sr3/subscribe/hpfx_amis.conf" ] }, { @@ -159,84 +169,88 @@ "execution_count": 5, "id": "insured-fetish", "metadata": { - "scrolled": true + "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "2022-03-19 13:19:48,073 2724872 [INFO] sarracenia.config fill_missing_options overriding batch for consistency with messageCountMax: 10\n", - ".2022-03-19 13:19:48,285 [INFO] 2724874 sarracenia.config fill_missing_options overriding batch for consistency with messageCountMax: 10\n", - "2022-03-19 13:19:48,287 [INFO] 2724874 sarracenia.config fill_missing_options overriding batch for consistency with messageCountMax: 10\n", - "2022-03-19 13:19:48,287 [INFO] 2724874 sarracenia.flow loadCallbacks plugins to load: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'sarracenia.flowcb.wistree.Wistree', 'sarracenia.flowcb.log.Log']\n", - "2022-03-19 13:19:48,690 [INFO] 2724874 sarracenia.flowcb.log __init__ subscribe initialized with: {'after_work', 'after_accept', 'on_housekeeping'}\n", - "2022-03-19 13:19:48,690 [INFO] 2724874 sarracenia.flow run options:\n", - "_Config__admin=\"...st/ None True True False False None None\",\n", - "_Config__broker=\"...ca/ None True True False False None None\",\n", - "_Config__post_broker=None, accel_threshold=0, acceptSizeWrong=False,\n", - "acceptUnmatched=False, action='foreground', attempts=3, auto_delete=False,\n", - "baseDir=None, baseUrl_relPath=False, batch=10,\n", - "bindings=\"... ['*.WXO-DD.bulletins.alphanumeric.#'])]\", bufsize=1048576,\n", - "bytes_per_second=None, bytes_ps=0,\n", - "cfg_run_dir=\"...me/peter/.cache/sr3/subscribe/hpfx_amis'\",\n", - "component='subscribe', config='hpfx_amis.conf',\n", - "configurations=['subscribe/hpfx_amis.conf'], currentDir=None,\n", - "dangerWillRobinson=False, debug=False, declared_exchanges=[],\n", - "declared_users=\"...': 'source', 'eggmeister': 'subscriber'}\", delete=False,\n", - "destfn_script=None, directory='/tmp/hpfx_amis/', discard=False,\n", - "documentRoot=None, download=True, durable=True,\n", - "env_declared=\"...OKER', 'MQP', 'SFTPUSER', 'TESTDOCROOT']\", exchange='xpublic',\n", - "exchangeDeclare=True, expire=600.0, feeder=amqp://tfeed@localhost,\n", - "fileEvents={'modify', 'link', 'delete', 'create'}, filename='WHATFN',\n", - "fixed_headers={}, flatten='/', hostdir='fractal', hostname='fractal',\n", - "housekeeping=300, imports=[], inflight=None, inline=False,\n", - "inline_encoding='guess', inline_max=4096, inline_only=False, instances=5,\n", - "integrity_arbitrary_value=None, integrity_method='sha512',\n", - "logEvents=\"...ork', 'after_accept', 'on_housekeeping'}\",\n", - "logFormat=\"...me)s] %(name)s %(funcName)s %(message)s'\", logLevel='info',\n", - "logStdout=False, log_flowcb_needed=False, lr_backupCount=5, lr_interval=1,\n", - "lr_when='midnight', masks=\"...pile('.*'), True, False, 0, False, '/')]\",\n", - "messageAgeMax=0, messageCountMax=10, messageDebugDump=False, messageRateMax=0,\n", - "messageRateMin=0,\n", - "message_strategy=\"...ubborn': True, 'failure_duration': '5m'}\", message_ttl=0,\n", - "mirror=False, no=0, nodupe_fileAgeMax=0, nodupe_ttl=0, overwrite=True,\n", - "permCopy=True, permDefault=0, permDirDefault=509, permLog=384,\n", - "pid_filename=\"...e/hpfx_amis//subscribe_hpfx_amis_00.pid'\", plugins_early=[],\n", - "plugins_late=\"...e.Wistree', 'sarracenia.flowcb.log.Log']\", post_baseDir=None,\n", - "post_baseUrl=None, post_documentRoot=None, post_exchanges=[],\n", - "post_topicPrefix=['v02', 'post'], prefetch=25, pstrip=False, queueBind=True,\n", - "queueDeclare=True, queueName=\"...s_subscribe.hpfx_amis.73590194.51440323'\",\n", - "queue_filename=\"...mis/subscribe.hpfx_amis.anonymous.qname'\", randid='4529',\n", - "randomize=False, realpathPost=False, rename=None, report_back=False,\n", - "reset=False, retry_path=\"...hpfx_amis//subscribe_hpfx_amis_00.retry'\",\n", - "retry_ttl=600.0, settings={}, sleep=0.1, statehost=False, strip=0, subtopic=[],\n", - "timeCopy=True, timeout=300, timezone='UTC', tls_rigour='normal',\n", - "topicPrefix=['v02', 'post'], undeclared=[], users=False, v2plugin_options=[],\n", - "v2plugins={}, vhost='/', vip=None\n", - "2022-03-19 13:19:48,691 [INFO] 2724874 sarracenia.flow run callbacks loaded: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'sarracenia.flowcb.wistree.Wistree', 'sarracenia.flowcb.log.Log']\n", - "2022-03-19 13:19:48,691 [INFO] 2724874 sarracenia.flow run pid: 2724874 subscribe/hpfx_amis.conf instance: 0\n", - "2022-03-19 13:19:50,780 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 3.37 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SX/KWAL/17/SXCN40_KWAL_191719___37614 \n", - "2022-03-19 13:19:50,780 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 3.37 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SX/KWAL/17/SXCN40_KWAL_191719___24613 \n", - "2022-03-19 13:19:51,013 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/miscellaneous/ca/SXCN40_KWAL_191719___37614.txt \n", - "2022-03-19 13:19:51,013 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/miscellaneous/ca/SXCN40_KWAL_191719___24613.txt \n", - "2022-03-19 13:19:52,058 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 1.72 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SR/KWAL/17/SRCN40_KWAL_191719___9404 \n", - "2022-03-19 13:19:52,167 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/hydrological/river/ca/SRCN40_KWAL_191719___9404.txt \n", - "2022-03-19 13:19:53,236 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 0.63 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SR/KWAL/17/SRCN40_KWAL_191719___24050 \n", - "2022-03-19 13:19:53,236 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 0.62 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SX/KWAL/17/SXCN40_KWAL_191719___20524 \n", - "2022-03-19 13:19:53,476 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/hydrological/river/ca/SRCN40_KWAL_191719___24050.txt \n", - "2022-03-19 13:19:53,476 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/miscellaneous/ca/SXCN40_KWAL_191719___20524.txt \n", - "2022-03-19 13:19:53,715 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 1.03 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SR/KWAL/17/SRND20_KWAL_191719___45279 \n", - "2022-03-19 13:19:53,839 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/hydrological/river/SRND20_KWAL_191719___45279.txt \n", - "2022-03-19 13:19:54,094 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 0.41 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SR/KWAL/17/SRCN40_KWAL_191719___64176 \n", - "2022-03-19 13:19:54,094 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 0.74 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SR/KWAL/17/SRCN40_KWAL_191719___58640 \n", - "2022-03-19 13:19:54,346 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/hydrological/river/ca/SRCN40_KWAL_191719___64176.txt \n", - "2022-03-19 13:19:54,346 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/hydrological/river/ca/SRCN40_KWAL_191719___58640.txt \n", - "2022-03-19 13:19:54,384 [INFO] 2724874 sarracenia.flowcb.log after_accept accepted: (lag: 1.04 ) https://hpfx.collab.science.gc.ca /20220319/WXO-DD/bulletins/alphanumeric/20220319/SX/KWAL/17/SXCN40_KWAL_191719___16476 \n", - "2022-03-19 13:19:54,492 [INFO] 2724874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20220319T17/WIS/us/wallops_i__wallops_station_va/surface/miscellaneous/ca/SXCN40_KWAL_191719___16476.txt \n", - "2022-03-19 13:19:54,510 [INFO] 2724874 sarracenia.flowcb.gather.message on_stop closing\n", - "2022-03-19 13:19:54,510 [INFO] 2724874 sarracenia.flow close flow/close completed cleanly pid: 2724874 subscribe/hpfx_amis.conf instance: 0\n", - "\n" + "2023-05-28 16:35:34,235 1919873 [INFO] sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\n", + ".2023-05-28 16:35:34,349 [INFO] 1919874 sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\n", + "2023-05-28 16:35:34,355 [INFO] 1919874 sarracenia.config finalize overriding batch for consistency with messageCountMax: 10\n", + "2023-05-28 16:35:34,355 [INFO] 1919874 sarracenia.flow loadCallbacks flowCallback plugins to load: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'accept.posthourtree', 'log']\n", + "2023-05-28 16:35:34,670 [INFO] 1919874 sarracenia.moth.amqp __getSetup queue declared q_anonymous_subscribe.hpfx_amis.33557703.14415188 (as: amqps://anonymous@hpfx.collab.science.gc.ca/) \n", + "2023-05-28 16:35:34,670 [INFO] 1919874 sarracenia.moth.amqp __getSetup binding q_anonymous_subscribe.hpfx_amis.33557703.14415188 with v02.post.*.WXO-DD.bulletins.alphanumeric.# to xpublic (as: amqps://anonymous@hpfx.collab.science.gc.ca/)\n", + "2023-05-28 16:35:34,723 [INFO] 1919874 sarracenia.flowcb.log __init__ subscribe initialized with: {'on_housekeeping', 'post', 'after_post', 'after_work', 'after_accept'}\n", + "2023-05-28 16:35:34,723 [INFO] 1919874 sarracenia.flow run callbacks loaded: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'accept.posthourtree', 'log']\n", + "2023-05-28 16:35:34,723 [INFO] 1919874 sarracenia.flow run pid: 1919874 subscribe/hpfx_amis.conf instance: 0\n", + "2023-05-28 16:35:34,764 [INFO] 1919874 sarracenia.flow run now active on vip None\n", + "2023-05-28 16:35:35,009 [INFO] 1919874 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:35:35,009 [INFO] 1919874 sarracenia.flowcb.log after_accept accepted: (lag: 0.80 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/CA/CWAO/20/CACN45_CWAO_281300__OBQ_20965 \n", + "2023-05-28 16:35:35,154 [INFO] 1919874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/CACN45_CWAO_281300__OBQ_20965 \n", + "2023-05-28 16:35:35,677 [INFO] 1919874 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:35:35,677 [INFO] 1919874 sarracenia.flowcb.log after_accept accepted: (lag: 0.56 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SR/KWAL/20/SRCN40_KWAL_282035___1729 \n", + "2023-05-28 16:35:35,796 [INFO] 1919874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SRCN40_KWAL_282035___1729 \n", + "2023-05-28 16:35:36,314 [INFO] 1919874 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:35:36,314 [INFO] 1919874 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:35:36,314 [INFO] 1919874 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:35:36,314 [INFO] 1919874 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:35:36,314 [INFO] 1919874 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:35:36,314 [INFO] 1919874 sarracenia.flowcb.log after_accept accepted: (lag: 0.55 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SR/KWAL/20/SRWA20_KWAL_282035___43515 \n", + "2023-05-28 16:35:36,314 [INFO] 1919874 sarracenia.flowcb.log after_accept accepted: (lag: 0.53 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SN/KWNB/20/SNVD15_KWNB_282000_RRO__50372 \n", + "2023-05-28 16:35:36,314 [INFO] 1919874 sarracenia.flowcb.log after_accept accepted: (lag: 0.53 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SX/KWAL/20/SXAK50_KWAL_282035___51354 \n", + "2023-05-28 16:35:36,314 [INFO] 1919874 sarracenia.flowcb.log after_accept accepted: (lag: 1.81 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SR/KWAL/20/SRCN40_KWAL_282035___32251 \n", + "2023-05-28 16:35:36,314 [INFO] 1919874 sarracenia.flowcb.log after_accept accepted: (lag: 1.80 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SR/KWAL/20/SRCN40_KWAL_282035___62598 \n", + "2023-05-28 16:35:36,951 [INFO] 1919874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SRWA20_KWAL_282035___43515 \n", + "2023-05-28 16:35:36,951 [INFO] 1919874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SNVD15_KWNB_282000_RRO__50372 \n", + "2023-05-28 16:35:36,951 [INFO] 1919874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SXAK50_KWAL_282035___51354 \n", + "2023-05-28 16:35:36,951 [INFO] 1919874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SRCN40_KWAL_282035___32251 \n", + "2023-05-28 16:35:36,951 [INFO] 1919874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SRCN40_KWAL_282035___62598 \n", + "2023-05-28 16:35:37,133 [INFO] 1919874 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:35:37,133 [INFO] 1919874 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:35:37,133 [INFO] 1919874 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:35:37,133 [INFO] 1919874 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:35:37,133 [INFO] 1919874 sarracenia.flowcb.accept.posthourtree after_accept post_hour_tree: new_dir: /tmp/hpfx_amis/20\n", + "2023-05-28 16:35:37,133 [INFO] 1919874 sarracenia.flowcb.log after_accept accepted: (lag: 3.26 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/WA/KKCI/20/WAUS41_KKCI_282045___14468 \n", + "2023-05-28 16:35:37,133 [INFO] 1919874 sarracenia.flowcb.log after_accept accepted: (lag: 3.26 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SR/KWAL/20/SRCN40_KWAL_282035___20765 \n", + "2023-05-28 16:35:37,133 [INFO] 1919874 sarracenia.flowcb.log after_accept accepted: (lag: 3.26 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/WA/KKCI/20/WAUS46_KKCI_282045___65023 \n", + "2023-05-28 16:35:37,133 [INFO] 1919874 sarracenia.flowcb.log after_accept accepted: (lag: 3.26 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/WA/KKCI/20/WAUS44_KKCI_282045___40622 \n", + "2023-05-28 16:35:37,133 [INFO] 1919874 sarracenia.flowcb.log after_accept accepted: (lag: 3.25 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/bulletins/alphanumeric/20230528/SR/KWAL/20/SRCN40_KWAL_282035___41115 \n", + "2023-05-28 16:35:37,636 [INFO] 1919874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/WAUS41_KKCI_282045___14468 \n", + "2023-05-28 16:35:37,636 [INFO] 1919874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SRCN40_KWAL_282035___20765 \n", + "2023-05-28 16:35:37,636 [INFO] 1919874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/WAUS46_KKCI_282045___65023 \n", + "2023-05-28 16:35:37,636 [INFO] 1919874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/WAUS44_KKCI_282045___40622 \n", + "2023-05-28 16:35:37,636 [INFO] 1919874 sarracenia.flowcb.log after_work downloaded ok: /tmp/hpfx_amis/20/SRCN40_KWAL_282035___41115 \n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.flow please_stop ok, telling 5 callbacks about it.\n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.flow run starting last pass (without gather) through loop for cleanup.\n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.flow please_stop ok, telling 5 callbacks about it.\n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.flow run on_housekeeping pid: 1919874 subscribe/hpfx_amis.conf instance: 0\n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.flowcb.gather.message on_housekeeping messages: good: 12 bad: 0 bytes: 1.6 KiB average: 139 Bytes\n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.flowcb.retry on_housekeeping on_housekeeping\n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.diskqueue on_housekeeping work_retry_00 on_housekeeping\n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.diskqueue on_housekeeping No retry in list\n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.diskqueue on_housekeeping on_housekeeping elapse 0.000117\n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.diskqueue on_housekeeping post_retry_000 on_housekeeping\n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.diskqueue on_housekeeping No retry in list\n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.diskqueue on_housekeeping on_housekeeping elapse 0.000081\n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.flowcb.housekeeping.resources on_housekeeping Current Memory cpu_times: user=0.18 system=0.01\n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.flowcb.housekeeping.resources on_housekeeping Current mem usage: 54.8 MiB, accumulating count (12 or 12/100 so far) before self-setting threshold\n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.flowcb.log stats version: 3.00.40, started: 2 seconds ago, last_housekeeping: 2.9 seconds ago \n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.flowcb.log stats messages received: 12, accepted: 12, rejected: 0 rate accepted: 100.0% or 4.1 m/s\n", + "2023-05-28 16:35:37,637 [INFO] 1919874 sarracenia.flowcb.log stats files transferred: 12 bytes: 4.5 KiB rate: 1.5 KiB/sec\n", + "2023-05-28 16:35:37,638 [INFO] 1919874 sarracenia.flowcb.log stats lag: average: 1.91, maximum: 3.26 \n", + "2023-05-28 16:35:37,638 [INFO] 1919874 sarracenia.flowcb.log on_housekeeping housekeeping\n", + "2023-05-28 16:35:37,638 [INFO] 1919874 sarracenia.flow run clean stop from run loop\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-05-28 16:35:37,669 [INFO] 1919874 sarracenia.flowcb.gather.message on_stop closing\r\n", + "2023-05-28 16:35:37,669 [INFO] 1919874 sarracenia.flow close flow/close completed cleanly pid: 1919874 subscribe/hpfx_amis.conf instance: 0\r\n", + "\r\n" ] } ], @@ -436,7 +450,19 @@ "execution_count": 6, "id": "related-consensus", "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'GTStoWIS2'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[6], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01msarracenia\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mflowcb\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m FlowCB\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mlogging\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mGTStoWIS2\u001b[39;00m\n\u001b[1;32m 5\u001b[0m logger \u001b[38;5;241m=\u001b[39m logging\u001b[38;5;241m.\u001b[39mgetLogger(\u001b[38;5;18m__name__\u001b[39m)\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mWistree\u001b[39;00m(FlowCB):\n", + "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'GTStoWIS2'" + ] + } + ], "source": [ " from sarracenia.flowcb import FlowCB\n", " import logging\n", @@ -520,7 +546,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 7, "id": "disciplinary-dublin", "metadata": {}, "outputs": [], @@ -629,7 +655,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -643,7 +669,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.10.6" } }, "nbformat": 4, diff --git a/docs/source/fr/Tutoriel/3_api_flow_demo.ipynb b/docs/source/fr/Tutoriel/3_api_flow_demo.ipynb index 10b15983f..b58cd8e5c 100644 --- a/docs/source/fr/Tutoriel/3_api_flow_demo.ipynb +++ b/docs/source/fr/Tutoriel/3_api_flow_demo.ipynb @@ -21,7 +21,15 @@ "execution_count": 1, "id": "weekly-terminology", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "mkdir: cannot create directory ‘/tmp/flow_demo’: File exists\r\n" + ] + } + ], "source": [ "!mkdir /tmp/flow_demo" ] @@ -36,7 +44,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "aggregate-election", "metadata": {}, "outputs": [], @@ -49,7 +57,7 @@ "\n", "cfg = sarracenia.config.no_file_config()\n", "\n", - "cfg.broker = sarracenia.credentials.Credential('amqps://anonymous:anonymous@hpfx.collab.science.gc.ca')\n", + "cfg.broker = 'amqps://anonymous:anonymous@hpfx.collab.science.gc.ca'\n", "cfg.topicPrefix = [ 'v02', 'post']\n", "cfg.component = 'subscribe'\n", "cfg.config = 'flow_demo'\n", @@ -62,12 +70,11 @@ "# set the instance number for the flow class.\n", "cfg.no=0\n", "\n", - "# set flow class to put working files in ~/.cache/sr3/subscrribe/flow_demo directory.\n", - "cfg.pid_filename = sarracenia.config.get_pid_filename( None, cfg.component, cfg.config, 0)\n", + "cfg.finalize()\n", "\n", "# accept/reject patterns:\n", "pattern=\".*\"\n", - "# to_match, write_to_dir, DESTFN, regex_to_match, accept=True,mirror,strip, pstrip,flatten\n", + "# to_match, write_to_dir, DESTFN, regex_to_match, accept=True,mirror,strip, pstrip,flatten\n", "cfg.masks= [ ( pattern, \"/tmp/flow_demo\", None, re.compile(pattern), True, False, False, False, '/' ) ]\n", "\n", "\n" @@ -125,7 +132,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "musical-discrimination", "metadata": { "scrolled": false @@ -135,53 +142,71 @@ "name": "stderr", "output_type": "stream", "text": [ - "2022-03-19 13:21:09,354 [INFO] sarracenia.flow loadCallbacks plugins to load: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources']\n", - "2022-03-19 13:21:09,586 [DEBUG] amqp _on_start Start from server, version: 0.9, properties: {'capabilities': {'publisher_confirms': True, 'exchange_exchange_bindings': True, 'basic.nack': True, 'consumer_cancel_notify': True, 'connection.blocked': True, 'consumer_priorities': True, 'authentication_failure_close': True, 'per_consumer_qos': True, 'direct_reply_to': True}, 'cluster_name': 'rabbit@hpfx2.collab.science.gc.ca', 'copyright': 'Copyright (C) 2007-2019 Pivotal Software, Inc.', 'information': 'Licensed under the MPL. See http://www.rabbitmq.com/', 'platform': 'Erlang/OTP 21.3', 'product': 'RabbitMQ', 'version': '3.7.13'}, mechanisms: [b'AMQPLAIN', b'PLAIN'], locales: ['en_US']\n", - "2022-03-19 13:21:09,640 [DEBUG] amqp __init__ using channel_id: 1\n", - "2022-03-19 13:21:09,661 [DEBUG] amqp _on_open_ok Channel open\n", - "2022-03-19 13:21:09,751 [DEBUG] sarracenia.flowcb.retry __init__ sr_retry __init__\n", - "2022-03-19 13:21:09,751 [DEBUG] sarracenia.diskqueue __init__ work_retry_00 __init__\n", - "2022-03-19 13:21:09,759 [DEBUG] sarracenia.config add_option MemoryMax declared as type: value:0\n", - "2022-03-19 13:21:09,759 [DEBUG] sarracenia.config add_option MemoryBaseLineFile declared as type: value:100\n", - "2022-03-19 13:21:09,759 [DEBUG] sarracenia.config add_option MemoryMultiplier declared as type: value:3\n", - "2022-03-19 13:21:09,760 [DEBUG] sarracenia.config check_undeclared_options missing defaults: {'notify_only', 'retry_mode', 'dry_run', 'restore', 'header', 'nodupe_basis', 'realpathFilter', 'MemoryMax', 'MemoryBaseLineFile', 'force_polling', 'pipe', 'exchange_suffix', 'post_exchange_split', 'set_passwords', 'cache_stat', 'reconnect', 'source_from_exchange', 'sanity_log_dead', 'follow_symlinks', 'exchange_split', 'feeder', 'post_on_start', 'save', 'report_daemons', 'MemoryMultiplier', 'destination', 'inplace', 'pump_flag', 'retry_ttl', 'post_exchange', 'report_exchange', 'blocksize', 'post_exchange_suffix', 'path'}\n", - "2022-03-19 13:21:09,760 [INFO] sarracenia.flow run options:\n", - "2022-03-19 13:21:09,770 [INFO] sarracenia.flow run callbacks loaded: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources']\n", - "2022-03-19 13:21:09,770 [INFO] sarracenia.flow run pid: 2725204 subscribe/flow_demo instance: 0\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "_Config__admin=None, _Config__broker=\"...ymous:anonymous@hpfx.collab.science.gc.ca None True True False False None None\", _Config__post_broker=None,\n", - "accel_threshold=0, acceptSizeWrong=False, acceptUnmatched=False, attempts=3, auto_delete=False, baseDir=None, baseUrl_relPath=False, batch=1,\n", - "bindings=\"...('xpublic', ['v02', 'post'], ['*', 'WXO-DD', 'observations', 'swob-ml', '#'])]\", bufsize=1048576, bytes_per_second=None, bytes_ps=0,\n", - "cfg_run_dir='.', component='subscribe', config='flow_demo', currentDir=None, debug=False, declared_exchanges=[], declared_users={}, delete=False,\n", - "destfn_script=None, directory=None, discard=False, documentRoot=None, download=True, durable=True, env_declared=[], exchange=None, exchangeDeclare=True,\n", - "expire=300, fileEvents={'modify', 'delete', 'create', 'link'}, filename='WHATFN', fixed_headers={}, flatten='/', hostdir='fractal', hostname='fractal',\n", - "housekeeping=300, imports=[], inflight=None, inline=False, inline_encoding='guess', inline_max=4096, inline_only=False, instances=1,\n", - "integrity_arbitrary_value=None, integrity_method='sha512', logEvents={'after_work', 'after_accept', 'on_housekeeping'},\n", - "logFormat='%(asctime)s [%(levelname)s] %(name)s %(funcName)s %(message)s', logLevel='info', logStdout=False, log_flowcb_needed=False, log_reject=False,\n", - "lr_backupCount=5, lr_interval=1, lr_when='midnight', masks=\"...*', '/tmp/flow_demo', None, re.compile('.*'), True, False, False, False, '/')]\",\n", - "messageAgeMax=0, messageCountMax=5, messageDebugDump=False, messageRateMax=0, messageRateMin=0,\n", - "message_strategy={'reset': True, 'stubborn': True, 'failure_duration': '5m'}, message_ttl=0, mirror=True, no=0, nodupe_fileAgeMax=0, nodupe_ttl=0,\n", - "overwrite=True, permCopy=True, permDefault=0, permDirDefault=509, permLog=384,\n", - "pid_filename='/home/peter/.cache/sr3/subscribe/flow_demo//subscribe_flow_demo_00.pid', plugins_early=[], plugins_late=[], post_baseDir=None,\n", - "post_baseUrl=None, post_documentRoot=None, post_exchanges=[], post_topicPrefix=['v02', 'post'], prefetch=25, pstrip=False, queueBind=True, queueDeclare=True,\n", - "queueName='q_anonymous.subscriber_test2', randid='f32a', randomize=False, realpathPost=False, rename=None, report_back=False, reset=False, retry_path='.',\n", - "settings={}, sleep=0.1, statehost=False, strip=0, subtopic=[], timeCopy=True, timeout=300, timezone='UTC', tls_rigour='normal', topicPrefix=['v02', 'post'],\n", - "undeclared=[], users=False, v2plugin_options=[], v2plugins={}, vhost='/', vip=None\n" + "2023-05-28 16:52:19,861 [INFO] sarracenia.flow loadCallbacks flowCallback plugins to load: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'log']\n", + "2023-05-28 16:52:19,940 [DEBUG] amqp _on_start Start from server, version: 0.9, properties: {'capabilities': {'publisher_confirms': True, 'exchange_exchange_bindings': True, 'basic.nack': True, 'consumer_cancel_notify': True, 'connection.blocked': True, 'consumer_priorities': True, 'authentication_failure_close': True, 'per_consumer_qos': True, 'direct_reply_to': True}, 'cluster_name': 'rabbit@hpfx2.collab.science.gc.ca', 'copyright': 'Copyright (c) 2007-2022 VMware, Inc. or its affiliates.', 'information': 'Licensed under the MPL 2.0. Website: https://rabbitmq.com', 'platform': 'Erlang/OTP 24.2.1', 'product': 'RabbitMQ', 'version': '3.9.13'}, mechanisms: [b'AMQPLAIN', b'PLAIN'], locales: ['en_US']\n", + "2023-05-28 16:52:19,984 [DEBUG] amqp __init__ using channel_id: 1\n", + "2023-05-28 16:52:20,002 [DEBUG] amqp _on_open_ok Channel open\n", + "2023-05-28 16:52:20,048 [INFO] sarracenia.moth.amqp __getSetup queue declared q_anonymous.subscriber_test2 (as: amqps://anonymous@hpfx.collab.science.gc.ca) \n", + "2023-05-28 16:52:20,048 [INFO] sarracenia.moth.amqp __getSetup binding q_anonymous.subscriber_test2 with v02.post.*.WXO-DD.observations.swob-ml.# to xpublic (as: amqps://anonymous@hpfx.collab.science.gc.ca)\n", + "2023-05-28 16:52:20,072 [DEBUG] sarracenia.moth.amqp __getSetup getSetup ... Done!\n", + "2023-05-28 16:52:20,073 [DEBUG] sarracenia.flowcb.retry __init__ sr_retry __init__\n", + "2023-05-28 16:52:20,074 [DEBUG] sarracenia.config add_option retry_driver declared as type: value:disk\n", + "2023-05-28 16:52:20,101 [DEBUG] sarracenia.diskqueue __init__ work_retry_00 __init__\n", + "2023-05-28 16:52:20,103 [DEBUG] sarracenia.config add_option MemoryMax declared as type: value:0\n", + "2023-05-28 16:52:20,103 [DEBUG] sarracenia.config add_option MemoryBaseLineFile declared as type: value:100\n", + "2023-05-28 16:52:20,103 [DEBUG] sarracenia.config add_option MemoryMultiplier declared as type: value:3\n", + "2023-05-28 16:52:20,104 [DEBUG] sarracenia.config add_option logEvents declared as type: value:{'after_work', 'after_accept', 'on_housekeeping'}\n", + "2023-05-28 16:52:20,104 [DEBUG] sarracenia.config add_option logMessageDump declared as type: value:False\n", + "2023-05-28 16:52:20,105 [INFO] sarracenia.flowcb.log __init__ subscribe initialized with: {'after_work', 'after_accept', 'on_housekeeping'}\n", + "2023-05-28 16:52:20,105 [DEBUG] sarracenia.config check_undeclared_options missing defaults: {'post_exchangeSuffix', 'exchangeSplit', 'integrity', 'pollUrl', 'post_exchangeSplit', 'MemoryMax', 'notify_only', 'cluster', 'blocksize', 'exchange_suffix', 'report_exchange', 'realpathFilter', 'action', 'logMessageDump', 'retry_driver', 'source', 'nodupe_basis', 'MemoryMultiplier', 'reconnect', 'force_polling', 'header', 'inplace', 'post_exchange', 'save', 'post_on_start', 'follow_symlinks', 'count', 'MemoryBaseLineFile', 'feeder', 'sendTo', 'restore'}\n", + "2023-05-28 16:52:20,105 [INFO] sarracenia.flow run callbacks loaded: ['sarracenia.flowcb.gather.message.Message', 'sarracenia.flowcb.retry.Retry', 'sarracenia.flowcb.housekeeping.resources.Resources', 'log']\n", + "2023-05-28 16:52:20,105 [INFO] sarracenia.flow run pid: 1921103 subscribe/flow_demo instance: 0\n", + "2023-05-28 16:52:20,128 [DEBUG] sarracenia.moth.amqp getNewMessage new msg: {'_format': 'v02', '_deleteOnPost': {'_format', 'exchange', 'ack_id', 'local_offset', 'subtopic'}, 'sundew_extension': 'DMS:WXO_RENAMED_SWOB2:MSC:XML::20230528202430', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_ef8614a54e610cd50588f448a9632244:DMS:WXO_RENAMED_SWOB2:MSC:XML::20230528202430', 'source': 'WXO-DD', 'mtime': '20230528T202432.81', 'atime': '20230528T202432.81', 'pubTime': '20230528T202432.81', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230528/WXO-DD/observations/swob-ml/20230528/CWRZ/2023-05-28-2023-CWRZ-AUTO-minute-swob.xml', 'subtopic': ['20230528', 'WXO-DD', 'observations', 'swob-ml', '20230528', 'CWRZ'], 'integrity': {'method': 'md5', 'value': '30K1LtKs+91neD6625tbcg=='}, 'size': 7665, 'exchange': 'xpublic', 'ack_id': 1, 'local_offset': 0}\n", + "2023-05-28 16:52:20,129 [INFO] sarracenia.flowcb.log after_accept accepted: (lag: 1667.32 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/observations/swob-ml/20230528/CWRZ/2023-05-28-2023-CWRZ-AUTO-minute-swob.xml \n", + "2023-05-28 16:52:20,129 [INFO] sarracenia.flow run now active on vip None\n", + "2023-05-28 16:52:20,130 [DEBUG] sarracenia.config add_option accelWgetCommand declared as type: value:/usr/bin/wget %s -o - -O %d\n", + "2023-05-28 16:52:20,213 [INFO] sarracenia.flowcb.log after_work downloaded ok: /tmp/flow_demo/2023-05-28-2023-CWRZ-AUTO-minute-swob.xml \n", + "2023-05-28 16:52:20,233 [DEBUG] sarracenia.moth.amqp getNewMessage new msg: {'_format': 'v02', '_deleteOnPost': {'_format', 'exchange', 'ack_id', 'local_offset', 'subtopic'}, 'sundew_extension': 'DMS:CMC:SWOB_FORESTRY:XML::20230528202436', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_b82b2479d8c115cf0eb4c82bfcc59981:DMS:CMC:SWOB_FORESTRY:XML::20230528202436', 'source': 'WXO-DD', 'mtime': '20230528T202437.541', 'atime': '20230528T202437.541', 'pubTime': '20230528T202437.541', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230528/WXO-DD/observations/swob-ml/partners/on-firewx/20230528/ban/2023-05-28-2023-on-mnrf-affes-ban-ban-AUTO-swob.xml', 'subtopic': ['20230528', 'WXO-DD', 'observations', 'swob-ml', 'partners', 'on-firewx', '20230528', 'ban'], 'integrity': {'method': 'md5', 'value': 'QGDX+gsirC8l8hnDfEHa1w=='}, 'size': 5203, 'exchange': 'xpublic', 'ack_id': 2, 'local_offset': 0}\n", + "2023-05-28 16:52:20,233 [INFO] sarracenia.flowcb.log after_accept accepted: (lag: 1662.69 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/observations/swob-ml/partners/on-firewx/20230528/ban/2023-05-28-2023-on-mnrf-affes-ban-ban-AUTO-swob.xml \n", + "2023-05-28 16:52:20,311 [INFO] sarracenia.flowcb.log after_work downloaded ok: /tmp/flow_demo/2023-05-28-2023-on-mnrf-affes-ban-ban-AUTO-swob.xml \n", + "2023-05-28 16:52:20,336 [DEBUG] sarracenia.moth.amqp getNewMessage new msg: {'_format': 'v02', '_deleteOnPost': {'_format', 'exchange', 'ack_id', 'local_offset', 'subtopic'}, 'sundew_extension': 'DMS:CMC:SWOB_FORESTRY:XML::20230528202436', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_b96cfe8f3a477e2c4a39af99a97b6429:DMS:CMC:SWOB_FORESTRY:XML::20230528202436', 'source': 'WXO-DD', 'mtime': '20230528T202437.541', 'atime': '20230528T202437.541', 'pubTime': '20230528T202437.541', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230528/WXO-DD/observations/swob-ml/partners/on-firewx/20230528/ple/2023-05-28-2023-on-mnrf-affes-ple-ple-AUTO-swob.xml', 'subtopic': ['20230528', 'WXO-DD', 'observations', 'swob-ml', 'partners', 'on-firewx', '20230528', 'ple'], 'integrity': {'method': 'md5', 'value': 'eL80Iw/3MaCqipWJOT7LeQ=='}, 'size': 5091, 'exchange': 'xpublic', 'ack_id': 3, 'local_offset': 0}\n", + "2023-05-28 16:52:20,336 [INFO] sarracenia.flowcb.log after_accept accepted: (lag: 1662.80 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/observations/swob-ml/partners/on-firewx/20230528/ple/2023-05-28-2023-on-mnrf-affes-ple-ple-AUTO-swob.xml \n", + "2023-05-28 16:52:20,433 [INFO] sarracenia.flowcb.log after_work downloaded ok: /tmp/flow_demo/2023-05-28-2023-on-mnrf-affes-ple-ple-AUTO-swob.xml \n", + "2023-05-28 16:52:20,456 [DEBUG] sarracenia.moth.amqp getNewMessage new msg: {'_format': 'v02', '_deleteOnPost': {'_format', 'exchange', 'ack_id', 'local_offset', 'subtopic'}, 'sundew_extension': 'DMS:WXO_RENAMED_SWOB2:MSC:XML::20230528202442', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_0e2c0d07f3648f9956db0a2b1523e6d7:DMS:WXO_RENAMED_SWOB2:MSC:XML::20230528202442', 'source': 'WXO-DD', 'mtime': '20230528T202443.46', 'atime': '20230528T202443.46', 'pubTime': '20230528T202443.46', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230528/WXO-DD/observations/swob-ml/20230528/CXHI/2023-05-28-2024-CXHI-AUTO-minute-swob.xml', 'subtopic': ['20230528', 'WXO-DD', 'observations', 'swob-ml', '20230528', 'CXHI'], 'integrity': {'method': 'md5', 'value': 'bGYYmVHKuo3JSRDOjiC7NA=='}, 'size': 9353, 'exchange': 'xpublic', 'ack_id': 4, 'local_offset': 0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2022-03-19 13:21:10,792 [DEBUG] sarracenia.config add_option accel_wget_command declared as type: value:/usr/bin/wget %s -O %d\n", - "2022-03-19 13:21:11,380 [DEBUG] amqp collect Closed channel #1\n", - "2022-03-19 13:21:11,380 [INFO] sarracenia.flowcb.gather.message on_stop closing\n", - "2022-03-19 13:21:11,381 [INFO] sarracenia.flow close flow/close completed cleanly pid: 2725204 subscribe/flow_demo instance: 0\n" + "2023-05-28 16:52:20,457 [INFO] sarracenia.flowcb.log after_accept accepted: (lag: 1657.00 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/observations/swob-ml/20230528/CXHI/2023-05-28-2024-CXHI-AUTO-minute-swob.xml \n", + "2023-05-28 16:52:20,534 [INFO] sarracenia.flowcb.log after_work downloaded ok: /tmp/flow_demo/2023-05-28-2024-CXHI-AUTO-minute-swob.xml \n", + "2023-05-28 16:52:20,558 [DEBUG] sarracenia.moth.amqp getNewMessage new msg: {'_format': 'v02', '_deleteOnPost': {'_format', 'exchange', 'ack_id', 'local_offset', 'subtopic'}, 'sundew_extension': 'DMS:WXO_RENAMED_SWOB2:MSC:XML::20230528202442', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_1198e5a492e9a42cd6aadbbe92bcb788:DMS:WXO_RENAMED_SWOB2:MSC:XML::20230528202442', 'source': 'WXO-DD', 'mtime': '20230528T202443.47', 'atime': '20230528T202443.47', 'pubTime': '20230528T202443.47', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230528/WXO-DD/observations/swob-ml/20230528/CWBM/2023-05-28-2024-CWBM-AUTO-minute-swob.xml', 'subtopic': ['20230528', 'WXO-DD', 'observations', 'swob-ml', '20230528', 'CWBM'], 'integrity': {'method': 'md5', 'value': 'sIqUJmWCsX5BVfplkZA75Q=='}, 'size': 9354, 'exchange': 'xpublic', 'ack_id': 5, 'local_offset': 0}\n", + "2023-05-28 16:52:20,559 [INFO] sarracenia.flowcb.log after_accept accepted: (lag: 1657.09 ) https://hpfx.collab.science.gc.ca /20230528/WXO-DD/observations/swob-ml/20230528/CWBM/2023-05-28-2024-CWBM-AUTO-minute-swob.xml \n", + "2023-05-28 16:52:20,652 [INFO] sarracenia.flowcb.log after_work downloaded ok: /tmp/flow_demo/2023-05-28-2024-CWBM-AUTO-minute-swob.xml \n", + "2023-05-28 16:52:20,653 [INFO] sarracenia.flow please_stop ok, telling 4 callbacks about it.\n", + "2023-05-28 16:52:20,653 [INFO] sarracenia.flow run starting last pass (without gather) through loop for cleanup.\n", + "2023-05-28 16:52:20,654 [INFO] sarracenia.flow please_stop ok, telling 4 callbacks about it.\n", + "2023-05-28 16:52:20,654 [INFO] sarracenia.flow run on_housekeeping pid: 1921103 subscribe/flow_demo instance: 0\n", + "2023-05-28 16:52:20,655 [INFO] sarracenia.flowcb.gather.message on_housekeeping messages: good: 5 bad: 0 bytes: 783 Bytes average: 156 Bytes\n", + "2023-05-28 16:52:20,655 [INFO] sarracenia.flowcb.retry on_housekeeping on_housekeeping\n", + "2023-05-28 16:52:20,655 [INFO] sarracenia.diskqueue on_housekeeping work_retry_00 on_housekeeping\n", + "2023-05-28 16:52:20,656 [INFO] sarracenia.diskqueue on_housekeeping No retry in list\n", + "2023-05-28 16:52:20,656 [INFO] sarracenia.diskqueue on_housekeeping on_housekeeping elapse 0.000612\n", + "2023-05-28 16:52:20,656 [INFO] sarracenia.diskqueue on_housekeeping post_retry_000 on_housekeeping\n", + "2023-05-28 16:52:20,657 [INFO] sarracenia.diskqueue on_housekeeping No retry in list\n", + "2023-05-28 16:52:20,657 [INFO] sarracenia.diskqueue on_housekeeping on_housekeeping elapse 0.000450\n", + "2023-05-28 16:52:20,657 [INFO] sarracenia.flowcb.housekeeping.resources on_housekeeping Current Memory cpu_times: user=0.47 system=0.07\n", + "2023-05-28 16:52:20,657 [INFO] sarracenia.flowcb.housekeeping.resources on_housekeeping Current mem usage: 759.0 MiB, accumulating count (5 or 5/100 so far) before self-setting threshold\n", + "2023-05-28 16:52:20,658 [INFO] sarracenia.flowcb.log stats version: 3.00.40, started: now, last_housekeeping: 0.6 seconds ago \n", + "2023-05-28 16:52:20,658 [INFO] sarracenia.flowcb.log stats messages received: 5, accepted: 5, rejected: 0 rate accepted: 100.0% or 9.0 m/s\n", + "2023-05-28 16:52:20,658 [INFO] sarracenia.flowcb.log stats files transferred: 5 bytes: 35.8 KiB rate: 64.8 KiB/sec\n", + "2023-05-28 16:52:20,658 [INFO] sarracenia.flowcb.log stats lag: average: 1661.38, maximum: 1667.32 \n", + "2023-05-28 16:52:20,658 [INFO] sarracenia.flowcb.log on_housekeeping housekeeping\n", + "2023-05-28 16:52:20,659 [INFO] sarracenia.flow run clean stop from run loop\n", + "2023-05-28 16:52:20,679 [DEBUG] amqp collect Closed channel #1\n", + "2023-05-28 16:52:20,680 [INFO] sarracenia.flowcb.gather.message on_stop closing\n", + "2023-05-28 16:52:20,680 [INFO] sarracenia.flow close flow/close completed cleanly pid: 1921103 subscribe/flow_demo instance: 0\n" ] } ], @@ -213,7 +238,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -227,7 +252,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.10.6" } }, "nbformat": 4, diff --git a/docs/source/fr/Tutoriel/4_api_moth_sub_demo.ipynb b/docs/source/fr/Tutoriel/4_api_moth_sub_demo.ipynb index 1e07e0361..9baa3f2b1 100644 --- a/docs/source/fr/Tutoriel/4_api_moth_sub_demo.ipynb +++ b/docs/source/fr/Tutoriel/4_api_moth_sub_demo.ipynb @@ -38,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 9, "id": "romance-handy", "metadata": { "scrolled": true, @@ -51,7 +51,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "options: {'acceptUnmatched': True, 'batch': 25, 'bindings': [('xpublic', ['v02', 'post'], ['#'])], 'broker': None, 'exchange': None, 'expire': 300, 'inline': False, 'inline_encoding': 'guess', 'inline_max': 4096, 'logFormat': '%(asctime)s [%(levelname)s] %(name)s %(funcName)s %(message)s', 'logLevel': 'info', 'messageDebugDump': False, 'message_strategy': {'reset': True, 'stubborn': True, 'failure_duration': '5m'}, 'message_ttl': 0, 'topicPrefix': ['v02', 'post'], 'tls_rigour': 'normal', 'queue_name': 'q_anonymous_fractal_SomethingHelpfulToYou', 'subtopic': [], 'durable': True, 'prefetch': 25, 'auto_delete': False, 'vhost': '/', 'reset': False, 'declare': True, 'bind': True}\n" + "options: {'acceptUnmatched': True, 'batch': 25, 'bindings': [('xpublic', ['v02', 'post'], ['#'])], 'broker': , 'dry_run': False, 'exchange': None, 'expire': None, 'inline': False, 'inlineEncoding': 'guess', 'inlineByteMax': 4096, 'logFormat': '%(asctime)s [%(levelname)s] %(name)s %(funcName)s %(message)s', 'logLevel': 'info', 'messageDebugDump': False, 'message_strategy': {'reset': True, 'stubborn': True, 'failure_duration': '5m'}, 'message_ttl': 0, 'topicPrefix': ['v02', 'post'], 'tlsRigour': 'normal', 'auto_delete': False, 'durable': True, 'exchangeDeclare': True, 'prefetch': 25, 'queueName': 'q_anonymous_fractal_QuelquechoseDUtile', 'queueBind': True, 'queueDeclare': True, 'reset': False, 'subtopic': [], 'vhost': '/', 'queue_name': 'q_anonymous_fractal_QuelquechoseDUtile'}\n" ] } ], @@ -63,13 +63,13 @@ "import time\n", "import socket\n", "\n", - "broker = sarracenia.credentials.Credential('amqps://anonymous:anonymous@hpfx.collab.science.gc.ca')\n", "\n", "options = sarracenia.moth.default_options\n", "options.update(sarracenia.moth.amqp.default_options)\n", + "options['broker'] = sarracenia.credentials.Credential('amqps://anonymous:anonymous@hpfx.collab.science.gc.ca')\n", "options['topicPrefix'] = [ 'v02', 'post' ]\n", "options['bindings'] = [('xpublic', ['v02', 'post'], ['#'])]\n", - "options['queue_name'] = 'q_anonymous_' + socket.getfqdn() + '_SomethingHelpfulToYou'\n", + "options['queueName'] = 'q_anonymous_' + socket.getfqdn() + '_QuelquechoseDUtile'\n", "\n", "print('options: %s' % options)\n", "\n" @@ -148,52 +148,71 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 10, "id": "little-louis", "metadata": { "scrolled": true }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-05-28 15:01:16,250 [INFO] sarracenia.moth.amqp __getSetup queue declared q_anonymous_fractal_QuelquechoseDUtile (as: amqps://anonymous@hpfx.collab.science.gc.ca) \n", + "2023-05-28 15:01:16,251 [INFO] sarracenia.moth.amqp __getSetup binding q_anonymous_fractal_QuelquechoseDUtile with v02.post.# to xpublic (as: amqps://anonymous@hpfx.collab.science.gc.ca)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "message 0: {'_format': 'v02', '_deleteOnPost': {'subtopic', 'ack_id', '_format', 'local_offset', 'exchange'}, 'sundew_extension': 'CMC:HRDPS:GRIB2:BIN::20230528190111', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_5eebe93b78f7f20d6c58dff7079f17f8:CMC:HRDPS:GRIB2:BIN::20230528190111', 'source': 'WXO-DD', 'mtime': '20230528T190113.733', 'atime': '20230528T190113.733', 'pubTime': '20230528T190113.733', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230528/WXO-DD/model_hrdps/north/grib2/12/006/CMC_hrdps_north_HGT_ISBL_1000_ps2.5km_2023052812_P006-00.grib2', 'subtopic': ['20230528', 'WXO-DD', 'model_hrdps', 'north', 'grib2', '12', '006'], 'integrity': {'method': 'md5', 'value': 'DcEZ6+fx637myOUf83VyDQ=='}, 'size': 236654, 'exchange': 'xpublic', 'ack_id': 1, 'local_offset': 0}\n", + "le premier 50 octets du fichier annoncé: b'GRIB\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x03\\x9cn\\x00\\x00\\x00\\x15\\x01\\x006\\x00\\x00\\x04\\x00\\x01\\x07\\xe7\\x05\\x1c\\x0c\\x00\\x00\\x01\\x02\\x00\\x00\\x00A\\x03\\x00\\x00\\x12q1\\x00\\x00\\x00'\n", + "message 1: {'_format': 'v02', '_deleteOnPost': {'subtopic', 'ack_id', '_format', 'local_offset', 'exchange'}, 'sundew_extension': 'CMC:HRDPS:GRIB2:BIN::20230528190111', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_abed0a37b3c8b8511cc78b6f8c5c6a82:CMC:HRDPS:GRIB2:BIN::20230528190111', 'source': 'WXO-DD', 'mtime': '20230528T190114.13', 'atime': '20230528T190114.13', 'pubTime': '20230528T190114.13', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230528/WXO-DD/model_hrdps/north/grib2/12/006/CMC_hrdps_north_SPFH_ISBL_0150_ps2.5km_2023052812_P006-00.grib2', 'subtopic': ['20230528', 'WXO-DD', 'model_hrdps', 'north', 'grib2', '12', '006'], 'integrity': {'method': 'md5', 'value': 'oMQDWV/QlF9aLLGOu+Tumw=='}, 'size': 330883, 'exchange': 'xpublic', 'ack_id': 2, 'local_offset': 0}\n", + "le premier 50 octets du fichier annoncé: b'GRIB\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x05\\x0c\\x83\\x00\\x00\\x00\\x15\\x01\\x006\\x00\\x00\\x04\\x00\\x01\\x07\\xe7\\x05\\x1c\\x0c\\x00\\x00\\x01\\x02\\x00\\x00\\x00A\\x03\\x00\\x00\\x12q1\\x00\\x00\\x00'\n", + "message 2: {'_format': 'v02', '_deleteOnPost': {'subtopic', 'ack_id', '_format', 'local_offset', 'exchange'}, 'sundew_extension': 'CMC:HRDPS:GRIB2:BIN::20230528190111', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_7af6caa9fd11fae11919525225783fad:CMC:HRDPS:GRIB2:BIN::20230528190111', 'source': 'WXO-DD', 'mtime': '20230528T190113.823', 'atime': '20230528T190113.823', 'pubTime': '20230528T190113.823', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230528/WXO-DD/model_hrdps/north/grib2/12/006/CMC_hrdps_north_DEPR_ISBL_0850_ps2.5km_2023052812_P006-00.grib2', 'subtopic': ['20230528', 'WXO-DD', 'model_hrdps', 'north', 'grib2', '12', '006'], 'integrity': {'method': 'md5', 'value': 'zSw+zw6P1XlQayy+CjoLAg=='}, 'size': 194315, 'exchange': 'xpublic', 'ack_id': 3, 'local_offset': 0}\n", + "le premier 50 octets du fichier annoncé: b'GRIB\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x02\\xf7\\x0b\\x00\\x00\\x00\\x15\\x01\\x006\\x00\\x00\\x04\\x00\\x01\\x07\\xe7\\x05\\x1c\\x0c\\x00\\x00\\x01\\x02\\x00\\x00\\x00A\\x03\\x00\\x00\\x12q1\\x00\\x00\\x00'\n", + "message 3: {'_format': 'v02', '_deleteOnPost': {'subtopic', 'ack_id', '_format', 'local_offset', 'exchange'}, 'sundew_extension': 'CMC:HRDPS:GRIB2:BIN::20230528190112', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'ALL', 'filename': 'msg_ddsr-WXO-DD_e02209b0564eb4b19dd746af9eb5ee9c:CMC:HRDPS:GRIB2:BIN::20230528190112', 'source': 'WXO-DD', 'mtime': '20230528T190114.89', 'atime': '20230528T190114.89', 'pubTime': '20230528T190114.89', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20230528/WXO-DD/model_hrdps/north/grib2/12/006/CMC_hrdps_north_WDIR_TGL_40_ps2.5km_2023052812_P006-00.grib2', 'subtopic': ['20230528', 'WXO-DD', 'model_hrdps', 'north', 'grib2', '12', '006'], 'integrity': {'method': 'md5', 'value': 'GpPL5qQEOn0ALfuOzQrIHw=='}, 'size': 529466, 'exchange': 'xpublic', 'ack_id': 4, 'local_offset': 0}\n", + "le premier 50 octets du fichier annoncé: b'GRIB\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x08\\x14:\\x00\\x00\\x00\\x15\\x01\\x006\\x00\\x00\\x04\\x00\\x01\\x07\\xe7\\x05\\x1c\\x0c\\x00\\x00\\x01\\x02\\x00\\x00\\x00A\\x03\\x00\\x00\\x12q1\\x00\\x00\\x00'\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-05-28 15:01:17,548 [INFO] sarracenia.moth.amqp getCleanUp deleteing queue q_anonymous_fractal_QuelquechoseDUtile\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "message 3: {'sundew_extension': 'DMS:CMC:WXR-TRANSCODER-OBS-DATAMART-IVR:AUDIO', 'to_clusters': 'DDI,DDSR', 'x-delay': 0, 'source': 'MSC-DMS-OP', 'from_cluster': 'DDSR.CMC', 'subtopic': ['20220215', 'WXO-DD', 'hello_weather', 'observations', '03'], '_deleteOnPost': {'local_offset', 'exchange', 'ack_id', 'subtopic'}, 'pubTime': '20220215T132115.073', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20220215/WXO-DD/.hello_weather/observations/03/03097.1.OBS.PUB.FR.SPX~20220215130500-20220215150500-1464', 'integrity': {'method': 'arbitrary', 'value': 'b8e417f28fc1f42557f4f633dad2b433'}, 'size': 65812, 'exchange': 'xpublic', 'ack_id': 1, 'local_offset': 0}\n", - "first 50 bytes of corresponding file: b'OggS\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x13\\xb5>\\x99\\x00\\x00\\x00\\x00\\xaa5\\xfff\\x01PSpeex 1.2rc1\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00'\n", - "message 4: {'sundew_extension': 'cvt_nws_bulletins-sr:ENMI:SA:3:Direct:20220215132114', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'DDSR.CMC,DDI.CMC,CMC,SCIENCE,EDM', 'filename': 'msg_ddsr-WXO-DD3_8c1a10e7fe978f07309d237be3f1fddb:cvt_nws_bulletins-sr:ENMI:SA:3:Direct:20220215132114', 'source': 'WXO-DD', 'mtime': '20220215T132116.584', 'atime': '20220215T132116.584', 'subtopic': ['20220215', 'WXO-DD', 'bulletins', 'alphanumeric', '20220215', 'SA', 'ENMI', '13'], '_deleteOnPost': {'local_offset', 'exchange', 'ack_id', 'subtopic'}, 'pubTime': '20220215T132116.584', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20220215/WXO-DD/bulletins/alphanumeric/20220215/SA/ENMI/13/SANO31_ENMI_151320___52674', 'integrity': {'method': 'md5', 'value': 'Icka1HDlRhJOmEzfOAoQpg=='}, 'size': 684, 'exchange': 'xpublic', 'ack_id': 2, 'local_offset': 0}\n", - "first 50 bytes of corresponding file: b'SANO31 ENMI 151320\\nMETAR ENAN 151320Z 13007KT 9999'\n", - "message 5: {'sundew_extension': 'cvt_nws_bulletins-sr:KWAL:SX:3:Direct:20220215132114', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'DDSR.CMC,DDI.CMC,CMC,SCIENCE,EDM', 'filename': 'msg_ddsr-WXO-DD3_e0ed771ab87a1a0b8e68f5a9b91a65c9:cvt_nws_bulletins-sr:KWAL:SX:3:Direct:20220215132114', 'source': 'WXO-DD', 'mtime': '20220215T132116.585', 'atime': '20220215T132116.585', 'subtopic': ['20220215', 'WXO-DD', 'bulletins', 'alphanumeric', '20220215', 'SX', 'KWAL', '13'], '_deleteOnPost': {'local_offset', 'exchange', 'ack_id', 'subtopic'}, 'pubTime': '20220215T132116.585', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20220215/WXO-DD/bulletins/alphanumeric/20220215/SX/KWAL/13/SXCN40_KWAL_151320___38969', 'integrity': {'method': 'md5', 'value': 'ccUpl7Q8zGzHIf4SUbv3og=='}, 'size': 144, 'exchange': 'xpublic', 'ack_id': 3, 'local_offset': 0}\n", - "first 50 bytes of corresponding file: b'SXCN40 KWAL 151320\\nCAC2186E 046132053\\n-032.9\\n057\\n0'\n", - "message 6: {'sundew_extension': 'cvt_nws_bulletins-sr:KWNB:SX:3:Direct:20220215132115', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'DDSR.CMC,DDI.CMC,CMC,SCIENCE,EDM', 'filename': 'msg_ddsr-WXO-DD3_4bb24c4edaf3275b72c6ed792abe3df5:cvt_nws_bulletins-sr:KWNB:SX:3:Direct:20220215132115', 'source': 'WXO-DD', 'mtime': '20220215T132117.587', 'atime': '20220215T132117.587', 'subtopic': ['20220215', 'WXO-DD', 'bulletins', 'alphanumeric', '20220215', 'SX', 'KWNB', '13'], '_deleteOnPost': {'local_offset', 'exchange', 'ack_id', 'subtopic'}, 'pubTime': '20220215T132117.587', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20220215/WXO-DD/bulletins/alphanumeric/20220215/SX/KWNB/13/SXUS23_KWNB_151300_RRP__54105', 'integrity': {'method': 'md5', 'value': '34bEHeB5cwI9WWKKk97OCw=='}, 'size': 2451, 'exchange': 'xpublic', 'ack_id': 4, 'local_offset': 0}\n", - "first 50 bytes of corresponding file: b'SXUS23 KWNB 151300 RRP\\nCMAN 15134\\nLCNA2 46/// /261'\n", - "message 7: {'sundew_extension': 'cvt_nws_bulletins-sr:KWAL:SR:3:Direct:20220215132115', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'DDSR.CMC,DDI.CMC,CMC,SCIENCE,EDM', 'filename': 'msg_ddsr-WXO-DD3_60f837832b38c3d9acc8b2d1f3190be1:cvt_nws_bulletins-sr:KWAL:SR:3:Direct:20220215132115', 'source': 'WXO-DD', 'mtime': '20220215T132117.588', 'atime': '20220215T132117.588', 'subtopic': ['20220215', 'WXO-DD', 'bulletins', 'alphanumeric', '20220215', 'SR', 'KWAL', '13'], '_deleteOnPost': {'local_offset', 'exchange', 'ack_id', 'subtopic'}, 'pubTime': '20220215T132117.588', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20220215/WXO-DD/bulletins/alphanumeric/20220215/SR/KWAL/13/SRCN40_KWAL_151320___62678', 'integrity': {'method': 'md5', 'value': 'IP+n0cisIJMGn5VXJy5WeQ=='}, 'size': 262, 'exchange': 'xpublic', 'ack_id': 5, 'local_offset': 0}\n", - "first 50 bytes of corresponding file: b'SRCN40 KWAL 151320\\n484010DC 046132052 :HG 5 #5 0.'\n", - "message 8: {'sundew_extension': 'DMS:WXO_RENAMED_SWOB2:MSC:XML::20220215132117', 'from_cluster': 'DDSR.CMC', 'to_clusters': 'DDSR.CMC,DDI.CMC,CMC,SCIENCE,EDM', 'filename': 'msg_ddsr-WXO-DD_7aee783b17658fbf30e138f39d872a6b:DMS:WXO_RENAMED_SWOB2:MSC:XML::20220215132117', 'source': 'WXO-DD', 'mtime': '20220215T132119.385', 'atime': '20220215T132119.385', 'subtopic': ['20220215', 'WXO-DD', 'observations', 'swob-ml', '20220215', 'CWTN'], '_deleteOnPost': {'local_offset', 'exchange', 'ack_id', 'subtopic'}, 'pubTime': '20220215T132119.385', 'baseUrl': 'https://hpfx.collab.science.gc.ca', 'relPath': '/20220215/WXO-DD/observations/swob-ml/20220215/CWTN/2022-02-15-1320-CWTN-AUTO-minute-swob.xml', 'integrity': {'method': 'md5', 'value': 'S1HVKvy8ELiUs40fJF7Wow=='}, 'size': 8528, 'exchange': 'xpublic', 'ack_id': 6, 'local_offset': 0}\n", - "first 50 bytes of corresponding file: b' 'Configuration': + def __init__(self, parent=None) -> 'Config': """ instantiate an empty Configuration """ @@ -1557,13 +1588,22 @@ def parse_file(self, cfg, component=None): setattr(self, k, v) self.undeclared.append(k) - def fill_missing_options(self, component, config): + def finalize(self, component=None, config=None): """ + Before final use, take the existing settings, and infer any missing needed defaults from what is provided. + Should be called prior to using a configuration. + There are default options that apply only if they are not overridden... """ self._parse_sum(None) + if not component and self.component: + component = self.component + + if not config and self.config: + config = self.config + if hasattr(self, 'nodupe_ttl'): if (type(self.nodupe_ttl) is str): if isTrue(self.nodupe_ttl): @@ -2489,7 +2529,7 @@ def one_config(component, config, isPost=False): if not hasattr(cfg,'broker') or (cfg.broker is None): cfg.broker = cfg.post_broker - cfg.fill_missing_options(component, config) + cfg.finalize(component, config) if component in ['post', 'watch']: cfg.postpath = list( map( os.path.expanduser, cfg.configurations[1:])) diff --git a/sarracenia/credentials.py b/sarracenia/credentials.py index 1da0f6460..f2da182b2 100755 --- a/sarracenia/credentials.py +++ b/sarracenia/credentials.py @@ -63,6 +63,17 @@ class Credential: bearer_token (str): bearer token for HTTP authentication login_method (str): force a specific login method for AMQP (PLAIN, AMQPLAIN, EXTERNAL or GSSAPI) + + Usage: + + # build a credential from a url string: + + from sarracenia.credentials import Credential + + broker = Credential('amqps://anonymous:anonymous@hpfx.collab.science.gc.ca') + + + """ def __init__(self, urlstr=None): @@ -120,6 +131,17 @@ class CredentialDB: Attributes: credentials (dict): contains all sarracenia.credentials.Credential objects managed by the CredentialDB. + + Usage: + # build a credential via lookup in the normal files: + import CredentialDB from sarracenia.credentials + + credentials = CredentialDB.read( "/the/path/to/the/credentials.conf" ) + + # if there are corresponding passwords or modulation of login information look it up. + + broker = credentials.get( "amqps://hpfx.collab.science.gc.ca" ) + remote = credentials.get( "sftp://hoho@theserver" ) """ def __init__(self, Unused_logger=None): @@ -188,7 +210,7 @@ def get(self, urlstr): # resolved from defined credentials - ok, details = self.resolve(urlstr, url) + ok, details = self._resolve(urlstr, url) if ok: return True, details # not found... is it valid ? @@ -278,7 +300,7 @@ def isValid(self, url, details=None): return True - def parse(self, line): + def _parse(self, line): """Parse a line of a credentials file, add it to the CredentialDB. Args: @@ -370,13 +392,13 @@ def read(self, path): lines = f.readlines() for line in lines: - self.parse(line) + self._parse(line) except: logger.error("credentials/read path = %s" % path) logger.debug('Exception details: ', exc_info=True) #logger.debug("Credentials = %s\n" % self.credentials) - def resolve(self, urlstr, url=None): + def _resolve(self, urlstr, url=None): """Resolve credentials for AMQP vhost from ones passed as a string, and optionally a urllib.parse.ParseResult object, into a sarracenia.credentials.Credential object. diff --git a/sarracenia/examples/flow_api_consumer.py b/sarracenia/examples/flow_api_consumer.py index 4da5c91b0..b200e9052 100644 --- a/sarracenia/examples/flow_api_consumer.py +++ b/sarracenia/examples/flow_api_consumer.py @@ -21,9 +21,8 @@ # set the instance number for the flow class. cfg.no = 0 -# set flow class to put working files in ~/.cache/sr3/subscrribe/flow_demo directory. -cfg.pid_filename = sarracenia.config.get_pid_filename(None, cfg.component, - cfg.config, 0) +# set other settings based on what is provided, for things like state files. +cfg.finalize() # accept/reject patterns: pattern = ".*" diff --git a/sarracenia/examples/moth_api_consumer.py b/sarracenia/examples/moth_api_consumer.py index 2276efa60..cf0d6f506 100644 --- a/sarracenia/examples/moth_api_consumer.py +++ b/sarracenia/examples/moth_api_consumer.py @@ -13,7 +13,7 @@ options = sarracenia.moth.default_options options.update(sarracenia.moth.amqp.default_options) -broker = sarracenia.credentials.Credential( +options['broker'] = sarracenia.credentials.Credential( 'amqps://anonymous:anonymous@hpfx.collab.science.gc.ca') # binding tuple: consists of prefix, exchange, rest. @@ -32,7 +32,7 @@ print('options: %s' % options) -h = sarracenia.moth.Moth.subFactory(broker, options) +h = sarracenia.moth.Moth.subFactory(options) count = 0 while count < 5: diff --git a/sarracenia/examples/moth_api_producer.py b/sarracenia/examples/moth_api_producer.py index 2346b6593..c52d29efd 100644 --- a/sarracenia/examples/moth_api_producer.py +++ b/sarracenia/examples/moth_api_producer.py @@ -33,7 +33,7 @@ print('cfg: %s' % cfg) # moth wants a dict as options, rather than sarracenia.config.Config instance. -posting_engine = sarracenia.moth.Moth.pubFactory(cfg.broker, cfg.dictify()) +posting_engine = sarracenia.moth.Moth.pubFactory(cfg.dictify()) sample_fileName = '/tmp/sample.txt' sample_file = open(sample_fileName, 'w') diff --git a/sarracenia/examples/moth_http_retrieval.py b/sarracenia/examples/moth_http_retrieval.py index 9273092af..ad0964f1d 100644 --- a/sarracenia/examples/moth_http_retrieval.py +++ b/sarracenia/examples/moth_http_retrieval.py @@ -11,11 +11,11 @@ import urllib.request import xml.etree.ElementTree as ET -broker = sarracenia.credentials.Credential( - 'amqps://anonymous:anonymous@hpfx.collab.science.gc.ca') options = sarracenia.moth.default_options options.update(sarracenia.moth.amqp.default_options) +options['broker'] = sarracenia.credentials.Credential( + 'amqps://anonymous:anonymous@hpfx.collab.science.gc.ca') options['topicPrefix'] = ['v02', 'post'] options['bindings'] = [('xpublic', ['v02', 'post'], ['*', 'WXO-DD', 'observations', 'swob-ml', '#'])] @@ -24,7 +24,7 @@ #print('options: %s' % options) -h = sarracenia.moth.Moth.subFactory(broker, options) +h = sarracenia.moth.Moth.subFactory(options) count = 0 diff --git a/sarracenia/flow/__init__.py b/sarracenia/flow/__init__.py index 584cc5955..ae694574a 100644 --- a/sarracenia/flow/__init__.py +++ b/sarracenia/flow/__init__.py @@ -539,8 +539,7 @@ def run(self): run_time = now - start_time total_messages += last_gather_len - if (self.o.messageCountMax > 0) and (total_messages >= - self.o.messageCountMax): + if (self.o.messageCountMax > 0) and (total_messages > self.o.messageCountMax): self.please_stop() current_rate = total_messages / run_time diff --git a/sarracenia/flowcb/accept/posthourtree.py b/sarracenia/flowcb/accept/posthourtree.py index 19037544b..8ec968a92 100755 --- a/sarracenia/flowcb/accept/posthourtree.py +++ b/sarracenia/flowcb/accept/posthourtree.py @@ -6,7 +6,7 @@ input A/B/c.gif --> output A/B//c.gif Usage: - flowcb sarracenia.flowcb.accept.posthourtree.PostHourTree + callback accept.posthourtree """ import logging @@ -16,14 +16,11 @@ logger = logging.getLogger(__name__) -class PostHourTree(FlowCB): +class Posthourtree(FlowCB): def after_accept(self, worklist): for message in worklist.incoming: - datestr = time.strftime('%H', time.localtime()) # pick the hour + datestr = time.strftime('%H', time.gmtime()) # pick the hour # insert the hour into the rename header of the message to be posted. - new_fname = message['headers']['rename'].split('/') - message['headers']['rename'] = '/'.join( - new_fname[0:-1]) + '/' + datestr + '/' + new_fname[-1] - logger.info("post_hour_tree: rename: %s" % - message['headers']['rename']) + message['new_dir'] += '/' + datestr + logger.info( f"post_hour_tree: new_dir: {message['new_dir']}" ) diff --git a/sarracenia/flowcb/gather/message.py b/sarracenia/flowcb/gather/message.py index ed0a692c5..9d7c5ff34 100755 --- a/sarracenia/flowcb/gather/message.py +++ b/sarracenia/flowcb/gather/message.py @@ -23,7 +23,7 @@ def __init__(self, options) -> None: if hasattr(self.o, 'broker') and self.o.broker: self.od = sarracenia.moth.default_options self.od.update(self.o.dictify()) - self.consumer = sarracenia.moth.Moth.subFactory(self.o.broker, self.od) + self.consumer = sarracenia.moth.Moth.subFactory(self.od) else: logger.critical('missing required broker specification') @@ -35,7 +35,7 @@ def gather(self) -> list: return self.consumer.newMessages() else: logger.warning( f'not connected. Trying to connect to {self.o.broker}') - self.consumer = sarracenia.moth.Moth.subFactory(self.o.broker, self.od) + self.consumer = sarracenia.moth.Moth.subFactory(self.od) return [] def ack(self, mlist) -> None: diff --git a/sarracenia/flowcb/post/message.py b/sarracenia/flowcb/post/message.py index 399980525..fa681f5b7 100755 --- a/sarracenia/flowcb/post/message.py +++ b/sarracenia/flowcb/post/message.py @@ -31,8 +31,7 @@ def __init__(self, options): props.update({ 'exchangeSplit': self.o.post_exchangeSplit, }) - self.poster = sarracenia.moth.Moth.pubFactory( - self.o.post_broker, props) + self.poster = sarracenia.moth.Moth.pubFactory(props) def post(self, worklist): diff --git a/sarracenia/moth/__init__.py b/sarracenia/moth/__init__.py index 3afc93378..620e7ff12 100755 --- a/sarracenia/moth/__init__.py +++ b/sarracenia/moth/__init__.py @@ -84,20 +84,31 @@ class Moth(): * subTopic - * id (queue for amqp, id for mqtt) + * queueName (for amqp, used as client-id for mqtt) this library knows nothing about Sarracenia, the only code used from sarracenia is to interpret duration properties, from the root sarracenia/__init__.py, the broker argument from sarracenia.credentials usage:: - c= Moth( broker, True, '5m', { 'batch':1 } ) + import sarracenia.moth + import sarracenia.credentials + + + props = sarracenia.moth.default_options + props['broker'] = sarracenia.credentials.Credential('amqps://anonymous:anonymous@hpfx.collab.science.gc.ca') + props['expire'] = 300 + props['batch'] = 1 + is_subscriber=True + + c= Moth( props, is_subscriber ) + + messages = c.newMessages() - c.newMessages() # if there are new messages from a publisher, return them, otherwise return # an empty list []]. - p=Moth( broker, True, '5m', { 'batch':1 }, True ) + p=Moth( { 'batch':1 }, False ) p.putNewMessage() @@ -175,7 +186,7 @@ class Moth(): * 'broker' : an sr_broker ? - * 'Queue' : Mandatory, name of a queue. (only in AMQP... hmm...) + * 'queueName' : Mandatory, name of a queue. (only in AMQP... hmm...) * 'bindings' : [ list of bindings ] @@ -192,53 +203,53 @@ class Moth(): """ @staticmethod - def subFactory(broker, props) -> 'Moth': + def subFactory(props) -> 'Moth': - if not broker : + if not props['broker'] : logger.error('no broker specified') return None - if not hasattr(broker,'url'): + if not hasattr(props['broker'],'url'): logger.error('invalid broker url') return None - if not ProtocolPresent(broker.url.scheme): + if not ProtocolPresent(props['broker'].url.scheme): logger.error('unknown broker scheme/protocol specified') return None for sc in Moth.__subclasses__(): - if (broker.url.scheme == sc.__name__.lower()) or ( - (broker.url.scheme[0:-1] == sc.__name__.lower()) and - (broker.url.scheme[-1] == 's')): - return sc(broker, props, True) + if (props['broker'].url.scheme == sc.__name__.lower()) or ( + (props['broker'].url.scheme[0:-1] == sc.__name__.lower()) and + (props['broker'].url.scheme[-1] == 's')): + return sc(props, True) logger.error('broker intialization failure') return None @staticmethod - def pubFactory(broker, props) -> 'Moth': - if not broker: + def pubFactory(props) -> 'Moth': + if not props['broker']: logger.error('no broker specified') return None - if not hasattr(broker,'url'): + if not hasattr(props['broker'],'url'): logger.error('invalid broker url') return None - if not ProtocolPresent(broker.url.scheme): + if not ProtocolPresent(props['broker'].url.scheme): logger.error('unknown broker scheme/protocol specified') return None for sc in Moth.__subclasses__(): - if (broker.url.scheme == sc.__name__.lower()) or ( - (broker.url.scheme[0:-1] == sc.__name__.lower()) and - (broker.url.scheme[-1] == 's')): - return sc(broker, props, False) + if (props['broker'].url.scheme == sc.__name__.lower()) or ( + (props['broker'].url.scheme[0:-1] == sc.__name__.lower()) and + (props['broker'].url.scheme[-1] == 's')): + return sc(props, False) # ProtocolPresent test should ensure that we never get here... logger.error('broker intialization failure') return None - def __init__(self, broker, props=None, is_subscriber=True) -> None: + def __init__(self, props=None, is_subscriber=True) -> None: """ If is_subscriber=True, then this is a consuming instance. expect calls to get* routines. @@ -264,8 +275,6 @@ def __init__(self, broker, props=None, is_subscriber=True) -> None: if props is not None: self.o.update(props) - self.broker = broker - me = 'sarracenia.moth.Moth' # apply settings from props. diff --git a/sarracenia/moth/amqp.py b/sarracenia/moth/amqp.py index c55eea1b5..a6e34f072 100755 --- a/sarracenia/moth/amqp.py +++ b/sarracenia/moth/amqp.py @@ -147,13 +147,13 @@ def _msgRawToDict(self, raw_msg) -> sarracenia.Message: # length of an AMQP short string (used for headers and many properties) amqp_ss_maxlen = 255 - def __init__(self, broker, props, is_subscriber) -> None: + def __init__(self, props, is_subscriber) -> None: """ connect to broker, depending on message_strategy stubborness, remain connected. """ - super().__init__(broker, props, is_subscriber) + super().__init__(props, is_subscriber) logging.basicConfig( format= @@ -253,7 +253,7 @@ def __getSetup(self) -> None: # tear the whole thing down, and start over. try: # from sr_consumer.build_connection... - if not self.__connect(self.broker): + if not self.__connect(self.o['broker']): logger.critical('could not connect') break @@ -263,8 +263,8 @@ def __getSetup(self) -> None: self.channel.basic_qos(0, self.o['prefetch'], True) #FIXME: test self.first_setup and props['reset']... delete queue... - broker_str = self.broker.url.geturl().replace( - ':' + self.broker.url.password + '@', '@') + broker_str = self.o['broker'].url.geturl().replace( + ':' + self.o['broker'].url.password + '@', '@') # from Queue declare if self.o['queueDeclare'] and self.o['queueName']: @@ -317,7 +317,7 @@ def __getSetup(self) -> None: f'connecting to: {self.o["queueName"]}, durable: {self.o["durable"]}, expire: {self.o["expire"]}, auto_delete={self.o["auto_delete"]}' ) logger.error("AMQP getSetup failed to {} with {}".format( - self.broker.url.hostname, err)) + self.o['broker'].url.hostname, err)) logger.debug('Exception details: ', exc_info=True) if not self.o['message_strategy']['stubborn']: return @@ -358,8 +358,8 @@ def __putSetup(self) -> None: # transaction mode... confirms would be better... self.channel.tx_select() - broker_str = self.broker.url.geturl().replace( - ':' + self.broker.url.password + '@', '@') + broker_str = self.o['broker'].url.geturl().replace( + ':' + self.o['broker'].url.password + '@', '@') #logger.debug('putSetup ... 1. connected to {}'.format(broker_str ) ) diff --git a/sarracenia/moth/mqtt.py b/sarracenia/moth/mqtt.py index cfa2a3366..8bbe41a96 100755 --- a/sarracenia/moth/mqtt.py +++ b/sarracenia/moth/mqtt.py @@ -125,6 +125,7 @@ def __init__(self, broker, options, is_subscriber): #self.o = options self.o.update(default_options) self.o.update(options) + self.o['broker'] = broker if 'qos' in self.o: if type(self.o['qos']) is not int: diff --git a/sarracenia/sr.py b/sarracenia/sr.py index 39acaab72..c1ce2f739 100755 --- a/sarracenia/sr.py +++ b/sarracenia/sr.py @@ -322,7 +322,7 @@ def _read_configs(self): }) cfgbody.applyComponentDefaults( c ) cfgbody.parse_file(cfg,c) - cfgbody.fill_missing_options(c, cfg) + cfgbody.finalize(c, cfg) self.configs[c][cbase]['options'] = cfgbody # ensure there is a known value of instances to run. if c in ['poll', 'post', 'cpost']: @@ -1270,7 +1270,7 @@ def declare(self): o, 'resolved_exchanges') and o.resolved_exchanges is not None: xdc = sarracenia.moth.Moth.pubFactory( - o.post_broker, { + { 'broker': o.post_broker, 'dry_run': self.options.dry_run, 'exchange': o.resolved_exchanges, @@ -1292,9 +1292,10 @@ def declare(self): o = self.configs[c][cfg]['options'] od = o.dictify() if hasattr(o, 'resolved_qname'): + od['broker'] = o.broker od['queueName'] = o.resolved_qname od['dry_run'] = self.options.dry_run - qdc = sarracenia.moth.Moth.subFactory(o.broker, od) + qdc = sarracenia.moth.Moth.subFactory(od) qdc.close() def disable(self): @@ -1456,7 +1457,8 @@ def cleanup(self): if hasattr(o, 'resolved_qname'): #print('deleting: %s is: %s @ %s' % (f, o.resolved_qname, o.broker.url.hostname )) qdc = sarracenia.moth.Moth.subFactory( - o.broker, { + { + 'broker': o.broker, 'dry_run': self.options.dry_run, 'echangeDeclare': False, 'queueDeclare': False, @@ -1487,7 +1489,8 @@ def cleanup(self): if o.post_broker and len(xx) < 1: print("No local queues found for exchange %s, attemping to remove it..." % x) qdc = sarracenia.moth.Moth.pubFactory( - o.post_broker, { + { + 'broker': o.post_broker, 'declare': False, 'exchange': x, 'dry_run': self.options.dry_run, @@ -2379,7 +2382,7 @@ def convert(self): logger.info("obsolete v2: " + v) continue else: - logger.error( f"unknown {k} {v}, manual conversion required.") + logger.warning( f"unknown {k} {v}, manual conversion required.") v3_cfg.write( f"# PROBLEM: unknown {k} {v}, manual conversion required.\n") else: line = convert_to_v3[k] @@ -2592,7 +2595,7 @@ def main(): cfg.parse_args() #FIXME... hmm... so... - #cfg.fill_missing_options() + #cfg.finalize() if not hasattr(cfg, 'action'): print('USAGE: %s [ -h ] (%s)' % (sys.argv[0], '|'.join(actions)))