Merge branch 'development' of https://github.com/MetPX/Sarracenia int…

…o development

debian/changelog

@@ -1,4 +1,11 @@
-metpx-sr3 (3.00.54rc3) UNRELEASED; urgency=medium
+metpx-sr3 (3.00.55rc1) unstable; urgency=medium
+
+  * fix #1151 calculate on download not working.
+  * fix #1150 bc transport api refactor.
+
+ -- SSC-5CD2310S60 <[email protected]>  Tue, 08 Aug 2024 11:24:36 -0400
+
+metpx-sr3 (3.00.54rc3) unstable; urgency=medium
 
   * fix #1142 token authentication re-factor
   * fix #1141 subscribe crash with random checksum.

sarracenia/__init__.py

@@ -426,8 +426,8 @@ def computeIdentity(msg, path, o, offset=0, data=None) -> None:
                     logger.debug("mtime remembered by xattr")
                     fxainteg = xattr.get('identity')
                     if fxainteg['method'] == o.identity_method: 
-                         msg['identity'] = fxainteg
-                         return
+                        msg['identity'] = fxainteg
+                        return
                     logger.debug("xattr different method than on disk")
                     calc_method = o.identity_method
                 else:

sarracenia/_version.py

@@ -1 +1 @@
-__version__ = "3.00.54rc3"
+__version__ = "3.00.55rc1"

sarracenia/examples/flow/bc_trans.conf

@@ -0,0 +1,27 @@
+#
+# Post the BC Trans URL to the sarra.
+# The sarra will authenticate through the authenticate plugin after
+#
+
+vip # Enter your vip here
+
+# You want to callback the scheduled flow AND the post plugin so the URL gets posted downstream
+callback scheduled
+callback post.message
+
+
+# Run everyhour
+scheduled_interval 3600
+
+# post_baseUrl + path combined make up the URL to be posted downstream
+post_baseUrl https://sawsx.api.gov.bc.ca/
+path /api/v1/motisite/report7100/all/3/
+
+# to ignore colons in the filenames generated.
+filename None
+
+# Where to announce the messages
+post_broker amqp://PROVINCIAL@localhost/
+post_exchange xs_PROVINCIAL
+
+logLevel debug

sarracenia/examples/sarra/bc_trans.conf

@@ -0,0 +1,38 @@
+# Flow callbacks
+callback authenticate.bc_trans
+callback accept.bc_trans
+filename                None
+
+# Source
+broker                  amqp://PROVINCIAL@localhost/
+exchange                xs_PROVINCIAL
+queueName               q_${BROKER_USER}_${PROGRAM}_${CONFIG}.ddsr-shared
+source PROVINCIAL
+
+
+# Filtering
+# Subtopic is the path from the scheduled flow
+subtopic               api.v1.motisite.report7100.#
+directory /tmp/${%Y%m%d}/${SOURCE}/AIRNOW/CSV/RWIN/${%H}
+accept                  .*
+
+# Destination
+post_broker		amqp://feeder@localhost/
+post_exchange		xpublic
+post_baseUrl		http://${HOSTNAME}
+post_baseDir		/
+
+
+# Additional options
+acceptUnmatched         False
+delete                  False
+mirror                  False
+inflight                .tmp
+instances               1
+timeout                 60
+
+# Set nodupe so that files over 5 mins are expired. The bearer token only lasts 5 minutes, so no point in having it longer in cache.
+nodupe_basis                  path
+nodupe_ttl                    300
+
+logLevel                debug

sarracenia/flow/__init__.py

@@ -2096,7 +2096,10 @@ def download(self, msg, options) -> bool:
             if self.o.identity_method.startswith('cod,'):
                 download_algo = self.o.identity_method[4:]
             elif 'identity' in msg:
-                download_algo = msg['identity']['method']
+                if msg['identity']['method'] == 'cod':
+                    download_algo = msg['identity']['value']
+                else:
+                    download_algo = msg['identity']['method']
             else:
                 download_algo = None
 

sarracenia/flowcb/accept/bc_trans.py

@@ -0,0 +1,50 @@
+#!/usr/bin/python3
+
+"""
+Renamer for new BC Trans API
+=====================================================================================
+
+Description
+
+    Rename the file fetched from the BC Trans scheduled flow to a CSV format, date included in filename.
+
+
+How to set up in your config:
+--------------------------------
+ 
+    Use ``callback accept.bc_trans``, and read about the config options above.
+    
+    For an example, see https://github.com/MetPX/sarracenia/tree/development/sarracenia/examples/flow files named ``*bc_trans*.conf``. 
+
+    Your ``subtopic`` should match the ``path`` from the scheduled flow plugin.
+
+Change log:
+-----------
+    - 2024-08-06: Inception of plugin.
+"""
+
+from sarracenia.flowcb import FlowCB
+import logging
+import requests,os,datetime,sys,time
+
+logger = logging.getLogger(__name__)
+
+class Bc_trans(FlowCB):
+    def __init__(self, options):
+        super().__init__(options, logger)
+
+        # end __init__
+
+
+    def after_accept(self, worklist):
+
+        for msg in worklist.incoming:
+
+            # Modify the filename with the specified format
+            try: 
+
+                now = datetime.datetime.now()
+                msg['new_file'] = f"OB.BC.MOT.BC_TRAN.{now.strftime('%Y%m%d%H%M%S')}.csv"
+
+            except Exception as e:
+                logger.debug("Exception details:", exc_info=True)

sarracenia/flowcb/authenticate/bc_trans.py

@@ -0,0 +1,126 @@
+#!/usr/bin/python3
+
+"""
+Authentication for new BC Trans API
+=====================================================================================
+
+Description
+
+    The BC Trans data provider is migrating to a public API. We used to have sarracenia polls fetching from their HTTP site. This has since been discontinued.
+    The API works by
+    1. Providing the client secret to the endpoint server.
+    2. The server responds with an access token.
+    3. Pass the access token to https://sawsx-services-dev-api-gov-bc-ca.test.api.gov.bc.ca/api/v1/motisite/report7100/al/X to get last X hours of data from provider.
+
+    The access token has a life span of 5 minutes. You can also generate however many access tokens you'd like given you have access to the client_secret and client_id.
+    As of now, the only data that can be fetched is 3 hours worth of past data.
+    The pollUrl should remain the same , unless other timestamps are specified in the future.
+
+    This plugin uses the BearerToken authentication class entry points.
+
+Documentation available from the Government of BC website
+    API tutorial (with examples) : https://developer.gov.bc.ca/docs/default/component/aps-infra-platform-docs/tutorials/quick-start/
+    Developer portal : https://api.gov.bc.ca/devportal/api-directory
+
+
+Configurable Options:
+----------------------
+
+``tokenEndpoint_baseUrl`` URL:
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+    URL where to fetch the access token.
+
+``tokenEndpoint_path`` URL:
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+    Path where to fetch the access token. To use in combination with token_Endpoint_baseUrl
+
+
+*Potential Room For Improvement*:
+----------------------------------
+
+    - Download from multiple different hour marks?
+
+How to set up your flow config:
+--------------------------------
+
+    Use ``callback authenticate.bc_trans``, and read about the config options above.
+
+    For example, see https://github.com/MetPX/sarracenia/tree/development/sarracenia/examples/flow files named ``*bc_trans*.conf``.
+
+Change log:
+-----------
+    - 2024-08-06: Inception of authenticate plugin.
+"""
+
+
+import sarracenia
+from sarracenia.flowcb.authenticate import BearerToken
+import logging
+import requests
+
+logger = logging.getLogger(__name__)
+
+class Bc_trans(BearerToken):
+    def __init__(self, options):
+        super().__init__(options, logger)
+
+        # Allow setting a logLevel *only* for this plugin in the config file:
+        # set accept.auth_eumetsat.logLevel debug
+        if hasattr(self.o, 'logLevel'):
+            logger.setLevel(self.o.logLevel.upper())
+
+        # For additional options (defined by user)
+        self.o.add_option('tokenEndpoint_baseUrl', 'str', 'https://loginproxy.gov.bc.ca/') # required
+        self.o.add_option('tokenEndpoint_path', 'str', 'auth/realms/apigw/protocol/openid-connect/token') # required
+
+        # Set initial values
+        self._bearer_token = None
+
+        # Make sure values already exist
+        if not self.o.tokenEndpoint_baseUrl or not self.o.tokenEndpoint_path:
+            logger.error(f"tokenEndpoint_baseUrl and tokenEndpoint_path is a required option.")
+            sys.exit(1)
+
+        # Get credentials from credentials.conf.
+        # In this case the client ID is the traditional url username and the client secret is the traditional password
+        # This avoids creating new fields in the credentials class
+        if self.o.tokenEndpoint_baseUrl and self.o.tokenEndpoint_path:
+            ok, self.details = sarracenia.config.Config.credentials.get(
+                self.o.tokenEndpoint_baseUrl + self.o.tokenEndpoint_path)
+
+        self.client_id = self.details.url.username
+        self.client_secret = self.details.url.password
+        self.grant_type = 'client_credentials'
+
+        # end __init__
+
+
+    def get_token(self):
+        """
+        Fetches the bearer token from the token endpoint. Uses the parent class entry point.
+        NOTE : Multiple bearer tokens can be fetched at the same time.
+        NOTE : The bearer token has a lifespan of 5 minutes.
+        """
+        try:
+            logger.info("Requesting a new bearer token")
+
+            response = requests.post(self.o.tokenEndpoint_baseUrl + self.o.tokenEndpoint_path,\
+                data={'grant_type': f'{self.grant_type}'},\
+                auth=(self.client_id, self.client_secret))
+
+            # Get response information
+            response_status = response.status_code
+            response_text = response.text
+
+            if response_status == 200:
+                self._bearer_token = response.json()["access_token"]
+            else:
+                logger.error( f"Request status received: {response_status}. Response message: {response_text}" )
+                self._bearer_token = None
+
+        except Exception as e:
+            logger.debug("Exception details:", exc_info=True)
+
+        return self._bearer_token

tests/sarracenia/flowcb/accept/auth_copernicus_test.py → tests/sarracenia/flowcb/authenticate/bc_trans_test.py

@@ -3,4 +3,4 @@
 #from unittest.mock import Mock
 
 import sarracenia.config
-import sarracenia.flowcb.accept.auth_copernicus
+import sarracenia.flowcb.authenticate.bc_trans

tests/sarracenia/flowcb/authenticate/copernicus_test.py

@@ -0,0 +1,6 @@
+import pytest
+from tests.conftest import *
+#from unittest.mock import Mock
+
+import sarracenia.config
+import sarracenia.flowcb.authenticate.copernicus

tests/sarracenia/flowcb/accept/auth_eumetstat_test.py → tests/sarracenia/flowcb/authenticate/eumetsat_test.py

@@ -3,4 +3,4 @@
 #from unittest.mock import Mock
 
 import sarracenia.config
-import sarracenia.flowcb.accept.auth_eumetsat
+import sarracenia.flowcb.authenticate.eumetsat