-
Notifications
You must be signed in to change notification settings - Fork 22
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'development' of https://github.com/MetPX/Sarracenia int…
…o development
- Loading branch information
Showing
11 changed files
with
264 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,11 @@ | ||
metpx-sr3 (3.00.54rc3) UNRELEASED; urgency=medium | ||
metpx-sr3 (3.00.55rc1) unstable; urgency=medium | ||
|
||
* fix #1151 calculate on download not working. | ||
* fix #1150 bc transport api refactor. | ||
|
||
-- SSC-5CD2310S60 <[email protected]> Tue, 08 Aug 2024 11:24:36 -0400 | ||
|
||
metpx-sr3 (3.00.54rc3) unstable; urgency=medium | ||
|
||
* fix #1142 token authentication re-factor | ||
* fix #1141 subscribe crash with random checksum. | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1 @@ | ||
__version__ = "3.00.54rc3" | ||
__version__ = "3.00.55rc1" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
# | ||
# Post the BC Trans URL to the sarra. | ||
# The sarra will authenticate through the authenticate plugin after | ||
# | ||
|
||
vip # Enter your vip here | ||
|
||
# You want to callback the scheduled flow AND the post plugin so the URL gets posted downstream | ||
callback scheduled | ||
callback post.message | ||
|
||
|
||
# Run everyhour | ||
scheduled_interval 3600 | ||
|
||
# post_baseUrl + path combined make up the URL to be posted downstream | ||
post_baseUrl https://sawsx.api.gov.bc.ca/ | ||
path /api/v1/motisite/report7100/all/3/ | ||
|
||
# to ignore colons in the filenames generated. | ||
filename None | ||
|
||
# Where to announce the messages | ||
post_broker amqp://PROVINCIAL@localhost/ | ||
post_exchange xs_PROVINCIAL | ||
|
||
logLevel debug |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
# Flow callbacks | ||
callback authenticate.bc_trans | ||
callback accept.bc_trans | ||
filename None | ||
|
||
# Source | ||
broker amqp://PROVINCIAL@localhost/ | ||
exchange xs_PROVINCIAL | ||
queueName q_${BROKER_USER}_${PROGRAM}_${CONFIG}.ddsr-shared | ||
source PROVINCIAL | ||
|
||
|
||
# Filtering | ||
# Subtopic is the path from the scheduled flow | ||
subtopic api.v1.motisite.report7100.# | ||
directory /tmp/${%Y%m%d}/${SOURCE}/AIRNOW/CSV/RWIN/${%H} | ||
accept .* | ||
|
||
# Destination | ||
post_broker amqp://feeder@localhost/ | ||
post_exchange xpublic | ||
post_baseUrl http://${HOSTNAME} | ||
post_baseDir / | ||
|
||
|
||
# Additional options | ||
acceptUnmatched False | ||
delete False | ||
mirror False | ||
inflight .tmp | ||
instances 1 | ||
timeout 60 | ||
|
||
# Set nodupe so that files over 5 mins are expired. The bearer token only lasts 5 minutes, so no point in having it longer in cache. | ||
nodupe_basis path | ||
nodupe_ttl 300 | ||
|
||
logLevel debug |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
#!/usr/bin/python3 | ||
|
||
""" | ||
Renamer for new BC Trans API | ||
===================================================================================== | ||
Description | ||
Rename the file fetched from the BC Trans scheduled flow to a CSV format, date included in filename. | ||
How to set up in your config: | ||
-------------------------------- | ||
Use ``callback accept.bc_trans``, and read about the config options above. | ||
For an example, see https://github.com/MetPX/sarracenia/tree/development/sarracenia/examples/flow files named ``*bc_trans*.conf``. | ||
Your ``subtopic`` should match the ``path`` from the scheduled flow plugin. | ||
Change log: | ||
----------- | ||
- 2024-08-06: Inception of plugin. | ||
""" | ||
|
||
from sarracenia.flowcb import FlowCB | ||
import logging | ||
import requests,os,datetime,sys,time | ||
|
||
logger = logging.getLogger(__name__) | ||
|
||
class Bc_trans(FlowCB): | ||
def __init__(self, options): | ||
super().__init__(options, logger) | ||
|
||
# end __init__ | ||
|
||
|
||
def after_accept(self, worklist): | ||
|
||
for msg in worklist.incoming: | ||
|
||
# Modify the filename with the specified format | ||
try: | ||
|
||
now = datetime.datetime.now() | ||
msg['new_file'] = f"OB.BC.MOT.BC_TRAN.{now.strftime('%Y%m%d%H%M%S')}.csv" | ||
|
||
except Exception as e: | ||
logger.debug("Exception details:", exc_info=True) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,126 @@ | ||
#!/usr/bin/python3 | ||
|
||
""" | ||
Authentication for new BC Trans API | ||
===================================================================================== | ||
Description | ||
The BC Trans data provider is migrating to a public API. We used to have sarracenia polls fetching from their HTTP site. This has since been discontinued. | ||
The API works by | ||
1. Providing the client secret to the endpoint server. | ||
2. The server responds with an access token. | ||
3. Pass the access token to https://sawsx-services-dev-api-gov-bc-ca.test.api.gov.bc.ca/api/v1/motisite/report7100/al/X to get last X hours of data from provider. | ||
The access token has a life span of 5 minutes. You can also generate however many access tokens you'd like given you have access to the client_secret and client_id. | ||
As of now, the only data that can be fetched is 3 hours worth of past data. | ||
The pollUrl should remain the same , unless other timestamps are specified in the future. | ||
This plugin uses the BearerToken authentication class entry points. | ||
Documentation available from the Government of BC website | ||
API tutorial (with examples) : https://developer.gov.bc.ca/docs/default/component/aps-infra-platform-docs/tutorials/quick-start/ | ||
Developer portal : https://api.gov.bc.ca/devportal/api-directory | ||
Configurable Options: | ||
---------------------- | ||
``tokenEndpoint_baseUrl`` URL: | ||
^^^^^^^^^^^^^^^^^^^^^^^^ | ||
URL where to fetch the access token. | ||
``tokenEndpoint_path`` URL: | ||
^^^^^^^^^^^^^^^^^^^^^^^^ | ||
Path where to fetch the access token. To use in combination with token_Endpoint_baseUrl | ||
*Potential Room For Improvement*: | ||
---------------------------------- | ||
- Download from multiple different hour marks? | ||
How to set up your flow config: | ||
-------------------------------- | ||
Use ``callback authenticate.bc_trans``, and read about the config options above. | ||
For example, see https://github.com/MetPX/sarracenia/tree/development/sarracenia/examples/flow files named ``*bc_trans*.conf``. | ||
Change log: | ||
----------- | ||
- 2024-08-06: Inception of authenticate plugin. | ||
""" | ||
|
||
|
||
import sarracenia | ||
from sarracenia.flowcb.authenticate import BearerToken | ||
import logging | ||
import requests | ||
|
||
logger = logging.getLogger(__name__) | ||
|
||
class Bc_trans(BearerToken): | ||
def __init__(self, options): | ||
super().__init__(options, logger) | ||
|
||
# Allow setting a logLevel *only* for this plugin in the config file: | ||
# set accept.auth_eumetsat.logLevel debug | ||
if hasattr(self.o, 'logLevel'): | ||
logger.setLevel(self.o.logLevel.upper()) | ||
|
||
# For additional options (defined by user) | ||
self.o.add_option('tokenEndpoint_baseUrl', 'str', 'https://loginproxy.gov.bc.ca/') # required | ||
self.o.add_option('tokenEndpoint_path', 'str', 'auth/realms/apigw/protocol/openid-connect/token') # required | ||
|
||
# Set initial values | ||
self._bearer_token = None | ||
|
||
# Make sure values already exist | ||
if not self.o.tokenEndpoint_baseUrl or not self.o.tokenEndpoint_path: | ||
logger.error(f"tokenEndpoint_baseUrl and tokenEndpoint_path is a required option.") | ||
sys.exit(1) | ||
|
||
# Get credentials from credentials.conf. | ||
# In this case the client ID is the traditional url username and the client secret is the traditional password | ||
# This avoids creating new fields in the credentials class | ||
if self.o.tokenEndpoint_baseUrl and self.o.tokenEndpoint_path: | ||
ok, self.details = sarracenia.config.Config.credentials.get( | ||
self.o.tokenEndpoint_baseUrl + self.o.tokenEndpoint_path) | ||
|
||
self.client_id = self.details.url.username | ||
self.client_secret = self.details.url.password | ||
self.grant_type = 'client_credentials' | ||
|
||
# end __init__ | ||
|
||
|
||
def get_token(self): | ||
""" | ||
Fetches the bearer token from the token endpoint. Uses the parent class entry point. | ||
NOTE : Multiple bearer tokens can be fetched at the same time. | ||
NOTE : The bearer token has a lifespan of 5 minutes. | ||
""" | ||
try: | ||
logger.info("Requesting a new bearer token") | ||
|
||
response = requests.post(self.o.tokenEndpoint_baseUrl + self.o.tokenEndpoint_path,\ | ||
data={'grant_type': f'{self.grant_type}'},\ | ||
auth=(self.client_id, self.client_secret)) | ||
|
||
# Get response information | ||
response_status = response.status_code | ||
response_text = response.text | ||
|
||
if response_status == 200: | ||
self._bearer_token = response.json()["access_token"] | ||
else: | ||
logger.error( f"Request status received: {response_status}. Response message: {response_text}" ) | ||
self._bearer_token = None | ||
|
||
except Exception as e: | ||
logger.debug("Exception details:", exc_info=True) | ||
|
||
return self._bearer_token |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
import pytest | ||
from tests.conftest import * | ||
#from unittest.mock import Mock | ||
|
||
import sarracenia.config | ||
import sarracenia.flowcb.authenticate.copernicus |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters