ruff uprev

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 default_install_hook_types: [pre-commit, pre-push]
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.7
+    rev: v0.8.6
     hooks:
       - name: Ruff linting
         id: ruff

pyproject.toml

@@ -53,7 +53,7 @@ test = [
 ]
 dev = [
     "pre-commit",
-    "ruff < 0.6",
+    "ruff < 0.9",
     "sqlfluff >= 3.2.5"
 ]
 

src/dashboard/get_chart_data/get_chart_data.py

@@ -10,6 +10,7 @@
 import boto3
 import jinja2
 import pandas
+
 from shared import decorators, enums, errors, functions
 
 log_level = os.environ.get("LAMBDA_LOG_LEVEL", "INFO")

src/dashboard/get_csv/get_csv.py

@@ -2,6 +2,7 @@
 
 import boto3
 import botocore
+
 from shared import decorators, enums, functions
 
 

src/dashboard/get_metadata/get_metadata.py

@@ -3,6 +3,7 @@
 import os
 
 import boto3
+
 from shared.decorators import generic_error_handler
 from shared.functions import http_response, read_metadata
 

src/dashboard/get_study_periods/get_study_periods.py

@@ -3,6 +3,7 @@
 import os
 
 import boto3
+
 from shared import decorators, enums, functions
 
 

src/dashboard/post_distribute/post_distribute.py

@@ -1,30 +1,30 @@
-""" Handler for validating the payload contents and submitting to queue
+"""Handler for validating the payload contents and submitting to queue"""
 
-"""
 import json
 import os
 
 import boto3
 import requests
+
 from shared import functions
 
 sns_client = boto3.client("sns", os.environ.get("AWS_REGION"))
-valid_chars ="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~:/?#[]@!$&'()*+,"
+valid_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~:/?#[]@!$&'()*+,"
+
 
 def validate_github_url(config):
-    if (
-        not config['url'].startswith("https://github.com/smart-on-fhir/")
-        or any(c not in valid_chars for c in config['url'])
+    if not config["url"].startswith("https://github.com/smart-on-fhir/") or any(
+        c not in valid_chars for c in config["url"]
     ):
         raise ValueError(f"{config['url']} is not an official Cumulus study.")
-    res = requests.get(config['url'], timeout=10)
+    res = requests.get(config["url"], timeout=10)
     if res.status_code != 200:
         raise ValueError(f"{config['url']} is not a valid git repository")
-    if 'tag' in config and config['tag'] is not None:
-        res = requests.get(config['url'] + f'/tree/{config["tag"]}', timeout=10)
+    if "tag" in config and config["tag"] is not None:
+        res = requests.get(config["url"] + f'/tree/{config["tag"]}', timeout=10)
         if res.status_code != 200:
             raise ValueError(f"{config['tag']} is not a valid tag")
-    
+
 
 def validate_body(body: dict):
     """Selects the appropriate handler for processing study requests"""
@@ -38,7 +38,7 @@ def validate_body(body: dict):
                 raise ValueError(f"Invalid key {key} received.")
 
 
-#@decorators.generic_error_handler(msg="Error generating distributed request")
+# @decorators.generic_error_handler(msg="Error generating distributed request")
 def distribute_handler(event: dict, context):
     """Creates a distribution packages and queues for delivery"""
     del context

src/dashboard/queue_distribute/queue_distribute.py

@@ -1,4 +1,4 @@
-""" Handler for submitting studies to an SNS queue for distribution to remote sites.
+"""Handler for submitting studies to an SNS queue for distribution to remote sites.
 
 In the long term, this module (or a submodule imported by this module) will be responsible
 for parsing the output of a dashboard guided workflow/query builder generated payload,
@@ -14,13 +14,15 @@
 only writable location.
 
 """
+
 import json
 import os
 import pathlib
 import subprocess
 
 import boto3
 from cumulus_library import cli
+
 from shared import decorators, functions
 
 # lambda performance tuning - moving these two variables to be global in the module
@@ -34,19 +36,21 @@
 sns_client = boto3.client("sns", os.environ.get("AWS_REGION"))
 BASE_DIR = "/tmp"  # noqa: S108
 
+
 def get_study_from_github(config):
     try:
-        subprocess.run(['/usr/bin/git', 'clone', config['url'], f"{BASE_DIR}/studies"], check=True)  # noqa: S603
-        if 'tag' in config and config['tag'] is not None:
+        subprocess.run(["/usr/bin/git", "clone", config["url"], f"{BASE_DIR}/studies"], check=True)  # noqa: S603
+        if "tag" in config and config["tag"] is not None:
             subprocess.run(  # noqa: S603
-                ['/usr/bin/git', 'checkout','tag'], 
-                cwd = f"{BASE_DIR}/studies/{config['url'].split('/')[-2]}",
+                ["/usr/bin/git", "checkout", "tag"],
+                cwd=f"{BASE_DIR}/studies/{config['url'].split('/')[-2]}",
             )
-            
+
     except subprocess.CalledProcessError:
         # TODO: retry/backoff logic? or do we just have a user queue again?
         raise ValueError(f"{config['url']} is unavailable")
 
+
 def prepare_study(body: dict):
     write_path = pathlib.Path(f"{BASE_DIR}/prepared")
     write_path.mkdir(parents=True, exist_ok=True)
@@ -89,15 +93,10 @@ def queue_handler(event: dict, context):
         file = f.read()
         sns_client.publish(
             TopicArn=topic_sns_arn,
-            Message=json.dumps({'study': body["study_name"]}),
+            Message=json.dumps({"study": body["study_name"]}),
             MessageGroupId=body["study_name"],
             Subject=body["study_name"],
-            MessageAttributes={
-                'study':{
-                    "DataType": "Binary",
-                    "BinaryValue": file
-                }
-            }
+            MessageAttributes={"study": {"DataType": "Binary", "BinaryValue": file}},
         )
     res = functions.http_response(200, f'Study {body["study_name"]} queued.')
     return res

src/site_upload/cache_api/cache_api.py

@@ -5,6 +5,7 @@
 
 import awswrangler
 import boto3
+
 from shared import decorators, enums, functions
 
 

src/site_upload/fetch_upload_url/fetch_upload_url.py

@@ -6,6 +6,7 @@
 
 import boto3
 import botocore.exceptions
+
 from shared.decorators import generic_error_handler
 from shared.enums import BucketPath
 from shared.functions import get_s3_json_as_dict, http_response

src/site_upload/powerset_merge/powerset_merge.py

@@ -7,6 +7,7 @@
 import awswrangler
 import pandas
 from pandas.core.indexes.range import RangeIndex
+
 from shared import awswrangler_functions, decorators, enums, functions, pandas_functions, s3_manager
 
 log_level = os.environ.get("LAMBDA_LOG_LEVEL", "INFO")

src/site_upload/process_flat/process_flat.py

@@ -2,6 +2,7 @@
 import os
 
 import awswrangler
+
 from shared import awswrangler_functions, decorators, enums, functions, pandas_functions, s3_manager
 
 log_level = os.environ.get("LAMBDA_LOG_LEVEL", "INFO")

src/site_upload/process_upload/process_upload.py

@@ -4,6 +4,7 @@
 import os
 
 import boto3
+
 from shared import decorators, enums, functions
 
 log_level = os.environ.get("LAMBDA_LOG_LEVEL", "INFO")

src/site_upload/study_period/study_period.py

@@ -5,6 +5,7 @@
 
 import awswrangler
 import boto3
+
 from shared import awswrangler_functions, decorators, enums, functions
 
 

tests/dashboard/test_post_distribute.py

@@ -33,33 +33,20 @@
             None,
             500,
         ),
-        (
-            "non_cumulus_repo",
-            "https://github.com/user/non_cumulus_repo", 
-            None,
-            500
-        ),
+        ("non_cumulus_repo", "https://github.com/user/non_cumulus_repo", None, 500),
     ],
 )
 @responses.activate
 @freeze_time("2020-01-01")
 def test_process_github(mock_notification, tmp_path, name, url, tag, expected_status, monkeypatch):
-    responses.add(
-        responses.GET,
-        "https://github.com/smart-on-fhir/test_study",
-        status=200
-    )
-    responses.add(
-        responses.GET,
-        "https://github.com/smart-on-fhir/test_study/tree/tag",
-        status=200
-    )
+    responses.add(responses.GET, "https://github.com/smart-on-fhir/test_study", status=200)
+    responses.add(responses.GET, "https://github.com/smart-on-fhir/test_study/tree/tag", status=200)
     responses.add(
         responses.GET,
         "https://github.com/smart-on-fhir/invalid_study",
         status=404,
     )
-    
+
     mock_sns = mock.MagicMock()
     monkeypatch.setattr(post_distribute, "sns_client", mock_sns)
     res = post_distribute.distribute_handler(
@@ -72,12 +59,11 @@ def test_process_github(mock_notification, tmp_path, name, url, tag, expected_st
             "TopicArn": "test-payload",
             "MessageGroupId": "test_study",
             "Subject": "test_study",
-            "Message": json.dumps({"github": {"url":url, "tag":tag}, "study_name": name})
+            "Message": json.dumps({"github": {"url": url, "tag": tag}, "study_name": name}),
         }
         for k, v in mock_sns.publish.call_args[1].items():
-
             assert expected_message[k] == v
-        
+
 
 def test_invalid_key():
     res = post_distribute.distribute_handler({"body": json.dumps({"bad_para,": "foo"})}, {})

tests/dashboard/test_queue_distribute.py

@@ -16,6 +16,7 @@ def error_callback(process):
     process.returncode = 1
     raise subprocess.CalledProcessError(1, "Error cloning")
 
+
 @mock.patch.dict(
     os.environ, {"TOPIC_STUDY_PAYLOAD_ARN": "test-payload", "AWS_REGION": "us-east-1"}, clear=True
 )
@@ -34,66 +35,57 @@ def error_callback(process):
             "tag",
             200,
         ),
-        (
-            "invalid_study",
-            "https://github.com/smart-on-fhir/invalid_study",
-            None,
-            500
-        ),
-
+        ("invalid_study", "https://github.com/smart-on-fhir/invalid_study", None, 500),
     ],
 )
 @responses.activate
 def test_process_github(
     mock_notification, tmp_path, name, url, tag, expected_status, monkeypatch, fp
 ):
-    fp.allow_unregistered(True) # fp is provided by pytest-subprocess
+    fp.allow_unregistered(True)  # fp is provided by pytest-subprocess
     if name == "invalid_study":
-        fp.register(["/usr/bin/git","clone",url,f"{tmp_path}/studies"], callback=error_callback)
+        fp.register(["/usr/bin/git", "clone", url, f"{tmp_path}/studies"], callback=error_callback)
     else:
-        fp.register(["/usr/bin/git","clone",url,f"{tmp_path}/studies"])
-        (tmp_path /"studies").mkdir()
+        fp.register(["/usr/bin/git", "clone", url, f"{tmp_path}/studies"])
+        (tmp_path / "studies").mkdir()
         shutil.copytree(
-            pathlib.Path.cwd() / f"./tests/test_data/mock_payloads/{name}/", 
-            tmp_path / f"studies/{name}"
+            pathlib.Path.cwd() / f"./tests/test_data/mock_payloads/{name}/",
+            tmp_path / f"studies/{name}",
         )
-    
+
     monkeypatch.setattr(queue_distribute, "BASE_DIR", tmp_path)
     mock_sns = mock.MagicMock()
     monkeypatch.setattr(queue_distribute, "sns_client", mock_sns)
-    with freeze_time("2020-01-01"): #Don't use as a fixture for this test; collides with fp mock
+    with freeze_time("2020-01-01"):  # Don't use as a fixture for this test; collides with fp mock
         res = queue_distribute.queue_handler(
-            { 
+            {
                 "Records": [
                     {
-                        "Sns":{
-                            "Message": json.dumps({
-                                "github": {
-                                    "url": url, "tag": tag
-                                },
-                                "study_name": name})
+                        "Sns": {
+                            "Message": json.dumps(
+                                {"github": {"url": url, "tag": tag}, "study_name": name}
+                            )
                         }
                     }
                 ]
             },
-            {}
+            {},
         )
     assert res["statusCode"] == expected_status
     if expected_status == 200:
         assert mock_sns.publish.is_called()
         expected_message = {
             "TopicArn": "test-payload",
             "MessageGroupId": "test_study",
-            'Message': '{"study": "test_study"}',
+            "Message": '{"study": "test_study"}',
             "Subject": "test_study",
-
         }
         for k, v in mock_sns.publish.call_args[1].items():
             if k == "MessageAttributes":
-                assert name in str(v['study']['BinaryValue'])
-            else: 
+                assert name in str(v["study"]["BinaryValue"])
+            else:
                 assert expected_message[k] == v
-    if tag=='tag':
+    if tag == "tag":
         files = [p for p in (tmp_path / f"studies/{name}").iterdir() if p.is_file()]
         files = [file.stem for file in files]
-        assert 'tag' in files
+        assert "tag" in files