Attachments upload with 429

Problem:
if an attachments upload encounters a 429 then the connection is closed
resulting in a zero length file.

Solution:
Make sure that Verify is True which will enable keep-alive.
Additionally upgrade all dependencies and reformat code to latest
standard.

Signed-off-by: Paul Hewlett <[email protected]>

archivist/applications.py

@@ -162,7 +162,6 @@ def __params(
         display_name: Optional[str] = None,
         custom_claims: Optional[dict[str, str]] = None,
     ) -> dict[str, Any]:
-
         params = {}
 
         if display_name is not None:

archivist/archivist.py

@@ -222,7 +222,6 @@ def __copy__(self) -> Archivist:
         )
 
     def _add_headers(self, headers: dict[str, str] | None) -> dict[str, Any]:
-
         if isinstance(headers, dict):
             newheaders = {**headers}
         else:

archivist/archivistpublic.py

@@ -83,7 +83,6 @@ def __init__(
         verify: bool = True,
         max_time: float = MAX_TIME,
     ):
-
         self._verify = verify
         self._response_ring_buffer = deque(maxlen=self.RING_BUFFER_MAX_LEN)
         self._session = None

archivist/cmds/runner/run.py

@@ -17,7 +17,6 @@
 
 
 def run(arch: archivist.Archivist, args):
-
     LOGGER.info("Using version %s of rkvst-archivist", about.__version__)
     LOGGER.info("Namespace %s", args.namespace)
 

archivist/cmds/template/run.py

@@ -18,7 +18,6 @@
 
 
 def run(arch: "type_helper.Archivist", args):
-
     LOGGER.info("Using version %s of rkvst-archivist", about.__version__)
     LOGGER.info("Namespace %s", args.namespace)
 

archivist/compliance.py

@@ -106,7 +106,6 @@ def compliant_at_report(self, compliance: dict[str, Any]):
 
         LOGGER.info("Compliant %s", compliance["compliant"])
         for outcome in compliance["compliance"]:
-
             if outcome["compliant"]:
                 continue
 

archivist/parser.py

@@ -145,7 +145,6 @@ def common_parser(description: str):
 
 
 def endpoint(args):
-
     if args.verbose:
         set_logger("DEBUG")
     else:
@@ -190,7 +189,7 @@ def endpoint(args):
         LOGGER.error("Critical error.  Aborting.")
         sys_exit(1)
 
-    arch = Archivist(args.url, auth, verify=False, fixtures=fixtures)
+    arch = Archivist(args.url, auth, fixtures=fixtures)
     if arch is None:
         LOGGER.error("Critical error.  Aborting.")
         sys_exit(1)

archivist/runner.py

@@ -280,7 +280,6 @@ def args(self, identity_method, step):
             label = self.get(f"{noun}_label")
             func = self.label("use", noun)
             if label is not None and func:
-
                 identity = self.identity_from_label(noun, identity_method)
                 if identity is None:
                     raise ArchivistInvalidOperationError(f"unknown {noun} '{label}'")

archivist/subjects.py

@@ -267,7 +267,6 @@ def __params(
         wallet_pub_key: Optional[list[str]] = None,
         tessera_pub_key: Optional[list[str]] = None,
     ) -> dict[str, Any]:
-
         params = {}
 
         if display_name is not None:

examples/access_policies_filter.py

@@ -38,7 +38,6 @@ def main():
         "https://app.rkvst.io",
         auth,
     ) as arch:
-
         # count access_policies...
         print(
             "no.of access_policies",

examples/access_policy_create.py

@@ -38,7 +38,6 @@ def main():
         "https://app.rkvst.io",
         (client_id, client_secret),
     ) as arch:
-
         props = {
             "display_name": "Friendly name of the policy",
             "description": "Description of the policy",

examples/applications_registration.py

@@ -33,7 +33,6 @@ def main():
         "https://app.rkvst.io",
         authtoken,
     ) as arch:
-
         # create application
         application = arch.applications.create(
             f"Application display name {uuid4()}",

examples/create_event.py

@@ -156,7 +156,6 @@ def main():
         "https://app.rkvst.io",
         (client_id, client_secret),
     ) as arch:
-
         # Create a new asset
         asset = create_asset(arch)
         print("Asset", json_dumps(asset, sort_keys=True, indent=4))

examples/filter_assets.py

@@ -39,7 +39,6 @@ def main():
         "https://app.rkvst.io",
         (client_id, client_secret),
     ) as arch:
-
         # list all assets with required attributes and properties
         props = {"confirmation_status": "CONFIRMED"}
         attrs = {"arc_display_type": "Traffic light"}

examples/filter_publicevents.py

@@ -15,7 +15,6 @@ def main():
     """
     # Initialize connection to ArchivistPublic
     with ArchivistPublic() as public:
-
         # Get all public events with required attributes and properties
         props = {"confirmation_status": "CONFIRMED"}
         attrs = {"arc_display_type": "Traffic light"}

examples/get_publicevent.py

@@ -13,7 +13,6 @@ def main():
     """Main function of getting public events."""
     # Initialize connection to ArchivistPublic
     with ArchivistPublic() as public:
-
         # URL is the fully-attested URL returned by archivist
         event = public.events.read(
             "https://app.rkvst.io/archivist/"

examples/runner.py

@@ -17,7 +17,6 @@
 
 
 def run(arch: Archivist, args):
-
     LOGGER.info("Using version %s of rkvst-archivist", about.__version__)
     LOGGER.info("Namespace %s", args.namespace)
 

examples/sbom_release.py

@@ -98,8 +98,7 @@ def main():
         client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
     )
 
-    with Archivist(rkvst_url, auth, verify=False, max_time=300) as arch:
-
+    with Archivist(rkvst_url, auth, max_time=300) as arch:
         asset, event = sbom_release(
             arch, getenv("BUILD_BUILDNUMBER"), getenv("SBOM_FILEPATH")
         )

examples/scan_test.py

@@ -137,8 +137,7 @@ def main():
         client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
     )
 
-    with Archivist(getenv("RKVST_URL"), auth, verify=False, max_time=300) as arch:
-
+    with Archivist(getenv("RKVST_URL"), auth, max_time=300) as arch:
         print("##[group]Today")
         today = date.today()
         scan_test(arch, today.strftime("%Y-%m-%d"))

examples/subject_create.py

@@ -38,7 +38,6 @@ def main():
         "https://app.rkvst.io",
         (client_id, client_secret),
     ) as arch:
-
         # subject_string is the base64 encoding of the self subject of the other organization
         subject_string = (
             "eyJpZGVudGl0eSI6ICJzdWJqZWN0cy8wMDAwMDAwMC0wMDAwLTAwMDAtMDA"

examples/subjects_filter.py

@@ -32,7 +32,6 @@ def main():
         "https://app.rkvst.io",
         (client_id, client_secret),
     ) as arch:
-
         # count subjects...
         print("no.of subjects", arch.subjects.count(display_name="Some display name"))
 

functests/execaccess_policies.py

@@ -120,7 +120,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False)
+        self.arch = Archivist(getenv("RKVST_URL"), auth)
 
         # these are for access_policies
         self.ac_props = deepcopy(PROPS)
@@ -301,7 +301,7 @@ def setUp(self):
         super().setUp()
         with open(getenv("RKVST_AUTHTOKEN_FILENAME_2"), encoding="utf-8") as fd:
             auth_2 = fd.read().strip()
-        self.arch_2 = Archivist(getenv("RKVST_URL"), auth_2, verify=False)
+        self.arch_2 = Archivist(getenv("RKVST_URL"), auth_2)
 
         # creates reciprocal subjects for arch 1 and arch 2.
         # subject 1 contains details of subject 2 to be shared

functests/execapplications.py

@@ -55,7 +55,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False)
+        self.arch = Archivist(getenv("RKVST_URL"), auth)
         self.display_name = f"{DISPLAY_NAME} {uuid4()}"
 
     def tearDown(self):
@@ -231,9 +231,7 @@ def test_archivist_token(self):
         with Archivist(
             getenv("RKVST_URL"),
             (application["client_id"], application["credentials"][0]["secret"]),
-            verify=False,
         ) as new_arch:
-
             # now we create an asset and add events 10 times with a 60s sleep
             # this should trigger a token refresh
             traffic_light = deepcopy(ATTRS)

functests/execassets.py

@@ -85,7 +85,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False, max_time=600)
+        self.arch = Archivist(getenv("RKVST_URL"), auth, max_time=600)
         self.attrs = deepcopy(ATTRS)
         self.traffic_light = deepcopy(ATTRS)
         self.traffic_light["arc_display_type"] = "Traffic light with violation camera"
@@ -248,7 +248,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False, max_time=600)
+        self.arch = Archivist(getenv("RKVST_URL"), auth, max_time=600)
 
     def tearDown(self):
         self.arch.close()

functests/execattachments.py

@@ -44,7 +44,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False)
+        self.arch = Archivist(getenv("RKVST_URL"), auth)
         self.file_uuid: str = ""
 
         with suppress(FileNotFoundError):
@@ -195,7 +195,7 @@ def setUp(self):
             client_id=getenv("RKVST_APPREG_CLIENT"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False)
+        self.arch = Archivist(getenv("RKVST_URL"), auth)
 
     def tearDown(self):
         self.arch.close()

functests/execcompliance_policies.py

@@ -98,7 +98,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False)
+        self.arch = Archivist(getenv("RKVST_URL"), auth)
 
     def tearDown(self):
         self.arch.close()

functests/execnotebooks.py

@@ -32,9 +32,7 @@ def setUp(self):
         self.client_id = getenv("RKVST_APPREG_CLIENT")
         self.client_secret = getenv("RKVST_APPREG_SECRET")
         self.url = getenv("RKVST_URL")
-        self.arch = Archivist(
-            self.url, (self.client_id, self.client_secret), verify=False
-        )
+        self.arch = Archivist(self.url, (self.client_id, self.client_secret))
 
     def tearDown(self):
         self.arch.close()

functests/execpublicassets.py

@@ -87,7 +87,7 @@ def setUp(self):
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
         self.url = getenv("RKVST_URL")
-        self.arch = Archivist(self.url, auth, verify=False, max_time=600)
+        self.arch = Archivist(self.url, auth, max_time=600)
         self.attrs = deepcopy(ATTRS)
         self.traffic_light = deepcopy(ATTRS)
         self.traffic_light["arc_display_type"] = "Traffic light with violation camera"

functests/execrunner.py

@@ -40,7 +40,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False, max_time=300)
+        self.arch = Archivist(getenv("RKVST_URL"), auth, max_time=300)
 
     def tearDown(self):
         self.arch.close()

functests/execsboms.py

@@ -52,7 +52,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False)
+        self.arch = Archivist(getenv("RKVST_URL"), auth)
         self.file_uuid: str = ""
         self.title = "TestSBOM"
 

functests/execsubjects.py

@@ -59,7 +59,7 @@ def setUp(self):
             auth_token_filename=getenv("RKVST_AUTHTOKEN_FILENAME"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False)
+        self.arch = Archivist(getenv("RKVST_URL"), auth)
         self.display_name = f"{DISPLAY_NAME} {uuid4()}"
 
     def tearDown(self):

requirements-dev.txt

@@ -1,21 +1,21 @@
 -r requirements.txt
 
 # code quality
-autopep8~=1.6
-black[jupyter]~=22.6
-coverage[toml]~=6.4
+autopep8~=2.0
+black[jupyter]~=23.1
+coverage[toml]~=7.1
 pip-audit~=2.4
-pycodestyle~=2.9
-pylint~=2.14
+pycodestyle~=2.10
+pylint~=2.16
 pyright~=1.1
-unittest-xml-reporting~=3.2.0
+unittest-xml-reporting~=3.2
 testbook~=0.4
 
 # analyze dependencies
-pipdeptree~=2.2
+pipdeptree~=2.3
 
 # uploading to pypi
-build~=0.8
+build~=0.10
 twine~=4.0
 
 # for sbom.xml file

requirements.txt

@@ -2,14 +2,14 @@
 # for the published wheel - the file docs/requirements.txt
 # must be kept in sync with this file.
 #
+# We can only upgrade to 2.2 of backoff when support for py3.7 is dropped
 backoff~=1.11
 certifi
 flatten-dict~=0.4
-iso8601~=1.0
+iso8601~=1.1
 Jinja2~=3.0
 pyaml-env~=1.1
 requests~=2.28
-requests-toolbelt~=0.9
 rfc3339~=6.2
 xmltodict~=0.13
 

scripts/builder.sh

@@ -14,7 +14,7 @@ then
 fi
 
 docker run \
-    --rm \
+    --rm -it \
     -v $(pwd):/home/builder \
     -u $(id -u):$(id -g) \
     -e FUNCTEST \

unittests/testassets.py

@@ -321,7 +321,6 @@ def common_assets_create_if_not_exists_nonexistent_asset(
         ) as mock_location, mock.patch.object(
             self.arch.attachments, "create"
         ) as mock_attachments:
-
             mock_get.side_effect = ArchivistNotFoundError
             mock_post.return_value = MockResponse(200, **resp)
             if loc_resp is not None:

unittests/testeventsconfirm.py

@@ -41,7 +41,6 @@ def test_events_create_with_confirmation(self):
         with mock.patch.object(
             self.arch.session, "post"
         ) as mock_post, mock.patch.object(self.arch.session, "get") as mock_get:
-
             mock_post.return_value = MockResponse(200, **RESPONSE)
             mock_get.return_value = MockResponse(200, **RESPONSE)
 

unittests/testeventscreate.py

@@ -376,7 +376,6 @@ def test_events_create_with_explicit_confirmation(self):
         with mock.patch.object(
             self.arch.session, "post"
         ) as mock_post, mock.patch.object(self.arch.session, "get") as mock_get:
-
             mock_post.return_value = MockResponse(200, **RESPONSE)
             mock_get.return_value = MockResponse(200, **RESPONSE)
 

unittests/testpublic.py

@@ -59,7 +59,7 @@ def test_public(self):
             self.assertEqual(
                 public.public,
                 True,
-                msg="verify must be True",
+                msg="public must be True",
             )
             with self.assertRaises(AttributeError):
                 e = public.Illegal_endpoint