fix logic to prevent failures

python/arcticdb/storage_fixtures/s3.py

@@ -237,7 +237,9 @@ def cleanup_bucket(self, b: S3Bucket):
         b.slow_cleanup(failure_consequence="We will be charged unless we manually delete it. ")
 
 
-def real_s3_from_environment_variables(shared_path: bool, native_config: Optional[NativeVariantStorage] = None, additional_suffix: str = ""):
+def real_s3_from_environment_variables(shared_path: bool, 
+                                       native_config: Optional[NativeVariantStorage] = None, 
+                                       additional_suffix: str = "") -> BaseS3StorageFixtureFactory:
     out = BaseS3StorageFixtureFactory(native_config=native_config)
     out.endpoint = os.getenv("ARCTICDB_REAL_S3_ENDPOINT")
     out.region = os.getenv("ARCTICDB_REAL_S3_REGION")
@@ -254,7 +256,12 @@ def real_s3_from_environment_variables(shared_path: bool, native_config: Optiona
     return out
 
 
-def real_s3_sts_from_environment_variables(user_name: str, role_name: str, policy_name: str, profile_name: str, native_config: NativeVariantStorage, additional_suffix: str):
+def real_s3_sts_from_environment_variables(user_name: str, 
+                                           role_name: str, 
+                                           policy_name: str, 
+                                           profile_name: str, 
+                                           native_config: NativeVariantStorage, 
+                                           additional_suffix: str) -> BaseS3StorageFixtureFactory:
     out = real_s3_from_environment_variables(False, native_config, additional_suffix)
     iam_client = boto3.client("iam", aws_access_key_id=out.default_key.id, aws_secret_access_key=out.default_key.secret)
     # Create IAM user
@@ -370,6 +377,7 @@ def real_s3_sts_from_environment_variables(user_name: str, role_name: str, polic
 
     try:
         access_key_response = iam_client.create_access_key(UserName=user_name)
+        logger.info(f"Response access key: {access_key_response}")
         out.sts_test_key.id = access_key_response["AccessKey"]["AccessKeyId"]
         out.sts_test_key.secret = access_key_response["AccessKey"]["SecretAccessKey"]
         logger.info("Access key created successfully.")

python/tests/conftest.py

@@ -27,6 +27,7 @@
 from arcticdb.storage_fixtures.azure import AzuriteStorageFixtureFactory
 from arcticdb.storage_fixtures.lmdb import LmdbStorageFixture
 from arcticdb.storage_fixtures.s3 import (
+    BaseS3StorageFixtureFactory,
     MotoS3StorageFixtureFactory,
     MotoNfsBackedS3StorageFixtureFactory,
     NfsS3Bucket,
@@ -236,7 +237,7 @@ def real_s3_storage(real_s3_storage_factory):
 
 
 @pytest.fixture(scope="session") # Config loaded at the first ArcticDB binary import, so we need to set it up before any tests
-def real_s3_sts_storage_factory():
+def real_s3_sts_storage_factory() -> Generator[BaseS3StorageFixtureFactory, None, None]:
     sts_test_credentials_prefix = os.getenv("ARCTICDB_REAL_S3_STS_TEST_CREDENTIALS_POSTFIX", f"{random.randint(0, 999)}_{datetime.utcnow().strftime('%Y-%m-%dT%H_%M_%S_%f')}")
     username = os.getenv("ARCTICDB_REAL_S3_STS_TEST_USERNAME", f"gh_sts_test_user_{sts_test_credentials_prefix}")
     role_name = os.getenv("ARCTICDB_REAL_S3_STS_TEST_ROLE", f"gh_sts_test_role_{sts_test_credentials_prefix}")
@@ -260,7 +261,7 @@ def real_s3_sts_storage_factory():
 
 
 @pytest.fixture
-def real_s3_sts_storage(real_s3_sts_storage_factory):
+def real_s3_sts_storage(real_s3_sts_storage_factory) -> Generator[S3Bucket, None, None]:
     with real_s3_sts_storage_factory.create_fixture() as f:
         yield f
 

python/tests/integration/arcticdb/test_arctic.py

@@ -117,18 +117,21 @@ def test_s3_no_ssl_verification(monkeypatch, s3_no_ssl_storage, client_cert_file
 
 @REAL_S3_TESTS_MARK
 def test_s3_sts_auth(real_s3_sts_storage):
+    library = "test"
     ac = Arctic(real_s3_sts_storage.arctic_uri)
-    lib = ac.create_library("test")
+    ac.delete_library(library) # make sure we delete any previously existing library
+    lib = ac.create_library(library)
     df = pd.DataFrame({'a': [1, 2, 3]})
     lib.write("sym", df)
     assert_frame_equal(lib.read("sym").data, df)
-    lib = ac.get_library("test")
+    lib = ac.get_library(library)
     assert_frame_equal(lib.read("sym").data, df)
 
     # Reload for testing a different codepath
     ac = Arctic(real_s3_sts_storage.arctic_uri)
-    lib = ac.get_library("test")
+    lib = ac.get_library(library)
     assert_frame_equal(lib.read("sym").data, df)
+    ac.delete_library(library) # make sure we delete any previously existing library
 
 @SLOW_TESTS_MARK
 @REAL_S3_TESTS_MARK
@@ -140,6 +143,7 @@ def test_s3_sts_expiry_check(real_s3_sts_storage):
     has been renewed.
     """
     symbol = "sym"
+    library = "expire"
     # Precondition check
     min_exp_time_min = 15 # This is minimum expiry time, set at fixture level
     value = get_config_int("S3Storage.STSTokenExpiryMin")
@@ -149,22 +153,26 @@ def test_s3_sts_expiry_check(real_s3_sts_storage):
     assert 15 <= value 
 
     ac = Arctic(real_s3_sts_storage.arctic_uri)
-    lib = ac.create_library("test")
+    ac.delete_library(library) # make sure we delete any previously existing library
+    lib = ac.create_library(library)
     df = pd.DataFrame({'a': [1, 2, 3]})
     lib.write(symbol, df)
 
     now = datetime.now()
     complete_at = now + timedelta(minutes=min_exp_time_min+3)
     logger.info(f"Test will complete at {complete_at}")
 
+    data: pd.DataFrame = lib.read(symbol).data
+    assert_frame_equal(df, data)
     while (datetime.now() < complete_at):
         logger.info(f"sleeping 30 sec")
         time.sleep(30)
-        data: pd.DataFrame = lib.read(symbol).data
-        assert_frame_equal(df, data)
         logger.info(f"Read successful at {datetime.now()}. Time remaining: {complete_at - datetime.now()}")
 
+    data: pd.DataFrame = lib.read(symbol).data
+    assert_frame_equal(df, data)
     logger.info(f"Connection did not expire")
+    ac.delete_library(library)
 
 @REAL_S3_TESTS_MARK
 def test_s3_sts_auth_store(real_s3_sts_version_store):