diff --git a/setup.py b/setup.py index f14d50e..f548197 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setuptools.setup( name="keboola.component", - version="1.6.10", + version="1.6.11", author="Keboola KDS Team", project_urls=project_urls, setup_requires=['pytest-runner', 'flake8'], diff --git a/src/keboola/component/dao.py b/src/keboola/component/dao.py index 89b5ef4..86276af 100644 --- a/src/keboola/component/dao.py +++ b/src/keboola/component/dao.py @@ -798,7 +798,10 @@ def __init__(self, name: str, write_always: Optional[bool] = False, has_header: Optional[bool] = None, description: Optional[str] = None, - # input + + # storage staging + s3: Optional[dict] = None, + abs: Optional[dict] = None, **kwargs ): """ @@ -890,6 +893,10 @@ def __init__(self, name: str, self.stage = stage self.has_header = has_header or self._has_header_in_file() + # storage staging + self._s3 = s3 + self._abs = abs + def __get_stage_inferred(self): if self._uri: return 'in' @@ -974,6 +981,10 @@ def build_input_definition(cls, name: str, created: Optional[str] = None, last_change_date: Optional[str] = None, last_import_date: Optional[str] = None, + + # storage staging + s3: Optional[dict] = None, + abs: Optional[dict] = None, **kwargs ): """ @@ -1005,6 +1016,8 @@ def build_input_definition(cls, name: str, created (Optional[str]): The creation timestamp of the table. Defaults to None. last_change_date (Optional[str]): The last modification timestamp of the table. Defaults to None. last_import_date (Optional[str]): The last import timestamp of the table. Defaults to None. + s3 (Optional[dict]): A dictionary containing Amazon S3 storage details. Defaults to None. + abs (Optional[dict]): A dictionary containing Azure Blob Storage details. Defaults to None. Returns: TableDefinition: An instance of TableDefinition configured for input tables. @@ -1031,6 +1044,8 @@ def build_input_definition(cls, name: str, created=created, last_change_date=last_change_date, last_import_date=last_import_date, + s3=s3, + abs=abs, **kwargs ) @@ -1156,6 +1171,10 @@ def build_from_manifest(cls, is_alias=manifest.get('is_alias'), attributes=manifest.get('attributes'), indexed_columns=manifest.get('indexed_columns'), + + # storage staging + s3=manifest.get('s3'), + abs=manifest.get('abs'), ) else: diff --git a/src/keboola/component/interface.py b/src/keboola/component/interface.py index bc9f4b1..98c6e69 100644 --- a/src/keboola/component/interface.py +++ b/src/keboola/component/interface.py @@ -1195,7 +1195,7 @@ def files_input_mapping(self) -> List[dao.FileInputMapping]: Returns: List[FileInputMapping] """ - defs = self.config_data.get('storage', {}).get('output', {}).get('files', []) + defs = self.config_data.get('storage', {}).get('input', {}).get('files', []) files = [] for file in defs: om = dao.build_dataclass_from_dict(dao.FileInputMapping, file) diff --git a/tests/data_examples/data_storage_staging_abs/config.json b/tests/data_examples/data_storage_staging_abs/config.json new file mode 100644 index 0000000..15e8bf1 --- /dev/null +++ b/tests/data_examples/data_storage_staging_abs/config.json @@ -0,0 +1,49 @@ +{ + "storage": { + "input": { + "files": [ + ], + "tables": [ + { + "source": "in.c-main.test", + "destination": "sample.csv", + "columns": [], + "where_values": [], + "where_operator": "eq" + } + ] + }, + "output": { + "tables": [ + ], + "files": [ + ] + } + }, + "parameters": { + "fooBar": { + "foo": 42, + "bar": 24 + }, + "baz": "bazBar" + }, + "action": "run", + "authorization": { + "oauth_api": { + "id": "123456", + "credentials": { + "id": "main", + "authorizedFor": "Myself", + "creator": { + "id": "1234", + "description": "me@keboola.com" + }, + "created": "2016-01-31 00:13:30", + "oauthVersion": "2.0", + "appKey": "myappkey", + "#data": "{\"mykey\":\"myval\"}", + "#appSecret": "myappsecret" + } + } + } +} diff --git a/tests/data_examples/data_storage_staging_abs/in/tables/sample.csv.manifest b/tests/data_examples/data_storage_staging_abs/in/tables/sample.csv.manifest new file mode 100644 index 0000000..b8d29d3 --- /dev/null +++ b/tests/data_examples/data_storage_staging_abs/in/tables/sample.csv.manifest @@ -0,0 +1,39 @@ +{ + "id": "in.c-main.test", + "uri": "https:\/\/connection.keboola.com\/\/v2\/storage\/tables\/in.c-main.test", + "name": "sample.csv", + "primary_key": [], + "indexed_columns": [], + "created": "2015-11-02T09:11:37+0100", + "last_change_date": "2015-11-02T09:11:37+0100", + "last_import_date": "2015-11-02T09:11:37+0100", + "rows_count": 400, + "data_size_bytes": 81920, + "is_alias": false, + "columns": [ + "x", + "Sales", + "CompPrice", + "Income", + "Advertising", + "Population", + "Price", + "ShelveLoc", + "Age", + "Education", + "Urban", + "US", + "High" + ], + "attributes": [], + "abs": { + "is_sliced": true, + "region": "us-east-1", + "container": "exp-2-export-test-test", + "name": "12345.csv.gzmanifest", + "credentials": { + "sas_connection_string": "BlobEndpoint=https://asdf.blob.core.windows.net;SharedAccessSignature=sv=2017-11-09&sr=c&st=2020-08-27T08:42:08Z&se=2020-08-27T20:42:08Z&sp=rl&sig=UJW4DPh%2Baaaaaaaaaa", + "expiration": "2020-08-27T22:42:08+0200" + } + } +} \ No newline at end of file diff --git a/tests/data_examples/data_storage_staging_s3/config.json b/tests/data_examples/data_storage_staging_s3/config.json new file mode 100644 index 0000000..15e8bf1 --- /dev/null +++ b/tests/data_examples/data_storage_staging_s3/config.json @@ -0,0 +1,49 @@ +{ + "storage": { + "input": { + "files": [ + ], + "tables": [ + { + "source": "in.c-main.test", + "destination": "sample.csv", + "columns": [], + "where_values": [], + "where_operator": "eq" + } + ] + }, + "output": { + "tables": [ + ], + "files": [ + ] + } + }, + "parameters": { + "fooBar": { + "foo": 42, + "bar": 24 + }, + "baz": "bazBar" + }, + "action": "run", + "authorization": { + "oauth_api": { + "id": "123456", + "credentials": { + "id": "main", + "authorizedFor": "Myself", + "creator": { + "id": "1234", + "description": "me@keboola.com" + }, + "created": "2016-01-31 00:13:30", + "oauthVersion": "2.0", + "appKey": "myappkey", + "#data": "{\"mykey\":\"myval\"}", + "#appSecret": "myappsecret" + } + } + } +} diff --git a/tests/data_examples/data_storage_staging_s3/in/tables/sample.csv.manifest b/tests/data_examples/data_storage_staging_s3/in/tables/sample.csv.manifest new file mode 100644 index 0000000..4c496e7 --- /dev/null +++ b/tests/data_examples/data_storage_staging_s3/in/tables/sample.csv.manifest @@ -0,0 +1,40 @@ +{ + "id": "in.c-main.test", + "uri": "https:\/\/connection.keboola.com\/\/v2\/storage\/tables\/in.c-main.test", + "name": "sample.csv", + "primary_key": [], + "indexed_columns": [], + "created": "2015-11-02T09:11:37+0100", + "last_change_date": "2015-11-02T09:11:37+0100", + "last_import_date": "2015-11-02T09:11:37+0100", + "rows_count": 400, + "data_size_bytes": 81920, + "is_alias": false, + "columns": [ + "x", + "Sales", + "CompPrice", + "Income", + "Advertising", + "Population", + "Price", + "ShelveLoc", + "Age", + "Education", + "Urban", + "US", + "High" + ], + "attributes": [], + "s3": { + "isSliced": true, + "region": "eu-central-1", + "bucket": "test", + "key": "test\/asdf\/12345.csv.gzmanifest", + "credentials": { + "access_key_id": "ASDF", + "secret_access_key": "1234", + "session_token": "abcd1234" + } + } +} \ No newline at end of file diff --git a/tests/test_dao.py b/tests/test_dao.py index 6f589a3..f33d82f 100644 --- a/tests/test_dao.py +++ b/tests/test_dao.py @@ -808,3 +808,30 @@ def test_all_tags(self): ) self.assertEqual(all_tags, file_def.tags) + + def test_build_from_manifest_s3_staging(self): + sample_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'data_examples', 'data_storage_staging_s3', 'in', 'tables') + + table_def = TableDefinition.build_from_manifest(os.path.join(sample_path, 'sample.csv.manifest')) + + self.assertEqual(table_def.s3_staging.bucket, "test") + self.assertEqual(table_def.s3_staging.credentials_access_key_id, "ASDF") + self.assertEqual(table_def.s3_staging.credentials_secret_access_key, "1234") + self.assertEqual(table_def.s3_staging.credentials_session_token ,"abcd1234") + self.assertEqual(table_def.s3_staging.is_sliced, True) + self.assertEqual(table_def.s3_staging.key, "test/asdf/12345.csv.gzmanifest") + self.assertEqual(table_def.s3_staging.region, "eu-central-1") + + def test_build_from_manifest_abs_staging(self): + sample_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'data_examples', 'data_storage_staging_abs', 'in', 'tables') + + table_def = TableDefinition.build_from_manifest(os.path.join(sample_path, 'sample.csv.manifest')) + + self.assertEqual(table_def.abs_staging.container, "exp-2-export-test-test") + self.assertEqual(table_def.abs_staging.credentials_expiration, "2020-08-27T22:42:08+0200") + self.assertEqual(table_def.abs_staging.credentials_sas_connection_string, "BlobEndpoint=https://asdf.blob.core.windows.net;SharedAccessSignature=sv=2017-11-09&sr=c&st=2020-08-27T08:42:08Z&se=2020-08-27T20:42:08Z&sp=rl&sig=UJW4DPh%2Baaaaaaaaaa") + self.assertEqual(table_def.abs_staging.is_sliced, True) + self.assertEqual(table_def.abs_staging.name, "12345.csv.gzmanifest") + self.assertEqual(table_def.abs_staging.region, "us-east-1") \ No newline at end of file