diff --git a/api/src/main/java/org/apache/gravitino/credential/AzureAccountKeyCredential.java b/api/src/main/java/org/apache/gravitino/credential/AzureAccountKeyCredential.java new file mode 100644 index 00000000000..be24d7cda0e --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/credential/AzureAccountKeyCredential.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; + +/** Azure account key credential. */ +public class AzureAccountKeyCredential implements Credential { + + /** Azure account key credential type. */ + public static final String AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE = "azure-account-key"; + /** Azure storage account name */ + public static final String GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME = "azure-storage-account-name"; + /** Azure storage account key */ + public static final String GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY = "azure-storage-account-key"; + + private String accountName; + private String accountKey; + + /** + * Constructs an instance of {@link AzureAccountKeyCredential}. + * + * @param accountName The Azure account name. + * @param accountKey The Azure account key. + */ + public AzureAccountKeyCredential(String accountName, String accountKey) { + validate(accountName, accountKey); + this.accountName = accountName; + this.accountKey = accountKey; + } + + /** + * This is the constructor that is used by credential factory to create an instance of credential + * according to the credential information. + */ + public AzureAccountKeyCredential() {} + + @Override + public String credentialType() { + return AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE; + } + + @Override + public long expireTimeInMs() { + return 0; + } + + @Override + public Map credentialInfo() { + return (new ImmutableMap.Builder()) + .put(GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME, accountName) + .put(GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY, accountKey) + .build(); + } + + @Override + public void initialize(Map credentialInfo, long expireTimeInMS) { + String accountName = credentialInfo.get(GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME); + String accountKey = credentialInfo.get(GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY); + validate(accountName, accountKey); + this.accountName = accountName; + this.accountKey = accountKey; + } + + /** + * Get Azure account name + * + * @return The Azure account name + */ + public String accountName() { + return accountName; + } + + /** + * Get Azure account key + * + * @return The Azure account key + */ + public String accountKey() { + return accountKey; + } + + private void validate(String accountName, String accountKey) { + Preconditions.checkArgument( + StringUtils.isNotBlank(accountName), "Azure account name should not be empty."); + Preconditions.checkArgument( + StringUtils.isNotBlank(accountKey), "Azure account key should not be empty."); + } +} diff --git a/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential b/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential index f130b4b6423..6071cb916ae 100644 --- a/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential +++ b/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential @@ -23,3 +23,4 @@ org.apache.gravitino.credential.GCSTokenCredential org.apache.gravitino.credential.OSSTokenCredential org.apache.gravitino.credential.OSSSecretKeyCredential org.apache.gravitino.credential.ADLSTokenCredential +org.apache.gravitino.credential.AzureAccountKeyCredential diff --git a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java index e2ee3ed82a3..c2b684acbde 100644 --- a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java +++ b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java @@ -38,7 +38,7 @@ import org.apache.gravitino.credential.CredentialContext; import org.apache.gravitino.credential.CredentialProvider; import org.apache.gravitino.credential.PathBasedCredentialContext; -import org.apache.gravitino.credential.config.ADLSCredentialConfig; +import org.apache.gravitino.credential.config.AzureCredentialConfig; /** Generates ADLS token to access ADLS data. */ public class ADLSTokenProvider implements CredentialProvider { @@ -51,14 +51,14 @@ public class ADLSTokenProvider implements CredentialProvider { @Override public void initialize(Map properties) { - ADLSCredentialConfig adlsCredentialConfig = new ADLSCredentialConfig(properties); - this.storageAccountName = adlsCredentialConfig.storageAccountName(); - this.tenantId = adlsCredentialConfig.tenantId(); - this.clientId = adlsCredentialConfig.clientId(); - this.clientSecret = adlsCredentialConfig.clientSecret(); + AzureCredentialConfig azureCredentialConfig = new AzureCredentialConfig(properties); + this.storageAccountName = azureCredentialConfig.storageAccountName(); + this.tenantId = azureCredentialConfig.tenantId(); + this.clientId = azureCredentialConfig.clientId(); + this.clientSecret = azureCredentialConfig.clientSecret(); this.endpoint = String.format("https://%s.%s", storageAccountName, ADLSTokenCredential.ADLS_DOMAIN); - this.tokenExpireSecs = adlsCredentialConfig.tokenExpireInSecs(); + this.tokenExpireSecs = azureCredentialConfig.adlsTokenExpireInSecs(); } @Override diff --git a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java new file mode 100644 index 00000000000..726c4f2d996 --- /dev/null +++ b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.abs.credential; + +import java.util.Map; +import org.apache.gravitino.credential.AzureAccountKeyCredential; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.credential.CredentialContext; +import org.apache.gravitino.credential.CredentialProvider; +import org.apache.gravitino.credential.config.AzureCredentialConfig; + +/** Generates Azure account key to access data. */ +public class AzureAccountKeyProvider implements CredentialProvider { + private String accountName; + private String accountKey; + + @Override + public void initialize(Map properties) { + AzureCredentialConfig azureCredentialConfig = new AzureCredentialConfig(properties); + this.accountName = azureCredentialConfig.storageAccountName(); + this.accountKey = azureCredentialConfig.storageAccountKey(); + } + + @Override + public void close() {} + + @Override + public String credentialType() { + return CredentialConstants.AZURE_ACCOUNT_KEY_CREDENTIAL_PROVIDER_TYPE; + } + + @Override + public Credential getCredential(CredentialContext context) { + return new AzureAccountKeyCredential(accountName, accountKey); + } +} diff --git a/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider index fb53efffa63..4c7e7982cb1 100644 --- a/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider +++ b/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider @@ -16,4 +16,5 @@ # specific language governing permissions and limitations # under the License. # -org.apache.gravitino.abs.credential.ADLSTokenProvider \ No newline at end of file +org.apache.gravitino.abs.credential.ADLSTokenProvider +org.apache.gravitino.abs.credential.AzureAccountKeyProvider \ No newline at end of file diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java index 7dd74d08484..29f9241c890 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java @@ -32,5 +32,7 @@ public class CredentialConstants { public static final String ADLS_TOKEN_CREDENTIAL_PROVIDER_TYPE = "adls-token"; public static final String ADLS_TOKEN_EXPIRE_IN_SECS = "adls-token-expire-in-secs"; + public static final String AZURE_ACCOUNT_KEY_CREDENTIAL_PROVIDER_TYPE = "azure-account-key"; + private CredentialConstants() {} } diff --git a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java index e1803a6ddf1..d7a3caf067f 100644 --- a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java +++ b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java @@ -33,12 +33,19 @@ public class CredentialPropertyUtils { @VisibleForTesting static final String ICEBERG_S3_SECRET_ACCESS_KEY = "s3.secret-access-key"; @VisibleForTesting static final String ICEBERG_S3_TOKEN = "s3.session-token"; @VisibleForTesting static final String ICEBERG_GCS_TOKEN = "gcs.oauth2.token"; - @VisibleForTesting static final String ICEBERG_ADLS_TOKEN = "adls.sas-token"; @VisibleForTesting static final String ICEBERG_OSS_ACCESS_KEY_ID = "client.access-key-id"; @VisibleForTesting static final String ICEBERG_OSS_ACCESS_KEY_SECRET = "client.access-key-secret"; @VisibleForTesting static final String ICEBERG_OSS_SECURITY_TOKEN = "client.security-token"; + @VisibleForTesting static final String ICEBERG_ADLS_TOKEN = "adls.sas-token"; + + @VisibleForTesting + static final String ICEBERG_ADLS_ACCOUNT_NAME = "adls.auth.shared-key.account.name"; + + @VisibleForTesting + static final String ICEBERG_ADLS_ACCOUNT_KEY = "adls.auth.shared-key.account.key"; + private static Map icebergCredentialPropertyMap = ImmutableMap.of( GCSTokenCredential.GCS_TOKEN_NAME, @@ -54,7 +61,11 @@ public class CredentialPropertyUtils { OSSTokenCredential.GRAVITINO_OSS_SESSION_ACCESS_KEY_ID, ICEBERG_OSS_ACCESS_KEY_ID, OSSTokenCredential.GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY, - ICEBERG_OSS_ACCESS_KEY_SECRET); + ICEBERG_OSS_ACCESS_KEY_SECRET, + AzureAccountKeyCredential.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME, + ICEBERG_ADLS_ACCOUNT_NAME, + AzureAccountKeyCredential.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY, + ICEBERG_ADLS_ACCOUNT_KEY); /** * Transforms a specific credential into a map of Iceberg properties. @@ -63,6 +74,14 @@ public class CredentialPropertyUtils { * @return a map of Iceberg properties derived from the credential */ public static Map toIcebergProperties(Credential credential) { + if (credential instanceof S3TokenCredential + || credential instanceof S3SecretKeyCredential + || credential instanceof OSSTokenCredential + || credential instanceof OSSSecretKeyCredential + || credential instanceof AzureAccountKeyCredential) { + return transformProperties(credential.credentialInfo(), icebergCredentialPropertyMap); + } + if (credential instanceof GCSTokenCredential) { Map icebergGCSCredentialProperties = transformProperties(credential.credentialInfo(), icebergCredentialPropertyMap); @@ -70,12 +89,7 @@ public static Map toIcebergProperties(Credential credential) { "gcs.oauth2.token-expires-at", String.valueOf(credential.expireTimeInMs())); return icebergGCSCredentialProperties; } - if (credential instanceof S3TokenCredential || credential instanceof S3SecretKeyCredential) { - return transformProperties(credential.credentialInfo(), icebergCredentialPropertyMap); - } - if (credential instanceof OSSTokenCredential || credential instanceof OSSSecretKeyCredential) { - return transformProperties(credential.credentialInfo(), icebergCredentialPropertyMap); - } + if (credential instanceof ADLSTokenCredential) { ADLSTokenCredential adlsCredential = (ADLSTokenCredential) credential; String sasTokenKey = @@ -87,6 +101,7 @@ public static Map toIcebergProperties(Credential credential) { icebergADLSCredentialProperties.put(sasTokenKey, adlsCredential.sasToken()); return icebergADLSCredentialProperties; } + return credential.toProperties(); } diff --git a/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java b/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java index 75a669e3887..6291b8857d7 100644 --- a/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java +++ b/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java @@ -165,4 +165,31 @@ void testADLSTokenCredential() { Assertions.assertEquals(sasToken, adlsTokenCredential.sasToken()); Assertions.assertEquals(expireTime, adlsTokenCredential.expireTimeInMs()); } + + @Test + void testAzureAccountKeyCredential() { + String storageAccountName = "storage-account-name"; + String storageAccountKey = "storage-account-key"; + + Map azureAccountKeyCredentialInfo = + ImmutableMap.of( + AzureAccountKeyCredential.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME, + storageAccountName, + AzureAccountKeyCredential.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY, + storageAccountKey); + long expireTime = 0; + Credential credential = + CredentialFactory.create( + AzureAccountKeyCredential.AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE, + azureAccountKeyCredentialInfo, + expireTime); + Assertions.assertEquals( + AzureAccountKeyCredential.AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE, credential.credentialType()); + Assertions.assertInstanceOf(AzureAccountKeyCredential.class, credential); + + AzureAccountKeyCredential azureAccountKeyCredential = (AzureAccountKeyCredential) credential; + Assertions.assertEquals(storageAccountName, azureAccountKeyCredential.accountName()); + Assertions.assertEquals(storageAccountKey, azureAccountKeyCredential.accountKey()); + Assertions.assertEquals(expireTime, azureAccountKeyCredential.expireTimeInMs()); + } } diff --git a/core/src/main/java/org/apache/gravitino/credential/config/ADLSCredentialConfig.java b/core/src/main/java/org/apache/gravitino/credential/config/AzureCredentialConfig.java similarity index 96% rename from core/src/main/java/org/apache/gravitino/credential/config/ADLSCredentialConfig.java rename to core/src/main/java/org/apache/gravitino/credential/config/AzureCredentialConfig.java index e9d368e6752..155cc6806e0 100644 --- a/core/src/main/java/org/apache/gravitino/credential/config/ADLSCredentialConfig.java +++ b/core/src/main/java/org/apache/gravitino/credential/config/AzureCredentialConfig.java @@ -29,7 +29,7 @@ import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.storage.AzureProperties; -public class ADLSCredentialConfig extends Config { +public class AzureCredentialConfig extends Config { public static final ConfigEntry AZURE_STORAGE_ACCOUNT_NAME = new ConfigBuilder(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME) @@ -79,7 +79,7 @@ public class ADLSCredentialConfig extends Config { .intConf() .createWithDefault(3600); - public ADLSCredentialConfig(Map properties) { + public AzureCredentialConfig(Map properties) { super(false); loadFromMap(properties, k -> true); } @@ -110,7 +110,7 @@ public String clientSecret() { } @NotNull - public Integer tokenExpireInSecs() { + public Integer adlsTokenExpireInSecs() { return this.get(ADLS_TOKEN_EXPIRE_IN_SECS); } } diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index 8d9d49745c2..f31aa13685a 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -106,18 +106,18 @@ The detailed configuration items are as follows: Gravitino Iceberg REST service supports using static S3 secret key or generating temporary token to access S3 data. -| Configuration item | Description | Default value | Required | Since Version | -|---------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------------------------------------------------|------------------| -| `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.aws.s3.S3FileIO` for S3. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.credential-provider-type` | Supports `s3-token` and `s3-secret-key` for S3. `s3-token` generates a temporary token according to the query data path while `s3-secret-key` using the s3 secret access key to access S3 data. | (none) | No | 0.7.0-incubating | -| `gravitino.iceberg-rest.s3-access-key-id` | The static access key ID used to access S3 data. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.s3-secret-access-key` | The static secret access key used to access S3 data. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.s3-endpoint` | An alternative endpoint of the S3 service, This could be used for S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.s3-region` | The region of the S3 service, like `us-west-2`. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.s3-role-arn` | The ARN of the role to access the S3 data. | (none) | Yes, when `credential-provider-type` is `s3-token` | 0.7.0-incubating | -| `gravitino.iceberg-rest.s3-external-id` | The S3 external id to generate token, only used when `credential-provider-type` is `s3-token`. | (none) | No | 0.7.0-incubating | -| `gravitino.iceberg-rest.s3-token-expire-in-secs` | The S3 session token expire time in secs, it couldn't exceed the max session time of the assumed role, only used when `credential-provider-type` is `s3-token`. | 3600 | No | 0.7.0-incubating | -| `gravitino.iceberg-rest.s3-token-service-endpoint` | An alternative endpoint of the S3 token service, This could be used with s3-compatible object storage service like MINIO that has a different STS endpoint. | (none) | No | 0.8.0-incubating | +| Configuration item | Description | Default value | Required | Since Version | +|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------------------------------------------------|------------------| +| `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.aws.s3.S3FileIO` for S3. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.credential-provider-type` | Supports `s3-token` and `s3-secret-key` for S3. `s3-token` generates a temporary token according to the query data path while `s3-secret-key` using the s3 secret access key to access S3 data. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.s3-access-key-id` | The static access key ID used to access S3 data. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.s3-secret-access-key` | The static secret access key used to access S3 data. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.s3-endpoint` | An alternative endpoint of the S3 service, This could be used for S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.s3-region` | The region of the S3 service, like `us-west-2`. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.s3-role-arn` | The ARN of the role to access the S3 data. | (none) | Yes, when `credential-provider-type` is `s3-token` | 0.7.0-incubating | +| `gravitino.iceberg-rest.s3-external-id` | The S3 external id to generate token, only used when `credential-provider-type` is `s3-token`. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.s3-token-expire-in-secs` | The S3 session token expire time in secs, it couldn't exceed the max session time of the assumed role, only used when `credential-provider-type` is `s3-token`. | 3600 | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.s3-token-service-endpoint` | An alternative endpoint of the S3 token service, This could be used with s3-compatible object storage service like MINIO that has a different STS endpoint. | (none) | No | 0.8.0-incubating | For other Iceberg s3 properties not managed by Gravitino like `s3.sse.type`, you could config it directly by `gravitino.iceberg-rest.s3.sse.type`. @@ -175,15 +175,16 @@ Please set `gravitino.iceberg-rest.warehouse` to `gs://{bucket_name}/${prefix_na Gravitino Iceberg REST service supports generating SAS token to access ADLS data. -| Configuration item | Description | Default value | Required | Since Version | -|-----------------------------------------------------|-----------------------------------------------------------------------------------------------------------|---------------|----------|------------------| -| `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.azure.adlsv2.ADLSFileIO` for ADLS. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.credential-provider-type` | Supports `adls-token`, generates a temporary token according to the query data path. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.azure-storage-account-name` | The static storage account name used to access ADLS data. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.azure-storage-account-key` | The static storage account key used to access ADLS data. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.azure-tenant-id` | Azure Active Directory (AAD) tenant ID. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.azure-client-id` | Azure Active Directory (AAD) client ID used for authentication. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.azure-client-secret` | Azure Active Directory (AAD) client secret used for authentication. | (none) | Yes | 0.8.0-incubating | +| Configuration item | Description | Default value | Required | Since Version | +|-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------|------------------| +| `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.azure.adlsv2.ADLSFileIO` for ADLS. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.credential-provider-type` | Supports `adls-token` and `azure-account-key`. `adls-token` generates a temporary token according to the query data path while `azure-account-key` uses a storage account key to access ADLS data. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.azure-storage-account-name` | The static storage account name used to access ADLS data. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.azure-storage-account-key` | The static storage account key used to access ADLS data. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.azure-tenant-id` | Azure Active Directory (AAD) tenant ID, only used when `credential-provider-type` is `adls-token`. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.azure-client-id` | Azure Active Directory (AAD) client ID used for authentication, only used when `credential-provider-type` is `adls-token`. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.azure-client-secret` | Azure Active Directory (AAD) client secret used for authentication, only used when `credential-provider-type` is `adls-token`. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.adls-token-expire-in-secs` | The ADLS SAS token expire time in secs, only used when `credential-provider-type` is `adls-token`. | 3600 | No | 0.8.0-incubating | For other Iceberg ADLS properties not managed by Gravitino like `adls.read.block-size-bytes`, you could config it directly by `gravitino.iceberg-rest.adls.read.block-size-bytes`. diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java similarity index 92% rename from iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSIT.java rename to iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java index 570298d050b..b16d504e1ea 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java @@ -36,7 +36,7 @@ @SuppressWarnings("FormatStringAnnotation") @EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches = "true") -public class IcebergRESTADLSIT extends IcebergRESTJdbcCatalogIT { +public class IcebergRESTADLSTokenIT extends IcebergRESTJdbcCatalogIT { private String storageAccountName; private String storageAccountKey; @@ -49,13 +49,14 @@ public class IcebergRESTADLSIT extends IcebergRESTJdbcCatalogIT { void initEnv() { this.storageAccountName = System.getenv() - .getOrDefault("GRAVITINO_ADLS_STORAGE_ACCOUNT_NAME", "{STORAGE_ACCOUNT_NAME}"); + .getOrDefault("GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME", "{STORAGE_ACCOUNT_NAME}"); this.storageAccountKey = - System.getenv().getOrDefault("GRAVITINO_ADLS_STORAGE_ACCOUNT_KEY", "{STORAGE_ACCOUNT_KEY}"); - this.tenantId = System.getenv().getOrDefault("GRAVITINO_ADLS_TENANT_ID", "{TENANT_ID}"); - this.clientId = System.getenv().getOrDefault("GRAVITINO_ADLS_CLIENT_ID", "{CLIENT_ID}"); + System.getenv() + .getOrDefault("GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY", "{STORAGE_ACCOUNT_KEY}"); + this.tenantId = System.getenv().getOrDefault("GRAVITINO_AZURE_TENANT_ID", "{TENANT_ID}"); + this.clientId = System.getenv().getOrDefault("GRAVITINO_AZURE_CLIENT_ID", "{CLIENT_ID}"); this.clientSecret = - System.getenv().getOrDefault("GRAVITINO_ADLS_CLIENT_SECRET", "{CLIENT_SECRET}"); + System.getenv().getOrDefault("GRAVITINO_AZURE_CLIENT_SECRET", "{CLIENT_SECRET}"); this.warehousePath = String.format( "abfss://%s@%s.dfs.core.windows.net/data/test", diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java new file mode 100644 index 00000000000..42709162aaa --- /dev/null +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.integration.test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.gravitino.integration.test.util.BaseIT; +import org.apache.gravitino.integration.test.util.DownloaderUtils; +import org.apache.gravitino.integration.test.util.ITUtils; +import org.apache.gravitino.storage.AzureProperties; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; + +@SuppressWarnings("FormatStringAnnotation") +@EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches = "true") +public class IcebergRESTAzureAccountKeyIT extends IcebergRESTJdbcCatalogIT { + + private String storageAccountName; + private String storageAccountKey; + private String warehousePath; + + @Override + void initEnv() { + this.storageAccountName = + System.getenv() + .getOrDefault("GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME", "{STORAGE_ACCOUNT_NAME}"); + this.storageAccountKey = + System.getenv() + .getOrDefault("GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY", "{STORAGE_ACCOUNT_KEY}"); + this.warehousePath = + String.format( + "abfss://%s@%s.dfs.core.windows.net/data/test", + System.getenv().getOrDefault("GRAVITINO_ADLS_CONTAINER", "{ADLS_CONTAINER}"), + storageAccountName); + + if (ITUtils.isEmbedded()) { + return; + } + try { + downloadIcebergAzureBundleJar(); + } catch (IOException e) { + LOG.warn("Download Iceberg Azure bundle jar failed,", e); + throw new RuntimeException(e); + } + copyAzureBundleJar(); + } + + @Override + public Map getCatalogConfig() { + HashMap m = new HashMap(); + m.putAll(getCatalogJdbcConfig()); + m.putAll(getADLSConfig()); + return m; + } + + public boolean supportsCredentialVending() { + return true; + } + + private Map getADLSConfig() { + Map configMap = new HashMap(); + + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, + CredentialConstants.AZURE_ACCOUNT_KEY_CREDENTIAL_PROVIDER_TYPE); + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME, + storageAccountName); + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY, + storageAccountKey); + + configMap.put( + IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConstants.IO_IMPL, + "org.apache.iceberg.azure.adlsv2.ADLSFileIO"); + configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConstants.WAREHOUSE, warehousePath); + + return configMap; + } + + private void downloadIcebergAzureBundleJar() throws IOException { + String icebergBundleJarName = "iceberg-azure-bundle-1.5.2.jar"; + String icebergBundleJarUri = + "https://repo1.maven.org/maven2/org/apache/iceberg/" + + "iceberg-azure-bundle/1.5.2/" + + icebergBundleJarName; + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome); + DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir); + } + + private void copyAzureBundleJar() { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome); + BaseIT.copyBundleJarsToDirectory("azure-bundle", targetDir); + } +}