Skip to content

Commit

Permalink
[#5778] feat(aliyun-bundles)support OSS secret key credential (#5814)
Browse files Browse the repository at this point in the history
### What changes were proposed in this pull request?

Support OSS secret key credential

### Why are the changes needed?

Fix: [# (5778)](#5778)

### How was this patch tested?

IcebergRESTOSSSecretIT
  • Loading branch information
sunxiaojian authored Dec 11, 2024
1 parent 31d3b4a commit 26a8b37
Show file tree
Hide file tree
Showing 9 changed files with 336 additions and 15 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.gravitino.credential;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;

/** OSS secret key credential. */
public class OSSSecretKeyCredential implements Credential {

/** OSS secret key credential type. */
public static final String OSS_SECRET_KEY_CREDENTIAL_TYPE = "oss-secret-key";
/** The static access key ID used to access OSS data. */
public static final String GRAVITINO_OSS_STATIC_ACCESS_KEY_ID = "oss-access-key-id";
/** The static secret access key used to access OSS data. */
public static final String GRAVITINO_OSS_STATIC_SECRET_ACCESS_KEY = "oss-secret-access-key";

private String accessKeyId;
private String secretAccessKey;

/**
* Constructs an instance of {@link OSSSecretKeyCredential} with the static OSS access key ID and
* secret access key.
*
* @param accessKeyId The OSS static access key ID.
* @param secretAccessKey The OSS static secret access key.
*/
public OSSSecretKeyCredential(String accessKeyId, String secretAccessKey) {
validate(accessKeyId, secretAccessKey, 0);
this.accessKeyId = accessKeyId;
this.secretAccessKey = secretAccessKey;
}

/**
* This is the constructor that is used by credential factory to create an instance of credential
* according to the credential information.
*/
public OSSSecretKeyCredential() {}

@Override
public String credentialType() {
return OSS_SECRET_KEY_CREDENTIAL_TYPE;
}

@Override
public long expireTimeInMs() {
return 0;
}

@Override
public Map<String, String> credentialInfo() {
return (new ImmutableMap.Builder<String, String>())
.put(GRAVITINO_OSS_STATIC_ACCESS_KEY_ID, accessKeyId)
.put(GRAVITINO_OSS_STATIC_SECRET_ACCESS_KEY, secretAccessKey)
.build();
}

/**
* Initialize the credential with the credential information.
*
* <p>This method is invoked to deserialize the credential in client side.
*
* @param credentialInfo The credential information from {@link #credentialInfo}.
* @param expireTimeInMs The expire-time from {@link #expireTimeInMs()}.
*/
@Override
public void initialize(Map<String, String> credentialInfo, long expireTimeInMs) {
String accessKeyId = credentialInfo.get(GRAVITINO_OSS_STATIC_ACCESS_KEY_ID);
String secretAccessKey = credentialInfo.get(GRAVITINO_OSS_STATIC_SECRET_ACCESS_KEY);
validate(accessKeyId, secretAccessKey, expireTimeInMs);
this.accessKeyId = accessKeyId;
this.secretAccessKey = secretAccessKey;
}

/**
* Get OSS static access key ID.
*
* @return The OSS access key ID.
*/
public String accessKeyId() {
return accessKeyId;
}

/**
* Get OSS static secret access key.
*
* @return The OSS secret access key.
*/
public String secretAccessKey() {
return secretAccessKey;
}

private void validate(String accessKeyId, String secretAccessKey, long expireTimeInMs) {
Preconditions.checkArgument(
StringUtils.isNotBlank(accessKeyId), "OSS access key Id should not empty");
Preconditions.checkArgument(
StringUtils.isNotBlank(secretAccessKey), "OSS secret access key should not empty");
Preconditions.checkArgument(
expireTimeInMs == 0, "The expire time of OSSSecretKeyCredential is not 0");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,4 @@ org.apache.gravitino.credential.S3TokenCredential
org.apache.gravitino.credential.S3SecretKeyCredential
org.apache.gravitino.credential.GCSTokenCredential
org.apache.gravitino.credential.OSSTokenCredential
org.apache.gravitino.credential.OSSSecretKeyCredential
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.gravitino.oss.credential;

import java.util.Map;
import org.apache.gravitino.credential.Credential;
import org.apache.gravitino.credential.CredentialContext;
import org.apache.gravitino.credential.CredentialProvider;
import org.apache.gravitino.credential.OSSSecretKeyCredential;
import org.apache.gravitino.credential.config.OSSCredentialConfig;

/** Generate OSS access key and secret key to access OSS data. */
public class OSSSecretKeyProvider implements CredentialProvider {

private String accessKey;
private String secretKey;

@Override
public void initialize(Map<String, String> properties) {
OSSCredentialConfig ossCredentialConfig = new OSSCredentialConfig(properties);
this.accessKey = ossCredentialConfig.accessKeyID();
this.secretKey = ossCredentialConfig.secretAccessKey();
}

@Override
public void close() {}

@Override
public String credentialType() {
return OSSSecretKeyCredential.OSS_SECRET_KEY_CREDENTIAL_TYPE;
}

@Override
public Credential getCredential(CredentialContext context) {
return new OSSSecretKeyCredential(accessKey, secretKey);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,5 @@
# specific language governing permissions and limitations
# under the License.
#
org.apache.gravitino.oss.credential.OSSTokenProvider
org.apache.gravitino.oss.credential.OSSTokenProvider
org.apache.gravitino.oss.credential.OSSSecretKeyProvider
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ public static Map<String, String> toIcebergProperties(Credential credential) {
if (credential instanceof S3TokenCredential || credential instanceof S3SecretKeyCredential) {
return transformProperties(credential.credentialInfo(), icebergCredentialPropertyMap);
}
if (credential instanceof OSSTokenCredential) {
if (credential instanceof OSSTokenCredential || credential instanceof OSSSecretKeyCredential) {
return transformProperties(credential.credentialInfo(), icebergCredentialPropertyMap);
}
return credential.toProperties();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,4 +113,29 @@ void testOSSTokenCredential() {
Assertions.assertEquals("token", ossTokenCredential1.securityToken());
Assertions.assertEquals(expireTime, ossTokenCredential1.expireTimeInMs());
}

@Test
void testOSSSecretKeyTokenCredential() {
Map<String, String> ossSecretKeyCredentialInfo =
ImmutableMap.of(
OSSSecretKeyCredential.GRAVITINO_OSS_STATIC_ACCESS_KEY_ID,
"accessKeyId",
OSSSecretKeyCredential.GRAVITINO_OSS_STATIC_SECRET_ACCESS_KEY,
"secretAccessKey");
long expireTime = 0;
Credential ossSecretKeyCredential =
CredentialFactory.create(
OSSSecretKeyCredential.OSS_SECRET_KEY_CREDENTIAL_TYPE,
ossSecretKeyCredentialInfo,
expireTime);
Assertions.assertEquals(
OSSSecretKeyCredential.OSS_SECRET_KEY_CREDENTIAL_TYPE,
ossSecretKeyCredential.credentialType());
Assertions.assertTrue(ossSecretKeyCredential instanceof OSSSecretKeyCredential);
OSSSecretKeyCredential ossSecretKeyCredential1 =
(OSSSecretKeyCredential) ossSecretKeyCredential;
Assertions.assertEquals("accessKeyId", ossSecretKeyCredential1.accessKeyId());
Assertions.assertEquals("secretAccessKey", ossSecretKeyCredential1.secretAccessKey());
Assertions.assertEquals(expireTime, ossSecretKeyCredential1.expireTimeInMs());
}
}
21 changes: 11 additions & 10 deletions docs/iceberg-rest-service.md
Original file line number Diff line number Diff line change
Expand Up @@ -130,16 +130,17 @@ To configure the JDBC catalog backend, set the `gravitino.iceberg-rest.warehouse

Gravitino Iceberg REST service supports using static access-key-id and secret-access-key or generating temporary token to access OSS data.

| Configuration item | Description | Default value | Required | Since Version |
|---------------------------------------------------|------------------------------------------------------------------------------------------------------------------|-----------------|----------|------------------|
| `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.aliyun.oss.OSSFileIO` for OSS. | (none) | No | 0.6.0-incubating |
| `gravitino.iceberg-rest.oss-access-key-id` | The static access key ID used to access OSS data. | (none) | No | 0.7.0-incubating |
| `gravitino.iceberg-rest.oss-secret-access-key` | The static secret access key used to access OSS data. | (none) | No | 0.7.0-incubating |
| `gravitino.iceberg-rest.oss-endpoint` | The endpoint of Aliyun OSS service. | (none) | No | 0.7.0-incubating |
| `gravitino.iceberg-rest.oss-region` | The region of the OSS service, like `oss-cn-hangzhou`, only used when `credential-provider-type` is `oss-token`. | (none) | No | 0.8.0-incubating |
| `gravitino.iceberg-rest.oss-role-arn` | The ARN of the role to access the OSS data, only used when `credential-provider-type` is `oss-token`. | (none) | No | 0.8.0-incubating |
| `gravitino.iceberg-rest.oss-external-id` | The OSS external id to generate token, only used when `credential-provider-type` is `oss-token`. | (none) | No | 0.8.0-incubating |
| `gravitino.iceberg-rest.oss-token-expire-in-secs` | The OSS security token expire time in secs, only used when `credential-provider-type` is `oss-token`. | 3600 | No | 0.8.0-incubating |
| Configuration item | Description | Default value | Required | Since Version |
|---------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------|------------------------------------------------------|------------------|
| `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.aliyun.oss.OSSFileIO` for OSS. | (none) | No | 0.6.0-incubating |
| `gravitino.iceberg-rest.credential-provider-type` | Supports `oss-token` and `oss-secret-key` for OSS. `oss-token` generates a temporary token according to the query data path while `oss-secret-key` using the oss secret access key to access S3 data. | (none) | No | 0.7.0-incubating |
| `gravitino.iceberg-rest.oss-access-key-id` | The static access key ID used to access OSS data. | (none) | No | 0.7.0-incubating |
| `gravitino.iceberg-rest.oss-secret-access-key` | The static secret access key used to access OSS data. | (none) | No | 0.7.0-incubating |
| `gravitino.iceberg-rest.oss-endpoint` | The endpoint of Aliyun OSS service. | (none) | No | 0.7.0-incubating |
| `gravitino.iceberg-rest.oss-region` | The region of the OSS service, like `oss-cn-hangzhou`, only used when `credential-provider-type` is `oss-token`. | (none) | No | 0.8.0-incubating |
| `gravitino.iceberg-rest.oss-role-arn` | The ARN of the role to access the OSS data, only used when `credential-provider-type` is `oss-token`. | (none) | Yes, when `credential-provider-type` is `oss-token`. | 0.8.0-incubating |
| `gravitino.iceberg-rest.oss-external-id` | The OSS external id to generate token, only used when `credential-provider-type` is `oss-token`. | (none) | No | 0.8.0-incubating |
| `gravitino.iceberg-rest.oss-token-expire-in-secs` | The OSS security token expire time in secs, only used when `credential-provider-type` is `oss-token`. | 3600 | No | 0.8.0-incubating |

For other Iceberg OSS properties not managed by Gravitino like `client.security-token`, you could config it directly by `gravitino.iceberg-rest.client.security-token`.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,7 @@ public class IcebergCatalogWrapper implements AutoCloseable {
IcebergConstants.IO_IMPL,
IcebergConstants.AWS_S3_REGION,
IcebergConstants.ICEBERG_S3_ENDPOINT,
IcebergConstants.ICEBERG_OSS_ENDPOINT,
IcebergConstants.ICEBERG_OSS_ACCESS_KEY_ID,
IcebergConstants.ICEBERG_OSS_ACCESS_KEY_SECRET);
IcebergConstants.ICEBERG_OSS_ENDPOINT);

public IcebergCatalogWrapper(IcebergConfig icebergConfig) {
this.catalogBackend =
Expand Down
Loading

0 comments on commit 26a8b37

Please sign in to comment.