Merge pull request #590 from awslabs/patch/v1.1.0-ft

feat(be+fe): Features for FUTU

.github/workflows/pull-request-check.yaml

@@ -33,7 +33,7 @@ jobs:
             be+fe
             cdk
             others
-          subjectPattern: ^[\s\w-\.]{5,100}$
+          subjectPattern: ^[\s\w-\./]{5,100}$
           subjectPatternError: |
             The subject "{subject}" found in the pull request title "{title}"
             didn't match the configured pattern. Please ensure that the subject

.semgrepignore

@@ -0,0 +1,19 @@
+# 忽略所有在 tests 目录下的文件
+tests/
+
+# 忽略所有的 .json 文件
+*.json
+
+# 忽略特定的文件
+.gitignore
+deployment/cdk-solution-helper/index.js
+deployment/cdk-solution-helper/index.js
+deployment/cdk-solution-helper/index.js
+deployment/helper.py
+source/portal/config/env.js
+source/portal/config/modules.js
+source/portal/config/modules.js
+source/portal/config/modules.js
+source/portal/config/paths.js
+source/portal/nginx-config/start_nginx.sh
+.github/*

buildspec.yml

@@ -23,7 +23,7 @@ phases:
       - chmod +x ./run-all-tests.sh && ./run-all-tests.sh
       - echo "Installing dependencies and executing unit tests completed `date`"
       - export BSS_IMAGE_ASSET_REPOSITORY_NAME='aws-sensitive-data-protection'
-      - export BUILD_VERSION=1.1.1-${CODEBUILD_RESOLVED_SOURCE_VERSION:0:7}
+      - export BUILD_VERSION=1.1.2-${CODEBUILD_RESOLVED_SOURCE_VERSION:0:7}
       - export CN_ASSETS='cn/'
       - |-
         set -euxo pipefail

deployment/build-s3-dist.sh

@@ -42,6 +42,8 @@ title "cdk synth"
 run cd ${SRC_PATH}
 
 # Replace before building
+sed -i "s|DEBUG|INFO|"g api/logging.conf
+
 sed -i "s|@TEMPLATE_SOLUTION_VERSION@|$SOLUTION_VERSION|"g lib/admin/database/*/*.sql
 sed -i "s|@TEMPLATE_SOLUTION_VERSION@|$SOLUTION_VERSION|"g lib/agent/DiscoveryJob.json
 

source/.viperlightignore

@@ -18,6 +18,10 @@ constructs/api/pytest/test_data_source.py
 constructs/api/pytest/test_labels.py
 constructs/api/pytest/test_query.py
 constructs/lib/common/solution-info.ts:30
+constructs/config/batch_create/datasource/template/batch_create_jdbc_datasource-cn.xlsx
+constructs/config/batch_create/datasource/template/batch_create_jdbc_datasource-en.xlsx
+constructs/config/batch_create/identifier/template/batch_create_identifier-cn.xlsx
+constructs/config/batch_create/identifier/template/batch_create_identifier-en.xlsx
 
 [python-pipoutdated]
 pip=v21.1.2

source/constructs/api/README.md

@@ -23,16 +23,31 @@ As shown in the following format. Please follow this configuration. Select 'Othe
 ```json
 {"username":"db username","password":"db password","engine":"mysql","host":"127.0.0.1","port":6306}
 ```
-### 4. Configuration development mode
-The following command is configured as development mode to bypass authentication.
+### 4. Configure development parameters
 ```shell
+export AdminBucketName="Your admin bucket name"
+# The two subnets where the API lambda is located
+export SubnetIds="subnet-xxxxxx,subnet-xxxxxx"
+# Development mode that bypasses authentication
 export mode=dev
 ```
-### 5. Starting web services locally
+### 5. Run as API role
+Firstly, use `aws configure -- profile cn` to configure a user authorization information. This user needs to have `sts:AssumeRole` permission.  
+Secondly, modify the trust relationships of the SDPS API role in the Admin account and add the first step user in the principal.  
+Thirdly, modify `.aws/config` file and configure the default profile using the following command
+```
+[default]
+region = cn-northwest-1
+source_profile = cn
+role_arn = arn:aws-cn:iam::{AdminAccountId}:role/SDPSAPIRole-cn-northwest-1
+output = json
+```
+Finally, validate using `aws sts get-caller-identity`. If the returned content contains `arn:aws-cn:sts::{AdminAccountId}:assumed-role/SDPSAPIRole-{Region}`, it indicates that the configuration is correct.
+### 6. Starting web services locally
 ```shell
 uvicorn main:app --reload
 ```
-### 6. View API
+### 7. View API
 http://127.0.0.1:8000/docs
 
 ## File Naming

source/constructs/api/catalog/crud.py

@@ -669,6 +669,16 @@ def delete_catalog_table_level_classification_by_database_region(database: str,
     ).delete()
     session.commit()
 
+def delete_catalog_table_level_classification_by_database_region_batch(database: str, region: str, type: str):
+    session = get_session()
+    session.query(models.CatalogTableLevelClassification).filter(
+        models.CatalogTableLevelClassification.database_name == database,
+        models.CatalogTableLevelClassification.database_type == type
+    ).filter(
+        models.CatalogTableLevelClassification.region == region
+    ).delete(synchronize_session=False)
+    session.commit()
+
 
 def delete_catalog_table_level_classification_by_database(database: str, region: str, type: str):
     session = get_session()
@@ -698,6 +708,16 @@ def delete_catalog_database_level_classification_by_database_region(database: st
     ).delete()
     session.commit()
 
+def delete_catalog_database_level_classification_by_database_region_batch(database: str, region: str, type: str):
+    session = get_session()
+    session.query(models.CatalogDatabaseLevelClassification).filter(
+        models.CatalogDatabaseLevelClassification.database_name == database,
+        models.CatalogDatabaseLevelClassification.database_type == type
+    ).filter(
+        models.CatalogDatabaseLevelClassification.region == region
+    ).delete(synchronize_session=False)
+    session.commit()
+
 
 def delete_catalog_column_level_classification_by_database_region(database: str, region: str, type: str):
     session = get_session()
@@ -709,6 +729,16 @@ def delete_catalog_column_level_classification_by_database_region(database: str,
     ).delete()
     session.commit()
 
+def delete_catalog_column_level_classification_by_database_region_batch(database: str, region: str, type: str):
+    session = get_session()
+    session.query(models.CatalogColumnLevelClassification).filter(
+        models.CatalogColumnLevelClassification.database_name == database,
+        models.CatalogDatabaseLevelClassification.database_type == type
+    ).filter(
+        models.CatalogColumnLevelClassification.region == region
+    ).delete(synchronize_session=False)
+    session.commit()
+
 
 def delete_catalog_column_level_classification_by_database(database: str, region: str, type: str):
     session = get_session()
@@ -1056,11 +1086,14 @@ def update_catalog_table_labels(
 
 
 def get_export_catalog_data():
-    return get_session().query(models.CatalogColumnLevelClassification.account_id,
+    return get_session().query(models.CatalogColumnLevelClassification.database_type,
+                               models.CatalogColumnLevelClassification.account_id,
                                models.CatalogColumnLevelClassification.region,
-                               models.CatalogColumnLevelClassification.database_type,
                                models.CatalogColumnLevelClassification.database_name,
+                               models.CatalogDatabaseLevelClassification.description,
+                               models.CatalogDatabaseLevelClassification.url,
                                models.CatalogColumnLevelClassification.table_name,
+                               models.CatalogTableLevelClassification.storage_location,
                                models.CatalogColumnLevelClassification.column_name,
                                models.CatalogColumnLevelClassification.column_path,
                                models.CatalogColumnLevelClassification.identifier,

source/constructs/api/catalog/main.py

@@ -395,7 +395,6 @@ def agg_catalog_summary_by_privacy(database_type: str):
 @router.get("/dashboard/agg-catalog-top-n", response_model=BaseResponse)
 @inject_session
 def agg_catalog_top_n(database_type: str, top_n: int):
-
     return service_dashboard.agg_catalog_data_source_top_n(database_type, top_n)