Fix report csv (#127)

* Fix report csv

* adding comment

---------

Co-authored-by: Nader <[email protected]>

Author: nadersip

arch/templates/EvidenceCollectionComponents.yaml

@@ -235,13 +235,15 @@ Resources:
                   - "s3:GetObject"
                   - "s3:CreateBucket"
                   - "s3:ListBucket"
+                  - "s3:DeleteObject"
                 Resource: !Sub "arn:aws:s3:::gc-fedclient-${AWS::AccountId}-${AWS::Region}/*"
               - Effect: Allow
                 Action:
                   - "s3:PutObject"
                   - "s3:GetObject"
                   - "s3:CreateBucket"
                   - "s3:ListBucket"
+                  - "s3:DeleteObject"
                 Resource: !Sub "arn:aws:s3:::gc-fedclient-${AWS::AccountId}-${AWS::Region}"
               - Effect: Allow
                 Action:

src/lambda/aws_bucket_watcher/app.py

@@ -19,12 +19,17 @@ def lambda_handler(event, context):
     key = event["Records"][0]["s3"]["object"]["key"]
     s3_resource = boto3.resource("s3")
     copy_source = {"Bucket": bucket, "Key": key}
+
+    #print(key)
 
     #Todo: Not sure why this exist, its not being used anywhere? 
     #account_id = context.invoked_function_arn.split(":")[4]
     target_key = f"{org_id}/{key}"
     logger.info("Attempting to copy audit data to GC managed s3 bucket: %s", target_key)
-    
-    s3_resource.Bucket(os.environ["target_bucket"]).Object(target_key).copy(
-        copy_source, ExtraArgs={"ACL": "bucket-owner-full-control"}
-    )
+
+    if key.startswith("chunks/") or key.startswith("state/"):
+        pass
+    else:
+        s3_resource.Bucket(os.environ["target_bucket"]).Object(target_key).copy(
+            copy_source, ExtraArgs={"ACL": "bucket-owner-full-control"}
+        )

src/lambda/aws_compile_audit_report/app.py

@@ -62,9 +62,32 @@ def create_boto3_clients():
 
 def lambda_handler(event, context):
     logger.info("Lambda invocation started (structured).")
-
+    
     clients = create_boto3_clients()
 
+    # Delete state file before proceeding
+    try:
+        s3 = boto3.client("s3")
+        objects = s3.list_objects_v2(Bucket=config["SOURCE_TARGET_BUCKET"], Prefix="state")
+        if 'Contents' in objects:
+            for obj in objects['Contents']:
+                s3.delete_object(Bucket=config["SOURCE_TARGET_BUCKET"], Key=obj["Key"])
+               # logger.info(f"Delete complete{obj}")
+    except Exception as e:
+        logger.info("Failed to delete S3 states folder: %s", str(e))
+ 
+    # Delete Chunk files before proceeding
+
+    try:
+        s3 = boto3.client("s3")
+        objects = s3.list_objects_v2(Bucket=config["SOURCE_TARGET_BUCKET"], Prefix="chunks")
+        if 'Contents' in objects:
+            for obj in objects['Contents']:
+                s3.delete_object(Bucket=config["SOURCE_TARGET_BUCKET"], Key=obj["Key"])
+               # logger.info(f"Delete complete{obj}")
+    except Exception as e:
+        logger.info("Failed to delete S3 chunks folder: %s", str(e))
+
     # Handle concurrency limit
     current_concurrency = event.get("current_concurrency", 1)
     if current_concurrency > config["MAX_CONCURRENCY"]:
@@ -149,6 +172,9 @@ def process_assessments(event, context, current_concurrency, clients):
             with open(chunk_file_local, "w", newline="") as csvfile:
                 writer = csv.writer(csvfile)
                 writer.writerow(OUTPUT_HEADER)
+                # Log the creation of the chunk file
+                logger.info(f"Created chunk file at {chunk_file_local}")
+
 
                 all_evidence_pages = get_all_evidence_paginated(
                     clients["auditmanager"], assessment_id, control_set_id, folder_id