Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(detect-secrets): get secrets plugins from config.yaml #6544

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

feat(detect-secrets): get secrets plugins from config.yaml #6544

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
18bb7b9
feat(detect-secrets): get secrets plugins from config.yaml
pedrooot Jan 15, 2025
7d19b73
feat(detect_secrets): resolve comments
pedrooot Jan 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions prowler/config/config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,37 @@ aws:
# Minimum retention period in hours for Kinesis streams
min_kinesis_stream_retention_hours: 168 # 7 days

# Detect Secrets plugin configuration
detect_secrets_plugins: [
{"name": "ArtifactoryDetector"},
{"name": "AWSKeyDetector"},
{"name": "AzureStorageKeyDetector"},
{"name": "BasicAuthDetector"},
{"name": "CloudantDetector"},
{"name": "DiscordBotTokenDetector"},
{"name": "GitHubTokenDetector"},
{"name": "GitLabTokenDetector"},
{"name": "Base64HighEntropyString", "limit": 6.0},
{"name": "HexHighEntropyString", "limit": 3.0},
{"name": "IbmCloudIamDetector"},
{"name": "IbmCosHmacDetector"},
# {"name": "IPPublicDetector"}, https://github.com/Yelp/detect-secrets/pull/885
{"name": "JwtTokenDetector"},
{"name": "KeywordDetector"},
{"name": "MailchimpDetector"},
{"name": "NpmDetector"},
{"name": "OpenAIDetector"},
{"name": "PrivateKeyDetector"},
{"name": "PypiTokenDetector"},
{"name": "SendGridDetector"},
{"name": "SlackDetector"},
{"name": "SoftlayerDetector"},
{"name": "SquareOAuthDetector"},
{"name": "StripeDetector"},
# {"name": "TelegramBotTokenDetector"}, https://github.com/Yelp/detect-secrets/pull/878
{"name": "TwilioKeyDetector"},
]


# Azure Configuration
azure:
Expand Down
70 changes: 40 additions & 30 deletions prowler/lib/utils/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,36 @@
from prowler.config.config import encoding_format_utf_8
from prowler.lib.logger import logger

default_detect_secrets_plugins = [
{"name": "ArtifactoryDetector"},
{"name": "AWSKeyDetector"},
{"name": "AzureStorageKeyDetector"},
{"name": "BasicAuthDetector"},
{"name": "CloudantDetector"},
{"name": "DiscordBotTokenDetector"},
{"name": "GitHubTokenDetector"},
{"name": "GitLabTokenDetector"},
{"name": "Base64HighEntropyString", "limit": 6.0},
{"name": "HexHighEntropyString", "limit": 3.0},
{"name": "IbmCloudIamDetector"},
{"name": "IbmCosHmacDetector"},
# {"name": "IPPublicDetector"}, https://github.com/Yelp/detect-secrets/pull/885
{"name": "JwtTokenDetector"},
{"name": "KeywordDetector"},
{"name": "MailchimpDetector"},
{"name": "NpmDetector"},
{"name": "OpenAIDetector"},
{"name": "PrivateKeyDetector"},
{"name": "PypiTokenDetector"},
{"name": "SendGridDetector"},
{"name": "SlackDetector"},
{"name": "SoftlayerDetector"},
{"name": "SquareOAuthDetector"},
{"name": "StripeDetector"},
# {"name": "TelegramBotTokenDetector"}, https://github.com/Yelp/detect-secrets/pull/878
{"name": "TwilioKeyDetector"},
]


def open_file(input_file: str, mode: str = "r") -> TextIOWrapper:
"""open_file returns a handler to the file using the specified mode."""
Expand Down Expand Up @@ -82,13 +112,17 @@ def hash_sha512(string: str) -> str:


def detect_secrets_scan(
data: str = None, file=None, excluded_secrets: list[str] = None
data: str = None,
file=None,
excluded_secrets: list[str] = None,
detect_secrets_plugins: dict = None,
) -> list[dict[str, str]]:
"""detect_secrets_scan scans the data or file for secrets using the detect-secrets library.
Args:
data (str): The data to scan for secrets.
file (str): The file to scan for secrets.
excluded_secrets (list): A list of regex patterns to exclude from the scan.
detect_secrets_plugins (dict): The settings to use for the scan.
Returns:
dict: The secrets found in the
Raises:
Expand All @@ -107,43 +141,19 @@ def detect_secrets_scan(

secrets = SecretsCollection()

if not detect_secrets_plugins:
detect_secrets_plugins = default_detect_secrets_plugins

settings = {
"plugins_used": [
{"name": "ArtifactoryDetector"},
{"name": "AWSKeyDetector"},
{"name": "AzureStorageKeyDetector"},
{"name": "BasicAuthDetector"},
{"name": "CloudantDetector"},
{"name": "DiscordBotTokenDetector"},
{"name": "GitHubTokenDetector"},
{"name": "GitLabTokenDetector"},
{"name": "Base64HighEntropyString", "limit": 6.0},
{"name": "HexHighEntropyString", "limit": 3.0},
{"name": "IbmCloudIamDetector"},
{"name": "IbmCosHmacDetector"},
# {"name": "IPPublicDetector"}, https://github.com/Yelp/detect-secrets/pull/885
{"name": "JwtTokenDetector"},
{"name": "KeywordDetector"},
{"name": "MailchimpDetector"},
{"name": "NpmDetector"},
{"name": "OpenAIDetector"},
{"name": "PrivateKeyDetector"},
{"name": "PypiTokenDetector"},
{"name": "SendGridDetector"},
{"name": "SlackDetector"},
{"name": "SoftlayerDetector"},
{"name": "SquareOAuthDetector"},
{"name": "StripeDetector"},
# {"name": "TelegramBotTokenDetector"}, https://github.com/Yelp/detect-secrets/pull/878
{"name": "TwilioKeyDetector"},
],
"plugins_used": detect_secrets_plugins,
"filters_used": [
{"path": "detect_secrets.filters.common.is_invalid_file"},
{"path": "detect_secrets.filters.common.is_known_false_positive"},
{"path": "detect_secrets.filters.heuristic.is_likely_id_string"},
{"path": "detect_secrets.filters.heuristic.is_potential_secret"},
],
}

if excluded_secrets and len(excluded_secrets) > 0:
settings["filters_used"].append(
{
Expand Down
6 changes: 5 additions & 1 deletion ...ind_secrets_ec2_launch_configuration/autoscaling_find_secrets_ec2_launch_configuration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,11 @@ def execute(self):
continue

has_secrets = detect_secrets_scan(
data=user_data, excluded_secrets=secrets_ignore_patterns
data=user_data,
excluded_secrets=secrets_ignore_patterns,
detect_secrets_plugins=autoscaling_client.audit_config.get(
"detect_secrets_plugins", None
),
)

if has_secrets:
Expand Down
3 changes: 3 additions & 0 deletions .../awslambda/awslambda_function_no_secrets_in_code/awslambda_function_no_secrets_in_code.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ def execute(self):
detect_secrets_output = detect_secrets_scan(
file=f"{tmp_dir_name}/{file}",
excluded_secrets=secrets_ignore_patterns,
detect_secrets_plugins=awslambda_client.audit_config.get(
"detect_secrets_plugins", {}
),
)
if detect_secrets_output:
for (
Expand Down
3 changes: 3 additions & 0 deletions .../awslambda_function_no_secrets_in_variables/awslambda_function_no_secrets_in_variables.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ def execute(self):
detect_secrets_output = detect_secrets_scan(
data=json.dumps(function.environment, indent=2),
excluded_secrets=secrets_ignore_patterns,
detect_secrets_plugins=awslambda_client.audit_config.get(
"detect_secrets_plugins", {}
),
)
if detect_secrets_output:
environment_variable_names = list(function.environment.keys())
Expand Down
6 changes: 5 additions & 1 deletion ...on/cloudformation_stack_outputs_find_secrets/cloudformation_stack_outputs_find_secrets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,11 @@ def execute(self):
data += f"{output}\n"

detect_secrets_output = detect_secrets_scan(
data=data, excluded_secrets=secrets_ignore_patterns
data=data,
excluded_secrets=secrets_ignore_patterns,
detect_secrets_plugins=cloudformation_client.audit_config.get(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you should get the default list of plugins instead of None or {}. Please review it in all checks.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree with you, changes on the next commit.

"detect_secrets_plugins", None
),
)
# If secrets are found, update the report status
if detect_secrets_output:
Expand Down
8 changes: 7 additions & 1 deletion ...dwatch/cloudwatch_log_group_no_secrets_in_logs/cloudwatch_log_group_no_secrets_in_logs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ def execute(self):
log_stream_secrets_output = detect_secrets_scan(
data=log_stream_data,
excluded_secrets=secrets_ignore_patterns,
detect_secrets_plugins=logs_client.audit_config.get(
"detect_secrets_plugins", {}
),
)

if log_stream_secrets_output:
Expand Down Expand Up @@ -68,7 +71,10 @@ def execute(self):
# Can get more informative output if there is more than 1 line.
# Will rescan just this event to get the type of secret and the line number
event_detect_secrets_output = detect_secrets_scan(
data=log_event_data
data=log_event_data,
detect_secrets_plugins=logs_client.audit_config.get(
"detect_secrets_plugins", {}
),
)
if event_detect_secrets_output:
for secret in event_detect_secrets_output:
Expand Down
3 changes: 3 additions & 0 deletions ...ld/codebuild_project_no_secrets_in_variables/codebuild_project_no_secrets_in_variables.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ def execute(self):
detect_secrets_output = detect_secrets_scan(
data=json.dumps({env_var.name: env_var.value}),
excluded_secrets=secrets_ignore_patterns,
detect_secrets_plugins=codebuild_client.audit_config.get(
"detect_secrets_plugins", {}
),
)
if detect_secrets_output:
secrets_info = [
Expand Down
6 changes: 5 additions & 1 deletion ...oviders/aws/services/ec2/ec2_instance_secrets_user_data/ec2_instance_secrets_user_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,11 @@ def execute(self):
)
continue
detect_secrets_output = detect_secrets_scan(
data=user_data, excluded_secrets=secrets_ignore_patterns
data=user_data,
excluded_secrets=secrets_ignore_patterns,
detect_secrets_plugins=ec2_client.audit_config.get(
"detect_secrets_plugins", None
),
)
if detect_secrets_output:
secrets_string = ", ".join(
Expand Down
6 changes: 5 additions & 1 deletion ...oviders/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,11 @@ def execute(self):
continue

version_secrets = detect_secrets_scan(
data=user_data, excluded_secrets=secrets_ignore_patterns
data=user_data,
excluded_secrets=secrets_ignore_patterns,
detect_secrets_plugins=ec2_client.audit_config.get(
"detect_secrets_plugins", None
),
)

if version_secrets:
Expand Down
6 changes: 5 additions & 1 deletion ...cs_task_definitions_no_environment_secrets/ecs_task_definitions_no_environment_secrets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,11 @@ def execute(self):

env_data = dumps(dump_env_vars, indent=2)
detect_secrets_output = detect_secrets_scan(
data=env_data, excluded_secrets=secrets_ignore_patterns
data=env_data,
excluded_secrets=secrets_ignore_patterns,
detect_secrets_plugins=ecs_client.audit_config.get(
"detect_secrets_plugins", None
),
)
if detect_secrets_output:
secrets_string = ", ".join(
Expand Down
3 changes: 3 additions & 0 deletions prowler/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ def execute(self):
detect_secrets_output = detect_secrets_scan(
data=json.dumps(document.content, indent=2),
excluded_secrets=secrets_ignore_patterns,
detect_secrets_plugins=ssm_client.audit_config.get(
"detect_secrets_plugins", None
),
)
if detect_secrets_output:
secrets_string = ", ".join(
Expand Down
6 changes: 4 additions & 2 deletions tests/lib/utils/utils_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ def test_validate_ip_address(self):
class Test_detect_secrets_scan:
def test_detect_secrets_scan_data(self):
data = "password=password"
secrets_detected = detect_secrets_scan(data=data)
secrets_detected = detect_secrets_scan(data=data, excluded_secrets=[])
assert type(secrets_detected) is list
assert len(secrets_detected) == 1
assert "filename" in secrets_detected[0]
Expand All @@ -128,7 +128,9 @@ def test_detect_secrets_scan_file_with_secrets(self):
temp_data_file = tempfile.NamedTemporaryFile(delete=False)
temp_data_file.write(b"password=password")
temp_data_file.seek(0)
secrets_detected = detect_secrets_scan(file=temp_data_file.name)
secrets_detected = detect_secrets_scan(
file=temp_data_file.name, excluded_secrets=[]
)
assert type(secrets_detected) is list
assert len(secrets_detected) == 1
assert "filename" in secrets_detected[0]
Expand Down
4 changes: 4 additions & 0 deletions ...rs/aws/services/ec2/ec2_launch_template_no_secrets/ec2_launch_template_no_secrets_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ def test_one_launch_template_with_secrets_in_multiple_versions(self):
)

ec2_client.launch_templates = [launch_template]
ec2_client.audit_config = {"detect_secrets_plugins": None}

with mock.patch(
"prowler.providers.common.provider.Provider.get_global_provider",
Expand Down Expand Up @@ -277,6 +278,7 @@ def test_one_launch_template_with_secrets_in_single_version(self):
)

ec2_client.launch_templates = [launch_template]
ec2_client.audit_config = {"detect_secrets_plugins": None}

with mock.patch(
"prowler.providers.common.provider.Provider.get_global_provider",
Expand Down Expand Up @@ -341,6 +343,7 @@ def test_one_launch_template_with_secrets_gzip(self):
)

ec2_client.launch_templates = [launch_template]
ec2_client.audit_config = {"detect_secrets_plugins": None}

with mock.patch(
"prowler.providers.common.provider.Provider.get_global_provider",
Expand Down Expand Up @@ -482,6 +485,7 @@ def test_two_launch_templates_one_template_with_secrets(self):
launch_template_secrets,
launch_template_no_secrets,
]
ec2_client.audit_config = {"detect_secrets_plugins": None}

with mock.patch(
"prowler.providers.common.provider.Provider.get_global_provider",
Expand Down
1 change: 1 addition & 0 deletions tests/providers/aws/services/ssm/ssm_document_secrets/ssm_document_secrets_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ def test_document_with_secrets(self):
document_name = "test-document"
document_arn = f"arn:aws:ssm:{AWS_REGION_US_EAST_1}:{AWS_ACCOUNT_NUMBER}:document/{document_name}"
ssm_client.audited_account = AWS_ACCOUNT_NUMBER
ssm_client.audit_config = {"detect_secrets_plugins": None}
ssm_client.documents = {
document_name: Document(
arn=document_arn,
Expand Down
Loading