Skip to content

⚡️ Speed up function filter_bad_extensions by 19% #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
codeflash-ai wants to merge 1 commit into
base: main
Choose a base branch
from
Open

⚡️ Speed up function filter_bad_extensions by 19% #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 21 additions & 13 deletions pr_agent/algo/language_handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,26 +6,25 @@

def filter_bad_extensions(files):
# Bad Extensions, source: https://github.com/EleutherAI/github-downloader/blob/345e7c4cbb9e0dc8a0615fd995a08bf9d73b3fe6/download_repo_text.py # noqa: E501
bad_extensions = get_settings().bad_extensions.default
if get_settings().config.use_extra_bad_extensions:
bad_extensions += get_settings().bad_extensions.extra
settings = get_settings()
bad_extensions = _get_combined_bad_extensions(settings)
return [f for f in files if f.filename is not None and is_valid_file(f.filename, bad_extensions)]


def is_valid_file(filename:str, bad_extensions=None) -> bool:
def is_valid_file(filename: str, bad_extensions=None) -> bool:
if not filename:
return False
if not bad_extensions:
bad_extensions = get_settings().bad_extensions.default
if get_settings().config.use_extra_bad_extensions:
bad_extensions += get_settings().bad_extensions.extra

auto_generated_files = ['package-lock.json', 'yarn.lock', 'composer.lock', 'Gemfile.lock', 'poetry.lock']
for forbidden_file in auto_generated_files:
if filename.endswith(forbidden_file):
return False
# Only call get_settings() if bad_extensions is not passed in
if bad_extensions is None:
settings = get_settings()
bad_extensions = _get_combined_bad_extensions(settings)

return filename.split('.')[-1] not in bad_extensions
auto_generated_files = ("package-lock.json", "yarn.lock", "composer.lock", "Gemfile.lock", "poetry.lock")
if filename.endswith(auto_generated_files):
return False

return filename.split(".")[-1] not in bad_extensions


def sort_files_by_main_languages(languages: Dict, files: list):
Expand Down Expand Up @@ -75,3 +74,12 @@ def sort_files_by_main_languages(languages: Dict, files: list):
files_sorted.append({"language": lang, "files": tmp})
files_sorted.append({"language": "Other", "files": list(rest_files.values())})
return files_sorted


def _get_combined_bad_extensions(settings):
# Helper that safely creates a combined bad extension list
bad_ext = settings.bad_extensions.default
if settings.config.use_extra_bad_extensions:
# Avoid mutating the settings.bad_extensions.default list
bad_ext = bad_ext + settings.bad_extensions.extra
return set(bad_ext) # Convert to set for fast lookup
71 changes: 40 additions & 31 deletions pr_agent/config_loader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,35 +5,38 @@
from dynaconf import Dynaconf
from starlette_context import context

PR_AGENT_TOML_KEY = 'pr-agent'
PR_AGENT_TOML_KEY = "pr-agent"

current_dir = dirname(abspath(__file__))
global_settings = Dynaconf(
envvar_prefix=False,
merge_enabled=True,
settings_files=[join(current_dir, f) for f in [
"settings/configuration.toml",
"settings/ignore.toml",
"settings/generated_code_ignore.toml",
"settings/language_extensions.toml",
"settings/pr_reviewer_prompts.toml",
"settings/pr_questions_prompts.toml",
"settings/pr_line_questions_prompts.toml",
"settings/pr_description_prompts.toml",
"settings/code_suggestions/pr_code_suggestions_prompts.toml",
"settings/code_suggestions/pr_code_suggestions_prompts_not_decoupled.toml",
"settings/code_suggestions/pr_code_suggestions_reflect_prompts.toml",
"settings/pr_information_from_user_prompts.toml",
"settings/pr_update_changelog_prompts.toml",
"settings/pr_custom_labels.toml",
"settings/pr_add_docs.toml",
"settings/custom_labels.toml",
"settings/pr_help_prompts.toml",
"settings/pr_help_docs_prompts.toml",
"settings/pr_help_docs_headings_prompts.toml",
"settings/.secrets.toml",
"settings_prod/.secrets.toml",
]]
settings_files=[
join(current_dir, f)
for f in [
"settings/configuration.toml",
"settings/ignore.toml",
"settings/generated_code_ignore.toml",
"settings/language_extensions.toml",
"settings/pr_reviewer_prompts.toml",
"settings/pr_questions_prompts.toml",
"settings/pr_line_questions_prompts.toml",
"settings/pr_description_prompts.toml",
"settings/code_suggestions/pr_code_suggestions_prompts.toml",
"settings/code_suggestions/pr_code_suggestions_prompts_not_decoupled.toml",
"settings/code_suggestions/pr_code_suggestions_reflect_prompts.toml",
"settings/pr_information_from_user_prompts.toml",
"settings/pr_update_changelog_prompts.toml",
"settings/pr_custom_labels.toml",
"settings/pr_add_docs.toml",
"settings/custom_labels.toml",
"settings/pr_help_prompts.toml",
"settings/pr_help_docs_prompts.toml",
"settings/pr_help_docs_headings_prompts.toml",
"settings/.secrets.toml",
"settings_prod/.secrets.toml",
]
],
)


Expand Down Expand Up @@ -81,7 +84,7 @@ def _find_pyproject() -> Optional[Path]:

pyproject_path = _find_pyproject()
if pyproject_path is not None:
get_settings().load_file(pyproject_path, env=f'tool.{PR_AGENT_TOML_KEY}')
get_settings().load_file(pyproject_path, env=f"tool.{PR_AGENT_TOML_KEY}")


def apply_secrets_manager_config():
Expand All @@ -90,15 +93,17 @@ def apply_secrets_manager_config():
"""
try:
# Dynamic imports to avoid circular dependency (secret_providers imports config_loader)
from pr_agent.secret_providers import get_secret_provider
from pr_agent.log import get_logger
from pr_agent.secret_providers import get_secret_provider

secret_provider = get_secret_provider()
if not secret_provider:
return

if (hasattr(secret_provider, 'get_all_secrets') and
get_settings().get("CONFIG.SECRET_PROVIDER") == 'aws_secrets_manager'):
if (
hasattr(secret_provider, "get_all_secrets")
and get_settings().get("CONFIG.SECRET_PROVIDER") == "aws_secrets_manager"
):
try:
secrets = secret_provider.get_all_secrets()
if secrets:
Expand All @@ -109,6 +114,7 @@ def apply_secrets_manager_config():
except Exception as e:
try:
from pr_agent.log import get_logger

get_logger().debug(f"Secret provider not configured: {e}")
except:
# Fail completely silently if log module is not available
Expand All @@ -123,14 +129,17 @@ def apply_secrets_to_config(secrets: dict):
# Dynamic import to avoid potential circular dependency
from pr_agent.log import get_logger
except:

def get_logger():
class DummyLogger:
def debug(self, msg): pass
def debug(self, msg):
pass

return DummyLogger()

for key, value in secrets.items():
if '.' in key: # nested key like "openai.key"
parts = key.split('.')
if "." in key: # nested key like "openai.key"
parts = key.split(".")
if len(parts) == 2:
section, setting = parts
section_upper = section.upper()
Expand Down