Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use Google Cloud log for stages/agents logger #812

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion common/cloud_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,6 @@ def _get_build_log(self, build_id: str) -> str:
bucket = self.storage_client.bucket(self.bucket_name)
blob = bucket.blob(log_file_uri)
log_content = self._extract_chat_history(blob.download_as_text())
logging.warning(log_content)
return log_content
except NotFound as e:
logging.error('Cloud build log %s not found: %s', log_file_uri, e)
Expand Down
16 changes: 15 additions & 1 deletion logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from typing import Mapping
from urllib.parse import urlparse

from google.cloud import logging as cloud_logging
from google.cloud import storage

from results import Result, RunResult, TrialResult
Expand Down Expand Up @@ -185,8 +186,21 @@ def error(msg: object,

def get_trial_logger(name: str = __name__,
trial: int = 0,
level=logging.DEBUG) -> CustomLoggerAdapter:
level: int = logging.DEBUG,
is_cloud: bool = False) -> CustomLoggerAdapter:
"""Sets up or retrieves a thread-local CustomLoggerAdapter for each thread."""

if is_cloud:
try:
client = cloud_logging.Client()
client.setup_logging()
except Exception as e:
logging.error(
'[Trial ID: %02d] Google Cloud log client initialization failure: %s',
trial, e)
else:
logging.debug('[Trial ID: %02d] Using local log.', trial)

logger = logging.getLogger(name)
if not logger.handlers:
formatter = logging.Formatter(
Expand Down
5 changes: 4 additions & 1 deletion stage/base_stage.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,10 @@ def __init__(self,
self.args = args
self.trial = trail
self.agents: list[BaseAgent] = agents or []
self.logger = logger.get_trial_logger(trial=trail)
self.logger = logger.get_trial_logger(
trial=trail,
is_cloud=args.cloud_experiment_name != '',
)
self.name: str = name or self.__class__.__name__

def __repr__(self) -> str:
Expand Down
2 changes: 1 addition & 1 deletion stage/writing_stage.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,6 @@ def execute(self, result_history: list[Result]) -> Result:
self.logger.write_fuzz_target(build_result)
self.logger.write_build_script(build_result)
self.logger.write_chat_history(build_result)
self.logger.debug('Writing stage completed with with result:\n%s',
self.logger.debug('Writing stage completed with with result: %s',
build_result)
return build_result