From 3fb413c3894e5fbc2a426e5d6c4da66673bfbe82 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Fri, 29 May 2026 20:15:30 +0800 Subject: [PATCH 01/20] Enhance unit test script and CI pipeline for AI failure analysis - Updated run_ut.sh to support additional command-line arguments for failure context and failed test cases. - Implemented functions to handle rerunning failed test cases based on their categories (base, inc, llmc). - Improved environment setup functions for INC and LLMC unit tests. - Modified the unit test execution logic to accommodate reruns of failed tests. - Enhanced the Azure Pipelines template (ut-template.yml) to include parameters for failure log context and failed test cases. - Added a new AI analysis stage in the unit-test.yml pipeline to handle failure context merging and analysis. - Introduced new scripts for AI failure analysis, including analyze_and_suggest.py, merge_failure_context.py, and post_pr_comment.py. - Created a new template (ai-analysis-template.yml) for AI analysis steps in the CI pipeline. - Implemented logic to post analysis results as comments on pull requests. Signed-off-by: chensuyue --- .../analyze_and_suggest.py | 435 ++++++++++++++++++ .../merge_failure_context.py | 134 ++++++ .../ai_failure_analysis/post_pr_comment.py | 125 +++++ .azure-pipelines/scripts/ut/collect_result.py | 135 ++++++ .azure-pipelines/scripts/ut/run_ut.sh | 228 ++++++--- .../template/ai-analysis-template.yml | 107 +++++ .azure-pipelines/template/ut-template.yml | 27 +- .azure-pipelines/unit-test.yml | 23 + 8 files changed, 1153 insertions(+), 61 deletions(-) create mode 100644 .azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py create mode 100644 .azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py create mode 100644 .azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py create mode 100644 .azure-pipelines/template/ai-analysis-template.yml diff --git a/.azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py b/.azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py new file mode 100644 index 000000000..68a85d3e0 --- /dev/null +++ b/.azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py @@ -0,0 +1,435 @@ +import argparse +import json +import os +import re +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + + +def load_json(path: Path): + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def write_text(path: Path, content: str): + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + f.write(content) + + +def load_text(path: Path) -> str: + if not path.exists(): + return "" + with open(path, "r", encoding="utf-8") as f: + return f.read() + + +def get_token_from_env() -> str: + return ( + os.environ.get("COPILOT_GITHUB_TOKEN", "") + or os.environ.get("GH_TOKEN", "") + or os.environ.get("GITHUB_TOKEN", "") + ) + + +def run_cmd(cmd: list[str], cwd: Path | None = None, env: dict | None = None) -> tuple[int, str, str]: + result = subprocess.run(cmd, cwd=cwd, env=env, check=False, capture_output=True, text=True) + return result.returncode, result.stdout or "", result.stderr or "" + + +def truncate_text(value: str, limit: int = 6000) -> str: + if len(value) <= limit: + return value + return value[: limit - 3] + "..." + + +def collect_failed_log_paths(payload: dict, failed_logs_root: Path | None, project_root: Path) -> list[str]: + requested = [ + (entry.get("log_file") or "").strip() + for entry in payload.get("failures", []) + if (entry.get("log_file") or "").strip() + ] + + candidate_roots = [] + if failed_logs_root: + candidate_roots.append(failed_logs_root) + + # Default location used by CI failure packaging. + candidate_roots.append(project_root / "log_dir" / "failure_logs") + + found: list[str] = [] + seen: set[str] = set() + for root in candidate_roots: + if not root.exists(): + continue + + for name in requested: + direct = root / name + matches: list[Path] = [] + if direct.exists(): + matches = [direct] + else: + matches = list(root.rglob(name)) + + for match in matches: + resolved = str(match.resolve()) + if resolved in seen: + continue + seen.add(resolved) + found.append(resolved) + + return sorted(found) + + +def generate_pr_patch(project_root: Path, output_path: Path, base_ref: str = "main") -> tuple[bool, str]: + diff_specs = [ + f"origin/{base_ref}...HEAD", + f"{base_ref}...HEAD", + "HEAD~1..HEAD", + ] + + for spec in diff_specs: + code, stdout, _stderr = run_cmd(["git", "diff", "--binary", "--minimal", "--no-color", spec], cwd=project_root) + if code != 0: + continue + + output_path.parent.mkdir(parents=True, exist_ok=True) + write_text(output_path, stdout) + return True, f"Generated PR patch using git diff spec: {spec}" + + write_text(output_path, "") + return False, "Failed to generate PR patch via git diff" + + +def extract_json_from_response(text: str) -> dict | None: + if not text.strip(): + return None + + fenced = re.search(r"```json\s*(\{.*?\})\s*```", text, flags=re.DOTALL | re.IGNORECASE) + if fenced: + try: + return json.loads(fenced.group(1)) + except json.JSONDecodeError: + pass + + bare = re.search(r"(\{.*\})", text, flags=re.DOTALL) + if bare: + try: + return json.loads(bare.group(1)) + except json.JSONDecodeError: + return None + + return None + + +def build_agent_prompt( + merged_failure_context: Path, + failed_log_paths_file: Path, + pr_patch_file: Path, + project_root: Path, +) -> str: + return "\n".join( + [ + "You are a non-interactive coding agent for CI failure analysis.", + "Your task MUST follow these steps in order:", + "1) Error classification: summarize failure types and group failed tests by root symptom.", + "2) Root-cause analysis: reason about likely root cause with PR changes and project code context.", + "3) Fix suggestion: provide a minimal and safe code change proposal.", + "4) Static checks: run at least one static check command and report command and result.", + "", + "Use these context sources:", + f"- merged_failure_context_json: {merged_failure_context}", + f"- failed_log_paths_list: {failed_log_paths_file}", + f"- pr_code_changes_patch: {pr_patch_file}", + f"- project_source_root: {project_root}", + "", + "Tooling policy:", + "- Non-interactive only; do not request user input.", + "- Prefer read-only inspection of files and git history.", + "- Keep suggestions minimal and low risk.", + "", + "Output STRICT JSON only with this schema:", + "{", + ' "error_classification": [', + " {", + ' "category": "string",', + ' "evidence": ["string"],', + ' "tests": ["string"]', + " }", + " ],", + ' "root_cause": "string",', + ' "confidence": "low|medium|high",', + ' "suggestion": "string",', + ' "patch": "unified diff string or empty",', + ' "static_checks": [', + " {", + ' "command": "string",', + ' "status": "pass|fail|not_run",', + ' "output": "string"', + " }", + " ],", + ' "selected_model": "string (optional if available)"', + "}", + ] + ) + + +def run_copilot_cli(prompt: str, project_root: Path, github_token: str) -> tuple[bool, str, dict | None, str]: + env = os.environ.copy() + if github_token: + env["COPILOT_GITHUB_TOKEN"] = github_token + + cmd = [ + "copilot", + "-p", + prompt, + "--allow-tool=shell(git:*)", + "--allow-tool=shell(python:*)", + "--allow-tool=shell(rg:*)", + "--allow-tool=read", + "--allow-tool=write", + "--no-ask-user", + ] + code, stdout, stderr = run_cmd(cmd, cwd=project_root, env=env) + raw_text = (stdout or "").strip() + parsed = extract_json_from_response(raw_text) + + message = f"copilot cli exit={code}" + if stderr.strip(): + message += f"; stderr={truncate_text(stderr.strip(), 1200)}" + + if code == 0 and parsed: + return True, message, parsed, raw_text + return False, message, parsed, raw_text + + +def fallback_analysis(payload: dict) -> dict: + failures = payload.get("failures", []) + if not failures: + return { + "error_classification": [], + "root_cause": "No failed cases were found in merged failure context.", + "confidence": "low", + "suggestion": "No patch generated.", + "patch": "", + "static_checks": [ + { + "command": "not_run", + "status": "not_run", + "output": "Copilot CLI analysis was not available; no agent static checks executed.", + } + ], + } + + first = failures[0] + excerpt = (first.get("excerpt", "") or first.get("tail", ""))[:1200] + return { + "error_classification": [ + { + "category": "test_failure", + "evidence": ["first failure excerpt"], + "tests": [first.get("test_name", "")], + } + ], + "root_cause": "Likely regression in the changed code path exercised by failed unit tests. Inspect the first traceback for precise failure location.", + "confidence": "medium", + "suggestion": "Review traceback, apply minimal fix, and rerun only failed test files.", + "patch": "", + "static_checks": [ + { + "command": "not_run", + "status": "not_run", + "output": "Copilot CLI analysis was not available; no agent static checks executed.", + } + ], + "first_failure_excerpt": excerpt, + } + + +def run_local_static_checks(project_root: Path, pr_patch_path: Path) -> list[dict]: + checks: list[dict] = [] + patch_text = load_text(pr_patch_path) + changed_files = sorted(set(re.findall(r"^\+\+\+ b/(.*\.py)$", patch_text, flags=re.MULTILINE))) + + for rel_path in changed_files[:20]: + file_path = project_root / rel_path + if not file_path.exists(): + continue + code, stdout, stderr = run_cmd([sys.executable, "-m", "py_compile", str(file_path)], cwd=project_root) + checks.append( + { + "command": f"{sys.executable} -m py_compile {rel_path}", + "status": "pass" if code == 0 else "fail", + "output": truncate_text((stdout + "\n" + stderr).strip(), 2000), + } + ) + + if not checks: + checks.append( + { + "command": f"{sys.executable} -m py_compile .azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py", + "status": "pass", + "output": "No changed Python files detected in PR patch for local static checks.", + } + ) + return checks + + +def main(): + parser = argparse.ArgumentParser(description="Analyze merged failure context and generate fix artifacts") + parser.add_argument("--failure-context", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--failed-test-cases", required=True, type=Path) + parser.add_argument("--project-root", type=Path, default=Path.cwd()) + parser.add_argument("--failed-logs-root", type=Path, default=None) + parser.add_argument("--base-ref", default="main") + args = parser.parse_args() + + args.output_dir.mkdir(parents=True, exist_ok=True) + payload = load_json(args.failure_context) + project_root = args.project_root.resolve() + token = get_token_from_env() + + failed_log_paths = collect_failed_log_paths(payload, args.failed_logs_root, project_root) + failed_log_paths_file = args.output_dir / "failed_log_paths.txt" + write_text(failed_log_paths_file, "\n".join(failed_log_paths) + ("\n" if failed_log_paths else "")) + + pr_patch_path = args.output_dir / "pr_code_changes.patch" + patch_ok, patch_message = generate_pr_patch(project_root, pr_patch_path, base_ref=args.base_ref) + + prompt = build_agent_prompt( + merged_failure_context=args.failure_context, + failed_log_paths_file=failed_log_paths_file, + pr_patch_file=pr_patch_path, + project_root=project_root, + ) + prompt_path = args.output_dir / "copilot_prompt.txt" + write_text(prompt_path, prompt) + + cli_ok, cli_message, cli_result, cli_raw = run_copilot_cli(prompt, project_root, token) + + fallback = fallback_analysis(payload) + analysis = fallback.copy() + if cli_ok and cli_result: + analysis.update( + { + "error_classification": cli_result.get("error_classification", analysis.get("error_classification", [])), + "root_cause": cli_result.get("root_cause", analysis.get("root_cause", "")), + "confidence": cli_result.get("confidence", analysis.get("confidence", "medium")), + "suggestion": cli_result.get("suggestion", analysis.get("suggestion", "")), + "patch": cli_result.get("patch", analysis.get("patch", "")), + "static_checks": cli_result.get("static_checks", analysis.get("static_checks", [])), + } + ) + if cli_result.get("selected_model"): + analysis["selected_model"] = cli_result["selected_model"] + elif cli_raw: + analysis["raw_response"] = truncate_text(cli_raw, 6000) + + analysis["local_static_checks"] = run_local_static_checks(project_root, pr_patch_path) + + patch_path = args.output_dir / "suggested_fix.patch" + report_path = args.output_dir / "ai_failure_report.md" + result_path = args.output_dir / "analysis_result.json" + + if not patch_path.exists(): + write_text(patch_path, analysis.get("patch", "")) + + report_lines = [ + "# Copilot Failure Analysis", + "", + f"- Model: auto (resolved: {analysis.get('selected_model', 'unknown')})", + f"- Copilot CLI status: {'success' if cli_ok else 'fallback'}", + f"- Copilot CLI message: {cli_message}", + "- External Copilot command executed: no", + "- External command status: disabled", + f"- PR patch generated: {'yes' if patch_ok else 'no'} ({patch_message})", + f"- Project source root: {project_root}", + f"- Failed logs provided: {len(failed_log_paths)}", + "", + "## Error Classification", + ] + + classes = analysis.get("error_classification", []) + if classes: + for item in classes: + category = item.get("category", "unknown") + tests = ", ".join(item.get("tests", [])[:10]) + evidence = "; ".join(item.get("evidence", [])[:3]) + report_lines.append(f"- {category}: tests=[{tests}] evidence={evidence}") + else: + report_lines.append("- No structured classification returned.") + + report_lines.extend([ + "", + "## Root Cause", + analysis.get("root_cause", "No root cause available."), + "", + "## Suggested Fix", + analysis.get("suggestion", "No suggestion available."), + "", + "## Static Checks (Agent)", + ]) + + for check in analysis.get("static_checks", [])[:20]: + report_lines.append(f"- [{check.get('status', 'not_run')}] {check.get('command', 'unknown')}") + + report_lines.extend([ + "", + "## Static Checks (Local)", + ]) + for check in analysis.get("local_static_checks", [])[:20]: + report_lines.append(f"- [{check.get('status', 'not_run')}] {check.get('command', 'unknown')}") + + report_lines.append("") + + excerpt = analysis.get("first_failure_excerpt", "") + if excerpt: + report_lines.extend([ + "## First Failure Excerpt", + "```text", + excerpt, + "```", + "", + ]) + + write_text(report_path, "\n".join(report_lines)) + + result_payload = { + "success": cli_ok, + "copilot_cli_ok": cli_ok, + "copilot_cli_message": cli_message, + "external_copilot_ok": False, + "external_message": "disabled", + "model": "auto", + "selected_model": analysis.get("selected_model", ""), + "error_classification": analysis.get("error_classification", []), + "root_cause": analysis.get("root_cause", ""), + "confidence": analysis.get("confidence", ""), + "suggestion": analysis.get("suggestion", ""), + "static_checks": analysis.get("static_checks", []), + "local_static_checks": analysis.get("local_static_checks", []), + "prompt_path": str(prompt_path), + "failed_log_paths_file": str(failed_log_paths_file), + "pr_patch_reference": str(pr_patch_path), + "report_path": str(report_path), + "patch_path": str(patch_path), + "failed_test_cases": str(args.failed_test_cases), + "project_root": str(project_root), + "generated_at": datetime.now(timezone.utc).isoformat(), + "source_commit": os.environ.get("SYSTEM_PULLREQUEST_SOURCECOMMITID", os.environ.get("BUILD_SOURCEVERSION", "")), + "pr_number": os.environ.get("SYSTEM_PULLREQUEST_PULLREQUESTNUMBER", ""), + } + + with open(result_path, "w", encoding="utf-8") as f: + json.dump(result_payload, f, indent=2) + + print(f"Analysis report: {report_path}") + print(f"Suggested patch: {patch_path}") + print(f"Analysis result: {result_path}") + + +if __name__ == "__main__": + main() diff --git a/.azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py b/.azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py new file mode 100644 index 000000000..69ecc76da --- /dev/null +++ b/.azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py @@ -0,0 +1,134 @@ +import argparse +import json +import os +import re +from pathlib import Path + + +def load_json(path: Path): + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def normalize_failure(entry: dict, source_file: str, artifact: str) -> dict: + return { + "source_file": source_file, + "artifact": artifact, + "test_name": entry.get("test_name", ""), + "status": entry.get("status", ""), + "log_file": entry.get("log_file", ""), + "duration": entry.get("duration", ""), + "excerpt": entry.get("excerpt", ""), + "tail": entry.get("tail", ""), + } + + +def get_artifact_name(context_file: Path, input_root: Path) -> str: + rel = context_file.relative_to(input_root) + if rel.parts: + return rel.parts[0] + return context_file.parent.name + + +def parse_artifact_attempt(artifact_name: str) -> tuple[str, str, int] | None: + # Expected artifact format: -- + match = re.match(r"^(?P.+)-(?P[^-]+)-(?P\d+)$", artifact_name) + if not match: + return None + return match.group("prefix"), match.group("part"), int(match.group("attempt")) + + +def select_latest_context_files(context_files: list[Path], input_root: Path) -> list[Path]: + latest_by_part: dict[tuple[str, str], tuple[int, Path]] = {} + passthrough: list[Path] = [] + + for context_file in context_files: + artifact_name = get_artifact_name(context_file, input_root) + parsed = parse_artifact_attempt(artifact_name) + if parsed is None: + passthrough.append(context_file) + continue + + prefix, part, attempt = parsed + key = (prefix, part) + current = latest_by_part.get(key) + if current is None or attempt > current[0]: + latest_by_part[key] = (attempt, context_file) + + selected = passthrough + [item[1] for item in latest_by_part.values()] + return sorted(selected) + + +def merge_contexts(input_root: Path) -> tuple[list[dict], list[str]]: + all_context_files = sorted(input_root.rglob("failure_context*.json")) + context_files = select_latest_context_files(all_context_files, input_root) + merged_failures: list[dict] = [] + seen: set[tuple[str, str]] = set() + + for context_file in context_files: + artifact = get_artifact_name(context_file, input_root) + payload = load_json(context_file) + for entry in payload.get("failures", []): + normalized = normalize_failure(entry, str(context_file), artifact) + key = (normalized.get("test_name", ""), normalized.get("log_file", "")) + if key in seen: + continue + seen.add(key) + merged_failures.append(normalized) + + rerun_targets: list[str] = [] + for item in merged_failures: + test_name = item.get("test_name", "") + if not test_name: + continue + if not test_name.startswith("test_"): + continue + rerun_targets.append(f"test/test_cpu/{test_name}.py") + + rerun_targets = sorted(set(rerun_targets)) + return merged_failures, rerun_targets + + +def main(): + parser = argparse.ArgumentParser(description="Merge failure context files from all test parts") + parser.add_argument("--input-root", required=True, type=Path, help="Root folder containing downloaded failure artifacts") + parser.add_argument("--output", required=True, type=Path, help="Merged failure context JSON path") + parser.add_argument("--failed-test-cases", required=True, type=Path, help="Output failed test cases list") + args = parser.parse_args() + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.failed_test_cases.parent.mkdir(parents=True, exist_ok=True) + + failures, rerun_targets = merge_contexts(args.input_root) + + payload = { + "schema_version": "1.0", + "build": { + "build_id": os.environ.get("BUILD_BUILDID", ""), + "source_commit": os.environ.get("SYSTEM_PULLREQUEST_SOURCECOMMITID", os.environ.get("BUILD_SOURCEVERSION", "")), + "pr_number": os.environ.get("SYSTEM_PULLREQUEST_PULLREQUESTNUMBER", ""), + }, + "stats": { + "failed_cases": len(failures), + "rerun_targets": len(rerun_targets), + }, + "failures": failures, + "rerun": { + "test_files": rerun_targets, + }, + } + + with open(args.output, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + + with open(args.failed_test_cases, "w", encoding="utf-8") as f: + if rerun_targets: + f.write("\n".join(rerun_targets) + "\n") + + print(f"Merged {len(failures)} failures from {args.input_root}") + print(f"Merged context: {args.output}") + print(f"Rerun targets: {args.failed_test_cases}") + + +if __name__ == "__main__": + main() diff --git a/.azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py b/.azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py new file mode 100644 index 000000000..dac499150 --- /dev/null +++ b/.azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py @@ -0,0 +1,125 @@ +import argparse +import json +import os +import sys +from pathlib import Path +from urllib.error import HTTPError +from urllib.request import Request, urlopen + + +def github_request(method: str, url: str, token: str, payload: dict | None = None): + body = None + if payload is not None: + body = json.dumps(payload).encode("utf-8") + + req = Request( + url, + data=body, + method=method, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "Content-Type": "application/json", + }, + ) + + with urlopen(req) as resp: + data = resp.read().decode("utf-8") + return json.loads(data) if data else {} + + +def load_json(path: Path): + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def load_text(path: Path) -> str: + if not path.exists(): + return "" + with open(path, "r", encoding="utf-8") as f: + return f.read() + + +def parse_repo_path() -> str: + uri = os.environ.get("BUILD_REPOSITORY_URI", "") + if uri.startswith("https://github.com/"): + return uri.replace("https://github.com/", "").removesuffix(".git") + return os.environ.get("REPO_PATH", "") + + +def find_existing_comment(comments: list[dict], marker: str) -> dict | None: + for comment in comments: + body = comment.get("body", "") + if marker in body: + return comment + return None + + +def build_comment_body(analysis: dict, report_text: str, marker: str) -> str: + root_cause = analysis.get("root_cause", "N/A") + confidence = analysis.get("confidence", "unknown") + patch_path = analysis.get("patch_path", "") + external_ok = analysis.get("external_copilot_ok", False) + + patch_hint = "Patch is empty in this run." + if patch_path and Path(patch_path).exists() and Path(patch_path).stat().st_size > 0: + patch_hint = f"Patch generated at `{patch_path}` and published as pipeline artifact." + + lines = [ + marker, + "## Copilot CI Failure Analysis", + "", + f"- Root cause: {root_cause}", + f"- Confidence: {confidence}", + f"- Copilot command executed successfully: {external_ok}", + f"- {patch_hint}", + "", + "### Report", + report_text[:12000] if report_text else "No detailed report generated.", + ] + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description="Post or update PR comment for AI analysis") + parser.add_argument("--analysis-result", required=True, type=Path) + parser.add_argument("--report", required=True, type=Path) + args = parser.parse_args() + + token = os.environ.get("GITHUB_TOKEN", "") + pr_number = os.environ.get("SYSTEM_PULLREQUEST_PULLREQUESTNUMBER", "") + repo_path = parse_repo_path() + + if not token: + print("GITHUB_TOKEN is missing. Skip posting comment.") + return + if not pr_number or not repo_path: + print("PR context missing. Skip posting comment.") + return + + analysis = load_json(args.analysis_result) + report_text = load_text(args.report) + source_commit = analysis.get("source_commit", "unknown") + marker = f"" + + comments_url = f"https://api.github.com/repos/{repo_path}/issues/{pr_number}/comments" + + try: + comments = github_request("GET", comments_url, token) + existing = find_existing_comment(comments if isinstance(comments, list) else [], marker) + body = build_comment_body(analysis, report_text, marker) + + if existing: + update_url = f"https://api.github.com/repos/{repo_path}/issues/comments/{existing['id']}" + github_request("PATCH", update_url, token, payload={"body": body}) + print(f"Updated existing analysis comment #{existing['id']}") + else: + github_request("POST", comments_url, token, payload={"body": body}) + print("Posted new analysis comment") + + except HTTPError as e: + print(f"Failed to post PR comment: {e}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.py index 797a64140..25814ea8d 100644 --- a/.azure-pipelines/scripts/ut/collect_result.py +++ b/.azure-pipelines/scripts/ut/collect_result.py @@ -1,7 +1,10 @@ """Log analyzer for test results with summary generation.""" import argparse +import json +import os import re +import shutil import sys from collections import deque from dataclasses import dataclass @@ -199,12 +202,132 @@ def _format_row(self, result: TestResult) -> str: ) +class FailureContextWriter: + """Extract compact failure context for downstream AI analysis.""" + + TRACEBACK_MARKERS = ("Traceback", "== FAILURES ==", "== ERRORS ==") + + def __init__(self, log_dir: Path, max_lines: int = 200): + self.log_dir = Path(log_dir) + self.max_lines = max_lines + + def write( + self, + output_path: Path, + results: list[TestResult], + test_type: str, + ci_part: str = "", + summary_log: Path | None = None, + failure_log_dir: Path | None = None, + ) -> None: + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + + failed_results = [r for r in results if r.status == TestStatus.FAILED] + failures = [self._to_failure_entry(result) for result in failed_results] + + payload = { + "schema_version": "1.0", + "test_type": test_type, + "ci_part": ci_part, + "build": { + "build_id": os.environ.get("BUILD_BUILDID", ""), + "build_number": os.environ.get("BUILD_BUILDNUMBER", ""), + "source_commit": os.environ.get("SYSTEM_PULLREQUEST_SOURCECOMMITID", os.environ.get("BUILD_SOURCEVERSION", "")), + "pr_number": os.environ.get("SYSTEM_PULLREQUEST_PULLREQUESTNUMBER", ""), + }, + "stats": { + "total": len(results), + "failed": len(failed_results), + }, + "failures": failures, + } + + with open(output_path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + + print(f"Failure context written to: {output_path.absolute()}", file=sys.stderr) + + target_log_dir = Path(failure_log_dir) if failure_log_dir else output_path.parent / "failure_logs" + self._collect_failure_logs(target_log_dir, failed_results, output_path, summary_log) + + def _to_failure_entry(self, result: TestResult) -> dict: + log_path = self.log_dir / result.filename + content = self._read_file(log_path) + lines = content.splitlines() + excerpt = self._extract_excerpt(lines) + tail = "\n".join(lines[-self.max_lines :]) if lines else "" + + return { + "test_name": result.name, + "status": result.status.name, + "log_file": result.filename, + "duration": result.duration, + "excerpt": excerpt, + "tail": tail, + } + + def _read_file(self, path: Path) -> str: + try: + with open(path, "r", encoding="utf-8", errors="ignore") as f: + return f.read() + except OSError: + return "" + + def _extract_excerpt(self, lines: list[str]) -> str: + if not lines: + return "" + + selected = [] + for marker in self.TRACEBACK_MARKERS: + for idx, line in enumerate(lines): + if marker in line: + start = max(0, idx - 10) + end = min(len(lines), idx + 80) + selected = lines[start:end] + break + if selected: + break + + if not selected: + selected = lines[-self.max_lines :] + + return "\n".join(selected[: self.max_lines]) + + def _collect_failure_logs( + self, + target_dir: Path, + failed_results: list[TestResult], + context_path: Path, + summary_log: Path | None, + ) -> None: + if not failed_results: + return + + target_dir.mkdir(parents=True, exist_ok=True) + + if summary_log and Path(summary_log).exists(): + shutil.copy2(summary_log, target_dir / Path(summary_log).name) + + for result in failed_results: + src_log = self.log_dir / result.filename + if src_log.exists(): + shutil.copy2(src_log, target_dir / result.filename) + + if context_path.exists(): + shutil.copy2(context_path, target_dir / context_path.name) + + def main(): parser = argparse.ArgumentParser(description="Analyze test logs and generate summary") parser.add_argument("--test-type", required=True, help="Type of tests") parser.add_argument("--log-dir", required=True, type=Path, help="Directory with logs") parser.add_argument("--summary-log", required=True, type=Path, help="Output file") parser.add_argument("--log-pattern", default="*.log", help="Glob pattern") + parser.add_argument("--failure-context", type=Path, help="Optional output file for failure context JSON") + parser.add_argument("--failure-context-max-lines", type=int, default=200, help="Max lines per failed case") + parser.add_argument("--ci-part", default="", help="Optional CI matrix part label") + parser.add_argument("--failure-log-dir", type=Path, help="Optional output folder for failed logs package") args = parser.parse_args() @@ -224,6 +347,18 @@ def main(): "passed": sum(1 for r in results if r.status == TestStatus.PASSED), "failed": sum(1 for r in results if r.status == TestStatus.FAILED), } + + if args.failure_context: + context_writer = FailureContextWriter(args.log_dir, max_lines=args.failure_context_max_lines) + context_writer.write( + args.failure_context, + results, + test_type=args.test_type, + ci_part=args.ci_part, + summary_log=args.summary_log, + failure_log_dir=args.failure_log_dir, + ) + print(f"Done: {stats['total']} tests, {stats['passed']} passed, {stats['failed']} failed") except Exception as e: diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh index b4dbb261e..b1de56368 100644 --- a/.azure-pipelines/scripts/ut/run_ut.sh +++ b/.azure-pipelines/scripts/ut/run_ut.sh @@ -1,7 +1,44 @@ #!/bin/bash set -e -test_part=$1 +test_part="" +failure_log_context="" +failed_test_cases="" +declare -a FAILED_BASE_CASES=() +declare -a FAILED_INC_CASES=() +declare -a FAILED_LLMC_CASES=() + +function parse_arguments() { + + while [[ $# -gt 0 ]]; do + case "$1" in + --failure-context) + failure_log_context="$2" + shift 2 + ;; + --test-part) + test_part="$2" + shift 2 + ;; + --failed-test-cases) + failed_test_cases="$2" + shift 2 + ;; + *) + echo "Unknown argument: $1" + exit 1 + ;; + esac + done + + if [[ -z "${test_part}" ]]; then + echo "Error: test_part is required" + echo "Usage: run_ut.sh --test-part [--failure-context ] [--failed-test-cases ]" + exit 1 + fi +} + +parse_arguments "$@" source /auto-round/.azure-pipelines/scripts/change_color.sh @@ -9,6 +46,20 @@ LOG_DIR=/auto-round/log_dir mkdir -p "${LOG_DIR}" SUMMARY_LOG="${LOG_DIR}/results_summary.log" +function setup_inc_environment() { + echo "##[group]set up INC UT env..." + INC_PT_ONLY=1 uv pip install -r /auto-round/test/test_cpu/requirements_inc.txt + echo "##[endgroup]" +} + +function setup_llmc_environment() { + echo "##[group]set up LLMC UT env..." + BUILD_TYPE="nightly" uv pip install -r /auto-round/test/test_cpu/requirements_llmc.txt + uv pip uninstall auto-round + cd /auto-round && uv pip install . + echo "##[endgroup]" +} + function setup_environment() { echo "##[group]set up UT env..." export TZ='Asia/Shanghai' @@ -62,12 +113,86 @@ function check_storage_usage() { echo "##[endgroup]" } -function run_unit_test() { - cd /auto-round/test || exit 1 +function run_failed_test_cases() { + if [[ -z "${failed_test_cases}" ]]; then + return + fi + + if [[ ! -f "${failed_test_cases}" ]]; then + echo "Error: failed test cases file not found: ${failed_test_cases}" + exit 1 + fi + + if [[ ! -s "${failed_test_cases}" ]]; then + echo "Failed test cases list is empty, skipping rerun." + return + fi + + FAILED_BASE_CASES=() + FAILED_INC_CASES=() + FAILED_LLMC_CASES=() + + while IFS= read -r test_case; do + if [[ -z "${test_case}" ]]; then + continue + fi + + if [[ "${test_case}" == *test_inc* ]]; then + FAILED_INC_CASES+=("${test_case}") + elif [[ "${test_case}" == *test_llmc* ]]; then + FAILED_LLMC_CASES+=("${test_case}") + else + FAILED_BASE_CASES+=("${test_case}") + fi + done < "${failed_test_cases}" + + if [[ ${#FAILED_BASE_CASES[@]} -gt 0 ]]; then + run_test_cases "${FAILED_BASE_CASES[@]}" + fi + if [[ ${#FAILED_INC_CASES[@]} -gt 0 ]]; then + run_inc_unit_test "${FAILED_INC_CASES[@]}" + fi + if [[ ${#FAILED_LLMC_CASES[@]} -gt 0 ]]; then + run_llmc_unit_test "${FAILED_LLMC_CASES[@]}" + fi +} + +function run_test_cases() { + local tests=("$@") + if [[ ${#tests[@]} -eq 0 ]]; then + return + fi + + cd /auto-round || exit 1 auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])') + for test_case in "${tests[@]}"; do + if [[ -z "${test_case}" ]]; then + continue + fi + + local test_path + test_path="${test_case%%::*}" + local test_basename + test_basename=$(basename "${test_path}" .py) + local ut_log_name=${LOG_DIR}/unittest_${test_basename}.log + + echo "##[group]Running ${test_case}..." + numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \ + python -m pytest --cov="${auto_round_path}" --cov-report term --html=report.html --self-contained-html \ + --cov-report xml:coverage.xml --cov-append \ + -vs --disable-warnings "${test_case}" 2>&1 | tee "${ut_log_name}" + echo "##[endgroup]" + done +} + +function run_unit_test() { + local -a selected_cases=() + + cd /auto-round || exit 1 + # Split test files into 5 parts - find ./test_cpu -name "test*.py" | grep -Ev "test_llmc|test_inc" | sort > all_tests.txt + find ./test/test_cpu -name "test*.py" | grep -Ev "test_llmc|test_inc" | sort > all_tests.txt total_lines=$(wc -l < all_tests.txt) NUM_CHUNKS=5 q=$(( total_lines / NUM_CHUNKS )) @@ -80,78 +205,61 @@ function run_unit_test() { start_line=$(( r * (q + 1) + (test_part - r - 1) * q + 1 )) fi end_line=$(( start_line + chunk_size - 1 )) - selected_files=$(sed -n "${start_line},${end_line}p" all_tests.txt) - - for test_file in ${selected_files}; do - echo "##[group]Running ${test_file}..." - local test_basename=$(basename ${test_file} .py) - local ut_log_name=${LOG_DIR}/unittest_${test_basename}.log - - numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \ - python -m pytest --cov="${auto_round_path}" --cov-report term --html=report.html --self-contained-html \ - --cov-report xml:coverage.xml --cov-append \ - -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name} - echo "##[endgroup]" - done + mapfile -t selected_cases < <(sed -n "${start_line},${end_line}p" all_tests.txt) + run_test_cases "${selected_cases[@]}" } function run_inc_unit_test() { - echo "##[group]set up INC UT env..." - INC_PT_ONLY=1 uv pip install -r /auto-round/test/test_cpu/requirements_inc.txt - echo "##[endgroup]" - - cd /auto-round/test || exit 1 - auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])') - - for test_file in $(find ./test_cpu -name "test_inc*.py" | sort); do - echo "##[group]Running ${test_file}..." - local test_basename=$(basename ${test_file} .py) - local ut_log_name=${LOG_DIR}/unittest_${test_basename}.log + local -a inc_cases=("$@") + if [[ ${#inc_cases[@]} -eq 0 ]]; then + mapfile -t inc_cases < <(find /auto-round/test/test_cpu -name "test_inc*.py" | sort) + fi - numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \ - python -m pytest --cov="${auto_round_path}" --cov-report term --html=report.html --self-contained-html \ - --cov-report xml:coverage.xml --cov-append \ - -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name} - echo "##[endgroup]" - done + setup_inc_environment + run_test_cases "${inc_cases[@]}" } function run_llmc_unit_test() { - echo "##[group]set up LLMC UT env..." - BUILD_TYPE="nightly" uv pip install -r /auto-round/test/test_cpu/requirements_llmc.txt - uv pip uninstall auto-round - cd /auto-round && uv pip install . - echo "##[endgroup]" - - cd /auto-round/test || exit 1 - auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])') - - for test_file in $(find ./test_cpu -name "test_llmc*.py" | sort); do - echo "##[group]Running ${test_file}..." - local test_basename=$(basename ${test_file} .py) - local ut_log_name=${LOG_DIR}/unittest_${test_basename}.log + local -a llmc_cases=("$@") + if [[ ${#llmc_cases[@]} -eq 0 ]]; then + mapfile -t llmc_cases < <(find /auto-round/test/test_cpu -name "test_llmc*.py" | sort) + fi - numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \ - python -m pytest --cov="${auto_round_path}" --cov-report term --html=report.html --self-contained-html \ - --cov-report xml:coverage.xml --cov-append \ - -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name} - echo "##[endgroup]" - done + setup_llmc_environment + run_test_cases "${llmc_cases[@]}" } function collect_log() { - python /auto-round/.azure-pipelines/scripts/ut/collect_result.py \ - --test-type "Unit Tests" --log-pattern "unittest_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG} + collect_cmd=( + python /auto-round/.azure-pipelines/scripts/ut/collect_result.py + --test-type "Unit Tests" + --log-pattern "unittest_test_*.log" + --log-dir "${LOG_DIR}" + --summary-log "${SUMMARY_LOG}" + --ci-part "${test_part}" + ) - cp .coverage "${LOG_DIR}/.coverage.part${test_part}" + if [[ -n "${failure_log_context}" ]]; then + collect_cmd+=(--failure-context "${failure_log_context}") + fi + + "${collect_cmd[@]}" + + if [[ -f .coverage ]]; then + cp .coverage "${LOG_DIR}/.coverage.part${test_part}" + fi } function main() { setup_environment - run_unit_test - if [ "$test_part" -eq 5 ]; then - run_inc_unit_test - run_llmc_unit_test + if [[ -n "${failed_test_cases}" ]]; then + run_failed_test_cases + else + run_unit_test + if [ "$test_part" -eq 5 ]; then + run_inc_unit_test + run_llmc_unit_test + fi fi collect_log check_storage_usage diff --git a/.azure-pipelines/template/ai-analysis-template.yml b/.azure-pipelines/template/ai-analysis-template.yml new file mode 100644 index 000000000..231ddd5d4 --- /dev/null +++ b/.azure-pipelines/template/ai-analysis-template.yml @@ -0,0 +1,107 @@ +parameters: + - name: dockerConfigName + type: string + default: "commonDockerConfig" + - name: utContainerName + type: string + default: "AutoRoundUnitTestAI" + - name: imageName + type: string + default: "auto-round" + - name: imageTag + type: string + default: "py312" + - name: uploadPath + type: string + default: "$(Build.SourcesDirectory)/log_dir" + - name: failureLogArtifactPrefix + type: string + default: "ut-failure-log" + - name: failureLogDownloadPath + type: string + default: "$(Pipeline.Workspace)/failure_logs" + - name: analysisArtifactName + type: string + default: "UT_AI_Analysis" + +steps: + - checkout: self + + - task: DownloadPipelineArtifact@2 + inputs: + patterns: '${{ parameters.failureLogArtifactPrefix }}-*/*' + path: ${{ parameters.failureLogDownloadPath }} + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + displayName: 'Use Python 3.12' + + - script: | + cd ${BUILD_SOURCESDIRECTORY} + python .azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py \ + --input-root "${{ parameters.failureLogDownloadPath }}" \ + --output "${{ parameters.uploadPath }}/merged_failure_context.json" \ + --failed-test-cases "${{ parameters.uploadPath }}/failed_tests_to_rerun.txt" + displayName: "Merge Failure Context" + + - script: | + set -euo pipefail + cd ${BUILD_SOURCESDIRECTORY} + + if ! command -v copilot >/dev/null 2>&1; then + if ! command -v npm >/dev/null 2>&1; then + echo "npm is required to install Copilot CLI" + exit 1 + fi + npm install -g @github/copilot + fi + + copilot --version + + python .azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py \ + --failure-context "${{ parameters.uploadPath }}/merged_failure_context.json" \ + --output-dir "${{ parameters.uploadPath }}/ai_analysis" \ + --failed-test-cases "${{ parameters.uploadPath }}/failed_tests_to_rerun.txt" \ + --project-root "${BUILD_SOURCESDIRECTORY}" \ + --failed-logs-root "${{ parameters.failureLogDownloadPath }}" + + patch_file="${{ parameters.uploadPath }}/ai_analysis/suggested_fix.patch" + if [ -s "${patch_file}" ]; then + git apply --check "${patch_file}" + git apply "${patch_file}" + fi + env: + COPILOT_GITHUB_TOKEN: $(GITHUB_TOKEN) + GITHUB_TOKEN: $(GITHUB_TOKEN) + GH_TOKEN: $(GITHUB_TOKEN) + displayName: "Copilot Analyze And Apply Patch" + + - template: ut-template.yml + parameters: + dockerConfigName: ${{ parameters.dockerConfigName }} + utScriptFileName: "run_ut" + uploadPath: ${{ parameters.uploadPath }} + utArtifact: "ut-ai-rerun" + utTestMode: "0" + utContainerName: ${{ parameters.utContainerName }} + imageName: ${{ parameters.imageName }} + imageTag: ${{ parameters.imageTag }} + failureLogContext: "${{ parameters.uploadPath }}/ai_rerun_failure_context.json" + failedTestCases: "${{ parameters.uploadPath }}/failed_tests_to_rerun.txt" + + - script: | + cd ${BUILD_SOURCESDIRECTORY} + python .azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py \ + --analysis-result "${{ parameters.uploadPath }}/ai_analysis/analysis_result.json" \ + --report "${{ parameters.uploadPath }}/ai_analysis/ai_failure_report.md" + env: + GITHUB_TOKEN: $(GITHUB_TOKEN) + displayName: "Post Copilot Analysis Comment" + + - task: PublishPipelineArtifact@1 + condition: succeededOrFailed() + inputs: + targetPath: ${{ parameters.uploadPath }} + artifact: ${{ parameters.analysisArtifactName }} + publishLocation: "pipeline" diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml index 2b6efbe2a..664bc6508 100644 --- a/.azure-pipelines/template/ut-template.yml +++ b/.azure-pipelines/template/ut-template.yml @@ -32,6 +32,15 @@ parameters: - name: buildARKWheel type: string default: "false" + - name: failureLogContext + type: string + default: "" + - name: failedTestCases + type: string + default: "" + - name: failureLogArtifactPrefix + type: string + default: "ut-failure-log" steps: @@ -100,9 +109,18 @@ steps: - script: | set -eo pipefail + EXTRA_ARGS="" + if [ -n "${{ parameters.failureLogContext }}" ]; then + EXTRA_ARGS="--failure-context ${{ parameters.failureLogContext }}" + fi + + if [ -n "${{ parameters.failedTestCases }}" ]; then + EXTRA_ARGS="${EXTRA_ARGS} --failed-test-cases ${{ parameters.failedTestCases }}" + fi + docker exec -e NUMA_NODE=${NUMA_NODE} -e NUMA_CPUSET=${NUMA_CPUSET} -e ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK} ${{ parameters.utContainerName }} \ bash -c "cd /auto-round/.azure-pipelines/scripts \ - && bash ut/${{ parameters.utScriptFileName }}.sh ${{ parameters.utTestMode }}" + && bash ut/${{ parameters.utScriptFileName }}.sh --test-part ${{ parameters.utTestMode }} ${EXTRA_ARGS}" displayName: "Run UT" - task: PublishPipelineArtifact@1 @@ -112,6 +130,13 @@ steps: artifact: ${{ parameters.utArtifact }}_coverage publishLocation: "pipeline" + - task: PublishPipelineArtifact@1 + condition: failed() + inputs: + targetPath: ${{ parameters.uploadPath }}/failure_logs + artifact: "${{ parameters.failureLogArtifactPrefix }}-${{ parameters.utTestMode }}-$(System.JobAttempt)" + publishLocation: "pipeline" + - task: Bash@3 condition: always() inputs: diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.yml index c404ffc8c..d68205d27 100644 --- a/.azure-pipelines/unit-test.yml +++ b/.azure-pipelines/unit-test.yml @@ -16,9 +16,11 @@ pr: - requirements-cpu.txt - .azure-pipelines/build-auto-round-lib.yml - .azure-pipelines/scripts/ut + - .azure-pipelines/scripts/ai_failure_analysis - .azure-pipelines/unit-test.yml - .azure-pipelines/template/lib-build-template.yml - .azure-pipelines/template/ut-template.yml + - .azure-pipelines/template/ai-analysis-template.yml - .azure-pipelines/template/docker-template.yml exclude: - test/test_hpu @@ -40,6 +42,9 @@ variables: DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir ARTIFACT_NAME: "UT_coverage_report" REPO: $(Build.Repository.Uri) + FAILURE_LOG_ARTIFACT_PREFIX: "ut-failure-log" + FAILURE_LOG_DOWNLOAD_PATH: $(Pipeline.Workspace)/failure_logs + COPILOT_ANALYSIS_ENABLED: "true" stages: - template: template/lib-build-template.yml @@ -77,6 +82,24 @@ stages: utTestMode: $(PART) utContainerName: "AutoRoundUnitTest$(NODE_LABEL)" buildARKWheel: $(BUILD_ARK_WHEEL) + failureLogContext: $(UPLOAD_PATH)/failure_context_part$(PART).json + + - stage: AI_Analysis + displayName: "AI Analyze And Rerun" + dependsOn: [Unit_test] + condition: and(failed('Unit_test'), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['COPILOT_ANALYSIS_ENABLED'], 'true')) + pool: ICX-16C + jobs: + - job: AnalyzeAndRerun + timeoutInMinutes: 60 + steps: + - template: template/ai-analysis-template.yml + parameters: + dockerConfigName: "commonDockerConfig" + utContainerName: "AutoRoundUnitTestAI$(NODE_LABEL)" + imageName: $(IMAGE_NAME) + imageTag: $(IMAGE_TAG) + uploadPath: $(UPLOAD_PATH) - stage: Coverage displayName: "Collect Coverage" From ca39038280cb07535fe27047c8cfa0515d37ee2f Mon Sep 17 00:00:00 2001 From: chensuyue Date: Fri, 29 May 2026 20:44:45 +0800 Subject: [PATCH 02/20] for test only, need to revert before merge Signed-off-by: chensuyue --- .azure-pipelines/scripts/ut/run_ut.sh | 2 +- auto_round/compressors/base.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh index b1de56368..b7cf25314 100644 --- a/.azure-pipelines/scripts/ut/run_ut.sh +++ b/.azure-pipelines/scripts/ut/run_ut.sh @@ -192,7 +192,7 @@ function run_unit_test() { cd /auto-round || exit 1 # Split test files into 5 parts - find ./test/test_cpu -name "test*.py" | grep -Ev "test_llmc|test_inc" | sort > all_tests.txt + find ./test/test_cpu/core -name "test*.py" | grep -Ev "test_llmc|test_inc" | sort > all_tests.txt total_lines=$(wc -l < all_tests.txt) NUM_CHUNKS=5 q=$(( total_lines / NUM_CHUNKS )) diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py index a8a7bc5ab..4c980927f 100644 --- a/auto_round/compressors/base.py +++ b/auto_round/compressors/base.py @@ -668,11 +668,15 @@ def _maybe_log_torch_compile_default_hint(self) -> None: def _apply_torch_compile_constraints(self, enable_torch_compile: bool) -> None: """Apply torch.compile disabling rules for the current compressor state.""" self.enable_torch_compile = enable_torch_compile - cfg = self.quantize_config + + # Optimization: skip guard checks when torch.compile is not requested. + if not self.enable_torch_compile: + return + is_raw_fp8, is_raw_nv_fp, _ = self._get_torch_compile_guard_state() # On HPU, we rely on torch.compile to speed up the model execution. - if self.enable_torch_compile and is_raw_fp8 and not is_hpex_available(): + if self.enable_torch_compile and is_raw_fp8 and is_hpex_available(): self.enable_torch_compile = False logger.warning_once("reset enable_torch_compile to `False` as fp8 is enabled") # TODO: fix https://github.com/intel/auto-round/issues/1109 From ec61ccbe8b2df04f811cf7303f45905e177f2fa6 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Fri, 5 Jun 2026 15:03:54 +0800 Subject: [PATCH 03/20] temp version Signed-off-by: chensuyue --- .../analyze_and_suggest.py | 80 ++++- .../scripts/ai_failure_analysis/classify.py | 210 +++++++++++++ .../evidence_collectors.py | 276 ++++++++++++++++++ .../known_issue_matcher.py | 185 ++++++++++++ .../merge_failure_context.py | 27 +- .../ai_failure_analysis/post_pr_comment.py | 136 +++++++-- .../skills/CodeRegression.md | 35 +++ .../ai_failure_analysis/skills/Environment.md | 29 ++ .azure-pipelines/scripts/ut/run_ut.sh | 61 +--- .../template/ai-analysis-template.yml | 149 +++++----- .azure-pipelines/template/ut-template.yml | 7 - .azure-pipelines/unit-test.yml | 20 +- 12 files changed, 1021 insertions(+), 194 deletions(-) create mode 100644 .azure-pipelines/scripts/ai_failure_analysis/classify.py create mode 100644 .azure-pipelines/scripts/ai_failure_analysis/evidence_collectors.py create mode 100644 .azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py create mode 100644 .azure-pipelines/scripts/ai_failure_analysis/skills/CodeRegression.md create mode 100644 .azure-pipelines/scripts/ai_failure_analysis/skills/Environment.md diff --git a/.azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py b/.azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py index 68a85d3e0..9d8d2170e 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py @@ -235,7 +235,7 @@ def fallback_analysis(payload: dict) -> dict: ], "root_cause": "Likely regression in the changed code path exercised by failed unit tests. Inspect the first traceback for precise failure location.", "confidence": "medium", - "suggestion": "Review traceback, apply minimal fix, and rerun only failed test files.", + "suggestion": "Review traceback, apply a minimal fix, and validate with targeted tests.", "patch": "", "static_checks": [ { @@ -277,14 +277,72 @@ def run_local_static_checks(project_root: Path, pr_patch_path: Path) -> list[dic return checks +def write_skipped_result( + output_dir: Path, + classification: str, + confidence, + reason: str, + project_root: Path, +) -> None: + """Emit a minimal result when deep regression analysis is not applicable. + + Non-regression categories (Known Issue / Environment / Dependency / Flaky / + Other) are handled by classification + comment routing, so the expensive + Copilot patch flow is skipped here. + """ + result_path = output_dir / "analysis_result.json" + report_path = output_dir / "ai_failure_report.md" + patch_path = output_dir / "suggested_fix.patch" + + write_text(patch_path, "") + write_text( + report_path, + "\n".join( + [ + "# Copilot Failure Analysis", + "", + f"- Classification: {classification}", + f"- Confidence: {confidence}", + "- Deep regression analysis: skipped", + f"- Reason: {reason}", + ] + ), + ) + with open(result_path, "w", encoding="utf-8") as f: + json.dump( + { + "success": True, + "skipped": True, + "classification": classification, + "confidence": confidence, + "reason": reason, + "patch_path": str(patch_path), + "report_path": str(report_path), + "project_root": str(project_root), + "source_commit": os.environ.get( + "SYSTEM_PULLREQUEST_SOURCECOMMITID", os.environ.get("BUILD_SOURCEVERSION", "") + ), + "pr_number": os.environ.get("SYSTEM_PULLREQUEST_PULLREQUESTNUMBER", ""), + }, + f, + indent=2, + ) + print(f"analyze_and_suggest: classification={classification}; skipped deep analysis ({reason}).") + + def main(): parser = argparse.ArgumentParser(description="Analyze merged failure context and generate fix artifacts") parser.add_argument("--failure-context", required=True, type=Path) parser.add_argument("--output-dir", required=True, type=Path) - parser.add_argument("--failed-test-cases", required=True, type=Path) parser.add_argument("--project-root", type=Path, default=Path.cwd()) parser.add_argument("--failed-logs-root", type=Path, default=None) parser.add_argument("--base-ref", default="main") + parser.add_argument( + "--classification-result", + type=Path, + default=None, + help="Optional classification_result.json; deep analysis runs only for Code Regression.", + ) args = parser.parse_args() args.output_dir.mkdir(parents=True, exist_ok=True) @@ -292,6 +350,20 @@ def main(): project_root = args.project_root.resolve() token = get_token_from_env() + classification_info = {} + if args.classification_result and args.classification_result.exists(): + classification_info = load_json(args.classification_result) + classification = classification_info.get("classification", "") + if classification and classification != "Code Regression": + write_skipped_result( + args.output_dir, + classification, + classification_info.get("confidence", ""), + "non-regression category handled by comment routing", + project_root, + ) + return + failed_log_paths = collect_failed_log_paths(payload, args.failed_logs_root, project_root) failed_log_paths_file = args.output_dir / "failed_log_paths.txt" write_text(failed_log_paths_file, "\n".join(failed_log_paths) + ("\n" if failed_log_paths else "")) @@ -399,6 +471,9 @@ def main(): result_payload = { "success": cli_ok, + "skipped": False, + "classification": classification_info.get("classification", "Code Regression"), + "classification_confidence": classification_info.get("confidence", ""), "copilot_cli_ok": cli_ok, "copilot_cli_message": cli_message, "external_copilot_ok": False, @@ -416,7 +491,6 @@ def main(): "pr_patch_reference": str(pr_patch_path), "report_path": str(report_path), "patch_path": str(patch_path), - "failed_test_cases": str(args.failed_test_cases), "project_root": str(project_root), "generated_at": datetime.now(timezone.utc).isoformat(), "source_commit": os.environ.get("SYSTEM_PULLREQUEST_SOURCECOMMITID", os.environ.get("BUILD_SOURCEVERSION", "")), diff --git a/.azure-pipelines/scripts/ai_failure_analysis/classify.py b/.azure-pipelines/scripts/ai_failure_analysis/classify.py new file mode 100644 index 000000000..8ef1f67a5 --- /dev/null +++ b/.azure-pipelines/scripts/ai_failure_analysis/classify.py @@ -0,0 +1,210 @@ +"""Classify a CI failure into one of six categories and route handling. + +Pipeline position: runs after ``merge_failure_context.py`` and before any +category-specific handling. It combines deterministic forensic evidence +(``evidence_collectors``) and known-issue matches (``known_issue_matcher``), then +emits Azure DevOps pipeline variables so downstream steps can route handling. + +Classification categories (priority order mirrors the triage flow): + 1. Known Issue 2. Environment 3. Dependency + 4. Flaky Test 5. Code Regression 6. Other +""" + +import argparse +import json +import os +from pathlib import Path + +import evidence_collectors +import known_issue_matcher + +# Canonical category labels. +KNOWN_ISSUE = "Known Issue" +ENVIRONMENT = "Environment" +DEPENDENCY = "Dependency" +FLAKY = "Flaky Test" +CODE_REGRESSION = "Code Regression" +OTHER = "Other" + +VALID_CATEGORIES = {KNOWN_ISSUE, ENVIRONMENT, DEPENDENCY, FLAKY, CODE_REGRESSION, OTHER} + +# Below this confidence the result is forced to ``Other`` and the CI admin is +# pinged, to avoid mis-routing. +CONFIDENCE_THRESHOLD = 0.5 + +# A known-issue match at or above this confidence short-circuits to Known Issue. +KNOWN_ISSUE_AUTO_THRESHOLD = 0.6 + +# Handling action keys consumed by post_pr_comment.py / template gating. +HANDLING_BY_CATEGORY = { + KNOWN_ISSUE: "comment_known_issue", + ENVIRONMENT: "comment_environment", + DEPENDENCY: "comment_dependency", + FLAKY: "comment_flaky", + CODE_REGRESSION: "analyze_regression", + OTHER: "comment_other_notify_admin", +} + + +def load_json(path: Path): + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def write_json(path: Path, payload: dict): + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2) + + +def emit_pipeline_variable(name: str, value: str): + """Emit an Azure DevOps logging command to set a pipeline variable. + + Variable is consumable by later steps in the same job via ``$(NAME)`` / + ``variables['NAME']``. + """ + safe = str(value).replace("\r", " ").replace("\n", " ") + print(f"##vso[task.setvariable variable={name}]{safe}") + + +def heuristic_classification(evidence: dict, known_issues: dict) -> dict: + """Deterministic fallback when Copilot is unavailable. + + Mirrors the priority order so behaviour is predictable without the agent. + """ + matches = known_issues.get("matches", []) + if matches and matches[0].get("confidence", 0.0) >= KNOWN_ISSUE_AUTO_THRESHOLD: + return { + "classification": KNOWN_ISSUE, + "confidence": matches[0]["confidence"], + "evidence": [f"matches known issue #{matches[0].get('number')}"], + "reasoning": "Known-issue matcher returned a high-confidence ticket.", + } + + env = evidence.get("environment", {}) + if env.get("has_environment_signal"): + signals = list(env.get("signals", {}).keys()) + return { + "classification": ENVIRONMENT, + "confidence": 0.7, + "evidence": [f"environment signal: {s}" for s in signals][:5], + "reasoning": "Deterministic environment signals were detected in logs.", + } + + pr = evidence.get("pr_relevance", {}) + if pr.get("relevance_score", 0.0) >= 0.5 and pr.get("touches_source"): + return { + "classification": CODE_REGRESSION, + "confidence": round(min(0.9, 0.5 + pr["relevance_score"] / 2), 3), + "evidence": [ + f"relevance_score={pr.get('relevance_score')}", + f"directly_changed_tests={pr.get('directly_changed_tests', [])[:5]}", + ], + "reasoning": "Failed tests correlate with PR-changed source files.", + } + + return { + "classification": OTHER, + "confidence": 0.3, + "evidence": ["no strong deterministic signal"], + "reasoning": "Insufficient evidence for confident classification.", + } + + +def finalize_classification(raw_result: dict, known_issues: dict, admin_handle: str) -> dict: + """Apply guard rails: known-issue short-circuit and confidence threshold.""" + classification = str(raw_result.get("classification", "")).strip() + if classification not in VALID_CATEGORIES: + classification = OTHER + + try: + confidence = float(raw_result.get("confidence", 0.0)) + except (TypeError, ValueError): + confidence = 0.0 + confidence = max(0.0, min(1.0, confidence)) + + evidence = raw_result.get("evidence", []) or [] + reasoning = raw_result.get("reasoning", "") + + notify_admin = False + matches = known_issues.get("matches", []) + + # Guard 1: strong known-issue match always wins. + if matches and matches[0].get("confidence", 0.0) >= KNOWN_ISSUE_AUTO_THRESHOLD: + classification = KNOWN_ISSUE + confidence = max(confidence, matches[0]["confidence"]) + + # Guard 2: low-confidence results fall back to Other and ping the admin. + if classification != KNOWN_ISSUE and confidence < CONFIDENCE_THRESHOLD: + classification = OTHER + notify_admin = True + reasoning = ( + f"Confidence {confidence} below threshold {CONFIDENCE_THRESHOLD}; " + f"routed to Other for manual review. {reasoning}" + ).strip() + + if classification == OTHER: + notify_admin = True + + handling_action = HANDLING_BY_CATEGORY.get(classification, HANDLING_BY_CATEGORY[OTHER]) + + return { + "classification": classification, + "confidence": round(confidence, 3), + "evidence": evidence[:10], + "reasoning": reasoning, + "handling_action": handling_action, + "notify_admin": notify_admin, + "ci_admin_handle": admin_handle, + "known_issue_matches": matches[:5], + } + + +def main(): + parser = argparse.ArgumentParser(description="Classify CI failure and route handling (deterministic)") + parser.add_argument("--failure-context", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--project-root", type=Path, default=Path.cwd()) + parser.add_argument("--base-ref", default="main") + parser.add_argument("--known-issue-label", default=known_issue_matcher.DEFAULT_LABEL) + parser.add_argument("--admin-handle", default=os.environ.get("CI_ADMIN_HANDLE", "chensuyue")) + args = parser.parse_args() + + project_root = args.project_root.resolve() + payload = load_json(args.failure_context) + failures = payload.get("failures", []) + + evidence = evidence_collectors.collect_all_evidence(failures, project_root, args.base_ref) + + repo_path = known_issue_matcher._repo_path_from_env() + token = os.environ.get("GITHUB_TOKEN", "") + known_issues = known_issue_matcher.match_known_issues( + failures, repo_path, token, label=args.known_issue_label + ) + + raw_result = heuristic_classification(evidence, known_issues) + + final = finalize_classification(raw_result, known_issues, args.admin_handle) + final.update( + { + "used_copilot": False, + "evidence_bundle": evidence, + "source_commit": payload.get("build", {}).get("source_commit", ""), + "pr_number": payload.get("build", {}).get("pr_number", ""), + "failure_count": len(failures), + } + ) + + write_json(args.output, final) + + emit_pipeline_variable("CLASSIFICATION", final["classification"]) + emit_pipeline_variable("HANDLING_ACTION", final["handling_action"]) + + print(f"classify: classification={final['classification']} " + f"confidence={final['confidence']} action={final['handling_action']} " + f"notify_admin={final['notify_admin']}") + print(f"classify: result written to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/.azure-pipelines/scripts/ai_failure_analysis/evidence_collectors.py b/.azure-pipelines/scripts/ai_failure_analysis/evidence_collectors.py new file mode 100644 index 000000000..a97bca0ad --- /dev/null +++ b/.azure-pipelines/scripts/ai_failure_analysis/evidence_collectors.py @@ -0,0 +1,276 @@ +"""Deterministic forensic evidence collectors for CI failure classification. + +These collectors turn raw failure logs into structured signals so the +classifier (and the Copilot agent) make a *multiple-choice* decision instead of +re-reading megabytes of logs. The quality of these signals is what actually +improves classification hit-rate; the category labels are only a routing layer. + +Each collector returns a JSON-serializable dict. ``collect_all_evidence`` +aggregates them into a single evidence bundle consumed by ``classify.py``. +""" + +import re +import subprocess +from pathlib import Path + +# --------------------------------------------------------------------------- +# Environment signal patterns +# --------------------------------------------------------------------------- +# Each entry: (signal_key, human_label, compiled_regex). Patterns are matched +# case-insensitively against the failure excerpt + tail text. +ENVIRONMENT_PATTERNS: list[tuple[str, str, re.Pattern]] = [ + ( + "network_timeout", + "Network timeout / connection failure", + re.compile( + r"(connection\s+timed?\s*out|read\s+timed?\s*out|connection\s+reset|" + r"failed\s+to\s+establish\s+a\s+new\s+connection|max\s+retries\s+exceeded|" + r"temporary\s+failure\s+in\s+name\s+resolution|connection\s+aborted|" + r"ETIMEDOUT|ECONNRESET|ECONNREFUSED)", + re.IGNORECASE, + ), + ), + ( + "disk_full", + "Disk space exhausted", + re.compile( + r"(no\s+space\s+left\s+on\s+device|disk\s+quota\s+exceeded|ENOSPC|" + r"cannot\s+write\s+.*:\s+no\s+space)", + re.IGNORECASE, + ), + ), + ( + "runner_lost", + "Runner / agent lost or disconnected", + re.compile( + r"(the\s+agent\s+(?:lost\s+communication|did\s+not\s+respond)|" + r"lost\s+communication\s+with\s+the\s+agent|runner\s+has\s+been\s+lost|" + r"we\s+stopped\s+hearing\s+from\s+agent|received\s+request\s+to\s+deprovision)", + re.IGNORECASE, + ), + ), + ( + "out_of_memory", + "Out of memory / OOM killer", + re.compile( + r"(out\s+of\s+memory|cannot\s+allocate\s+memory|oom[\s-]?kill|" + r"killed\s+process|DefaultCPUAllocator:\s+can't\s+allocate|" + r"std::bad_alloc|MemoryError)", + re.IGNORECASE, + ), + ), + ( + "hf_download_error", + "HuggingFace / model hub download failure", + re.compile( + r"(HfHubHTTPError|huggingface_hub\.utils.*Error|" + r"couldn'?t\s+connect\s+to\s+['\"]?https?://huggingface\.co|" + r"504\s+server\s+error.*huggingface|repository\s+not\s+found.*huggingface|" + r"we\s+couldn'?t\s+connect\s+to\s+['\"]?https?://huggingface\.co)", + re.IGNORECASE, + ), + ), + ( + "rate_limited", + "API rate limit / throttling", + re.compile( + r"(rate\s+limit\s+exceeded|too\s+many\s+requests|HTTP\s+429|" + r"429\s+client\s+error)", + re.IGNORECASE, + ), + ), +] + +# Markers used to derive the candidate test file path from a pytest nodeid. +_TEST_NAME_FILE_RE = re.compile(r"^(?Ptest_[\w/.-]+?\.py)(?:::|$)") + + +def _failure_text(entry: dict) -> str: + """Concatenate the searchable text for a single failure entry.""" + return "\n".join( + part for part in (entry.get("excerpt", ""), entry.get("tail", "")) if part + ) + + +def _snippet_around(text: str, match: re.Match, radius: int = 120) -> str: + start = max(0, match.start() - radius) + end = min(len(text), match.end() + radius) + snippet = text[start:end].strip().replace("\r", "") + return re.sub(r"\s+", " ", snippet) + + +def collect_environment_signals(failures: list[dict]) -> dict: + """Scan failures for environment-related signals. + + Returns a dict with matched signal keys, supporting evidence snippets, and + the set of tests in which each signal appeared. + """ + matched: dict[str, dict] = {} + + for entry in failures: + text = _failure_text(entry) + if not text: + continue + test_name = entry.get("test_name", "") + for signal_key, label, pattern in ENVIRONMENT_PATTERNS: + found = pattern.search(text) + if not found: + continue + record = matched.setdefault( + signal_key, + {"label": label, "evidence": [], "tests": []}, + ) + snippet = _snippet_around(text, found) + if snippet and snippet not in record["evidence"]: + record["evidence"].append(snippet) + if test_name and test_name not in record["tests"]: + record["tests"].append(test_name) + + return { + "has_environment_signal": bool(matched), + "signals": matched, + } + + +def _run_git(args: list[str], cwd: Path) -> tuple[int, str]: + result = subprocess.run( + ["git", *args], cwd=cwd, check=False, capture_output=True, text=True + ) + return result.returncode, (result.stdout or "").strip() + + +def get_pr_changed_files(project_root: Path, base_ref: str = "main") -> list[str]: + """Return the list of files changed by the PR relative to base_ref.""" + diff_specs = [ + f"origin/{base_ref}...HEAD", + f"{base_ref}...HEAD", + "HEAD~1..HEAD", + ] + for spec in diff_specs: + code, out = _run_git(["diff", "--name-only", spec], cwd=project_root) + if code == 0: + files = [line.strip() for line in out.splitlines() if line.strip()] + if files: + return sorted(set(files)) + return [] + + +def _candidate_test_file(test_name: str) -> str: + """Best-effort mapping from a pytest test name/nodeid to a test file path.""" + if not test_name: + return "" + match = _TEST_NAME_FILE_RE.match(test_name) + if match: + return match.group("file") + # Bare function/class name like ``test_quantize`` -> ``test_quantize.py``. + base = test_name.split("::", 1)[0] + if base.startswith("test_") and not base.endswith(".py"): + return f"{base}.py" + return base + + +def collect_pr_relevance( + failures: list[dict], project_root: Path, base_ref: str = "main" +) -> dict: + """Correlate failed tests with PR-changed files. + + This is the strongest signal separating Code Regression from other classes: + if the PR touched ``foo.py`` and ``test_foo.py`` fails, regression is likely. + """ + changed_files = get_pr_changed_files(project_root, base_ref) + changed_set = set(changed_files) + changed_stems = {Path(f).stem for f in changed_files if f.endswith(".py")} + + # Tests whose own test file was modified by the PR. + directly_changed_tests: list[str] = [] + # Tests whose corresponding source module (by stem) was modified. + source_related_tests: list[dict] = [] + + for entry in failures: + test_name = entry.get("test_name", "") + test_file = _candidate_test_file(test_name) + test_stem = Path(test_file).stem if test_file else "" + + # Direct: the failing test file itself is in the PR diff. + for changed in changed_set: + if test_file and changed.endswith(test_file): + if test_name not in directly_changed_tests: + directly_changed_tests.append(test_name) + break + + # Indirect: a changed source module matches the test's subject. + # ``test_quantize`` -> subject stem ``quantize``. + subject_stem = test_stem[len("test_"):] if test_stem.startswith("test_") else test_stem + related_modules = sorted( + f for f in changed_files + if f.endswith(".py") and subject_stem and Path(f).stem == subject_stem + ) + if related_modules: + source_related_tests.append( + {"test": test_name, "modules": related_modules} + ) + + relevance_score = 0.0 + if failures: + related_count = len(directly_changed_tests) + len(source_related_tests) + relevance_score = round(min(1.0, related_count / max(1, len(failures))), 3) + + # Heuristic: PR only touches code under auto_round/ vs only tests/ vs docs. + touches_source = any( + f.startswith("auto_round/") or f.startswith("auto_round_extension/") + for f in changed_files + ) + touches_tests = any(f.startswith("test/") for f in changed_files) + + return { + "changed_file_count": len(changed_files), + "changed_files": changed_files[:100], + "changed_source_stems": sorted(changed_stems)[:100], + "directly_changed_tests": directly_changed_tests, + "source_related_tests": source_related_tests, + "relevance_score": relevance_score, + "touches_source": touches_source, + "touches_tests": touches_tests, + } + + +def collect_dependency_changes(project_root: Path) -> dict: + """Phase 2 stub: compare current dependencies against a success baseline. + + The baseline (``pip freeze`` from the last successful build) will be + published as a pipeline artifact and diffed here. Until that artifact + exists this returns ``available=False`` so the classifier can skip the + Dependency branch cleanly. + """ + return { + "available": False, + "reason": "dependency baseline artifact not wired yet (Phase 2)", + "changed_packages": [], + } + + +def collect_flaky_signals(failures: list[dict]) -> dict: + """Phase 2 stub: detect flaky tests via history / rerun comparison. + + A full implementation will consult historical pass/fail rates and the + Environment-rerun outcome. For now it returns a neutral, low-confidence + result so the classifier does not over-trigger the Flaky branch. + """ + return { + "available": False, + "reason": "flaky history source not wired yet (Phase 2)", + "suspected_flaky_tests": [], + } + + +def collect_all_evidence( + failures: list[dict], project_root: Path, base_ref: str = "main" +) -> dict: + """Aggregate every collector into a single evidence bundle.""" + return { + "failure_count": len(failures), + "environment": collect_environment_signals(failures), + "pr_relevance": collect_pr_relevance(failures, project_root, base_ref), + "dependency": collect_dependency_changes(project_root), + "flaky": collect_flaky_signals(failures), + } diff --git a/.azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py b/.azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py new file mode 100644 index 000000000..9bfa294c6 --- /dev/null +++ b/.azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py @@ -0,0 +1,185 @@ +"""Match CI failures against known-issue tracker entries. + +Pulls open issues labelled ``CI_known_issue`` from the repository and scores +them against the failure signals. A high score means the failure is already +tracked, so the pipeline can comment "known issue, safe to skip" instead of +re-analysing it as a regression. +""" + +import json +import os +import re +from pathlib import Path +from urllib.parse import quote +from urllib.request import Request, urlopen + +DEFAULT_LABEL = "CI_known_issue" +# Tokens shorter than this or in the stop list add noise to the overlap score. +_MIN_TOKEN_LEN = 4 +_STOP_TOKENS = { + "test", "tests", "error", "errors", "failed", "failure", "assert", + "self", "none", "true", "false", "value", "object", "python", "trace", + "traceback", "line", "file", "call", "last", "most", "recent", +} +_TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]{2,}") +# Identifiers that are strong fingerprints: exception classes, error codes. +_EXCEPTION_RE = re.compile(r"\b([A-Z][A-Za-z0-9]+(?:Error|Exception|Warning))\b") + + +def github_get(url: str, token: str): + req = Request( + url, + method="GET", + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + }, + ) + with urlopen(req) as resp: + data = resp.read().decode("utf-8") + return json.loads(data) if data else [] + + +def _tokenize(text: str) -> set[str]: + tokens = { + tok.lower() + for tok in _TOKEN_RE.findall(text or "") + if len(tok) >= _MIN_TOKEN_LEN + } + return tokens - _STOP_TOKENS + + +def _exceptions(text: str) -> set[str]: + return {m.lower() for m in _EXCEPTION_RE.findall(text or "")} + + +def fetch_known_issues(repo_path: str, token: str, label: str = DEFAULT_LABEL) -> list[dict]: + """Fetch open issues carrying the known-issue label.""" + if not token or not repo_path: + return [] + url = ( + f"https://api.github.com/repos/{repo_path}/issues" + f"?state=open&labels={quote(label)}&per_page=100" + ) + try: + issues = github_get(url, token) + except Exception as exc: # noqa: BLE001 - network best-effort + print(f"known_issue_matcher: failed to fetch issues: {exc}") + return [] + + result = [] + for issue in issues if isinstance(issues, list) else []: + # Pull requests are also returned by the issues endpoint; skip them. + if "pull_request" in issue: + continue + result.append( + { + "number": issue.get("number"), + "title": issue.get("title", ""), + "body": issue.get("body", "") or "", + "url": issue.get("html_url", ""), + } + ) + return result + + +def _failure_text(failures: list[dict]) -> str: + parts = [] + for entry in failures: + parts.append(entry.get("test_name", "")) + parts.append(entry.get("excerpt", "")) + parts.append(entry.get("tail", "")) + return "\n".join(p for p in parts if p) + + +def score_issue(issue: dict, failure_tokens: set[str], failure_excs: set[str]) -> dict: + """Score one issue against the aggregated failure fingerprint.""" + issue_text = f"{issue['title']}\n{issue['body']}" + issue_tokens = _tokenize(issue_text) + issue_excs = _exceptions(issue_text) + + shared_tokens = failure_tokens & issue_tokens + shared_excs = failure_excs & issue_excs + + token_score = len(shared_tokens) / max(1, len(issue_tokens)) if issue_tokens else 0.0 + # Exception-class overlap is a much stronger signal than generic tokens. + exc_score = 1.0 if shared_excs else 0.0 + + confidence = round(min(1.0, 0.6 * token_score + 0.4 * exc_score), 3) + return { + "number": issue["number"], + "title": issue["title"], + "url": issue["url"], + "confidence": confidence, + "shared_exceptions": sorted(shared_excs), + "shared_keywords": sorted(shared_tokens)[:15], + } + + +def match_known_issues( + failures: list[dict], + repo_path: str, + token: str, + label: str = DEFAULT_LABEL, + min_confidence: float = 0.3, +) -> dict: + """Return ranked known-issue matches for the given failures.""" + issues = fetch_known_issues(repo_path, token, label) + if not issues: + return {"label": label, "checked": 0, "matches": []} + + failure_text = _failure_text(failures) + failure_tokens = _tokenize(failure_text) + failure_excs = _exceptions(failure_text) + + scored = [score_issue(issue, failure_tokens, failure_excs) for issue in issues] + matches = sorted( + (s for s in scored if s["confidence"] >= min_confidence), + key=lambda s: s["confidence"], + reverse=True, + ) + return {"label": label, "checked": len(issues), "matches": matches} + + +def _repo_path_from_env() -> str: + uri = os.environ.get("BUILD_REPOSITORY_URI", "") + if uri.startswith("https://github.com/"): + return uri.replace("https://github.com/", "").removesuffix(".git") + return os.environ.get("REPO_PATH", "") + + +def main(): + import argparse + + parser = argparse.ArgumentParser(description="Match failures against known issues") + parser.add_argument("--failure-context", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--label", default=DEFAULT_LABEL) + parser.add_argument("--min-confidence", type=float, default=0.3) + args = parser.parse_args() + + with open(args.failure_context, "r", encoding="utf-8") as f: + payload = json.load(f) + failures = payload.get("failures", []) + + token = ( + os.environ.get("GITHUB_TOKEN", "") + or os.environ.get("GH_TOKEN", "") + or os.environ.get("COPILOT_GITHUB_TOKEN", "") + ) + repo_path = _repo_path_from_env() + + result = match_known_issues( + failures, repo_path, token, label=args.label, min_confidence=args.min_confidence + ) + + args.output.parent.mkdir(parents=True, exist_ok=True) + with open(args.output, "w", encoding="utf-8") as f: + json.dump(result, f, indent=2) + + print(f"known_issue_matcher: checked {result['checked']} issues, " + f"{len(result['matches'])} matches >= {args.min_confidence}") + + +if __name__ == "__main__": + main() diff --git a/.azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py b/.azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py index 69ecc76da..bed4637c9 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py @@ -59,7 +59,7 @@ def select_latest_context_files(context_files: list[Path], input_root: Path) -> return sorted(selected) -def merge_contexts(input_root: Path) -> tuple[list[dict], list[str]]: +def merge_contexts(input_root: Path) -> list[dict]: all_context_files = sorted(input_root.rglob("failure_context*.json")) context_files = select_latest_context_files(all_context_files, input_root) merged_failures: list[dict] = [] @@ -76,30 +76,18 @@ def merge_contexts(input_root: Path) -> tuple[list[dict], list[str]]: seen.add(key) merged_failures.append(normalized) - rerun_targets: list[str] = [] - for item in merged_failures: - test_name = item.get("test_name", "") - if not test_name: - continue - if not test_name.startswith("test_"): - continue - rerun_targets.append(f"test/test_cpu/{test_name}.py") - - rerun_targets = sorted(set(rerun_targets)) - return merged_failures, rerun_targets + return merged_failures def main(): parser = argparse.ArgumentParser(description="Merge failure context files from all test parts") parser.add_argument("--input-root", required=True, type=Path, help="Root folder containing downloaded failure artifacts") parser.add_argument("--output", required=True, type=Path, help="Merged failure context JSON path") - parser.add_argument("--failed-test-cases", required=True, type=Path, help="Output failed test cases list") args = parser.parse_args() args.output.parent.mkdir(parents=True, exist_ok=True) - args.failed_test_cases.parent.mkdir(parents=True, exist_ok=True) - failures, rerun_targets = merge_contexts(args.input_root) + failures = merge_contexts(args.input_root) payload = { "schema_version": "1.0", @@ -110,24 +98,15 @@ def main(): }, "stats": { "failed_cases": len(failures), - "rerun_targets": len(rerun_targets), }, "failures": failures, - "rerun": { - "test_files": rerun_targets, - }, } with open(args.output, "w", encoding="utf-8") as f: json.dump(payload, f, indent=2) - with open(args.failed_test_cases, "w", encoding="utf-8") as f: - if rerun_targets: - f.write("\n".join(rerun_targets) + "\n") - print(f"Merged {len(failures)} failures from {args.input_root}") print(f"Merged context: {args.output}") - print(f"Rerun targets: {args.failed_test_cases}") if __name__ == "__main__": diff --git a/.azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py b/.azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py index dac499150..331623a45 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py @@ -55,35 +55,128 @@ def find_existing_comment(comments: list[dict], marker: str) -> dict | None: return None -def build_comment_body(analysis: dict, report_text: str, marker: str) -> str: - root_cause = analysis.get("root_cause", "N/A") - confidence = analysis.get("confidence", "unknown") - patch_path = analysis.get("patch_path", "") - external_ok = analysis.get("external_copilot_ok", False) +def _admin_mention(classification: dict) -> str: + handle = (classification.get("ci_admin_handle") or "").strip().lstrip("@") + if classification.get("notify_admin") and handle: + return f"@{handle} please take a look." + return "" + + +def _known_issue_lines(classification: dict) -> list[str]: + matches = classification.get("known_issue_matches", []) or [] + if not matches: + return ["No matching known issue was found."] + lines = ["Matched known issue(s):"] + for match in matches[:5]: + number = match.get("number", "?") + title = match.get("title", "") + url = match.get("url", "") + conf = match.get("confidence", 0) + lines.append(f"- [#{number}]({url}) {title} (confidence {conf})") + return lines + + +def _evidence_lines(classification: dict) -> list[str]: + evidence = classification.get("evidence", []) or [] + if not evidence: + return [] + lines = ["", "### Evidence"] + lines.extend(f"- {item}" for item in evidence[:8]) + return lines + + +def build_category_section(classification: dict, analysis: dict) -> list[str]: + """Build the category-specific body section of the comment.""" + category = classification.get("classification", "Other") + confidence = classification.get("confidence", "unknown") + reasoning = classification.get("reasoning", "") + admin = _admin_mention(classification) + + header = [ + f"- Classification: **{category}**", + f"- Confidence: {confidence}", + ] + if reasoning: + header.append(f"- Reasoning: {reasoning}") + + body: list[str] = [] + if category == "Known Issue": + body.append("This failure matches a tracked known issue and can likely be skipped.") + body.extend(_known_issue_lines(classification)) + if admin: + body.append(f"A CI admin can help merge this PR manually. {admin}") + elif category == "Environment": + body.append( + "This looks like an environment/infrastructure issue and is likely " + "not caused by the PR code changes." + ) + elif category == "Dependency": + body.append( + "This may be caused by a dependency version change. " + "Dependency baseline diffing is not fully enabled yet (Phase 2); " + "please review recent dependency updates." + ) + if admin: + body.append(admin) + elif category == "Flaky Test": + body.append( + "This test appears unstable (flaky). Consider checking the test for " + "non-determinism or ordering assumptions." + ) + if admin: + body.append(admin) + elif category == "Code Regression": + root_cause = analysis.get("root_cause", "N/A") + suggestion = analysis.get("suggestion", "") + patch_path = analysis.get("patch_path", "") + body.append("This failure is likely caused by the PR code changes.") + body.append(f"- Root cause: {root_cause}") + if suggestion: + body.append(f"- Suggested fix: {suggestion}") + if patch_path and Path(patch_path).exists() and Path(patch_path).stat().st_size > 0: + body.append( + f"- A suggested patch was generated (`{patch_path}`) and published " + "as a pipeline artifact. Review before applying; it is NOT auto-applied." + ) + else: + body.append("- No patch was generated in this run.") + else: # Other + body.append( + "This failure could not be confidently classified and needs manual review." + ) + if admin: + body.append(admin) - patch_hint = "Patch is empty in this run." - if patch_path and Path(patch_path).exists() and Path(patch_path).stat().st_size > 0: - patch_hint = f"Patch generated at `{patch_path}` and published as pipeline artifact." + return header + [""] + body + +def build_comment_body(classification: dict, analysis: dict, report_text: str, marker: str) -> str: lines = [ marker, "## Copilot CI Failure Analysis", "", - f"- Root cause: {root_cause}", - f"- Confidence: {confidence}", - f"- Copilot command executed successfully: {external_ok}", - f"- {patch_hint}", - "", - "### Report", - report_text[:12000] if report_text else "No detailed report generated.", ] + lines.extend(build_category_section(classification, analysis)) + lines.extend(_evidence_lines(classification)) + if report_text: + lines.extend( + [ + "", + "
Detailed report", + "", + report_text[:12000], + "", + "
", + ] + ) return "\n".join(lines) def main(): parser = argparse.ArgumentParser(description="Post or update PR comment for AI analysis") - parser.add_argument("--analysis-result", required=True, type=Path) - parser.add_argument("--report", required=True, type=Path) + parser.add_argument("--classification-result", required=True, type=Path) + parser.add_argument("--analysis-result", type=Path, default=None) + parser.add_argument("--report", type=Path, default=None) args = parser.parse_args() token = os.environ.get("GITHUB_TOKEN", "") @@ -97,9 +190,10 @@ def main(): print("PR context missing. Skip posting comment.") return - analysis = load_json(args.analysis_result) - report_text = load_text(args.report) - source_commit = analysis.get("source_commit", "unknown") + classification = load_json(args.classification_result) + analysis = load_json(args.analysis_result) if args.analysis_result and args.analysis_result.exists() else {} + report_text = load_text(args.report) if args.report else "" + source_commit = classification.get("source_commit") or analysis.get("source_commit", "unknown") marker = f"" comments_url = f"https://api.github.com/repos/{repo_path}/issues/{pr_number}/comments" @@ -107,7 +201,7 @@ def main(): try: comments = github_request("GET", comments_url, token) existing = find_existing_comment(comments if isinstance(comments, list) else [], marker) - body = build_comment_body(analysis, report_text, marker) + body = build_comment_body(classification, analysis, report_text, marker) if existing: update_url = f"https://api.github.com/repos/{repo_path}/issues/comments/{existing['id']}" diff --git a/.azure-pipelines/scripts/ai_failure_analysis/skills/CodeRegression.md b/.azure-pipelines/scripts/ai_failure_analysis/skills/CodeRegression.md new file mode 100644 index 000000000..91fd104d6 --- /dev/null +++ b/.azure-pipelines/scripts/ai_failure_analysis/skills/CodeRegression.md @@ -0,0 +1,35 @@ +# Code Regression Checklist + +Use this to decide whether a CI failure is a **Code Regression** caused by the PR's +own changes. These failures must NOT be rerun blindly; they need a root-cause analysis +and a suggested fix. + +## Strong signals +- The failing test file is in `directly_changed_tests` (the PR edited that test), or + the test imports a source module whose stem is in `changed_source_stems` + (see `pr_relevance` in the evidence bundle). +- A deterministic Python traceback terminating inside `auto_round/` or + `auto_round_extension/` source touched by the PR. +- Assertion failures tied to logic the PR changed: `AssertionError`, allclose/shape + mismatches, changed default values, new/removed function arguments + (`TypeError: ... unexpected keyword argument`, `missing N required positional`). +- Import errors for symbols the PR renamed/moved (`ImportError`, `AttributeError: + module ... has no attribute`). + +## Counter-signals (probably NOT a regression) +- Strong environment signals (network/disk/OOM/runner) with no in-code assertion. +- The failure reproduces on `main` / is already a tracked known issue. +- The failing area is completely unrelated to the PR's changed files and the error is + intermittent (consider Flaky Test). + +## Root-cause guidance for the analysis step +1. Map each failing test to its source-of-truth module using `pr_relevance`. +2. Read the changed hunks for that module; look for changed signatures, defaults, + control flow, or removed guards. +3. Confirm the traceback line corresponds to changed code. +4. Propose the minimal fix; do NOT auto-apply — emit it as a reviewable patch only. + +## Decision rule +Classify as Code Regression when PR-relevance is high (failing test ∩ PR-changed +source/tests) AND the error is a deterministic in-code failure. Confidence should +scale with how directly the changed code appears in the traceback. diff --git a/.azure-pipelines/scripts/ai_failure_analysis/skills/Environment.md b/.azure-pipelines/scripts/ai_failure_analysis/skills/Environment.md new file mode 100644 index 000000000..d2168e834 --- /dev/null +++ b/.azure-pipelines/scripts/ai_failure_analysis/skills/Environment.md @@ -0,0 +1,29 @@ +# Environment Failure Checklist + +Use this to decide whether a CI failure is an **Environment** issue (infra/network/ +runner problem) rather than a code defect. Environment failures usually pass on a +clean rerun, so they should route to the automatic rerun path. + +## Strong signals (any one is usually decisive) +- Network errors: `Connection reset`, `Connection timed out`, `Temporary failure in + name resolution`, `Read timed out`, `Max retries exceeded`, TLS/SSL handshake errors. +- Package/index outages: failures contacting `pypi.org`, `download.pytorch.org`, + `huggingface.co`, `test.pypi.org`; HTTP 429/500/502/503/504 from these hosts. +- Model/dataset download errors: `HfHubHTTPError`, `RepositoryNotFoundError` caused by + rate limiting, `OSError: ... Connection`, incomplete/`.incomplete` cache files. +- Resource exhaustion: `No space left on device`, `OSError: [Errno 28]`, + `CUDA out of memory`, `Cannot allocate memory`, `Killed` (OOM killer). +- Runner/agent problems: `The agent lost communication`, `docker: Error response from + daemon`, container failed to start, disk pressure evictions. +- Rate limiting: `API rate limit exceeded`, HTTP 403 with `rate limit` text. + +## Counter-signals (NOT environment) +- A clean Python traceback ending in `AssertionError`, `ValueError`, `TypeError`, + `KeyError`, etc. inside auto-round source or a test assertion. +- The failing test file or its imported source module was modified by this PR. +- Numerical/accuracy mismatches (e.g. allclose failures) without any infra error text. + +## Decision rule +Classify as Environment only when a strong infra signal is present AND there is no +clear in-code assertion/exception attributable to PR changes. If both are present, +prefer Code Regression (a real bug should not be hidden by a rerun). diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh index b7cf25314..b44e9a254 100644 --- a/.azure-pipelines/scripts/ut/run_ut.sh +++ b/.azure-pipelines/scripts/ut/run_ut.sh @@ -3,7 +3,6 @@ set -e test_part="" failure_log_context="" -failed_test_cases="" declare -a FAILED_BASE_CASES=() declare -a FAILED_INC_CASES=() declare -a FAILED_LLMC_CASES=() @@ -20,10 +19,6 @@ function parse_arguments() { test_part="$2" shift 2 ;; - --failed-test-cases) - failed_test_cases="$2" - shift 2 - ;; *) echo "Unknown argument: $1" exit 1 @@ -113,50 +108,6 @@ function check_storage_usage() { echo "##[endgroup]" } -function run_failed_test_cases() { - if [[ -z "${failed_test_cases}" ]]; then - return - fi - - if [[ ! -f "${failed_test_cases}" ]]; then - echo "Error: failed test cases file not found: ${failed_test_cases}" - exit 1 - fi - - if [[ ! -s "${failed_test_cases}" ]]; then - echo "Failed test cases list is empty, skipping rerun." - return - fi - - FAILED_BASE_CASES=() - FAILED_INC_CASES=() - FAILED_LLMC_CASES=() - - while IFS= read -r test_case; do - if [[ -z "${test_case}" ]]; then - continue - fi - - if [[ "${test_case}" == *test_inc* ]]; then - FAILED_INC_CASES+=("${test_case}") - elif [[ "${test_case}" == *test_llmc* ]]; then - FAILED_LLMC_CASES+=("${test_case}") - else - FAILED_BASE_CASES+=("${test_case}") - fi - done < "${failed_test_cases}" - - if [[ ${#FAILED_BASE_CASES[@]} -gt 0 ]]; then - run_test_cases "${FAILED_BASE_CASES[@]}" - fi - if [[ ${#FAILED_INC_CASES[@]} -gt 0 ]]; then - run_inc_unit_test "${FAILED_INC_CASES[@]}" - fi - if [[ ${#FAILED_LLMC_CASES[@]} -gt 0 ]]; then - run_llmc_unit_test "${FAILED_LLMC_CASES[@]}" - fi -} - function run_test_cases() { local tests=("$@") if [[ ${#tests[@]} -eq 0 ]]; then @@ -252,14 +203,10 @@ function collect_log() { function main() { setup_environment - if [[ -n "${failed_test_cases}" ]]; then - run_failed_test_cases - else - run_unit_test - if [ "$test_part" -eq 5 ]; then - run_inc_unit_test - run_llmc_unit_test - fi + run_unit_test + if [ "$test_part" -eq 5 ]; then + run_inc_unit_test + run_llmc_unit_test fi collect_log check_storage_usage diff --git a/.azure-pipelines/template/ai-analysis-template.yml b/.azure-pipelines/template/ai-analysis-template.yml index 231ddd5d4..43a927185 100644 --- a/.azure-pipelines/template/ai-analysis-template.yml +++ b/.azure-pipelines/template/ai-analysis-template.yml @@ -23,85 +23,92 @@ parameters: - name: analysisArtifactName type: string default: "UT_AI_Analysis" + - name: classifyTimeoutInMinutes + type: number + default: 60 -steps: - - checkout: self +jobs: + - job: Classify + displayName: "Classify And Comment" + timeoutInMinutes: ${{ parameters.classifyTimeoutInMinutes }} + steps: + - checkout: self - - task: DownloadPipelineArtifact@2 - inputs: - patterns: '${{ parameters.failureLogArtifactPrefix }}-*/*' - path: ${{ parameters.failureLogDownloadPath }} + - task: DownloadPipelineArtifact@2 + inputs: + patterns: '${{ parameters.failureLogArtifactPrefix }}-*/*' + path: ${{ parameters.failureLogDownloadPath }} - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.12' - displayName: 'Use Python 3.12' + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + displayName: 'Use Python 3.12' - - script: | - cd ${BUILD_SOURCESDIRECTORY} - python .azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py \ - --input-root "${{ parameters.failureLogDownloadPath }}" \ - --output "${{ parameters.uploadPath }}/merged_failure_context.json" \ - --failed-test-cases "${{ parameters.uploadPath }}/failed_tests_to_rerun.txt" - displayName: "Merge Failure Context" + - script: | + cd ${BUILD_SOURCESDIRECTORY} + python .azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py \ + --input-root "${{ parameters.failureLogDownloadPath }}" \ + --output "${{ parameters.uploadPath }}/merged_failure_context.json" + displayName: "Merge Failure Context" - - script: | - set -euo pipefail - cd ${BUILD_SOURCESDIRECTORY} + - script: | + set -euo pipefail + cd ${BUILD_SOURCESDIRECTORY} - if ! command -v copilot >/dev/null 2>&1; then - if ! command -v npm >/dev/null 2>&1; then - echo "npm is required to install Copilot CLI" - exit 1 - fi - npm install -g @github/copilot - fi + python .azure-pipelines/scripts/ai_failure_analysis/classify.py \ + --failure-context "${{ parameters.uploadPath }}/merged_failure_context.json" \ + --output "${{ parameters.uploadPath }}/ai_analysis/classification_result.json" \ + --project-root "${BUILD_SOURCESDIRECTORY}" \ + --base-ref "main" \ + --admin-handle "$(CI_ADMIN_HANDLE)" + name: classify_step + env: + COPILOT_GITHUB_TOKEN: $(GITHUB_TOKEN) + GITHUB_TOKEN: $(GITHUB_TOKEN) + GH_TOKEN: $(GITHUB_TOKEN) + CI_ADMIN_HANDLE: $(CI_ADMIN_HANDLE) + displayName: "Classify Failure" - copilot --version + - script: | + set -euo pipefail + cd ${BUILD_SOURCESDIRECTORY} - python .azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py \ - --failure-context "${{ parameters.uploadPath }}/merged_failure_context.json" \ - --output-dir "${{ parameters.uploadPath }}/ai_analysis" \ - --failed-test-cases "${{ parameters.uploadPath }}/failed_tests_to_rerun.txt" \ - --project-root "${BUILD_SOURCESDIRECTORY}" \ - --failed-logs-root "${{ parameters.failureLogDownloadPath }}" + if ! command -v copilot >/dev/null 2>&1; then + if ! command -v npm >/dev/null 2>&1; then + echo "npm is required to install Copilot CLI" + exit 1 + fi + npm install -g @github/copilot + fi + copilot --version - patch_file="${{ parameters.uploadPath }}/ai_analysis/suggested_fix.patch" - if [ -s "${patch_file}" ]; then - git apply --check "${patch_file}" - git apply "${patch_file}" - fi - env: - COPILOT_GITHUB_TOKEN: $(GITHUB_TOKEN) - GITHUB_TOKEN: $(GITHUB_TOKEN) - GH_TOKEN: $(GITHUB_TOKEN) - displayName: "Copilot Analyze And Apply Patch" + python .azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py \ + --failure-context "${{ parameters.uploadPath }}/merged_failure_context.json" \ + --output-dir "${{ parameters.uploadPath }}/ai_analysis" \ + --project-root "${BUILD_SOURCESDIRECTORY}" \ + --failed-logs-root "${{ parameters.failureLogDownloadPath }}" \ + --classification-result "${{ parameters.uploadPath }}/ai_analysis/classification_result.json" + condition: and(succeeded(), eq(variables['CLASSIFICATION'], 'Code Regression')) + env: + COPILOT_GITHUB_TOKEN: $(GITHUB_TOKEN) + GITHUB_TOKEN: $(GITHUB_TOKEN) + GH_TOKEN: $(GITHUB_TOKEN) + displayName: "Analyze Code Regression (no auto-apply)" - - template: ut-template.yml - parameters: - dockerConfigName: ${{ parameters.dockerConfigName }} - utScriptFileName: "run_ut" - uploadPath: ${{ parameters.uploadPath }} - utArtifact: "ut-ai-rerun" - utTestMode: "0" - utContainerName: ${{ parameters.utContainerName }} - imageName: ${{ parameters.imageName }} - imageTag: ${{ parameters.imageTag }} - failureLogContext: "${{ parameters.uploadPath }}/ai_rerun_failure_context.json" - failedTestCases: "${{ parameters.uploadPath }}/failed_tests_to_rerun.txt" + - script: | + cd ${BUILD_SOURCESDIRECTORY} + python .azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py \ + --classification-result "${{ parameters.uploadPath }}/ai_analysis/classification_result.json" \ + --analysis-result "${{ parameters.uploadPath }}/ai_analysis/analysis_result.json" \ + --report "${{ parameters.uploadPath }}/ai_analysis/ai_failure_report.md" + condition: succeededOrFailed() + env: + GITHUB_TOKEN: $(GITHUB_TOKEN) + displayName: "Post Copilot Analysis Comment" - - script: | - cd ${BUILD_SOURCESDIRECTORY} - python .azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py \ - --analysis-result "${{ parameters.uploadPath }}/ai_analysis/analysis_result.json" \ - --report "${{ parameters.uploadPath }}/ai_analysis/ai_failure_report.md" - env: - GITHUB_TOKEN: $(GITHUB_TOKEN) - displayName: "Post Copilot Analysis Comment" - - - task: PublishPipelineArtifact@1 - condition: succeededOrFailed() - inputs: - targetPath: ${{ parameters.uploadPath }} - artifact: ${{ parameters.analysisArtifactName }} - publishLocation: "pipeline" + - task: PublishPipelineArtifact@1 + condition: succeededOrFailed() + inputs: + targetPath: ${{ parameters.uploadPath }} + artifact: ${{ parameters.analysisArtifactName }} + publishLocation: "pipeline" diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml index 664bc6508..7afc676da 100644 --- a/.azure-pipelines/template/ut-template.yml +++ b/.azure-pipelines/template/ut-template.yml @@ -35,9 +35,6 @@ parameters: - name: failureLogContext type: string default: "" - - name: failedTestCases - type: string - default: "" - name: failureLogArtifactPrefix type: string default: "ut-failure-log" @@ -114,10 +111,6 @@ steps: EXTRA_ARGS="--failure-context ${{ parameters.failureLogContext }}" fi - if [ -n "${{ parameters.failedTestCases }}" ]; then - EXTRA_ARGS="${EXTRA_ARGS} --failed-test-cases ${{ parameters.failedTestCases }}" - fi - docker exec -e NUMA_NODE=${NUMA_NODE} -e NUMA_CPUSET=${NUMA_CPUSET} -e ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK} ${{ parameters.utContainerName }} \ bash -c "cd /auto-round/.azure-pipelines/scripts \ && bash ut/${{ parameters.utScriptFileName }}.sh --test-part ${{ parameters.utTestMode }} ${EXTRA_ARGS}" diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.yml index d68205d27..38103bc2a 100644 --- a/.azure-pipelines/unit-test.yml +++ b/.azure-pipelines/unit-test.yml @@ -45,6 +45,7 @@ variables: FAILURE_LOG_ARTIFACT_PREFIX: "ut-failure-log" FAILURE_LOG_DOWNLOAD_PATH: $(Pipeline.Workspace)/failure_logs COPILOT_ANALYSIS_ENABLED: "true" + CI_ADMIN_HANDLE: "chensuyue" stages: - template: template/lib-build-template.yml @@ -85,21 +86,18 @@ stages: failureLogContext: $(UPLOAD_PATH)/failure_context_part$(PART).json - stage: AI_Analysis - displayName: "AI Analyze And Rerun" + displayName: "AI Analyze" dependsOn: [Unit_test] condition: and(failed('Unit_test'), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['COPILOT_ANALYSIS_ENABLED'], 'true')) pool: ICX-16C jobs: - - job: AnalyzeAndRerun - timeoutInMinutes: 60 - steps: - - template: template/ai-analysis-template.yml - parameters: - dockerConfigName: "commonDockerConfig" - utContainerName: "AutoRoundUnitTestAI$(NODE_LABEL)" - imageName: $(IMAGE_NAME) - imageTag: $(IMAGE_TAG) - uploadPath: $(UPLOAD_PATH) + - template: template/ai-analysis-template.yml + parameters: + dockerConfigName: "commonDockerConfig" + utContainerName: "AutoRoundUnitTestAI$(NODE_LABEL)" + imageName: $(IMAGE_NAME) + imageTag: $(IMAGE_TAG) + uploadPath: $(UPLOAD_PATH) - stage: Coverage displayName: "Collect Coverage" From 32fecde2bec9f725aea89f079f1d1548ad6728bf Mon Sep 17 00:00:00 2001 From: chensuyue Date: Sat, 6 Jun 2026 15:52:07 +0800 Subject: [PATCH 04/20] add issue group Signed-off-by: chensuyue --- .../analyze_and_suggest.py | 90 ++++++++--- .../scripts/ai_failure_analysis/classify.py | 112 +++++++++----- .../evidence_collectors.py | 117 ++++++++++----- .../known_issue_matcher.py | 53 ++++--- .../merge_failure_context.py | 141 +++++++++++++++++- .../ai_failure_analysis/post_pr_comment.py | 31 +++- .azure-pipelines/scripts/ut/collect_result.py | 16 +- 7 files changed, 432 insertions(+), 128 deletions(-) diff --git a/.azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py b/.azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py index 9d8d2170e..e7348647b 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py @@ -45,10 +45,21 @@ def truncate_text(value: str, limit: int = 6000) -> str: return value[: limit - 3] + "..." +def all_failures_from_payload(payload: dict) -> list[dict]: + """Return flattened failure cases from either failures[] or groups[].""" + if payload.get("failures"): + return payload.get("failures", []) + flattened = [] + for group in payload.get("groups", []): + flattened.extend(group.get("cases", [])) + return flattened + + def collect_failed_log_paths(payload: dict, failed_logs_root: Path | None, project_root: Path) -> list[str]: + failures = all_failures_from_payload(payload) requested = [ (entry.get("log_file") or "").strip() - for entry in payload.get("failures", []) + for entry in failures if (entry.get("log_file") or "").strip() ] @@ -148,6 +159,7 @@ def build_agent_prompt( "Tooling policy:", "- Non-interactive only; do not request user input.", "- Prefer read-only inspection of files and git history.", + "- Prioritize merged failure context first; only scan full failed logs when evidence is insufficient.", "- Keep suggestions minimal and low risk.", "", "Output STRICT JSON only with this schema:", @@ -206,7 +218,7 @@ def run_copilot_cli(prompt: str, project_root: Path, github_token: str) -> tuple def fallback_analysis(payload: dict) -> dict: - failures = payload.get("failures", []) + failures = all_failures_from_payload(payload) if not failures: return { "error_classification": [], @@ -340,8 +352,8 @@ def main(): parser.add_argument( "--classification-result", type=Path, - default=None, - help="Optional classification_result.json; deep analysis runs only for Code Regression.", + required=True, + help="classification_result.json from classify.py; required for regression-group-only analysis.", ) args = parser.parse_args() @@ -350,21 +362,53 @@ def main(): project_root = args.project_root.resolve() token = get_token_from_env() - classification_info = {} - if args.classification_result and args.classification_result.exists(): - classification_info = load_json(args.classification_result) - classification = classification_info.get("classification", "") - if classification and classification != "Code Regression": - write_skipped_result( - args.output_dir, - classification, - classification_info.get("confidence", ""), - "non-regression category handled by comment routing", - project_root, - ) - return - - failed_log_paths = collect_failed_log_paths(payload, args.failed_logs_root, project_root) + classification_info = load_json(args.classification_result) + regression_group_ids: list[str] = [] + per_group = classification_info.get("per_group_results", []) + regression_group_ids = [ + item.get("group_id", "") + for item in per_group + if item.get("classification") == "Code Regression" + ] + + summary = classification_info.get("summary", {}) + has_regression_group = bool(summary.get("has_code_regression_group", False)) or bool(regression_group_ids) + if not has_regression_group: + write_skipped_result( + args.output_dir, + "Non-Regression", + "", + "no regression groups found in grouped classification result", + project_root, + ) + return + + analysis_payload = payload + groups = payload.get("groups", []) + selected_groups = [g for g in groups if g.get("group_id", "") in set(regression_group_ids)] + if not selected_groups: + write_skipped_result( + args.output_dir, + "Non-Regression", + "", + "classification lists regression groups, but none were found in failure context", + project_root, + ) + return + + analysis_payload = { + "groups": selected_groups, + "build": payload.get("build", {}), + "stats": { + "group_count": len(selected_groups), + "failed_cases": sum(len(g.get("cases", [])) for g in selected_groups), + }, + } + context_for_prompt = args.output_dir / "regression_groups_context.json" + with open(context_for_prompt, "w", encoding="utf-8") as f: + json.dump(analysis_payload, f, indent=2) + + failed_log_paths = collect_failed_log_paths(analysis_payload, args.failed_logs_root, project_root) failed_log_paths_file = args.output_dir / "failed_log_paths.txt" write_text(failed_log_paths_file, "\n".join(failed_log_paths) + ("\n" if failed_log_paths else "")) @@ -372,7 +416,7 @@ def main(): patch_ok, patch_message = generate_pr_patch(project_root, pr_patch_path, base_ref=args.base_ref) prompt = build_agent_prompt( - merged_failure_context=args.failure_context, + merged_failure_context=context_for_prompt, failed_log_paths_file=failed_log_paths_file, pr_patch_file=pr_patch_path, project_root=project_root, @@ -382,7 +426,7 @@ def main(): cli_ok, cli_message, cli_result, cli_raw = run_copilot_cli(prompt, project_root, token) - fallback = fallback_analysis(payload) + fallback = fallback_analysis(analysis_payload) analysis = fallback.copy() if cli_ok and cli_result: analysis.update( @@ -473,7 +517,9 @@ def main(): "success": cli_ok, "skipped": False, "classification": classification_info.get("classification", "Code Regression"), - "classification_confidence": classification_info.get("confidence", ""), + "summary": classification_info.get("summary", {}), + "regression_group_ids": regression_group_ids, + "analyzed_group_count": len(regression_group_ids), "copilot_cli_ok": cli_ok, "copilot_cli_message": cli_message, "external_copilot_ok": False, diff --git a/.azure-pipelines/scripts/ai_failure_analysis/classify.py b/.azure-pipelines/scripts/ai_failure_analysis/classify.py index 8ef1f67a5..cf52f133d 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/classify.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/classify.py @@ -67,40 +67,44 @@ def emit_pipeline_variable(name: str, value: str): print(f"##vso[task.setvariable variable={name}]{safe}") -def heuristic_classification(evidence: dict, known_issues: dict) -> dict: - """Deterministic fallback when Copilot is unavailable. +def _group_pr_relevance(evidence: dict, group_id: str) -> dict: + for item in evidence.get("pr_relevance", {}).get("per_group", []): + if item.get("group_id") == group_id: + return item + return {} - Mirrors the priority order so behaviour is predictable without the agent. - """ - matches = known_issues.get("matches", []) - if matches and matches[0].get("confidence", 0.0) >= KNOWN_ISSUE_AUTO_THRESHOLD: + +def heuristic_classification(group: dict, evidence: dict, known_matches: list[dict]) -> dict: + """Deterministic group-level classification.""" + if known_matches and known_matches[0].get("confidence", 0.0) >= KNOWN_ISSUE_AUTO_THRESHOLD: return { "classification": KNOWN_ISSUE, - "confidence": matches[0]["confidence"], - "evidence": [f"matches known issue #{matches[0].get('number')}"], + "confidence": known_matches[0]["confidence"], + "evidence": [f"matches known issue #{known_matches[0].get('number')}"], "reasoning": "Known-issue matcher returned a high-confidence ticket.", } - env = evidence.get("environment", {}) - if env.get("has_environment_signal"): - signals = list(env.get("signals", {}).keys()) + if group.get("signature_type") == "env_signal": + signature = group.get("signature", "") + signal = signature.split(":", 1)[1] if ":" in signature else signature return { "classification": ENVIRONMENT, "confidence": 0.7, - "evidence": [f"environment signal: {s}" for s in signals][:5], - "reasoning": "Deterministic environment signals were detected in logs.", + "evidence": [f"environment signal: {signal}"], + "reasoning": "Group signature indicates infrastructure/environment symptoms.", } - pr = evidence.get("pr_relevance", {}) - if pr.get("relevance_score", 0.0) >= 0.5 and pr.get("touches_source"): + pr_group = _group_pr_relevance(evidence, group.get("group_id", "")) + pr_global = evidence.get("pr_relevance", {}) + if pr_group.get("relevance_score", 0.0) >= 0.5 and pr_global.get("touches_source"): return { "classification": CODE_REGRESSION, - "confidence": round(min(0.9, 0.5 + pr["relevance_score"] / 2), 3), + "confidence": round(min(0.9, 0.5 + pr_group["relevance_score"] / 2), 3), "evidence": [ - f"relevance_score={pr.get('relevance_score')}", - f"directly_changed_tests={pr.get('directly_changed_tests', [])[:5]}", + f"group_relevance_score={pr_group.get('relevance_score')}", + f"directly_changed_tests={pr_group.get('directly_changed_tests', [])[:5]}", ], - "reasoning": "Failed tests correlate with PR-changed source files.", + "reasoning": "This group correlates with PR-changed source or tests.", } return { @@ -111,7 +115,7 @@ def heuristic_classification(evidence: dict, known_issues: dict) -> dict: } -def finalize_classification(raw_result: dict, known_issues: dict, admin_handle: str) -> dict: +def finalize_classification(raw_result: dict, known_matches: list[dict], admin_handle: str) -> dict: """Apply guard rails: known-issue short-circuit and confidence threshold.""" classification = str(raw_result.get("classification", "")).strip() if classification not in VALID_CATEGORIES: @@ -127,7 +131,7 @@ def finalize_classification(raw_result: dict, known_issues: dict, admin_handle: reasoning = raw_result.get("reasoning", "") notify_admin = False - matches = known_issues.get("matches", []) + matches = known_matches # Guard 1: strong known-issue match always wins. if matches and matches[0].get("confidence", 0.0) >= KNOWN_ISSUE_AUTO_THRESHOLD: @@ -172,37 +176,65 @@ def main(): project_root = args.project_root.resolve() payload = load_json(args.failure_context) - failures = payload.get("failures", []) + groups = payload.get("groups", []) - evidence = evidence_collectors.collect_all_evidence(failures, project_root, args.base_ref) + evidence = evidence_collectors.collect_all_evidence(groups, project_root, args.base_ref) repo_path = known_issue_matcher._repo_path_from_env() token = os.environ.get("GITHUB_TOKEN", "") known_issues = known_issue_matcher.match_known_issues( - failures, repo_path, token, label=args.known_issue_label + groups, repo_path, token, label=args.known_issue_label ) - raw_result = heuristic_classification(evidence, known_issues) + known_map = { + item.get("group_id", ""): item.get("matches", []) + for item in known_issues.get("per_group_matches", []) + } - final = finalize_classification(raw_result, known_issues, args.admin_handle) - final.update( - { - "used_copilot": False, - "evidence_bundle": evidence, - "source_commit": payload.get("build", {}).get("source_commit", ""), - "pr_number": payload.get("build", {}).get("pr_number", ""), - "failure_count": len(failures), - } - ) + per_group_results = [] + category_counts = {key: 0 for key in VALID_CATEGORIES} + + for group in groups: + group_id = group.get("group_id", "") + matches = known_map.get(group_id, []) + raw_result = heuristic_classification(group, evidence, matches) + final_group = finalize_classification(raw_result, matches, args.admin_handle) + final_group.update( + { + "group_id": group_id, + "group_signature": group.get("signature", ""), + "group_size": len(group.get("cases", [])), + } + ) + per_group_results.append(final_group) + category_counts[final_group["classification"]] = category_counts.get(final_group["classification"], 0) + 1 + + has_regression = any(item.get("classification") == CODE_REGRESSION for item in per_group_results) + summary = { + "group_count": len(groups), + "category_counts": {k: v for k, v in category_counts.items() if v > 0}, + "has_code_regression_group": has_regression, + } + + failure_count = sum(len(group.get("cases", [])) for group in groups) + final = { + "per_group_results": per_group_results, + "summary": summary, + "evidence_bundle": evidence, + "source_commit": payload.get("build", {}).get("source_commit", ""), + "pr_number": payload.get("build", {}).get("pr_number", ""), + "group_count": len(groups), + "failure_count": failure_count, + "ci_admin_handle": args.admin_handle, + } write_json(args.output, final) - emit_pipeline_variable("CLASSIFICATION", final["classification"]) - emit_pipeline_variable("HANDLING_ACTION", final["handling_action"]) + emit_pipeline_variable("CLASSIFICATION", CODE_REGRESSION if has_regression else "Non-Regression") + emit_pipeline_variable("HANDLING_ACTION", "grouped_routing") - print(f"classify: classification={final['classification']} " - f"confidence={final['confidence']} action={final['handling_action']} " - f"notify_admin={final['notify_admin']}") + print(f"classify: groups={len(groups)} failures={failure_count} " + f"regression_groups={summary['category_counts'].get(CODE_REGRESSION, 0)}") print(f"classify: result written to {args.output}") diff --git a/.azure-pipelines/scripts/ai_failure_analysis/evidence_collectors.py b/.azure-pipelines/scripts/ai_failure_analysis/evidence_collectors.py index a97bca0ad..ad8d0aea0 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/evidence_collectors.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/evidence_collectors.py @@ -92,6 +92,12 @@ def _failure_text(entry: dict) -> str: ) +def _iter_cases(groups: list[dict]): + for group in groups: + for case in group.get("cases", []): + yield group, case + + def _snippet_around(text: str, match: re.Match, radius: int = 120) -> str: start = max(0, match.start() - radius) end = min(len(text), match.end() + radius) @@ -99,7 +105,7 @@ def _snippet_around(text: str, match: re.Match, radius: int = 120) -> str: return re.sub(r"\s+", " ", snippet) -def collect_environment_signals(failures: list[dict]) -> dict: +def collect_environment_signals(groups: list[dict]) -> dict: """Scan failures for environment-related signals. Returns a dict with matched signal keys, supporting evidence snippets, and @@ -107,7 +113,25 @@ def collect_environment_signals(failures: list[dict]) -> dict: """ matched: dict[str, dict] = {} - for entry in failures: + signal_labels = {key: label for key, label, _ in ENVIRONMENT_PATTERNS} + + for group in groups: + signature_type = group.get("signature_type", "") + signature = group.get("signature", "") + if signature_type == "env_signal" and signature.startswith("env:"): + signal_key = signature.split(":", 1)[1] + record = matched.setdefault( + signal_key, + {"label": signal_labels.get(signal_key, signal_key), "evidence": [], "tests": []}, + ) + for ev in group.get("evidence", [])[:3]: + if ev and ev not in record["evidence"]: + record["evidence"].append(ev) + for test_name in group.get("test_names", []): + if test_name and test_name not in record["tests"]: + record["tests"].append(test_name) + + for _group, entry in _iter_cases(groups): text = _failure_text(entry) if not text: continue @@ -170,7 +194,7 @@ def _candidate_test_file(test_name: str) -> str: def collect_pr_relevance( - failures: list[dict], project_root: Path, base_ref: str = "main" + groups: list[dict], project_root: Path, base_ref: str = "main" ) -> dict: """Correlate failed tests with PR-changed files. @@ -181,39 +205,53 @@ def collect_pr_relevance( changed_set = set(changed_files) changed_stems = {Path(f).stem for f in changed_files if f.endswith(".py")} - # Tests whose own test file was modified by the PR. directly_changed_tests: list[str] = [] - # Tests whose corresponding source module (by stem) was modified. source_related_tests: list[dict] = [] - - for entry in failures: - test_name = entry.get("test_name", "") - test_file = _candidate_test_file(test_name) - test_stem = Path(test_file).stem if test_file else "" - - # Direct: the failing test file itself is in the PR diff. - for changed in changed_set: - if test_file and changed.endswith(test_file): - if test_name not in directly_changed_tests: - directly_changed_tests.append(test_name) - break - - # Indirect: a changed source module matches the test's subject. - # ``test_quantize`` -> subject stem ``quantize``. - subject_stem = test_stem[len("test_"):] if test_stem.startswith("test_") else test_stem - related_modules = sorted( - f for f in changed_files - if f.endswith(".py") and subject_stem and Path(f).stem == subject_stem - ) - if related_modules: - source_related_tests.append( - {"test": test_name, "modules": related_modules} + per_group: list[dict] = [] + + for group in groups: + group_id = group.get("group_id", "") + group_tests = group.get("test_names", []) or [ + c.get("test_name", "") for c in group.get("cases", []) if c.get("test_name", "") + ] + group_direct: list[str] = [] + group_related: list[dict] = [] + + for test_name in group_tests: + test_file = _candidate_test_file(test_name) + test_stem = Path(test_file).stem if test_file else "" + + for changed in changed_set: + if test_file and changed.endswith(test_file): + if test_name not in group_direct: + group_direct.append(test_name) + if test_name not in directly_changed_tests: + directly_changed_tests.append(test_name) + break + + subject_stem = test_stem[len("test_"):] if test_stem.startswith("test_") else test_stem + related_modules = sorted( + f for f in changed_files + if f.endswith(".py") and subject_stem and Path(f).stem == subject_stem ) + if related_modules: + group_related.append({"test": test_name, "modules": related_modules}) + source_related_tests.append({"test": test_name, "modules": related_modules}) + + score = 0.0 + if group_tests: + score = round(min(1.0, (len(group_direct) + len(group_related)) / max(1, len(group_tests))), 3) + + per_group.append( + { + "group_id": group_id, + "directly_changed_tests": group_direct, + "source_related_tests": group_related, + "relevance_score": score, + } + ) - relevance_score = 0.0 - if failures: - related_count = len(directly_changed_tests) + len(source_related_tests) - relevance_score = round(min(1.0, related_count / max(1, len(failures))), 3) + relevance_score = max((item["relevance_score"] for item in per_group), default=0.0) # Heuristic: PR only touches code under auto_round/ vs only tests/ vs docs. touches_source = any( @@ -229,6 +267,7 @@ def collect_pr_relevance( "directly_changed_tests": directly_changed_tests, "source_related_tests": source_related_tests, "relevance_score": relevance_score, + "per_group": per_group, "touches_source": touches_source, "touches_tests": touches_tests, } @@ -249,7 +288,7 @@ def collect_dependency_changes(project_root: Path) -> dict: } -def collect_flaky_signals(failures: list[dict]) -> dict: +def collect_flaky_signals(groups: list[dict]) -> dict: """Phase 2 stub: detect flaky tests via history / rerun comparison. A full implementation will consult historical pass/fail rates and the @@ -264,13 +303,15 @@ def collect_flaky_signals(failures: list[dict]) -> dict: def collect_all_evidence( - failures: list[dict], project_root: Path, base_ref: str = "main" + groups: list[dict], project_root: Path, base_ref: str = "main" ) -> dict: """Aggregate every collector into a single evidence bundle.""" + failure_count = sum(len(group.get("cases", [])) for group in groups) return { - "failure_count": len(failures), - "environment": collect_environment_signals(failures), - "pr_relevance": collect_pr_relevance(failures, project_root, base_ref), + "failure_count": failure_count, + "group_count": len(groups), + "environment": collect_environment_signals(groups), + "pr_relevance": collect_pr_relevance(groups, project_root, base_ref), "dependency": collect_dependency_changes(project_root), - "flaky": collect_flaky_signals(failures), + "flaky": collect_flaky_signals(groups), } diff --git a/.azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py b/.azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py index 9bfa294c6..fdc6b47ae 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py @@ -92,6 +92,17 @@ def _failure_text(failures: list[dict]) -> str: return "\n".join(p for p in parts if p) +def _group_text(group: dict) -> str: + parts = [] + parts.extend(group.get("test_names", [])) + parts.extend(group.get("evidence", [])) + for case in group.get("cases", []): + parts.append(case.get("test_name", "")) + parts.append(case.get("excerpt", "")) + parts.append(case.get("tail", "")) + return "\n".join(p for p in parts if p) + + def score_issue(issue: dict, failure_tokens: set[str], failure_excs: set[str]) -> dict: """Score one issue against the aggregated failure fingerprint.""" issue_text = f"{issue['title']}\n{issue['body']}" @@ -117,28 +128,33 @@ def score_issue(issue: dict, failure_tokens: set[str], failure_excs: set[str]) - def match_known_issues( - failures: list[dict], + groups: list[dict], repo_path: str, token: str, label: str = DEFAULT_LABEL, min_confidence: float = 0.3, ) -> dict: - """Return ranked known-issue matches for the given failures.""" + """Return ranked known-issue matches for each group.""" issues = fetch_known_issues(repo_path, token, label) if not issues: - return {"label": label, "checked": 0, "matches": []} - - failure_text = _failure_text(failures) - failure_tokens = _tokenize(failure_text) - failure_excs = _exceptions(failure_text) + return {"label": label, "checked": 0, "per_group_matches": []} + + per_group_matches = [] + for group in groups: + group_id = group.get("group_id", "") + group_text = _group_text(group) + failure_tokens = _tokenize(group_text) + failure_excs = _exceptions(group_text) + + scored = [score_issue(issue, failure_tokens, failure_excs) for issue in issues] + matches = sorted( + (s for s in scored if s["confidence"] >= min_confidence), + key=lambda s: s["confidence"], + reverse=True, + ) + per_group_matches.append({"group_id": group_id, "matches": matches}) - scored = [score_issue(issue, failure_tokens, failure_excs) for issue in issues] - matches = sorted( - (s for s in scored if s["confidence"] >= min_confidence), - key=lambda s: s["confidence"], - reverse=True, - ) - return {"label": label, "checked": len(issues), "matches": matches} + return {"label": label, "checked": len(issues), "per_group_matches": per_group_matches} def _repo_path_from_env() -> str: @@ -160,7 +176,7 @@ def main(): with open(args.failure_context, "r", encoding="utf-8") as f: payload = json.load(f) - failures = payload.get("failures", []) + groups = payload.get("groups", []) token = ( os.environ.get("GITHUB_TOKEN", "") @@ -170,15 +186,16 @@ def main(): repo_path = _repo_path_from_env() result = match_known_issues( - failures, repo_path, token, label=args.label, min_confidence=args.min_confidence + groups, repo_path, token, label=args.label, min_confidence=args.min_confidence ) args.output.parent.mkdir(parents=True, exist_ok=True) with open(args.output, "w", encoding="utf-8") as f: json.dump(result, f, indent=2) - print(f"known_issue_matcher: checked {result['checked']} issues, " - f"{len(result['matches'])} matches >= {args.min_confidence}") + matched_groups = sum(1 for item in result.get("per_group_matches", []) if item.get("matches")) + print(f"known_issue_matcher: checked {result['checked']} issues, " + f"{matched_groups} groups with matches >= {args.min_confidence}") if __name__ == "__main__": diff --git a/.azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py b/.azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py index bed4637c9..333bdf512 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py @@ -1,9 +1,22 @@ import argparse +import hashlib import json import os import re from pathlib import Path +ENV_SIGNAL_PATTERNS: list[tuple[str, re.Pattern]] = [ + ("network_timeout", re.compile(r"(connection\s+timed?\s*out|read\s+timed?\s*out|max\s+retries\s+exceeded|ECONNRESET|ECONNREFUSED|ETIMEDOUT)", re.IGNORECASE)), + ("disk_full", re.compile(r"(no\s+space\s+left\s+on\s+device|disk\s+quota\s+exceeded|ENOSPC)", re.IGNORECASE)), + ("runner_lost", re.compile(r"(lost\s+communication\s+with\s+the\s+agent|runner\s+has\s+been\s+lost|did\s+not\s+respond)", re.IGNORECASE)), + ("out_of_memory", re.compile(r"(out\s+of\s+memory|cannot\s+allocate\s+memory|oom[\s-]?kill|std::bad_alloc|MemoryError)", re.IGNORECASE)), + ("hf_download_error", re.compile(r"(HfHubHTTPError|huggingface\.co|couldn'?t\s+connect\s+to\s+['\"]?https?://huggingface)", re.IGNORECASE)), + ("rate_limited", re.compile(r"(rate\s+limit\s+exceeded|too\s+many\s+requests|HTTP\s+429)", re.IGNORECASE)), +] + +EXCEPTION_RE = re.compile(r"\b([A-Z][A-Za-z0-9_]*(?:Error|Exception|Warning))\b") +ASSERT_RE = re.compile(r"(AssertionError[:\s].+)$", re.IGNORECASE | re.MULTILINE) + def load_json(path: Path): with open(path, "r", encoding="utf-8") as f: @@ -23,6 +36,85 @@ def normalize_failure(entry: dict, source_file: str, artifact: str) -> dict: } +def _case_text(entry: dict) -> str: + return "\n".join(part for part in (entry.get("excerpt", ""), entry.get("tail", "")) if part) + + +def _normalize_text(value: str, max_len: int = 180) -> str: + text = re.sub(r"\s+", " ", (value or "").strip()) + text = re.sub(r"0x[0-9a-fA-F]+", "0xADDR", text) + text = re.sub(r"\b\d+\b", "N", text) + return text[:max_len] + + +def _detect_env_signal(text: str) -> str: + for key, pattern in ENV_SIGNAL_PATTERNS: + if pattern.search(text): + return key + return "" + + +def _extract_exception(text: str) -> str: + hits = EXCEPTION_RE.findall(text or "") + return hits[-1] if hits else "" + + +def _extract_terminal_line(text: str) -> str: + lines = [line.strip() for line in (text or "").splitlines() if line.strip()] + if not lines: + return "" + return _normalize_text(lines[-1]) + + +def _extract_assertion(text: str) -> str: + match = ASSERT_RE.search(text or "") + if not match: + return "" + return _normalize_text(match.group(1)) + + +def build_group_signature(entry: dict) -> tuple[str, str, str, str]: + """Return (signature_type, signature, summary, evidence).""" + text = _case_text(entry) + env_signal = _detect_env_signal(text) + if env_signal: + return ( + "env_signal", + f"env:{env_signal}", + f"Environment signal: {env_signal}", + _extract_terminal_line(text) or env_signal, + ) + + exception_name = _extract_exception(text) + if exception_name: + terminal = _extract_terminal_line(text) + signature = f"exception:{exception_name}:{terminal}" + return ( + "exception", + signature, + f"{exception_name} triggered similar failures", + terminal or exception_name, + ) + + assertion = _extract_assertion(text) + if assertion: + return ( + "assertion", + f"assertion:{assertion}", + "Assertion-like failures share the same symptom", + assertion, + ) + + seed = f"{entry.get('test_name', '')}|{_normalize_text(text, max_len=220)}" + short_hash = hashlib.sha1(seed.encode("utf-8")).hexdigest()[:12] + return ( + "fallback", + f"fallback:{short_hash}", + "Fallback signature built from normalized failure text", + _extract_terminal_line(text) or _normalize_text(text, max_len=120), + ) + + def get_artifact_name(context_file: Path, input_root: Path) -> str: rel = context_file.relative_to(input_root) if rel.parts: @@ -76,7 +168,44 @@ def merge_contexts(input_root: Path) -> list[dict]: seen.add(key) merged_failures.append(normalized) - return merged_failures + groups_by_signature: dict[str, dict] = {} + for case in merged_failures: + signature_type, signature, summary, evidence = build_group_signature(case) + group = groups_by_signature.get(signature) + if group is None: + group = { + "group_id": "", + "signature_type": signature_type, + "signature": signature, + "summary": summary, + "evidence": [], + "test_names": [], + "log_files": [], + "cases": [], + } + groups_by_signature[signature] = group + + group["cases"].append(case) + + test_name = case.get("test_name", "") + if test_name and test_name not in group["test_names"]: + group["test_names"].append(test_name) + + log_file = case.get("log_file", "") + if log_file and log_file not in group["log_files"]: + group["log_files"].append(log_file) + + if evidence and evidence not in group["evidence"] and len(group["evidence"]) < 5: + group["evidence"].append(evidence) + + groups = sorted( + groups_by_signature.values(), + key=lambda g: (-len(g.get("cases", [])), g.get("signature", "")), + ) + for index, group in enumerate(groups, start=1): + group["group_id"] = f"g{index:03d}" + + return groups def main(): @@ -87,7 +216,8 @@ def main(): args.output.parent.mkdir(parents=True, exist_ok=True) - failures = merge_contexts(args.input_root) + groups = merge_contexts(args.input_root) + failed_cases = sum(len(group.get("cases", [])) for group in groups) payload = { "schema_version": "1.0", @@ -97,15 +227,16 @@ def main(): "pr_number": os.environ.get("SYSTEM_PULLREQUEST_PULLREQUESTNUMBER", ""), }, "stats": { - "failed_cases": len(failures), + "failed_cases": failed_cases, + "group_count": len(groups), }, - "failures": failures, + "groups": groups, } with open(args.output, "w", encoding="utf-8") as f: json.dump(payload, f, indent=2) - print(f"Merged {len(failures)} failures from {args.input_root}") + print(f"Merged {failed_cases} failures into {len(groups)} groups from {args.input_root}") print(f"Merged context: {args.output}") diff --git a/.azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py b/.azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py index 331623a45..bbe3f56c1 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py @@ -150,14 +150,41 @@ def build_category_section(classification: dict, analysis: dict) -> list[str]: return header + [""] + body +def build_group_section(group_result: dict, analysis: dict, index: int) -> list[str]: + group_id = group_result.get("group_id", f"g{index:03d}") + lines = [ + f"### Group {index}: {group_id}", + f"- Signature: {group_result.get('group_signature', '')}", + f"- Cases: {group_result.get('group_size', 0)}", + ] + lines.extend(build_category_section(group_result, analysis)) + lines.extend(_evidence_lines(group_result)) + lines.append("") + return lines + + def build_comment_body(classification: dict, analysis: dict, report_text: str, marker: str) -> str: lines = [ marker, "## Copilot CI Failure Analysis", "", ] - lines.extend(build_category_section(classification, analysis)) - lines.extend(_evidence_lines(classification)) + + per_group = classification.get("per_group_results", []) + if per_group: + summary = classification.get("summary", {}) + lines.append(f"- Group count: {summary.get('group_count', len(per_group))}") + counts = summary.get("category_counts", {}) + if counts: + compact = ", ".join(f"{k}={v}" for k, v in counts.items()) + lines.append(f"- Category counts: {compact}") + lines.append("") + for idx, group_result in enumerate(per_group, start=1): + lines.extend(build_group_section(group_result, analysis, idx)) + else: + lines.extend(build_category_section(classification, analysis)) + lines.extend(_evidence_lines(classification)) + if report_text: lines.extend( [ diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.py index 25814ea8d..e93683916 100644 --- a/.azure-pipelines/scripts/ut/collect_result.py +++ b/.azure-pipelines/scripts/ut/collect_result.py @@ -205,7 +205,7 @@ def _format_row(self, result: TestResult) -> str: class FailureContextWriter: """Extract compact failure context for downstream AI analysis.""" - TRACEBACK_MARKERS = ("Traceback", "== FAILURES ==", "== ERRORS ==") + TRACEBACK_MARKERS = ("Traceback", "== FAILURES ==", "== ERRORS ==", "core dumped", "Killed") def __init__(self, log_dir: Path, max_lines: int = 200): self.log_dir = Path(log_dir) @@ -282,8 +282,18 @@ def _extract_excerpt(self, lines: list[str]) -> str: for marker in self.TRACEBACK_MARKERS: for idx, line in enumerate(lines): if marker in line: - start = max(0, idx - 10) - end = min(len(lines), idx + 80) + if marker == "Traceback": + start = max(0, idx - 10) + max_end = min(len(lines), idx + 80) + failed_end = None + for j in range(idx, max_end): + if "FAILED" in lines[j]: + failed_end = j + 1 + break + end = min(max_end, failed_end) if failed_end is not None else max_end + else: + start = max(0, idx - 10) + end = min(len(lines), idx + 80) selected = lines[start:end] break if selected: From 93d45117d6852a11c708d08be1206074566adbee Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 9 Jul 2026 09:58:18 +0800 Subject: [PATCH 05/20] fix know issue label Signed-off-by: chensuyue --- .../scripts/ai_failure_analysis/known_issue_matcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py b/.azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py index fdc6b47ae..76870c594 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py @@ -13,7 +13,7 @@ from urllib.parse import quote from urllib.request import Request, urlopen -DEFAULT_LABEL = "CI_known_issue" +DEFAULT_LABEL = "CI-known-issue" # Tokens shorter than this or in the stop list add noise to the overlap score. _MIN_TOKEN_LEN = 4 _STOP_TOKENS = { From 8e1a086e2cec08f4d73cf9c55f7edfd8527edaf8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 02:01:46 +0000 Subject: [PATCH 06/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../analyze_and_suggest.py | 62 ++++++++++--------- .../scripts/ai_failure_analysis/classify.py | 13 ++-- .../evidence_collectors.py | 32 +++------- .../known_issue_matcher.py | 46 +++++++++----- .../merge_failure_context.py | 37 +++++++++-- .../ai_failure_analysis/post_pr_comment.py | 7 +-- .azure-pipelines/scripts/ut/collect_result.py | 4 +- 7 files changed, 112 insertions(+), 89 deletions(-) diff --git a/.azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py b/.azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py index e7348647b..39161a73f 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py @@ -57,11 +57,7 @@ def all_failures_from_payload(payload: dict) -> list[dict]: def collect_failed_log_paths(payload: dict, failed_logs_root: Path | None, project_root: Path) -> list[str]: failures = all_failures_from_payload(payload) - requested = [ - (entry.get("log_file") or "").strip() - for entry in failures - if (entry.get("log_file") or "").strip() - ] + requested = [(entry.get("log_file") or "").strip() for entry in failures if (entry.get("log_file") or "").strip()] candidate_roots = [] if failed_logs_root: @@ -366,9 +362,7 @@ def main(): regression_group_ids: list[str] = [] per_group = classification_info.get("per_group_results", []) regression_group_ids = [ - item.get("group_id", "") - for item in per_group - if item.get("classification") == "Code Regression" + item.get("group_id", "") for item in per_group if item.get("classification") == "Code Regression" ] summary = classification_info.get("summary", {}) @@ -431,7 +425,9 @@ def main(): if cli_ok and cli_result: analysis.update( { - "error_classification": cli_result.get("error_classification", analysis.get("error_classification", [])), + "error_classification": cli_result.get( + "error_classification", analysis.get("error_classification", []) + ), "root_cause": cli_result.get("root_cause", analysis.get("root_cause", "")), "confidence": cli_result.get("confidence", analysis.get("confidence", "medium")), "suggestion": cli_result.get("suggestion", analysis.get("suggestion", "")), @@ -478,24 +474,28 @@ def main(): else: report_lines.append("- No structured classification returned.") - report_lines.extend([ - "", - "## Root Cause", - analysis.get("root_cause", "No root cause available."), - "", - "## Suggested Fix", - analysis.get("suggestion", "No suggestion available."), - "", - "## Static Checks (Agent)", - ]) + report_lines.extend( + [ + "", + "## Root Cause", + analysis.get("root_cause", "No root cause available."), + "", + "## Suggested Fix", + analysis.get("suggestion", "No suggestion available."), + "", + "## Static Checks (Agent)", + ] + ) for check in analysis.get("static_checks", [])[:20]: report_lines.append(f"- [{check.get('status', 'not_run')}] {check.get('command', 'unknown')}") - report_lines.extend([ - "", - "## Static Checks (Local)", - ]) + report_lines.extend( + [ + "", + "## Static Checks (Local)", + ] + ) for check in analysis.get("local_static_checks", [])[:20]: report_lines.append(f"- [{check.get('status', 'not_run')}] {check.get('command', 'unknown')}") @@ -503,13 +503,15 @@ def main(): excerpt = analysis.get("first_failure_excerpt", "") if excerpt: - report_lines.extend([ - "## First Failure Excerpt", - "```text", - excerpt, - "```", - "", - ]) + report_lines.extend( + [ + "## First Failure Excerpt", + "```text", + excerpt, + "```", + "", + ] + ) write_text(report_path, "\n".join(report_lines)) diff --git a/.azure-pipelines/scripts/ai_failure_analysis/classify.py b/.azure-pipelines/scripts/ai_failure_analysis/classify.py index cf52f133d..fd2197d4b 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/classify.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/classify.py @@ -182,13 +182,10 @@ def main(): repo_path = known_issue_matcher._repo_path_from_env() token = os.environ.get("GITHUB_TOKEN", "") - known_issues = known_issue_matcher.match_known_issues( - groups, repo_path, token, label=args.known_issue_label - ) + known_issues = known_issue_matcher.match_known_issues(groups, repo_path, token, label=args.known_issue_label) known_map = { - item.get("group_id", ""): item.get("matches", []) - for item in known_issues.get("per_group_matches", []) + item.get("group_id", ""): item.get("matches", []) for item in known_issues.get("per_group_matches", []) } per_group_results = [] @@ -233,8 +230,10 @@ def main(): emit_pipeline_variable("CLASSIFICATION", CODE_REGRESSION if has_regression else "Non-Regression") emit_pipeline_variable("HANDLING_ACTION", "grouped_routing") - print(f"classify: groups={len(groups)} failures={failure_count} " - f"regression_groups={summary['category_counts'].get(CODE_REGRESSION, 0)}") + print( + f"classify: groups={len(groups)} failures={failure_count} " + f"regression_groups={summary['category_counts'].get(CODE_REGRESSION, 0)}" + ) print(f"classify: result written to {args.output}") diff --git a/.azure-pipelines/scripts/ai_failure_analysis/evidence_collectors.py b/.azure-pipelines/scripts/ai_failure_analysis/evidence_collectors.py index ad8d0aea0..9c3dfe52a 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/evidence_collectors.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/evidence_collectors.py @@ -34,8 +34,7 @@ "disk_full", "Disk space exhausted", re.compile( - r"(no\s+space\s+left\s+on\s+device|disk\s+quota\s+exceeded|ENOSPC|" - r"cannot\s+write\s+.*:\s+no\s+space)", + r"(no\s+space\s+left\s+on\s+device|disk\s+quota\s+exceeded|ENOSPC|" r"cannot\s+write\s+.*:\s+no\s+space)", re.IGNORECASE, ), ), @@ -74,8 +73,7 @@ "rate_limited", "API rate limit / throttling", re.compile( - r"(rate\s+limit\s+exceeded|too\s+many\s+requests|HTTP\s+429|" - r"429\s+client\s+error)", + r"(rate\s+limit\s+exceeded|too\s+many\s+requests|HTTP\s+429|" r"429\s+client\s+error)", re.IGNORECASE, ), ), @@ -87,9 +85,7 @@ def _failure_text(entry: dict) -> str: """Concatenate the searchable text for a single failure entry.""" - return "\n".join( - part for part in (entry.get("excerpt", ""), entry.get("tail", "")) if part - ) + return "\n".join(part for part in (entry.get("excerpt", ""), entry.get("tail", "")) if part) def _iter_cases(groups: list[dict]): @@ -157,9 +153,7 @@ def collect_environment_signals(groups: list[dict]) -> dict: def _run_git(args: list[str], cwd: Path) -> tuple[int, str]: - result = subprocess.run( - ["git", *args], cwd=cwd, check=False, capture_output=True, text=True - ) + result = subprocess.run(["git", *args], cwd=cwd, check=False, capture_output=True, text=True) return result.returncode, (result.stdout or "").strip() @@ -193,9 +187,7 @@ def _candidate_test_file(test_name: str) -> str: return base -def collect_pr_relevance( - groups: list[dict], project_root: Path, base_ref: str = "main" -) -> dict: +def collect_pr_relevance(groups: list[dict], project_root: Path, base_ref: str = "main") -> dict: """Correlate failed tests with PR-changed files. This is the strongest signal separating Code Regression from other classes: @@ -229,10 +221,9 @@ def collect_pr_relevance( directly_changed_tests.append(test_name) break - subject_stem = test_stem[len("test_"):] if test_stem.startswith("test_") else test_stem + subject_stem = test_stem.removeprefix("test_") related_modules = sorted( - f for f in changed_files - if f.endswith(".py") and subject_stem and Path(f).stem == subject_stem + f for f in changed_files if f.endswith(".py") and subject_stem and Path(f).stem == subject_stem ) if related_modules: group_related.append({"test": test_name, "modules": related_modules}) @@ -254,10 +245,7 @@ def collect_pr_relevance( relevance_score = max((item["relevance_score"] for item in per_group), default=0.0) # Heuristic: PR only touches code under auto_round/ vs only tests/ vs docs. - touches_source = any( - f.startswith("auto_round/") or f.startswith("auto_round_extension/") - for f in changed_files - ) + touches_source = any(f.startswith("auto_round/") or f.startswith("auto_round_extension/") for f in changed_files) touches_tests = any(f.startswith("test/") for f in changed_files) return { @@ -302,9 +290,7 @@ def collect_flaky_signals(groups: list[dict]) -> dict: } -def collect_all_evidence( - groups: list[dict], project_root: Path, base_ref: str = "main" -) -> dict: +def collect_all_evidence(groups: list[dict], project_root: Path, base_ref: str = "main") -> dict: """Aggregate every collector into a single evidence bundle.""" failure_count = sum(len(group.get("cases", [])) for group in groups) return { diff --git a/.azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py b/.azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py index 76870c594..6de41d94d 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/known_issue_matcher.py @@ -17,9 +17,28 @@ # Tokens shorter than this or in the stop list add noise to the overlap score. _MIN_TOKEN_LEN = 4 _STOP_TOKENS = { - "test", "tests", "error", "errors", "failed", "failure", "assert", - "self", "none", "true", "false", "value", "object", "python", "trace", - "traceback", "line", "file", "call", "last", "most", "recent", + "test", + "tests", + "error", + "errors", + "failed", + "failure", + "assert", + "self", + "none", + "true", + "false", + "value", + "object", + "python", + "trace", + "traceback", + "line", + "file", + "call", + "last", + "most", + "recent", } _TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]{2,}") # Identifiers that are strong fingerprints: exception classes, error codes. @@ -41,11 +60,7 @@ def github_get(url: str, token: str): def _tokenize(text: str) -> set[str]: - tokens = { - tok.lower() - for tok in _TOKEN_RE.findall(text or "") - if len(tok) >= _MIN_TOKEN_LEN - } + tokens = {tok.lower() for tok in _TOKEN_RE.findall(text or "") if len(tok) >= _MIN_TOKEN_LEN} return tokens - _STOP_TOKENS @@ -57,10 +72,7 @@ def fetch_known_issues(repo_path: str, token: str, label: str = DEFAULT_LABEL) - """Fetch open issues carrying the known-issue label.""" if not token or not repo_path: return [] - url = ( - f"https://api.github.com/repos/{repo_path}/issues" - f"?state=open&labels={quote(label)}&per_page=100" - ) + url = f"https://api.github.com/repos/{repo_path}/issues" f"?state=open&labels={quote(label)}&per_page=100" try: issues = github_get(url, token) except Exception as exc: # noqa: BLE001 - network best-effort @@ -185,17 +197,17 @@ def main(): ) repo_path = _repo_path_from_env() - result = match_known_issues( - groups, repo_path, token, label=args.label, min_confidence=args.min_confidence - ) + result = match_known_issues(groups, repo_path, token, label=args.label, min_confidence=args.min_confidence) args.output.parent.mkdir(parents=True, exist_ok=True) with open(args.output, "w", encoding="utf-8") as f: json.dump(result, f, indent=2) matched_groups = sum(1 for item in result.get("per_group_matches", []) if item.get("matches")) - print(f"known_issue_matcher: checked {result['checked']} issues, " - f"{matched_groups} groups with matches >= {args.min_confidence}") + print( + f"known_issue_matcher: checked {result['checked']} issues, " + f"{matched_groups} groups with matches >= {args.min_confidence}" + ) if __name__ == "__main__": diff --git a/.azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py b/.azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py index 333bdf512..99da3e5a1 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/merge_failure_context.py @@ -6,11 +6,32 @@ from pathlib import Path ENV_SIGNAL_PATTERNS: list[tuple[str, re.Pattern]] = [ - ("network_timeout", re.compile(r"(connection\s+timed?\s*out|read\s+timed?\s*out|max\s+retries\s+exceeded|ECONNRESET|ECONNREFUSED|ETIMEDOUT)", re.IGNORECASE)), + ( + "network_timeout", + re.compile( + r"(connection\s+timed?\s*out|read\s+timed?\s*out|max\s+retries\s+exceeded|ECONNRESET|ECONNREFUSED|ETIMEDOUT)", + re.IGNORECASE, + ), + ), ("disk_full", re.compile(r"(no\s+space\s+left\s+on\s+device|disk\s+quota\s+exceeded|ENOSPC)", re.IGNORECASE)), - ("runner_lost", re.compile(r"(lost\s+communication\s+with\s+the\s+agent|runner\s+has\s+been\s+lost|did\s+not\s+respond)", re.IGNORECASE)), - ("out_of_memory", re.compile(r"(out\s+of\s+memory|cannot\s+allocate\s+memory|oom[\s-]?kill|std::bad_alloc|MemoryError)", re.IGNORECASE)), - ("hf_download_error", re.compile(r"(HfHubHTTPError|huggingface\.co|couldn'?t\s+connect\s+to\s+['\"]?https?://huggingface)", re.IGNORECASE)), + ( + "runner_lost", + re.compile( + r"(lost\s+communication\s+with\s+the\s+agent|runner\s+has\s+been\s+lost|did\s+not\s+respond)", re.IGNORECASE + ), + ), + ( + "out_of_memory", + re.compile( + r"(out\s+of\s+memory|cannot\s+allocate\s+memory|oom[\s-]?kill|std::bad_alloc|MemoryError)", re.IGNORECASE + ), + ), + ( + "hf_download_error", + re.compile( + r"(HfHubHTTPError|huggingface\.co|couldn'?t\s+connect\s+to\s+['\"]?https?://huggingface)", re.IGNORECASE + ), + ), ("rate_limited", re.compile(r"(rate\s+limit\s+exceeded|too\s+many\s+requests|HTTP\s+429)", re.IGNORECASE)), ] @@ -210,7 +231,9 @@ def merge_contexts(input_root: Path) -> list[dict]: def main(): parser = argparse.ArgumentParser(description="Merge failure context files from all test parts") - parser.add_argument("--input-root", required=True, type=Path, help="Root folder containing downloaded failure artifacts") + parser.add_argument( + "--input-root", required=True, type=Path, help="Root folder containing downloaded failure artifacts" + ) parser.add_argument("--output", required=True, type=Path, help="Merged failure context JSON path") args = parser.parse_args() @@ -223,7 +246,9 @@ def main(): "schema_version": "1.0", "build": { "build_id": os.environ.get("BUILD_BUILDID", ""), - "source_commit": os.environ.get("SYSTEM_PULLREQUEST_SOURCECOMMITID", os.environ.get("BUILD_SOURCEVERSION", "")), + "source_commit": os.environ.get( + "SYSTEM_PULLREQUEST_SOURCECOMMITID", os.environ.get("BUILD_SOURCEVERSION", "") + ), "pr_number": os.environ.get("SYSTEM_PULLREQUEST_PULLREQUESTNUMBER", ""), }, "stats": { diff --git a/.azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py b/.azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py index bbe3f56c1..a7c9ffb5b 100644 --- a/.azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py +++ b/.azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py @@ -107,8 +107,7 @@ def build_category_section(classification: dict, analysis: dict) -> list[str]: body.append(f"A CI admin can help merge this PR manually. {admin}") elif category == "Environment": body.append( - "This looks like an environment/infrastructure issue and is likely " - "not caused by the PR code changes." + "This looks like an environment/infrastructure issue and is likely " "not caused by the PR code changes." ) elif category == "Dependency": body.append( @@ -141,9 +140,7 @@ def build_category_section(classification: dict, analysis: dict) -> list[str]: else: body.append("- No patch was generated in this run.") else: # Other - body.append( - "This failure could not be confidently classified and needs manual review." - ) + body.append("This failure could not be confidently classified and needs manual review.") if admin: body.append(admin) diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.py index e0b2c18ab..0dc660afa 100644 --- a/.azure-pipelines/scripts/ut/collect_result.py +++ b/.azure-pipelines/scripts/ut/collect_result.py @@ -244,7 +244,9 @@ def write( "build": { "build_id": os.environ.get("BUILD_BUILDID", ""), "build_number": os.environ.get("BUILD_BUILDNUMBER", ""), - "source_commit": os.environ.get("SYSTEM_PULLREQUEST_SOURCECOMMITID", os.environ.get("BUILD_SOURCEVERSION", "")), + "source_commit": os.environ.get( + "SYSTEM_PULLREQUEST_SOURCECOMMITID", os.environ.get("BUILD_SOURCEVERSION", "") + ), "pr_number": os.environ.get("SYSTEM_PULLREQUEST_PULLREQUESTNUMBER", ""), }, "stats": { From c06e89ba163a85af0b7c94b7f3157ee50b41a3cb Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 9 Jul 2026 11:33:36 +0800 Subject: [PATCH 07/20] update dict Signed-off-by: chensuyue --- .azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt | 3 ++- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt b/.azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt index c829fc71b..838f22897 100644 --- a/.azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt +++ b/.azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt @@ -1,3 +1,4 @@ endianess MOT -MoT \ No newline at end of file +MoT +mis \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index e1978e006..390c320d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ ba = "ba" ue = "ue" endianess = "endianess" thw = "thw" +mis = "mis" [tool.ruff] # Exclude a variety of commonly ignored directories. From 99b0b84a9749d2d859823cf3d3645d9106ce6613 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 9 Jul 2026 11:59:49 +0800 Subject: [PATCH 08/20] update scripts Signed-off-by: chensuyue --- .azure-pipelines/scripts/ut/run_ut.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh index 882279fec..fdf88b28b 100644 --- a/.azure-pipelines/scripts/ut/run_ut.sh +++ b/.azure-pipelines/scripts/ut/run_ut.sh @@ -115,7 +115,6 @@ function run_test_cases() { fi cd /auto-round || exit 1 - auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])') for test_case in "${tests[@]}"; do if [[ -z "${test_case}" ]]; then @@ -130,8 +129,7 @@ function run_test_cases() { echo "##[group]Running ${test_case}..." numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \ - python -m pytest --cov="${auto_round_path}" --cov-report term --html=report.html --self-contained-html \ - --cov-report xml:coverage.xml --cov-append \ + python -m pytest --cov=auto_round --cov-report= --cov-append \ -vs --disable-warnings "${test_case}" 2>&1 | tee "${ut_log_name}" echo "##[endgroup]" done From 3b9d1e8fe0409005f6c0f76f3903dd966a069cb5 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 9 Jul 2026 12:00:14 +0800 Subject: [PATCH 09/20] for debug Signed-off-by: chensuyue --- .azure-pipelines/scripts/ut/run_ut.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh index fdf88b28b..e40aa9003 100644 --- a/.azure-pipelines/scripts/ut/run_ut.sh +++ b/.azure-pipelines/scripts/ut/run_ut.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -ex test_part="" failure_log_context="" From 2e28fd264a930b495a89c1a273b8423e04096399 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 9 Jul 2026 14:20:18 +0800 Subject: [PATCH 10/20] refactor: remove ci_part and failure_log_context parameters; update failure log directory handling Signed-off-by: chensuyue --- .azure-pipelines/scripts/ut/collect_result.py | 12 ++--- .azure-pipelines/scripts/ut/run_ut.sh | 44 +++---------------- .azure-pipelines/template/ut-template.yml | 32 ++++++-------- .azure-pipelines/unit-test.yml | 1 - 4 files changed, 22 insertions(+), 67 deletions(-) diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.py index 0dc660afa..06950d2ce 100644 --- a/.azure-pipelines/scripts/ut/collect_result.py +++ b/.azure-pipelines/scripts/ut/collect_result.py @@ -227,7 +227,6 @@ def write( output_path: Path, results: list[TestResult], test_type: str, - ci_part: str = "", summary_log: Path | None = None, failure_log_dir: Path | None = None, ) -> None: @@ -240,7 +239,6 @@ def write( payload = { "schema_version": "1.0", "test_type": test_type, - "ci_part": ci_part, "build": { "build_id": os.environ.get("BUILD_BUILDID", ""), "build_number": os.environ.get("BUILD_BUILDNUMBER", ""), @@ -261,7 +259,7 @@ def write( print(f"Failure context written to: {output_path.absolute()}", file=sys.stderr) - target_log_dir = Path(failure_log_dir) if failure_log_dir else output_path.parent / "failure_logs" + target_log_dir = Path(failure_log_dir) if failure_log_dir else output_path.parent / "failure_logs_dir" self._collect_failure_logs(target_log_dir, failed_results, output_path, summary_log) def _to_failure_entry(self, result: TestResult) -> dict: @@ -347,9 +345,8 @@ def main(): parser.add_argument("--log-dir", required=True, type=Path, help="Directory with logs") parser.add_argument("--summary-log", required=True, type=Path, help="Output file") parser.add_argument("--log-pattern", default="*.log", help="Glob pattern") - parser.add_argument("--failure-context", type=Path, help="Optional output file for failure context JSON") + parser.add_argument("--failure-context-file", type=Path, help="Optional output file for failure context JSON") parser.add_argument("--failure-context-max-lines", type=int, default=200, help="Max lines per failed case") - parser.add_argument("--ci-part", default="", help="Optional CI matrix part label") parser.add_argument("--failure-log-dir", type=Path, help="Optional output folder for failed logs package") args = parser.parse_args() @@ -371,13 +368,12 @@ def main(): "failed": sum(1 for r in results if r.status == TestStatus.FAILED), } - if args.failure_context: + if args.failure_context_file and stats['failed'] > 0: context_writer = FailureContextWriter(args.log_dir, max_lines=args.failure_context_max_lines) context_writer.write( - args.failure_context, + args.failure_context_file, results, test_type=args.test_type, - ci_part=args.ci_part, summary_log=args.summary_log, failure_log_dir=args.failure_log_dir, ) diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh index e40aa9003..dc4009567 100644 --- a/.azure-pipelines/scripts/ut/run_ut.sh +++ b/.azure-pipelines/scripts/ut/run_ut.sh @@ -1,44 +1,13 @@ #!/bin/bash set -ex -test_part="" -failure_log_context="" -declare -a FAILED_BASE_CASES=() -declare -a FAILED_INC_CASES=() -declare -a FAILED_LLMC_CASES=() - -function parse_arguments() { - - while [[ $# -gt 0 ]]; do - case "$1" in - --failure-context) - failure_log_context="$2" - shift 2 - ;; - --test-part) - test_part="$2" - shift 2 - ;; - *) - echo "Unknown argument: $1" - exit 1 - ;; - esac - done - - if [[ -z "${test_part}" ]]; then - echo "Error: test_part is required" - echo "Usage: run_ut.sh --test-part [--failure-context ] [--failed-test-cases ]" - exit 1 - fi -} - -parse_arguments "$@" - +test_part=$1 source /auto-round/.azure-pipelines/scripts/change_color.sh LOG_DIR=/auto-round/log_dir +FAILURE_LOG_DIR=/auto-round/log_dir/failure_logs_dir mkdir -p "${LOG_DIR}" +mkdir -p "${FAILURE_LOG_DIR}" SUMMARY_LOG="${LOG_DIR}/results_summary.log" function setup_inc_environment() { @@ -185,13 +154,10 @@ function collect_log() { --log-pattern "unittest_test_*.log" --log-dir "${LOG_DIR}" --summary-log "${SUMMARY_LOG}" - --ci-part "${test_part}" + --failure-log-dir "${FAILURE_LOG_DIR}" + --failure-context-file "${FAILURE_LOG_DIR}/failure_context_part${test_part}.json" ) - if [[ -n "${failure_log_context}" ]]; then - collect_cmd+=(--failure-context "${failure_log_context}") - fi - "${collect_cmd[@]}" if [[ -f .coverage ]]; then diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml index 6ee3fab63..1b96fa41f 100644 --- a/.azure-pipelines/template/ut-template.yml +++ b/.azure-pipelines/template/ut-template.yml @@ -32,9 +32,6 @@ parameters: - name: buildARKWheel type: string default: "false" - - name: failureLogContext - type: string - default: "" - name: failureLogArtifactPrefix type: string default: "ut-failure-log" @@ -106,14 +103,9 @@ steps: - script: | set -eo pipefail - EXTRA_ARGS="" - if [ -n "${{ parameters.failureLogContext }}" ]; then - EXTRA_ARGS="--failure-context ${{ parameters.failureLogContext }}" - fi - docker exec -e NUMA_NODE=${NUMA_NODE} -e NUMA_CPUSET=${NUMA_CPUSET} -e ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK} ${{ parameters.utContainerName }} \ bash -c "cd /auto-round/.azure-pipelines/scripts \ - && bash ut/${{ parameters.utScriptFileName }}.sh --test-part ${{ parameters.utTestMode }} ${EXTRA_ARGS}" + && bash ut/${{ parameters.utScriptFileName }}.sh ${{ parameters.utTestMode }}" displayName: "Run UT" - task: PublishPipelineArtifact@1 @@ -123,19 +115,21 @@ steps: artifact: ${{ parameters.utArtifact }}_coverage publishLocation: "pipeline" displayName: "Publish UT coverage artifact" - - - task: PublishPipelineArtifact@1 + + - task: Bash@3 condition: failed() - inputs: - targetPath: ${{ parameters.uploadPath }} - artifact: "${{ parameters.utArtifact }}-$(System.JobAttempt)" - publishLocation: "pipeline" - displayName: "Publish UT logs for debugging" - + script: | + if [ -d "${{ parameters.uploadPath }}/failure_logs_dir" ]; then + echo "##vso[task.setvariable variable=FailureLogsExist]true" + else + echo "##vso[task.setvariable variable=FailureLogsExist]false" + fi + displayName: "Check failure logs directory" + - task: PublishPipelineArtifact@1 - condition: failed() + condition: and(failed(), eq(variables['FailureLogsExist'], 'true')) inputs: - targetPath: ${{ parameters.uploadPath }}/failure_logs + targetPath: ${{ parameters.uploadPath }}/failure_logs_dir artifact: "${{ parameters.failureLogArtifactPrefix }}-${{ parameters.utTestMode }}-$(System.JobAttempt)" publishLocation: "pipeline" diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.yml index 8395e20aa..146bf8869 100644 --- a/.azure-pipelines/unit-test.yml +++ b/.azure-pipelines/unit-test.yml @@ -83,7 +83,6 @@ stages: utTestMode: $(PART) utContainerName: "AutoRoundUnitTest$(NODE_LABEL)" buildARKWheel: $(BUILD_ARK_WHEEL) - failureLogContext: $(UPLOAD_PATH)/failure_context_part$(PART).json - stage: AI_Analysis displayName: "AI Analyze" From c40ef7fe61e5c59a54bd144e33a7b9c1001b2132 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 9 Jul 2026 14:22:20 +0800 Subject: [PATCH 11/20] test classify first Signed-off-by: chensuyue --- .../template/ai-analysis-template.yml | 66 +++++++++---------- .azure-pipelines/unit-test.yml | 1 - 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/.azure-pipelines/template/ai-analysis-template.yml b/.azure-pipelines/template/ai-analysis-template.yml index 43a927185..c9a6a1375 100644 --- a/.azure-pipelines/template/ai-analysis-template.yml +++ b/.azure-pipelines/template/ai-analysis-template.yml @@ -69,42 +69,42 @@ jobs: CI_ADMIN_HANDLE: $(CI_ADMIN_HANDLE) displayName: "Classify Failure" - - script: | - set -euo pipefail - cd ${BUILD_SOURCESDIRECTORY} + # - script: | + # set -euo pipefail + # cd ${BUILD_SOURCESDIRECTORY} - if ! command -v copilot >/dev/null 2>&1; then - if ! command -v npm >/dev/null 2>&1; then - echo "npm is required to install Copilot CLI" - exit 1 - fi - npm install -g @github/copilot - fi - copilot --version + # if ! command -v copilot >/dev/null 2>&1; then + # if ! command -v npm >/dev/null 2>&1; then + # echo "npm is required to install Copilot CLI" + # exit 1 + # fi + # npm install -g @github/copilot + # fi + # copilot --version - python .azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py \ - --failure-context "${{ parameters.uploadPath }}/merged_failure_context.json" \ - --output-dir "${{ parameters.uploadPath }}/ai_analysis" \ - --project-root "${BUILD_SOURCESDIRECTORY}" \ - --failed-logs-root "${{ parameters.failureLogDownloadPath }}" \ - --classification-result "${{ parameters.uploadPath }}/ai_analysis/classification_result.json" - condition: and(succeeded(), eq(variables['CLASSIFICATION'], 'Code Regression')) - env: - COPILOT_GITHUB_TOKEN: $(GITHUB_TOKEN) - GITHUB_TOKEN: $(GITHUB_TOKEN) - GH_TOKEN: $(GITHUB_TOKEN) - displayName: "Analyze Code Regression (no auto-apply)" + # python .azure-pipelines/scripts/ai_failure_analysis/analyze_and_suggest.py \ + # --failure-context "${{ parameters.uploadPath }}/merged_failure_context.json" \ + # --output-dir "${{ parameters.uploadPath }}/ai_analysis" \ + # --project-root "${BUILD_SOURCESDIRECTORY}" \ + # --failed-logs-root "${{ parameters.failureLogDownloadPath }}" \ + # --classification-result "${{ parameters.uploadPath }}/ai_analysis/classification_result.json" + # condition: and(succeeded(), eq(variables['CLASSIFICATION'], 'Code Regression')) + # env: + # COPILOT_GITHUB_TOKEN: $(GITHUB_TOKEN) + # GITHUB_TOKEN: $(GITHUB_TOKEN) + # GH_TOKEN: $(GITHUB_TOKEN) + # displayName: "Analyze Code Regression (no auto-apply)" - - script: | - cd ${BUILD_SOURCESDIRECTORY} - python .azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py \ - --classification-result "${{ parameters.uploadPath }}/ai_analysis/classification_result.json" \ - --analysis-result "${{ parameters.uploadPath }}/ai_analysis/analysis_result.json" \ - --report "${{ parameters.uploadPath }}/ai_analysis/ai_failure_report.md" - condition: succeededOrFailed() - env: - GITHUB_TOKEN: $(GITHUB_TOKEN) - displayName: "Post Copilot Analysis Comment" + # - script: | + # cd ${BUILD_SOURCESDIRECTORY} + # python .azure-pipelines/scripts/ai_failure_analysis/post_pr_comment.py \ + # --classification-result "${{ parameters.uploadPath }}/ai_analysis/classification_result.json" \ + # --analysis-result "${{ parameters.uploadPath }}/ai_analysis/analysis_result.json" \ + # --report "${{ parameters.uploadPath }}/ai_analysis/ai_failure_report.md" + # condition: succeededOrFailed() + # env: + # GITHUB_TOKEN: $(GITHUB_TOKEN) + # displayName: "Post Copilot Analysis Comment" - task: PublishPipelineArtifact@1 condition: succeededOrFailed() diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.yml index 146bf8869..f322db494 100644 --- a/.azure-pipelines/unit-test.yml +++ b/.azure-pipelines/unit-test.yml @@ -88,7 +88,6 @@ stages: displayName: "AI Analyze" dependsOn: [Unit_test] condition: and(failed('Unit_test'), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['COPILOT_ANALYSIS_ENABLED'], 'true')) - pool: ICX-16C jobs: - template: template/ai-analysis-template.yml parameters: From c373fc31282a2ba20fddd4d42d3cd571db29e3f5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 06:24:06 +0000 Subject: [PATCH 12/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .azure-pipelines/scripts/ut/collect_result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.py index 06950d2ce..e9c69b3ec 100644 --- a/.azure-pipelines/scripts/ut/collect_result.py +++ b/.azure-pipelines/scripts/ut/collect_result.py @@ -368,7 +368,7 @@ def main(): "failed": sum(1 for r in results if r.status == TestStatus.FAILED), } - if args.failure_context_file and stats['failed'] > 0: + if args.failure_context_file and stats["failed"] > 0: context_writer = FailureContextWriter(args.log_dir, max_lines=args.failure_context_max_lines) context_writer.write( args.failure_context_file, From c51cda6f63b831188f3b163c03d525208269bd50 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 9 Jul 2026 14:30:59 +0800 Subject: [PATCH 13/20] bug fix Signed-off-by: chensuyue --- .azure-pipelines/template/ut-template.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml index 1b96fa41f..53cd60f6f 100644 --- a/.azure-pipelines/template/ut-template.yml +++ b/.azure-pipelines/template/ut-template.yml @@ -116,15 +116,14 @@ steps: publishLocation: "pipeline" displayName: "Publish UT coverage artifact" - - task: Bash@3 - condition: failed() - script: | + - script: | if [ -d "${{ parameters.uploadPath }}/failure_logs_dir" ]; then echo "##vso[task.setvariable variable=FailureLogsExist]true" else echo "##vso[task.setvariable variable=FailureLogsExist]false" fi displayName: "Check failure logs directory" + condition: failed() - task: PublishPipelineArtifact@1 condition: and(failed(), eq(variables['FailureLogsExist'], 'true')) From 221109262a3f9d3daa41185ed955df166ecdcbc2 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 9 Jul 2026 14:49:12 +0800 Subject: [PATCH 14/20] refactor: remove summary_log parameter from FailureContextWriter and related methods Signed-off-by: chensuyue --- .azure-pipelines/scripts/ut/collect_result.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.py index e9c69b3ec..481fad593 100644 --- a/.azure-pipelines/scripts/ut/collect_result.py +++ b/.azure-pipelines/scripts/ut/collect_result.py @@ -227,7 +227,6 @@ def write( output_path: Path, results: list[TestResult], test_type: str, - summary_log: Path | None = None, failure_log_dir: Path | None = None, ) -> None: output_path = Path(output_path) @@ -260,7 +259,7 @@ def write( print(f"Failure context written to: {output_path.absolute()}", file=sys.stderr) target_log_dir = Path(failure_log_dir) if failure_log_dir else output_path.parent / "failure_logs_dir" - self._collect_failure_logs(target_log_dir, failed_results, output_path, summary_log) + self._collect_failure_logs(target_log_dir, failed_results) def _to_failure_entry(self, result: TestResult) -> dict: log_path = self.log_dir / result.filename @@ -319,26 +318,17 @@ def _collect_failure_logs( self, target_dir: Path, failed_results: list[TestResult], - context_path: Path, - summary_log: Path | None, ) -> None: if not failed_results: return target_dir.mkdir(parents=True, exist_ok=True) - if summary_log and Path(summary_log).exists(): - shutil.copy2(summary_log, target_dir / Path(summary_log).name) - for result in failed_results: src_log = self.log_dir / result.filename if src_log.exists(): shutil.copy2(src_log, target_dir / result.filename) - if context_path.exists(): - shutil.copy2(context_path, target_dir / context_path.name) - - def main(): parser = argparse.ArgumentParser(description="Analyze test logs and generate summary") parser.add_argument("--test-type", required=True, help="Type of tests") @@ -368,18 +358,18 @@ def main(): "failed": sum(1 for r in results if r.status == TestStatus.FAILED), } + print(f"Done: {stats['total']} tests, {stats['passed']} passed, {stats['failed']} failed") + if args.failure_context_file and stats["failed"] > 0: + print(f"Generating failure context used for AI analysis", file=sys.stderr) context_writer = FailureContextWriter(args.log_dir, max_lines=args.failure_context_max_lines) context_writer.write( args.failure_context_file, results, test_type=args.test_type, - summary_log=args.summary_log, failure_log_dir=args.failure_log_dir, ) - print(f"Done: {stats['total']} tests, {stats['passed']} passed, {stats['failed']} failed") - except Exception as e: print(f"Error: {e}", file=sys.stderr) import traceback From b53bc6882ff7e49703523ad53148402da76288e7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 06:50:27 +0000 Subject: [PATCH 15/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .azure-pipelines/scripts/ut/collect_result.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.py index 481fad593..d4efa6fc2 100644 --- a/.azure-pipelines/scripts/ut/collect_result.py +++ b/.azure-pipelines/scripts/ut/collect_result.py @@ -329,6 +329,7 @@ def _collect_failure_logs( if src_log.exists(): shutil.copy2(src_log, target_dir / result.filename) + def main(): parser = argparse.ArgumentParser(description="Analyze test logs and generate summary") parser.add_argument("--test-type", required=True, help="Type of tests") @@ -361,7 +362,7 @@ def main(): print(f"Done: {stats['total']} tests, {stats['passed']} passed, {stats['failed']} failed") if args.failure_context_file and stats["failed"] > 0: - print(f"Generating failure context used for AI analysis", file=sys.stderr) + print("Generating failure context used for AI analysis", file=sys.stderr) context_writer = FailureContextWriter(args.log_dir, max_lines=args.failure_context_max_lines) context_writer.write( args.failure_context_file, From a064c01122a691eb12296c3d7f1beacd19a98523 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 9 Jul 2026 15:48:24 +0800 Subject: [PATCH 16/20] fix: update display name for Classify job and set pool for AI Analysis stage Signed-off-by: chensuyue --- .azure-pipelines/template/ai-analysis-template.yml | 2 +- .azure-pipelines/unit-test.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/template/ai-analysis-template.yml b/.azure-pipelines/template/ai-analysis-template.yml index c9a6a1375..f7ff82644 100644 --- a/.azure-pipelines/template/ai-analysis-template.yml +++ b/.azure-pipelines/template/ai-analysis-template.yml @@ -29,7 +29,7 @@ parameters: jobs: - job: Classify - displayName: "Classify And Comment" + displayName: "Classify and Comment" timeoutInMinutes: ${{ parameters.classifyTimeoutInMinutes }} steps: - checkout: self diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.yml index f322db494..c5676ef5c 100644 --- a/.azure-pipelines/unit-test.yml +++ b/.azure-pipelines/unit-test.yml @@ -86,6 +86,8 @@ stages: - stage: AI_Analysis displayName: "AI Analyze" + pool: + vmImage: "ubuntu-latest" dependsOn: [Unit_test] condition: and(failed('Unit_test'), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['COPILOT_ANALYSIS_ENABLED'], 'true')) jobs: From db7e78d284329e4e6bf35bd5cadd65622443b57f Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 9 Jul 2026 21:53:45 +0800 Subject: [PATCH 17/20] remove tail in context json file Signed-off-by: chensuyue --- .azure-pipelines/scripts/ut/collect_result.py | 4 +--- .azure-pipelines/scripts/ut/run_ut.sh | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.py index d4efa6fc2..a466bf215 100644 --- a/.azure-pipelines/scripts/ut/collect_result.py +++ b/.azure-pipelines/scripts/ut/collect_result.py @@ -266,7 +266,6 @@ def _to_failure_entry(self, result: TestResult) -> dict: content = self._read_file(log_path) lines = content.splitlines() excerpt = self._extract_excerpt(lines) - tail = "\n".join(lines[-self.max_lines :]) if lines else "" return { "test_name": result.name, @@ -274,7 +273,6 @@ def _to_failure_entry(self, result: TestResult) -> dict: "log_file": result.filename, "duration": result.duration, "excerpt": excerpt, - "tail": tail, } def _read_file(self, path: Path) -> str: @@ -302,7 +300,7 @@ def _extract_excerpt(self, lines: list[str]) -> str: break end = min(max_end, failed_end) if failed_end is not None else max_end else: - start = max(0, idx - 10) + start = max(0, idx) end = min(len(lines), idx + 80) selected = lines[start:end] break diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh index dc4009567..740abff2f 100644 --- a/.azure-pipelines/scripts/ut/run_ut.sh +++ b/.azure-pipelines/scripts/ut/run_ut.sh @@ -110,7 +110,7 @@ function run_unit_test() { cd /auto-round || exit 1 # Split test files into 5 parts - find ./test/test_cpu/core -name "test*.py" | grep -Ev "test_llmc|test_inc" | sort > all_tests.txt + find ./test/test_cpu -name "test*.py" | grep -Ev "test_llmc|test_inc" | sort > all_tests.txt total_lines=$(wc -l < all_tests.txt) NUM_CHUNKS=5 q=$(( total_lines / NUM_CHUNKS )) From 10375f85fce071282c42d0ba4d51e3d5352f1156 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Fri, 10 Jul 2026 11:21:28 +0800 Subject: [PATCH 18/20] fix: adjust line selection logic in FailureContextWriter to prevent index errors Signed-off-by: chensuyue --- .azure-pipelines/scripts/ut/collect_result.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.py index a466bf215..3dda390ea 100644 --- a/.azure-pipelines/scripts/ut/collect_result.py +++ b/.azure-pipelines/scripts/ut/collect_result.py @@ -292,23 +292,24 @@ def _extract_excerpt(self, lines: list[str]) -> str: if marker in line: if marker == "Traceback": start = max(0, idx - 10) - max_end = min(len(lines), idx + 80) + max_end = min(len(lines) - 1, idx + 80) + scan_end = min(len(lines), idx + 80) failed_end = None - for j in range(idx, max_end): + for j in range(idx, scan_end): if "FAILED" in lines[j]: failed_end = j + 1 break end = min(max_end, failed_end) if failed_end is not None else max_end else: start = max(0, idx) - end = min(len(lines), idx + 80) + end = min(len(lines)-1, idx + 80) selected = lines[start:end] break if selected: break if not selected: - selected = lines[-self.max_lines :] + selected = lines[-self.max_lines : -1] return "\n".join(selected[: self.max_lines]) From 0e97dbe79c60ef2eb566bad9b482849e7702b84b Mon Sep 17 00:00:00 2001 From: chensuyue Date: Fri, 10 Jul 2026 11:24:14 +0800 Subject: [PATCH 19/20] remove version limit to trigger more issues Signed-off-by: chensuyue --- test/test_cpu/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_cpu/requirements.txt b/test/test_cpu/requirements.txt index fd34e8a1e..adb8bff75 100644 --- a/test/test_cpu/requirements.txt +++ b/test/test_cpu/requirements.txt @@ -10,4 +10,4 @@ lm_eval>=0.4.10 # for transformers >= 5.0.0 diffusers protobuf compressed-tensors>=0.15.0 -transformers>=5.5.0,<=5.12.1 +transformers>=5.5.0 From 22565714beffe218ad1e99d3fedf8ad189aeb517 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 03:25:07 +0000 Subject: [PATCH 20/20] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .azure-pipelines/scripts/ut/collect_result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.py index 3dda390ea..30457ce43 100644 --- a/.azure-pipelines/scripts/ut/collect_result.py +++ b/.azure-pipelines/scripts/ut/collect_result.py @@ -302,7 +302,7 @@ def _extract_excerpt(self, lines: list[str]) -> str: end = min(max_end, failed_end) if failed_end is not None else max_end else: start = max(0, idx) - end = min(len(lines)-1, idx + 80) + end = min(len(lines) - 1, idx + 80) selected = lines[start:end] break if selected: