diff --git a/.gitignore b/.gitignore index 3072c1ffe..68b274472 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ toc/elections/2021/private.csv __pycache__ orgs.out.yml branchprotection.out.yml -/.idea \ No newline at end of file +/.idea +tmp/ \ No newline at end of file diff --git a/README.md b/README.md index 9a6a7b18e..7505417f3 100644 --- a/README.md +++ b/README.md @@ -39,3 +39,11 @@ The evolution of the Cloud Foundry Technical Community Governance is explained i The technical community's activities are structured into working groups by the TOC. For a listing of the active working groups, see [WORKING-GROUPS.md](toc/working-groups/WORKING-GROUPS.md). + +### Working Group Activity Reports + +Automated working group activity reports are generated quarterly for TOC review. These reports analyze GitHub activity across all repositories managed by each working group and provide strategic insights into development progress. + +- **View Reports**: [toc/working-groups/updates/](toc/working-groups/updates/) +- **Generate Reports**: See [automation documentation](toc/working-groups/updates/README.md) +- **OpenCode Integration**: Supports automated report generation using [OpenCode Run](https://github.com/sst/opencode) diff --git a/scripts/extract_wg_activity.py b/scripts/extract_wg_activity.py new file mode 100755 index 000000000..0b5fe41da --- /dev/null +++ b/scripts/extract_wg_activity.py @@ -0,0 +1,719 @@ +#!/usr/bin/env python3 +""" +Extract raw repository activity data for Cloud Foundry working groups using GitHub GraphQL API. +This script extracts pure activity data without analysis or formatting. +""" + +import yaml +import json +import subprocess +import sys +import argparse +from datetime import datetime, timedelta +from pathlib import Path +import re +from dateutil import parser as date_parser + +def extract_repos_from_charter(charter_file): + """Extract repository information from a working group charter file.""" + with open(charter_file, 'r') as f: + content = f.read() + + # Try YAML frontmatter first (new format) + if content.startswith('---'): + parts = content.split('---', 2) + if len(parts) >= 3: + yaml_content = parts[1] + try: + metadata = yaml.safe_load(yaml_content) + # Convert from frontmatter format to old config format + if 'repositories' in metadata: + config = { + 'name': metadata.get('name', charter_file.split('/')[-1].replace('.md', '')), + 'areas': [ + { + 'name': 'main', + 'repositories': metadata['repositories'] + } + ] + } + return config + except yaml.YAMLError as e: + print(f"Error parsing YAML frontmatter: {e}") + + # Fall back to old YAML block format + yaml_match = re.search(r'```yaml\n(.*?)\n```', content, re.DOTALL) + if not yaml_match: + print(f"No YAML configuration found in {charter_file}") + return None + + try: + config = yaml.safe_load(yaml_match.group(1)) + return config + except yaml.YAMLError as e: + print(f"Error parsing YAML: {e}") + return None + +def get_repo_activity_graphql(repo, since_date): + """Get raw repository activity using GitHub GraphQL API.""" + org, name = repo.split('/') + since_iso = since_date.isoformat() + + activity = { + 'repository': repo, + 'commits': [], + 'pull_requests': [], + 'issues': [], + 'releases': [] + } + + print(f" Fetching activity for {repo} since {since_date.strftime('%Y-%m-%d')}...") + + try: + # GraphQL query for commits + commits_query = f''' + {{ + repository(owner: "{org}", name: "{name}") {{ + defaultBranchRef {{ + target {{ + ... on Commit {{ + history(first: 100, since: "{since_iso}") {{ + edges {{ + node {{ + oid + messageHeadline + messageBody + author {{ + name + email + date + }} + url + associatedPullRequests(first: 5) {{ + edges {{ + node {{ + number + title + url + }} + }} + }} + }} + }} + }} + }} + }} + }} + }} + }} + ''' + + result = subprocess.run(['gh', 'api', 'graphql', '--cache', '1h', '-f', f'query={commits_query}'], + capture_output=True, text=True, timeout=30) + if result.returncode == 0: + data = json.loads(result.stdout) + if data.get('data', {}).get('repository', {}).get('defaultBranchRef'): + commits = data['data']['repository']['defaultBranchRef']['target']['history']['edges'] + for edge in commits: + commit = edge['node'] + associated_prs = [pr_edge['node'] for pr_edge in commit.get('associatedPullRequests', {}).get('edges', [])] + activity['commits'].append({ + 'sha': commit['oid'], + 'message': commit['messageHeadline'], + 'body': commit.get('messageBody', ''), + 'author': commit['author']['name'], + 'email': commit['author'].get('email', ''), + 'date': commit['author']['date'], + 'url': commit['url'], + 'associated_prs': associated_prs + }) + + # GraphQL query for pull requests + prs_query = f''' + {{ + repository(owner: "{org}", name: "{name}") {{ + pullRequests(first: 100, orderBy: {{field: UPDATED_AT, direction: DESC}}) {{ + edges {{ + node {{ + number + title + state + createdAt + updatedAt + closedAt + mergedAt + author {{ + login + }} + url + body + labels(first: 20) {{ + edges {{ + node {{ + name + }} + }} + }} + milestone {{ + title + }} + assignees(first: 10) {{ + edges {{ + node {{ + login + }} + }} + }} + reviews(first: 20) {{ + edges {{ + node {{ + state + author {{ + login + }} + }} + }} + }} + comments(first: 20) {{ + edges {{ + node {{ + body + author {{ + login + }} + createdAt + }} + }} + }} + }} + }} + }} + }} + }} + ''' + + result = subprocess.run(['gh', 'api', 'graphql', '--cache', '1h', '-f', f'query={prs_query}'], + capture_output=True, text=True, timeout=30) + if result.returncode == 0: + data = json.loads(result.stdout) + if data.get('data', {}).get('repository', {}).get('pullRequests'): + prs = data['data']['repository']['pullRequests']['edges'] + for edge in prs: + pr = edge['node'] + # Filter by date + try: + created_date = date_parser.parse(pr['createdAt']).replace(tzinfo=None) + updated_date = date_parser.parse(pr['updatedAt']).replace(tzinfo=None) + + if created_date >= since_date or updated_date >= since_date: + labels = [label_edge['node']['name'] for label_edge in pr.get('labels', {}).get('edges', [])] + assignees = [assignee_edge['node']['login'] for assignee_edge in pr.get('assignees', {}).get('edges', [])] + reviews = [{'state': review_edge['node']['state'], 'author': review_edge['node']['author']['login'] if review_edge['node']['author'] else None} + for review_edge in pr.get('reviews', {}).get('edges', [])] + comments = [{'body': comment_edge['node']['body'], 'author': comment_edge['node']['author']['login'] if comment_edge['node']['author'] else None, 'date': comment_edge['node']['createdAt']} + for comment_edge in pr.get('comments', {}).get('edges', [])] + + activity['pull_requests'].append({ + 'number': pr['number'], + 'title': pr['title'], + 'state': pr['state'], + 'created_at': pr['createdAt'], + 'updated_at': pr['updatedAt'], + 'closed_at': pr.get('closedAt'), + 'merged_at': pr.get('mergedAt'), + 'user': pr['author']['login'] if pr['author'] else None, + 'url': pr['url'], + 'body': pr['body'], + 'labels': labels, + 'milestone': pr.get('milestone', {}).get('title') if pr.get('milestone') else None, + 'assignees': assignees, + 'reviews': reviews, + 'comments': comments + }) + except Exception as e: + print(f" Date parsing error for PR in {repo}: {e}") + + # GraphQL query for issues + issues_query = f''' + {{ + repository(owner: "{org}", name: "{name}") {{ + issues(first: 100, orderBy: {{field: UPDATED_AT, direction: DESC}}) {{ + edges {{ + node {{ + number + title + state + createdAt + updatedAt + closedAt + author {{ + login + }} + url + body + labels(first: 20) {{ + edges {{ + node {{ + name + }} + }} + }} + milestone {{ + title + }} + assignees(first: 10) {{ + edges {{ + node {{ + login + }} + }} + }} + comments(first: 20) {{ + edges {{ + node {{ + body + author {{ + login + }} + createdAt + }} + }} + }} + }} + }} + }} + }} + }} + ''' + + result = subprocess.run(['gh', 'api', 'graphql', '--cache', '1h', '-f', f'query={issues_query}'], + capture_output=True, text=True, timeout=30) + if result.returncode == 0: + data = json.loads(result.stdout) + if data.get('data', {}).get('repository', {}).get('issues'): + issues = data['data']['repository']['issues']['edges'] + for edge in issues: + issue = edge['node'] + # Filter by date + try: + created_date = date_parser.parse(issue['createdAt']).replace(tzinfo=None) + updated_date = date_parser.parse(issue['updatedAt']).replace(tzinfo=None) + + if created_date >= since_date or updated_date >= since_date: + labels = [label_edge['node']['name'] for label_edge in issue.get('labels', {}).get('edges', [])] + assignees = [assignee_edge['node']['login'] for assignee_edge in issue.get('assignees', {}).get('edges', [])] + comments = [{'body': comment_edge['node']['body'], 'author': comment_edge['node']['author']['login'] if comment_edge['node']['author'] else None, 'date': comment_edge['node']['createdAt']} + for comment_edge in issue.get('comments', {}).get('edges', [])] + + activity['issues'].append({ + 'number': issue['number'], + 'title': issue['title'], + 'state': issue['state'], + 'created_at': issue['createdAt'], + 'updated_at': issue['updatedAt'], + 'closed_at': issue.get('closedAt'), + 'user': issue['author']['login'] if issue['author'] else None, + 'url': issue['url'], + 'body': issue['body'], + 'labels': labels, + 'milestone': issue.get('milestone', {}).get('title') if issue.get('milestone') else None, + 'assignees': assignees, + 'comments': comments + }) + except Exception as e: + print(f" Date parsing error for issue in {repo}: {e}") + + # GraphQL query for releases + releases_query = f''' + {{ + repository(owner: "{org}", name: "{name}") {{ + releases(first: 50, orderBy: {{field: CREATED_AT, direction: DESC}}) {{ + edges {{ + node {{ + tagName + name + createdAt + publishedAt + author {{ + login + }} + url + isPrerelease + isDraft + description + releaseAssets(first: 10) {{ + edges {{ + node {{ + name + downloadCount + }} + }} + }} + }} + }} + }} + }} + }} + ''' + + result = subprocess.run(['gh', 'api', 'graphql', '--cache', '1h', '-f', f'query={releases_query}'], + capture_output=True, text=True, timeout=30) + if result.returncode == 0: + data = json.loads(result.stdout) + if data.get('data', {}).get('repository', {}).get('releases'): + releases = data['data']['repository']['releases']['edges'] + for edge in releases: + release = edge['node'] + # Filter by date + try: + created_date = date_parser.parse(release['createdAt']).replace(tzinfo=None) + + if created_date >= since_date: + assets = [{'name': asset_edge['node']['name'], 'downloads': asset_edge['node']['downloadCount']} + for asset_edge in release.get('releaseAssets', {}).get('edges', [])] + + activity['releases'].append({ + 'tag_name': release['tagName'], + 'name': release['name'], + 'created_at': release['createdAt'], + 'published_at': release.get('publishedAt'), + 'author': release['author']['login'] if release['author'] else None, + 'url': release['url'], + 'prerelease': release['isPrerelease'], + 'draft': release['isDraft'], + 'description': release['description'], + 'assets': assets + }) + except Exception as e: + print(f" Date parsing error for release in {repo}: {e}") + + except subprocess.TimeoutExpired: + print(f" Timeout while fetching data for {repo}") + except subprocess.CalledProcessError as e: + print(f" Error fetching data for {repo}: {e}") + except json.JSONDecodeError as e: + print(f" JSON decode error for {repo}: {e}") + except Exception as e: + print(f" Unexpected error for {repo}: {e}") + + # Print activity summary for this repo + total_activity = len(activity['commits']) + len(activity['pull_requests']) + len(activity['issues']) + len(activity['releases']) + if total_activity > 0: + print(f" Found: {len(activity['commits'])} commits, {len(activity['pull_requests'])} PRs, {len(activity['issues'])} issues, {len(activity['releases'])} releases") + + return activity + +def get_community_rfcs(since_date): + """Get RFCs from cloudfoundry/community repo that are relevant to the reporting period.""" + rfcs = [] + + try: + print(" Fetching RFCs from cloudfoundry/community repository...") + + # Query for PRs and issues in community repo + prs_query = f''' + query {{ + repository(owner: "cloudfoundry", name: "community") {{ + pullRequests(first: 50, states: [OPEN, MERGED], orderBy: {{field: UPDATED_AT, direction: DESC}}) {{ + edges {{ + node {{ + number + title + state + url + createdAt + updatedAt + mergedAt + author {{ + login + }} + body + labels(first: 10) {{ + edges {{ + node {{ + name + }} + }} + }} + files(first: 10) {{ + edges {{ + node {{ + path + }} + }} + }} + }} + }} + }} + issues(first: 50, states: [OPEN, CLOSED], orderBy: {{field: UPDATED_AT, direction: DESC}}) {{ + edges {{ + node {{ + number + title + state + url + createdAt + updatedAt + closedAt + author {{ + login + }} + body + labels(first: 10) {{ + edges {{ + node {{ + name + }} + }} + }} + }} + }} + }} + }} + }} + ''' + + result = subprocess.run(['gh', 'api', 'graphql', '--cache', '1h', '-f', f'query={prs_query}'], + capture_output=True, text=True, timeout=30) + + if result.returncode == 0: + data = json.loads(result.stdout) + repo_data = data.get('data', {}).get('repository', {}) + + # Process PRs + prs = repo_data.get('pullRequests', {}).get('edges', []) + for edge in prs: + pr = edge['node'] + + # Filter by date + try: + updated_date = date_parser.parse(pr['updatedAt']).replace(tzinfo=None) + if updated_date >= since_date: + # Get labels and files + labels = [l['node']['name'] for l in pr.get('labels', {}).get('edges', [])] + files = [f['node']['path'] for f in pr.get('files', {}).get('edges', [])] + + # Check if it's RFC-related or working group related + is_rfc = (any('rfc' in file.lower() for file in files) or + 'rfc' in pr['title'].lower() or + any('rfc' in label.lower() for label in labels) or + any(label.startswith('wg-') for label in labels)) + + if is_rfc: + rfcs.append({ + 'number': pr['number'], + 'title': pr['title'], + 'state': pr['state'], + 'url': pr['url'], + 'created_at': pr['createdAt'], + 'updated_at': pr['updatedAt'], + 'merged_at': pr.get('mergedAt'), + 'author': pr['author']['login'] if pr['author'] else 'Unknown', + 'body': pr.get('body') or '', + 'type': 'RFC Pull Request', + 'files': files, + 'labels': labels + }) + except Exception as e: + print(f" Date parsing error for community PR: {e}") + + # Process Issues + issues = repo_data.get('issues', {}).get('edges', []) + for edge in issues: + issue = edge['node'] + + # Filter by date + try: + updated_date = date_parser.parse(issue['updatedAt']).replace(tzinfo=None) + if updated_date >= since_date: + # Get labels + labels = [l['node']['name'] for l in issue.get('labels', {}).get('edges', [])] + + # Check if it's RFC-related or working group related + is_rfc = ('rfc' in issue['title'].lower() or + any('rfc' in label.lower() for label in labels) or + any(label.startswith('wg-') for label in labels)) + + if is_rfc: + rfcs.append({ + 'number': issue['number'], + 'title': issue['title'], + 'state': issue['state'], + 'url': issue['url'], + 'created_at': issue['createdAt'], + 'updated_at': issue['updatedAt'], + 'closed_at': issue.get('closedAt'), + 'author': issue['author']['login'] if issue['author'] else 'Unknown', + 'body': issue.get('body') or '', + 'type': 'RFC Tracking Issue', + 'labels': labels + }) + except Exception as e: + print(f" Date parsing error for community issue: {e}") + + elif 'API rate limit exceeded' in result.stderr: + print(f" Warning: GitHub API rate limit exceeded. RFC data unavailable.") + else: + print(f" Error fetching RFCs: {result.stderr}") + + except Exception as e: + print(f" Error fetching community RFCs: {e}") + + return rfcs + +def find_previous_update_date(wg_name): + """Find the most recent update file for the working group and extract its date.""" + updates_dir = Path("toc/working-groups/updates") + if not updates_dir.exists(): + return None + + # Look for files matching the pattern: YYYY-MM-DD-{wg_name}.md + pattern = f"*-{wg_name}.md" + update_files = list(updates_dir.glob(pattern)) + + if not update_files: + return None + + # Sort by filename (which starts with date) to get the most recent + latest_file = sorted(update_files)[-1] + + # Extract date from filename + filename = latest_file.name + date_match = re.match(r'(\d{4}-\d{2}-\d{2})', filename) + if date_match: + try: + date_str = date_match.group(1) + return datetime.strptime(date_str, '%Y-%m-%d') + except ValueError: + print(f"Warning: Could not parse date from {filename}") + + return None + +def main(): + parser = argparse.ArgumentParser(description='Extract raw working group repository activity data') + + # Support both old charter file format and new working group name format + parser.add_argument('charter_or_wg', help='Path to working group charter file OR working group name (e.g., foundational-infrastructure)') + parser.add_argument('target_date', nargs='?', help='Target date for report (YYYY-MM-DD, defaults to today)') + parser.add_argument('--months', type=int, default=3, help='Number of months to look back from target date (default: 3, ignored if previous update found)') + parser.add_argument('--output', help='Output file for raw activity data (default: tmp/{wg}_activity.json)') + + args = parser.parse_args() + + # Determine if this is a charter file or working group name + if args.charter_or_wg.endswith('.md') or '/' in args.charter_or_wg: + # This is a file path + charter_file = args.charter_or_wg + wg_name = Path(charter_file).stem + else: + # This is a working group name + wg_name = args.charter_or_wg + charter_file = f"toc/working-groups/{wg_name}.md" + + if not Path(charter_file).exists(): + print(f"Error: Working group charter not found at {charter_file}") + print("Available working groups:") + for wg_file in Path("toc/working-groups").glob("*.md"): + if wg_file.stem not in ['WORKING-GROUPS']: + print(f" - {wg_file.stem}") + sys.exit(1) + + # Handle target date + if args.target_date: + try: + target_date = datetime.strptime(args.target_date, '%Y-%m-%d') + except ValueError: + print(f"Error: Invalid date format '{args.target_date}'. Use YYYY-MM-DD") + sys.exit(1) + else: + target_date = datetime.now() + + # Determine start date: check for previous updates first, then fallback to months + previous_update_date = find_previous_update_date(wg_name) + if previous_update_date: + since_date = previous_update_date.replace(microsecond=0, tzinfo=None) + print(f"Found previous update from {since_date.strftime('%Y-%m-%d')}, using as start date") + else: + # Calculate date range (using timezone-naive datetime) + since_date = target_date.replace(microsecond=0, tzinfo=None) - timedelta(days=args.months * 30) + print(f"No previous update found, using {args.months} months lookback") + + print(f"Extracting activity data for {wg_name} working group") + print(f"Fetching activity since {since_date.strftime('%Y-%m-%d %H:%M:%S')}") + + # Extract configuration from charter + config = extract_repos_from_charter(charter_file) + if not config: + sys.exit(1) + + # Collect all repositories + all_repos = [] + for area in config.get('areas', []): + repos = area.get('repositories', []) + for repo in repos: + all_repos.append({ + 'repository': repo, + 'area': area.get('name', 'Unknown') + }) + + print(f"Found {len(all_repos)} repositories across {len(config.get('areas', []))} areas") + + # Generate activity report + wg_activity = { + 'working_group': wg_name, + 'generated_at': datetime.now().isoformat(), + 'period_start': since_date.isoformat(), + 'period_end': target_date.isoformat(), + 'areas': [] + } + + total_commits = 0 + total_prs = 0 + total_issues = 0 + total_releases = 0 + + for area in config.get('areas', []): + print(f"\nProcessing area: {area.get('name', 'Unknown')}") + area_data = { + 'name': area.get('name', 'Unknown'), + 'repositories': [] + } + + repos = area.get('repositories', []) + for repo in repos: + activity = get_repo_activity_graphql(repo, since_date) + area_data['repositories'].append(activity) + + total_commits += len(activity['commits']) + total_prs += len(activity['pull_requests']) + total_issues += len(activity['issues']) + total_releases += len(activity['releases']) + + wg_activity['areas'].append(area_data) + + # Fetch RFCs from community repo + print(f"\nFetching RFCs from cloudfoundry/community...") + rfcs = get_community_rfcs(since_date) + wg_activity['rfcs'] = rfcs + + print(f"\nTotal activity found:") + print(f" Commits: {total_commits}") + print(f" Pull Requests: {total_prs}") + print(f" Issues: {total_issues}") + print(f" Releases: {total_releases}") + print(f" RFCs: {len(rfcs)}") + + # Set default output path + if not args.output: + args.output = f"tmp/{wg_name}_activity.json" + + # Output raw activity data + output_data = json.dumps(wg_activity, indent=2, default=str) + with open(args.output, 'w') as f: + f.write(output_data) + print(f"\nRaw activity data written to {args.output}") + + return args.output + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/scripts/generate_working_group_update.py b/scripts/generate_working_group_update.py new file mode 100755 index 000000000..119285c94 --- /dev/null +++ b/scripts/generate_working_group_update.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +""" +Cloud Foundry Working Group Update Generator - OpenCode Run Integration + +This script generates comprehensive working group activity reports by: +1. Using extract_wg_activity.py to extract raw GitHub data +2. Calling OpenCode Run with a structured prompt to analyze the data +3. Generating strategic, community-focused reports for TOC consumption + +The approach separates data extraction from analysis, allowing OpenCode Run's +AI capabilities to provide intelligent interpretation of raw GitHub activity. + +Usage: + python generate_working_group_update.py [date] + +Examples: + python generate_working_group_update.py foundational-infrastructure + python generate_working_group_update.py app-runtime-platform 2025-08-15 + +Requirements: +- OpenCode CLI installed and available in PATH +- GitHub CLI (gh) installed and authenticated with GitHub +- Network access to GitHub API + +The script leverages OpenCode Run's AI capabilities to transform raw data into +strategic insights that celebrate community collaboration and technical achievements. +""" + +import sys +import os +import subprocess +from datetime import datetime +from pathlib import Path + +def check_opencode_available(): + """Check if OpenCode CLI is available.""" + try: + result = subprocess.run(['opencode', '--version'], + capture_output=True, text=True, check=True) + return True + except (subprocess.CalledProcessError, FileNotFoundError): + return False + +def validate_working_group(wg_name): + """Validate that the working group exists.""" + valid_wgs = [ + 'foundational-infrastructure', 'app-runtime-platform', + 'app-runtime-deployments', 'app-runtime-interfaces', + 'cf-on-k8s', 'concourse', 'docs', 'paketo', + 'service-management', 'vulnerability-management' + ] + + if wg_name not in valid_wgs: + print(f"Error: Unknown working group '{wg_name}'") + print(f"Available working groups: {', '.join(valid_wgs)}") + return False + + # Check if charter file exists + charter_path = Path(f"toc/working-groups/{wg_name}.md") + if not charter_path.exists(): + print(f"Error: Working group charter not found at {charter_path}") + return False + + return True + +def validate_date(date_str): + """Validate date format.""" + try: + datetime.strptime(date_str, '%Y-%m-%d') + return True + except ValueError: + print(f"Error: Invalid date format '{date_str}'. Use YYYY-MM-DD") + return False + +def generate_opencode_prompt(wg_name, target_date=None): + """Generate the OpenCode Run prompt for working group analysis.""" + if target_date is None: + target_date = datetime.now().strftime('%Y-%m-%d') + + prompt = f"""I need to generate a Cloud Foundry working group activity update report for the {wg_name} working group that celebrates community collaboration and strategic initiatives. + +**Raw Activity Data Available:** +The file `tmp/{wg_name}_activity.json` contains comprehensive GitHub activity data for this working group, extracted for the period ending {target_date}. This JSON file has the following structure: + +```json +{{ + "working_group": "working-group-name", + "period_end": "YYYY-MM-DD", + "repositories": [ + {{ + "name": "org/repo-name", + "commits": [ + {{ + "sha": "commit-hash", + "author": "username", + "date": "YYYY-MM-DD", + "message": "commit message", + "url": "github-url" + }} + ], + "pull_requests": [ + {{ + "number": 123, + "title": "PR title", + "author": "username", + "state": "open|closed|merged", + "created_at": "YYYY-MM-DD", + "url": "github-url", + "comments": 5 + }} + ], + "issues": [ + {{ + "number": 456, + "title": "Issue title", + "author": "username", + "state": "open|closed", + "created_at": "YYYY-MM-DD", + "url": "github-url", + "comments": 3 + }} + ], + "releases": [ + {{ + "tag_name": "v1.2.3", + "name": "Release Name", + "published_at": "YYYY-MM-DD", + "url": "github-url" + }} + ] + }} + ], + "rfcs": [ + {{ + "number": "rfc-0001", + "title": "RFC Title", + "status": "accepted|draft|withdrawn", + "authors": ["author1", "author2"], + "url": "github-url" + }} + ] +}} +``` + +**Analysis Instructions:** + + 1. **Analyze Raw Activity Data with jq** + - Use `jq` commands to extract and analyze the JSON data from `tmp/{wg_name}_activity.json` + - **PRIORITIZE RFC-RELATED ACTIVITY**: Give highest priority to PRs, issues, and commits related to RFCs + - Identify major features, cross-repository collaboration, key contributors, and technology themes + - Look for patterns in commit messages, PR titles, and issue descriptions that mention RFCs + - Find related work spanning multiple repositories, especially RFC implementations + +2. **Analyze the Working Group Charter** + - Read the working group charter from `toc/working-groups/{wg_name}.md` + - Parse the YAML frontmatter to understand the working group's mission and scope + - Note the working group leads to avoid highlighting them in the report (no self-praise) + +3. **Analyze Existing Report Templates** + - Look for existing reports in `toc/working-groups/updates/` to understand format and style + - Use the most recent report for the same working group as a template if available + - Follow established patterns for structure, tone, and technical depth + + 4. **Filter and Prioritize RFCs for Working Group Relevance** + - From the RFC data in the JSON, identify RFCs most relevant to this working group + - **MANDATORY**: Include all in-progress RFCs that affect this working group + - Consider working group labels, keywords related to the WG's scope, and organizational changes + - Give RFC-related PRs and implementation work top priority in the analysis + +5. **Create Community-Focused Strategic Report** + Generate a markdown report at `toc/working-groups/updates/{target_date}-{wg_name}.md` with: + + **Report Structure:** + - **Title**: "{wg_name.replace('-', ' ').title()} Working Group Update" + - **Frontmatter**: Include title, date, and period + - **Major Initiatives**: + * Focus on completed and in-progress strategic work + * Highlight specific contributors and their organizations (avoid WG leads) + * **Include contributor GitHub profile links**: Use format [Name](https://github.com/username) + * **Integrate PR/Issue links directly in text**: Link specific work to PRs/issues inline within descriptions + * **Include RFC coverage as top priority**: Always mention in-progress RFCs relevant to the working group + * **Prioritize RFC-related activity**: Give highest priority to PRs, issues, and commits related to RFCs + * **Limit each initiative to exactly 3 paragraphs** for conciseness and focus + * **Keep paragraphs short**: Maximum 40 words per paragraph for readability + * Do NOT include separate "Related Work" sections - all links should be integrated into the text + +6. **Apply Community-Focused Writing Guidelines** + - **Celebrate Collaboration**: Emphasize how contributors from different organizations work together + - **Avoid Self-Praise**: Never highlight working group leads when they're giving the update + - **Focus on Impact**: Prioritize "why this matters" over "what was done" + - **Technical Depth**: Provide substantial detail about major initiatives + - **Comprehensive Linking**: Link to specific PRs, issues, and related work throughout + - **Issue Format**: Use descriptive links primarily, with shorthand format: `[Description](url) - org/repo#number` + - **No Duplicate Links**: Never link to the same PR/issue twice within the same paragraph + - **Prefer Descriptive Links**: Use PR change descriptions as link text rather than repo#PR format + - **Inline Integration**: Integrate all PR/issue links directly into descriptive text, not in separate sections + - **Community Language**: Use open-source, collaborative terminology rather than business speak + - **RFC Priority**: Always include in-progress RFCs and prioritize RFC-related activity highest + - **Concise Format**: Each major initiative must be exactly 2 paragraphs, maximum 40 words per paragraph + +**Key Success Criteria:** +- **RFC Coverage**: All in-progress RFCs relevant to the working group are prominently featured +- **RFC Activity Priority**: RFC-related PRs, issues, and commits receive top priority in analysis +- Major technical achievements are prominently featured with detailed context +- Non-lead contributors are recognized inline with GitHub profile links: [Name](https://github.com/username) +- Cross-organizational collaboration is highlighted +- Strategic themes (IPv6, security, modernization) are clearly articulated +- All PR/Issue links use descriptive text and avoid duplication within paragraphs +- Report reads as a celebration of open-source innovation +- Technical depth demonstrates the working group's strategic impact + +Generate a comprehensive report that helps the TOC understand both the technical progress and collaborative community dynamics of this working group.""" + + return prompt + +def run_opencode_analysis(wg_name, target_date=None): + """Execute OpenCode Run with the generated prompt.""" + if target_date is None: + target_date = datetime.now().strftime('%Y-%m-%d') + + print(f"Generating working group update for {wg_name} (target date: {target_date})") + + # First, extract raw activity data + print("Extracting raw activity data...") + activity_file = Path(f"tmp/{wg_name}_activity.json") + activity_file.parent.mkdir(exist_ok=True) + + try: + result = subprocess.run(['python3', 'scripts/extract_wg_activity.py', wg_name, target_date], + capture_output=True, text=True, check=True) + print(f"āœ… Raw activity data extracted: {activity_file}") + except subprocess.CalledProcessError as e: + print(f"Error extracting activity data: {e}") + print(f"stdout: {e.stdout}") + print(f"stderr: {e.stderr}") + return None + + # Check if activity file was generated + if not activity_file.exists(): + print(f"Error: Activity file not found at {activity_file}") + return None + + print("Running OpenCode analysis...") + + prompt = generate_opencode_prompt(wg_name, target_date) + + try: + # Run OpenCode with the generated prompt + result = subprocess.run(['opencode', 'run', prompt], + capture_output=False, text=True, check=True) + + # Check if the expected report was generated + expected_report = Path(f"toc/working-groups/updates/{target_date}-{wg_name}.md") + if expected_report.exists(): + print(f"āœ… Working group update generated: {expected_report}") + return str(expected_report) + else: + print(f"āš ļø OpenCode analysis completed, but report not found at expected location: {expected_report}") + print("The report may have been generated with a different filename or date.") + return None + + except subprocess.CalledProcessError as e: + print(f"Error running OpenCode analysis: {e}") + return None + +def main(): + if len(sys.argv) < 2 or sys.argv[1] in ['--help', '-h', 'help']: + print("Usage: python generate_working_group_update.py [date]") + print("\nAvailable working groups:") + print("- foundational-infrastructure") + print("- app-runtime-platform") + print("- app-runtime-deployments") + print("- app-runtime-interfaces") + print("- cf-on-k8s") + print("- concourse") + print("- docs") + print("- paketo") + print("- service-management") + print("- vulnerability-management") + print("\nExamples:") + print(" python generate_working_group_update.py foundational-infrastructure") + print(" python generate_working_group_update.py app-runtime-platform 2025-08-15") + print("\nRequirements:") + print("- OpenCode CLI installed and available in PATH") + print("- GitHub CLI (gh) installed and authenticated with GitHub") + print("\nSee toc/working-groups/updates/README.md for detailed documentation.") + sys.exit(0) + + if not check_opencode_available(): + print("Error: OpenCode CLI not found in PATH") + print("Please install OpenCode CLI: https://github.com/sst/opencode") + print("Or use the core analysis script directly: python3 scripts/extract_wg_activity.py") + sys.exit(1) + + # Validate arguments + wg_name = sys.argv[1] + if not validate_working_group(wg_name): + sys.exit(1) + + target_date = sys.argv[2] if len(sys.argv) > 2 else None + if target_date and not validate_date(target_date): + sys.exit(1) + + # Run the analysis + report_path = run_opencode_analysis(wg_name, target_date) + + if report_path: + print(f"\nšŸŽ‰ Working group update completed!") + print(f"šŸ“„ Report: {report_path}") + print(f"šŸ“ All reports: toc/working-groups/updates/") + else: + print(f"\nāš ļø Analysis completed but report location unclear") + print(f"Check toc/working-groups/updates/ for generated files") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/toc/working-groups/updates/2025-09-02-foundational-infrastructure.md b/toc/working-groups/updates/2025-09-02-foundational-infrastructure.md new file mode 100644 index 000000000..c57102728 --- /dev/null +++ b/toc/working-groups/updates/2025-09-02-foundational-infrastructure.md @@ -0,0 +1,47 @@ +--- +title: "Foundational Infrastructure Working Group Update" +date: 2025-09-02 +period: June 4 - September 2, 2025 +--- + +# Foundational Infrastructure Working Group Update + +## RFC Implementation: Cloud Controller Storage CLI Integration + +The working group's marquee RFC implementation advances the Cloud Controller blobstore architecture through [RFC-0043: Cloud Controller Blobstore Type: storage-cli](https://github.com/cloudfoundry/community/blob/main/toc/rfc/rfc-0043-cc-blobstore-storage-cli.md). [Stephan Merkel](https://github.com/stephanme) led the specification that unifies storage interfaces, enabling operators to leverage existing BOSH storage CLI tooling. + +The implementation received cross-organizational support from BOSH and Cloud Controller teams through [collaborative working group restructuring](https://github.com/cloudfoundry/community/pull/1275) - cloudfoundry/community#1275. [Aram Price](https://github.com/aramprice) consolidated the davcli repository placement in [organizational cleanup efforts](https://github.com/cloudfoundry/community/pull/1286) - cloudfoundry/community#1286 that streamlined storage CLI management. + +The Storage CLI area now bridges Foundational Infrastructure and App Runtime Interfaces working groups, demonstrating the strategic value of [shared technical ownership](https://github.com/cloudfoundry/community/pull/1292) - cloudfoundry/community#1292. This collaboration enables unified blobstore management across CF deployment architectures. + +## BOSH Infrastructure Modernization and IPv6 Progress + +BOSH continues to lead Cloud Foundry's infrastructure modernization with 1,166 commits across the VM deployment lifecycle. [Julian Hjortshoj](https://github.com/julian-hj) earned promotion to approver status through [extensive BOSH contributions](https://github.com/cloudfoundry/community/pull/1285) - cloudfoundry/community#1285, strengthening the technical leadership team. + +The working group welcomed new reviewers [Ivaylo Ivanov](https://github.com/ivaylogi98) and [Ned Petrov](https://github.com/neddp) through [community onboarding initiatives](https://github.com/cloudfoundry/community/pull/1279) - cloudfoundry/community#1279 and [expertise expansion](https://github.com/cloudfoundry/community/pull/1287) - cloudfoundry/community#1287. This growing reviewer base ensures sustainable code review capacity across BOSH's extensive repository ecosystem. + +IPv6 dual-stack support implementation progresses through [RFC-0038 tracking efforts](https://github.com/cloudfoundry/community/issues/1107), with cross-component coordination spanning BOSH Director, networking layers, and stemcell builders addressing the foundational requirements for modern network architectures. + +## Identity and Authentication Strategic Advances + +The UAA ecosystem delivered significant modernization milestones with Spring 6.2.8, Spring Security 6.5.1, and Spring Boot 3.5.3 upgrades in the [v78.0.0 major release](https://github.com/cloudfoundry/uaa/releases/tag/v78.0.0). [Filip Hanik](https://github.com/fhanik) and [Markus Strehle](https://github.com/strehle) coordinated the Java 21 development upgrade that positions UAA for long-term maintainability. + +Security enhancements addressed critical vulnerabilities through [HTTP response splitting protection](https://github.com/cloudfoundry/uaa/pull/3504) and [secure cookie enforcement](https://github.com/cloudfoundry/uaa/pull/3503). [Duane May](https://github.com/duanemay) led comprehensive dependency modernization spanning 238 commits across UAA repositories. + +The CredHub ecosystem maintains robust credential management capabilities with regular security updates through the [2.9.49 CLI release](https://github.com/cloudfoundry/credhub-cli/releases/tag/2.9.49) and consistent dependency management. Community adoption metrics show 561 Linux downloads and 102 ARM64 downloads for the latest CLI release. + +## Database and Monitoring Infrastructure Evolution + +MySQL and PostgreSQL database releases demonstrate continuous improvement with [v10.29.0 monitoring enhancements](https://github.com/cloudfoundry/mysql-monitoring-release/releases/tag/v10.29.0) and MySQL 8.4 compatibility through SHOW REPLICA STATUS migrations. [Andrew Garner](https://github.com/abg) and the database team delivered 122 commits focusing on operational reliability. + +Prometheus monitoring capabilities expanded through [BOSH exporter improvements](https://github.com/cloudfoundry/bosh_exporter) and [Firehose exporter enhancements](https://github.com/cloudfoundry/firehose_exporter), providing operators with comprehensive infrastructure observability. The Prometheus BOSH release ecosystem shows active development with 67 commits and 55 pull requests during the reporting period. + +System logging infrastructure received modernization through rsyslog and event-log components, with 45 commits addressing contemporary logging requirements. [Ben Fuller](https://github.com/Benjamintf1) coordinated logging modernization efforts that improve operator experience across CF deployments. + +## Community Growth and Organizational Development + +The working group demonstrates healthy community growth through strategic role management initiatives. [Procedural inactive member removal](https://github.com/cloudfoundry/community/pull/1271) - cloudfoundry/community#1271 followed RFC-0025 guidelines while [new contributor onboarding](https://github.com/cloudfoundry/community/pull/1295) - cloudfoundry/community#1295 brings fresh expertise to logging and metrics areas. + +Cross-organizational collaboration strengthened through shared Storage CLI ownership between Foundational Infrastructure and App Runtime Interfaces working groups. This model enables technical expertise sharing while maintaining clear accountability structures. + +The period concluded with robust technical delivery across all infrastructure domains, positioning Cloud Foundry for continued enterprise adoption and community innovation. \ No newline at end of file diff --git a/toc/working-groups/updates/README.md b/toc/working-groups/updates/README.md new file mode 100644 index 000000000..7bd2cb09f --- /dev/null +++ b/toc/working-groups/updates/README.md @@ -0,0 +1,223 @@ +# Cloud Foundry Working Group Community Activity Reports + +This directory contains working group updates generated by using the LLM based generate_working_group_update.py script. + +## Overview + +The automation system analyzes GitHub activity across all repositories managed by Cloud Foundry working groups and generates standardized quarterly community update reports. It uses the GitHub GraphQL API to extract detailed information about community contributions, collaboration patterns, and ecosystem improvements, then applies intelligent feature detection to identify major community initiatives and beneficial themes. + +The focus is on celebrating community contributions, highlighting collaboration, and showing how working group efforts benefit the entire Cloud Foundry ecosystem. + +## Quick Start with OpenCode Run + +**Prerequisites:** +- [OpenCode CLI](https://github.com/sst/opencode) installed and configured +- [GitHub CLI (`gh`)](https://cli.github.com/) installed and authenticated with GitHub +- Network access to GitHub API + +**Generate a report for any working group:** + +```bash +# Simple command that calls OpenCode Run with optimized prompts +python3 scripts/generate_working_group_update.py foundational-infrastructure + +# For a specific date +python3 scripts/generate_working_group_update.py foundational-infrastructure 2025-08-15 +``` + +The `generate_working_group_update.py` script automatically: +- Validates the working group name and date format +- Constructs the optimal OpenCode Run prompt with specific instructions +- Executes OpenCode Run with strategic analysis guidance +- Confirms report generation and provides file location + +## How It Works + +### 1. Repository Discovery +- Reads working group charters with YAML frontmatter. + +### 2. GitHub Data Collection +- Uses GraphQL API for efficient data retrieval +- Extracts 3-month rolling window of activity by default +- Collects comprehensive metadata: + - Commit messages, authors, timestamps + - PR titles, bodies, labels, milestones, comments, reviews + - Issue descriptions, labels, comments + - Release notes and artifacts + +### 3. Feature Analysis +- **Theme Detection**: 30+ strategic keyword patterns + - Security: `security`, `tls`, `ssl`, `certificate`, `encryption` + - Infrastructure: `deployment`, `bosh`, `infrastructure`, `scaling` + - Networking: `networking`, `dns`, `ipv6`, `connectivity` + - Performance: `performance`, `optimization`, `monitoring` + - Developer Experience: `developer`, `build`, `ci/cd`, `testing` + +- **Cross-Repository Initiative Detection**: Groups related PRs across repositories +- **Impact Assessment**: Prioritizes based on comments, reviews, labels + +### 4. Report Generation +- **Executive Summary**: High-level metrics and key achievements +- **Major Initiatives**: + - Completed: Merged PRs with significant impact + - In-Flight: Open PRs under active development +- **Cross-Cutting Themes**: Strategic focus areas with specific examples +- **Activity Breakdown**: Repository-level metrics table +- **Looking Ahead**: Trend analysis and future focus areas + +## Report Format + +Each generated report follows this structure: + +```markdown +--- +title: "Working Group Name Update" +date: 2025-09-02 +period: "June 02, 2025 - September 02, 2025" +--- + +# Working Group Name Update + +## Executive Summary +[High-level overview with key metrics] + +## Major Initiatives +### Completed Initiatives +[Merged PRs with impact analysis] + +### In-Flight Initiatives +[Active development efforts] + +## Cross-Cutting Themes +[Strategic focus areas with examples] + +## Repository Activity Breakdown +[Table of metrics by repository] + +## Looking Ahead +[Trend analysis and future priorities] +``` + +## Environment Setup + +### Required Tools +- **OpenCode CLI**: [Install from GitHub](https://github.com/sst/opencode) +- **GitHub CLI (gh)**: [Install from official docs](https://cli.github.com/) + - Must be authenticated: `gh auth login` + - Used for GitHub GraphQL API access + +### Python Dependencies +```bash +pip install requests pyyaml python-dateutil +``` + +### Authentication +The scripts use GitHub CLI (`gh`) for API access, which handles authentication automatically once configured: + +```bash +# Authenticate with GitHub (one-time setup) +gh auth login + +# Verify authentication +gh auth status +``` + +No additional environment variables are required when using GitHub CLI. + +## Customization + +### Adjusting Analysis Window +```bash +# 6-month analysis window +python scripts/extract_wg_activity.py foundational-infrastructure --months 6 + +# Specific date range +python scripts/extract_wg_activity.py foundational-infrastructure 2025-08-15 --months 3 +``` + +### Custom Feature Keywords +Edit the `feature_keywords` list in `extract_wg_activity.py`: + +```python +feature_keywords = [ + # Add your domain-specific keywords + 'custom-feature', 'special-integration', 'new-capability', + # ... existing keywords +] +``` + +### Output Customization +```bash +# Custom output paths +python scripts/extract_wg_activity.py foundational-infrastructure \ + --output /custom/path/activity.json \ + --features-output /custom/path/features.json \ + --report-output /custom/path/report.md +``` + +## Troubleshooting + +### Common Issues + +**1. GitHub API Rate Limiting** +``` +Error: API rate limit exceeded +``` +- Solution: Ensure `GITHUB_TOKEN` is set with sufficient rate limits +- GitHub API allows 1,000 requests/hour for authenticated users + +**2. Repository Access Issues** +``` +Error: Repository not found or access denied +``` +- Solution: Verify token has read access to CloudFoundry organization repositories +- Check that repository names in working group charters are correct + +**3. No Activity Found** +``` +Warning: Zero activity found for working group +``` +- Solution: Check date range with `--months` parameter +- Verify repositories are actively maintained +- Confirm working group charter has correct repository list + +### Debug Mode +```bash +# Enable verbose output +python scripts/extract_wg_activity.py foundational-infrastructure --verbose + +# Skip report generation to debug data collection +python scripts/extract_wg_activity.py foundational-infrastructure --no-report +``` + +## Integration Examples + +### TOC Meeting Preparation +```bash +# Generate reports for all working groups +for wg in foundational-infrastructure app-runtime-platform paketo; do + python scripts/generate_working_group_update.py $wg +done +``` + +### Quarterly Reviews +```bash +# Generate historical quarterly report +python scripts/generate_working_group_update.py foundational-infrastructure 2025-06-30 +``` + +### Custom Analysis +```bash +# 6-month deep dive for annual planning +python scripts/extract_wg_activity.py foundational-infrastructure --months 6 +cat /tmp/foundational-infrastructure_features.json | jq '.cross_cutting_themes' +``` + +## Contributing + +To enhance the automation system: + +1. **Add New Feature Keywords**: Update `feature_keywords` in `extract_wg_activity.py` +2. **Improve Theme Grouping**: Modify `theme_groups` in `generate_update_report()` +3. **Enhance Report Format**: Update the report template in `generate_update_report()` +4. **Add New Working Groups**: Create charter files with proper YAML frontmatter