Speed up repness 11x

delphi/polismath/benchmarks/__init__.py

@@ -0,0 +1 @@
+"""Benchmark scripts for polismath performance testing."""

delphi/polismath/benchmarks/bench_repness.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+"""
+Benchmark script for repness (representativeness) computation performance.
+
+Usage:
+    cd delphi
+    ../.venv/bin/python -m polismath.benchmarks.bench_repness <votes_csv_path> [--runs N]
+    ../.venv/bin/python -m polismath.benchmarks.bench_repness <votes_csv_path> --profile
+
+Example:
+    ../.venv/bin/python -m polismath.benchmarks.bench_repness real_data/.local/r7wehfsmutrwndviddnii-bg2050/2025-11-25-1909-r7wehfsmutrwndviddnii-votes.csv --runs 3
+    ../.venv/bin/python -m polismath.benchmarks.bench_repness real_data/.local/r7wehfsmutrwndviddnii-bg2050/2025-11-25-1909-r7wehfsmutrwndviddnii-votes.csv --profile
+"""
+# TODO(datasets): Once PR https://github.com/compdemocracy/polis/pull/2312 is merged,
+# use the datasets package with include_local=True instead of requiring a path argument.
+
+import time
+from pathlib import Path
+
+import click
+
+from polismath.benchmarks.benchmark_utils import (
+    load_votes_from_csv,
+    extract_dataset_name,
+    votes_csv_argument,
+    runs_option,
+)
+from polismath.conversation import Conversation
+from polismath.pca_kmeans_rep.repness import (
+    conv_repness,
+    comment_stats,
+    add_comparative_stats,
+    finalize_cmt_stats,
+    select_rep_comments,
+    compute_group_comment_stats_df,
+    select_rep_comments_df,
+    select_consensus_comments_df,
+    prop_test_vectorized,
+    two_prop_test_vectorized
+)
+
+
+profile_option = click.option(
+    '--profile', '-p',
+    is_flag=True,
+    help='Run with line profiler on conv_repness',
+)
+
+
+def setup_conversation(votes_csv: Path) -> tuple[Conversation, str, int, float]:
+    """
+    Load votes and setup conversation with PCA and clusters.
+
+    Args:
+        votes_csv: Path to votes CSV file
+
+    Returns:
+        Tuple of (conversation, dataset_name, n_votes, setup_time)
+    """
+    dataset_name = extract_dataset_name(votes_csv)
+
+    print(f"Loading votes from '{votes_csv}'...")
+    votes_dict = load_votes_from_csv(votes_csv)
+    n_votes = len(votes_dict['votes'])
+    print(f"Loaded {n_votes:,} votes")
+    print()
+
+    print("Setting up conversation with votes and clusters...")
+    setup_start = time.perf_counter()
+    conv = Conversation(dataset_name)
+    conv = conv.update_votes(votes_dict, recompute=False)
+    conv._compute_pca()
+    conv._compute_clusters()
+    setup_time = time.perf_counter() - setup_start
+
+    print(f"Setup completed in {setup_time:.2f}s")
+    print(f"  Matrix shape: {conv.raw_rating_mat.shape}")
+    print(f"  Number of groups: {len(conv.group_clusters)}")
+    print()
+
+    return conv, dataset_name, n_votes, setup_time
+
+
+def benchmark_repness(votes_csv: Path, runs: int = 3) -> dict:
+    """
+    Benchmark repness computation on a dataset.
+
+    Args:
+        votes_csv: Path to votes CSV file
+        runs: Number of runs to average
+
+    Returns:
+        Dictionary with benchmark results
+    """
+    conv, dataset_name, n_votes, setup_time = setup_conversation(votes_csv)
+
+    # Benchmark repness computation
+    print(f"Benchmarking repness computation ({runs} runs)...")
+    times = []
+    for i in range(runs):
+        start = time.perf_counter()
+        conv._compute_repness()
+        elapsed = time.perf_counter() - start
+        times.append(elapsed)
+        n_rep_comments = sum(len(v) for v in conv.repness.get('group_repness', {}).values())
+        print(f"  Run {i+1}: {elapsed:.3f}s ({n_rep_comments} representative comments)")
+
+    avg = sum(times) / len(times)
+    min_time = min(times)
+    max_time = max(times)
+
+    print()
+    print("=" * 50)
+    print(f"Dataset: {dataset_name}")
+    print(f"Votes: {n_votes:,}")
+    print(f"Matrix shape: {conv.raw_rating_mat.shape}")
+    print(f"Groups: {len(conv.group_clusters)}")
+    print(f"Average repness time: {avg:.3f}s")
+    print(f"Min/Max: {min_time:.3f}s / {max_time:.3f}s")
+
+    # Calculate comments per second
+    n_comments = conv.raw_rating_mat.shape[1]
+    n_participants = conv.raw_rating_mat.shape[0]
+    n_groups = len(conv.group_clusters)
+
+    # Repness complexity is roughly O(groups * comments * participants)
+    operations = n_groups * n_comments * n_participants
+    print(f"Throughput: {operations/avg:,.0f} ops/sec (groups × comments × participants)")
+
+    return {
+        'dataset': dataset_name,
+        'n_votes': n_votes,
+        'shape': conv.raw_rating_mat.shape,
+        'n_groups': n_groups,
+        'times': times,
+        'avg': avg,
+        'min': min_time,
+        'max': max_time,
+        'setup_time': setup_time,
+    }
+
+
+def profile_repness(votes_csv: Path) -> None:
+    """
+    Run line profiler on conv_repness.
+
+    Args:
+        votes_csv: Path to votes CSV file
+    """
+    from line_profiler import LineProfiler
+    conv, _, _, _ = setup_conversation(votes_csv)
+
+    # Setup line profiler
+    profiler = LineProfiler()
+    profiler.add_function(conv_repness)
+    profiler.add_function(compute_group_comment_stats_df)
+    profiler.add_function(select_rep_comments_df)
+    profiler.add_function(select_consensus_comments_df)
+    profiler.add_function(prop_test_vectorized)
+    profiler.add_function(two_prop_test_vectorized)
+
+    # Run profiled
+    print("Running conv_repness with line profiler...")
+    profiler.runcall(conv_repness, conv.rating_mat, conv.group_clusters)
+
+    # Print results
+    print()
+    print("=" * 70)
+    print("LINE PROFILE RESULTS")
+    print("=" * 70)
+    profiler.print_stats()
+
+
+@click.command()
+@votes_csv_argument
+@runs_option
+@profile_option
+def main(votes_csv: Path, runs: int, profile: bool):
+    """Benchmark repness computation performance."""
+    if profile:
+        profile_repness(votes_csv)
+    else:
+        benchmark_repness(votes_csv, runs)
+
+
+if __name__ == '__main__':
+    main()

delphi/polismath/benchmarks/bench_update_votes.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""
+Benchmark script for update_votes performance.
+
+Usage:
+    cd delphi
+    ../.venv/bin/python -m polismath.benchmarks.bench_update_votes <votes_csv_path> [--runs N]
+
+Example:
+    ../.venv/bin/python -m polismath.benchmarks.bench_update_votes real_data/.local/r7wehfsmutrwndviddnii-bg2050/2025-11-25-1909-r7wehfsmutrwndviddnii-votes.csv --runs 3
+"""
+# TODO(datasets): Once PR https://github.com/compdemocracy/polis/pull/2312 is merged,
+# use the datasets package with include_local=True instead of requiring a path argument.
+
+import time
+from pathlib import Path
+
+import click
+
+from polismath.benchmarks.benchmark_utils import (
+    load_votes_from_csv,
+    extract_dataset_name,
+    votes_csv_argument,
+    runs_option,
+)
+
+
+def benchmark_update_votes(votes_csv: Path, runs: int = 3) -> dict:
+    """
+    Benchmark update_votes on a dataset.
+
+    Args:
+        votes_csv: Path to votes CSV file
+        runs: Number of runs to average
+
+    Returns:
+        Dictionary with benchmark results
+    """
+    from polismath.conversation import Conversation
+
+    dataset_name = extract_dataset_name(votes_csv)
+
+    print(f"Loading votes from '{votes_csv}'...")
+    votes_dict = load_votes_from_csv(votes_csv)
+    n_votes = len(votes_dict['votes'])
+    print(f"Loaded {n_votes:,} votes")
+    print()
+
+    times = []
+    for i in range(runs):
+        conv = Conversation(dataset_name)
+        start = time.perf_counter()
+        conv = conv.update_votes(votes_dict, recompute=False)
+        elapsed = time.perf_counter() - start
+        times.append(elapsed)
+        print(f"  Run {i+1}: {elapsed:.2f}s")
+
+    avg = sum(times) / len(times)
+    min_time = min(times)
+    max_time = max(times)
+
+    print()
+    print(f"Dataset: {dataset_name}")
+    print(f"Votes: {n_votes:,}")
+    print(f"Matrix shape: {conv.raw_rating_mat.shape}")
+    print(f"Average time: {avg:.2f}s")
+    print(f"Min/Max: {min_time:.2f}s / {max_time:.2f}s")
+    print(f"Throughput: {n_votes/avg:,.0f} votes/sec")
+
+    return {
+        'dataset': dataset_name,
+        'n_votes': n_votes,
+        'shape': conv.raw_rating_mat.shape,
+        'times': times,
+        'avg': avg,
+        'min': min_time,
+        'max': max_time,
+        'throughput': n_votes / avg,
+    }
+
+
+@click.command()
+@votes_csv_argument
+@runs_option
+def main(votes_csv: Path, runs: int):
+    """Benchmark update_votes performance."""
+    benchmark_update_votes(votes_csv, runs)
+
+
+if __name__ == '__main__':
+    main()

delphi/polismath/benchmarks/benchmark_utils.py

@@ -0,0 +1,110 @@
+"""
+Shared utilities for benchmark scripts.
+"""
+
+import time
+from pathlib import Path
+from typing import Callable, Dict, Any
+
+import click
+import pandas as pd
+
+
+def load_votes_from_csv(votes_csv: Path) -> dict:
+    """
+    Load votes from a CSV file into the format expected by Conversation.update_votes().
+
+    Args:
+        votes_csv: Path to votes CSV file with columns: voter-id, comment-id, vote, timestamp
+
+    Returns:
+        Dictionary with 'votes' list and 'lastVoteTimestamp'
+    """
+    df = pd.read_csv(votes_csv)
+
+    # Fixed timestamp for reproducibility
+    fixed_timestamp = 1700000000000
+
+    # Use vectorized pandas operations instead of iterrows() for efficiency
+    df = df.rename(columns={
+        'voter-id': 'pid',
+        'comment-id': 'tid',
+    })
+    if 'timestamp' in df.columns:
+        df['created'] = df['timestamp'].astype(int)
+    else:
+        df['created'] = fixed_timestamp
+
+    votes_list = df[['pid', 'tid', 'vote', 'created']].to_dict('records')
+
+    return {
+        'votes': votes_list,
+        'lastVoteTimestamp': fixed_timestamp
+    }
+
+
+def extract_dataset_name(votes_path: Path) -> str:
+    """
+    Extract dataset name from path.
+
+    Args:
+        votes_path: Path to votes CSV file
+
+    Returns:
+        Dataset name (e.g., "r7wehfsmutrwndviddnii-bg2050" -> "bg2050")
+    """
+    parent_name = votes_path.parent.name
+    if '-' in parent_name:
+        return parent_name.split('-', 1)[1]
+    return parent_name
+
+
+def run_benchmark(
+    func: Callable[[], Any],
+    runs: int,
+    description: str = "operation"
+) -> Dict[str, Any]:
+    """
+    Run a benchmark function multiple times and collect timing statistics.
+
+    Args:
+        func: Function to benchmark (called with no arguments)
+        runs: Number of runs
+        description: Description for printing
+
+    Returns:
+        Dictionary with timing statistics and results from last run
+    """
+    times = []
+    result = None
+    for i in range(runs):
+        start = time.perf_counter()
+        result = func()
+        elapsed = time.perf_counter() - start
+        times.append(elapsed)
+        print(f"  Run {i+1}: {elapsed:.3f}s")
+
+    avg = sum(times) / len(times)
+    min_time = min(times)
+    max_time = max(times)
+
+    return {
+        'times': times,
+        'avg': avg,
+        'min': min_time,
+        'max': max_time,
+        'result': result,
+    }
+
+
+# Common click options for benchmark scripts
+votes_csv_argument = click.argument(
+    'votes_csv',
+    type=click.Path(exists=True, path_type=Path),
+)
+
+runs_option = click.option(
+    '--runs', '-n',
+    default=3,
+    help='Number of benchmark runs (default: 3)',
+)