|
| 1 | +#!/usr/bin/env python3 |
| 2 | + |
| 3 | +import json |
| 4 | +import shlex |
| 5 | +import subprocess |
| 6 | +import sys |
| 7 | +import tomllib |
| 8 | +import urllib.request |
| 9 | +from pathlib import Path |
| 10 | + |
| 11 | + |
| 12 | +REPO_ROOT = Path(__file__).resolve().parents[2] |
| 13 | +RUST_EXTRACTOR_MANIFEST = REPO_ROOT / "rust/extractor/Cargo.toml" |
| 14 | + |
| 15 | +# The files that mention the fixed Rust toolchain version |
| 16 | +TOOLCHAIN_RS = REPO_ROOT / "rust/extractor/src/toolchain.rs" |
| 17 | +INTEGRATION_TEST_CONFTEST = REPO_ROOT / "rust/ql/integration-tests/conftest.py" |
| 18 | +QL_TEST_SETUP = REPO_ROOT / "rust/ql/test/setup.sh" |
| 19 | + |
| 20 | + |
| 21 | +def print_step(number: int, description: str) -> None: |
| 22 | + print(f"\nStep {number}: {description}", flush=True) |
| 23 | + |
| 24 | + |
| 25 | +def run(*command: str) -> None: |
| 26 | + print(f"+ {shlex.join(command)}", flush=True) |
| 27 | + subprocess.run(command, cwd=REPO_ROOT, check=True) |
| 28 | + |
| 29 | + |
| 30 | +def run_codegen() -> None: |
| 31 | + try: |
| 32 | + run("bazel", "run", "//rust/codegen") |
| 33 | + except subprocess.CalledProcessError as error: |
| 34 | + print( |
| 35 | + "\nCodegen failed. Carry out the instructions from step 4 in rust/updating-rust-analyzer.md manually.", |
| 36 | + file=sys.stderr, |
| 37 | + flush=True, |
| 38 | + ) |
| 39 | + raise SystemExit(error.returncode) from None |
| 40 | + |
| 41 | + |
| 42 | +def get_min_rust_analyzer_version(manifest: str) -> str: |
| 43 | + """Get the minimum version of all ra_ap dependencies from `Cargo.toml`.""" |
| 44 | + dependencies = tomllib.loads(manifest)["dependencies"] |
| 45 | + return min( |
| 46 | + ( |
| 47 | + version |
| 48 | + for name, version in dependencies.items() |
| 49 | + if name.startswith("ra_ap_") |
| 50 | + ), |
| 51 | + key=lambda value: tuple(map(int, value.split("."))), |
| 52 | + ) |
| 53 | + |
| 54 | + |
| 55 | +def fetch(url: str) -> bytes: |
| 56 | + request = urllib.request.Request( |
| 57 | + url, |
| 58 | + headers={"User-Agent": "github/codeql rust-analyzer updater"}, |
| 59 | + ) |
| 60 | + with urllib.request.urlopen(request) as response: |
| 61 | + return response.read() |
| 62 | + |
| 63 | + |
| 64 | +def get_compatible_rust_toolchain(rust_analyzer_version: str) -> str: |
| 65 | + """Get the latest Rust toolchain released no later than rust-analyzer.""" |
| 66 | + # Get the release date of the rust-analyzer version |
| 67 | + crate_url = f"https://crates.io/api/v1/crates/ra_ap_syntax/{rust_analyzer_version}" |
| 68 | + crate = json.loads(fetch(crate_url)) |
| 69 | + rust_analyzer_release = crate["version"]["created_at"].split("T")[0] |
| 70 | + |
| 71 | + # `manifests.txt` is a list of all toolchains. The one we're interested in looks like |
| 72 | + # ``` |
| 73 | + # static.rust-lang.org/dist/YYYY-MM-DD/channel-rust-stable.toml |
| 74 | + # ``` |
| 75 | + # where `YYYY-MM-DD` is no later than the rust-analyzer release date. |
| 76 | + manifests = ( |
| 77 | + fetch("https://static.rust-lang.org/manifests.txt").decode().splitlines() |
| 78 | + ) |
| 79 | + rust_manifest = next( |
| 80 | + manifest |
| 81 | + for manifest in reversed(manifests) |
| 82 | + if manifest.endswith("/channel-rust-stable.toml") |
| 83 | + and manifest.split("/")[2] <= rust_analyzer_release |
| 84 | + ) |
| 85 | + manifest = tomllib.loads(fetch(f"https://{rust_manifest}").decode()) |
| 86 | + # The version looks like `version = "0.99.0 (797e8a9bc 2026-08-05)"` - we only want the first part. |
| 87 | + return manifest["pkg"]["rust"]["version"].split()[0] |
| 88 | + |
| 89 | + |
| 90 | +def update_fixed_rust_toolchain_versions(version: str) -> None: |
| 91 | + """Change the fixed toolchain in the places where it's hardcoded""" |
| 92 | + toolchain_rs = TOOLCHAIN_RS.read_text() |
| 93 | + prefix = 'const FIXED_RUST_TOOLCHAIN: &str = "' |
| 94 | + old_version = toolchain_rs.split(prefix, 1)[1].split('"', 1)[0] |
| 95 | + TOOLCHAIN_RS.write_text( |
| 96 | + toolchain_rs.replace( |
| 97 | + f'{prefix}{old_version}"', |
| 98 | + f'{prefix}{version}"', |
| 99 | + ) |
| 100 | + ) |
| 101 | + |
| 102 | + for test_setup in (INTEGRATION_TEST_CONFTEST, QL_TEST_SETUP): |
| 103 | + contents = test_setup.read_text() |
| 104 | + test_setup.write_text( |
| 105 | + contents.replace( |
| 106 | + f"rustup toolchain install {old_version} ", |
| 107 | + f"rustup toolchain install {version} ", |
| 108 | + ) |
| 109 | + ) |
| 110 | + |
| 111 | + |
| 112 | +def align_rust_analyzer_versions(manifest: str) -> tuple[str, bool]: |
| 113 | + """Align ra_ap dependency versions in a Cargo manifest. |
| 114 | +
|
| 115 | + rust-analyzer crates may be published gradually, so this selects the newest |
| 116 | + version that is available for every ra_ap dependency. |
| 117 | + """ |
| 118 | + dependencies = { |
| 119 | + name: version |
| 120 | + for name, version in tomllib.loads(manifest)["dependencies"].items() |
| 121 | + if name.startswith("ra_ap_") |
| 122 | + } |
| 123 | + if len(set(dependencies.values())) == 1: |
| 124 | + # All the `ra_ap_` dependencies agree |
| 125 | + return manifest, False |
| 126 | + |
| 127 | + version = get_min_rust_analyzer_version(manifest) |
| 128 | + |
| 129 | + for name, old_version in dependencies.items(): |
| 130 | + manifest = manifest.replace( |
| 131 | + f'{name} = "{old_version}"', |
| 132 | + f'{name} = "{version}"', |
| 133 | + ) |
| 134 | + return manifest, True |
| 135 | + |
| 136 | + |
| 137 | +def commit_all(title: str) -> None: |
| 138 | + run("git", "add", "--all") |
| 139 | + run("git", "commit", "--no-verify", "--allow-empty", "-m", title) |
| 140 | + |
| 141 | + |
| 142 | +def main() -> None: |
| 143 | + status = subprocess.run( |
| 144 | + ["git", "status", "--porcelain"], |
| 145 | + cwd=REPO_ROOT, |
| 146 | + check=True, |
| 147 | + capture_output=True, |
| 148 | + text=True, |
| 149 | + ).stdout |
| 150 | + if status: |
| 151 | + raise RuntimeError("the working tree must be clean") |
| 152 | + |
| 153 | + print_step(1, "Update dependencies") |
| 154 | + manifest = RUST_EXTRACTOR_MANIFEST.read_text() |
| 155 | + old_rust_analyzer_version = get_min_rust_analyzer_version(manifest) |
| 156 | + run("cargo", "upgrade", "--incompatible", "--pinned") |
| 157 | + # Re-read the (potentially) changed manifest |
| 158 | + manifest = RUST_EXTRACTOR_MANIFEST.read_text() |
| 159 | + new_rust_analyzer_version = get_min_rust_analyzer_version(manifest) |
| 160 | + if new_rust_analyzer_version == old_rust_analyzer_version: |
| 161 | + run("git", "restore", ".") |
| 162 | + print("No new rust-analyzer version available.") |
| 163 | + return |
| 164 | + |
| 165 | + aligned_manifest, manifest_changed = align_rust_analyzer_versions(manifest) |
| 166 | + if manifest_changed: |
| 167 | + RUST_EXTRACTOR_MANIFEST.write_text(aligned_manifest) |
| 168 | + run("cargo", "update") |
| 169 | + commit_all("Cargo: Upgrade dependencies") |
| 170 | + |
| 171 | + print_step(2, "Update the fixed Rust toolchain used by the extractor") |
| 172 | + rust_toolchain = get_compatible_rust_toolchain(new_rust_analyzer_version) |
| 173 | + update_fixed_rust_toolchain_versions(rust_toolchain) |
| 174 | + commit_all("Rust: Update fixed toolchain") |
| 175 | + |
| 176 | + print_step(3, "Regenerate vendored bazel files") |
| 177 | + run("misc/bazel/3rdparty/update_tree_sitter_extractors_deps.sh") |
| 178 | + commit_all("Bazel: Regenerate vendored cargo dependencies") |
| 179 | + |
| 180 | + print_step(4, "Run codegen") |
| 181 | + run_codegen() |
| 182 | + commit_all("Rust: Run codegen") |
| 183 | + |
| 184 | + print_step(5, "Try compiling") |
| 185 | + run("bazel", "run", "//rust:install") |
| 186 | + |
| 187 | + |
| 188 | +if __name__ == "__main__": |
| 189 | + main() |
0 commit comments