Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add bazel support (with tests), add ability to cache downloads locally for increased iteration speed. #19

Merged
merged 4 commits into from
Oct 22, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions .bazelrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
try-import user.bazelrc
try-import .buildbuddy-auth.rc

test --test_output=errors

# Build Buddy Cache Setup
build:build_buddy --bes_results_url=https://app.buildbuddy.io/invocation/
build:build_buddy --bes_backend=grpcs://remote.buildbuddy.io
build:build_buddy --remote_cache=grpcs://remote.buildbuddy.io
build:build_buddy --remote_timeout=3600

# Additional suggestions from buildbuddy for speed
build:build_buddy --experimental_remote_cache_compression
build:build_buddy --experimental_remote_cache_compression_threshold=100
build:build_buddy --noslim_profile
build:build_buddy --experimental_profile_include_target_label
build:build_buddy --experimental_profile_include_primary_output
build:build_buddy --nolegacy_important_outputs

build:ci --config=build_buddy
build:ci --remote_download_minimal
build:ci --build_metadata=ROLE=CI
build:ci --build_metadata=VISIBILITY=PUBLIC
1 change: 1 addition & 0 deletions .bazelversion
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7.2.0
35 changes: 35 additions & 0 deletions .github/workflows/bazel.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# This workflow runs the JSON checker on each vendordep JSON file.

name: Bazel CI

# Controls when the action will run. Triggers the workflow on push or pull request
# events but only for the main branch
on:
push:
pull_request:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
# This workflow contains a single job called "run-json-checker"
run-json-checker:
# The type of runner that the job will run on
runs-on: ubuntu-latest

# Steps represent a sequence of tasks that will be executed as part of the job
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # Fetch all history so we can use git diff

- name: Maybe setup BuildBuddy key
env:
API_KEY: ${{ secrets.API_KEY }}
if: ${{ env.API_KEY != '' }}
shell: bash
run: |
echo "API Key detected!"
echo "build:build_buddy --remote_header=x-buildbuddy-api-key=${{ env.API_KEY }}" > .buildbuddy-auth.rc

- name: Run check
run: |
bazel test //... -k --config=ci
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
*.sw?

bazel-*
user.bazelrc
39 changes: 39 additions & 0 deletions BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
load("@rules_python//python:defs.bzl", "py_binary")
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
load("@vendor-json-repo-pip//:requirements.bzl", "requirement")
load("//:test_utils.bzl", "vendordep_check_test")

# bazel run //:requirements.update
compile_pip_requirements(
name = "requirements",
extra_args = ["--allow-unsafe"],
requirements_in = "requirements.txt",
requirements_txt = "requirements_lock.txt",
)

py_binary(
name = "check",
srcs = ["check.py"],
visibility = ["//visibility:public"],
deps = [
requirement("pyelftools"),
requirement("pefile"),
],
)

# Change this for local testing only.
cache_directory = None

[vendordep_check_test(
allowable_errors = 1,
allowable_warnings = None,
cache_directory = cache_directory,
vendor_file = f,
) for f in glob(["2024/*.json"])]

[vendordep_check_test(
allowable_errors = 0,
allowable_warnings = None,
cache_directory = cache_directory,
vendor_file = f,
) for f in glob(["2025/*.json"])]
25 changes: 25 additions & 0 deletions MODULE.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
module(
name = "vendor-json-repo",
version = "",
compatibility_level = 1,
)

bazel_dep(name = "rules_python", version = "0.37.0")

python = use_extension("@rules_python//python/extensions:python.bzl", "python")
python.toolchain(
python_version = "3.10",
)
use_repo(python, "python_versions")

register_toolchains(
"@python_versions//:all",
)

pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
pip.parse(
hub_name = "vendor-json-repo-pip",
python_version = "3.10",
requirements_lock = "//:requirements_lock.txt",
)
use_repo(pip, "vendor-json-repo-pip")
131 changes: 131 additions & 0 deletions MODULE.bazel.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,11 @@ Currently only one option is supported: `no_debug_suffix`. Normally debug libra

The check.py script requires the `pyelftools` and `pefile` dependencies be installed; use `pip3 install` to install these.

## Bazel Testing
Pyunit tests are automatically auto generated run using the checker tool against all of the vendordep json files in the repository by bazel.

### Prerequisites
- Install [Bazelisk](https://github.com/bazelbuild/bazelisk/releases) and add it to your path. Bazelisk is a wrapper that will download the correct version of bazel specified in the repository. Note: You can alias/rename the binary to `bazel` if you want to keep the familiar `bazel build` vs `bazelisk build` syntax.

### Running the tests
To run the tests, simply run `bazel test //...`. Alternatively, you can run the `checker.py` tool in a standalone mode by running `bazel run //:checker -- <command line arguments from above>`
Empty file added WORKSPACE.bzlmod
Empty file.
23 changes: 19 additions & 4 deletions check.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import sys
import urllib.request
import uuid
import pathlib
from zipfile import ZipFile, BadZipFile

try:
Expand Down Expand Up @@ -38,6 +39,7 @@
got_error = 0
got_warn = 0
message_context = []
cache_directory = None

def msg(t, s):
ctx = ': '.join(message_context) + ': ' if message_context else ''
Expand All @@ -64,19 +66,21 @@ def info(s):
local_maven = None
year = "2025"

def parse_args():
def parse_args(argv):
"""Parse command line arguments. Returns list of filenames."""
parser = argparse.ArgumentParser(description='Checks a vendor json file')
parser.add_argument('--verbose', '-v', action='count', help='increase the verbosity of output')
parser.add_argument('--local-maven', help='directory to use for artifacts instead of fetching from mavenUrls')
parser.add_argument('--year', '-y', help='FRC competition season year (used to set known libraries)')
parser.add_argument('--cache_directory', type=pathlib.Path, help='Optional. If present will set up a download cache in this directory to prevent re-downloading artifacts. Should be used for debugging purposes only.')
parser.add_argument('file', nargs='+', help='json file to parse')
args = parser.parse_args()
args = parser.parse_args(argv)

global verbose, local_maven, year
global verbose, local_maven, year, cache_directory
verbose = args.verbose or 0
local_maven = args.local_maven
year = args.year or "2025"
cache_directory = args.cache_directory

return args.file

Expand Down Expand Up @@ -208,11 +212,22 @@ def fetch(self, classifier, failok=False):
else:
for baseurl in self.urls:
url = baseurl + self.path + fn
maybe_cached_file = None
if cache_directory:
maybe_cached_file = cache_directory / (self.path + fn)
if maybe_cached_file.exists():
if verbose >= 2:
print(f"Found a cache hit for {maybe_cached_file}")
return fn, maybe_cached_file.read_bytes()

if verbose >= 1:
print('downloading "{0}"'.format(url))
try:
with urlopener.open(url) as f:
result = f.read()
if maybe_cached_file:
maybe_cached_file.parent.mkdir(parents=True, exist_ok=True)
maybe_cached_file.write_bytes(result)
except urllib.error.HTTPError as e:
if not failok:
warn('could not fetch url "{0}": {1}'.format(url, e))
Expand Down Expand Up @@ -629,7 +644,7 @@ def check_file(filename):

def main():
had_errors = False
for fn in parse_args():
for fn in parse_args(sys.argv[1:]):
global json_filename, got_error, got_warn
json_filename = fn
got_error = 0
Expand Down
2 changes: 2 additions & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
pyelftools
pefile
Loading