Attempt to get some sort of CI for NeoVim specifically

.github/workflows/ci.yaml

@@ -18,7 +18,12 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: 18
-      - run: pip install ruff
+      - name: Install dependencies
+        run: |
+          pip install ruff black
+          sudo apt-get update
+          sudo apt-get install -y luarocks
+          sudo luarocks install luacheck
       - name: Get npm cache directory
         id: npm-cache-dir
         shell: bash
@@ -58,6 +63,10 @@ jobs:
           path: ${{ steps.npm-cache-dir.outputs.dir }}
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
           restore-keys: ${{ runner.os }}-node-
+      - if: ${{ matrix.os }} == "windows-latest"
+        run: winget install shasum
+      - name: Setup nvim
+        run: just --verbose ci-setup-nvim
       - run: just setup --locked
       - name: Configure
         run: just --verbose configure-tree-sitter

justfile

@@ -7,30 +7,67 @@ obj_dir := target / "obj"
 debug_out := bin_dir / "debug.out"
 fuzz_out := bin_dir / "fuzz.out"
 
-ts_path := justfile_directory() / "repositories" / "tree-sitter"
+downloads_path := justfile_directory() / "repositories"
+
+ts_path := downloads_path / "tree-sitter"
 ts_repo := "https://github.com/tree-sitter/tree-sitter"
 ts_sha := "1c55abb5308fe3891da545662e5df7ba28ade275" # v0.21.0
 
-just_path := justfile_directory() / "repositories" / "just"
+just_path := downloads_path / "just"
 just_repo := "https://github.com/casey/just.git"
 just_sha := "a2ff42e6c37ba5c429d444f3a18d3633e59f9a34" # 1.24.0
 
+nvim_ts_path := downloads_path / "nvim-treesitter"
+nvim_ts_repo := "https://github.com/nvim-treesitter/nvim-treesitter.git"
+nvim_ts_sha := "f197a15b0d1e8d555263af20add51450e5aaa1f0" # v0.9.2
+
 include_args := "-Isrc/ -I" + ts_path + "/lib/include -Inode_modules/nan"
 general_cflags := "-Wall -Werror --pedantic -Wno-format-pedantic"
 
 fuzzer_flags := env("FUZZER_FLAGS", "-fsanitize=fuzzer,address,undefined")
 fuzz_time := env("FUZZ_TOTAL_TIME", "1200")
+in_ci := env("CI", "0")
 
 # Source files needed to build a parser
 parser_sources := src + "/scanner.c " + src + "/parser.c " + ts_path + "/lib/src/lib.c"
 
 base_cache_key := sha256_file(src / "scanner.c") + sha256_file(src / "parser.c") + sha256(parser_sources) + sha256(include_args) + sha256(general_cflags) + sha256(fuzzer_flags)
 
-verbose_flag := if env("CI", "") == "1" { "--verbose" } else { "" }
-
-# `timeout` is not available on all platforms, but perl often is. This needs a
-# bash shell.
-make_timeout_fn := '''timeout () { perl -e 'alarm shift; exec @ARGV' "$@"; }'''
+verbose_flag := if in_ci != "0" { "--verbose" } else { "" }
+run_if_installed := "just " + verbose_flag + " _run-if-installed "
+
+format_queries_url := "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/63ca90eaa3ce1cc668add8828a9e3d6728dbbdf1/scripts/format-queries.lua"
+format_queries_sha := "a37344c87a0b9affa1d46b117e48442a205845776c4cdac31c57f591770cd522"
+format_queries_fname := downloads_path / "format-queries.lua"
+
+check_queries_url := "https://raw.githubusercontent.com/nvim-treesitter/nvim-treesitter/63ca90eaa3ce1cc668add8828a9e3d6728dbbdf1/scripts/check-queries.lua"
+check_queries_sha := "e4d7888aae1656e2a103f5d868c4f155eca20e554605bd3e6d1c2e8c8e4541f8"
+check_queries_fname := downloads_path / "check-queries.lua"
+
+nvim_tag := "v0.9.5"
+nvim_fsfx := if os() == "linux" {
+		"linux64.tar.gz"
+	} else if os() == "macos" {
+		"macos.tar.gz"
+	} else if os() == "windows" {
+		"win64.zip"
+	} else {
+		error("unsupported platform")
+	}
+nvim_sha := if os() == "linux" {
+		"44ee395d9b5f8a14be8ec00d3b8ead34e18fe6461e40c9c8c50e6956d643b6ca"
+	} else if os() == "macos" {
+		"19d2366e0d6da001583bd0b8a3db59f69ce3dda5fa41f3064c6778cef3edd34c"
+	} else if os() == "windows" {
+		"de6dc1f0edb45f5f225ee24ce80a4fcbc3a337932037e98ae143975fca2556bf"
+	} else {
+		error("unsupported platform")
+	}
+nvim_url := "https://github.com/neovim/neovim/releases/download/" + nvim_tag + "/nvim-" + nvim_fsfx
+nvim_default_path := downloads_path / "nvim"
+nvim_install_prefix := "$HOME/.local"
+nvim_exe := "VIM='" + nvim_install_prefix + "'/share " + nvim_install_prefix + "/bin/nvim"
+nvim_download_fname := downloads_path / "nvim-linux64.tar.gz"
 
 # Files that should parse with errors but not crash
 errors_expected := '''
@@ -50,6 +87,23 @@ no_just_parsing := '''
 default:
 	just --list
 
+# Only run a command if the tool is installed
+_run-if-installed CMD *ARGS="":
+	#!/bin/sh
+	set -eau
+
+	if command -v {{ CMD }} > /dev/null; then
+		echo "Running '{{ CMD }} {{ ARGS }}'" 1>&2
+		{{ CMD }} {{ ARGS }}
+	else
+		if [ '{{ in_ci }}' != 0 ]; then
+			echo 'NOT FOUND: {{ CMD }} required for CI'
+			exit 1
+		fi
+
+		echo "NOT FOUND: {{ CMD }}. Skipping check (this will get verified in CI)."
+	fi
+
 # Install needed packages and make sure tools are setup
 setup *npm-args:
 	#!/bin/sh
@@ -71,46 +125,49 @@ setup *npm-args:
 	check_installed clang
 	check_installed clang-tidy
 	check_installed clang-format
+	check_installed nvim
 
-	if which npm > /dev/null; then
-		npm install --include=dev {{ npm-args }}
-	else
-		echo "npm not found: skipping install"
-	fi
+	{{ run_if_installed }} npm install --include=dev {{ npm-args }}
 
 # Lint with more minimal dependencies that can be run during pre-commit
 _lint-min: _clone-repo-tree-sitter configure-compile-database
-	npm run lint:check
+	@{{ run_if_installed }} npm run lint:check
 	git ls-files '**.c' | grep -v 'parser\.c' | \
-		xargs -IFNAME sh -c 'echo "\nchecking file FNAME" && clang-tidy FNAME'
+		xargs -IFNAME sh -c \
+		'echo "\nchecking file FNAME" && {{ run_if_installed }} clang-tidy FNAME'
 
-# Run the linter for JS, C, Cargo, and Python. Requires clang-tidy, clippy, and ruff.
+# Run the linter for JS, C, Cargo, and Python. Will skip tools that are not installed.
 lint: _lint-min
-	cargo clippy
-	ruff .
+	@{{ run_if_installed }} cargo clippy
+	@{{ run_if_installed }} ruff .
+	@{{ run_if_installed }} luacheck .
 
 _out-dirs:
 	mkdir -p "{{ bin_dir }}"
 	mkdir -p "{{ obj_dir }}"
 
 alias fmt := format
 
-# Autoformat code. Requires Cargo, clang-format, and black.
-format: configure-compile-database
+# Autoformat code. Requires Cargo, clang-format and black are optional
+format: configure-compile-database \
+	(_ensure-downloaded format_queries_url format_queries_sha format_queries_fname)
 	npm run format:write
-	git ls-files '**.c' | grep -v 'parser\.c' | \
+	git ls-files '**.c' | grep -Fv 'parser.c' | \
 		xargs -IFNAME sh -c \
-		'echo "\nformatting 'FNAME'" && clang-format -i FNAME --verbose'
-	cargo fmt
-	black .
+		'echo "\nformatting 'FNAME'" && \
+		{{ run_if_installed }} clang-format -i FNAME --verbose'
+	{{ run_if_installed }} cargo fmt
+	{{ run_if_installed }} black .
 
 # Check formatting without editing
 format-check: configure-compile-database
 	npm run format:check
-	git ls-files '**.c' | grep -v 'parser\.c' | \
+	git ls-files '**.c' | grep -Fv 'parser.c' | \
 		xargs -IFNAME sh -c \
-		'echo "\nchecking formatting for 'FNAME'" && clang-format FNAME | diff -up - FNAME'
-	cargo fmt --check
+		'echo "\nchecking formatting for 'FNAME'" && \
+		{{ run_if_installed }} clang-format FNAME | diff -up - FNAME'
+	{{ run_if_installed }} cargo fmt --check
+	{{ run_if_installed }} black . --check
 
 # Generate the parser
 gen *extra-args:
@@ -120,10 +177,7 @@ gen *extra-args:
 	# Run formatting only on generated files
 	npx prettier --write src/
 
-	# Only clang-format if it is available
-	which clang-format > /dev/null && \
-		clang-format -i src/parser.c || \
-		echo "skipping clang-format"
+	@{{ run_if_installed }} clang-format -i src/parser.c
 
 alias t := test
 
@@ -133,10 +187,21 @@ test *ts-test-args: gen
 	just {{ verbose_flag }} test-parse-highlight
 	just {{ verbose_flag }} verify-no-error-tests
 
+	if command -v nvim > /dev/null; then \
+		just {{ verbose_flag }} test-nvim; \
+	else \
+		echo "NeoVim not found; skipping tests"; \
+	fi
+
 	echo '\nRunning Cargo tests'
 
 	# FIXME: xfail Rust CI on Windows because we are getting STATUS_DLL_NOT_FOUND
-	{{ if os_family() + env("CI", "1") == "windows1" { "echo skipping tests on Windows" } else { "cargo test" } }}
+	{{ if os_family() + env("CI", "1") == "windows1" { \
+		"echo skipping tests on Windows" \
+		} else { \
+		"cargo test" \
+	} }}
+
 
 # Verify that tree-sitter can parse and highlight all files in the repo. Requires a tree-sitter configuration.
 test-parse-highlight: _clone-repo-just
@@ -242,6 +307,10 @@ test-parse-highlight: _clone-repo-just
 verify-no-error-tests:
 	! grep -nr -C4 -E '(ERROR|MISSING|UNEXPECTED)' test
 
+test-nvim: _clone-repo-nvim-treesitter
+	@echo "Running NeoVim tests"
+	# FIXME: do something
+
 # Helper to rebuild helix grammars
 hx-build:
 	hx --grammar build
@@ -274,30 +343,6 @@ configure-tree-sitter:
 
 		f.truncate()
 
-# Run lint and check formatting
-ci-codestyle: lint format-check
-
-# Make sure that files have not changed
-ci-validate-generated-files exit-code="1":
-	#!/bin/sh
-	set -eaux
-
-	git tag ci-tmp-pre-updates
-
-	just {{ verbose_flag }} gen
-
-	failed=false
-	git diff ci-tmp-pre-updates --exit-code || failed=true
-	git tag -d ci-tmp-pre-updates
-
-	if ! [ "$failed" = "false" ]; then
-		echo '::warning::Generated files are out of date!'
-		echo '::warning::run `just gen` and commit the changes'
-
-		# We use an exit code so that we can use this as either a warning or error
-		exit {{ exit-code }}
-	fi
-
 # Run a subset of CI checks before committing.
 pre-commit: _lint-min format-check
 
@@ -320,7 +365,7 @@ _clone-repo url path sha:
 
 	if [ ! -d '{{ path }}' ]; then
 		echo "Cloning {{ url }}"
-		git clone '{{ url }}' '{{ path }}' --depth=100
+		git clone '{{ url }}' '{{ path }}' --depth=500
 	fi
 
 	actual_sha=$(git -C '{{ path }}' rev-parse HEAD)
@@ -330,11 +375,21 @@ _clone-repo url path sha:
 		git -C '{{ path }}' reset --hard '{{ sha }}'
 	fi
 
-# Clone the tree-sitter repo
+# Helpers to clone specific repos
 _clone-repo-tree-sitter: (_clone-repo ts_repo ts_path ts_sha)
-
-# Clone the just repo
 _clone-repo-just: (_clone-repo just_repo just_path just_sha)
+_clone-repo-nvim-treesitter: (_clone-repo nvim_ts_repo nvim_ts_path nvim_ts_sha)
+
+_ensure-downloaded URL SHA OUTPUT_FNAME:
+	#!/bin/sh
+	set -eau
+	# Exit if already installed
+	echo '{{ SHA }}  {{ OUTPUT_FNAME }}' | shasum -c && exit 0
+
+	set -x
+	mkdir -p '{{ parent_directory(OUTPUT_FNAME) }}'
+	curl -L '{{ URL }}' -o '{{ OUTPUT_FNAME }}'
+	echo '{{ SHA }}  {{ OUTPUT_FNAME }}' | shasum -c
 
 # Build a simple debug executable
 debug-build: _clone-repo-tree-sitter _out-dirs
@@ -411,3 +466,50 @@ configure-compile-database:
 
 	with open("compile_commands.json", "w") as f:
 		json.dump(results, f, indent=4)
+
+# Set up NeoVim for use in CI
+ci-setup-nvim:
+	#!/bin/sh
+	set -eaux
+
+	# Exit if installed
+	if ! command -v nvim; then
+		if [ {{ in_ci }} != '1' ]; then
+			echo "refusing to install nvim when not in CI"
+			exit 1
+		fi
+
+		mkdir -p "{{ nvim_install_prefix }}"
+		just {{ verbose_flag }} _ensure-downloaded '{{ nvim_url }}' \
+			'{{ nvim_sha }}' '{{ nvim_download_fname }}'
+
+		tar -xzf "{{ nvim_download_fname }}" -C "{{ nvim_install_prefix }}" \
+			--strip-components=1
+	fi
+
+	echo 'installed version:'
+	{{ nvim_exe }} --version
+
+# Run lint and check formatting
+ci-codestyle: lint format-check
+
+# Make sure that files have not changed
+ci-validate-generated-files exit-code="1":
+	#!/bin/sh
+	set -eaux
+
+	git tag ci-tmp-pre-updates
+
+	just {{ verbose_flag }} gen
+
+	failed=false
+	git diff ci-tmp-pre-updates --exit-code || failed=true
+	git tag -d ci-tmp-pre-updates
+
+	if ! [ "$failed" = "false" ]; then
+		echo '::warning::Generated files are out of date!'
+		echo '::warning::run `just gen` and commit the changes'
+
+		# We use an exit code so that we can use this as either a warning or error
+		exit {{ exit-code }}
+	fi

lua/tree-sitter-just/init.lua

@@ -22,6 +22,7 @@ function M.setup(arg)
 	parser_config.just = {
 		install_info = {
 			url = arg["local"] and join_paths(
+				-- luacheck: ignore
 				vim.fn.stdpath("data"),
 				"site",
 				"pack",