Skip to content

Commit

Permalink
Merge branch 'main' into switch-yaml
Browse files Browse the repository at this point in the history
  • Loading branch information
@morgante
morgante committed Jul 10, 2024
2 parents 90bcdd6 + 3a0edc2 commit 2a35e6c
Show file tree
Hide file tree
Showing 25 changed files with 2,645 additions and 108 deletions.
Binary file modified crates/wasm-bindings/wasm_parsers/tree-sitter-toml.wasm
View file
Binary file not shown.
155 changes: 47 additions & 108 deletions resources/edit_grammars.mjs
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,17 @@
import { exec } from "child_process";
import fs from "fs/promises";
import path from "path";
import { fileURLToPath } from "url";

///////////////////////////////////////////////////
// Scripting helpers
///////////////////////////////////////////////////

let childProcesses = new Set();
//a tweaked promisify(exec) to keep track of running children
const execPromise = (command, cwd) => {
const execPromise = (command) => {
return new Promise((resolve, reject) => {
console.log(command);
const child = exec(command, { cwd }, (error, stdout, stderr) => {
const child = exec(command, (error, stdout, stderr) => {
childProcesses.delete(child);
if (error) {
reject(error);
Expand Down Expand Up @@ -70,16 +69,9 @@ const allLanguages = [
// steps, but makes the code smaller for now
const languagesWithoutMetaVariables = ["html", "hcl"];

// Set up paths
const resourceDir = path.dirname(fileURLToPath(import.meta.url));
const METAVARIABLE_GRAMMARS_DIR = path.join(
resourceDir,
"metavariable-grammars"
);
const LANGUAGE_METAVARIABLES_DIR = path.join(
resourceDir,
"language-metavariables"
);
// assumes that this script is run from marzano/resources directory
const METAVARIABLE_GRAMMARS = `../metavariable-grammars`;
const LANGUAGE_METAVARIABLES = "language-metavariables";

///////////////////////////////////////////////////
// Build steps
Expand All @@ -93,59 +85,43 @@ const copyMvGrammar = async (lang, dest) => {
return;
}
await fs.copyFile(
`${METAVARIABLE_GRAMMARS_DIR}/${lang}-metavariable-grammar.js`,
path.join(
LANGUAGE_METAVARIABLES_DIR,
`tree-sitter-${dest ?? lang}/grammar.js`
)
`${METAVARIABLE_GRAMMARS}/${lang}-metavariable-grammar.js`,
`tree-sitter-${dest ?? lang}/grammar.js`
);
};

const copyMvScanner = async (lang, dest) =>
fs.copyFile(
`${METAVARIABLE_GRAMMARS_DIR}/${lang}-metavariable-scanner.cc`,
path.join(
LANGUAGE_METAVARIABLES_DIR,
`tree-sitter-${dest ?? lang}/src/scanner.cc`
)
`${METAVARIABLE_GRAMMARS}/${lang}-metavariable-scanner.cc`,
`tree-sitter-${dest ?? lang}/src/scanner.cc`
);

const treeSitterGenerate = async (dir, buildWasm = true) => {
const andMaybeBuildWasm = buildWasm ? "&& npx tree-sitter build-wasm " : "";
await execPromise(
`tree-sitter generate ${andMaybeBuildWasm} && echo "Generated grammar for ${dir}"`,
path.join(LANGUAGE_METAVARIABLES_DIR, `tree-sitter-${dir}`)
`cd tree-sitter-${dir} && npx tree-sitter generate ${andMaybeBuildWasm} && echo "Generated grammar for ${dir}"`
).catch((e) => console.log("swallowed error, ", e));
};

const copyNodeTypes = async (lang, dest) =>
fs.copyFile(
path.join(
LANGUAGE_METAVARIABLES_DIR,
`tree-sitter-${lang}/src/node-types.json`
),
path.join(resourceDir, `node-types/${dest ?? lang}-node-types.json`)
`tree-sitter-${lang}/src/node-types.json`,
`../node-types/${dest ?? lang}-node-types.json`
);

const copyWasmParser = async (lang, prefix) =>
fs.rename(
path.join(
LANGUAGE_METAVARIABLES_DIR,
`${prefix ?? "tree-sitter-"}${lang}/tree-sitter-${lang}.wasm`
),
path.join(
resourceDir,
`../crates/wasm-bindings/wasm_parsers/tree-sitter-${lang}.wasm`
)
`${prefix ?? "tree-sitter-"}${lang}/tree-sitter-${lang}.wasm`,
`../../crates/wasm-bindings/wasm_parsers/tree-sitter-${lang}.wasm`
);

async function rsyncGrammars(language) {
//If a language is given, only sync that language
//Otherwise, rm -rf the entire language-metavariables dir and sync it from scratch
const treeSitterLang = language ? `tree-sitter-${language}` : ".";
const mvDir = language
? `${LANGUAGE_METAVARIABLES_DIR}/${treeSitterLang}`
: LANGUAGE_METAVARIABLES_DIR;
? `${LANGUAGE_METAVARIABLES}/${treeSitterLang}`
: LANGUAGE_METAVARIABLES;

if (languagesWithoutMetaVariables.includes(language)) {
return;
Expand All @@ -154,21 +130,17 @@ async function rsyncGrammars(language) {
await fs.rmdir(mvDir, { recursive: true });
await fs.mkdir(mvDir, { recursive: true });

const submodulesDir = path.join(
resourceDir,
language ? `language-submodules/${treeSitterLang}` : "language-submodules/."
);
const submodulesDir = language
? `language-submodules/${treeSitterLang}`
: "language-submodules/.";
const blobsToExclude = [
".git*",
"tree-sitter-*/example",
"tree-sitter-*/test",
"tree-sitter-*/corpus",
];

console.log(`Copying ${submodulesDir} to ${mvDir}`);

await execPromise(
`rsync -r -l ${submodulesDir} ${mvDir} --exclude={${blobsToExclude.join(
`rsync -r -l ${submodulesDir} language-metavariables --exclude={${blobsToExclude.join(
","
)}}`
);
Expand Down Expand Up @@ -197,15 +169,11 @@ async function buildLanguage(language) {
const log = (message, ...args) =>
console.log(`[${language}] ` + message, ...args);
log(`Starting`);
const tsLangDir = path.join(
LANGUAGE_METAVARIABLES_DIR,
`tree-sitter-${language}`
);
const tsLangDir = `tree-sitter-${language}`;

if (language == "toml") {
await execPromise(
`npm install regexp-util && npx tree-sitter generate`,
tsLangDir
`cd ${tsLangDir} && npm install regexp-util && npx tree-sitter generate && cd ..`
);
}
//Force cargo.toml to use the correct version of tree-sitter
Expand Down Expand Up @@ -239,44 +207,31 @@ async function buildLanguage(language) {
]);

await fs.rename(
path.join(
LANGUAGE_METAVARIABLES_DIR,
`tree-sitter-markdown/tree-sitter-markdown/tree-sitter-markdown.wasm`
),
path.join(
resourceDir,
`../crates/wasm-bindings/wasm_parsers/tree-sitter-markdown-block.wasm`
)
`tree-sitter-markdown/tree-sitter-markdown/tree-sitter-markdown.wasm`,
`../../crates/wasm-bindings/wasm_parsers/tree-sitter-markdown-block.wasm`
);

await fs.rename(
path.join(
LANGUAGE_METAVARIABLES_DIR,
`tree-sitter-markdown/tree-sitter-markdown-inline/tree-sitter-markdown_inline.wasm`
),
path.join(
resourceDir,
`../crates/wasm-bindings/wasm_parsers/tree-sitter-markdown_inline.wasm`
)
`tree-sitter-markdown/tree-sitter-markdown-inline/tree-sitter-markdown_inline.wasm`,
`../../crates/wasm-bindings/wasm_parsers/tree-sitter-markdown_inline.wasm`
);
} else if (language === "typescript") {
// typescript is special
// we edit its package.json to point to our local version of the js grammar
log(`Copying files`);
await fs.copyFile(
`${METAVARIABLE_GRAMMARS_DIR}/typescript-package.json`,
`${METAVARIABLE_GRAMMARS}/typescript-package.json`,
`${tsLangDir}/package.json`
);

// typescript defines a typescript and tsx grammar, the grammar we care about is in common/define-grammar.js
await fs.copyFile(
`${METAVARIABLE_GRAMMARS_DIR}/typescript-metavariable-define-grammar.js`,
`${METAVARIABLE_GRAMMARS}/typescript-metavariable-define-grammar.js`,
`${tsLangDir}/common/define-grammar.js`
);

await execPromise(
`yarn && yarn build && echo "Generated grammar for ${language}"`,
tsLangDir
`cd ${tsLangDir} && yarn && yarn build && echo "Generated grammar for ${language}"`
);
// await Promise.all([
// execPromise(`cd ${tsLangDir}/tsx && npx tree-sitter build-wasm`),
Expand All @@ -291,41 +246,36 @@ async function buildLanguage(language) {
} else if (language === "vue") {
log(`Copying files`);
await fs.copyFile(
`${METAVARIABLE_GRAMMARS_DIR}/vue-package.json`,
`${METAVARIABLE_GRAMMARS}/vue-package.json`,
`${tsLangDir}/package.json`
);
await fs.copyFile(
`${METAVARIABLE_GRAMMARS_DIR}/vue-metavariable-grammar.js`,
`${METAVARIABLE_GRAMMARS}/vue-metavariable-grammar.js`,
`${tsLangDir}/grammar.js`
);

await execPromise(
`yarn && yarn prepack && npx tree-sitter build-wasm && echo "Generated grammar for ${language}"`,
tsLangDir
`cd ${tsLangDir} && yarn && yarn prepack && npx tree-sitter build-wasm && echo "Generated grammar for ${language}"`
);

await copyNodeTypes(language);
await copyWasmParser(language);
await fs.copyFile(
`${METAVARIABLE_GRAMMARS_DIR}/cc_build.rs`,
`${METAVARIABLE_GRAMMARS}/cc_build.rs`,
`${tsLangDir}/bindings/rust/build.rs`
);
} else if (language === "yaml") {
await copyMvScanner(language);
await buildSimpleLanguage(log, language);
await fs.copyFile(
`${METAVARIABLE_GRAMMARS_DIR}/cc_build.rs`,
`${METAVARIABLE_GRAMMARS}/cc_build.rs`,
`${tsLangDir}/bindings/rust/build.rs`
);
} else if (language === "hcl") {
//HCL's mv grammar goes into `make_grammar.js`, not `grammar.js`
await fs.copyFile(
`${METAVARIABLE_GRAMMARS_DIR}/${language}-metavariable-grammar.js`,
path.join(
resourceDir,
`language-metavariables`,
`tree-sitter-${language}/make_grammar.js`
)
`${METAVARIABLE_GRAMMARS}/${language}-metavariable-grammar.js`,
`tree-sitter-${language}/make_grammar.js`
);
await buildSimpleLanguage(log, language);
} else if (language === "sql") {
Expand All @@ -334,7 +284,7 @@ async function buildLanguage(language) {
await treeSitterGenerate(language, false);
await copyNodeTypes(language);
await fs.copyFile(
`${METAVARIABLE_GRAMMARS_DIR}/c_build.rs`,
`${METAVARIABLE_GRAMMARS}/c_build.rs`,
`${tsLangDir}/bindings/rust/build.rs`
);
} else if (language === "toml") {
Expand All @@ -344,11 +294,8 @@ async function buildLanguage(language) {
log(`Copying files`);
await Promise.all([
await fs.copyFile(
`${METAVARIABLE_GRAMMARS_DIR}/${language}-common-metavariable-grammar.js`,
path.join(
resourceDir,
`language-metavariables/tree-sitter-${language}/common/grammar.js`
)
`${METAVARIABLE_GRAMMARS}/${language}-common-metavariable-grammar.js`,
`tree-sitter-${language}/common/define-grammar.js`
),
// copyMvGrammar('php_only', 'php/php_only'),
// copyMvGrammar('php', 'php/php'),
Expand All @@ -368,24 +315,12 @@ async function buildLanguage(language) {

log(`Copying wasm parser`);
await fs.rename(
path.join(
LANGUAGE_METAVARIABLES_DIR,
`tree-sitter-php/php_only/tree-sitter-php_only.wasm`
),
path.join(
resourceDir,
`../crates/wasm-bindings/wasm_parsers/tree-sitter-php_only.wasm`
)
`tree-sitter-php/php_only/tree-sitter-php_only.wasm`,
`../../crates/wasm-bindings/wasm_parsers/tree-sitter-php_only.wasm`
);
await fs.rename(
path.join(
LANGUAGE_METAVARIABLES_DIR,
`tree-sitter-php/php/tree-sitter-php.wasm`
),
path.join(
resourceDir,
`../crates/wasm-bindings/wasm_parsers/tree-sitter-php.wasm`
)
`tree-sitter-php/php/tree-sitter-php.wasm`,
`../../crates/wasm-bindings/wasm_parsers/tree-sitter-php.wasm`
);
} else {
await buildSimpleLanguage(log, language);
Expand All @@ -399,16 +334,20 @@ async function buildLanguage(language) {
}

async function run() {
const wd = path.dirname(process.argv[1]);
const args = process.argv.slice(2);
const buildAll = args.length == 0;
const languagesTobuild = buildAll ? allLanguages : args;

process.chdir(wd);

console.log("Syncing upstream grammars");
if (buildAll) {
await rsyncGrammars();
} else {
await Promise.all(languagesTobuild.map(rsyncGrammars));
}
process.chdir(LANGUAGE_METAVARIABLES);
await Promise.all(languagesTobuild.map(buildLanguage));
}

Expand Down
11 changes: 11 additions & 0 deletions resources/language-metavariables/tree-sitter-toml/.gitattributes
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
* text eol=lf

src/*.json linguist-generated
src/parser.c linguist-generated
src/tree_sitter/* linguist-generated

bindings/** linguist-generated
binding.gyp linguist-generated
setup.py linguist-generated
Makefile linguist-generated
Package.swift linguist-generated
38 changes: 38 additions & 0 deletions resources/language-metavariables/tree-sitter-toml/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# Rust artifacts
Cargo.lock
target/

# Node artifacts
build/
prebuilds/
node_modules/
*.tgz

# Swift artifacts
.build/

# Go artifacts
go.sum
_obj/

# Python artifacts
.venv/
dist/
*.egg-info
*.whl

# C artifacts
*.a
*.so
*.so.*
*.dylib
*.dll
*.pc

# Example dirs
/examples/*/

# Grammar volatiles
*.wasm
*.obj
*.o
Loading

0 comments on commit 2a35e6c

Please sign in to comment.