Skip to content

Commit

Permalink
Refactor grammar generation
Browse files Browse the repository at this point in the history
All the grammar files are maintained in yaml format.
  • Loading branch information
jlelong committed Apr 20, 2024
1 parent a868688 commit fa5ffb6
Show file tree
Hide file tree
Showing 22 changed files with 645 additions and 339 deletions.
3 changes: 2 additions & 1 deletion dev/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,4 +122,5 @@ This script generates intellisense data for LaTeX stored in [`../data/expl3.json

## Grammar files

All the grammar files `*.tmLanguage.json` in `../syntax/` except `latexblock.tmLanguage.json` are retrieved from https://github.com/jlelong/vscode-latex-basics. They are updated by running the script `update-grammar.js`.
- `update-grammar.js` retrieves the latest grammar files from https://github.com/jlelong/vscode-latex-basics.
-`build-grammar.js` generates the `.json` grammar files from `.yaml`.
37 changes: 37 additions & 0 deletions dev/build-grammar.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
const fs = require('fs')
const path = require('path')
const yaml = require('js-yaml')

const syntaxDir = path.join(__dirname, '..', 'syntax')

/**
* Convert an input yaml file to a json output file
* @param {string} inputfile a yaml file name
* @param {string} outputfile a json file name
*/
function convertYamlToJson(inputfile, outputfile) {
try {
const grammar = yaml.load(fs.readFileSync(inputfile, {encoding: 'utf-8'}))
fs.writeFileSync(outputfile, JSON.stringify(grammar, undefined, 4))
} catch (error) {
console.log(error)
}
}

async function main() {
const grammarSrcFiles = [
'src/BibTeX-style.tmLanguage.yaml',
'src/DocTeX.tmLanguage.yaml',
'src/JLweave.tmLanguage.yaml',
'src/LaTeX-Expl3.tmLanguage.yaml',
'src/Pweave.tmLanguage.yaml',
'src/RSweave.tmLanguage.yaml'
]
for (const file of grammarSrcFiles) {
const baseFile = path.basename(file, '.yaml')
console.log(`Generating ${baseFile} from src/`)
convertYamlToJson(path.join(syntaxDir, file), path.join(syntaxDir, baseFile + '.json'))
}
}

main()
16 changes: 15 additions & 1 deletion dev/update-grammar.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,16 @@ const fs = require('fs')
const path = require('path')
const vel = require('vscode-extend-language')

const syntaxDir = path.join(__dirname, '..', 'syntax')

async function downloadFile(repo, file, version='main') {
const url = 'https://raw.githubusercontent.com/' + repo + '/' + version + '/' + file
var content = await vel.download(url)
if (!content) {
console.log('Cannot retrieve ', url)
return
}
const syntaxFilePath = '../syntax/' + path.basename(file)
const syntaxFilePath = path.join(syntaxDir, path.basename(file))
fs.writeFileSync(syntaxFilePath, content)
console.log('Updating', syntaxFilePath)
}
Expand All @@ -33,6 +35,18 @@ async function main() {
for (const file of grammarFiles) {
downloadFile(latexBasicsRepo, 'syntaxes/' + file)
}

// The order of the files matters!
languageFiles = [
'latex-cpp-embedded-language-configuration.json',
'markdown-latex-combined-language-configuration.json',
'latex-language-configuration.json',
'latex3-language-configuration.json'
]
for (const file of languageFiles) {
console.log('Expanding', path.join(syntaxDir, file))
await vel.expandConfigurationFile(path.join(syntaxDir, 'src', file), path.join(syntaxDir, file))
}
}

main()
18 changes: 0 additions & 18 deletions dev/update-language.js

This file was deleted.

4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -2559,7 +2559,9 @@
"release": "npm run clean && npm run lint && npm run compile && vsce package",
"test": "node ./out/test/runTest.js",
"watch-src": "tsc -watch -p tsconfig.json",
"watch-viewer": "tsc -watch -p viewer/tsconfig.json"
"watch-viewer": "tsc -watch -p viewer/tsconfig.json",
"build-grammar": "node ./dev/build-grammar.js",
"update-grammar": "node ./dev/update-grammar.js"
},
"dependencies": {
"cross-spawn": "7.0.3",
Expand Down
192 changes: 96 additions & 96 deletions syntax/BibTeX-style.tmLanguage.json
Original file line number Diff line number Diff line change
@@ -1,116 +1,116 @@
{
"name": "BibTeX Style",
"scopeName": "source.bst",
"patterns": [
{
"include": "#comments"
},
{
"include": "#function-declaration"
},
{
"include": "#command-keywords"
},
{
"include": "#operators"
},
{
"include": "#builtin-objects"
},
{
"include": "#string"
},
{
"include": "#number"
}
],
"repository": {
"function-declaration": {
"name": "meta.function.bst",
"begin": "\\b(FUNCTION)\\s*\\{\\s*([[:alpha:]_.\\$]*\\$?)",
"end": "\\}",
"beginCaptures": {
"1": {
"name": "storage.type.function.bst"
},
"2": {
"name": "entity.name.function.bst"
}
}
},
"command-keywords": {
"patterns": [
"name": "BibTeX Style",
"patterns": [
{
"name": "storage.type.int.bst",
"match": "\\b(?<!\\.)INTEGERS\\b(?!\\.)"
"include": "#comments"
},
{
"name": "storage.type.var.bst",
"match": "\\b(?<!\\.)(ENTRY|MACRO|STRINGS)\\b(?!\\.)"
"include": "#function-declaration"
},
{
"name": "support.function.bst",
"match": "\\b(?<!\\.)(READ|EXECUTE|ITERATE|REVERSE|SORT)\\b(?!\\.)"
}
]
},
"operators": {
"patterns": [
{
"name": "keyword.operator.relational.bst",
"match": "<|>"
"include": "#command-keywords"
},
{
"name": "keyword.operator.comparison.bst",
"match": "\\="
"include": "#operators"
},
{
"name": "keyword.operator.arithmetic.bst",
"match": "\\+|-|\\*"
"include": "#builtin-objects"
},
{
"name": "keyword.operator.assignment.bst",
"match": ":\\="
"include": "#string"
},
{
"name": "keyword.operator.logic.bst",
"match": "\\b(?<!\\.)(and|or|not)(?!(\\w|\\.))"
"include": "#number"
}
]
},
"builtin-objects": {
"patterns": [
{
"name": "keyword.control.bst",
"match": "\\b(?<!\\.)(if\\$|while\\$)(?!(\\w|\\.))"
],
"repository": {
"builtin-objects": {
"patterns": [
{
"match": "\\b(?<!\\.)(if\\$|while\\$)(?!(\\w|\\.))",
"name": "keyword.control.bst"
},
{
"match": "\\b(?<!\\.)(add\\.period\\$|call\\.type\\$|change\\.case\\$|chr\\.to\\.int\\$|cite\\$|duplicate\\$|empty\\$|format\\.name\\$|int\\.to\\.chr\\$|int\\.to\\.chr\\$|int\\.to\\.str\\$|missing\\$|newline\\$|num\\.names\\$|pop\\$|preamble\\$|purify\\$|quote\\$|skip\\$|stack\\$|substring\\$|swap\\$|text\\.length\\$|text\\.prefix\\$|top\\$|type\\$|warning\\$|width\\$|write\\$)(?!(\\w|\\.))",
"name": "support.function.bst"
},
{
"match": "\\b(?<!\\.)(entry\\.max\\$|global\\.max\\$)(?!(\\w|\\.))",
"name": "support.constant.bst"
},
{
"match": "\\b(?<!\\.)sort\\.key\\$(?!(\\w|\\.))",
"name": "support.variable.bst"
}
]
},
{
"name": "support.function.bst",
"match": "\\b(?<!\\.)(add\\.period\\$|call\\.type\\$|change\\.case\\$|chr\\.to\\.int\\$|cite\\$|duplicate\\$|empty\\$|format\\.name\\$|int\\.to\\.chr\\$|int\\.to\\.chr\\$|int\\.to\\.str\\$|missing\\$|newline\\$|num\\.names\\$|pop\\$|preamble\\$|purify\\$|quote\\$|skip\\$|stack\\$|substring\\$|swap\\$|text\\.length\\$|text\\.prefix\\$|top\\$|type\\$|warning\\$|width\\$|write\\$)(?!(\\w|\\.))"
"command-keywords": {
"patterns": [
{
"match": "\\b(?<!\\.)INTEGERS\\b(?!\\.)",
"name": "storage.type.int.bst"
},
{
"match": "\\b(?<!\\.)(ENTRY|MACRO|STRINGS)\\b(?!\\.)",
"name": "storage.type.var.bst"
},
{
"match": "\\b(?<!\\.)(READ|EXECUTE|ITERATE|REVERSE|SORT)\\b(?!\\.)",
"name": "support.function.bst"
}
]
},
{
"name": "support.constant.bst",
"match": "\\b(?<!\\.)(entry\\.max\\$|global\\.max\\$)(?!(\\w|\\.))"
"comments": {
"begin": "%",
"end": "$",
"name": "comment.line.percentage.bst"
},
{
"name": "support.variable.bst",
"match": "\\b(?<!\\.)sort\\.key\\$(?!(\\w|\\.))"
"function-declaration": {
"begin": "\\b(FUNCTION)\\s*\\{\\s*([[:alpha:]_.\\$]*\\$?)",
"beginCaptures": {
"1": {
"name": "storage.type.function.bst"
},
"2": {
"name": "entity.name.function.bst"
}
},
"end": "\\}",
"name": "meta.function.bst"
},
"number": {
"match": "#-?\\d+\\b",
"name": "constant.numeric.bst"
},
"operators": {
"patterns": [
{
"match": "<|>",
"name": "keyword.operator.relational.bst"
},
{
"match": "\\=",
"name": "keyword.operator.comparison.bst"
},
{
"match": "\\+|-|\\*",
"name": "keyword.operator.arithmetic.bst"
},
{
"match": ":\\=",
"name": "keyword.operator.assignment.bst"
},
{
"match": "\\b(?<!\\.)(and|or|not)(?!(\\w|\\.))",
"name": "keyword.operator.logic.bst"
}
]
},
"string": {
"begin": "\"",
"end": "\"",
"name": "string.quoted.double.bst"
}
]
},
"string": {
"name": "string.quoted.double.bst",
"begin": "\"",
"end": "\""
},
"number": {
"name": "constant.numeric.bst",
"match": "#-?\\d+\\b"
},
"comments": {
"name": "comment.line.percentage.bst",
"begin": "%",
"end": "$"
}
}
}
"scopeName": "source.bst"
}
Loading

0 comments on commit fa5ffb6

Please sign in to comment.