diff --git a/README.md b/README.md index 61e1a2ad..58cf581c 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,8 @@ npm install yaml The API provided by `yaml` has three layers, depending on how deep you need to go: [Parse & Stringify](https://eemeli.org/yaml/#parse-amp-stringify), [Documents](https://eemeli.org/yaml/#documents), and the underlying [Lexer/Parser/Composer](https://eemeli.org/yaml/#parsing-yaml). The first has the simplest API and "just works", the second gets you all the bells and whistles supported by the library along with a decent [AST](https://eemeli.org/yaml/#content-nodes), and the third lets you get progressively closer to YAML source, if that's your thing. +A [command-line tool](https://eemeli.org/yaml/#command-line-tool) is also included. + ```js import { parse, stringify } from 'yaml' // or @@ -55,26 +57,26 @@ const YAML = require('yaml') - [`#directives`](https://eemeli.org/yaml/#stream-directives) - [`#errors`](https://eemeli.org/yaml/#errors) - [`#warnings`](https://eemeli.org/yaml/#errors) -- [`isDocument(foo): boolean`](https://eemeli.org/yaml/#identifying-nodes) +- [`isDocument(foo): boolean`](https://eemeli.org/yaml/#identifying-node-types) - [`parseAllDocuments(str, options?): Document[]`](https://eemeli.org/yaml/#parsing-documents) - [`parseDocument(str, options?): Document`](https://eemeli.org/yaml/#parsing-documents) ### Content Nodes -- [`isAlias(foo): boolean`](https://eemeli.org/yaml/#identifying-nodes) -- [`isCollection(foo): boolean`](https://eemeli.org/yaml/#identifying-nodes) -- [`isMap(foo): boolean`](https://eemeli.org/yaml/#identifying-nodes) -- [`isNode(foo): boolean`](https://eemeli.org/yaml/#identifying-nodes) -- [`isPair(foo): boolean`](https://eemeli.org/yaml/#identifying-nodes) -- [`isScalar(foo): boolean`](https://eemeli.org/yaml/#identifying-nodes) -- [`isSeq(foo): boolean`](https://eemeli.org/yaml/#identifying-nodes) +- [`isAlias(foo): boolean`](https://eemeli.org/yaml/#identifying-node-types) +- [`isCollection(foo): boolean`](https://eemeli.org/yaml/#identifying-node-types) +- [`isMap(foo): boolean`](https://eemeli.org/yaml/#identifying-node-types) +- [`isNode(foo): boolean`](https://eemeli.org/yaml/#identifying-node-types) +- [`isPair(foo): boolean`](https://eemeli.org/yaml/#identifying-node-types) +- [`isScalar(foo): boolean`](https://eemeli.org/yaml/#identifying-node-types) +- [`isSeq(foo): boolean`](https://eemeli.org/yaml/#identifying-node-types) - [`new Scalar(value)`](https://eemeli.org/yaml/#scalar-values) - [`new YAMLMap()`](https://eemeli.org/yaml/#collections) - [`new YAMLSeq()`](https://eemeli.org/yaml/#collections) - [`doc.createAlias(node, name?): Alias`](https://eemeli.org/yaml/#working-with-anchors) - [`doc.createNode(value, options?): Node`](https://eemeli.org/yaml/#creating-nodes) - [`doc.createPair(key, value): Pair`](https://eemeli.org/yaml/#creating-nodes) -- [`visit(node, visitor)`](https://eemeli.org/yaml/#modifying-nodes) +- [`visit(node, visitor)`](https://eemeli.org/yaml/#finding-and-modifying-nodes) ### Parsing YAML diff --git a/bin.mjs b/bin.mjs new file mode 100755 index 00000000..7504ae13 --- /dev/null +++ b/bin.mjs @@ -0,0 +1,11 @@ +#!/usr/bin/env node + +import { UserError, cli, help } from './dist/cli.mjs' + +cli(process.stdin, error => { + if (error instanceof UserError) { + if (error.code === UserError.ARGS) console.error(`${help}\n`) + console.error(error.message) + process.exitCode = error.code + } else if (error) throw error +}) diff --git a/config/jest.config.js b/config/jest.config.js index bc83aae8..39b16c48 100644 --- a/config/jest.config.js +++ b/config/jest.config.js @@ -1,6 +1,6 @@ let moduleNameMapper const transform = { - '[/\\\\]tests[/\\\\].*\\.(js|ts)$': [ + '[/\\\\]tests[/\\\\].*\\.(m?js|ts)$': [ 'babel-jest', { configFile: './config/babel.config.js' } ] @@ -12,9 +12,14 @@ switch (process.env.npm_lifecycle_event) { console.log('Testing build output from dist/') moduleNameMapper = { '^yaml$': '/dist/index.js', + '^yaml/cli$': '/dist/cli.mjs', '^yaml/util$': '/dist/util.js', '^../src/test-events$': '/dist/test-events.js' } + transform['[/\\\\]dist[/\\\\].*\\.mjs$'] = [ + 'babel-jest', + { configFile: './config/babel.config.js' } + ] break case 'test': @@ -22,6 +27,7 @@ switch (process.env.npm_lifecycle_event) { process.env.TRACE_LEVEL = 'log' moduleNameMapper = { '^yaml$': '/src/index.ts', + '^yaml/cli$': '/src/cli.ts', '^yaml/util$': '/src/util.ts' } transform['[/\\\\]src[/\\\\].*\\.ts$'] = [ diff --git a/config/rollup.node-config.mjs b/config/rollup.node-config.mjs index 1857ce32..e42dbc73 100644 --- a/config/rollup.node-config.mjs +++ b/config/rollup.node-config.mjs @@ -1,17 +1,26 @@ +import { chmod, stat } from 'node:fs/promises' import typescript from '@rollup/plugin-typescript' -export default { - input: { - index: 'src/index.ts', - 'test-events': 'src/test-events.ts', - util: 'src/util.ts' +export default [ + { + input: { + index: 'src/index.ts', + 'test-events': 'src/test-events.ts', + util: 'src/util.ts' + }, + output: { + dir: 'dist', + format: 'cjs', + esModule: false, + preserveModules: true + }, + plugins: [typescript()], + treeshake: { moduleSideEffects: false, propertyReadSideEffects: false } }, - output: { - dir: 'dist', - format: 'cjs', - esModule: false, - preserveModules: true - }, - plugins: [typescript()], - treeshake: { moduleSideEffects: false, propertyReadSideEffects: false } -} + { + input: 'src/cli.ts', + output: { file: 'dist/cli.mjs' }, + external: () => true, + plugins: [typescript()] + } +] diff --git a/docs/01_intro.md b/docs/01_intro.md index 23e52158..6a4cfafb 100644 --- a/docs/01_intro.md +++ b/docs/01_intro.md @@ -43,6 +43,8 @@ This requirement may be updated between minor versions of the library. The API provided by `yaml` has three layers, depending on how deep you need to go: [Parse & Stringify](#parse-amp-stringify), [Documents](#documents), and the underlying [Lexer/Parser/Composer](#parsing-yaml). The first has the simplest API and "just works", the second gets you all the bells and whistles supported by the library along with a decent [AST](#content-nodes), and the third lets you get progressively closer to YAML source, if that's your thing. +A [command-line tool](#command-line-tool) is also included. +

Parse & Stringify

```js diff --git a/docs/09_cli.md b/docs/09_cli.md new file mode 100644 index 00000000..123cf991 --- /dev/null +++ b/docs/09_cli.md @@ -0,0 +1,26 @@ +# Command-line Tool + +Available as `npx yaml` or `npm exec yaml`: + +
+yaml: A command-line YAML processor and inspector
+
+Reads stdin and writes output to stdout and errors & warnings to stderr.
+
+Usage:
+  yaml          Process a YAML stream, outputting it as YAML
+  yaml cst      Parse the CST of a YAML stream
+  yaml lex      Parse the lexical tokens of a YAML stream
+  yaml valid    Validate a YAML stream, returning 0 on success
+
+Options:
+  --help, -h    Show this message.
+  --json, -j    Output JSON.
+
+Additional options for bare "yaml" command:
+  --doc, -d     Output pretty-printed JS Document objects.
+  --single, -1  Require the input to consist of a single YAML document.
+  --strict, -s  Stop on errors.
+  --visit, -v   Apply a visitor to each document (requires a path to import)
+  --yaml 1.1    Set the YAML version. (default: 1.2)
+
diff --git a/docs/09_yaml_syntax.md b/docs/10_yaml_syntax.md similarity index 100% rename from docs/09_yaml_syntax.md rename to docs/10_yaml_syntax.md diff --git a/docs/index.html.md b/docs/index.html.md index c457f7b6..03a9cce9 100644 --- a/docs/index.html.md +++ b/docs/index.html.md @@ -15,7 +15,8 @@ includes: - 06_custom_tags - 07_parsing_yaml - 08_errors - - 09_yaml_syntax + - 09_cli + - 10_yaml_syntax search: true --- diff --git a/docs/prepare-docs.mjs b/docs/prepare-docs.mjs index bf9ab3f6..d7109b15 100755 --- a/docs/prepare-docs.mjs +++ b/docs/prepare-docs.mjs @@ -1,12 +1,30 @@ #!/usr/bin/env node -import { lstat, mkdir, readdir, readFile, symlink, rm } from 'node:fs/promises' +import { + lstat, + mkdir, + readdir, + readFile, + symlink, + rm, + writeFile +} from 'node:fs/promises' import { resolve } from 'node:path' +import { help } from '../dist/cli.mjs' import { parseAllDocuments } from '../dist/index.js' const source = 'docs' const target = 'docs-slate/source' +// Update CLI help +const cli = resolve(source, '09_cli.md') +const docs = await readFile(cli, 'utf-8') +const update = docs.replace( + /(
).*?(<\/pre>)/s,
+  '$1\n' + help + '\n$2'
+)
+if (update !== docs) await writeFile(cli, update)
+
 // Create symlink for index.html.md
 const indexSource = resolve(source, 'index.html.md')
 const indexTarget = resolve(target, 'index.html.md')
diff --git a/package-lock.json b/package-lock.json
index 7bc84636..f541aee3 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -8,6 +8,9 @@
       "name": "yaml",
       "version": "2.3.4",
       "license": "ISC",
+      "bin": {
+        "yaml": "bin.mjs"
+      },
       "devDependencies": {
         "@babel/core": "^7.12.10",
         "@babel/plugin-transform-class-properties": "^7.23.3",
diff --git a/package.json b/package.json
index d673e8d8..56728838 100644
--- a/package.json
+++ b/package.json
@@ -18,6 +18,7 @@
   ],
   "type": "commonjs",
   "main": "./dist/index.js",
+  "bin": "./bin.mjs",
   "browser": {
     "./dist/index.js": "./browser/index.js",
     "./dist/util.js": "./browser/dist/util.js",
diff --git a/src/cli.ts b/src/cli.ts
new file mode 100644
index 00000000..a05ede81
--- /dev/null
+++ b/src/cli.ts
@@ -0,0 +1,206 @@
+import { resolve } from 'node:path'
+import { parseArgs } from 'node:util'
+
+import { type Token, prettyToken } from './parse/cst.js'
+import { Lexer } from './parse/lexer.js'
+import { Parser } from './parse/parser.js'
+import { Composer } from './compose/composer.js'
+import { LineCounter } from './parse/line-counter.js'
+import { type Document } from './doc/Document.js'
+import { prettifyError } from './errors.js'
+import { visit, type visitor } from './visit.js'
+
+export const help = `\
+yaml: A command-line YAML processor and inspector
+
+Reads stdin and writes output to stdout and errors & warnings to stderr.
+
+Usage:
+  yaml          Process a YAML stream, outputting it as YAML
+  yaml cst      Parse the CST of a YAML stream
+  yaml lex      Parse the lexical tokens of a YAML stream
+  yaml valid    Validate a YAML stream, returning 0 on success
+
+Options:
+  --help, -h    Show this message.
+  --json, -j    Output JSON.
+
+Additional options for bare "yaml" command:
+  --doc, -d     Output pretty-printed JS Document objects.
+  --single, -1  Require the input to consist of a single YAML document.
+  --strict, -s  Stop on errors.
+  --visit, -v   Apply a visitor to each document (requires a path to import)
+  --yaml 1.1    Set the YAML version. (default: 1.2)`
+
+export class UserError extends Error {
+  static ARGS = 2
+  static SINGLE = 3
+  code: number
+  constructor(code: number, message: string) {
+    super(`Error: ${message}`)
+    this.code = code
+  }
+}
+
+export async function cli(
+  stdin: NodeJS.ReadableStream,
+  done: (error?: Error) => void,
+  argv?: string[]
+) {
+  let args
+  try {
+    args = parseArgs({
+      args: argv,
+      allowPositionals: true,
+      options: {
+        doc: { type: 'boolean', short: 'd' },
+        help: { type: 'boolean', short: 'h' },
+        json: { type: 'boolean', short: 'j' },
+        single: { type: 'boolean', short: '1' },
+        strict: { type: 'boolean', short: 's' },
+        visit: { type: 'string', short: 'v' },
+        yaml: { type: 'string', default: '1.2' }
+      }
+    })
+  } catch (error) {
+    return done(new UserError(UserError.ARGS, (error as Error).message))
+  }
+
+  const {
+    positionals: [mode],
+    values: opt
+  } = args
+
+  stdin.setEncoding('utf-8')
+
+  // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing
+  switch (opt.help || mode) {
+    /* istanbul ignore next */
+    case true: // --help
+      console.log(help)
+      break
+
+    case 'lex': {
+      const lexer = new Lexer()
+      const data: string[] = []
+      const add = (tok: string) => {
+        if (opt.json) data.push(tok)
+        else console.log(prettyToken(tok))
+      }
+      stdin.on('data', (chunk: string) => {
+        for (const tok of lexer.lex(chunk, true)) add(tok)
+      })
+      stdin.on('end', () => {
+        for (const tok of lexer.lex('', false)) add(tok)
+        if (opt.json) console.log(JSON.stringify(data))
+        done()
+      })
+      break
+    }
+
+    case 'cst': {
+      const parser = new Parser()
+      const data: Token[] = []
+      const add = (tok: Token) => {
+        if (opt.json) data.push(tok)
+        else console.dir(tok, { depth: null })
+      }
+      stdin.on('data', (chunk: string) => {
+        for (const tok of parser.parse(chunk, true)) add(tok)
+      })
+      stdin.on('end', () => {
+        for (const tok of parser.parse('', false)) add(tok)
+        if (opt.json) console.log(JSON.stringify(data))
+        done()
+      })
+      break
+    }
+
+    case undefined:
+    case 'valid': {
+      const lineCounter = new LineCounter()
+      const parser = new Parser(lineCounter.addNewLine)
+      // @ts-expect-error Version is validated at runtime
+      const composer = new Composer({ version: opt.yaml })
+      const visitor: visitor | null = opt.visit
+        ? (await import(resolve(opt.visit))).default
+        : null
+      let source = ''
+      let hasDoc = false
+      let reqDocEnd = false
+      const data: Document[] = []
+      const add = (doc: Document) => {
+        if (hasDoc && opt.single) {
+          return done(
+            new UserError(
+              UserError.SINGLE,
+              'Input stream contains multiple documents'
+            )
+          )
+        }
+        for (const error of doc.errors) {
+          prettifyError(source, lineCounter)(error)
+          if (opt.strict || mode === 'valid') return done(error)
+          console.error(error)
+        }
+        for (const warning of doc.warnings) {
+          prettifyError(source, lineCounter)(warning)
+          console.error(warning)
+        }
+        if (visitor) visit(doc, visitor)
+        if (mode === 'valid') doc.toJS()
+        else if (opt.json) data.push(doc)
+        else if (opt.doc) {
+          Object.defineProperties(doc, {
+            options: { enumerable: false },
+            schema: { enumerable: false }
+          })
+          console.dir(doc, { depth: null })
+        } else {
+          if (reqDocEnd) console.log('...')
+          try {
+            const str = String(doc)
+            console.log(str.endsWith('\n') ? str.slice(0, -1) : str)
+          } catch (error) {
+            done(error as Error)
+          }
+        }
+        hasDoc = true
+        reqDocEnd = !doc.directives?.docEnd
+      }
+      stdin.on('data', (chunk: string) => {
+        source += chunk
+        for (const tok of parser.parse(chunk, true)) {
+          for (const doc of composer.next(tok)) add(doc)
+        }
+      })
+      stdin.on('end', () => {
+        for (const tok of parser.parse('', false)) {
+          for (const doc of composer.next(tok)) add(doc)
+        }
+        for (const doc of composer.end(false)) add(doc)
+        if (opt.single && !hasDoc) {
+          return done(
+            new UserError(
+              UserError.SINGLE,
+              'Input stream contained no documents'
+            )
+          )
+        }
+        if (mode !== 'valid' && opt.json) {
+          console.log(JSON.stringify(opt.single ? data[0] : data))
+        }
+        done()
+      })
+      break
+    }
+
+    default:
+      done(
+        new UserError(
+          UserError.ARGS,
+          `Unknown command: ${JSON.stringify(mode)}`
+        )
+      )
+  }
+}
diff --git a/tests/artifacts/cli-singlequote.mjs b/tests/artifacts/cli-singlequote.mjs
new file mode 100644
index 00000000..e291eaf7
--- /dev/null
+++ b/tests/artifacts/cli-singlequote.mjs
@@ -0,0 +1,6 @@
+/** @type {import('../../src/index').visitor} */
+export default {
+  Scalar(_, node) {
+    node.type = 'QUOTE_SINGLE'
+  }
+}
diff --git a/tests/artifacts/cli-unstyle.cjs b/tests/artifacts/cli-unstyle.cjs
new file mode 100644
index 00000000..5a103a6b
--- /dev/null
+++ b/tests/artifacts/cli-unstyle.cjs
@@ -0,0 +1,10 @@
+/** @type {import('../../src/index').visitor} */
+module.exports = {
+  Collection(_, node) {
+    delete node.flow
+  },
+  Scalar(_, node) {
+    delete node.format
+    delete node.type
+  }
+}
diff --git a/tests/cli.ts b/tests/cli.ts
new file mode 100644
index 00000000..f0da5172
--- /dev/null
+++ b/tests/cli.ts
@@ -0,0 +1,173 @@
+/* eslint-disable @typescript-eslint/no-floating-promises */
+
+import { Readable } from 'node:stream'
+import { cli } from 'yaml/cli'
+
+const [major] = process.versions.node.split('.')
+const skip = Number(major) < 20
+
+;(skip ? describe.skip : describe)('CLI', () => {
+  const stdout: unknown[] = []
+  const stderr: unknown[] = []
+  beforeAll(() => {
+    jest.spyOn(global.console, 'log').mockImplementation(thing => {
+      stdout.push(thing)
+    })
+    jest.spyOn(global.console, 'dir').mockImplementation(thing => {
+      stdout.push(thing)
+    })
+    jest.spyOn(global.console, 'error').mockImplementation(thing => {
+      stderr.push(thing)
+    })
+  })
+
+  function ok(
+    name: string,
+    input: string,
+    args: string[],
+    output: unknown[],
+    errors: unknown[] = []
+  ) {
+    test(name, done => {
+      stdout.length = 0
+      stderr.length = 0
+      cli(
+        Readable.from([input]),
+        error => {
+          try {
+            expect(stdout).toMatchObject(output)
+            expect(stderr).toMatchObject(errors)
+            expect(error).toBeUndefined()
+          } finally {
+            done()
+          }
+        },
+        args
+      )
+    })
+  }
+
+  function fail(
+    name: string,
+    input: string,
+    args: string[],
+    errors: unknown[]
+  ) {
+    test(name, done => {
+      stderr.length = 0
+      let doned = false
+      cli(
+        Readable.from([input]),
+        error => {
+          if (doned) return
+          try {
+            expect(stderr).toMatchObject(errors)
+            expect(error).not.toBeUndefined()
+          } finally {
+            done()
+            doned = true
+          }
+        },
+        args
+      )
+    })
+  }
+
+  describe('Bad arguments', () => {
+    fail('command', '42', ['nonesuch'], [])
+    fail('option', '42', ['--nonesuch'], [])
+  })
+
+  describe('Stream processing', () => {
+    ok('empty', '', [], [])
+    ok('basic', 'hello: world', [], ['hello: world'])
+    ok('valid ok', 'hello: world', ['valid'], [])
+    fail('valid fail', 'hello: world: 2', ['valid'], [])
+    ok(
+      'multiple',
+      'hello: world\n---\n42',
+      [],
+      ['hello: world', '...', '---\n42']
+    )
+    ok(
+      'warn',
+      'hello: !foo world',
+      [],
+      ['hello: !foo world'],
+      [{ name: 'YAMLWarning' }]
+    )
+    fail('error', 'hello: world: 2', [], [{ name: 'YAMLParseError' }])
+    fail('--single + empty', '', ['--single'], [])
+    fail('--single + multiple', 'hello: world\n---\n42', ['--single'], [])
+    describe('--json', () => {
+      ok('basic', 'hello: world', ['--json'], ['[{"hello":"world"}]'])
+      ok(
+        '--single',
+        'hello: world',
+        ['--json', '--single'],
+        ['{"hello":"world"}']
+      )
+      ok(
+        'multiple',
+        'hello: world\n---\n42',
+        ['--json'],
+        ['[{"hello":"world"},42]']
+      )
+    })
+    describe('--doc', () => {
+      ok('basic', 'hello: world', ['--doc'], [{ contents: { items: [{}] } }])
+      ok(
+        'multiple',
+        'hello: world\n---\n42',
+        ['--doc'],
+        [{ contents: { items: [{}] } }, { contents: { value: 42 } }]
+      )
+      ok(
+        'error',
+        'hello: world: 2',
+        ['--doc'],
+        [{ contents: { items: [{}] } }],
+        [{ name: 'YAMLParseError' }]
+      )
+    })
+    describe('--visit', () => {
+      ok(
+        'unstyle',
+        '{"hello":"world"}',
+        ['--visit', './tests/artifacts/cli-unstyle.cjs'],
+        ['hello: world']
+      )
+      ok(
+        'singlequote',
+        '{"hello":"world"}',
+        ['--visit', './tests/artifacts/cli-singlequote.mjs'],
+        ["{ 'hello': 'world' }"]
+      )
+    })
+  })
+
+  describe('CST parser', () => {
+    ok('basic', 'hello: world', ['cst'], [{ type: 'document' }])
+    ok(
+      'multiple',
+      'hello: world\n---\n42',
+      ['cst'],
+      [{ type: 'document' }, { type: 'document' }]
+    )
+  })
+
+  describe('Lexer', () => {
+    ok(
+      'basic',
+      'hello: world',
+      ['lex'],
+      ['', '', '"hello"', '":"', '" "', '', '"world"']
+    )
+    ok(
+      '--json',
+      'hello: world',
+      ['lex', '--json'],
+      ['["\\u0002","\\u001f","hello",":"," ","\\u001f","world"]']
+    )
+  })
+})
diff --git a/tests/tsconfig.json b/tests/tsconfig.json
index 342ab7a2..4e46645b 100644
--- a/tests/tsconfig.json
+++ b/tests/tsconfig.json
@@ -5,10 +5,11 @@
     "declaration": false,
     "paths": {
       "yaml": ["src/index.ts"],
+      "yaml/cli": ["src/cli.ts"],
       "yaml/util": ["src/util.ts"]
     },
     "rootDir": "..",
     "types": ["jest", "node"]
   },
-  "include": ["**/*.ts"]
+  "include": ["**/*.ts", "artifacts/cli-*"]
 }
diff --git a/tsconfig.json b/tsconfig.json
index dc36dc17..e735a19e 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -1,6 +1,7 @@
 {
   "compilerOptions": {
     "declaration": true,
+    "module": "ESNext",
     "moduleResolution": "node",
     "noUnusedLocals": true,
     "noUnusedParameters": true,