diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index f17a389f..a2c9dfe5 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -8,6 +8,15 @@ jobs: with: node-version: 20.x + - name: Generate Readme + run: | + node build/src/readme.generate.js + + # Validate no READMEs have been updated + - name: Validate Readme generation + run: | + git diff --exit-code + # Only build containers on branches otherwise container builds are duplicated deploy-nonprod-containers - name: Set up Docker Buildx if: ${{ github.ref != 'refs/heads/master' }} diff --git a/src/commands/copy/README.md b/src/commands/copy/README.md new file mode 100644 index 00000000..6c014baf --- /dev/null +++ b/src/commands/copy/README.md @@ -0,0 +1,32 @@ +# copy + +Copy a manifest of files + +## Usage + +copy [...location] + +### Arguments + +| Usage | Description | Options | +| ------------- | ------------------------ | ------- | +| [...location] | Manifest of file to copy | | + +### Options + +| Usage | Description | Options | +| ---------------------- | ----------------------------------- | -------- | +| --config | Location of role configuration file | optional | +| --concurrency | a number | optional | + +### Flags + +| Usage | Description | Options | +| ------------------ | ---------------------------------------------------------------------- | -------------- | +| --verbose | Verbose logging | | +| --force | Overwrite existing files | default: false | +| --no-clobber | Skip existing files | default: false | +| --force-no-clobber | Overwrite changed files | default: false | +| --fix-content-type | Correct content-type from "application/octet-stream" to common formats | default: false | + + diff --git a/src/commands/create-manifest/README.md b/src/commands/create-manifest/README.md new file mode 100644 index 00000000..60a83e4b --- /dev/null +++ b/src/commands/create-manifest/README.md @@ -0,0 +1,36 @@ +# create-manifest + +Create a list of files to copy and pass as a manifest + +## Usage + +create-manifest [...source] + +### Arguments + +| Usage | Description | Options | +| ----------- | ------------- | ------- | +| [...source] | Where to list | | + +### Options + +| Usage | Description | Options | +| ------------------ | ------------------------------------------------------- | -------- | +| --config | Location of role configuration file | optional | +| --transform | Transform/rename files | optional | +| --include | Include files eg ".\*.tiff?$" | optional | +| --exclude | Exclude files eg ".\*.prj$" | optional | +| --group-size | Group files into this size per group, eg "5Gi" or "3TB" | optional | +| --group | Group files into this number per group | optional | +| --limit | Limit the file count to this amount, -1 is no limit | optional | +| --output | Output location for the listing | | +| --target | Copy destination | | + +### Flags + +| Usage | Description | Options | +| --------- | ---------------------------------------- | ------- | +| --verbose | Verbose logging | | +| --flatten | Flatten the files in the target location | | + + diff --git a/src/commands/group/README.md b/src/commands/group/README.md new file mode 100644 index 00000000..04f945b7 --- /dev/null +++ b/src/commands/group/README.md @@ -0,0 +1,30 @@ +# group + +group a array of inputs into a set + +## Usage + +group [...items] + +### Arguments + +| Usage | Description | Options | +| ---------- | ------------------------------------------- | ------- | +| [...items] | list of items to group, can be a JSON array | | + +### Options + +| Usage | Description | Options | +| ----------------- | --------------------------------------------------- | ----------- | +| --config | Location of role configuration file | optional | +| --size | Group items into this number of items group | default: 50 | +| --from-file | JSON file to load inputs from, must be a JSON Array | optional | + +### Flags + +| Usage | Description | Options | +| -------------- | ----------------------------- | -------------- | +| --verbose | Verbose logging | | +| --force-output | force output additional files | default: false | + + diff --git a/src/commands/index.ts b/src/commands/index.ts index da15fe93..d83c2fc1 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -16,40 +16,42 @@ import { commandStacSync } from './stac-sync/stac.sync.js'; import { commandStacValidate } from './stac-validate/stac.validate.js'; import { commandTileIndexValidate } from './tileindex-validate/tileindex.validate.js'; +export const AllCommands = { + copy: commandCopy, + 'create-manifest': commandCreateManifest, + group: commandGroup, + flatten: commandCreateManifest, + 'lds-fetch-layer': commandLdsFetch, + list: commandList, + ls: commandList, + 'stac-catalog': commandStacCatalog, + 'stac-github-import': commandStacGithubImport, + 'stac-sync': commandStacSync, + 'stac-validate': commandStacValidate, + 'tileindex-validate': commandTileIndexValidate, + stac: subcommands({ + name: 'stac', + cmds: { + catalog: commandStacCatalog, + 'github-import': commandStacGithubImport, + sync: commandStacSync, + validate: commandStacValidate, + }, + }), + bmc: subcommands({ + name: 'bmc', + cmds: { + 'create-pr': basemapsCreatePullRequest, + 'create-mapsheet': basemapsCreateMapSheet, + }, + }), + 'pretty-print': commandPrettyPrint, + 'generate-path': commandGeneratePath, +}; + export const cmd = subcommands({ name: 'argo-tasks', version: CliInfo.version, description: 'Utility tasks for argo', - cmds: { - copy: commandCopy, - 'create-manifest': commandCreateManifest, - group: commandGroup, - flatten: commandCreateManifest, - 'lds-fetch-layer': commandLdsFetch, - list: commandList, - ls: commandList, - 'stac-catalog': commandStacCatalog, - 'stac-github-import': commandStacGithubImport, - 'stac-sync': commandStacSync, - 'stac-validate': commandStacValidate, - 'tileindex-validate': commandTileIndexValidate, - stac: subcommands({ - name: 'stac', - cmds: { - catalog: commandStacCatalog, - 'github-import': commandStacGithubImport, - sync: commandStacSync, - validate: commandStacValidate, - }, - }), - bmc: subcommands({ - name: 'bmc', - cmds: { - 'create-pr': basemapsCreatePullRequest, - 'create-mapsheet': basemapsCreateMapSheet, - }, - }), - 'pretty-print': commandPrettyPrint, - 'generate-path': commandGeneratePath, - }, + cmds: AllCommands, }); diff --git a/src/commands/list/README.md b/src/commands/list/README.md new file mode 100644 index 00000000..84c3f170 --- /dev/null +++ b/src/commands/list/README.md @@ -0,0 +1,33 @@ +# list + +List and group files into collections of tasks + +## Usage + +list [...location] + +### Arguments + +| Usage | Description | Options | +| ------------- | ------------- | ------- | +| [...location] | Where to list | | + +### Options + +| Usage | Description | Options | +| ------------------ | ------------------------------------------------------- | -------- | +| --config | Location of role configuration file | optional | +| --include | Include files eg ".\*.tiff?$" | optional | +| --exclude | Exclude files eg ".\*.prj$" | optional | +| --group-size | Group files into this size per group, eg "5Gi" or "3TB" | optional | +| --group | Group files into this number per group | optional | +| --limit | Limit the file count to this amount, -1 is no limit | optional | +| --output | Output location for the listing | optional | + +### Flags + +| Usage | Description | Options | +| --------- | --------------- | ------- | +| --verbose | Verbose logging | | + + diff --git a/src/commands/stac-catalog/README.md b/src/commands/stac-catalog/README.md new file mode 100644 index 00000000..7263c189 --- /dev/null +++ b/src/commands/stac-catalog/README.md @@ -0,0 +1,29 @@ +# stac-catalog + +Construct STAC catalog + +## Usage + +stac-catalog + +### Arguments + +| Usage | Description | Options | +| ----- | -------------------------------------------- | ------- | +| | Location to search for collection.json paths | | + +### Options + +| Usage | Description | Options | +| ---------------- | ---------------------------------------------------- | -------- | +| --config | Location of role configuration file | optional | +| --template | JSON template file location for the Catalog metadata | | +| --output | Output location for the catalog | | + +### Flags + +| Usage | Description | Options | +| --------- | --------------- | ------- | +| --verbose | Verbose logging | | + + diff --git a/src/commands/stac-github-import/README.md b/src/commands/stac-github-import/README.md new file mode 100644 index 00000000..802d7137 --- /dev/null +++ b/src/commands/stac-github-import/README.md @@ -0,0 +1,26 @@ +# stac-github-import + +Format and push a STAC collection.json file and Argo Workflows parameters file to a GitHub repository + +## Usage + +stac-github-import + +### Options + +| Usage | Description | Options | +| --------------------- | ------------------------------------------------------ | --------------------- | +| --config | Location of role configuration file | optional | +| --source | Source location of the collection.json file | | +| --target | Target location for the collection.json file | | +| --repo-name | One of 'linz/elevation', 'linz/imagery' | default: linz/imagery | +| --copy-option | One of '--force', '--no-clobber', '--force-no-clobber' | default: --no-clobber | +| --ticket | Associated JIRA ticket e.g. AIP-74 | default: | + +### Flags + +| Usage | Description | Options | +| --------- | --------------- | ------- | +| --verbose | Verbose logging | | + + diff --git a/src/commands/stac-sync/README.md b/src/commands/stac-sync/README.md new file mode 100644 index 00000000..5184375f --- /dev/null +++ b/src/commands/stac-sync/README.md @@ -0,0 +1,28 @@ +# stac-sync + +Sync STAC files + +## Usage + +stac-sync + +### Arguments + +| Usage | Description | Options | +| ----- | -------------------------------------------------------- | ------- | +| | Location of the source STAC to synchronise from | | +| | Location of the destination STAC in S3 to synchronise to | | + +### Options + +| Usage | Description | Options | +| -------------- | ----------------------------------- | -------- | +| --config | Location of role configuration file | optional | + +### Flags + +| Usage | Description | Options | +| --------- | --------------- | ------- | +| --verbose | Verbose logging | | + + diff --git a/src/commands/stac-validate/README.md b/src/commands/stac-validate/README.md new file mode 100644 index 00000000..4f97c50b --- /dev/null +++ b/src/commands/stac-validate/README.md @@ -0,0 +1,32 @@ +# stac-validate + +Validate STAC files + +## Usage + +stac-validate [...location] + +### Arguments + +| Usage | Description | Options | +| ------------- | -------------------------------------- | ------- | +| [...location] | Location of the STAC files to validate | | + +### Options + +| Usage | Description | Options | +| ---------------------- | -------------------------------------- | -------- | +| --config | Location of role configuration file | optional | +| --concurrency | Number of requests to run concurrently | optional | + +### Flags + +| Usage | Description | Options | +| ----------------- | --------------------------------------------------------- | ------- | +| --verbose | Verbose logging | | +| --checksum-assets | Validate the file:checksum of each asset if it exists | | +| --checksum-links | Validate the file:checksum of each STAC link if it exists | | +| --recursive | Follow and validate STAC links | | +| --strict | Strict checking | | + + diff --git a/src/commands/tileindex-validate/README.md b/src/commands/tileindex-validate/README.md new file mode 100644 index 00000000..b21091d8 --- /dev/null +++ b/src/commands/tileindex-validate/README.md @@ -0,0 +1,34 @@ +# tileindex-validate + +List input files and validate there are no duplicates. + +## Usage + +tileindex-validate [...location] + +### Arguments + +| Usage | Description | Options | +| ------------- | ---------------------------- | ------- | +| [...location] | Location of the source files | | + +### Options + +| Usage | Description | Options | +| ---------------------- | ---------------------------------------------------- | ------------- | +| --config | Location of role configuration file | optional | +| --include | Include files eg ".\*.tiff?$" | optional | +| --scale | Tile grid scale to align output tile to | | +| --source-epsg | Force epsg code for input tiffs | optional | +| --preset | Validate the input tiffs with a configuration preset | default: none | + +### Flags + +| Usage | Description | Options | +| -------------- | ---------------------------------------------------------- | -------------- | +| --verbose | Verbose logging | | +| --retile | Output tile configuration for retiling | default: false | +| --validate | Validate that all input tiffs perfectly align to tile grid | default: true | +| --force-output | force output additional files | default: false | + + diff --git a/src/readme.generate.ts b/src/readme.generate.ts new file mode 100644 index 00000000..52d86aa8 --- /dev/null +++ b/src/readme.generate.ts @@ -0,0 +1,83 @@ +import { fsa } from '@chunkd/fs'; +import type { HelpTopic, ProvidesHelp } from 'cmd-ts/dist/cjs/helpdoc.js'; +import { writeFileSync } from 'fs'; +import * as prettier from 'prettier'; + +import { AllCommands } from './commands/index.js'; +const AnsiRemove = /\u001b\[.*?m/g; + +function hasHelp(f: unknown): f is ProvidesHelp { + if (f == null) return false; + return typeof (f as ProvidesHelp).helpTopics === 'function'; +} + +async function generateReadme(): Promise { + for (const [key, cmd] of Object.entries(AllCommands)) { + if (key !== cmd.name) { + console.log(`Command name mismatch ${key} vs "${cmd.name}".. skipping`); + continue; + } + + if (!hasHelp(cmd)) { + console.log(`Command missing help "${cmd.name}".. skipping`); + continue; + } + + const targetPath = `./src/commands/${cmd.name}`; + const stat = await fsa.exists(targetPath); + if (stat === false) { + console.log(`Command missing folder ${targetPath}`); + continue; + } + + const args: HelpTopic[] = []; + const options: HelpTopic[] = []; + const flags: HelpTopic[] = []; + + for (const topic of cmd.helpTopics()) { + if (topic.category === 'arguments') args.push(topic); + else if (topic.category === 'options') options.push(topic); + else if (topic.category === 'flags') flags.push(topic); + else throw new Error('Unknown help topic: ' + topic.category); + } + + const data: string[] = []; + + data.push(`# ${cmd.name}`); + data.push(); + data.push(cmd.description ?? 'No description'); + data.push(); + data.push('## Usage'); + data.push(); + data.push(`${cmd.name} ` + args.map((m) => m.usage).join(' ')); + + for (const topics of [args, options, flags]) { + const firstTopic = topics[0]?.category; + + if (firstTopic == null) continue; + data.push(`### ${firstTopic.charAt(0).toUpperCase()}${firstTopic.slice(1)}`); // upper case flags -> Flags + data.push(); + data.push('|Usage|Description|Options|'); + data.push('|---------|---------|'); + for (const topic of topics) { + // all commands have --help + if (topic.usage.startsWith('--help')) continue; + + const context = [topic.usage, topic.description, topic.defaults?.[0]?.replace(AnsiRemove, '') ?? ''].join('|'); + data.push(`|${context}|`); + } + } + data.push(); + data.push(''); + data.push(); + + const text = data.join('\n'); + + const formatted = await prettier.format(text, { filepath: 'readme.md' }); + const targetReadme = fsa.join(targetPath, 'README.md'); + + writeFileSync(targetReadme, formatted); + } +} + +generateReadme().catch((e) => console.log('Failed', e));