From 07cc93a2e22ce72353b91740885ca0683ceb1fc2 Mon Sep 17 00:00:00 2001 From: Nicholas Lim <18374483+niclim@users.noreply.github.com> Date: Tue, 16 Jul 2024 14:43:43 -0400 Subject: [PATCH] Allow url accessible rules (#2848) --- .../__snapshots__/diff.test.ts.snap | 3 +- .../src/commands/diff/generate-rule-runner.ts | 52 ++++++++++++++----- projects/optic/src/config.ts | 29 ++++++++--- .../__tests__/download-ruleset.test.ts | 9 ++-- .../__tests__/prepare-ruleset.test.ts | 2 + .../src/custom-rulesets/download-ruleset.ts | 15 ++++-- .../src/custom-rulesets/prepare-rulesets.ts | 4 +- 7 files changed, 83 insertions(+), 31 deletions(-) diff --git a/projects/optic/src/__tests__/integration/__snapshots__/diff.test.ts.snap b/projects/optic/src/__tests__/integration/__snapshots__/diff.test.ts.snap index 3656c4e844..13438fbc14 100644 --- a/projects/optic/src/__tests__/integration/__snapshots__/diff.test.ts.snap +++ b/projects/optic/src/__tests__/integration/__snapshots__/diff.test.ts.snap @@ -533,8 +533,7 @@ exports[`diff with mock server custom rules 1`] = ` `; exports[`diff with mock server extends 1`] = ` -"Extending ruleset from @test-org/ruleset-id -x Empty example-api-v0.json +"x Empty example-api-v0.json Operations: 1 operation added, 3 changed, 1 removed x  Checks: 3/5 passed diff --git a/projects/optic/src/commands/diff/generate-rule-runner.ts b/projects/optic/src/commands/diff/generate-rule-runner.ts index 940d4ed465..9590ffdbd8 100644 --- a/projects/optic/src/commands/diff/generate-rule-runner.ts +++ b/projects/optic/src/commands/diff/generate-rule-runner.ts @@ -1,5 +1,5 @@ import path from 'path'; -import { ConfigRuleset, OpticCliConfig } from '../../config'; +import { ConfigRuleset, OpticCliConfig, initializeRules } from '../../config'; import { StandardRulesets } from '@useoptic/standard-rulesets'; import { RuleRunner, @@ -19,6 +19,16 @@ export function setRulesets(rulesets: (Ruleset | ExternalRule)[]) { } const isLocalJsFile = (name: string) => name.endsWith('.js'); +const isUrl = (name: string) => { + try { + const parsed = new URL(name); + if (parsed.protocol === 'http:' || parsed.protocol === 'https:') + return true; + return false; + } catch (e) { + return false; + } +}; type InputPayload = Parameters[0]; @@ -35,19 +45,28 @@ const getStandardToUse = async (options: { ); return config.ruleset; } else if (options.specRuleset) { - try { - const ruleset = await options.config.client.getStandard( - options.specRuleset - ); - return ruleset.config.ruleset; - } catch (e) { - logger.warn( - `${chalk.red('Warning:')} Could not download standard ${ + if (options.specRuleset.startsWith('@')) { + try { + const ruleset = await options.config.client.getStandard( options.specRuleset - }. Please check the ruleset name and whether you are authenticated (run: optic login).` - ); - process.exitCode = 1; - return []; + ); + return ruleset.config.ruleset; + } catch (e) { + logger.warn( + `${chalk.red('Warning:')} Could not download standard ${ + options.specRuleset + }. Please check the ruleset name and whether you are authenticated (run: optic login).` + ); + process.exitCode = 1; + return []; + } + } else { + const rules: any = { + extends: options.specRuleset, + }; + await initializeRules(rules, options.config.client); + + return rules.ruleset; } } else { return options.config.ruleset; @@ -105,6 +124,12 @@ export const generateRuleRunner = async ( ? path.dirname(options.config.configPath) : process.cwd(); localRulesets[rule.name] = path.resolve(rootPath, rule.name); // the path is the name + } else if (isUrl(rule.name)) { + hostedRulesets[rule.name] = { + uploaded_at: String(Math.random()), + url: rule.name, + should_decompress: false, + }; } else { rulesToFetch.push(rule.name); } @@ -118,6 +143,7 @@ export const generateRuleRunner = async ( hostedRulesets[hostedRuleset.name] = { uploaded_at: hostedRuleset.uploaded_at, url: hostedRuleset.url, + should_decompress: true, }; } } diff --git a/projects/optic/src/config.ts b/projects/optic/src/config.ts index b601e3a6d2..34bbdf448c 100644 --- a/projects/optic/src/config.ts +++ b/projects/optic/src/config.ts @@ -1,4 +1,5 @@ import fs from 'node:fs/promises'; +import fetch from 'node-fetch'; import yaml from 'js-yaml'; import { UserError, isTruthyStringValue } from '@useoptic/openapi-utilities'; import Ajv from 'ajv'; @@ -217,14 +218,28 @@ export const initializeRules = async ( client: OpticBackendClient ) => { let rulesetMap: Map = new Map(); + let rawRulesets = config.ruleset ? config.ruleset : []; if (config.extends) { - console.log(`Extending ruleset from ${config.extends}`); + logger.debug(`Extending ruleset from ${config.extends}`); try { - const response = await client.getStandard(config.extends); - rulesetMap = new Map( - response.config.ruleset.map((conf) => [conf.name, conf]) - ); + if (config.extends.startsWith('@')) { + const response = await client.getStandard(config.extends); + rulesetMap = new Map( + response.config.ruleset.map((conf) => [conf.name, conf]) + ); + } else { + // Assumption is that we're fetching a yaml file + const response = await fetch(config.extends).then((response) => { + if (response.status !== 200) { + throw new Error(`received status code ${response.status}`); + } else { + return response.text(); + } + }); + const parsed = yaml.load(response); + rawRulesets.push(...(parsed as any).ruleset); + } } catch (e) { console.error(e); console.log( @@ -233,8 +248,8 @@ export const initializeRules = async ( } } - if (config.ruleset) { - for (const ruleset of config.ruleset) { + if (rawRulesets.length) { + for (const ruleset of rawRulesets) { if (typeof ruleset === 'string') { rulesetMap.set(ruleset, { name: ruleset, config: {} }); } else if (typeof ruleset === 'object' && ruleset !== null) { diff --git a/projects/rulesets-base/src/custom-rulesets/__tests__/download-ruleset.test.ts b/projects/rulesets-base/src/custom-rulesets/__tests__/download-ruleset.test.ts index c8c8d785b0..d973264931 100644 --- a/projects/rulesets-base/src/custom-rulesets/__tests__/download-ruleset.test.ts +++ b/projects/rulesets-base/src/custom-rulesets/__tests__/download-ruleset.test.ts @@ -20,7 +20,8 @@ describe('downloadRuleset', () => { downloadRuleset( 'test-ruleset', 'https://some-url.com', - '2022-11-01T19:32:22.148Z' + '2022-11-01T19:32:22.148Z', + true ) ).rejects.toThrow(new Error('Downloading ruleset failed (404): Missing')); }); @@ -35,7 +36,8 @@ describe('downloadRuleset', () => { await downloadRuleset( 'test-ruleset', 'https://some-url.com', - '2022-11-01T19:32:22.148Z' + '2022-11-01T19:32:22.148Z', + true ); expect(fs.mkdir).toBeCalled(); expect(fs.writeFile).toHaveBeenCalledWith( @@ -49,7 +51,8 @@ describe('downloadRuleset', () => { await downloadRuleset( 'test-ruleset', 'https://some-url.com', - '2022-11-01T19:32:22.148Z' + '2022-11-01T19:32:22.148Z', + true ); expect(fetch).not.toBeCalled(); diff --git a/projects/rulesets-base/src/custom-rulesets/__tests__/prepare-ruleset.test.ts b/projects/rulesets-base/src/custom-rulesets/__tests__/prepare-ruleset.test.ts index 12cdd11e4e..b7d1f94e40 100644 --- a/projects/rulesets-base/src/custom-rulesets/__tests__/prepare-ruleset.test.ts +++ b/projects/rulesets-base/src/custom-rulesets/__tests__/prepare-ruleset.test.ts @@ -32,6 +32,7 @@ describe('prepareRulesets', () => { '@team/custom-ruleset': { url: 'https://some-url.com', uploaded_at: '123', + should_decompress: true, }, }, standardRulesets: { @@ -76,6 +77,7 @@ describe('prepareRulesets', () => { '@team/custom-ruleset': { url: 'https://some-url.com', uploaded_at: '123', + should_decompress: true, }, }, standardRulesets: { diff --git a/projects/rulesets-base/src/custom-rulesets/download-ruleset.ts b/projects/rulesets-base/src/custom-rulesets/download-ruleset.ts index da968a5b5d..2354433f7f 100644 --- a/projects/rulesets-base/src/custom-rulesets/download-ruleset.ts +++ b/projects/rulesets-base/src/custom-rulesets/download-ruleset.ts @@ -7,7 +7,8 @@ import path from 'node:path'; export async function downloadRuleset( name: string, url: string, - uploaded_at: string + uploaded_at: string, + should_decompress: boolean ): Promise { const filepath = path.join(os.tmpdir(), name, `${uploaded_at}.js`); try { @@ -22,14 +23,18 @@ export async function downloadRuleset( `Downloading ruleset failed (${resp.status}): ${await resp.text()}` ); } - - const compressed = await resp.buffer(); - const decompressed = zlib.brotliDecompressSync(compressed); + let raw: Buffer; + if (should_decompress) { + const compressed = await resp.buffer(); + raw = zlib.brotliDecompressSync(compressed); + } else { + raw = await resp.buffer(); + } const filefolder = path.dirname(filepath); // Does not error if folder exists when recursive = true await fs.mkdir(filefolder, { recursive: true }); - await fs.writeFile(filepath, decompressed); + await fs.writeFile(filepath, raw); return filepath; } diff --git a/projects/rulesets-base/src/custom-rulesets/prepare-rulesets.ts b/projects/rulesets-base/src/custom-rulesets/prepare-rulesets.ts index fa655243e6..3c68d6ec26 100644 --- a/projects/rulesets-base/src/custom-rulesets/prepare-rulesets.ts +++ b/projects/rulesets-base/src/custom-rulesets/prepare-rulesets.ts @@ -15,6 +15,7 @@ export type RulesetPayload = { { uploaded_at: string; url: string; + should_decompress: boolean; } >; standardRulesets: Record< @@ -81,7 +82,8 @@ export async function prepareRulesets( rulesetPath = await downloadRuleset( ruleset.name, hostedRuleset.url, - hostedRuleset.uploaded_at + hostedRuleset.uploaded_at, + hostedRuleset.should_decompress ); } catch (e) { warnings.push(`Loading ruleset ${ruleset.name} failed`);