feat: fetch urls, fix #67 (#95)

* feat: add new load util, add file support

* feat: add url plugin

* refactor: new pipeline api (wip)

* feat: support urls in specifications (wip)

* fix: details action is broken

* refactor: make all pipeline actions work with the queue

* refactor: move files around

* refactor: rename resolve to dereference

* docs(changeset): refactor!: when using the pipeline syntax all tasks are queued and only executed when you add .get(), .toJson() or toYaml()

* docs(changeset): refactor!: resolve is renamed to dereference

* docs: rewrite the section about file references

* chore: remove loadFiles utility

* docs(changeset): chore!: remove `loadFiles` utility, use `load` instead

* chore: export load utility and all plugins

* chore: add failing test

* fix: file references inside referenced files are broken

* fix: TS issues

* fix: build

* chore: exclude all node modules

* docs: wording

* docs: fix broken import in example

* feat: new pipeline action to get the filesystem

* refactor: simplify the action queue

* fix: TS issues

* test: filesystem with multiple files has the correct structure

* feat: better error messages

* chore: clean up

* fix: build

* chore: better types

* fix: TS issues

* chore: update lock file

* docs: fix example

* fix: tests

* fix: typo

* fix: doesn’t load urls in passed strings

.changeset/afraid-parents-burn.md

@@ -0,0 +1,5 @@
+---
+'@scalar/openapi-parser': minor
+---
+
+refactor!: resolve is renamed to dereference

.changeset/pink-birds-trade.md

@@ -0,0 +1,5 @@
+---
+'@scalar/openapi-parser': minor
+---
+
+refactor!: when using the pipeline syntax all tasks are queued and only executed when you add .get(), .toJson() or toYaml()

.changeset/tasty-donuts-smash.md

@@ -0,0 +1,5 @@
+---
+'@scalar/openapi-parser': minor
+---
+
+chore!: remove `loadFiles` utility, use `load` instead

README.md

@@ -56,7 +56,7 @@ if (!result.valid) {
 ### Resolve references
 
 ```ts
-import { resolve } from '@scalar/openapi-parser'
+import { dereference } from '@scalar/openapi-parser'
 
 const specification = `{
   "openapi": "3.1.0",
@@ -67,7 +67,7 @@ const specification = `{
   "paths": {}
 }`
 
-const result = await resolve(specification)
+const result = await dereference(specification)
 ```
 
 ## Modify an OpenAPI specification
@@ -128,21 +128,28 @@ const result = openapi()
   .get()
 ```
 
-## Advanced: File references
+## Advanced: URL and file references
 
 You can reference other files, too. To do that, the parser needs to know what files are available.
 
 ```ts
-import { loadFiles } from '@scalar/openapi-parser'
+import {
+  dereference,
+  fetchUrlsPlugin,
+  load,
+  readFilesPlugin,
+} from '@scalar/openapi-parser'
+
+// Load a file and all referenced files
+const filesystem = await load('./openapi.yaml', {
+  plugins: [readFilesPlugin, fetchUrlsPlugin],
+})
 
-// load a file and all referenced files
-const filesystem = loadFiles('./openapi.yaml')
-// instead of just passing a single specification, pass the whole “filesystem”
-const result = await resolve(filesystem)
+// Instead of just passing a single specification, pass the whole “filesystem”
+const result = await dereference(filesystem)
 ```
 
-You don’t have to use `loadFiles`, though. You just need to stick to the format. That enables you store the files
-wherever you want (maybe in a database?) or to use the package in a browser environment.
+As you see, `load()` supports plugin. You can write your own plugin, if you’d like to fetch API defintions from another data source, for example your database. Look at the source code of the `readFilesPlugin` to learn how this could look like.
 
 ## Community
 

demo/src/App.vue

@@ -1,16 +1,28 @@
 <script setup lang="ts">
-import { normalize, resolveReferences, toJson } from '@scalar/openapi-parser'
+import { fetchUrlsPlugin, openapi } from '@scalar/openapi-parser'
 import { onMounted, ref, watch } from 'vue'
 
 const value = ref(
   JSON.stringify(
     {
       openapi: '3.1.0',
-      info: {
-        title: 'Hello World',
-        version: '1.0.0',
+      info: {},
+      paths: {
+        '/foobar': {
+          post: {
+            requestBody: {
+              $ref: '#/components/requestBodies/Foobar',
+            },
+          },
+        },
+      },
+      components: {
+        requestBodies: {
+          Foobar: {
+            content: {},
+          },
+        },
       },
-      paths: {},
     },
     null,
     2,
@@ -34,7 +46,14 @@ watch(value, async (newValue) => {
 watch(
   value,
   async (newValue) => {
-    result.value = toJson(resolveReferences(normalize(newValue)))
+    result.value = (
+      await openapi()
+        .load(newValue, {
+          plugins: [fetchUrlsPlugin],
+        })
+        .dereference()
+        .get()
+    )?.schema
   },
   {
     immediate: true,

demo/vite.config.ts

@@ -1,21 +1,28 @@
 import vue from '@vitejs/plugin-vue'
+import { builtinModules } from 'node:module'
 import path from 'node:path'
 import { defineConfig } from 'vite'
 
 // https://vitejs.dev/config/
 export default defineConfig({
   plugins: [vue()],
+  build: {
+    rollupOptions: {
+      external: [...builtinModules, ...builtinModules.map((m) => `node:${m}`)],
+    },
+  },
   resolve: {
     alias: [
       // Resolve the uncompiled source code for all @scalar packages
       // It’s working with the alias, too. It’s just required to enable HMR.
       // It also does not match components since we want the built version
       {
-        // Resolve the uncompiled source code for all @scalar packages
-        // @scalar/* -> packages/*/
-        // (not @scalar/components/*/style.css)
-        find: /^@scalar\/(?!(openapi-parser|snippetz|components\/style\.css|components\b))(.+)/,
-        replacement: path.resolve(__dirname, '../packages/$2/src/index.ts'),
+        // Resolve the uncompiled source code for @scalar/openapi-parser packages
+        find: '@scalar/openapi-parser',
+        replacement: path.resolve(
+          __dirname,
+          '../packages/openapi-parser/src/index.ts',
+        ),
       },
     ],
   },

packages/openapi-parser/README.md

@@ -19,7 +19,7 @@ npm add @scalar/openapi-parser
 ### Parse
 
 ```ts
-import { resolve } from '@scalar/openapi-parser'
+import { dereference } from '@scalar/openapi-parser'
 
 const file = `{
   "openapi": "3.1.0",
@@ -30,7 +30,7 @@ const file = `{
   "paths": {}
 }`
 
-const result = await resolve(file)
+const result = await dereference(file)
 ```
 
 ### Validate
@@ -59,7 +59,7 @@ if (!result.valid) {
 ### Version
 
 ```ts
-import { resolve } from '@scalar/openapi-parser'
+import { dereference } from '@scalar/openapi-parser'
 
 const file = `{
   "openapi": "3.1.0",
@@ -70,7 +70,7 @@ const file = `{
   "paths": {}
 }`
 
-const result = await resolve(file)
+const result = await dereference(file)
 
 console.log(result.version)
 ```

packages/openapi-parser/createProfile.ts

@@ -11,6 +11,6 @@ const specification = {
   paths: {},
 }
 
-await openapi().load(structuredClone(specification)).resolve()
+await openapi().load(structuredClone(specification)).dereference().get()
 
 console.profileEnd()

packages/openapi-parser/package.json

@@ -53,7 +53,6 @@
     "ajv-formats": "^3.0.1",
     "jsonpointer": "^5.0.1",
     "leven": "^4.0.0",
-    "openapi-types": "^12.1.3",
     "yaml": "^2.4.2"
   }
 }

packages/openapi-parser/src/configuration.ts

@@ -38,6 +38,7 @@ export const ERRORS = {
     'Cannot find supported Swagger/OpenAPI version in specification, version must be a string.',
   INVALID_REFERENCE: 'Can’t resolve reference: %s',
   EXTERNAL_REFERENCE_NOT_FOUND: 'Can’t resolve external reference: %s',
+  FILE_DOES_NOT_EXIST: 'File does not exist: %s',
 } as const
 
 export type VALIDATOR_ERROR = keyof typeof ERRORS