Egress is authorized with UCAN (stubbed) (#126)

Implements the UCAN validation for egress, except: can't yet get Spaces
from the Indexing Service or get/store delegations, but both are stubbed
so we can build from here.

## To test/use:

1. Start with a Freeway URL, like
`https://bafybeib7l5an3dsnr65gvei4n3x64ihlqkhg4iytcrlyxkbse6m5e6zufm.ipfs.w3s.link`.
* To test on localhost, convert from a subdomain gateway URL to a path
gateway URL—eg,
`http://localhost:8787/ipfs/bafybeib7l5an3dsnr65gvei4n3x64ihlqkhg4iytcrlyxkbse6m5e6zufm`
2. Grab the Space's DID. (This doesn't actually have to be the correct
Space, as long as it's consistent, since we're stubbing the Space lookup
*and* the delegation store.)
3. Make sure you're logged into the `w3` CLI.
4. Run `node scripts/delegate-serve.js <space-did> | pbcopy` to copy a
`base64url` delegation string.
* Or, use `node scripts/delegate-serve.js <space-did> <token> | pbcopy`
to require a token.
5. Add `?stub_space=<space-did>&stub_delegation=<delegation>` to the
URL.
    * Add `authToken=<token>` if a token was required.
6. The content should load!
    * If the token does not match, it shouldn't load.
    * If the Space does not match, it shouldn't load.
    * If the delegation is missing, it shouldn't load.
    * If additional delegations are provided, it should still load.

## The Stubs

* **`stub_space`:** The Space DID to "find" the content in. The lookup
will really locate the content, but will add the given Space to whatever
results it gets back.
* **`stub_delegation`:** A delegation to add to the "delegation store"
for the request, `.archive()`ed to a CAR and `base64url`ed. May be given
multiple times to add multiple delegations. Every call to
`delegationsStore.find()` will find all given delegations.

## Remaining
- [x] Update to new ability names and wildcard delegation

package.json

@@ -51,10 +51,11 @@
     "@types/mocha": "^10.0.9",
     "@types/node-fetch": "^2.6.11",
     "@types/sinon": "^17.0.3",
-    "@ucanto/principal": "^8.1.0",
+    "@ucanto/principal": "^9.0.1",
     "@web3-storage/content-claims": "^5.0.0",
     "@web3-storage/public-bucket": "^1.1.0",
     "@web3-storage/upload-client": "^16.1.1",
+    "@web3-storage/w3cli": "^7.8.2",
     "carstream": "^2.1.0",
     "chai": "^5.1.1",
     "esbuild": "^0.18.20",

scripts/delegate-serve.d.ts

@@ -0,0 +1,4 @@
+declare module '@web3-storage/w3cli/lib.js' {
+  import { Client } from '@web3-storage/w3up-client'
+  export declare function getClient(): Promise<Client>
+}

scripts/delegate-serve.js

@@ -0,0 +1,38 @@
+import sade from 'sade'
+import { getClient } from '@web3-storage/w3cli/lib.js'
+import * as ed25519 from '@ucanto/principal/ed25519'
+import * as serve from '../src/capabilities/serve.js'
+
+const cli = sade('delegate-serve.js <space> [token]')
+
+cli
+  .describe(
+    `Delegates ${serve.star.can} to the Gateway for <space>, with an optional token. Outputs a base64url string suitable for the stub_delegation query parameter. Pipe the output to pbcopy or similar for the quickest workflow.`
+  )
+  .action(async (space, token) => {
+    const client = await getClient()
+
+    const gatewayIdentity = (await ed25519.Signer.generate()).withDID(
+      'did:web:w3s.link'
+    )
+
+    const delegation = await serve.star.delegate({
+      issuer: client.agent.issuer,
+      audience: gatewayIdentity,
+      with: space,
+      nb: { token: token ?? null },
+      expiration: Infinity,
+      proofs: client.proofs([
+        {
+          can: serve.star.can,
+          with: space
+        }
+      ])
+    })
+
+    const carResult = await delegation.archive()
+    if (carResult.error) throw carResult.error
+    process.stdout.write(Buffer.from(carResult.ok).toString('base64url'))
+  })
+
+cli.parse(process.argv)

src/capabilities/serve.js

@@ -0,0 +1,51 @@
+import { capability, Schema, DID, nullable, string } from '@ucanto/validator'
+
+/**
+ * "Manage the serving of content owned by the subject Space."
+ *
+ * A Principal who may `space/content/serve/*` is permitted to perform all
+ * operations related to serving content owned by the Space, including actually
+ * serving it and recording egress charges.
+ */
+export const star = capability({
+  can: 'space/content/serve/*',
+  /**
+   * The Space which contains the content. This Space will be charged egress
+   * fees if content is actually retrieved by way of this invocation.
+   */
+  with: DID,
+  nb: Schema.struct({
+    /** The authorization token, if any, used for this request. */
+    token: nullable(string())
+  })
+})
+
+/**
+ * "Serve content owned by the subject Space over HTTP."
+ *
+ * A Principal who may `space/content/serve/transport/http` is permitted to
+ * serve any content owned by the Space, in the manner of an [IPFS Gateway]. The
+ * content may be a Blob stored by a Storage Node, or indexed content stored
+ * within such Blobs (ie, Shards).
+ *
+ * Note that the args do not currently specify *what* content should be served.
+ * Invoking this command does not currently *serve* the content in any way, but
+ * merely validates the authority to do so. Currently, the entirety of a Space
+ * must use the same authorization, thus the content does not need to be
+ * identified. In the future, this command may refer directly to a piece of
+ * content by CID.
+ *
+ * [IPFS Gateway]: https://specs.ipfs.tech/http-gateways/path-gateway/
+ */
+export const transportHttp = capability({
+  can: 'space/content/serve/transport/http',
+  /**
+   * The Space which contains the content. This Space will be charged egress
+   * fees if content is actually retrieved by way of this invocation.
+   */
+  with: DID,
+  nb: Schema.struct({
+    /** The authorization token, if any, used for this request. */
+    token: nullable(string())
+  })
+})

src/index.js

@@ -21,9 +21,10 @@ import {
   withAuthToken,
   withCarBlockHandler,
   withRateLimit,
-  withNotFound,
+  withEgressTracker,
+  withAuthorizedSpace,
   withLocator,
-  withEgressTracker
+  withDelegationStubs
 } from './middleware/index.js'
 import { instrument } from '@microlabs/otel-cf-workers'
 import { NoopSpanProcessor } from '@opentelemetry/sdk-trace-base'
@@ -56,6 +57,7 @@ const handler = {
       createWithHttpMethod('GET', 'HEAD'),
       withAuthToken,
       withLocator,
+      withDelegationStubs,
 
       // Rate-limit requests
       withRateLimit,
@@ -65,7 +67,7 @@ const handler = {
 
       // Fetch data
       withCarBlockHandler,
-      withNotFound,
+      withAuthorizedSpace,
       withContentClaimsDagula,
       withFormatRawHandler,
       withFormatCarHandler,

src/middleware/index.js

@@ -3,6 +3,7 @@ export { withCarBlockHandler } from './withCarBlockHandler.js'
 export { withContentClaimsDagula } from './withContentClaimsDagula.js'
 export { withRateLimit } from './withRateLimit.js'
 export { withVersionHeader } from './withVersionHeader.js'
-export { withNotFound } from './withNotFound.js'
+export { withAuthorizedSpace } from './withAuthorizedSpace.js'
 export { withLocator } from './withLocator.js'
 export { withEgressTracker } from './withEgressTracker.js'
+export { withDelegationStubs } from './withDelegationStubs.js'

src/middleware/withAuthToken.js

@@ -5,16 +5,15 @@
  *  } from '@web3-storage/gateway-lib'
  * @import {
  *   Environment,
- *   InContext,
- *   OutContext,
+ *   AuthTokenContext,
  * } from './withAuthToken.types.js'
  */
 
 /**
  * Finds an authentication token in the URL query parameters or the
  * `Authorization` header and adds it to the context as `authToken`.
  *
- * @type {Middleware<OutContext<MiddlewareContext>, InContext, Environment>}
+ * @type {Middleware<AuthTokenContext, MiddlewareContext, Environment>}
  */
 export function withAuthToken (handler) {
   return async (req, env, ctx) => {

src/middleware/withAuthToken.types.ts

@@ -5,8 +5,8 @@ import {
 
 export interface Environment extends MiddlewareEnvironment {}
 
-export interface InContext extends MiddlewareContext {}
+export interface AuthTokenContextIn extends MiddlewareContext {}
 
-export type OutContext<IncomingContext> = IncomingContext & {
+export interface AuthTokenContext extends MiddlewareContext {
   authToken: string | null
 }

src/middleware/withAuthorizedSpace.js

@@ -0,0 +1,154 @@
+import { Verifier } from '@ucanto/principal'
+import { ok, access, Unauthorized } from '@ucanto/validator'
+import { HttpError } from '@web3-storage/gateway-lib/util'
+import * as serve from '../capabilities/serve.js'
+
+/**
+ * @import * as Ucanto from '@ucanto/interface'
+ * @import { Locator } from '@web3-storage/blob-fetcher'
+ * @import { IpfsUrlContext, Middleware } from '@web3-storage/gateway-lib'
+ * @import { LocatorContext } from './withLocator.types.js'
+ * @import { AuthTokenContext } from './withAuthToken.types.js'
+ * @import { SpaceContext, DelegationsStorageContext } from './withAuthorizedSpace.types.js'
+ */
+
+/**
+ * Attempts to locate the {@link IpfsUrlContext.dataCid}. If it's able to,
+ * attempts to authorize the request to access the data.
+ *
+ * @throws {HttpError} (404) If the locator tells us the data is not found, or
+ * if no located space is one the request is authorized to access.
+ * @throws {Error} If the locator fails in any other way.
+ * @type {(
+ *   Middleware<
+ *     LocatorContext & IpfsUrlContext & AuthTokenContext & DelegationsStorageContext & SpaceContext,
+ *     LocatorContext & IpfsUrlContext & AuthTokenContext & DelegationsStorageContext,
+ *     {}
+ *   >
+ * )}
+ */
+export function withAuthorizedSpace (handler) {
+  return async (request, env, ctx) => {
+    const { locator, dataCid } = ctx
+    const locRes = await locator.locate(dataCid.multihash)
+    if (locRes.error) {
+      if (locRes.error.name === 'NotFound') {
+        throw new HttpError('Not Found', { status: 404, cause: locRes.error })
+      }
+      throw new Error(`failed to locate: ${dataCid}`, { cause: locRes.error })
+    }
+
+    // Legacy behavior: Site results which have no Space attached are from
+    // before we started authorizing serving content explicitly. For these, we
+    // always serve the content, but only if the request has no authorization
+    // token.
+    const shouldServeLegacy =
+      locRes.ok.site.some((site) => site.space === undefined) &&
+      ctx.authToken === null
+
+    if (shouldServeLegacy) {
+      return handler(request, env, { ...ctx, space: null })
+    }
+
+    // These Spaces all have the content we're to serve, if we're allowed to.
+    const spaces = locRes.ok.site
+      .map((site) => site.space)
+      .filter((s) => s !== undefined)
+
+    try {
+      // First space to successfully authorize is the one we'll use.
+      const space = await Promise.any(
+        spaces.map(async (space) => {
+          const result = await authorize(space, ctx)
+          if (result.error) throw result.error
+          return space
+        })
+      )
+      return handler(request, env, {
+        ...ctx,
+        space,
+        locator: spaceScopedLocator(locator, space)
+      })
+    } catch (error) {
+      // If all Spaces failed to authorize, throw the first error.
+      if (
+        error instanceof AggregateError &&
+        error.errors.every((e) => e instanceof Unauthorized)
+      ) {
+        throw new HttpError('Not Found', { status: 404, cause: error })
+      } else {
+        throw error
+      }
+    }
+  }
+}
+
+/**
+ * Authorizes the request to serve content from the given Space. Looks for
+ * authorizing delegations in the
+ * {@link DelegationsStorageContext.delegationsStorage}.
+ *
+ * @param {Ucanto.DID} space
+ * @param {AuthTokenContext & DelegationsStorageContext} ctx
+ * @returns {Promise<Ucanto.Result<{}, Ucanto.Failure>>}
+ */
+const authorize = async (space, ctx) => {
+  // Look up delegations that might authorize us to serve the content.
+  const relevantDelegationsResult = await ctx.delegationsStorage.find({
+    audience: ctx.gatewayIdentity.did(),
+    can: serve.transportHttp.can,
+    with: space
+  })
+
+  if (relevantDelegationsResult.error) return relevantDelegationsResult
+
+  // Create an invocation of the serve capability.
+  const invocation = await serve.transportHttp
+    .invoke({
+      issuer: ctx.gatewayIdentity,
+      audience: ctx.gatewayIdentity,
+      with: space,
+      nb: {
+        token: ctx.authToken
+      },
+      proofs: relevantDelegationsResult.ok
+    })
+    .delegate()
+
+  // Validate the invocation.
+  const accessResult = await access(invocation, {
+    capability: serve.transportHttp,
+    authority: ctx.gatewayIdentity,
+    principal: Verifier,
+    validateAuthorization: () => ok({})
+  })
+
+  if (accessResult.error) {
+    return accessResult
+  }
+
+  return { ok: {} }
+}
+
+/**
+ * Wraps a {@link Locator} and locates content only from a specific Space.
+ *
+ * @param {Locator} locator
+ * @param {Ucanto.DID} space
+ * @returns {Locator}
+ */
+const spaceScopedLocator = (locator, space) => ({
+  locate: async (digest) => {
+    const locateResult = await locator.locate(digest)
+    if (locateResult.error) {
+      return locateResult
+    } else {
+      return {
+        ok: {
+          ...locateResult.ok,
+          site: locateResult.ok.site.filter((site) => site.space === space)
+        }
+      }
+    }
+  }
+})

src/middleware/withAuthorizedSpace.types.ts

@@ -0,0 +1,36 @@
+import * as Ucanto from '@ucanto/interface'
+import { Context as MiddlewareContext } from '@web3-storage/gateway-lib'
+
+export interface DelegationsStorageContext extends MiddlewareContext {
+  /** The identity of the gateway itself */
+  gatewayIdentity: Ucanto.Signer
+  delegationsStorage: DelegationsStorage
+}
+
+export interface SpaceContext extends MiddlewareContext {
+  space: Ucanto.DID | null
+}
+
+// TEMP: https://github.com/storacha/blob-fetcher/pull/13/files
+declare module '@web3-storage/blob-fetcher' {
+  interface Site {
+    space?: Ucanto.DID
+  }
+}
+
+// TEMP
+
+export interface Query {
+  audience?: Ucanto.DID
+  can: string
+  with: Ucanto.Resource
+}
+
+export interface DelegationsStorage {
+  /**
+   * find all items that match the query
+   */
+  find: (
+    query: Query
+  ) => Promise<Ucanto.Result<Ucanto.Delegation[], Ucanto.Failure>>
+}