feat: implement car check API

[laurentsenta]

Tooling to test trustless gateway - ipfs/specs#402

• bump go version to 1.20
• Implement a first car check API
• Review with @guanzo
• Implemented flexible "Exactly" and "Ordered" checks
• Implement fixture.GetCidForPath => fixture.MustGetCID("subdir", "ascii,txt")
• fixture.GetAllCidsForNode => fixture.MustGetChildrenCids("subdir")

	c1 := IsCar().
		HasBlock("bafkreidfdrlkeq4m4xnxuyx6iae76fdm4wgl5d4xzsb77ixhyqwumhz244").
		HasBlocks(
		    "bafkreidfdrlkeq4m4xnxuyx6iae76fdm4wgl5d4xzsb77ixhyqwumhz244",
		    "...",
		).
		HasRoot("bafkreidfdrlkeq4m4xnxuyx6iae76fdm4wgl5d4xzsb77ixhyqwumhz244").
		HasRoots(
		    "bafkreidfdrlkeq4m4xnxuyx6iae76fdm4wgl5d4xzsb77ixhyqwumhz244",
		    "...",
		).
		Exactly().
		InThatOrder()

In order to traverse a fixture:

     f1 := car.MustOpenUnixfsCar("some.car")
     f1.MustGetCid() // root cid
     f1.MustGetCid("subdir", "file.txt")
     f1.MustGetChildren("subdir") // returns array of children in depth-first traversal
     f1.MustGetChildrenCids("subdir")

Which means that the check for "my car should have the blocks for dir, subdir, and all the children of subdir" would look like:

...
Body(
    IsCar().
    HasBlocks(
         fixture.MustGetCid("dir"),
         fixture.MustGetCid("dir", "subdir"),
         fixture.MustGetChildrenCids("dir", "subdir")...
    ).
    Exactly().
    InThatOrder()
)

[guanzo]

Hey, here's the potential API we discussed

package tests

import (
	"testing"

	"github.com/ipfs/gateway-conformance/tooling/car"
	. "github.com/ipfs/gateway-conformance/tooling/test"
)

func TestGatewayCar(t *testing.T) {
	fixture := car.MustOpenUnixfsCar("t0118-test-dag.car")
	
	tests := SugarTests{
		{
			Name: "GET response for application/vnd.ipld.car",
			Hint: `
				CAR stream is not deterministic, as blocks can arrive in random order,
				but if we have a small file that fits into a single block, and export its CID
				we will get a CAR that is a deterministic array of bytes.
			`,
			Request: Request().
				Path("ipfs/%s/subdir/ascii.txt", fixture.MustGetCid()).
				Headers(
					Header("Accept", "application/vnd.ipld.car"),
				),
			Response: Expect().
				Status(200).
				Headers(
					Header("Content-Type").
						Hint("Expected content type to be application/vnd.ipld.car").
						Contains("application/vnd.ipld.car"),
					Header("Content-Length").
						Hint("CAR is streamed, gateway may not have the entire thing, unable to calculate total size").
						IsEmpty(),
					Header("Content-Disposition").
						Hint("Expected content disposition to be attachment; filename=\"<cid>.car\"").
						Contains("attachment; filename=\"%s.car\"", fixture.MustGetCid("subdir", "ascii.txt")),
					Header("X-Content-Type-Options").
						Hint("CAR is streamed, gateway may not have the entire thing, unable to calculate total size").
						Equals("nosniff"),
					Header("Accept-Ranges").
						Hint("CAR is streamed, gateway may not have the entire thing, unable to support range-requests. Partial downloads and resumes should be handled using IPLD selectors: https://github.com/ipfs/go-ipfs/issues/8769").
						Equals("none"),
				),
		},
		{
			Name: "GET with ?format=car&car-scope=block params returns expected blocks",
			Hint: `
				car-scope=block should return a CAR file with only the root block and a
				block for each optional path component.
			`,
			Request: Request().
				Path("ipfs/%s/subdir/ascii.txt", fixture.MustGetCid()).
				Query("format", "car").
				Query("car-scope", "block"),
			Response: Expect().
				Status(200).
				// TODO: Check if CAR response contains blocks for: root cid, subdir, ascii.txt
				Body(
					IsCar().
						HasOnlyRoots(fixture.MustGetCid()).
						HasOnlyBlocks([
							fixture.MustGetCid(),
							fixture.GetCidForPath("subdir"),
							fixture.GetCidForPath("ascii.txt")
						])
				),
		},
		{
			Name: "GET with ?format=car&car-scope=block params returns expected blocks",
			Hint: `
				car-scope=block should return a CAR file with only the root block and a
				block for each optional path component.
			`,
			Request: Request().
				Path("ipfs/%s/subdir/ascii.txt", fixture.MustGetCid()).
				Query("format", "car").
				Query("car-scope", "file"),
			Response: Expect().
				Status(200).
				// TODO: Check if CAR response contains blocks for: root cid, subdir, ascii.txt
				Body(
					IsCar().
						HasOnlyRoots(fixture.MustGetCid()).
						HasOnlyBlocks([
							fixture.MustGetCid(),
							fixture.GetCidForPath("subdir"),
							...fixture.GetAllCidsForNode("ascii.txt")
						])),
		},
		{
			Name: "GET with ?format=car&car-scope=block params returns expected blocks",
			Hint: `
				car-scope=block should return a CAR file with only the root block and a
				block for each optional path component.
			`,
			Request: Request().
				Path("ipfs/%s/subdir/ascii.txt", fixture.MustGetCid()).
				Query("format", "car").
				Query("car-scope", "all"),
			Response: Expect().
				Status(200).
				// TODO: Check if CAR response contains blocks for: root cid, subdir, ascii.txt
				Body(
					IsCar().
						HasOnlyRoots(fixture.MustGetCid()).
						HasOnlyBlocks([
							fixture.MustGetCid(),
							fixture.GetCidForPath("subdir"),
							...fixture.GetAllCidsForNode("ascii.txt")
						])),
		},
	}

	Run(t, tests)
}

[laurentsenta]

Thanks for sharing the code @guanzo, could you try the API in this branch?

See the PR description for the full API, small differences with what we discussed:

• use Exactly().InThisOrder() to test the order explicitly - I noticed there are discussion about determinism with car response; we're ready to support different expectation with this API (cc @aschmahmann)
• use MustGetChildrenCids("some", "path") to get the list of children, it should return the depth-first list of children as expected in the spec.