test: deterministic ipns fixtures during sharness gateway tests (#9667)

Author: laurentsenta

core/commands/routing.go

@@ -10,6 +10,8 @@ import (
 
 	cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
 
+	iface "github.com/ipfs/boxo/coreiface"
+	"github.com/ipfs/boxo/coreiface/options"
 	dag "github.com/ipfs/boxo/ipld/merkledag"
 	path "github.com/ipfs/boxo/path"
 	cid "github.com/ipfs/go-cid"
@@ -19,6 +21,16 @@ import (
 	routing "github.com/libp2p/go-libp2p/core/routing"
 )
 
+var (
+	errAllowOffline = errors.New("can't put while offline: pass `--allow-offline` to override")
+)
+
+const (
+	dhtVerboseOptionName   = "verbose"
+	numProvidersOptionName = "num-providers"
+	allowOfflineOptionName = "allow-offline"
+)
+
 var RoutingCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline:          "Issue routing commands.",
@@ -34,14 +46,6 @@ var RoutingCmd = &cmds.Command{
 	},
 }
 
-const (
-	dhtVerboseOptionName = "verbose"
-)
-
-const (
-	numProvidersOptionName = "num-providers"
-)
-
 var findProvidersRoutingCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline:          "Find peers that can provide a specific value, given a key.",
@@ -420,6 +424,9 @@ identified by QmFoo.
 		cmds.StringArg("key", true, false, "The key to store the value at."),
 		cmds.FileArg("value-file", true, false, "A path to a file containing the value to store.").EnableStdin(),
 	},
+	Options: []cmds.Option{
+		cmds.BoolOption(allowOfflineOptionName, "When offline, save the IPNS record to the the local datastore without broadcasting to the network instead of simply failing."),
+	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
 		api, err := cmdenv.GetApi(env, req)
 		if err != nil {
@@ -437,13 +444,22 @@ identified by QmFoo.
 			return err
 		}
 
-		err = api.Routing().Put(req.Context, req.Arguments[0], data)
+		allowOffline, _ := req.Options[allowOfflineOptionName].(bool)
+
+		opts := []options.RoutingPutOption{
+			options.Put.AllowOffline(allowOffline),
+		}
+
+		err = api.Routing().Put(req.Context, req.Arguments[0], data, opts...)
 		if err != nil {
 			return err
 		}
 
 		id, err := api.Key().Self(req.Context)
 		if err != nil {
+			if err == iface.ErrOffline {
+				err = errAllowOffline
+			}
 			return err
 		}
 

core/coreapi/routing.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 
 	coreiface "github.com/ipfs/boxo/coreiface"
+	caopts "github.com/ipfs/boxo/coreiface/options"
 	"github.com/ipfs/boxo/path"
 	peer "github.com/libp2p/go-libp2p/core/peer"
 )
@@ -24,9 +25,15 @@ func (r *RoutingAPI) Get(ctx context.Context, key string) ([]byte, error) {
 	return r.routing.GetValue(ctx, dhtKey)
 }
 
-func (r *RoutingAPI) Put(ctx context.Context, key string, value []byte) error {
-	if !r.nd.IsOnline {
-		return coreiface.ErrOffline
+func (r *RoutingAPI) Put(ctx context.Context, key string, value []byte, opts ...caopts.RoutingPutOption) error {
+	options, err := caopts.RoutingPutOptions(opts...)
+	if err != nil {
+		return err
+	}
+
+	err = r.checkOnline(options.AllowOffline)
+	if err != nil {
+		return err
 	}
 
 	dhtKey, err := normalizeKey(key)

core/coreapi/test/api_test.go

@@ -31,7 +31,7 @@ const testPeerID = "QmTFauExutTsy4XP6JbMFcw2Wa9645HJt2bTqL6qYDCKfe"
 
 type NodeProvider struct{}
 
-func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int) ([]coreiface.CoreAPI, error) {
+func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, online bool, n int) ([]coreiface.CoreAPI, error) {
 	mn := mocknet.New()
 
 	nodes := make([]*core.IpfsNode, n)
@@ -82,7 +82,7 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 			Routing: libp2p.DHTServerOption,
 			Repo:    r,
 			Host:    mock.MockHostOption(mn),
-			Online:  fullIdentity,
+			Online:  online,
 			ExtraOpts: map[string]bool{
 				"pubsub": true,
 			},
@@ -102,15 +102,17 @@ func (NodeProvider) MakeAPISwarm(ctx context.Context, fullIdentity bool, n int)
 		return nil, err
 	}
 
-	bsinf := bootstrap.BootstrapConfigWithPeers(
-		[]peer.AddrInfo{
-			nodes[0].Peerstore.PeerInfo(nodes[0].Identity),
-		},
-	)
+	if online {
+		bsinf := bootstrap.BootstrapConfigWithPeers(
+			[]peer.AddrInfo{
+				nodes[0].Peerstore.PeerInfo(nodes[0].Identity),
+			},
+		)
 
-	for _, n := range nodes[1:] {
-		if err := n.Bootstrap(bsinf); err != nil {
-			return nil, err
+		for _, n := range nodes[1:] {
+			if err := n.Bootstrap(bsinf); err != nil {
+				return nil, err
+			}
 		}
 	}
 

core/coreapi/test/path_test.go

@@ -19,7 +19,7 @@ func TestPathUnixFSHAMTPartial(t *testing.T) {
 	defer cancel()
 
 	// Create a node
-	apis, err := NodeProvider{}.MakeAPISwarm(ctx, true, 1)
+	apis, err := NodeProvider{}.MakeAPISwarm(ctx, true, true, 1)
 	if err != nil {
 		t.Fatal(err)
 	}

docs/examples/kubo-as-a-library/go.mod

@@ -7,7 +7,7 @@ go 1.18
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/boxo v0.8.1
+	github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866
 	github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
 	github.com/libp2p/go-libp2p v0.27.1
 	github.com/multiformats/go-multiaddr v0.9.0

docs/examples/kubo-as-a-library/go.sum

@@ -321,8 +321,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.8.1 h1:3DkKBCK+3rdEB5t77WDShUXXhktYwH99mkAsgajsKrU=
-github.com/ipfs/boxo v0.8.1/go.mod h1:xJ2hVb4La5WyD7GvKYE0lq2g1rmQZoCD2K4WNrV6aZI=
+github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866 h1:ThRTXD/EyoLb/jz+YW+ZlOLbjX9FyaxP0dEpgUp3cCE=
+github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866/go.mod h1:bORAHrH6hUtDZjbzTEaLrSpTdyhHKDIpjDRT+A14B7w=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/ipfs/boxo v0.8.1
+	github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866
 	github.com/ipfs/go-block-format v0.1.2
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-cidutil v0.1.0

go.sum

@@ -356,8 +356,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.8.1 h1:3DkKBCK+3rdEB5t77WDShUXXhktYwH99mkAsgajsKrU=
-github.com/ipfs/boxo v0.8.1/go.mod h1:xJ2hVb4La5WyD7GvKYE0lq2g1rmQZoCD2K4WNrV6aZI=
+github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866 h1:ThRTXD/EyoLb/jz+YW+ZlOLbjX9FyaxP0dEpgUp3cCE=
+github.com/ipfs/boxo v0.8.2-0.20230503105907-8059f183d866/go.mod h1:bORAHrH6hUtDZjbzTEaLrSpTdyhHKDIpjDRT+A14B7w=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=

test/sharness/t0114-gateway-subdomains.sh

@@ -93,40 +93,29 @@ test_launch_ipfs_daemon_without_network
 
 # Import test case
 # See the static fixtures in ./t0114-gateway-subdomains/
-test_expect_success "Add the test fixtures" '
-  ipfs dag import ../t0114-gateway-subdomains/fixtures.car
-'
-CID_VAL="hello"
+CID_VAL=hello
 CIDv1=bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am
 CIDv0=QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN
-# CIDv0to1 is necessary because raw-leaves are enabled by default during
-# "ipfs add" with CIDv1 and disabled with CIDv0
 CIDv0to1=bafybeiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4
 CIDv1_TOO_LONG=bafkrgqhhyivzstcz3hhswshfjgy6ertgmnqeleynhwt4dlfsthi4hn7zgh4uvlsb5xncykzapi3ocd4lzogukir6ksdy6wzrnz6ohnv4aglcs
-DIR_CID=bafybeiht6dtwk3les7vqm6ibpvz6qpohidvlshsfyr7l5mpysdw2vmbbhe # ./testdirlisting
-
-test_expect_success "Publish test text file to IPNS using RSA keys" '
-  RSA_KEY=$(ipfs key gen --ipns-base=b58mh --type=rsa --size=2048 test_key_rsa | head -n1 | tr -d "\n")
-  RSA_IPNS_IDv0=$(echo "$RSA_KEY" | ipfs cid format -v 0)
-  RSA_IPNS_IDv1=$(echo "$RSA_KEY" | ipfs cid format -v 1 --mc libp2p-key -b base36)
-  RSA_IPNS_IDv1_DAGPB=$(echo "$RSA_IPNS_IDv0" | ipfs cid format -v 1 -b base36)
-  test_check_peerid "${RSA_KEY}" &&
-  ipfs name publish --key test_key_rsa --allow-offline -Q "/ipfs/$CIDv1" > name_publish_out &&
-  ipfs name resolve "$RSA_KEY"  > output &&
-  printf "/ipfs/%s\n" "$CIDv1" > expected2 &&
-  test_cmp expected2 output
-'
+DIR_CID=bafybeiht6dtwk3les7vqm6ibpvz6qpohidvlshsfyr7l5mpysdw2vmbbhe
+
+RSA_KEY=QmVujd5Vb7moysJj8itnGufN7MEtPRCNHkKpNuA4onsRa3
+RSA_IPNS_IDv0=QmVujd5Vb7moysJj8itnGufN7MEtPRCNHkKpNuA4onsRa3
+RSA_IPNS_IDv1=k2k4r8m7xvggw5pxxk3abrkwyer625hg01hfyggrai7lk1m63fuihi7w
+RSA_IPNS_IDv1_DAGPB=k2jmtxu61bnhrtj301lw7zizknztocdbeqhxgv76l2q9t36fn9jbzipo
 
-test_expect_success "Publish test text file to IPNS using ED25519 keys" '
-  ED25519_KEY=$(ipfs key gen --ipns-base=b58mh --type=ed25519 test_key_ed25519 | head -n1 | tr -d "\n")
-  ED25519_IPNS_IDv0=$ED25519_KEY
-  ED25519_IPNS_IDv1=$(ipfs key list -l --ipns-base=base36 | grep test_key_ed25519 | cut -d " " -f1 | tr -d "\n")
-  ED25519_IPNS_IDv1_DAGPB=$(echo "$ED25519_IPNS_IDv1" | ipfs cid format -v 1 -b base36 --mc dag-pb)
-  test_check_peerid "${ED25519_KEY}" &&
-  ipfs name publish --key test_key_ed25519 --allow-offline -Q "/ipfs/$CIDv1" > name_publish_out &&
-  ipfs name resolve "$ED25519_KEY"  > output &&
-  printf "/ipfs/%s\n" "$CIDv1" > expected2 &&
-  test_cmp expected2 output
+ED25519_KEY=12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d
+ED25519_IPNS_IDv0=12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d
+ED25519_IPNS_IDv1=k51qzi5uqu5dk3v4rmjber23h16xnr23bsggmqqil9z2gduiis5se8dht36dam
+ED25519_IPNS_IDv1_DAGPB=k50rm9yjlt0jey4fqg6wafvqprktgbkpgkqdg27tpqje6iimzxewnhvtin9hhq
+IPNS_ED25519_B58MH=12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d
+IPNS_ED25519_B36CID=k51qzi5uqu5dk3v4rmjber23h16xnr23bsggmqqil9z2gduiis5se8dht36dam
+
+test_expect_success "Add the test fixtures" '
+  ipfs dag import ../t0114-gateway-subdomains/fixtures.car &&
+  ipfs routing put --allow-offline /ipns/${RSA_KEY} ../t0114-gateway-subdomains/${RSA_KEY}.ipns-record &&
+  ipfs routing put --allow-offline /ipns/${ED25519_KEY} ../t0114-gateway-subdomains/${ED25519_KEY}.ipns-record
 '
 
 # ensure we start with empty Gateway.PublicGateways
@@ -588,11 +577,6 @@ test_expect_success \
 ## https://github.com/ipfs/go-ipfs/issues/7318
 ## ============================================================================
 
-# ed25519 fits under 63 char limit when represented in base36
-IPNS_KEY="test_key_ed25519"
-IPNS_ED25519_B58MH=$(ipfs key list -l --ipns-base b58mh | grep $IPNS_KEY | cut -d" " -f1 | tr -d "\n")
-IPNS_ED25519_B36CID=$(ipfs key list -l --ipns-base base36 | grep $IPNS_KEY | cut -d" " -f1 | tr -d "\n")
-
 # local: *.localhost
 test_localhost_gateway_response_should_contain \
   "request for a ED25519 libp2p-key at localhost/ipns/{b58mh} returns Location HTTP header for DNS-safe subdomain redirect in browsers" \