Merge 'origin/master' into feat/libp2p-sharedtcp

.github/workflows/gotest.yml

@@ -45,7 +45,7 @@ jobs:
           make -j "$PARALLEL" test/unit/gotest.junit.xml &&
             [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]]
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # v5.0.7
+        uses: codecov/codecov-action@7f8b4b4bde536c465e797be725718b88c5d95e0e # v5.1.1
         if: failure() || success()
         with:
           name: unittests

.github/workflows/sharness.yml

@@ -55,7 +55,7 @@ jobs:
           # increasing parallelism beyond 10 doesn't speed up the tests much
           PARALLEL: ${{ github.repository == 'ipfs/kubo' && 10 || 3 }}
       - name: Upload coverage report
-        uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # v5.0.7
+        uses: codecov/codecov-action@7f8b4b4bde536c465e797be725718b88c5d95e0e # v5.1.1
         if: failure() || success()
         with:
           name: sharness

cmd/ipfs/kubo/daemon.go

@@ -295,7 +295,7 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 		if !domigrate {
 			fmt.Println("Not running migrations of fs-repo now.")
 			fmt.Println("Please get fs-repo-migrations from https://dist.ipfs.tech")
-			return fmt.Errorf("fs-repo requires migration")
+			return errors.New("fs-repo requires migration")
 		}
 
 		// Read Migration section of IPFS config
@@ -436,7 +436,7 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 		ncfg.Routing = libp2p.NilRouterOption
 	case routingOptionCustomKwd:
 		if cfg.Routing.AcceleratedDHTClient.WithDefault(config.DefaultAcceleratedDHTClient) {
-			return fmt.Errorf("Routing.AcceleratedDHTClient option is set even tho Routing.Type is custom, using custom .AcceleratedDHTClient needs to be set on DHT routers individually")
+			return errors.New("Routing.AcceleratedDHTClient option is set even tho Routing.Type is custom, using custom .AcceleratedDHTClient needs to be set on DHT routers individually")
 		}
 		ncfg.Routing = libp2p.ConstructDelegatedRouting(
 			cfg.Routing.Routers,

cmd/ipfs/kubo/init.go

@@ -88,11 +88,11 @@ environment variable:
 				if it.Err() != nil {
 					return it.Err()
 				}
-				return fmt.Errorf("file argument was nil")
+				return errors.New("file argument was nil")
 			}
 			file := files.FileFromEntry(it)
 			if file == nil {
-				return fmt.Errorf("expected a regular file")
+				return errors.New("expected a regular file")
 			}
 
 			conf = &config.Config{}

cmd/ipfs/kubo/pinmfs.go

@@ -90,34 +90,46 @@ func pinMFSOnChange(cctx pinMFSContext, configPollInterval time.Duration, node p
 		case <-cctx.Context().Done():
 			return
 		case <-tmo.C:
-			tmo.Reset(configPollInterval)
-		}
-
-		// reread the config, which may have changed in the meantime
-		cfg, err := cctx.GetConfig()
-		if err != nil {
-			mfslog.Errorf("pinning reading config (%v)", err)
-			continue
-		}
-		mfslog.Debugf("pinning loop is awake, %d remote services", len(cfg.Pinning.RemoteServices))
+			// reread the config, which may have changed in the meantime
+			cfg, err := cctx.GetConfig()
+			if err != nil {
+				mfslog.Errorf("pinning reading config (%v)", err)
+				continue
+			}
+			mfslog.Debugf("pinning loop is awake, %d remote services", len(cfg.Pinning.RemoteServices))
 
-		// get the most recent MFS root cid
-		rootNode, err := node.RootNode()
-		if err != nil {
-			mfslog.Errorf("pinning reading MFS root (%v)", err)
-			continue
+			// pin to all remote services in parallel
+			pinAllMFS(cctx.Context(), node, cfg, lastPins)
 		}
-
-		// pin to all remote services in parallel
-		pinAllMFS(cctx.Context(), node, cfg, rootNode.Cid(), lastPins)
+		// pinAllMFS may take long. Reset interval only when we are done doing it
+		// so that we are not pinning constantly.
+		tmo.Reset(configPollInterval)
 	}
 }
 
 // pinAllMFS pins on all remote services in parallel to overcome DoS attacks.
-func pinAllMFS(ctx context.Context, node pinMFSNode, cfg *config.Config, rootCid cid.Cid, lastPins map[string]lastPin) {
+func pinAllMFS(ctx context.Context, node pinMFSNode, cfg *config.Config, lastPins map[string]lastPin) {
 	ch := make(chan lastPin)
 	var started int
 
+	// Bail out to mitigate issue below when not needing to do anything.
+	if len(cfg.Pinning.RemoteServices) == 0 {
+		return
+	}
+
+	// get the most recent MFS root cid.
+	// Warning! This can be super expensive.
+	// See https://github.com/ipfs/boxo/pull/751
+	// and https://github.com/ipfs/kubo/issues/8694
+	// Reading an MFS-directory nodes can take minutes due to
+	// ever growing cache being synced to unixfs.
+	rootNode, err := node.RootNode()
+	if err != nil {
+		mfslog.Errorf("pinning reading MFS root (%v)", err)
+		return
+	}
+	rootCid := rootNode.Cid()
+
 	for svcName, svcConfig := range cfg.Pinning.RemoteServices {
 		if ctx.Err() != nil {
 			break

cmd/ipfs/kubo/pinmfs_test.go

@@ -94,11 +94,24 @@ func TestPinMFSRootNodeError(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 2*testConfigPollInterval)
 	defer cancel()
 
+	// need at least one config to trigger
+	cfg := &config.Config{
+		Pinning: config.Pinning{
+			RemoteServices: map[string]config.RemotePinningService{
+				"A": {
+					Policies: config.RemotePinningServicePolicies{
+						MFS: config.RemotePinningServiceMFSPolicy{
+							Enable: false,
+						},
+					},
+				},
+			},
+		},
+	}
+
 	cctx := &testPinMFSContext{
 		ctx: ctx,
-		cfg: &config.Config{
-			Pinning: config.Pinning{},
-		},
+		cfg: cfg,
 		err: nil,
 	}
 	node := &testPinMFSNode{

core/commands/add.go

@@ -25,7 +25,7 @@ import (
 )
 
 // ErrDepthLimitExceeded indicates that the max depth has been exceeded.
-var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")
+var ErrDepthLimitExceeded = errors.New("depth limit exceeded")
 
 type TimeParts struct {
 	t *time.Time

core/commands/cat.go

@@ -2,7 +2,7 @@ package commands
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"io"
 	"os"
 
@@ -43,13 +43,13 @@ var CatCmd = &cmds.Command{
 
 		offset, _ := req.Options[offsetOptionName].(int64)
 		if offset < 0 {
-			return fmt.Errorf("cannot specify negative offset")
+			return errors.New("cannot specify negative offset")
 		}
 
 		max, found := req.Options[lengthOptionName].(int64)
 
 		if max < 0 {
-			return fmt.Errorf("cannot specify negative length")
+			return errors.New("cannot specify negative length")
 		}
 		if !found {
 			max = -1

core/commands/cid.go

@@ -1,6 +1,7 @@
 package commands
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"sort"
@@ -85,10 +86,10 @@ The optional format string is a printf style format string:
 			}
 		case "0":
 			if opts.newCodec != 0 && opts.newCodec != cid.DagProtobuf {
-				return fmt.Errorf("cannot convert to CIDv0 with any codec other than dag-pb")
+				return errors.New("cannot convert to CIDv0 with any codec other than dag-pb")
 			}
 			if baseStr != "" && baseStr != "base58btc" {
-				return fmt.Errorf("cannot convert to CIDv0 with any multibase other than the implicit base58btc")
+				return errors.New("cannot convert to CIDv0 with any multibase other than the implicit base58btc")
 			}
 			opts.verConv = toCidV0
 		case "1":

core/commands/dht.go

@@ -78,7 +78,7 @@ var queryDhtCmd = &cmds.Command{
 		}
 
 		if d, ok := client.(kademlia); !ok {
-			return fmt.Errorf("dht client does not support GetClosestPeers")
+			return errors.New("dht client does not support GetClosestPeers")
 		} else {
 			errCh := make(chan error, 1)
 			go func() {

core/commands/files.go

@@ -327,7 +327,7 @@ func statNode(nd ipld.Node, enc cidenc.Encoder) (*statOutput, error) {
 			Type:           "file",
 		}, nil
 	default:
-		return nil, fmt.Errorf("not unixfs node (proto or raw)")
+		return nil, errors.New("not unixfs node (proto or raw)")
 	}
 }
 

core/commands/id.go

@@ -81,7 +81,7 @@ EXAMPLE:
 			var err error
 			id, err = peer.Decode(req.Arguments[0])
 			if err != nil {
-				return fmt.Errorf("invalid peer id")
+				return errors.New("invalid peer id")
 			}
 		} else {
 			id = n.Identity

core/commands/keystore.go

@@ -5,6 +5,7 @@ import (
 	"crypto/ed25519"
 	"crypto/x509"
 	"encoding/pem"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -101,12 +102,12 @@ var keyGenCmd = &cmds.Command{
 
 		typ, f := req.Options[keyStoreTypeOptionName].(string)
 		if !f {
-			return fmt.Errorf("please specify a key type with --type")
+			return errors.New("please specify a key type with --type")
 		}
 
 		name := req.Arguments[0]
 		if name == "self" {
-			return fmt.Errorf("cannot create key with name 'self'")
+			return errors.New("cannot create key with name 'self'")
 		}
 
 		opts := []options.KeyGenerateOption{options.Key.Type(typ)}

core/commands/p2p.go

@@ -250,7 +250,7 @@ func checkPort(target ma.Multiaddr) error {
 		if sport != "" {
 			return sport, nil
 		}
-		return "", fmt.Errorf("address does not contain tcp or udp protocol")
+		return "", errors.New("address does not contain tcp or udp protocol")
 	}
 
 	sport, err := getPort()
@@ -264,7 +264,7 @@ func checkPort(target ma.Multiaddr) error {
 	}
 
 	if port == 0 {
-		return fmt.Errorf("port can not be 0")
+		return errors.New("port can not be 0")
 	}
 
 	return nil

core/commands/routing.go

@@ -70,7 +70,7 @@ var findProvidersRoutingCmd = &cmds.Command{
 
 		numProviders, _ := req.Options[numProvidersOptionName].(int)
 		if numProviders < 1 {
-			return fmt.Errorf("number of providers must be greater than 0")
+			return errors.New("number of providers must be greater than 0")
 		}
 
 		c, err := cid.Parse(req.Arguments[0])

core/commands/stat.go

@@ -1,6 +1,7 @@
 package commands
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -100,7 +101,7 @@ Example:
 		}
 
 		if nd.Reporter == nil {
-			return fmt.Errorf("bandwidth reporter disabled in config")
+			return errors.New("bandwidth reporter disabled in config")
 		}
 
 		pstr, pfound := req.Options[statPeerOptionName].(string)

core/coreapi/coreapi.go

@@ -218,7 +218,7 @@ func (api *CoreAPI) WithOptions(opts ...options.ApiOption) (coreiface.CoreAPI, e
 			cs = node.DefaultIpnsCacheSize
 		}
 		if cs < 0 {
-			return nil, fmt.Errorf("cannot specify negative resolve cache size")
+			return nil, errors.New("cannot specify negative resolve cache size")
 		}
 
 		nsOptions := []namesys.Option{

core/coreapi/key.go

@@ -65,7 +65,7 @@ func (api *KeyAPI) Generate(ctx context.Context, name string, opts ...caopts.Key
 	}
 
 	if name == "self" {
-		return nil, fmt.Errorf("cannot create key with name 'self'")
+		return nil, errors.New("cannot create key with name 'self'")
 	}
 
 	_, err = api.repo.Keystore().Get(name)
@@ -168,11 +168,11 @@ func (api *KeyAPI) Rename(ctx context.Context, oldName string, newName string, o
 	ks := api.repo.Keystore()
 
 	if oldName == "self" {
-		return nil, false, fmt.Errorf("cannot rename key with name 'self'")
+		return nil, false, errors.New("cannot rename key with name 'self'")
 	}
 
 	if newName == "self" {
-		return nil, false, fmt.Errorf("cannot overwrite key with name 'self'")
+		return nil, false, errors.New("cannot overwrite key with name 'self'")
 	}
 
 	oldKey, err := ks.Get(oldName)
@@ -232,7 +232,7 @@ func (api *KeyAPI) Remove(ctx context.Context, name string) (coreiface.Key, erro
 	ks := api.repo.Keystore()
 
 	if name == "self" {
-		return nil, fmt.Errorf("cannot remove key with name 'self'")
+		return nil, errors.New("cannot remove key with name 'self'")
 	}
 
 	removed, err := ks.Get(name)

core/coreapi/name.go

@@ -2,6 +2,7 @@ package coreapi
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -214,5 +215,5 @@ func keylookup(self ci.PrivKey, kstore keystore.Keystore, k string) (ci.PrivKey,
 		}
 	}
 
-	return nil, fmt.Errorf("no key by the given name or PeerID was found")
+	return nil, errors.New("no key by the given name or PeerID was found")
 }

core/coreapi/routing.go

@@ -109,7 +109,7 @@ func (api *RoutingAPI) FindProviders(ctx context.Context, p path.Path, opts ...c
 
 	numProviders := settings.NumProviders
 	if numProviders < 1 {
-		return nil, fmt.Errorf("number of providers must be greater than 0")
+		return nil, errors.New("number of providers must be greater than 0")
 	}
 
 	pchan := api.routing.FindProvidersAsync(ctx, rp.RootCid(), numProviders)

core/coreapi/unixfs.go

@@ -72,7 +72,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
 	//}
 
 	if settings.NoCopy && !(cfg.Experimental.FilestoreEnabled || cfg.Experimental.UrlstoreEnabled) {
-		return path.ImmutablePath{}, fmt.Errorf("either the filestore or the urlstore must be enabled to use nocopy, see: https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-filestore")
+		return path.ImmutablePath{}, errors.New("either the filestore or the urlstore must be enabled to use nocopy, see: https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#ipfs-filestore")
 	}
 
 	addblockstore := api.blockstore

core/node/groups.go

@@ -301,7 +301,7 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
 		ipnsCacheSize = DefaultIpnsCacheSize
 	}
 	if ipnsCacheSize < 0 {
-		return fx.Error(fmt.Errorf("cannot specify negative resolve cache size"))
+		return fx.Error(errors.New("cannot specify negative resolve cache size"))
 	}
 
 	// Republisher params

core/node/libp2p/rcmgr.go

@@ -3,6 +3,7 @@ package libp2p
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -26,7 +27,7 @@ var rcmgrLogger = logging.Logger("rcmgr")
 
 const NetLimitTraceFilename = "rcmgr.json.gz"
 
-var ErrNoResourceMgr = fmt.Errorf("missing ResourceMgr: make sure the daemon is running with Swarm.ResourceMgr.Enabled")
+var ErrNoResourceMgr = errors.New("missing ResourceMgr: make sure the daemon is running with Swarm.ResourceMgr.Enabled")
 
 func ResourceManager(repoPath string, cfg config.SwarmConfig, userResourceOverrides rcmgr.PartialLimitConfig) interface{} {
 	return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo repo.Repo) (network.ResourceManager, Libp2pOpts, error) {