diff --git a/.github/workflows/docker-build-publish.yml b/.github/workflows/docker-build-publish.yml index 00565ffd0f..1c0abb9d18 100644 --- a/.github/workflows/docker-build-publish.yml +++ b/.github/workflows/docker-build-publish.yml @@ -20,3 +20,11 @@ jobs: uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.2.2 # yamllint disable-line rule:line-length with: dockerfile: Dockerfile + + docker-txsim-build: + permissions: + contents: write + packages: write + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.2.2 + with: + dockerfile: docker/Dockerfile_txsim diff --git a/Makefile b/Makefile index e3c9896021..e2233bb280 100644 --- a/Makefile +++ b/Makefile @@ -147,6 +147,12 @@ test-short: @go test -mod=readonly ./... -short .PHONY: test-short +## test-e2e: Run end to end tests via knuu. +test-e2e: + @echo "--> Running e2e tests" + @KNUU_NAMESPACE=celestia-app E2E=true go test -mod=readonly ./test/e2e/... -timeout 30m +.PHONY: test-e2e + ## test-race: Run unit tests in race mode. test-race: @echo "--> Running tests in race mode" @@ -165,6 +171,26 @@ test-coverage: @export VERSION=$(VERSION); bash -x scripts/test_cover.sh .PHONY: test-coverage +## txsim-install: Install the tx simulator. +txsim-install: + @echo "--> Installing tx simulator" + @go install -mod=readonly $(BUILD_FLAGS) ./test/cmd/txsim +.PHONY: txsim-install + +## txsim-build: Build the tx simulator binary into the ./build directory. +txsim-build: + @echo "--> Building tx simulator" + @cd ./test/cmd/txsim + @mkdir -p build/ + @go build $(BUILD_FLAGS) -o build/ ./test/cmd/txsim + @go mod tidy -compat=1.20 +.PHONY: txsim-build + +## docker-txsim-build: Build the Docker image tx simulator. +txsim-build-docker: + docker build -t ghcr.io/celestiaorg/txsim -f docker/Dockerfile_txsim . +.PHONY: txsim-build-docker + ## adr-gen: Download the ADR template from the celestiaorg/.github repo. Ex. `make adr-gen` adr-gen: @echo "--> Downloading ADR template" diff --git a/app/errors/errors.go b/app/errors/errors.go new file mode 100644 index 0000000000..c45658f664 --- /dev/null +++ b/app/errors/errors.go @@ -0,0 +1,80 @@ +package errors + +import ( + "errors" + "fmt" + "regexp" + "strconv" + + sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" +) + +var ( + // This is relatively brittle. It would be better if going below the min gas price + // had a specific error type. + regexpMinGasPrice = regexp.MustCompile(`insufficient fees; got: \d+utia required: \d+utia`) + regexpInt = regexp.MustCompile(`[0-9]+`) +) + +// ParseInsufficientMinGasPrice checks if the error is due to the gas price being too low. +// Given the previous gas price and gas limit, it returns the new minimum gas price that +// the node should accept. +// If the error is not due to the gas price being too low, it returns 0, nil +func ParseInsufficientMinGasPrice(err error, gasPrice float64, gasLimit uint64) (float64, error) { + // first work out if the error is ErrInsufficientFunds + if err == nil || !sdkerrors.ErrInsufficientFee.Is(err) { + return 0, nil + } + + // As there are multiple cases of ErrInsufficientFunds, we need to check the error message + // matches the regexp + substr := regexpMinGasPrice.FindAllString(err.Error(), -1) + if len(substr) != 1 { + return 0, nil + } + + // extract the first and second numbers from the error message (got and required) + numbers := regexpInt.FindAllString(substr[0], -1) + if len(numbers) != 2 { + return 0, fmt.Errorf("expected two numbers in error message got %d", len(numbers)) + } + + // attempt to parse them into float64 values + got, err := strconv.ParseFloat(numbers[0], 64) + if err != nil { + return 0, err + } + required, err := strconv.ParseFloat(numbers[1], 64) + if err != nil { + return 0, err + } + + // catch rare condition that required is zero. This should theoretically + // never happen as a min gas price of zero should always be accepted. + if required == 0 { + return 0, errors.New("unexpected case: required gas price is zero (why was an error returned)") + } + + // calculate the actual min gas price of the node based on the difference + // between the got and required values. If gas price was zero, we need to use + // the gasLimit to infer this. + if gasPrice == 0 || got == 0 { + if gasLimit == 0 { + return 0, fmt.Errorf("gas limit and gas price cannot be zero") + } + return required / float64(gasLimit), nil + } + return required / got * gasPrice, nil +} + +// IsInsufficientMinGasPrice checks if the error is due to the gas price being too low. +func IsInsufficientMinGasPrice(err error) bool { + // first work out if the error is ErrInsufficientFunds + if err == nil || !sdkerrors.ErrInsufficientFee.Is(err) { + return false + } + + // As there are multiple cases of ErrInsufficientFunds, we need to check the error message + // matches the regexp + return regexpMinGasPrice.MatchString(err.Error()) +} diff --git a/app/errors/errors_test.go b/app/errors/errors_test.go new file mode 100644 index 0000000000..071f110aac --- /dev/null +++ b/app/errors/errors_test.go @@ -0,0 +1,145 @@ +package errors_test + +import ( + "fmt" + "testing" + + "cosmossdk.io/errors" + "github.com/celestiaorg/celestia-app/app" + apperr "github.com/celestiaorg/celestia-app/app/errors" + "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/namespace" + testutil "github.com/celestiaorg/celestia-app/test/util" + blob "github.com/celestiaorg/celestia-app/x/blob/types" + sdk "github.com/cosmos/cosmos-sdk/types" + sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" + "github.com/cosmos/cosmos-sdk/x/auth/ante" + "github.com/stretchr/testify/require" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" +) + +// This will detect any changes to the DeductFeeDecorator which may cause a +// different error message that does not match the regexp. +func TestInsufficientMinGasPriceIntegration(t *testing.T) { + var ( + gasLimit uint64 = 1_000_000 + feeAmount int64 = 10 + gasPrice = float64(feeAmount) / float64(gasLimit) + ) + account := "test" + testApp, kr := testutil.SetupTestAppWithGenesisValSet(app.DefaultConsensusParams(), account) + minGasPrice, err := sdk.ParseDecCoins(fmt.Sprintf("%v%s", appconsts.DefaultMinGasPrice, app.BondDenom)) + require.NoError(t, err) + ctx := testApp.NewContext(true, tmproto.Header{}).WithMinGasPrices(minGasPrice) + signer := blob.NewKeyringSigner(kr, account, testutil.ChainID) + builder := signer.NewTxBuilder() + builder.SetGasLimit(gasLimit) + builder.SetFeeAmount(sdk.NewCoins(sdk.NewCoin(app.BondDenom, sdk.NewInt(feeAmount)))) + + address, err := signer.GetSignerInfo().GetAddress() + require.NoError(t, err) + + _, err = sdk.AccAddressFromBech32(address.String()) + require.NoError(t, err, address) + + b, err := blob.NewBlob(namespace.RandomNamespace(), []byte("hello world"), 0) + require.NoError(t, err) + + pfb, err := blob.NewMsgPayForBlobs(address.String(), b) + require.NoError(t, err, address) + + tx, err := signer.BuildSignedTx(builder, pfb) + require.NoError(t, err) + + decorator := ante.NewDeductFeeDecorator(testApp.AccountKeeper, testApp.BankKeeper, testApp.FeeGrantKeeper, nil) + anteHandler := sdk.ChainAnteDecorators(decorator) + + _, err = anteHandler(ctx, tx, false) + require.True(t, apperr.IsInsufficientMinGasPrice(err)) + actualGasPrice, err := apperr.ParseInsufficientMinGasPrice(err, gasPrice, gasLimit) + require.NoError(t, err) + require.Equal(t, appconsts.DefaultMinGasPrice, actualGasPrice, err) +} + +func TestInsufficientMinGasPriceTable(t *testing.T) { + testCases := []struct { + name string + err error + inputGasPrice float64 + inputGasLimit uint64 + isInsufficientMinGasPriceErr bool + expectParsingError bool + expectedGasPrice float64 + }{ + { + name: "nil error", + err: nil, + isInsufficientMinGasPriceErr: false, + }, + { + name: "not insufficient fee error", + err: errors.Wrap(sdkerrors.ErrInsufficientFee, "not enough gas to pay for blobs (minimum: 1000000, got: 100000)"), + isInsufficientMinGasPriceErr: false, + }, + { + name: "not insufficient fee error 2", + err: errors.Wrap(sdkerrors.ErrInsufficientFunds, "not enough gas to pay for blobs (got: 1000000, required: 100000)"), + isInsufficientMinGasPriceErr: false, + }, + { + name: "insufficient fee error", + err: errors.Wrap(sdkerrors.ErrInsufficientFee, "insufficient fees; got: 10utia required: 100utia"), + inputGasPrice: 0.01, + expectedGasPrice: 0.1, + isInsufficientMinGasPriceErr: true, + }, + { + name: "insufficient fee error with zero gas price", + err: errors.Wrap(sdkerrors.ErrInsufficientFee, "insufficient fees; got: 0utia required: 100utia"), + inputGasPrice: 0, + inputGasLimit: 100, + expectedGasPrice: 1, + isInsufficientMinGasPriceErr: true, + }, + { + name: "insufficient fee error with zero gas price and zero gas limit", + err: errors.Wrap(sdkerrors.ErrInsufficientFee, "insufficient fees; got: 0utia required: 100utia"), + inputGasPrice: 0, + inputGasLimit: 0, + isInsufficientMinGasPriceErr: true, + expectParsingError: true, + }, + { + name: "incorrectly formatted error", + err: errors.Wrap(sdkerrors.ErrInsufficientFee, "insufficient fees; got: 0uatom required: 100uatom"), + isInsufficientMinGasPriceErr: false, + }, + { + name: "error with zero required gas price", + err: errors.Wrap(sdkerrors.ErrInsufficientFee, "insufficient fees; got: 10utia required: 0utia"), + isInsufficientMinGasPriceErr: true, + expectParsingError: true, + }, + { + name: "error with extra wrapping", + err: errors.Wrap(errors.Wrap(sdkerrors.ErrInsufficientFee, "insufficient fees; got: 10utia required: 100utia"), "extra wrapping"), + inputGasPrice: 0.01, + expectedGasPrice: 0.1, + isInsufficientMinGasPriceErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.isInsufficientMinGasPriceErr, apperr.IsInsufficientMinGasPrice(tc.err)) + actualGasPrice, err := apperr.ParseInsufficientMinGasPrice(tc.err, tc.inputGasPrice, tc.inputGasLimit) + if tc.expectParsingError { + require.Error(t, err) + require.Zero(t, actualGasPrice) + } else { + require.NoError(t, err) + require.Equal(t, tc.expectedGasPrice, actualGasPrice) + } + }) + } +} diff --git a/docker/Dockerfile_txsim b/docker/Dockerfile_txsim new file mode 100644 index 0000000000..e5c07ffcb2 --- /dev/null +++ b/docker/Dockerfile_txsim @@ -0,0 +1,49 @@ +# Stage 1: generate celestia-appd binary +FROM docker.io/golang:1.20.6-alpine3.17 as builder +# hadolint ignore=DL3018 +RUN apk update && apk add --no-cache \ + gcc \ + git \ + make \ + musl-dev +COPY . /celestia-app +WORKDIR /celestia-app +# we need the celestia-appd build as we might want to create an account +# internally for txsimulation +RUN make build && make txsim-build + +# Stage 2: create a minimal image with the binary +FROM docker.io/alpine:3.18.2 + +# Use UID 10,001 because UIDs below 10,000 are a security risk. +# Ref: https://github.com/hexops/dockerfile/blob/main/README.md#do-not-use-a-uid-below-10000 +ARG UID=10001 +ARG USER_NAME=celestia + +ENV CELESTIA_HOME=/home/${USER_NAME} + +# hadolint ignore=DL3018 +RUN apk update && apk add --no-cache \ + bash \ + curl \ + jq \ + # Creates a user with $UID and $GID=$UID + && adduser ${USER_NAME} \ + -D \ + -g ${USER_NAME} \ + -h ${CELESTIA_HOME} \ + -s /sbin/nologin \ + -u ${UID} + +# Copy in the celestia-appd binary +COPY --from=builder /celestia-app/build/celestia-appd /bin/celestia-appd +COPY --from=builder /celestia-app/build/txsim /bin/txsim + +COPY --chown=${USER_NAME}:${USER_NAME} docker/txsim.sh /opt/entrypoint.sh + +USER ${USER_NAME} + +# grpc, rpc, api ports +EXPOSE 26657 1317 9090 + +ENTRYPOINT [ "/bin/bash", "/opt/entrypoint.sh" ] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000..29b81db775 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,122 @@ +# Celestia `txsim` Docker Image Usage Guide + +The `txsim` binary is a tool that can be used to simulate transactions on the Celestia network. It can be used to test the performance of the Celestia network. +This guide provides instructions on how to use the Celestia `txsim` Docker image. The `txsim` Docker image is designed to run the `txsim` binary with a variety of configurable options. + +## Table of Contents + +1. [Prerequisites](#prerequisites) +2. [Running the Docker Image](#running-the-docker-image) + - [Docker Run](#docker-run) + - [Docker Compose](#docker-compose) + - [Kubernetes Deployment](#kubernetes-deployment) +3. [Flag Breakdown](#flag-breakdown) + +## Prerequisites + +Before you can use the `txsim` Docker image, you must have a prefunded account set up. The `txsim` binary requires a prefunded account to function correctly. The keyring for this account should be stored in a file that can be accessed by the Docker container. + +## Running the Docker Image + +### Docker Run + +You can run the `txsim` Docker image using the `docker run` command. Here's an example: + +```bash +docker run -it -v ${HOME}/.celestia-app:/home/celestia ghcr.io/celestiaorg/txsim -k 0 -r http://consensus-validator-robusta-rc6.celestia-robusta.com:26657,http://consensus-full-robusta-rc6.celestia-robusta.com:26657 -g consensus-validator-robusta-rc6.celestia-robusta.com:9090 -t 10s -b 10 -d 100 -e 10 +``` + +In this command, the `-v` option is used to mount the `${HOME}/.celestia-app` directory from the host to the `/home/celestia` directory in the Docker container. This allows the `txsim` binary to access the keyring for the prefunded account. + +### Docker Compose + +You can also run the `txsim` Docker image using Docker Compose. Here's an example `docker-compose.yml` file: + +```yaml +version: '3' +services: + txsim: + image: ghcr.io/celestiaorg/txsim + command: > + -k 0 + -r http://consensus-validator-robusta-rc6.celestia-robusta.com:26657,http://consensus-full-robusta-rc6.celestia-robusta.com:26657 + -g consensus-validator-robusta-rc6.celestia-robusta.com:9090 + -t 10s + -b 10 + -d 100 + -e 10 + volumes: + - /Users/txsimp/.celestia-app:/home/celestia +``` + +In this file, the `volumes` key is used to mount the `/Users/txsimp/.celestia-app` directory from the host to the `/home/celestia` directory in the Docker container. + +### Kubernetes Deployment + +Finally, you can run the `txsim` Docker image in a Kubernetes cluster. Here's an example `deployment.yaml` file: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: txsim-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: txsim + template: + metadata: + labels: + app: txsim + spec: + containers: + - name: txsim + image: ghcr.io/celestiaorg/txsim + args: + - "-k" + - "0" + - "-r" + - "http://consensus-validator-robusta-rc6.celestia-robusta.com:26657,http://consensus-full-robusta-rc6.celestia-robusta.com:26657" + - "-g" + - "consensus-validator-robusta-rc6.celestia-robusta.com:9090" + - "-t" + - "10s" + - "-b" + - "10" + - "-d" + - "100" + - "-e" + - "10" + volumeMounts: + - name: keyring-volume + mountPath: /home/celestia + volumes: + - name: keyring-volume + hostPath: + path: /Users/txsimp/.celestia-app +``` + +In this file, the `volumeMounts` and `volumes` keys are used to mount the `/Users/txsimp/.celestia-app` directory from the host to the `/home/celestia` directory in the Docker container. + +## Flag Breakdown + +Here's a breakdown of what each flag means: + +- `-k`: Whether a new key should be created (1 for yes, 0 for no) +- `-p`: The path to the keyring for the prefunded account +- `-r`: The RPC endpoints for the `txsim` binary +- `-g`: The gRPC endpoints for the `txsim` binary +- `-t`: The poll time for the `txsim` binary +- `-b`: The number of blob sequences to run +- `-a`: The range of blobs to send per PFB in a sequence +- `-s`: The range of blob sizes to send +- `-m`: The mnemonic for the keyring +- `-d`: The seed for the random number generator +- `-e`: The number of send sequences to run +- `-i`: The amount to send from one account to another +- `-v`: The number of send iterations to run per sequence +- `-u`: The number of stake sequences to run +- `-w`: The amount of initial stake per sequence + +Please replace the placeholders in the examples with the actual values you want to use. diff --git a/docker/txsim.sh b/docker/txsim.sh new file mode 100644 index 0000000000..37b681a7af --- /dev/null +++ b/docker/txsim.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +CREATE_KEY=0 +KEY_PATH="/home/celestia" +RPC_ENDPOINTS="" +GRPC_ENDPOINTS="" +POLL_TIME="" +BLOB=0 +BLOB_AMOUNTS="1" +BLOB_SIZES="100-1000" +KEY_MNEMONIC="" +SEED=0 +SEND=0 +SEND_AMOUNT=1000 +SEND_ITERATIONS=1000 +STAKE=0 +STAKE_VALUE=1000 + +while getopts "k:p:r:g:t:b:a:s:m:d:e:i:v:u:w:" opt; do + case ${opt} in + k ) + CREATE_KEY=$OPTARG + ;; + p ) + KEY_PATH=$OPTARG + ;; + r ) + RPC_ENDPOINTS=$OPTARG + ;; + g ) + GRPC_ENDPOINTS=$OPTARG + ;; + t ) + POLL_TIME=$OPTARG + ;; + b ) + BLOB=$OPTARG + ;; + a ) + BLOB_AMOUNTS=$OPTARG + ;; + s ) + BLOB_SIZES=$OPTARG + ;; + m ) + KEY_MNEMONIC=$OPTARG + ;; + d ) + SEED=$OPTARG + ;; + e ) + SEND=$OPTARG + ;; + i ) + SEND_AMOUNT=$OPTARG + ;; + v ) + SEND_ITERATIONS=$OPTARG + ;; + u ) + STAKE=$OPTARG + ;; + w ) + STAKE_VALUE=$OPTARG + ;; + \? ) + echo "Invalid option: $OPTARG" 1>&2 + exit 1 + ;; + : ) + echo "Invalid option: $OPTARG requires an argument" 1>&2 + exit 1 + ;; + esac +done +shift $((OPTIND -1)) + +if [ "$CREATE_KEY" -eq 1 ]; then + echo "Creating a new keyring-test for the txsim" + /bin/celestia-appd keys add sim --keyring-backend test --home $KEY_PATH + sleep 5 +fi + +# Running a tx simulator +txsim --key-path $KEY_PATH \ + --rpc-endpoints $RPC_ENDPOINTS \ + --grpc-endpoints $GRPC_ENDPOINTS \ + --poll-time $POLL_TIME \ + --blob $BLOB \ + --blob-amounts $BLOB_AMOUNTS \ + --blob-sizes $BLOB_SIZES \ + --key-mnemonic "$KEY_MNEMONIC" \ + --seed $SEED \ + --send $SEND \ + --send-amount $SEND_AMOUNT \ + --send-iterations $SEND_ITERATIONS \ + --stake $STAKE \ + --stake-value $STAKE_VALUE diff --git a/docs/architecture/adr-021-restricted-block-size.md b/docs/architecture/adr-021-restricted-block-size.md index 98076818e5..6905a7486c 100644 --- a/docs/architecture/adr-021-restricted-block-size.md +++ b/docs/architecture/adr-021-restricted-block-size.md @@ -17,7 +17,7 @@ will determine what the protocol considers to be a valid block size. | Parameter | Size | Unit | Description | Value Control | |------------------|--------|--------|--------------|---------------| | `MaxBlockSizeBytes` | 100 | MiB | Maximum total size of the protobuf encoded block, a hard coded constant acting as a cap for `MaxBytes`. | Hard coded | -| `MaxBytes` | 21 | MiB | Determines the valid size of the entire protobuf encoded block. Is a governance modifiable parameter and is capped by `MaxBytes`. Used to regulate the amount of data gossiped in the consensus portion of the network (using the current block gossiping mechanism). | Modifiable | +| `MaxBytes` | 21 | MiB | Determines the valid size of the entire protobuf encoded block. Is a governance modifiable parameter and is capped by `MaxBlockSizeBytes`. Used to regulate the amount of data gossiped in the consensus portion of the network (using the current block gossiping mechanism). | Modifiable | | `MaxSquareSize` | ~7.5 | MiB | Determines the maximum size of the original data square. Used to regulate storage requirements for celestia-node, and the size of the data availability header. Default set to 128. | Hard coded | Using the currently set/default values, `MaxSquareSize` is limiting the block @@ -78,7 +78,7 @@ Option 2: Introduce a new parameter, `GovMaxSquareSize`. After implemented, the | Parameter | Size | Unit | Description | Value Control | |------------------|--------|--------|--------------|---------------| | `MaxBlockSizeBytes` | 100 | MiB | Maximum total size of the protobuf encoded block, a hard coded constant acting as a cap for `MaxBytes`. | Hard coded | -| `MaxBytes` | ~1.8 | MiB | Determines the valid size of the entire protobuf encoded block. Is a governance modifiable parameter and is capped by `MaxBytes`. Used to regulate the amount of data gossiped in the consensus portion of the network (using the current block gossiping mechanism). | Modifiable | +| `MaxBytes` | ~1.8 | MiB | Determines the valid size of the entire protobuf encoded block. Is a governance modifiable parameter and is capped by `MaxBlockSizeBytes`. Used to regulate the amount of data gossiped in the consensus portion of the network (using the current block gossiping mechanism). | Modifiable | | `MaxSquareSize` | ~7.5 | MiB | Determines the maximum size of the original data square. Used to regulate storage requirements for celestia-node, and the size of the data availability header. Default set to 128. | Hard coded | | `GovMaxSquareSize` | ~1.8 | MiB | Governance modifiable parameter that determines valid square sizes. Must be smaller than the `MaxSquareSize`. Default set to 64. | Modifiable | diff --git a/go.mod b/go.mod index c3d3451bb8..2d3e196e5f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/celestiaorg/celestia-app go 1.20 require ( - github.com/celestiaorg/nmt v0.17.0 + github.com/celestiaorg/nmt v0.18.0 github.com/celestiaorg/quantum-gravity-bridge v1.3.0 github.com/ethereum/go-ethereum v1.12.0 github.com/gogo/protobuf v1.3.3 @@ -39,15 +39,40 @@ require ( ) require ( - github.com/Microsoft/go-winio v0.6.0 // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.1 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/imdario/mergo v0.3.13 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/moby/spdystream v0.2.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc2 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect golang.org/x/mod v0.9.0 // indirect + golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect golang.org/x/tools v0.7.0 // indirect - gotest.tools/v3 v3.4.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/api v0.27.3 // indirect + k8s.io/apimachinery v0.27.3 // indirect + k8s.io/client-go v0.27.3 // indirect + k8s.io/klog/v2 v2.90.1 // indirect + k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect + k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) require ( @@ -59,7 +84,6 @@ require ( filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.1 // indirect - github.com/BurntSushi/toml v1.3.2 github.com/ChainSafe/go-schnorrkel v1.0.0 // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/Workiva/go-datastructures v1.0.53 // indirect @@ -69,6 +93,7 @@ require ( github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/bgentry/speakeasy v0.1.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect + github.com/celestiaorg/knuu v0.8.2 github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect @@ -93,7 +118,7 @@ require ( github.com/dgraph-io/badger/v2 v2.2007.4 // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect - github.com/docker/docker v24.0.2+incompatible + github.com/docker/docker v24.0.4+incompatible // indirect github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac // indirect github.com/dvsekhvalnov/jose2go v1.5.0 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect diff --git a/go.sum b/go.sum index 1fb21ab9fb..a11e8801db 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,6 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSu github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ChainSafe/go-schnorrkel v1.0.0 h1:3aDA67lAykLaG1y3AOjs88dMxC88PgUuHRrLeDnvGIM= github.com/ChainSafe/go-schnorrkel v1.0.0/go.mod h1:dpzHYVxLZcp8pjlV+O+UR8K0Hp/z7vcchBSbMBEhCw4= @@ -78,8 +76,8 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= -github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -116,6 +114,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= @@ -174,10 +173,12 @@ github.com/celestiaorg/celestia-core v1.25.0-tm-v0.34.28 h1:2/b6288LWPPJqb8vxdia github.com/celestiaorg/celestia-core v1.25.0-tm-v0.34.28/go.mod h1:J/GsBjoTZaFz71VeyrLZbG8rV+Rzi6oFEUZUipQ97hQ= github.com/celestiaorg/cosmos-sdk v1.16.1-sdk-v0.46.13 h1:CxEQDQEQR1ypB+VUmCISIqFVmHfb+mx8x+zh7rHbyU8= github.com/celestiaorg/cosmos-sdk v1.16.1-sdk-v0.46.13/go.mod h1:xpBZc/OYZ736hp0IZlBGNUhEgCD9C+bKs8yNLZibyv0= +github.com/celestiaorg/knuu v0.8.2 h1:x63nYTO46j293VkNP7VcRHtYHSize6dM/h7xME2Vgi0= +github.com/celestiaorg/knuu v0.8.2/go.mod h1:zBrMXhFl7irCPOVJyHS8sBg7K9Hux5dLy9VChA40LTo= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= -github.com/celestiaorg/nmt v0.17.0 h1:/k8YLwJvuHgT/jQ435zXKaDX811+sYEMXL4B/vYdSLU= -github.com/celestiaorg/nmt v0.17.0/go.mod h1:ZndCeAR4l9lxm7W51ouoyTo1cxhtFgK+4DpEIkxRA3A= +github.com/celestiaorg/nmt v0.18.0 h1:JUiKbAYVwGhPCN8Xn8+25A++luEb3q5K6KnReXihkIk= +github.com/celestiaorg/nmt v0.18.0/go.mod h1:0l8q6UYRju1xNrxtvV6NwPdW3lfsN6KuZ0htRnModdc= github.com/celestiaorg/quantum-gravity-bridge v1.3.0 h1:9zPIp7w1FWfkPnn16y3S4FpFLnQtS7rm81CUVcHEts0= github.com/celestiaorg/quantum-gravity-bridge v1.3.0/go.mod h1:6WOajINTDEUXpSj5UZzod16UZ96ZVB/rFNKyM+Mt1gI= github.com/celestiaorg/rsmt2d v0.10.0 h1:8dprr6CW5mCk5YPnbiLdirojw9YsJOE+XB+GORb8sT0= @@ -299,15 +300,16 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v24.0.2+incompatible h1:eATx+oLz9WdNVkQrr0qjQ8HvRJ4bOOxfzEo8R+dA3cg= -github.com/docker/docker v24.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.4+incompatible h1:s/LVDftw9hjblvqIeTiGYXBCD95nOEEl7qRsRrIOuQI= +github.com/docker/docker v24.0.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -321,6 +323,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -377,12 +381,21 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= +github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= @@ -397,6 +410,7 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= @@ -448,6 +462,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -461,6 +476,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -478,6 +495,7 @@ github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -495,6 +513,7 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -593,6 +612,8 @@ github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/improbable-eng/grpc-web v0.15.0 h1:BN+7z6uNXZ1tQGcNAuaU1YjsLTApzkjt2tzCixLaUPQ= github.com/improbable-eng/grpc-web v0.15.0/go.mod h1:1sy9HKV4Jt9aEs9JSnkWlRJPuPtwNr0l57L4f878wP8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -628,6 +649,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -636,6 +659,7 @@ github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= @@ -668,6 +692,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -695,6 +720,8 @@ github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamh github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= @@ -751,18 +778,23 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= -github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -791,11 +823,12 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q= +github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -894,7 +927,7 @@ github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRr github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= @@ -923,7 +956,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -953,6 +987,7 @@ github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= @@ -1292,6 +1327,7 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1319,6 +1355,7 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1460,6 +1497,7 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1521,6 +1559,8 @@ gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= @@ -1543,12 +1583,14 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= -gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1558,6 +1600,18 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +k8s.io/api v0.27.3 h1:yR6oQXXnUEBWEWcvPWS0jQL575KoAboQPfJAuKNrw5Y= +k8s.io/api v0.27.3/go.mod h1:C4BNvZnQOF7JA/0Xed2S+aUyJSfTGkGFxLXz9MnpIpg= +k8s.io/apimachinery v0.27.3 h1:Ubye8oBufD04l9QnNtW05idcOe9Z3GQN8+7PqmuVcUM= +k8s.io/apimachinery v0.27.3/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= +k8s.io/client-go v0.27.3 h1:7dnEGHZEJld3lYwxvLl7WoehK6lAq7GvgjxpA3nv1E8= +k8s.io/client-go v0.27.3/go.mod h1:2MBEKuTo6V1lbKy3z1euEGnhPfGZLKTS9tiJ2xodM48= +k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= +k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= +k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= +k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k= nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= pgregory.net/rapid v0.5.3 h1:163N50IHFqr1phZens4FQOdPgfJscR7a562mjQqeo4M= @@ -1566,6 +1620,10 @@ rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/pkg/namespace/namespace.go b/pkg/namespace/namespace.go index 5da38bc398..9e690ba023 100644 --- a/pkg/namespace/namespace.go +++ b/pkg/namespace/namespace.go @@ -148,7 +148,7 @@ func (n Namespace) IsPayForBlob() bool { func (n Namespace) Repeat(times int) []Namespace { ns := make([]Namespace, times) for i := 0; i < times; i++ { - ns[i] = n + ns[i] = n.deepCopy() } return ns } @@ -173,6 +173,8 @@ func (n Namespace) IsGreaterOrEqualThan(n2 Namespace) bool { return bytes.Compare(n.Bytes(), n2.Bytes()) > -1 } +// leftPad returns a new byte slice with the provided byte slice left-padded to the provided size. +// If the provided byte slice is already larger than the provided size, the original byte slice is returned. func leftPad(b []byte, size int) []byte { if len(b) >= size { return b @@ -180,3 +182,18 @@ func leftPad(b []byte, size int) []byte { pad := make([]byte, size-len(b)) return append(pad, b...) } + +// deepCopy returns a deep copy of the Namespace object. +func (n Namespace) deepCopy() Namespace { + // Create a deep copy of the ID slice + copyID := make([]byte, len(n.ID)) + copy(copyID, n.ID) + + // Create a new Namespace object with the copied fields + copyNamespace := Namespace{ + Version: n.Version, + ID: copyID, + } + + return copyNamespace +} diff --git a/pkg/namespace/namespace_test.go b/pkg/namespace/namespace_test.go index f7b97487dc..fcda8472d5 100644 --- a/pkg/namespace/namespace_test.go +++ b/pkg/namespace/namespace_test.go @@ -2,6 +2,7 @@ package namespace import ( "bytes" + "reflect" "testing" "github.com/stretchr/testify/assert" @@ -73,6 +74,19 @@ func TestNew(t *testing.T) { } } +// TestRepeatNonMutability ensures that the output of Repeat method is not mutated when the original namespace is mutated. +func TestRepeatNonMutability(t *testing.T) { + n := 10 + namespace := Namespace{Version: NamespaceVersionMax, ID: []byte{1, 2, 3, 4}} + repeated := namespace.Repeat(n) + // mutate the original namespace + namespace.ID[0] = 5 + // ensure the repeated namespaces are not mutated + for i := 0; i < n; i++ { + assert.NotEqual(t, repeated[i], namespace) + } +} + func TestNewV0(t *testing.T) { type testCase struct { name string @@ -195,3 +209,33 @@ func TestBytes(t *testing.T) { assert.Equal(t, want, got) } + +func TestLeftPad(t *testing.T) { + tests := []struct { + input []byte + size int + expected []byte + }{ + // input smaller than pad size + {[]byte{1, 2, 3}, 10, []byte{0, 0, 0, 0, 0, 0, 0, 1, 2, 3}}, + {[]byte{1}, 5, []byte{0, 0, 0, 0, 1}}, + {[]byte{1, 2}, 4, []byte{0, 0, 1, 2}}, + + // input equal to pad size + {[]byte{1, 2, 3}, 3, []byte{1, 2, 3}}, + {[]byte{1, 2, 3, 4}, 4, []byte{1, 2, 3, 4}}, + + // input larger than pad size + {[]byte{1, 2, 3, 4, 5}, 4, []byte{1, 2, 3, 4, 5}}, + {[]byte{1, 2, 3, 4, 5, 6, 7}, 3, []byte{1, 2, 3, 4, 5, 6, 7}}, + + // input size 0 + {[]byte{}, 8, []byte{0, 0, 0, 0, 0, 0, 0, 0}}, + {[]byte{}, 0, []byte{}}, + } + + for _, test := range tests { + result := leftPad(test.input, test.size) + assert.True(t, reflect.DeepEqual(result, test.expected)) + } +} diff --git a/pkg/shares/padding.go b/pkg/shares/padding.go index d2ee22bb73..aa5c2e425c 100644 --- a/pkg/shares/padding.go +++ b/pkg/shares/padding.go @@ -10,10 +10,11 @@ import ( // NamespacePaddingShare returns a share that acts as padding. Namespace padding // shares follow a blob so that the next blob may start at an index that -// conforms to blob share commitment rules. The ns parameter provided should -// be the namespace of the blob that precedes this padding in the data square. -func NamespacePaddingShare(ns appns.Namespace) (Share, error) { - b, err := NewBuilder(ns, appconsts.ShareVersionZero, true).Init() +// conforms to blob share commitment rules. The ns and shareVersion parameters +// provided should be the namespace and shareVersion of the blob that precedes +// this padding in the data square. +func NamespacePaddingShare(ns appns.Namespace, shareVersion uint8) (Share, error) { + b, err := NewBuilder(ns, shareVersion, true).Init() if err != nil { return Share{}, err } @@ -32,14 +33,14 @@ func NamespacePaddingShare(ns appns.Namespace) (Share, error) { } // NamespacePaddingShares returns n namespace padding shares. -func NamespacePaddingShares(ns appns.Namespace, n int) ([]Share, error) { +func NamespacePaddingShares(ns appns.Namespace, shareVersion uint8, n int) ([]Share, error) { var err error if n < 0 { return nil, errors.New("n must be positive") } shares := make([]Share, n) for i := 0; i < n; i++ { - shares[i], err = NamespacePaddingShare(ns) + shares[i], err = NamespacePaddingShare(ns, shareVersion) if err != nil { return shares, err } @@ -52,7 +53,7 @@ func NamespacePaddingShares(ns appns.Namespace, n int) ([]Share, error) { // first blob can start at an index that conforms to non-interactive default // rules. func ReservedPaddingShare() Share { - share, err := NamespacePaddingShare(appns.ReservedPaddingNamespace) + share, err := NamespacePaddingShare(appns.ReservedPaddingNamespace, appconsts.ShareVersionZero) if err != nil { panic(err) } @@ -61,7 +62,7 @@ func ReservedPaddingShare() Share { // ReservedPaddingShare returns n reserved padding shares. func ReservedPaddingShares(n int) []Share { - shares, err := NamespacePaddingShares(appns.ReservedPaddingNamespace, n) + shares, err := NamespacePaddingShares(appns.ReservedPaddingNamespace, appconsts.ShareVersionZero, n) if err != nil { panic(err) } @@ -72,7 +73,7 @@ func ReservedPaddingShares(n int) []Share { // square size. Tail padding shares follow the last blob share in the data // square. func TailPaddingShare() Share { - share, err := NamespacePaddingShare(appns.TailPaddingNamespace) + share, err := NamespacePaddingShare(appns.TailPaddingNamespace, appconsts.ShareVersionZero) if err != nil { panic(err) } @@ -81,7 +82,7 @@ func TailPaddingShare() Share { // TailPaddingShares returns n tail padding shares. func TailPaddingShares(n int) []Share { - shares, err := NamespacePaddingShares(appns.TailPaddingNamespace, n) + shares, err := NamespacePaddingShares(appns.TailPaddingNamespace, appconsts.ShareVersionZero, n) if err != nil { panic(err) } diff --git a/pkg/shares/padding_test.go b/pkg/shares/padding_test.go index abf04e8515..0d0317953e 100644 --- a/pkg/shares/padding_test.go +++ b/pkg/shares/padding_test.go @@ -40,13 +40,13 @@ var tailPadding, _ = zeroPadIfNecessary( ), appconsts.ShareSize) func TestNamespacePaddingShare(t *testing.T) { - got, err := NamespacePaddingShare(ns1) + got, err := NamespacePaddingShare(ns1, appconsts.ShareVersionZero) assert.NoError(t, err) assert.Equal(t, nsOnePadding, got.ToBytes()) } func TestNamespacePaddingShares(t *testing.T) { - shares, err := NamespacePaddingShares(ns1, 2) + shares, err := NamespacePaddingShares(ns1, appconsts.ShareVersionZero, 2) assert.NoError(t, err) for _, share := range shares { assert.Equal(t, nsOnePadding, share.ToBytes()) diff --git a/pkg/shares/parse_test.go b/pkg/shares/parse_test.go index 402b8714b7..152d262658 100644 --- a/pkg/shares/parse_test.go +++ b/pkg/shares/parse_test.go @@ -41,7 +41,7 @@ func TestParseShares(t *testing.T) { // because it takes more than one share to store a sequence of 1000 bytes tooLargeSequenceLen := generateRawShare(t, ns1, true, uint32(1000)) - ns1Padding, err := NamespacePaddingShare(ns1) + ns1Padding, err := NamespacePaddingShare(ns1, appconsts.ShareVersionZero) require.NoError(t, err) type testCase struct { diff --git a/pkg/shares/share_sequence_test.go b/pkg/shares/share_sequence_test.go index 31d790cdb4..3e98304217 100644 --- a/pkg/shares/share_sequence_test.go +++ b/pkg/shares/share_sequence_test.go @@ -151,7 +151,7 @@ func Test_validSequenceLen(t *testing.T) { } ns1 := appns.MustNewV0(bytes.Repeat([]byte{0x1}, appns.NamespaceVersionZeroIDSize)) - share, err := NamespacePaddingShare(ns1) + share, err := NamespacePaddingShare(ns1, appconsts.ShareVersionZero) require.NoError(t, err) namespacePadding := ShareSequence{ Namespace: ns1, diff --git a/pkg/shares/shares_test.go b/pkg/shares/shares_test.go index 10a277e7d4..cc7ee1bb19 100644 --- a/pkg/shares/shares_test.go +++ b/pkg/shares/shares_test.go @@ -241,7 +241,7 @@ func TestIsPadding(t *testing.T) { ), appconsts.ShareSize) - nsPadding, err := NamespacePaddingShare(ns1) + nsPadding, err := NamespacePaddingShare(ns1, appconsts.ShareVersionZero) require.NoError(t, err) testCases := []testCase{ diff --git a/pkg/shares/split_sparse_shares.go b/pkg/shares/split_sparse_shares.go index 0a54ffd7f1..eb3d8a1d57 100644 --- a/pkg/shares/split_sparse_shares.go +++ b/pkg/shares/split_sparse_shares.go @@ -86,7 +86,11 @@ func (sss *SparseShareSplitter) WriteNamespacePaddingShares(count int) error { if err != nil { return err } - nsPaddingShares, err := NamespacePaddingShares(lastBlobNs, count) + lastBlobInfo, err := lastBlob.InfoByte() + if err != nil { + return err + } + nsPaddingShares, err := NamespacePaddingShares(lastBlobNs, lastBlobInfo.Version(), count) if err != nil { return err } diff --git a/pkg/shares/split_sparse_shares_test.go b/pkg/shares/split_sparse_shares_test.go index db4ff394c8..ca3c55853c 100644 --- a/pkg/shares/split_sparse_shares_test.go +++ b/pkg/shares/split_sparse_shares_test.go @@ -4,6 +4,7 @@ import ( "bytes" "testing" + "github.com/celestiaorg/celestia-app/pkg/appconsts" appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/stretchr/testify/assert" coretypes "github.com/tendermint/tendermint/types" @@ -38,3 +39,38 @@ func TestSparseShareSplitter(t *testing.T) { got := sss.Export() assert.Len(t, got, 2) } + +func TestWriteNamespacePaddingShares(t *testing.T) { + ns1 := appns.MustNewV0(bytes.Repeat([]byte{1}, appns.NamespaceVersionZeroIDSize)) + blob1 := newBlob(ns1, appconsts.ShareVersionZero) + + sss := NewSparseShareSplitter() + + err := sss.Write(blob1) + assert.NoError(t, err) + err = sss.WriteNamespacePaddingShares(1) + assert.NoError(t, err) + + // got is expected to be [blob1, padding] + got := sss.Export() + assert.Len(t, got, 2) + + // verify that the second share is padding + isPadding, err := got[1].IsPadding() + assert.NoError(t, err) + assert.True(t, isPadding) + + // verify that the padding share has the same share version as blob1 + info, err := got[1].InfoByte() + assert.NoError(t, err) + assert.Equal(t, info.Version(), appconsts.ShareVersionZero) +} + +func newBlob(ns appns.Namespace, shareVersion uint8) coretypes.Blob { + return coretypes.Blob{ + NamespaceVersion: ns.Version, + NamespaceID: ns.ID, + ShareVersion: shareVersion, + Data: []byte("data"), + } +} diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index 4f00e2f6db..c07d630cba 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -213,8 +213,7 @@ A proof for a leaf in a [binary Merkle tree](#binary-merkle-tree), as per Sectio ### Namespace Merkle Tree - - + [Shares](./shares.md) in Celestia are associated with a provided _namespace_. The Namespace Merkle Tree (NMT) is a variation of the [Merkle Interval Tree](https://eprint.iacr.org/2018/642), which is itself an extension of the [Merkle Sum Tree](https://bitcointalk.org/index.php?topic=845978.0). It allows for compact proofs around the inclusion or exclusion of shares with particular namespace IDs. diff --git a/specs/src/specs/fraud_proofs.md b/specs/src/specs/fraud_proofs.md index 8dde0d33f5..2ec0802b00 100644 --- a/specs/src/specs/fraud_proofs.md +++ b/specs/src/specs/fraud_proofs.md @@ -22,4 +22,4 @@ State fraud proofs allow light clients to avoid making an honest majority assump state validity. While these are not incorporated into the protocol as of v1.0.0, there are example implementations that can be found in [Rollkit](https://github.com/rollkit/rollkit). More info in -[rollkit-ADR009](https://github.com/rollkit/rollkit/blob/4fd97ba8b8352771f2e66454099785d06fd0c31b/docs/lazy-adr/adr-009-state-fraud-proofs.md). \ No newline at end of file +[rollkit-ADR009](https://github.com/rollkit/rollkit/blob/4fd97ba8b8352771f2e66454099785d06fd0c31b/docs/lazy-adr/adr-009-state-fraud-proofs.md). diff --git a/specs/src/specs/namespace.md b/specs/src/specs/namespace.md index b259516cdb..d5fb595f3a 100644 --- a/specs/src/specs/namespace.md +++ b/specs/src/specs/namespace.md @@ -4,27 +4,40 @@ ## Abstract -One of Celestia's core data structures is the namespace. When a user submits a `MsgPayForBlobs` transaction to Celestia they MUST associate each blob with exactly one namespace. After their transaction has been included in a block, the namespace enables users to take an interest in a subset of the blobs published to Celestia by allowing the user to query for blobs by namespace. +One of Celestia's core data structures is the namespace. +When a user submits a transaction encapsulating a `MsgPayForBlobs` message to Celestia, they MUST associate each blob with exactly one namespace. +After their transaction has been included in a block, the namespace enables users to take an interest in a subset of the blobs published to Celestia by allowing the user to query for blobs by namespace. -In order to enable efficient retrieval of blobs by namespace, Celestia makes use of a [Namespaced Merkle Tree](https://github.com/celestiaorg/nmt). See section 5.2 of the [LazyLedger whitepaper](https://arxiv.org/pdf/1905.09274.pdf) for more details. +In order to enable efficient retrieval of blobs by namespace, Celestia makes use of a [Namespaced Merkle Tree](https://github.com/celestiaorg/nmt). +See section 5.2 of the [LazyLedger whitepaper](https://arxiv.org/pdf/1905.09274.pdf) for more details. ## Overview -A namespace is composed of two fields: [version](#version) and [id](#id). A namespace is encoded as a byte slice with the version and id concatenated. Each [share](./shares.md) is prefixed with exactly one namespace. +A namespace is composed of two fields: [version](#version) and [id](#id). +A namespace is encoded as a byte slice with the version and id concatenated. ![namespace](./figures/namespace.svg) ### Version -The namespace version is an 8-bit unsigned integer that indicates the version of the namespace. The version is used to determine the format of the namespace. The only supported user-specifiable namespace version is `0`. The version is encoded as a single byte. +The namespace version is an 8-bit unsigned integer that indicates the version of the namespace. +The version is used to determine the format of the namespace and +is encoded as a single byte. +A new namespace version MUST be introduced if the namespace format changes in a backwards incompatible way. -Note: The `PARITY_SHARE_NAMESPACE` uses the namespace version `255` so that it can be ignored via the `IgnoreMaxNamespace` feature from [nmt](https://github.com/celestiaorg/nmt). The `TAIL_PADDING_NAMESPACE` uses the namespace version `255` so that it remains ordered after all blob namespaces even in the case a new namespace version is introduced. +Below we explain supported user-specifiable namespace versions, +however, we note that Celestia MAY utilize other namespace versions for internal use. +For more details, see the [Reserved Namespaces](#reserved-namespaces) section. -A namespace with version `0` must contain an id with a prefix of 18 leading `0` bytes. The remaining 10 bytes of the id are user-specified. +#### Version 0 + +The only supported user-specifiable namespace version is `0`. +A namespace with version `0` MUST contain an id with a prefix of 18 leading `0` bytes. +The remaining 10 bytes of the id are user-specified. +Below, we provide examples of valid and invalid encoded user-supplied namespaces with version `0`. ```go // Valid encoded namespaces -0x0000000000000000000000000000000000000000000000000000000001 // transaction namespace 0x0000000000000000000000000000000000000001010101010101010101 // valid blob namespace 0x0000000000000000000000000000000000000011111111111111111111 // valid blob namespace @@ -34,14 +47,27 @@ A namespace with version `0` must contain an id with a prefix of 18 leading `0` 0x1111111111111111111111111111111111111111111111111111111111 // invalid because it does not have version 0 ``` -A new namespace version MUST be introduced if the namespace format changes in a backwards incompatible way (i.e. the number of leading `0` bytes in the id prefix is reduced). +Any change in the number of leading `0` bytes in the id of a namespace with version `0` is considered a backwards incompatible change and MUST be introduced as a new namespace version. ### ID -The namespace ID is a 28 byte identifier that uniquely identifies a namespace. The ID is encoded as a byte slice of length 28. +The namespace ID is a 28 byte identifier that uniquely identifies a namespace. +The ID is encoded as a byte slice of length 28. + ## Reserved Namespaces +Celestia reserves certain namespaces with specific meanings. +Celestia makes use of the reserved namespaces to properly organize and order transactions and blobs inside the [data square](./data_square_layout.md). +Applications MUST NOT use these reserved namespaces for their blob data. + +Below is a list of reserved namespaces, along with a brief description of each. +In addition to the items listed in this table, it should be noted that namespaces with values less than `0x00000000000000000000000000000000000000000000000000000000FF` are exclusively reserved for use within the Celestia protocols. +In the table, you will notice that the `PARITY_SHARE_NAMESPACE` and `TAIL_PADDING_NAMESPACE` utilize the namespace version `255`, which differs from the supported user-specified versions. +The reason for employing version `255` for the `PARITY_SHARE_NAMESPACE` is to enable more efficient proof generation within the context of [nmt](https://github.com/celestiaorg/nmt), where it is used in conjunction with the `IgnoreMaxNamespace` feature. +Similarly, the `TAIL_PADDING_NAMESPACE` utilizes the namespace version `255` to ensure that padding shares are always properly ordered and placed at the end of the Celestia data square even if a new namespace version is introduced. +For additional information on the significance and application of the reserved namespaces, please refer to the [Data Square Layout](./data_square_layout.md) specifications. + | name | type | value | description | |-------------------------------------|-------------|----------------------------------------------------------------|------------------------------------------------------------------------------------------------------| | `TRANSACTION_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000001` | Transactions: requests that modify the state. | @@ -54,11 +80,20 @@ The namespace ID is a 28 byte identifier that uniquely identifies a namespace. T ## Assumptions and Considerations +Applications MUST refrain from using the [reserved namespaces](#reserved-namespaces) for their blob data. + ## Implementation See [pkg/namespace](../../../pkg/namespace). +## Protobuf Definition + + + ## References 1. [ADR-014](../../../docs/architecture/adr-014-versioned-namespaces.md) 1. [ADR-015](../../../docs/architecture/adr-015-namespace-id-size.md) +1. [Namespaced Merkle Tree](https://github.com/celestiaorg/nmt) +1. [LazyLedger whitepaper](https://arxiv.org/pdf/1905.09274.pdf) +1. [Data Square Layout](./data_square_layout.md) diff --git a/test/e2e/Makefile b/test/e2e/Makefile deleted file mode 100644 index 1b63d30a3b..0000000000 --- a/test/e2e/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -all: cli docker - -cli: - go build -o build/e2e ./cmd/e2e - -docker: - docker build --tag ghcr.io/celestiaorg/celestia-app:current -f ../../Dockerfile ../.. - -PHONY: all cli docker diff --git a/test/e2e/README.md b/test/e2e/README.md deleted file mode 100644 index 56b6c75232..0000000000 --- a/test/e2e/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# E2E Framework - -## Purpose - -The e2e package provides a framework for integration testing of the Celestia consensus network. -It consists of a simple CLI and a series of TOML testnet files which manage the running of -several instances within the same network. The e2e test suite has the following purposes in mind: - -- **Compatibility testing**: Ensuring that multiple minor versions can operate successfully together within the same network. -- **Upgrade testing**: Ensure upgrades, whether major or minor, can perform seamlessly. -- **Sync testing**: Ensure that the latest version can sync data from the entire chain. -- **Non determinism check**: Ensure that the state machine is free of non-determinism that could compromise replication. -- **Invariant checking**: Ensure that system wide invariants hold. - -The e2e package is designed predominantly for correctness based testing of small clusters of node. -It is designed to be relatively quick and can be used locally. It relies on docker and docker compose -to orchestrate the nodes. - -## Usage - -To get started, run `make` within the e2e package directory. This builds the image referring to the current -branch as well as the cli (To build just the cli run `make cli`). Then, to run the complete suite: - -```bash -./build/e2e -f networks/simple.toml -``` - -You should see something like - -```bash -Setting up network simple-56602 -Spinning up testnet -Starting validator01 on -Starting validator02 on -Starting validator03 on -Starting validator04 on -Starting full01 on -Waiting for the network to reach height 20 -Stopping testnet -Finished testnet successfully -``` - -Alternatively you can use the commands: `setup`, `start`, `stop`, and `cleanup`. diff --git a/test/e2e/cmd/e2e/main.go b/test/e2e/cmd/e2e/main.go deleted file mode 100644 index 551ce64fa6..0000000000 --- a/test/e2e/cmd/e2e/main.go +++ /dev/null @@ -1,124 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "os/signal" - "syscall" - - e2e "github.com/celestiaorg/celestia-app/test/e2e/pkg" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" -) - -func main() { - NewCLI().Run() -} - -type CLI struct { - root *cobra.Command - testnet *e2e.Testnet -} - -func NewCLI() *CLI { - cli := &CLI{} - cli.root = &cobra.Command{ - Use: "e2e", - Short: "Command line runner for celestia app e2e framework", - SilenceUsage: true, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - file, err := cmd.Flags().GetString("file") - if err != nil { - return err - } - - manifest, err := e2e.LoadManifest(file) - if err != nil { - return fmt.Errorf("loading manifest: %w", err) - } - - testnet, err := e2e.LoadTestnet(manifest, file) - if err != nil { - return fmt.Errorf("building testnet: %w", err) - } - - cli.testnet = testnet - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - if err := e2e.Cleanup(ctx, cli.testnet); err != nil { - return fmt.Errorf("preparing testnet: %w", err) - } - - if err := e2e.Setup(ctx, cli.testnet); err != nil { - return fmt.Errorf("setting up testnet: %w", err) - } - defer func() { _ = e2e.Cleanup(ctx, cli.testnet) }() - - if err := e2e.Start(ctx, cli.testnet); err != nil { - return fmt.Errorf("starting network: %w", err) - } - - if err := e2e.WaitForNBlocks(ctx, cli.testnet, 10); err != nil { - return fmt.Errorf("waiting for the network to produce blocks: %w", err) - } - - if err := e2e.Stop(ctx, cli.testnet); err != nil { - return fmt.Errorf("stopping network: %w", err) - } - - fmt.Println("Finished testnet successfully") - return nil - }, - } - cli.root.PersistentFlags().StringP("file", "f", "", "Testnet TOML manifest") - _ = cli.root.MarkPersistentFlagRequired("file") - - cli.root.AddCommand(&cobra.Command{ - Use: "setup", - Short: "Setups a testnet", - RunE: func(cmd *cobra.Command, args []string) error { - return e2e.Setup(cmd.Context(), cli.testnet) - }, - }) - - cli.root.AddCommand(&cobra.Command{ - Use: "start", - Short: "Starts a testnet", - RunE: func(cmd *cobra.Command, args []string) error { - return e2e.Start(cmd.Context(), cli.testnet) - }, - }) - - cli.root.AddCommand(&cobra.Command{ - Use: "stop", - Short: "Stops a testnet", - RunE: func(cmd *cobra.Command, args []string) error { - return e2e.Stop(cmd.Context(), cli.testnet) - }, - }) - - cli.root.AddCommand(&cobra.Command{ - Use: "cleanup", - Short: "Tears down network and removes all resources", - RunE: func(cmd *cobra.Command, args []string) error { - return e2e.Cleanup(cmd.Context(), cli.testnet) - }, - }) - - return cli -} - -func (cli *CLI) Run() { - ctx, cancel := signal.NotifyContext(context.Background(), - syscall.SIGINT, - syscall.SIGTERM, - ) - defer cancel() - if err := cli.root.ExecuteContext(ctx); err != nil { - log.Err(err) - os.Exit(1) - } -} diff --git a/test/e2e/networks/simple.toml b/test/e2e/networks/simple.toml deleted file mode 100644 index 26ef084c64..0000000000 --- a/test/e2e/networks/simple.toml +++ /dev/null @@ -1,13 +0,0 @@ -[node.validator01] -self_delegation = 1000000 -[node.validator02] -self_delegation = 1000000 -[node.validator03] -self_delegation = 1000000 -[node.validator04] -self_delegation = 1000000 -[node.full01] -start_height = 10 - -[account.user1] -[account.user2] \ No newline at end of file diff --git a/test/e2e/node.go b/test/e2e/node.go new file mode 100644 index 0000000000..57bd165d6e --- /dev/null +++ b/test/e2e/node.go @@ -0,0 +1,262 @@ +package e2e + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/celestiaorg/knuu/pkg/knuu" + serverconfig "github.com/cosmos/cosmos-sdk/server/config" + "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/privval" + "github.com/tendermint/tendermint/rpc/client/http" + "github.com/tendermint/tendermint/types" +) + +const ( + rpcPort = 26657 + p2pPort = 26656 + grpcPort = 9090 + dockerSrcURL = "ghcr.io/celestiaorg/celestia-app" + secp256k1Type = "secp256k1" + ed25519Type = "ed25519" + remoteRootDir = "/home/celestia/.celestia-app" +) + +type Node struct { + Name string + Version string + StartHeight int64 + InitialPeers []string + SignerKey crypto.PrivKey + NetworkKey crypto.PrivKey + AccountKey crypto.PrivKey + SelfDelegation int64 + Instance *knuu.Instance + + rpcProxyPort int + grpcProxyPort int +} + +func NewNode( + name, version string, + startHeight, selfDelegation int64, + peers []string, + signerKey, networkKey, accountKey crypto.PrivKey, +) (*Node, error) { + instance, err := knuu.NewInstance(name) + if err != nil { + return nil, err + } + err = instance.SetImage(fmt.Sprintf("%s:%s", dockerSrcURL, version)) + if err != nil { + return nil, err + } + if err := instance.AddPortTCP(rpcPort); err != nil { + return nil, err + } + if err := instance.AddPortTCP(p2pPort); err != nil { + return nil, err + } + if err := instance.AddPortTCP(grpcPort); err != nil { + return nil, err + } + err = instance.SetMemory("200Mi", "200Mi") + if err != nil { + return nil, err + } + err = instance.SetCPU("300m") + if err != nil { + return nil, err + } + err = instance.AddVolumeWithOwner(remoteRootDir, "1Gi", 10001) + if err != nil { + return nil, err + } + err = instance.SetArgs("start", fmt.Sprintf("--home=%s", remoteRootDir), "--rpc.laddr=tcp://0.0.0.0:26657") + if err != nil { + return nil, err + } + + return &Node{ + Name: name, + Instance: instance, + Version: version, + StartHeight: startHeight, + InitialPeers: peers, + SignerKey: signerKey, + NetworkKey: networkKey, + AccountKey: accountKey, + SelfDelegation: selfDelegation, + }, nil +} + +func (n *Node) Init(genesis types.GenesisDoc, peers []string) error { + if len(peers) == 0 { + return fmt.Errorf("no peers provided") + } + + // Initialize file directories + rootDir := os.TempDir() + nodeDir := filepath.Join(rootDir, n.Name) + for _, dir := range []string{ + filepath.Join(nodeDir, "config"), + filepath.Join(nodeDir, "data"), + } { + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return fmt.Errorf("error creating directory %s: %w", dir, err) + } + } + + // Create and write the config file + cfg, err := MakeConfig(n) + if err != nil { + return fmt.Errorf("making config: %w", err) + } + configFilePath := filepath.Join(nodeDir, "config", "config.toml") + config.WriteConfigFile(configFilePath, cfg) + + // Store the genesis file + genesisFilePath := filepath.Join(nodeDir, "config", "genesis.json") + err = genesis.SaveAs(genesisFilePath) + if err != nil { + return fmt.Errorf("saving genesis: %w", err) + } + + // Create the app.toml file + appConfig, err := MakeAppConfig(n) + if err != nil { + return fmt.Errorf("making app config: %w", err) + } + appConfigFilePath := filepath.Join(nodeDir, "config", "app.toml") + serverconfig.WriteConfigFile(appConfigFilePath, appConfig) + + // Store the node key for the p2p handshake + nodeKeyFilePath := filepath.Join(nodeDir, "config", "node_key.json") + err = (&p2p.NodeKey{PrivKey: n.NetworkKey}).SaveAs(nodeKeyFilePath) + if err != nil { + return err + } + + err = os.Chmod(nodeKeyFilePath, 0o777) + if err != nil { + return fmt.Errorf("chmod node key: %w", err) + } + + // Store the validator signer key for consensus + pvKeyPath := filepath.Join(nodeDir, "config", "priv_validator_key.json") + pvStatePath := filepath.Join(nodeDir, "data", "priv_validator_state.json") + (privval.NewFilePV(n.SignerKey, pvKeyPath, pvStatePath)).Save() + + addrBookFile := filepath.Join(nodeDir, "config", "addrbook.json") + err = WriteAddressBook(peers, addrBookFile) + if err != nil { + return fmt.Errorf("writing address book: %w", err) + } + + _, err = n.Instance.ExecuteCommand(fmt.Sprintf("mkdir -p %s/config", remoteRootDir)) + if err != nil { + return fmt.Errorf("creating config directory: %w", err) + } + _, err = n.Instance.ExecuteCommand(fmt.Sprintf("mkdir -p %s/data", remoteRootDir)) + if err != nil { + return fmt.Errorf("creating data directory: %w", err) + } + + err = n.Instance.AddFile(configFilePath, filepath.Join(remoteRootDir, "config", "config.toml"), "10001:10001") + if err != nil { + return fmt.Errorf("adding config file: %w", err) + } + + err = n.Instance.AddFile(genesisFilePath, filepath.Join(remoteRootDir, "config", "genesis.json"), "10001:10001") + if err != nil { + return fmt.Errorf("adding genesis file: %w", err) + } + + err = n.Instance.AddFile(appConfigFilePath, filepath.Join(remoteRootDir, "config", "app.toml"), "10001:10001") + if err != nil { + return fmt.Errorf("adding app config file: %w", err) + } + + err = n.Instance.AddFile(pvKeyPath, filepath.Join(remoteRootDir, "config", "priv_validator_key.json"), "10001:10001") + if err != nil { + return fmt.Errorf("adding priv_validator_key file: %w", err) + } + + err = n.Instance.AddFile(pvStatePath, filepath.Join(remoteRootDir, "data", "priv_validator_state.json"), "10001:10001") + if err != nil { + return fmt.Errorf("adding priv_validator_state file: %w", err) + } + + err = n.Instance.AddFile(nodeKeyFilePath, filepath.Join(remoteRootDir, "config", "node_key.json"), "10001:10001") + if err != nil { + return fmt.Errorf("adding node_key file: %w", err) + } + + err = n.Instance.AddFile(addrBookFile, filepath.Join(remoteRootDir, "config", "addrbook.json"), "10001:10001") + if err != nil { + return fmt.Errorf("adding addrbook file: %w", err) + } + + return n.Instance.Commit() +} + +// AddressP2P returns a P2P endpoint address for the node. This is used for +// populating the address book. This will look something like: +// 3314051954fc072a0678ec0cbac690ad8676ab98@61.108.66.220:26656 +func (n Node) AddressP2P(withID bool) string { + ip, err := n.Instance.GetIP() + if err != nil { + panic(err) + } + addr := fmt.Sprintf("%v:%d", ip, p2pPort) + if withID { + addr = fmt.Sprintf("%x@%v", n.NetworkKey.PubKey().Address().Bytes(), addr) + } + return addr +} + +// AddressRPC returns an RPC endpoint address for the node. +// This returns the local proxy port that can be used to communicate with the node +func (n Node) AddressRPC() string { + return fmt.Sprintf("http://127.0.0.1:%d", n.rpcProxyPort) +} + +// AddressGRPC returns a GRPC endpoint address for the node. This returns the +// local proxy port that can be used to communicate with the node +func (n Node) AddressGRPC() string { + return fmt.Sprintf("127.0.0.1:%d", n.grpcProxyPort) +} + +func (n Node) IsValidator() bool { + return n.SelfDelegation != 0 +} + +func (n Node) Client() (*http.HTTP, error) { + return http.New(n.AddressRPC(), "/websocket") +} + +func (n *Node) Start() error { + if err := n.Instance.Start(); err != nil { + return err + } + + if err := n.Instance.WaitInstanceIsRunning(); err != nil { + return err + } + + rpcProxyPort, err := n.Instance.PortForwardTCP(rpcPort) + if err != nil { + return fmt.Errorf("forwarding port %d: %w", rpcPort, err) + } + + grpcProxyPort, err := n.Instance.PortForwardTCP(grpcPort) + if err != nil { + return fmt.Errorf("forwarding port %d: %w", grpcPort, err) + } + n.rpcProxyPort = rpcProxyPort + n.grpcProxyPort = grpcProxyPort + return nil +} diff --git a/test/e2e/pkg/exec.go b/test/e2e/pkg/exec.go deleted file mode 100644 index 2410b53193..0000000000 --- a/test/e2e/pkg/exec.go +++ /dev/null @@ -1,33 +0,0 @@ -package e2e - -import ( - "fmt" - osexec "os/exec" - "path/filepath" -) - -// execute executes a shell command. -func exec(args ...string) error { - cmd := osexec.Command(args[0], args[1:]...) - out, err := cmd.CombinedOutput() - switch err := err.(type) { - case nil: - return nil - case *osexec.ExitError: - return fmt.Errorf("failed to run %q:\n%v", args, string(out)) - default: - return err - } -} - -// execCompose runs a Docker Compose command for a testnet. -func execCompose(dir string, args ...string) error { - return exec(append( - []string{"docker compose", "-f", filepath.Join(dir, "docker-compose.yml")}, - args...)...) -} - -// execDocker runs a Docker command. -func execDocker(args ...string) error { - return exec(append([]string{"docker"}, args...)...) -} diff --git a/test/e2e/pkg/lifecycle.go b/test/e2e/pkg/lifecycle.go deleted file mode 100644 index 0f169b498f..0000000000 --- a/test/e2e/pkg/lifecycle.go +++ /dev/null @@ -1,106 +0,0 @@ -package e2e - -import ( - "context" - "errors" - "fmt" - "os" - "path/filepath" -) - -// Start commences the testnet. -func Start(ctx context.Context, testnet *Testnet) error { - fmt.Println("Spinning up testnet") - nodes := testnet.NodesByStartHeight() - for _, node := range nodes { - - if node.StartHeight != 0 { - if err := WaitForHeight(ctx, testnet, node.StartHeight); err != nil { - return err - } - } - - fmt.Printf("Starting %s at height %d on %s\n", node.Name, node.StartHeight, fmt.Sprintf("http://localhost:%d", node.ProxyPort)) - - if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil { - return err - } - } - - return nil -} - -// Stop stops the currently running network -func Stop(_ context.Context, testnet *Testnet) error { - fmt.Println("Stopping testnet") - return execCompose(testnet.Dir, "down") -} - -// Cleanup removes the Docker Compose containers and testnet directory. -func Cleanup(_ context.Context, testnet *Testnet) error { - err := cleanupDocker() - if err != nil { - return err - } - err = cleanupDir(testnet.Dir) - if err != nil { - return err - } - return nil -} - -// cleanupDocker removes all E2E resources (with label e2e=True), regardless -// of testnet. -func cleanupDocker() error { - // GNU xargs requires the -r flag to not run when input is empty, macOS - // does this by default. Ugly, but works. - xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)` - - err := exec("bash", "-c", fmt.Sprintf( - "docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR)) - if err != nil { - return err - } - - err = exec("bash", "-c", fmt.Sprintf( - "docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR)) - if err != nil { - return err - } - - return nil -} - -// cleanupDir cleans up a testnet directory -func cleanupDir(dir string) error { - if dir == "" { - return errors.New("no directory set") - } - - _, err := os.Stat(dir) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return err - } - - // On Linux, some local files in the volume will be owned by root since Tendermint - // runs as root inside the container, so we need to clean them up from within a - // container running as root too. - absDir, err := filepath.Abs(dir) - if err != nil { - return err - } - err = execDocker("run", "--rm", "--entrypoint", "", "-v", fmt.Sprintf("%v:/network", absDir), - "tendermint/e2e-node", "sh", "-c", "rm -rf /network/*/") - if err != nil { - return err - } - - err = os.RemoveAll(dir) - if err != nil { - return err - } - - return nil -} diff --git a/test/e2e/pkg/manifest.go b/test/e2e/pkg/manifest.go deleted file mode 100644 index 82d64f2f89..0000000000 --- a/test/e2e/pkg/manifest.go +++ /dev/null @@ -1,70 +0,0 @@ -package e2e - -import ( - "fmt" - "os" - - "github.com/BurntSushi/toml" -) - -type Manifest struct { - Nodes map[string]*ManifestNode `toml:"node"` - Accounts map[string]*ManifestAccount `toml:"account"` -} - -type ManifestNode struct { - // Versions is an array of binary versions that the node - // will run in it's lifetime. A series of upgrades can be - // triggered by the upgrade command or will happen automatically - // across a testnet. - // - // Version strings are used to pull docker images from ghcr.io. - // Alternatively, use "current", if you want to use the binary - // based on the current branch. You must have the image built by - // running "make docker". Default set to "current" - Versions []string `toml:"versions"` - - // The height that the node will start at - StartHeight int64 `toml:"start_height"` - - // Peers are the set of peers that initially populate the address - // book. Persistent peers and seeds are currently not supported - // By default all nodes declared in the manifest are included - Peers []string `toml:"peers"` - - // SelfDelegation is the delegation of the validator when they - // first come up. If set to 0, the node is not considered a validator - SelfDelegation int64 `toml:"self_delegation"` -} - -// ManifestAccounts represent SDK accounts that sign and -// submit transactions. If the account has the same name as -// the node it is the operator address for the validator. -// Unless specified it will have a default self delegation -// All accounts specfied are created at genesis. -type ManifestAccount struct { - // Tokens symbolizes the genesis supply of liquid tokens to that account - Tokens int64 `toml:"tokens"` - - // The key type to derive the account key from. Defaults to secp256k1 - KeyType string `toml:"key_type"` -} - -// Save saves the testnet manifest to a file. -func (m Manifest) Save(file string) error { - f, err := os.Create(file) - if err != nil { - return fmt.Errorf("failed to create manifest file %q: %w", file, err) - } - return toml.NewEncoder(f).Encode(m) -} - -// LoadManifest loads a testnet manifest from a file. -func LoadManifest(file string) (Manifest, error) { - manifest := Manifest{} - _, err := toml.DecodeFile(file, &manifest) - if err != nil { - return manifest, fmt.Errorf("failed to load testnet manifest %q: %w", file, err) - } - return manifest, nil -} diff --git a/test/e2e/pkg/rpc.go b/test/e2e/pkg/rpc.go deleted file mode 100644 index 070020cbb0..0000000000 --- a/test/e2e/pkg/rpc.go +++ /dev/null @@ -1,129 +0,0 @@ -package e2e - -import ( - "context" - "errors" - "fmt" - "sort" - "time" - - rpchttp "github.com/tendermint/tendermint/rpc/client/http" -) - -const ( - waitForHeightTimeout = 20 * time.Second -) - -// WaitForNBlocks queries the current latest height and waits until the network -// progresses to "blocks" blocks ahead. -func WaitForNBlocks(ctx context.Context, testnet *Testnet, blocks int64) error { - height, err := GetHeights(ctx, testnet) - if err != nil { - return err - } - - fmt.Printf("Waiting for the network to reach height %d\n", height[0]+blocks) - - return WaitForHeight(ctx, testnet, height[0]+blocks) -} - -// WaitForHeight waits until the first node reaches the height specified. If -// no progress is made within a 20 second window then the function times out -// with an error. -func WaitForHeight(ctx context.Context, testnet *Testnet, height int64) error { - var ( - err error - maxHeight int64 - clients = map[string]*rpchttp.HTTP{} - lastIncrease = time.Now() - ) - - for { - for _, node := range testnet.Nodes { - if node.StartHeight > height { - continue - } - client, ok := clients[node.Name] - if !ok { - client, err = node.Client() - if err != nil { - continue - } - clients[node.Name] = client - } - subctx, cancel := context.WithTimeout(ctx, 1*time.Second) - defer cancel() - - // request the status of the node - result, err := client.Status(subctx) - if err != nil { - if errors.Is(err, context.DeadlineExceeded) { - return err - } - continue - } - - if result.SyncInfo.LatestBlockHeight >= height { - return nil - } - - if result.SyncInfo.LatestBlockHeight > maxHeight { - maxHeight = result.SyncInfo.LatestBlockHeight - lastIncrease = time.Now() - } - - // If no progress has been made in the last 20 seconds, return an error. - if time.Since(lastIncrease) > waitForHeightTimeout { - return fmt.Errorf("network unable to reach next height within %s (max height: %d, target height: %d)", - waitForHeightTimeout, maxHeight, height, - ) - } - - time.Sleep(1 * time.Second) - } - } -} - -// GetHeights loops through all running nodes and returns an array of heights -// in order of highest to lowest. -func GetHeights(ctx context.Context, testnet *Testnet) ([]int64, error) { - var ( - err error - heights = make([]int64, 0, len(testnet.Nodes)) - clients = map[string]*rpchttp.HTTP{} - ) - - for _, node := range testnet.Nodes { - client, ok := clients[node.Name] - if !ok { - client, err = node.Client() - if err != nil { - continue - } - clients[node.Name] = client - } - subctx, cancel := context.WithTimeout(ctx, 1*time.Second) - defer cancel() - - // request the status of the node - result, err := client.Status(subctx) - if err != nil { - if errors.Is(err, context.DeadlineExceeded) { - return nil, err - } - continue - } - - heights = append(heights, result.SyncInfo.LatestBlockHeight) - } - if len(heights) == 0 { - return nil, errors.New("network is not running") - } - - // return heights in descending order - sort.Slice(heights, func(i, j int) bool { - return heights[i] > heights[j] - }) - - return heights, nil -} diff --git a/test/e2e/pkg/setup.go b/test/e2e/pkg/setup.go deleted file mode 100644 index 2861a7d508..0000000000 --- a/test/e2e/pkg/setup.go +++ /dev/null @@ -1,351 +0,0 @@ -package e2e - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "html/template" - "io" - "os" - "path/filepath" - "strings" - "time" - - "github.com/celestiaorg/celestia-app/app" - "github.com/celestiaorg/celestia-app/app/encoding" - codectypes "github.com/cosmos/cosmos-sdk/codec/types" - cryptocodec "github.com/cosmos/cosmos-sdk/crypto/codec" - serverconfig "github.com/cosmos/cosmos-sdk/server/config" - sdk "github.com/cosmos/cosmos-sdk/types" - auth "github.com/cosmos/cosmos-sdk/x/auth/types" - bank "github.com/cosmos/cosmos-sdk/x/bank/types" - slashing "github.com/cosmos/cosmos-sdk/x/slashing/types" - staking "github.com/cosmos/cosmos-sdk/x/staking/types" - dockertypes "github.com/docker/docker/api/types" - "github.com/docker/docker/client" - "github.com/ethereum/go-ethereum/common" - "github.com/tendermint/tendermint/config" - "github.com/tendermint/tendermint/crypto" - "github.com/tendermint/tendermint/p2p" - "github.com/tendermint/tendermint/p2p/pex" - "github.com/tendermint/tendermint/privval" - "github.com/tendermint/tendermint/types" -) - -func Setup(ctx context.Context, testnet *Testnet) error { - // Ensure that all the requisite images are available - if err := SetupImages(ctx, testnet); err != nil { - return fmt.Errorf("setting up images: %w", err) - } - - fmt.Printf("Setting up network %s\n", testnet.Name) - - _, err := os.Stat(testnet.Dir) - if err == nil { - return errors.New("testnet directory already exists") - } else if !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("checking if testnet directory exists: %w", err) - } - - // Create the directory for the testnet - if err := os.MkdirAll(testnet.Dir, os.ModePerm); err != nil { - return err - } - cleanup := func() { os.RemoveAll(testnet.Dir) } - - // Create the docker compose file - if err := WriteDockerCompose(testnet, filepath.Join(testnet.Dir, "docker-compose.yml")); err != nil { - cleanup() - return fmt.Errorf("setting up docker compose: %w", err) - } - - // Make the genesis file for the testnet - genesis, err := MakeGenesis(testnet) - if err != nil { - cleanup() - return fmt.Errorf("making genesis: %w", err) - } - - // Initialize the file system and configs for each node - for name, node := range testnet.Nodes { - err := InitNode(node, genesis, testnet.Dir) - if err != nil { - cleanup() - return fmt.Errorf("initializing node %s: %w", name, err) - } - } - - return nil -} - -// SetupImages ensures that all the requisite docker images for each -// used celestia consensus version. -func SetupImages(ctx context.Context, testnet *Testnet) error { - c, err := client.NewClientWithOpts() - if err != nil { - return fmt.Errorf("establishing docker client: %v", err) - } - - versions := testnet.GetAllVersions() - - for _, v := range versions { - if v == "current" { - // we assume that the user has locally downloaded - // the current docker image - continue - } - refStr := dockerSrcURL + ":" + v - fmt.Printf("Pulling in docker image: %s\n", refStr) - rc, err := c.ImagePull(ctx, refStr, dockertypes.ImagePullOptions{}) - if err != nil { - return fmt.Errorf("error pulling image %s: %w", refStr, err) - } - _, _ = io.Copy(io.Discard, rc) - _ = rc.Close() - } - - return nil -} - -func MakeGenesis(testnet *Testnet) (types.GenesisDoc, error) { - encCdc := encoding.MakeConfig(app.ModuleEncodingRegisters...) - appGenState := app.ModuleBasics.DefaultGenesis(encCdc.Codec) - bankGenesis := bank.DefaultGenesisState() - stakingGenesis := staking.DefaultGenesisState() - slashingGenesis := slashing.DefaultGenesisState() - genAccs := []auth.GenesisAccount{} - stakingGenesis.Params.BondDenom = app.BondDenom - delegations := make([]staking.Delegation, 0, len(testnet.Nodes)) - valInfo := make([]slashing.SigningInfo, 0, len(testnet.Nodes)) - balances := make([]bank.Balance, 0, len(testnet.Accounts)+1) - var ( - validators staking.Validators - totalBonded int64 - ) - - // setup the validator information on the state machine - for name, node := range testnet.Nodes { - if !node.IsValidator() || node.StartHeight != 0 { - continue - } - - addr := node.AccountKey.PubKey().Address() - pk, err := cryptocodec.FromTmPubKeyInterface(node.SignerKey.PubKey()) - if err != nil { - return types.GenesisDoc{}, fmt.Errorf("converting public key for node %s: %w", node.Name, err) - } - pkAny, err := codectypes.NewAnyWithValue(pk) - if err != nil { - return types.GenesisDoc{}, err - } - evmAddress := common.HexToAddress(crypto.CRandHex(common.AddressLength)) - - validators = append(validators, staking.Validator{ - OperatorAddress: sdk.ValAddress(addr).String(), - ConsensusPubkey: pkAny, - Description: staking.Description{ - Moniker: name, - }, - Status: staking.Bonded, - Tokens: sdk.NewInt(node.SelfDelegation), - DelegatorShares: sdk.OneDec(), - // 5% commission - Commission: staking.NewCommission(sdk.NewDecWithPrec(5, 2), sdk.OneDec(), sdk.OneDec()), - MinSelfDelegation: sdk.ZeroInt(), - EvmAddress: evmAddress.Hex(), - }) - totalBonded += node.SelfDelegation - consensusAddr := pk.Address() - delegations = append(delegations, staking.NewDelegation(sdk.AccAddress(addr), sdk.ValAddress(addr), sdk.OneDec())) - valInfo = append(valInfo, slashing.SigningInfo{ - Address: sdk.ConsAddress(consensusAddr).String(), - ValidatorSigningInfo: slashing.NewValidatorSigningInfo(sdk.ConsAddress(consensusAddr), 1, 0, time.Unix(0, 0), false, 0), - }) - } - stakingGenesis.Delegations = delegations - stakingGenesis.Validators = validators - slashingGenesis.SigningInfos = valInfo - - accountNumber := uint64(0) - for _, account := range testnet.Accounts { - pk, err := cryptocodec.FromTmPubKeyInterface(account.Key.PubKey()) - if err != nil { - return types.GenesisDoc{}, fmt.Errorf("converting public key for account %s: %w", account.Name, err) - } - - addr := pk.Address() - acc := auth.NewBaseAccount(addr.Bytes(), pk, accountNumber, 0) - genAccs = append(genAccs, acc) - balances = append(balances, bank.Balance{ - Address: sdk.AccAddress(addr).String(), - Coins: sdk.NewCoins( - sdk.NewCoin(app.BondDenom, sdk.NewInt(account.Tokens)), - ), - }) - } - // add bonded amount to bonded pool module account - balances = append(balances, bank.Balance{ - Address: auth.NewModuleAddress(staking.BondedPoolName).String(), - Coins: sdk.Coins{sdk.NewCoin(app.BondDenom, sdk.NewInt(totalBonded))}, - }) - bankGenesis.Balances = bank.SanitizeGenesisBalances(balances) - authGenesis := auth.NewGenesisState(auth.DefaultParams(), genAccs) - - // update the original genesis state - appGenState[bank.ModuleName] = encCdc.Codec.MustMarshalJSON(bankGenesis) - appGenState[auth.ModuleName] = encCdc.Codec.MustMarshalJSON(authGenesis) - appGenState[staking.ModuleName] = encCdc.Codec.MustMarshalJSON(stakingGenesis) - appGenState[slashing.ModuleName] = encCdc.Codec.MustMarshalJSON(slashingGenesis) - - if err := app.ModuleBasics.ValidateGenesis(encCdc.Codec, encCdc.TxConfig, appGenState); err != nil { - return types.GenesisDoc{}, fmt.Errorf("validating genesis: %w", err) - } - - appState, err := json.MarshalIndent(appGenState, "", " ") - if err != nil { - return types.GenesisDoc{}, fmt.Errorf("marshaling app state: %w", err) - } - - // Validator set and app hash are set in InitChain - return types.GenesisDoc{ - ChainID: testnet.Name, - GenesisTime: time.Now().UTC(), - ConsensusParams: types.DefaultConsensusParams(), - AppState: appState, - // AppHash is not provided but computed after InitChain - }, nil -} - -func MakeConfig(node *Node) (*config.Config, error) { - cfg := config.DefaultConfig() - cfg.Moniker = node.Name - cfg.RPC.ListenAddress = "tcp://0.0.0.0:26657" - cfg.P2P.ExternalAddress = fmt.Sprintf("tcp://%v", node.AddressP2P(false)) - cfg.P2P.PersistentPeers = strings.Join(node.Peers, ",") - - // TODO: when we use adaptive timeouts, add a parameter in the testnet manifest - // to set block times - // FIXME: This values get overridden by the timeout consts in the app package. - // We should modify this if we want to quicken the time of the blocks. - cfg.Consensus.TimeoutPropose = 1000 * time.Millisecond - cfg.Consensus.TimeoutCommit = 100 * time.Millisecond - return cfg, nil -} - -func InitNode(node *Node, genesis types.GenesisDoc, rootDir string) error { - // Initialize file directories - nodeDir := filepath.Join(rootDir, node.Name) - for _, dir := range []string{ - filepath.Join(nodeDir, "config"), - filepath.Join(nodeDir, "data"), - } { - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return fmt.Errorf("error creating directory %s: %w", dir, err) - } - } - - // Create and write the config file - cfg, err := MakeConfig(node) - if err != nil { - return fmt.Errorf("making config: %w", err) - } - config.WriteConfigFile(filepath.Join(nodeDir, "config", "config.toml"), cfg) - - // Store the genesis file - err = genesis.SaveAs(filepath.Join(nodeDir, "config", "genesis.json")) - if err != nil { - return fmt.Errorf("saving genesis: %w", err) - } - - // Create the app.toml file - appConfig, err := MakeAppConfig(node) - if err != nil { - return fmt.Errorf("making app config: %w", err) - } - serverconfig.WriteConfigFile(filepath.Join(nodeDir, "config", "app.toml"), appConfig) - - // Store the node key for the p2p handshake - err = (&p2p.NodeKey{PrivKey: node.NetworkKey}).SaveAs(filepath.Join(nodeDir, "config", "node_key.json")) - if err != nil { - return err - } - - // Store the validator signer key for consensus - (privval.NewFilePV(node.SignerKey, - filepath.Join(nodeDir, "config", "priv_validator_key.json"), - filepath.Join(nodeDir, "data", "priv_validator_state.json"), - )).Save() - - return nil -} - -func WriteDockerCompose(testnet *Testnet, file string) error { - tmpl, err := template.New("docker-compose").Parse(`version: '2.4' - -networks: - {{ .Name }}: - labels: - e2e: true - driver: bridge - ipam: - driver: default - config: - - subnet: {{ .IP }} - -services: -{{- range .Nodes }} - {{ .Name }}: - labels: - e2e: true - container_name: {{ .Name }} - image: ghcr.io/celestiaorg/celestia-app:{{ index .Versions 0 }} - entrypoint: ["/bin/celestia-appd"] - command: ["start"] - init: true - ports: - - 26656 - - {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657 - - 6060 - - 9090 - - 1317 - volumes: - - ./{{ .Name }}:/home/celestia/.celestia-app - networks: - {{ $.Name }}: - ipv4_address: {{ .IP }} - -{{end}}`) - if err != nil { - return err - } - var buf bytes.Buffer - err = tmpl.Execute(&buf, testnet) - if err != nil { - return err - } - return os.WriteFile(file, buf.Bytes(), 0o644) -} - -func WriteAddressBook(peers []string, file string) error { - book := pex.NewAddrBook(file, true) - for _, peer := range peers { - addr, err := p2p.NewNetAddressString(peer) - if err != nil { - return fmt.Errorf("parsing peer address %s: %w", peer, err) - } - err = book.AddAddress(addr, addr) - if err != nil { - return fmt.Errorf("adding peer address %s: %w", peer, err) - } - } - book.Save() - return nil -} - -func MakeAppConfig(_ *Node) (*serverconfig.Config, error) { - srvCfg := serverconfig.DefaultConfig() - srvCfg.MinGasPrices = fmt.Sprintf("0.001%s", app.BondDenom) - return srvCfg, srvCfg.ValidateBasic() -} diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go deleted file mode 100644 index d7b15d7fda..0000000000 --- a/test/e2e/pkg/testnet.go +++ /dev/null @@ -1,316 +0,0 @@ -package e2e - -import ( - "errors" - "fmt" - "io" - "math" - "math/rand" - "net" - "path/filepath" - "sort" - "strings" - - "github.com/tendermint/tendermint/crypto" - "github.com/tendermint/tendermint/crypto/ed25519" - "github.com/tendermint/tendermint/crypto/secp256k1" - "github.com/tendermint/tendermint/rpc/client/http" -) - -const ( - secp256k1Type = "secp256k1" - ed25519Type = "ed25519" - networkIPv4 = "10.186.73.0/24" - firstProxyPort uint32 = 4201 - dockerSrcURL = "ghcr.io/celestiaorg/celestia-app" - randomSeed int64 = 589308084734268 - defaultAccountTokens = 1e6 - rpcPort = 26657 -) - -type Testnet struct { - Name string // also used as the chain-id - Dir string - IP *net.IPNet - Nodes map[string]*Node - Accounts map[string]*Account -} - -type Node struct { - Name string - Versions []string - StartHeight int64 - Peers []string - SignerKey crypto.PrivKey - NetworkKey crypto.PrivKey - AccountKey crypto.PrivKey - IP net.IP - ProxyPort uint32 - SelfDelegation int64 -} - -type Account struct { - Name string - Tokens int64 - Key crypto.PrivKey -} - -func LoadTestnet(manifest Manifest, file string) (*Testnet, error) { - // the directory that the toml file is located in - dir := strings.TrimSuffix(file, filepath.Ext(file)) - name := fmt.Sprintf("%s-%d", filepath.Base(dir), rand.Intn(math.MaxUint16)) - _, ipNet, err := net.ParseCIDR(networkIPv4) - if err != nil { - return nil, fmt.Errorf("invalid IP network address %q: %w", networkIPv4, err) - } - ipGen := newIPGenerator(ipNet) - keyGen := newKeyGenerator(randomSeed) - proxyPort := firstProxyPort - - testnet := &Testnet{ - Dir: dir, - Name: name, - Nodes: make(map[string]*Node), - Accounts: make(map[string]*Account), - IP: ipGen.Network(), - } - - // deterministically sort names in alphabetical order - nodeNames := []string{} - for name := range manifest.Nodes { - nodeNames = append(nodeNames, name) - } - sort.Strings(nodeNames) - - for _, name := range nodeNames { - nodeManifest := manifest.Nodes[name] - if _, ok := testnet.Nodes[name]; ok { - return nil, fmt.Errorf("duplicate node name %s", name) - } - node := &Node{ - Name: name, - Versions: nodeManifest.Versions, - StartHeight: nodeManifest.StartHeight, - Peers: nodeManifest.Peers, - SignerKey: keyGen.Generate(ed25519Type), - NetworkKey: keyGen.Generate(ed25519Type), - AccountKey: keyGen.Generate(secp256k1Type), - SelfDelegation: nodeManifest.SelfDelegation, - IP: ipGen.Next(), - ProxyPort: proxyPort, - } - if len(node.Versions) == 0 { - node.Versions = []string{"current"} - } - - testnet.Nodes[name] = node - proxyPort++ - } - - for name, node := range testnet.Nodes { - // fill up the peers field if it is empty - if len(node.Peers) == 0 { - for otherName := range testnet.Nodes { - if otherName == name { - continue - } - node.Peers = append(node.Peers, otherName) - } - } - // replace the peer names with the P2P address. - for idx, peer := range node.Peers { - node.Peers[idx] = testnet.Nodes[peer].AddressP2P(true) - } - } - - // deterministically sort accounts in alphabetical order - accountNames := []string{} - for name := range manifest.Accounts { - accountNames = append(accountNames, name) - } - sort.Strings(accountNames) - - for _, name := range accountNames { - accountManifest := manifest.Accounts[name] - if _, ok := testnet.Accounts[name]; ok { - return nil, fmt.Errorf("duplicate account name %s", name) - } - account := &Account{ - Name: name, - Tokens: accountManifest.Tokens, - Key: keyGen.Generate(accountManifest.KeyType), - } - if account.Tokens == 0 { - account.Tokens = defaultAccountTokens - } - testnet.Accounts[name] = account - } - - return testnet, testnet.Validate() -} - -func (t *Testnet) Validate() (err error) { - if len(t.Accounts) == 0 { - return errors.New("at least one account is required") - } - if len(t.Nodes) == 0 { - return errors.New("at least one node is required") - } - validators := 0 - for name, node := range t.Nodes { - if err := node.Validate(); err != nil { - return fmt.Errorf("invalid node %s: %w", name, err) - } - // must have at least one validator - if node.SelfDelegation > 0 { - validators++ - } - } - if validators == 0 { - return errors.New("at least one node must a validator by having an associated account") - } - for _, account := range t.Accounts { - if err := account.Validate(); err != nil { - return err - } - } - - return nil -} - -func (t *Testnet) GetAllVersions() []string { - versions := make(map[string]struct{}) - // filter duplicate version strings - for _, node := range t.Nodes { - for _, version := range node.Versions { - versions[version] = struct{}{} - } - } - - // convert to list - versionsList := []string{} - for version := range versions { - versionsList = append(versionsList, version) - } - return versionsList -} - -func (t *Testnet) NodesByStartHeight() []*Node { - nodes := make([]*Node, 0, len(t.Nodes)) - for _, node := range t.Nodes { - nodes = append(nodes, node) - } - sort.Slice(nodes, func(i, j int) bool { - if nodes[i].StartHeight == nodes[j].StartHeight { - return nodes[i].Name < nodes[j].Name - } - return nodes[i].StartHeight < nodes[j].StartHeight - }) - return nodes -} - -// Address returns a P2P endpoint address for the node. -func (n Node) AddressP2P(withID bool) string { - addr := fmt.Sprintf("%v:26656", n.IP.String()) - if withID { - addr = fmt.Sprintf("%x@%v", n.NetworkKey.PubKey().Address().Bytes(), addr) - } - return addr -} - -// Address returns an RPC endpoint address for the node. -func (n Node) AddressRPC() string { - return fmt.Sprintf("%v:%d", n.IP.String(), rpcPort) -} - -func (n Node) IsValidator() bool { - return n.SelfDelegation != 0 -} - -func (n Node) Client() (*http.HTTP, error) { - return http.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort), "/websocket") -} - -func (n Node) Validate() error { - if len(n.Versions) == 0 { - return errors.New("at least one version is required") - } - if n.StartHeight < 0 { - return errors.New("start height must be non-negative") - } - if n.SelfDelegation < 0 { - return errors.New("self delegation must be non-negative") - } - return nil -} - -func (a Account) Validate() error { - if a.Tokens < 0 { - return errors.New("tokens must be non-negative") - } - return nil -} - -type keyGenerator struct { - random *rand.Rand -} - -func newKeyGenerator(seed int64) *keyGenerator { - return &keyGenerator{ - random: rand.New(rand.NewSource(seed)), //nolint:gosec - } -} - -func (g *keyGenerator) Generate(keyType string) crypto.PrivKey { - seed := make([]byte, ed25519.SeedSize) - - _, err := io.ReadFull(g.random, seed) - if err != nil { - panic(err) // this shouldn't happen - } - switch keyType { - case "secp256k1": - return secp256k1.GenPrivKeySecp256k1(seed) - case "", "ed25519": - return ed25519.GenPrivKeyFromSecret(seed) - default: - panic("KeyType not supported") // should not make it this far - } -} - -type ipGenerator struct { - network *net.IPNet - nextIP net.IP -} - -func newIPGenerator(network *net.IPNet) *ipGenerator { - nextIP := make([]byte, len(network.IP)) - copy(nextIP, network.IP) - gen := &ipGenerator{network: network, nextIP: nextIP} - // Skip network and gateway addresses - gen.Next() - gen.Next() - return gen -} - -func (g *ipGenerator) Network() *net.IPNet { - n := &net.IPNet{ - IP: make([]byte, len(g.network.IP)), - Mask: make([]byte, len(g.network.Mask)), - } - copy(n.IP, g.network.IP) - copy(n.Mask, g.network.Mask) - return n -} - -func (g *ipGenerator) Next() net.IP { - ip := make([]byte, len(g.nextIP)) - copy(ip, g.nextIP) - for i := len(g.nextIP) - 1; i >= 0; i-- { - g.nextIP[i]++ - if g.nextIP[i] != 0 { - break - } - } - return ip -} diff --git a/test/e2e/readme.md b/test/e2e/readme.md new file mode 100644 index 0000000000..dadd80bbbf --- /dev/null +++ b/test/e2e/readme.md @@ -0,0 +1,15 @@ +# End to End Testing + +Celestia uses the [knuu](https://github.com/celestiaorg/knuu) framework to orchestrate clusters of nodes in a network for end to end testing. This relies on Docker and a kubeconfig (in `~/.kube/config`) to access the Kubernetes cluster. + +## Usage + +E2E tests can be simply run through go tests. They are distinguished from unit tets through an environment variable. To run all e2e tests run: + +```shell +E2E=true KNUU_NAMESPACE=celestia-app go test ./test/e2e/... -timeout 30m +``` + +## Observation + +Logs of each of the nodes are posted to Grafana and can be accessed through Celestia's dashboard (using the `celestia-app` namespace). diff --git a/test/e2e/setup.go b/test/e2e/setup.go new file mode 100644 index 0000000000..ce9ba5173c --- /dev/null +++ b/test/e2e/setup.go @@ -0,0 +1,170 @@ +package e2e + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/celestiaorg/celestia-app/app" + "github.com/celestiaorg/celestia-app/app/encoding" + codectypes "github.com/cosmos/cosmos-sdk/codec/types" + cryptocodec "github.com/cosmos/cosmos-sdk/crypto/codec" + cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" + serverconfig "github.com/cosmos/cosmos-sdk/server/config" + sdk "github.com/cosmos/cosmos-sdk/types" + auth "github.com/cosmos/cosmos-sdk/x/auth/types" + bank "github.com/cosmos/cosmos-sdk/x/bank/types" + slashing "github.com/cosmos/cosmos-sdk/x/slashing/types" + staking "github.com/cosmos/cosmos-sdk/x/staking/types" + "github.com/ethereum/go-ethereum/common" + "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/p2p/pex" + "github.com/tendermint/tendermint/types" +) + +type GenesisAccount struct { + PubKey cryptotypes.PubKey + InitialTokens int64 +} + +func MakeGenesis(nodes []*Node, accounts []*GenesisAccount) (types.GenesisDoc, error) { + encCdc := encoding.MakeConfig(app.ModuleEncodingRegisters...) + appGenState := app.ModuleBasics.DefaultGenesis(encCdc.Codec) + bankGenesis := bank.DefaultGenesisState() + stakingGenesis := staking.DefaultGenesisState() + slashingGenesis := slashing.DefaultGenesisState() + genAccs := []auth.GenesisAccount{} + stakingGenesis.Params.BondDenom = app.BondDenom + delegations := make([]staking.Delegation, 0, len(nodes)) + valInfo := make([]slashing.SigningInfo, 0, len(nodes)) + balances := make([]bank.Balance, 0, len(accounts)) + var ( + validators staking.Validators + totalBonded int64 + ) + + // setup the validator information on the state machine + for _, node := range nodes { + if !node.IsValidator() || node.StartHeight != 0 { + continue + } + + addr := node.AccountKey.PubKey().Address() + pk, err := cryptocodec.FromTmPubKeyInterface(node.SignerKey.PubKey()) + if err != nil { + return types.GenesisDoc{}, fmt.Errorf("converting public key for node %s: %w", node.Name, err) + } + pkAny, err := codectypes.NewAnyWithValue(pk) + if err != nil { + return types.GenesisDoc{}, err + } + evmAddress := common.HexToAddress(crypto.CRandHex(common.AddressLength)) + + validators = append(validators, staking.Validator{ + OperatorAddress: sdk.ValAddress(addr).String(), + ConsensusPubkey: pkAny, + Description: staking.Description{ + Moniker: node.Name, + }, + Status: staking.Bonded, + Tokens: sdk.NewInt(node.SelfDelegation), + DelegatorShares: sdk.OneDec(), + // 5% commission + Commission: staking.NewCommission(sdk.NewDecWithPrec(5, 2), sdk.OneDec(), sdk.OneDec()), + MinSelfDelegation: sdk.ZeroInt(), + EvmAddress: evmAddress.Hex(), + }) + totalBonded += node.SelfDelegation + consensusAddr := pk.Address() + delegations = append(delegations, staking.NewDelegation(sdk.AccAddress(addr), sdk.ValAddress(addr), sdk.OneDec())) + valInfo = append(valInfo, slashing.SigningInfo{ + Address: sdk.ConsAddress(consensusAddr).String(), + ValidatorSigningInfo: slashing.NewValidatorSigningInfo(sdk.ConsAddress(consensusAddr), 1, 0, time.Unix(0, 0), false, 0), + }) + } + stakingGenesis.Delegations = delegations + stakingGenesis.Validators = validators + slashingGenesis.SigningInfos = valInfo + + for idx, account := range accounts { + addr := account.PubKey.Address() + acc := auth.NewBaseAccount(addr.Bytes(), account.PubKey, uint64(idx), 0) + genAccs = append(genAccs, acc) + if account.InitialTokens == 0 { + return types.GenesisDoc{}, fmt.Errorf("account %s has no initial tokens", addr) + } + balances = append(balances, bank.Balance{ + Address: sdk.AccAddress(addr).String(), + Coins: sdk.NewCoins( + sdk.NewCoin(app.BondDenom, sdk.NewInt(account.InitialTokens)), + ), + }) + } + // add bonded amount to bonded pool module account + balances = append(balances, bank.Balance{ + Address: auth.NewModuleAddress(staking.BondedPoolName).String(), + Coins: sdk.Coins{sdk.NewCoin(app.BondDenom, sdk.NewInt(totalBonded))}, + }) + bankGenesis.Balances = bank.SanitizeGenesisBalances(balances) + authGenesis := auth.NewGenesisState(auth.DefaultParams(), genAccs) + + // update the original genesis state + appGenState[bank.ModuleName] = encCdc.Codec.MustMarshalJSON(bankGenesis) + appGenState[auth.ModuleName] = encCdc.Codec.MustMarshalJSON(authGenesis) + appGenState[staking.ModuleName] = encCdc.Codec.MustMarshalJSON(stakingGenesis) + appGenState[slashing.ModuleName] = encCdc.Codec.MustMarshalJSON(slashingGenesis) + + if err := app.ModuleBasics.ValidateGenesis(encCdc.Codec, encCdc.TxConfig, appGenState); err != nil { + return types.GenesisDoc{}, fmt.Errorf("validating genesis: %w", err) + } + + appState, err := json.MarshalIndent(appGenState, "", " ") + if err != nil { + return types.GenesisDoc{}, fmt.Errorf("marshaling app state: %w", err) + } + + // Validator set and app hash are set in InitChain + return types.GenesisDoc{ + ChainID: "testnet", + GenesisTime: time.Now().UTC(), + ConsensusParams: app.DefaultConsensusParams(), + AppState: appState, + // AppHash is not provided but computed after InitChain + }, nil +} + +func MakeConfig(node *Node) (*config.Config, error) { + cfg := config.DefaultConfig() + cfg.Moniker = node.Name + cfg.RPC.ListenAddress = "tcp://0.0.0.0:26657" + cfg.P2P.ExternalAddress = fmt.Sprintf("tcp://%v", node.AddressP2P(false)) + cfg.P2P.PersistentPeers = strings.Join(node.InitialPeers, ",") + cfg.Consensus.TimeoutPropose = time.Second + cfg.Consensus.TimeoutCommit = time.Second + return cfg, nil +} + +func WriteAddressBook(peers []string, file string) error { + book := pex.NewAddrBook(file, false) + for _, peer := range peers { + addr, err := p2p.NewNetAddressString(peer) + if err != nil { + return fmt.Errorf("parsing peer address %s: %w", peer, err) + } + err = book.AddAddress(addr, addr) + if err != nil { + return fmt.Errorf("adding peer address %s: %w", peer, err) + } + } + book.Save() + return nil +} + +func MakeAppConfig(_ *Node) (*serverconfig.Config, error) { + srvCfg := serverconfig.DefaultConfig() + srvCfg.MinGasPrices = fmt.Sprintf("0.001%s", app.BondDenom) + return srvCfg, srvCfg.ValidateBasic() +} diff --git a/test/e2e/simple_test.go b/test/e2e/simple_test.go new file mode 100644 index 0000000000..1fd2af3292 --- /dev/null +++ b/test/e2e/simple_test.go @@ -0,0 +1,55 @@ +package e2e + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/celestiaorg/celestia-app/test/txsim" + "github.com/celestiaorg/celestia-app/test/util/testnode" + "github.com/stretchr/testify/require" +) + +const ( + latestVersion = "v1.0.0-rc9" + seed = 42 // the meaning of life +) + +// This test runs a simple testnet with 4 validators. It submits both MsgPayForBlobs +// and MsgSends over 30 seconds and then asserts that at least 10 transactions were +// committed. +func TestE2ESimple(t *testing.T) { + if os.Getenv("E2E") == "" { + t.Skip("skipping e2e test") + } + + testnet, err := New(t.Name(), seed) + require.NoError(t, err) + t.Cleanup(testnet.Cleanup) + require.NoError(t, testnet.CreateGenesisNodes(4, latestVersion, 10000000)) + + kr, err := testnet.CreateGenesisAccount("alice", 1e12) + require.NoError(t, err) + + require.NoError(t, testnet.Setup()) + require.NoError(t, testnet.Start()) + + sequences := txsim.NewBlobSequence(txsim.NewRange(200, 4000), txsim.NewRange(1, 3)).Clone(5) + sequences = append(sequences, txsim.NewSendSequence(4, 1000, 100).Clone(5)...) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + err = txsim.Run(ctx, testnet.RPCEndpoints(), testnet.GRPCEndpoints(), kr, seed, 3*time.Second, sequences...) + require.True(t, errors.Is(err, context.DeadlineExceeded), err.Error()) + + blockchain, err := testnode.ReadBlockchain(context.Background(), testnet.Node(0).AddressRPC()) + require.NoError(t, err) + + totalTxs := 0 + for _, block := range blockchain { + totalTxs += len(block.Data.Txs) + } + require.Greater(t, totalTxs, 10) +} diff --git a/test/e2e/testnet.go b/test/e2e/testnet.go new file mode 100644 index 0000000000..ad3a104662 --- /dev/null +++ b/test/e2e/testnet.go @@ -0,0 +1,179 @@ +package e2e + +import ( + "context" + "fmt" + "time" + + "github.com/celestiaorg/celestia-app/app" + "github.com/celestiaorg/celestia-app/app/encoding" + "github.com/celestiaorg/knuu/pkg/knuu" + "github.com/cosmos/cosmos-sdk/crypto/hd" + "github.com/cosmos/cosmos-sdk/crypto/keyring" + "github.com/rs/zerolog/log" +) + +type Testnet struct { + seed int64 + nodes []*Node + genesisAccounts []*GenesisAccount + keygen *keyGenerator +} + +func New(name string, seed int64) (*Testnet, error) { + identifier := fmt.Sprintf("%s_%s", name, time.Now().Format("20060102_150405")) + if err := knuu.InitializeWithIdentifier(identifier); err != nil { + return nil, err + } + + return &Testnet{ + seed: seed, + nodes: make([]*Node, 0), + genesisAccounts: make([]*GenesisAccount, 0), + keygen: newKeyGenerator(seed), + }, nil +} + +func (t *Testnet) CreateGenesisNode(version string, selfDelegation int64) error { + signerKey := t.keygen.Generate(ed25519Type) + networkKey := t.keygen.Generate(ed25519Type) + accountKey := t.keygen.Generate(secp256k1Type) + node, err := NewNode(fmt.Sprintf("val%d", len(t.nodes)), version, 0, selfDelegation, nil, signerKey, networkKey, accountKey) + if err != nil { + return err + } + t.nodes = append(t.nodes, node) + return nil +} + +func (t *Testnet) CreateGenesisNodes(num int, version string, selfDelegation int64) error { + for i := -0; i < num; i++ { + if err := t.CreateGenesisNode(version, selfDelegation); err != nil { + return err + } + } + return nil +} + +func (t *Testnet) CreateNode(version string, startHeight int64) error { + signerKey := t.keygen.Generate(ed25519Type) + networkKey := t.keygen.Generate(ed25519Type) + accountKey := t.keygen.Generate(secp256k1Type) + node, err := NewNode(fmt.Sprintf("val%d", len(t.nodes)), version, startHeight, 0, nil, signerKey, networkKey, accountKey) + if err != nil { + return err + } + t.nodes = append(t.nodes, node) + return nil +} + +func (t *Testnet) CreateGenesisAccount(name string, tokens int64) (keyring.Keyring, error) { + cdc := encoding.MakeConfig(app.ModuleEncodingRegisters...).Codec + kr := keyring.NewInMemory(cdc) + key, _, err := kr.NewMnemonic(name, keyring.English, "", "", hd.Secp256k1) + if err != nil { + return nil, err + } + pk, err := key.GetPubKey() + if err != nil { + return nil, err + } + t.genesisAccounts = append(t.genesisAccounts, &GenesisAccount{ + PubKey: pk, + InitialTokens: tokens, + }) + return kr, nil +} + +func (t *Testnet) Setup() error { + genesisNodes := make([]*Node, 0) + for _, node := range t.nodes { + if node.StartHeight == 0 && node.SelfDelegation > 0 { + genesisNodes = append(genesisNodes, node) + } + } + genesis, err := MakeGenesis(genesisNodes, t.genesisAccounts) + if err != nil { + return err + } + for _, node := range t.nodes { + // nodes are initialized with the addresses of all other + // nodes in their addressbook + peers := make([]string, 0, len(t.nodes)-1) + for _, peer := range t.nodes { + if peer.Name != node.Name { + peers = append(peers, peer.AddressP2P(true)) + } + } + + err = node.Init(genesis, peers) + if err != nil { + return err + } + } + return nil +} + +func (t *Testnet) RPCEndpoints() []string { + rpcEndpoints := make([]string, len(t.nodes)) + for idx, node := range t.nodes { + rpcEndpoints[idx] = node.AddressRPC() + } + return rpcEndpoints +} + +func (t *Testnet) GRPCEndpoints() []string { + grpcEndpoints := make([]string, len(t.nodes)) + for idx, node := range t.nodes { + grpcEndpoints[idx] = node.AddressGRPC() + } + return grpcEndpoints +} + +func (t *Testnet) Start() error { + genesisNodes := make([]*Node, 0) + for _, node := range t.nodes { + if node.StartHeight == 0 { + genesisNodes = append(genesisNodes, node) + } + } + for _, node := range genesisNodes { + err := node.Start() + if err != nil { + return fmt.Errorf("node %s failed to start: %w", node.Name, err) + } + } + for _, node := range genesisNodes { + client, err := node.Client() + if err != nil { + return fmt.Errorf("failed to initialized node %s: %w", node.Name, err) + } + for i := 0; i < 10; i++ { + resp, err := client.Status(context.Background()) + if err != nil { + return fmt.Errorf("node %s status response: %w", node.Name, err) + } + if resp.SyncInfo.LatestBlockHeight > 0 { + break + } + if i == 9 { + return fmt.Errorf("failed to start node %s", node.Name) + } + time.Sleep(time.Second) + } + } + return nil +} + +func (t *Testnet) Cleanup() { + for _, node := range t.nodes { + err := node.Instance.Destroy() + if err != nil { + log.Err(err).Msg(fmt.Sprintf("node %s failed to cleanup", node.Name)) + } + } +} + +func (t *Testnet) Node(i int) *Node { + return t.nodes[i] +} diff --git a/test/e2e/util.go b/test/e2e/util.go new file mode 100644 index 0000000000..b291bba559 --- /dev/null +++ b/test/e2e/util.go @@ -0,0 +1,37 @@ +package e2e + +import ( + "io" + "math/rand" + + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/ed25519" + "github.com/tendermint/tendermint/crypto/secp256k1" +) + +type keyGenerator struct { + random *rand.Rand +} + +func newKeyGenerator(seed int64) *keyGenerator { + return &keyGenerator{ + random: rand.New(rand.NewSource(seed)), //nolint:gosec + } +} + +func (g *keyGenerator) Generate(keyType string) crypto.PrivKey { + seed := make([]byte, ed25519.SeedSize) + + _, err := io.ReadFull(g.random, seed) + if err != nil { + panic(err) // this shouldn't happen + } + switch keyType { + case "secp256k1": + return secp256k1.GenPrivKeySecp256k1(seed) + case "", "ed25519": + return ed25519.GenPrivKeyFromSecret(seed) + default: + panic("KeyType not supported") // should not make it this far + } +} diff --git a/test/txsim/blob.go b/test/txsim/blob.go index db8d081024..33f2a03db2 100644 --- a/test/txsim/blob.go +++ b/test/txsim/blob.go @@ -5,7 +5,6 @@ import ( "fmt" "math/rand" - "github.com/celestiaorg/celestia-app/pkg/appconsts" ns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/test/util/blobfactory" blob "github.com/celestiaorg/celestia-app/x/blob/types" @@ -86,7 +85,7 @@ func (s *BlobSequence) Next(_ context.Context, _ grpc.ClientConn, rand *rand.Ran return Operation{ Msgs: []types.Msg{msg}, Blobs: blobs, - GasLimit: EstimateGas(sizes), + GasLimit: estimateGas(sizes), }, nil } @@ -107,18 +106,12 @@ func (r Range) Rand(rand *rand.Rand) int { return rand.Intn(r.Max-r.Min) + r.Min } -const ( - perByteGasTolerance = 2 - pfbGasFixedCost = 80000 -) - -// EstimateGas estimates the gas required to pay for a set of blobs in a PFB. -func EstimateGas(blobSizes []int) uint64 { - totalByteCount := 0 - for _, size := range blobSizes { - totalByteCount += size +// estimateGas estimates the gas required to pay for a set of blobs in a PFB. +func estimateGas(blobSizes []int) uint64 { + size := make([]uint32, len(blobSizes)) + for i, s := range blobSizes { + size[i] = uint32(s) } - variableGasAmount := (appconsts.DefaultGasPerBlobByte + perByteGasTolerance) * totalByteCount - return uint64(variableGasAmount + pfbGasFixedCost) + return blob.DefaultEstimateGas(size) } diff --git a/test/txsim/sequence.go b/test/txsim/sequence.go index 126be3c985..b3206fe73b 100644 --- a/test/txsim/sequence.go +++ b/test/txsim/sequence.go @@ -42,9 +42,9 @@ type Operation struct { } const ( - // set default gas limit to cover the costs of most transactions - // At 0.001 utia per gas, this equates to 1000utia per transaction - DefaultGasLimit = 1000000 + // Set the default gas limit to cover the costs of most transactions. + // At 0.1 utia per gas, this equates to 20_000utia per transaction. + DefaultGasLimit = 200_000 ) // ErrEndOfSequence is a special error which indicates that the sequence has been terminated diff --git a/test/util/testnode/config.go b/test/util/testnode/config.go index 7c83c3c317..b8ac3408a5 100644 --- a/test/util/testnode/config.go +++ b/test/util/testnode/config.go @@ -19,6 +19,8 @@ import ( type Config struct { // ChainID is the chain ID of the network. ChainID string + // GenesisTime is the genesis block time of the network. + GenesisTime time.Time // TmConfig is the Tendermint configuration used for the network. TmConfig *tmconfig.Config // AppConfig is the application configuration of the test node. @@ -91,11 +93,18 @@ func (c *Config) WithSupressLogs(sl bool) *Config { return c } +// WithGensisTime sets the GenesisTime and returns the Config. +func (c *Config) WithGensisTime(t time.Time) *Config { + c.GenesisTime = t + return c +} + func DefaultConfig() *Config { tmcfg := DefaultTendermintConfig() tmcfg.Consensus.TimeoutCommit = 1 * time.Millisecond cfg := &Config{} return cfg. + WithGensisTime(time.Now()). WithAccounts([]string{}). WithChainID(tmrand.Str(6)). WithTendermintConfig(DefaultTendermintConfig()). diff --git a/test/util/testnode/full_node.go b/test/util/testnode/full_node.go index 38c98631dc..3175516067 100644 --- a/test/util/testnode/full_node.go +++ b/test/util/testnode/full_node.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/crypto/keyring" @@ -81,6 +82,7 @@ func InitFiles( genState map[string]json.RawMessage, kr keyring.Keyring, chainID string, + genesisTime time.Time, ) (string, keyring.Keyring, error) { baseDir, err := initFileStructure(t, tmCfg) if err != nil { @@ -99,12 +101,12 @@ func InitFiles( return baseDir, kr, err } - err = initGenFiles(cparams, genState, encCfg.Codec, tmCfg.GenesisFile(), chainID) + err = initGenFiles(cparams, genState, encCfg.Codec, tmCfg.GenesisFile(), chainID, genesisTime) if err != nil { return baseDir, kr, err } - return baseDir, kr, collectGenFiles(tmCfg, encCfg, pubKey, nodeID, chainID, baseDir) + return baseDir, kr, collectGenFiles(tmCfg, encCfg, pubKey, nodeID, baseDir) } // DefaultGenesisState returns a default genesis state and a keyring with @@ -168,8 +170,9 @@ func NewNetwork(t testing.TB, cfg *Config) (cctx Context, rpcAddr, grpcAddr stri } chainID := cfg.ChainID + genTime := cfg.GenesisTime - baseDir, kr, err := InitFiles(t, cfg.ConsensusParams, tmCfg, genState, kr, chainID) + baseDir, kr, err := InitFiles(t, cfg.ConsensusParams, tmCfg, genState, kr, chainID, genTime) require.NoError(t, err) tmNode, app, err := NewCometNode(t, baseDir, cfg) diff --git a/test/util/testnode/node_init.go b/test/util/testnode/node_init.go index 594ae70d52..6cca3710d6 100644 --- a/test/util/testnode/node_init.go +++ b/test/util/testnode/node_init.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" @@ -26,26 +27,23 @@ import ( tmos "github.com/tendermint/tendermint/libs/os" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" "github.com/tendermint/tendermint/types" - tmtime "github.com/tendermint/tendermint/types/time" ) // NodeEVMPrivateKey the key used to initialize the test node validator. // Its corresponding address is: "0x9c2B12b5a07FC6D719Ed7646e5041A7E85758329". var NodeEVMPrivateKey, _ = crypto.HexToECDSA("64a1d6f0e760a8d62b4afdde4096f16f51b401eaaecc915740f71770ea76a8ad") -func collectGenFiles(tmCfg *config.Config, encCfg encoding.Config, pubKey cryptotypes.PubKey, nodeID, chainID, rootDir string) error { - genTime := tmtime.Now() - +func collectGenFiles(tmCfg *config.Config, encCfg encoding.Config, pubKey cryptotypes.PubKey, nodeID, rootDir string) error { gentxsDir := filepath.Join(rootDir, "gentxs") - initCfg := genutiltypes.NewInitConfig(chainID, gentxsDir, nodeID, pubKey) - genFile := tmCfg.GenesisFile() genDoc, err := types.GenesisDocFromFile(genFile) if err != nil { return err } + initCfg := genutiltypes.NewInitConfig(genDoc.ChainID, gentxsDir, nodeID, pubKey) + appState, err := genutil.GenAppStateFromConfig( encCfg.Codec, encCfg.TxConfig, @@ -59,8 +57,8 @@ func collectGenFiles(tmCfg *config.Config, encCfg encoding.Config, pubKey crypto } genDoc = &types.GenesisDoc{ - GenesisTime: genTime, - ChainID: chainID, + GenesisTime: genDoc.GenesisTime, + ChainID: genDoc.ChainID, Validators: nil, AppState: appState, ConsensusParams: genDoc.ConsensusParams, @@ -79,6 +77,7 @@ func initGenFiles( _ codec.Codec, file, chainID string, + genTime time.Time, ) error { appGenStateJSON, err := json.MarshalIndent(state, "", " ") if err != nil { @@ -86,6 +85,7 @@ func initGenFiles( } genDoc := types.GenesisDoc{ + GenesisTime: genTime, ChainID: chainID, AppState: appGenStateJSON, ConsensusParams: cparams, @@ -95,6 +95,8 @@ func initGenFiles( return genDoc.SaveAs(file) } +// createValidator creates a genesis transaction for adding a validator account. +// The transaction is stored in the `test.json` file under the 'baseDir/gentxs`. func createValidator( kr keyring.Keyring, encCfg encoding.Config, diff --git a/tools/blocktime/readme.md b/tools/blocktime/readme.md index e8d9ce4873..0ea48e9220 100644 --- a/tools/blocktime/readme.md +++ b/tools/blocktime/readme.md @@ -8,13 +8,13 @@ To read up on starting a node and exposing the RPC endpoint go to the docs [here To compile the binary, run either `go install` or `go build`. The binary can then be used as follows: -``` -$ ./blocktime [query_range] +```bash +./blocktime [query_range] ``` As an example -``` +```bash $ ./blocktime http://localhost:26657 1000 Chain: mocha-3 diff --git a/x/blob/ante/ante.go b/x/blob/ante/ante.go index 1893db35e2..c5eb82de1b 100644 --- a/x/blob/ante/ante.go +++ b/x/blob/ante/ante.go @@ -1,8 +1,6 @@ package ante import ( - "github.com/celestiaorg/celestia-app/pkg/appconsts" - "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-app/x/blob/types" "cosmossdk.io/errors" @@ -38,7 +36,7 @@ func (d MinGasPFBDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool // lazily fetch the gas per byte param gasPerByte = d.k.GasPerBlobByte(ctx) } - gasToConsume := gasToConsume(pfb, gasPerByte) + gasToConsume := pfb.Gas(gasPerByte) if gasToConsume > txGas { return ctx, errors.Wrapf(sdkerrors.ErrInsufficientFee, "not enough gas to pay for blobs (minimum: %d, got: %d)", gasToConsume, txGas) } @@ -48,15 +46,6 @@ func (d MinGasPFBDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool return next(ctx, tx, simulate) } -func gasToConsume(pfb *types.MsgPayForBlobs, gasPerByte uint32) uint64 { - var totalSharesUsed uint64 - for _, size := range pfb.BlobSizes { - totalSharesUsed += uint64(shares.SparseSharesNeeded(size)) - } - - return totalSharesUsed * appconsts.ShareSize * uint64(gasPerByte) -} - type BlobKeeper interface { GasPerBlobByte(ctx sdk.Context) uint32 } diff --git a/x/blob/keeper/keeper.go b/x/blob/keeper/keeper.go index af40268318..34ad7f5e01 100644 --- a/x/blob/keeper/keeper.go +++ b/x/blob/keeper/keeper.go @@ -4,8 +4,6 @@ import ( "context" "fmt" - "github.com/celestiaorg/celestia-app/pkg/appconsts" - "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/cosmos/cosmos-sdk/codec" storetypes "github.com/cosmos/cosmos-sdk/store/types" @@ -52,12 +50,7 @@ func (k Keeper) Logger(ctx sdk.Context) log.Logger { func (k Keeper) PayForBlobs(goCtx context.Context, msg *types.MsgPayForBlobs) (*types.MsgPayForBlobsResponse, error) { ctx := sdk.UnwrapSDKContext(goCtx) - var totalSharesUsed uint64 - for _, size := range msg.BlobSizes { - totalSharesUsed += uint64(shares.SparseSharesNeeded(size)) - } - - gasToConsume := totalSharesUsed * appconsts.ShareSize * uint64(k.GasPerBlobByte(ctx)) + gasToConsume := types.GasToConsume(msg.BlobSizes, k.GasPerBlobByte(ctx)) ctx.GasMeter().ConsumeGas(gasToConsume, payForBlobGasDescriptor) err := ctx.EventManager().EmitTypedEvent( diff --git a/x/blob/types/payforblob.go b/x/blob/types/payforblob.go index 5f72856393..9d541d6db8 100644 --- a/x/blob/types/payforblob.go +++ b/x/blob/types/payforblob.go @@ -12,6 +12,7 @@ import ( "github.com/celestiaorg/nmt" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/auth/migrations/legacytx" + auth "github.com/cosmos/cosmos-sdk/x/auth/types" "github.com/tendermint/tendermint/crypto/merkle" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" coretypes "github.com/tendermint/tendermint/types" @@ -21,6 +22,26 @@ import ( const ( URLMsgPayForBlobs = "/celestia.blob.v1.MsgPayForBlobs" ShareSize = appconsts.ShareSize + + // PFBGasFixedCost is a rough estimate for the "fixed cost" in the gas cost + // formula: gas cost = gas per byte * bytes per share * shares occupied by + // blob + "fixed cost". In this context, "fixed cost" accounts for the gas + // consumed by operations outside the blob's GasToConsume function (i.e. + // signature verification, tx size, read access to accounts). + // + // Since the gas cost of these operations is not easy to calculate, linear + // regression was performed on a set of observed data points to derive an + // approximate formula for gas cost. Assuming gas per byte = 8 and bytes per + // share = 512, we can solve for "fixed cost" and arrive at 65,000. gas cost + // = 8 * 512 * number of shares occupied by the blob + 65,000 has a + // correlation coefficient of 0.996. To be conservative, we round up "fixed + // cost" to 75,000 because the first tx always takes up 10,000 more gas than + // subsequent txs. + PFBGasFixedCost = 75000 + + // BytesPerBlobInfo is a rough estimation for the amount of extra bytes in + // information a blob adds to the size of the underlying transaction. + BytesPerBlobInfo = 70 ) // MsgPayForBlobs implements the `LegacyMsg` interface. @@ -130,6 +151,36 @@ func (msg *MsgPayForBlobs) ValidateBasic() error { return nil } +func (msg *MsgPayForBlobs) Gas(gasPerByte uint32) uint64 { + return GasToConsume(msg.BlobSizes, gasPerByte) +} + +// GasToConsume works out the extra gas charged to pay for a set of blobs in a PFB. +// Note that tranasctions will incur other gas costs, such as the signature verification +// and reads to the user's account. +func GasToConsume(blobSizes []uint32, gasPerByte uint32) uint64 { + var totalSharesUsed uint64 + for _, size := range blobSizes { + totalSharesUsed += uint64(appshares.SparseSharesNeeded(size)) + } + + return totalSharesUsed * appconsts.ShareSize * uint64(gasPerByte) +} + +// EstimateGas estimates the total gas required to pay for a set of blobs in a PFB. +// It is based on a linear model that is dependent on the governance parameters: +// gasPerByte and txSizeCost. It assumes other variables are constant. This includes +// assuming the PFB is the only message in the transaction. +func EstimateGas(blobSizes []uint32, gasPerByte uint32, txSizeCost uint64) uint64 { + return GasToConsume(blobSizes, gasPerByte) + (txSizeCost * BytesPerBlobInfo * uint64(len(blobSizes))) + PFBGasFixedCost +} + +// DefaultEstimateGas runs EstimateGas with the system defaults. The network may change these values +// through governance, thus this function should predominantly be used in testing. +func DefaultEstimateGas(blobSizes []uint32) uint64 { + return EstimateGas(blobSizes, appconsts.DefaultGasPerBlobByte, auth.DefaultTxSizeCostPerByte) +} + // ValidateBlobNamespace returns an error if the provided namespace is reserved, // parity shares, or tail padding. func ValidateBlobNamespace(ns appns.Namespace) error { diff --git a/x/mint/types/constants.go b/x/mint/types/constants.go index c8bb270575..4082a9f442 100644 --- a/x/mint/types/constants.go +++ b/x/mint/types/constants.go @@ -11,8 +11,8 @@ const ( // value isn't 365 because 97 out of 400 years are leap years. See // https://en.wikipedia.org/wiki/Year DaysPerYear = 365.2425 - SecondsPerYear = int(SecondsPerMinute * MinutesPerHour * HoursPerDay * DaysPerYear) // 31,556,952 - NanosecondsPerYear = int64(NanosecondsPerSecond * SecondsPerYear) // 31,556,952,000,000,000 + SecondsPerYear = int64(SecondsPerMinute * MinutesPerHour * HoursPerDay * DaysPerYear) // 31,556,952 + NanosecondsPerYear = int64(NanosecondsPerSecond * SecondsPerYear) // 31,556,952,000,000,000 InitialInflationRate = 0.08 DisinflationRate = 0.1