From 25b2df79c6d9ab47220be2b32f3daa5df9cc237e Mon Sep 17 00:00:00 2001 From: Rootul P Date: Mon, 14 Aug 2023 10:29:36 -0400 Subject: [PATCH] feat!: reserve the last 256 namespaces for protocol use (#2257) Closes https://github.com/celestiaorg/celestia-app/issues/2253 (cherry picked from commit cb8247ead807a52bb8915d450982544755efc1e6) # Conflicts: # pkg/namespace/consts.go # pkg/namespace/namespace.go # pkg/namespace/namespace_test.go # pkg/shares/padding.go # specs/src/specs/namespace.md --- pkg/namespace/consts.go | 52 +++++++++++++---- pkg/namespace/namespace.go | 16 +++++- pkg/namespace/namespace_test.go | 95 +++++++++++++++++++++++++++++++ pkg/shares/padding.go | 8 +++ pkg/shares/padding_test.go | 2 +- pkg/shares/parse_test.go | 4 +- pkg/shares/share_sequence_test.go | 2 +- pkg/shares/shares.go | 8 +-- specs/src/specs/namespace.md | 28 +++++++++ specs/src/specs/shares.md | 10 ++-- 10 files changed, 198 insertions(+), 27 deletions(-) diff --git a/pkg/namespace/consts.go b/pkg/namespace/consts.go index 2a71299592..d0ae48aec7 100644 --- a/pkg/namespace/consts.go +++ b/pkg/namespace/consts.go @@ -36,35 +36,63 @@ var ( // NamespaceVersionZeroPrefix is the prefix of a namespace ID for version 0. NamespaceVersionZeroPrefix = bytes.Repeat([]byte{0}, NamespaceVersionZeroPrefixSize) - // TxNamespace is the namespace reserved for transaction data. - TxNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 1}) + // TxNamespace is the namespace reserved for ordinary Cosmos SDK transactions. + TxNamespace = primaryReservedNamespace(0x01) // IntermediateStateRootsNamespace is the namespace reserved for // intermediate state root data. - IntermediateStateRootsNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 2}) + IntermediateStateRootsNamespace = primaryReservedNamespace(0x02) // PayForBlobNamespace is the namespace reserved for PayForBlobs transactions. - PayForBlobNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 4}) + PayForBlobNamespace = primaryReservedNamespace(0x04) - // ReservedPaddingNamespace is the namespace used for padding after all - // reserved namespaces. In practice this padding is after transactions - // (ordinary and PFBs) but before blobs. - ReservedPaddingNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) + // PrimaryReservedPaddingNamespace is the namespace used for padding after all + // primary reserved namespaces. + PrimaryReservedPaddingNamespace = primaryReservedNamespace(0xFF) +<<<<<<< HEAD // MaxReservedNamespace is lexicographically the largest namespace that is // reserved for protocol use. MaxReservedNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255}) +======= + // MaxPrimaryReservedNamespace is the highest primary reserved namespace. + // Namespaces lower than this are reserved for protocol use. + MaxPrimaryReservedNamespace = primaryReservedNamespace(0xFF) + + // MinSecondaryReservedNamespace is the lowest secondary reserved namespace + // reserved for protocol use. Namespaces higher than this are reserved for + // protocol use. + MinSecondaryReservedNamespace = secondaryReservedNamespace(0x00) +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) // TailPaddingNamespace is the namespace reserved for tail padding. All data // with this namespace will be ignored. - TailPaddingNamespace = Namespace{ - Version: math.MaxUint8, - ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), 0xFE), - } + TailPaddingNamespace = secondaryReservedNamespace(0xFE) // ParitySharesNamespace is the namespace reserved for erasure coded data. +<<<<<<< HEAD ParitySharesNamespace = Namespace{ Version: math.MaxUint8, ID: bytes.Repeat([]byte{0xFF}, NamespaceIDSize), } +======= + ParitySharesNamespace = secondaryReservedNamespace(0xFF) + + // SupportedBlobNamespaceVersions is a list of namespace versions that can be specified by a user for blobs. + SupportedBlobNamespaceVersions = []uint8{NamespaceVersionZero} +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) ) + +func primaryReservedNamespace(lastByte byte) Namespace { + return Namespace{ + Version: NamespaceVersionZero, + ID: append(bytes.Repeat([]byte{0x00}, NamespaceIDSize-1), lastByte), + } +} + +func secondaryReservedNamespace(lastByte byte) Namespace { + return Namespace{ + Version: NamespaceVersionMax, + ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), lastByte), + } +} diff --git a/pkg/namespace/namespace.go b/pkg/namespace/namespace.go index 5da38bc398..4eee45b1b3 100644 --- a/pkg/namespace/namespace.go +++ b/pkg/namespace/namespace.go @@ -122,7 +122,19 @@ func validateID(version uint8, id []byte) error { } func (n Namespace) IsReserved() bool { +<<<<<<< HEAD return bytes.Compare(n.Bytes(), MaxReservedNamespace.Bytes()) < 1 +======= + return n.IsPrimaryReserved() || n.IsSecondaryReserved() +} + +func (n Namespace) IsPrimaryReserved() bool { + return n.IsLessOrEqualThan(MaxPrimaryReservedNamespace) +} + +func (n Namespace) IsSecondaryReserved() bool { + return n.IsGreaterOrEqualThan(MinSecondaryReservedNamespace) +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) } func (n Namespace) IsParityShares() bool { @@ -133,8 +145,8 @@ func (n Namespace) IsTailPadding() bool { return bytes.Equal(n.Bytes(), TailPaddingNamespace.Bytes()) } -func (n Namespace) IsReservedPadding() bool { - return bytes.Equal(n.Bytes(), ReservedPaddingNamespace.Bytes()) +func (n Namespace) IsPrimaryReservedPadding() bool { + return bytes.Equal(n.Bytes(), PrimaryReservedPaddingNamespace.Bytes()) } func (n Namespace) IsTx() bool { diff --git a/pkg/namespace/namespace_test.go b/pkg/namespace/namespace_test.go index f7b97487dc..ff36a37e07 100644 --- a/pkg/namespace/namespace_test.go +++ b/pkg/namespace/namespace_test.go @@ -2,6 +2,11 @@ package namespace import ( "bytes" +<<<<<<< HEAD +======= + "math" + "reflect" +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) "testing" "github.com/stretchr/testify/assert" @@ -195,3 +200,93 @@ func TestBytes(t *testing.T) { assert.Equal(t, want, got) } +<<<<<<< HEAD +======= + +func TestLeftPad(t *testing.T) { + tests := []struct { + input []byte + size int + expected []byte + }{ + // input smaller than pad size + {[]byte{1, 2, 3}, 10, []byte{0, 0, 0, 0, 0, 0, 0, 1, 2, 3}}, + {[]byte{1}, 5, []byte{0, 0, 0, 0, 1}}, + {[]byte{1, 2}, 4, []byte{0, 0, 1, 2}}, + + // input equal to pad size + {[]byte{1, 2, 3}, 3, []byte{1, 2, 3}}, + {[]byte{1, 2, 3, 4}, 4, []byte{1, 2, 3, 4}}, + + // input larger than pad size + {[]byte{1, 2, 3, 4, 5}, 4, []byte{1, 2, 3, 4, 5}}, + {[]byte{1, 2, 3, 4, 5, 6, 7}, 3, []byte{1, 2, 3, 4, 5, 6, 7}}, + + // input size 0 + {[]byte{}, 8, []byte{0, 0, 0, 0, 0, 0, 0, 0}}, + {[]byte{}, 0, []byte{}}, + } + + for _, test := range tests { + result := leftPad(test.input, test.size) + assert.True(t, reflect.DeepEqual(result, test.expected)) + } +} + +func TestIsReserved(t *testing.T) { + type testCase struct { + ns Namespace + want bool + } + testCases := []testCase{ + { + ns: MustNewV0(bytes.Repeat([]byte{1}, NamespaceVersionZeroIDSize)), + want: false, + }, + { + ns: TxNamespace, + want: true, + }, + { + ns: IntermediateStateRootsNamespace, + want: true, + }, + { + ns: PayForBlobNamespace, + want: true, + }, + { + ns: PrimaryReservedPaddingNamespace, + want: true, + }, + { + ns: MaxPrimaryReservedNamespace, + want: true, + }, + { + ns: MinSecondaryReservedNamespace, + want: true, + }, + { + ns: TailPaddingNamespace, + want: true, + }, + { + ns: ParitySharesNamespace, + want: true, + }, + { + ns: Namespace{ + Version: math.MaxUint8, + ID: append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), 1), + }, + want: true, + }, + } + + for _, tc := range testCases { + got := tc.ns.IsReserved() + assert.Equal(t, tc.want, got) + } +} +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) diff --git a/pkg/shares/padding.go b/pkg/shares/padding.go index d2ee22bb73..25fc91bedf 100644 --- a/pkg/shares/padding.go +++ b/pkg/shares/padding.go @@ -52,7 +52,11 @@ func NamespacePaddingShares(ns appns.Namespace, n int) ([]Share, error) { // first blob can start at an index that conforms to non-interactive default // rules. func ReservedPaddingShare() Share { +<<<<<<< HEAD share, err := NamespacePaddingShare(appns.ReservedPaddingNamespace) +======= + share, err := NamespacePaddingShare(appns.PrimaryReservedPaddingNamespace, appconsts.ShareVersionZero) +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) if err != nil { panic(err) } @@ -61,7 +65,11 @@ func ReservedPaddingShare() Share { // ReservedPaddingShare returns n reserved padding shares. func ReservedPaddingShares(n int) []Share { +<<<<<<< HEAD shares, err := NamespacePaddingShares(appns.ReservedPaddingNamespace, n) +======= + shares, err := NamespacePaddingShares(appns.PrimaryReservedPaddingNamespace, appconsts.ShareVersionZero, n) +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) if err != nil { panic(err) } diff --git a/pkg/shares/padding_test.go b/pkg/shares/padding_test.go index abf04e8515..1830ddcf0a 100644 --- a/pkg/shares/padding_test.go +++ b/pkg/shares/padding_test.go @@ -23,7 +23,7 @@ var nsOnePadding, _ = zeroPadIfNecessary( var reservedPadding, _ = zeroPadIfNecessary( append( - appns.ReservedPaddingNamespace.Bytes(), + appns.PrimaryReservedPaddingNamespace.Bytes(), []byte{ 1, // info byte 0, 0, 0, 0, // sequence len diff --git a/pkg/shares/parse_test.go b/pkg/shares/parse_test.go index 402b8714b7..3aba052739 100644 --- a/pkg/shares/parse_test.go +++ b/pkg/shares/parse_test.go @@ -162,11 +162,11 @@ func TestParseShares(t *testing.T) { ignorePadding: false, want: []ShareSequence{ { - Namespace: appns.ReservedPaddingNamespace, + Namespace: appns.PrimaryReservedPaddingNamespace, Shares: []Share{ReservedPaddingShare()}, }, { - Namespace: appns.ReservedPaddingNamespace, + Namespace: appns.PrimaryReservedPaddingNamespace, Shares: []Share{ReservedPaddingShare()}, }, }, diff --git a/pkg/shares/share_sequence_test.go b/pkg/shares/share_sequence_test.go index 31d790cdb4..9ad10376bb 100644 --- a/pkg/shares/share_sequence_test.go +++ b/pkg/shares/share_sequence_test.go @@ -159,7 +159,7 @@ func Test_validSequenceLen(t *testing.T) { } reservedPadding := ShareSequence{ - Namespace: appns.ReservedPaddingNamespace, + Namespace: appns.PrimaryReservedPaddingNamespace, Shares: []Share{ReservedPaddingShare()}, } diff --git a/pkg/shares/shares.go b/pkg/shares/shares.go index 1f805d4e17..93711cc639 100644 --- a/pkg/shares/shares.go +++ b/pkg/shares/shares.go @@ -121,11 +121,11 @@ func (s *Share) IsPadding() (bool, error) { if err != nil { return false, err } - isReservedPadding, err := s.isReservedPadding() + isPrimaryReservedPadding, err := s.isPrimaryReservedPadding() if err != nil { return false, err } - return isNamespacePadding || isTailPadding || isReservedPadding, nil + return isNamespacePadding || isTailPadding || isPrimaryReservedPadding, nil } func (s *Share) isNamespacePadding() (bool, error) { @@ -149,12 +149,12 @@ func (s *Share) isTailPadding() (bool, error) { return ns.IsTailPadding(), nil } -func (s *Share) isReservedPadding() (bool, error) { +func (s *Share) isPrimaryReservedPadding() (bool, error) { ns, err := s.Namespace() if err != nil { return false, err } - return ns.IsReservedPadding(), nil + return ns.IsPrimaryReservedPadding(), nil } func (s *Share) ToBytes() []byte { diff --git a/specs/src/specs/namespace.md b/specs/src/specs/namespace.md index b259516cdb..3fd9436b9a 100644 --- a/specs/src/specs/namespace.md +++ b/specs/src/specs/namespace.md @@ -42,6 +42,7 @@ The namespace ID is a 28 byte identifier that uniquely identifies a namespace. T ## Reserved Namespaces +<<<<<<< HEAD | name | type | value | description | |-------------------------------------|-------------|----------------------------------------------------------------|------------------------------------------------------------------------------------------------------| | `TRANSACTION_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000001` | Transactions: requests that modify the state. | @@ -51,6 +52,33 @@ The namespace ID is a 28 byte identifier that uniquely identifies a namespace. T | `MAX_RESERVED_NAMESPACE` | `Namespace` | `0x00000000000000000000000000000000000000000000000000000000FF` | Max reserved namespace is lexicographically the largest namespace that is reserved for protocol use. | | `TAIL_PADDING_NAMESPACE` | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Tail padding for blobs: padding after all blobs to fill up the original data square. | | `PARITY_SHARE_NAMESPACE` | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Parity shares: extended shares in the available data matrix. | +======= +Celestia reserves some namespaces for protocol use. +These namespaces are called "reserved namespaces". +Reserved namespaces are used to arrange the contents of the [data square](./data_square_layout.md). +Applications MUST NOT use reserved namespaces for their blob data. +Reserved namespaces fall into two categories: _Primary_ and _Secondary_. + +- Primary: Namespaces with values less than or equal to `0x00000000000000000000000000000000000000000000000000000000FF`. Primary namespaces always have a version of `0`. +- Secondary: Namespaces with values greater than or equal to `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00`. +Secondary namespaces always have a version of `255` (`0xFF`) so that they are placed after all user specifiable namespaces in a sorted data square. +The `PARITY_SHARE_NAMESPACE` uses version `255` (`0xFF`) to enable more efficient proof generation within the context of [nmt](https://github.com/celestiaorg/nmt), where it is used in conjunction with the `IgnoreMaxNamespace` feature. +The `TAIL_PADDING_NAMESPACE` uses the version `255` to ensure that padding shares are always placed at the end of the Celestia data square even if a new user-specifiable version is introduced. + +Below is a list of the current reserved namespaces. +For additional information on the significance and application of the reserved namespaces, please refer to the [Data Square Layout](./data_square_layout.md) specifications. + +| name | type | category | value | description | +|--------------------------------------|-------------|-----------|----------------------------------------------------------------|----------------------------------------------------------------------------| +| `TRANSACTION_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000001` | Namespace for ordinary Cosmos SDK transactions. | +| `INTERMEDIATE_STATE_ROOT_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000002` | Namespace for intermediate state roots (not currently utilized). | +| `PAY_FOR_BLOB_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000004` | Namespace for transactions that contain a PayForBlob. | +| `PRIMARY_RESERVED_PADDING_NAMESPACE` | `Namespace` | Primary | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for padding after all primary reserved namespaces. | +| `MAX_PRIMARY_RESERVED_NAMESPACE` | `Namespace` | Primary | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for the highest primary reserved namespace. | +| `MIN_SECONDARY_RESERVED_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00` | Namespace for the lowest secondary reserved namespace. | +| `TAIL_PADDING_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Namespace for padding after all blobs to fill up the original data square. | +| `PARITY_SHARE_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Namespace for parity shares. | +>>>>>>> cb8247e (feat!: reserve the last 256 namespaces for protocol use (#2257)) ## Assumptions and Considerations diff --git a/specs/src/specs/shares.md b/specs/src/specs/shares.md index 1c9fcfb2e4..cfa57e2f25 100644 --- a/specs/src/specs/shares.md +++ b/specs/src/specs/shares.md @@ -18,7 +18,7 @@ User submitted transactions are split into shares (see [share splitting](#share- [Padding](#padding) shares are added to the `k * k` matrix to ensure: -1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#reserved-padding-share)) +1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#primary-reserved-padding-share)) 1. The number of shares in the matrix is a perfect square (see [tail padding share](#tail-padding-share)) ## Share Format @@ -85,17 +85,17 @@ Padding shares vary based on namespace but they conform to the [share format](#s A namespace padding share uses the namespace of the blob that precedes it in the data square so that the data square can retain the property that all shares are ordered by namespace. A namespace padding share acts as padding between blobs so that the subsequent blob begins at an index that conforms to the [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules). Clients MAY ignore the contents of these shares because they don't contain any significant data. -### Reserved Padding Share +### Primary Reserved Padding Share -Reserved padding shares use the [`RESERVED_PADDING_NAMESPACE`](./consensus.md#constants). Reserved padding shares are placed after the last reserved namespace share in the data square so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data. +Primary reserved padding shares use the [`PRIMARY_RESERVED_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Primary reserved padding shares are placed after shares in the primary reserved namespace range so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data. ### Tail Padding Share -Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./consensus.md#constants). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data. +Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data. ## Parity Share -Parity shares use the [`PARITY_SHARE_NAMESPACE`](./consensus.md#constants). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning. +Parity shares use the [`PARITY_SHARE_NAMESPACE`](./namespace.md#reserved-namespaces). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning. ## Share Splitting