feat!: reserve the last 256 namespaces for protocol use (#2257)

Closes #2253

pkg/namespace/consts.go

@@ -36,39 +36,50 @@ var (
 	// NamespaceVersionZeroPrefix is the prefix of a namespace ID for version 0.
 	NamespaceVersionZeroPrefix = bytes.Repeat([]byte{0}, NamespaceVersionZeroPrefixSize)
 
-	// TxNamespace is the namespace reserved for transaction data.
-	TxNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 1})
+	// TxNamespace is the namespace reserved for ordinary Cosmos SDK transactions.
+	TxNamespace = primaryReservedNamespace(0x01)
 
 	// IntermediateStateRootsNamespace is the namespace reserved for
 	// intermediate state root data.
-	IntermediateStateRootsNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 2})
+	IntermediateStateRootsNamespace = primaryReservedNamespace(0x02)
 
 	// PayForBlobNamespace is the namespace reserved for PayForBlobs transactions.
-	PayForBlobNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 4})
+	PayForBlobNamespace = primaryReservedNamespace(0x04)
 
-	// ReservedPaddingNamespace is the namespace used for padding after all
-	// reserved namespaces. In practice this padding is after transactions
-	// (ordinary and PFBs) but before blobs.
-	ReservedPaddingNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255})
+	// PrimaryReservedPaddingNamespace is the namespace used for padding after all
+	// primary reserved namespaces.
+	PrimaryReservedPaddingNamespace = primaryReservedNamespace(0xFF)
 
-	// MaxPrimaryReservedNamespace represents the largest primary reserved
-	// namespace reserved for protocol use. Note that there may be other
-	// non-primary reserved namespaces beyond this upper limit.
-	MaxPrimaryReservedNamespace = MustNewV0([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 255})
+	// MaxPrimaryReservedNamespace is the highest primary reserved namespace.
+	// Namespaces lower than this are reserved for protocol use.
+	MaxPrimaryReservedNamespace = primaryReservedNamespace(0xFF)
+
+	// MinSecondaryReservedNamespace is the lowest secondary reserved namespace
+	// reserved for protocol use. Namespaces higher than this are reserved for
+	// protocol use.
+	MinSecondaryReservedNamespace = secondaryReservedNamespace(0x00)
 
 	// TailPaddingNamespace is the namespace reserved for tail padding. All data
 	// with this namespace will be ignored.
-	TailPaddingNamespace = Namespace{
-		Version: math.MaxUint8,
-		ID:      append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), 0xFE),
-	}
+	TailPaddingNamespace = secondaryReservedNamespace(0xFE)
 
 	// ParitySharesNamespace is the namespace reserved for erasure coded data.
-	ParitySharesNamespace = Namespace{
-		Version: math.MaxUint8,
-		ID:      bytes.Repeat([]byte{0xFF}, NamespaceIDSize),
-	}
+	ParitySharesNamespace = secondaryReservedNamespace(0xFF)
 
 	// SupportedBlobNamespaceVersions is a list of namespace versions that can be specified by a user for blobs.
 	SupportedBlobNamespaceVersions = []uint8{NamespaceVersionZero}
 )
+
+func primaryReservedNamespace(lastByte byte) Namespace {
+	return Namespace{
+		Version: NamespaceVersionZero,
+		ID:      append(bytes.Repeat([]byte{0x00}, NamespaceIDSize-1), lastByte),
+	}
+}
+
+func secondaryReservedNamespace(lastByte byte) Namespace {
+	return Namespace{
+		Version: NamespaceVersionMax,
+		ID:      append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), lastByte),
+	}
+}

pkg/namespace/namespace.go

@@ -106,10 +106,15 @@ func validateID(version uint8, id []byte) error {
 
 // IsReserved returns true if the namespace is reserved for protocol-use.
 func (n Namespace) IsReserved() bool {
-	isLessThanOrEqualToMaxPrimaryReservedNamespace := bytes.Compare(n.Bytes(), MaxPrimaryReservedNamespace.Bytes()) < 1
-	isParityNamespace := n.IsParityShares()
-	isTailPadding := n.IsTailPadding()
-	return isLessThanOrEqualToMaxPrimaryReservedNamespace || isParityNamespace || isTailPadding
+	return n.IsPrimaryReserved() || n.IsSecondaryReserved()
+}
+
+func (n Namespace) IsPrimaryReserved() bool {
+	return n.IsLessOrEqualThan(MaxPrimaryReservedNamespace)
+}
+
+func (n Namespace) IsSecondaryReserved() bool {
+	return n.IsGreaterOrEqualThan(MinSecondaryReservedNamespace)
 }
 
 func (n Namespace) IsParityShares() bool {
@@ -120,8 +125,8 @@ func (n Namespace) IsTailPadding() bool {
 	return bytes.Equal(n.Bytes(), TailPaddingNamespace.Bytes())
 }
 
-func (n Namespace) IsReservedPadding() bool {
-	return bytes.Equal(n.Bytes(), ReservedPaddingNamespace.Bytes())
+func (n Namespace) IsPrimaryReservedPadding() bool {
+	return bytes.Equal(n.Bytes(), PrimaryReservedPaddingNamespace.Bytes())
 }
 
 func (n Namespace) IsTx() bool {

pkg/namespace/namespace_test.go

@@ -2,6 +2,8 @@ package namespace
 
 import (
 	"bytes"
+	"math"
+	"reflect"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -195,3 +197,90 @@ func TestBytes(t *testing.T) {
 
 	assert.Equal(t, want, got)
 }
+
+func TestLeftPad(t *testing.T) {
+	tests := []struct {
+		input    []byte
+		size     int
+		expected []byte
+	}{
+		// input smaller than pad size
+		{[]byte{1, 2, 3}, 10, []byte{0, 0, 0, 0, 0, 0, 0, 1, 2, 3}},
+		{[]byte{1}, 5, []byte{0, 0, 0, 0, 1}},
+		{[]byte{1, 2}, 4, []byte{0, 0, 1, 2}},
+
+		// input equal to pad size
+		{[]byte{1, 2, 3}, 3, []byte{1, 2, 3}},
+		{[]byte{1, 2, 3, 4}, 4, []byte{1, 2, 3, 4}},
+
+		// input larger than pad size
+		{[]byte{1, 2, 3, 4, 5}, 4, []byte{1, 2, 3, 4, 5}},
+		{[]byte{1, 2, 3, 4, 5, 6, 7}, 3, []byte{1, 2, 3, 4, 5, 6, 7}},
+
+		// input size 0
+		{[]byte{}, 8, []byte{0, 0, 0, 0, 0, 0, 0, 0}},
+		{[]byte{}, 0, []byte{}},
+	}
+
+	for _, test := range tests {
+		result := leftPad(test.input, test.size)
+		assert.True(t, reflect.DeepEqual(result, test.expected))
+	}
+}
+
+func TestIsReserved(t *testing.T) {
+	type testCase struct {
+		ns   Namespace
+		want bool
+	}
+	testCases := []testCase{
+		{
+			ns:   MustNewV0(bytes.Repeat([]byte{1}, NamespaceVersionZeroIDSize)),
+			want: false,
+		},
+		{
+			ns:   TxNamespace,
+			want: true,
+		},
+		{
+			ns:   IntermediateStateRootsNamespace,
+			want: true,
+		},
+		{
+			ns:   PayForBlobNamespace,
+			want: true,
+		},
+		{
+			ns:   PrimaryReservedPaddingNamespace,
+			want: true,
+		},
+		{
+			ns:   MaxPrimaryReservedNamespace,
+			want: true,
+		},
+		{
+			ns:   MinSecondaryReservedNamespace,
+			want: true,
+		},
+		{
+			ns:   TailPaddingNamespace,
+			want: true,
+		},
+		{
+			ns:   ParitySharesNamespace,
+			want: true,
+		},
+		{
+			ns: Namespace{
+				Version: math.MaxUint8,
+				ID:      append(bytes.Repeat([]byte{0xFF}, NamespaceIDSize-1), 1),
+			},
+			want: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		got := tc.ns.IsReserved()
+		assert.Equal(t, tc.want, got)
+	}
+}

pkg/shares/padding.go

@@ -53,7 +53,7 @@ func NamespacePaddingShares(ns appns.Namespace, shareVersion uint8, n int) ([]Sh
 // first blob can start at an index that conforms to non-interactive default
 // rules.
 func ReservedPaddingShare() Share {
-	share, err := NamespacePaddingShare(appns.ReservedPaddingNamespace, appconsts.ShareVersionZero)
+	share, err := NamespacePaddingShare(appns.PrimaryReservedPaddingNamespace, appconsts.ShareVersionZero)
 	if err != nil {
 		panic(err)
 	}
@@ -62,7 +62,7 @@ func ReservedPaddingShare() Share {
 
 // ReservedPaddingShare returns n reserved padding shares.
 func ReservedPaddingShares(n int) []Share {
-	shares, err := NamespacePaddingShares(appns.ReservedPaddingNamespace, appconsts.ShareVersionZero, n)
+	shares, err := NamespacePaddingShares(appns.PrimaryReservedPaddingNamespace, appconsts.ShareVersionZero, n)
 	if err != nil {
 		panic(err)
 	}

pkg/shares/padding_test.go

@@ -23,7 +23,7 @@ var nsOnePadding, _ = zeroPadIfNecessary(
 
 var reservedPadding, _ = zeroPadIfNecessary(
 	append(
-		appns.ReservedPaddingNamespace.Bytes(),
+		appns.PrimaryReservedPaddingNamespace.Bytes(),
 		[]byte{
 			1,          // info byte
 			0, 0, 0, 0, // sequence len

pkg/shares/parse_test.go

@@ -162,11 +162,11 @@ func TestParseShares(t *testing.T) {
 			ignorePadding: false,
 			want: []ShareSequence{
 				{
-					Namespace: appns.ReservedPaddingNamespace,
+					Namespace: appns.PrimaryReservedPaddingNamespace,
 					Shares:    []Share{ReservedPaddingShare()},
 				},
 				{
-					Namespace: appns.ReservedPaddingNamespace,
+					Namespace: appns.PrimaryReservedPaddingNamespace,
 					Shares:    []Share{ReservedPaddingShare()},
 				},
 			},

pkg/shares/share_sequence_test.go

@@ -159,7 +159,7 @@ func Test_validSequenceLen(t *testing.T) {
 	}
 
 	reservedPadding := ShareSequence{
-		Namespace: appns.ReservedPaddingNamespace,
+		Namespace: appns.PrimaryReservedPaddingNamespace,
 		Shares:    []Share{ReservedPaddingShare()},
 	}
 

pkg/shares/shares.go

@@ -121,11 +121,11 @@ func (s *Share) IsPadding() (bool, error) {
 	if err != nil {
 		return false, err
 	}
-	isReservedPadding, err := s.isReservedPadding()
+	isPrimaryReservedPadding, err := s.isPrimaryReservedPadding()
 	if err != nil {
 		return false, err
 	}
-	return isNamespacePadding || isTailPadding || isReservedPadding, nil
+	return isNamespacePadding || isTailPadding || isPrimaryReservedPadding, nil
 }
 
 func (s *Share) isNamespacePadding() (bool, error) {
@@ -149,12 +149,12 @@ func (s *Share) isTailPadding() (bool, error) {
 	return ns.IsTailPadding(), nil
 }
 
-func (s *Share) isReservedPadding() (bool, error) {
+func (s *Share) isPrimaryReservedPadding() (bool, error) {
 	ns, err := s.Namespace()
 	if err != nil {
 		return false, err
 	}
-	return ns.IsReservedPadding(), nil
+	return ns.IsPrimaryReservedPadding(), nil
 }
 
 func (s *Share) ToBytes() []byte {

specs/src/specs/namespace.md

@@ -42,30 +42,31 @@ The namespace ID is a 28 byte identifier that uniquely identifies a namespace. T
 
 ## Reserved Namespaces
 
-Celestia reserves certain namespaces with specific meanings.
-Reserved namespaces are used to properly organize and order transactions and blobs inside the [data square](./data_square_layout.md).
-Applications MUST NOT use these reserved namespaces for their blob data.
-
-Reserved namespaces fall into two categories, _Primary Reserved Namespaces_ and _Secondary Reserved Namespaces_.
-
-- Primary Reserved Namespaces: Namespaces with values less than or equal to `0x00000000000000000000000000000000000000000000000000000000FF` are referred to as Primary Reserved Namespaces and are exclusively reserved for use within the Celestia protocols.
-- Secondary Reserved Namespaces: Currently, there are two secondary reserved namespaces, namely, `PARITY_SHARE_NAMESPACE` and `TAIL_PADDING_NAMESPACE` which fall out of the range of primary reserved namespaces.
-In the table, you will notice that the `PARITY_SHARE_NAMESPACE` and `TAIL_PADDING_NAMESPACE` utilize the namespace version `255`, which differs from the supported user-specified versions.
-The reason for employing version `255` for the `PARITY_SHARE_NAMESPACE` is to enable more efficient proof generation within the context of [nmt](https://github.com/celestiaorg/nmt), where it is used in conjunction with the `IgnoreMaxNamespace` feature.
-Similarly, the `TAIL_PADDING_NAMESPACE` utilizes the namespace version `255` to ensure that padding shares are always properly ordered and placed at the end of the Celestia data square even if a new namespace version is introduced.
-
-Below is a list of currently used primary and secondary reserved namespaces, along with a brief description of each.
+Celestia reserves some namespaces for protocol use.
+These namespaces are called "reserved namespaces".
+Reserved namespaces are used to arrange the contents of the [data square](./data_square_layout.md).
+Applications MUST NOT use reserved namespaces for their blob data.
+Reserved namespaces fall into two categories: _Primary_ and _Secondary_.
+
+- Primary: Namespaces with values less than or equal to `0x00000000000000000000000000000000000000000000000000000000FF`. Primary namespaces always have a version of `0`.
+- Secondary: Namespaces with values greater than or equal to `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00`.
+Secondary namespaces always have a version of `255` (`0xFF`) so that they are placed after all user specifiable namespaces in a sorted data square.
+The `PARITY_SHARE_NAMESPACE` uses version `255` (`0xFF`) to enable more efficient proof generation within the context of [nmt](https://github.com/celestiaorg/nmt), where it is used in conjunction with the `IgnoreMaxNamespace` feature.
+The `TAIL_PADDING_NAMESPACE` uses the version `255` to ensure that padding shares are always placed at the end of the Celestia data square even if a new user-specifiable version is introduced.
+
+Below is a list of the current reserved namespaces.
 For additional information on the significance and application of the reserved namespaces, please refer to the [Data Square Layout](./data_square_layout.md) specifications.
 
-| name                                | type        | value                                                          | description                                                                                           |
-|-------------------------------------|-------------|----------------------------------------------------------------|-------------------------------------------------------------------------------------------------------|
-| `TRANSACTION_NAMESPACE`             | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000001` | Transactions: requests that modify the state.                                                         |
-| `INTERMEDIATE_STATE_ROOT_NAMESPACE` | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000002` | Intermediate state roots, committed after every transaction.                                          |
-| `PAY_FOR_BLOB_NAMESPACE`            | `Namespace` | `0x0000000000000000000000000000000000000000000000000000000004` | Namespace reserved for transactions that contain a PayForBlob.                                        |
-| `RESERVED_PADDING_NAMESPACE`        | `Namespace` | `0x00000000000000000000000000000000000000000000000000000000FF` | Padding after all reserved namespaces but before blobs.                                               |
-| `MAX_PRIMARY_RESERVED_NAMESPACE`    | `Namespace` | `0x00000000000000000000000000000000000000000000000000000000FF` | Max primary reserved namespace is the largest primary reserved namespace designated for protocol use. |
-| `TAIL_PADDING_NAMESPACE`            | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Tail padding for blobs: padding after all blobs to fill up the original data square.                  |
-| `PARITY_SHARE_NAMESPACE`            | `Namespace` | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Parity shares: extended shares in the available data matrix.                                          |
+| name                                 | type        | category  | value                                                          | description                                                                |
+|--------------------------------------|-------------|-----------|----------------------------------------------------------------|----------------------------------------------------------------------------|
+| `TRANSACTION_NAMESPACE`              | `Namespace` | Primary   | `0x0000000000000000000000000000000000000000000000000000000001` | Namespace for ordinary Cosmos SDK transactions.                            |
+| `INTERMEDIATE_STATE_ROOT_NAMESPACE`  | `Namespace` | Primary   | `0x0000000000000000000000000000000000000000000000000000000002` | Namespace for intermediate state roots (not currently utilized).           |
+| `PAY_FOR_BLOB_NAMESPACE`             | `Namespace` | Primary   | `0x0000000000000000000000000000000000000000000000000000000004` | Namespace for transactions that contain a PayForBlob.                      |
+| `PRIMARY_RESERVED_PADDING_NAMESPACE` | `Namespace` | Primary   | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for padding after all primary reserved namespaces.               |
+| `MAX_PRIMARY_RESERVED_NAMESPACE`     | `Namespace` | Primary   | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for the highest primary reserved namespace.                      |
+| `MIN_SECONDARY_RESERVED_NAMESPACE`   | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00` | Namespace for the lowest secondary reserved namespace.                     |
+| `TAIL_PADDING_NAMESPACE`             | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Namespace for padding after all blobs to fill up the original data square. |
+| `PARITY_SHARE_NAMESPACE`             | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Namespace for parity shares.                                               |
 
 ## Assumptions and Considerations
 

specs/src/specs/shares.md

@@ -18,7 +18,7 @@ User submitted transactions are split into shares (see [share splitting](#share-
 
 [Padding](#padding) shares are added to the `k * k` matrix to ensure:
 
-1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#reserved-padding-share))
+1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#primary-reserved-padding-share))
 1. The number of shares in the matrix is a perfect square (see [tail padding share](#tail-padding-share))
 
 ## Share Format
@@ -85,17 +85,17 @@ Padding shares vary based on namespace but they conform to the [share format](#s
 A namespace padding share uses the namespace of the blob that precedes it in the data square so that the data square can retain the property that all shares are ordered by namespace.
 A namespace padding share acts as padding between blobs so that the subsequent blob begins at an index that conforms to the [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules). Clients MAY ignore the contents of these shares because they don't contain any significant data.
 
-### Reserved Padding Share
+### Primary Reserved Padding Share
 
-Reserved padding shares use the [`RESERVED_PADDING_NAMESPACE`](./consensus.md#constants). Reserved padding shares are placed after the last reserved namespace share in the data square so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data.
+Primary reserved padding shares use the [`PRIMARY_RESERVED_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Primary reserved padding shares are placed after shares in the primary reserved namespace range so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data.
 
 ### Tail Padding Share
 
-Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./consensus.md#constants). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data.
+Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data.
 
 ## Parity Share
 
-Parity shares use the [`PARITY_SHARE_NAMESPACE`](./consensus.md#constants). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning.
+Parity shares use the [`PARITY_SHARE_NAMESPACE`](./namespace.md#reserved-namespaces). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning.
 
 ## Share Splitting