Split storable.go & typeinfo.go into smaller files

This change makes the code easier to read and maintain.

This commit splits typeinfo.go into:
- compactmap_extradata.go
- extradata.go
- typeinfo.go

This commit also moves:
- SlabIDStorable to slab_id_storable.go
- CBOR tag numbers allocated for atree to cbor_tag_nums.go
- etc

array.go

@@ -385,70 +385,6 @@ func (a *Array) Set(index uint64, value Value) (Storable, error) {
 	return existingStorable, nil
 }
 
-// uninlineStorableIfNeeded uninlines given storable if needed, and
-// returns uninlined Storable and its ValueID.
-// If given storable is a WrapperStorable, this function uninlines
-// wrapped storable if needed and returns a new WrapperStorable
-// with wrapped uninlined storable and its ValidID.
-func uninlineStorableIfNeeded(storage SlabStorage, storable Storable) (Storable, ValueID, bool, error) {
-	if storable == nil {
-		return storable, emptyValueID, false, nil
-	}
-
-	switch s := storable.(type) {
-	case ArraySlab: // inlined array slab
-		err := s.Uninline(storage)
-		if err != nil {
-			return nil, emptyValueID, false, err
-		}
-
-		slabID := s.SlabID()
-
-		newStorable := SlabIDStorable(slabID)
-		valueID := slabIDToValueID(slabID)
-
-		return newStorable, valueID, true, nil
-
-	case MapSlab: // inlined map slab
-		err := s.Uninline(storage)
-		if err != nil {
-			return nil, emptyValueID, false, err
-		}
-
-		slabID := s.SlabID()
-
-		newStorable := SlabIDStorable(slabID)
-		valueID := slabIDToValueID(slabID)
-
-		return newStorable, valueID, true, nil
-
-	case SlabIDStorable: // uninlined slab
-		valueID := slabIDToValueID(SlabID(s))
-
-		return storable, valueID, false, nil
-
-	case WrapperStorable:
-		unwrappedStorable := unwrapStorable(s)
-
-		// Uninline wrapped storable if needed.
-		uninlinedWrappedStorable, valueID, uninlined, err := uninlineStorableIfNeeded(storage, unwrappedStorable)
-		if err != nil {
-			return nil, emptyValueID, false, err
-		}
-
-		if !uninlined {
-			return storable, valueID, uninlined, nil
-		}
-
-		// Create a new WrapperStorable with uninlinedWrappedStorable
-		newStorable := s.WrapAtreeStorable(uninlinedWrappedStorable)
-
-		return newStorable, valueID, uninlined, nil
-	}
-
-	return storable, emptyValueID, false, nil
-}
-
 func (a *Array) set(index uint64, value Value) (Storable, error) {
 	existingStorable, err := a.root.Set(a.Storage, a.Address(), index, value)
 	if err != nil {

cbor_tag_nums.go

@@ -0,0 +1,78 @@
+/*
+ * Atree - Scalable Arrays and Ordered Maps
+ *
+ * Copyright Flow Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package atree
+
+import "fmt"
+
+const (
+	// WARNING: tag numbers defined in here in github.com/onflow/atree
+	// MUST not overlap with tag numbers used by Cadence internal value encoding.
+	// As of Aug. 14, 2024, Cadence uses tag numbers from 128 to 230.
+	// See runtime/interpreter/encode.go at github.com/onflow/cadence.
+
+	// Atree reserves CBOR tag numbers [240, 255] for internal use.
+	// Applications must use non-overlapping CBOR tag numbers to encode
+	// elements managed by atree containers.
+	minInternalCBORTagNumber = 240
+	maxInternalCBORTagNumber = 255
+
+	// Reserved CBOR tag numbers for atree internal use.
+
+	// Replace _ when new tag number is needed (use higher tag numbers first).
+	// Atree will use higher tag numbers first because Cadence will use lower tag numbers first.
+	// This approach allows more flexibility in case we need to revisit ranges used by Atree and Cadence.
+
+	_ = 240
+	_ = 241
+	_ = 242
+	_ = 243
+	_ = 244
+	_ = 245
+
+	CBORTagTypeInfoRef = 246
+
+	CBORTagInlinedArrayExtraData      = 247
+	CBORTagInlinedMapExtraData        = 248
+	CBORTagInlinedCompactMapExtraData = 249
+
+	CBORTagInlinedArray      = 250
+	CBORTagInlinedMap        = 251
+	CBORTagInlinedCompactMap = 252
+
+	CBORTagInlineCollisionGroup   = 253
+	CBORTagExternalCollisionGroup = 254
+
+	CBORTagSlabID = 255
+)
+
+// IsCBORTagNumberRangeAvailable returns true if the specified range is not reserved for internal use by atree.
+// Applications must only use available (unreserved) CBOR tag numbers to encode elements in atree managed containers.
+func IsCBORTagNumberRangeAvailable(minTagNum, maxTagNum uint64) (bool, error) {
+	if minTagNum > maxTagNum {
+		return false, NewUserError(fmt.Errorf("min CBOR tag number %d must be <= max CBOR tag number %d", minTagNum, maxTagNum))
+	}
+
+	return maxTagNum < minInternalCBORTagNumber || minTagNum > maxInternalCBORTagNumber, nil
+}
+
+// ReservedCBORTagNumberRange returns minTagNum and maxTagNum of the range of CBOR tag numbers
+// reserved for internal use by atree.
+func ReservedCBORTagNumberRange() (minTagNum, maxTagNum uint64) {
+	return minInternalCBORTagNumber, maxInternalCBORTagNumber
+}

compactmap_extradata.go

@@ -0,0 +1,245 @@
+/*
+ * Atree - Scalable Arrays and Ordered Maps
+ *
+ * Copyright Flow Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package atree
+
+import (
+	"encoding/binary"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/fxamacker/cbor/v2"
+)
+
+// compactMapExtraData is used for inlining compact values.
+// compactMapExtraData includes hkeys and keys with map extra data
+// because hkeys and keys are the same in order and content for
+// all values with the same compact type and map seed.
+type compactMapExtraData struct {
+	mapExtraData *MapExtraData
+	hkeys        []Digest             // hkeys is ordered by mapExtraData.Seed
+	keys         []ComparableStorable // keys is ordered by mapExtraData.Seed
+}
+
+var _ ExtraData = &compactMapExtraData{}
+
+const compactMapExtraDataLength = 3
+
+func newCompactMapExtraData(
+	dec *cbor.StreamDecoder,
+	decodeTypeInfo TypeInfoDecoder,
+	decodeStorable StorableDecoder,
+) (*compactMapExtraData, error) {
+
+	length, err := dec.DecodeArrayHead()
+	if err != nil {
+		return nil, NewDecodingError(err)
+	}
+
+	if length != compactMapExtraDataLength {
+		return nil, NewDecodingError(
+			fmt.Errorf(
+				"compact extra data has invalid length %d, want %d",
+				length,
+				arrayExtraDataLength,
+			))
+	}
+
+	// element 0: map extra data
+	mapExtraData, err := newMapExtraData(dec, decodeTypeInfo)
+	if err != nil {
+		// err is already categorized by newMapExtraData().
+		return nil, err
+	}
+
+	// element 1: digests
+	digestBytes, err := dec.DecodeBytes()
+	if err != nil {
+		return nil, NewDecodingError(err)
+	}
+
+	if len(digestBytes)%digestSize != 0 {
+		return nil, NewDecodingError(
+			fmt.Errorf(
+				"decoding digests failed: number of bytes %d is not multiple of %d",
+				len(digestBytes),
+				digestSize))
+	}
+
+	digestCount := len(digestBytes) / digestSize
+
+	// element 2: keys
+	keyCount, err := dec.DecodeArrayHead()
+	if err != nil {
+		return nil, NewDecodingError(err)
+	}
+
+	if keyCount != uint64(digestCount) {
+		return nil, NewDecodingError(
+			fmt.Errorf(
+				"decoding compact map key failed: number of keys %d is different from number of digests %d",
+				keyCount,
+				digestCount))
+	}
+
+	hkeys := make([]Digest, digestCount)
+	for i := 0; i < digestCount; i++ {
+		hkeys[i] = Digest(binary.BigEndian.Uint64(digestBytes[i*digestSize:]))
+	}
+
+	keys := make([]ComparableStorable, keyCount)
+	for i := uint64(0); i < keyCount; i++ {
+		// Decode compact map key
+		key, err := decodeStorable(dec, SlabIDUndefined, nil)
+		if err != nil {
+			// Wrap err as external error (if needed) because err is returned by StorableDecoder callback.
+			return nil, wrapErrorfAsExternalErrorIfNeeded(err, "failed to decode key's storable")
+		}
+		compactMapKey, ok := key.(ComparableStorable)
+		if !ok {
+			return nil, NewDecodingError(fmt.Errorf("failed to decode key's storable: got %T, expect ComparableStorable", key))
+		}
+		keys[i] = compactMapKey
+	}
+
+	return &compactMapExtraData{mapExtraData: mapExtraData, hkeys: hkeys, keys: keys}, nil
+}
+
+func (c *compactMapExtraData) Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error {
+	err := enc.CBOR.EncodeArrayHead(compactMapExtraDataLength)
+	if err != nil {
+		return NewEncodingError(err)
+	}
+
+	// element 0: map extra data
+	err = c.mapExtraData.Encode(enc, encodeTypeInfo)
+	if err != nil {
+		// err is already categorized by MapExtraData.Encode().
+		return err
+	}
+
+	// element 1: digests
+	totalDigestSize := len(c.hkeys) * digestSize
+
+	var digests []byte
+	if totalDigestSize <= len(enc.Scratch) {
+		digests = enc.Scratch[:totalDigestSize]
+	} else {
+		digests = make([]byte, totalDigestSize)
+	}
+
+	for i := 0; i < len(c.hkeys); i++ {
+		binary.BigEndian.PutUint64(digests[i*digestSize:], uint64(c.hkeys[i]))
+	}
+
+	err = enc.CBOR.EncodeBytes(digests)
+	if err != nil {
+		return NewEncodingError(err)
+	}
+
+	// element 2: field names
+	err = enc.CBOR.EncodeArrayHead(uint64(len(c.keys)))
+	if err != nil {
+		return NewEncodingError(err)
+	}
+
+	for _, key := range c.keys {
+		err = key.Encode(enc)
+		if err != nil {
+			// Wrap err as external error (if needed) because err is returned by ComparableStorable.Encode().
+			return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode key's storable")
+		}
+	}
+
+	err = enc.CBOR.Flush()
+	if err != nil {
+		return NewEncodingError(err)
+	}
+
+	return nil
+}
+
+func (c *compactMapExtraData) isExtraData() bool {
+	return true
+}
+
+func (c *compactMapExtraData) Type() TypeInfo {
+	return c.mapExtraData.TypeInfo
+}
+
+// makeCompactMapTypeID returns id of concatenated t.ID() with sorted names with "," as separator.
+func makeCompactMapTypeID(encodedTypeInfo string, names []ComparableStorable) string {
+	const separator = ","
+
+	if len(names) == 0 {
+		return encodedTypeInfo
+	}
+
+	if len(names) == 1 {
+		return encodedTypeInfo + separator + names[0].ID()
+	}
+
+	sorter := newFieldNameSorter(names)
+
+	sort.Sort(sorter)
+
+	return encodedTypeInfo + separator + sorter.join(separator)
+}
+
+// fieldNameSorter sorts names by index (not in place sort).
+type fieldNameSorter struct {
+	names []ComparableStorable
+	index []int
+}
+
+func newFieldNameSorter(names []ComparableStorable) *fieldNameSorter {
+	index := make([]int, len(names))
+	for i := 0; i < len(names); i++ {
+		index[i] = i
+	}
+	return &fieldNameSorter{
+		names: names,
+		index: index,
+	}
+}
+
+func (fn *fieldNameSorter) Len() int {
+	return len(fn.names)
+}
+
+func (fn *fieldNameSorter) Less(i, j int) bool {
+	i = fn.index[i]
+	j = fn.index[j]
+	return fn.names[i].Less(fn.names[j])
+}
+
+func (fn *fieldNameSorter) Swap(i, j int) {
+	fn.index[i], fn.index[j] = fn.index[j], fn.index[i]
+}
+
+func (fn *fieldNameSorter) join(sep string) string {
+	var sb strings.Builder
+	for i, index := range fn.index {
+		if i > 0 {
+			sb.WriteString(sep)
+		}
+		sb.WriteString(fn.names[index].ID())
+	}
+	return sb.String()
+}