fix: sparse inc mast less than uint32 max (milvus-io#38250)

relate: milvus-io#35853

Signed-off-by: aoiasd <[email protected]>

Author: aoiasd

internal/util/function/bm25_function.go

@@ -97,7 +97,7 @@ func (v *BM25FunctionRunner) run(data []string, dst []map[uint32]float32) error
 		for tokenStream.Advance() {
 			token := tokenStream.Token()
 			// TODO More Hash Option
-			hash := typeutil.HashString2Uint32(token)
+			hash := typeutil.HashString2LessUint32(token)
 			embeddingMap[hash] += 1
 		}
 		dst[i] = embeddingMap

pkg/util/typeutil/hash.go

@@ -19,6 +19,7 @@ package typeutil
 import (
 	"fmt"
 	"hash/crc32"
+	"math"
 	"strconv"
 	"strings"
 	"unsafe"
@@ -75,6 +76,16 @@ func HashString2Uint32(v string) uint32 {
 	return crc32.ChecksumIEEE([]byte(subString))
 }
 
+// HashString2LessUint32 hashing a string to uint32 but less than uint32 max
+func HashString2LessUint32(v string) uint32 {
+	subString := v
+	if len(v) > substringLengthForCRC {
+		subString = v[:substringLengthForCRC]
+	}
+
+	return crc32.ChecksumIEEE([]byte(subString)) % math.MaxUint32
+}
+
 // HashPK2Channels hash primary keys to channels
 func HashPK2Channels(primaryKeys *schemapb.IDs, shardNames []string) []uint32 {
 	numShard := uint32(len(shardNames))