diff --git a/go.mod b/go.mod index 47bffb9dc4b18..c5736a513f13c 100644 --- a/go.mod +++ b/go.mod @@ -91,7 +91,8 @@ require ( github.com/tidwall/btree v1.7.0 github.com/tidwall/pretty v1.2.1 github.com/tmc/langchaingo v0.1.13 - github.com/unum-cloud/usearch/golang v0.0.0-20251010193336-541e882da5a9 + github.com/unum-cloud/usearch/golang v0.0.0-20251130095425-a2f175991017 + github.com/viterin/vek v0.4.3 go.starlark.net v0.0.0-20250701195324-d457b4515e0e go.uber.org/automaxprocs v1.5.3 go.uber.org/ratelimit v0.2.0 @@ -133,6 +134,7 @@ require ( github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect + github.com/chewxy/math32 v1.10.1 // indirect github.com/cilium/ebpf v0.9.1 // indirect github.com/clbanning/mxj v1.8.4 // indirect github.com/cloudwego/base64x v0.1.6 // indirect @@ -216,6 +218,7 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/valyala/fastrand v1.1.0 // indirect github.com/valyala/histogram v1.2.0 // indirect + github.com/viterin/partial v1.1.0 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/atomic v1.11.0 // indirect diff --git a/go.sum b/go.sum index a371965827208..62c29a8dac803 100644 --- a/go.sum +++ b/go.sum @@ -140,6 +140,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chewxy/math32 v1.10.1 h1:LFpeY0SLJXeaiej/eIp2L40VYfscTvKh/FSEZ68uMkU= +github.com/chewxy/math32 v1.10.1/go.mod h1:dOB2rcuFrCn6UHrze36WSLVPKtzPMRAQvBvUwkSsLqs= github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= @@ -847,8 +849,8 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/unum-cloud/usearch/golang v0.0.0-20251010193336-541e882da5a9 h1:JrHCee+uqpF2zXooiKu7ymvKgnzlUIXtTlZ7vi21Tr0= -github.com/unum-cloud/usearch/golang v0.0.0-20251010193336-541e882da5a9/go.mod h1:NxBpQibuBBeA/V8RGbrNzVAv4OyWWL5yNao7mVz656k= +github.com/unum-cloud/usearch/golang v0.0.0-20251130095425-a2f175991017 h1:HoupsZEQjg+s7Dip8iTeCa87OcDGrGda/xOleoNXm4A= +github.com/unum-cloud/usearch/golang v0.0.0-20251130095425-a2f175991017/go.mod h1:NxBpQibuBBeA/V8RGbrNzVAv4OyWWL5yNao7mVz656k= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= @@ -861,6 +863,10 @@ github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tz github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/viterin/partial v1.1.0 h1:iH1l1xqBlapXsYzADS1dcbizg3iQUKTU1rbwkHv/80E= +github.com/viterin/partial v1.1.0/go.mod h1:oKGAo7/wylWkJTLrWX8n+f4aDPtQMQ6VG4dd2qur5QA= +github.com/viterin/vek v0.4.3 h1:cogdlNjd6EJYtNbmTN0lJCey2htrfSo1AHWpc6DVncQ= +github.com/viterin/vek v0.4.3/go.mod h1:A4JRAe8OvbhdzBL5ofzjBS0J29FyUrf95tQogvtHHUc= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= diff --git a/pkg/common/assertx/float32.go b/pkg/common/assertx/float32.go new file mode 100644 index 0000000000000..12fc47b88764d --- /dev/null +++ b/pkg/common/assertx/float32.go @@ -0,0 +1,56 @@ +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package assertx + +import ( + "math" +) + +const float32EqualityThreshold = 1e-5 + +// InEpsilonF32Slices returns true if all the elements in v1 and v2 are within epsilon of each other. +func InEpsilonF32Slices(want, got [][]float32) bool { + if len(want) != len(got) { + return false + } + + for i := range want { + if !InEpsilonF32Slice(want[i], got[i]) { + return false + } + } + return true +} + +// InEpsilonF32Slice returns true if all the elements in v1 and v2 are within epsilon of each other. +// assert.InEpsilonSlice requires v1 to be non-zero. +func InEpsilonF32Slice(want, got []float32) bool { + if len(want) != len(got) { + return false + } + + for i := range want { + if !InEpsilonF32(want[i], got[i]) { + return false + } + } + return true +} + +// InEpsilonF32 returns true if v1 and v2 are within epsilon of each other. +// assert.InEpsilon requires v1 to be non-zero. +func InEpsilonF32(want, got float32) bool { + return want == got || math.Abs(float64(want)-float64(got)) < float32EqualityThreshold || (math.IsNaN(float64(want)) && math.IsNaN(float64(got))) +} diff --git a/pkg/common/assertx/float32_test.go b/pkg/common/assertx/float32_test.go new file mode 100644 index 0000000000000..8d7f89be47dcf --- /dev/null +++ b/pkg/common/assertx/float32_test.go @@ -0,0 +1,158 @@ +// Copyright 2023 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package assertx + +import "testing" + +func TestInEpsilonF32(t *testing.T) { + type args struct { + want float32 + got float32 + } + tests := []struct { + name string + args args + want bool + }{ + + { + name: "Test 1 - difference is epsilon", + args: args{ + want: 1.0, + got: 1.0 + float32EqualityThreshold, + }, + want: false, + }, + { + name: "Test 2.a - difference is less than epsilon", + args: args{ + want: 2.0, + got: 2.0 + 0.5*float32EqualityThreshold, + }, + want: true, + }, + { + name: "Test 2.b - difference is more than epsilon", + args: args{ + want: 2.0, + got: 2.0 + 2*float32EqualityThreshold, + }, + want: false, + }, + { + name: "Test 2.c - difference is -ve of epsilon", + args: args{ + want: 2.0, + got: 2.0 - float32EqualityThreshold, + }, + want: false, + }, + { + name: "Test 3 - till 9th digit is same, 10th digit is different", + args: args{ + want: 1.732050800_0, + got: 1.732050800_9, + }, + want: true, + }, + { + name: "Test 4 - 5th digit is different", + args: args{ + want: 1.7320_0, + got: 1.7320_9, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := InEpsilonF32(tt.args.want, tt.args.got); got != tt.want { + t.Errorf("%s InEpsilonF32() = %v, want %v", tt.name, got, tt.want) + } + }) + } +} + +func TestInEpsilonF32Slice(t *testing.T) { + type args struct { + want []float32 + got []float32 + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "Test 1 - difference is epsilon", + args: args{ + want: []float32{2.0, 3.0}, + got: []float32{2.0 + float32EqualityThreshold, 3.0 + float32EqualityThreshold}, + }, + want: false, + }, + { + name: "Test 2 - 6th digit is different", + args: args{ + want: []float32{1.7320_0}, + got: []float32{1.7320_1}, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := InEpsilonF32Slice(tt.args.want, tt.args.got); got != tt.want { + t.Errorf("InEpsilonF32Slice() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestInEpsilonF32Slices(t *testing.T) { + type args struct { + want [][]float32 + got [][]float32 + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "Test 1 - difference is epsilon", + args: args{ + want: [][]float32{{1.0, 3.0}, {4.0}}, + got: [][]float32{{1.0 + float32EqualityThreshold, 3.0 + float32EqualityThreshold}, {4.0}}, + }, + want: false, + }, + { + name: "Test 2 - difference is less than epsilon (next neg-power of epsilon & epsilon/2)", + args: args{ + want: [][]float32{{2.0, 3.0}, {4.0}}, + got: [][]float32{{2.0 + 1e-1*float32EqualityThreshold, 3.0 + float32EqualityThreshold/2}, {4.0}}, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := InEpsilonF32Slices(tt.args.want, tt.args.got); got != tt.want { + t.Errorf("InEpsilonF32Slices() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/common/assertx/float64.go b/pkg/common/assertx/float64.go index 36dd2bfa37f77..7aa864d3ab033 100644 --- a/pkg/common/assertx/float64.go +++ b/pkg/common/assertx/float64.go @@ -36,7 +36,7 @@ func InEpsilonF64Slices(want, got [][]float64) bool { return false } - for i := 0; i < len(want); i++ { + for i := range want { if !InEpsilonF64Slice(want[i], got[i]) { return false } @@ -51,7 +51,7 @@ func InEpsilonF64Slice(want, got []float64) bool { return false } - for i := 0; i < len(want); i++ { + for i := range want { if !InEpsilonF64(want[i], got[i]) { return false } diff --git a/pkg/compare/arraycompare.go b/pkg/compare/arraycompare.go index 18029176fa23c..9d077a98c7377 100644 --- a/pkg/compare/arraycompare.go +++ b/pkg/compare/arraycompare.go @@ -15,10 +15,11 @@ package compare import ( + "slices" + "github.com/matrixorigin/matrixone/pkg/container/nulls" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" - "github.com/matrixorigin/matrixone/pkg/vectorize/moarray" "github.com/matrixorigin/matrixone/pkg/vm/process" ) @@ -72,7 +73,7 @@ func CompareArrayFromBytes[T types.RealNumbers](_x, _y []byte, desc bool) int { y := types.BytesToArray[T](_y) if desc { - return moarray.Compare[T](y, x) + return slices.Compare(y, x) } - return moarray.Compare[T](x, y) + return slices.Compare(x, y) } diff --git a/pkg/container/vector/utils.go b/pkg/container/vector/utils.go index 627192672962b..ca0cfe3eed7ae 100644 --- a/pkg/container/vector/utils.go +++ b/pkg/container/vector/utils.go @@ -16,9 +16,9 @@ package vector import ( "bytes" + "slices" "github.com/matrixorigin/matrixone/pkg/container/types" - "github.com/matrixorigin/matrixone/pkg/vectorize/moarray" ) // FindFirstIndexInSortedSlice finds the first index of v in a sorted slice s @@ -308,10 +308,10 @@ func ArrayGetMinMax[T types.RealNumbers](vec *Vector) (minv, maxv []T) { minv, maxv = val, val first = false } else { - if moarray.Compare[T](minv, val) > 0 { + if slices.Compare(minv, val) > 0 { minv = val } - if moarray.Compare[T](maxv, val) < 0 { + if slices.Compare(maxv, val) < 0 { maxv = val } } @@ -321,10 +321,10 @@ func ArrayGetMinMax[T types.RealNumbers](vec *Vector) (minv, maxv []T) { minv, maxv = val, val for i, j := 1, vec.Length(); i < j; i++ { val := types.GetArray[T](&col[i], area) - if moarray.Compare[T](minv, val) > 0 { + if slices.Compare(minv, val) > 0 { minv = val } - if moarray.Compare[T](maxv, val) < 0 { + if slices.Compare(maxv, val) < 0 { maxv = val } } diff --git a/pkg/container/vector/vector.go b/pkg/container/vector/vector.go index cce6b838c6476..894d280de30b0 100644 --- a/pkg/container/vector/vector.go +++ b/pkg/container/vector/vector.go @@ -28,7 +28,6 @@ import ( "github.com/matrixorigin/matrixone/pkg/container/bytejson" "github.com/matrixorigin/matrixone/pkg/container/nulls" "github.com/matrixorigin/matrixone/pkg/container/types" - "github.com/matrixorigin/matrixone/pkg/vectorize/moarray" "github.com/matrixorigin/matrixone/pkg/vectorize/shuffle" ) @@ -4249,13 +4248,13 @@ func (v *Vector) InplaceSortAndCompact() { case types.T_array_float32: col, area := MustVarlenaRawData(v) sort.Slice(col, func(i, j int) bool { - return moarray.Compare( + return slices.Compare( types.GetArray[float32](&col[i], area), types.GetArray[float32](&col[j], area), ) < 0 }) newCol := slices.CompactFunc(col, func(a, b types.Varlena) bool { - return moarray.Compare( + return slices.Compare( types.GetArray[float32](&a, area), types.GetArray[float32](&b, area), ) == 0 @@ -4268,13 +4267,13 @@ func (v *Vector) InplaceSortAndCompact() { case types.T_array_float64: col, area := MustVarlenaRawData(v) sort.Slice(col, func(i, j int) bool { - return moarray.Compare( + return slices.Compare( types.GetArray[float64](&col[i], area), types.GetArray[float64](&col[j], area), ) < 0 }) newCol := slices.CompactFunc(col, func(a, b types.Varlena) bool { - return moarray.Compare( + return slices.Compare( types.GetArray[float64](&a, area), types.GetArray[float64](&b, area), ) == 0 @@ -4429,7 +4428,7 @@ func (v *Vector) InplaceSort() { case types.T_array_float32: col, area := MustVarlenaRawData(v) sort.Slice(col, func(i, j int) bool { - return moarray.Compare[float32]( + return slices.Compare( types.GetArray[float32](&col[i], area), types.GetArray[float32](&col[j], area), ) < 0 @@ -4437,7 +4436,7 @@ func (v *Vector) InplaceSort() { case types.T_array_float64: col, area := MustVarlenaRawData(v) sort.Slice(col, func(i, j int) bool { - return moarray.Compare[float64]( + return slices.Compare( types.GetArray[float64](&col[i], area), types.GetArray[float64](&col[j], area), ) < 0 diff --git a/pkg/sort/sort.go b/pkg/sort/sort.go index ce867b31c6229..a25a7efec7c0b 100644 --- a/pkg/sort/sort.go +++ b/pkg/sort/sort.go @@ -16,12 +16,12 @@ package sort import ( "math/bits" + "slices" "github.com/matrixorigin/matrixone/pkg/container/bytejson" "github.com/matrixorigin/matrixone/pkg/container/nulls" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" - "github.com/matrixorigin/matrixone/pkg/vectorize/moarray" ) const ( @@ -360,7 +360,7 @@ func uuidLess(data []types.Uuid, i, j int64) bool { } func arrayLess[T types.RealNumbers](data [][]T, i, j int64) bool { - return moarray.Compare[T](data[i], data[j]) < 0 + return slices.Compare(data[i], data[j]) < 0 } func uuidGreater(data []types.Uuid, i, j int64) bool { @@ -368,7 +368,7 @@ func uuidGreater(data []types.Uuid, i, j int64) bool { } func arrayGreater[T types.RealNumbers](data [][]T, i, j int64) bool { - return moarray.Compare[T](data[i], data[j]) > 0 + return slices.Compare(data[i], data[j]) > 0 } func genericLess[T types.OrderedT](data []T, i, j int64) bool { diff --git a/pkg/sql/plan/function/arithmetic.go b/pkg/sql/plan/function/arithmetic.go index 326541d49ad7c..e5dc5f32b2da0 100644 --- a/pkg/sql/plan/function/arithmetic.go +++ b/pkg/sql/plan/function/arithmetic.go @@ -158,12 +158,12 @@ func vectorScalarOp[T types.RealNumbers](ivecs []*vector.Vector, result vector.F return err } } else { - out, err := moarray.ScalarOp[T](types.BytesToArray[T](vec), op, float64(sca)) + out, err := moarray.ScalarOp(types.BytesToArray[T](vec), op, sca) if err != nil { return err } - if err = rs.AppendBytes(types.ArrayToBytes[T](out), false); err != nil { + if err = rs.AppendBytes(types.ArrayToBytes(out), false); err != nil { return err } } @@ -184,12 +184,12 @@ func vectorScalarOp[T types.RealNumbers](ivecs []*vector.Vector, result vector.F return err } } else { - out, err := moarray.ScalarOp[T](types.BytesToArray[T](vec), op, float64(sca)) + out, err := moarray.ScalarOp(types.BytesToArray[T](vec), op, sca) if err != nil { return err } - if err = rs.AppendBytes(types.ArrayToBytes[T](out), false); err != nil { + if err = rs.AppendBytes(types.ArrayToBytes(out), false); err != nil { return err } } diff --git a/pkg/sql/plan/function/func_binary.go b/pkg/sql/plan/function/func_binary.go index 3de511b94ddba..ebe9407e98924 100644 --- a/pkg/sql/plan/function/func_binary.go +++ b/pkg/sql/plan/function/func_binary.go @@ -29,9 +29,6 @@ import ( "strings" "time" - "github.com/matrixorigin/matrixone/pkg/util/fault" - "go.uber.org/zap" - "github.com/matrixorigin/matrixone/pkg/clusterservice" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/nulls" @@ -41,11 +38,14 @@ import ( "github.com/matrixorigin/matrixone/pkg/pb/metadata" fj "github.com/matrixorigin/matrixone/pkg/sql/plan/function/fault" "github.com/matrixorigin/matrixone/pkg/sql/plan/function/functionUtil" + "github.com/matrixorigin/matrixone/pkg/util/fault" + "github.com/matrixorigin/matrixone/pkg/vectorindex/metric" "github.com/matrixorigin/matrixone/pkg/vectorize/floor" "github.com/matrixorigin/matrixone/pkg/vectorize/format" "github.com/matrixorigin/matrixone/pkg/vectorize/instr" "github.com/matrixorigin/matrixone/pkg/vectorize/moarray" "github.com/matrixorigin/matrixone/pkg/vm/process" + "go.uber.org/zap" "golang.org/x/exp/constraints" ) @@ -7295,7 +7295,7 @@ func SplitSingle(str, sep string, cnt uint32) (string, bool) { } func InnerProductArray[T types.RealNumbers](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { - return opBinaryBytesBytesToFixedWithErrorCheck[float64](ivecs, result, proc, length, func(v1, v2 []byte) (out float64, err error) { + return opBinaryBytesBytesToFixedWithErrorCheck[T](ivecs, result, proc, length, func(v1, v2 []byte) (out T, err error) { _v1 := types.BytesToArray[T](v1) _v2 := types.BytesToArray[T](v2) @@ -7304,34 +7304,38 @@ func InnerProductArray[T types.RealNumbers](ivecs []*vector.Vector, result vecto } func CosineSimilarityArray[T types.RealNumbers](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { - return opBinaryBytesBytesToFixedWithErrorCheck[float64](ivecs, result, proc, length, func(v1, v2 []byte) (out float64, err error) { + return opBinaryBytesBytesToFixedWithErrorCheck[T](ivecs, result, proc, length, func(v1, v2 []byte) (out T, err error) { _v1 := types.BytesToArray[T](v1) _v2 := types.BytesToArray[T](v2) - return moarray.CosineSimilarity[T](_v1, _v2) + dist, err := metric.CosineSimilarity(_v1, _v2) + return T(dist), err }, selectList) } func L2DistanceArray[T types.RealNumbers](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { - return opBinaryBytesBytesToFixedWithErrorCheck[float64](ivecs, result, proc, length, func(v1, v2 []byte) (out float64, err error) { + return opBinaryBytesBytesToFixedWithErrorCheck[T](ivecs, result, proc, length, func(v1, v2 []byte) (out T, err error) { _v1 := types.BytesToArray[T](v1) _v2 := types.BytesToArray[T](v2) - return moarray.L2Distance[T](_v1, _v2) + dist, err := metric.L2Distance(_v1, _v2) + return T(dist), err }, selectList) } func L2DistanceSqArray[T types.RealNumbers](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { - return opBinaryBytesBytesToFixedWithErrorCheck[float64](ivecs, result, proc, length, func(v1, v2 []byte) (out float64, err error) { + return opBinaryBytesBytesToFixedWithErrorCheck[T](ivecs, result, proc, length, func(v1, v2 []byte) (out T, err error) { _v1 := types.BytesToArray[T](v1) _v2 := types.BytesToArray[T](v2) - return moarray.L2DistanceSq[T](_v1, _v2) + dist, err := metric.L2DistanceSq(_v1, _v2) + return T(dist), err }, selectList) } func CosineDistanceArray[T types.RealNumbers](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { - return opBinaryBytesBytesToFixedWithErrorCheck[float64](ivecs, result, proc, length, func(v1, v2 []byte) (out float64, err error) { + return opBinaryBytesBytesToFixedWithErrorCheck[T](ivecs, result, proc, length, func(v1, v2 []byte) (out T, err error) { _v1 := types.BytesToArray[T](v1) _v2 := types.BytesToArray[T](v2) - return moarray.CosineDistance[T](_v1, _v2) + dist, err := metric.CosineDistance(_v1, _v2) + return T(dist), err }, selectList) } diff --git a/pkg/sql/plan/function/func_binary_test.go b/pkg/sql/plan/function/func_binary_test.go index a58e0cb217362..0d033f753fb02 100644 --- a/pkg/sql/plan/function/func_binary_test.go +++ b/pkg/sql/plan/function/func_binary_test.go @@ -4900,9 +4900,8 @@ func initSqrtArrayTestCase() []tcTemp { [][]float32{{4, 9, 16}, {0, 25, 49}}, []bool{false, false}), }, - expect: NewFunctionTestResult(types.T_array_float64.ToType(), false, - //NOTE: SQRT(vecf32) --> vecf64 - [][]float64{{2, 3, 4}, {0, 5, 7}}, + expect: NewFunctionTestResult(types.T_array_float32.ToType(), false, + [][]float32{{2, 3, 4}, {0, 5, 7}}, []bool{false, false}), }, { @@ -4947,8 +4946,8 @@ func initInnerProductArrayTestCase() []tcTemp { NewFunctionTestInput(types.T_array_float32.ToType(), [][]float32{{1, 2, 3}, {4, 5, 6}}, []bool{false, false}), NewFunctionTestInput(types.T_array_float32.ToType(), [][]float32{{1, 2, 3}, {4, 5, 6}}, []bool{false, false}), }, - expect: NewFunctionTestResult(types.T_float64.ToType(), false, - []float64{-14, -77}, + expect: NewFunctionTestResult(types.T_float32.ToType(), false, + []float32{-14, -77}, []bool{false, false}), }, { @@ -4992,8 +4991,8 @@ func initCosineSimilarityArrayTestCase() []tcTemp { NewFunctionTestInput(types.T_array_float32.ToType(), [][]float32{{1, 2, 3}, {4, 5, 6}}, []bool{false, false}), NewFunctionTestInput(types.T_array_float32.ToType(), [][]float32{{1, 2, 3}, {4, 5, 6}}, []bool{false, false}), }, - expect: NewFunctionTestResult(types.T_float64.ToType(), false, - []float64{1, 1}, + expect: NewFunctionTestResult(types.T_float32.ToType(), false, + []float32{1, 1}, []bool{false, false}), }, { @@ -5037,8 +5036,8 @@ func initL2DistanceArrayTestCase() []tcTemp { NewFunctionTestInput(types.T_array_float32.ToType(), [][]float32{{1, 2, 3}, {4, 5, 6}}, []bool{false, false}), NewFunctionTestInput(types.T_array_float32.ToType(), [][]float32{{10, 20, 30}, {40, 50, 60}}, []bool{false, false}), }, - expect: NewFunctionTestResult(types.T_float64.ToType(), false, - []float64{33.6749153137207, 78.97467803955078}, + expect: NewFunctionTestResult(types.T_float32.ToType(), false, + []float32{33.6749153137207, 78.97467803955078}, []bool{false, false}), }, { @@ -5082,8 +5081,8 @@ func initCosineDistanceArrayTestCase() []tcTemp { NewFunctionTestInput(types.T_array_float32.ToType(), [][]float32{{1, 2, 3}, {4, 5, 6}}, []bool{false, false}), NewFunctionTestInput(types.T_array_float32.ToType(), [][]float32{{10, 20, 30}, {5, 6, 7}}, []bool{false, false}), }, - expect: NewFunctionTestResult(types.T_float64.ToType(), false, - []float64{0, 0.00035416888764172594}, + expect: NewFunctionTestResult(types.T_float32.ToType(), false, + []float32{0, 0.0003542900085449219}, []bool{false, false}), }, { diff --git a/pkg/sql/plan/function/func_builtin.go b/pkg/sql/plan/function/func_builtin.go index d4eb86a1b8f42..edf057bf63324 100644 --- a/pkg/sql/plan/function/func_builtin.go +++ b/pkg/sql/plan/function/func_builtin.go @@ -25,7 +25,6 @@ import ( "time" "github.com/google/uuid" - "github.com/matrixorigin/matrixone/pkg/common/hashmap" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/runtime" @@ -2674,7 +2673,7 @@ func builtInSqrtArray[T types.RealNumbers](parameters []*vector.Vector, result v if err != nil { return nil, err } - return types.ArrayToBytes[float64](_out), nil + return types.ArrayToBytes[T](_out), nil }, selectList) } diff --git a/pkg/sql/plan/function/func_builtin_leastgreatest.go b/pkg/sql/plan/function/func_builtin_leastgreatest.go index b83320db6ba51..72994b52e5eb4 100644 --- a/pkg/sql/plan/function/func_builtin_leastgreatest.go +++ b/pkg/sql/plan/function/func_builtin_leastgreatest.go @@ -16,11 +16,11 @@ package function import ( "bytes" + "slices" "github.com/matrixorigin/matrixone/pkg/container/nulls" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" - "github.com/matrixorigin/matrixone/pkg/vectorize/moarray" "github.com/matrixorigin/matrixone/pkg/vm/process" ) @@ -334,7 +334,7 @@ func leastFn(parameters []*vector.Vector, _v1 := types.BytesToArray[float32](v1) _v2 := types.BytesToArray[float32](v2) - return moarray.Compare(_v1, _v2) < 0 + return slices.Compare(_v1, _v2) < 0 }) case types.T_array_float64: @@ -347,7 +347,7 @@ func leastFn(parameters []*vector.Vector, func(v1, v2 []byte) bool { _v1 := types.BytesToArray[float64](v1) _v2 := types.BytesToArray[float64](v2) - return moarray.Compare(_v1, _v2) < 0 + return slices.Compare(_v1, _v2) < 0 }) case types.T_date: @@ -605,7 +605,7 @@ func greatestFn(parameters []*vector.Vector, func(v1, v2 []byte) bool { _v1 := types.BytesToArray[float32](v1) _v2 := types.BytesToArray[float32](v2) - return moarray.Compare(_v1, _v2) > 0 + return slices.Compare(_v1, _v2) > 0 }) case types.T_array_float64: @@ -618,7 +618,7 @@ func greatestFn(parameters []*vector.Vector, func(v1, v2 []byte) bool { _v1 := types.BytesToArray[float64](v1) _v2 := types.BytesToArray[float64](v2) - return moarray.Compare(_v1, _v2) > 0 + return slices.Compare(_v1, _v2) > 0 }) case types.T_date: diff --git a/pkg/sql/plan/function/func_compare.go b/pkg/sql/plan/function/func_compare.go index f00e54ffd9b09..ba959d4c52c98 100644 --- a/pkg/sql/plan/function/func_compare.go +++ b/pkg/sql/plan/function/func_compare.go @@ -16,11 +16,12 @@ package function import ( "bytes" + "slices" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/nulls" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" - "github.com/matrixorigin/matrixone/pkg/vectorize/moarray" "github.com/matrixorigin/matrixone/pkg/vm/process" ) @@ -148,7 +149,7 @@ func equalFn(parameters []*vector.Vector, result vector.FunctionResultWrapper, p _v1 := types.BytesToArray[float32](v1) _v2 := types.BytesToArray[float32](v2) - return moarray.Compare[float32](_v1, _v2) == 0 + return slices.Compare(_v1, _v2) == 0 }, selectList) case types.T_array_float64: if parameters[0].GetArea() == nil && parameters[1].GetArea() == nil && (selectList == nil) { @@ -158,7 +159,7 @@ func equalFn(parameters []*vector.Vector, result vector.FunctionResultWrapper, p _v1 := types.BytesToArray[float64](v1) _v2 := types.BytesToArray[float64](v2) - return moarray.Compare[float64](_v1, _v2) == 0 + return slices.Compare(_v1, _v2) == 0 }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { @@ -643,14 +644,14 @@ func greatThanFn(parameters []*vector.Vector, result vector.FunctionResultWrappe _v1 := types.BytesToArray[float32](v1) _v2 := types.BytesToArray[float32](v2) - return moarray.Compare[float32](_v1, _v2) > 0 + return slices.Compare(_v1, _v2) > 0 }, selectList) case types.T_array_float64: return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { _v1 := types.BytesToArray[float64](v1) _v2 := types.BytesToArray[float64](v2) - return moarray.Compare[float64](_v1, _v2) > 0 + return slices.Compare(_v1, _v2) > 0 }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { @@ -753,14 +754,14 @@ func greatEqualFn(parameters []*vector.Vector, result vector.FunctionResultWrapp _v1 := types.BytesToArray[float32](v1) _v2 := types.BytesToArray[float32](v2) - return moarray.Compare[float32](_v1, _v2) >= 0 + return slices.Compare(_v1, _v2) >= 0 }, selectList) case types.T_array_float64: return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { _v1 := types.BytesToArray[float64](v1) _v2 := types.BytesToArray[float64](v2) - return moarray.Compare[float64](_v1, _v2) >= 0 + return slices.Compare(_v1, _v2) >= 0 }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { @@ -863,14 +864,14 @@ func notEqualFn(parameters []*vector.Vector, result vector.FunctionResultWrapper _v1 := types.BytesToArray[float32](v1) _v2 := types.BytesToArray[float32](v2) - return moarray.Compare[float32](_v1, _v2) != 0 + return slices.Compare(_v1, _v2) != 0 }, selectList) case types.T_array_float64: return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { _v1 := types.BytesToArray[float64](v1) _v2 := types.BytesToArray[float64](v2) - return moarray.Compare[float64](_v1, _v2) != 0 + return slices.Compare(_v1, _v2) != 0 }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { @@ -973,14 +974,14 @@ func lessThanFn(parameters []*vector.Vector, result vector.FunctionResultWrapper _v1 := types.BytesToArray[float32](v1) _v2 := types.BytesToArray[float32](v2) - return moarray.Compare[float32](_v1, _v2) < 0 + return slices.Compare(_v1, _v2) < 0 }, selectList) case types.T_array_float64: return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { _v1 := types.BytesToArray[float64](v1) _v2 := types.BytesToArray[float64](v2) - return moarray.Compare[float64](_v1, _v2) < 0 + return slices.Compare(_v1, _v2) < 0 }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { @@ -1083,14 +1084,14 @@ func lessEqualFn(parameters []*vector.Vector, result vector.FunctionResultWrappe _v1 := types.BytesToArray[float32](v1) _v2 := types.BytesToArray[float32](v2) - return moarray.Compare[float32](_v1, _v2) <= 0 + return slices.Compare(_v1, _v2) <= 0 }, selectList) case types.T_array_float64: return opBinaryBytesBytesToFixed[bool](parameters, rs, proc, length, func(v1, v2 []byte) bool { _v1 := types.BytesToArray[float64](v1) _v2 := types.BytesToArray[float64](v2) - return moarray.Compare[float64](_v1, _v2) <= 0 + return slices.Compare(_v1, _v2) <= 0 }, selectList) case types.T_date: return opBinaryFixedFixedToFixed[types.Date, types.Date, bool](parameters, rs, proc, length, func(a, b types.Date) bool { diff --git a/pkg/sql/plan/function/func_testcase.go b/pkg/sql/plan/function/func_testcase.go index 0103b6594ad8e..c02d16f6362b2 100644 --- a/pkg/sql/plan/function/func_testcase.go +++ b/pkg/sql/plan/function/func_testcase.go @@ -16,13 +16,13 @@ package function import ( "fmt" + "slices" "github.com/matrixorigin/matrixone/pkg/common/assertx" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/nulls" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" - "github.com/matrixorigin/matrixone/pkg/vectorize/moarray" "github.com/matrixorigin/matrixone/pkg/vm/process" ) @@ -616,7 +616,7 @@ func (fc *FunctionTestCase) Run() (succeed bool, errInfo string) { if null2 { return false, fmt.Sprintf("the %dth row expected %s, but get NULL", i+1, string(want)) } - if moarray.Compare[float32](types.BytesToArray[float32](want), types.BytesToArray[float32](get)) != 0 { + if slices.Compare(types.BytesToArray[float32](want), types.BytesToArray[float32](get)) != 0 { return false, fmt.Sprintf("the %dth row expected %v, but get %v", i+1, types.BytesToArray[float32](want), types.BytesToArray[float32](get)) } diff --git a/pkg/sql/plan/function/func_unary.go b/pkg/sql/plan/function/func_unary.go index 35ca367e1872d..b628ca2c0dc37 100644 --- a/pkg/sql/plan/function/func_unary.go +++ b/pkg/sql/plan/function/func_unary.go @@ -41,27 +41,26 @@ import ( "unicode/utf8" "unsafe" - "github.com/matrixorigin/matrixone/pkg/common/util" - "github.com/matrixorigin/matrixone/pkg/datalink" - "github.com/RoaringBitmap/roaring" - "golang.org/x/exp/constraints" - "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/common/system" + "github.com/matrixorigin/matrixone/pkg/common/util" "github.com/matrixorigin/matrixone/pkg/container/nulls" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/datalink" "github.com/matrixorigin/matrixone/pkg/fileservice" "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/sql/plan/function/functionUtil" "github.com/matrixorigin/matrixone/pkg/util/fault" + "github.com/matrixorigin/matrixone/pkg/vectorindex/metric" "github.com/matrixorigin/matrixone/pkg/vectorize/lengthutf8" "github.com/matrixorigin/matrixone/pkg/vectorize/moarray" "github.com/matrixorigin/matrixone/pkg/vectorize/momath" "github.com/matrixorigin/matrixone/pkg/version" "github.com/matrixorigin/matrixone/pkg/vm/process" + "golang.org/x/exp/constraints" ) func AbsUInt64(ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { @@ -234,7 +233,7 @@ func NormalizeL2Array[T types.RealNumbers](parameters []*vector.Vector, result v } else { outArrayF32 = outArrayF32[:len(inArrayF32)] } - _ = moarray.NormalizeL2(inArrayF32, outArrayF32) + _ = metric.NormalizeL2(inArrayF32, outArrayF32) _ = rs.AppendBytes(types.ArrayToBytes[float32](outArrayF32), false) *outArrayF32Ptr = outArrayF32 @@ -250,7 +249,7 @@ func NormalizeL2Array[T types.RealNumbers](parameters []*vector.Vector, result v } else { outArrayF64 = outArrayF64[:len(inArrayF64)] } - _ = moarray.NormalizeL2(inArrayF64, outArrayF64) + _ = metric.NormalizeL2(inArrayF64, outArrayF64) _ = rs.AppendBytes(types.ArrayToBytes[float64](outArrayF64), false) *outArrayF64Ptr = outArrayF64 @@ -263,14 +262,14 @@ func NormalizeL2Array[T types.RealNumbers](parameters []*vector.Vector, result v } func L1NormArray[T types.RealNumbers](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { - return opUnaryBytesToFixedWithErrorCheck[float64](ivecs, result, proc, length, func(in []byte) (float64, error) { + return opUnaryBytesToFixedWithErrorCheck[T](ivecs, result, proc, length, func(in []byte) (T, error) { _in := types.BytesToArray[T](in) return moarray.L1Norm(_in) }, selectList) } func L2NormArray[T types.RealNumbers](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { - return opUnaryBytesToFixedWithErrorCheck[float64](ivecs, result, proc, length, func(in []byte) (out float64, err error) { + return opUnaryBytesToFixedWithErrorCheck[T](ivecs, result, proc, length, func(in []byte) (T, error) { _in := types.BytesToArray[T](in) return moarray.L2Norm(_in) }, selectList) @@ -284,7 +283,7 @@ func VectorDimsArray[T types.RealNumbers](ivecs []*vector.Vector, result vector. } func SummationArray[T types.RealNumbers](ivecs []*vector.Vector, result vector.FunctionResultWrapper, proc *process.Process, length int, selectList *FunctionSelectList) error { - return opUnaryBytesToFixedWithErrorCheck[float64](ivecs, result, proc, length, func(in []byte) (out float64, err error) { + return opUnaryBytesToFixedWithErrorCheck[T](ivecs, result, proc, length, func(in []byte) (out T, err error) { _in := types.BytesToArray[T](in) return moarray.Summation[T](_in) diff --git a/pkg/sql/plan/function/func_unary_test.go b/pkg/sql/plan/function/func_unary_test.go index c21f2d1e0f991..5951dfcd591d3 100644 --- a/pkg/sql/plan/function/func_unary_test.go +++ b/pkg/sql/plan/function/func_unary_test.go @@ -22,16 +22,14 @@ import ( "testing" "time" - "github.com/matrixorigin/matrixone/pkg/sql/plan/function/functionUtil" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/matrixorigin/matrixone/pkg/sql/plan/function/functionUtil" "github.com/matrixorigin/matrixone/pkg/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type tcTemp struct { @@ -219,7 +217,7 @@ func initNormalizeL2ArrayTestCase() []tcTemp { {}, {0.18257418, 0.36514837, 0.5477226, 0.73029673}, {-0.18257418, 0.36514837, 0.5477226, 0.73029673}, - {0.8108108, 0.27027026, 0.32432434, 0.4054054}, + {0.8108108, 0.27027026, 0.3243243, 0.4054054}, {0.1576765, 0.315353, 0.5781472, 0.73582363}, }, []bool{true, false, false, false, false, false}), @@ -278,8 +276,8 @@ func initSummationArrayTestCase() []tcTemp { [][]float32{{1, 2, 3}, {4, 5, 6}}, []bool{false, false}), }, - expect: NewFunctionTestResult(types.T_float64.ToType(), false, - []float64{6, 15}, + expect: NewFunctionTestResult(types.T_float32.ToType(), false, + []float32{6, 15}, []bool{false, false}), }, { @@ -324,8 +322,8 @@ func initL1NormArrayTestCase() []tcTemp { [][]float32{{1, 2, 3}, {4, 5, 6}}, []bool{false, false}), }, - expect: NewFunctionTestResult(types.T_float64.ToType(), false, - []float64{6, 15}, + expect: NewFunctionTestResult(types.T_float32.ToType(), false, + []float32{6, 15}, []bool{false, false}), }, { @@ -370,8 +368,8 @@ func initL2NormArrayTestCase() []tcTemp { [][]float32{{1, 2, 3}, {4, 5, 6}}, []bool{false, false}), }, - expect: NewFunctionTestResult(types.T_float64.ToType(), false, - []float64{3.741657257080078, 8.774964332580566}, + expect: NewFunctionTestResult(types.T_float32.ToType(), false, + []float32{3.741657257080078, 8.774964332580566}, []bool{false, false}), }, { diff --git a/pkg/sql/plan/function/list_builtIn.go b/pkg/sql/plan/function/list_builtIn.go index 0afc148af3561..0e1103b9609c7 100644 --- a/pkg/sql/plan/function/list_builtIn.go +++ b/pkg/sql/plan/function/list_builtIn.go @@ -3292,8 +3292,7 @@ var supportedArrayOperations = []FuncNew{ overloadId: 0, args: []types.T{types.T_array_float32}, retType: func(parameters []types.Type) types.Type { - // NOTE summation(vecf32) --> float64 - return types.T_float64.ToType() + return types.T_float32.ToType() }, newOp: func() executeLogicOfOverload { return SummationArray[float32] @@ -3324,7 +3323,7 @@ var supportedArrayOperations = []FuncNew{ overloadId: 0, args: []types.T{types.T_array_float32}, retType: func(parameters []types.Type) types.Type { - return types.T_float64.ToType() + return types.T_float32.ToType() }, newOp: func() executeLogicOfOverload { return L1NormArray[float32] @@ -3355,7 +3354,7 @@ var supportedArrayOperations = []FuncNew{ overloadId: 0, args: []types.T{types.T_array_float32}, retType: func(parameters []types.Type) types.Type { - return types.T_float64.ToType() + return types.T_float32.ToType() }, newOp: func() executeLogicOfOverload { return L2NormArray[float32] @@ -3417,7 +3416,7 @@ var supportedArrayOperations = []FuncNew{ overloadId: 0, args: []types.T{types.T_array_float32, types.T_array_float32}, retType: func(parameters []types.Type) types.Type { - return types.T_float64.ToType() + return types.T_float32.ToType() }, newOp: func() executeLogicOfOverload { return InnerProductArray[float32] @@ -3448,7 +3447,7 @@ var supportedArrayOperations = []FuncNew{ overloadId: 0, args: []types.T{types.T_array_float32, types.T_array_float32}, retType: func(parameters []types.Type) types.Type { - return types.T_float64.ToType() + return types.T_float32.ToType() }, newOp: func() executeLogicOfOverload { return CosineSimilarityArray[float32] @@ -3479,7 +3478,7 @@ var supportedArrayOperations = []FuncNew{ overloadId: 0, args: []types.T{types.T_array_float32, types.T_array_float32}, retType: func(parameters []types.Type) types.Type { - return types.T_float64.ToType() + return types.T_float32.ToType() }, newOp: func() executeLogicOfOverload { return L2DistanceArray[float32] @@ -3541,7 +3540,7 @@ var supportedArrayOperations = []FuncNew{ overloadId: 0, args: []types.T{types.T_array_float32, types.T_array_float32}, retType: func(parameters []types.Type) types.Type { - return types.T_float64.ToType() + return types.T_float32.ToType() }, newOp: func() executeLogicOfOverload { return L2DistanceSqArray[float32] @@ -3603,7 +3602,7 @@ var supportedArrayOperations = []FuncNew{ overloadId: 0, args: []types.T{types.T_array_float32, types.T_array_float32}, retType: func(parameters []types.Type) types.Type { - return types.T_float64.ToType() + return types.T_float32.ToType() }, newOp: func() executeLogicOfOverload { return CosineDistanceArray[float32] @@ -3806,7 +3805,7 @@ var supportedMathBuiltIns = []FuncNew{ overloadId: 1, args: []types.T{types.T_array_float32}, retType: func(parameters []types.Type) types.Type { - return types.T_array_float64.ToType() + return types.T_array_float32.ToType() }, newOp: func() executeLogicOfOverload { return builtInSqrtArray[float32] diff --git a/pkg/vectorindex/metric/distance_func.go b/pkg/vectorindex/metric/distance_func.go index 496e5e3c650e2..24d3df34e8406 100644 --- a/pkg/vectorindex/metric/distance_func.go +++ b/pkg/vectorindex/metric/distance_func.go @@ -19,139 +19,33 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/types" - "gonum.org/v1/gonum/blas/blas32" - "gonum.org/v1/gonum/blas/blas64" + "github.com/matrixorigin/matrixone/pkg/vectorize/moarray" + "github.com/viterin/vek" + "github.com/viterin/vek/vek32" ) func L2Distance[T types.RealNumbers](v1, v2 []T) (T, error) { - switch any(v1).(type) { - case []float32: - _v1 := any(v1).([]float32) - _v2 := any(v2).([]float32) - - diff := blas32.Vector{ - N: len(_v1), - Inc: 1, - Data: make([]float32, len(_v1)), - } - - for i := range _v1 { - diff.Data[i] = _v1[i] - _v2[i] - } - return T(blas32.Nrm2(diff)), nil - - case []float64: - _v1 := any(v1).([]float64) - _v2 := any(v2).([]float64) - - diff := blas64.Vector{ - N: len(_v1), - Inc: 1, - Data: make([]float64, len(_v1)), - } - - for i := range _v1 { - diff.Data[i] = _v1[i] - _v2[i] - } - return T(blas64.Nrm2(diff)), nil - default: - return 0, moerr.NewInternalErrorNoCtx("L2Distance type not supported") - } - + return moarray.L2Distance(v1, v2) } func L2DistanceSq[T types.RealNumbers](v1, v2 []T) (T, error) { - var sumOfSquares T - for i := range v1 { - diff := v1[i] - v2[i] - sumOfSquares += diff * diff - } - return sumOfSquares, nil - + return moarray.L2DistanceSq(v1, v2) } func L1Distance[T types.RealNumbers](v1, v2 []T) (T, error) { - switch any(v1).(type) { - case []float32: - _v1 := any(v1).([]float32) - _v2 := any(v2).([]float32) - - diff := blas32.Vector{ - N: len(_v1), - Inc: 1, - Data: make([]float32, len(_v1)), - } - - for i := range _v1 { - diff.Data[i] = _v1[i] - _v2[i] - } - - return T(blas32.Asum(diff)), nil - - case []float64: - _v1 := any(v1).([]float64) - _v2 := any(v2).([]float64) - - diff := blas64.Vector{ - N: len(_v1), - Inc: 1, - Data: make([]float64, len(_v1)), - } - - for i := range _v1 { - diff.Data[i] = _v1[i] - _v2[i] - } - return T(blas64.Asum(diff)), nil - default: - return 0, moerr.NewInternalErrorNoCtx("L1Distance type not supported") - } + return moarray.L1Distance(v1, v2) } func InnerProduct[T types.RealNumbers](v1, v2 []T) (T, error) { - switch any(v1).(type) { - case []float32: - _v1 := blas32.Vector{N: len(v1), Inc: 1, Data: any(v1).([]float32)} - _v2 := blas32.Vector{N: len(v2), Inc: 1, Data: any(v2).([]float32)} - - return T(-blas32.Dot(_v1, _v2)), nil - - case []float64: - _v1 := blas64.Vector{N: len(v1), Inc: 1, Data: any(v1).([]float64)} - _v2 := blas64.Vector{N: len(v2), Inc: 1, Data: any(v2).([]float64)} - return T(-blas64.Dot(_v1, _v2)), nil - default: - return 0, moerr.NewInternalErrorNoCtx("InnerProduct type not supported") - } + return moarray.InnerProduct(v1, v2) } func CosineDistance[T types.RealNumbers](v1, v2 []T) (T, error) { - switch any(v1).(type) { - case []float32: - _v1 := blas32.Vector{N: len(v1), Inc: 1, Data: any(v1).([]float32)} - _v2 := blas32.Vector{N: len(v2), Inc: 1, Data: any(v2).([]float32)} - - mag1 := blas32.Nrm2(_v1) - mag2 := blas32.Nrm2(_v2) - if mag1 == 0 || mag2 == 0 { - return 0, moerr.NewInternalErrorNoCtx("cannot compute cosine similarity with zero vector") - } - score := blas32.Dot(_v1, _v2) / (mag1 * mag2) - return T(1 - score), nil - - case []float64: - _v1 := blas64.Vector{N: len(v1), Inc: 1, Data: any(v1).([]float64)} - _v2 := blas64.Vector{N: len(v2), Inc: 1, Data: any(v2).([]float64)} - mag1 := blas64.Nrm2(_v1) - mag2 := blas64.Nrm2(_v2) - if mag1 == 0 || mag2 == 0 { - return 0, moerr.NewInternalErrorNoCtx("cannot compute cosine similarity with zero vector") - } - score := blas64.Dot(_v1, _v2) / (mag1 * mag2) - return T(1 - score), nil - default: - return 0, moerr.NewInternalErrorNoCtx("CosineDistance type not supported") - } + return moarray.CosineDistance(v1, v2) +} +func CosineSimilarity[T types.RealNumbers](v1, v2 []T) (T, error) { + return moarray.CosineSimilarity(v1, v2) } // SphericalDistance is used for InnerProduct and CosineDistance in Spherical Kmeans. @@ -167,14 +61,17 @@ func SphericalDistance[T types.RealNumbers](v1, v2 []T) (T, error) { switch any(v1).(type) { case []float32: - _v1 := blas32.Vector{N: len(v1), Inc: 1, Data: any(v1).([]float32)} - _v2 := blas32.Vector{N: len(v2), Inc: 1, Data: any(v2).([]float32)} - dp = float64(blas32.Dot(_v1, _v2)) + _v1 := any(v1).([]float32) + _v2 := any(v2).([]float32) + + dp = float64(vek32.Dot(_v1, _v2)) case []float64: - _v1 := blas64.Vector{N: len(v1), Inc: 1, Data: any(v1).([]float64)} - _v2 := blas64.Vector{N: len(v2), Inc: 1, Data: any(v2).([]float64)} - dp = blas64.Dot(_v1, _v2) + _v1 := any(v1).([]float64) + _v2 := any(v2).([]float64) + + dp = vek.Dot(_v1, _v2) + default: return 0, moerr.NewInternalErrorNoCtx("SphericalDistance type not supported") } @@ -193,33 +90,21 @@ func SphericalDistance[T types.RealNumbers](v1, v2 []T) (T, error) { } func NormalizeL2[T types.RealNumbers](v1 []T, normalized []T) error { - - if len(v1) == 0 { - return moerr.NewInternalErrorNoCtx("cannot normalize empty vector") - } - - // Compute the norm of the vector - var sumSquares float64 - for _, val := range v1 { - sumSquares += float64(val) * float64(val) - } - norm := math.Sqrt(sumSquares) - if norm == 0 { - copy(normalized, v1) - return nil - } - - // Divide each element by the norm - for i, val := range v1 { - normalized[i] = T(float64(val) / norm) - } - - return nil + return moarray.NormalizeL2(v1, normalized) } func ScaleInPlace[T types.RealNumbers](v []T, scale T) { - for i := range v { - v[i] *= scale + switch _v := any(v).(type) { + case []float32: + _scale := float32(scale) + vek32.MulNumber_Inplace(_v, _scale) + + case []float64: + _scale := float64(scale) + vek.MulNumber_Inplace(_v, _scale) + + default: + panic("Scale type not supported") } } diff --git a/pkg/vectorindex/metric/distance_func_bench_test.go b/pkg/vectorindex/metric/distance_func_bench_test.go index 506d602d116cd..8040ec0f98193 100644 --- a/pkg/vectorindex/metric/distance_func_bench_test.go +++ b/pkg/vectorindex/metric/distance_func_bench_test.go @@ -25,14 +25,14 @@ Benchmark_L2Distance/Normalize_L2-10 1277733 1 Benchmark_L2Distance/L2_Distance(v1,_NormalizeL2)-10 589376 1883 ns/op */ func Benchmark_L2Distance(b *testing.B) { - dim := 128 + dim := 1024 b.Run("L2 Distance", func(b *testing.B) { v1, v2 := randomVectors(b.N, dim), randomVectors(b.N, dim) b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = L2Distance[float64](v1[i], v2[i]) + for i := range b.N { + _, _ = L2Distance(v1[i], v2[i]) } }) @@ -40,9 +40,9 @@ func Benchmark_L2Distance(b *testing.B) { v1 := randomVectors(b.N, dim) b.ResetTimer() - for i := 0; i < b.N; i++ { - res := make([]float64, dim) - _ = NormalizeL2[float64](v1[i], res) + for i := range b.N { + res := make([]float32, dim) + _ = NormalizeL2(v1[i], res) } }) @@ -50,20 +50,20 @@ func Benchmark_L2Distance(b *testing.B) { v1, v2 := randomVectors(b.N, dim), randomVectors(b.N, dim) b.ResetTimer() - for i := 0; i < b.N; i++ { - res := make([]float64, dim) - _ = NormalizeL2[float64](v2[i], res) - _, _ = L2Distance[float64](v1[i], res) + for i := range b.N { + res := make([]float32, dim) + _ = NormalizeL2(v2[i], res) + _, _ = L2Distance(v1[i], res) } }) } -func randomVectors(size, dim int) [][]float64 { - vectors := make([][]float64, size) +func randomVectors(size, dim int) [][]float32 { + vectors := make([][]float32, size) for i := range vectors { - for j := 0; j < dim; j++ { - vectors[i] = append(vectors[i], rand.Float64()) + for range dim { + vectors[i] = append(vectors[i], rand.Float32()) } } return vectors diff --git a/pkg/vectorindex/metric/distance_func_test.go b/pkg/vectorindex/metric/distance_func_test.go index e7b1c6d76078d..39f8e1e1d2d87 100644 --- a/pkg/vectorindex/metric/distance_func_test.go +++ b/pkg/vectorindex/metric/distance_func_test.go @@ -21,7 +21,6 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/assertx" "github.com/stretchr/testify/require" - "gonum.org/v1/gonum/blas/blas32" ) func Test_MaxFloat(t *testing.T) { @@ -32,22 +31,14 @@ func Test_MaxFloat(t *testing.T) { require.Equal(t, max64, float64(math.MaxFloat64)) } -func Test_Blas32(t *testing.T) { +func Test_Vek(t *testing.T) { - v1 := blas32.Vector{ - N: 3, - Inc: 1, - Data: []float32{4, 5, 6}, - } - v2 := blas32.Vector{ - N: 3, - Inc: 1, - Data: []float32{1, 2, 3}, - } + v1 := []float32{4, 5, 6} + v2 := []float32{1, 2, 3} distfn, _, err := ResolveKmeansDistanceFn[float32](Metric_L2Distance, false) require.Nil(t, err) - v, err := distfn(v1.Data, v2.Data) + v, err := distfn(v1, v2) require.Nil(t, err) fmt.Printf("blas32 v = %v\n", v) @@ -140,17 +131,21 @@ func Test_ResolveFun(t *testing.T) { } func Test_ZeroVector(t *testing.T) { - v1 := []float64{0, 0, 0} v2 := []float64{0, 0, 0} - _, err := CosineDistance[float64](v1, v2) - require.NotNil(t, err) + got64, err := CosineDistance(v1, v2) + require.Nil(t, err) + if !math.IsNaN(got64) { + t.Errorf("CosineDistance(%v, %v) = %v, want NaN", v1, v2, got64) + } v1f32 := []float32{0, 0, 0} v2f32 := []float32{0, 0, 0} - _, err = CosineDistance[float32](v1f32, v2f32) - require.NotNil(t, err) - + got32, err := CosineDistance(v1f32, v2f32) + require.Nil(t, err) + if !math.IsNaN(float64(got32)) { + t.Errorf("CosineDistance(%v, %v) = %v, want NaN", v1f32, v2f32, got32) + } } func Test_L2Distance(t *testing.T) { @@ -206,7 +201,7 @@ func Test_L2Distance(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got, err := L2Distance[float64](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + if got, err := L2Distance(tt.args.v1, tt.args.v2); err != nil || !assertx.InEpsilonF64(got, tt.want) { t.Errorf("L2Distance() = %v, want %v", got, tt.want) } }) @@ -266,7 +261,7 @@ func Test_L1Distance(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got, err := L1Distance[float64](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + if got, err := L1Distance(tt.args.v1, tt.args.v2); err != nil || !assertx.InEpsilonF64(got, tt.want) { t.Errorf("L1Distance() = %v, want %v", got, tt.want) } }) @@ -326,7 +321,7 @@ func Test_CosineDistance(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got, err := CosineDistance[float64](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + if got, err := CosineDistance(tt.args.v1, tt.args.v2); err != nil || !assertx.InEpsilonF64(got, tt.want) { t.Errorf("CosineDistance() = %v, want %v", got, tt.want) } }) @@ -386,7 +381,7 @@ func Test_InnerProduct(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got, err := InnerProduct[float64](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + if got, err := InnerProduct(tt.args.v1, tt.args.v2); err != nil || !assertx.InEpsilonF64(got, tt.want) { t.Errorf("InnerProduct() = %v, want %v", got, tt.want) } }) @@ -446,7 +441,7 @@ func Test_L2DistanceSq(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got, err := L2DistanceSq[float64](tt.args.v1, tt.args.v2); err != nil || got != tt.want { + if got, err := L2DistanceSq(tt.args.v1, tt.args.v2); err != nil || !assertx.InEpsilonF64(got, tt.want) { t.Errorf("L2DistanceSq() = %v, want %v", got, tt.want) } }) @@ -535,7 +530,70 @@ func Test_AngularDistance(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got, err := SphericalDistance[float64](tt.args.v1, tt.args.v2); err != nil || !assertx.InEpsilonF64(got, tt.want) { + if got, err := SphericalDistance(tt.args.v1, tt.args.v2); err != nil || !assertx.InEpsilonF64(got, tt.want) { + t.Errorf("SphericalDistance() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_AngularDistance32(t *testing.T) { + type args struct { + v1 []float32 + v2 []float32 + } + tests := []struct { + name string + args args + want float64 + }{ + { + name: "Test 1", + args: args{ + v1: []float32{1, 2, 3, 4}, + v2: []float32{1, 2, 4, 5}, + }, + want: 0, + }, + { + name: "Test 2", + args: args{ + v1: []float32{10, 20, 30, 40}, + v2: []float32{10.5, 21.5, 31.5, 43.5}, + }, + want: 0, + }, + // Test 3: Triangle Inequality check on **un-normalized** vector + // A(1,0),B(2,2), C(0,1) => AB + AC !>= BC => 0 + 0 !>= 0.5 + { + name: "Test 3.a", + args: args{ + v1: []float32{1, 0}, + v2: []float32{2, 2}, + }, + want: 0, + }, + { + name: "Test 3.b", + args: args{ + v1: []float32{2, 2}, + v2: []float32{0, 1}, + }, + want: 0, + }, + { + name: "Test 3.c", + args: args{ + v1: []float32{0, 1}, + v2: []float32{1, 0}, + }, + want: 0.5, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + if got, err := SphericalDistance(tt.args.v1, tt.args.v2); err != nil || !assertx.InEpsilonF64(float64(got), tt.want) { t.Errorf("SphericalDistance() = %v, want %v", got, tt.want) } }) diff --git a/pkg/vectorize/moarray/external.go b/pkg/vectorize/moarray/external.go index bd50cc078e57e..bc2c37f654fcc 100644 --- a/pkg/vectorize/moarray/external.go +++ b/pkg/vectorize/moarray/external.go @@ -15,14 +15,12 @@ package moarray import ( - "math" + "slices" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/types" - "github.com/matrixorigin/matrixone/pkg/vectorindex/metric" - "github.com/matrixorigin/matrixone/pkg/vectorize/momath" - "gonum.org/v1/gonum/blas/blas32" - "gonum.org/v1/gonum/blas/blas64" + "github.com/viterin/vek" + "github.com/viterin/vek/vek32" ) // These functions are exposed externally via SQL API. @@ -32,28 +30,19 @@ func Add[T types.RealNumbers](v1, v2 []T) ([]T, error) { return nil, moerr.NewArrayInvalidOpNoCtx(len(v1), len(v2)) } - switch any(v1).(type) { + switch _v1 := any(v1).(type) { case []float32: - _v1 := blas32.Vector{N: len(v1), Inc: 1, Data: any(v1).([]float32)} - _v2 := blas32.Vector{N: len(v2), Inc: 1, Data: any(v2).([]float32)} - data := make([]T, len(v1)) - f32 := any(data).([]float32) - ret := blas32.Vector{N: len(v1), Inc: 1, Data: f32} - blas32.Copy(_v1, ret) - blas32.Axpy(1, _v2, ret) - return data, nil + _v2 := any(v2).([]float32) + res := vek32.Add(_v1, _v2) + return any(res).([]T), nil + case []float64: - _v1 := blas64.Vector{N: len(v1), Inc: 1, Data: any(v1).([]float64)} - _v2 := blas64.Vector{N: len(v2), Inc: 1, Data: any(v2).([]float64)} - data := make([]T, len(v1)) - f64 := any(data).([]float64) - ret := blas64.Vector{N: len(v1), Inc: 1, Data: f64} - blas64.Copy(_v1, ret) - blas64.Axpy(1, _v2, ret) - return data, nil + _v2 := any(v2).([]float64) + res := vek.Add(_v1, _v2) + return any(res).([]T), nil + default: panic("Add type not supported") - } } @@ -62,28 +51,19 @@ func Subtract[T types.RealNumbers](v1, v2 []T) ([]T, error) { return nil, moerr.NewArrayInvalidOpNoCtx(len(v1), len(v2)) } - switch any(v1).(type) { + switch _v1 := any(v1).(type) { case []float32: - _v1 := blas32.Vector{N: len(v1), Inc: 1, Data: any(v1).([]float32)} - _v2 := blas32.Vector{N: len(v2), Inc: 1, Data: any(v2).([]float32)} - data := make([]T, len(v1)) - f32 := any(data).([]float32) - ret := blas32.Vector{N: len(v1), Inc: 1, Data: f32} - blas32.Copy(_v1, ret) - blas32.Axpy(-1, _v2, ret) - return data, nil + _v2 := any(v2).([]float32) + res := vek32.Sub(_v1, _v2) + return any(res).([]T), nil + case []float64: - _v1 := blas64.Vector{N: len(v1), Inc: 1, Data: any(v1).([]float64)} - _v2 := blas64.Vector{N: len(v2), Inc: 1, Data: any(v2).([]float64)} - data := make([]T, len(v1)) - f64 := any(data).([]float64) - ret := blas64.Vector{N: len(v1), Inc: 1, Data: f64} - blas64.Copy(_v1, ret) - blas64.Axpy(-1, _v2, ret) - return data, nil + _v2 := any(v2).([]float64) + res := vek.Sub(_v1, _v2) + return any(res).([]T), nil + default: panic("Subtract type not supported") - } } @@ -92,12 +72,20 @@ func Multiply[T types.RealNumbers](v1, v2 []T) ([]T, error) { return nil, moerr.NewArrayInvalidOpNoCtx(len(v1), len(v2)) } - ret := make([]T, len(v1)) + switch _v1 := any(v1).(type) { + case []float32: + _v2 := any(v2).([]float32) + res := vek32.Mul(_v1, _v2) + return any(res).([]T), nil + + case []float64: + _v2 := any(v2).([]float64) + res := vek.Mul(_v1, _v2) + return any(res).([]T), nil - for i := range v1 { - ret[i] = v1[i] * v2[i] + default: + panic("Multiply type not supported") } - return ret, nil } func Divide[T types.RealNumbers](v1, v2 []T) ([]T, error) { @@ -106,17 +94,97 @@ func Divide[T types.RealNumbers](v1, v2 []T) ([]T, error) { } // pre-check for division by zero - for i := 0; i < len(v2); i++ { - if v2[i] == 0 { - return nil, moerr.NewDivByZeroNoCtx() - } + if slices.Contains(v2, 0) { + return nil, moerr.NewDivByZeroNoCtx() + } + + switch _v1 := any(v1).(type) { + case []float32: + _v2 := any(v2).([]float32) + res := vek32.Div(_v1, _v2) + return any(res).([]T), nil + + case []float64: + _v2 := any(v2).([]float64) + res := vek.Div(_v1, _v2) + return any(res).([]T), nil + + default: + panic("Divide type not supported") + } +} + +func AddScalar[T types.RealNumbers](v1 []T, v2 T) ([]T, error) { + switch _v1 := any(v1).(type) { + case []float32: + _v2 := float32(v2) + res := vek32.AddNumber(_v1, _v2) + return any(res).([]T), nil + + case []float64: + _v2 := float64(v2) + res := vek.AddNumber(_v1, _v2) + return any(res).([]T), nil + + default: + panic("AddScalar type not supported") } +} + +func SubtractScalar[T types.RealNumbers](v1 []T, v2 T) ([]T, error) { + switch _v1 := any(v1).(type) { + case []float32: + _v2 := float32(v2) + res := vek32.SubNumber(_v1, _v2) + return any(res).([]T), nil + + case []float64: + _v2 := float64(v2) + res := vek.SubNumber(_v1, _v2) + return any(res).([]T), nil - ret := make([]T, len(v1)) - for i := range v1 { - ret[i] = v1[i] / v2[i] + default: + panic("SubtractScalar type not supported") + } +} + +func MultiplyScalar[T types.RealNumbers](v1 []T, v2 T) ([]T, error) { + switch _v1 := any(v1).(type) { + case []float32: + _v2 := float32(v2) + res := vek32.MulNumber(_v1, _v2) + return any(res).([]T), nil + + case []float64: + _v2 := float64(v2) + res := vek.MulNumber(_v1, _v2) + return any(res).([]T), nil + + default: + panic("MultiplyScalar type not supported") + } +} + +func DivideScalar[T types.RealNumbers](v1 []T, v2 T) ([]T, error) { + // pre-check for division by zero + if v2 == 0 { + return nil, moerr.NewDivByZeroNoCtx() + } + + switch _v1 := any(v1).(type) { + case []float32: + _v2 := float32(v2) + res := vek32.DivNumber(_v1, _v2) + return any(res).([]T), nil + + case []float64: + _v2 := float64(v2) + res := vek.DivNumber(_v1, _v2) + return any(res).([]T), nil + + default: + panic("DivideScalar type not supported") } - return ret, nil } // Compare returns an integer comparing two arrays/vectors lexicographically. @@ -146,235 +214,188 @@ func Compare[T types.RealNumbers](v1, v2 []T) int { /* ------------ [START] Performance critical functions. ------- */ -func InnerProduct[T types.RealNumbers](v1, v2 []T) (float64, error) { - +func InnerProduct[T types.RealNumbers](v1, v2 []T) (T, error) { if len(v1) != len(v2) { return 0, moerr.NewArrayInvalidOpNoCtx(len(v1), len(v2)) } - switch any(v1).(type) { + switch _v1 := any(v1).(type) { case []float32: - _v1 := blas32.Vector{N: len(v1), Inc: 1, Data: any(v1).([]float32)} - _v2 := blas32.Vector{N: len(v2), Inc: 1, Data: any(v2).([]float32)} - return -blas32.DDot(_v1, _v2), nil + _v2 := any(v2).([]float32) + return T(-vek32.Dot(_v1, _v2)), nil + case []float64: - _v1 := blas64.Vector{N: len(v1), Inc: 1, Data: any(v1).([]float64)} - _v2 := blas64.Vector{N: len(v2), Inc: 1, Data: any(v2).([]float64)} - return -blas64.Dot(_v1, _v2), nil + _v2 := any(v2).([]float64) + return T(-vek.Dot(_v1, _v2)), nil + default: panic("InnerProduct type not supported") + } +} +func L1Distance[T types.RealNumbers](v1, v2 []T) (T, error) { + if len(v1) != len(v2) { + return 0, moerr.NewArrayInvalidOpNoCtx(len(v1), len(v2)) + } + + switch _v1 := any(v1).(type) { + case []float32: + _v2 := any(v2).([]float32) + return T(vek32.ManhattanDistance(_v1, _v2)), nil + + case []float64: + _v2 := any(v2).([]float64) + return T(vek.ManhattanDistance(_v1, _v2)), nil + + default: + panic("L1Distance type not supported") } } -func L2Distance[T types.RealNumbers](v1, v2 []T) (float64, error) { +func L2Distance[T types.RealNumbers](v1, v2 []T) (T, error) { if len(v1) != len(v2) { return 0, moerr.NewArrayInvalidOpNoCtx(len(v1), len(v2)) } - ret, err := metric.L2Distance[T](v1, v2) - return float64(ret), err + switch _v1 := any(v1).(type) { + case []float32: + _v2 := any(v2).([]float32) + return T(vek32.Distance(_v1, _v2)), nil + + case []float64: + _v2 := any(v2).([]float64) + return T(vek.Distance(_v1, _v2)), nil + + default: + panic("L2Distance type not supported") + } } // L2DistanceSq returns the squared L2 distance between two vectors. // It is an optimized version of L2Distance used in Index Scan -func L2DistanceSq[T types.RealNumbers](v1, v2 []T) (float64, error) { +func L2DistanceSq[T types.RealNumbers](v1, v2 []T) (T, error) { if len(v1) != len(v2) { return 0, moerr.NewArrayInvalidOpNoCtx(len(v1), len(v2)) } - ret, err := metric.L2DistanceSq[T](v1, v2) - return float64(ret), err + switch _v1 := any(v1).(type) { + case []float32: + _v2 := any(v2).([]float32) + dist := vek32.Distance(_v1, _v2) + return T(dist * dist), nil + + case []float64: + _v2 := any(v2).([]float64) + dist := vek.Distance(_v1, _v2) + return T(dist * dist), nil + + default: + panic("L2DistanceSq type not supported") + } } -func CosineDistance[T types.RealNumbers](v1, v2 []T) (float64, error) { +func CosineDistance[T types.RealNumbers](v1, v2 []T) (T, error) { if len(v1) != len(v2) { return 0, moerr.NewArrayInvalidOpNoCtx(len(v1), len(v2)) } - cosine, err := CosineSimilarity[T](v1, v2) + cosine, err := CosineSimilarity(v1, v2) if err != nil { return 0, err } - return float64(1 - cosine), nil + return 1 - cosine, nil } -func CosineSimilarity[T types.RealNumbers](v1, v2 []T) (float64, error) { +func CosineSimilarity[T types.RealNumbers](v1, v2 []T) (T, error) { if len(v1) != len(v2) { return 0, moerr.NewArrayInvalidOpNoCtx(len(v1), len(v2)) } - var dot, normV1, normV2 float64 - - switch any(v1).(type) { + switch _v1 := any(v1).(type) { case []float32: - _v1 := blas32.Vector{N: len(v1), Inc: 1, Data: any(v1).([]float32)} - _v2 := blas32.Vector{N: len(v2), Inc: 1, Data: any(v2).([]float32)} - - dot = float64(blas32.Dot(_v1, _v2)) - - normV1 = float64(blas32.Nrm2(_v1)) - normV2 = float64(blas32.Nrm2(_v2)) + _v2 := any(v2).([]float32) + return T(vek32.CosineSimilarity(_v1, _v2)), nil case []float64: - _v1 := blas64.Vector{N: len(v1), Inc: 1, Data: any(v1).([]float64)} - _v2 := blas64.Vector{N: len(v2), Inc: 1, Data: any(v2).([]float64)} - - dot = blas64.Dot(_v1, _v2) - - normV1 = blas64.Nrm2(_v1) - normV2 = blas64.Nrm2(_v2) - } - - if normV1 == 0 || normV2 == 0 { - return 0, moerr.NewInternalErrorNoCtx("cosine similarity: one of the vector is zero") - } - - cosine := dot / (normV1 * normV2) - - // handle precision issues. Clamp the cosine simliarity to the range [-1, 1]. - if cosine > 1.0 { - cosine = 1.0 - } else if cosine < -1.0 { - cosine = -1.0 - } - - // NOTE: Downcast the float64 cosine_similarity to float32 and check if it is - // 1.0 or -1.0 to avoid precision issue. - // - // Example for corner case: - // - cosine_similarity(a,a) = 1: - // - Without downcasting check, we get the following results: - // cosine_similarity( [0.46323407, 23.498016, 563.923, 56.076736, 8732.958] , - // [0.46323407, 23.498016, 563.923, 56.076736, 8732.958] ) = 0.9999999999999998 - // - With downcasting, we get the following results: - // cosine_similarity( [0.46323407, 23.498016, 563.923, 56.076736, 8732.958] , - // [0.46323407, 23.498016, 563.923, 56.076736, 8732.958] ) = 1 - // - // Reason: - // The reason for this check is - // 1. gonums mat.Dot, mat.Norm returns float64. In other databases, we mostly do float32 operations. - // 2. float64 operations are not exact. - // mysql> select 76586261.65813679/(8751.35770370157 *8751.35770370157); - //+-----------------------------------------------------------+ - //| 76586261.65813679 / (8751.35770370157 * 8751.35770370157) | - //+-----------------------------------------------------------+ - //| 1.000000000000 | - //+-----------------------------------------------------------+ - //mysql> select cast(76586261.65813679 as double)/(8751.35770370157 * 8751.35770370157); - //+---------------------------------------------------------------------------+ - //| cast(76586261.65813679 as double) / (8751.35770370157 * 8751.35770370157) | - //+---------------------------------------------------------------------------+ - //| 0.9999999999999996 | - //+---------------------------------------------------------------------------+ - // 3. We only need to handle the case for 1.0 and -1.0 with float32 precision. - // Rest of the cases can have float64 precision. - - cosinef32 := float32(cosine) - if cosinef32 == 1 { - cosine = 1 - } else if cosinef32 == -1 { - cosine = -1 - } - - return cosine, nil + _v2 := any(v2).([]float64) + return T(vek.CosineSimilarity(_v1, _v2)), nil + + default: + panic("CosineSimilarity type not supported") + } } func NormalizeL2[T types.RealNumbers](v1 []T, normalized []T) error { - if len(v1) == 0 { return moerr.NewInternalErrorNoCtx("cannot normalize empty vector") } - // Compute the norm of the vector - var sumSquares float64 - for _, val := range v1 { - sumSquares += float64(val) * float64(val) - } - norm := math.Sqrt(sumSquares) - if norm == 0 { - copy(normalized, v1) - return nil - } + switch _v1 := any(v1).(type) { + case []float32: + _normalized := any(normalized).([]float32) + copy(_normalized, _v1) + norm := vek32.Norm(_v1) + if norm > 0 { + vek32.DivNumber_Inplace(_normalized, norm) + } + + case []float64: + _normalized := any(normalized).([]float64) + copy(_normalized, _v1) + norm := vek.Norm(_v1) + if norm > 0 { + vek.DivNumber_Inplace(_normalized, norm) + } - // Divide each element by the norm - for i, val := range v1 { - normalized[i] = T(float64(val) / norm) + default: + panic("NormalizeL2 type not supported") } return nil } // L1Norm returns l1 distance to origin. -func L1Norm[T types.RealNumbers](v []T) (float64, error) { - switch any(v).(type) { +func L1Norm[T types.RealNumbers](v []T) (T, error) { + switch _v := any(v).(type) { case []float32: - _v := blas32.Vector{N: len(v), Inc: 1, Data: any(v).([]float32)} - return float64(blas32.Asum(_v)), nil + return T(vek32.ManhattanNorm(_v)), nil + case []float64: - _v := blas64.Vector{N: len(v), Inc: 1, Data: any(v).([]float64)} - return blas64.Asum(_v), nil + return T(vek.ManhattanNorm(_v)), nil + default: - return 0, moerr.NewInternalErrorNoCtx("L1Norm type not supported") + panic("L1Norm type not supported") } } // L2Norm returns l2 distance to origin. -func L2Norm[T types.RealNumbers](v []T) (float64, error) { - switch any(v).(type) { +func L2Norm[T types.RealNumbers](v []T) (T, error) { + switch _v := any(v).(type) { case []float32: - _v := blas32.Vector{N: len(v), Inc: 1, Data: any(v).([]float32)} - return float64(blas32.Nrm2(_v)), nil + return T(vek32.Norm(_v)), nil + case []float64: - _v := blas64.Vector{N: len(v), Inc: 1, Data: any(v).([]float64)} - return blas64.Nrm2(_v), nil + return T(vek.Norm(_v)), nil + default: - return 0, moerr.NewInternalErrorNoCtx("L2Norm type not supported") + panic("L2Norm type not supported") } } -func ScalarOp[T types.RealNumbers](v []T, operation string, scalar float64) ([]T, error) { - - ret := make([]T, len(v)) - +func ScalarOp[T types.RealNumbers](v []T, operation string, scalar T) ([]T, error) { switch operation { - case "+", "-": - //TODO: optimize this in future. - if operation == "+" { - for i := range v { - ret[i] = v[i] + T(scalar) - } - } else { - for i := range v { - ret[i] = v[i] - T(scalar) - } - } - case "*", "/": - var scale float64 - if operation == "/" { - if scalar == 0 { - return nil, moerr.NewDivByZeroNoCtx() - } - scale = float64(1) / scalar - } else { - scale = scalar - } - - for i := range v { - ret[i] = v[i] * T(scale) - } + case "+": + return AddScalar(v, scalar) + case "-": + return SubtractScalar(v, scalar) + case "*": + return MultiplyScalar(v, scalar) + case "/": + return DivideScalar(v, scalar) default: return nil, moerr.NewInternalErrorNoCtx("scale_vector: invalid operation") } - - // check overflow - for i := range ret { - if math.IsInf(float64(ret[i]), 0) { - return nil, moerr.NewInternalErrorNoCtx("vector contains infinity values") - } - } - return ret, nil } /* ------------ [END] Performance critical functions. ------- */ @@ -382,43 +403,58 @@ func ScalarOp[T types.RealNumbers](v []T, operation string, scalar float64) ([]T /* ------------ [START] mat.VecDense not supported functions ------- */ func Abs[T types.RealNumbers](v []T) (res []T, err error) { - n := len(v) - res = make([]T, n) - for i := 0; i < n; i++ { - res[i], err = momath.AbsSigned[T](v[i]) - if err != nil { - return nil, err - } + switch _v := any(v).(type) { + case []float32: + res = any(vek32.Abs(_v)).([]T) + + case []float64: + res = any(vek.Abs(_v)).([]T) + + default: + panic("Abs type not supported") } - return res, nil + + return } -func Sqrt[T types.RealNumbers](v []T) (res []float64, err error) { - n := len(v) - res = make([]float64, n) - for i := 0; i < n; i++ { - res[i], err = momath.Sqrt(float64(v[i])) - if err != nil { - return nil, err - } +func Sqrt[T types.RealNumbers](v []T) ([]T, error) { + firstNegIdx := slices.IndexFunc(v, func(n T) bool { + return n < 0 + }) + if firstNegIdx != -1 { + return nil, moerr.NewInvalidArgNoCtx("Sqrt", v[firstNegIdx]) + } + + switch _v := any(v).(type) { + case []float32: + return any(vek32.Sqrt(_v)).([]T), nil + + case []float64: + return any(vek.Sqrt(_v)).([]T), nil + + default: + panic("Sqrt type not supported") } - return res, nil } -func Summation[T types.RealNumbers](v []T) (float64, error) { - n := len(v) - var sum float64 = 0 - for i := 0; i < n; i++ { - sum += float64(v[i]) +func Summation[T types.RealNumbers](v []T) (T, error) { + switch _v := any(v).(type) { + case []float32: + return T(vek32.Sum(_v)), nil + + case []float64: + return T(vek.Sum(_v)), nil + + default: + panic("Sum type not supported") } - return sum, nil } func Cast[I types.RealNumbers, O types.RealNumbers](in []I) (out []O, err error) { n := len(in) out = make([]O, n) - for i := 0; i < n; i++ { + for i := range in { out[i] = O(in[i]) } diff --git a/pkg/vectorize/moarray/external_test.go b/pkg/vectorize/moarray/external_test.go index cb3a4a895a74e..e23eb734d1878 100644 --- a/pkg/vectorize/moarray/external_test.go +++ b/pkg/vectorize/moarray/external_test.go @@ -51,7 +51,7 @@ func TestAdd(t *testing.T) { t.Run(tt.name, func(t *testing.T) { if tt.args.rightArgF32 != nil { - if gotRes, err := Add[float32](tt.args.leftArgF32, tt.args.rightArgF32); err != nil || !reflect.DeepEqual(gotRes, tt.wantF32) { + if gotRes, err := Add[float32](tt.args.leftArgF32, tt.args.rightArgF32); err != nil || !assertx.InEpsilonF32Slice(gotRes, tt.wantF32) { t.Errorf("Add() = %v, want %v", gotRes, tt.wantF32) } } @@ -94,7 +94,7 @@ func TestSubtract(t *testing.T) { t.Run(tt.name, func(t *testing.T) { if tt.args.rightArgF32 != nil { - if gotRes, err := Subtract[float32](tt.args.leftArgF32, tt.args.rightArgF32); err != nil || !reflect.DeepEqual(gotRes, tt.wantF32) { + if gotRes, err := Subtract[float32](tt.args.leftArgF32, tt.args.rightArgF32); err != nil || !assertx.InEpsilonF32Slice(tt.wantF32, gotRes) { t.Errorf("Subtract() = %v, want %v", gotRes, tt.wantF32) } } @@ -142,7 +142,7 @@ func TestMultiply(t *testing.T) { t.Run(tt.name, func(t *testing.T) { if tt.args.rightArgF32 != nil { - if gotRes, err := Multiply[float32](tt.args.leftArgF32, tt.args.rightArgF32); err != nil || !reflect.DeepEqual(tt.wantF32, gotRes) { + if gotRes, err := Multiply[float32](tt.args.leftArgF32, tt.args.rightArgF32); err != nil || !assertx.InEpsilonF32Slice(tt.wantF32, gotRes) { t.Errorf("Multiply() = %v, want %v", gotRes, tt.wantF32) } } @@ -205,7 +205,7 @@ func TestDivide(t *testing.T) { if _, gotErr := Divide[float32](tt.args.leftArgF32, tt.args.rightArgF32); gotErr == nil { t.Errorf("Divide() should throw error") } - } else if gotRes, err := Divide[float32](tt.args.leftArgF32, tt.args.rightArgF32); err != nil || !reflect.DeepEqual(gotRes, tt.wantF32) { + } else if gotRes, err := Divide[float32](tt.args.leftArgF32, tt.args.rightArgF32); err != nil || !assertx.InEpsilonF32Slice(tt.wantF32, gotRes) { t.Errorf("Divide() = %v, want %v", gotRes, tt.wantF32) } } @@ -327,7 +327,7 @@ func TestCast(t *testing.T) { } } if tt.args.argF64 != nil && tt.wantF32 != nil { - if gotResF32, err := Cast[float64, float32](tt.args.argF64); err != nil || !reflect.DeepEqual(gotResF32, tt.wantF32) { + if gotResF32, err := Cast[float64, float32](tt.args.argF64); err != nil || !assertx.InEpsilonF32Slice(gotResF32, tt.wantF32) { t.Errorf("Cast() = %v, want %v", gotResF32, tt.wantF32) } } @@ -363,7 +363,7 @@ func TestAbs(t *testing.T) { t.Run(tt.name, func(t *testing.T) { if tt.args.argF32 != nil { - if gotRes, err := Abs[float32](tt.args.argF32); err != nil || !reflect.DeepEqual(gotRes, tt.wantF32) { + if gotRes, err := Abs[float32](tt.args.argF32); err != nil || !assertx.InEpsilonF32Slice(gotRes, tt.wantF32) { t.Errorf("Abs() = %v, want %v", gotRes, tt.wantF32) } } @@ -440,7 +440,7 @@ func TestNormalizeL2(t *testing.T) { if err := NormalizeL2[float32](tt.args.argF32, res); err == nil { t.Errorf("NormalizeL2() should throw error") } - } else if err := NormalizeL2[float32](tt.args.argF32, res); err != nil || !reflect.DeepEqual(tt.wantF32, res) { + } else if err := NormalizeL2[float32](tt.args.argF32, res); err != nil || !assertx.InEpsilonF32Slice(tt.wantF32, res) { t.Errorf("NormalizeL2() = %v, want %v", res, tt.wantF32) } } @@ -467,7 +467,7 @@ func TestSqrt(t *testing.T) { name string args args - wantF32 []float64 // ie result for argF32 is []float32 + wantF32 []float32 // ie result for argF32 is []float32 wantF64 []float64 // ie result for argF64 is []float64 wantErr bool } @@ -475,7 +475,7 @@ func TestSqrt(t *testing.T) { { name: "Test1 - float32", args: args{argF32: []float32{1, 0, 4}}, - wantF32: []float64{1, 0, 2}, + wantF32: []float32{1, 0, 2}, }, { name: "Test2 - float32 error case", @@ -501,7 +501,7 @@ func TestSqrt(t *testing.T) { if _, err := Sqrt[float32](tt.args.argF32); err == nil { t.Errorf("Sqrt() should throw error") } - } else if gotRes, err := Sqrt[float32](tt.args.argF32); err != nil || !reflect.DeepEqual(gotRes, tt.wantF32) { + } else if gotRes, err := Sqrt[float32](tt.args.argF32); err != nil || !assertx.InEpsilonF32Slice(gotRes, tt.wantF32) { t.Errorf("Sqrt() = %v, want %v", err, tt.wantErr) t.Errorf("Sqrt() = %v, want %v", gotRes, tt.wantF32) } @@ -546,7 +546,7 @@ func TestSummation(t *testing.T) { t.Run(tt.name, func(t *testing.T) { if tt.args.argF32 != nil { - if gotRes, err := Summation[float32](tt.args.argF32); err != nil || !assertx.InEpsilonF64(tt.want, gotRes) { + if gotRes, err := Summation[float32](tt.args.argF32); err != nil || !assertx.InEpsilonF64(float64(gotRes), tt.want) { t.Errorf("Summation() = %v, want %v", gotRes, tt.want) } } @@ -589,7 +589,7 @@ func TestInnerProduct(t *testing.T) { t.Run(tt.name, func(t *testing.T) { if tt.args.argLeftF32 != nil { - if gotRes, _ := InnerProduct[float32](tt.args.argLeftF32, tt.args.argRightF32); !assertx.InEpsilonF64(tt.want, gotRes) { + if gotRes, _ := InnerProduct[float32](tt.args.argLeftF32, tt.args.argRightF32); !assertx.InEpsilonF64(tt.want, float64(gotRes)) { t.Errorf("InnerProduct() = %v, want %v", gotRes, tt.want) } } @@ -629,7 +629,7 @@ func TestL1Norm(t *testing.T) { t.Run(tt.name, func(t *testing.T) { if tt.args.argF32 != nil { - if gotRes, _ := L1Norm[float32](tt.args.argF32); !assertx.InEpsilonF64(tt.want, gotRes) { + if gotRes, _ := L1Norm[float32](tt.args.argF32); !assertx.InEpsilonF64(tt.want, float64(gotRes)) { t.Errorf("L1Norm() = %v, want %v", gotRes, tt.want) } } @@ -669,7 +669,7 @@ func TestL2Norm(t *testing.T) { t.Run(tt.name, func(t *testing.T) { if tt.args.argF32 != nil { - if gotRes, _ := L2Norm[float32](tt.args.argF32); !assertx.InEpsilonF64(tt.want, gotRes) { + if gotRes, _ := L2Norm[float32](tt.args.argF32); !assertx.InEpsilonF64(tt.want, float64(gotRes)) { t.Errorf("L2Norm() = %v, want %v", gotRes, tt.want) } } @@ -747,12 +747,12 @@ func TestCosineSimilarity(t *testing.T) { t.Run(tt.name, func(t *testing.T) { if tt.args.argLeftF32 != nil { - if gotRes, _ := CosineSimilarity[float32](tt.args.argLeftF32, tt.args.argRightF32); !reflect.DeepEqual(tt.want, gotRes) { + if gotRes, _ := CosineSimilarity[float32](tt.args.argLeftF32, tt.args.argRightF32); !assertx.InEpsilonF64(tt.want, float64(gotRes)) { t.Errorf("CosineSimilarity() = %v, want %v", gotRes, tt.want) } } if tt.args.argLeftF64 != nil { - if gotRes, _ := CosineSimilarity[float64](tt.args.argLeftF64, tt.args.argRightF64); !reflect.DeepEqual(tt.want, gotRes) { + if gotRes, _ := CosineSimilarity[float64](tt.args.argLeftF64, tt.args.argRightF64); !assertx.InEpsilonF64(tt.want, gotRes) { t.Errorf("CosineSimilarity() = %v, want %v", gotRes, tt.want) } } @@ -790,7 +790,7 @@ func TestL2Distance(t *testing.T) { t.Run(tt.name, func(t *testing.T) { if tt.args.argLeftF32 != nil { - if gotRes, _ := L2Distance[float32](tt.args.argLeftF32, tt.args.argRightF32); !assertx.InEpsilonF64(tt.want, gotRes) { + if gotRes, _ := L2Distance[float32](tt.args.argLeftF32, tt.args.argRightF32); !assertx.InEpsilonF64(tt.want, float64(gotRes)) { t.Errorf("L2Distance() = %v, want %v", gotRes, tt.want) } } @@ -833,7 +833,7 @@ func TestCosineDistance(t *testing.T) { t.Run(tt.name, func(t *testing.T) { if tt.args.argLeftF32 != nil { - if gotRes, _ := CosineDistance[float32](tt.args.argLeftF32, tt.args.argRightF32); !assertx.InEpsilonF64(tt.want, gotRes) { + if gotRes, _ := CosineDistance[float32](tt.args.argLeftF32, tt.args.argRightF32); !assertx.InEpsilonF64(tt.want, float64(gotRes)) { t.Errorf("CosineDistance() = %v, want %v", gotRes, tt.want) } } @@ -907,11 +907,11 @@ func TestScalarOp(t *testing.T) { t.Run(tt.name, func(t *testing.T) { if tt.args.argVecF32 != nil { - if gotRes, _ := ScalarOp[float32](tt.args.argVecF32, tt.args.argOp, tt.args.argSca); !reflect.DeepEqual(gotRes, tt.wantVecF32) { + if gotRes, _ := ScalarOp[float32](tt.args.argVecF32, tt.args.argOp, float32(tt.args.argSca)); !assertx.InEpsilonF32Slice(tt.wantVecF32, gotRes) { t.Errorf("ScalarOp() = %v, want %v", gotRes, tt.wantVecF32) } } else if tt.args.argVecF64 != nil { - if gotRes, _ := ScalarOp[float64](tt.args.argVecF64, tt.args.argOp, tt.args.argSca); !reflect.DeepEqual(gotRes, tt.wantVecF64) { + if gotRes, _ := ScalarOp[float64](tt.args.argVecF64, tt.args.argOp, tt.args.argSca); !assertx.InEpsilonF64Slice(tt.wantVecF64, gotRes) { t.Errorf("ScalarOp() = %v, want %v", gotRes, tt.wantVecF64) } } diff --git a/pkg/vectorize/momath/math.go b/pkg/vectorize/momath/math.go index 62c92925571bd..659b3b9c9ddb8 100644 --- a/pkg/vectorize/momath/math.go +++ b/pkg/vectorize/momath/math.go @@ -17,9 +17,8 @@ package momath import ( "math" - "golang.org/x/exp/constraints" - "github.com/matrixorigin/matrixone/pkg/common/moerr" + "golang.org/x/exp/constraints" ) func Acos(v float64) (float64, error) { diff --git a/pkg/vm/engine/readutil/reader.go b/pkg/vm/engine/readutil/reader.go index f16e39324a2a4..54831b795b338 100644 --- a/pkg/vm/engine/readutil/reader.go +++ b/pkg/vm/engine/readutil/reader.go @@ -479,9 +479,16 @@ func (r *reader) SetIndexParam(param *plan.IndexReaderParam) { if param.DistRange != nil { r.orderByLimit.LowerBoundType = param.DistRange.LowerBoundType - r.orderByLimit.LowerBound = param.DistRange.LowerBound.GetLit().GetDval() r.orderByLimit.UpperBoundType = param.DistRange.UpperBoundType - r.orderByLimit.UpperBound = param.DistRange.UpperBound.GetLit().GetDval() + + switch r.orderByLimit.Typ { + case types.T_array_float32: + r.orderByLimit.LowerBound = float64(param.DistRange.LowerBound.GetLit().GetFval()) + r.orderByLimit.UpperBound = float64(param.DistRange.UpperBound.GetLit().GetFval()) + case types.T_array_float64: + r.orderByLimit.LowerBound = param.DistRange.LowerBound.GetLit().GetDval() + r.orderByLimit.UpperBound = param.DistRange.UpperBound.GetLit().GetDval() + } if param.OrigFuncName == metric.DistFn_L2Distance { r.orderByLimit.LowerBound *= r.orderByLimit.LowerBound diff --git a/pkg/vm/engine/tae/compute/compare.go b/pkg/vm/engine/tae/compute/compare.go index b9e8c5c1d0eb3..090b4ffb68272 100644 --- a/pkg/vm/engine/tae/compute/compare.go +++ b/pkg/vm/engine/tae/compute/compare.go @@ -18,10 +18,10 @@ import ( "bytes" "cmp" "fmt" + "slices" "unsafe" "github.com/matrixorigin/matrixone/pkg/container/types" - "github.com/matrixorigin/matrixone/pkg/vectorize/moarray" ) func CompareOrdered[T types.OrderedT](a, b T) int { @@ -42,7 +42,7 @@ func CompareBytes(a, b []byte) int { } func compareArrayFromBytes[T types.RealNumbers](a, b []byte) int { - return moarray.Compare[T](types.BytesToArray[T](a), types.BytesToArray[T](b)) + return slices.Compare(types.BytesToArray[T](a), types.BytesToArray[T](b)) } func Compare(a, b []byte, t types.T, scale1, scale2 int32) int { diff --git a/pkg/vm/engine/tae/index/zm.go b/pkg/vm/engine/tae/index/zm.go index fd56a8bac0645..b911ce0219fbb 100644 --- a/pkg/vm/engine/tae/index/zm.go +++ b/pkg/vm/engine/tae/index/zm.go @@ -19,6 +19,7 @@ import ( "encoding/hex" "fmt" "math" + "slices" "sort" "strconv" "strings" @@ -28,7 +29,6 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/container/vector" - "github.com/matrixorigin/matrixone/pkg/vectorize/moarray" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/compute" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" ) @@ -970,10 +970,10 @@ func (zm ZM) SubVecIn(vec *vector.Vector) (int, int) { col := vector.MustArrayCol[float32](vec) minVal, maxVal := types.BytesToArray[float32](zm.GetMinBuf()), types.BytesToArray[float32](zm.GetMaxBuf()) lowerBound := sort.Search(len(col), func(i int) bool { - return moarray.Compare[float32](minVal, col[i]) <= 0 + return slices.Compare(minVal, col[i]) <= 0 }) upperBound := sort.Search(len(col), func(i int) bool { - return moarray.Compare[float32](maxVal, col[i]) < 0 + return slices.Compare(maxVal, col[i]) < 0 }) return lowerBound, upperBound @@ -981,10 +981,10 @@ func (zm ZM) SubVecIn(vec *vector.Vector) (int, int) { col := vector.MustArrayCol[float64](vec) minVal, maxVal := types.BytesToArray[float64](zm.GetMinBuf()), types.BytesToArray[float64](zm.GetMaxBuf()) lowerBound := sort.Search(len(col), func(i int) bool { - return moarray.Compare[float64](minVal, col[i]) <= 0 + return slices.Compare(minVal, col[i]) <= 0 }) upperBound := sort.Search(len(col), func(i int) bool { - return moarray.Compare[float64](maxVal, col[i]) < 0 + return slices.Compare(maxVal, col[i]) < 0 }) return lowerBound, upperBound @@ -1206,19 +1206,19 @@ func (zm ZM) AnyIn(vec *vector.Vector) bool { col := vector.MustArrayCol[float32](vec) minVal, maxVal := types.BytesToArray[float32](zm.GetMinBuf()), types.BytesToArray[float32](zm.GetMaxBuf()) lowerBound := sort.Search(len(col), func(i int) bool { - return moarray.Compare[float32](minVal, col[i]) <= 0 + return slices.Compare(minVal, col[i]) <= 0 }) - return lowerBound < len(col) && moarray.Compare[float32](maxVal, col[lowerBound]) >= 0 + return lowerBound < len(col) && slices.Compare(maxVal, col[lowerBound]) >= 0 case types.T_array_float64: col := vector.MustArrayCol[float64](vec) minVal, maxVal := types.BytesToArray[float64](zm.GetMinBuf()), types.BytesToArray[float64](zm.GetMaxBuf()) lowerBound := sort.Search(len(col), func(i int) bool { - return moarray.Compare[float64](minVal, col[i]) <= 0 + return slices.Compare(minVal, col[i]) <= 0 }) - return lowerBound < len(col) && moarray.Compare[float64](maxVal, col[lowerBound]) >= 0 + return lowerBound < len(col) && slices.Compare(maxVal, col[lowerBound]) >= 0 default: return true diff --git a/test/distributed/cases/array/array.result b/test/distributed/cases/array/array.result index 24f96fb2b8d8d..325afeb12d16d 100644 --- a/test/distributed/cases/array/array.result +++ b/test/distributed/cases/array/array.result @@ -5,9 +5,9 @@ drop table if exists vec_table; create table vec_table(a int, b vecf32(3), c vecf64(3)); desc vec_table; Field Type Null Key Default Extra Comment -a INT(32) YES null -b VECF32(3) YES null -c VECF64(3) YES null +a INT(32) YES null +b VECF32(3) YES null +c VECF64(3) YES null insert into vec_table values(1, "[1,2,3]", "[4,5,6]"); select * from vec_table; a b c @@ -65,7 +65,7 @@ b + [1,2,3] [2, 4, 6] select b + sqrt(b) from vec_table; b + sqrt(b) -[2, 3.414213562373095, 4.732050807568877] +[2, 3.4142137, 4.732051] select b + c from vec_table; b + c [5, 7, 9] @@ -77,7 +77,7 @@ abs(cast([-1,-2,3] as vecf32(3))) [1, 2, 3] select sqrt(b) from vec_table; sqrt(b) -[1, 1.4142135623730951, 1.7320508075688772] +[1, 1.4142135, 1.7320508] select summation(b) from vec_table; summation(b) 6.0 @@ -104,7 +104,7 @@ cosine_distance(b, [1,2,3]) 0.0 select normalize_l2(b) from vec_table; normalize_l2(b) -[0.26726124, 0.5345225, 0.80178374] +[0.26726124, 0.5345225, 0.8017837] select * FROM vec_table ORDER BY cosine_similarity(b, '[3,1,2]') LIMIT 5; a b c 1 [1, 2, 3] [4, 5, 6] @@ -306,8 +306,8 @@ INSERT INTO `t8` VALUES(4,'[8.560689, 6.790359, 821.9778]','[0.46323407, 23.4980 select cosine_similarity(b,b), cosine_similarity(c,c) from t8; cosine_similarity(b, b) cosine_similarity(c, c) null null -1.0 null -0.9999998589068639 0.9999999473078075 +0.99999994 null +1.0 0.99999994 1.0 1.0 create table t9(a int, b vecf64(3), c vecf64(5)); INSERT INTO `t9` VALUES (1,NULL,NULL); @@ -542,8 +542,6 @@ select * from t8 except select * from t9; a b c 3 [0.1726299, 3.2908857, 30.433094] [0.45052445, 2.1984527, 9.579752, 123.48039, 4635.894] 4 [8.560689, 6.790359, 821.9778] [0.46323407, 23.498016, 563.923, 56.076736, 8732.958] -select cast("[76875768584509877574546435800000005,8955885757767774774774774456466]" as vecf32(2)) *623585864455; -internal error: vector contains infinity values select l2_distance("[0, 0, 0]", "[3, 4, 0]"), l2_distance_sq("[0, 0, 0]", "[3, 4, 0]"); l2_distance([0, 0, 0], [3, 4, 0]) l2_distance_sq([0, 0, 0], [3, 4, 0]) 5.0 25.0 diff --git a/test/distributed/cases/array/array.sql b/test/distributed/cases/array/array.sql index 926ebf3fe289d..dba50631e6271 100644 --- a/test/distributed/cases/array/array.sql +++ b/test/distributed/cases/array/array.sql @@ -246,9 +246,6 @@ select c2+c3 from t1; -- Except select * from t8 except select * from t9; --- infinity scenario -select cast("[76875768584509877574546435800000005,8955885757767774774774774456466]" as vecf32(2)) *623585864455; - -- l2_distance_sq select l2_distance("[0, 0, 0]", "[3, 4, 0]"), l2_distance_sq("[0, 0, 0]", "[3, 4, 0]"); @@ -259,4 +256,4 @@ set @a="[1,2,3]"; execute s1 using @a; -- post -drop database vecdb; \ No newline at end of file +drop database vecdb; diff --git a/test/distributed/cases/pessimistic_transaction/vector/vector_hnsw.result b/test/distributed/cases/pessimistic_transaction/vector/vector_hnsw.result index ecbf27d7cc0e8..91c6af3ad7449 100644 --- a/test/distributed/cases/pessimistic_transaction/vector/vector_hnsw.result +++ b/test/distributed/cases/pessimistic_transaction/vector/vector_hnsw.result @@ -13,9 +13,9 @@ Table Create Table vector_index_01 CREATE TABLE `vector_index_01` (\n `a` bigint NOT NULL,\n `b` vecf32(128) DEFAULT NULL,\n `c` int DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `c_k` (`c`),\n KEY `idx01` USING hnsw (`b`) m = 48 ef_construction = 64 ef_search = 64 op_type 'vector_l2_ops' \n) desc vector_index_01; Field Type Null Key Default Extra Comment -a BIGINT(64) NO PRI null -b VECF32(128) YES MUL null -c INT(32) YES MUL null +a BIGINT(64) NO PRI null +b VECF32(128) YES MUL null +c INT(32) YES MUL null drop table vector_index_01; create table vector_index_02(a bigint primary key, b vecf32(128),c int,key c_k(c)); insert into vector_index_02 values(9774 ,"[1, 0, 1, 6, 6, 17, 47, 39, 2, 0, 1, 25, 27, 10, 56, 130, 18, 5, 2, 6, 15, 2, 19, 130, 42, 28, 1, 1, 2, 1, 0, 5, 0, 2, 4, 4, 31, 34, 44, 35, 9, 3, 8, 11, 33, 12, 61, 130, 130, 17, 0, 1, 6, 2, 9, 130, 111, 36, 0, 0, 11, 9, 1, 12, 2, 100, 130, 28, 7, 2, 6, 7, 9, 27, 130, 83, 5, 0, 1, 18, 130, 130, 84, 9, 0, 0, 2, 24, 111, 24, 0, 1, 37, 24, 2, 10, 12, 62, 33, 3, 0, 0, 0, 1, 3, 16, 106, 28, 0, 0, 0, 0, 17, 46, 85, 10, 0, 0, 1, 4, 11, 4, 2, 2, 9, 14, 8, 8]",3),(9775,"[0, 1, 1, 3, 0, 3, 46, 20, 1, 4, 17, 9, 1, 17, 108, 15, 0, 3, 37, 17, 6, 15, 116, 16, 6, 1, 4, 7, 7, 7, 9, 6, 0, 8, 10, 4, 26, 129, 27, 9, 0, 0, 5, 2, 11, 129, 129, 12, 103, 4, 0, 0, 2, 31, 129, 129, 94, 4, 0, 0, 0, 3, 13, 42, 0, 15, 38, 2, 70, 129, 1, 0, 5, 10, 40, 12, 74, 129, 6, 1, 129, 39, 6, 1, 2, 22, 9, 33, 122, 13, 0, 0, 0, 0, 5, 23, 4, 11, 9, 12, 45, 38, 1, 0, 0, 4, 36, 38, 57, 32, 0, 0, 82, 22, 9, 5, 13, 11, 3, 94, 35, 3, 0, 0, 0, 1, 16, 97]",5),(9776,"[10, 3, 8, 5, 48, 26, 5, 16, 17, 0, 0, 2, 132, 53, 1, 16, 112, 6, 0, 0, 7, 2, 1, 48, 48, 15, 18, 31, 3, 0, 0, 9, 6, 10, 19, 27, 50, 46, 17, 9, 18, 1, 4, 48, 132, 23, 3, 5, 132, 9, 4, 3, 11, 0, 2, 46, 84, 12, 10, 10, 1, 0, 12, 76, 26, 22, 16, 26, 35, 15, 3, 16, 15, 1, 51, 132, 125, 8, 1, 2, 132, 51, 67, 91, 8, 0, 0, 30, 126, 39, 32, 38, 4, 0, 1, 12, 24, 2, 2, 2, 4, 7, 2, 19, 93, 19, 70, 92, 2, 3, 1, 21, 36, 58, 132, 94, 0, 0, 0, 0, 21, 25, 57, 48, 1, 0, 0, 1]",3); @@ -26,9 +26,9 @@ Table Create Table vector_index_02 CREATE TABLE `vector_index_02` (\n `a` bigint NOT NULL,\n `b` vecf32(128) DEFAULT NULL,\n `c` int DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `c_k` (`c`),\n KEY `idx01` USING hnsw (`b`) op_type 'vector_l2_ops' \n) desc vector_index_02; Field Type Null Key Default Extra Comment -a BIGINT(64) NO PRI null -b VECF32(128) YES MUL null -c INT(32) YES MUL null +a BIGINT(64) NO PRI null +b VECF32(128) YES MUL null +c INT(32) YES MUL null select * from vector_index_02 order by L2_DISTANCE(b, "[16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]") ASC LIMIT 2; a b c 9777 [16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13] 4 @@ -68,9 +68,9 @@ Table Create Table vector_index_04 CREATE TABLE `vector_index_04` (\n `a` bigint NOT NULL,\n `b` vecf32(3) DEFAULT NULL,\n `c` vecf32(4) DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `idx01` USING hnsw (`b`) op_type 'vector_l2_ops' ,\n KEY `idx02` USING hnsw (`c`) op_type 'vector_l2_ops' \n) desc vector_index_04; Field Type Null Key Default Extra Comment -a BIGINT(64) NO PRI null -b VECF32(3) YES MUL null -c VECF32(4) YES MUL null +a BIGINT(64) NO PRI null +b VECF32(3) YES MUL null +c VECF32(4) YES MUL null insert into vector_index_04 values(4,"[156,213,61]","[10.25,0.14,0.88,10.0001]"),(5,"[177,425,30]","[11.25,51.25,80.699,44.25]"),(6,"[80,56,3]","[90.686,5.212,19.22,7.02]"); insert into vector_index_04 values(7,"[5.6,2.13,6.1]","[1.25,8.14,4.88,18.0001]"),(8,"[17.7,4.25,3.0]","[1.215,5.25,8.9,47.25]"),(9,"[8,5.6,13]","[9.6,57.22,19.2,78.02]"); alter table vector_index_04 alter reindex idx01 hnsw; @@ -80,9 +80,9 @@ Table Create Table vector_index_04 CREATE TABLE `vector_index_04` (\n `a` bigint NOT NULL,\n `b` vecf32(3) DEFAULT NULL,\n `c` vecf32(4) DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `idx01` USING hnsw (`b`) op_type 'vector_l2_ops' ,\n KEY `idx02` USING hnsw (`c`) op_type 'vector_l2_ops' \n) desc vector_index_04; Field Type Null Key Default Extra Comment -a BIGINT(64) NO PRI null -b VECF32(3) YES MUL null -c VECF32(4) YES MUL null +a BIGINT(64) NO PRI null +b VECF32(3) YES MUL null +c VECF32(4) YES MUL null select * from vector_index_04 order by L2_DISTANCE(c,"[9.6,57.22,19.2,78.02]") limit 3; a b c 9 [8, 5.6, 13] [9.6, 57.22, 19.2, 78.02] @@ -266,17 +266,17 @@ Table Create Table vector_ip_01 CREATE TABLE `vector_ip_01` (\n `a` bigint NOT NULL,\n `b` vecf32(128) DEFAULT NULL,\n `c` int DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `c_k` (`c`),\n KEY `idx01` USING hnsw (`b`) op_type 'vector_ip_ops' \n) desc vector_ip_01; Field Type Null Key Default Extra Comment -a BIGINT(64) NO PRI null -b VECF32(128) YES MUL null -c INT(32) YES MUL null +a BIGINT(64) NO PRI null +b VECF32(128) YES MUL null +c INT(32) YES MUL null select * from vector_ip_01 order by inner_product(b, normalize_l2("[1, 0, 1, 6, 6, 17, 47, 39, 2, 0, 1, 25, 27, 10, 56, 130, 18, 5, 2, 6, 15, 2, 19, 130, 42, 28, 1, 1, 2, 1, 0, 5, 0, 2, 4, 4, 31, 34, 44, 35, 9, 3, 8, 11, 33, 12, 61, 130, 130, 17, 0, 1, 6, 2, 9, 130, 111, 36, 0, 0, 11, 9, 1, 12, 2, 100, 130, 28, 7, 2, 6, 7, 9, 27, 130, 83, 5, 0, 1, 18, 130, 130, 84, 9, 0, 0, 2, 24, 111, 24, 0, 1, 37, 24, 2, 10, 12, 62, 33, 3, 0, 0, 0, 1, 3, 16, 106, 28, 0, 0, 0, 0, 17, 46, 85, 10, 0, 0, 1, 4, 11, 4, 2, 2, 9, 14, 8, 8]")) ASC LIMIT 2; a b c -9774 [0.0019665654, 0, 0.0019665654, 0.011799393, 0.011799393, 0.033431612, 0.09242858, 0.07669605, 0.0039331308, 0, 0.0019665654, 0.04916414, 0.053097267, 0.019665655, 0.110127665, 0.2556535, 0.035398178, 0.009832827, 0.0039331308, 0.011799393, 0.029498482, 0.0039331308, 0.037364744, 0.2556535, 0.08259575, 0.055063833, 0.0019665654, 0.0019665654, 0.0039331308, 0.0019665654, 0, 0.009832827, 0, 0.0039331308, 0.0078662615, 0.0078662615, 0.06096353, 0.066863224, 0.08652888, 0.06882979, 0.017699089, 0.0058996966, 0.015732523, 0.02163222, 0.06489666, 0.023598786, 0.119960494, 0.2556535, 0.2556535, 0.033431612, 0, 0.0019665654, 0.011799393, 0.0039331308, 0.017699089, 0.2556535, 0.21828876, 0.070796356, 0, 0, 0.02163222, 0.017699089, 0.0019665654, 0.023598786, 0.0039331308, 0.19665655, 0.2556535, 0.055063833, 0.013765958, 0.0039331308, 0.011799393, 0.013765958, 0.017699089, 0.053097267, 0.2556535, 0.16322494, 0.009832827, 0, 0.0019665654, 0.035398178, 0.2556535, 0.2556535, 0.1651915, 0.017699089, 0, 0, 0.0039331308, 0.047197573, 0.21828876, 0.047197573, 0, 0.0019665654, 0.07276292, 0.047197573, 0.0039331308, 0.019665655, 0.023598786, 0.12192706, 0.06489666, 0.0058996966, 0, 0, 0, 0.0019665654, 0.0058996966, 0.031465046, 0.20845594, 0.055063833, 0, 0, 0, 0, 0.033431612, 0.090462014, 0.16715807, 0.019665655, 0, 0, 0.0019665654, 0.0078662615, 0.02163222, 0.0078662615, 0.0039331308, 0.0039331308, 0.017699089, 0.027531916, 0.015732523, 0.015732523] 3 -9778 [0.08063698, 0, 0, 0.01376729, 0.0019667556, 0.0019667556, 0.039335113, 0.13177262, 0.0177008, 0, 0, 0, 0, 0.060969424, 0.23601067, 0.119972095, 0.049168892, 0, 0, 0, 0, 0.019667557, 0.23601067, 0.177008, 0.06293618, 0, 0, 0.0019667556, 0.025567824, 0.021634312, 0.043268625, 0.098337784, 0.007867022, 0, 0.003933511, 0.18290828, 0.078670226, 0.029501334, 0.07276996, 0.0354016, 0.023601068, 0.003933511, 0.003933511, 0.037368357, 0.015734045, 0.08653725, 0.23601067, 0.049168892, 0.23601067, 0.009833778, 0, 0, 0, 0.003933511, 0.09440427, 0.1907753, 0.20060907, 0.02753458, 0.005900267, 0.005900267, 0.021634312, 0.0177008, 0.06686969, 0.08063698, 0, 0, 0.007867022, 0.23601067, 0.11013832, 0.005900267, 0.007867022, 0.009833778, 0.011800534, 0.029501334, 0.07276996, 0.22814366, 0.05506916, 0, 0, 0.005900267, 0.23601067, 0.23601067, 0.047202136, 0.011800534, 0.003933511, 0, 0.0019667556, 0.05506916, 0.10423805, 0.177008, 0.10030454, 0.021634312, 0.021634312, 0.003933511, 0.023601068, 0.02753458, 0.015734045, 0.011800534, 0.007867022, 0.059002668, 0.0177008, 0.0019667556, 0.007867022, 0.043268625, 0.049168892, 0.15537369, 0.23601067, 0.12980588, 0.009833778, 0, 0, 0.011800534, 0.08260374, 0.23601067, 0.17897476, 0.08457049, 0.029501334, 0.003933511, 0.007867022, 0.076703474, 0.023601068, 0.0177008, 0.0177008, 0.023601068, 0.029501334, 0.009833778, 0.047202136, 0.0708032] 4 +9774 [0.0019665654, 0, 0.0019665654, 0.011799392, 0.011799392, 0.033431612, 0.09242857, 0.07669605, 0.0039331308, 0, 0.0019665654, 0.049164135, 0.053097267, 0.019665655, 0.11012766, 0.2556535, 0.035398178, 0.009832827, 0.0039331308, 0.011799392, 0.02949848, 0.0039331308, 0.037364744, 0.2556535, 0.08259574, 0.05506383, 0.0019665654, 0.0019665654, 0.0039331308, 0.0019665654, 0, 0.009832827, 0, 0.0039331308, 0.0078662615, 0.0078662615, 0.060963526, 0.066863224, 0.086528875, 0.06882979, 0.017699089, 0.005899696, 0.015732523, 0.021632219, 0.06489666, 0.023598785, 0.11996049, 0.2556535, 0.2556535, 0.033431612, 0, 0.0019665654, 0.011799392, 0.0039331308, 0.017699089, 0.2556535, 0.21828876, 0.070796356, 0, 0, 0.021632219, 0.017699089, 0.0019665654, 0.023598785, 0.0039331308, 0.19665654, 0.2556535, 0.05506383, 0.013765957, 0.0039331308, 0.011799392, 0.013765957, 0.017699089, 0.053097267, 0.2556535, 0.16322492, 0.009832827, 0, 0.0019665654, 0.035398178, 0.2556535, 0.2556535, 0.16519149, 0.017699089, 0, 0, 0.0039331308, 0.04719757, 0.21828876, 0.04719757, 0, 0.0019665654, 0.07276292, 0.04719757, 0.0039331308, 0.019665655, 0.023598785, 0.12192705, 0.06489666, 0.005899696, 0, 0, 0, 0.0019665654, 0.005899696, 0.031465046, 0.20845594, 0.05506383, 0, 0, 0, 0, 0.033431612, 0.09046201, 0.16715805, 0.019665655, 0, 0, 0.0019665654, 0.0078662615, 0.021632219, 0.0078662615, 0.0039331308, 0.0039331308, 0.017699089, 0.027531914, 0.015732523, 0.015732523] 3 +9778 [0.08063698, 0, 0, 0.013767289, 0.0019667556, 0.0019667556, 0.039335113, 0.13177262, 0.0177008, 0, 0, 0, 0, 0.060969424, 0.23601067, 0.119972095, 0.04916889, 0, 0, 0, 0, 0.019667557, 0.23601067, 0.177008, 0.06293618, 0, 0, 0.0019667556, 0.025567822, 0.021634312, 0.043268625, 0.09833778, 0.007867022, 0, 0.003933511, 0.18290827, 0.078670226, 0.029501334, 0.072769955, 0.0354016, 0.023601066, 0.003933511, 0.003933511, 0.037368357, 0.015734045, 0.08653725, 0.23601067, 0.04916889, 0.23601067, 0.009833778, 0, 0, 0, 0.003933511, 0.094404265, 0.19077529, 0.20060907, 0.027534578, 0.0059002666, 0.0059002666, 0.021634312, 0.0177008, 0.06686969, 0.08063698, 0, 0, 0.007867022, 0.23601067, 0.11013831, 0.0059002666, 0.007867022, 0.009833778, 0.011800533, 0.029501334, 0.072769955, 0.22814365, 0.055069156, 0, 0, 0.0059002666, 0.23601067, 0.23601067, 0.047202133, 0.011800533, 0.003933511, 0, 0.0019667556, 0.055069156, 0.10423805, 0.177008, 0.10030454, 0.021634312, 0.021634312, 0.003933511, 0.023601066, 0.027534578, 0.015734045, 0.011800533, 0.007867022, 0.059002668, 0.0177008, 0.0019667556, 0.007867022, 0.043268625, 0.04916889, 0.15537369, 0.23601067, 0.12980586, 0.009833778, 0, 0, 0.011800533, 0.08260374, 0.23601067, 0.17897476, 0.08457049, 0.029501334, 0.003933511, 0.007867022, 0.07670347, 0.023601066, 0.0177008, 0.0177008, 0.023601066, 0.029501334, 0.009833778, 0.047202133, 0.0708032] 4 select * from vector_ip_01 order by inner_product(b, normalize_l2('[41, 0, 0, 7, 1, 1, 20, 67, 9, 0, 0, 0, 0, 31, 120, 61, 25, 0, 0, 0, 0, 10, 120, 90, 32, 0, 0, 1, 13, 11, 22, 50, 4, 0, 2, 93, 40, 15, 37, 18, 12, 2, 2, 19, 8, 44, 120, 25, 120, 5, 0, 0, 0, 2, 48, 97, 102, 14, 3, 3, 11, 9, 34, 41, 0, 0, 4, 120, 56, 3, 4, 5, 6, 15, 37, 116, 28, 0, 0, 3, 120, 120, 24, 6, 2, 0, 1, 28, 53, 90, 51, 11, 11, 2, 12, 14, 8, 6, 4, 30, 9, 1, 4, 22, 25, 79, 120, 66, 5, 0, 0, 6, 42, 120, 91, 43, 15, 2, 4, 39, 12, 9, 9, 12, 15, 5, 24, 36]')) ASC LIMIT 2; a b c -9778 [0.08063698, 0, 0, 0.01376729, 0.0019667556, 0.0019667556, 0.039335113, 0.13177262, 0.0177008, 0, 0, 0, 0, 0.060969424, 0.23601067, 0.119972095, 0.049168892, 0, 0, 0, 0, 0.019667557, 0.23601067, 0.177008, 0.06293618, 0, 0, 0.0019667556, 0.025567824, 0.021634312, 0.043268625, 0.098337784, 0.007867022, 0, 0.003933511, 0.18290828, 0.078670226, 0.029501334, 0.07276996, 0.0354016, 0.023601068, 0.003933511, 0.003933511, 0.037368357, 0.015734045, 0.08653725, 0.23601067, 0.049168892, 0.23601067, 0.009833778, 0, 0, 0, 0.003933511, 0.09440427, 0.1907753, 0.20060907, 0.02753458, 0.005900267, 0.005900267, 0.021634312, 0.0177008, 0.06686969, 0.08063698, 0, 0, 0.007867022, 0.23601067, 0.11013832, 0.005900267, 0.007867022, 0.009833778, 0.011800534, 0.029501334, 0.07276996, 0.22814366, 0.05506916, 0, 0, 0.005900267, 0.23601067, 0.23601067, 0.047202136, 0.011800534, 0.003933511, 0, 0.0019667556, 0.05506916, 0.10423805, 0.177008, 0.10030454, 0.021634312, 0.021634312, 0.003933511, 0.023601068, 0.02753458, 0.015734045, 0.011800534, 0.007867022, 0.059002668, 0.0177008, 0.0019667556, 0.007867022, 0.043268625, 0.049168892, 0.15537369, 0.23601067, 0.12980588, 0.009833778, 0, 0, 0.011800534, 0.08260374, 0.23601067, 0.17897476, 0.08457049, 0.029501334, 0.003933511, 0.007867022, 0.076703474, 0.023601068, 0.0177008, 0.0177008, 0.023601068, 0.029501334, 0.009833778, 0.047202136, 0.0708032] 4 -9774 [0.0019665654, 0, 0.0019665654, 0.011799393, 0.011799393, 0.033431612, 0.09242858, 0.07669605, 0.0039331308, 0, 0.0019665654, 0.04916414, 0.053097267, 0.019665655, 0.110127665, 0.2556535, 0.035398178, 0.009832827, 0.0039331308, 0.011799393, 0.029498482, 0.0039331308, 0.037364744, 0.2556535, 0.08259575, 0.055063833, 0.0019665654, 0.0019665654, 0.0039331308, 0.0019665654, 0, 0.009832827, 0, 0.0039331308, 0.0078662615, 0.0078662615, 0.06096353, 0.066863224, 0.08652888, 0.06882979, 0.017699089, 0.0058996966, 0.015732523, 0.02163222, 0.06489666, 0.023598786, 0.119960494, 0.2556535, 0.2556535, 0.033431612, 0, 0.0019665654, 0.011799393, 0.0039331308, 0.017699089, 0.2556535, 0.21828876, 0.070796356, 0, 0, 0.02163222, 0.017699089, 0.0019665654, 0.023598786, 0.0039331308, 0.19665655, 0.2556535, 0.055063833, 0.013765958, 0.0039331308, 0.011799393, 0.013765958, 0.017699089, 0.053097267, 0.2556535, 0.16322494, 0.009832827, 0, 0.0019665654, 0.035398178, 0.2556535, 0.2556535, 0.1651915, 0.017699089, 0, 0, 0.0039331308, 0.047197573, 0.21828876, 0.047197573, 0, 0.0019665654, 0.07276292, 0.047197573, 0.0039331308, 0.019665655, 0.023598786, 0.12192706, 0.06489666, 0.0058996966, 0, 0, 0, 0.0019665654, 0.0058996966, 0.031465046, 0.20845594, 0.055063833, 0, 0, 0, 0, 0.033431612, 0.090462014, 0.16715807, 0.019665655, 0, 0, 0.0019665654, 0.0078662615, 0.02163222, 0.0078662615, 0.0039331308, 0.0039331308, 0.017699089, 0.027531916, 0.015732523, 0.015732523] 3 +9778 [0.08063698, 0, 0, 0.013767289, 0.0019667556, 0.0019667556, 0.039335113, 0.13177262, 0.0177008, 0, 0, 0, 0, 0.060969424, 0.23601067, 0.119972095, 0.04916889, 0, 0, 0, 0, 0.019667557, 0.23601067, 0.177008, 0.06293618, 0, 0, 0.0019667556, 0.025567822, 0.021634312, 0.043268625, 0.09833778, 0.007867022, 0, 0.003933511, 0.18290827, 0.078670226, 0.029501334, 0.072769955, 0.0354016, 0.023601066, 0.003933511, 0.003933511, 0.037368357, 0.015734045, 0.08653725, 0.23601067, 0.04916889, 0.23601067, 0.009833778, 0, 0, 0, 0.003933511, 0.094404265, 0.19077529, 0.20060907, 0.027534578, 0.0059002666, 0.0059002666, 0.021634312, 0.0177008, 0.06686969, 0.08063698, 0, 0, 0.007867022, 0.23601067, 0.11013831, 0.0059002666, 0.007867022, 0.009833778, 0.011800533, 0.029501334, 0.072769955, 0.22814365, 0.055069156, 0, 0, 0.0059002666, 0.23601067, 0.23601067, 0.047202133, 0.011800533, 0.003933511, 0, 0.0019667556, 0.055069156, 0.10423805, 0.177008, 0.10030454, 0.021634312, 0.021634312, 0.003933511, 0.023601066, 0.027534578, 0.015734045, 0.011800533, 0.007867022, 0.059002668, 0.0177008, 0.0019667556, 0.007867022, 0.043268625, 0.04916889, 0.15537369, 0.23601067, 0.12980586, 0.009833778, 0, 0, 0.011800533, 0.08260374, 0.23601067, 0.17897476, 0.08457049, 0.029501334, 0.003933511, 0.007867022, 0.07670347, 0.023601066, 0.0177008, 0.0177008, 0.023601066, 0.029501334, 0.009833778, 0.047202133, 0.0708032] 4 +9774 [0.0019665654, 0, 0.0019665654, 0.011799392, 0.011799392, 0.033431612, 0.09242857, 0.07669605, 0.0039331308, 0, 0.0019665654, 0.049164135, 0.053097267, 0.019665655, 0.11012766, 0.2556535, 0.035398178, 0.009832827, 0.0039331308, 0.011799392, 0.02949848, 0.0039331308, 0.037364744, 0.2556535, 0.08259574, 0.05506383, 0.0019665654, 0.0019665654, 0.0039331308, 0.0019665654, 0, 0.009832827, 0, 0.0039331308, 0.0078662615, 0.0078662615, 0.060963526, 0.066863224, 0.086528875, 0.06882979, 0.017699089, 0.005899696, 0.015732523, 0.021632219, 0.06489666, 0.023598785, 0.11996049, 0.2556535, 0.2556535, 0.033431612, 0, 0.0019665654, 0.011799392, 0.0039331308, 0.017699089, 0.2556535, 0.21828876, 0.070796356, 0, 0, 0.021632219, 0.017699089, 0.0019665654, 0.023598785, 0.0039331308, 0.19665654, 0.2556535, 0.05506383, 0.013765957, 0.0039331308, 0.011799392, 0.013765957, 0.017699089, 0.053097267, 0.2556535, 0.16322492, 0.009832827, 0, 0.0019665654, 0.035398178, 0.2556535, 0.2556535, 0.16519149, 0.017699089, 0, 0, 0.0039331308, 0.04719757, 0.21828876, 0.04719757, 0, 0.0019665654, 0.07276292, 0.04719757, 0.0039331308, 0.019665655, 0.023598785, 0.12192705, 0.06489666, 0.005899696, 0, 0, 0, 0.0019665654, 0.005899696, 0.031465046, 0.20845594, 0.05506383, 0, 0, 0, 0, 0.033431612, 0.09046201, 0.16715805, 0.019665655, 0, 0, 0.0019665654, 0.0078662615, 0.021632219, 0.0078662615, 0.0039331308, 0.0039331308, 0.017699089, 0.027531914, 0.015732523, 0.015732523] 3 drop table vector_ip_01; drop table if exists vector_cos_01; create table vector_cos_01(a bigint primary key, b vecf32(128),c int,key c_k(c)); @@ -288,9 +288,9 @@ Table Create Table vector_cos_01 CREATE TABLE `vector_cos_01` (\n `a` bigint NOT NULL,\n `b` vecf32(128) DEFAULT NULL,\n `c` int DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `c_k` (`c`),\n KEY `idx01` USING hnsw (`b`) op_type 'vector_cosine_ops' \n) desc vector_cos_01; Field Type Null Key Default Extra Comment -a BIGINT(64) NO PRI null -b VECF32(128) YES MUL null -c INT(32) YES MUL null +a BIGINT(64) NO PRI null +b VECF32(128) YES MUL null +c INT(32) YES MUL null select * from vector_cos_01 order by cosine_distance(b, "[16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]") ASC LIMIT 2; a b c 9777 [16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13] 4 @@ -305,8 +305,8 @@ a b c orderbyfn 9776 [10, 3, 8, 5, 48, 26, 5, 16, 17, 0, 0, 2, 132, 53, 1, 16, 112, 6, 0, 0, 7, 2, 1, 48, 48, 15, 18, 31, 3, 0, 0, 9, 6, 10, 19, 27, 50, 46, 17, 9, 18, 1, 4, 48, 132, 23, 3, 5, 132, 9, 4, 3, 11, 0, 2, 46, 84, 12, 10, 10, 1, 0, 12, 76, 26, 22, 16, 26, 35, 15, 3, 16, 15, 1, 51, 132, 125, 8, 1, 2, 132, 51, 67, 91, 8, 0, 0, 30, 126, 39, 32, 38, 4, 0, 1, 12, 24, 2, 2, 2, 4, 7, 2, 19, 93, 19, 70, 92, 2, 3, 1, 21, 36, 58, 132, 94, 0, 0, 0, 0, 21, 25, 57, 48, 1, 0, 0, 1] 3 0.26629316806793213 select *, l2_distance(b, "[1, 15, 15, 0, 5, 7, 5, 5, 4, 0, 0, 0, 28, 1, 12, 5, 75, 20, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]") as orderbyfn from vector_cos_01 order by cosine_distance(b, "[1, 15, 15, 0, 5, 7, 5, 5, 4, 0, 0, 0, 28, 1, 12, 5, 75, 20, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]") ASC LIMIT 2; a b c orderbyfn -9777 [16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13] 4 127.4205551147461 -9776 [10, 3, 8, 5, 48, 26, 5, 16, 17, 0, 0, 2, 132, 53, 1, 16, 112, 6, 0, 0, 7, 2, 1, 48, 48, 15, 18, 31, 3, 0, 0, 9, 6, 10, 19, 27, 50, 46, 17, 9, 18, 1, 4, 48, 132, 23, 3, 5, 132, 9, 4, 3, 11, 0, 2, 46, 84, 12, 10, 10, 1, 0, 12, 76, 26, 22, 16, 26, 35, 15, 3, 16, 15, 1, 51, 132, 125, 8, 1, 2, 132, 51, 67, 91, 8, 0, 0, 30, 126, 39, 32, 38, 4, 0, 1, 12, 24, 2, 2, 2, 4, 7, 2, 19, 93, 19, 70, 92, 2, 3, 1, 21, 36, 58, 132, 94, 0, 0, 0, 0, 21, 25, 57, 48, 1, 0, 0, 1] 3 364.642333984375 +9777 [16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13] 4 127.42056274414062 +9776 [10, 3, 8, 5, 48, 26, 5, 16, 17, 0, 0, 2, 132, 53, 1, 16, 112, 6, 0, 0, 7, 2, 1, 48, 48, 15, 18, 31, 3, 0, 0, 9, 6, 10, 19, 27, 50, 46, 17, 9, 18, 1, 4, 48, 132, 23, 3, 5, 132, 9, 4, 3, 11, 0, 2, 46, 84, 12, 10, 10, 1, 0, 12, 76, 26, 22, 16, 26, 35, 15, 3, 16, 15, 1, 51, 132, 125, 8, 1, 2, 132, 51, 67, 91, 8, 0, 0, 30, 126, 39, 32, 38, 4, 0, 1, 12, 24, 2, 2, 2, 4, 7, 2, 19, 93, 19, 70, 92, 2, 3, 1, 21, 36, 58, 132, 94, 0, 0, 0, 0, 21, 25, 57, 48, 1, 0, 0, 1] 3 364.6423034667969 select *, cosine_distance(b, "[2, 15, 15, 0, 5, 7, 5, 5, 4, 0, 0, 0, 28, 1, 12, 5, 75, 20, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]") as orderbyfn from vector_cos_01 order by cosine_distance(b, "[1, 15, 15, 0, 5, 7, 5, 5, 4, 0, 0, 0, 28, 1, 12, 5, 75, 20, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]") ASC LIMIT 2; a b c orderbyfn 9777 [16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13] 4 0.031903373234243526 diff --git a/test/distributed/cases/pessimistic_transaction/vector/vector_hnsw_f64.result b/test/distributed/cases/pessimistic_transaction/vector/vector_hnsw_f64.result index 0c18cba685a96..1186931616b61 100644 --- a/test/distributed/cases/pessimistic_transaction/vector/vector_hnsw_f64.result +++ b/test/distributed/cases/pessimistic_transaction/vector/vector_hnsw_f64.result @@ -13,9 +13,9 @@ Table Create Table vector_index_01 CREATE TABLE `vector_index_01` (\n `a` bigint NOT NULL,\n `b` vecf64(128) DEFAULT NULL,\n `c` int DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `c_k` (`c`),\n KEY `idx01` USING hnsw (`b`) m = 48 ef_construction = 64 ef_search = 64 op_type 'vector_l2_ops' \n) desc vector_index_01; Field Type Null Key Default Extra Comment -a BIGINT(64) NO PRI null -b VECF64(128) YES MUL null -c INT(32) YES MUL null +a BIGINT(64) NO PRI null +b VECF64(128) YES MUL null +c INT(32) YES MUL null drop table vector_index_01; create table vector_index_02(a bigint primary key, b vecf64(128),c int,key c_k(c)); insert into vector_index_02 values(9774 ,"[1, 0, 1, 6, 6, 17, 47, 39, 2, 0, 1, 25, 27, 10, 56, 130, 18, 5, 2, 6, 15, 2, 19, 130, 42, 28, 1, 1, 2, 1, 0, 5, 0, 2, 4, 4, 31, 34, 44, 35, 9, 3, 8, 11, 33, 12, 61, 130, 130, 17, 0, 1, 6, 2, 9, 130, 111, 36, 0, 0, 11, 9, 1, 12, 2, 100, 130, 28, 7, 2, 6, 7, 9, 27, 130, 83, 5, 0, 1, 18, 130, 130, 84, 9, 0, 0, 2, 24, 111, 24, 0, 1, 37, 24, 2, 10, 12, 62, 33, 3, 0, 0, 0, 1, 3, 16, 106, 28, 0, 0, 0, 0, 17, 46, 85, 10, 0, 0, 1, 4, 11, 4, 2, 2, 9, 14, 8, 8]",3),(9775,"[0, 1, 1, 3, 0, 3, 46, 20, 1, 4, 17, 9, 1, 17, 108, 15, 0, 3, 37, 17, 6, 15, 116, 16, 6, 1, 4, 7, 7, 7, 9, 6, 0, 8, 10, 4, 26, 129, 27, 9, 0, 0, 5, 2, 11, 129, 129, 12, 103, 4, 0, 0, 2, 31, 129, 129, 94, 4, 0, 0, 0, 3, 13, 42, 0, 15, 38, 2, 70, 129, 1, 0, 5, 10, 40, 12, 74, 129, 6, 1, 129, 39, 6, 1, 2, 22, 9, 33, 122, 13, 0, 0, 0, 0, 5, 23, 4, 11, 9, 12, 45, 38, 1, 0, 0, 4, 36, 38, 57, 32, 0, 0, 82, 22, 9, 5, 13, 11, 3, 94, 35, 3, 0, 0, 0, 1, 16, 97]",5),(9776,"[10, 3, 8, 5, 48, 26, 5, 16, 17, 0, 0, 2, 132, 53, 1, 16, 112, 6, 0, 0, 7, 2, 1, 48, 48, 15, 18, 31, 3, 0, 0, 9, 6, 10, 19, 27, 50, 46, 17, 9, 18, 1, 4, 48, 132, 23, 3, 5, 132, 9, 4, 3, 11, 0, 2, 46, 84, 12, 10, 10, 1, 0, 12, 76, 26, 22, 16, 26, 35, 15, 3, 16, 15, 1, 51, 132, 125, 8, 1, 2, 132, 51, 67, 91, 8, 0, 0, 30, 126, 39, 32, 38, 4, 0, 1, 12, 24, 2, 2, 2, 4, 7, 2, 19, 93, 19, 70, 92, 2, 3, 1, 21, 36, 58, 132, 94, 0, 0, 0, 0, 21, 25, 57, 48, 1, 0, 0, 1]",3); @@ -26,9 +26,9 @@ Table Create Table vector_index_02 CREATE TABLE `vector_index_02` (\n `a` bigint NOT NULL,\n `b` vecf64(128) DEFAULT NULL,\n `c` int DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `c_k` (`c`),\n KEY `idx01` USING hnsw (`b`) op_type 'vector_l2_ops' \n) desc vector_index_02; Field Type Null Key Default Extra Comment -a BIGINT(64) NO PRI null -b VECF64(128) YES MUL null -c INT(32) YES MUL null +a BIGINT(64) NO PRI null +b VECF64(128) YES MUL null +c INT(32) YES MUL null select * from vector_index_02 order by L2_DISTANCE(b, "[16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]") ASC LIMIT 2; a b c 9777 [16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13] 4 @@ -68,9 +68,9 @@ Table Create Table vector_index_04 CREATE TABLE `vector_index_04` (\n `a` bigint NOT NULL,\n `b` vecf64(3) DEFAULT NULL,\n `c` vecf64(4) DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `idx01` USING hnsw (`b`) op_type 'vector_l2_ops' ,\n KEY `idx02` USING hnsw (`c`) op_type 'vector_l2_ops' \n) desc vector_index_04; Field Type Null Key Default Extra Comment -a BIGINT(64) NO PRI null -b VECF64(3) YES MUL null -c VECF64(4) YES MUL null +a BIGINT(64) NO PRI null +b VECF64(3) YES MUL null +c VECF64(4) YES MUL null insert into vector_index_04 values(4,"[156,213,61]","[10.25,0.14,0.88,10.0001]"),(5,"[177,425,30]","[11.25,51.25,80.699,44.25]"),(6,"[80,56,3]","[90.686,5.212,19.22,7.02]"); insert into vector_index_04 values(7,"[5.6,2.13,6.1]","[1.25,8.14,4.88,18.0001]"),(8,"[17.7,4.25,3.0]","[1.215,5.25,8.9,47.25]"),(9,"[8,5.6,13]","[9.6,57.22,19.2,78.02]"); alter table vector_index_04 alter reindex idx01 hnsw; @@ -80,9 +80,9 @@ Table Create Table vector_index_04 CREATE TABLE `vector_index_04` (\n `a` bigint NOT NULL,\n `b` vecf64(3) DEFAULT NULL,\n `c` vecf64(4) DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `idx01` USING hnsw (`b`) op_type 'vector_l2_ops' ,\n KEY `idx02` USING hnsw (`c`) op_type 'vector_l2_ops' \n) desc vector_index_04; Field Type Null Key Default Extra Comment -a BIGINT(64) NO PRI null -b VECF64(3) YES MUL null -c VECF64(4) YES MUL null +a BIGINT(64) NO PRI null +b VECF64(3) YES MUL null +c VECF64(4) YES MUL null select * from vector_index_04 order by L2_DISTANCE(c,"[9.6,57.22,19.2,78.02]") limit 3; a b c 9 [8, 5.6, 13] [9.6, 57.22, 19.2, 78.02] @@ -266,17 +266,17 @@ Table Create Table vector_ip_01 CREATE TABLE `vector_ip_01` (\n `a` bigint NOT NULL,\n `b` vecf64(128) DEFAULT NULL,\n `c` int DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `c_k` (`c`),\n KEY `idx01` USING hnsw (`b`) op_type 'vector_ip_ops' \n) desc vector_ip_01; Field Type Null Key Default Extra Comment -a BIGINT(64) NO PRI null -b VECF64(128) YES MUL null -c INT(32) YES MUL null +a BIGINT(64) NO PRI null +b VECF64(128) YES MUL null +c INT(32) YES MUL null select * from vector_ip_01 order by inner_product(b, normalize_l2("[1, 0, 1, 6, 6, 17, 47, 39, 2, 0, 1, 25, 27, 10, 56, 130, 18, 5, 2, 6, 15, 2, 19, 130, 42, 28, 1, 1, 2, 1, 0, 5, 0, 2, 4, 4, 31, 34, 44, 35, 9, 3, 8, 11, 33, 12, 61, 130, 130, 17, 0, 1, 6, 2, 9, 130, 111, 36, 0, 0, 11, 9, 1, 12, 2, 100, 130, 28, 7, 2, 6, 7, 9, 27, 130, 83, 5, 0, 1, 18, 130, 130, 84, 9, 0, 0, 2, 24, 111, 24, 0, 1, 37, 24, 2, 10, 12, 62, 33, 3, 0, 0, 0, 1, 3, 16, 106, 28, 0, 0, 0, 0, 17, 46, 85, 10, 0, 0, 1, 4, 11, 4, 2, 2, 9, 14, 8, 8]")) ASC LIMIT 2; a b c -9774 [0.0019665653817355633, 0, 0.0019665653817355633, 0.011799393221735954, 0.011799393221735954, 0.03343161195516586, 0.09242857992649078, 0.07669605314731598, 0.0039331307634711266, 0, 0.0019665653817355633, 0.04916413873434067, 0.05309726670384407, 0.019665654748678207, 0.11012766510248184, 0.2556535005569458, 0.035398177802562714, 0.009832827374339104, 0.0039331307634711266, 0.011799393221735954, 0.02949848212301731, 0.0039331307634711266, 0.037364743649959564, 0.2556535005569458, 0.08259575068950653, 0.05506383255124092, 0.0019665653817355633, 0.0019665653817355633, 0.0039331307634711266, 0.0019665653817355633, 0, 0.009832827374339104, 0, 0.0039331307634711266, 0.007866261526942253, 0.007866261526942253, 0.06096353009343147, 0.06686322391033173, 0.08652888238430023, 0.06882978975772858, 0.017699088901281357, 0.005899696610867977, 0.015732523053884506, 0.021632220596075058, 0.06489665806293488, 0.02359878644347191, 0.1199604943394661, 0.2556535005569458, 0.2556535005569458, 0.03343161195516586, 0, 0.0019665653817355633, 0.011799393221735954, 0.0039331307634711266, 0.017699088901281357, 0.2556535005569458, 0.21828876435756683, 0.07079635560512543, 0, 0, 0.021632220596075058, 0.017699088901281357, 0.0019665653817355633, 0.02359878644347191, 0.0039331307634711266, 0.19665655493736267, 0.2556535005569458, 0.05506383255124092, 0.01376595813781023, 0.0039331307634711266, 0.011799393221735954, 0.01376595813781023, 0.017699088901281357, 0.05309726670384407, 0.2556535005569458, 0.1632249355316162, 0.009832827374339104, 0, 0.0019665653817355633, 0.035398177802562714, 0.2556535005569458, 0.2556535005569458, 0.16519150137901306, 0.017699088901281357, 0, 0, 0.0039331307634711266, 0.04719757288694382, 0.21828876435756683, 0.04719757288694382, 0, 0.0019665653817355633, 0.07276292145252228, 0.04719757288694382, 0.0039331307634711266, 0.019665654748678207, 0.02359878644347191, 0.12192706018686295, 0.06489665806293488, 0.005899696610867977, 0, 0, 0, 0.0019665653817355633, 0.005899696610867977, 0.03146504610776901, 0.20845593512058258, 0.05506383255124092, 0, 0, 0, 0, 0.03343161195516586, 0.09046201407909393, 0.1671580672264099, 0.019665654748678207, 0, 0, 0.0019665653817355633, 0.007866261526942253, 0.021632220596075058, 0.007866261526942253, 0.0039331307634711266, 0.0039331307634711266, 0.017699088901281357, 0.02753191627562046, 0.015732523053884506, 0.015732523053884506] 3 -9778 [0.08063697814941406, 0, 0, 0.01376728992909193, 0.0019667556043714285, 0.0019667556043714285, 0.039335113018751144, 0.13177262246608734, 0.0177008006721735, 0, 0, 0, 0, 0.06096942350268364, 0.23601067066192627, 0.1199720948934555, 0.049168892204761505, 0, 0, 0, 0, 0.019667556509375572, 0.23601067066192627, 0.1770080029964447, 0.06293617933988571, 0, 0, 0.0019667556043714285, 0.02556782402098179, 0.021634312346577644, 0.04326862469315529, 0.09833778440952301, 0.007867022417485714, 0, 0.003933511208742857, 0.18290828168392181, 0.07867022603750229, 0.029501333832740784, 0.07276996225118637, 0.035401601344347, 0.023601068183779716, 0.003933511208742857, 0.003933511208742857, 0.03736835718154907, 0.015734044834971428, 0.08653724938631058, 0.23601067066192627, 0.049168892204761505, 0.23601067066192627, 0.009833778254687786, 0, 0, 0, 0.003933511208742857, 0.09440427273511887, 0.1907753050327301, 0.20060907304286957, 0.02753457985818386, 0.005900267045944929, 0.005900267045944929, 0.021634312346577644, 0.0177008006721735, 0.06686969101428986, 0.08063697814941406, 0, 0, 0.007867022417485714, 0.23601067066192627, 0.11013831943273544, 0.005900267045944929, 0.007867022417485714, 0.009833778254687786, 0.011800534091889858, 0.029501333832740784, 0.07276996225118637, 0.22814366221427917, 0.05506915971636772, 0, 0, 0.005900267045944929, 0.23601067066192627, 0.23601067066192627, 0.04720213636755943, 0.011800534091889858, 0.003933511208742857, 0, 0.0019667556043714285, 0.05506915971636772, 0.10423804819583893, 0.1770080029964447, 0.10030453652143478, 0.021634312346577644, 0.021634312346577644, 0.003933511208742857, 0.023601068183779716, 0.02753457985818386, 0.015734044834971428, 0.011800534091889858, 0.007867022417485714, 0.05900266766548157, 0.0177008006721735, 0.0019667556043714285, 0.007867022417485714, 0.04326862469315529, 0.049168892204761505, 0.1553736925125122, 0.23601067066192627, 0.12980587780475616, 0.009833778254687786, 0, 0, 0.011800534091889858, 0.08260373771190643, 0.23601067066192627, 0.17897476255893707, 0.0845704898238182, 0.029501333832740784, 0.003933511208742857, 0.007867022417485714, 0.07670347392559052, 0.023601068183779716, 0.0177008006721735, 0.0177008006721735, 0.023601068183779716, 0.029501333832740784, 0.009833778254687786, 0.04720213636755943, 0.070803202688694] 4 +9774 [0.0019665653817355633, 0, 0.0019665653817355633, 0.01179939229041338, 0.01179939229041338, 0.03343161195516586, 0.09242857247591019, 0.07669605314731598, 0.0039331307634711266, 0, 0.0019665653817355633, 0.04916413500905037, 0.05309726670384407, 0.019665654748678207, 0.11012765765190125, 0.2556535005569458, 0.035398177802562714, 0.009832827374339104, 0.0039331307634711266, 0.01179939229041338, 0.029498480260372162, 0.0039331307634711266, 0.037364743649959564, 0.2556535005569458, 0.08259574323892593, 0.05506382882595062, 0.0019665653817355633, 0.0019665653817355633, 0.0039331307634711266, 0.0019665653817355633, 0, 0.009832827374339104, 0, 0.0039331307634711266, 0.007866261526942253, 0.007866261526942253, 0.060963526368141174, 0.06686322391033173, 0.08652887493371964, 0.06882978975772858, 0.017699088901281357, 0.00589969614520669, 0.015732523053884506, 0.02163221873342991, 0.06489665806293488, 0.02359878458082676, 0.1199604868888855, 0.2556535005569458, 0.2556535005569458, 0.03343161195516586, 0, 0.0019665653817355633, 0.01179939229041338, 0.0039331307634711266, 0.017699088901281357, 0.2556535005569458, 0.21828876435756683, 0.07079635560512543, 0, 0, 0.02163221873342991, 0.017699088901281357, 0.0019665653817355633, 0.02359878458082676, 0.0039331307634711266, 0.19665654003620148, 0.2556535005569458, 0.05506382882595062, 0.013765957206487656, 0.0039331307634711266, 0.01179939229041338, 0.013765957206487656, 0.017699088901281357, 0.05309726670384407, 0.2556535005569458, 0.16322492063045502, 0.009832827374339104, 0, 0.0019665653817355633, 0.035398177802562714, 0.2556535005569458, 0.2556535005569458, 0.16519148647785187, 0.017699088901281357, 0, 0, 0.0039331307634711266, 0.04719756916165352, 0.21828876435756683, 0.04719756916165352, 0, 0.0019665653817355633, 0.07276292145252228, 0.04719756916165352, 0.0039331307634711266, 0.019665654748678207, 0.02359878458082676, 0.12192705273628235, 0.06489665806293488, 0.00589969614520669, 0, 0, 0, 0.0019665653817355633, 0.00589969614520669, 0.03146504610776901, 0.20845593512058258, 0.05506382882595062, 0, 0, 0, 0, 0.03343161195516586, 0.09046200662851334, 0.16715805232524872, 0.019665654748678207, 0, 0, 0.0019665653817355633, 0.007866261526942253, 0.02163221873342991, 0.007866261526942253, 0.0039331307634711266, 0.0039331307634711266, 0.017699088901281357, 0.02753191441297531, 0.015732523053884506, 0.015732523053884506] 3 +9778 [0.08063697814941406, 0, 0, 0.013767288997769356, 0.0019667556043714285, 0.0019667556043714285, 0.039335113018751144, 0.13177262246608734, 0.0177008006721735, 0, 0, 0, 0, 0.06096942350268364, 0.23601067066192627, 0.1199720948934555, 0.04916888847947121, 0, 0, 0, 0, 0.019667556509375572, 0.23601067066192627, 0.1770080029964447, 0.06293617933988571, 0, 0, 0.0019667556043714285, 0.02556782215833664, 0.021634312346577644, 0.04326862469315529, 0.09833777695894241, 0.007867022417485714, 0, 0.003933511208742857, 0.18290826678276062, 0.07867022603750229, 0.029501333832740784, 0.07276995480060577, 0.035401601344347, 0.023601066321134567, 0.003933511208742857, 0.003933511208742857, 0.03736835718154907, 0.015734044834971428, 0.08653724938631058, 0.23601067066192627, 0.04916888847947121, 0.23601067066192627, 0.009833778254687786, 0, 0, 0, 0.003933511208742857, 0.09440426528453827, 0.1907752901315689, 0.20060907304286957, 0.02753457799553871, 0.005900266580283642, 0.005900266580283642, 0.021634312346577644, 0.0177008006721735, 0.06686969101428986, 0.08063697814941406, 0, 0, 0.007867022417485714, 0.23601067066192627, 0.11013831198215485, 0.005900266580283642, 0.007867022417485714, 0.009833778254687786, 0.011800533160567284, 0.029501333832740784, 0.07276995480060577, 0.22814364731311798, 0.05506915599107742, 0, 0, 0.005900266580283642, 0.23601067066192627, 0.23601067066192627, 0.047202132642269135, 0.011800533160567284, 0.003933511208742857, 0, 0.0019667556043714285, 0.05506915599107742, 0.10423804819583893, 0.1770080029964447, 0.10030453652143478, 0.021634312346577644, 0.021634312346577644, 0.003933511208742857, 0.023601066321134567, 0.02753457799553871, 0.015734044834971428, 0.011800533160567284, 0.007867022417485714, 0.05900266766548157, 0.0177008006721735, 0.0019667556043714285, 0.007867022417485714, 0.04326862469315529, 0.04916888847947121, 0.1553736925125122, 0.23601067066192627, 0.12980586290359497, 0.009833778254687786, 0, 0, 0.011800533160567284, 0.08260373771190643, 0.23601067066192627, 0.17897476255893707, 0.0845704898238182, 0.029501333832740784, 0.003933511208742857, 0.007867022417485714, 0.07670346647500992, 0.023601066321134567, 0.0177008006721735, 0.0177008006721735, 0.023601066321134567, 0.029501333832740784, 0.009833778254687786, 0.047202132642269135, 0.070803202688694] 4 select * from vector_ip_01 order by inner_product(b, normalize_l2('[41, 0, 0, 7, 1, 1, 20, 67, 9, 0, 0, 0, 0, 31, 120, 61, 25, 0, 0, 0, 0, 10, 120, 90, 32, 0, 0, 1, 13, 11, 22, 50, 4, 0, 2, 93, 40, 15, 37, 18, 12, 2, 2, 19, 8, 44, 120, 25, 120, 5, 0, 0, 0, 2, 48, 97, 102, 14, 3, 3, 11, 9, 34, 41, 0, 0, 4, 120, 56, 3, 4, 5, 6, 15, 37, 116, 28, 0, 0, 3, 120, 120, 24, 6, 2, 0, 1, 28, 53, 90, 51, 11, 11, 2, 12, 14, 8, 6, 4, 30, 9, 1, 4, 22, 25, 79, 120, 66, 5, 0, 0, 6, 42, 120, 91, 43, 15, 2, 4, 39, 12, 9, 9, 12, 15, 5, 24, 36]')) ASC LIMIT 2; a b c -9778 [0.08063697814941406, 0, 0, 0.01376728992909193, 0.0019667556043714285, 0.0019667556043714285, 0.039335113018751144, 0.13177262246608734, 0.0177008006721735, 0, 0, 0, 0, 0.06096942350268364, 0.23601067066192627, 0.1199720948934555, 0.049168892204761505, 0, 0, 0, 0, 0.019667556509375572, 0.23601067066192627, 0.1770080029964447, 0.06293617933988571, 0, 0, 0.0019667556043714285, 0.02556782402098179, 0.021634312346577644, 0.04326862469315529, 0.09833778440952301, 0.007867022417485714, 0, 0.003933511208742857, 0.18290828168392181, 0.07867022603750229, 0.029501333832740784, 0.07276996225118637, 0.035401601344347, 0.023601068183779716, 0.003933511208742857, 0.003933511208742857, 0.03736835718154907, 0.015734044834971428, 0.08653724938631058, 0.23601067066192627, 0.049168892204761505, 0.23601067066192627, 0.009833778254687786, 0, 0, 0, 0.003933511208742857, 0.09440427273511887, 0.1907753050327301, 0.20060907304286957, 0.02753457985818386, 0.005900267045944929, 0.005900267045944929, 0.021634312346577644, 0.0177008006721735, 0.06686969101428986, 0.08063697814941406, 0, 0, 0.007867022417485714, 0.23601067066192627, 0.11013831943273544, 0.005900267045944929, 0.007867022417485714, 0.009833778254687786, 0.011800534091889858, 0.029501333832740784, 0.07276996225118637, 0.22814366221427917, 0.05506915971636772, 0, 0, 0.005900267045944929, 0.23601067066192627, 0.23601067066192627, 0.04720213636755943, 0.011800534091889858, 0.003933511208742857, 0, 0.0019667556043714285, 0.05506915971636772, 0.10423804819583893, 0.1770080029964447, 0.10030453652143478, 0.021634312346577644, 0.021634312346577644, 0.003933511208742857, 0.023601068183779716, 0.02753457985818386, 0.015734044834971428, 0.011800534091889858, 0.007867022417485714, 0.05900266766548157, 0.0177008006721735, 0.0019667556043714285, 0.007867022417485714, 0.04326862469315529, 0.049168892204761505, 0.1553736925125122, 0.23601067066192627, 0.12980587780475616, 0.009833778254687786, 0, 0, 0.011800534091889858, 0.08260373771190643, 0.23601067066192627, 0.17897476255893707, 0.0845704898238182, 0.029501333832740784, 0.003933511208742857, 0.007867022417485714, 0.07670347392559052, 0.023601068183779716, 0.0177008006721735, 0.0177008006721735, 0.023601068183779716, 0.029501333832740784, 0.009833778254687786, 0.04720213636755943, 0.070803202688694] 4 -9774 [0.0019665653817355633, 0, 0.0019665653817355633, 0.011799393221735954, 0.011799393221735954, 0.03343161195516586, 0.09242857992649078, 0.07669605314731598, 0.0039331307634711266, 0, 0.0019665653817355633, 0.04916413873434067, 0.05309726670384407, 0.019665654748678207, 0.11012766510248184, 0.2556535005569458, 0.035398177802562714, 0.009832827374339104, 0.0039331307634711266, 0.011799393221735954, 0.02949848212301731, 0.0039331307634711266, 0.037364743649959564, 0.2556535005569458, 0.08259575068950653, 0.05506383255124092, 0.0019665653817355633, 0.0019665653817355633, 0.0039331307634711266, 0.0019665653817355633, 0, 0.009832827374339104, 0, 0.0039331307634711266, 0.007866261526942253, 0.007866261526942253, 0.06096353009343147, 0.06686322391033173, 0.08652888238430023, 0.06882978975772858, 0.017699088901281357, 0.005899696610867977, 0.015732523053884506, 0.021632220596075058, 0.06489665806293488, 0.02359878644347191, 0.1199604943394661, 0.2556535005569458, 0.2556535005569458, 0.03343161195516586, 0, 0.0019665653817355633, 0.011799393221735954, 0.0039331307634711266, 0.017699088901281357, 0.2556535005569458, 0.21828876435756683, 0.07079635560512543, 0, 0, 0.021632220596075058, 0.017699088901281357, 0.0019665653817355633, 0.02359878644347191, 0.0039331307634711266, 0.19665655493736267, 0.2556535005569458, 0.05506383255124092, 0.01376595813781023, 0.0039331307634711266, 0.011799393221735954, 0.01376595813781023, 0.017699088901281357, 0.05309726670384407, 0.2556535005569458, 0.1632249355316162, 0.009832827374339104, 0, 0.0019665653817355633, 0.035398177802562714, 0.2556535005569458, 0.2556535005569458, 0.16519150137901306, 0.017699088901281357, 0, 0, 0.0039331307634711266, 0.04719757288694382, 0.21828876435756683, 0.04719757288694382, 0, 0.0019665653817355633, 0.07276292145252228, 0.04719757288694382, 0.0039331307634711266, 0.019665654748678207, 0.02359878644347191, 0.12192706018686295, 0.06489665806293488, 0.005899696610867977, 0, 0, 0, 0.0019665653817355633, 0.005899696610867977, 0.03146504610776901, 0.20845593512058258, 0.05506383255124092, 0, 0, 0, 0, 0.03343161195516586, 0.09046201407909393, 0.1671580672264099, 0.019665654748678207, 0, 0, 0.0019665653817355633, 0.007866261526942253, 0.021632220596075058, 0.007866261526942253, 0.0039331307634711266, 0.0039331307634711266, 0.017699088901281357, 0.02753191627562046, 0.015732523053884506, 0.015732523053884506] 3 +9778 [0.08063697814941406, 0, 0, 0.013767288997769356, 0.0019667556043714285, 0.0019667556043714285, 0.039335113018751144, 0.13177262246608734, 0.0177008006721735, 0, 0, 0, 0, 0.06096942350268364, 0.23601067066192627, 0.1199720948934555, 0.04916888847947121, 0, 0, 0, 0, 0.019667556509375572, 0.23601067066192627, 0.1770080029964447, 0.06293617933988571, 0, 0, 0.0019667556043714285, 0.02556782215833664, 0.021634312346577644, 0.04326862469315529, 0.09833777695894241, 0.007867022417485714, 0, 0.003933511208742857, 0.18290826678276062, 0.07867022603750229, 0.029501333832740784, 0.07276995480060577, 0.035401601344347, 0.023601066321134567, 0.003933511208742857, 0.003933511208742857, 0.03736835718154907, 0.015734044834971428, 0.08653724938631058, 0.23601067066192627, 0.04916888847947121, 0.23601067066192627, 0.009833778254687786, 0, 0, 0, 0.003933511208742857, 0.09440426528453827, 0.1907752901315689, 0.20060907304286957, 0.02753457799553871, 0.005900266580283642, 0.005900266580283642, 0.021634312346577644, 0.0177008006721735, 0.06686969101428986, 0.08063697814941406, 0, 0, 0.007867022417485714, 0.23601067066192627, 0.11013831198215485, 0.005900266580283642, 0.007867022417485714, 0.009833778254687786, 0.011800533160567284, 0.029501333832740784, 0.07276995480060577, 0.22814364731311798, 0.05506915599107742, 0, 0, 0.005900266580283642, 0.23601067066192627, 0.23601067066192627, 0.047202132642269135, 0.011800533160567284, 0.003933511208742857, 0, 0.0019667556043714285, 0.05506915599107742, 0.10423804819583893, 0.1770080029964447, 0.10030453652143478, 0.021634312346577644, 0.021634312346577644, 0.003933511208742857, 0.023601066321134567, 0.02753457799553871, 0.015734044834971428, 0.011800533160567284, 0.007867022417485714, 0.05900266766548157, 0.0177008006721735, 0.0019667556043714285, 0.007867022417485714, 0.04326862469315529, 0.04916888847947121, 0.1553736925125122, 0.23601067066192627, 0.12980586290359497, 0.009833778254687786, 0, 0, 0.011800533160567284, 0.08260373771190643, 0.23601067066192627, 0.17897476255893707, 0.0845704898238182, 0.029501333832740784, 0.003933511208742857, 0.007867022417485714, 0.07670346647500992, 0.023601066321134567, 0.0177008006721735, 0.0177008006721735, 0.023601066321134567, 0.029501333832740784, 0.009833778254687786, 0.047202132642269135, 0.070803202688694] 4 +9774 [0.0019665653817355633, 0, 0.0019665653817355633, 0.01179939229041338, 0.01179939229041338, 0.03343161195516586, 0.09242857247591019, 0.07669605314731598, 0.0039331307634711266, 0, 0.0019665653817355633, 0.04916413500905037, 0.05309726670384407, 0.019665654748678207, 0.11012765765190125, 0.2556535005569458, 0.035398177802562714, 0.009832827374339104, 0.0039331307634711266, 0.01179939229041338, 0.029498480260372162, 0.0039331307634711266, 0.037364743649959564, 0.2556535005569458, 0.08259574323892593, 0.05506382882595062, 0.0019665653817355633, 0.0019665653817355633, 0.0039331307634711266, 0.0019665653817355633, 0, 0.009832827374339104, 0, 0.0039331307634711266, 0.007866261526942253, 0.007866261526942253, 0.060963526368141174, 0.06686322391033173, 0.08652887493371964, 0.06882978975772858, 0.017699088901281357, 0.00589969614520669, 0.015732523053884506, 0.02163221873342991, 0.06489665806293488, 0.02359878458082676, 0.1199604868888855, 0.2556535005569458, 0.2556535005569458, 0.03343161195516586, 0, 0.0019665653817355633, 0.01179939229041338, 0.0039331307634711266, 0.017699088901281357, 0.2556535005569458, 0.21828876435756683, 0.07079635560512543, 0, 0, 0.02163221873342991, 0.017699088901281357, 0.0019665653817355633, 0.02359878458082676, 0.0039331307634711266, 0.19665654003620148, 0.2556535005569458, 0.05506382882595062, 0.013765957206487656, 0.0039331307634711266, 0.01179939229041338, 0.013765957206487656, 0.017699088901281357, 0.05309726670384407, 0.2556535005569458, 0.16322492063045502, 0.009832827374339104, 0, 0.0019665653817355633, 0.035398177802562714, 0.2556535005569458, 0.2556535005569458, 0.16519148647785187, 0.017699088901281357, 0, 0, 0.0039331307634711266, 0.04719756916165352, 0.21828876435756683, 0.04719756916165352, 0, 0.0019665653817355633, 0.07276292145252228, 0.04719756916165352, 0.0039331307634711266, 0.019665654748678207, 0.02359878458082676, 0.12192705273628235, 0.06489665806293488, 0.00589969614520669, 0, 0, 0, 0.0019665653817355633, 0.00589969614520669, 0.03146504610776901, 0.20845593512058258, 0.05506382882595062, 0, 0, 0, 0, 0.03343161195516586, 0.09046200662851334, 0.16715805232524872, 0.019665654748678207, 0, 0, 0.0019665653817355633, 0.007866261526942253, 0.02163221873342991, 0.007866261526942253, 0.0039331307634711266, 0.0039331307634711266, 0.017699088901281357, 0.02753191441297531, 0.015732523053884506, 0.015732523053884506] 3 drop table vector_ip_01; drop table if exists vector_cos_01; create table vector_cos_01(a bigint primary key, b vecf64(128),c int,key c_k(c)); @@ -288,9 +288,9 @@ Table Create Table vector_cos_01 CREATE TABLE `vector_cos_01` (\n `a` bigint NOT NULL,\n `b` vecf64(128) DEFAULT NULL,\n `c` int DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `c_k` (`c`),\n KEY `idx01` USING hnsw (`b`) op_type 'vector_cosine_ops' \n) desc vector_cos_01; Field Type Null Key Default Extra Comment -a BIGINT(64) NO PRI null -b VECF64(128) YES MUL null -c INT(32) YES MUL null +a BIGINT(64) NO PRI null +b VECF64(128) YES MUL null +c INT(32) YES MUL null select * from vector_cos_01 order by cosine_distance(b, "[16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]") ASC LIMIT 2; a b c 9777 [16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13] 4 diff --git a/test/distributed/cases/vector/vector_func.result b/test/distributed/cases/vector/vector_func.result index 513396d1a0e39..82cdbd12342eb 100644 --- a/test/distributed/cases/vector/vector_func.result +++ b/test/distributed/cases/vector/vector_func.result @@ -61,7 +61,7 @@ null 837.3288311958313 select sum(SUMMATION(vecf32_3)) from vtab32; sum(SUMMATION(vecf32_3)) -873.1968674659729 +873.1968944072723 select min(SUMMATION(vecf32_3)) from vtab32; min(SUMMATION(vecf32_3)) 1.9714266657829285 @@ -70,7 +70,7 @@ max(SUMMATION(vecf32_3)) 837.3288311958313 select avg(SUMMATION(vecf32_3)) from vtab32; avg(SUMMATION(vecf32_3)) -291.06562248865765 +291.06563146909076 select count(SUMMATION(vecf32_3)) from vtab32; count(SUMMATION(vecf32_3)) 3 @@ -412,7 +412,7 @@ null 822.0504150390625 select sum(l2_norm(vecf32_3)) from vtab32; sum(l2_norm(vecf32_3)) -853.8230618238449 +853.8231209516525 select min(l2_norm(vecf32_3)) from vtab32; min(l2_norm(vecf32_3)) 1.161650538444519 @@ -421,7 +421,7 @@ max(l2_norm(vecf32_3)) 822.0504150390625 select avg(l2_norm(vecf32_3)) from vtab32; avg(l2_norm(vecf32_3)) -284.607687274615 +284.60770698388416 select count(l2_norm(vecf32_3)) from vtab32; count(l2_norm(vecf32_3)) 3 @@ -757,8 +757,8 @@ null null select cosine_similarity(vecf32_3,vecf32_3), cosine_similarity(vecf32_5,vecf32_5) from vtab32; cosine_similarity(vecf32_3, vecf32_3) cosine_similarity(vecf32_5, vecf32_5) null null -1.0 null -0.9999998589068639 0.9999999473078075 +0.99999994 null +1.0 0.99999994 1.0 1.0 select cosine_similarity(vecf64_3,vecf64_3), cosine_similarity(vecf64_5,vecf64_5) from vtab64; cosine_similarity(vecf64_3, vecf64_3) cosine_similarity(vecf64_5, vecf64_5) diff --git a/test/distributed/cases/vector/vector_index.result b/test/distributed/cases/vector/vector_index.result index f9abf16a9c969..0ba872dbc4f52 100644 --- a/test/distributed/cases/vector/vector_index.result +++ b/test/distributed/cases/vector/vector_index.result @@ -10,9 +10,9 @@ Table Create Table vector_index_01 CREATE TABLE `vector_index_01` (\n `a` int NOT NULL,\n `b` vecf32(128) DEFAULT NULL,\n `c` int DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `c_k` (`c`),\n KEY `idx01` USING ivfflat (`b`) lists = 5 op_type 'vector_l2_ops' \n) desc vector_index_01; Field Type Null Key Default Extra Comment -a INT(32) NO PRI null -b VECF32(128) YES MUL null -c INT(32) YES MUL null +a INT(32) NO PRI null +b VECF32(128) YES MUL null +c INT(32) YES MUL null select * from vector_index_01 order by L2_DISTANCE(b, "[16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]") ASC LIMIT 2; a b c 9777 [16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13] 4 @@ -28,9 +28,9 @@ Table Create Table vector_index_02 CREATE TABLE `vector_index_02` (\n `a` int NOT NULL,\n `b` vecf32(128) DEFAULT NULL,\n `c` int DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `c_k` (`c`),\n KEY `idx01` USING ivfflat (`b`) lists = 5 op_type 'vector_l2_ops' \n) desc vector_index_02; Field Type Null Key Default Extra Comment -a INT(32) NO PRI null -b VECF32(128) YES MUL null -c INT(32) YES MUL null +a INT(32) NO PRI null +b VECF32(128) YES MUL null +c INT(32) YES MUL null select * from vector_index_02 order by L2_DISTANCE(b, "[16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]") ASC LIMIT 2; a b c 9777 [16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13] 4 @@ -73,9 +73,9 @@ Table Create Table vector_index_04 CREATE TABLE `vector_index_04` (\n `a` int NOT NULL,\n `b` vecf32(3) DEFAULT NULL,\n `c` vecf32(4) DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `idx01` USING ivfflat (`c`) lists = 3 op_type 'vector_l2_ops' \n) desc vector_index_04; Field Type Null Key Default Extra Comment -a INT(32) NO PRI null -b VECF32(3) YES null -c VECF32(4) YES MUL null +a INT(32) NO PRI null +b VECF32(3) YES null +c VECF32(4) YES MUL null insert into vector_index_04 values(4,"[156,213,61]","[10.25,0.14,0.88,10.0001]"),(5,"[177,425,30]","[11.25,51.25,80.699,44.25]"),(6,"[80,56,3]","[90.686,5.212,19.22,7.02]"); insert into vector_index_04 values(7,"[5.6,2.13,6.1]","[1.25,8.14,4.88,18.0001]"),(8,"[17.7,4.25,3.0]","[1.215,5.25,8.9,47.25]"),(9,"[8,5.6,13]","[9.6,57.22,19.2,78.02]"); alter table vector_index_04 alter reindex idx ivfflat lists=8; @@ -86,9 +86,9 @@ Table Create Table vector_index_04 CREATE TABLE `vector_index_04` (\n `a` int NOT NULL,\n `b` vecf32(3) DEFAULT NULL,\n `c` vecf32(4) DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `idx01` USING ivfflat (`c`) lists = 8 op_type 'vector_l2_ops' \n) desc vector_index_04; Field Type Null Key Default Extra Comment -a INT(32) NO PRI null -b VECF32(3) YES null -c VECF32(4) YES MUL null +a INT(32) NO PRI null +b VECF32(3) YES null +c VECF32(4) YES MUL null select * from vector_index_04 order by L2_DISTANCE(c,"[9.6,57.22,19.2,78.02]") limit 3; a b c 9 [8, 5.6, 13] [9.6, 57.22, 19.2, 78.02] @@ -263,15 +263,15 @@ Table Create Table vector_ip_01 CREATE TABLE `vector_ip_01` (\n `a` bigint NOT NULL,\n `b` vecf32(128) DEFAULT NULL,\n `c` int DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `c_k` (`c`),\n KEY `idx01` USING ivfflat (`b`) lists = 5 op_type 'vector_ip_ops' \n) desc vector_ip_01; Field Type Null Key Default Extra Comment -a BIGINT(64) NO PRI null -b VECF32(128) YES MUL null -c INT(32) YES MUL null +a BIGINT(64) NO PRI null +b VECF32(128) YES MUL null +c INT(32) YES MUL null select * from vector_ip_01 order by inner_product(b, normalize_l2("[1, 0, 1, 6, 6, 17, 47, 39, 2, 0, 1, 25, 27, 10, 56, 130, 18, 5, 2, 6, 15, 2, 19, 130, 42, 28, 1, 1, 2, 1, 0, 5, 0, 2, 4, 4, 31, 34, 44, 35, 9, 3, 8, 11, 33, 12, 61, 130, 130, 17, 0, 1, 6, 2, 9, 130, 111, 36, 0, 0, 11, 9, 1, 12, 2, 100, 130, 28, 7, 2, 6, 7, 9, 27, 130, 83, 5, 0, 1, 18, 130, 130, 84, 9, 0, 0, 2, 24, 111, 24, 0, 1, 37, 24, 2, 10, 12, 62, 33, 3, 0, 0, 0, 1, 3, 16, 106, 28, 0, 0, 0, 0, 17, 46, 85, 10, 0, 0, 1, 4, 11, 4, 2, 2, 9, 14, 8, 8]")) ASC LIMIT 2; a b c -9774 [0.0019665654, 0, 0.0019665654, 0.011799393, 0.011799393, 0.033431612, 0.09242858, 0.07669605, 0.0039331308, 0, 0.0019665654, 0.04916414, 0.053097267, 0.019665655, 0.110127665, 0.2556535, 0.035398178, 0.009832827, 0.0039331308, 0.011799393, 0.029498482, 0.0039331308, 0.037364744, 0.2556535, 0.08259575, 0.055063833, 0.0019665654, 0.0019665654, 0.0039331308, 0.0019665654, 0, 0.009832827, 0, 0.0039331308, 0.0078662615, 0.0078662615, 0.06096353, 0.066863224, 0.08652888, 0.06882979, 0.017699089, 0.0058996966, 0.015732523, 0.02163222, 0.06489666, 0.023598786, 0.119960494, 0.2556535, 0.2556535, 0.033431612, 0, 0.0019665654, 0.011799393, 0.0039331308, 0.017699089, 0.2556535, 0.21828876, 0.070796356, 0, 0, 0.02163222, 0.017699089, 0.0019665654, 0.023598786, 0.0039331308, 0.19665655, 0.2556535, 0.055063833, 0.013765958, 0.0039331308, 0.011799393, 0.013765958, 0.017699089, 0.053097267, 0.2556535, 0.16322494, 0.009832827, 0, 0.0019665654, 0.035398178, 0.2556535, 0.2556535, 0.1651915, 0.017699089, 0, 0, 0.0039331308, 0.047197573, 0.21828876, 0.047197573, 0, 0.0019665654, 0.07276292, 0.047197573, 0.0039331308, 0.019665655, 0.023598786, 0.12192706, 0.06489666, 0.0058996966, 0, 0, 0, 0.0019665654, 0.0058996966, 0.031465046, 0.20845594, 0.055063833, 0, 0, 0, 0, 0.033431612, 0.090462014, 0.16715807, 0.019665655, 0, 0, 0.0019665654, 0.0078662615, 0.02163222, 0.0078662615, 0.0039331308, 0.0039331308, 0.017699089, 0.027531916, 0.015732523, 0.015732523] 3 +9774 [0.0019665654, 0, 0.0019665654, 0.011799392, 0.011799392, 0.033431612, 0.09242857, 0.07669605, 0.0039331308, 0, 0.0019665654, 0.049164135, 0.053097267, 0.019665655, 0.11012766, 0.2556535, 0.035398178, 0.009832827, 0.0039331308, 0.011799392, 0.02949848, 0.0039331308, 0.037364744, 0.2556535, 0.08259574, 0.05506383, 0.0019665654, 0.0019665654, 0.0039331308, 0.0019665654, 0, 0.009832827, 0, 0.0039331308, 0.0078662615, 0.0078662615, 0.060963526, 0.066863224, 0.086528875, 0.06882979, 0.017699089, 0.005899696, 0.015732523, 0.021632219, 0.06489666, 0.023598785, 0.11996049, 0.2556535, 0.2556535, 0.033431612, 0, 0.0019665654, 0.011799392, 0.0039331308, 0.017699089, 0.2556535, 0.21828876, 0.070796356, 0, 0, 0.021632219, 0.017699089, 0.0019665654, 0.023598785, 0.0039331308, 0.19665654, 0.2556535, 0.05506383, 0.013765957, 0.0039331308, 0.011799392, 0.013765957, 0.017699089, 0.053097267, 0.2556535, 0.16322492, 0.009832827, 0, 0.0019665654, 0.035398178, 0.2556535, 0.2556535, 0.16519149, 0.017699089, 0, 0, 0.0039331308, 0.04719757, 0.21828876, 0.04719757, 0, 0.0019665654, 0.07276292, 0.04719757, 0.0039331308, 0.019665655, 0.023598785, 0.12192705, 0.06489666, 0.005899696, 0, 0, 0, 0.0019665654, 0.005899696, 0.031465046, 0.20845594, 0.05506383, 0, 0, 0, 0, 0.033431612, 0.09046201, 0.16715805, 0.019665655, 0, 0, 0.0019665654, 0.0078662615, 0.021632219, 0.0078662615, 0.0039331308, 0.0039331308, 0.017699089, 0.027531914, 0.015732523, 0.015732523] 3 select * from vector_ip_01 order by inner_product(b, normalize_l2('[41, 0, 0, 7, 1, 1, 20, 67, 9, 0, 0, 0, 0, 31, 120, 61, 25, 0, 0, 0, 0, 10, 120, 90, 32, 0, 0, 1, 13, 11, 22, 50, 4, 0, 2, 93, 40, 15, 37, 18, 12, 2, 2, 19, 8, 44, 120, 25, 120, 5, 0, 0, 0, 2, 48, 97, 102, 14, 3, 3, 11, 9, 34, 41, 0, 0, 4, 120, 56, 3, 4, 5, 6, 15, 37, 116, 28, 0, 0, 3, 120, 120, 24, 6, 2, 0, 1, 28, 53, 90, 51, 11, 11, 2, 12, 14, 8, 6, 4, 30, 9, 1, 4, 22, 25, 79, 120, 66, 5, 0, 0, 6, 42, 120, 91, 43, 15, 2, 4, 39, 12, 9, 9, 12, 15, 5, 24, 36]')) ASC LIMIT 2; a b c -9778 [0.08063698, 0, 0, 0.01376729, 0.0019667556, 0.0019667556, 0.039335113, 0.13177262, 0.0177008, 0, 0, 0, 0, 0.060969424, 0.23601067, 0.119972095, 0.049168892, 0, 0, 0, 0, 0.019667557, 0.23601067, 0.177008, 0.06293618, 0, 0, 0.0019667556, 0.025567824, 0.021634312, 0.043268625, 0.098337784, 0.007867022, 0, 0.003933511, 0.18290828, 0.078670226, 0.029501334, 0.07276996, 0.0354016, 0.023601068, 0.003933511, 0.003933511, 0.037368357, 0.015734045, 0.08653725, 0.23601067, 0.049168892, 0.23601067, 0.009833778, 0, 0, 0, 0.003933511, 0.09440427, 0.1907753, 0.20060907, 0.02753458, 0.005900267, 0.005900267, 0.021634312, 0.0177008, 0.06686969, 0.08063698, 0, 0, 0.007867022, 0.23601067, 0.11013832, 0.005900267, 0.007867022, 0.009833778, 0.011800534, 0.029501334, 0.07276996, 0.22814366, 0.05506916, 0, 0, 0.005900267, 0.23601067, 0.23601067, 0.047202136, 0.011800534, 0.003933511, 0, 0.0019667556, 0.05506916, 0.10423805, 0.177008, 0.10030454, 0.021634312, 0.021634312, 0.003933511, 0.023601068, 0.02753458, 0.015734045, 0.011800534, 0.007867022, 0.059002668, 0.0177008, 0.0019667556, 0.007867022, 0.043268625, 0.049168892, 0.15537369, 0.23601067, 0.12980588, 0.009833778, 0, 0, 0.011800534, 0.08260374, 0.23601067, 0.17897476, 0.08457049, 0.029501334, 0.003933511, 0.007867022, 0.076703474, 0.023601068, 0.0177008, 0.0177008, 0.023601068, 0.029501334, 0.009833778, 0.047202136, 0.0708032] 4 +9778 [0.08063698, 0, 0, 0.013767289, 0.0019667556, 0.0019667556, 0.039335113, 0.13177262, 0.0177008, 0, 0, 0, 0, 0.060969424, 0.23601067, 0.119972095, 0.04916889, 0, 0, 0, 0, 0.019667557, 0.23601067, 0.177008, 0.06293618, 0, 0, 0.0019667556, 0.025567822, 0.021634312, 0.043268625, 0.09833778, 0.007867022, 0, 0.003933511, 0.18290827, 0.078670226, 0.029501334, 0.072769955, 0.0354016, 0.023601066, 0.003933511, 0.003933511, 0.037368357, 0.015734045, 0.08653725, 0.23601067, 0.04916889, 0.23601067, 0.009833778, 0, 0, 0, 0.003933511, 0.094404265, 0.19077529, 0.20060907, 0.027534578, 0.0059002666, 0.0059002666, 0.021634312, 0.0177008, 0.06686969, 0.08063698, 0, 0, 0.007867022, 0.23601067, 0.11013831, 0.0059002666, 0.007867022, 0.009833778, 0.011800533, 0.029501334, 0.072769955, 0.22814365, 0.055069156, 0, 0, 0.0059002666, 0.23601067, 0.23601067, 0.047202133, 0.011800533, 0.003933511, 0, 0.0019667556, 0.055069156, 0.10423805, 0.177008, 0.10030454, 0.021634312, 0.021634312, 0.003933511, 0.023601066, 0.027534578, 0.015734045, 0.011800533, 0.007867022, 0.059002668, 0.0177008, 0.0019667556, 0.007867022, 0.043268625, 0.04916889, 0.15537369, 0.23601067, 0.12980586, 0.009833778, 0, 0, 0.011800533, 0.08260374, 0.23601067, 0.17897476, 0.08457049, 0.029501334, 0.003933511, 0.007867022, 0.07670347, 0.023601066, 0.0177008, 0.0177008, 0.023601066, 0.029501334, 0.009833778, 0.047202133, 0.0708032] 4 drop table vector_ip_01; create table vector_cos_01(a int primary key, b vecf32(128),c int,key c_k(c)); insert into vector_cos_01 values(9774 ,"[1, 0, 1, 6, 6, 17, 47, 39, 2, 0, 1, 25, 27, 10, 56, 130, 18, 5, 2, 6, 15, 2, 19, 130, 42, 28, 1, 1, 2, 1, 0, 5, 0, 2, 4, 4, 31, 34, 44, 35, 9, 3, 8, 11, 33, 12, 61, 130, 130, 17, 0, 1, 6, 2, 9, 130, 111, 36, 0, 0, 11, 9, 1, 12, 2, 100, 130, 28, 7, 2, 6, 7, 9, 27, 130, 83, 5, 0, 1, 18, 130, 130, 84, 9, 0, 0, 2, 24, 111, 24, 0, 1, 37, 24, 2, 10, 12, 62, 33, 3, 0, 0, 0, 1, 3, 16, 106, 28, 0, 0, 0, 0, 17, 46, 85, 10, 0, 0, 1, 4, 11, 4, 2, 2, 9, 14, 8, 8]",3),(9775,"[0, 1, 1, 3, 0, 3, 46, 20, 1, 4, 17, 9, 1, 17, 108, 15, 0, 3, 37, 17, 6, 15, 116, 16, 6, 1, 4, 7, 7, 7, 9, 6, 0, 8, 10, 4, 26, 129, 27, 9, 0, 0, 5, 2, 11, 129, 129, 12, 103, 4, 0, 0, 2, 31, 129, 129, 94, 4, 0, 0, 0, 3, 13, 42, 0, 15, 38, 2, 70, 129, 1, 0, 5, 10, 40, 12, 74, 129, 6, 1, 129, 39, 6, 1, 2, 22, 9, 33, 122, 13, 0, 0, 0, 0, 5, 23, 4, 11, 9, 12, 45, 38, 1, 0, 0, 4, 36, 38, 57, 32, 0, 0, 82, 22, 9, 5, 13, 11, 3, 94, 35, 3, 0, 0, 0, 1, 16, 97]",5),(9776,"[10, 3, 8, 5, 48, 26, 5, 16, 17, 0, 0, 2, 132, 53, 1, 16, 112, 6, 0, 0, 7, 2, 1, 48, 48, 15, 18, 31, 3, 0, 0, 9, 6, 10, 19, 27, 50, 46, 17, 9, 18, 1, 4, 48, 132, 23, 3, 5, 132, 9, 4, 3, 11, 0, 2, 46, 84, 12, 10, 10, 1, 0, 12, 76, 26, 22, 16, 26, 35, 15, 3, 16, 15, 1, 51, 132, 125, 8, 1, 2, 132, 51, 67, 91, 8, 0, 0, 30, 126, 39, 32, 38, 4, 0, 1, 12, 24, 2, 2, 2, 4, 7, 2, 19, 93, 19, 70, 92, 2, 3, 1, 21, 36, 58, 132, 94, 0, 0, 0, 0, 21, 25, 57, 48, 1, 0, 0, 1]",3); @@ -282,9 +282,9 @@ Table Create Table vector_cos_01 CREATE TABLE `vector_cos_01` (\n `a` int NOT NULL,\n `b` vecf32(128) DEFAULT NULL,\n `c` int DEFAULT NULL,\n PRIMARY KEY (`a`),\n KEY `c_k` (`c`),\n KEY `idx01` USING ivfflat (`b`) lists = 5 op_type 'vector_cosine_ops' \n) desc vector_cos_01; Field Type Null Key Default Extra Comment -a INT(32) NO PRI null -b VECF32(128) YES MUL null -c INT(32) YES MUL null +a INT(32) NO PRI null +b VECF32(128) YES MUL null +c INT(32) YES MUL null select * from vector_cos_01 order by cosine_distance(b, "[16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]") ASC LIMIT 2; a b c 9777 [16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13] 4 @@ -296,7 +296,7 @@ a b c orderbyfn 9777 [16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13] 4 0.03196156024932861 select *, l2_distance(b, "[1, 15, 15, 0, 5, 7, 5, 5, 4, 0, 0, 0, 28, 1, 12, 5, 75, 20, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]") as orderbyfn from vector_cos_01 order by cosine_distance(b, "[1, 15, 15, 0, 5, 7, 5, 5, 4, 0, 0, 0, 28, 1, 12, 5, 75, 20, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]") ASC LIMIT 2; a b c orderbyfn -9777 [16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13] 4 127.4205551147461 +9777 [16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13] 4 127.42056274414062 select *, cosine_distance(b, "[2, 15, 15, 0, 5, 7, 5, 5, 4, 0, 0, 0, 28, 1, 12, 5, 75, 20, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]") as orderbyfn from vector_cos_01 order by cosine_distance(b, "[1, 15, 15, 0, 5, 7, 5, 5, 4, 0, 0, 0, 28, 1, 12, 5, 75, 20, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13]") ASC LIMIT 2; a b c orderbyfn 9777 [16, 15, 0, 0, 5, 46, 5, 5, 4, 0, 0, 0, 28, 118, 12, 5, 75, 44, 5, 0, 6, 32, 6, 49, 41, 74, 9, 1, 0, 0, 0, 9, 1, 9, 16, 41, 71, 80, 3, 0, 0, 4, 3, 5, 51, 106, 11, 3, 112, 28, 13, 1, 4, 8, 3, 104, 118, 14, 1, 1, 0, 0, 0, 88, 3, 27, 46, 118, 108, 49, 2, 0, 1, 46, 118, 118, 27, 12, 0, 0, 33, 118, 118, 8, 0, 0, 0, 4, 118, 95, 40, 0, 0, 0, 1, 11, 27, 38, 12, 12, 18, 29, 3, 2, 13, 30, 94, 78, 30, 19, 9, 3, 31, 45, 70, 42, 15, 1, 3, 12, 14, 22, 16, 2, 3, 17, 24, 13] 4 0.031903373234243526 diff --git a/test/distributed/cases/vector/vector_ivf_mode.result b/test/distributed/cases/vector/vector_ivf_mode.result index 46e06670dfd52..9dee73906ac5f 100644 --- a/test/distributed/cases/vector/vector_ivf_mode.result +++ b/test/distributed/cases/vector/vector_ivf_mode.result @@ -409,9 +409,9 @@ LIMIT 1 by rank with option 'mode=pre') ORDER BY dist LIMIT 4; id category dist 1 A 0.14142133490046585 -2 A 0.14142136782769238 -3 B 0.8485281542825963 -8 B 1.1045360607064578 +2 A 0.1414213612422477 +3 B 0.8485281894049685 +4 B 1.1045360607064578 WITH q1 AS ( SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist FROM mini_vector_data diff --git a/test/distributed/cases/vector/vector_ivf_mode.sql b/test/distributed/cases/vector/vector_ivf_mode.sql index a7af074467d5e..908aec4264b27 100644 --- a/test/distributed/cases/vector/vector_ivf_mode.sql +++ b/test/distributed/cases/vector/vector_ivf_mode.sql @@ -241,106 +241,106 @@ LIMIT 3 by rank with option 'mode=pre'; -- ============================================================================ -- Test Case: Simple UNION with mode=pre on same table -(SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist - FROM mini_vector_data +(SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist + FROM mini_vector_data WHERE id IN ('id1', 'id2', 'id3') - ORDER BY l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') + ORDER BY l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') LIMIT 2 by rank with option 'mode=pre') UNION -(SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist - FROM mini_vector_data +(SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist + FROM mini_vector_data WHERE id IN ('id7', 'id8', 'id9') - ORDER BY l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') + ORDER BY l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') LIMIT 2 by rank with option 'mode=pre') ORDER BY dist LIMIT 3; -- Test Case: UNION ALL with mode=pre (allows duplicates) -(SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist - FROM mini_vector_data +(SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist + FROM mini_vector_data WHERE id LIKE 'id%' - ORDER BY l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') + ORDER BY l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') LIMIT 3 by rank with option 'mode=pre') UNION ALL -(SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist - FROM mini_vector_data +(SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist + FROM mini_vector_data WHERE text LIKE '%test%' - ORDER BY l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') + ORDER BY l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') LIMIT 2 by rank with option 'mode=pre') ORDER BY dist LIMIT 5; -- Test Case: UNION with mode=pre and mode=post (mixed modes) -(SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist - FROM mini_vector_data +(SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist + FROM mini_vector_data WHERE id IN ('id1', 'id2', 'id3', 'id4') - ORDER BY l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') + ORDER BY l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') LIMIT 2 by rank with option 'mode=pre') UNION -(SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist - FROM mini_vector_data +(SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist + FROM mini_vector_data WHERE id IN ('id6', 'id7', 'id8', 'id9') - ORDER BY l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') + ORDER BY l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') LIMIT 2 by rank with option 'mode=post') ORDER BY dist LIMIT 4; -- Test Case: UNION with mode=pre on different tables (mini_vector_data and mini_embed_data) (SELECT id, text AS content - FROM mini_vector_data + FROM mini_vector_data ORDER BY id, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') LIMIT 2 by rank with option 'mode=pre') UNION (SELECT id, content - FROM mini_embed_data - ORDER BY cosine_distance(embedding, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') + FROM mini_embed_data + ORDER BY cosine_distance(embedding, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') LIMIT 2 by rank with option 'mode=pre') LIMIT 3; -- Test Case: UNION with mode=pre and complex WHERE conditions -(SELECT id, category, l2_distance(vec, '[0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]') AS dist - FROM vec_with_multi_idx +(SELECT id, category, l2_distance(vec, '[0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]') AS dist + FROM vec_with_multi_idx WHERE category = 'A' AND status = 1 - ORDER BY l2_distance(vec, '[0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]') + ORDER BY l2_distance(vec, '[0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]') LIMIT 2 by rank with option 'mode=pre') UNION -(SELECT id, category, l2_distance(vec, '[0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]') AS dist - FROM vec_with_multi_idx +(SELECT id, category, l2_distance(vec, '[0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]') AS dist + FROM vec_with_multi_idx WHERE category = 'B' AND status = 1 - ORDER BY l2_distance(vec, '[0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]') + ORDER BY l2_distance(vec, '[0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9]') LIMIT 2 by rank with option 'mode=pre') ORDER BY dist LIMIT 3; -- Test Case: Three-way UNION with mode=pre -(SELECT id, category, l2_distance(vec, '[0.15,0.25,0.35,0.45,0.55,0.65,0.75,0.85]') AS dist - FROM vec_with_regular_idx +(SELECT id, category, l2_distance(vec, '[0.15,0.25,0.35,0.45,0.55,0.65,0.75,0.85]') AS dist + FROM vec_with_regular_idx WHERE category = 'A' - ORDER BY l2_distance(vec, '[0.15,0.25,0.35,0.45,0.55,0.65,0.75,0.85]') + ORDER BY l2_distance(vec, '[0.15,0.25,0.35,0.45,0.55,0.65,0.75,0.85]') LIMIT 2 by rank with option 'mode=pre') UNION -(SELECT id, category, l2_distance(vec, '[0.15,0.25,0.35,0.45,0.55,0.65,0.75,0.85]') AS dist - FROM vec_with_regular_idx +(SELECT id, category, l2_distance(vec, '[0.15,0.25,0.35,0.45,0.55,0.65,0.75,0.85]') AS dist + FROM vec_with_regular_idx WHERE category = 'B' - ORDER BY l2_distance(vec, '[0.15,0.25,0.35,0.45,0.55,0.65,0.75,0.85]') + ORDER BY l2_distance(vec, '[0.15,0.25,0.35,0.45,0.55,0.65,0.75,0.85]') LIMIT 2 by rank with option 'mode=pre') UNION -(SELECT id, category, l2_distance(vec, '[0.15,0.25,0.35,0.45,0.55,0.65,0.75,0.85]') AS dist - FROM vec_with_regular_idx +(SELECT id, category, l2_distance(vec, '[0.15,0.25,0.35,0.45,0.55,0.65,0.75,0.85]') AS dist + FROM vec_with_regular_idx WHERE category = 'C' - ORDER BY l2_distance(vec, '[0.15,0.25,0.35,0.45,0.55,0.65,0.75,0.85]') + ORDER BY l2_distance(vec, '[0.15,0.25,0.35,0.45,0.55,0.65,0.75,0.85]') LIMIT 1 by rank with option 'mode=pre') ORDER BY dist LIMIT 4; -- Test Case: UNION with mode=pre using CTE (WITH clause) WITH q1 AS ( - SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist - FROM mini_vector_data + SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist + FROM mini_vector_data WHERE id IN ('id1', 'id2', 'id3') - ORDER BY dist + ORDER BY dist LIMIT 2 by rank with option 'mode=pre' ), q2 AS ( - SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist - FROM mini_vector_data + SELECT id, text, l2_distance(vec, '[0.1,-0.2,0.3,0.4,-0.1,0.2,0.0,0.5]') AS dist + FROM mini_vector_data WHERE id IN ('id8', 'id9', 'id10') - ORDER BY dist + ORDER BY dist LIMIT 2 by rank with option 'mode=pre' ) SELECT * FROM q1 @@ -350,77 +350,77 @@ ORDER BY dist LIMIT 3; -- Test Case: UNION ALL with mode=pre and different LIMIT sizes (testing over-fetch) (SELECT id, content - FROM mini_embed_data + FROM mini_embed_data WHERE file_id = 'file01' - ORDER BY cosine_distance(embedding, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') + ORDER BY cosine_distance(embedding, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') LIMIT 1 by rank with option 'mode=pre') UNION ALL (SELECT id, content - FROM mini_embed_data + FROM mini_embed_data WHERE file_id = 'file02' - ORDER BY cosine_distance(embedding, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') + ORDER BY cosine_distance(embedding, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') LIMIT 2 by rank with option 'mode=pre') LIMIT 3; -- Test Case: UNION with mode=pre and DESC ordering -(SELECT id, text, l2_distance(vec, '[0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5]') AS dist - FROM mini_vector_data +(SELECT id, text, l2_distance(vec, '[0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5]') AS dist + FROM mini_vector_data WHERE id LIKE 'id1%' ORDER BY l2_distance(vec, '[0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5]') DESC LIMIT 2 by rank with option 'mode=pre') UNION -(SELECT id, text, l2_distance(vec, '[0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5]') AS dist - FROM mini_vector_data +(SELECT id, text, l2_distance(vec, '[0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5]') AS dist + FROM mini_vector_data WHERE id LIKE 'id%' AND id NOT LIKE 'id1%' ORDER BY l2_distance(vec, '[0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5]') DESC LIMIT 3 by rank with option 'mode=pre') ORDER BY dist DESC LIMIT 4; -- Test Case: UNION with mode=pre and OFFSET -(SELECT id, category, status, l2_distance(vec, '[0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.1]') AS dist - FROM vec_with_multi_idx +(SELECT id, category, status, l2_distance(vec, '[0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.1]') AS dist + FROM vec_with_multi_idx WHERE status = 1 - ORDER BY l2_distance(vec, '[0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.1]') + ORDER BY l2_distance(vec, '[0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.1]') LIMIT 3 OFFSET 1 by rank with option 'mode=pre') UNION -(SELECT id, category, status, l2_distance(vec, '[0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.1]') AS dist - FROM vec_with_multi_idx +(SELECT id, category, status, l2_distance(vec, '[0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.1]') AS dist + FROM vec_with_multi_idx WHERE status = 2 - ORDER BY l2_distance(vec, '[0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.1]') + ORDER BY l2_distance(vec, '[0.3,0.4,0.5,0.6,0.7,0.8,0.9,0.1]') LIMIT 2 by rank with option 'mode=pre') ORDER BY dist LIMIT 4; -- Test Case: UNION with mode=pre, mode=post, and mode=force (all three modes) -(SELECT id, text, l2_distance(vec, '[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]') AS dist - FROM mini_vector_data +(SELECT id, text, l2_distance(vec, '[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]') AS dist + FROM mini_vector_data WHERE id IN ('id1', 'id2') - ORDER BY l2_distance(vec, '[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]') + ORDER BY l2_distance(vec, '[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]') LIMIT 1 by rank with option 'mode=pre') UNION ALL -(SELECT id, text, l2_distance(vec, '[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]') AS dist - FROM mini_vector_data +(SELECT id, text, l2_distance(vec, '[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]') AS dist + FROM mini_vector_data WHERE id IN ('id3', 'id4') - ORDER BY l2_distance(vec, '[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]') + ORDER BY l2_distance(vec, '[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]') LIMIT 1 by rank with option 'mode=post') UNION ALL -(SELECT id, text, l2_distance(vec, '[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]') AS dist - FROM mini_vector_data +(SELECT id, text, l2_distance(vec, '[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]') AS dist + FROM mini_vector_data WHERE id IN ('id5', 'id6') - ORDER BY l2_distance(vec, '[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]') + ORDER BY l2_distance(vec, '[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]') LIMIT 1 by rank with option 'mode=force') ORDER BY dist LIMIT 3; -- Test Case: UNION with composite index and mode=pre -(SELECT id, category, status, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') AS dist - FROM vec_with_composite_idx +(SELECT id, category, status, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') AS dist + FROM vec_with_composite_idx WHERE category = 'A' AND status = 1 - ORDER BY id, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') + ORDER BY id, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') LIMIT 1 by rank with option 'mode=pre') UNION -(SELECT id, category, status, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') AS dist - FROM vec_with_composite_idx +(SELECT id, category, status, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') AS dist + FROM vec_with_composite_idx WHERE category = 'B' - ORDER BY id, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') + ORDER BY id, l2_distance(vec, '[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8]') LIMIT 2 by rank with option 'mode=pre') ORDER BY dist LIMIT 3; diff --git a/thirdparties/Makefile b/thirdparties/Makefile index b25a4f48b9bd5..32cab28a698ae 100644 --- a/thirdparties/Makefile +++ b/thirdparties/Makefile @@ -15,11 +15,11 @@ PWD=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) UNAME_S=$(shell uname -s | tr A-Z a-z) UNAME_M=$(shell uname -m) -USEARCH_DIR=USearch-2.21.1 -USEARCH_TAR=usearch-2.21.1.tar.gz -STRINGZILLA_DIR=StringZilla-4.2.1 +USEARCH_DIR=USearch-2.21.4 +USEARCH_TAR=usearch-2.21.4.tar.gz +STRINGZILLA_DIR=StringZilla-4.4.2 STRINGZILLA_TAR=$(STRINGZILLA_DIR).tar.gz -SIMSIMD_DIR=SimSIMD-6.5.3 +SIMSIMD_DIR=SimSIMD-6.5.5 SIMSIMD_TAR=$(SIMSIMD_DIR).tar.gz FP16_DIR=fp16 FP16_TAR=$(FP16_DIR).tar.gz diff --git a/thirdparties/SimSIMD-6.5.3.tar.gz b/thirdparties/SimSIMD-6.5.3.tar.gz deleted file mode 100644 index 4195983878991..0000000000000 Binary files a/thirdparties/SimSIMD-6.5.3.tar.gz and /dev/null differ diff --git a/thirdparties/SimSIMD-6.5.5.tar.gz b/thirdparties/SimSIMD-6.5.5.tar.gz new file mode 100644 index 0000000000000..4b6503c214d17 Binary files /dev/null and b/thirdparties/SimSIMD-6.5.5.tar.gz differ diff --git a/thirdparties/StringZilla-4.2.1.tar.gz b/thirdparties/StringZilla-4.2.1.tar.gz deleted file mode 100644 index 73be36cc9fbb3..0000000000000 Binary files a/thirdparties/StringZilla-4.2.1.tar.gz and /dev/null differ diff --git a/thirdparties/StringZilla-4.4.2.tar.gz b/thirdparties/StringZilla-4.4.2.tar.gz new file mode 100644 index 0000000000000..816781e83dbcf Binary files /dev/null and b/thirdparties/StringZilla-4.4.2.tar.gz differ diff --git a/thirdparties/usearch-2.21.1.tar.gz b/thirdparties/usearch-2.21.1.tar.gz deleted file mode 100644 index 3e70aaf663688..0000000000000 Binary files a/thirdparties/usearch-2.21.1.tar.gz and /dev/null differ diff --git a/thirdparties/usearch-2.21.4.tar.gz b/thirdparties/usearch-2.21.4.tar.gz new file mode 100644 index 0000000000000..2b5644c0ceb54 Binary files /dev/null and b/thirdparties/usearch-2.21.4.tar.gz differ