Merge branch 'master' of https://github.com/milvus-io/milvus into 241…

…2-opt-index-meta-mutex
milvus-io · Jan 23, 2025 · 00cc771 · 00cc771
2 parents 5cb0180 + 70c53f7
commit 00cc771
Show file tree

Hide file tree

Showing 206 changed files with 6,160 additions and 2,899 deletions.
diff --git a/.env b/.env
@@ -5,8 +5,8 @@ IMAGE_ARCH=amd64
 OS_NAME=ubuntu22.04
 
 # for services.builder.image in docker-compose.yml
-DATE_VERSION=20240816-1275005
-LATEST_DATE_VERSION=20240816-1275005
+DATE_VERSION=20250120-cb959cd
+LATEST_DATE_VERSION=20250120-cb959cd
 # for services.gpubuilder.image in docker-compose.yml
 GPU_DATE_VERSION=20240806-d8668fe
 LATEST_GPU_DATE_VERSION=20240806-d8668fe

diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml
@@ -67,7 +67,7 @@ jobs:
           fi
           make milvus
       - name: Upload Cmake log
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ failure() }}
         with:
           name: cmake-log

diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -100,7 +100,7 @@ jobs:
       - run: |
           zip -r code.zip . -x "./.docker/*" -x "./cmake_build/thirdparty/**" -x ".git/**"
       - name: Archive code
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: code
           path: code.zip
@@ -129,7 +129,7 @@ jobs:
           remove-android: 'true'
           remove-haskell: 'true'
       - name: Download code
-        uses: actions/download-artifact@v3.0.1
+        uses: actions/download-artifact@v4
         with:
           name: code
       - run: |
@@ -181,7 +181,7 @@ jobs:
           remove-android: 'true'
           remove-haskell: 'true'
       - name: Download code
-        uses: actions/download-artifact@v3.0.1
+        uses: actions/download-artifact@v4
         with:
           name: code
       - run: |
@@ -233,7 +233,7 @@ jobs:
           remove-android: 'true'
           remove-haskell: 'true'
       - name: Download code
-        uses: actions/download-artifact@v3.0.1
+        uses: actions/download-artifact@v4
         with:
           name: code
       - run: |

diff --git a/.gitignore b/.gitignore
@@ -101,6 +101,8 @@ cwrapper_rocksdb_build/
 **/data/*
 
 internal/proto/**/*.pb.go
+pkg/streaming/**/*.pb.go
 internal/core/src/pb/*.pb.h
 internal/core/src/pb/*.pb.cc
 **/legacypb/*.pb.go
+pkg/streaming/proto/**/*.pb.go
diff --git a/README.md b/README.md
@@ -244,7 +244,7 @@ Reference to cite when you use Milvus in a research paper:
 }
 ```
 <!-- Do not remove start of hero-bot -->
-<img src="https://img.shields.io/badge/all--contributors-427-orange"><br>
+<img src="https://img.shields.io/badge/all--contributors-429-orange"><br>
 <a href="https://github.com/0xflotus"><img src="https://avatars.githubusercontent.com/u/26602940?v=4" width="30px" /></a>
 <a href="https://github.com/ABNER-1"><img src="https://avatars.githubusercontent.com/u/24547351?v=4" width="30px" /></a>
 <a href="https://github.com/Abhijnan-Bajpai"><img src="https://avatars.githubusercontent.com/u/57059194?v=4" width="30px" /></a>
@@ -272,6 +272,7 @@ Reference to cite when you use Milvus in a research paper:
 <a href="https://github.com/CodeInDreams"><img src="https://avatars.githubusercontent.com/u/17664279?v=4" width="30px" /></a>
 <a href="https://github.com/CsterKuroi"><img src="https://avatars.githubusercontent.com/u/12230174?v=4" width="30px" /></a>
 <a href="https://github.com/Cupchen"><img src="https://avatars.githubusercontent.com/u/34762375?v=4" width="30px" /></a>
+<a href="https://github.com/DLT1412"><img src="https://avatars.githubusercontent.com/u/8294986?v=4" width="30px" /></a>
 <a href="https://github.com/DanielHuang1983"><img src="https://avatars.githubusercontent.com/u/4417873?v=4" width="30px" /></a>
 <a href="https://github.com/Deep1Shikha"><img src="https://avatars.githubusercontent.com/u/47516502?v=4" width="30px" /></a>
 <a href="https://github.com/DerekChia"><img src="https://avatars.githubusercontent.com/u/1457728?v=4" width="30px" /></a>
@@ -448,6 +449,7 @@ Reference to cite when you use Milvus in a research paper:
 <a href="https://github.com/freestsoul"><img src="https://avatars.githubusercontent.com/u/3909908?v=4" width="30px" /></a>
 <a href="https://github.com/gcmutator"><img src="https://avatars.githubusercontent.com/u/134900551?v=4" width="30px" /></a>
 <a href="https://github.com/ggaaooppeenngg"><img src="https://avatars.githubusercontent.com/u/4769989?v=4" width="30px" /></a>
+<a href="https://github.com/gifi-siby"><img src="https://avatars.githubusercontent.com/u/176461003?v=4" width="30px" /></a>
 <a href="https://github.com/git-hulk"><img src="https://avatars.githubusercontent.com/u/4987594?v=4" width="30px" /></a>
 <a href="https://github.com/godchen0212"><img src="https://avatars.githubusercontent.com/u/67679556?v=4" width="30px" /></a>
 <a href="https://github.com/goodhamgupta"><img src="https://avatars.githubusercontent.com/u/14368181?v=4" width="30px" /></a>

diff --git a/README_CN.md b/README_CN.md
@@ -183,7 +183,7 @@ Milvus [训练营](https://github.com/milvus-io/bootcamp)能够帮助你了解
 ### All contributors
 
 <br><!-- Do not remove start of hero-bot -->
-<img src="https://img.shields.io/badge/all--contributors-427-orange"><br>
+<img src="https://img.shields.io/badge/all--contributors-429-orange"><br>
 <a href="https://github.com/0xflotus"><img src="https://avatars.githubusercontent.com/u/26602940?v=4" width="30px" /></a>
 <a href="https://github.com/ABNER-1"><img src="https://avatars.githubusercontent.com/u/24547351?v=4" width="30px" /></a>
 <a href="https://github.com/Abhijnan-Bajpai"><img src="https://avatars.githubusercontent.com/u/57059194?v=4" width="30px" /></a>
@@ -211,6 +211,7 @@ Milvus [训练营](https://github.com/milvus-io/bootcamp)能够帮助你了解
 <a href="https://github.com/CodeInDreams"><img src="https://avatars.githubusercontent.com/u/17664279?v=4" width="30px" /></a>
 <a href="https://github.com/CsterKuroi"><img src="https://avatars.githubusercontent.com/u/12230174?v=4" width="30px" /></a>
 <a href="https://github.com/Cupchen"><img src="https://avatars.githubusercontent.com/u/34762375?v=4" width="30px" /></a>
+<a href="https://github.com/DLT1412"><img src="https://avatars.githubusercontent.com/u/8294986?v=4" width="30px" /></a>
 <a href="https://github.com/DanielHuang1983"><img src="https://avatars.githubusercontent.com/u/4417873?v=4" width="30px" /></a>
 <a href="https://github.com/Deep1Shikha"><img src="https://avatars.githubusercontent.com/u/47516502?v=4" width="30px" /></a>
 <a href="https://github.com/DerekChia"><img src="https://avatars.githubusercontent.com/u/1457728?v=4" width="30px" /></a>
@@ -387,6 +388,7 @@ Milvus [训练营](https://github.com/milvus-io/bootcamp)能够帮助你了解
 <a href="https://github.com/freestsoul"><img src="https://avatars.githubusercontent.com/u/3909908?v=4" width="30px" /></a>
 <a href="https://github.com/gcmutator"><img src="https://avatars.githubusercontent.com/u/134900551?v=4" width="30px" /></a>
 <a href="https://github.com/ggaaooppeenngg"><img src="https://avatars.githubusercontent.com/u/4769989?v=4" width="30px" /></a>
+<a href="https://github.com/gifi-siby"><img src="https://avatars.githubusercontent.com/u/176461003?v=4" width="30px" /></a>
 <a href="https://github.com/git-hulk"><img src="https://avatars.githubusercontent.com/u/4987594?v=4" width="30px" /></a>
 <a href="https://github.com/godchen0212"><img src="https://avatars.githubusercontent.com/u/67679556?v=4" width="30px" /></a>
 <a href="https://github.com/goodhamgupta"><img src="https://avatars.githubusercontent.com/u/14368181?v=4" width="30px" /></a>

diff --git a/build/docker/builder/cpu/amazonlinux2023/Dockerfile b/build/docker/builder/cpu/amazonlinux2023/Dockerfile
@@ -58,7 +58,7 @@ RUN mkdir -p /home/milvus/.vscode-server/extensions \
 COPY --chown=0:0 build/docker/builder/entrypoint.sh /
 
 RUN curl https://sh.rustup.rs -sSf | \
-    sh -s -- --default-toolchain=1.73 -y
+    sh -s -- --default-toolchain=1.83 -y
 
 ENV PATH=/root/.cargo/bin:$PATH
 

diff --git a/build/docker/builder/cpu/rockylinux8/Dockerfile b/build/docker/builder/cpu/rockylinux8/Dockerfile
@@ -45,7 +45,7 @@ RUN dnf -y update && \
 RUN pip3 install conan==1.64.1
 RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.11.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go 
 RUN curl https://sh.rustup.rs -sSf | \
-    sh -s -- --default-toolchain=1.73 -y
+    sh -s -- --default-toolchain=1.83 -y
 
 ENV PATH=/root/.cargo/bin:/usr/local/bin:/usr/local/go/bin:$PATH
 

diff --git a/build/docker/builder/cpu/ubuntu20.04/Dockerfile b/build/docker/builder/cpu/ubuntu20.04/Dockerfile
@@ -53,7 +53,7 @@ RUN mkdir -p /home/milvus/.vscode-server/extensions \
 COPY --chown=0:0 build/docker/builder/entrypoint.sh /
 
 RUN curl https://sh.rustup.rs -sSf | \
-    sh -s -- --default-toolchain=1.73 -y
+    sh -s -- --default-toolchain=1.83 -y
 
 ENV PATH=/root/.cargo/bin:$PATH
 

diff --git a/build/docker/builder/cpu/ubuntu22.04/Dockerfile b/build/docker/builder/cpu/ubuntu22.04/Dockerfile
@@ -59,7 +59,7 @@ RUN mkdir -p /home/milvus/.vscode-server/extensions \
 COPY --chown=0:0 build/docker/builder/entrypoint.sh /
 
 RUN curl https://sh.rustup.rs -sSf | \
-    sh -s -- --default-toolchain=1.73 -y
+    sh -s -- --default-toolchain=1.83 -y
 
 ENV PATH=/root/.cargo/bin:$PATH
 

diff --git a/build/docker/builder/gpu/ubuntu20.04/Dockerfile b/build/docker/builder/gpu/ubuntu20.04/Dockerfile
@@ -87,7 +87,7 @@ RUN wget -O /tini https://github.com/krallin/tini/releases/download/v0.19.0/tini
     chmod +x /tini
 
 RUN curl https://sh.rustup.rs -sSf | \
-    sh -s -- --default-toolchain=1.73 -y
+    sh -s -- --default-toolchain=1.83 -y
 
 ENV PATH=/root/.cargo/bin:$PATH
 

diff --git a/build/docker/builder/gpu/ubuntu22.04/Dockerfile b/build/docker/builder/gpu/ubuntu22.04/Dockerfile
@@ -18,7 +18,7 @@ RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.11.linux-$TARG
 RUN pip3 install conan==1.64.1
 # Install rust
 RUN curl https://sh.rustup.rs -sSf | \
-    sh -s -- --default-toolchain=1.73 -y
+    sh -s -- --default-toolchain=1.83 -y
 ENV PATH=/root/.cargo/bin:/usr/local/bin:/usr/local/go/bin:$PATH
 
 RUN mkdir /opt/vcpkg &&  \

diff --git a/client/milvusclient/read_options.go b/client/milvusclient/read_options.go
@@ -523,9 +523,10 @@ func (opt *queryOption) Request() (*milvuspb.QueryRequest, error) {
 		PartitionNames: opt.partitionNames,
 		OutputFields:   opt.outputFields,
 
-		Expr:             opt.expr,
-		QueryParams:      entity.MapKvPairs(opt.queryParams),
-		ConsistencyLevel: opt.consistencyLevel.CommonConsistencyLevel(),
+		Expr:                  opt.expr,
+		QueryParams:           entity.MapKvPairs(opt.queryParams),
+		ConsistencyLevel:      opt.consistencyLevel.CommonConsistencyLevel(),
+		UseDefaultConsistency: opt.useDefaultConsistencyLevel,
 	}
 
 	req.ExprTemplateValues = make(map[string]*schemapb.TemplateValue)

diff --git a/configs/milvus.yaml b/configs/milvus.yaml
@@ -176,6 +176,9 @@ mq:
     mergeCheckInterval: 1 # the interval time(in seconds) for dispatcher to check whether to merge
     targetBufSize: 16 # the lenth of channel buffer for targe
     maxTolerantLag: 3 # Default value: "3", the timeout(in seconds) that target sends msgPack
+    maxDispatcherNumPerPchannel: 5 # The maximum number of dispatchers per physical channel, primarily to limit the number of consumers and prevent performance issues(e.g., during recovery when a large number of channels are watched).
+    retrySleep: 3 # register retry sleep time in seconds
+    retryTimeout: 60 # register retry timeout in seconds
 
 # Related configuration of pulsar, used to manage Milvus logs of recent mutation operations, output streaming log, and provide log publish-subscribe services.
 pulsar:
@@ -520,7 +523,6 @@ dataCoord:
     balanceInterval: 360 # The interval with which the channel manager check dml channel balance status
     checkInterval: 1 # The interval in seconds with which the channel manager advances channel states
     notifyChannelOperationTimeout: 5 # Timeout notifing channel operations (in seconds).
-    maxConcurrentChannelTaskNumPerDN: 32 # The maximum concurrency for each DataNode executing channel tasks (watch, release).
   segment:
     maxSize: 1024 # The maximum size of a segment, unit: MB. datacoord.segment.maxSize and datacoord.segment.sealProportion together determine if a segment can be sealed.
     diskSegmentMaxSize: 2048 # Maximun size of a segment in MB for collection which has Disk index
@@ -570,6 +572,7 @@ dataCoord:
     maxParallelTaskNum: -1 # Deprecated, see datanode.slot.slotCap
     dropTolerance: 86400 # Compaction task will be cleaned after finish longer than this time(in seconds)
     gcInterval: 1800 # The time interval in seconds for compaction gc
+    scheduleInterval: 500 # The time interval in milliseconds for scheduling compaction tasks. If the configuration setting is below 100ms, it will be ajusted upwards to 100ms
     mix:
       triggerInterval: 60 # The time interval in seconds to trigger mix compaction
     levelzero:
@@ -828,6 +831,7 @@ common:
     # like the old password verification when updating the credential
     superUsers: 
     defaultRootPassword: "Milvus" # default password for root user. The maximum length is 72 characters, and double quotes are required.
+    rootShouldBindRole: false # Whether the root user should bind a role when the authorization is enabled.
     rbac:
       overrideBuiltInPrivilegeGroups:
         enabled: false # Whether to override build-in privilege groups

diff --git a/internal/core/build-support/add_cmake_license.sh b/internal/core/build-support/add_cmake_license.sh
@@ -9,7 +9,7 @@ fi
 cat ${LICENSE} > /dev/null || exit -1
 
 FILES=`find ${FOLDER} \
-| grep -E "(*CMakeLists.txt$)" \
+| grep "CMakeLists.txt$" \
 | grep -v thirdparty \
 | grep -v cmake_build \
 | grep -v cmake-build \

diff --git a/internal/core/build-support/add_cpp_license.sh b/internal/core/build-support/add_cpp_license.sh
@@ -9,7 +9,7 @@ fi
 cat ${LICENSE} > /dev/null || exit -1
 
 FILES=`find ${FOLDER} \
-| grep -E "(*\.cpp$|*\.h$|*\.cu$)" \
+| grep "(\.cpp\|\.h\|\.cu)$" \
 | grep -v thirdparty \
 | grep -v cmake_build \
 | grep -v cmake-build \

diff --git a/internal/core/run_clang_format.sh b/internal/core/run_clang_format.sh
@@ -6,8 +6,10 @@ else
 fi
 CorePath=$1
 
+CLANG_FORMAT=clang-format-12
+
 formatThis() {
-    find "$1" | grep -E "(*\.cpp|*\.h|*\.cc)$" | grep -v "gen_tools/templates" | grep -v "\.pb\." | grep -v "tantivy-binding.h" | xargs clang-format-12 -i
+    find "$1" | grep "(\.cpp\|\.h\|\.cc)$" | grep -v "gen_tools/templates" | grep -v "\.pb\." | grep -v "tantivy-binding.h" | xargs $CLANG_FORMAT -i
 }
 
 formatThis "${CorePath}/src"

diff --git a/internal/core/src/clustering/KmeansClustering.h b/internal/core/src/clustering/KmeansClustering.h
@@ -58,20 +58,20 @@ class KmeansClustering {
         return cluster_result_;
     }
 
-    // ut
     inline std::string
     GetRemoteCentroidsObjectPrefix() const {
         auto index_meta_ = file_manager_->GetIndexMeta();
         auto field_meta_ = file_manager_->GetFieldDataMeta();
         boost::filesystem::path prefix =
             file_manager_->GetChunkManager()->GetRootPath();
-        boost::filesystem::path path =
+        boost::filesystem::path path = std::string(ANALYZE_ROOT_PATH);
+        boost::filesystem::path path1 =
             std::to_string(index_meta_.build_id) + "/" +
             std::to_string(index_meta_.index_version) + "/" +
             std::to_string(field_meta_.collection_id) + "/" +
             std::to_string(field_meta_.partition_id) + "/" +
             std::to_string(field_meta_.field_id);
-        return (prefix / path).string();
+        return (prefix / path / path1).string();
     }
 
     inline std::string

diff --git a/internal/core/src/common/ChunkWriter.cpp b/internal/core/src/common/ChunkWriter.cpp
@@ -356,13 +356,14 @@ create_chunk(const FieldMeta& field_meta,
         }
         case milvus::DataType::VECTOR_FLOAT: {
             w = std::make_shared<
-                ChunkWriter<arrow::FixedSizeBinaryArray, float>>(dim, nullable);
+                ChunkWriter<arrow::FixedSizeBinaryArray, knowhere::fp32>>(
+                dim, nullable);
             break;
         }
         case milvus::DataType::VECTOR_BINARY: {
             w = std::make_shared<
-                ChunkWriter<arrow::FixedSizeBinaryArray, uint8_t>>(dim / 8,
-                                                                   nullable);
+                ChunkWriter<arrow::FixedSizeBinaryArray, knowhere::bin1>>(
+                dim / 8, nullable);
             break;
         }
         case milvus::DataType::VECTOR_FLOAT16: {
@@ -377,6 +378,12 @@ create_chunk(const FieldMeta& field_meta,
                 dim, nullable);
             break;
         }
+        case milvus::DataType::VECTOR_INT8: {
+            w = std::make_shared<
+                ChunkWriter<arrow::FixedSizeBinaryArray, knowhere::int8>>(
+                dim, nullable);
+            break;
+        }
         case milvus::DataType::VARCHAR:
         case milvus::DataType::STRING: {
             w = std::make_shared<StringChunkWriter>(nullable);
@@ -450,13 +457,13 @@ create_chunk(const FieldMeta& field_meta,
         }
         case milvus::DataType::VECTOR_FLOAT: {
             w = std::make_shared<
-                ChunkWriter<arrow::FixedSizeBinaryArray, float>>(
+                ChunkWriter<arrow::FixedSizeBinaryArray, knowhere::fp32>>(
                 dim, file, file_offset, nullable);
             break;
         }
         case milvus::DataType::VECTOR_BINARY: {
             w = std::make_shared<
-                ChunkWriter<arrow::FixedSizeBinaryArray, uint8_t>>(
+                ChunkWriter<arrow::FixedSizeBinaryArray, knowhere::bin1>>(
                 dim / 8, file, file_offset, nullable);
             break;
         }
@@ -472,6 +479,12 @@ create_chunk(const FieldMeta& field_meta,
                 dim, file, file_offset, nullable);
             break;
         }
+        case milvus::DataType::VECTOR_INT8: {
+            w = std::make_shared<
+                ChunkWriter<arrow::FixedSizeBinaryArray, knowhere::int8>>(
+                dim, file, file_offset, nullable);
+            break;
+        }
         case milvus::DataType::VARCHAR:
         case milvus::DataType::STRING: {
             w = std::make_shared<StringChunkWriter>(

diff --git a/internal/core/src/common/FieldData.cpp b/internal/core/src/common/FieldData.cpp
@@ -238,6 +238,7 @@ FieldDataImpl<Type, is_type_entire_row>::FillFieldData(
         case DataType::VECTOR_FLOAT:
         case DataType::VECTOR_FLOAT16:
         case DataType::VECTOR_BFLOAT16:
+        case DataType::VECTOR_INT8:
         case DataType::VECTOR_BINARY: {
             auto array_info =
                 GetDataInfoFromArray<arrow::FixedSizeBinaryArray,

diff --git a/internal/core/src/common/FieldData.h b/internal/core/src/common/FieldData.h
@@ -141,6 +141,17 @@ class FieldData<SparseFloatVector> : public FieldDataSparseVectorImpl {
     }
 };
 
+template <>
+class FieldData<Int8Vector> : public FieldDataImpl<int8, false> {
+ public:
+    explicit FieldData(int64_t dim,
+                       DataType data_type,
+                       int64_t buffered_num_rows = 0)
+        : FieldDataImpl<int8, false>::FieldDataImpl(
+              dim, data_type, false, buffered_num_rows) {
+    }
+};
+
 using FieldDataPtr = std::shared_ptr<FieldDataBase>;
 using FieldDataChannel = Channel<FieldDataPtr>;
 using FieldDataChannelPtr = std::shared_ptr<FieldDataChannel>;