[HUDI-8769] Limit instantsToArchive to avoid expensive locks blocking ingestion and table services

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v1/TimelineArchiverV1.java

@@ -80,6 +80,7 @@
 import static org.apache.hudi.common.table.timeline.InstantComparison.LESSER_THAN;
 import static org.apache.hudi.common.table.timeline.InstantComparison.LESSER_THAN_OR_EQUALS;
 import static org.apache.hudi.common.table.timeline.InstantComparison.compareTimestamps;
+import static org.apache.hudi.config.HoodieArchivalConfig.ARCHIVE_LIMIT_INSTANTS;
 
 /**
  * Archiver to bound the growth of files under .hoodie meta path.
@@ -353,6 +354,8 @@ private List<HoodieInstant> getInstantsToArchive() throws IOException {
             InstantComparatorV1.getComparableAction(i.getAction()))));
 
     return instantsToArchive.stream()
+        .sorted()
+        .limit(config.getLongOrDefault(ARCHIVE_LIMIT_INSTANTS))
         .flatMap(hoodieInstant ->
             groupByTsAction.getOrDefault(Pair.of(hoodieInstant.requestedTime(),
                 InstantComparatorV1.getComparableAction(hoodieInstant.getAction())), Collections.emptyList()).stream())

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieArchivalConfig.java

@@ -102,6 +102,14 @@ public class HoodieArchivalConfig extends HoodieConfig {
       .withDocumentation("If enabled, archival will proceed beyond savepoint, skipping savepoint commits."
           + " If disabled, archival will stop at the earliest savepoint commit.");
 
+  public static final ConfigProperty<Long> ARCHIVE_LIMIT_INSTANTS = ConfigProperty
+      .key("hoodie.archive.limit.instants")
+      .defaultValue(Long.MAX_VALUE)
+      .markAdvanced()
+      .sinceVersion("0.16.0")
+      .withDocumentation("If enabled, archival will limit the instants upto this value."
+          + "This is useful when a large timeline containing tens of thousands instants that need to archived");
+
   /**
    * @deprecated Use {@link #MAX_COMMITS_TO_KEEP} and its methods instead
    */
@@ -193,6 +201,11 @@ public Builder withArchiveBeyondSavepoint(boolean archiveBeyondSavepoint) {
       return this;
     }
 
+    public Builder withArchiveLimitInstants(long limitInstants) {
+      archivalConfig.setValue(ARCHIVE_LIMIT_INSTANTS, String.valueOf(limitInstants));
+      return this;
+    }
+
     public HoodieArchivalConfig build() {
       archivalConfig.setDefaults(HoodieArchivalConfig.class.getName());
       return archivalConfig;

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestHoodieTimelineArchiver.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.client.timeline;
+
+import org.apache.hudi.client.timeline.versioning.v1.TimelineArchiverV1;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieLocalEngineContext;
+import org.apache.hudi.common.table.timeline.versioning.v1.ActiveTimelineV1;
+import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
+import org.apache.hudi.common.testutils.InProcessTimeGenerator;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.config.HoodieArchivalConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
+import org.apache.hudi.table.HoodieTable;
+import org.apache.hudi.testutils.HoodieWriteableTestTable;
+
+import org.apache.avro.Schema;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+
+import static org.apache.hudi.common.testutils.SchemaTestUtil.getSchemaFromResource;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class TestHoodieTimelineArchiver extends HoodieCommonTestHarness {
+  private static final Schema SCHEMA = getSchemaFromResource(TestHoodieTimelineArchiver.class, "/exampleSchema.avsc", true);
+
+  @BeforeEach
+  void setUp() throws Exception {
+    initPath();
+    initMetaClient();
+  }
+
+  @Test
+  void archiveIfRequired_instantsAreArchived() throws Exception {
+    HoodieWriteConfig writeConfig = HoodieWriteConfig
+        .newBuilder()
+        .withPath(tempDir.toString())
+        .withArchivalConfig(HoodieArchivalConfig.newBuilder()
+            .archiveCommitsWith(2, 3)
+            .withArchiveLimitInstants(1)
+            .build())
+        .withMarkersType("DIRECT")
+        .build();
+    HoodieEngineContext context = new HoodieLocalEngineContext(metaClient.getStorageConf());
+    HoodieStorage hoodieStorage = new HoodieHadoopStorage(basePath, metaClient.getStorageConf());
+    HoodieWriteableTestTable testTable = new HoodieWriteableTestTable(basePath, hoodieStorage, metaClient, SCHEMA, null, null, Option.of(context));
+    testTable.addCommit(InProcessTimeGenerator.createNewInstantTime());
+    testTable.addCommit(InProcessTimeGenerator.createNewInstantTime());
+    testTable.addCommit(InProcessTimeGenerator.createNewInstantTime());
+    testTable.addCommit(InProcessTimeGenerator.createNewInstantTime());
+    testTable.addCommit(InProcessTimeGenerator.createNewInstantTime());
+    HoodieTable hoodieTable = setupMockHoodieTable(context, writeConfig);
+
+    TimelineArchiverV1 archiver = new TimelineArchiverV1<>(writeConfig, hoodieTable);
+    archiver.archiveIfRequired(context);
+    assertEquals(4, metaClient.reloadActiveTimeline().countInstants());
+    // Run archive again with reloaded metaClient results.
+    hoodieTable = setupMockHoodieTable(context, writeConfig);
+    archiver = new TimelineArchiverV1<>(writeConfig, hoodieTable);
+    archiver.archiveIfRequired(context);
+    assertEquals(3, metaClient.reloadActiveTimeline().countInstants());
+    // Run archive again with reloaded metaClient results but it's no op as timeline instants is not above max limit of 3.
+    hoodieTable = setupMockHoodieTable(context, writeConfig);
+    archiver = new TimelineArchiverV1<>(writeConfig, hoodieTable);
+    archiver.archiveIfRequired(context);
+    assertEquals(3, metaClient.reloadActiveTimeline().countInstants());
+
+    ActiveTimelineV1 rawActiveTimeline = new ActiveTimelineV1(metaClient, false);
+    assertEquals(9, rawActiveTimeline.countInstants());
+  }
+
+  private HoodieTable setupMockHoodieTable(HoodieEngineContext context, HoodieWriteConfig writeConfig) {
+    HoodieTable hoodieTable = mock(HoodieTable.class, RETURNS_DEEP_STUBS);
+    when(hoodieTable.getContext()).thenReturn(context);
+    when(hoodieTable.getConfig()).thenReturn(writeConfig);
+    when(hoodieTable.getMetaClient()).thenReturn(metaClient);
+    when(hoodieTable.getActiveTimeline()).thenReturn(metaClient.getActiveTimeline());
+    when(hoodieTable.getCompletedCommitsTimeline()).thenReturn(metaClient.getCommitsTimeline().filterCompletedInstants());
+    when(hoodieTable.getCompletedCleanTimeline()).thenReturn(metaClient.getActiveTimeline().getCleanerTimeline().filterCompletedInstants());
+    when(hoodieTable.getCompletedSavepointTimeline()).thenReturn(metaClient.getActiveTimeline().getSavePointTimeline().filterCompletedInstants());
+    when(hoodieTable.getSavepointTimestamps()).thenReturn(Collections.emptySet());
+    return hoodieTable;
+  }
+}