From 27a1cffb774f101321d03ffc3ecf20acfbd90765 Mon Sep 17 00:00:00 2001 From: Sourabh Badhya Date: Fri, 2 Feb 2024 18:59:36 +0530 Subject: [PATCH] Revert "HIVE-27637: Compare highest write ID of compaction records when trying to perform abort cleanup (Zsolt Miskolczi reviewed by Attila Turoczy, Sourabh Badhya)" (#5058) This reverts commit f3439697343f3d5e1f1d007d8c878a6eb821713b. --- .../txn/TestCompactionTxnHandler.java | 44 ------------------- .../handler/TestAbortedTxnCleaner.java | 17 +------ .../queries/ReadyToCleanAbortHandler.java | 12 +---- 3 files changed, 3 insertions(+), 70 deletions(-) diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java index cd02eb1ba3eb..d26f3774af70 100644 --- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java +++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java @@ -1056,50 +1056,6 @@ public void testFindReadyToCleanAborts_limitFetchSize() throws Exception { assertEquals(1, potentials.size()); } - @Test - public void testFindReadyToCleanAborts() throws Exception { - long txnId = openTxn(); - - List components = new ArrayList<>(); - components.add(createLockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb", "mytable", "mypartition=myvalue", DataOperationType.UPDATE)); - components.add(createLockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb", "yourtable", "mypartition=myvalue", DataOperationType.UPDATE)); - - allocateTableWriteIds("mydb", "mytable", txnId); - allocateTableWriteIds("mydb", "yourtable", txnId); - - LockRequest req = new LockRequest(components, "me", "localhost"); - req.setTxnid(txnId); - LockResponse res = txnHandler.lock(req); - assertSame(res.getState(), LockState.ACQUIRED); - - txnHandler.abortTxn(new AbortTxnRequest((txnId))); - - txnId = openTxn(); - components = new ArrayList<>(); - components.add(createLockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb", "mytable", "mypartition=myvalue", DataOperationType.UPDATE)); - allocateTableWriteIds("mydb", "mytable", txnId); - - req = new LockRequest(components, "me", "localhost"); - req.setTxnid(txnId); - res = txnHandler.lock(req); - assertSame(res.getState(), LockState.ACQUIRED); - - CompactionRequest rqst = new CompactionRequest("mydb", "mytable", CompactionType.MINOR); - rqst.setPartitionname("mypartition=myvalue"); - txnHandler.compact(rqst); - - CompactionInfo ci = txnHandler.findNextToCompact(aFindNextCompactRequest("fred", WORKER_VERSION)); - assertNotNull(ci); - ci.highestWriteId = 41; - txnHandler.updateCompactorState(ci, 0); - - List potentials = txnHandler.findReadyToCleanAborts(1, 0); - assertEquals(1, potentials.size()); - CompactionInfo potentialToCleanAbort = potentials.get(0); - assertEquals("mydb", potentialToCleanAbort.dbname); - assertEquals("yourtable", potentialToCleanAbort.tableName); - } - private static FindNextCompactRequest aFindNextCompactRequest(String workerId, String workerVersion) { FindNextCompactRequest request = new FindNextCompactRequest(); request.setWorkerId(workerId); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/handler/TestAbortedTxnCleaner.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/handler/TestAbortedTxnCleaner.java index 1d31aae94343..8f6814d4890f 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/handler/TestAbortedTxnCleaner.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/handler/TestAbortedTxnCleaner.java @@ -44,9 +44,7 @@ import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @@ -284,21 +282,8 @@ public void testCleaningOfAbortedDirectoriesBelowBase() throws Exception { cleaner.setCleanupHandlers(Arrays.asList(mockedTaskHandler)); cleaner.run(); - Mockito.verifyNoInteractions(mockedFSRemover); + Mockito.verify(mockedFSRemover, Mockito.times(1)).clean(any(CleanupRequest.class)); Mockito.verify(mockedTaskHandler, Mockito.times(1)).getTasks(); - String compactionQueuePresence = "SELECT COUNT(*) FROM \"COMPACTION_QUEUE\" " + - " WHERE \"CQ_DATABASE\" = '" + dbName+ "' AND \"CQ_TABLE\" = '" + tableName + "' AND \"CQ_PARTITION\" IS NULL"; - Assert.assertEquals(1, TestTxnDbUtil.countQueryAgent(conf, compactionQueuePresence)); - - directories = getDirectories(conf, t, null); - // Both base and delta files are present since the cleaner skips them as there is a newer write. - Assert.assertEquals(5, directories.size()); - Assert.assertEquals(1, directories.stream().filter(dir -> dir.getName().startsWith(AcidUtils.BASE_PREFIX)).count()); - - // Run compaction and clean up - startInitiator(); - startWorker(); - startCleaner(); directories = getDirectories(conf, t, null); // The table is already compacted, so we must see 1 base delta diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/queries/ReadyToCleanAbortHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/queries/ReadyToCleanAbortHandler.java index eebe29dc441a..4940d384095b 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/queries/ReadyToCleanAbortHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/queries/ReadyToCleanAbortHandler.java @@ -52,7 +52,7 @@ public class ReadyToCleanAbortHandler implements QueryHandler \"res1\".\"MAX_ABORTED_WRITE_ID\"" + - " AND \"cq\".\"CQ_STATE\" " + - " IN ('i', 'w', 'r'))"; + " WHERE \"res3\".\"RETRY_RECORD_CHECK\" <= 0 OR \"res3\".\"RETRY_RECORD_CHECK\" IS NULL"; private final long abortedTimeThreshold; private final int abortedThreshold;