..
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index 923884ec505b..d3fcb13f632a 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -25,7 +25,7 @@
org.apache.hive
hive-standalone-metastore
- 3.1.2.fk.11
+ 3.1.2.fk.12
jar
Hive Standalone Metastore
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
index 1c1b3ce5eeb6..875e272eb4aa 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
@@ -61,6 +61,10 @@ hive_metastoreConstants::hive_metastoreConstants() {
TABLE_BUCKETING_VERSION = "bucketing_version";
+ EXPECTED_PARAMETER_KEY = "expected_parameter_key";
+
+ EXPECTED_PARAMETER_VALUE = "expected_parameter_value";
+
}
}}} // namespace
diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h
index 1f062530e4d8..4cffc9491fb2 100644
--- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h
+++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h
@@ -40,6 +40,8 @@ class hive_metastoreConstants {
std::string TABLE_NO_AUTO_COMPACT;
std::string TABLE_TRANSACTIONAL_PROPERTIES;
std::string TABLE_BUCKETING_VERSION;
+ std::string EXPECTED_PARAMETER_KEY;
+ std::string EXPECTED_PARAMETER_VALUE;
};
extern const hive_metastoreConstants g_hive_metastore_constants;
diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
index 2ee81df1dc11..76be19a7d0e9 100644
--- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
+++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java
@@ -86,4 +86,8 @@
public static final String TABLE_BUCKETING_VERSION = "bucketing_version";
+ public static final String EXPECTED_PARAMETER_KEY = "expected_parameter_key";
+
+ public static final String EXPECTED_PARAMETER_VALUE = "expected_parameter_value";
+
}
diff --git a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
index 84f7e3320c87..8c65e346a279 100644
--- a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
+++ b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php
@@ -31377,6 +31377,8 @@ final class Constant extends \Thrift\Type\TConstant {
static protected $TABLE_NO_AUTO_COMPACT;
static protected $TABLE_TRANSACTIONAL_PROPERTIES;
static protected $TABLE_BUCKETING_VERSION;
+ static protected $EXPECTED_PARAMETER_KEY;
+ static protected $EXPECTED_PARAMETER_VALUE;
static protected function init_DDL_TIME() {
return "transient_lastDdlTime";
@@ -31477,6 +31479,14 @@ static protected function init_TABLE_TRANSACTIONAL_PROPERTIES() {
static protected function init_TABLE_BUCKETING_VERSION() {
return "bucketing_version";
}
+
+ static protected function init_EXPECTED_PARAMETER_KEY() {
+ return "expected_parameter_key";
+ }
+
+ static protected function init_EXPECTED_PARAMETER_VALUE() {
+ return "expected_parameter_value";
+ }
}
diff --git a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/constants.py b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/constants.py
index c27745a6078e..261b3bf82aa1 100644
--- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/constants.py
+++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/constants.py
@@ -34,3 +34,5 @@
TABLE_NO_AUTO_COMPACT = "no_auto_compaction"
TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties"
TABLE_BUCKETING_VERSION = "bucketing_version"
+EXPECTED_PARAMETER_KEY = "expected_parameter_key"
+EXPECTED_PARAMETER_VALUE = "expected_parameter_value"
diff --git a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb
index 9e6cedd43d72..8cc22cfcaceb 100644
--- a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb
+++ b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb
@@ -57,3 +57,7 @@
TABLE_BUCKETING_VERSION = %q"bucketing_version"
+EXPECTED_PARAMETER_KEY = %q"expected_parameter_key"
+
+EXPECTED_PARAMETER_VALUE = %q"expected_parameter_value"
+
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index f328ad181558..f010f0d2cca8 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -49,6 +49,7 @@
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import javax.jdo.Constants;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
@@ -146,7 +147,17 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam
rename = true;
}
- msdb.openTransaction();
+ String expectedKey = environmentContext != null && environmentContext.getProperties() != null ?
+ environmentContext.getProperties().get(hive_metastoreConstants.EXPECTED_PARAMETER_KEY) : null;
+ String expectedValue = environmentContext != null && environmentContext.getProperties() != null ?
+ environmentContext.getProperties().get(hive_metastoreConstants.EXPECTED_PARAMETER_VALUE) : null;
+
+ if (expectedKey != null) {
+ // If we have to check the expected state of the table we have to prevent nonrepeatable reads.
+ msdb.openTransaction(Constants.TX_REPEATABLE_READ);
+ } else {
+ msdb.openTransaction();
+ }
// get old table
oldt = msdb.getTable(catName, dbname, name);
if (oldt == null) {
@@ -154,6 +165,12 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam
Warehouse.getCatalogQualifiedTableName(catName, dbname, name) + " doesn't exist");
}
+ if (expectedKey != null && expectedValue != null
+ && !expectedValue.equals(oldt.getParameters().get(expectedKey))) {
+ throw new MetaException("The table has been modified. The parameter value for key '" + expectedKey + "' is '"
+ + oldt.getParameters().get(expectedKey) + "'. The expected was value was '" + expectedValue + "'");
+ }
+
if (oldt.getPartitionKeysSize() != 0) {
isPartitionedTable = true;
}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 7bf5cef84197..cc2588747164 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -703,17 +703,31 @@ public void shutdown() {
}
/**
- * Opens a new one or the one already created Every call of this function must
- * have corresponding commit or rollback function call
+ * Opens a new one or the one already created. Every call of this function must
+ * have corresponding commit or rollback function call.
*
* @return an active transaction
*/
-
@Override
public boolean openTransaction() {
+ return openTransaction(null);
+ }
+
+ /**
+ * Opens a new one or the one already created. Every call of this function must
+ * have corresponding commit or rollback function call.
+ *
+ * @param isolationLevel The transaction isolation level. Only possible to set on the first call.
+ * @return an active transaction
+ */
+ @Override
+ public boolean openTransaction(String isolationLevel) {
openTrasactionCalls++;
if (openTrasactionCalls == 1) {
currentTransaction = pm.currentTransaction();
+ if (isolationLevel != null) {
+ currentTransaction.setIsolationLevel(isolationLevel);
+ }
currentTransaction.begin();
transactionStatus = TXN_STATUS.OPEN;
} else {
@@ -723,10 +737,16 @@ public boolean openTransaction() {
throw new RuntimeException("openTransaction called in an interior"
+ " transaction scope, but currentTransaction is not active.");
}
+
+ // Can not change the isolation level on an already open transaction
+ if (isolationLevel != null && !isolationLevel.equals(currentTransaction.getIsolationLevel())) {
+ throw new RuntimeException("Can not set isolation level on an open transaction");
+ }
}
boolean result = currentTransaction.isActive();
- debugLog("Open transaction: count = " + openTrasactionCalls + ", isActive = " + result);
+ debugLog("Open transaction: count = " + openTrasactionCalls + ", isActive = " + result + ", isolationLevel = "
+ + currentTransaction.getIsolationLevel());
return result;
}
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
index f350aa9fd7c0..a6ffcf417e92 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java
@@ -104,14 +104,24 @@ public interface RawStore extends Configurable {
void shutdown();
/**
- * Opens a new one or the one already created Every call of this function must
- * have corresponding commit or rollback function call
+ * Opens a new one or the one already created. Every call of this function must
+ * have corresponding commit or rollback function call.
*
* @return an active transaction
*/
-
boolean openTransaction();
+ /**
+ * Opens a new one or the one already created. Every call of this function must
+ * have corresponding commit or rollback function call.
+ *
+ * @param isolationLevel The transaction isolation level. Only possible to set on the first call.
+ * @return an active transaction
+ */
+ default boolean openTransaction(String isolationLevel) {
+ throw new UnsupportedOperationException("Setting isolation level for this Store is not supported");
+ }
+
/**
* if this is the commit of the first open call then an actual commit is
* called.
diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift
index ad1dc1f76966..c08a03c0e1a9 100644
--- a/standalone-metastore/src/main/thrift/hive_metastore.thrift
+++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift
@@ -2243,3 +2243,6 @@ const string TABLE_NO_AUTO_COMPACT = "no_auto_compaction",
const string TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties",
const string TABLE_BUCKETING_VERSION = "bucketing_version",
+// Keys for alter table environment context parameters
+const string EXPECTED_PARAMETER_KEY = "expected_parameter_key",
+const string EXPECTED_PARAMETER_VALUE = "expected_parameter_value",
\ No newline at end of file
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java
index c1eff55b11c3..a2ab9e807f37 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java
@@ -23,7 +23,9 @@
import org.apache.hadoop.hive.metastore.ColumnType;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.metastore.RawStore;
import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest;
import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.Catalog;
@@ -33,6 +35,7 @@
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
@@ -65,15 +68,21 @@
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME;
import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME;
+import static org.junit.Assert.assertTrue;
/**
* Test class for IMetaStoreClient API. Testing the Table related functions for metadata
@@ -1075,6 +1084,99 @@ public void testAlterTableAlreadyExists() throws Exception {
}
}
+ @Test
+ public void testAlterTableExpectedPropertyMatch() throws Exception {
+ Table originalTable = testTables[0];
+
+ EnvironmentContext context = new EnvironmentContext();
+ context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_KEY, "transient_lastDdlTime");
+ context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_VALUE,
+ originalTable.getParameters().get("transient_lastDdlTime"));
+
+ client.alter_table(originalTable.getCatName(), originalTable.getDbName(), originalTable.getTableName(),
+ originalTable, context);
+ }
+
+ @Test(expected = MetaException.class)
+ public void testAlterTableExpectedPropertyDifferent() throws Exception {
+ Table originalTable = testTables[0];
+
+ EnvironmentContext context = new EnvironmentContext();
+ context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_KEY, "transient_lastDdlTime");
+ context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_VALUE, "alma");
+
+ client.alter_table(originalTable.getCatName(), originalTable.getDbName(), originalTable.getTableName(),
+ originalTable, context);
+ }
+
+ /**
+ * This tests ensures that concurrent Iceberg commits will fail. Acceptable as a first sanity check.
+ *
+ * I have not found a good way to check that HMS side database commits are parallel in the
+ * {@link org.apache.hadoop.hive.metastore.HiveAlterHandler#alterTable(RawStore, Warehouse, String, String, String, Table, EnvironmentContext)}
+ * call, but this test could be used to manually ensure that using breakpoints.
+ */
+ @Test
+ public void testAlterTableExpectedPropertyConcurrent() throws Exception {
+ Table originalTable = testTables[0];
+
+ originalTable.getParameters().put("snapshot", "0");
+ client.alter_table(originalTable.getCatName(), originalTable.getDbName(), originalTable.getTableName(),
+ originalTable, null);
+
+ ExecutorService threads = null;
+ try {
+ threads = Executors.newFixedThreadPool(2);
+ for (int i = 0; i < 3; i++) {
+ EnvironmentContext context = new EnvironmentContext();
+ context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_KEY, "snapshot");
+ context.putToProperties(hive_metastoreConstants.EXPECTED_PARAMETER_VALUE, String.valueOf(i));
+
+ Table newTable = originalTable.deepCopy();
+ newTable.getParameters().put("snapshot", String.valueOf(i + 1));
+
+ IMetaStoreClient client1 = metaStore.getClient();
+ IMetaStoreClient client2 = metaStore.getClient();
+
+ Collection> concurrentTasks = new ArrayList<>(2);
+ concurrentTasks.add(alterTask(client1, newTable, context));
+ concurrentTasks.add(alterTask(client2, newTable, context));
+
+ Collection> results = threads.invokeAll(concurrentTasks);
+
+ boolean foundSuccess = false;
+ boolean foundFailure = false;
+
+ for (Future result : results) {
+ if (result.get()) {
+ foundSuccess = true;
+ } else {
+ foundFailure = true;
+ }
+ }
+
+ assertTrue("At least one success is expected", foundSuccess);
+ assertTrue("At least one failure is expected", foundFailure);
+ }
+ } finally {
+ if (threads != null) {
+ threads.shutdown();
+ }
+ }
+ }
+
+ private Callable alterTask(IMetaStoreClient hmsClient, Table newTable, EnvironmentContext context) {
+ return () -> {
+ try {
+ hmsClient.alter_table(newTable.getCatName(), newTable.getDbName(), newTable.getTableName(),
+ newTable, context);
+ } catch (Throwable e) {
+ return false;
+ }
+ return true;
+ };
+ }
+
@Test
public void tablesInOtherCatalogs() throws TException, URISyntaxException {
String catName = "create_etc_tables_in_other_catalogs";
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 52919fc5617b..7c7fcdffe2d5 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -20,7 +20,7 @@
org.apache.hive
hive
- 3.1.2.fk.11
+ 3.1.2.fk.12
../pom.xml
diff --git a/testutils/pom.xml b/testutils/pom.xml
index 8b8e76f87cde..ec981e77b4a2 100644
--- a/testutils/pom.xml
+++ b/testutils/pom.xml
@@ -19,7 +19,7 @@
org.apache.hive
hive
- 3.1.2.fk.11
+ 3.1.2.fk.12
../pom.xml
diff --git a/upgrade-acid/pom.xml b/upgrade-acid/pom.xml
index 313cd2a1a3c2..d0d554a9066d 100644
--- a/upgrade-acid/pom.xml
+++ b/upgrade-acid/pom.xml
@@ -26,7 +26,7 @@
4.0.0
org.apache.hive
- 3.1.2.fk.11
+ 3.1.2.fk.12
hive-upgrade-acid
Hive Upgrade Acid
jar
diff --git a/vector-code-gen/pom.xml b/vector-code-gen/pom.xml
index be68ae1126d8..93401a91ba01 100644
--- a/vector-code-gen/pom.xml
+++ b/vector-code-gen/pom.xml
@@ -19,7 +19,7 @@
org.apache.hive
hive
- 3.1.2.fk.11
+ 3.1.2.fk.12
../pom.xml