Fixed locale issue with some 'set' statements that used numeric values.

dstreev · dstreev · commit 6c3119e0214a · 2025-03-31T09:12:03.000-04:00
diff --git a/Writerside/topics/Hive-Conversions.md b/Writerside/topics/Hive-Conversions.md
@@ -27,7 +27,7 @@ Modify the `hms-mirror` configuration to include the following settings:
 ``` yaml
 clusters:
   LEFT|RIGHT:
-    platformType: HDP2|HDP3|CHD5|CDH6|CDP7.1|CDP7_2|...
+    platformType: HDP2|HDP3|CHD5|CDH6|CDP7_1|CDP7_2|...
 ```
 </tab>
 </tabs>
diff --git a/Writerside/topics/cli-options.md b/Writerside/topics/cli-options.md
diff --git a/Writerside/writerside.cfg b/Writerside/writerside.cfg
@@ -4,5 +4,5 @@
 <ihp version="2.0">
     <topics dir="topics" web-path="topics"/>
     <images dir="images" web-path="images"/>
-    <instance src="hms-mirror.tree" web-path="/hms-mirror" version="v2.3.0.x"/>
+    <instance src="hms-mirror.tree" web-path="/hms-mirror" version="v2.3.1.x"/>
 </ihp>
diff --git a/pom.xml b/pom.xml
@@ -28,7 +28,7 @@
 
     <groupId>com.cloudera.utils.hadoop</groupId>
     <artifactId>hms-mirror</artifactId>
-    <version>2.3.1.2</version>
+    <version>2.3.1.4</version>
     <packaging>jar</packaging>
 
     <name>hms-mirror</name>
diff --git a/src/main/java/com/cloudera/utils/hms/mirror/SessionVars.java b/src/main/java/com/cloudera/utils/hms/mirror/SessionVars.java
@@ -21,7 +21,8 @@ public interface SessionVars {
     String TEZ_EXECUTION_DESC = "SET 'tez' as the execution engine";
     String SET_TEZ_AS_EXECUTION_ENGINE = "SET hive.execution.engine=tez";
 
-    String SET_SESSION_VALUE = "SET {0}={1}";
+    String SET_SESSION_VALUE_INT = "SET {0}={1,number,0}";
+    String SET_SESSION_VALUE_STRING = "SET {0}={1}";
 
     String SORT_DYNAMIC_PARTITION = "hive.optimize.sort.dynamic.partition";
     String SORT_DYNAMIC_PARTITION_THRESHOLD = "hive.optimize.sort.dynamic.partition.threshold";
diff --git a/src/main/java/com/cloudera/utils/hms/mirror/cli/config/CliInit.java b/src/main/java/com/cloudera/utils/hms/mirror/cli/config/CliInit.java
@@ -138,6 +138,10 @@ public HmsMirrorConfig loadHmsMirrorConfig(@Value("${hms-mirror.config.path}") S
             fullConfigPath = configPath + File.separator + configFile;
         }
         HmsMirrorConfig config = domainService.deserializeConfig(fullConfigPath);
+        if (config == null) {
+            log.error("Couldn't locate configuration file: {}", fullConfigPath);
+            throw new RuntimeException("Couldn't locate configuration file: " + fullConfigPath);
+        }
         return config;
     }
 
diff --git a/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/DataStrategyBase.java b/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/DataStrategyBase.java
@@ -597,7 +597,7 @@ protected Boolean buildMigrationSql(TableMirror tableMirror, Environment origina
                 if (config.getOptimization().isSkip()) {
                     if (!config.getCluster(Environment.LEFT).isLegacyHive()) {
                         assert targetEnvTable != null;
-                        targetEnvTable.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE, SORT_DYNAMIC_PARTITION, "false"));
+                        targetEnvTable.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE_STRING, SORT_DYNAMIC_PARTITION, "false"));
                     }
                     String partElement = TableUtils.getPartitionElements(original);
                     String transferSql = MessageFormat.format(MirrorConf.SQL_DATA_TRANSFER_WITH_PARTITIONS_DECLARATIVE,
@@ -607,9 +607,9 @@ protected Boolean buildMigrationSql(TableMirror tableMirror, Environment origina
                     targetEnvTable.addSql(new Pair(transferDesc, transferSql));
                 } else if (config.getOptimization().isSortDynamicPartitionInserts()) {
                     if (!config.getCluster(Environment.LEFT).isLegacyHive()) {
-                        targetEnvTable.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE, SORT_DYNAMIC_PARTITION, "true"));
+                        targetEnvTable.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE_STRING, SORT_DYNAMIC_PARTITION, "true"));
                         if (!config.getCluster(Environment.LEFT).isHdpHive3()) {
-                            targetEnvTable.addSql("Setting " + SORT_DYNAMIC_PARTITION_THRESHOLD, MessageFormat.format(SET_SESSION_VALUE, SORT_DYNAMIC_PARTITION_THRESHOLD, "0"));
+                            targetEnvTable.addSql("Setting " + SORT_DYNAMIC_PARTITION_THRESHOLD, MessageFormat.format(SET_SESSION_VALUE_INT, SORT_DYNAMIC_PARTITION_THRESHOLD, 0));
                         }
                     }
                     String partElement = TableUtils.getPartitionElements(original);
@@ -619,9 +619,9 @@ protected Boolean buildMigrationSql(TableMirror tableMirror, Environment origina
                     targetEnvTable.addSql(new Pair(transferDesc, transferSql));
                 } else {
                     if (!config.getCluster(Environment.LEFT).isLegacyHive()) {
-                        targetEnvTable.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE, SORT_DYNAMIC_PARTITION, "false"));
+                        targetEnvTable.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE_STRING, SORT_DYNAMIC_PARTITION, "false"));
                         if (!config.getCluster(Environment.LEFT).isHdpHive3()) {
-                            targetEnvTable.addSql("Setting " + SORT_DYNAMIC_PARTITION_THRESHOLD, MessageFormat.format(SET_SESSION_VALUE, SORT_DYNAMIC_PARTITION_THRESHOLD, "-1"));
+                            targetEnvTable.addSql("Setting " + SORT_DYNAMIC_PARTITION_THRESHOLD, MessageFormat.format(SET_SESSION_VALUE_INT, SORT_DYNAMIC_PARTITION_THRESHOLD, -1));
                         }
                     }
                     String partElement = TableUtils.getPartitionElements(original);
diff --git a/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/SQLAcidInPlaceDataStrategy.java b/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/SQLAcidInPlaceDataStrategy.java
@@ -166,7 +166,7 @@ public Boolean buildOutSql(TableMirror tableMirror) throws MissingDataPointExcep
         if (let.getPartitioned()) {
             if (hmsMirrorConfig.getOptimization().isSkip()) {
                 if (!hmsMirrorConfig.getCluster(Environment.LEFT).isLegacyHive()) {
-                    let.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE,SORT_DYNAMIC_PARTITION,"false"));
+                    let.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE_STRING,SORT_DYNAMIC_PARTITION,"false"));
                 }
                 String partElement = TableUtils.getPartitionElements(let);
                 String transferSql = MessageFormat.format(MirrorConf.SQL_DATA_TRANSFER_WITH_PARTITIONS_DECLARATIVE,
@@ -175,9 +175,9 @@ public Boolean buildOutSql(TableMirror tableMirror) throws MissingDataPointExcep
                 let.addSql(new Pair(transferDesc, transferSql));
             } else if (hmsMirrorConfig.getOptimization().isSortDynamicPartitionInserts()) {
                 if (!hmsMirrorConfig.getCluster(Environment.LEFT).isLegacyHive()) {
-                    let.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE,SORT_DYNAMIC_PARTITION ,"true"));
+                    let.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE_STRING,SORT_DYNAMIC_PARTITION ,"true"));
                     if (!hmsMirrorConfig.getCluster(Environment.LEFT).isHdpHive3()) {
-                        let.addSql("Setting " + SORT_DYNAMIC_PARTITION_THRESHOLD, MessageFormat.format(SET_SESSION_VALUE,SORT_DYNAMIC_PARTITION_THRESHOLD ,"0"));
+                        let.addSql("Setting " + SORT_DYNAMIC_PARTITION_THRESHOLD, MessageFormat.format(SET_SESSION_VALUE_INT,SORT_DYNAMIC_PARTITION_THRESHOLD , 0));
                     }
                 }
                 String partElement = TableUtils.getPartitionElements(let);
@@ -188,9 +188,9 @@ public Boolean buildOutSql(TableMirror tableMirror) throws MissingDataPointExcep
             } else {
                 // Prescriptive Optimization.
                 if (!hmsMirrorConfig.getCluster(Environment.LEFT).isLegacyHive()) {
-                    let.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE,SORT_DYNAMIC_PARTITION ,"false"));
+                    let.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE_STRING,SORT_DYNAMIC_PARTITION ,"false"));
                     if (!hmsMirrorConfig.getCluster(Environment.LEFT).isHdpHive3()) {
-                        let.addSql("Setting " + SORT_DYNAMIC_PARTITION_THRESHOLD, MessageFormat.format(SET_SESSION_VALUE,SORT_DYNAMIC_PARTITION_THRESHOLD ,"-1"));
+                        let.addSql("Setting " + SORT_DYNAMIC_PARTITION_THRESHOLD, MessageFormat.format(SET_SESSION_VALUE_INT,SORT_DYNAMIC_PARTITION_THRESHOLD , -1));
                     }
                 }
 
diff --git a/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/StorageMigrationDataStrategy.java b/src/main/java/com/cloudera/utils/hms/mirror/datastrategy/StorageMigrationDataStrategy.java
@@ -563,7 +563,7 @@ public Boolean build(TableMirror tableMirror) {
                         // Build Partition Elements.
                         if (config.getOptimization().isSkip()) {
                             if (!config.getCluster(Environment.LEFT).isLegacyHive()) {
-                                let.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE, SORT_DYNAMIC_PARTITION, "false"));
+                                let.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE_STRING, SORT_DYNAMIC_PARTITION, "false"));
                             }
                             String partElement = TableUtils.getPartitionElements(let);
                             String transferSql = MessageFormat.format(MirrorConf.SQL_DATA_TRANSFER_WITH_PARTITIONS_DECLARATIVE,
@@ -573,9 +573,9 @@ public Boolean build(TableMirror tableMirror) {
                         } else if (config.getOptimization().isSortDynamicPartitionInserts()) {
                             // Declarative
                             if (!config.getCluster(Environment.LEFT).isLegacyHive()) {
-                                let.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE, SORT_DYNAMIC_PARTITION, "true"));
+                                let.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE_STRING, SORT_DYNAMIC_PARTITION, "true"));
                                 if (!config.getCluster(Environment.LEFT).isHdpHive3()) {
-                                    let.addSql("Setting " + SORT_DYNAMIC_PARTITION_THRESHOLD, MessageFormat.format(SET_SESSION_VALUE, SORT_DYNAMIC_PARTITION_THRESHOLD, "0"));
+                                    let.addSql("Setting " + SORT_DYNAMIC_PARTITION_THRESHOLD, MessageFormat.format(SET_SESSION_VALUE_INT, SORT_DYNAMIC_PARTITION_THRESHOLD, 0));
                                 }
                             }
                             String partElement = TableUtils.getPartitionElements(let);
@@ -586,9 +586,9 @@ public Boolean build(TableMirror tableMirror) {
                         } else {
                             // Prescriptive
                             if (!config.getCluster(Environment.LEFT).isLegacyHive()) {
-                                let.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE, SORT_DYNAMIC_PARTITION, "false"));
+                                let.addSql("Setting " + SORT_DYNAMIC_PARTITION, MessageFormat.format(SET_SESSION_VALUE_STRING, SORT_DYNAMIC_PARTITION, "false"));
                                 if (!config.getCluster(Environment.LEFT).isHdpHive3()) {
-                                    let.addSql("Setting " + SORT_DYNAMIC_PARTITION_THRESHOLD, MessageFormat.format(SET_SESSION_VALUE, SORT_DYNAMIC_PARTITION_THRESHOLD, "-1"));
+                                    let.addSql("Setting " + SORT_DYNAMIC_PARTITION_THRESHOLD, MessageFormat.format(SET_SESSION_VALUE_INT, SORT_DYNAMIC_PARTITION_THRESHOLD, -1));
                                 }
                             }
                             String partElement = TableUtils.getPartitionElements(let);
diff --git a/src/main/java/com/cloudera/utils/hms/mirror/service/DomainService.java b/src/main/java/com/cloudera/utils/hms/mirror/service/DomainService.java
@@ -128,9 +128,13 @@ protected String fileToString(String configFilename) {
     }
 
     public HmsMirrorConfig deserializeConfig(String configFilename) {
-        HmsMirrorConfig config;
+        HmsMirrorConfig config = null;
         log.info("Initializing Config.");
         String yamlCfgFile = fileToString(configFilename);
+        if (isNull(yamlCfgFile)) {
+            log.error("Couldn't locate configuration file: " + configFilename);
+            return null;
+        }
         try {
             config = yamlMapper.readerFor(HmsMirrorConfig.class).readValue(yamlCfgFile);
         } catch (JsonProcessingException e) {
diff --git a/src/main/java/com/cloudera/utils/hms/mirror/service/StatsCalculatorService.java b/src/main/java/com/cloudera/utils/hms/mirror/service/StatsCalculatorService.java
@@ -155,7 +155,7 @@ public void setSessionOptions(Cluster cluster, EnvironmentTable controlEnv, Envi
                 // Set the tez group max size.
                 Long maxGrouping = getTezMaxGrouping(controlEnv);
                 applyEnv.addSql("Setting the " + TEZ_GROUP_MAX_SIZE,
-                        MessageFormat.format(SET_SESSION_VALUE,TEZ_GROUP_MAX_SIZE , Long.toString(maxGrouping)));
+                        MessageFormat.format(SET_SESSION_VALUE_INT,TEZ_GROUP_MAX_SIZE , maxGrouping));
             }
         }
 
@@ -166,17 +166,17 @@ public void setSessionOptions(Cluster cluster, EnvironmentTable controlEnv, Envi
             if (controlEnv.getPartitions().size() > 1000) {
                 applyEnv.addIssue("Setting " + HIVE_MAX_DYNAMIC_PARTITIONS);
                 applyEnv.addSql("Setting " + HIVE_MAX_DYNAMIC_PARTITIONS,
-                        MessageFormat.format(SET_SESSION_VALUE,HIVE_MAX_DYNAMIC_PARTITIONS ,
+                        MessageFormat.format(SET_SESSION_VALUE_INT,HIVE_MAX_DYNAMIC_PARTITIONS ,
                                 (int) (controlEnv.getPartitions().size() * 1.2)));
                 applyEnv.addIssue("Adjusting " + HIVE_MAX_REDUCERS + " to handle partition load");
                 int ratio = getPartitionDistributionRatio(controlEnv).intValue();
                 if (ratio >= 1) {
                     applyEnv.addSql("Setting " + HIVE_MAX_REDUCERS,
-                            MessageFormat.format(SET_SESSION_VALUE,HIVE_MAX_REDUCERS ,
+                            MessageFormat.format(SET_SESSION_VALUE_INT,HIVE_MAX_REDUCERS ,
                                     (ratio * controlEnv.getPartitions().size()) + 20));
                 } else {
                     applyEnv.addSql("Setting " + HIVE_MAX_REDUCERS,
-                            MessageFormat.format(SET_SESSION_VALUE, HIVE_MAX_REDUCERS ,
+                            MessageFormat.format(SET_SESSION_VALUE_INT, HIVE_MAX_REDUCERS ,
                                     (int) (controlEnv.getPartitions().size() * 1.2)));
                 }
             }
@@ -186,28 +186,28 @@ public void setSessionOptions(Cluster cluster, EnvironmentTable controlEnv, Envi
         if (serdeType == SerdeType.TEXT) {
             if (hmsMirrorConfig.getOptimization().isCompressTextOutput()) {
                 applyEnv.addIssue("Setting " + HIVE_COMPRESS_OUTPUT + " because you've setting that optimization");
-                applyEnv.addSql("Setting: " + HIVE_COMPRESS_OUTPUT, MessageFormat.format(SET_SESSION_VALUE, HIVE_COMPRESS_OUTPUT, "true"));
+                applyEnv.addSql("Setting: " + HIVE_COMPRESS_OUTPUT, MessageFormat.format(SET_SESSION_VALUE_STRING, HIVE_COMPRESS_OUTPUT, "true"));
             } else {
                 applyEnv.addIssue("Setting " + HIVE_COMPRESS_OUTPUT + " because you HAVEN'T set that optimization");
-                applyEnv.addSql("Setting: " + HIVE_COMPRESS_OUTPUT, MessageFormat.format(SET_SESSION_VALUE,HIVE_COMPRESS_OUTPUT,"false"));
+                applyEnv.addSql("Setting: " + HIVE_COMPRESS_OUTPUT, MessageFormat.format(SET_SESSION_VALUE_STRING,HIVE_COMPRESS_OUTPUT,"false"));
             }
         }
 
         // Handle Auto Stats Gathering.
         if (!cluster.isLegacyHive()) {
             if (cluster.isEnableAutoTableStats()) {
                 applyEnv.addIssue("Setting " + HIVE_AUTO_TABLE_STATS + " because you've set that optimization");
-                applyEnv.addSql("Setting: " + HIVE_AUTO_TABLE_STATS, MessageFormat.format(SET_SESSION_VALUE,HIVE_AUTO_TABLE_STATS ,"true"));
+                applyEnv.addSql("Setting: " + HIVE_AUTO_TABLE_STATS, MessageFormat.format(SET_SESSION_VALUE_STRING,HIVE_AUTO_TABLE_STATS ,"true"));
             } else {
                 applyEnv.addIssue("Setting " + HIVE_AUTO_TABLE_STATS + " because you've set that optimization");
-                applyEnv.addSql("Setting: " + HIVE_AUTO_TABLE_STATS, MessageFormat.format(SET_SESSION_VALUE,HIVE_AUTO_TABLE_STATS ,"false"));
+                applyEnv.addSql("Setting: " + HIVE_AUTO_TABLE_STATS, MessageFormat.format(SET_SESSION_VALUE_STRING,HIVE_AUTO_TABLE_STATS ,"false"));
             }
             if (cluster.isEnableAutoColumnStats()) {
                 applyEnv.addIssue("Setting " + HIVE_AUTO_COLUMN_STATS + " because you've set that optimization");
-                applyEnv.addSql("Setting: " + HIVE_AUTO_COLUMN_STATS, MessageFormat.format(SET_SESSION_VALUE,HIVE_AUTO_COLUMN_STATS,"true"));
+                applyEnv.addSql("Setting: " + HIVE_AUTO_COLUMN_STATS, MessageFormat.format(SET_SESSION_VALUE_STRING,HIVE_AUTO_COLUMN_STATS,"true"));
             } else {
                 applyEnv.addIssue("Setting " + HIVE_AUTO_COLUMN_STATS + " because you've set that optimization");
-                applyEnv.addSql("Setting: " + HIVE_AUTO_COLUMN_STATS, MessageFormat.format(SET_SESSION_VALUE,HIVE_AUTO_COLUMN_STATS, "false"));
+                applyEnv.addSql("Setting: " + HIVE_AUTO_COLUMN_STATS, MessageFormat.format(SET_SESSION_VALUE_STRING,HIVE_AUTO_COLUMN_STATS, "false"));
             }
         }
     }
diff --git a/src/test/java/com/cloudera/utils/hms/mirror/TestSessionVars.java b/src/test/java/com/cloudera/utils/hms/mirror/TestSessionVars.java
@@ -2,6 +2,8 @@
 
 import lombok.extern.slf4j.Slf4j;
 import org.junit.Test;
+
+import static com.cloudera.utils.hms.mirror.SessionVars.*;
 import static org.junit.Assert.assertEquals;
 
 import java.text.MessageFormat;
@@ -12,7 +14,22 @@ public class TestSessionVars {
     @Test
     public void testSetSessionVars() {
         String result = MessageFormat.
-                format(SessionVars.SET_SESSION_VALUE, SessionVars.SORT_DYNAMIC_PARTITION_THRESHOLD, "-1");
+                format(SessionVars.SET_SESSION_VALUE_INT, SessionVars.SORT_DYNAMIC_PARTITION_THRESHOLD, -1);
         assertEquals("SET hive.optimize.sort.dynamic.partition.threshold=-1", result);
     }
+
+    @Test
+    public void testSetSessionVars_01() {
+        String result = MessageFormat.format(SET_SESSION_VALUE_INT,HIVE_MAX_DYNAMIC_PARTITIONS ,
+                        (int)(1322 * 1.2));
+        assertEquals("SET hive.exec.max.dynamic.partitions=1586", result);
+    }
+
+    @Test
+    public void testSetSessionVars_02() {
+        String result = MessageFormat.format(SET_SESSION_VALUE_STRING,SORT_DYNAMIC_PARTITION ,
+                "true");
+        assertEquals("SET hive.optimize.sort.dynamic.partition=true", result);
+    }
+
 }

Original file line number	Diff line number	Diff line change
`@@ -138,6 +138,10 @@ public HmsMirrorConfig loadHmsMirrorConfig(@Value("${hms-mirror.config.path}") S`
`138`	`138`	`fullConfigPath = configPath + File.separator + configFile;`
`139`	`139`	`}`
`140`	`140`	`HmsMirrorConfig config = domainService.deserializeConfig(fullConfigPath);`
	`141`	`+ if (config == null) {`
	`142`	`+ log.error("Couldn't locate configuration file: {}", fullConfigPath);`
	`143`	`+ throw new RuntimeException("Couldn't locate configuration file: " + fullConfigPath);`
	`144`	`+ }`
`141`	`145`	`return config;`
`142`	`146`	`}`
`143`	`147`