Skip to content

Commit 4a649aa

Browse files
dstreevdstreev
committed
Fixed property overrides placement to correct cluster.
1 parent e22cd41 commit 4a649aa

File tree

7 files changed

+36
-30
lines changed

7 files changed

+36
-30
lines changed

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828

2929
<groupId>com.cloudera.utils.hadoop</groupId>
3030
<artifactId>hms-mirror</artifactId>
31-
<version>2.2.0.18.0</version>
31+
<version>2.2.0.18.1</version>
3232
<packaging>jar</packaging>
3333

3434
<name>hms-mirror</name>

‎src/main/java/com/cloudera/utils/hms/mirror/datastrategy/DataStrategyBase.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -569,8 +569,9 @@ protected Boolean buildMigrationSql(TableMirror tableMirror, Environment origina
569569

570570
// Set Override Properties.
571571
if (config.getOptimization().getOverrides() != null) {
572-
for (String key : config.getOptimization().getOverrides().getLeft().keySet()) {
573-
targetEnvTable.addSql("Setting " + key, "set " + key + "=" + config.getOptimization().getOverrides().getLeft().get(key));
572+
Map<String, String> overrides = config.getOptimization().getOverrides().getFor(targetEnv2);
573+
for (String key : overrides.keySet()) {
574+
targetEnvTable.addSql("Setting " + key, "set " + key + "=" + overrides.get(key));
574575
}
575576
}
576577

‎src/main/java/com/cloudera/utils/hms/mirror/datastrategy/SQLAcidDowngradeInPlaceDataStrategy.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.springframework.stereotype.Component;
3838

3939
import java.text.MessageFormat;
40+
import java.util.Map;
4041

4142
import static com.cloudera.utils.hms.mirror.SessionVars.SORT_DYNAMIC_PARTITION;
4243
import static com.cloudera.utils.hms.mirror.SessionVars.SORT_DYNAMIC_PARTITION_THRESHOLD;
@@ -125,9 +126,11 @@ public Boolean buildOutSql(TableMirror tableMirror) throws MissingDataPointExcep
125126

126127
let.addSql(TableUtils.USE_DESC, useDb);
127128
// Set Override Properties.
128-
if (!hmsMirrorConfig.getOptimization().getOverrides().getLeft().isEmpty()) {
129-
for (String key : hmsMirrorConfig.getOptimization().getOverrides().getLeft().keySet()) {
130-
let.addSql("Setting " + key, "set " + key + "=" + hmsMirrorConfig.getOptimization().getOverrides().getLeft().get(key));
129+
// Get the LEFT overrides for the DOWNGRADE.
130+
Map<String, String> overrides = hmsMirrorConfig.getOptimization().getOverrides().getFor(Environment.LEFT);
131+
if (!overrides.isEmpty()) {
132+
for (String key : overrides.keySet()) {
133+
let.addSql("Setting " + key, "set " + key + "=" + overrides.get(key));
131134
}
132135
}
133136

‎src/main/java/com/cloudera/utils/hms/mirror/datastrategy/StorageMigrationDataStrategy.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -489,9 +489,10 @@ public Boolean execute(TableMirror tableMirror) {
489489
let.addSql(new Pair(TEZ_EXECUTION_DESC, SET_TEZ_AS_EXECUTION_ENGINE));
490490
}
491491
// Set Override Properties.
492-
if (config.getOptimization().getOverrides() != null) {
493-
for (String key : config.getOptimization().getOverrides().getLeft().keySet()) {
494-
let.addSql("Setting " + key, "set " + key + "=" + config.getOptimization().getOverrides().getLeft().get(key));
492+
Map<String, String> overrides = config.getOptimization().getOverrides().getFor(Environment.LEFT);
493+
if (!overrides.isEmpty()) {
494+
for (String key : overrides.keySet()) {
495+
let.addSql("Setting " + key, "set " + key + "=" + overrides.get(key));
495496
}
496497
}
497498

‎src/main/java/com/cloudera/utils/hms/mirror/domain/Overrides.java‎

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package com.cloudera.utils.hms.mirror.domain;
1919

20+
import com.cloudera.utils.hms.mirror.domain.support.Environment;
2021
import com.cloudera.utils.hms.mirror.domain.support.SideType;
2122
import com.fasterxml.jackson.annotation.JsonIgnore;
2223
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@@ -47,31 +48,28 @@ public void addProperty(String key, String value, SideType side) {
4748
}
4849

4950
@JsonIgnore
50-
public Map<String, String> getLeft() {
51-
Map<String, String> left = new TreeMap<String, String>();
52-
for (Map.Entry<String, Map<SideType, String>> entry : properties.entrySet()) {
53-
if (entry.getValue().containsKey(SideType.LEFT)) {
54-
left.put(entry.getKey(), entry.getValue().get(SideType.LEFT));
55-
}
56-
if (entry.getValue().containsKey(SideType.BOTH)) {
57-
left.put(entry.getKey(), entry.getValue().get(SideType.BOTH));
58-
}
59-
}
60-
return left;
61-
}
51+
public Map<String, String> getFor(Environment environment) {
6252

63-
@JsonIgnore
64-
public Map<String, String> getRight() {
65-
Map<String, String> right = new TreeMap<String, String>();
53+
Map<String, String> rtn = new TreeMap<String, String>();
6654
for (Map.Entry<String, Map<SideType, String>> entry : properties.entrySet()) {
67-
if (entry.getValue().containsKey(SideType.RIGHT)) {
68-
right.put(entry.getKey(), entry.getValue().get(SideType.RIGHT));
55+
switch (environment) {
56+
case LEFT:
57+
if (entry.getValue().containsKey(SideType.LEFT)) {
58+
rtn.put(entry.getKey(), entry.getValue().get(SideType.LEFT));
59+
}
60+
break;
61+
case RIGHT:
62+
if (entry.getValue().containsKey(SideType.RIGHT)) {
63+
rtn.put(entry.getKey(), entry.getValue().get(SideType.RIGHT));
64+
}
65+
break;
6966
}
7067
if (entry.getValue().containsKey(SideType.BOTH)) {
71-
right.put(entry.getKey(), entry.getValue().get(SideType.BOTH));
68+
rtn.put(entry.getKey(), entry.getValue().get(SideType.BOTH));
7269
}
7370
}
74-
return right;
71+
72+
return rtn;
7573
}
7674

7775
public void setPropertyOverridesStr(String[] inPropsStr, SideType side) {

‎src/test/java/com/cloudera/utils/hms/mirror/domain/HmsMirrorConfigCloneFullTest_01.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,8 +147,8 @@ public void cloneTest_01() {
147147
assertEquals(clone.getOptimization().isAutoTune(), config.getOptimization().isAutoTune());
148148
assertEquals(clone.getOptimization().isCompressTextOutput(), config.getOptimization().isCompressTextOutput());
149149
assertEquals(clone.getOptimization().isSkipStatsCollection(), config.getOptimization().isSkipStatsCollection());
150-
assertEquals(clone.getOptimization().getOverrides().getLeft(), config.getOptimization().getOverrides().getLeft());
151-
assertEquals(clone.getOptimization().getOverrides().getRight(), config.getOptimization().getOverrides().getRight());
150+
assertEquals(clone.getOptimization().getOverrides().getFor(Environment.LEFT), config.getOptimization().getOverrides().getFor(Environment.LEFT));
151+
assertEquals(clone.getOptimization().getOverrides().getFor(Environment.RIGHT), config.getOptimization().getOverrides().getFor(Environment.RIGHT));
152152
assertEquals(clone.getOptimization().isBuildShadowStatistics(), config.getOptimization().isBuildShadowStatistics());
153153

154154
assertEquals(clone.getOutputDirectory(), config.getOutputDirectory());

‎src/test/java/com/cloudera/utils/hms/mirror/integration/end_to_end/legacy_to_cdp/Test_sql_is.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@
4343
// "--hms-mirror.config.target-namespace=s3a://my_cs_bucket",
4444
// "--hms-mirror.config.reset-to-default-location=true",
4545
// "--hms-mirror.config.distcp=true",
46+
"--hms-mirror.config.property-overrides=hive.exec.orc.split.strategy=BI",
47+
"--hms-mirror.config.property-overrides-left=tez.queue.name=left-side",
48+
"--hms-mirror.config.property-overrides-right=tez.queue.name=right-side",
4649
"--hms-mirror.conversion.test-filename=/test_data/assorted_tbls_01.yaml",
4750
"--hms-mirror.config.filename=/config/default.yaml.hdp2-cdp",
4851
"--hms-mirror.config.output-dir=${user.home}/.hms-mirror/test-output/e2e/legacy_cdp/sql_is"

0 commit comments

Comments
 (0)