Skip to content

Commit 0da4e4d

Browse files
dstreevdstreev
committed
Handle an NPE during in-place scenarios with SQL datastrategy.
1 parent a710be4 commit 0da4e4d

File tree

5 files changed

+75
-61
lines changed

5 files changed

+75
-61
lines changed

‎configs/default.template.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ clusters:
6161
max: 5
6262
RIGHT:
6363
# Removed, see platformType. ...legacyHive: false
64-
platformType: 'CDP_7_1'
64+
platformType: 'CDP7_1'
6565
# Is the 'Hadoop COMPATIBLE File System' used to prefix data locations for this cluster.
6666
# It is mainly used to as a baseline for where "DATA" will be transfered in the
6767
# STORAGE stage. The data location in the source location will be move to this

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828

2929
<groupId>com.cloudera.utils.hadoop</groupId>
3030
<artifactId>hms-mirror</artifactId>
31-
<version>2.3.1.1</version>
31+
<version>2.3.1.2</version>
3232
<packaging>jar</packaging>
3333

3434
<name>hms-mirror</name>

‎src/main/java/com/cloudera/utils/hms/mirror/datastrategy/DataStrategyBase.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,10 @@ public Boolean buildTableSchema(CopySpec copySpec) throws RequiredConfigurationE
191191
// Cluster as an intermediate holder.
192192
intermediateCluster = config.getCluster(Environment.LEFT).clone();
193193
}
194-
config.getClusters().put(copySpec.getTarget(), intermediateCluster);
194+
if (nonNull(intermediateCluster) && nonNull(intermediateCluster.getHiveServer2())) {
195+
intermediateCluster.getHiveServer2().setDisconnected(Boolean.TRUE);
196+
config.getClusters().put(copySpec.getTarget(), intermediateCluster);
197+
}
195198
}
196199

197200
try {
@@ -356,7 +359,8 @@ public Boolean buildTableSchema(CopySpec copySpec) throws RequiredConfigurationE
356359
TableUtils.removeTblProperty("last_modified_time", target);
357360

358361
// 6. Set 'discover.partitions' if config and non-acid
359-
if (config.getCluster(copySpec.getTarget()).getPartitionDiscovery().isAuto() && TableUtils.isPartitioned(target)) {
362+
if (nonNull(config.getCluster(copySpec.getTarget())) &&
363+
config.getCluster(copySpec.getTarget()).getPartitionDiscovery().isAuto() && TableUtils.isPartitioned(target)) {
360364

361365
if (converted) {
362366
target.addProperty(DISCOVER_PARTITIONS, Boolean.TRUE.toString());
@@ -492,7 +496,8 @@ public Boolean buildTableSchema(CopySpec copySpec) throws RequiredConfigurationE
492496
TableUtils.removeTblProperty(EXTERNAL_TABLE_PURGE, target);
493497
}
494498

495-
if (config.getCluster(copySpec.getTarget()).isLegacyHive() && config.getDataStrategy() != DataStrategyEnum.STORAGE_MIGRATION) {
499+
if (nonNull(config.getCluster(copySpec.getTarget())) &&
500+
config.getCluster(copySpec.getTarget()).isLegacyHive() && config.getDataStrategy() != DataStrategyEnum.STORAGE_MIGRATION) {
496501
// remove newer flags;
497502
TableUtils.removeTblProperty(EXTERNAL_TABLE_PURGE, target);
498503
TableUtils.removeTblProperty(DISCOVER_PARTITIONS, target);

‎src/main/java/com/cloudera/utils/hms/mirror/service/ConfigService.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -962,7 +962,6 @@ public Boolean validate(ExecuteSession session, CliEnvironment cli) {
962962
runStatus.addWarning(RELATIVE_MANUAL);
963963
}
964964
default:
965-
966965
if (nonNull(config.getCluster(Environment.RIGHT)) && config.getCluster(Environment.RIGHT).isHdpHive3()) {
967966
config.getTranslator().setForceExternalLocation(Boolean.TRUE);
968967
runStatus.addWarning(HDP3_HIVE);
@@ -976,6 +975,9 @@ public Boolean validate(ExecuteSession session, CliEnvironment cli) {
976975
runStatus.addError(NON_LEGACY_TO_LEGACY);
977976
rtn.set(Boolean.FALSE);
978977
}
978+
} else {
979+
// Drop the Right cluster to prevent confusion.
980+
config.getClusters().remove(Environment.RIGHT);
979981
}
980982
}
981983

@@ -1172,6 +1174,7 @@ public Boolean validate(ExecuteSession session, CliEnvironment cli) {
11721174
case DUMP:
11731175
// No check needed.
11741176
break;
1177+
case EXPORT_IMPORT:
11751178
case SQL:
11761179
if (config.getMigrateACID().isInplace()) {
11771180
break;

‎src/main/java/com/cloudera/utils/hms/mirror/service/TableService.java‎

Lines changed: 61 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -843,77 +843,83 @@ public Boolean runTableSql(TableMirror tblMirror, Environment environment) {
843843
public Boolean runTableSql(List<Pair> sqlList, TableMirror tblMirror, Environment environment) {
844844
Boolean rtn = Boolean.TRUE;
845845
HmsMirrorConfig config = executeSessionService.getSession().getConfig();
846+
// Check if there is anything to run.
847+
if (nonNull(sqlList) && !sqlList.isEmpty()) {
848+
// Check if the cluster is connected. This could happen if the cluster is a virtual cluster created as a place holder for processing.
849+
if (nonNull(config.getCluster(environment)) && nonNull(config.getCluster(environment).getHiveServer2()) &&
850+
!config.getCluster(environment).getHiveServer2().isDisconnected()) {
851+
// Skip this if using test data.
852+
if (!config.isLoadingTestData()) {
853+
854+
try (Connection conn = getConnectionPoolService().getHS2EnvironmentConnection(environment)) {
855+
if (isNull(conn) && config.isExecute() && !config.getCluster(environment).getHiveServer2().isDisconnected()) {
856+
// this is a problem.
857+
rtn = Boolean.FALSE;
858+
tblMirror.addIssue(environment, "Connection missing. This is a bug.");
859+
}
846860

847-
// Skip this if using test data.
848-
if (!config.isLoadingTestData()) {
849-
850-
try (Connection conn = getConnectionPoolService().getHS2EnvironmentConnection(environment)) {
851-
if (isNull(conn) && config.isExecute() && !config.getCluster(environment).getHiveServer2().isDisconnected()) {
852-
// this is a problem.
853-
rtn = Boolean.FALSE;
854-
tblMirror.addIssue(environment, "Connection missing. This is a bug.");
855-
}
856-
857-
if (isNull(conn) && config.getCluster(environment).getHiveServer2().isDisconnected()) {
858-
tblMirror.addIssue(environment, "Running in 'disconnected' mode. NO RIGHT operations will be done. " +
859-
"The scripts will need to be run 'manually'.");
860-
}
861+
if (isNull(conn) && config.getCluster(environment).getHiveServer2().isDisconnected()) {
862+
tblMirror.addIssue(environment, "Running in 'disconnected' mode. NO RIGHT operations will be done. " +
863+
"The scripts will need to be run 'manually'.");
864+
}
861865

862-
if (rtn && nonNull(conn)) {
863-
try (Statement stmt = conn.createStatement()) {
864-
for (Pair pair : sqlList) {
865-
String action = pair.getAction();
866-
if (action.trim().isEmpty() || action.trim().startsWith("--")) {
867-
continue;
868-
} else {
869-
log.debug("{}:SQL:{}:{}", environment, pair.getDescription(), pair.getAction());
870-
tblMirror.setMigrationStageMessage("Executing SQL: " + pair.getDescription());
871-
if (config.isExecute()) {
872-
// Log the Return of 'set' commands.
873-
if (pair.getAction().trim().toLowerCase().startsWith("set")) {
874-
stmt.execute(pair.getAction());
875-
try {
876-
// Check for a result set and print result if present.
877-
ResultSet resultSet = stmt.getResultSet();
878-
if (!isNull(resultSet)) {
879-
while (resultSet.next()) {
880-
tblMirror.addStep(environment.toString(), "Sql Run Complete for: " + pair.getDescription() + " : " + resultSet.getString(1));
881-
log.info("{}:{}", pair.getAction(), resultSet.getString(1));
866+
if (rtn && nonNull(conn)) {
867+
try (Statement stmt = conn.createStatement()) {
868+
for (Pair pair : sqlList) {
869+
String action = pair.getAction();
870+
if (action.trim().isEmpty() || action.trim().startsWith("--")) {
871+
continue;
872+
} else {
873+
log.debug("{}:SQL:{}:{}", environment, pair.getDescription(), pair.getAction());
874+
tblMirror.setMigrationStageMessage("Executing SQL: " + pair.getDescription());
875+
if (config.isExecute()) {
876+
// Log the Return of 'set' commands.
877+
if (pair.getAction().trim().toLowerCase().startsWith("set")) {
878+
stmt.execute(pair.getAction());
879+
try {
880+
// Check for a result set and print result if present.
881+
ResultSet resultSet = stmt.getResultSet();
882+
if (!isNull(resultSet)) {
883+
while (resultSet.next()) {
884+
tblMirror.addStep(environment.toString(), "Sql Run Complete for: " + pair.getDescription() + " : " + resultSet.getString(1));
885+
log.info("{}:{}", pair.getAction(), resultSet.getString(1));
886+
}
887+
} else {
888+
tblMirror.addStep(environment.toString(), "Sql Run Complete for: " + pair.getDescription());
889+
}
890+
} catch (SQLException se) {
891+
// Otherwise, just log command.
892+
tblMirror.addStep(environment.toString(), "Sql Run Complete for: " + pair.getDescription());
882893
}
883894
} else {
895+
stmt.execute(pair.getAction());
884896
tblMirror.addStep(environment.toString(), "Sql Run Complete for: " + pair.getDescription());
885897
}
886-
} catch (SQLException se) {
887-
// Otherwise, just log command.
888-
tblMirror.addStep(environment.toString(), "Sql Run Complete for: " + pair.getDescription());
898+
} else {
899+
tblMirror.addStep(environment.toString(), "Sql Run SKIPPED (DRY-RUN) for: " + pair.getDescription());
889900
}
890-
} else {
891-
stmt.execute(pair.getAction());
892-
tblMirror.addStep(environment.toString(), "Sql Run Complete for: " + pair.getDescription());
893901
}
894-
} else {
895-
tblMirror.addStep(environment.toString(), "Sql Run SKIPPED (DRY-RUN) for: " + pair.getDescription());
896902
}
903+
} catch (SQLException throwables) {
904+
log.error("{}:{}", environment.toString(), throwables.getMessage(), throwables);
905+
String message = throwables.getMessage();
906+
if (throwables.getMessage().contains("HiveAccessControlException Permission denied")) {
907+
message = message + " See [Hive SQL Exception / HDFS Permissions Issues](https://github.com/cloudera-labs/hms-mirror#hive-sql-exception--hdfs-permissions-issues)";
908+
}
909+
if (throwables.getMessage().contains("AvroSerdeException")) {
910+
message = message + ". It's possible the `avro.schema.url` referenced file doesn't exist at the target. " +
911+
"Use the `-asm` option and hms-mirror will attempt to copy it to the new cluster.";
912+
}
913+
tblMirror.getEnvironmentTable(environment).addError(message);
914+
rtn = Boolean.FALSE;
897915
}
898916
}
899917
} catch (SQLException throwables) {
918+
tblMirror.getEnvironmentTable(environment).addError("Connecting: " + throwables.getMessage());
900919
log.error("{}:{}", environment.toString(), throwables.getMessage(), throwables);
901-
String message = throwables.getMessage();
902-
if (throwables.getMessage().contains("HiveAccessControlException Permission denied")) {
903-
message = message + " See [Hive SQL Exception / HDFS Permissions Issues](https://github.com/cloudera-labs/hms-mirror#hive-sql-exception--hdfs-permissions-issues)";
904-
}
905-
if (throwables.getMessage().contains("AvroSerdeException")) {
906-
message = message + ". It's possible the `avro.schema.url` referenced file doesn't exist at the target. " +
907-
"Use the `-asm` option and hms-mirror will attempt to copy it to the new cluster.";
908-
}
909-
tblMirror.getEnvironmentTable(environment).addError(message);
910920
rtn = Boolean.FALSE;
911921
}
912922
}
913-
} catch (SQLException throwables) {
914-
tblMirror.getEnvironmentTable(environment).addError("Connecting: " + throwables.getMessage());
915-
log.error("{}:{}", environment.toString(), throwables.getMessage(), throwables);
916-
rtn = Boolean.FALSE;
917923
}
918924
}
919925
return rtn;

0 commit comments

Comments
 (0)