SQL Strategy only uses MSCK for partition discovery for Shadow Table: #145

Author: dstreev

Writerside/images/cluster_tabs.png

@@ -27,7 +27,7 @@ Modify the `hms-mirror` configuration to include the following settings:
 ``` yaml
 clusters:
   LEFT|RIGHT:
-    platformType: HDP2|HDP3|CHD5|CDH6|CDP7.1|CDP7_2|..
+    platformType: HDP2|HDP3|CHD5|CDH6|CDP7.1|CDP7_2|...
 ```
 </tab>
 </tabs>

Writerside/images/hive_version_flags.png

@@ -28,7 +28,7 @@
 
     <groupId>com.cloudera.utils.hadoop</groupId>
     <artifactId>hms-mirror</artifactId>
-    <version>2.2.0.12</version>
+    <version>2.2.0.12.1</version>
     <packaging>jar</packaging>
 
     <name>hms-mirror</name>

Writerside/topics/Hive-Conversions.md

@@ -225,10 +225,31 @@ public Boolean buildOutSql(TableMirror tableMirror) throws MissingDataPointExcep
                 ret.addSql(TableUtils.CREATE_SHADOW_DESC, shadowCreateStmt);
                 // Repair Partitions
                 // TODO: Handle odd partitions.
-                if (let.getPartitioned()) {
+                if (config.loadMetadataDetails()) {
+                    // TODO: Write out the SQL to build the partitions.  NOTE: We need to get the partition locations and modify them
+                    //       to the new namespace.
+                    String tableParts = getTranslatorService().buildPartitionAddStatement(let);
+                    // This will be empty when there's no data and we need to handle that.
+                    if (!isBlank(tableParts)) {
+                        String addPartSql = MessageFormat.format(MirrorConf.ALTER_TABLE_PARTITION_ADD_LOCATION, set.getName(), tableParts);
+                        ret.addSql(MirrorConf.ALTER_TABLE_PARTITION_ADD_LOCATION_DESC, addPartSql);
+                    }
+                } else {// if (config.getCluster(Environment.RIGHT).getPartitionDiscovery().isInitMSCK()) {
                     String shadowMSCKStmt = MessageFormat.format(MirrorConf.MSCK_REPAIR_TABLE, set.getName());
                     ret.addSql(TableUtils.REPAIR_DESC, shadowMSCKStmt);
+                    //                    String msckStmt = MessageFormat.format(MirrorConf.MSCK_REPAIR_TABLE, ret.getName());
+//                    // Add the MSCK repair to both initial and cleanup.
+//                    ret.addSql(TableUtils.REPAIR_DESC, msckStmt);
+//                    if (config.getTransfer().getStorageMigration().isDistcp()) {
+//                        ret.addCleanUpSql(TableUtils.REPAIR_DESC, msckStmt);
+//                    }
                 }
+
+
+//                if (let.getPartitioned()) {
+//                    String shadowMSCKStmt = MessageFormat.format(MirrorConf.MSCK_REPAIR_TABLE, set.getName());
+//                    ret.addSql(TableUtils.REPAIR_DESC, shadowMSCKStmt);
+//                }
             }
         }
 

pom.xml

@@ -292,8 +292,10 @@ public static boolean save(HmsMirrorConfig config, String configFilename, Boolea
     @JsonIgnore
     public Boolean loadMetadataDetails() {
         // When we're ALIGNED and asking fir DISTCP, we need to load the partition metadata.
-        if (transfer.getStorageMigration().getTranslationType() == TranslationTypeEnum.ALIGNED
-                && getTransfer().getStorageMigration().isDistcp()) {
+        if ((transfer.getStorageMigration().getTranslationType() == TranslationTypeEnum.ALIGNED
+                && getTransfer().getStorageMigration().isDistcp()) ||
+                (dataStrategy == DataStrategyEnum.SQL
+                        && getTransfer().getStorageMigration().getDataMovementStrategy() == DataMovementStrategyEnum.SQL)) {
             switch (dataStrategy) {
                 case LINKED:
                 case COMMON:

src/main/java/com/cloudera/utils/hms/mirror/datastrategy/IntermediateDataStrategy.java

@@ -1005,10 +1005,10 @@ public Boolean validate(ExecuteSession session, CliEnvironment cli) {
                         rtn.set(Boolean.FALSE);
                     }
                 }
-                if (!isBlank(config.getTransfer().getTargetNamespace())) {
-                    runStatus.addError(SQL_DISTCP_ACID_W_STORAGE_OPTS);
-                    rtn.set(Boolean.FALSE);
-                }
+//                if (!isBlank(config.getTransfer().getTargetNamespace())) {
+//                    runStatus.addError(SQL_DISTCP_ACID_W_STORAGE_OPTS);
+//                    rtn.set(Boolean.FALSE);
+//                }
             }