Added feature to transfer table owner detail with -to.

Author: dstreev

README.md

@@ -758,7 +758,7 @@ After running the `setup.sh` script, `hms-mirror` will be available in the `$PAT
 ### Assumptions
 
 1. This process will only 'migrate' EXTERNAL and MANAGED (non-ACID/Transactional) table METADATA (not data, except with [SQL](#sql) and [EXPORT_IMPORT](#export-import) ).
-2. MANAGED tables replicated to the **RIGHT** cluster will be converted to "EXTERNAL" tables for the 'metadata' stage.  They will be tagged as 'legacy managed' in the **RIGHT** cluster and will NOT be assigned the `external.table.purge=true` flag, yet.  Once the table's data has been migrated, the table will be adjusted to be purgeable via `external.table.purge=true` to match the classic `MANAGED` table behavior.
+2. MANAGED tables replicated to the **RIGHT** cluster will be converted to "EXTERNAL" tables for the 'metadata' stage.  They will be tagged as 'legacy managed' in the **RIGHT** cluster.  They will be assigned the `external.table.purge=true` flag, to continue the behaviors of the legacy managed tables.
 1. The **RIGHT** cluster has 'line of sight' to the **LEFT** cluster.
 2. The **RIGHT** cluster has been configured to access the **LEFT** cluster storage. See [link clusters](#linking-clusters-storage-layers).  This is the same configuration required to support `distcp` from the **RIGHT** cluster to the **LEFT** cluster.
 3. The movement of metadata/data is from the **LEFT** cluster to the **RIGHT** cluster.
@@ -779,7 +779,7 @@ When you do need to move data, `hms-mirror` create a workbook of 'source' and 't
 
 ```
 usage: hms-mirror <options>
-                  version:1.5.3.1.0-SNAPSHOT
+                  version: *******
 Hive Metastore Migration Utility
  -accept,--accept                                                          Accept ALL confirmations
                                                                            and silence prompts
@@ -1059,6 +1059,22 @@ Hive Metastore Migration Utility
                                                                            double-quote the
                                                                            expression on the
                                                                            commandline.
+ -to,--transfer-ownership                                                  If available (supported)
+                                                                           on LEFT cluster, extract
+                                                                           and transfer the tables
+                                                                           owner to the RIGHT
+                                                                           cluster. Note: This will
+                                                                           make an 'exta' SQL call
+                                                                           on the LEFT cluster to
+                                                                           determine the ownership.
+                                                                           This won't be supported
+                                                                           on CDH 5 and some other
+                                                                           legacy Hive platforms.
+                                                                           Beware the cost of this
+                                                                           extra call for EVERY
+                                                                           table, as it may slow
+                                                                           down the process for a
+                                                                           large volume of tables.
  -v,--views-only                                                           Process VIEWs ONLY
  -wd,--warehouse-directory <warehouse-path>                                The warehouse directory
                                                                            path.  Should not include

pom.xml

@@ -22,7 +22,7 @@
 
     <groupId>com.cloudera.utils.hadoop</groupId>
     <artifactId>hms-mirror</artifactId>
-    <version>1.5.3.1.3-SNAPSHOT</version>
+    <version>1.5.3.2.0-SNAPSHOT</version>
     <name>hms-mirror</name>
 
     <url>https://github.com/dstreev/hms_mirror</url>

src/main/java/com/cloudera/utils/hadoop/hms/Mirror.java

@@ -396,6 +396,10 @@ public long init(String[] args) {
                 config.setCopyAvroSchemaUrls(Boolean.TRUE);
             }
 
+            if (cmd.hasOption("to")) {
+                config.setTransferOwnership(Boolean.TRUE);
+            }
+
             String dataStrategyStr = cmd.getOptionValue("d");
             // default is SCHEMA_ONLY
             if (dataStrategyStr != null) {
@@ -1381,6 +1385,15 @@ private Options getOptions() {
         asmOption.setRequired(Boolean.FALSE);
         options.addOption(asmOption);
 
+        Option transferOwnershipOption = new Option("to", "transfer-ownership", false,
+                "If available (supported) on LEFT cluster, extract and transfer the tables owner to the " +
+                        "RIGHT cluster. Note: This will make an 'exta' SQL call on the LEFT cluster to determine " +
+                        "the ownership.  This won't be supported on CDH 5 and some other legacy Hive platforms. " +
+                        "Beware the cost of this extra call for EVERY table, as it may slow down the process for " +
+                        "a large volume of tables.");
+        transferOwnershipOption.setRequired(Boolean.FALSE);
+        options.addOption(transferOwnershipOption);
+
         OptionGroup dbAdjustOptionGroup = new OptionGroup();
 
         Option dbPrefixOption = new Option("dbp", "db-prefix", true,

src/main/java/com/cloudera/utils/hadoop/hms/mirror/Cluster.java

@@ -287,6 +287,7 @@ public void getTables(Config config, DBMirror dbMirror) throws SQLException {
         }
     }
 
+
     public void getTableDefinition(Config config, String database, TableMirror tableMirror) throws SQLException {
         // The connection should already be in the database;
         Connection conn = null;
@@ -358,6 +359,33 @@ public void getTableDefinition(Config config, String database, TableMirror table
                         }
                     }
 
+                    if (config.getTransferOwnership()) {
+                        try {
+                            String ownerStatement = MessageFormat.format(MirrorConf.SHOW_TABLE_EXTENDED, tableMirror.getName());
+                            resultSet = stmt.executeQuery(ownerStatement);
+                            String owner = null;
+                            while (resultSet.next()) {
+
+                                if (resultSet.getString(1).startsWith("owner")) {
+                                    String[] ownerLine = resultSet.getString(1).split(":");
+                                    try {
+                                        owner = ownerLine[1];
+                                    } catch (Throwable t) {
+                                        // Parsing issue.
+                                        LOG.error("Couldn't parse 'owner' value from: " + resultSet.getString(1) +
+                                                " for table: " + tableMirror.getDbName() + "." + tableMirror.getName());
+                                    }
+                                    break;
+                                }
+                            }
+                            if (owner != null) {
+                                et.setOwner(owner);
+                            }
+                        }  catch (SQLException sed) {
+                            // Failed to gather owner details.
+                        }
+                    }
+
                     LOG.debug(getEnvironment() + ":" + database + "." + et.getName() +
                             ": Loaded Table Definition");
                 } catch (SQLException throwables) {

src/main/java/com/cloudera/utils/hadoop/hms/mirror/Config.java

@@ -140,6 +140,7 @@ public class Config {
     private String tblExcludeRegEx = null;
 
     private TransferConfig transfer = new TransferConfig();
+    private Boolean transferOwnership = Boolean.FALSE;
     @JsonIgnore
     private ScheduledExecutorService transferThreadPool;
     @JsonIgnore
@@ -1066,6 +1067,14 @@ public void setTransfer(TransferConfig transfer) {
         this.transfer = transfer;
     }
 
+    public Boolean getTransferOwnership() {
+        return transferOwnership;
+    }
+
+    public void setTransferOwnership(Boolean transferOwnership) {
+        this.transferOwnership = transferOwnership;
+    }
+
     public Map<Environment, Cluster> getClusters() {
         return clusters;
     }

src/main/java/com/cloudera/utils/hadoop/hms/mirror/EnvironmentTable.java

@@ -27,6 +27,8 @@ public class EnvironmentTable {
     private CreateStrategy createStrategy = CreateStrategy.NOTHING;
     private List<String> definition = new ArrayList<String>();
     private final Boolean partitioned = Boolean.FALSE;
+    private String owner = null;
+
     private List<String> partitions = new ArrayList<String>();
     private List<String> actions = new ArrayList<String>();
     private Map<String, String> addProperties = new TreeMap<String, String>();
@@ -88,6 +90,14 @@ public List<String> getPartitions() {
         return partitions;
     }
 
+    public String getOwner() {
+        return owner;
+    }
+
+    public void setOwner(String owner) {
+        this.owner = owner;
+    }
+
     public void setPartitions(List<String> partitions) {
         this.partitions = partitions;
     }

src/main/java/com/cloudera/utils/hadoop/hms/mirror/MirrorConf.java

@@ -21,7 +21,9 @@ public class MirrorConf {
 
     public static final String SHOW_TABLES = "SHOW TABLES";
     public static final String SHOW_VIEWS = "SHOW VIEWS";
+    public static final String SHOW_TABLE_EXTENDED = "SHOW TABLE EXTENDED LIKE {0}";
     public static final String SHOW_CREATE_TABLE = "SHOW CREATE TABLE {0}";
+    public static final String DESCRIBE_FORMATTED_TABLE = "DESCRIBE FORMATTED {0}";
     public static final String SHOW_PARTITIONS = "SHOW PARTITIONS {0}.{1}";
     public static final String MSCK_REPAIR_TABLE = "MSCK REPAIR TABLE {0}";
     public static final String MSCK_REPAIR_TABLE_DESC = "MSCK Repair Table";
@@ -55,6 +57,8 @@ public class MirrorConf {
     public static final String DROP_VIEW = "DROP VIEW IF EXISTS {0}";
     public static final String RENAME_TABLE_DESC = "Rename table";
     public static final String RENAME_TABLE = " ALTER TABLE {0} RENAME TO {1}";
+    public static final String SET_OWNER_DESC = "Set table owner";
+    public static final String SET_OWNER = "ALTER TABLE {0} SET OWNER USER {1}";
     public static final String EXPORT_TABLE =
             "EXPORT TABLE {0} TO \"{1}\"";
     public static final String IMPORT_EXTERNAL_TABLE =

src/main/java/com/cloudera/utils/hadoop/hms/mirror/TableMirror.java

@@ -972,6 +972,10 @@ public Boolean buildoutDUMPSql(Config config, DBMirror dbMirror) {
 
         createTbl = this.getCreateStatement(Environment.LEFT);
         let.addSql(TableUtils.CREATE_DESC, createTbl);
+        if (!config.getCluster(Environment.LEFT).getLegacyHive() && config.getTransferOwnership() && let.getOwner() != null) {
+            String ownerSql = MessageFormat.format(MirrorConf.SET_OWNER, let.getName(), let.getOwner());
+            let.addSql(MirrorConf.SET_OWNER_DESC, ownerSql);
+        }
 
         // If partitioned, !ACID, repair
         if (let.getPartitioned() && !TableUtils.isACID(let) &&
@@ -1028,6 +1032,10 @@ public Boolean buildoutSCHEMA_ONLYSql(Config config, DBMirror dbMirror) {
                 ret.addSql(TableUtils.USE_DESC, useDb);
                 String createStmt2 = getCreateStatement(Environment.RIGHT);
                 ret.addSql(TableUtils.CREATE_DESC, createStmt2);
+                if (!config.getCluster(Environment.RIGHT).getLegacyHive() && config.getTransferOwnership() && let.getOwner() != null) {
+                    String ownerSql = MessageFormat.format(MirrorConf.SET_OWNER, ret.getName(), let.getOwner());
+                    ret.addSql(MirrorConf.SET_OWNER_DESC, ownerSql);
+                }
                 break;
         }
 
@@ -1154,6 +1162,10 @@ public Boolean buildoutSQLSql(Config config, DBMirror dbMirror) {
                 case CREATE:
                     String createStmt2 = getCreateStatement(Environment.RIGHT);
                     ret.addSql(TableUtils.CREATE_DESC, createStmt2);
+                    if (!config.getCluster(Environment.RIGHT).getLegacyHive() && config.getTransferOwnership() && let.getOwner() != null) {
+                        String ownerSql = MessageFormat.format(MirrorConf.SET_OWNER, ret.getName(), let.getOwner());
+                        ret.addSql(MirrorConf.SET_OWNER_DESC, ownerSql);
+                    }
                     break;
             }
 
@@ -1189,6 +1201,10 @@ public Boolean buildoutSTORAGEMIGRATIONSql(Config config, DBMirror dbMirror) {
         // Create table with New Location
         String createStmt2 = getCreateStatement(Environment.RIGHT);
         let.addSql(TableUtils.CREATE_DESC, createStmt2);
+        if (!config.getCluster(Environment.LEFT).getLegacyHive() && config.getTransferOwnership() && let.getOwner() != null) {
+            String ownerSql = MessageFormat.format(MirrorConf.SET_OWNER, let.getName(), let.getOwner());
+            let.addSql(MirrorConf.SET_OWNER_DESC, ownerSql);
+        }
 
         // Drop Renamed Table.
         String dropOriginalTable = MessageFormat.format(MirrorConf.DROP_TABLE, let.getName());
@@ -1266,6 +1282,10 @@ public Boolean buildoutIntermediateSql(Config config, DBMirror dbMirror) {
             case CREATE:
                 String createStmt2 = getCreateStatement(Environment.RIGHT);
                 ret.addSql(TableUtils.CREATE_DESC, createStmt2);
+                if (!config.getCluster(Environment.RIGHT).getLegacyHive() && config.getTransferOwnership() && let.getOwner() != null) {
+                    String ownerSql = MessageFormat.format(MirrorConf.SET_OWNER, ret.getName(), let.getOwner());
+                    ret.addSql(MirrorConf.SET_OWNER_DESC, ownerSql);
+                }
                 if (let.getPartitioned()) {
                     if (config.getTransfer().getCommonStorage() != null) {
                         if (!TableUtils.isACID(let) || (TableUtils.isACID(let) && config.getMigrateACID().isDowngrade())) {

src/test/java/com/cloudera/utils/hadoop/hms/LegacySchemaMigrationTest.java

@@ -219,6 +219,26 @@ public void test_so_leg() {
         assertTrue("Return Code Failure: " + rtn + " doesn't match: " + check, rtn == check);
     }
 
+    @Test
+    public void test_so_owner_leg() {
+        String nameofCurrMethod = new Throwable()
+                .getStackTrace()[0]
+                .getMethodName();
+
+        String outputDir = outputDirBase + nameofCurrMethod;
+
+        String[] args = new String[]{"-db", DataState.getInstance().getWorking_db(),
+                "-to",
+                "-o", outputDir, "-cfg", DataState.getInstance().getConfiguration()};
+        args = toExecute(args, execArgs, Boolean.FALSE);
+
+        long rtn = 0;
+        Mirror mirror = new Mirror();
+        rtn = mirror.go(args);
+        int check = 0;
+        assertTrue("Return Code Failure: " + rtn + " doesn't match: " + check, rtn == check);
+    }
+
     @Test
     public void test_so_distcp_leg() {
         String nameofCurrMethod = new Throwable()

src/test/java/com/cloudera/utils/hadoop/hms/MirrorTestBase.java

@@ -90,6 +90,7 @@ public Boolean dataSetup01() {
                 dataset = getDataset(2, 2000, null);
             }
             build_n_populate(CREATE_EXTERNAL_TBL, TBL_INSERT, dataset, leftSql,new String[]{"ext_part_02"});
+            build_n_populate(CREATE_LEGACY_MNGD_TBL, TBL_INSERT, dataset, leftSql,new String[]{"legacy_mngd_01"});
 
             Mirror cfgMirror = new Mirror();
             long rtn = cfgMirror.setupSqlLeft(args, leftSql);