Metastore Direct Process to cleanup partition/fs discrepancies. #44

dstreev · dstreev · commit f49e3db2d12d · 2023-08-21T14:21:21.000-04:00
diff --git a/bin/setup.sh b/bin/setup.sh
@@ -45,6 +45,8 @@ rm -f $BASE_DIR/bin/*
 
 cp -f hive-sre $BASE_DIR/bin
 cp -f hive-sre-cli $BASE_DIR/bin
+cp -f u3/get_part_ids.sh $BASE_DIR/bin
+cp -f u3/mysql_missing_parts.sh $BASE_DIR/bin
 
 if [ -f hive-sre-shaded.jar ]; then
     cp -f hive-sre-shaded.jar $BASE_DIR/lib
@@ -57,15 +59,21 @@ fi
 chmod -R +r $BASE_DIR
 chmod +x $BASE_DIR/bin/hive-sre
 chmod +x $BASE_DIR/bin/hive-sre-cli
+chmod +x $BASE_DIR/bin/get_part_ids.sh
+chmod +x $BASE_DIR/bin/mysql_missing_parts.sh
 
 if (( $EUID == 0 )); then
   echo "Setting up global links"
   ln -sf $BASE_DIR/bin/hive-sre /usr/local/bin/hive-sre
   ln -sf $BASE_DIR/bin/hive-sre-cli /usr/local/bin/hive-sre-cli
+  ln -sf $BASE_DIR/bin/get_part_ids.sh /usr/local/bin/get_part_ids.sh
+  ln -sf $BASE_DIR/bin/mysql_missing_parts.sh /usr/local/bin/mysql_missing_parts.sh
 else
   mkdir -p $HOME/bin
   ln -sf $BASE_DIR/bin/hive-sre $HOME/bin/hive-sre
   ln -sf $BASE_DIR/bin/hive-sre-cli $HOME/bin/hive-sre-cli
+  ln -sf $BASE_DIR/bin/get_part_ids.sh $HOME/bin/get_part_ids.sh
+  ln -sf $BASE_DIR/bin/mysql_missing_parts.sh $HOME/bin/mysql_missing_parts.sh
   echo "Executable in $HOME/bin .  Add this to the environment path."
 fi
 
diff --git a/dependency-reduced-pom.xml b/dependency-reduced-pom.xml
@@ -4,7 +4,7 @@
   <groupId>com.cloudera.utils.hive</groupId>
   <artifactId>hive-sre</artifactId>
   <name>hive-sre</name>
-  <version>3.0.0.0</version>
+  <version>3.0.1.0</version>
   <url>https://github.com/cloudera-labs/hive-sre</url>
   <build>
     <finalName>${project.artifactId}</finalName>
diff --git a/pom.xml b/pom.xml
@@ -22,7 +22,7 @@
 
     <groupId>com.cloudera.utils.hive</groupId>
     <artifactId>hive-sre</artifactId>
-    <version>3.0.0.0</version>
+    <version>3.0.1.0</version>
 
     <name>hive-sre</name>
 
diff --git a/src/main/assembly/assembly.xml b/src/main/assembly/assembly.xml
@@ -39,6 +39,15 @@
             <outputDirectory>hive-sre-install</outputDirectory>
             <useDefaultExcludes>true</useDefaultExcludes>
         </fileSet>
+        <fileSet>
+            <directory>${project.basedir}/src/main/bash</directory>
+            <includes>
+                <include>get_part_ids.sh</include>
+                <include>mysql_missing_parts.sh</include>
+            </includes>
+            <outputDirectory>hive-sre-install/u3</outputDirectory>
+            <useDefaultExcludes>true</useDefaultExcludes>
+        </fileSet>
         <fileSet>
             <directory>${project.basedir}/configs</directory>
             <includes>
diff --git a/src/main/bash/get_part_ids.sh b/src/main/bash/get_part_ids.sh
@@ -0,0 +1,39 @@
+#! /bin/bash
+PARTIDS=()
+
+# Parse the loc missing dirs file for the partition id.
+# Build a list of id's that can be added to an 'in' clause
+# in SQL to modify the need tables.
+
+while IFS="|" read -r one two three four five six
+do
+  nospaces=${six// } # remove leading spaces
+
+  re='^[0-9]+$'
+  if  [[ $nospaces =~ $re ]] ; then
+     # echo "$nospaces"
+     PARTIDS+=("$nospaces")
+  fi
+done < <(grep ^\| $1)
+
+arraylength=${#PARTIDS[@]}
+#echo "Total Partition Ids: $arraylength"
+
+# Set the initial item.
+PARTS_LIST=${PARTIDS[0]}
+for (( i=1; i<${arraylength}; i++ ));
+do
+  # New Line every 10000 records.
+  if [ `expr $i % 10000` -eq 0 ];
+  then
+    # Print the list so far.
+    echo "${PARTS_LIST}" >> $2
+    # Reset the list
+    PARTS_LIST=${PARTIDS[$i]}
+  else
+    PARTS_LIST="$PARTS_LIST,${PARTIDS[$i]}"
+  fi
+done
+
+# Print the list
+echo "${PARTS_LIST}" >> $2
diff --git a/src/main/bash/mysql_missing_parts.sh b/src/main/bash/mysql_missing_parts.sh
@@ -0,0 +1,26 @@
+#! /bin/bash
+
+# Build Mysql Statements for Metastore to remove partitions that didn't have directories in HDFS.
+
+TMPDIR=$(mktemp -d)
+
+echo "Temp Dir: $TMPDIR"
+
+$(dirname $0)/get_part_ids.sh $1 $TMPDIR/part_ids.txt
+
+while IFS="|" read -r line
+do
+  echo "Line: $line"
+  # PART_COL_PRIVS
+  echo "DELETE FROM PART_COL_PRIVS WHERE PART_ID IN ($line);" >> mysql_missing_parts.sql
+  # PART_COL_STATS
+  echo "DELETE FROM PART_COL_STATS WHERE PART_ID IN ($line);" >> mysql_missing_parts.sql
+  # PART_PRIVS
+  echo "DELETE FROM PART_PRIVS WHERE PART_ID IN ($line);" >> mysql_missing_parts.sql
+  # PARTITION_KEY_VALS
+  echo "DELETE FROM PARTITION_KEY_VALS WHERE PART_ID IN ($line);" >> mysql_missing_parts.sql
+  # PARTITION_PARAMS
+  echo "DELETE FROM PARTITION_PARAMS WHERE PART_ID IN ($line);" >> mysql_missing_parts.sql
+  # PARTITIONS
+  echo "DELETE FROM PARTITIONS WHERE PART_ID IN ($line);" >> mysql_missing_parts.sql
+done <$TMPDIR/part_ids.txt
diff --git a/src/main/resources/u3/MYSQL/hive_u3_queries.yaml b/src/main/resources/u3/MYSQL/hive_u3_queries.yaml
@@ -41,7 +41,8 @@ query_definitions:
                         WHEN PS.LOCATION IS NULL
                         THEN S.LOCATION
                         ELSE PS.LOCATION
-                    END AS PATH_LOCATION
+                    END AS PATH_LOCATION,
+                    P.PART_ID
                 FROM
                     DBS D
                 INNER JOIN
diff --git a/src/main/resources/u3/ORACLE/hive_u3_queries.yaml b/src/main/resources/u3/ORACLE/hive_u3_queries.yaml
@@ -41,7 +41,8 @@ query_definitions:
                         WHEN PS.LOCATION IS NULL
                         THEN S.LOCATION
                         ELSE PS.LOCATION
-                    END AS PATH_LOCATION
+                    END AS PATH_LOCATION,
+                    P.PART_ID
                 FROM
                     DBS D
                 INNER JOIN
diff --git a/src/main/resources/u3/POSTGRES/hive_u3_queries.yaml b/src/main/resources/u3/POSTGRES/hive_u3_queries.yaml
@@ -38,7 +38,8 @@ query_definitions:
                         WHEN \"PS\".\"LOCATION\" IS NULL
                             THEN \"S\".\"LOCATION\"
                         ELSE \"PS\".\"LOCATION\"
-                        END AS \"PATH_LOCATION\"
+                        END AS \"PATH_LOCATION\",
+                    \"P\".\"PART_ID\",
                 FROM
                     \"DBS\" \"D\"
                         INNER JOIN
diff --git a/src/main/resources/u3/proc/hive_u3_procs.yaml b/src/main/resources/u3/proc/hive_u3_procs.yaml
@@ -19,11 +19,11 @@ processes:
     displayName:              "Locations Scan"
     queryDefinitionReference: "/hive_u3_queries.yaml"
     dbListingQuery:           "db_tbl_count"
-    listingColumns: [ "name" ,"tbl_name" , "tbl_type" ,"part_name" , "path_location" ]
+    listingColumns: [ "name" ,"tbl_name" , "tbl_type" ,"part_name" , "path_location", "part_id" ]
     pathsListingQuery:        "tbl_part_locations"
     commandChecks:
       - displayName:        "Hive 3 Upgrade Check - Missing Directory Locations Scan"
-        title:              "# Hive 3 Upgrade Check - Missing Direcotories (v.${Implementation-Version})\n\n## Missing Directory Locations Scan"
+        title:              "# Hive 3 Upgrade Check - Missing Directories (v.${Implementation-Version})\n\n## Missing Directory Locations Scan"
         note:               "
          *Remediation Options*\n
                 _NOTE:_ Beware of \"Smart Quote\" AND other conversions via Markdown Renderers. Copy/paste from 'raw' text to ensure compatibility.\n\n
@@ -39,7 +39,7 @@ processes:
          always a good practice.\n\n
          This process should be run frequently, up to the point of the upgrade, to ensure all items have been addressed.  Before
          the upgrade can succeed, this report should be *empty*.\n\n"
-        header:             "| DB.Table:Partition Spec | Hive SQL (recommended) | HDFS | Hive MSCK (Post Upgrade) |\n|:---|:---|:---|:---|"
+        header:             "| DB.Table:Partition Spec | Hive SQL (recommended) | HDFS | Hive MSCK (Post Upgrade) | PART ID |\n|:---|:---|:---|:---|:---|"
         errorDescription:   "Scan Actions"
         successDescription: "Issues"
         errorFilename:      loc_scan_missing_dirs.md
@@ -54,8 +54,8 @@ processes:
           partitionCheck:
             RECORDS:
               test: "if (\"%4$s\".equals(\" \")) true; else false;"
-              pass: "\"| %1$s.%2$s | DROP TABLE IF EXISTS `%1$s`.`%2$s`; | mkdir -p \\\"%5$s\\\" | MSCK REPAIR TABLE `%1$s`.`%2$s` SYNC PARTITIONS; |\""
-              fail: "\"| %1$s.%2$s:\" + com.cloudera.utils.hive.sre.Utils.dirToPartitionSpec('%4$s') + \" | ALTER TABLE `%1$s`.`%2$s` DROP IF EXISTS PARTITION (\" + com.cloudera.utils.hive.sre.Utils.dirToPartitionSpec('%4$s') + \"); | mkdir -p \\\"%5$s\\\" | |\""
+              pass: "\"| %1$s.%2$s | DROP TABLE IF EXISTS `%1$s`.`%2$s`; | mkdir -p \\\"%5$s\\\" | MSCK REPAIR TABLE `%1$s`.`%2$s` SYNC PARTITIONS; | |\""
+              fail: "\"| %1$s.%2$s:\" + com.cloudera.utils.hive.sre.Utils.dirToPartitionSpec('%4$s') + \" | ALTER TABLE `%1$s`.`%2$s` DROP IF EXISTS PARTITION (\" + com.cloudera.utils.hive.sre.Utils.dirToPartitionSpec('%4$s') + \"); | mkdir -p \\\"%5$s\\\" | | %6$s |\""
               params: [ "hive" ]
 
   - type:                     "dbSet"
diff --git a/u3e.md b/u3e.md
@@ -27,3 +27,20 @@ These scripts will execute directly against the METASTORE DB, skipping the HiveS
 
 - Kudu Serde / Table Upgrades.  See: https://kudu.apache.org/docs/hive_metastore.html
 - Legacy Decimal (Hive 0.12) issues.  Refer to description of these in the `u3` report.
+
+### Shortcut the Partition Missing Directories Process
+
+The output of the `u3 -hdp2|-cdh` process creates an output file called `loc_scan_missing_dirs.md` that contains a list of all the existing partitions that have no matching directory on the filesystem.  In the report, there are several options available to correct these inconsistencies.  You can create empty directories that match the partition definitions, or remove the partitions via HiveSQL.
+
+Or these two options, dropping the partitions in Hive is preferred.  But a large number of these inconsistencies can take a lot of time to run via HiveSQL.  An alternative is available that builds a list of the partition ids and then build out the `MYSQL` scripts to drop them from the metastore directly.  This process is much faster than running the HiveSQL scripts.
+
+Run the `u3` process as normal to generate the `loc_scan_missing_dirs.md` file.  Installed in the path is a script called `mysql_missing_parts.sh`, which should be on the path (in same directory as the `hive-sre` script. 
+
+**Recommend starting small with a test hive database first.**
+
+**BACKUP THE METASTORE DB BEFORE RUNNING THIS PROCESS**
+
+1. `cd <report_output_directory>` (where the `loc_scan_missing_dirs.md` file is).
+2. run: `mysql_missing_parts.sh loc_scan_missing_dirs.md`.  In the output directory this generates an `mysql` script file called `mysql_missing_parts.sql`.
+3. run the `mysql_missing_parts.sql` script through your favorite **MYSQL** cli application against that metastore RDBMS.
+4. rerun the `u3` process to confirm the partition discrepancies are gone.
