Fixed mysql partition cleanup scripts. #52

Author: dstreev

bin/deploy.yaml

@@ -41,6 +41,18 @@
         dest: /usr/local/hive-sre/bin
         mode: 0755
 
+    - name: Copy parts
+      copy:
+        src: ../src/main/bash/get_part_ids.sh
+        dest: /usr/local/hive-sre/bin
+        mode: 0755
+
+    - name: Copy missing parts
+      copy:
+        src: ../src/main/bash/mysql_missing_parts.sh
+        dest: /usr/local/hive-sre/bin
+        mode: 0755
+
     - name: Copy lib
       copy:
         src: ../target/hive-sre-shaded.jar
@@ -66,3 +78,17 @@
         dest: /usr/local/bin/hive-sre-cli
         mode: 0755
         state: link
+
+    - name: Link Parts
+      file:
+        src: /usr/local/hive-sre/bin/get_part_ids.sh
+        dest: /usr/local/bin/get_part_ids.sh
+        mode: 0755
+        state: link
+
+    - name: Link Missing Parts
+      file:
+        src: /usr/local/hive-sre/bin/mysql_missing_parts.sh
+        dest: /usr/local/bin/mysql_missing_parts.sh
+        mode: 0755
+        state: link

pom.xml

@@ -22,7 +22,7 @@
 
     <groupId>com.cloudera.utils.hive</groupId>
     <artifactId>hive-sre</artifactId>
-    <version>3.0.1.3</version>
+    <version>3.0.1.4</version>
 
     <name>hive-sre</name>
 

src/main/bash/get_part_ids.sh

@@ -5,16 +5,20 @@ PARTIDS=()
 # Build a list of id's that can be added to an 'in' clause
 # in SQL to modify the need tables.
 
-while IFS="|" read -r one two three four five six
+awk 'match($0, /\| [0-9]+ \|/) {
+    print substr($0, RSTART, RLENGTH)
+}' $1 > $1.part
+
+while IFS=" " read -r one two three
 do
-  nospaces=${six// } # remove leading spaces
+  nospaces=${two// } # remove leading spaces
 
   re='^[0-9]+$'
   if  [[ $nospaces =~ $re ]] ; then
      # echo "$nospaces"
      PARTIDS+=("$nospaces")
   fi
-done < <(grep ^\| $1)
+done < $1.part
 
 arraylength=${#PARTIDS[@]}
 #echo "Total Partition Ids: $arraylength"

src/main/bash/mysql_missing_parts.sh

@@ -8,7 +8,7 @@ echo "Temp Dir: $TMPDIR"
 
 $(dirname $0)/get_part_ids.sh $1 $TMPDIR/part_ids.txt
 
-while IFS="|" read -r line
+for line in `cat $TMPDIR/part_ids.txt`
 do
   echo "Line: $line"
   # PART_COL_PRIVS
@@ -23,4 +23,4 @@ do
   echo "DELETE FROM PARTITION_PARAMS WHERE PART_ID IN ($line);" >> mysql_missing_parts.sql
   # PARTITIONS
   echo "DELETE FROM PARTITIONS WHERE PART_ID IN ($line);" >> mysql_missing_parts.sql
-done <$TMPDIR/part_ids.txt
+done