HIVE-28292: Optimize SHOW TABLES|VIEWS statements

ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/tables/ShowTablesFormatter.java

@@ -19,12 +19,13 @@
 package org.apache.hadoop.hive.ql.ddl.table.info.show.tables;
 
 import com.google.common.collect.ImmutableMap;
+
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.ddl.ShowUtils;
 import org.apache.hadoop.hive.ql.ddl.ShowUtils.TextMetaDataTable;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.metadata.formatting.MapBuilder;
 import org.apache.hadoop.hive.ql.metadata.formatting.MetaDataFormatUtils;
 import org.apache.hadoop.hive.ql.session.SessionState;
@@ -51,7 +52,7 @@ public static ShowTablesFormatter getFormatter(HiveConf conf) {
 
   public abstract void showTables(DataOutputStream out, List<String> tables) throws HiveException;
 
-  abstract void showTablesExtended(DataOutputStream out, List<Table> tables) throws HiveException;
+  abstract void showTablesExtended(DataOutputStream out, List<Pair<String, String>> tables) throws HiveException;
 
   // ------ Implementations ------
 
@@ -62,16 +63,16 @@ public void showTables(DataOutputStream out, List<String> tables) throws HiveExc
     }
 
     @Override
-    void showTablesExtended(DataOutputStream out, List<Table> tables) throws HiveException {
+    void showTablesExtended(DataOutputStream out, List<Pair<String, String>> tables) throws HiveException {
       if (tables.isEmpty()) {
         return;
       }
 
       List<Map<String, Object>> tableDataList = new ArrayList<>();
-      for (Table table : tables) {
+      for (Pair<String, String> table : tables) {
         Map<String, Object> tableData = ImmutableMap.of(
-            "Table Name", table.getTableName(),
-            "Table Type", table.getTableType().toString());
+            "Table Name", table.getLeft(),
+            "Table Type", table.getRight());
         tableDataList.add(tableData);
       }
 
@@ -96,7 +97,7 @@ public void showTables(DataOutputStream out, List<String> tables) throws HiveExc
     }
 
     @Override
-    void showTablesExtended(DataOutputStream out, List<Table> tables) throws HiveException {
+    void showTablesExtended(DataOutputStream out, List<Pair<String, String>> tables) throws HiveException {
       if (tables.isEmpty()) {
         return;
       }
@@ -106,8 +107,8 @@ void showTablesExtended(DataOutputStream out, List<Table> tables) throws HiveExc
         if (!SessionState.get().isHiveServerQuery()) {
           mdt.addRow("# Table Name", "Table Type");
         }
-        for (Table table : tables) {
-          mdt.addRow(table.getTableName(), table.getTableType().toString());
+        for (Pair<String, String> table : tables) {
+          mdt.addRow(table.getLeft(), table.getRight());
         }
         // In case the query is served by HiveServer2, don't pad it with spaces,
         // as HiveServer2 output is consumed by JDBC/ODBC clients.

ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/show/tables/ShowTablesOperation.java

@@ -26,15 +26,15 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.ddl.DDLOperation;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.udf.UDFLike;
 
 /**
  * Operation process showing the tables.
@@ -60,12 +60,9 @@ public int execute() throws HiveException {
   }
 
   private void showTables() throws HiveException {
+    String pattern = MetaStoreUtils.convertSqlPatternToRegExp(desc.getPattern());
     List<String> tableNames = new ArrayList<>(
-        context.getDb().getTablesByType(desc.getDbName(), null, desc.getTypeFilter()));
-    if (desc.getPattern() != null) {
-      Pattern pattern = Pattern.compile(UDFLike.likePatternToRegExp(desc.getPattern()), Pattern.CASE_INSENSITIVE);
-      tableNames = tableNames.stream().filter(name -> pattern.matcher(name).matches()).collect(Collectors.toList());
-    }
+        context.getDb().getTablesByType(desc.getDbName(), pattern, desc.getTypeFilter()));
     Collections.sort(tableNames);
     LOG.debug("Found {} table(s) matching the SHOW TABLES statement.", tableNames.size());
 
@@ -78,20 +75,22 @@ private void showTables() throws HiveException {
   }
 
   private void showTablesExtended() throws HiveException {
-    List<Table> tableObjects = new ArrayList<>();
-    tableObjects.addAll(context.getDb().getTableObjects(desc.getDbName(), null, desc.getTypeFilter()));
-    if (desc.getPattern() != null) {
-      Pattern pattern = Pattern.compile(UDFLike.likePatternToRegExp(desc.getPattern()), Pattern.CASE_INSENSITIVE);
-      tableObjects = tableObjects.stream()
-          .filter(object -> pattern.matcher(object.getTableName()).matches())
-          .collect(Collectors.toList());
+    List<Pair<String, String>> tableTypePairs = new ArrayList<>();
+    String pattern = MetaStoreUtils.convertSqlPatternToRegExp(desc.getPattern());
+    TableType typeFilter = desc.getTypeFilter();
+    TableType[] tableTypes = typeFilter == null ? TableType.values() : new TableType[]{typeFilter};
+    for (TableType tableType : tableTypes) {
+      String typeString = tableType.toString();
+      List<String> tables = context.getDb().getTablesByType(desc.getDbName(), pattern, tableType);
+      tableTypePairs.addAll(tables.stream()
+          .map(table -> Pair.of(table, typeString)).collect(Collectors.toList()));
     }
-    Collections.sort(tableObjects, Comparator.comparing(Table::getTableName));
-    LOG.debug("Found {} table(s) matching the SHOW EXTENDED TABLES statement.", tableObjects.size());
+    Collections.sort(tableTypePairs, Comparator.comparing(Pair::getLeft));
+    LOG.debug("Found {} table(s) matching the SHOW EXTENDED TABLES statement.", tableTypePairs.size());
 
     try (DataOutputStream os = ShowUtils.getOutputStream(new Path(desc.getResFile()), context)) {
       ShowTablesFormatter formatter = ShowTablesFormatter.getFormatter(context.getConf());
-      formatter.showTablesExtended(os, tableObjects);
+      formatter.showTablesExtended(os, tableTypePairs);
     } catch (Exception e) {
       throw new HiveException(e, ErrorMsg.GENERIC_ERROR, "in database " + desc.getDbName());
     }

ql/src/java/org/apache/hadoop/hive/ql/ddl/view/show/ShowViewsOperation.java

@@ -21,18 +21,16 @@
 import java.io.DataOutputStream;
 import java.util.Collections;
 import java.util.List;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
 
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
 import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.ddl.DDLOperation;
 import org.apache.hadoop.hive.ql.ddl.DDLOperationContext;
 import org.apache.hadoop.hive.ql.ddl.ShowUtils;
 import org.apache.hadoop.hive.ql.ddl.table.info.show.tables.ShowTablesFormatter;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.udf.UDFLike;
 
 /**
  * Operation process showing the views.
@@ -48,13 +46,8 @@ public int execute() throws HiveException {
       throw new HiveException(ErrorMsg.DATABASE_NOT_EXISTS, desc.getDbName());
     }
 
-    List<String> viewNames = context.getDb().getTablesByType(desc.getDbName(), null, TableType.VIRTUAL_VIEW);
-    if (desc.getPattern() != null) {
-      Pattern pattern = Pattern.compile(UDFLike.likePatternToRegExp(desc.getPattern()), Pattern.CASE_INSENSITIVE);
-      viewNames = viewNames.stream()
-          .filter(name -> pattern.matcher(name).matches())
-          .collect(Collectors.toList());
-    }
+    String pattern = MetaStoreUtils.convertSqlPatternToRegExp(desc.getPattern());
+    List<String> viewNames = context.getDb().getTablesByType(desc.getDbName(), pattern, TableType.VIRTUAL_VIEW);
     Collections.sort(viewNames);
     LOG.debug("Found {} view(s) matching the SHOW VIEWS statement.", viewNames.size());
 

standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java

@@ -2897,28 +2897,12 @@ public List<String> getTables(String dbname, String tablePattern) throws MetaExc
   @Override
   public List<String> getTables(String catName, String dbName, String tablePattern)
       throws TException {
-    List<String> tables = new ArrayList<>();
-    GetProjectionsSpec projectionsSpec = new GetProjectionsSpec();
-    projectionsSpec.setFieldList(Arrays.asList("dbName", "tableName", "owner", "ownerType"));
-    GetTablesRequest req = new GetTablesRequest(dbName);
-    req.setCatName(catName);
-    req.setCapabilities(version);
-    req.setTblNames(null);
-    if(tablePattern == null){
+    if (tablePattern == null) {
       tablePattern = ".*";
     }
-    req.setTablesPattern(tablePattern);
-    if (processorCapabilities != null)
-      req.setProcessorCapabilities(new ArrayList<String>(Arrays.asList(processorCapabilities)));
-    if (processorIdentifier != null)
-      req.setProcessorIdentifier(processorIdentifier);
-    req.setProjectionSpec(projectionsSpec);
-    List<Table> tableObjects = client.get_table_objects_by_name_req(req).getTables();
-    tableObjects = deepCopyTables(FilterUtils.filterTablesIfEnabled(isClientFilterEnabled, filterHook, tableObjects));
-    for (Table tbl : tableObjects) {
-      tables.add(tbl.getTableName());
-    }
-    return tables;
+    String filter = hive_metastoreConstants.HIVE_FILTER_FIELD_TABLE_NAME + " like \"" + tablePattern + "\"";
+    List<String> tables = listTableNamesByFilter(catName, dbName, filter, (short) -1);
+    return FilterUtils.filterTableNamesIfEnabled(isClientFilterEnabled, filterHook, catName, dbName, tables);
   }
 
   @Override

standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java

@@ -1315,4 +1315,31 @@ public static String getHttpPath(String httpPath) {
     }
     return httpPath;
   }
+
+  public static String convertSqlPatternToRegExp(String likePattern) {
+    if (likePattern == null) {
+      return null;
+    }
+    StringBuilder sb = new StringBuilder();
+    for (int i = 0; i < likePattern.length(); i++) {
+      // Make a special case for "\\_" and "\\%"
+      char n = likePattern.charAt(i);
+      if (n == '\\'
+          && i + 1 < likePattern.length()
+          && (likePattern.charAt(i + 1) == '_' || likePattern.charAt(i + 1) == '%')) {
+        sb.append(likePattern.charAt(i + 1));
+        i++;
+        continue;
+      }
+
+      if (n == '_') {
+        sb.append(".");
+      } else if (n == '%') {
+        sb.append(".*");
+      } else {
+        sb.append(n);
+      }
+    }
+    return sb.toString();
+  }
 }

standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.1.0.derby.sql

@@ -269,7 +269,7 @@ CREATE UNIQUE INDEX "APP"."ROLEENTITYINDEX" ON "APP"."ROLES" ("ROLE_NAME");
 
 CREATE INDEX "APP"."TABLEPRIVILEGEINDEX" ON "APP"."TBL_PRIVS" ("AUTHORIZER", "TBL_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "TBL_PRIV", "GRANTOR", "GRANTOR_TYPE");
 
-CREATE UNIQUE INDEX "APP"."UNIQUETABLE" ON "APP"."TBLS" ("TBL_NAME", "DB_ID");
+CREATE UNIQUE INDEX "APP"."UNIQUETABLE" ON "APP"."TBLS" ("DB_ID", "TBL_NAME");
 
 CREATE UNIQUE INDEX "APP"."UNIQUE_DATABASE" ON "APP"."DBS" ("NAME", "CTLG_NAME");
 

standalone-metastore/metastore-server/src/main/sql/derby/upgrade-4.0.0-to-4.1.0.derby.sql

@@ -12,5 +12,9 @@ ALTER TABLE "APP"."PART_COL_STATS" DROP COLUMN "DB_NAME";
 ALTER TABLE "APP"."PART_COL_STATS" DROP COLUMN "TABLE_NAME";
 ALTER TABLE "APP"."PART_COL_STATS" DROP COLUMN "PARTITION_NAME";
 
+-- HIVE-28292
+DROP INDEX "APP"."UNIQUETABLE";
+CREATE UNIQUE INDEX "APP"."UNIQUETABLE" ON "APP"."TBLS" ("DB_ID", "TBL_NAME");
+
 -- This needs to be the last thing done.  Insert any changes above this line.
 UPDATE "APP".VERSION SET SCHEMA_VERSION='4.1.0', VERSION_COMMENT='Hive release version 4.1.0' where VER_ID=1;

standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.1.0.mssql.sql

@@ -818,9 +818,7 @@ ALTER TABLE TBLS ADD CONSTRAINT TBLS_FK1 FOREIGN KEY (DB_ID) REFERENCES DBS (DB_
 
 CREATE INDEX TBLS_N50 ON TBLS (SD_ID);
 
-CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (TBL_NAME,DB_ID);
-
-CREATE INDEX TBLS_N49 ON TBLS (DB_ID);
+CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (DB_ID,TBL_NAME);
 
 
 -- Constraints for table SDS for class(es) [org.apache.hadoop.hive.metastore.model.MStorageDescriptor]

standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-4.0.0-to-4.1.0.mssql.sql

@@ -11,6 +11,11 @@ CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS (PART_ID,COLUMN_NAME);
 DROP INDEX PART_COL_STATS_N49 on PART_COL_STATS;
 ALTER TABLE PART_COL_STATS DROP COLUMN CAT_NAME, DB_NAME, TABLE_NAME, PARTITION_NAME;
 
+-- HIVE-28292
+DROP INDEX UNIQUETABLE ON TBLS;
+CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (DB_ID,TBL_NAME);
+DROP INDEX TBLS_N49 ON TBLS;
+
 -- These lines need to be last.  Insert any changes above.
 UPDATE VERSION SET SCHEMA_VERSION='4.1.0', VERSION_COMMENT='Hive release version 4.1.0' where VER_ID=1;
 SELECT 'Finished upgrading MetaStore schema from 4.0.0 to 4.1.0' AS MESSAGE;

standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.1.0.mysql.sql

@@ -597,9 +597,8 @@ CREATE TABLE IF NOT EXISTS `TBLS` (
   `IS_REWRITE_ENABLED` bit(1) NOT NULL DEFAULT 0,
   `WRITE_ID` bigint(20) DEFAULT 0,
   PRIMARY KEY (`TBL_ID`),
-  UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`),
+  UNIQUE KEY `UNIQUETABLE` (`DB_ID`,`TBL_NAME`),
   KEY `TBLS_N50` (`SD_ID`),
-  KEY `TBLS_N49` (`DB_ID`),
   CONSTRAINT `TBLS_FK1` FOREIGN KEY (`SD_ID`) REFERENCES `SDS` (`SD_ID`),
   CONSTRAINT `TBLS_FK2` FOREIGN KEY (`DB_ID`) REFERENCES `DBS` (`DB_ID`)
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;

standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-4.0.0-to-4.1.0.mysql.sql

@@ -9,6 +9,11 @@ DROP INDEX PCS_STATS_IDX ON PART_COL_STATS;
 CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS (PART_ID, COLUMN_NAME) USING BTREE;
 ALTER TABLE PART_COL_STATS DROP COLUMN CAT_NAME, DROP COLUMN DB_NAME, DROP COLUMN TABLE_NAME, DROP COLUMN PARTITION_NAME;
 
+-- HIVE-28292
+DROP INDEX UNIQUETABLE ON TBLS;
+CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (DB_ID, TBL_NAME);
+DROP INDEX TBLS_N49 ON TBLS;
+
 -- These lines need to be last.  Insert any changes above.
 UPDATE VERSION SET SCHEMA_VERSION='4.1.0', VERSION_COMMENT='Hive release version 4.1.0' where VER_ID=1;
 SELECT 'Finished upgrading MetaStore schema from 4.0.0 to 4.1.0' AS MESSAGE;

standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.1.0.oracle.sql

@@ -852,9 +852,7 @@ ALTER TABLE TBLS ADD CONSTRAINT TBLS_FK2 FOREIGN KEY (DB_ID) REFERENCES DBS (DB_
 
 ALTER TABLE TBLS ADD CONSTRAINT TBLS_FK1 FOREIGN KEY (SD_ID) REFERENCES SDS (SD_ID) INITIALLY DEFERRED ;
 
-CREATE INDEX TBLS_N49 ON TBLS (DB_ID);
-
-CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (TBL_NAME,DB_ID);
+CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (DB_ID,TBL_NAME);
 
 CREATE INDEX TBLS_N50 ON TBLS (SD_ID);
 

standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-4.0.0-to-4.1.0.oracle.sql

@@ -11,6 +11,11 @@ CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS (PART_ID, COLUMN_NAME);
 DROP INDEX PART_COL_STATS_N49;
 ALTER TABLE PART_COL_STATS DROP (CAT_NAME, DB_NAME, TABLE_NAME, PARTITION_NAME);
 
+-- HIVE-28292
+DROP INDEX UNIQUETABLE;
+CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (DB_ID, TBL_NAME);
+DROP INDEX TBLS_N49;
+
 -- These lines need to be last.  Insert any changes above.
 UPDATE VERSION SET SCHEMA_VERSION='4.1.0', VERSION_COMMENT='Hive release version 4.1.0' where VER_ID=1;
 SELECT 'Finished upgrading MetaStore schema from 4.0.0 to 4.1.0' AS Status from dual;

standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.1.0.postgres.sql

@@ -1028,7 +1028,7 @@ ALTER TABLE ONLY "PARTITIONS"
 --
 
 ALTER TABLE ONLY "TBLS"
-    ADD CONSTRAINT "UNIQUETABLE" UNIQUE ("TBL_NAME", "DB_ID");
+    ADD CONSTRAINT "UNIQUETABLE" UNIQUE ("DB_ID", "TBL_NAME");
 
 
 --
@@ -1245,13 +1245,6 @@ CREATE INDEX "TABLEPRIVILEGEINDEX" ON "TBL_PRIVS" USING btree ("AUTHORIZER", "TB
 CREATE INDEX "TABLE_PARAMS_N49" ON "TABLE_PARAMS" USING btree ("TBL_ID");
 
 
---
--- Name: TBLS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace:
---
-
-CREATE INDEX "TBLS_N49" ON "TBLS" USING btree ("DB_ID");
-
-
 --
 -- Name: TBLS_N50; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace:
 --

standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-4.0.0-to-4.1.0.postgres.sql

@@ -11,6 +11,11 @@ CREATE INDEX "PCS_STATS_IDX" ON "PART_COL_STATS" USING btree ("PART_ID","COLUMN_
 DROP INDEX "PART_COL_STATS_N49";
 ALTER TABLE "PART_COL_STATS" DROP COLUMN "CAT_NAME", DROP COLUMN "DB_NAME", DROP COLUMN "TABLE_NAME", DROP COLUMN "PARTITION_NAME";
 
+-- HIVE-28292
+ALTER TABLE ONLY "TBLS" DROP CONSTRAINT "UNIQUETABLE";
+ALTER TABLE ONLY "TBLS" ADD CONSTRAINT "UNIQUETABLE" UNIQUE ("DB_ID", "TBL_NAME");
+DROP INDEX "TBLS_N49";
+
 -- These lines need to be last. Insert any changes above.
 UPDATE "VERSION" SET "SCHEMA_VERSION"='4.1.0', "VERSION_COMMENT"='Hive release version 4.1.0' where "VER_ID"=1;
 SELECT 'Finished upgrading MetaStore schema from 4.0.0 to 4.1.0';