Implement create catalog.database syntax

Author: zhangbutao

parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g

@@ -1165,7 +1165,7 @@ createDatabaseStatement
 @after { popMsg(state); }
     : KW_CREATE (KW_DATABASE|KW_SCHEMA)
         ifNotExists?
-        name=identifier
+        name=databaseName
         databaseComment?
         dbLocation?
         dbManagedLocation?
@@ -1174,7 +1174,7 @@ createDatabaseStatement
 
     | KW_CREATE KW_REMOTE (KW_DATABASE|KW_SCHEMA)
         ifNotExists?
-        name=identifier
+        name=databaseName
         databaseComment?
         dbConnectorName
         (KW_WITH KW_DBPROPERTIES dbprops=dbProperties)?

ql/src/java/org/apache/hadoop/hive/ql/ddl/database/create/CreateDatabaseAnalyzer.java

@@ -20,10 +20,12 @@
 
 import java.util.Map;
 
+import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.hive.metastore.api.DataConnector;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.DatabaseType;
 import org.apache.hadoop.hive.metastore.api.PrincipalType;
+import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.QueryState;
 import org.apache.hadoop.hive.ql.exec.TaskFactory;
 import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType;
@@ -47,7 +49,12 @@ public CreateDatabaseAnalyzer(QueryState queryState) throws SemanticException {
 
   @Override
   public void analyzeInternal(ASTNode root) throws SemanticException {
-    String databaseName = unescapeIdentifier(root.getChild(0).getText());
+    Pair<String, String> catDbNamePair = getCatDbNamePair((ASTNode) root.getChild(0));
+    String catalogName = catDbNamePair.getLeft();
+    if (catalogName != null && getCatalog(catalogName) == null) {
+      throw new SemanticException(ErrorMsg.CATALOG_NOT_EXISTS, catalogName);
+    }
+    String databaseName = catDbNamePair.getRight();
 
     boolean ifNotExists = false;
     String comment = null;
@@ -92,14 +99,15 @@ public void analyzeInternal(ASTNode root) throws SemanticException {
       }
     }
 
-    if (ifNotExists && getDatabase(databaseName, false) != null) {
+    if (ifNotExists && getDatabase(catalogName, databaseName, false) != null) {
       return;
     }
 
     CreateDatabaseDesc desc = null;
     Database database = new Database(databaseName, comment, locationUri, props);
+    database.setCatalogName(catalogName);
     if (type.equalsIgnoreCase(DatabaseType.NATIVE.name())) {
-      desc = new CreateDatabaseDesc(databaseName, comment, locationUri, managedLocationUri, ifNotExists, props);
+      desc = new CreateDatabaseDesc(catalogName, databaseName, comment, locationUri, managedLocationUri, ifNotExists, props);
       database.setType(DatabaseType.NATIVE);
       // database = new Database(databaseName, comment, locationUri, props);
       if (managedLocationUri != null) {
@@ -109,7 +117,7 @@ public void analyzeInternal(ASTNode root) throws SemanticException {
       String remoteDbName = databaseName;
       if (props != null && props.get("connector.remoteDbName") != null) // TODO finalize the property name
         remoteDbName = props.get("connector.remoteDbName");
-      desc = new CreateDatabaseDesc(databaseName, comment, locationUri, null, ifNotExists, props, type,
+      desc = new CreateDatabaseDesc(catalogName, databaseName, comment, locationUri, null, ifNotExists, props, type,
           connectorName, remoteDbName);
       database.setConnector_name(connectorName);
       database.setType(DatabaseType.REMOTE);

ql/src/java/org/apache/hadoop/hive/ql/ddl/database/create/CreateDatabaseDesc.java

@@ -35,6 +35,7 @@
 public class CreateDatabaseDesc implements DDLDesc, Serializable {
   private static final long serialVersionUID = 1L;
 
+  private final String catalogName;
   private final String databaseName;
   private final String comment;
   private final String locationUri;
@@ -45,13 +46,14 @@ public class CreateDatabaseDesc implements DDLDesc, Serializable {
   private final String remoteDbName;
   private final Map<String, String> dbProperties;
 
-  public CreateDatabaseDesc(String databaseName, String comment, String locationUri, String managedLocationUri,
+  public CreateDatabaseDesc(String catalogName, String databaseName, String comment, String locationUri, String managedLocationUri,
       boolean ifNotExists, Map<String, String> dbProperties) {
-    this(databaseName, comment, locationUri, managedLocationUri, ifNotExists, dbProperties, "NATIVE", null, null);
+    this(catalogName, databaseName, comment, locationUri, managedLocationUri, ifNotExists, dbProperties, "NATIVE", null, null);
   }
 
-  public CreateDatabaseDesc(String databaseName, String comment, String locationUri, String managedLocationUri,
+  public CreateDatabaseDesc(String catalogName, String databaseName, String comment, String locationUri, String managedLocationUri,
       boolean ifNotExists, Map<String, String> dbProperties, String dbtype, String connectorName, String remoteDbName) {
+    this.catalogName = catalogName;
     this.databaseName = databaseName;
     this.comment = comment;
     if (dbtype != null && dbtype.equalsIgnoreCase("REMOTE")) {
@@ -80,6 +82,11 @@ public Map<String, String> getDatabaseProperties() {
     return dbProperties;
   }
 
+  @Explain(displayName="catalogName", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
+  public String getCatalogName() {
+    return catalogName;
+  }
+
   @Explain(displayName="name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
   public String getName() {
     return databaseName;

ql/src/java/org/apache/hadoop/hive/ql/ddl/database/create/CreateDatabaseOperation.java

@@ -47,6 +47,7 @@ public CreateDatabaseOperation(DDLOperationContext context, CreateDatabaseDesc d
   public int execute() throws HiveException {
     Database database = new Database(desc.getName(), desc.getComment(), desc.getLocationUri(),
         desc.getDatabaseProperties());
+    database.setCatalogName(desc.getCatalogName());
     database.setOwnerName(SessionState.getUserFromAuthenticator());
     database.setOwnerType(PrincipalType.USER);
     database.setType(desc.getDatabaseType());
@@ -55,12 +56,12 @@ public int execute() throws HiveException {
         if (desc.getManagedLocationUri() != null) {
           database.setManagedLocationUri(desc.getManagedLocationUri());
         }
-        makeLocationQualified(database); // TODO add catalog prefix for db location
+        makeLocationQualified(database); // TODO catalog. Add catalog prefix for location
         if (database.getLocationUri().equalsIgnoreCase(database.getManagedLocationUri())) {
           throw new HiveException("Managed and external locations for database cannot be the same");
         }
       } else if (desc.getDatabaseType() == DatabaseType.REMOTE) {
-        makeLocationQualified(database); // TODO add catalog prefix for db location
+        makeLocationQualified(database); // TODO catalog. Add catalog prefix for location
         database.setConnector_name(desc.getConnectorName());
         database.setRemote_dbname(desc.getRemoteDbName());
       } else { // should never be here

ql/src/java/org/apache/hadoop/hive/ql/ddl/database/drop/DropDatabaseAnalyzer.java

@@ -59,7 +59,7 @@ public void analyzeInternal(ASTNode root) throws SemanticException {
     boolean isSoftDelete = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED);
 
     String catalogName = catDbNamePair.getLeft();
-    if (getCatalog(catalogName) == null) {
+    if (catalogName != null && getCatalog(catalogName) == null) {
       throw new SemanticException(ErrorMsg.CATALOG_NOT_EXISTS, catalogName);
     }
     String databaseName = catDbNamePair.getRight();

ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java

@@ -150,7 +150,7 @@ private boolean isDbEmpty(String dbName) throws HiveException {
 
   private Task<?> createDbTask(Database dbObj) throws MetaException {
     // note that we do not set location - for repl load, we want that auto-created.
-    CreateDatabaseDesc createDbDesc = new CreateDatabaseDesc(dbObj.getName(), dbObj.getDescription(),
+    CreateDatabaseDesc createDbDesc = new CreateDatabaseDesc(dbObj.getCatalogName(), dbObj.getName(), dbObj.getDescription(),
             getDbLocation(dbObj), getDbManagedLocation(dbObj), false, updateDbProps(dbObj, context.dumpDirectory));
     // If it exists, we want this to be an error condition. Repl Load is not intended to replace a
     // db.

ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java

@@ -668,7 +668,7 @@ public void dropCatalog(String catName, boolean ignoreUnknownCat)
   public void createDatabase(Database db, boolean ifNotExist)
       throws AlreadyExistsException, HiveException {
     try {
-      db.setCatalogName(SessionState.get().getCurrentCatalog());
+      db.setCatalogName(Objects.requireNonNullElse(db.getCatalogName(), SessionState.get().getCurrentCatalog()));
       getMSC().createDatabase(db);
     } catch (AlreadyExistsException e) {
       if (!ifNotExist) {
@@ -725,7 +725,7 @@ public void dropDatabase(String name, boolean deleteData, boolean ignoreUnknownD
    */
   public void dropDatabase(String name, boolean deleteData, boolean ignoreUnknownDb, boolean cascade)
       throws HiveException, NoSuchObjectException {
-    dropDatabase(new DropDatabaseDesc(getDefaultCatalog(conf) ,name, ignoreUnknownDb, cascade, deleteData)); //TODO check the actual catalog
+    dropDatabase(new DropDatabaseDesc(getDefaultCatalog(conf) ,name, ignoreUnknownDb, cascade, deleteData)); // TODO catalog. check the actual catalog
   }
 
   public void dropDatabase(DropDatabaseDesc desc) 

ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateDatabaseHandler.java

@@ -56,9 +56,10 @@ public List<Task<?>> handle(Context context)
     Database db = metaData.getDatabase();
     String destinationDBName =
         context.dbName == null ? db.getName() : context.dbName;
+    String destinationCatalogName = db.getCatalogName(); // TODO catalog. Need to double check the catalog here.
 
     CreateDatabaseDesc createDatabaseDesc =
-        new CreateDatabaseDesc(destinationDBName, db.getDescription(), null, null, true, db.getParameters());
+        new CreateDatabaseDesc(destinationCatalogName, destinationDBName, db.getDescription(), null, null, true, db.getParameters());
     Task<DDLWork> createDBTask = TaskFactory.get(
         new DDLWork(new HashSet<>(), new HashSet<>(), createDatabaseDesc, true,
                 context.getDumpDirectory(), context.getMetricCollector()), context.hiveConf);

ql/src/test/queries/clientpositive/catalog_database.q

@@ -11,22 +11,25 @@ CREATE CATALOG testcat LOCATION '/tmp/testcat' COMMENT 'Hive test catalog';
 -- Check catalogs list
 SHOW CATALOGS;
 
+-- CREATE DATABASE in new catalog testcat by catalog.db pattern
+CREATE DATABASE testcat.testdb_1;
+
 -- Switch the catalog from hive to 'testcat'
 SET CATALOG testcat;
 
--- CREATE DATABASE in default catalog 'hive'
-CREATE DATABASE testdb_new;
+-- CREATE DATABASE in new catalog testcat
+CREATE DATABASE testdb_2;
 
 -- Check databases in catalog 'testcat',
--- The list of databases in the catalog 'hive' should only contain the default and the testdb_new.
+-- The list of databases in the catalog 'hive' should contain default and testdb_1 and testdb_2.
 SHOW DATABASES;
 
 -- Switch database by catalog.db pattern
-USE testcat.testdb_new;
+USE testcat.testdb_1;
 
 -- Drop database by catalog.db pattern
-DROP DATABASE testcat.testdb_new;
+DROP DATABASE testcat.testdb_1;
 
 -- Check databases in catalog 'testcat',
--- The list of databases in the catalog 'hive' should only contain the default.
+-- The list of databases in the catalog 'hive' should contain default and testdb_2.
 SHOW DATABASES;

ql/src/test/results/clientpositive/llap/catalog_database.q.out

@@ -24,40 +24,48 @@ POSTHOOK: query: SHOW CATALOGS
 POSTHOOK: type: SHOWCATALOGS
 hive
 testcat
+PREHOOK: query: CREATE DATABASE testcat.testdb_1
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:testdb_1
+POSTHOOK: query: CREATE DATABASE testcat.testdb_1
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:testdb_1
 PREHOOK: query: SET CATALOG testcat
 PREHOOK: type: SWITCHCATALOG
 PREHOOK: Input: catalog:testcat
 POSTHOOK: query: SET CATALOG testcat
 POSTHOOK: type: SWITCHCATALOG
 POSTHOOK: Input: catalog:testcat
-PREHOOK: query: CREATE DATABASE testdb_new
+PREHOOK: query: CREATE DATABASE testdb_2
 PREHOOK: type: CREATEDATABASE
-PREHOOK: Output: database:testdb_new
-POSTHOOK: query: CREATE DATABASE testdb_new
+PREHOOK: Output: database:testdb_2
+POSTHOOK: query: CREATE DATABASE testdb_2
 POSTHOOK: type: CREATEDATABASE
-POSTHOOK: Output: database:testdb_new
+POSTHOOK: Output: database:testdb_2
 PREHOOK: query: SHOW DATABASES
 PREHOOK: type: SHOWDATABASES
 POSTHOOK: query: SHOW DATABASES
 POSTHOOK: type: SHOWDATABASES
 default
-testdb_new
-PREHOOK: query: USE testcat.testdb_new
+testdb_1
+testdb_2
+PREHOOK: query: USE testcat.testdb_1
 PREHOOK: type: SWITCHDATABASE
-PREHOOK: Input: database:testdb_new
-POSTHOOK: query: USE testcat.testdb_new
+PREHOOK: Input: database:testdb_1
+POSTHOOK: query: USE testcat.testdb_1
 POSTHOOK: type: SWITCHDATABASE
-POSTHOOK: Input: database:testdb_new
-PREHOOK: query: DROP DATABASE testcat.testdb_new
+POSTHOOK: Input: database:testdb_1
+PREHOOK: query: DROP DATABASE testcat.testdb_1
 PREHOOK: type: DROPDATABASE
-PREHOOK: Input: database:testdb_new
-PREHOOK: Output: database:testdb_new
-POSTHOOK: query: DROP DATABASE testcat.testdb_new
+PREHOOK: Input: database:testdb_1
+PREHOOK: Output: database:testdb_1
+POSTHOOK: query: DROP DATABASE testcat.testdb_1
 POSTHOOK: type: DROPDATABASE
-POSTHOOK: Input: database:testdb_new
-POSTHOOK: Output: database:testdb_new
+POSTHOOK: Input: database:testdb_1
+POSTHOOK: Output: database:testdb_1
 PREHOOK: query: SHOW DATABASES
 PREHOOK: type: SHOWDATABASES
 POSTHOOK: query: SHOW DATABASES
 POSTHOOK: type: SHOWDATABASES
 default
+testdb_2