From 618b0e5b695e6d259e95bc82e4278022fa103035 Mon Sep 17 00:00:00 2001 From: srielau Date: Wed, 12 Aug 2026 12:30:22 +0000 Subject: [PATCH 1/9] [SPARK-58738][SQL] Add parse_command function for SQL statement JSON analysis --- .../catalyst/analysis/FunctionRegistry.scala | 1 + .../catalyst/expressions/ParseCommand.scala | 77 ++++++ .../catalyst/parser/ParseCommandResult.scala | 222 ++++++++++++++++++ .../catalyst/parser/SqlStatementCodes.scala | 173 ++++++++++++++ .../expressions/ParseCommandSuite.scala | 59 +++++ .../parser/ParseCommandResultSuite.scala | 139 +++++++++++ 6 files changed, 671 insertions(+) create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala create mode 100644 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseCommandSuite.scala create mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index 5eef532ab20b..b6e30985ef0c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -846,6 +846,7 @@ object FunctionRegistry { // misc functions expression[AssertTrue]("assert_true"), expressionBuilder("raise_error", RaiseErrorExpressionBuilder), + expression[ParseCommand]("parse_command"), expression[Crc32]("crc32"), expression[Md5]("md5"), expression[Uuid]("uuid"), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala new file mode 100644 index 000000000000..50607df7f054 --- /dev/null +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.expressions + +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.parser.ParseCommandResult +import org.apache.spark.sql.internal.types.StringTypeWithCollation +import org.apache.spark.sql.types.{AbstractDataType, DataType, StringType} +import org.apache.spark.unsafe.types.UTF8String + +/** + * Parses a SQL statement string and returns a compact JSON description of the + * unresolved statement (classification, references, select list, parameters), + * or a STANDARD-format error object when the statement does not parse. + * + * Designed for batch evaluation over DataFrames of SQL text; never throws on + * syntax errors so a single bad row does not fail the query. + */ +// scalastyle:off line.size.limit +@ExpressionDescription( + usage = """_FUNC_(sqlStmt) - Parses `sqlStmt` and returns a JSON string describing the + statement (parse success, Table 39 statement classification, table and function + references, select-list columns, and parameter markers). On syntax error returns + JSON with `parse_success` false and a nested STANDARD error object instead of + throwing.""", + arguments = """ + Arguments: + * sqlStmt - A SQL statement string to parse. + An expression that evaluates to a string. + """, + examples = """ + Examples: + > SELECT _FUNC_('SELECT a, b FROM t'); + {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,...} + > SELECT _FUNC_('SELEC'); + {"parse_success":false,"error":{"errorClass":"PARSE_SYNTAX_ERROR",...}} + """, + group = "misc_funcs", + since = "4.3.0") +// scalastyle:on line.size.limit +case class ParseCommand(child: Expression) + extends UnaryExpression + with ImplicitCastInputTypes + with CodegenFallback { + + override def prettyName: String = "parse_command" + + override def nullable: Boolean = true + + override def dataType: DataType = StringType + + override def inputTypes: Seq[AbstractDataType] = + Seq(StringTypeWithCollation(supportsTrimCollation = true)) + + override def nullSafeEval(input: Any): Any = { + val sql = input.asInstanceOf[UTF8String].toString + UTF8String.fromString(ParseCommandResult.fromSql(sql)) + } + + override protected def withNewChildInternal(newChild: Expression): ParseCommand = + copy(child = newChild) +} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala new file mode 100644 index 000000000000..3f545708fe4e --- /dev/null +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.parser + +import scala.collection.mutable +import scala.util.control.NonFatal + +import org.json4s._ +import org.json4s.jackson.JsonMethods.{compact, parse => parseJson, render} + +import org.apache.spark.{ErrorMessageFormat, SparkThrowable, SparkThrowableHelper} +import org.apache.spark.sql.catalyst.analysis._ +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.plans.logical._ + +/** + * Parses a SQL statement string and returns a compact JSON description of the + * unresolved plan (parse-only; no catalog resolution). + * + * On success the JSON includes statement classification (ISO/IEC 9075-2:2023 + * Table 39), table/function references, select-list items, and parameter + * markers. On parse failure it returns `parse_success: false` with a nested + * STANDARD-format error object and does not throw. + */ +object ParseCommandResult { + + private val parser: ThreadLocal[CatalystSqlParser] = + ThreadLocal.withInitial(() => new CatalystSqlParser()) + + /** Parse `sql` and render the JSON result string. Never throws for bad SQL. */ + def fromSql(sql: String): String = { + try { + val plan = parser.get().parsePlan(sql) + fromPlan(plan) + } catch { + case e: ParseException => + errorJson(e) + case e: SparkThrowable with Throwable => + errorJson(e) + case NonFatal(e) => + // Unexpected failures still must not fail a batch row. + compact(render(JObject( + "parse_success" -> JBool(false), + "error" -> JObject( + "errorClass" -> JString("LEGACY"), + "messageParameters" -> JObject( + "message" -> JString(Option(e.getMessage).getOrElse(e.toString)) + ) + ) + ))) + } + } + + /** Build success JSON from an already-parsed unresolved plan. */ + def fromPlan(plan: LogicalPlan): String = { + val classification = SqlStatementCodes.classify(plan) + val fields = mutable.ListBuffer.empty[JField] + fields += "parse_success" -> JBool(true) + fields += "statement_identifier" -> JString(classification.statementIdentifier) + fields += "statement_code" -> JInt(classification.statementCode) + fields += "statement_type" -> JString(classification.statementType) + fields += "statement_class" -> JString(classification.statementClass) + if (classification.asSubquery) { + fields += "as_subquery" -> JBool(true) + } + fields += "table_references" -> JArray( + collectTableReferences(plan).map(partsToJArray).toList) + fields += "function_references" -> JArray( + collectFunctionReferences(plan).map(partsToJArray).toList) + fields += "select_list" -> JArray(collectSelectList(plan).toList) + fields += "parameter_markers" -> parameterMarkersJson(plan) + compact(render(JObject(fields.toList))) + } + + private def errorJson(e: SparkThrowable with Throwable): String = { + val errorObj = parseJson( + SparkThrowableHelper.getMessage(e, ErrorMessageFormat.STANDARD)) + compact(render(JObject( + "parse_success" -> JBool(false), + "error" -> errorObj + ))) + } + + private def partsToJArray(parts: Seq[String]): JArray = + JArray(parts.map(JString).toList) + + /** + * Collect multipart table/view identifiers as written in the SQL. + * Deduplicates while preserving first-seen order. + */ + def collectTableReferences(plan: LogicalPlan): Seq[Seq[String]] = { + val seen = mutable.LinkedHashSet.empty[Seq[String]] + def add(parts: Seq[String]): Unit = { + if (parts.nonEmpty) seen += parts + } + def collectFrom(p: LogicalPlan): Unit = { + p.collectWithSubqueries { + case u: UnresolvedRelation => add(u.multipartIdentifier) + case u: UnresolvedTable => add(u.multipartIdentifier) + case u: UnresolvedView => add(u.multipartIdentifier) + case u: UnresolvedTableOrView => add(u.multipartIdentifier) + case u: UnresolvedIdentifier => add(u.nameParts) + } + } + collectFrom(plan) + // InsertIntoStatement.table is not a child of the plan tree. + plan match { + case InsertIntoStatement(table, _, _, _, _, _, _, _, _) => + collectFrom(table) + case _ => + } + seen.toSeq + } + + /** Collect multipart function names, including table-valued functions. */ + def collectFunctionReferences(plan: LogicalPlan): Seq[Seq[String]] = { + val seen = mutable.LinkedHashSet.empty[Seq[String]] + def add(parts: Seq[String]): Unit = { + if (parts.nonEmpty) seen += parts + } + def collectInExpression(e: Expression): Unit = e.foreach { + case f: UnresolvedFunction => add(f.nameParts) + case _ => + } + plan.collectWithSubqueries { + case p: LogicalPlan => + p.expressions.foreach(collectInExpression) + p match { + case u: UnresolvedTableValuedFunction => add(u.name) + case _ => + } + } + seen.toSeq + } + + /** + * Collect the primary select list as `{name, expression}` objects. + * Empty for non-query statements without a projected query body. + */ + def collectSelectList(plan: LogicalPlan): Seq[JObject] = { + val query = primaryQueryPlan(plan) + val named: Seq[NamedExpression] = query match { + case p: Project => p.projectList + case a: Aggregate => a.aggregateExpressions + case _ => Nil + } + named.map(selectListItem) + } + + private def primaryQueryPlan(plan: LogicalPlan): LogicalPlan = plan match { + case UnresolvedWith(child, _, _) => primaryQueryPlan(child) + case InsertIntoStatement(_, _, _, query, _, _, _, _, _) => + primaryQueryPlan(query) + case c: CreateTableAsSelect => primaryQueryPlan(c.query) + case r: ReplaceTableAsSelect => primaryQueryPlan(r.query) + case SubqueryAlias(_, child) => primaryQueryPlan(child) + case other => other + } + + private def selectListItem(ne: NamedExpression): JObject = ne match { + case Alias(child, name) => + JObject( + "name" -> partsToJArray(Seq(name)), + "expression" -> JString(child.sql)) + case u: UnresolvedAlias => + JObject( + "name" -> partsToJArray(Nil), + "expression" -> JString(u.child.sql)) + case s: UnresolvedStar => + val name = s.target.map(_ :+ "*").getOrElse(Seq("*")) + JObject( + "name" -> partsToJArray(name), + "expression" -> JString(s.sql)) + case a: UnresolvedAttribute => + JObject( + "name" -> partsToJArray(a.nameParts), + "expression" -> JString(a.sql)) + case other => + JObject( + "name" -> partsToJArray(Seq(other.name)), + "expression" -> JString(other.sql)) + } + + private def parameterMarkersJson(plan: LogicalPlan): JObject = { + val named = mutable.LinkedHashSet.empty[String] + var unnamedCount = 0 + def visitExpr(e: Expression): Unit = e.foreach { + case n: NamedParameter => named += n.name + case _: PosParameter => unnamedCount += 1 + case _ => + } + plan.collectWithSubqueries { + case p: LogicalPlan => + p.expressions.foreach(visitExpr) + } + // InsertIntoStatement.table is outside the child tree. + plan match { + case InsertIntoStatement(table, _, _, _, _, _, _, _, _) => + table.expressions.foreach(visitExpr) + case _ => + } + JObject( + "named" -> JArray(named.toList.map(JString)), + "unnamed_count" -> JInt(unnamedCount) + ) + } +} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala new file mode 100644 index 000000000000..1ba005cb4754 --- /dev/null +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.parser + +import org.apache.spark.sql.catalyst.analysis.UnresolvedExecuteImmediate +import org.apache.spark.sql.catalyst.plans.logical._ + +/** + * Classification of a parsed SQL statement using ISO/IEC 9075-2:2023 Table 39, + * "SQL-statement codes" (clause 23.1 <get diagnostics statement>). + * + * @param statementType BNF production name without angle brackets + * @param statementIdentifier Table 39 Identifier column (or Spark product name) + * @param statementCode Table 39 Code column; Spark-only statements use negative + * implementation-defined codes (Table 39 IE005 / IV190) + * @param statementClass Clause 4.41.2 function class, or + * "implementation-defined statement" for Spark extensions + * @param asSubquery True when <table definition> contains an <as subquery clause> + */ +case class SqlStatementClassification( + statementType: String, + statementIdentifier: String, + statementCode: Int, + statementClass: String, + asSubquery: Boolean = false) + +/** + * Maps unresolved [[LogicalPlan]]s to Table 39 statement codes. + * + * Spark-only statements use the standard's implementation-defined escape hatch: + * a product-specific identifier and a distinct negative code. Codes are + * append-only and must never be renumbered. + */ +object SqlStatementCodes { + + // Standard Table 39 entries used by Spark SQL (ISO/IEC 9075-2:2023). + val Select: SqlStatementClassification = SqlStatementClassification( + "direct select statement: multiple rows", "SELECT", 21, "SQL-data statement") + val Insert: SqlStatementClassification = SqlStatementClassification( + "insert statement", "INSERT", 50, "SQL-data change statement") + val DeleteWhere: SqlStatementClassification = SqlStatementClassification( + "delete statement: searched", "DELETE WHERE", 19, "SQL-data change statement") + val UpdateWhere: SqlStatementClassification = SqlStatementClassification( + "update statement: searched", "UPDATE WHERE", 82, "SQL-data change statement") + val Merge: SqlStatementClassification = SqlStatementClassification( + "merge statement", "MERGE", 128, "SQL-data change statement") + val CreateTable: SqlStatementClassification = SqlStatementClassification( + "table definition", "CREATE TABLE", 77, "SQL-schema statement") + val CreateView: SqlStatementClassification = SqlStatementClassification( + "view definition", "CREATE VIEW", 84, "SQL-schema statement") + val DropTable: SqlStatementClassification = SqlStatementClassification( + "drop table statement", "DROP TABLE", 32, "SQL-schema statement") + val DropView: SqlStatementClassification = SqlStatementClassification( + "drop view statement", "DROP VIEW", 36, "SQL-schema statement") + val AlterTable: SqlStatementClassification = SqlStatementClassification( + "alter table statement", "ALTER TABLE", 4, "SQL-schema statement") + val CreateSchema: SqlStatementClassification = SqlStatementClassification( + "schema definition", "CREATE SCHEMA", 64, "SQL-schema statement") + val DropSchema: SqlStatementClassification = SqlStatementClassification( + "drop schema statement", "DROP SCHEMA", 31, "SQL-schema statement") + val SetSchema: SqlStatementClassification = SqlStatementClassification( + "set schema statement", "SET SCHEMA", 74, "SQL-session statement") + val TruncateTable: SqlStatementClassification = SqlStatementClassification( + "truncate table statement", "TRUNCATE TABLE", 139, "SQL-data change statement") + val CreateRoutine: SqlStatementClassification = SqlStatementClassification( + "schema routine", "CREATE ROUTINE", 14, "SQL-schema statement") + val DropRoutine: SqlStatementClassification = SqlStatementClassification( + "drop routine statement", "DROP ROUTINE", 30, "SQL-schema statement") + val ExecuteImmediate: SqlStatementClassification = SqlStatementClassification( + "execute immediate statement", "EXECUTE IMMEDIATE", 43, "SQL-dynamic statement") + val Call: SqlStatementClassification = SqlStatementClassification( + "call statement", "CALL", 7, "SQL-control statement") + + // Table 39 "Unrecognized statements": empty identifier, code 0. + val Unrecognized: SqlStatementClassification = SqlStatementClassification( + "", "", 0, "implementation-defined statement") + + // Spark product-specific identifiers with append-only negative codes + // (Table 39 implementation-defined / IE005 row: negative Code values). + val CacheTable: SqlStatementClassification = spark("CACHE TABLE", -1) + val CacheTableAsSelect: SqlStatementClassification = spark("CACHE TABLE AS SELECT", -2) + val UncacheTable: SqlStatementClassification = spark("UNCACHE TABLE", -3) + val RefreshTable: SqlStatementClassification = spark("REFRESH TABLE", -4) + val ShowTables: SqlStatementClassification = spark("SHOW TABLES", -5) + val DescribeTable: SqlStatementClassification = spark("DESCRIBE TABLE", -6) + val AnalyzeTable: SqlStatementClassification = spark("ANALYZE TABLE", -7) + val DeclareVariable: SqlStatementClassification = spark("DECLARE VARIABLE", -8) + val SetVariable: SqlStatementClassification = spark("SET VARIABLE", -9) + val DropVariable: SqlStatementClassification = spark("DROP VARIABLE", -10) + val ShowTableProperties: SqlStatementClassification = spark("SHOW TBLPROPERTIES", -11) + val DescribeNamespace: SqlStatementClassification = spark("DESCRIBE NAMESPACE", -12) + val ShowFunctions: SqlStatementClassification = spark("SHOW FUNCTIONS", -13) + val DescribeFunction: SqlStatementClassification = spark("DESCRIBE FUNCTION", -14) + val ShowCreateTable: SqlStatementClassification = spark("SHOW CREATE TABLE", -15) + val ShowColumns: SqlStatementClassification = spark("SHOW COLUMNS", -16) + val ShowPartitions: SqlStatementClassification = spark("SHOW PARTITIONS", -17) + val ShowViews: SqlStatementClassification = spark("SHOW VIEWS", -18) + val RefreshFunction: SqlStatementClassification = spark("REFRESH FUNCTION", -19) + val CommentOnNamespace: SqlStatementClassification = spark("COMMENT ON NAMESPACE", -20) + val CommentOnTable: SqlStatementClassification = spark("COMMENT ON TABLE", -21) + + private def spark(identifier: String, code: Int): SqlStatementClassification = { + assert(code < 0, s"Spark statement codes must be negative, got $code") + SqlStatementClassification( + statementType = identifier.toLowerCase(java.util.Locale.ROOT), + statementIdentifier = identifier, + statementCode = code, + statementClass = "implementation-defined statement") + } + + /** Classify an unresolved logical plan. */ + def classify(plan: LogicalPlan): SqlStatementClassification = plan match { + case UnresolvedWith(child, _, _) => classify(child) + case _: InsertIntoStatement => Insert + case _: DeleteFromTable | _: DeleteFromTableWithFilters => DeleteWhere + case _: UpdateTable => UpdateWhere + case _: MergeIntoTable => Merge + case _: CreateTableAsSelect | _: ReplaceTableAsSelect => + CreateTable.copy(asSubquery = true) + case _: CreateTable | _: CreateTableLike | _: ReplaceTable => CreateTable + case _: CreateView => CreateView + case _: DropTable => DropTable + case _: DropView => DropView + case _: CreateNamespace => CreateSchema + case _: DropNamespace => DropSchema + case _: SetCatalogAndNamespace => SetSchema + case _: TruncateTable => TruncateTable + case _: CreateFunction => CreateRoutine + case _: DropFunction => DropRoutine + case _: UnresolvedExecuteImmediate => ExecuteImmediate + case _: Call => Call + case _: CommentOnTable => CommentOnTable + case _: AlterTableCommand | _: RenameTable => AlterTable + case _: CacheTable => CacheTable + case _: CacheTableAsSelect => CacheTableAsSelect + case _: UncacheTable => UncacheTable + case _: RefreshTable => RefreshTable + case _: ShowTables | _: ShowTablesExtended => ShowTables + case _: DescribeRelation | _: DescribeTablePartition | _: DescribeColumn => + DescribeTable + case _: AnalyzeTable | _: AnalyzeTables | _: AnalyzeColumn => AnalyzeTable + case _: CreateVariable => DeclareVariable + case _: SetVariable => SetVariable + case _: DropVariable => DropVariable + case _: ShowTableProperties => ShowTableProperties + case _: DescribeNamespace => DescribeNamespace + case _: ShowFunctions => ShowFunctions + case _: DescribeFunction => DescribeFunction + case _: ShowCreateTable => ShowCreateTable + case _: ShowColumns => ShowColumns + case _: ShowPartitions | _: ShowTablePartition => ShowPartitions + case _: ShowViews => ShowViews + case _: RefreshFunction => RefreshFunction + case _: CommentOnNamespace => CommentOnNamespace + case _: Command => Unrecognized + case _ => Select + } +} diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseCommandSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseCommandSuite.scala new file mode 100644 index 000000000000..84d00a97f5cc --- /dev/null +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseCommandSuite.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.expressions + +import org.json4s._ +import org.json4s.jackson.JsonMethods.parse + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.types.StringType +import org.apache.spark.unsafe.types.UTF8String + +class ParseCommandSuite extends SparkFunSuite with ExpressionEvalHelper { + + private def evalJson(sql: String): JValue = { + val result = ParseCommand(Literal(sql)).eval().asInstanceOf[UTF8String].toString + parse(result) + } + + test("parse_command returns JSON for a valid SELECT") { + val j = evalJson("SELECT 1 AS a") + assert(j \ "parse_success" === JBool(true)) + assert(j \ "statement_identifier" === JString("SELECT")) + assert(j \ "statement_code" === JInt(21)) + } + + test("parse_command returns null for null input") { + checkEvaluation(ParseCommand(Literal.create(null, StringType)), null) + } + + test("parse_command does not throw on syntax error") { + val j = evalJson("NOT A STATEMENT !!!") + assert(j \ "parse_success" === JBool(false)) + assert(j \ "error" \ "errorClass" === JString("PARSE_SYNTAX_ERROR")) + assert((j \ "error" \ "messageTemplate") != JNothing) + } + + test("parse_command works with CodegenFallback path") { + val expr = ParseCommand(Literal("INSERT INTO t SELECT 1")) + assert(expr.isInstanceOf[CodegenFallback]) + val j = evalJson("INSERT INTO t SELECT 1") + assert(j \ "statement_identifier" === JString("INSERT")) + } +} diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala new file mode 100644 index 000000000000..38870fc4a6b3 --- /dev/null +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.parser + +import org.json4s._ +import org.json4s.jackson.JsonMethods.parse + +import org.apache.spark.SparkFunSuite + +class ParseCommandResultSuite extends SparkFunSuite { + + private def json(sql: String): JValue = parse(ParseCommandResult.fromSql(sql)) + + private def obj(sql: String): JObject = json(sql).asInstanceOf[JObject] + + private def field(sql: String, name: String): JValue = + obj(sql).obj.find(_._1 == name).map(_._2).getOrElse(JNothing) + + test("SELECT classification uses Table 39 SELECT / code 21") { + val j = obj("SELECT a FROM t") + assert(j \ "parse_success" === JBool(true)) + assert(j \ "statement_identifier" === JString("SELECT")) + assert(j \ "statement_code" === JInt(21)) + assert(j \ "statement_type" === + JString("direct select statement: multiple rows")) + assert(j \ "statement_class" === JString("SQL-data statement")) + } + + test("CREATE TABLE and CTAS share CREATE TABLE code 77") { + val create = obj("CREATE TABLE t (a INT)") + assert(create \ "statement_identifier" === JString("CREATE TABLE")) + assert(create \ "statement_code" === JInt(77)) + assert(create \ "as_subquery" === JNothing) + + val ctas = obj("CREATE TABLE t AS SELECT 1 AS a") + assert(ctas \ "statement_identifier" === JString("CREATE TABLE")) + assert(ctas \ "statement_code" === JInt(77)) + assert(ctas \ "as_subquery" === JBool(true)) + } + + test("DML statement identifiers and codes") { + assert(field("INSERT INTO t SELECT 1", "statement_identifier") === + JString("INSERT")) + assert(field("INSERT INTO t SELECT 1", "statement_code") === JInt(50)) + + assert(field("DELETE FROM t WHERE a = 1", "statement_identifier") === + JString("DELETE WHERE")) + assert(field("DELETE FROM t WHERE a = 1", "statement_code") === JInt(19)) + + assert(field("UPDATE t SET a = 1 WHERE b = 2", "statement_identifier") === + JString("UPDATE WHERE")) + assert(field("UPDATE t SET a = 1 WHERE b = 2", "statement_code") === JInt(82)) + + assert(field( + "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE", + "statement_identifier") === JString("MERGE")) + assert(field( + "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE", + "statement_code") === JInt(128)) + } + + test("table and function references are multipart sequences") { + val j = obj("SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2") + assert(j \ "table_references" === JArray(List( + JArray(List(JString("cat"), JString("ns"), JString("t1"))), + JArray(List(JString("t2"))) + ))) + val funcs = (j \ "function_references").asInstanceOf[JArray].arr + assert(funcs.contains( + JArray(List(JString("db"), JString("my_func"))))) + assert(funcs.contains(JArray(List(JString("count"))))) + } + + test("select_list exposes name parts and expression text") { + val j = obj("SELECT t.a AS x, b + 1 FROM t") + val list = (j \ "select_list").asInstanceOf[JArray].arr + assert(list.length === 2) + assert(list.head \ "name" === JArray(List(JString("x")))) + assert((list.head \ "expression").asInstanceOf[JString].s.contains("a")) + } + + test("named and unnamed parameter markers") { + val named = obj("SELECT * FROM t WHERE a = :foo AND b = :bar") + assert(named \ "parameter_markers" \ "named" === + JArray(List(JString("foo"), JString("bar")))) + assert(named \ "parameter_markers" \ "unnamed_count" === JInt(0)) + + val unnamed = obj("SELECT * FROM t WHERE a = ? AND b = ?") + assert(unnamed \ "parameter_markers" \ "named" === JArray(Nil)) + assert(unnamed \ "parameter_markers" \ "unnamed_count" === JInt(2)) + } + + test("syntax error returns STANDARD error JSON without throwing") { + val j = obj("SELEC FROM t") + assert(j \ "parse_success" === JBool(false)) + assert(j \ "error" \ "errorClass" === JString("PARSE_SYNTAX_ERROR")) + assert((j \ "error" \ "messageTemplate") != JNothing) + assert(j \ "error" \ "sqlState" === JString("42601")) + } + + test("Spark-only statements use negative implementation-defined codes") { + val j = obj("CACHE TABLE t") + assert(j \ "parse_success" === JBool(true)) + assert(j \ "statement_identifier" === JString("CACHE TABLE")) + assert(j \ "statement_code" === JInt(-1)) + assert(j \ "statement_class" === + JString("implementation-defined statement")) + } + + test("Table 39 standard code pairs are pinned") { + assert(SqlStatementCodes.Select.statementCode === 21) + assert(SqlStatementCodes.Insert.statementCode === 50) + assert(SqlStatementCodes.DeleteWhere.statementCode === 19) + assert(SqlStatementCodes.UpdateWhere.statementCode === 82) + assert(SqlStatementCodes.Merge.statementCode === 128) + assert(SqlStatementCodes.CreateTable.statementCode === 77) + assert(SqlStatementCodes.CreateView.statementCode === 84) + assert(SqlStatementCodes.DropTable.statementCode === 32) + assert(SqlStatementCodes.AlterTable.statementCode === 4) + assert(SqlStatementCodes.TruncateTable.statementCode === 139) + assert(SqlStatementCodes.Unrecognized.statementCode === 0) + assert(SqlStatementCodes.CacheTable.statementCode < 0) + } +} From ca819b1155e6a7eb9a5ad15d416b579dc422139b Mon Sep 17 00:00:00 2001 From: srielau Date: Wed, 12 Aug 2026 12:53:31 +0000 Subject: [PATCH 2/9] [SPARK-58738][SQL] Walk CTE bodies when collecting parse_command references UnresolvedWith keeps CTE definitions in innerChildren, so collectWithSubqueries missed tables/functions/params inside WITH. Add foreachPlanDeep and broader CTE and nested-subquery tests. --- .../catalyst/parser/ParseCommandResult.scala | 66 ++++++----- .../parser/ParseCommandResultSuite.scala | 108 ++++++++++++++++++ 2 files changed, 144 insertions(+), 30 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala index 3f545708fe4e..96a2dde5a5ab 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala @@ -99,6 +99,28 @@ object ParseCommandResult { private def partsToJArray(parts: Seq[String]): JArray = JArray(parts.map(JString).toList) + /** + * Deep plan walk covering tree slots that standard `collect` / + * `collectWithSubqueries` miss: + * - [[UnresolvedWith]] CTE definitions (`innerChildren`, not `children`) + * - [[InsertIntoStatement]].table (non-child plan slot) + * Nested expression subqueries are still covered by `foreachWithSubqueries`. + */ + private def foreachPlanDeep(plan: LogicalPlan)(f: LogicalPlan => Unit): Unit = { + plan.foreachWithSubqueries { p => + f(p) + p match { + case w: UnresolvedWith => + w.cteRelations.foreach { case (_, ctePlan, _) => + foreachPlanDeep(ctePlan)(f) + } + case InsertIntoStatement(table, _, _, _, _, _, _, _, _) => + foreachPlanDeep(table)(f) + case _ => + } + } + } + /** * Collect multipart table/view identifiers as written in the SQL. * Deduplicates while preserving first-seen order. @@ -108,20 +130,12 @@ object ParseCommandResult { def add(parts: Seq[String]): Unit = { if (parts.nonEmpty) seen += parts } - def collectFrom(p: LogicalPlan): Unit = { - p.collectWithSubqueries { - case u: UnresolvedRelation => add(u.multipartIdentifier) - case u: UnresolvedTable => add(u.multipartIdentifier) - case u: UnresolvedView => add(u.multipartIdentifier) - case u: UnresolvedTableOrView => add(u.multipartIdentifier) - case u: UnresolvedIdentifier => add(u.nameParts) - } - } - collectFrom(plan) - // InsertIntoStatement.table is not a child of the plan tree. - plan match { - case InsertIntoStatement(table, _, _, _, _, _, _, _, _) => - collectFrom(table) + foreachPlanDeep(plan) { + case u: UnresolvedRelation => add(u.multipartIdentifier) + case u: UnresolvedTable => add(u.multipartIdentifier) + case u: UnresolvedView => add(u.multipartIdentifier) + case u: UnresolvedTableOrView => add(u.multipartIdentifier) + case u: UnresolvedIdentifier => add(u.nameParts) case _ => } seen.toSeq @@ -137,13 +151,12 @@ object ParseCommandResult { case f: UnresolvedFunction => add(f.nameParts) case _ => } - plan.collectWithSubqueries { - case p: LogicalPlan => - p.expressions.foreach(collectInExpression) - p match { - case u: UnresolvedTableValuedFunction => add(u.name) - case _ => - } + foreachPlanDeep(plan) { p => + p.expressions.foreach(collectInExpression) + p match { + case u: UnresolvedTableValuedFunction => add(u.name) + case _ => + } } seen.toSeq } @@ -204,15 +217,8 @@ object ParseCommandResult { case _: PosParameter => unnamedCount += 1 case _ => } - plan.collectWithSubqueries { - case p: LogicalPlan => - p.expressions.foreach(visitExpr) - } - // InsertIntoStatement.table is outside the child tree. - plan match { - case InsertIntoStatement(table, _, _, _, _, _, _, _, _) => - table.expressions.foreach(visitExpr) - case _ => + foreachPlanDeep(plan) { p => + p.expressions.foreach(visitExpr) } JObject( "named" -> JArray(named.toList.map(JString)), diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala index 38870fc4a6b3..7459195ee746 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala @@ -136,4 +136,112 @@ class ParseCommandResultSuite extends SparkFunSuite { assert(SqlStatementCodes.Unrecognized.statementCode === 0) assert(SqlStatementCodes.CacheTable.statementCode < 0) } + + private def tableRefs(sql: String): Set[Seq[String]] = + (obj(sql) \ "table_references").asInstanceOf[JArray].arr.map { + case JArray(parts) => parts.map(_.asInstanceOf[JString].s) + case other => fail(s"unexpected table_references entry: $other") + }.toSet + + private def funcRefs(sql: String): Set[Seq[String]] = + (obj(sql) \ "function_references").asInstanceOf[JArray].arr.map { + case JArray(parts) => parts.map(_.asInstanceOf[JString].s) + case other => fail(s"unexpected function_references entry: $other") + }.toSet + + test("CTE body tables are collected (UnresolvedWith innerChildren)") { + // CTE definitions are innerChildren, not children - easy to miss in walks. + val sql = + """WITH cte AS (SELECT a FROM hidden_base) + |SELECT a FROM cte""".stripMargin + assert(tableRefs(sql) === Set(Seq("hidden_base"), Seq("cte"))) + } + + test("multi-CTE chain and nested CTE definitions") { + val sql = + """WITH + | a AS (SELECT id FROM base_a), + | b AS ( + | WITH nested AS (SELECT id FROM base_nested) + | SELECT n.id FROM nested n JOIN base_b b ON n.id = b.id + | ) + |SELECT a.id, b.id FROM a JOIN b ON a.id = b.id""".stripMargin + assert(tableRefs(sql) === Set( + Seq("base_a"), + Seq("base_nested"), + Seq("nested"), + Seq("base_b"), + Seq("a"), + Seq("b"))) + } + + test("CTE with expression subqueries, functions, and parameters") { + val sql = + """WITH filtered AS ( + | SELECT upper(x) AS u, my_schema.my_udf(y) AS v + | FROM src + | WHERE z IN (SELECT z FROM lookup WHERE flag = :flag) + | AND EXISTS (SELECT 1 FROM probe WHERE probe.id = src.id) + |) + |SELECT u, count(v) FROM filtered WHERE u = ? GROUP BY u""".stripMargin + assert(tableRefs(sql) === Set( + Seq("src"), Seq("lookup"), Seq("probe"), Seq("filtered"))) + assert(funcRefs(sql).contains(Seq("upper"))) + assert(funcRefs(sql).contains(Seq("my_schema", "my_udf"))) + assert(funcRefs(sql).contains(Seq("count"))) + val params = obj(sql) \ "parameter_markers" + assert(params \ "named" === JArray(List(JString("flag")))) + assert(params \ "unnamed_count" === JInt(1)) + } + + test("WITH on INSERT / CTAS reaches CTE and target tables") { + val insertSql = + """INSERT INTO dest + |WITH s AS (SELECT a FROM src WHERE a > 0) + |SELECT a FROM s""".stripMargin + assert(tableRefs(insertSql) === Set(Seq("dest"), Seq("src"), Seq("s"))) + + val ctasSql = + """CREATE TABLE dest AS + |WITH s AS (SELECT a FROM src) + |SELECT a FROM s""".stripMargin + val ctas = obj(ctasSql) + assert(ctas \ "as_subquery" === JBool(true)) + assert(tableRefs(ctasSql).contains(Seq("src"))) + assert(tableRefs(ctasSql).contains(Seq("s"))) + } + + test("nested FROM / scalar / EXISTS subqueries outside CTEs") { + val sql = + """SELECT + | (SELECT max(v) FROM scalar_src) AS m, + | t.a + |FROM outer_t t + |JOIN (SELECT id FROM join_src) j ON t.id = j.id + |WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id) + | AND t.a IN (SELECT a FROM in_src)""".stripMargin + assert(tableRefs(sql) === Set( + Seq("scalar_src"), + Seq("outer_t"), + Seq("join_src"), + Seq("exists_src"), + Seq("in_src"))) + assert(funcRefs(sql).contains(Seq("max"))) + } + + test("MERGE collects target, source, and action-expression tables") { + val sql = + """MERGE INTO tgt t + |USING (SELECT id FROM src) s + |ON t.id = s.id + |WHEN MATCHED AND t.flag IN (SELECT flag FROM flags) THEN + | UPDATE SET t.v = (SELECT v FROM vals WHERE vals.id = t.id) + |WHEN NOT MATCHED THEN + | INSERT (id) VALUES (s.id)""".stripMargin + val refs = tableRefs(sql) + assert(refs.contains(Seq("tgt"))) + assert(refs.contains(Seq("src"))) + assert(refs.contains(Seq("flags"))) + assert(refs.contains(Seq("vals"))) + } } From dc2457c4b64140eeded6a83aba19792920136ec1 Mon Sep 17 00:00:00 2001 From: srielau Date: Wed, 12 Aug 2026 14:03:51 +0000 Subject: [PATCH 3/9] [SPARK-58738][SQL] Support BEGIN END scripts in parse_command Classify CompoundBody as BEGIN END (-22) and deep-walk SingleStatement roots, exception handlers, and simple CASE else bodies so nested refs are not dropped. --- .../catalyst/parser/ParseCommandResult.scala | 10 +++ .../catalyst/parser/SqlStatementCodes.scala | 3 + .../parser/ParseCommandResultSuite.scala | 78 +++++++++++++++++++ 3 files changed, 91 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala index 96a2dde5a5ab..48d37f072022 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala @@ -104,6 +104,9 @@ object ParseCommandResult { * `collectWithSubqueries` miss: * - [[UnresolvedWith]] CTE definitions (`innerChildren`, not `children`) * - [[InsertIntoStatement]].table (non-child plan slot) + * - [[SingleStatement]].parsedPlan (children expose only nested children) + * - [[CompoundBody]].handlers (not in `children`) + * - [[SimpleCaseStatement]].elseBody (not in `children`) * Nested expression subqueries are still covered by `foreachWithSubqueries`. */ private def foreachPlanDeep(plan: LogicalPlan)(f: LogicalPlan => Unit): Unit = { @@ -116,6 +119,13 @@ object ParseCommandResult { } case InsertIntoStatement(table, _, _, _, _, _, _, _, _) => foreachPlanDeep(table)(f) + case s: SingleStatement => + // Root of the wrapped statement is skipped by SingleStatement.children. + foreachPlanDeep(s.parsedPlan)(f) + case c: CompoundBody => + c.handlers.foreach(h => foreachPlanDeep(h)(f)) + case s: SimpleCaseStatement => + s.elseBody.foreach(b => foreachPlanDeep(b)(f)) case _ => } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala index 1ba005cb4754..3e7fbf746ffc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala @@ -113,6 +113,8 @@ object SqlStatementCodes { val RefreshFunction: SqlStatementClassification = spark("REFRESH FUNCTION", -19) val CommentOnNamespace: SqlStatementClassification = spark("COMMENT ON NAMESPACE", -20) val CommentOnTable: SqlStatementClassification = spark("COMMENT ON TABLE", -21) + // SQL/PSM-style scripting (9075-4); not in Foundation Table 39. + val BeginEnd: SqlStatementClassification = spark("BEGIN END", -22) private def spark(identifier: String, code: Int): SqlStatementClassification = { assert(code < 0, s"Spark statement codes must be negative, got $code") @@ -126,6 +128,7 @@ object SqlStatementCodes { /** Classify an unresolved logical plan. */ def classify(plan: LogicalPlan): SqlStatementClassification = plan match { case UnresolvedWith(child, _, _) => classify(child) + case _: CompoundBody => BeginEnd case _: InsertIntoStatement => Insert case _: DeleteFromTable | _: DeleteFromTableWithFilters => DeleteWhere case _: UpdateTable => UpdateWhere diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala index 7459195ee746..fb6b5e0fae9c 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala @@ -135,6 +135,7 @@ class ParseCommandResultSuite extends SparkFunSuite { assert(SqlStatementCodes.TruncateTable.statementCode === 139) assert(SqlStatementCodes.Unrecognized.statementCode === 0) assert(SqlStatementCodes.CacheTable.statementCode < 0) + assert(SqlStatementCodes.BeginEnd.statementCode === -22) } private def tableRefs(sql: String): Set[Seq[String]] = @@ -244,4 +245,81 @@ class ParseCommandResultSuite extends SparkFunSuite { assert(refs.contains(Seq("flags"))) assert(refs.contains(Seq("vals"))) } + + test("BEGIN END script classification uses Spark code -22") { + val j = obj("BEGIN SELECT 1; END") + assert(j \ "parse_success" === JBool(true)) + assert(j \ "statement_identifier" === JString("BEGIN END")) + assert(j \ "statement_code" === JInt(-22)) + assert(j \ "statement_class" === + JString("implementation-defined statement")) + // Compound scripts have no single primary select list. + assert(j \ "select_list" === JArray(Nil)) + } + + test("BEGIN END walks SingleStatement.parsedPlan for tables and functions") { + // SingleStatement.children skips the statement root (e.g. Project), so a + // naive collectWithSubqueries misses project-list functions. + val sql = + """BEGIN + | SELECT count(a), upper(b) FROM script_t WHERE c = :p; + |END""".stripMargin + assert(tableRefs(sql) === Set(Seq("script_t"))) + assert(funcRefs(sql).contains(Seq("count"))) + assert(funcRefs(sql).contains(Seq("upper"))) + assert(obj(sql) \ "parameter_markers" \ "named" === + JArray(List(JString("p")))) + } + + test("BEGIN END with IF / WHILE / FOR collects nested statement refs") { + val sql = + """BEGIN + | IF (SELECT flag FROM gate) THEN + | INSERT INTO dest SELECT * FROM src_if; + | ELSE + | DELETE FROM src_else WHERE id IN (SELECT id FROM doomed); + | END IF; + | WHILE (SELECT cont FROM ctrl) DO + | UPDATE tgt SET v = 1 WHERE id IN (SELECT id FROM while_src); + | END WHILE; + | FOR x AS SELECT id FROM for_src DO + | SELECT my_udf(id) FROM for_body WHERE id = x.id; + | END FOR; + |END""".stripMargin + val refs = tableRefs(sql) + assert(refs === Set( + Seq("gate"), + Seq("dest"), + Seq("src_if"), + Seq("src_else"), + Seq("doomed"), + Seq("ctrl"), + Seq("tgt"), + Seq("while_src"), + Seq("for_src"), + Seq("for_body"))) + assert(funcRefs(sql).contains(Seq("my_udf"))) + } + + test("BEGIN END exception handler body tables are collected") { + val sql = + """BEGIN + | DECLARE EXIT HANDLER FOR SQLEXCEPTION + | BEGIN + | INSERT INTO err_log SELECT * FROM failing_row; + | END; + | SELECT a FROM main_t; + |END""".stripMargin + assert(tableRefs(sql) === Set( + Seq("err_log"), Seq("failing_row"), Seq("main_t"))) + } + + test("BEGIN END with CTE inside script body") { + val sql = + """BEGIN + | WITH c AS (SELECT a FROM cte_base) + | SELECT a FROM c; + |END""".stripMargin + assert(tableRefs(sql) === Set(Seq("cte_base"), Seq("c"))) + } } From bbdd79f46621f0c972e81af399728770a4d672c4 Mon Sep 17 00:00:00 2001 From: srielau Date: Wed, 12 Aug 2026 14:24:18 +0000 Subject: [PATCH 4/9] [SPARK-58738][SQL] Add parse_command SQLQueryTestSuite golden tests Cover SELECT/DML/DDL, CTEs, scripts, errors, and batch evaluation via sql-tests inputs with regenerated result and analyzer golden files. --- .../analyzer-results/parse-command.sql.out | 169 +++++++++++++++ .../sql-tests/inputs/parse-command.sql | 71 +++++++ .../sql-tests/results/parse-command.sql.out | 193 ++++++++++++++++++ 3 files changed, 433 insertions(+) create mode 100644 sql/core/src/test/resources/sql-tests/analyzer-results/parse-command.sql.out create mode 100644 sql/core/src/test/resources/sql-tests/inputs/parse-command.sql create mode 100644 sql/core/src/test/resources/sql-tests/results/parse-command.sql.out diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/parse-command.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-command.sql.out new file mode 100644 index 000000000000..b43dc9abcd49 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-command.sql.out @@ -0,0 +1,169 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +SELECT parse_command(NULL) +-- !query analysis +Project [parse_command(cast(null as string)) AS parse_command(NULL)#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('SELECT a, b FROM t') +-- !query analysis +Project [parse_command(SELECT a, b FROM t) AS parse_command(SELECT a, b FROM t)#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2') +-- !query analysis +Project [parse_command(SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2) AS parse_command(SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2)#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('INSERT INTO t SELECT 1') +-- !query analysis +Project [parse_command(INSERT INTO t SELECT 1) AS parse_command(INSERT INTO t SELECT 1)#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('DELETE FROM t WHERE a = 1') +-- !query analysis +Project [parse_command(DELETE FROM t WHERE a = 1) AS parse_command(DELETE FROM t WHERE a = 1)#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('UPDATE t SET a = 1 WHERE b = 2') +-- !query analysis +Project [parse_command(UPDATE t SET a = 1 WHERE b = 2) AS parse_command(UPDATE t SET a = 1 WHERE b = 2)#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE') +-- !query analysis +Project [parse_command(MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE) AS parse_command(MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE)#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('CREATE TABLE t (a INT)') +-- !query analysis +Project [parse_command(CREATE TABLE t (a INT)) AS parse_command(CREATE TABLE t (a INT))#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('CREATE TABLE t AS SELECT 1 AS a') +-- !query analysis +Project [parse_command(CREATE TABLE t AS SELECT 1 AS a) AS parse_command(CREATE TABLE t AS SELECT 1 AS a)#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('DROP TABLE t') +-- !query analysis +Project [parse_command(DROP TABLE t) AS parse_command(DROP TABLE t)#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('CACHE TABLE t') +-- !query analysis +Project [parse_command(CACHE TABLE t) AS parse_command(CACHE TABLE t)#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('SELECT * FROM t WHERE a = :foo AND b = ?') +-- !query analysis +Project [parse_command(SELECT * FROM t WHERE a = :foo AND b = ?) AS parse_command(SELECT * FROM t WHERE a = :foo AND b = ?)#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte') +-- !query analysis +Project [parse_command(WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte) AS parse_command(WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte)#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)') +-- !query analysis +Project [parse_command(SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)) AS parse_command(SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id))#x] ++- OneRowRelation + + +-- !query +SELECT get_json_object(parse_command('SELEC FROM t'), '$.parse_success') +-- !query analysis +Project [get_json_object(parse_command(SELEC FROM t), $.parse_success) AS get_json_object(parse_command(SELEC FROM t), $.parse_success)#x] ++- OneRowRelation + + +-- !query +SELECT get_json_object(parse_command('SELEC FROM t'), '$.error.errorClass') +-- !query analysis +Project [get_json_object(parse_command(SELEC FROM t), $.error.errorClass) AS get_json_object(parse_command(SELEC FROM t), $.error.errorClass)#x] ++- OneRowRelation + + +-- !query +SELECT get_json_object(parse_command('SELEC FROM t'), '$.error.sqlState') +-- !query analysis +Project [get_json_object(parse_command(SELEC FROM t), $.error.sqlState) AS get_json_object(parse_command(SELEC FROM t), $.error.sqlState)#x] ++- OneRowRelation + + +-- !query +SELECT sql_text, parse_command(sql_text) FROM VALUES + ('SELECT 1'), + ('INSERT INTO t SELECT 1'), + ('CACHE TABLE t') +AS t(sql_text) +-- !query analysis +Project [sql_text#x, parse_command(sql_text#x) AS parse_command(sql_text)#x] ++- SubqueryAlias t + +- LocalRelation [sql_text#x] + + +-- !query +SELECT parse_command('BEGIN SELECT 1; END') +-- !query analysis +Project [parse_command(BEGIN SELECT 1; END) AS parse_command(BEGIN SELECT 1; END)#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END') +-- !query analysis +Project [parse_command(BEGIN SELECT count(a) FROM script_t WHERE c = :p; END) AS parse_command(BEGIN SELECT count(a) FROM script_t WHERE c = :p; END)#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END') +-- !query analysis +Project [parse_command(BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END) AS parse_command(BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END)#x] ++- OneRowRelation + + +-- !query +SELECT parse_command('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END') +-- !query analysis +Project [parse_command(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END) AS parse_command(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END)#x] ++- OneRowRelation + + +-- !query +SELECT + get_json_object(parse_command('BEGIN SELECT 1; END'), '$.statement_identifier') AS statement_identifier, + get_json_object(parse_command('BEGIN SELECT 1; END'), '$.statement_code') AS statement_code, + get_json_object(parse_command('BEGIN SELECT count(a) FROM script_t; END'), '$.table_references') AS table_references, + get_json_object(parse_command('BEGIN SELECT count(a) FROM script_t; END'), '$.function_references') AS function_references +-- !query analysis +Project [get_json_object(parse_command(BEGIN SELECT 1; END), $.statement_identifier) AS statement_identifier#x, get_json_object(parse_command(BEGIN SELECT 1; END), $.statement_code) AS statement_code#x, get_json_object(parse_command(BEGIN SELECT count(a) FROM script_t; END), $.table_references) AS table_references#x, get_json_object(parse_command(BEGIN SELECT count(a) FROM script_t; END), $.function_references) AS function_references#x] ++- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/inputs/parse-command.sql b/sql/core/src/test/resources/sql-tests/inputs/parse-command.sql new file mode 100644 index 000000000000..a637ce6ab161 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/parse-command.sql @@ -0,0 +1,71 @@ +-- End-to-end coverage for parse_command (SPARK-58738). +-- Returns compact JSON for parse-only statement analysis. + +-- null input +SELECT parse_command(NULL); + +-- basic SELECT classification and references +SELECT parse_command('SELECT a, b FROM t'); +SELECT parse_command('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2'); + +-- DML +SELECT parse_command('INSERT INTO t SELECT 1'); +SELECT parse_command('DELETE FROM t WHERE a = 1'); +SELECT parse_command('UPDATE t SET a = 1 WHERE b = 2'); +SELECT parse_command('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE'); + +-- DDL / CTAS +SELECT parse_command('CREATE TABLE t (a INT)'); +SELECT parse_command('CREATE TABLE t AS SELECT 1 AS a'); +SELECT parse_command('DROP TABLE t'); + +-- Spark-only statements (negative Table 39 codes) +SELECT parse_command('CACHE TABLE t'); + +-- parameter markers +SELECT parse_command('SELECT * FROM t WHERE a = :foo AND b = ?'); + +-- CTE: UnresolvedWith CTE bodies are innerChildren +SELECT parse_command('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte'); + +-- nested subqueries +SELECT parse_command('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)'); + +-- syntax error: never throws; STANDARD error nested under parse_success=false +SELECT get_json_object(parse_command('SELEC FROM t'), '$.parse_success'); +SELECT get_json_object(parse_command('SELEC FROM t'), '$.error.errorClass'); +SELECT get_json_object(parse_command('SELEC FROM t'), '$.error.sqlState'); + +-- batch over a column of SQL text +SELECT sql_text, parse_command(sql_text) FROM VALUES + ('SELECT 1'), + ('INSERT INTO t SELECT 1'), + ('CACHE TABLE t') +AS t(sql_text); + +-- BEGIN END scripts contain ';' inside the string literal; use query delimiters +-- so the test harness does not split on those semicolons. +--QUERY-DELIMITER-START +SELECT parse_command('BEGIN SELECT 1; END'); +--QUERY-DELIMITER-END + +--QUERY-DELIMITER-START +SELECT parse_command('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END'); +--QUERY-DELIMITER-END + +--QUERY-DELIMITER-START +SELECT parse_command('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END'); +--QUERY-DELIMITER-END + +--QUERY-DELIMITER-START +SELECT parse_command('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END'); +--QUERY-DELIMITER-END + +-- extract key fields from a script for readable assertions +--QUERY-DELIMITER-START +SELECT + get_json_object(parse_command('BEGIN SELECT 1; END'), '$.statement_identifier') AS statement_identifier, + get_json_object(parse_command('BEGIN SELECT 1; END'), '$.statement_code') AS statement_code, + get_json_object(parse_command('BEGIN SELECT count(a) FROM script_t; END'), '$.table_references') AS table_references, + get_json_object(parse_command('BEGIN SELECT count(a) FROM script_t; END'), '$.function_references') AS function_references; +--QUERY-DELIMITER-END diff --git a/sql/core/src/test/resources/sql-tests/results/parse-command.sql.out b/sql/core/src/test/resources/sql-tests/results/parse-command.sql.out new file mode 100644 index 000000000000..256510ae3b21 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/parse-command.sql.out @@ -0,0 +1,193 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +SELECT parse_command(NULL) +-- !query schema +struct +-- !query output +NULL + + +-- !query +SELECT parse_command('SELECT a, b FROM t') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"statement_type":"direct select statement: multiple rows","statement_class":"SQL-data statement","table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"],"expression":"a"},{"name":["b"],"expression":"b"}],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"statement_type":"direct select statement: multiple rows","statement_class":"SQL-data statement","table_references":[["cat","ns","t1"],["t2"]],"function_references":[["db","my_func"],["count"]],"select_list":[{"name":[],"expression":"db.my_func(a)"},{"name":[],"expression":"count(b)"}],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command('INSERT INTO t SELECT 1') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"statement_type":"insert statement","statement_class":"SQL-data change statement","table_references":[["t"]],"function_references":[],"select_list":[{"name":[],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command('DELETE FROM t WHERE a = 1') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"DELETE WHERE","statement_code":19,"statement_type":"delete statement: searched","statement_class":"SQL-data change statement","table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command('UPDATE t SET a = 1 WHERE b = 2') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"UPDATE WHERE","statement_code":82,"statement_type":"update statement: searched","statement_class":"SQL-data change statement","table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"MERGE","statement_code":128,"statement_type":"merge statement","statement_class":"SQL-data change statement","table_references":[["t"],["s"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command('CREATE TABLE t (a INT)') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"statement_type":"table definition","statement_class":"SQL-schema statement","table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command('CREATE TABLE t AS SELECT 1 AS a') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"statement_type":"table definition","statement_class":"SQL-schema statement","as_subquery":true,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command('DROP TABLE t') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"DROP TABLE","statement_code":32,"statement_type":"drop table statement","statement_class":"SQL-schema statement","table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command('CACHE TABLE t') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"CACHE TABLE","statement_code":-1,"statement_type":"cache table","statement_class":"implementation-defined statement","table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command('SELECT * FROM t WHERE a = :foo AND b = ?') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"statement_type":"direct select statement: multiple rows","statement_class":"SQL-data statement","table_references":[["t"]],"function_references":[],"select_list":[{"name":["*"],"expression":"unresolvedstar()"}],"parameter_markers":{"named":["foo"],"unnamed_count":1}} + + +-- !query +SELECT parse_command('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"statement_type":"direct select statement: multiple rows","statement_class":"SQL-data statement","table_references":[["hidden_base"],["cte"]],"function_references":[],"select_list":[{"name":["a"],"expression":"a"}],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"statement_type":"direct select statement: multiple rows","statement_class":"SQL-data statement","table_references":[["scalar_src"],["exists_src"],["outer_t"]],"function_references":[["max"]],"select_list":[{"name":["m"],"expression":"scalarsubquery()"},{"name":["t","a"],"expression":"t.a"}],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT get_json_object(parse_command('SELEC FROM t'), '$.parse_success') +-- !query schema +struct +-- !query output +false + + +-- !query +SELECT get_json_object(parse_command('SELEC FROM t'), '$.error.errorClass') +-- !query schema +struct +-- !query output +PARSE_SYNTAX_ERROR + + +-- !query +SELECT get_json_object(parse_command('SELEC FROM t'), '$.error.sqlState') +-- !query schema +struct +-- !query output +42601 + + +-- !query +SELECT sql_text, parse_command(sql_text) FROM VALUES + ('SELECT 1'), + ('INSERT INTO t SELECT 1'), + ('CACHE TABLE t') +AS t(sql_text) +-- !query schema +struct +-- !query output +CACHE TABLE t {"parse_success":true,"statement_identifier":"CACHE TABLE","statement_code":-1,"statement_type":"cache table","statement_class":"implementation-defined statement","table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +INSERT INTO t SELECT 1 {"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"statement_type":"insert statement","statement_class":"SQL-data change statement","table_references":[["t"]],"function_references":[],"select_list":[{"name":[],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} +SELECT 1 {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"statement_type":"direct select statement: multiple rows","statement_class":"SQL-data statement","table_references":[],"function_references":[],"select_list":[{"name":[],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command('BEGIN SELECT 1; END') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"statement_type":"begin end","statement_class":"implementation-defined statement","table_references":[],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"statement_type":"begin end","statement_class":"implementation-defined statement","table_references":[["script_t"]],"function_references":[["count"]],"select_list":[],"parameter_markers":{"named":["p"],"unnamed_count":0}} + + +-- !query +SELECT parse_command('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"statement_type":"begin end","statement_class":"implementation-defined statement","table_references":[["gate"],["dest"],["src_if"],["src_else"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"statement_type":"begin end","statement_class":"implementation-defined statement","table_references":[["err_log"],["failing_row"],["main_t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT + get_json_object(parse_command('BEGIN SELECT 1; END'), '$.statement_identifier') AS statement_identifier, + get_json_object(parse_command('BEGIN SELECT 1; END'), '$.statement_code') AS statement_code, + get_json_object(parse_command('BEGIN SELECT count(a) FROM script_t; END'), '$.table_references') AS table_references, + get_json_object(parse_command('BEGIN SELECT count(a) FROM script_t; END'), '$.function_references') AS function_references +-- !query schema +struct +-- !query output +BEGIN END -22 [["script_t"]] [["count"]] From 0359507603217fb4a4fe83cd7ebddd727e871fd0 Mon Sep 17 00:00:00 2001 From: srielau Date: Wed, 12 Aug 2026 17:36:33 +0000 Subject: [PATCH 5/9] [SPARK-58738][SQL] Slim parse_command classification and expand coverage Keep only statement_identifier/code on success, expose error line/position, and broaden complex SQL, scripting, and non-syntax parser error tests. --- .../catalyst/expressions/ParseCommand.scala | 8 +- .../catalyst/parser/ParseCommandResult.scala | 48 ++- .../catalyst/parser/SqlStatementCodes.scala | 77 ++-- .../parser/ParseCommandResultSuite.scala | 139 ++++++- .../analyzer-results/parse-command.sql.out | 383 ++++++++++++++++++ .../sql-tests/inputs/parse-command.sql | 187 +++++++++ .../sql-tests/results/parse-command.sql.out | 324 ++++++++++++++- 7 files changed, 1067 insertions(+), 99 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala index 50607df7f054..9e5acf6cf8ef 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala @@ -25,7 +25,7 @@ import org.apache.spark.unsafe.types.UTF8String /** * Parses a SQL statement string and returns a compact JSON description of the - * unresolved statement (classification, references, select list, parameters), + * unresolved statement (identifier/code, references, select list, parameters), * or a STANDARD-format error object when the statement does not parse. * * Designed for batch evaluation over DataFrames of SQL text; never throws on @@ -34,10 +34,10 @@ import org.apache.spark.unsafe.types.UTF8String // scalastyle:off line.size.limit @ExpressionDescription( usage = """_FUNC_(sqlStmt) - Parses `sqlStmt` and returns a JSON string describing the - statement (parse success, Table 39 statement classification, table and function + statement (parse success, Table 39 statement identifier/code, table and function references, select-list columns, and parameter markers). On syntax error returns - JSON with `parse_success` false and a nested STANDARD error object instead of - throwing.""", + JSON with `parse_success` false, source location, and a nested STANDARD error object + instead of throwing.""", arguments = """ Arguments: * sqlStmt - A SQL statement string to parse. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala index 48d37f072022..327b8e9a3f27 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala @@ -27,15 +27,17 @@ import org.apache.spark.{ErrorMessageFormat, SparkThrowable, SparkThrowableHelpe import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.catalyst.trees.Origin +import org.apache.spark.sql.exceptions.SqlScriptingException /** * Parses a SQL statement string and returns a compact JSON description of the * unresolved plan (parse-only; no catalog resolution). * - * On success the JSON includes statement classification (ISO/IEC 9075-2:2023 - * Table 39), table/function references, select-list items, and parameter - * markers. On parse failure it returns `parse_success: false` with a nested - * STANDARD-format error object and does not throw. + * On success the JSON includes the statement identifier/code (ISO/IEC + * 9075-2:2023 Table 39), table/function references, select-list items, and + * parameter markers. On parse failure it returns `parse_success: false` with + * source location and a nested STANDARD-format error object, and does not throw. */ object ParseCommandResult { @@ -73,11 +75,6 @@ object ParseCommandResult { fields += "parse_success" -> JBool(true) fields += "statement_identifier" -> JString(classification.statementIdentifier) fields += "statement_code" -> JInt(classification.statementCode) - fields += "statement_type" -> JString(classification.statementType) - fields += "statement_class" -> JString(classification.statementClass) - if (classification.asSubquery) { - fields += "as_subquery" -> JBool(true) - } fields += "table_references" -> JArray( collectTableReferences(plan).map(partsToJArray).toList) fields += "function_references" -> JArray( @@ -89,16 +86,41 @@ object ParseCommandResult { private def errorJson(e: SparkThrowable with Throwable): String = { val errorObj = parseJson( - SparkThrowableHelper.getMessage(e, ErrorMessageFormat.STANDARD)) + SparkThrowableHelper.getMessage(e, ErrorMessageFormat.STANDARD)).asInstanceOf[JObject] + val origin = e match { + case p: ParseException => Some(p.start) + case s: SqlScriptingException => Some(s.origin) + case _ => None + } + val locationFields = origin.toSeq.flatMap(originFields) compact(render(JObject( "parse_success" -> JBool(false), - "error" -> errorObj + "error" -> JObject(errorObj.obj ++ locationFields) ))) } + private def originFields(origin: Origin): Seq[JField] = Seq( + origin.line.map(line => "line" -> JInt(line)), + origin.startPosition.map(position => "position" -> JInt(position))).flatten + private def partsToJArray(parts: Seq[String]): JArray = JArray(parts.map(JString).toList) + /** + * Walk expressions in all product fields, including wrappers such as column + * definitions that [[LogicalPlan.expressions]] does not descend into. + */ + private def foreachExpressionDeep(plan: LogicalPlan)(f: Expression => Unit): Unit = { + def visit(value: Any): Unit = value match { + case e: Expression => f(e) + case _: LogicalPlan => + case values: Iterable[_] => values.foreach(visit) + case value: Product => value.productIterator.foreach(visit) + case _ => + } + plan.productIterator.foreach(visit) + } + /** * Deep plan walk covering tree slots that standard `collect` / * `collectWithSubqueries` miss: @@ -162,7 +184,7 @@ object ParseCommandResult { case _ => } foreachPlanDeep(plan) { p => - p.expressions.foreach(collectInExpression) + foreachExpressionDeep(p)(collectInExpression) p match { case u: UnresolvedTableValuedFunction => add(u.name) case _ => @@ -228,7 +250,7 @@ object ParseCommandResult { case _ => } foreachPlanDeep(plan) { p => - p.expressions.foreach(visitExpr) + foreachExpressionDeep(p)(visitExpr) } JObject( "named" -> JArray(named.toList.map(JString)), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala index 3e7fbf746ffc..fd8126028967 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala @@ -24,20 +24,13 @@ import org.apache.spark.sql.catalyst.plans.logical._ * Classification of a parsed SQL statement using ISO/IEC 9075-2:2023 Table 39, * "SQL-statement codes" (clause 23.1 <get diagnostics statement>). * - * @param statementType BNF production name without angle brackets * @param statementIdentifier Table 39 Identifier column (or Spark product name) * @param statementCode Table 39 Code column; Spark-only statements use negative * implementation-defined codes (Table 39 IE005 / IV190) - * @param statementClass Clause 4.41.2 function class, or - * "implementation-defined statement" for Spark extensions - * @param asSubquery True when <table definition> contains an <as subquery clause> */ case class SqlStatementClassification( - statementType: String, statementIdentifier: String, - statementCode: Int, - statementClass: String, - asSubquery: Boolean = false) + statementCode: Int) /** * Maps unresolved [[LogicalPlan]]s to Table 39 statement codes. @@ -49,46 +42,29 @@ case class SqlStatementClassification( object SqlStatementCodes { // Standard Table 39 entries used by Spark SQL (ISO/IEC 9075-2:2023). - val Select: SqlStatementClassification = SqlStatementClassification( - "direct select statement: multiple rows", "SELECT", 21, "SQL-data statement") - val Insert: SqlStatementClassification = SqlStatementClassification( - "insert statement", "INSERT", 50, "SQL-data change statement") - val DeleteWhere: SqlStatementClassification = SqlStatementClassification( - "delete statement: searched", "DELETE WHERE", 19, "SQL-data change statement") - val UpdateWhere: SqlStatementClassification = SqlStatementClassification( - "update statement: searched", "UPDATE WHERE", 82, "SQL-data change statement") - val Merge: SqlStatementClassification = SqlStatementClassification( - "merge statement", "MERGE", 128, "SQL-data change statement") - val CreateTable: SqlStatementClassification = SqlStatementClassification( - "table definition", "CREATE TABLE", 77, "SQL-schema statement") - val CreateView: SqlStatementClassification = SqlStatementClassification( - "view definition", "CREATE VIEW", 84, "SQL-schema statement") - val DropTable: SqlStatementClassification = SqlStatementClassification( - "drop table statement", "DROP TABLE", 32, "SQL-schema statement") - val DropView: SqlStatementClassification = SqlStatementClassification( - "drop view statement", "DROP VIEW", 36, "SQL-schema statement") - val AlterTable: SqlStatementClassification = SqlStatementClassification( - "alter table statement", "ALTER TABLE", 4, "SQL-schema statement") - val CreateSchema: SqlStatementClassification = SqlStatementClassification( - "schema definition", "CREATE SCHEMA", 64, "SQL-schema statement") - val DropSchema: SqlStatementClassification = SqlStatementClassification( - "drop schema statement", "DROP SCHEMA", 31, "SQL-schema statement") - val SetSchema: SqlStatementClassification = SqlStatementClassification( - "set schema statement", "SET SCHEMA", 74, "SQL-session statement") - val TruncateTable: SqlStatementClassification = SqlStatementClassification( - "truncate table statement", "TRUNCATE TABLE", 139, "SQL-data change statement") - val CreateRoutine: SqlStatementClassification = SqlStatementClassification( - "schema routine", "CREATE ROUTINE", 14, "SQL-schema statement") - val DropRoutine: SqlStatementClassification = SqlStatementClassification( - "drop routine statement", "DROP ROUTINE", 30, "SQL-schema statement") - val ExecuteImmediate: SqlStatementClassification = SqlStatementClassification( - "execute immediate statement", "EXECUTE IMMEDIATE", 43, "SQL-dynamic statement") - val Call: SqlStatementClassification = SqlStatementClassification( - "call statement", "CALL", 7, "SQL-control statement") + val Select: SqlStatementClassification = SqlStatementClassification("SELECT", 21) + val Insert: SqlStatementClassification = SqlStatementClassification("INSERT", 50) + val DeleteWhere: SqlStatementClassification = SqlStatementClassification("DELETE WHERE", 19) + val UpdateWhere: SqlStatementClassification = SqlStatementClassification("UPDATE WHERE", 82) + val Merge: SqlStatementClassification = SqlStatementClassification("MERGE", 128) + val CreateTable: SqlStatementClassification = SqlStatementClassification("CREATE TABLE", 77) + val CreateView: SqlStatementClassification = SqlStatementClassification("CREATE VIEW", 84) + val DropTable: SqlStatementClassification = SqlStatementClassification("DROP TABLE", 32) + val DropView: SqlStatementClassification = SqlStatementClassification("DROP VIEW", 36) + val AlterTable: SqlStatementClassification = SqlStatementClassification("ALTER TABLE", 4) + val CreateSchema: SqlStatementClassification = SqlStatementClassification("CREATE SCHEMA", 64) + val DropSchema: SqlStatementClassification = SqlStatementClassification("DROP SCHEMA", 31) + val SetSchema: SqlStatementClassification = SqlStatementClassification("SET SCHEMA", 74) + val TruncateTable: SqlStatementClassification = + SqlStatementClassification("TRUNCATE TABLE", 139) + val CreateRoutine: SqlStatementClassification = SqlStatementClassification("CREATE ROUTINE", 14) + val DropRoutine: SqlStatementClassification = SqlStatementClassification("DROP ROUTINE", 30) + val ExecuteImmediate: SqlStatementClassification = + SqlStatementClassification("EXECUTE IMMEDIATE", 43) + val Call: SqlStatementClassification = SqlStatementClassification("CALL", 7) // Table 39 "Unrecognized statements": empty identifier, code 0. - val Unrecognized: SqlStatementClassification = SqlStatementClassification( - "", "", 0, "implementation-defined statement") + val Unrecognized: SqlStatementClassification = SqlStatementClassification("", 0) // Spark product-specific identifiers with append-only negative codes // (Table 39 implementation-defined / IE005 row: negative Code values). @@ -118,11 +94,7 @@ object SqlStatementCodes { private def spark(identifier: String, code: Int): SqlStatementClassification = { assert(code < 0, s"Spark statement codes must be negative, got $code") - SqlStatementClassification( - statementType = identifier.toLowerCase(java.util.Locale.ROOT), - statementIdentifier = identifier, - statementCode = code, - statementClass = "implementation-defined statement") + SqlStatementClassification(statementIdentifier = identifier, statementCode = code) } /** Classify an unresolved logical plan. */ @@ -133,8 +105,7 @@ object SqlStatementCodes { case _: DeleteFromTable | _: DeleteFromTableWithFilters => DeleteWhere case _: UpdateTable => UpdateWhere case _: MergeIntoTable => Merge - case _: CreateTableAsSelect | _: ReplaceTableAsSelect => - CreateTable.copy(asSubquery = true) + case _: CreateTableAsSelect | _: ReplaceTableAsSelect => CreateTable case _: CreateTable | _: CreateTableLike | _: ReplaceTable => CreateTable case _: CreateView => CreateView case _: DropTable => DropTable diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala index fb6b5e0fae9c..fbfb7df6ffcb 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala @@ -36,9 +36,8 @@ class ParseCommandResultSuite extends SparkFunSuite { assert(j \ "parse_success" === JBool(true)) assert(j \ "statement_identifier" === JString("SELECT")) assert(j \ "statement_code" === JInt(21)) - assert(j \ "statement_type" === - JString("direct select statement: multiple rows")) - assert(j \ "statement_class" === JString("SQL-data statement")) + assert(j \ "statement_type" === JNothing) + assert(j \ "statement_class" === JNothing) } test("CREATE TABLE and CTAS share CREATE TABLE code 77") { @@ -50,7 +49,7 @@ class ParseCommandResultSuite extends SparkFunSuite { val ctas = obj("CREATE TABLE t AS SELECT 1 AS a") assert(ctas \ "statement_identifier" === JString("CREATE TABLE")) assert(ctas \ "statement_code" === JInt(77)) - assert(ctas \ "as_subquery" === JBool(true)) + assert(ctas \ "as_subquery" === JNothing) } test("DML statement identifiers and codes") { @@ -113,13 +112,70 @@ class ParseCommandResultSuite extends SparkFunSuite { assert(j \ "error" \ "sqlState" === JString("42601")) } + test("parse errors expose line and position") { + val sql = + """SELECT * + |FROM t + |ORDER BY a + |CLUSTER BY b""".stripMargin + val error = obj(sql) \ "error" + assert(error \ "errorClass" === + JString("UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES")) + assert(error \ "line" === JInt(3)) + assert(error \ "position" === JInt(0)) + val context = (error \ "queryContext").asInstanceOf[JArray].arr.head + assert(context \ "startIndex" === JInt(17)) + } + + test("multiline script errors expose the script line") { + val sql = + """BEGIN + | SELECT 1; + | SELEC 2; + |END""".stripMargin + val error = obj(sql) \ "error" + assert(error \ "errorClass" === JString("PARSE_SYNTAX_ERROR")) + assert(error \ "line" === JInt(3)) + assert(error \ "position" === JInt(8)) + } + + test("SQL scripting validation errors expose their origin") { + val sql = + """BEGIN + | lbl_begin: BEGIN + | SELECT 1; + | END lbl_end; + |END""".stripMargin + val error = obj(sql) \ "error" + assert(error \ "errorClass" === JString("LABELS_MISMATCH")) + assert(error \ "line" === JInt(2)) + assert(error \ "position" === JInt(2)) + } + + test("parse-only validation returns error classes beyond syntax errors") { + val cases = Seq( + "" -> "PARSE_EMPTY_STATEMENT", + "USE bad-name" -> "INVALID_IDENTIFIER", + "WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c" -> + "DUPLICATED_CTE_NAMES", + "MERGE INTO target USING source ON target.id = source.id" -> + "MERGE_WITHOUT_WHEN", + "DROP FUNCTION catalog.schema.func" -> + "INVALID_SQL_SYNTAX.UNSUPPORTED_SQL_STATEMENT", + "SELECT 1 AS IDENTIFIER('alias.field')" -> + "IDENTIFIER_TOO_MANY_NAME_PARTS", + "SELECT DATE 'not-a-date'" -> "INVALID_TYPED_LITERAL") + cases.foreach { case (sql, errorClass) => + assert(obj(sql) \ "error" \ "errorClass" === JString(errorClass), sql) + } + } + test("Spark-only statements use negative implementation-defined codes") { val j = obj("CACHE TABLE t") assert(j \ "parse_success" === JBool(true)) assert(j \ "statement_identifier" === JString("CACHE TABLE")) assert(j \ "statement_code" === JInt(-1)) - assert(j \ "statement_class" === - JString("implementation-defined statement")) + assert(j \ "statement_class" === JNothing) } test("Table 39 standard code pairs are pinned") { @@ -207,7 +263,7 @@ class ParseCommandResultSuite extends SparkFunSuite { |WITH s AS (SELECT a FROM src) |SELECT a FROM s""".stripMargin val ctas = obj(ctasSql) - assert(ctas \ "as_subquery" === JBool(true)) + assert(ctas \ "statement_identifier" === JString("CREATE TABLE")) assert(tableRefs(ctasSql).contains(Seq("src"))) assert(tableRefs(ctasSql).contains(Seq("s"))) } @@ -230,6 +286,42 @@ class ParseCommandResultSuite extends SparkFunSuite { assert(funcRefs(sql).contains(Seq("max"))) } + test("functions are collected from expression positions and table-valued functions") { + val sql = + """SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( + | PARTITION BY lower(t.c) ORDER BY length(t.d)) + |FROM left_t t + |JOIN right_t r ON hash(t.id) = hash(r.id) + |JOIN LATERAL range(cast(t.n AS BIGINT)) rng + |WHERE startswith(t.c, 'x') + | AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id) + |GROUP BY coalesce(t.a, 0), t.b, t.c, t.d + |HAVING count_if(t.b > 0) > 0 + |ORDER BY greatest(t.a, 1)""".stripMargin + assert(tableRefs(sql) === Set(Seq("left_t"), Seq("right_t"), Seq("scalar_t"))) + assert(funcRefs(sql) === Set( + Seq("coalesce"), + Seq("sum"), + Seq("abs"), + Seq("lower"), + Seq("length"), + Seq("hash"), + Seq("range"), + Seq("startswith"), + Seq("max"), + Seq("count_if"), + Seq("greatest"))) + } + + test("functions in wrapped DDL expressions are collected") { + val sql = + """CREATE TABLE target ( + | created DATE DEFAULT current_date(), + | normalized STRING DEFAULT upper('x') + |)""".stripMargin + assert(funcRefs(sql) === Set(Seq("current_date"), Seq("upper"))) + } + test("MERGE collects target, source, and action-expression tables") { val sql = """MERGE INTO tgt t @@ -251,8 +343,6 @@ class ParseCommandResultSuite extends SparkFunSuite { assert(j \ "parse_success" === JBool(true)) assert(j \ "statement_identifier" === JString("BEGIN END")) assert(j \ "statement_code" === JInt(-22)) - assert(j \ "statement_class" === - JString("implementation-defined statement")) // Compound scripts have no single primary select list. assert(j \ "select_list" === JArray(Nil)) } @@ -322,4 +412,35 @@ class ParseCommandResultSuite extends SparkFunSuite { |END""".stripMargin assert(tableRefs(sql) === Set(Seq("cte_base"), Seq("c"))) } + + test("multiline script collects functions and tables from all control-flow branches") { + val sql = + """BEGIN + | CASE upper(:kind) + | WHEN lower('a') THEN + | SELECT max(a) FROM case_a; + | ELSE + | SELECT min(b) FROM case_else; + | END CASE; + | REPEAT + | INSERT INTO repeat_target + | SELECT transform(items, x -> abs(x)) FROM repeat_source; + | UNTIL EXISTS (SELECT 1 FROM repeat_done WHERE ready()) + | END REPEAT; + |END""".stripMargin + assert(tableRefs(sql) === Set( + Seq("case_a"), + Seq("case_else"), + Seq("repeat_target"), + Seq("repeat_source"), + Seq("repeat_done"))) + assert(funcRefs(sql) === Set( + Seq("upper"), + Seq("lower"), + Seq("max"), + Seq("min"), + Seq("transform"), + Seq("abs"), + Seq("ready"))) + } } diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/parse-command.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-command.sql.out index b43dc9abcd49..b73ab10c306b 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/parse-command.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-command.sql.out @@ -97,6 +97,96 @@ Project [parse_command(SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM out +- OneRowRelation +-- !query +SELECT parse_command( +'SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( + PARTITION BY lower(t.c) ORDER BY length(t.d)) + FROM left_t t + JOIN right_t r ON hash(t.id) = hash(r.id) + JOIN LATERAL range(cast(t.n AS BIGINT)) rng + WHERE startswith(t.c, ''x'') + AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id) + GROUP BY coalesce(t.a, 0), t.b, t.c, t.d + HAVING count_if(t.b > 0) > 0 + ORDER BY greatest(t.a, 1)') +-- !query analysis +Project [parse_command(SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( + PARTITION BY lower(t.c) ORDER BY length(t.d)) + FROM left_t t + JOIN right_t r ON hash(t.id) = hash(r.id) + JOIN LATERAL range(cast(t.n AS BIGINT)) rng + WHERE startswith(t.c, 'x') + AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id) + GROUP BY coalesce(t.a, 0), t.b, t.c, t.d + HAVING count_if(t.b > 0) > 0 + ORDER BY greatest(t.a, 1)) AS parse_command(SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( + PARTITION BY lower(t.c) ORDER BY length(t.d)) + FROM left_t t + JOIN right_t r ON hash(t.id) = hash(r.id) + JOIN LATERAL range(cast(t.n AS BIGINT)) rng + WHERE startswith(t.c, 'x') + AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id) + GROUP BY coalesce(t.a, 0), t.b, t.c, t.d + HAVING count_if(t.b > 0) > 0 + ORDER BY greatest(t.a, 1))#x] ++- OneRowRelation + + +-- !query +SELECT parse_command( +'MERGE INTO target t + USING ( + SELECT id, normalize_name(name) AS name + FROM source + WHERE is_valid(id) + ) s + ON hash(t.id) = hash(s.id) + WHEN MATCHED AND should_update(t.name, s.name) THEN + UPDATE SET name = coalesce(s.name, upper(t.name)) + WHEN NOT MATCHED THEN + INSERT (id, name) VALUES (s.id, lower(s.name))') +-- !query analysis +Project [parse_command(MERGE INTO target t + USING ( + SELECT id, normalize_name(name) AS name + FROM source + WHERE is_valid(id) + ) s + ON hash(t.id) = hash(s.id) + WHEN MATCHED AND should_update(t.name, s.name) THEN + UPDATE SET name = coalesce(s.name, upper(t.name)) + WHEN NOT MATCHED THEN + INSERT (id, name) VALUES (s.id, lower(s.name))) AS parse_command(MERGE INTO target t + USING ( + SELECT id, normalize_name(name) AS name + FROM source + WHERE is_valid(id) + ) s + ON hash(t.id) = hash(s.id) + WHEN MATCHED AND should_update(t.name, s.name) THEN + UPDATE SET name = coalesce(s.name, upper(t.name)) + WHEN NOT MATCHED THEN + INSERT (id, name) VALUES (s.id, lower(s.name)))#x] ++- OneRowRelation + + +-- !query +SELECT parse_command( +'CREATE TABLE defaults ( + created DATE DEFAULT current_date(), + normalized STRING DEFAULT upper(''x'') + )') +-- !query analysis +Project [parse_command(CREATE TABLE defaults ( + created DATE DEFAULT current_date(), + normalized STRING DEFAULT upper('x') + )) AS parse_command(CREATE TABLE defaults ( + created DATE DEFAULT current_date(), + normalized STRING DEFAULT upper('x') + ))#x] ++- OneRowRelation + + -- !query SELECT get_json_object(parse_command('SELEC FROM t'), '$.parse_success') -- !query analysis @@ -118,6 +208,172 @@ Project [get_json_object(parse_command(SELEC FROM t), $.error.sqlState) AS get_j +- OneRowRelation +-- !query +SELECT + get_json_object(parse_command( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b'), '$.error.errorClass') AS error_class, + get_json_object(parse_command( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b'), '$.error.line') AS line, + get_json_object(parse_command( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b'), '$.error.position') AS position, + get_json_object(parse_command( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b'), '$.error.queryContext[0].startIndex') AS start_index +-- !query analysis +Project [get_json_object(parse_command(SELECT * + FROM t + ORDER BY a + CLUSTER BY b), $.error.errorClass) AS error_class#x, get_json_object(parse_command(SELECT * + FROM t + ORDER BY a + CLUSTER BY b), $.error.line) AS line#x, get_json_object(parse_command(SELECT * + FROM t + ORDER BY a + CLUSTER BY b), $.error.position) AS position#x, get_json_object(parse_command(SELECT * + FROM t + ORDER BY a + CLUSTER BY b), $.error.queryContext[0].startIndex) AS start_index#x] ++- OneRowRelation + + +-- !query +SELECT get_json_object(parse_command(''), '$.error.errorClass') +-- !query analysis +Project [get_json_object(parse_command(), $.error.errorClass) AS get_json_object(parse_command(), $.error.errorClass)#x] ++- OneRowRelation + + +-- !query +SELECT get_json_object(parse_command('USE bad-name'), '$.error.errorClass') +-- !query analysis +Project [get_json_object(parse_command(USE bad-name), $.error.errorClass) AS get_json_object(parse_command(USE bad-name), $.error.errorClass)#x] ++- OneRowRelation + + +-- !query +SELECT get_json_object( + parse_command('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c'), + '$.error.errorClass') +-- !query analysis +Project [get_json_object(parse_command(WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c), $.error.errorClass) AS get_json_object(parse_command(WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c), $.error.errorClass)#x] ++- OneRowRelation + + +-- !query +SELECT get_json_object( + parse_command('MERGE INTO target USING source ON target.id = source.id'), + '$.error.errorClass') +-- !query analysis +Project [get_json_object(parse_command(MERGE INTO target USING source ON target.id = source.id), $.error.errorClass) AS get_json_object(parse_command(MERGE INTO target USING source ON target.id = source.id), $.error.errorClass)#x] ++- OneRowRelation + + +-- !query +SELECT get_json_object( + parse_command('DROP FUNCTION catalog.schema.func'), + '$.error.errorClass') +-- !query analysis +Project [get_json_object(parse_command(DROP FUNCTION catalog.schema.func), $.error.errorClass) AS get_json_object(parse_command(DROP FUNCTION catalog.schema.func), $.error.errorClass)#x] ++- OneRowRelation + + +-- !query +SELECT get_json_object( + parse_command('SELECT 1 AS IDENTIFIER(''alias.field'')'), + '$.error.errorClass') +-- !query analysis +Project [get_json_object(parse_command(SELECT 1 AS IDENTIFIER('alias.field')), $.error.errorClass) AS get_json_object(parse_command(SELECT 1 AS IDENTIFIER('alias.field')), $.error.errorClass)#x] ++- OneRowRelation + + +-- !query +SELECT get_json_object( + parse_command('SELECT DATE ''not-a-date'''), + '$.error.errorClass') +-- !query analysis +Project [get_json_object(parse_command(SELECT DATE 'not-a-date'), $.error.errorClass) AS get_json_object(parse_command(SELECT DATE 'not-a-date'), $.error.errorClass)#x] ++- OneRowRelation + + +-- !query +SELECT + get_json_object(parse_command( +'BEGIN + SELECT 1; + SELEC 2; + END'), '$.error.errorClass') AS error_class, + get_json_object(parse_command( +'BEGIN + SELECT 1; + SELEC 2; + END'), '$.error.line') AS line, + get_json_object(parse_command( +'BEGIN + SELECT 1; + SELEC 2; + END'), '$.error.position') AS position +-- !query analysis +Project [get_json_object(parse_command(BEGIN + SELECT 1; + SELEC 2; + END), $.error.errorClass) AS error_class#x, get_json_object(parse_command(BEGIN + SELECT 1; + SELEC 2; + END), $.error.line) AS line#x, get_json_object(parse_command(BEGIN + SELECT 1; + SELEC 2; + END), $.error.position) AS position#x] ++- OneRowRelation + + +-- !query +SELECT + get_json_object(parse_command( +'BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END'), '$.error.errorClass') AS error_class, + get_json_object(parse_command( +'BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END'), '$.error.line') AS line, + get_json_object(parse_command( +'BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END'), '$.error.position') AS position +-- !query analysis +Project [get_json_object(parse_command(BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END), $.error.errorClass) AS error_class#x, get_json_object(parse_command(BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END), $.error.line) AS line#x, get_json_object(parse_command(BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END), $.error.position) AS position#x] ++- OneRowRelation + + -- !query SELECT sql_text, parse_command(sql_text) FROM VALUES ('SELECT 1'), @@ -158,6 +414,133 @@ Project [parse_command(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT +- OneRowRelation +-- !query +SELECT + get_json_object(parse_command( +'BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + INSERT INTO error_log + SELECT format_string(''%s'', message) FROM error_source; + END; + + WITH prepared AS ( + SELECT id, normalize_name(name) AS name + FROM input_names + WHERE is_valid(id) + ) + INSERT INTO output_names + SELECT id, upper(name) FROM prepared; + + IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN + UPDATE update_target + SET value = coalesce((SELECT max(value) FROM update_source), 0) + WHERE should_update(id); + ELSE + DELETE FROM delete_target + WHERE id IN (SELECT id FROM delete_source WHERE expired(ts)); + END IF; + + FOR row AS + SELECT id FROM loop_source WHERE ready(id) + DO + SELECT audit(row.id), count(*) FROM loop_body; + END FOR; + END'), '$.table_references') AS table_references, + get_json_object(parse_command( +'BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + INSERT INTO error_log + SELECT format_string(''%s'', message) FROM error_source; + END; + + WITH prepared AS ( + SELECT id, normalize_name(name) AS name + FROM input_names + WHERE is_valid(id) + ) + INSERT INTO output_names + SELECT id, upper(name) FROM prepared; + + IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN + UPDATE update_target + SET value = coalesce((SELECT max(value) FROM update_source), 0) + WHERE should_update(id); + ELSE + DELETE FROM delete_target + WHERE id IN (SELECT id FROM delete_source WHERE expired(ts)); + END IF; + + FOR row AS + SELECT id FROM loop_source WHERE ready(id) + DO + SELECT audit(row.id), count(*) FROM loop_body; + END FOR; + END'), '$.function_references') AS function_references +-- !query analysis +Project [get_json_object(parse_command(BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + INSERT INTO error_log + SELECT format_string('%s', message) FROM error_source; + END; + + WITH prepared AS ( + SELECT id, normalize_name(name) AS name + FROM input_names + WHERE is_valid(id) + ) + INSERT INTO output_names + SELECT id, upper(name) FROM prepared; + + IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN + UPDATE update_target + SET value = coalesce((SELECT max(value) FROM update_source), 0) + WHERE should_update(id); + ELSE + DELETE FROM delete_target + WHERE id IN (SELECT id FROM delete_source WHERE expired(ts)); + END IF; + + FOR row AS + SELECT id FROM loop_source WHERE ready(id) + DO + SELECT audit(row.id), count(*) FROM loop_body; + END FOR; + END), $.table_references) AS table_references#x, get_json_object(parse_command(BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + INSERT INTO error_log + SELECT format_string('%s', message) FROM error_source; + END; + + WITH prepared AS ( + SELECT id, normalize_name(name) AS name + FROM input_names + WHERE is_valid(id) + ) + INSERT INTO output_names + SELECT id, upper(name) FROM prepared; + + IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN + UPDATE update_target + SET value = coalesce((SELECT max(value) FROM update_source), 0) + WHERE should_update(id); + ELSE + DELETE FROM delete_target + WHERE id IN (SELECT id FROM delete_source WHERE expired(ts)); + END IF; + + FOR row AS + SELECT id FROM loop_source WHERE ready(id) + DO + SELECT audit(row.id), count(*) FROM loop_body; + END FOR; + END), $.function_references) AS function_references#x] ++- OneRowRelation + + -- !query SELECT get_json_object(parse_command('BEGIN SELECT 1; END'), '$.statement_identifier') AS statement_identifier, diff --git a/sql/core/src/test/resources/sql-tests/inputs/parse-command.sql b/sql/core/src/test/resources/sql-tests/inputs/parse-command.sql index a637ce6ab161..313fc996c0ac 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/parse-command.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/parse-command.sql @@ -31,11 +31,130 @@ SELECT parse_command('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte' -- nested subqueries SELECT parse_command('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)'); +-- functions in projection, window, join, TVF, predicates, subquery, grouping, and ordering +SELECT parse_command( +'SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( + PARTITION BY lower(t.c) ORDER BY length(t.d)) + FROM left_t t + JOIN right_t r ON hash(t.id) = hash(r.id) + JOIN LATERAL range(cast(t.n AS BIGINT)) rng + WHERE startswith(t.c, ''x'') + AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id) + GROUP BY coalesce(t.a, 0), t.b, t.c, t.d + HAVING count_if(t.b > 0) > 0 + ORDER BY greatest(t.a, 1)'); + +-- functions and tables throughout a multiline MERGE +SELECT parse_command( +'MERGE INTO target t + USING ( + SELECT id, normalize_name(name) AS name + FROM source + WHERE is_valid(id) + ) s + ON hash(t.id) = hash(s.id) + WHEN MATCHED AND should_update(t.name, s.name) THEN + UPDATE SET name = coalesce(s.name, upper(t.name)) + WHEN NOT MATCHED THEN + INSERT (id, name) VALUES (s.id, lower(s.name))'); + +-- functions embedded in DDL column defaults +SELECT parse_command( +'CREATE TABLE defaults ( + created DATE DEFAULT current_date(), + normalized STRING DEFAULT upper(''x'') + )'); + -- syntax error: never throws; STANDARD error nested under parse_success=false SELECT get_json_object(parse_command('SELEC FROM t'), '$.parse_success'); SELECT get_json_object(parse_command('SELEC FROM t'), '$.error.errorClass'); SELECT get_json_object(parse_command('SELEC FROM t'), '$.error.sqlState'); +-- source location from a multiline parse-time validation error +SELECT + get_json_object(parse_command( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b'), '$.error.errorClass') AS error_class, + get_json_object(parse_command( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b'), '$.error.line') AS line, + get_json_object(parse_command( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b'), '$.error.position') AS position, + get_json_object(parse_command( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b'), '$.error.queryContext[0].startIndex') AS start_index; + +-- parse-only validation errors beyond PARSE_SYNTAX_ERROR +SELECT get_json_object(parse_command(''), '$.error.errorClass'); +SELECT get_json_object(parse_command('USE bad-name'), '$.error.errorClass'); +SELECT get_json_object( + parse_command('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c'), + '$.error.errorClass'); +SELECT get_json_object( + parse_command('MERGE INTO target USING source ON target.id = source.id'), + '$.error.errorClass'); +SELECT get_json_object( + parse_command('DROP FUNCTION catalog.schema.func'), + '$.error.errorClass'); +SELECT get_json_object( + parse_command('SELECT 1 AS IDENTIFIER(''alias.field'')'), + '$.error.errorClass'); +SELECT get_json_object( + parse_command('SELECT DATE ''not-a-date'''), + '$.error.errorClass'); + +-- location for an error inside a multiline script +--QUERY-DELIMITER-START +SELECT + get_json_object(parse_command( +'BEGIN + SELECT 1; + SELEC 2; + END'), '$.error.errorClass') AS error_class, + get_json_object(parse_command( +'BEGIN + SELECT 1; + SELEC 2; + END'), '$.error.line') AS line, + get_json_object(parse_command( +'BEGIN + SELECT 1; + SELEC 2; + END'), '$.error.position') AS position; +--QUERY-DELIMITER-END + +-- location for a SQL scripting semantic validation error +--QUERY-DELIMITER-START +SELECT + get_json_object(parse_command( +'BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END'), '$.error.errorClass') AS error_class, + get_json_object(parse_command( +'BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END'), '$.error.line') AS line, + get_json_object(parse_command( +'BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END'), '$.error.position') AS position; +--QUERY-DELIMITER-END + -- batch over a column of SQL text SELECT sql_text, parse_command(sql_text) FROM VALUES ('SELECT 1'), @@ -61,6 +180,74 @@ SELECT parse_command('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SEL SELECT parse_command('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END'); --QUERY-DELIMITER-END +-- Complex, genuinely multiline script. Extract collections to keep the +-- expected output focused on complete tree walking. +--QUERY-DELIMITER-START +SELECT + get_json_object(parse_command( +'BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + INSERT INTO error_log + SELECT format_string(''%s'', message) FROM error_source; + END; + + WITH prepared AS ( + SELECT id, normalize_name(name) AS name + FROM input_names + WHERE is_valid(id) + ) + INSERT INTO output_names + SELECT id, upper(name) FROM prepared; + + IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN + UPDATE update_target + SET value = coalesce((SELECT max(value) FROM update_source), 0) + WHERE should_update(id); + ELSE + DELETE FROM delete_target + WHERE id IN (SELECT id FROM delete_source WHERE expired(ts)); + END IF; + + FOR row AS + SELECT id FROM loop_source WHERE ready(id) + DO + SELECT audit(row.id), count(*) FROM loop_body; + END FOR; + END'), '$.table_references') AS table_references, + get_json_object(parse_command( +'BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + INSERT INTO error_log + SELECT format_string(''%s'', message) FROM error_source; + END; + + WITH prepared AS ( + SELECT id, normalize_name(name) AS name + FROM input_names + WHERE is_valid(id) + ) + INSERT INTO output_names + SELECT id, upper(name) FROM prepared; + + IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN + UPDATE update_target + SET value = coalesce((SELECT max(value) FROM update_source), 0) + WHERE should_update(id); + ELSE + DELETE FROM delete_target + WHERE id IN (SELECT id FROM delete_source WHERE expired(ts)); + END IF; + + FOR row AS + SELECT id FROM loop_source WHERE ready(id) + DO + SELECT audit(row.id), count(*) FROM loop_body; + END FOR; + END'), '$.function_references') AS function_references; +--QUERY-DELIMITER-END + -- extract key fields from a script for readable assertions --QUERY-DELIMITER-START SELECT diff --git a/sql/core/src/test/resources/sql-tests/results/parse-command.sql.out b/sql/core/src/test/resources/sql-tests/results/parse-command.sql.out index 256510ae3b21..340e7d4a9d70 100644 --- a/sql/core/src/test/resources/sql-tests/results/parse-command.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/parse-command.sql.out @@ -12,7 +12,7 @@ SELECT parse_command('SELECT a, b FROM t') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"statement_type":"direct select statement: multiple rows","statement_class":"SQL-data statement","table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"],"expression":"a"},{"name":["b"],"expression":"b"}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"],"expression":"a"},{"name":["b"],"expression":"b"}],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query @@ -20,7 +20,7 @@ SELECT parse_command('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"statement_type":"direct select statement: multiple rows","statement_class":"SQL-data statement","table_references":[["cat","ns","t1"],["t2"]],"function_references":[["db","my_func"],["count"]],"select_list":[{"name":[],"expression":"db.my_func(a)"},{"name":[],"expression":"count(b)"}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["cat","ns","t1"],["t2"]],"function_references":[["db","my_func"],["count"]],"select_list":[{"name":[],"expression":"db.my_func(a)"},{"name":[],"expression":"count(b)"}],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query @@ -28,7 +28,7 @@ SELECT parse_command('INSERT INTO t SELECT 1') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"statement_type":"insert statement","statement_class":"SQL-data change statement","table_references":[["t"]],"function_references":[],"select_list":[{"name":[],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"table_references":[["t"]],"function_references":[],"select_list":[{"name":[],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query @@ -36,7 +36,7 @@ SELECT parse_command('DELETE FROM t WHERE a = 1') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"DELETE WHERE","statement_code":19,"statement_type":"delete statement: searched","statement_class":"SQL-data change statement","table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"DELETE WHERE","statement_code":19,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query @@ -44,7 +44,7 @@ SELECT parse_command('UPDATE t SET a = 1 WHERE b = 2') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"UPDATE WHERE","statement_code":82,"statement_type":"update statement: searched","statement_class":"SQL-data change statement","table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"UPDATE WHERE","statement_code":82,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query @@ -52,7 +52,7 @@ SELECT parse_command('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELE -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"MERGE","statement_code":128,"statement_type":"merge statement","statement_class":"SQL-data change statement","table_references":[["t"],["s"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"MERGE","statement_code":128,"table_references":[["t"],["s"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query @@ -60,7 +60,7 @@ SELECT parse_command('CREATE TABLE t (a INT)') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"statement_type":"table definition","statement_class":"SQL-schema statement","table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query @@ -68,7 +68,7 @@ SELECT parse_command('CREATE TABLE t AS SELECT 1 AS a') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"statement_type":"table definition","statement_class":"SQL-schema statement","as_subquery":true,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query @@ -76,7 +76,7 @@ SELECT parse_command('DROP TABLE t') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"DROP TABLE","statement_code":32,"statement_type":"drop table statement","statement_class":"SQL-schema statement","table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"DROP TABLE","statement_code":32,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query @@ -84,7 +84,7 @@ SELECT parse_command('CACHE TABLE t') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"CACHE TABLE","statement_code":-1,"statement_type":"cache table","statement_class":"implementation-defined statement","table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"CACHE TABLE","statement_code":-1,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query @@ -92,7 +92,7 @@ SELECT parse_command('SELECT * FROM t WHERE a = :foo AND b = ?') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"statement_type":"direct select statement: multiple rows","statement_class":"SQL-data statement","table_references":[["t"]],"function_references":[],"select_list":[{"name":["*"],"expression":"unresolvedstar()"}],"parameter_markers":{"named":["foo"],"unnamed_count":1}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["*"],"expression":"unresolvedstar()"}],"parameter_markers":{"named":["foo"],"unnamed_count":1}} -- !query @@ -100,7 +100,7 @@ SELECT parse_command('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte' -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"statement_type":"direct select statement: multiple rows","statement_class":"SQL-data statement","table_references":[["hidden_base"],["cte"]],"function_references":[],"select_list":[{"name":["a"],"expression":"a"}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["hidden_base"],["cte"]],"function_references":[],"select_list":[{"name":["a"],"expression":"a"}],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query @@ -108,7 +108,78 @@ SELECT parse_command('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM oute -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"statement_type":"direct select statement: multiple rows","statement_class":"SQL-data statement","table_references":[["scalar_src"],["exists_src"],["outer_t"]],"function_references":[["max"]],"select_list":[{"name":["m"],"expression":"scalarsubquery()"},{"name":["t","a"],"expression":"t.a"}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["scalar_src"],["exists_src"],["outer_t"]],"function_references":[["max"]],"select_list":[{"name":["m"],"expression":"scalarsubquery()"},{"name":["t","a"],"expression":"t.a"}],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command( +'SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( + PARTITION BY lower(t.c) ORDER BY length(t.d)) + FROM left_t t + JOIN right_t r ON hash(t.id) = hash(r.id) + JOIN LATERAL range(cast(t.n AS BIGINT)) rng + WHERE startswith(t.c, ''x'') + AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id) + GROUP BY coalesce(t.a, 0), t.b, t.c, t.d + HAVING count_if(t.b > 0) > 0 + ORDER BY greatest(t.a, 1)') +-- !query schema +struct 0) > 0 + ORDER BY greatest(t.a, 1)):string> +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["scalar_t"],["left_t"],["right_t"]],"function_references":[["greatest"],["count_if"],["coalesce"],["sum"],["abs"],["lower"],["length"],["startswith"],["max"],["range"],["hash"]],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command( +'MERGE INTO target t + USING ( + SELECT id, normalize_name(name) AS name + FROM source + WHERE is_valid(id) + ) s + ON hash(t.id) = hash(s.id) + WHEN MATCHED AND should_update(t.name, s.name) THEN + UPDATE SET name = coalesce(s.name, upper(t.name)) + WHEN NOT MATCHED THEN + INSERT (id, name) VALUES (s.id, lower(s.name))') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"MERGE","statement_code":128,"table_references":[["target"],["source"]],"function_references":[["hash"],["should_update"],["coalesce"],["upper"],["lower"],["normalize_name"],["is_valid"]],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_command( +'CREATE TABLE defaults ( + created DATE DEFAULT current_date(), + normalized STRING DEFAULT upper(''x'') + )') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"table_references":[["defaults"]],"function_references":[["current_date"],["upper"]],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query @@ -135,6 +206,149 @@ struct 42601 +-- !query +SELECT + get_json_object(parse_command( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b'), '$.error.errorClass') AS error_class, + get_json_object(parse_command( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b'), '$.error.line') AS line, + get_json_object(parse_command( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b'), '$.error.position') AS position, + get_json_object(parse_command( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b'), '$.error.queryContext[0].startIndex') AS start_index +-- !query schema +struct +-- !query output +UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES 3 1 19 + + +-- !query +SELECT get_json_object(parse_command(''), '$.error.errorClass') +-- !query schema +struct +-- !query output +PARSE_EMPTY_STATEMENT + + +-- !query +SELECT get_json_object(parse_command('USE bad-name'), '$.error.errorClass') +-- !query schema +struct +-- !query output +INVALID_IDENTIFIER + + +-- !query +SELECT get_json_object( + parse_command('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c'), + '$.error.errorClass') +-- !query schema +struct +-- !query output +DUPLICATED_CTE_NAMES + + +-- !query +SELECT get_json_object( + parse_command('MERGE INTO target USING source ON target.id = source.id'), + '$.error.errorClass') +-- !query schema +struct +-- !query output +MERGE_WITHOUT_WHEN + + +-- !query +SELECT get_json_object( + parse_command('DROP FUNCTION catalog.schema.func'), + '$.error.errorClass') +-- !query schema +struct +-- !query output +INVALID_SQL_SYNTAX.UNSUPPORTED_SQL_STATEMENT + + +-- !query +SELECT get_json_object( + parse_command('SELECT 1 AS IDENTIFIER(''alias.field'')'), + '$.error.errorClass') +-- !query schema +struct +-- !query output +IDENTIFIER_TOO_MANY_NAME_PARTS + + +-- !query +SELECT get_json_object( + parse_command('SELECT DATE ''not-a-date'''), + '$.error.errorClass') +-- !query schema +struct +-- !query output +INVALID_TYPED_LITERAL + + +-- !query +SELECT + get_json_object(parse_command( +'BEGIN + SELECT 1; + SELEC 2; + END'), '$.error.errorClass') AS error_class, + get_json_object(parse_command( +'BEGIN + SELECT 1; + SELEC 2; + END'), '$.error.line') AS line, + get_json_object(parse_command( +'BEGIN + SELECT 1; + SELEC 2; + END'), '$.error.position') AS position +-- !query schema +struct +-- !query output +PARSE_SYNTAX_ERROR 3 9 + + +-- !query +SELECT + get_json_object(parse_command( +'BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END'), '$.error.errorClass') AS error_class, + get_json_object(parse_command( +'BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END'), '$.error.line') AS line, + get_json_object(parse_command( +'BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END'), '$.error.position') AS position +-- !query schema +struct +-- !query output +LABELS_MISMATCH 2 3 + + -- !query SELECT sql_text, parse_command(sql_text) FROM VALUES ('SELECT 1'), @@ -144,9 +358,9 @@ AS t(sql_text) -- !query schema struct -- !query output -CACHE TABLE t {"parse_success":true,"statement_identifier":"CACHE TABLE","statement_code":-1,"statement_type":"cache table","statement_class":"implementation-defined statement","table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -INSERT INTO t SELECT 1 {"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"statement_type":"insert statement","statement_class":"SQL-data change statement","table_references":[["t"]],"function_references":[],"select_list":[{"name":[],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} -SELECT 1 {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"statement_type":"direct select statement: multiple rows","statement_class":"SQL-data statement","table_references":[],"function_references":[],"select_list":[{"name":[],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} +CACHE TABLE t {"parse_success":true,"statement_identifier":"CACHE TABLE","statement_code":-1,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +INSERT INTO t SELECT 1 {"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"table_references":[["t"]],"function_references":[],"select_list":[{"name":[],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} +SELECT 1 {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[],"function_references":[],"select_list":[{"name":[],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query @@ -154,7 +368,7 @@ SELECT parse_command('BEGIN SELECT 1; END') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"statement_type":"begin end","statement_class":"implementation-defined statement","table_references":[],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query @@ -162,7 +376,7 @@ SELECT parse_command('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"statement_type":"begin end","statement_class":"implementation-defined statement","table_references":[["script_t"]],"function_references":[["count"]],"select_list":[],"parameter_markers":{"named":["p"],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["script_t"]],"function_references":[["count"]],"select_list":[],"parameter_markers":{"named":["p"],"unnamed_count":0}} -- !query @@ -170,7 +384,7 @@ SELECT parse_command('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SEL -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"statement_type":"begin end","statement_class":"implementation-defined statement","table_references":[["gate"],["dest"],["src_if"],["src_else"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["gate"],["dest"],["src_if"],["src_else"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query @@ -178,7 +392,77 @@ SELECT parse_command('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT I -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"statement_type":"begin end","statement_class":"implementation-defined statement","table_references":[["err_log"],["failing_row"],["main_t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["err_log"],["failing_row"],["main_t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT + get_json_object(parse_command( +'BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + INSERT INTO error_log + SELECT format_string(''%s'', message) FROM error_source; + END; + + WITH prepared AS ( + SELECT id, normalize_name(name) AS name + FROM input_names + WHERE is_valid(id) + ) + INSERT INTO output_names + SELECT id, upper(name) FROM prepared; + + IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN + UPDATE update_target + SET value = coalesce((SELECT max(value) FROM update_source), 0) + WHERE should_update(id); + ELSE + DELETE FROM delete_target + WHERE id IN (SELECT id FROM delete_source WHERE expired(ts)); + END IF; + + FOR row AS + SELECT id FROM loop_source WHERE ready(id) + DO + SELECT audit(row.id), count(*) FROM loop_body; + END FOR; + END'), '$.table_references') AS table_references, + get_json_object(parse_command( +'BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + INSERT INTO error_log + SELECT format_string(''%s'', message) FROM error_source; + END; + + WITH prepared AS ( + SELECT id, normalize_name(name) AS name + FROM input_names + WHERE is_valid(id) + ) + INSERT INTO output_names + SELECT id, upper(name) FROM prepared; + + IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN + UPDATE update_target + SET value = coalesce((SELECT max(value) FROM update_source), 0) + WHERE should_update(id); + ELSE + DELETE FROM delete_target + WHERE id IN (SELECT id FROM delete_source WHERE expired(ts)); + END IF; + + FOR row AS + SELECT id FROM loop_source WHERE ready(id) + DO + SELECT audit(row.id), count(*) FROM loop_body; + END FOR; + END'), '$.function_references') AS function_references +-- !query schema +struct +-- !query output +[["error_log"],["error_source"],["input_names"],["output_names"],["prepared"],["control_flags"],["update_source"],["update_target"],["delete_source"],["delete_target"],["loop_source"],["loop_body"]] [["format_string"],["normalize_name"],["is_valid"],["upper"],["enabled"],["coalesce"],["should_update"],["max"],["expired"],["ready"],["audit"],["count"]] -- !query From f1b9acc3f7aeb211775d5ee122417dc7dff45a06 Mon Sep 17 00:00:00 2001 From: srielau Date: Thu, 13 Aug 2026 09:51:55 +0000 Subject: [PATCH 6/9] [SPARK-58738][SQL] Fix parse_command expression metadata test failures Declare nullIntolerant since ParseCommand implements nullSafeEval, replace the elided example outputs with exact reproducible ones, and register the function in the expression schema golden file. --- .../spark/sql/catalyst/expressions/ParseCommand.scala | 8 +++++--- .../test/resources/sql-functions/sql-expression-schema.md | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala index 9e5acf6cf8ef..ba0a0f5ff47d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala @@ -46,9 +46,9 @@ import org.apache.spark.unsafe.types.UTF8String examples = """ Examples: > SELECT _FUNC_('SELECT a, b FROM t'); - {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,...} - > SELECT _FUNC_('SELEC'); - {"parse_success":false,"error":{"errorClass":"PARSE_SYNTAX_ERROR",...}} + {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"],"expression":"a"},{"name":["b"],"expression":"b"}],"parameter_markers":{"named":[],"unnamed_count":0}} + > SELECT get_json_object(_FUNC_('SELEC'), '$.error.errorClass'); + PARSE_SYNTAX_ERROR """, group = "misc_funcs", since = "4.3.0") @@ -62,6 +62,8 @@ case class ParseCommand(child: Expression) override def nullable: Boolean = true + override def nullIntolerant: Boolean = true + override def dataType: DataType = StringType override def inputTypes: Seq[AbstractDataType] = diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md index 7c389ed360ab..48b426f7685e 100644 --- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md +++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md @@ -274,6 +274,7 @@ | org.apache.spark.sql.catalyst.expressions.OctetLength | octet_length | SELECT octet_length('Spark SQL') | struct | | org.apache.spark.sql.catalyst.expressions.Or | or | SELECT true or false | struct<(true OR false):boolean> | | org.apache.spark.sql.catalyst.expressions.Overlay | overlay | SELECT overlay('Spark SQL' PLACING '_' FROM 6) | struct | +| org.apache.spark.sql.catalyst.expressions.ParseCommand | parse_command | SELECT parse_command('SELECT a, b FROM t') | struct | | org.apache.spark.sql.catalyst.expressions.ParseToDate | to_date | SELECT to_date('2009-07-30 04:17:52') | struct | | org.apache.spark.sql.catalyst.expressions.ParseToTimestamp | to_timestamp | SELECT to_timestamp('2016-12-31 00:12:00') | struct | | org.apache.spark.sql.catalyst.expressions.ParseToTimestampLTZExpressionBuilder | to_timestamp_ltz | SELECT to_timestamp_ltz('2016-12-31 00:12:00') | struct | From 617940c27dbce4755f7c90879bfd4e36fa8b7e4e Mon Sep 17 00:00:00 2001 From: srielau Date: Thu, 13 Aug 2026 12:01:38 +0000 Subject: [PATCH 7/9] [SPARK-58738][SQL] Rename to parse_sql and address review feedback Move parsing onto SparkSqlParser in sql/core, rename parse_command to parse_sql, drop select_list expression text, tighten error handling and classification, and keep table_references lineage-focused. --- .../catalyst/analysis/FunctionRegistry.scala | 1 - .../parser/ParseCommandResultSuite.scala | 446 ------------------ .../sql/catalyst/expressions/ParseSql.scala} | 41 +- .../sql/catalyst/parser/ParseSqlResult.scala} | 104 ++-- .../catalyst/parser/SqlStatementCodes.scala | 66 ++- .../internal/BaseSessionStateBuilder.scala | 9 +- .../sql-functions/sql-expression-schema.md | 2 +- ...arse-command.sql.out => parse-sql.sql.out} | 215 +++++---- .../{parse-command.sql => parse-sql.sql} | 107 ++--- ...arse-command.sql.out => parse-sql.sql.out} | 208 ++++---- .../catalyst/expressions/ParseSqlSuite.scala} | 17 +- .../catalyst/parser/ParseSqlResultSuite.scala | 105 +++++ 12 files changed, 555 insertions(+), 766 deletions(-) delete mode 100644 sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala rename sql/{catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala => core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala} (65%) rename sql/{catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala => core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala} (74%) rename sql/{catalyst => core}/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala (70%) rename sql/core/src/test/resources/sql-tests/analyzer-results/{parse-command.sql.out => parse-sql.sql.out} (51%) rename sql/core/src/test/resources/sql-tests/inputs/{parse-command.sql => parse-sql.sql} (61%) rename sql/core/src/test/resources/sql-tests/results/{parse-command.sql.out => parse-sql.sql.out} (61%) rename sql/{catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseCommandSuite.scala => core/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseSqlSuite.scala} (74%) create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index b6e30985ef0c..5eef532ab20b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -846,7 +846,6 @@ object FunctionRegistry { // misc functions expression[AssertTrue]("assert_true"), expressionBuilder("raise_error", RaiseErrorExpressionBuilder), - expression[ParseCommand]("parse_command"), expression[Crc32]("crc32"), expression[Md5]("md5"), expression[Uuid]("uuid"), diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala deleted file mode 100644 index fbfb7df6ffcb..000000000000 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResultSuite.scala +++ /dev/null @@ -1,446 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.catalyst.parser - -import org.json4s._ -import org.json4s.jackson.JsonMethods.parse - -import org.apache.spark.SparkFunSuite - -class ParseCommandResultSuite extends SparkFunSuite { - - private def json(sql: String): JValue = parse(ParseCommandResult.fromSql(sql)) - - private def obj(sql: String): JObject = json(sql).asInstanceOf[JObject] - - private def field(sql: String, name: String): JValue = - obj(sql).obj.find(_._1 == name).map(_._2).getOrElse(JNothing) - - test("SELECT classification uses Table 39 SELECT / code 21") { - val j = obj("SELECT a FROM t") - assert(j \ "parse_success" === JBool(true)) - assert(j \ "statement_identifier" === JString("SELECT")) - assert(j \ "statement_code" === JInt(21)) - assert(j \ "statement_type" === JNothing) - assert(j \ "statement_class" === JNothing) - } - - test("CREATE TABLE and CTAS share CREATE TABLE code 77") { - val create = obj("CREATE TABLE t (a INT)") - assert(create \ "statement_identifier" === JString("CREATE TABLE")) - assert(create \ "statement_code" === JInt(77)) - assert(create \ "as_subquery" === JNothing) - - val ctas = obj("CREATE TABLE t AS SELECT 1 AS a") - assert(ctas \ "statement_identifier" === JString("CREATE TABLE")) - assert(ctas \ "statement_code" === JInt(77)) - assert(ctas \ "as_subquery" === JNothing) - } - - test("DML statement identifiers and codes") { - assert(field("INSERT INTO t SELECT 1", "statement_identifier") === - JString("INSERT")) - assert(field("INSERT INTO t SELECT 1", "statement_code") === JInt(50)) - - assert(field("DELETE FROM t WHERE a = 1", "statement_identifier") === - JString("DELETE WHERE")) - assert(field("DELETE FROM t WHERE a = 1", "statement_code") === JInt(19)) - - assert(field("UPDATE t SET a = 1 WHERE b = 2", "statement_identifier") === - JString("UPDATE WHERE")) - assert(field("UPDATE t SET a = 1 WHERE b = 2", "statement_code") === JInt(82)) - - assert(field( - "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE", - "statement_identifier") === JString("MERGE")) - assert(field( - "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE", - "statement_code") === JInt(128)) - } - - test("table and function references are multipart sequences") { - val j = obj("SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2") - assert(j \ "table_references" === JArray(List( - JArray(List(JString("cat"), JString("ns"), JString("t1"))), - JArray(List(JString("t2"))) - ))) - val funcs = (j \ "function_references").asInstanceOf[JArray].arr - assert(funcs.contains( - JArray(List(JString("db"), JString("my_func"))))) - assert(funcs.contains(JArray(List(JString("count"))))) - } - - test("select_list exposes name parts and expression text") { - val j = obj("SELECT t.a AS x, b + 1 FROM t") - val list = (j \ "select_list").asInstanceOf[JArray].arr - assert(list.length === 2) - assert(list.head \ "name" === JArray(List(JString("x")))) - assert((list.head \ "expression").asInstanceOf[JString].s.contains("a")) - } - - test("named and unnamed parameter markers") { - val named = obj("SELECT * FROM t WHERE a = :foo AND b = :bar") - assert(named \ "parameter_markers" \ "named" === - JArray(List(JString("foo"), JString("bar")))) - assert(named \ "parameter_markers" \ "unnamed_count" === JInt(0)) - - val unnamed = obj("SELECT * FROM t WHERE a = ? AND b = ?") - assert(unnamed \ "parameter_markers" \ "named" === JArray(Nil)) - assert(unnamed \ "parameter_markers" \ "unnamed_count" === JInt(2)) - } - - test("syntax error returns STANDARD error JSON without throwing") { - val j = obj("SELEC FROM t") - assert(j \ "parse_success" === JBool(false)) - assert(j \ "error" \ "errorClass" === JString("PARSE_SYNTAX_ERROR")) - assert((j \ "error" \ "messageTemplate") != JNothing) - assert(j \ "error" \ "sqlState" === JString("42601")) - } - - test("parse errors expose line and position") { - val sql = - """SELECT * - |FROM t - |ORDER BY a - |CLUSTER BY b""".stripMargin - val error = obj(sql) \ "error" - assert(error \ "errorClass" === - JString("UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES")) - assert(error \ "line" === JInt(3)) - assert(error \ "position" === JInt(0)) - val context = (error \ "queryContext").asInstanceOf[JArray].arr.head - assert(context \ "startIndex" === JInt(17)) - } - - test("multiline script errors expose the script line") { - val sql = - """BEGIN - | SELECT 1; - | SELEC 2; - |END""".stripMargin - val error = obj(sql) \ "error" - assert(error \ "errorClass" === JString("PARSE_SYNTAX_ERROR")) - assert(error \ "line" === JInt(3)) - assert(error \ "position" === JInt(8)) - } - - test("SQL scripting validation errors expose their origin") { - val sql = - """BEGIN - | lbl_begin: BEGIN - | SELECT 1; - | END lbl_end; - |END""".stripMargin - val error = obj(sql) \ "error" - assert(error \ "errorClass" === JString("LABELS_MISMATCH")) - assert(error \ "line" === JInt(2)) - assert(error \ "position" === JInt(2)) - } - - test("parse-only validation returns error classes beyond syntax errors") { - val cases = Seq( - "" -> "PARSE_EMPTY_STATEMENT", - "USE bad-name" -> "INVALID_IDENTIFIER", - "WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c" -> - "DUPLICATED_CTE_NAMES", - "MERGE INTO target USING source ON target.id = source.id" -> - "MERGE_WITHOUT_WHEN", - "DROP FUNCTION catalog.schema.func" -> - "INVALID_SQL_SYNTAX.UNSUPPORTED_SQL_STATEMENT", - "SELECT 1 AS IDENTIFIER('alias.field')" -> - "IDENTIFIER_TOO_MANY_NAME_PARTS", - "SELECT DATE 'not-a-date'" -> "INVALID_TYPED_LITERAL") - cases.foreach { case (sql, errorClass) => - assert(obj(sql) \ "error" \ "errorClass" === JString(errorClass), sql) - } - } - - test("Spark-only statements use negative implementation-defined codes") { - val j = obj("CACHE TABLE t") - assert(j \ "parse_success" === JBool(true)) - assert(j \ "statement_identifier" === JString("CACHE TABLE")) - assert(j \ "statement_code" === JInt(-1)) - assert(j \ "statement_class" === JNothing) - } - - test("Table 39 standard code pairs are pinned") { - assert(SqlStatementCodes.Select.statementCode === 21) - assert(SqlStatementCodes.Insert.statementCode === 50) - assert(SqlStatementCodes.DeleteWhere.statementCode === 19) - assert(SqlStatementCodes.UpdateWhere.statementCode === 82) - assert(SqlStatementCodes.Merge.statementCode === 128) - assert(SqlStatementCodes.CreateTable.statementCode === 77) - assert(SqlStatementCodes.CreateView.statementCode === 84) - assert(SqlStatementCodes.DropTable.statementCode === 32) - assert(SqlStatementCodes.AlterTable.statementCode === 4) - assert(SqlStatementCodes.TruncateTable.statementCode === 139) - assert(SqlStatementCodes.Unrecognized.statementCode === 0) - assert(SqlStatementCodes.CacheTable.statementCode < 0) - assert(SqlStatementCodes.BeginEnd.statementCode === -22) - } - - private def tableRefs(sql: String): Set[Seq[String]] = - (obj(sql) \ "table_references").asInstanceOf[JArray].arr.map { - case JArray(parts) => parts.map(_.asInstanceOf[JString].s) - case other => fail(s"unexpected table_references entry: $other") - }.toSet - - private def funcRefs(sql: String): Set[Seq[String]] = - (obj(sql) \ "function_references").asInstanceOf[JArray].arr.map { - case JArray(parts) => parts.map(_.asInstanceOf[JString].s) - case other => fail(s"unexpected function_references entry: $other") - }.toSet - - test("CTE body tables are collected (UnresolvedWith innerChildren)") { - // CTE definitions are innerChildren, not children - easy to miss in walks. - val sql = - """WITH cte AS (SELECT a FROM hidden_base) - |SELECT a FROM cte""".stripMargin - assert(tableRefs(sql) === Set(Seq("hidden_base"), Seq("cte"))) - } - - test("multi-CTE chain and nested CTE definitions") { - val sql = - """WITH - | a AS (SELECT id FROM base_a), - | b AS ( - | WITH nested AS (SELECT id FROM base_nested) - | SELECT n.id FROM nested n JOIN base_b b ON n.id = b.id - | ) - |SELECT a.id, b.id FROM a JOIN b ON a.id = b.id""".stripMargin - assert(tableRefs(sql) === Set( - Seq("base_a"), - Seq("base_nested"), - Seq("nested"), - Seq("base_b"), - Seq("a"), - Seq("b"))) - } - - test("CTE with expression subqueries, functions, and parameters") { - val sql = - """WITH filtered AS ( - | SELECT upper(x) AS u, my_schema.my_udf(y) AS v - | FROM src - | WHERE z IN (SELECT z FROM lookup WHERE flag = :flag) - | AND EXISTS (SELECT 1 FROM probe WHERE probe.id = src.id) - |) - |SELECT u, count(v) FROM filtered WHERE u = ? GROUP BY u""".stripMargin - assert(tableRefs(sql) === Set( - Seq("src"), Seq("lookup"), Seq("probe"), Seq("filtered"))) - assert(funcRefs(sql).contains(Seq("upper"))) - assert(funcRefs(sql).contains(Seq("my_schema", "my_udf"))) - assert(funcRefs(sql).contains(Seq("count"))) - val params = obj(sql) \ "parameter_markers" - assert(params \ "named" === JArray(List(JString("flag")))) - assert(params \ "unnamed_count" === JInt(1)) - } - - test("WITH on INSERT / CTAS reaches CTE and target tables") { - val insertSql = - """INSERT INTO dest - |WITH s AS (SELECT a FROM src WHERE a > 0) - |SELECT a FROM s""".stripMargin - assert(tableRefs(insertSql) === Set(Seq("dest"), Seq("src"), Seq("s"))) - - val ctasSql = - """CREATE TABLE dest AS - |WITH s AS (SELECT a FROM src) - |SELECT a FROM s""".stripMargin - val ctas = obj(ctasSql) - assert(ctas \ "statement_identifier" === JString("CREATE TABLE")) - assert(tableRefs(ctasSql).contains(Seq("src"))) - assert(tableRefs(ctasSql).contains(Seq("s"))) - } - - test("nested FROM / scalar / EXISTS subqueries outside CTEs") { - val sql = - """SELECT - | (SELECT max(v) FROM scalar_src) AS m, - | t.a - |FROM outer_t t - |JOIN (SELECT id FROM join_src) j ON t.id = j.id - |WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id) - | AND t.a IN (SELECT a FROM in_src)""".stripMargin - assert(tableRefs(sql) === Set( - Seq("scalar_src"), - Seq("outer_t"), - Seq("join_src"), - Seq("exists_src"), - Seq("in_src"))) - assert(funcRefs(sql).contains(Seq("max"))) - } - - test("functions are collected from expression positions and table-valued functions") { - val sql = - """SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( - | PARTITION BY lower(t.c) ORDER BY length(t.d)) - |FROM left_t t - |JOIN right_t r ON hash(t.id) = hash(r.id) - |JOIN LATERAL range(cast(t.n AS BIGINT)) rng - |WHERE startswith(t.c, 'x') - | AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id) - |GROUP BY coalesce(t.a, 0), t.b, t.c, t.d - |HAVING count_if(t.b > 0) > 0 - |ORDER BY greatest(t.a, 1)""".stripMargin - assert(tableRefs(sql) === Set(Seq("left_t"), Seq("right_t"), Seq("scalar_t"))) - assert(funcRefs(sql) === Set( - Seq("coalesce"), - Seq("sum"), - Seq("abs"), - Seq("lower"), - Seq("length"), - Seq("hash"), - Seq("range"), - Seq("startswith"), - Seq("max"), - Seq("count_if"), - Seq("greatest"))) - } - - test("functions in wrapped DDL expressions are collected") { - val sql = - """CREATE TABLE target ( - | created DATE DEFAULT current_date(), - | normalized STRING DEFAULT upper('x') - |)""".stripMargin - assert(funcRefs(sql) === Set(Seq("current_date"), Seq("upper"))) - } - - test("MERGE collects target, source, and action-expression tables") { - val sql = - """MERGE INTO tgt t - |USING (SELECT id FROM src) s - |ON t.id = s.id - |WHEN MATCHED AND t.flag IN (SELECT flag FROM flags) THEN - | UPDATE SET t.v = (SELECT v FROM vals WHERE vals.id = t.id) - |WHEN NOT MATCHED THEN - | INSERT (id) VALUES (s.id)""".stripMargin - val refs = tableRefs(sql) - assert(refs.contains(Seq("tgt"))) - assert(refs.contains(Seq("src"))) - assert(refs.contains(Seq("flags"))) - assert(refs.contains(Seq("vals"))) - } - - test("BEGIN END script classification uses Spark code -22") { - val j = obj("BEGIN SELECT 1; END") - assert(j \ "parse_success" === JBool(true)) - assert(j \ "statement_identifier" === JString("BEGIN END")) - assert(j \ "statement_code" === JInt(-22)) - // Compound scripts have no single primary select list. - assert(j \ "select_list" === JArray(Nil)) - } - - test("BEGIN END walks SingleStatement.parsedPlan for tables and functions") { - // SingleStatement.children skips the statement root (e.g. Project), so a - // naive collectWithSubqueries misses project-list functions. - val sql = - """BEGIN - | SELECT count(a), upper(b) FROM script_t WHERE c = :p; - |END""".stripMargin - assert(tableRefs(sql) === Set(Seq("script_t"))) - assert(funcRefs(sql).contains(Seq("count"))) - assert(funcRefs(sql).contains(Seq("upper"))) - assert(obj(sql) \ "parameter_markers" \ "named" === - JArray(List(JString("p")))) - } - - test("BEGIN END with IF / WHILE / FOR collects nested statement refs") { - val sql = - """BEGIN - | IF (SELECT flag FROM gate) THEN - | INSERT INTO dest SELECT * FROM src_if; - | ELSE - | DELETE FROM src_else WHERE id IN (SELECT id FROM doomed); - | END IF; - | WHILE (SELECT cont FROM ctrl) DO - | UPDATE tgt SET v = 1 WHERE id IN (SELECT id FROM while_src); - | END WHILE; - | FOR x AS SELECT id FROM for_src DO - | SELECT my_udf(id) FROM for_body WHERE id = x.id; - | END FOR; - |END""".stripMargin - val refs = tableRefs(sql) - assert(refs === Set( - Seq("gate"), - Seq("dest"), - Seq("src_if"), - Seq("src_else"), - Seq("doomed"), - Seq("ctrl"), - Seq("tgt"), - Seq("while_src"), - Seq("for_src"), - Seq("for_body"))) - assert(funcRefs(sql).contains(Seq("my_udf"))) - } - - test("BEGIN END exception handler body tables are collected") { - val sql = - """BEGIN - | DECLARE EXIT HANDLER FOR SQLEXCEPTION - | BEGIN - | INSERT INTO err_log SELECT * FROM failing_row; - | END; - | SELECT a FROM main_t; - |END""".stripMargin - assert(tableRefs(sql) === Set( - Seq("err_log"), Seq("failing_row"), Seq("main_t"))) - } - - test("BEGIN END with CTE inside script body") { - val sql = - """BEGIN - | WITH c AS (SELECT a FROM cte_base) - | SELECT a FROM c; - |END""".stripMargin - assert(tableRefs(sql) === Set(Seq("cte_base"), Seq("c"))) - } - - test("multiline script collects functions and tables from all control-flow branches") { - val sql = - """BEGIN - | CASE upper(:kind) - | WHEN lower('a') THEN - | SELECT max(a) FROM case_a; - | ELSE - | SELECT min(b) FROM case_else; - | END CASE; - | REPEAT - | INSERT INTO repeat_target - | SELECT transform(items, x -> abs(x)) FROM repeat_source; - | UNTIL EXISTS (SELECT 1 FROM repeat_done WHERE ready()) - | END REPEAT; - |END""".stripMargin - assert(tableRefs(sql) === Set( - Seq("case_a"), - Seq("case_else"), - Seq("repeat_target"), - Seq("repeat_source"), - Seq("repeat_done"))) - assert(funcRefs(sql) === Set( - Seq("upper"), - Seq("lower"), - Seq("max"), - Seq("min"), - Seq("transform"), - Seq("abs"), - Seq("ready"))) - } -} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala similarity index 65% rename from sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala rename to sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala index ba0a0f5ff47d..25f26c0a91b7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseCommand.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala @@ -17,27 +17,29 @@ package org.apache.spark.sql.catalyst.expressions +import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, FunctionRegistryBase} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -import org.apache.spark.sql.catalyst.parser.ParseCommandResult +import org.apache.spark.sql.catalyst.parser.ParseSqlResult import org.apache.spark.sql.internal.types.StringTypeWithCollation import org.apache.spark.sql.types.{AbstractDataType, DataType, StringType} import org.apache.spark.unsafe.types.UTF8String /** * Parses a SQL statement string and returns a compact JSON description of the - * unresolved statement (identifier/code, references, select list, parameters), - * or a STANDARD-format error object when the statement does not parse. + * unresolved statement (identifier/code, lineage references, select-list names, + * parameters), or a STANDARD-format error object when the statement does not + * parse. * - * Designed for batch evaluation over DataFrames of SQL text; never throws on - * syntax errors so a single bad row does not fail the query. + * Designed for batch evaluation over DataFrames of SQL text. User-facing parse + * errors become JSON; unexpected internal failures propagate. */ // scalastyle:off line.size.limit @ExpressionDescription( usage = """_FUNC_(sqlStmt) - Parses `sqlStmt` and returns a JSON string describing the statement (parse success, Table 39 statement identifier/code, table and function - references, select-list columns, and parameter markers). On syntax error returns - JSON with `parse_success` false, source location, and a nested STANDARD error object - instead of throwing.""", + references for lineage, select-list column names, and parameter markers). On + syntax / parse error returns JSON with `parse_success` false, source location, + and a nested STANDARD error object instead of throwing.""", arguments = """ Arguments: * sqlStmt - A SQL statement string to parse. @@ -46,19 +48,19 @@ import org.apache.spark.unsafe.types.UTF8String examples = """ Examples: > SELECT _FUNC_('SELECT a, b FROM t'); - {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"],"expression":"a"},{"name":["b"],"expression":"b"}],"parameter_markers":{"named":[],"unnamed_count":0}} + {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"]},{"name":["b"]}],"parameter_markers":{"named":[],"unnamed_count":0}} > SELECT get_json_object(_FUNC_('SELEC'), '$.error.errorClass'); PARSE_SYNTAX_ERROR """, group = "misc_funcs", - since = "4.3.0") + since = "5.0.0") // scalastyle:on line.size.limit -case class ParseCommand(child: Expression) +case class ParseSql(child: Expression) extends UnaryExpression with ImplicitCastInputTypes with CodegenFallback { - override def prettyName: String = "parse_command" + override def prettyName: String = "parse_sql" override def nullable: Boolean = true @@ -71,9 +73,20 @@ case class ParseCommand(child: Expression) override def nullSafeEval(input: Any): Any = { val sql = input.asInstanceOf[UTF8String].toString - UTF8String.fromString(ParseCommandResult.fromSql(sql)) + UTF8String.fromString(ParseSqlResult.fromSql(sql)) } - override protected def withNewChildInternal(newChild: Expression): ParseCommand = + override protected def withNewChildInternal(newChild: Expression): ParseSql = copy(child = newChild) } + +object ParseSql { + /** Register the builtin with a session function registry. */ + def register(registry: FunctionRegistry): Unit = { + val (info, builder) = FunctionRegistryBase.build[ParseSql]("parse_sql", Some("5.0.0")) + registry.registerFunction( + FunctionRegistry.builtinFunctionIdentifier("parse_sql"), + info, + builder) + } +} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala similarity index 74% rename from sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala rename to sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala index 327b8e9a3f27..892e908a458c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseCommandResult.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.catalyst.parser import scala.collection.mutable -import scala.util.control.NonFatal import org.json4s._ import org.json4s.jackson.JsonMethods.{compact, parse => parseJson, render} @@ -29,42 +28,40 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.sql.exceptions.SqlScriptingException +import org.apache.spark.sql.execution.SparkSqlParser +import org.apache.spark.sql.execution.command.{CreateViewCommand, DescribeQueryCommand, ExplainCommand} /** * Parses a SQL statement string and returns a compact JSON description of the * unresolved plan (parse-only; no catalog resolution). * + * Uses [[SparkSqlParser]] so coverage matches the production session parser + * (EXPLAIN / SET / ADD JAR / temp views / etc.). + * * On success the JSON includes the statement identifier/code (ISO/IEC - * 9075-2:2023 Table 39), table/function references, select-list items, and + * 9075-2:2023 Table 39), table/function references, select-list names, and * parameter markers. On parse failure it returns `parse_success: false` with - * source location and a nested STANDARD-format error object, and does not throw. + * source location and a nested STANDARD-format error object, and does not + * throw. Unexpected / internal failures propagate so the function fails. */ -object ParseCommandResult { +object ParseSqlResult { - private val parser: ThreadLocal[CatalystSqlParser] = - ThreadLocal.withInitial(() => new CatalystSqlParser()) + private val parser: ThreadLocal[SparkSqlParser] = + ThreadLocal.withInitial(() => new SparkSqlParser()) - /** Parse `sql` and render the JSON result string. Never throws for bad SQL. */ + /** Parse `sql` and render the JSON result string. */ def fromSql(sql: String): String = { try { val plan = parser.get().parsePlan(sql) fromPlan(plan) } catch { + // User-facing parse / scripting failures become JSON; internal errors fail. case e: ParseException => errorJson(e) + case e: SqlScriptingException => + errorJson(e) case e: SparkThrowable with Throwable => errorJson(e) - case NonFatal(e) => - // Unexpected failures still must not fail a batch row. - compact(render(JObject( - "parse_success" -> JBool(false), - "error" -> JObject( - "errorClass" -> JString("LEGACY"), - "messageParameters" -> JObject( - "message" -> JString(Option(e.getMessage).getOrElse(e.toString)) - ) - ) - ))) } } @@ -148,20 +145,46 @@ object ParseCommandResult { c.handlers.foreach(h => foreachPlanDeep(h)(f)) case s: SimpleCaseStatement => s.elseBody.foreach(b => foreachPlanDeep(b)(f)) + case ExplainCommand(logicalPlan, _) => + foreachPlanDeep(logicalPlan)(f) + case DescribeQueryCommand(_, queryPlan) => + foreachPlanDeep(queryPlan)(f) case _ => } } } /** - * Collect multipart table/view identifiers as written in the SQL. - * Deduplicates while preserving first-seen order. + * Collect multipart table/view identifiers for lineage (as written in the + * SQL). CTE definition names and correlation aliases are omitted; tables + * referenced inside CTE bodies are still included. Deduplicates while + * preserving first-seen order. */ - def collectTableReferences(plan: LogicalPlan): Seq[Seq[String]] = { + private def collectTableReferences(plan: LogicalPlan): Seq[Seq[String]] = { val seen = mutable.LinkedHashSet.empty[Seq[String]] + val cteNames = mutable.HashSet.empty[String] + + def addCteNames(w: UnresolvedWith): Unit = { + w.cteRelations.foreach { case (name, _, _) => + cteNames += name.toLowerCase(java.util.Locale.ROOT) + } + } + + def isCteName(parts: Seq[String]): Boolean = parts match { + case Seq(name) => cteNames.contains(name.toLowerCase(java.util.Locale.ROOT)) + case _ => false + } + def add(parts: Seq[String]): Unit = { - if (parts.nonEmpty) seen += parts + if (parts.nonEmpty && !isCteName(parts)) seen += parts } + + // First pass: gather CTE names in scope (including nested). + foreachPlanDeep(plan) { + case w: UnresolvedWith => addCteNames(w) + case _ => + } + foreachPlanDeep(plan) { case u: UnresolvedRelation => add(u.multipartIdentifier) case u: UnresolvedTable => add(u.multipartIdentifier) @@ -174,7 +197,7 @@ object ParseCommandResult { } /** Collect multipart function names, including table-valued functions. */ - def collectFunctionReferences(plan: LogicalPlan): Seq[Seq[String]] = { + private def collectFunctionReferences(plan: LogicalPlan): Seq[Seq[String]] = { val seen = mutable.LinkedHashSet.empty[Seq[String]] def add(parts: Seq[String]): Unit = { if (parts.nonEmpty) seen += parts @@ -194,10 +217,10 @@ object ParseCommandResult { } /** - * Collect the primary select list as `{name, expression}` objects. - * Empty for non-query statements without a projected query body. + * Collect the primary select list as `{name}` objects (multipart name + * parts only). Empty for non-query statements without a projected query body. */ - def collectSelectList(plan: LogicalPlan): Seq[JObject] = { + private def collectSelectList(plan: LogicalPlan): Seq[JObject] = { val query = primaryQueryPlan(plan) val named: Seq[NamedExpression] = query match { case p: Project => p.projectList @@ -213,32 +236,27 @@ object ParseCommandResult { primaryQueryPlan(query) case c: CreateTableAsSelect => primaryQueryPlan(c.query) case r: ReplaceTableAsSelect => primaryQueryPlan(r.query) + case c: CreateView => primaryQueryPlan(c.query) + case c: CreateViewCommand => primaryQueryPlan(c.plan) + case c: CacheTableAsSelect => primaryQueryPlan(c.plan) + case ExplainCommand(logicalPlan, _) => primaryQueryPlan(logicalPlan) + case DescribeQueryCommand(_, queryPlan) => primaryQueryPlan(queryPlan) case SubqueryAlias(_, child) => primaryQueryPlan(child) case other => other } private def selectListItem(ne: NamedExpression): JObject = ne match { - case Alias(child, name) => - JObject( - "name" -> partsToJArray(Seq(name)), - "expression" -> JString(child.sql)) - case u: UnresolvedAlias => - JObject( - "name" -> partsToJArray(Nil), - "expression" -> JString(u.child.sql)) + case Alias(_, name) => + JObject("name" -> partsToJArray(Seq(name))) + case _: UnresolvedAlias => + JObject("name" -> partsToJArray(Nil)) case s: UnresolvedStar => val name = s.target.map(_ :+ "*").getOrElse(Seq("*")) - JObject( - "name" -> partsToJArray(name), - "expression" -> JString(s.sql)) + JObject("name" -> partsToJArray(name)) case a: UnresolvedAttribute => - JObject( - "name" -> partsToJArray(a.nameParts), - "expression" -> JString(a.sql)) + JObject("name" -> partsToJArray(a.nameParts)) case other => - JObject( - "name" -> partsToJArray(Seq(other.name)), - "expression" -> JString(other.sql)) + JObject("name" -> partsToJArray(Seq(other.name))) } private def parameterMarkersJson(plan: LogicalPlan): JObject = { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala similarity index 70% rename from sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala rename to sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala index fd8126028967..1a62d6a30352 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala @@ -17,8 +17,10 @@ package org.apache.spark.sql.catalyst.parser -import org.apache.spark.sql.catalyst.analysis.UnresolvedExecuteImmediate +import org.apache.spark.sql.catalyst.analysis.{UnresolvedExecuteImmediate, UnresolvedHaving} import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.execution.command._ +import org.apache.spark.sql.execution.datasources.{CreateTempViewUsing, RefreshResource} /** * Classification of a parsed SQL statement using ISO/IEC 9075-2:2023 Table 39, @@ -38,6 +40,9 @@ case class SqlStatementClassification( * Spark-only statements use the standard's implementation-defined escape hatch: * a product-specific identifier and a distinct negative code. Codes are * append-only and must never be renumbered. + * + * Unknown plans map to [[Unrecognized]] (empty identifier, code 0). Query + * shapes are allowlisted; unknown non-commands are not assumed to be SELECT. */ object SqlStatementCodes { @@ -91,6 +96,22 @@ object SqlStatementCodes { val CommentOnTable: SqlStatementClassification = spark("COMMENT ON TABLE", -21) // SQL/PSM-style scripting (9075-4); not in Foundation Table 39. val BeginEnd: SqlStatementClassification = spark("BEGIN END", -22) + // SparkSqlParser-only session / resource commands (append-only). + val Explain: SqlStatementClassification = spark("EXPLAIN", -23) + val Set: SqlStatementClassification = spark("SET", -24) + val Reset: SqlStatementClassification = spark("RESET", -25) + val AddJar: SqlStatementClassification = spark("ADD JAR", -26) + val AddFile: SqlStatementClassification = spark("ADD FILE", -27) + val AddArchive: SqlStatementClassification = spark("ADD ARCHIVE", -28) + val ListJar: SqlStatementClassification = spark("LIST JAR", -29) + val ListFile: SqlStatementClassification = spark("LIST FILE", -30) + val ClearCache: SqlStatementClassification = spark("CLEAR CACHE", -31) + val RefreshResourceCmd: SqlStatementClassification = spark("REFRESH RESOURCE", -32) + val DescribeQuery: SqlStatementClassification = spark("DESCRIBE QUERY", -33) + val ShowCatalogs: SqlStatementClassification = spark("SHOW CATALOGS", -34) + val ShowCurrentNamespace: SqlStatementClassification = + spark("SHOW CURRENT NAMESPACE", -35) + val SetCatalog: SqlStatementClassification = spark("SET CATALOG", -36) private def spark(identifier: String, code: Int): SqlStatementClassification = { assert(code < 0, s"Spark statement codes must be negative, got $code") @@ -107,15 +128,18 @@ object SqlStatementCodes { case _: MergeIntoTable => Merge case _: CreateTableAsSelect | _: ReplaceTableAsSelect => CreateTable case _: CreateTable | _: CreateTableLike | _: ReplaceTable => CreateTable - case _: CreateView => CreateView + case _: CreateView | _: CreateViewCommand | _: CreateTempViewUsing => CreateView case _: DropTable => DropTable case _: DropView => DropView case _: CreateNamespace => CreateSchema case _: DropNamespace => DropSchema - case _: SetCatalogAndNamespace => SetSchema + case _: SetCatalogAndNamespace | _: SetNamespaceCommand => SetSchema + case _: SetCatalogCommand => SetCatalog case _: TruncateTable => TruncateTable - case _: CreateFunction => CreateRoutine - case _: DropFunction => DropRoutine + case _: CreateFunction | _: CreateFunctionCommand | + _: CreateUserDefinedFunction | _: CreateUserDefinedFunctionCommand => + CreateRoutine + case _: DropFunction | _: DropFunctionCommand => DropRoutine case _: UnresolvedExecuteImmediate => ExecuteImmediate case _: Call => Call case _: CommentOnTable => CommentOnTable @@ -127,6 +151,7 @@ object SqlStatementCodes { case _: ShowTables | _: ShowTablesExtended => ShowTables case _: DescribeRelation | _: DescribeTablePartition | _: DescribeColumn => DescribeTable + case _: DescribeQueryCommand => DescribeQuery case _: AnalyzeTable | _: AnalyzeTables | _: AnalyzeColumn => AnalyzeTable case _: CreateVariable => DeclareVariable case _: SetVariable => SetVariable @@ -141,7 +166,36 @@ object SqlStatementCodes { case _: ShowViews => ShowViews case _: RefreshFunction => RefreshFunction case _: CommentOnNamespace => CommentOnNamespace + case _: ExplainCommand => Explain + case _: SetCommand => Set + case _: ResetCommand => Reset + case _: AddJarsCommand => AddJar + case _: AddFilesCommand => AddFile + case _: AddArchivesCommand => AddArchive + case _: ListJarsCommand => ListJar + case _: ListFilesCommand => ListFile + case ClearCacheCommand => ClearCache + case _: RefreshResource => RefreshResourceCmd + case _: ShowCatalogsCommand => ShowCatalogs + case _: ShowCurrentNamespaceCommand => ShowCurrentNamespace case _: Command => Unrecognized - case _ => Select + case p if isQueryPlan(p) => Select + case _ => Unrecognized + } + + /** + * Allowlisted query-shaped plans. Unknown non-command plans are not assumed + * to be SELECT. + */ + private def isQueryPlan(plan: LogicalPlan): Boolean = plan match { + case _: Project | _: Aggregate | _: Distinct | _: Filter | _: Sort | + _: GlobalLimit | _: LocalLimit | _: Join | _: Union | _: Except | + _: Intersect | _: SubqueryAlias | _: Repartition | + _: RepartitionByExpression | _: Sample | _: Range | + _: OneRowRelation | _: LocalRelation | _: Deduplicate | + _: Expand | _: Generate | _: Window | _: Tail | _: Offset | + _: LateralJoin | _: UnresolvedHaving | _: CollectMetrics | + _: WithCTE => true + case _ => false } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala index 52c6821d0001..1ee7699fff42 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala @@ -22,7 +22,7 @@ import org.apache.spark.sql.artifact.ArtifactManager import org.apache.spark.sql.catalyst.analysis.{Analyzer, EvalSubqueriesForTimeTravel, FunctionRegistry, InvokeProcedures, ReplaceCharWithVarchar, ResolveDataSource, ResolveEventTimeWatermark, ResolveExecuteImmediate, ResolveMetricView, ResolveSessionCatalog, ResolveSetCatalogCommand, ResolveTranspose, TableFunctionRegistry} import org.apache.spark.sql.catalyst.analysis.resolver.ResolverExtension import org.apache.spark.sql.catalyst.catalog.{FunctionExpressionBuilder, SessionCatalog} -import org.apache.spark.sql.catalyst.expressions.{Expression, ExtractSemiStructuredFields} +import org.apache.spark.sql.catalyst.expressions.{Expression, ExtractSemiStructuredFields, ParseSql} import org.apache.spark.sql.catalyst.normalizer.NormalizeCTEIds import org.apache.spark.sql.catalyst.optimizer.Optimizer import org.apache.spark.sql.catalyst.parser.ParserInterface @@ -96,7 +96,12 @@ abstract class BaseSessionStateBuilder( */ protected lazy val functionRegistry: FunctionRegistry = { parentState.map(_.functionRegistry.clone()) - .getOrElse(extensions.registerFunctions(FunctionRegistry.builtin.clone())) + .getOrElse { + val registry = FunctionRegistry.builtin.clone() + // sql/core-only builtins that need SparkSqlParser. + ParseSql.register(registry) + extensions.registerFunctions(registry) + } } /** diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md index 48b426f7685e..8a4d226fd94a 100644 --- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md +++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md @@ -274,7 +274,7 @@ | org.apache.spark.sql.catalyst.expressions.OctetLength | octet_length | SELECT octet_length('Spark SQL') | struct | | org.apache.spark.sql.catalyst.expressions.Or | or | SELECT true or false | struct<(true OR false):boolean> | | org.apache.spark.sql.catalyst.expressions.Overlay | overlay | SELECT overlay('Spark SQL' PLACING '_' FROM 6) | struct | -| org.apache.spark.sql.catalyst.expressions.ParseCommand | parse_command | SELECT parse_command('SELECT a, b FROM t') | struct | +| org.apache.spark.sql.catalyst.expressions.ParseSql | parse_sql | SELECT parse_sql('SELECT a, b FROM t') | struct | | org.apache.spark.sql.catalyst.expressions.ParseToDate | to_date | SELECT to_date('2009-07-30 04:17:52') | struct | | org.apache.spark.sql.catalyst.expressions.ParseToTimestamp | to_timestamp | SELECT to_timestamp('2016-12-31 00:12:00') | struct | | org.apache.spark.sql.catalyst.expressions.ParseToTimestampLTZExpressionBuilder | to_timestamp_ltz | SELECT to_timestamp_ltz('2016-12-31 00:12:00') | struct | diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/parse-command.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out similarity index 51% rename from sql/core/src/test/resources/sql-tests/analyzer-results/parse-command.sql.out rename to sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out index b73ab10c306b..104c4d5660bc 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/parse-command.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out @@ -1,104 +1,104 @@ -- Automatically generated by SQLQueryTestSuite -- !query -SELECT parse_command(NULL) +SELECT parse_sql(NULL) -- !query analysis -Project [parse_command(cast(null as string)) AS parse_command(NULL)#x] +Project [parse_sql(cast(null as string)) AS parse_sql(NULL)#x] +- OneRowRelation -- !query -SELECT parse_command('SELECT a, b FROM t') +SELECT parse_sql('SELECT a, b FROM t') -- !query analysis -Project [parse_command(SELECT a, b FROM t) AS parse_command(SELECT a, b FROM t)#x] +Project [parse_sql(SELECT a, b FROM t) AS parse_sql(SELECT a, b FROM t)#x] +- OneRowRelation -- !query -SELECT parse_command('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2') +SELECT parse_sql('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2') -- !query analysis -Project [parse_command(SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2) AS parse_command(SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2)#x] +Project [parse_sql(SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2) AS parse_sql(SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2)#x] +- OneRowRelation -- !query -SELECT parse_command('INSERT INTO t SELECT 1') +SELECT parse_sql('INSERT INTO t SELECT 1') -- !query analysis -Project [parse_command(INSERT INTO t SELECT 1) AS parse_command(INSERT INTO t SELECT 1)#x] +Project [parse_sql(INSERT INTO t SELECT 1) AS parse_sql(INSERT INTO t SELECT 1)#x] +- OneRowRelation -- !query -SELECT parse_command('DELETE FROM t WHERE a = 1') +SELECT parse_sql('DELETE FROM t WHERE a = 1') -- !query analysis -Project [parse_command(DELETE FROM t WHERE a = 1) AS parse_command(DELETE FROM t WHERE a = 1)#x] +Project [parse_sql(DELETE FROM t WHERE a = 1) AS parse_sql(DELETE FROM t WHERE a = 1)#x] +- OneRowRelation -- !query -SELECT parse_command('UPDATE t SET a = 1 WHERE b = 2') +SELECT parse_sql('UPDATE t SET a = 1 WHERE b = 2') -- !query analysis -Project [parse_command(UPDATE t SET a = 1 WHERE b = 2) AS parse_command(UPDATE t SET a = 1 WHERE b = 2)#x] +Project [parse_sql(UPDATE t SET a = 1 WHERE b = 2) AS parse_sql(UPDATE t SET a = 1 WHERE b = 2)#x] +- OneRowRelation -- !query -SELECT parse_command('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE') +SELECT parse_sql('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE') -- !query analysis -Project [parse_command(MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE) AS parse_command(MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE)#x] +Project [parse_sql(MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE) AS parse_sql(MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE)#x] +- OneRowRelation -- !query -SELECT parse_command('CREATE TABLE t (a INT)') +SELECT parse_sql('CREATE TABLE t (a INT)') -- !query analysis -Project [parse_command(CREATE TABLE t (a INT)) AS parse_command(CREATE TABLE t (a INT))#x] +Project [parse_sql(CREATE TABLE t (a INT)) AS parse_sql(CREATE TABLE t (a INT))#x] +- OneRowRelation -- !query -SELECT parse_command('CREATE TABLE t AS SELECT 1 AS a') +SELECT parse_sql('CREATE TABLE t AS SELECT 1 AS a') -- !query analysis -Project [parse_command(CREATE TABLE t AS SELECT 1 AS a) AS parse_command(CREATE TABLE t AS SELECT 1 AS a)#x] +Project [parse_sql(CREATE TABLE t AS SELECT 1 AS a) AS parse_sql(CREATE TABLE t AS SELECT 1 AS a)#x] +- OneRowRelation -- !query -SELECT parse_command('DROP TABLE t') +SELECT parse_sql('DROP TABLE t') -- !query analysis -Project [parse_command(DROP TABLE t) AS parse_command(DROP TABLE t)#x] +Project [parse_sql(DROP TABLE t) AS parse_sql(DROP TABLE t)#x] +- OneRowRelation -- !query -SELECT parse_command('CACHE TABLE t') +SELECT parse_sql('CACHE TABLE t') -- !query analysis -Project [parse_command(CACHE TABLE t) AS parse_command(CACHE TABLE t)#x] +Project [parse_sql(CACHE TABLE t) AS parse_sql(CACHE TABLE t)#x] +- OneRowRelation -- !query -SELECT parse_command('SELECT * FROM t WHERE a = :foo AND b = ?') +SELECT parse_sql('SELECT * FROM t WHERE a = :foo AND b = ?') -- !query analysis -Project [parse_command(SELECT * FROM t WHERE a = :foo AND b = ?) AS parse_command(SELECT * FROM t WHERE a = :foo AND b = ?)#x] +Project [parse_sql(SELECT * FROM t WHERE a = :foo AND b = ?) AS parse_sql(SELECT * FROM t WHERE a = :foo AND b = ?)#x] +- OneRowRelation -- !query -SELECT parse_command('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte') +SELECT parse_sql('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte') -- !query analysis -Project [parse_command(WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte) AS parse_command(WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte)#x] +Project [parse_sql(WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte) AS parse_sql(WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte)#x] +- OneRowRelation -- !query -SELECT parse_command('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)') +SELECT parse_sql('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)') -- !query analysis -Project [parse_command(SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)) AS parse_command(SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id))#x] +Project [parse_sql(SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)) AS parse_sql(SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id))#x] +- OneRowRelation -- !query -SELECT parse_command( +SELECT parse_sql( 'SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( PARTITION BY lower(t.c) ORDER BY length(t.d)) FROM left_t t @@ -110,7 +110,7 @@ SELECT parse_command( HAVING count_if(t.b > 0) > 0 ORDER BY greatest(t.a, 1)') -- !query analysis -Project [parse_command(SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( +Project [parse_sql(SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( PARTITION BY lower(t.c) ORDER BY length(t.d)) FROM left_t t JOIN right_t r ON hash(t.id) = hash(r.id) @@ -119,7 +119,7 @@ Project [parse_command(SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id) GROUP BY coalesce(t.a, 0), t.b, t.c, t.d HAVING count_if(t.b > 0) > 0 - ORDER BY greatest(t.a, 1)) AS parse_command(SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( + ORDER BY greatest(t.a, 1)) AS parse_sql(SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( PARTITION BY lower(t.c) ORDER BY length(t.d)) FROM left_t t JOIN right_t r ON hash(t.id) = hash(r.id) @@ -133,7 +133,7 @@ Project [parse_command(SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( -- !query -SELECT parse_command( +SELECT parse_sql( 'MERGE INTO target t USING ( SELECT id, normalize_name(name) AS name @@ -146,7 +146,7 @@ SELECT parse_command( WHEN NOT MATCHED THEN INSERT (id, name) VALUES (s.id, lower(s.name))') -- !query analysis -Project [parse_command(MERGE INTO target t +Project [parse_sql(MERGE INTO target t USING ( SELECT id, normalize_name(name) AS name FROM source @@ -156,7 +156,7 @@ Project [parse_command(MERGE INTO target t WHEN MATCHED AND should_update(t.name, s.name) THEN UPDATE SET name = coalesce(s.name, upper(t.name)) WHEN NOT MATCHED THEN - INSERT (id, name) VALUES (s.id, lower(s.name))) AS parse_command(MERGE INTO target t + INSERT (id, name) VALUES (s.id, lower(s.name))) AS parse_sql(MERGE INTO target t USING ( SELECT id, normalize_name(name) AS name FROM source @@ -171,16 +171,16 @@ Project [parse_command(MERGE INTO target t -- !query -SELECT parse_command( +SELECT parse_sql( 'CREATE TABLE defaults ( created DATE DEFAULT current_date(), normalized STRING DEFAULT upper(''x'') )') -- !query analysis -Project [parse_command(CREATE TABLE defaults ( +Project [parse_sql(CREATE TABLE defaults ( created DATE DEFAULT current_date(), normalized STRING DEFAULT upper('x') - )) AS parse_command(CREATE TABLE defaults ( + )) AS parse_sql(CREATE TABLE defaults ( created DATE DEFAULT current_date(), normalized STRING DEFAULT upper('x') ))#x] @@ -188,59 +188,59 @@ Project [parse_command(CREATE TABLE defaults ( -- !query -SELECT get_json_object(parse_command('SELEC FROM t'), '$.parse_success') +SELECT get_json_object(parse_sql('SELEC FROM t'), '$.parse_success') -- !query analysis -Project [get_json_object(parse_command(SELEC FROM t), $.parse_success) AS get_json_object(parse_command(SELEC FROM t), $.parse_success)#x] +Project [get_json_object(parse_sql(SELEC FROM t), $.parse_success) AS get_json_object(parse_sql(SELEC FROM t), $.parse_success)#x] +- OneRowRelation -- !query -SELECT get_json_object(parse_command('SELEC FROM t'), '$.error.errorClass') +SELECT get_json_object(parse_sql('SELEC FROM t'), '$.error.errorClass') -- !query analysis -Project [get_json_object(parse_command(SELEC FROM t), $.error.errorClass) AS get_json_object(parse_command(SELEC FROM t), $.error.errorClass)#x] +Project [get_json_object(parse_sql(SELEC FROM t), $.error.errorClass) AS get_json_object(parse_sql(SELEC FROM t), $.error.errorClass)#x] +- OneRowRelation -- !query -SELECT get_json_object(parse_command('SELEC FROM t'), '$.error.sqlState') +SELECT get_json_object(parse_sql('SELEC FROM t'), '$.error.sqlState') -- !query analysis -Project [get_json_object(parse_command(SELEC FROM t), $.error.sqlState) AS get_json_object(parse_command(SELEC FROM t), $.error.sqlState)#x] +Project [get_json_object(parse_sql(SELEC FROM t), $.error.sqlState) AS get_json_object(parse_sql(SELEC FROM t), $.error.sqlState)#x] +- OneRowRelation -- !query SELECT - get_json_object(parse_command( + get_json_object(parse_sql( 'SELECT * FROM t ORDER BY a CLUSTER BY b'), '$.error.errorClass') AS error_class, - get_json_object(parse_command( + get_json_object(parse_sql( 'SELECT * FROM t ORDER BY a CLUSTER BY b'), '$.error.line') AS line, - get_json_object(parse_command( + get_json_object(parse_sql( 'SELECT * FROM t ORDER BY a CLUSTER BY b'), '$.error.position') AS position, - get_json_object(parse_command( + get_json_object(parse_sql( 'SELECT * FROM t ORDER BY a CLUSTER BY b'), '$.error.queryContext[0].startIndex') AS start_index -- !query analysis -Project [get_json_object(parse_command(SELECT * +Project [get_json_object(parse_sql(SELECT * FROM t ORDER BY a - CLUSTER BY b), $.error.errorClass) AS error_class#x, get_json_object(parse_command(SELECT * + CLUSTER BY b), $.error.errorClass) AS error_class#x, get_json_object(parse_sql(SELECT * FROM t ORDER BY a - CLUSTER BY b), $.error.line) AS line#x, get_json_object(parse_command(SELECT * + CLUSTER BY b), $.error.line) AS line#x, get_json_object(parse_sql(SELECT * FROM t ORDER BY a - CLUSTER BY b), $.error.position) AS position#x, get_json_object(parse_command(SELECT * + CLUSTER BY b), $.error.position) AS position#x, get_json_object(parse_sql(SELECT * FROM t ORDER BY a CLUSTER BY b), $.error.queryContext[0].startIndex) AS start_index#x] @@ -248,89 +248,108 @@ Project [get_json_object(parse_command(SELECT * -- !query -SELECT get_json_object(parse_command(''), '$.error.errorClass') +SELECT get_json_object(parse_sql(''), '$.error.errorClass') -- !query analysis -Project [get_json_object(parse_command(), $.error.errorClass) AS get_json_object(parse_command(), $.error.errorClass)#x] +Project [get_json_object(parse_sql(), $.error.errorClass) AS get_json_object(parse_sql(), $.error.errorClass)#x] +- OneRowRelation -- !query -SELECT get_json_object(parse_command('USE bad-name'), '$.error.errorClass') +SELECT get_json_object(parse_sql('USE bad-name'), '$.error.errorClass') -- !query analysis -Project [get_json_object(parse_command(USE bad-name), $.error.errorClass) AS get_json_object(parse_command(USE bad-name), $.error.errorClass)#x] +Project [get_json_object(parse_sql(USE bad-name), $.error.errorClass) AS get_json_object(parse_sql(USE bad-name), $.error.errorClass)#x] +- OneRowRelation -- !query SELECT get_json_object( - parse_command('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c'), + parse_sql('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c'), '$.error.errorClass') -- !query analysis -Project [get_json_object(parse_command(WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c), $.error.errorClass) AS get_json_object(parse_command(WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c), $.error.errorClass)#x] +Project [get_json_object(parse_sql(WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c), $.error.errorClass) AS get_json_object(parse_sql(WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c), $.error.errorClass)#x] +- OneRowRelation -- !query SELECT get_json_object( - parse_command('MERGE INTO target USING source ON target.id = source.id'), + parse_sql('MERGE INTO target USING source ON target.id = source.id'), '$.error.errorClass') -- !query analysis -Project [get_json_object(parse_command(MERGE INTO target USING source ON target.id = source.id), $.error.errorClass) AS get_json_object(parse_command(MERGE INTO target USING source ON target.id = source.id), $.error.errorClass)#x] +Project [get_json_object(parse_sql(MERGE INTO target USING source ON target.id = source.id), $.error.errorClass) AS get_json_object(parse_sql(MERGE INTO target USING source ON target.id = source.id), $.error.errorClass)#x] +- OneRowRelation -- !query -SELECT get_json_object( - parse_command('DROP FUNCTION catalog.schema.func'), - '$.error.errorClass') +SELECT get_json_object(parse_sql('EXPLAIN SELECT 1'), '$.statement_identifier') +-- !query analysis +Project [get_json_object(parse_sql(EXPLAIN SELECT 1), $.statement_identifier) AS get_json_object(parse_sql(EXPLAIN SELECT 1), $.statement_identifier)#x] ++- OneRowRelation + + +-- !query +SELECT get_json_object(parse_sql('SET spark.sql.adaptive.enabled=true'), '$.statement_code') +-- !query analysis +Project [get_json_object(parse_sql(SET spark.sql.adaptive.enabled=true), $.statement_code) AS get_json_object(parse_sql(SET spark.sql.adaptive.enabled=true), $.statement_code)#x] ++- OneRowRelation + + +-- !query +SELECT get_json_object(parse_sql('ADD JAR /tmp/x.jar'), '$.statement_identifier') +-- !query analysis +Project [get_json_object(parse_sql(ADD JAR /tmp/x.jar), $.statement_identifier) AS get_json_object(parse_sql(ADD JAR /tmp/x.jar), $.statement_identifier)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('CREATE VIEW v AS SELECT a, b FROM t') -- !query analysis -Project [get_json_object(parse_command(DROP FUNCTION catalog.schema.func), $.error.errorClass) AS get_json_object(parse_command(DROP FUNCTION catalog.schema.func), $.error.errorClass)#x] +Project [parse_sql(CREATE VIEW v AS SELECT a, b FROM t) AS parse_sql(CREATE VIEW v AS SELECT a, b FROM t)#x] +- OneRowRelation -- !query SELECT get_json_object( - parse_command('SELECT 1 AS IDENTIFIER(''alias.field'')'), + parse_sql('SELECT 1 AS IDENTIFIER(''alias.field'')'), '$.error.errorClass') -- !query analysis -Project [get_json_object(parse_command(SELECT 1 AS IDENTIFIER('alias.field')), $.error.errorClass) AS get_json_object(parse_command(SELECT 1 AS IDENTIFIER('alias.field')), $.error.errorClass)#x] +Project [get_json_object(parse_sql(SELECT 1 AS IDENTIFIER('alias.field')), $.error.errorClass) AS get_json_object(parse_sql(SELECT 1 AS IDENTIFIER('alias.field')), $.error.errorClass)#x] +- OneRowRelation -- !query SELECT get_json_object( - parse_command('SELECT DATE ''not-a-date'''), + parse_sql('SELECT DATE ''not-a-date'''), '$.error.errorClass') -- !query analysis -Project [get_json_object(parse_command(SELECT DATE 'not-a-date'), $.error.errorClass) AS get_json_object(parse_command(SELECT DATE 'not-a-date'), $.error.errorClass)#x] +Project [get_json_object(parse_sql(SELECT DATE 'not-a-date'), $.error.errorClass) AS get_json_object(parse_sql(SELECT DATE 'not-a-date'), $.error.errorClass)#x] +- OneRowRelation -- !query SELECT - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN SELECT 1; SELEC 2; END'), '$.error.errorClass') AS error_class, - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN SELECT 1; SELEC 2; END'), '$.error.line') AS line, - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN SELECT 1; SELEC 2; END'), '$.error.position') AS position -- !query analysis -Project [get_json_object(parse_command(BEGIN +Project [get_json_object(parse_sql(BEGIN SELECT 1; SELEC 2; - END), $.error.errorClass) AS error_class#x, get_json_object(parse_command(BEGIN + END), $.error.errorClass) AS error_class#x, get_json_object(parse_sql(BEGIN SELECT 1; SELEC 2; - END), $.error.line) AS line#x, get_json_object(parse_command(BEGIN + END), $.error.line) AS line#x, get_json_object(parse_sql(BEGIN SELECT 1; SELEC 2; END), $.error.position) AS position#x] @@ -339,34 +358,34 @@ Project [get_json_object(parse_command(BEGIN -- !query SELECT - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; END'), '$.error.errorClass') AS error_class, - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; END'), '$.error.line') AS line, - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; END'), '$.error.position') AS position -- !query analysis -Project [get_json_object(parse_command(BEGIN +Project [get_json_object(parse_sql(BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; - END), $.error.errorClass) AS error_class#x, get_json_object(parse_command(BEGIN + END), $.error.errorClass) AS error_class#x, get_json_object(parse_sql(BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; - END), $.error.line) AS line#x, get_json_object(parse_command(BEGIN + END), $.error.line) AS line#x, get_json_object(parse_sql(BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; @@ -375,48 +394,48 @@ Project [get_json_object(parse_command(BEGIN -- !query -SELECT sql_text, parse_command(sql_text) FROM VALUES +SELECT sql_text, parse_sql(sql_text) FROM VALUES ('SELECT 1'), ('INSERT INTO t SELECT 1'), ('CACHE TABLE t') AS t(sql_text) -- !query analysis -Project [sql_text#x, parse_command(sql_text#x) AS parse_command(sql_text)#x] +Project [sql_text#x, parse_sql(sql_text#x) AS parse_sql(sql_text)#x] +- SubqueryAlias t +- LocalRelation [sql_text#x] -- !query -SELECT parse_command('BEGIN SELECT 1; END') +SELECT parse_sql('BEGIN SELECT 1; END') -- !query analysis -Project [parse_command(BEGIN SELECT 1; END) AS parse_command(BEGIN SELECT 1; END)#x] +Project [parse_sql(BEGIN SELECT 1; END) AS parse_sql(BEGIN SELECT 1; END)#x] +- OneRowRelation -- !query -SELECT parse_command('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END') +SELECT parse_sql('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END') -- !query analysis -Project [parse_command(BEGIN SELECT count(a) FROM script_t WHERE c = :p; END) AS parse_command(BEGIN SELECT count(a) FROM script_t WHERE c = :p; END)#x] +Project [parse_sql(BEGIN SELECT count(a) FROM script_t WHERE c = :p; END) AS parse_sql(BEGIN SELECT count(a) FROM script_t WHERE c = :p; END)#x] +- OneRowRelation -- !query -SELECT parse_command('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END') +SELECT parse_sql('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END') -- !query analysis -Project [parse_command(BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END) AS parse_command(BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END)#x] +Project [parse_sql(BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END) AS parse_sql(BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END)#x] +- OneRowRelation -- !query -SELECT parse_command('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END') +SELECT parse_sql('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END') -- !query analysis -Project [parse_command(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END) AS parse_command(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END)#x] +Project [parse_sql(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END) AS parse_sql(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END)#x] +- OneRowRelation -- !query SELECT - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN @@ -447,7 +466,7 @@ SELECT SELECT audit(row.id), count(*) FROM loop_body; END FOR; END'), '$.table_references') AS table_references, - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN @@ -479,7 +498,7 @@ SELECT END FOR; END'), '$.function_references') AS function_references -- !query analysis -Project [get_json_object(parse_command(BEGIN +Project [get_json_object(parse_sql(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO error_log @@ -508,7 +527,7 @@ Project [get_json_object(parse_command(BEGIN DO SELECT audit(row.id), count(*) FROM loop_body; END FOR; - END), $.table_references) AS table_references#x, get_json_object(parse_command(BEGIN + END), $.table_references) AS table_references#x, get_json_object(parse_sql(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO error_log @@ -543,10 +562,10 @@ Project [get_json_object(parse_command(BEGIN -- !query SELECT - get_json_object(parse_command('BEGIN SELECT 1; END'), '$.statement_identifier') AS statement_identifier, - get_json_object(parse_command('BEGIN SELECT 1; END'), '$.statement_code') AS statement_code, - get_json_object(parse_command('BEGIN SELECT count(a) FROM script_t; END'), '$.table_references') AS table_references, - get_json_object(parse_command('BEGIN SELECT count(a) FROM script_t; END'), '$.function_references') AS function_references + get_json_object(parse_sql('BEGIN SELECT 1; END'), '$.statement_identifier') AS statement_identifier, + get_json_object(parse_sql('BEGIN SELECT 1; END'), '$.statement_code') AS statement_code, + get_json_object(parse_sql('BEGIN SELECT count(a) FROM script_t; END'), '$.table_references') AS table_references, + get_json_object(parse_sql('BEGIN SELECT count(a) FROM script_t; END'), '$.function_references') AS function_references -- !query analysis -Project [get_json_object(parse_command(BEGIN SELECT 1; END), $.statement_identifier) AS statement_identifier#x, get_json_object(parse_command(BEGIN SELECT 1; END), $.statement_code) AS statement_code#x, get_json_object(parse_command(BEGIN SELECT count(a) FROM script_t; END), $.table_references) AS table_references#x, get_json_object(parse_command(BEGIN SELECT count(a) FROM script_t; END), $.function_references) AS function_references#x] +Project [get_json_object(parse_sql(BEGIN SELECT 1; END), $.statement_identifier) AS statement_identifier#x, get_json_object(parse_sql(BEGIN SELECT 1; END), $.statement_code) AS statement_code#x, get_json_object(parse_sql(BEGIN SELECT count(a) FROM script_t; END), $.table_references) AS table_references#x, get_json_object(parse_sql(BEGIN SELECT count(a) FROM script_t; END), $.function_references) AS function_references#x] +- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/inputs/parse-command.sql b/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql similarity index 61% rename from sql/core/src/test/resources/sql-tests/inputs/parse-command.sql rename to sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql index 313fc996c0ac..2b267dbd6e3a 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/parse-command.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql @@ -1,38 +1,38 @@ --- End-to-end coverage for parse_command (SPARK-58738). --- Returns compact JSON for parse-only statement analysis. +-- End-to-end coverage for parse_sql (SPARK-58738). +-- Returns compact JSON for parse-only statement analysis via SparkSqlParser. -- null input -SELECT parse_command(NULL); +SELECT parse_sql(NULL); -- basic SELECT classification and references -SELECT parse_command('SELECT a, b FROM t'); -SELECT parse_command('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2'); +SELECT parse_sql('SELECT a, b FROM t'); +SELECT parse_sql('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2'); -- DML -SELECT parse_command('INSERT INTO t SELECT 1'); -SELECT parse_command('DELETE FROM t WHERE a = 1'); -SELECT parse_command('UPDATE t SET a = 1 WHERE b = 2'); -SELECT parse_command('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE'); +SELECT parse_sql('INSERT INTO t SELECT 1'); +SELECT parse_sql('DELETE FROM t WHERE a = 1'); +SELECT parse_sql('UPDATE t SET a = 1 WHERE b = 2'); +SELECT parse_sql('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE'); -- DDL / CTAS -SELECT parse_command('CREATE TABLE t (a INT)'); -SELECT parse_command('CREATE TABLE t AS SELECT 1 AS a'); -SELECT parse_command('DROP TABLE t'); +SELECT parse_sql('CREATE TABLE t (a INT)'); +SELECT parse_sql('CREATE TABLE t AS SELECT 1 AS a'); +SELECT parse_sql('DROP TABLE t'); -- Spark-only statements (negative Table 39 codes) -SELECT parse_command('CACHE TABLE t'); +SELECT parse_sql('CACHE TABLE t'); -- parameter markers -SELECT parse_command('SELECT * FROM t WHERE a = :foo AND b = ?'); +SELECT parse_sql('SELECT * FROM t WHERE a = :foo AND b = ?'); --- CTE: UnresolvedWith CTE bodies are innerChildren -SELECT parse_command('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte'); +-- CTE: lineage excludes CTE names; still walks CTE bodies for real tables +SELECT parse_sql('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte'); -- nested subqueries -SELECT parse_command('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)'); +SELECT parse_sql('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)'); -- functions in projection, window, join, TVF, predicates, subquery, grouping, and ordering -SELECT parse_command( +SELECT parse_sql( 'SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( PARTITION BY lower(t.c) ORDER BY length(t.d)) FROM left_t t @@ -45,7 +45,7 @@ SELECT parse_command( ORDER BY greatest(t.a, 1)'); -- functions and tables throughout a multiline MERGE -SELECT parse_command( +SELECT parse_sql( 'MERGE INTO target t USING ( SELECT id, normalize_name(name) AS name @@ -59,73 +59,74 @@ SELECT parse_command( INSERT (id, name) VALUES (s.id, lower(s.name))'); -- functions embedded in DDL column defaults -SELECT parse_command( +SELECT parse_sql( 'CREATE TABLE defaults ( created DATE DEFAULT current_date(), normalized STRING DEFAULT upper(''x'') )'); -- syntax error: never throws; STANDARD error nested under parse_success=false -SELECT get_json_object(parse_command('SELEC FROM t'), '$.parse_success'); -SELECT get_json_object(parse_command('SELEC FROM t'), '$.error.errorClass'); -SELECT get_json_object(parse_command('SELEC FROM t'), '$.error.sqlState'); +SELECT get_json_object(parse_sql('SELEC FROM t'), '$.parse_success'); +SELECT get_json_object(parse_sql('SELEC FROM t'), '$.error.errorClass'); +SELECT get_json_object(parse_sql('SELEC FROM t'), '$.error.sqlState'); -- source location from a multiline parse-time validation error SELECT - get_json_object(parse_command( + get_json_object(parse_sql( 'SELECT * FROM t ORDER BY a CLUSTER BY b'), '$.error.errorClass') AS error_class, - get_json_object(parse_command( + get_json_object(parse_sql( 'SELECT * FROM t ORDER BY a CLUSTER BY b'), '$.error.line') AS line, - get_json_object(parse_command( + get_json_object(parse_sql( 'SELECT * FROM t ORDER BY a CLUSTER BY b'), '$.error.position') AS position, - get_json_object(parse_command( + get_json_object(parse_sql( 'SELECT * FROM t ORDER BY a CLUSTER BY b'), '$.error.queryContext[0].startIndex') AS start_index; -- parse-only validation errors beyond PARSE_SYNTAX_ERROR -SELECT get_json_object(parse_command(''), '$.error.errorClass'); -SELECT get_json_object(parse_command('USE bad-name'), '$.error.errorClass'); +SELECT get_json_object(parse_sql(''), '$.error.errorClass'); +SELECT get_json_object(parse_sql('USE bad-name'), '$.error.errorClass'); SELECT get_json_object( - parse_command('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c'), + parse_sql('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c'), '$.error.errorClass'); SELECT get_json_object( - parse_command('MERGE INTO target USING source ON target.id = source.id'), + parse_sql('MERGE INTO target USING source ON target.id = source.id'), '$.error.errorClass'); +SELECT get_json_object(parse_sql('EXPLAIN SELECT 1'), '$.statement_identifier'); +SELECT get_json_object(parse_sql('SET spark.sql.adaptive.enabled=true'), '$.statement_code'); +SELECT get_json_object(parse_sql('ADD JAR /tmp/x.jar'), '$.statement_identifier'); +SELECT parse_sql('CREATE VIEW v AS SELECT a, b FROM t'); SELECT get_json_object( - parse_command('DROP FUNCTION catalog.schema.func'), + parse_sql('SELECT 1 AS IDENTIFIER(''alias.field'')'), '$.error.errorClass'); SELECT get_json_object( - parse_command('SELECT 1 AS IDENTIFIER(''alias.field'')'), - '$.error.errorClass'); -SELECT get_json_object( - parse_command('SELECT DATE ''not-a-date'''), + parse_sql('SELECT DATE ''not-a-date'''), '$.error.errorClass'); -- location for an error inside a multiline script --QUERY-DELIMITER-START SELECT - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN SELECT 1; SELEC 2; END'), '$.error.errorClass') AS error_class, - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN SELECT 1; SELEC 2; END'), '$.error.line') AS line, - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN SELECT 1; SELEC 2; @@ -135,19 +136,19 @@ SELECT -- location for a SQL scripting semantic validation error --QUERY-DELIMITER-START SELECT - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; END'), '$.error.errorClass') AS error_class, - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; END'), '$.error.line') AS line, - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN lbl_begin: BEGIN SELECT 1; @@ -156,7 +157,7 @@ SELECT --QUERY-DELIMITER-END -- batch over a column of SQL text -SELECT sql_text, parse_command(sql_text) FROM VALUES +SELECT sql_text, parse_sql(sql_text) FROM VALUES ('SELECT 1'), ('INSERT INTO t SELECT 1'), ('CACHE TABLE t') @@ -165,26 +166,26 @@ AS t(sql_text); -- BEGIN END scripts contain ';' inside the string literal; use query delimiters -- so the test harness does not split on those semicolons. --QUERY-DELIMITER-START -SELECT parse_command('BEGIN SELECT 1; END'); +SELECT parse_sql('BEGIN SELECT 1; END'); --QUERY-DELIMITER-END --QUERY-DELIMITER-START -SELECT parse_command('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END'); +SELECT parse_sql('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END'); --QUERY-DELIMITER-END --QUERY-DELIMITER-START -SELECT parse_command('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END'); +SELECT parse_sql('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END'); --QUERY-DELIMITER-END --QUERY-DELIMITER-START -SELECT parse_command('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END'); +SELECT parse_sql('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END'); --QUERY-DELIMITER-END -- Complex, genuinely multiline script. Extract collections to keep the -- expected output focused on complete tree walking. --QUERY-DELIMITER-START SELECT - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN @@ -215,7 +216,7 @@ SELECT SELECT audit(row.id), count(*) FROM loop_body; END FOR; END'), '$.table_references') AS table_references, - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN @@ -251,8 +252,8 @@ SELECT -- extract key fields from a script for readable assertions --QUERY-DELIMITER-START SELECT - get_json_object(parse_command('BEGIN SELECT 1; END'), '$.statement_identifier') AS statement_identifier, - get_json_object(parse_command('BEGIN SELECT 1; END'), '$.statement_code') AS statement_code, - get_json_object(parse_command('BEGIN SELECT count(a) FROM script_t; END'), '$.table_references') AS table_references, - get_json_object(parse_command('BEGIN SELECT count(a) FROM script_t; END'), '$.function_references') AS function_references; + get_json_object(parse_sql('BEGIN SELECT 1; END'), '$.statement_identifier') AS statement_identifier, + get_json_object(parse_sql('BEGIN SELECT 1; END'), '$.statement_code') AS statement_code, + get_json_object(parse_sql('BEGIN SELECT count(a) FROM script_t; END'), '$.table_references') AS table_references, + get_json_object(parse_sql('BEGIN SELECT count(a) FROM script_t; END'), '$.function_references') AS function_references; --QUERY-DELIMITER-END diff --git a/sql/core/src/test/resources/sql-tests/results/parse-command.sql.out b/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out similarity index 61% rename from sql/core/src/test/resources/sql-tests/results/parse-command.sql.out rename to sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out index 340e7d4a9d70..699f60bac26e 100644 --- a/sql/core/src/test/resources/sql-tests/results/parse-command.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out @@ -1,118 +1,118 @@ -- Automatically generated by SQLQueryTestSuite -- !query -SELECT parse_command(NULL) +SELECT parse_sql(NULL) -- !query schema -struct +struct -- !query output NULL -- !query -SELECT parse_command('SELECT a, b FROM t') +SELECT parse_sql('SELECT a, b FROM t') -- !query schema -struct +struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"],"expression":"a"},{"name":["b"],"expression":"b"}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"]},{"name":["b"]}],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query -SELECT parse_command('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2') +SELECT parse_sql('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2') -- !query schema -struct +struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["cat","ns","t1"],["t2"]],"function_references":[["db","my_func"],["count"]],"select_list":[{"name":[],"expression":"db.my_func(a)"},{"name":[],"expression":"count(b)"}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["cat","ns","t1"],["t2"]],"function_references":[["db","my_func"],["count"]],"select_list":[{"name":[]},{"name":[]}],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query -SELECT parse_command('INSERT INTO t SELECT 1') +SELECT parse_sql('INSERT INTO t SELECT 1') -- !query schema -struct +struct -- !query output -{"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"table_references":[["t"]],"function_references":[],"select_list":[{"name":[],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"table_references":[["t"]],"function_references":[],"select_list":[{"name":[]}],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query -SELECT parse_command('DELETE FROM t WHERE a = 1') +SELECT parse_sql('DELETE FROM t WHERE a = 1') -- !query schema -struct +struct -- !query output {"parse_success":true,"statement_identifier":"DELETE WHERE","statement_code":19,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query -SELECT parse_command('UPDATE t SET a = 1 WHERE b = 2') +SELECT parse_sql('UPDATE t SET a = 1 WHERE b = 2') -- !query schema -struct +struct -- !query output {"parse_success":true,"statement_identifier":"UPDATE WHERE","statement_code":82,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query -SELECT parse_command('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE') +SELECT parse_sql('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE') -- !query schema -struct +struct -- !query output {"parse_success":true,"statement_identifier":"MERGE","statement_code":128,"table_references":[["t"],["s"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query -SELECT parse_command('CREATE TABLE t (a INT)') +SELECT parse_sql('CREATE TABLE t (a INT)') -- !query schema -struct +struct -- !query output {"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query -SELECT parse_command('CREATE TABLE t AS SELECT 1 AS a') +SELECT parse_sql('CREATE TABLE t AS SELECT 1 AS a') -- !query schema -struct +struct -- !query output -{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"]}],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query -SELECT parse_command('DROP TABLE t') +SELECT parse_sql('DROP TABLE t') -- !query schema -struct +struct -- !query output {"parse_success":true,"statement_identifier":"DROP TABLE","statement_code":32,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query -SELECT parse_command('CACHE TABLE t') +SELECT parse_sql('CACHE TABLE t') -- !query schema -struct +struct -- !query output {"parse_success":true,"statement_identifier":"CACHE TABLE","statement_code":-1,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query -SELECT parse_command('SELECT * FROM t WHERE a = :foo AND b = ?') +SELECT parse_sql('SELECT * FROM t WHERE a = :foo AND b = ?') -- !query schema -struct +struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["*"],"expression":"unresolvedstar()"}],"parameter_markers":{"named":["foo"],"unnamed_count":1}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["*"]}],"parameter_markers":{"named":["foo"],"unnamed_count":1}} -- !query -SELECT parse_command('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte') +SELECT parse_sql('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte') -- !query schema -struct +struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["hidden_base"],["cte"]],"function_references":[],"select_list":[{"name":["a"],"expression":"a"}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["hidden_base"]],"function_references":[],"select_list":[{"name":["a"]}],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query -SELECT parse_command('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)') +SELECT parse_sql('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)') -- !query schema -struct +struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["scalar_src"],["exists_src"],["outer_t"]],"function_references":[["max"]],"select_list":[{"name":["m"],"expression":"scalarsubquery()"},{"name":["t","a"],"expression":"t.a"}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["scalar_src"],["exists_src"],["outer_t"]],"function_references":[["max"]],"select_list":[{"name":["m"]},{"name":["t","a"]}],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query -SELECT parse_command( +SELECT parse_sql( 'SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( PARTITION BY lower(t.c) ORDER BY length(t.d)) FROM left_t t @@ -124,7 +124,7 @@ SELECT parse_command( HAVING count_if(t.b > 0) > 0 ORDER BY greatest(t.a, 1)') -- !query schema -struct @@ -183,47 +183,47 @@ struct +struct -- !query output false -- !query -SELECT get_json_object(parse_command('SELEC FROM t'), '$.error.errorClass') +SELECT get_json_object(parse_sql('SELEC FROM t'), '$.error.errorClass') -- !query schema -struct +struct -- !query output PARSE_SYNTAX_ERROR -- !query -SELECT get_json_object(parse_command('SELEC FROM t'), '$.error.sqlState') +SELECT get_json_object(parse_sql('SELEC FROM t'), '$.error.sqlState') -- !query schema -struct +struct -- !query output 42601 -- !query SELECT - get_json_object(parse_command( + get_json_object(parse_sql( 'SELECT * FROM t ORDER BY a CLUSTER BY b'), '$.error.errorClass') AS error_class, - get_json_object(parse_command( + get_json_object(parse_sql( 'SELECT * FROM t ORDER BY a CLUSTER BY b'), '$.error.line') AS line, - get_json_object(parse_command( + get_json_object(parse_sql( 'SELECT * FROM t ORDER BY a CLUSTER BY b'), '$.error.position') AS position, - get_json_object(parse_command( + get_json_object(parse_sql( 'SELECT * FROM t ORDER BY a @@ -235,84 +235,106 @@ UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES 3 1 19 -- !query -SELECT get_json_object(parse_command(''), '$.error.errorClass') +SELECT get_json_object(parse_sql(''), '$.error.errorClass') -- !query schema -struct +struct -- !query output PARSE_EMPTY_STATEMENT -- !query -SELECT get_json_object(parse_command('USE bad-name'), '$.error.errorClass') +SELECT get_json_object(parse_sql('USE bad-name'), '$.error.errorClass') -- !query schema -struct +struct -- !query output INVALID_IDENTIFIER -- !query SELECT get_json_object( - parse_command('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c'), + parse_sql('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c'), '$.error.errorClass') -- !query schema -struct +struct -- !query output DUPLICATED_CTE_NAMES -- !query SELECT get_json_object( - parse_command('MERGE INTO target USING source ON target.id = source.id'), + parse_sql('MERGE INTO target USING source ON target.id = source.id'), '$.error.errorClass') -- !query schema -struct +struct -- !query output MERGE_WITHOUT_WHEN -- !query -SELECT get_json_object( - parse_command('DROP FUNCTION catalog.schema.func'), - '$.error.errorClass') +SELECT get_json_object(parse_sql('EXPLAIN SELECT 1'), '$.statement_identifier') +-- !query schema +struct +-- !query output +EXPLAIN + + +-- !query +SELECT get_json_object(parse_sql('SET spark.sql.adaptive.enabled=true'), '$.statement_code') +-- !query schema +struct +-- !query output +-24 + + +-- !query +SELECT get_json_object(parse_sql('ADD JAR /tmp/x.jar'), '$.statement_identifier') +-- !query schema +struct +-- !query output +ADD JAR + + +-- !query +SELECT parse_sql('CREATE VIEW v AS SELECT a, b FROM t') -- !query schema -struct +struct -- !query output -INVALID_SQL_SYNTAX.UNSUPPORTED_SQL_STATEMENT +{"parse_success":true,"statement_identifier":"CREATE VIEW","statement_code":84,"table_references":[["v"],["t"]],"function_references":[],"select_list":[{"name":["a"]},{"name":["b"]}],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query SELECT get_json_object( - parse_command('SELECT 1 AS IDENTIFIER(''alias.field'')'), + parse_sql('SELECT 1 AS IDENTIFIER(''alias.field'')'), '$.error.errorClass') -- !query schema -struct +struct -- !query output IDENTIFIER_TOO_MANY_NAME_PARTS -- !query SELECT get_json_object( - parse_command('SELECT DATE ''not-a-date'''), + parse_sql('SELECT DATE ''not-a-date'''), '$.error.errorClass') -- !query schema -struct +struct -- !query output INVALID_TYPED_LITERAL -- !query SELECT - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN SELECT 1; SELEC 2; END'), '$.error.errorClass') AS error_class, - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN SELECT 1; SELEC 2; END'), '$.error.line') AS line, - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN SELECT 1; SELEC 2; @@ -325,19 +347,19 @@ PARSE_SYNTAX_ERROR 3 9 -- !query SELECT - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; END'), '$.error.errorClass') AS error_class, - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; END'), '$.error.line') AS line, - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN lbl_begin: BEGIN SELECT 1; @@ -350,54 +372,54 @@ LABELS_MISMATCH 2 3 -- !query -SELECT sql_text, parse_command(sql_text) FROM VALUES +SELECT sql_text, parse_sql(sql_text) FROM VALUES ('SELECT 1'), ('INSERT INTO t SELECT 1'), ('CACHE TABLE t') AS t(sql_text) -- !query schema -struct +struct -- !query output CACHE TABLE t {"parse_success":true,"statement_identifier":"CACHE TABLE","statement_code":-1,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -INSERT INTO t SELECT 1 {"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"table_references":[["t"]],"function_references":[],"select_list":[{"name":[],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} -SELECT 1 {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[],"function_references":[],"select_list":[{"name":[],"expression":"1"}],"parameter_markers":{"named":[],"unnamed_count":0}} +INSERT INTO t SELECT 1 {"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"table_references":[["t"]],"function_references":[],"select_list":[{"name":[]}],"parameter_markers":{"named":[],"unnamed_count":0}} +SELECT 1 {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[],"function_references":[],"select_list":[{"name":[]}],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query -SELECT parse_command('BEGIN SELECT 1; END') +SELECT parse_sql('BEGIN SELECT 1; END') -- !query schema -struct +struct -- !query output {"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query -SELECT parse_command('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END') +SELECT parse_sql('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END') -- !query schema -struct +struct -- !query output {"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["script_t"]],"function_references":[["count"]],"select_list":[],"parameter_markers":{"named":["p"],"unnamed_count":0}} -- !query -SELECT parse_command('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END') +SELECT parse_sql('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END') -- !query schema -struct +struct -- !query output {"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["gate"],["dest"],["src_if"],["src_else"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query -SELECT parse_command('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END') +SELECT parse_sql('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END') -- !query schema -struct +struct -- !query output {"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["err_log"],["failing_row"],["main_t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -- !query SELECT - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN @@ -428,7 +450,7 @@ SELECT SELECT audit(row.id), count(*) FROM loop_body; END FOR; END'), '$.table_references') AS table_references, - get_json_object(parse_command( + get_json_object(parse_sql( 'BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN @@ -462,15 +484,15 @@ SELECT -- !query schema struct -- !query output -[["error_log"],["error_source"],["input_names"],["output_names"],["prepared"],["control_flags"],["update_source"],["update_target"],["delete_source"],["delete_target"],["loop_source"],["loop_body"]] [["format_string"],["normalize_name"],["is_valid"],["upper"],["enabled"],["coalesce"],["should_update"],["max"],["expired"],["ready"],["audit"],["count"]] +[["error_log"],["error_source"],["input_names"],["output_names"],["control_flags"],["update_source"],["update_target"],["delete_source"],["delete_target"],["loop_source"],["loop_body"]] [["format_string"],["normalize_name"],["is_valid"],["upper"],["enabled"],["coalesce"],["should_update"],["max"],["expired"],["ready"],["audit"],["count"]] -- !query SELECT - get_json_object(parse_command('BEGIN SELECT 1; END'), '$.statement_identifier') AS statement_identifier, - get_json_object(parse_command('BEGIN SELECT 1; END'), '$.statement_code') AS statement_code, - get_json_object(parse_command('BEGIN SELECT count(a) FROM script_t; END'), '$.table_references') AS table_references, - get_json_object(parse_command('BEGIN SELECT count(a) FROM script_t; END'), '$.function_references') AS function_references + get_json_object(parse_sql('BEGIN SELECT 1; END'), '$.statement_identifier') AS statement_identifier, + get_json_object(parse_sql('BEGIN SELECT 1; END'), '$.statement_code') AS statement_code, + get_json_object(parse_sql('BEGIN SELECT count(a) FROM script_t; END'), '$.table_references') AS table_references, + get_json_object(parse_sql('BEGIN SELECT count(a) FROM script_t; END'), '$.function_references') AS function_references -- !query schema struct -- !query output diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseCommandSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseSqlSuite.scala similarity index 74% rename from sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseCommandSuite.scala rename to sql/core/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseSqlSuite.scala index 84d00a97f5cc..10b8e5156b11 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseCommandSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseSqlSuite.scala @@ -25,33 +25,32 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.types.StringType import org.apache.spark.unsafe.types.UTF8String -class ParseCommandSuite extends SparkFunSuite with ExpressionEvalHelper { +class ParseSqlSuite extends SparkFunSuite with ExpressionEvalHelper { private def evalJson(sql: String): JValue = { - val result = ParseCommand(Literal(sql)).eval().asInstanceOf[UTF8String].toString + val result = ParseSql(Literal(sql)).eval().asInstanceOf[UTF8String].toString parse(result) } - test("parse_command returns JSON for a valid SELECT") { + test("parse_sql returns JSON for a valid SELECT") { val j = evalJson("SELECT 1 AS a") assert(j \ "parse_success" === JBool(true)) assert(j \ "statement_identifier" === JString("SELECT")) assert(j \ "statement_code" === JInt(21)) } - test("parse_command returns null for null input") { - checkEvaluation(ParseCommand(Literal.create(null, StringType)), null) + test("parse_sql returns null for null input") { + checkEvaluation(ParseSql(Literal.create(null, StringType)), null) } - test("parse_command does not throw on syntax error") { + test("parse_sql does not throw on syntax error") { val j = evalJson("NOT A STATEMENT !!!") assert(j \ "parse_success" === JBool(false)) assert(j \ "error" \ "errorClass" === JString("PARSE_SYNTAX_ERROR")) - assert((j \ "error" \ "messageTemplate") != JNothing) } - test("parse_command works with CodegenFallback path") { - val expr = ParseCommand(Literal("INSERT INTO t SELECT 1")) + test("parse_sql works with CodegenFallback path") { + val expr = ParseSql(Literal("INSERT INTO t SELECT 1")) assert(expr.isInstanceOf[CodegenFallback]) val j = evalJson("INSERT INTO t SELECT 1") assert(j \ "statement_identifier" === JString("INSERT")) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala new file mode 100644 index 000000000000..9fadc228663e --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.parser + +import org.json4s._ +import org.json4s.jackson.JsonMethods.parse + +import org.apache.spark.SparkFunSuite + +/** + * Pin Table 39 codes and parser-surface contracts that goldens do not cover. + * Behavioral coverage lives in sql-tests/inputs/parse-sql.sql. + */ +class ParseSqlResultSuite extends SparkFunSuite { + + private def obj(sql: String): JObject = + parse(ParseSqlResult.fromSql(sql)).asInstanceOf[JObject] + + private def tableRefs(sql: String): Set[Seq[String]] = + (obj(sql) \ "table_references").asInstanceOf[JArray].arr.map { + case JArray(parts) => parts.map(_.asInstanceOf[JString].s) + case other => fail(s"unexpected table_references entry: $other") + }.toSet + + test("Table 39 standard code pairs are pinned") { + assert(SqlStatementCodes.Select.statementCode === 21) + assert(SqlStatementCodes.Insert.statementCode === 50) + assert(SqlStatementCodes.DeleteWhere.statementCode === 19) + assert(SqlStatementCodes.UpdateWhere.statementCode === 82) + assert(SqlStatementCodes.Merge.statementCode === 128) + assert(SqlStatementCodes.CreateTable.statementCode === 77) + assert(SqlStatementCodes.CreateView.statementCode === 84) + assert(SqlStatementCodes.DropTable.statementCode === 32) + assert(SqlStatementCodes.AlterTable.statementCode === 4) + assert(SqlStatementCodes.TruncateTable.statementCode === 139) + assert(SqlStatementCodes.Unrecognized.statementCode === 0) + assert(SqlStatementCodes.CacheTable.statementCode < 0) + assert(SqlStatementCodes.BeginEnd.statementCode === -22) + assert(SqlStatementCodes.Explain.statementCode === -23) + assert(SqlStatementCodes.Set.statementCode === -24) + } + + test("SELECT classification uses Table 39 SELECT / code 21") { + val j = obj("SELECT a FROM t") + assert(j \ "parse_success" === JBool(true)) + assert(j \ "statement_identifier" === JString("SELECT")) + assert(j \ "statement_code" === JInt(21)) + } + + test("SparkSqlParser-only statements get Spark codes") { + val explain = obj("EXPLAIN SELECT 1") + assert(explain \ "statement_identifier" === JString("EXPLAIN")) + assert(explain \ "statement_code" === JInt(-23)) + + val set = obj("SET spark.sql.adaptive.enabled=true") + assert(set \ "statement_identifier" === JString("SET")) + assert(set \ "statement_code" === JInt(-24)) + + val addJar = obj("ADD JAR /tmp/x.jar") + assert(addJar \ "statement_identifier" === JString("ADD JAR")) + assert(addJar \ "statement_code" === JInt(-26)) + } + + test("CTE names and correlation aliases are omitted from table_references") { + val sql = + """WITH cte AS (SELECT a FROM hidden_base) + |SELECT a FROM cte""".stripMargin + assert(tableRefs(sql) === Set(Seq("hidden_base"))) + } + + test("CREATE VIEW exposes select_list from the query body") { + val j = obj("CREATE VIEW v AS SELECT a, b FROM t") + assert(j \ "statement_identifier" === JString("CREATE VIEW")) + assert(j \ "statement_code" === JInt(84)) + // View target and source table both count as lineage refs. + assert(tableRefs("CREATE VIEW v AS SELECT a, b FROM t") === + Set(Seq("v"), Seq("t"))) + assert(j \ "select_list" === JArray(List( + JObject("name" -> JArray(List(JString("a")))), + JObject("name" -> JArray(List(JString("b")))) + ))) + assert((j \ "select_list")(0) \ "expression" === JNothing) + } + + test("syntax error returns STANDARD error JSON without throwing") { + val j = obj("SELEC FROM t") + assert(j \ "parse_success" === JBool(false)) + assert(j \ "error" \ "errorClass" === JString("PARSE_SYNTAX_ERROR")) + } +} From 5d64974c0191f11bfaae22c41771d5b3491725ad Mon Sep 17 00:00:00 2001 From: srielau Date: Thu, 13 Aug 2026 13:39:18 +0000 Subject: [PATCH 8/9] [SPARK-58738][SQL] Tighten parse_sql catch, lineage, and query allowlist Narrow errors to ParseException/SqlScriptingException, classify TABLE/VALUES as SELECT, collect only table/view lineage targets, document stock-parser limits, and add CREATE METRIC VIEW code -37. --- .../sql/catalyst/expressions/ParseSql.scala | 9 +-- .../sql/catalyst/parser/ParseSqlResult.scala | 55 +++++++++++++--- .../catalyst/parser/SqlStatementCodes.scala | 17 ++++- .../analyzer-results/parse-sql.sql.out | 29 +++++++++ .../resources/sql-tests/inputs/parse-sql.sql | 9 +++ .../sql-tests/results/parse-sql.sql.out | 33 ++++++++++ .../catalyst/parser/ParseSqlResultSuite.scala | 63 +++++++------------ 7 files changed, 161 insertions(+), 54 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala index 25f26c0a91b7..e479a51555b7 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala @@ -35,10 +35,11 @@ import org.apache.spark.unsafe.types.UTF8String */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = """_FUNC_(sqlStmt) - Parses `sqlStmt` and returns a JSON string describing the - statement (parse success, Table 39 statement identifier/code, table and function - references for lineage, select-list column names, and parameter markers). On - syntax / parse error returns JSON with `parse_success` false, source location, + usage = """_FUNC_(sqlStmt) - Parses `sqlStmt` with the stock Spark SQL parser and + returns a JSON string describing the statement (parse success, Table 39 statement + identifier/code, table and function references for lineage, select-list column + names, and parameter markers). Session parser extensions are not applied. + On syntax / parse error returns JSON with `parse_success` false, source location, and a nested STANDARD error object instead of throwing.""", arguments = """ Arguments: diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala index 892e908a458c..cab6a94da9da 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala @@ -30,19 +30,25 @@ import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.sql.exceptions.SqlScriptingException import org.apache.spark.sql.execution.SparkSqlParser import org.apache.spark.sql.execution.command.{CreateViewCommand, DescribeQueryCommand, ExplainCommand} +import org.apache.spark.sql.execution.datasources.CreateTempViewUsing /** * Parses a SQL statement string and returns a compact JSON description of the * unresolved plan (parse-only; no catalog resolution). * - * Uses [[SparkSqlParser]] so coverage matches the production session parser - * (EXPLAIN / SET / ADD JAR / temp views / etc.). + * Uses a stock [[SparkSqlParser]] (ThreadLocal) so statement coverage matches + * the default production parser (EXPLAIN / SET / ADD JAR / temp views / etc.). + * Session-specific [[org.apache.spark.sql.SparkSessionExtensions]] parser + * wrappers are intentionally not applied: `parse_sql` must evaluate on + * executors without a session, so only the stock parser is available under + * distributed eval. * * On success the JSON includes the statement identifier/code (ISO/IEC * 9075-2:2023 Table 39), table/function references, select-list names, and * parameter markers. On parse failure it returns `parse_success: false` with * source location and a nested STANDARD-format error object, and does not - * throw. Unexpected / internal failures propagate so the function fails. + * throw. Only [[ParseException]] / [[SqlScriptingException]] are converted to + * JSON; unexpected / internal failures propagate so the function fails. */ object ParseSqlResult { @@ -55,13 +61,11 @@ object ParseSqlResult { val plan = parser.get().parsePlan(sql) fromPlan(plan) } catch { - // User-facing parse / scripting failures become JSON; internal errors fail. + // User-facing parse / scripting failures become JSON; everything else fails. case e: ParseException => errorJson(e) case e: SqlScriptingException => errorJson(e) - case e: SparkThrowable with Throwable => - errorJson(e) } } @@ -154,14 +158,30 @@ object ParseSqlResult { } } + /** Multipart name from a table/view-shaped plan node, if any. */ + private def tableOrViewParts(plan: LogicalPlan): Option[Seq[String]] = plan match { + case u: UnresolvedRelation => Some(u.multipartIdentifier) + case u: UnresolvedTable => Some(u.multipartIdentifier) + case u: UnresolvedView => Some(u.multipartIdentifier) + case u: UnresolvedTableOrView => Some(u.multipartIdentifier) + case u: UnresolvedIdentifier => Some(u.nameParts) + case _ => None + } + + private def tableIdentifierParts(id: org.apache.spark.sql.catalyst.TableIdentifier): Seq[String] = + id.catalog.toSeq ++ id.database.toSeq :+ id.table + /** * Collect multipart table/view identifiers for lineage (as written in the * SQL). CTE definition names and correlation aliases are omitted; tables - * referenced inside CTE bodies are still included. Deduplicates while - * preserving first-seen order. + * referenced inside CTE bodies are still included. Function / variable + * identifiers are not collected. Deduplicates while preserving first-seen + * order. */ private def collectTableReferences(plan: LogicalPlan): Seq[Seq[String]] = { val seen = mutable.LinkedHashSet.empty[Seq[String]] + // CTE names are always single-part in the grammar; UnresolvedRelation refs + // to CTEs are likewise single-part, so filtering matches that shape. val cteNames = mutable.HashSet.empty[String] def addCteNames(w: UnresolvedWith): Unit = { @@ -190,7 +210,24 @@ object ParseSqlResult { case u: UnresolvedTable => add(u.multipartIdentifier) case u: UnresolvedView => add(u.multipartIdentifier) case u: UnresolvedTableOrView => add(u.multipartIdentifier) - case u: UnresolvedIdentifier => add(u.nameParts) + // Table/view DDL targets only — not CreateFunction / CreateVariable names. + case c: CreateView => tableOrViewParts(c.child).foreach(add) + case c: CreateViewCommand => add(tableIdentifierParts(c.name)) + case c: CreateTempViewUsing => add(tableIdentifierParts(c.tableIdent)) + case c: CreateTable => tableOrViewParts(c.name).foreach(add) + case c: CreateTableAsSelect => tableOrViewParts(c.name).foreach(add) + case c: ReplaceTable => tableOrViewParts(c.name).foreach(add) + case c: ReplaceTableAsSelect => tableOrViewParts(c.name).foreach(add) + case c: DropTable => tableOrViewParts(c.child).foreach(add) + case c: DropView => tableOrViewParts(c.child).foreach(add) + case c: TruncateTable => tableOrViewParts(c.table).foreach(add) + case c: TruncatePartition => tableOrViewParts(c.table).foreach(add) + case c: CacheTable => + if (c.multipartIdentifier.nonEmpty) add(c.multipartIdentifier) + else tableOrViewParts(c.table).foreach(add) + case c: UncacheTable => tableOrViewParts(c.table).foreach(add) + case c: RefreshTable => tableOrViewParts(c.child).foreach(add) + case c: CommentOnTable => tableOrViewParts(c.table).foreach(add) case _ => } seen.toSeq diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala index 1a62d6a30352..cb2fae1d9af9 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala @@ -17,10 +17,19 @@ package org.apache.spark.sql.catalyst.parser -import org.apache.spark.sql.catalyst.analysis.{UnresolvedExecuteImmediate, UnresolvedHaving} +import org.apache.spark.sql.catalyst.analysis.{ + RelationTimeTravel, + ResolvedInlineTable, + UnresolvedExecuteImmediate, + UnresolvedHaving, + UnresolvedInlineTable, + UnresolvedRelation, + UnresolvedTableValuedFunction +} import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.execution.command._ import org.apache.spark.sql.execution.datasources.{CreateTempViewUsing, RefreshResource} +import org.apache.spark.sql.metricview.logical.CreateMetricView /** * Classification of a parsed SQL statement using ISO/IEC 9075-2:2023 Table 39, @@ -112,6 +121,7 @@ object SqlStatementCodes { val ShowCurrentNamespace: SqlStatementClassification = spark("SHOW CURRENT NAMESPACE", -35) val SetCatalog: SqlStatementClassification = spark("SET CATALOG", -36) + val CreateMetricViewStmt: SqlStatementClassification = spark("CREATE METRIC VIEW", -37) private def spark(identifier: String, code: Int): SqlStatementClassification = { assert(code < 0, s"Spark statement codes must be negative, got $code") @@ -178,6 +188,7 @@ object SqlStatementCodes { case _: RefreshResource => RefreshResourceCmd case _: ShowCatalogsCommand => ShowCatalogs case _: ShowCurrentNamespaceCommand => ShowCurrentNamespace + case _: CreateMetricView | _: CreateMetricViewCommand => CreateMetricViewStmt case _: Command => Unrecognized case p if isQueryPlan(p) => Select case _ => Unrecognized @@ -195,7 +206,9 @@ object SqlStatementCodes { _: OneRowRelation | _: LocalRelation | _: Deduplicate | _: Expand | _: Generate | _: Window | _: Tail | _: Offset | _: LateralJoin | _: UnresolvedHaving | _: CollectMetrics | - _: WithCTE => true + _: WithCTE | _: UnresolvedRelation | _: UnresolvedInlineTable | + _: ResolvedInlineTable | _: RelationTimeTravel | + _: UnresolvedTableValuedFunction => true case _ => false } } diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out index 104c4d5660bc..2ad111a44370 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out @@ -76,6 +76,35 @@ Project [parse_sql(CACHE TABLE t) AS parse_sql(CACHE TABLE t)#x] +- OneRowRelation +-- !query +SELECT parse_sql('TABLE t') +-- !query analysis +Project [parse_sql(TABLE t) AS parse_sql(TABLE t)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('VALUES (1), (2)') +-- !query analysis +Project [parse_sql(VALUES (1), (2)) AS parse_sql(VALUES (1), (2))#x] ++- OneRowRelation + + +-- !query +SELECT get_json_object(parse_sql('CREATE FUNCTION f AS ''x'' USING JAR ''y.jar'''), + '$.table_references') +-- !query analysis +Project [get_json_object(parse_sql(CREATE FUNCTION f AS 'x' USING JAR 'y.jar'), $.table_references) AS get_json_object(parse_sql(CREATE FUNCTION f AS 'x' USING JAR 'y.jar'), $.table_references)#x] ++- OneRowRelation + + +-- !query +SELECT get_json_object(parse_sql('DECLARE VARIABLE x INT'), '$.table_references') +-- !query analysis +Project [get_json_object(parse_sql(DECLARE VARIABLE x INT), $.table_references) AS get_json_object(parse_sql(DECLARE VARIABLE x INT), $.table_references)#x] ++- OneRowRelation + + -- !query SELECT parse_sql('SELECT * FROM t WHERE a = :foo AND b = ?') -- !query analysis diff --git a/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql b/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql index 2b267dbd6e3a..29ed4c67dfe1 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql @@ -22,6 +22,15 @@ SELECT parse_sql('DROP TABLE t'); -- Spark-only statements (negative Table 39 codes) SELECT parse_sql('CACHE TABLE t'); +-- TABLE / VALUES are SELECT-shaped (not Unrecognized) +SELECT parse_sql('TABLE t'); +SELECT parse_sql('VALUES (1), (2)'); + +-- function / variable names are not table_references +SELECT get_json_object(parse_sql('CREATE FUNCTION f AS ''x'' USING JAR ''y.jar'''), + '$.table_references'); +SELECT get_json_object(parse_sql('DECLARE VARIABLE x INT'), '$.table_references'); + -- parameter markers SELECT parse_sql('SELECT * FROM t WHERE a = :foo AND b = ?'); diff --git a/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out b/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out index 699f60bac26e..64b48f3a9bcc 100644 --- a/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out @@ -87,6 +87,39 @@ struct {"parse_success":true,"statement_identifier":"CACHE TABLE","statement_code":-1,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +-- !query +SELECT parse_sql('TABLE t') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT parse_sql('VALUES (1), (2)') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} + + +-- !query +SELECT get_json_object(parse_sql('CREATE FUNCTION f AS ''x'' USING JAR ''y.jar'''), + '$.table_references') +-- !query schema +struct +-- !query output +[] + + +-- !query +SELECT get_json_object(parse_sql('DECLARE VARIABLE x INT'), '$.table_references') +-- !query schema +struct +-- !query output +[] + + -- !query SELECT parse_sql('SELECT * FROM t WHERE a = :foo AND b = ?') -- !query schema diff --git a/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala index 9fadc228663e..f7c412f62545 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala @@ -21,9 +21,10 @@ import org.json4s._ import org.json4s.jackson.JsonMethods.parse import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.internal.SQLConf /** - * Pin Table 39 codes and parser-surface contracts that goldens do not cover. + * Pin Table 39 codes and contracts that goldens do not cover. * Behavioral coverage lives in sql-tests/inputs/parse-sql.sql. */ class ParseSqlResultSuite extends SparkFunSuite { @@ -37,7 +38,7 @@ class ParseSqlResultSuite extends SparkFunSuite { case other => fail(s"unexpected table_references entry: $other") }.toSet - test("Table 39 standard code pairs are pinned") { + test("Table 39 standard and Spark code pairs are pinned") { assert(SqlStatementCodes.Select.statementCode === 21) assert(SqlStatementCodes.Insert.statementCode === 50) assert(SqlStatementCodes.DeleteWhere.statementCode === 19) @@ -53,48 +54,32 @@ class ParseSqlResultSuite extends SparkFunSuite { assert(SqlStatementCodes.BeginEnd.statementCode === -22) assert(SqlStatementCodes.Explain.statementCode === -23) assert(SqlStatementCodes.Set.statementCode === -24) + assert(SqlStatementCodes.CreateMetricViewStmt.statementCode === -37) } - test("SELECT classification uses Table 39 SELECT / code 21") { - val j = obj("SELECT a FROM t") - assert(j \ "parse_success" === JBool(true)) - assert(j \ "statement_identifier" === JString("SELECT")) - assert(j \ "statement_code" === JInt(21)) - } - - test("SparkSqlParser-only statements get Spark codes") { - val explain = obj("EXPLAIN SELECT 1") - assert(explain \ "statement_identifier" === JString("EXPLAIN")) - assert(explain \ "statement_code" === JInt(-23)) - - val set = obj("SET spark.sql.adaptive.enabled=true") - assert(set \ "statement_identifier" === JString("SET")) - assert(set \ "statement_code" === JInt(-24)) - - val addJar = obj("ADD JAR /tmp/x.jar") - assert(addJar \ "statement_identifier" === JString("ADD JAR")) - assert(addJar \ "statement_code" === JInt(-26)) - } + test("TABLE and VALUES classify as SELECT") { + val table = obj("TABLE t") + assert(table \ "statement_identifier" === JString("SELECT")) + assert(table \ "statement_code" === JInt(21)) + assert(tableRefs("TABLE t") === Set(Seq("t"))) - test("CTE names and correlation aliases are omitted from table_references") { - val sql = - """WITH cte AS (SELECT a FROM hidden_base) - |SELECT a FROM cte""".stripMargin - assert(tableRefs(sql) === Set(Seq("hidden_base"))) + // Eager inlining must not flip VALUES between SELECT and Unrecognized. + Seq(true, false).foreach { eager => + SQLConf.withExistingConf(new SQLConf) { + SQLConf.get.setConf(SQLConf.EAGER_EVAL_OF_UNRESOLVED_INLINE_TABLE_ENABLED, eager) + val values = obj("VALUES (1), (2)") + assert(values \ "statement_identifier" === JString("SELECT"), + s"eager=$eager") + assert(values \ "statement_code" === JInt(21), s"eager=$eager") + } + } } - test("CREATE VIEW exposes select_list from the query body") { - val j = obj("CREATE VIEW v AS SELECT a, b FROM t") - assert(j \ "statement_identifier" === JString("CREATE VIEW")) - assert(j \ "statement_code" === JInt(84)) - // View target and source table both count as lineage refs. - assert(tableRefs("CREATE VIEW v AS SELECT a, b FROM t") === - Set(Seq("v"), Seq("t"))) - assert(j \ "select_list" === JArray(List( - JObject("name" -> JArray(List(JString("a")))), - JObject("name" -> JArray(List(JString("b")))) - ))) - assert((j \ "select_list")(0) \ "expression" === JNothing) + test("CREATE FUNCTION and DECLARE VARIABLE are not table_references") { + assert(tableRefs("CREATE FUNCTION f AS 'x' USING JAR 'y.jar'").isEmpty) + assert(tableRefs("DECLARE VARIABLE x INT").isEmpty) + // Contrast: CREATE VIEW still reports the view target. + assert(tableRefs("CREATE VIEW v AS SELECT 1 AS a") === Set(Seq("v"))) } test("syntax error returns STANDARD error JSON without throwing") { From 722eb01080a1626b22f4a7b82bb56ce06d348b7d Mon Sep 17 00:00:00 2001 From: srielau Date: Fri, 14 Aug 2026 10:46:42 +0000 Subject: [PATCH 9/9] [SPARK-58738][SQL] Gate parse_sql and tighten JSON/golden coverage Add spark.sql.parseSql.enabled (default false) while iterating, omit unused JSON fields, fix select-list/error context, and dump full JSON goldens. --- .../apache/spark/sql/internal/SQLConf.scala | 12 + .../sql/catalyst/expressions/ParseSql.scala | 28 +- .../sql/catalyst/parser/ParseSqlResult.scala | 98 ++++-- .../analyzer-results/parse-sql-gating.sql.out | 14 + .../analyzer-results/parse-sql.sql.out | 276 ++++++++--------- .../sql-tests/inputs/parse-sql-gating.sql | 4 + .../resources/sql-tests/inputs/parse-sql.sql | 180 +++++------ .../results/parse-sql-gating.sql.out | 16 + .../sql-tests/results/parse-sql.sql.out | 287 +++++++++--------- .../catalyst/expressions/ParseSqlSuite.scala | 58 +++- .../catalyst/parser/ParseSqlResultSuite.scala | 12 +- .../sql/expressions/ExpressionInfoSuite.scala | 2 + 12 files changed, 553 insertions(+), 434 deletions(-) create mode 100644 sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql-gating.sql.out create mode 100644 sql/core/src/test/resources/sql-tests/inputs/parse-sql-gating.sql create mode 100644 sql/core/src/test/resources/sql-tests/results/parse-sql-gating.sql.out diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 8803546e53f8..208d9a365782 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -5610,6 +5610,16 @@ object SQLConf { .booleanConf .createWithDefault(false) + val PARSE_SQL_ENABLED = + buildConf("spark.sql.parseSql.enabled") + .doc("When true, enables the parse_sql function. This feature is under active " + + "development; the JSON contract may change across releases while the flag remains " + + "off by default.") + .version("5.0.0") + .withBindingPolicy(ConfigBindingPolicy.SESSION) + .booleanConf + .createWithDefault(false) + val ELT_OUTPUT_AS_STRING = buildConf("spark.sql.function.eltOutputAsString") .doc("When this option is set to false and all inputs are binary, `elt` returns " + "an output as binary. Otherwise, it returns as a string.") @@ -9411,6 +9421,8 @@ class SQLConf extends Serializable with Logging with SqlApiConf { def concatBinaryAsString: Boolean = getConf(CONCAT_BINARY_AS_STRING) + def parseSqlEnabled: Boolean = getConf(PARSE_SQL_ENABLED) + def eltOutputAsString: Boolean = getConf(ELT_OUTPUT_AS_STRING) def validatePartitionColumns: Boolean = getConf(VALIDATE_PARTITION_COLUMNS) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala index e479a51555b7..3c85b21bda8a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala @@ -17,9 +17,11 @@ package org.apache.spark.sql.catalyst.expressions -import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, FunctionRegistryBase} +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, FunctionRegistryBase, TypeCheckResult} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.parser.ParseSqlResult +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.types.StringTypeWithCollation import org.apache.spark.sql.types.{AbstractDataType, DataType, StringType} import org.apache.spark.unsafe.types.UTF8String @@ -30,8 +32,9 @@ import org.apache.spark.unsafe.types.UTF8String * parameters), or a STANDARD-format error object when the statement does not * parse. * - * Designed for batch evaluation over DataFrames of SQL text. User-facing parse - * errors become JSON; unexpected internal failures propagate. + * Behind [[SQLConf.PARSE_SQL_ENABLED]] while the JSON contract is still + * evolving. Designed for batch evaluation over DataFrames of SQL text. + * User-facing parse errors become JSON; unexpected internal failures propagate. */ // scalastyle:off line.size.limit @ExpressionDescription( @@ -39,8 +42,9 @@ import org.apache.spark.unsafe.types.UTF8String returns a JSON string describing the statement (parse success, Table 39 statement identifier/code, table and function references for lineage, select-list column names, and parameter markers). Session parser extensions are not applied. - On syntax / parse error returns JSON with `parse_success` false, source location, - and a nested STANDARD error object instead of throwing.""", + Requires spark.sql.parseSql.enabled=true. On syntax / parse error returns JSON + with `parse_success` false, source location, and a nested STANDARD error object + instead of throwing.""", arguments = """ Arguments: * sqlStmt - A SQL statement string to parse. @@ -49,7 +53,7 @@ import org.apache.spark.unsafe.types.UTF8String examples = """ Examples: > SELECT _FUNC_('SELECT a, b FROM t'); - {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"]},{"name":["b"]}],"parameter_markers":{"named":[],"unnamed_count":0}} + {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"select_list":[{"name":["a"]},{"name":["b"]}]} > SELECT get_json_object(_FUNC_('SELEC'), '$.error.errorClass'); PARSE_SYNTAX_ERROR """, @@ -72,6 +76,18 @@ case class ParseSql(child: Expression) override def inputTypes: Seq[AbstractDataType] = Seq(StringTypeWithCollation(supportsTrimCollation = true)) + override def checkInputDataTypes(): TypeCheckResult = { + if (!SQLConf.get.parseSqlEnabled) { + throw new AnalysisException( + errorClass = "FEATURE_NOT_ENABLED", + messageParameters = Map( + "featureName" -> "parse_sql", + "configKey" -> SQLConf.PARSE_SQL_ENABLED.key, + "configValue" -> "true")) + } + super.checkInputDataTypes() + } + override def nullSafeEval(input: Any): Any = { val sql = input.asInstanceOf[UTF8String].toString UTF8String.fromString(ParseSqlResult.fromSql(sql)) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala index cab6a94da9da..94287bc486ba 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala @@ -26,7 +26,7 @@ import org.apache.spark.{ErrorMessageFormat, SparkThrowable, SparkThrowableHelpe import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.catalyst.trees.Origin +import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin, SQLQueryContext} import org.apache.spark.sql.exceptions.SqlScriptingException import org.apache.spark.sql.execution.SparkSqlParser import org.apache.spark.sql.execution.command.{CreateViewCommand, DescribeQueryCommand, ExplainCommand} @@ -43,12 +43,14 @@ import org.apache.spark.sql.execution.datasources.CreateTempViewUsing * executors without a session, so only the stock parser is available under * distributed eval. * - * On success the JSON includes the statement identifier/code (ISO/IEC - * 9075-2:2023 Table 39), table/function references, select-list names, and - * parameter markers. On parse failure it returns `parse_success: false` with - * source location and a nested STANDARD-format error object, and does not - * throw. Only [[ParseException]] / [[SqlScriptingException]] are converted to - * JSON; unexpected / internal failures propagate so the function fails. + * On success the JSON always includes `parse_success`, the statement + * identifier/code (ISO/IEC 9075-2:2023 Table 39), and omits unused optional + * fields (`table_references`, `function_references`, `select_list`, + * `parameter_markers`) when empty. On parse failure it returns + * `parse_success: false` with source location and a nested STANDARD-format + * error object, and does not throw. Only [[ParseException]] / + * [[SqlScriptingException]] are converted to JSON; unexpected / internal + * failures propagate so the function fails. */ object ParseSqlResult { @@ -58,8 +60,17 @@ object ParseSqlResult { /** Parse `sql` and render the JSON result string. */ def fromSql(sql: String): String = { try { - val plan = parser.get().parsePlan(sql) - fromPlan(plan) + // Do not inherit the outer query's origin from the parse_sql expression. + // Errors and parsed nodes must refer to the SQL string passed to this function. + val origin = if (sql.nonEmpty) { + Origin(startIndex = Some(0), stopIndex = Some(sql.length - 1), sqlText = Some(sql)) + } else { + Origin(sqlText = Some(sql)) + } + CurrentOrigin.withOrigin(origin) { + val plan = parser.get().parsePlan(sql) + fromPlan(plan) + } } catch { // User-facing parse / scripting failures become JSON; everything else fails. case e: ParseException => @@ -76,12 +87,20 @@ object ParseSqlResult { fields += "parse_success" -> JBool(true) fields += "statement_identifier" -> JString(classification.statementIdentifier) fields += "statement_code" -> JInt(classification.statementCode) - fields += "table_references" -> JArray( - collectTableReferences(plan).map(partsToJArray).toList) - fields += "function_references" -> JArray( - collectFunctionReferences(plan).map(partsToJArray).toList) - fields += "select_list" -> JArray(collectSelectList(plan).toList) - fields += "parameter_markers" -> parameterMarkersJson(plan) + // Omit unused collections / markers so consumers can treat absence as empty. + val tables = collectTableReferences(plan) + if (tables.nonEmpty) { + fields += "table_references" -> JArray(tables.map(partsToJArray).toList) + } + val functions = collectFunctionReferences(plan) + if (functions.nonEmpty) { + fields += "function_references" -> JArray(functions.map(partsToJArray).toList) + } + val selectList = collectSelectList(plan) + if (selectList.nonEmpty) { + fields += "select_list" -> JArray(selectList.toList) + } + parameterMarkersJson(plan).foreach(markers => fields += "parameter_markers" -> markers) compact(render(JObject(fields.toList))) } @@ -94,12 +113,29 @@ object ParseSqlResult { case _ => None } val locationFields = origin.toSeq.flatMap(originFields) + val contextFields = if (errorObj.obj.exists(_._1 == "queryContext")) { + Nil + } else { + origin.toSeq.flatMap(queryContextField) + } compact(render(JObject( "parse_success" -> JBool(false), - "error" -> JObject(errorObj.obj ++ locationFields) + "error" -> JObject(errorObj.obj ++ contextFields ++ locationFields) ))) } + private def queryContextField(origin: Origin): Option[JField] = origin.context match { + case context: SQLQueryContext if context.isValid => + Some("queryContext" -> JArray(List(JObject( + "objectType" -> JString(context.objectType), + "objectName" -> JString(context.objectName), + "startIndex" -> JInt(context.startIndex + 1), + "stopIndex" -> JInt(context.stopIndex + 1), + "fragment" -> JString(context.fragment) + )))) + case _ => None + } + private def originFields(origin: Origin): Seq[JField] = Seq( origin.line.map(line => "line" -> JInt(line)), origin.startPosition.map(position => "position" -> JInt(position))).flatten @@ -279,6 +315,17 @@ object ParseSqlResult { case ExplainCommand(logicalPlan, _) => primaryQueryPlan(logicalPlan) case DescribeQueryCommand(_, queryPlan) => primaryQueryPlan(queryPlan) case SubqueryAlias(_, child) => primaryQueryPlan(child) + case Sort(_, _, child, _) => primaryQueryPlan(child) + case Filter(_, child) => primaryQueryPlan(child) + case UnresolvedHaving(_, child) => primaryQueryPlan(child) + case UnresolvedQualify(_, child) => primaryQueryPlan(child) + case Distinct(child) => primaryQueryPlan(child) + case GlobalLimit(_, child) => primaryQueryPlan(child) + case LocalLimit(_, child) => primaryQueryPlan(child) + case Offset(_, child) => primaryQueryPlan(child) + case Repartition(_, _, child) => primaryQueryPlan(child) + case RepartitionByExpression(_, child, _, _) => primaryQueryPlan(child) + case Sample(_, _, _, _, child, _) => primaryQueryPlan(child) case other => other } @@ -296,7 +343,12 @@ object ParseSqlResult { JObject("name" -> partsToJArray(Seq(other.name))) } - private def parameterMarkersJson(plan: LogicalPlan): JObject = { + /** + * Parameter-marker object, or None when the statement has neither named nor + * positional markers. Nested empty members are also omitted: `named` only + * when non-empty, `unnamed_count` only when > 0. + */ + private def parameterMarkersJson(plan: LogicalPlan): Option[JObject] = { val named = mutable.LinkedHashSet.empty[String] var unnamedCount = 0 def visitExpr(e: Expression): Unit = e.foreach { @@ -307,9 +359,13 @@ object ParseSqlResult { foreachPlanDeep(plan) { p => foreachExpressionDeep(p)(visitExpr) } - JObject( - "named" -> JArray(named.toList.map(JString)), - "unnamed_count" -> JInt(unnamedCount) - ) + if (named.isEmpty && unnamedCount == 0) { + None + } else { + val fields = mutable.ListBuffer.empty[JField] + if (named.nonEmpty) fields += "named" -> JArray(named.toList.map(JString)) + if (unnamedCount > 0) fields += "unnamed_count" -> JInt(unnamedCount) + Some(JObject(fields.toList)) + } } } diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql-gating.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql-gating.sql.out new file mode 100644 index 000000000000..dbf89961fa3a --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql-gating.sql.out @@ -0,0 +1,14 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +SELECT parse_sql('SELECT 1') +-- !query analysis +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "FEATURE_NOT_ENABLED", + "sqlState" : "56038", + "messageParameters" : { + "configKey" : "spark.sql.parseSql.enabled", + "configValue" : "true", + "featureName" : "parse_sql" + } +} diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out index 2ad111a44370..733b41c24b88 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out @@ -20,6 +20,19 @@ Project [parse_sql(SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2) AS par +- OneRowRelation +-- !query +SELECT + get_json_object(result, '$.statement_identifier') AS statement_identifier, + get_json_object(result, '$.table_references[0][0]') AS first_table, + get_json_object(result, '$.select_list[1].name[0]') AS second_column +FROM (SELECT parse_sql('SELECT a, b FROM t') AS result) +-- !query analysis +Project [get_json_object(result#x, $.statement_identifier) AS statement_identifier#x, get_json_object(result#x, $.table_references[0][0]) AS first_table#x, get_json_object(result#x, $.select_list[1].name[0]) AS second_column#x] ++- SubqueryAlias __auto_generated_subquery_name + +- Project [parse_sql(SELECT a, b FROM t) AS result#x] + +- OneRowRelation + + -- !query SELECT parse_sql('INSERT INTO t SELECT 1') -- !query analysis @@ -91,17 +104,16 @@ Project [parse_sql(VALUES (1), (2)) AS parse_sql(VALUES (1), (2))#x] -- !query -SELECT get_json_object(parse_sql('CREATE FUNCTION f AS ''x'' USING JAR ''y.jar'''), - '$.table_references') +SELECT parse_sql('CREATE FUNCTION f AS ''x'' USING JAR ''y.jar''') -- !query analysis -Project [get_json_object(parse_sql(CREATE FUNCTION f AS 'x' USING JAR 'y.jar'), $.table_references) AS get_json_object(parse_sql(CREATE FUNCTION f AS 'x' USING JAR 'y.jar'), $.table_references)#x] +Project [parse_sql(CREATE FUNCTION f AS 'x' USING JAR 'y.jar') AS parse_sql(CREATE FUNCTION f AS 'x' USING JAR 'y.jar')#x] +- OneRowRelation -- !query -SELECT get_json_object(parse_sql('DECLARE VARIABLE x INT'), '$.table_references') +SELECT parse_sql('DECLARE VARIABLE x INT') -- !query analysis -Project [get_json_object(parse_sql(DECLARE VARIABLE x INT), $.table_references) AS get_json_object(parse_sql(DECLARE VARIABLE x INT), $.table_references)#x] +Project [parse_sql(DECLARE VARIABLE x INT) AS parse_sql(DECLARE VARIABLE x INT)#x] +- OneRowRelation @@ -217,115 +229,111 @@ Project [parse_sql(CREATE TABLE defaults ( -- !query -SELECT get_json_object(parse_sql('SELEC FROM t'), '$.parse_success') +SELECT parse_sql('SELEC FROM t') -- !query analysis -Project [get_json_object(parse_sql(SELEC FROM t), $.parse_success) AS get_json_object(parse_sql(SELEC FROM t), $.parse_success)#x] +Project [parse_sql(SELEC FROM t) AS parse_sql(SELEC FROM t)#x] +- OneRowRelation -- !query -SELECT get_json_object(parse_sql('SELEC FROM t'), '$.error.errorClass') --- !query analysis -Project [get_json_object(parse_sql(SELEC FROM t), $.error.errorClass) AS get_json_object(parse_sql(SELEC FROM t), $.error.errorClass)#x] -+- OneRowRelation - - --- !query -SELECT get_json_object(parse_sql('SELEC FROM t'), '$.error.sqlState') +SELECT + get_json_object(result, '$.parse_success') AS parse_success, + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM (SELECT parse_sql('SELEC FROM t') AS result) -- !query analysis -Project [get_json_object(parse_sql(SELEC FROM t), $.error.sqlState) AS get_json_object(parse_sql(SELEC FROM t), $.error.sqlState)#x] -+- OneRowRelation +Project [get_json_object(result#x, $.parse_success) AS parse_success#x, get_json_object(result#x, $.error.errorClass) AS error_class#x, get_json_object(result#x, $.error.queryContext[0].fragment) AS fragment#x] ++- SubqueryAlias __auto_generated_subquery_name + +- Project [parse_sql(SELEC FROM t) AS result#x] + +- OneRowRelation -- !query -SELECT - get_json_object(parse_sql( +SELECT parse_sql( 'SELECT * FROM t ORDER BY a - CLUSTER BY b'), '$.error.errorClass') AS error_class, - get_json_object(parse_sql( -'SELECT * + CLUSTER BY b') +-- !query analysis +Project [parse_sql(SELECT * FROM t ORDER BY a - CLUSTER BY b'), '$.error.line') AS line, - get_json_object(parse_sql( -'SELECT * + CLUSTER BY b) AS parse_sql(SELECT * FROM t ORDER BY a - CLUSTER BY b'), '$.error.position') AS position, - get_json_object(parse_sql( + CLUSTER BY b)#x] ++- OneRowRelation + + +-- !query +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].startIndex') AS start_index +FROM ( + SELECT parse_sql( 'SELECT * FROM t ORDER BY a - CLUSTER BY b'), '$.error.queryContext[0].startIndex') AS start_index + CLUSTER BY b') AS result +) -- !query analysis -Project [get_json_object(parse_sql(SELECT * - FROM t - ORDER BY a - CLUSTER BY b), $.error.errorClass) AS error_class#x, get_json_object(parse_sql(SELECT * - FROM t - ORDER BY a - CLUSTER BY b), $.error.line) AS line#x, get_json_object(parse_sql(SELECT * - FROM t - ORDER BY a - CLUSTER BY b), $.error.position) AS position#x, get_json_object(parse_sql(SELECT * +Project [get_json_object(result#x, $.error.errorClass) AS error_class#x, get_json_object(result#x, $.error.line) AS line#x, get_json_object(result#x, $.error.position) AS position#x, get_json_object(result#x, $.error.queryContext[0].startIndex) AS start_index#x] ++- SubqueryAlias __auto_generated_subquery_name + +- Project [parse_sql(SELECT * FROM t ORDER BY a - CLUSTER BY b), $.error.queryContext[0].startIndex) AS start_index#x] -+- OneRowRelation + CLUSTER BY b) AS result#x] + +- OneRowRelation -- !query -SELECT get_json_object(parse_sql(''), '$.error.errorClass') +SELECT parse_sql('') -- !query analysis -Project [get_json_object(parse_sql(), $.error.errorClass) AS get_json_object(parse_sql(), $.error.errorClass)#x] +Project [parse_sql() AS parse_sql()#x] +- OneRowRelation -- !query -SELECT get_json_object(parse_sql('USE bad-name'), '$.error.errorClass') +SELECT parse_sql('USE bad-name') -- !query analysis -Project [get_json_object(parse_sql(USE bad-name), $.error.errorClass) AS get_json_object(parse_sql(USE bad-name), $.error.errorClass)#x] +Project [parse_sql(USE bad-name) AS parse_sql(USE bad-name)#x] +- OneRowRelation -- !query -SELECT get_json_object( - parse_sql('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c'), - '$.error.errorClass') +SELECT parse_sql('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c') -- !query analysis -Project [get_json_object(parse_sql(WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c), $.error.errorClass) AS get_json_object(parse_sql(WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c), $.error.errorClass)#x] +Project [parse_sql(WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c) AS parse_sql(WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c)#x] +- OneRowRelation -- !query -SELECT get_json_object( - parse_sql('MERGE INTO target USING source ON target.id = source.id'), - '$.error.errorClass') +SELECT parse_sql('MERGE INTO target USING source ON target.id = source.id') -- !query analysis -Project [get_json_object(parse_sql(MERGE INTO target USING source ON target.id = source.id), $.error.errorClass) AS get_json_object(parse_sql(MERGE INTO target USING source ON target.id = source.id), $.error.errorClass)#x] +Project [parse_sql(MERGE INTO target USING source ON target.id = source.id) AS parse_sql(MERGE INTO target USING source ON target.id = source.id)#x] +- OneRowRelation -- !query -SELECT get_json_object(parse_sql('EXPLAIN SELECT 1'), '$.statement_identifier') +SELECT parse_sql('EXPLAIN SELECT 1') -- !query analysis -Project [get_json_object(parse_sql(EXPLAIN SELECT 1), $.statement_identifier) AS get_json_object(parse_sql(EXPLAIN SELECT 1), $.statement_identifier)#x] +Project [parse_sql(EXPLAIN SELECT 1) AS parse_sql(EXPLAIN SELECT 1)#x] +- OneRowRelation -- !query -SELECT get_json_object(parse_sql('SET spark.sql.adaptive.enabled=true'), '$.statement_code') +SELECT parse_sql('SET spark.sql.adaptive.enabled=true') -- !query analysis -Project [get_json_object(parse_sql(SET spark.sql.adaptive.enabled=true), $.statement_code) AS get_json_object(parse_sql(SET spark.sql.adaptive.enabled=true), $.statement_code)#x] +Project [parse_sql(SET spark.sql.adaptive.enabled=true) AS parse_sql(SET spark.sql.adaptive.enabled=true)#x] +- OneRowRelation -- !query -SELECT get_json_object(parse_sql('ADD JAR /tmp/x.jar'), '$.statement_identifier') +SELECT parse_sql('ADD JAR /tmp/x.jar') -- !query analysis -Project [get_json_object(parse_sql(ADD JAR /tmp/x.jar), $.statement_identifier) AS get_json_object(parse_sql(ADD JAR /tmp/x.jar), $.statement_identifier)#x] +Project [parse_sql(ADD JAR /tmp/x.jar) AS parse_sql(ADD JAR /tmp/x.jar)#x] +- OneRowRelation @@ -337,89 +345,102 @@ Project [parse_sql(CREATE VIEW v AS SELECT a, b FROM t) AS parse_sql(CREATE VIEW -- !query -SELECT get_json_object( - parse_sql('SELECT 1 AS IDENTIFIER(''alias.field'')'), - '$.error.errorClass') +SELECT parse_sql('SELECT 1 AS IDENTIFIER(''alias.field'')') -- !query analysis -Project [get_json_object(parse_sql(SELECT 1 AS IDENTIFIER('alias.field')), $.error.errorClass) AS get_json_object(parse_sql(SELECT 1 AS IDENTIFIER('alias.field')), $.error.errorClass)#x] +Project [parse_sql(SELECT 1 AS IDENTIFIER('alias.field')) AS parse_sql(SELECT 1 AS IDENTIFIER('alias.field'))#x] +- OneRowRelation -- !query -SELECT get_json_object( - parse_sql('SELECT DATE ''not-a-date'''), - '$.error.errorClass') +SELECT parse_sql('SELECT DATE ''not-a-date''') -- !query analysis -Project [get_json_object(parse_sql(SELECT DATE 'not-a-date'), $.error.errorClass) AS get_json_object(parse_sql(SELECT DATE 'not-a-date'), $.error.errorClass)#x] +Project [parse_sql(SELECT DATE 'not-a-date') AS parse_sql(SELECT DATE 'not-a-date')#x] +- OneRowRelation -- !query -SELECT - get_json_object(parse_sql( -'BEGIN - SELECT 1; - SELEC 2; - END'), '$.error.errorClass') AS error_class, - get_json_object(parse_sql( -'BEGIN - SELECT 1; - SELEC 2; - END'), '$.error.line') AS line, - get_json_object(parse_sql( +SELECT parse_sql( 'BEGIN SELECT 1; SELEC 2; - END'), '$.error.position') AS position + END') -- !query analysis -Project [get_json_object(parse_sql(BEGIN +Project [parse_sql(BEGIN SELECT 1; SELEC 2; - END), $.error.errorClass) AS error_class#x, get_json_object(parse_sql(BEGIN + END) AS parse_sql(BEGIN SELECT 1; SELEC 2; - END), $.error.line) AS line#x, get_json_object(parse_sql(BEGIN - SELECT 1; - SELEC 2; - END), $.error.position) AS position#x] + END)#x] +- OneRowRelation -- !query SELECT - get_json_object(parse_sql( + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM ( + SELECT parse_sql( 'BEGIN - lbl_begin: BEGIN - SELECT 1; - END lbl_end; - END'), '$.error.errorClass') AS error_class, - get_json_object(parse_sql( + SELECT 1; + SELEC 2; + END') AS result +) +-- !query analysis +Project [get_json_object(result#x, $.error.errorClass) AS error_class#x, get_json_object(result#x, $.error.line) AS line#x, get_json_object(result#x, $.error.position) AS position#x, get_json_object(result#x, $.error.queryContext[0].fragment) AS fragment#x] ++- SubqueryAlias __auto_generated_subquery_name + +- Project [parse_sql(BEGIN + SELECT 1; + SELEC 2; + END) AS result#x] + +- OneRowRelation + + +-- !query +SELECT parse_sql( 'BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; - END'), '$.error.line') AS line, - get_json_object(parse_sql( -'BEGIN + END') +-- !query analysis +Project [parse_sql(BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; - END'), '$.error.position') AS position --- !query analysis -Project [get_json_object(parse_sql(BEGIN + END) AS parse_sql(BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; - END), $.error.errorClass) AS error_class#x, get_json_object(parse_sql(BEGIN + END)#x] ++- OneRowRelation + + +-- !query +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM ( + SELECT parse_sql( +'BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; - END), $.error.line) AS line#x, get_json_object(parse_sql(BEGIN + END') AS result +) +-- !query analysis +Project [get_json_object(result#x, $.error.errorClass) AS error_class#x, get_json_object(result#x, $.error.line) AS line#x, get_json_object(result#x, $.error.position) AS position#x, get_json_object(result#x, $.error.queryContext[0].fragment) AS fragment#x] ++- SubqueryAlias __auto_generated_subquery_name + +- Project [parse_sql(BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; - END), $.error.position) AS position#x] -+- OneRowRelation + END) AS result#x] + +- OneRowRelation -- !query @@ -463,39 +484,7 @@ Project [parse_sql(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO -- !query -SELECT - get_json_object(parse_sql( -'BEGIN - DECLARE EXIT HANDLER FOR SQLEXCEPTION - BEGIN - INSERT INTO error_log - SELECT format_string(''%s'', message) FROM error_source; - END; - - WITH prepared AS ( - SELECT id, normalize_name(name) AS name - FROM input_names - WHERE is_valid(id) - ) - INSERT INTO output_names - SELECT id, upper(name) FROM prepared; - - IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN - UPDATE update_target - SET value = coalesce((SELECT max(value) FROM update_source), 0) - WHERE should_update(id); - ELSE - DELETE FROM delete_target - WHERE id IN (SELECT id FROM delete_source WHERE expired(ts)); - END IF; - - FOR row AS - SELECT id FROM loop_source WHERE ready(id) - DO - SELECT audit(row.id), count(*) FROM loop_body; - END FOR; - END'), '$.table_references') AS table_references, - get_json_object(parse_sql( +SELECT parse_sql( 'BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN @@ -525,9 +514,9 @@ SELECT DO SELECT audit(row.id), count(*) FROM loop_body; END FOR; - END'), '$.function_references') AS function_references + END') -- !query analysis -Project [get_json_object(parse_sql(BEGIN +Project [parse_sql(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO error_log @@ -556,7 +545,7 @@ Project [get_json_object(parse_sql(BEGIN DO SELECT audit(row.id), count(*) FROM loop_body; END FOR; - END), $.table_references) AS table_references#x, get_json_object(parse_sql(BEGIN + END) AS parse_sql(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO error_log @@ -585,16 +574,5 @@ Project [get_json_object(parse_sql(BEGIN DO SELECT audit(row.id), count(*) FROM loop_body; END FOR; - END), $.function_references) AS function_references#x] -+- OneRowRelation - - --- !query -SELECT - get_json_object(parse_sql('BEGIN SELECT 1; END'), '$.statement_identifier') AS statement_identifier, - get_json_object(parse_sql('BEGIN SELECT 1; END'), '$.statement_code') AS statement_code, - get_json_object(parse_sql('BEGIN SELECT count(a) FROM script_t; END'), '$.table_references') AS table_references, - get_json_object(parse_sql('BEGIN SELECT count(a) FROM script_t; END'), '$.function_references') AS function_references --- !query analysis -Project [get_json_object(parse_sql(BEGIN SELECT 1; END), $.statement_identifier) AS statement_identifier#x, get_json_object(parse_sql(BEGIN SELECT 1; END), $.statement_code) AS statement_code#x, get_json_object(parse_sql(BEGIN SELECT count(a) FROM script_t; END), $.table_references) AS table_references#x, get_json_object(parse_sql(BEGIN SELECT count(a) FROM script_t; END), $.function_references) AS function_references#x] + END)#x] +- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/inputs/parse-sql-gating.sql b/sql/core/src/test/resources/sql-tests/inputs/parse-sql-gating.sql new file mode 100644 index 000000000000..7f2c147afc0f --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/parse-sql-gating.sql @@ -0,0 +1,4 @@ +-- parse_sql is off by default while the JSON contract is still evolving. +--SET spark.sql.parseSql.enabled=false + +SELECT parse_sql('SELECT 1'); diff --git a/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql b/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql index 29ed4c67dfe1..a11de25e8e5d 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql @@ -1,5 +1,7 @@ -- End-to-end coverage for parse_sql (SPARK-58738). -- Returns compact JSON for parse-only statement analysis via SparkSqlParser. +-- Off by default while the JSON contract is still evolving. +--SET spark.sql.parseSql.enabled=true -- null input SELECT parse_sql(NULL); @@ -8,6 +10,13 @@ SELECT parse_sql(NULL); SELECT parse_sql('SELECT a, b FROM t'); SELECT parse_sql('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2'); +-- JSON-path access over one shared successful parse result +SELECT + get_json_object(result, '$.statement_identifier') AS statement_identifier, + get_json_object(result, '$.table_references[0][0]') AS first_table, + get_json_object(result, '$.select_list[1].name[0]') AS second_column +FROM (SELECT parse_sql('SELECT a, b FROM t') AS result); + -- DML SELECT parse_sql('INSERT INTO t SELECT 1'); SELECT parse_sql('DELETE FROM t WHERE a = 1'); @@ -27,9 +36,8 @@ SELECT parse_sql('TABLE t'); SELECT parse_sql('VALUES (1), (2)'); -- function / variable names are not table_references -SELECT get_json_object(parse_sql('CREATE FUNCTION f AS ''x'' USING JAR ''y.jar'''), - '$.table_references'); -SELECT get_json_object(parse_sql('DECLARE VARIABLE x INT'), '$.table_references'); +SELECT parse_sql('CREATE FUNCTION f AS ''x'' USING JAR ''y.jar'''); +SELECT parse_sql('DECLARE VARIABLE x INT'); -- parameter markers SELECT parse_sql('SELECT * FROM t WHERE a = :foo AND b = ?'); @@ -74,95 +82,99 @@ SELECT parse_sql( normalized STRING DEFAULT upper(''x'') )'); --- syntax error: never throws; STANDARD error nested under parse_success=false -SELECT get_json_object(parse_sql('SELEC FROM t'), '$.parse_success'); -SELECT get_json_object(parse_sql('SELEC FROM t'), '$.error.errorClass'); -SELECT get_json_object(parse_sql('SELEC FROM t'), '$.error.sqlState'); +-- syntax error: dump the complete STANDARD error, including query context +SELECT parse_sql('SELEC FROM t'); --- source location from a multiline parse-time validation error +-- JSON-path access over one shared parse result SELECT - get_json_object(parse_sql( -'SELECT * - FROM t - ORDER BY a - CLUSTER BY b'), '$.error.errorClass') AS error_class, - get_json_object(parse_sql( -'SELECT * - FROM t - ORDER BY a - CLUSTER BY b'), '$.error.line') AS line, - get_json_object(parse_sql( + get_json_object(result, '$.parse_success') AS parse_success, + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM (SELECT parse_sql('SELEC FROM t') AS result); + +-- full multiline parse-time validation error, including context and location +SELECT parse_sql( 'SELECT * FROM t ORDER BY a - CLUSTER BY b'), '$.error.position') AS position, - get_json_object(parse_sql( + CLUSTER BY b'); + +-- JSON-path access over one shared multiline parse result +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].startIndex') AS start_index +FROM ( + SELECT parse_sql( 'SELECT * FROM t ORDER BY a - CLUSTER BY b'), '$.error.queryContext[0].startIndex') AS start_index; + CLUSTER BY b') AS result +); -- parse-only validation errors beyond PARSE_SYNTAX_ERROR -SELECT get_json_object(parse_sql(''), '$.error.errorClass'); -SELECT get_json_object(parse_sql('USE bad-name'), '$.error.errorClass'); -SELECT get_json_object( - parse_sql('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c'), - '$.error.errorClass'); -SELECT get_json_object( - parse_sql('MERGE INTO target USING source ON target.id = source.id'), - '$.error.errorClass'); -SELECT get_json_object(parse_sql('EXPLAIN SELECT 1'), '$.statement_identifier'); -SELECT get_json_object(parse_sql('SET spark.sql.adaptive.enabled=true'), '$.statement_code'); -SELECT get_json_object(parse_sql('ADD JAR /tmp/x.jar'), '$.statement_identifier'); +SELECT parse_sql(''); +SELECT parse_sql('USE bad-name'); +SELECT parse_sql('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c'); +SELECT parse_sql('MERGE INTO target USING source ON target.id = source.id'); +SELECT parse_sql('EXPLAIN SELECT 1'); +SELECT parse_sql('SET spark.sql.adaptive.enabled=true'); +SELECT parse_sql('ADD JAR /tmp/x.jar'); SELECT parse_sql('CREATE VIEW v AS SELECT a, b FROM t'); -SELECT get_json_object( - parse_sql('SELECT 1 AS IDENTIFIER(''alias.field'')'), - '$.error.errorClass'); -SELECT get_json_object( - parse_sql('SELECT DATE ''not-a-date'''), - '$.error.errorClass'); +SELECT parse_sql('SELECT 1 AS IDENTIFIER(''alias.field'')'); +SELECT parse_sql('SELECT DATE ''not-a-date'''); -- location for an error inside a multiline script --QUERY-DELIMITER-START -SELECT - get_json_object(parse_sql( -'BEGIN - SELECT 1; - SELEC 2; - END'), '$.error.errorClass') AS error_class, - get_json_object(parse_sql( +SELECT parse_sql( 'BEGIN SELECT 1; SELEC 2; - END'), '$.error.line') AS line, - get_json_object(parse_sql( + END'); +--QUERY-DELIMITER-END + +-- JSON-path access over one shared scripting parse result +--QUERY-DELIMITER-START +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM ( + SELECT parse_sql( 'BEGIN SELECT 1; SELEC 2; - END'), '$.error.position') AS position; + END') AS result +); --QUERY-DELIMITER-END -- location for a SQL scripting semantic validation error --QUERY-DELIMITER-START -SELECT - get_json_object(parse_sql( -'BEGIN - lbl_begin: BEGIN - SELECT 1; - END lbl_end; - END'), '$.error.errorClass') AS error_class, - get_json_object(parse_sql( +SELECT parse_sql( 'BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; - END'), '$.error.line') AS line, - get_json_object(parse_sql( + END'); +--QUERY-DELIMITER-END + +-- JSON-path access over one shared scripting validation result +--QUERY-DELIMITER-START +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM ( + SELECT parse_sql( 'BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; - END'), '$.error.position') AS position; + END') AS result +); --QUERY-DELIMITER-END -- batch over a column of SQL text @@ -190,42 +202,9 @@ SELECT parse_sql('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT SELECT parse_sql('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END'); --QUERY-DELIMITER-END --- Complex, genuinely multiline script. Extract collections to keep the --- expected output focused on complete tree walking. +-- Complex, genuinely multiline script: dump the complete JSON result. --QUERY-DELIMITER-START -SELECT - get_json_object(parse_sql( -'BEGIN - DECLARE EXIT HANDLER FOR SQLEXCEPTION - BEGIN - INSERT INTO error_log - SELECT format_string(''%s'', message) FROM error_source; - END; - - WITH prepared AS ( - SELECT id, normalize_name(name) AS name - FROM input_names - WHERE is_valid(id) - ) - INSERT INTO output_names - SELECT id, upper(name) FROM prepared; - - IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN - UPDATE update_target - SET value = coalesce((SELECT max(value) FROM update_source), 0) - WHERE should_update(id); - ELSE - DELETE FROM delete_target - WHERE id IN (SELECT id FROM delete_source WHERE expired(ts)); - END IF; - - FOR row AS - SELECT id FROM loop_source WHERE ready(id) - DO - SELECT audit(row.id), count(*) FROM loop_body; - END FOR; - END'), '$.table_references') AS table_references, - get_json_object(parse_sql( +SELECT parse_sql( 'BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN @@ -255,14 +234,5 @@ SELECT DO SELECT audit(row.id), count(*) FROM loop_body; END FOR; - END'), '$.function_references') AS function_references; ---QUERY-DELIMITER-END - --- extract key fields from a script for readable assertions ---QUERY-DELIMITER-START -SELECT - get_json_object(parse_sql('BEGIN SELECT 1; END'), '$.statement_identifier') AS statement_identifier, - get_json_object(parse_sql('BEGIN SELECT 1; END'), '$.statement_code') AS statement_code, - get_json_object(parse_sql('BEGIN SELECT count(a) FROM script_t; END'), '$.table_references') AS table_references, - get_json_object(parse_sql('BEGIN SELECT count(a) FROM script_t; END'), '$.function_references') AS function_references; + END'); --QUERY-DELIMITER-END diff --git a/sql/core/src/test/resources/sql-tests/results/parse-sql-gating.sql.out b/sql/core/src/test/resources/sql-tests/results/parse-sql-gating.sql.out new file mode 100644 index 000000000000..41de434b003b --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/parse-sql-gating.sql.out @@ -0,0 +1,16 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +SELECT parse_sql('SELECT 1') +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "FEATURE_NOT_ENABLED", + "sqlState" : "56038", + "messageParameters" : { + "configKey" : "spark.sql.parseSql.enabled", + "configValue" : "true", + "featureName" : "parse_sql" + } +} diff --git a/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out b/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out index 64b48f3a9bcc..1f27bceace01 100644 --- a/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out @@ -12,7 +12,7 @@ SELECT parse_sql('SELECT a, b FROM t') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"]},{"name":["b"]}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"select_list":[{"name":["a"]},{"name":["b"]}]} -- !query @@ -20,7 +20,19 @@ SELECT parse_sql('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["cat","ns","t1"],["t2"]],"function_references":[["db","my_func"],["count"]],"select_list":[{"name":[]},{"name":[]}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["cat","ns","t1"],["t2"]],"function_references":[["db","my_func"],["count"]],"select_list":[{"name":[]},{"name":[]}]} + + +-- !query +SELECT + get_json_object(result, '$.statement_identifier') AS statement_identifier, + get_json_object(result, '$.table_references[0][0]') AS first_table, + get_json_object(result, '$.select_list[1].name[0]') AS second_column +FROM (SELECT parse_sql('SELECT a, b FROM t') AS result) +-- !query schema +struct +-- !query output +SELECT t b -- !query @@ -28,7 +40,7 @@ SELECT parse_sql('INSERT INTO t SELECT 1') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"table_references":[["t"]],"function_references":[],"select_list":[{"name":[]}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"table_references":[["t"]],"select_list":[{"name":[]}]} -- !query @@ -36,7 +48,7 @@ SELECT parse_sql('DELETE FROM t WHERE a = 1') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"DELETE WHERE","statement_code":19,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"DELETE WHERE","statement_code":19,"table_references":[["t"]]} -- !query @@ -44,7 +56,7 @@ SELECT parse_sql('UPDATE t SET a = 1 WHERE b = 2') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"UPDATE WHERE","statement_code":82,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"UPDATE WHERE","statement_code":82,"table_references":[["t"]]} -- !query @@ -52,7 +64,7 @@ SELECT parse_sql('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"MERGE","statement_code":128,"table_references":[["t"],["s"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"MERGE","statement_code":128,"table_references":[["t"],["s"]]} -- !query @@ -60,7 +72,7 @@ SELECT parse_sql('CREATE TABLE t (a INT)') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"table_references":[["t"]]} -- !query @@ -68,7 +80,7 @@ SELECT parse_sql('CREATE TABLE t AS SELECT 1 AS a') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["a"]}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"table_references":[["t"]],"select_list":[{"name":["a"]}]} -- !query @@ -76,7 +88,7 @@ SELECT parse_sql('DROP TABLE t') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"DROP TABLE","statement_code":32,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"DROP TABLE","statement_code":32,"table_references":[["t"]]} -- !query @@ -84,7 +96,7 @@ SELECT parse_sql('CACHE TABLE t') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"CACHE TABLE","statement_code":-1,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"CACHE TABLE","statement_code":-1,"table_references":[["t"]]} -- !query @@ -92,7 +104,7 @@ SELECT parse_sql('TABLE t') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]]} -- !query @@ -100,24 +112,23 @@ SELECT parse_sql('VALUES (1), (2)') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21} -- !query -SELECT get_json_object(parse_sql('CREATE FUNCTION f AS ''x'' USING JAR ''y.jar'''), - '$.table_references') +SELECT parse_sql('CREATE FUNCTION f AS ''x'' USING JAR ''y.jar''') -- !query schema -struct +struct -- !query output -[] +{"parse_success":true,"statement_identifier":"CREATE ROUTINE","statement_code":14} -- !query -SELECT get_json_object(parse_sql('DECLARE VARIABLE x INT'), '$.table_references') +SELECT parse_sql('DECLARE VARIABLE x INT') -- !query schema -struct +struct -- !query output -[] +{"parse_success":true,"statement_identifier":"DECLARE VARIABLE","statement_code":-8} -- !query @@ -125,7 +136,7 @@ SELECT parse_sql('SELECT * FROM t WHERE a = :foo AND b = ?') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"function_references":[],"select_list":[{"name":["*"]}],"parameter_markers":{"named":["foo"],"unnamed_count":1}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"select_list":[{"name":["*"]}],"parameter_markers":{"named":["foo"],"unnamed_count":1}} -- !query @@ -133,7 +144,7 @@ SELECT parse_sql('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["hidden_base"]],"function_references":[],"select_list":[{"name":["a"]}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["hidden_base"]],"select_list":[{"name":["a"]}]} -- !query @@ -141,7 +152,7 @@ SELECT parse_sql('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["scalar_src"],["exists_src"],["outer_t"]],"function_references":[["max"]],"select_list":[{"name":["m"]},{"name":["t","a"]}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["scalar_src"],["exists_src"],["outer_t"]],"function_references":[["max"]],"select_list":[{"name":["m"]},{"name":["t","a"]}]} -- !query @@ -168,7 +179,7 @@ struct 0) > 0 ORDER BY greatest(t.a, 1)):string> -- !query output -{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["scalar_t"],["left_t"],["right_t"]],"function_references":[["greatest"],["count_if"],["coalesce"],["sum"],["abs"],["lower"],["length"],["startswith"],["max"],["range"],["hash"]],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["scalar_t"],["left_t"],["right_t"]],"function_references":[["greatest"],["count_if"],["coalesce"],["sum"],["abs"],["lower"],["length"],["startswith"],["max"],["range"],["hash"]],"select_list":[{"name":[]},{"name":[]}]} -- !query @@ -197,7 +208,7 @@ struct -- !query output -{"parse_success":true,"statement_identifier":"MERGE","statement_code":128,"table_references":[["target"],["source"]],"function_references":[["hash"],["should_update"],["coalesce"],["upper"],["lower"],["normalize_name"],["is_valid"]],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"MERGE","statement_code":128,"table_references":[["target"],["source"]],"function_references":[["hash"],["should_update"],["coalesce"],["upper"],["lower"],["normalize_name"],["is_valid"]]} -- !query @@ -212,55 +223,57 @@ struct -- !query output -{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"table_references":[["defaults"]],"function_references":[["current_date"],["upper"]],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"table_references":[["defaults"]],"function_references":[["current_date"],["upper"]]} -- !query -SELECT get_json_object(parse_sql('SELEC FROM t'), '$.parse_success') +SELECT parse_sql('SELEC FROM t') -- !query schema -struct +struct -- !query output -false +{"parse_success":false,"error":{"errorClass":"PARSE_SYNTAX_ERROR","messageTemplate":"Syntax error at or near .","sqlState":"42601","messageParameters":{"error":"'SELEC'","hint":""},"queryContext":[{"objectType":"","objectName":"","startIndex":1,"stopIndex":12,"fragment":"SELEC FROM t"}],"line":1,"position":0}} -- !query -SELECT get_json_object(parse_sql('SELEC FROM t'), '$.error.errorClass') +SELECT + get_json_object(result, '$.parse_success') AS parse_success, + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM (SELECT parse_sql('SELEC FROM t') AS result) -- !query schema -struct +struct -- !query output -PARSE_SYNTAX_ERROR +false PARSE_SYNTAX_ERROR SELEC FROM t -- !query -SELECT get_json_object(parse_sql('SELEC FROM t'), '$.error.sqlState') +SELECT parse_sql( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b') -- !query schema -struct +struct -- !query output -42601 +{"parse_success":false,"error":{"errorClass":"UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES","messageTemplate":"The feature is not supported: Combination of ORDER BY/SORT BY/DISTRIBUTE BY/CLUSTER BY.","sqlState":"0A000","queryContext":[{"objectType":"","objectName":"","startIndex":19,"stopIndex":42,"fragment":"ORDER BY a\n CLUSTER BY b"}],"line":3,"position":1}} -- !query SELECT - get_json_object(parse_sql( + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].startIndex') AS start_index +FROM ( + SELECT parse_sql( 'SELECT * FROM t ORDER BY a - CLUSTER BY b'), '$.error.errorClass') AS error_class, - get_json_object(parse_sql( -'SELECT * - FROM t - ORDER BY a - CLUSTER BY b'), '$.error.line') AS line, - get_json_object(parse_sql( -'SELECT * - FROM t - ORDER BY a - CLUSTER BY b'), '$.error.position') AS position, - get_json_object(parse_sql( -'SELECT * - FROM t - ORDER BY a - CLUSTER BY b'), '$.error.queryContext[0].startIndex') AS start_index + CLUSTER BY b') AS result +) -- !query schema struct -- !query output @@ -268,63 +281,59 @@ UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES 3 1 19 -- !query -SELECT get_json_object(parse_sql(''), '$.error.errorClass') +SELECT parse_sql('') -- !query schema -struct +struct -- !query output -PARSE_EMPTY_STATEMENT +{"parse_success":false,"error":{"errorClass":"PARSE_EMPTY_STATEMENT","messageTemplate":"Syntax error, unexpected empty statement.","sqlState":"42617","line":1,"position":0}} -- !query -SELECT get_json_object(parse_sql('USE bad-name'), '$.error.errorClass') +SELECT parse_sql('USE bad-name') -- !query schema -struct +struct -- !query output -INVALID_IDENTIFIER +{"parse_success":false,"error":{"errorClass":"INVALID_IDENTIFIER","messageTemplate":"The unquoted identifier is invalid and must be back quoted as: ``.\nUnquoted identifiers can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_').\nUnquoted identifiers must also not start with a digit.\nDifferent data sources and meta stores may impose additional restrictions on valid identifiers.","sqlState":"42602","messageParameters":{"ident":"bad-name"},"queryContext":[{"objectType":"","objectName":"","startIndex":1,"stopIndex":12,"fragment":"USE bad-name"}],"line":1,"position":7}} -- !query -SELECT get_json_object( - parse_sql('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c'), - '$.error.errorClass') +SELECT parse_sql('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c') -- !query schema -struct +struct -- !query output -DUPLICATED_CTE_NAMES +{"parse_success":false,"error":{"errorClass":"DUPLICATED_CTE_NAMES","messageTemplate":"CTE definition can't have duplicate names: .","sqlState":"42602","messageParameters":{"duplicateNames":"`c`"},"queryContext":[{"objectType":"","objectName":"","startIndex":1,"stopIndex":53,"fragment":"WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c"}],"line":1,"position":0}} -- !query -SELECT get_json_object( - parse_sql('MERGE INTO target USING source ON target.id = source.id'), - '$.error.errorClass') +SELECT parse_sql('MERGE INTO target USING source ON target.id = source.id') -- !query schema -struct +struct -- !query output -MERGE_WITHOUT_WHEN +{"parse_success":false,"error":{"errorClass":"MERGE_WITHOUT_WHEN","messageTemplate":"There must be at least one WHEN clause in a MERGE statement.","sqlState":"42601","queryContext":[{"objectType":"","objectName":"","startIndex":1,"stopIndex":55,"fragment":"MERGE INTO target USING source ON target.id = source.id"}],"line":1,"position":0}} -- !query -SELECT get_json_object(parse_sql('EXPLAIN SELECT 1'), '$.statement_identifier') +SELECT parse_sql('EXPLAIN SELECT 1') -- !query schema -struct +struct -- !query output -EXPLAIN +{"parse_success":true,"statement_identifier":"EXPLAIN","statement_code":-23,"select_list":[{"name":[]}]} -- !query -SELECT get_json_object(parse_sql('SET spark.sql.adaptive.enabled=true'), '$.statement_code') +SELECT parse_sql('SET spark.sql.adaptive.enabled=true') -- !query schema -struct +struct -- !query output --24 +{"parse_success":true,"statement_identifier":"SET","statement_code":-24} -- !query -SELECT get_json_object(parse_sql('ADD JAR /tmp/x.jar'), '$.statement_identifier') +SELECT parse_sql('ADD JAR /tmp/x.jar') -- !query schema -struct +struct -- !query output -ADD JAR +{"parse_success":true,"statement_identifier":"ADD JAR","statement_code":-26} -- !query @@ -332,76 +341,97 @@ SELECT parse_sql('CREATE VIEW v AS SELECT a, b FROM t') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"CREATE VIEW","statement_code":84,"table_references":[["v"],["t"]],"function_references":[],"select_list":[{"name":["a"]},{"name":["b"]}],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"CREATE VIEW","statement_code":84,"table_references":[["v"],["t"]],"select_list":[{"name":["a"]},{"name":["b"]}]} -- !query -SELECT get_json_object( - parse_sql('SELECT 1 AS IDENTIFIER(''alias.field'')'), - '$.error.errorClass') +SELECT parse_sql('SELECT 1 AS IDENTIFIER(''alias.field'')') -- !query schema -struct +struct -- !query output -IDENTIFIER_TOO_MANY_NAME_PARTS +{"parse_success":false,"error":{"errorClass":"IDENTIFIER_TOO_MANY_NAME_PARTS","messageTemplate":" is not a valid identifier as it has more than name parts.","sqlState":"42601","messageParameters":{"identifier":"`alias`.`field`","limit":"1"},"queryContext":[{"objectType":"","objectName":"","startIndex":8,"stopIndex":37,"fragment":"1 AS IDENTIFIER('alias.field')"}],"line":1,"position":12}} -- !query -SELECT get_json_object( - parse_sql('SELECT DATE ''not-a-date'''), - '$.error.errorClass') +SELECT parse_sql('SELECT DATE ''not-a-date''') -- !query schema -struct +struct -- !query output -INVALID_TYPED_LITERAL +{"parse_success":false,"error":{"errorClass":"INVALID_TYPED_LITERAL","messageTemplate":"The value of the typed literal is invalid: .","sqlState":"42604","messageParameters":{"value":"'not-a-date'","valueType":"\"DATE\""},"queryContext":[{"objectType":"","objectName":"","startIndex":8,"stopIndex":24,"fragment":"DATE 'not-a-date'"}],"line":1,"position":7}} -- !query -SELECT - get_json_object(parse_sql( +SELECT parse_sql( 'BEGIN SELECT 1; SELEC 2; - END'), '$.error.errorClass') AS error_class, - get_json_object(parse_sql( -'BEGIN + END') +-- !query schema +struct +-- !query output +{"parse_success":false,"error":{"errorClass":"PARSE_SYNTAX_ERROR","messageTemplate":"Syntax error at or near .","sqlState":"42601","messageParameters":{"error":"'2'","hint":""},"queryContext":[{"objectType":"","objectName":"","startIndex":1,"stopIndex":35,"fragment":"BEGIN\n SELECT 1;\n SELEC 2;\n END"}],"line":3,"position":9}} + + +-- !query +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM ( + SELECT parse_sql( 'BEGIN SELECT 1; SELEC 2; - END'), '$.error.position') AS position + END') AS result +) -- !query schema -struct +struct -- !query output -PARSE_SYNTAX_ERROR 3 9 +PARSE_SYNTAX_ERROR 3 9 BEGIN + SELECT 1; + SELEC 2; + END -- !query -SELECT - get_json_object(parse_sql( +SELECT parse_sql( 'BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; - END'), '$.error.errorClass') AS error_class, - get_json_object(parse_sql( -'BEGIN + END') +-- !query schema +struct +-- !query output +{"parse_success":false,"error":{"errorClass":"LABELS_MISMATCH","messageTemplate":"Begin label does not match the end label .","sqlState":"42K0L","messageParameters":{"beginLabel":"`lbl_begin`","endLabel":"`lbl_end`"},"queryContext":[{"objectType":"","objectName":"","startIndex":10,"stopIndex":19,"fragment":"lbl_begin:"}],"line":2,"position":3}} + + +-- !query +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM ( + SELECT parse_sql( 'BEGIN lbl_begin: BEGIN SELECT 1; END lbl_end; - END'), '$.error.position') AS position + END') AS result +) -- !query schema -struct +struct -- !query output -LABELS_MISMATCH 2 3 +LABELS_MISMATCH 2 3 lbl_begin: -- !query @@ -413,9 +443,9 @@ AS t(sql_text) -- !query schema struct -- !query output -CACHE TABLE t {"parse_success":true,"statement_identifier":"CACHE TABLE","statement_code":-1,"table_references":[["t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} -INSERT INTO t SELECT 1 {"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"table_references":[["t"]],"function_references":[],"select_list":[{"name":[]}],"parameter_markers":{"named":[],"unnamed_count":0}} -SELECT 1 {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[],"function_references":[],"select_list":[{"name":[]}],"parameter_markers":{"named":[],"unnamed_count":0}} +CACHE TABLE t {"parse_success":true,"statement_identifier":"CACHE TABLE","statement_code":-1,"table_references":[["t"]]} +INSERT INTO t SELECT 1 {"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"table_references":[["t"]],"select_list":[{"name":[]}]} +SELECT 1 {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"select_list":[{"name":[]}]} -- !query @@ -423,7 +453,7 @@ SELECT parse_sql('BEGIN SELECT 1; END') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22} -- !query @@ -431,7 +461,7 @@ SELECT parse_sql('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END') -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["script_t"]],"function_references":[["count"]],"select_list":[],"parameter_markers":{"named":["p"],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["script_t"]],"function_references":[["count"]],"parameter_markers":{"named":["p"]}} -- !query @@ -439,7 +469,7 @@ SELECT parse_sql('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["gate"],["dest"],["src_if"],["src_else"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["gate"],["dest"],["src_if"],["src_else"]]} -- !query @@ -447,12 +477,11 @@ SELECT parse_sql('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO -- !query schema struct -- !query output -{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["err_log"],["failing_row"],["main_t"]],"function_references":[],"select_list":[],"parameter_markers":{"named":[],"unnamed_count":0}} +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["err_log"],["failing_row"],["main_t"]]} -- !query -SELECT - get_json_object(parse_sql( +SELECT parse_sql( 'BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN @@ -482,13 +511,13 @@ SELECT DO SELECT audit(row.id), count(*) FROM loop_body; END FOR; - END'), '$.table_references') AS table_references, - get_json_object(parse_sql( -'BEGIN + END') +-- !query schema +struct --- !query output -[["error_log"],["error_source"],["input_names"],["output_names"],["control_flags"],["update_source"],["update_target"],["delete_source"],["delete_target"],["loop_source"],["loop_body"]] [["format_string"],["normalize_name"],["is_valid"],["upper"],["enabled"],["coalesce"],["should_update"],["max"],["expired"],["ready"],["audit"],["count"]] - - --- !query -SELECT - get_json_object(parse_sql('BEGIN SELECT 1; END'), '$.statement_identifier') AS statement_identifier, - get_json_object(parse_sql('BEGIN SELECT 1; END'), '$.statement_code') AS statement_code, - get_json_object(parse_sql('BEGIN SELECT count(a) FROM script_t; END'), '$.table_references') AS table_references, - get_json_object(parse_sql('BEGIN SELECT count(a) FROM script_t; END'), '$.function_references') AS function_references --- !query schema -struct + END):string> -- !query output -BEGIN END -22 [["script_t"]] [["count"]] +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["error_log"],["error_source"],["input_names"],["output_names"],["control_flags"],["update_source"],["update_target"],["delete_source"],["delete_target"],["loop_source"],["loop_body"]],"function_references":[["format_string"],["normalize_name"],["is_valid"],["upper"],["enabled"],["coalesce"],["should_update"],["max"],["expired"],["ready"],["audit"],["count"]]} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseSqlSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseSqlSuite.scala index 10b8e5156b11..f44a409a0d64 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseSqlSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseSqlSuite.scala @@ -21,38 +21,70 @@ import org.json4s._ import org.json4s.jackson.JsonMethods.parse import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.plans.SQLHelper +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StringType import org.apache.spark.unsafe.types.UTF8String -class ParseSqlSuite extends SparkFunSuite with ExpressionEvalHelper { +class ParseSqlSuite extends SparkFunSuite with ExpressionEvalHelper with SQLHelper { private def evalJson(sql: String): JValue = { val result = ParseSql(Literal(sql)).eval().asInstanceOf[UTF8String].toString parse(result) } + test("parse_sql is disabled by default") { + assert(!SQLConf.get.parseSqlEnabled) + checkError( + exception = intercept[AnalysisException] { + ParseSql(Literal("SELECT 1")).checkInputDataTypes() + }, + condition = "FEATURE_NOT_ENABLED", + parameters = Map( + "featureName" -> "parse_sql", + "configKey" -> SQLConf.PARSE_SQL_ENABLED.key, + "configValue" -> "true")) + } + + test("parse_sql type check succeeds when enabled") { + withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> "true") { + assert(ParseSql(Literal("SELECT 1")).checkInputDataTypes() === + TypeCheckResult.TypeCheckSuccess) + } + } + test("parse_sql returns JSON for a valid SELECT") { - val j = evalJson("SELECT 1 AS a") - assert(j \ "parse_success" === JBool(true)) - assert(j \ "statement_identifier" === JString("SELECT")) - assert(j \ "statement_code" === JInt(21)) + withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> "true") { + val j = evalJson("SELECT 1 AS a") + assert(j \ "parse_success" === JBool(true)) + assert(j \ "statement_identifier" === JString("SELECT")) + assert(j \ "statement_code" === JInt(21)) + } } test("parse_sql returns null for null input") { - checkEvaluation(ParseSql(Literal.create(null, StringType)), null) + withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> "true") { + checkEvaluation(ParseSql(Literal.create(null, StringType)), null) + } } test("parse_sql does not throw on syntax error") { - val j = evalJson("NOT A STATEMENT !!!") - assert(j \ "parse_success" === JBool(false)) - assert(j \ "error" \ "errorClass" === JString("PARSE_SYNTAX_ERROR")) + withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> "true") { + val j = evalJson("NOT A STATEMENT !!!") + assert(j \ "parse_success" === JBool(false)) + assert(j \ "error" \ "errorClass" === JString("PARSE_SYNTAX_ERROR")) + } } test("parse_sql works with CodegenFallback path") { - val expr = ParseSql(Literal("INSERT INTO t SELECT 1")) - assert(expr.isInstanceOf[CodegenFallback]) - val j = evalJson("INSERT INTO t SELECT 1") - assert(j \ "statement_identifier" === JString("INSERT")) + withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> "true") { + val expr = ParseSql(Literal("INSERT INTO t SELECT 1")) + assert(expr.isInstanceOf[CodegenFallback]) + val j = evalJson("INSERT INTO t SELECT 1") + assert(j \ "statement_identifier" === JString("INSERT")) + } } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala index f7c412f62545..62b0e4cd5c74 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala @@ -33,10 +33,14 @@ class ParseSqlResultSuite extends SparkFunSuite { parse(ParseSqlResult.fromSql(sql)).asInstanceOf[JObject] private def tableRefs(sql: String): Set[Seq[String]] = - (obj(sql) \ "table_references").asInstanceOf[JArray].arr.map { - case JArray(parts) => parts.map(_.asInstanceOf[JString].s) - case other => fail(s"unexpected table_references entry: $other") - }.toSet + obj(sql) \ "table_references" match { + case JNothing => Set.empty + case JArray(arr) => arr.map { + case JArray(parts) => parts.map(_.asInstanceOf[JString].s) + case other => fail(s"unexpected table_references entry: $other") + }.toSet + case other => fail(s"unexpected table_references: $other") + } test("Table 39 standard and Spark code pairs are pinned") { assert(SqlStatementCodes.Select.statementCode === 21) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala index dab117fa6713..7b5402119b40 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala @@ -292,6 +292,8 @@ class ExpressionInfoSuite extends SharedSparkSession { val clonedSpark = spark.cloneSession() // Coalescing partitions can change result order, so disable it. clonedSpark.conf.set(SQLConf.COALESCE_PARTITIONS_ENABLED.key, false) + // parse_sql examples require the experimental feature flag. + clonedSpark.conf.set(SQLConf.PARSE_SQL_ENABLED.key, true) val info = clonedSpark.sessionState.catalog.lookupFunctionInfo(funcId) val className = info.getClassName if (!ignoreSet.contains(className)) {