diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 8803546e53f84..208d9a3657822 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -5610,6 +5610,16 @@ object SQLConf { .booleanConf .createWithDefault(false) + val PARSE_SQL_ENABLED = + buildConf("spark.sql.parseSql.enabled") + .doc("When true, enables the parse_sql function. This feature is under active " + + "development; the JSON contract may change across releases while the flag remains " + + "off by default.") + .version("5.0.0") + .withBindingPolicy(ConfigBindingPolicy.SESSION) + .booleanConf + .createWithDefault(false) + val ELT_OUTPUT_AS_STRING = buildConf("spark.sql.function.eltOutputAsString") .doc("When this option is set to false and all inputs are binary, `elt` returns " + "an output as binary. Otherwise, it returns as a string.") @@ -9411,6 +9421,8 @@ class SQLConf extends Serializable with Logging with SqlApiConf { def concatBinaryAsString: Boolean = getConf(CONCAT_BINARY_AS_STRING) + def parseSqlEnabled: Boolean = getConf(PARSE_SQL_ENABLED) + def eltOutputAsString: Boolean = getConf(ELT_OUTPUT_AS_STRING) def validatePartitionColumns: Boolean = getConf(VALIDATE_PARTITION_COLUMNS) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala new file mode 100644 index 0000000000000..3c85b21bda8a9 --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/ParseSql.scala @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.expressions + +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, FunctionRegistryBase, TypeCheckResult} +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.parser.ParseSqlResult +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.types.StringTypeWithCollation +import org.apache.spark.sql.types.{AbstractDataType, DataType, StringType} +import org.apache.spark.unsafe.types.UTF8String + +/** + * Parses a SQL statement string and returns a compact JSON description of the + * unresolved statement (identifier/code, lineage references, select-list names, + * parameters), or a STANDARD-format error object when the statement does not + * parse. + * + * Behind [[SQLConf.PARSE_SQL_ENABLED]] while the JSON contract is still + * evolving. Designed for batch evaluation over DataFrames of SQL text. + * User-facing parse errors become JSON; unexpected internal failures propagate. + */ +// scalastyle:off line.size.limit +@ExpressionDescription( + usage = """_FUNC_(sqlStmt) - Parses `sqlStmt` with the stock Spark SQL parser and + returns a JSON string describing the statement (parse success, Table 39 statement + identifier/code, table and function references for lineage, select-list column + names, and parameter markers). Session parser extensions are not applied. + Requires spark.sql.parseSql.enabled=true. On syntax / parse error returns JSON + with `parse_success` false, source location, and a nested STANDARD error object + instead of throwing.""", + arguments = """ + Arguments: + * sqlStmt - A SQL statement string to parse. + An expression that evaluates to a string. + """, + examples = """ + Examples: + > SELECT _FUNC_('SELECT a, b FROM t'); + {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"select_list":[{"name":["a"]},{"name":["b"]}]} + > SELECT get_json_object(_FUNC_('SELEC'), '$.error.errorClass'); + PARSE_SYNTAX_ERROR + """, + group = "misc_funcs", + since = "5.0.0") +// scalastyle:on line.size.limit +case class ParseSql(child: Expression) + extends UnaryExpression + with ImplicitCastInputTypes + with CodegenFallback { + + override def prettyName: String = "parse_sql" + + override def nullable: Boolean = true + + override def nullIntolerant: Boolean = true + + override def dataType: DataType = StringType + + override def inputTypes: Seq[AbstractDataType] = + Seq(StringTypeWithCollation(supportsTrimCollation = true)) + + override def checkInputDataTypes(): TypeCheckResult = { + if (!SQLConf.get.parseSqlEnabled) { + throw new AnalysisException( + errorClass = "FEATURE_NOT_ENABLED", + messageParameters = Map( + "featureName" -> "parse_sql", + "configKey" -> SQLConf.PARSE_SQL_ENABLED.key, + "configValue" -> "true")) + } + super.checkInputDataTypes() + } + + override def nullSafeEval(input: Any): Any = { + val sql = input.asInstanceOf[UTF8String].toString + UTF8String.fromString(ParseSqlResult.fromSql(sql)) + } + + override protected def withNewChildInternal(newChild: Expression): ParseSql = + copy(child = newChild) +} + +object ParseSql { + /** Register the builtin with a session function registry. */ + def register(registry: FunctionRegistry): Unit = { + val (info, builder) = FunctionRegistryBase.build[ParseSql]("parse_sql", Some("5.0.0")) + registry.registerFunction( + FunctionRegistry.builtinFunctionIdentifier("parse_sql"), + info, + builder) + } +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala new file mode 100644 index 0000000000000..94287bc486baa --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResult.scala @@ -0,0 +1,371 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.parser + +import scala.collection.mutable + +import org.json4s._ +import org.json4s.jackson.JsonMethods.{compact, parse => parseJson, render} + +import org.apache.spark.{ErrorMessageFormat, SparkThrowable, SparkThrowableHelper} +import org.apache.spark.sql.catalyst.analysis._ +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.catalyst.trees.{CurrentOrigin, Origin, SQLQueryContext} +import org.apache.spark.sql.exceptions.SqlScriptingException +import org.apache.spark.sql.execution.SparkSqlParser +import org.apache.spark.sql.execution.command.{CreateViewCommand, DescribeQueryCommand, ExplainCommand} +import org.apache.spark.sql.execution.datasources.CreateTempViewUsing + +/** + * Parses a SQL statement string and returns a compact JSON description of the + * unresolved plan (parse-only; no catalog resolution). + * + * Uses a stock [[SparkSqlParser]] (ThreadLocal) so statement coverage matches + * the default production parser (EXPLAIN / SET / ADD JAR / temp views / etc.). + * Session-specific [[org.apache.spark.sql.SparkSessionExtensions]] parser + * wrappers are intentionally not applied: `parse_sql` must evaluate on + * executors without a session, so only the stock parser is available under + * distributed eval. + * + * On success the JSON always includes `parse_success`, the statement + * identifier/code (ISO/IEC 9075-2:2023 Table 39), and omits unused optional + * fields (`table_references`, `function_references`, `select_list`, + * `parameter_markers`) when empty. On parse failure it returns + * `parse_success: false` with source location and a nested STANDARD-format + * error object, and does not throw. Only [[ParseException]] / + * [[SqlScriptingException]] are converted to JSON; unexpected / internal + * failures propagate so the function fails. + */ +object ParseSqlResult { + + private val parser: ThreadLocal[SparkSqlParser] = + ThreadLocal.withInitial(() => new SparkSqlParser()) + + /** Parse `sql` and render the JSON result string. */ + def fromSql(sql: String): String = { + try { + // Do not inherit the outer query's origin from the parse_sql expression. + // Errors and parsed nodes must refer to the SQL string passed to this function. + val origin = if (sql.nonEmpty) { + Origin(startIndex = Some(0), stopIndex = Some(sql.length - 1), sqlText = Some(sql)) + } else { + Origin(sqlText = Some(sql)) + } + CurrentOrigin.withOrigin(origin) { + val plan = parser.get().parsePlan(sql) + fromPlan(plan) + } + } catch { + // User-facing parse / scripting failures become JSON; everything else fails. + case e: ParseException => + errorJson(e) + case e: SqlScriptingException => + errorJson(e) + } + } + + /** Build success JSON from an already-parsed unresolved plan. */ + def fromPlan(plan: LogicalPlan): String = { + val classification = SqlStatementCodes.classify(plan) + val fields = mutable.ListBuffer.empty[JField] + fields += "parse_success" -> JBool(true) + fields += "statement_identifier" -> JString(classification.statementIdentifier) + fields += "statement_code" -> JInt(classification.statementCode) + // Omit unused collections / markers so consumers can treat absence as empty. + val tables = collectTableReferences(plan) + if (tables.nonEmpty) { + fields += "table_references" -> JArray(tables.map(partsToJArray).toList) + } + val functions = collectFunctionReferences(plan) + if (functions.nonEmpty) { + fields += "function_references" -> JArray(functions.map(partsToJArray).toList) + } + val selectList = collectSelectList(plan) + if (selectList.nonEmpty) { + fields += "select_list" -> JArray(selectList.toList) + } + parameterMarkersJson(plan).foreach(markers => fields += "parameter_markers" -> markers) + compact(render(JObject(fields.toList))) + } + + private def errorJson(e: SparkThrowable with Throwable): String = { + val errorObj = parseJson( + SparkThrowableHelper.getMessage(e, ErrorMessageFormat.STANDARD)).asInstanceOf[JObject] + val origin = e match { + case p: ParseException => Some(p.start) + case s: SqlScriptingException => Some(s.origin) + case _ => None + } + val locationFields = origin.toSeq.flatMap(originFields) + val contextFields = if (errorObj.obj.exists(_._1 == "queryContext")) { + Nil + } else { + origin.toSeq.flatMap(queryContextField) + } + compact(render(JObject( + "parse_success" -> JBool(false), + "error" -> JObject(errorObj.obj ++ contextFields ++ locationFields) + ))) + } + + private def queryContextField(origin: Origin): Option[JField] = origin.context match { + case context: SQLQueryContext if context.isValid => + Some("queryContext" -> JArray(List(JObject( + "objectType" -> JString(context.objectType), + "objectName" -> JString(context.objectName), + "startIndex" -> JInt(context.startIndex + 1), + "stopIndex" -> JInt(context.stopIndex + 1), + "fragment" -> JString(context.fragment) + )))) + case _ => None + } + + private def originFields(origin: Origin): Seq[JField] = Seq( + origin.line.map(line => "line" -> JInt(line)), + origin.startPosition.map(position => "position" -> JInt(position))).flatten + + private def partsToJArray(parts: Seq[String]): JArray = + JArray(parts.map(JString).toList) + + /** + * Walk expressions in all product fields, including wrappers such as column + * definitions that [[LogicalPlan.expressions]] does not descend into. + */ + private def foreachExpressionDeep(plan: LogicalPlan)(f: Expression => Unit): Unit = { + def visit(value: Any): Unit = value match { + case e: Expression => f(e) + case _: LogicalPlan => + case values: Iterable[_] => values.foreach(visit) + case value: Product => value.productIterator.foreach(visit) + case _ => + } + plan.productIterator.foreach(visit) + } + + /** + * Deep plan walk covering tree slots that standard `collect` / + * `collectWithSubqueries` miss: + * - [[UnresolvedWith]] CTE definitions (`innerChildren`, not `children`) + * - [[InsertIntoStatement]].table (non-child plan slot) + * - [[SingleStatement]].parsedPlan (children expose only nested children) + * - [[CompoundBody]].handlers (not in `children`) + * - [[SimpleCaseStatement]].elseBody (not in `children`) + * Nested expression subqueries are still covered by `foreachWithSubqueries`. + */ + private def foreachPlanDeep(plan: LogicalPlan)(f: LogicalPlan => Unit): Unit = { + plan.foreachWithSubqueries { p => + f(p) + p match { + case w: UnresolvedWith => + w.cteRelations.foreach { case (_, ctePlan, _) => + foreachPlanDeep(ctePlan)(f) + } + case InsertIntoStatement(table, _, _, _, _, _, _, _, _) => + foreachPlanDeep(table)(f) + case s: SingleStatement => + // Root of the wrapped statement is skipped by SingleStatement.children. + foreachPlanDeep(s.parsedPlan)(f) + case c: CompoundBody => + c.handlers.foreach(h => foreachPlanDeep(h)(f)) + case s: SimpleCaseStatement => + s.elseBody.foreach(b => foreachPlanDeep(b)(f)) + case ExplainCommand(logicalPlan, _) => + foreachPlanDeep(logicalPlan)(f) + case DescribeQueryCommand(_, queryPlan) => + foreachPlanDeep(queryPlan)(f) + case _ => + } + } + } + + /** Multipart name from a table/view-shaped plan node, if any. */ + private def tableOrViewParts(plan: LogicalPlan): Option[Seq[String]] = plan match { + case u: UnresolvedRelation => Some(u.multipartIdentifier) + case u: UnresolvedTable => Some(u.multipartIdentifier) + case u: UnresolvedView => Some(u.multipartIdentifier) + case u: UnresolvedTableOrView => Some(u.multipartIdentifier) + case u: UnresolvedIdentifier => Some(u.nameParts) + case _ => None + } + + private def tableIdentifierParts(id: org.apache.spark.sql.catalyst.TableIdentifier): Seq[String] = + id.catalog.toSeq ++ id.database.toSeq :+ id.table + + /** + * Collect multipart table/view identifiers for lineage (as written in the + * SQL). CTE definition names and correlation aliases are omitted; tables + * referenced inside CTE bodies are still included. Function / variable + * identifiers are not collected. Deduplicates while preserving first-seen + * order. + */ + private def collectTableReferences(plan: LogicalPlan): Seq[Seq[String]] = { + val seen = mutable.LinkedHashSet.empty[Seq[String]] + // CTE names are always single-part in the grammar; UnresolvedRelation refs + // to CTEs are likewise single-part, so filtering matches that shape. + val cteNames = mutable.HashSet.empty[String] + + def addCteNames(w: UnresolvedWith): Unit = { + w.cteRelations.foreach { case (name, _, _) => + cteNames += name.toLowerCase(java.util.Locale.ROOT) + } + } + + def isCteName(parts: Seq[String]): Boolean = parts match { + case Seq(name) => cteNames.contains(name.toLowerCase(java.util.Locale.ROOT)) + case _ => false + } + + def add(parts: Seq[String]): Unit = { + if (parts.nonEmpty && !isCteName(parts)) seen += parts + } + + // First pass: gather CTE names in scope (including nested). + foreachPlanDeep(plan) { + case w: UnresolvedWith => addCteNames(w) + case _ => + } + + foreachPlanDeep(plan) { + case u: UnresolvedRelation => add(u.multipartIdentifier) + case u: UnresolvedTable => add(u.multipartIdentifier) + case u: UnresolvedView => add(u.multipartIdentifier) + case u: UnresolvedTableOrView => add(u.multipartIdentifier) + // Table/view DDL targets only — not CreateFunction / CreateVariable names. + case c: CreateView => tableOrViewParts(c.child).foreach(add) + case c: CreateViewCommand => add(tableIdentifierParts(c.name)) + case c: CreateTempViewUsing => add(tableIdentifierParts(c.tableIdent)) + case c: CreateTable => tableOrViewParts(c.name).foreach(add) + case c: CreateTableAsSelect => tableOrViewParts(c.name).foreach(add) + case c: ReplaceTable => tableOrViewParts(c.name).foreach(add) + case c: ReplaceTableAsSelect => tableOrViewParts(c.name).foreach(add) + case c: DropTable => tableOrViewParts(c.child).foreach(add) + case c: DropView => tableOrViewParts(c.child).foreach(add) + case c: TruncateTable => tableOrViewParts(c.table).foreach(add) + case c: TruncatePartition => tableOrViewParts(c.table).foreach(add) + case c: CacheTable => + if (c.multipartIdentifier.nonEmpty) add(c.multipartIdentifier) + else tableOrViewParts(c.table).foreach(add) + case c: UncacheTable => tableOrViewParts(c.table).foreach(add) + case c: RefreshTable => tableOrViewParts(c.child).foreach(add) + case c: CommentOnTable => tableOrViewParts(c.table).foreach(add) + case _ => + } + seen.toSeq + } + + /** Collect multipart function names, including table-valued functions. */ + private def collectFunctionReferences(plan: LogicalPlan): Seq[Seq[String]] = { + val seen = mutable.LinkedHashSet.empty[Seq[String]] + def add(parts: Seq[String]): Unit = { + if (parts.nonEmpty) seen += parts + } + def collectInExpression(e: Expression): Unit = e.foreach { + case f: UnresolvedFunction => add(f.nameParts) + case _ => + } + foreachPlanDeep(plan) { p => + foreachExpressionDeep(p)(collectInExpression) + p match { + case u: UnresolvedTableValuedFunction => add(u.name) + case _ => + } + } + seen.toSeq + } + + /** + * Collect the primary select list as `{name}` objects (multipart name + * parts only). Empty for non-query statements without a projected query body. + */ + private def collectSelectList(plan: LogicalPlan): Seq[JObject] = { + val query = primaryQueryPlan(plan) + val named: Seq[NamedExpression] = query match { + case p: Project => p.projectList + case a: Aggregate => a.aggregateExpressions + case _ => Nil + } + named.map(selectListItem) + } + + private def primaryQueryPlan(plan: LogicalPlan): LogicalPlan = plan match { + case UnresolvedWith(child, _, _) => primaryQueryPlan(child) + case InsertIntoStatement(_, _, _, query, _, _, _, _, _) => + primaryQueryPlan(query) + case c: CreateTableAsSelect => primaryQueryPlan(c.query) + case r: ReplaceTableAsSelect => primaryQueryPlan(r.query) + case c: CreateView => primaryQueryPlan(c.query) + case c: CreateViewCommand => primaryQueryPlan(c.plan) + case c: CacheTableAsSelect => primaryQueryPlan(c.plan) + case ExplainCommand(logicalPlan, _) => primaryQueryPlan(logicalPlan) + case DescribeQueryCommand(_, queryPlan) => primaryQueryPlan(queryPlan) + case SubqueryAlias(_, child) => primaryQueryPlan(child) + case Sort(_, _, child, _) => primaryQueryPlan(child) + case Filter(_, child) => primaryQueryPlan(child) + case UnresolvedHaving(_, child) => primaryQueryPlan(child) + case UnresolvedQualify(_, child) => primaryQueryPlan(child) + case Distinct(child) => primaryQueryPlan(child) + case GlobalLimit(_, child) => primaryQueryPlan(child) + case LocalLimit(_, child) => primaryQueryPlan(child) + case Offset(_, child) => primaryQueryPlan(child) + case Repartition(_, _, child) => primaryQueryPlan(child) + case RepartitionByExpression(_, child, _, _) => primaryQueryPlan(child) + case Sample(_, _, _, _, child, _) => primaryQueryPlan(child) + case other => other + } + + private def selectListItem(ne: NamedExpression): JObject = ne match { + case Alias(_, name) => + JObject("name" -> partsToJArray(Seq(name))) + case _: UnresolvedAlias => + JObject("name" -> partsToJArray(Nil)) + case s: UnresolvedStar => + val name = s.target.map(_ :+ "*").getOrElse(Seq("*")) + JObject("name" -> partsToJArray(name)) + case a: UnresolvedAttribute => + JObject("name" -> partsToJArray(a.nameParts)) + case other => + JObject("name" -> partsToJArray(Seq(other.name))) + } + + /** + * Parameter-marker object, or None when the statement has neither named nor + * positional markers. Nested empty members are also omitted: `named` only + * when non-empty, `unnamed_count` only when > 0. + */ + private def parameterMarkersJson(plan: LogicalPlan): Option[JObject] = { + val named = mutable.LinkedHashSet.empty[String] + var unnamedCount = 0 + def visitExpr(e: Expression): Unit = e.foreach { + case n: NamedParameter => named += n.name + case _: PosParameter => unnamedCount += 1 + case _ => + } + foreachPlanDeep(plan) { p => + foreachExpressionDeep(p)(visitExpr) + } + if (named.isEmpty && unnamedCount == 0) { + None + } else { + val fields = mutable.ListBuffer.empty[JField] + if (named.nonEmpty) fields += "named" -> JArray(named.toList.map(JString)) + if (unnamedCount > 0) fields += "unnamed_count" -> JInt(unnamedCount) + Some(JObject(fields.toList)) + } + } +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala new file mode 100644 index 0000000000000..cb2fae1d9af9c --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/parser/SqlStatementCodes.scala @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.parser + +import org.apache.spark.sql.catalyst.analysis.{ + RelationTimeTravel, + ResolvedInlineTable, + UnresolvedExecuteImmediate, + UnresolvedHaving, + UnresolvedInlineTable, + UnresolvedRelation, + UnresolvedTableValuedFunction +} +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.execution.command._ +import org.apache.spark.sql.execution.datasources.{CreateTempViewUsing, RefreshResource} +import org.apache.spark.sql.metricview.logical.CreateMetricView + +/** + * Classification of a parsed SQL statement using ISO/IEC 9075-2:2023 Table 39, + * "SQL-statement codes" (clause 23.1 <get diagnostics statement>). + * + * @param statementIdentifier Table 39 Identifier column (or Spark product name) + * @param statementCode Table 39 Code column; Spark-only statements use negative + * implementation-defined codes (Table 39 IE005 / IV190) + */ +case class SqlStatementClassification( + statementIdentifier: String, + statementCode: Int) + +/** + * Maps unresolved [[LogicalPlan]]s to Table 39 statement codes. + * + * Spark-only statements use the standard's implementation-defined escape hatch: + * a product-specific identifier and a distinct negative code. Codes are + * append-only and must never be renumbered. + * + * Unknown plans map to [[Unrecognized]] (empty identifier, code 0). Query + * shapes are allowlisted; unknown non-commands are not assumed to be SELECT. + */ +object SqlStatementCodes { + + // Standard Table 39 entries used by Spark SQL (ISO/IEC 9075-2:2023). + val Select: SqlStatementClassification = SqlStatementClassification("SELECT", 21) + val Insert: SqlStatementClassification = SqlStatementClassification("INSERT", 50) + val DeleteWhere: SqlStatementClassification = SqlStatementClassification("DELETE WHERE", 19) + val UpdateWhere: SqlStatementClassification = SqlStatementClassification("UPDATE WHERE", 82) + val Merge: SqlStatementClassification = SqlStatementClassification("MERGE", 128) + val CreateTable: SqlStatementClassification = SqlStatementClassification("CREATE TABLE", 77) + val CreateView: SqlStatementClassification = SqlStatementClassification("CREATE VIEW", 84) + val DropTable: SqlStatementClassification = SqlStatementClassification("DROP TABLE", 32) + val DropView: SqlStatementClassification = SqlStatementClassification("DROP VIEW", 36) + val AlterTable: SqlStatementClassification = SqlStatementClassification("ALTER TABLE", 4) + val CreateSchema: SqlStatementClassification = SqlStatementClassification("CREATE SCHEMA", 64) + val DropSchema: SqlStatementClassification = SqlStatementClassification("DROP SCHEMA", 31) + val SetSchema: SqlStatementClassification = SqlStatementClassification("SET SCHEMA", 74) + val TruncateTable: SqlStatementClassification = + SqlStatementClassification("TRUNCATE TABLE", 139) + val CreateRoutine: SqlStatementClassification = SqlStatementClassification("CREATE ROUTINE", 14) + val DropRoutine: SqlStatementClassification = SqlStatementClassification("DROP ROUTINE", 30) + val ExecuteImmediate: SqlStatementClassification = + SqlStatementClassification("EXECUTE IMMEDIATE", 43) + val Call: SqlStatementClassification = SqlStatementClassification("CALL", 7) + + // Table 39 "Unrecognized statements": empty identifier, code 0. + val Unrecognized: SqlStatementClassification = SqlStatementClassification("", 0) + + // Spark product-specific identifiers with append-only negative codes + // (Table 39 implementation-defined / IE005 row: negative Code values). + val CacheTable: SqlStatementClassification = spark("CACHE TABLE", -1) + val CacheTableAsSelect: SqlStatementClassification = spark("CACHE TABLE AS SELECT", -2) + val UncacheTable: SqlStatementClassification = spark("UNCACHE TABLE", -3) + val RefreshTable: SqlStatementClassification = spark("REFRESH TABLE", -4) + val ShowTables: SqlStatementClassification = spark("SHOW TABLES", -5) + val DescribeTable: SqlStatementClassification = spark("DESCRIBE TABLE", -6) + val AnalyzeTable: SqlStatementClassification = spark("ANALYZE TABLE", -7) + val DeclareVariable: SqlStatementClassification = spark("DECLARE VARIABLE", -8) + val SetVariable: SqlStatementClassification = spark("SET VARIABLE", -9) + val DropVariable: SqlStatementClassification = spark("DROP VARIABLE", -10) + val ShowTableProperties: SqlStatementClassification = spark("SHOW TBLPROPERTIES", -11) + val DescribeNamespace: SqlStatementClassification = spark("DESCRIBE NAMESPACE", -12) + val ShowFunctions: SqlStatementClassification = spark("SHOW FUNCTIONS", -13) + val DescribeFunction: SqlStatementClassification = spark("DESCRIBE FUNCTION", -14) + val ShowCreateTable: SqlStatementClassification = spark("SHOW CREATE TABLE", -15) + val ShowColumns: SqlStatementClassification = spark("SHOW COLUMNS", -16) + val ShowPartitions: SqlStatementClassification = spark("SHOW PARTITIONS", -17) + val ShowViews: SqlStatementClassification = spark("SHOW VIEWS", -18) + val RefreshFunction: SqlStatementClassification = spark("REFRESH FUNCTION", -19) + val CommentOnNamespace: SqlStatementClassification = spark("COMMENT ON NAMESPACE", -20) + val CommentOnTable: SqlStatementClassification = spark("COMMENT ON TABLE", -21) + // SQL/PSM-style scripting (9075-4); not in Foundation Table 39. + val BeginEnd: SqlStatementClassification = spark("BEGIN END", -22) + // SparkSqlParser-only session / resource commands (append-only). + val Explain: SqlStatementClassification = spark("EXPLAIN", -23) + val Set: SqlStatementClassification = spark("SET", -24) + val Reset: SqlStatementClassification = spark("RESET", -25) + val AddJar: SqlStatementClassification = spark("ADD JAR", -26) + val AddFile: SqlStatementClassification = spark("ADD FILE", -27) + val AddArchive: SqlStatementClassification = spark("ADD ARCHIVE", -28) + val ListJar: SqlStatementClassification = spark("LIST JAR", -29) + val ListFile: SqlStatementClassification = spark("LIST FILE", -30) + val ClearCache: SqlStatementClassification = spark("CLEAR CACHE", -31) + val RefreshResourceCmd: SqlStatementClassification = spark("REFRESH RESOURCE", -32) + val DescribeQuery: SqlStatementClassification = spark("DESCRIBE QUERY", -33) + val ShowCatalogs: SqlStatementClassification = spark("SHOW CATALOGS", -34) + val ShowCurrentNamespace: SqlStatementClassification = + spark("SHOW CURRENT NAMESPACE", -35) + val SetCatalog: SqlStatementClassification = spark("SET CATALOG", -36) + val CreateMetricViewStmt: SqlStatementClassification = spark("CREATE METRIC VIEW", -37) + + private def spark(identifier: String, code: Int): SqlStatementClassification = { + assert(code < 0, s"Spark statement codes must be negative, got $code") + SqlStatementClassification(statementIdentifier = identifier, statementCode = code) + } + + /** Classify an unresolved logical plan. */ + def classify(plan: LogicalPlan): SqlStatementClassification = plan match { + case UnresolvedWith(child, _, _) => classify(child) + case _: CompoundBody => BeginEnd + case _: InsertIntoStatement => Insert + case _: DeleteFromTable | _: DeleteFromTableWithFilters => DeleteWhere + case _: UpdateTable => UpdateWhere + case _: MergeIntoTable => Merge + case _: CreateTableAsSelect | _: ReplaceTableAsSelect => CreateTable + case _: CreateTable | _: CreateTableLike | _: ReplaceTable => CreateTable + case _: CreateView | _: CreateViewCommand | _: CreateTempViewUsing => CreateView + case _: DropTable => DropTable + case _: DropView => DropView + case _: CreateNamespace => CreateSchema + case _: DropNamespace => DropSchema + case _: SetCatalogAndNamespace | _: SetNamespaceCommand => SetSchema + case _: SetCatalogCommand => SetCatalog + case _: TruncateTable => TruncateTable + case _: CreateFunction | _: CreateFunctionCommand | + _: CreateUserDefinedFunction | _: CreateUserDefinedFunctionCommand => + CreateRoutine + case _: DropFunction | _: DropFunctionCommand => DropRoutine + case _: UnresolvedExecuteImmediate => ExecuteImmediate + case _: Call => Call + case _: CommentOnTable => CommentOnTable + case _: AlterTableCommand | _: RenameTable => AlterTable + case _: CacheTable => CacheTable + case _: CacheTableAsSelect => CacheTableAsSelect + case _: UncacheTable => UncacheTable + case _: RefreshTable => RefreshTable + case _: ShowTables | _: ShowTablesExtended => ShowTables + case _: DescribeRelation | _: DescribeTablePartition | _: DescribeColumn => + DescribeTable + case _: DescribeQueryCommand => DescribeQuery + case _: AnalyzeTable | _: AnalyzeTables | _: AnalyzeColumn => AnalyzeTable + case _: CreateVariable => DeclareVariable + case _: SetVariable => SetVariable + case _: DropVariable => DropVariable + case _: ShowTableProperties => ShowTableProperties + case _: DescribeNamespace => DescribeNamespace + case _: ShowFunctions => ShowFunctions + case _: DescribeFunction => DescribeFunction + case _: ShowCreateTable => ShowCreateTable + case _: ShowColumns => ShowColumns + case _: ShowPartitions | _: ShowTablePartition => ShowPartitions + case _: ShowViews => ShowViews + case _: RefreshFunction => RefreshFunction + case _: CommentOnNamespace => CommentOnNamespace + case _: ExplainCommand => Explain + case _: SetCommand => Set + case _: ResetCommand => Reset + case _: AddJarsCommand => AddJar + case _: AddFilesCommand => AddFile + case _: AddArchivesCommand => AddArchive + case _: ListJarsCommand => ListJar + case _: ListFilesCommand => ListFile + case ClearCacheCommand => ClearCache + case _: RefreshResource => RefreshResourceCmd + case _: ShowCatalogsCommand => ShowCatalogs + case _: ShowCurrentNamespaceCommand => ShowCurrentNamespace + case _: CreateMetricView | _: CreateMetricViewCommand => CreateMetricViewStmt + case _: Command => Unrecognized + case p if isQueryPlan(p) => Select + case _ => Unrecognized + } + + /** + * Allowlisted query-shaped plans. Unknown non-command plans are not assumed + * to be SELECT. + */ + private def isQueryPlan(plan: LogicalPlan): Boolean = plan match { + case _: Project | _: Aggregate | _: Distinct | _: Filter | _: Sort | + _: GlobalLimit | _: LocalLimit | _: Join | _: Union | _: Except | + _: Intersect | _: SubqueryAlias | _: Repartition | + _: RepartitionByExpression | _: Sample | _: Range | + _: OneRowRelation | _: LocalRelation | _: Deduplicate | + _: Expand | _: Generate | _: Window | _: Tail | _: Offset | + _: LateralJoin | _: UnresolvedHaving | _: CollectMetrics | + _: WithCTE | _: UnresolvedRelation | _: UnresolvedInlineTable | + _: ResolvedInlineTable | _: RelationTimeTravel | + _: UnresolvedTableValuedFunction => true + case _ => false + } +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala index 52c6821d00011..1ee7699fff429 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala @@ -22,7 +22,7 @@ import org.apache.spark.sql.artifact.ArtifactManager import org.apache.spark.sql.catalyst.analysis.{Analyzer, EvalSubqueriesForTimeTravel, FunctionRegistry, InvokeProcedures, ReplaceCharWithVarchar, ResolveDataSource, ResolveEventTimeWatermark, ResolveExecuteImmediate, ResolveMetricView, ResolveSessionCatalog, ResolveSetCatalogCommand, ResolveTranspose, TableFunctionRegistry} import org.apache.spark.sql.catalyst.analysis.resolver.ResolverExtension import org.apache.spark.sql.catalyst.catalog.{FunctionExpressionBuilder, SessionCatalog} -import org.apache.spark.sql.catalyst.expressions.{Expression, ExtractSemiStructuredFields} +import org.apache.spark.sql.catalyst.expressions.{Expression, ExtractSemiStructuredFields, ParseSql} import org.apache.spark.sql.catalyst.normalizer.NormalizeCTEIds import org.apache.spark.sql.catalyst.optimizer.Optimizer import org.apache.spark.sql.catalyst.parser.ParserInterface @@ -96,7 +96,12 @@ abstract class BaseSessionStateBuilder( */ protected lazy val functionRegistry: FunctionRegistry = { parentState.map(_.functionRegistry.clone()) - .getOrElse(extensions.registerFunctions(FunctionRegistry.builtin.clone())) + .getOrElse { + val registry = FunctionRegistry.builtin.clone() + // sql/core-only builtins that need SparkSqlParser. + ParseSql.register(registry) + extensions.registerFunctions(registry) + } } /** diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md index 7c389ed360aba..8a4d226fd94af 100644 --- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md +++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md @@ -274,6 +274,7 @@ | org.apache.spark.sql.catalyst.expressions.OctetLength | octet_length | SELECT octet_length('Spark SQL') | struct | | org.apache.spark.sql.catalyst.expressions.Or | or | SELECT true or false | struct<(true OR false):boolean> | | org.apache.spark.sql.catalyst.expressions.Overlay | overlay | SELECT overlay('Spark SQL' PLACING '_' FROM 6) | struct | +| org.apache.spark.sql.catalyst.expressions.ParseSql | parse_sql | SELECT parse_sql('SELECT a, b FROM t') | struct | | org.apache.spark.sql.catalyst.expressions.ParseToDate | to_date | SELECT to_date('2009-07-30 04:17:52') | struct | | org.apache.spark.sql.catalyst.expressions.ParseToTimestamp | to_timestamp | SELECT to_timestamp('2016-12-31 00:12:00') | struct | | org.apache.spark.sql.catalyst.expressions.ParseToTimestampLTZExpressionBuilder | to_timestamp_ltz | SELECT to_timestamp_ltz('2016-12-31 00:12:00') | struct | diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql-gating.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql-gating.sql.out new file mode 100644 index 0000000000000..dbf89961fa3ac --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql-gating.sql.out @@ -0,0 +1,14 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +SELECT parse_sql('SELECT 1') +-- !query analysis +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "FEATURE_NOT_ENABLED", + "sqlState" : "56038", + "messageParameters" : { + "configKey" : "spark.sql.parseSql.enabled", + "configValue" : "true", + "featureName" : "parse_sql" + } +} diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out new file mode 100644 index 0000000000000..733b41c24b884 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/parse-sql.sql.out @@ -0,0 +1,578 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +SELECT parse_sql(NULL) +-- !query analysis +Project [parse_sql(cast(null as string)) AS parse_sql(NULL)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('SELECT a, b FROM t') +-- !query analysis +Project [parse_sql(SELECT a, b FROM t) AS parse_sql(SELECT a, b FROM t)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2') +-- !query analysis +Project [parse_sql(SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2) AS parse_sql(SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2)#x] ++- OneRowRelation + + +-- !query +SELECT + get_json_object(result, '$.statement_identifier') AS statement_identifier, + get_json_object(result, '$.table_references[0][0]') AS first_table, + get_json_object(result, '$.select_list[1].name[0]') AS second_column +FROM (SELECT parse_sql('SELECT a, b FROM t') AS result) +-- !query analysis +Project [get_json_object(result#x, $.statement_identifier) AS statement_identifier#x, get_json_object(result#x, $.table_references[0][0]) AS first_table#x, get_json_object(result#x, $.select_list[1].name[0]) AS second_column#x] ++- SubqueryAlias __auto_generated_subquery_name + +- Project [parse_sql(SELECT a, b FROM t) AS result#x] + +- OneRowRelation + + +-- !query +SELECT parse_sql('INSERT INTO t SELECT 1') +-- !query analysis +Project [parse_sql(INSERT INTO t SELECT 1) AS parse_sql(INSERT INTO t SELECT 1)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('DELETE FROM t WHERE a = 1') +-- !query analysis +Project [parse_sql(DELETE FROM t WHERE a = 1) AS parse_sql(DELETE FROM t WHERE a = 1)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('UPDATE t SET a = 1 WHERE b = 2') +-- !query analysis +Project [parse_sql(UPDATE t SET a = 1 WHERE b = 2) AS parse_sql(UPDATE t SET a = 1 WHERE b = 2)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE') +-- !query analysis +Project [parse_sql(MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE) AS parse_sql(MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('CREATE TABLE t (a INT)') +-- !query analysis +Project [parse_sql(CREATE TABLE t (a INT)) AS parse_sql(CREATE TABLE t (a INT))#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('CREATE TABLE t AS SELECT 1 AS a') +-- !query analysis +Project [parse_sql(CREATE TABLE t AS SELECT 1 AS a) AS parse_sql(CREATE TABLE t AS SELECT 1 AS a)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('DROP TABLE t') +-- !query analysis +Project [parse_sql(DROP TABLE t) AS parse_sql(DROP TABLE t)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('CACHE TABLE t') +-- !query analysis +Project [parse_sql(CACHE TABLE t) AS parse_sql(CACHE TABLE t)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('TABLE t') +-- !query analysis +Project [parse_sql(TABLE t) AS parse_sql(TABLE t)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('VALUES (1), (2)') +-- !query analysis +Project [parse_sql(VALUES (1), (2)) AS parse_sql(VALUES (1), (2))#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('CREATE FUNCTION f AS ''x'' USING JAR ''y.jar''') +-- !query analysis +Project [parse_sql(CREATE FUNCTION f AS 'x' USING JAR 'y.jar') AS parse_sql(CREATE FUNCTION f AS 'x' USING JAR 'y.jar')#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('DECLARE VARIABLE x INT') +-- !query analysis +Project [parse_sql(DECLARE VARIABLE x INT) AS parse_sql(DECLARE VARIABLE x INT)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('SELECT * FROM t WHERE a = :foo AND b = ?') +-- !query analysis +Project [parse_sql(SELECT * FROM t WHERE a = :foo AND b = ?) AS parse_sql(SELECT * FROM t WHERE a = :foo AND b = ?)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte') +-- !query analysis +Project [parse_sql(WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte) AS parse_sql(WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)') +-- !query analysis +Project [parse_sql(SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)) AS parse_sql(SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id))#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql( +'SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( + PARTITION BY lower(t.c) ORDER BY length(t.d)) + FROM left_t t + JOIN right_t r ON hash(t.id) = hash(r.id) + JOIN LATERAL range(cast(t.n AS BIGINT)) rng + WHERE startswith(t.c, ''x'') + AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id) + GROUP BY coalesce(t.a, 0), t.b, t.c, t.d + HAVING count_if(t.b > 0) > 0 + ORDER BY greatest(t.a, 1)') +-- !query analysis +Project [parse_sql(SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( + PARTITION BY lower(t.c) ORDER BY length(t.d)) + FROM left_t t + JOIN right_t r ON hash(t.id) = hash(r.id) + JOIN LATERAL range(cast(t.n AS BIGINT)) rng + WHERE startswith(t.c, 'x') + AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id) + GROUP BY coalesce(t.a, 0), t.b, t.c, t.d + HAVING count_if(t.b > 0) > 0 + ORDER BY greatest(t.a, 1)) AS parse_sql(SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( + PARTITION BY lower(t.c) ORDER BY length(t.d)) + FROM left_t t + JOIN right_t r ON hash(t.id) = hash(r.id) + JOIN LATERAL range(cast(t.n AS BIGINT)) rng + WHERE startswith(t.c, 'x') + AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id) + GROUP BY coalesce(t.a, 0), t.b, t.c, t.d + HAVING count_if(t.b > 0) > 0 + ORDER BY greatest(t.a, 1))#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql( +'MERGE INTO target t + USING ( + SELECT id, normalize_name(name) AS name + FROM source + WHERE is_valid(id) + ) s + ON hash(t.id) = hash(s.id) + WHEN MATCHED AND should_update(t.name, s.name) THEN + UPDATE SET name = coalesce(s.name, upper(t.name)) + WHEN NOT MATCHED THEN + INSERT (id, name) VALUES (s.id, lower(s.name))') +-- !query analysis +Project [parse_sql(MERGE INTO target t + USING ( + SELECT id, normalize_name(name) AS name + FROM source + WHERE is_valid(id) + ) s + ON hash(t.id) = hash(s.id) + WHEN MATCHED AND should_update(t.name, s.name) THEN + UPDATE SET name = coalesce(s.name, upper(t.name)) + WHEN NOT MATCHED THEN + INSERT (id, name) VALUES (s.id, lower(s.name))) AS parse_sql(MERGE INTO target t + USING ( + SELECT id, normalize_name(name) AS name + FROM source + WHERE is_valid(id) + ) s + ON hash(t.id) = hash(s.id) + WHEN MATCHED AND should_update(t.name, s.name) THEN + UPDATE SET name = coalesce(s.name, upper(t.name)) + WHEN NOT MATCHED THEN + INSERT (id, name) VALUES (s.id, lower(s.name)))#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql( +'CREATE TABLE defaults ( + created DATE DEFAULT current_date(), + normalized STRING DEFAULT upper(''x'') + )') +-- !query analysis +Project [parse_sql(CREATE TABLE defaults ( + created DATE DEFAULT current_date(), + normalized STRING DEFAULT upper('x') + )) AS parse_sql(CREATE TABLE defaults ( + created DATE DEFAULT current_date(), + normalized STRING DEFAULT upper('x') + ))#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('SELEC FROM t') +-- !query analysis +Project [parse_sql(SELEC FROM t) AS parse_sql(SELEC FROM t)#x] ++- OneRowRelation + + +-- !query +SELECT + get_json_object(result, '$.parse_success') AS parse_success, + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM (SELECT parse_sql('SELEC FROM t') AS result) +-- !query analysis +Project [get_json_object(result#x, $.parse_success) AS parse_success#x, get_json_object(result#x, $.error.errorClass) AS error_class#x, get_json_object(result#x, $.error.queryContext[0].fragment) AS fragment#x] ++- SubqueryAlias __auto_generated_subquery_name + +- Project [parse_sql(SELEC FROM t) AS result#x] + +- OneRowRelation + + +-- !query +SELECT parse_sql( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b') +-- !query analysis +Project [parse_sql(SELECT * + FROM t + ORDER BY a + CLUSTER BY b) AS parse_sql(SELECT * + FROM t + ORDER BY a + CLUSTER BY b)#x] ++- OneRowRelation + + +-- !query +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].startIndex') AS start_index +FROM ( + SELECT parse_sql( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b') AS result +) +-- !query analysis +Project [get_json_object(result#x, $.error.errorClass) AS error_class#x, get_json_object(result#x, $.error.line) AS line#x, get_json_object(result#x, $.error.position) AS position#x, get_json_object(result#x, $.error.queryContext[0].startIndex) AS start_index#x] ++- SubqueryAlias __auto_generated_subquery_name + +- Project [parse_sql(SELECT * + FROM t + ORDER BY a + CLUSTER BY b) AS result#x] + +- OneRowRelation + + +-- !query +SELECT parse_sql('') +-- !query analysis +Project [parse_sql() AS parse_sql()#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('USE bad-name') +-- !query analysis +Project [parse_sql(USE bad-name) AS parse_sql(USE bad-name)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c') +-- !query analysis +Project [parse_sql(WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c) AS parse_sql(WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('MERGE INTO target USING source ON target.id = source.id') +-- !query analysis +Project [parse_sql(MERGE INTO target USING source ON target.id = source.id) AS parse_sql(MERGE INTO target USING source ON target.id = source.id)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('EXPLAIN SELECT 1') +-- !query analysis +Project [parse_sql(EXPLAIN SELECT 1) AS parse_sql(EXPLAIN SELECT 1)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('SET spark.sql.adaptive.enabled=true') +-- !query analysis +Project [parse_sql(SET spark.sql.adaptive.enabled=true) AS parse_sql(SET spark.sql.adaptive.enabled=true)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('ADD JAR /tmp/x.jar') +-- !query analysis +Project [parse_sql(ADD JAR /tmp/x.jar) AS parse_sql(ADD JAR /tmp/x.jar)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('CREATE VIEW v AS SELECT a, b FROM t') +-- !query analysis +Project [parse_sql(CREATE VIEW v AS SELECT a, b FROM t) AS parse_sql(CREATE VIEW v AS SELECT a, b FROM t)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('SELECT 1 AS IDENTIFIER(''alias.field'')') +-- !query analysis +Project [parse_sql(SELECT 1 AS IDENTIFIER('alias.field')) AS parse_sql(SELECT 1 AS IDENTIFIER('alias.field'))#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('SELECT DATE ''not-a-date''') +-- !query analysis +Project [parse_sql(SELECT DATE 'not-a-date') AS parse_sql(SELECT DATE 'not-a-date')#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql( +'BEGIN + SELECT 1; + SELEC 2; + END') +-- !query analysis +Project [parse_sql(BEGIN + SELECT 1; + SELEC 2; + END) AS parse_sql(BEGIN + SELECT 1; + SELEC 2; + END)#x] ++- OneRowRelation + + +-- !query +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM ( + SELECT parse_sql( +'BEGIN + SELECT 1; + SELEC 2; + END') AS result +) +-- !query analysis +Project [get_json_object(result#x, $.error.errorClass) AS error_class#x, get_json_object(result#x, $.error.line) AS line#x, get_json_object(result#x, $.error.position) AS position#x, get_json_object(result#x, $.error.queryContext[0].fragment) AS fragment#x] ++- SubqueryAlias __auto_generated_subquery_name + +- Project [parse_sql(BEGIN + SELECT 1; + SELEC 2; + END) AS result#x] + +- OneRowRelation + + +-- !query +SELECT parse_sql( +'BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END') +-- !query analysis +Project [parse_sql(BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END) AS parse_sql(BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END)#x] ++- OneRowRelation + + +-- !query +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM ( + SELECT parse_sql( +'BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END') AS result +) +-- !query analysis +Project [get_json_object(result#x, $.error.errorClass) AS error_class#x, get_json_object(result#x, $.error.line) AS line#x, get_json_object(result#x, $.error.position) AS position#x, get_json_object(result#x, $.error.queryContext[0].fragment) AS fragment#x] ++- SubqueryAlias __auto_generated_subquery_name + +- Project [parse_sql(BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END) AS result#x] + +- OneRowRelation + + +-- !query +SELECT sql_text, parse_sql(sql_text) FROM VALUES + ('SELECT 1'), + ('INSERT INTO t SELECT 1'), + ('CACHE TABLE t') +AS t(sql_text) +-- !query analysis +Project [sql_text#x, parse_sql(sql_text#x) AS parse_sql(sql_text)#x] ++- SubqueryAlias t + +- LocalRelation [sql_text#x] + + +-- !query +SELECT parse_sql('BEGIN SELECT 1; END') +-- !query analysis +Project [parse_sql(BEGIN SELECT 1; END) AS parse_sql(BEGIN SELECT 1; END)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END') +-- !query analysis +Project [parse_sql(BEGIN SELECT count(a) FROM script_t WHERE c = :p; END) AS parse_sql(BEGIN SELECT count(a) FROM script_t WHERE c = :p; END)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END') +-- !query analysis +Project [parse_sql(BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END) AS parse_sql(BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END') +-- !query analysis +Project [parse_sql(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END) AS parse_sql(BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END)#x] ++- OneRowRelation + + +-- !query +SELECT parse_sql( +'BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + INSERT INTO error_log + SELECT format_string(''%s'', message) FROM error_source; + END; + + WITH prepared AS ( + SELECT id, normalize_name(name) AS name + FROM input_names + WHERE is_valid(id) + ) + INSERT INTO output_names + SELECT id, upper(name) FROM prepared; + + IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN + UPDATE update_target + SET value = coalesce((SELECT max(value) FROM update_source), 0) + WHERE should_update(id); + ELSE + DELETE FROM delete_target + WHERE id IN (SELECT id FROM delete_source WHERE expired(ts)); + END IF; + + FOR row AS + SELECT id FROM loop_source WHERE ready(id) + DO + SELECT audit(row.id), count(*) FROM loop_body; + END FOR; + END') +-- !query analysis +Project [parse_sql(BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + INSERT INTO error_log + SELECT format_string('%s', message) FROM error_source; + END; + + WITH prepared AS ( + SELECT id, normalize_name(name) AS name + FROM input_names + WHERE is_valid(id) + ) + INSERT INTO output_names + SELECT id, upper(name) FROM prepared; + + IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN + UPDATE update_target + SET value = coalesce((SELECT max(value) FROM update_source), 0) + WHERE should_update(id); + ELSE + DELETE FROM delete_target + WHERE id IN (SELECT id FROM delete_source WHERE expired(ts)); + END IF; + + FOR row AS + SELECT id FROM loop_source WHERE ready(id) + DO + SELECT audit(row.id), count(*) FROM loop_body; + END FOR; + END) AS parse_sql(BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + INSERT INTO error_log + SELECT format_string('%s', message) FROM error_source; + END; + + WITH prepared AS ( + SELECT id, normalize_name(name) AS name + FROM input_names + WHERE is_valid(id) + ) + INSERT INTO output_names + SELECT id, upper(name) FROM prepared; + + IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN + UPDATE update_target + SET value = coalesce((SELECT max(value) FROM update_source), 0) + WHERE should_update(id); + ELSE + DELETE FROM delete_target + WHERE id IN (SELECT id FROM delete_source WHERE expired(ts)); + END IF; + + FOR row AS + SELECT id FROM loop_source WHERE ready(id) + DO + SELECT audit(row.id), count(*) FROM loop_body; + END FOR; + END)#x] ++- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/inputs/parse-sql-gating.sql b/sql/core/src/test/resources/sql-tests/inputs/parse-sql-gating.sql new file mode 100644 index 0000000000000..7f2c147afc0f6 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/parse-sql-gating.sql @@ -0,0 +1,4 @@ +-- parse_sql is off by default while the JSON contract is still evolving. +--SET spark.sql.parseSql.enabled=false + +SELECT parse_sql('SELECT 1'); diff --git a/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql b/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql new file mode 100644 index 0000000000000..a11de25e8e5d5 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/parse-sql.sql @@ -0,0 +1,238 @@ +-- End-to-end coverage for parse_sql (SPARK-58738). +-- Returns compact JSON for parse-only statement analysis via SparkSqlParser. +-- Off by default while the JSON contract is still evolving. +--SET spark.sql.parseSql.enabled=true + +-- null input +SELECT parse_sql(NULL); + +-- basic SELECT classification and references +SELECT parse_sql('SELECT a, b FROM t'); +SELECT parse_sql('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2'); + +-- JSON-path access over one shared successful parse result +SELECT + get_json_object(result, '$.statement_identifier') AS statement_identifier, + get_json_object(result, '$.table_references[0][0]') AS first_table, + get_json_object(result, '$.select_list[1].name[0]') AS second_column +FROM (SELECT parse_sql('SELECT a, b FROM t') AS result); + +-- DML +SELECT parse_sql('INSERT INTO t SELECT 1'); +SELECT parse_sql('DELETE FROM t WHERE a = 1'); +SELECT parse_sql('UPDATE t SET a = 1 WHERE b = 2'); +SELECT parse_sql('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE'); + +-- DDL / CTAS +SELECT parse_sql('CREATE TABLE t (a INT)'); +SELECT parse_sql('CREATE TABLE t AS SELECT 1 AS a'); +SELECT parse_sql('DROP TABLE t'); + +-- Spark-only statements (negative Table 39 codes) +SELECT parse_sql('CACHE TABLE t'); + +-- TABLE / VALUES are SELECT-shaped (not Unrecognized) +SELECT parse_sql('TABLE t'); +SELECT parse_sql('VALUES (1), (2)'); + +-- function / variable names are not table_references +SELECT parse_sql('CREATE FUNCTION f AS ''x'' USING JAR ''y.jar'''); +SELECT parse_sql('DECLARE VARIABLE x INT'); + +-- parameter markers +SELECT parse_sql('SELECT * FROM t WHERE a = :foo AND b = ?'); + +-- CTE: lineage excludes CTE names; still walks CTE bodies for real tables +SELECT parse_sql('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte'); + +-- nested subqueries +SELECT parse_sql('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)'); + +-- functions in projection, window, join, TVF, predicates, subquery, grouping, and ordering +SELECT parse_sql( +'SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( + PARTITION BY lower(t.c) ORDER BY length(t.d)) + FROM left_t t + JOIN right_t r ON hash(t.id) = hash(r.id) + JOIN LATERAL range(cast(t.n AS BIGINT)) rng + WHERE startswith(t.c, ''x'') + AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id) + GROUP BY coalesce(t.a, 0), t.b, t.c, t.d + HAVING count_if(t.b > 0) > 0 + ORDER BY greatest(t.a, 1)'); + +-- functions and tables throughout a multiline MERGE +SELECT parse_sql( +'MERGE INTO target t + USING ( + SELECT id, normalize_name(name) AS name + FROM source + WHERE is_valid(id) + ) s + ON hash(t.id) = hash(s.id) + WHEN MATCHED AND should_update(t.name, s.name) THEN + UPDATE SET name = coalesce(s.name, upper(t.name)) + WHEN NOT MATCHED THEN + INSERT (id, name) VALUES (s.id, lower(s.name))'); + +-- functions embedded in DDL column defaults +SELECT parse_sql( +'CREATE TABLE defaults ( + created DATE DEFAULT current_date(), + normalized STRING DEFAULT upper(''x'') + )'); + +-- syntax error: dump the complete STANDARD error, including query context +SELECT parse_sql('SELEC FROM t'); + +-- JSON-path access over one shared parse result +SELECT + get_json_object(result, '$.parse_success') AS parse_success, + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM (SELECT parse_sql('SELEC FROM t') AS result); + +-- full multiline parse-time validation error, including context and location +SELECT parse_sql( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b'); + +-- JSON-path access over one shared multiline parse result +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].startIndex') AS start_index +FROM ( + SELECT parse_sql( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b') AS result +); + +-- parse-only validation errors beyond PARSE_SYNTAX_ERROR +SELECT parse_sql(''); +SELECT parse_sql('USE bad-name'); +SELECT parse_sql('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c'); +SELECT parse_sql('MERGE INTO target USING source ON target.id = source.id'); +SELECT parse_sql('EXPLAIN SELECT 1'); +SELECT parse_sql('SET spark.sql.adaptive.enabled=true'); +SELECT parse_sql('ADD JAR /tmp/x.jar'); +SELECT parse_sql('CREATE VIEW v AS SELECT a, b FROM t'); +SELECT parse_sql('SELECT 1 AS IDENTIFIER(''alias.field'')'); +SELECT parse_sql('SELECT DATE ''not-a-date'''); + +-- location for an error inside a multiline script +--QUERY-DELIMITER-START +SELECT parse_sql( +'BEGIN + SELECT 1; + SELEC 2; + END'); +--QUERY-DELIMITER-END + +-- JSON-path access over one shared scripting parse result +--QUERY-DELIMITER-START +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM ( + SELECT parse_sql( +'BEGIN + SELECT 1; + SELEC 2; + END') AS result +); +--QUERY-DELIMITER-END + +-- location for a SQL scripting semantic validation error +--QUERY-DELIMITER-START +SELECT parse_sql( +'BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END'); +--QUERY-DELIMITER-END + +-- JSON-path access over one shared scripting validation result +--QUERY-DELIMITER-START +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM ( + SELECT parse_sql( +'BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END') AS result +); +--QUERY-DELIMITER-END + +-- batch over a column of SQL text +SELECT sql_text, parse_sql(sql_text) FROM VALUES + ('SELECT 1'), + ('INSERT INTO t SELECT 1'), + ('CACHE TABLE t') +AS t(sql_text); + +-- BEGIN END scripts contain ';' inside the string literal; use query delimiters +-- so the test harness does not split on those semicolons. +--QUERY-DELIMITER-START +SELECT parse_sql('BEGIN SELECT 1; END'); +--QUERY-DELIMITER-END + +--QUERY-DELIMITER-START +SELECT parse_sql('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END'); +--QUERY-DELIMITER-END + +--QUERY-DELIMITER-START +SELECT parse_sql('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END'); +--QUERY-DELIMITER-END + +--QUERY-DELIMITER-START +SELECT parse_sql('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END'); +--QUERY-DELIMITER-END + +-- Complex, genuinely multiline script: dump the complete JSON result. +--QUERY-DELIMITER-START +SELECT parse_sql( +'BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + INSERT INTO error_log + SELECT format_string(''%s'', message) FROM error_source; + END; + + WITH prepared AS ( + SELECT id, normalize_name(name) AS name + FROM input_names + WHERE is_valid(id) + ) + INSERT INTO output_names + SELECT id, upper(name) FROM prepared; + + IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN + UPDATE update_target + SET value = coalesce((SELECT max(value) FROM update_source), 0) + WHERE should_update(id); + ELSE + DELETE FROM delete_target + WHERE id IN (SELECT id FROM delete_source WHERE expired(ts)); + END IF; + + FOR row AS + SELECT id FROM loop_source WHERE ready(id) + DO + SELECT audit(row.id), count(*) FROM loop_body; + END FOR; + END'); +--QUERY-DELIMITER-END diff --git a/sql/core/src/test/resources/sql-tests/results/parse-sql-gating.sql.out b/sql/core/src/test/resources/sql-tests/results/parse-sql-gating.sql.out new file mode 100644 index 0000000000000..41de434b003b1 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/parse-sql-gating.sql.out @@ -0,0 +1,16 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +SELECT parse_sql('SELECT 1') +-- !query schema +struct<> +-- !query output +org.apache.spark.sql.AnalysisException +{ + "errorClass" : "FEATURE_NOT_ENABLED", + "sqlState" : "56038", + "messageParameters" : { + "configKey" : "spark.sql.parseSql.enabled", + "configValue" : "true", + "featureName" : "parse_sql" + } +} diff --git a/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out b/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out new file mode 100644 index 0000000000000..1f27bceace017 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/parse-sql.sql.out @@ -0,0 +1,547 @@ +-- Automatically generated by SQLQueryTestSuite +-- !query +SELECT parse_sql(NULL) +-- !query schema +struct +-- !query output +NULL + + +-- !query +SELECT parse_sql('SELECT a, b FROM t') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"select_list":[{"name":["a"]},{"name":["b"]}]} + + +-- !query +SELECT parse_sql('SELECT db.my_func(a), count(b) FROM cat.ns.t1 JOIN t2') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["cat","ns","t1"],["t2"]],"function_references":[["db","my_func"],["count"]],"select_list":[{"name":[]},{"name":[]}]} + + +-- !query +SELECT + get_json_object(result, '$.statement_identifier') AS statement_identifier, + get_json_object(result, '$.table_references[0][0]') AS first_table, + get_json_object(result, '$.select_list[1].name[0]') AS second_column +FROM (SELECT parse_sql('SELECT a, b FROM t') AS result) +-- !query schema +struct +-- !query output +SELECT t b + + +-- !query +SELECT parse_sql('INSERT INTO t SELECT 1') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"table_references":[["t"]],"select_list":[{"name":[]}]} + + +-- !query +SELECT parse_sql('DELETE FROM t WHERE a = 1') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"DELETE WHERE","statement_code":19,"table_references":[["t"]]} + + +-- !query +SELECT parse_sql('UPDATE t SET a = 1 WHERE b = 2') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"UPDATE WHERE","statement_code":82,"table_references":[["t"]]} + + +-- !query +SELECT parse_sql('MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN DELETE') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"MERGE","statement_code":128,"table_references":[["t"],["s"]]} + + +-- !query +SELECT parse_sql('CREATE TABLE t (a INT)') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"table_references":[["t"]]} + + +-- !query +SELECT parse_sql('CREATE TABLE t AS SELECT 1 AS a') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"table_references":[["t"]],"select_list":[{"name":["a"]}]} + + +-- !query +SELECT parse_sql('DROP TABLE t') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"DROP TABLE","statement_code":32,"table_references":[["t"]]} + + +-- !query +SELECT parse_sql('CACHE TABLE t') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"CACHE TABLE","statement_code":-1,"table_references":[["t"]]} + + +-- !query +SELECT parse_sql('TABLE t') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]]} + + +-- !query +SELECT parse_sql('VALUES (1), (2)') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21} + + +-- !query +SELECT parse_sql('CREATE FUNCTION f AS ''x'' USING JAR ''y.jar''') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"CREATE ROUTINE","statement_code":14} + + +-- !query +SELECT parse_sql('DECLARE VARIABLE x INT') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"DECLARE VARIABLE","statement_code":-8} + + +-- !query +SELECT parse_sql('SELECT * FROM t WHERE a = :foo AND b = ?') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["t"]],"select_list":[{"name":["*"]}],"parameter_markers":{"named":["foo"],"unnamed_count":1}} + + +-- !query +SELECT parse_sql('WITH cte AS (SELECT a FROM hidden_base) SELECT a FROM cte') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["hidden_base"]],"select_list":[{"name":["a"]}]} + + +-- !query +SELECT parse_sql('SELECT (SELECT max(v) FROM scalar_src) AS m, t.a FROM outer_t t WHERE EXISTS (SELECT 1 FROM exists_src e WHERE e.id = t.id)') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["scalar_src"],["exists_src"],["outer_t"]],"function_references":[["max"]],"select_list":[{"name":["m"]},{"name":["t","a"]}]} + + +-- !query +SELECT parse_sql( +'SELECT coalesce(t.a, 0), sum(abs(t.b)) OVER ( + PARTITION BY lower(t.c) ORDER BY length(t.d)) + FROM left_t t + JOIN right_t r ON hash(t.id) = hash(r.id) + JOIN LATERAL range(cast(t.n AS BIGINT)) rng + WHERE startswith(t.c, ''x'') + AND EXISTS (SELECT max(s.v) FROM scalar_t s WHERE s.id = t.id) + GROUP BY coalesce(t.a, 0), t.b, t.c, t.d + HAVING count_if(t.b > 0) > 0 + ORDER BY greatest(t.a, 1)') +-- !query schema +struct 0) > 0 + ORDER BY greatest(t.a, 1)):string> +-- !query output +{"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"table_references":[["scalar_t"],["left_t"],["right_t"]],"function_references":[["greatest"],["count_if"],["coalesce"],["sum"],["abs"],["lower"],["length"],["startswith"],["max"],["range"],["hash"]],"select_list":[{"name":[]},{"name":[]}]} + + +-- !query +SELECT parse_sql( +'MERGE INTO target t + USING ( + SELECT id, normalize_name(name) AS name + FROM source + WHERE is_valid(id) + ) s + ON hash(t.id) = hash(s.id) + WHEN MATCHED AND should_update(t.name, s.name) THEN + UPDATE SET name = coalesce(s.name, upper(t.name)) + WHEN NOT MATCHED THEN + INSERT (id, name) VALUES (s.id, lower(s.name))') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"MERGE","statement_code":128,"table_references":[["target"],["source"]],"function_references":[["hash"],["should_update"],["coalesce"],["upper"],["lower"],["normalize_name"],["is_valid"]]} + + +-- !query +SELECT parse_sql( +'CREATE TABLE defaults ( + created DATE DEFAULT current_date(), + normalized STRING DEFAULT upper(''x'') + )') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"CREATE TABLE","statement_code":77,"table_references":[["defaults"]],"function_references":[["current_date"],["upper"]]} + + +-- !query +SELECT parse_sql('SELEC FROM t') +-- !query schema +struct +-- !query output +{"parse_success":false,"error":{"errorClass":"PARSE_SYNTAX_ERROR","messageTemplate":"Syntax error at or near .","sqlState":"42601","messageParameters":{"error":"'SELEC'","hint":""},"queryContext":[{"objectType":"","objectName":"","startIndex":1,"stopIndex":12,"fragment":"SELEC FROM t"}],"line":1,"position":0}} + + +-- !query +SELECT + get_json_object(result, '$.parse_success') AS parse_success, + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM (SELECT parse_sql('SELEC FROM t') AS result) +-- !query schema +struct +-- !query output +false PARSE_SYNTAX_ERROR SELEC FROM t + + +-- !query +SELECT parse_sql( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b') +-- !query schema +struct +-- !query output +{"parse_success":false,"error":{"errorClass":"UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES","messageTemplate":"The feature is not supported: Combination of ORDER BY/SORT BY/DISTRIBUTE BY/CLUSTER BY.","sqlState":"0A000","queryContext":[{"objectType":"","objectName":"","startIndex":19,"stopIndex":42,"fragment":"ORDER BY a\n CLUSTER BY b"}],"line":3,"position":1}} + + +-- !query +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].startIndex') AS start_index +FROM ( + SELECT parse_sql( +'SELECT * + FROM t + ORDER BY a + CLUSTER BY b') AS result +) +-- !query schema +struct +-- !query output +UNSUPPORTED_FEATURE.COMBINATION_QUERY_RESULT_CLAUSES 3 1 19 + + +-- !query +SELECT parse_sql('') +-- !query schema +struct +-- !query output +{"parse_success":false,"error":{"errorClass":"PARSE_EMPTY_STATEMENT","messageTemplate":"Syntax error, unexpected empty statement.","sqlState":"42617","line":1,"position":0}} + + +-- !query +SELECT parse_sql('USE bad-name') +-- !query schema +struct +-- !query output +{"parse_success":false,"error":{"errorClass":"INVALID_IDENTIFIER","messageTemplate":"The unquoted identifier is invalid and must be back quoted as: ``.\nUnquoted identifiers can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_').\nUnquoted identifiers must also not start with a digit.\nDifferent data sources and meta stores may impose additional restrictions on valid identifiers.","sqlState":"42602","messageParameters":{"ident":"bad-name"},"queryContext":[{"objectType":"","objectName":"","startIndex":1,"stopIndex":12,"fragment":"USE bad-name"}],"line":1,"position":7}} + + +-- !query +SELECT parse_sql('WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c') +-- !query schema +struct +-- !query output +{"parse_success":false,"error":{"errorClass":"DUPLICATED_CTE_NAMES","messageTemplate":"CTE definition can't have duplicate names: .","sqlState":"42602","messageParameters":{"duplicateNames":"`c`"},"queryContext":[{"objectType":"","objectName":"","startIndex":1,"stopIndex":53,"fragment":"WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c"}],"line":1,"position":0}} + + +-- !query +SELECT parse_sql('MERGE INTO target USING source ON target.id = source.id') +-- !query schema +struct +-- !query output +{"parse_success":false,"error":{"errorClass":"MERGE_WITHOUT_WHEN","messageTemplate":"There must be at least one WHEN clause in a MERGE statement.","sqlState":"42601","queryContext":[{"objectType":"","objectName":"","startIndex":1,"stopIndex":55,"fragment":"MERGE INTO target USING source ON target.id = source.id"}],"line":1,"position":0}} + + +-- !query +SELECT parse_sql('EXPLAIN SELECT 1') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"EXPLAIN","statement_code":-23,"select_list":[{"name":[]}]} + + +-- !query +SELECT parse_sql('SET spark.sql.adaptive.enabled=true') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"SET","statement_code":-24} + + +-- !query +SELECT parse_sql('ADD JAR /tmp/x.jar') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"ADD JAR","statement_code":-26} + + +-- !query +SELECT parse_sql('CREATE VIEW v AS SELECT a, b FROM t') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"CREATE VIEW","statement_code":84,"table_references":[["v"],["t"]],"select_list":[{"name":["a"]},{"name":["b"]}]} + + +-- !query +SELECT parse_sql('SELECT 1 AS IDENTIFIER(''alias.field'')') +-- !query schema +struct +-- !query output +{"parse_success":false,"error":{"errorClass":"IDENTIFIER_TOO_MANY_NAME_PARTS","messageTemplate":" is not a valid identifier as it has more than name parts.","sqlState":"42601","messageParameters":{"identifier":"`alias`.`field`","limit":"1"},"queryContext":[{"objectType":"","objectName":"","startIndex":8,"stopIndex":37,"fragment":"1 AS IDENTIFIER('alias.field')"}],"line":1,"position":12}} + + +-- !query +SELECT parse_sql('SELECT DATE ''not-a-date''') +-- !query schema +struct +-- !query output +{"parse_success":false,"error":{"errorClass":"INVALID_TYPED_LITERAL","messageTemplate":"The value of the typed literal is invalid: .","sqlState":"42604","messageParameters":{"value":"'not-a-date'","valueType":"\"DATE\""},"queryContext":[{"objectType":"","objectName":"","startIndex":8,"stopIndex":24,"fragment":"DATE 'not-a-date'"}],"line":1,"position":7}} + + +-- !query +SELECT parse_sql( +'BEGIN + SELECT 1; + SELEC 2; + END') +-- !query schema +struct +-- !query output +{"parse_success":false,"error":{"errorClass":"PARSE_SYNTAX_ERROR","messageTemplate":"Syntax error at or near .","sqlState":"42601","messageParameters":{"error":"'2'","hint":""},"queryContext":[{"objectType":"","objectName":"","startIndex":1,"stopIndex":35,"fragment":"BEGIN\n SELECT 1;\n SELEC 2;\n END"}],"line":3,"position":9}} + + +-- !query +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM ( + SELECT parse_sql( +'BEGIN + SELECT 1; + SELEC 2; + END') AS result +) +-- !query schema +struct +-- !query output +PARSE_SYNTAX_ERROR 3 9 BEGIN + SELECT 1; + SELEC 2; + END + + +-- !query +SELECT parse_sql( +'BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END') +-- !query schema +struct +-- !query output +{"parse_success":false,"error":{"errorClass":"LABELS_MISMATCH","messageTemplate":"Begin label does not match the end label .","sqlState":"42K0L","messageParameters":{"beginLabel":"`lbl_begin`","endLabel":"`lbl_end`"},"queryContext":[{"objectType":"","objectName":"","startIndex":10,"stopIndex":19,"fragment":"lbl_begin:"}],"line":2,"position":3}} + + +-- !query +SELECT + get_json_object(result, '$.error.errorClass') AS error_class, + get_json_object(result, '$.error.line') AS line, + get_json_object(result, '$.error.position') AS position, + get_json_object(result, '$.error.queryContext[0].fragment') AS fragment +FROM ( + SELECT parse_sql( +'BEGIN + lbl_begin: BEGIN + SELECT 1; + END lbl_end; + END') AS result +) +-- !query schema +struct +-- !query output +LABELS_MISMATCH 2 3 lbl_begin: + + +-- !query +SELECT sql_text, parse_sql(sql_text) FROM VALUES + ('SELECT 1'), + ('INSERT INTO t SELECT 1'), + ('CACHE TABLE t') +AS t(sql_text) +-- !query schema +struct +-- !query output +CACHE TABLE t {"parse_success":true,"statement_identifier":"CACHE TABLE","statement_code":-1,"table_references":[["t"]]} +INSERT INTO t SELECT 1 {"parse_success":true,"statement_identifier":"INSERT","statement_code":50,"table_references":[["t"]],"select_list":[{"name":[]}]} +SELECT 1 {"parse_success":true,"statement_identifier":"SELECT","statement_code":21,"select_list":[{"name":[]}]} + + +-- !query +SELECT parse_sql('BEGIN SELECT 1; END') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22} + + +-- !query +SELECT parse_sql('BEGIN SELECT count(a) FROM script_t WHERE c = :p; END') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["script_t"]],"function_references":[["count"]],"parameter_markers":{"named":["p"]}} + + +-- !query +SELECT parse_sql('BEGIN IF (SELECT flag FROM gate) THEN INSERT INTO dest SELECT * FROM src_if; ELSE DELETE FROM src_else; END IF; END') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["gate"],["dest"],["src_if"],["src_else"]]} + + +-- !query +SELECT parse_sql('BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN INSERT INTO err_log SELECT * FROM failing_row; END; SELECT a FROM main_t; END') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["err_log"],["failing_row"],["main_t"]]} + + +-- !query +SELECT parse_sql( +'BEGIN + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + INSERT INTO error_log + SELECT format_string(''%s'', message) FROM error_source; + END; + + WITH prepared AS ( + SELECT id, normalize_name(name) AS name + FROM input_names + WHERE is_valid(id) + ) + INSERT INTO output_names + SELECT id, upper(name) FROM prepared; + + IF EXISTS (SELECT 1 FROM control_flags WHERE enabled()) THEN + UPDATE update_target + SET value = coalesce((SELECT max(value) FROM update_source), 0) + WHERE should_update(id); + ELSE + DELETE FROM delete_target + WHERE id IN (SELECT id FROM delete_source WHERE expired(ts)); + END IF; + + FOR row AS + SELECT id FROM loop_source WHERE ready(id) + DO + SELECT audit(row.id), count(*) FROM loop_body; + END FOR; + END') +-- !query schema +struct +-- !query output +{"parse_success":true,"statement_identifier":"BEGIN END","statement_code":-22,"table_references":[["error_log"],["error_source"],["input_names"],["output_names"],["control_flags"],["update_source"],["update_target"],["delete_source"],["delete_target"],["loop_source"],["loop_body"]],"function_references":[["format_string"],["normalize_name"],["is_valid"],["upper"],["enabled"],["coalesce"],["should_update"],["max"],["expired"],["ready"],["audit"],["count"]]} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseSqlSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseSqlSuite.scala new file mode 100644 index 0000000000000..f44a409a0d641 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/expressions/ParseSqlSuite.scala @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.expressions + +import org.json4s._ +import org.json4s.jackson.JsonMethods.parse + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.plans.SQLHelper +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.StringType +import org.apache.spark.unsafe.types.UTF8String + +class ParseSqlSuite extends SparkFunSuite with ExpressionEvalHelper with SQLHelper { + + private def evalJson(sql: String): JValue = { + val result = ParseSql(Literal(sql)).eval().asInstanceOf[UTF8String].toString + parse(result) + } + + test("parse_sql is disabled by default") { + assert(!SQLConf.get.parseSqlEnabled) + checkError( + exception = intercept[AnalysisException] { + ParseSql(Literal("SELECT 1")).checkInputDataTypes() + }, + condition = "FEATURE_NOT_ENABLED", + parameters = Map( + "featureName" -> "parse_sql", + "configKey" -> SQLConf.PARSE_SQL_ENABLED.key, + "configValue" -> "true")) + } + + test("parse_sql type check succeeds when enabled") { + withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> "true") { + assert(ParseSql(Literal("SELECT 1")).checkInputDataTypes() === + TypeCheckResult.TypeCheckSuccess) + } + } + + test("parse_sql returns JSON for a valid SELECT") { + withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> "true") { + val j = evalJson("SELECT 1 AS a") + assert(j \ "parse_success" === JBool(true)) + assert(j \ "statement_identifier" === JString("SELECT")) + assert(j \ "statement_code" === JInt(21)) + } + } + + test("parse_sql returns null for null input") { + withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> "true") { + checkEvaluation(ParseSql(Literal.create(null, StringType)), null) + } + } + + test("parse_sql does not throw on syntax error") { + withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> "true") { + val j = evalJson("NOT A STATEMENT !!!") + assert(j \ "parse_success" === JBool(false)) + assert(j \ "error" \ "errorClass" === JString("PARSE_SYNTAX_ERROR")) + } + } + + test("parse_sql works with CodegenFallback path") { + withSQLConf(SQLConf.PARSE_SQL_ENABLED.key -> "true") { + val expr = ParseSql(Literal("INSERT INTO t SELECT 1")) + assert(expr.isInstanceOf[CodegenFallback]) + val j = evalJson("INSERT INTO t SELECT 1") + assert(j \ "statement_identifier" === JString("INSERT")) + } + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala new file mode 100644 index 0000000000000..62b0e4cd5c744 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/catalyst/parser/ParseSqlResultSuite.scala @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.parser + +import org.json4s._ +import org.json4s.jackson.JsonMethods.parse + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.internal.SQLConf + +/** + * Pin Table 39 codes and contracts that goldens do not cover. + * Behavioral coverage lives in sql-tests/inputs/parse-sql.sql. + */ +class ParseSqlResultSuite extends SparkFunSuite { + + private def obj(sql: String): JObject = + parse(ParseSqlResult.fromSql(sql)).asInstanceOf[JObject] + + private def tableRefs(sql: String): Set[Seq[String]] = + obj(sql) \ "table_references" match { + case JNothing => Set.empty + case JArray(arr) => arr.map { + case JArray(parts) => parts.map(_.asInstanceOf[JString].s) + case other => fail(s"unexpected table_references entry: $other") + }.toSet + case other => fail(s"unexpected table_references: $other") + } + + test("Table 39 standard and Spark code pairs are pinned") { + assert(SqlStatementCodes.Select.statementCode === 21) + assert(SqlStatementCodes.Insert.statementCode === 50) + assert(SqlStatementCodes.DeleteWhere.statementCode === 19) + assert(SqlStatementCodes.UpdateWhere.statementCode === 82) + assert(SqlStatementCodes.Merge.statementCode === 128) + assert(SqlStatementCodes.CreateTable.statementCode === 77) + assert(SqlStatementCodes.CreateView.statementCode === 84) + assert(SqlStatementCodes.DropTable.statementCode === 32) + assert(SqlStatementCodes.AlterTable.statementCode === 4) + assert(SqlStatementCodes.TruncateTable.statementCode === 139) + assert(SqlStatementCodes.Unrecognized.statementCode === 0) + assert(SqlStatementCodes.CacheTable.statementCode < 0) + assert(SqlStatementCodes.BeginEnd.statementCode === -22) + assert(SqlStatementCodes.Explain.statementCode === -23) + assert(SqlStatementCodes.Set.statementCode === -24) + assert(SqlStatementCodes.CreateMetricViewStmt.statementCode === -37) + } + + test("TABLE and VALUES classify as SELECT") { + val table = obj("TABLE t") + assert(table \ "statement_identifier" === JString("SELECT")) + assert(table \ "statement_code" === JInt(21)) + assert(tableRefs("TABLE t") === Set(Seq("t"))) + + // Eager inlining must not flip VALUES between SELECT and Unrecognized. + Seq(true, false).foreach { eager => + SQLConf.withExistingConf(new SQLConf) { + SQLConf.get.setConf(SQLConf.EAGER_EVAL_OF_UNRESOLVED_INLINE_TABLE_ENABLED, eager) + val values = obj("VALUES (1), (2)") + assert(values \ "statement_identifier" === JString("SELECT"), + s"eager=$eager") + assert(values \ "statement_code" === JInt(21), s"eager=$eager") + } + } + } + + test("CREATE FUNCTION and DECLARE VARIABLE are not table_references") { + assert(tableRefs("CREATE FUNCTION f AS 'x' USING JAR 'y.jar'").isEmpty) + assert(tableRefs("DECLARE VARIABLE x INT").isEmpty) + // Contrast: CREATE VIEW still reports the view target. + assert(tableRefs("CREATE VIEW v AS SELECT 1 AS a") === Set(Seq("v"))) + } + + test("syntax error returns STANDARD error JSON without throwing") { + val j = obj("SELEC FROM t") + assert(j \ "parse_success" === JBool(false)) + assert(j \ "error" \ "errorClass" === JString("PARSE_SYNTAX_ERROR")) + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala index dab117fa67137..7b5402119b409 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala @@ -292,6 +292,8 @@ class ExpressionInfoSuite extends SharedSparkSession { val clonedSpark = spark.cloneSession() // Coalescing partitions can change result order, so disable it. clonedSpark.conf.set(SQLConf.COALESCE_PARTITIONS_ENABLED.key, false) + // parse_sql examples require the experimental feature flag. + clonedSpark.conf.set(SQLConf.PARSE_SQL_ENABLED.key, true) val info = clonedSpark.sessionState.catalog.lookupFunctionInfo(funcId) val className = info.getClassName if (!ignoreSet.contains(className)) {