[FEA] Add json_tuple function as supported in qualification tool (#589)

* add json_tuple support Signed-off-by: cindyyuanjiang <[email protected]> * updated unit test name Signed-off-by: cindyyuanjiang <[email protected]> * updated unit test name Signed-off-by: cindyyuanjiang <[email protected]> --------- Signed-off-by: cindyyuanjiang <[email protected]>
NVIDIA · Sep 27, 2023 · 1437b7a · 1437b7a
1 parent 3610023
commit 1437b7a
Show file tree

Hide file tree

Showing 10 changed files with 13 additions and 3 deletions.
diff --git a/core/src/main/resources/operatorsScore-databricks-aws.csv b/core/src/main/resources/operatorsScore-databricks-aws.csv
@@ -130,6 +130,7 @@ IntegralDivide,2.45
 IsNaN,2.45
 IsNotNull,2.45
 IsNull,2.45
+JsonTuple,2.45
 KnownFloatingPointNormalized,2.45
 KnownNotNull,2.45
 Lag,2.45

diff --git a/core/src/main/resources/operatorsScore-databricks-azure.csv b/core/src/main/resources/operatorsScore-databricks-azure.csv
@@ -130,6 +130,7 @@ IntegralDivide,2.73
 IsNaN,2.73
 IsNotNull,2.73
 IsNull,2.73
+JsonTuple,2.73
 KnownFloatingPointNormalized,2.73
 KnownNotNull,2.73
 Lag,2.73

diff --git a/core/src/main/resources/operatorsScore-dataproc-l4.csv b/core/src/main/resources/operatorsScore-dataproc-l4.csv
@@ -130,6 +130,7 @@ IntegralDivide,4.16
 IsNaN,4.16
 IsNotNull,4.16
 IsNull,4.16
+JsonTuple,4.16
 KnownFloatingPointNormalized,4.16
 KnownNotNull,4.16
 Lag,4.16

diff --git a/core/src/main/resources/operatorsScore-dataproc-t4.csv b/core/src/main/resources/operatorsScore-dataproc-t4.csv
@@ -130,6 +130,7 @@ IntegralDivide,4.88
 IsNaN,4.88
 IsNotNull,4.88
 IsNull,4.88
+JsonTuple,4.88
 KnownFloatingPointNormalized,4.88
 KnownNotNull,4.88
 Lag,4.88

diff --git a/core/src/main/resources/operatorsScore-emr-a10.csv b/core/src/main/resources/operatorsScore-emr-a10.csv
@@ -130,6 +130,7 @@ IntegralDivide,2.59
 IsNaN,2.59
 IsNotNull,2.59
 IsNull,2.59
+JsonTuple,2.59
 KnownFloatingPointNormalized,2.59
 KnownNotNull,2.59
 Lag,2.59

diff --git a/core/src/main/resources/operatorsScore-emr-t4.csv b/core/src/main/resources/operatorsScore-emr-t4.csv
@@ -130,6 +130,7 @@ IntegralDivide,2.07
 IsNaN,2.07
 IsNotNull,2.07
 IsNull,2.07
+JsonTuple,2.07
 KnownFloatingPointNormalized,2.07
 KnownNotNull,2.07
 Lag,2.07

diff --git a/core/src/main/resources/operatorsScore.csv b/core/src/main/resources/operatorsScore.csv
@@ -135,6 +135,7 @@ IntegralDivide,4
 IsNaN,4
 IsNotNull,4
 IsNull,4
+JsonTuple,4
 KnownFloatingPointNormalized,4
 KnownNotNull,4
 Lag,4

diff --git a/core/src/main/resources/supportedExprs.csv b/core/src/main/resources/supportedExprs.csv
@@ -270,6 +270,9 @@ IsNotNull,S,`isnotnull`,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,P
 IsNotNull,S,`isnotnull`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA
 IsNull,S,`isnull`,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,S,S,NS,PS,PS,PS,NS
 IsNull,S,`isnull`,None,project,result,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA
+JsonTuple,S,`json_tuple`,None,project,json,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA,NA,NA,NA,NA,NA
+JsonTuple,S,`json_tuple`,None,project,field,NA,NA,NA,NA,NA,NA,NA,NA,NA,PS,NA,NA,NA,NA,NA,NA,NA,NA
+JsonTuple,S,`json_tuple`,None,project,result,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,S,NA,NA,NA
 KnownFloatingPointNormalized,S, ,None,project,input,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA
 KnownFloatingPointNormalized,S, ,None,project,result,NA,NA,NA,NA,NA,S,S,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA
 KnownNotNull,S, ,None,project,input,S,S,S,S,S,S,S,S,PS,S,S,NS,S,S,PS,PS,PS,NS

diff --git a/core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/SqlPlanParserSuite.scala b/core/src/test/scala/com/nvidia/spark/rapids/tool/planparser/SqlPlanParserSuite.scala
@@ -781,7 +781,7 @@ class SQLPlanParserSuite extends BaseTestSuite {
     }
   }
 
-  test("Expression not supported in Generate") {
+  test("json_tuple is supported in Generate") {
     TrampolineUtil.withTempDir { eventLogDir =>
       val (eventLog, _) = ToolTestUtils.generateEventLog(eventLogDir,
         "Expressions in Generate") { spark =>
@@ -791,7 +791,6 @@ class SQLPlanParserSuite extends BaseTestSuite {
             |"City":"ABCDE","State":"YZ"}""".stripMargin
         val data = Seq((1, jsonString))
         val df = data.toDF("id", "jsonValues")
-        //json_tuple which is called from GenerateExec is not supported in GPU yet.
         df.select(col("id"), json_tuple(col("jsonValues"), "Zipcode", "ZipCodeType", "City"))
       }
       val pluginTypeChecker = new PluginTypeChecker()
@@ -802,7 +801,7 @@ class SQLPlanParserSuite extends BaseTestSuite {
       }
       val execInfo = getAllExecsFromPlan(parsedPlans.toSeq)
       val generateExprs = execInfo.filter(_.exec == "Generate")
-      assertSizeAndNotSupported(1, generateExprs)
+      assertSizeAndSupported(1, generateExprs)
     }
   }
 

diff --git a/user_tools/custom_speedup_factors/operatorsList.csv b/user_tools/custom_speedup_factors/operatorsList.csv
@@ -123,6 +123,7 @@ IntegralDivide
 IsNaN
 IsNotNull
 IsNull
+JsonTuple
 KnownFloatingPointNormalized
 KnownNotNull
 Lag