-
Notifications
You must be signed in to change notification settings - Fork 1.9k
[Kernel connector] Handle unsupported data types in kernel-spark connector #5273
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
fb1da64
b26fef6
6f2a286
8637281
03a4ce6
b6296f9
0831eb2
95e7b43
981fd42
19945ca
1f40d47
3bff5fc
b8b71c3
0b86813
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
{"commitInfo":{"timestamp":1723768497710,"operation":"CREATE OR REPLACE TABLE AS SELECT","operationParameters":{"partitionBy":"[]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.checkpointInterval\":\"2\"}"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"2","numOutputRows":"100","numOutputBytes":"14767"},"engineInfo":"Apache-Spark/4.0.0-SNAPSHOT Delta-Lake/3.3.0-SNAPSHOT","txnId":"2cc10429-f586-4c74-805c-8d19fd180c87"}} | ||
{"metaData":{"id":"d7eb0848-b002-4e0b-9d8d-dd335c90946f","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_variants\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"variant\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_struct_of_variants\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":\"variant\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_of_array_of_variants\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"v\",\"type\":{\"type\":\"array\",\"elementType\":\"variant\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{"delta.checkpointInterval":"2"},"createdTime":1723768495302}} | ||
{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["variantType-preview"],"writerFeatures":["variantType-preview","appendOnly","invariants"]}} | ||
{"add":{"path":"part-00000-16c852df-ba66-4080-be25-530a05922422-c000.snappy.parquet","partitionValues":{},"size":7443,"modificationTime":1723768496908,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"}} | ||
{"add":{"path":"part-00001-664313d3-14b4-4dbf-8110-77001b877182-c000.snappy.parquet","partitionValues":{},"size":7324,"modificationTime":1723768496908,"dataChange":true,"stats":"{\"numRecords\":50,\"minValues\":{\"id\":50},\"maxValues\":{\"id\":99},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"}} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
{"commitInfo":{"timestamp":1723768498557,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5072"},"engineInfo":"Apache-Spark/4.0.0-SNAPSHOT Delta-Lake/3.3.0-SNAPSHOT","txnId":"78417efa-a13f-45df-add0-f96aa113fd68"}} | ||
{"add":{"path":"part-00000-9a9c570c-ee32-4322-ad2f-8c837a77d398-c000.snappy.parquet","partitionValues":{},"size":5072,"modificationTime":1723768498551,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":0},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"}} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
{"commitInfo":{"timestamp":1723768498990,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"5072"},"engineInfo":"Apache-Spark/4.0.0-SNAPSHOT Delta-Lake/3.3.0-SNAPSHOT","txnId":"d90393d5-9cdd-40f1-8861-121f2169808b"}} | ||
{"add":{"path":"part-00000-1e14ba22-3114-46d1-96fb-48b4912507ce-c000.snappy.parquet","partitionValues":{},"size":5072,"modificationTime":1723768498986,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1},\"nullCount\":{\"id\":0,\"v\":0,\"array_of_variants\":0,\"struct_of_variants\":{\"v\":0},\"map_of_variants\":0,\"array_of_struct_of_variants\":0,\"struct_of_array_of_variants\":{\"v\":0}}}"}} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
{"version":2,"size":6,"sizeInBytes":21929,"numOfAddFiles":4,"checkpointSchema":{"type":"struct","fields":[{"name":"txn","type":{"type":"struct","fields":[{"name":"appId","type":"string","nullable":true,"metadata":{}},{"name":"version","type":"long","nullable":true,"metadata":{}},{"name":"lastUpdated","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"add","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"modificationTime","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"tags","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}},{"name":"clusteringProvider","type":"string","nullable":true,"metadata":{}},{"name":"stats","type":"string","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"remove","type":{"type":"struct","fields":[{"name":"path","type":"string","nullable":true,"metadata":{}},{"name":"deletionTimestamp","type":"long","nullable":true,"metadata":{}},{"name":"dataChange","type":"boolean","nullable":true,"metadata":{}},{"name":"extendedFileMetadata","type":"boolean","nullable":true,"metadata":{}},{"name":"partitionValues","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"size","type":"long","nullable":true,"metadata":{}},{"name":"deletionVector","type":{"type":"struct","fields":[{"name":"storageType","type":"string","nullable":true,"metadata":{}},{"name":"pathOrInlineDv","type":"string","nullable":true,"metadata":{}},{"name":"offset","type":"integer","nullable":true,"metadata":{}},{"name":"sizeInBytes","type":"integer","nullable":true,"metadata":{}},{"name":"cardinality","type":"long","nullable":true,"metadata":{}},{"name":"maxRowIndex","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"baseRowId","type":"long","nullable":true,"metadata":{}},{"name":"defaultRowCommitVersion","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"metaData","type":{"type":"struct","fields":[{"name":"id","type":"string","nullable":true,"metadata":{}},{"name":"name","type":"string","nullable":true,"metadata":{}},{"name":"description","type":"string","nullable":true,"metadata":{}},{"name":"format","type":{"type":"struct","fields":[{"name":"provider","type":"string","nullable":true,"metadata":{}},{"name":"options","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"schemaString","type":"string","nullable":true,"metadata":{}},{"name":"partitionColumns","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"configuration","type":{"type":"map","keyType":"string","valueType":"string","valueContainsNull":true},"nullable":true,"metadata":{}},{"name":"createdTime","type":"long","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"protocol","type":{"type":"struct","fields":[{"name":"minReaderVersion","type":"integer","nullable":true,"metadata":{}},{"name":"minWriterVersion","type":"integer","nullable":true,"metadata":{}},{"name":"readerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}},{"name":"writerFeatures","type":{"type":"array","elementType":"string","containsNull":true},"nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}},{"name":"domainMetadata","type":{"type":"struct","fields":[{"name":"domain","type":"string","nullable":true,"metadata":{}},{"name":"configuration","type":"string","nullable":true,"metadata":{}},{"name":"removed","type":"boolean","nullable":true,"metadata":{}}]},"nullable":true,"metadata":{}}]},"checksum":"a8d400a03ead8a86dbb412f2a693e26e"} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -16,6 +16,7 @@ | |
package io.delta.kernel.spark.read; | ||
|
||
import static org.junit.jupiter.api.Assertions.assertEquals; | ||
import static org.junit.jupiter.api.Assertions.assertThrows; | ||
import static org.junit.jupiter.api.Assertions.assertTrue; | ||
|
||
import io.delta.golden.GoldenTableUtils$; | ||
|
@@ -575,6 +576,32 @@ public Dataset<Row> apply() { | |
checkAnswer(dfFunc, expectedSeq); | ||
} | ||
|
||
@Test | ||
public void testVariantUnsupportedDataType() { | ||
String tablePath = goldenTablePath("spark-variant-checkpoint"); | ||
|
||
RuntimeException exception = | ||
assertThrows( | ||
RuntimeException.class, | ||
() -> { | ||
spark.sql("SELECT * FROM `dsv2`.`delta`.`" + tablePath + "`").collect(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The error comes from:
It seesm that it is already handled in There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In the future, when getting table schema from Unity catalog, we can check whether the UC schema is supported in Kernel. For example, the Time type is not supported yet. |
||
}); | ||
|
||
Throwable rootCause = getRootCause(exception); | ||
String errorMessage = rootCause.getMessage(); | ||
|
||
assertTrue(errorMessage.contains("Unsupported data type")); | ||
assertTrue(errorMessage.contains("variant")); | ||
} | ||
|
||
private Throwable getRootCause(Throwable throwable) { | ||
Throwable cause = throwable; | ||
while (cause.getCause() != null) { | ||
cause = cause.getCause(); | ||
} | ||
return cause; | ||
} | ||
|
||
@Test | ||
public void testAllGoldenTables() { | ||
List<String> tableNames = getAllGoldenTableNames(); | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
shall we rename
spark-variant-checkpoint
to justvariant
?