-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Schema match checking (#27) * initial look at introducing a schema match check * update targeted spark version (excluding 3.4 currently) * Spark 3.4 update (#29) * implement required changes for the Spark 3.4.0 release * Update GitHub Action * Update spark.yml move to non-deprecated versions * update remaining action and remove deprecated setting * add new test to cover more of the options setting * add missing copyright header * add unit test to cover exception case * update README.md * remove TOC * update version to final
- Loading branch information
Showing
21 changed files
with
821 additions
and
147 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
42 changes: 42 additions & 0 deletions
42
src/main/3.0/scala/org/apache/spark/sql/catalyst/FileSourceOptions.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package org.apache.spark.sql.catalyst | ||
|
|
||
| import org.apache.spark.sql.catalyst.FileSourceOptions.{IGNORE_CORRUPT_FILES, IGNORE_MISSING_FILES} | ||
| import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap | ||
| import org.apache.spark.sql.internal.SQLConf | ||
|
|
||
| /** | ||
| * Common options for the file-based data source. | ||
| */ | ||
| class FileSourceOptions( | ||
| @transient private val parameters: CaseInsensitiveMap[String]) | ||
| extends Serializable { | ||
|
|
||
| def this(parameters: Map[String, String]) = this(CaseInsensitiveMap(parameters)) | ||
|
|
||
| val ignoreCorruptFiles: Boolean = parameters.get(IGNORE_CORRUPT_FILES).map(_.toBoolean) | ||
| .getOrElse(SQLConf.get.ignoreCorruptFiles) | ||
|
|
||
| val ignoreMissingFiles: Boolean = parameters.get(IGNORE_MISSING_FILES).map(_.toBoolean) | ||
| .getOrElse(SQLConf.get.ignoreMissingFiles) | ||
| } | ||
|
|
||
| object FileSourceOptions { | ||
| val IGNORE_CORRUPT_FILES = "ignoreCorruptFiles" | ||
| val IGNORE_MISSING_FILES = "ignoreMissingFiles" | ||
| } |
40 changes: 40 additions & 0 deletions
40
src/main/3.3/scala/com/elastacloud/spark/excel/ExcelDataIterator.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| /* | ||
| * Copyright 2021 Elastacloud Limited | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.elastacloud.spark.excel | ||
|
|
||
| import com.elastacloud.spark.excel.parser.ExcelParser | ||
| import org.apache.hadoop.conf.Configuration | ||
| import org.apache.hadoop.fs.Path | ||
| import org.apache.spark.TaskContext | ||
| import org.apache.spark.internal.Logging | ||
| import org.apache.spark.sql.catalyst.InternalRow | ||
| import org.apache.spark.sql.execution.datasources.{CodecStreams, PartitionedFile} | ||
| import org.apache.spark.sql.types.StructType | ||
|
|
||
| import java.net.URI | ||
|
|
||
| private[excel] class ExcelDataIterator(options: ExcelParserOptions) extends Logging { | ||
| def readFile(conf: Configuration, file: PartitionedFile, schema: StructType): Iterator[InternalRow] = { | ||
| val filePath = new Path(new URI(file.filePath)) | ||
| val inputStream = CodecStreams.createInputStreamWithCloseResource(conf, filePath) | ||
|
|
||
| Option(TaskContext.get()).foreach(_.addTaskCompletionListener[Unit](_ => inputStream.close())) | ||
|
|
||
| val parser = new ExcelParser(inputStream, options, Some(schema), Some(schema)) | ||
| parser.getDataIterator.map(it => InternalRow.fromSeq(it)) | ||
| } | ||
| } |
53 changes: 53 additions & 0 deletions
53
src/main/3.3/scala/com/elastacloud/spark/excel/ExcelPartitionReader.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| /* | ||
| * Copyright 2021 Elastacloud Limited | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.elastacloud.spark.excel | ||
|
|
||
| import com.elastacloud.spark.excel.parser.ExcelParser | ||
| import org.apache.hadoop.fs.Path | ||
| import org.apache.spark.broadcast.Broadcast | ||
| import org.apache.spark.sql.catalyst.InternalRow | ||
| import org.apache.spark.sql.connector.read.PartitionReader | ||
| import org.apache.spark.sql.execution.datasources.{CodecStreams, PartitionedFile} | ||
| import org.apache.spark.sql.types.StructType | ||
| import org.apache.spark.util.SerializableConfiguration | ||
|
|
||
| import java.net.URI | ||
|
|
||
| private[excel] class ExcelPartitionReader( | ||
| path: PartitionedFile, | ||
| broadcastConf: Broadcast[SerializableConfiguration], | ||
| excelOptions: ExcelParserOptions, | ||
| schema: StructType, | ||
| readSchema: StructType | ||
| ) extends PartitionReader[InternalRow] { | ||
| private val filePath = new Path(new URI(path.filePath)) | ||
| private val inputStream = CodecStreams.createInputStreamWithCloseResource(broadcastConf.value.value, filePath) | ||
| private val excelParser = new ExcelParser(inputStream, excelOptions, Some(schema), Some(readSchema)) | ||
| private val dataIterator = excelParser.getDataIterator | ||
|
|
||
| override def next(): Boolean = dataIterator.hasNext | ||
|
|
||
| override def get(): InternalRow = { | ||
| val rowData = dataIterator.next() | ||
| InternalRow.fromSeq(rowData) | ||
| } | ||
|
|
||
| override def close(): Unit = { | ||
| excelParser.close() | ||
| inputStream.close() | ||
| } | ||
| } |
40 changes: 40 additions & 0 deletions
40
src/main/3.3/scala/com/elastacloud/spark/excel/ExcelPartitionReaderFactory.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| /* | ||
| * Copyright 2021 Elastacloud Limited | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.elastacloud.spark.excel | ||
|
|
||
| import org.apache.spark.broadcast.Broadcast | ||
| import org.apache.spark.sql.catalyst.{FileSourceOptions, InternalRow} | ||
| import org.apache.spark.sql.connector.read.PartitionReader | ||
| import org.apache.spark.sql.execution.datasources.PartitionedFile | ||
| import org.apache.spark.sql.execution.datasources.v2.{FilePartitionReaderFactory, PartitionReaderWithPartitionValues} | ||
| import org.apache.spark.sql.internal.SQLConf | ||
| import org.apache.spark.sql.types.StructType | ||
| import org.apache.spark.util.SerializableConfiguration | ||
|
|
||
| private[excel] case class ExcelPartitionReaderFactory( | ||
| sqlConf: SQLConf, | ||
| broadcastConf: Broadcast[SerializableConfiguration], | ||
| dataSchema: StructType, | ||
| readDataSchema: StructType, | ||
| partitionSchema: StructType, | ||
| excelOptions: ExcelParserOptions | ||
| ) extends FilePartitionReaderFactory { | ||
| override def buildReader(partitionedFile: PartitionedFile): PartitionReader[InternalRow] = { | ||
| val reader = new ExcelPartitionReader(partitionedFile, broadcastConf, excelOptions, dataSchema, readDataSchema) | ||
| new PartitionReaderWithPartitionValues(reader, readDataSchema, partitionSchema, partitionedFile.partitionValues) | ||
| } | ||
| } |
Oops, something went wrong.