diff --git a/build.sbt b/build.sbt index 2022792..a06be6a 100644 --- a/build.sbt +++ b/build.sbt @@ -23,7 +23,7 @@ lazy val root = (project in file(".")) .settings( organization := "ai.hunters", name := "spark-adaptive-file-connector", - version := "1.0.0") + version := "1.0.1") ThisBuild / versionScheme := Some("semver-spec") diff --git a/scalastyle-config.xml b/scalastyle-config.xml index 06ee4c8..f7b622a 100644 --- a/scalastyle-config.xml +++ b/scalastyle-config.xml @@ -60,7 +60,7 @@ This file is divided into 3 sections: - + true diff --git a/src/main/scala/ai/hunters/spark/sql/streaming/path/DynamicPathGenerator.scala b/src/main/scala/ai/hunters/spark/sql/streaming/path/DynamicPathGenerator.scala index aa72da5..9bc9b7d 100644 --- a/src/main/scala/ai/hunters/spark/sql/streaming/path/DynamicPathGenerator.scala +++ b/src/main/scala/ai/hunters/spark/sql/streaming/path/DynamicPathGenerator.scala @@ -17,16 +17,11 @@ * limitations under the License. */ - package ai.hunters.spark.sql.streaming.path import java.time.{LocalDate, ZoneId} -import DynamicPathGenerator.{ - DAY_WILDCARD, - MONTH_WILDCARD, - YEAR_WILDCARD -} +import DynamicPathGenerator.{DAY_WILDCARD, MONTH_WILDCARD, YEAR_WILDCARD} class DynamicPathGenerator(wildcardedGlob: String, maxNumberOfDaysToRead: Int, timeZone: ZoneId) extends PathGenerator { diff --git a/src/main/scala/org/apache/spark/sql/streaming/fs/DynamicPathsFileStreamSourceProvider.scala b/src/main/scala/org/apache/spark/sql/streaming/fs/DynamicPathsFileStreamSourceProvider.scala index b57f27c..74f6dce 100644 --- a/src/main/scala/org/apache/spark/sql/streaming/fs/DynamicPathsFileStreamSourceProvider.scala +++ b/src/main/scala/org/apache/spark/sql/streaming/fs/DynamicPathsFileStreamSourceProvider.scala @@ -69,6 +69,10 @@ class DynamicPathsFileStreamSourceProvider DynamicPathsFileStreamSourceProvider.MAX_NUMBER_OF_DAYS_TO_READ_PARAMETER_NAME, DynamicPathsFileStreamSourceProvider.DEFAULT_MAX_NUMBER_OF_DAYS_TO_READ) .toInt + require( + maxNumberOfDaysToRead >= DynamicPathsFileStreamSourceProvider.DEFAULT_MAX_NUMBER_OF_DAYS_TO_READ.toInt, + s"maxNumberOfDaysToRead must be at least " + + s"${DynamicPathsFileStreamSourceProvider.DEFAULT_MAX_NUMBER_OF_DAYS_TO_READ} for no data loss in midnight.") val sourcePathsTimezone = ZoneId.of( parameters.getOrElse(