From 81dcfd78dcaa6573f60bca79f9874e41c86cae2e Mon Sep 17 00:00:00 2001 From: paras-sehrawat Date: Thu, 29 Jan 2026 12:58:14 +0530 Subject: [PATCH] FileSystem to be chosen based on the file system of Input Path A. --- bigfiles/src/main/scala/za/co/absa/parser/ArgsParser.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigfiles/src/main/scala/za/co/absa/parser/ArgsParser.scala b/bigfiles/src/main/scala/za/co/absa/parser/ArgsParser.scala index b27fc1b..40ffa2b 100644 --- a/bigfiles/src/main/scala/za/co/absa/parser/ArgsParser.scala +++ b/bigfiles/src/main/scala/za/co/absa/parser/ArgsParser.scala @@ -83,7 +83,8 @@ object ArgsParser { */ def validate(args: Arguments)(implicit spark: SparkSession): Boolean = { val config = spark.sparkContext.hadoopConfiguration - val fs = FileSystem.get(config) + val fsFilePath = new Path(args.inputA) + val fs = fsFilePath.getFileSystem(config) if (!fs.exists(new Path(args.inputA))) throw new IllegalArgumentException(s"Input ${args.inputA} does not exist") if (!fs.exists(new Path(args.inputB))) throw new IllegalArgumentException(s"Input ${args.inputB} does not exist") if (fs.exists(new Path(args.out))) throw new IllegalArgumentException(s"Output ${args.out} already exist")