update tests to add tests for zstd

Updated tests to include test files for zstd compression.
Originally created by @ldsands in JuliaIO#41 and available at https://github.com/JuliaIO/parquet-compatibility now.

Author: tanmaykm

.gitignore

@@ -2,4 +2,5 @@
 *.jl.*.cov
 *.jl.mem
 parquet-compatibility/
+julia-parquet-compatibility/
 .vscode/settings.json

test/get_parcompat.jl

@@ -4,9 +4,16 @@ using Test
 function get_parcompat(parcompat=joinpath(dirname(@__FILE__), "parquet-compatibility"))
     # look for parquet-compatibility in test folder, clone to tempdir if not found
     if !isdir(parcompat)
-        parcompat = joinpath(dirname(@__FILE__), "parquet-compatibility")
         run(`git clone https://github.com/Parquet/parquet-compatibility.git $parcompat`)
     end
 end
 
+function get_juliaparcompat(juliaparcompat=joinpath(dirname(@__FILE__), "julia-parquet-compatibility"))
+    # look for julia-parquet-compatibility in test folder, clone to tempdir if not found
+    if !isdir(juliaparcompat)
+        run(`git clone https://github.com/JuliaIO/parquet-compatibility.git $juliaparcompat`)
+    end
+end
+
 get_parcompat()
+get_juliaparcompat()

test/test_decode.jl

@@ -1,8 +1,8 @@
 using Parquet
 using Test
 
-function test_decode(file, parcompat=joinpath(dirname(@__FILE__), "parquet-compatibility"))
-    p = ParFile(joinpath(parcompat, file))
+function test_decode(file)
+    p = ParFile(file)
     println("loaded $file")
     @test isa(p.meta, Parquet.FileMetaData)
 
@@ -46,13 +46,23 @@ function test_decode(file, parcompat=joinpath(dirname(@__FILE__), "parquet-compa
 end
 
 function test_decode_all_pages()
+    testfolder = joinpath(@__DIR__, "parquet-compatibility")
     for encformat in ("SNAPPY", "GZIP", "NONE")
         for fname in ("nation", "customer")
-            test_decode("parquet-testdata/impala/1.1.1-$encformat/$fname.impala.parquet")
+            testfile = joinpath(testfolder, "parquet-testdata", "impala", "1.1.1-$encformat", "$fname.impala.parquet")
+            test_decode(testfile)
         end
     end
+
+    testfolder = joinpath(@__DIR__, "julia-parquet-compatibility")
+    for encformat in ("ZSTD", "SNAPPY", "GZIP", "NONE")
+        for fname in ("nation", "customer")
+            testfile = joinpath(testfolder, "Parquet_Files", "$(encformat)_pandas_pyarrow_$(fname).parquet")
+            test_decode(testfile)
+        end
+    end
+
+    test_decode(joinpath(@__DIR__, "missingvalues", "synthetic_data.parquet"))
 end
 
 test_decode_all_pages()
-
-test_decode("synthetic_data.parquet", "missingvalues")

test/test_load.jl

@@ -2,8 +2,8 @@ using Parquet
 using Test
 using Dates
 
-function test_load(file, parcompat=joinpath(dirname(@__FILE__), "parquet-compatibility"))
-    p = ParFile(joinpath(parcompat, file))
+function test_load(file::String)
+    p = ParFile(file)
     println("loaded $file")
     @test isa(p.meta, Parquet.FileMetaData)
 
@@ -47,8 +47,8 @@ function print_names(indent, t)
     end
 end
 
-function test_schema(file, schema_name::Symbol,  parcompat=joinpath(dirname(@__FILE__), "parquet-compatibility"))
-    p = ParFile(joinpath(parcompat, file))
+function test_schema(file, schema_name::Symbol)
+    p = ParFile(file)
     println("loaded $file")
 
     #mod_name = string(schema_name) * "Module"
@@ -77,10 +77,21 @@ function test_schema(file, schema_name::Symbol,  parcompat=joinpath(dirname(@__F
 end
 
 function test_load_all_pages()
+    testfolder = joinpath(@__DIR__, "parquet-compatibility")
     for encformat in ("SNAPPY", "GZIP", "NONE")
         for fname in ("nation", "customer")
-            test_load("parquet-testdata/impala/1.1.1-$encformat/$fname.impala.parquet")
-            test_schema("parquet-testdata/impala/1.1.1-$encformat/$fname.impala.parquet", Symbol(fname * "_" * encformat))
+            testfile = joinpath(testfolder, "parquet-testdata", "impala", "1.1.1-$encformat", "$fname.impala.parquet")
+            test_load(testfile)
+            test_schema(testfile, Symbol(fname * "_" * encformat))
+        end
+    end
+
+    testfolder = joinpath(@__DIR__, "julia-parquet-compatibility")
+    for encformat in ("ZSTD", "SNAPPY", "GZIP", "NONE")
+        for fname in ("nation", "customer")
+            testfile = joinpath(testfolder, "Parquet_Files", "$(encformat)_pandas_pyarrow_$(fname).parquet")
+            test_load(testfile)
+            test_schema(testfile, Symbol(fname * "_jl_" * encformat))
         end
     end
 end