Skip to content

Make the full test suite runnable #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
asp24 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Make the full test suite runnable #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
/.idea/
/.vscode/

# Corpora that loadFile fetches on demand rather than keeping in the repo.
/testdata/parking-citations-1M.json.zst
/testdata/RC_2009-01.json.zst

# Build and profiling output.
*.test
*.out
*.prof

# Editor leftovers.
*.swp
*.swo
60 changes: 37 additions & 23 deletions ndjson_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ func TestNdjsonCountWhere(t *testing.T) {
if testing.Short() {
t.Skip("skipping... too long")
}
ndjson := loadFile("testdata/parking-citations.json.zst")
ndjson := loadFile(t, "testdata/parking-citations.json.zst")
pj, err := ParseND(ndjson, nil)
if err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -322,7 +322,7 @@ func TestNdjsonCountWhere2(t *testing.T) {
if testing.Short() {
t.Skip("skipping... too long")
}
ndjson := loadFile("testdata/RC_2009-01.json.zst")
ndjson := loadFile(t, "testdata/RC_2009-01.json.zst")
// Test trimming
b := make([]byte, 0, len(ndjson)+4)
b = append(b, '\n', '\n')
Expand Down Expand Up @@ -363,11 +363,20 @@ func TestNdjsonCountWhere2(t *testing.T) {
})
}

func loadFile(filename string) []byte {
// loadFile returns a test corpus, fetching and caching it when the repository does
// not ship the file.
//
// The corpora too large to commit live on an external host, and it no longer serves
// all of them - parking-citations-1M.json.zst and RC_2009-01.json.zst both answer
// 404. A corpus that cannot be obtained skips the caller rather than failing it:
// there is nothing wrong with the code under test, and a hard failure there made
// `go test ./...` unusable. Anything else is still fatal.
func loadFile(tb testing.TB, filename string) []byte {
tb.Helper()
if !strings.HasSuffix(filename, ".zst") {
ndjson, err := ioutil.ReadFile(filename)
if err != nil {
panic("Failed to load file")
tb.Skipf("corpus %s is unavailable: %v", filename, err)
}
return ndjson
}
Expand All @@ -379,31 +388,36 @@ func loadFile(filename string) []byte {
defer f.Close()
break
}
if os.IsNotExist(err) {
fmt.Println("downloading file", filename)
resp, err := http.DefaultClient.Get("https://dl.minio.io/assets/" + filepath.Base(filename))
if err == nil && resp.StatusCode == http.StatusOK {
b, err := ioutil.ReadAll(resp.Body)
if err == nil {
err = ioutil.WriteFile(filename, b, os.ModePerm)
if err == nil {
continue
}
panic("Failed to write file:" + err.Error())
}
panic("Failed to read file:" + err.Error())
}
panic("Failed to download file:" + err.Error())
if !os.IsNotExist(err) {
tb.Fatalf("opening %s: %v", filename, err)
}
fmt.Println("downloading file", filename)
resp, err := http.DefaultClient.Get("https://dl.minio.io/assets/" + filepath.Base(filename))
if err != nil {
tb.Skipf("corpus %s could not be downloaded: %v", filename, err)
}
// err is nil here, so it cannot be used to describe a bad status.
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
tb.Skipf("corpus %s could not be downloaded: %s", filename, resp.Status)
}
b, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
tb.Skipf("corpus %s could not be read: %v", filename, err)
}
if err = ioutil.WriteFile(filename, b, os.ModePerm); err != nil {
tb.Fatalf("writing %s: %v", filename, err)
}
}
dec, err := zstd.NewReader(f)
if err != nil {
panic("Failed to create decompressor")
tb.Fatalf("creating decompressor for %s: %v", filename, err)
}
defer dec.Close()
ndjson, err := ioutil.ReadAll(dec)
if err != nil {
panic("Failed to load file")
tb.Fatalf("decompressing %s: %v", filename, err)
}
return ndjson
}
Expand Down Expand Up @@ -490,7 +504,7 @@ func BenchmarkNdjsonWarmCountStar(b *testing.B) {
b.SkipNow()
}

ndjson := loadFile("testdata/parking-citations-1M.json.zst")
ndjson := loadFile(b, "testdata/parking-citations-1M.json.zst")

pj, err := ParseND(ndjson, nil)
if err != nil {
Expand All @@ -510,7 +524,7 @@ func BenchmarkNdjsonWarmCountStarWithWhere(b *testing.B) {
b.SkipNow()
}

ndjson := loadFile("testdata/parking-citations-1M.json.zst")
ndjson := loadFile(b, "testdata/parking-citations-1M.json.zst")

pj, err := ParseND(ndjson, nil)
if err != nil {
Expand Down
8 changes: 4 additions & 4 deletions parse_json_amd64_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ func BenchmarkNdjsonStage2(b *testing.B) {
b.SkipNow()
}

ndjson := loadFile("testdata/parking-citations-1M.json.zst")
ndjson := loadFile(b, "testdata/parking-citations-1M.json.zst")
pj := internalParsedJson{}

b.SetBytes(int64(len(ndjson)))
Expand All @@ -96,7 +96,7 @@ func BenchmarkNdjsonStage1(b *testing.B) {
if !SupportedCPU() {
b.SkipNow()
}
ndjson := loadFile("testdata/parking-citations-1M.json.zst")
ndjson := loadFile(b, "testdata/parking-citations-1M.json.zst")

pj := internalParsedJson{}

Expand All @@ -116,7 +116,7 @@ func BenchmarkNdjsonColdCountStar(b *testing.B) {
if !SupportedCPU() {
b.SkipNow()
}
ndjson := loadFile("testdata/parking-citations-1M.json.zst")
ndjson := loadFile(b, "testdata/parking-citations-1M.json.zst")

b.SetBytes(int64(len(ndjson)))
b.ReportAllocs()
Expand All @@ -135,7 +135,7 @@ func BenchmarkNdjsonColdCountStarWithWhere(b *testing.B) {
if !SupportedCPU() {
b.SkipNow()
}
ndjson := loadFile("testdata/parking-citations-1M.json.zst")
ndjson := loadFile(b, "testdata/parking-citations-1M.json.zst")
const want = 110349
runtime.GC()
pj := internalParsedJson{}
Expand Down
6 changes: 3 additions & 3 deletions parsed_serialize_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ func BenchmarkSerializeNDJSON(b *testing.B) {
b.SkipNow()
}

ndjson := loadFile("testdata/parking-citations-1M.json.zst")
ndjson := loadFile(b, "testdata/parking-citations-1M.json.zst")

pj, err := ParseND(ndjson, nil)
if err != nil {
Expand Down Expand Up @@ -172,7 +172,7 @@ func BenchmarkDeSerializeNDJSON(b *testing.B) {
b.SkipNow()
}

ndjson := loadFile("testdata/parking-citations-1M.json.zst")
ndjson := loadFile(b, "testdata/parking-citations-1M.json.zst")

pj, err := ParseND(ndjson, nil)
if err != nil {
Expand Down Expand Up @@ -224,7 +224,7 @@ func TestDeSerializeNDJSON(t *testing.T) {
if testing.Short() {
t.Skip("skipping... too long")
}
ndjson := loadFile("testdata/parking-citations.json.zst")
ndjson := loadFile(t, "testdata/parking-citations.json.zst")

pj, err := ParseND(ndjson, nil)
if err != nil {
Expand Down
Loading