Merge pull request #7 from flant/fix_jq_call_loop_problem

fix: explicit JqCallLoop can lead to runtime C errors

Author: diafour

README.md

@@ -7,45 +7,48 @@ CGO bindings for jq with cache for compiled programs
 ```
 import (
   "fmt"
-  . "github.com/flant/libjq-go" // import Jq, JqMainThread and JqCallLoop
+  . "github.com/flant/libjq-go" // import Jq() shortcut
 )
 
 func main() {
-  // Jq instance with direct calls of libjq methods. Note that it cannot be used in go routines.
-  var jq = JqMainThread
-
-  // Run one program with one input.
-  res, err := jq().Program(".foo").Run(`{"foo":"bar"}`)
-
-  // Use directory with jq modules.
-  res, err := jq().WithLibPath("./jq_lib").
-    Program(...).
-    Run(...)
-
-  // Use jq state cache to speedup handling of multiple inputs.
-  prg, err := jq().Program(...).Precompile()
-  for _, data := range InputJsons {
-    res, err = prg.Run(data)
-    // do something with filter result ...
-  }
-
-  // Use jq from go-routines.
-  // Jq() helper returns instance that use LockOsThread trick to run libjq methods in main thread.
-  done := make(chan struct{})
-
-  go func() {
-    res, err := Jq().Program(".foo").Run(`{"foo":"bar"}`)
-    done <- struct{}{}
-  }()
-
-  // main is locked here.
-  JqCallLoop(done)
+	// 1. Run one program with one input.
+	res, err = Jq().Program(".foo").Run(`{"foo":"bar"}`)
+
+	// 2. Use directory with jq modules.
+	res, err = Jq().WithLibPath("./jq_lib").
+		Program(`....`).
+		Run(`...`)
+	
+	// 3. Use program text as a key for a cache.
+	for _, data := range inputJsons {
+		res, err = Jq().Program(".foo").Cached().Run(data)
+		// Do something with result ...
+	}
+	
+	// 4. Explicitly precompile jq expression.
+	prg, err := Jq().Program(".foo").Precompile()
+	for _, data := range inputJsons {
+		res, err = prg.Run(data)
+		// Do something with result ...
+	}
+	
+	// 5. It is safe to use Jq() from multiple go-routines.
+	//    Note however that programs are executed synchronously.
+	go func() {
+		res, err = Jq().Program(".foo").Run(`{"foo":"bar"}`)
+	}()
+	go func() {
+		res, err = Jq().Program(".foo").Cached().Run(`{"foo":"bar"}`)
+	}()
 }
 ```
 
+This code is available in [example.go](example/example.go) as a working example.
 
 ## Build
 
+1. Local build
+
 To build your program with this library, you should install some build dependencies and statically compile oniguruma and jq libraries:
 
 ```
@@ -64,6 +67,10 @@ Now you can build your application:
 CGO_ENABLED=1 CGO_CFLAGS="${LIBJQ_CFLAGS}" CGO_LDFLAGS="${LIBJQ_LDFLAGS}" go build <your arguments>
 ```
 
+2. Docker build
+
+If you want to build your program with docker, you can build oniguruma and jq in artifact image and then copy them to go builder image. See example of this approach in [Dockerfile](https://github.com/flant/shell-operator/blob/master/Dockerfile) of a shell-operator — the real project that use this library.
+
 ## Inspired projects
 
 There are other `jq` bindings in Go:

example/example.go

@@ -2,52 +2,143 @@ package main
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
+	"sync"
 
 	. "github.com/flant/libjq-go"
 )
 
+/*
+Run it locally if oniguruma and jq are compiled:
+
+CGO_ENABLED=1 \
+CGO_CFLAGS="-Ipath-to-jq_lib/include" \
+CGO_LDFLAGS="-Lpath-to-oniguruma_lib/lib -Lpath-to-jq_lib/lib" \
+go run example.go
+
+1. "bar"
+2. kebab-string-here "kebabStringHere"
+3. "bar-quux"
+3. "baz-baz"
+4. "Foo quux"
+4. "Foo baz"
+5. "bar"
+5. "bar"
+5. "bar"
+
+*/
+
 func main() {
 	var res string
 	var err error
+	var inputJsons []string
 
-	// Jq instance with direct calls of libjq methods — cannot be used is go routines.
-	var jq = JqMainThread
-
-	// Run one program with one input.
-	res, err = jq().Program(".foo").Run(`{"foo":"bar"}`)
+	// 1. Run one program with one input.
+	res, err = Jq().Program(".foo").Run(`{"foo":"bar"}`)
 	if err != nil {
 		panic(err)
 	}
-	fmt.Printf("filter result: %s\n", res)
+	fmt.Printf("1. %s\n", res)
+	// Should print
+	// 1. "bar"
+
+	// 2. Use directory with jq modules.
+	prepareJqLib()
+	res, err = Jq().WithLibPath("./jq_lib").
+		Program(`include "mylibrary"; .foo|mymethod`).
+		Run(`{"foo":"kebab-string-here"}`)
+	fmt.Printf("2. %s %s\n", "kebab-string-here", res)
+	removeJqLib()
+	// Should print
+	// 2. kebab-string-here "kebabStringHere"
+
+	// 3. Use program text as a key for a cache.
+	inputJsons = []string{
+		`{ "foo":"bar-quux" }`,
+		`{ "foo":"baz-baz" }`,
+		// ...
+	}
+	for _, data := range inputJsons {
+		res, err = Jq().Program(".foo").Cached().Run(data)
+		if err != nil {
+			panic(err)
+		}
+		// Now do something with filter result ...
+		fmt.Printf("3. %s\n", res)
+	}
+	// Should print
+	// 3. "bar-quux"
+	// 3. "baz-baz"
 
-	// Use jq state cache to speedup handling of multiple inputs.
-	InputJson := []string{
-		`{..fields..}`,
-		`{..fields..}`,
+	// 4. Explicitly precompile jq expression.
+	inputJsons = []string{
+		`{ "bar":"Foo quux" }`,
+		`{ "bar":"Foo baz" }`,
+		// ...
 	}
-	jqp, err := jq().Program(".[]|.bar").Precompile()
+	prg, err := Jq().Program(".bar").Precompile()
 	if err != nil {
 		panic(err)
 	}
-	for _, data := range InputJson {
-		res, err = jqp.Run(data)
-		// do something with filter result ...
+	for _, data := range inputJsons {
+		res, err = prg.Run(data)
+		if err != nil {
+			panic(err)
+		}
+		// Now do something with filter result ...
+		fmt.Printf("4. %s\n", res)
 	}
+	// Should print
+	// 4. "Foo quux"
+	// 4. "Foo baz"
 
-	// Use directory with jq modules.
-	res, err = jq().WithLibPath("./jq_lib").
-		Program(`include "libname"; .foo|libmethod`).
-		Run(`{"foo":"json here"}`)
-
-	// Use jq from go-routines.
-	// Jq() returns instance that use LockOsThread trick to run libjq methods in main thread.
-	done := make(chan struct{})
-
+	// 5. It is safe to use Jq() from multiple go-routines.
+	//    Note however that programs are executed synchronously.
+	wg := sync.WaitGroup{}
+	wg.Add(3)
 	go func() {
 		res, err = Jq().Program(".foo").Run(`{"foo":"bar"}`)
-		done <- struct{}{}
+		if err != nil {
+			panic(err)
+		}
+		fmt.Printf("5. %s\n", res)
+		wg.Done()
+	}()
+	go func() {
+		res, err = Jq().Program(".foo").Cached().Run(`{"foo":"bar"}`)
+		if err != nil {
+			panic(err)
+		}
+		fmt.Printf("5. %s\n", res)
+		wg.Done()
+	}()
+	go func() {
+		res, err = Jq().Program(".foo").Cached().Run(`{"foo":"bar"}`)
+		if err != nil {
+			panic(err)
+		}
+		fmt.Printf("5. %s\n", res)
+		wg.Done()
 	}()
+	wg.Wait()
+	// Should print
+	// 5. "bar"
+	// 5. "bar"
+	// 5. "bar"
+}
+
+func prepareJqLib() {
+	if err := os.MkdirAll("./jq_lib", 0755); err != nil {
+		panic(err)
+	}
+	if err := ioutil.WriteFile("./jq_lib/mylibrary.jq", []byte(`def mymethod: gsub("-(?<a>[a-z])"; .a|ascii_upcase);`), 0644); err != nil {
+		panic(err)
+	}
+}
 
-	// main is locked here.
-	JqCallLoop(done)
+func removeJqLib() {
+	if err := os.RemoveAll("./jq_lib"); err != nil {
+		panic(err)
+	}
 }

go.mod

@@ -2,4 +2,4 @@ module github.com/flant/libjq-go
 
 go 1.12
 
-require github.com/stretchr/testify v1.4.0
+require github.com/onsi/gomega v1.5.0

go.sum

@@ -1,10 +1,26 @@
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

jq.go

@@ -1,22 +1,11 @@
 package libjq_go
 
-import "github.com/flant/libjq-go/pkg/jq"
+import (
+	"github.com/flant/libjq-go/pkg/jq"
+)
 
-// Jq is a default jq invoker with a cache for programs and a jq calls proxy
+// Jq is handy shortcut to use a default jq invoker with enabled cache for programs
 func Jq() *jq.Jq {
-	return jq.NewJq().
-		WithCache(jq.JqDefaultCache()).
-		WithCallProxy(jq.JqCall)
-}
-
-func JqMainThread() *jq.Jq {
 	return jq.NewJq().
 		WithCache(jq.JqDefaultCache())
 }
-
-// JqCallLoop should be called from main.main method to make Jq.Program.Run calls from go routines.
-//
-// Note: this method locks thread execution.
-func JqCallLoop(done chan struct{}) {
-	jq.JqCallLoop(done)
-}

jq_test.go

@@ -1 +1,19 @@
 package libjq_go
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+)
+
+func Test_OneProgram_OneInput(t *testing.T) {
+	g := NewWithT(t)
+
+	res, err := Jq().Program(".foo").Run(`{"foo":"bar"}`)
+	g.Expect(err).ShouldNot(HaveOccurred())
+	g.Expect(res).To(Equal(`"bar"`))
+
+	res, err = Jq().Program(".foo").RunRaw(`{"foo":"bar"}`)
+	g.Expect(err).ShouldNot(HaveOccurred())
+	g.Expect(res).To(Equal(`bar`))
+}