batching for stdout

README.md

@@ -11,29 +11,36 @@ without relying on polling.
 
 ## Installation
 
-Download the precompiled binaries at the [release page](https://github.com/AgentCosmic/xnotify/releases).
+Download the pre-compiled binaries at the [release page](https://github.com/AgentCosmic/xnotify/releases).
 
 Or if you have Go installed you can run:
 ```go get github.com/AgentCosmic/xnotify```
 
 ## Tutorial
 
 ```
+NAME:
+   xnotify - Watch files for changes.
+   File changes will be printed to stdout in the format <operation_name> <file_path>.
+   stdin accepts a list of files to watch.
+   Use -- to execute 1 or more commands in sequence, stopping if any command exits unsuccessfully. It will kill the old tasks if a new event is triggered.
+
 USAGE:
    xnotify [options] [-- <command> [args...]...]
 
 VERSION:
-   0.2.4
+   0.3.0
 
 GLOBAL OPTIONS:
    --include value, -i value  Include path to watch recursively.
-   --exclude value, -e value  Exclude files from the search using Regular Expression. This only applies to files that were passed as arguments.
+   --exclude value, -e value  Exclude changes from files that match the Regular Expression. This will also apply to events received in server mode.
    --shallow                  Disable recursive file globbing. If the path is a directory, the contents will not be included.
    --listen value             Listen on address for file changes e.g. localhost:8080 or just :8080. See --client on how to send file changes.
-   --base value               Use this base path instead of the working directory. This will affect where --include finds the files. If using --listen, it will replace the original base path that was used at the sender. (default: "./")
+   --base value               Use this base path instead of the working directory. This changes the root directory used by --include. If using --listen, it will replace the original base path that was used at the sender. (default: "./")
    --client value             Send file changes to the address e.g. localhost:8080 or just :8080. See --listen on how to receive events.
-   --batch milliseconds       Send the events together if they occur within given milliseconds. The program will only execute given milliseconds after the last event was fired. Only valid with -- argument. (default: 0)
-   --trigger                  Run the given command immediately even if there is no file change. Only valid with -- argument.
+   --batch value              Delay emitting all events until it is idle for the given time in milliseconds (also known as debouncing). The --client argument does not support batching. (default: 0)
+   --terminator value         Terminator used to terminate each batch when printing to stdout. Only active when --batch option is used. (default: "\x00")
+   --trigger                  Run the given command immediately even if there is no file change. Only valid with the -- argument.
    --verbose                  Print verbose logs.
    --help, -h                 Print this help.
    --version, -v              print the version
@@ -71,25 +78,24 @@ Watch all files in the current directory on the host machine and send events to
 
 On the VM:
 ```
-./xnotify --listen "0.0.0.0:8090" --base "/opt/wwww/project" | xargs -L 1 ./build.sh
+./xnotify --listen "0.0.0.0:8090" --base "/home/john/project" | xargs -L 1 ./build.sh
 ```
 You need to set `--base` if the working directory path is different on the host and VM. Remember to use `0.0.0.0`
 because the traffic is coming from outside the system.
 
-Since the client is triggered using HTTP, you can manually send a request to the client address to trigger a file
-change. Send a JSON request in the following format: `{"path": "path/to/file", "operation": "event name"}`. The `operation`
+Since the client is triggered using HTTP, you can manually send a request to the client address to trigger an event.
+Send a JSON request in the following format: `{"path": "path/to/file", "operation": "event name"}`. The `operation`
 field is optional as it's only used for logging. Some possible use cases would be triggering a task after a script has
 finished running, or setting up multiple clients for different events.
 
 ### Task Runner
 
 Run multiple commands when a file changes. Kills and runs the commands again if a new event comes before the commands
-finish. Use `--batch 100` to run the command only 100ms after the last event happened. This will batch multiple
-events together and execute the command only once instead of restarting it for every single event. Commands will run in
-order as if the `&&` operator is used. Be careful not to run commands that spawn child processes as the child processes
-_might not_ terminate with the parent processes.
+finish. Commands will run in
+the same order as if the `&&` operator is used. Be careful not to run commands that spawn child processes as the child
+processes _might not_ terminate with the parent processes.
 ```
-./xnotify -i . -e "\.git$" --batch 100 -- my_lint arg1 arg2 -- ./compile.sh --flag value -- ./run.sh
+./xnotify -i . -e "\.git$" -- my_lint arg1 arg2 -- ./compile.sh --flag value -- ./run.sh
 ```
 This will run the commands in the same manner as:
 ```
@@ -100,6 +106,33 @@ You can also set the `--trigger` option if you want your command to run immediat
 ./xnotify -i . --trigger -- run_server.sh 8080
 ```
 
+### Batching
+
+Sometimes multiple file events are triggered within a very short timespan. This might cause too many processes to
+spawn. To solve this we can use the `--batch` argument. This will delay the events from emiting until a certain
+duration has passed since the last event &mdash; also known as debouncing. For example, by using `--batch 100`, the
+events will only be emitted once the last file change is 100ms old.
+
+Each batch will be terminated with a null character by default. You can change this using `--terminator`. Each event
+will still be terminated with new lines. Here are some examples with `xargs`.
+
+The `-0` or `--null` flags allow `xargs` to recognize each batch using the null character:
+
+```
+./xnotify -i . --batch 1000 | xargs -0 -L 1 ./build.sh
+```
+
+Using a different terminator such as `xxx`:
+```
+./xnotify -i . --batch 1000 --terminator xxx | xargs -d xxx -L 1 ./build.sh
+```
+
+Since the events are batched, the `$1` argument will now contain a list of events. You will have to parse this text to
+extract the path information if you need it.
+
+Batching works with the task runner too. It will only restart the tasks after the last event is emitted. The
+`--terminator` flag does not apply here.
+
 ## Real World Examples
 
 [How to Get File Notification Working on Docker, VirtualBox, VMWare or Vagrant](https://daltontan.com/file-notification-docker-virtualbox-vmware-vagrant/27)

go.mod

@@ -2,6 +2,7 @@ module github.com/AgentCosmic/xnotify
 
 require (
 	github.com/fsnotify/fsnotify v1.4.9
+	gopkg.in/alessio/shellescape.v1 v1.0.0-20170105083845-52074bc9df61
 	gopkg.in/urfave/cli.v1 v1.20.0
 )
 

go.sum

@@ -2,5 +2,7 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9 h1:L2auWcuQIvxz9xSEqzESnV/QN/gNRXNApHi3fYwl2w0=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+gopkg.in/alessio/shellescape.v1 v1.0.0-20170105083845-52074bc9df61 h1:8ajkpB4hXVftY5ko905id+dOnmorcS2CHNxxHLLDcFM=
+gopkg.in/alessio/shellescape.v1 v1.0.0-20170105083845-52074bc9df61/go.mod h1:IfMagxm39Ys4ybJrDb7W3Ob8RwxftP0Yy+or/NVz1O8=
 gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0=
 gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=

xnotify.go

@@ -15,11 +15,13 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/fsnotify/fsnotify"
 	"gopkg.in/urfave/cli.v1"
+	"gopkg.in/alessio/shellescape.v1"
 )
 
 // Event for each file change
@@ -31,19 +33,27 @@ type Event struct {
 
 type program struct {
 	eventChannel    chan Event // track file change events
-	tasks           [][]string
-	process         *os.Process
-	processChannel  chan bool // track the process we are spawning
 	clientAddress   string
 	batchMS         int
-	batchSize       int32
 	base            string
 	defaultBase     string
-	trigger         bool
-	hasTasks        bool
 	excludePatterns []string
+	// used for print runner
+	terminator  string // used to terminate each batch
+	mu          sync.Mutex
+	timer       *time.Timer // used for debouncing
+	batchEvents []Event     // collect events for next batch
+	// used for task runner
+	trigger        bool        //  whether to trigger tasks immediately on startup
+	hasTasks       bool        // if there is any task to run
+	batchSize      int32       // keep track of the last event to trigger the runner
+	tasks          [][]string  // tasks to run
+	process        *os.Process // task process
+	processChannel chan bool   // track the process we are spawning
 }
 
+const NullChar = "\000"
+
 func main() {
 	log.SetPrefix("[xnotify] ")
 	log.SetFlags(0)
@@ -54,11 +64,11 @@ func main() {
 	}
 	app := cli.NewApp()
 	app.Name = "xnotify"
-	app.Version = "0.2.4"
+	app.Version = "0.3.0"
 	app.Usage = "Watch files for changes." +
 		"\n   File changes will be printed to stdout in the format <operation_name> <file_path>." +
 		"\n   stdin accepts a list of files to watch." +
-		"\n   Use -- to execute 1 or more commands in sequence, stopping if any command exits unsuccessfully."
+		"\n   Use -- to execute 1 or more commands in sequence, stopping if any command exits unsuccessfully. It will kill the old tasks if a new event is triggered."
 	app.UsageText = "xnotify [options] [-- <command> [args...]...]"
 	app.HideHelp = true
 	app.Flags = []cli.Flag{
@@ -68,7 +78,7 @@ func main() {
 		},
 		cli.StringSliceFlag{
 			Name:  "exclude, e",
-			Usage: "Exclude changes from files that match the Regular Expression. This will also apply to events recieved in server mode.",
+			Usage: "Exclude changes from files that match the Regular Expression. This will also apply to events received in server mode.",
 		},
 		cli.BoolFlag{
 			Name:  "shallow",
@@ -81,7 +91,7 @@ func main() {
 		cli.StringFlag{
 			Name:        "base",
 			Value:       prog.defaultBase,
-			Usage:       "Use this base path instead of the working directory. This will affect where --include finds the files. If using --listen, it will replace the original base path that was used at the sender.",
+			Usage:       "Use this base path instead of the working directory. This changes the root directory used by --include. If using --listen, it will replace the original base path that was used at the sender.",
 			Destination: &prog.base,
 		},
 		cli.StringFlag{
@@ -91,12 +101,18 @@ func main() {
 		},
 		cli.IntFlag{
 			Name:        "batch",
-			Usage:       "Send the events together if they occur within given `milliseconds`. The program will only execute given milliseconds after the last event was fired. Only valid with -- argument.",
+			Usage:       "Delay emitting all events until it is idle for the given time in milliseconds (also known as debouncing). The --client argument does not support batching.",
 			Destination: &prog.batchMS,
 		},
+		cli.StringFlag{
+			Name:        "terminator",
+			Usage:       "Terminator used to terminate each batch when printing to stdout. Only active when --batch option is used.",
+			Destination: &prog.terminator,
+			Value:       NullChar,
+		},
 		cli.BoolFlag{
 			Name:        "trigger",
-			Usage:       "Run the given command immediately even if there is no file change. Only valid with -- argument.",
+			Usage:       "Run the given command immediately even if there is no file change. Only valid with the -- argument.",
 			Destination: &prog.trigger,
 		},
 		cli.BoolFlag{
@@ -148,8 +164,8 @@ func (prog *program) action(c *cli.Context) (err error) {
 	if prog.base != prog.defaultBase && c.String("listen") == "" {
 		noEffect("base")
 	}
-	if prog.batchMS != 0 && !prog.hasTasks {
-		noEffect("batch")
+	if prog.terminator != NullChar && prog.batchMS == 0 {
+		noEffect("terminator")
 	}
 	if prog.trigger && !prog.hasTasks {
 		noEffect("trigger")
@@ -367,11 +383,6 @@ func (prog *program) fileChanged(e Event) {
 	}
 }
 
-// runner that prints to stdout
-func (prog *program) printRunner(e Event) {
-	fmt.Printf("%s %s\n", e.Operation, e.Path)
-}
-
 // runner that sends to a another client via http
 func (prog *program) httpRunner(e Event) {
 	b, err := json.Marshal(&e)
@@ -392,6 +403,29 @@ func (prog *program) httpRunner(e Event) {
 	}
 }
 
+// runner that prints to stdout
+func (prog *program) printRunner(e Event) {
+	prog.mu.Lock()
+	defer prog.mu.Unlock()
+	dur, err := time.ParseDuration(fmt.Sprint(prog.batchMS, "ms"))
+	if err != nil {
+		panic(err)
+	}
+	if prog.timer != nil {
+		prog.timer.Stop()
+	}
+	prog.batchEvents = append(prog.batchEvents, e)
+	prog.timer = time.AfterFunc(dur, func() {
+		for _, e := range prog.batchEvents {
+			fmt.Printf("%s %s\n", e.Operation, shellescape.Quote(e.Path))
+		}
+		if dur > 0 {
+			fmt.Print(prog.terminator)
+		}
+		prog.batchEvents = make([]Event, 0)
+	})
+}
+
 //
 // ----- Batch program runner -----
 //
@@ -405,6 +439,7 @@ func (prog *program) programRunner(eventChannel chan Event, e Event) {
 	}
 	atomic.AddInt32(&prog.batchSize, 1)
 	time.AfterFunc(dur, func() {
+		// only need to execute once the last task is here, means the batchMS time has passed since last event
 		if atomic.LoadInt32(&prog.batchSize) == 1 {
 			// if program is already done, there will be no effect
 			if prog.process != nil {
@@ -415,6 +450,7 @@ func (prog *program) programRunner(eventChannel chan Event, e Event) {
 				// need to clear anything from previous run
 				<-prog.processChannel
 			}
+			// tell the loop there's new event
 			eventChannel <- e
 			// wait until the process is captured before proceeding so we can kill it later
 			<-prog.processChannel