Release 1.6.0

README.md

@@ -48,7 +48,7 @@ USAGE:
    scc DIRECTORY
 
 VERSION:
-   1.5.0
+   1.6.0
 
 COMMANDS:
      help, h  Shows a list of commands or help for one command
@@ -66,7 +66,8 @@ GLOBAL OPTIONS:
    --complexity, -c                    Set to skip complexity calculations note will be overridden if wide is set
    --wide, -w                          Set to check produce more output such as complexity and code vs complexity ranking. Same as setting format to wide
    --averagewage value, --aw value     Set as integer to set the average wage used for basic COCOMO calculation (default: 56286)
-   --cocomo, --co                      Set to check remove cocomo calculation output
+   --cocomo, --co                      Set to check remove COCOMO calculation output
+   --filegccount value, --fgc value    How many files to parse before turning the GC on (default: 10000)
    --debug                             Set to enable debug output
    --trace                             Set to enable trace output, not recommended for multiple files
    --help, -h                          show help
@@ -107,7 +108,11 @@ Estimated People Required 19.867141
 
 ### Performance
 
-Generally `scc` will be very close to the runtime of `tokei` or faster than any other code counter out there. However if you want greater performance and you have RAM to spare you can disable the garbage collector like the following on linux `GOGC=-1 scc .` which should speed things up considerably. See the below for example runtimes on a 16 CPU Linux machine running against the linux kernel source.
+Generally `scc` will be very close to the runtime of `tokei` or faster than any other code counter out there. It is designed to scale to as many CPU's cores as you can provide.
+
+However if you want greater performance and you have RAM to spare you can disable the garbage collector like the following on linux `GOGC=-1 scc .` which should speed things up considerably.
+
+See the below for example runtimes on a 16 CPU Linux machine running against the linux kernel source.
 
 ```
 scc                        1.489 s ± 0.055 s
@@ -136,6 +141,12 @@ Its possible that you may see the counts vary between runs. This usually means o
 
 To help identify this issue run scc like so `scc -v .` and look for the message `too many open files` in the output. If it is there you can rectify it by setting your ulimit to a higher value.
 
+### Low Memory
+
+If you are running `scc` in a low memory envrionment < 512 MB of RAM you may need to set `--filegccount` or `--fgc` to a lower value such as `0` to force the garbage collector to be on at all times.
+
+A sign that this is required will be `scc` crashing with panic errors.
+
 ### Tests
 
 scc is pretty well tested with many unit, integration and benchmarks to ensure that it is fast and complete.

main.go

@@ -17,7 +17,7 @@ func main() {
 	app := cli.NewApp()
 	app.EnableBashCompletion = true
 	app.Name = "scc"
-	app.Version = "1.5.0"
+	app.Version = "1.6.0"
 	app.Usage = "Sloc, Cloc and Code. Count lines of code in a directory with complexity estimation."
 	app.UsageText = "scc DIRECTORY"
 
@@ -92,6 +92,12 @@ func main() {
 			Usage:       "Set to check remove COCOMO calculation output",
 			Destination: &processor.Cocomo,
 		},
+		cli.IntFlag{
+			Name:        "filegccount, fgc",
+			Usage:       "How many files to parse before turning the GC on",
+			Destination: &processor.GcFileCount,
+			Value:       10000,
+		},
 		cli.BoolFlag{
 			Name:        "debug",
 			Usage:       "Set to enable debug output",

processor/file.go

@@ -6,6 +6,7 @@ import (
 	"github.com/monochromegane/go-gitignore"
 	"io/ioutil"
 	"path/filepath"
+	"runtime/debug"
 	"strings"
 	"sync"
 )
@@ -67,6 +68,9 @@ func walkDirectoryParallel(root string, output *chan *FileJob) {
 		extensionLookup = wlExtensionLookup
 	}
 
+	var mutex = &sync.Mutex{}
+	totalCount := 0
+
 	var wg sync.WaitGroup
 	all, _ := ioutil.ReadDir(root)
 	// TODO the gitignore should check for futher gitignores deeper in the tree
@@ -127,7 +131,15 @@ func walkDirectoryParallel(root string, output *chan *FileJob) {
 									}
 
 									if ok {
+										mutex.Lock()
+										totalCount++
+										mutex.Unlock()
 										*output <- &FileJob{Location: root, Filename: info.Name(), Extension: extension, Language: language}
+
+										// Turn GC back to what it was before if we have parsed enough files
+										if totalCount >= GcFileCount {
+											debug.SetGCPercent(gcPercent)
+										}
 									} else if Verbose {
 										printWarn(fmt.Sprintf("skipping file unknown extension: %s", info.Name()))
 									}
@@ -152,6 +164,9 @@ func walkDirectoryParallel(root string, output *chan *FileJob) {
 				language, ok := extensionLookup[extension]
 
 				if ok {
+					mutex.Lock()
+					totalCount++
+					mutex.Unlock()
 					*output <- &FileJob{Location: filepath.Join(root, f.Name()), Filename: f.Name(), Extension: extension, Language: language}
 				} else if Verbose {
 					printWarn(fmt.Sprintf("skipping file unknown extension: %s", f.Name()))

processor/processor.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"runtime"
+	"runtime/debug"
 	"sort"
 	"strings"
 )
@@ -31,6 +32,8 @@ var FileProcessJobQueueSize = runtime.NumCPU()
 var FileSummaryJobQueueSize = runtime.NumCPU()
 var WhiteListExtensions = ""
 var AverageWage int64 = 56286
+var GcFileCount = 10000
+var gcPercent = -1
 
 // Not set via flags but by arguments following the the flags
 var DirFilePaths = []string{}
@@ -43,6 +46,7 @@ var LanguageFeatures = map[string]LanguageFeature{}
 // Needs to be called at least once in order for anything to actually happen
 func ProcessConstants() {
 	var database = loadDatabase()
+	gcPercent = debug.SetGCPercent(gcPercent)
 
 	startTime := makeTimestampNano()
 	for name, value := range database {