Merge pull request #93 from boyter/rgitignore

Recursive ignore file check

.gitignore

@@ -30,4 +30,5 @@ examples/performance_tests/7
 examples/performance_tests/8
 
 # Ignore test
-ignored.xml
+ignored.xml
+gitignorefile.txt

.ignore

@@ -1 +1,2 @@
 vendor/
+ignorefile.txt

README.md

@@ -7,7 +7,9 @@ A tool similar to cloc, sloccount and tokei. For counting physical the lines of
 
 Goal is to be the fastest code counter possible, but also perform COCOMO calculation like sloccount and to estimate code complexity similar to cyclomatic complexity calculators. In short one tool to rule them all.
 
-Also it has a very short name which is easy to type `scc`.
+Also it has a very short name which is easy to type `scc`. 
+
+If you don't like sloc cloc and code feel free to use the name `Succinct Code Counter`.
 
 [![Build Status](https://travis-ci.org/boyter/scc.svg?branch=master)](https://travis-ci.org/boyter/scc)
 [![Go Report Card](https://goreportcard.com/badge/github.com/boyter/scc)](https://goreportcard.com/report/github.com/boyter/scc)

examples/ignore/README.md

@@ -0,0 +1 @@
+Files in here are to ensure that .ignore and .gitignore work recursively

processor/file.go

@@ -1,7 +1,6 @@
 package processor
 
 import (
-	"errors"
 	"fmt"
 	"github.com/karrick/godirwalk"
 	"github.com/monochromegane/go-gitignore"
@@ -98,37 +97,15 @@ func walkDirectoryParallel(root string, output chan *FileJob) {
 		all, _ = ioutil.ReadDir(root)
 	}
 
-	var gitIgnore gitignore.IgnoreMatcher
-	gitIgnoreError := errors.New("")
-
+	ignores := []gitignore.IgnoreMatcher{}
 	if !GitIgnore {
-		// TODO the gitIgnore should check for further gitignores deeper in the tree
-		gitIgnore, gitIgnoreError = gitignore.NewGitIgnore(filepath.Join(root, ".gitignore"))
-		if Verbose {
-			if gitIgnoreError == nil {
-				printWarn(fmt.Sprintf("found and loaded gitignore file: %s", filepath.Join(root, ".gitignore")))
-			} else {
-				printWarn(fmt.Sprintf("no gitignore found: %s", filepath.Join(root, ".gitignore")))
-			}
-		}
+		ignores = loadIgnoreFile(root, ".gitignore", ignores)
 	}
-
-	var ignore gitignore.IgnoreMatcher
-	ignoreError := errors.New("")
-
 	if !Ignore {
-		ignore, ignoreError = gitignore.NewGitIgnore(filepath.Join(root, ".ignore"))
-		if Verbose {
-			if ignoreError == nil {
-				printWarn(fmt.Sprintf("found and loaded ignore file: %s", filepath.Join(root, ".ignore")))
-			} else {
-				printWarn(fmt.Sprintf("no ignore found: %s", filepath.Join(root, ".ignore")))
-			}
-		}
+		ignores = loadIgnoreFile(root, ".ignore", ignores)
 	}
 
 	resetGc := false
-
 	var excludes []*regexp.Regexp
 
 	for _, exclude := range Exclude {
@@ -162,24 +139,20 @@ func walkDirectoryParallel(root string, output chan *FileJob) {
 				}
 			}
 
-			if gitIgnoreError == nil && gitIgnore.Match(filepath.Join(root, f.Name()), true) {
-				if Verbose {
-					printWarn("skipping directory due to git ignore: " + filepath.Join(root, f.Name()))
-				}
-				shouldSkip = true
-			}
-
-			if ignoreError == nil && ignore.Match(filepath.Join(root, f.Name()), true) {
-				if Verbose {
-					printWarn("skipping directory due to ignore: " + filepath.Join(root, f.Name()))
+			for _, i := range ignores {
+				if i.Match(filepath.Join(root, f.Name()), true) {
+					if Verbose {
+						printWarn("skipping directory due to ignore: " + filepath.Join(root, f.Name()))
+					}
+					shouldSkip = true
+					break
 				}
-				shouldSkip = true
 			}
 
 			if !shouldSkip {
 				wg.Add(1)
 				go func(toWalk string) {
-					filejobs := walkDirectory(toWalk, PathBlacklist, extensionLookup)
+					filejobs := walkDirectory(toWalk, PathBlacklist, extensionLookup, ignores)
 					for i := 0; i < len(filejobs); i++ {
 						for _, lan := range filejobs[i].PossibleLanguages {
 							LoadLanguageFeature(lan)
@@ -206,18 +179,13 @@ func walkDirectoryParallel(root string, output chan *FileJob) {
 
 			shouldSkip := false
 
-			if gitIgnoreError == nil && gitIgnore.Match(fpath, false) {
-				if Verbose {
-					printWarn("skipping file due to git ignore: " + f.Name())
-				}
-				shouldSkip = true
-			}
-
-			if ignoreError == nil && ignore.Match(fpath, false) {
-				if Verbose {
-					printWarn("skipping file due to ignore: " + f.Name())
+			for _, ignore := range ignores {
+				if ignore.Match(filepath.Join(root, f.Name()), false) {
+					if Verbose {
+						printWarn("skipping file due to ignore: " + filepath.Join(root, f.Name()))
+					}
+					shouldSkip = true
 				}
-				shouldSkip = true
 			}
 
 			for _, exclude := range excludes {
@@ -268,7 +236,24 @@ func walkDirectoryParallel(root string, output chan *FileJob) {
 	}
 }
 
-func walkDirectory(toWalk string, blackList []string, extensionLookup map[string][]string) []FileJob {
+func loadIgnoreFile(root string, filename string, ignores []gitignore.IgnoreMatcher) []gitignore.IgnoreMatcher {
+	ignore, err := gitignore.NewGitIgnore(filepath.Join(root, filename))
+
+	if err == nil {
+		ignores = append(ignores, ignore)
+	}
+
+	if Verbose {
+		if err == nil {
+			printWarn(fmt.Sprintf("found and loaded ignore file: %s", filepath.Join(root, filename)))
+		} else {
+			printWarn(fmt.Sprintf("no ignore found: %s", filepath.Join(root, filename)))
+		}
+	}
+	return ignores
+}
+
+func walkDirectory(toWalk string, blackList []string, extensionLookup map[string][]string, ignores []gitignore.IgnoreMatcher) []FileJob {
 	extension := ""
 	var filejobs []FileJob
 
@@ -308,9 +293,25 @@ func walkDirectory(toWalk string, blackList []string, extensionLookup map[string
 						return filepath.SkipDir
 					}
 				}
-			}
 
-			if !info.IsDir() {
+				for _, i := range ignores {
+					if i.Match(root, true) {
+						if Verbose {
+							printWarn("skipping directory due to ignore: " + root)
+						}
+						return filepath.SkipDir
+					}
+				}
+			} else {
+				for _, i := range ignores {
+					if i.Match(filepath.Join(root, info.Name()), false) {
+						if Verbose {
+							printWarn("skipping file due to ignore: " + filepath.Join(root, info.Name()))
+						}
+						return nil
+					}
+				}
+
 				// Lookup in case the full name matches
 				language, ok := extensionLookup[strings.ToLower(info.Name())]
 

processor/file_test.go

@@ -1,6 +1,7 @@
 package processor
 
 import (
+	"github.com/monochromegane/go-gitignore"
 	"math/rand"
 	"path/filepath"
 	"strings"
@@ -191,7 +192,7 @@ func TestWalkDirectory(t *testing.T) {
 	Debug = true
 	Exclude = []string{"test"}
 	ProcessConstants()
-	files := walkDirectory(".", []string{}, ExtensionToLanguage)
+	files := walkDirectory(".", []string{}, ExtensionToLanguage, []gitignore.IgnoreMatcher{})
 
 	if len(files) == 0 {
 		t.Error("Expected at least one file")

test-all.sh

@@ -249,7 +249,7 @@ fi
 
 # Turn off gitignore https://github.com/boyter/scc/issues/53
 touch ignored.xml
-a=$(./scc  | grep Total)
+a=$(./scc | grep Total)
 b=$(./scc --no-gitignore | grep Total)
 if [ "$a" == "$b" ]; then
     echo -e "${RED}======================================================="
@@ -260,9 +260,11 @@ else
     echo -e "${GREEN}PASSED git ignore filter"
 fi
 
-a=$(./scc  | grep Total)
+a=$(./scc | grep Total)
 b=$(./scc --no-ignore | grep Total)
 if [ "$a" == "$b" ]; then
+    echo "$a"
+    echo "$b"
     echo -e "${RED}======================================================="
     echo -e "FAILED ignore filter"
     echo -e "=================================================${NC}"
@@ -271,6 +273,34 @@ else
     echo -e "${GREEN}PASSED ignore filter"
 fi
 
+touch ./examples/ignore/ignorefile.txt
+a=$(./scc --by-file | grep ignorefile)
+b=$(./scc --by-file --no-ignore | grep ignorefile)
+if [ "$a" == "$b" ]; then
+    echo "$a"
+    echo "$b"
+    echo -e "${RED}======================================================="
+    echo -e "FAILED ignore recursive filter"
+    echo -e "=================================================${NC}"
+    exit
+else
+    echo -e "${GREEN}PASSED ignore recursive filter"
+fi
+
+touch ./examples/ignore/gitignorefile.txt
+a=$(./scc --by-file | grep gitignorefile)
+b=$(./scc --by-file --no-gitignore | grep gitignorefile)
+if [ "$a" == "$b" ]; then
+    echo "$a"
+    echo "$b"
+    echo -e "${RED}======================================================="
+    echo -e "FAILED gitignore recursive filter"
+    echo -e "=================================================${NC}"
+    exit
+else
+    echo -e "${GREEN}PASSED gitignore recursive filter"
+fi
+
 # Try out specific languages
 for i in 'Bosque ' 'Flow9 ' 'Bitbucket Pipeline ' 'Docker ignore ' 'Q# ' 'Futhark ' 'Alloy ' 'Wren ' 'Monkey C ' 'Alchemist ' 'Luna ' 'ignore '
 do
@@ -288,6 +318,8 @@ echo -e "${NC}Cleaning up..."
 rm ./scc
 rm ./ignored.xml
 rm .tmp_scc_yaml
+rm ./examples/ignore/gitignorefile.txt
+rm ./examples/ignore/ignorefile.txt
 
 echo -e "${GREEN}================================================="
 echo -e "ALL TESTS PASSED"