Fix sending multiple request to same endpoint bug

Author: C-Sto

librecursebuster/logic.go

@@ -43,7 +43,6 @@ func ManageNewURLs() {
 	//decides on whether to add to the directory list, or add to file output
 	for {
 		candidate := <-gState.Chans.newPagesChan
-
 		//check the candidate is an actual URL
 		u, err := url.Parse(strings.TrimSpace(candidate.URL))
 
@@ -185,8 +184,7 @@ func dirBust(page SpiderPage) {
 	}
 
 	atomic.StoreUint32(gState.DirbProgress, 0)
-
-	tested := make(map[string]bool)        //ensure we don't send things more than once
+	//ensure we don't send things more than once
 	for _, word := range gState.WordList { //will receive from the channel until it's closed
 		//read words off the channel, and test it OR close out because we wanna skip it
 		if word == "" {
@@ -204,31 +202,40 @@ func dirBust(page SpiderPage) {
 			for _, method := range gState.Methods {
 				if len(gState.Extensions) > 0 && gState.Extensions[0] != "" {
 					for _, ext := range gState.Extensions {
-						if tested[method+page.URL+word+"."+ext] {
+						gState.CMut.Lock()
+						if gState.Checked[method+page.URL+word+"."+ext] {
+							gState.CMut.Unlock()
 							continue
 						}
 						gState.Chans.workersChan <- struct{}{}
 						gState.wg.Add(1)
 						go testURL(method, page.URL+word+"."+ext, gState.Client)
-						tested[method+page.URL+word+"."+ext] = true
+						gState.Checked[method+page.URL+word+"."+ext] = true
+						gState.CMut.Unlock()
 					}
 				}
 				if gState.Cfg.AppendDir {
-					if tested[method+page.URL+word+"/"] {
+					gState.CMut.Lock()
+					if gState.Checked[method+page.URL+word+"/"] {
+						gState.CMut.Unlock()
 						continue
 					}
 					gState.Chans.workersChan <- struct{}{}
 					gState.wg.Add(1)
 					go testURL(method, page.URL+word+"/", gState.Client)
-					tested[method+page.URL+word+"/"] = true
+					gState.Checked[method+page.URL+word+"/"] = true
+					gState.CMut.Unlock()
 				}
-				if tested[method+page.URL+word] {
+				gState.CMut.Lock()
+				if gState.Checked[method+page.URL+word] {
+					gState.CMut.Unlock()
 					continue
 				}
 				gState.Chans.workersChan <- struct{}{}
 				gState.wg.Add(1)
 				go testURL(method, page.URL+word, gState.Client)
-				tested[method+page.URL+word] = true
+				gState.Checked[method+page.URL+word] = true
+				gState.CMut.Unlock()
 				//if gState.Cfg.MaxDirs == 1 {
 				atomic.AddUint32(gState.DirbProgress, 1)
 				//}

librecursebuster/main_test.go

@@ -46,7 +46,7 @@ func TestBasicFunctionality(t *testing.T) {
 
 	finished := make(chan struct{})
 	cfg := getDefaultConfig()
-	urlSlice := preSetupTest(cfg, "2001", finished)
+	urlSlice := preSetupTest(cfg, "2001", finished, t)
 	found := postSetupTest(urlSlice)
 
 	//waitgroup check (if test times out, the waitgroup is broken... somewhere)
@@ -104,7 +104,7 @@ func TestAppendSlash(t *testing.T) {
 	//add an appendslash value to the wordlist that should _only_ be found if the appendslash var is set
 	finished := make(chan struct{})
 	cfg := getDefaultConfig()
-	urlSlice := preSetupTest(cfg, "2002", finished)
+	urlSlice := preSetupTest(cfg, "2002", finished, t)
 	gState.Cfg.AppendDir = true
 	gState.WordList = append(gState.WordList, "appendslash")
 	found := postSetupTest(urlSlice)
@@ -121,7 +121,7 @@ func TestBasicAuth(t *testing.T) {
 	finished := make(chan struct{})
 	cfg := getDefaultConfig()
 	cfg.Auth = "dGVzdDp0ZXN0"
-	urlSlice := preSetupTest(cfg, "2003", finished)
+	urlSlice := preSetupTest(cfg, "2003", finished, t)
 	gState.WordList = append(gState.WordList, "basicauth")
 	found := postSetupTest(urlSlice)
 	gState.Wait()
@@ -137,7 +137,7 @@ func TestBadCodes(t *testing.T) {
 	finished := make(chan struct{})
 	cfg := getDefaultConfig()
 	cfg.BadResponses = "404,500"
-	urlSlice := preSetupTest(cfg, "2004", finished)
+	urlSlice := preSetupTest(cfg, "2004", finished, t)
 	gState.WordList = append(gState.WordList, "badcode")
 	found := postSetupTest(urlSlice)
 	gState.Wait()
@@ -156,7 +156,7 @@ func TestBadHeaders(t *testing.T) {
 	cfg := getDefaultConfig()
 	cfg.BadHeader = ArrayStringFlag{}
 	cfg.BadHeader.Set("X-Bad-Header: test123")
-	urlSlice := preSetupTest(cfg, "2005", finished)
+	urlSlice := preSetupTest(cfg, "2005", finished, t)
 	gState.WordList = append(gState.WordList, "badheader")
 	found := postSetupTest(urlSlice)
 	gState.Wait()
@@ -175,7 +175,7 @@ func TestAjax(t *testing.T) {
 	cfg := getDefaultConfig()
 	cfg.Ajax = true
 	cfg.Methods = "GET,POST"
-	urlSlice := preSetupTest(cfg, "2006", finished)
+	urlSlice := preSetupTest(cfg, "2006", finished, t)
 	gState.WordList = append(gState.WordList, "ajaxonly")
 	gState.WordList = append(gState.WordList, "onlynoajax")
 	gState.WordList = append(gState.WordList, "ajaxpost")
@@ -252,7 +252,7 @@ func postSetupTest(urlSlice []string) (found map[string]bool) {
 	return
 }
 
-func preSetupTest(cfg *Config, servPort string, finished chan struct{}) (urlSlice []string) {
+func preSetupTest(cfg *Config, servPort string, finished chan struct{}, t *testing.T) (urlSlice []string) {
 	//Test default functions. Basic dirb should work, and all files should be found as expected
 
 	//basic state setup
@@ -264,7 +264,7 @@ func preSetupTest(cfg *Config, servPort string, finished chan struct{}) (urlSlic
 
 	//start the test server
 	setup := make(chan struct{})
-	go testserver.Start(servPort, finished, setup)
+	go testserver.Start(servPort, finished, setup, t)
 	<-setup //whoa, concurrency sucks???
 	//test URL
 	globalState.Cfg.URL = localURL + servPort

librecursebuster/testserver/main.go

@@ -5,6 +5,8 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"testing"
+	"time"
 )
 
 /*wordlist
@@ -56,7 +58,8 @@ func handler(w http.ResponseWriter, r *http.Request) {
 	//if it's been visited more than once, instant fail
 	if visited[r.Method+":"+r.URL.Path] && r.URL.Path != "/" {
 		//we can visit the base url more than once
-		//panic("Path visited more than once: " + r.Method + ":" + r.URL.Path)
+		tes.Fail()
+		panic("Path visited more than once: " + r.Method + ":" + r.URL.Path)
 	}
 	visited[r.Method+":"+r.URL.Path] = true
 	vMut.Unlock()
@@ -172,13 +175,31 @@ func handler(w http.ResponseWriter, r *http.Request) {
 	fmt.Println(r.Method, r.URL, respCode, bod)
 }
 
+var tes *testing.T
+
 //Start starts the test HTTP server
-func Start(port string, finishedTest, setup chan struct{}) {
+func Start(port string, finishedTest, setup chan struct{}, t *testing.T) {
 	s := http.NewServeMux()
 	visited = make(map[string]bool)
 	vMut = &sync.RWMutex{}
+	tes = t
 	s.HandleFunc("/", handler)
 	go http.ListenAndServe("127.0.0.1:"+port, s)
+
+	for {
+		time.Sleep(time.Second * 1)
+		resp, err := http.Get("http://localhost:" + port)
+		if err != nil {
+			continue
+		}
+		resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			continue
+		}
+		// Reached this point: server is up and running!
+		break
+	}
+
 	close(setup)
 	<-finishedTest //this is an ultra gross hack :(
 }

librecursebuster/ui.go

@@ -11,7 +11,6 @@ import (
 //StartUI is called to begin the UI... stuff
 func (s *State) StartUI(uiWG *sync.WaitGroup, quitChan chan struct{}) {
 	g, err := ui.NewGui(ui.OutputNormal)
-	g.Mouse = true
 	if err != nil {
 		panic(err)
 	}
@@ -47,15 +46,15 @@ func (s *State) StartUI(uiWG *sync.WaitGroup, quitChan chan struct{}) {
 	err = s.ui.SetKeybinding("", ui.KeyArrowDown, ui.ModNone, scrollDown)
 	if err != nil {
 		panic(err)
-	}
+	} /* Mouse stuff broke copying out of the terminal... not ideal
 	err = s.ui.SetKeybinding("", ui.MouseWheelUp, ui.ModNone, scrollUp)
 	if err != nil {
 		panic(err)
 	}
 	err = s.ui.SetKeybinding("", ui.MouseWheelDown, ui.ModNone, scrollDown)
 	if err != nil {
 		panic(err)
-	}
+	}*/
 	uiWG.Done()
 	err = s.ui.MainLoop()
 	if err != nil && err != ui.ErrQuit {

main.go

@@ -14,7 +14,7 @@ import (
 	"github.com/fatih/color"
 )
 
-const version = "1.5.10"
+const version = "1.5.11"
 
 func main() {
 	if runtime.GOOS == "windows" { //lol goos