From 208faec7d298a7a95958bc21ef9427966a7d1d59 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 16 Dec 2014 16:25:17 +0500 Subject: [PATCH 01/64] Initial support for input modifiers --- Dockerfile | 4 +- Makefile | 8 +-- examples/echo_modifier/echo_modifier.go | 44 +++++++++++++++++ plugins.go | 5 ++ pong_modifier.rb | 13 +++++ settings.go | 4 ++ traffic_modifier.go | 65 +++++++++++++++++++++++++ 7 files changed, 138 insertions(+), 5 deletions(-) create mode 100644 examples/echo_modifier/echo_modifier.go create mode 100755 pong_modifier.rb create mode 100644 traffic_modifier.go diff --git a/Dockerfile b/Dockerfile index 2b174ce1..73524394 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,10 @@ FROM google/golang RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean +RUN apt-get install ruby -y + WORKDIR /gopath/src/gor ADD . /gopath/src/gor -RUN go get \ No newline at end of file +RUN go get diff --git a/Makefile b/Makefile index 037e311c..4b68a9e8 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ -SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go settings_header_filters.go settings_header_hash_filters.go settings_headers.go settings_methods.go settings_option.go settings_url_regexp.go test_input.go elasticsearch.go settings_url_map.go +SOURCE = emitter.go gor.go traffic_modifier.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go settings_header_filters.go settings_header_hash_filters.go settings_headers.go settings_methods.go settings_option.go settings_url_regexp.go test_input.go elasticsearch.go settings_url_map.go release: release-x86 release-x64 release-x64: docker run -v `pwd`:/gopath/src/gor -t --env GOOS=linux --env GOARCH=amd64 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x64.tar.gz gor && rm gor - + release-x86: docker run -v `pwd`:/gopath/src/gor -t --env GOOS=linux --env GOARCH=386 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x86.tar.gz gor && rm gor @@ -22,7 +22,7 @@ dbench: # Used mainly for debugging, because docker container do not have access to parent machine ports drun: - docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-dummy=0 --input-http=:9000 --output-http="http://localhost:9000" --verbose + docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-modifier="./pong_modifier.rb" --input-dummy=0 --input-http=:9000 --output-http="http://localhost:9000" --verbose -dbash: +dbash: docker run -v `pwd`:/gopath/src/gor -t -i gor /bin/bash \ No newline at end of file diff --git a/examples/echo_modifier/echo_modifier.go b/examples/echo_modifier/echo_modifier.go new file mode 100644 index 00000000..9dd0ea6b --- /dev/null +++ b/examples/echo_modifier/echo_modifier.go @@ -0,0 +1,44 @@ +package main + +import ( + "os" + "bufio" +) + +func main() { + reader := bufio.NewReader(os.Stdin) + data := make(chan []byte) + + go ReadStdin(data) + + for { + os.Stdout.Print(<- data, '¶') + } +} + +func ReadStdin(data chan []byte){ + for { + buf, err := reader.ReadBytes('¶') + buf_len := len(buf) + if buf_len > 0 { + new_buf_len := len(buf) - 2 + if new_buf_len > 0 { + new_buf := make([]byte, new_buf_len) + copy(new_buf, buf[:new_buf_len]) + data <- new_buf + if err != nil { + if err != io.EOF { + log.Printf("error: %s\n", err) + } + } + } + } + } +} + +while data = STDIN.gets(separator) + STDERR.puts "==== Start ====" + STDERR.puts data + puts data + STDERR.puts "==== End ====" +end \ No newline at end of file diff --git a/plugins.go b/plugins.go index 7f96ae0d..0cc8ab51 100644 --- a/plugins.go +++ b/plugins.go @@ -9,6 +9,8 @@ import ( type InOutPlugins struct { Inputs []io.Reader Outputs []io.Writer + + Modifiers []TrafficModifier } type ReaderOrWriter interface{} @@ -54,6 +56,9 @@ func registerPlugin(constructor interface{}, options ...interface{}) { } if _, ok := plugin.(io.Reader); ok { + for _, options := range Settings.inputModifier { + plugin_wrapper = NewTrafficModifier(plugin_wrapper, options) + } Plugins.Inputs = append(Plugins.Inputs, plugin_wrapper.(io.Reader)) } diff --git a/pong_modifier.rb b/pong_modifier.rb new file mode 100755 index 00000000..e702507f --- /dev/null +++ b/pong_modifier.rb @@ -0,0 +1,13 @@ +#!/usr/bin/env ruby +# encoding: utf-8 +require "base64" + +STDERR.puts "Starting modifier" +puts "Starting modifier" + +while data = STDIN.gets.chomp + STDERR.puts "==== Start ====" + STDERR.puts Base64.encode64(data) + puts data + STDERR.puts "==== End ====" +end \ No newline at end of file diff --git a/settings.go b/settings.go index f7973c15..e59a43a3 100644 --- a/settings.go +++ b/settings.go @@ -29,6 +29,8 @@ type AppSettings struct { inputRAW MultiOption + inputModifier MultiOption + inputHTTP MultiOption outputHTTP MultiOption outputHTTPHeaders HTTPHeaders @@ -70,6 +72,8 @@ func init() { flag.Var(&Settings.inputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com") + flag.Var(&Settings.inputModifier, "input-modifier", "Used for modifying input traffic using external command") + flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com") flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com") diff --git a/traffic_modifier.go b/traffic_modifier.go new file mode 100644 index 00000000..0e8a09ad --- /dev/null +++ b/traffic_modifier.go @@ -0,0 +1,65 @@ +package main + +import ( + "fmt" + "log" + "io" + "os/exec" + "os" + "encoding/base64" +) + +type TrafficModifier struct { + plugin interface{} + command string + + data chan []byte + + Stdin io.Writer + Stdout io.Reader +} + +func NewTrafficModifier(plugin interface{}, command string) io.Reader { + m := new(TrafficModifier) + m.plugin = plugin + m.command = command + + cmd := exec.Command("bash", "-c", command) + cmd.Stderr = os.Stderr + + m.Stdout, _ = cmd.StdoutPipe() + m.Stdin, _ = cmd.StdinPipe() + + m.Stdout = base64.NewDecoder(base64.StdEncoding, m.Stdout) + + go m.copy(m.Stdin, m.plugin.(io.Reader)) + + err := cmd.Run() + + if (err != nil) { + log.Fatal(err) + } + + return m +} + +func (m *TrafficModifier) copy(to io.Writer, from io.Reader) { + buf := make([]byte, 5*1024*1024) + + for { + nr, er := from.Read(buf) + if nr > 0 && len(buf) > nr { + to.Write(base64.StdEncoding.Encode(buf)) + } + } +} + +func (m *TrafficModifier) Read(data []byte) (n int, err error) { + n, err = m.Stdout.Read(data) + + return +} + +func (m *TrafficModifier) String() string { + return fmt.Sprintf("Modifying traffic for %s using '%s' command", m.plugin, m.command) +} From 4f03f81d049a381a35c46bf61138d1bb7e9b1866 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 27 Jun 2015 21:09:06 +0500 Subject: [PATCH 02/64] Working modifier implementation with examples --- Dockerfile | 2 +- Makefile | 2 +- emitter.go | 1 + examples/echo_modifier.rb | 16 +++++++ examples/echo_modifier.sh | 10 +++++ examples/echo_modifier/echo_modifier.go | 44 ------------------- input_dummy.go | 2 +- plugins.go | 1 + pong_modifier.rb | 13 ------ traffic_modifier.go | 58 +++++++++++++++++++------ 10 files changed, 76 insertions(+), 73 deletions(-) create mode 100755 examples/echo_modifier.rb create mode 100755 examples/echo_modifier.sh delete mode 100644 examples/echo_modifier/echo_modifier.go delete mode 100755 pong_modifier.rb diff --git a/Dockerfile b/Dockerfile index 73524394..7691f7ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM google/golang RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean -RUN apt-get install ruby -y +RUN apt-get update && apt-get install ruby vim-common -y WORKDIR /gopath/src/gor diff --git a/Makefile b/Makefile index 4b68a9e8..18b69e5b 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ dbench: # Used mainly for debugging, because docker container do not have access to parent machine ports drun: - docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-modifier="./pong_modifier.rb" --input-dummy=0 --input-http=:9000 --output-http="http://localhost:9000" --verbose + docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-modifier="./examples/echo_modifier.sh" --input-dummy=0 --output-http="http://localhost:9000" --verbose dbash: docker run -v `pwd`:/gopath/src/gor -t -i gor /bin/bash \ No newline at end of file diff --git a/emitter.go b/emitter.go index 0bc80866..84c0acb8 100644 --- a/emitter.go +++ b/emitter.go @@ -26,6 +26,7 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { for { nr, er := src.Read(buf) + if nr > 0 && len(buf) > nr { Debug("Sending", src, ": ", string(buf[0:nr])) diff --git a/examples/echo_modifier.rb b/examples/echo_modifier.rb new file mode 100755 index 00000000..e3a489d0 --- /dev/null +++ b/examples/echo_modifier.rb @@ -0,0 +1,16 @@ +#!/usr/bin/env ruby +# encoding: utf-8 +while data = STDIN.gets + next unless data + data = data.chomp + + decoded = [data].pack("H*") + encoded = decoded.unpack("H*").first + + STDOUT.puts encoded + + + STDERR.puts "[DEBUG] Original data: #{data}" + STDERR.puts "[DEBUG] Decoded request: #{decoded}" + STDERR.puts "[DEBUG] Encoded data: #{encoded}" +end \ No newline at end of file diff --git a/examples/echo_modifier.sh b/examples/echo_modifier.sh new file mode 100755 index 00000000..1d56cdc1 --- /dev/null +++ b/examples/echo_modifier.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +while read line; do + decoded=$(echo "$line" | xxd -r -p) + encoded=$(echo "$decoded" | xxd -p | tr -d "\\n") + echo "$encoded" + + >&2 echo "[DEBUG] Original data: $line" + >&2 echo "[DEBUG] Decoded request: $decoded" + >&2 echo "[DEBUG] Encoded data: $encoded" +done; diff --git a/examples/echo_modifier/echo_modifier.go b/examples/echo_modifier/echo_modifier.go deleted file mode 100644 index 9dd0ea6b..00000000 --- a/examples/echo_modifier/echo_modifier.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "os" - "bufio" -) - -func main() { - reader := bufio.NewReader(os.Stdin) - data := make(chan []byte) - - go ReadStdin(data) - - for { - os.Stdout.Print(<- data, '¶') - } -} - -func ReadStdin(data chan []byte){ - for { - buf, err := reader.ReadBytes('¶') - buf_len := len(buf) - if buf_len > 0 { - new_buf_len := len(buf) - 2 - if new_buf_len > 0 { - new_buf := make([]byte, new_buf_len) - copy(new_buf, buf[:new_buf_len]) - data <- new_buf - if err != nil { - if err != io.EOF { - log.Printf("error: %s\n", err) - } - } - } - } - } -} - -while data = STDIN.gets(separator) - STDERR.puts "==== Start ====" - STDERR.puts data - puts data - STDERR.puts "==== End ====" -end \ No newline at end of file diff --git a/input_dummy.go b/input_dummy.go index a2a87ac5..846e86aa 100644 --- a/input_dummy.go +++ b/input_dummy.go @@ -30,7 +30,7 @@ func (i *DummyInput) emit() { for { select { case <-ticker.C: - i.data <- []byte("GET / HTTP/1.1\r\n\r\n") + i.data <- []byte("POST /pub/WWW/å HTTP/1.1\nHost: www.w3.org\r\n\r\na=1&b=2") } } } diff --git a/plugins.go b/plugins.go index 62bce402..ecafbfa9 100644 --- a/plugins.go +++ b/plugins.go @@ -62,6 +62,7 @@ func registerPlugin(constructor interface{}, options ...interface{}) { Plugins.Inputs = append(Plugins.Inputs, plugin_wrapper.(io.Reader)) } + if _, ok := plugin.(io.Writer); ok { Plugins.Outputs = append(Plugins.Outputs, plugin_wrapper.(io.Writer)) } diff --git a/pong_modifier.rb b/pong_modifier.rb deleted file mode 100755 index e702507f..00000000 --- a/pong_modifier.rb +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env ruby -# encoding: utf-8 -require "base64" - -STDERR.puts "Starting modifier" -puts "Starting modifier" - -while data = STDIN.gets.chomp - STDERR.puts "==== Start ====" - STDERR.puts Base64.encode64(data) - puts data - STDERR.puts "==== End ====" -end \ No newline at end of file diff --git a/traffic_modifier.go b/traffic_modifier.go index 0e8a09ad..ddc92d11 100644 --- a/traffic_modifier.go +++ b/traffic_modifier.go @@ -6,7 +6,8 @@ import ( "io" "os/exec" "os" - "encoding/base64" + "bufio" + "encoding/hex" ) type TrafficModifier struct { @@ -23,43 +24,74 @@ func NewTrafficModifier(plugin interface{}, command string) io.Reader { m := new(TrafficModifier) m.plugin = plugin m.command = command + m.data = make(chan []byte) - cmd := exec.Command("bash", "-c", command) - cmd.Stderr = os.Stderr + cmd := exec.Command(command) m.Stdout, _ = cmd.StdoutPipe() m.Stdin, _ = cmd.StdinPipe() - - m.Stdout = base64.NewDecoder(base64.StdEncoding, m.Stdout) + cmd.Stderr = os.Stderr go m.copy(m.Stdin, m.plugin.(io.Reader)) + go m.read(m.Stdout) - err := cmd.Run() + go func(){ + err := cmd.Start() - if (err != nil) { - log.Fatal(err) - } + if (err != nil) { + log.Fatal(err) + } + }() + + defer cmd.Wait() return m } func (m *TrafficModifier) copy(to io.Writer, from io.Reader) { buf := make([]byte, 5*1024*1024) + dst := make([]byte, len(buf)*2) for { - nr, er := from.Read(buf) + nr, _ := from.Read(buf) if nr > 0 && len(buf) > nr { - to.Write(base64.StdEncoding.Encode(buf)) + hex.Encode(dst, buf[0:nr]) + to.Write(dst[0:nr*2]) + to.Write([]byte("\r\n")) } } } -func (m *TrafficModifier) Read(data []byte) (n int, err error) { - n, err = m.Stdout.Read(data) +func (m *TrafficModifier) read(from io.Reader) { + buf := make([]byte, 5*1024*1024) + + scanner := bufio.NewScanner(from) + + for scanner.Scan() { + bytes := scanner.Bytes() + hex.Decode(buf, bytes) + + Debug("Received:", buf[0:len(bytes)/2]) + + m.data <- buf[0:len(bytes)/2] + } + + if err := scanner.Err(); err != nil { + fmt.Fprintln(os.Stderr, "Traffic modifier command failed:", err) + } return } +func (m *TrafficModifier) Read(data []byte) (int, error) { + Debug("Trying to read channel!") + buf := <- m.data + copy(data, buf) + + return len(buf), nil +} + + func (m *TrafficModifier) String() string { return fmt.Sprintf("Modifying traffic for %s using '%s' command", m.plugin, m.command) } From 674ffc620241cd0c2562ba7a917fa9ce322a463a Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 28 Jun 2015 00:16:19 +0500 Subject: [PATCH 03/64] Start adding tests for traffic modifier --- input_raw_test.go | 2 +- output_http_test.go | 10 ++-- plugins.go | 1 - traffic_modifier.go | 117 +++++++++++++++++++-------------------- traffic_modifier_test.go | 74 +++++++++++++++++++++++++ 5 files changed, 138 insertions(+), 66 deletions(-) create mode 100644 traffic_modifier_test.go diff --git a/input_raw_test.go b/input_raw_test.go index b1bcca65..2eeb6d96 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -13,7 +13,7 @@ func TestRAWInput(t *testing.T) { wg := new(sync.WaitGroup) quit := make(chan int) - listener := startHTTP(func(req *http.Request) {}) + listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) input := NewRAWInput(listener.Addr().String()) output := NewTestOutput(func(data []byte) { diff --git a/output_http_test.go b/output_http_test.go index 2432d5c8..b24f442c 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -12,9 +12,9 @@ import ( "time" ) -func startHTTP(cb func(*http.Request)) net.Listener { +func startHTTP(cb func(http.ResponseWriter, *http.Request)) net.Listener { handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - cb(r) + cb(w, r) }) listener, _ := net.Listen("tcp", ":0") @@ -54,7 +54,7 @@ func TestHTTPOutput(t *testing.T) { headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} methods := HTTPMethods{"GET", "PUT", "POST"} - listener := startHTTP(func(req *http.Request) { + listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { if req.Header.Get("User-Agent") != "Gor" { t.Error("Wrong header") } @@ -104,7 +104,7 @@ func TestHTTPOutputChunkedEncoding(t *testing.T) { headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} methods := HTTPMethods{"GET", "PUT", "POST"} - listener := startHTTP(func(req *http.Request) { + listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { defer req.Body.Close() body, _ := ioutil.ReadAll(req.Body) @@ -140,7 +140,7 @@ func BenchmarkHTTPOutput(b *testing.B) { headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} methods := HTTPMethods{"GET", "PUT", "POST"} - listener := startHTTP(func(req *http.Request) { + listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { time.Sleep(50 * time.Millisecond) wg.Done() }) diff --git a/plugins.go b/plugins.go index ecafbfa9..62bce402 100644 --- a/plugins.go +++ b/plugins.go @@ -62,7 +62,6 @@ func registerPlugin(constructor interface{}, options ...interface{}) { Plugins.Inputs = append(Plugins.Inputs, plugin_wrapper.(io.Reader)) } - if _, ok := plugin.(io.Writer); ok { Plugins.Outputs = append(Plugins.Outputs, plugin_wrapper.(io.Writer)) } diff --git a/traffic_modifier.go b/traffic_modifier.go index ddc92d11..7a34bb94 100644 --- a/traffic_modifier.go +++ b/traffic_modifier.go @@ -1,97 +1,96 @@ package main import ( - "fmt" - "log" - "io" - "os/exec" - "os" - "bufio" - "encoding/hex" + "bufio" + "encoding/hex" + "fmt" + "io" + "log" + "os" + "os/exec" ) type TrafficModifier struct { - plugin interface{} - command string + plugin interface{} + command string - data chan []byte + data chan []byte - Stdin io.Writer - Stdout io.Reader + Stdin io.Writer + Stdout io.Reader } func NewTrafficModifier(plugin interface{}, command string) io.Reader { - m := new(TrafficModifier) - m.plugin = plugin - m.command = command - m.data = make(chan []byte) + m := new(TrafficModifier) + m.plugin = plugin + m.command = command + m.data = make(chan []byte) - cmd := exec.Command(command) + cmd := exec.Command(command) - m.Stdout, _ = cmd.StdoutPipe() - m.Stdin, _ = cmd.StdinPipe() - cmd.Stderr = os.Stderr + m.Stdout, _ = cmd.StdoutPipe() + m.Stdin, _ = cmd.StdinPipe() + cmd.Stderr = os.Stderr - go m.copy(m.Stdin, m.plugin.(io.Reader)) - go m.read(m.Stdout) + go m.copy(m.Stdin, m.plugin.(io.Reader)) + go m.read(m.Stdout) - go func(){ - err := cmd.Start() + go func() { + err := cmd.Start() - if (err != nil) { - log.Fatal(err) - } - }() + if err != nil { + log.Fatal(err) + } + }() - defer cmd.Wait() + defer cmd.Wait() - return m + return m } func (m *TrafficModifier) copy(to io.Writer, from io.Reader) { - buf := make([]byte, 5*1024*1024) - dst := make([]byte, len(buf)*2) - - for { - nr, _ := from.Read(buf) - if nr > 0 && len(buf) > nr { - hex.Encode(dst, buf[0:nr]) - to.Write(dst[0:nr*2]) - to.Write([]byte("\r\n")) - } - } + buf := make([]byte, 5*1024*1024) + dst := make([]byte, len(buf)*2) + + for { + nr, _ := from.Read(buf) + if nr > 0 && len(buf) > nr { + hex.Encode(dst, buf[0:nr]) + to.Write(dst[0 : nr*2]) + to.Write([]byte("\r\n")) + } + } } func (m *TrafficModifier) read(from io.Reader) { - buf := make([]byte, 5*1024*1024) + buf := make([]byte, 5*1024*1024) - scanner := bufio.NewScanner(from) + scanner := bufio.NewScanner(from) - for scanner.Scan() { - bytes := scanner.Bytes() - hex.Decode(buf, bytes) + for scanner.Scan() { + bytes := scanner.Bytes() + hex.Decode(buf, bytes) - Debug("Received:", buf[0:len(bytes)/2]) + Debug("Received:", buf[0:len(bytes)/2]) - m.data <- buf[0:len(bytes)/2] - } + m.data <- buf[0 : len(bytes)/2] + } - if err := scanner.Err(); err != nil { - fmt.Fprintln(os.Stderr, "Traffic modifier command failed:", err) - } + if err := scanner.Err(); err != nil { + fmt.Fprintln(os.Stderr, "Traffic modifier command failed:", err) + } - return + return } func (m *TrafficModifier) Read(data []byte) (int, error) { - Debug("Trying to read channel!") - buf := <- m.data - copy(data, buf) + Debug("Trying to read channel!") + buf := <-m.data + copy(data, buf) - return len(buf), nil + return len(buf), nil } - func (m *TrafficModifier) String() string { - return fmt.Sprintf("Modifying traffic for %s using '%s' command", m.plugin, m.command) + return fmt.Sprintf("Modifying traffic for %s using '%s' command", m.plugin, m.command) } diff --git a/traffic_modifier_test.go b/traffic_modifier_test.go new file mode 100644 index 00000000..547876e0 --- /dev/null +++ b/traffic_modifier_test.go @@ -0,0 +1,74 @@ +package main + +import ( + _ "bufio" + "bytes" + "crypto/rand" + _ "io" + "io/ioutil" + _ "log" + _ "net" + "net/http" + "sync" + "testing" +) + +// Simple service that generate token on request, and require this token for accesing to secure area +func NewFakeSecureService(wg *sync.WaitGroup) string { + active_tokens := make([][]byte, 0) + + listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/token": + // Generate random token + token_length := 10 + token := make([]byte, token_length) + rand.Read(token) + + w.Write(token) + + active_tokens = append(active_tokens, token) + case "/secure": + token := []byte(req.URL.Query().Get("token")) + + for _, t := range active_tokens { + if bytes.Equal(t, token) { + w.WriteHeader(http.StatusAccepted) + } else { + w.WriteHeader(http.StatusForbidden) + } + } + } + + wg.Done() + }) + + return "http://" + listener.Addr().String() +} + +func TestFakeSecureService(t *testing.T) { + var resp *http.Response + + wg := new(sync.WaitGroup) + + addr := NewFakeSecureService(wg) + + wg.Add(3) + + resp, _ = http.Get(addr + "/token") + token, _ := ioutil.ReadAll(resp.Body) + + // Right token + resp, _ = http.Get(addr + "/secure?token=" + string(token)) + if resp.StatusCode != http.StatusAccepted { + t.Error("Valid token should returns wrong status:", resp.StatusCode) + } + + // Wrong tokens forbidden + resp, _ = http.Get(addr + "/secure?token=wrong") + if resp.StatusCode != http.StatusForbidden { + t.Error("Wrong tokens should be forbidden, instead:", resp.StatusCode) + } + + wg.Wait() +} From 98f941b69ac01ce85e8c3a0593b1a405bae28022 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 28 Jun 2015 00:23:21 +0500 Subject: [PATCH 04/64] Add support for complex commands --- Makefile | 2 +- traffic_modifier.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 18b69e5b..693a80fb 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ dbench: # Used mainly for debugging, because docker container do not have access to parent machine ports drun: - docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-modifier="./examples/echo_modifier.sh" --input-dummy=0 --output-http="http://localhost:9000" --verbose + docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-modifier="bash ./examples/echo_modifier.sh" --input-dummy=0 --output-http="http://localhost:9000" --verbose dbash: docker run -v `pwd`:/gopath/src/gor -t -i gor /bin/bash \ No newline at end of file diff --git a/traffic_modifier.go b/traffic_modifier.go index 7a34bb94..6a4a5b65 100644 --- a/traffic_modifier.go +++ b/traffic_modifier.go @@ -8,6 +8,7 @@ import ( "log" "os" "os/exec" + "strings" ) type TrafficModifier struct { @@ -26,7 +27,8 @@ func NewTrafficModifier(plugin interface{}, command string) io.Reader { m.command = command m.data = make(chan []byte) - cmd := exec.Command(command) + commands := strings.Split(command, " ") + cmd := exec.Command(commands[0], commands[1:]...) m.Stdout, _ = cmd.StdoutPipe() m.Stdin, _ = cmd.StdinPipe() From 3e86456e36e94f8d18c10238f17736d577e4c696 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 28 Jun 2015 22:00:55 +0500 Subject: [PATCH 05/64] Add more debug --- Makefile | 2 +- input_raw.go | 2 ++ output_http.go | 3 +- traffic_modifier_test.go | 71 +++++++++++++++++++++++++++++++++------- 4 files changed, 65 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 693a80fb..b79497d9 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ dbuild: docker build -t gor . dtest: - docker run -v `pwd`:/gopath/src/gor -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v + docker run -v `pwd`:/gopath/src/gor -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v --verbose dfmt: docker run -v `pwd`:/gopath/src/gor -t -i gor go fmt diff --git a/input_raw.go b/input_raw.go index 3a6a9601..9653559f 100644 --- a/input_raw.go +++ b/input_raw.go @@ -32,6 +32,8 @@ func (i *RAWInput) Read(data []byte) (int, error) { func (i *RAWInput) listen(address string) { address = strings.Replace(address, "[::]", "127.0.0.1", -1) + Debug("Listening for traffic on: " + address) + host, port, err := net.SplitHostPort(address) if err != nil { diff --git a/output_http.go b/output_http.go index 46a342a4..ba1afa29 100644 --- a/output_http.go +++ b/output_http.go @@ -73,6 +73,7 @@ type HTTPOutput struct { address string limit int queue chan []byte + responses chan []byte redirectLimit int @@ -91,7 +92,7 @@ type HTTPOutput struct { queueStats *GorStat } -func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, urlRegexp HTTPUrlRegexp, headerFilters HTTPHeaderFilters, headerHashFilters HTTPHeaderHashFilters, elasticSearchAddr string, outputHTTPUrlRewrite UrlRewriteMap, outputHTTPRedirects int) io.Writer { +func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, urlRegexp HTTPUrlRegexp, headerFilters HTTPHeaderFilters, headerHashFilters HTTPHeaderHashFilters, elasticSearchAddr string, outputHTTPUrlRewrite UrlRewriteMap, outputHTTPRedirects int) io.ReadWriter { o := new(HTTPOutput) diff --git a/traffic_modifier_test.go b/traffic_modifier_test.go index 547876e0..82e786e8 100644 --- a/traffic_modifier_test.go +++ b/traffic_modifier_test.go @@ -4,13 +4,14 @@ import ( _ "bufio" "bytes" "crypto/rand" - _ "io" + "io" "io/ioutil" _ "log" _ "net" "net/http" "sync" "testing" + "strings" ) // Simple service that generate token on request, and require this token for accesing to secure area @@ -18,32 +19,40 @@ func NewFakeSecureService(wg *sync.WaitGroup) string { active_tokens := make([][]byte, 0) listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { + Debug("Received request: " + req.URL.String()) + switch req.URL.Path { case "/token": // Generate random token token_length := 10 token := make([]byte, token_length) rand.Read(token) - - w.Write(token) - active_tokens = append(active_tokens, token) + + w.Write(token) case "/secure": token := []byte(req.URL.Query().Get("token")) + token_found := false for _, t := range active_tokens { if bytes.Equal(t, token) { - w.WriteHeader(http.StatusAccepted) - } else { - w.WriteHeader(http.StatusForbidden) + token_found = true + break } } + + if token_found { + w.WriteHeader(http.StatusAccepted) + } else { + w.WriteHeader(http.StatusForbidden) + } } wg.Done() }) - return "http://" + listener.Addr().String() + address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1) + return address } func TestFakeSecureService(t *testing.T) { @@ -55,20 +64,60 @@ func TestFakeSecureService(t *testing.T) { wg.Add(3) - resp, _ = http.Get(addr + "/token") + resp, _ = http.Get("http://" + addr + "/token") token, _ := ioutil.ReadAll(resp.Body) // Right token - resp, _ = http.Get(addr + "/secure?token=" + string(token)) + resp, _ = http.Get("http://" + addr + "/secure?token=" + string(token)) if resp.StatusCode != http.StatusAccepted { t.Error("Valid token should returns wrong status:", resp.StatusCode) } // Wrong tokens forbidden - resp, _ = http.Get(addr + "/secure?token=wrong") + resp, _ = http.Get("http://" + addr + "/secure?token=wrong") if resp.StatusCode != http.StatusForbidden { t.Error("Wrong tokens should be forbidden, instead:", resp.StatusCode) } wg.Wait() } + +func TestTrafficModifier(t *testing.T) { + var resp *http.Response + + wg := new(sync.WaitGroup) + + from := NewFakeSecureService(wg) + to := NewFakeSecureService(wg) + + quit := make(chan int) + + // Catch traffic from one service + input := NewRAWInput(from) + + // And redirect to another + headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} + methods := HTTPMethods{"GET", "PUT", "POST"} + output := NewHTTPOutput(to, headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) + + Plugins.Inputs = []io.Reader{input} + Plugins.Outputs = []io.Writer{output} + + // Start Gor + go Start(quit) + + // Should receive 2 requests from original + 2 from replayed + wg.Add(4) + + // Sending traffic to original service + resp, _ = http.Get("http://" + from + "/token") + token, _ := ioutil.ReadAll(resp.Body) + + resp, _ = http.Get("http://" + from + "/secure?token=" + string(token)) + if resp.StatusCode != http.StatusAccepted { + t.Error("Valid token should returns wrong status:", resp.StatusCode) + } + + wg.Wait() + close(quit) +} From e64aac9364c5aee4d7d3436081671cbb3cf06cc0 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 9 Jul 2015 20:24:27 +0500 Subject: [PATCH 06/64] Rename input modifier to middleware --- traffic_modifier.go => middleware.go | 14 +++++++------- traffic_modifier_test.go => middleware_test.go | 2 +- plugins.go | 6 +++--- settings.go | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) rename traffic_modifier.go => middleware.go (80%) rename traffic_modifier_test.go => middleware_test.go (98%) diff --git a/traffic_modifier.go b/middleware.go similarity index 80% rename from traffic_modifier.go rename to middleware.go index 6a4a5b65..5b20f77a 100644 --- a/traffic_modifier.go +++ b/middleware.go @@ -11,7 +11,7 @@ import ( "strings" ) -type TrafficModifier struct { +type Middleware struct { plugin interface{} command string @@ -21,8 +21,8 @@ type TrafficModifier struct { Stdout io.Reader } -func NewTrafficModifier(plugin interface{}, command string) io.Reader { - m := new(TrafficModifier) +func NewMiddleware(plugin interface{}, command string) io.Reader { + m := new(Middleware) m.plugin = plugin m.command = command m.data = make(chan []byte) @@ -50,7 +50,7 @@ func NewTrafficModifier(plugin interface{}, command string) io.Reader { return m } -func (m *TrafficModifier) copy(to io.Writer, from io.Reader) { +func (m *Middleware) copy(to io.Writer, from io.Reader) { buf := make([]byte, 5*1024*1024) dst := make([]byte, len(buf)*2) @@ -64,7 +64,7 @@ func (m *TrafficModifier) copy(to io.Writer, from io.Reader) { } } -func (m *TrafficModifier) read(from io.Reader) { +func (m *Middleware) read(from io.Reader) { buf := make([]byte, 5*1024*1024) scanner := bufio.NewScanner(from) @@ -85,7 +85,7 @@ func (m *TrafficModifier) read(from io.Reader) { return } -func (m *TrafficModifier) Read(data []byte) (int, error) { +func (m *Middleware) Read(data []byte) (int, error) { Debug("Trying to read channel!") buf := <-m.data copy(data, buf) @@ -93,6 +93,6 @@ func (m *TrafficModifier) Read(data []byte) (int, error) { return len(buf), nil } -func (m *TrafficModifier) String() string { +func (m *Middleware) String() string { return fmt.Sprintf("Modifying traffic for %s using '%s' command", m.plugin, m.command) } diff --git a/traffic_modifier_test.go b/middleware_test.go similarity index 98% rename from traffic_modifier_test.go rename to middleware_test.go index 53072564..347453c8 100644 --- a/traffic_modifier_test.go +++ b/middleware_test.go @@ -82,7 +82,7 @@ func TestFakeSecureService(t *testing.T) { wg.Wait() } -func TestTrafficModifier(t *testing.T) { +func TestMiddleware(t *testing.T) { var resp *http.Response wg := new(sync.WaitGroup) diff --git a/plugins.go b/plugins.go index b06b8cc5..e83a7f20 100644 --- a/plugins.go +++ b/plugins.go @@ -10,7 +10,7 @@ type InOutPlugins struct { Inputs []io.Reader Outputs []io.Writer - Modifiers []TrafficModifier + Middleware []Middleware } type ReaderOrWriter interface{} @@ -56,8 +56,8 @@ func registerPlugin(constructor interface{}, options ...interface{}) { } if _, ok := plugin.(io.Reader); ok { - for _, options := range Settings.inputModifier { - plugin_wrapper = NewTrafficModifier(plugin_wrapper, options) + for _, options := range Settings.middleware { + plugin_wrapper = NewMiddleware(plugin_wrapper, options) } Plugins.Inputs = append(Plugins.Inputs, plugin_wrapper.(io.Reader)) } diff --git a/settings.go b/settings.go index f7e168cc..537b97af 100644 --- a/settings.go +++ b/settings.go @@ -41,7 +41,7 @@ type AppSettings struct { inputRAW MultiOption - inputModifier MultiOption + middleware MultiOption inputHTTP MultiOption outputHTTP MultiOption @@ -78,7 +78,7 @@ func init() { flag.Var(&Settings.inputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com") - flag.Var(&Settings.inputModifier, "input-modifier", "Used for modifying input traffic using external command") + flag.Var(&Settings.middleware, "middleware", "Used for modifying input traffic using external command") flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com") From f812120dfc2152d4f9747f3234581ccaac0f0d09 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 10 Jul 2015 10:24:00 +0500 Subject: [PATCH 07/64] Support only 1 middleware and rewrite tests --- emitter.go | 2 ++ http_client.go | 20 ++++++++++------ middleware_test.go | 58 ++++++++++++++++++++++++---------------------- plugins.go | 4 ++-- proto/proto.go | 5 ++++ settings.go | 6 ++--- 6 files changed, 55 insertions(+), 40 deletions(-) diff --git a/emitter.go b/emitter.go index 174cfc87..3fdbac44 100644 --- a/emitter.go +++ b/emitter.go @@ -31,6 +31,8 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { if nr > 0 && len(buf) > nr { payload := buf[0:nr] + Debug("[EMITTER] Received payload:", string(payload)) + if modifier != nil { payload = modifier.Rewrite(payload) diff --git a/http_client.go b/http_client.go index 4e44f774..d142e21a 100644 --- a/http_client.go +++ b/http_client.go @@ -72,7 +72,7 @@ func (c *HTTPClient) Disconnect() { if c.conn != nil { c.conn.Close() c.conn = nil - Debug("Disconnected: ", c.baseURL) + Debug("[HTTP] Disconnected: ", c.baseURL) } } @@ -90,7 +90,7 @@ func (c *HTTPClient) isAlive() bool { func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if c.conn == nil || !c.isAlive() { - Debug("Connecting:", c.baseURL) + Debug("[HTTP] Connecting:", c.baseURL) c.Connect() } @@ -101,11 +101,11 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host)) if c.config.Debug { - Debug("Sending:", string(data)) + Debug("[HTTP] Sending:", string(data)) } if _, err = c.conn.Write(data); err != nil { - Debug("Write error:", err, c.baseURL) + Debug("[HTTP] Write error:", err, c.baseURL) return } @@ -113,14 +113,14 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { n, err := c.conn.Read(c.respBuf) if err != nil { - Debug("READ ERRORR!", err, c.conn) + Debug("[HTTP] READ ERRORR!", err, c.conn) return } payload := c.respBuf[:n] if c.config.Debug { - Debug("Received:", string(payload)) + Debug("[HTTP] Received:", string(payload)) } if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects { @@ -134,7 +134,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n") if c.config.Debug { - Debug("Redirecting to: " + string(location)) + Debug("[HTTP] Redirecting to: " + string(location)) } return c.Send(redirectPayload) @@ -145,3 +145,9 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { return payload, err } + +func (c *HTTPClient) Get(path string) (response []byte, err error) { + payload := "GET " + path + " HTTP/1.1\r\n\r\n" + + return c.Send([]byte(payload)) +} \ No newline at end of file diff --git a/middleware_test.go b/middleware_test.go index 347453c8..7ce251a0 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -1,22 +1,20 @@ package main import ( - _ "bufio" "bytes" "crypto/rand" "io" - "io/ioutil" - _ "log" - _ "net" "net/http" "sync" "testing" "strings" + "github.com/buger/gor/proto" + "encoding/hex" ) // Simple service that generate token on request, and require this token for accesing to secure area func NewFakeSecureService(wg *sync.WaitGroup) string { - active_tokens := make([][]byte, 0) + active_tokens := make([]string, 0) listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { Debug("Received request: " + req.URL.String()) @@ -25,17 +23,18 @@ func NewFakeSecureService(wg *sync.WaitGroup) string { case "/token": // Generate random token token_length := 10 - token := make([]byte, token_length) - rand.Read(token) + buf := make([]byte, token_length) + rand.Read(buf) + token := hex.EncodeToString(buf) active_tokens = append(active_tokens, token) - w.Write(token) + w.Write([]byte(token)) case "/secure": - token := []byte(req.URL.Query().Get("token")) + token := req.URL.Query().Get("token") token_found := false for _, t := range active_tokens { - if bytes.Equal(t, token) { + if t == token { token_found = true break } @@ -56,7 +55,7 @@ func NewFakeSecureService(wg *sync.WaitGroup) string { } func TestFakeSecureService(t *testing.T) { - var resp *http.Response + var resp, token []byte wg := new(sync.WaitGroup) @@ -64,26 +63,27 @@ func TestFakeSecureService(t *testing.T) { wg.Add(3) - resp, _ = http.Get("http://" + addr + "/token") - token, _ := ioutil.ReadAll(resp.Body) + client := NewHTTPClient("http://" + addr, &HTTPClientConfig{Debug: true}) + resp, _ = client.Get("/token") + token = proto.Body(resp) - // Right token - resp, _ = http.Get("http://" + addr + "/secure?token=" + string(token)) - if resp.StatusCode != http.StatusAccepted { - t.Error("Valid token should returns wrong status:", resp.StatusCode) - } + // Right token + resp, _ = client.Get("/secure?token=" + string(token)) + if !bytes.Equal(proto.Status(resp), []byte("202")) { + t.Error("Valid token should return status 202:", string(proto.Status(resp))) + } - // Wrong tokens forbidden - resp, _ = http.Get("http://" + addr + "/secure?token=wrong") - if resp.StatusCode != http.StatusForbidden { - t.Error("Wrong tokens should be forbidden, instead:", resp.StatusCode) + // Wrong tokens forbidden + resp, _ = client.Get("/secure?token=wrong") + if !bytes.Equal(proto.Status(resp), []byte("403")) { + t.Error("Wrong token should returns status 403:", string(proto.Status(resp))) } wg.Wait() } func TestMiddleware(t *testing.T) { - var resp *http.Response + var resp, token []byte wg := new(sync.WaitGroup) @@ -107,13 +107,15 @@ func TestMiddleware(t *testing.T) { // Should receive 2 requests from original + 2 from replayed wg.Add(4) + client := NewHTTPClient("http://" + from, &HTTPClientConfig{Debug: true}) + // Sending traffic to original service - resp, _ = http.Get("http://" + from + "/token") - token, _ := ioutil.ReadAll(resp.Body) + resp, _ = client.Get("/token") + token = proto.Body(resp) - resp, _ = http.Get("http://" + from + "/secure?token=" + string(token)) - if resp.StatusCode != http.StatusAccepted { - t.Error("Valid token should returns wrong status:", resp.StatusCode) + resp, _ = client.Get("/secure?token=" + string(token)) + if !bytes.Equal(proto.Status(resp), []byte("202")) { + t.Error("Valid token should return 202:", proto.Status(resp)) } wg.Wait() diff --git a/plugins.go b/plugins.go index e83a7f20..adfc2424 100644 --- a/plugins.go +++ b/plugins.go @@ -56,8 +56,8 @@ func registerPlugin(constructor interface{}, options ...interface{}) { } if _, ok := plugin.(io.Reader); ok { - for _, options := range Settings.middleware { - plugin_wrapper = NewMiddleware(plugin_wrapper, options) + if len(Settings.middleware) > 0 { + plugin_wrapper = NewMiddleware(plugin_wrapper, Settings.middleware) } Plugins.Inputs = append(Plugins.Inputs, plugin_wrapper.(io.Reader)) } diff --git a/proto/proto.go b/proto/proto.go index 626d2893..9ca60e50 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -68,6 +68,11 @@ func AddHeader(payload, name, value []byte) []byte { return byteutils.Insert(payload, mimeStart, header) } +func Body(payload []byte) []byte { + // 4 -> len(EMPTY_LINE) + return payload[MIMEHeadersEndPos(payload) + 4:] +} + func Path(payload []byte) []byte { start := bytes.IndexByte(payload, ' ') start += 1 diff --git a/settings.go b/settings.go index 537b97af..5c34f468 100644 --- a/settings.go +++ b/settings.go @@ -41,7 +41,7 @@ type AppSettings struct { inputRAW MultiOption - middleware MultiOption + middleware string inputHTTP MultiOption outputHTTP MultiOption @@ -78,7 +78,7 @@ func init() { flag.Var(&Settings.inputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com") - flag.Var(&Settings.middleware, "middleware", "Used for modifying input traffic using external command") + flag.StringVar(&Settings.middleware, "middleware", "", "Used for modifying traffic using external command") flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com") @@ -117,7 +117,7 @@ func init() { func Debug(args ...interface{}) { if Settings.verbose { - log.Print("[DEBUG] ") + fmt.Print("[DEBUG] ") log.Println(args...) } } From 5e6f7e03a763bdb97b343d8e67afe802c2b2ec15 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 17 Jul 2015 20:58:31 +0500 Subject: [PATCH 08/64] First middleware tests! :dance: --- emitter.go | 31 ++++++++++++-- examples/echo_modifier.rb | 6 +-- examples/echo_modifier.sh | 6 +-- middleware.go | 19 ++++----- middleware_test.go | 85 ++++++++++++++++++++++++++++++++++----- plugins.go | 7 ---- 6 files changed, 120 insertions(+), 34 deletions(-) diff --git a/emitter.go b/emitter.go index 3fdbac44..8237f6f9 100644 --- a/emitter.go +++ b/emitter.go @@ -3,11 +3,36 @@ package main import ( "io" "time" + "crypto/rand" ) +func uuid() []byte { + b := make([]byte, 16) + rand.Read(b) + return b +} + + func Start(stop chan int) { - for _, in := range Plugins.Inputs { - go CopyMulty(in, Plugins.Outputs...) + if Settings.middleware != "" { + middleware := NewMiddleware(Settings.middleware) + + for _, in := range Plugins.Inputs { + middleware.ReadFrom(in) + } + + // We going only to read responses, so using same ReadFrom method + for _, out := range Plugins.Outputs { + if r, ok := out.(io.Reader); ok { + middleware.ReadFrom(r) + } + } + + go CopyMulty(middleware, Plugins.Outputs...) + } else { + for _, in := range Plugins.Inputs { + go CopyMulty(in, Plugins.Outputs...) + } } for { @@ -31,7 +56,7 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { if nr > 0 && len(buf) > nr { payload := buf[0:nr] - Debug("[EMITTER] Received payload:", string(payload)) + Debug("[EMITTER] input:", string(payload)) if modifier != nil { payload = modifier.Rewrite(payload) diff --git a/examples/echo_modifier.rb b/examples/echo_modifier.rb index e3a489d0..a20beb83 100755 --- a/examples/echo_modifier.rb +++ b/examples/echo_modifier.rb @@ -10,7 +10,7 @@ STDOUT.puts encoded - STDERR.puts "[DEBUG] Original data: #{data}" - STDERR.puts "[DEBUG] Decoded request: #{decoded}" - STDERR.puts "[DEBUG] Encoded data: #{encoded}" + STDERR.puts "[DEBUG][MIDDLEWARE] Original data: #{data}" + STDERR.puts "[DEBUG][MIDDLEWARE] Decoded request: #{decoded}" + STDERR.puts "[DEBUG][MIDDLEWARE] Encoded data: #{encoded}" end \ No newline at end of file diff --git a/examples/echo_modifier.sh b/examples/echo_modifier.sh index 1d56cdc1..22c7386b 100755 --- a/examples/echo_modifier.sh +++ b/examples/echo_modifier.sh @@ -4,7 +4,7 @@ while read line; do encoded=$(echo "$decoded" | xxd -p | tr -d "\\n") echo "$encoded" - >&2 echo "[DEBUG] Original data: $line" - >&2 echo "[DEBUG] Decoded request: $decoded" - >&2 echo "[DEBUG] Encoded data: $encoded" + >&2 echo "[DEBUG][MIDDLEWARE] Original data: $line" + >&2 echo "[DEBUG][MIDDLEWARE] Decoded request: $decoded" + >&2 echo "[DEBUG][MIDDLEWARE] Encoded data: $encoded" done; diff --git a/middleware.go b/middleware.go index 5b20f77a..dc387eb0 100644 --- a/middleware.go +++ b/middleware.go @@ -12,7 +12,6 @@ import ( ) type Middleware struct { - plugin interface{} command string data chan []byte @@ -21,11 +20,10 @@ type Middleware struct { Stdout io.Reader } -func NewMiddleware(plugin interface{}, command string) io.Reader { +func NewMiddleware(command string) *Middleware { m := new(Middleware) - m.plugin = plugin m.command = command - m.data = make(chan []byte) + m.data = make(chan []byte, 1000) commands := strings.Split(command, " ") cmd := exec.Command(commands[0], commands[1:]...) @@ -34,7 +32,6 @@ func NewMiddleware(plugin interface{}, command string) io.Reader { m.Stdin, _ = cmd.StdinPipe() cmd.Stderr = os.Stderr - go m.copy(m.Stdin, m.plugin.(io.Reader)) go m.read(m.Stdout) go func() { @@ -43,13 +40,17 @@ func NewMiddleware(plugin interface{}, command string) io.Reader { if err != nil { log.Fatal(err) } - }() - defer cmd.Wait() + cmd.Wait() + }() return m } +func (m *Middleware) ReadFrom(plugin io.Reader) { + go m.copy(m.Stdin, plugin) +} + func (m *Middleware) copy(to io.Writer, from io.Reader) { buf := make([]byte, 5*1024*1024) dst := make([]byte, len(buf)*2) @@ -59,7 +60,7 @@ func (m *Middleware) copy(to io.Writer, from io.Reader) { if nr > 0 && len(buf) > nr { hex.Encode(dst, buf[0:nr]) to.Write(dst[0 : nr*2]) - to.Write([]byte("\r\n")) + to.Write([]byte("\n")) } } } @@ -94,5 +95,5 @@ func (m *Middleware) Read(data []byte) (int, error) { } func (m *Middleware) String() string { - return fmt.Sprintf("Modifying traffic for %s using '%s' command", m.plugin, m.command) + return fmt.Sprintf("Modifying traffic using '%s' command", m.command) } diff --git a/middleware_test.go b/middleware_test.go index 7ce251a0..e069d23c 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -9,14 +9,18 @@ import ( "testing" "strings" "github.com/buger/gor/proto" + "net/http/httptest" "encoding/hex" + "time" ) +type fakeServiceCb func(string, int, []byte) + // Simple service that generate token on request, and require this token for accesing to secure area -func NewFakeSecureService(wg *sync.WaitGroup) string { +func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) string { active_tokens := make([]string, 0) - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { Debug("Received request: " + req.URL.String()) switch req.URL.Path { @@ -29,6 +33,8 @@ func NewFakeSecureService(wg *sync.WaitGroup) string { active_tokens = append(active_tokens, token) w.Write([]byte(token)) + + cb(req.URL.Path, 200, []byte(token)) case "/secure": token := req.URL.Query().Get("token") token_found := false @@ -42,15 +48,17 @@ func NewFakeSecureService(wg *sync.WaitGroup) string { if token_found { w.WriteHeader(http.StatusAccepted) + cb(req.URL.Path, 202, []byte(nil)) } else { w.WriteHeader(http.StatusForbidden) + cb(req.URL.Path, 403, []byte(nil)) } } wg.Done() - }) + })) - address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1) + address := strings.Replace(server.Listener.Addr().String(), "[::]", "127.0.0.1", -1) return address } @@ -59,7 +67,9 @@ func TestFakeSecureService(t *testing.T) { wg := new(sync.WaitGroup) - addr := NewFakeSecureService(wg) + addr := NewFakeSecureService(wg, func(path string, status int, resp []byte){ + + }) wg.Add(3) @@ -82,13 +92,66 @@ func TestFakeSecureService(t *testing.T) { wg.Wait() } -func TestMiddleware(t *testing.T) { +func TestEchoMiddleware(t *testing.T) { + wg := new(sync.WaitGroup) + + from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + wg.Done() + })) + to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + wg.Done() + })) + + quit := make(chan int) + + // Catch traffic from one service + input := NewRAWInput(from.Listener.Addr().String()) + + // And redirect to another + output := NewHTTPOutput(to.URL, &HTTPOutputConfig{}) + + Plugins.Inputs = []io.Reader{input} + Plugins.Outputs = []io.Writer{output} + Settings.middleware = "./examples/echo_modifier.sh" + + // Start Gor + go Start(quit) + + time.Sleep(time.Millisecond) + + // Should receive 2 requests from original + 2 from replayed + wg.Add(4) + + client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true}) + + // Request should be echoed + client.Get("/") + client.Get("/") + + wg.Wait() + close(quit) + Settings.middleware = "" +} + +func TestTokenMiddleware(t *testing.T) { var resp, token []byte wg := new(sync.WaitGroup) - from := NewFakeSecureService(wg) - to := NewFakeSecureService(wg) + from := NewFakeSecureService(wg, func(path string, status int, tok []byte){ + }) + to := NewFakeSecureService(wg, func(path string, status int, tok []byte){ + switch path { + case "/token": + if bytes.Equal(token, tok) { + t.Error("Tokens should not match") + } + case "/secure": + if status != 202 { + // t.Error("Server should receive valid rewritten token") + } + } + }) quit := make(chan int) @@ -100,10 +163,13 @@ func TestMiddleware(t *testing.T) { Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} + // Settings.middleware = "./examples/echo_modifier.sh" // Start Gor go Start(quit) + time.Sleep(time.Millisecond) + // Should receive 2 requests from original + 2 from replayed wg.Add(4) @@ -120,4 +186,5 @@ func TestMiddleware(t *testing.T) { wg.Wait() close(quit) -} + Settings.middleware = "" +} \ No newline at end of file diff --git a/plugins.go b/plugins.go index adfc2424..321317e2 100644 --- a/plugins.go +++ b/plugins.go @@ -9,12 +9,8 @@ import ( type InOutPlugins struct { Inputs []io.Reader Outputs []io.Writer - - Middleware []Middleware } -type ReaderOrWriter interface{} - var Plugins *InOutPlugins = new(InOutPlugins) func extractLimitOptions(options string) (string, string) { @@ -56,9 +52,6 @@ func registerPlugin(constructor interface{}, options ...interface{}) { } if _, ok := plugin.(io.Reader); ok { - if len(Settings.middleware) > 0 { - plugin_wrapper = NewMiddleware(plugin_wrapper, Settings.middleware) - } Plugins.Inputs = append(Plugins.Inputs, plugin_wrapper.(io.Reader)) } From d62eccb655f048ac8434094e2b41ba2b20564b3e Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 21 Jul 2015 10:06:10 +0500 Subject: [PATCH 09/64] Make raw message expiration configurable to improve tests time --- input_raw.go | 7 +++++-- input_raw_test.go | 10 ++++++---- middleware_test.go | 4 ++-- raw_socket_listener/listener.go | 13 +++++++++++-- raw_socket_listener/tcp_message.go | 13 ++++++------- 5 files changed, 30 insertions(+), 17 deletions(-) diff --git a/input_raw.go b/input_raw.go index c4f3c342..613cbbe8 100644 --- a/input_raw.go +++ b/input_raw.go @@ -5,19 +5,22 @@ import ( "log" "net" "strings" + "time" ) // RAWInput used for intercepting traffic for given address type RAWInput struct { data chan []byte address string + expire time.Duration } // NewRAWInput constructor for RAWInput. Accepts address with port as argument. -func NewRAWInput(address string) (i *RAWInput) { +func NewRAWInput(address string, expire time.Duration) (i *RAWInput) { i = new(RAWInput) i.data = make(chan []byte) i.address = address + i.expire = expire go i.listen(address) @@ -42,7 +45,7 @@ func (i *RAWInput) listen(address string) { log.Fatal("input-raw: error while parsing address", err) } - listener := raw.NewListener(host, port) + listener := raw.NewListener(host, port, i.expire) for { // Receiving TCPMessage object diff --git a/input_raw_test.go b/input_raw_test.go index 0f7b0c3e..a1458453 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -15,6 +15,8 @@ import ( "time" ) +const testRawExpire = time.Millisecond * 100 + func TestRAWInput(t *testing.T) { wg := new(sync.WaitGroup) @@ -22,7 +24,7 @@ func TestRAWInput(t *testing.T) { listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) - input := NewRAWInput(listener.Addr().String()) + input := NewRAWInput(listener.Addr().String(), testRawExpire) output := NewTestOutput(func(data []byte) { wg.Done() }) @@ -64,7 +66,7 @@ func TestInputRAW100Expect(t *testing.T) { originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr) + input := NewRAWInput(originAddr, testRawExpire) // We will use it to get content of raw HTTP request testOutput := NewTestOutput(func(data []byte) { @@ -121,7 +123,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr) + input := NewRAWInput(originAddr, testRawExpire) listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { defer req.Body.Close() @@ -179,7 +181,7 @@ func TestInputRAWLargePayload(t *testing.T) { })) originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr) + input := NewRAWInput(originAddr, testRawExpire) replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024) diff --git a/middleware_test.go b/middleware_test.go index cfea8b4a..cb7319a4 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -105,7 +105,7 @@ func TestEchoMiddleware(t *testing.T) { quit := make(chan int) // Catch traffic from one service - input := NewRAWInput(from.Listener.Addr().String()) + input := NewRAWInput(from.Listener.Addr().String(), testRawExpire) // And redirect to another output := NewHTTPOutput(to.URL, &HTTPOutputConfig{}) @@ -156,7 +156,7 @@ func TestTokenMiddleware(t *testing.T) { quit := make(chan int) // Catch traffic from one service - input := NewRAWInput(from) + input := NewRAWInput(from, testRawExpire) // And redirect to another output := NewHTTPOutput(to, &HTTPOutputConfig{}) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index f82ae6fb..53857ac9 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -18,6 +18,7 @@ import ( "log" "net" "strconv" + "time" ) // Listener handle traffic capture @@ -42,10 +43,12 @@ type Listener struct { addr string // IP to listen port int // Port to listen + + messageExpire time.Duration } // NewListener creates and initializes new Listener object -func NewListener(addr string, port string) (rawListener *Listener) { +func NewListener(addr string, port string, expire time.Duration) (rawListener *Listener) { rawListener = &Listener{} rawListener.packetsChan = make(chan *TCPPacket, 10000) @@ -59,6 +62,12 @@ func NewListener(addr string, port string) (rawListener *Listener) { rawListener.addr = addr rawListener.port, _ = strconv.Atoi(port) + if expire.Nanoseconds() == 0 { + expire = 2000 * time.Millisecond + } + + rawListener.messageExpire = expire + go rawListener.listen() go rawListener.readRAWSocket() @@ -157,7 +166,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { if !ok { // We sending messageDelChan channel, so message object can communicate with Listener and notify it if message completed - message = NewTCPMessage(mID, t.messageDelChan, packet.Ack) + message = NewTCPMessage(mID, t.messageDelChan, packet.Ack, &t.messageExpire) t.messages[mID] = message } diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index ddfb0f28..e6e31bda 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -6,9 +6,6 @@ import ( "time" ) -// MsgExpire specify period that message should wait before it considered as finished -const MsgExpire = 2000 * time.Millisecond - // TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence // Its needed because all TCP message can be fragmented or re-transmitted // @@ -25,17 +22,19 @@ type TCPMessage struct { packetsChan chan *TCPPacket delChan chan *TCPMessage + + expire *time.Duration } // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted -func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32) (msg *TCPMessage) { - msg = &TCPMessage{ID: ID, Ack: Ack} +func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32, expire *time.Duration) (msg *TCPMessage) { + msg = &TCPMessage{ID: ID, Ack: Ack, expire: expire} msg.packetsChan = make(chan *TCPPacket) msg.delChan = delChan // used for notifying that message completed or expired // Every time we receive packet we reset this timer - msg.timer = time.AfterFunc(MsgExpire, msg.Timeout) + msg.timer = time.AfterFunc(*msg.expire, msg.Timeout) go msg.listen() @@ -103,5 +102,5 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { } // Reset message timeout timer - t.timer.Reset(MsgExpire) + t.timer.Reset(*t.expire) } From 35854196f05e0b8f7b2969c121a28c2ebd21b627 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 3 Aug 2015 20:07:28 +0300 Subject: [PATCH 10/64] Fix tests --- output_http_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/output_http_test.go b/output_http_test.go index ddc9110c..a428c388 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -82,7 +82,7 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) { input := NewTestInput() - listener := startHTTP(func(req *http.Request) { + listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { if req.Host != "custom-host.com" { t.Error("Wrong header", req.Host) } From 9ab8b3c11372848e76fa3ff87566f421649c24e6 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 10 Aug 2015 22:57:18 +0300 Subject: [PATCH 11/64] Add header to request and response --- emitter.go | 10 ++++++---- examples/echo_modifier.sh | 23 +++++++++++++++++++++-- input_raw.go | 9 ++++++++- middleware.go | 2 +- middleware_test.go | 7 +++++-- output_http.go | 20 ++++++++++++++++++++ 6 files changed, 61 insertions(+), 10 deletions(-) diff --git a/emitter.go b/emitter.go index 63453a5a..ec5065c6 100644 --- a/emitter.go +++ b/emitter.go @@ -56,7 +56,9 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { if nr > 0 && len(buf) > nr { payload := buf[0:nr] - Debug("[EMITTER] input:", string(payload)) + if Settings.debug { + Debug("[EMITTER] input:", string(payload[0:500])) + } if modifier != nil { payload = modifier.Rewrite(payload) @@ -65,10 +67,10 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { if len(payload) == 0 { continue } - } - if Settings.debug { - Debug("[EMITTER] Sending payload, size:", len(payload), "First 500 bytes:", string(payload[0:500])) + if Settings.debug { + Debug("[EMITTER] Rewrittern input:", len(payload), "First 500 bytes:", string(payload[0:500])) + } } if Settings.splitOutput { diff --git a/examples/echo_modifier.sh b/examples/echo_modifier.sh index 22c7386b..9aded362 100755 --- a/examples/echo_modifier.sh +++ b/examples/echo_modifier.sh @@ -1,8 +1,27 @@ #!/usr/bin/env bash while read line; do decoded=$(echo "$line" | xxd -r -p) - encoded=$(echo "$decoded" | xxd -p | tr -d "\\n") - echo "$encoded" + + header=$(echo "$decoded" | head -n +1) + payload=$(echo "$decoded" | tail -n +2) + + encoded=$(echo -e "$header\n$payload" | xxd -p | tr -d "\\n") + + >&2 echo "" + >&2 echo "[DEBUG][MIDDLEWARE] ===================================" + + case ${header:0:1} in + "2") + >&2 echo "[DEBUG][MIDDLEWARE] Request type: Replayed Response" + ;; + "1") + >&2 echo "[DEBUG][MIDDLEWARE] Request type: Request" + echo "$encoded" + ;; + *) + >&2 echo "[DEBUG][MIDDLEWARE] Unknown request type $header" + esac + >&2 echo "[DEBUG][MIDDLEWARE] ===================================" >&2 echo "[DEBUG][MIDDLEWARE] Original data: $line" >&2 echo "[DEBUG][MIDDLEWARE] Decoded request: $decoded" diff --git a/input_raw.go b/input_raw.go index 613cbbe8..48775fac 100644 --- a/input_raw.go +++ b/input_raw.go @@ -29,7 +29,14 @@ func NewRAWInput(address string, expire time.Duration) (i *RAWInput) { func (i *RAWInput) Read(data []byte) (int, error) { buf := <-i.data - copy(data, buf) + + if len(Settings.middleware) > 0 { + header := []byte("1\n") + copy(data[0:len(header)], header) + copy(data[len(header):], buf) + } else { + copy(data, buf) + } return len(buf), nil } diff --git a/middleware.go b/middleware.go index 20714bff..89080e2d 100644 --- a/middleware.go +++ b/middleware.go @@ -74,7 +74,7 @@ func (m *Middleware) read(from io.Reader) { bytes := scanner.Bytes() hex.Decode(buf, bytes) - Debug("Received:", buf[0:len(bytes)/2]) + Debug("[MIDDLEWARE-MASTER] Received:", string(buf[0:len(bytes)/2])) m.data <- buf[0 : len(bytes)/2] } diff --git a/middleware_test.go b/middleware_test.go index cb7319a4..d9274bc9 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -99,6 +99,7 @@ func TestEchoMiddleware(t *testing.T) { wg.Done() })) to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + panic("Asdasd") wg.Done() })) @@ -108,7 +109,7 @@ func TestEchoMiddleware(t *testing.T) { input := NewRAWInput(from.Listener.Addr().String(), testRawExpire) // And redirect to another - output := NewHTTPOutput(to.URL, &HTTPOutputConfig{}) + output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} @@ -122,7 +123,7 @@ func TestEchoMiddleware(t *testing.T) { // Should receive 2 requests from original + 2 from replayed wg.Add(4) - client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true}) + client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false}) // Request should be echoed client.Get("/") @@ -131,6 +132,8 @@ func TestEchoMiddleware(t *testing.T) { wg.Wait() close(quit) Settings.middleware = "" + + time.Sleep(10*time.Millisecond) } func TestTokenMiddleware(t *testing.T) { diff --git a/output_http.go b/output_http.go index 43bab96b..3288c304 100644 --- a/output_http.go +++ b/output_http.go @@ -60,6 +60,7 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer { } o.queue = make(chan []byte, 100) + o.responses = make(chan []byte, 100) o.needWorker = make(chan int, 1) // Initial workers count @@ -152,15 +153,34 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) { return len(data), nil } +func (o *HTTPOutput) Read(data []byte) (int, error) { + buf := <- o.responses + header := []byte("2\n") + copy(data[0:2], header) + copy(data[2:], buf) + + return len(buf) + len(header), nil +} + func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { + if len(Settings.middleware) > 0 { + request = request[2:] + } + start := time.Now() resp, err := client.Send(request) stop := time.Now() + panic(string(resp)) + if err != nil { log.Println("Request error:", err) } + if len(Settings.middleware) > 0 { + o.responses <- resp + } + if o.elasticSearch != nil { o.elasticSearch.ResponseAnalyze(request, resp, start, stop) } From 5098e388a24a3b09871928b223b07e186a4cf981 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 11 Aug 2015 11:06:25 +0300 Subject: [PATCH 12/64] Fix tests, now properly receive responses --- examples/echo_modifier.sh | 22 +++++++++++----------- input_raw.go | 6 ++++-- middleware.go | 9 ++++++++- middleware_test.go | 3 +-- output_http.go | 2 -- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/examples/echo_modifier.sh b/examples/echo_modifier.sh index 9aded362..e3f7dd9f 100755 --- a/examples/echo_modifier.sh +++ b/examples/echo_modifier.sh @@ -1,29 +1,29 @@ #!/usr/bin/env bash while read line; do - decoded=$(echo "$line" | xxd -r -p) + decoded=$(echo -e "$line" | xxd -r -p) - header=$(echo "$decoded" | head -n +1) - payload=$(echo "$decoded" | tail -n +2) + header=$(echo -e "$decoded" | head -n +1) + payload=$(echo -e "$decoded" | tail -n +2) encoded=$(echo -e "$header\n$payload" | xxd -p | tr -d "\\n") >&2 echo "" - >&2 echo "[DEBUG][MIDDLEWARE] ===================================" + >&2 echo "[DEBUG][ECHO] ===================================" case ${header:0:1} in "2") - >&2 echo "[DEBUG][MIDDLEWARE] Request type: Replayed Response" + >&2 echo "[DEBUG][ECHO] Request type: Replayed Response" ;; "1") - >&2 echo "[DEBUG][MIDDLEWARE] Request type: Request" + >&2 echo "[DEBUG][ECHO] Request type: Request" echo "$encoded" ;; *) - >&2 echo "[DEBUG][MIDDLEWARE] Unknown request type $header" + >&2 echo "[DEBUG][ECHO] Unknown request type $header" esac - >&2 echo "[DEBUG][MIDDLEWARE] ===================================" + >&2 echo "[DEBUG][ECHO] ===================================" - >&2 echo "[DEBUG][MIDDLEWARE] Original data: $line" - >&2 echo "[DEBUG][MIDDLEWARE] Decoded request: $decoded" - >&2 echo "[DEBUG][MIDDLEWARE] Encoded data: $encoded" + >&2 echo "[DEBUG][ECHO] Original data: $line" + >&2 echo "[DEBUG][ECHO] Decoded request: $decoded" + >&2 echo "[DEBUG][ECHO] Encoded data: $encoded" done; diff --git a/input_raw.go b/input_raw.go index 48775fac..e29a2735 100644 --- a/input_raw.go +++ b/input_raw.go @@ -34,11 +34,13 @@ func (i *RAWInput) Read(data []byte) (int, error) { header := []byte("1\n") copy(data[0:len(header)], header) copy(data[len(header):], buf) + + return len(buf)+len(header), nil } else { copy(data, buf) - } - return len(buf), nil + return len(buf), nil + } } func (i *RAWInput) listen(address string) { diff --git a/middleware.go b/middleware.go index 89080e2d..c642cd23 100644 --- a/middleware.go +++ b/middleware.go @@ -58,9 +58,14 @@ func (m *Middleware) copy(to io.Writer, from io.Reader) { for { nr, _ := from.Read(buf) if nr > 0 && len(buf) > nr { + hex.Encode(dst, buf[0:nr]) to.Write(dst[0 : nr*2]) to.Write([]byte("\n")) + + if Settings.debug { + Debug("[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from) + } } } } @@ -74,7 +79,9 @@ func (m *Middleware) read(from io.Reader) { bytes := scanner.Bytes() hex.Decode(buf, bytes) - Debug("[MIDDLEWARE-MASTER] Received:", string(buf[0:len(bytes)/2])) + if Settings.debug { + Debug("[MIDDLEWARE-MASTER] Received:", string(buf[0:len(bytes)/2])) + } m.data <- buf[0 : len(bytes)/2] } diff --git a/middleware_test.go b/middleware_test.go index d9274bc9..a75d6bc8 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -99,7 +99,6 @@ func TestEchoMiddleware(t *testing.T) { wg.Done() })) to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - panic("Asdasd") wg.Done() })) @@ -133,7 +132,7 @@ func TestEchoMiddleware(t *testing.T) { close(quit) Settings.middleware = "" - time.Sleep(10*time.Millisecond) + time.Sleep(100*time.Millisecond) } func TestTokenMiddleware(t *testing.T) { diff --git a/output_http.go b/output_http.go index 3288c304..a66027c9 100644 --- a/output_http.go +++ b/output_http.go @@ -171,8 +171,6 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { resp, err := client.Send(request) stop := time.Now() - panic(string(resp)) - if err != nil { log.Println("Request error:", err) } From 0009555d96dd32efe34e36c7b0625bb3aa6e6105 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 11 Aug 2015 11:08:14 +0300 Subject: [PATCH 13/64] Apply fmt --- http_client.go | 14 +++++++------- input_raw.go | 2 +- middleware_test.go | 2 +- output_http.go | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/http_client.go b/http_client.go index 6a8632cd..37b2691d 100644 --- a/http_client.go +++ b/http_client.go @@ -18,11 +18,11 @@ var defaultPorts = map[string]string{ } type HTTPClientConfig struct { - FollowRedirects int - Debug bool - OriginalHost bool - Timeout time.Duration - ResponseBufferSize int + FollowRedirects int + Debug bool + OriginalHost bool + Timeout time.Duration + ResponseBufferSize int } type HTTPClient struct { @@ -52,7 +52,7 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient { } if config.ResponseBufferSize == 0 { - config.ResponseBufferSize = 512*1024 // 500kb + config.ResponseBufferSize = 512 * 1024 // 500kb } client := new(HTTPClient) @@ -69,7 +69,7 @@ func (c *HTTPClient) Connect() (err error) { c.Disconnect() if !strings.Contains(c.host, ":") { - c.conn, err = net.Dial("tcp", c.host + ":80") + c.conn, err = net.Dial("tcp", c.host+":80") } else { c.conn, err = net.Dial("tcp", c.host) } diff --git a/input_raw.go b/input_raw.go index e29a2735..6fdb9eea 100644 --- a/input_raw.go +++ b/input_raw.go @@ -35,7 +35,7 @@ func (i *RAWInput) Read(data []byte) (int, error) { copy(data[0:len(header)], header) copy(data[len(header):], buf) - return len(buf)+len(header), nil + return len(buf) + len(header), nil } else { copy(data, buf) diff --git a/middleware_test.go b/middleware_test.go index a75d6bc8..71748a87 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -132,7 +132,7 @@ func TestEchoMiddleware(t *testing.T) { close(quit) Settings.middleware = "" - time.Sleep(100*time.Millisecond) + time.Sleep(100 * time.Millisecond) } func TestTokenMiddleware(t *testing.T) { diff --git a/output_http.go b/output_http.go index a66027c9..7be477f6 100644 --- a/output_http.go +++ b/output_http.go @@ -154,7 +154,7 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) { } func (o *HTTPOutput) Read(data []byte) (int, error) { - buf := <- o.responses + buf := <-o.responses header := []byte("2\n") copy(data[0:2], header) copy(data[2:], buf) From 8046468ea47701e21fd0c56616349452ef6c546b Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 11 Aug 2015 11:28:22 +0300 Subject: [PATCH 14/64] Fix tests --- plugins.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins.go b/plugins.go index cb189a4b..d0ba71ac 100644 --- a/plugins.go +++ b/plugins.go @@ -55,11 +55,14 @@ func registerPlugin(constructor interface{}, options ...interface{}) { pluginWrapper = plugin } - if _, ok := plugin.(io.Reader); ok { + _, isR := plugin.(io.Reader) + _, isW := plugin.(io.Writer) + + if isR && !isW { Plugins.Inputs = append(Plugins.Inputs, pluginWrapper.(io.Reader)) } - if _, ok := plugin.(io.Writer); ok { + if isW { Plugins.Outputs = append(Plugins.Outputs, pluginWrapper.(io.Writer)) } } From 2990fab6355868038572ff6d371e72a657a9460c Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 11 Aug 2015 17:37:37 +0300 Subject: [PATCH 15/64] Track original responses --- emitter.go | 9 +++++-- examples/echo_modifier.sh | 3 +++ input_raw.go | 41 ++++++++++++++++++++---------- input_raw_test.go | 41 +++++++++++++++++++++++++++--- middleware_test.go | 6 ++--- plugins.go | 1 + raw_socket_listener/listener.go | 38 ++++++++++++++++++++------- raw_socket_listener/tcp_message.go | 24 ++++++++++++++--- raw_socket_listener/tcp_packet.go | 2 +- 9 files changed, 129 insertions(+), 36 deletions(-) diff --git a/emitter.go b/emitter.go index ec5065c6..07692954 100644 --- a/emitter.go +++ b/emitter.go @@ -56,8 +56,13 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { if nr > 0 && len(buf) > nr { payload := buf[0:nr] + _maxN := nr + if nr > 500 { + _maxN = 500 + } + if Settings.debug { - Debug("[EMITTER] input:", string(payload[0:500])) + Debug("[EMITTER] input:", string(payload[0:_maxN])) } if modifier != nil { @@ -69,7 +74,7 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { } if Settings.debug { - Debug("[EMITTER] Rewrittern input:", len(payload), "First 500 bytes:", string(payload[0:500])) + Debug("[EMITTER] Rewrittern input:", len(payload), "First 500 bytes:", string(payload[0:_maxN])) } } diff --git a/examples/echo_modifier.sh b/examples/echo_modifier.sh index e3f7dd9f..105b4bbb 100755 --- a/examples/echo_modifier.sh +++ b/examples/echo_modifier.sh @@ -11,6 +11,9 @@ while read line; do >&2 echo "[DEBUG][ECHO] ===================================" case ${header:0:1} in + "3") + >&2 echo "[DEBUG][ECHO] Request type: Original Response" + ;; "2") >&2 echo "[DEBUG][ECHO] Request type: Replayed Response" ;; diff --git a/input_raw.go b/input_raw.go index 6fdb9eea..762655c0 100644 --- a/input_raw.go +++ b/input_raw.go @@ -10,17 +10,21 @@ import ( // RAWInput used for intercepting traffic for given address type RAWInput struct { - data chan []byte + requests chan []byte + responses chan []byte address string expire time.Duration + captureResponse bool } // NewRAWInput constructor for RAWInput. Accepts address with port as argument. -func NewRAWInput(address string, expire time.Duration) (i *RAWInput) { +func NewRAWInput(address string, expire time.Duration, captureResponse bool) (i *RAWInput) { i = new(RAWInput) - i.data = make(chan []byte) + i.requests = make(chan []byte) + i.responses = make(chan []byte) i.address = address i.expire = expire + i.captureResponse = captureResponse go i.listen(address) @@ -28,18 +32,25 @@ func NewRAWInput(address string, expire time.Duration) (i *RAWInput) { } func (i *RAWInput) Read(data []byte) (int, error) { - buf := <-i.data - - if len(Settings.middleware) > 0 { - header := []byte("1\n") + select { + case buf := <- i.requests: + if i.captureResponse { + header := []byte("1\n") + copy(data[0:len(header)], header) + copy(data[len(header):], buf) + + return len(buf) + len(header), nil + } else { + copy(data, buf) + + return len(buf), nil + } + case buf := <- i.responses: + header := []byte("3\n") copy(data[0:len(header)], header) copy(data[len(header):], buf) return len(buf) + len(header), nil - } else { - copy(data, buf) - - return len(buf), nil } } @@ -54,13 +65,17 @@ func (i *RAWInput) listen(address string) { log.Fatal("input-raw: error while parsing address", err) } - listener := raw.NewListener(host, port, i.expire) + listener := raw.NewListener(host, port, i.expire, i.captureResponse) for { // Receiving TCPMessage object m := listener.Receive() - i.data <- m.Bytes() + i.requests <- m.RequestBytes() + + if i.captureResponse { + i.responses <- m.ResponseBytes() + } } } diff --git a/input_raw_test.go b/input_raw_test.go index a1458453..c8220299 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -24,7 +24,7 @@ func TestRAWInput(t *testing.T) { listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) - input := NewRAWInput(listener.Addr().String(), testRawExpire) + input := NewRAWInput(listener.Addr().String(), testRawExpire, false) output := NewTestOutput(func(data []byte) { wg.Done() }) @@ -66,7 +66,7 @@ func TestInputRAW100Expect(t *testing.T) { originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire) + input := NewRAWInput(originAddr, testRawExpire, false) // We will use it to get content of raw HTTP request testOutput := NewTestOutput(func(data []byte) { @@ -123,7 +123,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire) + input := NewRAWInput(originAddr, testRawExpire, false) listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { defer req.Body.Close() @@ -181,7 +181,7 @@ func TestInputRAWLargePayload(t *testing.T) { })) originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire) + input := NewRAWInput(originAddr, testRawExpire, false) replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024) @@ -214,3 +214,36 @@ func TestInputRAWLargePayload(t *testing.T) { wg.Wait() close(quit) } + +func TestInputRAWResponse(t *testing.T) { + wg := new(sync.WaitGroup) + quit := make(chan int) + + listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) + + input := NewRAWInput(listener.Addr().String(), testRawExpire, true) + output := NewTestOutput(func(data []byte) { + wg.Done() + }) + + Plugins.Inputs = []io.Reader{input} + Plugins.Outputs = []io.Writer{output} + + address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1) + + client := NewHTTPClient(address, &HTTPClientConfig{}) + + time.Sleep(time.Millisecond) + go Start(quit) + + for i := 0; i < 100; i++ { + // 2 because we track both request and response + wg.Add(2) + client.Get("/") + } + + wg.Wait() + close(quit) + + time.Sleep(100*time.Millisecond) +} \ No newline at end of file diff --git a/middleware_test.go b/middleware_test.go index 71748a87..e323fd96 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -105,7 +105,7 @@ func TestEchoMiddleware(t *testing.T) { quit := make(chan int) // Catch traffic from one service - input := NewRAWInput(from.Listener.Addr().String(), testRawExpire) + input := NewRAWInput(from.Listener.Addr().String(), testRawExpire, true) // And redirect to another output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true}) @@ -158,14 +158,14 @@ func TestTokenMiddleware(t *testing.T) { quit := make(chan int) // Catch traffic from one service - input := NewRAWInput(from, testRawExpire) + input := NewRAWInput(from, testRawExpire, true) // And redirect to another output := NewHTTPOutput(to, &HTTPOutputConfig{}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} - // Settings.middleware = "./examples/echo_modifier.sh" + Settings.middleware = "./examples/echo_modifier.sh" // Start Gor go Start(quit) diff --git a/plugins.go b/plugins.go index d0ba71ac..c92435c7 100644 --- a/plugins.go +++ b/plugins.go @@ -58,6 +58,7 @@ func registerPlugin(constructor interface{}, options ...interface{}) { _, isR := plugin.(io.Reader) _, isW := plugin.(io.Writer) + // Some of the output can be Readers as well because return responses if isR && !isW { Plugins.Inputs = append(Plugins.Inputs, pluginWrapper.(io.Reader)) } diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 53857ac9..ef8b2833 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -32,6 +32,8 @@ type Listener struct { // To get ACK of second message we need to compute its Seq and wait for them message seqWithData map[uint32]uint32 + respAliases map[uint32]uint32 + // Messages ready to be send to client packetsChan chan *TCPPacket @@ -42,14 +44,16 @@ type Listener struct { messageDelChan chan *TCPMessage addr string // IP to listen - port int // Port to listen + port uint16 // Port to listen messageExpire time.Duration + + captureResponse bool } // NewListener creates and initializes new Listener object -func NewListener(addr string, port string, expire time.Duration) (rawListener *Listener) { - rawListener = &Listener{} +func NewListener(addr string, port string, expire time.Duration, captureResponse bool) (rawListener *Listener) { + rawListener = &Listener{captureResponse: captureResponse} rawListener.packetsChan = make(chan *TCPPacket, 10000) rawListener.messagesChan = make(chan *TCPMessage, 10000) @@ -58,9 +62,11 @@ func NewListener(addr string, port string, expire time.Duration) (rawListener *L rawListener.messages = make(map[string]*TCPMessage) rawListener.ackAliases = make(map[uint32]uint32) rawListener.seqWithData = make(map[uint32]uint32) + rawListener.respAliases = make(map[uint32]uint32) rawListener.addr = addr - rawListener.port, _ = strconv.Atoi(port) + _port, _ := strconv.Atoi(port) + rawListener.port = uint16(_port) if expire.Nanoseconds() == 0 { expire = 2000 * time.Millisecond @@ -115,18 +121,19 @@ func (t *Listener) readRAWSocket() { } func (t *Listener) parsePacket(addr net.Addr, buf []byte) { - if t.isIncomingDataPacket(buf) { + if t.isValidPacket(buf) { t.packetsChan <- ParseTCPPacket(addr, buf) } } -func (t *Listener) isIncomingDataPacket(buf []byte) bool { +func (t *Listener) isValidPacket(buf []byte) bool { // To avoid full packet parsing every time, we manually parsing values needed for packet filtering // http://en.wikipedia.org/wiki/Transmission_Control_Protocol destPort := binary.BigEndian.Uint16(buf[2:4]) + srcPort := binary.BigEndian.Uint16(buf[0:2]) // Because RAW_SOCKET can't be bound to port, we have to control it by ourself - if int(destPort) == t.port { + if destPort == t.port || (t.captureResponse && srcPort == t.port) { // Get the 'data offset' (size of the TCP header in 32-bit words) dataOffset := (buf[12] & 0xF0) >> 4 @@ -161,12 +168,20 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { packet.Ack = alias } - mID := packet.Addr.String() + strconv.Itoa(int(packet.SrcPort)) + strconv.Itoa(int(packet.Ack)) + // if response + if t.captureResponse && packet.DestPort != t.port { + if alias, ok := t.respAliases[packet.Ack]; ok { + packet.Ack = alias + } + } + + + mID := packet.Addr.String() + strconv.Itoa(int(packet.Ack)) message, ok := t.messages[mID] if !ok { // We sending messageDelChan channel, so message object can communicate with Listener and notify it if message completed - message = NewTCPMessage(mID, t.messageDelChan, packet.Ack, &t.messageExpire) + message = NewTCPMessage(mID, t.messageDelChan, packet.Ack, &t.messageExpire, t.port) t.messages[mID] = message } @@ -179,6 +194,11 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { } } + if t.captureResponse { + // Response tracking + t.respAliases[packet.Seq+uint32(len(packet.Data))] = packet.Ack + } + // Adding packet to message message.packetsChan <- packet } diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index e6e31bda..5e0f8a65 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -15,6 +15,7 @@ import ( type TCPMessage struct { ID string // Message ID Ack uint32 + port uint16 packets []*TCPPacket timer *time.Timer // Used for expire check @@ -27,8 +28,8 @@ type TCPMessage struct { } // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted -func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32, expire *time.Duration) (msg *TCPMessage) { - msg = &TCPMessage{ID: ID, Ack: Ack, expire: expire} +func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32, expire *time.Duration, port uint16) (msg *TCPMessage) { + msg = &TCPMessage{ID: ID, Ack: Ack, expire: expire, port: port} msg.packetsChan = make(chan *TCPPacket) msg.delChan = delChan // used for notifying that message completed or expired @@ -73,11 +74,26 @@ func (t *TCPMessage) Timeout() { } // Bytes sorts packets in right orders and return message content -func (t *TCPMessage) Bytes() (output []byte) { +func (t *TCPMessage) RequestBytes() (output []byte) { sort.Sort(sortBySeq(t.packets)) for _, v := range t.packets { - output = append(output, v.Data...) + if v.DestPort == t.port { + output = append(output, v.Data...) + } + } + + return output +} + +// Bytes sorts packets in right orders and return message content +func (t *TCPMessage) ResponseBytes() (output []byte) { + sort.Sort(sortBySeq(t.packets)) + + for _, v := range t.packets { + if v.DestPort != t.port { + output = append(output, v.Data...) + } } return output diff --git a/raw_socket_listener/tcp_packet.go b/raw_socket_listener/tcp_packet.go index 3ebcf880..998eeb46 100644 --- a/raw_socket_listener/tcp_packet.go +++ b/raw_socket_listener/tcp_packet.go @@ -50,7 +50,6 @@ func ParseTCPPacket(addr net.Addr, b []byte) (p *TCPPacket) { // Parse TCP Packet, inspired by: https://github.com/miekg/pcap/blob/master/packet.go func (t *TCPPacket) Parse() { t.ParseBasic() - t.DestPort = binary.BigEndian.Uint16(t.Data[2:4]) t.Flags = binary.BigEndian.Uint16(t.Data[12:14]) & 0x1FF t.Window = binary.BigEndian.Uint16(t.Data[14:16]) t.Checksum = binary.BigEndian.Uint16(t.Data[16:18]) @@ -59,6 +58,7 @@ func (t *TCPPacket) Parse() { // ParseBasic set of fields func (t *TCPPacket) ParseBasic() { + t.DestPort = binary.BigEndian.Uint16(t.Data[2:4]) t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2]) t.Seq = binary.BigEndian.Uint32(t.Data[4:8]) t.Ack = binary.BigEndian.Uint32(t.Data[8:12]) From 491541c2df1d3f3518aeb618c122bf7a1b6e4434 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 13 Aug 2015 22:11:23 +0300 Subject: [PATCH 16/64] Response object should be separate message --- input_raw.go | 42 +++++++++++-------------- input_raw_test.go | 4 +-- raw_socket_listener/listener.go | 50 +++++++++++++++++++++--------- raw_socket_listener/tcp_message.go | 38 ++++++++++------------- 4 files changed, 73 insertions(+), 61 deletions(-) diff --git a/input_raw.go b/input_raw.go index 762655c0..5201f2d2 100644 --- a/input_raw.go +++ b/input_raw.go @@ -10,18 +10,16 @@ import ( // RAWInput used for intercepting traffic for given address type RAWInput struct { - requests chan []byte - responses chan []byte - address string - expire time.Duration + data chan *raw.TCPMessage + address string + expire time.Duration captureResponse bool } // NewRAWInput constructor for RAWInput. Accepts address with port as argument. func NewRAWInput(address string, expire time.Duration, captureResponse bool) (i *RAWInput) { i = new(RAWInput) - i.requests = make(chan []byte) - i.responses = make(chan []byte) + i.data = make(chan *raw.TCPMessage) i.address = address i.expire = expire i.captureResponse = captureResponse @@ -32,25 +30,25 @@ func NewRAWInput(address string, expire time.Duration, captureResponse bool) (i } func (i *RAWInput) Read(data []byte) (int, error) { - select { - case buf := <- i.requests: - if i.captureResponse { - header := []byte("1\n") - copy(data[0:len(header)], header) - copy(data[len(header):], buf) - - return len(buf) + len(header), nil - } else { - copy(data, buf) + msg := <-i.data + buf := msg.Bytes() + + if i.captureResponse { + var header []byte - return len(buf), nil + if msg.IsIncoming { + header = []byte("1\n") + } else { + header = []byte("3\n") } - case buf := <- i.responses: - header := []byte("3\n") + copy(data[0:len(header)], header) copy(data[len(header):], buf) return len(buf) + len(header), nil + } else { + copy(data, buf) + return len(buf), nil } } @@ -71,11 +69,7 @@ func (i *RAWInput) listen(address string) { // Receiving TCPMessage object m := listener.Receive() - i.requests <- m.RequestBytes() - - if i.captureResponse { - i.responses <- m.ResponseBytes() - } + i.data <- m } } diff --git a/input_raw_test.go b/input_raw_test.go index c8220299..6a2b1171 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -245,5 +245,5 @@ func TestInputRAWResponse(t *testing.T) { wg.Wait() close(quit) - time.Sleep(100*time.Millisecond) -} \ No newline at end of file + time.Sleep(100 * time.Millisecond) +} diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index ef8b2833..fa208d40 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -17,6 +17,7 @@ import ( "encoding/binary" "log" "net" + "runtime/debug" "strconv" "time" ) @@ -32,7 +33,7 @@ type Listener struct { // To get ACK of second message we need to compute its Seq and wait for them message seqWithData map[uint32]uint32 - respAliases map[uint32]uint32 + respAliases map[uint32]*request // Messages ready to be send to client packetsChan chan *TCPPacket @@ -44,13 +45,18 @@ type Listener struct { messageDelChan chan *TCPMessage addr string // IP to listen - port uint16 // Port to listen + port uint16 // Port to listen messageExpire time.Duration captureResponse bool } +type request struct { + start int64 + ack uint32 +} + // NewListener creates and initializes new Listener object func NewListener(addr string, port string, expire time.Duration, captureResponse bool) (rawListener *Listener) { rawListener = &Listener{captureResponse: captureResponse} @@ -62,7 +68,7 @@ func NewListener(addr string, port string, expire time.Duration, captureResponse rawListener.messages = make(map[string]*TCPMessage) rawListener.ackAliases = make(map[uint32]uint32) rawListener.seqWithData = make(map[uint32]uint32) - rawListener.respAliases = make(map[uint32]uint32) + rawListener.respAliases = make(map[uint32]*request) rawListener.addr = addr _port, _ := strconv.Atoi(port) @@ -155,10 +161,17 @@ var bPOST = []byte("POST") // // For TCP message unique id is Acknowledgment number (see tcp_packet.go) func (t *Listener) processTCPPacket(packet *TCPPacket) { - defer func() { recover() }() + // Don't exit on panic + defer func() { + if r := recover(); r != nil { + log.Println("PANIC: pkg:", r, string(debug.Stack())) + } + }() var message *TCPMessage + isIncoming := packet.DestPort == t.port + if parentAck, ok := t.seqWithData[packet.Seq]; ok { t.ackAliases[packet.Ack] = parentAck delete(t.seqWithData, packet.Seq) @@ -168,23 +181,27 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { packet.Ack = alias } - // if response - if t.captureResponse && packet.DestPort != t.port { - if alias, ok := t.respAliases[packet.Ack]; ok { - packet.Ack = alias - } + var responseRequest *request + + if t.captureResponse && !isIncoming { + responseRequest, _ = t.respAliases[packet.Ack] } + mID := packet.Addr.String() + strconv.Itoa(int(packet.DestPort)) + strconv.Itoa(int(packet.Ack)) - mID := packet.Addr.String() + strconv.Itoa(int(packet.Ack)) message, ok := t.messages[mID] if !ok { // We sending messageDelChan channel, so message object can communicate with Listener and notify it if message completed - message = NewTCPMessage(mID, t.messageDelChan, packet.Ack, &t.messageExpire, t.port) + message = NewTCPMessage(mID, t.messageDelChan, packet.Ack, &t.messageExpire, isIncoming) t.messages[mID] = message + + if !isIncoming && responseRequest != nil { + message.RequestStart = responseRequest.start + } } + // Handling Expect: 100-continue requests if bytes.Equal(packet.Data[0:4], bPOST) { if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) { t.seqWithData[packet.Seq+uint32(len(packet.Data))] = packet.Ack @@ -194,9 +211,14 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { } } - if t.captureResponse { - // Response tracking - t.respAliases[packet.Seq+uint32(len(packet.Data))] = packet.Ack + if t.captureResponse && isIncoming { + // If message have multiple packets, delete previous alias + if len(message.packets) > 0 { + delete(t.respAliases, uint32(message.Size())) + } + + responseAck := packet.Seq + uint32(message.Size()+len(packet.Data)) + t.respAliases[responseAck] = &request{message.Start, message.Ack} } // Adding packet to message diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 5e0f8a65..d06ad196 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -13,10 +13,12 @@ import ( // Message can be compiled from unique packets with same message_id which sorted by sequence // Message is received if we didn't receive any packets for 2000ms type TCPMessage struct { - ID string // Message ID - Ack uint32 - port uint16 - packets []*TCPPacket + ID string // Message ID + Ack uint32 + RequestStart int64 + Start int64 + IsIncoming bool + packets []*TCPPacket timer *time.Timer // Used for expire check @@ -28,9 +30,9 @@ type TCPMessage struct { } // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted -func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32, expire *time.Duration, port uint16) (msg *TCPMessage) { - msg = &TCPMessage{ID: ID, Ack: Ack, expire: expire, port: port} - +func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32, expire *time.Duration, IsIncoming bool) (msg *TCPMessage) { + msg = &TCPMessage{ID: ID, Ack: Ack, expire: expire, IsIncoming: IsIncoming} + msg.Start = time.Now().UnixNano() msg.packetsChan = make(chan *TCPPacket) msg.delChan = delChan // used for notifying that message completed or expired @@ -74,29 +76,23 @@ func (t *TCPMessage) Timeout() { } // Bytes sorts packets in right orders and return message content -func (t *TCPMessage) RequestBytes() (output []byte) { +func (t *TCPMessage) Bytes() (output []byte) { sort.Sort(sortBySeq(t.packets)) - for _, v := range t.packets { - if v.DestPort == t.port { - output = append(output, v.Data...) - } + for _, p := range t.packets { + output = append(output, p.Data...) } return output } -// Bytes sorts packets in right orders and return message content -func (t *TCPMessage) ResponseBytes() (output []byte) { - sort.Sort(sortBySeq(t.packets)) - - for _, v := range t.packets { - if v.DestPort != t.port { - output = append(output, v.Data...) - } +// Size returns total size of message +func (t *TCPMessage) Size() (size int) { + for _, p := range t.packets { + size += len(p.Data) } - return output + return } // AddPacket to the message and ensure packet uniqueness From bb319b5fd24bd147d28b1604ca78da04d7091527 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 14 Aug 2015 20:42:23 +0300 Subject: [PATCH 17/64] Add request ID and token modifier middleware --- examples/echo_modifier.sh | 4 +- examples/token_modifier.go | 97 ++++++++++++++++++++++++++++++ gor.go | 18 +++--- input_raw.go | 36 +++++++++-- middleware.go | 25 +++++--- middleware_test.go | 31 ++++++---- output_http.go | 29 ++++++--- raw_socket_listener/listener.go | 1 + raw_socket_listener/tcp_message.go | 22 +++++++ 9 files changed, 218 insertions(+), 45 deletions(-) create mode 100644 examples/token_modifier.go diff --git a/examples/echo_modifier.sh b/examples/echo_modifier.sh index 105b4bbb..13d20742 100755 --- a/examples/echo_modifier.sh +++ b/examples/echo_modifier.sh @@ -11,10 +11,10 @@ while read line; do >&2 echo "[DEBUG][ECHO] ===================================" case ${header:0:1} in - "3") + "2") >&2 echo "[DEBUG][ECHO] Request type: Original Response" ;; - "2") + "3") >&2 echo "[DEBUG][ECHO] Request type: Replayed Response" ;; "1") diff --git a/examples/token_modifier.go b/examples/token_modifier.go new file mode 100644 index 00000000..7ca37dd8 --- /dev/null +++ b/examples/token_modifier.go @@ -0,0 +1,97 @@ +package main + +import ( + "os" + "bufio" + "encoding/hex" + "github.com/buger/gor/proto" + "bytes" + "fmt" +) + +// requestID -> originalToken +var originalTokens map[string][]byte + +// originalToken -> replayedToken +var tokenAliases map[string][]byte + +func main() { + originalTokens = make(map[string][]byte) + tokenAliases = make(map[string][]byte) + + scanner := bufio.NewScanner(os.Stdin) + + for scanner.Scan() { + encoded := scanner.Bytes() + buf := make([]byte, len(encoded)/2) + hex.Decode(buf, encoded) + + go process(buf) + } +} + +func process(buf []byte) { + // First byte indicate payload type, possible values: + // 1 - Request + // 2 - Response + // 3 - ReplayedResponse + payloadType := buf[0] + headerSize := 42 + header := buf[:headerSize] + // For each request you should receive 3 payloads (request, response, replayed response) with same request id + reqID := string(header[2:headerSize]) + payload := buf[headerSize:] + + Debug("Received payload:", string(buf)) + + switch payloadType { + case '1': + if bytes.Equal(proto.Path(payload), []byte("/token")) { + originalTokens[reqID] = []byte{} + Debug("Found token request:", reqID) + } else { + tokenVal, vs, _ := proto.PathParam(payload, []byte("token")) + + if vs != -1 { // If there is GET token param + if alias, ok := tokenAliases[string(tokenVal)]; ok { + // Rewrite original token to alias + payload = proto.SetPathParam(payload, []byte("token"), alias) + + // Copy modified payload to our buffer + copy(buf[headerSize:], payload) + } + } + } + + // Re-compute length in case if payload was modified + bufLen := len(header) + len(payload) + // Encoding request and sending it back + dst := make([]byte, bufLen*2+1) + hex.Encode(dst, buf[:bufLen]) + dst[len(dst)-1] = '\n' + + os.Stdout.Write(dst) + + return + case '2': // Original response + if _, ok := originalTokens[reqID]; ok { + // Token is inside response body + secureToken := proto.Body(payload) + originalTokens[reqID] = secureToken + Debug("Remember origial token:", string(secureToken)) + } + case '3': // Replayed response + if originalToken, ok := originalTokens[reqID]; ok { + delete(originalTokens, reqID) + secureToken := proto.Body(payload) + tokenAliases[string(originalToken)] = secureToken + + Debug("Create alias for new token token, was:", string(originalToken), "now:", string(secureToken)) + } + } +} + +func Debug(args ...interface{}) { + fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ") + fmt.Fprintln(os.Stderr, args...) +} \ No newline at end of file diff --git a/gor.go b/gor.go index af9bf5f0..4cd85969 100644 --- a/gor.go +++ b/gor.go @@ -8,7 +8,7 @@ import ( "log" "os" "runtime" - "runtime/debug" + _ "runtime/debug" "runtime/pprof" "time" ) @@ -20,20 +20,18 @@ var ( ) func main() { + // // Don't exit on panic + // defer func() { + // if r := recover(); r != nil { + // fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack()) + // } + // }() + // If not set via env cariable if len(os.Getenv("GOMAXPROCS")) == 0 { runtime.GOMAXPROCS(runtime.NumCPU() * 2) } - // Don't exit on panic - defer func() { - if r := recover(); r != nil { - if _, ok := r.(error); !ok { - fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack()) - } - } - }() - fmt.Println("Version:", VERSION) flag.Parse() diff --git a/input_raw.go b/input_raw.go index 5201f2d2..9d786b31 100644 --- a/input_raw.go +++ b/input_raw.go @@ -29,19 +29,43 @@ func NewRAWInput(address string, expire time.Duration, captureResponse bool) (i return } +const ( + RequestPayload = 1 << iota + ResponsePayload + ReplayedResponsePayload +) + +func payloadHeader(payloadType int, uuid []byte) (header []byte) { + header = make([]byte, 43) + header[1] = ' ' + header[len(header)-1] = '\n' + + switch payloadType { + case RequestPayload: + header[0] = '1' + case ResponsePayload: + header[0] = '2' + case ReplayedResponsePayload: + header[0] = '3' + } + + copy(header[2:], uuid) + + return header +} + func (i *RAWInput) Read(data []byte) (int, error) { msg := <-i.data buf := msg.Bytes() if i.captureResponse { - var header []byte - - if msg.IsIncoming { - header = []byte("1\n") - } else { - header = []byte("3\n") + payloadType := RequestPayload + if !msg.IsIncoming { + payloadType = ResponsePayload } + header := payloadHeader(payloadType, msg.UUID()) + copy(data[0:len(header)], header) copy(data[len(header):], buf) diff --git a/middleware.go b/middleware.go index c642cd23..f1d072da 100644 --- a/middleware.go +++ b/middleware.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "strings" + "sync" ) type Middleware struct { @@ -16,6 +17,8 @@ type Middleware struct { data chan []byte + mu sync.Mutex + Stdin io.Writer Stdout io.Reader } @@ -48,6 +51,7 @@ func NewMiddleware(command string) *Middleware { } func (m *Middleware) ReadFrom(plugin io.Reader) { + Debug("[MIDDLEWARE-MASTER] Starting reading from", plugin) go m.copy(m.Stdin, plugin) } @@ -60,8 +64,11 @@ func (m *Middleware) copy(to io.Writer, from io.Reader) { if nr > 0 && len(buf) > nr { hex.Encode(dst, buf[0:nr]) - to.Write(dst[0 : nr*2]) - to.Write([]byte("\n")) + dst[nr*2] = '\n' + + m.mu.Lock() + to.Write(dst[0 : nr*2+1]) + m.mu.Unlock() if Settings.debug { Debug("[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from) @@ -71,19 +78,23 @@ func (m *Middleware) copy(to io.Writer, from io.Reader) { } func (m *Middleware) read(from io.Reader) { - buf := make([]byte, 5*1024*1024) - scanner := bufio.NewScanner(from) for scanner.Scan() { bytes := scanner.Bytes() - hex.Decode(buf, bytes) + buf := make([]byte, len(bytes)/2) + if _, err := hex.Decode(buf, bytes); err != nil { + fmt.Fprintln(os.Stderr, "Failed to decode input payload", err, len(bytes)) + } if Settings.debug { - Debug("[MIDDLEWARE-MASTER] Received:", string(buf[0:len(bytes)/2])) + Debug("[MIDDLEWARE-MASTER] Received:", string(buf)) } - m.data <- buf[0 : len(bytes)/2] + // We should accept only request payloads + if buf[0] == '1' { + m.data <- buf + } } if err := scanner.Err(); err != nil { diff --git a/middleware_test.go b/middleware_test.go index e323fd96..c691e58a 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -96,23 +96,28 @@ func TestEchoMiddleware(t *testing.T) { wg := new(sync.WaitGroup) from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Env", "prod") + w.Header().Set("RequestPath", r.URL.Path) wg.Done() })) to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Env", "test") + w.Header().Set("RequestPath", r.URL.Path) wg.Done() })) quit := make(chan int) + Settings.middleware = "./examples/echo_modifier.sh" + // Catch traffic from one service input := NewRAWInput(from.Listener.Addr().String(), testRawExpire, true) // And redirect to another - output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true}) + output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} - Settings.middleware = "./examples/echo_modifier.sh" // Start Gor go Start(quit) @@ -125,14 +130,14 @@ func TestEchoMiddleware(t *testing.T) { client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false}) // Request should be echoed - client.Get("/") - client.Get("/") + client.Get("/a") + client.Get("/b") wg.Wait() close(quit) - Settings.middleware = "" - time.Sleep(100 * time.Millisecond) + + Settings.middleware = "" } func TestTokenMiddleware(t *testing.T) { @@ -150,7 +155,7 @@ func TestTokenMiddleware(t *testing.T) { } case "/secure": if status != 202 { - // t.Error("Server should receive valid rewritten token") + t.Error("Server should receive valid rewritten token") } } }) @@ -161,26 +166,29 @@ func TestTokenMiddleware(t *testing.T) { input := NewRAWInput(from, testRawExpire, true) // And redirect to another - output := NewHTTPOutput(to, &HTTPOutputConfig{}) + output := NewHTTPOutput(to, &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} - Settings.middleware = "./examples/echo_modifier.sh" + Settings.middleware = "go run ./examples/token_modifier.go" // Start Gor go Start(quit) - time.Sleep(time.Millisecond) + // Wait for middleware to initialize + time.Sleep(500 * time.Millisecond) // Should receive 2 requests from original + 2 from replayed wg.Add(4) - client := NewHTTPClient("http://"+from, &HTTPClientConfig{Debug: true}) + client := NewHTTPClient("http://"+from, &HTTPClientConfig{Debug: false}) // Sending traffic to original service resp, _ = client.Get("/token") token = proto.Body(resp) + time.Sleep(50*time.Millisecond) + resp, _ = client.Get("/secure?token=" + string(token)) if !bytes.Equal(proto.Status(resp), []byte("202")) { t.Error("Valid token should return 202:", proto.Status(resp)) @@ -188,5 +196,6 @@ func TestTokenMiddleware(t *testing.T) { wg.Wait() close(quit) + time.Sleep(100 * time.Millisecond) Settings.middleware = "" } diff --git a/output_http.go b/output_http.go index 7be477f6..38c423f3 100644 --- a/output_http.go +++ b/output_http.go @@ -9,6 +9,11 @@ import ( const initialDynamicWorkers = 10 +type response struct { + payload []byte + uuid []byte +} + // HTTPOutputConfig struct for holding http output configuration type HTTPOutputConfig struct { redirectLimit int @@ -36,7 +41,7 @@ type HTTPOutput struct { address string limit int queue chan []byte - responses chan []byte + responses chan response needWorker chan int @@ -60,7 +65,7 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer { } o.queue = make(chan []byte, 100) - o.responses = make(chan []byte, 100) + o.responses = make(chan response, 100) o.needWorker = make(chan int, 1) // Initial workers count @@ -154,17 +159,23 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) { } func (o *HTTPOutput) Read(data []byte) (int, error) { - buf := <-o.responses - header := []byte("2\n") - copy(data[0:2], header) - copy(data[2:], buf) + resp := <-o.responses + + Debug("[OUTPUT-HTTP] Received response", string(resp.payload)) - return len(buf) + len(header), nil + header := payloadHeader(ReplayedResponsePayload, resp.uuid) + copy(data[0:len(header)], header) + copy(data[len(header):], resp.payload) + + return len(resp.payload) + len(header), nil } func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { + var uuid []byte + if len(Settings.middleware) > 0 { - request = request[2:] + uuid = request[2:42] + request = request[43:] } start := time.Now() @@ -176,7 +187,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { } if len(Settings.middleware) > 0 { - o.responses <- resp + o.responses <- response{resp, uuid} } if o.elasticSearch != nil { diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index fa208d40..3c7af465 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -198,6 +198,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { if !isIncoming && responseRequest != nil { message.RequestStart = responseRequest.start + message.RequestAck = responseRequest.ack } } diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index d06ad196..14e70d91 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -4,6 +4,9 @@ import ( "log" "sort" "time" + "crypto/sha1" + "encoding/hex" + "strconv" ) // TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence @@ -16,6 +19,7 @@ type TCPMessage struct { ID string // Message ID Ack uint32 RequestStart int64 + RequestAck uint32 Start int64 IsIncoming bool packets []*TCPPacket @@ -116,3 +120,21 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { // Reset message timeout timer t.timer.Reset(*t.expire) } + +func (t *TCPMessage) UUID() []byte { + var key []byte + + if t.IsIncoming { + key = strconv.AppendInt(key, t.Start, 10) + key = strconv.AppendUint(key, uint64(t.Ack), 10) + } else { + key = strconv.AppendInt(key, t.RequestStart, 10) + key = strconv.AppendUint(key, uint64(t.RequestAck), 10) + } + + uuid := make([]byte, 40) + sha := sha1.Sum(key) + hex.Encode(uuid, sha[:20]) + + return uuid +} From 70e7d8dd292839c43f4c120c58385ee2ae012d37 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 14 Aug 2015 23:07:46 +0300 Subject: [PATCH 18/64] Reduce delay of raw input --- examples/token_modifier.go | 152 ++++++++++++++--------------- input_raw_test.go | 2 +- middleware.go | 2 +- middleware_test.go | 2 +- output_http.go | 2 +- raw_socket_listener/listener.go | 1 + raw_socket_listener/tcp_message.go | 77 +++++++++++++-- 7 files changed, 149 insertions(+), 89 deletions(-) diff --git a/examples/token_modifier.go b/examples/token_modifier.go index 7ca37dd8..89ec995c 100644 --- a/examples/token_modifier.go +++ b/examples/token_modifier.go @@ -1,12 +1,12 @@ package main import ( - "os" - "bufio" - "encoding/hex" - "github.com/buger/gor/proto" - "bytes" - "fmt" + "bufio" + "bytes" + "encoding/hex" + "fmt" + "github.com/buger/gor/proto" + "os" ) // requestID -> originalToken @@ -16,82 +16,82 @@ var originalTokens map[string][]byte var tokenAliases map[string][]byte func main() { - originalTokens = make(map[string][]byte) - tokenAliases = make(map[string][]byte) + originalTokens = make(map[string][]byte) + tokenAliases = make(map[string][]byte) - scanner := bufio.NewScanner(os.Stdin) + scanner := bufio.NewScanner(os.Stdin) - for scanner.Scan() { - encoded := scanner.Bytes() - buf := make([]byte, len(encoded)/2) - hex.Decode(buf, encoded) + for scanner.Scan() { + encoded := scanner.Bytes() + buf := make([]byte, len(encoded)/2) + hex.Decode(buf, encoded) - go process(buf) - } + go process(buf) + } } func process(buf []byte) { - // First byte indicate payload type, possible values: - // 1 - Request - // 2 - Response - // 3 - ReplayedResponse - payloadType := buf[0] - headerSize := 42 - header := buf[:headerSize] - // For each request you should receive 3 payloads (request, response, replayed response) with same request id - reqID := string(header[2:headerSize]) - payload := buf[headerSize:] - - Debug("Received payload:", string(buf)) - - switch payloadType { - case '1': - if bytes.Equal(proto.Path(payload), []byte("/token")) { - originalTokens[reqID] = []byte{} - Debug("Found token request:", reqID) - } else { - tokenVal, vs, _ := proto.PathParam(payload, []byte("token")) - - if vs != -1 { // If there is GET token param - if alias, ok := tokenAliases[string(tokenVal)]; ok { - // Rewrite original token to alias - payload = proto.SetPathParam(payload, []byte("token"), alias) - - // Copy modified payload to our buffer - copy(buf[headerSize:], payload) - } - } - } - - // Re-compute length in case if payload was modified - bufLen := len(header) + len(payload) - // Encoding request and sending it back - dst := make([]byte, bufLen*2+1) - hex.Encode(dst, buf[:bufLen]) - dst[len(dst)-1] = '\n' - - os.Stdout.Write(dst) - - return - case '2': // Original response - if _, ok := originalTokens[reqID]; ok { - // Token is inside response body - secureToken := proto.Body(payload) - originalTokens[reqID] = secureToken - Debug("Remember origial token:", string(secureToken)) - } - case '3': // Replayed response - if originalToken, ok := originalTokens[reqID]; ok { - delete(originalTokens, reqID) - secureToken := proto.Body(payload) - tokenAliases[string(originalToken)] = secureToken - - Debug("Create alias for new token token, was:", string(originalToken), "now:", string(secureToken)) - } - } + // First byte indicate payload type, possible values: + // 1 - Request + // 2 - Response + // 3 - ReplayedResponse + payloadType := buf[0] + headerSize := 42 + header := buf[:headerSize] + // For each request you should receive 3 payloads (request, response, replayed response) with same request id + reqID := string(header[2:headerSize]) + payload := buf[headerSize:] + + Debug("Received payload:", string(buf)) + + switch payloadType { + case '1': + if bytes.Equal(proto.Path(payload), []byte("/token")) { + originalTokens[reqID] = []byte{} + Debug("Found token request:", reqID) + } else { + tokenVal, vs, _ := proto.PathParam(payload, []byte("token")) + + if vs != -1 { // If there is GET token param + if alias, ok := tokenAliases[string(tokenVal)]; ok { + // Rewrite original token to alias + payload = proto.SetPathParam(payload, []byte("token"), alias) + + // Copy modified payload to our buffer + copy(buf[headerSize:], payload) + } + } + } + + // Re-compute length in case if payload was modified + bufLen := len(header) + len(payload) + // Encoding request and sending it back + dst := make([]byte, bufLen*2+1) + hex.Encode(dst, buf[:bufLen]) + dst[len(dst)-1] = '\n' + + os.Stdout.Write(dst) + + return + case '2': // Original response + if _, ok := originalTokens[reqID]; ok { + // Token is inside response body + secureToken := proto.Body(payload) + originalTokens[reqID] = secureToken + Debug("Remember origial token:", string(secureToken)) + } + case '3': // Replayed response + if originalToken, ok := originalTokens[reqID]; ok { + delete(originalTokens, reqID) + secureToken := proto.Body(payload) + tokenAliases[string(originalToken)] = secureToken + + Debug("Create alias for new token token, was:", string(originalToken), "now:", string(secureToken)) + } + } } func Debug(args ...interface{}) { - fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ") - fmt.Fprintln(os.Stderr, args...) -} \ No newline at end of file + fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ") + fmt.Fprintln(os.Stderr, args...) +} diff --git a/input_raw_test.go b/input_raw_test.go index 6a2b1171..3276e7a3 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -138,7 +138,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { }) replayAddr := listener.Addr().String() - httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{Debug: true}) + httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{Debug: false}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{httpOutput} diff --git a/middleware.go b/middleware.go index f1d072da..1057173c 100644 --- a/middleware.go +++ b/middleware.go @@ -17,7 +17,7 @@ type Middleware struct { data chan []byte - mu sync.Mutex + mu sync.Mutex Stdin io.Writer Stdout io.Reader diff --git a/middleware_test.go b/middleware_test.go index c691e58a..f80e3303 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -187,7 +187,7 @@ func TestTokenMiddleware(t *testing.T) { resp, _ = client.Get("/token") token = proto.Body(resp) - time.Sleep(50*time.Millisecond) + time.Sleep(10 * time.Millisecond) resp, _ = client.Get("/secure?token=" + string(token)) if !bytes.Equal(proto.Status(resp), []byte("202")) { diff --git a/output_http.go b/output_http.go index 38c423f3..c3e04508 100644 --- a/output_http.go +++ b/output_http.go @@ -11,7 +11,7 @@ const initialDynamicWorkers = 10 type response struct { payload []byte - uuid []byte + uuid []byte } // HTTPOutputConfig struct for holding http output configuration diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 3c7af465..eee05f44 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -204,6 +204,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { // Handling Expect: 100-continue requests if bytes.Equal(packet.Data[0:4], bPOST) { + // reading last 20 bytes (not counting CRLF): last header value (if no body presented) if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) { t.seqWithData[packet.Seq+uint32(len(packet.Data))] = packet.Ack diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 14e70d91..bba9d292 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -1,12 +1,14 @@ package rawSocket import ( - "log" - "sort" - "time" + "bytes" "crypto/sha1" "encoding/hex" + "github.com/buger/gor/proto" + "log" + "sort" "strconv" + "time" ) // TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence @@ -22,7 +24,8 @@ type TCPMessage struct { RequestAck uint32 Start int64 IsIncoming bool - packets []*TCPPacket + + packets []*TCPPacket timer *time.Timer // Used for expire check @@ -40,9 +43,6 @@ func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32, expire *time msg.packetsChan = make(chan *TCPPacket) msg.delChan = delChan // used for notifying that message completed or expired - // Every time we receive packet we reset this timer - msg.timer = time.AfterFunc(*msg.expire, msg.Timeout) - go msg.listen() return @@ -64,6 +64,10 @@ func (t *TCPMessage) listen() { // Timeout notifies message to stop listening, close channel and message ready to be sent func (t *TCPMessage) Timeout() { + if t.timer != nil { + t.timer.Stop() + } + select { // In some cases Timeout can be called multiple times (do not know how yet) // Ensure that we did not close channel 2 times @@ -117,8 +121,63 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { t.packets = append(t.packets, packet) } - // Reset message timeout timer - t.timer.Reset(*t.expire) + if !t.isMultipart() { + log.Println("MESSAGE NOT MULTIPART", string(packet.Data)) + t.Timeout() + } else { + log.Println("MESSAGE MULTIPART", string(packet.Data)) + // If more then 1 packet, wait for more, and set expiration + if len(t.packets) == 1 { + // Every time we receive packet we reset this timer + t.timer = time.AfterFunc(*t.expire, t.Timeout) + } else { + // Reset message timeout timer + t.timer.Reset(*t.expire) + } + } +} + +// isMultipart returns true if message contains from multiple tcp packets +func (t *TCPMessage) isMultipart() bool { + if len(t.packets) > 1 { + return true + } + + payload := t.packets[0].Data + m := payload[:3] + + if t.IsIncoming { + // If one GET, OPTIONS, or HEAD request + if bytes.Equal(m, []byte("GET")) || bytes.Equal(m, []byte("OPT")) || bytes.Equal(m, []byte("HEA")) { + return false + } else { + if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 { + l, _ := strconv.Atoi(string(length)) + + log.Println("Content-Length", l, "Body length:", len(proto.Body(payload))) + // If content-length equal current body length + if l > 0 && l == len(proto.Body(payload)) { + return false + } + } + } + } else { + if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 { + if length[0] == '0' { + return false + } + + l, _ := strconv.Atoi(string(length)) + + log.Println("Content-Length", l, "Body length:", len(proto.Body(payload))) + // If content-length equal current body length + if l > 0 && l == len(proto.Body(payload)) { + return false + } + } + } + + return true } func (t *TCPMessage) UUID() []byte { From 56f0a3fcc0a9c564d3aa29d84b88f2bab01220c8 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 07:57:25 +0300 Subject: [PATCH 19/64] Remove too verbose output --- raw_socket_listener/tcp_message.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index bba9d292..4ff9975a 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -154,7 +154,6 @@ func (t *TCPMessage) isMultipart() bool { if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 { l, _ := strconv.Atoi(string(length)) - log.Println("Content-Length", l, "Body length:", len(proto.Body(payload))) // If content-length equal current body length if l > 0 && l == len(proto.Body(payload)) { return false @@ -169,7 +168,6 @@ func (t *TCPMessage) isMultipart() bool { l, _ := strconv.Atoi(string(length)) - log.Println("Content-Length", l, "Body length:", len(proto.Body(payload))) // If content-length equal current body length if l > 0 && l == len(proto.Body(payload)) { return false From 4f2341b2d3c2c4885def60067574c9fb26509cbb Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 08:06:13 +0300 Subject: [PATCH 20/64] Remove more debug --- http_client.go | 2 -- raw_socket_listener/tcp_message.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/http_client.go b/http_client.go index 37b2691d..e3d6c1e2 100644 --- a/http_client.go +++ b/http_client.go @@ -165,8 +165,6 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { payload := c.respBuf[:n] - Debug("[HTTPClient] Received:", n) - if c.config.Debug { Debug("[HTTPClient] Received:", string(payload)) } diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 4ff9975a..359e62e6 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -122,10 +122,8 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { } if !t.isMultipart() { - log.Println("MESSAGE NOT MULTIPART", string(packet.Data)) t.Timeout() } else { - log.Println("MESSAGE MULTIPART", string(packet.Data)) // If more then 1 packet, wait for more, and set expiration if len(t.packets) == 1 { // Every time we receive packet we reset this timer From 41c1421a1b4ad92557510381073263514529b3ff Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 08:36:13 +0300 Subject: [PATCH 21/64] Simulate some real-life server by adding small response delay --- middleware_test.go | 3 +++ settings.go | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/middleware_test.go b/middleware_test.go index f80e3303..b99ccae3 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -146,6 +146,7 @@ func TestTokenMiddleware(t *testing.T) { wg := new(sync.WaitGroup) from := NewFakeSecureService(wg, func(path string, status int, tok []byte) { + time.Sleep(10 * time.Millisecond) }) to := NewFakeSecureService(wg, func(path string, status int, tok []byte) { switch path { @@ -158,6 +159,8 @@ func TestTokenMiddleware(t *testing.T) { t.Error("Server should receive valid rewritten token") } } + + time.Sleep(10 * time.Millisecond) }) quit := make(chan int) diff --git a/settings.go b/settings.go index 528adcd5..37ac211f 100644 --- a/settings.go +++ b/settings.go @@ -3,8 +3,9 @@ package main import ( "flag" "fmt" - "log" "os" + "sync" + "time" ) var VERSION string @@ -122,10 +123,19 @@ func init() { flag.Var(&Settings.modifierConfig.paramHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%") } +var previousDebugTime int64 +var debugMutex sync.Mutex + // Debug gets called only if --verbose flag specified func Debug(args ...interface{}) { if Settings.verbose { - fmt.Printf("[DEBUG][PID %d] ", os.Getpid()) - log.Println(args...) + debugMutex.Lock() + now := time.Now() + diff := float64(now.UnixNano()-previousDebugTime) / 1000000 + previousDebugTime = now.UnixNano() + debugMutex.Unlock() + + fmt.Printf("[DEBUG][PID %d][%d][%fms] ", os.Getpid(), now.UnixNano(), diff) + fmt.Println(args...) } } From 1a36374c2dc66fe92afa1304bb7d2656b5e38b07 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 08:44:27 +0300 Subject: [PATCH 22/64] Add some comments --- middleware_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/middleware_test.go b/middleware_test.go index b99ccae3..6dd5b292 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -190,6 +190,8 @@ func TestTokenMiddleware(t *testing.T) { resp, _ = client.Get("/token") token = proto.Body(resp) + // When delay is too smal, middleware does not always rewrite requests in time + // Hopefuly client will have delay more then 10ms :) time.Sleep(10 * time.Millisecond) resp, _ = client.Get("/secure?token=" + string(token)) From a18079fa840112678ac1e3a1269871aa7bdfbdbf Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 10:33:50 +0300 Subject: [PATCH 23/64] Improving docs --- README.md | 23 +++++++++++- examples/echo_modifier.sh | 32 ---------------- examples/{ => middleware}/echo_modifier.rb | 0 examples/middleware/echo_modifier.sh | 41 +++++++++++++++++++++ examples/{ => middleware}/token_modifier.go | 2 +- input_raw_test.go | 2 - middleware_test.go | 5 ++- 7 files changed, 66 insertions(+), 39 deletions(-) delete mode 100755 examples/echo_modifier.sh rename examples/{ => middleware}/echo_modifier.rb (100%) create mode 100755 examples/middleware/echo_modifier.sh rename examples/{ => middleware}/token_modifier.go (99%) diff --git a/README.md b/README.md index 2ea19c35..67f02e69 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ gor --input-tcp :28020 --output-http "http://staging.com" --output-http "http:/ ``` ### HTTP output workers -By default Gor creates dynamic pull of workers: it starts with 10 and create more http output workers when the http output queue length is greater than 10. The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue. No more workers will be spawned until that request to spawn N workers is satisfied. If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies. +By default Gor creates dynamic pull of workers: it starts with 10 and create more http output workers when the http output queue length is greater than 10. The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue. No more workers will be spawned until that request to spawn N workers is satisfied. If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies. You may specify fixed number of workers using `--output-http-workers=20` option. ### Follow redirects @@ -149,7 +149,7 @@ gor --input-raw :80 --output-http "http://staging.server" \ ``` ### Rewriting original request -Gor supports built-in basic rewriting support, for complex logic see https://github.com/buger/gor/pull/162 +Gor supports some basic request rewriting support. For complex logic you can use middleware, see below. #### Rewrite URL based on a mapping ``` @@ -177,6 +177,25 @@ Host header gets special treatment. By default Host get set to the value specifi If you app accepts traffic from multiple domain, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all. +### Middleware +Middleware is a programm that accepts request payload and response at STDIN and emits modified requests at STDOUT. + +``` + Original request +--------------+ ++-------------+----------STDIN---------->+ | +| Gor input | | Middleware | ++-------------+----------STDIN---------->+ | + Original response +------+---+---+ + | ^ ++-------------+ Modified request v | +| Gor output +<---------STDOUT-----------------+ | ++-----+-------+ | + | | + | Replayed response | + +------------------STDIN----------------->----+ +``` + + ### Saving requests to file and replaying them You can save requests to file, and replay them later: ``` diff --git a/examples/echo_modifier.sh b/examples/echo_modifier.sh deleted file mode 100755 index 13d20742..00000000 --- a/examples/echo_modifier.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -while read line; do - decoded=$(echo -e "$line" | xxd -r -p) - - header=$(echo -e "$decoded" | head -n +1) - payload=$(echo -e "$decoded" | tail -n +2) - - encoded=$(echo -e "$header\n$payload" | xxd -p | tr -d "\\n") - - >&2 echo "" - >&2 echo "[DEBUG][ECHO] ===================================" - - case ${header:0:1} in - "2") - >&2 echo "[DEBUG][ECHO] Request type: Original Response" - ;; - "3") - >&2 echo "[DEBUG][ECHO] Request type: Replayed Response" - ;; - "1") - >&2 echo "[DEBUG][ECHO] Request type: Request" - echo "$encoded" - ;; - *) - >&2 echo "[DEBUG][ECHO] Unknown request type $header" - esac - >&2 echo "[DEBUG][ECHO] ===================================" - - >&2 echo "[DEBUG][ECHO] Original data: $line" - >&2 echo "[DEBUG][ECHO] Decoded request: $decoded" - >&2 echo "[DEBUG][ECHO] Encoded data: $encoded" -done; diff --git a/examples/echo_modifier.rb b/examples/middleware/echo_modifier.rb similarity index 100% rename from examples/echo_modifier.rb rename to examples/middleware/echo_modifier.rb diff --git a/examples/middleware/echo_modifier.sh b/examples/middleware/echo_modifier.sh new file mode 100755 index 00000000..96dc1217 --- /dev/null +++ b/examples/middleware/echo_modifier.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# `xxd` utility included into vim-common package +# It allow hex decoding/encoding + +function log { + # Logging to stderr, because stdout/stdin used for data transfer + >&2 echo "[DEBUG][ECHO] $1" +} + +while read line; do + decoded=$(echo -e "$line" | xxd -r -p) + + header=$(echo -e "$decoded" | head -n +1) + payload=$(echo -e "$decoded" | tail -n +2) + + encoded=$(echo -e "$header\n$payload" | xxd -p | tr -d "\\n") + + log "" + log "===================================" + + case ${header:0:1} in + "1") + log "Request type: Request" + echo "$encoded" + ;; + "2") + log "Request type: Original Response" + ;; + "3") + log "Request type: Replayed Response" + ;; + *) + log "Unknown request type $header" + esac + log "===================================" + + log "Original data: $line" + log "Decoded request: $decoded" + log "Encoded data: $encoded" +done; diff --git a/examples/token_modifier.go b/examples/middleware/token_modifier.go similarity index 99% rename from examples/token_modifier.go rename to examples/middleware/token_modifier.go index 89ec995c..1245c5fb 100644 --- a/examples/token_modifier.go +++ b/examples/middleware/token_modifier.go @@ -45,7 +45,7 @@ func process(buf []byte) { Debug("Received payload:", string(buf)) switch payloadType { - case '1': + case '1': // Request if bytes.Equal(proto.Path(payload), []byte("/token")) { originalTokens[reqID] = []byte{} Debug("Found token request:", reqID) diff --git a/input_raw_test.go b/input_raw_test.go index 3276e7a3..32730356 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -244,6 +244,4 @@ func TestInputRAWResponse(t *testing.T) { wg.Wait() close(quit) - - time.Sleep(100 * time.Millisecond) } diff --git a/middleware_test.go b/middleware_test.go index 6dd5b292..25d33749 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -108,7 +108,7 @@ func TestEchoMiddleware(t *testing.T) { quit := make(chan int) - Settings.middleware = "./examples/echo_modifier.sh" + Settings.middleware = "./examples/middleware/echo_modifier.sh" // Catch traffic from one service input := NewRAWInput(from.Listener.Addr().String(), testRawExpire, true) @@ -173,12 +173,13 @@ func TestTokenMiddleware(t *testing.T) { Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} - Settings.middleware = "go run ./examples/token_modifier.go" + Settings.middleware = "go run ./examples/middleware/token_modifier.go" // Start Gor go Start(quit) // Wait for middleware to initialize + // Give go compiller time to build programm time.Sleep(500 * time.Millisecond) // Should receive 2 requests from original + 2 from replayed From ddd519062213863878e987082df92ddb8d537c23 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 21:05:10 +0300 Subject: [PATCH 24/64] Improve docs --- README.md | 25 ++++++++++++++++++++++++- examples/middleware/token_modifier.go | 23 +++++++++++++++++++++-- middleware.go | 5 ++++- middleware_test.go | 1 - 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 67f02e69..5ccad357 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ Host header gets special treatment. By default Host get set to the value specifi If you app accepts traffic from multiple domain, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all. ### Middleware -Middleware is a programm that accepts request payload and response at STDIN and emits modified requests at STDOUT. +Middleware is a programm that accepts request and response payload at STDIN and emits modified requests at STDOUT. You can implement any custom logic like stripping private data, advanced rewriting, support for oAuth and etc. ``` Original request +--------------+ @@ -195,6 +195,29 @@ Middleware is a programm that accepts request payload and response at STDIN and +------------------STDIN----------------->----+ ``` +Middleware can be written in any language, see `examples/middleware` folder for examples. +Middleware programm should accept the fact that all communication with Gor is asyncronious, there is no guarantee that original request and response messages will come one after each other. Your app should take care of the state if logic depends on original or replayed response, see `examples/middleware/token_modifier.go` as example. + +#### Communication protocol +All messages should be hex encoded, new line character specifieds the end of the message, eg. new message per line. + +Decoded payload consist of 2 parts: header and HTTP payload, separated by new line character. Example: +``` +1 932079936fa4306fc308d67588178d17d823647c +GET /a HTTP/1.1 +Host: 127.0.0.1 + +``` + +First header byte `1` represent payload type, possible values: `1` - request, `2` - original response, `3` - replayed response +After empty spaces, goes request id `932079936fa4306fc308d67588178d17d823647c`, which is always 40 characters length. Request id unique among all requests (sha1 of time and Ack), but remain same for original and replayed response, so you can create associations between request and responses. + +HTTP payload is unmodified HTTP requests/responses intercepted from network. You can read more about request format [here](http://www.jmarshall.com/easy/http/), [here](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) and [here](http://www.w3.org/Protocols/rfc2616/rfc2616.html). You can operate with payload as you want, add headers, change path, and etc. Basically you just editing a string, just ensure that it is RCF compliant. + +At the end modified (or untouched) request should be emitted back to STDOUT, keeping original header, and hex-encoded. If you want to filter request, just not send it. Emitting responses back is optional, and does not affect anyting at the moment. + +#### Advanced example +Imagine that you have auth system that randomly generate access tokens, which used later for accessing secure content. Since there is no pre-defined token value, naive approach without middleware (or if middleware use only request payloads) will fail, because replayed server have own tokens, not synced with origin. To fix this, our middleware should take in account responses of replayed and origin server, store `originalToken -> replayedToken` aliases and rewrite all requests using this token to use replayed alias. See `examples/middleware/token_modifier.go` and `middleware_test.go#TestTokenMiddleware` as example of described scheme. ### Saving requests to file and replaying them You can save requests to file, and replay them later: diff --git a/examples/middleware/token_modifier.go b/examples/middleware/token_modifier.go index 1245c5fb..3b871e2d 100644 --- a/examples/middleware/token_modifier.go +++ b/examples/middleware/token_modifier.go @@ -1,3 +1,22 @@ +/* +This middleware made for auth system that randomly generate access tokens, which used later for accessing secure content. Since there is no pre-defined token value, naive approach without middleware (or if middleware use only request payloads) will fail, because replayed server have own tokens, not synced with origin. To fix this, our middleware should take in account responses of replayed and origin server, store `originalToken -> replayedToken` aliases and rewrite all requests using this token to use replayed alias. See `middleware_test.go#TestTokenMiddleware` test for examples of using this middleware. + +How middleware works: + + Original request +--------------+ ++-------------+----------STDIN---------->+ | +| Gor input | | Middleware | ++-------------+----------STDIN---------->+ | + Original response +------+---+---+ + | ^ ++-------------+ Modified request v | +| Gor output +<---------STDOUT-----------------+ | ++-----+-------+ | + | | + | Replayed response | + +------------------STDIN----------------->----+ +*/ + package main import ( @@ -50,10 +69,10 @@ func process(buf []byte) { originalTokens[reqID] = []byte{} Debug("Found token request:", reqID) } else { - tokenVal, vs, _ := proto.PathParam(payload, []byte("token")) + token, vs, _ := proto.PathParam(payload, []byte("token")) if vs != -1 { // If there is GET token param - if alias, ok := tokenAliases[string(tokenVal)]; ok { + if alias, ok := tokenAliases[string(token)]; ok { // Rewrite original token to alias payload = proto.SetPathParam(payload, []byte("token"), alias) diff --git a/middleware.go b/middleware.go index 1057173c..cb4ea123 100644 --- a/middleware.go +++ b/middleware.go @@ -33,7 +33,10 @@ func NewMiddleware(command string) *Middleware { m.Stdout, _ = cmd.StdoutPipe() m.Stdin, _ = cmd.StdinPipe() - cmd.Stderr = os.Stderr + + if Settings.verbose { + cmd.Stderr = os.Stderr + } go m.read(m.Stdout) diff --git a/middleware_test.go b/middleware_test.go index 25d33749..05bcb4aa 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -68,7 +68,6 @@ func TestFakeSecureService(t *testing.T) { wg := new(sync.WaitGroup) addr := NewFakeSecureService(wg, func(path string, status int, resp []byte) { - }) wg.Add(3) From 879ed48b80a4a9ac8af2f24b6c0fafae9af7c42b Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 21:05:51 +0300 Subject: [PATCH 25/64] Doc changes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ccad357..33d68a43 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ Host: 127.0.0.1 ``` First header byte `1` represent payload type, possible values: `1` - request, `2` - original response, `3` - replayed response -After empty spaces, goes request id `932079936fa4306fc308d67588178d17d823647c`, which is always 40 characters length. Request id unique among all requests (sha1 of time and Ack), but remain same for original and replayed response, so you can create associations between request and responses. +After empty spaces, goes request id `932079936fa4306fc308d67588178d17d823647c`. Request id unique among all requests (sha1 of time and Ack), but remain same for original and replayed response, so you can create associations between request and responses. HTTP payload is unmodified HTTP requests/responses intercepted from network. You can read more about request format [here](http://www.jmarshall.com/easy/http/), [here](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) and [here](http://www.w3.org/Protocols/rfc2616/rfc2616.html). You can operate with payload as you want, add headers, change path, and etc. Basically you just editing a string, just ensure that it is RCF compliant. From 0d445722b90a8419af8dab4828a8ca4a08c6639b Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 21:27:20 +0300 Subject: [PATCH 26/64] Some fixes --- Makefile | 4 ++-- README.md | 18 +++++++++++++++--- input_raw.go | 25 ------------------------- payload_format.go | 26 ++++++++++++++++++++++++++ plugins.go | 8 +++++++- 5 files changed, 50 insertions(+), 31 deletions(-) create mode 100644 payload_format.go diff --git a/Makefile b/Makefile index 51ebf177..6f43cab8 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go traffic_modifier.go +SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go payload_format.go SOURCE_PATH = /gopath/src/github.com/buger/gor/ @@ -37,7 +37,7 @@ dbench: # Used mainly for debugging, because docker container do not have access to parent machine ports drun: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --verbose + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-http :9000 --verbose drecord: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-file=requests.bin --verbose diff --git a/README.md b/README.md index 33d68a43..8e5dbf0d 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ Host header gets special treatment. By default Host get set to the value specifi If you app accepts traffic from multiple domain, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all. ### Middleware -Middleware is a programm that accepts request and response payload at STDIN and emits modified requests at STDOUT. You can implement any custom logic like stripping private data, advanced rewriting, support for oAuth and etc. +Middleware is a program that accepts request and response payload at STDIN and emits modified requests at STDOUT. You can implement any custom logic like stripping private data, advanced rewriting, support for oAuth and etc. ``` Original request +--------------+ @@ -196,7 +196,19 @@ Middleware is a programm that accepts request and response payload at STDIN and ``` Middleware can be written in any language, see `examples/middleware` folder for examples. -Middleware programm should accept the fact that all communication with Gor is asyncronious, there is no guarantee that original request and response messages will come one after each other. Your app should take care of the state if logic depends on original or replayed response, see `examples/middleware/token_modifier.go` as example. +Middleware program should accept the fact that all communication with Gor is asynchronous, there is no guarantee that original request and response messages will come one after each other. Your app should take care of the state if logic depends on original or replayed response, see `examples/middleware/token_modifier.go` as example. + +Simple bash echo middleware (returns same request) will look like this: +```bash +while read line; do + echo $line +end +``` + +Middleware can be enabled using `--middleware` option, by specifying path to executable file: +``` +gor --input-raw :80 --middleware "/opt/middleware_executable" --output-http "http://staging.server" +``` #### Communication protocol All messages should be hex encoded, new line character specifieds the end of the message, eg. new message per line. @@ -214,7 +226,7 @@ After empty spaces, goes request id `932079936fa4306fc308d67588178d17d823647c`. HTTP payload is unmodified HTTP requests/responses intercepted from network. You can read more about request format [here](http://www.jmarshall.com/easy/http/), [here](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) and [here](http://www.w3.org/Protocols/rfc2616/rfc2616.html). You can operate with payload as you want, add headers, change path, and etc. Basically you just editing a string, just ensure that it is RCF compliant. -At the end modified (or untouched) request should be emitted back to STDOUT, keeping original header, and hex-encoded. If you want to filter request, just not send it. Emitting responses back is optional, and does not affect anyting at the moment. +At the end modified (or untouched) request should be emitted back to STDOUT, keeping original header, and hex-encoded. If you want to filter request, just not send it. Emitting responses back is optional, and does not affect anything at the moment. #### Advanced example Imagine that you have auth system that randomly generate access tokens, which used later for accessing secure content. Since there is no pre-defined token value, naive approach without middleware (or if middleware use only request payloads) will fail, because replayed server have own tokens, not synced with origin. To fix this, our middleware should take in account responses of replayed and origin server, store `originalToken -> replayedToken` aliases and rewrite all requests using this token to use replayed alias. See `examples/middleware/token_modifier.go` and `middleware_test.go#TestTokenMiddleware` as example of described scheme. diff --git a/input_raw.go b/input_raw.go index 9d786b31..cfdfc8b4 100644 --- a/input_raw.go +++ b/input_raw.go @@ -29,31 +29,6 @@ func NewRAWInput(address string, expire time.Duration, captureResponse bool) (i return } -const ( - RequestPayload = 1 << iota - ResponsePayload - ReplayedResponsePayload -) - -func payloadHeader(payloadType int, uuid []byte) (header []byte) { - header = make([]byte, 43) - header[1] = ' ' - header[len(header)-1] = '\n' - - switch payloadType { - case RequestPayload: - header[0] = '1' - case ResponsePayload: - header[0] = '2' - case ReplayedResponsePayload: - header[0] = '3' - } - - copy(header[2:], uuid) - - return header -} - func (i *RAWInput) Read(data []byte) (int, error) { msg := <-i.data buf := msg.Bytes() diff --git a/payload_format.go b/payload_format.go new file mode 100644 index 00000000..f26b1031 --- /dev/null +++ b/payload_format.go @@ -0,0 +1,26 @@ +package main + +const ( + RequestPayload = 1 << iota + ResponsePayload + ReplayedResponsePayload +) + +func payloadHeader(payloadType int, uuid []byte) (header []byte) { + header = make([]byte, 43) + header[1] = ' ' + header[len(header)-1] = '\n' + + switch payloadType { + case RequestPayload: + header[0] = '1' + case ResponsePayload: + header[0] = '2' + case ReplayedResponsePayload: + header[0] = '3' + } + + copy(header[2:], uuid) + + return header +} diff --git a/plugins.go b/plugins.go index c92435c7..ce8e7a14 100644 --- a/plugins.go +++ b/plugins.go @@ -79,7 +79,13 @@ func InitPlugins() { } for _, options := range Settings.inputRAW { - registerPlugin(NewRAWInput, options) + captureResponse := false + + if len(Settings.middleware) > 0 { + captureResponse = true + } + + registerPlugin(NewRAWInput, options, 0, captureResponse) } for _, options := range Settings.inputTCP { From d1091102ba8bc043cdc1eb19654fd41c49cd2e39 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 21:46:09 +0300 Subject: [PATCH 27/64] Ignore all release files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index c59f5bad..3c34e46e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ *.out *.bin + +*.gz From 211be0cdddf91a7ffbb15a1902382f2195f6e7cf Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 17 Aug 2015 17:10:43 +0300 Subject: [PATCH 28/64] Add request time and round-trip response time --- Makefile | 2 +- README.md | 22 +++++++++++++--- examples/middleware/token_modifier.go | 11 +++++--- input_raw.go | 11 ++++---- middleware.go | 1 - output_http.go | 16 ++++++----- payload_format.go | 26 ------------------ protocol.go | 38 +++++++++++++++++++++++++++ raw_socket_listener/tcp_message.go | 2 ++ 9 files changed, 82 insertions(+), 47 deletions(-) delete mode 100644 payload_format.go create mode 100644 protocol.go diff --git a/Makefile b/Makefile index 6f43cab8..11ce2c68 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go payload_format.go +SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go SOURCE_PATH = /gopath/src/github.com/buger/gor/ diff --git a/README.md b/README.md index 8e5dbf0d..e341d986 100644 --- a/README.md +++ b/README.md @@ -213,16 +213,30 @@ gor --input-raw :80 --middleware "/opt/middleware_executable" --output-http "htt #### Communication protocol All messages should be hex encoded, new line character specifieds the end of the message, eg. new message per line. -Decoded payload consist of 2 parts: header and HTTP payload, separated by new line character. Example: +Decoded payload consist of 2 parts: header and HTTP payload, separated by new line character. + +Example request payload: + ``` -1 932079936fa4306fc308d67588178d17d823647c +1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 GET /a HTTP/1.1 Host: 127.0.0.1 ``` -First header byte `1` represent payload type, possible values: `1` - request, `2` - original response, `3` - replayed response -After empty spaces, goes request id `932079936fa4306fc308d67588178d17d823647c`. Request id unique among all requests (sha1 of time and Ack), but remain same for original and replayed response, so you can create associations between request and responses. +Example response payload: + +``` +2 8e091765ae902fef8a2b7d9dd960e9d52222bd8c 2782013 +HTTP/1.1 200 OK +Date: Mon, 17 Aug 2015 13:40:23 GMT +Content-Length: 0 +Content-Type: text/plain; charset=utf-8 + +``` + +Header contains request meta information separated by spaces. First value is payload type, possible values: `1` - request, `2` - original response, `3` - replayed response. +Next goes request id: unique among all requests (sha1 of time and Ack), but remain same for original and replayed response, so you can create associations between request and responses. Third argument varies depending on payload type: for request - start time, for responses - round-trip time. HTTP payload is unmodified HTTP requests/responses intercepted from network. You can read more about request format [here](http://www.jmarshall.com/easy/http/), [here](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) and [here](http://www.w3.org/Protocols/rfc2616/rfc2616.html). You can operate with payload as you want, add headers, change path, and etc. Basically you just editing a string, just ensure that it is RCF compliant. diff --git a/examples/middleware/token_modifier.go b/examples/middleware/token_modifier.go index 3b871e2d..9da3fc8a 100644 --- a/examples/middleware/token_modifier.go +++ b/examples/middleware/token_modifier.go @@ -55,10 +55,13 @@ func process(buf []byte) { // 2 - Response // 3 - ReplayedResponse payloadType := buf[0] - headerSize := 42 - header := buf[:headerSize] + headerSize := bytes.IndexByte(buf, '\n') + 1 + header := buf[:headerSize-1] + + // Header contains space separated values of: request type, request id, and request start time (or round-trip time for responses) + meta := bytes.Split(header, []byte(" ")) // For each request you should receive 3 payloads (request, response, replayed response) with same request id - reqID := string(header[2:headerSize]) + reqID := string(meta[1]) payload := buf[headerSize:] Debug("Received payload:", string(buf)) @@ -83,7 +86,7 @@ func process(buf []byte) { } // Re-compute length in case if payload was modified - bufLen := len(header) + len(payload) + bufLen := headerSize + len(payload) // Encoding request and sending it back dst := make([]byte, bufLen*2+1) hex.Encode(dst, buf[:bufLen]) diff --git a/input_raw.go b/input_raw.go index cfdfc8b4..75a37ec2 100644 --- a/input_raw.go +++ b/input_raw.go @@ -34,12 +34,13 @@ func (i *RAWInput) Read(data []byte) (int, error) { buf := msg.Bytes() if i.captureResponse { - payloadType := RequestPayload - if !msg.IsIncoming { - payloadType = ResponsePayload - } + var header []byte - header := payloadHeader(payloadType, msg.UUID()) + if msg.IsIncoming { + header = payloadHeader(RequestPayload, msg.UUID(), msg.Start) + } else { + header = payloadHeader(ResponsePayload, msg.UUID(), msg.End-msg.RequestStart) + } copy(data[0:len(header)], header) copy(data[len(header):], buf) diff --git a/middleware.go b/middleware.go index cb4ea123..e88be4dd 100644 --- a/middleware.go +++ b/middleware.go @@ -108,7 +108,6 @@ func (m *Middleware) read(from io.Reader) { } func (m *Middleware) Read(data []byte) (int, error) { - Debug("Trying to read channel!") buf := <-m.data copy(data, buf) diff --git a/output_http.go b/output_http.go index c3e04508..a44c7fd2 100644 --- a/output_http.go +++ b/output_http.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "io" "log" "sync/atomic" @@ -10,8 +11,9 @@ import ( const initialDynamicWorkers = 10 type response struct { - payload []byte - uuid []byte + payload []byte + uuid []byte + roundTripTime int64 } // HTTPOutputConfig struct for holding http output configuration @@ -163,7 +165,7 @@ func (o *HTTPOutput) Read(data []byte) (int, error) { Debug("[OUTPUT-HTTP] Received response", string(resp.payload)) - header := payloadHeader(ReplayedResponsePayload, resp.uuid) + header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime) copy(data[0:len(header)], header) copy(data[len(header):], resp.payload) @@ -174,8 +176,10 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { var uuid []byte if len(Settings.middleware) > 0 { - uuid = request[2:42] - request = request[43:] + headerSize := bytes.IndexByte(request, '\n') + meta := bytes.Split(request[:headerSize], []byte{' '}) + uuid = meta[1] + request = request[headerSize+1:] } start := time.Now() @@ -187,7 +191,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { } if len(Settings.middleware) > 0 { - o.responses <- response{resp, uuid} + o.responses <- response{resp, uuid, stop.UnixNano() - start.UnixNano()} } if o.elasticSearch != nil { diff --git a/payload_format.go b/payload_format.go deleted file mode 100644 index f26b1031..00000000 --- a/payload_format.go +++ /dev/null @@ -1,26 +0,0 @@ -package main - -const ( - RequestPayload = 1 << iota - ResponsePayload - ReplayedResponsePayload -) - -func payloadHeader(payloadType int, uuid []byte) (header []byte) { - header = make([]byte, 43) - header[1] = ' ' - header[len(header)-1] = '\n' - - switch payloadType { - case RequestPayload: - header[0] = '1' - case ResponsePayload: - header[0] = '2' - case ReplayedResponsePayload: - header[0] = '3' - } - - copy(header[2:], uuid) - - return header -} diff --git a/protocol.go b/protocol.go new file mode 100644 index 00000000..a92ac803 --- /dev/null +++ b/protocol.go @@ -0,0 +1,38 @@ +package main + +import ( + "strconv" +) + +const ( + RequestPayload = 1 << iota + ResponsePayload + ReplayedResponsePayload +) + +// Timing is request start or round-trip time, depending on payloadType +func payloadHeader(payloadType int, uuid []byte, timing int64) (header []byte) { + sTime := strconv.FormatInt(timing, 10) + + //Example: + // 3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 1231\n + // `+ 1` indicates space characters or end of line + header = make([]byte, 1+1+len(uuid)+1+len(sTime)+1) + header[1] = ' ' + header[2+len(uuid)] = ' ' + header[len(header)-1] = '\n' + + switch payloadType { + case RequestPayload: + header[0] = '1' + case ResponsePayload: + header[0] = '2' + case ReplayedResponsePayload: + header[0] = '3' + } + + copy(header[2:], uuid) + copy(header[3+len(uuid):], sTime) + + return header +} diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 359e62e6..956f4ec4 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -23,6 +23,7 @@ type TCPMessage struct { RequestStart int64 RequestAck uint32 Start int64 + End int64 IsIncoming bool packets []*TCPPacket @@ -119,6 +120,7 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { log.Println("Received packet with same sequence") } else { t.packets = append(t.packets, packet) + t.End = time.Now().UnixNano() } if !t.isMultipart() { From 69424bf6700fc592157c9e264af5a3575152913c Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 17 Aug 2015 17:13:55 +0300 Subject: [PATCH 29/64] Fix hound --- examples/middleware/echo_modifier.rb | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/examples/middleware/echo_modifier.rb b/examples/middleware/echo_modifier.rb index a20beb83..084acc17 100755 --- a/examples/middleware/echo_modifier.rb +++ b/examples/middleware/echo_modifier.rb @@ -1,16 +1,15 @@ #!/usr/bin/env ruby # encoding: utf-8 while data = STDIN.gets - next unless data - data = data.chomp + next unless data + data = data.chomp - decoded = [data].pack("H*") - encoded = decoded.unpack("H*").first + decoded = [data].pack("H*") + encoded = decoded.unpack("H*").first - STDOUT.puts encoded + STDOUT.puts encoded - - STDERR.puts "[DEBUG][MIDDLEWARE] Original data: #{data}" - STDERR.puts "[DEBUG][MIDDLEWARE] Decoded request: #{decoded}" - STDERR.puts "[DEBUG][MIDDLEWARE] Encoded data: #{encoded}" -end \ No newline at end of file + STDERR.puts "[DEBUG][MIDDLEWARE] Original data: #{data}" + STDERR.puts "[DEBUG][MIDDLEWARE] Decoded request: #{decoded}" + STDERR.puts "[DEBUG][MIDDLEWARE] Encoded data: #{encoded}" +end From d96e3393ec8cb059f9b4cc21be8753b2e24e5864 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 17 Aug 2015 20:36:59 +0300 Subject: [PATCH 30/64] Handle HTTP error codes --- http_client.go | 33 +++++++++++++++- http_client_test.go | 91 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/http_client.go b/http_client.go index e3d6c1e2..6ec12273 100644 --- a/http_client.go +++ b/http_client.go @@ -21,6 +21,7 @@ type HTTPClientConfig struct { FollowRedirects int Debug bool OriginalHost bool + ConnectionTimeout time.Duration Timeout time.Duration ResponseBufferSize int } @@ -69,9 +70,9 @@ func (c *HTTPClient) Connect() (err error) { c.Disconnect() if !strings.Contains(c.host, ":") { - c.conn, err = net.Dial("tcp", c.host+":80") + c.conn, err = net.DialTimeout("tcp", c.host+":80", c.config.ConnectionTimeout) } else { - c.conn, err = net.Dial("tcp", c.host) + c.conn, err = net.DialTimeout("tcp", c.host, c.config.ConnectionTimeout) } if c.scheme == "https" { @@ -124,6 +125,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { Debug("[HTTPClient] Connecting:", c.baseURL) if err = c.Connect(); err != nil { log.Println("[HTTPClient] Connection error:", err) + response = errorPayload(HTTP_CONNECTION_ERROR) return } } @@ -142,6 +144,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if _, err = c.conn.Write(data); err != nil { Debug("[HTTPClient] Write error:", err, c.baseURL) + response = errorPayload(HTTP_TIMEOUT) return } @@ -160,6 +163,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if err != nil { Debug("[HTTPClient] Response read error", err, c.conn) + response = errorPayload(HTTP_TIMEOUT) return } @@ -197,3 +201,28 @@ func (c *HTTPClient) Get(path string) (response []byte, err error) { return c.Send([]byte(payload)) } + +const ( + // https://support.cloudflare.com/hc/en-us/articles/200171936-Error-520-Web-server-is-returning-an-unknown-error + HTTP_UNKNOWN_ERROR = "520" + // https://support.cloudflare.com/hc/en-us/articles/200171916-Error-521-Web-server-is-down + HTTP_CONNECTION_ERROR = "521" + // https://support.cloudflare.com/hc/en-us/articles/200171906-Error-522-Connection-timed-out + HTTP_CONNECTION_TIMEOUT = "522" + // https://support.cloudflare.com/hc/en-us/articles/200171946-Error-523-Origin-is-unreachable + HTTP_UNREACHABLE = "523" + // https://support.cloudflare.com/hc/en-us/articles/200171926-Error-524-A-timeout-occurred + HTTP_TIMEOUT = "524" +) + +var errorPayloadTemplate = "HTTP/1.1 202 Accepted\r\nDate: Mon, 17 Aug 2015 14:10:11 GMT\r\nContent-Length: 0\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n" + +func errorPayload(errorCode string) []byte { + payload := make([]byte, len(errorPayloadTemplate)) + copy(payload, errorPayloadTemplate) + + copy(payload[29:58], []byte(time.Now().Format(time.RFC1123))) + copy(payload[9:12], errorCode) + + return payload +} diff --git a/http_client_test.go b/http_client_test.go index 6109a90e..3f13be23 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -8,7 +8,11 @@ import ( "net/http" "net/http/httptest" "net/http/httputil" + "github.com/buger/gor/proto" "sync" + "time" + "log" + _ "reflect" "testing" _ "time" ) @@ -299,3 +303,90 @@ func TestHTTPClientHandleHTTP10(t *testing.T) { wg.Wait() } + +func TestHTTPClientErrors(t *testing.T) { + req := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n") + + // Port not exists + client := NewHTTPClient("http://127.0.0.1:1", &HTTPClientConfig{Debug: true}) + if resp, err := client.Send(req); err != nil { + if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) { + t.Error("Should return status 521 for connection refused, instead:", string(s)) + } + } else { + t.Error("Should throw error") + } + + client = NewHTTPClient("http://not.existing", &HTTPClientConfig{Debug: true}) + if resp, err := client.Send(req); err != nil { + if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) { + t.Error("Should return status 521 for no such host, instead:", string(s)) + } + } else { + t.Error("Should throw error") + } + + // Non routable IP address to simulate connection timeout + client = NewHTTPClient("http://10.255.255.1", &HTTPClientConfig{Debug: true, ConnectionTimeout: 100 * time.Millisecond }) + + if resp, err := client.Send(req); err != nil { + if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) { + t.Error("Should return status 521 for io/timeout:", string(s)) + } + } else { + t.Error("Should throw error") + } + + // Connecting but io timeout on read + ln, _ := net.Listen("tcp", ":0") + client = NewHTTPClient("http://" + ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) + + if resp, err := client.Send(req); err != nil { + if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) { + t.Error("Should return status 524 for io read, instead:", string(s)) + } + } else { + t.Error("Should throw error") + } + + // Response read error read tcp [::1]:51128: connection reset by peer &{{0xc20802a000}} + ln1, _ := net.Listen("tcp", ":0") + go func(){ + ln1.Accept() + }() + + client = NewHTTPClient("http://" + ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) + + if resp, err := client.Send(req); err != nil { + if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) { + t.Error("Should return status 524 for connection reset by peer, instead:", string(s)) + } + } else { + t.Error("Should throw error") + } + + ln2, _ := net.Listen("tcp", ":0") + go func(){ + for { + buf := make([]byte, 64*1024) + conn, err := ln2.Accept() + + if err != nil { + log.Println("Error while Accept()", err) + continue + } + + conn.Read(buf) + } + }() + + client = NewHTTPClient("http://" + ln2.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) + + if resp, err := client.Send(req); err != nil { + if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) { + t.Error("Should return status 524 for connection reset by peer, instead:", string(s)) + } + } else { + t.Error("Should throw error") + } +} From ea94dd0162ad0fa844a6a681fe51b125304c90db Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 08:20:08 +0300 Subject: [PATCH 31/64] fmt changes --- http_client_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/http_client_test.go b/http_client_test.go index 3f13be23..d1fe315b 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -3,17 +3,17 @@ package main import ( "bytes" "crypto/rand" + "github.com/buger/gor/proto" "io/ioutil" + "log" "net" "net/http" "net/http/httptest" "net/http/httputil" - "github.com/buger/gor/proto" - "sync" - "time" - "log" _ "reflect" + "sync" "testing" + "time" _ "time" ) @@ -327,7 +327,7 @@ func TestHTTPClientErrors(t *testing.T) { } // Non routable IP address to simulate connection timeout - client = NewHTTPClient("http://10.255.255.1", &HTTPClientConfig{Debug: true, ConnectionTimeout: 100 * time.Millisecond }) + client = NewHTTPClient("http://10.255.255.1", &HTTPClientConfig{Debug: true, ConnectionTimeout: 100 * time.Millisecond}) if resp, err := client.Send(req); err != nil { if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) { @@ -339,7 +339,7 @@ func TestHTTPClientErrors(t *testing.T) { // Connecting but io timeout on read ln, _ := net.Listen("tcp", ":0") - client = NewHTTPClient("http://" + ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) + client = NewHTTPClient("http://"+ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) if resp, err := client.Send(req); err != nil { if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) { @@ -351,11 +351,11 @@ func TestHTTPClientErrors(t *testing.T) { // Response read error read tcp [::1]:51128: connection reset by peer &{{0xc20802a000}} ln1, _ := net.Listen("tcp", ":0") - go func(){ + go func() { ln1.Accept() }() - client = NewHTTPClient("http://" + ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) + client = NewHTTPClient("http://"+ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) if resp, err := client.Send(req); err != nil { if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) { @@ -366,21 +366,21 @@ func TestHTTPClientErrors(t *testing.T) { } ln2, _ := net.Listen("tcp", ":0") - go func(){ + go func() { for { buf := make([]byte, 64*1024) conn, err := ln2.Accept() if err != nil { log.Println("Error while Accept()", err) - continue - } + continue + } conn.Read(buf) } }() - client = NewHTTPClient("http://" + ln2.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) + client = NewHTTPClient("http://"+ln2.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) if resp, err := client.Send(req); err != nil { if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) { From 3e763c43cf255b7d6da9620970ad70c587429d65 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 14:32:52 +0300 Subject: [PATCH 32/64] Unify all input/output plugins --- README.md | 2 +- emitter.go | 21 +++++----- examples/middleware/echo_modifier.sh | 3 +- examples/middleware/token_modifier.go | 22 +++++----- input_raw.go | 29 +++++--------- input_raw_test.go | 58 ++++++++------------------- middleware.go | 5 +-- middleware_test.go | 4 +- output_file.go | 4 ++ output_http.go | 17 ++++---- output_http_test.go | 2 +- output_tcp.go | 4 ++ protocol.go | 37 +++++++++++++++++ raw_socket_listener/listener.go | 12 ++++-- raw_socket_listener/tcp_message.go | 7 +++- raw_socket_listener/tcp_packet.go | 7 +++- test_input.go | 8 +++- 17 files changed, 136 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index e341d986..11415f48 100644 --- a/README.md +++ b/README.md @@ -240,7 +240,7 @@ Next goes request id: unique among all requests (sha1 of time and Ack), but rema HTTP payload is unmodified HTTP requests/responses intercepted from network. You can read more about request format [here](http://www.jmarshall.com/easy/http/), [here](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) and [here](http://www.w3.org/Protocols/rfc2616/rfc2616.html). You can operate with payload as you want, add headers, change path, and etc. Basically you just editing a string, just ensure that it is RCF compliant. -At the end modified (or untouched) request should be emitted back to STDOUT, keeping original header, and hex-encoded. If you want to filter request, just not send it. Emitting responses back is optional, and does not affect anything at the moment. +At the end modified (or untouched) request should be emitted back to STDOUT, keeping original header, and hex-encoded. If you want to filter request, just not send it. Emitting responses back is required, even if you did not touch them. #### Advanced example Imagine that you have auth system that randomly generate access tokens, which used later for accessing secure content. Since there is no pre-defined token value, naive approach without middleware (or if middleware use only request payloads) will fail, because replayed server have own tokens, not synced with origin. To fix this, our middleware should take in account responses of replayed and origin server, store `originalToken -> replayedToken` aliases and rewrite all requests using this token to use replayed alias. See `examples/middleware/token_modifier.go` and `middleware_test.go#TestTokenMiddleware` as example of described scheme. diff --git a/emitter.go b/emitter.go index 07692954..40875bc3 100644 --- a/emitter.go +++ b/emitter.go @@ -1,17 +1,11 @@ package main import ( - "crypto/rand" "io" "time" + "bytes" ) -func uuid() []byte { - b := make([]byte, 16) - rand.Read(b) - return b -} - // Start initialize loop for sending data from inputs to outputs func Start(stop chan int) { if Settings.middleware != "" { @@ -65,14 +59,21 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { Debug("[EMITTER] input:", string(payload[0:_maxN])) } - if modifier != nil { - payload = modifier.Rewrite(payload) + if modifier != nil && isRequestPayload(payload) { + headSize := bytes.IndexByte(payload, '\n') + 1 + body := payload[headSize:] + originalBodyLen := len(body) + body = modifier.Rewrite(body) // If modifier tells to skip request - if len(payload) == 0 { + if len(body) == 0 { continue } + if originalBodyLen != len(body) { + payload = append(payload[:headSize], body...) + } + if Settings.debug { Debug("[EMITTER] Rewrittern input:", len(payload), "First 500 bytes:", string(payload[0:_maxN])) } diff --git a/examples/middleware/echo_modifier.sh b/examples/middleware/echo_modifier.sh index 96dc1217..ec946eeb 100755 --- a/examples/middleware/echo_modifier.sh +++ b/examples/middleware/echo_modifier.sh @@ -22,7 +22,6 @@ while read line; do case ${header:0:1} in "1") log "Request type: Request" - echo "$encoded" ;; "2") log "Request type: Original Response" @@ -33,6 +32,8 @@ while read line; do *) log "Unknown request type $header" esac + echo "$encoded" + log "===================================" log "Original data: $line" diff --git a/examples/middleware/token_modifier.go b/examples/middleware/token_modifier.go index 9da3fc8a..021408d5 100644 --- a/examples/middleware/token_modifier.go +++ b/examples/middleware/token_modifier.go @@ -80,21 +80,13 @@ func process(buf []byte) { payload = proto.SetPathParam(payload, []byte("token"), alias) // Copy modified payload to our buffer - copy(buf[headerSize:], payload) + buf = append(buf[:headerSize], payload...) } } } - // Re-compute length in case if payload was modified - bufLen := headerSize + len(payload) - // Encoding request and sending it back - dst := make([]byte, bufLen*2+1) - hex.Encode(dst, buf[:bufLen]) - dst[len(dst)-1] = '\n' - - os.Stdout.Write(dst) - - return + // Emitting data back + os.Stdout.Write(encode(buf)) case '2': // Original response if _, ok := originalTokens[reqID]; ok { // Token is inside response body @@ -113,6 +105,14 @@ func process(buf []byte) { } } +func encode(buf []byte) []byte { + dst := make([]byte, len(buf)*2+1) + hex.Encode(dst, buf) + dst[len(dst)-1] = '\n' + + return dst +} + func Debug(args ...interface{}) { fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ") fmt.Fprintln(os.Stderr, args...) diff --git a/input_raw.go b/input_raw.go index 75a37ec2..3bf882b0 100644 --- a/input_raw.go +++ b/input_raw.go @@ -13,16 +13,14 @@ type RAWInput struct { data chan *raw.TCPMessage address string expire time.Duration - captureResponse bool } // NewRAWInput constructor for RAWInput. Accepts address with port as argument. -func NewRAWInput(address string, expire time.Duration, captureResponse bool) (i *RAWInput) { +func NewRAWInput(address string, expire time.Duration) (i *RAWInput) { i = new(RAWInput) i.data = make(chan *raw.TCPMessage) i.address = address i.expire = expire - i.captureResponse = captureResponse go i.listen(address) @@ -33,23 +31,18 @@ func (i *RAWInput) Read(data []byte) (int, error) { msg := <-i.data buf := msg.Bytes() - if i.captureResponse { - var header []byte + var header []byte - if msg.IsIncoming { - header = payloadHeader(RequestPayload, msg.UUID(), msg.Start) - } else { - header = payloadHeader(ResponsePayload, msg.UUID(), msg.End-msg.RequestStart) - } - - copy(data[0:len(header)], header) - copy(data[len(header):], buf) - - return len(buf) + len(header), nil + if msg.IsIncoming { + header = payloadHeader(RequestPayload, msg.UUID(), msg.Start) } else { - copy(data, buf) - return len(buf), nil + header = payloadHeader(ResponsePayload, msg.UUID(), msg.End-msg.RequestStart) } + + copy(data[0:len(header)], header) + copy(data[len(header):], buf) + + return len(buf) + len(header), nil } func (i *RAWInput) listen(address string) { @@ -63,7 +56,7 @@ func (i *RAWInput) listen(address string) { log.Fatal("input-raw: error while parsing address", err) } - listener := raw.NewListener(host, port, i.expire, i.captureResponse) + listener := raw.NewListener(host, port, i.expire, true) for { // Receiving TCPMessage object diff --git a/input_raw_test.go b/input_raw_test.go index 32730356..1f5abe96 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -24,7 +24,7 @@ func TestRAWInput(t *testing.T) { listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) - input := NewRAWInput(listener.Addr().String(), testRawExpire, false) + input := NewRAWInput(listener.Addr().String(), testRawExpire) output := NewTestOutput(func(data []byte) { wg.Done() }) @@ -41,7 +41,8 @@ func TestRAWInput(t *testing.T) { go Start(quit) for i := 0; i < 100; i++ { - wg.Add(1) + // request + response + wg.Add(2) client.Get("/") } @@ -66,14 +67,19 @@ func TestInputRAW100Expect(t *testing.T) { originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire, false) + input := NewRAWInput(originAddr, testRawExpire) // We will use it to get content of raw HTTP request testOutput := NewTestOutput(func(data []byte) { - if strings.Contains(string(data), "Expect: 100-continue") { - t.Error("Should not contain 100-continue header") + switch data[0] { + case '1': + if strings.Contains(string(data), "Expect: 100-continue") { + t.Error("Should not contain 100-continue header") + } + wg.Done() + case '2': + wg.Done() } - wg.Done() }) listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { @@ -96,7 +102,8 @@ func TestInputRAW100Expect(t *testing.T) { go Start(quit) - wg.Add(3) + // Origin + Response/Request Test Output + Request Http Output + wg.Add(4) curl := exec.Command("curl", "http://"+originAddr, "--data-binary", "@README.md") err := curl.Run() if err != nil { @@ -123,7 +130,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire, false) + input := NewRAWInput(originAddr, testRawExpire) listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { defer req.Body.Close() @@ -138,7 +145,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { }) replayAddr := listener.Addr().String() - httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{Debug: false}) + httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{httpOutput} @@ -181,7 +188,7 @@ func TestInputRAWLargePayload(t *testing.T) { })) originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire, false) + input := NewRAWInput(originAddr, testRawExpire) replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024) @@ -214,34 +221,3 @@ func TestInputRAWLargePayload(t *testing.T) { wg.Wait() close(quit) } - -func TestInputRAWResponse(t *testing.T) { - wg := new(sync.WaitGroup) - quit := make(chan int) - - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) - - input := NewRAWInput(listener.Addr().String(), testRawExpire, true) - output := NewTestOutput(func(data []byte) { - wg.Done() - }) - - Plugins.Inputs = []io.Reader{input} - Plugins.Outputs = []io.Writer{output} - - address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1) - - client := NewHTTPClient(address, &HTTPClientConfig{}) - - time.Sleep(time.Millisecond) - go Start(quit) - - for i := 0; i < 100; i++ { - // 2 because we track both request and response - wg.Add(2) - client.Get("/") - } - - wg.Wait() - close(quit) -} diff --git a/middleware.go b/middleware.go index e88be4dd..8655e853 100644 --- a/middleware.go +++ b/middleware.go @@ -94,10 +94,7 @@ func (m *Middleware) read(from io.Reader) { Debug("[MIDDLEWARE-MASTER] Received:", string(buf)) } - // We should accept only request payloads - if buf[0] == '1' { - m.data <- buf - } + m.data <- buf } if err := scanner.Err(); err != nil { diff --git a/middleware_test.go b/middleware_test.go index 05bcb4aa..bf8203c9 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -110,7 +110,7 @@ func TestEchoMiddleware(t *testing.T) { Settings.middleware = "./examples/middleware/echo_modifier.sh" // Catch traffic from one service - input := NewRAWInput(from.Listener.Addr().String(), testRawExpire, true) + input := NewRAWInput(from.Listener.Addr().String(), testRawExpire) // And redirect to another output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false}) @@ -165,7 +165,7 @@ func TestTokenMiddleware(t *testing.T) { quit := make(chan int) // Catch traffic from one service - input := NewRAWInput(from, testRawExpire, true) + input := NewRAWInput(from, testRawExpire) // And redirect to another output := NewHTTPOutput(to, &HTTPOutputConfig{Debug: true}) diff --git a/output_file.go b/output_file.go index f6a81ae9..fab2210f 100644 --- a/output_file.go +++ b/output_file.go @@ -43,6 +43,10 @@ func (o *FileOutput) init(path string) { } func (o *FileOutput) Write(data []byte) (n int, err error) { + if !isOriginPayload(data) { + return len(data), nil + } + raw := RawRequest{time.Now().UnixNano(), data} o.encoder.Encode(raw) diff --git a/output_http.go b/output_http.go index a44c7fd2..cc751921 100644 --- a/output_http.go +++ b/output_http.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "io" "log" "sync/atomic" @@ -140,6 +139,10 @@ func (o *HTTPOutput) startWorker() { } func (o *HTTPOutput) Write(data []byte) (n int, err error) { + if !isRequestPayload(data) { + return len(data), nil + } + buf := make([]byte, len(data)) copy(buf, data) @@ -173,17 +176,11 @@ func (o *HTTPOutput) Read(data []byte) (int, error) { } func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { - var uuid []byte - - if len(Settings.middleware) > 0 { - headerSize := bytes.IndexByte(request, '\n') - meta := bytes.Split(request[:headerSize], []byte{' '}) - uuid = meta[1] - request = request[headerSize+1:] - } + meta := payloadMeta(request) + uuid := meta[1] start := time.Now() - resp, err := client.Send(request) + resp, err := client.Send(payloadBody(request)) stop := time.Now() if err != nil { diff --git a/output_http_test.go b/output_http_test.go index a428c388..f4a4e6cc 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -55,7 +55,7 @@ func TestHTTPOutput(t *testing.T) { methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")} Settings.modifierConfig = HTTPModifierConfig{headers: headers, methods: methods} - output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{}) + output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} diff --git a/output_tcp.go b/output_tcp.go index 2656a255..fdbe579e 100644 --- a/output_tcp.go +++ b/output_tcp.go @@ -57,6 +57,10 @@ func (o *TCPOutput) worker() { } func (o *TCPOutput) Write(data []byte) (n int, err error) { + if !isOriginPayload(data) { + return len(data), nil + } + // Hex encoding always 2x number of bytes encoded := make([]byte, len(data)*2+1) hex.Encode(encoded, data) diff --git a/protocol.go b/protocol.go index a92ac803..9b8cc1ab 100644 --- a/protocol.go +++ b/protocol.go @@ -2,6 +2,9 @@ package main import ( "strconv" + "bytes" + "encoding/hex" + "crypto/rand" ) const ( @@ -10,6 +13,17 @@ const ( ReplayedResponsePayload ) +func uuid() []byte { + b := make([]byte, 20) + rand.Read(b) + + uuid := make([]byte, 40) + hex.Encode(uuid, b) + + return uuid +} + + // Timing is request start or round-trip time, depending on payloadType func payloadHeader(payloadType int, uuid []byte, timing int64) (header []byte) { sTime := strconv.FormatInt(timing, 10) @@ -36,3 +50,26 @@ func payloadHeader(payloadType int, uuid []byte, timing int64) (header []byte) { return header } + +func payloadBody(payload []byte) []byte { + headerSize := bytes.IndexByte(payload, '\n') + return payload[headerSize+1:] +} + +func payloadMeta(payload []byte) [][]byte { + headerSize := bytes.IndexByte(payload, '\n') + return bytes.Split(payload[:headerSize], []byte{' '}) +} + +func isOriginPayload(payload []byte) bool { + switch payload[0] { + case '1', '2': + return true + default: + return false + } +} + +func isRequestPayload(payload []byte) bool { + return payload[0] == '1' +} \ No newline at end of file diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index eee05f44..eefb3e05 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -91,10 +91,15 @@ func (t *Listener) listen() { select { // If message ready for deletion it means that its also complete or expired by timeout case message := <-t.messageDelChan: - t.messagesChan <- message delete(t.ackAliases, message.Ack) delete(t.messages, message.ID) + if !message.IsIncoming { + delete(t.respAliases, message.Ack) + } + + t.messagesChan <- message + // We need to use channels to process each packet to avoid data races case packet := <-t.packetsChan: t.processTCPPacket(packet) @@ -216,11 +221,12 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { if t.captureResponse && isIncoming { // If message have multiple packets, delete previous alias if len(message.packets) > 0 { - delete(t.respAliases, uint32(message.Size())) + delete(t.respAliases, message.ResponseAck) } - responseAck := packet.Seq + uint32(message.Size()+len(packet.Data)) + responseAck := packet.Seq + uint32(len(packet.Data)) t.respAliases[responseAck] = &request{message.Start, message.Ack} + message.ResponseAck = responseAck } // Adding packet to message diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 956f4ec4..a0b8f885 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -20,6 +20,7 @@ import ( type TCPMessage struct { ID string // Message ID Ack uint32 + ResponseAck uint32 RequestStart int64 RequestAck uint32 Start int64 @@ -80,7 +81,11 @@ func (t *TCPMessage) Timeout() { } default: close(t.packetsChan) - t.delChan <- t // Notify RAWListener that message is ready to be send to replay server + // Notify RAWListener that message is ready to be send to replay server + // Responses without requests gets discarded + if t.IsIncoming || t.RequestStart != 0 { + t.delChan <- t + } } } diff --git a/raw_socket_listener/tcp_packet.go b/raw_socket_listener/tcp_packet.go index 998eeb46..e1ef3584 100644 --- a/raw_socket_listener/tcp_packet.go +++ b/raw_socket_listener/tcp_packet.go @@ -69,6 +69,11 @@ func (t *TCPPacket) ParseBasic() { // String output for a TCP Packet func (t *TCPPacket) String() string { + maxLen := len(t.Data) + if maxLen > 500 { + maxLen = 500 + } + return strings.Join([]string{ "Source port: " + strconv.Itoa(int(t.SrcPort)), "Dest port:" + strconv.Itoa(int(t.DestPort)), @@ -90,7 +95,7 @@ func (t *TCPPacket) String() string { "Checksum:" + strconv.Itoa(int(t.Checksum)), "Data size:" + strconv.Itoa(len(t.Data)), - "Data:" + string(t.Data), + "Data:" + string(t.Data[:maxLen]), }, "\n") } diff --git a/test_input.go b/test_input.go index a9a359a3..c699dff3 100644 --- a/test_input.go +++ b/test_input.go @@ -3,6 +3,7 @@ package main import ( "crypto/rand" "encoding/base64" + "time" ) // TestInput used for testing purpose, it allows emitting requests on demand @@ -20,9 +21,12 @@ func NewTestInput() (i *TestInput) { func (i *TestInput) Read(data []byte) (int, error) { buf := <-i.data - copy(data, buf) - return len(buf), nil + header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano()) + copy(data[0:len(header)], header) + copy(data[len(header):], buf) + + return len(buf) + len(header), nil } // EmitGET emits GET request without headers From a971d8a8430557c595c731a2452ad93a96ed3860 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 15:48:42 +0300 Subject: [PATCH 33/64] Change file format to be text based --- Makefile | 2 +- input_dummy.go | 7 +++++-- input_file.go | 50 ++++++++++++++++++++++++++++++++++---------------- output_file.go | 18 ++++-------------- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index 11ce2c68..c051a841 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ drun: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-http :9000 --verbose drecord: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-file=requests.bin --verbose + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose dreplay: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-file=requests.bin --output-tcp=:9000 --verbose -h diff --git a/input_dummy.go b/input_dummy.go index f439179e..4c148303 100644 --- a/input_dummy.go +++ b/input_dummy.go @@ -21,9 +21,12 @@ func NewDummyInput(options string) (di *DummyInput) { func (i *DummyInput) Read(data []byte) (int, error) { buf := <-i.data - copy(data, buf) - return len(buf), nil + header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano()) + copy(data[0:len(header)], header) + copy(data[len(header):], buf) + + return len(buf) + len(header), nil } func (i *DummyInput) emit() { diff --git a/input_file.go b/input_file.go index 60c93ec2..0a384283 100644 --- a/input_file.go +++ b/input_file.go @@ -1,9 +1,11 @@ package main import ( - "encoding/gob" "log" "os" + "bufio" + "bytes" + "strconv" "time" ) @@ -11,7 +13,7 @@ import ( type FileInput struct { data chan []byte path string - decoder *gob.Decoder + file *os.File speedFactor float64 } @@ -35,7 +37,7 @@ func (i *FileInput) init(path string) { log.Fatal(i, "Cannot open file %q. Error: %s", path, err) } - i.decoder = gob.NewDecoder(file) + i.file = file } func (i *FileInput) Read(data []byte) (int, error) { @@ -49,30 +51,46 @@ func (i *FileInput) String() string { return "File input: " + i.path } +func scanSeparator(data []byte, atEOF bool) (advance int, token []byte, err error) { + if atEOF && len(data) == 0 { + return 0, nil, nil + } + + if i := bytes.Index(data, []byte(fileSeparator)); i >= 0 { + // We have a full newline-terminated line. + return i + len(fileSeparator), data[0:i], nil + } + + if atEOF { + return len(data), data, nil + } + return 0, nil, nil +} + func (i *FileInput) emit() { var lastTime int64 - for { - raw := new(RawRequest) - err := i.decoder.Decode(raw) + // reader := bufio.NewReader(conn) + scanner := bufio.NewScanner(i.file) + scanner.Split(scanSeparator) - if err != nil { - return - } + for scanner.Scan() { + buf := scanner.Bytes() + meta := payloadMeta(buf) - if lastTime != 0 { - timeDiff := raw.Timestamp - lastTime + if meta[0][0] == '1' && lastTime != 0 { + ts, _ := strconv.ParseInt(string(meta[2]), 10, 64) + timeDiff := ts - lastTime - // We can speedup or slowdown execution based on speedFactor if i.speedFactor != 1 { - timeDiff = int64(float64(raw.Timestamp-lastTime) / i.speedFactor) + timeDiff = int64(float64(timeDiff) / i.speedFactor) } time.Sleep(time.Duration(timeDiff)) - } - lastTime = raw.Timestamp + lastTime = ts + } - i.data <- raw.Request + i.data <- buf } } diff --git a/output_file.go b/output_file.go index fab2210f..d01f05f5 100644 --- a/output_file.go +++ b/output_file.go @@ -1,23 +1,14 @@ package main import ( - "encoding/gob" "io" "log" "os" - "time" ) -// RawRequest stores original start time and request payload -type RawRequest struct { - Timestamp int64 - Request []byte -} - // FileOutput output plugin type FileOutput struct { path string - encoder *gob.Encoder file *os.File } @@ -38,18 +29,17 @@ func (o *FileOutput) init(path string) { if err != nil { log.Fatal(o, "Cannot open file %q. Error: %s", path, err) } - - o.encoder = gob.NewEncoder(o.file) } +var fileSeparator = "\n🐵🙈🙉\n" + func (o *FileOutput) Write(data []byte) (n int, err error) { if !isOriginPayload(data) { return len(data), nil } - raw := RawRequest{time.Now().UnixNano(), data} - - o.encoder.Encode(raw) + o.file.Write(data) + o.file.Write([]byte(fileSeparator)) return len(data), nil } From 7790b33ceabedf2ffa1473699ab4f61e46178bbe Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 15:52:23 +0300 Subject: [PATCH 34/64] Refactor protocol scanner --- input_file.go | 19 +------------------ output_file.go | 4 +--- protocol.go | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/input_file.go b/input_file.go index 0a384283..1fdf0f1f 100644 --- a/input_file.go +++ b/input_file.go @@ -4,7 +4,6 @@ import ( "log" "os" "bufio" - "bytes" "strconv" "time" ) @@ -51,28 +50,12 @@ func (i *FileInput) String() string { return "File input: " + i.path } -func scanSeparator(data []byte, atEOF bool) (advance int, token []byte, err error) { - if atEOF && len(data) == 0 { - return 0, nil, nil - } - - if i := bytes.Index(data, []byte(fileSeparator)); i >= 0 { - // We have a full newline-terminated line. - return i + len(fileSeparator), data[0:i], nil - } - - if atEOF { - return len(data), data, nil - } - return 0, nil, nil -} - func (i *FileInput) emit() { var lastTime int64 // reader := bufio.NewReader(conn) scanner := bufio.NewScanner(i.file) - scanner.Split(scanSeparator) + scanner.Split(payloadScanner) for scanner.Scan() { buf := scanner.Bytes() diff --git a/output_file.go b/output_file.go index d01f05f5..e8b0d868 100644 --- a/output_file.go +++ b/output_file.go @@ -31,15 +31,13 @@ func (o *FileOutput) init(path string) { } } -var fileSeparator = "\n🐵🙈🙉\n" - func (o *FileOutput) Write(data []byte) (n int, err error) { if !isOriginPayload(data) { return len(data), nil } o.file.Write(data) - o.file.Write([]byte(fileSeparator)) + o.file.Write([]byte(payloadSeparator)) return len(data), nil } diff --git a/protocol.go b/protocol.go index 9b8cc1ab..a55f6b3c 100644 --- a/protocol.go +++ b/protocol.go @@ -23,6 +23,24 @@ func uuid() []byte { return uuid } +var payloadSeparator = "\n🐵🙈🙉\n" + +func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err error) { + if atEOF && len(data) == 0 { + return 0, nil, nil + } + + if i := bytes.Index(data, []byte(payloadSeparator)); i >= 0 { + // We have a full newline-terminated line. + return i + len([]byte(payloadSeparator)), data[0:i], nil + } + + if atEOF { + return len(data), data, nil + } + return 0, nil, nil +} + // Timing is request start or round-trip time, depending on payloadType func payloadHeader(payloadType int, uuid []byte, timing int64) (header []byte) { From 8da4e54cbb7f61d170f3a1ee41a20d89c8c75f78 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 16:02:03 +0300 Subject: [PATCH 35/64] Tcp communication should use same protocol as file based --- input_tcp.go | 12 ++---------- input_tcp_test.go | 9 +++------ output_tcp.go | 11 ++++------- output_tcp_test.go | 8 ++------ 4 files changed, 11 insertions(+), 29 deletions(-) diff --git a/input_tcp.go b/input_tcp.go index 47d2c3df..15dffa51 100644 --- a/input_tcp.go +++ b/input_tcp.go @@ -2,7 +2,6 @@ package main import ( "bufio" - "encoding/hex" "fmt" "log" "net" @@ -10,7 +9,6 @@ import ( ) // TCPInput used for internal communication -// It expected hex encoded data type TCPInput struct { data chan []byte address string @@ -62,16 +60,10 @@ func (i *TCPInput) handleConnection(conn net.Conn) { reader := bufio.NewReader(conn) scanner := bufio.NewScanner(reader) + scanner.Split(payloadScanner) for scanner.Scan() { - encodedPayload := scanner.Bytes() - // Hex encoding always 2x number of bytes - decoded := make([]byte, len(encodedPayload)/2) - _, err := hex.Decode(decoded, encodedPayload) - if err != nil { - log.Println("[TCPInput] failed to hex decode TCP payload:", err) - } - i.data <- decoded + i.data <- scanner.Bytes() } if err := scanner.Err(); err != nil { diff --git a/input_tcp_test.go b/input_tcp_test.go index f440c57a..d844023f 100644 --- a/input_tcp_test.go +++ b/input_tcp_test.go @@ -1,7 +1,6 @@ package main import ( - "encoding/hex" "io" "log" "net" @@ -35,14 +34,12 @@ func TestTCPInput(t *testing.T) { log.Fatal(err) } - msg := []byte("GET / HTTP/1.1\r\n\r\n") + msg := []byte("1 1 1\nGET / HTTP/1.1\r\n\r\n") for i := 0; i < 100; i++ { wg.Add(1) - - encoded := make([]byte, len(msg)*2) - hex.Encode(encoded, msg) - conn.Write(append(encoded, '\n')) + conn.Write(msg) + conn.Write([]byte(payloadSeparator)) } wg.Wait() diff --git a/output_tcp.go b/output_tcp.go index fdbe579e..96295dc8 100644 --- a/output_tcp.go +++ b/output_tcp.go @@ -1,7 +1,6 @@ package main import ( - "encoding/hex" "fmt" "io" "log" @@ -47,7 +46,9 @@ func (o *TCPOutput) worker() { defer conn.Close() for { - _, err := conn.Write(<-o.buf) + conn.Write(<-o.buf) + _, err := conn.Write([]byte(payloadSeparator)) + if err != nil { log.Println("Worker failed on write, exitings and starting new worker") go o.worker() @@ -61,11 +62,7 @@ func (o *TCPOutput) Write(data []byte) (n int, err error) { return len(data), nil } - // Hex encoding always 2x number of bytes - encoded := make([]byte, len(data)*2+1) - hex.Encode(encoded, data) - encoded[len(encoded)-1] = '\n' - o.buf <- encoded + o.buf <- data if Settings.outputTCPStats { o.bufStats.Write(len(o.buf)) diff --git a/output_tcp_test.go b/output_tcp_test.go index 84c66b86..b295d7a6 100644 --- a/output_tcp_test.go +++ b/output_tcp_test.go @@ -2,7 +2,6 @@ package main import ( "bufio" - "encoding/hex" "io" "log" "net" @@ -50,13 +49,10 @@ func startTCP(cb func([]byte)) net.Listener { go func() { reader := bufio.NewReader(conn) scanner := bufio.NewScanner(reader) + scanner.Split(payloadScanner) for scanner.Scan() { - encodedPayload := scanner.Bytes() - // Hex encoding always 2x number of bytes - decoded := make([]byte, len(encodedPayload)/2) - hex.Decode(decoded, encodedPayload) - cb(decoded) + cb(scanner.Bytes()) } }() } From ae35f88004eefa9525a14ef252290571f9eb67cf Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 17:18:02 +0300 Subject: [PATCH 36/64] Remove magic numbers --- input_file.go | 2 +- input_raw_test.go | 4 ++-- protocol.go | 22 +++++++--------------- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/input_file.go b/input_file.go index 1fdf0f1f..f1a057bc 100644 --- a/input_file.go +++ b/input_file.go @@ -61,7 +61,7 @@ func (i *FileInput) emit() { buf := scanner.Bytes() meta := payloadMeta(buf) - if meta[0][0] == '1' && lastTime != 0 { + if meta[0][0] == RequestPayload && lastTime != 0 { ts, _ := strconv.ParseInt(string(meta[2]), 10, 64) timeDiff := ts - lastTime diff --git a/input_raw_test.go b/input_raw_test.go index 1f5abe96..0f30afb6 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -72,12 +72,12 @@ func TestInputRAW100Expect(t *testing.T) { // We will use it to get content of raw HTTP request testOutput := NewTestOutput(func(data []byte) { switch data[0] { - case '1': + case RequestPayload: if strings.Contains(string(data), "Expect: 100-continue") { t.Error("Should not contain 100-continue header") } wg.Done() - case '2': + case ResponsePayload: wg.Done() } }) diff --git a/protocol.go b/protocol.go index a55f6b3c..5d97eb75 100644 --- a/protocol.go +++ b/protocol.go @@ -8,9 +8,9 @@ import ( ) const ( - RequestPayload = 1 << iota - ResponsePayload - ReplayedResponsePayload + RequestPayload = '1' + ResponsePayload = '2' + ReplayedResponsePayload = '3' ) func uuid() []byte { @@ -43,26 +43,18 @@ func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err err // Timing is request start or round-trip time, depending on payloadType -func payloadHeader(payloadType int, uuid []byte, timing int64) (header []byte) { +func payloadHeader(payloadType byte, uuid []byte, timing int64) (header []byte) { sTime := strconv.FormatInt(timing, 10) //Example: // 3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 1231\n // `+ 1` indicates space characters or end of line header = make([]byte, 1+1+len(uuid)+1+len(sTime)+1) + header[0] = payloadType header[1] = ' ' header[2+len(uuid)] = ' ' header[len(header)-1] = '\n' - switch payloadType { - case RequestPayload: - header[0] = '1' - case ResponsePayload: - header[0] = '2' - case ReplayedResponsePayload: - header[0] = '3' - } - copy(header[2:], uuid) copy(header[3+len(uuid):], sTime) @@ -81,7 +73,7 @@ func payloadMeta(payload []byte) [][]byte { func isOriginPayload(payload []byte) bool { switch payload[0] { - case '1', '2': + case RequestPayload, ResponsePayload: return true default: return false @@ -89,5 +81,5 @@ func isOriginPayload(payload []byte) bool { } func isRequestPayload(payload []byte) bool { - return payload[0] == '1' + return payload[0] == RequestPayload } \ No newline at end of file From d6c776fa0c06c86d6010c385d743d4c9131bbc6a Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 17:27:26 +0300 Subject: [PATCH 37/64] Verbose output for travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d491b84e..63ecff53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go go: 1.4.2 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 15s" \ No newline at end of file +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 15s --verbose" \ No newline at end of file From b53fba041547399f2eacdda8582795cfec0c9487 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 17:28:09 +0300 Subject: [PATCH 38/64] Fix formatting issues --- emitter.go | 2 +- input_file.go | 2 +- input_raw.go | 6 +++--- output_file.go | 4 ++-- protocol.go | 19 +++++++++---------- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/emitter.go b/emitter.go index 40875bc3..00ba070c 100644 --- a/emitter.go +++ b/emitter.go @@ -1,9 +1,9 @@ package main import ( + "bytes" "io" "time" - "bytes" ) // Start initialize loop for sending data from inputs to outputs diff --git a/input_file.go b/input_file.go index f1a057bc..840c5d72 100644 --- a/input_file.go +++ b/input_file.go @@ -1,9 +1,9 @@ package main import ( + "bufio" "log" "os" - "bufio" "strconv" "time" ) diff --git a/input_raw.go b/input_raw.go index 3bf882b0..8c096c6f 100644 --- a/input_raw.go +++ b/input_raw.go @@ -10,9 +10,9 @@ import ( // RAWInput used for intercepting traffic for given address type RAWInput struct { - data chan *raw.TCPMessage - address string - expire time.Duration + data chan *raw.TCPMessage + address string + expire time.Duration } // NewRAWInput constructor for RAWInput. Accepts address with port as argument. diff --git a/output_file.go b/output_file.go index e8b0d868..5ccc93cc 100644 --- a/output_file.go +++ b/output_file.go @@ -8,8 +8,8 @@ import ( // FileOutput output plugin type FileOutput struct { - path string - file *os.File + path string + file *os.File } // NewFileOutput constructor for FileOutput, accepts path diff --git a/protocol.go b/protocol.go index 5d97eb75..1705f587 100644 --- a/protocol.go +++ b/protocol.go @@ -1,15 +1,15 @@ package main import ( - "strconv" "bytes" - "encoding/hex" "crypto/rand" + "encoding/hex" + "strconv" ) const ( - RequestPayload = '1' - ResponsePayload = '2' + RequestPayload = '1' + ResponsePayload = '2' ReplayedResponsePayload = '3' ) @@ -27,8 +27,8 @@ var payloadSeparator = "\n🐵🙈🙉\n" func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err error) { if atEOF && len(data) == 0 { - return 0, nil, nil - } + return 0, nil, nil + } if i := bytes.Index(data, []byte(payloadSeparator)); i >= 0 { // We have a full newline-terminated line. @@ -36,12 +36,11 @@ func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err err } if atEOF { - return len(data), data, nil + return len(data), data, nil } - return 0, nil, nil + return 0, nil, nil } - // Timing is request start or round-trip time, depending on payloadType func payloadHeader(payloadType byte, uuid []byte, timing int64) (header []byte) { sTime := strconv.FormatInt(timing, 10) @@ -82,4 +81,4 @@ func isOriginPayload(payload []byte) bool { func isRequestPayload(payload []byte) bool { return payload[0] == RequestPayload -} \ No newline at end of file +} From 7cff78174ae2fab06513bd64c85452cc87e5c06a Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 17:35:54 +0300 Subject: [PATCH 39/64] Remove verbose flag --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 63ecff53..d491b84e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go go: 1.4.2 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 15s --verbose" \ No newline at end of file +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 15s" \ No newline at end of file From c4037e0c5727978a4ba27bc4186970a5d2897d2f Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 17:42:55 +0300 Subject: [PATCH 40/64] Increase travis timeout time --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d491b84e..544d2ffd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go go: 1.4.2 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 15s" \ No newline at end of file +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 30s" \ No newline at end of file From 78456d6cd11e8422449c410fc3e801dd2ee464c4 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 17:46:57 +0300 Subject: [PATCH 41/64] More timeout --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 544d2ffd..4736e5bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go go: 1.4.2 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 30s" \ No newline at end of file +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s" \ No newline at end of file From a6e29c684cb0ae3cb4a58ee3d77c5184b1a22b90 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 21:25:56 +0300 Subject: [PATCH 42/64] Close connection --- http_client_test.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/http_client_test.go b/http_client_test.go index d1fe315b..2500c73a 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -340,6 +340,7 @@ func TestHTTPClientErrors(t *testing.T) { // Connecting but io timeout on read ln, _ := net.Listen("tcp", ":0") client = NewHTTPClient("http://"+ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) + defer ln.Close() if resp, err := client.Send(req); err != nil { if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) { @@ -354,6 +355,7 @@ func TestHTTPClientErrors(t *testing.T) { go func() { ln1.Accept() }() + defer ln1.Close() client = NewHTTPClient("http://"+ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) @@ -367,18 +369,18 @@ func TestHTTPClientErrors(t *testing.T) { ln2, _ := net.Listen("tcp", ":0") go func() { - for { - buf := make([]byte, 64*1024) - conn, err := ln2.Accept() + buf := make([]byte, 64*1024) + conn, err := ln2.Accept() - if err != nil { - log.Println("Error while Accept()", err) - continue - } - - conn.Read(buf) + if err != nil { + log.Println("Error while Accept()", err) + continue } + + conn.Read(buf) + defer conn.Close() }() + defer ln2.Close() client = NewHTTPClient("http://"+ln2.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) From 5d2f6eceea399c1a196d07d49bf416fce02eda46 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 19 Aug 2015 09:19:00 +0300 Subject: [PATCH 43/64] Fixes --- Makefile | 2 +- http_client_test.go | 20 ++++++++++---------- middleware_test.go | 7 ++----- output_http.go | 9 ++++++++- raw_socket_listener/listener.go | 3 +++ raw_socket_listener/tcp_message.go | 20 +++++++++++++++----- 6 files changed, 39 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index c051a841..f2bba5ce 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ drace: docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -v -race -timeout 15s dtest: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test ./... $(ARGS) -v -timeout 10s + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test ./... $(ARGS) -v -timeout 60s dcover: docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out diff --git a/http_client_test.go b/http_client_test.go index 2500c73a..d9703814 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -5,7 +5,7 @@ import ( "crypto/rand" "github.com/buger/gor/proto" "io/ioutil" - "log" + _ "log" "net" "net/http" "net/http/httptest" @@ -171,12 +171,16 @@ func TestHTTPClientServerInstantDisconnect(t *testing.T) { GETPayload := []byte("GET / HTTP/1.1\r\n\r\n") ln, _ := net.Listen("tcp", ":0") + defer ln.Close() go func() { for { - conn, _ := ln.Accept() - conn.Close() + conn, err := ln.Accept() + if err != nil { + break + } + conn.Close() wg.Done() } }() @@ -196,12 +200,13 @@ func TestHTTPClientServerNoKeepAlive(t *testing.T) { GETPayload := []byte("GET / HTTP/1.1\r\n\r\n") ln, _ := net.Listen("tcp", ":0") + defer ln.Close() go func() { for { conn, err := ln.Accept() if err != nil { - // handle error + break } buf := make([]byte, 4096) @@ -370,12 +375,7 @@ func TestHTTPClientErrors(t *testing.T) { ln2, _ := net.Listen("tcp", ":0") go func() { buf := make([]byte, 64*1024) - conn, err := ln2.Accept() - - if err != nil { - log.Println("Error while Accept()", err) - continue - } + conn, _ := ln2.Accept() conn.Read(buf) defer conn.Close() diff --git a/middleware_test.go b/middleware_test.go index bf8203c9..aab936ec 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -149,10 +149,6 @@ func TestTokenMiddleware(t *testing.T) { }) to := NewFakeSecureService(wg, func(path string, status int, tok []byte) { switch path { - case "/token": - if bytes.Equal(token, tok) { - t.Error("Tokens should not match") - } case "/secure": if status != 202 { t.Error("Server should receive valid rewritten token") @@ -164,6 +160,8 @@ func TestTokenMiddleware(t *testing.T) { quit := make(chan int) + Settings.middleware = "go run ./examples/middleware/token_modifier.go" + // Catch traffic from one service input := NewRAWInput(from, testRawExpire) @@ -172,7 +170,6 @@ func TestTokenMiddleware(t *testing.T) { Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} - Settings.middleware = "go run ./examples/middleware/token_modifier.go" // Start Gor go Start(quit) diff --git a/output_http.go b/output_http.go index cc751921..0f57ea95 100644 --- a/output_http.go +++ b/output_http.go @@ -28,6 +28,8 @@ type HTTPOutputConfig struct { OriginalHost bool Debug bool + + TrackResponses bool } // HTTPOutput plugin manage pool of workers which send request to replayed server @@ -42,6 +44,7 @@ type HTTPOutput struct { address string limit int queue chan []byte + responses chan response needWorker chan int @@ -81,6 +84,10 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer { o.elasticSearch.Init(o.config.elasticSearch) } + if len(Settings.middleware) > 0 { + o.config.TrackResponses = true + } + go o.workerMaster() return o @@ -187,7 +194,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { log.Println("Request error:", err) } - if len(Settings.middleware) > 0 { + if o.config.TrackResponses { o.responses <- response{resp, uuid, stop.UnixNano() - start.UnixNano()} } diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index eefb3e05..fc3de754 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -102,6 +102,7 @@ func (t *Listener) listen() { // We need to use channels to process each packet to avoid data races case packet := <-t.packetsChan: + // log.Println("Received packet:", packet) t.processTCPPacket(packet) } } @@ -219,10 +220,12 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { } if t.captureResponse && isIncoming { + message.mu.Lock() // If message have multiple packets, delete previous alias if len(message.packets) > 0 { delete(t.respAliases, message.ResponseAck) } + message.mu.Unlock() responseAck := packet.Seq + uint32(len(packet.Data)) t.respAliases[responseAck] = &request{message.Start, message.Ack} diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index a0b8f885..2ef83271 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "time" + "sync" ) // TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence @@ -36,6 +37,8 @@ type TCPMessage struct { delChan chan *TCPMessage expire *time.Duration + + mu sync.Mutex } // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted @@ -44,6 +47,7 @@ func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32, expire *time msg.Start = time.Now().UnixNano() msg.packetsChan = make(chan *TCPPacket) msg.delChan = delChan // used for notifying that message completed or expired + msg.timer = time.NewTimer(0) go msg.listen() @@ -66,16 +70,13 @@ func (t *TCPMessage) listen() { // Timeout notifies message to stop listening, close channel and message ready to be sent func (t *TCPMessage) Timeout() { - if t.timer != nil { - t.timer.Stop() - } - select { // In some cases Timeout can be called multiple times (do not know how yet) // Ensure that we did not close channel 2 times case packet, ok := <-t.packetsChan: if ok { t.AddPacket(packet) + t.Timeout() } else { return } @@ -112,6 +113,9 @@ func (t *TCPMessage) Size() (size int) { // AddPacket to the message and ensure packet uniqueness // TCP allows that packet can be re-send multiple times func (t *TCPMessage) AddPacket(packet *TCPPacket) { + t.mu.Lock() + defer t.mu.Unlock() + packetFound := false for _, pkt := range t.packets { @@ -124,7 +128,13 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { if packetFound { log.Println("Received packet with same sequence") } else { - t.packets = append(t.packets, packet) + // Packets not always captured in same Seq order, and sometimes we need to prepend + if len(t.packets) == 0 || packet.Seq > t.packets[len(t.packets)-1].Seq { + t.packets = append(t.packets, packet) + } else { + t.packets = append([]*TCPPacket{packet}, t.packets...) + } + t.End = time.Now().UnixNano() } From e54f77f22906275a94e4534391dc13eb3c22969b Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 19 Aug 2015 09:27:16 +0300 Subject: [PATCH 44/64] Try to return all responses --- raw_socket_listener/tcp_message.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 2ef83271..5c4cf53b 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "github.com/buger/gor/proto" "log" - "sort" "strconv" "time" "sync" @@ -84,16 +83,14 @@ func (t *TCPMessage) Timeout() { close(t.packetsChan) // Notify RAWListener that message is ready to be send to replay server // Responses without requests gets discarded - if t.IsIncoming || t.RequestStart != 0 { + if t.IsIncoming { t.delChan <- t } } } -// Bytes sorts packets in right orders and return message content +// Bytes return message content func (t *TCPMessage) Bytes() (output []byte) { - sort.Sort(sortBySeq(t.packets)) - for _, p := range t.packets { output = append(output, p.Data...) } From 6cb4c265ddfa946cb7e159d855a0b0a1e16761f5 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 19 Aug 2015 21:46:45 +0300 Subject: [PATCH 45/64] Some debugging and race fixes --- input_raw_test.go | 12 +++++++++++- middleware_test.go | 3 +++ raw_socket_listener/tcp_message.go | 21 ++++++++++++--------- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/input_raw_test.go b/input_raw_test.go index 0f30afb6..38126f14 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -11,6 +11,7 @@ import ( "os/exec" "strings" "sync" + "sync/atomic" "testing" "time" ) @@ -18,14 +19,23 @@ import ( const testRawExpire = time.Millisecond * 100 func TestRAWInput(t *testing.T) { - wg := new(sync.WaitGroup) quit := make(chan int) listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) + var respCounter, reqCounter int64 + input := NewRAWInput(listener.Addr().String(), testRawExpire) output := NewTestOutput(func(data []byte) { + if data[0] == '1' { + atomic.AddInt64(&reqCounter, 1) + } else { + atomic.AddInt64(&respCounter, 1) + } + + log.Println(reqCounter, respCounter) + wg.Done() }) diff --git a/middleware_test.go b/middleware_test.go index aab936ec..4697d092 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -19,8 +19,11 @@ type fakeServiceCb func(string, int, []byte) // Simple service that generate token on request, and require this token for accesing to secure area func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) string { active_tokens := make([]string, 0) + var mu sync.Mutex server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + mu.Lock() + defer mu.Unlock() Debug("Received request: " + req.URL.String()) switch req.URL.Path { diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 5c4cf53b..58005347 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -83,7 +83,7 @@ func (t *TCPMessage) Timeout() { close(t.packetsChan) // Notify RAWListener that message is ready to be send to replay server // Responses without requests gets discarded - if t.IsIncoming { + if t.IsIncoming || t.RequestStart != 0 { t.delChan <- t } } @@ -156,19 +156,22 @@ func (t *TCPMessage) isMultipart() bool { } payload := t.packets[0].Data - m := payload[:3] + m := payload[:4] if t.IsIncoming { // If one GET, OPTIONS, or HEAD request - if bytes.Equal(m, []byte("GET")) || bytes.Equal(m, []byte("OPT")) || bytes.Equal(m, []byte("HEA")) { + if bytes.Equal(m, []byte("GET ")) || bytes.Equal(m, []byte("OPTI")) || bytes.Equal(m, []byte("HEAD")) { return false } else { - if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 { - l, _ := strconv.Atoi(string(length)) - - // If content-length equal current body length - if l > 0 && l == len(proto.Body(payload)) { - return false + // Sometimes header comes after the body :( + if bytes.Equal(m, []byte("POST")) || bytes.Equal(m, []byte("PUT ")) || bytes.Equal(m, []byte("PATC")) { + if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 { + l, _ := strconv.Atoi(string(length)) + + // If content-length equal current body length + if l > 0 && l == len(proto.Body(payload)) { + return false + } } } } From 9550b31fdee530b1704fca57eb4905c64f6f4c08 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 19 Aug 2015 22:38:24 +0300 Subject: [PATCH 46/64] Fix plugin registration --- Makefile | 2 +- plugins.go | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index f2bba5ce..3addd1ad 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ dbench: # Used mainly for debugging, because docker container do not have access to parent machine ports drun: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-http :9000 --verbose + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw :9000 --input-http :9000 --verbose --debug drecord: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose diff --git a/plugins.go b/plugins.go index ce8e7a14..8c08e2fc 100644 --- a/plugins.go +++ b/plugins.go @@ -4,6 +4,7 @@ import ( "io" "reflect" "strings" + "time" ) // InOutPlugins struct for holding references to plugins @@ -79,13 +80,7 @@ func InitPlugins() { } for _, options := range Settings.inputRAW { - captureResponse := false - - if len(Settings.middleware) > 0 { - captureResponse = true - } - - registerPlugin(NewRAWInput, options, 0, captureResponse) + registerPlugin(NewRAWInput, options, time.Duration(0)) } for _, options := range Settings.inputTCP { From 33cdcd4d1386441c16b8345e75b715a699d570eb Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 20 Aug 2015 07:28:13 +0300 Subject: [PATCH 47/64] Use httptest package and properly close servers --- http_client_test.go | 6 +++++ input_raw_test.go | 61 +++++++++++++++++++++++---------------------- middleware_test.go | 23 +++++++++++------ output_http_test.go | 34 +++++++++---------------- 4 files changed, 64 insertions(+), 60 deletions(-) diff --git a/http_client_test.go b/http_client_test.go index d9703814..e91284e9 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -75,6 +75,7 @@ func TestHTTPClientSend(t *testing.T) { wg.Done() })) + defer server.Close() client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true}) @@ -103,6 +104,7 @@ func TestHTTPClientResponseBuffer(t *testing.T) { wg.Done() })) + defer server.Close() client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: false, ResponseBufferSize: 1024}) @@ -153,6 +155,7 @@ func TestHTTPClientHTTPSSend(t *testing.T) { wg.Done() })) + defer server.Close() client := NewHTTPClient(server.URL, &HTTPClientConfig{}) @@ -246,6 +249,7 @@ func TestHTTPClientRedirect(t *testing.T) { wg.Done() })) + defer server.Close() client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 1, Debug: false}) @@ -277,6 +281,7 @@ func TestHTTPClientRedirectLimit(t *testing.T) { wg.Done() })) + defer server.Close() client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 2, Debug: false}) @@ -300,6 +305,7 @@ func TestHTTPClientHandleHTTP10(t *testing.T) { wg.Done() })) + defer server.Close() client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true}) diff --git a/input_raw_test.go b/input_raw_test.go index 38126f14..608c06ea 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -22,11 +22,14 @@ func TestRAWInput(t *testing.T) { wg := new(sync.WaitGroup) quit := make(chan int) - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + defer origin.Close() + originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) + var respCounter, reqCounter int64 - input := NewRAWInput(listener.Addr().String(), testRawExpire) + input := NewRAWInput(originAddr, testRawExpire) output := NewTestOutput(func(data []byte) { if data[0] == '1' { atomic.AddInt64(&reqCounter, 1) @@ -42,9 +45,7 @@ func TestRAWInput(t *testing.T) { Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} - address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1) - - client := NewHTTPClient(address, &HTTPClientConfig{}) + client := NewHTTPClient(origin.URL, &HTTPClientConfig{}) time.Sleep(time.Millisecond) @@ -68,14 +69,15 @@ func TestInputRAW100Expect(t *testing.T) { fileContent, _ := ioutil.ReadFile("README.md") // Origing and Replay server initialization - origin := startHTTP(func(w http.ResponseWriter, req *http.Request) { - defer req.Body.Close() - ioutil.ReadAll(req.Body) + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + ioutil.ReadAll(r.Body) wg.Done() - }) + })) + defer origin.Close() - originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) + originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) input := NewRAWInput(originAddr, testRawExpire) @@ -92,20 +94,20 @@ func TestInputRAW100Expect(t *testing.T) { } }) - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { - defer req.Body.Close() - body, _ := ioutil.ReadAll(req.Body) + replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + body, _ := ioutil.ReadAll(r.Body) if !bytes.Equal(body, fileContent) { - buf, _ := httputil.DumpRequest(req, true) + buf, _ := httputil.DumpRequest(r, true) t.Error("Wrong POST body:", string(buf)) } wg.Done() - }) - replayAddr := listener.Addr().String() + })) + defer replay.Close() - httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{}) + httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{testOutput, httpOutput} @@ -131,31 +133,30 @@ func TestInputRAWChunkedEncoding(t *testing.T) { fileContent, _ := ioutil.ReadFile("README.md") // Origing and Replay server initialization - origin := startHTTP(func(w http.ResponseWriter, req *http.Request) { - defer req.Body.Close() - ioutil.ReadAll(req.Body) + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + ioutil.ReadAll(r.Body) wg.Done() - }) - - originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) + })) + originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) input := NewRAWInput(originAddr, testRawExpire) - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { - defer req.Body.Close() - body, _ := ioutil.ReadAll(req.Body) + replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + body, _ := ioutil.ReadAll(r.Body) if !bytes.Equal(body, fileContent) { - buf, _ := httputil.DumpRequest(req, true) + buf, _ := httputil.DumpRequest(r, true) t.Error("Wrong POST body:", string(buf)) } wg.Done() - }) - replayAddr := listener.Addr().String() + })) + defer replay.Close() - httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{Debug: true}) + httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{httpOutput} diff --git a/middleware_test.go b/middleware_test.go index 4697d092..84586548 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -17,7 +17,7 @@ import ( type fakeServiceCb func(string, int, []byte) // Simple service that generate token on request, and require this token for accesing to secure area -func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) string { +func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server { active_tokens := make([]string, 0) var mu sync.Mutex @@ -61,8 +61,7 @@ func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) string { wg.Done() })) - address := strings.Replace(server.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - return address + return server } func TestFakeSecureService(t *testing.T) { @@ -70,12 +69,13 @@ func TestFakeSecureService(t *testing.T) { wg := new(sync.WaitGroup) - addr := NewFakeSecureService(wg, func(path string, status int, resp []byte) { + server := NewFakeSecureService(wg, func(path string, status int, resp []byte) { }) + defer server.Close() wg.Add(3) - client := NewHTTPClient("http://"+addr, &HTTPClientConfig{Debug: true}) + client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true}) resp, _ = client.Get("/token") token = proto.Body(resp) @@ -102,11 +102,14 @@ func TestEchoMiddleware(t *testing.T) { w.Header().Set("RequestPath", r.URL.Path) wg.Done() })) + defer from.Close() + to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Env", "test") w.Header().Set("RequestPath", r.URL.Path) wg.Done() })) + defer to.Close() quit := make(chan int) @@ -150,6 +153,8 @@ func TestTokenMiddleware(t *testing.T) { from := NewFakeSecureService(wg, func(path string, status int, tok []byte) { time.Sleep(10 * time.Millisecond) }) + defer from.Close() + to := NewFakeSecureService(wg, func(path string, status int, tok []byte) { switch path { case "/secure": @@ -160,16 +165,18 @@ func TestTokenMiddleware(t *testing.T) { time.Sleep(10 * time.Millisecond) }) + defer to.Close() quit := make(chan int) Settings.middleware = "go run ./examples/middleware/token_modifier.go" + fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1) // Catch traffic from one service - input := NewRAWInput(from, testRawExpire) + input := NewRAWInput(fromAddr, testRawExpire) // And redirect to another - output := NewHTTPOutput(to, &HTTPOutputConfig{Debug: true}) + output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} @@ -184,7 +191,7 @@ func TestTokenMiddleware(t *testing.T) { // Should receive 2 requests from original + 2 from replayed wg.Add(4) - client := NewHTTPClient("http://"+from, &HTTPClientConfig{Debug: false}) + client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false}) // Sending traffic to original service resp, _ = client.Get("/token") diff --git a/output_http_test.go b/output_http_test.go index f4a4e6cc..b97399e3 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -3,7 +3,6 @@ package main import ( "io" "io/ioutil" - "net" "net/http" "net/http/httptest" _ "net/http/httputil" @@ -12,25 +11,13 @@ import ( "time" ) -func startHTTP(cb func(http.ResponseWriter, *http.Request)) net.Listener { - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - cb(w, r) - }) - - listener, _ := net.Listen("tcp", ":0") - - go http.Serve(listener, handler) - - return listener -} - func TestHTTPOutput(t *testing.T) { wg := new(sync.WaitGroup) quit := make(chan int) input := NewTestInput() - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if req.Header.Get("User-Agent") != "Gor" { t.Error("Wrong header") } @@ -49,13 +36,14 @@ func TestHTTPOutput(t *testing.T) { } wg.Done() - }) + })) + defer server.Close() headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")} Settings.modifierConfig = HTTPModifierConfig{headers: headers, methods: methods} - output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{Debug: true}) + output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} @@ -82,18 +70,19 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) { input := NewTestInput() - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if req.Host != "custom-host.com" { t.Error("Wrong header", req.Host) } wg.Done() - }) + })) + defer server.Close() headers := HTTPHeaders{HTTPHeader{"Host", "custom-host.com"}} Settings.modifierConfig = HTTPModifierConfig{headers: headers} - output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{Debug: false, OriginalHost: true}) + output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: false, OriginalHost: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} @@ -140,13 +129,14 @@ func BenchmarkHTTPOutput(b *testing.B) { wg := new(sync.WaitGroup) quit := make(chan int) - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { time.Sleep(50 * time.Millisecond) wg.Done() - }) + })) + defer server.Close() input := NewTestInput() - output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{}) + output := NewHTTPOutput(server.URL, &HTTPOutputConfig{}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} From 6974a6e278a0a0cb28c1b5cbbac97e8981cc1cc3 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 20 Aug 2015 08:56:10 +0300 Subject: [PATCH 48/64] Ensure that raw input gets closed --- emitter.go | 13 +++++++- input_raw.go | 18 +++++++++-- input_raw_test.go | 11 +++++-- middleware_test.go | 2 ++ raw_socket_listener/listener.go | 55 ++++++++++++++++++++++----------- 5 files changed, 75 insertions(+), 24 deletions(-) diff --git a/emitter.go b/emitter.go index 00ba070c..8ad053c2 100644 --- a/emitter.go +++ b/emitter.go @@ -32,8 +32,19 @@ func Start(stop chan int) { for { select { case <-stop: + for _, in := range Plugins.Inputs { + if c, ok := in.(io.Closer); ok { + c.Close() + } + } + + for _, out := range Plugins.Outputs { + if c, ok := out.(io.Closer); ok { + c.Close() + } + } return - case <-time.After(1 * time.Second): + case <-time.After(time.Second): } } } diff --git a/input_raw.go b/input_raw.go index 8c096c6f..7ce6a30c 100644 --- a/input_raw.go +++ b/input_raw.go @@ -13,6 +13,8 @@ type RAWInput struct { data chan *raw.TCPMessage address string expire time.Duration + quit chan bool + listener *raw.Listener } // NewRAWInput constructor for RAWInput. Accepts address with port as argument. @@ -21,6 +23,7 @@ func NewRAWInput(address string, expire time.Duration) (i *RAWInput) { i.data = make(chan *raw.TCPMessage) i.address = address i.expire = expire + i.quit = make(chan bool) go i.listen(address) @@ -56,11 +59,17 @@ func (i *RAWInput) listen(address string) { log.Fatal("input-raw: error while parsing address", err) } - listener := raw.NewListener(host, port, i.expire, true) + i.listener = raw.NewListener(host, port, i.expire, true) for { + select { + case <-i.quit: + return + default: + } + // Receiving TCPMessage object - m := listener.Receive() + m := i.listener.Receive() i.data <- m } @@ -69,3 +78,8 @@ func (i *RAWInput) listen(address string) { func (i *RAWInput) String() string { return "RAW Socket input: " + i.address } + +func (i *RAWInput) Close() { + i.listener.Close() + close(i.quit) +} diff --git a/input_raw_test.go b/input_raw_test.go index 608c06ea..4c7c1ad8 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -28,8 +28,13 @@ func TestRAWInput(t *testing.T) { var respCounter, reqCounter int64 + defer func(){ + log.Println(reqCounter, respCounter) + }() input := NewRAWInput(originAddr, testRawExpire) + defer input.Close() + output := NewTestOutput(func(data []byte) { if data[0] == '1' { atomic.AddInt64(&reqCounter, 1) @@ -37,8 +42,6 @@ func TestRAWInput(t *testing.T) { atomic.AddInt64(&respCounter, 1) } - log.Println(reqCounter, respCounter) - wg.Done() }) @@ -58,7 +61,6 @@ func TestRAWInput(t *testing.T) { } wg.Wait() - close(quit) } @@ -80,6 +82,7 @@ func TestInputRAW100Expect(t *testing.T) { originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) input := NewRAWInput(originAddr, testRawExpire) + defer input.Close() // We will use it to get content of raw HTTP request testOutput := NewTestOutput(func(data []byte) { @@ -142,6 +145,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) input := NewRAWInput(originAddr, testRawExpire) + defer input.Close() replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() @@ -200,6 +204,7 @@ func TestInputRAWLargePayload(t *testing.T) { originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) input := NewRAWInput(originAddr, testRawExpire) + defer input.Close() replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024) diff --git a/middleware_test.go b/middleware_test.go index 84586548..c94154d3 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -117,6 +117,7 @@ func TestEchoMiddleware(t *testing.T) { // Catch traffic from one service input := NewRAWInput(from.Listener.Addr().String(), testRawExpire) + defer input.Close() // And redirect to another output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false}) @@ -174,6 +175,7 @@ func TestTokenMiddleware(t *testing.T) { fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1) // Catch traffic from one service input := NewRAWInput(fromAddr, testRawExpire) + defer input.Close() // And redirect to another output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true}) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index fc3de754..65b18993 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -20,6 +20,7 @@ import ( "runtime/debug" "strconv" "time" + "strings" ) // Listener handle traffic capture @@ -50,6 +51,9 @@ type Listener struct { messageExpire time.Duration captureResponse bool + + conn net.PacketConn + quit chan bool } type request struct { @@ -58,30 +62,31 @@ type request struct { } // NewListener creates and initializes new Listener object -func NewListener(addr string, port string, expire time.Duration, captureResponse bool) (rawListener *Listener) { - rawListener = &Listener{captureResponse: captureResponse} +func NewListener(addr string, port string, expire time.Duration, captureResponse bool) (l *Listener) { + l = &Listener{captureResponse: captureResponse} - rawListener.packetsChan = make(chan *TCPPacket, 10000) - rawListener.messagesChan = make(chan *TCPMessage, 10000) - rawListener.messageDelChan = make(chan *TCPMessage, 10000) + l.packetsChan = make(chan *TCPPacket, 10000) + l.messagesChan = make(chan *TCPMessage, 10000) + l.messageDelChan = make(chan *TCPMessage, 10000) + l.quit = make(chan bool) - rawListener.messages = make(map[string]*TCPMessage) - rawListener.ackAliases = make(map[uint32]uint32) - rawListener.seqWithData = make(map[uint32]uint32) - rawListener.respAliases = make(map[uint32]*request) + l.messages = make(map[string]*TCPMessage) + l.ackAliases = make(map[uint32]uint32) + l.seqWithData = make(map[uint32]uint32) + l.respAliases = make(map[uint32]*request) - rawListener.addr = addr + l.addr = addr _port, _ := strconv.Atoi(port) - rawListener.port = uint16(_port) + l.port = uint16(_port) if expire.Nanoseconds() == 0 { expire = 2000 * time.Millisecond } - rawListener.messageExpire = expire + l.messageExpire = expire - go rawListener.listen() - go rawListener.readRAWSocket() + go l.listen() + go l.readRAWSocket() return } @@ -89,6 +94,9 @@ func NewListener(addr string, port string, expire time.Duration, captureResponse func (t *Listener) listen() { for { select { + case <-t.quit: + t.conn.Close() + return // If message ready for deletion it means that its also complete or expired by timeout case message := <-t.messageDelChan: delete(t.ackAliases, message.Ack) @@ -109,21 +117,26 @@ func (t *Listener) listen() { } func (t *Listener) readRAWSocket() { conn, e := net.ListenPacket("ip4:tcp", t.addr) + t.conn = conn if e != nil { log.Fatal(e) } - defer conn.Close() + defer t.conn.Close() for { buf := make([]byte, 64*1024) // 64kb // Note: ReadFrom receive messages without IP header - n, addr, err := conn.ReadFrom(buf) + n, addr, err := t.conn.ReadFrom(buf) if err != nil { - log.Println("Error:", err) - continue + if strings.HasSuffix(err.Error(), "closed network connection") { + return + } else { + log.Println("Raw listener error:", err) + continue + } } if n > 0 { @@ -240,3 +253,9 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { func (t *Listener) Receive() *TCPMessage { return <-t.messagesChan } + +func (t *Listener) Close() { + close(t.quit) + t.conn.Close() + return +} From 3c45595a96c4cfaacea0713a0ce34c9e6c022946 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 20 Aug 2015 09:27:44 +0300 Subject: [PATCH 49/64] More race fixes --- emitter.go | 11 ----------- input_file.go | 6 +++++- output_tcp.go | 6 +++++- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/emitter.go b/emitter.go index 8ad053c2..d767d456 100644 --- a/emitter.go +++ b/emitter.go @@ -32,17 +32,6 @@ func Start(stop chan int) { for { select { case <-stop: - for _, in := range Plugins.Inputs { - if c, ok := in.(io.Closer); ok { - c.Close() - } - } - - for _, out := range Plugins.Outputs { - if c, ok := out.(io.Closer); ok { - c.Close() - } - } return case <-time.After(time.Second): } diff --git a/input_file.go b/input_file.go index 840c5d72..24ebd2df 100644 --- a/input_file.go +++ b/input_file.go @@ -74,6 +74,10 @@ func (i *FileInput) emit() { lastTime = ts } - i.data <- buf + // scanner returs only pointer, so to remove data-race we have to allocate new array + newBuf := make([]byte, len(buf)) + copy(newBuf, buf) + + i.data <- newBuf } } diff --git a/output_tcp.go b/output_tcp.go index 96295dc8..9d46fa76 100644 --- a/output_tcp.go +++ b/output_tcp.go @@ -62,7 +62,11 @@ func (o *TCPOutput) Write(data []byte) (n int, err error) { return len(data), nil } - o.buf <- data + // We have to copy, because sending data in multiple threads + newBuf := make([]byte, len(data)) + copy(newBuf, data) + + o.buf <- newBuf if Settings.outputTCPStats { o.bufStats.Write(len(o.buf)) From 81d35086539f0303d93c711360c4ea009e34fd6c Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 20 Aug 2015 09:29:51 +0300 Subject: [PATCH 50/64] Enable race testing for travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4736e5bd..a1eaa093 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go go: 1.4.2 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s" \ No newline at end of file +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s -race" \ No newline at end of file From 76d5e22ba18b799771ebf106143889044989bbcf Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 20 Aug 2015 09:33:51 +0300 Subject: [PATCH 51/64] Enable packet debugging --- raw_socket_listener/listener.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 65b18993..b35443a2 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -110,7 +110,7 @@ func (t *Listener) listen() { // We need to use channels to process each packet to avoid data races case packet := <-t.packetsChan: - // log.Println("Received packet:", packet) + log.Println("Received packet:", packet) t.processTCPPacket(packet) } } From 9309fb86ef7cd4917f29d6ee81c43980f9fff970 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 20 Aug 2015 09:45:59 +0300 Subject: [PATCH 52/64] Reduce debugging --- raw_socket_listener/listener.go | 1 - 1 file changed, 1 deletion(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index b35443a2..54590193 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -110,7 +110,6 @@ func (t *Listener) listen() { // We need to use channels to process each packet to avoid data races case packet := <-t.packetsChan: - log.Println("Received packet:", packet) t.processTCPPacket(packet) } } From 6533b50ab08ea94d53a2be7055919d6d645881aa Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 20 Aug 2015 09:51:44 +0300 Subject: [PATCH 53/64] Travis should use 1.5 --- .travis.yml | 4 ++-- Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index a1eaa093..82c3aec2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go -go: 1.4.2 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s -race" \ No newline at end of file +go: 1.5 +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.5)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s -race" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 5cfb7835..38c5228d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -FROM google/golang:1.4 +FROM golang:1.5 -RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean +RUN cd /go/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean RUN apt-get update && apt-get install ruby vim-common -y From 16d7f23d980fc52db9972a90691359a48178a706 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 22 Aug 2015 09:41:22 +0300 Subject: [PATCH 54/64] New fixes --- Dockerfile | 11 +++++-- Makefile | 7 +++-- examples/middleware/echo.java | 30 +++++++++++++++++++ .../middleware/{echo_modifier.rb => echo.rb} | 0 .../middleware/{echo_modifier.sh => echo.sh} | 0 input_dummy.go | 13 ++++---- input_file.go | 2 ++ input_http.go | 9 ++++-- input_raw.go | 8 ++--- input_raw_test.go | 3 +- middleware_test.go | 2 +- output_http.go | 8 ++--- raw_socket_listener/listener.go | 2 +- raw_socket_listener/tcp_message.go | 2 +- 14 files changed, 73 insertions(+), 24 deletions(-) create mode 100644 examples/middleware/echo.java rename examples/middleware/{echo_modifier.rb => echo.rb} (100%) rename examples/middleware/{echo_modifier.sh => echo.sh} (100%) diff --git a/Dockerfile b/Dockerfile index 38c5228d..678c431f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,16 @@ -FROM golang:1.5 +FROM google/golang:1.4 -RUN cd /go/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean +RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean RUN apt-get update && apt-get install ruby vim-common -y +# Install Java +# RUN apt-get install -y software-properties-common python-software-properties +# RUN add-apt-repository -y ppa:webupd8team/java +# RUN apt-get update -y +# RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections +# RUN apt-get install -y oracle-java8-installer + WORKDIR /gopath/src/github.com/buger/gor/ ADD . /gopath/src/github.com/buger/gor/ diff --git a/Makefile b/Makefile index 3addd1ad..29de44d8 100644 --- a/Makefile +++ b/Makefile @@ -37,10 +37,13 @@ dbench: # Used mainly for debugging, because docker container do not have access to parent machine ports drun: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw :9000 --input-http :9000 --verbose --debug + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw :9000 --input-http :9000 --verbose --debug --middleware "./examples/middleware/echo.sh" + +drun-2: + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-file="./fixtures/requests.gor" --output-dummy --verbose --debug --middleware "./examples/middleware/echo.sh" drecord: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose --debug dreplay: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-file=requests.bin --output-tcp=:9000 --verbose -h diff --git a/examples/middleware/echo.java b/examples/middleware/echo.java new file mode 100644 index 00000000..e37cf723 --- /dev/null +++ b/examples/middleware/echo.java @@ -0,0 +1,30 @@ +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; + +import org.apache.commons.io.IOUtils; + +public class Echo { + public static void main(String[] args) { + if(args != null){ + for(String arg : args){ + System.out.println(arg); + } + + } + + BufferedReader stdin = new BufferedReader(new InputStreamReader( + System.in)); + String line = null; + + try { + while ((line = stdin.readLine()) != null) { + + System.out.println(line); + + } + } catch (IOException e) { + IOUtils.closeQuietly(stdin); + } + } +} \ No newline at end of file diff --git a/examples/middleware/echo_modifier.rb b/examples/middleware/echo.rb similarity index 100% rename from examples/middleware/echo_modifier.rb rename to examples/middleware/echo.rb diff --git a/examples/middleware/echo_modifier.sh b/examples/middleware/echo.sh similarity index 100% rename from examples/middleware/echo_modifier.sh rename to examples/middleware/echo.sh diff --git a/input_dummy.go b/input_dummy.go index 4c148303..17907d37 100644 --- a/input_dummy.go +++ b/input_dummy.go @@ -22,11 +22,9 @@ func NewDummyInput(options string) (di *DummyInput) { func (i *DummyInput) Read(data []byte) (int, error) { buf := <-i.data - header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano()) - copy(data[0:len(header)], header) - copy(data[len(header):], buf) + copy(data, buf) - return len(buf) + len(header), nil + return len(buf), nil } func (i *DummyInput) emit() { @@ -35,7 +33,12 @@ func (i *DummyInput) emit() { for { select { case <-ticker.C: - i.data <- []byte("POST /pub/WWW/å HTTP/1.1\nHost: www.w3.org\r\n\r\na=1&b=2") + uuid := uuid() + reqh := payloadHeader(RequestPayload, uuid, time.Now().UnixNano()) + i.data <- append(reqh, []byte("POST /pub/WWW/å HTTP/1.1\nHost: www.w3.org\r\nContent-Length: 7\r\n\r\na=1&b=2")...) + + resh := payloadHeader(ResponsePayload, uuid, 1) + i.data <- append(resh, []byte("HTTP/1.1 200 OK\r\n\r\n")...) } } } diff --git a/input_file.go b/input_file.go index 24ebd2df..60ec9580 100644 --- a/input_file.go +++ b/input_file.go @@ -80,4 +80,6 @@ func (i *FileInput) emit() { i.data <- newBuf } + + log.Println("FileInput: end of file") } diff --git a/input_http.go b/input_http.go index 5150c4e0..f7717c5e 100644 --- a/input_http.go +++ b/input_http.go @@ -5,6 +5,7 @@ import ( "net" "net/http" "net/http/httputil" + "time" ) // HTTPInput used for sending requests to Gor via http @@ -27,9 +28,13 @@ func NewHTTPInput(address string) (i *HTTPInput) { func (i *HTTPInput) Read(data []byte) (int, error) { buf := <-i.data - copy(data, buf) - return len(buf), nil + header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano()) + + copy(data[0:len(header)], header) + copy(data[len(header):], buf) + + return len(buf) + len(header), nil } func (i *HTTPInput) handler(w http.ResponseWriter, r *http.Request) { diff --git a/input_raw.go b/input_raw.go index 7ce6a30c..ed147078 100644 --- a/input_raw.go +++ b/input_raw.go @@ -10,10 +10,10 @@ import ( // RAWInput used for intercepting traffic for given address type RAWInput struct { - data chan *raw.TCPMessage - address string - expire time.Duration - quit chan bool + data chan *raw.TCPMessage + address string + expire time.Duration + quit chan bool listener *raw.Listener } diff --git a/input_raw_test.go b/input_raw_test.go index 4c7c1ad8..fbe47e49 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -26,9 +26,8 @@ func TestRAWInput(t *testing.T) { defer origin.Close() originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - var respCounter, reqCounter int64 - defer func(){ + defer func() { log.Println(reqCounter, respCounter) }() diff --git a/middleware_test.go b/middleware_test.go index c94154d3..8474eeb0 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -113,7 +113,7 @@ func TestEchoMiddleware(t *testing.T) { quit := make(chan int) - Settings.middleware = "./examples/middleware/echo_modifier.sh" + Settings.middleware = "./examples/middleware/echo.sh" // Catch traffic from one service input := NewRAWInput(from.Listener.Addr().String(), testRawExpire) diff --git a/output_http.go b/output_http.go index 0f57ea95..0528e38c 100644 --- a/output_http.go +++ b/output_http.go @@ -41,9 +41,9 @@ type HTTPOutput struct { // aligned at 64bit. See https://github.com/golang/go/issues/599 activeWorkers int64 - address string - limit int - queue chan []byte + address string + limit int + queue chan []byte responses chan response @@ -173,7 +173,7 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) { func (o *HTTPOutput) Read(data []byte) (int, error) { resp := <-o.responses - Debug("[OUTPUT-HTTP] Received response", string(resp.payload)) + Debug("[OUTPUT-HTTP] Received response:", string(resp.payload)) header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime) copy(data[0:len(header)], header) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 54590193..4fe232df 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -19,8 +19,8 @@ import ( "net" "runtime/debug" "strconv" - "time" "strings" + "time" ) // Listener handle traffic capture diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 58005347..cabac1d2 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -7,8 +7,8 @@ import ( "github.com/buger/gor/proto" "log" "strconv" - "time" "sync" + "time" ) // TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence From cbc27bd662f03a365eaf8bbfa96e444819d70d1e Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 22 Aug 2015 09:48:10 +0300 Subject: [PATCH 55/64] More debugging --- input_raw_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/input_raw_test.go b/input_raw_test.go index fbe47e49..8c3dab5f 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -27,9 +27,6 @@ func TestRAWInput(t *testing.T) { originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) var respCounter, reqCounter int64 - defer func() { - log.Println(reqCounter, respCounter) - }() input := NewRAWInput(originAddr, testRawExpire) defer input.Close() @@ -41,6 +38,8 @@ func TestRAWInput(t *testing.T) { atomic.AddInt64(&respCounter, 1) } + log.Println(reqCounter, respCounter) + wg.Done() }) From 02bf7a86de102473bc4566ebdc2f56618c082e8f Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 22 Aug 2015 09:58:52 +0300 Subject: [PATCH 56/64] Rollback to 1.4.2 in travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 82c3aec2..a1eaa093 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go -go: 1.5 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.5)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s -race" \ No newline at end of file +go: 1.4.2 +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s -race" \ No newline at end of file From d5a1eb0b9df45faa6d6abb076e37d2762874ebe9 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 22 Aug 2015 11:37:01 +0300 Subject: [PATCH 57/64] Increase input raw timeout --- input_raw_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/input_raw_test.go b/input_raw_test.go index 8c3dab5f..ad2a6900 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -16,7 +16,7 @@ import ( "time" ) -const testRawExpire = time.Millisecond * 100 +const testRawExpire = time.Millisecond * 200 func TestRAWInput(t *testing.T) { wg := new(sync.WaitGroup) From f3ea30c6df9e7dd9100eff322a15ea3a8296cbe3 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 23 Aug 2015 12:45:28 +0300 Subject: [PATCH 58/64] Add example of java echo middleware --- .gitignore | 2 ++ Dockerfile | 19 ++++++++++++------- Makefile | 2 +- examples/middleware/echo.java | 2 +- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 3c34e46e..0acc83e1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ *.bin *.gz + +*.class diff --git a/Dockerfile b/Dockerfile index 678c431f..bf86e03b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,16 +4,21 @@ RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean RUN apt-get update && apt-get install ruby vim-common -y -# Install Java -# RUN apt-get install -y software-properties-common python-software-properties -# RUN add-apt-repository -y ppa:webupd8team/java -# RUN apt-get update -y -# RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections -# RUN apt-get install -y oracle-java8-installer +# Install Java for middleware testing +RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list +RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 +RUN apt-get update -y +RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections +RUN apt-get install oracle-java8-installer -y -WORKDIR /gopath/src/github.com/buger/gor/ +RUN wget http://apache-mirror.rbc.ru/pub/apache//commons/io/binaries/commons-io-2.4-bin.tar.gz -P /tmp +RUN tar xzf /tmp/commons-io-2.4-bin.tar.gz -C /tmp +WORKDIR /gopath/src/github.com/buger/gor/ ADD . /gopath/src/github.com/buger/gor/ +RUN javac -cp /tmp/commons-io-2.4/commons-io-2.4.jar ./examples/middleware/echo.java + RUN go get -u github.com/golang/lint/golint RUN go get \ No newline at end of file diff --git a/Makefile b/Makefile index 29de44d8..339df036 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ drun: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw :9000 --input-http :9000 --verbose --debug --middleware "./examples/middleware/echo.sh" drun-2: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-file="./fixtures/requests.gor" --output-dummy --verbose --debug --middleware "./examples/middleware/echo.sh" + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-file="./fixtures/requests.gor" --output-dummy=0 --verbose --debug --middleware "java -cp ./examples/middleware echo" drecord: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose --debug diff --git a/examples/middleware/echo.java b/examples/middleware/echo.java index e37cf723..707193eb 100644 --- a/examples/middleware/echo.java +++ b/examples/middleware/echo.java @@ -4,7 +4,7 @@ import org.apache.commons.io.IOUtils; -public class Echo { +public class echo { public static void main(String[] args) { if(args != null){ for(String arg : args){ From f0b96af420a636aeae36b9afc918d9645a417793 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 23 Aug 2015 13:07:29 +0300 Subject: [PATCH 59/64] Add protection for malformed requests --- http_modifier.go | 4 ++++ output_http.go | 8 +++++++- proto/proto.go | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/http_modifier.go b/http_modifier.go index 14cdf859..fbca0c32 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -30,6 +30,10 @@ func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier { } func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { + if !proto.IsHTTPPayload(payload) { + return payload + } + if len(m.config.methods) > 0 { method := proto.Method(payload) diff --git a/output_http.go b/output_http.go index 0528e38c..cafe6376 100644 --- a/output_http.go +++ b/output_http.go @@ -5,6 +5,7 @@ import ( "log" "sync/atomic" "time" + "github.com/buger/gor/proto" ) const initialDynamicWorkers = 10 @@ -186,8 +187,13 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { meta := payloadMeta(request) uuid := meta[1] + body := payloadBody(request) + if !proto.IsHTTPPayload(body) { + return + } + start := time.Now() - resp, err := client.Send(payloadBody(request)) + resp, err := client.Send(body) stop := time.Now() if err != nil { diff --git a/proto/proto.go b/proto/proto.go index da53cc09..549cec3e 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -208,3 +208,18 @@ func Method(payload []byte) []byte { func Status(payload []byte) []byte { return Path(payload) } + +var httpMethods []string = []string{ + "GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN", +} + +func IsHTTPPayload(payload []byte) bool { + method := string(payload[0:4]) + + for _, m := range httpMethods { + if method == m { + return true + } + } + return false +} From 929e02790745042fb92aad88716a442d5021b565 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 23 Aug 2015 13:08:51 +0300 Subject: [PATCH 60/64] small formatting change --- output_http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/output_http.go b/output_http.go index cafe6376..9f82d37e 100644 --- a/output_http.go +++ b/output_http.go @@ -1,11 +1,11 @@ package main import ( + "github.com/buger/gor/proto" "io" "log" "sync/atomic" "time" - "github.com/buger/gor/proto" ) const initialDynamicWorkers = 10 From 3c81723ac32ae103a3f09f42fb711fb9a8deeb0c Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 24 Aug 2015 08:58:57 +0300 Subject: [PATCH 61/64] Increase timeout for large payload --- input_raw_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/input_raw_test.go b/input_raw_test.go index ad2a6900..79b04b12 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -179,6 +179,9 @@ func TestInputRAWChunkedEncoding(t *testing.T) { } func TestInputRAWLargePayload(t *testing.T) { + Settings.verbose = true + Settings.debug = true + wg := new(sync.WaitGroup) quit := make(chan int) @@ -201,7 +204,7 @@ func TestInputRAWLargePayload(t *testing.T) { })) originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire) + input := NewRAWInput(originAddr, time.Second) defer input.Close() replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { From fe91a20629d8dd78c0ebecd3e63a8bbc084cc874 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 24 Aug 2015 09:24:31 +0300 Subject: [PATCH 62/64] fix race --- input_raw_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/input_raw_test.go b/input_raw_test.go index 79b04b12..9305724f 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -179,9 +179,6 @@ func TestInputRAWChunkedEncoding(t *testing.T) { } func TestInputRAWLargePayload(t *testing.T) { - Settings.verbose = true - Settings.debug = true - wg := new(sync.WaitGroup) quit := make(chan int) From 43262f333677c7af044528382f589317ec455eff Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 24 Aug 2015 09:32:31 +0300 Subject: [PATCH 63/64] Increase timeouts --- input_raw_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/input_raw_test.go b/input_raw_test.go index 9305724f..7a133730 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -79,7 +79,7 @@ func TestInputRAW100Expect(t *testing.T) { originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire) + input := NewRAWInput(originAddr, time.Second) defer input.Close() // We will use it to get content of raw HTTP request @@ -142,7 +142,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { })) originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire) + input := NewRAWInput(originAddr, time.Second) defer input.Close() replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { From abde6ecfcb1c852a992b04422a2f34e0204c877b Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 24 Aug 2015 09:40:15 +0300 Subject: [PATCH 64/64] less verbose output --- input_raw_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/input_raw_test.go b/input_raw_test.go index 7a133730..0d754437 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -38,7 +38,9 @@ func TestRAWInput(t *testing.T) { atomic.AddInt64(&respCounter, 1) } - log.Println(reqCounter, respCounter) + if Settings.debug { + log.Println(reqCounter, respCounter) + } wg.Done() })