From f21c825b7454f980ac49c201bcc6d64438ae3887 Mon Sep 17 00:00:00 2001 From: Rath Rene Date: Sun, 1 Oct 2023 20:56:50 +0200 Subject: [PATCH] implemented http-proxy --- README.md | 6 +- config_example.yml | 1 + docs/source/info/getting_started.rst | 45 +++++++ docs/source/info/intro.rst | 33 ++--- docs/source/info/redirect.rst | 2 +- lib/cnf/cnf_file/rules_parse.go | 28 ++--- lib/cnf/cnf_file/validate.go | 4 +- lib/cnf/config.go | 22 ++-- lib/cnf/hardcoded.go | 29 +++++ lib/cnf/unmarshal.go | 11 ++ lib/cnf/unmarshal_test.go | 6 + lib/main/metrics.go | 8 +- lib/main/service.go | 76 ++++-------- lib/proc/filter/main.go | 22 +++- lib/proc/filter/match.go | 6 +- lib/proc/fwd/http.go | 161 +++++++++++++++++++++++++ lib/proc/fwd/main.go | 22 +++- lib/proc/fwd/util.go | 46 +++++++ lib/proc/fwd/util_test.go | 81 +++++++++++++ lib/proc/http/main.go | 12 -- lib/proc/parse/http.go | 19 +++ lib/proc/parse/pkt.go | 15 ++- lib/proc/parse/{helpers.go => util.go} | 18 +++ lib/rcv/http.go | 38 +++--- lib/rcv/main.go | 8 +- lib/rcv/server.go | 11 +- lib/send/main.go | 12 ++ lib/send/tcp.go | 18 ++- lib/u/helpers.go | 64 ++++++++++ lib/u/helpers_test.go | 40 ++++++ 30 files changed, 692 insertions(+), 172 deletions(-) create mode 100644 lib/proc/fwd/http.go create mode 100644 lib/proc/fwd/util.go create mode 100644 lib/proc/fwd/util_test.go delete mode 100644 lib/proc/http/main.go rename lib/proc/parse/{helpers.go => util.go} (61%) create mode 100644 lib/u/helpers_test.go diff --git a/README.md b/README.md index d05a365..5095b91 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Its focus is set on transparent security filtering. - [ ] Proxy-Protocol - - [ ] HTTP Proxy + - [x] HTTP Proxy - [ ] HTTPS Proxy @@ -33,7 +33,7 @@ Its focus is set on transparent security filtering. - [x] TCP - - [ ] HTTP + - [x] HTTP - [x] TLS @@ -55,7 +55,7 @@ Its focus is set on transparent security filtering. - [ ] Listener-Specific - [ ] Proxy-Protocol - - [ ] HTTP Proxy + - [x] HTTP Proxy - [ ] SOCKS5 Proxy - [ ] DNS diff --git a/config_example.yml b/config_example.yml index eec61a4..140afc0 100644 --- a/config_example.yml +++ b/config_example.yml @@ -24,6 +24,7 @@ service: interceptPublic: '/tmp/calamary.subca.crt' interceptPrivate: '/tmp/calamary.subca.key' + dnsNameservers: ['1.1.1.1', '8.8.8.8'] debug: false timeout: connect: 2000 diff --git a/docs/source/info/getting_started.rst b/docs/source/info/getting_started.rst index c79c670..f8f0e0e 100644 --- a/docs/source/info/getting_started.rst +++ b/docs/source/info/getting_started.rst @@ -31,6 +31,50 @@ Config-validation only: /usr/bin/calamary -v +Modes +##### + +Transparent +=========== + +**State:** Implemented/Testing + +Calamary focuses on transparent traffic interception. + +You will have to redirect the traffic: :ref:`Redirect ` + +This mode will work for TCP & UDP. + +HTTP/HTTPS Proxy +================ + +**State:** Implemented/Testing + +You can also choose to let Calamary act as a HTTP/S proxy. + +One commonly uses this feature if only some applications should send their traffic over the proxy. + +This mode only supports TCP. + +Note: Calamary uses TLS-SNI > Host-Header to find its actual target host. It will also check all IPs (IPv6 > IPv4) that are returned by the DNS query for their reachability, before establishing a connection. + +SOCKS5 Proxy +============ + +**State:** not implemented + +Like HTTP/S proxy, but it works for UDP as well. + +Proxy-Protocol +============== + +**State:** in development + +You can use the proxy-protcol mode if you want to send traffic from remote systems over the proxy. + +The commonly used `proxy-protocol `_ preserves the original source- & destination while minimizing overhead. + + Configuration ############# @@ -60,6 +104,7 @@ Basic config example: interceptPublic: '/tmp/calamary.subca.crt' interceptPrivate: '/tmp/calamary.subca.key' + dnsNameservers: ['1.1.1.1', '8.8.8.8'] debug: false timeout: # ms connect: 2000 diff --git a/docs/source/info/intro.rst b/docs/source/info/intro.rst index e8d9b23..633715d 100644 --- a/docs/source/info/intro.rst +++ b/docs/source/info/intro.rst @@ -11,22 +11,26 @@ Intro Calamary is a `squid `_-like proxy. -Its focus is set on **security filtering for HTTPS**. +Its focus is set on **security filtering for HTTPS/TLS**. -**It will not**: - -* act as caching proxy -* act as reverse proxy +The ruleset should be logical, transparent & easy to understand. **Features**: -* basic traffic filtering - see :ref:`Rules ` +* support for mainstream :ref:`proxy modes ` +* filtering ruleset - see :ref:`Rules ` + + * ability to filter on protocol-basis + * ability to enforce TLS (*deny any unencrypted connections*) + * certificate verification -* enforce TLS (*deny any unencrypted connections*) * detect plain HTTP and respond with generic HTTPS-redirect -* intercept-, http-proxy and proxy-proto-modes - * support for `proxy-protocol `_ +**It will not**: + +* act as caching proxy +* act as reverse proxy +* implement edge-case workarounds for unencrypted protocols Getting Started ############### @@ -77,17 +81,18 @@ I would much preferr a keep-it-simple approach. Even if that means that some nic How? #### -* Use TLS-SNI as target instead of HTTP Host-Header +* Plaintext HTTP is not that common anymore. + We are using TLS-SNI > Host-Header to resolve the target. -* Optionally use additional DNS-based verfication if TTL > 3 min + Plain HTTP is unsecure by default. So we won't check for Host-Header mangling. + The ruleset is applied 'postrouting' (*IP/Net matching*) and Host-Header domains are ignored by the ruleset. -* Whenever it is not possible to route the traffic through the proxy.. - To overcome the DNAT restriction, of losing the real target IP, the proxy will have a lightweight forwarder mode: +* Whenever it is not possible to route the traffic through the proxy.. - |proxy_forwarder| + To overcome the DNAT restriction, of losing the real target IP, there will be a :ref:`Redirector `! * **Transparent traffic interception will be the focus**. diff --git a/docs/source/info/redirect.rst b/docs/source/info/redirect.rst index 8c883ad..71675b8 100644 --- a/docs/source/info/redirect.rst +++ b/docs/source/info/redirect.rst @@ -15,7 +15,7 @@ You may want/need to redirect traffic to the proxy's listeners for some use-case This is essential for using the :code:`transparent` mode. -For modes like :code:`proxyproto`, :code:`http`, :code:`https` or :code:`socks5` this is not necessary. (*but it's also possible using the :ref:`Redirector `*) +For modes like :code:`proxyproto`, :code:`http`, :code:`https` or :code:`socks5` this is not necessary. (*but it's also possible using the* :ref:`Redirector `) You will have to choose between using **DNAT** and **TPROXY** to redirect the traffic on firewall-level. diff --git a/lib/cnf/cnf_file/rules_parse.go b/lib/cnf/cnf_file/rules_parse.go index a62cae2..aaa22dc 100644 --- a/lib/cnf/cnf_file/rules_parse.go +++ b/lib/cnf/cnf_file/rules_parse.go @@ -15,12 +15,10 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) { var v cnf.Var var vf bool var vn bool - var value string // todo: move duplicate lines into sub-functions - for i := range rawRules { - ruleRaw := rawRules[i] + for _, ruleRaw := range rawRules { rule := cnf.Rule{ Action: meta.RuleAction(ruleRaw.Action), } @@ -31,8 +29,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) { rule.Match.SrcNet = []*net.IPNet{} rule.Match.SrcNetN = []*net.IPNet{} } - for i2 := range ruleRaw.Match.SrcNet { - value = ruleRaw.Match.SrcNet[i2] + for _, value := range ruleRaw.Match.SrcNet { vf, vn, v = usedVar(value) if vf { for i3 := range v.Value { @@ -56,8 +53,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) { rule.Match.DestNet = []*net.IPNet{} rule.Match.DestNetN = []*net.IPNet{} } - for i2 := range ruleRaw.Match.DestNet { - value = ruleRaw.Match.DestNet[i2] + for _, value := range ruleRaw.Match.DestNet { vf, vn, v = usedVar(value) if vf { for i3 := range v.Value { @@ -81,8 +77,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) { rule.Match.SrcPort = []uint16{} rule.Match.SrcPortN = []uint16{} } - for i2 := range ruleRaw.Match.SrcPort { - value = ruleRaw.Match.SrcPort[i2] + for _, value := range ruleRaw.Match.SrcPort { vf, vn, v = usedVar(value) if vf { for i3 := range v.Value { @@ -106,8 +101,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) { rule.Match.DestPort = []uint16{} rule.Match.DestPortN = []uint16{} } - for i2 := range ruleRaw.Match.DestPort { - value = ruleRaw.Match.DestPort[i2] + for _, value := range ruleRaw.Match.DestPort { vf, vn, v = usedVar(value) if vf { for i3 := range v.Value { @@ -131,8 +125,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) { rule.Match.ProtoL3 = []meta.Proto{} rule.Match.ProtoL3N = []meta.Proto{} } - for i2 := range ruleRaw.Match.ProtoL3 { - value = ruleRaw.Match.ProtoL3[i2] + for _, value := range ruleRaw.Match.ProtoL3 { vf, vn, v = usedVar(value) if vf { for i3 := range v.Value { @@ -156,8 +149,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) { rule.Match.ProtoL4 = []meta.Proto{} rule.Match.ProtoL4N = []meta.Proto{} } - for i2 := range ruleRaw.Match.ProtoL4 { - value = ruleRaw.Match.ProtoL4[i2] + for _, value := range ruleRaw.Match.ProtoL4 { vf, vn, v = usedVar(value) if vf { for i3 := range v.Value { @@ -181,8 +173,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) { rule.Match.ProtoL5 = []meta.Proto{} rule.Match.ProtoL5N = []meta.Proto{} } - for i2 := range ruleRaw.Match.ProtoL5 { - value = ruleRaw.Match.ProtoL5[i2] + for _, value := range ruleRaw.Match.ProtoL5 { vf, vn, v = usedVar(value) if vf { for i3 := range v.Value { @@ -205,8 +196,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) { if len(ruleRaw.Match.Domains) > 0 { rule.Match.Domains = []string{} } - for i2 := range ruleRaw.Match.Domains { - value = ruleRaw.Match.Domains[i2] + for _, value := range ruleRaw.Match.Domains { vf, vn, v = usedVar(value) if vf { for i3 := range v.Value { diff --git a/lib/cnf/cnf_file/validate.go b/lib/cnf/cnf_file/validate.go index a8e3c50..3b17e26 100644 --- a/lib/cnf/cnf_file/validate.go +++ b/lib/cnf/cnf_file/validate.go @@ -15,8 +15,8 @@ import ( ) func validateConfig(newCnf cnf.Config, fail bool) bool { - for i := range newCnf.Service.Listen { - if !validateListener(newCnf.Service.Listen[i], fail) { + for _, ln := range newCnf.Service.Listen { + if !validateListener(ln, fail) { return false } } diff --git a/lib/cnf/config.go b/lib/cnf/config.go index 4fd643f..6c66f77 100644 --- a/lib/cnf/config.go +++ b/lib/cnf/config.go @@ -15,12 +15,13 @@ type Config struct { } type ServiceConfig struct { - Timeout ServiceTimeout `yaml:"timeout"` - Listen []ServiceListener `yaml:"listen"` - Certs ServiceCertificates `yaml:"certs"` - Output ServiceOutput `yaml:"output"` - Debug bool `yaml:"debug" default:"false"` - Metrics ServiceMetrics `yaml:"metrics"` + Timeout ServiceTimeout `yaml:"timeout"` + Listen []ServiceListener `yaml:"listen"` + Certs ServiceCertificates `yaml:"certs"` + Output ServiceOutput `yaml:"output"` + Debug bool `yaml:"debug" default:"false"` + Metrics ServiceMetrics `yaml:"metrics"` + DnsNameservers YamlStringArray `yaml:"dnsNameservers" default:"[\"1.1.1.1\", \"8.8.8.8\"]"` } // todo: implement default listen-ips = localhost @@ -37,9 +38,11 @@ type ServiceListener struct { } type ServiceTimeout struct { - Connect uint `yaml:"connect" default:"2000"` // dial - Process uint `yaml:"process" default:"1000"` // parsing packet - Idle uint `yaml:"idle" default:"30000"` // close connection if no data was sent or received + Connect uint `yaml:"connect" default:"2000"` // dial + Process uint `yaml:"process" default:"1000"` // parsing packet + Idle uint `yaml:"idle" default:"30000"` // close connection if no data was sent or received + Probe uint `yaml:"probe" default:"500"` // check if target port is reachable before establishing connection + DnsLookup uint `yaml:"dnsLookup" default:"200"` } var DefaultConnectRetryWait = uint(1000) // ms @@ -53,6 +56,7 @@ type ServiceOutput struct { } var DefaultMetricsPort = uint16(9512) +var InboundRetries = 3 type ServiceMetrics struct { Enabled bool `yaml:"enabled" default:"false"` diff --git a/lib/cnf/hardcoded.go b/lib/cnf/hardcoded.go index 2c2c957..4a60889 100644 --- a/lib/cnf/hardcoded.go +++ b/lib/cnf/hardcoded.go @@ -1,6 +1,7 @@ package cnf import ( + "net" "time" ) @@ -16,3 +17,31 @@ const ( ) var ConfigFileAbs string = "/etc/calamary/config.yml" + +var NetForwardDeny []*net.IPNet + +func InitNetForwardDeny() { + _, localhost1, _ := net.ParseCIDR("127.0.0.0/8") + _, localhost2, _ := net.ParseCIDR("::1/128") + _, localhost3, _ := net.ParseCIDR("::/128") + _, linklocal1, _ := net.ParseCIDR("169.254.0.0/16") + _, linklocal2, _ := net.ParseCIDR("fe80::/10") + _, linklocal3, _ := net.ParseCIDR("fc00::/7") + _, multicast1, _ := net.ParseCIDR("224.0.0.0/4") + _, multicast2, _ := net.ParseCIDR("ff00::/8") + _, broadcast1, _ := net.ParseCIDR("255.255.255.255/32") + _, blackhole1, _ := net.ParseCIDR("100::/64") + + NetForwardDeny = []*net.IPNet{ + localhost1, + localhost2, + localhost3, + linklocal1, + linklocal2, + linklocal3, + multicast1, + multicast2, + broadcast1, + blackhole1, + } +} diff --git a/lib/cnf/unmarshal.go b/lib/cnf/unmarshal.go index 6fc16a9..ecbeb47 100644 --- a/lib/cnf/unmarshal.go +++ b/lib/cnf/unmarshal.go @@ -36,6 +36,17 @@ func (s *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { return nil } +func (s *ServiceConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + defaults.Set(s) + + type plain ServiceConfig + if err := unmarshal((*plain)(s)); err != nil { + return err + } + + return nil +} + func (s *ServiceListener) UnmarshalYAML(unmarshal func(interface{}) error) error { defaults.Set(s) diff --git a/lib/cnf/unmarshal_test.go b/lib/cnf/unmarshal_test.go index ccbdfc8..dca2476 100644 --- a/lib/cnf/unmarshal_test.go +++ b/lib/cnf/unmarshal_test.go @@ -1,6 +1,7 @@ package cnf import ( + "fmt" "os" "path/filepath" "runtime" @@ -60,4 +61,9 @@ func TestConfigDefaults(t *testing.T) { if cnf2.Service.Metrics.Port != 9512 { t.Errorf("Unmarshal defaults-file #6 (%+v)", cnf2.Service.Metrics) } + if len(cnf2.Service.DnsNameservers) != 2 || + fmt.Sprintf("%v", cnf2.Service.DnsNameservers[0]) != "1.1.1.1" || + fmt.Sprintf("%v", cnf2.Service.DnsNameservers[1]) != "8.8.8.8" { + t.Errorf("Unmarshal defaults-file #7 (%+v)", cnf2.Service.DnsNameservers) + } } diff --git a/lib/main/metrics.go b/lib/main/metrics.go index 1defd43..eb8c450 100644 --- a/lib/main/metrics.go +++ b/lib/main/metrics.go @@ -33,8 +33,8 @@ func startPrometheusExporter() { if cnf.Metrics() { log.Info("service", "Starting prometheus metrics-exporter") - for i := range metricFuncs { - prometheus.Register(metricFuncs[i]) + for _, mf := range metricFuncs { + prometheus.Register(mf) } metricsSrv := http.NewServeMux() @@ -43,8 +43,8 @@ func startPrometheusExporter() { http.ListenAndServe(fmt.Sprintf("127.0.0.1:%v", cnf.C.Service.Metrics.Port), metricsSrv) http.ListenAndServe(fmt.Sprintf("[::1]:%v", cnf.C.Service.Metrics.Port), metricsSrv) - for i := range metricFuncs { - prometheus.MustRegister(metricFuncs[i]) + for _, mf := range metricFuncs { + prometheus.MustRegister(mf) } } } diff --git a/lib/main/service.go b/lib/main/service.go index 2a01ec4..98478df 100644 --- a/lib/main/service.go +++ b/lib/main/service.go @@ -46,23 +46,16 @@ func (svc *service) signalHandler() { func (svc *service) start() { svc.servers = rcv.BuildServers() - for i := range svc.servers { - listener := svc.servers[i] - go svc.serve(listener) + for _, srv := range svc.servers { + go svc.serve(srv) } log.Info("service", "Started") } func (svc *service) shutdown(cancel context.CancelFunc) { cancel() - for i := range svc.servers { - server := svc.servers[i] - if u.IsIn(string(server.Cnf.Mode), []string{"http", "https"}) { - server.HttpServer.Close() - time.Sleep(time.Millisecond * 500) - } else { - server.Listener.Close() - } + for _, srv := range svc.servers { + srv.Listener.Close() } log.Info("service", "Stopped") os.Exit(0) @@ -74,53 +67,28 @@ func (svc *service) shutdown(cancel context.CancelFunc) { } func (svc *service) serve(srv rcv.Server) (err error) { - // log.Info("service", fmt.Sprintf("Serving %s://%s", ln.Addr().Network(), ln.Addr().String())) - - if srv.Cnf.Mode == meta.ListenModeHttp { - err = srv.HttpServer.ListenAndServe() - if err != nil && !strings.Contains(fmt.Sprintf("%v", err), "Server closed") { - log.ErrorS("service", fmt.Sprintf("HTTP server failure: %v", err)) - return err - } - - } else if srv.Cnf.Mode == meta.ListenModeHttps { - err = srv.HttpServer.ListenAndServeTLS( - cnf.C.Service.Certs.ServerPublic, - cnf.C.Service.Certs.ServerPrivate, - ) - if err != nil && !strings.Contains(fmt.Sprintf("%v", err), "Server closed") { - log.ErrorS("service", fmt.Sprintf("HTTPS server failure: %v", err)) - return err - } - - } else { - var tempDelay time.Duration - - for { - conn, err := srv.Listener.Accept() - if err != nil { - if _, ok := err.(net.Error); ok { - if !strings.Contains(fmt.Sprintf("%v", err), "use of closed network connection") { - if tempDelay == 0 { - tempDelay = 1 * time.Second - } else { - tempDelay *= 2 - } - if max := 5 * time.Second; tempDelay > max { - tempDelay = max - } - log.Warn("service", fmt.Sprintf("Error: %v, retrying in %v", err, tempDelay)) - time.Sleep(tempDelay) - continue - } + for { + conn, err := srv.Listener.Accept() + if err != nil { + if _, ok := err.(net.Error); ok { + if !strings.Contains(fmt.Sprintf("%v", err), "use of closed network connection") { + // todo: retries + time.Sleep(u.Timeout(cnf.DefaultConnectRetryWait)) + continue } - return err } - tempDelay = 0 - log.Debug("service", fmt.Sprintf("Accept: %s://%s", srv.Listener.Addr().Network(), srv.Listener.Addr().String())) + return err + } + log.Debug("service", fmt.Sprintf("Accept: %s://%s", srv.Listener.Addr().Network(), srv.Listener.Addr().String())) + if isModeHttp(srv.Cnf.Mode) { + go fwd.ForwardHttp(srv.Cnf, srv.L4Proto, conn) + } else { go fwd.Forward(srv.Cnf, srv.L4Proto, conn) } } - return nil +} + +func isModeHttp(mode meta.ListenMode) bool { + return mode == meta.ListenModeHttp || mode == meta.ListenModeHttps } diff --git a/lib/proc/filter/main.go b/lib/proc/filter/main.go index 1616535..1730498 100644 --- a/lib/proc/filter/main.go +++ b/lib/proc/filter/main.go @@ -12,9 +12,17 @@ import ( // http://www.squid-cache.org/Doc/config/acl/ func Filter(pkt parse.ParsedPacket) bool { - for rid := range *cnf.RULES { - rule := (*cnf.RULES)[rid] + // anti-loop & security loopholes + if alwaysDeny(pkt) == meta.MatchPositive { + if cnf.Metrics() { + metrics.RuleMatches.WithLabelValues("always").Inc() + metrics.RuleActions.WithLabelValues(meta.RevRuleAction(meta.ActionDeny)).Inc() + } + parse.LogConnDebug("filter", pkt, "Matched always deny") + return applyAction(meta.ActionDeny) + } + for rid, rule := range *cnf.RULES { if cnf.Metrics() { metrics.RuleHits.WithLabelValues(fmt.Sprintf("%v", rid)).Inc() } @@ -68,3 +76,13 @@ func applyAction(action meta.Action) bool { } return false } + +func alwaysDeny(pkt parse.ParsedPacket) meta.Match { + if cnf.NetForwardDeny == nil { + cnf.InitNetForwardDeny() + } + return anyNetMatch( + cnf.NetForwardDeny, + pkt.L3.DestIP, + ) +} diff --git a/lib/proc/filter/match.go b/lib/proc/filter/match.go index 6e8228c..d8f5388 100644 --- a/lib/proc/filter/match.go +++ b/lib/proc/filter/match.go @@ -125,7 +125,8 @@ func matchDomain(pkt parse.ParsedPacket, rule cnf.Rule, rid int) meta.Match { if pkt.L5.Proto == meta.ProtoL5Tls { return anyDomainMatch(rule.Match.Domains, pkt.L5.TlsSni) } - // todo: add plain http domain-match + // NOTE: domains from plain http host-headers are ignored by design as they can be modified easily + // no important dataflow should use plain HTTP anyway - just move to HTTPS already.. } return meta.MatchNeutral } @@ -158,8 +159,7 @@ func anyNetMatch(nets []*net.IPNet, ip net.IP) meta.Match { } func anyDomainMatch(domains []string, domain string) meta.Match { - for i := range domains { - matchDomain := domains[i] + for _, matchDomain := range domains { if strings.HasPrefix(matchDomain, "*.") { matchDomain = strings.Replace(matchDomain, "*.", "", 1) if strings.HasSuffix(domain, matchDomain) { diff --git a/lib/proc/fwd/http.go b/lib/proc/fwd/http.go new file mode 100644 index 0000000..c0aa203 --- /dev/null +++ b/lib/proc/fwd/http.go @@ -0,0 +1,161 @@ +package fwd + +import ( + "bufio" + "fmt" + "io" + "net" + "net/http" + + "github.com/superstes/calamary/cnf" + "github.com/superstes/calamary/log" + "github.com/superstes/calamary/metrics" + "github.com/superstes/calamary/proc/meta" + "github.com/superstes/calamary/proc/parse" + "github.com/superstes/calamary/send" + "github.com/superstes/calamary/u" +) + +func ForwardHttp(srvCnf cnf.ServiceListener, l4Proto meta.Proto, conn net.Conn) { + defer conn.Close() + + if l4Proto != meta.ProtoL4Tcp { + log.ErrorS("forward", fmt.Sprintf("L4Proto %s not supported in http/https mode!", meta.RevProto(l4Proto))) + return + } + + if cnf.Metrics() { + metrics.ReqTcp.Inc() + metrics.CurrentConn.Inc() + defer metrics.CurrentConn.Dec() + } + + // todo: full parsing may not be needed if 'connect' + pktProxy, connProxyIo := parseConn(srvCnf, l4Proto, conn) + + if pktProxy.L5.Proto != meta.ProtoL5Http { + parse.LogConnError("forward", pktProxy, "Got non HTTP-Request on HTTP server") + return + } + + reqProxy := readRequest(pktProxy, connProxyIo) + + if reqProxy == nil { + return + } else if reqProxy.Method == http.MethodConnect { + forwardConnect(srvCnf, l4Proto, conn, pktProxy, connProxyIo, reqProxy) + } else { + // plaintext HTTP is unsafe by design and should not be allowed + // todo: implement generic https-redirection response + + forwardPlain(srvCnf, l4Proto, conn, pktProxy, connProxyIo, reqProxy) + } +} + +func forwardPlain( + srvCnf cnf.ServiceListener, l4Proto meta.Proto, conn net.Conn, + pkt parse.ParsedPacket, connIo io.ReadWriter, req *http.Request, +) { + pkt.L5Http = parse.ParseHttpPacket(pkt, req) + pkt.L4.DestPort = pkt.L5Http.Port + dest := resolveTargetHostname(pkt) + if dest == nil { + proxyResp := responseFailed() + proxyResp.Write(conn) + return + } + pkt.L3.DestIP = dest + parse.LogConnDebug("forward", pkt, "Updated destination IP") + + filterConn(pkt, conn, connIo) + send.ForwardHttp(pkt, conn, connIo, req) +} + +func forwardConnect( + srvCnf cnf.ServiceListener, l4Proto meta.Proto, conn net.Conn, + pktProxy parse.ParsedPacket, connProxyIo io.ReadWriter, reqProxy *http.Request, +) { + proxyResp := http.Response{ + StatusCode: 200, + Request: reqProxy, + ProtoMajor: reqProxy.ProtoMajor, + ProtoMinor: reqProxy.ProtoMinor, + } + err := proxyResp.Write(conn) + if err != nil { + parse.LogConnError("forward", pktProxy, "Failed to write proxy response") + return + } + + pkt, connIo := parseConn(srvCnf, l4Proto, conn) + host, port := parse.SplitHttpHost(reqProxy.Host, pkt.L5.Encrypted) + pkt.L5Http = &parse.ParsedHttp{ + Host: host, + Port: port, + } + pkt.L4.DestPort = pkt.L5Http.Port + + /* + req := readRequest(pkt, connIo) + pkt.L5Http = parse.ParseHttpPacket(pkt, req) + */ + dest := resolveTargetHostname(pkt) + if dest == nil { + proxyResp = responseFailed() + proxyResp.Write(conn) + return + } + pkt.L3.DestIP = dest + parse.LogConnDebug("forward", pkt, "Updated destination IP") + + filterConn(pkt, conn, connIo) + send.Forward(pkt, conn, connIo) + +} + +func resolveTargetHostname(pkt parse.ParsedPacket) net.IP { + var destIp4, destIp6 []net.IP + if pkt.L5.Encrypted == meta.OptBoolTrue { + // todo: enable tls-interception + if pkt.L5.TlsSni == "" { + parse.LogConnError("forward", pkt, "Target hostname not retrievable via TLS-SNI") + return nil + } + destIp4, destIp6 = u.DnsLookup46(pkt.L5.TlsSni) + + } else { + if pkt.L5Http.Host == "" { + parse.LogConnError("forward", pkt, "Target hostname not retrievable via Host-Header") + return nil + } + destIp4, destIp6 = u.DnsLookup46(pkt.L5Http.Host) + } + + dest := FirstReachableTarget(pkt, destIp4, destIp6, "tcp", pkt.L4.DestPort) + if dest == nil { + parse.LogConnError("send", pkt, + fmt.Sprintf("No target IP reachable after %v retries: IP4 %v, IP6 %v, Port %v", + cnf.C.Service.Output.Retries, destIp4, destIp6, pkt.L4.DestPort), + ) + return nil + } + return dest +} + +func responseFailed() http.Response { + return http.Response{ + StatusCode: http.StatusRequestTimeout, + ProtoMajor: 1, + ProtoMinor: 1, + } +} + +func readRequest(pkt parse.ParsedPacket, connIo io.ReadWriter) *http.Request { + reqRaw := bufio.NewReader(connIo) + req, err := http.ReadRequest(reqRaw) + if err != nil { + parse.LogConnError("forward", pkt, "Failed to parse HTTP request") + return nil + } + return req +} diff --git a/lib/proc/fwd/main.go b/lib/proc/fwd/main.go index d3f0438..804c6a1 100644 --- a/lib/proc/fwd/main.go +++ b/lib/proc/fwd/main.go @@ -23,18 +23,30 @@ func Forward(srvCnf cnf.ServiceListener, l4Proto meta.Proto, conn net.Conn) { defer metrics.CurrentConn.Dec() } - var connIo io.ReadWriter = conn + pkt, connIo := parseConn(srvCnf, l4Proto, conn) + filterConn(pkt, conn, connIo) + send.Forward(pkt, conn, connIo) +} + +func parseConn(srvCnf cnf.ServiceListener, l4Proto meta.Proto, conn net.Conn) (pkt parse.ParsedPacket, connIo io.ReadWriter) { + connIo = conn connIoBuf := new(bytes.Buffer) connIoTee := io.TeeReader(connIo, connIoBuf) - pkt := parse.Parse(srvCnf, l4Proto, conn, connIoTee) + + pkt = parse.Parse(srvCnf, l4Proto, conn, connIoTee) + + // write read bytes back to stream so we can forward them + connIo = u.NewReadWriter(io.MultiReader(bytes.NewReader(connIoBuf.Bytes()), connIo), connIo) + + return +} + +func filterConn(pkt parse.ParsedPacket, conn net.Conn, connIo io.ReadWriter) { if !filter.Filter(pkt) { parse.LogConnInfo("forward", pkt, "Denied") conn.Close() return } - // write read bytes back to stream so we can forward them - connIo = u.NewReadWriter(io.MultiReader(bytes.NewReader(connIoBuf.Bytes()), connIo), connIo) parse.LogConnInfo("forward", pkt, "Accept") - send.Forward(pkt, conn, connIo) } diff --git a/lib/proc/fwd/util.go b/lib/proc/fwd/util.go new file mode 100644 index 0000000..d02a6e2 --- /dev/null +++ b/lib/proc/fwd/util.go @@ -0,0 +1,46 @@ +package fwd + +import ( + "fmt" + "net" + "time" + + "github.com/superstes/calamary/cnf" + "github.com/superstes/calamary/proc/parse" + "github.com/superstes/calamary/u" +) + +func FirstReachableTarget(pkt parse.ParsedPacket, dest4 []net.IP, dest6 []net.IP, l4Proto string, port uint16) net.IP { + retries := 1 + for { + for _, ip6 := range dest6 { + if TargetIsReachable(l4Proto, ip6, port) { + return ip6 + } + } + for _, ip4 := range dest4 { + if TargetIsReachable(l4Proto, ip4, port) { + return ip4 + } + } + if retries >= int(cnf.C.Service.Output.Retries) { + break + } + parse.LogConnDebug( + "send", pkt, fmt.Sprintf("Connection probe to all available IPs failed (retry %v/%v)", + retries, cnf.C.Service.Output.Retries), + ) + retries++ + time.Sleep(u.Timeout(cnf.DefaultConnectRetryWait)) + } + return nil +} + +func TargetIsReachable(l4Proto string, target net.IP, port uint16) bool { + targetAddr := fmt.Sprintf("%s:%v", target.String(), port) + dummyConn, err := net.DialTimeout(l4Proto, targetAddr, u.Timeout(cnf.C.Service.Timeout.Probe)) + if dummyConn != nil { + dummyConn.Close() + } + return err == nil +} diff --git a/lib/proc/fwd/util_test.go b/lib/proc/fwd/util_test.go new file mode 100644 index 0000000..c2cad23 --- /dev/null +++ b/lib/proc/fwd/util_test.go @@ -0,0 +1,81 @@ +package fwd + +import ( + "net" + "testing" + + "github.com/creasty/defaults" + "github.com/superstes/calamary/cnf" + "github.com/superstes/calamary/proc/parse" +) + +func TestTargetReachable(t *testing.T) { + cnf.C = &cnf.Config{} + defaults.Set(cnf.C) // probe timeout + + testTarget := net.ParseIP("1.1.1.1") + if !TargetIsReachable("tcp", testTarget, 53) { + t.Error("Target reachability check failed #1") + } + if TargetIsReachable("tcp", testTarget, 50000) { + t.Error("Target reachability check failed #2") + } +} + +func TestFirstReachableTarget(t *testing.T) { + cnf.C = &cnf.Config{} + defaults.Set(cnf.C) // probe timeout + + testTarget1 := net.ParseIP("135.181.170.219") + testTarget2 := net.ParseIP("1.1.1.1") + testTarget3 := net.ParseIP("2001:db8::1") + testTarget4 := net.ParseIP("2606:4700:4700::1001") + + testerHasIPv6 := TargetIsReachable("tcp", testTarget4, 53) + + target := FirstReachableTarget( + parse.ParsedPacket{}, + []net.IP{testTarget1}, + []net.IP{}, + "tcp", + 53, + ) + if target != nil { + t.Errorf("Target first-reachable check failed #1 (%v)", target) + } + + target = FirstReachableTarget( + parse.ParsedPacket{}, + []net.IP{testTarget1, testTarget2}, + []net.IP{}, + "tcp", + 53, + ) + if target.String() != testTarget2.String() { + t.Errorf("Target first-reachable check failed #2 (%v)", target) + } + + target = FirstReachableTarget( + parse.ParsedPacket{}, + []net.IP{}, + []net.IP{testTarget3, testTarget4}, + "tcp", + 53, + ) + if testerHasIPv6 && target.String() != testTarget4.String() { + // nil => if testing client has no IPv6 + t.Errorf("Target first-reachable check failed #3 (%v)", target) + } + + target = FirstReachableTarget( + parse.ParsedPacket{}, + []net.IP{testTarget1, testTarget2}, + []net.IP{testTarget3, testTarget4}, + "tcp", + 53, + ) + if testerHasIPv6 && target.String() != testTarget4.String() { + // nil => if testing client has no IPv6 + t.Errorf("Target first-reachable check failed #4 (%v)", target) + } +} diff --git a/lib/proc/http/main.go b/lib/proc/http/main.go deleted file mode 100644 index 92a3d3f..0000000 --- a/lib/proc/http/main.go +++ /dev/null @@ -1,12 +0,0 @@ -package proc_http - -import "net/http" - -func HandleRequest(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotImplemented) -} - -func HandleRequestTls(w http.ResponseWriter, r *http.Request) { - w.Header().Add("Strict-Transport-Security", "max-age=63072000; includeSubDomains") - HandleRequest(w, r) -} diff --git a/lib/proc/parse/http.go b/lib/proc/parse/http.go index f84b60b..00b0113 100644 --- a/lib/proc/parse/http.go +++ b/lib/proc/parse/http.go @@ -25,3 +25,22 @@ func parseHttp(pkt ParsedPacket, hdr [cnf.BYTES_HDR_L5]byte) { pkt.L5Http = &ParsedHttp{} } } + +func ParseHttpPacket(pkt ParsedPacket, req *http.Request) *ParsedHttp { + if pkt.L5.Encrypted == meta.OptBoolTrue { + // todo: implement tls-interception + return &ParsedHttp{} + } + + host, port := SplitHttpHost(req.Host, pkt.L5.Encrypted) + + // todo: parse further useful information for fitlering + return &ParsedHttp{ + Host: host, + Port: port, + Method: req.Method, + ProtoMajor: req.ProtoMajor, + ProtoMinor: req.ProtoMinor, + Url: req.URL, + } +} diff --git a/lib/proc/parse/pkt.go b/lib/proc/parse/pkt.go index 3fec5cb..3d318dc 100644 --- a/lib/proc/parse/pkt.go +++ b/lib/proc/parse/pkt.go @@ -2,6 +2,7 @@ package parse import ( "net" + "net/url" "github.com/superstes/calamary/proc/meta" ) @@ -46,10 +47,16 @@ type ParsedHttp struct { /* method */ - Headers string - MimeType string - AuthUser string - AuthPwd string + Host string + Port uint16 + Method string + Url *url.URL + ProtoMajor int + ProtoMinor int + Headers string + MimeType string + AuthUser string + AuthPwd string } type ParsedTls struct { diff --git a/lib/proc/parse/helpers.go b/lib/proc/parse/util.go similarity index 61% rename from lib/proc/parse/helpers.go rename to lib/proc/parse/util.go index 60e3705..9abc386 100644 --- a/lib/proc/parse/helpers.go +++ b/lib/proc/parse/util.go @@ -3,6 +3,8 @@ package parse import ( "fmt" "net" + "strconv" + "strings" "github.com/superstes/calamary/proc/meta" "github.com/superstes/calamary/u" @@ -29,3 +31,19 @@ func parseIpProto(addr net.Addr) meta.Proto { } return meta.ProtoL3IP6 } + +func SplitHttpHost(host string, encypted meta.OptBool) (dns string, port uint16) { + if strings.Contains(host, ":") { + parts := strings.SplitN(host, ":", 2) + port, err := strconv.Atoi(parts[1]) + + if err == nil { + return parts[0], uint16(port) + } + } + + if encypted == meta.OptBoolTrue { + return host, 443 + } + return host, 80 +} diff --git a/lib/rcv/http.go b/lib/rcv/http.go index 19517ce..75a999e 100644 --- a/lib/rcv/http.go +++ b/lib/rcv/http.go @@ -3,32 +3,24 @@ package rcv import ( "crypto/tls" "fmt" - "net/http" "github.com/superstes/calamary/cnf" - proc_http "github.com/superstes/calamary/proc/http" "github.com/superstes/calamary/proc/meta" ) func newServerHttpTcp(addr string, lncnf cnf.ServiceListener) (Server, error) { - httpMux := http.NewServeMux() - httpMux.HandleFunc("/", proc_http.HandleRequest) - - httpSrv := &http.Server{ - Addr: fmt.Sprintf("%s:%v", addr, lncnf.Port), - Handler: httpMux, + transparentSrv, err := newServerTransparentTcp(addr, lncnf) + if err != nil { + return transparentSrv, err } return Server{ - HttpServer: httpSrv, - Cnf: lncnf, - L4Proto: meta.ProtoL4Tcp, + Listener: transparentSrv.Listener, + Cnf: lncnf, + L4Proto: meta.ProtoL4Tcp, }, nil } func newServerHttpsTcp(addr string, lncnf cnf.ServiceListener) (Server, error) { - httpMux := http.NewServeMux() - httpMux.HandleFunc("/", proc_http.HandleRequestTls) - tlsCnf := &tls.Config{ MinVersion: tls.VersionTLS11, NextProtos: []string{"h2", "http/1.1"}, @@ -43,15 +35,17 @@ func newServerHttpsTcp(addr string, lncnf cnf.ServiceListener) (Server, error) { }, */ } - httpSrv := &http.Server{ - Addr: fmt.Sprintf("%s:%v", addr, lncnf.Port), - Handler: httpMux, - TLSConfig: tlsCnf, - TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0), + ln, err := tls.Listen( + "tcp", + fmt.Sprintf("%v:%v", addr, lncnf.Port), + tlsCnf, + ) + if err != nil { + return Server{}, err } return Server{ - HttpServer: httpSrv, - Cnf: lncnf, - L4Proto: meta.ProtoL4Tcp, + Listener: ln, + Cnf: lncnf, + L4Proto: meta.ProtoL4Tcp, }, nil } diff --git a/lib/rcv/main.go b/lib/rcv/main.go index 091ac20..834cc2d 100644 --- a/lib/rcv/main.go +++ b/lib/rcv/main.go @@ -3,7 +3,6 @@ package rcv import ( "fmt" "net" - "net/http" "github.com/superstes/calamary/cnf" "github.com/superstes/calamary/proc/meta" @@ -16,10 +15,9 @@ type Listener interface { } type Server struct { - Listener Listener - HttpServer *http.Server - Cnf cnf.ServiceListener - L4Proto meta.Proto + Listener Listener + Cnf cnf.ServiceListener + L4Proto meta.Proto } type ServerInfo struct { diff --git a/lib/rcv/server.go b/lib/rcv/server.go index 2cfb498..05b2e33 100644 --- a/lib/rcv/server.go +++ b/lib/rcv/server.go @@ -2,7 +2,6 @@ package rcv import ( "fmt" - "strings" "github.com/superstes/calamary/cnf" "github.com/superstes/calamary/log" @@ -26,11 +25,8 @@ func newServersForIps( lnfuncUdp func(string, cnf.ServiceListener) (Server, error), ) (servers []Server) { if len(ips) > 0 { - for i := range ips { - ip := ips[i] - if !u.IsIPv4(ip) && !strings.Contains(ip, "[") { - ip = fmt.Sprintf("[%v]", ip) - } + for _, ip := range ips { + ip = u.FormatIPv6(ip) if lncnf.Tcp && lnfuncTcp != nil { srv, err := lnfuncTcp(ip, lncnf) if err != nil { @@ -85,8 +81,7 @@ func newServerSocks5(lncnf cnf.ServiceListener) (servers []Server) { } func BuildServers() (servers []Server) { - for i := range cnf.C.Service.Listen { - lncnf := cnf.C.Service.Listen[i] + for _, lncnf := range cnf.C.Service.Listen { servers = append( servers, serverModeMapping[lncnf.Mode](lncnf)..., diff --git a/lib/send/main.go b/lib/send/main.go index 0490f20..9324c58 100644 --- a/lib/send/main.go +++ b/lib/send/main.go @@ -3,6 +3,7 @@ package send import ( "io" "net" + "net/http" "github.com/superstes/calamary/proc/meta" "github.com/superstes/calamary/proc/parse" @@ -17,3 +18,14 @@ func Forward(pkt parse.ParsedPacket, conn net.Conn, connIo io.ReadWriter) { forwardTcp(pkt, conn, connIo) } } + +func ForwardHttp(pkt parse.ParsedPacket, conn net.Conn, connIo io.ReadWriter, req *http.Request) { + // http-proxy - rewrite request; only for plaintext http + connFwd := establishTcp(pkt) + + if err := req.Write(connFwd); err != nil { + return + } + + transportTcp(pkt, conn, connIo, connFwd) +} diff --git a/lib/send/tcp.go b/lib/send/tcp.go index ce8968e..efbd313 100644 --- a/lib/send/tcp.go +++ b/lib/send/tcp.go @@ -11,7 +11,7 @@ import ( "github.com/superstes/calamary/u" ) -func forwardTcp(pkt parse.ParsedPacket, conn net.Conn, connIo io.ReadWriter) { +func establishTcp(pkt parse.ParsedPacket) net.Conn { var connFwd net.Conn var err error retries := 1 @@ -19,26 +19,30 @@ func forwardTcp(pkt parse.ParsedPacket, conn net.Conn, connIo io.ReadWriter) { connFwd, err = net.DialTimeout("tcp", parse.PktDest(pkt), u.Timeout(cnf.C.Service.Timeout.Connect)) if err == nil { - defer connFwd.Close() break } else { if retries >= int(cnf.C.Service.Output.Retries) { parse.LogConnError( "send", pkt, fmt.Sprintf("Connect retry exceeded (%v/%v): %v", - cnf.C.Service.Output.Retries, cnf.C.Service.Output.Retries, err), + retries, cnf.C.Service.Output.Retries, err), ) - return + return nil } parse.LogConnDebug( "send", pkt, fmt.Sprintf("Connection failed (retry %v/%v): %v", - cnf.C.Service.Output.Retries, cnf.C.Service.Output.Retries, err), + retries, cnf.C.Service.Output.Retries, err), ) retries++ time.Sleep(u.Timeout(cnf.DefaultConnectRetryWait)) } } + parse.LogConnDebug("send", pkt, "Connection established") + return connFwd +} +func transportTcp(pkt parse.ParsedPacket, conn net.Conn, connIo io.ReadWriter, connFwd net.Conn) { + defer connFwd.Close() close := make(chan bool, 1) link := Link{src: connIo, close: close} parse.LogConnDebug("send", pkt, "Forwarding") @@ -47,3 +51,7 @@ func forwardTcp(pkt parse.ParsedPacket, conn net.Conn, connIo io.ReadWriter) { <-close parse.LogConnDebug("send", pkt, "Closed") } + +func forwardTcp(pkt parse.ParsedPacket, conn net.Conn, connIo io.ReadWriter) { + transportTcp(pkt, conn, connIo, establishTcp(pkt)) +} diff --git a/lib/u/helpers.go b/lib/u/helpers.go index 9b3a19e..861e751 100644 --- a/lib/u/helpers.go +++ b/lib/u/helpers.go @@ -1,11 +1,16 @@ package u import ( + "context" "fmt" + "net" "strings" "time" "slices" + + "github.com/superstes/calamary/cnf" + "github.com/superstes/calamary/log" ) func ToStr(data any) string { @@ -84,3 +89,62 @@ func IsIn(value string, list []string) bool { } return false } + +// just as shorter version +func IsInStr(searchFor string, searchIn string) bool { + return strings.Contains(searchIn, searchFor) +} + +func dnsResolveWithServer(srv string) *net.Resolver { + if !strings.Contains(srv, ":") { + srv = srv + ":53" + } + return &net.Resolver{ + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{ + Timeout: Timeout(cnf.C.Service.Timeout.DnsLookup), + } + return d.DialContext(ctx, network, srv) + }, + } +} + +func DnsLookup(dns string) (ips []net.IP) { + var err error + for _, srv := range cnf.C.Service.DnsNameservers { + ips, err = dnsResolveWithServer(srv).LookupIP( + context.Background(), "ip", dns, + ) + if err != nil { + log.Debug("util", fmt.Sprintf("Failed to lookup DNS '%s' via server %s: %v", dns, srv, err)) + continue + } + if len(ips) > 0 { + break + } + } + if len(ips) == 0 { + log.ErrorS("util", fmt.Sprintf("Failed to lookup DNS '%s'", dns)) + return + } + log.Debug("util", fmt.Sprintf("DNS '%s' resolved to: %v", dns, ips)) + return ips +} + +func DnsLookup46(dns string) (ip4 []net.IP, ip6 []net.IP) { + for _, ip := range DnsLookup(dns) { + if IsIPv4(ip.String()) { + ip4 = append(ip4, ip) + } else { + ip6 = append(ip6, ip) + } + } + return +} + +func FormatIPv6(ip string) string { + if !IsIPv4(ip) && !strings.Contains(ip, "[") { + return fmt.Sprintf("[%v]", ip) + } + return ip +} diff --git a/lib/u/helpers_test.go b/lib/u/helpers_test.go new file mode 100644 index 0000000..a070e44 --- /dev/null +++ b/lib/u/helpers_test.go @@ -0,0 +1,40 @@ +package u + +import ( + "net" + "testing" + + "github.com/superstes/calamary/cnf" +) + +func TestDnsLookup(t *testing.T) { + // NOTE: tests will fail if the IPs change.. should not be common + cnf.C = &cnf.Config{ + Service: cnf.ServiceConfig{ + DnsNameservers: []string{"1.1.1.1"}, + }, + } + resp := DnsLookup("one.one.one.one") + + if !isInIpList("1.1.1.1", resp) || !isInIpList("1.0.0.1", resp) || + !isInIpList("2606:4700:4700::1001", resp) || !isInIpList("2606:4700:4700::1111", resp) { + t.Errorf("DNS Query #1 has unexpected result: %v", resp) + } + + resp4, resp6 := DnsLookup46("one.one.one.one") + + if !isInIpList("1.1.1.1", resp4) || !isInIpList("1.0.0.1", resp4) || + !isInIpList("2606:4700:4700::1001", resp6) || !isInIpList("2606:4700:4700::1111", resp6) || + isInIpList("1.1.1.1", resp6) || isInIpList("2606:4700:4700::1001", resp4) { + t.Errorf("DNS Query #2 has unexpected result: ip4 %v, ip6 %v", resp4, resp6) + } +} + +func isInIpList(value string, list []net.IP) bool { + for i := range list { + if list[i].String() == value { + return true + } + } + return false +}