From ea5c46d1511fbdcf19de4a14fc4795d56e988c55 Mon Sep 17 00:00:00 2001 From: Rath Rene Date: Sun, 1 Oct 2023 01:19:58 +0200 Subject: [PATCH] connection logging, cleanup & more docs --- docs/source/info/redirect.rst | 105 +++++++++++++++++++------------- docs/source/info/redirector.rst | 26 ++++++++ lib/log/main.go | 34 +++-------- lib/main/main.go | 4 +- lib/main/metrics.go | 56 +++++++++-------- lib/metrics/main.go | 6 +- lib/proc/filter/main.go | 11 ++-- lib/proc/parse/conn_log.go | 19 ++++++ lib/proc/parse/helpers.go | 3 + lib/proc/parse/tls.go | 13 +--- lib/send/connect.go | 10 +-- lib/send/main.go | 3 +- scripts/buildrun.sh | 1 + 13 files changed, 170 insertions(+), 121 deletions(-) create mode 100644 docs/source/info/redirector.rst create mode 100644 lib/proc/parse/conn_log.go diff --git a/docs/source/info/redirect.rst b/docs/source/info/redirect.rst index 8125dab..8c883ad 100644 --- a/docs/source/info/redirect.rst +++ b/docs/source/info/redirect.rst @@ -8,57 +8,27 @@ Redirect Traffic ################ -TProxy +Basics ###### -To run Calamary as `TPROXY `_ target - you will have to set `CAP_NET_RAW `_: - -:: - - bind to any address for transparent proxying - -You can add it like this: - -.. code-block:: bash +You may want/need to redirect traffic to the proxy's listeners for some use-case. - setcap cap_net_raw=+ep /usr/bin/calamary +This is essential for using the :code:`transparent` mode. - # make sure only wanted users can execute the binary! - chown root:proxy /usr/bin/calamary - chmod 750 /usr/bin/calamary +For modes like :code:`proxyproto`, :code:`http`, :code:`https` or :code:`socks5` this is not necessary. (*but it's also possible using the :ref:`Redirector `*) -Read more about TPROXY here: +You will have to choose between using **DNAT** and **TPROXY** to redirect the traffic on firewall-level. -* `wiki.superstes.eu - NFTables - TProxy `_ +**TProxy** has the benefit that it won't modify the packets destination. This makes processing the traffic easier and can be benefitial in regards to performance. -* `kernel docs `_ +But it also has the drawback that traffic that originates from the proxy-server (*netfilter hook - output*) will have to be looped-back. -Output loopback -=============== +I personally like to use TProxy for filtering input/forward- and DNAT for output-traffic. -You will have to configure a loopback route if you want to proxy 'output' traffic: -.. code-block:: bash - - echo "200 proxy_loopback" > /etc/iproute2/rt_tables.d/proxy.conf - - # These need to be configured persistent: (maybe use an interface up-hook) - ip rule add fwmark 200 table proxy_loopback - ip -6 rule add fwmark 200 table proxy_loopback - ip route add local 0.0.0.0/0 dev lo table proxy_loopback - ip -6 route add local ::/0 dev lo table proxy_loopback - - # can be checked using: - ip rule list - ip -6 rule list - ip -d route show table all - - # you might need to set a sysctl: - sysctl -w net.ipv4.conf.all.route_localnet=1 - - # you might want to block 127.0.0.1 on non loopback interfaces if you enable it: - iptables -t raw -A PREROUTING ! -i lo -d 127.0.0.0/8 -j DROP - iptables -t mangle -A POSTROUTING ! -o lo -s 127.0.0.0/8 -j DROP +.. warning:: + The config-examples below may not be complete! + If you find issues with them - please `open an issue `_ NFTables @@ -154,3 +124,56 @@ TProxy Full TProxy example: `gist.github.com/superstes - TProxy IPTables `_ You might need to enable some iptables kernel modules: `Kernel docs - IPTables extensions `_ + + +TProxy +###### + +To run Calamary as `TPROXY `_ target - you will have to set `CAP_NET_RAW `_: + +:: + + bind to any address for transparent proxying + +You can add it like this: + +.. code-block:: bash + + setcap cap_net_raw=+ep /usr/bin/calamary + + # make sure only wanted users can execute the binary! + chown root:proxy /usr/bin/calamary + chmod 750 /usr/bin/calamary + +Read more about TPROXY here: + +* `wiki.superstes.eu - NFTables - TProxy `_ + +* `kernel docs `_ + +Output loopback +=============== + +You will have to configure a loopback route if you want to proxy 'output' traffic: + +.. code-block:: bash + + echo "200 proxy_loopback" > /etc/iproute2/rt_tables.d/proxy.conf + + # These need to be configured persistent: (maybe use an interface up-hook) + ip rule add fwmark 200 table proxy_loopback + ip -6 rule add fwmark 200 table proxy_loopback + ip route add local 0.0.0.0/0 dev lo table proxy_loopback + ip -6 route add local ::/0 dev lo table proxy_loopback + + # can be checked using: + ip rule list + ip -6 rule list + ip -d route show table all + + # you might need to set a sysctl: + sysctl -w net.ipv4.conf.all.route_localnet=1 + + # you might want to block 127.0.0.1 on non loopback interfaces if you enable it: + iptables -t raw -A PREROUTING ! -i lo -d 127.0.0.0/8 -j DROP + iptables -t mangle -A POSTROUTING ! -o lo -s 127.0.0.0/8 -j DROP diff --git a/docs/source/info/redirector.rst b/docs/source/info/redirector.rst new file mode 100644 index 0000000..fe9b2d9 --- /dev/null +++ b/docs/source/info/redirector.rst @@ -0,0 +1,26 @@ +.. _redirector: + +.. |proxy_forwarder| image:: https://wiki.superstes.eu/en/latest/_images/squid_remote.png + :class: wiki-img + +.. include:: ../_inc/head.rst + +########## +Redirector +########## + +The redirector will be a smaller version of Calamary. (*without filtering*) + +It can be used to forward traffic from remote locations to your proxy servers. + +|proxy_forwarder| + +Per example - this might be useful if you have: + +* Distributed systems + + * Cloud servers that are only connected to public WAN and should send all their outbound traffic over your proxy + + * Simple/dumb firewalls/routers that should send the networks outbound traffic over your proxy + +As it utilizes the commonly used `proxy-protocol `_ to redirect the traffic, it might also be useful in combination with other proxies. diff --git a/lib/log/main.go b/lib/log/main.go index 757bb58..958592d 100644 --- a/lib/log/main.go +++ b/lib/log/main.go @@ -9,7 +9,7 @@ import ( func log(lvl string, pkg string, msg string) { var base string - if cnf.C.Service.Debug { + if cnf.Debug() { base = fmt.Sprintf("%s | %s | %s\n", lvl, pkg, msg) } else { base = fmt.Sprintf("%s | %s\n", lvl, msg) @@ -22,9 +22,9 @@ func log(lvl string, pkg string, msg string) { } } -func logConn(lvl string, pkg string, src string, dst string, msg string) { +func Conn(lvl string, pkg string, src string, dst string, msg string) { var base string - if cnf.C.Service.Debug { + if cnf.Debug() { base = fmt.Sprintf("%s | %s | %s => %s | %s\n", lvl, pkg, src, dst, msg) } else { base = fmt.Sprintf("%s | %s => %s | %s\n", lvl, src, dst, msg) @@ -46,31 +46,19 @@ func Error(pkg string, err error) { log("ERROR", pkg, fmt.Sprintf("%s", err)) } -func ConnErrorS(pkg string, src string, dst string, msg string) { - logConn("ERROR", pkg, src, dst, msg) -} - -func ConnError(pkg string, src string, dst string, err error) { - logConn("ERROR", pkg, src, dst, fmt.Sprintf("%s", err)) -} - -func ConnWarnS(pkg string, src string, dst string, msg string) { - logConn("WARN", pkg, src, dst, msg) -} - -func ConnWarn(pkg string, src string, dst string, err error) { - logConn("WARN", pkg, src, dst, fmt.Sprintf("%s", err)) +func ConnError(pkg string, src string, dst string, msg string) { + Conn("ERROR", pkg, src, dst, msg) } func Debug(pkg string, msg string) { - if cnf.C.Service.Debug { + if cnf.Debug() { log("DEBUG", pkg, msg) } } func ConnDebug(pkg string, src string, dst string, msg string) { - if cnf.C.Service.Debug { - logConn("DEBUG", pkg, src, dst, msg) + if cnf.Debug() { + Conn("DEBUG", pkg, src, dst, msg) } } @@ -78,17 +66,13 @@ func Info(pkg string, msg string) { log("INFO", pkg, msg) } -func ConnInfo(pkg string, src string, dst string, msg string) { - logConn("INFO", pkg, src, dst, msg) -} - func Warn(pkg string, msg string) { log("WARN", pkg, msg) } func Fatal(pkg string, msg string) { var base string - if cnf.C.Service.Debug { + if cnf.Debug() { base = fmt.Sprintf("FATAL | %s | %s\n", pkg, msg) } else { base = fmt.Sprintf("FATAL | %s\n", msg) diff --git a/lib/main/main.go b/lib/main/main.go index 04b29e2..91b2c5c 100644 --- a/lib/main/main.go +++ b/lib/main/main.go @@ -30,13 +30,13 @@ func main() { if modeValidate { cnf.C.Service.Debug = true - cnf_file.Load(true) + cnf_file.Load(true, true) log.Info("config", "Config validated successfully") os.Exit(0) } welcome() - cnf_file.Load(false) + cnf_file.Load(false, true) service := &service{} go startPrometheusExporter() service.start() diff --git a/lib/main/metrics.go b/lib/main/metrics.go index c1d8875..1defd43 100644 --- a/lib/main/metrics.go +++ b/lib/main/metrics.go @@ -11,38 +11,40 @@ import ( "github.com/superstes/calamary/metrics" ) -func catchAll(w http.ResponseWriter, r *http.Request) { +// todo: clean-up redundant linking +var metricFuncs = []prometheus.Collector{ + metrics.BytesRcv, + metrics.BytesSent, + metrics.CurrentConn, + metrics.ReqTcp, + metrics.ReqL3Proto, + metrics.ReqL5Proto, + metrics.ReqTlsVersion, + metrics.RuleHits, + metrics.RuleMatches, + metrics.RuleActions, +} + +func denyAll(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusForbidden) } func startPrometheusExporter() { if cnf.Metrics() { log.Info("service", "Starting prometheus metrics-exporter") - prometheus.Register(metrics.BytesRcv) - prometheus.Register(metrics.BytesSent) - prometheus.Register(metrics.CurrentConn) - prometheus.Register(metrics.ReqTcp) - prometheus.Register(metrics.RuleReqL3Proto) - prometheus.Register(metrics.RuleReqL5Proto) - prometheus.Register(metrics.RuleReqTlsVersion) - prometheus.Register(metrics.RuleHits) - prometheus.Register(metrics.RuleMatches) - prometheus.Register(metrics.RuleActions) - - http.Handle("/metrics", promhttp.Handler()) - http.HandleFunc("/", catchAll) - http.ListenAndServe(fmt.Sprintf("127.0.0.1:%v", cnf.C.Service.Metrics.Port), nil) - http.ListenAndServe(fmt.Sprintf("[::1]:%v", cnf.C.Service.Metrics.Port), nil) - - prometheus.MustRegister(metrics.BytesRcv) - prometheus.MustRegister(metrics.BytesSent) - prometheus.MustRegister(metrics.CurrentConn) - prometheus.MustRegister(metrics.ReqTcp) - prometheus.MustRegister(metrics.RuleReqL3Proto) - prometheus.MustRegister(metrics.RuleReqL5Proto) - prometheus.MustRegister(metrics.RuleReqTlsVersion) - prometheus.MustRegister(metrics.RuleHits) - prometheus.MustRegister(metrics.RuleMatches) - prometheus.MustRegister(metrics.RuleActions) + + for i := range metricFuncs { + prometheus.Register(metricFuncs[i]) + } + + metricsSrv := http.NewServeMux() + metricsSrv.Handle("/metrics", promhttp.Handler()) + metricsSrv.HandleFunc("/", denyAll) + http.ListenAndServe(fmt.Sprintf("127.0.0.1:%v", cnf.C.Service.Metrics.Port), metricsSrv) + http.ListenAndServe(fmt.Sprintf("[::1]:%v", cnf.C.Service.Metrics.Port), metricsSrv) + + for i := range metricFuncs { + prometheus.MustRegister(metricFuncs[i]) + } } } diff --git a/lib/metrics/main.go b/lib/metrics/main.go index 6182e2b..556649e 100644 --- a/lib/metrics/main.go +++ b/lib/metrics/main.go @@ -26,7 +26,7 @@ var ( Name: "req_tcp", Help: "Count of tcp requests received", }) - RuleReqL3Proto = prometheus.NewCounterVec( + ReqL3Proto = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "calamary", Name: "req_protoL3", @@ -34,7 +34,7 @@ var ( }, []string{"protocol"}, ) - RuleReqL5Proto = prometheus.NewCounterVec( + ReqL5Proto = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "calamary", Name: "req_protoL5", @@ -42,7 +42,7 @@ var ( }, []string{"protocol"}, ) - RuleReqTlsVersion = prometheus.NewCounterVec( + ReqTlsVersion = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "calamary", Name: "req_tls_version", diff --git a/lib/proc/filter/main.go b/lib/proc/filter/main.go index 2509e6f..1616535 100644 --- a/lib/proc/filter/main.go +++ b/lib/proc/filter/main.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/superstes/calamary/cnf" - "github.com/superstes/calamary/log" "github.com/superstes/calamary/metrics" "github.com/superstes/calamary/proc/meta" "github.com/superstes/calamary/proc/parse" @@ -49,13 +48,17 @@ func Filter(pkt parse.ParsedPacket) bool { } // implicit deny - log.ConnDebug("filter", parse.PktSrc(pkt), parse.PktDest(pkt), "No rule matched - implicit deny") + if cnf.Metrics() { + metrics.RuleMatches.WithLabelValues("default").Inc() + metrics.RuleActions.WithLabelValues(meta.RevRuleAction(meta.ActionDeny)).Inc() + } + parse.LogConnDebug("filter", pkt, "No rule matched - implicit deny") return applyAction(meta.ActionDeny) } func ruleDebug(pkt parse.ParsedPacket, rule_id int, msg string) { - if cnf.C.Service.Debug { - log.ConnDebug("filter", parse.PktSrc(pkt), parse.PktDest(pkt), fmt.Sprintf("Rule %v - %s", rule_id, msg)) + if cnf.Debug() { + parse.LogConnDebug("filter", pkt, fmt.Sprintf("Rule %v - %s", rule_id, msg)) } } diff --git a/lib/proc/parse/conn_log.go b/lib/proc/parse/conn_log.go new file mode 100644 index 0000000..7f08f37 --- /dev/null +++ b/lib/proc/parse/conn_log.go @@ -0,0 +1,19 @@ +package parse + +import "github.com/superstes/calamary/log" + +func LogConnDebug(pkg string, pkt ParsedPacket, msg string) { + log.Conn("DEBUG", pkg, PktSrc(pkt), PktDest(pkt), msg) +} + +func LogConnInfo(pkg string, pkt ParsedPacket, msg string) { + log.Conn("INFO", pkg, PktSrc(pkt), PktDest(pkt), msg) +} + +func LogConnWarn(pkg string, pkt ParsedPacket, msg string) { + log.Conn("WARN", pkg, PktSrc(pkt), PktDest(pkt), msg) +} + +func LogConnError(pkg string, pkt ParsedPacket, msg string) { + log.Conn("ERROR", pkg, PktSrc(pkt), PktDest(pkt), msg) +} diff --git a/lib/proc/parse/helpers.go b/lib/proc/parse/helpers.go index 0dfd86f..60e3705 100644 --- a/lib/proc/parse/helpers.go +++ b/lib/proc/parse/helpers.go @@ -13,6 +13,9 @@ func PktSrc(pkt ParsedPacket) string { } func PktDest(pkt ParsedPacket) string { + if pkt.L3.DestIP == nil { + return "?" + } return fmt.Sprintf("%s:%v", pkt.L3.DestIP.String(), pkt.L4.DestPort) } diff --git a/lib/proc/parse/tls.go b/lib/proc/parse/tls.go index 9d02760..2dc8fed 100644 --- a/lib/proc/parse/tls.go +++ b/lib/proc/parse/tls.go @@ -6,7 +6,6 @@ import ( "net" "github.com/superstes/calamary/cnf" - "github.com/superstes/calamary/log" "github.com/superstes/calamary/proc/meta" tls_dissector "github.com/superstes/calamary/proc/parse/tls" ) @@ -22,17 +21,13 @@ func parseTls(pkt ParsedPacket, conn net.Conn, connIo io.Reader, hdr [cnf.BYTES_ if isTlsRaw { record, err := tls_dissector.ReadRecord(connIo) if err != nil { - log.ConnWarnS("parse", PktSrc(pkt), PktDest(pkt), fmt.Sprintf( - "Failed to parse TLS handshake: %v", err, - )) + LogConnWarn("parse", pkt, fmt.Sprintf("Failed to parse TLS handshake: %v", err)) return } clientHello := tls_dissector.ClientHelloMsg{} if err = clientHello.Decode(record.Opaque); err != nil { - log.ConnWarnS("parse", PktSrc(pkt), PktDest(pkt), fmt.Sprintf( - "Failed to parse TLS client-hello: %v", err, - )) + LogConnWarn("parse", pkt, fmt.Sprintf("Failed to parse TLS client-hello: %v", err)) return } @@ -51,8 +46,6 @@ func parseTls(pkt ParsedPacket, conn net.Conn, connIo io.Reader, hdr [cnf.BYTES_ } } } - log.ConnDebug("parse", PktSrc(pkt), PktDest(pkt), fmt.Sprintf( - "TLS information: IsTls=%v, TlsVersion=%v, TlsSni=%s", isTlsRaw, tlsVersion, sni, - )) + LogConnDebug("parse", pkt, fmt.Sprintf("TLS information: IsTls=%v, TlsVersion=%v, TlsSni=%s", isTlsRaw, tlsVersion, sni)) return } diff --git a/lib/send/connect.go b/lib/send/connect.go index 522812a..b922962 100644 --- a/lib/send/connect.go +++ b/lib/send/connect.go @@ -7,7 +7,6 @@ import ( "time" "github.com/superstes/calamary/cnf" - "github.com/superstes/calamary/log" "github.com/superstes/calamary/metrics" "github.com/superstes/calamary/proc/parse" "github.com/superstes/calamary/u" @@ -47,13 +46,13 @@ func (l *Link) pipe(pkt parse.ParsedPacket, srcConn net.Conn, src io.ReadWriter, metrics.BytesSent.Add(float64(n)) } l.sentBytes += uint64(n) - log.ConnDebug("send", parse.PktSrc(pkt), parse.PktDest(pkt), fmt.Sprintf("%d bytes sent", l.sentBytes)) + parse.LogConnDebug("send", pkt, fmt.Sprintf("%d bytes sent", l.sentBytes)) } else { if cnf.Metrics() { metrics.BytesRcv.Add(float64(n)) } l.rcvBytes += uint64(n) - log.ConnDebug("send", parse.PktSrc(pkt), parse.PktDest(pkt), fmt.Sprintf("%d bytes received", l.rcvBytes)) + parse.LogConnDebug("send", pkt, fmt.Sprintf("%d bytes received", l.rcvBytes)) } } } @@ -63,10 +62,7 @@ func (l *Link) closer(pkt parse.ParsedPacket, errMsg string, err error) { return } if err != io.EOF { - log.ConnErrorS( - "send", parse.PktSrc(pkt), parse.PktDest(pkt), - fmt.Sprintf("Read failed: %s - %v", errMsg, err), - ) + parse.LogConnError("send", pkt, fmt.Sprintf("Read failed: %s - %v", errMsg, err)) } l.close <- true l.closed = true diff --git a/lib/send/main.go b/lib/send/main.go index 6a730da..0490f20 100644 --- a/lib/send/main.go +++ b/lib/send/main.go @@ -4,7 +4,6 @@ import ( "io" "net" - "github.com/superstes/calamary/log" "github.com/superstes/calamary/proc/meta" "github.com/superstes/calamary/proc/parse" ) @@ -12,7 +11,7 @@ import ( func Forward(pkt parse.ParsedPacket, conn net.Conn, connIo io.ReadWriter) { if pkt.L4.Proto == meta.ProtoL4Udp { // connFwd, err = DialUdp(pkt) - log.ConnErrorS("send", parse.PktSrc(pkt), parse.PktDest(pkt), "UDP not yet implemented!") + parse.LogConnError("send", pkt, "UDP not yet implemented!") } else { forwardTcp(pkt, conn, connIo) diff --git a/scripts/buildrun.sh b/scripts/buildrun.sh index 527bdcf..f168b65 100755 --- a/scripts/buildrun.sh +++ b/scripts/buildrun.sh @@ -15,6 +15,7 @@ then echo 'GENERATING DUMMY CERTS...' echo '' openssl req -x509 -newkey rsa:4096 -keyout /tmp/calamary.key -out /tmp/calamary.crt -sha256 -days 60 -nodes -subj "$CERT_CN" + chmod 644 /tmp/calamary.key /tmp/calamary.crt fi echo ''