connection logging, cleanup & more docs

docs/source/info/redirect.rst

@@ -8,57 +8,27 @@
 Redirect Traffic
 ################
 
-TProxy
+Basics
 ######
 
-To run Calamary as `TPROXY <https://docs.kernel.org/networking/tproxy.html>`_ target - you will have to set `CAP_NET_RAW <https://man7.org/linux/man-pages/man7/capabilities.7.html>`_:
-
-::
-
-  bind to any address for transparent proxying
-
-You can add it like this:
-
-.. code-block:: bash
+You may want/need to redirect traffic to the proxy's listeners for some use-case.
 
-    setcap cap_net_raw=+ep /usr/bin/calamary
+This is essential for using the :code:`transparent` mode.
 
-    # make sure only wanted users can execute the binary!
-    chown root:proxy /usr/bin/calamary
-    chmod 750 /usr/bin/calamary
+For modes like :code:`proxyproto`, :code:`http`, :code:`https` or :code:`socks5` this is not necessary. (*but it's also possible using the :ref:`Redirector <redirector>`*)
 
-Read more about TPROXY here:
+You will have to choose between using **DNAT** and **TPROXY** to redirect the traffic on firewall-level.
 
-* `wiki.superstes.eu - NFTables - TProxy <https://wiki.superstes.eu/en/latest/1/network/firewall_nftables.html#tproxy>`_
+**TProxy** has the benefit that it won't modify the packets destination. This makes processing the traffic easier and can be benefitial in regards to performance.
 
-* `kernel docs <https://docs.kernel.org/networking/tproxy.html>`_
+But it also has the drawback that traffic that originates from the proxy-server (*netfilter hook - output*) will have to be looped-back.
 
-Output loopback
-===============
+I personally like to use TProxy for filtering input/forward- and DNAT for output-traffic.
 
-You will have to configure a loopback route if you want to proxy 'output' traffic:
 
-.. code-block:: bash
-
-    echo "200 proxy_loopback" > /etc/iproute2/rt_tables.d/proxy.conf
-
-    # These need to be configured persistent: (maybe use an interface up-hook)
-    ip rule add fwmark 200 table proxy_loopback
-    ip -6 rule add fwmark 200 table proxy_loopback
-    ip route add local 0.0.0.0/0 dev lo table proxy_loopback
-    ip -6 route add local ::/0 dev lo table proxy_loopback
-
-    # can be checked using:
-    ip rule list
-    ip -6 rule list
-    ip -d route show table all
-
-    # you might need to set a sysctl:
-    sysctl -w net.ipv4.conf.all.route_localnet=1
-
-    # you might want to block 127.0.0.1 on non loopback interfaces if you enable it:
-    iptables -t raw -A PREROUTING ! -i lo -d 127.0.0.0/8 -j DROP
-    iptables -t mangle -A POSTROUTING ! -o lo -s 127.0.0.0/8 -j DROP
+.. warning::
+    The config-examples below may not be complete!
+    If you find issues with them - please `open an issue <https://github.com/superstes/calamary/issues>`_
 
 
 NFTables
@@ -154,3 +124,56 @@ TProxy
 Full TProxy example: `gist.github.com/superstes - TProxy IPTables <https://gist.github.com/superstes/c4fefbf403f61812abf89165d7bc4000>`_
 
 You might need to enable some iptables kernel modules: `Kernel docs - IPTables extensions <https://docs.kernel.org/networking/tproxy.html#iptables-and-nf-tables-extensions>`_   
+
+
+TProxy
+######
+
+To run Calamary as `TPROXY <https://docs.kernel.org/networking/tproxy.html>`_ target - you will have to set `CAP_NET_RAW <https://man7.org/linux/man-pages/man7/capabilities.7.html>`_:
+
+::
+
+  bind to any address for transparent proxying
+
+You can add it like this:
+
+.. code-block:: bash
+
+    setcap cap_net_raw=+ep /usr/bin/calamary
+
+    # make sure only wanted users can execute the binary!
+    chown root:proxy /usr/bin/calamary
+    chmod 750 /usr/bin/calamary
+
+Read more about TPROXY here:
+
+* `wiki.superstes.eu - NFTables - TProxy <https://wiki.superstes.eu/en/latest/1/network/firewall_nftables.html#tproxy>`_
+
+* `kernel docs <https://docs.kernel.org/networking/tproxy.html>`_
+
+Output loopback
+===============
+
+You will have to configure a loopback route if you want to proxy 'output' traffic:
+
+.. code-block:: bash
+
+    echo "200 proxy_loopback" > /etc/iproute2/rt_tables.d/proxy.conf
+
+    # These need to be configured persistent: (maybe use an interface up-hook)
+    ip rule add fwmark 200 table proxy_loopback
+    ip -6 rule add fwmark 200 table proxy_loopback
+    ip route add local 0.0.0.0/0 dev lo table proxy_loopback
+    ip -6 route add local ::/0 dev lo table proxy_loopback
+
+    # can be checked using:
+    ip rule list
+    ip -6 rule list
+    ip -d route show table all
+
+    # you might need to set a sysctl:
+    sysctl -w net.ipv4.conf.all.route_localnet=1
+
+    # you might want to block 127.0.0.1 on non loopback interfaces if you enable it:
+    iptables -t raw -A PREROUTING ! -i lo -d 127.0.0.0/8 -j DROP
+    iptables -t mangle -A POSTROUTING ! -o lo -s 127.0.0.0/8 -j DROP

docs/source/info/redirector.rst

@@ -0,0 +1,26 @@
+.. _redirector:
+
+.. |proxy_forwarder| image:: https://wiki.superstes.eu/en/latest/_images/squid_remote.png
+   :class: wiki-img
+
+.. include:: ../_inc/head.rst
+
+##########
+Redirector
+##########
+
+The redirector will be a smaller version of Calamary. (*without filtering*)
+
+It can be used to forward traffic from remote locations to your proxy servers.
+
+|proxy_forwarder|
+
+Per example - this might be useful if you have:
+
+* Distributed systems
+
+  * Cloud servers that are only connected to public WAN and should send all their outbound traffic over your proxy
+
+  * Simple/dumb firewalls/routers that should send the networks outbound traffic over your proxy
+
+As it utilizes the commonly used `proxy-protocol <https://www.haproxy.com/blog/use-the-proxy-protocol-to-preserve-a-clients-ip-address>`_ to redirect the traffic, it might also be useful in combination with other proxies.

lib/log/main.go

@@ -9,7 +9,7 @@ import (
 
 func log(lvl string, pkg string, msg string) {
 	var base string
-	if cnf.C.Service.Debug {
+	if cnf.Debug() {
 		base = fmt.Sprintf("%s | %s | %s\n", lvl, pkg, msg)
 	} else {
 		base = fmt.Sprintf("%s | %s\n", lvl, msg)
@@ -22,9 +22,9 @@ func log(lvl string, pkg string, msg string) {
 	}
 }
 
-func logConn(lvl string, pkg string, src string, dst string, msg string) {
+func Conn(lvl string, pkg string, src string, dst string, msg string) {
 	var base string
-	if cnf.C.Service.Debug {
+	if cnf.Debug() {
 		base = fmt.Sprintf("%s | %s | %s => %s | %s\n", lvl, pkg, src, dst, msg)
 	} else {
 		base = fmt.Sprintf("%s | %s => %s | %s\n", lvl, src, dst, msg)
@@ -46,49 +46,33 @@ func Error(pkg string, err error) {
 	log("ERROR", pkg, fmt.Sprintf("%s", err))
 }
 
-func ConnErrorS(pkg string, src string, dst string, msg string) {
-	logConn("ERROR", pkg, src, dst, msg)
-}
-
-func ConnError(pkg string, src string, dst string, err error) {
-	logConn("ERROR", pkg, src, dst, fmt.Sprintf("%s", err))
-}
-
-func ConnWarnS(pkg string, src string, dst string, msg string) {
-	logConn("WARN", pkg, src, dst, msg)
-}
-
-func ConnWarn(pkg string, src string, dst string, err error) {
-	logConn("WARN", pkg, src, dst, fmt.Sprintf("%s", err))
+func ConnError(pkg string, src string, dst string, msg string) {
+	Conn("ERROR", pkg, src, dst, msg)
 }
 
 func Debug(pkg string, msg string) {
-	if cnf.C.Service.Debug {
+	if cnf.Debug() {
 		log("DEBUG", pkg, msg)
 	}
 }
 
 func ConnDebug(pkg string, src string, dst string, msg string) {
-	if cnf.C.Service.Debug {
-		logConn("DEBUG", pkg, src, dst, msg)
+	if cnf.Debug() {
+		Conn("DEBUG", pkg, src, dst, msg)
 	}
 }
 
 func Info(pkg string, msg string) {
 	log("INFO", pkg, msg)
 }
 
-func ConnInfo(pkg string, src string, dst string, msg string) {
-	logConn("INFO", pkg, src, dst, msg)
-}
-
 func Warn(pkg string, msg string) {
 	log("WARN", pkg, msg)
 }
 
 func Fatal(pkg string, msg string) {
 	var base string
-	if cnf.C.Service.Debug {
+	if cnf.Debug() {
 		base = fmt.Sprintf("FATAL | %s | %s\n", pkg, msg)
 	} else {
 		base = fmt.Sprintf("FATAL | %s\n", msg)

lib/main/main.go

@@ -30,13 +30,13 @@ func main() {
 
 	if modeValidate {
 		cnf.C.Service.Debug = true
-		cnf_file.Load(true)
+		cnf_file.Load(true, true)
 		log.Info("config", "Config validated successfully")
 		os.Exit(0)
 	}
 
 	welcome()
-	cnf_file.Load(false)
+	cnf_file.Load(false, true)
 	service := &service{}
 	go startPrometheusExporter()
 	service.start()

lib/main/metrics.go

@@ -11,38 +11,40 @@ import (
 	"github.com/superstes/calamary/metrics"
 )
 
-func catchAll(w http.ResponseWriter, r *http.Request) {
+// todo: clean-up redundant linking
+var metricFuncs = []prometheus.Collector{
+	metrics.BytesRcv,
+	metrics.BytesSent,
+	metrics.CurrentConn,
+	metrics.ReqTcp,
+	metrics.ReqL3Proto,
+	metrics.ReqL5Proto,
+	metrics.ReqTlsVersion,
+	metrics.RuleHits,
+	metrics.RuleMatches,
+	metrics.RuleActions,
+}
+
+func denyAll(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusForbidden)
 }
 
 func startPrometheusExporter() {
 	if cnf.Metrics() {
 		log.Info("service", "Starting prometheus metrics-exporter")
-		prometheus.Register(metrics.BytesRcv)
-		prometheus.Register(metrics.BytesSent)
-		prometheus.Register(metrics.CurrentConn)
-		prometheus.Register(metrics.ReqTcp)
-		prometheus.Register(metrics.RuleReqL3Proto)
-		prometheus.Register(metrics.RuleReqL5Proto)
-		prometheus.Register(metrics.RuleReqTlsVersion)
-		prometheus.Register(metrics.RuleHits)
-		prometheus.Register(metrics.RuleMatches)
-		prometheus.Register(metrics.RuleActions)
-
-		http.Handle("/metrics", promhttp.Handler())
-		http.HandleFunc("/", catchAll)
-		http.ListenAndServe(fmt.Sprintf("127.0.0.1:%v", cnf.C.Service.Metrics.Port), nil)
-		http.ListenAndServe(fmt.Sprintf("[::1]:%v", cnf.C.Service.Metrics.Port), nil)
-
-		prometheus.MustRegister(metrics.BytesRcv)
-		prometheus.MustRegister(metrics.BytesSent)
-		prometheus.MustRegister(metrics.CurrentConn)
-		prometheus.MustRegister(metrics.ReqTcp)
-		prometheus.MustRegister(metrics.RuleReqL3Proto)
-		prometheus.MustRegister(metrics.RuleReqL5Proto)
-		prometheus.MustRegister(metrics.RuleReqTlsVersion)
-		prometheus.MustRegister(metrics.RuleHits)
-		prometheus.MustRegister(metrics.RuleMatches)
-		prometheus.MustRegister(metrics.RuleActions)
+
+		for i := range metricFuncs {
+			prometheus.Register(metricFuncs[i])
+		}
+
+		metricsSrv := http.NewServeMux()
+		metricsSrv.Handle("/metrics", promhttp.Handler())
+		metricsSrv.HandleFunc("/", denyAll)
+		http.ListenAndServe(fmt.Sprintf("127.0.0.1:%v", cnf.C.Service.Metrics.Port), metricsSrv)
+		http.ListenAndServe(fmt.Sprintf("[::1]:%v", cnf.C.Service.Metrics.Port), metricsSrv)
+
+		for i := range metricFuncs {
+			prometheus.MustRegister(metricFuncs[i])
+		}
 	}
 }

lib/metrics/main.go

@@ -26,23 +26,23 @@ var (
 		Name:      "req_tcp",
 		Help:      "Count of tcp requests received",
 	})
-	RuleReqL3Proto = prometheus.NewCounterVec(
+	ReqL3Proto = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: "calamary",
 			Name:      "req_protoL3",
 			Help:      "Count of requests per L3 protocol",
 		},
 		[]string{"protocol"},
 	)
-	RuleReqL5Proto = prometheus.NewCounterVec(
+	ReqL5Proto = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: "calamary",
 			Name:      "req_protoL5",
 			Help:      "Count of requests per L5 protocol",
 		},
 		[]string{"protocol"},
 	)
-	RuleReqTlsVersion = prometheus.NewCounterVec(
+	ReqTlsVersion = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: "calamary",
 			Name:      "req_tls_version",

lib/proc/filter/main.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	"github.com/superstes/calamary/cnf"
-	"github.com/superstes/calamary/log"
 	"github.com/superstes/calamary/metrics"
 	"github.com/superstes/calamary/proc/meta"
 	"github.com/superstes/calamary/proc/parse"
@@ -49,13 +48,17 @@ func Filter(pkt parse.ParsedPacket) bool {
 	}
 
 	// implicit deny
-	log.ConnDebug("filter", parse.PktSrc(pkt), parse.PktDest(pkt), "No rule matched - implicit deny")
+	if cnf.Metrics() {
+		metrics.RuleMatches.WithLabelValues("default").Inc()
+		metrics.RuleActions.WithLabelValues(meta.RevRuleAction(meta.ActionDeny)).Inc()
+	}
+	parse.LogConnDebug("filter", pkt, "No rule matched - implicit deny")
 	return applyAction(meta.ActionDeny)
 }
 
 func ruleDebug(pkt parse.ParsedPacket, rule_id int, msg string) {
-	if cnf.C.Service.Debug {
-		log.ConnDebug("filter", parse.PktSrc(pkt), parse.PktDest(pkt), fmt.Sprintf("Rule %v - %s", rule_id, msg))
+	if cnf.Debug() {
+		parse.LogConnDebug("filter", pkt, fmt.Sprintf("Rule %v - %s", rule_id, msg))
 	}
 }
 

lib/proc/parse/conn_log.go

@@ -0,0 +1,19 @@
+package parse
+
+import "github.com/superstes/calamary/log"
+
+func LogConnDebug(pkg string, pkt ParsedPacket, msg string) {
+	log.Conn("DEBUG", pkg, PktSrc(pkt), PktDest(pkt), msg)
+}
+
+func LogConnInfo(pkg string, pkt ParsedPacket, msg string) {
+	log.Conn("INFO", pkg, PktSrc(pkt), PktDest(pkt), msg)
+}
+
+func LogConnWarn(pkg string, pkt ParsedPacket, msg string) {
+	log.Conn("WARN", pkg, PktSrc(pkt), PktDest(pkt), msg)
+}
+
+func LogConnError(pkg string, pkt ParsedPacket, msg string) {
+	log.Conn("ERROR", pkg, PktSrc(pkt), PktDest(pkt), msg)
+}

lib/proc/parse/helpers.go

@@ -13,6 +13,9 @@ func PktSrc(pkt ParsedPacket) string {
 }
 
 func PktDest(pkt ParsedPacket) string {
+	if pkt.L3.DestIP == nil {
+		return "?"
+	}
 	return fmt.Sprintf("%s:%v", pkt.L3.DestIP.String(), pkt.L4.DestPort)
 }