implemented http-proxy

README.md

@@ -23,7 +23,7 @@ Its focus is set on transparent security filtering.
 
   - [ ] Proxy-Protocol
 
-  - [ ] HTTP Proxy
+  - [x] HTTP Proxy
 
   - [ ] HTTPS Proxy
 
@@ -33,7 +33,7 @@ Its focus is set on transparent security filtering.
 
   - [x] TCP
 
-    - [ ] HTTP
+    - [x] HTTP
 
   - [x] TLS
 
@@ -55,7 +55,7 @@ Its focus is set on transparent security filtering.
   - [ ] Listener-Specific
 
     - [ ] Proxy-Protocol
-    - [ ] HTTP Proxy
+    - [x] HTTP Proxy
     - [ ] SOCKS5 Proxy
 
   - [ ] DNS

config_example.yml

@@ -24,6 +24,7 @@ service:
     interceptPublic: '/tmp/calamary.subca.crt'
     interceptPrivate: '/tmp/calamary.subca.key'
 
+  dnsNameservers: ['1.1.1.1', '8.8.8.8']
   debug: false
   timeout:
     connect: 2000

docs/source/info/getting_started.rst

@@ -31,6 +31,50 @@ Config-validation only:
     /usr/bin/calamary -v
 
 
+Modes
+#####
+
+Transparent
+===========
+
+**State:** Implemented/Testing
+
+Calamary focuses on transparent traffic interception.
+
+You will have to redirect the traffic: :ref:`Redirect <redirect>`
+
+This mode will work for TCP & UDP.
+
+HTTP/HTTPS Proxy
+================
+
+**State:** Implemented/Testing
+
+You can also choose to let Calamary act as a HTTP/S proxy.
+
+One commonly uses this feature if only some applications should send their traffic over the proxy.
+
+This mode only supports TCP.
+
+Note: Calamary uses TLS-SNI > Host-Header to find its actual target host. It will also check all IPs (IPv6 > IPv4) that are returned by the DNS query for their reachability, before establishing a connection.
+
+SOCKS5 Proxy
+============
+
+**State:** not implemented
+
+Like HTTP/S proxy, but it works for UDP as well.
+
+Proxy-Protocol
+==============
+
+**State:** in development
+
+You can use the proxy-protcol mode if you want to send traffic from remote systems over the proxy.
+
+The commonly used `proxy-protocol <https://www.haproxy.com/blog/use-the-proxy-protocol-to-preserve-a-clients-ip-address>`_ preserves the original source- & destination while minimizing overhead.
+
+
 Configuration
 #############
 
@@ -60,6 +104,7 @@ Basic config example:
         interceptPublic: '/tmp/calamary.subca.crt'
         interceptPrivate: '/tmp/calamary.subca.key'
 
+      dnsNameservers: ['1.1.1.1', '8.8.8.8']
       debug: false
       timeout:  # ms
         connect: 2000

docs/source/info/intro.rst

@@ -11,22 +11,26 @@ Intro
 
 Calamary is a `squid <http://www.squid-cache.org/>`_-like proxy.
 
-Its focus is set on **security filtering for HTTPS**.
+Its focus is set on **security filtering for HTTPS/TLS**.
 
-**It will not**:
-
-* act as caching proxy
-* act as reverse proxy
+The ruleset should be logical, transparent & easy to understand.
 
 **Features**:
 
-* basic traffic filtering - see :ref:`Rules <rules>`
+* support for mainstream :ref:`proxy modes <getting_started>`
+* filtering ruleset - see :ref:`Rules <rules>`
+
+  * ability to filter on protocol-basis
+  * ability to enforce TLS (*deny any unencrypted connections*)
+
 * certificate verification
-* enforce TLS (*deny any unencrypted connections*)
 * detect plain HTTP and respond with generic HTTPS-redirect
-* intercept-, http-proxy and proxy-proto-modes
 
-  * support for `proxy-protocol <https://github.com/pires/go-proxyproto>`_
+**It will not**:
+
+* act as caching proxy
+* act as reverse proxy
+* implement edge-case workarounds for unencrypted protocols
 
 Getting Started
 ###############
@@ -77,17 +81,18 @@ I would much preferr a keep-it-simple approach. Even if that means that some nic
 How?
 ####
 
-* Use TLS-SNI as target instead of HTTP Host-Header
+* Plaintext HTTP is not that common anymore.
 
+  We are using TLS-SNI > Host-Header to resolve the target.
 
-* Optionally use additional DNS-based verfication if TTL > 3 min
+  Plain HTTP is unsecure by default. So we won't check for Host-Header mangling.
 
+  The ruleset is applied 'postrouting' (*IP/Net matching*) and Host-Header domains are ignored by the ruleset.
 
-* Whenever it is not possible to route the traffic through the proxy..
 
-  To overcome the DNAT restriction, of losing the real target IP, the proxy will have a lightweight forwarder mode:
+* Whenever it is not possible to route the traffic through the proxy..
 
-  |proxy_forwarder|
+  To overcome the DNAT restriction, of losing the real target IP, there will be a :ref:`Redirector <redirector>`!
 
 
 * **Transparent traffic interception will be the focus**.

docs/source/info/redirect.rst

@@ -15,7 +15,7 @@ You may want/need to redirect traffic to the proxy's listeners for some use-case
 
 This is essential for using the :code:`transparent` mode.
 
-For modes like :code:`proxyproto`, :code:`http`, :code:`https` or :code:`socks5` this is not necessary. (*but it's also possible using the :ref:`Redirector <redirector>`*)
+For modes like :code:`proxyproto`, :code:`http`, :code:`https` or :code:`socks5` this is not necessary. (*but it's also possible using the* :ref:`Redirector <redirector>`)
 
 You will have to choose between using **DNAT** and **TPROXY** to redirect the traffic on firewall-level.
 

lib/cnf/cnf_file/rules_parse.go

@@ -15,12 +15,10 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) {
 	var v cnf.Var
 	var vf bool
 	var vn bool
-	var value string
 
 	// todo: move duplicate lines into sub-functions
 
-	for i := range rawRules {
-		ruleRaw := rawRules[i]
+	for _, ruleRaw := range rawRules {
 		rule := cnf.Rule{
 			Action: meta.RuleAction(ruleRaw.Action),
 		}
@@ -31,8 +29,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) {
 			rule.Match.SrcNet = []*net.IPNet{}
 			rule.Match.SrcNetN = []*net.IPNet{}
 		}
-		for i2 := range ruleRaw.Match.SrcNet {
-			value = ruleRaw.Match.SrcNet[i2]
+		for _, value := range ruleRaw.Match.SrcNet {
 			vf, vn, v = usedVar(value)
 			if vf {
 				for i3 := range v.Value {
@@ -56,8 +53,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) {
 			rule.Match.DestNet = []*net.IPNet{}
 			rule.Match.DestNetN = []*net.IPNet{}
 		}
-		for i2 := range ruleRaw.Match.DestNet {
-			value = ruleRaw.Match.DestNet[i2]
+		for _, value := range ruleRaw.Match.DestNet {
 			vf, vn, v = usedVar(value)
 			if vf {
 				for i3 := range v.Value {
@@ -81,8 +77,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) {
 			rule.Match.SrcPort = []uint16{}
 			rule.Match.SrcPortN = []uint16{}
 		}
-		for i2 := range ruleRaw.Match.SrcPort {
-			value = ruleRaw.Match.SrcPort[i2]
+		for _, value := range ruleRaw.Match.SrcPort {
 			vf, vn, v = usedVar(value)
 			if vf {
 				for i3 := range v.Value {
@@ -106,8 +101,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) {
 			rule.Match.DestPort = []uint16{}
 			rule.Match.DestPortN = []uint16{}
 		}
-		for i2 := range ruleRaw.Match.DestPort {
-			value = ruleRaw.Match.DestPort[i2]
+		for _, value := range ruleRaw.Match.DestPort {
 			vf, vn, v = usedVar(value)
 			if vf {
 				for i3 := range v.Value {
@@ -131,8 +125,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) {
 			rule.Match.ProtoL3 = []meta.Proto{}
 			rule.Match.ProtoL3N = []meta.Proto{}
 		}
-		for i2 := range ruleRaw.Match.ProtoL3 {
-			value = ruleRaw.Match.ProtoL3[i2]
+		for _, value := range ruleRaw.Match.ProtoL3 {
 			vf, vn, v = usedVar(value)
 			if vf {
 				for i3 := range v.Value {
@@ -156,8 +149,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) {
 			rule.Match.ProtoL4 = []meta.Proto{}
 			rule.Match.ProtoL4N = []meta.Proto{}
 		}
-		for i2 := range ruleRaw.Match.ProtoL4 {
-			value = ruleRaw.Match.ProtoL4[i2]
+		for _, value := range ruleRaw.Match.ProtoL4 {
 			vf, vn, v = usedVar(value)
 			if vf {
 				for i3 := range v.Value {
@@ -181,8 +173,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) {
 			rule.Match.ProtoL5 = []meta.Proto{}
 			rule.Match.ProtoL5N = []meta.Proto{}
 		}
-		for i2 := range ruleRaw.Match.ProtoL5 {
-			value = ruleRaw.Match.ProtoL5[i2]
+		for _, value := range ruleRaw.Match.ProtoL5 {
 			vf, vn, v = usedVar(value)
 			if vf {
 				for i3 := range v.Value {
@@ -205,8 +196,7 @@ func ParseRules(rawRules []cnf.RuleRaw) (rules []cnf.Rule) {
 		if len(ruleRaw.Match.Domains) > 0 {
 			rule.Match.Domains = []string{}
 		}
-		for i2 := range ruleRaw.Match.Domains {
-			value = ruleRaw.Match.Domains[i2]
+		for _, value := range ruleRaw.Match.Domains {
 			vf, vn, v = usedVar(value)
 			if vf {
 				for i3 := range v.Value {

lib/cnf/cnf_file/validate.go

@@ -15,8 +15,8 @@ import (
 )
 
 func validateConfig(newCnf cnf.Config, fail bool) bool {
-	for i := range newCnf.Service.Listen {
-		if !validateListener(newCnf.Service.Listen[i], fail) {
+	for _, ln := range newCnf.Service.Listen {
+		if !validateListener(ln, fail) {
 			return false
 		}
 	}

lib/cnf/config.go

@@ -15,12 +15,13 @@ type Config struct {
 }
 
 type ServiceConfig struct {
-	Timeout ServiceTimeout      `yaml:"timeout"`
-	Listen  []ServiceListener   `yaml:"listen"`
-	Certs   ServiceCertificates `yaml:"certs"`
-	Output  ServiceOutput       `yaml:"output"`
-	Debug   bool                `yaml:"debug" default:"false"`
-	Metrics ServiceMetrics      `yaml:"metrics"`
+	Timeout        ServiceTimeout      `yaml:"timeout"`
+	Listen         []ServiceListener   `yaml:"listen"`
+	Certs          ServiceCertificates `yaml:"certs"`
+	Output         ServiceOutput       `yaml:"output"`
+	Debug          bool                `yaml:"debug" default:"false"`
+	Metrics        ServiceMetrics      `yaml:"metrics"`
+	DnsNameservers YamlStringArray     `yaml:"dnsNameservers" default:"[\"1.1.1.1\", \"8.8.8.8\"]"`
 }
 
 // todo: implement default listen-ips = localhost
@@ -37,9 +38,11 @@ type ServiceListener struct {
 }
 
 type ServiceTimeout struct {
-	Connect uint `yaml:"connect" default:"2000"` // dial
-	Process uint `yaml:"process" default:"1000"` // parsing packet
-	Idle    uint `yaml:"idle" default:"30000"`   // close connection if no data was sent or received
+	Connect   uint `yaml:"connect" default:"2000"` // dial
+	Process   uint `yaml:"process" default:"1000"` // parsing packet
+	Idle      uint `yaml:"idle" default:"30000"`   // close connection if no data was sent or received
+	Probe     uint `yaml:"probe" default:"500"`    // check if target port is reachable before establishing connection
+	DnsLookup uint `yaml:"dnsLookup" default:"200"`
 }
 
 var DefaultConnectRetryWait = uint(1000) // ms
@@ -53,6 +56,7 @@ type ServiceOutput struct {
 }
 
 var DefaultMetricsPort = uint16(9512)
+var InboundRetries = 3
 
 type ServiceMetrics struct {
 	Enabled bool   `yaml:"enabled" default:"false"`

lib/cnf/hardcoded.go

@@ -1,6 +1,7 @@
 package cnf
 
 import (
+	"net"
 	"time"
 )
 
@@ -16,3 +17,31 @@ const (
 )
 
 var ConfigFileAbs string = "/etc/calamary/config.yml"
+
+var NetForwardDeny []*net.IPNet
+
+func InitNetForwardDeny() {
+	_, localhost1, _ := net.ParseCIDR("127.0.0.0/8")
+	_, localhost2, _ := net.ParseCIDR("::1/128")
+	_, localhost3, _ := net.ParseCIDR("::/128")
+	_, linklocal1, _ := net.ParseCIDR("169.254.0.0/16")
+	_, linklocal2, _ := net.ParseCIDR("fe80::/10")
+	_, linklocal3, _ := net.ParseCIDR("fc00::/7")
+	_, multicast1, _ := net.ParseCIDR("224.0.0.0/4")
+	_, multicast2, _ := net.ParseCIDR("ff00::/8")
+	_, broadcast1, _ := net.ParseCIDR("255.255.255.255/32")
+	_, blackhole1, _ := net.ParseCIDR("100::/64")
+
+	NetForwardDeny = []*net.IPNet{
+		localhost1,
+		localhost2,
+		localhost3,
+		linklocal1,
+		linklocal2,
+		linklocal3,
+		multicast1,
+		multicast2,
+		broadcast1,
+		blackhole1,
+	}
+}

lib/cnf/unmarshal.go

@@ -36,6 +36,17 @@ func (s *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
+func (s *ServiceConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	defaults.Set(s)
+
+	type plain ServiceConfig
+	if err := unmarshal((*plain)(s)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (s *ServiceListener) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	defaults.Set(s)
 

lib/cnf/unmarshal_test.go

@@ -1,6 +1,7 @@
 package cnf
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -60,4 +61,9 @@ func TestConfigDefaults(t *testing.T) {
 	if cnf2.Service.Metrics.Port != 9512 {
 		t.Errorf("Unmarshal defaults-file #6 (%+v)", cnf2.Service.Metrics)
 	}
+	if len(cnf2.Service.DnsNameservers) != 2 ||
+		fmt.Sprintf("%v", cnf2.Service.DnsNameservers[0]) != "1.1.1.1" ||
+		fmt.Sprintf("%v", cnf2.Service.DnsNameservers[1]) != "8.8.8.8" {
+		t.Errorf("Unmarshal defaults-file #7 (%+v)", cnf2.Service.DnsNameservers)
+	}
 }