Support DNS SRV Records within Ringpop (#4614)

Author: lindleywhite

common/config/config.go

@@ -114,7 +114,7 @@ type (
 	Ringpop struct {
 		// Name to be used in ringpop advertisement
 		Name string `yaml:"name" validate:"nonzero"`
-		// BootstrapMode is a enum that defines the ringpop bootstrap method
+		// BootstrapMode is a enum that defines the ringpop bootstrap method, currently supports: hosts, files, custom, dns, and dns-srv
 		BootstrapMode BootstrapMode `yaml:"bootstrapMode"`
 		// BootstrapHosts is a list of seed hosts to be used for ringpop bootstrap
 		BootstrapHosts []string `yaml:"bootstrapHosts"`

common/config/ringpop.go

@@ -57,6 +57,10 @@ const (
 	// BootstrapModeDNS represents a list of hosts passed in the configuration
 	// to be resolved, and the resulting addresses are used for bootstrap
 	BootstrapModeDNS
+	// BootstrapModeDNSSRV represents a list of DNS hosts passed in the configuration
+	// to resolve secondary addresses that DNS SRV record would return resulting in
+	// a host list that will contain multiple dynamic addresses and their unique ports
+	BootstrapModeDNSSRV
 )
 
 const (
@@ -130,6 +134,8 @@ func parseBootstrapMode(
 		return BootstrapModeCustom, nil
 	case "dns":
 		return BootstrapModeDNS, nil
+	case "dns-srv":
+		return BootstrapModeDNSSRV, nil
 	}
 	return BootstrapModeNone, errors.New("invalid or no ringpop bootstrap mode")
 }
@@ -143,7 +149,7 @@ func validateBootstrapMode(
 		if len(rpConfig.BootstrapFile) == 0 {
 			return fmt.Errorf("ringpop config missing bootstrap file param")
 		}
-	case BootstrapModeHosts, BootstrapModeDNS:
+	case BootstrapModeHosts, BootstrapModeDNS, BootstrapModeDNSSRV:
 		if len(rpConfig.BootstrapHosts) == 0 {
 			return fmt.Errorf("ringpop config missing boostrap hosts param")
 		}
@@ -263,6 +269,7 @@ func (factory *RingpopFactory) getChannel(
 
 type dnsHostResolver interface {
 	LookupHost(ctx context.Context, host string) (addrs []string, err error)
+	LookupSRV(ctx context.Context, service, proto, name string) (cname string, addrs []*net.SRV, err error)
 }
 
 type dnsProvider struct {
@@ -322,6 +329,84 @@ func (provider *dnsProvider) Hosts() ([]string, error) {
 	return results, nil
 }
 
+type dnsSRVProvider struct {
+	UnresolvedHosts []string
+	Resolver        dnsHostResolver
+	Logger          log.Logger
+}
+
+func newDNSSRVProvider(
+	hosts []string,
+	resolver dnsHostResolver,
+	logger log.Logger,
+) *dnsSRVProvider {
+
+	set := map[string]struct{}{}
+	for _, hostport := range hosts {
+		set[hostport] = struct{}{}
+	}
+
+	var keys []string
+	for key := range set {
+		keys = append(keys, key)
+	}
+
+	return &dnsSRVProvider{
+		UnresolvedHosts: keys,
+		Resolver:        resolver,
+		Logger:          logger,
+	}
+}
+
+func (provider *dnsSRVProvider) Hosts() ([]string, error) {
+	var results []string
+	resolvedHosts := map[string][]string{}
+
+	for _, service := range provider.UnresolvedHosts {
+		serviceParts := strings.Split(service, ".")
+		if len(serviceParts) <= 2 {
+			provider.Logger.Error("could not seperate service name from domain", tag.Address(service))
+			return nil, errors.New("could not seperate service name from domain. check host configuration")
+		}
+		serviceName := serviceParts[0]
+		domain := strings.Join(serviceParts[1:], ".")
+		resolved, exists := resolvedHosts[serviceName]
+		if !exists {
+			_, addrs, err := provider.Resolver.LookupSRV(context.Background(), serviceName, "tcp", domain)
+
+			if err != nil {
+				provider.Logger.Error("could not resolve host", tag.Address(serviceName), tag.Error(err))
+				return nil, errors.New(fmt.Sprintf("could not resolve host: %s.%s", serviceName, domain))
+			}
+
+			var targets []string
+			for _, record := range addrs {
+			   target, err := provider.Resolver.LookupHost(context.Background(), record.Target)
+
+			   if err != nil {
+					provider.Logger.Warn("could not resolve srv dns host", tag.Address(record.Target), tag.Error(err))
+					continue
+			   }
+			   for _, host := range target {
+			   		targets = append(targets, net.JoinHostPort(host, fmt.Sprintf("%d", record.Port)))
+			   }
+			}
+			resolvedHosts[serviceName] = targets
+			resolved = targets
+		}
+
+		for _, r := range resolved {
+			results = append(results, r)
+		}
+
+	}
+
+	if len(results) == 0 {
+		return nil, errors.New("no hosts found, and bootstrap requires at least one")
+	}
+	return results, nil
+}
+
 func newDiscoveryProvider(
 	cfg *Ringpop,
 	logger log.Logger,
@@ -339,6 +424,8 @@ func newDiscoveryProvider(
 		return jsonfile.New(cfg.BootstrapFile), nil
 	case BootstrapModeDNS:
 		return newDNSProvider(cfg.BootstrapHosts, net.DefaultResolver, logger), nil
+	case BootstrapModeDNSSRV:
+		return newDNSSRVProvider(cfg.BootstrapHosts, net.DefaultResolver, logger), nil
 	}
 	return nil, fmt.Errorf("unknown bootstrap mode")
 }

common/config/ringpop_test.go

@@ -25,6 +25,7 @@ import (
 	"fmt"
 	"testing"
 	"time"
+	"net"
 
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
@@ -93,6 +94,8 @@ func (s *RingpopSuite) TestCustomMode() {
 
 type mockResolver struct {
 	Hosts map[string][]string
+	SRV map[string][]net.SRV
+	suite *RingpopSuite
 }
 
 func (resolver *mockResolver) LookupHost(ctx context.Context, host string) ([]string, error) {
@@ -103,6 +106,22 @@ func (resolver *mockResolver) LookupHost(ctx context.Context, host string) ([]st
 	return addrs, nil
 }
 
+func (resolver *mockResolver) LookupSRV(ctx context.Context, service string, proto string, name string) (string, []*net.SRV, error) {
+	var records []*net.SRV
+	srvs, ok := resolver.SRV[service]
+	if !ok {
+		return "", nil, fmt.Errorf("Host was not resolved: %s", service)
+	}
+
+	for _, record := range srvs {
+		var srvRecord net.SRV
+		srvRecord = record
+		records = append(records, &srvRecord)
+	}
+
+	return "test", records, nil
+}
+
 func (s *RingpopSuite) TestDNSMode() {
 	var cfg Ringpop
 	err := yaml.Unmarshal([]byte(getDNSConfig()), &cfg)
@@ -165,6 +184,102 @@ func (s *RingpopSuite) TestDNSMode() {
 	s.NotNil(err, "error should be returned when no hosts")
 }
 
+func (s *RingpopSuite) TestDNSSRVMode() {
+	var cfg Ringpop
+	err := yaml.Unmarshal([]byte(getDNSSRVConfig()), &cfg)
+	s.Nil(err)
+	s.Equal("test", cfg.Name)
+	s.Equal(BootstrapModeDNSSRV, cfg.BootstrapMode)
+	s.Nil(cfg.validate())
+	logger := loggerimpl.NewNopLogger()
+	f, err := cfg.NewFactory(nil, "test", logger)
+	s.Nil(err)
+	s.NotNil(f)
+
+	s.ElementsMatch(
+		[]string{
+			"service-a.example.net",
+			"service-b.example.net",
+			"unknown-duplicate.example.net",
+			"unknown-duplicate.example.net",
+			"badhostport",
+		},
+		cfg.BootstrapHosts,
+	)
+
+	provider := newDNSSRVProvider(
+		cfg.BootstrapHosts,
+		&mockResolver{
+			SRV: map[string][]net.SRV{
+				"service-a": []net.SRV{{ Target:"az1-service-a.addr.example.net", Port: 7755}, {Target: "az2-service-a.addr.example.net", Port: 7566}},
+				"service-b": []net.SRV{{ Target:"az1-service-b.addr.example.net", Port: 7788}, {Target: "az2-service-b.addr.example.net", Port: 7896}},
+			},
+			Hosts: map[string][]string{
+				"az1-service-a.addr.example.net": []string{"10.0.0.1"},
+				"az2-service-a.addr.example.net": []string{"10.0.2.0", "10.0.2.3"},
+				"az1-service-b.addr.example.net": []string{"10.0.3.0", "10.0.3.3"},
+				"az2-service-b.addr.example.net": []string{"10.0.3.1"},
+			},
+			suite: s,
+		},
+		logger,
+	)
+	cfg.DiscoveryProvider = provider
+	s.ElementsMatch(
+		[]string{
+			"service-a.example.net",
+			"service-b.example.net",
+			"unknown-duplicate.example.net",
+			"badhostport",
+		},
+		provider.UnresolvedHosts,
+		"duplicate entries should be removed",
+	)
+
+	//Expect unknown-duplicate.example.net to not resolve
+	_, err = cfg.DiscoveryProvider.Hosts()
+	s.NotNil(err)
+
+	//Remove known bad hosts from Unresolved list
+	provider.UnresolvedHosts = []string{
+		"service-a.example.net",
+		"service-b.example.net",
+		"badhostport",
+	}
+
+	//Expect badhostport to not seperate service name
+	_, err = cfg.DiscoveryProvider.Hosts()
+	s.NotNil(err)
+
+
+	//Remove known bad hosts from Unresolved list
+	provider.UnresolvedHosts = []string{
+		"service-a.example.net",
+		"service-b.example.net",
+	}
+
+	hostports, err := cfg.DiscoveryProvider.Hosts()
+	s.Nil(err)
+	s.ElementsMatch(
+		[]string{
+			"10.0.0.1:7755",
+			"10.0.2.0:7566", "10.0.2.3:7566",
+			"10.0.3.0:7788", "10.0.3.3:7788",
+			"10.0.3.1:7896",
+		},
+		hostports,
+	)
+
+	cfg.DiscoveryProvider = newDNSProvider(
+		cfg.BootstrapHosts,
+		&mockResolver{Hosts: map[string][]string{}},
+		logger,
+	)
+	hostports, err = cfg.DiscoveryProvider.Hosts()
+	s.Nil(hostports)
+	s.NotNil(err, "error should be returned when no hosts")
+}
+
 func (s *RingpopSuite) TestInvalidConfig() {
 	var cfg Ringpop
 	s.NotNil(cfg.validate())
@@ -207,3 +322,15 @@ bootstrapHosts:
 - badhostport
 maxJoinDuration: 30s`
 }
+
+func getDNSSRVConfig() string {
+	return `name: "test"
+bootstrapMode: "dns-srv"
+bootstrapHosts:
+- service-a.example.net
+- service-b.example.net
+- unknown-duplicate.example.net
+- unknown-duplicate.example.net
+- badhostport
+maxJoinDuration: 30s`
+}