Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
The happy eyeballs sorting of a multi-address host's address list now happens once when the
address list is created or refreshed, instead of on every upstream connection attempt. The order
in which connection attempts are made is unchanged.
21 changes: 11 additions & 10 deletions source/common/network/happy_eyeballs_connection_impl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,23 @@ namespace Envoy {
namespace Network {

HappyEyeballsConnectionProvider::HappyEyeballsConnectionProvider(
Event::Dispatcher& dispatcher, const std::vector<Address::InstanceConstSharedPtr>& address_list,
Event::Dispatcher& dispatcher,
const Upstream::HostDescription::SharedConstAddressVector& sorted_address_list,
const std::shared_ptr<const Upstream::UpstreamLocalAddressSelector>&
upstream_local_address_selector,
UpstreamTransportSocketFactory& socket_factory,
TransportSocketOptionsConstSharedPtr transport_socket_options,
const Upstream::HostDescriptionConstSharedPtr& host,
const ConnectionSocket::OptionsSharedPtr options,
const envoy::config::cluster::v3::UpstreamConnectionOptions::HappyEyeballsConfig&
happy_eyeballs_config)
: dispatcher_(dispatcher), address_list_(sortAddresses(address_list, happy_eyeballs_config)),
const ConnectionSocket::OptionsSharedPtr options)
: dispatcher_(dispatcher), address_list_(sorted_address_list),
upstream_local_address_selector_(upstream_local_address_selector),
socket_factory_(socket_factory), transport_socket_options_(transport_socket_options),
host_(host), options_(options) {}
host_(host), options_(options) {
ASSERT(address_list_ != nullptr && !address_list_->empty());
}

bool HappyEyeballsConnectionProvider::hasNextConnection() {
return next_address_ < address_list_.size();
return next_address_ < address_list_->size();
}

ClientConnectionPtr HappyEyeballsConnectionProvider::createNextConnection(const uint64_t id) {
Expand All @@ -37,8 +38,8 @@ ClientConnectionPtr HappyEyeballsConnectionProvider::createNextConnection(const
first_connection_created_ = true;
ASSERT(hasNextConnection());
ENVOY_LOG_EVENT(debug, "happy_eyeballs_cx_attempt", "C[{}] address={}", id,
address_list_[next_address_]->asStringView());
auto& address = address_list_[next_address_++];
(*address_list_)[next_address_]->asStringView());
auto& address = (*address_list_)[next_address_++];
auto upstream_local_address = upstream_local_address_selector_->getUpstreamLocalAddress(
address, options_, makeOptRefFromPtr(transport_socket_options_.get()));

Expand All @@ -50,7 +51,7 @@ ClientConnectionPtr HappyEyeballsConnectionProvider::createNextConnection(const

size_t HappyEyeballsConnectionProvider::nextConnection() { return next_address_; }

size_t HappyEyeballsConnectionProvider::totalConnections() { return address_list_.size(); }
size_t HappyEyeballsConnectionProvider::totalConnections() { return address_list_->size(); }

namespace {

Expand Down
34 changes: 16 additions & 18 deletions source/common/network/happy_eyeballs_connection_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,22 @@ namespace Network {
/**
* Implementation of ConnectionProvider for HappyEyeballs. It provides client
* connections to multiple addresses in an specific order complying to
* HappyEyeballs.
* HappyEyeballs. The address list passed to the constructor must already be
* sorted with sortAddresses(); the host computes this once when its address
* list is created or refreshed rather than on every connection attempt.
*/
class HappyEyeballsConnectionProvider : public ConnectionProvider,
Logger::Loggable<Logger::Id::happy_eyeballs> {
public:
HappyEyeballsConnectionProvider(
Event::Dispatcher& dispatcher,
const std::vector<Address::InstanceConstSharedPtr>& address_list,
const Upstream::HostDescription::SharedConstAddressVector& sorted_address_list,
const std::shared_ptr<const Upstream::UpstreamLocalAddressSelector>&
upstream_local_address_selector,
UpstreamTransportSocketFactory& socket_factory,
TransportSocketOptionsConstSharedPtr transport_socket_options,
const Upstream::HostDescriptionConstSharedPtr& host,
const ConnectionSocket::OptionsSharedPtr options,
const envoy::config::cluster::v3::UpstreamConnectionOptions::HappyEyeballsConfig&
happy_eyeballs_config);
const ConnectionSocket::OptionsSharedPtr options);
bool hasNextConnection() override;
ClientConnectionPtr createNextConnection(const uint64_t id) override;
size_t nextConnection() override;
Expand All @@ -44,8 +44,8 @@ class HappyEyeballsConnectionProvider : public ConnectionProvider,

private:
Event::Dispatcher& dispatcher_;
// List of addresses to attempt to connect to.
const std::vector<Address::InstanceConstSharedPtr> address_list_;
// List of addresses to attempt to connect to, pre-sorted with sortAddresses().
const Upstream::HostDescription::SharedConstAddressVector address_list_;
const Upstream::UpstreamLocalAddressSelectorConstSharedPtr upstream_local_address_selector_;
UpstreamTransportSocketFactory& socket_factory_;
TransportSocketOptionsConstSharedPtr transport_socket_options_;
Expand All @@ -69,28 +69,26 @@ class HappyEyeballsConnectionProvider : public ConnectionProvider,
* they are applied to each open connection and applied when creating new ones.
*
* See the Happy Eyeballs RFC at https://datatracker.ietf.org/doc/html/rfc6555
* TODO(RyanTheOptimist): Implement the Happy Eyeballs address sorting algorithm
* either in the class or in the resolution code.
* The address list must already be sorted with
* HappyEyeballsConnectionProvider::sortAddresses(), which the host does once
* when the address list is created or refreshed.
*/
class HappyEyeballsConnectionImpl : public MultiConnectionBaseImpl,
Logger::Loggable<Logger::Id::happy_eyeballs> {
public:
HappyEyeballsConnectionImpl(
Event::Dispatcher& dispatcher,
const std::vector<Address::InstanceConstSharedPtr>& address_list,
const Upstream::HostDescription::SharedConstAddressVector& sorted_address_list,
const std::shared_ptr<const Upstream::UpstreamLocalAddressSelector>&
upstream_local_address_selector,
UpstreamTransportSocketFactory& socket_factory,
TransportSocketOptionsConstSharedPtr transport_socket_options,
const Upstream::HostDescriptionConstSharedPtr& host,
const ConnectionSocket::OptionsSharedPtr options,
const envoy::config::cluster::v3::UpstreamConnectionOptions::HappyEyeballsConfig&
happy_eyeballs_config)
: MultiConnectionBaseImpl(dispatcher,
std::make_unique<Network::HappyEyeballsConnectionProvider>(
dispatcher, address_list, upstream_local_address_selector,
socket_factory, transport_socket_options, host, options,
happy_eyeballs_config)) {}
const ConnectionSocket::OptionsSharedPtr options)
: MultiConnectionBaseImpl(
dispatcher, std::make_unique<Network::HappyEyeballsConnectionProvider>(
dispatcher, sorted_address_list, upstream_local_address_selector,
socket_factory, transport_socket_options, host, options)) {}
};

} // namespace Network
Expand Down
39 changes: 25 additions & 14 deletions source/common/upstream/upstream_impl.cc
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,7 @@ HostDescriptionImpl::HostDescriptionImpl(
uint32_t priority, const AddressVector& address_list, absl::string_view stat_name)
: HostDescriptionImplBase(cluster, hostname, dest_address, endpoint_metadata, locality_metadata,
locality, health_check_config, priority, creation_status),
sorted_address_list_or_null_(makeSortedAddressListOrNull(*cluster, address_list)),
address_(dest_address),
address_list_or_null_(makeAddressListOrNull(dest_address, address_list)),
health_check_address_(resolveHealthCheckAddress(health_check_config, dest_address)),
Expand Down Expand Up @@ -565,6 +566,20 @@ HostDescription::SharedConstAddressVector HostDescriptionImplBase::makeAddressLi
return std::make_shared<AddressVector>(address_list);
}

HostDescription::SharedConstAddressVector
HostDescriptionImplBase::makeSortedAddressListOrNull(const ClusterInfo& cluster,
const AddressVector& address_list) {
if (address_list.size() <= 1) {
return {};
}
const envoy::config::cluster::v3::UpstreamConnectionOptions::HappyEyeballsConfig&
happy_eyeballs_config =
cluster.happyEyeballsConfig().has_value() ? *cluster.happyEyeballsConfig()
: defaultHappyEyeballsConfig();
return std::make_shared<AddressVector>(
Network::HappyEyeballsConnectionProvider::sortAddresses(address_list, happy_eyeballs_config));
}

Network::UpstreamTransportSocketFactory& HostDescriptionImplBase::resolveTransportSocketFactory(
const Network::Address::InstanceConstSharedPtr& dest_address,
const envoy::config::core::v3::Metadata* endpoint_metadata,
Expand All @@ -590,8 +605,8 @@ Host::CreateConnectionData HostImplBase::createConnection(
? resolveTransportSocketFactory(address(), metadata().get(), transport_socket_options)
: transportSocketFactory();

return createConnection(dispatcher, cluster(), address(), addressListOrNull(), factory, options,
transport_socket_options, shared_from_this());
return createConnection(dispatcher, cluster(), address(), sortedAddressListOrNull(), factory,
options, transport_socket_options, shared_from_this());
}

void HostImplBase::setEdsHealthFlag(envoy::config::core::v3::HealthStatus health_status) {
Expand Down Expand Up @@ -637,7 +652,7 @@ Host::CreateConnectionData HostImplBase::createOrcaReportingConnection(
Network::UpstreamTransportSocketFactory& factory,
Network::Address::InstanceConstSharedPtr orca_address) const {
return createOrcaConnection(dispatcher, std::move(transport_socket_options), factory,
std::move(orca_address), address(), addressListOrNull(),
std::move(orca_address), address(), sortedAddressListOrNull(),
shared_from_this());
}

Expand All @@ -647,13 +662,13 @@ Host::CreateConnectionData HostImplBase::createOrcaConnection(
Network::UpstreamTransportSocketFactory& factory,
Network::Address::InstanceConstSharedPtr orca_address,
const Network::Address::InstanceConstSharedPtr& host_address,
const SharedConstAddressVector& address_list, HostDescriptionConstSharedPtr host) const {
const SharedConstAddressVector& sorted_address_list, HostDescriptionConstSharedPtr host) const {
// The original-port address list applies only when dialing the host's own address. Compare
// by value: pointer identity doesn't survive LogicalHost re-resolution.
const bool use_address_list = *orca_address == *host_address;
return createConnection(dispatcher, cluster(), orca_address,
use_address_list ? address_list : SharedConstAddressVector{}, factory,
/*options=*/nullptr, transport_socket_options, std::move(host));
use_address_list ? sorted_address_list : SharedConstAddressVector{},
factory, /*options=*/nullptr, transport_socket_options, std::move(host));
}

std::optional<Network::Address::InstanceConstSharedPtr> HostImplBase::maybeGetProxyRedirectAddress(
Expand Down Expand Up @@ -713,7 +728,7 @@ std::optional<Network::Address::InstanceConstSharedPtr> HostImplBase::maybeGetPr
Host::CreateConnectionData HostImplBase::createConnection(
Event::Dispatcher& dispatcher, const ClusterInfo& cluster,
const Network::Address::InstanceConstSharedPtr& address,
const SharedConstAddressVector& address_list_or_null,
const SharedConstAddressVector& sorted_address_list,
Network::UpstreamTransportSocketFactory& socket_factory,
const Network::ConnectionSocket::OptionsSharedPtr& options,
Network::TransportSocketOptionsConstSharedPtr transport_socket_options,
Expand All @@ -736,15 +751,11 @@ Host::CreateConnectionData HostImplBase::createConnection(
proxy_address.value(), upstream_local_address.address_,
socket_factory.createTransportSocket(transport_socket_options, host),
upstream_local_address.socket_options_, transport_socket_options);
} else if (address_list_or_null != nullptr && address_list_or_null->size() > 1) {
} else if (sorted_address_list != nullptr && sorted_address_list->size() > 1) {
ENVOY_LOG(debug, "Upstream using happy eyeballs config.");
const envoy::config::cluster::v3::UpstreamConnectionOptions::HappyEyeballsConfig&
happy_eyeballs_config =
cluster.happyEyeballsConfig().has_value() ? *cluster.happyEyeballsConfig()
: defaultHappyEyeballsConfig();
connection = std::make_unique<Network::HappyEyeballsConnectionImpl>(
dispatcher, *address_list_or_null, source_address_selector, socket_factory,
transport_socket_options, host, options, happy_eyeballs_config);
dispatcher, sorted_address_list, source_address_selector, socket_factory,
transport_socket_options, host, options);
} else {
auto upstream_local_address = source_address_selector->getUpstreamLocalAddress(
address, options, makeOptRefFromPtr(transport_socket_options.get()));
Expand Down
32 changes: 30 additions & 2 deletions source/common/upstream/upstream_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,17 @@ class HostDescriptionImplBase : virtual public HostDescription,
makeAddressListOrNull(const Network::Address::InstanceConstSharedPtr& address,
const AddressVector& address_list);

/**
* @return nullptr if address_list has fewer than 2 addresses (happy eyeballs does not
* apply), otherwise a shared_ptr to a copy of address_list sorted with
* Network::HappyEyeballsConnectionProvider::sortAddresses() using the cluster's happy
* eyeballs config, or the default config if the cluster does not specify one. This is
* computed once when the address list is created or refreshed so that connection
* attempts do not re-sort it.
*/
static SharedConstAddressVector makeSortedAddressListOrNull(const ClusterInfo& cluster,
const AddressVector& address_list);

private:
ClusterInfoConstSharedPtr cluster_;
const std::string hostname_;
Expand Down Expand Up @@ -347,6 +358,11 @@ class HostDescriptionImpl : public HostDescriptionImplBase {
const envoy::config::endpoint::v3::Endpoint::HealthCheckConfig& health_check_config,
uint32_t priority, const AddressVector& address_list = {}, absl::string_view stat_name = {});

// Happy eyeballs sorted copy of the address list, or nullptr if the host does not have
// multiple addresses. Set at construction and never changed; read by
// HostImpl::sortedAddressListOrNull().
const SharedConstAddressVector sorted_address_list_or_null_;

private:
// No locks are required in this implementation: all address-related member
// variables are set at construction and never change. See
Expand Down Expand Up @@ -478,10 +494,17 @@ class HostImplBase : public Host,
}

protected:
/**
* @return the address list sorted for happy eyeballs connection attempts, or nullptr if
* the host does not have multiple addresses. The list is computed once when the address
* list is created or refreshed rather than on every connection attempt.
*/
virtual SharedConstAddressVector sortedAddressListOrNull() const PURE;

static CreateConnectionData
createConnection(Event::Dispatcher& dispatcher, const ClusterInfo& cluster,
const Network::Address::InstanceConstSharedPtr& address,
const SharedConstAddressVector& address_list,
const SharedConstAddressVector& sorted_address_list,
Network::UpstreamTransportSocketFactory& socket_factory,
const Network::ConnectionSocket::OptionsSharedPtr& options,
Network::TransportSocketOptionsConstSharedPtr transport_socket_options,
Expand All @@ -497,7 +520,7 @@ class HostImplBase : public Host,
Network::UpstreamTransportSocketFactory& factory,
Network::Address::InstanceConstSharedPtr orca_address,
const Network::Address::InstanceConstSharedPtr& host_address,
const SharedConstAddressVector& address_list,
const SharedConstAddressVector& sorted_address_list,
HostDescriptionConstSharedPtr host) const;

private:
Expand Down Expand Up @@ -556,6 +579,11 @@ class HostImpl : public HostImplBase, public HostDescriptionImpl {
HostDescriptionImpl(creation_status, cluster, hostname, address, endpoint_metadata,
locality_metadata, locality, health_check_config, priority,
address_list, stat_name) {}

// Upstream::HostImplBase
SharedConstAddressVector sortedAddressListOrNull() const override {
return sorted_address_list_or_null_;
}
};

class HostsPerLocalityImpl : public HostsPerLocality {
Expand Down
Loading
Loading