dnsdist: Move the backends to the new configuration

pdns/dnsdistdist/Makefile.am

@@ -145,7 +145,7 @@ dnsdist_SOURCES = \
 	dns_random.hh \
 	dnscrypt.cc dnscrypt.hh \
 	dnsdist-async.cc dnsdist-async.hh \
-	dnsdist-backend.cc \
+	dnsdist-backend.cc dnsdist-backend.hh \
 	dnsdist-backoff.hh \
 	dnsdist-cache.cc dnsdist-cache.hh \
 	dnsdist-carbon.cc dnsdist-carbon.hh \
@@ -272,7 +272,7 @@ testrunner_SOURCES = \
 	dns.cc dns.hh \
 	dnscrypt.cc dnscrypt.hh \
 	dnsdist-async.cc dnsdist-async.hh \
-	dnsdist-backend.cc \
+	dnsdist-backend.cc dnsdist-backend.hh \
 	dnsdist-backoff.hh \
 	dnsdist-cache.cc dnsdist-cache.hh \
 	dnsdist-concurrent-connections.hh \

pdns/dnsdistdist/dnsdist-backend.cc

@@ -21,6 +21,7 @@
  */
 #include "config.h"
 #include "dnsdist.hh"
+#include "dnsdist-backend.hh"
 #include "dnsdist-backoff.hh"
 #include "dnsdist-metrics.hh"
 #include "dnsdist-nghttp2.hh"
@@ -1006,3 +1007,17 @@ void ServerPool::removeServer(shared_ptr<DownstreamState>& server)
   }
   *servers = std::move(newServers);
 }
+
+namespace dnsdist::backend
+{
+void registerNewBackend(std::shared_ptr<DownstreamState>& backend)
+{
+  dnsdist::configuration::updateRuntimeConfiguration([&backend](dnsdist::configuration::RuntimeConfiguration& config) {
+    auto& backends = config.d_backends;
+    backends.push_back(backend);
+    std::stable_sort(backends.begin(), backends.end(), [](const std::shared_ptr<DownstreamState>& lhs, const std::shared_ptr<DownstreamState>& rhs) {
+      return lhs->d_config.order < rhs->d_config.order;
+    });
+  });
+}
+}

pdns/dnsdistdist/dnsdist-backend.hh

@@ -0,0 +1,31 @@
+/*
+ * This file is part of PowerDNS or dnsdist.
+ * Copyright -- PowerDNS.COM B.V. and its contributors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * In addition, for the avoidance of any doubt, permission is granted to
+ * link this program with OpenSSL and to (re)distribute the binaries
+ * produced as the result of such linking.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+#pragma once
+
+#include <memory>
+
+struct DownstreamState;
+
+namespace dnsdist::backend
+{
+void registerNewBackend(std::shared_ptr<DownstreamState>& backend);
+}

pdns/dnsdistdist/dnsdist-carbon.cc

@@ -74,8 +74,7 @@ static bool doOneCarbonExport(const Carbon::Endpoint& endpoint)
       }
     }
 
-    auto states = g_dstates.getLocal();
-    for (const auto& state : *states) {
+    for (const auto& state : dnsdist::configuration::getCurrentRuntimeConfiguration().d_backends) {
       string serverName = state->getName().empty() ? state->d_config.remote.toStringWithPort() : state->getName();
       boost::replace_all(serverName, ".", "_");
       string base = namespace_name;

pdns/dnsdistdist/dnsdist-configuration.hh

@@ -136,6 +136,9 @@ public:
 
 class ServerPolicy;
 struct ServerPool;
+struct DownstreamState;
+
+using servers_t = std::vector<std::shared_ptr<DownstreamState>>;
 
 namespace dnsdist::configuration
 {
@@ -181,6 +184,7 @@ struct Configuration
    a RCU-like mechanism */
 struct RuntimeConfiguration
 {
+  servers_t d_backends;
   std::map<std::string, std::shared_ptr<ServerPool>> d_pools;
   std::shared_ptr<ServerPolicy> d_lbPolicy;
   NetmaskGroup d_ACL;

pdns/dnsdistdist/dnsdist-discovery.cc

@@ -22,6 +22,7 @@
 
 #include "config.h"
 #include "dnsdist-discovery.hh"
+#include "dnsdist-backend.hh"
 #include "dnsdist.hh"
 #include "dnsdist-random.hh"
 #include "dnsparser.hh"
@@ -459,16 +460,17 @@ bool ServiceDiscovery::tryToUpgradeBackend(const UpgradeableBackend& backend)
 
     newServer->start();
 
-    auto states = g_dstates.getCopy();
-    states.push_back(newServer);
     /* remove the existing backend if needed */
     if (!backend.keepAfterUpgrade) {
-      for (auto it = states.begin(); it != states.end(); ++it) {
-        if (*it == backend.d_ds) {
-          states.erase(it);
-          break;
+      dnsdist::configuration::updateRuntimeConfiguration([&backend](dnsdist::configuration::RuntimeConfiguration& config) {
+        auto& backends = config.d_backends;
+        for (auto backendIt = backends.begin(); backendIt != backends.end(); ++backendIt) {
+          if (*backendIt == backend.d_ds) {
+            backends.erase(backendIt);
+            break;
+          }
         }
-      }
+      });
 
       for (const string& poolName : backend.d_ds->d_config.pools) {
         removeServerFromPool(poolName, backend.d_ds);
@@ -477,11 +479,8 @@ bool ServiceDiscovery::tryToUpgradeBackend(const UpgradeableBackend& backend)
       removeServerFromPool("", backend.d_ds);
     }
 
-    std::stable_sort(states.begin(), states.end(), [](const decltype(newServer)& a, const decltype(newServer)& b) {
-      return a->d_config.order < b->d_config.order;
-    });
+    dnsdist::backend::registerNewBackend(newServer);
 
-    g_dstates.setState(states);
     if (!backend.keepAfterUpgrade) {
       backend.d_ds->stop();
     }

pdns/dnsdistdist/dnsdist-lua-inspection.cc

@@ -742,9 +742,8 @@ void setupLuaInspection(LuaContext& luaCtx)
     fmt = boost::format("%-3d %-20.20s %-20.20s %-20d %-20d %-25d %-25d %-20d %-20d %-20d %-20d %-20d %-20d %-20d %-20d %-20f %-20f");
     ret << (fmt % "#" % "Name" % "Address" % "Connections" % "Max concurrent conn" % "Died sending query" % "Died reading response" % "Gave up" % "Read timeouts" % "Write timeouts" % "Connect timeouts" % "Too many conn" % "Total connections" % "Reused connections" % "TLS resumptions" % "Avg queries/conn" % "Avg duration") << endl;
 
-    auto states = g_dstates.getLocal();
     counter = 0;
-    for (const auto& backend : *states) {
+    for (const auto& backend : dnsdist::configuration::getCurrentRuntimeConfiguration().d_backends) {
       ret << (fmt % counter % backend->getName() % backend->d_config.remote.toStringWithPort() % backend->tcpCurrentConnections % backend->tcpMaxConcurrentConnections % backend->tcpDiedSendingQuery % backend->tcpDiedReadingResponse % backend->tcpGaveUp % backend->tcpReadTimeouts % backend->tcpWriteTimeouts % backend->tcpConnectTimeouts % backend->tcpTooManyConcurrentConnections % backend->tcpNewConnections % backend->tcpReusedConnections % backend->tlsResumptions % backend->tcpAvgQueriesPerConnection % backend->tcpAvgConnectionDuration) << endl;
       ++counter;
     }

pdns/dnsdistdist/dnsdist-lua.cc

@@ -37,6 +37,7 @@
 #include <vector>
 
 #include "dnsdist.hh"
+#include "dnsdist-backend.hh"
 #include "dnsdist-carbon.hh"
 #include "dnsdist-concurrent-connections.hh"
 #include "dnsdist-configuration.hh"
@@ -709,12 +710,8 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
                            }
                          }
 
-                         auto states = g_dstates.getCopy();
-                         states.push_back(ret);
-                         std::stable_sort(states.begin(), states.end(), [](const decltype(ret)& lhs, const decltype(ret)& rhs) {
-                           return lhs->d_config.order < rhs->d_config.order;
-                         });
-                         g_dstates.setState(states);
+                         dnsdist::backend::registerNewBackend(ret);
+
                          checkAllParametersConsumed("newServer", vars);
                          return ret;
                        });
@@ -723,21 +720,20 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
                        [](boost::variant<std::shared_ptr<DownstreamState>, int, std::string> var) {
                          setLuaSideEffect();
                          shared_ptr<DownstreamState> server = nullptr;
-                         auto states = g_dstates.getCopy();
                          if (auto* rem = boost::get<shared_ptr<DownstreamState>>(&var)) {
                            server = *rem;
                          }
                          else if (auto* str = boost::get<std::string>(&var)) {
                            const auto uuid = getUniqueID(*str);
-                           for (auto& state : states) {
+                           for (auto& state : dnsdist::configuration::getCurrentRuntimeConfiguration().d_backends) {
                              if (*state->d_config.id == uuid) {
                                server = state;
                              }
                            }
                          }
                          else {
                            int idx = boost::get<int>(var);
-                           server = states.at(idx);
+                           server = dnsdist::configuration::getCurrentRuntimeConfiguration().d_backends.at(idx);
                          }
                          if (!server) {
                            throw std::runtime_error("unable to locate the requested server");
@@ -747,8 +743,11 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
                          }
                          /* the server might also be in the default pool */
                          removeServerFromPool("", server);
-                         states.erase(remove(states.begin(), states.end(), server), states.end());
-                         g_dstates.setState(states);
+
+                         dnsdist::configuration::updateRuntimeConfiguration([&server](dnsdist::configuration::RuntimeConfiguration& config) {
+                           config.d_backends.erase(std::remove(config.d_backends.begin(), config.d_backends.end(), server), config.d_backends.end());
+                         });
+
                          server->stop();
                        });
 
@@ -1179,8 +1178,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
       uint64_t totQueries{0};
       uint64_t totDrops{0};
       int counter = 0;
-      auto states = g_dstates.getLocal();
-      for (const auto& backend : *states) {
+      for (const auto& backend : dnsdist::configuration::getCurrentRuntimeConfiguration().d_backends) {
         string status = backend->getStatus();
         string pools;
         for (const auto& pool : backend->d_config.pools) {
@@ -1225,7 +1223,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
     setLuaNoSideEffect();
     LuaArray<std::shared_ptr<DownstreamState>> ret;
     int count = 1;
-    for (const auto& backend : g_dstates.getCopy()) {
+    for (const auto& backend : dnsdist::configuration::getCurrentRuntimeConfiguration().d_backends) {
       ret.emplace_back(count++, backend);
     }
     return ret;
@@ -1240,7 +1238,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
     if (client) {
       return std::make_shared<DownstreamState>(ComboAddress());
     }
-    auto states = g_dstates.getCopy();
+    const auto& states = dnsdist::configuration::getCurrentRuntimeConfiguration().d_backends;
     if (auto* str = boost::get<std::string>(&identifier)) {
       const auto uuid = getUniqueID(*str);
       for (auto& state : states) {

pdns/dnsdistdist/dnsdist-snmp.cc

@@ -278,11 +278,11 @@ static netsnmp_variable_list* backendStatTable_get_first_data_point(void** loop_
 
   /* get a copy of the shared_ptrs so they are not
      destroyed while we process the request */
-  auto dstates = g_dstates.getLocal();
+  auto backends = dnsdist::configuration::getCurrentRuntimeConfiguration().d_backends;
   s_servers.clear();
-  s_servers.reserve(dstates->size());
-  for (const auto& server : *dstates) {
-    s_servers.push_back(server);
+  s_servers.reserve(backends.size());
+  for (const auto& server : backends) {
+    s_servers.push_back(std::move(server));
   }
 
   return backendStatTable_get_next_data_point(loop_context,

pdns/dnsdistdist/dnsdist-web.cc

@@ -561,7 +561,6 @@ static void handlePrometheus(const YaHTTP::Request& req, YaHTTP::Response& resp)
   output << "dnsdist_latency_sum " << dnsdist::metrics::g_stats.latencySum << "\n";
   output << "dnsdist_latency_count " << dnsdist::metrics::g_stats.latencyCount << "\n";
 
-  auto states = g_dstates.getLocal();
   const string statesbase = "dnsdist_server_";
 
   // clang-format off
@@ -628,7 +627,7 @@ static void handlePrometheus(const YaHTTP::Request& req, YaHTTP::Response& resp)
   output << "# HELP " << statesbase << "healthcheckfailuresinvalid "      << "Number of health check attempts where the DNS response was invalid"                   << "\n";
   output << "# TYPE " << statesbase << "healthcheckfailuresinvalid "      << "counter"                                                                              << "\n";
 
-  for (const auto& state : *states) {
+  for (const auto& state : dnsdist::configuration::getCurrentRuntimeConfiguration().d_backends) {
     string serverName;
 
     if (state->getName().empty()) {
@@ -1153,9 +1152,9 @@ static void handleStats(const YaHTTP::Request& req, YaHTTP::Response& resp)
 
   Json::array servers;
   {
-    auto localServers = g_dstates.getLocal();
-    servers.reserve(localServers->size());
-    for (const auto& server : *localServers) {
+    const auto& localServers = dnsdist::configuration::getCurrentRuntimeConfiguration().d_backends;
+    servers.reserve(localServers.size());
+    for (const auto& server : localServers) {
       addServerToJSON(servers, num++, server);
     }
   }

pdns/dnsdistdist/dnsdist.cc

@@ -123,8 +123,6 @@ std::vector<uint32_t> g_TCPFastOpenKey;
 
 Rings g_rings;
 
-GlobalStateHolder<servers_t> g_dstates;
-
 // we are not willing to receive a bigger UDP response than that, no matter what
 static constexpr size_t s_maxUDPResponsePacketSize{4096U};
 static size_t const s_initialUDPPacketBufferSize = s_maxUDPResponsePacketSize + DNSCRYPT_MAX_RESPONSE_PADDING_AND_MAC_SIZE;
@@ -2360,7 +2358,6 @@ static void healthChecksThread()
     .tv_sec = 0,
     .tv_usec = 0
   };
-  auto states = g_dstates.getLocal(); // this points to the actual shared_ptrs!
 
   for (;;) {
     timeval now{};
@@ -2375,7 +2372,9 @@ static void healthChecksThread()
     }
 
     std::unique_ptr<FDMultiplexer> mplexer{nullptr};
-    for (const auto& dss : *states) {
+    // this points to the actual shared_ptrs!
+    const auto servers = dnsdist::configuration::getCurrentRuntimeConfiguration().d_backends;
+    for (const auto& dss : servers) {
       dss->updateStatisticsInfo();
 
       dss->handleUDPTimeouts();
@@ -2385,7 +2384,7 @@ static void healthChecksThread()
       }
 
       if (!mplexer) {
-        mplexer = std::unique_ptr<FDMultiplexer>(FDMultiplexer::getMultiplexerSilent(states->size()));
+        mplexer = std::unique_ptr<FDMultiplexer>(FDMultiplexer::getMultiplexerSilent(servers.size()));
       }
 
       if (!queueHealthCheck(mplexer, dss)) {
@@ -2458,16 +2457,16 @@ static void checkFileDescriptorsLimits(size_t udpBindsCount, size_t tcpBindsCoun
   const auto immutableConfig = dnsdist::configuration::getImmutableConfiguration();
   /* stdin, stdout, stderr */
   rlim_t requiredFDsCount = 3;
-  auto backends = g_dstates.getLocal();
+  const auto backends = dnsdist::configuration::getCurrentRuntimeConfiguration().d_backends;
   /* UDP sockets to backends */
   size_t backendUDPSocketsCount = 0;
-  for (const auto& backend : *backends) {
+  for (const auto& backend : backends) {
     backendUDPSocketsCount += backend->sockets.size();
   }
   requiredFDsCount += backendUDPSocketsCount;
   /* TCP sockets to backends */
   if (immutableConfig.d_maxTCPClientThreads > 0) {
-    requiredFDsCount += (backends->size() * immutableConfig.d_maxTCPClientThreads);
+    requiredFDsCount += (backends.size() * immutableConfig.d_maxTCPClientThreads);
   }
   /* listening sockets */
   requiredFDsCount += udpBindsCount;
@@ -2789,10 +2788,10 @@ static void cleanupLuaObjects()
   for (const auto& chain : dnsdist::rules::getResponseRuleChains()) {
     chain.holder.setState({});
   }
-  g_dstates.setState({});
   dnsdist::configuration::updateRuntimeConfiguration([](dnsdist::configuration::RuntimeConfiguration& config) {
     config.d_lbPolicy = std::make_shared<ServerPolicy>();
     config.d_pools.clear();
+    config.d_backends.clear();
   });
   clearWebHandlers();
   dnsdist::lua::hooks::clearMaintenanceHooks();
@@ -3063,8 +3062,7 @@ static void setupPools()
   if (precompute) {
     vinfolog("Pre-computing hashes for consistent hash load-balancing policy");
     // pre compute hashes
-    auto backends = g_dstates.getLocal();
-    for (const auto& backend : *backends) {
+    for (const auto& backend : dnsdist::configuration::getCurrentRuntimeConfiguration().d_backends) {
       if (backend->d_config.d_weight < 100) {
         vinfolog("Warning, the backend '%s' has a very low weight (%d), which will not yield a good distribution of queries with the 'chashed' policy. Please consider raising it to at least '100'.", backend->getName(), backend->d_config.d_weight);
       }
@@ -3436,19 +3434,21 @@ int main(int argc, char** argv)
         auto ret = std::make_shared<DownstreamState>(std::move(config), nullptr, true);
         addServerToPool("", ret);
         ret->start();
-        g_dstates.modify([&ret](servers_t& servers) { servers.push_back(std::move(ret)); });
+        dnsdist::configuration::updateRuntimeConfiguration([&ret](dnsdist::configuration::RuntimeConfiguration& config) {
+          config.d_backends.push_back(std::move(ret));
+        });
       }
     }
 
-    if (g_dstates.getLocal()->empty()) {
+    if (dnsdist::configuration::getCurrentRuntimeConfiguration().d_backends.empty()) {
       errlog("No downstream servers defined: all packets will get dropped");
       // you might define them later, but you need to know
     }
 
     checkFileDescriptorsLimits(udpBindsCount, tcpBindsCount);
 
     {
-      auto states = g_dstates.getCopy(); // it is a copy, but the internal shared_ptrs are the real deal
+      const auto states = dnsdist::configuration::getCurrentRuntimeConfiguration().d_backends; // it is a copy, but the internal shared_ptrs are the real deal
       auto mplexer = std::unique_ptr<FDMultiplexer>(FDMultiplexer::getMultiplexerSilent(states.size()));
       for (auto& dss : states) {