diff --git a/pdns/dnsdistdist/Makefile.am b/pdns/dnsdistdist/Makefile.am index 0e85df5c990a..19e57742ee29 100644 --- a/pdns/dnsdistdist/Makefile.am +++ b/pdns/dnsdistdist/Makefile.am @@ -162,6 +162,7 @@ dnsdist_SOURCES = \ dnsdist-dynbpf.cc dnsdist-dynbpf.hh \ dnsdist-ecs.cc dnsdist-ecs.hh \ dnsdist-edns.cc dnsdist-edns.hh \ + dnsdist-frontend.cc dnsdist-frontend.hh \ dnsdist-healthchecks.cc dnsdist-healthchecks.hh \ dnsdist-idstate.cc dnsdist-idstate.hh \ dnsdist-internal-queries.cc dnsdist-internal-queries.hh \ @@ -285,6 +286,7 @@ testrunner_SOURCES = \ dnsdist-dynbpf.cc dnsdist-dynbpf.hh \ dnsdist-ecs.cc dnsdist-ecs.hh \ dnsdist-edns.cc dnsdist-edns.hh \ + dnsdist-frontend.cc dnsdist-frontend.hh \ dnsdist-idstate.cc dnsdist-idstate.hh \ dnsdist-kvs.cc dnsdist-kvs.hh \ dnsdist-lbpolicies.cc dnsdist-lbpolicies.hh \ diff --git a/pdns/dnsdistdist/dnsdist-carbon.cc b/pdns/dnsdistdist/dnsdist-carbon.cc index 76b35e8f5131..6fa97a5341ef 100644 --- a/pdns/dnsdistdist/dnsdist-carbon.cc +++ b/pdns/dnsdistdist/dnsdist-carbon.cc @@ -27,6 +27,7 @@ #include "dnsdist.hh" #include "dnsdist-backoff.hh" #include "dnsdist-configuration.hh" +#include "dnsdist-frontend.hh" #include "dnsdist-metrics.hh" #ifndef DISABLE_CARBON @@ -113,7 +114,7 @@ static bool doOneCarbonExport(const Carbon::Endpoint& endpoint) } std::map frontendDuplicates; - for (const auto& front : g_frontends) { + for (const auto& front : dnsdist::getFrontends()) { if (front->udpFD == -1 && front->tcpFD == -1) { continue; } @@ -222,7 +223,7 @@ static bool doOneCarbonExport(const Carbon::Endpoint& endpoint) { std::map dohFrontendDuplicates; const string base = "dnsdist." + hostname + ".main.doh."; - for (const auto& doh : g_dohlocals) { + for (const auto& doh : dnsdist::getDoHFrontends()) { string name = doh->d_tlsContext.d_addr.toStringWithPort(); boost::replace_all(name, ".", "_"); boost::replace_all(name, ":", "_"); diff --git a/pdns/dnsdistdist/dnsdist-configuration.hh b/pdns/dnsdistdist/dnsdist-configuration.hh index a102fb6db5e6..bca05ef2a0e3 100644 --- a/pdns/dnsdistdist/dnsdist-configuration.hh +++ b/pdns/dnsdistdist/dnsdist-configuration.hh @@ -143,6 +143,7 @@ public: class ServerPolicy; struct ServerPool; struct DownstreamState; +struct ClientState; using servers_t = std::vector>; @@ -161,6 +162,7 @@ struct Configuration { std::set d_capabilitiesToRetain; std::vector d_tcpFastOpenKey; + std::vector> d_frontends; #ifdef __linux__ // On Linux this gives us 128k pending queries (default is 8192 queries), // which should be enough to deal with huge spikes diff --git a/pdns/dnsdistdist/dnsdist-frontend.cc b/pdns/dnsdistdist/dnsdist-frontend.cc new file mode 100644 index 000000000000..20bef4e0047c --- /dev/null +++ b/pdns/dnsdistdist/dnsdist-frontend.cc @@ -0,0 +1,88 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "dnsdist-frontend.hh" +#include "dnsdist.hh" + +namespace dnsdist +{ + +const std::vector>& getFrontends() +{ + return dnsdist::configuration::getImmutableConfiguration().d_frontends; +} + +std::vector> getDNSCryptFrontends() +{ + std::vector> results; + for (const auto& frontend : getFrontends()) { + if (frontend->getProtocol() == dnsdist::Protocol::DNSCryptUDP || frontend->getProtocol() == dnsdist::Protocol::DNSCryptTCP) { + results.push_back(frontend->dnscryptCtx); + } + } + return results; +} + +std::vector> getDoTFrontends() +{ + std::vector> results; + for (const auto& frontend : getFrontends()) { + if (frontend->getProtocol() == dnsdist::Protocol::DoT) { + results.push_back(frontend->tlsFrontend); + } + } + return results; +} + +std::vector> getDoHFrontends() +{ + std::vector> results; + for (const auto& frontend : getFrontends()) { + if (frontend->getProtocol() == dnsdist::Protocol::DoH) { + results.push_back(frontend->dohFrontend); + } + } + return results; +} + +std::vector> getDoQFrontends() +{ + std::vector> results; + for (const auto& frontend : getFrontends()) { + if (frontend->getProtocol() == dnsdist::Protocol::DoQ) { + results.push_back(frontend->doqFrontend); + } + } + return results; +} + +std::vector> getDoH3Frontends() +{ + std::vector> results; + for (const auto& frontend : getFrontends()) { + if (frontend->getProtocol() == dnsdist::Protocol::DoH3) { + results.push_back(frontend->doh3Frontend); + } + } + return results; +} +} diff --git a/pdns/dnsdistdist/dnsdist-frontend.hh b/pdns/dnsdistdist/dnsdist-frontend.hh new file mode 100644 index 000000000000..fe7f1c2a92f8 --- /dev/null +++ b/pdns/dnsdistdist/dnsdist-frontend.hh @@ -0,0 +1,42 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include +#include + +struct ClientState; +class DNSCryptContext; +class TLSFrontend; +struct DOHFrontend; +struct DOQFrontend; +struct DOH3Frontend; + +namespace dnsdist +{ +const std::vector>& getFrontends(); +std::vector> getDNSCryptFrontends(); +std::vector> getDoTFrontends(); +std::vector> getDoHFrontends(); +std::vector> getDoQFrontends(); +std::vector> getDoH3Frontends(); +} diff --git a/pdns/dnsdistdist/dnsdist-lua-bindings.cc b/pdns/dnsdistdist/dnsdist-lua-bindings.cc index a4b74ebc1c5f..52ee81d6a8cb 100644 --- a/pdns/dnsdistdist/dnsdist-lua-bindings.cc +++ b/pdns/dnsdistdist/dnsdist-lua-bindings.cc @@ -23,6 +23,7 @@ #include "config.h" #include "dnsdist.hh" #include "dnsdist-async.hh" +#include "dnsdist-frontend.hh" #include "dnsdist-lua.hh" #include "dnsdist-resolver.hh" #include "dnsdist-svc.hh" @@ -670,7 +671,7 @@ void setupLuaBindings(LuaContext& luaCtx, bool client, bool configCheck) return; } if (bpf) { - for (const auto& frontend : g_frontends) { + for (const auto& frontend : dnsdist::getFrontends()) { frontend->attachFilter(bpf, frontend->getSocket()); } } diff --git a/pdns/dnsdistdist/dnsdist-lua-inspection.cc b/pdns/dnsdistdist/dnsdist-lua-inspection.cc index ff3ba6090927..43d7e9b0828b 100644 --- a/pdns/dnsdistdist/dnsdist-lua-inspection.cc +++ b/pdns/dnsdistdist/dnsdist-lua-inspection.cc @@ -24,6 +24,7 @@ #include "dnsdist.hh" #include "dnsdist-console.hh" #include "dnsdist-dynblocks.hh" +#include "dnsdist-frontend.hh" #include "dnsdist-lua.hh" #include "dnsdist-nghttp2.hh" #include "dnsdist-rings.hh" @@ -732,7 +733,7 @@ void setupLuaInspection(LuaContext& luaCtx) ret << (fmt % "#" % "Address" % "Connections" % "Max concurrent conn" % "Died reading query" % "Died sending response" % "Gave up" % "Client timeouts" % "Downstream timeouts" % "Avg queries/conn" % "Avg duration" % "TLS new sessions" % "TLS Resumptions" % "TLS unknown ticket keys" % "TLS inactive ticket keys" % "TLS 1.0" % "TLS 1.1" % "TLS 1.2" % "TLS 1.3" % "TLS other") << endl; size_t counter = 0; - for (const auto& frontend : g_frontends) { + for (const auto& frontend : dnsdist::getFrontends()) { ret << (fmt % counter % frontend->local.toStringWithPort() % frontend->tcpCurrentConnections % frontend->tcpMaxConcurrentConnections % frontend->tcpDiedReadingQuery % frontend->tcpDiedSendingResponse % frontend->tcpGaveUp % frontend->tcpClientTimeouts % frontend->tcpDownstreamTimeouts % frontend->tcpAvgQueriesPerConnection % frontend->tcpAvgConnectionDuration % frontend->tlsNewSessions % frontend->tlsResumptions % frontend->tlsUnknownTicketKey % frontend->tlsInactiveTicketKey % frontend->tls10queries % frontend->tls11queries % frontend->tls12queries % frontend->tls13queries % frontend->tlsUnknownqueries) << endl; ++counter; } @@ -759,7 +760,7 @@ void setupLuaInspection(LuaContext& luaCtx) ret << (fmt % "#" % "Address" % "DH key too small" % "Inappropriate fallback" % "No shared cipher" % "Unknown cipher type" % "Unknown exchange type" % "Unknown protocol" % "Unsupported EC" % "Unsupported protocol") << endl; size_t counter = 0; - for (const auto& frontend : g_frontends) { + for (const auto& frontend : dnsdist::getFrontends()) { if (!frontend->hasTLS()) { continue; } diff --git a/pdns/dnsdistdist/dnsdist-lua.cc b/pdns/dnsdistdist/dnsdist-lua.cc index a660b7c1f85f..8449178a406c 100644 --- a/pdns/dnsdistdist/dnsdist-lua.cc +++ b/pdns/dnsdistdist/dnsdist-lua.cc @@ -46,6 +46,7 @@ #include "dnsdist-dynblocks.hh" #include "dnsdist-discovery.hh" #include "dnsdist-ecs.hh" +#include "dnsdist-frontend.hh" #include "dnsdist-healthchecks.hh" #include "dnsdist-lua.hh" #include "dnsdist-lua-hooks.hh" @@ -982,12 +983,13 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize, maxInFlightQueriesPerConn, tcpMaxConcurrentConnections, enableProxyProtocol); + auto frontends = dnsdist::configuration::getImmutableConfiguration().d_frontends; try { ComboAddress loc(addr, 53); - for (auto it = g_frontends.begin(); it != g_frontends.end();) { + for (auto it = frontends.begin(); it != frontends.end();) { /* DoH, DoT and DNSCrypt frontends are separate */ if ((*it)->tlsFrontend == nullptr && (*it)->dnscryptCtx == nullptr && (*it)->dohFrontend == nullptr) { - it = g_frontends.erase(it); + it = frontends.erase(it); } else { ++it; @@ -995,8 +997,8 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) } // only works pre-startup, so no sync necessary - auto udpCS = std::make_unique(loc, false, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); - auto tcpCS = std::make_unique(loc, true, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); + auto udpCS = std::make_shared(loc, false, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); + auto tcpCS = std::make_shared(loc, true, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); if (tcpListenQueueSize > 0) { tcpCS->tcpListenQueueSize = tcpListenQueueSize; } @@ -1018,10 +1020,13 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) vinfolog("Enabling XSK in %s mode for incoming UDP packets to %s", socket->getXDPMode(), loc.toStringWithPort()); } #endif /* HAVE_XSK */ - g_frontends.push_back(std::move(udpCS)); - g_frontends.push_back(std::move(tcpCS)); + frontends.push_back(std::move(udpCS)); + frontends.push_back(std::move(tcpCS)); checkAllParametersConsumed("setLocal", vars); + dnsdist::configuration::updateImmutableConfiguration([&frontends](dnsdist::configuration::Configuration& config) { + config.d_frontends = std::move(frontends); + }); } catch (const std::exception& e) { g_outputBuffer = "Error: " + string(e.what()) + "\n"; @@ -1051,8 +1056,8 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) try { ComboAddress loc(addr, 53); // only works pre-startup, so no sync necessary - auto udpCS = std::make_unique(loc, false, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); - auto tcpCS = std::make_unique(loc, true, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); + auto udpCS = std::make_shared(loc, false, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); + auto tcpCS = std::make_shared(loc, true, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); if (tcpListenQueueSize > 0) { tcpCS->tcpListenQueueSize = tcpListenQueueSize; } @@ -1073,8 +1078,10 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) vinfolog("Enabling XSK in %s mode for incoming UDP packets to %s", socket->getXDPMode(), loc.toStringWithPort()); } #endif /* HAVE_XSK */ - g_frontends.push_back(std::move(udpCS)); - g_frontends.push_back(std::move(tcpCS)); + dnsdist::configuration::updateImmutableConfiguration([&udpCS, &tcpCS](dnsdist::configuration::Configuration& config) { + config.d_frontends.push_back(std::move(udpCS)); + config.d_frontends.push_back(std::move(tcpCS)); + }); checkAllParametersConsumed("addLocal", vars); } @@ -1145,10 +1152,9 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) #if 0 // Useful for debugging leaks, but might lead to race under load // since other threads are still running. - for (auto& frontend : g_tlslocals) { + for (auto& frontend : getDoTFrontends()) { frontend->cleanup(); } - g_tlslocals.clear(); g_rings.clear(); #endif /* 0 */ pdns::coverage::dumpCoverageData(); @@ -1767,13 +1773,15 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) auto ctx = std::make_shared(providerName, certKeys); /* UDP */ - auto clientState = std::make_unique(ComboAddress(addr, 443), false, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); + auto clientState = std::make_shared(ComboAddress(addr, 443), false, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); clientState->dnscryptCtx = ctx; - g_dnsCryptLocals.push_back(ctx); - g_frontends.push_back(std::move(clientState)); + + dnsdist::configuration::updateImmutableConfiguration([&clientState](dnsdist::configuration::Configuration& config) { + config.d_frontends.push_back(std::move(clientState)); + }); /* TCP */ - clientState = std::make_unique(ComboAddress(addr, 443), true, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); + clientState = std::make_shared(ComboAddress(addr, 443), true, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); clientState->dnscryptCtx = std::move(ctx); if (tcpListenQueueSize > 0) { clientState->tcpListenQueueSize = tcpListenQueueSize; @@ -1785,7 +1793,9 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) clientState->d_tcpConcurrentConnectionsLimit = tcpMaxConcurrentConnections; } - g_frontends.push_back(std::move(clientState)); + dnsdist::configuration::updateImmutableConfiguration([&clientState](dnsdist::configuration::Configuration& config) { + config.d_frontends.push_back(std::move(clientState)); + }); } catch (const std::exception& e) { errlog("Error during addDNSCryptBind() processing: %s", e.what()); @@ -1801,7 +1811,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) size_t idx = 0; std::unordered_set> contexts; - for (const auto& frontend : g_frontends) { + for (const auto& frontend : dnsdist::getFrontends()) { const std::shared_ptr ctx = frontend->dnscryptCtx; if (!ctx || contexts.count(ctx) != 0) { continue; @@ -1817,15 +1827,16 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) luaCtx.writeFunction("getDNSCryptBind", [](uint64_t idx) { setLuaNoSideEffect(); std::shared_ptr ret = nullptr; - if (idx < g_dnsCryptLocals.size()) { - ret = g_dnsCryptLocals.at(idx); + auto frontends = dnsdist::getDNSCryptFrontends(); + if (idx < frontends.size()) { + ret = frontends.at(idx); } return ret; }); luaCtx.writeFunction("getDNSCryptBindCount", []() { setLuaNoSideEffect(); - return g_dnsCryptLocals.size(); + return dnsdist::getDNSCryptFrontends().size(); }); #endif /* HAVE_DNSCRYPT */ @@ -1934,7 +1945,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) ret << (fmt % "#" % "Address" % "Protocol" % "Queries") << endl; size_t counter = 0; - for (const auto& front : g_frontends) { + for (const auto& front : dnsdist::getFrontends()) { ret << (fmt % counter % front->local.toStringWithPort() % front->getType() % front->queries) << endl; counter++; } @@ -1949,15 +1960,16 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) luaCtx.writeFunction("getBind", [](uint64_t num) { setLuaNoSideEffect(); ClientState* ret = nullptr; - if (num < g_frontends.size()) { - ret = g_frontends[num].get(); + auto frontends = dnsdist::getFrontends(); + if (num < frontends.size()) { + ret = frontends[num].get(); } return ret; }); luaCtx.writeFunction("getBindCount", []() { setLuaNoSideEffect(); - return g_frontends.size(); + return dnsdist::getFrontends().size(); }); luaCtx.writeFunction("help", [](boost::optional command) { @@ -2552,8 +2564,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) } } - g_dohlocals.push_back(frontend); - auto clientState = std::make_unique(frontend->d_tlsContext.d_addr, true, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); + auto clientState = std::make_shared(frontend->d_tlsContext.d_addr, true, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); clientState->dohFrontend = std::move(frontend); clientState->d_additionalAddresses = std::move(additionalAddresses); @@ -2563,7 +2574,10 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) if (tcpMaxConcurrentConnections > 0) { clientState->d_tcpConcurrentConnectionsLimit = tcpMaxConcurrentConnections; } - g_frontends.push_back(std::move(clientState)); + + dnsdist::configuration::updateImmutableConfiguration([&clientState](dnsdist::configuration::Configuration& config) { + config.d_frontends.push_back(std::move(clientState)); + }); #else /* HAVE_DNS_OVER_HTTPS */ throw std::runtime_error("addDOHLocal() called but DNS over HTTPS support is not present!"); #endif /* HAVE_DNS_OVER_HTTPS */ @@ -2633,12 +2647,14 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) checkAllParametersConsumed("addDOH3Local", vars); } - g_doh3locals.push_back(frontend); - auto clientState = std::make_unique(frontend->d_local, false, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); + + auto clientState = std::make_shared(frontend->d_local, false, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); clientState->doh3Frontend = frontend; clientState->d_additionalAddresses = std::move(additionalAddresses); - g_frontends.push_back(std::move(clientState)); + dnsdist::configuration::updateImmutableConfiguration([&clientState](dnsdist::configuration::Configuration& config) { + config.d_frontends.push_back(std::move(clientState)); + }); #else throw std::runtime_error("addDOH3Local() called but DNS over HTTP/3 support is not present!"); #endif @@ -2708,12 +2724,14 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) checkAllParametersConsumed("addDOQLocal", vars); } - g_doqlocals.push_back(frontend); - auto clientState = std::make_unique(frontend->d_local, false, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); + + auto clientState = std::make_shared(frontend->d_local, false, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); clientState->doqFrontend = std::move(frontend); clientState->d_additionalAddresses = std::move(additionalAddresses); - g_frontends.push_back(std::move(clientState)); + dnsdist::configuration::updateImmutableConfiguration([&clientState](dnsdist::configuration::Configuration& config) { + config.d_frontends.push_back(std::move(clientState)); + }); #else throw std::runtime_error("addDOQLocal() called but DNS over QUIC support is not present!"); #endif @@ -2727,7 +2745,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) boost::format fmt("%-3d %-20.20s %-15d %-15d %-15d %-15d"); ret << (fmt % "#" % "Address" % "Bad Version" % "Invalid Token" % "Errors" % "Valid") << endl; size_t counter = 0; - for (const auto& ctx : g_doqlocals) { + for (const auto& ctx : dnsdist::getDoQFrontends()) { ret << (fmt % counter % ctx->d_local.toStringWithPort() % ctx->d_doqUnsupportedVersionErrors % ctx->d_doqInvalidTokensReceived % ctx->d_errorResponses % ctx->d_validResponses) << endl; counter++; } @@ -2750,12 +2768,13 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) } setLuaNoSideEffect(); try { - if (index < g_doqlocals.size()) { - result = g_doqlocals.at(index); + auto doqFrontends = dnsdist::getDoQFrontends(); + if (index < doqFrontends.size()) { + result = doqFrontends.at(index); } else { - errlog("Error: trying to get DOQ frontend with index %d but we only have %d frontend(s)\n", index, g_doqlocals.size()); - g_outputBuffer = "Error: trying to get DOQ frontend with index " + std::to_string(index) + " but we only have " + std::to_string(g_doqlocals.size()) + " frontend(s)\n"; + errlog("Error: trying to get DOQ frontend with index %d but we only have %d frontend(s)\n", index, doqFrontends.size()); + g_outputBuffer = "Error: trying to get DOQ frontend with index " + std::to_string(index) + " but we only have " + std::to_string(doqFrontends.size()) + " frontend(s)\n"; } } catch (const std::exception& e) { @@ -2767,7 +2786,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) luaCtx.writeFunction("getDOQFrontendCount", []() { setLuaNoSideEffect(); - return g_doqlocals.size(); + return dnsdist::getDoQFrontends().size(); }); luaCtx.registerFunction::*)()>("reloadCertificates", [](const std::shared_ptr& frontend) { @@ -2785,7 +2804,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) boost::format fmt("%-3d %-20.20s %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d"); ret << (fmt % "#" % "Address" % "HTTP" % "HTTP/1" % "HTTP/2" % "GET" % "POST" % "Bad" % "Errors" % "Redirects" % "Valid" % "# ticket keys" % "Rotation delay" % "Next rotation") << endl; size_t counter = 0; - for (const auto& ctx : g_dohlocals) { + for (const auto& ctx : dnsdist::getDoHFrontends()) { ret << (fmt % counter % ctx->d_tlsContext.d_addr.toStringWithPort() % ctx->d_httpconnects % ctx->d_http1Stats.d_nbQueries % ctx->d_http2Stats.d_nbQueries % ctx->d_getqueries % ctx->d_postqueries % ctx->d_badrequests % ctx->d_errorresponses % ctx->d_redirectresponses % ctx->d_validresponses % ctx->getTicketsKeysCount() % ctx->getTicketsKeyRotationDelay() % ctx->getNextTicketsKeyRotation()) << endl; counter++; } @@ -2808,7 +2827,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) boost::format fmt("%-3d %-20.20s %-15d %-15d %-15d %-15d"); ret << (fmt % "#" % "Address" % "Bad Version" % "Invalid Token" % "Errors" % "Valid") << endl; size_t counter = 0; - for (const auto& ctx : g_doh3locals) { + for (const auto& ctx : dnsdist::getDoH3Frontends()) { ret << (fmt % counter % ctx->d_local.toStringWithPort() % ctx->d_doh3UnsupportedVersionErrors % ctx->d_doh3InvalidTokensReceived % ctx->d_errorResponses % ctx->d_validResponses) << endl; counter++; } @@ -2831,12 +2850,13 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) } setLuaNoSideEffect(); try { - if (index < g_doh3locals.size()) { - result = g_doh3locals.at(index); + auto doh3Frontends = dnsdist::getDoH3Frontends(); + if (index < doh3Frontends.size()) { + result = doh3Frontends.at(index); } else { - errlog("Error: trying to get DOH3 frontend with index %d but we only have %d frontend(s)\n", index, g_doh3locals.size()); - g_outputBuffer = "Error: trying to get DOH3 frontend with index " + std::to_string(index) + " but we only have " + std::to_string(g_doh3locals.size()) + " frontend(s)\n"; + errlog("Error: trying to get DOH3 frontend with index %d but we only have %d frontend(s)\n", index, doh3Frontends.size()); + g_outputBuffer = "Error: trying to get DOH3 frontend with index " + std::to_string(index) + " but we only have " + std::to_string(doh3Frontends.size()) + " frontend(s)\n"; } } catch (const std::exception& e) { @@ -2848,7 +2868,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) luaCtx.writeFunction("getDOH3FrontendCount", []() { setLuaNoSideEffect(); - return g_doh3locals.size(); + return dnsdist::getDoH3Frontends().size(); }); luaCtx.registerFunction::*)()>("reloadCertificates", [](const std::shared_ptr& frontend) { @@ -2867,7 +2887,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) g_outputBuffer = "\n- HTTP/1:\n\n"; ret << (fmt % "#" % "Address" % "200" % "400" % "403" % "500" % "502" % "Others") << endl; size_t counter = 0; - for (const auto& ctx : g_dohlocals) { + for (const auto& ctx : dnsdist::getDoHFrontends()) { ret << (fmt % counter % ctx->d_tlsContext.d_addr.toStringWithPort() % ctx->d_http1Stats.d_nb200Responses % ctx->d_http1Stats.d_nb400Responses % ctx->d_http1Stats.d_nb403Responses % ctx->d_http1Stats.d_nb500Responses % ctx->d_http1Stats.d_nb502Responses % ctx->d_http1Stats.d_nbOtherResponses) << endl; counter++; } @@ -2877,7 +2897,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) g_outputBuffer += "\n- HTTP/2:\n\n"; ret << (fmt % "#" % "Address" % "200" % "400" % "403" % "500" % "502" % "Others") << endl; counter = 0; - for (const auto& ctx : g_dohlocals) { + for (const auto& ctx : dnsdist::getDoHFrontends()) { ret << (fmt % counter % ctx->d_tlsContext.d_addr.toStringWithPort() % ctx->d_http2Stats.d_nb200Responses % ctx->d_http2Stats.d_nb400Responses % ctx->d_http2Stats.d_nb403Responses % ctx->d_http2Stats.d_nb500Responses % ctx->d_http2Stats.d_nb502Responses % ctx->d_http2Stats.d_nbOtherResponses) << endl; counter++; } @@ -2900,12 +2920,13 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) #ifdef HAVE_DNS_OVER_HTTPS setLuaNoSideEffect(); try { - if (index < g_dohlocals.size()) { - result = g_dohlocals.at(index); + auto dohFrontends = dnsdist::getDoHFrontends(); + if (index < dohFrontends.size()) { + result = dohFrontends.at(index); } else { - errlog("Error: trying to get DOH frontend with index %d but we only have %d frontend(s)\n", index, g_dohlocals.size()); - g_outputBuffer = "Error: trying to get DOH frontend with index " + std::to_string(index) + " but we only have " + std::to_string(g_dohlocals.size()) + " frontend(s)\n"; + errlog("Error: trying to get DOH frontend with index %d but we only have %d frontend(s)\n", index, dohFrontends.size()); + g_outputBuffer = "Error: trying to get DOH frontend with index " + std::to_string(index) + " but we only have " + std::to_string(dohFrontends.size()) + " frontend(s)\n"; } } catch (const std::exception& e) { @@ -2920,7 +2941,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) luaCtx.writeFunction("getDOHFrontendCount", []() { setLuaNoSideEffect(); - return g_dohlocals.size(); + return dnsdist::getDoHFrontends().size(); }); luaCtx.registerFunction::*)()>("reloadCertificates", [](const std::shared_ptr& frontend) { @@ -3043,7 +3064,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) vinfolog("Loading default TLS provider '%s'", provider); } // only works pre-startup, so no sync necessary - auto clientState = std::make_unique(frontend->d_addr, true, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); + auto clientState = std::make_shared(frontend->d_addr, true, reusePort, tcpFastOpenQueueSize, interface, cpus, enableProxyProtocol); clientState->tlsFrontend = frontend; clientState->d_additionalAddresses = std::move(additionalAddresses); if (tcpListenQueueSize > 0) { @@ -3056,8 +3077,9 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) clientState->d_tcpConcurrentConnectionsLimit = tcpMaxConcurrentConns; } - g_tlslocals.push_back(clientState->tlsFrontend); - g_frontends.push_back(std::move(clientState)); + dnsdist::configuration::updateImmutableConfiguration([&clientState](dnsdist::configuration::Configuration& config) { + config.d_frontends.push_back(std::move(clientState)); + }); } catch (const std::exception& e) { g_outputBuffer = "Error: " + string(e.what()) + "\n"; @@ -3076,7 +3098,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) // 1 2 3 4 5 ret << (fmt % "#" % "Address" % "# ticket keys" % "Rotation delay" % "Next rotation") << endl; size_t counter = 0; - for (const auto& ctx : g_tlslocals) { + for (const auto& ctx : dnsdist::getDoTFrontends()) { ret << (fmt % counter % ctx->d_addr.toStringWithPort() % ctx->getTicketsKeysCount() % ctx->getTicketsKeyRotationDelay() % ctx->getNextTicketsKeyRotation()) << endl; counter++; } @@ -3095,13 +3117,14 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) std::shared_ptr result = nullptr; #ifdef HAVE_DNS_OVER_TLS setLuaNoSideEffect(); + const auto tlsFrontends = dnsdist::getDoTFrontends(); try { - if (index < g_tlslocals.size()) { - result = g_tlslocals.at(index)->getContext(); + if (index < tlsFrontends.size()) { + result = tlsFrontends.at(index)->getContext(); } else { - errlog("Error: trying to get TLS context with index %d but we only have %d context(s)\n", index, g_tlslocals.size()); - g_outputBuffer = "Error: trying to get TLS context with index " + std::to_string(index) + " but we only have " + std::to_string(g_tlslocals.size()) + " context(s)\n"; + errlog("Error: trying to get TLS context with index %d but we only have %d context(s)\n", index, tlsFrontends.size()); + g_outputBuffer = "Error: trying to get TLS context with index " + std::to_string(index) + " but we only have " + std::to_string(tlsFrontends.size()) + " context(s)\n"; } } catch (const std::exception& e) { @@ -3119,12 +3142,13 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) #ifdef HAVE_DNS_OVER_TLS setLuaNoSideEffect(); try { - if (index < g_tlslocals.size()) { - result = g_tlslocals.at(index); + auto tlsFrontends = dnsdist::getDoTFrontends(); + if (index < tlsFrontends.size()) { + result = tlsFrontends.at(index); } else { - errlog("Error: trying to get TLS frontend with index %d but we only have %d frontends\n", index, g_tlslocals.size()); - g_outputBuffer = "Error: trying to get TLS frontend with index " + std::to_string(index) + " but we only have " + std::to_string(g_tlslocals.size()) + " frontend(s)\n"; + errlog("Error: trying to get TLS frontend with index %d but we only have %d frontends\n", index, tlsFrontends.size()); + g_outputBuffer = "Error: trying to get TLS frontend with index " + std::to_string(index) + " but we only have " + std::to_string(tlsFrontends.size()) + " frontend(s)\n"; } } catch (const std::exception& e) { @@ -3139,7 +3163,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) luaCtx.writeFunction("getTLSFrontendCount", []() { setLuaNoSideEffect(); - return g_tlslocals.size(); + return dnsdist::getDoTFrontends().size(); }); luaCtx.registerFunction::*)()>("rotateTicketsKey", [](std::shared_ptr& ctx) { @@ -3197,7 +3221,7 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) }); luaCtx.writeFunction("reloadAllCertificates", []() { - for (auto& frontend : g_frontends) { + for (auto& frontend : dnsdist::getFrontends()) { if (!frontend) { continue; } diff --git a/pdns/dnsdistdist/dnsdist-svc.cc b/pdns/dnsdistdist/dnsdist-svc.cc index d09d8d7bf02f..a92fdb2072f9 100644 --- a/pdns/dnsdistdist/dnsdist-svc.cc +++ b/pdns/dnsdistdist/dnsdist-svc.cc @@ -175,7 +175,6 @@ bool generateSVCResponse(DNSQuestion& dnsQuestion, const std::vector frontendDuplicates; - for (const auto& front : g_frontends) { + for (const auto& front : dnsdist::getFrontends()) { if (front->udpFD == -1 && front->tcpFD == -1) { continue; } @@ -782,7 +783,7 @@ static void handlePrometheus(const YaHTTP::Request& req, YaHTTP::Response& resp) #ifdef HAVE_DNS_OVER_HTTPS std::map dohFrontendDuplicates; - for(const auto& doh : g_dohlocals) { + for(const auto& doh : dnsdist::getDoHFrontends()) { const string frontName = doh->d_tlsContext.d_addr.toStringWithPort(); uint64_t threadNumber = 0; auto dupPair = frontendDuplicates.emplace(frontName, 1); @@ -1144,8 +1145,8 @@ static void handleStats(const YaHTTP::Request& req, YaHTTP::Response& resp) Json::array frontends; num = 0; - frontends.reserve(g_frontends.size()); - for (const auto& front : g_frontends) { + frontends.reserve(dnsdist::getFrontends().size()); + for (const auto& front : dnsdist::getFrontends()) { if (front->udpFD == -1 && front->tcpFD == -1) { continue; } @@ -1200,9 +1201,10 @@ static void handleStats(const YaHTTP::Request& req, YaHTTP::Response& resp) Json::array dohs; #ifdef HAVE_DNS_OVER_HTTPS { - dohs.reserve(g_dohlocals.size()); + const auto dohFrontends = dnsdist::getDoHFrontends(); + dohs.reserve(dohFrontends.size()); num = 0; - for (const auto& doh : g_dohlocals) { + for (const auto& doh : dohFrontends) { dohs.emplace_back(Json::object{ {"id", num++}, {"address", doh->d_tlsContext.d_addr.toStringWithPort()}, @@ -1271,7 +1273,7 @@ static void handleStats(const YaHTTP::Request& req, YaHTTP::Response& resp) string localaddressesStr; { std::set localaddresses; - for (const auto& front : g_frontends) { + for (const auto& front : dnsdist::getFrontends()) { localaddresses.insert(front->local.toStringWithPort()); } for (const auto& addr : localaddresses) { diff --git a/pdns/dnsdistdist/dnsdist.cc b/pdns/dnsdistdist/dnsdist.cc index f0c6810540dd..1df9452273df 100644 --- a/pdns/dnsdistdist/dnsdist.cc +++ b/pdns/dnsdistdist/dnsdist.cc @@ -50,6 +50,7 @@ #include "dnsdist-dynblocks.hh" #include "dnsdist-ecs.hh" #include "dnsdist-edns.hh" +#include "dnsdist-frontend.hh" #include "dnsdist-healthchecks.hh" #include "dnsdist-lua.hh" #include "dnsdist-lua-hooks.hh" @@ -96,16 +97,9 @@ using std::thread; string g_outputBuffer; -std::vector> g_tlslocals; -std::vector> g_dohlocals; -std::vector> g_doqlocals; -std::vector> g_doh3locals; -std::vector> g_dnsCryptLocals; - shared_ptr g_defaultBPFFilter{nullptr}; std::vector> g_dynBPFFilters; -std::vector> g_frontends; /* UDP: the grand design. Per socket we listen on for incoming queries there is one thread. Then we have a bunch of connected sockets for talking to downstream servers. We send directly to those sockets. @@ -2685,37 +2679,37 @@ static void setupLocalSocket(ClientState& clientState, const ComboAddress& addr, } } -static void setUpLocalBind(std::unique_ptr& cstate) +static void setUpLocalBind(ClientState& cstate) { /* skip some warnings if there is an identical UDP context */ - bool warn = !cstate->tcp || cstate->tlsFrontend != nullptr || cstate->dohFrontend != nullptr; - int& descriptor = !cstate->tcp ? cstate->udpFD : cstate->tcpFD; + bool warn = !cstate.tcp || cstate.tlsFrontend != nullptr || cstate.dohFrontend != nullptr; + int& descriptor = !cstate.tcp ? cstate.udpFD : cstate.tcpFD; (void)warn; - setupLocalSocket(*cstate, cstate->local, descriptor, cstate->tcp, warn); + setupLocalSocket(cstate, cstate.local, descriptor, cstate.tcp, warn); - for (auto& [addr, socket] : cstate->d_additionalAddresses) { - setupLocalSocket(*cstate, addr, socket, true, false); + for (auto& [addr, socket] : cstate.d_additionalAddresses) { + setupLocalSocket(cstate, addr, socket, true, false); } - if (cstate->tlsFrontend != nullptr) { - if (!cstate->tlsFrontend->setupTLS()) { - errlog("Error while setting up TLS on local address '%s', exiting", cstate->local.toStringWithPort()); + if (cstate.tlsFrontend != nullptr) { + if (!cstate.tlsFrontend->setupTLS()) { + errlog("Error while setting up TLS on local address '%s', exiting", cstate.local.toStringWithPort()); _exit(EXIT_FAILURE); } } - if (cstate->dohFrontend != nullptr) { - cstate->dohFrontend->setup(); + if (cstate.dohFrontend != nullptr) { + cstate.dohFrontend->setup(); } - if (cstate->doqFrontend != nullptr) { - cstate->doqFrontend->setup(); + if (cstate.doqFrontend != nullptr) { + cstate.doqFrontend->setup(); } - if (cstate->doh3Frontend != nullptr) { - cstate->doh3Frontend->setup(); + if (cstate.doh3Frontend != nullptr) { + cstate.doh3Frontend->setup(); } - cstate->ready = true; + cstate.ready = true; } struct CommandLineParameters @@ -3131,11 +3125,13 @@ static void dropPrivileges(const CommandLineParameters& cmdLine) static void initFrontends(const CommandLineParameters& cmdLine) { + auto frontends = dnsdist::configuration::getImmutableConfiguration().d_frontends; + if (!cmdLine.locals.empty()) { - for (auto it = g_frontends.begin(); it != g_frontends.end();) { + for (auto it = frontends.begin(); it != frontends.end();) { /* DoH, DoT and DNSCrypt frontends are separate */ if ((*it)->dohFrontend == nullptr && (*it)->tlsFrontend == nullptr && (*it)->dnscryptCtx == nullptr && (*it)->doqFrontend == nullptr && (*it)->doh3Frontend == nullptr) { - it = g_frontends.erase(it); + it = frontends.erase(it); } else { ++it; @@ -3144,18 +3140,22 @@ static void initFrontends(const CommandLineParameters& cmdLine) for (const auto& loc : cmdLine.locals) { /* UDP */ - g_frontends.emplace_back(std::make_unique(ComboAddress(loc, 53), false, false, 0, "", std::set{}, true)); + frontends.emplace_back(std::make_unique(ComboAddress(loc, 53), false, false, 0, "", std::set{}, true)); /* TCP */ - g_frontends.emplace_back(std::make_unique(ComboAddress(loc, 53), true, false, 0, "", std::set{}, true)); + frontends.emplace_back(std::make_unique(ComboAddress(loc, 53), true, false, 0, "", std::set{}, true)); } } - if (g_frontends.empty()) { + if (frontends.empty()) { /* UDP */ - g_frontends.emplace_back(std::make_unique(ComboAddress("127.0.0.1", 53), false, false, 0, "", std::set{}, true)); + frontends.emplace_back(std::make_unique(ComboAddress("127.0.0.1", 53), false, false, 0, "", std::set{}, true)); /* TCP */ - g_frontends.emplace_back(std::make_unique(ComboAddress("127.0.0.1", 53), true, false, 0, "", std::set{}, true)); + frontends.emplace_back(std::make_unique(ComboAddress("127.0.0.1", 53), true, false, 0, "", std::set{}, true)); } + + dnsdist::configuration::updateImmutableConfiguration([&frontends](dnsdist::configuration::Configuration& config) { + config.d_frontends = std::move(frontends); + }); } namespace dnsdist @@ -3171,7 +3171,7 @@ static void startFrontends() std::vector tcpStates; std::vector udpStates; - for (auto& clientState : g_frontends) { + for (auto& clientState : dnsdist::getFrontends()) { #ifdef HAVE_XSK if (clientState->xskInfo) { dnsdist::xsk::addDestinationAddress(clientState->local); @@ -3389,7 +3389,7 @@ int main(int argc, char** argv) initFrontends(cmdLine); - for (const auto& frontend : g_frontends) { + for (const auto& frontend : dnsdist::getFrontends()) { if (!frontend->tcp) { ++udpBindsCount; } @@ -3423,8 +3423,8 @@ int main(int argc, char** argv) g_rings.init(config.d_ringsCapacity, config.d_ringsNumberOfShards, config.d_ringsNbLockTries, config.d_ringsRecordQueries, config.d_ringsRecordResponses); } - for (auto& frontend : g_frontends) { - setUpLocalBind(frontend); + for (auto& frontend : dnsdist::getFrontends()) { + setUpLocalBind(*frontend); } { diff --git a/pdns/dnsdistdist/dnsdist.hh b/pdns/dnsdistdist/dnsdist.hh index 163a508f1098..ece21758c1c6 100644 --- a/pdns/dnsdistdist/dnsdist.hh +++ b/pdns/dnsdistdist/dnsdist.hh @@ -1019,12 +1019,6 @@ enum ednsHeaderFlags EDNS_HEADER_FLAG_DO = 32768 }; -extern std::vector> g_tlslocals; -extern std::vector> g_dohlocals; -extern std::vector> g_doqlocals; -extern std::vector> g_doh3locals; -extern std::vector> g_frontends; - extern shared_ptr g_defaultBPFFilter; extern std::vector> g_dynBPFFilters; @@ -1039,7 +1033,6 @@ bool responseContentMatches(const PacketBuffer& response, const DNSName& qname, bool checkQueryHeaders(const struct dnsheader& dnsHeader, ClientState& clientState); -extern std::vector> g_dnsCryptLocals; bool handleDNSCryptQuery(PacketBuffer& packet, DNSCryptQuery& query, bool tcp, time_t now, PacketBuffer& response); bool checkDNSCryptQuery(const ClientState& clientState, PacketBuffer& query, std::unique_ptr& dnsCryptQuery, time_t now, bool tcp); diff --git a/pdns/dnsdistdist/test-dnsdistlbpolicies_cc.cc b/pdns/dnsdistdist/test-dnsdistlbpolicies_cc.cc index 0391f8fb4aa9..abe9a95c9e6e 100644 --- a/pdns/dnsdistdist/test-dnsdistlbpolicies_cc.cc +++ b/pdns/dnsdistdist/test-dnsdistlbpolicies_cc.cc @@ -23,8 +23,6 @@ std::unique_ptr g_snmpAgent{nullptr}; Rings g_rings; #endif /* BENCH_POLICIES */ -std::vector> g_frontends; - /* add stub implementations, we don't want to include the corresponding object files and their dependencies */