From cf97e3085fb31681bafd19d4d8470fdf21556490 Mon Sep 17 00:00:00 2001 From: "nastassia.dailidava" Date: Mon, 18 Sep 2023 10:35:35 +0200 Subject: [PATCH] logs #292 --- .../snapshot/EnvoySnapshotFactory.kt | 4 +- .../resource/clusters/EnvoyClustersFactory.kt | 17 +-- .../trafficsplitting/TrafficSplitting.kt | 17 +++ .../WeightedClustersRoutingTest.kt | 17 +++ .../WeightedClustersServiceTagRoutingTest.kt | 108 ++++++++++++++++++ 5 files changed, 149 insertions(+), 14 deletions(-) create mode 100644 envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersServiceTagRoutingTest.kt diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt index 289cd6914..0b009c1e2 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/EnvoySnapshotFactory.kt @@ -341,7 +341,9 @@ class EnvoySnapshotFactory( listenersVersion = version.listeners, routes = routes, routesVersion = version.routes - ) + ).also { + logger.info("Snapshot for group: $it") + } } private fun createRoutesWhenUsingTransparentProxy( diff --git a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt index 407da6b9e..7986b352c 100644 --- a/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt +++ b/envoy-control-core/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/snapshot/resource/clusters/EnvoyClustersFactory.kt @@ -86,11 +86,7 @@ class EnvoyClustersFactory( communicationMode: CommunicationMode ): List { return services.map { edsCluster(it, communicationMode) } - .also { - it.forEach { cluster -> - logger.debug(" Created cluster config for services: ${cluster.name}") - } - } + .onEach { logger.debug("Created cluster config for services: ${it.toString()}") } } fun getSecuredClusters(insecureClusters: List): List { @@ -247,6 +243,7 @@ class EnvoyClustersFactory( .setCommonHttpProtocolOptions(HttpProtocolOptions.newBuilder().setIdleTimeout(idleTimeoutPolicy)) .setName(clusterName) .build() + .also { logger.debug("Created regular cluster config ${it.toString()}") } } private fun createSetOfClustersForGroup( @@ -262,10 +259,8 @@ class EnvoyClustersFactory( val aggregateCluster = createAggregateCluster(mainCluster.name, linkedSetOf(secondaryCluster.name, mainCluster.name)) return listOf(mainCluster, secondaryCluster, aggregateCluster) - .also { - it.forEach { cl -> - logger.debug("Created traffic splitting cluster config with cluster name: {}", cl.name) - } + .onEach { + logger.debug("Created set of cluster configs for traffic splitting: {}", it.toString()) } } @@ -277,12 +272,8 @@ class EnvoyClustersFactory( ): Collection { return cluster?.let { if (enableTrafficSplitting(serviceName, clusterLoadAssignment)) { - logger.debug( - "Creating traffic splitting cluster config for ${cluster.name}, service: $serviceName" - ) createSetOfClustersForGroup(dependencySettings, cluster) } else { - logger.debug("Creating cluster config for ${cluster.name}, service: $serviceName") listOf(createClusterForGroup(dependencySettings, cluster)) } } ?: listOf() diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt index 02152f999..a49c7f996 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/TrafficSplitting.kt @@ -48,3 +48,20 @@ fun EnvoyExtension.callUpstreamServiceRepeatedly( ) return stats } + +fun EnvoyExtension.callUpstreamServiceRepeatedly( + vararg services: EchoServiceExtension, + numberOfCalls: Int = 100, + tag: String? +): CallStats { + val stats = CallStats(services.asList()) + this.egressOperations.callServiceRepeatedly( + service = upstreamServiceName, + stats = stats, + minRepeat = numberOfCalls, + maxRepeat = numberOfCalls, + repeatUntil = { true }, + headers = tag?.let { mapOf("x-service-tag" to it) } ?: emptyMap(), + ) + return stats +} diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt index be3d28cee..371ff8038 100644 --- a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersRoutingTest.kt @@ -10,6 +10,7 @@ import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulMultiCluster import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlClusteredExtension import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension +import pl.allegro.tech.servicemesh.envoycontrol.logger import verifyCallsCountCloseTo import verifyCallsCountGreaterThan import verifyIsReachable @@ -17,6 +18,7 @@ import java.time.Duration class WeightedClustersRoutingTest { companion object { + val logger by logger() private const val forceTrafficZone = "dc2" private val properties = mapOf( @@ -96,4 +98,19 @@ class WeightedClustersRoutingTest { .verifyCallsCountCloseTo(upstreamServiceDC1, 90) .verifyCallsCountGreaterThan(upstreamServiceDC2, 1) } + + @Test + fun `should route traffic according to weights with service tag`() { + consul.serverFirst.operations.registerServiceWithEnvoyOnEgress(echoEnvoyDC1, name = serviceName) + + consul.serverFirst.operations.registerService(upstreamServiceDC1, name = upstreamServiceName, tags = listOf("tag")) + echoEnvoyDC1.verifyIsReachable(upstreamServiceDC1, upstreamServiceName) + + consul.serverSecond.operations.registerService(upstreamServiceDC2, name = upstreamServiceName, tags = listOf("tag")) + echoEnvoyDC1.verifyIsReachable(upstreamServiceDC2, upstreamServiceName) + + echoEnvoyDC1.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2, tag = "tag") + .verifyCallsCountCloseTo(upstreamServiceDC1, 90) + .verifyCallsCountGreaterThan(upstreamServiceDC2, 1) + } } diff --git a/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersServiceTagRoutingTest.kt b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersServiceTagRoutingTest.kt new file mode 100644 index 000000000..0ac4982f2 --- /dev/null +++ b/envoy-control-tests/src/main/kotlin/pl/allegro/tech/servicemesh/envoycontrol/trafficsplitting/WeightedClustersServiceTagRoutingTest.kt @@ -0,0 +1,108 @@ +package pl.allegro.tech.servicemesh.envoycontrol.trafficsplitting + +import TrafficSplitting.serviceName +import TrafficSplitting.upstreamServiceName +import callUpstreamServiceRepeatedly +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.RegisterExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.Xds +import pl.allegro.tech.servicemesh.envoycontrol.config.consul.ConsulMultiClusterExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.envoy.EnvoyExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.envoycontrol.EnvoyControlClusteredExtension +import pl.allegro.tech.servicemesh.envoycontrol.config.service.EchoServiceExtension +import pl.allegro.tech.servicemesh.envoycontrol.logger +import verifyCallsCountCloseTo +import verifyCallsCountGreaterThan +import verifyIsReachable +import java.time.Duration + +class WeightedClustersServiceTagRoutingTest { + companion object { + val logger by logger() + private const val forceTrafficZone = "dc2" + + private val properties = mapOf( + "envoy-control.envoy.snapshot.routing.service-tags.enabled" to true, + "envoy-control.envoy.snapshot.routing.service-tags.metadata-key" to "tag", + "envoy-control.envoy.snapshot.routing.service-tags.auto-service-tag-enabled" to true, + "envoy-control.envoy.snapshot.outgoing-permissions.services-allowed-to-use-wildcard" to setOf("echo2", "test-service"), + "logging.level.pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.clusters.EnvoyClustersFactory" to "DEBUG", + "logging.level.pl.allegro.tech.servicemesh.envoycontrol.snapshot.resource.routes.EnvoyEgressRoutesFactory" to "DEBUG", + "logging.level.pl.allegro.tech.servicemesh.envoycontrol.snapshot.EnvoySnapshotFactory" to "DEBUG", + "envoy-control.envoy.snapshot.stateSampleDuration" to Duration.ofSeconds(0), + "envoy-control.sync.enabled" to true, + "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.zoneName" to forceTrafficZone, + "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.serviceByWeightsProperties.$serviceName.main" to 90, + "envoy-control.envoy.snapshot.loadBalancing.trafficSplitting.serviceByWeightsProperties.$serviceName.secondary" to 10, + "envoy-control.envoy.snapshot.loadBalancing.priorities.zonePriorities" to mapOf( + "dc1" to mapOf( + "dc1" to 0, + "dc2" to 1 + ), + "dc2" to mapOf( + "dc1" to 1, + "dc2" to 0, + ), + ) + ) + + private val echo2Config = """ + node: + metadata: + proxy_settings: + outgoing: + dependencies: + - service: "service-1" + """.trimIndent() + + private val config = Xds.copy(configOverride = echo2Config, serviceName = "echo2") + + @JvmField + @RegisterExtension + val consul = ConsulMultiClusterExtension() + + @JvmField + @RegisterExtension + val envoyControl = + EnvoyControlClusteredExtension(consul.serverFirst, { properties }, listOf(consul)) + + @JvmField + @RegisterExtension + val envoyControl2 = + EnvoyControlClusteredExtension(consul.serverSecond, { properties }, listOf(consul)) + + @JvmField + @RegisterExtension + val echoServiceDC1 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val upstreamServiceDC1 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val upstreamServiceDC2 = EchoServiceExtension() + + @JvmField + @RegisterExtension + val echoEnvoyDC1 = EnvoyExtension(envoyControl, localService = echoServiceDC1, config) + @JvmField + @RegisterExtension + val echoEnvoyDC2 = EnvoyExtension(envoyControl2) + } + + @Test + fun `should route traffic according to weights with service tag`() { + consul.serverFirst.operations.registerServiceWithEnvoyOnEgress(echoEnvoyDC1, name = serviceName) + + consul.serverFirst.operations.registerService(upstreamServiceDC1, name = upstreamServiceName, tags = listOf("global", "tag")) + echoEnvoyDC1.verifyIsReachable(upstreamServiceDC1, upstreamServiceName) + + consul.serverSecond.operations.registerService(upstreamServiceDC2, name = upstreamServiceName, tags = listOf("global", "tag")) + echoEnvoyDC1.verifyIsReachable(upstreamServiceDC2, upstreamServiceName) + + echoEnvoyDC1.callUpstreamServiceRepeatedly(upstreamServiceDC1, upstreamServiceDC2, tag = "tag") + .verifyCallsCountCloseTo(upstreamServiceDC1, 90) + .verifyCallsCountGreaterThan(upstreamServiceDC2, 1) + } +}