Merge pull request #518 from allegro/mg-ms-pluton-19231-optimize-render

PLUTON-19231 | Optimize vcl rendering fix #516

Author: ziollek

envs/base.env

@@ -2,5 +2,5 @@
 DEBUG=False
 TEMPLATE_DEBUG=False
 DJANGO_SETTINGS_MODULE=vaas.settings.docker
-PROMETHEUS_ENABLE=True
+PROMETHEUS_ENABLE=False
 LOG_LEVEL=INFO

envs/dev.env

@@ -2,5 +2,5 @@
 DEBUG=True
 TEMPLATE_DEBUG=True
 DJANGO_SETTINGS_MODULE=vaas.settings.docker
-PROMETHEUS_ENABLE=True
+PROMETHEUS_ENABLE=False
 LOG_LEVEL=DEBUG

vaas/vaas/resources/dev-data.yaml

@@ -38,94 +38,6 @@
     remove_path: false
     time_profile: 1
     cluster: [1, 2, 3, 4]
-- model: cluster.dc
-  pk: 1
-  fields: {name: First datacenter, symbol: dc1}
-- model: cluster.dc
-  pk: 2
-  fields: {name: Second datacenter, symbol: dc2}
-- model: manager.backend
-  pk: 1
-  fields: {address: 192.168.199.10, port: 80, weight: 1, dc: 1, max_connections: 5,
-    connect_timeout: '0.30', first_byte_timeout: '5.00', between_bytes_timeout: '1.00',
-    director: 1, enabled: true, inherit_time_profile: true}
-- model: manager.backend
-  pk: 2
-  fields: {address: 192.168.199.11, port: 80, weight: 1, dc: 1, max_connections: 5,
-    connect_timeout: '0.30', first_byte_timeout: '5.00', between_bytes_timeout: '1.00',
-    director: 1, enabled: true, inherit_time_profile: true}
-- model: manager.backend
-  pk: 3
-  fields: {address: 192.168.199.12, port: 80, weight: 2, dc: 1, max_connections: 5,
-    connect_timeout: '0.30', first_byte_timeout: '5.00', between_bytes_timeout: '1.00',
-    director: 1, enabled: true, inherit_time_profile: true}
-- model: manager.backend
-  pk: 4
-  fields: {address: 192.168.199.13, port: 80, weight: 3, dc: 1, max_connections: 5,
-    connect_timeout: '0.30', first_byte_timeout: '5.00', between_bytes_timeout: '1.00',
-    director: 2, enabled: true, inherit_time_profile: true}
-- model: manager.backend
-  pk: 5
-  fields: {address: 192.168.199.14, port: 80, weight: 4, dc: 1, max_connections: 5,
-    connect_timeout: '0.30', first_byte_timeout: '5.00', between_bytes_timeout: '1.00',
-    director: 2, enabled: true, inherit_time_profile: true}
-- model: manager.backend
-  pk: 6
-  fields: {address: 192.168.199.15, port: 80, weight: 5, dc: 1, max_connections: 5,
-    connect_timeout: '0.30', first_byte_timeout: '5.00', between_bytes_timeout: '1.00',
-    director: 2, enabled: true, inherit_time_profile: true}
-- model: cluster.logicalcluster
-  pk: 1
-  fields: {name: cluster1_siteA_test, reload_timestamp: '2019-11-19T08:02:15.994071+00:00',
-    error_timestamp: '2019-11-13T07:46:40.450785+00:00', last_error_info: null,
-           current_vcl_versions: '[]', labels_list: '["one:192.168.199.6"]'}
-- model: cluster.logicalcluster
-  pk: 2
-  fields: {name: cluster2_siteB_test, reload_timestamp: '2019-11-18T14:16:58.664933+00:00',
-    error_timestamp: '2019-11-13T07:46:40.453146+00:00', last_error_info: null,
-    current_vcl_versions: '[]', labels_list: '["two:192.168.199.4", "env:dev"]'}
-- model: cluster.logicalcluster
-  pk: 3
-  fields: {name: cluster3_siteA_dev, reload_timestamp: '2019-11-18T14:16:58.664933+00:00',
-    error_timestamp: '2019-11-13T07:46:40.453839+00:00', last_error_info: null,
-           current_vcl_versions: '[]', labels_list: '["example.com"]'}
-- model: cluster.logicalcluster
-  pk: 4
-  fields: {name: cluster4_siteC_prod, reload_timestamp: '2019-11-18T14:30:12.551574+00:00',
-    error_timestamp: '2019-11-13T07:46:40.454405+00:00', last_error_info: null,
-            current_vcl_versions: '[]', labels_list: '["placeholder:cluster4"]'}
-- model: cluster.domainmapping
-  pk: 1
-  fields: {domain: "mydomain.com", mappings_list: '["{one}:6081","{two}:6081"]', type: dynamic, clusters: []}
-- model: cluster.domainmapping
-  pk: 2
-  fields: {domain: "example.com", mappings_list: '["example.base.com"]', type: static, clusters: [3]}
-- model: cluster.vcltemplate
-  pk: 2
-  fields: {name: vagrant_template_4, content: "<VCL/>", version: '4.0', comment: wefwef}
-- model: cluster.varnishserver
-  pk: 3
-  fields: {ip: 192.168.199.4, hostname: varnish-4.1, cluster_weight: 1, http_port: 6081,
-    port: 6082, secret: edcf6c52-6f93-4d0d-82b9-cd74239146b0, status: active, dc: 1,
-    template: 2, cluster: 2, is_canary: false}
-- model: cluster.varnishserver
-  pk: 4
-  fields: {ip: 192.168.199.6, hostname: varnish-6, cluster_weight: 1, http_port: 6081,
-    port: 6082, secret: edcf6c52-6f93-4d0d-82b9-cd74239146b0, status: active, dc: 1,
-    template: 2, cluster: 1, is_canary: false}
-- model: cluster.varnishserver
-  pk: 2
-  fields: {ip: 192.168.199.7, hostname: varnish-7, cluster_weight: 1, http_port: 6081,
-    port: 6082, secret: edcf6c52-6f93-4d0d-82b9-cd74239146b0, status: active, dc: 1,
-    template: 2, cluster: 1, is_canary: false}
-- model: router.route
-  pk: 1
-  fields:
-    condition: req.url ~ "^\/flexibleee"
-    priority: 51
-    director: 2
-    action: pass
-    clusters: [2]
 - model: router.positiveurl
   pk: 1
   fields: {url: http://192.168.199.4:6081/flexibleee, route: 1}
@@ -147,7 +59,7 @@
 - model: router.redirect
   pk: 1
   fields:
-    src_domain: 1
+    src_domain: 3
     condition: req.url ~ "/source_url"
     destination: http://mydomain.com/destination_url
     action: 301

vaas/vaas/router/models.py

@@ -1,10 +1,9 @@
 # -*- coding: utf-8 -*-
 import uuid
-from typing import Dict, Tuple, List
+from typing import Dict
 from django.db import models
 from django.conf import settings
 from django.core.validators import MinValueValidator, MaxValueValidator
-from urllib.parse import urlsplit
 
 from vaas.cluster.models import DomainMapping, LogicalCluster
 from vaas.manager.models import Director
@@ -37,16 +36,6 @@ class ResponseStatusCode(models.IntegerChoices):
     def get_hashed_assertions_pks(self) -> Dict[int, int]:
         return {hash((a.given_url, a.expected_location)): a.pk for a in self.assertions.all()}
 
-    def fetch_all_destinations_mappings(self, cluster: LogicalCluster) -> Tuple[str, List[str]]:
-        """
-        Fetch tuple containing domain parsed from destination url and all found mappings for input cluster
-        """
-        all_mappings = set()
-        destination_domain = urlsplit(self.destination).netloc
-        for domain_mapping in DomainMapping.objects.filter(domain=destination_domain):
-            all_mappings = all_mappings.union(set(domain_mapping.mapped_domains(cluster)))
-        return destination_domain, list(all_mappings)
-
     @property
     def final_condition(self):
         if self.required_custom_header:

vaas/vaas/vcl/renderer.py

@@ -5,7 +5,7 @@
 import hashlib
 import time
 import functools
-from typing import Dict, List
+from urllib.parse import urlsplit
 
 from django.conf import settings
 from django.db.models import Prefetch
@@ -186,13 +186,26 @@ def __init__(self, varnish, input_data):
             'mesh_routing': varnish.cluster.service_mesh_routing
         }
 
+    def fetch_all_destinations_mappings(self, cluster: LogicalCluster, redirect: str,
+                                        domain_mappings: list[DomainMapping]) -> tuple[str, list[str]]:
+        """Fetch tuple containing domain parsed from destination url and all found mappings for input cluster
+        """
+        all_mappings = set()
+        destination_domain = urlsplit(redirect).netloc
+        for domain_mapping in domain_mappings:
+            all_mappings = all_mappings.union(set(domain_mapping.mapped_domains(cluster)))
+        return destination_domain, list(all_mappings)
+
     @collect_processing
-    def prepare_redirects(self) -> Dict[str, List[VclRedirect]]:
-        redirects = {}
-        related_domains = MappingProvider(DomainMapping.objects.all()).provide_related_domains(self.varnish.cluster)
-        for redirect in self.input.redirects:
-            destination_domain, destination_mappings = redirect.fetch_all_destinations_mappings(self.varnish.cluster)
-            if str(redirect.src_domain) in related_domains:
+    def prepare_redirects(self) -> dict[str, list[VclRedirect]]:
+        redirects = dict()
+        related_domains = self.input.mapping_provider.provide_related_domains(self.varnish.cluster)
+        for related_domain in related_domains:
+            if related_domain not in self.input.redirects.keys():
+                continue
+            for redirect in self.input.redirects.get(related_domain):
+                destination_domain, destination_mappings = self.fetch_all_destinations_mappings(
+                    self.varnish.cluster, redirect.destination, self.input.domain_mappings)
                 for mapped_domain in redirect.src_domain.mapped_domains(self.varnish.cluster):
                     destination = str(redirect.destination)
                     if destination_domain == redirect.src_domain.domain:
@@ -382,7 +395,7 @@ def fetch_render_data(self):
             Prefetch('clusters', queryset=LogicalCluster.objects.only('pk'), to_attr='cluster_ids'),
         ))
         self.routes.sort(key=lambda route: "{:03d}-{}".format(route.priority, route.director.name))
-        self.redirects = list(Redirect.objects.all().order_by('src_domain', 'priority'))
+        self.redirects = self.assemble_redirects()
         self.dcs = list(Dc.objects.all())
         self.template_blocks = list(VclTemplateBlock.objects.all().prefetch_related('template'))
         self.vcl_variables = list(VclVariable.objects.all())
@@ -392,6 +405,17 @@ def fetch_render_data(self):
         )
         self.distributed_backends = self.distribute_backends(backends)
         self.distributed_canary_backends = self.prepare_canary_backends(canary_backend_ids, backends)
+        self.domain_mappings = DomainMapping.objects.all()
+        self.mapping_provider = MappingProvider(self.domain_mappings)
+
+    @collect_processing
+    def assemble_redirects(self) -> dict[str, list[Redirect]]:
+        redirects = {}
+        for redirect in Redirect.objects.all().order_by('src_domain', 'priority'):
+            if redirect.src_domain.domain not in redirects.keys():
+                redirects[redirect.src_domain.domain] = []
+            redirects[redirect.src_domain.domain].append(redirect)
+        return redirects
 
     @collect_processing
     def distribute_backends(self, backends):

vaas/vaas/vcl/tests/expected-vcl-4.0-canary.vcl

@@ -339,7 +339,7 @@ sub vcl_synth {
     if (resp.status == 989) {
         set resp.status = 200;
         set resp.http.Content-Type = "application/json";
-        synthetic ( {"{ "vcl_version" : "0bbef", "varnish_status": "disabled" }"} );
+        synthetic ( {"{ "vcl_version" : "ce716", "varnish_status": "disabled" }"} );
         return (deliver);
     }
 }
@@ -424,12 +424,6 @@ sub vcl_recv {
         set req.http.x-response-code = "301";
         set req.http.x-action = "redirect";
     }
-    else if (req.url ~ "/external") {
-        set req.http.x-redirect = "3";
-        set req.http.x-destination = "http://example-external.com/external_destination";
-        set req.http.x-response-code = "301";
-        set req.http.x-action = "redirect";
-    }
     }
     if (req.http.host == "example.prod.org") {
     if (req.url ~ "/source") {
@@ -444,12 +438,6 @@ sub vcl_recv {
         set req.http.x-response-code = "301";
         set req.http.x-action = "redirect";
     }
-    else if (req.url ~ "/external") {
-        set req.http.x-redirect = "3";
-        set req.http.x-destination = "http://example-external.com/external_destination";
-        set req.http.x-response-code = "301";
-        set req.http.x-action = "redirect";
-    }
     }
 
 # Test ROUTER

vaas/vaas/vcl/tests/expected-vcl-4.0.vcl

@@ -366,7 +366,7 @@ sub vcl_synth {
     if (resp.status == 989) {
         set resp.status = 200;
         set resp.http.Content-Type = "application/json";
-        synthetic ( {"{ "vcl_version" : "d584a", "varnish_status": "disabled" }"} );
+        synthetic ( {"{ "vcl_version" : "721d3", "varnish_status": "disabled" }"} );
         return (deliver);
     }
 }
@@ -451,12 +451,6 @@ sub vcl_recv {
         set req.http.x-response-code = "301";
         set req.http.x-action = "redirect";
     }
-    else if (req.url ~ "/external") {
-        set req.http.x-redirect = "3";
-        set req.http.x-destination = "http://example-external.com/external_destination";
-        set req.http.x-response-code = "301";
-        set req.http.x-action = "redirect";
-    }
     }
     if (req.http.host == "example.prod.org") {
     if (req.url ~ "/source") {
@@ -471,12 +465,6 @@ sub vcl_recv {
         set req.http.x-response-code = "301";
         set req.http.x-action = "redirect";
     }
-    else if (req.url ~ "/external") {
-        set req.http.x-redirect = "3";
-        set req.http.x-destination = "http://example-external.com/external_destination";
-        set req.http.x-response-code = "301";
-        set req.http.x-action = "redirect";
-    }
     }
 
 # Test ROUTER

vaas/vaas/vcl/tests/test_renderer.py

@@ -280,7 +280,7 @@ def setUp(self):
         )
 
         Redirect.objects.create(
-            src_domain=self.example_domain_mapping,
+            src_domain=self.external_domain_mapping,
             condition='req.url ~ "/external"',
             destination='http://external.com/external_destination',
             action=301,
@@ -436,23 +436,26 @@ def test_should_decorate_set_backend_tag_with_fallback_service_in_dc1(self):
     def test_should_decorate_flexible_router_tag_with_properly_mapped_destination_domain(self):
         vcl_tag_builder = VclTagBuilder(self.varnish, VclRendererInput())
         tag = vcl_tag_builder.get_expanded_tags('FLEXIBLE_ROUTER').pop()
-        assert_set_equal({'example.prod.com', 'example.prod.org'}, set(tag.parameters['redirects'].keys()))
+        assert_set_equal({'example.prod.com', 'example-external.com', 'example.prod.org'},
+                         set(tag.parameters['redirects'].keys()))
         assert_equals('example.com', tag.parameters['redirects']['example.prod.com'][1].src_domain.domain)
         assert_equals('example.com', tag.parameters['redirects']['example.prod.org'][1].src_domain.domain)
         assert_equals('http://example.prod.com/destination',
                       tag.parameters['redirects']['example.prod.com'][1].destination)
         assert_equals('http://example.prod.org/destination',
                       tag.parameters['redirects']['example.prod.org'][1].destination)
         assert_equals('http://example-external.com/external_destination',
-                      tag.parameters['redirects']['example.prod.com'][2].destination)
+                      tag.parameters['redirects']['example-external.com'][0].destination)
 
     def test_should_sort_redirects_by_priority(self):
         vcl_tag_builder = VclTagBuilder(self.varnish, VclRendererInput())
         tag = vcl_tag_builder.get_expanded_tags('FLEXIBLE_ROUTER').pop()
-        assert_set_equal({'example.prod.com', 'example.prod.org'}, set(tag.parameters['redirects'].keys()))
+        assert_set_equal({'example.prod.com', 'example-external.com', 'example.prod.org'},
+                         set(tag.parameters['redirects'].keys()))
         assert_equals('2/example.prod.com', tag.parameters['redirects']['example.prod.com'][0].id)
         assert_equals('1/example.prod.com', tag.parameters['redirects']['example.prod.com'][1].id)
-        assert_equals('3/example.prod.com', tag.parameters['redirects']['example.prod.com'][2].id)
+        assert_equals('2/example.prod.org', tag.parameters['redirects']['example.prod.org'][0].id)
+        assert_equals('3/example-external.com', tag.parameters['redirects']['example-external.com'][0].id)
 
 
 class VclRendererInputTest(TestCase):