diff --git a/squad/core/comparison.py b/squad/core/comparison.py index f77d402b9..8c49e48e6 100644 --- a/squad/core/comparison.py +++ b/squad/core/comparison.py @@ -1,8 +1,9 @@ -from collections import OrderedDict +from collections import OrderedDict, defaultdict from django.db.models import F from itertools import groupby from functools import reduce import statistics +import time from squad.core.utils import parse_name, join_name, split_dict @@ -43,11 +44,14 @@ class BaseComparison(object): (name, env) """ - def __init__(self, *builds): + def __init__(self, *builds, suites=None, offset=0, per_page=50): self.builds = list(builds) self.environments = OrderedDict() self.all_environments = set() self.results = OrderedDict() + self.suites = suites + self.offset = offset + self.per_page = per_page for build in self.builds: self.environments[build] = set() @@ -164,13 +168,43 @@ class TestComparison(BaseComparison): __test__ = False - def __init__(self, *builds): + def __init__(self, *builds, suites=None, offset=0, per_page=50): self.__intermittent__ = {} self.tests_with_issues = {} self.__failures__ = OrderedDict() - BaseComparison.__init__(self, *builds) + BaseComparison.__init__(self, *builds, suites=suites, offset=offset, per_page=per_page) def __extract_results__(self): + + """ + + problem: + - comparing takes a huge amount of memory and time + - it does this because it blindly takes all tests from + the first build and try to find a matching test in the + second/target build + - after tests match, transitions are applied (pass to fail, fail to pass, n/a to pass, etc) + - after transitions are applied we paginate the whole thing just + to display a really small portion of it + + idea: + - i think i can improve it by running smaller comparisons by + comparing by suite, and then paginate tests in database + - compare results by suite means reducing number of testruns + and also reducing number of returned tests since we'd be using + suite.id + - if filtering tests by suite.id and a very small number of testruns + we'll be dealing with only a few million tests, which DB handles just fine + so I think it's ok to apply order by, offset and limit :) + - results look weird, but promising :) + - keep trying tomorrow! + + + """ + + start = time.time() + print('fetching testruns') + test_runs = models.TestRun.objects.filter( build__in=self.builds, ).prefetch_related( @@ -178,8 +212,31 @@ def __extract_results__(self): 'environment', ).only('build', 'environment') + statuses = models.Status.objects.filter(test_run__in=test_runs, suite__isnull=False) + if self.suites: + statuses = statuses.filter(suite__slug__in=self.suites) + + # Group testruns by suite.slug + partial_results = defaultdict(lambda: []) + test_runs = [] + for status in statuses.all(): + partial_results[status.suite.slug].append((status.suite, status.test_run)) + test_runs.append(status.test_run) + + results = defaultdict(lambda: []) + for suite_slug in partial_results.keys(): + results[suite_slug] = self.__extract_suite_results__(suite_slug, partial_results[suite_slug]) + + self.__resolve_intermittent_tests__() + + self.results = OrderedDict(sorted(self.results.items())) + for build in self.builds: + self.environments[build] = sorted(self.environments[build]) + + def __extract_suite_results__(self, suite_slug, suites_and_testruns): + test_runs_ids = {} - for test_run in test_runs: + for suite, test_run in suites_and_testruns: build = test_run.build env = test_run.environment.slug @@ -189,24 +246,29 @@ def __extract_results__(self): if test_runs_ids.get(test_run.id, None) is None: test_runs_ids[test_run.id] = (build, env) - for ids in split_dict(test_runs_ids, chunk_size=100): - self.__extract_test_results__(ids) + print('splitting tests') + start = time.time() - self.__resolve_intermittent_tests__() + self.__extract_test_results__(test_runs_ids, suite) - self.results = OrderedDict(sorted(self.results.items())) - for build in self.builds: - self.environments[build] = sorted(self.environments[build]) + duration = time.time() - start + print('finish splitting tests! took %f' % duration) - def __extract_test_results__(self, test_runs_ids): - tests = models.Test.objects.filter(test_run_id__in=test_runs_ids.keys()).annotate( - suite_slug=F('suite__slug'), - ).prefetch_related('metadata').defer('log') + def __extract_test_results__(self, test_runs_ids, suite): + print('\tfetching tests') + start = time.time() + + tests = models.Test.objects.filter(test_run_id__in=test_runs_ids.keys(), suite=suite) \ + .prefetch_related('metadata') \ + .defer('log') \ + .order_by('metadata__name') \ + .all()[self.offset:self.offset+self.per_page] for test in tests: + print('.', end='', flush=True) build, env = test_runs_ids.get(test.test_run_id) - full_name = join_name(test.suite_slug, test.name) + full_name = join_name(suite.slug, test.name) if full_name not in self.results: self.results[full_name] = OrderedDict() @@ -221,6 +283,9 @@ def __extract_test_results__(self, test_runs_ids): self.__failures__[env] = [] self.__failures__[env].append(test) + duration = time.time() - start + print('\tfinish fetching tests! took %f' % duration) + def __resolve_intermittent_tests__(self): if len(self.tests_with_issues) == 0: return diff --git a/squad/frontend/comparison.py b/squad/frontend/comparison.py index 9ee971dd4..4430eb56d 100644 --- a/squad/frontend/comparison.py +++ b/squad/frontend/comparison.py @@ -101,6 +101,9 @@ def compare_test(request): def compare_builds(request): + import time + outer_start = time.time() + print('comparing builds') project_slug = request.GET.get('project') comparison_type = request.GET.get('comparison_type', 'test') transitions = __get_transitions(request) @@ -112,15 +115,24 @@ def compare_builds(request): baseline_build = request.GET.get('baseline') target_build = request.GET.get('target') + print('comparing %s against %s' % (baseline_build, target_build)) if baseline_build and target_build: baseline = get_object_or_404(project.builds, version=baseline_build) target = get_object_or_404(project.builds, version=target_build) comparison_class = __get_comparison_class(comparison_type) - comparison = comparison_class.compare_builds(baseline, target) - + start = time.time() + print('starting comparison') + comparison = comparison_class(baseline, target, suites=['cts-lkft/arm64-v8a.CtsDeqpTestCases']) + duration = time.time() - start + print('finished comparison! took %f' % duration, flush=True) + + start = time.time() + print('applying transitions') if comparison_type == 'test' and len(transitions): comparison.apply_transitions([t for t, checked in transitions.items() if checked]) + duration = time.time() - start + print('finished applying transitions! took %f' % duration, flush=True) comparison.results = __paginate(comparison.results, request) @@ -131,4 +143,8 @@ def compare_builds(request): 'transitions': transitions, } - return render(request, 'squad/compare_builds.jinja2', context) + response = render(request, 'squad/compare_builds.jinja2', context) + + duration = time.time() - outer_start + print('finished comparing builds! took %f' % duration, flush=True) + return response diff --git a/squad/settings.py b/squad/settings.py index 778ece149..93688ca72 100644 --- a/squad/settings.py +++ b/squad/settings.py @@ -64,7 +64,7 @@ django_toolbar = None django_toolbar_middleware = None -if DEBUG: +if DEBUG and False: try: DEBUG_TOOLBAR_CONFIG = {