PRESUBMIT.py

# Copyright 2013 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""web_tests/ presubmit script for Blink.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for more details about the presubmit API built into gcl.
"""

import filecmp
import inspect
import os
import sys
import tempfile
import re
from html.parser import HTMLParser
from typing import List


def _CheckTestharnessWdspecResults(input_api, output_api):
    """Checks for all-PASS generic baselines for testharness/wdspec tests.

    These files are unnecessary because for testharness/wdspec tests, if there is no
    baseline file then the test is considered to pass when the output is all
    PASS. Note that only generic baselines are checked because platform specific
    and virtual baselines might be needed to prevent fallback.
    """
    baseline_files = _TxtGenericBaselinesToCheck(input_api)
    if not baseline_files:
        return []

    checker_path = input_api.os_path.join(input_api.PresubmitLocalPath(), '..',
                                          'tools', 'check_expected_pass.py')

    # When running git cl presubmit --all this presubmit may be asked to check
    # ~19,000 files. Passing these on the command line would far exceed Windows
    # limits, so we use --path-files instead.

    # We have to set delete=False and then let the object go out of scope so
    # that the file can be opened by name on Windows.
    with tempfile.NamedTemporaryFile('w+', newline='', delete=False) as f:
        for path in baseline_files:
            f.write('%s\n' % path)
        paths_name = f.name

    args = [
        input_api.python3_executable, checker_path, '--path-files', paths_name
    ]
    _, errs = input_api.subprocess.Popen(
        args,
        stdout=input_api.subprocess.PIPE,
        stderr=input_api.subprocess.PIPE,
        universal_newlines=True).communicate()

    os.remove(paths_name)
    if errs:
        return [output_api.PresubmitError(errs)]
    return []


def _TxtGenericBaselinesToCheck(input_api):
    """Returns a list of paths of generic baselines for testharness/wdspec tests."""
    baseline_files = []
    this_dir = input_api.PresubmitLocalPath()
    for f in input_api.AffectedFiles():
        if f.Action() == 'D':
            continue
        path = f.AbsoluteLocalPath()
        if not path.endswith('-expected.txt'):
            continue
        if (input_api.os_path.join(this_dir, 'platform') in path
                or input_api.os_path.join(this_dir, 'virtual') in path
                or input_api.os_path.join(this_dir, 'flag-specific') in path):
            continue
        baseline_files.append(path)
    return baseline_files


def _CheckFilesUsingEventSender(input_api, output_api):
    """Check if any new layout tests still use eventSender. If they do, we encourage replacing them with
       chrome.gpuBenchmarking.pointerActionSequence.
    """
    results = []
    actions = ["eventSender.touch", "eventSender.mouse", "eventSender.gesture"]
    for f in input_api.AffectedFiles():
        if f.Action() == 'A':
            for line_num, line in f.ChangedContents():
                if line.find("eventSender.beginDragWithFiles") != -1:
                    break
                if any(action in line for action in actions):
                    results.append(output_api.PresubmitPromptWarning(
                        'eventSender is deprecated, please use chrome.gpuBenchmarking.pointerActionSequence instead ' +
                        '(see https://crbug.com/711340 and http://goo.gl/BND75q).\n' +
                        'Files: %s:%d %s ' % (f.LocalPath(), line_num, line)))
    return results


def _CheckTestExpectations(input_api, output_api):
    results = []
    os_path = input_api.os_path
    sys.path.append(
        os_path.join(
            os_path.dirname(
                os_path.abspath(inspect.getfile(_CheckTestExpectations))),
                '..', 'tools'))
    from blinkpy.presubmit.lint_test_expectations import (
        PresubmitCheckTestExpectations)
    results.extend(PresubmitCheckTestExpectations(input_api, output_api))
    return results


def _CheckForRedundantBaselines(input_api, output_api, max_tests: int = 1000):
    tests = _TestsCorrespondingToAffectedBaselines(input_api, max_tests)
    if not tests:
        return []
    elif len(tests) > max_tests:
        return [
            output_api.PresubmitNotifyResult(
                'Too many tests to check for redundant baselines; skipping.',
                items=tests),
        ]
    path_to_blink_tool = input_api.os_path.join(input_api.PresubmitLocalPath(),
                                                input_api.os_path.pardir,
                                                'tools', 'blink_tool.py')
    with input_api.CreateTemporaryFile(mode='w+') as test_name_file:
        for test in tests:
            test_name_file.write(f'{test}\n')
        test_name_file.flush()
        command_args = [
            input_api.python3_executable,
            path_to_blink_tool,
            'optimize-baselines',
            '--no-manifest-update',
            '--check',
            f'--test-name-file={test_name_file.name}',
        ]
        command = input_api.Command(
            name='Checking for redundant affected baselines ...',
            cmd=command_args,
            kwargs={},
            message=output_api.PresubmitPromptWarning,
            python3=True)
        return input_api.RunTests([command])


def _TestsCorrespondingToAffectedBaselines(input_api,
                                           max_tests: int = 1000) -> List[str]:
    sep = input_api.re.escape(input_api.os_path.sep)
    baseline_pattern = input_api.re.compile(
        r'((platform|flag-specific)%s[^%s]+%s)?(virtual%s[^%s]+%s)?'
        r'(?P<test_prefix>.*)-expected\.(txt|png|wav)' % ((sep, ) * 6))
    test_paths = set()
    for affected_file in input_api.AffectedFiles():
        if len(test_paths) > max_tests:
            # Exit early; no need to glob for more tests.
            break
        baseline_path_from_web_tests = input_api.os_path.relpath(
            affected_file.AbsoluteLocalPath(), input_api.PresubmitLocalPath())
        baseline_match = baseline_pattern.fullmatch(
            baseline_path_from_web_tests)
        if not baseline_match:
            continue
        # Baselines for WPT-style variants have sanitized filenames with '?' and
        # '&' in the query parameter section converted to '_', like:
        #   a/b.html?c&d -> a/b_c_d-expected.txt
        #
        # Regrettably, this is a lossy coercion that cannot be distinguished
        # from tests with '_' in the original test ID, like:
        #   a/b_c_d.html -> a/b_c_d-expected.txt
        #
        # Here, we err on the side of not attempting to check such tests, which
        # could be unrelated.
        test_prefix = baseline_match['test_prefix']
        # Getting the test name from the baseline path is not as easy as the
        # other direction. Try all extensions as a heuristic instead.
        for extension in [
                'html', 'xml', 'xhtml', 'xht', 'pl', 'htm', 'php', 'svg',
                'mht', 'pdf', 'js'
        ]:
            abs_prefix = input_api.os_path.join(input_api.PresubmitLocalPath(),
                                                test_prefix)
            test_paths.update(input_api.glob(f'{abs_prefix}*.{extension}'))
    return [
        input_api.os_path.relpath(test_path, input_api.PresubmitLocalPath())
        for test_path in sorted(test_paths)
    ]


def _CheckForJSTest(input_api, output_api):
    """'js-test.js' is the past, 'testharness.js' is our glorious future"""
    jstest_re = input_api.re.compile(r'resources/js-test.js')

    def source_file_filter(path):
        return input_api.FilterSourceFile(path, files_to_check=[r'\.(html|js|php|pl|svg)$'])

    errors = input_api.canned_checks._FindNewViolationsOfRule(
        lambda _, x: not jstest_re.search(x), input_api, source_file_filter)
    errors = ['  * %s' % violation for violation in errors]
    if errors:
        return [output_api.PresubmitPromptOrNotify(
            '"resources/js-test.js" is deprecated; please write new layout '
            'tests using the assertions in "resources/testharness.js" '
            'instead, as these can be more easily upstreamed to Web Platform '
            'Tests for cross-vendor compatibility testing. If you\'re not '
            'already familiar with this framework, a tutorial is available at '
            'https://darobin.github.io/test-harness-tutorial/docs/using-testharness.html'
            '\n\n%s' % '\n'.join(errors))]
    return []


def _CheckForInvalidPreferenceError(input_api, output_api):
    pattern = input_api.re.compile('Invalid name for preference: (.+)')
    results = []

    for f in input_api.AffectedFiles():
        if not f.LocalPath().endswith('-expected.txt'):
            continue
        for line_num, line in f.ChangedContents():
            error = pattern.search(line)
            if error:
                results.append(output_api.PresubmitError('Found an invalid preference %s in expected result %s:%s' % (error.group(1), f, line_num)))
    return results


def _CheckRunAfterLayoutAndPaintJS(input_api, output_api):
    """Checks if resources/run-after-layout-and-paint.js and
       http/tests/resources/run-after-layout-and-paint.js are the same."""
    js_file = input_api.os_path.join(input_api.PresubmitLocalPath(),
        'resources', 'run-after-layout-and-paint.js')
    http_tests_js_file = input_api.os_path.join(input_api.PresubmitLocalPath(),
        'http', 'tests', 'resources', 'run-after-layout-and-paint.js')
    for f in input_api.AffectedFiles():
        path = f.AbsoluteLocalPath()
        if path == js_file or path == http_tests_js_file:
            if not filecmp.cmp(js_file, http_tests_js_file):
                return [output_api.PresubmitError(
                    '%s and %s must be kept exactly the same' %
                    (js_file, http_tests_js_file))]
            break
    return []


def _CheckForUnlistedTestFolder(input_api, output_api):
    """Checks all the test folders under web_tests are listed in BUILD.gn.
    """
    this_dir = input_api.PresubmitLocalPath()
    possible_new_dirs = set()
    for f in input_api.AffectedFiles():
        if f.Action() == 'A':
            # We only check added folders. For deleted folders, if BUILD.gn is
            # not updated, the build will fail at upload step. The reason is that
            # we can not know if the folder is deleted as there can be local
            # unchecked in files.
            path = f.AbsoluteLocalPath()
            fns = path[len(this_dir)+1:].split('/')
            if len(fns) > 1:
                possible_new_dirs.add(fns[0])

    if possible_new_dirs:
        path_build_gn = input_api.os_path.join(input_api.change.RepositoryRoot(), 'BUILD.gn')
        dirs_from_build_gn = []
        start_line = '# === List Test Cases folders here ==='
        end_line = '# === Test Case Folders Ends ==='
        end_line_count = 0
        find_start_line  = False
        for line in input_api.ReadFile(path_build_gn).splitlines():
            line = line.strip()
            if line.startswith(start_line):
                find_start_line = True
                continue
            if find_start_line:
                if line.startswith(end_line):
                    find_start_line = False
                    end_line_count += 1
                    if end_line_count == 2:
                        break
                    continue
                if len(line.split('/')) > 1:
                    dirs_from_build_gn.append(line.split('/')[-2])
        dirs_from_build_gn.extend(
            ['platform', 'FlagExpectations', 'flag-specific', 'TestLists'])

        new_dirs = [x for x in possible_new_dirs if x not in dirs_from_build_gn]
        if new_dirs:
            dir_plural = "directories" if len(new_dirs) > 1 else "directory"
            error_message = (
                'This CL adds new %s(%s) under //third_party/blink/web_tests/, but //BUILD.gn '
                'is not updated. Please add the %s to BUILD.gn.' % (dir_plural, ', '.join(new_dirs), dir_plural))
            if input_api.is_committing:
                return [output_api.PresubmitError(error_message)]
            else:
                return [output_api.PresubmitPromptWarning(error_message)]
    return []


def _CheckForExtraVirtualBaselines(input_api, output_api):
    """Checks that expectations in virtual test suites are for virtual test suites that exist
    """
    os_path = input_api.os_path

    local_dir = os_path.relpath(
        os_path.normpath('{0}/'.format(input_api.PresubmitLocalPath().replace(
            os_path.sep, '/'))), input_api.change.RepositoryRoot())

    check_all = False
    check_files = []
    for f in input_api.AffectedFiles(include_deletes=False):
        local_path = f.LocalPath()
        assert local_path.startswith(local_dir)
        local_path = os_path.relpath(local_path, local_dir)
        path_components = local_path.split(os_path.sep)
        if f.Action() == 'A':
            if len(path_components) > 2 and path_components[0] == 'virtual':
                check_files.append((local_path, path_components[1]))
            elif (len(path_components) > 4 and path_components[2] == 'virtual'
                  and (path_components[0] == 'platform'
                       or path_components[0] == 'flag-specific')):
                check_files.append((local_path, path_components[3]))
        elif local_path == 'VirtualTestSuites':
            check_all = True

    if not check_all and len(check_files) == 0:
        return []

    # The rest of this test fails on Windows because win32pipe is not available
    # and other errors.
    if os.name == 'nt':
        return []

    from blinkpy.common.host import Host
    port_factory = Host().port_factory
    known_virtual_suites = [
        suite.full_prefix[8:-1] for suite in port_factory.get(
            port_factory.all_port_names()[0]).virtual_test_suites()
    ]

    results = []
    if check_all:
        for f in input_api.change.AllFiles(
                os_path.join(input_api.PresubmitLocalPath(), "virtual")):
            suite = f.split('/')[0]
            if not suite in known_virtual_suites:
                path = os_path.relpath(
                    os_path.join(input_api.PresubmitLocalPath(), "virtual", f),
                    input_api.change.RepositoryRoot())
                results.append(
                    output_api.PresubmitError(
                        "Baseline %s exists, but %s is not a known virtual test suite."
                        % (path, suite)))
        for subdir in ["platform", "flag-specific"]:
            for f in input_api.change.AllFiles(
                    os_path.join(input_api.PresubmitLocalPath(), subdir)):
                path_components = f.split('/')
                if len(path_components) < 3 or path_components[1] != 'virtual':
                    continue
                suite = path_components[2]
                if not suite in known_virtual_suites:
                    path = os_path.relpath(
                        os_path.join(input_api.PresubmitLocalPath(), subdir,
                                     f), input_api.change.RepositoryRoot())
                    results.append(
                        output_api.PresubmitError(
                            "Baseline %s exists, but %s is not a known virtual test suite."
                            % (path, suite)))
    else:
        for (f, suite) in check_files:
            if not suite in known_virtual_suites:
                path = os_path.relpath(
                    os_path.join(input_api.PresubmitLocalPath(), f),
                    input_api.change.RepositoryRoot())
                results.append(
                    output_api.PresubmitError(
                        "This CL adds a new baseline %s, but %s is not a known virtual test suite."
                        % (path, suite)))
    return results


def _CheckWebViewExpectations(input_api, output_api):
    src_dir = os.path.join(input_api.PresubmitLocalPath(), os.pardir,
                           os.pardir, os.pardir)
    webview_data_dir = input_api.os_path.join(src_dir, 'android_webview',
                                              'tools', 'system_webview_shell',
                                              'test', 'data', 'webexposed')
    if webview_data_dir not in sys.path:
        sys.path.append(webview_data_dir)

    # pylint: disable=import-outside-toplevel
    from exposed_webview_interfaces_presubmit import (
        CheckNotWebViewExposedInterfaces)
    return CheckNotWebViewExposedInterfaces(input_api, output_api)


class _DoctypeParser(HTMLParser):
    """Parses HTML to check if there exists a DOCTYPE declaration before all other tags.
    """

    def __init__(self):
        super().__init__()
        self.encountered_tag = False
        self.doctype = ""

    def handle_starttag(self, *_):
        self.encountered_tag = True

    def handle_startendtag(self, *_):
        self.encountered_tag = True

    def handle_decl(self, decl):
        if not self.encountered_tag:
            self.doctype = decl
            self.encountered_tag = True


def _IsDoctypeHTMLSet(lines):
    """Returns true if the given HTML file starts with <!DOCTYPE html>.
    """
    parser = _DoctypeParser()
    for l in lines:
        parser.feed(l)

    return re.match("DOCTYPE\s*html\s*$", parser.doctype, re.IGNORECASE)


def _CheckForDoctypeHTML(input_api, output_api):
    """Checks that all changed HTML files start with the correct <!DOCTYPE html> tag.
    """
    results = []

    if input_api.no_diffs:
        return results

    wpt_path = input_api.os_path.join(input_api.PresubmitLocalPath(),
                                      "external", "wpt")

    for f in input_api.AffectedFiles(include_deletes=False):
        path = f.LocalPath()
        fname = input_api.os_path.basename(path)

        if not fname.endswith(".html") or "quirk" in fname:
            continue

        if not _IsDoctypeHTMLSet(f.NewContents()):
            error = "HTML file \"%s\" does not start with <!DOCTYPE html>. " \
                    "If you really intend to test in quirks mode, add \"quirk\" " \
                    "to the name of your test." % path

            if f.Action() == "A" or _IsDoctypeHTMLSet(f.OldContents()):
                # These tests are being imported from WPT, so <!DOCTYPE html> is
                # not required yet.
                no_errors = f.AbsoluteLocalPath().startswith(wpt_path)
                if no_errors:
                    results.append(output_api.PresubmitPromptWarning(error))
                else:
                    results.append(output_api.PresubmitError(error))

    return results


def _CheckNewVirtualSuites(input_api, output_api, max_suite_length: int = 48):
    """Validate new virtual test suites."""
    # TODO(crbug.com/1380165): Once all virtual suites adopt "owners", consider
    # making the field mandatory. In that case, we don't need to access the
    # change contents and can promote this check to `lint_test_expectations.py`.
    vts_path = input_api.os_path.join(input_api.PresubmitLocalPath(),
                                      'VirtualTestSuites')
    results = []
    for affected_file in input_api.AffectedFiles():
        if affected_file.AbsoluteLocalPath() != vts_path:
            continue
        old_contents = ''.join(affected_file.OldContents())
        new_contents = ''.join(affected_file.NewContents())
        try:
            old_suites = _FilterForSuites(input_api.json.loads(old_contents))
            new_suites = _FilterForSuites(input_api.json.loads(new_contents))
            old_suite_names = {suite['prefix'] for suite in old_suites}
            new_ownerless_suites, new_long_suites = [], []
            for suite in new_suites:
                prefix, owners = suite['prefix'], suite.get('owners', [])
                if prefix in old_suite_names:
                    continue
                if not owners:
                    new_ownerless_suites.append(prefix)
                if len(prefix) > max_suite_length:
                    new_long_suites.append(prefix)
            if new_ownerless_suites:
                results.append(
                    output_api.PresubmitPromptWarning(
                        'Consider specifying "owners" (a list of emails) '
                        'for the virtual suites added by this patch:',
                        new_ownerless_suites))
            if new_long_suites:
                results.append(
                    output_api.PresubmitPromptWarning(
                        'Consider shorter virtual suite names so that the '
                        "global filename length presubmit doesn't reject "
                        'future `*-expected.txt` under their directories. You '
                        'can add comments about these suites to '
                        'VirtualTestSuites.', new_long_suites))
        except (ValueError, KeyError):
            # Invalid JSON or missing required fields will be detected by
            # `lint_test_expectations.py`.
            pass
        break
    return results


def _FilterForSuites(suites):
    return [suite for suite in suites if not isinstance(suite, str)]


def CheckChangeOnUpload(input_api, output_api):
    results = []
    results.extend(_CheckTestharnessWdspecResults(input_api, output_api))
    results.extend(_CheckFilesUsingEventSender(input_api, output_api))
    results.extend(_CheckTestExpectations(input_api, output_api))
    # `_CheckTestExpectations()` updates the WPT manifests for
    # `_CheckForRedundantBaselines()`, so they must run in order. (Updating the
    # manifest is needed to correctly detect tests but takes 10-15s, so try
    # to only do so once; see crbug.com/1492238.)
    results.extend(_CheckForRedundantBaselines(input_api, output_api))
    results.extend(_CheckForJSTest(input_api, output_api))
    results.extend(_CheckForInvalidPreferenceError(input_api, output_api))
    results.extend(_CheckRunAfterLayoutAndPaintJS(input_api, output_api))
    results.extend(_CheckForUnlistedTestFolder(input_api, output_api))
    results.extend(_CheckForExtraVirtualBaselines(input_api, output_api))
    results.extend(_CheckWebViewExpectations(input_api, output_api))
    results.extend(_CheckForDoctypeHTML(input_api, output_api))
    results.extend(_CheckNewVirtualSuites(input_api, output_api))
    return results


def CheckChangeOnCommit(input_api, output_api):
    results = []
    results.extend(_CheckTestharnessWdspecResults(input_api, output_api))
    results.extend(_CheckFilesUsingEventSender(input_api, output_api))
    results.extend(_CheckTestExpectations(input_api, output_api))
    # `_CheckTestExpectations()` updates the WPT manifests for
    # `_CheckForRedundantBaselines()`, so they must run in order. (Updating the
    # manifest is needed to correctly detect tests but takes 10-15s, so try
    # to only do so once; see crbug.com/1492238.)
    results.extend(_CheckForRedundantBaselines(input_api, output_api))
    results.extend(_CheckForUnlistedTestFolder(input_api, output_api))
    results.extend(_CheckForExtraVirtualBaselines(input_api, output_api))
    results.extend(_CheckWebViewExpectations(input_api, output_api))
    results.extend(_CheckForDoctypeHTML(input_api, output_api))
    results.extend(_CheckNewVirtualSuites(input_api, output_api))
    return results