diff --git a/docker/test/util/process_functional_tests_result.py b/docker/test/util/process_functional_tests_result.py index fd4cc9f4bf76..30cba0d6690d 100755 --- a/docker/test/util/process_functional_tests_result.py +++ b/docker/test/util/process_functional_tests_result.py @@ -4,6 +4,7 @@ import logging import argparse import csv +import json OK_SIGN = "[ OK " FAIL_SIGN = "[ FAIL " @@ -206,6 +207,7 @@ def write_results(results_file, status_file, results, status): parser.add_argument("--out-results-file", default="/test_output/test_results.tsv") parser.add_argument("--out-status-file", default="/test_output/check_status.tsv") parser.add_argument("--broken-tests", default="/analyzer_tech_debt.txt") + parser.add_argument("--broken-tests-json", default="/broken_tests.json") args = parser.parse_args() broken_tests = list() @@ -213,6 +215,14 @@ def write_results(results_file, status_file, results, status): logging.info(f"File {args.broken_tests} with broken tests found") with open(args.broken_tests) as f: broken_tests = f.read().splitlines() + + if os.path.exists(args.broken_tests_json): + logging.info(f"File {args.broken_tests_json} with broken tests found") + + with open(args.broken_tests_json) as f: + broken_tests.extend(json.load(f).keys()) + + if broken_tests: logging.info(f"Broken tests in the list: {len(broken_tests)}") state, description, test_results = process_result(args.in_results_dir, broken_tests) diff --git a/tests/broken_tests.json b/tests/broken_tests.json new file mode 100644 index 000000000000..b0d374916c12 --- /dev/null +++ b/tests/broken_tests.json @@ -0,0 +1,22 @@ +{ + "test_postgresql_replica_database_engine_2/test.py::test_quoting_publication": { + "message": "DB::Exception: Syntax error:", + "reason": "syntax error" + }, + "test_distributed_inter_server_secret/test.py::test_secure_cluster_distributed_over_distributed_different_users": { + "message": "DB::NetException: Connection reset by peer, while reading from socket", + "reason": "network issue" + }, + "02920_alter_column_of_projections": { + "reason": "requires different settings" + }, + "02888_system_tables_with_inaccsessible_table_function": { + "reason": "todo investigate" + }, + "03094_grouparraysorted_memory": { + "reason": "flaky" + }, + "02700_s3_part_INT_MAX": { + "reason": "fails with asan" + } +} \ No newline at end of file diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 8f1ffb05ac3c..4cd022c6bf81 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -97,6 +97,9 @@ def get_run_command( if "analyzer" not in check_name else "" ) + volume_with_broken_test += ( + f"--volume={repo_path}/tests/broken_tests.json:/broken_tests.json " + ) return ( f"docker run --volume={builds_path}:/package_folder " diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py index f10c0a190e2f..18725a509095 100755 --- a/tests/ci/integration_tests_runner.py +++ b/tests/ci/integration_tests_runner.py @@ -474,6 +474,19 @@ def _get_parallel_tests_skip_list(repo_path): skip_list_tests = json.load(skip_list_file) return list(sorted(skip_list_tests)) + @staticmethod + def _get_broken_tests_list(repo_path: str) -> dict: + skip_list_file_path = f"{repo_path}/tests/broken_tests.json" + if ( + not os.path.isfile(skip_list_file_path) + or os.path.getsize(skip_list_file_path) == 0 + ): + return {} + + with open(skip_list_file_path, "r", encoding="utf-8") as skip_list_file: + skip_list_tests = json.load(skip_list_file) + return skip_list_tests + @staticmethod def group_test_by_file(tests): result = {} # type: Dict @@ -891,6 +904,8 @@ def run_impl(self, repo_path, build_path): " ".join(not_found_tests[:3]), ) + known_broken_tests = self._get_broken_tests_list(repo_path) + grouped_tests = self.group_test_by_file(filtered_sequential_tests) i = 0 for par_group in chunks(filtered_parallel_tests, PARALLEL_GROUP_SIZE): @@ -921,6 +936,26 @@ def run_impl(self, repo_path, build_path): group_counters, group_test_times, log_paths = self.try_run_test_group( repo_path, group, tests, MAX_RETRY, NUM_WORKERS ) + + for fail_status in ("ERROR", "FAILED"): + for failed_test in group_counters[fail_status]: + if failed_test in known_broken_tests.keys(): + fail_message = known_broken_tests[failed_test].get("message") + if not fail_message: + mark_as_broken = True + else: + mark_as_broken = False + for log_path in log_paths: + if log_path.endswith(".log"): + with open(log_path) as log_file: + if fail_message in log_file.read(): + mark_as_broken = True + break + + if mark_as_broken: + group_counters[fail_status].remove(failed_test) + group_counters["BROKEN"].append(failed_test) + total_tests = 0 for counter, value in group_counters.items(): logging.info( diff --git a/tests/integration/compose/docker_compose_hdfs.yml b/tests/integration/compose/docker_compose_hdfs.yml index 1cae54ad9e1a..40a10df01f7c 100644 --- a/tests/integration/compose/docker_compose_hdfs.yml +++ b/tests/integration/compose/docker_compose_hdfs.yml @@ -1,7 +1,7 @@ version: '2.3' services: hdfs1: - image: sequenceiq/hadoop-docker:2.7.0 + image: prasanthj/docker-hadoop:2.6.0 hostname: hdfs1 restart: always expose: