From 34b2b66b693c0919e4075e358e5eef8e2dfad2a9 Mon Sep 17 00:00:00 2001 From: Maxim Vafin Date: Tue, 12 Dec 2023 22:25:27 +0100 Subject: [PATCH] [PT FE] Report unsupported operations with model list (#21607) * [PT FE] Report unsupported operations with model list * Continue on error * Disable more models * Check if file exists * continue of error * Fix on failure * Fix file path * Apply suggestions from code review --- .../workflows/job_pytorch_models_tests.yml | 8 ++++- .../models_hub_common/test_convert_model.py | 1 + .../torch_tests/scripts/process_op_report.py | 27 +++++++++++++++++ .../torch_tests/torch_utils.py | 30 +++++++++++++++++-- 4 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 tests/model_hub_tests/torch_tests/scripts/process_op_report.py diff --git a/.github/workflows/job_pytorch_models_tests.yml b/.github/workflows/job_pytorch_models_tests.yml index 1640bea02c413b..8904b2212e9e33 100644 --- a/.github/workflows/job_pytorch_models_tests.yml +++ b/.github/workflows/job_pytorch_models_tests.yml @@ -110,7 +110,13 @@ jobs: TYPE: ${{ inputs.event == 'schedule' && 'nightly' || 'precommit'}} TEST_DEVICE: CPU USE_SYSTEM_CACHE: False + OP_REPORT_FILE: ${{ env.INSTALL_TEST_DIR }}/TEST-torch_unsupported_ops.log + - name: Reformat unsupported ops file + if: '!cancelled()' + run: | + python3 ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/scripts/process_op_report.py ${INSTALL_TEST_DIR}/TEST-torch_unsupported_ops.log + - name: Available storage after tests run: | echo "Available storage:" @@ -122,5 +128,5 @@ jobs: with: name: test-results-torch-models path: | - ${{ env.INSTALL_TEST_DIR }}/TEST*.html + ${{ env.INSTALL_TEST_DIR }}/TEST-torch* if-no-files-found: 'error' diff --git a/tests/model_hub_tests/models_hub_common/test_convert_model.py b/tests/model_hub_tests/models_hub_common/test_convert_model.py index 68e58a0658defc..ad09380daeb212 100644 --- a/tests/model_hub_tests/models_hub_common/test_convert_model.py +++ b/tests/model_hub_tests/models_hub_common/test_convert_model.py @@ -87,6 +87,7 @@ def teardown_method(self): gc.collect() def _run(self, model_name, model_link, ie_device): + self.model_name = model_name print("Load the model {} (url: {})".format(model_name, model_link)) fw_model = self.load_model(model_name, model_link) print("Retrieve inputs info") diff --git a/tests/model_hub_tests/torch_tests/scripts/process_op_report.py b/tests/model_hub_tests/torch_tests/scripts/process_op_report.py new file mode 100644 index 00000000000000..c9336c606b6858 --- /dev/null +++ b/tests/model_hub_tests/torch_tests/scripts/process_op_report.py @@ -0,0 +1,27 @@ +# Copyright (C) 2018-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +if __name__ == "__main__": + assert len(sys.argv) > 1, "Please provide filename" + filename = sys.argv[1] + if os.path.isfile(filename): + ops_dict = dict() + with open(filename, 'r') as f: + for line in f.readlines(): + r = line.split() + if r[0] in ops_dict: + ops_dict[r[0]].append(r[1]) + else: + ops_dict[r[0]] = [r[1]] + + with open(filename, 'w') as f: + for op in sorted(ops_dict.keys()): + models = ops_dict[op] + m_str = ', '.join(models) + f.write( + f"{op:<30} appears in {len(models):>2} models: {m_str}\n") + else: + print(f"File {filename} doesn't exist.") diff --git a/tests/model_hub_tests/torch_tests/torch_utils.py b/tests/model_hub_tests/torch_tests/torch_utils.py index e726068feff64d..d92462efaf6521 100644 --- a/tests/model_hub_tests/torch_tests/torch_utils.py +++ b/tests/model_hub_tests/torch_tests/torch_utils.py @@ -3,6 +3,7 @@ import pytest import torch +import os from models_hub_common.test_convert_model import TestConvertModel from models_hub_common.utils import get_models_list from openvino import convert_model @@ -27,10 +28,22 @@ def flattenize_structure(outputs): def process_pytest_marks(filepath: str): return [ - pytest.param(n, marks=pytest.mark.xfail(reason=r) if m == "xfail" else pytest.mark.skip(reason=r)) if m else n + pytest.param(n, marks=pytest.mark.xfail(reason=r) if m == + "xfail" else pytest.mark.skip(reason=r)) if m else n for n, _, m, r in get_models_list(filepath)] +def extract_unsupported_ops_from_exception(e: str) -> list: + exception_str = "No conversion rule found for operations:" + for s in e.splitlines(): + it = s.find(exception_str) + if it >= 0: + _s = s[it + len(exception_str):] + ops = _s.replace(" ", "").split(",") + return ops + return [] + + class TestTorchConvertModel(TestConvertModel): def setup_class(self): torch.set_grad_enabled(False) @@ -49,8 +62,19 @@ def prepare_inputs(self, inputs_info): return [i.numpy() for i in inputs] def convert_model(self, model_obj): - ov_model = convert_model( - model_obj, example_input=self.example, verbose=True) + try: + ov_model = convert_model( + model_obj, example_input=self.example, verbose=True) + except Exception as e: + report_filename = os.environ.get("OP_REPORT_FILE", None) + if report_filename: + mode = 'a' if os.path.exists(report_filename) else 'w' + with open(report_filename, mode) as f: + ops = extract_unsupported_ops_from_exception(str(e)) + if ops: + ops = [f"{op} {self.model_name}" for op in ops] + f.write("\n".join(ops) + "\n") + raise e return ov_model def infer_fw_model(self, model_obj, inputs):