diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml index b91c9ee3e5..021e892e2e 100644 --- a/.github/workflows/pr_stage_test.yml +++ b/.github/workflows/pr_stage_test.yml @@ -98,14 +98,12 @@ jobs: torch: '2.4.1' cuda: 'cu118' image: 'nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04' - - python-version: '3.10' + - python-version: '3.12' torch: '2.8.0' cuda: 'cu129' image: 'nvidia/cuda:12.9.1-cudnn-runtime-ubuntu22.04' container: image: ${{ matrix.image }} - permissions: - pull-requests: write steps: - name: Show disk usage run: df -h @@ -155,12 +153,7 @@ jobs: format: markdown output: both hide_branch_rate: false - - name: Add Coverage PR Comment - uses: marocchino/sticky-pull-request-comment@v2 - if: github.event_name == 'pull_request' - with: - recreate: true - path: code-coverage-results.md + # build_windows: # runs-on: windows-2022 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dc5234058f..ba8038d668 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,3 +65,8 @@ repos: | ^docs ) additional_dependencies: ["types-setuptools", "types-requests", "types-PyYAML"] + - repo: https://github.com/astral-sh/uv-pre-commit + # uv version. + rev: 0.9.5 + hooks: + - id: uv-lock diff --git a/mmengine/config/utils.py b/mmengine/config/utils.py index 001a2267ac..81e53aa637 100644 --- a/mmengine/config/utils.py +++ b/mmengine/config/utils.py @@ -177,6 +177,9 @@ def _is_builtin_module(module_name: str) -> bool: origin_path = getattr(spec, 'origin', None) if origin_path is None: return False + # Handle frozen modules (e.g., os module in some Python distributions) + if origin_path == 'frozen': + return True origin_path = osp.abspath(origin_path) if ('site-package' in origin_path or 'dist-package' in origin_path or not origin_path.startswith( diff --git a/pyproject.toml b/pyproject.toml index 43f92a1ad2..1e04481027 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "onedl-mmengine" -version = "0.10.8" +version = "0.10.9" description = "Engine of VBTI projects" readme = "README.md" authors = [ @@ -14,6 +14,8 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Utilities" ] dependencies = [ @@ -29,7 +31,7 @@ dependencies = [ [dependency-groups] test = [ - "aim<=3.17.5; sys_platform != 'win32'", + "aim; sys_platform != 'win32'", "bitsandbytes", "clearml", "coverage", @@ -77,7 +79,7 @@ all = [ "termcolor", "yapf", "regex; sys_platform == 'win32'", - "aim<=3.17.5; sys_platform != 'win32'", + "aim; sys_platform != 'win32'", "bitsandbytes", "clearml", "coverage", diff --git a/tests/data/config/lazy_module_config/test_ast_transform.py b/tests/data/config/lazy_module_config/test_ast_transform.py index a8803dde24..6f0ada1736 100644 --- a/tests/data/config/lazy_module_config/test_ast_transform.py +++ b/tests/data/config/lazy_module_config/test_ast_transform.py @@ -3,7 +3,7 @@ from importlib.util import find_spec as find_module import numpy -import numpy.compat +import numpy.fft import numpy.linalg as linalg from mmengine.config import Config diff --git a/tests/test_config/test_lazy.py b/tests/test_config/test_lazy.py index d69822814b..78676ed4c5 100644 --- a/tests/test_config/test_lazy.py +++ b/tests/test_config/test_lazy.py @@ -8,7 +8,7 @@ from unittest import TestCase import numpy -import numpy.compat +import numpy.fft import numpy.linalg as linalg from rich.progress import Progress @@ -56,17 +56,17 @@ def test_lazy_module(self): # 1.2 getattr as LazyAttr self.assertIsInstance(lazy_numpy.linalg, LazyAttr) - self.assertIsInstance(lazy_numpy.compat, LazyAttr) + self.assertIsInstance(lazy_numpy.fft, LazyAttr) # 1.3 Build module from LazyObject. amp and functional can be accessed imported_numpy = lazy_numpy.build() self.assertIs(imported_numpy.linalg, linalg) - self.assertIs(imported_numpy.compat, numpy.compat) + self.assertIs(imported_numpy.fft, numpy.fft) # 1.4.1 Build module from LazyAttr imported_linalg = lazy_numpy.linalg.build() - imported_compat = lazy_numpy.compat.build() - self.assertIs(imported_compat, numpy.compat) + imported_compat = lazy_numpy.fft.build() + self.assertIs(imported_compat, numpy.fft) self.assertIs(imported_linalg, linalg) # 1.4.2 build class method from LazyAttr diff --git a/tests/test_fileio/test_backends/test_petrel_backend.py b/tests/test_fileio/test_backends/test_petrel_backend.py index 6f379c3f23..0dbad20853 100644 --- a/tests/test_fileio/test_backends/test_petrel_backend.py +++ b/tests/test_fileio/test_backends/test_petrel_backend.py @@ -1,7 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved. import os import os.path as osp -import sys import tempfile from contextlib import contextmanager from copy import deepcopy @@ -49,54 +48,63 @@ def build_temporary_directory(): yield tmp_dir +# Check if petrel_client is available +PETREL_CLIENT_AVAILABLE = False try: - # Other unit tests may mock these modules so we need to pop them first. - sys.modules.pop('petrel_client', None) - sys.modules.pop('petrel_client.client', None) - - # If petrel_client is imported successfully, we can test PetrelBackend - # without mock. import petrel_client # noqa: F401 -except ImportError: - sys.modules['petrel_client'] = MagicMock() - sys.modules['petrel_client.client'] = MagicMock() - - class MockPetrelClient: - - def __init__(self, - enable_mc=True, - enable_multi_cluster=False, - conf_path=None): - self.enable_mc = enable_mc - self.enable_multi_cluster = enable_multi_cluster - self.conf_path = conf_path - - def Get(self, filepath): - with open(filepath, 'rb') as f: - content = f.read() - return content - - def put(self): - pass + PETREL_CLIENT_AVAILABLE = True +except (ImportError, ModuleNotFoundError): + PETREL_CLIENT_AVAILABLE = False - def delete(self): - pass - def contains(self): - pass +class MockPetrelClient: + """Mock PetrelClient for testing when petrel_client is not available.""" - def isdir(self): - pass + def __init__(self, + enable_mc=True, + enable_multi_cluster=False, + conf_path=None): + self.enable_mc = enable_mc + self.enable_multi_cluster = enable_multi_cluster + self.conf_path = conf_path + + def Get(self, filepath): + with open(filepath, 'rb') as f: + content = f.read() + return content + + def put(self, filepath, content): + pass + + def delete(self, filepath): + pass + + def contains(self, filepath): + pass + + def isdir(self, filepath): + pass + + def list(self, dir_path): + for entry in os.scandir(dir_path): + if entry.name.startswith('.'): + continue + if entry.is_file(): + yield entry.name + elif entry.is_dir(): + yield entry.name + '/' - def list(self, dir_path): - for entry in os.scandir(dir_path): - if not entry.name.startswith('.') and entry.is_file(): - yield entry.name - elif osp.isdir(entry.path): - yield entry.name + '/' - @contextmanager - def delete_and_reset_method(obj, method): +@contextmanager +def delete_and_reset_method(obj, method): + if hasattr(obj, '_mock_methods') or str(type(obj).__name__) == 'MagicMock': + method_obj = deepcopy(getattr(obj, method)) + try: + delattr(obj, method) + yield + finally: + setattr(obj, method, method_obj) + else: method_obj = deepcopy(getattr(type(obj), method)) try: delattr(type(obj), method) @@ -104,7 +112,11 @@ def delete_and_reset_method(obj, method): finally: setattr(type(obj), method, method_obj) - @patch('petrel_client.client.Client', MockPetrelClient) + +if not PETREL_CLIENT_AVAILABLE: + # Define the test class that uses mocking when + # petrel_client is not available + class TestPetrelBackend(TestCase): @classmethod @@ -118,6 +130,24 @@ def setUpClass(cls): cls.expected_dir = 's3://user/data' cls.expected_path = f'{cls.expected_dir}/test.jpg' + def setUp(self): + # Mock petrel_client for each test + self.mock_petrel_client = MagicMock() + self.mock_client_module = MagicMock() + self.mock_client_module.Client = MockPetrelClient + self.mock_petrel_client.client = self.mock_client_module + + self.patcher_petrel = patch.dict( + 'sys.modules', { + 'petrel_client': self.mock_petrel_client, + 'petrel_client.client': self.mock_client_module + }) + self.patcher_petrel.start() + + def tearDown(self): + # Clean up the mock + self.patcher_petrel.stop() + def test_name(self): backend = PetrelBackend() self.assertEqual(backend.name, 'PetrelBackend') @@ -563,6 +593,7 @@ def test_generate_presigned_url(self): pass else: + # Define the test class that uses real petrel_client when available class TestPetrelBackend(TestCase): # type: ignore diff --git a/tests/test_fileio/test_fileclient.py b/tests/test_fileio/test_fileclient.py index 5f71379c23..af7b7b47e5 100644 --- a/tests/test_fileio/test_fileclient.py +++ b/tests/test_fileio/test_fileclient.py @@ -16,8 +16,6 @@ from mmengine.utils import has_method sys.modules['ceph'] = MagicMock() -sys.modules['petrel_client'] = MagicMock() -sys.modules['petrel_client.client'] = MagicMock() sys.modules['mc'] = MagicMock() @@ -137,6 +135,24 @@ def setup_class(cls): cls.img_shape = (300, 400, 3) cls.text_path = cls.test_data_dir / 'filelist.txt' + def setup_method(self): + # Mock petrel_client for each test + self.mock_petrel_client = MagicMock() + self.mock_client_module = MagicMock() + self.mock_client_module.Client = MockPetrelClient + self.mock_petrel_client.client = self.mock_client_module + + self.patcher_petrel = patch.dict( + 'sys.modules', { + 'petrel_client': self.mock_petrel_client, + 'petrel_client.client': self.mock_client_module + }) + self.patcher_petrel.start() + + def teardown_method(self): + # Clean up the mock + self.patcher_petrel.stop() + def test_error(self): with pytest.raises(ValueError): FileClient('hadoop') @@ -289,7 +305,6 @@ def test_disk_backend(self): osp.join('dir2', 'img.jpg'), 'text1.txt', 'text2.txt' } - @patch('petrel_client.client.Client', MockPetrelClient) @pytest.mark.parametrize('backend,prefix', [('petrel', None), (None, 's3')]) def test_petrel_backend(self, backend, prefix): diff --git a/tests/test_visualizer/test_vis_backend.py b/tests/test_visualizer/test_vis_backend.py index c991462ef9..0e090583e2 100644 --- a/tests/test_visualizer/test_vis_backend.py +++ b/tests/test_visualizer/test_vis_backend.py @@ -462,37 +462,36 @@ def test_close(self): reason='Aim does not support Windows for now.') class TestAimVisBackend: + @classmethod + def setup_class(cls): + """Setup AimVisBackend instance once for all tests in this class.""" + cls.aim_vis_backend = AimVisBackend() + def test_init(self): AimVisBackend() VISBACKENDS.build(dict(type='AimVisBackend')) def test_experiment(self): - aim_vis_backend = AimVisBackend() - assert aim_vis_backend.experiment == aim_vis_backend._aim_run + assert self.aim_vis_backend.experiment == self.aim_vis_backend._aim_run def test_add_config(self): cfg = Config(dict(a=1, b=dict(b1=[0, 1]))) - aim_vis_backend = AimVisBackend() - aim_vis_backend.add_config(cfg) + self.aim_vis_backend.add_config(cfg) def test_add_image(self): image = np.random.randint(0, 256, size=(10, 10, 3)).astype(np.uint8) - aim_vis_backend = AimVisBackend() - aim_vis_backend.add_image('img', image) - aim_vis_backend.add_image('img', image, step=1) + self.aim_vis_backend.add_image('img', image) + self.aim_vis_backend.add_image('img', image, step=1) def test_add_scalar(self): - aim_vis_backend = AimVisBackend() - aim_vis_backend.add_scalar('map', 0.9) - aim_vis_backend.add_scalar('map', 0.9, step=1) - aim_vis_backend.add_scalar('map', 0.95, step=2) + self.aim_vis_backend.add_scalar('map', 0.9) + self.aim_vis_backend.add_scalar('map', 0.9, step=1) + self.aim_vis_backend.add_scalar('map', 0.95, step=2) def test_add_scalars(self): - aim_vis_backend = AimVisBackend() input_dict = {'map': 0.7, 'acc': 0.9} - aim_vis_backend.add_scalars(input_dict) + self.aim_vis_backend.add_scalars(input_dict) def test_close(self): - aim_vis_backend = AimVisBackend() - aim_vis_backend._init_env() - aim_vis_backend.close() + self.aim_vis_backend._init_env() + self.aim_vis_backend.close() diff --git a/uv.lock b/uv.lock index fb2cd91297..47bd3168f9 100644 --- a/uv.lock +++ b/uv.lock @@ -2802,7 +2802,7 @@ wheels = [ [[package]] name = "onedl-mmengine" -version = "0.10.8" +version = "0.10.9" source = { editable = "." } dependencies = [ { name = "addict" }, @@ -2887,7 +2887,7 @@ test-lite = [ requires-dist = [ { name = "addict" }, { name = "addict", marker = "extra == 'all'" }, - { name = "aim", marker = "sys_platform != 'win32' and extra == 'all'", specifier = "<=3.17.5" }, + { name = "aim", marker = "sys_platform != 'win32' and extra == 'all'" }, { name = "bitsandbytes", marker = "extra == 'all'" }, { name = "clearml", marker = "extra == 'all'" }, { name = "coverage", marker = "extra == 'all'" }, @@ -2935,7 +2935,7 @@ docs = [ { name = "urllib3" }, ] test = [ - { name = "aim", marker = "sys_platform != 'win32'", specifier = "<=3.17.5" }, + { name = "aim", marker = "sys_platform != 'win32'" }, { name = "bitsandbytes" }, { name = "clearml" }, { name = "coverage" },