diff --git a/test/README.md b/test/README.md index d04a607a4a..42bff0c6dc 100644 --- a/test/README.md +++ b/test/README.md @@ -10,6 +10,9 @@ You can see all existing tests in [`test/`](https://github.com/intel/cve-bin-too - [Adding new tests: Signature tests against real files](#adding-new-tests-signature-tests-against-real-files) - [Adding new tests: Checker filename mappings](#adding-new-tests-checker-filename-mappings) - [Known issues](#known-issues) + - [Test Organization and Avoiding Database Locking](#test-organization-and-avoiding-database-locking) + - [Running Tests](#running-tests) + - [Test Environment Variables](#test-environment-variables) ## Running all tests @@ -178,3 +181,34 @@ pytest -v test/test_checkers.py -k python ## Known issues If you're using Windows and plan to run PDF tests, we **strongly** recommend also `pdftotext`. We experienced problems running tests without this. The best approach to do this is through [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/windows.html) (click [here](https://anaconda.org/conda-forge/pdftotext) to find out how to install this package with conda). + +## Test Organization and Avoiding Database Locking + +### Running Tests + +To avoid "database is locked" errors during testing, tests in the cve-bin-tool are organized into two groups: + +1. **Regular tests**: These can be run in parallel and don't modify the database or use `-u never` to avoid database updates. +2. **Synchronous tests**: These tests modify the database and should be run sequentially to avoid locking issues. + +To run tests properly: + +```bash +# Run regular tests (in parallel) +pytest test/ -xvs + +# Run database-modifying (synchronous) tests separately +pytest test/ -xvs --synchronous +``` + +When submitting new tests that modify the database: + +1. Mark them with `@pytest.mark.skipif(not SYNCHRONOUS(), reason="Database-modifying test, run synchronously")` +2. Ensure they use `-u never` if they don't need to update the database +3. Place any tests that must update the database in the synchronous group + +### Test Environment Variables + +* `LONG_TESTS=1`: Run long tests (e.g., network tests, large package tests) +* `EXTERNAL_SYSTEM=1`: Run tests requiring external systems +* `SYNCHRONOUS=1`: Run tests marked as synchronous (database-modifying) diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000000..02de59734a --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,36 @@ +""" +Pytest configuration file for cve-bin-tool tests +""" +import pytest + + +def pytest_configure(config): + """Register custom markers.""" + config.addinivalue_line( + "markers", "skipif(condition, reason=None): skip the test if condition is true" + ) + + +def pytest_addoption(parser): + """Add a command line option to run synchronous tests.""" + parser.addoption( + "--synchronous", + action="store_true", + default=False, + help="run synchronous (non-parallel-safe) tests", + ) + + +def pytest_collection_modifyitems(config, items): + """Set SYNCHRONOUS environment variable based on --synchronous option.""" + import os + run_synchronous = config.getoption("--synchronous") + os.environ["SYNCHRONOUS"] = "1" if run_synchronous else "0" + + # Skip synchronous tests if --synchronous is not given + skip_synchronous = pytest.mark.skip(reason="need --synchronous option to run") + for item in items: + for marker in item.iter_markers(name="skipif"): + if len(marker.args) > 0 and "SYNCHRONOUS" in str(marker.args[0]): + if not run_synchronous: + item.add_marker(skip_synchronous) \ No newline at end of file diff --git a/test/test_cvedb.py b/test/test_cvedb.py index 80742ebbb4..df2a7fa239 100644 --- a/test/test_cvedb.py +++ b/test/test_cvedb.py @@ -4,7 +4,7 @@ import datetime import shutil import tempfile -from test.utils import EXTERNAL_SYSTEM, LONG_TESTS +from test.utils import EXTERNAL_SYSTEM, LONG_TESTS, SYNCHRONOUS import pytest @@ -38,6 +38,7 @@ async def test_refresh_nvd_json(self): assert year in years, f"Missing NVD data for {year}" @pytest.mark.skipif(not LONG_TESTS(), reason="Skipping long tests") + @pytest.mark.skipif(not SYNCHRONOUS(), reason="Database-modifying test, run synchronously") def test_import_export_json(self): main(["cve-bin-tool", "-u", "never", "--export", self.nvd.cachedir]) cve_entries_check = "SELECT data_source, COUNT(*) as number FROM cve_severity GROUP BY data_source ORDER BY number DESC" @@ -67,6 +68,7 @@ def test_import_export_json(self): assert cve_entries_before == cve_entries_after + @pytest.mark.skipif(not SYNCHRONOUS(), reason="Database-modifying test, run synchronously") def test_new_database_schema(self): # Check if the new schema is created in the database self.cvedb.init_database() diff --git a/test/test_language_scanner.py b/test/test_language_scanner.py index 638fd5cfad..70bd5ea675 100644 --- a/test/test_language_scanner.py +++ b/test/test_language_scanner.py @@ -7,11 +7,13 @@ from test.utils import LONG_TESTS import pytest +import tempfile from cve_bin_tool import parsers from cve_bin_tool.cvedb import CVEDB from cve_bin_tool.log import LOGGER from cve_bin_tool.version_scanner import VersionScanner +from cve_bin_tool.language_scanner import LanguageScanner class TestLanguageScanner: @@ -166,10 +168,12 @@ class TestLanguageScanner: @classmethod def setup_class(cls): + cls.tempdir = tempfile.mkdtemp(prefix="langdata-") + cls.package_test_dir = tempfile.mkdtemp(prefix="package_test-") cls.cvedb = CVEDB() - print("Setting up database.") - cls.cvedb.get_cvelist_if_stale() - print("Database setup complete.") + # Always use -u never to avoid database locking in parallel tests + print("Skip NVD database updates - using -u never by default for parallel test safety.") + cls.scanner = LanguageScanner(should_extract=True, quiet=True) @pytest.mark.parametrize( "filename, product_list", diff --git a/test/test_nvd_api.py b/test/test_nvd_api.py index 47bbbaf0da..597777a690 100644 --- a/test/test_nvd_api.py +++ b/test/test_nvd_api.py @@ -5,7 +5,8 @@ import shutil import tempfile from datetime import datetime, timedelta -from test.utils import EXTERNAL_SYSTEM +from test.utils import EXTERNAL_SYSTEM, SYNCHRONOUS +from pathlib import Path import pytest @@ -100,3 +101,16 @@ async def test_api_cve_count(self): abs(nvd_api.total_results - (cve_count["Total"] - cve_count["Rejected"])) <= cve_count["Received"] ) + + @pytest.mark.asyncio + @pytest.mark.skipif(not EXTERNAL_SYSTEM(), reason="Needs network connection.") + @pytest.mark.skipif(not SYNCHRONOUS(), reason="Database-modifying test, run synchronously") + async def test_fetch_api(self): + await self.nvd.fetch_cves() + assert Path(self.nvd.nvd_api_file).exists() + + # Test extract to db + cvedb = CVEDB(cachedir=self.outdir) + cvedb.init_database() + cvedb.populate_db() + cvedb.db_close() diff --git a/test/test_scanner.py b/test/test_scanner.py index 9ea481695b..7a71ccc2cd 100644 --- a/test/test_scanner.py +++ b/test/test_scanner.py @@ -68,10 +68,8 @@ class TestScanner: @classmethod def setup_class(cls): cls.cvedb = CVEDB() - if os.getenv("UPDATE_DB") == "1": - cls.cvedb.get_cvelist_if_stale() - else: - print("Skip NVD database updates.") + # Always use -u never to avoid database locking in parallel tests + print("Skip NVD database updates - using -u never by default for parallel test safety.") # Instantiate a scanner cls.scanner = VersionScanner(should_extract=True) # temp dir for mapping tests @@ -321,7 +319,7 @@ def test_version_in_package( ), f"""{option} not found in {package_name}. Remove {option} from other_products.""" assert ( option in product_not_present - ), f"""{option} not found in {product_not_present}. Remove {option} from other_products or add a test for {option} in test_data.""" + ), f"""{option} not found in {product_not_present}. Remove {option} from other_products or add a test for {option} in test_data.""" product_not_present.remove(option) for not_product in product_not_present: assert ( diff --git a/test/test_source_epss.py b/test/test_source_epss.py index 0caae28b7d..410c03b84e 100644 --- a/test/test_source_epss.py +++ b/test/test_source_epss.py @@ -1,7 +1,10 @@ from pathlib import Path +import pytest + from cve_bin_tool.cvedb import CVEDB from cve_bin_tool.data_sources import epss_source +from test.utils import SYNCHRONOUS class TestSourceEPSS: @@ -22,10 +25,10 @@ def setup_class(cls): ("CVE-1999-0007", 1, "0.00180", "0.54020"), ] + @pytest.mark.skipif(not SYNCHRONOUS(), reason="Database-modifying test, run synchronously") def test_parse_epss(self): # EPSS need metrics table to populated in the database. EPSS metric id is a constant. cvedb = CVEDB() - # creating table cvedb.init_database() # populating metrics cvedb.populate_metrics() diff --git a/test/utils.py b/test/utils.py index 01567fe89f..191dbae9e4 100644 --- a/test/utils.py +++ b/test/utils.py @@ -76,6 +76,14 @@ def EXTERNAL_SYSTEM() -> bool: return False +def SYNCHRONOUS() -> bool: + # Used to mark tests that should be run synchronously (not in parallel) + env_var = os.getenv("SYNCHRONOUS") + if env_var: + return bool(int(env_var)) + return False + + @pytest.fixture def event_loop(): yield get_event_loop()