Support Python 3.12 and drop 3.7 (#369)

* Need GIL for all thread deallocation
* Use spawn for testing
* Replace the deprecated build tool

.github/workflows/lint.yml

@@ -9,7 +9,7 @@ jobs:
   lint:
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.github/workflows/python-package.yml

@@ -14,7 +14,7 @@ jobs:
   build:
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
@@ -48,13 +48,13 @@ jobs:
     - name: Build dist and test with unittest
       if: matrix.os != 'windows-latest'
       run: |
-        python setup.py sdist bdist_wheel
+        python -m build
         pip install dist/*.whl
         python -m unittest
     - name: Build dist and test with unittest on Windows
       if: matrix.os == 'windows-latest'
       run: |
-        python setup.py sdist bdist_wheel
+        python -m build
         pip install (Get-ChildItem dist/*.whl)
         python -m unittest
     - name: Generate coverage report

.github/workflows/release.yml

@@ -11,14 +11,10 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [37, 38, 39, 310, 311]
+        python-version: [38, 39, 310, 311, 312]
         manylinux-image: [manylinux2014, manylinux_2_24]
         arch: [auto]
         include:
-          - os: ubuntu-latest
-            manylinux-image: manylinux2014
-            arch: aarch64
-            python-version: 37
           - os: ubuntu-latest
             manylinux-image: manylinux2014
             arch: aarch64
@@ -35,6 +31,10 @@ jobs:
             manylinux-image: manylinux2014
             arch: aarch64
             python-version: 311
+          - os: ubuntu-latest
+            manylinux-image: manylinux2014
+            arch: aarch64
+            python-version: 312
         exclude:
           # manyliunx image is not a valid variation on MacOS and Windows
           - os: macos-latest
@@ -54,7 +54,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install twine flake8 setuptools wheel
+          pip install build twine flake8 setuptools wheel
 
       - name: Install cibuildwheel
         run: python -m pip install cibuildwheel -U
@@ -97,10 +97,10 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install twine flake8 setuptools wheel
+          pip install build twine flake8 setuptools wheel
       - name: Build source tar
         run: |
-          python setup.py sdist
+          python -m build
       - name: Publish wheels to PyPI
         continue-on-error: true
         env:

MANIFEST.in

@@ -1,5 +1,5 @@
 include LICENSE
-include NOTICE
+include NOTICE.txt
 include README.md
 include setup.py
 

Makefile

@@ -3,14 +3,14 @@
 refresh: clean build install lint
 
 build:
-	python setup.py build
+	python -m build
 
 install:
-	python setup.py install
+	pip install .
 
 build_dist:
 	make clean
-	python setup.py sdist bdist_wheel
+	python -m build
 	pip install dist/*.whl
 	make test
 

pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "viztracer"
+authors = [{name = "Tian Gao", email = "[email protected]"}]
+description = "A debugging and profiling tool that can trace and visualize python code execution"
+dependencies = ["objprint>0.1.3"]
+readme = "README.md"
+requires-python = ">=3.8"
+dynamic = ["version"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: MacOS",
+    "Operating System :: POSIX :: Linux",
+    "Operating System :: Microsoft :: Windows",
+    "Topic :: Software Development :: Quality Assurance",
+    "Topic :: Software Development :: Bug Tracking",
+    "Topic :: System :: Logging",
+]
+
+[project.urls]
+HomePage = "https://github.com/gaogaotiantian/viztracer"
+Documentation = "https://viztracer.readthedocs.io"
+
+[project.optional-dependencies]
+full = ["rich", "orjson"]
+
+[project.scripts]
+viztracer = "viztracer:main"
+vizviewer = "viztracer:viewer_main"
+vdb = "viztracer:sim_main"
+
+[tool.setuptools.dynamic]
+version = {attr = "viztracer.__version__"}

requirements-dev.txt

@@ -1,4 +1,5 @@
 # Build Packages
+build
 setuptools
 wheel
 twine

setup.py

@@ -3,20 +3,6 @@
 
 import setuptools
 
-with open("README.md") as f:
-    long_description = f.read()
-
-with open("./src/viztracer/__init__.py") as f:
-    for line in f.readlines():
-        if line.startswith("__version__"):
-            # __version__ = "0.9"
-            delim = '"' if '"' in line else "'"
-            version = line.split(delim)[1]
-            break
-    else:
-        print("Can't find version! Stop Here!")
-        sys.exit(1)
-
 # Determine which attach binary to take into package
 package_data = {
     "viztracer": [
@@ -48,15 +34,7 @@
         ])
 
 setuptools.setup(
-    name="viztracer",
-    version=version,
-    author="Tian Gao",
-    author_email="[email protected]",
-    description="A debugging and profiling tool that can trace and visualize python code execution",
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    url="https://github.com/gaogaotiantian/viztracer",
-    packages=setuptools.find_packages("src"),
+    packages=setuptools.find_namespace_packages("src"),
     package_dir={"": "src"},
     package_data=package_data,
     ext_modules=[
@@ -79,32 +57,4 @@
             extra_compile_args={"win32": []}.get(sys.platform, ["-Werror", "-std=c99"]),
         ),
     ],
-    classifiers=[
-        "Development Status :: 4 - Beta",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
-        "Intended Audience :: Developers",
-        "License :: OSI Approved :: Apache Software License",
-        "Operating System :: MacOS",
-        "Operating System :: POSIX :: Linux",
-        "Operating System :: Microsoft :: Windows",
-        "Topic :: Software Development :: Quality Assurance",
-        "Topic :: Software Development :: Bug Tracking",
-        "Topic :: System :: Logging",
-    ],
-    python_requires=">=3.7",
-    install_requires=["objprint>=0.1.3"],
-    extras_require={
-        "full": ["rich", "orjson"],
-    },
-    entry_points={
-        "console_scripts": [
-            "viztracer = viztracer:main",
-            "vizviewer = viztracer:viewer_main",
-            "vdb = viztracer:sim_main",
-        ],
-    },
 )

src/viztracer/cellmagic.py

@@ -15,6 +15,7 @@ def viztracer(self, line, cell, local_ns) -> None:
 
             from .viewer import ServerThread
             from .viztracer import VizTracer
+            assert self.shell is not None
             code = self.shell.transform_cell(cell)
             file_path = "./viztracer_report.json"
             with VizTracer(verbose=0, output_file=file_path):

src/viztracer/code_monkey.py

@@ -40,11 +40,7 @@ def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.FunctionDef:
         elif self.inst_type in ("log_var", "log_number"):
             instrumented_nodes: List[ast.stmt] = []
             args = node.args
-            if sys.version_info >= (3, 8):
-                func_args_name = [a.arg for a in args.posonlyargs + args.args + args.kwonlyargs]
-            else:
-                # python 3.7 does not have posonlyargs
-                func_args_name = [a.arg for a in args.args + args.kwonlyargs]
+            func_args_name = [a.arg for a in args.posonlyargs + args.args + args.kwonlyargs]
             if "vararg" in args._fields and args.vararg:
                 func_args_name.append(args.vararg.arg)
             if "kwarg" in args._fields and args.kwarg:
@@ -251,10 +247,6 @@ def get_string_of_expr(self, node: Union[ast.expr, ast.slice]) -> str:
                 return f"'{node.value}'"
             else:
                 return f"{node.value}"
-        elif isinstance(node, ast.Num):
-            return f"{node.n}"
-        elif isinstance(node, ast.Str):
-            return f"'{node.s}'"
         elif isinstance(node, ast.Attribute):
             return f"{self.get_string_of_expr(node.value)}.{node.attr}"
         elif isinstance(node, ast.Subscript):

src/viztracer/modules/snaptrace.c

@@ -1519,6 +1519,7 @@ static void snaptrace_threaddestructor(void* key) {
     struct ThreadInfo* info = key;
     struct FunctionNode* tmp = NULL;
     if (info) {
+        PyGILState_STATE state = PyGILState_Ensure();
         info->paused = 0;
         info->curr_stack_depth = 0;
         info->ignore_stack_depth = 0;
@@ -1527,7 +1528,6 @@ static void snaptrace_threaddestructor(void* key) {
             while (info->stack_top->prev) {
                 info->stack_top = info->stack_top->prev;
             }
-            PyGILState_STATE state = PyGILState_Ensure();
             while (info->stack_top) {
                 tmp = info->stack_top;
                 if (tmp->args) {
@@ -1537,12 +1537,12 @@ static void snaptrace_threaddestructor(void* key) {
                 info->stack_top = info->stack_top->next;
                 PyMem_FREE(tmp);
             }
-            PyGILState_Release(state);
         }
         info->stack_top = NULL;
         info->curr_task = NULL;
         info->curr_task_frame = NULL;
         PyMem_FREE(info);
+        PyGILState_Release(state);
     }
 }
 

src/viztracer/patch.py

@@ -161,7 +161,7 @@ def patch_spawned_process(viztracer_kwargs: Dict[str, Any], cmdline_args: List[s
 
     @no_type_check
     @functools.wraps(multiprocessing.spawn._main)
-    def _main_3839(fd, parent_sentinel) -> Any:
+    def _main(fd, parent_sentinel) -> Any:
         with os.fdopen(fd, 'rb', closefd=True) as from_parent:
             process.current_process()._inheriting = True
             try:
@@ -174,25 +174,7 @@ def _main_3839(fd, parent_sentinel) -> Any:
                 del process.current_process()._inheriting
         return self._bootstrap(parent_sentinel)
 
-    @no_type_check
-    @functools.wraps(multiprocessing.spawn._main)
-    def _main_3637(fd) -> Any:
-        with os.fdopen(fd, 'rb', closefd=True) as from_parent:
-            process.current_process()._inheriting = True
-            try:
-                preparation_data = reduction.pickle.load(from_parent)
-                prepare(preparation_data)
-                self: Process = reduction.pickle.load(from_parent)
-                sp = SpawnProcess(viztracer_kwargs, self.run, self._target, self._args, self._kwargs, cmdline_args)
-                self.run = sp.run
-            finally:
-                del process.current_process()._inheriting
-        return self._bootstrap()
-
-    if sys.version_info >= (3, 8):
-        multiprocessing.spawn._main = _main_3839  # type: ignore
-    else:
-        multiprocessing.spawn._main = _main_3637  # type: ignore
+    multiprocessing.spawn._main = _main  # type: ignore
 
 
 def install_all_hooks(

tests/test_cmdline.py

@@ -406,7 +406,6 @@ def test_log_func_entry(self):
                       expected_output_file="result.json",
                       expected_entries=7)
 
-    @skipIf(sys.version_info < (3, 8), "sys.addaudithook does not exist on 3.8-")
     def test_log_audit(self):
         def check_func(include_names, exclude_names=[]):
             def inner(data):

tests/test_multiprocess.py

@@ -7,7 +7,6 @@
 import sys
 import tempfile
 import unittest
-from unittest.case import skipIf
 
 from .cmdline_tmpl import CmdlineTmpl
 
@@ -292,7 +291,7 @@ def check_func(data):
             self.template(["viztracer", "-o", "result.json", "cmdline_test.py"],
                           expected_output_file="result.json", script=file_fork, check_func=check_func)
 
-    @unittest.skipIf(sys.version_info < (3, 8) or sys.platform not in ["linux", "linux2"], "Only works on Linux + py3.8+")
+    @unittest.skipIf(sys.platform not in ["linux", "linux2"], "Only works on Linux")
     def test_os_fork_term(self):
         def check_func_wrapper(process_num):
             def check_func(data):
@@ -408,7 +407,6 @@ def check_func(data):
 
 
 class TestLoky(CmdlineTmpl):
-    @skipIf(sys.version_info < (3, 8), "fork + exec will make viztracer + loky deadlock")
     def test_loky_basic(self):
         def check_func(data):
             pids = set()

tests/test_remote.py

@@ -60,7 +60,7 @@ def test_attach(self):
             while True:
                 time.sleep(0.5)
         """)
-        output_file = os.path.abspath(f"./remote_{int(time.time()*1000)}.json")
+        output_file = os.path.abspath(f"./remote_{int(time.time() * 1000)}.json")
         attach_cmd = cmd_with_coverage(["viztracer", "-o", output_file, "--attach"])
 
         self.attach_check(file_to_attach, attach_cmd, output_file)
@@ -149,7 +149,7 @@ def test_uninstall(self):
             while True:
                 time.sleep(0.5)
         """)
-        output_file = os.path.abspath(f"remote_{int(time.time()*1000)}.json")
+        output_file = os.path.abspath(f"remote_{int(time.time() * 1000)}.json")
         uninstall_cmd = cmd_with_coverage(["viztracer", "-o", output_file, "--uninstall"])
         attach_cmd = cmd_with_coverage(["viztracer", "-o", output_file, "--attach"])
 

tests/test_viewer.py

@@ -121,7 +121,7 @@ def _wait_until_socket_on(self):
         self.fail(f"Can't connect to 127.0.0.1:{port}")
 
     def url(self, offset: int = 0) -> str:
-        return f'http://127.0.0.1:{self.port+offset}'
+        return f'http://127.0.0.1:{self.port + offset}'
 
 
 class MockOpen(unittest.TestCase):
@@ -143,7 +143,9 @@ def get_and_check(self, url, expected):
             os.kill(self.int_pid, signal.SIGINT)
 
     def __call__(self, url):
-        self.p = multiprocessing.Process(target=self.get_and_check, args=(url, self.file_content))
+        # fork in a multi-threaded program could result in dead lock
+        ctx = multiprocessing.get_context("spawn")
+        self.p = ctx.Process(target=self.get_and_check, args=(url, self.file_content))
         self.p.start()