From d95a132a2b78f6e8fea83b01c33baf82b77c4fb6 Mon Sep 17 00:00:00 2001 From: Seth Nickell Date: Wed, 7 Aug 2024 22:18:12 -1000 Subject: [PATCH 1/9] Register PyCall::LibPython::API.Py_FinalizeEx() --- ext/pycall/libpython.c | 1 + ext/pycall/pycall.c | 12 ++++++++++++ ext/pycall/pycall_internal.h | 1 + 3 files changed, 14 insertions(+) diff --git a/ext/pycall/libpython.c b/ext/pycall/libpython.c index 48ca1d2..e0c8245 100644 --- a/ext/pycall/libpython.c +++ b/ext/pycall/libpython.c @@ -89,6 +89,7 @@ pycall_init_libpython_api_table(VALUE libpython_handle) INIT_API_TABLE_ENTRY(PyUnicode_Type, required); INIT_API_TABLE_ENTRY(Py_InitializeEx, required); + INIT_API_TABLE_ENTRY(Py_FinalizeEx, required); INIT_API_TABLE_ENTRY(Py_IsInitialized, required); INIT_API_TABLE_ENTRY(Py_GetVersion, required); diff --git a/ext/pycall/pycall.c b/ext/pycall/pycall.c index 2c7d56e..b835f17 100644 --- a/ext/pycall/pycall.c +++ b/ext/pycall/pycall.c @@ -681,6 +681,15 @@ pycall_libpython_api_PyList_GetItem(VALUE mod, VALUE pyptr, VALUE idx) return pycall_pyptr_new(pyobj_item); } +static VALUE +pycall_libpython_api_Py_FinalizeEx(VALUE mod) +{ + assert(Py_API(Py_IsInitialized())); + + return Py_API(Py_FinalizeEx)(); +} + + /* ==== PyCall::Helpers ==== */ static VALUE @@ -2155,6 +2164,8 @@ init_python(void) { static char const *argv[1] = { "" }; + printf("PYCALL.C: init_python\n"); + /* optional functions */ if (! Py_API(PyObject_DelAttrString)) { /* The case of PyObject_DelAttrString as a macro */ @@ -2372,6 +2383,7 @@ Init_pycall(void) rb_define_module_function(mAPI, "PyObject_Dir", pycall_libpython_api_PyObject_Dir, 1); rb_define_module_function(mAPI, "PyList_Size", pycall_libpython_api_PyList_Size, 1); rb_define_module_function(mAPI, "PyList_GetItem", pycall_libpython_api_PyList_GetItem, 2); + rb_define_module_function(mAPI, "Py_FinalizeEx", pycall_libpython_api_Py_FinalizeEx, 0); /* PyCall::LibPython::Helpers */ diff --git a/ext/pycall/pycall_internal.h b/ext/pycall/pycall_internal.h index 4a7a06c..f1d2b63 100644 --- a/ext/pycall/pycall_internal.h +++ b/ext/pycall/pycall_internal.h @@ -565,6 +565,7 @@ typedef struct { PyObject *PyExc_TypeError; void (* Py_InitializeEx)(int); + int (* Py_FinalizeEx)(void); int (* Py_IsInitialized)(); char const * (* Py_GetVersion)(); From c75c233e13c61e2012eab0ee7aaef57d8d101b37 Mon Sep 17 00:00:00 2001 From: Seth Nickell Date: Wed, 7 Aug 2024 22:23:32 -1000 Subject: [PATCH 2/9] Remove debug printf --- ext/pycall/pycall.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/ext/pycall/pycall.c b/ext/pycall/pycall.c index b835f17..6b38d71 100644 --- a/ext/pycall/pycall.c +++ b/ext/pycall/pycall.c @@ -2164,8 +2164,6 @@ init_python(void) { static char const *argv[1] = { "" }; - printf("PYCALL.C: init_python\n"); - /* optional functions */ if (! Py_API(PyObject_DelAttrString)) { /* The case of PyObject_DelAttrString as a macro */ From 59295b333d5efa9a5493c116375d9b0f036b7079 Mon Sep 17 00:00:00 2001 From: Seth Nickell Date: Wed, 7 Aug 2024 22:58:09 -1000 Subject: [PATCH 3/9] at_exit from C: rb_set_end_proc(finalize_python) --- ext/pycall/pycall.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ext/pycall/pycall.c b/ext/pycall/pycall.c index 6b38d71..9d77553 100644 --- a/ext/pycall/pycall.c +++ b/ext/pycall/pycall.c @@ -2159,6 +2159,18 @@ pycall_PyObject_DelAttrString(PyObject *pyobj, const char *attr_name) return Py_API(PyObject_SetAttrString)(pyobj, attr_name, NULL); } +static void +finalize_python(VALUE arg) +{ + int res; + + assert(Py_API(Py_IsInitialized())); + res = Py_API(Py_FinalizeEx)(); + + if (res != 0) + rb_raise(rb_eRuntimeError, "Finalizing Python at_exit: Py_FinalizeEx() returned an error: %d", res); +} + static void init_python(void) { @@ -2171,6 +2183,7 @@ init_python(void) } Py_API(Py_InitializeEx)(0); + rb_set_end_proc(finalize_python, Qnil); Py_API(PySys_SetArgvEx)(0, (char **)argv, 0); if (!Py_API(PyEval_ThreadsInitialized)()) { From f57b569656ec5434b6b97ef962ccc9ca6ace6a04 Mon Sep 17 00:00:00 2001 From: Seth Nickell Date: Wed, 7 Aug 2024 23:19:12 -1000 Subject: [PATCH 4/9] cannot use at_exit block: must be called from the same thread --- ext/pycall/pycall.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/ext/pycall/pycall.c b/ext/pycall/pycall.c index 9d77553..6b38d71 100644 --- a/ext/pycall/pycall.c +++ b/ext/pycall/pycall.c @@ -2159,18 +2159,6 @@ pycall_PyObject_DelAttrString(PyObject *pyobj, const char *attr_name) return Py_API(PyObject_SetAttrString)(pyobj, attr_name, NULL); } -static void -finalize_python(VALUE arg) -{ - int res; - - assert(Py_API(Py_IsInitialized())); - res = Py_API(Py_FinalizeEx)(); - - if (res != 0) - rb_raise(rb_eRuntimeError, "Finalizing Python at_exit: Py_FinalizeEx() returned an error: %d", res); -} - static void init_python(void) { @@ -2183,7 +2171,6 @@ init_python(void) } Py_API(Py_InitializeEx)(0); - rb_set_end_proc(finalize_python, Qnil); Py_API(PySys_SetArgvEx)(0, (char **)argv, 0); if (!Py_API(PyEval_ThreadsInitialized)()) { From 239b06a75ee6a1be6f81805f9d200310dfac1321 Mon Sep 17 00:00:00 2001 From: Seth Nickell Date: Wed, 7 Aug 2024 23:30:50 -1000 Subject: [PATCH 5/9] init.rb: add PyCall.finalize() --- lib/pycall/init.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pycall/init.rb b/lib/pycall/init.rb index e14a107..63bff45 100644 --- a/lib/pycall/init.rb +++ b/lib/pycall/init.rb @@ -47,4 +47,8 @@ class << LibPython const_set(:PYTHON_DESCRIPTION, LibPython::PYTHON_DESCRIPTION) true end + + def self.finalize + LibPython::API.Py_FinalizeEx() + end end From ef4cbe2ff1fbf03c75f1efce14446936eb6c8c17 Mon Sep 17 00:00:00 2001 From: Seth Nickell Date: Wed, 7 Aug 2024 23:42:18 -1000 Subject: [PATCH 6/9] allow python to be re-initialized --- ext/pycall/pycall.c | 21 ++++++++++++++------- lib/pycall/init.rb | 2 ++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/ext/pycall/pycall.c b/ext/pycall/pycall.c index 6b38d71..bf015dc 100644 --- a/ext/pycall/pycall.c +++ b/ext/pycall/pycall.c @@ -2311,6 +2311,19 @@ init_tuple(void) rb_define_alias(cTuple, "to_ary", "to_a"); } +VALUE +init_m_python(VALUE mod) +{ + /* initialize Python interpreter */ + init_python(); + init_pyerror(); + init_tuple(); + pycall_init_gcguard(); + pycall_init_ruby_wrapper(); + + return Qnil; +} + void Init_pycall(void) { @@ -2438,11 +2451,5 @@ Init_pycall(void) } rb_define_const(mLibPython, "PYTHON_VERSION", python_version_string); - /* initialize Python interpreter */ - - init_python(); - init_pyerror(); - init_tuple(); - pycall_init_gcguard(); - pycall_init_ruby_wrapper(); + rb_define_module_function(mLibPython, "init_python", init_m_python, 0); } diff --git a/lib/pycall/init.rb b/lib/pycall/init.rb index 63bff45..6bcd454 100644 --- a/lib/pycall/init.rb +++ b/lib/pycall/init.rb @@ -38,6 +38,8 @@ class << LibPython require 'pycall.so' + LibPython.init_python + PyCall.sys.path.append(File.expand_path('../python', __FILE__)) require 'pycall/dict' From d35916d2754ae4636f8fb224b50ed8db7b8e9fed Mon Sep 17 00:00:00 2001 From: Seth Nickell Date: Wed, 7 Aug 2024 23:42:29 -1000 Subject: [PATCH 7/9] Revert "allow python to be re-initialized" Works, but gives warnings and of limited use. This reverts commit ef4cbe2ff1fbf03c75f1efce14446936eb6c8c17. --- ext/pycall/pycall.c | 21 +++++++-------------- lib/pycall/init.rb | 2 -- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/ext/pycall/pycall.c b/ext/pycall/pycall.c index bf015dc..6b38d71 100644 --- a/ext/pycall/pycall.c +++ b/ext/pycall/pycall.c @@ -2311,19 +2311,6 @@ init_tuple(void) rb_define_alias(cTuple, "to_ary", "to_a"); } -VALUE -init_m_python(VALUE mod) -{ - /* initialize Python interpreter */ - init_python(); - init_pyerror(); - init_tuple(); - pycall_init_gcguard(); - pycall_init_ruby_wrapper(); - - return Qnil; -} - void Init_pycall(void) { @@ -2451,5 +2438,11 @@ Init_pycall(void) } rb_define_const(mLibPython, "PYTHON_VERSION", python_version_string); - rb_define_module_function(mLibPython, "init_python", init_m_python, 0); + /* initialize Python interpreter */ + + init_python(); + init_pyerror(); + init_tuple(); + pycall_init_gcguard(); + pycall_init_ruby_wrapper(); } diff --git a/lib/pycall/init.rb b/lib/pycall/init.rb index 6bcd454..63bff45 100644 --- a/lib/pycall/init.rb +++ b/lib/pycall/init.rb @@ -38,8 +38,6 @@ class << LibPython require 'pycall.so' - LibPython.init_python - PyCall.sys.path.append(File.expand_path('../python', __FILE__)) require 'pycall/dict' From 0987d79dae1e06e11d74524e23baf9ff66dafa69 Mon Sep 17 00:00:00 2001 From: Seth Nickell Date: Thu, 8 Aug 2024 00:02:39 -1000 Subject: [PATCH 8/9] Automatically finalize when initialized on main thread --- lib/pycall/init.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/pycall/init.rb b/lib/pycall/init.rb index 63bff45..c735fb6 100644 --- a/lib/pycall/init.rb +++ b/lib/pycall/init.rb @@ -45,6 +45,13 @@ class << LibPython require 'pycall/slice' const_set(:PYTHON_VERSION, LibPython::PYTHON_VERSION) const_set(:PYTHON_DESCRIPTION, LibPython::PYTHON_DESCRIPTION) + + @init_thread_id = Thread.current.object_id + at_exit do + # Finalize is only safe to call if we're on the same thread that initialized PyCall + finalize if Thread.current.object_id == @init_thread_id + end + true end From 41cdfe213fd08ef644b93a26b2b7fb5da7c79b92 Mon Sep 17 00:00:00 2001 From: Seth Nickell Date: Sun, 11 Aug 2024 17:58:34 -1000 Subject: [PATCH 9/9] add temporary debug code --- pycall_hangs_main_thread.rb | 14 ++++++++++++++ test.sh | 5 +++++ 2 files changed, 19 insertions(+) create mode 100755 pycall_hangs_main_thread.rb create mode 100755 test.sh diff --git a/pycall_hangs_main_thread.rb b/pycall_hangs_main_thread.rb new file mode 100755 index 0000000..ffac8aa --- /dev/null +++ b/pycall_hangs_main_thread.rb @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby + +side_thread = Thread.new do + require 'pycall' + PyCall.import_module('sys') + PyCall.import_module('pandas') + + PyCall.finalize # if this line is commented out, the process will hang on exit + puts "side thread: exiting" +end +side_thread.join + +puts "main thread: exiting" +#=> process exits! \ No newline at end of file diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..9420e38 --- /dev/null +++ b/test.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -ex + +bundle exec ruby -I ext/pycall pycall_hangs_main_thread.rb 2> /dev/null \ No newline at end of file