Skip to content

Commit bc357f5

Browse files
benoitcbenoitc
committed
Refactor import caching to per-interpreter model
- Replace per-process import caching with global ETS registry - Imports are applied to sys.modules when interpreters are created - Remove deprecated loop import NIFs (loop_import_module, etc.) - Add interp_apply_imports/2 and interp_flush_imports/2 NIFs - flush_imports now removes modules from sys.modules in all interpreters - Contexts and loops sharing the same interpreter share sys.modules - Add py:init_import_registry/0 called on application startup - Add py:get_imports/0, py:del_import/1,2, py:clear_imports/0 - Add py_context:flush_imports/2 and py_context_router:flush_all_imports/1 - Add del_import_reimport_test verifying module removal from sys.modules
1 parent 4ac10a4 commit bc357f5

File tree

10 files changed

+1201
-223
lines changed

10 files changed

+1201
-223
lines changed

‎c_src/py_nif.c‎

Lines changed: 341 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3168,6 +3168,96 @@ static void owngil_execute_create_local_env(py_context_t *ctx) {
31683168
ctx->response_ok = true;
31693169
}
31703170

3171+
/**
3172+
* @brief Execute apply_imports in OWN_GIL context
3173+
*
3174+
* Applies a list of imports to the interpreter's sys.modules.
3175+
* The imports list is passed via request_term.
3176+
*
3177+
* Note: OWN_GIL contexts have their own dedicated interpreter,
3178+
* so sys.modules is per-context in this mode.
3179+
*/
3180+
static void owngil_execute_apply_imports(py_context_t *ctx) {
3181+
/* Process each import from request_term */
3182+
ERL_NIF_TERM head, tail = ctx->request_term;
3183+
int arity;
3184+
const ERL_NIF_TERM *tuple;
3185+
3186+
while (enif_get_list_cell(ctx->shared_env, tail, &head, &tail)) {
3187+
if (!enif_get_tuple(ctx->shared_env, head, &arity, &tuple) || arity != 2) {
3188+
continue;
3189+
}
3190+
3191+
ErlNifBinary module_bin;
3192+
if (!enif_inspect_binary(ctx->shared_env, tuple[0], &module_bin)) {
3193+
continue;
3194+
}
3195+
3196+
/* Convert to C string */
3197+
char *module_name = enif_alloc(module_bin.size + 1);
3198+
if (module_name == NULL) continue;
3199+
memcpy(module_name, module_bin.data, module_bin.size);
3200+
module_name[module_bin.size] = '\0';
3201+
3202+
/* Skip __main__ */
3203+
if (strcmp(module_name, "__main__") == 0) {
3204+
enif_free(module_name);
3205+
continue;
3206+
}
3207+
3208+
/* Import the module - caches in this interpreter's sys.modules */
3209+
PyObject *mod = PyImport_ImportModule(module_name);
3210+
if (mod != NULL) {
3211+
Py_DECREF(mod); /* sys.modules holds the reference */
3212+
} else {
3213+
/* Clear error - import failure is not fatal */
3214+
PyErr_Clear();
3215+
}
3216+
3217+
enif_free(module_name);
3218+
}
3219+
3220+
ctx->response_term = enif_make_atom(ctx->shared_env, "ok");
3221+
ctx->response_ok = true;
3222+
}
3223+
3224+
/**
3225+
* @brief Execute flush_imports in OWN_GIL context
3226+
*
3227+
* Clears the per-context module cache optimization layer.
3228+
* Does NOT clear sys.modules (that would break Python).
3229+
*/
3230+
static void owngil_execute_flush_imports(py_context_t *ctx) {
3231+
/* Remove specified modules from sys.modules */
3232+
PyObject *sys_modules = PyImport_GetModuleDict(); /* Borrowed ref */
3233+
if (sys_modules != NULL) {
3234+
ERL_NIF_TERM head, tail = ctx->request_data;
3235+
while (enif_get_list_cell(ctx->shared_env, tail, &head, &tail)) {
3236+
ErlNifBinary mod_bin;
3237+
if (enif_inspect_binary(ctx->shared_env, head, &mod_bin)) {
3238+
char *mod_name = enif_alloc(mod_bin.size + 1);
3239+
if (mod_name != NULL) {
3240+
memcpy(mod_name, mod_bin.data, mod_bin.size);
3241+
mod_name[mod_bin.size] = '\0';
3242+
/* Remove module from sys.modules if present */
3243+
if (PyDict_GetItemString(sys_modules, mod_name) != NULL) {
3244+
PyDict_DelItemString(sys_modules, mod_name);
3245+
}
3246+
enif_free(mod_name);
3247+
}
3248+
}
3249+
}
3250+
}
3251+
3252+
/* Clear the per-context optimization cache if it exists */
3253+
if (ctx->module_cache != NULL) {
3254+
PyDict_Clear(ctx->module_cache);
3255+
}
3256+
3257+
ctx->response_term = enif_make_atom(ctx->shared_env, "ok");
3258+
ctx->response_ok = true;
3259+
}
3260+
31713261
/**
31723262
* @brief Execute a request based on its type
31733263
*/
@@ -3203,6 +3293,12 @@ static void owngil_execute_request(py_context_t *ctx) {
32033293
case CTX_REQ_CREATE_LOCAL_ENV:
32043294
owngil_execute_create_local_env(ctx);
32053295
break;
3296+
case CTX_REQ_APPLY_IMPORTS:
3297+
owngil_execute_apply_imports(ctx);
3298+
break;
3299+
case CTX_REQ_FLUSH_IMPORTS:
3300+
owngil_execute_flush_imports(ctx);
3301+
break;
32063302
default:
32073303
ctx->response_term = enif_make_tuple2(ctx->shared_env,
32083304
enif_make_atom(ctx->shared_env, "error"),
@@ -3768,6 +3864,94 @@ static ERL_NIF_TERM dispatch_create_local_env_to_owngil(
37683864
return result;
37693865
}
37703866

3867+
/**
3868+
* @brief Dispatch apply_imports to OWN_GIL worker thread
3869+
*
3870+
* @param env NIF environment
3871+
* @param ctx Context resource
3872+
* @param imports_term List of {ModuleBin, FuncBin | all} tuples
3873+
* @return ok | {error, Reason}
3874+
*/
3875+
static ERL_NIF_TERM dispatch_apply_imports_to_owngil(
3876+
ErlNifEnv *env, py_context_t *ctx, ERL_NIF_TERM imports_term
3877+
) {
3878+
if (!atomic_load(&ctx->thread_running)) {
3879+
return make_error(env, "thread_not_running");
3880+
}
3881+
3882+
pthread_mutex_lock(&ctx->request_mutex);
3883+
3884+
enif_clear_env(ctx->shared_env);
3885+
ctx->request_term = enif_make_copy(ctx->shared_env, imports_term);
3886+
ctx->request_type = CTX_REQ_APPLY_IMPORTS;
3887+
3888+
pthread_cond_signal(&ctx->request_ready);
3889+
3890+
/* Wait for response with timeout */
3891+
struct timespec deadline;
3892+
clock_gettime(CLOCK_REALTIME, &deadline);
3893+
deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS;
3894+
3895+
while (ctx->request_type != CTX_REQ_NONE) {
3896+
int rc = pthread_cond_timedwait(&ctx->response_ready, &ctx->request_mutex, &deadline);
3897+
if (rc == ETIMEDOUT) {
3898+
atomic_store(&ctx->thread_running, false);
3899+
pthread_mutex_unlock(&ctx->request_mutex);
3900+
fprintf(stderr, "OWN_GIL apply_imports dispatch timeout: worker thread unresponsive\n");
3901+
return make_error(env, "worker_timeout");
3902+
}
3903+
}
3904+
3905+
ERL_NIF_TERM result = enif_make_copy(env, ctx->response_term);
3906+
pthread_mutex_unlock(&ctx->request_mutex);
3907+
3908+
return result;
3909+
}
3910+
3911+
/**
3912+
* @brief Dispatch flush_imports to OWN_GIL worker thread
3913+
*
3914+
* @param env NIF environment
3915+
* @param ctx Context resource
3916+
* @param modules_list List of module names to remove from sys.modules
3917+
* @return ok
3918+
*/
3919+
static ERL_NIF_TERM dispatch_flush_imports_to_owngil(
3920+
ErlNifEnv *env, py_context_t *ctx, ERL_NIF_TERM modules_list
3921+
) {
3922+
if (!atomic_load(&ctx->thread_running)) {
3923+
return make_error(env, "thread_not_running");
3924+
}
3925+
3926+
pthread_mutex_lock(&ctx->request_mutex);
3927+
3928+
enif_clear_env(ctx->shared_env);
3929+
ctx->request_type = CTX_REQ_FLUSH_IMPORTS;
3930+
ctx->request_data = enif_make_copy(ctx->shared_env, modules_list);
3931+
3932+
pthread_cond_signal(&ctx->request_ready);
3933+
3934+
/* Wait for response with timeout */
3935+
struct timespec deadline;
3936+
clock_gettime(CLOCK_REALTIME, &deadline);
3937+
deadline.tv_sec += OWNGIL_DISPATCH_TIMEOUT_SECS;
3938+
3939+
while (ctx->request_type != CTX_REQ_NONE) {
3940+
int rc = pthread_cond_timedwait(&ctx->response_ready, &ctx->request_mutex, &deadline);
3941+
if (rc == ETIMEDOUT) {
3942+
atomic_store(&ctx->thread_running, false);
3943+
pthread_mutex_unlock(&ctx->request_mutex);
3944+
fprintf(stderr, "OWN_GIL flush_imports dispatch timeout: worker thread unresponsive\n");
3945+
return make_error(env, "worker_timeout");
3946+
}
3947+
}
3948+
3949+
ERL_NIF_TERM result = enif_make_copy(env, ctx->response_term);
3950+
pthread_mutex_unlock(&ctx->request_mutex);
3951+
3952+
return result;
3953+
}
3954+
37713955
#endif /* HAVE_SUBINTERPRETERS */
37723956

37733957
/**
@@ -3786,6 +3970,9 @@ static int owngil_context_init(py_context_t *ctx) {
37863970
atomic_store(&ctx->init_error, false);
37873971
atomic_store(&ctx->shutdown_requested, false);
37883972
ctx->request_type = CTX_REQ_NONE;
3973+
ctx->request_term = 0;
3974+
ctx->request_data = 0;
3975+
ctx->response_term = 0;
37893976
ctx->response_ok = false;
37903977

37913978
/* Initialize mutex and condition variables */
@@ -4735,6 +4922,158 @@ static ERL_NIF_TERM nif_create_local_env(ErlNifEnv *env, int argc, const ERL_NIF
47354922
return enif_make_tuple2(env, ATOM_OK, ref);
47364923
}
47374924

4925+
/**
4926+
* @brief Apply a list of imports to an interpreter's sys.modules
4927+
*
4928+
* nif_interp_apply_imports(Ref, Imports) -> ok | {error, Reason}
4929+
*
4930+
* Imports: [{ModuleBin, FuncBin | 'all'}, ...]
4931+
* Imports modules into the interpreter's sys.modules (shared by all
4932+
* contexts/loops using this interpreter).
4933+
*
4934+
* Note: This imports into the INTERPRETER's module cache (sys.modules),
4935+
* not a per-context cache. All contexts using this interpreter will
4936+
* see the imported modules.
4937+
*/
4938+
static ERL_NIF_TERM nif_interp_apply_imports(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
4939+
(void)argc;
4940+
py_context_t *ctx;
4941+
4942+
if (!runtime_is_running()) {
4943+
return make_error(env, "python_not_running");
4944+
}
4945+
4946+
if (!enif_get_resource(env, argv[0], PY_CONTEXT_RESOURCE_TYPE, (void **)&ctx)) {
4947+
return make_error(env, "invalid_context");
4948+
}
4949+
4950+
if (ctx->destroyed) {
4951+
return make_error(env, "context_destroyed");
4952+
}
4953+
4954+
#ifdef HAVE_SUBINTERPRETERS
4955+
/* OWN_GIL mode: dispatch to the dedicated thread */
4956+
if (ctx->uses_own_gil) {
4957+
return dispatch_apply_imports_to_owngil(env, ctx, argv[1]);
4958+
}
4959+
#endif
4960+
4961+
py_context_guard_t guard = py_context_acquire(ctx);
4962+
if (!guard.acquired) {
4963+
return make_error(env, "acquire_failed");
4964+
}
4965+
4966+
/* Process each import - imports go into interpreter's sys.modules */
4967+
ERL_NIF_TERM head, tail = argv[1];
4968+
int arity;
4969+
const ERL_NIF_TERM *tuple;
4970+
4971+
while (enif_get_list_cell(env, tail, &head, &tail)) {
4972+
if (!enif_get_tuple(env, head, &arity, &tuple) || arity != 2) {
4973+
continue;
4974+
}
4975+
4976+
ErlNifBinary module_bin;
4977+
if (!enif_inspect_binary(env, tuple[0], &module_bin)) {
4978+
continue;
4979+
}
4980+
4981+
/* Convert to C string */
4982+
char *module_name = enif_alloc(module_bin.size + 1);
4983+
if (module_name == NULL) continue;
4984+
memcpy(module_name, module_bin.data, module_bin.size);
4985+
module_name[module_bin.size] = '\0';
4986+
4987+
/* Skip __main__ */
4988+
if (strcmp(module_name, "__main__") == 0) {
4989+
enif_free(module_name);
4990+
continue;
4991+
}
4992+
4993+
/* Import the module - this caches in interpreter's sys.modules
4994+
* which is shared by all contexts using this interpreter */
4995+
PyObject *mod = PyImport_ImportModule(module_name);
4996+
if (mod != NULL) {
4997+
Py_DECREF(mod); /* sys.modules holds the reference */
4998+
} else {
4999+
/* Clear error - import failure is not fatal */
5000+
PyErr_Clear();
5001+
}
5002+
5003+
enif_free(module_name);
5004+
}
5005+
5006+
py_context_release(&guard);
5007+
return ATOM_OK;
5008+
}
5009+
5010+
/**
5011+
* @brief Flush imports from an interpreter's sys.modules
5012+
*
5013+
* nif_interp_flush_imports(Ref, Modules) -> ok
5014+
*
5015+
* Removes specified modules from sys.modules and clears the per-context
5016+
* module_cache optimization layer.
5017+
*
5018+
* @param Ref Context reference
5019+
* @param Modules List of binary module names to remove from sys.modules
5020+
*/
5021+
static ERL_NIF_TERM nif_interp_flush_imports(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) {
5022+
(void)argc;
5023+
py_context_t *ctx;
5024+
5025+
if (!runtime_is_running()) {
5026+
return ATOM_OK; /* Nothing to flush if Python not running */
5027+
}
5028+
5029+
if (!enif_get_resource(env, argv[0], PY_CONTEXT_RESOURCE_TYPE, (void **)&ctx)) {
5030+
return make_error(env, "invalid_context");
5031+
}
5032+
5033+
if (ctx->destroyed) {
5034+
return ATOM_OK; /* Already destroyed */
5035+
}
5036+
5037+
#ifdef HAVE_SUBINTERPRETERS
5038+
/* OWN_GIL mode: dispatch to the dedicated thread */
5039+
if (ctx->uses_own_gil) {
5040+
return dispatch_flush_imports_to_owngil(env, ctx, argv[1]);
5041+
}
5042+
#endif
5043+
5044+
py_context_guard_t guard = py_context_acquire(ctx);
5045+
if (!guard.acquired) {
5046+
return ATOM_OK; /* Can't acquire - just return ok */
5047+
}
5048+
5049+
/* Remove specified modules from sys.modules */
5050+
PyObject *sys_modules = PyImport_GetModuleDict(); /* Borrowed ref */
5051+
if (sys_modules != NULL) {
5052+
ERL_NIF_TERM head, tail = argv[1];
5053+
while (enif_get_list_cell(env, tail, &head, &tail)) {
5054+
ErlNifBinary mod_bin;
5055+
if (enif_inspect_binary(env, head, &mod_bin)) {
5056+
char *mod_name = binary_to_string(&mod_bin);
5057+
if (mod_name != NULL) {
5058+
/* Remove module from sys.modules if present */
5059+
if (PyDict_GetItemString(sys_modules, mod_name) != NULL) {
5060+
PyDict_DelItemString(sys_modules, mod_name);
5061+
}
5062+
enif_free(mod_name);
5063+
}
5064+
}
5065+
}
5066+
}
5067+
5068+
/* Clear per-context optimization cache */
5069+
if (ctx->module_cache != NULL) {
5070+
PyDict_Clear(ctx->module_cache);
5071+
}
5072+
5073+
py_context_release(&guard);
5074+
return ATOM_OK;
5075+
}
5076+
47385077
/**
47395078
* @brief Execute Python statements using a process-local environment
47405079
*
@@ -6792,12 +7131,6 @@ static ErlNifFunc nif_funcs[] = {
67927131
/* Per-process namespace NIFs */
67937132
{"event_loop_exec", 2, nif_event_loop_exec, ERL_NIF_DIRTY_JOB_IO_BOUND},
67947133
{"event_loop_eval", 2, nif_event_loop_eval, ERL_NIF_DIRTY_JOB_IO_BOUND},
6795-
/* Module import caching NIFs */
6796-
{"loop_import_module", 2, nif_loop_import_module, ERL_NIF_DIRTY_JOB_IO_BOUND},
6797-
{"loop_import_function", 3, nif_loop_import_function, ERL_NIF_DIRTY_JOB_IO_BOUND},
6798-
{"loop_flush_import_cache", 1, nif_loop_flush_import_cache, 0},
6799-
{"loop_import_stats", 1, nif_loop_import_stats, 0},
6800-
{"loop_import_list", 1, nif_loop_import_list, 0},
68017134
{"add_reader", 3, nif_add_reader, 0},
68027135
{"remove_reader", 2, nif_remove_reader, 0},
68037136
{"add_writer", 3, nif_add_writer, 0},
@@ -6870,6 +7203,8 @@ static ErlNifFunc nif_funcs[] = {
68707203
{"context_eval", 4, nif_context_eval_with_env, ERL_NIF_DIRTY_JOB_CPU_BOUND},
68717204
{"context_call", 6, nif_context_call_with_env, ERL_NIF_DIRTY_JOB_CPU_BOUND},
68727205
{"create_local_env", 1, nif_create_local_env, 0},
7206+
{"interp_apply_imports", 2, nif_interp_apply_imports, ERL_NIF_DIRTY_JOB_CPU_BOUND},
7207+
{"interp_flush_imports", 2, nif_interp_flush_imports, ERL_NIF_DIRTY_JOB_CPU_BOUND},
68737208
{"context_call_method", 4, nif_context_call_method, ERL_NIF_DIRTY_JOB_CPU_BOUND},
68747209
{"context_to_term", 1, nif_context_to_term, 0},
68757210
{"context_interp_id", 1, nif_context_interp_id, 0},

0 commit comments

Comments
 (0)