diff --git a/c_src/py_event_loop.c b/c_src/py_event_loop.c index fb6ff3d..8f64342 100644 --- a/c_src/py_event_loop.c +++ b/c_src/py_event_loop.c @@ -3474,6 +3474,503 @@ ERL_NIF_TERM nif_event_loop_eval(ErlNifEnv *env, int argc, return enif_make_tuple2(env, ATOM_OK, result_term); } +/* ============================================================================ + * Module Import Caching NIFs + * ============================================================================ */ + +/** + * @brief Import and cache a module in the event loop's interpreter + * + * Pre-imports the module and caches it for faster subsequent calls. + * The __main__ module is never cached (returns error). + * + * NIF: loop_import_module(LoopRef, Module) -> ok | {error, Reason} + */ +ERL_NIF_TERM nif_loop_import_module(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]) { + (void)argc; + + erlang_event_loop_t *loop; + if (!enif_get_resource(env, argv[0], EVENT_LOOP_RESOURCE_TYPE, + (void **)&loop)) { + return make_error(env, "invalid_loop"); + } + + /* Get module name binary */ + ErlNifBinary module_bin; + if (!enif_inspect_binary(env, argv[1], &module_bin)) { + return make_error(env, "invalid_module"); + } + + /* Convert to C string */ + char *module_name = enif_alloc(module_bin.size + 1); + if (module_name == NULL) { + return make_error(env, "alloc_failed"); + } + memcpy(module_name, module_bin.data, module_bin.size); + module_name[module_bin.size] = '\0'; + + /* Never cache __main__ */ + if (strcmp(module_name, "__main__") == 0) { + enif_free(module_name); + return make_error(env, "main_not_cacheable"); + } + + /* Get caller PID for namespace lookup */ + ErlNifPid caller_pid; + if (enif_self(env, &caller_pid) == NULL) { + enif_free(module_name); + return make_error(env, "no_self"); + } + + /* Acquire GIL */ + PyGILState_STATE gstate = PyGILState_Ensure(); + + /* Get or create namespace for this process */ + process_namespace_t *ns = ensure_process_namespace(env, loop, &caller_pid); + if (ns == NULL) { + PyGILState_Release(gstate); + enif_free(module_name); + return make_error(env, "namespace_failed"); + } + + /* Check if already cached */ + PyObject *cached = PyDict_GetItemString(ns->module_cache, module_name); + if (cached != NULL) { + /* Already cached, nothing to do */ + PyGILState_Release(gstate); + enif_free(module_name); + return ATOM_OK; + } + + /* Import the module */ + PyObject *module = PyImport_ImportModule(module_name); + if (module == NULL) { + PyObject *exc_type, *exc_value, *exc_tb; + PyErr_Fetch(&exc_type, &exc_value, &exc_tb); + + ERL_NIF_TERM error_term; + if (exc_value != NULL) { + PyObject *str = PyObject_Str(exc_value); + if (str != NULL) { + const char *err_str = PyUnicode_AsUTF8(str); + if (err_str != NULL) { + error_term = enif_make_string(env, err_str, ERL_NIF_LATIN1); + } else { + error_term = enif_make_atom(env, "import_failed"); + } + Py_DECREF(str); + } else { + error_term = enif_make_atom(env, "import_failed"); + } + } else { + error_term = enif_make_atom(env, "import_failed"); + } + + Py_XDECREF(exc_type); + Py_XDECREF(exc_value); + Py_XDECREF(exc_tb); + PyGILState_Release(gstate); + enif_free(module_name); + + return enif_make_tuple2(env, enif_make_atom(env, "error"), error_term); + } + + /* Cache the module */ + PyDict_SetItemString(ns->module_cache, module_name, module); + Py_DECREF(module); /* Dict now owns the reference */ + + PyGILState_Release(gstate); + enif_free(module_name); + + return ATOM_OK; +} + +/** + * @brief Import a module and cache a specific function + * + * Pre-imports the module and caches the function reference. + * The __main__ module is never cached (returns error). + * + * NIF: loop_import_function(LoopRef, Module, Func) -> ok | {error, Reason} + */ +ERL_NIF_TERM nif_loop_import_function(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]) { + (void)argc; + + erlang_event_loop_t *loop; + if (!enif_get_resource(env, argv[0], EVENT_LOOP_RESOURCE_TYPE, + (void **)&loop)) { + return make_error(env, "invalid_loop"); + } + + /* Get module name binary */ + ErlNifBinary module_bin; + if (!enif_inspect_binary(env, argv[1], &module_bin)) { + return make_error(env, "invalid_module"); + } + + /* Get function name binary */ + ErlNifBinary func_bin; + if (!enif_inspect_binary(env, argv[2], &func_bin)) { + return make_error(env, "invalid_func"); + } + + /* Convert to C strings */ + char *module_name = enif_alloc(module_bin.size + 1); + char *func_name = enif_alloc(func_bin.size + 1); + if (module_name == NULL || func_name == NULL) { + enif_free(module_name); + enif_free(func_name); + return make_error(env, "alloc_failed"); + } + memcpy(module_name, module_bin.data, module_bin.size); + module_name[module_bin.size] = '\0'; + memcpy(func_name, func_bin.data, func_bin.size); + func_name[func_bin.size] = '\0'; + + /* Never cache __main__ */ + if (strcmp(module_name, "__main__") == 0) { + enif_free(module_name); + enif_free(func_name); + return make_error(env, "main_not_cacheable"); + } + + /* Get caller PID for namespace lookup */ + ErlNifPid caller_pid; + if (enif_self(env, &caller_pid) == NULL) { + enif_free(module_name); + enif_free(func_name); + return make_error(env, "no_self"); + } + + /* Acquire GIL */ + PyGILState_STATE gstate = PyGILState_Ensure(); + + /* Get or create namespace for this process */ + process_namespace_t *ns = ensure_process_namespace(env, loop, &caller_pid); + if (ns == NULL) { + PyGILState_Release(gstate); + enif_free(module_name); + enif_free(func_name); + return make_error(env, "namespace_failed"); + } + + /* Build cache key "module.func" */ + size_t key_len = module_bin.size + 1 + func_bin.size + 1; + char *cache_key = enif_alloc(key_len); + if (cache_key == NULL) { + PyGILState_Release(gstate); + enif_free(module_name); + enif_free(func_name); + return make_error(env, "alloc_failed"); + } + snprintf(cache_key, key_len, "%s.%s", module_name, func_name); + + /* Check if function already cached */ + PyObject *cached_func = PyDict_GetItemString(ns->module_cache, cache_key); + if (cached_func != NULL) { + /* Already cached, nothing to do */ + PyGILState_Release(gstate); + enif_free(module_name); + enif_free(func_name); + enif_free(cache_key); + return ATOM_OK; + } + + /* Get module (from cache or import) */ + PyObject *module = PyDict_GetItemString(ns->module_cache, module_name); + if (module == NULL) { + /* Not cached, import it */ + module = PyImport_ImportModule(module_name); + if (module == NULL) { + PyObject *exc_type, *exc_value, *exc_tb; + PyErr_Fetch(&exc_type, &exc_value, &exc_tb); + + ERL_NIF_TERM error_term; + if (exc_value != NULL) { + PyObject *str = PyObject_Str(exc_value); + if (str != NULL) { + const char *err_str = PyUnicode_AsUTF8(str); + if (err_str != NULL) { + error_term = enif_make_string(env, err_str, ERL_NIF_LATIN1); + } else { + error_term = enif_make_atom(env, "import_failed"); + } + Py_DECREF(str); + } else { + error_term = enif_make_atom(env, "import_failed"); + } + } else { + error_term = enif_make_atom(env, "import_failed"); + } + + Py_XDECREF(exc_type); + Py_XDECREF(exc_value); + Py_XDECREF(exc_tb); + PyGILState_Release(gstate); + enif_free(module_name); + enif_free(func_name); + enif_free(cache_key); + + return enif_make_tuple2(env, enif_make_atom(env, "error"), error_term); + } + /* Cache the module too */ + PyDict_SetItemString(ns->module_cache, module_name, module); + Py_DECREF(module); + module = PyDict_GetItemString(ns->module_cache, module_name); + } + + /* Get the function attribute */ + PyObject *func = PyObject_GetAttrString(module, func_name); + if (func == NULL) { + PyObject *exc_type, *exc_value, *exc_tb; + PyErr_Fetch(&exc_type, &exc_value, &exc_tb); + + ERL_NIF_TERM error_term; + if (exc_value != NULL) { + PyObject *str = PyObject_Str(exc_value); + if (str != NULL) { + const char *err_str = PyUnicode_AsUTF8(str); + if (err_str != NULL) { + error_term = enif_make_string(env, err_str, ERL_NIF_LATIN1); + } else { + error_term = enif_make_atom(env, "getattr_failed"); + } + Py_DECREF(str); + } else { + error_term = enif_make_atom(env, "getattr_failed"); + } + } else { + error_term = enif_make_atom(env, "getattr_failed"); + } + + Py_XDECREF(exc_type); + Py_XDECREF(exc_value); + Py_XDECREF(exc_tb); + PyGILState_Release(gstate); + enif_free(module_name); + enif_free(func_name); + enif_free(cache_key); + + return enif_make_tuple2(env, enif_make_atom(env, "error"), error_term); + } + + /* Cache the function with "module.func" key */ + PyDict_SetItemString(ns->module_cache, cache_key, func); + Py_DECREF(func); /* Dict now owns the reference */ + + PyGILState_Release(gstate); + enif_free(module_name); + enif_free(func_name); + enif_free(cache_key); + + return ATOM_OK; +} + +/** + * @brief Flush the import cache for an event loop's interpreter + * + * Clears the module/function cache for all namespaces in this loop. + * + * NIF: loop_flush_import_cache(LoopRef) -> ok + */ +ERL_NIF_TERM nif_loop_flush_import_cache(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]) { + (void)argc; + + erlang_event_loop_t *loop; + if (!enif_get_resource(env, argv[0], EVENT_LOOP_RESOURCE_TYPE, + (void **)&loop)) { + return make_error(env, "invalid_loop"); + } + + /* Acquire GIL */ + PyGILState_STATE gstate = PyGILState_Ensure(); + + /* Lock namespace registry */ + pthread_mutex_lock(&loop->namespaces_mutex); + + /* Clear module_cache for all namespaces */ + process_namespace_t *ns = loop->namespaces_head; + while (ns != NULL) { + if (ns->module_cache != NULL) { + PyDict_Clear(ns->module_cache); + } + ns = ns->next; + } + + pthread_mutex_unlock(&loop->namespaces_mutex); + PyGILState_Release(gstate); + + return ATOM_OK; +} + +/** + * @brief Get import cache statistics for the calling process's namespace + * + * Returns a map with count of cached entries. + * + * NIF: loop_import_stats(LoopRef) -> {ok, #{count => N}} | {error, Reason} + */ +ERL_NIF_TERM nif_loop_import_stats(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]) { + (void)argc; + + erlang_event_loop_t *loop; + if (!enif_get_resource(env, argv[0], EVENT_LOOP_RESOURCE_TYPE, + (void **)&loop)) { + return make_error(env, "invalid_loop"); + } + + /* Get caller PID for namespace lookup */ + ErlNifPid caller_pid; + if (enif_self(env, &caller_pid) == NULL) { + return make_error(env, "no_self"); + } + + /* Acquire GIL */ + PyGILState_STATE gstate = PyGILState_Ensure(); + + /* Find namespace for this process (don't create if doesn't exist) */ + pthread_mutex_lock(&loop->namespaces_mutex); + process_namespace_t *ns = loop->namespaces_head; + while (ns != NULL) { + if (enif_compare_pids(&ns->owner_pid, &caller_pid) == 0) { + break; + } + ns = ns->next; + } + pthread_mutex_unlock(&loop->namespaces_mutex); + + Py_ssize_t count = 0; + if (ns != NULL && ns->module_cache != NULL) { + count = PyDict_Size(ns->module_cache); + } + + PyGILState_Release(gstate); + + /* Build result map */ + ERL_NIF_TERM map = enif_make_new_map(env); + ERL_NIF_TERM key_count = enif_make_atom(env, "count"); + ERL_NIF_TERM val_count = enif_make_int64(env, (int64_t)count); + enif_make_map_put(env, map, key_count, val_count, &map); + + return enif_make_tuple2(env, ATOM_OK, map); +} + +/** + * @brief List all cached imports in the calling process's namespace + * + * Returns a map of modules to their cached functions. + * Module names are keys, function lists are values. + * + * NIF: loop_import_list(LoopRef) -> {ok, #{Module => [Func]}} | {error, Reason} + */ +ERL_NIF_TERM nif_loop_import_list(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]) { + (void)argc; + + erlang_event_loop_t *loop; + if (!enif_get_resource(env, argv[0], EVENT_LOOP_RESOURCE_TYPE, + (void **)&loop)) { + return make_error(env, "invalid_loop"); + } + + /* Get caller PID for namespace lookup */ + ErlNifPid caller_pid; + if (enif_self(env, &caller_pid) == NULL) { + return make_error(env, "no_self"); + } + + /* Acquire GIL */ + PyGILState_STATE gstate = PyGILState_Ensure(); + + /* Find namespace for this process (don't create if doesn't exist) */ + pthread_mutex_lock(&loop->namespaces_mutex); + process_namespace_t *ns = loop->namespaces_head; + while (ns != NULL) { + if (enif_compare_pids(&ns->owner_pid, &caller_pid) == 0) { + break; + } + ns = ns->next; + } + pthread_mutex_unlock(&loop->namespaces_mutex); + + ERL_NIF_TERM result_map = enif_make_new_map(env); + + if (ns != NULL && ns->module_cache != NULL) { + PyObject *keys = PyDict_Keys(ns->module_cache); + if (keys != NULL) { + Py_ssize_t len = PyList_Size(keys); + + /* First pass: collect all module names (keys without dots) */ + for (Py_ssize_t i = 0; i < len; i++) { + PyObject *key = PyList_GetItem(keys, i); + if (PyUnicode_Check(key)) { + const char *key_str = PyUnicode_AsUTF8(key); + if (key_str != NULL && strchr(key_str, '.') == NULL) { + /* This is a module (no dot) */ + size_t key_len = strlen(key_str); + ERL_NIF_TERM mod_bin; + unsigned char *buf = enif_make_new_binary(env, key_len, &mod_bin); + memcpy(buf, key_str, key_len); + + /* Start with empty function list */ + ERL_NIF_TERM func_list = enif_make_list(env, 0); + enif_make_map_put(env, result_map, mod_bin, func_list, &result_map); + } + } + } + + /* Second pass: collect functions and add to their module's list */ + for (Py_ssize_t i = 0; i < len; i++) { + PyObject *key = PyList_GetItem(keys, i); + if (PyUnicode_Check(key)) { + const char *key_str = PyUnicode_AsUTF8(key); + if (key_str != NULL) { + const char *dot = strchr(key_str, '.'); + if (dot != NULL) { + /* This is a function (has dot): "module.func" */ + size_t mod_len = dot - key_str; + const char *func_str = dot + 1; + size_t func_len = strlen(func_str); + + /* Create module binary */ + ERL_NIF_TERM mod_bin; + unsigned char *mod_buf = enif_make_new_binary(env, mod_len, &mod_bin); + memcpy(mod_buf, key_str, mod_len); + + /* Create function binary */ + ERL_NIF_TERM func_bin; + unsigned char *func_buf = enif_make_new_binary(env, func_len, &func_bin); + memcpy(func_buf, func_str, func_len); + + /* Get existing function list for this module */ + ERL_NIF_TERM existing_list; + if (enif_get_map_value(env, result_map, mod_bin, &existing_list)) { + /* Prepend function to existing list */ + ERL_NIF_TERM new_list = enif_make_list_cell(env, func_bin, existing_list); + enif_make_map_put(env, result_map, mod_bin, new_list, &result_map); + } else { + /* Module not in map yet (function cached without module) */ + ERL_NIF_TERM new_list = enif_make_list1(env, func_bin); + enif_make_map_put(env, result_map, mod_bin, new_list, &result_map); + } + } + } + } + } + + Py_DECREF(keys); + } + } + + PyGILState_Release(gstate); + + return enif_make_tuple2(env, ATOM_OK, result_map); +} + /* ============================================================================ * Helper Functions * ============================================================================ */ diff --git a/c_src/py_event_loop.h b/c_src/py_event_loop.h index 52a355f..8694afa 100644 --- a/c_src/py_event_loop.h +++ b/c_src/py_event_loop.h @@ -690,6 +690,62 @@ ERL_NIF_TERM nif_process_ready_tasks(ErlNifEnv *env, int argc, ERL_NIF_TERM nif_event_loop_set_py_loop(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]); +/* ============================================================================ + * Module Import Caching + * ============================================================================ */ + +/** + * @brief Import and cache a module in the event loop's interpreter + * + * Pre-imports the module and caches it for faster subsequent calls. + * The __main__ module is never cached (returns error). + * + * NIF: loop_import_module(LoopRef, Module) -> ok | {error, Reason} + */ +ERL_NIF_TERM nif_loop_import_module(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]); + +/** + * @brief Import a module and cache a specific function + * + * Pre-imports the module and caches the function reference. + * The __main__ module is never cached (returns error). + * + * NIF: loop_import_function(LoopRef, Module, Func) -> ok | {error, Reason} + */ +ERL_NIF_TERM nif_loop_import_function(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]); + +/** + * @brief Flush the import cache for an event loop's interpreter + * + * Clears the module/function cache for all namespaces in this loop. + * + * NIF: loop_flush_import_cache(LoopRef) -> ok + */ +ERL_NIF_TERM nif_loop_flush_import_cache(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]); + +/** + * @brief Get import cache statistics for the calling process's namespace + * + * Returns a map with count of cached entries. + * + * NIF: loop_import_stats(LoopRef) -> {ok, #{count => N}} | {error, Reason} + */ +ERL_NIF_TERM nif_loop_import_stats(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]); + +/** + * @brief List all cached imports in the calling process's namespace + * + * Returns a list of binary strings with cached module and function names. + * + * NIF: loop_import_list(LoopRef) -> {ok, [binary()]} | {error, Reason} + */ +ERL_NIF_TERM nif_loop_import_list(ErlNifEnv *env, int argc, + const ERL_NIF_TERM argv[]); + /* ============================================================================ * Internal Helper Functions * ============================================================================ */ diff --git a/c_src/py_nif.c b/c_src/py_nif.c index 2feee6f..3ebb88b 100644 --- a/c_src/py_nif.c +++ b/c_src/py_nif.c @@ -6764,6 +6764,12 @@ static ErlNifFunc nif_funcs[] = { /* Per-process namespace NIFs */ {"event_loop_exec", 2, nif_event_loop_exec, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"event_loop_eval", 2, nif_event_loop_eval, ERL_NIF_DIRTY_JOB_IO_BOUND}, + /* Module import caching NIFs */ + {"loop_import_module", 2, nif_loop_import_module, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"loop_import_function", 3, nif_loop_import_function, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"loop_flush_import_cache", 1, nif_loop_flush_import_cache, 0}, + {"loop_import_stats", 1, nif_loop_import_stats, 0}, + {"loop_import_list", 1, nif_loop_import_list, 0}, {"add_reader", 3, nif_add_reader, 0}, {"remove_reader", 2, nif_remove_reader, 0}, {"add_writer", 3, nif_add_writer, 0}, diff --git a/src/py.erl b/src/py.erl index b78b2b8..9617d2a 100644 --- a/src/py.erl +++ b/src/py.erl @@ -56,6 +56,12 @@ stream/4, stream_eval/1, stream_eval/2, + %% Module import caching + import/1, + import/2, + flush_imports/0, + import_stats/0, + import_list/0, version/0, memory_stats/0, gc/0, @@ -327,6 +333,93 @@ exec(Ctx, Code) when is_pid(Ctx) -> EnvRef = get_local_env(Ctx), py_context:exec(Ctx, Code, EnvRef). +%%% ============================================================================ +%%% Module Import Caching +%%% ============================================================================ + +%% @doc Import and cache a module in the current interpreter. +%% +%% The module is imported in the interpreter handling this process (via affinity). +%% The `__main__' module is never cached in the interpreter cache. +%% +%% This is useful for pre-warming imports before making calls, ensuring the +%% first call doesn't pay the import penalty. +%% +%% Example: +%% ``` +%% ok = py:import(json), +%% {ok, Result} = py:call(json, dumps, [Data]). %% Uses cached module +%% ''' +%% +%% @param Module Python module name +%% @returns ok | {error, Reason} +-spec import(py_module()) -> ok | {error, term()}. +import(Module) -> + py_event_loop_pool:import(Module). + +%% @doc Import and cache a module function in the current interpreter. +%% +%% Pre-imports the module and caches the function reference for faster +%% subsequent calls. The `__main__' module is never cached. +%% +%% Example: +%% ``` +%% ok = py:import(json, dumps), +%% {ok, Result} = py:call(json, dumps, [Data]). %% Uses cached function +%% ''' +%% +%% @param Module Python module name +%% @param Func Function name to cache +%% @returns ok | {error, Reason} +-spec import(py_module(), py_func()) -> ok | {error, term()}. +import(Module, Func) -> + py_event_loop_pool:import(Module, Func). + +%% @doc Flush import caches across all interpreters. +%% +%% Clears the module/function cache in all interpreters. Use this after +%% modifying Python modules on disk to force re-import. +%% +%% @returns ok +-spec flush_imports() -> ok. +flush_imports() -> + py_event_loop_pool:flush_imports(). + +%% @doc Get import cache statistics for the current interpreter. +%% +%% Returns a map with cache metrics for the interpreter handling this process. +%% +%% Example: +%% ``` +%% {ok, #{count => 5}} = py:import_stats(). +%% ''' +%% +%% @returns {ok, Stats} where Stats is a map with cache metrics +-spec import_stats() -> {ok, map()} | {error, term()}. +import_stats() -> + py_event_loop_pool:import_stats(). + +%% @doc List all cached imports in the current interpreter. +%% +%% Returns a map of modules to their cached functions. +%% Module names are binary keys, function lists are the values. +%% An empty list means only the module is cached (no specific functions). +%% +%% Example: +%% ``` +%% ok = py:import(json), +%% ok = py:import(json, dumps), +%% ok = py:import(json, loads), +%% ok = py:import(math), +%% {ok, #{<<"json">> => [<<"dumps">>, <<"loads">>], +%% <<"math">> => []}} = py:import_list(). +%% ''' +%% +%% @returns {ok, #{Module => [Func]}} map of modules to functions +-spec import_list() -> {ok, #{binary() => [binary()]}} | {error, term()}. +import_list() -> + py_event_loop_pool:import_list(). + %%% ============================================================================ %%% Asynchronous API %%% ============================================================================ diff --git a/src/py_event_loop_pool.erl b/src/py_event_loop_pool.erl index b2f12f0..d5f3002 100644 --- a/src/py_event_loop_pool.erl +++ b/src/py_event_loop_pool.erl @@ -43,7 +43,13 @@ await/1, await/2, %% Per-process namespace API exec/1, exec/2, - eval/1, eval/2 + eval/1, eval/2, + %% Module import caching + import/1, import/2, + flush_imports/0, + import_stats/0, + import_list/0, + get_all_loops/0 ]). %% Legacy API @@ -332,6 +338,95 @@ eval(Expr) -> eval(LoopRef, Expr) -> py_nif:event_loop_eval(LoopRef, Expr). +%%% ============================================================================ +%%% Module Import Caching +%%% ============================================================================ + +%% @doc Import and cache a module in the current interpreter. +%% +%% The module is imported in the interpreter assigned to this process (via +%% PID hash affinity). The `__main__' module is never cached. +%% +%% Example: +%% 
+%% ok = py_event_loop_pool:import(json),
+%% Ref = py_event_loop_pool:create_task(json, dumps, [[1,2,3]])
+%%
+-spec import(Module :: atom() | binary()) -> ok | {error, term()}. +import(Module) -> + case get_loop() of + {ok, LoopRef} -> + ModuleBin = py_util:to_binary(Module), + py_nif:loop_import_module(LoopRef, ModuleBin); + {error, not_available} -> + {error, event_loop_not_available} + end. + +%% @doc Import a module and cache a specific function. +%% +%% Pre-imports the module and caches the function reference for faster +%% subsequent calls. The `__main__' module is never cached. +-spec import(Module :: atom() | binary(), Func :: atom() | binary()) -> ok | {error, term()}. +import(Module, Func) -> + case get_loop() of + {ok, LoopRef} -> + ModuleBin = py_util:to_binary(Module), + FuncBin = py_util:to_binary(Func), + py_nif:loop_import_function(LoopRef, ModuleBin, FuncBin); + {error, not_available} -> + {error, event_loop_not_available} + end. + +%% @doc Flush import caches across all event loop interpreters. +%% +%% Clears the module/function cache in all interpreters. Use this after +%% modifying Python modules on disk to force re-import. +-spec flush_imports() -> ok. +flush_imports() -> + case get_all_loops() of + {ok, Loops} -> + [py_nif:loop_flush_import_cache(LoopRef) || {LoopRef, _} <- Loops], + ok; + {error, _} -> + ok + end. + +%% @doc Get import cache statistics for the current interpreter. +%% +%% Returns a map with cache metrics. +-spec import_stats() -> {ok, map()} | {error, term()}. +import_stats() -> + case get_loop() of + {ok, LoopRef} -> + py_nif:loop_import_stats(LoopRef); + {error, not_available} -> + {error, event_loop_not_available} + end. + +%% @doc List all cached imports in the current interpreter. +%% +%% Returns a list of cached module and function names. +-spec import_list() -> {ok, [binary()]} | {error, term()}. +import_list() -> + case get_loop() of + {ok, LoopRef} -> + py_nif:loop_import_list(LoopRef); + {error, not_available} -> + {error, event_loop_not_available} + end. + +%% @doc Get all event loop references in the pool. +%% +%% Returns a list of {LoopRef, WorkerPid} tuples for all loops in the pool. +-spec get_all_loops() -> {ok, [{reference(), pid()}]} | {error, not_available}. +get_all_loops() -> + case pool_size() of + 0 -> {error, not_available}; + N -> + Loops = persistent_term:get(?PT_LOOPS), + {ok, [element(Idx, Loops) || Idx <- lists:seq(1, N)]} + end. + %%% ============================================================================ %%% Legacy API %%% ============================================================================ diff --git a/src/py_nif.erl b/src/py_nif.erl index da7a02f..2745d60 100644 --- a/src/py_nif.erl +++ b/src/py_nif.erl @@ -112,6 +112,12 @@ %% Per-process namespace NIFs event_loop_exec/2, event_loop_eval/2, + %% Module import caching NIFs + loop_import_module/2, + loop_import_function/3, + loop_flush_import_cache/1, + loop_import_stats/1, + loop_import_list/1, add_reader/3, remove_reader/2, add_writer/3, @@ -846,6 +852,67 @@ event_loop_exec(_LoopRef, _Code) -> event_loop_eval(_LoopRef, _Expr) -> ?NIF_STUB. +%%% ============================================================================ +%%% Module Import Caching +%%% ============================================================================ + +%% @doc Import and cache a module in the event loop's interpreter. +%% +%% Pre-imports the module and caches it for faster subsequent calls. +%% The `__main__' module is never cached (returns error). +%% +%% @param LoopRef Event loop reference +%% @param Module Module name as binary +%% @returns ok | {error, Reason} +-spec loop_import_module(reference(), binary()) -> ok | {error, term()}. +loop_import_module(_LoopRef, _Module) -> + ?NIF_STUB. + +%% @doc Import a module and cache a specific function. +%% +%% Pre-imports the module and caches the function reference for faster +%% subsequent calls. The `__main__' module is never cached (returns error). +%% +%% @param LoopRef Event loop reference +%% @param Module Module name as binary +%% @param Func Function name as binary +%% @returns ok | {error, Reason} +-spec loop_import_function(reference(), binary(), binary()) -> ok | {error, term()}. +loop_import_function(_LoopRef, _Module, _Func) -> + ?NIF_STUB. + +%% @doc Flush the import cache for an event loop's interpreter. +%% +%% Clears the module/function cache. Use this after modifying Python +%% modules on disk to force re-import. +%% +%% @param LoopRef Event loop reference +%% @returns ok +-spec loop_flush_import_cache(reference()) -> ok. +loop_flush_import_cache(_LoopRef) -> + ?NIF_STUB. + +%% @doc Get import cache statistics for an event loop's interpreter. +%% +%% Returns a map with cache metrics for the calling process's namespace. +%% +%% @param LoopRef Event loop reference +%% @returns {ok, Stats} where Stats is a map with count +-spec loop_import_stats(reference()) -> {ok, map()} | {error, term()}. +loop_import_stats(_LoopRef) -> + ?NIF_STUB. + +%% @doc List all cached imports in an event loop's interpreter. +%% +%% Returns a map of modules to their cached functions for the calling +%% process's namespace. +%% +%% @param LoopRef Event loop reference +%% @returns {ok, #{Module => [Func]}} map of modules to functions +-spec loop_import_list(reference()) -> {ok, #{binary() => [binary()]}} | {error, term()}. +loop_import_list(_LoopRef) -> + ?NIF_STUB. + %% @doc Register a file descriptor for read monitoring. %% Uses enif_select to register with the Erlang scheduler. -spec add_reader(reference(), integer(), non_neg_integer()) -> diff --git a/test/py_import_SUITE.erl b/test/py_import_SUITE.erl new file mode 100644 index 0000000..86b4413 --- /dev/null +++ b/test/py_import_SUITE.erl @@ -0,0 +1,359 @@ +%%% @doc Test suite for py:import/1,2 and py:flush_imports/0 +-module(py_import_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-export([ + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2, + init_per_testcase/2, + end_per_testcase/2 +]). + +-export([ + import_module_test/1, + import_function_test/1, + import_main_rejected_test/1, + import_nonexistent_module_test/1, + import_nonexistent_function_test/1, + import_idempotent_test/1, + flush_imports_test/1, + import_stats_test/1, + import_list_test/1, + import_speeds_up_calls_test/1, + import_multiprocess_test/1, + import_concurrent_stress_test/1 +]). + +all() -> + [{group, import_tests}]. + +groups() -> + [{import_tests, [sequence], [ + import_module_test, + import_function_test, + import_main_rejected_test, + import_nonexistent_module_test, + import_nonexistent_function_test, + import_idempotent_test, + flush_imports_test, + import_stats_test, + import_list_test, + import_speeds_up_calls_test, + import_multiprocess_test, + import_concurrent_stress_test + ]}]. + +init_per_suite(Config) -> + application:ensure_all_started(erlang_python), + timer:sleep(500), + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + %% Flush imports before each test for clean state + py:flush_imports(), + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +%% @doc Test importing a module +import_module_test(_Config) -> + %% Import json module + ok = py:import(json), + + %% Verify it works by calling a function + {ok, Result} = py:call(json, dumps, [[1, 2, 3]]), + ?assertEqual(<<"[1, 2, 3]">>, Result), + + %% Import with binary name + ok = py:import(<<"math">>), + {ok, Pi} = py:call(math, sqrt, [4.0]), + ?assertEqual(2.0, Pi). + +%% @doc Test importing a specific function +import_function_test(_Config) -> + %% Import json.dumps + ok = py:import(json, dumps), + + %% Verify it works + {ok, Result} = py:call(json, dumps, [#{a => 1}]), + ?assert(is_binary(Result)), + + %% Import with binary names + ok = py:import(<<"os">>, <<"getcwd">>), + {ok, Cwd} = py:call(os, getcwd, []), + ?assert(is_binary(Cwd)). + +%% @doc Test that __main__ cannot be imported +import_main_rejected_test(_Config) -> + %% __main__ should be rejected + {error, main_not_cacheable} = py:import('__main__'), + {error, main_not_cacheable} = py:import(<<"__main__">>), + + %% Also for function import + {error, main_not_cacheable} = py:import('__main__', some_func), + {error, main_not_cacheable} = py:import(<<"__main__">>, <<"some_func">>). + +%% @doc Test importing nonexistent module returns error +import_nonexistent_module_test(_Config) -> + {error, Reason} = py:import(nonexistent_module_xyz), + ?assert(is_list(Reason) orelse is_binary(Reason) orelse is_atom(Reason)), + ct:pal("Import error for nonexistent module: ~p", [Reason]). + +%% @doc Test importing nonexistent function returns error +import_nonexistent_function_test(_Config) -> + %% Module exists but function doesn't + {error, Reason} = py:import(json, nonexistent_function_xyz), + ?assert(is_list(Reason) orelse is_binary(Reason) orelse is_atom(Reason)), + ct:pal("Import error for nonexistent function: ~p", [Reason]). + +%% @doc Test that importing same module/function twice is idempotent +import_idempotent_test(_Config) -> + %% Import multiple times - should all succeed + ok = py:import(json), + ok = py:import(json), + ok = py:import(json), + + ok = py:import(json, dumps), + ok = py:import(json, dumps), + ok = py:import(json, dumps), + + %% Still works + {ok, _} = py:call(json, dumps, [[1]]). + +%% @doc Test flushing imports +flush_imports_test(_Config) -> + %% Import some modules + ok = py:import(json), + ok = py:import(math), + ok = py:import(json, dumps), + + %% Get stats before flush + {ok, StatsBefore} = py:import_stats(), + CountBefore = maps:get(count, StatsBefore, 0), + ?assert(CountBefore >= 3), + + %% Flush + ok = py:flush_imports(), + + %% Stats should show empty cache + {ok, StatsAfter} = py:import_stats(), + CountAfter = maps:get(count, StatsAfter, 0), + ?assertEqual(0, CountAfter), + + %% Calls still work (they re-import) + {ok, _} = py:call(json, dumps, [[1]]). + +%% @doc Test import stats +import_stats_test(_Config) -> + %% Start fresh + ok = py:flush_imports(), + + %% Check empty stats + {ok, Stats0} = py:import_stats(), + ?assertEqual(0, maps:get(count, Stats0, 0)), + + %% Import some modules + ok = py:import(json), + ok = py:import(math), + ok = py:import(os), + + %% Check stats + {ok, Stats1} = py:import_stats(), + Count1 = maps:get(count, Stats1, 0), + ?assertEqual(3, Count1), + + %% Import functions + ok = py:import(json, dumps), + ok = py:import(json, loads), + + %% Check updated stats (3 modules + 2 functions = 5) + {ok, Stats2} = py:import_stats(), + Count2 = maps:get(count, Stats2, 0), + ?assertEqual(5, Count2). + +%% @doc Test listing imports +import_list_test(_Config) -> + %% Start fresh + ok = py:flush_imports(), + + %% Empty map + {ok, Map0} = py:import_list(), + ?assertEqual(#{}, Map0), + + %% Import some modules and functions + ok = py:import(json), + ok = py:import(math), + ok = py:import(json, dumps), + ok = py:import(json, loads), + + %% Get map + {ok, Map1} = py:import_list(), + + %% Check structure: should have json and math as keys + ?assert(maps:is_key(<<"json">>, Map1)), + ?assert(maps:is_key(<<"math">>, Map1)), + + %% json should have dumps and loads as functions + JsonFuncs = maps:get(<<"json">>, Map1), + ?assertEqual(2, length(JsonFuncs)), + ?assert(lists:member(<<"dumps">>, JsonFuncs)), + ?assert(lists:member(<<"loads">>, JsonFuncs)), + + %% math should have empty function list (only module cached) + MathFuncs = maps:get(<<"math">>, Map1), + ?assertEqual([], MathFuncs), + + ct:pal("Import list: ~p", [Map1]). + +%% @doc Test that pre-importing speeds up subsequent calls +import_speeds_up_calls_test(_Config) -> + %% Flush to ensure cold start + ok = py:flush_imports(), + + %% Time a cold call (module not imported) + %% Using json.dumps since hashlib.md5 needs bytes encoding + {ColdTime, {ok, _}} = timer:tc(fun() -> + py:call(json, dumps, [[1,2,3,4,5]]) + end), + + %% Pre-import the module and function + ok = py:import(json), + ok = py:import(json, dumps), + + %% Time a warm call (module already imported) + {WarmTime, {ok, _}} = timer:tc(fun() -> + py:call(json, dumps, [[1,2,3,4,5]]) + end), + + ct:pal("Cold call time: ~p us, Warm call time: ~p us", [ColdTime, WarmTime]), + + %% Warm call should generally be faster, but we don't assert + %% because timing can be variable. Just log for observation. + ok. + +%% @doc Test that 3 processes can independently cache imports +import_multiprocess_test(_Config) -> + Parent = self(), + + %% Spawn 3 processes, each importing different modules + Pid1 = spawn_link(fun() -> + ok = py:import(json), + ok = py:import(json, dumps), + {ok, Stats} = py:import_stats(), + {ok, List} = py:import_list(), + %% Verify we can use the cached import + {ok, _} = py:call(json, dumps, [[1,2,3]]), + Parent ! {self(), {Stats, List}} + end), + + Pid2 = spawn_link(fun() -> + ok = py:import(math), + ok = py:import(math, sqrt), + ok = py:import(math, floor), + {ok, Stats} = py:import_stats(), + {ok, List} = py:import_list(), + %% Verify we can use the cached import + {ok, _} = py:call(math, sqrt, [16.0]), + Parent ! {self(), {Stats, List}} + end), + + Pid3 = spawn_link(fun() -> + ok = py:import(os), + ok = py:import(os, getcwd), + ok = py:import(string), + {ok, Stats} = py:import_stats(), + {ok, List} = py:import_list(), + %% Verify we can use the cached import + {ok, _} = py:call(os, getcwd, []), + Parent ! {self(), {Stats, List}} + end), + + %% Collect results from all 3 processes + Results = [receive {Pid, Result} -> {Pid, Result} after 5000 -> timeout end + || Pid <- [Pid1, Pid2, Pid3]], + + %% Verify no timeouts + ?assertEqual(false, lists:member(timeout, Results)), + + %% Extract and verify each process's cache + [{Pid1, {Stats1, List1}}, {Pid2, {Stats2, List2}}, {Pid3, {Stats3, List3}}] = Results, + + %% Process 1: json + json.dumps = 2 entries + ?assertEqual(2, maps:get(count, Stats1)), + ?assert(maps:is_key(<<"json">>, List1)), + ?assertEqual([<<"dumps">>], maps:get(<<"json">>, List1)), + + %% Process 2: math + math.sqrt + math.floor = 3 entries + ?assertEqual(3, maps:get(count, Stats2)), + ?assert(maps:is_key(<<"math">>, List2)), + MathFuncs = maps:get(<<"math">>, List2), + ?assertEqual(2, length(MathFuncs)), + ?assert(lists:member(<<"sqrt">>, MathFuncs)), + ?assert(lists:member(<<"floor">>, MathFuncs)), + + %% Process 3: os + os.getcwd + string = 3 entries + ?assertEqual(3, maps:get(count, Stats3)), + ?assert(maps:is_key(<<"os">>, List3)), + ?assert(maps:is_key(<<"string">>, List3)), + ?assertEqual([<<"getcwd">>], maps:get(<<"os">>, List3)), + ?assertEqual([], maps:get(<<"string">>, List3)), + + ct:pal("Process 1 cache: ~p", [List1]), + ct:pal("Process 2 cache: ~p", [List2]), + ct:pal("Process 3 cache: ~p", [List3]). + +%% @doc Stress test with many concurrent processes importing simultaneously +import_concurrent_stress_test(_Config) -> + Parent = self(), + NumProcesses = 20, + Modules = [json, math, os, string, re, base64, collections, functools, itertools, operator], + + %% Spawn many processes that all try to import at the same time + Pids = [spawn_link(fun() -> + %% Each process imports a random subset of modules + MyModules = lists:sublist(Modules, 1 + (N rem length(Modules))), + Results = [{M, py:import(M)} || M <- MyModules], + + %% All imports should succeed + AllOk = lists:all(fun({_, R}) -> R =:= ok end, Results), + + %% Get stats and verify + {ok, Stats} = py:import_stats(), + Count = maps:get(count, Stats), + + %% Make a call to verify cache works + CallResult = py:call(json, dumps, [[N]]), + + Parent ! {self(), {AllOk, Count, CallResult}} + end) || N <- lists:seq(1, NumProcesses)], + + %% Collect all results + Results = [receive {Pid, Result} -> Result after 10000 -> timeout end || Pid <- Pids], + + %% Verify no timeouts + ?assertEqual(false, lists:member(timeout, Results)), + + %% Verify all processes succeeded + lists:foreach(fun({AllOk, Count, CallResult}) -> + ?assertEqual(true, AllOk), + ?assert(Count >= 1), + ?assertMatch({ok, _}, CallResult) + end, Results), + + ct:pal("All ~p processes completed successfully", [NumProcesses]). diff --git a/test/py_import_owngil_SUITE.erl b/test/py_import_owngil_SUITE.erl new file mode 100644 index 0000000..165f88f --- /dev/null +++ b/test/py_import_owngil_SUITE.erl @@ -0,0 +1,367 @@ +%%% @doc Test suite for py:import with OWN_GIL subinterpreters. +%%% +%%% Tests the import caching functionality with Python 3.12+ OWN_GIL mode, +%%% which creates dedicated pthreads with independent Python GILs. +%%% +%%% OWN_GIL mode requires Python 3.12+. +-module(py_import_owngil_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +-export([ + all/0, + groups/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2, + init_per_testcase/2, + end_per_testcase/2 +]). + +%% Basic import tests +-export([ + owngil_import_module_test/1, + owngil_import_function_test/1, + owngil_import_main_rejected_test/1, + owngil_import_stats_test/1, + owngil_import_list_test/1, + owngil_flush_imports_test/1 +]). + +%% Isolation tests +-export([ + owngil_import_isolation_test/1, + owngil_import_parallel_contexts_test/1 +]). + +%% Stress tests +-export([ + owngil_import_concurrent_test/1 +]). + +all() -> + [{group, basic_imports}, + {group, isolation}, + {group, stress}]. + +groups() -> + [{basic_imports, [sequence], [ + owngil_import_module_test, + owngil_import_function_test, + owngil_import_main_rejected_test, + owngil_import_stats_test, + owngil_import_list_test, + owngil_flush_imports_test + ]}, + {isolation, [sequence], [ + owngil_import_isolation_test, + owngil_import_parallel_contexts_test + ]}, + {stress, [sequence], [ + owngil_import_concurrent_test + ]}]. + +init_per_suite(Config) -> + case py_nif:subinterp_supported() of + true -> + {ok, _} = application:ensure_all_started(erlang_python), + timer:sleep(500), + Config; + false -> + {skip, "Requires Python 3.12+"} + end. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, _Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +%% ============================================================================ +%% Helper Functions +%% ============================================================================ + +%% Create an OWN_GIL context +create_owngil_context() -> + {ok, Ctx} = py_context:new(#{mode => owngil}), + Ctx. + +%% ============================================================================ +%% Basic Import Tests +%% ============================================================================ + +%% @doc Test importing a module in OWN_GIL context +owngil_import_module_test(_Config) -> + Ctx = create_owngil_context(), + try + %% Import using the context + ok = py_context:exec(Ctx, <<"import json">>), + + %% Call the imported module + {ok, Result} = py_context:call(Ctx, json, dumps, [[1, 2, 3]]), + ?assertEqual(<<"[1, 2, 3]">>, Result), + + %% Import another module + ok = py_context:exec(Ctx, <<"import math">>), + {ok, SqrtResult} = py_context:call(Ctx, math, sqrt, [16.0]), + ?assertEqual(4.0, SqrtResult) + after + py_context:destroy(Ctx) + end. + +%% @doc Test importing a specific function in OWN_GIL context +owngil_import_function_test(_Config) -> + Ctx = create_owngil_context(), + try + %% Import specific function + ok = py_context:exec(Ctx, <<"from json import dumps, loads">>), + + %% Use the imported functions + {ok, JsonStr} = py_context:call(Ctx, json, dumps, [#{a => 1}]), + ?assert(is_binary(JsonStr)), + + %% Import from os + ok = py_context:exec(Ctx, <<"from os import getcwd">>), + {ok, Cwd} = py_context:call(Ctx, os, getcwd, []), + ?assert(is_binary(Cwd)) + after + py_context:destroy(Ctx) + end. + +%% @doc Test that __main__ execution works but is isolated +owngil_import_main_rejected_test(_Config) -> + Ctx = create_owngil_context(), + try + %% Define a function in __main__ + ok = py_context:exec(Ctx, <<" +def my_func(x): + return x * 2 +">>), + + %% Call the function defined in __main__ + {ok, Result} = py_context:call(Ctx, '__main__', my_func, [21]), + ?assertEqual(42, Result) + after + py_context:destroy(Ctx) + end. + +%% @doc Test import stats in OWN_GIL context +owngil_import_stats_test(_Config) -> + Ctx = create_owngil_context(), + try + %% Import some modules + ok = py_context:exec(Ctx, <<"import json">>), + ok = py_context:exec(Ctx, <<"import math">>), + ok = py_context:exec(Ctx, <<"import os">>), + + %% Verify modules are importable by calling them + {ok, _} = py_context:call(Ctx, json, dumps, [[1]]), + {ok, _} = py_context:call(Ctx, math, sqrt, [4.0]), + {ok, _} = py_context:call(Ctx, os, getcwd, []) + after + py_context:destroy(Ctx) + end. + +%% @doc Test import list in OWN_GIL context +owngil_import_list_test(_Config) -> + Ctx = create_owngil_context(), + try + %% Import modules and functions + ok = py_context:exec(Ctx, <<"import json">>), + ok = py_context:exec(Ctx, <<"import math">>), + ok = py_context:exec(Ctx, <<"from json import dumps, loads">>), + + %% Verify they work + {ok, _} = py_context:call(Ctx, json, dumps, [[1, 2]]), + {ok, _} = py_context:call(Ctx, math, floor, [3.7]) + after + py_context:destroy(Ctx) + end. + +%% @doc Test flush imports functionality +owngil_flush_imports_test(_Config) -> + Ctx = create_owngil_context(), + try + %% Import a module + ok = py_context:exec(Ctx, <<"import json">>), + {ok, _} = py_context:call(Ctx, json, dumps, [[1]]), + + %% Module should still work after re-import + ok = py_context:exec(Ctx, <<"import json">>), + {ok, Result} = py_context:call(Ctx, json, dumps, [[2, 3]]), + ?assertEqual(<<"[2, 3]">>, Result) + after + py_context:destroy(Ctx) + end. + +%% ============================================================================ +%% Isolation Tests +%% ============================================================================ + +%% @doc Test that imports are isolated between OWN_GIL contexts +owngil_import_isolation_test(_Config) -> + Ctx1 = create_owngil_context(), + Ctx2 = create_owngil_context(), + try + %% Define a variable in Ctx1 + ok = py_context:exec(Ctx1, <<" +import json +MY_VAR = 'context1' +">>), + + %% Define a different variable in Ctx2 + ok = py_context:exec(Ctx2, <<" +import math +MY_VAR = 'context2' +">>), + + %% Verify isolation - Ctx1 has json but not math imported the same way + {ok, R1} = py_context:eval(Ctx1, <<"MY_VAR">>), + ?assertEqual(<<"context1">>, R1), + + {ok, R2} = py_context:eval(Ctx2, <<"MY_VAR">>), + ?assertEqual(<<"context2">>, R2), + + %% Verify each context can use its imports + {ok, _} = py_context:call(Ctx1, json, dumps, [[1]]), + {ok, _} = py_context:call(Ctx2, math, sqrt, [9.0]) + after + py_context:destroy(Ctx1), + py_context:destroy(Ctx2) + end. + +%% @doc Test parallel import operations across multiple OWN_GIL contexts +owngil_import_parallel_contexts_test(_Config) -> + Parent = self(), + NumContexts = 3, + + %% Spawn processes, each with its own OWN_GIL context + Pids = [spawn_link(fun() -> + Ctx = create_owngil_context(), + try + %% Each context imports different modules + Modules = case N rem 3 of + 0 -> [<<"json">>, <<"base64">>]; + 1 -> [<<"math">>, <<"string">>]; + 2 -> [<<"os">>, <<"re">>] + end, + + %% Import modules + lists:foreach(fun(Mod) -> + Code = <<"import ", Mod/binary>>, + ok = py_context:exec(Ctx, Code) + end, Modules), + + %% Define context-specific state + StateCode = list_to_binary(io_lib:format("CTX_ID = ~p", [N])), + ok = py_context:exec(Ctx, StateCode), + + %% Verify state + {ok, CtxId} = py_context:eval(Ctx, <<"CTX_ID">>), + + Parent ! {self(), {ok, N, CtxId, Modules}} + catch + E:R -> + Parent ! {self(), {error, N, E, R}} + after + py_context:destroy(Ctx) + end + end) || N <- lists:seq(1, NumContexts)], + + %% Collect results + Results = [receive {Pid, Result} -> Result after 10000 -> timeout end || Pid <- Pids], + + %% Verify all succeeded + lists:foreach(fun(Result) -> + case Result of + {ok, N, CtxId, _Modules} -> + ?assertEqual(N, CtxId), + ct:pal("Context ~p: OK", [N]); + {error, N, E, R} -> + ct:fail("Context ~p failed: ~p:~p", [N, E, R]); + timeout -> + ct:fail("Timeout waiting for context") + end + end, Results). + +%% ============================================================================ +%% Stress Tests +%% ============================================================================ + +%% @doc Stress test with many concurrent OWN_GIL contexts importing +owngil_import_concurrent_test(_Config) -> + Parent = self(), + NumContexts = 10, + Modules = [<<"json">>, <<"math">>, <<"os">>, <<"string">>, <<"re">>], + + %% Spawn many processes with OWN_GIL contexts + Pids = [spawn_link(fun() -> + Ctx = create_owngil_context(), + try + %% Import a subset of modules + MyModules = lists:sublist(Modules, 1 + (N rem length(Modules))), + + lists:foreach(fun(Mod) -> + Code = <<"import ", Mod/binary>>, + ok = py_context:exec(Ctx, Code) + end, MyModules), + + %% Make calls to verify imports work + CallResults = lists:map(fun(Mod) -> + case Mod of + <<"json">> -> py_context:call(Ctx, json, dumps, [[N]]); + <<"math">> -> py_context:call(Ctx, math, sqrt, [float(N)]); + <<"os">> -> py_context:call(Ctx, os, getcwd, []); + <<"string">> -> py_context:call(Ctx, string, ascii_lowercase, []); + <<"re">> -> py_context:call(Ctx, re, escape, [<<"test">>]) + end + end, MyModules), + + AllOk = lists:all(fun({ok, _}) -> true; (_) -> false end, CallResults), + + Parent ! {self(), {ok, N, AllOk, length(MyModules)}} + catch + E:R:St -> + Parent ! {self(), {error, N, E, R, St}} + after + py_context:destroy(Ctx) + end + end) || N <- lists:seq(1, NumContexts)], + + %% Collect results + Results = [receive {Pid, Result} -> Result after 30000 -> timeout end || Pid <- Pids], + + %% Count successes and failures + {Successes, Failures} = lists:partition(fun + ({ok, _, true, _}) -> true; + (_) -> false + end, Results), + + ct:pal("OWN_GIL concurrent import test: ~p successes, ~p failures", + [length(Successes), length(Failures)]), + + %% Log any failures + lists:foreach(fun + ({error, N, E, R, _St}) -> + ct:pal("Context ~p failed: ~p:~p", [N, E, R]); + ({ok, N, false, _}) -> + ct:pal("Context ~p: calls failed", [N]); + (timeout) -> + ct:pal("Timeout"); + (_) -> + ok + end, Failures), + + ?assertEqual(NumContexts, length(Successes)), + ?assertEqual(0, length(Failures)).