diff --git a/Doxyfile.in b/Doxyfile.in index c58162f5..0d0b9d11 100644 --- a/Doxyfile.in +++ b/Doxyfile.in @@ -1872,6 +1872,16 @@ ALIASES += DOC_DESC_ATOMICITY_XSTREAM_STATE="Management of states of execution s ALIASES += DOC_DESC_POOL_AUTOMATIC{1}="If \1 is not configured to be automatically freed, it is the user's responsibility to free \1 after its use unless \c 1 is associated with the main scheduler of the primary execution stream. If \1 is configured to be automatically freed, \1 is automatically freed when all schedulers associated with \1 is freed. If the user never associates \1 with a scheduler (e.g., by \c ABT_sched_create()), it is the user's responsibility to free \1." +ALIASES += DOC_DESC_POOL_CONTEXT_POP_PUSH="This context value is intended for both pop and push operations." + +ALIASES += DOC_DESC_POOL_CONTEXT_PUSH="This context value is intended for push operations." + +ALIASES += DOC_DESC_POOL_CONTEXT_IMPL{1}="The routine must perform the required functionality regardless of the value of \1." + +ALIASES += DOC_DESC_POOL_CONTEXT_IMPL_DEF_OWNER="Each pool implementation defines its specific behavior. Some pool implementations might not respect the user's owner setting." + +ALIASES += DOC_DESC_POOL_CONTEXT_IMPL_DEF_PRIO="Each pool implementation defines its specific behavior. Some pool implementations might not respect the user's priority setting." + ALIASES += DOC_DESC_SCHED_AUTOMATIC{1}="If \1 is not configured to be automatically freed, it is the user's responsibility to free \1 after its use unless \c 1 is associated with the main scheduler of the primary execution stream. If \1 is configured to be automatically freed, \1 is automatically freed when a work unit associated with \1 is freed. If the user never associates \1 with a work unit (e.g., by \c ABT_pool_add_sched() or \c ABT_xstream_set_main_sched()), it is the user's responsibility to free \1." ALIASES += DOC_DESC_TIMER_RESOLUTION="The resolution of elapsed time depends on the clock resolution of the system." @@ -2040,6 +2050,10 @@ ALIASES += DOC_ERROR_INV_POOL_KIND{1}="\c ABT_ERR_INV_POOL_KIND is returned if \ ALIASES += DOC_ERROR_INV_POOL_PTR{1}="\c ABT_ERR_INV_POOL is returned if \1 points to \c ABT_POOL_NULL.\n" +ALIASES += DOC_ERROR_INV_POOL_USER_DEF_HANDLE{1}="\c ABT_ERR_INV_POOL_USER_DEF is returned if \1 is \c ABT_POOL_USER_DEF_NULL.\n" + +ALIASES += DOC_ERROR_INV_POOL_USER_DEF_PTR{1}="\c ABT_ERR_INV_POOL_USER_DEF is returned if \1 points to \c ABT_POOL_USER_DEF_NULL.\n" + ALIASES += DOC_ERROR_INV_RWLOCK_HANDLE{1}="\c ABT_ERR_INV_RWLOCK is returned if \1 is \c ABT_RWLOCK_NULL.\n" ALIASES += DOC_ERROR_INV_RWLOCK_PTR{1}="\c ABT_ERR_INV_RWLOCK is returned if \1 points to \c ABT_RWLOCK_NULL.\n" diff --git a/examples/scheduling/sched_and_pool_user.c b/examples/scheduling/sched_and_pool_user.c index 20f88d1a..c4c52479 100644 --- a/examples/scheduling/sched_and_pool_user.c +++ b/examples/scheduling/sched_and_pool_user.c @@ -6,13 +6,14 @@ #include #include #include +#include #include "abt.h" #define NUM_XSTREAMS 4 #define NUM_THREADS 4 +static void create_pools(int num, ABT_pool *pools); static void create_scheds(int num, ABT_pool *pools, ABT_sched *scheds); -static int example_pool_get_def(ABT_pool_access access, ABT_pool_def *p_def); static void create_threads(void *arg); static void thread_hello(void *arg); @@ -22,16 +23,12 @@ int main(int argc, char *argv[]) ABT_sched scheds[NUM_XSTREAMS]; ABT_pool pools[NUM_XSTREAMS]; ABT_thread threads[NUM_XSTREAMS]; - ABT_pool_def pool_def; int i; ABT_init(argc, argv); /* Create pools */ - example_pool_get_def(ABT_POOL_ACCESS_MPMC, &pool_def); - for (i = 0; i < NUM_XSTREAMS; i++) { - ABT_pool_create(&pool_def, ABT_POOL_CONFIG_NULL, &pools[i]); - } + create_pools(NUM_XSTREAMS, pools); /* Create schedulers */ create_scheds(NUM_XSTREAMS, pools, scheds); @@ -229,364 +226,155 @@ static void thread_hello(void *arg) printf(" [U%d:E%d] Goodbye, world!%s\n", tid, cur_rank, msg); } -/* FIFO pool implementation +/* + * A simple random work-stealing pool implementation that is often used by + * Cilk-like task parallel runtime systems. * - * Based on src/pool/fifo.c, but modified to avoid the use of internal data - * structures. + * Create push Non-create push + * \ / + * (head) <- <- <- (tail) + * / \ + * Local pop Remote pop */ +typedef struct unit_t unit_t; +typedef struct pool_t pool_t; -/*******************************************************************/ - -struct example_unit { - struct example_unit *p_prev; - struct example_unit *p_next; - ABT_pool pool; - ABT_thread thread; +struct unit_t { + unit_t *p_prev; + unit_t *p_next; }; -struct example_pool_data { - ABT_mutex mutex; - size_t num_units; - struct example_unit *p_head; - struct example_unit *p_tail; +struct pool_t { + pthread_mutex_t lock; + unit_t *p_head; + unit_t *p_tail; }; -static int pool_init(ABT_pool pool, ABT_pool_config config); -static int pool_free(ABT_pool pool); -static size_t pool_get_size(ABT_pool pool); -static void pool_push_shared(ABT_pool pool, ABT_unit unit); -static void pool_push_private(ABT_pool pool, ABT_unit unit); -static ABT_unit pool_pop_shared(ABT_pool pool); -static ABT_unit pool_pop_private(ABT_pool pool); -static int pool_remove_shared(ABT_pool pool, ABT_unit unit); -static int pool_remove_private(ABT_pool pool, ABT_unit unit); - -typedef struct example_unit unit_t; -static ABT_bool unit_is_in_pool(ABT_unit unit); -static ABT_unit unit_create_from_thread(ABT_thread thread); -static void unit_free(ABT_unit *unit); - -typedef struct example_pool_data data_t; - -static inline data_t *pool_get_data_ptr(void *p_data) -{ - return (data_t *)p_data; -} - -/* Obtain the FIFO pool definition according to the access type */ -static int example_pool_get_def(ABT_pool_access access, ABT_pool_def *p_def) -{ - int abt_errno = ABT_SUCCESS; - - /* Definitions according to the access type */ - /* FIXME: need better implementation, e.g., lock-free one */ - switch (access) { - case ABT_POOL_ACCESS_PRIV: - p_def->p_push = pool_push_private; - p_def->p_pop = pool_pop_private; - p_def->p_remove = pool_remove_private; - break; - - case ABT_POOL_ACCESS_SPSC: - case ABT_POOL_ACCESS_MPSC: - case ABT_POOL_ACCESS_SPMC: - case ABT_POOL_ACCESS_MPMC: - p_def->p_push = pool_push_shared; - p_def->p_pop = pool_pop_shared; - p_def->p_remove = pool_remove_shared; - break; - - default: - return ABT_ERR_INV_POOL_ACCESS; - } - - /* Common definitions regardless of the access type */ - p_def->access = access; - p_def->p_init = pool_init; - p_def->p_free = pool_free; - p_def->p_get_size = pool_get_size; - p_def->u_get_type = NULL; /* Unused. */ - p_def->u_get_thread = NULL; /* Unused. */ - p_def->u_get_task = NULL; /* Unused. */ - p_def->u_is_in_pool = unit_is_in_pool; - p_def->u_create_from_thread = unit_create_from_thread; - p_def->u_create_from_task = NULL; /* Unused. */ - p_def->u_free = unit_free; - /* Unimplemented. */ - p_def->p_pop_timedwait = NULL; - p_def->p_print_all = NULL; - - return abt_errno; -} - /* Pool functions */ - -int pool_init(ABT_pool pool, ABT_pool_config config) +static ABT_unit pool_create_unit(ABT_pool pool, ABT_thread thread) { - int abt_errno = ABT_SUCCESS; - ABT_pool_access access; - - data_t *p_data = (data_t *)malloc(sizeof(data_t)); - if (!p_data) - return ABT_ERR_MEM; - - ABT_pool_get_access(pool, &access); - - p_data->mutex = ABT_MUTEX_NULL; - if (access != ABT_POOL_ACCESS_PRIV) { - /* Initialize the mutex */ - ABT_mutex_create(&p_data->mutex); - } - - p_data->num_units = 0; - p_data->p_head = NULL; - p_data->p_tail = NULL; - - ABT_pool_set_data(pool, p_data); - - return abt_errno; -} - -static int pool_free(ABT_pool pool) -{ - int abt_errno = ABT_SUCCESS; - void *data; - ABT_pool_get_data(pool, &data); - data_t *p_data = pool_get_data_ptr(data); - if (p_data->mutex != ABT_MUTEX_NULL) { - ABT_mutex_free(&p_data->mutex); - } - - free(p_data); - - return abt_errno; -} - -static size_t pool_get_size(ABT_pool pool) -{ - void *data; - ABT_pool_get_data(pool, &data); - data_t *p_data = pool_get_data_ptr(data); - return p_data->num_units; + unit_t *p_unit = (unit_t *)calloc(1, sizeof(unit_t)); + if (!p_unit) + return ABT_UNIT_NULL; + return (ABT_unit)p_unit; } -static void pool_push_shared(ABT_pool pool, ABT_unit unit) +static void pool_free_unit(ABT_pool pool, ABT_unit unit) { - void *data; - ABT_pool_get_data(pool, &data); - data_t *p_data = pool_get_data_ptr(data); unit_t *p_unit = (unit_t *)unit; - - ABT_mutex_spinlock(p_data->mutex); - if (p_data->num_units == 0) { - p_unit->p_prev = p_unit; - p_unit->p_next = p_unit; - p_data->p_head = p_unit; - p_data->p_tail = p_unit; - } else { - unit_t *p_head = p_data->p_head; - unit_t *p_tail = p_data->p_tail; - p_tail->p_next = p_unit; - p_head->p_prev = p_unit; - p_unit->p_prev = p_tail; - p_unit->p_next = p_head; - p_data->p_tail = p_unit; - } - p_data->num_units++; - - p_unit->pool = pool; - ABT_mutex_unlock(p_data->mutex); + free(p_unit); } -static void pool_push_private(ABT_pool pool, ABT_unit unit) +static ABT_bool pool_is_empty(ABT_pool pool) { - void *data; - ABT_pool_get_data(pool, &data); - data_t *p_data = pool_get_data_ptr(data); - unit_t *p_unit = (unit_t *)unit; - - if (p_data->num_units == 0) { - p_unit->p_prev = p_unit; - p_unit->p_next = p_unit; - p_data->p_head = p_unit; - p_data->p_tail = p_unit; - } else { - unit_t *p_head = p_data->p_head; - unit_t *p_tail = p_data->p_tail; - p_tail->p_next = p_unit; - p_head->p_prev = p_unit; - p_unit->p_prev = p_tail; - p_unit->p_next = p_head; - p_data->p_tail = p_unit; - } - p_data->num_units++; - - p_unit->pool = pool; + pool_t *p_pool; + ABT_pool_get_data(pool, (void **)&p_pool); + return p_pool->p_head ? ABT_FALSE : ABT_TRUE; } -static ABT_unit pool_pop_shared(ABT_pool pool) +static ABT_unit pool_pop(ABT_pool pool, ABT_pool_context context) { - void *data; - ABT_pool_get_data(pool, &data); - data_t *p_data = pool_get_data_ptr(data); + pool_t *p_pool; + ABT_pool_get_data(pool, (void **)&p_pool); unit_t *p_unit = NULL; - ABT_unit h_unit = ABT_UNIT_NULL; - - ABT_mutex_spinlock(p_data->mutex); - if (p_data->num_units > 0) { - p_unit = p_data->p_head; - if (p_data->num_units == 1) { - p_data->p_head = NULL; - p_data->p_tail = NULL; - } else { - p_unit->p_prev->p_next = p_unit->p_next; - p_unit->p_next->p_prev = p_unit->p_prev; - p_data->p_head = p_unit->p_next; - } - p_data->num_units--; - - p_unit->p_prev = NULL; - p_unit->p_next = NULL; - p_unit->pool = ABT_POOL_NULL; - - h_unit = (ABT_unit)p_unit; + pthread_mutex_lock(&p_pool->lock); + if (p_pool->p_head == NULL) { + /* Empty. */ + } else if (p_pool->p_head == p_pool->p_tail) { + /* Only one thread. */ + p_unit = p_pool->p_head; + p_pool->p_head = NULL; + p_pool->p_tail = NULL; + } else if (context & ABT_POOL_CONTEXT_OWNER_SECONDARY) { + /* Pop from the tail. */ + p_unit = p_pool->p_tail; + p_pool->p_tail = p_unit->p_next; + } else { + /* Pop from the head. */ + p_unit = p_pool->p_head; + p_pool->p_head = p_unit->p_prev; } - ABT_mutex_unlock(p_data->mutex); - - return h_unit; + pthread_mutex_unlock(&p_pool->lock); + if (!p_unit) + return ABT_UNIT_NULL; + return (ABT_unit)p_unit; } -static ABT_unit pool_pop_private(ABT_pool pool) +static void pool_push(ABT_pool pool, ABT_unit unit, ABT_pool_context context) { - void *data; - ABT_pool_get_data(pool, &data); - data_t *p_data = pool_get_data_ptr(data); - unit_t *p_unit = NULL; - ABT_unit h_unit = ABT_UNIT_NULL; - - if (p_data->num_units > 0) { - p_unit = p_data->p_head; - if (p_data->num_units == 1) { - p_data->p_head = NULL; - p_data->p_tail = NULL; + pool_t *p_pool; + ABT_pool_get_data(pool, (void **)&p_pool); + unit_t *p_unit = (unit_t *)unit; + pthread_mutex_lock(&p_pool->lock); + if (context & (ABT_POOL_CONTEXT_OP_THREAD_CREATE | + ABT_POOL_CONTEXT_OP_THREAD_CREATE_TO | + ABT_POOL_CONTEXT_OP_THREAD_REVIVE | + ABT_POOL_CONTEXT_OP_THREAD_REVIVE_TO)) { + /* Push to the head. */ + if (p_pool->p_head) { + p_unit->p_prev = p_pool->p_head; + p_pool->p_head->p_next = p_unit; } else { - p_unit->p_prev->p_next = p_unit->p_next; - p_unit->p_next->p_prev = p_unit->p_prev; - p_data->p_head = p_unit->p_next; + p_pool->p_tail = p_unit; } - p_data->num_units--; - - p_unit->p_prev = NULL; - p_unit->p_next = NULL; - p_unit->pool = ABT_POOL_NULL; - - h_unit = (ABT_unit)p_unit; - } - - return h_unit; -} - -static int pool_remove_shared(ABT_pool pool, ABT_unit unit) -{ - void *data; - ABT_pool_get_data(pool, &data); - data_t *p_data = pool_get_data_ptr(data); - unit_t *p_unit = (unit_t *)unit; - - if (p_data->num_units == 0) - return ABT_ERR_POOL; - if (p_unit->pool == ABT_POOL_NULL) - return ABT_ERR_POOL; - - if (p_unit->pool != pool) { - return ABT_ERR_INV_POOL; - } - - ABT_mutex_spinlock(p_data->mutex); - if (p_data->num_units == 1) { - p_data->p_head = NULL; - p_data->p_tail = NULL; + p_pool->p_head = p_unit; } else { - p_unit->p_prev->p_next = p_unit->p_next; - p_unit->p_next->p_prev = p_unit->p_prev; - if (p_unit == p_data->p_head) { - p_data->p_head = p_unit->p_next; - } else if (p_unit == p_data->p_tail) { - p_data->p_tail = p_unit->p_prev; + /* Push to the tail. */ + if (p_pool->p_tail) { + p_unit->p_next = p_pool->p_tail; + p_pool->p_tail->p_prev = p_unit; + } else { + p_pool->p_head = p_unit; } + p_pool->p_tail = p_unit; } - p_data->num_units--; - - p_unit->pool = ABT_POOL_NULL; - ABT_mutex_unlock(p_data->mutex); - - p_unit->p_prev = NULL; - p_unit->p_next = NULL; - - return ABT_SUCCESS; + pthread_mutex_unlock(&p_pool->lock); } -static int pool_remove_private(ABT_pool pool, ABT_unit unit) +static int pool_init(ABT_pool pool, ABT_pool_config config) { - void *data; - ABT_pool_get_data(pool, &data); - data_t *p_data = pool_get_data_ptr(data); - unit_t *p_unit = (unit_t *)unit; - - if (p_data->num_units == 0) - return ABT_ERR_POOL; - if (p_unit->pool == ABT_POOL_NULL) - return ABT_ERR_POOL; - - if (p_unit->pool != pool) { - return ABT_ERR_INV_POOL; - } + pool_t *p_pool = (pool_t *)calloc(1, sizeof(pool_t)); + if (!p_pool) + return ABT_ERR_MEM; - if (p_data->num_units == 1) { - p_data->p_head = NULL; - p_data->p_tail = NULL; - } else { - p_unit->p_prev->p_next = p_unit->p_next; - p_unit->p_next->p_prev = p_unit->p_prev; - if (p_unit == p_data->p_head) { - p_data->p_head = p_unit->p_next; - } else if (p_unit == p_data->p_tail) { - p_data->p_tail = p_unit->p_prev; - } + /* Initialize the spinlock */ + int ret = pthread_mutex_init(&p_pool->lock, 0); + if (ret != 0) { + free(p_pool); + return ABT_ERR_SYS; } - p_data->num_units--; - - p_unit->pool = ABT_POOL_NULL; - p_unit->p_prev = NULL; - p_unit->p_next = NULL; - + ABT_pool_set_data(pool, (void *)p_pool); return ABT_SUCCESS; } -/* Unit functions */ - -static ABT_bool unit_is_in_pool(ABT_unit unit) +static void pool_free(ABT_pool pool) { - unit_t *p_unit = (unit_t *)unit; - return (p_unit->pool != ABT_POOL_NULL) ? ABT_TRUE : ABT_FALSE; + pool_t *p_pool; + ABT_pool_get_data(pool, (void **)&p_pool); + pthread_mutex_destroy(&p_pool->lock); + free(p_pool); } -static ABT_unit unit_create_from_thread(ABT_thread thread) +static void create_pools(int num, ABT_pool *pools) { - unit_t *p_unit = malloc(sizeof(unit_t)); - if (!p_unit) - return ABT_UNIT_NULL; - - p_unit->p_prev = NULL; - p_unit->p_next = NULL; - p_unit->pool = ABT_POOL_NULL; - p_unit->thread = thread; - - return (ABT_unit)p_unit; -} + /* Pool definition */ + ABT_pool_user_def def; + ABT_pool_user_def_create(pool_create_unit, pool_free_unit, pool_is_empty, + pool_pop, pool_push, &def); + ABT_pool_user_def_set_init(def, pool_init); + ABT_pool_user_def_set_free(def, pool_free); + /* Pool configuration */ + ABT_pool_config config; + ABT_pool_config_create(&config); + /* The same as a pool created by ABT_pool_create_basic(). */ + const int automatic = 1; + ABT_pool_config_set(config, ABT_pool_config_automatic.key, + ABT_pool_config_automatic.type, &automatic); -static void unit_free(ABT_unit *unit) -{ - free(*unit); - *unit = ABT_UNIT_NULL; + int i; + for (i = 0; i < num; i++) { + ABT_pool_create(def, config, &pools[i]); + } + ABT_pool_user_def_free(&def); + ABT_pool_config_free(&config); } diff --git a/src/error.c b/src/error.c index a14acd44..576435ef 100644 --- a/src/error.c +++ b/src/error.c @@ -106,7 +106,8 @@ int ABT_error_get_str(int err, char *str, size_t *len) "ABT_ERR_INV_ARG", "ABT_ERR_SYS", "ABT_ERR_CPUID", - "ABT_ERR_INV_POOL_CONFIG" }; + "ABT_ERR_INV_POOL_CONFIG", + "ABT_ERR_INV_POOL_USER_DEF" }; #ifndef ABT_CONFIG_ENABLE_VER_20_API ABTI_CHECK_TRUE(err >= ABT_SUCCESS && diff --git a/src/include/Makefile.mk b/src/include/Makefile.mk index 1a2c6bc9..3b5d9871 100644 --- a/src/include/Makefile.mk +++ b/src/include/Makefile.mk @@ -32,6 +32,7 @@ noinst_HEADERS = \ include/abti_rwlock.h \ include/abti_pool.h \ include/abti_pool_config.h \ + include/abti_pool_user_def.h \ include/abti_sched.h \ include/abti_sched_config.h \ include/abti_self.h \ diff --git a/src/include/abt.h.in b/src/include/abt.h.in index 076d0562..17ac1bc5 100644 --- a/src/include/abt.h.in +++ b/src/include/abt.h.in @@ -169,6 +169,11 @@ extern "C" { * @brief Error code: invalid pool configuration. */ #define ABT_ERR_INV_POOL_CONFIG 56 +/** + * @ingroup ERROR_CODE + * @brief Error code: invalid pool definition. + */ +#define ABT_ERR_INV_POOL_USER_DEF 57 /** * @ingroup ERROR_CODE * @brief Error code: invalid work unit for scheduling. @@ -770,6 +775,7 @@ struct ABT_sched_opaque; struct ABT_sched_config_opaque; struct ABT_pool_opaque; struct ABT_pool_config_opaque; +struct ABT_pool_def; struct ABT_unit_opaque; struct ABT_thread_opaque; struct ABT_thread_attr_opaque; @@ -861,6 +867,11 @@ typedef enum ABT_pool_kind ABT_pool_kind; * @brief Pool access type. */ typedef enum ABT_pool_access ABT_pool_access; +/** + * @ingroup POOL + * @brief Pool definition. + */ +typedef struct ABT_pool_def * ABT_pool_user_def; /** * @ingroup UNIT * @brief Work unit handle type for scheduling. @@ -1056,6 +1067,7 @@ typedef enum ABT_sync_event_type ABT_sync_event_type; #define ABT_BARRIER_NULL ((ABT_barrier) NULL) #define ABT_TIMER_NULL ((ABT_timer) NULL) #define ABT_TOOL_CONTEXT_NULL ((ABT_tool_context) NULL) +#define ABT_POOL_USER_DEF_NULL ((ABT_pool_user_def) NULL) #else #define ABT_XSTREAM_NULL ((ABT_xstream) (0x01)) #define ABT_XSTREAM_BARRIER_NULL ((ABT_xstream_barrier)(0x02)) @@ -1077,6 +1089,7 @@ typedef enum ABT_sync_event_type ABT_sync_event_type; #define ABT_BARRIER_NULL ((ABT_barrier) (0x12)) #define ABT_TIMER_NULL ((ABT_timer) (0x13)) #define ABT_TOOL_CONTEXT_NULL ((ABT_tool_context) (0x14)) +#define ABT_POOL_USER_DEF_NULL ((ABT_pool_user_def) (0x15)) #endif /** @@ -1445,6 +1458,480 @@ typedef struct { */ extern const ABT_pool_config_var ABT_pool_config_automatic ABT_API_PUBLIC; +/** + * @ingroup POOL + * @brief A pool context value + * + * \c ABT_pool_context hints a context of a pool operation. The underlying pool + * implementation can use this hint to adapt its behavior. + * + * @note + * The built-in pools assume that at most one of the \c ABT_POOL_CONTEXT_PRIO + * flags, at most one of the \c ABT_POOL_CONTEXT_OP flags corresponding to a + * pool operation, and at most one of the \c ABT_POOL_CONTEXT_OWNER flags are + * set to a pool context value. A user-defined pool is recommended to follow + * the same hint as one used by the built-in pools, but it is not mandatory. + */ +typedef uint64_t ABT_pool_context; +/** + * @ingroup POOL + * @brief A flag that hints a default priority. + * + * \c ABT_POOL_CONTEXT_PRIO_DEFAULT_PRIO hints a default priority. + * + * @note + * \DOC_DESC_POOL_CONTEXT_POP_PUSH + * \DOC_DESC_POOL_CONTEXT_IMPL_DEF_PRIO + */ +#define ABT_POOL_CONTEXT_PRIO_DEFAULT_PRIO ((ABT_pool_context)0x0) +/** + * @ingroup POOL + * @brief A flag that hints a high priority. + * + * \c ABT_POOL_CONTEXT_PRIO_HIGH_PRIO hints a high priority. An expected + * behavior is that a pop operation with this flag gets a unit of high priority + * while a push operation with this flag sets a target unit to high priority so + * that the unit will be popped before other units of lower priority. + * + * @note + * \DOC_DESC_POOL_CONTEXT_POP_PUSH + * \DOC_DESC_POOL_CONTEXT_IMPL_DEF_PRIO + */ +#define ABT_POOL_CONTEXT_PRIO_HIGH_PRIO ((ABT_pool_context)0x1) +/** + * @ingroup POOL + * @brief A flag that hints a low priority. + * + * \c ABT_POOL_CONTEXT_PRIO_LOW_PRIO hints a low priority. An expected behavior + * is that a pop operation with this flag gets a unit of low priority while a + * push operation with this flag sets a target unit to low priority so that the + * unit will be popped after other units of higher priority. + * + * @note + * \DOC_DESC_POOL_CONTEXT_POP_PUSH + * \DOC_DESC_POOL_CONTEXT_IMPL_DEF_PRIO + */ +#define ABT_POOL_CONTEXT_PRIO_LOW_PRIO ((ABT_pool_context)0x2) +/** + * @ingroup POOL + * @brief A flag that hints a default ownership of a pool. + * + * \c ABT_POOL_CONTEXT_OWNER_DEFAULT hints that a caller has a default ownership + * of a pool. + * + * @note + * \DOC_DESC_POOL_CONTEXT_POP_PUSH + */ +#define ABT_POOL_CONTEXT_OWNER_DEFAULT ((ABT_pool_context)0x0) +/** + * @ingroup POOL + * @brief A flag that hints a primary ownership of a pool. + * + * \c ABT_POOL_CONTEXT_OWNER_DEFAULT hints that a caller is a primary owner of a + * pool. A primary owner accesses a pool more frequently than secondary owners. + * + * @note + * \DOC_DESC_POOL_CONTEXT_POP_PUSH + * + * @note + * In a typical random work-stealing setting, a scheduler would hint + * \c ABT_POOL_CONTEXT_OWNER_PRIMARY to access its own pool (i.e., a local + * pool). + */ +#define ABT_POOL_CONTEXT_OWNER_PRIMARY ((ABT_pool_context)0x100) +/** + * @ingroup POOL + * @brief A flag that hints a secondary ownership of a pool. + * + * \c ABT_POOL_CONTEXT_OWNER_DEFAULT hints that a caller is a secondary owner of + * a pool. A secondary owner accesses a pool less frequently than primary + * owners. + * + * @note + * \DOC_DESC_POOL_CONTEXT_POP_PUSH + * + * @note + * In a typical random work-stealing setting, a scheduler would hint + * \c ABT_POOL_CONTEXT_OWNER_SECONDARY to access a pool that is not owned (i.e., + * a remote pool). + */ +#define ABT_POOL_CONTEXT_OWNER_SECONDARY ((ABT_pool_context)0x200) +/** + * @ingroup POOL + * @brief A flag that hints an unspecified pool operation. + * + * \c ABT_POOL_CONTEXT_OP_POOL_OTHER hints an unspecified pool operation that is + * categorized as none of the other predefined pool operations. + * + * @note + * \DOC_DESC_POOL_CONTEXT_POP_PUSH + */ +#define ABT_POOL_CONTEXT_OP_POOL_OTHER ((ABT_pool_context)0x0) +/** + * @ingroup POOL + * @brief A flag that hints a push operation in a thread creation routine + * without a yield operation. + * + * \c ABT_POOL_CONTEXT_OP_THREAD_CREATE hints a pool push operation in a thread + * creation routine that does not follow a yield operation to a newly created + * thread. + * + * @note + * \DOC_DESC_POOL_CONTEXT_PUSH + */ +#define ABT_POOL_CONTEXT_OP_THREAD_CREATE ((ABT_pool_context)0x1000) +/** + * @ingroup POOL + * @brief A flag that hints a push operation in a thread creation routine with + * a yield operation. + * + * \c ABT_POOL_CONTEXT_OP_THREAD_CREATE_TO hints a pool push operation in a + * thread creation routine that follows a yield operation to a newly created + * thread. + * + * @note + * \DOC_DESC_POOL_CONTEXT_PUSH + */ +#define ABT_POOL_CONTEXT_OP_THREAD_CREATE_TO ((ABT_pool_context)0x2000) +/** + * @ingroup POOL + * @brief A flag that hints a push operation in a thread revival routine + * without a yield operation. + * + * \c ABT_POOL_CONTEXT_OP_THREAD_REVIVE hints a pool push operation in a thread + * revival routine that does not follow a yield operation to a revived thread. + * + * @note + * \DOC_DESC_POOL_CONTEXT_PUSH + */ +#define ABT_POOL_CONTEXT_OP_THREAD_REVIVE ((ABT_pool_context)0x4000) +/** + * @ingroup POOL + * @brief A flag that hints a push operation in a thread revival routine with + * a yield operation. + * + * \c ABT_POOL_CONTEXT_OP_THREAD_REVIVE_TO hints a pool push operation in a + * thread revival routine that follows a yield operation to a newly created + * thread. + * + * @note + * \DOC_DESC_POOL_CONTEXT_PUSH + */ +#define ABT_POOL_CONTEXT_OP_THREAD_REVIVE_TO ((ABT_pool_context)0x8000) +/** + * @ingroup POOL + * @brief A flag that hints a push operation in a thread yield routine. + * + * \c ABT_POOL_CONTEXT_OP_THREAD_YIELD hints a pool push operation in a thread + * yield routine. + * + * @note + * \DOC_DESC_POOL_CONTEXT_PUSH + */ +#define ABT_POOL_CONTEXT_OP_THREAD_YIELD ((ABT_pool_context)0x10000) +/** + * @ingroup POOL + * @brief A flag that hints a push operation in a thread yield-to routine. + * + * \c ABT_POOL_CONTEXT_OP_THREAD_YIELD_TO hints a pool push operation in a + * thread yield-to routine. + * + * @note + * \DOC_DESC_POOL_CONTEXT_PUSH + */ +#define ABT_POOL_CONTEXT_OP_THREAD_YIELD_TO ((ABT_pool_context)0x20000) +/** + * @ingroup POOL + * @brief A flag that hints a push operation in a thread resume-yield-to + * routine. + * + * \c ABT_POOL_CONTEXT_OP_THREAD_RESUME_YIELD_TO hints a pool push operation in + * a thread resume-yield-to routine. + * + * @note + * \DOC_DESC_POOL_CONTEXT_PUSH + */ +#define ABT_POOL_CONTEXT_OP_THREAD_RESUME_YIELD_TO ((ABT_pool_context)0x40000) +/** + * @ingroup POOL + * @brief A flag that hints a push operation in a yield operation in a + * synchronization loop. + * + * \c ABT_POOL_CONTEXT_OP_THREAD_YIELD_LOOP hints a pool push operation in + * an internal yield operation in a loop for synchronization. + * + * @note + * \DOC_DESC_POOL_CONTEXT_PUSH + */ +#define ABT_POOL_CONTEXT_OP_THREAD_YIELD_LOOP ((ABT_pool_context)0x80000) +/** + * @ingroup POOL + * @brief A flag that hints a push operation in a thread resume routine. + * + * \c ABT_POOL_CONTEXT_OP_THREAD_RESUME hints a pool push operation in a thread + * resume routine. + * + * @note + * \DOC_DESC_POOL_CONTEXT_PUSH + */ +#define ABT_POOL_CONTEXT_OP_THREAD_RESUME ((ABT_pool_context)0x100000) +/** + * @ingroup POOL + * @brief A flag that hints a push operation for thread migration. + * + * \c ABT_POOL_CONTEXT_OP_THREAD_MIGRATE hints a pool push operation for thread + * migration. + * + * @note + * \DOC_DESC_POOL_CONTEXT_PUSH + */ +#define ABT_POOL_CONTEXT_OP_THREAD_MIGRATE ((ABT_pool_context)0x200000) + +/* Pool Functions */ +/** + * @ingroup POOL_USER_DEF + * @fn ABT_unit (*ABT_pool_user_create_unit_fn)(ABT_pool, ABT_thread) + * @brief Function that creates an \c ABT_unit handle that is associated with an + * \c ABT_thread handle. + * + * \c ABT_pool_user_create_unit_fn() creates a user-defined \c ABT_unit handle + * that is associated with the work-unit \c thread where \c thread is associated + * with the pool \c pool. \c thread is neither \c ABT_THREAD_NULL nor + * \c ABT_TASK_NULL. The returned value must point to a 4-byte aligned memory + * address that is unique in the system. This function may return + * \c ABT_UNIT_NULL if creation fails. + * + * @note + * Specifically, the Argobots implementation assumes that 1. the first 2 least + * significant bits are zero and 2. all ABT_unit values are unique. Both + * requirements are satisfied if \c ABT_unit points to a 4-byte aligned memory + * block. An \c ABT_thread handle also satisfies these requirements. + * + * The user must implement \c ABT_pool_user_create_unit_fn() to implement a + * pool. + * + * A caller of \c ABT_pool_user_create_unit_fn() is undefined, so a program that + * relies on a caller of this routine is non-conforming. + */ +typedef ABT_unit (*ABT_pool_user_create_unit_fn)(ABT_pool, ABT_thread); +/** + * @ingroup POOL_USER_DEF + * @fn void (*ABT_pool_user_free_unit_fn)(ABT_pool pool, ABT_unit unit) + * @brief Function that frees an \c ABT_unit handle. + * + * \c ABT_pool_user_free_unit_fn() frees the work unit \c unit that is created + * by its corresponding \c ABT_pool_user_create_unit_fn(). \c unit belongs to + * the pool \c pool. Its associated entity (i.e., \c ABT_thread) is freed by + * the Argobots runtime. \c ABT_UNIT_NULL is never passed as \c unit to this + * routine. + * + * The user must implement \c ABT_pool_user_free_unit_fn() to implement a pool. + * + * A caller of \c ABT_pool_user_free_unit_fn() is undefined, so a program that + * relies on a caller of this routine is non-conforming. + */ +typedef void (*ABT_pool_user_free_unit_fn)(ABT_pool, ABT_unit); +/** + * @ingroup POOL_USER_DEF + * @fn ABT_bool (*ABT_pool_user_is_empty_fn)(ABT_pool pool) + * @brief Function that returns if a pool is empty. + * + * \c ABT_pool_user_is_empty_fn() returns \c ABT_TRUE if the pool \c pool is + * empty. Otherwise, it returns \c ABT_FALSE. The Argobots runtime assumes + * that the number of work units in \c pool is zero if this routine returns + * \c ABT_TRUE. + * + * The user must implement \c ABT_pool_user_is_empty_fn() to implement a pool. + * + * A caller of \c ABT_pool_user_is_empty_fn() is undefined, so a program that + * relies on a caller of this routine is non-conforming. + */ +typedef ABT_bool (*ABT_pool_user_is_empty_fn)(ABT_pool); +/** + * @ingroup POOL_USER_DEF + * @fn ABT_unit (*ABT_pool_user_pop_fn)(ABT_pool pool, + * ABT_pool_context context) + * @brief Function that pops a work unit from a pool. + * + * \c ABT_pool_user_pop_fn() pops a work unit from the pool \c pool and returns + * it. The implementation of this routine may change its behavior based on the + * pool context \c context. + * + * @note + * \DOC_DESC_POOL_CONTEXT_IMPL{\c context} + * + * The user must implement \c ABT_pool_user_pop_fn() to implement a pool. + * + * A caller of \c ABT_pool_user_pop_fn() is undefined, so a program that relies + * on a caller of this routine is non-conforming. + */ +typedef ABT_unit (*ABT_pool_user_pop_fn)(ABT_pool, ABT_pool_context); +/** + * @ingroup POOL_USER_DEF + * @fn void (*ABT_pool_user_push_fn)(ABT_pool pool, ABT_unit unit, + * ABT_pool_context context) + * @brief Function that pushes a work unit to a pool. + * + * \c ABT_pool_user_push_fn() pushes the work unit \c unit to the pool \c pool. + * \c unit must be stored in \c pool and able to be popped by a pop function + * (e.g., \c ABT_pool_user_pop_fn()) later. The implementation of this routine + * may change its behavior based on the pool context \c context. + * + * @note + * \DOC_DESC_POOL_CONTEXT_IMPL{\c context} + * + * The user must implement \c ABT_pool_user_push_fn() to implement a pool. + * + * A caller of \c ABT_pool_user_push_fn() is undefined, so a program that relies + * on a caller of this routine is non-conforming. + */ +typedef void (*ABT_pool_user_push_fn)(ABT_pool, ABT_unit, ABT_pool_context); +/** + * @ingroup POOL_USER_DEF + * @fn int (*ABT_pool_user_init_fn)(ABT_pool pool, + * ABT_pool_config pool_config) + * @brief Function that creates a pool. + * + * \c ABT_pool_user_init_fn() sets up the pool \c pool with the pool + * configuration \c pool_config. \c ABT_pool_user_init_fn() may return a value + * other than \c ABT_SUCCESS if creation fails. + * + * \c ABT_pool_user_init_fn() is optional, so the user may set + * \c ABT_pool_user_init_fn to \c NULL. + * + * A caller of \c ABT_pool_user_init_fn() is undefined, so a program that relies + * on a caller of this routine is non-conforming. + */ +typedef int (*ABT_pool_user_init_fn)(ABT_pool, ABT_pool_config); +/** + * @ingroup POOL_USER_DEF + * @fn void (*ABT_pool_user_free_fn)(ABT_pool pool) + * @brief Function that frees a pool. + * + * \c ABT_pool_user_free_fn() frees the pool \c pool and its associated + * resources. + * + * \c ABT_pool_user_free_fn() is optional, so the user may set + * \c ABT_pool_user_free_fn to \c NULL. + * + * A caller of \c ABT_pool_user_free_fn() is undefined, so a program that relies + * on a caller of this routine is non-conforming. + */ +typedef void (*ABT_pool_user_free_fn)(ABT_pool); +/** + * @ingroup POOL_USER_DEF + * @fn size_t (*ABT_pool_user_get_size_fn)(ABT_pool pool) + * @brief Function that returns the number of work units in a pool. + * + * \c ABT_pool_user_get_size_fn() returns the size of the pool \c pool. The + * Argobots runtime assumes that the size indicates the number of work units in + * \c pool. + * + * \c ABT_pool_user_get_size_fn() is optional, so the user may set + * \c ABT_pool_user_get_size_fn to \c NULL. + * + * A caller of \c ABT_pool_user_get_size_fn() is undefined, so a program that + * relies on a caller of this routine is non-conforming. + */ +typedef size_t (*ABT_pool_user_get_size_fn)(ABT_pool); +/** + * @ingroup POOL_USER_DEF + * @fn ABT_unit (*ABT_pool_user_pop_wait_fn)(ABT_pool pool, double time_secs, + * ABT_pool_context context) + * @brief Function that pops a work unit from a pool with wait. + * + * \c ABT_pool_user_pop_wait_fn() pops a work unit from the pool \c pool and + * returns it. If no work unit exists in \c pool, \c ABT_UNIT_NULL must be + * returned. \c time_secs hints at the duration in seconds that indicates how + * long the caller should wait on \c pool until a work unit is added to \c pool + * when \c pool is empty. The implementation of this routine may change its + * behavior based on the pool context \c context. + * + * @note + * \DOC_DESC_POOL_CONTEXT_IMPL{\c context} + * + * \c ABT_pool_user_pop_wait_fn() is optional, so the user may set + * ABT_pool_user_pop_wait_fn to \c NULL. + * + * A caller of \c ABT_pool_user_pop_wait_fn() is undefined, so a program that + * relies on a caller of this routine is non-conforming. + */ +typedef ABT_unit (*ABT_pool_user_pop_wait_fn)(ABT_pool, double, + ABT_pool_context); +/** + * @ingroup POOL_USER_DEF + * @fn void (*ABT_pool_user_pop_many_fn)(ABT_pool pool, ABT_unit *units, + * size_t max_units, + * size_t *num_popped, + * ABT_pool_context context) + * @brief Function that pops work units from a pool. + * + * \c ABT_pool_user_pop_many_fn() pops multiple work units from the pool \c pool + * and returns them through \c units. This routine pops at maximum \c max_units + * work units from the pool and returns the number of popped units to + * \c num_popped where \c num_popped is not \c NULL. The popped units are + * stored in \c units at indices from 0 to \c num_popped - 1. This routine may + * not modify elements of \c units at indices from \c num_popped to + * \c max_units. The implementation of this routine may change its behavior + * based on the pool context \c context. + * + * @note + * \DOC_DESC_POOL_CONTEXT_IMPL{\c context} + * + * \c ABT_pool_user_pop_many_fn() is optional, so the user may set + * \c ABT_pool_user_pop_many_fn to \c NULL. + * + * A caller of \c ABT_pool_user_pop_many_fn() is undefined, so a program that + * relies on a caller of this routine is non-conforming. + */ +typedef void (*ABT_pool_user_pop_many_fn)(ABT_pool, ABT_unit *, size_t, + size_t *, ABT_pool_context); +/** + * @ingroup POOL_USER_DEF + * @fn void (*ABT_pool_user_push_many_fn)(ABT_pool pool, + * const ABT_unit *units, + * size_t num_units, + * ABT_pool_context context) + * @brief Function that pushes work units to a pool. + * + * \c ABT_pool_user_push_many_fn() pushes multiple work units to the pool + * \c pool. \c num_units specifies the number of work units in \c units. + * \c units must be stored in \c pool and able to be popped by a pop function + * (e.g., \c ABT_pool_user_pop_fn()) later. The implementation of this routine + * may change its behavior based on the pool context \c context. + * + * @note + * \DOC_DESC_POOL_CONTEXT_IMPL{\c context} + * + * \c ABT_pool_user_push_many_fn() is optional, so the user may set + * \c ABT_pool_user_push_many_fn to \c NULL. + * + * A caller of \c ABT_pool_user_push_many_fn() is undefined, so a program that + * relies on a caller of this routine is non-conforming. + */ +typedef void (*ABT_pool_user_push_many_fn)(ABT_pool, const ABT_unit *, size_t, + ABT_pool_context); +/** + * @ingroup POOL_USER_DEF + * @fn void (*ABT_pool_user_print_all_fn)(ABT_pool pool, void *arg, + * void (*print_f)(void *, ABT_unit)) + * @brief Function that applies a user-given function to all work units in a + * pool. + * + * \c ABT_pool_user_print_all_fn() applies \c print_f() to all work units in the + * pool \c pool. \c print_f() is called with the user-given argument \c arg and + * a work unit handle. The caller guarantees that \c print_f() does not modify + * the states of \c pool and work units in \c pool. + * + * \c ABT_pool_user_print_all_fn() is optional, so the user may set + * \c ABT_pool_user_print_all_fn to \c NULL. + * + * A caller of \c ABT_pool_user_print_all_fn() is undefined, so a program that + * relies on a caller of this routine is non-conforming. + */ +typedef void (*ABT_pool_user_print_all_fn)(ABT_pool, void *arg, + void (*)(void *, ABT_unit)); + /* Pool Functions */ typedef ABT_unit_type (*ABT_unit_get_type_fn)(ABT_unit); typedef ABT_thread (*ABT_unit_get_thread_fn)(ABT_unit); @@ -1468,7 +1955,7 @@ typedef int (*ABT_pool_print_all_fn)(ABT_pool, void *arg, * @ingroup POOL * @brief A struct that defines a pool. */ -typedef struct { +struct ABT_pool_def { /** * @brief Access type. * @@ -1722,7 +2209,8 @@ typedef struct { * the caller of \c p_print_all() is non-conforming. */ ABT_pool_print_all_fn p_print_all; -} ABT_pool_def; +}; +typedef struct ABT_pool_def ABT_pool_def; /* Tool callback type. */ typedef void (*ABT_tool_thread_callback_fn)(ABT_thread, ABT_xstream, uint64_t event, @@ -1806,12 +2294,13 @@ int ABT_sched_config_get(ABT_sched_config config, int idx, ABT_sched_config_type *p_type, void *val) ABT_API_PUBLIC; /* Pool */ -int ABT_pool_create(ABT_pool_def *def, ABT_pool_config config, +int ABT_pool_create(ABT_pool_user_def def, ABT_pool_config config, ABT_pool *newpool) ABT_API_PUBLIC; int ABT_pool_create_basic(ABT_pool_kind kind, ABT_pool_access access, ABT_bool automatic, ABT_pool *newpool) ABT_API_PUBLIC; int ABT_pool_free(ABT_pool *pool) ABT_API_PUBLIC; int ABT_pool_get_access(ABT_pool pool, ABT_pool_access *access) ABT_API_PUBLIC; +int ABT_pool_is_empty(ABT_pool pool, ABT_bool *is_empty) ABT_API_PUBLIC; int ABT_pool_get_size(ABT_pool pool, size_t *size) ABT_API_PUBLIC; int ABT_pool_get_total_size(ABT_pool pool, size_t *size) ABT_API_PUBLIC; int ABT_pool_pop(ABT_pool pool, ABT_unit *unit) ABT_API_PUBLIC; @@ -1835,6 +2324,29 @@ int ABT_pool_config_set(ABT_pool_config config, int key, ABT_pool_config_type ty int ABT_pool_config_get(ABT_pool_config config, int key, ABT_pool_config_type *p_type, void *val) ABT_API_PUBLIC; +/* Pool definition */ +int ABT_pool_user_def_create(ABT_pool_user_create_unit_fn p_create_unit, + ABT_pool_user_free_unit_fn p_free_unit, + ABT_pool_user_is_empty_fn p_is_empty, + ABT_pool_user_pop_fn p_pop, + ABT_pool_user_push_fn p_push, + ABT_pool_user_def *newdef) ABT_API_PUBLIC; +int ABT_pool_user_def_free(ABT_pool_user_def *def) ABT_API_PUBLIC; +int ABT_pool_user_def_set_init(ABT_pool_user_def def, + ABT_pool_user_init_fn p_init) ABT_API_PUBLIC; +int ABT_pool_user_def_set_free(ABT_pool_user_def def, + ABT_pool_user_free_fn p_free) ABT_API_PUBLIC; +int ABT_pool_user_def_set_get_size(ABT_pool_user_def def, + ABT_pool_user_get_size_fn p_get_size) ABT_API_PUBLIC; +int ABT_pool_user_def_set_pop_wait(ABT_pool_user_def def, + ABT_pool_user_pop_wait_fn p_pop_wait) ABT_API_PUBLIC; +int ABT_pool_user_def_set_pop_many(ABT_pool_user_def def, + ABT_pool_user_pop_many_fn p_pop_many) ABT_API_PUBLIC; +int ABT_pool_user_def_set_push_many(ABT_pool_user_def def, + ABT_pool_user_push_many_fn p_push_many) ABT_API_PUBLIC; +int ABT_pool_user_def_set_print_all(ABT_pool_user_def def, + ABT_pool_user_print_all_fn p_print_all) ABT_API_PUBLIC; + /* Work Unit */ int ABT_unit_get_thread(ABT_unit unit, ABT_thread *thread) ABT_API_PUBLIC; int ABT_unit_set_associated_pool(ABT_unit unit, ABT_pool pool) ABT_API_PUBLIC; diff --git a/src/include/abti.h b/src/include/abti.h index 819587a6..742b7331 100644 --- a/src/include/abti.h +++ b/src/include/abti.h @@ -118,7 +118,11 @@ typedef enum ABTI_sched_used ABTI_sched_used; typedef void *ABTI_sched_id; /* Scheduler id */ typedef uintptr_t ABTI_sched_kind; /* Scheduler kind */ typedef struct ABTI_pool ABTI_pool; -typedef struct ABTI_pool_def ABTI_pool_def; +typedef struct ABTI_pool_required_def ABTI_pool_required_def; +typedef struct ABTI_pool_optional_def ABTI_pool_optional_def; +typedef struct ABTI_pool_deprecated_def ABTI_pool_deprecated_def; +typedef struct ABTI_pool_old_def ABTI_pool_old_def; +typedef struct ABTI_pool_user_def ABTI_pool_user_def; typedef struct ABTI_pool_config ABTI_pool_config; typedef struct ABTI_thread ABTI_thread; typedef struct ABTI_thread_attr ABTI_thread_attr; @@ -320,37 +324,36 @@ struct ABTI_sched_config { ABTU_hashtable *p_table; }; -struct ABTI_pool { - ABT_pool_access access; /* Access mode */ - ABT_bool automatic; /* To know if automatic data free */ - ABT_bool is_builtin; /* Built-in pool. */ - /* NOTE: int32_t to check if still positive */ - ABTD_atomic_int32 num_scheds; /* Number of associated schedulers */ - ABTD_atomic_int32 num_blocked; /* Number of blocked ULTs */ - void *data; /* Specific data */ - uint64_t id; /* ID */ +struct ABTI_pool_required_def { + /* Required pool operations */ + ABT_pool_user_create_unit_fn p_create_unit; + ABT_pool_user_free_unit_fn p_free_unit; + ABT_pool_user_is_empty_fn p_is_empty; + ABT_pool_user_pop_fn p_pop; + ABT_pool_user_push_fn p_push; +}; - /* Functions to manage units */ - ABT_unit_is_in_pool_fn u_is_in_pool; - ABT_unit_create_from_thread_fn u_create_from_thread; - ABT_unit_free_fn u_free; +struct ABTI_pool_optional_def { + /* Optional pool operations */ + ABT_pool_user_init_fn p_init; + ABT_pool_user_free_fn p_free; + ABT_pool_user_get_size_fn p_get_size; + ABT_pool_user_pop_wait_fn p_pop_wait; + ABT_pool_user_pop_many_fn p_pop_many; + ABT_pool_user_push_many_fn p_push_many; + ABT_pool_user_print_all_fn p_print_all; +}; - /* Functions to manage the pool */ - ABT_pool_init_fn p_init; - ABT_pool_get_size_fn p_get_size; - ABT_pool_push_fn p_push; - ABT_pool_pop_fn p_pop; - ABT_pool_pop_wait_fn p_pop_wait; +struct ABTI_pool_deprecated_def { + /* Pool operations that might be directly called even now, but + * deprecated. All operations are optional. */ + ABT_unit_is_in_pool_fn u_is_in_pool; ABT_pool_pop_timedwait_fn p_pop_timedwait; ABT_pool_remove_fn p_remove; - ABT_pool_free_fn p_free; - ABT_pool_print_all_fn p_print_all; }; -struct ABTI_pool_def { - ABT_pool_access access; - ABT_unit_get_thread_fn u_get_thread; - ABT_unit_is_in_pool_fn u_is_in_pool; +struct ABTI_pool_old_def { + /* Pool operations that are not directly called now. */ ABT_unit_create_from_thread_fn u_create_from_thread; ABT_unit_free_fn u_free; ABT_pool_init_fn p_init; @@ -358,12 +361,39 @@ struct ABTI_pool_def { ABT_pool_push_fn p_push; ABT_pool_pop_fn p_pop; ABT_pool_pop_wait_fn p_pop_wait; - ABT_pool_pop_timedwait_fn p_pop_timedwait; - ABT_pool_remove_fn p_remove; ABT_pool_free_fn p_free; ABT_pool_print_all_fn p_print_all; }; +struct ABTI_pool { + ABT_pool_access access; /* Access mode */ + ABT_bool automatic; /* To know if automatic data free */ + ABT_bool is_builtin; /* Built-in pool. */ + /* NOTE: int32_t to check if still positive */ + ABTD_atomic_int32 num_scheds; /* Number of associated schedulers */ + ABTD_atomic_int32 num_blocked; /* Number of blocked ULTs */ + void *data; /* Specific data */ + uint64_t id; /* ID */ + + ABTI_pool_required_def required_def; + ABTI_pool_optional_def optional_def; + ABTI_pool_deprecated_def deprecated_def; + ABTI_pool_old_def old_def; +}; + +struct ABTI_pool_user_def { + ABT_pool_access dummy_access; + ABT_unit_get_type_fn dummy_fn1; + ABT_unit_get_thread_fn dummy_fn2; + ABT_unit_get_task_fn dummy_fn3; + ABT_unit_is_in_pool_fn dummy_fn4; + ABT_unit_create_from_thread_fn + symbol; /* This value is to check if ABT_pool_user_def points to + ABTI_pool_user_def or ABT_pool_def. */ + ABTI_pool_required_def required_def; + ABTI_pool_optional_def optional_def; +}; + struct ABTI_pool_config { ABTU_hashtable *p_table; }; @@ -533,10 +563,8 @@ void ABTI_sched_free(ABTI_global *p_global, ABTI_local *p_local, ABTI_sched *p_sched, ABT_bool force_free); ABTU_ret_err int ABTI_sched_get_migration_pool(ABTI_sched *, ABTI_pool *, ABTI_pool **); -ABT_bool ABTI_sched_has_to_stop(ABTI_local **pp_local, ABTI_sched *p_sched); -size_t ABTI_sched_get_size(ABTI_sched *p_sched); -size_t ABTI_sched_get_total_size(ABTI_sched *p_sched); -size_t ABTI_sched_get_effective_size(ABTI_local *p_local, ABTI_sched *p_sched); +ABT_bool ABTI_sched_has_to_stop(ABTI_sched *p_sched); +ABT_bool ABTI_sched_has_unit(ABTI_sched *p_sched); void ABTI_sched_print(ABTI_sched *p_sched, FILE *p_os, int indent, ABT_bool print_sub); void ABTI_sched_reset_id(void); @@ -551,10 +579,16 @@ ABTU_ret_err int ABTI_pool_create_basic(ABT_pool_kind kind, ABT_bool automatic, ABTI_pool **pp_newpool); void ABTI_pool_free(ABTI_pool *p_pool); -ABTU_ret_err int ABTI_pool_get_fifo_def(ABT_pool_access access, - ABTI_pool_def *p_def); -ABTU_ret_err int ABTI_pool_get_fifo_wait_def(ABT_pool_access access, - ABTI_pool_def *p_def); +ABTU_ret_err int +ABTI_pool_get_fifo_def(ABT_pool_access access, + ABTI_pool_required_def *p_required_def, + ABTI_pool_optional_def *p_optional_def, + ABTI_pool_deprecated_def *p_deprecated_def); +ABTU_ret_err int +ABTI_pool_get_fifo_wait_def(ABT_pool_access access, + ABTI_pool_required_def *p_required_def, + ABTI_pool_optional_def *p_optional_def, + ABTI_pool_deprecated_def *p_deprecated_def); void ABTI_pool_print(ABTI_pool *p_pool, FILE *p_os, int indent); void ABTI_pool_reset_id(void); @@ -562,6 +596,9 @@ void ABTI_pool_reset_id(void); ABTU_ret_err int ABTI_pool_config_read(const ABTI_pool_config *p_config, int key, void *p_val); +/* Pool definition */ +ABT_bool ABTI_pool_user_def_is_new(const ABT_pool_user_def def); + /* Work Unit */ void ABTI_unit_init_hash_table(ABTI_global *p_global); void ABTI_unit_finalize_hash_table(ABTI_global *p_global); @@ -637,6 +674,7 @@ void ABTI_info_check_print_all_thread_stacks(void); #include "abti_self.h" #include "abti_pool.h" #include "abti_pool_config.h" +#include "abti_pool_user_def.h" #include "abti_sched.h" #include "abti_sched_config.h" #include "abti_stream.h" diff --git a/src/include/abti_error.h b/src/include/abti_error.h index 6aa468bd..9743c04f 100644 --- a/src/include/abti_error.h +++ b/src/include/abti_error.h @@ -199,6 +199,15 @@ } \ } while (0) +#define ABTI_CHECK_NULL_POOL_USER_DEF_PTR(p) \ + do { \ + if (ABTI_IS_ERROR_CHECK_ENABLED && \ + ABTU_unlikely(p == (ABTI_pool_user_def *)NULL)) { \ + HANDLE_ERROR_FUNC_WITH_CODE(ABT_ERR_INV_POOL_USER_DEF); \ + return ABT_ERR_INV_POOL_USER_DEF; \ + } \ + } while (0) + #define ABTI_CHECK_NULL_SCHED_PTR(p) \ do { \ if (ABTI_IS_ERROR_CHECK_ENABLED && \ diff --git a/src/include/abti_mutex.h b/src/include/abti_mutex.h index 6dcb6326..897c332b 100644 --- a/src/include/abti_mutex.h +++ b/src/include/abti_mutex.h @@ -85,6 +85,7 @@ static inline void ABTI_mutex_lock_no_recursion(ABTI_local **pp_local, if (p_ythread) { while (ABTD_spinlock_try_acquire(&p_mutex->lock)) { ABTI_ythread_yield(&p_local_xstream, p_ythread, + ABTI_YTHREAD_YIELD_KIND_YIELD_LOOP, ABT_SYNC_EVENT_TYPE_MUTEX, (void *)p_mutex); *pp_local = ABTI_xstream_get_local(p_local_xstream); } diff --git a/src/include/abti_pool.h b/src/include/abti_pool.h index a6729828..ef5a5cdf 100644 --- a/src/include/abti_pool.h +++ b/src/include/abti_pool.h @@ -50,34 +50,40 @@ static inline void ABTI_pool_dec_num_blocked(ABTI_pool *p_pool) ABTD_atomic_fetch_sub_int32(&p_pool->num_blocked, 1); } -static inline void ABTI_pool_push(ABTI_pool *p_pool, ABT_unit unit) +static inline void ABTI_pool_push(ABTI_pool *p_pool, ABT_unit unit, + ABT_pool_context context) { /* Push unit into pool */ LOG_DEBUG_POOL_PUSH(p_pool, unit); - p_pool->p_push(ABTI_pool_get_handle(p_pool), unit); + p_pool->required_def.p_push(ABTI_pool_get_handle(p_pool), unit, context); } -static inline void ABTI_pool_add_thread(ABTI_thread *p_thread) +static inline void ABTI_pool_add_thread(ABTI_thread *p_thread, + ABT_pool_context context) { /* Set the ULT's state as READY. The relaxed version is used since the state * is synchronized by the following pool operation. */ ABTD_atomic_relaxed_store_int(&p_thread->state, ABT_THREAD_STATE_READY); /* Add the ULT to the associated pool */ - ABTI_pool_push(p_thread->p_pool, p_thread->unit); + ABTI_pool_push(p_thread->p_pool, p_thread->unit, context); } ABTU_ret_err static inline int ABTI_pool_remove(ABTI_pool *p_pool, ABT_unit unit) { LOG_DEBUG_POOL_REMOVE(p_pool, unit); - return p_pool->p_remove(ABTI_pool_get_handle(p_pool), unit); + ABTI_UB_ASSERT(p_pool->deprecated_def.p_remove); + return p_pool->deprecated_def.p_remove(ABTI_pool_get_handle(p_pool), unit); } -static inline ABT_unit ABTI_pool_pop_wait(ABTI_pool *p_pool, double time_secs) +static inline ABT_unit ABTI_pool_pop_wait(ABTI_pool *p_pool, double time_secs, + ABT_pool_context context) { ABT_unit unit; - unit = p_pool->p_pop_wait(ABTI_pool_get_handle(p_pool), time_secs); + ABTI_UB_ASSERT(p_pool->optional_def.p_pop_wait); + unit = p_pool->optional_def.p_pop_wait(ABTI_pool_get_handle(p_pool), + time_secs, context); LOG_DEBUG_POOL_POP(p_pool, unit); return unit; @@ -88,17 +94,20 @@ static inline ABT_unit ABTI_pool_pop_timedwait(ABTI_pool *p_pool, { ABT_unit unit; - unit = p_pool->p_pop_timedwait(ABTI_pool_get_handle(p_pool), abstime_secs); + ABTI_UB_ASSERT(p_pool->deprecated_def.p_pop_timedwait); + unit = p_pool->deprecated_def.p_pop_timedwait(ABTI_pool_get_handle(p_pool), + abstime_secs); LOG_DEBUG_POOL_POP(p_pool, unit); return unit; } -static inline ABT_unit ABTI_pool_pop(ABTI_pool *p_pool) +static inline ABT_unit ABTI_pool_pop(ABTI_pool *p_pool, + ABT_pool_context context) { ABT_unit unit; - unit = p_pool->p_pop(ABTI_pool_get_handle(p_pool)); + unit = p_pool->required_def.p_pop(ABTI_pool_get_handle(p_pool), context); LOG_DEBUG_POOL_POP(p_pool, unit); return unit; @@ -119,9 +128,15 @@ static inline int32_t ABTI_pool_release(ABTI_pool *p_pool) return ABTD_atomic_fetch_sub_int32(&p_pool->num_scheds, 1) - 1; } +static inline ABT_bool ABTI_pool_is_empty(ABTI_pool *p_pool) +{ + return p_pool->required_def.p_is_empty(ABTI_pool_get_handle(p_pool)); +} + static inline size_t ABTI_pool_get_size(ABTI_pool *p_pool) { - return p_pool->p_get_size(ABTI_pool_get_handle(p_pool)); + ABTI_UB_ASSERT(p_pool->optional_def.p_get_size); + return p_pool->optional_def.p_get_size(ABTI_pool_get_handle(p_pool)); } static inline size_t ABTI_pool_get_total_size(ABTI_pool *p_pool) diff --git a/src/include/abti_pool_user_def.h b/src/include/abti_pool_user_def.h new file mode 100644 index 00000000..80c30ed7 --- /dev/null +++ b/src/include/abti_pool_user_def.h @@ -0,0 +1,43 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * See COPYRIGHT in top-level directory. + */ + +#ifndef ABTI_POOL_USER_DEF_H_INCLUDED +#define ABTI_POOL_USER_DEF_H_INCLUDED + +/* Inlined functions for pool user definitions */ + +static inline ABTI_pool_user_def * +ABTI_pool_user_def_get_ptr(ABT_pool_user_def def) +{ +#ifndef ABT_CONFIG_DISABLE_ERROR_CHECK + ABTI_pool_user_def *p_def; + if (def == ABT_POOL_USER_DEF_NULL) { + p_def = NULL; + } else { + p_def = (ABTI_pool_user_def *)def; + } + return p_def; +#else + return (ABTI_pool_user_def *)def; +#endif +} + +static inline ABT_pool_user_def +ABTI_pool_user_def_get_handle(ABTI_pool_user_def *p_def) +{ +#ifndef ABT_CONFIG_DISABLE_ERROR_CHECK + ABT_pool_user_def h_def; + if (p_def == NULL) { + h_def = ABT_POOL_USER_DEF_NULL; + } else { + h_def = (ABT_pool_user_def)p_def; + } + return h_def; +#else + return (ABT_pool_user_def)p_def; +#endif +} + +#endif /* ABTI_POOL_USER_DEF_H_INCLUDED */ diff --git a/src/include/abti_unit.h b/src/include/abti_unit.h index 8d4b6158..2b95dacb 100644 --- a/src/include/abti_unit.h +++ b/src/include/abti_unit.h @@ -65,13 +65,15 @@ ABTI_unit_set_associated_pool(ABTI_global *p_global, ABT_unit unit, return ABT_SUCCESS; } else { /* The new pool is a user-defined pool. */ + ABT_pool pool = ABTI_pool_get_handle(p_pool); ABT_unit new_unit = - p_pool->u_create_from_thread(ABTI_thread_get_handle(p_thread)); + p_pool->required_def.p_create_unit(pool, ABTI_thread_get_handle( + p_thread)); if (new_unit == ABT_UNIT_NULL) return ABT_ERR_OTHER; int ret = ABTI_unit_map_thread(p_global, new_unit, p_thread); if (ret != ABT_SUCCESS) { - p_pool->u_free(&new_unit); + p_pool->required_def.p_free_unit(pool, new_unit); return ret; } p_thread->unit = new_unit; @@ -87,7 +89,8 @@ ABTI_unit_set_associated_pool(ABTI_global *p_global, ABT_unit unit, /* The old unit is associated with a custom pool. Remove the * existing mapping. */ ABTI_unit_unmap_thread(p_global, unit); - p_thread->p_pool->u_free(&unit); + ABT_pool old_pool = ABTI_pool_get_handle(p_thread->p_pool); + p_thread->p_pool->required_def.p_free_unit(old_pool, unit); ABTI_unit_init_builtin(p_thread); p_thread->p_pool = p_pool; *pp_thread = p_thread; @@ -98,17 +101,20 @@ ABTI_unit_set_associated_pool(ABTI_global *p_global, ABT_unit unit, return ABT_SUCCESS; } else { /* Both are associated with different custom pools. */ + ABT_pool pool = ABTI_pool_get_handle(p_pool); ABT_unit new_unit = - p_pool->u_create_from_thread(ABTI_thread_get_handle(p_thread)); + p_pool->required_def.p_create_unit(pool, ABTI_thread_get_handle( + p_thread)); if (new_unit == ABT_UNIT_NULL) return ABT_ERR_OTHER; int ret = ABTI_unit_map_thread(p_global, new_unit, p_thread); if (ret != ABT_SUCCESS) { - p_pool->u_free(&new_unit); + p_pool->required_def.p_free_unit(pool, new_unit); return ret; } ABTI_unit_unmap_thread(p_global, unit); - p_thread->p_pool->u_free(&unit); + ABT_pool old_pool = ABTI_pool_get_handle(p_thread->p_pool); + p_thread->p_pool->required_def.p_free_unit(old_pool, unit); p_thread->unit = new_unit; p_thread->p_pool = p_pool; *pp_thread = p_thread; @@ -128,13 +134,15 @@ ABTU_ret_err static inline int ABTI_thread_init_pool(ABTI_global *p_global, p_thread->p_pool = p_pool; return ABT_SUCCESS; } else { + ABT_pool pool = ABTI_pool_get_handle(p_pool); ABT_unit new_unit = - p_pool->u_create_from_thread(ABTI_thread_get_handle(p_thread)); + p_pool->required_def.p_create_unit(pool, ABTI_thread_get_handle( + p_thread)); if (new_unit == ABT_UNIT_NULL) return ABT_ERR_OTHER; int ret = ABTI_unit_map_thread(p_global, new_unit, p_thread); if (ret != ABT_SUCCESS) { - p_pool->u_free(&new_unit); + p_pool->required_def.p_free_unit(pool, new_unit); return ret; } p_thread->unit = new_unit; @@ -155,13 +163,15 @@ ABTI_thread_set_associated_pool(ABTI_global *p_global, ABTI_thread *p_thread, return ABT_SUCCESS; } else if (ABTI_unit_is_builtin(unit)) { /* The new unit is associated with a custom pool. Add a new mapping. */ + ABT_pool pool = ABTI_pool_get_handle(p_pool); ABT_unit new_unit = - p_pool->u_create_from_thread(ABTI_thread_get_handle(p_thread)); + p_pool->required_def.p_create_unit(pool, ABTI_thread_get_handle( + p_thread)); if (new_unit == ABT_UNIT_NULL) return ABT_ERR_OTHER; int ret = ABTI_unit_map_thread(p_global, new_unit, p_thread); if (ret != ABT_SUCCESS) { - p_pool->u_free(&new_unit); + p_pool->required_def.p_free_unit(pool, new_unit); return ret; } p_thread->unit = new_unit; @@ -171,7 +181,8 @@ ABTI_thread_set_associated_pool(ABTI_global *p_global, ABTI_thread *p_thread, /* The old unit is associated with a custom pool. Remove the existing * mapping. */ ABTI_unit_unmap_thread(p_global, unit); - p_thread->p_pool->u_free(&unit); + ABT_pool old_pool = ABTI_pool_get_handle(p_thread->p_pool); + p_thread->p_pool->required_def.p_free_unit(old_pool, unit); ABTI_unit_init_builtin(p_thread); p_thread->p_pool = p_pool; return ABT_SUCCESS; @@ -180,17 +191,20 @@ ABTI_thread_set_associated_pool(ABTI_global *p_global, ABTI_thread *p_thread, return ABT_SUCCESS; } else { /* Both are associated with different custom pools. */ + ABT_pool pool = ABTI_pool_get_handle(p_pool); ABT_unit new_unit = - p_pool->u_create_from_thread(ABTI_thread_get_handle(p_thread)); + p_pool->required_def.p_create_unit(pool, ABTI_thread_get_handle( + p_thread)); if (new_unit == ABT_UNIT_NULL) return ABT_ERR_OTHER; int ret = ABTI_unit_map_thread(p_global, new_unit, p_thread); if (ret != ABT_SUCCESS) { - p_pool->u_free(&new_unit); + p_pool->required_def.p_free_unit(pool, new_unit); return ret; } ABTI_unit_unmap_thread(p_global, unit); - p_thread->p_pool->u_free(&unit); + ABT_pool old_pool = ABTI_pool_get_handle(p_thread->p_pool); + p_thread->p_pool->required_def.p_free_unit(old_pool, unit); p_thread->unit = new_unit; p_thread->p_pool = p_pool; return ABT_SUCCESS; @@ -203,7 +217,8 @@ static inline void ABTI_thread_unset_associated_pool(ABTI_global *p_global, ABT_unit unit = p_thread->unit; if (ABTU_unlikely(!ABTI_unit_is_builtin(unit))) { ABTI_unit_unmap_thread(p_global, unit); - p_thread->p_pool->u_free(&unit); + ABT_pool old_pool = ABTI_pool_get_handle(p_thread->p_pool); + p_thread->p_pool->required_def.p_free_unit(old_pool, unit); } #if ABTI_IS_ERROR_CHECK_ENABLED p_thread->unit = ABT_UNIT_NULL; diff --git a/src/include/abti_waitlist.h b/src/include/abti_waitlist.h index 95c88daa..fd83b31f 100644 --- a/src/include/abti_waitlist.h +++ b/src/include/abti_waitlist.h @@ -128,8 +128,9 @@ static inline ABT_bool ABTI_waitlist_wait_timedout_and_unlock( ABTD_spinlock_acquire(p_lock); goto timeout; } - ABTI_ythread_yield(&p_local_xstream, p_ythread, sync_event_type, - p_sync); + ABTI_ythread_yield(&p_local_xstream, p_ythread, + ABTI_YTHREAD_YIELD_KIND_YIELD_LOOP, + sync_event_type, p_sync); *pp_local = ABTI_xstream_get_local(p_local_xstream); } } else { diff --git a/src/include/abti_ythread.h b/src/include/abti_ythread.h index 98bfa773..9f6690aa 100644 --- a/src/include/abti_ythread.h +++ b/src/include/abti_ythread.h @@ -56,7 +56,7 @@ static inline void ABTI_ythread_resume_and_push(ABTI_local *p_local, ABTI_pool *p_pool = p_ythread->thread.p_pool; /* Add the ULT to its associated pool */ - ABTI_pool_add_thread(&p_ythread->thread); + ABTI_pool_add_thread(&p_ythread->thread, ABT_POOL_CONTEXT_OP_THREAD_RESUME); /* Decrease the number of blocked threads */ ABTI_pool_dec_num_blocked(p_pool); @@ -197,33 +197,66 @@ static inline void ABTI_ythread_run_child(ABTI_xstream **pp_local_xstream, ABTI_ythread_switch_to_child_internal(pp_local_xstream, p_self, p_child); } -void ABTI_ythread_callback_yield(void *arg); +typedef enum { + ABTI_YTHREAD_YIELD_KIND_USER, + ABTI_YTHREAD_YIELD_KIND_YIELD_LOOP, +} ABTI_ythread_yield_kind; + +typedef enum { + ABTI_YTHREAD_YIELD_TO_KIND_USER, + ABTI_YTHREAD_YIELD_TO_KIND_CREATE_TO, + ABTI_YTHREAD_YIELD_TO_KIND_REVIVE_TO, +} ABTI_ythread_yield_to_kind; + +void ABTI_ythread_callback_yield_user_yield(void *arg); +void ABTI_ythread_callback_yield_loop(void *arg); +void ABTI_ythread_callback_yield_user_yield_to(void *arg); +void ABTI_ythread_callback_yield_create_to(void *arg); +void ABTI_ythread_callback_yield_revive_to(void *arg); static inline void ABTI_ythread_yield(ABTI_xstream **pp_local_xstream, ABTI_ythread *p_self, + ABTI_ythread_yield_kind kind, ABT_sync_event_type sync_event_type, void *p_sync) { ABTI_event_ythread_yield(*pp_local_xstream, p_self, p_self->thread.p_parent, sync_event_type, p_sync); - ABTI_ythread_switch_to_parent_internal(pp_local_xstream, p_self, - ABTI_ythread_callback_yield, - (void *)p_self); + if (kind == ABTI_YTHREAD_YIELD_KIND_USER) { + ABTI_ythread_switch_to_parent_internal( + pp_local_xstream, p_self, ABTI_ythread_callback_yield_user_yield, + (void *)p_self); + } else { + ABTI_UB_ASSERT(kind == ABTI_YTHREAD_YIELD_KIND_YIELD_LOOP); + ABTI_ythread_switch_to_parent_internal(pp_local_xstream, p_self, + ABTI_ythread_callback_yield_loop, + (void *)p_self); + } } -static inline void ABTI_ythread_yield_to(ABTI_xstream **pp_local_xstream, - ABTI_ythread *p_self, - ABTI_ythread *p_target, - ABT_sync_event_type sync_event_type, - void *p_sync) +static inline void +ABTI_ythread_yield_to(ABTI_xstream **pp_local_xstream, ABTI_ythread *p_self, + ABTI_ythread *p_target, ABTI_ythread_yield_to_kind kind, + ABT_sync_event_type sync_event_type, void *p_sync) { ABTI_event_ythread_yield(*pp_local_xstream, p_self, p_self->thread.p_parent, sync_event_type, p_sync); ABTD_atomic_release_store_int(&p_target->thread.state, ABT_THREAD_STATE_RUNNING); - ABTI_ythread_switch_to_sibling_internal(pp_local_xstream, p_self, p_target, - ABTI_ythread_callback_yield, - (void *)p_self); + if (kind == ABTI_YTHREAD_YIELD_TO_KIND_USER) { + ABTI_ythread_switch_to_sibling_internal( + pp_local_xstream, p_self, p_target, + ABTI_ythread_callback_yield_user_yield_to, (void *)p_self); + } else if (kind == ABTI_YTHREAD_YIELD_TO_KIND_CREATE_TO) { + ABTI_ythread_switch_to_sibling_internal( + pp_local_xstream, p_self, p_target, + ABTI_ythread_callback_yield_create_to, (void *)p_self); + } else { + ABTI_UB_ASSERT(kind == ABTI_YTHREAD_YIELD_TO_KIND_REVIVE_TO); + ABTI_ythread_switch_to_sibling_internal( + pp_local_xstream, p_self, p_target, + ABTI_ythread_callback_yield_revive_to, (void *)p_self); + } } /* Old interface used for ABT_thread_yield_to() */ @@ -238,6 +271,7 @@ ABTI_ythread_thread_yield_to(ABTI_xstream **pp_local_xstream, sync_event_type, p_sync); ABTD_atomic_release_store_int(&p_target->thread.state, ABT_THREAD_STATE_RUNNING); + ABTI_ythread_switch_to_sibling_internal( pp_local_xstream, p_self, p_target, ABTI_ythread_callback_thread_yield_to, (void *)p_self); @@ -250,9 +284,14 @@ typedef struct { void ABTI_ythread_callback_resume_yield_to(void *arg); +typedef enum { + ABTI_YTHREAD_RESUME_YIELD_TO_KIND_USER, +} ABTI_ythread_resume_yield_to_kind; + static inline void ABTI_ythread_resume_yield_to(ABTI_xstream **pp_local_xstream, ABTI_ythread *p_self, ABTI_ythread *p_target, + ABTI_ythread_resume_yield_to_kind kind, ABT_sync_event_type sync_event_type, void *p_sync) { /* The ULT must be in BLOCKED state. */ @@ -265,6 +304,7 @@ ABTI_ythread_resume_yield_to(ABTI_xstream **pp_local_xstream, sync_event_type, p_sync); ABTD_atomic_release_store_int(&p_target->thread.state, ABT_THREAD_STATE_RUNNING); + ABTI_UB_ASSERT(kind == ABTI_YTHREAD_RESUME_YIELD_TO_KIND_USER); ABTI_ythread_callback_resume_yield_to_arg arg = { p_self, p_target }; ABTI_ythread_switch_to_sibling_internal( pp_local_xstream, p_self, p_target, @@ -606,7 +646,7 @@ static inline void ABTI_ythread_schedule(ABTI_global *p_global, /* If p_thread is cancelled, there's nothing to do. */ } else if (request_op == ABTI_THREAD_HANDLE_REQUEST_MIGRATED) { /* If p_thread is migrated, let's push p_thread back to its pool. */ - ABTI_pool_add_thread(p_thread); + ABTI_pool_add_thread(p_thread, ABT_POOL_CONTEXT_OP_THREAD_MIGRATE); } } diff --git a/src/info.c b/src/info.c index 3292ef31..666c9d0b 100644 --- a/src/info.c +++ b/src/info.c @@ -1281,7 +1281,7 @@ ABTU_ret_err static int info_print_thread_stacks_in_pool(ABTI_global *p_global, fflush(fp); return ABT_SUCCESS; } - ABTI_CHECK_TRUE(p_pool->p_print_all, ABT_ERR_POOL); + ABTI_CHECK_TRUE(p_pool->optional_def.p_print_all, ABT_ERR_POOL); ABT_pool pool = ABTI_pool_get_handle(p_pool); @@ -1289,7 +1289,7 @@ ABTU_ret_err static int info_print_thread_stacks_in_pool(ABTI_global *p_global, struct info_print_unit_arg_t arg; arg.p_global = p_global; arg.fp = fp; - p_pool->p_print_all(pool, &arg, info_print_unit); + p_pool->optional_def.p_print_all(pool, &arg, info_print_unit); fflush(fp); return ABT_SUCCESS; } diff --git a/src/pool/Makefile.mk b/src/pool/Makefile.mk index 5e36bb9b..11d42a61 100644 --- a/src/pool/Makefile.mk +++ b/src/pool/Makefile.mk @@ -7,4 +7,5 @@ abt_sources += \ pool/fifo.c \ pool/fifo_wait.c \ pool/pool.c \ - pool/pool_config.c + pool/pool_config.c \ + pool/pool_user_def.c diff --git a/src/pool/fifo.c b/src/pool/fifo.c index 3a88ec03..5559f293 100644 --- a/src/pool/fifo.c +++ b/src/pool/fifo.c @@ -9,22 +9,37 @@ /* FIFO pool implementation */ static int pool_init(ABT_pool pool, ABT_pool_config config); -static int pool_free(ABT_pool pool); +static void pool_free(ABT_pool pool); +static ABT_bool pool_is_empty(ABT_pool pool); static size_t pool_get_size(ABT_pool pool); -static void pool_push_shared(ABT_pool pool, ABT_unit unit); -static void pool_push_private(ABT_pool pool, ABT_unit unit); -static ABT_unit pool_pop_shared(ABT_pool pool); -static ABT_unit pool_pop_private(ABT_pool pool); -static ABT_unit pool_pop_wait(ABT_pool pool, double time_secs); -static ABT_unit pool_pop_timedwait(ABT_pool pool, double abstime_secs); +static void pool_push_shared(ABT_pool pool, ABT_unit unit, + ABT_pool_context context); +static void pool_push_private(ABT_pool pool, ABT_unit unit, + ABT_pool_context context); +static ABT_unit pool_pop_shared(ABT_pool pool, ABT_pool_context context); +static ABT_unit pool_pop_private(ABT_pool pool, ABT_pool_context context); +static ABT_unit pool_pop_wait(ABT_pool pool, double time_secs, + ABT_pool_context context); +static void pool_push_many_shared(ABT_pool pool, const ABT_unit *units, + size_t num_units, ABT_pool_context context); +static void pool_push_many_private(ABT_pool pool, const ABT_unit *units, + size_t num_units, ABT_pool_context context); +static void pool_pop_many_shared(ABT_pool pool, ABT_unit *units, + size_t max_units, size_t *num_popped, + ABT_pool_context context); +static void pool_pop_many_private(ABT_pool pool, ABT_unit *units, + size_t max_units, size_t *num_popped, + ABT_pool_context context); +static void pool_print_all(ABT_pool pool, void *arg, + void (*print_fn)(void *, ABT_unit)); +static ABT_unit pool_create_unit(ABT_pool pool, ABT_thread thread); +static void pool_free_unit(ABT_pool pool, ABT_unit unit); + +/* For backward compatibility */ static int pool_remove_shared(ABT_pool pool, ABT_unit unit); static int pool_remove_private(ABT_pool pool, ABT_unit unit); -static int pool_print_all(ABT_pool pool, void *arg, - void (*print_fn)(void *, ABT_unit)); - -static ABT_bool unit_is_in_pool(ABT_unit unit); -static ABT_unit unit_create_from_thread(ABT_thread thread); -static void unit_free(ABT_unit *unit); +static ABT_unit pool_pop_timedwait(ABT_pool pool, double abstime_secs); +static ABT_bool pool_unit_is_in_pool(ABT_unit unit); struct data { ABTD_spinlock mutex; @@ -65,25 +80,32 @@ ABTU_ret_err static inline int spinlock_acquire_if_not_empty(data_t *p_data) } /* Obtain the FIFO pool definition according to the access type */ -ABTU_ret_err int ABTI_pool_get_fifo_def(ABT_pool_access access, - ABTI_pool_def *p_def) +ABTU_ret_err int +ABTI_pool_get_fifo_def(ABT_pool_access access, + ABTI_pool_required_def *p_required_def, + ABTI_pool_optional_def *p_optional_def, + ABTI_pool_deprecated_def *p_deprecated_def) { /* Definitions according to the access type */ /* FIXME: need better implementation, e.g., lock-free one */ switch (access) { case ABT_POOL_ACCESS_PRIV: - p_def->p_push = pool_push_private; - p_def->p_pop = pool_pop_private; - p_def->p_remove = pool_remove_private; + p_required_def->p_push = pool_push_private; + p_required_def->p_pop = pool_pop_private; + p_optional_def->p_push_many = pool_push_many_private; + p_optional_def->p_pop_many = pool_pop_many_private; + p_deprecated_def->p_remove = pool_remove_private; break; case ABT_POOL_ACCESS_SPSC: case ABT_POOL_ACCESS_MPSC: case ABT_POOL_ACCESS_SPMC: case ABT_POOL_ACCESS_MPMC: - p_def->p_push = pool_push_shared; - p_def->p_pop = pool_pop_shared; - p_def->p_remove = pool_remove_shared; + p_required_def->p_push = pool_push_shared; + p_required_def->p_pop = pool_pop_shared; + p_optional_def->p_push_many = pool_push_many_shared; + p_optional_def->p_pop_many = pool_pop_many_shared; + p_deprecated_def->p_remove = pool_remove_shared; break; default: @@ -91,17 +113,17 @@ ABTU_ret_err int ABTI_pool_get_fifo_def(ABT_pool_access access, } /* Common definitions regardless of the access type */ - p_def->access = access; - p_def->p_init = pool_init; - p_def->p_free = pool_free; - p_def->p_get_size = pool_get_size; - p_def->p_pop_wait = pool_pop_wait; - p_def->p_pop_timedwait = pool_pop_timedwait; - p_def->p_print_all = pool_print_all; - p_def->u_is_in_pool = unit_is_in_pool; - p_def->u_create_from_thread = unit_create_from_thread; - p_def->u_free = unit_free; - + p_optional_def->p_init = pool_init; + p_optional_def->p_free = pool_free; + p_required_def->p_is_empty = pool_is_empty; + p_optional_def->p_get_size = pool_get_size; + p_optional_def->p_pop_wait = pool_pop_wait; + p_optional_def->p_print_all = pool_print_all; + p_required_def->p_create_unit = pool_create_unit; + p_required_def->p_free_unit = pool_free_unit; + + p_deprecated_def->p_pop_timedwait = pool_pop_timedwait; + p_deprecated_def->u_is_in_pool = pool_unit_is_in_pool; return ABT_SUCCESS; } @@ -135,31 +157,31 @@ static int pool_init(ABT_pool pool, ABT_pool_config config) return abt_errno; } -static int pool_free(ABT_pool pool) +static void pool_free(ABT_pool pool) { - int abt_errno = ABT_SUCCESS; ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); ABTU_free(p_data); - - return abt_errno; } -static size_t pool_get_size(ABT_pool pool) +static ABT_bool pool_is_empty(ABT_pool pool) { ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); - return p_data->num_threads; + return ABTD_atomic_acquire_load_int(&p_data->is_empty) ? ABT_TRUE + : ABT_FALSE; } -static void pool_push_shared(ABT_pool pool, ABT_unit unit) +static size_t pool_get_size(ABT_pool pool) { ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); - ABTI_thread *p_thread = ABTI_unit_get_thread_from_builtin_unit(unit); + return p_data->num_threads; +} - ABTD_spinlock_acquire(&p_data->mutex); +static inline void pool_push_unsafe(data_t *p_data, ABTI_thread *p_thread) +{ if (p_data->num_threads == 0) { p_thread->p_prev = p_thread; p_thread->p_next = p_thread; @@ -177,69 +199,98 @@ static void pool_push_shared(ABT_pool pool, ABT_unit unit) p_data->p_tail = p_thread; p_data->num_threads++; } - ABTD_atomic_release_store_int(&p_thread->is_in_pool, 1); +} + +static void pool_push_shared(ABT_pool pool, ABT_unit unit, + ABT_pool_context context) +{ + (void)context; + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + data_t *p_data = pool_get_data_ptr(p_pool->data); + ABTI_thread *p_thread = ABTI_unit_get_thread_from_builtin_unit(unit); + ABTD_spinlock_acquire(&p_data->mutex); + pool_push_unsafe(p_data, p_thread); ABTD_spinlock_release(&p_data->mutex); } -static void pool_push_private(ABT_pool pool, ABT_unit unit) +static void pool_push_private(ABT_pool pool, ABT_unit unit, + ABT_pool_context context) { + (void)context; ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); ABTI_thread *p_thread = ABTI_unit_get_thread_from_builtin_unit(unit); + pool_push_unsafe(p_data, p_thread); +} - if (p_data->num_threads == 0) { - p_thread->p_prev = p_thread; - p_thread->p_next = p_thread; - p_data->p_head = p_thread; - p_data->p_tail = p_thread; - p_data->num_threads = 1; - ABTD_atomic_release_store_int(&p_data->is_empty, 0); - } else { - ABTI_thread *p_head = p_data->p_head; - ABTI_thread *p_tail = p_data->p_tail; - p_tail->p_next = p_thread; - p_head->p_prev = p_thread; - p_thread->p_prev = p_tail; - p_thread->p_next = p_head; - p_data->p_tail = p_thread; - p_data->num_threads++; +static void pool_push_many_shared(ABT_pool pool, const ABT_unit *units, + size_t num_units, ABT_pool_context context) +{ + (void)context; + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + data_t *p_data = pool_get_data_ptr(p_pool->data); + if (num_units > 0) { + ABTD_spinlock_acquire(&p_data->mutex); + size_t i; + for (i = 0; i < num_units; i++) { + ABTI_thread *p_thread = + ABTI_unit_get_thread_from_builtin_unit(units[i]); + pool_push_unsafe(p_data, p_thread); + } + ABTD_spinlock_release(&p_data->mutex); } - - ABTD_atomic_release_store_int(&p_thread->is_in_pool, 1); } -static ABT_unit pool_pop_wait(ABT_pool pool, double time_secs) +static void pool_push_many_private(ABT_pool pool, const ABT_unit *units, + size_t num_units, ABT_pool_context context) { + (void)context; ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); - ABTI_thread *p_thread = NULL; + size_t i; + for (i = 0; i < num_units; i++) { + ABTI_thread *p_thread = + ABTI_unit_get_thread_from_builtin_unit(units[i]); + pool_push_unsafe(p_data, p_thread); + } +} - double time_start = 0.0; +static inline ABT_unit pool_pop_unsafe(data_t *p_data) +{ + if (p_data->num_threads > 0) { + ABTI_thread *p_thread = p_data->p_head; + if (p_data->num_threads == 1) { + p_data->p_head = NULL; + p_data->p_tail = NULL; + p_data->num_threads = 0; + ABTD_atomic_release_store_int(&p_data->is_empty, 1); + } else { + p_thread->p_prev->p_next = p_thread->p_next; + p_thread->p_next->p_prev = p_thread->p_prev; + p_data->p_head = p_thread->p_next; + p_data->num_threads--; + } + p_thread->p_prev = NULL; + p_thread->p_next = NULL; + ABTD_atomic_release_store_int(&p_thread->is_in_pool, 0); + return ABTI_unit_get_builtin_unit(p_thread); + } else { + return ABT_UNIT_NULL; + } +} + +static ABT_unit pool_pop_wait(ABT_pool pool, double time_secs, + ABT_pool_context context) +{ + (void)context; + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + data_t *p_data = pool_get_data_ptr(p_pool->data); + double time_start = 0.0; while (1) { if (spinlock_acquire_if_not_empty(p_data) == 0) { - ABT_unit h_unit = ABT_UNIT_NULL; - if (p_data->num_threads > 0) { - p_thread = p_data->p_head; - if (p_data->num_threads == 1) { - p_data->p_head = NULL; - p_data->p_tail = NULL; - p_data->num_threads = 0; - ABTD_atomic_release_store_int(&p_data->is_empty, 1); - } else { - p_thread->p_prev->p_next = p_thread->p_next; - p_thread->p_next->p_prev = p_thread->p_prev; - p_data->p_head = p_thread->p_next; - p_data->num_threads--; - } - - p_thread->p_prev = NULL; - p_thread->p_next = NULL; - ABTD_atomic_release_store_int(&p_thread->is_in_pool, 0); - - h_unit = ABTI_unit_get_builtin_unit(p_thread); - } + ABT_unit h_unit = pool_pop_unsafe(p_data); ABTD_spinlock_release(&p_data->mutex); if (h_unit != ABT_UNIT_NULL) return h_unit; @@ -262,31 +313,9 @@ static ABT_unit pool_pop_timedwait(ABT_pool pool, double abstime_secs) { ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); - ABTI_thread *p_thread = NULL; - while (1) { if (spinlock_acquire_if_not_empty(p_data) == 0) { - ABT_unit h_unit = ABT_UNIT_NULL; - if (p_data->num_threads > 0) { - p_thread = p_data->p_head; - if (p_data->num_threads == 1) { - p_data->p_head = NULL; - p_data->p_tail = NULL; - p_data->num_threads = 0; - ABTD_atomic_release_store_int(&p_data->is_empty, 1); - } else { - p_thread->p_prev->p_next = p_thread->p_next; - p_thread->p_next->p_prev = p_thread->p_prev; - p_data->p_head = p_thread->p_next; - p_data->num_threads--; - } - - p_thread->p_prev = NULL; - p_thread->p_next = NULL; - ABTD_atomic_release_store_int(&p_thread->is_in_pool, 0); - - h_unit = ABTI_unit_get_builtin_unit(p_thread); - } + ABT_unit h_unit = pool_pop_unsafe(p_data); ABTD_spinlock_release(&p_data->mutex); if (h_unit != ABT_UNIT_NULL) return h_unit; @@ -300,34 +329,13 @@ static ABT_unit pool_pop_timedwait(ABT_pool pool, double abstime_secs) } } -static ABT_unit pool_pop_shared(ABT_pool pool) +static ABT_unit pool_pop_shared(ABT_pool pool, ABT_pool_context context) { + (void)context; ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); - ABTI_thread *p_thread = NULL; - if (spinlock_acquire_if_not_empty(p_data) == 0) { - ABT_unit h_unit = ABT_UNIT_NULL; - if (p_data->num_threads > 0) { - p_thread = p_data->p_head; - if (p_data->num_threads == 1) { - p_data->p_head = NULL; - p_data->p_tail = NULL; - p_data->num_threads = 0; - ABTD_atomic_release_store_int(&p_data->is_empty, 1); - } else { - p_thread->p_prev->p_next = p_thread->p_next; - p_thread->p_next->p_prev = p_thread->p_prev; - p_data->p_head = p_thread->p_next; - p_data->num_threads--; - } - - p_thread->p_prev = NULL; - p_thread->p_next = NULL; - ABTD_atomic_release_store_int(&p_thread->is_in_pool, 0); - - h_unit = ABTI_unit_get_builtin_unit(p_thread); - } + ABT_unit h_unit = pool_pop_unsafe(p_data); ABTD_spinlock_release(&p_data->mutex); return h_unit; } else { @@ -335,47 +343,59 @@ static ABT_unit pool_pop_shared(ABT_pool pool) } } -static ABT_unit pool_pop_private(ABT_pool pool) +static ABT_unit pool_pop_private(ABT_pool pool, ABT_pool_context context) { + (void)context; ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); - ABTI_thread *p_thread = NULL; + return pool_pop_unsafe(p_data); +} - ABT_unit h_unit = ABT_UNIT_NULL; - if (p_data->num_threads > 0) { - p_thread = p_data->p_head; - if (p_data->num_threads == 1) { - p_data->p_head = NULL; - p_data->p_tail = NULL; - p_data->num_threads = 0; - ABTD_atomic_relaxed_store_int(&p_data->is_empty, 1); - } else { - p_thread->p_prev->p_next = p_thread->p_next; - p_thread->p_next->p_prev = p_thread->p_prev; - p_data->p_head = p_thread->p_next; - p_data->num_threads--; +static void pool_pop_many_shared(ABT_pool pool, ABT_unit *units, + size_t max_units, size_t *num_popped, + ABT_pool_context context) +{ + (void)context; + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + data_t *p_data = pool_get_data_ptr(p_pool->data); + if (max_units != 0 && spinlock_acquire_if_not_empty(p_data) == 0) { + size_t i; + for (i = 0; i < max_units; i++) { + ABT_unit h_unit = pool_pop_unsafe(p_data); + if (h_unit == ABT_UNIT_NULL) + break; + units[i] = h_unit; } - - p_thread->p_prev = NULL; - p_thread->p_next = NULL; - ABTD_atomic_release_store_int(&p_thread->is_in_pool, 0); - - h_unit = ABTI_unit_get_builtin_unit(p_thread); + *num_popped = i; + ABTD_spinlock_release(&p_data->mutex); + } else { + *num_popped = 0; } - return h_unit; } -static int pool_remove_shared(ABT_pool pool, ABT_unit unit) +static void pool_pop_many_private(ABT_pool pool, ABT_unit *units, + size_t max_units, size_t *num_popped, + ABT_pool_context context) { + (void)context; ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); - ABTI_thread *p_thread = ABTI_unit_get_thread_from_builtin_unit(unit); + size_t i; + for (i = 0; i < max_units; i++) { + ABT_unit h_unit = pool_pop_unsafe(p_data); + if (h_unit == ABT_UNIT_NULL) + break; + units[i] = h_unit; + } + *num_popped = i; +} +static inline int pool_remove_unsafe(data_t *p_data, ABTI_thread *p_thread) +{ ABTI_CHECK_TRUE(p_data->num_threads != 0, ABT_ERR_POOL); ABTI_CHECK_TRUE(ABTD_atomic_acquire_load_int(&p_thread->is_in_pool) == 1, ABT_ERR_POOL); - ABTD_spinlock_acquire(&p_data->mutex); if (p_data->num_threads == 1) { p_data->p_head = NULL; p_data->p_tail = NULL; @@ -391,51 +411,33 @@ static int pool_remove_shared(ABT_pool pool, ABT_unit unit) } p_data->num_threads--; } - ABTD_atomic_release_store_int(&p_thread->is_in_pool, 0); - ABTD_spinlock_release(&p_data->mutex); - p_thread->p_prev = NULL; p_thread->p_next = NULL; - return ABT_SUCCESS; } -static int pool_remove_private(ABT_pool pool, ABT_unit unit) +static int pool_remove_shared(ABT_pool pool, ABT_unit unit) { ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); ABTI_thread *p_thread = ABTI_unit_get_thread_from_builtin_unit(unit); + ABTD_spinlock_acquire(&p_data->mutex); + int abt_errno = pool_remove_unsafe(p_data, p_thread); + ABTD_spinlock_release(&p_data->mutex); + return abt_errno; +} - ABTI_CHECK_TRUE(p_data->num_threads != 0, ABT_ERR_POOL); - ABTI_CHECK_TRUE(ABTD_atomic_acquire_load_int(&p_thread->is_in_pool) == 1, - ABT_ERR_POOL); - - if (p_data->num_threads == 1) { - p_data->p_head = NULL; - p_data->p_tail = NULL; - p_data->num_threads = 0; - ABTD_atomic_relaxed_store_int(&p_data->is_empty, 1); - } else { - p_thread->p_prev->p_next = p_thread->p_next; - p_thread->p_next->p_prev = p_thread->p_prev; - if (p_thread == p_data->p_head) { - p_data->p_head = p_thread->p_next; - } else if (p_thread == p_data->p_tail) { - p_data->p_tail = p_thread->p_prev; - } - p_data->num_threads--; - } - - ABTD_atomic_release_store_int(&p_thread->is_in_pool, 0); - p_thread->p_prev = NULL; - p_thread->p_next = NULL; - - return ABT_SUCCESS; +static int pool_remove_private(ABT_pool pool, ABT_unit unit) +{ + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + data_t *p_data = pool_get_data_ptr(p_pool->data); + ABTI_thread *p_thread = ABTI_unit_get_thread_from_builtin_unit(unit); + return pool_remove_unsafe(p_data, p_thread); } -static int pool_print_all(ABT_pool pool, void *arg, - void (*print_fn)(void *, ABT_unit)) +static void pool_print_all(ABT_pool pool, void *arg, + void (*print_fn)(void *, ABT_unit)) { ABT_pool_access access; ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); @@ -458,27 +460,25 @@ static int pool_print_all(ABT_pool pool, void *arg, if (access != ABT_POOL_ACCESS_PRIV) { ABTD_spinlock_release(&p_data->mutex); } - - return ABT_SUCCESS; } /* Unit functions */ -static ABT_bool unit_is_in_pool(ABT_unit unit) +static ABT_bool pool_unit_is_in_pool(ABT_unit unit) { ABTI_thread *p_thread = ABTI_unit_get_thread_from_builtin_unit(unit); return ABTD_atomic_acquire_load_int(&p_thread->is_in_pool) ? ABT_TRUE : ABT_FALSE; } -static ABT_unit unit_create_from_thread(ABT_thread thread) +static ABT_unit pool_create_unit(ABT_pool pool, ABT_thread thread) { /* Call ABTI_unit_init_builtin() instead. */ ABTI_ASSERT(0); return ABT_UNIT_NULL; } -static void unit_free(ABT_unit *unit) +static void pool_free_unit(ABT_pool pool, ABT_unit unit) { /* A built-in unit does not need to be freed. This function may not be * called. */ diff --git a/src/pool/fifo_wait.c b/src/pool/fifo_wait.c index ca6d87f0..aed87f2a 100644 --- a/src/pool/fifo_wait.c +++ b/src/pool/fifo_wait.c @@ -8,18 +8,26 @@ /* FIFO_WAIT pool implementation */ static int pool_init(ABT_pool pool, ABT_pool_config config); -static int pool_free(ABT_pool pool); +static void pool_free(ABT_pool pool); +static ABT_bool pool_is_empty(ABT_pool pool); static size_t pool_get_size(ABT_pool pool); -static void pool_push(ABT_pool pool, ABT_unit unit); -static ABT_unit pool_pop(ABT_pool pool); -static ABT_unit pool_pop_wait(ABT_pool pool, double time_secs); -static ABT_unit pool_pop_timedwait(ABT_pool pool, double abstime_secs); +static void pool_push(ABT_pool pool, ABT_unit unit, ABT_pool_context context); +static ABT_unit pool_pop(ABT_pool pool, ABT_pool_context context); +static ABT_unit pool_pop_wait(ABT_pool pool, double time_secs, + ABT_pool_context context); +static void pool_push_many(ABT_pool pool, const ABT_unit *units, + size_t num_units, ABT_pool_context context); +static void pool_pop_many(ABT_pool pool, ABT_unit *units, size_t max_units, + size_t *num_popped, ABT_pool_context context); +static void pool_print_all(ABT_pool pool, void *arg, + void (*print_fn)(void *, ABT_unit)); +static ABT_unit pool_create_unit(ABT_pool pool, ABT_thread thread); +static void pool_free_unit(ABT_pool pool, ABT_unit unit); + +/* For backward compatibility */ static int pool_remove(ABT_pool pool, ABT_unit unit); -static int pool_print_all(ABT_pool pool, void *arg, - void (*print_fn)(void *, ABT_unit)); -static ABT_bool unit_is_in_pool(ABT_unit unit); -static ABT_unit unit_create_from_thread(ABT_thread thread); -static void unit_free(ABT_unit *unit); +static ABT_unit pool_pop_timedwait(ABT_pool pool, double abstime_secs); +static ABT_bool pool_unit_is_in_pool(ABT_unit unit); struct data { pthread_mutex_t mutex; @@ -36,23 +44,28 @@ static inline data_t *pool_get_data_ptr(void *p_data) return (data_t *)p_data; } -ABTU_ret_err int ABTI_pool_get_fifo_wait_def(ABT_pool_access access, - ABTI_pool_def *p_def) +ABTU_ret_err int +ABTI_pool_get_fifo_wait_def(ABT_pool_access access, + ABTI_pool_required_def *p_required_def, + ABTI_pool_optional_def *p_optional_def, + ABTI_pool_deprecated_def *p_deprecated_def) { - p_def->access = access; - p_def->p_init = pool_init; - p_def->p_free = pool_free; - p_def->p_get_size = pool_get_size; - p_def->p_push = pool_push; - p_def->p_pop = pool_pop; - p_def->p_pop_wait = pool_pop_wait; - p_def->p_pop_timedwait = pool_pop_timedwait; - p_def->p_remove = pool_remove; - p_def->p_print_all = pool_print_all; - p_def->u_is_in_pool = unit_is_in_pool; - p_def->u_create_from_thread = unit_create_from_thread; - p_def->u_free = unit_free; - + p_optional_def->p_init = pool_init; + p_optional_def->p_free = pool_free; + p_required_def->p_is_empty = pool_is_empty; + p_optional_def->p_get_size = pool_get_size; + p_required_def->p_push = pool_push; + p_required_def->p_pop = pool_pop; + p_optional_def->p_pop_wait = pool_pop_wait; + p_optional_def->p_push_many = pool_push_many; + p_optional_def->p_pop_many = pool_pop_many; + p_optional_def->p_print_all = pool_print_all; + p_required_def->p_create_unit = pool_create_unit; + p_required_def->p_free_unit = pool_free_unit; + + p_deprecated_def->p_pop_timedwait = pool_pop_timedwait; + p_deprecated_def->u_is_in_pool = pool_unit_is_in_pool; + p_deprecated_def->p_remove = pool_remove; return ABT_SUCCESS; } @@ -90,33 +103,33 @@ static int pool_init(ABT_pool pool, ABT_pool_config config) return abt_errno; } -static int pool_free(ABT_pool pool) +static void pool_free(ABT_pool pool) { - int abt_errno = ABT_SUCCESS; ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); pthread_mutex_destroy(&p_data->mutex); pthread_cond_destroy(&p_data->cond); ABTU_free(p_data); - - return abt_errno; } -static size_t pool_get_size(ABT_pool pool) +static ABT_bool pool_is_empty(ABT_pool pool) { ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); - return p_data->num_threads; + return ABTD_atomic_acquire_load_int(&p_data->is_empty) ? ABT_TRUE + : ABT_FALSE; } -static void pool_push(ABT_pool pool, ABT_unit unit) +static size_t pool_get_size(ABT_pool pool) { ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); - ABTI_thread *p_thread = ABTI_unit_get_thread_from_builtin_unit(unit); + return p_data->num_threads; +} - pthread_mutex_lock(&p_data->mutex); +static inline void pool_push_unsafe(data_t *p_data, ABTI_thread *p_thread) +{ if (p_data->num_threads == 0) { p_thread->p_prev = p_thread; p_thread->p_next = p_thread; @@ -134,21 +147,80 @@ static void pool_push(ABT_pool pool, ABT_unit unit) p_data->p_tail = p_thread; p_data->num_threads++; } - ABTD_atomic_release_store_int(&p_thread->is_in_pool, 1); +} + +static void pool_push(ABT_pool pool, ABT_unit unit, ABT_pool_context context) +{ + (void)context; + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + data_t *p_data = pool_get_data_ptr(p_pool->data); + ABTI_thread *p_thread = ABTI_unit_get_thread_from_builtin_unit(unit); + + pthread_mutex_lock(&p_data->mutex); + pool_push_unsafe(p_data, p_thread); pthread_cond_signal(&p_data->cond); pthread_mutex_unlock(&p_data->mutex); } -static ABT_unit pool_pop_wait(ABT_pool pool, double time_secs) +static void pool_push_many(ABT_pool pool, const ABT_unit *units, + size_t num_units, ABT_pool_context context) { + (void)context; ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); - ABTI_thread *p_thread = NULL; - ABT_unit h_unit = ABT_UNIT_NULL; - pthread_mutex_lock(&p_data->mutex); + if (num_units > 0) { + pthread_mutex_lock(&p_data->mutex); + size_t i; + for (i = 0; i < num_units; i++) { + ABTI_thread *p_thread = + ABTI_unit_get_thread_from_builtin_unit(units[i]); + pool_push_unsafe(p_data, p_thread); + } + if (num_units == 1) { + /* Wake up a single waiter. */ + pthread_cond_signal(&p_data->cond); + } else { + /* Wake up all the waiters. */ + pthread_cond_broadcast(&p_data->cond); + } + pthread_mutex_unlock(&p_data->mutex); + } +} +static inline ABT_unit pool_pop_unsafe(data_t *p_data) +{ + if (p_data->num_threads > 0) { + ABTI_thread *p_thread = p_data->p_head; + if (p_data->num_threads == 1) { + p_data->p_head = NULL; + p_data->p_tail = NULL; + p_data->num_threads = 0; + ABTD_atomic_release_store_int(&p_data->is_empty, 1); + } else { + p_thread->p_prev->p_next = p_thread->p_next; + p_thread->p_next->p_prev = p_thread->p_prev; + p_data->p_head = p_thread->p_next; + p_data->num_threads--; + } + + p_thread->p_prev = NULL; + p_thread->p_next = NULL; + ABTD_atomic_release_store_int(&p_thread->is_in_pool, 0); + return ABTI_unit_get_builtin_unit(p_thread); + } else { + return ABT_UNIT_NULL; + } +} + +static ABT_unit pool_pop_wait(ABT_pool pool, double time_secs, + ABT_pool_context context) +{ + (void)context; + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + data_t *p_data = pool_get_data_ptr(p_pool->data); + pthread_mutex_lock(&p_data->mutex); if (!p_data->num_threads) { #if defined(ABT_CONFIG_USE_CLOCK_GETTIME) struct timespec ts; @@ -175,29 +247,8 @@ static ABT_unit pool_pop_wait(ABT_pool pool, double time_secs) } #endif } - - if (p_data->num_threads > 0) { - p_thread = p_data->p_head; - if (p_data->num_threads == 1) { - p_data->p_head = NULL; - p_data->p_tail = NULL; - p_data->num_threads = 0; - ABTD_atomic_release_store_int(&p_data->is_empty, 1); - } else { - p_thread->p_prev->p_next = p_thread->p_next; - p_thread->p_next->p_prev = p_thread->p_prev; - p_data->p_head = p_thread->p_next; - p_data->num_threads--; - } - - p_thread->p_prev = NULL; - p_thread->p_next = NULL; - ABTD_atomic_release_store_int(&p_thread->is_in_pool, 0); - - h_unit = ABTI_unit_get_builtin_unit(p_thread); - } + ABT_unit h_unit = pool_pop_unsafe(p_data); pthread_mutex_unlock(&p_data->mutex); - return h_unit; } @@ -212,71 +263,25 @@ static ABT_unit pool_pop_timedwait(ABT_pool pool, double abstime_secs) { ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); - ABTI_thread *p_thread = NULL; - ABT_unit h_unit = ABT_UNIT_NULL; - pthread_mutex_lock(&p_data->mutex); - if (!p_data->num_threads) { struct timespec ts; convert_double_sec_to_timespec(&ts, abstime_secs); pthread_cond_timedwait(&p_data->cond, &p_data->mutex, &ts); } - - if (p_data->num_threads > 0) { - p_thread = p_data->p_head; - if (p_data->num_threads == 1) { - p_data->p_head = NULL; - p_data->p_tail = NULL; - p_data->num_threads = 0; - ABTD_atomic_release_store_int(&p_data->is_empty, 1); - } else { - p_thread->p_prev->p_next = p_thread->p_next; - p_thread->p_next->p_prev = p_thread->p_prev; - p_data->p_head = p_thread->p_next; - p_data->num_threads--; - } - - p_thread->p_prev = NULL; - p_thread->p_next = NULL; - ABTD_atomic_release_store_int(&p_thread->is_in_pool, 0); - - h_unit = ABTI_unit_get_builtin_unit(p_thread); - } + ABT_unit h_unit = pool_pop_unsafe(p_data); pthread_mutex_unlock(&p_data->mutex); - return h_unit; } -static ABT_unit pool_pop(ABT_pool pool) +static ABT_unit pool_pop(ABT_pool pool, ABT_pool_context context) { + (void)context; ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); - ABTI_thread *p_thread = NULL; - if (ABTD_atomic_acquire_load_int(&p_data->is_empty) == 0) { - ABT_unit h_unit = ABT_UNIT_NULL; pthread_mutex_lock(&p_data->mutex); - if (p_data->num_threads > 0) { - p_thread = p_data->p_head; - if (p_data->num_threads == 1) { - p_data->p_head = NULL; - p_data->p_tail = NULL; - p_data->num_threads = 0; - ABTD_atomic_release_store_int(&p_data->is_empty, 1); - } else { - p_thread->p_prev->p_next = p_thread->p_next; - p_thread->p_next->p_prev = p_thread->p_prev; - p_data->p_head = p_thread->p_next; - p_data->num_threads--; - } - - p_thread->p_prev = NULL; - p_thread->p_next = NULL; - ABTD_atomic_release_store_int(&p_thread->is_in_pool, 0); - - h_unit = ABTI_unit_get_builtin_unit(p_thread); - } + ABT_unit h_unit = pool_pop_unsafe(p_data); pthread_mutex_unlock(&p_data->mutex); return h_unit; } else { @@ -284,6 +289,29 @@ static ABT_unit pool_pop(ABT_pool pool) } } +static void pool_pop_many(ABT_pool pool, ABT_unit *units, size_t max_units, + size_t *num_popped, ABT_pool_context context) +{ + (void)context; + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + data_t *p_data = pool_get_data_ptr(p_pool->data); + if (max_units != 0 && + ABTD_atomic_acquire_load_int(&p_data->is_empty) == 0) { + pthread_mutex_lock(&p_data->mutex); + size_t i; + for (i = 0; i < max_units; i++) { + ABT_unit h_unit = pool_pop_unsafe(p_data); + if (h_unit == ABT_UNIT_NULL) + break; + units[i] = h_unit; + } + *num_popped = i; + pthread_mutex_unlock(&p_data->mutex); + } else { + *num_popped = 0; + } +} + static int pool_remove(ABT_pool pool, ABT_unit unit) { ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); @@ -310,24 +338,20 @@ static int pool_remove(ABT_pool pool, ABT_unit unit) } p_data->num_threads--; } - ABTD_atomic_release_store_int(&p_thread->is_in_pool, 0); pthread_mutex_unlock(&p_data->mutex); - p_thread->p_prev = NULL; p_thread->p_next = NULL; - return ABT_SUCCESS; } -static int pool_print_all(ABT_pool pool, void *arg, - void (*print_fn)(void *, ABT_unit)) +static void pool_print_all(ABT_pool pool, void *arg, + void (*print_fn)(void *, ABT_unit)) { ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); data_t *p_data = pool_get_data_ptr(p_pool->data); pthread_mutex_lock(&p_data->mutex); - size_t num_threads = p_data->num_threads; ABTI_thread *p_thread = p_data->p_head; while (num_threads--) { @@ -336,29 +360,26 @@ static int pool_print_all(ABT_pool pool, void *arg, print_fn(arg, unit); p_thread = p_thread->p_next; } - pthread_mutex_unlock(&p_data->mutex); - - return ABT_SUCCESS; } /* Unit functions */ -static ABT_bool unit_is_in_pool(ABT_unit unit) +static ABT_bool pool_unit_is_in_pool(ABT_unit unit) { ABTI_thread *p_thread = ABTI_unit_get_thread_from_builtin_unit(unit); return ABTD_atomic_acquire_load_int(&p_thread->is_in_pool) ? ABT_TRUE : ABT_FALSE; } -static ABT_unit unit_create_from_thread(ABT_thread thread) +static ABT_unit pool_create_unit(ABT_pool pool, ABT_thread thread) { /* Call ABTI_unit_init_builtin() instead. */ ABTI_ASSERT(0); return ABT_UNIT_NULL; } -static void unit_free(ABT_unit *unit) +static void pool_free_unit(ABT_pool pool, ABT_unit unit) { /* A built-in unit does not need to be freed. This function may not be * called. */ diff --git a/src/pool/pool.c b/src/pool/pool.c index 4db2e658..3dcc8752 100644 --- a/src/pool/pool.c +++ b/src/pool/pool.c @@ -5,10 +5,18 @@ #include "abti.h" -ABTU_ret_err static int pool_create(ABTI_pool_def *def, - ABTI_pool_config *p_config, - ABT_bool def_automatic, ABT_bool is_builtin, - ABTI_pool **pp_newpool); +ABTU_ret_err static int pool_create( + ABT_pool_access access, const ABTI_pool_required_def *p_required_def, + const ABTI_pool_optional_def *p_optional_def, + const ABTI_pool_deprecated_def *p_deprecated_def, + const ABTI_pool_old_def *p_old_def, ABTI_pool_config *p_config, + ABT_bool def_automatic, ABT_bool is_builtin, ABTI_pool **pp_newpool); +static void +pool_create_def_from_old_def(const ABT_pool_def *p_def, + ABTI_pool_old_def *p_old_def, + ABTI_pool_required_def *p_required_def, + ABTI_pool_optional_def *p_optional_def, + ABTI_pool_deprecated_def *p_deprecated_def); /** @defgroup POOL Pool * This group is for Pool. @@ -22,8 +30,12 @@ ABTU_ret_err static int pool_create(ABTI_pool_def *def, * (\c def) and a pool configuration (\c config), and returns its handle through * \c newpool. * - * \c def must define all the non-optional functions. See \c #ABT_pool_def for - * details. + * \c def must define all the non-optional functions. See + * \c ABT_pool_user_def_create() for details. + * + * @note + * \c #ABT_pool_def is kept for compatibility. The user is highly recommended + * to use \c ABT_pool_user_def instead. * * The caller of each pool function is undefined, so a program that relies on * the caller of pool functions is non-conforming. @@ -61,11 +73,11 @@ ABTU_ret_err static int pool_create(ABTI_pool_def *def, * @errors * \DOC_ERROR_SUCCESS * \DOC_ERROR_USR_POOL_INIT{\c p_init()} + * \DOC_ERROR_INV_POOL_USER_DEF_HANDLE{\c def} * \DOC_ERROR_RESOURCE * * @undefined * \DOC_UNDEFINED_UNINIT - * \DOC_UNDEFINED_NULL_PTR{\c def} * \DOC_UNDEFINED_NULL_PTR{any non-optional pool function of \c def} * \DOC_UNDEFINED_NULL_PTR{\c newpool} * @@ -74,48 +86,53 @@ ABTU_ret_err static int pool_create(ABTI_pool_def *def, * @param[out] newpool pool handle * @return Error code */ -int ABT_pool_create(ABT_pool_def *def, ABT_pool_config config, +int ABT_pool_create(ABT_pool_user_def def, ABT_pool_config config, ABT_pool *newpool) { ABTI_UB_ASSERT(ABTI_initialized()); ABTI_UB_ASSERT(newpool); ABTI_UB_ASSERT(def); - ABTI_UB_ASSERT(def->u_create_from_thread); - ABTI_UB_ASSERT(def->u_free); - ABTI_UB_ASSERT(def->p_get_size); - ABTI_UB_ASSERT(def->p_push); - ABTI_UB_ASSERT(def->p_pop); - #ifndef ABT_CONFIG_ENABLE_VER_20_API /* Argobots 1.x sets newpool to NULL on error. */ *newpool = ABT_POOL_NULL; #endif + ABT_pool_access access; + ABTI_pool_required_def required_def, *p_required_def; + ABTI_pool_optional_def optional_def, *p_optional_def; + ABTI_pool_deprecated_def deprecated_def, *p_deprecated_def; + ABTI_pool_old_def old_def, *p_old_def; /* Copy def */ - ABTI_pool_def internal_def; - - internal_def.access = def->access; - internal_def.u_is_in_pool = def->u_is_in_pool; - internal_def.u_create_from_thread = def->u_create_from_thread; - internal_def.u_free = def->u_free; - internal_def.p_init = def->p_init; - internal_def.p_get_size = def->p_get_size; - internal_def.p_push = def->p_push; - internal_def.p_pop = def->p_pop; -#ifdef ABT_CONFIG_ENABLE_VER_20_API - internal_def.p_pop_wait = def->p_pop_wait; -#else - internal_def.p_pop_wait = NULL; -#endif - internal_def.p_pop_timedwait = def->p_pop_timedwait; - internal_def.p_remove = def->p_remove; - internal_def.p_free = def->p_free; - internal_def.p_print_all = def->p_print_all; + if (ABTI_pool_user_def_is_new(def)) { + /* New ABTI_pool_user_def */ + access = ABT_POOL_ACCESS_MPMC; + ABTI_pool_user_def *p_def = ABTI_pool_user_def_get_ptr(def); + ABTI_CHECK_NULL_POOL_USER_DEF_PTR(p_def); + p_required_def = &p_def->required_def; + p_optional_def = &p_def->optional_def; + p_deprecated_def = NULL; + p_old_def = NULL; + } else { + /* Old ABT_pool_def */ + ABTI_UB_ASSERT(def->u_create_from_thread); + ABTI_UB_ASSERT(def->u_free); + ABTI_UB_ASSERT(def->p_get_size); + ABTI_UB_ASSERT(def->p_push); + ABTI_UB_ASSERT(def->p_pop); + access = def->access; + pool_create_def_from_old_def(def, &old_def, &required_def, + &optional_def, &deprecated_def); + p_required_def = &required_def; + p_optional_def = &optional_def; + p_deprecated_def = &deprecated_def; + p_old_def = &old_def; + } ABTI_pool *p_newpool; ABTI_pool_config *p_config = ABTI_pool_config_get_ptr(config); const ABT_bool def_automatic = ABT_FALSE; - int abt_errno = pool_create(&internal_def, p_config, def_automatic, - ABT_FALSE, &p_newpool); + int abt_errno = + pool_create(access, p_required_def, p_optional_def, p_deprecated_def, + p_old_def, p_config, def_automatic, ABT_FALSE, &p_newpool); ABTI_CHECK_ERROR(abt_errno); *newpool = ABTI_pool_get_handle(p_newpool); @@ -235,7 +252,7 @@ int ABT_pool_free(ABT_pool *pool) ABT_pool h_pool = *pool; ABTI_pool *p_pool = ABTI_pool_get_ptr(h_pool); ABTI_CHECK_NULL_POOL_PTR(p_pool); - ABTI_UB_ASSERT(p_pool->p_get_size(h_pool) == 0); + ABTI_UB_ASSERT(ABTI_pool_is_empty(p_pool)); ABTI_pool_free(p_pool); @@ -277,6 +294,41 @@ int ABT_pool_get_access(ABT_pool pool, ABT_pool_access *access) return ABT_SUCCESS; } +/** + * @ingroup POOL + * @brief Check if a pool is empty. + * + * \c ABT_pool_is_empty() returns whether the pool \c pool is or not through + * \c is_empty. If \c pool is empty, \c ABT_TRUE is set to \c is_empty. + * Otherwise, \c ABT_FALSE is set to \c is_empty. + * + * @contexts + * \DOC_CONTEXT_INIT \DOC_CONTEXT_NOCTXSWITCH + * + * @errors + * \DOC_ERROR_SUCCESS + * \DOC_ERROR_INV_POOL_HANDLE{\c pool} + * + * @undefined + * \DOC_UNDEFINED_UNINIT + * \DOC_UNDEFINED_NULL_PTR{\c is_empty} + * + * @param[in] pool pool handle + * @param[out] is_empty emptiness of a pool + * @return Error code + */ +int ABT_pool_is_empty(ABT_pool pool, ABT_bool *is_empty) +{ + ABTI_UB_ASSERT(ABTI_initialized()); + ABTI_UB_ASSERT(is_empty); + + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + ABTI_CHECK_NULL_POOL_PTR(p_pool); + + *is_empty = ABTI_pool_is_empty(p_pool); + return ABT_SUCCESS; +} + /** * @ingroup POOL * @brief Get the total size of a pool. @@ -305,6 +357,7 @@ int ABT_pool_get_access(ABT_pool pool, ABT_pool_access *access) * @errors * \DOC_ERROR_SUCCESS * \DOC_ERROR_INV_POOL_HANDLE{\c pool} + * \DOC_ERROR_POOL_UNSUPPORTED_FEATURE{\c pool, \c p_get_size()} * * @undefined * \DOC_UNDEFINED_UNINIT @@ -321,6 +374,7 @@ int ABT_pool_get_total_size(ABT_pool pool, size_t *size) ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); ABTI_CHECK_NULL_POOL_PTR(p_pool); + ABTI_CHECK_TRUE(p_pool->optional_def.p_get_size, ABT_ERR_POOL); *size = ABTI_pool_get_total_size(p_pool); return ABT_SUCCESS; @@ -351,6 +405,7 @@ int ABT_pool_get_total_size(ABT_pool pool, size_t *size) * @errors * \DOC_ERROR_SUCCESS * \DOC_ERROR_INV_POOL_HANDLE{\c pool} + * \DOC_ERROR_POOL_UNSUPPORTED_FEATURE{\c pool, \c p_get_size()} * * @undefined * \DOC_UNDEFINED_UNINIT @@ -367,6 +422,7 @@ int ABT_pool_get_size(ABT_pool pool, size_t *size) ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); ABTI_CHECK_NULL_POOL_PTR(p_pool); + ABTI_CHECK_TRUE(p_pool->optional_def.p_get_size, ABT_ERR_POOL); *size = ABTI_pool_get_size(p_pool); return ABT_SUCCESS; @@ -425,7 +481,7 @@ int ABT_pool_pop(ABT_pool pool, ABT_unit *p_unit) ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); ABTI_CHECK_NULL_POOL_PTR(p_pool); - *p_unit = ABTI_pool_pop(p_pool); + *p_unit = ABTI_pool_pop(p_pool, ABT_POOL_CONTEXT_OP_POOL_OTHER); return ABT_SUCCESS; } @@ -485,9 +541,10 @@ int ABT_pool_pop_wait(ABT_pool pool, ABT_unit *p_unit, double time_secs) ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); ABTI_CHECK_NULL_POOL_PTR(p_pool); - ABTI_CHECK_TRUE(p_pool->p_pop_wait, ABT_ERR_POOL); + ABTI_CHECK_TRUE(p_pool->optional_def.p_pop_wait, ABT_ERR_POOL); - *p_unit = ABTI_pool_pop_wait(p_pool, time_secs); + *p_unit = + ABTI_pool_pop_wait(p_pool, time_secs, ABT_POOL_CONTEXT_OP_POOL_OTHER); return ABT_SUCCESS; } @@ -552,7 +609,7 @@ int ABT_pool_pop_timedwait(ABT_pool pool, ABT_unit *p_unit, double abstime_secs) ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); ABTI_CHECK_NULL_POOL_PTR(p_pool); - ABTI_CHECK_TRUE(p_pool->p_pop_timedwait, ABT_ERR_POOL); + ABTI_CHECK_TRUE(p_pool->deprecated_def.p_pop_timedwait, ABT_ERR_POOL); *p_unit = ABTI_pool_pop_timedwait(p_pool, abstime_secs); return ABT_SUCCESS; @@ -613,7 +670,7 @@ int ABT_pool_push(ABT_pool pool, ABT_unit unit) ABTI_CHECK_ERROR(abt_errno); /* ABTI_unit_set_associated_pool() might change unit, so "unit" must be read * again from p_thread. */ - ABTI_pool_push(p_pool, p_thread->unit); + ABTI_pool_push(p_pool, p_thread->unit, ABT_POOL_CONTEXT_OP_POOL_OTHER); return ABT_SUCCESS; } @@ -663,7 +720,7 @@ int ABT_pool_remove(ABT_pool pool, ABT_unit unit) ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); ABTI_CHECK_NULL_POOL_PTR(p_pool); - ABTI_CHECK_TRUE(p_pool->p_remove, ABT_ERR_POOL); + ABTI_CHECK_TRUE(p_pool->deprecated_def.p_remove, ABT_ERR_POOL); /* unit must be in this pool, so we do not need to reset its associated * pool. */ @@ -723,9 +780,9 @@ int ABT_pool_print_all(ABT_pool pool, void *arg, ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); ABTI_CHECK_NULL_POOL_PTR(p_pool); - ABTI_CHECK_TRUE(p_pool->p_print_all, ABT_ERR_POOL); + ABTI_CHECK_TRUE(p_pool->optional_def.p_print_all, ABT_ERR_POOL); - p_pool->p_print_all(pool, arg, print_fn); + p_pool->optional_def.p_print_all(pool, arg, print_fn); return ABT_SUCCESS; } @@ -920,20 +977,25 @@ ABTU_ret_err int ABTI_pool_create_basic(ABT_pool_kind kind, ABTI_pool **pp_newpool) { int abt_errno; - ABTI_pool_def def; - ABTI_CHECK_TRUE(access == ABT_POOL_ACCESS_PRIV || access == ABT_POOL_ACCESS_SPSC || access == ABT_POOL_ACCESS_MPSC || access == ABT_POOL_ACCESS_SPMC || access == ABT_POOL_ACCESS_MPMC, ABT_ERR_INV_POOL_ACCESS); + + ABTI_pool_required_def required_def; + ABTI_pool_optional_def optional_def; + ABTI_pool_deprecated_def deprecated_def; switch (kind) { case ABT_POOL_FIFO: - abt_errno = ABTI_pool_get_fifo_def(access, &def); + abt_errno = ABTI_pool_get_fifo_def(access, &required_def, + &optional_def, &deprecated_def); break; case ABT_POOL_FIFO_WAIT: - abt_errno = ABTI_pool_get_fifo_wait_def(access, &def); + abt_errno = + ABTI_pool_get_fifo_wait_def(access, &required_def, + &optional_def, &deprecated_def); break; default: abt_errno = ABT_ERR_INV_POOL_KIND; @@ -941,7 +1003,9 @@ ABTU_ret_err int ABTI_pool_create_basic(ABT_pool_kind kind, } ABTI_CHECK_ERROR(abt_errno); - abt_errno = pool_create(&def, NULL, automatic, ABT_TRUE, pp_newpool); + abt_errno = + pool_create(access, &required_def, &optional_def, &deprecated_def, NULL, + NULL, automatic, ABT_TRUE, pp_newpool); ABTI_CHECK_ERROR(abt_errno); return ABT_SUCCESS; } @@ -949,8 +1013,8 @@ ABTU_ret_err int ABTI_pool_create_basic(ABT_pool_kind kind, void ABTI_pool_free(ABTI_pool *p_pool) { ABT_pool h_pool = ABTI_pool_get_handle(p_pool); - if (p_pool->p_free) { - p_pool->p_free(h_pool); + if (p_pool->optional_def.p_free) { + p_pool->optional_def.p_free(h_pool); } ABTU_free(p_pool); } @@ -989,6 +1053,7 @@ void ABTI_pool_print(ABTI_pool *p_pool, FILE *p_os, int indent) "%*saccess : %s\n" "%*sautomatic : %s\n" "%*snum_scheds : %d\n" + "%*sis_empty : %s\n" "%*ssize : %zu\n" "%*snum_blocked : %d\n" "%*sdata : %p\n", @@ -996,7 +1061,10 @@ void ABTI_pool_print(ABTI_pool *p_pool, FILE *p_os, int indent) access, indent, "", (p_pool->automatic == ABT_TRUE) ? "TRUE" : "FALSE", indent, "", ABTD_atomic_acquire_load_int32(&p_pool->num_scheds), indent, "", - ABTI_pool_get_size(p_pool), indent, "", + (ABTI_pool_is_empty(p_pool) ? "TRUE" : "FALSE"), indent, "", + (p_pool->optional_def.p_get_size ? ABTI_pool_get_size(p_pool) + : 0), + indent, "", ABTD_atomic_acquire_load_int32(&p_pool->num_blocked), indent, "", p_pool->data); } @@ -1013,11 +1081,158 @@ void ABTI_pool_reset_id(void) /* Internal static functions */ /*****************************************************************************/ +static ABT_unit pool_create_unit_wrapper(ABT_pool pool, ABT_thread thread) +{ + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + return p_pool->old_def.u_create_from_thread(thread); +} + +static void pool_free_unit_wrapper(ABT_pool pool, ABT_unit unit) +{ + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + p_pool->old_def.u_free(&unit); +} + +static ABT_bool pool_is_empty_wrapper(ABT_pool pool) +{ + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + size_t size = p_pool->old_def.p_get_size(pool); + return (size == 0) ? ABT_TRUE : ABT_FALSE; +} + +static ABT_unit pool_pop_wrapper(ABT_pool pool, ABT_pool_context context) +{ + (void)context; + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + return p_pool->old_def.p_pop(pool); +} + +static void pool_push_wrapper(ABT_pool pool, ABT_unit unit, + ABT_pool_context context) +{ + (void)context; + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + p_pool->old_def.p_push(pool, unit); +} + +static int pool_init_wrapper(ABT_pool pool, ABT_pool_config config) +{ + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + return p_pool->old_def.p_init(pool, config); +} + +static void pool_free_wrapper(ABT_pool pool) +{ + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + p_pool->old_def.p_free(pool); +} + +static size_t pool_get_size_wrapper(ABT_pool pool) +{ + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + return p_pool->old_def.p_get_size(pool); +} + +static ABT_unit pool_pop_wait_wrapper(ABT_pool pool, double time_secs, + ABT_pool_context context) +{ + (void)context; + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + return p_pool->old_def.p_pop_wait(pool, time_secs); +} + +static void pool_pop_many_wrapper(ABT_pool pool, ABT_unit *units, + size_t max_units, size_t *num_popped, + ABT_pool_context context) +{ + (void)context; + size_t i; + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + for (i = 0; i < max_units; i++) { + ABT_unit unit = p_pool->old_def.p_pop(pool); + if (unit != ABT_UNIT_NULL) { + units[i] = unit; + } else { + break; + } + } + *num_popped = i; +} + +static void pool_push_many_wrapper(ABT_pool pool, const ABT_unit *units, + size_t num_units, ABT_pool_context context) +{ + (void)context; + size_t i; + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + for (i = 0; i < num_units; i++) { + p_pool->old_def.p_push(pool, units[i]); + } +} + +static void pool_print_all_wrapper(ABT_pool pool, void *arg, + void (*print_f)(void *, ABT_unit)) +{ + ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); + p_pool->old_def.p_print_all(pool, arg, print_f); +} + +static void +pool_create_def_from_old_def(const ABT_pool_def *p_def, + ABTI_pool_old_def *p_old_def, + ABTI_pool_required_def *p_required_def, + ABTI_pool_optional_def *p_optional_def, + ABTI_pool_deprecated_def *p_deprecated_def) +{ + /* Create p_old_def*/ + p_old_def->u_create_from_thread = p_def->u_create_from_thread; + p_old_def->u_free = p_def->u_free; + p_old_def->p_init = p_def->p_init; + p_old_def->p_get_size = p_def->p_get_size; + p_old_def->p_push = p_def->p_push; + p_old_def->p_pop = p_def->p_pop; +#ifdef ABT_CONFIG_ENABLE_VER_20_API + p_old_def->p_pop_wait = p_def->p_pop_wait; +#else + p_old_def->p_pop_wait = NULL; +#endif + p_old_def->p_free = p_def->p_free; + p_old_def->p_print_all = p_def->p_print_all; + + /* Set up p_required_def */ + p_required_def->p_create_unit = pool_create_unit_wrapper; + p_required_def->p_free_unit = pool_free_unit_wrapper; + p_required_def->p_is_empty = pool_is_empty_wrapper; + p_required_def->p_pop = pool_pop_wrapper; + p_required_def->p_push = pool_push_wrapper; + + /* Set up p_optional_def */ + /* Must be created from ABT_pool_def */ + p_optional_def->p_get_size = pool_get_size_wrapper; + p_optional_def->p_pop_many = pool_pop_many_wrapper; + p_optional_def->p_push_many = pool_push_many_wrapper; + /* Optional */ + p_optional_def->p_init = p_old_def->p_init ? pool_init_wrapper : NULL; + p_optional_def->p_free = p_old_def->p_free ? pool_free_wrapper : NULL; + p_optional_def->p_pop_wait = + p_old_def->p_pop_wait ? pool_pop_wait_wrapper : NULL; + p_optional_def->p_print_all = + p_old_def->p_print_all ? pool_print_all_wrapper : NULL; + + /* Set up p_deprecated_def */ + p_deprecated_def->u_is_in_pool = p_def->u_is_in_pool; + p_deprecated_def->p_pop_timedwait = p_def->p_pop_timedwait; + p_deprecated_def->p_remove = p_def->p_remove; +} + static inline uint64_t pool_get_new_id(void); -ABTU_ret_err static int pool_create(ABTI_pool_def *def, - ABTI_pool_config *p_config, - ABT_bool def_automatic, ABT_bool is_builtin, - ABTI_pool **pp_newpool) +ABTU_ret_err static int +pool_create(ABT_pool_access access, + const ABTI_pool_required_def *p_required_def, + const ABTI_pool_optional_def *p_optional_def, + const ABTI_pool_deprecated_def *p_deprecated_def, + const ABTI_pool_old_def *p_old_def, ABTI_pool_config *p_config, + ABT_bool def_automatic, ABT_bool is_builtin, ABTI_pool **pp_newpool) { int abt_errno; ABTI_pool *p_pool; @@ -1036,32 +1251,38 @@ ABTU_ret_err static int pool_create(ABTI_pool_def *def, } } - p_pool->access = def->access; + p_pool->access = access; p_pool->automatic = automatic; p_pool->is_builtin = is_builtin; ABTD_atomic_release_store_int32(&p_pool->num_scheds, 0); ABTD_atomic_release_store_int32(&p_pool->num_blocked, 0); p_pool->data = NULL; - - /* Set up the pool functions from def */ - p_pool->u_is_in_pool = def->u_is_in_pool; - p_pool->u_create_from_thread = def->u_create_from_thread; - p_pool->u_free = def->u_free; - p_pool->p_init = def->p_init; - p_pool->p_get_size = def->p_get_size; - p_pool->p_push = def->p_push; - p_pool->p_pop = def->p_pop; - p_pool->p_pop_wait = def->p_pop_wait; - p_pool->p_pop_timedwait = def->p_pop_timedwait; - p_pool->p_remove = def->p_remove; - p_pool->p_free = def->p_free; - p_pool->p_print_all = def->p_print_all; + memcpy(&p_pool->required_def, p_required_def, + sizeof(ABTI_pool_required_def)); + if (p_optional_def) { + memcpy(&p_pool->optional_def, p_optional_def, + sizeof(ABTI_pool_optional_def)); + } else { + memset(&p_pool->optional_def, 0, sizeof(ABTI_pool_optional_def)); + } + if (p_deprecated_def) { + memcpy(&p_pool->deprecated_def, p_deprecated_def, + sizeof(ABTI_pool_deprecated_def)); + } else { + memset(&p_pool->deprecated_def, 0, sizeof(ABTI_pool_deprecated_def)); + } + if (p_old_def) { + memcpy(&p_pool->old_def, p_old_def, sizeof(ABTI_pool_old_def)); + } else { + memset(&p_pool->old_def, 0, sizeof(ABTI_pool_old_def)); + } p_pool->id = pool_get_new_id(); /* Configure the pool */ - if (p_pool->p_init) { + if (p_pool->optional_def.p_init) { ABT_pool_config config = ABTI_pool_config_get_handle(p_config); - abt_errno = p_pool->p_init(ABTI_pool_get_handle(p_pool), config); + abt_errno = + p_pool->optional_def.p_init(ABTI_pool_get_handle(p_pool), config); if (abt_errno != ABT_SUCCESS) { ABTU_free(p_pool); return abt_errno; diff --git a/src/pool/pool_user_def.c b/src/pool/pool_user_def.c new file mode 100644 index 00000000..c3370caf --- /dev/null +++ b/src/pool/pool_user_def.c @@ -0,0 +1,373 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * See COPYRIGHT in top-level directory. + */ + +#include "abti.h" + +#include +#include +#include + +/** @defgroup POOL_USER_DEF Pool definition + * This group is for Pool definition. + */ + +/** + * @ingroup POOL_USER_DEF + * @brief Create a new pool definition. + * + * \c ABT_pool_user_def_create() creates a new pool definition and returns its + * handle through \c newdef. \c p_create_unit, \c p_free_unit, \c p_is_empty, + * \c p_pop, \c p_push are registered to \c newdef. + * + * \c newdef must be freed by \c ABT_pool_user_def_free() after its use. + * + * @contexts + * \DOC_CONTEXT_INIT \DOC_CONTEXT_NOCTXSWITCH + * + * @errors + * \DOC_ERROR_SUCCESS + * \DOC_ERROR_RESOURCE + * + * @undefined + * \DOC_UNDEFINED_UNINIT + * \DOC_UNDEFINED_NULL_PTR{\c newdef} + * \DOC_UNDEFINED_NULL_PTR{\c p_create_unit} + * \DOC_UNDEFINED_NULL_PTR{\c p_free_unit} + * \DOC_UNDEFINED_NULL_PTR{\c p_is_empty} + * \DOC_UNDEFINED_NULL_PTR{\c p_pop} + * \DOC_UNDEFINED_NULL_PTR{\c p_push} + * + * @param[in] p_create_unit unit creation function + * @param[in] p_free_unit unit release function + * @param[in] p_is_empty emptiness check function + * @param[in] p_pop pop function + * @param[in] p_push push function + * @param[out] newdef pool definition handle + * @return Error code + */ +int ABT_pool_user_def_create(ABT_pool_user_create_unit_fn p_create_unit, + ABT_pool_user_free_unit_fn p_free_unit, + ABT_pool_user_is_empty_fn p_is_empty, + ABT_pool_user_pop_fn p_pop, + ABT_pool_user_push_fn p_push, + ABT_pool_user_def *newdef) +{ + ABTI_UB_ASSERT(ABTI_initialized()); + ABTI_UB_ASSERT(newdef); + ABTI_UB_ASSERT(p_create_unit); + ABTI_UB_ASSERT(p_free_unit); + ABTI_UB_ASSERT(p_is_empty); + ABTI_UB_ASSERT(p_pop); + ABTI_UB_ASSERT(p_push); + + ABTI_pool_user_def *p_newdef; + int abt_errno = + ABTU_calloc(1, sizeof(ABTI_pool_user_def), (void **)&p_newdef); + ABTI_CHECK_ERROR(abt_errno); + + ABTI_UB_ASSERT(p_newdef->symbol == NULL); /* This value must be NULL. */ + /* Set values */ + p_newdef->required_def.p_create_unit = p_create_unit; + p_newdef->required_def.p_free_unit = p_free_unit; + p_newdef->required_def.p_is_empty = p_is_empty; + p_newdef->required_def.p_pop = p_pop; + p_newdef->required_def.p_push = p_push; + + *newdef = ABTI_pool_user_def_get_handle(p_newdef); + return ABT_SUCCESS; +} + +/** + * @ingroup POOL_USER_DEF + * @brief Free a pool definition. + * + * \c ABT_pool_user_def_free() deallocates the resource used for the pool + * definition \c def and sets \c def to \c ABT_POOL_USER_DEF_NULL. + * + * @contexts + * \DOC_CONTEXT_INIT \DOC_CONTEXT_NOCTXSWITCH + * + * @errors + * \DOC_ERROR_SUCCESS + * \DOC_ERROR_INV_POOL_USER_DEF_PTR{\c def} + * + * @undefined + * \DOC_UNDEFINED_UNINIT + * \DOC_UNDEFINED_NULL_PTR{\c def} + * \DOC_UNDEFINED_THREAD_UNSAFE_FREE{\c def} + * + * @param[in,out] def pool definition handle + * @return Error code + */ +int ABT_pool_user_def_free(ABT_pool_user_def *def) +{ + ABTI_UB_ASSERT(ABTI_initialized()); + ABTI_UB_ASSERT(def); + + ABT_pool_user_def h_def = *def; + ABTI_pool_user_def *p_def = ABTI_pool_user_def_get_ptr(h_def); + ABTI_CHECK_NULL_POOL_USER_DEF_PTR(p_def); + + /* Free the memory */ + ABTU_free(p_def); + *def = ABT_POOL_USER_DEF_NULL; + return ABT_SUCCESS; +} + +/** + * @ingroup POOL_USER_DEF + * @brief Register a pool initialization function to a pool definition. + * + * \c ABT_pool_user_def_set_init() registers the pool initialization function + * \c p_init to a pool definition \c def. If \c p_init is \c NULL, the + * corresponding function is removed from \c def. + * + * @contexts + * \DOC_CONTEXT_INIT \DOC_CONTEXT_NOCTXSWITCH + * + * @errors + * \DOC_ERROR_SUCCESS + * \DOC_ERROR_INV_POOL_USER_DEF_HANDLE{\c def} + * + * @undefined + * \DOC_UNDEFINED_UNINIT + * \DOC_UNDEFINED_THREAD_UNSAFE_FREE{\c def} + * + * @param[in] def pool definition handle + * @param[in] p_init pool initialization function + * @return Error code + */ +int ABT_pool_user_def_set_init(ABT_pool_user_def def, + ABT_pool_user_init_fn p_init) +{ + ABTI_UB_ASSERT(ABTI_initialized()); + + ABTI_pool_user_def *p_def = ABTI_pool_user_def_get_ptr(def); + ABTI_CHECK_NULL_POOL_USER_DEF_PTR(p_def); + + p_def->optional_def.p_init = p_init; + return ABT_SUCCESS; +} + +/** + * @ingroup POOL_USER_DEF + * @brief Register a pool finalization function to a pool definition. + * + * \c ABT_pool_user_def_set_free() registers the pool finalization function + * \c p_free to a pool definition \c def. If \c p_free is \c NULL, the + * corresponding function is removed from \c def. + * + * @contexts + * \DOC_CONTEXT_INIT \DOC_CONTEXT_NOCTXSWITCH + * + * @errors + * \DOC_ERROR_SUCCESS + * \DOC_ERROR_INV_POOL_USER_DEF_HANDLE{\c def} + * + * @undefined + * \DOC_UNDEFINED_UNINIT + * \DOC_UNDEFINED_THREAD_UNSAFE_FREE{\c def} + * + * @param[in] def pool definition handle + * @param[in] p_free pool finalization function + * @return Error code + */ +int ABT_pool_user_def_set_free(ABT_pool_user_def def, + ABT_pool_user_free_fn p_free) +{ + ABTI_UB_ASSERT(ABTI_initialized()); + + ABTI_pool_user_def *p_def = ABTI_pool_user_def_get_ptr(def); + ABTI_CHECK_NULL_POOL_USER_DEF_PTR(p_def); + + p_def->optional_def.p_free = p_free; + return ABT_SUCCESS; +} + +/** + * @ingroup POOL_USER_DEF + * @brief Register a size inquiry function to a pool definition. + * + * \c ABT_pool_user_def_set_get_size() registers the size inquiry function + * \c p_get_size to a pool definition \c def. If \c p_get_size is \c NULL, the + * corresponding function is removed from \c def. + * + * @contexts + * \DOC_CONTEXT_INIT \DOC_CONTEXT_NOCTXSWITCH + * + * @errors + * \DOC_ERROR_SUCCESS + * \DOC_ERROR_INV_POOL_USER_DEF_HANDLE{\c def} + * + * @undefined + * \DOC_UNDEFINED_UNINIT + * \DOC_UNDEFINED_THREAD_UNSAFE_FREE{\c def} + * + * @param[in] def pool definition handle + * @param[in] p_get_size size inquiry function + * @return Error code + */ +int ABT_pool_user_def_set_get_size(ABT_pool_user_def def, + ABT_pool_user_get_size_fn p_get_size) +{ + ABTI_UB_ASSERT(ABTI_initialized()); + + ABTI_pool_user_def *p_def = ABTI_pool_user_def_get_ptr(def); + ABTI_CHECK_NULL_POOL_USER_DEF_PTR(p_def); + + p_def->optional_def.p_get_size = p_get_size; + return ABT_SUCCESS; +} + +/** + * @ingroup POOL_USER_DEF + * @brief Register a pop-wait function to a pool definition. + * + * \c ABT_pool_user_def_set_pop_wait() registers the pop-wait function + * \c p_pop_wait to a pool definition \c def. If \c p_pop_wait is \c NULL, the + * corresponding function is removed from \c def. + * + * @contexts + * \DOC_CONTEXT_INIT \DOC_CONTEXT_NOCTXSWITCH + * + * @errors + * \DOC_ERROR_SUCCESS + * \DOC_ERROR_INV_POOL_USER_DEF_HANDLE{\c def} + * + * @undefined + * \DOC_UNDEFINED_UNINIT + * \DOC_UNDEFINED_THREAD_UNSAFE_FREE{\c def} + * + * @param[in] def pool definition handle + * @param[in] p_pop_wait pop-wait function + * @return Error code + */ +int ABT_pool_user_def_set_pop_wait(ABT_pool_user_def def, + ABT_pool_user_pop_wait_fn p_pop_wait) +{ + ABTI_UB_ASSERT(ABTI_initialized()); + + ABTI_pool_user_def *p_def = ABTI_pool_user_def_get_ptr(def); + ABTI_CHECK_NULL_POOL_USER_DEF_PTR(p_def); + + p_def->optional_def.p_pop_wait = p_pop_wait; + return ABT_SUCCESS; +} + +/** + * @ingroup POOL_USER_DEF + * @brief Register a pop-many function to a pool definition. + * + * \c ABT_pool_user_def_set_pop_many() registers the pop-many function + * \c p_pop_many to a pool definition \c def. If \c p_pop_many is \c NULL, the + * corresponding function is removed from \c def. + * + * @contexts + * \DOC_CONTEXT_INIT \DOC_CONTEXT_NOCTXSWITCH + * + * @errors + * \DOC_ERROR_SUCCESS + * \DOC_ERROR_INV_POOL_USER_DEF_HANDLE{\c def} + * + * @undefined + * \DOC_UNDEFINED_UNINIT + * \DOC_UNDEFINED_THREAD_UNSAFE_FREE{\c def} + * + * @param[in] def pool definition handle + * @param[in] p_pop_many pop-many function + * @return Error code + */ +int ABT_pool_user_def_set_pop_many(ABT_pool_user_def def, + ABT_pool_user_pop_many_fn p_pop_many) +{ + ABTI_UB_ASSERT(ABTI_initialized()); + + ABTI_pool_user_def *p_def = ABTI_pool_user_def_get_ptr(def); + ABTI_CHECK_NULL_POOL_USER_DEF_PTR(p_def); + + p_def->optional_def.p_pop_many = p_pop_many; + return ABT_SUCCESS; +} + +/** + * @ingroup POOL_USER_DEF + * @brief Register a push-many function to a pool definition. + * + * \c ABT_pool_user_def_set_push_many() registers the push-many function + * \c p_push_many to a pool definition \c def. If \c p_push_many is \c NULL, + * the corresponding function is removed from \c def. + * + * @contexts + * \DOC_CONTEXT_INIT \DOC_CONTEXT_NOCTXSWITCH + * + * @errors + * \DOC_ERROR_SUCCESS + * \DOC_ERROR_INV_POOL_USER_DEF_HANDLE{\c def} + * + * @undefined + * \DOC_UNDEFINED_UNINIT + * \DOC_UNDEFINED_THREAD_UNSAFE_FREE{\c def} + * + * @param[in] def pool definition handle + * @param[in] p_push_many push-many function + * @return Error code + */ +int ABT_pool_user_def_set_push_many(ABT_pool_user_def def, + ABT_pool_user_push_many_fn p_push_many) +{ + ABTI_UB_ASSERT(ABTI_initialized()); + + ABTI_pool_user_def *p_def = ABTI_pool_user_def_get_ptr(def); + ABTI_CHECK_NULL_POOL_USER_DEF_PTR(p_def); + + p_def->optional_def.p_push_many = p_push_many; + return ABT_SUCCESS; +} + +/** + * @ingroup POOL_USER_DEF + * @brief Register a print-all function to a pool definition. + * + * \c ABT_pool_user_def_set_print_all() registers the print-all function + * \c p_print_all to a pool definition \c def. If \c p_print_all is \c NULL, + * the corresponding function is removed from \c def. + * + * @contexts + * \DOC_CONTEXT_INIT \DOC_CONTEXT_NOCTXSWITCH + * + * @errors + * \DOC_ERROR_SUCCESS + * \DOC_ERROR_INV_POOL_USER_DEF_HANDLE{\c def} + * + * @undefined + * \DOC_UNDEFINED_UNINIT + * \DOC_UNDEFINED_THREAD_UNSAFE_FREE{\c def} + * + * @param[in] def pool definition handle + * @param[in] p_print_all print-all function + * @return Error code + */ +int ABT_pool_user_def_set_print_all(ABT_pool_user_def def, + ABT_pool_user_print_all_fn p_print_all) +{ + ABTI_UB_ASSERT(ABTI_initialized()); + + ABTI_pool_user_def *p_def = ABTI_pool_user_def_get_ptr(def); + ABTI_CHECK_NULL_POOL_USER_DEF_PTR(p_def); + + p_def->optional_def.p_print_all = p_print_all; + return ABT_SUCCESS; +} + +/*****************************************************************************/ +/* Private APIs */ +/*****************************************************************************/ + +ABT_bool ABTI_pool_user_def_is_new(const ABT_pool_user_def def) +{ + /* If def points to ABT_pool_def, this u_create_from_thread not be NULL. + * Otherwise, it is "symbol", so it must be NULL. */ + return def->u_create_from_thread ? ABT_FALSE : ABT_TRUE; +} diff --git a/src/sched/basic.c b/src/sched/basic.c index 3b77b91c..c6a0b83d 100644 --- a/src/sched/basic.c +++ b/src/sched/basic.c @@ -115,7 +115,9 @@ static void sched_run(ABT_sched sched) for (i = 0; i < num_pools; i++) { ABTI_pool *p_pool = ABTI_pool_get_ptr(pools[i]); ++pop_count; - if ((unit = ABTI_pool_pop(p_pool)) != ABT_UNIT_NULL) { + if ((unit = + ABTI_pool_pop(p_pool, ABT_POOL_CONTEXT_OP_POOL_OTHER)) != + ABT_UNIT_NULL) { ABTI_thread *p_thread = ABTI_unit_get_thread(p_global, unit); ABTI_ythread_schedule(p_global, &p_local_xstream, p_thread); break; @@ -124,10 +126,8 @@ static void sched_run(ABT_sched sched) /* if we attempted event_freq pops, check for events */ if (pop_count >= event_freq) { ABTI_xstream_check_events(p_local_xstream, p_sched); - ABTI_local *p_local = ABTI_xstream_get_local(p_local_xstream); - if (ABTI_sched_has_to_stop(&p_local, p_sched) == ABT_TRUE) + if (ABTI_sched_has_to_stop(p_sched) == ABT_TRUE) break; - p_local_xstream = ABTI_local_get_xstream(p_local); SCHED_SLEEP(unit != ABT_UNIT_NULL, p_data->sleep_time); pop_count = 0; } diff --git a/src/sched/basic_wait.c b/src/sched/basic_wait.c index 1c9e9d5e..d7144ff9 100644 --- a/src/sched/basic_wait.c +++ b/src/sched/basic_wait.c @@ -111,7 +111,8 @@ static void sched_run(ABT_sched sched) ABT_pool pool = pools[i]; ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); /* Pop one work unit */ - ABT_unit unit = ABTI_pool_pop(p_pool); + ABT_unit unit = + ABTI_pool_pop(p_pool, ABT_POOL_CONTEXT_OP_POOL_OTHER); if (unit != ABT_UNIT_NULL) { ABTI_thread *p_thread = ABTI_unit_get_thread(p_global, unit); ABTI_ythread_schedule(p_global, &p_local_xstream, p_thread); @@ -125,13 +126,14 @@ static void sched_run(ABT_sched sched) if (!run_cnt_nowait) { ABTI_pool *p_pool = ABTI_pool_get_ptr(pools[0]); ABT_unit unit; - if (p_pool->p_pop_wait) { - unit = ABTI_pool_pop_wait(p_pool, 0.1); - } else if (p_pool->p_pop_timedwait) { + if (p_pool->optional_def.p_pop_wait) { + unit = ABTI_pool_pop_wait(p_pool, 0.1, + ABT_POOL_CONTEXT_OP_POOL_OTHER); + } else if (p_pool->deprecated_def.p_pop_timedwait) { unit = ABTI_pool_pop_timedwait(p_pool, ABTI_get_wtime() + 0.1); } else { /* No "wait" pop, so let's use a normal one. */ - unit = ABTI_pool_pop(p_pool); + unit = ABTI_pool_pop(p_pool, ABT_POOL_CONTEXT_OP_POOL_OTHER); } if (unit != ABT_UNIT_NULL) { ABTI_thread *p_thread = ABTI_unit_get_thread(p_global, unit); @@ -146,10 +148,8 @@ static void sched_run(ABT_sched sched) * be processed in a timely manner. */ if (!run_cnt_nowait || (++work_count >= event_freq)) { ABTI_xstream_check_events(p_local_xstream, p_sched); - ABTI_local *p_local = ABTI_xstream_get_local(p_local_xstream); - if (ABTI_sched_has_to_stop(&p_local, p_sched) == ABT_TRUE) + if (ABTI_sched_has_to_stop(p_sched) == ABT_TRUE) break; - p_local_xstream = ABTI_local_get_xstream(p_local); work_count = 0; } } diff --git a/src/sched/prio.c b/src/sched/prio.c index 06b97a06..149eb1ae 100644 --- a/src/sched/prio.c +++ b/src/sched/prio.c @@ -111,7 +111,8 @@ static void sched_run(ABT_sched sched) for (i = 0; i < num_pools; i++) { ABT_pool pool = pools[i]; ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); - ABT_unit unit = ABTI_pool_pop(p_pool); + ABT_unit unit = + ABTI_pool_pop(p_pool, ABT_POOL_CONTEXT_OP_POOL_OTHER); if (unit != ABT_UNIT_NULL) { ABTI_thread *p_thread = ABTI_unit_get_thread(p_global, unit); ABTI_ythread_schedule(p_global, &p_local_xstream, p_thread); @@ -122,10 +123,8 @@ static void sched_run(ABT_sched sched) if (++work_count >= event_freq) { ABTI_xstream_check_events(p_local_xstream, p_sched); - ABTI_local *p_local = ABTI_xstream_get_local(p_local_xstream); - if (ABTI_sched_has_to_stop(&p_local, p_sched) == ABT_TRUE) + if (ABTI_sched_has_to_stop(p_sched) == ABT_TRUE) break; - p_local_xstream = ABTI_local_get_xstream(p_local); work_count = 0; SCHED_SLEEP(run_cnt, p_data->sleep_time); } diff --git a/src/sched/randws.c b/src/sched/randws.c index 593301de..aa636119 100644 --- a/src/sched/randws.c +++ b/src/sched/randws.c @@ -106,7 +106,7 @@ static void sched_run(ABT_sched sched) /* Execute one work unit from the scheduler's pool */ ABT_pool pool = pools[0]; ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); - unit = ABTI_pool_pop(p_pool); + unit = ABTI_pool_pop(p_pool, ABT_POOL_CONTEXT_OWNER_PRIMARY); if (unit != ABT_UNIT_NULL) { ABTI_thread *p_thread = ABTI_unit_get_thread(p_global, unit); ABTI_ythread_schedule(p_global, &p_local_xstream, p_thread); @@ -117,7 +117,7 @@ static void sched_run(ABT_sched sched) (num_pools == 2) ? 1 : (rand_r(&seed) % (num_pools - 1) + 1); pool = pools[target]; p_pool = ABTI_pool_get_ptr(pool); - unit = ABTI_pool_pop(p_pool); + unit = ABTI_pool_pop(p_pool, ABT_POOL_CONTEXT_OWNER_SECONDARY); if (unit != ABT_UNIT_NULL) { ABTI_thread *p_thread = ABTI_unit_get_thread(p_global, unit); ABTI_ythread_schedule(p_global, &p_local_xstream, p_thread); @@ -127,10 +127,8 @@ static void sched_run(ABT_sched sched) if (++work_count >= p_data->event_freq) { ABTI_xstream_check_events(p_local_xstream, p_sched); - ABTI_local *p_local = ABTI_xstream_get_local(p_local_xstream); - if (ABTI_sched_has_to_stop(&p_local, p_sched) == ABT_TRUE) + if (ABTI_sched_has_to_stop(p_sched) == ABT_TRUE) break; - p_local_xstream = ABTI_local_get_xstream(p_local); work_count = 0; SCHED_SLEEP(run_cnt, p_data->sleep_time); } diff --git a/src/sched/sched.c b/src/sched/sched.c index 9bbc2c31..41c0351d 100644 --- a/src/sched/sched.c +++ b/src/sched/sched.c @@ -487,14 +487,13 @@ int ABT_sched_has_to_stop(ABT_sched sched, ABT_bool *stop) #ifndef ABT_CONFIG_ENABLE_VER_20_API *stop = ABT_FALSE; #endif - ABTI_local *p_local = ABTI_local_get_local(); ABTI_sched *p_sched = ABTI_sched_get_ptr(sched); ABTI_CHECK_NULL_SCHED_PTR(p_sched); #ifndef ABT_CONFIG_ENABLE_VER_20_API - ABTI_CHECK_TRUE(p_local, ABT_ERR_INV_XSTREAM); + ABTI_CHECK_TRUE(ABTI_local_get_local(), ABT_ERR_INV_XSTREAM); #endif - *stop = ABTI_sched_has_to_stop(&p_local, p_sched); + *stop = ABTI_sched_has_to_stop(p_sched); return ABT_SUCCESS; } @@ -590,6 +589,8 @@ int ABT_sched_get_data(ABT_sched sched, void **data) * @errors * \DOC_ERROR_SUCCESS * \DOC_ERROR_INV_SCHED_HANDLE{\c sched} + * \DOC_ERROR_POOL_UNSUPPORTED_FEATURE{one of the associated pools, + * \c p_get_size()} * * @undefined * \DOC_UNDEFINED_UNINIT @@ -610,8 +611,20 @@ int ABT_sched_get_size(ABT_sched sched, size_t *size) ABTI_sched *p_sched = ABTI_sched_get_ptr(sched); ABTI_CHECK_NULL_SCHED_PTR(p_sched); + /* Check availability of p_get_size() */ + size_t p; + for (p = 0; p < p_sched->num_pools; p++) { + ABTI_pool *p_pool = ABTI_pool_get_ptr(p_sched->pools[p]); + ABTI_CHECK_TRUE(p_pool->optional_def.p_get_size, ABT_ERR_POOL); + } - *size = ABTI_sched_get_size(p_sched); + /* Sum up all the sizes */ + size_t pool_size = 0; + for (p = 0; p < p_sched->num_pools; p++) { + ABTI_pool *p_pool = ABTI_pool_get_ptr(p_sched->pools[p]); + pool_size += ABTI_pool_get_size(p_pool); + } + *size = pool_size; return ABT_SUCCESS; } @@ -641,6 +654,8 @@ int ABT_sched_get_size(ABT_sched sched, size_t *size) * @errors * \DOC_ERROR_SUCCESS * \DOC_ERROR_INV_SCHED_HANDLE{\c sched} + * \DOC_ERROR_POOL_UNSUPPORTED_FEATURE{one of the associated pools, + * \c p_get_size()} * * @undefined * \DOC_UNDEFINED_UNINIT @@ -661,8 +676,20 @@ int ABT_sched_get_total_size(ABT_sched sched, size_t *size) ABTI_sched *p_sched = ABTI_sched_get_ptr(sched); ABTI_CHECK_NULL_SCHED_PTR(p_sched); + /* Check availability of p_get_size() */ + size_t p; + for (p = 0; p < p_sched->num_pools; p++) { + ABTI_pool *p_pool = ABTI_pool_get_ptr(p_sched->pools[p]); + ABTI_CHECK_TRUE(p_pool->optional_def.p_get_size, ABT_ERR_POOL); + } - *size = ABTI_sched_get_total_size(p_sched); + /* Sum up all the sizes */ + size_t pool_size = 0; + for (p = 0; p < p_sched->num_pools; p++) { + ABTI_pool *p_pool = ABTI_pool_get_ptr(p_sched->pools[p]); + pool_size += ABTI_pool_get_total_size(p_pool); + } + *size = pool_size; return ABT_SUCCESS; } @@ -670,18 +697,6 @@ int ABT_sched_get_total_size(ABT_sched sched, size_t *size) /* Private APIs */ /*****************************************************************************/ -size_t ABTI_sched_get_size(ABTI_sched *p_sched) -{ - size_t pool_size = 0, p; - - for (p = 0; p < p_sched->num_pools; p++) { - ABTI_pool *p_pool = ABTI_pool_get_ptr(p_sched->pools[p]); - pool_size += ABTI_pool_get_size(p_pool); - } - - return pool_size; -} - void ABTI_sched_finish(ABTI_sched *p_sched) { ABTI_sched_set_request(p_sched, ABTI_SCHED_REQ_FINISH); @@ -907,7 +922,7 @@ void ABTI_sched_free(ABTI_global *p_global, ABTI_local *p_local, ABTU_free(p_sched); } -ABT_bool ABTI_sched_has_to_stop(ABTI_local **pp_local, ABTI_sched *p_sched) +ABT_bool ABTI_sched_has_to_stop(ABTI_sched *p_sched) { /* Check exit request */ if (ABTD_atomic_acquire_load_uint32(&p_sched->request) & @@ -915,11 +930,11 @@ ABT_bool ABTI_sched_has_to_stop(ABTI_local **pp_local, ABTI_sched *p_sched) return ABT_TRUE; } - if (ABTI_sched_get_effective_size(*pp_local, p_sched) == 0) { + if (!ABTI_sched_has_unit(p_sched)) { if (ABTD_atomic_acquire_load_uint32(&p_sched->request) & (ABTI_SCHED_REQ_FINISH | ABTI_SCHED_REQ_REPLACE)) { /* Check join request */ - if (ABTI_sched_get_effective_size(*pp_local, p_sched) == 0) + if (!ABTI_sched_has_unit(p_sched)) return ABT_TRUE; } else if (p_sched->used == ABTI_SCHED_IN_POOL) { /* Let's finish it anyway. @@ -930,69 +945,56 @@ ABT_bool ABTI_sched_has_to_stop(ABTI_local **pp_local, ABTI_sched *p_sched) return ABT_FALSE; } -/* Get the pool suitable for receiving a migrating thread */ -ABTU_ret_err int ABTI_sched_get_migration_pool(ABTI_sched *p_sched, - ABTI_pool *source_pool, - ABTI_pool **pp_pool) -{ - /* Find a pool. If get_migr_pool is not defined, we pick the first pool */ - if (p_sched->get_migr_pool == NULL) { - ABTI_CHECK_TRUE(p_sched->num_pools > 0, ABT_ERR_MIGRATION_TARGET); - *pp_pool = ABTI_pool_get_ptr(p_sched->pools[0]); - } else { - ABT_sched sched = ABTI_sched_get_handle(p_sched); - ABTI_pool *p_pool = ABTI_pool_get_ptr(p_sched->get_migr_pool(sched)); - ABTI_CHECK_TRUE(p_pool, ABT_ERR_MIGRATION_TARGET); - *pp_pool = p_pool; - } - return ABT_SUCCESS; -} - -size_t ABTI_sched_get_total_size(ABTI_sched *p_sched) +ABT_bool ABTI_sched_has_unit(ABTI_sched *p_sched) { - size_t pool_size = 0, p; - - for (p = 0; p < p_sched->num_pools; p++) { - ABTI_pool *p_pool = ABTI_pool_get_ptr(p_sched->pools[p]); - pool_size += ABTI_pool_get_total_size(p_pool); - } - - return pool_size; -} - -/* Compared to \c ABTI_sched_get_total_size, ABTI_sched_get_effective_size does - * not count the number of blocked ULTs if a pool has more than one consumer or - * the caller ES is not the latest consumer. This is necessary when the ES - * associated with the target scheduler has to be joined and the pool is shared - * between different schedulers associated with different ESs. */ -size_t ABTI_sched_get_effective_size(ABTI_local *p_local, ABTI_sched *p_sched) -{ - size_t pool_size = 0, p; - - for (p = 0; p < p_sched->num_pools; p++) { + /* ABTI_sched_has_unit() does not count the number of blocked ULTs if a pool + * has more than one consumer or the caller ES is not the latest consumer. + * This is necessary when the ES associated with the target scheduler has to + * be joined and the pool is shared between different schedulers associated + * with different ESs. */ + size_t p, num_pools = p_sched->num_pools; + for (p = 0; p < num_pools; p++) { ABT_pool pool = p_sched->pools[p]; ABTI_pool *p_pool = ABTI_pool_get_ptr(pool); - pool_size += ABTI_pool_get_size(p_pool); + if (!ABTI_pool_is_empty(p_pool)) + return ABT_TRUE; switch (p_pool->access) { case ABT_POOL_ACCESS_PRIV: - pool_size += - ABTD_atomic_acquire_load_int32(&p_pool->num_blocked); + if (ABTD_atomic_acquire_load_int32(&p_pool->num_blocked)) + return ABT_TRUE; break; case ABT_POOL_ACCESS_SPSC: case ABT_POOL_ACCESS_MPSC: case ABT_POOL_ACCESS_SPMC: case ABT_POOL_ACCESS_MPMC: if (ABTD_atomic_acquire_load_int32(&p_pool->num_scheds) == 1) { - pool_size += - ABTD_atomic_acquire_load_int32(&p_pool->num_blocked); + if (ABTD_atomic_acquire_load_int32(&p_pool->num_blocked)) + return ABT_TRUE; } break; default: break; } } + return ABT_FALSE; +} - return pool_size; +/* Get the pool suitable for receiving a migrating thread */ +ABTU_ret_err int ABTI_sched_get_migration_pool(ABTI_sched *p_sched, + ABTI_pool *source_pool, + ABTI_pool **pp_pool) +{ + /* Find a pool. If get_migr_pool is not defined, we pick the first pool */ + if (p_sched->get_migr_pool == NULL) { + ABTI_CHECK_TRUE(p_sched->num_pools > 0, ABT_ERR_MIGRATION_TARGET); + *pp_pool = ABTI_pool_get_ptr(p_sched->pools[0]); + } else { + ABT_sched sched = ABTI_sched_get_handle(p_sched); + ABTI_pool *p_pool = ABTI_pool_get_ptr(p_sched->get_migr_pool(sched)); + ABTI_CHECK_TRUE(p_pool, ABT_ERR_MIGRATION_TARGET); + *pp_pool = p_pool; + } + return ABT_SUCCESS; } void ABTI_sched_print(ABTI_sched *p_sched, FILE *p_os, int indent, @@ -1042,8 +1044,7 @@ void ABTI_sched_print(ABTI_sched *p_sched, FILE *p_os, int indent, "%*sautomatic: %s\n" "%*srequest : 0x%x\n" "%*snum_pools: %zu\n" - "%*ssize : %zu\n" - "%*stot_size : %zu\n" + "%*shas_unit : %s\n" "%*sthread : %p\n" "%*sdata : %p\n", indent, "", (void *)p_sched, @@ -1054,8 +1055,7 @@ void ABTI_sched_print(ABTI_sched *p_sched, FILE *p_os, int indent, "", (p_sched->automatic == ABT_TRUE) ? "TRUE" : "FALSE", indent, "", ABTD_atomic_acquire_load_uint32(&p_sched->request), indent, "", p_sched->num_pools, indent, "", - ABTI_sched_get_size(p_sched), indent, "", - ABTI_sched_get_total_size(p_sched), indent, "", + (ABTI_sched_has_unit(p_sched) ? "TRUE" : "FALSE"), indent, "", (void *)p_sched->p_ythread, indent, "", p_sched->data); if (print_sub == ABT_TRUE) { size_t i; diff --git a/src/self.c b/src/self.c index 96d267ca..0480e4d1 100644 --- a/src/self.c +++ b/src/self.c @@ -631,7 +631,8 @@ int ABT_self_yield(void) ABTI_ythread *p_ythread; ABTI_SETUP_LOCAL_YTHREAD(&p_local_xstream, &p_ythread); - ABTI_ythread_yield(&p_local_xstream, p_ythread, ABT_SYNC_EVENT_TYPE_USER, + ABTI_ythread_yield(&p_local_xstream, p_ythread, + ABTI_YTHREAD_YIELD_KIND_USER, ABT_SYNC_EVENT_TYPE_USER, NULL); return ABT_SUCCESS; } @@ -686,12 +687,10 @@ int ABT_self_yield_to(ABT_thread thread) ABT_ERR_INV_THREAD); ABTI_CHECK_TRUE(!(p_tar_ythread->thread.type & ABTI_THREAD_TYPE_MAIN_SCHED), ABT_ERR_INV_THREAD); - ABTI_UB_ASSERT(!(p_tar_ythread->thread.p_pool->u_is_in_pool && - p_tar_ythread->thread.p_pool->u_is_in_pool( - p_tar_ythread->thread.unit) == ABT_TRUE)); /* Switch the context */ ABTI_ythread_yield_to(&p_local_xstream, p_cur_ythread, p_tar_ythread, + ABTI_YTHREAD_YIELD_TO_KIND_USER, ABT_SYNC_EVENT_TYPE_USER, NULL); return ABT_SUCCESS; } @@ -747,6 +746,7 @@ int ABT_self_resume_yield_to(ABT_thread thread) /* Switch the context */ ABTI_ythread_resume_yield_to(&p_local_xstream, p_cur_ythread, p_tar_ythread, + ABTI_YTHREAD_RESUME_YIELD_TO_KIND_USER, ABT_SYNC_EVENT_TYPE_USER, NULL); return ABT_SUCCESS; } @@ -845,9 +845,6 @@ int ABT_self_suspend_to(ABT_thread thread) ABT_ERR_INV_THREAD); ABTI_CHECK_TRUE(!(p_tar_ythread->thread.type & ABTI_THREAD_TYPE_MAIN_SCHED), ABT_ERR_INV_THREAD); - ABTI_UB_ASSERT(!(p_tar_ythread->thread.p_pool->u_is_in_pool && - p_tar_ythread->thread.p_pool->u_is_in_pool( - p_tar_ythread->thread.unit) == ABT_TRUE)); /* Switch the context */ ABTI_ythread_suspend_to(&p_local_xstream, p_cur_ythread, p_tar_ythread, @@ -997,9 +994,6 @@ int ABT_self_exit_to(ABT_thread thread) ABTI_CHECK_TRUE(!(p_tar_ythread->thread.type & (ABTI_THREAD_TYPE_MAIN_SCHED | ABTI_THREAD_TYPE_PRIMARY)), ABT_ERR_INV_THREAD); - ABTI_UB_ASSERT(!(p_tar_ythread->thread.p_pool->u_is_in_pool && - p_tar_ythread->thread.p_pool->u_is_in_pool( - p_tar_ythread->thread.unit) == ABT_TRUE)); /* Switch the context */ ABTI_ythread_exit_to(p_local_xstream, p_cur_ythread, p_tar_ythread); diff --git a/src/task.c b/src/task.c index ed1d6728..78a300ea 100644 --- a/src/task.c +++ b/src/task.c @@ -611,7 +611,7 @@ ABTU_ret_err static int task_create(ABTI_global *p_global, ABTI_local *p_local, p_pool); /* Add this task to the scheduler's pool */ - ABTI_pool_push(p_pool, p_newtask->unit); + ABTI_pool_push(p_pool, p_newtask->unit, ABT_POOL_CONTEXT_OP_THREAD_CREATE); /* Return value */ *pp_newtask = p_newtask; diff --git a/src/thread.c b/src/thread.c index 85ef1fe6..b59113f6 100644 --- a/src/thread.c +++ b/src/thread.c @@ -209,6 +209,7 @@ int ABT_thread_create_to(ABT_pool pool, void (*thread_func)(void *), void *arg, /* Yield to the target ULT. */ ABTI_ythread_yield_to(&p_local_xstream, p_cur_ythread, p_newthread, + ABTI_YTHREAD_YIELD_TO_KIND_CREATE_TO, ABT_SYNC_EVENT_TYPE_USER, NULL); return ABT_SUCCESS; } @@ -563,6 +564,7 @@ int ABT_thread_revive_to(ABT_pool pool, void (*thread_func)(void *), void *arg, /* Yield to the target ULT. */ ABTI_ythread_yield_to(&p_local_xstream, p_self, p_target, + ABTI_YTHREAD_YIELD_TO_KIND_REVIVE_TO, ABT_SYNC_EVENT_TYPE_USER, NULL); return ABT_SUCCESS; } @@ -1299,15 +1301,17 @@ int ABT_thread_yield_to(ABT_thread thread) ABTI_CHECK_TRUE(p_cur_ythread != p_tar_ythread, ABT_ERR_INV_THREAD); ABTI_CHECK_TRUE(!(p_cur_ythread->thread.type & ABTI_THREAD_TYPE_MAIN_SCHED), ABT_ERR_INV_THREAD); - ABTI_CHECK_TRUE(p_tar_ythread->thread.p_pool->u_is_in_pool, ABT_ERR_POOL); - ABTI_CHECK_TRUE(p_tar_ythread->thread.p_pool->p_remove, ABT_ERR_POOL); + ABTI_CHECK_TRUE(p_tar_ythread->thread.p_pool->deprecated_def.u_is_in_pool, + ABT_ERR_POOL); + ABTI_CHECK_TRUE(p_tar_ythread->thread.p_pool->deprecated_def.p_remove, + ABT_ERR_POOL); /* If the target thread is not in READY, we don't yield. Note that ULT can * be regarded as 'ready' only if its state is READY and it has been * pushed into a pool. Since we set ULT's state to READY and then push it * into a pool, we check them in the reverse order, i.e., check if the ULT * is inside a pool and the its state. */ - if (!(p_tar_ythread->thread.p_pool->u_is_in_pool( + if (!(p_tar_ythread->thread.p_pool->deprecated_def.u_is_in_pool( p_tar_ythread->thread.unit) == ABT_TRUE && ABTD_atomic_acquire_load_int(&p_tar_ythread->thread.state) == ABT_THREAD_STATE_READY)) { @@ -1385,7 +1389,8 @@ int ABT_thread_yield(void) ABTI_SETUP_LOCAL_YTHREAD(&p_local_xstream, &p_ythread); #endif - ABTI_ythread_yield(&p_local_xstream, p_ythread, ABT_SYNC_EVENT_TYPE_USER, + ABTI_ythread_yield(&p_local_xstream, p_ythread, + ABTI_YTHREAD_YIELD_KIND_USER, ABT_SYNC_EVENT_TYPE_USER, NULL); return ABT_SUCCESS; } @@ -2928,7 +2933,8 @@ ythread_create(ABTI_global *p_global, ABTI_local *p_local, ABTI_pool *p_pool, p_pool); if (pool_op == THREAD_POOL_OP_PUSH) { /* Add this thread to the pool */ - ABTI_pool_push(p_pool, p_newthread->thread.unit); + ABTI_pool_push(p_pool, p_newthread->thread.unit, + ABT_POOL_CONTEXT_OP_THREAD_CREATE); } } else { /* pool_op == THREAD_POOL_OP_NONE */ @@ -2981,7 +2987,8 @@ thread_revive(ABTI_global *p_global, ABTI_local *p_local, ABTI_pool *p_pool, if (pool_op == THREAD_POOL_OP_PUSH) { /* Add this thread to the pool */ - ABTI_pool_push(p_pool, p_thread->unit); + ABTI_pool_push(p_pool, p_thread->unit, + ABT_POOL_CONTEXT_OP_THREAD_REVIVE); } return ABT_SUCCESS; } @@ -3107,6 +3114,7 @@ static void thread_join_yield_thread(ABTI_xstream **pp_local_xstream, while (ABTD_atomic_acquire_load_int(&p_thread->state) != ABT_THREAD_STATE_TERMINATED) { ABTI_ythread_yield(pp_local_xstream, p_self, + ABTI_YTHREAD_YIELD_KIND_YIELD_LOOP, ABT_SYNC_EVENT_TYPE_THREAD_JOIN, (void *)p_thread); } ABTI_event_thread_join(ABTI_xstream_get_local(*pp_local_xstream), p_thread, @@ -3195,7 +3203,8 @@ static void thread_root_func(void *arg) ABTI_pool *p_root_pool = p_local_xstream->p_root_pool; do { - ABT_unit unit = ABTI_pool_pop(p_root_pool); + ABT_unit unit = + ABTI_pool_pop(p_root_pool, ABT_POOL_CONTEXT_OWNER_PRIMARY); if (unit != ABT_UNIT_NULL) { ABTI_xstream *p_xstream = p_local_xstream; ABTI_thread *p_thread = @@ -3270,7 +3279,7 @@ static void thread_main_sched_func(void *arg) * execution of all work units. */ if ((ABTD_atomic_relaxed_load_uint32(&p_sched->request) & ABTI_SCHED_REQ_FINISH) && - ABTI_sched_get_effective_size(p_local, p_sched) == 0) { + !ABTI_sched_has_unit(p_sched)) { break; } } diff --git a/src/ythread.c b/src/ythread.c index cda47b7e..8beb5b7f 100644 --- a/src/ythread.c +++ b/src/ythread.c @@ -18,7 +18,8 @@ static void ythread_unwind_stack(void *arg); /* Private APIs */ /*****************************************************************************/ -void ABTI_ythread_callback_yield(void *arg) +static inline void ythread_callback_yield_impl(void *arg, + ABT_pool_context context) { ABTI_ythread *p_prev = (ABTI_ythread *)arg; if (ABTI_thread_handle_request(&p_prev->thread, ABT_TRUE) & @@ -26,10 +27,35 @@ void ABTI_ythread_callback_yield(void *arg) /* p_prev is terminated. */ } else { /* Push p_prev back to the pool. */ - ABTI_pool_add_thread(&p_prev->thread); + ABTI_pool_add_thread(&p_prev->thread, context); } } +void ABTI_ythread_callback_yield_user_yield(void *arg) +{ + ythread_callback_yield_impl(arg, ABT_POOL_CONTEXT_OP_THREAD_YIELD); +} + +void ABTI_ythread_callback_yield_loop(void *arg) +{ + ythread_callback_yield_impl(arg, ABT_POOL_CONTEXT_OP_THREAD_YIELD_LOOP); +} + +void ABTI_ythread_callback_yield_user_yield_to(void *arg) +{ + ythread_callback_yield_impl(arg, ABT_POOL_CONTEXT_OP_THREAD_YIELD_TO); +} + +void ABTI_ythread_callback_yield_create_to(void *arg) +{ + ythread_callback_yield_impl(arg, ABT_POOL_CONTEXT_OP_THREAD_CREATE_TO); +} + +void ABTI_ythread_callback_yield_revive_to(void *arg) +{ + ythread_callback_yield_impl(arg, ABT_POOL_CONTEXT_OP_THREAD_REVIVE_TO); +} + /* Before yield_to, p_prev->thread.p_pool's num_blocked must be incremented to * avoid making a pool empty. */ void ABTI_ythread_callback_thread_yield_to(void *arg) @@ -45,7 +71,8 @@ void ABTI_ythread_callback_thread_yield_to(void *arg) /* p_prev is terminated. */ } else { /* Push p_prev back to the pool. */ - ABTI_pool_add_thread(&p_prev->thread); + ABTI_pool_add_thread(&p_prev->thread, + ABT_POOL_CONTEXT_OP_THREAD_YIELD_TO); } /* Decrease the number of blocked threads of the original pool (i.e., before * migration), which has been increased by p_prev to avoid making a pool @@ -66,7 +93,8 @@ void ABTI_ythread_callback_resume_yield_to(void *arg) /* p_prev is terminated. */ } else { /* Push this thread back to the pool. */ - ABTI_pool_add_thread(&p_prev->thread); + ABTI_pool_add_thread(&p_prev->thread, + ABT_POOL_CONTEXT_OP_THREAD_RESUME_YIELD_TO); } /* Decrease the number of blocked threads of p_next's pool. */ ABTI_pool_dec_num_blocked(p_next->thread.p_pool); diff --git a/test/.gitignore b/test/.gitignore index 37e2d18e..52427721 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -44,6 +44,7 @@ basic/sched_config basic/sched_user_ws basic/pool_config basic/pool_custom +basic/pool_user_def basic/sync_no_contention basic/main_sched basic/mutex diff --git a/test/basic/Makefile.am b/test/basic/Makefile.am index cd905a96..fa6e81ec 100644 --- a/test/basic/Makefile.am +++ b/test/basic/Makefile.am @@ -49,6 +49,7 @@ TESTS = \ sched_user_ws \ pool_config \ pool_custom \ + pool_user_def \ sync_no_contention \ main_sched \ mutex \ @@ -154,6 +155,7 @@ sched_config_SOURCES = sched_config.c sched_user_ws_SOURCES = sched_user_ws.c pool_config_SOURCES = pool_config.c pool_custom_SOURCES = pool_custom.c +pool_user_def_SOURCES = pool_user_def.c sync_no_contention_SOURCES = sync_no_contention.c main_sched_SOURCES = main_sched.c mutex_SOURCES = mutex.c @@ -245,6 +247,7 @@ testing: ./sched_user_ws ./pool_config ./pool_custom + ./pool_user_def ./sync_no_contention ./main_sched ./mutex diff --git a/test/basic/error.c b/test/basic/error.c index 35a48749..40e4575b 100644 --- a/test/basic/error.c +++ b/test/basic/error.c @@ -38,6 +38,7 @@ int main(int argc, char *argv[]) { "ABT_ERR_INV_POOL_KIND", ABT_ERR_INV_POOL_KIND }, { "ABT_ERR_INV_POOL_ACCESS", ABT_ERR_INV_POOL_ACCESS }, { "ABT_ERR_INV_POOL_CONFIG", ABT_ERR_INV_POOL_CONFIG }, + { "ABT_ERR_INV_POOL_USER_DEF", ABT_ERR_INV_POOL_USER_DEF }, { "ABT_ERR_INV_UNIT", ABT_ERR_INV_UNIT }, { "ABT_ERR_INV_THREAD", ABT_ERR_INV_THREAD }, { "ABT_ERR_INV_THREAD_ATTR", ABT_ERR_INV_THREAD_ATTR }, diff --git a/test/basic/pool_custom.c b/test/basic/pool_custom.c index 40ad9e2b..723eabf1 100644 --- a/test/basic/pool_custom.c +++ b/test/basic/pool_custom.c @@ -12,12 +12,14 @@ #include "abttest.h" void create_sched_def(ABT_sched_def *p_def); -void create_pool1_def(ABT_pool_def *p_def); -void create_pool2_def(ABT_pool_def *p_def); +ABT_pool create_pool1(void); +ABT_pool create_pool2(void); +ABT_pool create_pool3(void); +ABT_pool create_pool4(void); -#define DEFAULT_NUM_XSTREAMS 2 -#define DEFAULT_NUM_THREADS 100 -#define NUM_POOLS 4 +#define DEFAULT_NUM_XSTREAMS 3 +#define DEFAULT_NUM_THREADS 200 +#define NUM_POOLS 6 void thread_func(void *arg) { @@ -49,17 +51,17 @@ ABT_pool create_pool(int pool_type) ABT_FALSE, &newpool); ATS_ERROR(ret, "ABT_pool_create_basic"); } else if (pool_type == 2) { - /* User-defined basic pool 1. */ - ABT_pool_def pool_def; - create_pool1_def(&pool_def); - int ret = ABT_pool_create(&pool_def, ABT_POOL_CONFIG_NULL, &newpool); - ATS_ERROR(ret, "ABT_pool_create"); + /* ABT_pool_def-based pool (pool 1). */ + newpool = create_pool1(); } else if (pool_type == 3) { - /* User-defined basic pool 2. */ - ABT_pool_def pool_def; - create_pool2_def(&pool_def); - int ret = ABT_pool_create(&pool_def, ABT_POOL_CONFIG_NULL, &newpool); - ATS_ERROR(ret, "ABT_pool_create"); + /* ABT_pool_def-based pool (pool 2). */ + newpool = create_pool2(); + } else if (pool_type == 4) { + /* ABTI_pool_user_def-based pool (pool 3). */ + newpool = create_pool3(); + } else if (pool_type == 5) { + /* ABTI_pool_user_def-based pool (poo; 4). */ + newpool = create_pool4(); } return newpool; } @@ -374,25 +376,31 @@ int pool1_free(ABT_pool pool) return ABT_SUCCESS; } -void create_pool1_def(ABT_pool_def *p_def) +ABT_pool create_pool1(void) { - p_def->access = ABT_POOL_ACCESS_MPMC; - p_def->u_create_from_thread = pool1_unit_create_from_thread; - p_def->u_free = pool1_unit_free; - p_def->p_init = pool1_init; - p_def->p_get_size = pool1_get_size; - p_def->p_push = pool1_push; - p_def->p_pop = pool1_pop; - p_def->p_free = pool1_free; + ABT_pool_def def; + def.access = ABT_POOL_ACCESS_MPMC; + def.u_create_from_thread = pool1_unit_create_from_thread; + def.u_free = pool1_unit_free; + def.p_init = pool1_init; + def.p_get_size = pool1_get_size; + def.p_push = pool1_push; + def.p_pop = pool1_pop; + def.p_free = pool1_free; /* Optional. */ - p_def->u_is_in_pool = NULL; + def.u_is_in_pool = NULL; #ifdef ABT_ENABLE_VER_20_API - p_def->p_pop_wait = NULL; + def.p_pop_wait = NULL; #endif - p_def->p_pop_timedwait = NULL; - p_def->p_remove = NULL; - p_def->p_print_all = NULL; + def.p_pop_timedwait = NULL; + def.p_remove = NULL; + def.p_print_all = NULL; + + ABT_pool newpool; + int ret = ABT_pool_create(&def, ABT_POOL_CONFIG_NULL, &newpool); + ATS_ERROR(ret, "ABT_pool_create"); + return newpool; } /******************************************************************************/ @@ -447,23 +455,189 @@ int pool2_free(ABT_pool pool) return ABT_SUCCESS; } -void create_pool2_def(ABT_pool_def *p_def) +ABT_pool create_pool2(void) { - p_def->access = ABT_POOL_ACCESS_MPMC; - p_def->u_create_from_thread = pool2_unit_create_from_thread; - p_def->u_free = pool2_unit_free; - p_def->p_init = pool2_init; - p_def->p_get_size = pool2_get_size; - p_def->p_push = pool2_push; - p_def->p_pop = pool2_pop; - p_def->p_free = pool2_free; + ABT_pool_def def; + def.access = ABT_POOL_ACCESS_MPMC; + def.u_create_from_thread = pool2_unit_create_from_thread; + def.u_free = pool2_unit_free; + def.p_init = pool2_init; + def.p_get_size = pool2_get_size; + def.p_push = pool2_push; + def.p_pop = pool2_pop; + def.p_free = pool2_free; /* Optional. */ - p_def->u_is_in_pool = NULL; + def.u_is_in_pool = NULL; #ifdef ABT_ENABLE_VER_20_API - p_def->p_pop_wait = NULL; + def.p_pop_wait = NULL; #endif - p_def->p_pop_timedwait = NULL; - p_def->p_remove = NULL; - p_def->p_print_all = NULL; + def.p_pop_timedwait = NULL; + def.p_remove = NULL; + def.p_print_all = NULL; + + ABT_pool newpool; + int ret = ABT_pool_create(&def, ABT_POOL_CONFIG_NULL, &newpool); + ATS_ERROR(ret, "ABT_pool_create"); + return newpool; +} + +/******************************************************************************/ +/* Pool 3 */ +/******************************************************************************/ + +ABT_pool g_pool3; +queue_t pool3_queue; + +ABT_unit pool3_create_unit(ABT_pool pool, ABT_thread thread) +{ + assert(g_pool3 == pool); + return (ABT_unit)create_unit(&pool3_queue, thread, 1); +} + +void pool3_free_unit(ABT_pool pool, ABT_unit unit) +{ + assert(g_pool3 == pool); + free_unit(&pool3_queue, (unit_t *)(unit)); +} + +int pool3_init(ABT_pool pool, ABT_pool_config config) +{ + g_pool3 = pool; + pool3_queue.list.p_prev = &pool3_queue.list; + pool3_queue.list.p_next = &pool3_queue.list; + pool3_queue.size = 0; + pool3_queue.num_units = 0; + pthread_mutex_init(&pool3_queue.lock, NULL); + return ABT_SUCCESS; +} + +ABT_bool pool3_is_empty(ABT_pool pool) +{ + assert(g_pool3 == pool); + return (pool3_queue.size == 0) ? ABT_TRUE : ABT_FALSE; +} + +void pool3_push(ABT_pool pool, ABT_unit unit, ABT_pool_context context) +{ + assert(g_pool3 == pool); + unit_t *p_unit = (unit_t *)unit; + assert(p_unit->pool_type == 1); + queue_push(&pool3_queue, p_unit); +} + +ABT_unit pool3_pop(ABT_pool pool, ABT_pool_context context) +{ + assert(g_pool3 == pool); + unit_t *p_unit = queue_pop(&pool3_queue); + return p_unit ? ((ABT_unit)p_unit) : ABT_UNIT_NULL; +} + +void pool3_free(ABT_pool pool) +{ + assert(g_pool3 == pool); + assert(pool3_queue.size == 0); + assert(pool3_queue.num_units == 0); + pthread_mutex_destroy(&pool3_queue.lock); +} + +ABT_pool create_pool3(void) +{ + /* Pool definition */ + int ret; + ABT_pool_user_def def; + ret = ABT_pool_user_def_create(pool3_create_unit, pool3_free_unit, + pool3_is_empty, pool3_pop, pool3_push, &def); + ATS_ERROR(ret, "ABT_pool_user_def_create"); + ret = ABT_pool_user_def_set_init(def, pool3_init); + ATS_ERROR(ret, "ABT_pool_user_def_set_init"); + ret = ABT_pool_user_def_set_free(def, pool3_free); + ATS_ERROR(ret, "ABT_pool_user_def_set_free"); + + ABT_pool newpool; + ret = ABT_pool_create(def, ABT_POOL_CONFIG_NULL, &newpool); + ATS_ERROR(ret, "ABT_pool_create"); + ret = ABT_pool_user_def_free(&def); + ATS_ERROR(ret, "ABT_pool_user_def_free"); + return newpool; +} + +/******************************************************************************/ +/* Pool 4 */ +/******************************************************************************/ + +ABT_pool g_pool4; +queue_t pool4_queue; + +ABT_unit pool4_create_unit(ABT_pool pool, ABT_thread thread) +{ + assert(g_pool4 == pool); + return (ABT_unit)create_unit(&pool4_queue, thread, 1); +} + +void pool4_free_unit(ABT_pool pool, ABT_unit unit) +{ + assert(g_pool4 == pool); + free_unit(&pool4_queue, (unit_t *)(unit)); +} + +int pool4_init(ABT_pool pool, ABT_pool_config config) +{ + g_pool4 = pool; + pool4_queue.list.p_prev = &pool4_queue.list; + pool4_queue.list.p_next = &pool4_queue.list; + pool4_queue.size = 0; + pool4_queue.num_units = 0; + pthread_mutex_init(&pool4_queue.lock, NULL); + return ABT_SUCCESS; +} + +ABT_bool pool4_is_empty(ABT_pool pool) +{ + assert(g_pool4 == pool); + return (pool4_queue.size == 0) ? ABT_TRUE : ABT_FALSE; +} + +void pool4_push(ABT_pool pool, ABT_unit unit, ABT_pool_context context) +{ + assert(g_pool4 == pool); + unit_t *p_unit = (unit_t *)unit; + assert(p_unit->pool_type == 1); + queue_push(&pool4_queue, p_unit); +} + +ABT_unit pool4_pop(ABT_pool pool, ABT_pool_context context) +{ + assert(g_pool4 == pool); + unit_t *p_unit = queue_pop(&pool4_queue); + return p_unit ? ((ABT_unit)p_unit) : ABT_UNIT_NULL; +} + +void pool4_free(ABT_pool pool) +{ + assert(g_pool4 == pool); + assert(pool4_queue.size == 0); + assert(pool4_queue.num_units == 0); + pthread_mutex_destroy(&pool4_queue.lock); +} + +ABT_pool create_pool4(void) +{ + /* Pool definition */ + int ret; + ABT_pool_user_def def; + ret = ABT_pool_user_def_create(pool4_create_unit, pool4_free_unit, + pool4_is_empty, pool4_pop, pool4_push, &def); + ATS_ERROR(ret, "ABT_pool_user_def_create"); + ret = ABT_pool_user_def_set_init(def, pool4_init); + ATS_ERROR(ret, "ABT_pool_user_def_set_init"); + ret = ABT_pool_user_def_set_free(def, pool4_free); + ATS_ERROR(ret, "ABT_pool_user_def_set_free"); + + ABT_pool newpool; + ret = ABT_pool_create(def, ABT_POOL_CONFIG_NULL, &newpool); + ATS_ERROR(ret, "ABT_pool_create"); + ret = ABT_pool_user_def_free(&def); + ATS_ERROR(ret, "ABT_pool_user_def_free"); + return newpool; } diff --git a/test/basic/pool_user_def.c b/test/basic/pool_user_def.c new file mode 100644 index 00000000..13d18dcd --- /dev/null +++ b/test/basic/pool_user_def.c @@ -0,0 +1,707 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * See COPYRIGHT in top-level directory. + */ + +/* Check if a user-defined pool is registered properly. */ + +#include +#include +#include "abt.h" +#include "abttest.h" + +typedef struct { + /* create_unit */ + int create_unit_counter; + ABT_thread create_unit_thread_in; + ABT_unit create_unit_out; + /* free_unit */ + int free_unit_counter; + ABT_unit free_unit_in; + /* is_empty */ + int is_empty_counter; + ABT_bool is_empty_out; + /* pop */ + int pop_counter; + ABT_pool_context pop_context_in; + ABT_unit pop_out; + /* push */ + int push_counter; + ABT_unit push_unit_in; + ABT_pool_context push_context_in; + /* init */ + int init_counter; + ABT_pool_config init_config_in; + /* free */ + int free_counter; + /* get_size */ + int get_size_counter; + size_t get_size_out; + /* pop_wait */ + int pop_wait_counter; + double pop_wait_time_secs_in; + ABT_pool_context pop_wait_context_in; + ABT_unit pop_wait_out; + /* pop_many */ + int pop_many_counter; + size_t pop_many_max_units_in; + ABT_pool_context pop_many_context_in; + ABT_unit pop_many_units_out[10]; + size_t pop_many_num_popped_out; + /* push_many */ + int push_many_counter; + ABT_unit push_many_units_in[10]; + size_t push_many_num_units_in; + ABT_pool_context push_many_context_in; + /* print_all */ + int print_all_counter; + void *print_all_arg_in; + void (*print_all_print_f_in)(void *, ABT_unit); +} expect_t; +expect_t g_expect; + +ABT_pool create_pool(ABT_pool_config config); + +void empty_func(void *arg) +{ + ; /* Empty */ +} + +void create_revive_to_func(void *arg) +{ + int ret; + ABT_pool pool; + ret = ABT_self_get_last_pool(&pool); + ATS_ERROR(ret, "ABT_self_get_last_pool"); + + ABT_unit self_unit; + ABT_thread self_thread; + ret = ABT_self_get_thread(&self_thread); + ATS_ERROR(ret, "ABT_self_get_thread"); + ret = ABT_thread_get_unit(self_thread, &self_unit); + ATS_ERROR(ret, "ABT_thread_get_unit"); + + ABT_thread thread; + int create_unit_counter = g_expect.create_unit_counter; + int push_counter = g_expect.push_counter; + g_expect.push_unit_in = self_unit; + g_expect.push_context_in = ABT_POOL_CONTEXT_OP_THREAD_CREATE_TO; + ret = ABT_thread_create_to(pool, empty_func, NULL, ABT_THREAD_ATTR_NULL, + &thread); + ATS_ERROR(ret, "ABT_thread_create_to"); + assert(g_expect.create_unit_counter == create_unit_counter + 1); + assert(g_expect.push_counter == push_counter + 1); + + ABT_unit unit; + ret = ABT_thread_get_unit(thread, &unit); + ATS_ERROR(ret, "ABT_thread_get_unit"); + assert(g_expect.create_unit_out == unit); + + ret = ABT_thread_join(thread); + ATS_ERROR(ret, "ABT_thread_join"); + + g_expect.push_unit_in = self_unit; + g_expect.push_context_in = ABT_POOL_CONTEXT_OP_THREAD_REVIVE_TO; + ret = ABT_thread_revive_to(pool, empty_func, NULL, &thread); + ATS_ERROR(ret, "ABT_thread_revive_to"); + assert(g_expect.push_counter == push_counter + 2); + + int free_unit_counter = g_expect.free_unit_counter; + g_expect.free_unit_in = unit; + ret = ABT_thread_free(&thread); + ATS_ERROR(ret, "ABT_thread_free"); + assert(g_expect.free_unit_counter == free_unit_counter + 1); +} + +void suspend_func(void *arg) +{ + int ret; + ret = ABT_self_suspend(); + ATS_ERROR(ret, "ABT_self_suspend"); + /* Resumed by ABT_self_resume() and scheduled by ABT_self_yield_to() */ + ret = ABT_self_suspend(); + ATS_ERROR(ret, "ABT_self_suspend"); + /* Resumed and scheduled by ABT_self_resume_yield_to() */ +} + +void yield_func(void *arg) +{ + int ret; + ABT_pool pool; + ret = ABT_self_get_last_pool(&pool); + ATS_ERROR(ret, "ABT_self_get_last_pool"); + + ABT_unit self_unit; + ABT_thread self_thread; + ret = ABT_self_get_thread(&self_thread); + ATS_ERROR(ret, "ABT_self_get_thread"); + ret = ABT_thread_get_unit(self_thread, &self_unit); + ATS_ERROR(ret, "ABT_thread_get_unit"); + + int push_counter = g_expect.push_counter; + g_expect.push_unit_in = self_unit; + g_expect.push_context_in = ABT_POOL_CONTEXT_OP_THREAD_YIELD; + ret = ABT_self_yield(); + ATS_ERROR(ret, "ABT_self_yield"); + assert(g_expect.push_counter == push_counter + 1); + + ABT_thread thread; + int create_unit_counter = g_expect.create_unit_counter; + g_expect.push_context_in = ABT_POOL_CONTEXT_OP_THREAD_CREATE; + ret = ABT_thread_create(pool, suspend_func, NULL, ABT_THREAD_ATTR_NULL, + &thread); + ATS_ERROR(ret, "ABT_thread_create"); + assert(g_expect.create_unit_counter == create_unit_counter + 1); + assert(g_expect.push_counter == push_counter + 2); + + ABT_unit unit; + ret = ABT_thread_get_unit(thread, &unit); + ATS_ERROR(ret, "ABT_thread_get_unit"); + assert(g_expect.create_unit_out == unit); + + ABT_unit popped_unit; + int pop_counter = g_expect.pop_counter; + ret = ABT_pool_pop(pool, &popped_unit); + ATS_ERROR(ret, "ABT_pool_pop"); + assert(g_expect.pop_counter == pop_counter + 1); + assert(g_expect.pop_out == popped_unit && popped_unit == unit); + + g_expect.push_unit_in = self_unit; + g_expect.push_context_in = ABT_POOL_CONTEXT_OP_THREAD_YIELD_TO; + ret = ABT_self_yield_to(thread); + ATS_ERROR(ret, "ABT_self_yield_to"); + assert(g_expect.push_counter == push_counter + 3); + + g_expect.push_unit_in = unit; + g_expect.push_context_in = ABT_POOL_CONTEXT_OP_THREAD_RESUME; + ret = ABT_thread_resume(thread); + ATS_ERROR(ret, "ABT_thread_resume"); + assert(g_expect.push_counter == push_counter + 4); + + pop_counter = g_expect.pop_counter; + ret = ABT_pool_pop(pool, &popped_unit); + ATS_ERROR(ret, "ABT_pool_pop"); + assert(g_expect.pop_counter == pop_counter + 1); + assert(g_expect.pop_out == popped_unit && popped_unit == unit); + + g_expect.push_unit_in = self_unit; + g_expect.push_context_in = ABT_POOL_CONTEXT_OP_THREAD_YIELD_TO; + ret = ABT_self_yield_to(thread); + ATS_ERROR(ret, "ABT_self_yield_to"); + assert(g_expect.push_counter == push_counter + 5); + + g_expect.push_unit_in = self_unit; + g_expect.push_context_in = ABT_POOL_CONTEXT_OP_THREAD_RESUME_YIELD_TO; + ret = ABT_self_resume_yield_to(thread); + ATS_ERROR(ret, "ABT_self_resume_yield_to"); + assert(g_expect.push_counter == push_counter + 6); + + int free_unit_counter = g_expect.free_unit_counter; + g_expect.free_unit_in = unit; + ret = ABT_thread_free(&thread); + ATS_ERROR(ret, "ABT_thread_free"); + assert(g_expect.free_unit_counter == free_unit_counter + 1); +} + +void print_all_func(void *arg, ABT_unit unit) +{ + assert(0); +} + +int main(int argc, char *argv[]) +{ + int ret; + + /* Initialize */ + ATS_read_args(argc, argv); + /* Initialize Argobots. */ + ATS_init(argc, argv, 1); + + ABT_pool pool; + + /* ABT_pool_create() */ + { + ABT_pool_config config; + ret = ABT_pool_config_create(&config); + ATS_ERROR(ret, "ABT_pool_config_create"); + g_expect.init_config_in = config; + assert(g_expect.init_counter == 0); + pool = create_pool(config); + assert(g_expect.init_counter == 1); + ret = ABT_pool_config_free(&config); + ATS_ERROR(ret, "ABT_pool_config_free"); + } + + /* ABT_pool_is_empty() */ + { + int is_empty_counter = g_expect.is_empty_counter; + ABT_bool is_empty; + ret = ABT_pool_is_empty(pool, &is_empty); + ATS_ERROR(ret, "ABT_pool_is_empty"); + assert(g_expect.is_empty_counter == is_empty_counter + 1); + assert(is_empty == ABT_TRUE && g_expect.is_empty_out == is_empty); + } + + /* ABT_pool_get_size() */ + { + int get_size_counter = g_expect.get_size_counter; + size_t size; + ret = ABT_pool_get_size(pool, &size); + ATS_ERROR(ret, "ABT_pool_get_size"); + assert(g_expect.get_size_counter == get_size_counter + 1); + assert(size == 0 && g_expect.get_size_out == size); + } + + /* ABT_pool_get_total_size() */ + { + int get_size_counter = g_expect.get_size_counter; + size_t size; + ret = ABT_pool_get_total_size(pool, &size); + ATS_ERROR(ret, "ABT_pool_get_total_size"); + assert(g_expect.get_size_counter == get_size_counter + 1); + assert(size == 0 && g_expect.get_size_out == size); + } + + /* ABT_pool_pop() */ + { + int pop_counter = g_expect.pop_counter; + ABT_unit unit; + ret = ABT_pool_pop(pool, &unit); + ATS_ERROR(ret, "ABT_pool_pop"); + assert(g_expect.pop_counter == pop_counter + 1); + assert(unit == ABT_UNIT_NULL && g_expect.pop_out == unit); + } + + /* ABT_pool_pop_wait() */ + { + int pop_wait_counter = g_expect.pop_wait_counter; + g_expect.pop_wait_time_secs_in = 1.0; + ABT_unit unit; + ret = ABT_pool_pop_wait(pool, &unit, 1.0); + ATS_ERROR(ret, "ABT_pool_pop_wait"); + assert(g_expect.pop_wait_counter == pop_wait_counter + 1); + assert(unit == ABT_UNIT_NULL && g_expect.pop_wait_out == unit); + } + + /* ABT_pool_print_all() */ + { + int print_all_counter = g_expect.print_all_counter; + g_expect.print_all_arg_in = (void *)&g_expect.print_all_arg_in; + g_expect.print_all_print_f_in = print_all_func; + ret = ABT_pool_print_all(pool, g_expect.print_all_arg_in, + g_expect.print_all_print_f_in); + ATS_ERROR(ret, "ABT_pool_print_all"); + assert(g_expect.print_all_counter == print_all_counter + 1); + } + + /* ABT_pool_push(), ABT_thread_create(), and ABT_thread_revive() */ + { + ABT_thread thread; + int create_unit_counter = g_expect.create_unit_counter; + int push_counter = g_expect.push_counter; + g_expect.push_context_in = ABT_POOL_CONTEXT_OP_THREAD_CREATE; + ret = ABT_thread_create(pool, empty_func, NULL, ABT_THREAD_ATTR_NULL, + &thread); + ATS_ERROR(ret, "ABT_thread_create"); + assert(g_expect.create_unit_counter == create_unit_counter + 1); + assert(g_expect.push_counter == push_counter + 1); + + ABT_unit unit; + ret = ABT_thread_get_unit(thread, &unit); + ATS_ERROR(ret, "ABT_thread_get_unit"); + assert(g_expect.create_unit_out == unit); + + ABT_unit popped_unit; + int pop_counter = g_expect.pop_counter; + ret = ABT_pool_pop(pool, &popped_unit); + ATS_ERROR(ret, "ABT_pool_pop"); + assert(g_expect.pop_counter == pop_counter + 1); + assert(unit == popped_unit && g_expect.pop_out == popped_unit); + + g_expect.push_unit_in = unit; + ret = ABT_pool_push(pool, unit); + ATS_ERROR(ret, "ABT_pool_push"); + assert(g_expect.push_counter == push_counter + 2); + + ret = ABT_pool_pop(pool, &popped_unit); + ATS_ERROR(ret, "ABT_pool_pop"); + assert(g_expect.pop_counter == pop_counter + 2); + assert(unit == popped_unit && g_expect.pop_out == popped_unit); + + ret = ABT_self_schedule(thread, pool); + ATS_ERROR(ret, "ABT_self_schedule"); + + ret = ABT_thread_join(thread); + ATS_ERROR(ret, "ABT_thread_join"); + + g_expect.push_unit_in = unit; + g_expect.push_context_in = ABT_POOL_CONTEXT_OP_THREAD_REVIVE; + ret = ABT_thread_revive(pool, empty_func, NULL, &thread); + ATS_ERROR(ret, "ABT_thread_revive"); + assert(g_expect.push_counter == push_counter + 3); + + ret = ABT_pool_pop(pool, &popped_unit); + ATS_ERROR(ret, "ABT_pool_pop"); + assert(g_expect.pop_counter == pop_counter + 3); + assert(unit == popped_unit && g_expect.pop_out == popped_unit); + + ret = ABT_self_schedule(thread, pool); + ATS_ERROR(ret, "ABT_self_schedule"); + + int free_unit_counter = g_expect.free_unit_counter; + g_expect.free_unit_in = unit; + ret = ABT_thread_free(&thread); + ATS_ERROR(ret, "ABT_thread_free"); + assert(g_expect.free_unit_counter == free_unit_counter + 1); + } + + /* ABT_thread_create_to() and ABT_thread_revive_to() */ + { + ABT_thread thread; + int create_unit_counter = g_expect.create_unit_counter; + int push_counter = g_expect.push_counter; + g_expect.push_context_in = ABT_POOL_CONTEXT_OP_THREAD_CREATE; + ret = ABT_thread_create(pool, create_revive_to_func, NULL, + ABT_THREAD_ATTR_NULL, &thread); + ATS_ERROR(ret, "ABT_thread_create"); + assert(g_expect.create_unit_counter == create_unit_counter + 1); + assert(g_expect.push_counter == push_counter + 1); + + while (1) { + ABT_unit unit; + int pop_counter = g_expect.pop_counter; + ret = ABT_pool_pop(pool, &unit); + ATS_ERROR(ret, "ABT_pool_pop"); + assert(g_expect.pop_counter == pop_counter + 1); + assert(g_expect.pop_out == unit); + + if (unit == ABT_UNIT_NULL) + break; + ret = ABT_self_schedule(thread, pool); + ATS_ERROR(ret, "ABT_self_schedule"); + } + + int free_unit_counter = g_expect.free_unit_counter; + ret = ABT_thread_free(&thread); + ATS_ERROR(ret, "ABT_thread_free"); + assert(g_expect.free_unit_counter == free_unit_counter + 1); + } + + /* ABT_self_yield(), ABT_self_yield_to(), ABT_thread_resume(), and + * ABT_self_resume_yield_to() */ + { + ABT_thread thread; + int create_unit_counter = g_expect.create_unit_counter; + int push_counter = g_expect.push_counter; + g_expect.push_context_in = ABT_POOL_CONTEXT_OP_THREAD_CREATE; + ret = ABT_thread_create(pool, yield_func, NULL, ABT_THREAD_ATTR_NULL, + &thread); + ATS_ERROR(ret, "ABT_thread_create"); + assert(g_expect.create_unit_counter == create_unit_counter + 1); + assert(g_expect.push_counter == push_counter + 1); + + while (1) { + ABT_unit unit; + int pop_counter = g_expect.pop_counter; + ret = ABT_pool_pop(pool, &unit); + ATS_ERROR(ret, "ABT_pool_pop"); + assert(g_expect.pop_counter == pop_counter + 1); + assert(g_expect.pop_out == unit); + + if (unit == ABT_UNIT_NULL) + break; + ret = ABT_self_schedule(thread, pool); + ATS_ERROR(ret, "ABT_self_schedule"); + } + + int free_unit_counter = g_expect.free_unit_counter; + ret = ABT_thread_free(&thread); + ATS_ERROR(ret, "ABT_thread_free"); + assert(g_expect.free_unit_counter == free_unit_counter + 1); + } + + /* ABT_pool_free() */ + { + int free_counter = g_expect.free_counter; + ABT_pool_free(&pool); + assert(g_expect.free_counter == free_counter + 1); + } + + /* Finalize Argobots. */ + ret = ATS_finalize(0); + return ret; +} + +/* Pool implementation. */ + +#define POOL_BUFFER_LEN 16 +typedef struct { + ABT_pool pool; + int ptr; + ABT_unit units[POOL_BUFFER_LEN]; +} pool_t; +pool_t g_pool; + +ABT_unit pool_create_unit(ABT_pool pool, ABT_thread thread) +{ + assert(g_pool.pool == pool); + g_expect.create_unit_counter++; + if (g_expect.create_unit_thread_in != NULL) { + assert(g_expect.create_unit_thread_in == thread); + g_expect.create_unit_thread_in = NULL; + } + g_expect.create_unit_out = (ABT_unit)thread; + return (ABT_unit)thread; +} + +void pool_free_unit(ABT_pool pool, ABT_unit unit) +{ + assert(g_pool.pool == pool); + g_expect.free_unit_counter++; + if (g_expect.free_unit_in != NULL) { + assert(g_expect.free_unit_in == unit); + g_expect.free_unit_in = NULL; + } +} + +ABT_bool pool_is_empty(ABT_pool pool) +{ + assert(g_pool.pool == pool); + g_expect.is_empty_counter++; + int i; + for (i = 0; i < POOL_BUFFER_LEN; i++) { + if (g_pool.units[i] != ABT_UNIT_NULL) { + g_expect.is_empty_out = ABT_FALSE; + return ABT_FALSE; + } + } + g_expect.is_empty_out = ABT_TRUE; + return ABT_TRUE; +} + +ABT_unit pool_pop(ABT_pool pool, ABT_pool_context context) +{ + assert(g_pool.pool == pool); + g_expect.pop_counter++; + if (g_expect.pop_context_in != 0) { + assert(g_expect.pop_context_in & context); + g_expect.pop_context_in = 0; + } + + int ptr = g_pool.ptr++, i; + for (i = 0; i < POOL_BUFFER_LEN; i++) { + int thread_i = (ptr + i) % POOL_BUFFER_LEN; + if (g_pool.units[thread_i] != ABT_UNIT_NULL) { + ABT_unit unit = g_pool.units[thread_i]; + g_pool.units[thread_i] = ABT_UNIT_NULL; + g_expect.pop_out = unit; + return unit; + } + } + g_expect.pop_out = ABT_UNIT_NULL; + return ABT_UNIT_NULL; +} + +void pool_push(ABT_pool pool, ABT_unit unit, ABT_pool_context context) +{ + assert(g_pool.pool == pool); + g_expect.push_counter++; + if (g_expect.push_unit_in != NULL) { + assert(g_expect.push_unit_in == unit); + g_expect.push_unit_in = NULL; + } + if (g_expect.push_context_in != 0) { + assert(g_expect.push_context_in & context); + g_expect.push_context_in = 0; + } + + int ptr = g_pool.ptr++, i; + for (i = 0; i < POOL_BUFFER_LEN; i++) { + int thread_i = (ptr + i) % POOL_BUFFER_LEN; + if (g_pool.units[thread_i] == ABT_UNIT_NULL) { + g_pool.units[thread_i] = unit; + return; + } + } + /* This pool gets full. */ + assert(0); +} + +int pool_init(ABT_pool pool, ABT_pool_config config) +{ + g_pool.pool = pool; + g_expect.init_counter++; + if (g_expect.init_config_in != NULL) { + assert(g_expect.init_config_in == config); + g_expect.init_config_in = NULL; + } + g_pool.ptr = 0; + int i; + for (i = 0; i < POOL_BUFFER_LEN; i++) { + g_pool.units[i] = ABT_UNIT_NULL; + } + return ABT_SUCCESS; +} + +void pool_free(ABT_pool pool) +{ + assert(g_pool.pool == pool); + g_expect.free_counter++; + int i; + for (i = 0; i < POOL_BUFFER_LEN; i++) { + assert(g_pool.units[i] == ABT_UNIT_NULL); + } + g_pool.pool = ABT_POOL_NULL; +} + +size_t pool_get_size(ABT_pool pool) +{ + assert(g_pool.pool == pool); + g_expect.get_size_counter++; + int i; + size_t size = 0; + for (i = 0; i < POOL_BUFFER_LEN; i++) { + if (g_pool.units[i] != ABT_UNIT_NULL) + size++; + } + g_expect.get_size_out = size; + return size; +} + +ABT_unit pool_pop_wait(ABT_pool pool, double time_secs, + ABT_pool_context context) +{ + assert(g_pool.pool == pool); + g_expect.pop_wait_counter++; + if (g_expect.pop_wait_context_in != 0) { + assert(g_expect.pop_wait_context_in & context); + g_expect.pop_wait_context_in = 0; + } + if (g_expect.pop_wait_time_secs_in != 0.0) { + assert(g_expect.pop_wait_time_secs_in == time_secs); + g_expect.pop_wait_time_secs_in = 0.0; + } + + int ptr = g_pool.ptr++, i; + for (i = 0; i < POOL_BUFFER_LEN; i++) { + int thread_i = (ptr + i) % POOL_BUFFER_LEN; + if (g_pool.units[thread_i] != ABT_UNIT_NULL) { + ABT_unit unit = g_pool.units[thread_i]; + g_pool.units[thread_i] = ABT_UNIT_NULL; + g_expect.pop_wait_out = unit; + return unit; + } + } + g_expect.pop_wait_out = ABT_UNIT_NULL; + return ABT_UNIT_NULL; +} + +void pool_pop_many(ABT_pool pool, ABT_unit *units, size_t max_units, + size_t *num_popped, ABT_pool_context context) +{ + assert(g_pool.pool == pool); + g_expect.pop_many_counter++; + if (g_expect.pop_many_context_in != 0) { + assert(g_expect.pop_many_context_in & context); + g_expect.pop_many_context_in = 0; + } + if (g_expect.pop_many_max_units_in != 0) { + assert(g_expect.pop_many_max_units_in == max_units); + g_expect.pop_many_max_units_in = 0; + } + + *num_popped = 0; + int ptr = g_pool.ptr++, i; + for (i = 0; i < POOL_BUFFER_LEN; i++) { + if (*num_popped >= max_units) + break; + int thread_i = (ptr + i) % POOL_BUFFER_LEN; + if (g_pool.units[thread_i] != ABT_UNIT_NULL) { + ABT_unit unit = g_pool.units[thread_i]; + g_pool.units[thread_i] = ABT_UNIT_NULL; + units[*num_popped] = unit; + g_expect.pop_many_units_out[*num_popped] = unit; + (*num_popped)++; + } + } + g_expect.pop_many_num_popped_out = *num_popped; +} + +void pool_push_many(ABT_pool pool, const ABT_unit *units, size_t num_units, + ABT_pool_context context) +{ + assert(g_pool.pool == pool); + g_expect.push_many_counter++; + if (g_expect.push_many_context_in != 0) { + assert(g_expect.push_many_context_in & context); + g_expect.push_many_context_in = 0; + } + if (g_expect.push_many_num_units_in != 0) { + assert(g_expect.push_many_num_units_in == num_units); + g_expect.push_many_num_units_in = 0; + } + size_t unit_i = 0; + for (unit_i = 0; unit_i < num_units; unit_i++) { + if (g_expect.push_many_units_in[unit_i] != NULL) + assert(g_expect.push_many_units_in[unit_i] == units[unit_i]); + } + + unit_i = 0; + int ptr = g_pool.ptr++, i; + for (i = 0; i < POOL_BUFFER_LEN; i++) { + if (unit_i == num_units) + break; + int thread_i = (ptr + i) % POOL_BUFFER_LEN; + if (g_pool.units[thread_i] == ABT_UNIT_NULL) { + g_pool.units[thread_i] = units[unit_i]; + g_expect.push_many_units_in[i] = units[i]; + unit_i++; + } + } + assert(unit_i == num_units); +} + +void pool_print_all(ABT_pool pool, void *arg, void (*print_f)(void *, ABT_unit)) +{ + assert(g_pool.pool == pool); + g_expect.print_all_counter++; + if (g_expect.print_all_arg_in != NULL) { + assert(g_expect.print_all_arg_in == arg); + g_expect.print_all_arg_in = NULL; + } + if (g_expect.print_all_print_f_in != NULL) { + assert(g_expect.print_all_print_f_in == print_f); + g_expect.print_all_print_f_in = NULL; + } +} + +ABT_pool create_pool(ABT_pool_config config) +{ + /* Pool definition */ + int ret; + ABT_pool_user_def def; + ret = ABT_pool_user_def_create(pool_create_unit, pool_free_unit, + pool_is_empty, pool_pop, pool_push, &def); + ATS_ERROR(ret, "ABT_pool_user_def_create"); + ret = ABT_pool_user_def_set_init(def, pool_init); + ATS_ERROR(ret, "ABT_pool_user_def_set_init"); + ret = ABT_pool_user_def_set_free(def, pool_free); + ATS_ERROR(ret, "ABT_pool_user_def_set_free"); + ret = ABT_pool_user_def_set_get_size(def, pool_get_size); + ATS_ERROR(ret, "ABT_pool_user_def_set_get_size"); + ret = ABT_pool_user_def_set_pop_wait(def, pool_pop_wait); + ATS_ERROR(ret, "ABT_pool_user_def_set_pop_wait"); + ret = ABT_pool_user_def_set_pop_many(def, pool_pop_many); + ATS_ERROR(ret, "ABT_pool_user_def_set_pop_many"); + ret = ABT_pool_user_def_set_push_many(def, pool_push_many); + ATS_ERROR(ret, "ABT_pool_user_def_set_push_many"); + ret = ABT_pool_user_def_set_print_all(def, pool_print_all); + ATS_ERROR(ret, "ABT_pool_user_def_set_print_all"); + + ABT_pool newpool; + ret = ABT_pool_create(def, config, &newpool); + ATS_ERROR(ret, "ABT_pool_create"); + ret = ABT_pool_user_def_free(&def); + ATS_ERROR(ret, "ABT_pool_user_def_free"); + return newpool; +} diff --git a/test/leakcheck/pool.c b/test/leakcheck/pool.c index 14307a41..56b835f0 100644 --- a/test/leakcheck/pool.c +++ b/test/leakcheck/pool.c @@ -11,7 +11,8 @@ /* Check ABT_pool. */ -#define POOL_KIND_USER ((ABT_pool_kind)999) +#define POOL_KIND_USER ((ABT_pool_kind)998) +#define POOL_KIND_USER2 ((ABT_pool_kind)999) ABT_unit unit_create_from_thread(ABT_thread thread) { @@ -23,6 +24,16 @@ void unit_free(ABT_unit *p_unit) (void)p_unit; } +ABT_unit pool_create_unit(ABT_pool pool, ABT_thread thread) +{ + return (ABT_unit)thread; +} + +void pool_free_unit(ABT_pool pool, ABT_unit unit) +{ + (void)unit; +} + typedef struct { int num_units; ABT_unit units[16]; @@ -49,7 +60,15 @@ size_t pool_get_size(ABT_pool pool) return pool_data->num_units; } -void pool_push(ABT_pool pool, ABT_unit unit) +ABT_bool pool_is_empty(ABT_pool pool) +{ + pool_data_t *pool_data; + int ret = ABT_pool_get_data(pool, (void **)&pool_data); + assert(ret == ABT_SUCCESS); + return pool_data->num_units == 0 ? ABT_TRUE : ABT_FALSE; +} + +void pool_push_old(ABT_pool pool, ABT_unit unit) { /* Very simple: no lock, fixed size. This implementation is for simplicity, * so don't use it in a real program unless you know what you are really @@ -60,7 +79,12 @@ void pool_push(ABT_pool pool, ABT_unit unit) pool_data->units[pool_data->num_units++] = unit; } -ABT_unit pool_pop(ABT_pool pool) +void pool_push(ABT_pool pool, ABT_unit unit, ABT_pool_context context) +{ + pool_push_old(pool, unit); +} + +ABT_unit pool_pop_old(ABT_pool pool) { pool_data_t *pool_data; int ret = ABT_pool_get_data(pool, (void **)&pool_data); @@ -70,7 +94,12 @@ ABT_unit pool_pop(ABT_pool pool) return pool_data->units[--pool_data->num_units]; } -int pool_free(ABT_pool pool) +ABT_unit pool_pop(ABT_pool pool, ABT_pool_context context) +{ + return pool_pop_old(pool); +} + +int pool_free_old(ABT_pool pool) { pool_data_t *pool_data; int ret = ABT_pool_get_data(pool, (void **)&pool_data); @@ -79,11 +108,84 @@ int pool_free(ABT_pool pool) return ABT_SUCCESS; } +void pool_free(ABT_pool pool) +{ + pool_data_t *pool_data; + int ret = ABT_pool_get_data(pool, (void **)&pool_data); + assert(ret == ABT_SUCCESS); + free(pool_data); +} + ABT_pool create_pool(int automatic, int must_succeed) { int ret; ABT_pool pool = (ABT_pool)RAND_PTR; + ABT_pool_user_def def = (ABT_pool_user_def)RAND_PTR; + ret = ABT_pool_user_def_create(pool_create_unit, pool_free_unit, + pool_is_empty, pool_pop, pool_push, &def); + assert(!must_succeed || ret == ABT_SUCCESS); + if (ret != ABT_SUCCESS) { + assert(def == (ABT_pool_user_def)RAND_PTR); + return ABT_POOL_NULL; + } + ret = ABT_pool_user_def_set_init(def, pool_init); + assert(ret == ABT_SUCCESS); + ret = ABT_pool_user_def_set_free(def, pool_free); + assert(ret == ABT_SUCCESS); + + ABT_pool_config config = (ABT_pool_config)RAND_PTR; + if (automatic) { + /* Create a configuration. */ + ret = ABT_pool_config_create(&config); + assert(!must_succeed || ret == ABT_SUCCESS); + if (ret != ABT_SUCCESS) { + assert(config == (ABT_pool_config)RAND_PTR); + ret = ABT_pool_user_def_free(&def); + assert(ret == ABT_SUCCESS && def == ABT_POOL_USER_DEF_NULL); + return ABT_POOL_NULL; + } + const int automatic_val = 1; + ret = ABT_pool_config_set(config, ABT_pool_config_automatic.key, + ABT_pool_config_automatic.type, + (const void *)&automatic_val); + assert(!must_succeed || ret == ABT_SUCCESS); + if (ret != ABT_SUCCESS) { + ret = ABT_pool_config_free(&config); + assert(ret == ABT_SUCCESS && config == ABT_POOL_CONFIG_NULL); + ret = ABT_pool_user_def_free(&def); + assert(ret == ABT_SUCCESS && def == ABT_POOL_USER_DEF_NULL); + return ABT_POOL_NULL; + } + } else { + /* By default, a pool created by ABT_pool_create() is not automatically + * freed. */ + config = ABT_POOL_CONFIG_NULL; + } + ret = ABT_pool_create(def, config, &pool); + assert(!must_succeed || ret == ABT_SUCCESS); + if (ret != ABT_SUCCESS) { +#ifdef ABT_ENABLE_VER_20_API + assert(pool == (ABT_pool)RAND_PTR); +#else + assert(pool == ABT_POOL_NULL); +#endif + pool = ABT_POOL_NULL; + } + if (config != ABT_POOL_CONFIG_NULL) { + ret = ABT_pool_config_free(&config); + assert(ret == ABT_SUCCESS && config == ABT_POOL_CONFIG_NULL); + } + ret = ABT_pool_user_def_free(&def); + assert(ret == ABT_SUCCESS && def == ABT_POOL_USER_DEF_NULL); + return pool; +} + +ABT_pool create_pool_old(int automatic, int must_succeed) +{ + int ret; + ABT_pool pool = (ABT_pool)RAND_PTR; + ABT_pool_def pool_def; pool_def.access = ABT_POOL_ACCESS_MPMC; pool_def.u_get_type = NULL; @@ -95,14 +197,14 @@ ABT_pool create_pool(int automatic, int must_succeed) pool_def.u_free = unit_free; pool_def.p_init = pool_init; pool_def.p_get_size = pool_get_size; - pool_def.p_push = pool_push; - pool_def.p_pop = pool_pop; + pool_def.p_push = pool_push_old; + pool_def.p_pop = pool_pop_old; #ifdef ABT_ENABLE_VER_20_API pool_def.p_pop_wait = NULL; #endif pool_def.p_pop_timedwait = NULL; pool_def.p_remove = NULL; - pool_def.p_free = pool_free; + pool_def.p_free = pool_free_old; pool_def.p_print_all = NULL; ABT_pool_config config = (ABT_pool_config)RAND_PTR; @@ -176,6 +278,8 @@ void program(ABT_pool_kind kind, int automatic, int type, int must_succeed) ABT_pool pool; if (kind == POOL_KIND_USER) { pool = create_pool(automatic, must_succeed); + } else if (kind == POOL_KIND_USER2) { + pool = create_pool_old(automatic, must_succeed); } else { pool = create_pool_basic(kind, automatic, must_succeed); } @@ -269,7 +373,7 @@ int main() rtrace_init(); int i, automatic, type; - ABT_pool_kind kinds[] = { ABT_POOL_FIFO, POOL_KIND_USER }; + ABT_pool_kind kinds[] = { ABT_POOL_FIFO, POOL_KIND_USER, POOL_KIND_USER2 }; /* Checking all takes too much time. */ for (i = 0; i < (int)(sizeof(kinds) / sizeof(kinds[0])); i++) { for (automatic = 0; automatic <= 1; automatic++) { diff --git a/test/leakcheck/unit.c b/test/leakcheck/unit.c index 66af4685..2890dd74 100644 --- a/test/leakcheck/unit.c +++ b/test/leakcheck/unit.c @@ -32,6 +32,16 @@ void unit_free(ABT_unit *p_unit) free(*p_unit); } +ABT_unit pool_create_unit(ABT_pool pool, ABT_thread thread) +{ + return unit_create_from_thread(thread); +} + +void pool_free_unit(ABT_pool pool, ABT_unit unit) +{ + unit_free(&unit); +} + typedef struct { int num_units; ABT_unit units[16]; @@ -56,7 +66,15 @@ size_t pool_get_size(ABT_pool pool) return pool_data->num_units; } -void pool_push(ABT_pool pool, ABT_unit unit) +ABT_bool pool_is_empty(ABT_pool pool) +{ + pool_data_t *pool_data; + int ret = ABT_pool_get_data(pool, (void **)&pool_data); + assert(ret == ABT_SUCCESS); + return pool_data->num_units == 0 ? ABT_TRUE : ABT_FALSE; +} + +void pool_push_old(ABT_pool pool, ABT_unit unit) { /* Very simple: no lock, fixed size. This implementation is for simplicity, * so don't use it in a real program unless you know what you are really @@ -67,7 +85,12 @@ void pool_push(ABT_pool pool, ABT_unit unit) pool_data->units[pool_data->num_units++] = unit; } -ABT_unit pool_pop(ABT_pool pool) +void pool_push(ABT_pool pool, ABT_unit unit, ABT_pool_context context) +{ + pool_push_old(pool, unit); +} + +ABT_unit pool_pop_old(ABT_pool pool) { pool_data_t *pool_data; int ret = ABT_pool_get_data(pool, (void **)&pool_data); @@ -77,7 +100,12 @@ ABT_unit pool_pop(ABT_pool pool) return pool_data->units[--pool_data->num_units]; } -int pool_free(ABT_pool pool) +ABT_unit pool_pop(ABT_pool pool, ABT_pool_context context) +{ + return pool_pop_old(pool); +} + +int pool_free_old(ABT_pool pool) { pool_data_t *pool_data; int ret = ABT_pool_get_data(pool, (void **)&pool_data); @@ -86,11 +114,40 @@ int pool_free(ABT_pool pool) return ABT_SUCCESS; } +void pool_free(ABT_pool pool) +{ + pool_data_t *pool_data; + int ret = ABT_pool_get_data(pool, (void **)&pool_data); + assert(ret == ABT_SUCCESS); + free(pool_data); +} + ABT_pool create_pool(void) { int ret; ABT_pool pool; + ABT_pool_user_def def = (ABT_pool_user_def)RAND_PTR; + ret = ABT_pool_user_def_create(pool_create_unit, pool_free_unit, + pool_is_empty, pool_pop, pool_push, &def); + assert(ret == ABT_SUCCESS); + ret = ABT_pool_user_def_set_init(def, pool_init); + assert(ret == ABT_SUCCESS); + ret = ABT_pool_user_def_set_free(def, pool_free); + assert(ret == ABT_SUCCESS); + + ret = ABT_pool_create(def, ABT_POOL_CONFIG_NULL, &pool); + assert(ret == ABT_SUCCESS); + ret = ABT_pool_user_def_free(&def); + assert(ret == ABT_SUCCESS && def == ABT_POOL_USER_DEF_NULL); + return pool; +} + +ABT_pool create_pool_old(void) +{ + int ret; + ABT_pool pool; + ABT_pool_def pool_def; pool_def.access = ABT_POOL_ACCESS_MPMC; pool_def.u_get_type = NULL; @@ -102,14 +159,14 @@ ABT_pool create_pool(void) pool_def.u_free = unit_free; pool_def.p_init = pool_init; pool_def.p_get_size = pool_get_size; - pool_def.p_push = pool_push; - pool_def.p_pop = pool_pop; + pool_def.p_push = pool_push_old; + pool_def.p_pop = pool_pop_old; #ifdef ABT_ENABLE_VER_20_API pool_def.p_pop_wait = NULL; #endif pool_def.p_pop_timedwait = NULL; pool_def.p_remove = NULL; - pool_def.p_free = pool_free; + pool_def.p_free = pool_free_old; pool_def.p_print_all = NULL; ret = ABT_pool_create(&pool_def, ABT_POOL_CONFIG_NULL, &pool); @@ -144,10 +201,29 @@ void program(int use_predef, int must_succeed) assert(ret == ABT_SUCCESS); /* Pool creation should be covered by other tests. */ ABT_pool pools[2]; - pools[0] = create_pool(); - if (!use_predef) { + if (use_predef == 0) { + pools[0] = create_pool(); pools[1] = create_pool(); + } else if (use_predef == 1) { + pools[0] = create_pool(); + pools[1] = create_pool_old(); + } else if (use_predef == 2) { + pools[0] = create_pool_old(); + pools[1] = create_pool_old(); + } else if (use_predef == 3) { + pools[0] = create_pool(); + ret = ABT_pool_create_basic(ABT_POOL_FIFO, ABT_POOL_ACCESS_MPMC, + ABT_FALSE, &pools[1]); + assert(ret == ABT_SUCCESS); + } else if (use_predef == 4) { + pools[0] = create_pool_old(); + ret = ABT_pool_create_basic(ABT_POOL_FIFO, ABT_POOL_ACCESS_MPMC, + ABT_FALSE, &pools[1]); + assert(ret == ABT_SUCCESS); } else { + ret = ABT_pool_create_basic(ABT_POOL_FIFO, ABT_POOL_ACCESS_MPMC, + ABT_FALSE, &pools[0]); + assert(ret == ABT_SUCCESS); ret = ABT_pool_create_basic(ABT_POOL_FIFO, ABT_POOL_ACCESS_MPMC, ABT_FALSE, &pools[1]); assert(ret == ABT_SUCCESS); @@ -216,7 +292,7 @@ int main() rtrace_init(); int use_predef; - for (use_predef = 0; use_predef <= 1; use_predef++) { + for (use_predef = 0; use_predef <= 5; use_predef++) { do { rtrace_start(); program(use_predef, 0);