diff options
Diffstat (limited to 'main/astobj2.c')
| -rw-r--r-- | main/astobj2.c | 781 |
1 files changed, 552 insertions, 229 deletions
diff --git a/main/astobj2.c b/main/astobj2.c index 62e76dbf6..d0643fe5b 100644 --- a/main/astobj2.c +++ b/main/astobj2.c @@ -34,20 +34,19 @@ ASTERISK_FILE_VERSION(__FILE__, "$Revision$") /*! * astobj2 objects are always preceded by this data structure, - * which contains a lock, a reference counter, - * the flags and a pointer to a destructor. + * which contains a reference counter, + * option flags and a pointer to a destructor. * The refcount is used to decide when it is time to * invoke the destructor. * The magic number is used for consistency check. - * XXX the lock is not always needed, and its initialization may be - * expensive. Consider making it external. */ struct __priv_data { - ast_mutex_t lock; int ref_counter; ao2_destructor_fn destructor_fn; - /*! for stats */ + /*! User data size for stats */ size_t data_size; + /*! The ao2 object option flags */ + uint32_t options; /*! magic number. This is used to verify that a pointer passed in is a * valid astobj2 */ uint32_t magic; @@ -64,9 +63,29 @@ struct astobj2 { void *user_data[0]; }; -#ifdef AST_DEVMODE -/* #define AO2_DEBUG 1 */ -#endif +struct ao2_lock_priv { + ast_mutex_t lock; +}; + +/* AstObj2 with recursive lock. */ +struct astobj2_lock { + struct ao2_lock_priv mutex; + struct __priv_data priv_data; + void *user_data[0]; +}; + +struct ao2_rwlock_priv { + ast_rwlock_t lock; + /*! Count of the number of threads holding a lock on this object. -1 if it is the write lock. */ + int num_lockers; +}; + +/* AstObj2 with RW lock. */ +struct astobj2_rwlock { + struct ao2_rwlock_priv rwlock; + struct __priv_data priv_data; + void *user_data[0]; +}; #ifdef AO2_DEBUG struct ao2_stats { @@ -102,6 +121,12 @@ void ao2_bt(void) } #endif +#define INTERNAL_OBJ_MUTEX(user_data) \ + ((struct astobj2_lock *) (((char *) (user_data)) - sizeof(struct astobj2_lock))) + +#define INTERNAL_OBJ_RWLOCK(user_data) \ + ((struct astobj2_rwlock *) (((char *) (user_data)) - sizeof(struct astobj2_rwlock))) + /*! * \brief convert from a pointer _p to a user-defined object * @@ -119,7 +144,7 @@ static inline struct astobj2 *INTERNAL_OBJ(void *user_data) p = (struct astobj2 *) ((char *) user_data - sizeof(*p)); if (AO2_MAGIC != (p->priv_data.magic) ) { ast_log(LOG_ERROR, "bad magic number 0x%x for %p\n", p->priv_data.magic, p); - p = NULL; + return NULL; } return p; @@ -137,123 +162,269 @@ enum ao2_callback_type { */ #define EXTERNAL_OBJ(_p) ((_p) == NULL ? NULL : (_p)->user_data) -/* the underlying functions common to debug and non-debug versions */ - -static int internal_ao2_ref(void *user_data, const int delta); -static struct ao2_container *internal_ao2_container_alloc(struct ao2_container *c, const uint n_buckets, ao2_hash_fn *hash_fn, - ao2_callback_fn *cmp_fn); -static struct bucket_entry *internal_ao2_link(struct ao2_container *c, void *user_data, int flags, const char *file, int line, const char *func); -static void *internal_ao2_callback(struct ao2_container *c, - const enum search_flags flags, void *cb_fn, void *arg, void *data, enum ao2_callback_type type, - const char *tag, char *file, int line, const char *funcname); -static void *internal_ao2_iterator_next(struct ao2_iterator *a, struct bucket_entry **q); - -int __ao2_lock(void *user_data, const char *file, const char *func, int line, const char *var) +int __ao2_lock(void *user_data, enum ao2_lock_req lock_how, const char *file, const char *func, int line, const char *var) { - struct astobj2 *p = INTERNAL_OBJ(user_data); + struct astobj2 *obj = INTERNAL_OBJ(user_data); + struct astobj2_lock *obj_mutex; + struct astobj2_rwlock *obj_rwlock; + int res = 0; - if (p == NULL) + if (obj == NULL) { return -1; + } + switch (obj->priv_data.options & AO2_ALLOC_OPT_LOCK_MASK) { + case AO2_ALLOC_OPT_LOCK_MUTEX: + obj_mutex = INTERNAL_OBJ_MUTEX(user_data); + res = __ast_pthread_mutex_lock(file, line, func, var, &obj_mutex->mutex.lock); +#ifdef AO2_DEBUG + if (!res) { + ast_atomic_fetchadd_int(&ao2.total_locked, 1); + } +#endif + break; + case AO2_ALLOC_OPT_LOCK_RWLOCK: + obj_rwlock = INTERNAL_OBJ_RWLOCK(user_data); + switch (lock_how) { + case AO2_LOCK_REQ_MUTEX: + case AO2_LOCK_REQ_WRLOCK: + res = __ast_rwlock_wrlock(file, line, func, &obj_rwlock->rwlock.lock, var); + if (!res) { + ast_atomic_fetchadd_int(&obj_rwlock->rwlock.num_lockers, -1); +#ifdef AO2_DEBUG + ast_atomic_fetchadd_int(&ao2.total_locked, 1); +#endif + } + break; + case AO2_LOCK_REQ_RDLOCK: + res = __ast_rwlock_rdlock(file, line, func, &obj_rwlock->rwlock.lock, var); + if (!res) { + ast_atomic_fetchadd_int(&obj_rwlock->rwlock.num_lockers, +1); #ifdef AO2_DEBUG - ast_atomic_fetchadd_int(&ao2.total_locked, 1); + ast_atomic_fetchadd_int(&ao2.total_locked, 1); #endif + } + break; + } + break; + case AO2_ALLOC_OPT_LOCK_NOLOCK: + /* The ao2 object has no lock. */ + break; + default: + ast_log(__LOG_ERROR, file, line, func, "Invalid lock option on ao2 object %p\n", + user_data); + return -1; + } - return __ast_pthread_mutex_lock(file, line, func, var, &p->priv_data.lock); + return res; } int __ao2_unlock(void *user_data, const char *file, const char *func, int line, const char *var) { - struct astobj2 *p = INTERNAL_OBJ(user_data); + struct astobj2 *obj = INTERNAL_OBJ(user_data); + struct astobj2_lock *obj_mutex; + struct astobj2_rwlock *obj_rwlock; + int res = 0; + int current_value; - if (p == NULL) + if (obj == NULL) { return -1; + } + switch (obj->priv_data.options & AO2_ALLOC_OPT_LOCK_MASK) { + case AO2_ALLOC_OPT_LOCK_MUTEX: + obj_mutex = INTERNAL_OBJ_MUTEX(user_data); + res = __ast_pthread_mutex_unlock(file, line, func, var, &obj_mutex->mutex.lock); #ifdef AO2_DEBUG - ast_atomic_fetchadd_int(&ao2.total_locked, -1); + if (!res) { + ast_atomic_fetchadd_int(&ao2.total_locked, -1); + } #endif - - return __ast_pthread_mutex_unlock(file, line, func, var, &p->priv_data.lock); + break; + case AO2_ALLOC_OPT_LOCK_RWLOCK: + obj_rwlock = INTERNAL_OBJ_RWLOCK(user_data); + + current_value = ast_atomic_fetchadd_int(&obj_rwlock->rwlock.num_lockers, -1) - 1; + if (current_value < 0) { + /* It was a WRLOCK that we are unlocking. Fix the count. */ + ast_atomic_fetchadd_int(&obj_rwlock->rwlock.num_lockers, -current_value); + } + res = __ast_rwlock_unlock(file, line, func, &obj_rwlock->rwlock.lock, var); +#ifdef AO2_DEBUG + if (!res) { + ast_atomic_fetchadd_int(&ao2.total_locked, -1); + } +#endif + break; + case AO2_ALLOC_OPT_LOCK_NOLOCK: + /* The ao2 object has no lock. */ + break; + default: + ast_log(__LOG_ERROR, file, line, func, "Invalid lock option on ao2 object %p\n", + user_data); + res = -1; + break; + } + return res; } -int __ao2_trylock(void *user_data, const char *file, const char *func, int line, const char *var) +int __ao2_trylock(void *user_data, enum ao2_lock_req lock_how, const char *file, const char *func, int line, const char *var) { - struct astobj2 *p = INTERNAL_OBJ(user_data); - int ret; + struct astobj2 *obj = INTERNAL_OBJ(user_data); + struct astobj2_lock *obj_mutex; + struct astobj2_rwlock *obj_rwlock; + int res = 0; - if (p == NULL) + if (obj == NULL) { return -1; - ret = __ast_pthread_mutex_trylock(file, line, func, var, &p->priv_data.lock); + } + switch (obj->priv_data.options & AO2_ALLOC_OPT_LOCK_MASK) { + case AO2_ALLOC_OPT_LOCK_MUTEX: + obj_mutex = INTERNAL_OBJ_MUTEX(user_data); + res = __ast_pthread_mutex_trylock(file, line, func, var, &obj_mutex->mutex.lock); #ifdef AO2_DEBUG - if (!ret) - ast_atomic_fetchadd_int(&ao2.total_locked, 1); + if (!res) { + ast_atomic_fetchadd_int(&ao2.total_locked, 1); + } #endif - return ret; -} - -void *ao2_object_get_lockaddr(void *obj) -{ - struct astobj2 *p = INTERNAL_OBJ(obj); + break; + case AO2_ALLOC_OPT_LOCK_RWLOCK: + obj_rwlock = INTERNAL_OBJ_RWLOCK(user_data); + switch (lock_how) { + case AO2_LOCK_REQ_MUTEX: + case AO2_LOCK_REQ_WRLOCK: + res = __ast_rwlock_trywrlock(file, line, func, &obj_rwlock->rwlock.lock, var); + if (!res) { + ast_atomic_fetchadd_int(&obj_rwlock->rwlock.num_lockers, -1); +#ifdef AO2_DEBUG + ast_atomic_fetchadd_int(&ao2.total_locked, 1); +#endif + } + break; + case AO2_LOCK_REQ_RDLOCK: + res = __ast_rwlock_tryrdlock(file, line, func, &obj_rwlock->rwlock.lock, var); + if (!res) { + ast_atomic_fetchadd_int(&obj_rwlock->rwlock.num_lockers, +1); +#ifdef AO2_DEBUG + ast_atomic_fetchadd_int(&ao2.total_locked, 1); +#endif + } + break; + } + break; + case AO2_ALLOC_OPT_LOCK_NOLOCK: + /* The ao2 object has no lock. */ + return 0; + default: + ast_log(__LOG_ERROR, file, line, func, "Invalid lock option on ao2 object %p\n", + user_data); + return -1; + } - if (p == NULL) - return NULL; - return &p->priv_data.lock; + return res; } -/* - * The argument is a pointer to the user portion. +/*! + * \internal + * \brief Adjust an object's lock to the requested level. + * + * \param user_data An ao2 object to adjust lock level. + * \param lock_how What level to adjust lock. + * \param keep_stronger TRUE if keep original lock level if it is stronger. + * + * \pre The ao2 object is already locked. + * + * \details + * An ao2 object with a RWLOCK will have its lock level adjusted + * to the specified level if it is not already there. An ao2 + * object with a different type of lock is not affected. + * + * \return Original lock level. */ - - -int __ao2_ref_debug(void *user_data, const int delta, const char *tag, char *file, int line, const char *funcname) +static enum ao2_lock_req adjust_lock(void *user_data, enum ao2_lock_req lock_how, int keep_stronger) { struct astobj2 *obj = INTERNAL_OBJ(user_data); - - if (obj == NULL) - return -1; - - if (delta != 0) { - FILE *refo = fopen(REF_FILE, "a"); - if (refo) { - fprintf(refo, "%p %s%d %s:%d:%s (%s) [@%d]\n", user_data, (delta < 0 ? "" : "+"), - delta, file, line, funcname, tag, obj ? obj->priv_data.ref_counter : -1); - fclose(refo); + struct astobj2_rwlock *obj_rwlock; + enum ao2_lock_req orig_lock; + + switch (obj->priv_data.options & AO2_ALLOC_OPT_LOCK_MASK) { + case AO2_ALLOC_OPT_LOCK_RWLOCK: + obj_rwlock = INTERNAL_OBJ_RWLOCK(user_data); + if (obj_rwlock->rwlock.num_lockers < 0) { + orig_lock = AO2_LOCK_REQ_WRLOCK; + } else { + orig_lock = AO2_LOCK_REQ_RDLOCK; } - } - if (obj->priv_data.ref_counter + delta == 0 && obj->priv_data.destructor_fn != NULL) { /* this isn't protected with lock; just for o/p */ - FILE *refo = fopen(REF_FILE, "a"); - if (refo) { - fprintf(refo, "%p **call destructor** %s:%d:%s (%s)\n", user_data, file, line, funcname, tag); - fclose(refo); + switch (lock_how) { + case AO2_LOCK_REQ_MUTEX: + lock_how = AO2_LOCK_REQ_WRLOCK; + /* Fall through */ + case AO2_LOCK_REQ_WRLOCK: + if (lock_how != orig_lock) { + /* Switch from read lock to write lock. */ + ao2_unlock(user_data); + ao2_wrlock(user_data); + } + break; + case AO2_LOCK_REQ_RDLOCK: + if (!keep_stronger && lock_how != orig_lock) { + /* Switch from write lock to read lock. */ + ao2_unlock(user_data); + ao2_rdlock(user_data); + } + break; } + break; + default: + ast_log(LOG_ERROR, "Invalid lock option on ao2 object %p\n", user_data); + /* Fall through */ + case AO2_ALLOC_OPT_LOCK_NOLOCK: + case AO2_ALLOC_OPT_LOCK_MUTEX: + orig_lock = AO2_LOCK_REQ_MUTEX; + break; } - return internal_ao2_ref(user_data, delta); + + return orig_lock; } -int __ao2_ref(void *user_data, const int delta) +void *ao2_object_get_lockaddr(void *user_data) { struct astobj2 *obj = INTERNAL_OBJ(user_data); + struct astobj2_lock *obj_mutex; - if (obj == NULL) - return -1; + if (obj == NULL) { + return NULL; + } + + switch (obj->priv_data.options & AO2_ALLOC_OPT_LOCK_MASK) { + case AO2_ALLOC_OPT_LOCK_MUTEX: + obj_mutex = INTERNAL_OBJ_MUTEX(user_data); + return &obj_mutex->mutex.lock; + default: + break; + } - return internal_ao2_ref(user_data, delta); + return NULL; } -static int internal_ao2_ref(void *user_data, const int delta) +static int internal_ao2_ref(void *user_data, int delta, const char *file, int line, const char *funcname) { struct astobj2 *obj = INTERNAL_OBJ(user_data); + struct astobj2_lock *obj_mutex; + struct astobj2_rwlock *obj_rwlock; int current_value; int ret; - if (obj == NULL) + if (obj == NULL) { return -1; + } /* if delta is 0, just return the refcount */ - if (delta == 0) - return (obj->priv_data.ref_counter); + if (delta == 0) { + return obj->priv_data.ref_counter; + } /* we modify with an atomic operation the reference counter */ ret = ast_atomic_fetchadd_int(&obj->priv_data.ref_counter, delta); @@ -263,56 +434,169 @@ static int internal_ao2_ref(void *user_data, const int delta) ast_atomic_fetchadd_int(&ao2.total_refs, delta); #endif + if (0 < current_value) { + /* The object still lives. */ + return ret; + } + /* this case must never happen */ - if (current_value < 0) - ast_log(LOG_ERROR, "refcount %d on object %p\n", current_value, user_data); + if (current_value < 0) { + ast_log(__LOG_ERROR, file, line, funcname, + "Invalid refcount %d on ao2 object %p\n", current_value, user_data); + } - if (current_value <= 0) { /* last reference, destroy the object */ - if (obj->priv_data.destructor_fn != NULL) { - obj->priv_data.destructor_fn(user_data); - } + /* last reference, destroy the object */ + if (obj->priv_data.destructor_fn != NULL) { + obj->priv_data.destructor_fn(user_data); + } - ast_mutex_destroy(&obj->priv_data.lock); #ifdef AO2_DEBUG - ast_atomic_fetchadd_int(&ao2.total_mem, - obj->priv_data.data_size); - ast_atomic_fetchadd_int(&ao2.total_objects, -1); + ast_atomic_fetchadd_int(&ao2.total_mem, - obj->priv_data.data_size); + ast_atomic_fetchadd_int(&ao2.total_objects, -1); #endif - /* for safety, zero-out the astobj2 header and also the + + switch (obj->priv_data.options & AO2_ALLOC_OPT_LOCK_MASK) { + case AO2_ALLOC_OPT_LOCK_MUTEX: + obj_mutex = INTERNAL_OBJ_MUTEX(user_data); + ast_mutex_destroy(&obj_mutex->mutex.lock); + + /* + * For safety, zero-out the astobj2_lock header and also the + * first word of the user-data, which we make sure is always + * allocated. + */ + memset(obj_mutex, '\0', sizeof(*obj_mutex) + sizeof(void *) ); + ast_free(obj_mutex); + break; + case AO2_ALLOC_OPT_LOCK_RWLOCK: + obj_rwlock = INTERNAL_OBJ_RWLOCK(user_data); + ast_rwlock_destroy(&obj_rwlock->rwlock.lock); + + /* + * For safety, zero-out the astobj2_rwlock header and also the * first word of the user-data, which we make sure is always - * allocated. */ - memset(obj, '\0', sizeof(struct astobj2 *) + sizeof(void *) ); + * allocated. + */ + memset(obj_rwlock, '\0', sizeof(*obj_rwlock) + sizeof(void *) ); + ast_free(obj_rwlock); + break; + case AO2_ALLOC_OPT_LOCK_NOLOCK: + /* + * For safety, zero-out the astobj2 header and also the first + * word of the user-data, which we make sure is always + * allocated. + */ + memset(obj, '\0', sizeof(*obj) + sizeof(void *) ); ast_free(obj); + break; + default: + ast_log(__LOG_ERROR, file, line, funcname, + "Invalid lock option on ao2 object %p\n", user_data); + break; } return ret; } -/* - * We always alloc at least the size of a void *, - * for debugging purposes. - */ -static void *internal_ao2_alloc(size_t data_size, ao2_destructor_fn destructor_fn, const char *file, int line, const char *funcname) +int __ao2_ref_debug(void *user_data, int delta, const char *tag, const char *file, int line, const char *funcname) +{ + struct astobj2 *obj = INTERNAL_OBJ(user_data); + + if (obj == NULL) + return -1; + + if (delta != 0) { + FILE *refo = fopen(REF_FILE, "a"); + if (refo) { + fprintf(refo, "%p %s%d %s:%d:%s (%s) [@%d]\n", user_data, (delta < 0 ? "" : "+"), + delta, file, line, funcname, tag, obj ? obj->priv_data.ref_counter : -1); + fclose(refo); + } + } + if (obj->priv_data.ref_counter + delta == 0 && obj->priv_data.destructor_fn != NULL) { /* this isn't protected with lock; just for o/p */ + FILE *refo = fopen(REF_FILE, "a"); + if (refo) { + fprintf(refo, "%p **call destructor** %s:%d:%s (%s)\n", user_data, file, line, funcname, tag); + fclose(refo); + } + } + return internal_ao2_ref(user_data, delta, file, line, funcname); +} + +int __ao2_ref(void *user_data, int delta) +{ + struct astobj2 *obj = INTERNAL_OBJ(user_data); + + if (obj == NULL) + return -1; + + return internal_ao2_ref(user_data, delta, __FILE__, __LINE__, __FUNCTION__); +} + +static void *internal_ao2_alloc(size_t data_size, ao2_destructor_fn destructor_fn, unsigned int options, const char *file, int line, const char *funcname) { /* allocation */ struct astobj2 *obj; + struct astobj2_lock *obj_mutex; + struct astobj2_rwlock *obj_rwlock; - if (data_size < sizeof(void *)) + if (data_size < sizeof(void *)) { + /* + * We always alloc at least the size of a void *, + * for debugging purposes. + */ data_size = sizeof(void *); + } + switch (options & AO2_ALLOC_OPT_LOCK_MASK) { + case AO2_ALLOC_OPT_LOCK_MUTEX: #if defined(__AST_DEBUG_MALLOC) - obj = __ast_calloc(1, sizeof(*obj) + data_size, file, line, funcname); + obj_mutex = __ast_calloc(1, sizeof(*obj_mutex) + data_size, file, line, funcname); #else - obj = ast_calloc(1, sizeof(*obj) + data_size); + obj_mutex = ast_calloc(1, sizeof(*obj_mutex) + data_size); #endif + if (obj_mutex == NULL) { + return NULL; + } - if (obj == NULL) + ast_mutex_init(&obj_mutex->mutex.lock); + obj = (struct astobj2 *) &obj_mutex->priv_data; + break; + case AO2_ALLOC_OPT_LOCK_RWLOCK: +#if defined(__AST_DEBUG_MALLOC) + obj_rwlock = __ast_calloc(1, sizeof(*obj_rwlock) + data_size, file, line, funcname); +#else + obj_rwlock = ast_calloc(1, sizeof(*obj_rwlock) + data_size); +#endif + if (obj_rwlock == NULL) { + return NULL; + } + + ast_rwlock_init(&obj_rwlock->rwlock.lock); + obj = (struct astobj2 *) &obj_rwlock->priv_data; + break; + case AO2_ALLOC_OPT_LOCK_NOLOCK: +#if defined(__AST_DEBUG_MALLOC) + obj = __ast_calloc(1, sizeof(*obj) + data_size, file, line, funcname); +#else + obj = ast_calloc(1, sizeof(*obj) + data_size); +#endif + if (obj == NULL) { + return NULL; + } + break; + default: + /* Invalid option value. */ + ast_log(__LOG_DEBUG, file, line, funcname, "Invalid lock option requested\n"); return NULL; + } - ast_mutex_init(&obj->priv_data.lock); - obj->priv_data.magic = AO2_MAGIC; - obj->priv_data.data_size = data_size; + /* Initialize common ao2 values. */ obj->priv_data.ref_counter = 1; obj->priv_data.destructor_fn = destructor_fn; /* can be NULL */ + obj->priv_data.data_size = data_size; + obj->priv_data.options = options; + obj->priv_data.magic = AO2_MAGIC; #ifdef AO2_DEBUG ast_atomic_fetchadd_int(&ao2.total_objects, 1); @@ -324,14 +608,14 @@ static void *internal_ao2_alloc(size_t data_size, ao2_destructor_fn destructor_f return EXTERNAL_OBJ(obj); } -void *__ao2_alloc_debug(size_t data_size, ao2_destructor_fn destructor_fn, const char *tag, - const char *file, int line, const char *funcname, int ref_debug) +void *__ao2_alloc_debug(size_t data_size, ao2_destructor_fn destructor_fn, unsigned int options, const char *tag, + const char *file, int line, const char *funcname, int ref_debug) { /* allocation */ void *obj; FILE *refo; - if ((obj = internal_ao2_alloc(data_size, destructor_fn, file, line, funcname)) == NULL) { + if ((obj = internal_ao2_alloc(data_size, destructor_fn, options, file, line, funcname)) == NULL) { return NULL; } @@ -344,9 +628,9 @@ void *__ao2_alloc_debug(size_t data_size, ao2_destructor_fn destructor_fn, const return obj; } -void *__ao2_alloc(size_t data_size, ao2_destructor_fn destructor_fn) +void *__ao2_alloc(size_t data_size, ao2_destructor_fn destructor_fn, unsigned int options) { - return internal_ao2_alloc(data_size, destructor_fn, __FILE__, __LINE__, __FUNCTION__); + return internal_ao2_alloc(data_size, destructor_fn, options, __FILE__, __LINE__, __FUNCTION__); } @@ -356,6 +640,17 @@ static void container_destruct(void *c); /* internal callback to destroy a container. */ static void container_destruct_debug(void *c); +/*! + * A structure to create a linked list of entries, + * used within a bucket. + * XXX \todo this should be private to the container code + */ +struct bucket_entry { + AST_LIST_ENTRY(bucket_entry) entry; + int version; + struct astobj2 *astobj;/* pointer to internal data */ +}; + /* each bucket in the container is a tailq. */ AST_LIST_HEAD_NOLOCK(bucket, bucket_entry); @@ -410,14 +705,15 @@ static int hash_zero(const void *user_obj, const int flags) /* * A container is just an object, after all! */ -static struct ao2_container *internal_ao2_container_alloc(struct ao2_container *c, const unsigned int n_buckets, ao2_hash_fn *hash_fn, - ao2_callback_fn *cmp_fn) +static struct ao2_container *internal_ao2_container_alloc(struct ao2_container *c, + unsigned int n_buckets, ao2_hash_fn *hash_fn, ao2_callback_fn *cmp_fn) { /* XXX maybe consistency check on arguments ? */ /* compute the container size */ - if (!c) + if (!c) { return NULL; + } c->version = 1; /* 0 is a reserved value here */ c->n_buckets = hash_fn ? n_buckets : 1; @@ -431,28 +727,27 @@ static struct ao2_container *internal_ao2_container_alloc(struct ao2_container * return c; } -struct ao2_container *__ao2_container_alloc_debug(unsigned int n_buckets, ao2_hash_fn *hash_fn, - ao2_callback_fn *cmp_fn, const char *tag, char *file, int line, - const char *funcname, int ref_debug) +struct ao2_container *__ao2_container_alloc_debug(unsigned int options, + unsigned int n_buckets, ao2_hash_fn *hash_fn, ao2_callback_fn *cmp_fn, + const char *tag, const char *file, int line, const char *funcname, int ref_debug) { /* XXX maybe consistency check on arguments ? */ /* compute the container size */ - const unsigned int num_buckets = hash_fn ? n_buckets : 1; + unsigned int num_buckets = hash_fn ? n_buckets : 1; size_t container_size = sizeof(struct ao2_container) + num_buckets * sizeof(struct bucket); - struct ao2_container *c = __ao2_alloc_debug(container_size, container_destruct_debug, tag, file, line, funcname, ref_debug); + struct ao2_container *c = __ao2_alloc_debug(container_size, container_destruct_debug, options, tag, file, line, funcname, ref_debug); return internal_ao2_container_alloc(c, num_buckets, hash_fn, cmp_fn); } -struct ao2_container *__ao2_container_alloc(unsigned int n_buckets, ao2_hash_fn *hash_fn, - ao2_callback_fn *cmp_fn) +struct ao2_container *__ao2_container_alloc(unsigned int options, + unsigned int n_buckets, ao2_hash_fn *hash_fn, ao2_callback_fn *cmp_fn) { /* XXX maybe consistency check on arguments ? */ /* compute the container size */ - const unsigned int num_buckets = hash_fn ? n_buckets : 1; size_t container_size = sizeof(struct ao2_container) + num_buckets * sizeof(struct bucket); - struct ao2_container *c = __ao2_alloc(container_size, container_destruct); + struct ao2_container *c = __ao2_alloc(container_size, container_destruct, options); return internal_ao2_container_alloc(c, num_buckets, hash_fn, cmp_fn); } @@ -465,77 +760,68 @@ int ao2_container_count(struct ao2_container *c) return c->elements; } -/*! - * A structure to create a linked list of entries, - * used within a bucket. - * XXX \todo this should be private to the container code - */ -struct bucket_entry { - AST_LIST_ENTRY(bucket_entry) entry; - int version; - struct astobj2 *astobj; /* pointer to internal data */ -}; - /* * link an object to a container */ - -static struct bucket_entry *internal_ao2_link(struct ao2_container *c, void *user_data, int flags, const char *file, int line, const char *func) +static struct bucket_entry *internal_ao2_link(struct ao2_container *c, void *user_data, int flags, const char *tag, const char *file, int line, const char *funcname) { int i; + enum ao2_lock_req orig_lock; /* create a new list entry */ struct bucket_entry *p; struct astobj2 *obj = INTERNAL_OBJ(user_data); - if (obj == NULL) + if (obj == NULL) { return NULL; + } - if (INTERNAL_OBJ(c) == NULL) + if (INTERNAL_OBJ(c) == NULL) { return NULL; + } p = ast_calloc(1, sizeof(*p)); - if (!p) + if (!p) { return NULL; + } i = abs(c->hash_fn(user_data, OBJ_POINTER)); - if (!(flags & OBJ_NOLOCK)) { - ao2_lock(c); + if (flags & OBJ_NOLOCK) { + orig_lock = adjust_lock(c, AO2_LOCK_REQ_WRLOCK, 1); + } else { + ao2_wrlock(c); + orig_lock = AO2_LOCK_REQ_MUTEX; } + i %= c->n_buckets; p->astobj = obj; p->version = ast_atomic_fetchadd_int(&c->version, 1); AST_LIST_INSERT_TAIL(&c->buckets[i], p, entry); ast_atomic_fetchadd_int(&c->elements, 1); - /* the last two operations (ao2_ref, ao2_unlock) must be done by the calling func */ - return p; -} - -void *__ao2_link_debug(struct ao2_container *c, void *user_data, int flags, const char *tag, char *file, int line, const char *funcname) -{ - struct bucket_entry *p = internal_ao2_link(c, user_data, flags, file, line, funcname); - - if (p) { + if (tag) { __ao2_ref_debug(user_data, +1, tag, file, line, funcname); - if (!(flags & OBJ_NOLOCK)) { - ao2_unlock(c); - } + } else { + __ao2_ref(user_data, +1); } + + if (flags & OBJ_NOLOCK) { + adjust_lock(c, orig_lock, 0); + } else { + ao2_unlock(c); + } + return p; } -void *__ao2_link(struct ao2_container *c, void *user_data, int flags) +void *__ao2_link_debug(struct ao2_container *c, void *new_obj, int flags, const char *tag, const char *file, int line, const char *funcname) { - struct bucket_entry *p = internal_ao2_link(c, user_data, flags, __FILE__, __LINE__, __PRETTY_FUNCTION__); + return internal_ao2_link(c, new_obj, flags, tag, file, line, funcname); +} - if (p) { - __ao2_ref(user_data, +1); - if (!(flags & OBJ_NOLOCK)) { - ao2_unlock(c); - } - } - return p; +void *__ao2_link(struct ao2_container *c, void *new_obj, int flags) +{ + return internal_ao2_link(c, new_obj, flags, NULL, __FILE__, __LINE__, __PRETTY_FUNCTION__); } /*! @@ -550,14 +836,14 @@ int ao2_match_by_addr(void *user_data, void *arg, int flags) * Unlink an object from the container * and destroy the associated * bucket_entry structure. */ -void *__ao2_unlink_debug(struct ao2_container *c, void *user_data, int flags, const char *tag, - char *file, int line, const char *funcname) +void *__ao2_unlink_debug(struct ao2_container *c, void *user_data, int flags, + const char *tag, const char *file, int line, const char *funcname) { - if (INTERNAL_OBJ(user_data) == NULL) /* safety check on the argument */ + if (INTERNAL_OBJ(user_data) == NULL) { /* safety check on the argument */ return NULL; + } flags |= (OBJ_UNLINK | OBJ_POINTER | OBJ_NODATA); - __ao2_callback_debug(c, flags, ao2_match_by_addr, user_data, tag, file, line, funcname); return NULL; @@ -565,8 +851,9 @@ void *__ao2_unlink_debug(struct ao2_container *c, void *user_data, int flags, co void *__ao2_unlink(struct ao2_container *c, void *user_data, int flags) { - if (INTERNAL_OBJ(user_data) == NULL) /* safety check on the argument */ + if (INTERNAL_OBJ(user_data) == NULL) { /* safety check on the argument */ return NULL; + } flags |= (OBJ_UNLINK | OBJ_POINTER | OBJ_NODATA); __ao2_callback(c, flags, ao2_match_by_addr, user_data); @@ -598,19 +885,21 @@ static int cb_true_data(void *user_data, void *arg, void *data, int flags) * aren't an excessive load to the system, as the callback should not be * called as often as, say, the ao2_ref func is called. */ -static void *internal_ao2_callback(struct ao2_container *c, - const enum search_flags flags, void *cb_fn, void *arg, void *data, enum ao2_callback_type type, - const char *tag, char *file, int line, const char *funcname) +static void *internal_ao2_callback(struct ao2_container *c, enum search_flags flags, + void *cb_fn, void *arg, void *data, enum ao2_callback_type type, const char *tag, + const char *file, int line, const char *funcname) { int i, start, last; /* search boundaries */ + enum ao2_lock_req orig_lock; void *ret = NULL; ao2_callback_fn *cb_default = NULL; ao2_callback_data_fn *cb_withdata = NULL; struct ao2_container *multi_container = NULL; struct ao2_iterator *multi_iterator = NULL; - if (INTERNAL_OBJ(c) == NULL) /* safety check on the argument */ + if (INTERNAL_OBJ(c) == NULL) { /* safety check on the argument */ return NULL; + } /* * This logic is used so we can support OBJ_MULTIPLE with OBJ_NODATA @@ -625,7 +914,8 @@ static void *internal_ao2_callback(struct ao2_container *c, * is destroyed, the container will be automatically * destroyed as well. */ - if (!(multi_container = __ao2_container_alloc(1, NULL, NULL))) { + multi_container = __ao2_container_alloc(AO2_ALLOC_OPT_LOCK_NOLOCK, 1, NULL, NULL); + if (!multi_container) { return NULL; } if (!(multi_iterator = ast_calloc(1, sizeof(*multi_iterator)))) { @@ -657,10 +947,13 @@ static void *internal_ao2_callback(struct ao2_container *c, * run the hash function. Otherwise, scan the whole container * (this only for the time being. We need to optimize this.) */ - if ((flags & (OBJ_POINTER | OBJ_KEY))) /* we know hash can handle this case */ + if ((flags & (OBJ_POINTER | OBJ_KEY))) { + /* we know hash can handle this case */ start = i = c->hash_fn(arg, flags & (OBJ_POINTER | OBJ_KEY)) % c->n_buckets; - else /* don't know, let's scan all buckets */ + } else { + /* don't know, let's scan all buckets */ start = i = -1; /* XXX this must be fixed later. */ + } /* determine the search boundaries: i..last-1 */ if (i < 0) { @@ -672,9 +965,20 @@ static void *internal_ao2_callback(struct ao2_container *c, last = i + 1; } - - if (!(flags & OBJ_NOLOCK)) { - ao2_lock(c); /* avoid modifications to the content */ + /* avoid modifications to the content */ + if (flags & OBJ_NOLOCK) { + if (flags & OBJ_UNLINK) { + orig_lock = adjust_lock(c, AO2_LOCK_REQ_WRLOCK, 1); + } else { + orig_lock = adjust_lock(c, AO2_LOCK_REQ_RDLOCK, 1); + } + } else { + orig_lock = AO2_LOCK_REQ_MUTEX; + if (flags & OBJ_UNLINK) { + ao2_wrlock(c); + } else { + ao2_rdlock(c); + } } for (; i < last ; i++) { @@ -710,7 +1014,7 @@ static void *internal_ao2_callback(struct ao2_container *c, } } - /* If we are in OBJ_MULTIPLE mode and OBJ_NODATE is off, + /* If we are in OBJ_MULTIPLE mode and OBJ_NODATA is off, * link the object into the container that will hold the results. */ if (ret && (multi_container != NULL)) { @@ -764,14 +1068,16 @@ static void *internal_ao2_callback(struct ao2_container *c, } } - if (!(flags & OBJ_NOLOCK)) { + if (flags & OBJ_NOLOCK) { + adjust_lock(c, orig_lock, 0); + } else { ao2_unlock(c); } /* if multi_container was created, we are returning multiple objects */ if (multi_container != NULL) { *multi_iterator = ao2_iterator_init(multi_container, - AO2_ITERATOR_DONTLOCK | AO2_ITERATOR_UNLINK | AO2_ITERATOR_MALLOCD); + AO2_ITERATOR_UNLINK | AO2_ITERATOR_MALLOCD); ao2_ref(multi_container, -1); return multi_iterator; } else { @@ -780,7 +1086,7 @@ static void *internal_ao2_callback(struct ao2_container *c, } void *__ao2_callback_debug(struct ao2_container *c, enum search_flags flags, - ao2_callback_fn *cb_fn, void *arg, const char *tag, char *file, int line, + ao2_callback_fn *cb_fn, void *arg, const char *tag, const char *file, int line, const char *funcname) { return internal_ao2_callback(c,flags, cb_fn, arg, NULL, DEFAULT, tag, file, line, funcname); @@ -792,16 +1098,15 @@ void *__ao2_callback(struct ao2_container *c, enum search_flags flags, return internal_ao2_callback(c,flags, cb_fn, arg, NULL, DEFAULT, NULL, NULL, 0, NULL); } -void *__ao2_callback_data_debug(struct ao2_container *c, - enum search_flags flags, - ao2_callback_data_fn *cb_fn, void *arg, void *data, - const char *tag, char *file, int line, const char *funcname) +void *__ao2_callback_data_debug(struct ao2_container *c, enum search_flags flags, + ao2_callback_data_fn *cb_fn, void *arg, void *data, const char *tag, const char *file, + int line, const char *funcname) { return internal_ao2_callback(c, flags, cb_fn, arg, data, WITH_DATA, tag, file, line, funcname); } void *__ao2_callback_data(struct ao2_container *c, enum search_flags flags, - ao2_callback_data_fn *cb_fn, void *arg, void *data) + ao2_callback_data_fn *cb_fn, void *arg, void *data) { return internal_ao2_callback(c, flags, cb_fn, arg, data, WITH_DATA, NULL, NULL, 0, NULL); } @@ -809,7 +1114,8 @@ void *__ao2_callback_data(struct ao2_container *c, enum search_flags flags, /*! * the find function just invokes the default callback with some reasonable flags. */ -void *__ao2_find_debug(struct ao2_container *c, const void *arg, enum search_flags flags, const char *tag, char *file, int line, const char *funcname) +void *__ao2_find_debug(struct ao2_container *c, const void *arg, enum search_flags flags, + const char *tag, const char *file, int line, const char *funcname) { void *arged = (void *) arg;/* Done to avoid compiler const warning */ @@ -854,26 +1160,39 @@ void ao2_iterator_destroy(struct ao2_iterator *i) /* * move to the next element in the container. */ -static void *internal_ao2_iterator_next(struct ao2_iterator *a, struct bucket_entry **q) +static void *internal_ao2_iterator_next(struct ao2_iterator *a, const char *tag, const char *file, int line, const char *funcname) { int lim; + enum ao2_lock_req orig_lock; struct bucket_entry *p = NULL; - void *ret = NULL; - - *q = NULL; + void *ret; - if (INTERNAL_OBJ(a->c) == NULL) + if (INTERNAL_OBJ(a->c) == NULL) { return NULL; + } - if (!(a->flags & AO2_ITERATOR_DONTLOCK)) - ao2_lock(a->c); + if (a->flags & AO2_ITERATOR_DONTLOCK) { + if (a->flags & AO2_ITERATOR_UNLINK) { + orig_lock = adjust_lock(a->c, AO2_LOCK_REQ_WRLOCK, 1); + } else { + orig_lock = adjust_lock(a->c, AO2_LOCK_REQ_RDLOCK, 1); + } + } else { + orig_lock = AO2_LOCK_REQ_MUTEX; + if (a->flags & AO2_ITERATOR_UNLINK) { + ao2_wrlock(a->c); + } else { + ao2_rdlock(a->c); + } + } /* optimization. If the container is unchanged and * we have a pointer, try follow it */ if (a->c->version == a->c_version && (p = a->obj)) { - if ((p = AST_LIST_NEXT(p, entry))) + if ((p = AST_LIST_NEXT(p, entry))) { goto found; + } /* nope, start from the next bucket */ a->bucket++; a->version = 0; @@ -891,8 +1210,9 @@ static void *internal_ao2_iterator_next(struct ao2_iterator *a, struct bucket_en for (; a->bucket < lim; a->bucket++, a->version = 0) { /* scan the current bucket */ AST_LIST_TRAVERSE(&a->c->buckets[a->bucket], p, entry) { - if (p->version > a->version) + if (p->version > a->version) { goto found; + } } } @@ -913,48 +1233,35 @@ found: a->version = p->version; a->obj = p; a->c_version = a->c->version; + /* inc refcount of returned object */ - *q = p; + if (tag) { + __ao2_ref_debug(ret, 1, tag, file, line, funcname); + } else { + __ao2_ref(ret, 1); + } } + } else { + ret = NULL; + } + + if (a->flags & AO2_ITERATOR_DONTLOCK) { + adjust_lock(a->c, orig_lock, 0); + } else { + ao2_unlock(a->c); } return ret; } -void *__ao2_iterator_next_debug(struct ao2_iterator *a, const char *tag, char *file, int line, const char *funcname) +void *__ao2_iterator_next_debug(struct ao2_iterator *a, const char *tag, const char *file, int line, const char *funcname) { - struct bucket_entry *p; - void *ret = NULL; - - ret = internal_ao2_iterator_next(a, &p); - - if (p) { - /* inc refcount of returned object */ - __ao2_ref_debug(ret, 1, tag, file, line, funcname); - } - - if (!(a->flags & AO2_ITERATOR_DONTLOCK)) - ao2_unlock(a->c); - - return ret; + return internal_ao2_iterator_next(a, tag, file, line, funcname); } void *__ao2_iterator_next(struct ao2_iterator *a) { - struct bucket_entry *p = NULL; - void *ret = NULL; - - ret = internal_ao2_iterator_next(a, &p); - - if (p) { - /* inc refcount of returned object */ - __ao2_ref(ret, 1); - } - - if (!(a->flags & AO2_ITERATOR_DONTLOCK)) - ao2_unlock(a->c); - - return ret; + return internal_ao2_iterator_next(a, NULL, __FILE__, __LINE__, __PRETTY_FUNCTION__); } /* callback for destroying container. @@ -1037,8 +1344,8 @@ int ao2_container_dup(struct ao2_container *dest, struct ao2_container *src, enu int res = 0; if (!(flags & OBJ_NOLOCK)) { - ao2_lock(src); - ao2_lock(dest); + ao2_rdlock(src); + ao2_wrlock(dest); } obj = __ao2_callback(src, OBJ_NOLOCK, dup_obj_cb, dest); if (obj) { @@ -1061,16 +1368,24 @@ int ao2_container_dup(struct ao2_container *dest, struct ao2_container *src, enu struct ao2_container *__ao2_container_clone(struct ao2_container *orig, enum search_flags flags) { struct ao2_container *clone; + struct astobj2 *orig_obj; + unsigned int options; int failed; + orig_obj = INTERNAL_OBJ(orig); + if (!orig_obj) { + return NULL; + } + options = orig_obj->priv_data.options; + /* Create the clone container with the same properties as the original. */ - clone = __ao2_container_alloc(orig->n_buckets, orig->hash_fn, orig->cmp_fn); + clone = __ao2_container_alloc(options, orig->n_buckets, orig->hash_fn, orig->cmp_fn); if (!clone) { return NULL; } if (flags & OBJ_NOLOCK) { - ao2_lock(clone); + ao2_wrlock(clone); } failed = ao2_container_dup(clone, orig, flags); if (flags & OBJ_NOLOCK) { @@ -1084,20 +1399,28 @@ struct ao2_container *__ao2_container_clone(struct ao2_container *orig, enum sea return clone; } -struct ao2_container *__ao2_container_clone_debug(struct ao2_container *orig, enum search_flags flags, const char *tag, char *file, int line, const char *funcname, int ref_debug) +struct ao2_container *__ao2_container_clone_debug(struct ao2_container *orig, enum search_flags flags, const char *tag, const char *file, int line, const char *funcname, int ref_debug) { struct ao2_container *clone; + struct astobj2 *orig_obj; + unsigned int options; int failed; + orig_obj = INTERNAL_OBJ(orig); + if (!orig_obj) { + return NULL; + } + options = orig_obj->priv_data.options; + /* Create the clone container with the same properties as the original. */ - clone = __ao2_container_alloc_debug(orig->n_buckets, orig->hash_fn, orig->cmp_fn, tag, - file, line, funcname, ref_debug); + clone = __ao2_container_alloc_debug(options, orig->n_buckets, orig->hash_fn, + orig->cmp_fn, tag, file, line, funcname, ref_debug); if (!clone) { return NULL; } if (flags & OBJ_NOLOCK) { - ao2_lock(clone); + ao2_wrlock(clone); } failed = ao2_container_dup(clone, orig, flags); if (flags & OBJ_NOLOCK) { |