Enhance astobj2 to support other types of containers.

The new API allows for sorted containers, insertion options, duplicate
handling options, and traversal order options.

* Adds the ability for containers to be sorted when they are created.

* Adds container creation options to handle duplicates when they are
inserted.

* Adds container creation option to insert objects at the beginning or end
of the container traversal order.

* Adds OBJ_PARTIAL_KEY to allow searching with a partial key.  The partial
key works similarly to the OBJ_KEY flag.  (The real search speed
improvement with this flag will come when red-black trees are added.)

* Adds container traversal and iteration order options: Ascending and
Descending.

* Adds an AST_DEVMODE compile feature to check the stats and integrity of
registered containers using the CLI "astobj2 container stats <name>" and
"astobj2 container check <name>".  The channels container is normally
registered since it is one of the most important containers in the system.

* Adds ao2_iterator_restart() to allow iteration to be restarted from the
beginning.

* Changes the generic container object to have a v_method table pointer to
support other types of containers.

* Changes the container nodes holding objects to be ref counted.

The ref counted nodes and v_method table pointer changes pave the way to
allow other types of containers.

* Includes a large astobj2 unit test enhancement that tests the new
features.

(closes issue ASTERISK-19969)
Reported by: rmudgett

Review: https://reviewboard.asterisk.org/r/2078/


git-svn-id: https://origsvn.digium.com/svn/asterisk/trunk@372997 65c4cc65-6c06-0410-ace0-fbb531ad65f3

1 files changed, 1416 insertions, 149 deletions

diff --git a/tests/test_astobj2.c b/tests/test_astobj2.c
index 275c260af..95d7e753d 100644
--- a/tests/test_astobj2.c
+++ b/tests/test_astobj2.c
@@ -37,15 +37,59 @@ ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
 #include "asterisk/test.h"
 #include "asterisk/astobj2.h"
 
+enum test_container_type {
+	TEST_CONTAINER_LIST,
+	TEST_CONTAINER_HASH,
+};
+
+/*!
+ * \internal
+ * \brief Convert the container type enum to string.
+ * \since 12.0.0
+ *
+ * \param type Container type value to convert to string.
+ *
+ * \return String value of container type.
+ */
+static const char *test_container2str(enum test_container_type type)
+{
+	const char *c_type;
+
+	c_type = "Unknown";
+	switch (type) {
+	case TEST_CONTAINER_LIST:
+		c_type = "List";
+		break;
+	case TEST_CONTAINER_HASH:
+		c_type = "Hash";
+		break;
+	}
+	return c_type;
+}
+
 struct test_obj {
-	int i;
+	/*! What to decrement when object is destroyed. */
 	int *destructor_count;
+	/*! Container object key */
+	int i;
+	/*! Identifier for duplicate object key tests. */
+	int dup_number;
 };
 
-static void test_obj_destructor(void *obj)
+/*! Partial search key +/- matching range. */
+int partial_key_match_range;
+/*! Special iax2 OBJ_CONTINUE test.  Bucket selected. */
+int special_bucket;
+/*! Special iax2 OBJ_CONTINUE test.  Object number select. */
+int special_match;
+
+static void test_obj_destructor(void *v_obj)
 {
-	struct test_obj *test_obj = (struct test_obj *) obj;
-	*test_obj->destructor_count = *test_obj->destructor_count - 1;
+	struct test_obj *obj = (struct test_obj *) v_obj;
+
+	if (obj->destructor_count) {
+		--*obj->destructor_count;
+	}
 }
 
 static int increment_cb(void *obj, void *arg, int flag)
@@ -58,124 +102,140 @@ static int increment_cb(void *obj, void *arg, int flag)
 
 static int all_but_one_cb(void *obj, void *arg, int flag)
 {
-	struct test_obj *test_obj = (struct test_obj *) obj;
+	struct test_obj *cmp_obj = (struct test_obj *) obj;
 
-	return (test_obj->i > 1) ? CMP_MATCH : 0;
+	return (cmp_obj->i) ? CMP_MATCH : 0;
 }
 
 static int multiple_cb(void *obj, void *arg, int flag)
 {
 	int *i = (int *) arg;
-	struct test_obj *test_obj = (struct test_obj *) obj;
+	struct test_obj *cmp_obj = (struct test_obj *) obj;
 
-	return (test_obj->i <= *i) ? CMP_MATCH : 0;
+	return (cmp_obj->i < *i) ? CMP_MATCH : 0;
 }
 
 static int test_cmp_cb(void *obj, void *arg, int flags)
 {
 	struct test_obj *cmp_obj = (struct test_obj *) obj;
 
-	if (!arg) {
-		return 0;
-	}
-
 	if (flags & OBJ_KEY) {
 		int *i = (int *) arg;
-		return (cmp_obj->i == *i) ? CMP_MATCH | CMP_STOP : 0;
+
+		return (cmp_obj->i == *i) ? CMP_MATCH : 0;
+	} else if (flags & OBJ_PARTIAL_KEY) {
+		int *i = (int *) arg;
+
+		return (*i - partial_key_match_range <= cmp_obj->i
+			&& cmp_obj->i <= *i + partial_key_match_range) ? CMP_MATCH : 0;
 	} else {
-		struct test_obj *test_obj = (struct test_obj *) arg;
-		return (cmp_obj->i == test_obj->i) ? CMP_MATCH | CMP_STOP : 0;
+		struct test_obj *arg_obj = (struct test_obj *) arg;
+
+		if (!arg_obj) {
+			/* Never match on the special iax2 OBJ_CONTINUE test. */
+			return 0;
+		}
+
+		return (cmp_obj->i == arg_obj->i) ? CMP_MATCH : 0;
 	}
 }
 
 static int test_hash_cb(const void *obj, const int flags)
 {
-	if (!obj) {
-		return 0;
-	}
-
 	if (flags & OBJ_KEY) {
 		const int *i = obj;
 
 		return *i;
+	} else if (flags & OBJ_PARTIAL_KEY) {
+		/* This is absolutely wrong to be called with this flag value. */
+		abort();
+		/* Just in case abort() doesn't work or something else super silly */
+		*((int *) 0) = 0;
+		return 0;
 	} else {
-		const struct test_obj *test_obj = obj;
+		const struct test_obj *hash_obj = obj;
 
-		return test_obj->i;
+		if (!hash_obj) {
+			/*
+			 * Use the special_bucket as the bucket for the special iax2
+			 * OBJ_CONTINUE test.
+			 */
+			return special_bucket;
+		}
+
+		return hash_obj->i;
 	}
 }
 
-static int astobj2_test_helper(int use_hash, int use_cmp, unsigned int lim, struct ast_test *test)
+static int test_sort_cb(const void *obj_left, const void *obj_right, int flags)
 {
-	struct ao2_container *c1;
-	struct ao2_container *c2;
-	struct ao2_container *c3 = NULL;
-	struct ao2_iterator it;
-	struct ao2_iterator *mult_it;
-	struct test_obj *obj;
-	struct test_obj *obj2;
-	struct test_obj tmp_obj;
-	int bucket_size;
-	int increment = 0;
-	int destructor_count = 0;
-	int num;
-	int  res = AST_TEST_PASS;
-
-	/* This test needs at least 5 objects */
-	if (lim < 5) {
-		lim = 5;
-	}
+	const struct test_obj *test_left = obj_left;
 
-	bucket_size = (ast_random() % ((lim / 4) + 1)) + 1;
-	c1 = ao2_t_container_alloc(bucket_size, use_hash ? test_hash_cb : NULL, use_cmp ? test_cmp_cb : NULL, "test");
-	c2 = ao2_t_container_alloc(bucket_size, test_hash_cb, test_cmp_cb, "test");
+	if (flags & OBJ_KEY) {
+		const int *i = obj_right;
 
-	if (!c1 || !c2) {
-		ast_test_status_update(test, "ao2_container_alloc failed.\n");
-		res = AST_TEST_FAIL;
-		goto cleanup;
-	}
+		return test_left->i - *i;
+	} else if (flags & OBJ_PARTIAL_KEY) {
+		int *i = (int *) obj_right;
 
-	/* Create objects and link into container */
-	destructor_count = lim;
-	for (num = 1; num <= lim; num++) {
-		if (!(obj = ao2_t_alloc(sizeof(struct test_obj), test_obj_destructor, "making zombies"))) {
-			ast_test_status_update(test, "ao2_alloc failed.\n");
-			res = AST_TEST_FAIL;
-			goto cleanup;
+		if (*i - partial_key_match_range <= test_left->i
+			&& test_left->i <= *i + partial_key_match_range) {
+			return 0;
 		}
-		obj->destructor_count = &destructor_count;
-		obj->i = num;
-		ao2_link(c1, obj);
-		ao2_t_ref(obj, -1, "test");
-		if (ao2_container_count(c1) != num) {
-			ast_test_status_update(test, "container did not link correctly\n");
-			res = AST_TEST_FAIL;
+
+		return test_left->i - *i;
+	} else {
+		const struct test_obj *test_right = obj_right;
+
+		if (!test_right) {
+			/*
+			 * Compare with special_match in the special iax2 OBJ_CONTINUE
+			 * test.
+			 */
+			return test_left->i - special_match;
 		}
+
+		return test_left->i - test_right->i;
 	}
+}
 
-	ast_test_status_update(test, "Container created: random bucket size %d: number of items: %d\n", bucket_size, lim);
+/*!
+ * \internal
+ * \brief Test container cloning.
+ * \since 12.0.0
+ *
+ * \param res Passed in enum ast_test_result_state.
+ * \param orig Container to clone.
+ * \param test Test output controller.
+ *
+ * \return enum ast_test_result_state
+ */
+static int test_container_clone(int res, struct ao2_container *orig, struct ast_test *test)
+{
+	struct ao2_container *clone;
+	struct test_obj *obj;
+	struct test_obj *obj2;
+	struct ao2_iterator iter;
 
-	/* Testing ao2_container_clone */
-	c3 = ao2_container_clone(c1, 0);
-	if (!c3) {
+	clone = ao2_container_clone(orig, 0);
+	if (!clone) {
 		ast_test_status_update(test, "ao2_container_clone failed.\n");
-		res = AST_TEST_FAIL;
-		goto cleanup;
+		return AST_TEST_FAIL;
 	}
-	if (ao2_container_count(c1) != ao2_container_count(c3)) {
+	if (ao2_container_check(clone, 0)) {
+		ast_test_status_update(test, "container integrity check failed\n");
+		res = AST_TEST_FAIL;
+	} else if (ao2_container_count(orig) != ao2_container_count(clone)) {
 		ast_test_status_update(test, "Cloned container does not have the same number of objects.\n");
 		res = AST_TEST_FAIL;
 	} else {
-		it = ao2_iterator_init(c1, 0);
-		for (; (obj = ao2_t_iterator_next(&it, "test orig")); ao2_t_ref(obj, -1, "test orig")) {
+		iter = ao2_iterator_init(orig, 0);
+		for (; (obj = ao2_t_iterator_next(&iter, "test orig")); ao2_t_ref(obj, -1, "test orig")) {
 			/*
 			 * Unlink the matching object from the cloned container to make
 			 * the next search faster.  This is a big speed optimization!
-			 * It reduces the container with 100000 objects test time from
-			 * 18 seconds to 200 ms.
 			 */
-			obj2 = ao2_t_callback(c3, OBJ_POINTER | OBJ_UNLINK, ao2_match_by_addr, obj,
+			obj2 = ao2_t_callback(clone, OBJ_POINTER | OBJ_UNLINK, ao2_match_by_addr, obj,
 				"test clone");
 			if (obj2) {
 				ao2_t_ref(obj2, -1, "test clone");
@@ -185,98 +245,366 @@ static int astobj2_test_helper(int use_hash, int use_cmp, unsigned int lim, stru
 				"Orig container has an object %p not in the clone container.\n", obj);
 			res = AST_TEST_FAIL;
 		}
-		ao2_iterator_destroy(&it);
-		if (ao2_container_count(c3)) {
+		ao2_iterator_destroy(&iter);
+		if (ao2_container_count(clone)) {
 			ast_test_status_update(test, "Cloned container still has objects.\n");
 			res = AST_TEST_FAIL;
 		}
+		if (ao2_container_check(clone, 0)) {
+			ast_test_status_update(test, "container integrity check failed\n");
+			res = AST_TEST_FAIL;
+		}
 	}
-	ao2_t_ref(c3, -1, "bye c3");
-	c3 = NULL;
+	ao2_t_ref(clone, -1, "bye clone");
+
+	return res;
+}
+
+/*!
+ * \internal
+ * \brief Test ao2_find with no flags.
+ * \since 12.0.0
+ *
+ * \param res Passed in enum ast_test_result_state.
+ * \param look_in Container to search.
+ * \param limit Container contains objects 0 - (limit - 1).
+ * \param test Test output controller.
+ *
+ * \return enum ast_test_result_state
+ */
+static int test_ao2_find_w_no_flags(int res, struct ao2_container *look_in, int limit, struct ast_test *test)
+{
+	int i;
+	int num;
+	struct test_obj tmp_obj = { 0, };
+	struct test_obj *obj;
+
+	for (num = 100; num--;) {
+		i = ast_random() % limit; /* find a random object */
 
-	/* Testing ao2_find with no flags */
-	num = 100;
-	for (; num; num--) {
-		int i = (ast_random() % ((lim / 2)) + 1); /* find a random object */
 		tmp_obj.i = i;
-		if (!(obj = ao2_find(c1, &tmp_obj, 0))) {
-			res = AST_TEST_FAIL;
+		obj = ao2_find(look_in, &tmp_obj, 0);
+		if (!obj) {
 			ast_test_status_update(test, "COULD NOT FIND:%d, ao2_find() with no flags failed.\n", i);
+			res = AST_TEST_FAIL;
 		} else {
-			/* a correct match will only take place when the custom cmp function is used */
-			if (use_cmp && obj->i != i) {
-				ast_test_status_update(test, "object %d does not match object %d\n", obj->i, tmp_obj.i);
+			if (obj->i != i) {
+				ast_test_status_update(test, "object %d does not match %d\n", obj->i, i);
 				res = AST_TEST_FAIL;
 			}
 			ao2_t_ref(obj, -1, "test");
 		}
 	}
 
-	/* Testing ao2_find with OBJ_POINTER */
-	num = 75;
-	for (; num; num--) {
-		int i = (ast_random() % ((lim / 2)) + 1); /* find a random object */
+	return res;
+}
+
+/*!
+ * \internal
+ * \brief Test ao2_find with OBJ_POINTER.
+ * \since 12.0.0
+ *
+ * \param res Passed in enum ast_test_result_state.
+ * \param look_in Container to search.
+ * \param limit Container contains objects 0 - (limit - 1).
+ * \param test Test output controller.
+ *
+ * \return enum ast_test_result_state
+ */
+static int test_ao2_find_w_OBJ_POINTER(int res, struct ao2_container *look_in, int limit, struct ast_test *test)
+{
+	int i;
+	int num;
+	struct test_obj tmp_obj = { 0, };
+	struct test_obj *obj;
+
+	for (num = 75; num--;) {
+		i = ast_random() % limit; /* find a random object */
+
 		tmp_obj.i = i;
-		if (!(obj = ao2_find(c1, &tmp_obj, OBJ_POINTER))) {
-			res = AST_TEST_FAIL;
+		obj = ao2_find(look_in, &tmp_obj, OBJ_POINTER);
+		if (!obj) {
 			ast_test_status_update(test, "COULD NOT FIND:%d, ao2_find() with OBJ_POINTER flag failed.\n", i);
+			res = AST_TEST_FAIL;
 		} else {
-			/* a correct match will only take place when the custom cmp function is used */
-			if (use_cmp && obj->i != i) {
-				ast_test_status_update(test, "object %d does not match object %d\n", obj->i, tmp_obj.i);
+			if (obj->i != i) {
+				ast_test_status_update(test, "object %d does not match %d\n", obj->i, i);
 				res = AST_TEST_FAIL;
 			}
 			ao2_t_ref(obj, -1, "test");
 		}
 	}
 
-	/* Testing ao2_find with OBJ_KEY */
-	num = 75;
-	for (; num; num--) {
-		int i = (ast_random() % ((lim / 2)) + 1); /* find a random object */
-		if (!(obj = ao2_find(c1, &i, OBJ_KEY))) {
-			res = AST_TEST_FAIL;
+	return res;
+}
+
+/*!
+ * \internal
+ * \brief Test ao2_find with OBJ_KEY.
+ * \since 12.0.0
+ *
+ * \param res Passed in enum ast_test_result_state.
+ * \param look_in Container to search.
+ * \param limit Container contains objects 0 - (limit - 1).
+ * \param test Test output controller.
+ *
+ * \return enum ast_test_result_state
+ */
+static int test_ao2_find_w_OBJ_KEY(int res, struct ao2_container *look_in, int limit, struct ast_test *test)
+{
+	int i;
+	int num;
+	struct test_obj *obj;
+
+	for (num = 75; num--;) {
+		i = ast_random() % limit; /* find a random object */
+
+		obj = ao2_find(look_in, &i, OBJ_KEY);
+		if (!obj) {
 			ast_test_status_update(test, "COULD NOT FIND:%d, ao2_find() with OBJ_KEY flag failed.\n", i);
+			res = AST_TEST_FAIL;
 		} else {
-			/* a correct match will only take place when the custom cmp function is used */
-			if (use_cmp && obj->i != i) {
-				ast_test_status_update(test, "object %d does not match object %d\n", obj->i, tmp_obj.i);
+			if (obj->i != i) {
+				ast_test_status_update(test, "object %d does not match %d\n", obj->i, i);
 				res = AST_TEST_FAIL;
 			}
 			ao2_t_ref(obj, -1, "test");
 		}
 	}
 
-	/* Testing ao2_find with OBJ_POINTER | OBJ_UNLINK | OBJ_CONTINUE.
+	return res;
+}
+
+/*!
+ * \internal
+ * \brief Test ao2_find with OBJ_PARTIAL_KEY.
+ * \since 12.0.0
+ *
+ * \param res Passed in enum ast_test_result_state.
+ * \param look_in Container to search.
+ * \param limit Container contains objects 0 - (limit - 1).
+ * \param test Test output controller.
+ *
+ * \return enum ast_test_result_state
+ */
+static int test_ao2_find_w_OBJ_PARTIAL_KEY(int res, struct ao2_container *look_in, int limit, struct ast_test *test)
+{
+	int i;
+	int num;
+	struct test_obj *obj;
+
+	/* Set partial match to find exactly. */
+	partial_key_match_range = 0;
+
+	for (num = 100; num--;) {
+		i = ast_random() % limit; /* find a random object */
+
+		obj = ao2_find(look_in, &i, OBJ_PARTIAL_KEY);
+		if (!obj) {
+			ast_test_status_update(test, "COULD NOT FIND:%d, ao2_find() with OBJ_PARTIAL_KEY flag failed.\n", i);
+			res = AST_TEST_FAIL;
+		} else {
+			if (obj->i != i) {
+				ast_test_status_update(test, "object %d does not match %d\n", obj->i, i);
+				res = AST_TEST_FAIL;
+			}
+			ao2_t_ref(obj, -1, "test");
+		}
+	}
+
+	return res;
+}
+
+static int astobj2_test_1_helper(int tst_num, enum test_container_type type, int use_sort, unsigned int lim, struct ast_test *test)
+{
+	const char *c_type;
+	struct ao2_container *c1;
+	struct ao2_container *c2;
+	struct ao2_iterator it;
+	struct ao2_iterator *mult_it;
+	struct test_obj *obj;
+	ao2_callback_fn *cmp_fn;
+	int n_buckets;
+	int increment = 0;
+	int destructor_count = 0;
+	int count;
+	int num;
+	int res = AST_TEST_PASS;
+
+	c_type = test_container2str(type);
+	ast_test_status_update(test, "Test %d, %s containers (%s).\n",
+		tst_num, c_type, use_sort ? "sorted" : "non-sorted");
+
+	/* Need at least 12 objects for the special iax2 OBJ_CONTINUE test. */
+	if (lim < 12) {
+		lim = 12;
+	}
+
+	c1 = NULL;
+	switch (type) {
+	case TEST_CONTAINER_LIST:
+		/* Lists just have one bucket. */
+		n_buckets = 1;
+		c1 = ao2_t_container_alloc_list(AO2_ALLOC_OPT_LOCK_MUTEX, 0,
+			use_sort ? test_sort_cb : NULL, test_cmp_cb, "test");
+		break;
+	case TEST_CONTAINER_HASH:
+		n_buckets = (ast_random() % ((lim / 4) + 1)) + 1;
+		if (n_buckets < 6) {
+			/* Need at least 6 buckets for the special iax2 OBJ_CONTINUE test. */
+			n_buckets = 6;
+		}
+		c1 = ao2_t_container_alloc_hash(AO2_ALLOC_OPT_LOCK_MUTEX, 0, n_buckets,
+			test_hash_cb, use_sort ? test_sort_cb : NULL, test_cmp_cb, "test");
+		break;
+	}
+	c2 = ao2_t_container_alloc(1, NULL, NULL, "test");
+
+	if (!c1 || !c2) {
+		ast_test_status_update(test, "ao2_container_alloc failed.\n");
+		res = AST_TEST_FAIL;
+		goto cleanup;
+	}
+
+	/* Create objects and link into container */
+	destructor_count = lim;
+	for (num = 0; num < lim; ++num) {
+		if (!(obj = ao2_t_alloc(sizeof(struct test_obj), test_obj_destructor, "making zombies"))) {
+			ast_test_status_update(test, "ao2_alloc failed.\n");
+			res = AST_TEST_FAIL;
+			goto cleanup;
+		}
+		obj->destructor_count = &destructor_count;
+		obj->i = num;
+		ao2_link(c1, obj);
+		ao2_t_ref(obj, -1, "test");
+		if (ao2_container_count(c1) != num + 1) {
+			ast_test_status_update(test, "container did not link correctly\n");
+			res = AST_TEST_FAIL;
+		}
+	}
+	if (ao2_container_check(c1, 0)) {
+		ast_test_status_update(test, "container integrity check failed\n");
+		res = AST_TEST_FAIL;
+		goto cleanup;
+	}
+
+	ast_test_status_update(test, "%s container created: buckets: %d, items: %d\n",
+		c_type, n_buckets, lim);
+
+	/* Testing ao2_container_clone */
+	res = test_container_clone(res, c1, test);
+
+	/* Testing ao2_find with no flags */
+	res = test_ao2_find_w_no_flags(res, c1, lim, test);
+
+	/* Testing ao2_find with OBJ_POINTER */
+	res = test_ao2_find_w_OBJ_POINTER(res, c1, lim, test);
+
+	/* Testing ao2_find with OBJ_KEY */
+	res = test_ao2_find_w_OBJ_KEY(res, c1, lim, test);
+
+	/* Testing ao2_find with OBJ_PARTIAL_KEY */
+	res = test_ao2_find_w_OBJ_PARTIAL_KEY(res, c1, lim, test);
+
+	/*
+	 * Testing ao2_find with OBJ_POINTER | OBJ_UNLINK | OBJ_CONTINUE.
 	 * In this test items are unlinked from c1 and placed in c2.  Then
 	 * unlinked from c2 and placed back into c1.
 	 *
 	 * For this module and set of custom hash/cmp functions, an object
 	 * should only be found if the astobj2 default cmp function is used.
-	 * This test is designed to mimic the chan_iax.c call number use case. */
-	num = lim < 25 ? lim : 25;
-	for (; num; num--) {
-		if (!(obj = ao2_find(c1, NULL, OBJ_POINTER | OBJ_UNLINK | OBJ_CONTINUE))) {
-			if (!use_cmp) {
-				ast_test_status_update(test, "ao2_find with OBJ_POINTER | OBJ_UNLINK | OBJ_CONTINUE failed with default hash function.\n");
-				res = AST_TEST_FAIL;
+	 * This test is designed to mimic the chan_iax.c call number use case.
+	 *
+	 * Must test the custom cmp_cb case first since it should never
+	 * find and thus unlink anything for this test.
+	 */
+	for (cmp_fn = test_cmp_cb; ; cmp_fn = NULL) {
+		num = lim;
+		for (count = 0; num && count < 100; ++count) {
+			--num;
+
+			/* This special manipulation is needed for sorted hash buckets. */
+			special_bucket = num;
+			switch (count) {
+			case 0:
+				/* Beyond end of bucket list. */
+				special_match = lim;
+				break;
+			case 1:
+				/* At end of bucket list. */
+				special_match = num;
+				break;
+			case 2:
+				/* In between in middle of bucket list. */
+				special_match = num - 1;
+				break;
+			case 3:
+				/* Beginning of bucket list. */
+				special_match = num % n_buckets;
+				break;
+			case 4:
+				/* Before bucket list. */
+				special_match = -1;
+				break;
+			default:
+				/* Empty bucket list. (If possible to empty it.) */
+				special_match = -1;
+				special_bucket = lim - 1;
+				break;
 			}
-		} else {
-			if (use_cmp) {
-				ast_test_status_update(test, "ao2_find with OBJ_POINTER | OBJ_UNLINK | OBJ_CONTINUE failed with custom hash function.\n");
-				res = AST_TEST_FAIL;
+
+			/* ao2_find is just a shortcut notation for ao2_callback(). */
+			obj = ao2_callback(c1, OBJ_POINTER | OBJ_UNLINK | OBJ_CONTINUE, cmp_fn, NULL);
+			if (!obj) {
+				if (!cmp_fn) {
+					ast_test_status_update(test,
+						"ao2_find with OBJ_POINTER | OBJ_UNLINK | OBJ_CONTINUE failed with default cmp_cb.\n");
+					res = AST_TEST_FAIL;
+				}
+			} else {
+				if (cmp_fn) {
+					ast_test_status_update(test,
+						"ao2_find with OBJ_POINTER | OBJ_UNLINK | OBJ_CONTINUE failed with custom cmp_cb.\n");
+					res = AST_TEST_FAIL;
+				}
+				ao2_link(c2, obj);
+				ao2_t_ref(obj, -1, "test");
 			}
-			ao2_link(c2, obj);
+		}
+		if (ao2_container_check(c1, 0)) {
+			ast_test_status_update(test, "container integrity check failed\n");
+			res = AST_TEST_FAIL;
+			goto cleanup;
+		}
+		if (ao2_container_check(c2, 0)) {
+			ast_test_status_update(test, "container integrity check failed\n");
+			res = AST_TEST_FAIL;
+			goto cleanup;
+		}
+		it = ao2_iterator_init(c2, 0);
+		while ((obj = ao2_t_iterator_next(&it, "test"))) {
+			ao2_t_unlink(c2, obj, "test");
+			ao2_t_link(c1, obj, "test");
 			ao2_t_ref(obj, -1, "test");
 		}
+		ao2_iterator_destroy(&it);
+		if (ao2_container_check(c1, 0)) {
+			ast_test_status_update(test, "container integrity check failed\n");
+			res = AST_TEST_FAIL;
+			goto cleanup;
+		}
+		if (ao2_container_check(c2, 0)) {
+			ast_test_status_update(test, "container integrity check failed\n");
+			res = AST_TEST_FAIL;
+			goto cleanup;
+		}
+
+		if (!cmp_fn) {
+			/* Completed testing with custom cmp_cb and default cmp_cb */
+			break;
+		}
 	}
-	it = ao2_iterator_init(c2, 0);
-	while ((obj = ao2_t_iterator_next(&it, "test"))) {
-		ao2_t_unlink(c2, obj, "test");
-		ao2_t_link(c1, obj, "test");
-		ao2_t_ref(obj, -1, "test");
-	}
-	ao2_iterator_destroy(&it);
 
 	/* Test Callback with no flags. */
 	increment = 0;
@@ -294,10 +622,10 @@ static int astobj2_test_helper(int use_hash, int use_cmp, unsigned int lim, stru
 		res = AST_TEST_FAIL;
 	}
 
-	/* Test OBJ_MULTIPLE with OBJ_UNLINK*/
+	/* Test OBJ_MULTIPLE with OBJ_UNLINK, add items back afterwards */
 	num = lim < 25 ? lim : 25;
 	if (!(mult_it = ao2_t_callback(c1, OBJ_MULTIPLE | OBJ_UNLINK, multiple_cb, &num, "test multiple"))) {
-		ast_test_status_update(test, "OBJ_MULTIPLE iwth OBJ_UNLINK test failed.\n");
+		ast_test_status_update(test, "OBJ_MULTIPLE with OBJ_UNLINK test failed.\n");
 		res = AST_TEST_FAIL;
 	} else {
 		/* make sure num items unlinked is as expected */
@@ -305,6 +633,11 @@ static int astobj2_test_helper(int use_hash, int use_cmp, unsigned int lim, stru
 			ast_test_status_update(test, "OBJ_MULTIPLE | OBJ_UNLINK test failed, did not unlink correct number of objects.\n");
 			res = AST_TEST_FAIL;
 		}
+		if (ao2_container_check(c1, 0)) {
+			ast_test_status_update(test, "container integrity check failed\n");
+			res = AST_TEST_FAIL;
+			goto cleanup;
+		}
 
 		/* link what was unlinked back into c1 */
 		while ((obj = ao2_t_iterator_next(mult_it, "test"))) {
@@ -312,10 +645,15 @@ static int astobj2_test_helper(int use_hash, int use_cmp, unsigned int lim, stru
 			ao2_t_ref(obj, -1, "test"); /* remove ref from iterator */
 		}
 		ao2_iterator_destroy(mult_it);
+		if (ao2_container_check(c1, 0)) {
+			ast_test_status_update(test, "container integrity check failed\n");
+			res = AST_TEST_FAIL;
+			goto cleanup;
+		}
 	}
 
-	/* Test OBJ_MULTIPLE without unlink, add items back afterwards */
-	num = lim < 25 ? lim : 25;
+	/* Test OBJ_MULTIPLE without unlink and iterate the returned container */
+	num = 5;
 	if (!(mult_it = ao2_t_callback(c1, OBJ_MULTIPLE, multiple_cb, &num, "test multiple"))) {
 		ast_test_status_update(test, "OBJ_MULTIPLE without OBJ_UNLINK test failed.\n");
 		res = AST_TEST_FAIL;
@@ -327,7 +665,7 @@ static int astobj2_test_helper(int use_hash, int use_cmp, unsigned int lim, stru
 	}
 
 	/* Test OBJ_MULTIPLE without unlink and no iterating */
-	num = lim < 5 ? lim : 5;
+	num = 5;
 	if (!(mult_it = ao2_t_callback(c1, OBJ_MULTIPLE, multiple_cb, &num, "test multiple"))) {
 		ast_test_status_update(test, "OBJ_MULTIPLE with no OBJ_UNLINK and no iterating failed.\n");
 		res = AST_TEST_FAIL;
@@ -343,11 +681,18 @@ static int astobj2_test_helper(int use_hash, int use_cmp, unsigned int lim, stru
 
 	/* Testing iterator.  Unlink a single object and break. do not add item back */
 	it = ao2_iterator_init(c1, 0);
-	num = (lim / 4) + 1;
+	num = ast_random() % lim; /* remove a random object */
+	if (!num) {
+		/*
+		 * Well we cannot remove object zero because of test with
+		 * all_but_one_cb later.
+		 */
+		num = 1;
+	}
 	while ((obj = ao2_t_iterator_next(&it, "test"))) {
 		if (obj->i == num) {
-			ao2_t_ref(obj, -1, "test");
 			ao2_t_unlink(c1, obj, "test");
+			ao2_t_ref(obj, -1, "test");
 			break;
 		}
 		ao2_t_ref(obj, -1, "test");
@@ -359,6 +704,11 @@ static int astobj2_test_helper(int use_hash, int use_cmp, unsigned int lim, stru
 		ast_test_status_update(test, "unlink during iterator failed. Number %d was not removed.\n", num);
 		res = AST_TEST_FAIL;
 	}
+	if (ao2_container_check(c1, 0)) {
+		ast_test_status_update(test, "container integrity check failed\n");
+		res = AST_TEST_FAIL;
+		goto cleanup;
+	}
 
 	/* Test unlink all with OBJ_MULTIPLE, leave a single object for the container to destroy */
 	ao2_t_callback(c1, OBJ_MULTIPLE | OBJ_UNLINK | OBJ_NODATA, all_but_one_cb, NULL, "test multiple");
@@ -367,6 +717,10 @@ static int astobj2_test_helper(int use_hash, int use_cmp, unsigned int lim, stru
 		ast_test_status_update(test, "OBJ_MULTIPLE | OBJ_UNLINK | OBJ_NODATA failed. destructor count %d\n", destructor_count);
 		res = AST_TEST_FAIL;
 	}
+	if (ao2_container_check(c1, 0)) {
+		ast_test_status_update(test, "container integrity check failed\n");
+		res = AST_TEST_FAIL;
+	}
 
 cleanup:
 	/* destroy containers */
@@ -376,9 +730,6 @@ cleanup:
 	if (c2) {
 		ao2_t_ref(c2, -1, "bye c2");
 	}
-	if (c3) {
-		ao2_t_ref(c3, -1, "bye c3");
-	}
 
 	if (destructor_count > 0) {
 		ast_test_status_update(test, "all destructors were not called, destructor count is %d\n", destructor_count);
@@ -399,7 +750,7 @@ AST_TEST_DEFINE(astobj2_test_1)
 	case TEST_INIT:
 		info->name = "astobj2_test1";
 		info->category = "/main/astobj2/";
-		info->summary = "astobj2 test using ao2 objects, containers, callbacks, and iterators";
+		info->summary = "Test ao2 objects, containers, callbacks, and iterators";
 		info->description =
 			"Builds ao2_containers with various item numbers, bucket sizes, cmp and hash "
 			"functions. Runs a series of tests to manipulate the container using callbacks "
@@ -409,28 +760,20 @@ AST_TEST_DEFINE(astobj2_test_1)
 		break;
 	}
 
-
-	/* Test 1, 500 items with custom hash and cmp functions */
-	ast_test_status_update(test, "Test 1, astobj2 test with 500 items.\n");
-	if ((res = astobj2_test_helper(1, 1, 500, test)) == AST_TEST_FAIL) {
+	/* Test number, container_type, use_sort, number of objects. */
+	if ((res = astobj2_test_1_helper(1, TEST_CONTAINER_LIST, 0, 50, test)) == AST_TEST_FAIL) {
 		return res;
 	}
 
-	/* Test 2, 1000 items with custom hash and default cmp functions */
-	ast_test_status_update(test, "Test 2, astobj2 test with 1000 items.\n");
-	if ((res = astobj2_test_helper(1, 0, 1000, test)) == AST_TEST_FAIL) {
+	if ((res = astobj2_test_1_helper(2, TEST_CONTAINER_LIST, 1, 50, test)) == AST_TEST_FAIL) {
 		return res;
 	}
 
-	/* Test 3, 10000 items with default hash and custom cmp functions */
-	ast_test_status_update(test, "Test 3, astobj2 test with 10000 items.\n");
-	if ((res = astobj2_test_helper(0, 1, 10000, test)) == AST_TEST_FAIL) {
+	if ((res = astobj2_test_1_helper(3, TEST_CONTAINER_HASH, 0, 1000, test)) == AST_TEST_FAIL) {
 		return res;
 	}
 
-	/* Test 4, 100000 items with default hash and cmp functions */
-	ast_test_status_update(test, "Test 4, astobj2 test with 100000 items.\n");
-	if ((res = astobj2_test_helper(0, 0, 100000, test)) == AST_TEST_FAIL) {
+	if ((res = astobj2_test_1_helper(4, TEST_CONTAINER_HASH, 1, 1000, test)) == AST_TEST_FAIL) {
 		return res;
 	}
 
@@ -484,6 +827,11 @@ AST_TEST_DEFINE(astobj2_test_2)
 			res = AST_TEST_FAIL;
 		}
 	}
+	if (ao2_container_check(c, 0)) {
+		ast_test_status_update(test, "container integrity check failed\n");
+		res = AST_TEST_FAIL;
+		goto cleanup;
+	}
 
 	/*
 	 * Iteration take 1.  Just make sure we see all NUM_OBJS objects.
@@ -698,11 +1046,929 @@ cleanup:
 	return res;
 }
 
+/*!
+ * \internal
+ * \brief Make a nonsorted container for astobj2 testing.
+ * \since 12.0.0
+ *
+ * \param type Container type to create.
+ * \param options Container options
+ *
+ * \retval container on success.
+ * \retval NULL on error.
+ */
+static struct ao2_container *test_make_nonsorted(enum test_container_type type, int options)
+{
+	struct ao2_container *container;
+
+	container = NULL;
+	switch (type) {
+	case TEST_CONTAINER_LIST:
+		container = ao2_container_alloc_list(AO2_ALLOC_OPT_LOCK_MUTEX, options,
+			NULL, test_cmp_cb);
+		break;
+	case TEST_CONTAINER_HASH:
+		container = ao2_container_alloc_hash(AO2_ALLOC_OPT_LOCK_MUTEX, options, 5,
+			test_hash_cb, NULL, test_cmp_cb);
+		break;
+	}
+
+	return container;
+}
+
+/*!
+ * \internal
+ * \brief Make a sorted container for astobj2 testing.
+ * \since 12.0.0
+ *
+ * \param type Container type to create.
+ * \param options Container options
+ *
+ * \retval container on success.
+ * \retval NULL on error.
+ */
+static struct ao2_container *test_make_sorted(enum test_container_type type, int options)
+{
+	struct ao2_container *container;
+
+	container = NULL;
+	switch (type) {
+	case TEST_CONTAINER_LIST:
+		container = ao2_t_container_alloc_list(AO2_ALLOC_OPT_LOCK_MUTEX, options,
+			test_sort_cb, test_cmp_cb, "test");
+		break;
+	case TEST_CONTAINER_HASH:
+		container = ao2_t_container_alloc_hash(AO2_ALLOC_OPT_LOCK_MUTEX, options, 5,
+			test_hash_cb, test_sort_cb, test_cmp_cb, "test");
+		break;
+	}
+
+	return container;
+}
+
+/*!
+ * \internal
+ * \brief Insert the given test vector into the given container.
+ * \since 12.0.0
+ *
+ * \note The given test vector must not have any duplicates.
+ *
+ * \param container Container to insert the test vector.
+ * \param destroy_counter What to increment when the object is destroyed.
+ * \param vector Test vector to insert.
+ * \param count Number of objects in the vector.
+ * \param prefix Test output prefix string.
+ * \param test Test output controller.
+ *
+ * \retval 0 on success.
+ * \retval -1 on error.
+ */
+static int insert_test_vector(struct ao2_container *container, int *destroy_counter, const int *vector, int count, const char *prefix, struct ast_test *test)
+{
+	int idx;
+	struct test_obj *obj;
+
+	for (idx = 0; idx < count; ++idx) {
+		obj = ao2_alloc(sizeof(struct test_obj), test_obj_destructor);
+		if (!obj) {
+			ast_test_status_update(test, "%s: ao2_alloc failed.\n", prefix);
+			return -1;
+		}
+		if (destroy_counter) {
+			/* This object ultimately needs to be destroyed. */
+			++*destroy_counter;
+		}
+		obj->destructor_count = destroy_counter;
+		obj->i = vector[idx];
+		ao2_link(container, obj);
+		ao2_t_ref(obj, -1, "test");
+
+		if (ao2_container_count(container) != idx + 1) {
+			ast_test_status_update(test,
+				"%s: Unexpected container count.  Expected:%d Got:%d\n",
+				prefix, idx + 1, ao2_container_count(container));
+			return -1;
+		}
+	}
+	if (ao2_container_check(container, 0)) {
+		ast_test_status_update(test, "%s: Container integrity check failed\n", prefix);
+		return -1;
+	}
+
+	return 0;
+}
+
+/*!
+ * \internal
+ * \brief Insert duplicates of number into the given container.
+ * \since 12.0.0
+ *
+ * \note The given container must not already have the number in it.
+ *
+ * \param container Container to insert the duplicates.
+ * \param destroy_counter What to increment when the object is destroyed.
+ * \param number Number of object to duplicate.
+ * \param prefix Test output prefix string.
+ * \param test Test output controller.
+ *
+ * \retval 0 on success.
+ * \retval -1 on error.
+ */
+static int insert_test_duplicates(struct ao2_container *container, int *destroy_counter, int number, const char *prefix, struct ast_test *test)
+{
+	int count;
+	struct test_obj *obj;
+	struct test_obj *obj_dup;
+
+	/* Check if object already exists in the container. */
+	obj = ao2_find(container, &number, OBJ_KEY);
+	if (obj) {
+		ast_test_status_update(test, "%s: Object %d already exists.\n", prefix, number);
+		ao2_t_ref(obj, -1, "test");
+		return -1;
+	}
+
+	/* Add three duplicate keyed objects. */
+	obj_dup = NULL;
+	for (count = 0; count < 4; ++count) {
+		obj = ao2_alloc(sizeof(struct test_obj), test_obj_destructor);
+		if (!obj) {
+			ast_test_status_update(test, "%s: ao2_alloc failed.\n", prefix);
+			if (obj_dup) {
+				ao2_t_ref(obj_dup, -1, "test");
+			}
+			return -1;
+		}
+		if (destroy_counter) {
+			/* This object ultimately needs to be destroyed. */
+			++*destroy_counter;
+		}
+		obj->destructor_count = destroy_counter;
+		obj->i = number;
+		obj->dup_number = count;
+		ao2_link(container, obj);
+
+		if (count == 2) {
+			/* Duplicate this object. */
+			obj_dup = obj;
+		} else {
+			ao2_t_ref(obj, -1, "test");
+		}
+	}
+
+	/* Add the duplicate object. */
+	ao2_link(container, obj_dup);
+	ao2_t_ref(obj_dup, -1, "test");
+
+	if (ao2_container_check(container, 0)) {
+		ast_test_status_update(test, "%s: Container integrity check failed\n", prefix);
+		return -1;
+	}
+
+	return 0;
+}
+
+/*!
+ * \internal
+ * \brief Iterate over the container and compare the objects with the given vector.
+ * \since 12.0.0
+ *
+ * \param res Passed in enum ast_test_result_state.
+ * \param container Container to iterate.
+ * \param flags Flags controlling the iteration.
+ * \param vector Expected vector to find.
+ * \param count Number of objects in the vector.
+ * \param prefix Test output prefix string.
+ * \param test Test output controller.
+ *
+ * \return enum ast_test_result_state
+ */
+static int test_ao2_iteration(int res, struct ao2_container *container,
+	enum ao2_iterator_flags flags,
+	const int *vector, int count, const char *prefix, struct ast_test *test)
+{
+	struct ao2_iterator iter;
+	struct test_obj *obj;
+	int idx;
+
+	if (ao2_container_count(container) != count) {
+		ast_test_status_update(test, "%s: Container count doesn't match vector count.\n",
+			prefix);
+		res = AST_TEST_FAIL;
+	}
+
+	iter = ao2_iterator_init(container, flags);
+
+	/* Check iterated objects against the given vector. */
+	for (idx = 0; idx < count; ++idx) {
+		obj = ao2_iterator_next(&iter);
+		if (!obj) {
+			ast_test_status_update(test, "%s: Too few objects found.\n", prefix);
+			res = AST_TEST_FAIL;
+			break;
+		}
+		if (vector[idx] != obj->i) {
+			ast_test_status_update(test, "%s: Object %d != vector[%d] %d.\n",
+				prefix, obj->i, idx, vector[idx]);
+			res = AST_TEST_FAIL;
+		}
+		ao2_ref(obj, -1); /* remove ref from iterator */
+	}
+	obj = ao2_iterator_next(&iter);
+	if (obj) {
+		ast_test_status_update(test, "%s: Too many objects found.  Object %d\n",
+			prefix, obj->i);
+		ao2_ref(obj, -1); /* remove ref from iterator */
+		res = AST_TEST_FAIL;
+	}
+
+	ao2_iterator_destroy(&iter);
+
+	return res;
+}
+
+/*!
+ * \internal
+ * \brief Run an ao2_callback() and compare the returned vector with the given vector.
+ * \since 12.0.0
+ *
+ * \param res Passed in enum ast_test_result_state.
+ * \param container Container to traverse.
+ * \param flags Callback flags controlling the traversal.
+ * \param cmp_fn Compare function to select objects.
+ * \param arg Optional argument.
+ * \param vector Expected vector to find.
+ * \param count Number of objects in the vector.
+ * \param prefix Test output prefix string.
+ * \param test Test output controller.
+ *
+ * \return enum ast_test_result_state
+ */
+static int test_ao2_callback_traversal(int res, struct ao2_container *container,
+	enum search_flags flags, ao2_callback_fn *cmp_fn, void *arg,
+	const int *vector, int count, const char *prefix, struct ast_test *test)
+{
+	struct ao2_iterator *mult_iter;
+	struct test_obj *obj;
+	int idx;
+
+	mult_iter = ao2_callback(container, flags | OBJ_MULTIPLE, cmp_fn, arg);
+	if (!mult_iter) {
+		ast_test_status_update(test, "%s: Did not return iterator.\n", prefix);
+		return AST_TEST_FAIL;
+	}
+
+	/* Check matching objects against the given vector. */
+	for (idx = 0; idx < count; ++idx) {
+		obj = ao2_iterator_next(mult_iter);
+		if (!obj) {
+			ast_test_status_update(test, "%s: Too few objects found.\n", prefix);
+			res = AST_TEST_FAIL;
+			break;
+		}
+		if (vector[idx] != obj->i) {
+			ast_test_status_update(test, "%s: Object %d != vector[%d] %d.\n",
+				prefix, obj->i, idx, vector[idx]);
+			res = AST_TEST_FAIL;
+		}
+		ao2_ref(obj, -1); /* remove ref from iterator */
+	}
+	obj = ao2_iterator_next(mult_iter);
+	if (obj) {
+		ast_test_status_update(test, "%s: Too many objects found.  Object %d\n",
+			prefix, obj->i);
+		ao2_ref(obj, -1); /* remove ref from iterator */
+		res = AST_TEST_FAIL;
+	}
+	ao2_iterator_destroy(mult_iter);
+
+	return res;
+}
+
+/*!
+ * \internal
+ * \brief Run an ao2_find() for duplicates and compare the returned vector with the given vector.
+ * \since 12.0.0
+ *
+ * \param res Passed in enum ast_test_result_state.
+ * \param container Container to traverse.
+ * \param flags Callback flags controlling the traversal.
+ * \param number Number of object to find all duplicates.
+ * \param vector Expected vector to find.
+ * \param count Number of objects in the vector.
+ * \param prefix Test output prefix string.
+ * \param test Test output controller.
+ *
+ * \return enum ast_test_result_state
+ */
+static int test_expected_duplicates(int res, struct ao2_container *container,
+	enum search_flags flags, int number,
+	const int *vector, int count, const char *prefix, struct ast_test *test)
+{
+	struct ao2_iterator *mult_iter;
+	struct test_obj *obj;
+	int idx;
+
+	mult_iter = ao2_find(container, &number, flags | OBJ_MULTIPLE | OBJ_KEY);
+	if (!mult_iter) {
+		ast_test_status_update(test, "%s: Did not return iterator.\n", prefix);
+		return AST_TEST_FAIL;
+	}
+
+	/* Check matching objects against the given vector. */
+	for (idx = 0; idx < count; ++idx) {
+		obj = ao2_iterator_next(mult_iter);
+		if (!obj) {
+			ast_test_status_update(test, "%s: Too few objects found.\n", prefix);
+			res = AST_TEST_FAIL;
+			break;
+		}
+		if (number != obj->i) {
+			ast_test_status_update(test, "%s: Object %d != %d.\n",
+				prefix, obj->i, number);
+			res = AST_TEST_FAIL;
+		}
+		if (vector[idx] != obj->dup_number) {
+			ast_test_status_update(test, "%s: Object dup id %d != vector[%d] %d.\n",
+				prefix, obj->dup_number, idx, vector[idx]);
+			res = AST_TEST_FAIL;
+		}
+		ao2_ref(obj, -1); /* remove ref from iterator */
+	}
+	obj = ao2_iterator_next(mult_iter);
+	if (obj) {
+		ast_test_status_update(test,
+			"%s: Too many objects found.  Object %d, dup id %d\n",
+			prefix, obj->i, obj->dup_number);
+		ao2_ref(obj, -1); /* remove ref from iterator */
+		res = AST_TEST_FAIL;
+	}
+	ao2_iterator_destroy(mult_iter);
+
+	return res;
+}
+
+/*!
+ * \internal
+ * \brief Test nonsorted container traversal.
+ * \since 12.0.0
+ *
+ * \param res Passed in enum ast_test_result_state.
+ * \param tst_num Test number.
+ * \param type Container type to test.
+ * \param test Test output controller.
+ *
+ * \return enum ast_test_result_state
+ */
+static int test_traversal_nonsorted(int res, int tst_num, enum test_container_type type, struct ast_test *test)
+{
+	struct ao2_container *c1;
+	struct ao2_container *c2 = NULL;
+	int partial;
+	int destructor_count = 0;
+
+	/*! Container object insertion vector. */
+	static const int test_initial[] = {
+		1, 0, 2, 6, 4, 7, 5, 3, 9, 8
+	};
+
+	/*! Container object insertion vector reversed. */
+	static const int test_reverse[] = {
+		8, 9, 3, 5, 7, 4, 6, 2, 0, 1
+	};
+	static const int test_list_partial_forward[] = {
+		6, 7, 5
+	};
+	static const int test_list_partial_backward[] = {
+		5, 7, 6
+	};
+
+	/* The hash orders assume that there are 5 buckets. */
+	static const int test_hash_end_forward[] = {
+		0, 5, 1, 6, 2, 7, 3, 8, 4, 9
+	};
+	static const int test_hash_end_backward[] = {
+		9, 4, 8, 3, 7, 2, 6, 1, 5, 0
+	};
+	static const int test_hash_begin_forward[] = {
+		5, 0, 6, 1, 7, 2, 8, 3, 9, 4
+	};
+	static const int test_hash_begin_backward[] = {
+		4, 9, 3, 8, 2, 7, 1, 6, 0, 5
+	};
+	static const int test_hash_partial_forward[] = {
+		5, 6, 7
+	};
+	static const int test_hash_partial_backward[] = {
+		7, 6, 5
+	};
+
+	ast_test_status_update(test, "Test %d, %s containers.\n",
+		tst_num, test_container2str(type));
+
+	/* Create container that inserts objects at the end. */
+	c1 = test_make_nonsorted(type, 0);
+	if (!c1) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_vector(c1, &destructor_count, test_initial, ARRAY_LEN(test_initial), "c1", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+
+	/* Create container that inserts objects at the beginning. */
+	c2 = test_make_nonsorted(type, AO2_CONTAINER_ALLOC_OPT_INSERT_BEGIN);
+	if (!c2) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_vector(c2, &destructor_count, test_initial, ARRAY_LEN(test_initial), "c2", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+
+	/* Check container iteration directions */
+	switch (type) {
+	case TEST_CONTAINER_LIST:
+		res = test_ao2_iteration(res, c1, 0,
+			test_initial, ARRAY_LEN(test_initial),
+			"Iteration (ascending, insert end)", test);
+		res = test_ao2_iteration(res, c1, AO2_ITERATOR_DESCENDING,
+			test_reverse, ARRAY_LEN(test_reverse),
+			"Iteration (descending, insert end)", test);
+
+		res = test_ao2_iteration(res, c2, 0,
+			test_reverse, ARRAY_LEN(test_reverse),
+			"Iteration (ascending, insert begin)", test);
+		res = test_ao2_iteration(res, c2, AO2_ITERATOR_DESCENDING,
+			test_initial, ARRAY_LEN(test_initial),
+			"Iteration (descending, insert begin)", test);
+		break;
+	case TEST_CONTAINER_HASH:
+		res = test_ao2_iteration(res, c1, 0,
+			test_hash_end_forward, ARRAY_LEN(test_hash_end_forward),
+			"Iteration (ascending, insert end)", test);
+		res = test_ao2_iteration(res, c1, AO2_ITERATOR_DESCENDING,
+			test_hash_end_backward, ARRAY_LEN(test_hash_end_backward),
+			"Iteration (descending, insert end)", test);
+
+		res = test_ao2_iteration(res, c2, 0,
+			test_hash_begin_forward, ARRAY_LEN(test_hash_begin_forward),
+			"Iteration (ascending, insert begin)", test);
+		res = test_ao2_iteration(res, c2, AO2_ITERATOR_DESCENDING,
+			test_hash_begin_backward, ARRAY_LEN(test_hash_begin_backward),
+			"Iteration (descending, insert begin)", test);
+		break;
+	}
+
+	/* Check container traversal directions */
+	switch (type) {
+	case TEST_CONTAINER_LIST:
+		res = test_ao2_callback_traversal(res, c1, OBJ_ORDER_ASCENDING, NULL, NULL,
+			test_initial, ARRAY_LEN(test_initial),
+			"Traversal (ascending, insert end)", test);
+		res = test_ao2_callback_traversal(res, c1, OBJ_ORDER_DESCENDING, NULL, NULL,
+			test_reverse, ARRAY_LEN(test_reverse),
+			"Traversal (descending, insert end)", test);
+
+		res = test_ao2_callback_traversal(res, c2, OBJ_ORDER_ASCENDING, NULL, NULL,
+			test_reverse, ARRAY_LEN(test_reverse),
+			"Traversal (ascending, insert begin)", test);
+		res = test_ao2_callback_traversal(res, c2, OBJ_ORDER_DESCENDING, NULL, NULL,
+			test_initial, ARRAY_LEN(test_initial),
+			"Traversal (descending, insert begin)", test);
+		break;
+	case TEST_CONTAINER_HASH:
+		res = test_ao2_callback_traversal(res, c1, OBJ_ORDER_ASCENDING, NULL, NULL,
+			test_hash_end_forward, ARRAY_LEN(test_hash_end_forward),
+			"Traversal (ascending, insert end)", test);
+		res = test_ao2_callback_traversal(res, c1, OBJ_ORDER_DESCENDING, NULL, NULL,
+			test_hash_end_backward, ARRAY_LEN(test_hash_end_backward),
+			"Traversal (descending, insert end)", test);
+
+		res = test_ao2_callback_traversal(res, c2, OBJ_ORDER_ASCENDING, NULL, NULL,
+			test_hash_begin_forward, ARRAY_LEN(test_hash_begin_forward),
+			"Traversal (ascending, insert begin)", test);
+		res = test_ao2_callback_traversal(res, c2, OBJ_ORDER_DESCENDING, NULL, NULL,
+			test_hash_begin_backward, ARRAY_LEN(test_hash_begin_backward),
+			"Traversal (descending, insert begin)", test);
+		break;
+	}
+
+	/* Check traversal with OBJ_PARTIAL_KEY search range. */
+	partial = 6;
+	partial_key_match_range = 1;
+	switch (type) {
+	case TEST_CONTAINER_LIST:
+		res = test_ao2_callback_traversal(res, c1, OBJ_PARTIAL_KEY | OBJ_ORDER_ASCENDING,
+			test_cmp_cb, &partial,
+			test_list_partial_forward, ARRAY_LEN(test_list_partial_forward),
+			"Traversal OBJ_PARTIAL_KEY (ascending)", test);
+		res = test_ao2_callback_traversal(res, c1, OBJ_PARTIAL_KEY | OBJ_ORDER_DESCENDING,
+			test_cmp_cb, &partial,
+			test_list_partial_backward, ARRAY_LEN(test_list_partial_backward),
+			"Traversal OBJ_PARTIAL_KEY (descending)", test);
+		break;
+	case TEST_CONTAINER_HASH:
+		res = test_ao2_callback_traversal(res, c1, OBJ_PARTIAL_KEY | OBJ_ORDER_ASCENDING,
+			test_cmp_cb, &partial,
+			test_hash_partial_forward, ARRAY_LEN(test_hash_partial_forward),
+			"Traversal OBJ_PARTIAL_KEY (ascending)", test);
+		res = test_ao2_callback_traversal(res, c1, OBJ_PARTIAL_KEY | OBJ_ORDER_DESCENDING,
+			test_cmp_cb, &partial,
+			test_hash_partial_backward, ARRAY_LEN(test_hash_partial_backward),
+			"Traversal OBJ_PARTIAL_KEY (descending)", test);
+		break;
+	}
+
+test_cleanup:
+	/* destroy containers */
+	if (c1) {
+		ao2_t_ref(c1, -1, "bye c1");
+	}
+	if (c2) {
+		ao2_t_ref(c2, -1, "bye c2");
+	}
+
+	if (destructor_count > 0) {
+		ast_test_status_update(test,
+			"all destructors were not called, destructor count is %d\n",
+			destructor_count);
+		res = AST_TEST_FAIL;
+	} else if (destructor_count < 0) {
+		ast_test_status_update(test,
+			"Destructor was called too many times, destructor count is %d\n",
+			destructor_count);
+		res = AST_TEST_FAIL;
+	}
+
+	return res;
+}
+
+/*!
+ * \internal
+ * \brief Test sorted container traversal.
+ * \since 12.0.0
+ *
+ * \param res Passed in enum ast_test_result_state.
+ * \param tst_num Test number.
+ * \param type Container type to test.
+ * \param test Test output controller.
+ *
+ * \return enum ast_test_result_state
+ */
+static int test_traversal_sorted(int res, int tst_num, enum test_container_type type, struct ast_test *test)
+{
+	struct ao2_container *c1;
+	struct ao2_container *c2 = NULL;
+	int partial;
+	int destructor_count = 0;
+	int duplicate_number = 100;
+
+	/*! Container object insertion vector. */
+	static const int test_initial[] = {
+		1, 0, 2, 6, 4, 7, 5, 3, 9, 8
+	};
+
+	/*! Container forward traversal/iteration. */
+	static const int test_forward[] = {
+		0, 1, 2, 3, 4, 5, 6, 7, 8, 9
+	};
+	/*! Container backward traversal/iteration. */
+	static const int test_backward[] = {
+		9, 8, 7, 6, 5, 4, 3, 2, 1, 0
+	};
+
+	static const int test_partial_forward[] = {
+		5, 6, 7
+	};
+	static const int test_partial_backward[] = {
+		7, 6, 5
+	};
+
+	/* The hash orders assume that there are 5 buckets. */
+	static const int test_hash_forward[] = {
+		0, 5, 1, 6, 2, 7, 3, 8, 4, 9
+	};
+	static const int test_hash_backward[] = {
+		9, 4, 8, 3, 7, 2, 6, 1, 5, 0
+	};
+	static const int test_hash_partial_forward[] = {
+		5, 6, 7
+	};
+	static const int test_hash_partial_backward[] = {
+		7, 6, 5
+	};
+
+	/* Duplicate identifier order */
+	static const int test_dup_allow_forward[] = {
+		0, 1, 2, 3, 2
+	};
+	static const int test_dup_allow_backward[] = {
+		2, 3, 2, 1, 0
+	};
+	static const int test_dup_reject[] = {
+		0
+	};
+	static const int test_dup_obj_reject_forward[] = {
+		0, 1, 2, 3
+	};
+	static const int test_dup_obj_reject_backward[] = {
+		3, 2, 1, 0
+	};
+	static const int test_dup_replace[] = {
+		2
+	};
+
+	ast_test_status_update(test, "Test %d, %s containers.\n",
+		tst_num, test_container2str(type));
+
+	/* Create container that inserts duplicate objects after matching objects. */
+	c1 = test_make_sorted(type, AO2_CONTAINER_ALLOC_OPT_DUPS_ALLOW);
+	if (!c1) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_vector(c1, &destructor_count, test_initial, ARRAY_LEN(test_initial), "c1(DUPS_ALLOW)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+
+	/* Create container that inserts duplicate objects before matching objects. */
+	c2 = test_make_sorted(type, AO2_CONTAINER_ALLOC_OPT_INSERT_BEGIN | AO2_CONTAINER_ALLOC_OPT_DUPS_ALLOW);
+	if (!c2) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_vector(c2, &destructor_count, test_initial, ARRAY_LEN(test_initial), "c2(DUPS_ALLOW)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+
+	/* Check container iteration directions */
+	switch (type) {
+	case TEST_CONTAINER_LIST:
+		res = test_ao2_iteration(res, c1, 0,
+			test_forward, ARRAY_LEN(test_forward),
+			"Iteration (ascending)", test);
+		res = test_ao2_iteration(res, c1, AO2_ITERATOR_DESCENDING,
+			test_backward, ARRAY_LEN(test_backward),
+			"Iteration (descending)", test);
+		break;
+	case TEST_CONTAINER_HASH:
+		res = test_ao2_iteration(res, c1, 0,
+			test_hash_forward, ARRAY_LEN(test_hash_forward),
+			"Iteration (ascending)", test);
+		res = test_ao2_iteration(res, c1, AO2_ITERATOR_DESCENDING,
+			test_hash_backward, ARRAY_LEN(test_hash_backward),
+			"Iteration (descending)", test);
+		break;
+	}
+
+	/* Check container traversal directions */
+	switch (type) {
+	case TEST_CONTAINER_LIST:
+		res = test_ao2_callback_traversal(res, c1, OBJ_ORDER_ASCENDING, NULL, NULL,
+			test_forward, ARRAY_LEN(test_forward),
+			"Traversal (ascending)", test);
+		res = test_ao2_callback_traversal(res, c1, OBJ_ORDER_DESCENDING, NULL, NULL,
+			test_backward, ARRAY_LEN(test_backward),
+			"Traversal (descending)", test);
+		break;
+	case TEST_CONTAINER_HASH:
+		res = test_ao2_callback_traversal(res, c1, OBJ_ORDER_ASCENDING, NULL, NULL,
+			test_hash_forward, ARRAY_LEN(test_hash_forward),
+			"Traversal (ascending, insert end)", test);
+		res = test_ao2_callback_traversal(res, c1, OBJ_ORDER_DESCENDING, NULL, NULL,
+			test_hash_backward, ARRAY_LEN(test_hash_backward),
+			"Traversal (descending)", test);
+		break;
+	}
+
+	/* Check traversal with OBJ_PARTIAL_KEY search range. */
+	partial = 6;
+	partial_key_match_range = 1;
+	switch (type) {
+	case TEST_CONTAINER_LIST:
+		res = test_ao2_callback_traversal(res, c1, OBJ_PARTIAL_KEY | OBJ_ORDER_ASCENDING,
+			test_cmp_cb, &partial,
+			test_partial_forward, ARRAY_LEN(test_partial_forward),
+			"Traversal OBJ_PARTIAL_KEY (ascending)", test);
+		res = test_ao2_callback_traversal(res, c1, OBJ_PARTIAL_KEY | OBJ_ORDER_DESCENDING,
+			test_cmp_cb, &partial,
+			test_partial_backward, ARRAY_LEN(test_partial_backward),
+			"Traversal OBJ_PARTIAL_KEY (descending)", test);
+		break;
+	case TEST_CONTAINER_HASH:
+		res = test_ao2_callback_traversal(res, c1, OBJ_PARTIAL_KEY | OBJ_ORDER_ASCENDING,
+			test_cmp_cb, &partial,
+			test_hash_partial_forward, ARRAY_LEN(test_hash_partial_forward),
+			"Traversal OBJ_PARTIAL_KEY (ascending)", test);
+		res = test_ao2_callback_traversal(res, c1, OBJ_PARTIAL_KEY | OBJ_ORDER_DESCENDING,
+			test_cmp_cb, &partial,
+			test_hash_partial_backward, ARRAY_LEN(test_hash_partial_backward),
+			"Traversal OBJ_PARTIAL_KEY (descending)", test);
+		break;
+	}
+
+	/* Add duplicates to initial containers that allow duplicates */
+	if (insert_test_duplicates(c1, &destructor_count, duplicate_number, "c1(DUPS_ALLOW)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_duplicates(c2, &destructor_count, duplicate_number, "c2(DUPS_ALLOW)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+
+	/* Check duplicates in containers that allow duplicates. */
+	res = test_expected_duplicates(res, c1, OBJ_ORDER_ASCENDING, duplicate_number,
+		test_dup_allow_forward, ARRAY_LEN(test_dup_allow_forward),
+		"Duplicates (ascending, DUPS_ALLOW)", test);
+	res = test_expected_duplicates(res, c1, OBJ_ORDER_DESCENDING, duplicate_number,
+		test_dup_allow_backward, ARRAY_LEN(test_dup_allow_backward),
+		"Duplicates (descending, DUPS_ALLOW)", test);
+
+	ao2_t_ref(c1, -1, "bye c1");
+	c1 = NULL;
+	ao2_t_ref(c2, -1, "bye c2");
+	c2 = NULL;
+
+	/* Create containers that reject duplicate keyed objects. */
+	c1 = test_make_sorted(type, AO2_CONTAINER_ALLOC_OPT_DUPS_REJECT);
+	if (!c1) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_vector(c1, &destructor_count, test_initial, ARRAY_LEN(test_initial), "c1(DUPS_REJECT)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_duplicates(c1, &destructor_count, duplicate_number, "c1(DUPS_REJECT)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	c2 = test_make_sorted(type, AO2_CONTAINER_ALLOC_OPT_INSERT_BEGIN | AO2_CONTAINER_ALLOC_OPT_DUPS_REJECT);
+	if (!c2) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_vector(c2, &destructor_count, test_initial, ARRAY_LEN(test_initial), "c2(DUPS_REJECT)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_duplicates(c2, &destructor_count, duplicate_number, "c2(DUPS_REJECT)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+
+	/* Check duplicates in containers that reject duplicate keyed objects. */
+	res = test_expected_duplicates(res, c1, OBJ_ORDER_ASCENDING, duplicate_number,
+		test_dup_reject, ARRAY_LEN(test_dup_reject),
+		"Duplicates (ascending, DUPS_REJECT)", test);
+	res = test_expected_duplicates(res, c1, OBJ_ORDER_DESCENDING, duplicate_number,
+		test_dup_reject, ARRAY_LEN(test_dup_reject),
+		"Duplicates (descending, DUPS_REJECT)", test);
+
+	ao2_t_ref(c1, -1, "bye c1");
+	c1 = NULL;
+	ao2_t_ref(c2, -1, "bye c2");
+	c2 = NULL;
+
+	/* Create containers that reject duplicate objects. */
+	c1 = test_make_sorted(type, AO2_CONTAINER_ALLOC_OPT_DUPS_OBJ_REJECT);
+	if (!c1) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_vector(c1, &destructor_count, test_initial, ARRAY_LEN(test_initial), "c1(DUPS_OBJ_REJECT)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_duplicates(c1, &destructor_count, duplicate_number, "c1(DUPS_OBJ_REJECT)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	c2 = test_make_sorted(type, AO2_CONTAINER_ALLOC_OPT_INSERT_BEGIN | AO2_CONTAINER_ALLOC_OPT_DUPS_OBJ_REJECT);
+	if (!c2) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_vector(c2, &destructor_count, test_initial, ARRAY_LEN(test_initial), "c2(DUPS_OBJ_REJECT)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_duplicates(c2, &destructor_count, duplicate_number, "c2(DUPS_OBJ_REJECT)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+
+	/* Check duplicates in containers that reject duplicate objects. */
+	res = test_expected_duplicates(res, c1, OBJ_ORDER_ASCENDING, duplicate_number,
+		test_dup_obj_reject_forward, ARRAY_LEN(test_dup_obj_reject_forward),
+		"Duplicates (ascending, DUPS_OBJ_REJECT)", test);
+	res = test_expected_duplicates(res, c1, OBJ_ORDER_DESCENDING, duplicate_number,
+		test_dup_obj_reject_backward, ARRAY_LEN(test_dup_obj_reject_backward),
+		"Duplicates (descending, DUPS_OBJ_REJECT)", test);
+
+	ao2_t_ref(c1, -1, "bye c1");
+	c1 = NULL;
+	ao2_t_ref(c2, -1, "bye c2");
+	c2 = NULL;
+
+	/* Create container that replaces duplicate keyed objects. */
+	c1 = test_make_sorted(type, AO2_CONTAINER_ALLOC_OPT_DUPS_REPLACE);
+	if (!c1) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_vector(c1, &destructor_count, test_initial, ARRAY_LEN(test_initial), "c1(DUPS_REJECT)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_duplicates(c1, &destructor_count, duplicate_number, "c1(DUPS_REJECT)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	c2 = test_make_sorted(type, AO2_CONTAINER_ALLOC_OPT_INSERT_BEGIN | AO2_CONTAINER_ALLOC_OPT_DUPS_REPLACE);
+	if (!c2) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_vector(c2, &destructor_count, test_initial, ARRAY_LEN(test_initial), "c2(DUPS_REPLACE)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+	if (insert_test_duplicates(c2, &destructor_count, duplicate_number, "c2(DUPS_REPLACE)", test)) {
+		res = AST_TEST_FAIL;
+		goto test_cleanup;
+	}
+
+	/* Check duplicates in containers that replaces duplicate keyed objects. */
+	res = test_expected_duplicates(res, c1, OBJ_ORDER_ASCENDING, duplicate_number,
+		test_dup_replace, ARRAY_LEN(test_dup_replace),
+		"Duplicates (ascending, DUPS_REPLACE)", test);
+	res = test_expected_duplicates(res, c1, OBJ_ORDER_DESCENDING, duplicate_number,
+		test_dup_replace, ARRAY_LEN(test_dup_replace),
+		"Duplicates (descending, DUPS_REPLACE)", test);
+
+test_cleanup:
+	/* destroy containers */
+	if (c1) {
+		ao2_t_ref(c1, -1, "bye c1");
+	}
+	if (c2) {
+		ao2_t_ref(c2, -1, "bye c2");
+	}
+
+	if (destructor_count > 0) {
+		ast_test_status_update(test,
+			"all destructors were not called, destructor count is %d\n",
+			destructor_count);
+		res = AST_TEST_FAIL;
+	} else if (destructor_count < 0) {
+		ast_test_status_update(test,
+			"Destructor was called too many times, destructor count is %d\n",
+			destructor_count);
+		res = AST_TEST_FAIL;
+	}
+
+	return res;
+}
+
+AST_TEST_DEFINE(astobj2_test_4)
+{
+	int res = AST_TEST_PASS;
+
+	switch (cmd) {
+	case TEST_INIT:
+		info->name = "astobj2_test4";
+		info->category = "/main/astobj2/";
+		info->summary = "Test container traversal/iteration";
+		info->description =
+			"This test is to see if the container traversal/iteration works "
+			"as intended for each supported container type.";
+		return AST_TEST_NOT_RUN;
+	case TEST_EXECUTE:
+		break;
+	}
+
+	res = test_traversal_nonsorted(res, 1, TEST_CONTAINER_LIST, test);
+	res = test_traversal_nonsorted(res, 2, TEST_CONTAINER_HASH, test);
+
+	res = test_traversal_sorted(res, 3, TEST_CONTAINER_LIST, test);
+	res = test_traversal_sorted(res, 4, TEST_CONTAINER_HASH, test);
+
+	return res;
+}
+
 static int unload_module(void)
 {
 	AST_TEST_UNREGISTER(astobj2_test_1);
 	AST_TEST_UNREGISTER(astobj2_test_2);
 	AST_TEST_UNREGISTER(astobj2_test_3);
+	AST_TEST_UNREGISTER(astobj2_test_4);
 	return 0;
 }
 
@@ -711,6 +1977,7 @@ static int load_module(void)
 	AST_TEST_REGISTER(astobj2_test_1);
 	AST_TEST_REGISTER(astobj2_test_2);
 	AST_TEST_REGISTER(astobj2_test_3);
+	AST_TEST_REGISTER(astobj2_test_4);
 	return AST_MODULE_LOAD_SUCCESS;
 }