A new feature thanks to the fine folks at Switchvox!

If a deadlock is detected, then the typical lock information will be
printed along with a backtrace of the stack for the offending threads.
Use of this requires compiling with DETECT_DEADLOCKS and having glibc
installed.

Furthermore, issuing the "core show locks" CLI command will print the
normal lock information as well as a backtraces for each lock. This
requires that DEBUG_THREADS is enabled and that glibc is installed.

All the backtrace features may be disabled by running the configure
script with --without-execinfo as an argument



git-svn-id: https://origsvn.digium.com/svn/asterisk/trunk@118173 65c4cc65-6c06-0410-ace0-fbb531ad65f3

2 files changed, 99 insertions, 16 deletions

diff --git a/main/logger.c b/main/logger.c
index a4936badd..d20ac8bc5 100644
--- a/main/logger.c
+++ b/main/logger.c
@@ -1127,30 +1127,67 @@ void ast_log(int level, const char *file, int line, const char *function, const
 	return;
 }
 
+#ifdef HAVE_BKTR
+
+struct ast_bt *ast_bt_create(void) 
+{
+	struct ast_bt *bt = ast_calloc(1, sizeof(*bt));
+	if (!bt) {
+		ast_log(LOG_ERROR, "Unable to allocate memory for backtrace structure!\n");
+		return NULL;
+	}
+
+	bt->alloced = 1;
+
+	ast_bt_get_addresses(bt);
+
+	return bt;
+}
+
+int ast_bt_get_addresses(struct ast_bt *bt)
+{
+	bt->num_frames = backtrace(bt->addresses, AST_MAX_BT_FRAMES);
+
+	return 0;
+}
+
+void *ast_bt_destroy(struct ast_bt *bt)
+{
+	if (bt->alloced) {
+		ast_free(bt);
+	}
+
+	return NULL;
+}
+
+#endif /* HAVE_BKTR */
+
 void ast_backtrace(void)
 {
 #ifdef HAVE_BKTR
-	int count = 0, i = 0;
-	void **addresses;
+	struct ast_bt *backtrace;
+	int i = 0;
 	char **strings;
 
-	if ((addresses = ast_calloc(MAX_BACKTRACE_FRAMES, sizeof(*addresses)))) {
-		count = backtrace(addresses, MAX_BACKTRACE_FRAMES);
-		if ((strings = backtrace_symbols(addresses, count))) {
-			ast_debug(1, "Got %d backtrace record%c\n", count, count != 1 ? 's' : ' ');
-			for (i = 0; i < count; i++) {
+	if (!(backtrace = ast_bt_create())) {
+		ast_log(LOG_WARNING, "Unable to allocate space for backtrace structure\n");
+		return;
+	}
+
+	if ((strings = backtrace_symbols(backtrace->addresses, backtrace->num_frames))) {
+		ast_debug(1, "Got %d backtrace record%c\n", backtrace->num_frames, backtrace->num_frames != 1 ? 's' : ' ');
+		for (i = 0; i < backtrace->num_frames; i++) {
 #if __WORDSIZE == 32
-				ast_log(LOG_DEBUG, "#%d: [%08X] %s\n", i, (unsigned int)addresses[i], strings[i]);
+			ast_log(LOG_DEBUG, "#%d: [%08X] %s\n", i, (unsigned int)backtrace->addresses[i], strings[i]);
 #elif __WORDSIZE == 64
-				ast_log(LOG_DEBUG, "#%d: [%016lX] %s\n", i, (unsigned long)addresses[i], strings[i]);
+			ast_log(LOG_DEBUG, "#%d: [%016lX] %s\n", i, (unsigned long)backtrace->addresses[i], strings[i]);
 #endif
-			}
-			free(strings);
-		} else {
-			ast_debug(1, "Could not allocate memory for backtrace\n");
 		}
-		ast_free(addresses);
+		free(strings);
+	} else {
+		ast_debug(1, "Could not allocate memory for backtrace\n");
 	}
+	ast_bt_destroy(backtrace);
 #else
 	ast_log(LOG_WARNING, "Must run configure with '--with-execinfo' for stack backtraces.\n");
 #endif
diff --git a/main/utils.c b/main/utils.c
index 3946df9a9..dfa32eb58 100644
--- a/main/utils.c
+++ b/main/utils.c
@@ -540,6 +540,9 @@ struct thr_lock_info {
 		enum ast_lock_type type;
 		/*! This thread is waiting on this lock */
 		int pending:2;
+#ifdef HAVE_BKTR
+		struct ast_bt *backtrace;
+#endif
 	} locks[AST_MAX_LOCKS];
 	/*! This is the number of locks currently held by this thread.
 	 *  The index (num_locks - 1) has the info on the last one in the
@@ -583,9 +586,13 @@ static void lock_info_destroy(void *data)
  * \brief The thread storage key for per-thread lock info
  */
 AST_THREADSTORAGE_CUSTOM(thread_lock_info, NULL, lock_info_destroy);
-
+#ifdef HAVE_BKTR
+void ast_store_lock_info(enum ast_lock_type type, const char *filename,
+	int line_num, const char *func, const char *lock_name, void *lock_addr, struct ast_bt *bt)
+#else
 void ast_store_lock_info(enum ast_lock_type type, const char *filename,
 	int line_num, const char *func, const char *lock_name, void *lock_addr)
+#endif
 {
 	struct thr_lock_info *lock_info;
 	int i;
@@ -598,6 +605,9 @@ void ast_store_lock_info(enum ast_lock_type type, const char *filename,
 	for (i = 0; i < lock_info->num_locks; i++) {
 		if (lock_info->locks[i].lock_addr == lock_addr) {
 			lock_info->locks[i].times_locked++;
+#ifdef HAVE_BKTR
+			lock_info->locks[i].backtrace = bt;
+#endif
 			pthread_mutex_unlock(&lock_info->lock);
 			return;
 		}
@@ -628,6 +638,9 @@ void ast_store_lock_info(enum ast_lock_type type, const char *filename,
 	lock_info->locks[i].times_locked = 1;
 	lock_info->locks[i].type = type;
 	lock_info->locks[i].pending = 1;
+#ifdef HAVE_BKTR
+	lock_info->locks[i].backtrace = bt;
+#endif
 	lock_info->num_locks++;
 
 	pthread_mutex_unlock(&lock_info->lock);
@@ -661,8 +674,11 @@ void ast_mark_lock_failed(void *lock_addr)
 	}
 	pthread_mutex_unlock(&lock_info->lock);
 }
-
+#ifdef HAVE_BKTR
+void ast_remove_lock_info(void *lock_addr, struct ast_bt *bt)
+#else
 void ast_remove_lock_info(void *lock_addr)
+#endif
 {
 	struct thr_lock_info *lock_info;
 	int i = 0;
@@ -685,6 +701,9 @@ void ast_remove_lock_info(void *lock_addr)
 
 	if (lock_info->locks[i].times_locked > 1) {
 		lock_info->locks[i].times_locked--;
+#ifdef HAVE_BKTR
+		lock_info->locks[i].backtrace = bt;
+#endif
 		pthread_mutex_unlock(&lock_info->lock);
 		return;
 	}
@@ -714,6 +733,30 @@ static const char *locktype2str(enum ast_lock_type type)
 	return "UNKNOWN";
 }
 
+#ifdef HAVE_BKTR
+static void append_backtrace_information(struct ast_str **str, struct ast_bt *bt)
+{
+	char **symbols;
+
+	if (!bt) {
+		ast_str_append(str, 0, "\tNo backtrace to print\n");
+		return;
+	}
+
+	if ((symbols = backtrace_symbols(bt->addresses, bt->num_frames))) {
+		int frame_iterator;
+		
+		for (frame_iterator = 0; frame_iterator < bt->num_frames; ++frame_iterator) {
+			ast_str_append(str, 0, "\t%s\n", symbols[frame_iterator]);
+		}
+
+		free(symbols);
+	} else {
+		ast_str_append(str, 0, "\tCouldn't retrieve backtrace symbols\n");
+	}
+}
+#endif
+
 static void append_lock_information(struct ast_str **str, struct thr_lock_info *lock_info, int i)
 {
 	int j;
@@ -728,6 +771,9 @@ static void append_lock_information(struct ast_str **str, struct thr_lock_info *
 				   lock_info->locks[i].func, lock_info->locks[i].lock_name,
 				   lock_info->locks[i].lock_addr, 
 				   lock_info->locks[i].times_locked);
+#ifdef HAVE_BKTR
+	append_backtrace_information(str, lock_info->locks[i].backtrace);
+#endif
 	
 	if (!lock_info->locks[i].pending || lock_info->locks[i].pending == -1)
 		return;