Fix deadlocks in chan_sip in REFER and BYE handling

This resolves several deadlocks in chan_sip relating to usage of
ast_channel_bridge_peer and improves accessibility of lock debugging
function calls.

Review: https://reviewboard.asterisk.org/r/2756/
(closes issue ASTERISK-22215)


git-svn-id: https://origsvn.digium.com/svn/asterisk/trunk@396723 65c4cc65-6c06-0410-ace0-fbb531ad65f3

3 files changed, 156 insertions, 87 deletions

diff --git a/channels/chan_sip.c b/channels/chan_sip.c
index 3fa04b0d9..fc3fb4c49 100644
--- a/channels/chan_sip.c
+++ b/channels/chan_sip.c
@@ -18008,23 +18008,30 @@ static int get_refer_info(struct sip_pvt *transferer, struct sip_request *outgoi
 
 	/* Give useful transfer information to the dialplan */
 	if (transferer->owner) {
-		RAII_VAR(struct ast_channel *, peer, ast_channel_bridge_peer(transferer->owner), ast_channel_cleanup);
+		RAII_VAR(struct ast_channel *, peer, NULL, ast_channel_cleanup);
+		RAII_VAR(struct ast_channel *, owner_relock, NULL, ast_channel_cleanup);
+		RAII_VAR(struct ast_channel *, owner_ref, NULL, ast_channel_cleanup);
 
-		/*! pbx_builtin_setvar_helper will attempt to lock the channel. We need
-		 * to be sure it's already locked here so we don't deadlock.
-		 */
-		while (peer && ast_channel_trylock(peer)) {
-			sip_pvt_unlock(transferer);
-			do {
-				CHANNEL_DEADLOCK_AVOIDANCE(transferer->owner);
-			} while (sip_pvt_trylock(transferer));
-		}
+		/* Grab a reference to transferer->owner to prevent it from going away */
+		owner_ref = ast_channel_ref(transferer->owner);
+
+		/* Established locking order here is bridge, channel, pvt
+		 * and the bridge will be locked during ast_channel_bridge_peer */
+		ast_channel_unlock(owner_ref);
+		sip_pvt_unlock(transferer);
 
+		peer = ast_channel_bridge_peer(owner_ref);
 		if (peer) {
 			pbx_builtin_setvar_helper(peer, "SIPREFERRINGCONTEXT", transferer->context);
 			pbx_builtin_setvar_helper(peer, "SIPREFERREDBYHDR", p_referred_by);
 			ast_channel_unlock(peer);
 		}
+
+		owner_relock = sip_pvt_lock_full(transferer);
+		if (!owner_relock) {
+			ast_debug(3, "Unable to reacquire owner channel lock, channel is gone\n");
+			return -5;
+		}
 	}
 
 	if (!ast_strlen_zero(p_referred_by)) {
@@ -26091,7 +26098,7 @@ static int handle_request_refer(struct sip_pvt *p, struct sip_request *req, uint
 	int res = 0;
 	struct blind_transfer_cb_data cb_data;
 	enum ast_transfer_result transfer_res;
-	RAII_VAR(struct ast_channel *, transferer, NULL, ao2_cleanup);
+	RAII_VAR(struct ast_channel *, transferer, NULL, ast_channel_cleanup);
 	RAII_VAR(struct ast_str *, replaces_str, NULL, ast_free_ptr);
 
 	if (req->debug) {
@@ -26368,6 +26375,8 @@ static int handle_request_bye(struct sip_pvt *p, struct sip_request *req)
 	struct ast_channel *c=NULL;
 	int res;
 	const char *required;
+	RAII_VAR(struct ast_channel *, peer_channel, NULL, ast_channel_cleanup);
+	char quality_buf[AST_MAX_USER_FIELD], *quality;
 
 	/* If we have an INCOMING invite that we haven't answered, terminate that transaction */
 	if (p->pendinginvite && !ast_test_flag(&p->flags[0], SIP_OUTGOING) && !req->ignore) {
@@ -26384,70 +26393,88 @@ static int handle_request_bye(struct sip_pvt *p, struct sip_request *req)
 	check_via(p, req);
 	sip_alreadygone(p);
 
-	/* Get RTCP quality before end of call */
-	if (p->do_history || p->owner) {
-		char quality_buf[AST_MAX_USER_FIELD], *quality;
-		RAII_VAR(struct ast_channel *, bridge, p->owner ? ast_channel_bridge_peer(p->owner) : NULL, ast_channel_cleanup);
+	if (p->owner) {
+		RAII_VAR(struct ast_channel *, owner_relock, NULL, ast_channel_cleanup);
+		RAII_VAR(struct ast_channel *, owner_ref, NULL, ast_channel_cleanup);
 
-		/* We need to get the lock on bridge because ast_rtp_instance_set_stats_vars will attempt
-		 * to lock the bridge. This may get hairy...
-		 */
-		while (bridge && ast_channel_trylock(bridge)) {
-			ast_channel_unlock(p->owner);
-			do {
-				/* Can't use DEADLOCK_AVOIDANCE since p is an ao2 object */
-				sip_pvt_unlock(p);
-				usleep(1);
-				sip_pvt_lock(p);
-			} while (p->owner && ast_channel_trylock(p->owner));
-		}
+		/* Grab a reference to p->owner to prevent it from going away */
+		owner_ref = ast_channel_ref(p->owner);
+
+		/* Established locking order here is bridge, channel, pvt
+		 * and the bridge will be locked during ast_channel_bridge_peer */
+		ast_channel_unlock(owner_ref);
+		sip_pvt_unlock(p);
+
+		peer_channel = ast_channel_bridge_peer(owner_ref);
 
+		owner_relock = sip_pvt_lock_full(p);
+		if (!owner_relock) {
+			ast_debug(3, "Unable to reacquire owner channel lock, channel is gone\n");
+			return 0;
+		}
+	}
 
-		if (p->rtp && (quality = ast_rtp_instance_get_quality(p->rtp, AST_RTP_INSTANCE_STAT_FIELD_QUALITY, quality_buf, sizeof(quality_buf)))) {
-			if (p->do_history) {
+	/* Get RTCP quality before end of call */
+	if (p->rtp) {
+		if (p->do_history) {
+			if ((quality = ast_rtp_instance_get_quality(p->rtp, AST_RTP_INSTANCE_STAT_FIELD_QUALITY, quality_buf, sizeof(quality_buf)))) {
 				append_history(p, "RTCPaudio", "Quality:%s", quality);
+			}
+			if ((quality = ast_rtp_instance_get_quality(p->rtp, AST_RTP_INSTANCE_STAT_FIELD_QUALITY_JITTER, quality_buf, sizeof(quality_buf)))) {
+				append_history(p, "RTCPaudioJitter", "Quality:%s", quality);
+			}
+			if ((quality = ast_rtp_instance_get_quality(p->rtp, AST_RTP_INSTANCE_STAT_FIELD_QUALITY_LOSS, quality_buf, sizeof(quality_buf)))) {
+				append_history(p, "RTCPaudioLoss", "Quality:%s", quality);
+			}
+			if ((quality = ast_rtp_instance_get_quality(p->rtp, AST_RTP_INSTANCE_STAT_FIELD_QUALITY_RTT, quality_buf, sizeof(quality_buf)))) {
+				append_history(p, "RTCPaudioRTT", "Quality:%s", quality);
+			}
+		}
 
-				if ((quality = ast_rtp_instance_get_quality(p->rtp, AST_RTP_INSTANCE_STAT_FIELD_QUALITY_JITTER, quality_buf, sizeof(quality_buf)))) {
-					append_history(p, "RTCPaudioJitter", "Quality:%s", quality);
-				}
-				if ((quality = ast_rtp_instance_get_quality(p->rtp, AST_RTP_INSTANCE_STAT_FIELD_QUALITY_LOSS, quality_buf, sizeof(quality_buf)))) {
-					append_history(p, "RTCPaudioLoss", "Quality:%s", quality);
-				}
-				if ((quality = ast_rtp_instance_get_quality(p->rtp, AST_RTP_INSTANCE_STAT_FIELD_QUALITY_RTT, quality_buf, sizeof(quality_buf)))) {
-					append_history(p, "RTCPaudioRTT", "Quality:%s", quality);
+		if (p->owner) {
+			RAII_VAR(struct ast_channel *, owner_relock, NULL, ast_channel_cleanup);
+			RAII_VAR(struct ast_channel *, owner_ref, NULL, ast_channel_cleanup);
+
+			/* Grab a reference to p->owner to prevent it from going away */
+			owner_ref = ast_channel_ref(p->owner);
+
+			/* Established locking order here is bridge, channel, pvt
+			 * and the bridge and channel will be locked during
+			 * ast_rtp_instance_set_stats_vars */
+			ast_channel_unlock(owner_ref);
+			sip_pvt_unlock(p);
+
+			ast_rtp_instance_set_stats_vars(owner_ref, p->rtp);
+			if (peer_channel && IS_SIP_TECH(ast_channel_tech(peer_channel))) {
+				struct sip_pvt *q = ast_channel_tech_pvt(peer_channel);
+				if (q && q->rtp) {
+					ast_rtp_instance_set_stats_vars(peer_channel, q->rtp);
 				}
 			}
 
-			if (p->owner) {
-				ast_rtp_instance_set_stats_vars(p->owner, p->rtp);
+			owner_relock = sip_pvt_lock_full(p);
+			if (!owner_relock) {
+				ast_debug(3, "Unable to reacquire owner channel lock, channel is gone\n");
+				return 0;
 			}
-
 		}
+	}
 
-		if (bridge) {
-			struct sip_pvt *q = ast_channel_tech_pvt(bridge);
-
-			if (IS_SIP_TECH(ast_channel_tech(bridge)) && q && q->rtp) {
-				ast_rtp_instance_set_stats_vars(bridge, q->rtp);
-			}
-			ast_channel_unlock(bridge);
+	if (p->vrtp && (quality = ast_rtp_instance_get_quality(p->vrtp, AST_RTP_INSTANCE_STAT_FIELD_QUALITY, quality_buf, sizeof(quality_buf)))) {
+		if (p->do_history) {
+			append_history(p, "RTCPvideo", "Quality:%s", quality);
+		}
+		if (p->owner) {
+			pbx_builtin_setvar_helper(p->owner, "RTPVIDEOQOS", quality);
 		}
+	}
 
-		if (p->vrtp && (quality = ast_rtp_instance_get_quality(p->vrtp, AST_RTP_INSTANCE_STAT_FIELD_QUALITY, quality_buf, sizeof(quality_buf)))) {
-			if (p->do_history) {
-				append_history(p, "RTCPvideo", "Quality:%s", quality);
-			}
-			if (p->owner) {
-				pbx_builtin_setvar_helper(p->owner, "RTPVIDEOQOS", quality);
-			}
+	if (p->trtp && (quality = ast_rtp_instance_get_quality(p->trtp, AST_RTP_INSTANCE_STAT_FIELD_QUALITY, quality_buf, sizeof(quality_buf)))) {
+		if (p->do_history) {
+			append_history(p, "RTCPtext", "Quality:%s", quality);
 		}
-		if (p->trtp && (quality = ast_rtp_instance_get_quality(p->trtp, AST_RTP_INSTANCE_STAT_FIELD_QUALITY, quality_buf, sizeof(quality_buf)))) {
-			if (p->do_history) {
-				append_history(p, "RTCPtext", "Quality:%s", quality);
-			}
-			if (p->owner) {
-				pbx_builtin_setvar_helper(p->owner, "RTPTEXTQOS", quality);
-			}
+		if (p->owner) {
+			pbx_builtin_setvar_helper(p->owner, "RTPTEXTQOS", quality);
 		}
 	}
 
@@ -26463,15 +26490,30 @@ static int handle_request_bye(struct sip_pvt *p, struct sip_request *req)
 		if (!res) {
 			c = p->owner;
 			if (c) {
-				RAII_VAR(struct ast_channel *, bridged_to, ast_channel_bridge_peer(c), ast_channel_cleanup);
-				if (bridged_to) {
+				if (peer_channel) {
+					RAII_VAR(struct ast_channel *, owner_relock, NULL, ast_channel_cleanup);
+					char *local_context = ast_strdupa(p->context);
+					char *local_refer_to = ast_strdupa(p->refer->refer_to);
+
+					/* Grab a reference to p->owner to prevent it from going away */
+					ast_channel_ref(c);
+
 					/* Don't actually hangup here... */
 					ast_queue_unhold(c);
 					ast_channel_unlock(c);  /* async_goto can do a masquerade, no locks can be held during a masq */
-					ast_async_goto(bridged_to, p->context, p->refer->refer_to, 1);
-					ast_channel_lock(c);
-				} else
+					sip_pvt_unlock(p);
+
+					ast_async_goto(peer_channel, local_context, local_refer_to, 1);
+
+					owner_relock = sip_pvt_lock_full(p);
+					ast_channel_cleanup(c);
+					if (!owner_relock) {
+						ast_debug(3, "Unable to reacquire owner channel lock, channel is gone\n");
+						return 0;
+					}
+				} else {
 					ast_queue_hangup(p->owner);
+				}
 			}
 		} else {
 			ast_log(LOG_WARNING, "Invalid transfer information from '%s'\n", ast_sockaddr_stringify(&p->recv));
diff --git a/include/asterisk/lock.h b/include/asterisk/lock.h
index 6ede7de50..34e944476 100644
--- a/include/asterisk/lock.h
+++ b/include/asterisk/lock.h
@@ -316,7 +316,22 @@ static inline void __dump_backtrace(struct ast_bt *bt, int canlog)
  * \param this_lock_addr lock address to return lock information
  * \since 1.6.1
  */
-void log_show_lock(void *this_lock_addr);
+void ast_log_show_lock(void *this_lock_addr);
+
+/*!
+ * \brief Generate a lock dump equivalent to "core show locks".
+ *
+ * The lock dump generated is generally too large to be output by a
+ * single ast_verbose/log/debug/etc. call. Only ast_cli() handles it
+ * properly without changing BUFSIZ in logger.c.
+ *
+ * Note: This must be ast_free()d when you're done with it.
+ *
+ * \retval An ast_str containing the lock dump
+ * \retval NULL on error
+ * \since 12
+ */
+struct ast_str *ast_dump_locks(void);
 
 /*!
  * \brief retrieve lock info for the specified mutex
diff --git a/main/utils.c b/main/utils.c
index b19148b7d..1f0528556 100644
--- a/main/utils.c
+++ b/main/utils.c
@@ -927,7 +927,7 @@ static void append_lock_information(struct ast_str **str, struct thr_lock_info *
 	which will give a stack trace and continue. -- that aught to do the job!
 
 */
-void log_show_lock(void *this_lock_addr)
+void ast_log_show_lock(void *this_lock_addr)
 {
 	struct thr_lock_info *lock_info;
 	struct ast_str *str;
@@ -958,24 +958,12 @@ void log_show_lock(void *this_lock_addr)
 }
 
 
-static char *handle_show_locks(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
+struct ast_str *ast_dump_locks(void)
 {
 	struct thr_lock_info *lock_info;
 	struct ast_str *str;
 
-	if (!(str = ast_str_create(4096)))
-		return CLI_FAILURE;
-
-	switch (cmd) {
-	case CLI_INIT:
-		e->command = "core show locks";
-		e->usage =
-			"Usage: core show locks\n"
-			"       This command is for lock debugging.  It prints out which locks\n"
-			"are owned by each active thread.\n";
-		return NULL;
-
-	case CLI_GENERATE:
+	if (!(str = ast_str_create(4096))) {
 		return NULL;
 	}
 
@@ -988,8 +976,9 @@ static char *handle_show_locks(struct ast_cli_entry *e, int cmd, struct ast_cli_
 	               "=== <pending> <lock#> (<file>): <lock type> <line num> <function> <lock name> <lock addr> (times locked)\n"
 	               "===\n", ast_get_version());
 
-	if (!str)
-		return CLI_FAILURE;
+	if (!str) {
+		return NULL;
+	}
 
 	pthread_mutex_lock(&lock_infos_lock.mutex);
 	AST_LIST_TRAVERSE(&lock_infos, lock_info, entry) {
@@ -1012,14 +1001,37 @@ static char *handle_show_locks(struct ast_cli_entry *e, int cmd, struct ast_cli_
 	}
 	pthread_mutex_unlock(&lock_infos_lock.mutex);
 
-	if (!str)
-		return CLI_FAILURE;
+	if (!str) {
+		return NULL;
+	}
 
 	ast_str_append(&str, 0, "=======================================================================\n"
 	               "\n");
 
-	if (!str)
+	return str;
+}
+
+static char *handle_show_locks(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
+{
+	struct ast_str *str;
+
+	switch (cmd) {
+	case CLI_INIT:
+		e->command = "core show locks";
+		e->usage =
+			"Usage: core show locks\n"
+			"       This command is for lock debugging.  It prints out which locks\n"
+			"are owned by each active thread.\n";
+		return NULL;
+
+	case CLI_GENERATE:
+		return NULL;
+	}
+
+	str = ast_dump_locks();
+	if (!str) {
 		return CLI_FAILURE;
+	}
 
 	ast_cli(a->fd, "%s", ast_str_buffer(str));