HTTP: Stop accepting requests on final system shutdown.

There are three CLI commands to stop and restart Asterisk each.

1) core stop/restart now - Hangup all calls and stop or restart Asterisk.
New channels are prevented while the shutdown request is pending.

2) core stop/restart gracefully - Stop or restart Asterisk when there are
no calls remaining in the system.  New channels are prevented while the
shutdown request is pending.

3) core stop/restart when convenient - Stop or restart Asterisk when there
are no calls in the system.  New calls are not prevented while the
shutdown request is pending.

ARI has made stopping/restarting Asterisk more problematic.  While a
shutdown request is pending it is desirable to continue to process ARI
HTTP requests for current calls.  To handle the current calls while a
shutdown request is pending, a new committed to shutdown phase is needed
so ARI applications can deal with the calls until the system is fully
committed to shutdown.

* Added a new shutdown committed phase so ARI applications can deal with
calls until the final committed to shutdown phase is reached.

* Made refuse new HTTP requests when the system has reached the final
system shutdown phase.  Starting anything while the system is actively
releasing resources and unloading modules is not a good thing.

* Split the bridging framework shutdown to not cleanup the global bridging
containers when shutting down in a hurry.  This is similar to how other
modules prevent crashes on rapid system shutdown.

* Moved ast_begin_shutdown(), ast_cancel_shutdown(), and
ast_shutting_down().  You should not have to include channel.h just to
access these system functions.

ASTERISK-24752 #close
Reported by: Matthew Jordan

Review: https://reviewboard.asterisk.org/r/4399/


git-svn-id: https://origsvn.digium.com/svn/asterisk/branches/13@431692 65c4cc65-6c06-0410-ace0-fbb531ad65f3

4 files changed, 155 insertions, 58 deletions

diff --git a/main/asterisk.c b/main/asterisk.c
index 1e5829616..1817eefd8 100644
--- a/main/asterisk.c
+++ b/main/asterisk.c
@@ -432,16 +432,34 @@ static char ast_config_AST_CTL[PATH_MAX] = "asterisk.ctl";
 extern unsigned int ast_FD_SETSIZE;
 
 static char *_argv[256];
+
 typedef enum {
-	NOT_SHUTTING_DOWN = -2,
-	SHUTTING_DOWN = -1,
-	/* Valid values for quit_handler niceness below: */
+	/*! Normal operation */
+	NOT_SHUTTING_DOWN,
+	/*! Committed to shutting down.  Final phase */
+	SHUTTING_DOWN_FINAL,
+	/*! Committed to shutting down.  Initial phase */
+	SHUTTING_DOWN,
+	/*!
+	 * Valid values for quit_handler() niceness below.
+	 * These shutdown/restart levels can be cancelled.
+	 *
+	 * Remote console exit right now
+	 */
 	SHUTDOWN_FAST,
+	/*! core stop/restart now */
 	SHUTDOWN_NORMAL,
+	/*! core stop/restart gracefully */
 	SHUTDOWN_NICE,
+	/*! core stop/restart when convenient */
 	SHUTDOWN_REALLY_NICE
 } shutdown_nice_t;
+
 static shutdown_nice_t shuttingdown = NOT_SHUTTING_DOWN;
+
+/*! Prevent new channel allocation for shutdown. */
+static int shutdown_pending;
+
 static int restartnow;
 static pthread_t consolethread = AST_PTHREADT_NULL;
 static pthread_t mon_sig_flags;
@@ -1867,6 +1885,43 @@ int ast_set_priority(int pri)
 	return 0;
 }
 
+int ast_shutdown_final(void)
+{
+	return shuttingdown == SHUTTING_DOWN_FINAL;
+}
+
+int ast_shutting_down(void)
+{
+	return shutdown_pending;
+}
+
+int ast_cancel_shutdown(void)
+{
+	int shutdown_aborted = 0;
+
+	ast_mutex_lock(&safe_system_lock);
+	if (shuttingdown >= SHUTDOWN_FAST) {
+		shuttingdown = NOT_SHUTTING_DOWN;
+		shutdown_pending = 0;
+		shutdown_aborted = 1;
+	}
+	ast_mutex_unlock(&safe_system_lock);
+	return shutdown_aborted;
+}
+
+/*!
+ * \internal
+ * \brief Initiate system shutdown -- prevents new channels from being allocated.
+ */
+static void ast_begin_shutdown(void)
+{
+	ast_mutex_lock(&safe_system_lock);
+	if (shuttingdown != NOT_SHUTTING_DOWN) {
+		shutdown_pending = 1;
+	}
+	ast_mutex_unlock(&safe_system_lock);
+}
+
 static int can_safely_quit(shutdown_nice_t niceness, int restart);
 static void really_quit(int num, shutdown_nice_t niceness, int restart);
 
@@ -1879,8 +1934,53 @@ static void quit_handler(int num, shutdown_nice_t niceness, int restart)
 	/* It wasn't our time. */
 }
 
+#define SHUTDOWN_TIMEOUT	15	/* Seconds */
+
+/*!
+ * \internal
+ * \brief Wait for all channels to die, a timeout, or shutdown cancelled.
+ * \since 13.3.0
+ *
+ * \param niceness Shutdown niceness in effect
+ * \param seconds Number of seconds to wait or less than zero if indefinitely.
+ *
+ * \retval zero if waiting wasn't necessary.  We were idle.
+ * \retval non-zero if we had to wait.
+ */
+static int wait_for_channels_to_die(shutdown_nice_t niceness, int seconds)
+{
+	time_t start;
+	time_t now;
+	int waited = 0;
+
+	time(&start);
+	for (;;) {
+		if (!ast_undestroyed_channels() || shuttingdown != niceness) {
+			break;
+		}
+		if (seconds < 0) {
+			/* No timeout so just poll every second */
+			sleep(1);
+		} else {
+			time(&now);
+
+			/* Wait up to the given seconds for all channels to go away */
+			if (seconds < (now - start)) {
+				break;
+			}
+
+			/* Sleep 1/10 of a second */
+			usleep(100000);
+		}
+		waited = 1;
+	}
+	return waited;
+}
+
 static int can_safely_quit(shutdown_nice_t niceness, int restart)
 {
+	int waited = 0;
+
 	/* Check if someone else isn't already doing this. */
 	ast_mutex_lock(&safe_system_lock);
 	if (shuttingdown != NOT_SHUTTING_DOWN && niceness >= shuttingdown) {
@@ -1897,40 +1997,30 @@ static int can_safely_quit(shutdown_nice_t niceness, int restart)
 	 * the atexit handlers, otherwise this would be a bit early. */
 	ast_cdr_engine_term();
 
-	/* Shutdown the message queue for the technology agnostic message channel.
-	 * This has to occur before we pause shutdown pending ast_undestroyed_channels. */
+	/*
+	 * Shutdown the message queue for the technology agnostic message channel.
+	 * This has to occur before we pause shutdown pending ast_undestroyed_channels.
+	 *
+	 * XXX This is not reversed on shutdown cancel.
+	 */
 	ast_msg_shutdown();
 
 	if (niceness == SHUTDOWN_NORMAL) {
-		time_t s, e;
 		/* Begin shutdown routine, hanging up active channels */
-		ast_begin_shutdown(1);
+		ast_begin_shutdown();
 		if (ast_opt_console) {
 			ast_verb(0, "Beginning asterisk %s....\n", restart ? "restart" : "shutdown");
 		}
-		time(&s);
-		for (;;) {
-			time(&e);
-			/* Wait up to 15 seconds for all channels to go away */
-			if ((e - s) > 15 || !ast_undestroyed_channels() || shuttingdown != niceness) {
-				break;
-			}
-			/* Sleep 1/10 of a second */
-			usleep(100000);
-		}
+		ast_softhangup_all();
+		waited |= wait_for_channels_to_die(niceness, SHUTDOWN_TIMEOUT);
 	} else if (niceness >= SHUTDOWN_NICE) {
 		if (niceness != SHUTDOWN_REALLY_NICE) {
-			ast_begin_shutdown(0);
+			ast_begin_shutdown();
 		}
 		if (ast_opt_console) {
 			ast_verb(0, "Waiting for inactivity to perform %s...\n", restart ? "restart" : "halt");
 		}
-		for (;;) {
-			if (!ast_undestroyed_channels() || shuttingdown != niceness) {
-				break;
-			}
-			sleep(1);
-		}
+		waited |= wait_for_channels_to_die(niceness, -1);
 	}
 
 	/* Re-acquire lock and check if someone changed the niceness, in which
@@ -1944,9 +2034,28 @@ static int can_safely_quit(shutdown_nice_t niceness, int restart)
 		ast_mutex_unlock(&safe_system_lock);
 		return 0;
 	}
-	shuttingdown = SHUTTING_DOWN;
+
+	if (niceness >= SHUTDOWN_REALLY_NICE) {
+		shuttingdown = SHUTTING_DOWN;
+		ast_mutex_unlock(&safe_system_lock);
+
+		/* No more Mr. Nice guy.  We are committed to shutting down now. */
+		ast_begin_shutdown();
+		ast_softhangup_all();
+		waited |= wait_for_channels_to_die(SHUTTING_DOWN, SHUTDOWN_TIMEOUT);
+
+		ast_mutex_lock(&safe_system_lock);
+	}
+	shuttingdown = SHUTTING_DOWN_FINAL;
 	ast_mutex_unlock(&safe_system_lock);
 
+	if (niceness >= SHUTDOWN_NORMAL && waited) {
+		/*
+		 * We were not idle.  Give things in progress a chance to
+		 * recognize the final shutdown phase.
+		 */
+		sleep(1);
+	}
 	return 1;
 }
 
@@ -2454,8 +2563,6 @@ static char *handle_restart_when_convenient(struct ast_cli_entry *e, int cmd, st
 
 static char *handle_abort_shutdown(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
 {
-	int aborting_shutdown = 0;
-
 	switch (cmd) {
 	case CLI_INIT:
 		e->command = "core abort shutdown";
@@ -2471,16 +2578,8 @@ static char *handle_abort_shutdown(struct ast_cli_entry *e, int cmd, struct ast_
 	if (a->argc != e->args)
 		return CLI_SHOWUSAGE;
 
-	ast_mutex_lock(&safe_system_lock);
-	if (shuttingdown >= SHUTDOWN_FAST) {
-		aborting_shutdown = 1;
-		shuttingdown = NOT_SHUTTING_DOWN;
-	}
-	ast_mutex_unlock(&safe_system_lock);
+	ast_cancel_shutdown();
 
-	if (aborting_shutdown) {
-		ast_cancel_shutdown();
-	}
 	return CLI_SUCCESS;
 }
 
diff --git a/main/bridge.c b/main/bridge.c
index c1fc145c6..6737fa62f 100644
--- a/main/bridge.c
+++ b/main/bridge.c
@@ -5322,7 +5322,7 @@ static void bridge_prnt_obj(void *v_obj, void *where, ao2_prnt_fn *prnt)
 
 /*!
  * \internal
- * \brief Shutdown the bridging system.
+ * \brief Shutdown the bridging system.  Stuff to always do.
  * \since 12.0.0
  *
  * \return Nothing
@@ -5334,6 +5334,17 @@ static void bridge_shutdown(void)
 	ast_manager_unregister("BridgeTechnologyUnsuspend");
 	ast_cli_unregister_multiple(bridge_cli, ARRAY_LEN(bridge_cli));
 	ao2_container_unregister("bridges");
+}
+
+/*!
+ * \internal
+ * \brief Shutdown the bridging system.  More stuff to do on graceful shutdown.
+ * \since 13.3.0
+ *
+ * \return Nothing
+ */
+static void bridge_cleanup(void)
+{
 	ao2_cleanup(bridges);
 	bridges = NULL;
 	ao2_cleanup(bridge_manager);
@@ -5342,6 +5353,7 @@ static void bridge_shutdown(void)
 
 int ast_bridging_init(void)
 {
+	ast_register_cleanup(bridge_cleanup);
 	ast_register_atexit(bridge_shutdown);
 
 	if (ast_stasis_bridging_init()) {
diff --git a/main/channel.c b/main/channel.c
index 4bf2f61ef..da0ca558b 100644
--- a/main/channel.c
+++ b/main/channel.c
@@ -100,9 +100,6 @@ struct ast_epoll_data {
 #define MONITOR_DELAY	150 * 8		/*!< 150 ms of MONITORING DELAY */
 #endif
 
-/*! \brief Prevent new channel allocation if shutting down. */
-static int shutting_down;
-
 static int chancount;
 
 unsigned long global_fin, global_fout;
@@ -504,13 +501,9 @@ static int ast_channel_softhangup_cb(void *obj, void *arg, int flags)
 	return 0;
 }
 
-void ast_begin_shutdown(int hangup)
+void ast_softhangup_all(void)
 {
-	shutting_down = 1;
-
-	if (hangup) {
-		ao2_callback(channels, OBJ_NODATA | OBJ_MULTIPLE, ast_channel_softhangup_cb, NULL);
-	}
+	ao2_callback(channels, OBJ_NODATA | OBJ_MULTIPLE, ast_channel_softhangup_cb, NULL);
 }
 
 /*! \brief returns number of active/allocated channels */
@@ -524,18 +517,6 @@ int ast_undestroyed_channels(void)
 	return ast_atomic_fetchadd_int(&chancount, 0);
 }
 
-/*! \brief Cancel a shutdown in progress */
-void ast_cancel_shutdown(void)
-{
-	shutting_down = 0;
-}
-
-/*! \brief Returns non-zero if Asterisk is being shut down */
-int ast_shutting_down(void)
-{
-	return shutting_down;
-}
-
 /*! \brief Set when to hangup channel */
 void ast_channel_setwhentohangup_tv(struct ast_channel *chan, struct timeval offset)
 {
diff --git a/main/http.c b/main/http.c
index ef3b4b22b..3346cadb4 100644
--- a/main/http.c
+++ b/main/http.c
@@ -1873,6 +1873,11 @@ static int httpd_process_request(struct ast_tcptls_session_instance *ser)
 		return -1;
 	}
 
+	if (ast_shutdown_final()) {
+		ast_http_error(ser, 503, "Service Unavailable", "Shutdown in progress");
+		return -1;
+	}
+
 	/* process "Request Headers" lines */
 	if (http_request_headers_get(ser, &headers)) {
 		return -1;