From feddab7944a12746bcbdb6fc82cc9d4951d61eb5 Mon Sep 17 00:00:00 2001 From: Richard Mudgett Date: Wed, 11 Feb 2015 17:28:13 +0000 Subject: HTTP: Stop accepting requests on final system shutdown. There are three CLI commands to stop and restart Asterisk each. 1) core stop/restart now - Hangup all calls and stop or restart Asterisk. New channels are prevented while the shutdown request is pending. 2) core stop/restart gracefully - Stop or restart Asterisk when there are no calls remaining in the system. New channels are prevented while the shutdown request is pending. 3) core stop/restart when convenient - Stop or restart Asterisk when there are no calls in the system. New calls are not prevented while the shutdown request is pending. ARI has made stopping/restarting Asterisk more problematic. While a shutdown request is pending it is desirable to continue to process ARI HTTP requests for current calls. To handle the current calls while a shutdown request is pending, a new committed to shutdown phase is needed so ARI applications can deal with the calls until the system is fully committed to shutdown. * Added a new shutdown committed phase so ARI applications can deal with calls until the final committed to shutdown phase is reached. * Made refuse new HTTP requests when the system has reached the final system shutdown phase. Starting anything while the system is actively releasing resources and unloading modules is not a good thing. * Split the bridging framework shutdown to not cleanup the global bridging containers when shutting down in a hurry. This is similar to how other modules prevent crashes on rapid system shutdown. * Moved ast_begin_shutdown(), ast_cancel_shutdown(), and ast_shutting_down(). You should not have to include channel.h just to access these system functions. ASTERISK-24752 #close Reported by: Matthew Jordan Review: https://reviewboard.asterisk.org/r/4399/ git-svn-id: https://origsvn.digium.com/svn/asterisk/branches/13@431692 65c4cc65-6c06-0410-ace0-fbb531ad65f3 --- main/asterisk.c | 171 ++++++++++++++++++++++++++++++++++++++++++++------------ main/bridge.c | 14 ++++- main/channel.c | 23 +------- main/http.c | 5 ++ 4 files changed, 155 insertions(+), 58 deletions(-) (limited to 'main') diff --git a/main/asterisk.c b/main/asterisk.c index 1e5829616..1817eefd8 100644 --- a/main/asterisk.c +++ b/main/asterisk.c @@ -432,16 +432,34 @@ static char ast_config_AST_CTL[PATH_MAX] = "asterisk.ctl"; extern unsigned int ast_FD_SETSIZE; static char *_argv[256]; + typedef enum { - NOT_SHUTTING_DOWN = -2, - SHUTTING_DOWN = -1, - /* Valid values for quit_handler niceness below: */ + /*! Normal operation */ + NOT_SHUTTING_DOWN, + /*! Committed to shutting down. Final phase */ + SHUTTING_DOWN_FINAL, + /*! Committed to shutting down. Initial phase */ + SHUTTING_DOWN, + /*! + * Valid values for quit_handler() niceness below. + * These shutdown/restart levels can be cancelled. + * + * Remote console exit right now + */ SHUTDOWN_FAST, + /*! core stop/restart now */ SHUTDOWN_NORMAL, + /*! core stop/restart gracefully */ SHUTDOWN_NICE, + /*! core stop/restart when convenient */ SHUTDOWN_REALLY_NICE } shutdown_nice_t; + static shutdown_nice_t shuttingdown = NOT_SHUTTING_DOWN; + +/*! Prevent new channel allocation for shutdown. */ +static int shutdown_pending; + static int restartnow; static pthread_t consolethread = AST_PTHREADT_NULL; static pthread_t mon_sig_flags; @@ -1867,6 +1885,43 @@ int ast_set_priority(int pri) return 0; } +int ast_shutdown_final(void) +{ + return shuttingdown == SHUTTING_DOWN_FINAL; +} + +int ast_shutting_down(void) +{ + return shutdown_pending; +} + +int ast_cancel_shutdown(void) +{ + int shutdown_aborted = 0; + + ast_mutex_lock(&safe_system_lock); + if (shuttingdown >= SHUTDOWN_FAST) { + shuttingdown = NOT_SHUTTING_DOWN; + shutdown_pending = 0; + shutdown_aborted = 1; + } + ast_mutex_unlock(&safe_system_lock); + return shutdown_aborted; +} + +/*! + * \internal + * \brief Initiate system shutdown -- prevents new channels from being allocated. + */ +static void ast_begin_shutdown(void) +{ + ast_mutex_lock(&safe_system_lock); + if (shuttingdown != NOT_SHUTTING_DOWN) { + shutdown_pending = 1; + } + ast_mutex_unlock(&safe_system_lock); +} + static int can_safely_quit(shutdown_nice_t niceness, int restart); static void really_quit(int num, shutdown_nice_t niceness, int restart); @@ -1879,8 +1934,53 @@ static void quit_handler(int num, shutdown_nice_t niceness, int restart) /* It wasn't our time. */ } +#define SHUTDOWN_TIMEOUT 15 /* Seconds */ + +/*! + * \internal + * \brief Wait for all channels to die, a timeout, or shutdown cancelled. + * \since 13.3.0 + * + * \param niceness Shutdown niceness in effect + * \param seconds Number of seconds to wait or less than zero if indefinitely. + * + * \retval zero if waiting wasn't necessary. We were idle. + * \retval non-zero if we had to wait. + */ +static int wait_for_channels_to_die(shutdown_nice_t niceness, int seconds) +{ + time_t start; + time_t now; + int waited = 0; + + time(&start); + for (;;) { + if (!ast_undestroyed_channels() || shuttingdown != niceness) { + break; + } + if (seconds < 0) { + /* No timeout so just poll every second */ + sleep(1); + } else { + time(&now); + + /* Wait up to the given seconds for all channels to go away */ + if (seconds < (now - start)) { + break; + } + + /* Sleep 1/10 of a second */ + usleep(100000); + } + waited = 1; + } + return waited; +} + static int can_safely_quit(shutdown_nice_t niceness, int restart) { + int waited = 0; + /* Check if someone else isn't already doing this. */ ast_mutex_lock(&safe_system_lock); if (shuttingdown != NOT_SHUTTING_DOWN && niceness >= shuttingdown) { @@ -1897,40 +1997,30 @@ static int can_safely_quit(shutdown_nice_t niceness, int restart) * the atexit handlers, otherwise this would be a bit early. */ ast_cdr_engine_term(); - /* Shutdown the message queue for the technology agnostic message channel. - * This has to occur before we pause shutdown pending ast_undestroyed_channels. */ + /* + * Shutdown the message queue for the technology agnostic message channel. + * This has to occur before we pause shutdown pending ast_undestroyed_channels. + * + * XXX This is not reversed on shutdown cancel. + */ ast_msg_shutdown(); if (niceness == SHUTDOWN_NORMAL) { - time_t s, e; /* Begin shutdown routine, hanging up active channels */ - ast_begin_shutdown(1); + ast_begin_shutdown(); if (ast_opt_console) { ast_verb(0, "Beginning asterisk %s....\n", restart ? "restart" : "shutdown"); } - time(&s); - for (;;) { - time(&e); - /* Wait up to 15 seconds for all channels to go away */ - if ((e - s) > 15 || !ast_undestroyed_channels() || shuttingdown != niceness) { - break; - } - /* Sleep 1/10 of a second */ - usleep(100000); - } + ast_softhangup_all(); + waited |= wait_for_channels_to_die(niceness, SHUTDOWN_TIMEOUT); } else if (niceness >= SHUTDOWN_NICE) { if (niceness != SHUTDOWN_REALLY_NICE) { - ast_begin_shutdown(0); + ast_begin_shutdown(); } if (ast_opt_console) { ast_verb(0, "Waiting for inactivity to perform %s...\n", restart ? "restart" : "halt"); } - for (;;) { - if (!ast_undestroyed_channels() || shuttingdown != niceness) { - break; - } - sleep(1); - } + waited |= wait_for_channels_to_die(niceness, -1); } /* Re-acquire lock and check if someone changed the niceness, in which @@ -1944,9 +2034,28 @@ static int can_safely_quit(shutdown_nice_t niceness, int restart) ast_mutex_unlock(&safe_system_lock); return 0; } - shuttingdown = SHUTTING_DOWN; + + if (niceness >= SHUTDOWN_REALLY_NICE) { + shuttingdown = SHUTTING_DOWN; + ast_mutex_unlock(&safe_system_lock); + + /* No more Mr. Nice guy. We are committed to shutting down now. */ + ast_begin_shutdown(); + ast_softhangup_all(); + waited |= wait_for_channels_to_die(SHUTTING_DOWN, SHUTDOWN_TIMEOUT); + + ast_mutex_lock(&safe_system_lock); + } + shuttingdown = SHUTTING_DOWN_FINAL; ast_mutex_unlock(&safe_system_lock); + if (niceness >= SHUTDOWN_NORMAL && waited) { + /* + * We were not idle. Give things in progress a chance to + * recognize the final shutdown phase. + */ + sleep(1); + } return 1; } @@ -2454,8 +2563,6 @@ static char *handle_restart_when_convenient(struct ast_cli_entry *e, int cmd, st static char *handle_abort_shutdown(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a) { - int aborting_shutdown = 0; - switch (cmd) { case CLI_INIT: e->command = "core abort shutdown"; @@ -2471,16 +2578,8 @@ static char *handle_abort_shutdown(struct ast_cli_entry *e, int cmd, struct ast_ if (a->argc != e->args) return CLI_SHOWUSAGE; - ast_mutex_lock(&safe_system_lock); - if (shuttingdown >= SHUTDOWN_FAST) { - aborting_shutdown = 1; - shuttingdown = NOT_SHUTTING_DOWN; - } - ast_mutex_unlock(&safe_system_lock); + ast_cancel_shutdown(); - if (aborting_shutdown) { - ast_cancel_shutdown(); - } return CLI_SUCCESS; } diff --git a/main/bridge.c b/main/bridge.c index c1fc145c6..6737fa62f 100644 --- a/main/bridge.c +++ b/main/bridge.c @@ -5322,7 +5322,7 @@ static void bridge_prnt_obj(void *v_obj, void *where, ao2_prnt_fn *prnt) /*! * \internal - * \brief Shutdown the bridging system. + * \brief Shutdown the bridging system. Stuff to always do. * \since 12.0.0 * * \return Nothing @@ -5334,6 +5334,17 @@ static void bridge_shutdown(void) ast_manager_unregister("BridgeTechnologyUnsuspend"); ast_cli_unregister_multiple(bridge_cli, ARRAY_LEN(bridge_cli)); ao2_container_unregister("bridges"); +} + +/*! + * \internal + * \brief Shutdown the bridging system. More stuff to do on graceful shutdown. + * \since 13.3.0 + * + * \return Nothing + */ +static void bridge_cleanup(void) +{ ao2_cleanup(bridges); bridges = NULL; ao2_cleanup(bridge_manager); @@ -5342,6 +5353,7 @@ static void bridge_shutdown(void) int ast_bridging_init(void) { + ast_register_cleanup(bridge_cleanup); ast_register_atexit(bridge_shutdown); if (ast_stasis_bridging_init()) { diff --git a/main/channel.c b/main/channel.c index 4bf2f61ef..da0ca558b 100644 --- a/main/channel.c +++ b/main/channel.c @@ -100,9 +100,6 @@ struct ast_epoll_data { #define MONITOR_DELAY 150 * 8 /*!< 150 ms of MONITORING DELAY */ #endif -/*! \brief Prevent new channel allocation if shutting down. */ -static int shutting_down; - static int chancount; unsigned long global_fin, global_fout; @@ -504,13 +501,9 @@ static int ast_channel_softhangup_cb(void *obj, void *arg, int flags) return 0; } -void ast_begin_shutdown(int hangup) +void ast_softhangup_all(void) { - shutting_down = 1; - - if (hangup) { - ao2_callback(channels, OBJ_NODATA | OBJ_MULTIPLE, ast_channel_softhangup_cb, NULL); - } + ao2_callback(channels, OBJ_NODATA | OBJ_MULTIPLE, ast_channel_softhangup_cb, NULL); } /*! \brief returns number of active/allocated channels */ @@ -524,18 +517,6 @@ int ast_undestroyed_channels(void) return ast_atomic_fetchadd_int(&chancount, 0); } -/*! \brief Cancel a shutdown in progress */ -void ast_cancel_shutdown(void) -{ - shutting_down = 0; -} - -/*! \brief Returns non-zero if Asterisk is being shut down */ -int ast_shutting_down(void) -{ - return shutting_down; -} - /*! \brief Set when to hangup channel */ void ast_channel_setwhentohangup_tv(struct ast_channel *chan, struct timeval offset) { diff --git a/main/http.c b/main/http.c index ef3b4b22b..3346cadb4 100644 --- a/main/http.c +++ b/main/http.c @@ -1873,6 +1873,11 @@ static int httpd_process_request(struct ast_tcptls_session_instance *ser) return -1; } + if (ast_shutdown_final()) { + ast_http_error(ser, 503, "Service Unavailable", "Shutdown in progress"); + return -1; + } + /* process "Request Headers" lines */ if (http_request_headers_get(ser, &headers)) { return -1; -- cgit v1.2.3