Restrict CLI/AMI commands on shutdown.

During stress testing, we have frequently seen crashes occur because a
CLI or AMI command attempts to access information that is in the process
of being destroyed.

When addressing how to fix this issue, we initially considered fixing
individual crashes we observed. However, the changes required to fix
those problems would introduce considerable overhead to the nominal
case. This is not reasonable in order to prevent a crash from occurring
while Asterisk is already shutting down.

Instead, this change makes it so AMI and CLI commands cannot be executed
if Asterisk is being shut down. For AMI, this is absolute. For CLI,
though, certain commands can be registered so that they may be run
during Asterisk shutdown.

ASTERISK-25825 #close

Change-Id: I8887e215ac352fadf7f4c1e082da9089b1421990

1 files changed, 51 insertions, 2 deletions

diff --git a/main/cli.c b/main/cli.c
index 0ac5d612a..f2bedc91a 100644
--- a/main/cli.c
+++ b/main/cli.c
@@ -63,6 +63,7 @@ ASTERISK_REGISTER_FILE()
 #include "asterisk/bridge.h"
 #include "asterisk/stasis_channels.h"
 #include "asterisk/stasis_bridges.h"
+#include "asterisk/vector.h"
 
 /*!
  * \brief List of restrictions per user.
@@ -109,6 +110,9 @@ static struct module_level_list debug_modules = AST_RWLIST_HEAD_INIT_VALUE;
 
 AST_THREADSTORAGE(ast_cli_buf);
 
+AST_RWLOCK_DEFINE_STATIC(shutdown_commands_lock);
+static AST_VECTOR(, struct ast_cli_entry *) shutdown_commands;
+
 /*! \brief Initial buffer size for resulting strings in ast_cli() */
 #define AST_CLI_INITLEN   256
 
@@ -2031,6 +2035,7 @@ static void cli_shutdown(void)
 /*! \brief initialize the _full_cmd string in * each of the builtins. */
 void ast_builtins_init(void)
 {
+	AST_VECTOR_INIT(&shutdown_commands, 0);
 	ast_cli_register_multiple(cli_cli, ARRAY_LEN(cli_cli));
 	ast_register_cleanup(cli_shutdown);
 }
@@ -2209,6 +2214,13 @@ static int cli_is_registered(struct ast_cli_entry *e)
 	return 0;
 }
 
+static void remove_shutdown_command(struct ast_cli_entry *e)
+{
+	ast_rwlock_wrlock(&shutdown_commands_lock);
+	AST_VECTOR_REMOVE_ELEM_UNORDERED(&shutdown_commands, e, AST_VECTOR_ELEM_CLEANUP_NOOP);
+	ast_rwlock_unlock(&shutdown_commands_lock);
+}
+
 int ast_cli_unregister(struct ast_cli_entry *e)
 {
 	if (e->inuse) {
@@ -2217,6 +2229,7 @@ int ast_cli_unregister(struct ast_cli_entry *e)
 		AST_RWLIST_WRLOCK(&helpers);
 		AST_RWLIST_REMOVE(&helpers, e, list);
 		AST_RWLIST_UNLOCK(&helpers);
+		remove_shutdown_command(e);
 		ast_free(e->_full_cmd);
 		e->_full_cmd = NULL;
 		if (e->handler) {
@@ -2675,10 +2688,27 @@ char *ast_cli_generator(const char *text, const char *word, int state)
 	return __ast_cli_generator(text, word, state, 1);
 }
 
+static int allowed_on_shutdown(struct ast_cli_entry *e)
+{
+	int found = 0;
+	int i;
+
+	ast_rwlock_rdlock(&shutdown_commands_lock);
+	for (i = 0; i < AST_VECTOR_SIZE(&shutdown_commands); ++i) {
+		if (e == AST_VECTOR_GET(&shutdown_commands, i)) {
+			found = 1;
+			break;
+		}
+	}
+	ast_rwlock_unlock(&shutdown_commands_lock);
+
+	return found;
+}
+
 int ast_cli_command_full(int uid, int gid, int fd, const char *s)
 {
 	const char *args[AST_MAX_ARGS + 1];
-	struct ast_cli_entry *e;
+	struct ast_cli_entry *e = NULL;
 	int x;
 	char *duplicate = parse_args(s, &x, args + 1, AST_MAX_ARGS, NULL);
 	char tmp[AST_MAX_ARGS + 1];
@@ -2702,6 +2732,11 @@ int ast_cli_command_full(int uid, int gid, int fd, const char *s)
 		goto done;
 	}
 
+	if (ast_shutting_down() && !allowed_on_shutdown(e)) {
+		ast_cli(fd, "Command '%s' cannot be run during shutdown\n", s);
+		goto done;
+	}
+
 	ast_join(tmp, sizeof(tmp), args + 1);
 	/* Check if the user has rights to run this command. */
 	if (!cli_has_permissions(uid, gid, tmp)) {
@@ -2724,8 +2759,11 @@ int ast_cli_command_full(int uid, int gid, int fd, const char *s)
 	} else if (retval == CLI_FAILURE) {
 		ast_cli(fd, "Command '%s' failed.\n", s);
 	}
-	ast_atomic_fetchadd_int(&e->inuse, -1);
+
 done:
+	if (e) {
+		ast_atomic_fetchadd_int(&e->inuse, -1);
+	}
 	ast_free(duplicate);
 	return retval == CLI_SUCCESS ? RESULT_SUCCESS : RESULT_FAILURE;
 }
@@ -2751,3 +2789,14 @@ void ast_cli_print_timestr_fromseconds(int fd, int seconds, const char *prefix)
 {
 	print_uptimestr(fd, ast_tv(seconds, 0), prefix, 0);
 }
+
+int ast_cli_allow_at_shutdown(struct ast_cli_entry *e)
+{
+	int res;
+
+	ast_rwlock_wrlock(&shutdown_commands_lock);
+	res = AST_VECTOR_APPEND(&shutdown_commands, e);
+	ast_rwlock_unlock(&shutdown_commands_lock);
+
+	return res;
+}