res_http_websocket: Close websocket correctly and use careful fwrite

When a client takes a long time to process information received from Asterisk,
a write operation using fwrite may fail to write all information. This causes
the underlying file stream to be in an unknown state, such that the socket
must be disconnected. Unfortunately, there are two problems with this in
Asterisk's existing websocket code:
1. Periodically, during the read loop, Asterisk must write to the connected
   websocket to respond to pings. As such, Asterisk maintains a reference to
   the session during the loop. When ast_http_websocket_write fails, it may
   cause the session to decrement its ref count, but this in and of itself
   does not break the read loop. The read loop's write, on the other hand,
   does not break the loop if it fails. This causes the socket to get in a
   'stuck' state, preventing the client from reconnecting to the server.
2. More importantly, however, is that the fwrite in ast_http_websocket_write
   fails with a large volume of data when the client takes awhile to process
   the information. When it does fail, it fails writing only a portion of
   the bytes. With some debugging, it was shown that this was failing in a
   similar fashion to ASTERISK-12767. Switching this over to ast_careful_fwrite
   with a long enough timeout solved the problem.

Note that this version of the patch, unlike r417310 in Asterisk 11, exposes
configuration options beyond just chan_sip's sip.conf. Configuration options
to configure the write timeout have also been added to pjsip.conf and ari.conf.

#ASTERISK-23917 #close
Reported by: Matt Jordan

Review: https://reviewboard.asterisk.org/r/3624/
........

Merged revisions 417310 from http://svn.asterisk.org/svn/asterisk/branches/11
........

Merged revisions 417311 from http://svn.asterisk.org/svn/asterisk/branches/12


git-svn-id: https://origsvn.digium.com/svn/asterisk/trunk@417317 65c4cc65-6c06-0410-ace0-fbb531ad65f3

16 files changed, 157 insertions, 19 deletions

diff --git a/UPGRADE.txt b/UPGRADE.txt
index 82dad8c2e..b27ae85e4 100644
--- a/UPGRADE.txt
+++ b/UPGRADE.txt
@@ -225,5 +225,13 @@ Utilities:
  - The refcounter program has been removed in favor of the refcounter.py script
    in contrib/scripts.
 
+WebSockets:
+ - Added a compatibility option for ari, chan_sip, and chan_pjsip
+   'websocket_write_timeout'. When a websocket connection exists where Asterisk
+   writes a substantial amount of data to the connected client, and the connected
+   client is slow to process the received data, the socket may be disconnected.
+   In such cases, it may be necessary to adjust this value. Default is 100 ms.
+
+
 ===========================================================
 ===========================================================
diff --git a/channels/chan_sip.c b/channels/chan_sip.c
index 5e829357f..14b460c14 100644
--- a/channels/chan_sip.c
+++ b/channels/chan_sip.c
@@ -2665,6 +2665,10 @@ static void sip_websocket_callback(struct ast_websocket *session, struct ast_var
 		goto end;
 	}
 
+	if (ast_websocket_set_timeout(session, sip_cfg.websocket_write_timeout)) {
+		goto end;
+	}
+
 	while ((res = ast_wait_for_input(ast_websocket_fd(session), -1)) > 0) {
 		char *payload;
 		uint64_t payload_len;
@@ -32009,6 +32013,12 @@ static int reload_config(enum channelreloadreason reason)
 			ast_copy_string(default_parkinglot, v->value, sizeof(default_parkinglot));
 		} else if (!strcasecmp(v->name, "refer_addheaders")) {
 			global_refer_addheaders = ast_true(v->value);
+		} else if (!strcasecmp(v->name, "websocket_write_timeout")) {
+			if (sscanf(v->value, "%30d", &sip_cfg.websocket_write_timeout) != 1
+				|| sip_cfg.websocket_write_timeout < 0) {
+				ast_log(LOG_WARNING, "'%s' is not a valid websocket_write_timeout value at line %d. Using default '%d'.\n", v->value, v->lineno, AST_DEFAULT_WEBSOCKET_WRITE_TIMEOUT);
+				sip_cfg.websocket_write_timeout = AST_DEFAULT_WEBSOCKET_WRITE_TIMEOUT;
+			}
 		}
 	}
 
diff --git a/channels/sip/include/sip.h b/channels/sip/include/sip.h
index e2ab6e19a..ab151de10 100644
--- a/channels/sip/include/sip.h
+++ b/channels/sip/include/sip.h
@@ -773,6 +773,7 @@ struct sip_settings {
 	struct ast_format_cap *caps; /*!< Supported codecs */
 	int tcp_enabled;
 	int default_max_forwards;    /*!< Default max forwards (SIP Anti-loop) */
+	int websocket_write_timeout; /*!< Socket write timeout for websocket transports, in ms */
 };
 
 struct ast_websocket;
diff --git a/configs/ari.conf.sample b/configs/ari.conf.sample
index decdddc58..59f9a44e5 100644
--- a/configs/ari.conf.sample
+++ b/configs/ari.conf.sample
@@ -1,19 +1,25 @@
 [general]
-enabled = yes		; When set to no, ARI support is disabled.
-;pretty = no		; When set to yes, responses from ARI are
-;			; formatted to be human readable.
-;allowed_origins =	; Comma separated list of allowed origins, for
-;			; Cross-Origin Resource Sharing. May be set to * to
-;			; allow all origins.
-;auth_realm =		; Realm to use for authentication. Defaults to Asterisk
-;			; REST Interface.
+enabled = yes       ; When set to no, ARI support is disabled.
+;pretty = no        ; When set to yes, responses from ARI are
+;                   ; formatted to be human readable.
+;allowed_origins =  ; Comma separated list of allowed origins, for
+;                   ; Cross-Origin Resource Sharing. May be set to * to
+;                   ; allow all origins.
+;auth_realm =       ; Realm to use for authentication. Defaults to Asterisk
+;                   ; REST Interface.
+;
+; Default write timeout to set on websockets. This value may need to be adjusted
+; for connections where Asterisk must write a substantial amount of data and the
+; receiving clients are slow to process the received information. Value is in
+; milliseconds; default is 100 ms.
+;websocket_write_timeout = 100
 
 ;[username]
-;type = user		; Specifies user configuration
-;read_only = no		; When set to yes, user is only authorized for
-;			; read-only requests.
+;type = user        ; Specifies user configuration
+;read_only = no     ; When set to yes, user is only authorized for
+;                   ; read-only requests.
 ;
-;password =		; Crypted or plaintext password (see password_format).
+;password =         ; Crypted or plaintext password (see password_format).
 ;
 ; password_format may be set to plain (the default) or crypt. When set to crypt,
 ; crypt(3) is used to validate the password. A crypted password can be generated
@@ -22,3 +28,4 @@ enabled = yes		; When set to no, ARI support is disabled.
 ; When set to plain, the password is in plaintext.
 ;
 ;password_format = plain
+
diff --git a/configs/pjsip.conf.sample b/configs/pjsip.conf.sample
index 1bcfcb96b..3aa05a96b 100644
--- a/configs/pjsip.conf.sample
+++ b/configs/pjsip.conf.sample
@@ -616,7 +616,13 @@
                 ; "")
 ;tos=0  ; Enable TOS for the signalling sent over this transport (default: "0")
 ;cos=0  ; Enable COS for the signalling sent over this transport (default: "0")
-
+;websocket_write_timeout=100    ; Default write timeout to set on websocket
+                                ; transports. This value may need to be adjusted
+                                ; for connections where Asterisk must write a
+                                ; substantial amount of data and the receiving
+                                ; clients are slow to process the received
+                                ; information. Value is in milliseconds; default
+                                ; is 100 ms.
 
 ;==========================CONTACT SECTION OPTIONS=========================
 ;[contact]
diff --git a/configs/sip.conf.sample b/configs/sip.conf.sample
index 1175047b3..010137d72 100644
--- a/configs/sip.conf.sample
+++ b/configs/sip.conf.sample
@@ -229,6 +229,12 @@ tcpbindaddr=0.0.0.0             ; IP address for TCP server to bind to (0.0.0.0
 				; unauthenticated sessions that will be allowed
                                 ; to connect at any given time. (default: 100)
 
+;websocket_write_timeout = 100  ; Default write timeout to set on websocket transports.
+                                ; This value may need to be adjusted for connections where
+                                ; Asterisk must write a substantial amount of data and the
+                                ; receiving clients are slow to process the received information.
+                                ; Value is in milliseconds; default is 100 ms.
+
 transport=udp                   ; Set the default transports.  The order determines the primary default transport.
                                 ; If tcpenable=no and the transport set is tcp, we will fallback to UDP.
 
diff --git a/include/asterisk/http_websocket.h b/include/asterisk/http_websocket.h
index 074ae1202..3e07e608b 100644
--- a/include/asterisk/http_websocket.h
+++ b/include/asterisk/http_websocket.h
@@ -24,6 +24,12 @@
 
 #include <errno.h>
 
+/*! \brief Default websocket write timeout, in ms */
+#define AST_DEFAULT_WEBSOCKET_WRITE_TIMEOUT 100
+
+/*! \brief Default websocket write timeout, in ms (as a string) */
+#define AST_DEFAULT_WEBSOCKET_WRITE_TIMEOUT_STR "100"
+
 /*!
  * \file http_websocket.h
  * \brief Support for WebSocket connections within the Asterisk HTTP server and client
@@ -324,4 +330,16 @@ AST_OPTIONAL_API(struct ast_websocket *, ast_websocket_client_create,
  */
 AST_OPTIONAL_API(const char *, ast_websocket_client_accept_protocol,
 		 (struct ast_websocket *ws), { return NULL;});
+
+/*!
+ * \brief Set the timeout on a non-blocking WebSocket session.
+ *
+ * \since 11.11.0
+ * \since 12.4.0
+ *
+ * \retval 0 on success
+ * \retval -1 on failure
+ */
+AST_OPTIONAL_API(int, ast_websocket_set_timeout, (struct ast_websocket *session, int timeout), {return -1;});
+
 #endif
diff --git a/include/asterisk/res_pjsip.h b/include/asterisk/res_pjsip.h
index c7e99aded..de654ee8d 100644
--- a/include/asterisk/res_pjsip.h
+++ b/include/asterisk/res_pjsip.h
@@ -128,6 +128,8 @@ struct ast_sip_transport {
 	unsigned int tos;
 	/*! QOS COS value */
 	unsigned int cos;
+	/*! Write timeout */
+	int write_timeout;
 };
 
 /*!
diff --git a/res/ari/ari_websockets.c b/res/ari/ari_websockets.c
index 90d6f0fdb..ff0a53c4f 100644
--- a/res/ari/ari_websockets.c
+++ b/res/ari/ari_websockets.c
@@ -56,11 +56,16 @@ struct ast_ari_websocket_session *ast_ari_websocket_session_create(
 	struct ast_websocket *ws_session, int (*validator)(struct ast_json *))
 {
 	RAII_VAR(struct ast_ari_websocket_session *, session, NULL, ao2_cleanup);
+	RAII_VAR(struct ast_ari_conf *, config, ast_ari_config_get(), ao2_cleanup);
 
 	if (ws_session == NULL) {
 		return NULL;
 	}
 
+	if (config == NULL || config->general == NULL) {
+		return NULL;
+	}
+
 	if (validator == NULL) {
 		validator = null_validator;
 	}
@@ -72,6 +77,11 @@ struct ast_ari_websocket_session *ast_ari_websocket_session_create(
 		return NULL;
 	}
 
+	if (ast_websocket_set_timeout(ws_session, config->general->write_timeout)) {
+		ast_log(LOG_WARNING, "Failed to set write timeout %d on ARI web socket\n",
+			config->general->write_timeout);
+	}
+
 	session = ao2_alloc(sizeof(*session), websocket_session_dtor);
 	if (!session) {
 		return NULL;
diff --git a/res/ari/config.c b/res/ari/config.c
index 59c4d7d94..667d91ac0 100644
--- a/res/ari/config.c
+++ b/res/ari/config.c
@@ -27,6 +27,7 @@
 ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
 
 #include "asterisk/config_options.h"
+#include "asterisk/http_websocket.h"
 #include "internal.h"
 
 /*! \brief Locking container for safe configuration access. */
@@ -320,6 +321,9 @@ int ast_ari_config_init(void)
 	aco_option_register(&cfg_info, "allowed_origins", ACO_EXACT, general_options,
 		"", OPT_STRINGFIELD_T, 0,
 		STRFLDSET(struct ast_ari_conf_general, allowed_origins));
+	aco_option_register(&cfg_info, "websocket_write_timeout", ACO_EXACT, general_options,
+		AST_DEFAULT_WEBSOCKET_WRITE_TIMEOUT_STR, OPT_INT_T, PARSE_IN_RANGE,
+		FLDSET(struct ast_ari_conf_general, write_timeout), 1, INT_MAX);
 
 	aco_option_register(&cfg_info, "type", ACO_EXACT, user, NULL,
 		OPT_NOOP_T, 0, 0);
diff --git a/res/ari/internal.h b/res/ari/internal.h
index 8453747f1..93ea0b773 100644
--- a/res/ari/internal.h
+++ b/res/ari/internal.h
@@ -65,6 +65,8 @@ struct ast_ari_conf {
 struct ast_ari_conf_general {
 	/*! Enabled by default, disabled if false. */
 	int enabled;
+	/*! Write timeout for websocket connections */
+	int write_timeout;
 	/*! Encoding format used during output (default compact). */
 	enum ast_json_encoding_format format;
 	/*! Authentication realm */
diff --git a/res/res_ari.c b/res/res_ari.c
index ce7027e44..acdbbfe9a 100644
--- a/res/res_ari.c
+++ b/res/res_ari.c
@@ -95,6 +95,14 @@
 						<ref type="link">https://wiki.asterisk.org/wiki/display/AST/Asterisk+Builtin+mini-HTTP+Server</ref>
 					</see-also>
 				</configOption>
+				<configOption name="websocket_write_timeout">
+					<synopsis>The timeout (in milliseconds) to set on WebSocket connections.</synopsis>
+					<description>
+						<para>If a websocket connection accepts input slowly, the timeout
+						for writes to it can be increased to keep it from being disconnected.
+						Value is in milliseconds; default is 100 ms.</para>
+					</description>
+				</configOption>
 				<configOption name="pretty">
 					<synopsis>Responses from ARI are formatted to be human readable</synopsis>
 				</configOption>
diff --git a/res/res_http_websocket.c b/res/res_http_websocket.c
index 07fcd9e2e..90744a1b3 100644
--- a/res/res_http_websocket.c
+++ b/res/res_http_websocket.c
@@ -81,6 +81,7 @@ struct ast_websocket {
 	size_t payload_len;               /*!< Length of the payload */
 	char *payload;                    /*!< Pointer to the payload */
 	size_t reconstruct;               /*!< Number of bytes before a reconstructed payload will be returned and a new one started */
+	int timeout;                      /*!< The timeout for operations on the socket */
 	unsigned int secure:1;            /*!< Bit to indicate that the transport is secure */
 	unsigned int closing:1;           /*!< Bit to indicate that the session is in the process of being closed */
 	unsigned int close_sent:1;        /*!< Bit to indicate that the session close opcode has been sent and no further data will be sent */
@@ -260,7 +261,7 @@ int AST_OPTIONAL_API_NAME(ast_websocket_close)(struct ast_websocket *session, ui
 	session->close_sent = 1;
 
 	ao2_lock(session);
-	res = (fwrite(frame, 1, 4, session->f) == 4) ? 0 : -1;
+	res = ast_careful_fwrite(session->f, session->fd, frame, 4, session->timeout);
 	ao2_unlock(session);
 	return res;
 }
@@ -303,13 +304,12 @@ int AST_OPTIONAL_API_NAME(ast_websocket_write)(struct ast_websocket *session, en
 		ao2_unlock(session);
 		return -1;
 	}
-
-	if (fwrite(frame, 1, header_size, session->f) != header_size) {
+	if (ast_careful_fwrite(session->f, session->fd, frame, header_size, session->timeout)) {
 		ao2_unlock(session);
 		return -1;
 	}
 
-	if (fwrite(payload, 1, actual_length, session->f) != actual_length) {
+	if (ast_careful_fwrite(session->f, session->fd, payload, actual_length, session->timeout)) {
 		ao2_unlock(session);
 		return -1;
 	}
@@ -371,6 +371,13 @@ int AST_OPTIONAL_API_NAME(ast_websocket_set_nonblock)(struct ast_websocket *sess
 	return 0;
 }
 
+int AST_OPTIONAL_API_NAME(ast_websocket_set_timeout)(struct ast_websocket *session, int timeout)
+{
+	session->timeout = timeout;
+
+	return 0;
+}
+
 /* MAINTENANCE WARNING on ast_websocket_read()!
  *
  * We have to keep in mind during this function that the fact that session->fd seems ready
@@ -514,8 +521,10 @@ int AST_OPTIONAL_API_NAME(ast_websocket_read)(struct ast_websocket *session, cha
 		}
 
 		/* Per the RFC for PING we need to send back an opcode with the application data as received */
-		if (*opcode == AST_WEBSOCKET_OPCODE_PING) {
-			ast_websocket_write(session, AST_WEBSOCKET_OPCODE_PONG, *payload, *payload_len);
+		if ((*opcode == AST_WEBSOCKET_OPCODE_PING) && (ast_websocket_write(session, AST_WEBSOCKET_OPCODE_PONG, *payload, *payload_len))) {
+			*payload_len = 0;
+			ast_websocket_close(session, 1009);
+			return 0;
 		}
 
 		session->payload = new_payload;
@@ -696,6 +705,7 @@ int AST_OPTIONAL_API_NAME(ast_websocket_uri_cb)(struct ast_tcptls_session_instan
 			ao2_ref(protocol_handler, -1);
 			return 0;
 		}
+		session->timeout =  AST_DEFAULT_WEBSOCKET_WRITE_TIMEOUT;
 
 		fprintf(ser->f, "HTTP/1.1 101 Switching Protocols\r\n"
 			"Upgrade: %s\r\n"
diff --git a/res/res_pjsip.c b/res/res_pjsip.c
index 2602660ee..45b8e7e02 100644
--- a/res/res_pjsip.c
+++ b/res/res_pjsip.c
@@ -869,6 +869,14 @@
 					or the <replaceable>wss</replaceable> protocols.</para></note>
 					</description>
 				</configOption>
+				<configOption name="websocket_write_timeout">
+					<synopsis>The timeout (in milliseconds) to set on WebSocket connections.</synopsis>
+					<description>
+						<para>If a websocket connection accepts input slowly, the timeout
+						for writes to it can be increased to keep it from being disconnected.
+						Value is in milliseconds; default is 100 ms.</para>
+					</description>
+				</configOption>
 			</configObject>
 			<configObject name="contact">
 				<synopsis>A way of creating an aliased name to a SIP URI</synopsis>
diff --git a/res/res_pjsip/config_transport.c b/res/res_pjsip/config_transport.c
index 22581ca52..785fcc5ac 100644
--- a/res/res_pjsip/config_transport.c
+++ b/res/res_pjsip/config_transport.c
@@ -28,6 +28,7 @@
 #include "asterisk/sorcery.h"
 #include "asterisk/acl.h"
 #include "include/res_pjsip_private.h"
+#include "asterisk/http_websocket.h"
 
 static int sip_transport_to_ami(const struct ast_sip_transport *transport,
 				struct ast_str **buf)
@@ -668,6 +669,7 @@ int ast_sip_initialize_sorcery_transport(void)
 	ast_sorcery_object_field_register_custom(sorcery, "transport", "local_net", "", transport_localnet_handler, localnet_to_str, localnet_to_vl, 0, 0);
 	ast_sorcery_object_field_register_custom(sorcery, "transport", "tos", "0", transport_tos_handler, tos_to_str, NULL, 0, 0);
 	ast_sorcery_object_field_register(sorcery, "transport", "cos", "0", OPT_UINT_T, 0, FLDSET(struct ast_sip_transport, cos));
+	ast_sorcery_object_field_register(sorcery, "transport", "websocket_write_timeout", AST_DEFAULT_WEBSOCKET_WRITE_TIMEOUT_STR, OPT_INT_T, PARSE_IN_RANGE, FLDSET(struct ast_sip_transport, write_timeout), 1, INT_MAX);
 
 	ast_sip_register_endpoint_formatter(&endpoint_transport_formatter);
 
diff --git a/res/res_pjsip_transport_websocket.c b/res/res_pjsip_transport_websocket.c
index 22962dab0..bae120a19 100644
--- a/res/res_pjsip_transport_websocket.c
+++ b/res/res_pjsip_transport_websocket.c
@@ -207,6 +207,37 @@ static int transport_read(void *data)
 	return (read_data->payload_len == recvd) ? 0 : -1;
 }
 
+static int get_write_timeout(void)
+{
+	int write_timeout = -1;
+	struct ao2_container *transports;
+
+	transports = ast_sorcery_retrieve_by_fields(ast_sip_get_sorcery(), "transport", AST_RETRIEVE_FLAG_ALL, NULL);
+
+	if (transports) {
+		struct ao2_iterator it_transports = ao2_iterator_init(transports, 0);
+		struct ast_sip_transport *transport;
+
+		for (; (transport = ao2_iterator_next(&it_transports)); ao2_cleanup(transport)) {
+			if (transport->type != AST_TRANSPORT_WS && transport->type != AST_TRANSPORT_WSS) {
+				continue;
+			}
+			ast_debug(5, "Found %s transport with write timeout: %d\n",
+				transport->type == AST_TRANSPORT_WS ? "WS" : "WSS",
+				transport->write_timeout);
+			write_timeout = MAX(write_timeout, transport->write_timeout);
+		}
+		ao2_cleanup(transports);
+	}
+
+	if (write_timeout < 0) {
+		write_timeout = AST_DEFAULT_WEBSOCKET_WRITE_TIMEOUT;
+	}
+
+	ast_debug(1, "Write timeout for WS/WSS transports: %d\n", write_timeout);
+	return write_timeout;
+}
+
 /*!
  \brief WebSocket connection handler.
  */
@@ -222,6 +253,11 @@ static void websocket_cb(struct ast_websocket *session, struct ast_variable *par
 		return;
 	}
 
+	if (ast_websocket_set_timeout(session, get_write_timeout())) {
+		ast_websocket_unref(session);
+		return;
+	}
+
 	if (!(serializer = ast_sip_create_serializer())) {
 		ast_websocket_unref(session);
 		return;