Tested new ioqueue framework on Linux with select and epoll

git-svn-id: http://svn.pjsip.org/repos/pjproject/main@14 74dad513-b988-da41-8d7b-12977e46ad98

5 files changed, 265 insertions, 265 deletions

diff --git a/pjlib/include/pj/compat/os_linux.h b/pjlib/include/pj/compat/os_linux.h
index a2f97f65..efb661a4 100644
--- a/pjlib/include/pj/compat/os_linux.h
+++ b/pjlib/include/pj/compat/os_linux.h
@@ -54,22 +54,22 @@
 #define PJ_HAS_WINSOCK_H	    0
 #define PJ_HAS_WINSOCK2_H	    0
 
-#define PJ_SOCK_HAS_INET_ATON	    1

-

-/* When this macro is set, getsockopt(SOL_SOCKET, SO_ERROR) will return

- * the status of non-blocking connect() operation.

+#define PJ_SOCK_HAS_INET_ATON	    1
+
+/* When this macro is set, getsockopt(SOL_SOCKET, SO_ERROR) will return
+ * the status of non-blocking connect() operation.
+ */
+#define PJ_HAS_SO_ERROR             1
+
+/* This value specifies the value set in errno by the OS when a non-blocking
+ * socket recv() can not return immediate daata.
+ */
+#define PJ_BLOCKING_ERROR_VAL       EAGAIN
+
+/* This value specifies the value set in errno by the OS when a non-blocking
+ * socket connect() can not get connected immediately.
  */
-#define PJ_HAS_SO_ERROR             1

-

-/* This value specifies the value set in errno by the OS when a non-blocking

- * socket recv() can not return immediate daata.

- */

-#define PJ_BLOCKING_ERROR_VAL       EAGAIN

-

-/* This value specifies the value set in errno by the OS when a non-blocking

- * socket connect() can not get connected immediately.

- */

-#define PJ_BLOCKING_CONNECT_ERROR_VAL   EINPROGRESS

+#define PJ_BLOCKING_CONNECT_ERROR_VAL   EINPROGRESS
 
 /* Default threading is enabled, unless it's overridden. */
 #ifndef PJ_HAS_THREADS
diff --git a/pjlib/include/pj/compat/os_linux_kernel.h b/pjlib/include/pj/compat/os_linux_kernel.h
index 0d44ef0e..ccae3418 100644
--- a/pjlib/include/pj/compat/os_linux_kernel.h
+++ b/pjlib/include/pj/compat/os_linux_kernel.h
@@ -52,21 +52,21 @@
 #define PJ_HAS_WINSOCK2_H	    0
 
 #define PJ_SOCK_HAS_INET_ATON	    0
-

-/* When this macro is set, getsockopt(SOL_SOCKET, SO_ERROR) will return

- * the status of non-blocking connect() operation.

- */

-#define PJ_HAS_SO_ERROR             1

-

-/* This value specifies the value set in errno by the OS when a non-blocking

- * socket recv() can not return immediate daata.

- */

-#define PJ_BLOCKING_ERROR_VAL       EAGAIN

-

-/* This value specifies the value set in errno by the OS when a non-blocking

- * socket connect() can not get connected immediately.

- */

-#define PJ_BLOCKING_CONNECT_ERROR_VAL   EINPROGRESS

+
+/* When this macro is set, getsockopt(SOL_SOCKET, SO_ERROR) will return
+ * the status of non-blocking connect() operation.
+ */
+#define PJ_HAS_SO_ERROR             1
+
+/* This value specifies the value set in errno by the OS when a non-blocking
+ * socket recv() can not return immediate daata.
+ */
+#define PJ_BLOCKING_ERROR_VAL       EAGAIN
+
+/* This value specifies the value set in errno by the OS when a non-blocking
+ * socket connect() can not get connected immediately.
+ */
+#define PJ_BLOCKING_CONNECT_ERROR_VAL   EINPROGRESS
 
 #ifndef PJ_HAS_THREADS
 #  define PJ_HAS_THREADS	    (1)
diff --git a/pjlib/include/pj/compat/os_sunos.h b/pjlib/include/pj/compat/os_sunos.h
index 990fb57d..87c408ab 100644
--- a/pjlib/include/pj/compat/os_sunos.h
+++ b/pjlib/include/pj/compat/os_sunos.h
@@ -39,21 +39,21 @@
 #define PJ_HAS_WINSOCK2_H	    0
 
 #define PJ_SOCK_HAS_INET_ATON	    0
-

-/* When this macro is set, getsockopt(SOL_SOCKET, SO_ERROR) will return

- * the status of non-blocking connect() operation.

- */

-#define PJ_HAS_SO_ERROR             0

-

-/* This value specifies the value set in errno by the OS when a non-blocking

- * socket recv() can not return immediate daata.

- */

-#define PJ_BLOCKING_ERROR_VAL       EWOULDBLOCK

-

-/* This value specifies the value set in errno by the OS when a non-blocking

- * socket connect() can not get connected immediately.

- */

-#define PJ_BLOCKING_CONNECT_ERROR_VAL   EINPROGRESS

+
+/* When this macro is set, getsockopt(SOL_SOCKET, SO_ERROR) will return
+ * the status of non-blocking connect() operation.
+ */
+#define PJ_HAS_SO_ERROR             0
+
+/* This value specifies the value set in errno by the OS when a non-blocking
+ * socket recv() can not return immediate daata.
+ */
+#define PJ_BLOCKING_ERROR_VAL       EWOULDBLOCK
+
+/* This value specifies the value set in errno by the OS when a non-blocking
+ * socket connect() can not get connected immediately.
+ */
+#define PJ_BLOCKING_CONNECT_ERROR_VAL   EINPROGRESS
 
 /* Default threading is enabled, unless it's overridden. */
 #ifndef PJ_HAS_THREADS
diff --git a/pjlib/include/pj/compat/os_win32.h b/pjlib/include/pj/compat/os_win32.h
index 87ff7520..e8391b94 100644
--- a/pjlib/include/pj/compat/os_win32.h
+++ b/pjlib/include/pj/compat/os_win32.h
@@ -59,21 +59,21 @@
 #define PJ_HAS_WINSOCK2_H	    1
 
 #define PJ_SOCK_HAS_INET_ATON	    0
-

-/* When this macro is set, getsockopt(SOL_SOCKET, SO_ERROR) will return

- * the status of non-blocking connect() operation.

- */

-#define PJ_HAS_SO_ERROR             0

-

-/* This value specifies the value set in errno by the OS when a non-blocking

- * socket recv() or send() can not return immediately.

- */

-#define PJ_BLOCKING_ERROR_VAL       WSAEWOULDBLOCK

-

-/* This value specifies the value set in errno by the OS when a non-blocking

- * socket connect() can not get connected immediately.

- */

-#define PJ_BLOCKING_CONNECT_ERROR_VAL   WSAEWOULDBLOCK

+
+/* When this macro is set, getsockopt(SOL_SOCKET, SO_ERROR) will return
+ * the status of non-blocking connect() operation.
+ */
+#define PJ_HAS_SO_ERROR             0
+
+/* This value specifies the value set in errno by the OS when a non-blocking
+ * socket recv() or send() can not return immediately.
+ */
+#define PJ_BLOCKING_ERROR_VAL       WSAEWOULDBLOCK
+
+/* This value specifies the value set in errno by the OS when a non-blocking
+ * socket connect() can not get connected immediately.
+ */
+#define PJ_BLOCKING_CONNECT_ERROR_VAL   WSAEWOULDBLOCK
 
 /* Default threading is enabled, unless it's overridden. */
 #ifndef PJ_HAS_THREADS
diff --git a/pjlib/include/pj/ioqueue.h b/pjlib/include/pj/ioqueue.h
index 2e084fd3..ce30c9f9 100644
--- a/pjlib/include/pj/ioqueue.h
+++ b/pjlib/include/pj/ioqueue.h
@@ -47,53 +47,53 @@ PJ_BEGIN_DECL
  * @ingroup PJ_IO
  * @{
  *
- * I/O Queue provides API for performing asynchronous I/O operations. It

- * conforms to proactor pattern, which allows application to submit an

- * asynchronous operation and to be notified later when the operation has

+ * I/O Queue provides API for performing asynchronous I/O operations. It
+ * conforms to proactor pattern, which allows application to submit an
+ * asynchronous operation and to be notified later when the operation has
  * completed.
- *

- * The framework works natively in platforms where asynchronous operation API

- * exists, such as in Windows NT with IoCompletionPort/IOCP. In other 

- * platforms, the I/O queue abstracts the operating system's event poll API

- * to provide semantics similar to IoCompletionPort with minimal penalties

- * (i.e. per ioqueue and per handle mutex protection).

- *

- * The I/O queue provides more than just unified abstraction. It also:

- *  - makes sure that the operation uses the most effective way to utilize

- *    the underlying mechanism, to provide the maximum theoritical

- *    throughput possible on a given platform.

- *  - choose the most efficient mechanism for event polling on a given

- *    platform.

+ *
+ * The framework works natively in platforms where asynchronous operation API
+ * exists, such as in Windows NT with IoCompletionPort/IOCP. In other 
+ * platforms, the I/O queue abstracts the operating system's event poll API
+ * to provide semantics similar to IoCompletionPort with minimal penalties
+ * (i.e. per ioqueue and per handle mutex protection).
+ *
+ * The I/O queue provides more than just unified abstraction. It also:
+ *  - makes sure that the operation uses the most effective way to utilize
+ *    the underlying mechanism, to provide the maximum theoritical
+ *    throughput possible on a given platform.
+ *  - choose the most efficient mechanism for event polling on a given
+ *    platform.
  *
  * Currently, the I/O Queue is implemented using:
- *  - <tt><b>select()</b></tt>, as the common denominator, but the least 

- *    efficient. Also the number of descriptor is limited to 

- *    \c PJ_IOQUEUE_MAX_HANDLES (which by default is 64).

- *  - <tt><b>/dev/epoll</b></tt> on Linux (user mode and kernel mode), 

- *    a much faster replacement for select() on Linux (and more importantly

+ *  - <tt><b>select()</b></tt>, as the common denominator, but the least 
+ *    efficient. Also the number of descriptor is limited to 
+ *    \c PJ_IOQUEUE_MAX_HANDLES (which by default is 64).
+ *  - <tt><b>/dev/epoll</b></tt> on Linux (user mode and kernel mode), 
+ *    a much faster replacement for select() on Linux (and more importantly
  *    doesn't have limitation on number of descriptors).
- *  - <b>I/O Completion ports</b> on Windows NT/2000/XP, which is the most 

- *    efficient way to dispatch events in Windows NT based OSes, and most 

- *    importantly, it doesn't have the limit on how many handles to monitor.

+ *  - <b>I/O Completion ports</b> on Windows NT/2000/XP, which is the most 
+ *    efficient way to dispatch events in Windows NT based OSes, and most 
+ *    importantly, it doesn't have the limit on how many handles to monitor.
  *    And it works with files (not only sockets) as well.
- *

- *

- * \section pj_ioqueue_concurrency_sec Concurrency Rules

- *

- * The items below describe rules that must be obeyed when using the I/O 

- * queue, with regard to concurrency:

- *  - simultaneous operations (by different threads) to different key is safe.

- *  - simultaneous operations to the same key is also safe, except

- *    <b>unregistration</b>, which is described below.

- *  - <b>care must be taken when unregistering a key</b> from the

- *    ioqueue. Application must take care that when one thread is issuing

- *    an unregistration, other thread is not simultaneously invoking an

- *    operation <b>to the same key</b>.

- *\n

- *    This happens because the ioqueue functions are working with a pointer

- *    to the key, and there is a possible race condition where the pointer

- *    has been rendered invalid by other threads before the ioqueue has a

- *    chance to acquire mutex on it.

+ *
+ *
+ * \section pj_ioqueue_concurrency_sec Concurrency Rules
+ *
+ * The items below describe rules that must be obeyed when using the I/O 
+ * queue, with regard to concurrency:
+ *  - simultaneous operations (by different threads) to different key is safe.
+ *  - simultaneous operations to the same key is also safe, except
+ *    <b>unregistration</b>, which is described below.
+ *  - <b>care must be taken when unregistering a key</b> from the
+ *    ioqueue. Application must take care that when one thread is issuing
+ *    an unregistration, other thread is not simultaneously invoking an
+ *    operation <b>to the same key</b>.
+ *\n
+ *    This happens because the ioqueue functions are working with a pointer
+ *    to the key, and there is a possible race condition where the pointer
+ *    has been rendered invalid by other threads before the ioqueue has a
+ *    chance to acquire mutex on it.
  *
  * \section pj_ioqeuue_examples_sec Examples
  *
@@ -103,26 +103,26 @@ PJ_BEGIN_DECL
  *  - \ref page_pjlib_ioqueue_udp_test
  *  - \ref page_pjlib_ioqueue_perf_test
  */
-

-

-

-/**

- * This structure describes operation specific key to be submitted to

- * I/O Queue when performing the asynchronous operation. This key will

- * be returned to the application when completion callback is called.

- *

- * Application normally wants to attach it's specific data in the

- * \c user_data field so that it can keep track of which operation has

- * completed when the callback is called. Alternatively, application can

- * also extend this struct to include its data, because the pointer that

- * is returned in the completion callback will be exactly the same as

- * the pointer supplied when the asynchronous function is called.

- */

-typedef struct pj_ioqueue_op_key_t

-{ 

-    void *internal__[32];           /**< Internal I/O Queue data.   */

-    void *user_data;                /**< Application data.          */

-} pj_ioqueue_op_key_t;

+
+
+
+/**
+ * This structure describes operation specific key to be submitted to
+ * I/O Queue when performing the asynchronous operation. This key will
+ * be returned to the application when completion callback is called.
+ *
+ * Application normally wants to attach it's specific data in the
+ * \c user_data field so that it can keep track of which operation has
+ * completed when the callback is called. Alternatively, application can
+ * also extend this struct to include its data, because the pointer that
+ * is returned in the completion callback will be exactly the same as
+ * the pointer supplied when the asynchronous function is called.
+ */
+typedef struct pj_ioqueue_op_key_t
+{ 
+    void *internal__[32];           /**< Internal I/O Queue data.   */
+    void *user_data;                /**< Application data.          */
+} pj_ioqueue_op_key_t;
 
 /**
  * This structure describes the callbacks to be called when I/O operation
@@ -134,58 +134,58 @@ typedef struct pj_ioqueue_callback
      * This callback is called when #pj_ioqueue_recv or #pj_ioqueue_recvfrom
      * completes.
      *
-     * @param key	    The key.

+     * @param key	    The key.
      * @param op_key        Operation key.
-     * @param bytes_read    >= 0 to indicate the amount of data read, 

-     *                      otherwise negative value containing the error

-     *                      code. To obtain the pj_status_t error code, use

+     * @param bytes_read    >= 0 to indicate the amount of data read, 
+     *                      otherwise negative value containing the error
+     *                      code. To obtain the pj_status_t error code, use
      *                      (pj_status_t code = -bytes_read).
      */
-    void (*on_read_complete)(pj_ioqueue_key_t *key, 

-                             pj_ioqueue_op_key_t *op_key, 

+    void (*on_read_complete)(pj_ioqueue_key_t *key, 
+                             pj_ioqueue_op_key_t *op_key, 
                              pj_ssize_t bytes_read);
 
     /**
      * This callback is called when #pj_ioqueue_write or #pj_ioqueue_sendto
      * completes.
      *
-     * @param key	    The key.

+     * @param key	    The key.
      * @param op_key        Operation key.
-     * @param bytes_sent    >= 0 to indicate the amount of data written, 

-     *                      otherwise negative value containing the error

-     *                      code. To obtain the pj_status_t error code, use

-     *                      (pj_status_t code = -bytes_sent).

+     * @param bytes_sent    >= 0 to indicate the amount of data written, 
+     *                      otherwise negative value containing the error
+     *                      code. To obtain the pj_status_t error code, use
+     *                      (pj_status_t code = -bytes_sent).
      */
-    void (*on_write_complete)(pj_ioqueue_key_t *key, 

-                              pj_ioqueue_op_key_t *op_key, 

+    void (*on_write_complete)(pj_ioqueue_key_t *key, 
+                              pj_ioqueue_op_key_t *op_key, 
                               pj_ssize_t bytes_sent);
 
     /**
      * This callback is called when #pj_ioqueue_accept completes.
      *
-     * @param key	    The key.

+     * @param key	    The key.
      * @param op_key        Operation key.
      * @param sock          Newly connected socket.
      * @param status	    Zero if the operation completes successfully.
      */
-    void (*on_accept_complete)(pj_ioqueue_key_t *key, 

-                               pj_ioqueue_op_key_t *op_key, 

+    void (*on_accept_complete)(pj_ioqueue_key_t *key, 
+                               pj_ioqueue_op_key_t *op_key, 
                                pj_sock_t sock, 
                                pj_status_t status);
 
     /**
      * This callback is called when #pj_ioqueue_connect completes.
      *
-     * @param key	    The key.

+     * @param key	    The key.
      * @param status	    PJ_SUCCESS if the operation completes successfully.
      */
-    void (*on_connect_complete)(pj_ioqueue_key_t *key, 

+    void (*on_connect_complete)(pj_ioqueue_key_t *key, 
                                 pj_status_t status);
 } pj_ioqueue_callback;
 
 
 /**
- * Types of pending I/O Queue operation. This enumeration is only used

+ * Types of pending I/O Queue operation. This enumeration is only used
  * internally within the ioqueue.
  */
 typedef enum pj_ioqueue_operation_e
@@ -204,17 +204,17 @@ typedef enum pj_ioqueue_operation_e
 } pj_ioqueue_operation_e;
 
 
-/**

- * This macro specifies the maximum number of events that can be

- * processed by the ioqueue on a single poll cycle, on implementation

- * that supports it. The value is only meaningfull when specified

- * during PJLIB build.

- */

-#ifndef PJ_IOQUEUE_MAX_EVENTS_IN_SINGLE_POLL

-#   define PJ_IOQUEUE_MAX_EVENTS_IN_SINGLE_POLL     (16)

-#endif

+/**
+ * This macro specifies the maximum number of events that can be
+ * processed by the ioqueue on a single poll cycle, on implementation
+ * that supports it. The value is only meaningfull when specified
+ * during PJLIB build.
+ */
+#ifndef PJ_IOQUEUE_MAX_EVENTS_IN_SINGLE_POLL
+#   define PJ_IOQUEUE_MAX_EVENTS_IN_SINGLE_POLL     (16)
+#endif
+
 
-

 /**
  * Create a new I/O Queue framework.
  *
@@ -270,9 +270,9 @@ PJ_DECL(pj_status_t) pj_ioqueue_set_lock( pj_ioqueue_t *ioque,
  * @param sock	    The socket.
  * @param user_data User data to be associated with the key, which can be
  *		    retrieved later.
- * @param cb	    Callback to be called when I/O operation completes. 

- * @param key       Pointer to receive the key to be associated with this

- *                  socket. Subsequent I/O queue operation will need this

+ * @param cb	    Callback to be called when I/O operation completes. 
+ * @param key       Pointer to receive the key to be associated with this
+ *                  socket. Subsequent I/O queue operation will need this
  *                  key.
  *
  * @return	    PJ_SUCCESS on success, or the error code.
@@ -281,17 +281,17 @@ PJ_DECL(pj_status_t) pj_ioqueue_register_sock( pj_pool_t *pool,
 					       pj_ioqueue_t *ioque,
 					       pj_sock_t sock,
 					       void *user_data,
-					       const pj_ioqueue_callback *cb,

+					       const pj_ioqueue_callback *cb,
                                                pj_ioqueue_key_t **key );
 
 /**
- * Unregister from the I/O Queue framework. Caller must make sure that

- * the key doesn't have any pending operation before calling this function,

- * or otherwise the behaviour is undefined (either callback will be called

- * later when the data is sent/received, or the callback will not be called,

+ * Unregister from the I/O Queue framework. Caller must make sure that
+ * the key doesn't have any pending operation before calling this function,
+ * or otherwise the behaviour is undefined (either callback will be called
+ * later when the data is sent/received, or the callback will not be called,
  * or even something else).
  *
- * @param key	    The key that was previously obtained from registration.

+ * @param key	    The key that was previously obtained from registration.
  *
  * @return          PJ_SUCCESS on success or the error code.
  */
@@ -300,44 +300,44 @@ PJ_DECL(pj_status_t) pj_ioqueue_unregister( pj_ioqueue_key_t *key );
 
 /**
  * Get user data associated with an ioqueue key.
- *

- * @param key	    The key that was previously obtained from registration.

  *
- * @return          The user data associated with the descriptor, or NULL 

+ * @param key	    The key that was previously obtained from registration.
+ *
+ * @return          The user data associated with the descriptor, or NULL 
  *                  on error or if no data is associated with the key during
  *                  registration.
  */
 PJ_DECL(void*) pj_ioqueue_get_user_data( pj_ioqueue_key_t *key );
-

-/**

- * Set or change the user data to be associated with the file descriptor or

- * handle or socket descriptor.

- *

- * @param key	    The key that was previously obtained from registration.

- * @param user_data User data to be associated with the descriptor.

- * @param old_data  Optional parameter to retrieve the old user data.

- *

- * @return          PJ_SUCCESS on success or the error code.

- */

-PJ_DECL(pj_status_t) pj_ioqueue_set_user_data( pj_ioqueue_key_t *key,

-                                               void *user_data,

-                                               void **old_data);

+
+/**
+ * Set or change the user data to be associated with the file descriptor or
+ * handle or socket descriptor.
+ *
+ * @param key	    The key that was previously obtained from registration.
+ * @param user_data User data to be associated with the descriptor.
+ * @param old_data  Optional parameter to retrieve the old user data.
+ *
+ * @return          PJ_SUCCESS on success or the error code.
+ */
+PJ_DECL(pj_status_t) pj_ioqueue_set_user_data( pj_ioqueue_key_t *key,
+                                               void *user_data,
+                                               void **old_data);
 
 
 #if defined(PJ_HAS_TCP) && PJ_HAS_TCP != 0
 /**
  * Instruct I/O Queue to accept incoming connection on the specified 
- * listening socket. This function will return immediately (i.e. non-blocking)

- * regardless whether a connection is immediately available. If the function

- * can't complete immediately, the caller will be notified about the incoming

- * connection when it calls pj_ioqueue_poll(). If a new connection is

- * immediately available, the function returns PJ_SUCCESS with the new

+ * listening socket. This function will return immediately (i.e. non-blocking)
+ * regardless whether a connection is immediately available. If the function
+ * can't complete immediately, the caller will be notified about the incoming
+ * connection when it calls pj_ioqueue_poll(). If a new connection is
+ * immediately available, the function returns PJ_SUCCESS with the new
  * connection; in this case, the callback WILL NOT be called.
  *
- * @param key	    The key which registered to the server socket.

- * @param op_key    An operation specific key to be associated with the

- *                  pending operation, so that application can keep track of

- *                  which operation has been completed when the callback is

+ * @param key	    The key which registered to the server socket.
+ * @param op_key    An operation specific key to be associated with the
+ *                  pending operation, so that application can keep track of
+ *                  which operation has been completed when the callback is
  *                  called.
  * @param new_sock  Argument which contain pointer to receive the new socket
  *                  for the incoming connection.
@@ -349,15 +349,15 @@ PJ_DECL(pj_status_t) pj_ioqueue_set_user_data( pj_ioqueue_key_t *key,
  *		    address, and on output, contains the actual length of the
  *		    address. This argument is optional.
  * @return
- *  - PJ_SUCCESS    When connection is available immediately, and the 

- *                  parameters will be updated to contain information about 

- *                  the new connection. In this case, a completion callback

+ *  - PJ_SUCCESS    When connection is available immediately, and the 
+ *                  parameters will be updated to contain information about 
+ *                  the new connection. In this case, a completion callback
  *                  WILL NOT be called.
- *  - PJ_EPENDING   If no connection is available immediately. When a new

+ *  - PJ_EPENDING   If no connection is available immediately. When a new
  *                  connection arrives, the callback will be called.
  *  - non-zero      which indicates the appropriate error code.
  */
-PJ_DECL(pj_status_t) pj_ioqueue_accept( pj_ioqueue_key_t *key,

+PJ_DECL(pj_status_t) pj_ioqueue_accept( pj_ioqueue_key_t *key,
                                         pj_ioqueue_op_key_t *op_key,
 					pj_sock_t *sock,
 					pj_sockaddr_t *local,
@@ -366,9 +366,9 @@ PJ_DECL(pj_status_t) pj_ioqueue_accept( pj_ioqueue_key_t *key,
 
 /**
  * Initiate non-blocking socket connect. If the socket can NOT be connected
- * immediately, asynchronous connect() will be scheduled and caller will be

- * notified via completion callback when it calls pj_ioqueue_poll(). If

- * socket is connected immediately, the function returns PJ_SUCCESS and

+ * immediately, asynchronous connect() will be scheduled and caller will be
+ * notified via completion callback when it calls pj_ioqueue_poll(). If
+ * socket is connected immediately, the function returns PJ_SUCCESS and
  * completion callback WILL NOT be called.
  *
  * @param key	    The key associated with TCP socket
@@ -376,7 +376,7 @@ PJ_DECL(pj_status_t) pj_ioqueue_accept( pj_ioqueue_key_t *key,
  * @param addrlen   The remote address length.
  *
  * @return
- *  - PJ_SUCCESS    If socket is connected immediately. In this case, the

+ *  - PJ_SUCCESS    If socket is connected immediately. In this case, the
  *                  completion callback WILL NOT be called.
  *  - PJ_EPENDING   If operation is queued, or 
  *  - non-zero      Indicates the error code.
@@ -404,40 +404,40 @@ PJ_DECL(int) pj_ioqueue_poll( pj_ioqueue_t *ioque,
 
 
 /**
- * Instruct the I/O Queue to read from the specified handle. This function

- * returns immediately (i.e. non-blocking) regardless whether some data has 

- * been transfered. If the operation can't complete immediately, caller will 

- * be notified about the completion when it calls pj_ioqueue_poll(). If data

- * is immediately available, the function will return PJ_SUCCESS and the

- * callback WILL NOT be called.

+ * Instruct the I/O Queue to read from the specified handle. This function
+ * returns immediately (i.e. non-blocking) regardless whether some data has 
+ * been transfered. If the operation can't complete immediately, caller will 
+ * be notified about the completion when it calls pj_ioqueue_poll(). If data
+ * is immediately available, the function will return PJ_SUCCESS and the
+ * callback WILL NOT be called.
  *
  * @param key	    The key that uniquely identifies the handle.
- * @param op_key    An operation specific key to be associated with the

- *                  pending operation, so that application can keep track of

- *                  which operation has been completed when the callback is

- *                  called. Caller must make sure that this key remains 

- *                  valid until the function completes.

+ * @param op_key    An operation specific key to be associated with the
+ *                  pending operation, so that application can keep track of
+ *                  which operation has been completed when the callback is
+ *                  called. Caller must make sure that this key remains 
+ *                  valid until the function completes.
  * @param buffer    The buffer to hold the read data. The caller MUST make sure
  *		    that this buffer remain valid until the framework completes
  *		    reading the handle.
- * @param length    On input, it specifies the size of the buffer. If data is

- *                  available to be read immediately, the function returns

- *                  PJ_SUCCESS and this argument will be filled with the

- *                  amount of data read. If the function is pending, caller

- *                  will be notified about the amount of data read in the

- *                  callback. This parameter can point to local variable in

- *                  caller's stack and doesn't have to remain valid for the

+ * @param length    On input, it specifies the size of the buffer. If data is
+ *                  available to be read immediately, the function returns
+ *                  PJ_SUCCESS and this argument will be filled with the
+ *                  amount of data read. If the function is pending, caller
+ *                  will be notified about the amount of data read in the
+ *                  callback. This parameter can point to local variable in
+ *                  caller's stack and doesn't have to remain valid for the
  *                  duration of pending operation.
  * @param flags     Recv flag.
  *
  * @return
- *  - PJ_SUCCESS    If immediate data has been received in the buffer. In this

+ *  - PJ_SUCCESS    If immediate data has been received in the buffer. In this
  *                  case, the callback WILL NOT be called.
- *  - PJ_EPENDING   If the operation has been queued, and the callback will be

+ *  - PJ_EPENDING   If the operation has been queued, and the callback will be
  *                  called when data has been received.
  *  - non-zero      The return value indicates the error code.
  */
-PJ_DECL(pj_status_t) pj_ioqueue_recv( pj_ioqueue_key_t *key,

+PJ_DECL(pj_status_t) pj_ioqueue_recv( pj_ioqueue_key_t *key,
                                       pj_ioqueue_op_key_t *op_key,
 				      void *buffer,
 				      pj_ssize_t *length,
@@ -450,26 +450,26 @@ PJ_DECL(pj_status_t) pj_ioqueue_recv( pj_ioqueue_key_t *key,
  * remain valid until the framework completes reading the data.
  *
  * @param key	    The key that uniquely identifies the handle.
- * @param op_key    An operation specific key to be associated with the

- *                  pending operation, so that application can keep track of

- *                  which operation has been completed when the callback is

- *                  called.

+ * @param op_key    An operation specific key to be associated with the
+ *                  pending operation, so that application can keep track of
+ *                  which operation has been completed when the callback is
+ *                  called.
  * @param buffer    The buffer to hold the read data. The caller MUST make sure
  *		    that this buffer remain valid until the framework completes
  *		    reading the handle.
- * @param length    On input, it specifies the size of the buffer. If data is

- *                  available to be read immediately, the function returns

- *                  PJ_SUCCESS and this argument will be filled with the

- *                  amount of data read. If the function is pending, caller

- *                  will be notified about the amount of data read in the

- *                  callback. This parameter can point to local variable in

- *                  caller's stack and doesn't have to remain valid for the

- *                  duration of pending operation.

+ * @param length    On input, it specifies the size of the buffer. If data is
+ *                  available to be read immediately, the function returns
+ *                  PJ_SUCCESS and this argument will be filled with the
+ *                  amount of data read. If the function is pending, caller
+ *                  will be notified about the amount of data read in the
+ *                  callback. This parameter can point to local variable in
+ *                  caller's stack and doesn't have to remain valid for the
+ *                  duration of pending operation.
  * @param flags     Recv flag.
  * @param addr      Optional Pointer to buffer to receive the address.
  * @param addrlen   On input, specifies the length of the address buffer.
  *                  On output, it will be filled with the actual length of
- *                  the address. This argument can be NULL if \c addr is not

+ *                  the address. This argument can be NULL if \c addr is not
  *                  specified.
  *
  * @return
@@ -479,7 +479,7 @@ PJ_DECL(pj_status_t) pj_ioqueue_recv( pj_ioqueue_key_t *key,
  *  - PJ_EPENDING   If the operation has been queued.
  *  - non-zero      The return value indicates the error code.
  */
-PJ_DECL(pj_status_t) pj_ioqueue_recvfrom( pj_ioqueue_key_t *key,

+PJ_DECL(pj_status_t) pj_ioqueue_recvfrom( pj_ioqueue_key_t *key,
                                           pj_ioqueue_op_key_t *op_key,
 					  void *buffer,
 					  pj_ssize_t *length,
@@ -489,39 +489,39 @@ PJ_DECL(pj_status_t) pj_ioqueue_recvfrom( pj_ioqueue_key_t *key,
 
 
 /**
- * Instruct the I/O Queue to write to the handle. This function will return

- * immediately (i.e. non-blocking) regardless whether some data has been 

- * transfered. If the function can't complete immediately, the caller will

- * be notified about the completion when it calls pj_ioqueue_poll(). If 

- * operation completes immediately and data has been transfered, the function

- * returns PJ_SUCCESS and the callback will NOT be called.

- *

+ * Instruct the I/O Queue to write to the handle. This function will return
+ * immediately (i.e. non-blocking) regardless whether some data has been 
+ * transfered. If the function can't complete immediately, the caller will
+ * be notified about the completion when it calls pj_ioqueue_poll(). If 
+ * operation completes immediately and data has been transfered, the function
+ * returns PJ_SUCCESS and the callback will NOT be called.
+ *
  * @param key	    The key that identifies the handle.
- * @param op_key    An operation specific key to be associated with the

- *                  pending operation, so that application can keep track of

- *                  which operation has been completed when the callback is

- *                  called.

+ * @param op_key    An operation specific key to be associated with the
+ *                  pending operation, so that application can keep track of
+ *                  which operation has been completed when the callback is
+ *                  called.
  * @param data	    The data to send. Caller MUST make sure that this buffer 
  *		    remains valid until the write operation completes.
- * @param length    On input, it specifies the length of data to send. When

- *                  data was sent immediately, this function returns PJ_SUCCESS

- *                  and this parameter contains the length of data sent. If

- *                  data can not be sent immediately, an asynchronous operation

- *                  is scheduled and caller will be notified via callback the

- *                  number of bytes sent. This parameter can point to local 

- *                  variable on caller's stack and doesn't have to remain 

+ * @param length    On input, it specifies the length of data to send. When
+ *                  data was sent immediately, this function returns PJ_SUCCESS
+ *                  and this parameter contains the length of data sent. If
+ *                  data can not be sent immediately, an asynchronous operation
+ *                  is scheduled and caller will be notified via callback the
+ *                  number of bytes sent. This parameter can point to local 
+ *                  variable on caller's stack and doesn't have to remain 
  *                  valid until the operation has completed.
  * @param flags     Send flags.
  *
  * @return
- *  - PJ_SUCCESS    If data was immediately transfered. In this case, no

- *                  pending operation has been scheduled and the callback

+ *  - PJ_SUCCESS    If data was immediately transfered. In this case, no
+ *                  pending operation has been scheduled and the callback
  *                  WILL NOT be called.
- *  - PJ_EPENDING   If the operation has been queued. Once data base been

+ *  - PJ_EPENDING   If the operation has been queued. Once data base been
  *                  transfered, the callback will be called.
  *  - non-zero      The return value indicates the error code.
  */
-PJ_DECL(pj_status_t) pj_ioqueue_send( pj_ioqueue_key_t *key,

+PJ_DECL(pj_status_t) pj_ioqueue_send( pj_ioqueue_key_t *key,
                                       pj_ioqueue_op_key_t *op_key,
 				      const void *data,
 				      pj_ssize_t *length,
@@ -533,20 +533,20 @@ PJ_DECL(pj_status_t) pj_ioqueue_send( pj_ioqueue_key_t *key,
  * pj_sock_sendto() (or equivalent) will be called to send the data.
  *
  * @param key	    the key that identifies the handle.
- * @param op_key    An operation specific key to be associated with the

- *                  pending operation, so that application can keep track of

- *                  which operation has been completed when the callback is

- *                  called.

+ * @param op_key    An operation specific key to be associated with the
+ *                  pending operation, so that application can keep track of
+ *                  which operation has been completed when the callback is
+ *                  called.
  * @param data	    the data to send. Caller MUST make sure that this buffer 
  *		    remains valid until the write operation completes.
- * @param length    On input, it specifies the length of data to send. When

- *                  data was sent immediately, this function returns PJ_SUCCESS

- *                  and this parameter contains the length of data sent. If

- *                  data can not be sent immediately, an asynchronous operation

- *                  is scheduled and caller will be notified via callback the

- *                  number of bytes sent. This parameter can point to local 

- *                  variable on caller's stack and doesn't have to remain 

- *                  valid until the operation has completed.

+ * @param length    On input, it specifies the length of data to send. When
+ *                  data was sent immediately, this function returns PJ_SUCCESS
+ *                  and this parameter contains the length of data sent. If
+ *                  data can not be sent immediately, an asynchronous operation
+ *                  is scheduled and caller will be notified via callback the
+ *                  number of bytes sent. This parameter can point to local 
+ *                  variable on caller's stack and doesn't have to remain 
+ *                  valid until the operation has completed.
  * @param flags     send flags.
  * @param addr      Optional remote address.
  * @param addrlen   Remote address length, \c addr is specified.
@@ -556,7 +556,7 @@ PJ_DECL(pj_status_t) pj_ioqueue_send( pj_ioqueue_key_t *key,
  *  - PJ_EPENDING   If the operation has been queued.
  *  - non-zero      The return value indicates the error code.
  */
-PJ_DECL(pj_status_t) pj_ioqueue_sendto( pj_ioqueue_key_t *key,

+PJ_DECL(pj_status_t) pj_ioqueue_sendto( pj_ioqueue_key_t *key,
                                         pj_ioqueue_op_key_t *op_key,
 					const void *data,
 					pj_ssize_t *length,