Changed ioqueue to allow simultaneous operations on the same key

git-svn-id: http://svn.pjsip.org/repos/pjproject/main@11 74dad513-b988-da41-8d7b-12977e46ad98

11 files changed, 393 insertions, 183 deletions

diff --git a/pjlib/include/pj/compat/os_linux.h b/pjlib/include/pj/compat/os_linux.h
index c75589d6..a2f97f65 100644
--- a/pjlib/include/pj/compat/os_linux.h
+++ b/pjlib/include/pj/compat/os_linux.h
@@ -54,7 +54,22 @@
 #define PJ_HAS_WINSOCK_H	    0
 #define PJ_HAS_WINSOCK2_H	    0
 
-#define PJ_SOCK_HAS_INET_ATON	    1
+#define PJ_SOCK_HAS_INET_ATON	    1

+

+/* When this macro is set, getsockopt(SOL_SOCKET, SO_ERROR) will return

+ * the status of non-blocking connect() operation.

+ */
+#define PJ_HAS_SO_ERROR             1

+

+/* This value specifies the value set in errno by the OS when a non-blocking

+ * socket recv() can not return immediate daata.

+ */

+#define PJ_BLOCKING_ERROR_VAL       EAGAIN

+

+/* This value specifies the value set in errno by the OS when a non-blocking

+ * socket connect() can not get connected immediately.

+ */

+#define PJ_BLOCKING_CONNECT_ERROR_VAL   EINPROGRESS

 
 /* Default threading is enabled, unless it's overridden. */
 #ifndef PJ_HAS_THREADS
diff --git a/pjlib/include/pj/compat/os_linux_kernel.h b/pjlib/include/pj/compat/os_linux_kernel.h
index 91c41773..0d44ef0e 100644
--- a/pjlib/include/pj/compat/os_linux_kernel.h
+++ b/pjlib/include/pj/compat/os_linux_kernel.h
@@ -52,6 +52,21 @@
 #define PJ_HAS_WINSOCK2_H	    0
 
 #define PJ_SOCK_HAS_INET_ATON	    0
+

+/* When this macro is set, getsockopt(SOL_SOCKET, SO_ERROR) will return

+ * the status of non-blocking connect() operation.

+ */

+#define PJ_HAS_SO_ERROR             1

+

+/* This value specifies the value set in errno by the OS when a non-blocking

+ * socket recv() can not return immediate daata.

+ */

+#define PJ_BLOCKING_ERROR_VAL       EAGAIN

+

+/* This value specifies the value set in errno by the OS when a non-blocking

+ * socket connect() can not get connected immediately.

+ */

+#define PJ_BLOCKING_CONNECT_ERROR_VAL   EINPROGRESS

 
 #ifndef PJ_HAS_THREADS
 #  define PJ_HAS_THREADS	    (1)
diff --git a/pjlib/include/pj/compat/os_palmos.h b/pjlib/include/pj/compat/os_palmos.h
index 63cd8273..e2ac4b52 100644
--- a/pjlib/include/pj/compat/os_palmos.h
+++ b/pjlib/include/pj/compat/os_palmos.h
@@ -44,6 +44,21 @@
 #define PJ_HAS_WINSOCK2_H	    0
 
 #define PJ_SOCK_HAS_INET_ATON	    0
+

+/* When this macro is set, getsockopt(SOL_SOCKET, SO_ERROR) will return

+ * the status of non-blocking connect() operation.

+ */

+#define PJ_HAS_SO_ERROR             0

+

+/* This value specifies the value set in errno by the OS when a non-blocking

+ * socket recv() can not return immediate daata.

+ */

+#define PJ_BLOCKING_ERROR_VAL       xxx

+

+/* This value specifies the value set in errno by the OS when a non-blocking

+ * socket connect() can not get connected immediately.

+ */

+#define PJ_BLOCKING_CONNECT_ERROR_VAL   xxx

 
 /* Default threading is enabled, unless it's overridden. */
 #ifndef PJ_HAS_THREADS
diff --git a/pjlib/include/pj/compat/os_sunos.h b/pjlib/include/pj/compat/os_sunos.h
index b0f48c38..990fb57d 100644
--- a/pjlib/include/pj/compat/os_sunos.h
+++ b/pjlib/include/pj/compat/os_sunos.h
@@ -39,6 +39,21 @@
 #define PJ_HAS_WINSOCK2_H	    0
 
 #define PJ_SOCK_HAS_INET_ATON	    0
+

+/* When this macro is set, getsockopt(SOL_SOCKET, SO_ERROR) will return

+ * the status of non-blocking connect() operation.

+ */

+#define PJ_HAS_SO_ERROR             0

+

+/* This value specifies the value set in errno by the OS when a non-blocking

+ * socket recv() can not return immediate daata.

+ */

+#define PJ_BLOCKING_ERROR_VAL       EWOULDBLOCK

+

+/* This value specifies the value set in errno by the OS when a non-blocking

+ * socket connect() can not get connected immediately.

+ */

+#define PJ_BLOCKING_CONNECT_ERROR_VAL   EINPROGRESS

 
 /* Default threading is enabled, unless it's overridden. */
 #ifndef PJ_HAS_THREADS
diff --git a/pjlib/include/pj/compat/os_win32.h b/pjlib/include/pj/compat/os_win32.h
index db7b1b4f..87ff7520 100644
--- a/pjlib/include/pj/compat/os_win32.h
+++ b/pjlib/include/pj/compat/os_win32.h
@@ -59,6 +59,21 @@
 #define PJ_HAS_WINSOCK2_H	    1
 
 #define PJ_SOCK_HAS_INET_ATON	    0
+

+/* When this macro is set, getsockopt(SOL_SOCKET, SO_ERROR) will return

+ * the status of non-blocking connect() operation.

+ */

+#define PJ_HAS_SO_ERROR             0

+

+/* This value specifies the value set in errno by the OS when a non-blocking

+ * socket recv() or send() can not return immediately.

+ */

+#define PJ_BLOCKING_ERROR_VAL       WSAEWOULDBLOCK

+

+/* This value specifies the value set in errno by the OS when a non-blocking

+ * socket connect() can not get connected immediately.

+ */

+#define PJ_BLOCKING_CONNECT_ERROR_VAL   WSAEWOULDBLOCK

 
 /* Default threading is enabled, unless it's overridden. */
 #ifndef PJ_HAS_THREADS
diff --git a/pjlib/include/pj/doxygen.h b/pjlib/include/pj/doxygen.h
index 9c32e4e2..30f642a6 100644
--- a/pjlib/include/pj/doxygen.h
+++ b/pjlib/include/pj/doxygen.h
@@ -87,9 +87,12 @@
  * Alternatively, to get the list of all examples, you can click on 
  * <b>Related Pages</b> on the top of HTML document or on 
  * <b>PJLIB Page Documentation</b> on navigation pane of your PDF reader.
- *
- *
- *
+ *

+ *  - <b>How to Submit Code to PJLIB Project</b>
+ *\n
+ * Please read \ref pjlib_coding_convention_page before submitting

+ * your code. Send your code as patch against current Subversion tree
+ * to the appropriate mailing list.

  *
  *
  * @section features_sec Features
@@ -395,6 +398,52 @@
 
 
 
+/*////////////////////////////////////////////////////////////////////////// */

+/*

+         CODING CONVENTION

+ */

+

+/**

+ * @page pjlib_coding_convention_page Coding Convention

+ *

+ * Before you submit your code/patches to be included with PJLIB, you must

+ * make sure that your code is compliant with PJLIB coding convention.

+ * <b>This is very important!</b> Otherwise we would not accept your code.

+ *

+ * @section coding_conv_editor_sec Editor Settings

+ *

+ * The single most important thing in the whole coding convention is editor 

+ * settings. It's more important than the correctness of your code (bugs will

+ * only crash the system, but incorrect tab size is mental!).

+ *

+ * Kindly set your editor as follows:

+ *  - tab size to \b 8.

+ *  - indentation to \b 4.

+ *

+ * With \c vi, you can do it with:

+ * <pre>

+ *  :se ts=8

+ *  :se sts=4

+ * </pre>

+ *

+ * You should replace tab with eight spaces.

+ *

+ * @section coding_conv_detail_sec Coding Style

+ *

+ * Coding style MUST strictly follow K&R style. The rest of coding style

+ * must follow current style. You SHOULD be able to observe the style

+ * currently used by PJLIB from PJLIB sources, and apply the style to your 

+ * code. If you're not able to do simple thing like to observe PJLIB

+ * coding style from the sources, then logic dictates that your ability to

+ * observe more difficult area in PJLIB such as memory allocation strategy, 

+ * concurrency, etc is questionable.

+ *

+ * @section coding_conv_comment_sec Commenting Your Code

+ *

+ * Public API (e.g. in header files) MUST have doxygen compliant comments.

+ *

+ */

+

 
 /*////////////////////////////////////////////////////////////////////////// */
 /*
diff --git a/pjlib/include/pj/ioqueue.h b/pjlib/include/pj/ioqueue.h
index 9d26b082..6a7b827e 100644
--- a/pjlib/include/pj/ioqueue.h
+++ b/pjlib/include/pj/ioqueue.h
@@ -1,5 +1,4 @@
 /* $Id$
- *
  */
 
 #ifndef __PJ_IOQUEUE_H__
@@ -48,17 +47,54 @@ PJ_BEGIN_DECL
  * @ingroup PJ_IO
  * @{
  *
- * This file provides abstraction for various event dispatching mechanisms. 
- * The interfaces for event dispatching vary alot, even in a single
- * operating system. The abstraction here hopefully is suitable for most of
- * the event dispatching available.
- *
- * Currently, the I/O Queue supports:
- * - select(), as the common denominator, but the least efficient.
- * - I/O Completion ports in Windows NT/2000/XP, which is the most efficient
- *      way to dispatch events in Windows NT based OSes, and most importantly,
- *      it doesn't have the limit on how many handles to monitor. And it works
- *      with files (not only sockets) as well.
+ * I/O Queue provides API for performing asynchronous I/O operations. It

+ * conforms to proactor pattern, which allows application to submit an

+ * asynchronous operation and to be notified later when the operation has

+ * completed.
+ *

+ * The framework works natively in platforms where asynchronous operation API

+ * exists, such as in Windows NT with IoCompletionPort/IOCP. In other 

+ * platforms, the I/O queue abstracts the operating system's event poll API

+ * to provide semantics similar to IoCompletionPort with minimal penalties

+ * (i.e. per ioqueue and per handle mutex protection).

+ *

+ * The I/O queue provides more than just unified abstraction. It also:

+ *  - makes sure that the operation uses the most effective way to utilize

+ *    the underlying mechanism, to provide the maximum theoritical

+ *    throughput possible on a given platform.

+ *  - choose the most efficient mechanism for event polling on a given

+ *    platform.

+ *
+ * Currently, the I/O Queue is implemented using:
+ *  - <tt><b>select()</b></tt>, as the common denominator, but the least 

+ *    efficient. Also the number of descriptor is limited to 

+ *    \c PJ_IOQUEUE_MAX_HANDLES (which by default is 64).

+ *  - <tt><b>/dev/epoll</b></tt> on Linux (user mode and kernel mode), 

+ *    a much faster replacement for select() on Linux (and more importantly

+ *    doesn't have limitation on number of descriptors).
+ *  - <b>I/O Completion ports</b> on Windows NT/2000/XP, which is the most 

+ *    efficient way to dispatch events in Windows NT based OSes, and most 

+ *    importantly, it doesn't have the limit on how many handles to monitor.

+ *    And it works with files (not only sockets) as well.
+ *

+ *

+ * \section pj_ioqueue_concurrency_sec Concurrency Rules

+ *

+ * The items below describe rules that must be obeyed when using the I/O 

+ * queue, with regard to concurrency:

+ *  - in general, the I/O queue is thread safe (assuming the lock strategy

+ *    is not changed to disable mutex protection). All operations, except

+ *    unregistration which is described below, can be safely invoked 

+ *    simultaneously by multiple  threads.

+ *  - however, <b>care must be taken when unregistering a key</b> from the

+ *    ioqueue. Application must take care that when one thread is issuing

+ *    an unregistration, other thread is not simultaneously invoking an

+ *    operation <b>to the same key</b>.

+ *\n

+ *    This happens because the ioqueue functions are working with a pointer

+ *    to the key, and there is a possible race condition where the pointer

+ *    has been rendered invalid by other threads before the ioqueue has a

+ *    chance to acquire mutex on it.

  *
  * \section pj_ioqeuue_examples_sec Examples
  *
@@ -68,53 +104,90 @@ PJ_BEGIN_DECL
  *  - \ref page_pjlib_ioqueue_udp_test
  *  - \ref page_pjlib_ioqueue_perf_test
  */
+

+

+

+/**

+ * This structure describes operation specific key to be submitted to

+ * I/O Queue when performing the asynchronous operation. This key will

+ * be returned to the application when completion callback is called.

+ *

+ * Application normally wants to attach it's specific data in the

+ * \c user_data field so that it can keep track of which operation has

+ * completed when the callback is called. Alternatively, application can

+ * also extend this struct to include its data, because the pointer that

+ * is returned in the completion callback will be exactly the same as

+ * the pointer supplied when the asynchronous function is called.

+ */

+typedef struct pj_ioqueue_op_key_t

+{ 

+    void *internal__[32];           /**< Internal I/O Queue data.   */

+    void *user_data;                /**< Application data.          */

+} pj_ioqueue_op_key_t;

 
- /**
-  * This structure describes the callbacks to be called when I/O operation
-  * completes.
-  */
+/**
+ * This structure describes the callbacks to be called when I/O operation
+ * completes.
+ */
 typedef struct pj_ioqueue_callback
 {
     /**
-     * This callback is called when #pj_ioqueue_read or #pj_ioqueue_recvfrom
+     * This callback is called when #pj_ioqueue_recv or #pj_ioqueue_recvfrom
      * completes.
      *
-     * @param key	    The key.
-     * @param bytes_read    The size of data that has just been read.
+     * @param key	    The key.

+     * @param op_key        Operation key.
+     * @param bytes_read    >= 0 to indicate the amount of data read, 

+     *                      otherwise negative value containing the error

+     *                      code. To obtain the pj_status_t error code, use

+     *                      (pj_status_t code = -bytes_read).
      */
-    void (*on_read_complete)(pj_ioqueue_key_t *key, pj_ssize_t bytes_read);
+    void (*on_read_complete)(pj_ioqueue_key_t *key, 

+                             pj_ioqueue_op_key_t *op_key, 

+                             pj_ssize_t bytes_read);
 
     /**
      * This callback is called when #pj_ioqueue_write or #pj_ioqueue_sendto
      * completes.
      *
-     * @param key	    The key.
-     * @param bytes_read    The size of data that has just been read.
+     * @param key	    The key.

+     * @param op_key        Operation key.
+     * @param bytes_sent    >= 0 to indicate the amount of data written, 

+     *                      otherwise negative value containing the error

+     *                      code. To obtain the pj_status_t error code, use

+     *                      (pj_status_t code = -bytes_sent).

      */
-    void (*on_write_complete)(pj_ioqueue_key_t *key, pj_ssize_t bytes_sent);
+    void (*on_write_complete)(pj_ioqueue_key_t *key, 

+                              pj_ioqueue_op_key_t *op_key, 

+                              pj_ssize_t bytes_sent);
 
     /**
      * This callback is called when #pj_ioqueue_accept completes.
      *
-     * @param key	    The key.
+     * @param key	    The key.

+     * @param op_key        Operation key.
      * @param sock          Newly connected socket.
      * @param status	    Zero if the operation completes successfully.
      */
-    void (*on_accept_complete)(pj_ioqueue_key_t *key, pj_sock_t sock, 
-                               int status);
+    void (*on_accept_complete)(pj_ioqueue_key_t *key, 

+                               pj_ioqueue_op_key_t *op_key, 

+                               pj_sock_t sock, 
+                               pj_status_t status);
 
     /**
      * This callback is called when #pj_ioqueue_connect completes.
      *
-     * @param key	    The key.
-     * @param status	    Zero if the operation completes successfully.
+     * @param key	    The key.

+     * @param status	    PJ_SUCCESS if the operation completes successfully.
      */
-    void (*on_connect_complete)(pj_ioqueue_key_t *key, int status);
+    void (*on_connect_complete)(pj_ioqueue_key_t *key, 

+                                pj_status_t status);
 } pj_ioqueue_callback;
 
 
 /**
- * Types of I/O Queue operation.
+ * Types of pending I/O Queue operation. This enumeration is only used

+ * internally within the ioqueue.
  */
 typedef enum pj_ioqueue_operation_e
 {
@@ -137,25 +210,19 @@ typedef enum pj_ioqueue_operation_e
  * number of threads.
  */
 #define PJ_IOQUEUE_DEFAULT_THREADS  0
-
-
+

 /**
  * Create a new I/O Queue framework.
  *
  * @param pool		The pool to allocate the I/O queue structure. 
  * @param max_fd	The maximum number of handles to be supported, which 
  *			should not exceed PJ_IOQUEUE_MAX_HANDLES.
- * @param max_threads	The maximum number of threads that are allowed to
- *			operate on a single descriptor simultaneously. If
- *                      the value is zero, the framework will set it
- *                      to a reasonable value.
  * @param ioqueue	Pointer to hold the newly created I/O Queue.
  *
  * @return		PJ_SUCCESS on success.
  */
 PJ_DECL(pj_status_t) pj_ioqueue_create( pj_pool_t *pool, 
 					pj_size_t max_fd,
-					int max_threads,
 					pj_ioqueue_t **ioqueue);
 
 /**
@@ -199,8 +266,10 @@ PJ_DECL(pj_status_t) pj_ioqueue_set_lock( pj_ioqueue_t *ioque,
  * @param sock	    The socket.
  * @param user_data User data to be associated with the key, which can be
  *		    retrieved later.
- * @param cb	    Callback to be called when I/O operation completes. 
- * @param key	    Pointer to receive the returned key.
+ * @param cb	    Callback to be called when I/O operation completes. 

+ * @param key       Pointer to receive the key to be associated with this

+ *                  socket. Subsequent I/O queue operation will need this

+ *                  key.
  *
  * @return	    PJ_SUCCESS on success, or the error code.
  */
@@ -208,49 +277,66 @@ PJ_DECL(pj_status_t) pj_ioqueue_register_sock( pj_pool_t *pool,
 					       pj_ioqueue_t *ioque,
 					       pj_sock_t sock,
 					       void *user_data,
-					       const pj_ioqueue_callback *cb,
-					       pj_ioqueue_key_t **key);
+					       const pj_ioqueue_callback *cb,

+                                               pj_ioqueue_key_t **key );
 
 /**
- * Unregister a handle from the I/O Queue framework.
+ * Unregister from the I/O Queue framework. Caller must make sure that

+ * the key doesn't have any pending operation before calling this function,

+ * or otherwise the behaviour is undefined (either callback will be called

+ * later when the data is sent/received, or the callback will not be called,

+ * or even something else).
  *
- * @param ioque     The I/O Queue.
- * @param key	    The key that uniquely identifies the handle, which is 
- *                  returned from the function #pj_ioqueue_register_sock()
- *                  or other registration functions.
+ * @param key	    The key that was previously obtained from registration.

  *
  * @return          PJ_SUCCESS on success or the error code.
  */
-PJ_DECL(pj_status_t) pj_ioqueue_unregister( pj_ioqueue_t *ioque,
-					    pj_ioqueue_key_t *key );
+PJ_DECL(pj_status_t) pj_ioqueue_unregister( pj_ioqueue_key_t *key );
 
 
 /**
- * Get user data associated with the I/O Queue key.
+ * Get user data associated with an ioqueue key.
+ *

+ * @param key	    The key that was previously obtained from registration.

  *
- * @param key	    The key previously associated with the socket/handle with
- *		    #pj_ioqueue_register_sock() (or other registration 
- *                  functions).
- *
- * @return          The user data associated with the key, or NULL on error
- *                  of if no data is associated with the key during 
+ * @return          The user data associated with the descriptor, or NULL 

+ *                  on error or if no data is associated with the key during
  *                  registration.
  */
 PJ_DECL(void*) pj_ioqueue_get_user_data( pj_ioqueue_key_t *key );
+

+/**

+ * Set or change the user data to be associated with the file descriptor or

+ * handle or socket descriptor.

+ *

+ * @param key	    The key that was previously obtained from registration.

+ * @param user_data User data to be associated with the descriptor.

+ * @param old_data  Optional parameter to retrieve the old user data.

+ *

+ * @return          PJ_SUCCESS on success or the error code.

+ */

+PJ_DECL(pj_status_t) pj_ioqueue_set_user_data( pj_ioqueue_key_t *key,

+                                               void *user_data,

+                                               void **old_data);

 
 
 #if defined(PJ_HAS_TCP) && PJ_HAS_TCP != 0
 /**
- * Instruct I/O Queue to wait for incoming connections on the specified 
- * listening socket. This function will return
- * immediately (i.e. non-blocking) regardless whether some data has been 
- * transfered. If the function can't complete immediately, and the caller will
- * be notified about the completion when it calls pj_ioqueue_poll().
- *
- * @param ioqueue   The I/O Queue
- * @param key	    The key which registered to the server socket.
- * @param sock	    Argument which contain pointer to receive 
- *                  the socket for the incoming connection.
+ * Instruct I/O Queue to accept incoming connection on the specified 
+ * listening socket. This function will return immediately (i.e. non-blocking)

+ * regardless whether a connection is immediately available. If the function

+ * can't complete immediately, the caller will be notified about the incoming

+ * connection when it calls pj_ioqueue_poll(). If a new connection is

+ * immediately available, the function returns PJ_SUCCESS with the new

+ * connection; in this case, the callback WILL NOT be called.
+ *
+ * @param key	    The key which registered to the server socket.

+ * @param op_key    An operation specific key to be associated with the

+ *                  pending operation, so that application can keep track of

+ *                  which operation has been completed when the callback is

+ *                  called.
+ * @param new_sock  Argument which contain pointer to receive the new socket
+ *                  for the incoming connection.
  * @param local	    Optional argument which contain pointer to variable to 
  *                  receive local address.
  * @param remote    Optional argument which contain pointer to variable to 
@@ -259,14 +345,16 @@ PJ_DECL(void*) pj_ioqueue_get_user_data( pj_ioqueue_key_t *key );
  *		    address, and on output, contains the actual length of the
  *		    address. This argument is optional.
  * @return
- *  - PJ_SUCCESS    If there's a connection available immediately, which 
- *                  in this case the callback should have been called before 
- *                  the function returns.
- *  - PJ_EPENDING   If accept is queued, or 
+ *  - PJ_SUCCESS    When connection is available immediately, and the 

+ *                  parameters will be updated to contain information about 

+ *                  the new connection. In this case, a completion callback

+ *                  WILL NOT be called.
+ *  - PJ_EPENDING   If no connection is available immediately. When a new

+ *                  connection arrives, the callback will be called.
  *  - non-zero      which indicates the appropriate error code.
  */
-PJ_DECL(pj_status_t) pj_ioqueue_accept( pj_ioqueue_t *ioqueue,
-					pj_ioqueue_key_t *key,
+PJ_DECL(pj_status_t) pj_ioqueue_accept( pj_ioqueue_key_t *key,

+                                        pj_ioqueue_op_key_t *op_key,
 					pj_sock_t *sock,
 					pj_sockaddr_t *local,
 					pj_sockaddr_t *remote,
@@ -274,21 +362,22 @@ PJ_DECL(pj_status_t) pj_ioqueue_accept( pj_ioqueue_t *ioqueue,
 
 /**
  * Initiate non-blocking socket connect. If the socket can NOT be connected
- * immediately, the result will be reported during poll.
+ * immediately, asynchronous connect() will be scheduled and caller will be

+ * notified via completion callback when it calls pj_ioqueue_poll(). If

+ * socket is connected immediately, the function returns PJ_SUCCESS and

+ * completion callback WILL NOT be called.
  *
- * @param ioqueue   The ioqueue
  * @param key	    The key associated with TCP socket
  * @param addr	    The remote address.
  * @param addrlen   The remote address length.
  *
  * @return
- *  - PJ_SUCCESS    If socket is connected immediately, which in this case 
- *                  the callback should have been called.
+ *  - PJ_SUCCESS    If socket is connected immediately. In this case, the

+ *                  completion callback WILL NOT be called.
  *  - PJ_EPENDING   If operation is queued, or 
  *  - non-zero      Indicates the error code.
  */
-PJ_DECL(pj_status_t) pj_ioqueue_connect( pj_ioqueue_t *ioqueue,
-					 pj_ioqueue_key_t *key,
+PJ_DECL(pj_status_t) pj_ioqueue_connect( pj_ioqueue_key_t *key,
 					 const pj_sockaddr_t *addr,
 					 int addrlen );
 
@@ -309,74 +398,75 @@ PJ_DECL(pj_status_t) pj_ioqueue_connect( pj_ioqueue_t *ioqueue,
 PJ_DECL(int) pj_ioqueue_poll( pj_ioqueue_t *ioque,
 			      const pj_time_val *timeout);
 
-/**
- * Instruct the I/O Queue to read from the specified handle. This function
- * returns immediately (i.e. non-blocking) regardless whether some data has 
- * been transfered. If the operation can't complete immediately, caller will 
- * be notified about the completion when it calls pj_ioqueue_poll().
- *
- * @param ioque	    The I/O Queue.
- * @param key	    The key that uniquely identifies the handle.
- * @param buffer    The buffer to hold the read data. The caller MUST make sure
- *		    that this buffer remain valid until the framework completes
- *		    reading the handle.
- * @param buflen    The maximum size to be read.
- *
- * @return
- *  - PJ_SUCCESS    If immediate data has been received. In this case, the 
- *		    callback must have been called before this function 
- *		    returns, and no pending operation is scheduled.
- *  - PJ_EPENDING   If the operation has been queued.
- *  - non-zero      The return value indicates the error code.
- */
-PJ_DECL(pj_status_t) pj_ioqueue_read( pj_ioqueue_t *ioque,
-				      pj_ioqueue_key_t *key,
-				      void *buffer,
-				      pj_size_t buflen);
-
 
 /**
- * This function behaves similarly as #pj_ioqueue_read(), except that it is
- * normally called for socket.
+ * Instruct the I/O Queue to read from the specified handle. This function

+ * returns immediately (i.e. non-blocking) regardless whether some data has 

+ * been transfered. If the operation can't complete immediately, caller will 

+ * be notified about the completion when it calls pj_ioqueue_poll(). If data

+ * is immediately available, the function will return PJ_SUCCESS and the

+ * callback WILL NOT be called.

  *
- * @param ioque	    The I/O Queue.
  * @param key	    The key that uniquely identifies the handle.
+ * @param op_key    An operation specific key to be associated with the

+ *                  pending operation, so that application can keep track of

+ *                  which operation has been completed when the callback is

+ *                  called. Caller must make sure that this key remains 

+ *                  valid until the function completes.

  * @param buffer    The buffer to hold the read data. The caller MUST make sure
  *		    that this buffer remain valid until the framework completes
  *		    reading the handle.
- * @param buflen    The maximum size to be read.
+ * @param length    On input, it specifies the size of the buffer. If data is

+ *                  available to be read immediately, the function returns

+ *                  PJ_SUCCESS and this argument will be filled with the

+ *                  amount of data read. If the function is pending, caller

+ *                  will be notified about the amount of data read in the

+ *                  callback. This parameter can point to local variable in

+ *                  caller's stack and doesn't have to remain valid for the

+ *                  duration of pending operation.
  * @param flags     Recv flag.
  *
  * @return
- *  - PJ_SUCCESS    If immediate data has been received. In this case, the 
- *		    callback must have been called before this function 
- *		    returns, and no pending operation is scheduled.
- *  - PJ_EPENDING   If the operation has been queued.
+ *  - PJ_SUCCESS    If immediate data has been received in the buffer. In this

+ *                  case, the callback WILL NOT be called.
+ *  - PJ_EPENDING   If the operation has been queued, and the callback will be

+ *                  called when data has been received.
  *  - non-zero      The return value indicates the error code.
  */
-PJ_DECL(pj_status_t) pj_ioqueue_recv( pj_ioqueue_t *ioque,
-				      pj_ioqueue_key_t *key,
+PJ_DECL(pj_status_t) pj_ioqueue_recv( pj_ioqueue_key_t *key,

+                                      pj_ioqueue_op_key_t *op_key,
 				      void *buffer,
-				      pj_size_t buflen,
+				      pj_ssize_t *length,
 				      unsigned flags );
 
 /**
- * This function behaves similarly as #pj_ioqueue_read(), except that it is
+ * This function behaves similarly as #pj_ioqueue_recv(), except that it is
  * normally called for socket, and the remote address will also be returned
  * along with the data. Caller MUST make sure that both buffer and addr
  * remain valid until the framework completes reading the data.
  *
- * @param ioque	    The I/O Queue.
  * @param key	    The key that uniquely identifies the handle.
+ * @param op_key    An operation specific key to be associated with the

+ *                  pending operation, so that application can keep track of

+ *                  which operation has been completed when the callback is

+ *                  called.

  * @param buffer    The buffer to hold the read data. The caller MUST make sure
  *		    that this buffer remain valid until the framework completes
  *		    reading the handle.
- * @param buflen    The maximum size to be read.
+ * @param length    On input, it specifies the size of the buffer. If data is

+ *                  available to be read immediately, the function returns

+ *                  PJ_SUCCESS and this argument will be filled with the

+ *                  amount of data read. If the function is pending, caller

+ *                  will be notified about the amount of data read in the

+ *                  callback. This parameter can point to local variable in

+ *                  caller's stack and doesn't have to remain valid for the

+ *                  duration of pending operation.

  * @param flags     Recv flag.
- * @param addr      Pointer to buffer to receive the address, or NULL.
+ * @param addr      Optional Pointer to buffer to receive the address.
  * @param addrlen   On input, specifies the length of the address buffer.
  *                  On output, it will be filled with the actual length of
- *                  the address.
+ *                  the address. This argument can be NULL if \c addr is not

+ *                  specified.
  *
  * @return
  *  - PJ_SUCCESS    If immediate data has been received. In this case, the 
@@ -385,56 +475,52 @@ PJ_DECL(pj_status_t) pj_ioqueue_recv( pj_ioqueue_t *ioque,
  *  - PJ_EPENDING   If the operation has been queued.
  *  - non-zero      The return value indicates the error code.
  */
-PJ_DECL(pj_status_t) pj_ioqueue_recvfrom( pj_ioqueue_t *ioque,
-					  pj_ioqueue_key_t *key,
+PJ_DECL(pj_status_t) pj_ioqueue_recvfrom( pj_ioqueue_key_t *key,

+                                          pj_ioqueue_op_key_t *op_key,
 					  void *buffer,
-					  pj_size_t buflen,
+					  pj_ssize_t *length,
                                           unsigned flags,
 					  pj_sockaddr_t *addr,
 					  int *addrlen);
 
-/**
- * Instruct the I/O Queue to write to the handle. This function will return
- * immediately (i.e. non-blocking) regardless whether some data has been 
- * transfered. If the function can't complete immediately, and the caller will
- * be notified about the completion when it calls pj_ioqueue_poll().
- *
- * @param ioque	    the I/O Queue.
- * @param key	    the key that identifies the handle.
- * @param data	    the data to send. Caller MUST make sure that this buffer 
- *		    remains valid until the write operation completes.
- * @param datalen   the length of the data.
- *
- * @return
- *  - PJ_SUCCESS    If data was immediately written.
- *  - PJ_EPENDING   If the operation has been queued.
- *  - non-zero      The return value indicates the error code.
- */
-PJ_DECL(pj_status_t) pj_ioqueue_write( pj_ioqueue_t *ioque,
-				       pj_ioqueue_key_t *key,
-				       const void *data,
-				       pj_size_t datalen);
 
 /**
- * This function behaves similarly as #pj_ioqueue_write(), except that
- * pj_sock_send() (or equivalent) will be called to send the data.
- *
- * @param ioque	    the I/O Queue.
- * @param key	    the key that identifies the handle.
- * @param data	    the data to send. Caller MUST make sure that this buffer 
+ * Instruct the I/O Queue to write to the handle. This function will return

+ * immediately (i.e. non-blocking) regardless whether some data has been 

+ * transfered. If the function can't complete immediately, the caller will

+ * be notified about the completion when it calls pj_ioqueue_poll(). If 

+ * operation completes immediately and data has been transfered, the function

+ * returns PJ_SUCCESS and the callback will NOT be called.

+ *

+ * @param key	    The key that identifies the handle.
+ * @param op_key    An operation specific key to be associated with the

+ *                  pending operation, so that application can keep track of

+ *                  which operation has been completed when the callback is

+ *                  called.

+ * @param data	    The data to send. Caller MUST make sure that this buffer 
  *		    remains valid until the write operation completes.
- * @param datalen   the length of the data.
- * @param flags     send flags.
+ * @param length    On input, it specifies the length of data to send. When

+ *                  data was sent immediately, this function returns PJ_SUCCESS

+ *                  and this parameter contains the length of data sent. If

+ *                  data can not be sent immediately, an asynchronous operation

+ *                  is scheduled and caller will be notified via callback the

+ *                  number of bytes sent. This parameter can point to local 

+ *                  variable on caller's stack and doesn't have to remain 

+ *                  valid until the operation has completed.
+ * @param flags     Send flags.
  *
  * @return
- *  - PJ_SUCCESS    If data was immediately written.
- *  - PJ_EPENDING   If the operation has been queued.
+ *  - PJ_SUCCESS    If data was immediately transfered. In this case, no

+ *                  pending operation has been scheduled and the callback

+ *                  WILL NOT be called.
+ *  - PJ_EPENDING   If the operation has been queued. Once data base been

+ *                  transfered, the callback will be called.
  *  - non-zero      The return value indicates the error code.
  */
-PJ_DECL(pj_status_t) pj_ioqueue_send( pj_ioqueue_t *ioque,
-				      pj_ioqueue_key_t *key,
+PJ_DECL(pj_status_t) pj_ioqueue_send( pj_ioqueue_key_t *key,

+                                      pj_ioqueue_op_key_t *op_key,
 				      const void *data,
-				      pj_size_t datalen,
+				      pj_ssize_t *length,
 				      unsigned flags );
 
 
@@ -442,24 +528,34 @@ PJ_DECL(pj_status_t) pj_ioqueue_send( pj_ioqueue_t *ioque,
  * This function behaves similarly as #pj_ioqueue_write(), except that
  * pj_sock_sendto() (or equivalent) will be called to send the data.
  *
- * @param ioque	    the I/O Queue.
  * @param key	    the key that identifies the handle.
+ * @param op_key    An operation specific key to be associated with the

+ *                  pending operation, so that application can keep track of

+ *                  which operation has been completed when the callback is

+ *                  called.

  * @param data	    the data to send. Caller MUST make sure that this buffer 
  *		    remains valid until the write operation completes.
- * @param datalen   the length of the data.
+ * @param length    On input, it specifies the length of data to send. When

+ *                  data was sent immediately, this function returns PJ_SUCCESS

+ *                  and this parameter contains the length of data sent. If

+ *                  data can not be sent immediately, an asynchronous operation

+ *                  is scheduled and caller will be notified via callback the

+ *                  number of bytes sent. This parameter can point to local 

+ *                  variable on caller's stack and doesn't have to remain 

+ *                  valid until the operation has completed.

  * @param flags     send flags.
- * @param addr      remote address.
- * @param addrlen   remote address length.
+ * @param addr      Optional remote address.
+ * @param addrlen   Remote address length, \c addr is specified.
  *
  * @return
  *  - PJ_SUCCESS    If data was immediately written.
  *  - PJ_EPENDING   If the operation has been queued.
  *  - non-zero      The return value indicates the error code.
  */
-PJ_DECL(pj_status_t) pj_ioqueue_sendto( pj_ioqueue_t *ioque,
-					pj_ioqueue_key_t *key,
+PJ_DECL(pj_status_t) pj_ioqueue_sendto( pj_ioqueue_key_t *key,

+                                        pj_ioqueue_op_key_t *op_key,
 					const void *data,
-					pj_size_t datalen,
+					pj_ssize_t *length,
                                         unsigned flags,
 					const pj_sockaddr_t *addr,
 					int addrlen);
diff --git a/pjlib/include/pj/list.h b/pjlib/include/pj/list.h
index 32bd5aec..95ae99fd 100644
--- a/pjlib/include/pj/list.h
+++ b/pjlib/include/pj/list.h
@@ -46,7 +46,7 @@ PJ_BEGIN_DECL
  * @hideinitializer
  */
 #define PJ_DECL_LIST_MEMBER(type)  type *prev; /** List @a prev. */ \
-                                   type *next; /** List @a next. */ 
+                                   type *next  /** List @a next. */ 
 
 
 /**
@@ -56,7 +56,7 @@ PJ_BEGIN_DECL
  */
 struct pj_list
 {
-    PJ_DECL_LIST_MEMBER(void)
+    PJ_DECL_LIST_MEMBER(void);
 };
 
 
diff --git a/pjlib/include/pj/pool.h b/pjlib/include/pj/pool.h
index ca6ac20a..4001079f 100644
--- a/pjlib/include/pj/pool.h
+++ b/pjlib/include/pj/pool.h
@@ -113,7 +113,7 @@ typedef void pj_pool_callback(pj_pool_t *pool, pj_size_t size);
  */
 typedef struct pj_pool_block
 {
-    PJ_DECL_LIST_MEMBER(struct pj_pool_block)   /**< List's prev and next.  */
+    PJ_DECL_LIST_MEMBER(struct pj_pool_block);  /**< List's prev and next.  */
     unsigned char    *buf;                      /**< Start of buffer.       */
     unsigned char    *cur;                      /**< Current alloc ptr.     */
     unsigned char    *end;                      /**< End of buffer.         */
@@ -126,7 +126,7 @@ typedef struct pj_pool_block
  */
 struct pj_pool_t
 {
-    PJ_DECL_LIST_MEMBER(struct pj_pool_t)
+    PJ_DECL_LIST_MEMBER(struct pj_pool_t);
 
     /** Pool name */
     char	    obj_name[PJ_MAX_OBJ_NAME];
diff --git a/pjlib/include/pj/sock_select.h b/pjlib/include/pj/sock_select.h
index 730d05e8..5b0c7002 100644
--- a/pjlib/include/pj/sock_select.h
+++ b/pjlib/include/pj/sock_select.h
@@ -97,16 +97,6 @@ PJ_DECL(pj_bool_t) PJ_FD_ISSET(pj_sock_t fd, const pj_fd_set_t *fdsetp);
 
 
 /**
- * Get the number of descriptors in the set.
- *
- * @param fdsetp    The descriptor set.
- *
- * @return          Number of descriptors in the set.
- */
-PJ_DECL(pj_size_t) PJ_FD_COUNT(const pj_fd_set_t *fdsetp);
-
-
-/**
  * This function wait for a number of file  descriptors to change status.
  * The behaviour is the same as select() function call which appear in
  * standard BSD socket libraries.
diff --git a/pjlib/include/pj/xml.h b/pjlib/include/pj/xml.h
index d487cc88..9777a514 100644
--- a/pjlib/include/pj/xml.h
+++ b/pjlib/include/pj/xml.h
@@ -30,7 +30,7 @@ typedef struct pj_xml_node pj_xml_node;
 /** This structure declares XML attribute. */
 struct pj_xml_attr
 {
-    PJ_DECL_LIST_MEMBER(pj_xml_attr)
+    PJ_DECL_LIST_MEMBER(pj_xml_attr);
     pj_str_t	name;	    /**< Attribute name. */
     pj_str_t	value;	    /**< Attribute value. */
 };
@@ -39,13 +39,13 @@ struct pj_xml_attr
  */
 typedef struct pj_xml_node_head
 {
-    PJ_DECL_LIST_MEMBER(pj_xml_node)
+    PJ_DECL_LIST_MEMBER(pj_xml_node);
 } pj_xml_node_head;
 
 /** This structure describes XML node. */
 struct pj_xml_node
 {
-    PJ_DECL_LIST_MEMBER(pj_xml_node)    /** List @a prev and @a next member */
+    PJ_DECL_LIST_MEMBER(pj_xml_node);   /** List @a prev and @a next member */
     pj_str_t		name;		/** Node name. */
     pj_xml_attr		attr_head;      /** Attribute list. */
     pj_xml_node_head	node_head;      /** Node list. */