nginx-0.1.16-RELEASE importrelease-0.1.16

    *) Bugfix: if the response were transferred by chunks, then on the HEAD
       request the final chunk was issued.

    *) Bugfix: the "Connection: keep-alive" header were issued, even if the
       keepalive_timeout directive forbade the keep-alive use.

    *) Bugfix: the errors in the ngx_http_fastcgi_module caused the
       segmentation faults.

    *) Bugfix: the compressed response encrypted by SSL may not transferred
       complete.

    *) Bugfix: the TCP-specific TCP_NODELAY, TCP_NOPSUH, and TCP_CORK
       options, are not used for the unix domain sockets.

    *) Feature: the rewrite directive supports the arguments rewriting.

    *) Bugfix: the response code 400 was returned for the POST request with
       the "Content-Length: 0" header; the bug had appeared in 0.1.14.

8 files changed, 214 insertions, 158 deletions

diff --git a/src/event/modules/ngx_devpoll_module.c b/src/event/modules/ngx_devpoll_module.c
index 34775f2b1..aa61df7de 100644
--- a/src/event/modules/ngx_devpoll_module.c
+++ b/src/event/modules/ngx_devpoll_module.c
@@ -310,7 +310,7 @@ static int ngx_devpoll_set_event(ngx_event_t *ev, int event, u_int flags)
 
 int ngx_devpoll_process_events(ngx_cycle_t *cycle)
 {
-    int                 events;
+    int                 events, revents;
     ngx_int_t           i;
     ngx_uint_t          j, lock, accept_lock, expire;
     size_t              n;
@@ -463,30 +463,40 @@ int ngx_devpoll_process_events(ngx_cycle_t *cycle)
         }
 #endif
 
+        revents = event_list[i].revents;
+
         ngx_log_debug3(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
                        "devpoll: fd:%d, ev:%04Xd, rev:%04Xd",
-                       event_list[i].fd,
-                       event_list[i].events, event_list[i].revents);
+                       event_list[i].fd, event_list[i].events, revents);
 
-        if (event_list[i].revents & (POLLERR|POLLHUP|POLLNVAL)) {
+        if (revents & (POLLERR|POLLHUP|POLLNVAL)) {
             ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
                           "ioctl(DP_POLL) error fd:%d ev:%04Xd rev:%04Xd",
-                          event_list[i].fd,
-                          event_list[i].events, event_list[i].revents);
+                          event_list[i].fd, event_list[i].events, revents);
         }
 
-        if (event_list[i].revents & ~(POLLIN|POLLOUT|POLLERR|POLLHUP|POLLNVAL))
-        {
+        if (revents & ~(POLLIN|POLLOUT|POLLERR|POLLHUP|POLLNVAL)) {
             ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
                           "strange ioctl(DP_POLL) events "
                           "fd:%d ev:%04Xd rev:%04Xd",
-                          event_list[i].fd,
-                          event_list[i].events, event_list[i].revents);
+                          event_list[i].fd, event_list[i].events, revents);
+        }
+
+        if ((revents & (POLLERR|POLLHUP|POLLNVAL))
+             && (revents & (POLLIN|POLLOUT)) == 0)
+        {
+            /*
+             * if the error events were returned without POLLIN or POLLOUT,
+             * then add these flags to handle the events at least in one
+             * active handler 
+             */
+
+            revents |= POLLIN|POLLOUT;
         }
 
         wev = c->write;
 
-        if ((event_list[i].events & (POLLOUT|POLLERR|POLLHUP)) && wev->active) {
+        if ((revents & POLLOUT) && wev->active) {
             wev->ready = 1;
 
             if (!ngx_threaded && !ngx_accept_mutex_held) {
@@ -505,7 +515,7 @@ int ngx_devpoll_process_events(ngx_cycle_t *cycle)
 
         rev = c->read;
 
-        if ((event_list[i].events & (POLLIN|POLLERR|POLLHUP)) && rev->active) {
+        if ((revents & POLLIN) && rev->active) {
             rev->ready = 1;
 
             if (!ngx_threaded && !ngx_accept_mutex_held) {
diff --git a/src/event/modules/ngx_epoll_module.c b/src/event/modules/ngx_epoll_module.c
index 907d6aeb5..af2781851 100644
--- a/src/event/modules/ngx_epoll_module.c
+++ b/src/event/modules/ngx_epoll_module.c
@@ -198,37 +198,40 @@ static void ngx_epoll_done(ngx_cycle_t *cycle)
 
 static int ngx_epoll_add_event(ngx_event_t *ev, int event, u_int flags)
 {
-    int                  op, prev;
+    int                  op;
+    uint32_t             events, prev;
     ngx_event_t         *e;
     ngx_connection_t    *c;
     struct epoll_event   ee;
 
     c = ev->data;
 
+    events = (uint32_t) event;
+
     if (event == NGX_READ_EVENT) {
         e = c->write;
         prev = EPOLLOUT;
 #if (NGX_READ_EVENT != EPOLLIN)
-        event = EPOLLIN;
+        events = EPOLLIN;
 #endif
 
     } else {
         e = c->read;
         prev = EPOLLIN;
 #if (NGX_WRITE_EVENT != EPOLLOUT)
-        event = EPOLLOUT;
+        events = EPOLLOUT;
 #endif
     }
 
     if (e->active) {
         op = EPOLL_CTL_MOD;
-        event |= prev;
+        events |= prev;
 
     } else {
         op = EPOLL_CTL_ADD;
     }
 
-    ee.events = event | flags;
+    ee.events = events | flags;
     ee.data.u64 = (uintptr_t) c | ev->instance;
 
     ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0,
@@ -252,14 +255,15 @@ static int ngx_epoll_add_event(ngx_event_t *ev, int event, u_int flags)
 
 static int ngx_epoll_del_event(ngx_event_t *ev, int event, u_int flags)
 {
-    int                  op, prev;
+    int                  op;
+    uint32_t             prev;
     ngx_event_t         *e;
     ngx_connection_t    *c;
     struct epoll_event   ee;
 
     /*
-     * when the file descriptor is closed the epoll automatically deletes
-     * it from its queue so we do not need to delete explicity the event
+     * when the file descriptor is closed, the epoll automatically deletes
+     * it from its queue, so we do not need to delete explicity the event
      * before the closing the file descriptor
      */
 
@@ -370,6 +374,7 @@ int ngx_epoll_process_events(ngx_cycle_t *cycle)
 {
     int                events;
     size_t             n;
+    uint32_t           revents;
     ngx_int_t          instance, i;
     ngx_uint_t         lock, accept_lock, expire;
     ngx_err_t          err;
@@ -521,27 +526,40 @@ int ngx_epoll_process_events(ngx_cycle_t *cycle)
         log = c->log ? c->log : cycle->log;
 #endif
 
+        revents = event_list[i].events;
+
         ngx_log_debug3(NGX_LOG_DEBUG_EVENT, log, 0,
                        "epoll: fd:%d ev:%04XD d:%p",
-                       c->fd, event_list[i].events, event_list[i].data);
+                       c->fd, revents, event_list[i].data);
 
-        if (event_list[i].events & (EPOLLERR|EPOLLHUP)) {
+        if (revents & (EPOLLERR|EPOLLHUP)) {
             ngx_log_debug2(NGX_LOG_DEBUG_EVENT, log, 0,
                            "epoll_wait() error on fd:%d ev:%04XD",
-                           c->fd, event_list[i].events);
+                           c->fd, revents);
         }
 
-        if (event_list[i].events & ~(EPOLLIN|EPOLLOUT|EPOLLERR|EPOLLHUP)) {
+        if (revents & ~(EPOLLIN|EPOLLOUT|EPOLLERR|EPOLLHUP)) {
             ngx_log_error(NGX_LOG_ALERT, log, 0,
                           "strange epoll_wait() events fd:%d ev:%04XD",
-                          c->fd, event_list[i].events);
+                          c->fd, revents);
+        }
+
+        if ((revents & (EPOLLERR|EPOLLHUP))
+             && (revents & (EPOLLIN|EPOLLOUT)) == 0)
+        {
+            /*
+             * if the error events were returned without EPOLLIN or EPOLLOUT,
+             * then add these flags to handle the events at least in one
+             * active handler
+             */
+
+            revents |= EPOLLIN|EPOLLOUT;
         }
 
         wev = c->write;
 
-        if ((event_list[i].events & (EPOLLOUT|EPOLLERR|EPOLLHUP))
-            && wev->active)
-        {
+        if ((revents & EPOLLOUT) && wev->active) {
+
             if (ngx_threaded) {
                 wev->posted_ready = 1;
                 ngx_post_event(wev);
@@ -564,9 +582,8 @@ int ngx_epoll_process_events(ngx_cycle_t *cycle)
          * if the accept event is the last one.
          */
 
-        if ((event_list[i].events & (EPOLLIN|EPOLLERR|EPOLLHUP))
-            && rev->active)
-        {
+        if ((revents & EPOLLIN) && rev->active) {
+
             if (ngx_threaded && !rev->accept) {
                 rev->posted_ready = 1;
 
diff --git a/src/event/modules/ngx_poll_module.c b/src/event/modules/ngx_poll_module.c
index 18f72c3b5..12726b58d 100644
--- a/src/event/modules/ngx_poll_module.c
+++ b/src/event/modules/ngx_poll_module.c
@@ -262,7 +262,7 @@ static ngx_int_t ngx_poll_del_event(ngx_event_t *ev, int event, u_int flags)
 
 static ngx_int_t ngx_poll_process_events(ngx_cycle_t *cycle)
 {
-    int                 ready;
+    int                 ready, revents;
     ngx_int_t           i, nready;
     ngx_uint_t          n, found, lock, expire;
     ngx_msec_t          timer;
@@ -378,33 +378,30 @@ static ngx_int_t ngx_poll_process_events(ngx_cycle_t *cycle)
 
     for (i = 0; i < nevents && ready; i++) {
 
+        revents = event_list[i].revents;
+
 #if 0
         ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
                        "poll: %d: fd:%d ev:%04Xd rev:%04Xd",
-                       i, event_list[i].fd,
-                       event_list[i].events, event_list[i].revents);
+                       i, event_list[i].fd, event_list[i].events, revents);
 #else
-        if (event_list[i].revents) {
+        if (revents) {
             ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
                            "poll: %d: fd:%d ev:%04Xd rev:%04Xd",
-                           i, event_list[i].fd,
-                           event_list[i].events, event_list[i].revents);
+                           i, event_list[i].fd, event_list[i].events, revents);
         }
 #endif
 
-        if (event_list[i].revents & POLLNVAL) {
+        if (revents & POLLNVAL) {
             ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
                           "poll() error fd:%d ev:%04Xd rev:%04Xd",
-                          event_list[i].fd,
-                          event_list[i].events, event_list[i].revents);
+                          event_list[i].fd, event_list[i].events, revents);
         }
 
-        if (event_list[i].revents & ~(POLLIN|POLLOUT|POLLERR|POLLHUP|POLLNVAL))
-        {
+        if (revents & ~(POLLIN|POLLOUT|POLLERR|POLLHUP|POLLNVAL)) {
             ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
                           "strange poll() events fd:%d ev:%04Xd rev:%04Xd",
-                          event_list[i].fd,
-                          event_list[i].events, event_list[i].revents);
+                          event_list[i].fd, event_list[i].events, revents);
         }
 
         if (event_list[i].fd == -1) {
@@ -447,9 +444,21 @@ static ngx_int_t ngx_poll_process_events(ngx_cycle_t *cycle)
             continue;
         }
 
+        if ((revents & (POLLERR|POLLHUP|POLLNVAL))
+             && (revents & (POLLIN|POLLOUT)) == 0)
+        {
+            /*
+             * if the error events were returned without POLLIN or POLLOUT,
+             * then add these flags to handle the events at least in one
+             * active handler
+             */
+
+            revents |= POLLIN|POLLOUT;
+        }
+
         found = 0;
 
-        if (event_list[i].revents & (POLLIN|POLLERR|POLLHUP|POLLNVAL)) {
+        if (revents & POLLIN) {
             found = 1;
 
             ev = c->read;
@@ -474,7 +483,7 @@ static ngx_int_t ngx_poll_process_events(ngx_cycle_t *cycle)
 #endif
         }
 
-        if (event_list[i].revents & (POLLOUT|POLLERR|POLLHUP|POLLNVAL)) {
+        if (revents & POLLOUT) {
             found = 1;
             ev = c->write;
             ev->ready = 1;
diff --git a/src/event/ngx_event.c b/src/event/ngx_event.c
index 9960225fc..d79e5ff71 100644
--- a/src/event/ngx_event.c
+++ b/src/event/ngx_event.c
@@ -201,7 +201,7 @@ static ngx_int_t ngx_event_module_init(ngx_cycle_t *cycle)
     }
 
 
-    /* TODO: 128 is cache line size */
+    /* TODO: adjust cache line size, 128 is P4 cache line size */
 
     size = 128            /* ngx_accept_mutex */
            + 128;         /* ngx_connection_counter */
diff --git a/src/event/ngx_event_accept.c b/src/event/ngx_event_accept.c
index 70a7c89a4..d809a77aa 100644
--- a/src/event/ngx_event_accept.c
+++ b/src/event/ngx_event_accept.c
@@ -10,29 +10,22 @@
 #include <nginx.h>
 
 
-typedef struct {
-    int         flag;
-    ngx_str_t  *name;
-} ngx_accept_log_ctx_t;
-
-
 static void ngx_close_accepted_socket(ngx_socket_t s, ngx_log_t *log);
-static u_char *ngx_accept_log_error(void *data, u_char *buf, size_t len);
+static u_char *ngx_accept_log_error(ngx_log_t *log, u_char *buf, size_t len);
 
 
 void ngx_event_accept(ngx_event_t *ev)
 {
-    ngx_uint_t             instance, accepted;
-    socklen_t              len;
-    struct sockaddr       *sa;
-    ngx_err_t              err;
-    ngx_log_t             *log;
-    ngx_pool_t            *pool;
-    ngx_socket_t           s;
-    ngx_event_t           *rev, *wev;
-    ngx_connection_t      *c, *ls;
-    ngx_event_conf_t      *ecf;
-    ngx_accept_log_ctx_t  *ctx;
+    ngx_uint_t         instance, accepted;
+    socklen_t          len;
+    struct sockaddr   *sa;
+    ngx_err_t          err;
+    ngx_log_t         *log;
+    ngx_pool_t        *pool;
+    ngx_socket_t       s;
+    ngx_event_t       *rev, *wev;
+    ngx_connection_t  *c, *ls;
+    ngx_event_conf_t  *ecf;
 
     ecf = ngx_event_get_conf(ngx_cycle->conf_ctx, ngx_event_core_module);
 
@@ -81,16 +74,7 @@ void ngx_event_accept(ngx_event_t *ev)
         ngx_memcpy(log, ls->log, sizeof(ngx_log_t));
         pool->log = log;
 
-        if (!(ctx = ngx_palloc(pool, sizeof(ngx_accept_log_ctx_t)))) {
-            ngx_destroy_pool(pool);
-            return;
-        }
-
-        /* -1 disables the connection number logging */
-        ctx->flag = -1;
-        ctx->name = &ls->listening->addr_text;
-
-        log->data = ctx;
+        log->data = &ls->listening->addr_text;
         log->handler = ngx_accept_log_error;
 
         len = ls->listening->socklen;
@@ -467,9 +451,7 @@ static void ngx_close_accepted_socket(ngx_socket_t s, ngx_log_t *log)
 }
 
 
-static u_char *ngx_accept_log_error(void *data, u_char *buf, size_t len)
+static u_char *ngx_accept_log_error(ngx_log_t *log, u_char *buf, size_t len)
 {
-    ngx_accept_log_ctx_t  *ctx = data;
-
-    return ngx_snprintf(buf, len, " while accept() on %V", ctx->name);
+    return ngx_snprintf(buf, len, " while accept() on %V", log->data);
 }
diff --git a/src/event/ngx_event_connect.c b/src/event/ngx_event_connect.c
index 75277a7bd..1a3e8f806 100644
--- a/src/event/ngx_event_connect.c
+++ b/src/event/ngx_event_connect.c
@@ -8,11 +8,8 @@
 #include <ngx_core.h>
 #include <ngx_event.h>
 #include <ngx_event_connect.h>
-#include <nginx.h>
 
 
-/* AF_INET only */
-
 ngx_int_t ngx_event_connect_peer(ngx_peer_connection_t *pc)
 {
     int                  rc;
@@ -170,6 +167,7 @@ ngx_int_t ngx_event_connect_peer(ngx_peer_connection_t *pc)
     }
 
 #if (NGX_WIN32)
+
     /*
      * Winsock assignes a socket number divisible by 4
      * so to find a connection we divide a socket number by 4.
@@ -232,6 +230,11 @@ ngx_int_t ngx_event_connect_peer(ngx_peer_connection_t *pc)
 
     c->log_error = pc->log_error;
 
+    if (peer->sockaddr->sa_family != AF_INET) {
+        c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED;
+        c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED;
+    }
+
     pc->connection = c;
 
     /*
@@ -289,6 +292,9 @@ ngx_int_t ngx_event_connect_peer(ngx_peer_connection_t *pc)
         }
  
         ngx_log_debug0(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connected");
+
+        wev->ready = 1;
+
         return NGX_OK;
     }
 
diff --git a/src/event/ngx_event_pipe.c b/src/event/ngx_event_pipe.c
index ad67931c1..0206bf8fe 100644
--- a/src/event/ngx_event_pipe.c
+++ b/src/event/ngx_event_pipe.c
@@ -17,8 +17,6 @@ static ngx_int_t ngx_event_pipe_write_chain_to_temp_file(ngx_event_pipe_t *p);
 static ngx_inline void ngx_event_pipe_remove_shadow_links(ngx_buf_t *buf);
 static ngx_inline void ngx_event_pipe_free_shadow_raw_buf(ngx_chain_t **free,
                                                           ngx_buf_t *buf);
-static ngx_inline void ngx_event_pipe_add_free_buf(ngx_chain_t **chain,
-                                                   ngx_chain_t *cl);
 static ngx_int_t ngx_event_pipe_drain_chains(ngx_event_pipe_t *p);
 
 
@@ -29,6 +27,8 @@ ngx_int_t ngx_event_pipe(ngx_event_pipe_t *p, int do_write)
 
     for ( ;; ) {
         if (do_write) {
+            p->log->action = "sending to client";
+
             if (ngx_event_pipe_write_to_downstream(p) == NGX_ABORT) {
                 return NGX_ABORT;
             }
@@ -37,6 +37,8 @@ ngx_int_t ngx_event_pipe(ngx_event_pipe_t *p, int do_write)
         p->read = 0;
         p->upstream_blocked = 0;
 
+        p->log->action = "reading upstream";
+
         if (ngx_event_pipe_read_upstream(p) == NGX_ABORT) {
             return NGX_ABORT;
         }
@@ -77,12 +79,12 @@ ngx_int_t ngx_event_pipe(ngx_event_pipe_t *p, int do_write)
 }
 
 
-ngx_int_t ngx_event_pipe_read_upstream(ngx_event_pipe_t *p)
+static ngx_int_t ngx_event_pipe_read_upstream(ngx_event_pipe_t *p)
 {
     ssize_t       n, size;
     ngx_int_t     rc;
     ngx_buf_t    *b;
-    ngx_chain_t  *chain, *cl, *tl;
+    ngx_chain_t  *chain, *cl;
 
     if (p->upstream_eof || p->upstream_error || p->upstream_done) {
         return NGX_OK;
@@ -169,8 +171,12 @@ ngx_int_t ngx_event_pipe_read_upstream(ngx_event_pipe_t *p)
 
                 p->allocated++;
 
-                ngx_alloc_link_and_set_buf(tl, b, p->pool, NGX_ABORT);
-                chain = tl;
+                if (!(chain = ngx_alloc_chain_link(p->pool))) {
+                    return NGX_ABORT;
+                }
+
+                chain->buf = b;
+                chain->next = NULL;
 
             } else if (!p->cachable && p->downstream->write->ready) {
 
@@ -191,7 +197,7 @@ ngx_int_t ngx_event_pipe_read_upstream(ngx_event_pipe_t *p)
             {
 
                 /*
-                 * if it's allowed then save some bufs from r->in
+                 * if it is allowed, then save some bufs from r->in
                  * to a temporary file, and add them to a r->out chain
                  */
 
@@ -227,7 +233,7 @@ ngx_int_t ngx_event_pipe_read_upstream(ngx_event_pipe_t *p)
 
             } else {
 
-                /* if there're no bufs to read in then disable a level event */
+                /* there are no bufs to read in */
 
                 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, p->log, 0,
                                "no pipe bufs to read in");
@@ -298,52 +304,56 @@ ngx_int_t ngx_event_pipe_read_upstream(ngx_event_pipe_t *p)
 
 #if (NGX_DEBUG)
 
-    if (p->in || p->busy || p->free_raw_bufs) {
-        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, p->log, 0, "pipe buf");
-    }
-
     for (cl = p->busy; cl; cl = cl->next) {
-        ngx_log_debug3(NGX_LOG_DEBUG_EVENT, p->log, 0,
-                       "pipe buf busy %p, pos %p, size: %z",
+        ngx_log_debug8(NGX_LOG_DEBUG_EVENT, p->log, 0,
+                       "pipe buf busy s:%d t:%d f:%d "
+                       "%p, pos %p, size: %z "
+                       "file: %O, size: %z",
+                       (cl->buf->shadow ? 1 : 0),
+                       cl->buf->temporary, cl->buf->in_file,
                        cl->buf->start, cl->buf->pos,
-                       cl->buf->last - cl->buf->pos);
+                       cl->buf->last - cl->buf->pos,
+                       cl->buf->file_pos,
+                       cl->buf->file_last - cl->buf->file_pos);
     }
 
     for (cl = p->out; cl; cl = cl->next) {
-        if (cl->buf->in_file && cl->buf->temporary) {
-            ngx_log_debug5(NGX_LOG_DEBUG_EVENT, p->log, 0,
-                           "pipe buf out shadow %p, pos %p, size: %z "
-                           "file: %O, size: %z",
-                           cl->buf->start, cl->buf->pos,
-                           cl->buf->last - cl->buf->pos,
-                           cl->buf->file_pos,
-                           cl->buf->file_last - cl->buf->file_pos);
-
-        } else if (cl->buf->in_file) {
-            ngx_log_debug2(NGX_LOG_DEBUG_EVENT, p->log, 0,
-                           "pipe buf out file %O, size: %z",
-                           cl->buf->file_pos,
-                           cl->buf->file_last - cl->buf->file_pos);
-        } else {
-            ngx_log_debug3(NGX_LOG_DEBUG_EVENT, p->log, 0,
-                           "pipe buf out %p, pos %p, size: %z",
-                           cl->buf->start, cl->buf->pos,
-                           cl->buf->last - cl->buf->pos);
-        }
+        ngx_log_debug8(NGX_LOG_DEBUG_EVENT, p->log, 0,
+                       "pipe buf out  s:%d t:%d f:%d "
+                       "%p, pos %p, size: %z "
+                       "file: %O, size: %z",
+                       (cl->buf->shadow ? 1 : 0),
+                       cl->buf->temporary, cl->buf->in_file,
+                       cl->buf->start, cl->buf->pos,
+                       cl->buf->last - cl->buf->pos,
+                       cl->buf->file_pos,
+                       cl->buf->file_last - cl->buf->file_pos);
     }
 
     for (cl = p->in; cl; cl = cl->next) {
-        ngx_log_debug3(NGX_LOG_DEBUG_EVENT, p->log, 0,
-                       "pipe buf in %p, pos %p, size: %z",
+        ngx_log_debug8(NGX_LOG_DEBUG_EVENT, p->log, 0,
+                       "pipe buf in   s:%d t:%d f:%d "
+                       "%p, pos %p, size: %z "
+                       "file: %O, size: %z",
+                       (cl->buf->shadow ? 1 : 0),
+                       cl->buf->temporary, cl->buf->in_file,
                        cl->buf->start, cl->buf->pos,
-                       cl->buf->last - cl->buf->pos);
+                       cl->buf->last - cl->buf->pos,
+                       cl->buf->file_pos,
+                       cl->buf->file_last - cl->buf->file_pos);
     }
 
     for (cl = p->free_raw_bufs; cl; cl = cl->next) {
-        ngx_log_debug3(NGX_LOG_DEBUG_EVENT, p->log, 0,
-                       "pipe buf free %p, last %p, size: %z",
-                       cl->buf->start, cl->buf->last,
-                       cl->buf->end - cl->buf->last);
+        ngx_log_debug8(NGX_LOG_DEBUG_EVENT, p->log, 0,
+                       "pipe buf free s:%d t:%d f:%d "
+                       "%p, pos %p, size: %z "
+                       "file: %O, size: %z",
+                       (cl->buf->shadow ? 1 : 0),
+                       cl->buf->temporary, cl->buf->in_file,
+                       cl->buf->start, cl->buf->pos,
+                       cl->buf->last - cl->buf->pos,
+                       cl->buf->file_pos,
+                       cl->buf->file_last - cl->buf->file_pos);
     }
 
 #endif
@@ -377,12 +387,11 @@ ngx_int_t ngx_event_pipe_read_upstream(ngx_event_pipe_t *p)
 }
 
 
-ngx_int_t ngx_event_pipe_write_to_downstream(ngx_event_pipe_t *p)
+static ngx_int_t ngx_event_pipe_write_to_downstream(ngx_event_pipe_t *p)
 {
     size_t        bsize;
     ngx_uint_t    flush;
-    ngx_buf_t    *b;
-    ngx_chain_t  *out, **ll, *cl, *tl;
+    ngx_chain_t  *out, **ll, *cl;
 
     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, p->log, 0,
                    "pipe write downstream: %d", p->downstream->write->ready);
@@ -522,11 +531,9 @@ ngx_int_t ngx_event_pipe_write_to_downstream(ngx_event_pipe_t *p)
             /* add the free shadow raw buf to p->free_raw_bufs */
 
             if (cl->buf->last_shadow) {
-                b = cl->buf->shadow;
-                b->pos = b->last = b->start;
-                b->shadow = NULL;
-                ngx_alloc_link_and_set_buf(tl, b, p->pool, NGX_ABORT);
-                ngx_event_pipe_add_free_buf(&p->free_raw_bufs, tl);
+                if (ngx_event_pipe_add_free_buf(p, cl->buf->shadow) != NGX_OK) {
+                    return NGX_ABORT;
+                }
 
                 cl->buf->last_shadow = 0;
             }
@@ -631,13 +638,21 @@ static ngx_int_t ngx_event_pipe_write_chain_to_temp_file(ngx_event_pipe_t *p)
         ngx_chain_add_link(p->out, p->last_out, cl);
 
         if (b->last_shadow) {
-            b->shadow->pos = b->shadow->start;
-            b->shadow->last = b->shadow->start;
 
-            ngx_alloc_link_and_set_buf(tl, b->shadow, p->pool, NGX_ABORT);
+            if (!(tl = ngx_alloc_chain_link(p->pool))) {
+                return NGX_ABORT;
+            }
+
+            tl->buf = b->shadow;
+            tl->next = NULL;
 
             *last_free = tl;
             last_free = &tl->next;
+
+            b->shadow->pos = b->shadow->start;
+            b->shadow->last = b->shadow->start;
+
+            ngx_event_pipe_remove_shadow_links(b->shadow);
         }
     }
 
@@ -673,7 +688,12 @@ ngx_int_t ngx_event_pipe_copy_input_filter(ngx_event_pipe_t *p, ngx_buf_t *buf)
     b->recycled = 1;
     buf->shadow = b;
 
-    ngx_alloc_link_and_set_buf(cl, b, p->pool, NGX_ERROR);
+    if (!(cl = ngx_alloc_chain_link(p->pool))) {
+        return NGX_ERROR;
+    }
+
+    cl->buf = b;
+    cl->next = NULL;
 
     ngx_log_debug1(NGX_LOG_DEBUG_EVENT, p->log, 0, "input buf #%d", b->num);
 
@@ -742,28 +762,48 @@ static ngx_inline void ngx_event_pipe_free_shadow_raw_buf(ngx_chain_t **free,
 }
 
 
-static ngx_inline void ngx_event_pipe_add_free_buf(ngx_chain_t **chain,
-                                                   ngx_chain_t *cl)
+ngx_int_t ngx_event_pipe_add_free_buf(ngx_event_pipe_t *p, ngx_buf_t *b)
 {
-    if (*chain == NULL) {
-        *chain = cl;
-        return;
+    ngx_chain_t  *cl;
+
+    if (!(cl = ngx_alloc_chain_link(p->pool))) {
+        return NGX_ERROR;
     }
 
-    if ((*chain)->buf->pos != (*chain)->buf->last) {
-        cl->next = (*chain)->next;
-        (*chain)->next = cl;
+    b->pos = b->start;
+    b->last = b->start;
+    b->shadow = NULL;
 
-    } else {
-        cl->next = (*chain);
-        (*chain) = cl;
+    cl->buf = b;
+
+    if (p->free_raw_bufs == NULL) {
+        p->free_raw_bufs = cl;
+        cl->next = NULL;
+
+        return NGX_OK;
     }
+
+    if (p->free_raw_bufs->buf->pos == p->free_raw_bufs->buf->last) {
+
+        /* add the free buf to the list start */
+
+        cl->next = p->free_raw_bufs;
+        p->free_raw_bufs = cl;
+
+        return NGX_OK;
+    }
+
+    /* the first free buf is partialy filled, thus add the free buf after it */
+
+    cl->next = p->free_raw_bufs->next;
+    p->free_raw_bufs->next = cl;
+
+    return NGX_OK;
 }
 
 
 static ngx_int_t ngx_event_pipe_drain_chains(ngx_event_pipe_t *p)
 {
-    ngx_buf_t    *b;
     ngx_chain_t  *cl, *tl;
 
     for ( ;; ) {
@@ -785,19 +825,10 @@ static ngx_int_t ngx_event_pipe_drain_chains(ngx_event_pipe_t *p)
 
         while (cl) {
             if (cl->buf->last_shadow) {
-                b = cl->buf->shadow;
-                b->pos = b->last = b->start;
-                b->shadow = NULL;
-
-                if (!(tl = ngx_alloc_chain_link(p->pool))) {
+                if (ngx_event_pipe_add_free_buf(p, cl->buf->shadow) != NGX_OK) {
                     return NGX_ABORT;
                 }
 
-                tl->buf = b;
-                tl->next = NULL;
-
-                ngx_event_pipe_add_free_buf(&p->free_raw_bufs, tl);
-
                 cl->buf->last_shadow = 0;
             }
 
diff --git a/src/event/ngx_event_pipe.h b/src/event/ngx_event_pipe.h
index 9781820d9..887f8b7f2 100644
--- a/src/event/ngx_event_pipe.h
+++ b/src/event/ngx_event_pipe.h
@@ -88,6 +88,7 @@ struct ngx_event_pipe_s {
 
 ngx_int_t ngx_event_pipe(ngx_event_pipe_t *p, int do_write);
 ngx_int_t ngx_event_pipe_copy_input_filter(ngx_event_pipe_t *p, ngx_buf_t *buf);
+ngx_int_t ngx_event_pipe_add_free_buf(ngx_event_pipe_t *p, ngx_buf_t *b);
 
 
 #endif /* _NGX_EVENT_PIPE_H_INCLUDED_ */