diff options
| -rw-r--r-- | auto/modules | 1 | ||||
| -rw-r--r-- | auto/options | 3 | ||||
| -rw-r--r-- | auto/os/features | 40 | ||||
| -rw-r--r-- | auto/sources | 2 | ||||
| -rw-r--r-- | src/core/ngx_buf.h | 15 | ||||
| -rw-r--r-- | src/core/ngx_core.h | 1 | ||||
| -rw-r--r-- | src/core/ngx_file.h | 5 | ||||
| -rw-r--r-- | src/core/ngx_output_chain.c | 24 | ||||
| -rw-r--r-- | src/event/modules/ngx_epoll_module.c | 217 | ||||
| -rw-r--r-- | src/event/ngx_event.h | 27 | ||||
| -rw-r--r-- | src/event/ngx_event_pipe.c | 25 | ||||
| -rw-r--r-- | src/http/ngx_http_cache.h | 1 | ||||
| -rw-r--r-- | src/http/ngx_http_copy_filter_module.c | 53 | ||||
| -rw-r--r-- | src/http/ngx_http_core_module.c | 17 | ||||
| -rw-r--r-- | src/http/ngx_http_core_module.h | 3 | ||||
| -rw-r--r-- | src/http/ngx_http_file_cache.c | 109 | ||||
| -rw-r--r-- | src/http/ngx_http_request.c | 28 | ||||
| -rw-r--r-- | src/http/ngx_http_request.h | 9 | ||||
| -rw-r--r-- | src/http/ngx_http_upstream.c | 74 | ||||
| -rw-r--r-- | src/os/unix/ngx_file_aio_read.c | 210 | ||||
| -rw-r--r-- | src/os/unix/ngx_files.h | 8 | ||||
| -rw-r--r-- | src/os/unix/ngx_freebsd_config.h | 9 | ||||
| -rw-r--r-- | src/os/unix/ngx_linux_aio_read.c | 131 | ||||
| -rw-r--r-- | src/os/unix/ngx_linux_config.h | 7 | ||||
| -rw-r--r-- | src/os/unix/ngx_posix_config.h | 6 | ||||
| -rw-r--r-- | src/os/unix/ngx_solaris_config.h | 15 |
26 files changed, 958 insertions, 82 deletions
diff --git a/auto/modules b/auto/modules index 01951c738..146b7d7f2 100644 --- a/auto/modules +++ b/auto/modules @@ -41,6 +41,7 @@ fi if [ $NGX_TEST_BUILD_EPOLL = YES ]; then have=NGX_HAVE_EPOLL . auto/have + have=NGX_HAVE_EVENTFD . auto/have have=NGX_TEST_BUILD_EPOLL . auto/have EVENT_MODULES="$EVENT_MODULES $EPOLL_MODULE" CORE_SRCS="$CORE_SRCS $EPOLL_SRCS" diff --git a/auto/options b/auto/options index bc9e80917..6bc01cd9f 100644 --- a/auto/options +++ b/auto/options @@ -43,6 +43,7 @@ EVENT_AIO=NO USE_THREADS=NO +NGX_FILE_AIO=NO NGX_IPV6=NO HTTP=YES @@ -170,6 +171,7 @@ do #--with-threads=*) USE_THREADS="$value" ;; #--with-threads) USE_THREADS="pthreads" ;; + --with-file-aio) NGX_FILE_AIO=YES ;; --with-ipv6) NGX_IPV6=YES ;; --without-http) HTTP=NO ;; @@ -305,6 +307,7 @@ cat << END --with-poll_module enable poll module --without-poll_module disable poll module + --with-file-aio enable file aio support --with-ipv6 enable ipv6 support --with-http_ssl_module enable ngx_http_ssl_module diff --git a/auto/os/features b/auto/os/features index 140e26b89..72ad534c3 100644 --- a/auto/os/features +++ b/auto/os/features @@ -274,3 +274,43 @@ if [ $ngx_found != yes ]; then CORE_LIBS="$CORE_LIBS -lrt" fi fi + + +if [ $NGX_FILE_AIO = YES ]; then + ngx_feature="kqueue AIO support" + ngx_feature_name="NGX_HAVE_FILE_AIO" + ngx_feature_run=no + ngx_feature_incs="#include <aio.h>" + ngx_feature_path= + ngx_feature_libs= + ngx_feature_test="int n; struct aiocb iocb; + iocb.aio_sigevent.sigev_notify = SIGEV_KEVENT; + n = aio_read(&iocb)" + . auto/feature + + if [ $ngx_found = yes ]; then + CORE_SRCS="$CORE_SRCS $FILE_AIO_SRCS" + fi +fi + + +if [ $NGX_FILE_AIO = YES ]; then + ngx_feature="Linux AIO support" + ngx_feature_name="NGX_HAVE_FILE_AIO" + ngx_feature_run=no + ngx_feature_incs="#include <linux/aio_abi.h> + #include <sys/syscall.h>" + ngx_feature_path= + ngx_feature_libs= + ngx_feature_test="int n = SYS_eventfd; + struct iocb iocb; + iocb.aio_lio_opcode = IOCB_CMD_PREAD; + iocb.aio_flags = IOCB_FLAG_RESFD; + iocb.aio_resfd = -1;" + . auto/feature + + if [ $ngx_found = yes ]; then + have=NGX_HAVE_EVENTFD . auto/have + CORE_SRCS="$CORE_SRCS $LINUX_AIO_SRCS" + fi +fi diff --git a/auto/sources b/auto/sources index 616b9f11e..73afc9276 100644 --- a/auto/sources +++ b/auto/sources @@ -125,6 +125,8 @@ AIO_SRCS="src/event/modules/ngx_aio_module.c \ src/os/unix/ngx_aio_read_chain.c \ src/os/unix/ngx_aio_write_chain.c" +FILE_AIO_SRCS="src/os/unix/ngx_file_aio_read.c" +LINUX_AIO_SRCS="src/os/unix/ngx_linux_aio_read.c" UNIX_INCS="$CORE_INCS $EVENT_INCS src/os/unix" diff --git a/src/core/ngx_buf.h b/src/core/ngx_buf.h index 56e03992d..b455270ad 100644 --- a/src/core/ngx_buf.h +++ b/src/core/ngx_buf.h @@ -67,9 +67,16 @@ typedef struct { } ngx_bufs_t; +typedef struct ngx_output_chain_ctx_s ngx_output_chain_ctx_t; + typedef ngx_int_t (*ngx_output_chain_filter_pt)(void *ctx, ngx_chain_t *in); -typedef struct { +#if (NGX_HAVE_FILE_AIO) +typedef void (*ngx_output_chain_aio_pt)(ngx_output_chain_ctx_t *ctx, + ngx_file_t *file); +#endif + +struct ngx_output_chain_ctx_s { ngx_buf_t *buf; ngx_chain_t *in; ngx_chain_t *free; @@ -90,7 +97,11 @@ typedef struct { ngx_output_chain_filter_pt output_filter; void *filter_ctx; -} ngx_output_chain_ctx_t; + +#if (NGX_HAVE_FILE_AIO) + ngx_output_chain_aio_pt aio; +#endif +}; typedef struct { diff --git a/src/core/ngx_core.h b/src/core/ngx_core.h index d5f18b84c..58d0b8030 100644 --- a/src/core/ngx_core.h +++ b/src/core/ngx_core.h @@ -19,6 +19,7 @@ typedef struct ngx_open_file_s ngx_open_file_t; typedef struct ngx_command_s ngx_command_t; typedef struct ngx_file_s ngx_file_t; typedef struct ngx_event_s ngx_event_t; +typedef struct ngx_event_aio_s ngx_event_aio_t; typedef struct ngx_connection_s ngx_connection_t; typedef void (*ngx_event_handler_pt)(ngx_event_t *ev); diff --git a/src/core/ngx_file.h b/src/core/ngx_file.h index 56315433b..8b502539a 100644 --- a/src/core/ngx_file.h +++ b/src/core/ngx_file.h @@ -22,10 +22,15 @@ struct ngx_file_s { ngx_log_t *log; +#if (NGX_HAVE_FILE_AIO) + ngx_event_aio_t *aio; +#endif + unsigned valid_info:1; unsigned directio:1; }; + #define NGX_MAX_PATH_LEVEL 3 diff --git a/src/core/ngx_output_chain.c b/src/core/ngx_output_chain.c index 6d6d241c1..99e0a9434 100644 --- a/src/core/ngx_output_chain.c +++ b/src/core/ngx_output_chain.c @@ -519,8 +519,26 @@ ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx) #endif +#if (NGX_HAVE_FILE_AIO) + + if (ctx->aio) { + n = ngx_file_aio_read(src->file, dst->pos, (size_t) size, + src->file_pos, ctx->pool); + if (n == NGX_AGAIN) { + ctx->aio(ctx, src->file); + return NGX_AGAIN; + } + + } else { + n = ngx_read_file(src->file, dst->pos, (size_t) size, + src->file_pos); + } +#else + n = ngx_read_file(src->file, dst->pos, (size_t) size, src->file_pos); +#endif + #if (NGX_HAVE_ALIGNED_DIRECTIO) if (ctx->unaligned) { @@ -545,12 +563,6 @@ ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx) return (ngx_int_t) n; } -#if (NGX_FILE_AIO_READ) - if (n == NGX_AGAIN) { - return (ngx_int_t) n; - } -#endif - if (n != size) { ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0, ngx_read_file_n " read only %z of %O from \"%s\"", diff --git a/src/event/modules/ngx_epoll_module.c b/src/event/modules/ngx_epoll_module.c index 4e016798a..dc5d4226e 100644 --- a/src/event/modules/ngx_epoll_module.c +++ b/src/event/modules/ngx_epoll_module.c @@ -58,6 +58,29 @@ int epoll_wait(int epfd, struct epoll_event *events, int nevents, int timeout) return -1; } +#if (NGX_HAVE_FILE_AIO) + +#define SYS_io_setup 245 +#define SYS_io_destroy 246 +#define SYS_io_getevents 247 +#define SYS_eventfd 323 + +typedef u_int aio_context_t; + +struct io_event { + uint64_t data; /* the data field from the iocb */ + uint64_t obj; /* what iocb this event came from */ + int64_t res; /* result code for this event */ + int64_t res2; /* secondary result */ +}; + + +int eventfd(u_int initval) +{ + return -1; +} + +#endif #endif @@ -78,6 +101,10 @@ static ngx_int_t ngx_epoll_del_connection(ngx_connection_t *c, static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags); +#if (NGX_HAVE_FILE_AIO) +static void ngx_epoll_eventfd_handler(ngx_event_t *ev); +#endif + static void *ngx_epoll_create_conf(ngx_cycle_t *cycle); static char *ngx_epoll_init_conf(ngx_cycle_t *cycle, void *conf); @@ -85,6 +112,15 @@ static int ep = -1; static struct epoll_event *event_list; static ngx_uint_t nevents; +#if (NGX_HAVE_FILE_AIO) + +int ngx_eventfd = -1; +aio_context_t ngx_aio_ctx = 0; + +static ngx_event_t ngx_eventfd_event; +static ngx_connection_t ngx_eventfd_conn; + +#endif static ngx_str_t epoll_name = ngx_string("epoll"); @@ -136,6 +172,42 @@ ngx_module_t ngx_epoll_module = { }; +#if (NGX_HAVE_FILE_AIO) + +/* + * We call io_setup(), io_destroy() io_submit(), and io_getevents() directly + * as syscalls instead of libaio usage, because the library header file + * supports eventfd() since 0.3.107 version only. + * + * Also we do not use eventfd() in glibc, because glibc supports it + * since 2.8 version and glibc maps two syscalls eventfd() and eventfd2() + * into single eventfd() function with different number of parameters. + */ + +static long +io_setup(u_int nr_reqs, aio_context_t *ctx) +{ + return syscall(SYS_io_setup, nr_reqs, ctx); +} + + +static int +io_destroy(aio_context_t ctx) +{ + return syscall(SYS_io_destroy, ctx); +} + + +static long +io_getevents(aio_context_t ctx, long min_nr, long nr, struct io_event *events, + struct timespec *tmo) +{ + return syscall(SYS_io_getevents, ctx, min_nr, nr, events, tmo); +} + +#endif + + static ngx_int_t ngx_epoll_init(ngx_cycle_t *cycle, ngx_msec_t timer) { @@ -151,6 +223,55 @@ ngx_epoll_init(ngx_cycle_t *cycle, ngx_msec_t timer) "epoll_create() failed"); return NGX_ERROR; } + +#if (NGX_HAVE_FILE_AIO) + { + int n; + struct epoll_event ee; + + ngx_eventfd = syscall(SYS_eventfd, 0); + + if (ngx_eventfd == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "eventfd() failed"); + return NGX_ERROR; + } + + n = 1; + + if (ioctl(ngx_eventfd, FIONBIO, &n) == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "ioctl(eventfd, FIONBIO) failed"); + } + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "eventfd: %d", ngx_eventfd); + + n = io_setup(1024, &ngx_aio_ctx); + + if (n != 0) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, -n, "io_setup() failed"); + return NGX_ERROR; + } + + ngx_eventfd_event.data = &ngx_eventfd_conn; + ngx_eventfd_event.handler = ngx_epoll_eventfd_handler; + ngx_eventfd_event.log = cycle->log; + ngx_eventfd_event.active = 1; + ngx_eventfd_conn.fd = ngx_eventfd; + ngx_eventfd_conn.read = &ngx_eventfd_event; + ngx_eventfd_conn.log = cycle->log; + + ee.events = EPOLLIN|EPOLLET; + ee.data.ptr = &ngx_eventfd_conn; + + if (epoll_ctl(ep, EPOLL_CTL_ADD, ngx_eventfd, &ee) == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "epoll_ctl(EPOLL_CTL_ADD, eventfd) failed"); + return NGX_ERROR; + } + } +#endif } if (nevents < epcf->events) { @@ -193,6 +314,17 @@ ngx_epoll_done(ngx_cycle_t *cycle) ep = -1; +#if (NGX_HAVE_FILE_AIO) + + if (io_destroy(ngx_aio_ctx) != 0) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "io_destroy() failed"); + } + + ngx_aio_ctx = 0; + +#endif + ngx_free(event_list); event_list = NULL; @@ -537,6 +669,91 @@ ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) } +#if (NGX_HAVE_FILE_AIO) + +static void +ngx_epoll_eventfd_handler(ngx_event_t *ev) +{ + int n; + long i, events; + uint64_t ready; + ngx_err_t err; + ngx_event_t *e; + ngx_event_aio_t *aio; + struct io_event event[64]; + struct timespec ts; + + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd handler"); + + n = read(ngx_eventfd, &ready, 8); + + err = ngx_errno; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0, "eventfd: %d", n); + + if (n != 8) { + if (n == -1) { + if (err == NGX_EAGAIN) { + return; + } + + ngx_log_error(NGX_LOG_ALERT, ev->log, err, "read(eventfd) failed"); + return; + } + + ngx_log_error(NGX_LOG_ALERT, ev->log, 0, + "read(eventfd) returned only %d bytes", n); + return; + } + + ts.tv_sec = 0; + ts.tv_nsec = 0; + + while (ready) { + + events = io_getevents(ngx_aio_ctx, 1, 64, event, &ts); + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "io_getevents: %l", events); + + if (events > 0) { + ready -= events; + + for (i = 0; i < events; i++) { + + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "io_event: %uXL %uXL %L %L", + event[i].data, event[i].obj, + event[i].res, event[i].res2); + + e = (ngx_event_t *) (uintptr_t) event[i].data; + + e->complete = 1; + e->active = 0; + e->ready = 1; + + aio = e->data; + aio->res = event[i].res; + + ngx_post_event(e, &ngx_posted_events); + } + + continue; + } + + if (events == 0) { + return; + } + + /* events < 0 */ + ngx_log_error(NGX_LOG_ALERT, ev->log, -events, "io_getevents() failed"); + return; + } +} + +#endif + + static void * ngx_epoll_create_conf(ngx_cycle_t *cycle) { diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h index 33c8cdc2d..ab38546a0 100644 --- a/src/event/ngx_event.h +++ b/src/event/ngx_event.h @@ -189,6 +189,33 @@ struct ngx_event_s { }; +#if (NGX_HAVE_FILE_AIO) + +struct ngx_event_aio_s { + void *data; + ngx_event_handler_pt handler; + ngx_file_t *file; + + ngx_fd_t fd; + +#if (NGX_HAVE_EVENTFD) + int64_t res; +#if (NGX_TEST_BUILD_EPOLL) + ngx_err_t err; + size_t nbytes; +#endif +#else + ngx_err_t err; + size_t nbytes; +#endif + + ngx_aiocb_t aiocb; + ngx_event_t event; +}; + +#endif + + typedef struct { in_addr_t mask; in_addr_t addr; diff --git a/src/event/ngx_event_pipe.c b/src/event/ngx_event_pipe.c index bc9fb26ab..d01b20446 100644 --- a/src/event/ngx_event_pipe.c +++ b/src/event/ngx_event_pipe.c @@ -24,15 +24,22 @@ ngx_int_t ngx_event_pipe(ngx_event_pipe_t *p, ngx_int_t do_write) { u_int flags; + ngx_int_t rc; ngx_event_t *rev, *wev; for ( ;; ) { if (do_write) { p->log->action = "sending to client"; - if (ngx_event_pipe_write_to_downstream(p) == NGX_ABORT) { + rc = ngx_event_pipe_write_to_downstream(p); + + if (rc == NGX_ABORT) { return NGX_ABORT; } + + if (rc == NGX_BUSY) { + return NGX_OK; + } } p->read = 0; @@ -422,7 +429,7 @@ ngx_event_pipe_write_to_downstream(ngx_event_pipe_t *p) u_char *prev; size_t bsize; ngx_int_t rc; - ngx_uint_t flush, prev_last_shadow; + ngx_uint_t flush, flushed, prev_last_shadow; ngx_chain_t *out, **ll, *cl, file; ngx_connection_t *downstream; @@ -431,6 +438,8 @@ ngx_event_pipe_write_to_downstream(ngx_event_pipe_t *p) ngx_log_debug1(NGX_LOG_DEBUG_EVENT, p->log, 0, "pipe write downstream: %d", downstream->write->ready); + flushed = 0; + for ( ;; ) { if (p->downstream_error) { return ngx_event_pipe_drain_chains(p); @@ -610,8 +619,16 @@ ngx_event_pipe_write_to_downstream(ngx_event_pipe_t *p) ngx_log_debug2(NGX_LOG_DEBUG_EVENT, p->log, 0, "pipe write: out:%p, f:%d", out, flush); - if (out == NULL && !flush) { - break; + if (out == NULL) { + + if (!flush) { + break; + } + + /* a workaround for AIO */ + if (flushed++ > 10) { + return NGX_BUSY; + } } rc = p->output_filter(p->output_ctx, out); diff --git a/src/http/ngx_http_cache.h b/src/http/ngx_http_cache.h index 6ac6fd876..e04eba3a4 100644 --- a/src/http/ngx_http_cache.h +++ b/src/http/ngx_http_cache.h @@ -18,6 +18,7 @@ #define NGX_HTTP_CACHE_STALE 3 #define NGX_HTTP_CACHE_UPDATING 4 #define NGX_HTTP_CACHE_HIT 5 +#define NGX_HTTP_CACHE_SCARCE 6 #define NGX_HTTP_CACHE_KEY_LEN 16 diff --git a/src/http/ngx_http_copy_filter_module.c b/src/http/ngx_http_copy_filter_module.c index 688677f50..411e731f3 100644 --- a/src/http/ngx_http_copy_filter_module.c +++ b/src/http/ngx_http_copy_filter_module.c @@ -14,6 +14,12 @@ typedef struct { } ngx_http_copy_filter_conf_t; +#if (NGX_HAVE_FILE_AIO) +static void ngx_http_copy_aio_handler(ngx_output_chain_ctx_t *ctx, + ngx_file_t *file); +static void ngx_http_copy_aio_event_handler(ngx_event_t *ev); +#endif + static void *ngx_http_copy_filter_create_conf(ngx_conf_t *cf); static char *ngx_http_copy_filter_merge_conf(ngx_conf_t *cf, void *parent, void *child); @@ -73,10 +79,15 @@ ngx_http_copy_filter(ngx_http_request_t *r, ngx_chain_t *in) ngx_int_t rc; ngx_connection_t *c; ngx_output_chain_ctx_t *ctx; + ngx_http_core_loc_conf_t *clcf; ngx_http_copy_filter_conf_t *conf; c = r->connection; + if (r->aio) { + return NGX_AGAIN; + } + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, "copy filter: \"%V?%V\"", &r->uri, &r->args); @@ -104,6 +115,13 @@ ngx_http_copy_filter(ngx_http_request_t *r, ngx_chain_t *in) ctx->output_filter = (ngx_output_chain_filter_pt) ngx_http_next_filter; ctx->filter_ctx = r; +#if (NGX_HAVE_FILE_AIO) + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + if (clcf->aio) { + ctx->aio = ngx_http_copy_aio_handler; + } +#endif + r->request_output = 1; } @@ -123,6 +141,41 @@ ngx_http_copy_filter(ngx_http_request_t *r, ngx_chain_t *in) } +#if (NGX_HAVE_FILE_AIO) + +static void +ngx_http_copy_aio_handler(ngx_output_chain_ctx_t *ctx, ngx_file_t *file) +{ + ngx_http_request_t *r; + + r = ctx->filter_ctx; + + file->aio->data = r; + file->aio->handler = ngx_http_copy_aio_event_handler; + + r->main->blocked++; + r->aio = 1; +} + + +static void +ngx_http_copy_aio_event_handler(ngx_event_t *ev) +{ + ngx_event_aio_t *aio; + ngx_http_request_t *r; + + aio = ev->data; + r = aio->data; + + r->main->blocked--; + r->aio = 0; + + r->connection->write->handler(r->connection->write); +} + +#endif + + static void * ngx_http_copy_filter_create_conf(ngx_conf_t *cf) { diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c index 7b47fbb61..da91ee66a 100644 --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -383,6 +383,17 @@ static ngx_command_t ngx_http_core_commands[] = { offsetof(ngx_http_core_loc_conf_t, sendfile_max_chunk), NULL }, +#if (NGX_HAVE_FILE_AIO) + + { ngx_string("aio"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_core_loc_conf_t, aio), + NULL }, + +#endif + { ngx_string("directio"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ngx_http_core_directio, @@ -2916,6 +2927,9 @@ ngx_http_core_create_loc_conf(ngx_conf_t *cf) lcf->internal = NGX_CONF_UNSET; lcf->sendfile = NGX_CONF_UNSET; lcf->sendfile_max_chunk = NGX_CONF_UNSET_SIZE; +#if (NGX_HAVE_FILE_AIO) + lcf->aio = NGX_CONF_UNSET; +#endif lcf->directio = NGX_CONF_UNSET; lcf->tcp_nopush = NGX_CONF_UNSET; lcf->tcp_nodelay = NGX_CONF_UNSET; @@ -3113,6 +3127,9 @@ ngx_http_core_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->sendfile, prev->sendfile, 0); ngx_conf_merge_size_value(conf->sendfile_max_chunk, prev->sendfile_max_chunk, 0); +#if (NGX_HAVE_FILE_AIO) + ngx_conf_merge_value(conf->aio, prev->aio, 0); +#endif ngx_conf_merge_off_value(conf->directio, prev->directio, NGX_MAX_OFF_T_VALUE); ngx_conf_merge_value(conf->tcp_nopush, prev->tcp_nopush, 0); diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h index 71d5b953c..522509b41 100644 --- a/src/http/ngx_http_core_module.h +++ b/src/http/ngx_http_core_module.h @@ -347,6 +347,9 @@ struct ngx_http_core_loc_conf_s { /* client_body_in_singe_buffer */ ngx_flag_t internal; /* internal */ ngx_flag_t sendfile; /* sendfile */ +#if (NGX_HAVE_FILE_AIO) + ngx_flag_t aio; /* aio */ +#endif ngx_flag_t tcp_nopush; /* tcp_nopush */ ngx_flag_t tcp_nodelay; /* tcp_nodelay */ ngx_flag_t reset_timedout_connection; /* reset_timedout_connection */ diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c index 8972f47dc..a6271a8e9 100644 --- a/src/http/ngx_http_file_cache.c +++ b/src/http/ngx_http_file_cache.c @@ -10,6 +10,11 @@ #include <ngx_md5.h> +static ngx_int_t ngx_http_file_cache_read(ngx_http_request_t *r, + ngx_http_cache_t *c); +#if (NGX_HAVE_FILE_AIO) +static void ngx_http_cache_aio_event_handler(ngx_event_t *ev); +#endif static ngx_int_t ngx_http_file_cache_exists(ngx_http_file_cache_t *cache, ngx_http_cache_t *c); static ngx_http_file_cache_node_t * @@ -173,20 +178,22 @@ ngx_http_file_cache_create_key(ngx_http_request_t *r) ngx_int_t ngx_http_file_cache_open(ngx_http_request_t *r) { - u_char *p; - time_t now; - ssize_t n; - ngx_int_t rc, rv; - ngx_uint_t cold, test; - ngx_path_t *path; - ngx_http_cache_t *c; - ngx_pool_cleanup_t *cln; - ngx_open_file_info_t of; - ngx_http_file_cache_t *cache; - ngx_http_core_loc_conf_t *clcf; - ngx_http_file_cache_header_t *h; + u_char *p; + ngx_int_t rc, rv; + ngx_uint_t cold, test; + ngx_path_t *path; + ngx_http_cache_t *c; + ngx_pool_cleanup_t *cln; + ngx_open_file_info_t of; + ngx_http_file_cache_t *cache; + ngx_http_core_loc_conf_t *clcf; c = r->cache; + + if (c->buf) { + return ngx_http_file_cache_read(r, c); + } + cache = c->file_cache; cln = ngx_pool_cleanup_add(r->pool, 0); @@ -207,7 +214,7 @@ ngx_http_file_cache_open(ngx_http_request_t *r) cln->data = c; if (rc == NGX_AGAIN) { - return rc; + return NGX_HTTP_CACHE_SCARCE; } cold = cache->sh->cold; @@ -227,11 +234,11 @@ ngx_http_file_cache_open(ngx_http_request_t *r) if (c->min_uses > 1) { if (!cold) { - return NGX_AGAIN; + return NGX_HTTP_CACHE_SCARCE; } test = 1; - rv = NGX_AGAIN; + rv = NGX_HTTP_CACHE_SCARCE; } else { c->temp_file = 1; @@ -299,14 +306,58 @@ ngx_http_file_cache_open(ngx_http_request_t *r) c->file.fd = of.fd; c->file.log = r->connection->log; + c->uniq = of.uniq; + c->length = of.size; c->buf = ngx_create_temp_buf(r->pool, c->body_start); if (c->buf == NULL) { return NGX_ERROR; } + return ngx_http_file_cache_read(r, c); +} + + +static ngx_int_t +ngx_http_file_cache_read(ngx_http_request_t *r, ngx_http_cache_t *c) +{ + time_t now; + ssize_t n; + ngx_int_t rc; + ngx_http_file_cache_t *cache; + ngx_http_file_cache_header_t *h; + + c = r->cache; + +#if (NGX_HAVE_FILE_AIO) + { + ngx_http_core_loc_conf_t *clcf; + + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + + if (clcf->aio) { + n = ngx_file_aio_read(&c->file, c->buf->pos, c->body_start, 0, r->pool); + + if (n == NGX_AGAIN) { + c->file.aio->data = r; + c->file.aio->handler = ngx_http_cache_aio_event_handler; + + r->main->blocked++; + r->aio = 1; + + return NGX_AGAIN; + } + + } else { + n = ngx_read_file(&c->file, c->buf->pos, c->body_start, 0); + } + } +#else + n = ngx_read_file(&c->file, c->buf->pos, c->body_start, 0); +#endif + if (n == NGX_ERROR) { return n; } @@ -331,12 +382,13 @@ ngx_http_file_cache_open(ngx_http_request_t *r) c->last_modified = h->last_modified; c->date = h->date; c->valid_msec = h->valid_msec; - c->length = of.size; c->body_start = h->body_start; r->cached = 1; - if (cold) { + cache = c->file_cache; + + if (cache->sh->cold) { ngx_shmtx_lock(&cache->shpool->mutex); @@ -344,7 +396,7 @@ ngx_http_file_cache_open(ngx_http_request_t *r) c->node->uses = 1; c->node->body_start = c->body_start; c->node->exists = 1; - c->node->uniq = of.uniq; + c->node->uniq = c->uniq; cache->sh->size += (c->length + cache->bsize - 1) / cache->bsize; } @@ -379,6 +431,27 @@ ngx_http_file_cache_open(ngx_http_request_t *r) } +#if (NGX_HAVE_FILE_AIO) + + +static void +ngx_http_cache_aio_event_handler(ngx_event_t *ev) +{ + ngx_event_aio_t *aio; + ngx_http_request_t *r; + + aio = ev->data; + r = aio->data; + + r->main->blocked--; + r->aio = 0; + + r->connection->write->handler(r->connection->write); +} + +#endif + + static ngx_int_t ngx_http_file_cache_exists(ngx_http_file_cache_t *cache, ngx_http_cache_t *c) { diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c index ea8843bfc..5f46e61cc 100644 --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c @@ -1868,6 +1868,10 @@ ngx_http_finalize_request(ngx_http_request_t *r, ngx_int_t rc) return; } + if (r->main->blocked) { + r->write_event_handler = ngx_http_request_finalizer; + } + ngx_http_terminate_request(r, rc); return; } @@ -1969,7 +1973,7 @@ ngx_http_finalize_request(ngx_http_request_t *r, ngx_int_t rc) return; } - if (r->buffered || c->buffered || r->postponed) { + if (r->buffered || c->buffered || r->postponed || r->blocked) { if (ngx_http_set_write_handler(r) != NGX_OK) { ngx_http_terminate_request(r, 0); @@ -2022,7 +2026,7 @@ ngx_http_terminate_request(ngx_http_request_t *r, ngx_int_t rc) mr = r->main; ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "http terminate request count: %d", mr->count); + "http terminate request count:%d", mr->count); cln = mr->cleanup; mr->cleanup = NULL; @@ -2035,10 +2039,16 @@ ngx_http_terminate_request(ngx_http_request_t *r, ngx_int_t rc) cln = cln->next; } - ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "http terminate cleanup count: %d", mr->count); + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http terminate cleanup count:%d blk:%d", + mr->count, mr->blocked); if (mr->write_event_handler) { + + if (mr->blocked) { + return; + } + mr->posted_requests = NULL; mr->write_event_handler = ngx_http_terminate_handler; (void) ngx_http_post_request(mr); @@ -2053,7 +2063,7 @@ static void ngx_http_terminate_handler(ngx_http_request_t *r) { ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "http terminate handler count: %d", r->count); + "http terminate handler count:%d", r->count); r->count = 1; @@ -2161,7 +2171,7 @@ ngx_http_writer(ngx_http_request_t *r) } } else { - if (wev->delayed) { + if (wev->delayed || r->aio) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, wev->log, 0, "http writer delayed"); @@ -2830,8 +2840,8 @@ ngx_http_close_request(ngx_http_request_t *r, ngx_int_t rc) r = r->main; c = r->connection; - ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, - "http request count: %d", r->count); + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, + "http request count:%d blk:%d", r->count, r->blocked); if (r->count == 0) { ngx_log_error(NGX_LOG_ALERT, c->log, 0, "http request count is zero"); @@ -2839,7 +2849,7 @@ ngx_http_close_request(ngx_http_request_t *r, ngx_int_t rc) r->count--; - if (r->count) { + if (r->count || r->blocked) { return; } diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h index 7b984fb27..52ac90c23 100644 --- a/src/http/ngx_http_request.h +++ b/src/http/ngx_http_request.h @@ -416,6 +416,12 @@ struct ngx_http_request_s { ngx_http_cleanup_t *cleanup; + unsigned subrequests:8; + unsigned count:8; + unsigned blocked:8; + + unsigned aio:1; + unsigned http_state:4; /* URI with "/." and on Win32 with "//" */ @@ -501,9 +507,6 @@ struct ngx_http_request_s { unsigned stat_writing:1; #endif - unsigned subrequests:8; - unsigned count:8; - /* used to parse HTTP headers */ ngx_uint_t state; diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index f552d2e76..a864fb16d 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c @@ -18,6 +18,7 @@ static ngx_int_t ngx_http_upstream_cache_status(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); #endif +static void ngx_http_upstream_init_request(ngx_http_request_t *r); static void ngx_http_upstream_resolve_handler(ngx_resolver_ctx_t *ctx); static void ngx_http_upstream_rd_check_broken_connection(ngx_http_request_t *r); static void ngx_http_upstream_wr_check_broken_connection(ngx_http_request_t *r); @@ -386,15 +387,7 @@ ngx_http_upstream_create(ngx_http_request_t *r) void ngx_http_upstream_init(ngx_http_request_t *r) { - ngx_str_t *host; - ngx_uint_t i; - ngx_connection_t *c; - ngx_resolver_ctx_t *ctx, temp; - ngx_http_cleanup_t *cln; - ngx_http_upstream_t *u; - ngx_http_core_loc_conf_t *clcf; - ngx_http_upstream_srv_conf_t *uscf, **uscfp; - ngx_http_upstream_main_conf_t *umcf; + ngx_connection_t *c; c = r->connection; @@ -405,15 +398,6 @@ ngx_http_upstream_init(ngx_http_request_t *r) ngx_del_timer(c->read); } - u = r->upstream; - - u->store = (u->conf->store || u->conf->store_lengths); - - if (!u->store && !r->post_action && !u->conf->ignore_client_abort) { - r->read_event_handler = ngx_http_upstream_rd_check_broken_connection; - r->write_event_handler = ngx_http_upstream_wr_check_broken_connection; - } - if (ngx_event_flags & NGX_USE_CLEAR_EVENT) { if (!c->write->active) { @@ -426,10 +410,28 @@ ngx_http_upstream_init(ngx_http_request_t *r) } } - if (r->request_body) { - u->request_bufs = r->request_body->bufs; + ngx_http_upstream_init_request(r); +} + + +static void +ngx_http_upstream_init_request(ngx_http_request_t *r) +{ + ngx_str_t *host; + ngx_uint_t i; + ngx_resolver_ctx_t *ctx, temp; + ngx_http_cleanup_t *cln; + ngx_http_upstream_t *u; + ngx_http_core_loc_conf_t *clcf; + ngx_http_upstream_srv_conf_t *uscf, **uscfp; + ngx_http_upstream_main_conf_t *umcf; + + if (r->aio) { + return; } + u = r->upstream; + #if (NGX_HTTP_CACHE) if (u->conf->cache) { @@ -437,6 +439,11 @@ ngx_http_upstream_init(ngx_http_request_t *r) rc = ngx_http_upstream_cache(r, u); + if (rc == NGX_AGAIN) { + r->write_event_handler = ngx_http_upstream_init_request; + return; + } + if (rc == NGX_DONE) { return; } @@ -449,6 +456,17 @@ ngx_http_upstream_init(ngx_http_request_t *r) #endif + u->store = (u->conf->store || u->conf->store_lengths); + + if (!u->store && !r->post_action && !u->conf->ignore_client_abort) { + r->read_event_handler = ngx_http_upstream_rd_check_broken_connection; + r->write_event_handler = ngx_http_upstream_wr_check_broken_connection; + } + + if (r->request_body) { + u->request_bufs = r->request_body->bufs; + } + if (u->create_request(r) != NGX_OK) { ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); return; @@ -543,7 +561,7 @@ ngx_http_upstream_init(ngx_http_request_t *r) } if (ctx == NGX_NO_RESOLVER) { - ngx_log_error(NGX_LOG_ERR, c->log, 0, + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "no resolver defined to resolve %V", host); ngx_http_finalize_request(r, NGX_HTTP_BAD_GATEWAY); @@ -657,10 +675,6 @@ ngx_http_upstream_cache(ngx_http_request_t *r, ngx_http_upstream_t *u) break; - case NGX_ERROR: - - return NGX_ERROR; - case NGX_HTTP_CACHE_STALE: c->valid_sec = 0; @@ -681,12 +695,20 @@ ngx_http_upstream_cache(ngx_http_request_t *r, ngx_http_upstream_t *u) break; - case NGX_AGAIN: + case NGX_HTTP_CACHE_SCARCE: u->cacheable = 0; break; + case NGX_AGAIN: + + return NGX_AGAIN; + + case NGX_ERROR: + + return NGX_ERROR; + default: /* cached NGX_HTTP_BAD_GATEWAY, NGX_HTTP_GATEWAY_TIME_OUT, etc. */ diff --git a/src/os/unix/ngx_file_aio_read.c b/src/os/unix/ngx_file_aio_read.c new file mode 100644 index 000000000..bf089ec5d --- /dev/null +++ b/src/os/unix/ngx_file_aio_read.c @@ -0,0 +1,210 @@ + +/* + * Copyright (C) Igor Sysoev + */ + + +#include <ngx_config.h> +#include <ngx_core.h> +#include <ngx_event.h> + + +/* + * FreeBSD file AIO features and quirks: + * + * if an asked data are already in VM cache, then aio_error() returns 0, + * and the data are already copied in buffer; + * + * aio_read() preread in VM cache as minimum 32K; + * + * aio_read/aio_error() may return EINPROGRESS for just written data; + * + * kqueue EVFILT_AIO filter is level triggered only: an event repeats + * until aio_return() will be called; + * + * aio_cancel() can not cancel file AIO: it returns AIO_NOTCANCELED always. + */ + + +extern int ngx_kqueue; + + +static ssize_t ngx_file_aio_result(ngx_file_t *file, ngx_event_aio_t *aio, + ngx_event_t *ev); +static void ngx_file_aio_event_handler(ngx_event_t *ev); + + +ssize_t +ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset, + ngx_pool_t *pool) +{ + int n; + ngx_event_t *ev; + ngx_event_aio_t *aio; + static ngx_uint_t enosys = 0; + + if (enosys) { + return ngx_read_file(file, buf, size, offset); + } + + aio = file->aio; + + if (aio == NULL) { + aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t)); + if (aio == NULL) { + return NGX_ERROR; + } + + aio->file = file; + aio->fd = file->fd; + aio->event.data = aio; + aio->event.ready = 1; + aio->event.log = file->log; + file->aio = aio; + } + + ev = &aio->event; + + if (!ev->ready) { + ngx_log_error(NGX_LOG_ALERT, file->log, 0, + "second aio post for \"%V\"", &file->name); + return NGX_AGAIN; + } + + ngx_log_debug4(NGX_LOG_DEBUG_CORE, file->log, 0, + "aio complete:%d @%O:%z %V", + ev->complete, offset, size, &file->name); + + if (ev->complete) { + ev->complete = 0; + ngx_set_errno(aio->err); + + if (aio->err == 0) { + return aio->nbytes; + } + + return NGX_ERROR; + } + + ngx_memzero(&aio->aiocb, sizeof(struct aiocb)); + + aio->aiocb.aio_fildes = file->fd; + aio->aiocb.aio_offset = offset; + aio->aiocb.aio_buf = buf; + aio->aiocb.aio_nbytes = size; +#if (NGX_HAVE_KQUEUE) + aio->aiocb.aio_sigevent.sigev_notify_kqueue = ngx_kqueue; + aio->aiocb.aio_sigevent.sigev_notify = SIGEV_KEVENT; + aio->aiocb.aio_sigevent.sigev_value.sigval_ptr = ev; +#endif + ev->handler = ngx_file_aio_event_handler; + + n = aio_read(&aio->aiocb); + + if (n == -1) { + n = ngx_errno; + + if (n == NGX_EAGAIN) { + return ngx_read_file(file, buf, size, offset); + } + + ngx_log_error(NGX_LOG_CRIT, file->log, n, + "aio_read(\"%V\") failed", &file->name); + + if (n == NGX_ENOSYS) { + enosys = 1; + return ngx_read_file(file, buf, size, offset); + } + + return NGX_ERROR; + } + + ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0, + "aio_read: fd:%d %d", file->fd, n); + + ev->active = 1; + ev->ready = 0; + ev->complete = 0; + + return ngx_file_aio_result(aio->file, aio, ev); +} + + +static ssize_t +ngx_file_aio_result(ngx_file_t *file, ngx_event_aio_t *aio, ngx_event_t *ev) +{ + int n; + ngx_err_t err; + + n = aio_error(&aio->aiocb); + + ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0, + "aio_error: fd:%d %d", file->fd, n); + + if (n == -1) { + err = ngx_errno; + aio->err = err; + + ngx_log_error(NGX_LOG_ALERT, file->log, err, + "aio_error(\"%V\") failed", &file->name); + return NGX_ERROR; + } + + if (n != 0) { + if (n == NGX_EINPROGRESS) { + if (ev->ready) { + ev->ready = 0; + ngx_log_error(NGX_LOG_ALERT, file->log, n, + "aio_read(\"%V\") still in progress", + &file->name); + } + + return NGX_AGAIN; + } + + aio->err = n; + ev->ready = 0; + + ngx_log_error(NGX_LOG_CRIT, file->log, n, + "aio_read(\"%V\") failed", &file->name); + return NGX_ERROR; + } + + n = aio_return(&aio->aiocb); + + if (n == -1) { + err = ngx_errno; + aio->err = err; + ev->ready = 0; + + ngx_log_error(NGX_LOG_ALERT, file->log, err, + "aio_return(\"%V\") failed", &file->name); + return NGX_ERROR; + } + + aio->err = 0; + aio->nbytes = n; + ev->ready = 1; + ev->active = 0; + + ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0, + "aio_return: fd:%d %d", file->fd, n); + + return n; +} + + +static void +ngx_file_aio_event_handler(ngx_event_t *ev) +{ + ngx_event_aio_t *aio; + + aio = ev->data; + + ngx_log_debug2(NGX_LOG_DEBUG_CORE, ev->log, 0, + "aio event handler fd:%d %V", aio->fd, &aio->file->name); + + if (ngx_file_aio_result(aio->file, aio, ev) != NGX_AGAIN) { + aio->handler(ev); + } +} diff --git a/src/os/unix/ngx_files.h b/src/os/unix/ngx_files.h index 5395acd03..993ce8f93 100644 --- a/src/os/unix/ngx_files.h +++ b/src/os/unix/ngx_files.h @@ -287,4 +287,12 @@ size_t ngx_fs_bsize(u_char *name); #define ngx_set_stderr_n "dup2(STDERR_FILENO)" +#if (NGX_HAVE_FILE_AIO) + +ssize_t ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, + off_t offset, ngx_pool_t *pool); + +#endif + + #endif /* _NGX_FILES_H_INCLUDED_ */ diff --git a/src/os/unix/ngx_freebsd_config.h b/src/os/unix/ngx_freebsd_config.h index 997581a6b..20cb66640 100644 --- a/src/os/unix/ngx_freebsd_config.h +++ b/src/os/unix/ngx_freebsd_config.h @@ -73,13 +73,14 @@ #endif -#if (NGX_HAVE_AIO) -#include <aio.h> +#if (NGX_HAVE_KQUEUE) +#include <sys/event.h> #endif -#if (NGX_HAVE_KQUEUE) -#include <sys/event.h> +#if (NGX_HAVE_FILE_AIO || NGX_HAVE_AIO) +#include <aio.h> +typedef struct aiocb ngx_aiocb_t; #endif diff --git a/src/os/unix/ngx_linux_aio_read.c b/src/os/unix/ngx_linux_aio_read.c new file mode 100644 index 000000000..3b4185c8b --- /dev/null +++ b/src/os/unix/ngx_linux_aio_read.c @@ -0,0 +1,131 @@ + +/* + * Copyright (C) Igor Sysoev + */ + + +#include <ngx_config.h> +#include <ngx_core.h> +#include <ngx_event.h> + + +extern int ngx_eventfd; +extern aio_context_t ngx_aio_ctx; + + +static void ngx_file_aio_event_handler(ngx_event_t *ev); + + +static long +io_submit(aio_context_t ctx, long n, struct iocb **paiocb) +{ + return syscall(SYS_io_submit, ctx, n, paiocb); +} + + +ssize_t +ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset, + ngx_pool_t *pool) +{ + long n; + struct iocb *piocb[1]; + ngx_event_t *ev; + ngx_event_aio_t *aio; + static ngx_uint_t enosys = 0; + + if (enosys) { + return ngx_read_file(file, buf, size, offset); + } + + aio = file->aio; + + if (aio == NULL) { + aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t)); + if (aio == NULL) { + return NGX_ERROR; + } + + aio->file = file; + aio->fd = file->fd; + aio->event.data = aio; + aio->event.ready = 1; + aio->event.log = file->log; + file->aio = aio; + } + + ev = &aio->event; + + if (!ev->ready) { + ngx_log_error(NGX_LOG_ALERT, file->log, 0, + "second aio post for \"%V\"", &file->name); + return NGX_AGAIN; + } + + ngx_log_debug4(NGX_LOG_DEBUG_CORE, file->log, 0, + "aio complete:%d @%O:%z %V", + ev->complete, offset, size, &file->name); + + if (ev->complete) { + ev->active = 0; + ev->complete = 0; + + if (aio->res >= 0) { + ngx_set_errno(0); + return aio->res; + } + + ngx_set_errno(-aio->res); + return NGX_ERROR; + } + + ngx_memzero(&aio->aiocb, sizeof(struct iocb)); + + aio->aiocb.aio_data = (uint64_t) (uintptr_t) ev; + aio->aiocb.aio_lio_opcode = IOCB_CMD_PREAD; + aio->aiocb.aio_fildes = file->fd; + aio->aiocb.aio_buf = (uint64_t) (uintptr_t) buf; + aio->aiocb.aio_nbytes = size; + aio->aiocb.aio_offset = offset; + aio->aiocb.aio_flags = IOCB_FLAG_RESFD; + aio->aiocb.aio_resfd = ngx_eventfd; + + ev->handler = ngx_file_aio_event_handler; + + piocb[0] = &aio->aiocb; + + n = io_submit(ngx_aio_ctx, 1, piocb); + + if (n == 1) { + return NGX_AGAIN; + } + + n = -n; + + if (n == NGX_EAGAIN) { + return ngx_read_file(file, buf, size, offset); + } + + ngx_log_error(NGX_LOG_CRIT, file->log, n, + "io_submit(\"%V\") failed", &file->name); + + if (n == NGX_ENOSYS) { + enosys = 1; + return ngx_read_file(file, buf, size, offset); + } + + return NGX_ERROR; +} + + +static void +ngx_file_aio_event_handler(ngx_event_t *ev) +{ + ngx_event_aio_t *aio; + + aio = ev->data; + + ngx_log_debug2(NGX_LOG_DEBUG_CORE, ev->log, 0, + "aio event handler fd:%d %V", aio->fd, &aio->file->name); + + aio->handler(ev); +} diff --git a/src/os/unix/ngx_linux_config.h b/src/os/unix/ngx_linux_config.h index fb978c46e..cf45e54a1 100644 --- a/src/os/unix/ngx_linux_config.h +++ b/src/os/unix/ngx_linux_config.h @@ -81,6 +81,13 @@ extern ssize_t sendfile(int s, int fd, int32_t *offset, size_t size); #endif +#if (NGX_HAVE_FILE_AIO) +#include <sys/syscall.h> +#include <linux/aio_abi.h> +typedef struct iocb ngx_aiocb_t; +#endif + + #define NGX_LISTEN_BACKLOG 511 diff --git a/src/os/unix/ngx_posix_config.h b/src/os/unix/ngx_posix_config.h index 3644151ee..86a80d701 100644 --- a/src/os/unix/ngx_posix_config.h +++ b/src/os/unix/ngx_posix_config.h @@ -112,6 +112,12 @@ #endif +#if (NGX_HAVE_FILE_AIO) +#include <aio.h> +typedef struct aiocb ngx_aiocb_t; +#endif + + #define NGX_LISTEN_BACKLOG 511 diff --git a/src/os/unix/ngx_solaris_config.h b/src/os/unix/ngx_solaris_config.h index 663f26570..ae8212d8a 100644 --- a/src/os/unix/ngx_solaris_config.h +++ b/src/os/unix/ngx_solaris_config.h @@ -62,16 +62,6 @@ #endif -#if (NGX_HAVE_SENDFILE) -#include <sys/sendfile.h> -#endif - - -#if (NGX_HAVE_AIO) -#include <aio.h> -#endif - - #if (NGX_HAVE_DEVPOLL) #include <sys/ioctl.h> #include <sys/devpoll.h> @@ -83,6 +73,11 @@ #endif +#if (NGX_HAVE_SENDFILE) +#include <sys/sendfile.h> +#endif + + #define NGX_LISTEN_BACKLOG 511 |