ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/libev/ev_linuxaio.c
(Generate patch)

Comparing libev/ev_linuxaio.c (file contents):
Revision 1.34 by root, Wed Jun 26 00:01:46 2019 UTC vs.
Revision 1.38 by root, Mon Jul 1 19:56:30 2019 UTC

… …
100 * not only is this totally undocumented, not even the source code 100 * not only is this totally undocumented, not even the source code
101 * can tell you what the future semantics of compat_features and 101 * can tell you what the future semantics of compat_features and
102 * incompat_features are, or what header_length actually is for. 102 * incompat_features are, or what header_length actually is for.
103 */ 103 */
104#define AIO_RING_MAGIC 0xa10a10a1 104#define AIO_RING_MAGIC 0xa10a10a1
105#define AIO_RING_INCOMPAT_FEATURES 0 105#define EV_AIO_RING_INCOMPAT_FEATURES 0
106struct aio_ring 106struct aio_ring
107{ 107{
108 unsigned id; /* kernel internal index number */ 108 unsigned id; /* kernel internal index number */
109 unsigned nr; /* number of io_events */ 109 unsigned nr; /* number of io_events */
110 unsigned head; /* Written to by userland or by kernel. */ 110 unsigned head; /* Written to by userland or by kernel. */
… …
299 299
300 if (iocb->io.aio_buf) 300 if (iocb->io.aio_buf)
301 { 301 {
302 evsys_io_cancel (linuxaio_ctx, &iocb->io, (struct io_event *)0); 302 evsys_io_cancel (linuxaio_ctx, &iocb->io, (struct io_event *)0);
303 /* on relevant kernels, io_cancel fails with EINPROGRES if everything is fine */ 303 /* on relevant kernels, io_cancel fails with EINPROGRES if everything is fine */
304 assert (("libev: linuxaio unexpected io_cancel failed", errno != EINPROGRESS)); 304 assert (("libev: linuxaio unexpected io_cancel failed", errno == EINPROGRESS || errno == EINTR));
305 } 305 }
306 306
307 if (nev) 307 if (nev)
308 { 308 {
309 iocb->io.aio_buf = 309 iocb->io.aio_buf =
… …
322linuxaio_epoll_cb (EV_P_ struct ev_io *w, int revents) 322linuxaio_epoll_cb (EV_P_ struct ev_io *w, int revents)
323{ 323{
324 epoll_poll (EV_A_ 0); 324 epoll_poll (EV_A_ 0);
325} 325}
326 326
327static void 327inline_speed
328void
328linuxaio_fd_rearm (EV_P_ int fd) 329linuxaio_fd_rearm (EV_P_ int fd)
329{ 330{
330 anfds [fd].events = 0; 331 anfds [fd].events = 0;
331 linuxaio_iocbps [fd]->io.aio_buf = 0; 332 linuxaio_iocbps [fd]->io.aio_buf = 0;
332 fd_change (EV_A_ fd, EV_ANFD_REIFY); 333 fd_change (EV_A_ fd, EV_ANFD_REIFY);
… …
348 fd, 349 fd,
349 (res & (POLLOUT | POLLERR | POLLHUP) ? EV_WRITE : 0) 350 (res & (POLLOUT | POLLERR | POLLHUP) ? EV_WRITE : 0)
350 | (res & (POLLIN | POLLERR | POLLHUP) ? EV_READ : 0) 351 | (res & (POLLIN | POLLERR | POLLHUP) ? EV_READ : 0)
351 ); 352 );
352 353
353 /* linux aio is oneshot: rearm fd. TODO: this does more work than needed */ 354 /* linux aio is oneshot: rearm fd. TODO: this does more work than strictly needed */
354 linuxaio_fd_rearm (EV_A_ fd); 355 linuxaio_fd_rearm (EV_A_ fd);
355 356
356 --nr; 357 --nr;
357 ++ev; 358 ++ev;
358 } 359 }
… …
371 unsigned tail = *(volatile unsigned *)&ring->tail; 372 unsigned tail = *(volatile unsigned *)&ring->tail;
372 373
373 if (head == tail) 374 if (head == tail)
374 return 0; 375 return 0;
375 376
376 /* bail out if the ring buffer doesn't match the expected layout */
377 if (expect_false (ring->magic != AIO_RING_MAGIC)
378 || ring->incompat_features != AIO_RING_INCOMPAT_FEATURES
379 || ring->header_length != sizeof (struct aio_ring)) /* TODO: or use it to find io_event[0]? */
380 return 0;
381
382 /* make sure the events up to tail are visible */ 377 /* make sure the events up to tail are visible */
383 ECB_MEMORY_FENCE_ACQUIRE; 378 ECB_MEMORY_FENCE_ACQUIRE;
384 379
385 /* parse all available events, but only once, to avoid starvation */ 380 /* parse all available events, but only once, to avoid starvation */
386 if (tail > head) /* normal case around */ 381 if (tail > head) /* normal case around */
… …
396 *(volatile unsigned *)&ring->head = tail; 391 *(volatile unsigned *)&ring->head = tail;
397 392
398 return 1; 393 return 1;
399} 394}
400 395
396inline_size
397int
398linuxaio_ringbuf_valid (EV_P)
399{
400 struct aio_ring *ring = (struct aio_ring *)linuxaio_ctx;
401
402 return expect_true (ring->magic == AIO_RING_MAGIC)
403 && ring->incompat_features == EV_AIO_RING_INCOMPAT_FEATURES
404 && ring->header_length == sizeof (struct aio_ring); /* TODO: or use it to find io_event[0]? */
405}
406
401/* read at least one event from kernel, or timeout */ 407/* read at least one event from kernel, or timeout */
402inline_size 408inline_size
403void 409void
404linuxaio_get_events (EV_P_ ev_tstamp timeout) 410linuxaio_get_events (EV_P_ ev_tstamp timeout)
405{ 411{
406 struct timespec ts; 412 struct timespec ts;
407 struct io_event ioev[1]; 413 struct io_event ioev[8]; /* 256 octet stack space */
408 int res; 414 int want = 1; /* how many events to request */
415 int ringbuf_valid = linuxaio_ringbuf_valid (EV_A);
409 416
417 if (expect_true (ringbuf_valid))
418 {
419 /* if the ring buffer has any events, we don't wait or call the kernel at all */
410 if (linuxaio_get_events_from_ring (EV_A)) 420 if (linuxaio_get_events_from_ring (EV_A))
411 return; 421 return;
412 422
413 /* no events, so wait for at least one, then poll ring buffer again */ 423 /* if the ring buffer is empty, and we don't have a timeout, then don't call the kernel */
414 /* this degrades to one event per loop iteration */ 424 if (!timeout)
415 /* if the ring buffer changes layout, but so be it */ 425 return;
426 }
427 else
428 /* no ringbuffer, request slightly larger batch */
429 want = sizeof (ioev) / sizeof (ioev [0]);
416 430
431 /* no events, so wait for some
432 * for fairness reasons, we do this in a loop, to fetch all events
433 */
434 for (;;)
435 {
436 int res;
437
417 EV_RELEASE_CB; 438 EV_RELEASE_CB;
418 439
419 ts.tv_sec = (long)timeout; 440 ts.tv_sec = (long)timeout;
420 ts.tv_nsec = (long)((timeout - ts.tv_sec) * 1e9); 441 ts.tv_nsec = (long)((timeout - ts.tv_sec) * 1e9);
421 442
422 res = evsys_io_getevents (linuxaio_ctx, 1, sizeof (ioev) / sizeof (ioev [0]), ioev, &ts); 443 res = evsys_io_getevents (linuxaio_ctx, 1, want, ioev, &ts);
423 444
424 EV_ACQUIRE_CB; 445 EV_ACQUIRE_CB;
425 446
426 if (res < 0) 447 if (res < 0)
427 if (errno == EINTR) 448 if (errno == EINTR)
428 /* ignored */; 449 /* ignored, retry */;
429 else 450 else
430 ev_syserr ("(libev) linuxaio io_getevents"); 451 ev_syserr ("(libev) linuxaio io_getevents");
431 else if (res) 452 else if (res)
432 { 453 {
433 /* at least one event available, handle it and any remaining ones in the ring buffer */ 454 /* at least one event available, handle them */
434 linuxaio_parse_events (EV_A_ ioev, res); 455 linuxaio_parse_events (EV_A_ ioev, res);
456
457 if (expect_true (ringbuf_valid))
458 {
459 /* if we have a ring buffer, handle any remaining events in it */
435 linuxaio_get_events_from_ring (EV_A); 460 linuxaio_get_events_from_ring (EV_A);
461
462 /* at this point, we should have handled all outstanding events */
463 break;
464 }
465 else if (res < want)
466 /* otherwise, if there were fewere events than we wanted, we assume there are no more */
467 break;
468 }
469 else
470 break; /* no events from the kernel, we are done */
471
472 timeout = 0; /* only wait in the first iteration */
436 } 473 }
437} 474}
438 475
439inline_size 476inline_size
440int 477int
… …
516 assert (("libev: event loop rejected bad fd", errno != EBADF)); 553 assert (("libev: event loop rejected bad fd", errno != EBADF));
517 fd_kill (EV_A_ linuxaio_submits [submitted]->aio_fildes); 554 fd_kill (EV_A_ linuxaio_submits [submitted]->aio_fildes);
518 555
519 res = 1; /* skip this iocb */ 556 res = 1; /* skip this iocb */
520 } 557 }
558 else if (errno == EINTR)
559 /* silently ignored */;
521 else 560 else
522 ev_syserr ("(libev) linuxaio io_submit"); 561 ev_syserr ("(libev) linuxaio io_submit");
523 562
524 submitted += res; 563 submitted += res;
525 } 564 }

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines