--- libeio/eio.c 2008/06/21 00:13:13 1.21 +++ libeio/eio.c 2008/10/12 22:30:33 1.26 @@ -122,6 +122,17 @@ /*****************************************************************************/ +#if __GNUC__ >= 3 +# define expect(expr,value) __builtin_expect ((expr),(value)) +#else +# define expect(expr,value) (expr) +#endif + +#define expect_false(expr) expect ((expr) != 0, 0) +#define expect_true(expr) expect ((expr) != 0, 1) + +/*****************************************************************************/ + #define ETP_PRI_MIN EIO_PRI_MIN #define ETP_PRI_MAX EIO_PRI_MAX @@ -179,6 +190,15 @@ static mutex_t reqlock = X_MUTEX_INIT; static cond_t reqwait = X_COND_INIT; +#if !HAVE_PREADWRITE +/* + * make our pread/pwrite emulation safe against themselves, but not against + * normal read/write by using a mutex. slows down execution a lot, + * but that's your problem, not mine. + */ +static mutex_t preadwritelock = X_MUTEX_INIT; +#endif + typedef struct etp_worker { /* locked by wrklock */ @@ -404,11 +424,11 @@ static void etp_maybe_start_thread (void) { - if (etp_nthreads () >= wanted) + if (expect_true (etp_nthreads () >= wanted)) return; /* todo: maybe use idle here, but might be less exact */ - if (0 <= (int)etp_nthreads () + (int)etp_npending () - (int)etp_nreqs ()) + if (expect_true (0 <= (int)etp_nthreads () + (int)etp_npending () - (int)etp_nreqs ())) return; etp_start_thread (); @@ -471,7 +491,7 @@ --nreqs; X_UNLOCK (reqlock); - if (req->type == EIO_GROUP && req->size) + if (expect_false (req->type == EIO_GROUP && req->size)) { req->int1 = 1; /* mark request as delayed */ continue; @@ -479,11 +499,11 @@ else { int res = ETP_FINISH (req); - if (res) + if (expect_false (res)) return res; } - if (maxreqs && !--maxreqs) + if (expect_false (maxreqs && !--maxreqs)) break; if (maxtime) @@ -512,17 +532,36 @@ { req->pri -= ETP_PRI_MIN; - if (req->pri < ETP_PRI_MIN - ETP_PRI_MIN) req->pri = ETP_PRI_MIN - ETP_PRI_MIN; - if (req->pri > ETP_PRI_MAX - ETP_PRI_MIN) req->pri = ETP_PRI_MAX - ETP_PRI_MIN; + if (expect_false (req->pri < ETP_PRI_MIN - ETP_PRI_MIN)) req->pri = ETP_PRI_MIN - ETP_PRI_MIN; + if (expect_false (req->pri > ETP_PRI_MAX - ETP_PRI_MIN)) req->pri = ETP_PRI_MAX - ETP_PRI_MIN; - X_LOCK (reqlock); - ++nreqs; - ++nready; - reqq_push (&req_queue, req); - X_COND_SIGNAL (reqwait); - X_UNLOCK (reqlock); + if (expect_false (req->type == EIO_GROUP)) + { + /* I hope this is worth it :/ */ + X_LOCK (reqlock); + ++nreqs; + X_UNLOCK (reqlock); + + X_LOCK (reslock); + + ++npending; + + if (!reqq_push (&res_queue, req) && want_poll_cb) + want_poll_cb (); + + X_UNLOCK (reslock); + } + else + { + X_LOCK (reqlock); + ++nreqs; + ++nready; + reqq_push (&req_queue, req); + X_COND_SIGNAL (reqwait); + X_UNLOCK (reqlock); - etp_maybe_start_thread (); + etp_maybe_start_thread (); + } } static void etp_set_max_poll_time (double nseconds) @@ -567,12 +606,12 @@ { while (grp->size < grp->int2 && !EIO_CANCELLED (grp)) { - int old_len = grp->size; + grp->flags &= ~EIO_FLAG_GROUPADD; EIO_FEED (grp); /* stop if no progress has been made */ - if (old_len == grp->size) + if (!(grp->flags & EIO_FLAG_GROUPADD)) { grp->feed = 0; break; @@ -702,13 +741,6 @@ # define pread eio__pread # define pwrite eio__pwrite -/* - * make our pread/pwrite safe against themselves, but not against - * normal read/write by using a mutex. slows down execution a lot, - * but that's your problem, not mine. - */ -static mutex_t preadwritelock = X_MUTEX_INIT; - static ssize_t eio__pread (int fd, void *buf, size_t count, off_t offset) { @@ -950,6 +982,35 @@ req->result = res; } +#if !(_POSIX_MAPPED_FILES && _POSIX_SYNCHRONIZED_IO) +# define msync(a,b,c) ENOSYS +#endif + +int +eio__mtouch (void *mem, size_t len, int flags) +{ + intptr_t addr = (intptr_t)mem; + intptr_t end = addr + len; +#ifdef PAGESIZE + const intptr_t page = PAGESIZE; +#else + static intptr_t page; + + if (!page) + page = sysconf (_SC_PAGESIZE); +#endif + + addr &= ~(page - 1); /* assume page size is always a power of two */ + + if (addr < end) + if (flags) /* modify */ + do { *((volatile sig_atomic_t *)addr) |= 0; } while ((addr += page) < len); + else + do { *((volatile sig_atomic_t *)addr) ; } while ((addr += page) < len); + + return 0; +} + /*****************************************************************************/ #define ALLOC(len) \ @@ -1124,6 +1185,8 @@ case EIO_SYNC: req->result = 0; sync (); break; case EIO_FSYNC: req->result = fsync (req->int1); break; case EIO_FDATASYNC: req->result = fdatasync (req->int1); break; + case EIO_MSYNC: req->result = msync (req->ptr2, req->size, req->int1); break; + case EIO_MTOUCH: req->result = eio__mtouch (req->ptr2, req->size, req->int1); break; case EIO_READDIR: eio__scandir (req, self); break; @@ -1165,14 +1228,17 @@ ? futimes (req->int1, times) : utimes (req->ptr1, times); } + break; case EIO_GROUP: + abort (); /* handled in eio_request */ + case EIO_NOP: req->result = 0; break; case EIO_CUSTOM: - req->feed (req); + ((void (*)(eio_req *))req->feed) (req); break; default: @@ -1205,6 +1271,16 @@ REQ (EIO_FSYNC); req->int1 = fd; SEND; } +eio_req *eio_msync (void *addr, size_t length, int flags, int pri, eio_cb cb, void *data) +{ + REQ (EIO_MSYNC); req->ptr2 = addr; req->size = length; req->int1 = flags; SEND; +} + +eio_req *eio_mtouch (void *addr, size_t length, int flags, int pri, eio_cb cb, void *data) +{ + REQ (EIO_MTOUCH); req->ptr2 = addr; req->size = length; req->int1 = flags; SEND; +} + eio_req *eio_fdatasync (int fd, int pri, eio_cb cb, void *data) { REQ (EIO_FDATASYNC); req->int1 = fd; SEND; @@ -1369,7 +1445,7 @@ eio_req *eio_custom (eio_cb execute, int pri, eio_cb cb, void *data) { - REQ (EIO_CUSTOM); req->feed = execute; SEND; + REQ (EIO_CUSTOM); req->feed = (void (*)(eio_req *))execute; SEND; } #endif @@ -1407,6 +1483,8 @@ { assert (("cannot add requests to IO::AIO::GRP after the group finished", grp->int1 != 2)); + grp->flags |= EIO_FLAG_GROUPADD; + ++grp->size; req->grp = grp;