ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/libeio/eio.c
(Generate patch)

Comparing libeio/eio.c (file contents):
Revision 1.64 by root, Thu May 26 04:05:18 2011 UTC vs.
Revision 1.89 by root, Thu Jul 14 22:36:17 2011 UTC

35 * and other provisions required by the GPL. If you do not delete the 35 * and other provisions required by the GPL. If you do not delete the
36 * provisions above, a recipient may use your version of this file under 36 * provisions above, a recipient may use your version of this file under
37 * either the BSD or the GPL. 37 * either the BSD or the GPL.
38 */ 38 */
39 39
40#ifndef _WIN32
41# include "config.h"
42#endif
43
40#include "eio.h" 44#include "eio.h"
45#include "ecb.h"
41 46
42#ifdef EIO_STACKSIZE 47#ifdef EIO_STACKSIZE
43# define XTHREAD_STACKSIZE EIO_STACKSIZE 48# define XTHREAD_STACKSIZE EIO_STACKSIZE
44#endif 49#endif
45#include "xthread.h" 50#include "xthread.h"
49#include <stdlib.h> 54#include <stdlib.h>
50#include <string.h> 55#include <string.h>
51#include <errno.h> 56#include <errno.h>
52#include <sys/types.h> 57#include <sys/types.h>
53#include <sys/stat.h> 58#include <sys/stat.h>
54#include <sys/statvfs.h>
55#include <limits.h> 59#include <limits.h>
56#include <fcntl.h> 60#include <fcntl.h>
57#include <assert.h> 61#include <assert.h>
58 62
63/* intptr_t comes from unistd.h, says POSIX/UNIX/tradition */
64/* intptr_t only comes from stdint.h, says idiot openbsd coder */
65#if HAVE_STDINT_H
66# include <stdint.h>
67#endif
68
69#ifndef ECANCELED
70# define ECANCELED EDOM
71#endif
72
73static void eio_destroy (eio_req *req);
74
59#ifndef EIO_FINISH 75#ifndef EIO_FINISH
60# define EIO_FINISH(req) ((req)->finish) && !EIO_CANCELLED (req) ? (req)->finish (req) : 0 76# define EIO_FINISH(req) ((req)->finish) && !EIO_CANCELLED (req) ? (req)->finish (req) : 0
61#endif 77#endif
62 78
63#ifndef EIO_DESTROY 79#ifndef EIO_DESTROY
66 82
67#ifndef EIO_FEED 83#ifndef EIO_FEED
68# define EIO_FEED(req) do { if ((req)->feed ) (req)->feed (req); } while (0) 84# define EIO_FEED(req) do { if ((req)->feed ) (req)->feed (req); } while (0)
69#endif 85#endif
70 86
87#ifndef EIO_FD_TO_WIN32_HANDLE
88# define EIO_FD_TO_WIN32_HANDLE(fd) _get_osfhandle (fd)
89#endif
90#ifndef EIO_WIN32_HANDLE_TO_FD
91# define EIO_WIN32_HANDLE_TO_FD(handle) _open_osfhandle (handle, 0)
92#endif
93
94#define EIO_ERRNO(errval,retval) ((errno = errval), retval)
95
96#define EIO_ENOSYS() EIO_ERRNO (ENOSYS, -1)
97
71#ifdef _WIN32 98#ifdef _WIN32
72 99
73 /*doh*/ 100 #define PAGESIZE 4096 /* GetSystemInfo? */
101
102 #ifdef EIO_STRUCT_STATI64
103 #define stat(path,buf) _stati64 (path,buf)
104 #define fstat(fd,buf) _fstati64 (path,buf)
105 #endif
106 #define lstat(path,buf) stat (path,buf)
107 #define fsync(fd) (FlushFileBuffers (EIO_FD_TO_WIN32_HANDLE (fd)) ? 0 : EIO_ERRNO (EBADF, -1))
108 #define mkdir(path,mode) _mkdir (path)
109 #define link(old,neu) (CreateHardLink (neu, old, 0) ? 0 : EIO_ERRNO (ENOENT, -1))
110
111 #define chown(path,uid,gid) EIO_ENOSYS ()
112 #define fchown(fd,uid,gid) EIO_ENOSYS ()
113 #define truncate(path,offs) EIO_ENOSYS () /* far-miss: SetEndOfFile */
114 #define ftruncate(fd,offs) EIO_ENOSYS () /* near-miss: SetEndOfFile */
115 #define mknod(path,mode,dev) EIO_ENOSYS ()
116 #define sync() EIO_ENOSYS ()
117
118 /* we could even stat and see if it exists */
119 static int
120 symlink (const char *old, const char *neu)
121 {
122 if (CreateSymbolicLink (neu, old, 1))
123 return 0;
124
125 if (CreateSymbolicLink (neu, old, 0))
126 return 0;
127
128 return EIO_ERRNO (ENOENT, -1);
129 }
130
74#else 131#else
75 132
76# include "config.h"
77# include <sys/time.h> 133 #include <sys/time.h>
78# include <sys/select.h> 134 #include <sys/select.h>
135 #include <sys/statvfs.h>
79# include <unistd.h> 136 #include <unistd.h>
80# include <utime.h> 137 #include <utime.h>
81# include <signal.h> 138 #include <signal.h>
82# include <dirent.h> 139 #include <dirent.h>
83 140
84#if _POSIX_MEMLOCK || _POSIX_MEMLOCK_RANGE || _POSIX_MAPPED_FILES 141 #if _POSIX_MEMLOCK || _POSIX_MEMLOCK_RANGE || _POSIX_MAPPED_FILES
85# include <sys/mman.h> 142 #include <sys/mman.h>
86#endif 143 #endif
87 144
88/* POSIX_SOURCE is useless on bsd's, and XOPEN_SOURCE is unreliable there, too */ 145 /* POSIX_SOURCE is useless on bsd's, and XOPEN_SOURCE is unreliable there, too */
89# if __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__ 146 #if __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__
90# define _DIRENT_HAVE_D_TYPE /* sigh */ 147 #define _DIRENT_HAVE_D_TYPE /* sigh */
91# define D_INO(de) (de)->d_fileno 148 #define D_INO(de) (de)->d_fileno
92# define D_NAMLEN(de) (de)->d_namlen 149 #define D_NAMLEN(de) (de)->d_namlen
93# elif __linux || defined d_ino || _XOPEN_SOURCE >= 600 150 #elif __linux || defined d_ino || _XOPEN_SOURCE >= 600
94# define D_INO(de) (de)->d_ino 151 #define D_INO(de) (de)->d_ino
95# endif 152 #endif
96 153
97#ifdef _D_EXACT_NAMLEN 154 #ifdef _D_EXACT_NAMLEN
98# undef D_NAMLEN 155 #undef D_NAMLEN
99# define D_NAMLEN(de) _D_EXACT_NAMLEN (de) 156 #define D_NAMLEN(de) _D_EXACT_NAMLEN (de)
100#endif 157 #endif
101 158
102# ifdef _DIRENT_HAVE_D_TYPE 159 #ifdef _DIRENT_HAVE_D_TYPE
103# define D_TYPE(de) (de)->d_type 160 #define D_TYPE(de) (de)->d_type
104# endif 161 #endif
105 162
106# ifndef EIO_STRUCT_DIRENT 163 #ifndef EIO_STRUCT_DIRENT
107# define EIO_STRUCT_DIRENT struct dirent 164 #define EIO_STRUCT_DIRENT struct dirent
108# endif 165 #endif
109 166
110#endif 167#endif
111 168
112#if HAVE_SENDFILE 169#if HAVE_SENDFILE
113# if __linux 170# if __linux
156 if (!eio_buf) \ 213 if (!eio_buf) \
157 return -1; 214 return -1;
158 215
159#define EIO_TICKS ((1000000 + 1023) >> 10) 216#define EIO_TICKS ((1000000 + 1023) >> 10)
160 217
161/*****************************************************************************/
162
163#if __GNUC__ >= 3
164# define expect(expr,value) __builtin_expect ((expr),(value))
165#else
166# define expect(expr,value) (expr)
167#endif
168
169#define expect_false(expr) expect ((expr) != 0, 0)
170#define expect_true(expr) expect ((expr) != 0, 1)
171
172/*****************************************************************************/
173
174#define ETP_PRI_MIN EIO_PRI_MIN 218#define ETP_PRI_MIN EIO_PRI_MIN
175#define ETP_PRI_MAX EIO_PRI_MAX 219#define ETP_PRI_MAX EIO_PRI_MAX
176 220
177struct etp_worker; 221struct etp_worker;
178 222
203/*****************************************************************************/ 247/*****************************************************************************/
204 248
205#define ETP_NUM_PRI (ETP_PRI_MAX - ETP_PRI_MIN + 1) 249#define ETP_NUM_PRI (ETP_PRI_MAX - ETP_PRI_MIN + 1)
206 250
207/* calculate time difference in ~1/EIO_TICKS of a second */ 251/* calculate time difference in ~1/EIO_TICKS of a second */
252ecb_inline int
208static int tvdiff (struct timeval *tv1, struct timeval *tv2) 253tvdiff (struct timeval *tv1, struct timeval *tv2)
209{ 254{
210 return (tv2->tv_sec - tv1->tv_sec ) * EIO_TICKS 255 return (tv2->tv_sec - tv1->tv_sec ) * EIO_TICKS
211 + ((tv2->tv_usec - tv1->tv_usec) >> 10); 256 + ((tv2->tv_usec - tv1->tv_usec) >> 10);
212} 257}
213 258
257#define ETP_WORKER_LOCK(wrk) X_LOCK (wrklock) 302#define ETP_WORKER_LOCK(wrk) X_LOCK (wrklock)
258#define ETP_WORKER_UNLOCK(wrk) X_UNLOCK (wrklock) 303#define ETP_WORKER_UNLOCK(wrk) X_UNLOCK (wrklock)
259 304
260/* worker threads management */ 305/* worker threads management */
261 306
307static void ecb_cold
262static void etp_worker_clear (etp_worker *wrk) 308etp_worker_clear (etp_worker *wrk)
263{ 309{
264 ETP_WORKER_CLEAR (wrk); 310 ETP_WORKER_CLEAR (wrk);
265} 311}
266 312
313static void ecb_cold
267static void etp_worker_free (etp_worker *wrk) 314etp_worker_free (etp_worker *wrk)
268{ 315{
269 wrk->next->prev = wrk->prev; 316 wrk->next->prev = wrk->prev;
270 wrk->prev->next = wrk->next; 317 wrk->prev->next = wrk->next;
271 318
272 free (wrk); 319 free (wrk);
273} 320}
274 321
275static unsigned int etp_nreqs (void) 322static unsigned int
323etp_nreqs (void)
276{ 324{
277 int retval; 325 int retval;
278 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 326 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
279 retval = nreqs; 327 retval = nreqs;
280 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock); 328 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
281 return retval; 329 return retval;
282} 330}
283 331
284static unsigned int etp_nready (void) 332static unsigned int
333etp_nready (void)
285{ 334{
286 unsigned int retval; 335 unsigned int retval;
287 336
288 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 337 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
289 retval = nready; 338 retval = nready;
290 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock); 339 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
291 340
292 return retval; 341 return retval;
293} 342}
294 343
295static unsigned int etp_npending (void) 344static unsigned int
345etp_npending (void)
296{ 346{
297 unsigned int retval; 347 unsigned int retval;
298 348
299 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 349 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
300 retval = npending; 350 retval = npending;
301 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock); 351 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
302 352
303 return retval; 353 return retval;
304} 354}
305 355
306static unsigned int etp_nthreads (void) 356static unsigned int
357etp_nthreads (void)
307{ 358{
308 unsigned int retval; 359 unsigned int retval;
309 360
310 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 361 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
311 retval = started; 362 retval = started;
325} etp_reqq; 376} etp_reqq;
326 377
327static etp_reqq req_queue; 378static etp_reqq req_queue;
328static etp_reqq res_queue; 379static etp_reqq res_queue;
329 380
381static int ecb_noinline
330static int reqq_push (etp_reqq *q, ETP_REQ *req) 382reqq_push (etp_reqq *q, ETP_REQ *req)
331{ 383{
332 int pri = req->pri; 384 int pri = req->pri;
333 req->next = 0; 385 req->next = 0;
334 386
335 if (q->qe[pri]) 387 if (q->qe[pri])
341 q->qe[pri] = q->qs[pri] = req; 393 q->qe[pri] = q->qs[pri] = req;
342 394
343 return q->size++; 395 return q->size++;
344} 396}
345 397
398static ETP_REQ * ecb_noinline
346static ETP_REQ *reqq_shift (etp_reqq *q) 399reqq_shift (etp_reqq *q)
347{ 400{
348 int pri; 401 int pri;
349 402
350 if (!q->size) 403 if (!q->size)
351 return 0; 404 return 0;
366 } 419 }
367 420
368 abort (); 421 abort ();
369} 422}
370 423
424static void ecb_cold
371static void etp_thread_init (void) 425etp_thread_init (void)
372{ 426{
427#if !HAVE_PREADWRITE
428 X_MUTEX_CREATE (preadwritelock);
429#endif
373 X_MUTEX_CREATE (wrklock); 430 X_MUTEX_CREATE (wrklock);
374 X_MUTEX_CREATE (reslock); 431 X_MUTEX_CREATE (reslock);
375 X_MUTEX_CREATE (reqlock); 432 X_MUTEX_CREATE (reqlock);
376 X_COND_CREATE (reqwait); 433 X_COND_CREATE (reqwait);
377} 434}
378 435
436static void ecb_cold
379static void etp_atfork_prepare (void) 437etp_atfork_prepare (void)
380{ 438{
381 X_LOCK (wrklock);
382 X_LOCK (reqlock);
383 X_LOCK (reslock);
384#if !HAVE_PREADWRITE
385 X_LOCK (preadwritelock);
386#endif
387} 439}
388 440
441static void ecb_cold
389static void etp_atfork_parent (void) 442etp_atfork_parent (void)
390{ 443{
391#if !HAVE_PREADWRITE
392 X_UNLOCK (preadwritelock);
393#endif
394 X_UNLOCK (reslock);
395 X_UNLOCK (reqlock);
396 X_UNLOCK (wrklock);
397} 444}
398 445
446static void ecb_cold
399static void etp_atfork_child (void) 447etp_atfork_child (void)
400{ 448{
401 ETP_REQ *prv; 449 ETP_REQ *prv;
402 450
403 while ((prv = reqq_shift (&req_queue))) 451 while ((prv = reqq_shift (&req_queue)))
404 ETP_DESTROY (prv); 452 ETP_DESTROY (prv);
424 npending = 0; 472 npending = 0;
425 473
426 etp_thread_init (); 474 etp_thread_init ();
427} 475}
428 476
429static void 477static void ecb_cold
430etp_once_init (void) 478etp_once_init (void)
431{ 479{
432 etp_thread_init (); 480 etp_thread_init ();
433 X_THREAD_ATFORK (etp_atfork_prepare, etp_atfork_parent, etp_atfork_child); 481 X_THREAD_ATFORK (etp_atfork_prepare, etp_atfork_parent, etp_atfork_child);
434} 482}
435 483
436static int 484static int ecb_cold
437etp_init (void (*want_poll)(void), void (*done_poll)(void)) 485etp_init (void (*want_poll)(void), void (*done_poll)(void))
438{ 486{
439 static pthread_once_t doinit = PTHREAD_ONCE_INIT; 487 static pthread_once_t doinit = PTHREAD_ONCE_INIT;
440 488
441 pthread_once (&doinit, etp_once_init); 489 pthread_once (&doinit, etp_once_init);
446 return 0; 494 return 0;
447} 495}
448 496
449X_THREAD_PROC (etp_proc); 497X_THREAD_PROC (etp_proc);
450 498
499static void ecb_cold
451static void etp_start_thread (void) 500etp_start_thread (void)
452{ 501{
453 etp_worker *wrk = calloc (1, sizeof (etp_worker)); 502 etp_worker *wrk = calloc (1, sizeof (etp_worker));
454 503
455 /*TODO*/ 504 /*TODO*/
456 assert (("unable to allocate worker thread data", wrk)); 505 assert (("unable to allocate worker thread data", wrk));
469 free (wrk); 518 free (wrk);
470 519
471 X_UNLOCK (wrklock); 520 X_UNLOCK (wrklock);
472} 521}
473 522
523static void
474static void etp_maybe_start_thread (void) 524etp_maybe_start_thread (void)
475{ 525{
476 if (expect_true (etp_nthreads () >= wanted)) 526 if (ecb_expect_true (etp_nthreads () >= wanted))
477 return; 527 return;
478 528
479 /* todo: maybe use idle here, but might be less exact */ 529 /* todo: maybe use idle here, but might be less exact */
480 if (expect_true (0 <= (int)etp_nthreads () + (int)etp_npending () - (int)etp_nreqs ())) 530 if (ecb_expect_true (0 <= (int)etp_nthreads () + (int)etp_npending () - (int)etp_nreqs ()))
481 return; 531 return;
482 532
483 etp_start_thread (); 533 etp_start_thread ();
484} 534}
485 535
536static void ecb_cold
486static void etp_end_thread (void) 537etp_end_thread (void)
487{ 538{
488 eio_req *req = calloc (1, sizeof (eio_req)); 539 eio_req *req = calloc (1, sizeof (eio_req));
489 540
490 req->type = -1; 541 req->type = -1;
491 req->pri = ETP_PRI_MAX - ETP_PRI_MIN; 542 req->pri = ETP_PRI_MAX - ETP_PRI_MIN;
498 X_LOCK (wrklock); 549 X_LOCK (wrklock);
499 --started; 550 --started;
500 X_UNLOCK (wrklock); 551 X_UNLOCK (wrklock);
501} 552}
502 553
503static int etp_poll (void) 554static int
555etp_poll (void)
504{ 556{
505 unsigned int maxreqs; 557 unsigned int maxreqs;
506 unsigned int maxtime; 558 unsigned int maxtime;
507 struct timeval tv_start, tv_now; 559 struct timeval tv_start, tv_now;
508 560
538 590
539 X_LOCK (reqlock); 591 X_LOCK (reqlock);
540 --nreqs; 592 --nreqs;
541 X_UNLOCK (reqlock); 593 X_UNLOCK (reqlock);
542 594
543 if (expect_false (req->type == EIO_GROUP && req->size)) 595 if (ecb_expect_false (req->type == EIO_GROUP && req->size))
544 { 596 {
545 req->int1 = 1; /* mark request as delayed */ 597 req->int1 = 1; /* mark request as delayed */
546 continue; 598 continue;
547 } 599 }
548 else 600 else
549 { 601 {
550 int res = ETP_FINISH (req); 602 int res = ETP_FINISH (req);
551 if (expect_false (res)) 603 if (ecb_expect_false (res))
552 return res; 604 return res;
553 } 605 }
554 606
555 if (expect_false (maxreqs && !--maxreqs)) 607 if (ecb_expect_false (maxreqs && !--maxreqs))
556 break; 608 break;
557 609
558 if (maxtime) 610 if (maxtime)
559 { 611 {
560 gettimeofday (&tv_now, 0); 612 gettimeofday (&tv_now, 0);
566 618
567 errno = EAGAIN; 619 errno = EAGAIN;
568 return -1; 620 return -1;
569} 621}
570 622
623static void
571static void etp_cancel (ETP_REQ *req) 624etp_cancel (ETP_REQ *req)
572{ 625{
573 X_LOCK (wrklock); 626 req->cancelled = 1;
574 req->flags |= EIO_FLAG_CANCELLED;
575 X_UNLOCK (wrklock);
576 627
577 eio_grp_cancel (req); 628 eio_grp_cancel (req);
578} 629}
579 630
631static void
580static void etp_submit (ETP_REQ *req) 632etp_submit (ETP_REQ *req)
581{ 633{
582 req->pri -= ETP_PRI_MIN; 634 req->pri -= ETP_PRI_MIN;
583 635
584 if (expect_false (req->pri < ETP_PRI_MIN - ETP_PRI_MIN)) req->pri = ETP_PRI_MIN - ETP_PRI_MIN; 636 if (ecb_expect_false (req->pri < ETP_PRI_MIN - ETP_PRI_MIN)) req->pri = ETP_PRI_MIN - ETP_PRI_MIN;
585 if (expect_false (req->pri > ETP_PRI_MAX - ETP_PRI_MIN)) req->pri = ETP_PRI_MAX - ETP_PRI_MIN; 637 if (ecb_expect_false (req->pri > ETP_PRI_MAX - ETP_PRI_MIN)) req->pri = ETP_PRI_MAX - ETP_PRI_MIN;
586 638
587 if (expect_false (req->type == EIO_GROUP)) 639 if (ecb_expect_false (req->type == EIO_GROUP))
588 { 640 {
589 /* I hope this is worth it :/ */ 641 /* I hope this is worth it :/ */
590 X_LOCK (reqlock); 642 X_LOCK (reqlock);
591 ++nreqs; 643 ++nreqs;
592 X_UNLOCK (reqlock); 644 X_UNLOCK (reqlock);
611 663
612 etp_maybe_start_thread (); 664 etp_maybe_start_thread ();
613 } 665 }
614} 666}
615 667
668static void ecb_cold
616static void etp_set_max_poll_time (double nseconds) 669etp_set_max_poll_time (double nseconds)
617{ 670{
618 if (WORDACCESS_UNSAFE) X_LOCK (reslock); 671 if (WORDACCESS_UNSAFE) X_LOCK (reslock);
619 max_poll_time = nseconds * EIO_TICKS; 672 max_poll_time = nseconds * EIO_TICKS;
620 if (WORDACCESS_UNSAFE) X_UNLOCK (reslock); 673 if (WORDACCESS_UNSAFE) X_UNLOCK (reslock);
621} 674}
622 675
676static void ecb_cold
623static void etp_set_max_poll_reqs (unsigned int maxreqs) 677etp_set_max_poll_reqs (unsigned int maxreqs)
624{ 678{
625 if (WORDACCESS_UNSAFE) X_LOCK (reslock); 679 if (WORDACCESS_UNSAFE) X_LOCK (reslock);
626 max_poll_reqs = maxreqs; 680 max_poll_reqs = maxreqs;
627 if (WORDACCESS_UNSAFE) X_UNLOCK (reslock); 681 if (WORDACCESS_UNSAFE) X_UNLOCK (reslock);
628} 682}
629 683
684static void ecb_cold
630static void etp_set_max_idle (unsigned int nthreads) 685etp_set_max_idle (unsigned int nthreads)
631{ 686{
632 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 687 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
633 max_idle = nthreads; 688 max_idle = nthreads;
634 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock); 689 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
635} 690}
636 691
692static void ecb_cold
637static void etp_set_idle_timeout (unsigned int seconds) 693etp_set_idle_timeout (unsigned int seconds)
638{ 694{
639 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 695 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
640 idle_timeout = seconds; 696 idle_timeout = seconds;
641 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock); 697 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
642} 698}
643 699
700static void ecb_cold
644static void etp_set_min_parallel (unsigned int nthreads) 701etp_set_min_parallel (unsigned int nthreads)
645{ 702{
646 if (wanted < nthreads) 703 if (wanted < nthreads)
647 wanted = nthreads; 704 wanted = nthreads;
648} 705}
649 706
707static void ecb_cold
650static void etp_set_max_parallel (unsigned int nthreads) 708etp_set_max_parallel (unsigned int nthreads)
651{ 709{
652 if (wanted > nthreads) 710 if (wanted > nthreads)
653 wanted = nthreads; 711 wanted = nthreads;
654 712
655 while (started > wanted) 713 while (started > wanted)
656 etp_end_thread (); 714 etp_end_thread ();
657} 715}
658 716
659/*****************************************************************************/ 717/*****************************************************************************/
660 718
719static void
661static void grp_try_feed (eio_req *grp) 720grp_try_feed (eio_req *grp)
662{ 721{
663 while (grp->size < grp->int2 && !EIO_CANCELLED (grp)) 722 while (grp->size < grp->int2 && !EIO_CANCELLED (grp))
664 { 723 {
665 grp->flags &= ~EIO_FLAG_GROUPADD; 724 grp->flags &= ~EIO_FLAG_GROUPADD;
666 725
673 break; 732 break;
674 } 733 }
675 } 734 }
676} 735}
677 736
737static int
678static int grp_dec (eio_req *grp) 738grp_dec (eio_req *grp)
679{ 739{
680 --grp->size; 740 --grp->size;
681 741
682 /* call feeder, if applicable */ 742 /* call feeder, if applicable */
683 grp_try_feed (grp); 743 grp_try_feed (grp);
687 return eio_finish (grp); 747 return eio_finish (grp);
688 else 748 else
689 return 0; 749 return 0;
690} 750}
691 751
752static void
692void eio_destroy (eio_req *req) 753eio_destroy (eio_req *req)
693{ 754{
694 if ((req)->flags & EIO_FLAG_PTR1_FREE) free (req->ptr1); 755 if ((req)->flags & EIO_FLAG_PTR1_FREE) free (req->ptr1);
695 if ((req)->flags & EIO_FLAG_PTR2_FREE) free (req->ptr2); 756 if ((req)->flags & EIO_FLAG_PTR2_FREE) free (req->ptr2);
696 757
697 EIO_DESTROY (req); 758 EIO_DESTROY (req);
698} 759}
699 760
761static int
700static int eio_finish (eio_req *req) 762eio_finish (eio_req *req)
701{ 763{
702 int res = EIO_FINISH (req); 764 int res = EIO_FINISH (req);
703 765
704 if (req->grp) 766 if (req->grp)
705 { 767 {
713 if (grp->grp_first == req) 775 if (grp->grp_first == req)
714 grp->grp_first = req->grp_next; 776 grp->grp_first = req->grp_next;
715 777
716 res2 = grp_dec (grp); 778 res2 = grp_dec (grp);
717 779
718 if (!res && res2) 780 if (!res)
719 res = res2; 781 res = res2;
720 } 782 }
721 783
722 eio_destroy (req); 784 eio_destroy (req);
723 785
724 return res; 786 return res;
725} 787}
726 788
789void
727void eio_grp_cancel (eio_req *grp) 790eio_grp_cancel (eio_req *grp)
728{ 791{
729 for (grp = grp->grp_first; grp; grp = grp->grp_next) 792 for (grp = grp->grp_first; grp; grp = grp->grp_next)
730 eio_cancel (grp); 793 eio_cancel (grp);
731} 794}
732 795
796void
733void eio_cancel (eio_req *req) 797eio_cancel (eio_req *req)
734{ 798{
735 etp_cancel (req); 799 etp_cancel (req);
736} 800}
737 801
802void
738void eio_submit (eio_req *req) 803eio_submit (eio_req *req)
739{ 804{
740 etp_submit (req); 805 etp_submit (req);
741} 806}
742 807
743unsigned int eio_nreqs (void) 808unsigned int
809eio_nreqs (void)
744{ 810{
745 return etp_nreqs (); 811 return etp_nreqs ();
746} 812}
747 813
748unsigned int eio_nready (void) 814unsigned int
815eio_nready (void)
749{ 816{
750 return etp_nready (); 817 return etp_nready ();
751} 818}
752 819
753unsigned int eio_npending (void) 820unsigned int
821eio_npending (void)
754{ 822{
755 return etp_npending (); 823 return etp_npending ();
756} 824}
757 825
758unsigned int eio_nthreads (void) 826unsigned int ecb_cold
827eio_nthreads (void)
759{ 828{
760 return etp_nthreads (); 829 return etp_nthreads ();
761} 830}
762 831
832void ecb_cold
763void eio_set_max_poll_time (double nseconds) 833eio_set_max_poll_time (double nseconds)
764{ 834{
765 etp_set_max_poll_time (nseconds); 835 etp_set_max_poll_time (nseconds);
766} 836}
767 837
838void ecb_cold
768void eio_set_max_poll_reqs (unsigned int maxreqs) 839eio_set_max_poll_reqs (unsigned int maxreqs)
769{ 840{
770 etp_set_max_poll_reqs (maxreqs); 841 etp_set_max_poll_reqs (maxreqs);
771} 842}
772 843
844void ecb_cold
773void eio_set_max_idle (unsigned int nthreads) 845eio_set_max_idle (unsigned int nthreads)
774{ 846{
775 etp_set_max_idle (nthreads); 847 etp_set_max_idle (nthreads);
776} 848}
777 849
850void ecb_cold
778void eio_set_idle_timeout (unsigned int seconds) 851eio_set_idle_timeout (unsigned int seconds)
779{ 852{
780 etp_set_idle_timeout (seconds); 853 etp_set_idle_timeout (seconds);
781} 854}
782 855
856void ecb_cold
783void eio_set_min_parallel (unsigned int nthreads) 857eio_set_min_parallel (unsigned int nthreads)
784{ 858{
785 etp_set_min_parallel (nthreads); 859 etp_set_min_parallel (nthreads);
786} 860}
787 861
862void ecb_cold
788void eio_set_max_parallel (unsigned int nthreads) 863eio_set_max_parallel (unsigned int nthreads)
789{ 864{
790 etp_set_max_parallel (nthreads); 865 etp_set_max_parallel (nthreads);
791} 866}
792 867
793int eio_poll (void) 868int eio_poll (void)
802# undef pread 877# undef pread
803# undef pwrite 878# undef pwrite
804# define pread eio__pread 879# define pread eio__pread
805# define pwrite eio__pwrite 880# define pwrite eio__pwrite
806 881
807static ssize_t 882static eio_ssize_t
808eio__pread (int fd, void *buf, size_t count, off_t offset) 883eio__pread (int fd, void *buf, size_t count, off_t offset)
809{ 884{
810 ssize_t res; 885 eio_ssize_t res;
811 off_t ooffset; 886 off_t ooffset;
812 887
813 X_LOCK (preadwritelock); 888 X_LOCK (preadwritelock);
814 ooffset = lseek (fd, 0, SEEK_CUR); 889 ooffset = lseek (fd, 0, SEEK_CUR);
815 lseek (fd, offset, SEEK_SET); 890 lseek (fd, offset, SEEK_SET);
818 X_UNLOCK (preadwritelock); 893 X_UNLOCK (preadwritelock);
819 894
820 return res; 895 return res;
821} 896}
822 897
823static ssize_t 898static eio_ssize_t
824eio__pwrite (int fd, void *buf, size_t count, off_t offset) 899eio__pwrite (int fd, void *buf, size_t count, off_t offset)
825{ 900{
826 ssize_t res; 901 eio_ssize_t res;
827 off_t ooffset; 902 off_t ooffset;
828 903
829 X_LOCK (preadwritelock); 904 X_LOCK (preadwritelock);
830 ooffset = lseek (fd, 0, SEEK_CUR); 905 ooffset = lseek (fd, 0, SEEK_CUR);
831 lseek (fd, offset, SEEK_SET); 906 lseek (fd, offset, SEEK_SET);
863#ifndef HAVE_FUTIMES 938#ifndef HAVE_FUTIMES
864 939
865# undef futimes 940# undef futimes
866# define futimes(fd,times) eio__futimes (fd, times) 941# define futimes(fd,times) eio__futimes (fd, times)
867 942
943static int
868static int eio__futimes (int fd, const struct timeval tv[2]) 944eio__futimes (int fd, const struct timeval tv[2])
869{ 945{
870 errno = ENOSYS; 946 errno = ENOSYS;
871 return -1; 947 return -1;
872} 948}
873 949
877# undef fdatasync 953# undef fdatasync
878# define fdatasync(fd) fsync (fd) 954# define fdatasync(fd) fsync (fd)
879#endif 955#endif
880 956
881/* sync_file_range always needs emulation */ 957/* sync_file_range always needs emulation */
882int 958static int
883eio__sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags) 959eio__sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags)
884{ 960{
885#if HAVE_SYNC_FILE_RANGE 961#if HAVE_SYNC_FILE_RANGE
886 int res; 962 int res;
887 963
904 /* even though we could play tricks with the flags, it's better to always 980 /* even though we could play tricks with the flags, it's better to always
905 * call fdatasync, as that matches the expectation of its users best */ 981 * call fdatasync, as that matches the expectation of its users best */
906 return fdatasync (fd); 982 return fdatasync (fd);
907} 983}
908 984
985static int
986eio__fallocate (int fd, int mode, off_t offset, size_t len)
987{
988#if HAVE_FALLOCATE
989 return fallocate (fd, mode, offset, len);
990#else
991 errno = ENOSYS;
992 return -1;
993#endif
994}
995
909#if !HAVE_READAHEAD 996#if !HAVE_READAHEAD
910# undef readahead 997# undef readahead
911# define readahead(fd,offset,count) eio__readahead (fd, offset, count, self) 998# define readahead(fd,offset,count) eio__readahead (fd, offset, count, self)
912 999
913static ssize_t 1000static eio_ssize_t
914eio__readahead (int fd, off_t offset, size_t count, etp_worker *self) 1001eio__readahead (int fd, off_t offset, size_t count, etp_worker *self)
915{ 1002{
916 size_t todo = count; 1003 size_t todo = count;
917 dBUF; 1004 dBUF;
918 1005
930} 1017}
931 1018
932#endif 1019#endif
933 1020
934/* sendfile always needs emulation */ 1021/* sendfile always needs emulation */
935static ssize_t 1022static eio_ssize_t
936eio__sendfile (int ofd, int ifd, off_t offset, size_t count, etp_worker *self) 1023eio__sendfile (int ofd, int ifd, off_t offset, size_t count, etp_worker *self)
937{ 1024{
1025 eio_ssize_t written = 0;
938 ssize_t res; 1026 eio_ssize_t res;
939 1027
940 if (!count) 1028 if (!count)
941 return 0; 1029 return 0;
942 1030
1031 for (;;)
1032 {
1033#ifdef __APPLE__
1034# undef HAVE_SENDFILE /* broken, as everything on os x */
1035#endif
943#if HAVE_SENDFILE 1036#if HAVE_SENDFILE
944# if __linux 1037# if __linux
1038 off_t soffset = offset;
945 res = sendfile (ofd, ifd, &offset, count); 1039 res = sendfile (ofd, ifd, &soffset, count);
946 1040
947# elif __FreeBSD__ 1041# elif __FreeBSD__
948 /* 1042 /*
949 * Of course, the freebsd sendfile is a dire hack with no thoughts 1043 * Of course, the freebsd sendfile is a dire hack with no thoughts
950 * wasted on making it similar to other I/O functions. 1044 * wasted on making it similar to other I/O functions.
951 */ 1045 */
952 {
953 off_t sbytes; 1046 off_t sbytes;
954 res = sendfile (ifd, ofd, offset, count, 0, &sbytes, 0); 1047 res = sendfile (ifd, ofd, offset, count, 0, &sbytes, 0);
955 1048
956 #if 0 /* according to the manpage, this is correct, but broken behaviour */ 1049 #if 0 /* according to the manpage, this is correct, but broken behaviour */
957 /* freebsd' sendfile will return 0 on success */ 1050 /* freebsd' sendfile will return 0 on success */
958 /* freebsd 8 documents it as only setting *sbytes on EINTR and EAGAIN, but */ 1051 /* freebsd 8 documents it as only setting *sbytes on EINTR and EAGAIN, but */
959 /* not on e.g. EIO or EPIPE - sounds broken */ 1052 /* not on e.g. EIO or EPIPE - sounds broken */
960 if ((res < 0 && (errno == EAGAIN || errno == EINTR) && sbytes) || res == 0) 1053 if ((res < 0 && (errno == EAGAIN || errno == EINTR) && sbytes) || res == 0)
961 res = sbytes; 1054 res = sbytes;
962 #endif 1055 #endif
963 1056
964 /* according to source inspection, this is correct, and useful behaviour */ 1057 /* according to source inspection, this is correct, and useful behaviour */
965 if (sbytes) 1058 if (sbytes)
966 res = sbytes; 1059 res = sbytes;
967 }
968 1060
969# elif defined (__APPLE__) 1061# elif defined (__APPLE__)
970
971 {
972 off_t sbytes = count; 1062 off_t sbytes = count;
973 res = sendfile (ifd, ofd, offset, &sbytes, 0, 0); 1063 res = sendfile (ifd, ofd, offset, &sbytes, 0, 0);
974 1064
975 /* according to the manpage, sbytes is always valid */ 1065 /* according to the manpage, sbytes is always valid */
976 if (sbytes) 1066 if (sbytes)
977 res = sbytes; 1067 res = sbytes;
978 }
979 1068
980# elif __hpux 1069# elif __hpux
981 res = sendfile (ofd, ifd, offset, count, 0, 0); 1070 res = sendfile (ofd, ifd, offset, count, 0, 0);
982 1071
983# elif __solaris 1072# elif __solaris
984 {
985 struct sendfilevec vec; 1073 struct sendfilevec vec;
986 size_t sbytes; 1074 size_t sbytes;
987 1075
988 vec.sfv_fd = ifd; 1076 vec.sfv_fd = ifd;
989 vec.sfv_flag = 0; 1077 vec.sfv_flag = 0;
990 vec.sfv_off = offset; 1078 vec.sfv_off = offset;
991 vec.sfv_len = count; 1079 vec.sfv_len = count;
992 1080
993 res = sendfilev (ofd, &vec, 1, &sbytes); 1081 res = sendfilev (ofd, &vec, 1, &sbytes);
994 1082
995 if (res < 0 && sbytes) 1083 if (res < 0 && sbytes)
996 res = sbytes; 1084 res = sbytes;
997 }
998 1085
999# endif 1086# endif
1000 1087
1001#elif defined (_WIN32) 1088#elif defined (_WIN32) && 0
1002
1003 /* does not work, just for documentation of what would need to be done */ 1089 /* does not work, just for documentation of what would need to be done */
1004 { 1090 /* actually, cannot be done like this, as TransmitFile changes the file offset, */
1091 /* libeio guarantees that the file offset does not change, and windows */
1092 /* has no way to get an independent handle to the same file description */
1005 HANDLE h = TO_SOCKET (ifd); 1093 HANDLE h = TO_SOCKET (ifd);
1006 SetFilePointer (h, offset, 0, FILE_BEGIN); 1094 SetFilePointer (h, offset, 0, FILE_BEGIN);
1007 res = TransmitFile (TO_SOCKET (ofd), h, count, 0, 0, 0, 0); 1095 res = TransmitFile (TO_SOCKET (ofd), h, count, 0, 0, 0, 0);
1008 }
1009 1096
1010#else 1097#else
1011 res = -1; 1098 res = -1;
1012 errno = ENOSYS; 1099 errno = ENOSYS;
1013#endif 1100#endif
1014 1101
1102 /* we assume sendfile can copy at least 128mb in one go */
1103 if (res <= 128 * 1024 * 1024)
1104 {
1105 if (res > 0)
1106 written += res;
1107
1108 if (written)
1109 return written;
1110
1111 break;
1112 }
1113 else
1114 {
1115 /* if we requested more, then probably the kernel was lazy */
1116 written += res;
1117 offset += res;
1118 count -= res;
1119
1120 if (!count)
1121 return written;
1122 }
1123 }
1124
1015 if (res < 0 1125 if (res < 0
1016 && (errno == ENOSYS || errno == EINVAL || errno == ENOTSOCK 1126 && (errno == ENOSYS || errno == EINVAL || errno == ENOTSOCK
1017 /* BSDs */ 1127 /* BSDs */
1018#ifdef ENOTSUP /* sigh, if the steenking pile called openbsd would only try to at least compile posix code... */ 1128#ifdef ENOTSUP /* sigh, if the steenking pile called openbsd would only try to at least compile posix code... */
1019 || errno == ENOTSUP 1129 || errno == ENOTSUP
1020#endif 1130#endif
1030 1140
1031 res = 0; 1141 res = 0;
1032 1142
1033 while (count) 1143 while (count)
1034 { 1144 {
1035 ssize_t cnt; 1145 eio_ssize_t cnt;
1036 1146
1037 cnt = pread (ifd, eio_buf, count > EIO_BUFSIZE ? EIO_BUFSIZE : count, offset); 1147 cnt = pread (ifd, eio_buf, count > EIO_BUFSIZE ? EIO_BUFSIZE : count, offset);
1038 1148
1039 if (cnt <= 0) 1149 if (cnt <= 0)
1040 { 1150 {
1055 count -= cnt; 1165 count -= cnt;
1056 } 1166 }
1057 } 1167 }
1058 1168
1059 return res; 1169 return res;
1170}
1171
1172#ifdef PAGESIZE
1173# define eio_pagesize() PAGESIZE
1174#else
1175static intptr_t
1176eio_pagesize (void)
1177{
1178 static intptr_t page;
1179
1180 if (!page)
1181 page = sysconf (_SC_PAGESIZE);
1182
1183 return page;
1184}
1185#endif
1186
1187static void
1188eio_page_align (void **addr, size_t *length)
1189{
1190 intptr_t mask = eio_pagesize () - 1;
1191
1192 /* round down addr */
1193 intptr_t adj = mask & (intptr_t)*addr;
1194
1195 *addr = (void *)((intptr_t)*addr - adj);
1196 *length += adj;
1197
1198 /* round up length */
1199 *length = (*length + mask) & ~mask;
1200}
1201
1202#if !_POSIX_MEMLOCK
1203# define eio__mlockall(a) EIO_ENOSYS ()
1204#else
1205
1206static int
1207eio__mlockall (int flags)
1208{
1209 #if __GLIBC__ == 2 && __GLIBC_MINOR__ <= 7
1210 extern int mallopt (int, int);
1211 mallopt (-6, 238); /* http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=473812 */
1212 #endif
1213
1214 if (EIO_MCL_CURRENT != MCL_CURRENT
1215 || EIO_MCL_FUTURE != MCL_FUTURE)
1216 {
1217 flags = 0
1218 | (flags & EIO_MCL_CURRENT ? MCL_CURRENT : 0)
1219 | (flags & EIO_MCL_FUTURE ? MCL_FUTURE : 0);
1220 }
1221
1222 return mlockall (flags);
1223}
1224#endif
1225
1226#if !_POSIX_MEMLOCK_RANGE
1227# define eio__mlock(a,b) EIO_ENOSYS ()
1228#else
1229
1230static int
1231eio__mlock (void *addr, size_t length)
1232{
1233 eio_page_align (&addr, &length);
1234
1235 return mlock (addr, length);
1236}
1237
1238#endif
1239
1240#if !(_POSIX_MAPPED_FILES && _POSIX_SYNCHRONIZED_IO)
1241# define eio__msync(a,b,c) EIO_ENOSYS ()
1242#else
1243
1244static int
1245eio__msync (void *mem, size_t len, int flags)
1246{
1247 eio_page_align (&mem, &len);
1248
1249 if (EIO_MS_ASYNC != MS_SYNC
1250 || EIO_MS_INVALIDATE != MS_INVALIDATE
1251 || EIO_MS_SYNC != MS_SYNC)
1252 {
1253 flags = 0
1254 | (flags & EIO_MS_ASYNC ? MS_ASYNC : 0)
1255 | (flags & EIO_MS_INVALIDATE ? MS_INVALIDATE : 0)
1256 | (flags & EIO_MS_SYNC ? MS_SYNC : 0);
1257 }
1258
1259 return msync (mem, len, flags);
1260}
1261
1262#endif
1263
1264static int
1265eio__mtouch (eio_req *req)
1266{
1267 void *mem = req->ptr2;
1268 size_t len = req->size;
1269 int flags = req->int1;
1270
1271 eio_page_align (&mem, &len);
1272
1273 {
1274 intptr_t addr = (intptr_t)mem;
1275 intptr_t end = addr + len;
1276 intptr_t page = eio_pagesize ();
1277
1278 if (addr < end)
1279 if (flags & EIO_MT_MODIFY) /* modify */
1280 do { *((volatile sig_atomic_t *)addr) |= 0; } while ((addr += page) < len && !EIO_CANCELLED (req));
1281 else
1282 do { *((volatile sig_atomic_t *)addr) ; } while ((addr += page) < len && !EIO_CANCELLED (req));
1283 }
1284
1285 return 0;
1286}
1287
1288/*****************************************************************************/
1289/* requests implemented outside eio_execute, because they are so large */
1290
1291static void
1292eio__realpath (eio_req *req, etp_worker *self)
1293{
1294 char *rel = req->ptr1;
1295 char *res;
1296 char *tmp1, *tmp2;
1297#if SYMLOOP_MAX > 32
1298 int symlinks = SYMLOOP_MAX;
1299#else
1300 int symlinks = 32;
1301#endif
1302
1303 req->result = -1;
1304
1305 errno = EINVAL;
1306 if (!rel)
1307 return;
1308
1309 errno = ENOENT;
1310 if (!*rel)
1311 return;
1312
1313 if (!req->ptr2)
1314 {
1315 X_LOCK (wrklock);
1316 req->flags |= EIO_FLAG_PTR2_FREE;
1317 X_UNLOCK (wrklock);
1318 req->ptr2 = malloc (PATH_MAX * 3);
1319
1320 errno = ENOMEM;
1321 if (!req->ptr2)
1322 return;
1323 }
1324
1325 res = req->ptr2;
1326 tmp1 = res + PATH_MAX;
1327 tmp2 = tmp1 + PATH_MAX;
1328
1329#if 0 /* disabled, the musl way to do things is just too racy */
1330#if __linux && defined(O_NONBLOCK) && defined(O_NOATIME)
1331 /* on linux we may be able to ask the kernel */
1332 {
1333 int fd = open (rel, O_RDONLY | O_NONBLOCK | O_NOCTTY | O_NOATIME);
1334
1335 if (fd >= 0)
1336 {
1337 sprintf (tmp1, "/proc/self/fd/%d", fd);
1338 req->result = readlink (tmp1, res, PATH_MAX);
1339 close (fd);
1340
1341 /* here we should probably stat the open file and the disk file, to make sure they still match */
1342
1343 if (req->result > 0)
1344 goto done;
1345 }
1346 else if (errno == ELOOP || errno == ENAMETOOLONG || errno == ENOENT || errno == ENOTDIR || errno == EIO)
1347 return;
1348 }
1349#endif
1350#endif
1351
1352 if (*rel != '/')
1353 {
1354 if (!getcwd (res, PATH_MAX))
1355 return;
1356
1357 if (res [1]) /* only use if not / */
1358 res += strlen (res);
1359 }
1360
1361 while (*rel)
1362 {
1363 eio_ssize_t len, linklen;
1364 char *beg = rel;
1365
1366 while (*rel && *rel != '/')
1367 ++rel;
1368
1369 len = rel - beg;
1370
1371 if (!len) /* skip slashes */
1372 {
1373 ++rel;
1374 continue;
1375 }
1376
1377 if (beg [0] == '.')
1378 {
1379 if (len == 1)
1380 continue; /* . - nop */
1381
1382 if (beg [1] == '.' && len == 2)
1383 {
1384 /* .. - back up one component, if possible */
1385
1386 while (res != req->ptr2)
1387 if (*--res == '/')
1388 break;
1389
1390 continue;
1391 }
1392 }
1393
1394 errno = ENAMETOOLONG;
1395 if (res + 1 + len + 1 >= tmp1)
1396 return;
1397
1398 /* copy one component */
1399 *res = '/';
1400 memcpy (res + 1, beg, len);
1401
1402 /* zero-terminate, for readlink */
1403 res [len + 1] = 0;
1404
1405 /* now check if it's a symlink */
1406 linklen = readlink (req->ptr2, tmp1, PATH_MAX);
1407
1408 if (linklen < 0)
1409 {
1410 if (errno != EINVAL)
1411 return;
1412
1413 /* it's a normal directory. hopefully */
1414 res += len + 1;
1415 }
1416 else
1417 {
1418 /* yay, it was a symlink - build new path in tmp2 */
1419 int rellen = strlen (rel);
1420
1421 errno = ENAMETOOLONG;
1422 if (linklen + 1 + rellen >= PATH_MAX)
1423 return;
1424
1425 errno = ELOOP;
1426 if (!--symlinks)
1427 return;
1428
1429 if (*tmp1 == '/')
1430 res = req->ptr2; /* symlink resolves to an absolute path */
1431
1432 /* we need to be careful, as rel might point into tmp2 already */
1433 memmove (tmp2 + linklen + 1, rel, rellen + 1);
1434 tmp2 [linklen] = '/';
1435 memcpy (tmp2, tmp1, linklen);
1436
1437 rel = tmp2;
1438 }
1439 }
1440
1441 /* special case for the lone root path */
1442 if (res == req->ptr2)
1443 *res++ = '/';
1444
1445 req->result = res - (char *)req->ptr2;
1446
1447done:
1448 req->ptr2 = realloc (req->ptr2, req->result); /* trade time for space savings */
1060} 1449}
1061 1450
1062static signed char 1451static signed char
1063eio_dent_cmp (const eio_dirent *a, const eio_dirent *b) 1452eio_dent_cmp (const eio_dirent *a, const eio_dirent *b)
1064{ 1453{
1246 1635
1247 X_LOCK (wrklock); 1636 X_LOCK (wrklock);
1248 /* the corresponding closedir is in ETP_WORKER_CLEAR */ 1637 /* the corresponding closedir is in ETP_WORKER_CLEAR */
1249 self->dirp = dirp = opendir (req->ptr1); 1638 self->dirp = dirp = opendir (req->ptr1);
1250 1639
1640 if (req->flags & EIO_FLAG_PTR1_FREE)
1641 free (req->ptr1);
1642
1251 req->flags |= EIO_FLAG_PTR1_FREE | EIO_FLAG_PTR2_FREE; 1643 req->flags |= EIO_FLAG_PTR1_FREE | EIO_FLAG_PTR2_FREE;
1252 req->ptr1 = dents = flags ? malloc (dentalloc * sizeof (eio_dirent)) : 0; 1644 req->ptr1 = dents = flags ? malloc (dentalloc * sizeof (eio_dirent)) : 0;
1253 req->ptr2 = names = malloc (namesalloc); 1645 req->ptr2 = names = malloc (namesalloc);
1254 X_UNLOCK (wrklock); 1646 X_UNLOCK (wrklock);
1255 1647
1306 /* skip . and .. entries */ 1698 /* skip . and .. entries */
1307 if (name [0] != '.' || (name [1] && (name [1] != '.' || name [2]))) 1699 if (name [0] != '.' || (name [1] && (name [1] != '.' || name [2])))
1308 { 1700 {
1309 int len = D_NAMLEN (entp) + 1; 1701 int len = D_NAMLEN (entp) + 1;
1310 1702
1311 while (expect_false (namesoffs + len > namesalloc)) 1703 while (ecb_expect_false (namesoffs + len > namesalloc))
1312 { 1704 {
1313 namesalloc *= 2; 1705 namesalloc *= 2;
1314 X_LOCK (wrklock); 1706 X_LOCK (wrklock);
1315 req->ptr2 = names = realloc (names, namesalloc); 1707 req->ptr2 = names = realloc (names, namesalloc);
1316 X_UNLOCK (wrklock); 1708 X_UNLOCK (wrklock);
1323 1715
1324 if (dents) 1716 if (dents)
1325 { 1717 {
1326 struct eio_dirent *ent; 1718 struct eio_dirent *ent;
1327 1719
1328 if (expect_false (dentoffs == dentalloc)) 1720 if (ecb_expect_false (dentoffs == dentalloc))
1329 { 1721 {
1330 dentalloc *= 2; 1722 dentalloc *= 2;
1331 X_LOCK (wrklock); 1723 X_LOCK (wrklock);
1332 req->ptr1 = dents = realloc (dents, dentalloc * sizeof (eio_dirent)); 1724 req->ptr1 = dents = realloc (dents, dentalloc * sizeof (eio_dirent));
1333 X_UNLOCK (wrklock); 1725 X_UNLOCK (wrklock);
1421 break; 1813 break;
1422 } 1814 }
1423 } 1815 }
1424} 1816}
1425 1817
1426#ifdef PAGESIZE
1427# define eio_pagesize() PAGESIZE
1428#else
1429static intptr_t
1430eio_pagesize (void)
1431{
1432 static intptr_t page;
1433
1434 if (!page)
1435 page = sysconf (_SC_PAGESIZE);
1436
1437 return page;
1438}
1439#endif
1440
1441static void
1442eio_page_align (void **addr, size_t *length)
1443{
1444 intptr_t mask = eio_pagesize () - 1;
1445
1446 /* round down addr */
1447 intptr_t adj = mask & (intptr_t)*addr;
1448
1449 *addr = (void *)((intptr_t)*addr - adj);
1450 *length += adj;
1451
1452 /* round up length */
1453 *length = (*length + mask) & ~mask;
1454}
1455
1456#if !_POSIX_MEMLOCK
1457# define eio__mlockall(a) ((errno = ENOSYS), -1)
1458#else
1459
1460static int
1461eio__mlockall (int flags)
1462{
1463 #if __GLIBC__ == 2 && __GLIBC_MINOR__ <= 7
1464 extern int mallopt (int, int);
1465 mallopt (-6, 238); /* http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=473812 */
1466 #endif
1467
1468 if (EIO_MCL_CURRENT != MCL_CURRENT
1469 || EIO_MCL_FUTURE != MCL_FUTURE)
1470 {
1471 flags = 0
1472 | (flags & EIO_MCL_CURRENT ? MCL_CURRENT : 0)
1473 | (flags & EIO_MCL_FUTURE ? MCL_FUTURE : 0);
1474 }
1475
1476 return mlockall (flags);
1477}
1478#endif
1479
1480#if !_POSIX_MEMLOCK_RANGE
1481# define eio__mlock(a,b) ((errno = ENOSYS), -1)
1482#else
1483
1484static int
1485eio__mlock (void *addr, size_t length)
1486{
1487 eio_page_align (&addr, &length);
1488
1489 return mlock (addr, length);
1490}
1491
1492#endif
1493
1494#if !(_POSIX_MAPPED_FILES && _POSIX_SYNCHRONIZED_IO)
1495# define eio__msync(a,b,c) ((errno = ENOSYS), -1)
1496#else
1497
1498int
1499eio__msync (void *mem, size_t len, int flags)
1500{
1501 eio_page_align (&mem, &len);
1502
1503 if (EIO_MS_ASYNC != MS_SYNC
1504 || EIO_MS_INVALIDATE != MS_INVALIDATE
1505 || EIO_MS_SYNC != MS_SYNC)
1506 {
1507 flags = 0
1508 | (flags & EIO_MS_ASYNC ? MS_ASYNC : 0)
1509 | (flags & EIO_MS_INVALIDATE ? MS_INVALIDATE : 0)
1510 | (flags & EIO_MS_SYNC ? MS_SYNC : 0);
1511 }
1512
1513 return msync (mem, len, flags);
1514}
1515
1516#endif
1517
1518int
1519eio__mtouch (void *mem, size_t len, int flags)
1520{
1521 eio_page_align (&mem, &len);
1522
1523 {
1524 intptr_t addr = (intptr_t)mem;
1525 intptr_t end = addr + len;
1526 intptr_t page = eio_pagesize ();
1527
1528 if (addr < end)
1529 if (flags & EIO_MT_MODIFY) /* modify */
1530 do { *((volatile sig_atomic_t *)addr) |= 0; } while ((addr += page) < len);
1531 else
1532 do { *((volatile sig_atomic_t *)addr) ; } while ((addr += page) < len);
1533 }
1534
1535 return 0;
1536}
1537
1538/*****************************************************************************/ 1818/*****************************************************************************/
1539 1819
1540#define ALLOC(len) \ 1820#define ALLOC(len) \
1541 if (!req->ptr2) \ 1821 if (!req->ptr2) \
1542 { \ 1822 { \
1555X_THREAD_PROC (etp_proc) 1835X_THREAD_PROC (etp_proc)
1556{ 1836{
1557 ETP_REQ *req; 1837 ETP_REQ *req;
1558 struct timespec ts; 1838 struct timespec ts;
1559 etp_worker *self = (etp_worker *)thr_arg; 1839 etp_worker *self = (etp_worker *)thr_arg;
1840 int timeout;
1560 1841
1561 /* try to distribute timeouts somewhat randomly */ 1842 /* try to distribute timeouts somewhat evenly */
1562 ts.tv_nsec = ((unsigned long)self & 1023UL) * (1000000000UL / 1024UL); 1843 ts.tv_nsec = ((unsigned long)self & 1023UL) * (1000000000UL / 1024UL);
1563 1844
1564 for (;;) 1845 for (;;)
1565 { 1846 {
1847 ts.tv_sec = 0;
1848
1566 X_LOCK (reqlock); 1849 X_LOCK (reqlock);
1567 1850
1568 for (;;) 1851 for (;;)
1569 { 1852 {
1570 self->req = req = reqq_shift (&req_queue); 1853 self->req = req = reqq_shift (&req_queue);
1571 1854
1572 if (req) 1855 if (req)
1573 break; 1856 break;
1574 1857
1858 if (ts.tv_sec == 1) /* no request, but timeout detected, let's quit */
1859 {
1860 X_UNLOCK (reqlock);
1861 X_LOCK (wrklock);
1862 --started;
1863 X_UNLOCK (wrklock);
1864 goto quit;
1865 }
1866
1575 ++idle; 1867 ++idle;
1576 1868
1577 ts.tv_sec = time (0) + idle_timeout; 1869 if (idle <= max_idle)
1578 if (X_COND_TIMEDWAIT (reqwait, reqlock, ts) == ETIMEDOUT) 1870 /* we are allowed to idle, so do so without any timeout */
1871 X_COND_WAIT (reqwait, reqlock);
1872 else
1579 { 1873 {
1580 if (idle > max_idle) 1874 /* initialise timeout once */
1581 { 1875 if (!ts.tv_sec)
1582 --idle; 1876 ts.tv_sec = time (0) + idle_timeout;
1583 X_UNLOCK (reqlock);
1584 X_LOCK (wrklock);
1585 --started;
1586 X_UNLOCK (wrklock);
1587 goto quit;
1588 }
1589 1877
1590 /* we are allowed to idle, so do so without any timeout */
1591 X_COND_WAIT (reqwait, reqlock); 1878 if (X_COND_TIMEDWAIT (reqwait, reqlock, ts) == ETIMEDOUT)
1879 ts.tv_sec = 1; /* assuming this is not a value computed above.,.. */
1592 } 1880 }
1593 1881
1594 --idle; 1882 --idle;
1595 } 1883 }
1596 1884
1599 X_UNLOCK (reqlock); 1887 X_UNLOCK (reqlock);
1600 1888
1601 if (req->type < 0) 1889 if (req->type < 0)
1602 goto quit; 1890 goto quit;
1603 1891
1604 if (!EIO_CANCELLED (req))
1605 ETP_EXECUTE (self, req); 1892 ETP_EXECUTE (self, req);
1606 1893
1607 X_LOCK (reslock); 1894 X_LOCK (reslock);
1608 1895
1609 ++npending; 1896 ++npending;
1610 1897
1625 return 0; 1912 return 0;
1626} 1913}
1627 1914
1628/*****************************************************************************/ 1915/*****************************************************************************/
1629 1916
1917int ecb_cold
1630int eio_init (void (*want_poll)(void), void (*done_poll)(void)) 1918eio_init (void (*want_poll)(void), void (*done_poll)(void))
1631{ 1919{
1632 return etp_init (want_poll, done_poll); 1920 return etp_init (want_poll, done_poll);
1633} 1921}
1634 1922
1923ecb_inline void
1635static void eio_api_destroy (eio_req *req) 1924eio_api_destroy (eio_req *req)
1636{ 1925{
1637 free (req); 1926 free (req);
1638} 1927}
1639 1928
1640#define REQ(rtype) \ 1929#define REQ(rtype) \
1659 { \ 1948 { \
1660 eio_api_destroy (req); \ 1949 eio_api_destroy (req); \
1661 return 0; \ 1950 return 0; \
1662 } 1951 }
1663 1952
1953static void
1664static void eio_execute (etp_worker *self, eio_req *req) 1954eio_execute (etp_worker *self, eio_req *req)
1665{ 1955{
1956 if (ecb_expect_false (EIO_CANCELLED (req)))
1957 {
1958 req->result = -1;
1959 req->errorno = ECANCELED;
1960 return;
1961 }
1962
1666 switch (req->type) 1963 switch (req->type)
1667 { 1964 {
1668 case EIO_READ: ALLOC (req->size); 1965 case EIO_READ: ALLOC (req->size);
1669 req->result = req->offs >= 0 1966 req->result = req->offs >= 0
1670 ? pread (req->int1, req->ptr2, req->size, req->offs) 1967 ? pread (req->int1, req->ptr2, req->size, req->offs)
1704 case EIO_RENAME: req->result = rename (req->ptr1, req->ptr2); break; 2001 case EIO_RENAME: req->result = rename (req->ptr1, req->ptr2); break;
1705 case EIO_LINK: req->result = link (req->ptr1, req->ptr2); break; 2002 case EIO_LINK: req->result = link (req->ptr1, req->ptr2); break;
1706 case EIO_SYMLINK: req->result = symlink (req->ptr1, req->ptr2); break; 2003 case EIO_SYMLINK: req->result = symlink (req->ptr1, req->ptr2); break;
1707 case EIO_MKNOD: req->result = mknod (req->ptr1, (mode_t)req->int2, (dev_t)req->offs); break; 2004 case EIO_MKNOD: req->result = mknod (req->ptr1, (mode_t)req->int2, (dev_t)req->offs); break;
1708 2005
2006 case EIO_REALPATH: eio__realpath (req, self); break;
2007
1709 case EIO_READLINK: ALLOC (PATH_MAX); 2008 case EIO_READLINK: ALLOC (PATH_MAX);
1710 req->result = readlink (req->ptr1, req->ptr2, PATH_MAX); break; 2009 req->result = readlink (req->ptr1, req->ptr2, PATH_MAX); break;
1711 2010
1712 case EIO_SYNC: req->result = 0; sync (); break; 2011 case EIO_SYNC: req->result = 0; sync (); break;
1713 case EIO_FSYNC: req->result = fsync (req->int1); break; 2012 case EIO_FSYNC: req->result = fsync (req->int1); break;
1714 case EIO_FDATASYNC: req->result = fdatasync (req->int1); break; 2013 case EIO_FDATASYNC: req->result = fdatasync (req->int1); break;
1715 case EIO_MSYNC: req->result = eio__msync (req->ptr2, req->size, req->int1); break; 2014 case EIO_MSYNC: req->result = eio__msync (req->ptr2, req->size, req->int1); break;
1716 case EIO_MTOUCH: req->result = eio__mtouch (req->ptr2, req->size, req->int1); break; 2015 case EIO_MTOUCH: req->result = eio__mtouch (req); break;
1717 case EIO_MLOCK: req->result = eio__mlock (req->ptr2, req->size); break; 2016 case EIO_MLOCK: req->result = eio__mlock (req->ptr2, req->size); break;
1718 case EIO_MLOCKALL: req->result = eio__mlockall (req->int1); break; 2017 case EIO_MLOCKALL: req->result = eio__mlockall (req->int1); break;
1719 case EIO_SYNC_FILE_RANGE: req->result = eio__sync_file_range (req->int1, req->offs, req->size, req->int2); break; 2018 case EIO_SYNC_FILE_RANGE: req->result = eio__sync_file_range (req->int1, req->offs, req->size, req->int2); break;
2019 case EIO_FALLOCATE: req->result = eio__fallocate (req->int1, req->int2, req->offs, req->size); break;
1720 2020
1721 case EIO_READDIR: eio__scandir (req, self); break; 2021 case EIO_READDIR: eio__scandir (req, self); break;
1722 2022
1723 case EIO_BUSY: 2023 case EIO_BUSY:
1724#ifdef _WIN32 2024#ifdef _WIN32
1765 case EIO_NOP: 2065 case EIO_NOP:
1766 req->result = 0; 2066 req->result = 0;
1767 break; 2067 break;
1768 2068
1769 case EIO_CUSTOM: 2069 case EIO_CUSTOM:
1770 ((void (*)(eio_req *))req->feed) (req); 2070 req->feed (req);
1771 break; 2071 break;
1772 2072
1773 default: 2073 default:
1774 errno = ENOSYS; 2074 errno = ENOSYS;
1775 req->result = -1; 2075 req->result = -1;
1824eio_req *eio_sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags, int pri, eio_cb cb, void *data) 2124eio_req *eio_sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags, int pri, eio_cb cb, void *data)
1825{ 2125{
1826 REQ (EIO_SYNC_FILE_RANGE); req->int1 = fd; req->offs = offset; req->size = nbytes; req->int2 = flags; SEND; 2126 REQ (EIO_SYNC_FILE_RANGE); req->int1 = fd; req->offs = offset; req->size = nbytes; req->int2 = flags; SEND;
1827} 2127}
1828 2128
2129eio_req *eio_fallocate (int fd, int mode, off_t offset, size_t len, int pri, eio_cb cb, void *data)
2130{
2131 REQ (EIO_FALLOCATE); req->int1 = fd; req->int2 = mode; req->offs = offset; req->size = len; SEND;
2132}
2133
1829eio_req *eio_fdatasync (int fd, int pri, eio_cb cb, void *data) 2134eio_req *eio_fdatasync (int fd, int pri, eio_cb cb, void *data)
1830{ 2135{
1831 REQ (EIO_FDATASYNC); req->int1 = fd; SEND; 2136 REQ (EIO_FDATASYNC); req->int1 = fd; SEND;
1832} 2137}
1833 2138
1874eio_req *eio_fchmod (int fd, mode_t mode, int pri, eio_cb cb, void *data) 2179eio_req *eio_fchmod (int fd, mode_t mode, int pri, eio_cb cb, void *data)
1875{ 2180{
1876 REQ (EIO_FCHMOD); req->int1 = fd; req->int2 = (long)mode; SEND; 2181 REQ (EIO_FCHMOD); req->int1 = fd; req->int2 = (long)mode; SEND;
1877} 2182}
1878 2183
1879eio_req *eio_fchown (int fd, uid_t uid, gid_t gid, int pri, eio_cb cb, void *data) 2184eio_req *eio_fchown (int fd, eio_uid_t uid, eio_gid_t gid, int pri, eio_cb cb, void *data)
1880{ 2185{
1881 REQ (EIO_FCHOWN); req->int1 = fd; req->int2 = (long)uid; req->int3 = (long)gid; SEND; 2186 REQ (EIO_FCHOWN); req->int1 = fd; req->int2 = (long)uid; req->int3 = (long)gid; SEND;
1882} 2187}
1883 2188
1884eio_req *eio_dup2 (int fd, int fd2, int pri, eio_cb cb, void *data) 2189eio_req *eio_dup2 (int fd, int fd2, int pri, eio_cb cb, void *data)
1904eio_req *eio_truncate (const char *path, off_t offset, int pri, eio_cb cb, void *data) 2209eio_req *eio_truncate (const char *path, off_t offset, int pri, eio_cb cb, void *data)
1905{ 2210{
1906 REQ (EIO_TRUNCATE); PATH; req->offs = offset; SEND; 2211 REQ (EIO_TRUNCATE); PATH; req->offs = offset; SEND;
1907} 2212}
1908 2213
1909eio_req *eio_chown (const char *path, uid_t uid, gid_t gid, int pri, eio_cb cb, void *data) 2214eio_req *eio_chown (const char *path, eio_uid_t uid, eio_gid_t gid, int pri, eio_cb cb, void *data)
1910{ 2215{
1911 REQ (EIO_CHOWN); PATH; req->int2 = (long)uid; req->int3 = (long)gid; SEND; 2216 REQ (EIO_CHOWN); PATH; req->int2 = (long)uid; req->int3 = (long)gid; SEND;
1912} 2217}
1913 2218
1914eio_req *eio_chmod (const char *path, mode_t mode, int pri, eio_cb cb, void *data) 2219eio_req *eio_chmod (const char *path, mode_t mode, int pri, eio_cb cb, void *data)
1928} 2233}
1929 2234
1930eio_req *eio_readlink (const char *path, int pri, eio_cb cb, void *data) 2235eio_req *eio_readlink (const char *path, int pri, eio_cb cb, void *data)
1931{ 2236{
1932 return eio__1path (EIO_READLINK, path, pri, cb, data); 2237 return eio__1path (EIO_READLINK, path, pri, cb, data);
2238}
2239
2240eio_req *eio_realpath (const char *path, int pri, eio_cb cb, void *data)
2241{
2242 return eio__1path (EIO_REALPATH, path, pri, cb, data);
1933} 2243}
1934 2244
1935eio_req *eio_stat (const char *path, int pri, eio_cb cb, void *data) 2245eio_req *eio_stat (const char *path, int pri, eio_cb cb, void *data)
1936{ 2246{
1937 return eio__1path (EIO_STAT, path, pri, cb, data); 2247 return eio__1path (EIO_STAT, path, pri, cb, data);
1996eio_req *eio_rename (const char *path, const char *new_path, int pri, eio_cb cb, void *data) 2306eio_req *eio_rename (const char *path, const char *new_path, int pri, eio_cb cb, void *data)
1997{ 2307{
1998 return eio__2path (EIO_RENAME, path, new_path, pri, cb, data); 2308 return eio__2path (EIO_RENAME, path, new_path, pri, cb, data);
1999} 2309}
2000 2310
2001eio_req *eio_custom (eio_cb execute, int pri, eio_cb cb, void *data) 2311eio_req *eio_custom (void (*execute)(eio_req *), int pri, eio_cb cb, void *data)
2002{ 2312{
2003 REQ (EIO_CUSTOM); req->feed = (void (*)(eio_req *))execute; SEND; 2313 REQ (EIO_CUSTOM); req->feed = execute; SEND;
2004} 2314}
2005 2315
2006#endif 2316#endif
2007 2317
2008eio_req *eio_grp (eio_cb cb, void *data) 2318eio_req *eio_grp (eio_cb cb, void *data)
2017#undef SEND 2327#undef SEND
2018 2328
2019/*****************************************************************************/ 2329/*****************************************************************************/
2020/* grp functions */ 2330/* grp functions */
2021 2331
2332void
2022void eio_grp_feed (eio_req *grp, void (*feed)(eio_req *req), int limit) 2333eio_grp_feed (eio_req *grp, void (*feed)(eio_req *req), int limit)
2023{ 2334{
2024 grp->int2 = limit; 2335 grp->int2 = limit;
2025 grp->feed = feed; 2336 grp->feed = feed;
2026 2337
2027 grp_try_feed (grp); 2338 grp_try_feed (grp);
2028} 2339}
2029 2340
2341void
2030void eio_grp_limit (eio_req *grp, int limit) 2342eio_grp_limit (eio_req *grp, int limit)
2031{ 2343{
2032 grp->int2 = limit; 2344 grp->int2 = limit;
2033 2345
2034 grp_try_feed (grp); 2346 grp_try_feed (grp);
2035} 2347}
2036 2348
2349void
2037void eio_grp_add (eio_req *grp, eio_req *req) 2350eio_grp_add (eio_req *grp, eio_req *req)
2038{ 2351{
2039 assert (("cannot add requests to IO::AIO::GRP after the group finished", grp->int1 != 2)); 2352 assert (("cannot add requests to IO::AIO::GRP after the group finished", grp->int1 != 2));
2040 2353
2041 grp->flags |= EIO_FLAG_GROUPADD; 2354 grp->flags |= EIO_FLAG_GROUPADD;
2042 2355
2053} 2366}
2054 2367
2055/*****************************************************************************/ 2368/*****************************************************************************/
2056/* misc garbage */ 2369/* misc garbage */
2057 2370
2371eio_ssize_t
2058ssize_t eio_sendfile_sync (int ofd, int ifd, off_t offset, size_t count) 2372eio_sendfile_sync (int ofd, int ifd, off_t offset, size_t count)
2059{ 2373{
2060 etp_worker wrk; 2374 etp_worker wrk;
2061 ssize_t ret; 2375 eio_ssize_t ret;
2062 2376
2063 wrk.dbuf = 0; 2377 wrk.dbuf = 0;
2064 2378
2065 ret = eio__sendfile (ofd, ifd, offset, count, &wrk); 2379 ret = eio__sendfile (ofd, ifd, offset, count, &wrk);
2066 2380

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines