ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/libeio/eio.c
(Generate patch)

Comparing libeio/eio.c (file contents):
Revision 1.67 by root, Sun Jun 5 20:25:00 2011 UTC vs.
Revision 1.89 by root, Thu Jul 14 22:36:17 2011 UTC

40#ifndef _WIN32 40#ifndef _WIN32
41# include "config.h" 41# include "config.h"
42#endif 42#endif
43 43
44#include "eio.h" 44#include "eio.h"
45#include "ecb.h"
45 46
46#ifdef EIO_STACKSIZE 47#ifdef EIO_STACKSIZE
47# define XTHREAD_STACKSIZE EIO_STACKSIZE 48# define XTHREAD_STACKSIZE EIO_STACKSIZE
48#endif 49#endif
49#include "xthread.h" 50#include "xthread.h"
53#include <stdlib.h> 54#include <stdlib.h>
54#include <string.h> 55#include <string.h>
55#include <errno.h> 56#include <errno.h>
56#include <sys/types.h> 57#include <sys/types.h>
57#include <sys/stat.h> 58#include <sys/stat.h>
58#include <sys/statvfs.h>
59#include <limits.h> 59#include <limits.h>
60#include <fcntl.h> 60#include <fcntl.h>
61#include <assert.h> 61#include <assert.h>
62 62
63/* intptr_t comes from unistd.h, says POSIX/UNIX/tradition */ 63/* intptr_t comes from unistd.h, says POSIX/UNIX/tradition */
64/* intptr_t only comes form stdint.h, says idiot openbsd coder */ 64/* intptr_t only comes from stdint.h, says idiot openbsd coder */
65#if HAVE_STDINT_H 65#if HAVE_STDINT_H
66# include <stdint.h> 66# include <stdint.h>
67#endif 67#endif
68 68
69#ifndef ECANCELED
70# define ECANCELED EDOM
71#endif
72
73static void eio_destroy (eio_req *req);
74
69#ifndef EIO_FINISH 75#ifndef EIO_FINISH
70# define EIO_FINISH(req) ((req)->finish) && !EIO_CANCELLED (req) ? (req)->finish (req) : 0 76# define EIO_FINISH(req) ((req)->finish) && !EIO_CANCELLED (req) ? (req)->finish (req) : 0
71#endif 77#endif
72 78
73#ifndef EIO_DESTROY 79#ifndef EIO_DESTROY
76 82
77#ifndef EIO_FEED 83#ifndef EIO_FEED
78# define EIO_FEED(req) do { if ((req)->feed ) (req)->feed (req); } while (0) 84# define EIO_FEED(req) do { if ((req)->feed ) (req)->feed (req); } while (0)
79#endif 85#endif
80 86
87#ifndef EIO_FD_TO_WIN32_HANDLE
88# define EIO_FD_TO_WIN32_HANDLE(fd) _get_osfhandle (fd)
89#endif
90#ifndef EIO_WIN32_HANDLE_TO_FD
91# define EIO_WIN32_HANDLE_TO_FD(handle) _open_osfhandle (handle, 0)
92#endif
93
94#define EIO_ERRNO(errval,retval) ((errno = errval), retval)
95
96#define EIO_ENOSYS() EIO_ERRNO (ENOSYS, -1)
97
81#ifdef _WIN32 98#ifdef _WIN32
82 99
83 /*doh*/ 100 #define PAGESIZE 4096 /* GetSystemInfo? */
101
102 #ifdef EIO_STRUCT_STATI64
103 #define stat(path,buf) _stati64 (path,buf)
104 #define fstat(fd,buf) _fstati64 (path,buf)
105 #endif
106 #define lstat(path,buf) stat (path,buf)
107 #define fsync(fd) (FlushFileBuffers (EIO_FD_TO_WIN32_HANDLE (fd)) ? 0 : EIO_ERRNO (EBADF, -1))
108 #define mkdir(path,mode) _mkdir (path)
109 #define link(old,neu) (CreateHardLink (neu, old, 0) ? 0 : EIO_ERRNO (ENOENT, -1))
110
111 #define chown(path,uid,gid) EIO_ENOSYS ()
112 #define fchown(fd,uid,gid) EIO_ENOSYS ()
113 #define truncate(path,offs) EIO_ENOSYS () /* far-miss: SetEndOfFile */
114 #define ftruncate(fd,offs) EIO_ENOSYS () /* near-miss: SetEndOfFile */
115 #define mknod(path,mode,dev) EIO_ENOSYS ()
116 #define sync() EIO_ENOSYS ()
117
118 /* we could even stat and see if it exists */
119 static int
120 symlink (const char *old, const char *neu)
121 {
122 if (CreateSymbolicLink (neu, old, 1))
123 return 0;
124
125 if (CreateSymbolicLink (neu, old, 0))
126 return 0;
127
128 return EIO_ERRNO (ENOENT, -1);
129 }
130
84#else 131#else
85 132
86# include <sys/time.h> 133 #include <sys/time.h>
87# include <sys/select.h> 134 #include <sys/select.h>
135 #include <sys/statvfs.h>
88# include <unistd.h> 136 #include <unistd.h>
89# include <utime.h> 137 #include <utime.h>
90# include <signal.h> 138 #include <signal.h>
91# include <dirent.h> 139 #include <dirent.h>
92 140
93#if _POSIX_MEMLOCK || _POSIX_MEMLOCK_RANGE || _POSIX_MAPPED_FILES 141 #if _POSIX_MEMLOCK || _POSIX_MEMLOCK_RANGE || _POSIX_MAPPED_FILES
94# include <sys/mman.h> 142 #include <sys/mman.h>
95#endif 143 #endif
96 144
97/* POSIX_SOURCE is useless on bsd's, and XOPEN_SOURCE is unreliable there, too */ 145 /* POSIX_SOURCE is useless on bsd's, and XOPEN_SOURCE is unreliable there, too */
98# if __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__ 146 #if __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__
99# define _DIRENT_HAVE_D_TYPE /* sigh */ 147 #define _DIRENT_HAVE_D_TYPE /* sigh */
100# define D_INO(de) (de)->d_fileno 148 #define D_INO(de) (de)->d_fileno
101# define D_NAMLEN(de) (de)->d_namlen 149 #define D_NAMLEN(de) (de)->d_namlen
102# elif __linux || defined d_ino || _XOPEN_SOURCE >= 600 150 #elif __linux || defined d_ino || _XOPEN_SOURCE >= 600
103# define D_INO(de) (de)->d_ino 151 #define D_INO(de) (de)->d_ino
104# endif 152 #endif
105 153
106#ifdef _D_EXACT_NAMLEN 154 #ifdef _D_EXACT_NAMLEN
107# undef D_NAMLEN 155 #undef D_NAMLEN
108# define D_NAMLEN(de) _D_EXACT_NAMLEN (de) 156 #define D_NAMLEN(de) _D_EXACT_NAMLEN (de)
109#endif 157 #endif
110 158
111# ifdef _DIRENT_HAVE_D_TYPE 159 #ifdef _DIRENT_HAVE_D_TYPE
112# define D_TYPE(de) (de)->d_type 160 #define D_TYPE(de) (de)->d_type
113# endif 161 #endif
114 162
115# ifndef EIO_STRUCT_DIRENT 163 #ifndef EIO_STRUCT_DIRENT
116# define EIO_STRUCT_DIRENT struct dirent 164 #define EIO_STRUCT_DIRENT struct dirent
117# endif 165 #endif
118 166
119#endif 167#endif
120 168
121#if HAVE_SENDFILE 169#if HAVE_SENDFILE
122# if __linux 170# if __linux
165 if (!eio_buf) \ 213 if (!eio_buf) \
166 return -1; 214 return -1;
167 215
168#define EIO_TICKS ((1000000 + 1023) >> 10) 216#define EIO_TICKS ((1000000 + 1023) >> 10)
169 217
170/*****************************************************************************/
171
172#if __GNUC__ >= 3
173# define expect(expr,value) __builtin_expect ((expr),(value))
174#else
175# define expect(expr,value) (expr)
176#endif
177
178#define expect_false(expr) expect ((expr) != 0, 0)
179#define expect_true(expr) expect ((expr) != 0, 1)
180
181/*****************************************************************************/
182
183#define ETP_PRI_MIN EIO_PRI_MIN 218#define ETP_PRI_MIN EIO_PRI_MIN
184#define ETP_PRI_MAX EIO_PRI_MAX 219#define ETP_PRI_MAX EIO_PRI_MAX
185 220
186struct etp_worker; 221struct etp_worker;
187 222
212/*****************************************************************************/ 247/*****************************************************************************/
213 248
214#define ETP_NUM_PRI (ETP_PRI_MAX - ETP_PRI_MIN + 1) 249#define ETP_NUM_PRI (ETP_PRI_MAX - ETP_PRI_MIN + 1)
215 250
216/* calculate time difference in ~1/EIO_TICKS of a second */ 251/* calculate time difference in ~1/EIO_TICKS of a second */
252ecb_inline int
217static int tvdiff (struct timeval *tv1, struct timeval *tv2) 253tvdiff (struct timeval *tv1, struct timeval *tv2)
218{ 254{
219 return (tv2->tv_sec - tv1->tv_sec ) * EIO_TICKS 255 return (tv2->tv_sec - tv1->tv_sec ) * EIO_TICKS
220 + ((tv2->tv_usec - tv1->tv_usec) >> 10); 256 + ((tv2->tv_usec - tv1->tv_usec) >> 10);
221} 257}
222 258
266#define ETP_WORKER_LOCK(wrk) X_LOCK (wrklock) 302#define ETP_WORKER_LOCK(wrk) X_LOCK (wrklock)
267#define ETP_WORKER_UNLOCK(wrk) X_UNLOCK (wrklock) 303#define ETP_WORKER_UNLOCK(wrk) X_UNLOCK (wrklock)
268 304
269/* worker threads management */ 305/* worker threads management */
270 306
307static void ecb_cold
271static void etp_worker_clear (etp_worker *wrk) 308etp_worker_clear (etp_worker *wrk)
272{ 309{
273 ETP_WORKER_CLEAR (wrk); 310 ETP_WORKER_CLEAR (wrk);
274} 311}
275 312
313static void ecb_cold
276static void etp_worker_free (etp_worker *wrk) 314etp_worker_free (etp_worker *wrk)
277{ 315{
278 wrk->next->prev = wrk->prev; 316 wrk->next->prev = wrk->prev;
279 wrk->prev->next = wrk->next; 317 wrk->prev->next = wrk->next;
280 318
281 free (wrk); 319 free (wrk);
282} 320}
283 321
284static unsigned int etp_nreqs (void) 322static unsigned int
323etp_nreqs (void)
285{ 324{
286 int retval; 325 int retval;
287 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 326 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
288 retval = nreqs; 327 retval = nreqs;
289 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock); 328 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
290 return retval; 329 return retval;
291} 330}
292 331
293static unsigned int etp_nready (void) 332static unsigned int
333etp_nready (void)
294{ 334{
295 unsigned int retval; 335 unsigned int retval;
296 336
297 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 337 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
298 retval = nready; 338 retval = nready;
299 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock); 339 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
300 340
301 return retval; 341 return retval;
302} 342}
303 343
304static unsigned int etp_npending (void) 344static unsigned int
345etp_npending (void)
305{ 346{
306 unsigned int retval; 347 unsigned int retval;
307 348
308 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 349 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
309 retval = npending; 350 retval = npending;
310 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock); 351 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
311 352
312 return retval; 353 return retval;
313} 354}
314 355
315static unsigned int etp_nthreads (void) 356static unsigned int
357etp_nthreads (void)
316{ 358{
317 unsigned int retval; 359 unsigned int retval;
318 360
319 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 361 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
320 retval = started; 362 retval = started;
334} etp_reqq; 376} etp_reqq;
335 377
336static etp_reqq req_queue; 378static etp_reqq req_queue;
337static etp_reqq res_queue; 379static etp_reqq res_queue;
338 380
381static int ecb_noinline
339static int reqq_push (etp_reqq *q, ETP_REQ *req) 382reqq_push (etp_reqq *q, ETP_REQ *req)
340{ 383{
341 int pri = req->pri; 384 int pri = req->pri;
342 req->next = 0; 385 req->next = 0;
343 386
344 if (q->qe[pri]) 387 if (q->qe[pri])
350 q->qe[pri] = q->qs[pri] = req; 393 q->qe[pri] = q->qs[pri] = req;
351 394
352 return q->size++; 395 return q->size++;
353} 396}
354 397
398static ETP_REQ * ecb_noinline
355static ETP_REQ *reqq_shift (etp_reqq *q) 399reqq_shift (etp_reqq *q)
356{ 400{
357 int pri; 401 int pri;
358 402
359 if (!q->size) 403 if (!q->size)
360 return 0; 404 return 0;
375 } 419 }
376 420
377 abort (); 421 abort ();
378} 422}
379 423
424static void ecb_cold
380static void etp_thread_init (void) 425etp_thread_init (void)
381{ 426{
427#if !HAVE_PREADWRITE
428 X_MUTEX_CREATE (preadwritelock);
429#endif
382 X_MUTEX_CREATE (wrklock); 430 X_MUTEX_CREATE (wrklock);
383 X_MUTEX_CREATE (reslock); 431 X_MUTEX_CREATE (reslock);
384 X_MUTEX_CREATE (reqlock); 432 X_MUTEX_CREATE (reqlock);
385 X_COND_CREATE (reqwait); 433 X_COND_CREATE (reqwait);
386} 434}
387 435
436static void ecb_cold
388static void etp_atfork_prepare (void) 437etp_atfork_prepare (void)
389{ 438{
390 X_LOCK (wrklock);
391 X_LOCK (reqlock);
392 X_LOCK (reslock);
393#if !HAVE_PREADWRITE
394 X_LOCK (preadwritelock);
395#endif
396} 439}
397 440
441static void ecb_cold
398static void etp_atfork_parent (void) 442etp_atfork_parent (void)
399{ 443{
400#if !HAVE_PREADWRITE
401 X_UNLOCK (preadwritelock);
402#endif
403 X_UNLOCK (reslock);
404 X_UNLOCK (reqlock);
405 X_UNLOCK (wrklock);
406} 444}
407 445
446static void ecb_cold
408static void etp_atfork_child (void) 447etp_atfork_child (void)
409{ 448{
410 ETP_REQ *prv; 449 ETP_REQ *prv;
411 450
412 while ((prv = reqq_shift (&req_queue))) 451 while ((prv = reqq_shift (&req_queue)))
413 ETP_DESTROY (prv); 452 ETP_DESTROY (prv);
433 npending = 0; 472 npending = 0;
434 473
435 etp_thread_init (); 474 etp_thread_init ();
436} 475}
437 476
438static void 477static void ecb_cold
439etp_once_init (void) 478etp_once_init (void)
440{ 479{
441 etp_thread_init (); 480 etp_thread_init ();
442 X_THREAD_ATFORK (etp_atfork_prepare, etp_atfork_parent, etp_atfork_child); 481 X_THREAD_ATFORK (etp_atfork_prepare, etp_atfork_parent, etp_atfork_child);
443} 482}
444 483
445static int 484static int ecb_cold
446etp_init (void (*want_poll)(void), void (*done_poll)(void)) 485etp_init (void (*want_poll)(void), void (*done_poll)(void))
447{ 486{
448 static pthread_once_t doinit = PTHREAD_ONCE_INIT; 487 static pthread_once_t doinit = PTHREAD_ONCE_INIT;
449 488
450 pthread_once (&doinit, etp_once_init); 489 pthread_once (&doinit, etp_once_init);
455 return 0; 494 return 0;
456} 495}
457 496
458X_THREAD_PROC (etp_proc); 497X_THREAD_PROC (etp_proc);
459 498
499static void ecb_cold
460static void etp_start_thread (void) 500etp_start_thread (void)
461{ 501{
462 etp_worker *wrk = calloc (1, sizeof (etp_worker)); 502 etp_worker *wrk = calloc (1, sizeof (etp_worker));
463 503
464 /*TODO*/ 504 /*TODO*/
465 assert (("unable to allocate worker thread data", wrk)); 505 assert (("unable to allocate worker thread data", wrk));
478 free (wrk); 518 free (wrk);
479 519
480 X_UNLOCK (wrklock); 520 X_UNLOCK (wrklock);
481} 521}
482 522
523static void
483static void etp_maybe_start_thread (void) 524etp_maybe_start_thread (void)
484{ 525{
485 if (expect_true (etp_nthreads () >= wanted)) 526 if (ecb_expect_true (etp_nthreads () >= wanted))
486 return; 527 return;
487 528
488 /* todo: maybe use idle here, but might be less exact */ 529 /* todo: maybe use idle here, but might be less exact */
489 if (expect_true (0 <= (int)etp_nthreads () + (int)etp_npending () - (int)etp_nreqs ())) 530 if (ecb_expect_true (0 <= (int)etp_nthreads () + (int)etp_npending () - (int)etp_nreqs ()))
490 return; 531 return;
491 532
492 etp_start_thread (); 533 etp_start_thread ();
493} 534}
494 535
536static void ecb_cold
495static void etp_end_thread (void) 537etp_end_thread (void)
496{ 538{
497 eio_req *req = calloc (1, sizeof (eio_req)); 539 eio_req *req = calloc (1, sizeof (eio_req));
498 540
499 req->type = -1; 541 req->type = -1;
500 req->pri = ETP_PRI_MAX - ETP_PRI_MIN; 542 req->pri = ETP_PRI_MAX - ETP_PRI_MIN;
507 X_LOCK (wrklock); 549 X_LOCK (wrklock);
508 --started; 550 --started;
509 X_UNLOCK (wrklock); 551 X_UNLOCK (wrklock);
510} 552}
511 553
512static int etp_poll (void) 554static int
555etp_poll (void)
513{ 556{
514 unsigned int maxreqs; 557 unsigned int maxreqs;
515 unsigned int maxtime; 558 unsigned int maxtime;
516 struct timeval tv_start, tv_now; 559 struct timeval tv_start, tv_now;
517 560
547 590
548 X_LOCK (reqlock); 591 X_LOCK (reqlock);
549 --nreqs; 592 --nreqs;
550 X_UNLOCK (reqlock); 593 X_UNLOCK (reqlock);
551 594
552 if (expect_false (req->type == EIO_GROUP && req->size)) 595 if (ecb_expect_false (req->type == EIO_GROUP && req->size))
553 { 596 {
554 req->int1 = 1; /* mark request as delayed */ 597 req->int1 = 1; /* mark request as delayed */
555 continue; 598 continue;
556 } 599 }
557 else 600 else
558 { 601 {
559 int res = ETP_FINISH (req); 602 int res = ETP_FINISH (req);
560 if (expect_false (res)) 603 if (ecb_expect_false (res))
561 return res; 604 return res;
562 } 605 }
563 606
564 if (expect_false (maxreqs && !--maxreqs)) 607 if (ecb_expect_false (maxreqs && !--maxreqs))
565 break; 608 break;
566 609
567 if (maxtime) 610 if (maxtime)
568 { 611 {
569 gettimeofday (&tv_now, 0); 612 gettimeofday (&tv_now, 0);
575 618
576 errno = EAGAIN; 619 errno = EAGAIN;
577 return -1; 620 return -1;
578} 621}
579 622
623static void
580static void etp_cancel (ETP_REQ *req) 624etp_cancel (ETP_REQ *req)
581{ 625{
582 X_LOCK (wrklock); 626 req->cancelled = 1;
583 req->flags |= EIO_FLAG_CANCELLED;
584 X_UNLOCK (wrklock);
585 627
586 eio_grp_cancel (req); 628 eio_grp_cancel (req);
587} 629}
588 630
631static void
589static void etp_submit (ETP_REQ *req) 632etp_submit (ETP_REQ *req)
590{ 633{
591 req->pri -= ETP_PRI_MIN; 634 req->pri -= ETP_PRI_MIN;
592 635
593 if (expect_false (req->pri < ETP_PRI_MIN - ETP_PRI_MIN)) req->pri = ETP_PRI_MIN - ETP_PRI_MIN; 636 if (ecb_expect_false (req->pri < ETP_PRI_MIN - ETP_PRI_MIN)) req->pri = ETP_PRI_MIN - ETP_PRI_MIN;
594 if (expect_false (req->pri > ETP_PRI_MAX - ETP_PRI_MIN)) req->pri = ETP_PRI_MAX - ETP_PRI_MIN; 637 if (ecb_expect_false (req->pri > ETP_PRI_MAX - ETP_PRI_MIN)) req->pri = ETP_PRI_MAX - ETP_PRI_MIN;
595 638
596 if (expect_false (req->type == EIO_GROUP)) 639 if (ecb_expect_false (req->type == EIO_GROUP))
597 { 640 {
598 /* I hope this is worth it :/ */ 641 /* I hope this is worth it :/ */
599 X_LOCK (reqlock); 642 X_LOCK (reqlock);
600 ++nreqs; 643 ++nreqs;
601 X_UNLOCK (reqlock); 644 X_UNLOCK (reqlock);
620 663
621 etp_maybe_start_thread (); 664 etp_maybe_start_thread ();
622 } 665 }
623} 666}
624 667
668static void ecb_cold
625static void etp_set_max_poll_time (double nseconds) 669etp_set_max_poll_time (double nseconds)
626{ 670{
627 if (WORDACCESS_UNSAFE) X_LOCK (reslock); 671 if (WORDACCESS_UNSAFE) X_LOCK (reslock);
628 max_poll_time = nseconds * EIO_TICKS; 672 max_poll_time = nseconds * EIO_TICKS;
629 if (WORDACCESS_UNSAFE) X_UNLOCK (reslock); 673 if (WORDACCESS_UNSAFE) X_UNLOCK (reslock);
630} 674}
631 675
676static void ecb_cold
632static void etp_set_max_poll_reqs (unsigned int maxreqs) 677etp_set_max_poll_reqs (unsigned int maxreqs)
633{ 678{
634 if (WORDACCESS_UNSAFE) X_LOCK (reslock); 679 if (WORDACCESS_UNSAFE) X_LOCK (reslock);
635 max_poll_reqs = maxreqs; 680 max_poll_reqs = maxreqs;
636 if (WORDACCESS_UNSAFE) X_UNLOCK (reslock); 681 if (WORDACCESS_UNSAFE) X_UNLOCK (reslock);
637} 682}
638 683
684static void ecb_cold
639static void etp_set_max_idle (unsigned int nthreads) 685etp_set_max_idle (unsigned int nthreads)
640{ 686{
641 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 687 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
642 max_idle = nthreads; 688 max_idle = nthreads;
643 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock); 689 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
644} 690}
645 691
692static void ecb_cold
646static void etp_set_idle_timeout (unsigned int seconds) 693etp_set_idle_timeout (unsigned int seconds)
647{ 694{
648 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 695 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
649 idle_timeout = seconds; 696 idle_timeout = seconds;
650 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock); 697 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
651} 698}
652 699
700static void ecb_cold
653static void etp_set_min_parallel (unsigned int nthreads) 701etp_set_min_parallel (unsigned int nthreads)
654{ 702{
655 if (wanted < nthreads) 703 if (wanted < nthreads)
656 wanted = nthreads; 704 wanted = nthreads;
657} 705}
658 706
707static void ecb_cold
659static void etp_set_max_parallel (unsigned int nthreads) 708etp_set_max_parallel (unsigned int nthreads)
660{ 709{
661 if (wanted > nthreads) 710 if (wanted > nthreads)
662 wanted = nthreads; 711 wanted = nthreads;
663 712
664 while (started > wanted) 713 while (started > wanted)
665 etp_end_thread (); 714 etp_end_thread ();
666} 715}
667 716
668/*****************************************************************************/ 717/*****************************************************************************/
669 718
719static void
670static void grp_try_feed (eio_req *grp) 720grp_try_feed (eio_req *grp)
671{ 721{
672 while (grp->size < grp->int2 && !EIO_CANCELLED (grp)) 722 while (grp->size < grp->int2 && !EIO_CANCELLED (grp))
673 { 723 {
674 grp->flags &= ~EIO_FLAG_GROUPADD; 724 grp->flags &= ~EIO_FLAG_GROUPADD;
675 725
682 break; 732 break;
683 } 733 }
684 } 734 }
685} 735}
686 736
737static int
687static int grp_dec (eio_req *grp) 738grp_dec (eio_req *grp)
688{ 739{
689 --grp->size; 740 --grp->size;
690 741
691 /* call feeder, if applicable */ 742 /* call feeder, if applicable */
692 grp_try_feed (grp); 743 grp_try_feed (grp);
696 return eio_finish (grp); 747 return eio_finish (grp);
697 else 748 else
698 return 0; 749 return 0;
699} 750}
700 751
752static void
701void eio_destroy (eio_req *req) 753eio_destroy (eio_req *req)
702{ 754{
703 if ((req)->flags & EIO_FLAG_PTR1_FREE) free (req->ptr1); 755 if ((req)->flags & EIO_FLAG_PTR1_FREE) free (req->ptr1);
704 if ((req)->flags & EIO_FLAG_PTR2_FREE) free (req->ptr2); 756 if ((req)->flags & EIO_FLAG_PTR2_FREE) free (req->ptr2);
705 757
706 EIO_DESTROY (req); 758 EIO_DESTROY (req);
707} 759}
708 760
761static int
709static int eio_finish (eio_req *req) 762eio_finish (eio_req *req)
710{ 763{
711 int res = EIO_FINISH (req); 764 int res = EIO_FINISH (req);
712 765
713 if (req->grp) 766 if (req->grp)
714 { 767 {
722 if (grp->grp_first == req) 775 if (grp->grp_first == req)
723 grp->grp_first = req->grp_next; 776 grp->grp_first = req->grp_next;
724 777
725 res2 = grp_dec (grp); 778 res2 = grp_dec (grp);
726 779
727 if (!res && res2) 780 if (!res)
728 res = res2; 781 res = res2;
729 } 782 }
730 783
731 eio_destroy (req); 784 eio_destroy (req);
732 785
733 return res; 786 return res;
734} 787}
735 788
789void
736void eio_grp_cancel (eio_req *grp) 790eio_grp_cancel (eio_req *grp)
737{ 791{
738 for (grp = grp->grp_first; grp; grp = grp->grp_next) 792 for (grp = grp->grp_first; grp; grp = grp->grp_next)
739 eio_cancel (grp); 793 eio_cancel (grp);
740} 794}
741 795
796void
742void eio_cancel (eio_req *req) 797eio_cancel (eio_req *req)
743{ 798{
744 etp_cancel (req); 799 etp_cancel (req);
745} 800}
746 801
802void
747void eio_submit (eio_req *req) 803eio_submit (eio_req *req)
748{ 804{
749 etp_submit (req); 805 etp_submit (req);
750} 806}
751 807
752unsigned int eio_nreqs (void) 808unsigned int
809eio_nreqs (void)
753{ 810{
754 return etp_nreqs (); 811 return etp_nreqs ();
755} 812}
756 813
757unsigned int eio_nready (void) 814unsigned int
815eio_nready (void)
758{ 816{
759 return etp_nready (); 817 return etp_nready ();
760} 818}
761 819
762unsigned int eio_npending (void) 820unsigned int
821eio_npending (void)
763{ 822{
764 return etp_npending (); 823 return etp_npending ();
765} 824}
766 825
767unsigned int eio_nthreads (void) 826unsigned int ecb_cold
827eio_nthreads (void)
768{ 828{
769 return etp_nthreads (); 829 return etp_nthreads ();
770} 830}
771 831
832void ecb_cold
772void eio_set_max_poll_time (double nseconds) 833eio_set_max_poll_time (double nseconds)
773{ 834{
774 etp_set_max_poll_time (nseconds); 835 etp_set_max_poll_time (nseconds);
775} 836}
776 837
838void ecb_cold
777void eio_set_max_poll_reqs (unsigned int maxreqs) 839eio_set_max_poll_reqs (unsigned int maxreqs)
778{ 840{
779 etp_set_max_poll_reqs (maxreqs); 841 etp_set_max_poll_reqs (maxreqs);
780} 842}
781 843
844void ecb_cold
782void eio_set_max_idle (unsigned int nthreads) 845eio_set_max_idle (unsigned int nthreads)
783{ 846{
784 etp_set_max_idle (nthreads); 847 etp_set_max_idle (nthreads);
785} 848}
786 849
850void ecb_cold
787void eio_set_idle_timeout (unsigned int seconds) 851eio_set_idle_timeout (unsigned int seconds)
788{ 852{
789 etp_set_idle_timeout (seconds); 853 etp_set_idle_timeout (seconds);
790} 854}
791 855
856void ecb_cold
792void eio_set_min_parallel (unsigned int nthreads) 857eio_set_min_parallel (unsigned int nthreads)
793{ 858{
794 etp_set_min_parallel (nthreads); 859 etp_set_min_parallel (nthreads);
795} 860}
796 861
862void ecb_cold
797void eio_set_max_parallel (unsigned int nthreads) 863eio_set_max_parallel (unsigned int nthreads)
798{ 864{
799 etp_set_max_parallel (nthreads); 865 etp_set_max_parallel (nthreads);
800} 866}
801 867
802int eio_poll (void) 868int eio_poll (void)
811# undef pread 877# undef pread
812# undef pwrite 878# undef pwrite
813# define pread eio__pread 879# define pread eio__pread
814# define pwrite eio__pwrite 880# define pwrite eio__pwrite
815 881
816static ssize_t 882static eio_ssize_t
817eio__pread (int fd, void *buf, size_t count, off_t offset) 883eio__pread (int fd, void *buf, size_t count, off_t offset)
818{ 884{
819 ssize_t res; 885 eio_ssize_t res;
820 off_t ooffset; 886 off_t ooffset;
821 887
822 X_LOCK (preadwritelock); 888 X_LOCK (preadwritelock);
823 ooffset = lseek (fd, 0, SEEK_CUR); 889 ooffset = lseek (fd, 0, SEEK_CUR);
824 lseek (fd, offset, SEEK_SET); 890 lseek (fd, offset, SEEK_SET);
827 X_UNLOCK (preadwritelock); 893 X_UNLOCK (preadwritelock);
828 894
829 return res; 895 return res;
830} 896}
831 897
832static ssize_t 898static eio_ssize_t
833eio__pwrite (int fd, void *buf, size_t count, off_t offset) 899eio__pwrite (int fd, void *buf, size_t count, off_t offset)
834{ 900{
835 ssize_t res; 901 eio_ssize_t res;
836 off_t ooffset; 902 off_t ooffset;
837 903
838 X_LOCK (preadwritelock); 904 X_LOCK (preadwritelock);
839 ooffset = lseek (fd, 0, SEEK_CUR); 905 ooffset = lseek (fd, 0, SEEK_CUR);
840 lseek (fd, offset, SEEK_SET); 906 lseek (fd, offset, SEEK_SET);
872#ifndef HAVE_FUTIMES 938#ifndef HAVE_FUTIMES
873 939
874# undef futimes 940# undef futimes
875# define futimes(fd,times) eio__futimes (fd, times) 941# define futimes(fd,times) eio__futimes (fd, times)
876 942
943static int
877static int eio__futimes (int fd, const struct timeval tv[2]) 944eio__futimes (int fd, const struct timeval tv[2])
878{ 945{
879 errno = ENOSYS; 946 errno = ENOSYS;
880 return -1; 947 return -1;
881} 948}
882 949
886# undef fdatasync 953# undef fdatasync
887# define fdatasync(fd) fsync (fd) 954# define fdatasync(fd) fsync (fd)
888#endif 955#endif
889 956
890/* sync_file_range always needs emulation */ 957/* sync_file_range always needs emulation */
891int 958static int
892eio__sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags) 959eio__sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags)
893{ 960{
894#if HAVE_SYNC_FILE_RANGE 961#if HAVE_SYNC_FILE_RANGE
895 int res; 962 int res;
896 963
913 /* even though we could play tricks with the flags, it's better to always 980 /* even though we could play tricks with the flags, it's better to always
914 * call fdatasync, as that matches the expectation of its users best */ 981 * call fdatasync, as that matches the expectation of its users best */
915 return fdatasync (fd); 982 return fdatasync (fd);
916} 983}
917 984
985static int
986eio__fallocate (int fd, int mode, off_t offset, size_t len)
987{
988#if HAVE_FALLOCATE
989 return fallocate (fd, mode, offset, len);
990#else
991 errno = ENOSYS;
992 return -1;
993#endif
994}
995
918#if !HAVE_READAHEAD 996#if !HAVE_READAHEAD
919# undef readahead 997# undef readahead
920# define readahead(fd,offset,count) eio__readahead (fd, offset, count, self) 998# define readahead(fd,offset,count) eio__readahead (fd, offset, count, self)
921 999
922static ssize_t 1000static eio_ssize_t
923eio__readahead (int fd, off_t offset, size_t count, etp_worker *self) 1001eio__readahead (int fd, off_t offset, size_t count, etp_worker *self)
924{ 1002{
925 size_t todo = count; 1003 size_t todo = count;
926 dBUF; 1004 dBUF;
927 1005
939} 1017}
940 1018
941#endif 1019#endif
942 1020
943/* sendfile always needs emulation */ 1021/* sendfile always needs emulation */
944static ssize_t 1022static eio_ssize_t
945eio__sendfile (int ofd, int ifd, off_t offset, size_t count, etp_worker *self) 1023eio__sendfile (int ofd, int ifd, off_t offset, size_t count, etp_worker *self)
946{ 1024{
1025 eio_ssize_t written = 0;
947 ssize_t res; 1026 eio_ssize_t res;
948 1027
949 if (!count) 1028 if (!count)
950 return 0; 1029 return 0;
951 1030
1031 for (;;)
1032 {
1033#ifdef __APPLE__
1034# undef HAVE_SENDFILE /* broken, as everything on os x */
1035#endif
952#if HAVE_SENDFILE 1036#if HAVE_SENDFILE
953# if __linux 1037# if __linux
1038 off_t soffset = offset;
954 res = sendfile (ofd, ifd, &offset, count); 1039 res = sendfile (ofd, ifd, &soffset, count);
955 1040
956# elif __FreeBSD__ 1041# elif __FreeBSD__
957 /* 1042 /*
958 * Of course, the freebsd sendfile is a dire hack with no thoughts 1043 * Of course, the freebsd sendfile is a dire hack with no thoughts
959 * wasted on making it similar to other I/O functions. 1044 * wasted on making it similar to other I/O functions.
960 */ 1045 */
961 {
962 off_t sbytes; 1046 off_t sbytes;
963 res = sendfile (ifd, ofd, offset, count, 0, &sbytes, 0); 1047 res = sendfile (ifd, ofd, offset, count, 0, &sbytes, 0);
964 1048
965 #if 0 /* according to the manpage, this is correct, but broken behaviour */ 1049 #if 0 /* according to the manpage, this is correct, but broken behaviour */
966 /* freebsd' sendfile will return 0 on success */ 1050 /* freebsd' sendfile will return 0 on success */
967 /* freebsd 8 documents it as only setting *sbytes on EINTR and EAGAIN, but */ 1051 /* freebsd 8 documents it as only setting *sbytes on EINTR and EAGAIN, but */
968 /* not on e.g. EIO or EPIPE - sounds broken */ 1052 /* not on e.g. EIO or EPIPE - sounds broken */
969 if ((res < 0 && (errno == EAGAIN || errno == EINTR) && sbytes) || res == 0) 1053 if ((res < 0 && (errno == EAGAIN || errno == EINTR) && sbytes) || res == 0)
970 res = sbytes; 1054 res = sbytes;
971 #endif 1055 #endif
972 1056
973 /* according to source inspection, this is correct, and useful behaviour */ 1057 /* according to source inspection, this is correct, and useful behaviour */
974 if (sbytes) 1058 if (sbytes)
975 res = sbytes; 1059 res = sbytes;
976 }
977 1060
978# elif defined (__APPLE__) 1061# elif defined (__APPLE__)
979
980 {
981 off_t sbytes = count; 1062 off_t sbytes = count;
982 res = sendfile (ifd, ofd, offset, &sbytes, 0, 0); 1063 res = sendfile (ifd, ofd, offset, &sbytes, 0, 0);
983 1064
984 /* according to the manpage, sbytes is always valid */ 1065 /* according to the manpage, sbytes is always valid */
985 if (sbytes) 1066 if (sbytes)
986 res = sbytes; 1067 res = sbytes;
987 }
988 1068
989# elif __hpux 1069# elif __hpux
990 res = sendfile (ofd, ifd, offset, count, 0, 0); 1070 res = sendfile (ofd, ifd, offset, count, 0, 0);
991 1071
992# elif __solaris 1072# elif __solaris
993 {
994 struct sendfilevec vec; 1073 struct sendfilevec vec;
995 size_t sbytes; 1074 size_t sbytes;
996 1075
997 vec.sfv_fd = ifd; 1076 vec.sfv_fd = ifd;
998 vec.sfv_flag = 0; 1077 vec.sfv_flag = 0;
999 vec.sfv_off = offset; 1078 vec.sfv_off = offset;
1000 vec.sfv_len = count; 1079 vec.sfv_len = count;
1001 1080
1002 res = sendfilev (ofd, &vec, 1, &sbytes); 1081 res = sendfilev (ofd, &vec, 1, &sbytes);
1003 1082
1004 if (res < 0 && sbytes) 1083 if (res < 0 && sbytes)
1005 res = sbytes; 1084 res = sbytes;
1006 }
1007 1085
1008# endif 1086# endif
1009 1087
1010#elif defined (_WIN32) 1088#elif defined (_WIN32) && 0
1011
1012 /* does not work, just for documentation of what would need to be done */ 1089 /* does not work, just for documentation of what would need to be done */
1013 { 1090 /* actually, cannot be done like this, as TransmitFile changes the file offset, */
1091 /* libeio guarantees that the file offset does not change, and windows */
1092 /* has no way to get an independent handle to the same file description */
1014 HANDLE h = TO_SOCKET (ifd); 1093 HANDLE h = TO_SOCKET (ifd);
1015 SetFilePointer (h, offset, 0, FILE_BEGIN); 1094 SetFilePointer (h, offset, 0, FILE_BEGIN);
1016 res = TransmitFile (TO_SOCKET (ofd), h, count, 0, 0, 0, 0); 1095 res = TransmitFile (TO_SOCKET (ofd), h, count, 0, 0, 0, 0);
1017 }
1018 1096
1019#else 1097#else
1020 res = -1; 1098 res = -1;
1021 errno = ENOSYS; 1099 errno = ENOSYS;
1022#endif 1100#endif
1023 1101
1102 /* we assume sendfile can copy at least 128mb in one go */
1103 if (res <= 128 * 1024 * 1024)
1104 {
1105 if (res > 0)
1106 written += res;
1107
1108 if (written)
1109 return written;
1110
1111 break;
1112 }
1113 else
1114 {
1115 /* if we requested more, then probably the kernel was lazy */
1116 written += res;
1117 offset += res;
1118 count -= res;
1119
1120 if (!count)
1121 return written;
1122 }
1123 }
1124
1024 if (res < 0 1125 if (res < 0
1025 && (errno == ENOSYS || errno == EINVAL || errno == ENOTSOCK 1126 && (errno == ENOSYS || errno == EINVAL || errno == ENOTSOCK
1026 /* BSDs */ 1127 /* BSDs */
1027#ifdef ENOTSUP /* sigh, if the steenking pile called openbsd would only try to at least compile posix code... */ 1128#ifdef ENOTSUP /* sigh, if the steenking pile called openbsd would only try to at least compile posix code... */
1028 || errno == ENOTSUP 1129 || errno == ENOTSUP
1029#endif 1130#endif
1039 1140
1040 res = 0; 1141 res = 0;
1041 1142
1042 while (count) 1143 while (count)
1043 { 1144 {
1044 ssize_t cnt; 1145 eio_ssize_t cnt;
1045 1146
1046 cnt = pread (ifd, eio_buf, count > EIO_BUFSIZE ? EIO_BUFSIZE : count, offset); 1147 cnt = pread (ifd, eio_buf, count > EIO_BUFSIZE ? EIO_BUFSIZE : count, offset);
1047 1148
1048 if (cnt <= 0) 1149 if (cnt <= 0)
1049 { 1150 {
1064 count -= cnt; 1165 count -= cnt;
1065 } 1166 }
1066 } 1167 }
1067 1168
1068 return res; 1169 return res;
1170}
1171
1172#ifdef PAGESIZE
1173# define eio_pagesize() PAGESIZE
1174#else
1175static intptr_t
1176eio_pagesize (void)
1177{
1178 static intptr_t page;
1179
1180 if (!page)
1181 page = sysconf (_SC_PAGESIZE);
1182
1183 return page;
1184}
1185#endif
1186
1187static void
1188eio_page_align (void **addr, size_t *length)
1189{
1190 intptr_t mask = eio_pagesize () - 1;
1191
1192 /* round down addr */
1193 intptr_t adj = mask & (intptr_t)*addr;
1194
1195 *addr = (void *)((intptr_t)*addr - adj);
1196 *length += adj;
1197
1198 /* round up length */
1199 *length = (*length + mask) & ~mask;
1200}
1201
1202#if !_POSIX_MEMLOCK
1203# define eio__mlockall(a) EIO_ENOSYS ()
1204#else
1205
1206static int
1207eio__mlockall (int flags)
1208{
1209 #if __GLIBC__ == 2 && __GLIBC_MINOR__ <= 7
1210 extern int mallopt (int, int);
1211 mallopt (-6, 238); /* http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=473812 */
1212 #endif
1213
1214 if (EIO_MCL_CURRENT != MCL_CURRENT
1215 || EIO_MCL_FUTURE != MCL_FUTURE)
1216 {
1217 flags = 0
1218 | (flags & EIO_MCL_CURRENT ? MCL_CURRENT : 0)
1219 | (flags & EIO_MCL_FUTURE ? MCL_FUTURE : 0);
1220 }
1221
1222 return mlockall (flags);
1223}
1224#endif
1225
1226#if !_POSIX_MEMLOCK_RANGE
1227# define eio__mlock(a,b) EIO_ENOSYS ()
1228#else
1229
1230static int
1231eio__mlock (void *addr, size_t length)
1232{
1233 eio_page_align (&addr, &length);
1234
1235 return mlock (addr, length);
1236}
1237
1238#endif
1239
1240#if !(_POSIX_MAPPED_FILES && _POSIX_SYNCHRONIZED_IO)
1241# define eio__msync(a,b,c) EIO_ENOSYS ()
1242#else
1243
1244static int
1245eio__msync (void *mem, size_t len, int flags)
1246{
1247 eio_page_align (&mem, &len);
1248
1249 if (EIO_MS_ASYNC != MS_SYNC
1250 || EIO_MS_INVALIDATE != MS_INVALIDATE
1251 || EIO_MS_SYNC != MS_SYNC)
1252 {
1253 flags = 0
1254 | (flags & EIO_MS_ASYNC ? MS_ASYNC : 0)
1255 | (flags & EIO_MS_INVALIDATE ? MS_INVALIDATE : 0)
1256 | (flags & EIO_MS_SYNC ? MS_SYNC : 0);
1257 }
1258
1259 return msync (mem, len, flags);
1260}
1261
1262#endif
1263
1264static int
1265eio__mtouch (eio_req *req)
1266{
1267 void *mem = req->ptr2;
1268 size_t len = req->size;
1269 int flags = req->int1;
1270
1271 eio_page_align (&mem, &len);
1272
1273 {
1274 intptr_t addr = (intptr_t)mem;
1275 intptr_t end = addr + len;
1276 intptr_t page = eio_pagesize ();
1277
1278 if (addr < end)
1279 if (flags & EIO_MT_MODIFY) /* modify */
1280 do { *((volatile sig_atomic_t *)addr) |= 0; } while ((addr += page) < len && !EIO_CANCELLED (req));
1281 else
1282 do { *((volatile sig_atomic_t *)addr) ; } while ((addr += page) < len && !EIO_CANCELLED (req));
1283 }
1284
1285 return 0;
1286}
1287
1288/*****************************************************************************/
1289/* requests implemented outside eio_execute, because they are so large */
1290
1291static void
1292eio__realpath (eio_req *req, etp_worker *self)
1293{
1294 char *rel = req->ptr1;
1295 char *res;
1296 char *tmp1, *tmp2;
1297#if SYMLOOP_MAX > 32
1298 int symlinks = SYMLOOP_MAX;
1299#else
1300 int symlinks = 32;
1301#endif
1302
1303 req->result = -1;
1304
1305 errno = EINVAL;
1306 if (!rel)
1307 return;
1308
1309 errno = ENOENT;
1310 if (!*rel)
1311 return;
1312
1313 if (!req->ptr2)
1314 {
1315 X_LOCK (wrklock);
1316 req->flags |= EIO_FLAG_PTR2_FREE;
1317 X_UNLOCK (wrklock);
1318 req->ptr2 = malloc (PATH_MAX * 3);
1319
1320 errno = ENOMEM;
1321 if (!req->ptr2)
1322 return;
1323 }
1324
1325 res = req->ptr2;
1326 tmp1 = res + PATH_MAX;
1327 tmp2 = tmp1 + PATH_MAX;
1328
1329#if 0 /* disabled, the musl way to do things is just too racy */
1330#if __linux && defined(O_NONBLOCK) && defined(O_NOATIME)
1331 /* on linux we may be able to ask the kernel */
1332 {
1333 int fd = open (rel, O_RDONLY | O_NONBLOCK | O_NOCTTY | O_NOATIME);
1334
1335 if (fd >= 0)
1336 {
1337 sprintf (tmp1, "/proc/self/fd/%d", fd);
1338 req->result = readlink (tmp1, res, PATH_MAX);
1339 close (fd);
1340
1341 /* here we should probably stat the open file and the disk file, to make sure they still match */
1342
1343 if (req->result > 0)
1344 goto done;
1345 }
1346 else if (errno == ELOOP || errno == ENAMETOOLONG || errno == ENOENT || errno == ENOTDIR || errno == EIO)
1347 return;
1348 }
1349#endif
1350#endif
1351
1352 if (*rel != '/')
1353 {
1354 if (!getcwd (res, PATH_MAX))
1355 return;
1356
1357 if (res [1]) /* only use if not / */
1358 res += strlen (res);
1359 }
1360
1361 while (*rel)
1362 {
1363 eio_ssize_t len, linklen;
1364 char *beg = rel;
1365
1366 while (*rel && *rel != '/')
1367 ++rel;
1368
1369 len = rel - beg;
1370
1371 if (!len) /* skip slashes */
1372 {
1373 ++rel;
1374 continue;
1375 }
1376
1377 if (beg [0] == '.')
1378 {
1379 if (len == 1)
1380 continue; /* . - nop */
1381
1382 if (beg [1] == '.' && len == 2)
1383 {
1384 /* .. - back up one component, if possible */
1385
1386 while (res != req->ptr2)
1387 if (*--res == '/')
1388 break;
1389
1390 continue;
1391 }
1392 }
1393
1394 errno = ENAMETOOLONG;
1395 if (res + 1 + len + 1 >= tmp1)
1396 return;
1397
1398 /* copy one component */
1399 *res = '/';
1400 memcpy (res + 1, beg, len);
1401
1402 /* zero-terminate, for readlink */
1403 res [len + 1] = 0;
1404
1405 /* now check if it's a symlink */
1406 linklen = readlink (req->ptr2, tmp1, PATH_MAX);
1407
1408 if (linklen < 0)
1409 {
1410 if (errno != EINVAL)
1411 return;
1412
1413 /* it's a normal directory. hopefully */
1414 res += len + 1;
1415 }
1416 else
1417 {
1418 /* yay, it was a symlink - build new path in tmp2 */
1419 int rellen = strlen (rel);
1420
1421 errno = ENAMETOOLONG;
1422 if (linklen + 1 + rellen >= PATH_MAX)
1423 return;
1424
1425 errno = ELOOP;
1426 if (!--symlinks)
1427 return;
1428
1429 if (*tmp1 == '/')
1430 res = req->ptr2; /* symlink resolves to an absolute path */
1431
1432 /* we need to be careful, as rel might point into tmp2 already */
1433 memmove (tmp2 + linklen + 1, rel, rellen + 1);
1434 tmp2 [linklen] = '/';
1435 memcpy (tmp2, tmp1, linklen);
1436
1437 rel = tmp2;
1438 }
1439 }
1440
1441 /* special case for the lone root path */
1442 if (res == req->ptr2)
1443 *res++ = '/';
1444
1445 req->result = res - (char *)req->ptr2;
1446
1447done:
1448 req->ptr2 = realloc (req->ptr2, req->result); /* trade time for space savings */
1069} 1449}
1070 1450
1071static signed char 1451static signed char
1072eio_dent_cmp (const eio_dirent *a, const eio_dirent *b) 1452eio_dent_cmp (const eio_dirent *a, const eio_dirent *b)
1073{ 1453{
1255 1635
1256 X_LOCK (wrklock); 1636 X_LOCK (wrklock);
1257 /* the corresponding closedir is in ETP_WORKER_CLEAR */ 1637 /* the corresponding closedir is in ETP_WORKER_CLEAR */
1258 self->dirp = dirp = opendir (req->ptr1); 1638 self->dirp = dirp = opendir (req->ptr1);
1259 1639
1640 if (req->flags & EIO_FLAG_PTR1_FREE)
1641 free (req->ptr1);
1642
1260 req->flags |= EIO_FLAG_PTR1_FREE | EIO_FLAG_PTR2_FREE; 1643 req->flags |= EIO_FLAG_PTR1_FREE | EIO_FLAG_PTR2_FREE;
1261 req->ptr1 = dents = flags ? malloc (dentalloc * sizeof (eio_dirent)) : 0; 1644 req->ptr1 = dents = flags ? malloc (dentalloc * sizeof (eio_dirent)) : 0;
1262 req->ptr2 = names = malloc (namesalloc); 1645 req->ptr2 = names = malloc (namesalloc);
1263 X_UNLOCK (wrklock); 1646 X_UNLOCK (wrklock);
1264 1647
1315 /* skip . and .. entries */ 1698 /* skip . and .. entries */
1316 if (name [0] != '.' || (name [1] && (name [1] != '.' || name [2]))) 1699 if (name [0] != '.' || (name [1] && (name [1] != '.' || name [2])))
1317 { 1700 {
1318 int len = D_NAMLEN (entp) + 1; 1701 int len = D_NAMLEN (entp) + 1;
1319 1702
1320 while (expect_false (namesoffs + len > namesalloc)) 1703 while (ecb_expect_false (namesoffs + len > namesalloc))
1321 { 1704 {
1322 namesalloc *= 2; 1705 namesalloc *= 2;
1323 X_LOCK (wrklock); 1706 X_LOCK (wrklock);
1324 req->ptr2 = names = realloc (names, namesalloc); 1707 req->ptr2 = names = realloc (names, namesalloc);
1325 X_UNLOCK (wrklock); 1708 X_UNLOCK (wrklock);
1332 1715
1333 if (dents) 1716 if (dents)
1334 { 1717 {
1335 struct eio_dirent *ent; 1718 struct eio_dirent *ent;
1336 1719
1337 if (expect_false (dentoffs == dentalloc)) 1720 if (ecb_expect_false (dentoffs == dentalloc))
1338 { 1721 {
1339 dentalloc *= 2; 1722 dentalloc *= 2;
1340 X_LOCK (wrklock); 1723 X_LOCK (wrklock);
1341 req->ptr1 = dents = realloc (dents, dentalloc * sizeof (eio_dirent)); 1724 req->ptr1 = dents = realloc (dents, dentalloc * sizeof (eio_dirent));
1342 X_UNLOCK (wrklock); 1725 X_UNLOCK (wrklock);
1430 break; 1813 break;
1431 } 1814 }
1432 } 1815 }
1433} 1816}
1434 1817
1435#ifdef PAGESIZE
1436# define eio_pagesize() PAGESIZE
1437#else
1438static intptr_t
1439eio_pagesize (void)
1440{
1441 static intptr_t page;
1442
1443 if (!page)
1444 page = sysconf (_SC_PAGESIZE);
1445
1446 return page;
1447}
1448#endif
1449
1450static void
1451eio_page_align (void **addr, size_t *length)
1452{
1453 intptr_t mask = eio_pagesize () - 1;
1454
1455 /* round down addr */
1456 intptr_t adj = mask & (intptr_t)*addr;
1457
1458 *addr = (void *)((intptr_t)*addr - adj);
1459 *length += adj;
1460
1461 /* round up length */
1462 *length = (*length + mask) & ~mask;
1463}
1464
1465#if !_POSIX_MEMLOCK
1466# define eio__mlockall(a) ((errno = ENOSYS), -1)
1467#else
1468
1469static int
1470eio__mlockall (int flags)
1471{
1472 #if __GLIBC__ == 2 && __GLIBC_MINOR__ <= 7
1473 extern int mallopt (int, int);
1474 mallopt (-6, 238); /* http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=473812 */
1475 #endif
1476
1477 if (EIO_MCL_CURRENT != MCL_CURRENT
1478 || EIO_MCL_FUTURE != MCL_FUTURE)
1479 {
1480 flags = 0
1481 | (flags & EIO_MCL_CURRENT ? MCL_CURRENT : 0)
1482 | (flags & EIO_MCL_FUTURE ? MCL_FUTURE : 0);
1483 }
1484
1485 return mlockall (flags);
1486}
1487#endif
1488
1489#if !_POSIX_MEMLOCK_RANGE
1490# define eio__mlock(a,b) ((errno = ENOSYS), -1)
1491#else
1492
1493static int
1494eio__mlock (void *addr, size_t length)
1495{
1496 eio_page_align (&addr, &length);
1497
1498 return mlock (addr, length);
1499}
1500
1501#endif
1502
1503#if !(_POSIX_MAPPED_FILES && _POSIX_SYNCHRONIZED_IO)
1504# define eio__msync(a,b,c) ((errno = ENOSYS), -1)
1505#else
1506
1507int
1508eio__msync (void *mem, size_t len, int flags)
1509{
1510 eio_page_align (&mem, &len);
1511
1512 if (EIO_MS_ASYNC != MS_SYNC
1513 || EIO_MS_INVALIDATE != MS_INVALIDATE
1514 || EIO_MS_SYNC != MS_SYNC)
1515 {
1516 flags = 0
1517 | (flags & EIO_MS_ASYNC ? MS_ASYNC : 0)
1518 | (flags & EIO_MS_INVALIDATE ? MS_INVALIDATE : 0)
1519 | (flags & EIO_MS_SYNC ? MS_SYNC : 0);
1520 }
1521
1522 return msync (mem, len, flags);
1523}
1524
1525#endif
1526
1527int
1528eio__mtouch (eio_req *req)
1529{
1530 void *mem = req->ptr2;
1531 size_t len = req->size;
1532 int flags = req->int1;
1533
1534 eio_page_align (&mem, &len);
1535
1536 {
1537 intptr_t addr = (intptr_t)mem;
1538 intptr_t end = addr + len;
1539 intptr_t page = eio_pagesize ();
1540
1541 if (addr < end)
1542 if (flags & EIO_MT_MODIFY) /* modify */
1543 do { *((volatile sig_atomic_t *)addr) |= 0; } while ((addr += page) < len && !EIO_CANCELLED (req));
1544 else
1545 do { *((volatile sig_atomic_t *)addr) ; } while ((addr += page) < len && !EIO_CANCELLED (req));
1546 }
1547
1548 return 0;
1549}
1550
1551/*****************************************************************************/ 1818/*****************************************************************************/
1552 1819
1553#define ALLOC(len) \ 1820#define ALLOC(len) \
1554 if (!req->ptr2) \ 1821 if (!req->ptr2) \
1555 { \ 1822 { \
1568X_THREAD_PROC (etp_proc) 1835X_THREAD_PROC (etp_proc)
1569{ 1836{
1570 ETP_REQ *req; 1837 ETP_REQ *req;
1571 struct timespec ts; 1838 struct timespec ts;
1572 etp_worker *self = (etp_worker *)thr_arg; 1839 etp_worker *self = (etp_worker *)thr_arg;
1840 int timeout;
1573 1841
1574 /* try to distribute timeouts somewhat randomly */ 1842 /* try to distribute timeouts somewhat evenly */
1575 ts.tv_nsec = ((unsigned long)self & 1023UL) * (1000000000UL / 1024UL); 1843 ts.tv_nsec = ((unsigned long)self & 1023UL) * (1000000000UL / 1024UL);
1576 1844
1577 for (;;) 1845 for (;;)
1578 { 1846 {
1847 ts.tv_sec = 0;
1848
1579 X_LOCK (reqlock); 1849 X_LOCK (reqlock);
1580 1850
1581 for (;;) 1851 for (;;)
1582 { 1852 {
1583 self->req = req = reqq_shift (&req_queue); 1853 self->req = req = reqq_shift (&req_queue);
1584 1854
1585 if (req) 1855 if (req)
1586 break; 1856 break;
1587 1857
1858 if (ts.tv_sec == 1) /* no request, but timeout detected, let's quit */
1859 {
1860 X_UNLOCK (reqlock);
1861 X_LOCK (wrklock);
1862 --started;
1863 X_UNLOCK (wrklock);
1864 goto quit;
1865 }
1866
1588 ++idle; 1867 ++idle;
1589 1868
1590 ts.tv_sec = time (0) + idle_timeout; 1869 if (idle <= max_idle)
1591 if (X_COND_TIMEDWAIT (reqwait, reqlock, ts) == ETIMEDOUT) 1870 /* we are allowed to idle, so do so without any timeout */
1871 X_COND_WAIT (reqwait, reqlock);
1872 else
1592 { 1873 {
1593 if (idle > max_idle) 1874 /* initialise timeout once */
1594 { 1875 if (!ts.tv_sec)
1595 --idle; 1876 ts.tv_sec = time (0) + idle_timeout;
1596 X_UNLOCK (reqlock);
1597 X_LOCK (wrklock);
1598 --started;
1599 X_UNLOCK (wrklock);
1600 goto quit;
1601 }
1602 1877
1603 /* we are allowed to idle, so do so without any timeout */
1604 X_COND_WAIT (reqwait, reqlock); 1878 if (X_COND_TIMEDWAIT (reqwait, reqlock, ts) == ETIMEDOUT)
1879 ts.tv_sec = 1; /* assuming this is not a value computed above.,.. */
1605 } 1880 }
1606 1881
1607 --idle; 1882 --idle;
1608 } 1883 }
1609 1884
1612 X_UNLOCK (reqlock); 1887 X_UNLOCK (reqlock);
1613 1888
1614 if (req->type < 0) 1889 if (req->type < 0)
1615 goto quit; 1890 goto quit;
1616 1891
1617 if (!EIO_CANCELLED (req))
1618 ETP_EXECUTE (self, req); 1892 ETP_EXECUTE (self, req);
1619 1893
1620 X_LOCK (reslock); 1894 X_LOCK (reslock);
1621 1895
1622 ++npending; 1896 ++npending;
1623 1897
1638 return 0; 1912 return 0;
1639} 1913}
1640 1914
1641/*****************************************************************************/ 1915/*****************************************************************************/
1642 1916
1917int ecb_cold
1643int eio_init (void (*want_poll)(void), void (*done_poll)(void)) 1918eio_init (void (*want_poll)(void), void (*done_poll)(void))
1644{ 1919{
1645 return etp_init (want_poll, done_poll); 1920 return etp_init (want_poll, done_poll);
1646} 1921}
1647 1922
1923ecb_inline void
1648static void eio_api_destroy (eio_req *req) 1924eio_api_destroy (eio_req *req)
1649{ 1925{
1650 free (req); 1926 free (req);
1651} 1927}
1652 1928
1653#define REQ(rtype) \ 1929#define REQ(rtype) \
1672 { \ 1948 { \
1673 eio_api_destroy (req); \ 1949 eio_api_destroy (req); \
1674 return 0; \ 1950 return 0; \
1675 } 1951 }
1676 1952
1953static void
1677static void eio_execute (etp_worker *self, eio_req *req) 1954eio_execute (etp_worker *self, eio_req *req)
1678{ 1955{
1956 if (ecb_expect_false (EIO_CANCELLED (req)))
1957 {
1958 req->result = -1;
1959 req->errorno = ECANCELED;
1960 return;
1961 }
1962
1679 switch (req->type) 1963 switch (req->type)
1680 { 1964 {
1681 case EIO_READ: ALLOC (req->size); 1965 case EIO_READ: ALLOC (req->size);
1682 req->result = req->offs >= 0 1966 req->result = req->offs >= 0
1683 ? pread (req->int1, req->ptr2, req->size, req->offs) 1967 ? pread (req->int1, req->ptr2, req->size, req->offs)
1717 case EIO_RENAME: req->result = rename (req->ptr1, req->ptr2); break; 2001 case EIO_RENAME: req->result = rename (req->ptr1, req->ptr2); break;
1718 case EIO_LINK: req->result = link (req->ptr1, req->ptr2); break; 2002 case EIO_LINK: req->result = link (req->ptr1, req->ptr2); break;
1719 case EIO_SYMLINK: req->result = symlink (req->ptr1, req->ptr2); break; 2003 case EIO_SYMLINK: req->result = symlink (req->ptr1, req->ptr2); break;
1720 case EIO_MKNOD: req->result = mknod (req->ptr1, (mode_t)req->int2, (dev_t)req->offs); break; 2004 case EIO_MKNOD: req->result = mknod (req->ptr1, (mode_t)req->int2, (dev_t)req->offs); break;
1721 2005
2006 case EIO_REALPATH: eio__realpath (req, self); break;
2007
1722 case EIO_READLINK: ALLOC (PATH_MAX); 2008 case EIO_READLINK: ALLOC (PATH_MAX);
1723 req->result = readlink (req->ptr1, req->ptr2, PATH_MAX); break; 2009 req->result = readlink (req->ptr1, req->ptr2, PATH_MAX); break;
1724 2010
1725 case EIO_SYNC: req->result = 0; sync (); break; 2011 case EIO_SYNC: req->result = 0; sync (); break;
1726 case EIO_FSYNC: req->result = fsync (req->int1); break; 2012 case EIO_FSYNC: req->result = fsync (req->int1); break;
1728 case EIO_MSYNC: req->result = eio__msync (req->ptr2, req->size, req->int1); break; 2014 case EIO_MSYNC: req->result = eio__msync (req->ptr2, req->size, req->int1); break;
1729 case EIO_MTOUCH: req->result = eio__mtouch (req); break; 2015 case EIO_MTOUCH: req->result = eio__mtouch (req); break;
1730 case EIO_MLOCK: req->result = eio__mlock (req->ptr2, req->size); break; 2016 case EIO_MLOCK: req->result = eio__mlock (req->ptr2, req->size); break;
1731 case EIO_MLOCKALL: req->result = eio__mlockall (req->int1); break; 2017 case EIO_MLOCKALL: req->result = eio__mlockall (req->int1); break;
1732 case EIO_SYNC_FILE_RANGE: req->result = eio__sync_file_range (req->int1, req->offs, req->size, req->int2); break; 2018 case EIO_SYNC_FILE_RANGE: req->result = eio__sync_file_range (req->int1, req->offs, req->size, req->int2); break;
2019 case EIO_FALLOCATE: req->result = eio__fallocate (req->int1, req->int2, req->offs, req->size); break;
1733 2020
1734 case EIO_READDIR: eio__scandir (req, self); break; 2021 case EIO_READDIR: eio__scandir (req, self); break;
1735 2022
1736 case EIO_BUSY: 2023 case EIO_BUSY:
1737#ifdef _WIN32 2024#ifdef _WIN32
1778 case EIO_NOP: 2065 case EIO_NOP:
1779 req->result = 0; 2066 req->result = 0;
1780 break; 2067 break;
1781 2068
1782 case EIO_CUSTOM: 2069 case EIO_CUSTOM:
1783 ((void (*)(eio_req *))req->feed) (req); 2070 req->feed (req);
1784 break; 2071 break;
1785 2072
1786 default: 2073 default:
1787 errno = ENOSYS; 2074 errno = ENOSYS;
1788 req->result = -1; 2075 req->result = -1;
1837eio_req *eio_sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags, int pri, eio_cb cb, void *data) 2124eio_req *eio_sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags, int pri, eio_cb cb, void *data)
1838{ 2125{
1839 REQ (EIO_SYNC_FILE_RANGE); req->int1 = fd; req->offs = offset; req->size = nbytes; req->int2 = flags; SEND; 2126 REQ (EIO_SYNC_FILE_RANGE); req->int1 = fd; req->offs = offset; req->size = nbytes; req->int2 = flags; SEND;
1840} 2127}
1841 2128
2129eio_req *eio_fallocate (int fd, int mode, off_t offset, size_t len, int pri, eio_cb cb, void *data)
2130{
2131 REQ (EIO_FALLOCATE); req->int1 = fd; req->int2 = mode; req->offs = offset; req->size = len; SEND;
2132}
2133
1842eio_req *eio_fdatasync (int fd, int pri, eio_cb cb, void *data) 2134eio_req *eio_fdatasync (int fd, int pri, eio_cb cb, void *data)
1843{ 2135{
1844 REQ (EIO_FDATASYNC); req->int1 = fd; SEND; 2136 REQ (EIO_FDATASYNC); req->int1 = fd; SEND;
1845} 2137}
1846 2138
1887eio_req *eio_fchmod (int fd, mode_t mode, int pri, eio_cb cb, void *data) 2179eio_req *eio_fchmod (int fd, mode_t mode, int pri, eio_cb cb, void *data)
1888{ 2180{
1889 REQ (EIO_FCHMOD); req->int1 = fd; req->int2 = (long)mode; SEND; 2181 REQ (EIO_FCHMOD); req->int1 = fd; req->int2 = (long)mode; SEND;
1890} 2182}
1891 2183
1892eio_req *eio_fchown (int fd, uid_t uid, gid_t gid, int pri, eio_cb cb, void *data) 2184eio_req *eio_fchown (int fd, eio_uid_t uid, eio_gid_t gid, int pri, eio_cb cb, void *data)
1893{ 2185{
1894 REQ (EIO_FCHOWN); req->int1 = fd; req->int2 = (long)uid; req->int3 = (long)gid; SEND; 2186 REQ (EIO_FCHOWN); req->int1 = fd; req->int2 = (long)uid; req->int3 = (long)gid; SEND;
1895} 2187}
1896 2188
1897eio_req *eio_dup2 (int fd, int fd2, int pri, eio_cb cb, void *data) 2189eio_req *eio_dup2 (int fd, int fd2, int pri, eio_cb cb, void *data)
1917eio_req *eio_truncate (const char *path, off_t offset, int pri, eio_cb cb, void *data) 2209eio_req *eio_truncate (const char *path, off_t offset, int pri, eio_cb cb, void *data)
1918{ 2210{
1919 REQ (EIO_TRUNCATE); PATH; req->offs = offset; SEND; 2211 REQ (EIO_TRUNCATE); PATH; req->offs = offset; SEND;
1920} 2212}
1921 2213
1922eio_req *eio_chown (const char *path, uid_t uid, gid_t gid, int pri, eio_cb cb, void *data) 2214eio_req *eio_chown (const char *path, eio_uid_t uid, eio_gid_t gid, int pri, eio_cb cb, void *data)
1923{ 2215{
1924 REQ (EIO_CHOWN); PATH; req->int2 = (long)uid; req->int3 = (long)gid; SEND; 2216 REQ (EIO_CHOWN); PATH; req->int2 = (long)uid; req->int3 = (long)gid; SEND;
1925} 2217}
1926 2218
1927eio_req *eio_chmod (const char *path, mode_t mode, int pri, eio_cb cb, void *data) 2219eio_req *eio_chmod (const char *path, mode_t mode, int pri, eio_cb cb, void *data)
1941} 2233}
1942 2234
1943eio_req *eio_readlink (const char *path, int pri, eio_cb cb, void *data) 2235eio_req *eio_readlink (const char *path, int pri, eio_cb cb, void *data)
1944{ 2236{
1945 return eio__1path (EIO_READLINK, path, pri, cb, data); 2237 return eio__1path (EIO_READLINK, path, pri, cb, data);
2238}
2239
2240eio_req *eio_realpath (const char *path, int pri, eio_cb cb, void *data)
2241{
2242 return eio__1path (EIO_REALPATH, path, pri, cb, data);
1946} 2243}
1947 2244
1948eio_req *eio_stat (const char *path, int pri, eio_cb cb, void *data) 2245eio_req *eio_stat (const char *path, int pri, eio_cb cb, void *data)
1949{ 2246{
1950 return eio__1path (EIO_STAT, path, pri, cb, data); 2247 return eio__1path (EIO_STAT, path, pri, cb, data);
2009eio_req *eio_rename (const char *path, const char *new_path, int pri, eio_cb cb, void *data) 2306eio_req *eio_rename (const char *path, const char *new_path, int pri, eio_cb cb, void *data)
2010{ 2307{
2011 return eio__2path (EIO_RENAME, path, new_path, pri, cb, data); 2308 return eio__2path (EIO_RENAME, path, new_path, pri, cb, data);
2012} 2309}
2013 2310
2014eio_req *eio_custom (eio_cb execute, int pri, eio_cb cb, void *data) 2311eio_req *eio_custom (void (*execute)(eio_req *), int pri, eio_cb cb, void *data)
2015{ 2312{
2016 REQ (EIO_CUSTOM); req->feed = (void (*)(eio_req *))execute; SEND; 2313 REQ (EIO_CUSTOM); req->feed = execute; SEND;
2017} 2314}
2018 2315
2019#endif 2316#endif
2020 2317
2021eio_req *eio_grp (eio_cb cb, void *data) 2318eio_req *eio_grp (eio_cb cb, void *data)
2030#undef SEND 2327#undef SEND
2031 2328
2032/*****************************************************************************/ 2329/*****************************************************************************/
2033/* grp functions */ 2330/* grp functions */
2034 2331
2332void
2035void eio_grp_feed (eio_req *grp, void (*feed)(eio_req *req), int limit) 2333eio_grp_feed (eio_req *grp, void (*feed)(eio_req *req), int limit)
2036{ 2334{
2037 grp->int2 = limit; 2335 grp->int2 = limit;
2038 grp->feed = feed; 2336 grp->feed = feed;
2039 2337
2040 grp_try_feed (grp); 2338 grp_try_feed (grp);
2041} 2339}
2042 2340
2341void
2043void eio_grp_limit (eio_req *grp, int limit) 2342eio_grp_limit (eio_req *grp, int limit)
2044{ 2343{
2045 grp->int2 = limit; 2344 grp->int2 = limit;
2046 2345
2047 grp_try_feed (grp); 2346 grp_try_feed (grp);
2048} 2347}
2049 2348
2349void
2050void eio_grp_add (eio_req *grp, eio_req *req) 2350eio_grp_add (eio_req *grp, eio_req *req)
2051{ 2351{
2052 assert (("cannot add requests to IO::AIO::GRP after the group finished", grp->int1 != 2)); 2352 assert (("cannot add requests to IO::AIO::GRP after the group finished", grp->int1 != 2));
2053 2353
2054 grp->flags |= EIO_FLAG_GROUPADD; 2354 grp->flags |= EIO_FLAG_GROUPADD;
2055 2355
2066} 2366}
2067 2367
2068/*****************************************************************************/ 2368/*****************************************************************************/
2069/* misc garbage */ 2369/* misc garbage */
2070 2370
2371eio_ssize_t
2071ssize_t eio_sendfile_sync (int ofd, int ifd, off_t offset, size_t count) 2372eio_sendfile_sync (int ofd, int ifd, off_t offset, size_t count)
2072{ 2373{
2073 etp_worker wrk; 2374 etp_worker wrk;
2074 ssize_t ret; 2375 eio_ssize_t ret;
2075 2376
2076 wrk.dbuf = 0; 2377 wrk.dbuf = 0;
2077 2378
2078 ret = eio__sendfile (ofd, ifd, offset, count, &wrk); 2379 ret = eio__sendfile (ofd, ifd, offset, count, &wrk);
2079 2380

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines