ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/libeio/eio.c
(Generate patch)

Comparing libeio/eio.c (file contents):
Revision 1.65 by root, Mon May 30 12:56:50 2011 UTC vs.
Revision 1.89 by root, Thu Jul 14 22:36:17 2011 UTC

40#ifndef _WIN32 40#ifndef _WIN32
41# include "config.h" 41# include "config.h"
42#endif 42#endif
43 43
44#include "eio.h" 44#include "eio.h"
45#include "ecb.h"
45 46
46#ifdef EIO_STACKSIZE 47#ifdef EIO_STACKSIZE
47# define XTHREAD_STACKSIZE EIO_STACKSIZE 48# define XTHREAD_STACKSIZE EIO_STACKSIZE
48#endif 49#endif
49#include "xthread.h" 50#include "xthread.h"
53#include <stdlib.h> 54#include <stdlib.h>
54#include <string.h> 55#include <string.h>
55#include <errno.h> 56#include <errno.h>
56#include <sys/types.h> 57#include <sys/types.h>
57#include <sys/stat.h> 58#include <sys/stat.h>
58#include <sys/statvfs.h>
59#include <limits.h> 59#include <limits.h>
60#include <fcntl.h> 60#include <fcntl.h>
61#include <assert.h> 61#include <assert.h>
62 62
63/* intptr_t comes from unistd.h, says POSIX/UNIX/tradition */
64/* intptr_t only comes from stdint.h, says idiot openbsd coder */
65#if HAVE_STDINT_H
66# include <stdint.h>
67#endif
68
69#ifndef ECANCELED
70# define ECANCELED EDOM
71#endif
72
73static void eio_destroy (eio_req *req);
74
63#ifndef EIO_FINISH 75#ifndef EIO_FINISH
64# define EIO_FINISH(req) ((req)->finish) && !EIO_CANCELLED (req) ? (req)->finish (req) : 0 76# define EIO_FINISH(req) ((req)->finish) && !EIO_CANCELLED (req) ? (req)->finish (req) : 0
65#endif 77#endif
66 78
67#ifndef EIO_DESTROY 79#ifndef EIO_DESTROY
70 82
71#ifndef EIO_FEED 83#ifndef EIO_FEED
72# define EIO_FEED(req) do { if ((req)->feed ) (req)->feed (req); } while (0) 84# define EIO_FEED(req) do { if ((req)->feed ) (req)->feed (req); } while (0)
73#endif 85#endif
74 86
87#ifndef EIO_FD_TO_WIN32_HANDLE
88# define EIO_FD_TO_WIN32_HANDLE(fd) _get_osfhandle (fd)
89#endif
90#ifndef EIO_WIN32_HANDLE_TO_FD
91# define EIO_WIN32_HANDLE_TO_FD(handle) _open_osfhandle (handle, 0)
92#endif
93
94#define EIO_ERRNO(errval,retval) ((errno = errval), retval)
95
96#define EIO_ENOSYS() EIO_ERRNO (ENOSYS, -1)
97
75#ifdef _WIN32 98#ifdef _WIN32
76 99
77 /*doh*/ 100 #define PAGESIZE 4096 /* GetSystemInfo? */
101
102 #ifdef EIO_STRUCT_STATI64
103 #define stat(path,buf) _stati64 (path,buf)
104 #define fstat(fd,buf) _fstati64 (path,buf)
105 #endif
106 #define lstat(path,buf) stat (path,buf)
107 #define fsync(fd) (FlushFileBuffers (EIO_FD_TO_WIN32_HANDLE (fd)) ? 0 : EIO_ERRNO (EBADF, -1))
108 #define mkdir(path,mode) _mkdir (path)
109 #define link(old,neu) (CreateHardLink (neu, old, 0) ? 0 : EIO_ERRNO (ENOENT, -1))
110
111 #define chown(path,uid,gid) EIO_ENOSYS ()
112 #define fchown(fd,uid,gid) EIO_ENOSYS ()
113 #define truncate(path,offs) EIO_ENOSYS () /* far-miss: SetEndOfFile */
114 #define ftruncate(fd,offs) EIO_ENOSYS () /* near-miss: SetEndOfFile */
115 #define mknod(path,mode,dev) EIO_ENOSYS ()
116 #define sync() EIO_ENOSYS ()
117
118 /* we could even stat and see if it exists */
119 static int
120 symlink (const char *old, const char *neu)
121 {
122 if (CreateSymbolicLink (neu, old, 1))
123 return 0;
124
125 if (CreateSymbolicLink (neu, old, 0))
126 return 0;
127
128 return EIO_ERRNO (ENOENT, -1);
129 }
130
78#else 131#else
79 132
80# include <sys/time.h> 133 #include <sys/time.h>
81# include <sys/select.h> 134 #include <sys/select.h>
135 #include <sys/statvfs.h>
82# include <unistd.h> 136 #include <unistd.h>
83# include <utime.h> 137 #include <utime.h>
84# include <signal.h> 138 #include <signal.h>
85# include <dirent.h> 139 #include <dirent.h>
86 140
87#if _POSIX_MEMLOCK || _POSIX_MEMLOCK_RANGE || _POSIX_MAPPED_FILES 141 #if _POSIX_MEMLOCK || _POSIX_MEMLOCK_RANGE || _POSIX_MAPPED_FILES
88# include <sys/mman.h> 142 #include <sys/mman.h>
89#endif 143 #endif
90 144
91/* POSIX_SOURCE is useless on bsd's, and XOPEN_SOURCE is unreliable there, too */ 145 /* POSIX_SOURCE is useless on bsd's, and XOPEN_SOURCE is unreliable there, too */
92# if __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__ 146 #if __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__
93# define _DIRENT_HAVE_D_TYPE /* sigh */ 147 #define _DIRENT_HAVE_D_TYPE /* sigh */
94# define D_INO(de) (de)->d_fileno 148 #define D_INO(de) (de)->d_fileno
95# define D_NAMLEN(de) (de)->d_namlen 149 #define D_NAMLEN(de) (de)->d_namlen
96# elif __linux || defined d_ino || _XOPEN_SOURCE >= 600 150 #elif __linux || defined d_ino || _XOPEN_SOURCE >= 600
97# define D_INO(de) (de)->d_ino 151 #define D_INO(de) (de)->d_ino
98# endif 152 #endif
99 153
100#ifdef _D_EXACT_NAMLEN 154 #ifdef _D_EXACT_NAMLEN
101# undef D_NAMLEN 155 #undef D_NAMLEN
102# define D_NAMLEN(de) _D_EXACT_NAMLEN (de) 156 #define D_NAMLEN(de) _D_EXACT_NAMLEN (de)
103#endif 157 #endif
104 158
105# ifdef _DIRENT_HAVE_D_TYPE 159 #ifdef _DIRENT_HAVE_D_TYPE
106# define D_TYPE(de) (de)->d_type 160 #define D_TYPE(de) (de)->d_type
107# endif 161 #endif
108 162
109# ifndef EIO_STRUCT_DIRENT 163 #ifndef EIO_STRUCT_DIRENT
110# define EIO_STRUCT_DIRENT struct dirent 164 #define EIO_STRUCT_DIRENT struct dirent
111# endif 165 #endif
112 166
113#endif 167#endif
114 168
115#if HAVE_SENDFILE 169#if HAVE_SENDFILE
116# if __linux 170# if __linux
159 if (!eio_buf) \ 213 if (!eio_buf) \
160 return -1; 214 return -1;
161 215
162#define EIO_TICKS ((1000000 + 1023) >> 10) 216#define EIO_TICKS ((1000000 + 1023) >> 10)
163 217
164/*****************************************************************************/
165
166#if __GNUC__ >= 3
167# define expect(expr,value) __builtin_expect ((expr),(value))
168#else
169# define expect(expr,value) (expr)
170#endif
171
172#define expect_false(expr) expect ((expr) != 0, 0)
173#define expect_true(expr) expect ((expr) != 0, 1)
174
175/*****************************************************************************/
176
177#define ETP_PRI_MIN EIO_PRI_MIN 218#define ETP_PRI_MIN EIO_PRI_MIN
178#define ETP_PRI_MAX EIO_PRI_MAX 219#define ETP_PRI_MAX EIO_PRI_MAX
179 220
180struct etp_worker; 221struct etp_worker;
181 222
206/*****************************************************************************/ 247/*****************************************************************************/
207 248
208#define ETP_NUM_PRI (ETP_PRI_MAX - ETP_PRI_MIN + 1) 249#define ETP_NUM_PRI (ETP_PRI_MAX - ETP_PRI_MIN + 1)
209 250
210/* calculate time difference in ~1/EIO_TICKS of a second */ 251/* calculate time difference in ~1/EIO_TICKS of a second */
252ecb_inline int
211static int tvdiff (struct timeval *tv1, struct timeval *tv2) 253tvdiff (struct timeval *tv1, struct timeval *tv2)
212{ 254{
213 return (tv2->tv_sec - tv1->tv_sec ) * EIO_TICKS 255 return (tv2->tv_sec - tv1->tv_sec ) * EIO_TICKS
214 + ((tv2->tv_usec - tv1->tv_usec) >> 10); 256 + ((tv2->tv_usec - tv1->tv_usec) >> 10);
215} 257}
216 258
260#define ETP_WORKER_LOCK(wrk) X_LOCK (wrklock) 302#define ETP_WORKER_LOCK(wrk) X_LOCK (wrklock)
261#define ETP_WORKER_UNLOCK(wrk) X_UNLOCK (wrklock) 303#define ETP_WORKER_UNLOCK(wrk) X_UNLOCK (wrklock)
262 304
263/* worker threads management */ 305/* worker threads management */
264 306
307static void ecb_cold
265static void etp_worker_clear (etp_worker *wrk) 308etp_worker_clear (etp_worker *wrk)
266{ 309{
267 ETP_WORKER_CLEAR (wrk); 310 ETP_WORKER_CLEAR (wrk);
268} 311}
269 312
313static void ecb_cold
270static void etp_worker_free (etp_worker *wrk) 314etp_worker_free (etp_worker *wrk)
271{ 315{
272 wrk->next->prev = wrk->prev; 316 wrk->next->prev = wrk->prev;
273 wrk->prev->next = wrk->next; 317 wrk->prev->next = wrk->next;
274 318
275 free (wrk); 319 free (wrk);
276} 320}
277 321
278static unsigned int etp_nreqs (void) 322static unsigned int
323etp_nreqs (void)
279{ 324{
280 int retval; 325 int retval;
281 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 326 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
282 retval = nreqs; 327 retval = nreqs;
283 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock); 328 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
284 return retval; 329 return retval;
285} 330}
286 331
287static unsigned int etp_nready (void) 332static unsigned int
333etp_nready (void)
288{ 334{
289 unsigned int retval; 335 unsigned int retval;
290 336
291 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 337 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
292 retval = nready; 338 retval = nready;
293 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock); 339 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
294 340
295 return retval; 341 return retval;
296} 342}
297 343
298static unsigned int etp_npending (void) 344static unsigned int
345etp_npending (void)
299{ 346{
300 unsigned int retval; 347 unsigned int retval;
301 348
302 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 349 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
303 retval = npending; 350 retval = npending;
304 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock); 351 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
305 352
306 return retval; 353 return retval;
307} 354}
308 355
309static unsigned int etp_nthreads (void) 356static unsigned int
357etp_nthreads (void)
310{ 358{
311 unsigned int retval; 359 unsigned int retval;
312 360
313 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 361 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
314 retval = started; 362 retval = started;
328} etp_reqq; 376} etp_reqq;
329 377
330static etp_reqq req_queue; 378static etp_reqq req_queue;
331static etp_reqq res_queue; 379static etp_reqq res_queue;
332 380
381static int ecb_noinline
333static int reqq_push (etp_reqq *q, ETP_REQ *req) 382reqq_push (etp_reqq *q, ETP_REQ *req)
334{ 383{
335 int pri = req->pri; 384 int pri = req->pri;
336 req->next = 0; 385 req->next = 0;
337 386
338 if (q->qe[pri]) 387 if (q->qe[pri])
344 q->qe[pri] = q->qs[pri] = req; 393 q->qe[pri] = q->qs[pri] = req;
345 394
346 return q->size++; 395 return q->size++;
347} 396}
348 397
398static ETP_REQ * ecb_noinline
349static ETP_REQ *reqq_shift (etp_reqq *q) 399reqq_shift (etp_reqq *q)
350{ 400{
351 int pri; 401 int pri;
352 402
353 if (!q->size) 403 if (!q->size)
354 return 0; 404 return 0;
369 } 419 }
370 420
371 abort (); 421 abort ();
372} 422}
373 423
424static void ecb_cold
374static void etp_thread_init (void) 425etp_thread_init (void)
375{ 426{
427#if !HAVE_PREADWRITE
428 X_MUTEX_CREATE (preadwritelock);
429#endif
376 X_MUTEX_CREATE (wrklock); 430 X_MUTEX_CREATE (wrklock);
377 X_MUTEX_CREATE (reslock); 431 X_MUTEX_CREATE (reslock);
378 X_MUTEX_CREATE (reqlock); 432 X_MUTEX_CREATE (reqlock);
379 X_COND_CREATE (reqwait); 433 X_COND_CREATE (reqwait);
380} 434}
381 435
436static void ecb_cold
382static void etp_atfork_prepare (void) 437etp_atfork_prepare (void)
383{ 438{
384 X_LOCK (wrklock);
385 X_LOCK (reqlock);
386 X_LOCK (reslock);
387#if !HAVE_PREADWRITE
388 X_LOCK (preadwritelock);
389#endif
390} 439}
391 440
441static void ecb_cold
392static void etp_atfork_parent (void) 442etp_atfork_parent (void)
393{ 443{
394#if !HAVE_PREADWRITE
395 X_UNLOCK (preadwritelock);
396#endif
397 X_UNLOCK (reslock);
398 X_UNLOCK (reqlock);
399 X_UNLOCK (wrklock);
400} 444}
401 445
446static void ecb_cold
402static void etp_atfork_child (void) 447etp_atfork_child (void)
403{ 448{
404 ETP_REQ *prv; 449 ETP_REQ *prv;
405 450
406 while ((prv = reqq_shift (&req_queue))) 451 while ((prv = reqq_shift (&req_queue)))
407 ETP_DESTROY (prv); 452 ETP_DESTROY (prv);
427 npending = 0; 472 npending = 0;
428 473
429 etp_thread_init (); 474 etp_thread_init ();
430} 475}
431 476
432static void 477static void ecb_cold
433etp_once_init (void) 478etp_once_init (void)
434{ 479{
435 etp_thread_init (); 480 etp_thread_init ();
436 X_THREAD_ATFORK (etp_atfork_prepare, etp_atfork_parent, etp_atfork_child); 481 X_THREAD_ATFORK (etp_atfork_prepare, etp_atfork_parent, etp_atfork_child);
437} 482}
438 483
439static int 484static int ecb_cold
440etp_init (void (*want_poll)(void), void (*done_poll)(void)) 485etp_init (void (*want_poll)(void), void (*done_poll)(void))
441{ 486{
442 static pthread_once_t doinit = PTHREAD_ONCE_INIT; 487 static pthread_once_t doinit = PTHREAD_ONCE_INIT;
443 488
444 pthread_once (&doinit, etp_once_init); 489 pthread_once (&doinit, etp_once_init);
449 return 0; 494 return 0;
450} 495}
451 496
452X_THREAD_PROC (etp_proc); 497X_THREAD_PROC (etp_proc);
453 498
499static void ecb_cold
454static void etp_start_thread (void) 500etp_start_thread (void)
455{ 501{
456 etp_worker *wrk = calloc (1, sizeof (etp_worker)); 502 etp_worker *wrk = calloc (1, sizeof (etp_worker));
457 503
458 /*TODO*/ 504 /*TODO*/
459 assert (("unable to allocate worker thread data", wrk)); 505 assert (("unable to allocate worker thread data", wrk));
472 free (wrk); 518 free (wrk);
473 519
474 X_UNLOCK (wrklock); 520 X_UNLOCK (wrklock);
475} 521}
476 522
523static void
477static void etp_maybe_start_thread (void) 524etp_maybe_start_thread (void)
478{ 525{
479 if (expect_true (etp_nthreads () >= wanted)) 526 if (ecb_expect_true (etp_nthreads () >= wanted))
480 return; 527 return;
481 528
482 /* todo: maybe use idle here, but might be less exact */ 529 /* todo: maybe use idle here, but might be less exact */
483 if (expect_true (0 <= (int)etp_nthreads () + (int)etp_npending () - (int)etp_nreqs ())) 530 if (ecb_expect_true (0 <= (int)etp_nthreads () + (int)etp_npending () - (int)etp_nreqs ()))
484 return; 531 return;
485 532
486 etp_start_thread (); 533 etp_start_thread ();
487} 534}
488 535
536static void ecb_cold
489static void etp_end_thread (void) 537etp_end_thread (void)
490{ 538{
491 eio_req *req = calloc (1, sizeof (eio_req)); 539 eio_req *req = calloc (1, sizeof (eio_req));
492 540
493 req->type = -1; 541 req->type = -1;
494 req->pri = ETP_PRI_MAX - ETP_PRI_MIN; 542 req->pri = ETP_PRI_MAX - ETP_PRI_MIN;
501 X_LOCK (wrklock); 549 X_LOCK (wrklock);
502 --started; 550 --started;
503 X_UNLOCK (wrklock); 551 X_UNLOCK (wrklock);
504} 552}
505 553
506static int etp_poll (void) 554static int
555etp_poll (void)
507{ 556{
508 unsigned int maxreqs; 557 unsigned int maxreqs;
509 unsigned int maxtime; 558 unsigned int maxtime;
510 struct timeval tv_start, tv_now; 559 struct timeval tv_start, tv_now;
511 560
541 590
542 X_LOCK (reqlock); 591 X_LOCK (reqlock);
543 --nreqs; 592 --nreqs;
544 X_UNLOCK (reqlock); 593 X_UNLOCK (reqlock);
545 594
546 if (expect_false (req->type == EIO_GROUP && req->size)) 595 if (ecb_expect_false (req->type == EIO_GROUP && req->size))
547 { 596 {
548 req->int1 = 1; /* mark request as delayed */ 597 req->int1 = 1; /* mark request as delayed */
549 continue; 598 continue;
550 } 599 }
551 else 600 else
552 { 601 {
553 int res = ETP_FINISH (req); 602 int res = ETP_FINISH (req);
554 if (expect_false (res)) 603 if (ecb_expect_false (res))
555 return res; 604 return res;
556 } 605 }
557 606
558 if (expect_false (maxreqs && !--maxreqs)) 607 if (ecb_expect_false (maxreqs && !--maxreqs))
559 break; 608 break;
560 609
561 if (maxtime) 610 if (maxtime)
562 { 611 {
563 gettimeofday (&tv_now, 0); 612 gettimeofday (&tv_now, 0);
569 618
570 errno = EAGAIN; 619 errno = EAGAIN;
571 return -1; 620 return -1;
572} 621}
573 622
623static void
574static void etp_cancel (ETP_REQ *req) 624etp_cancel (ETP_REQ *req)
575{ 625{
576 X_LOCK (wrklock); 626 req->cancelled = 1;
577 req->flags |= EIO_FLAG_CANCELLED;
578 X_UNLOCK (wrklock);
579 627
580 eio_grp_cancel (req); 628 eio_grp_cancel (req);
581} 629}
582 630
631static void
583static void etp_submit (ETP_REQ *req) 632etp_submit (ETP_REQ *req)
584{ 633{
585 req->pri -= ETP_PRI_MIN; 634 req->pri -= ETP_PRI_MIN;
586 635
587 if (expect_false (req->pri < ETP_PRI_MIN - ETP_PRI_MIN)) req->pri = ETP_PRI_MIN - ETP_PRI_MIN; 636 if (ecb_expect_false (req->pri < ETP_PRI_MIN - ETP_PRI_MIN)) req->pri = ETP_PRI_MIN - ETP_PRI_MIN;
588 if (expect_false (req->pri > ETP_PRI_MAX - ETP_PRI_MIN)) req->pri = ETP_PRI_MAX - ETP_PRI_MIN; 637 if (ecb_expect_false (req->pri > ETP_PRI_MAX - ETP_PRI_MIN)) req->pri = ETP_PRI_MAX - ETP_PRI_MIN;
589 638
590 if (expect_false (req->type == EIO_GROUP)) 639 if (ecb_expect_false (req->type == EIO_GROUP))
591 { 640 {
592 /* I hope this is worth it :/ */ 641 /* I hope this is worth it :/ */
593 X_LOCK (reqlock); 642 X_LOCK (reqlock);
594 ++nreqs; 643 ++nreqs;
595 X_UNLOCK (reqlock); 644 X_UNLOCK (reqlock);
614 663
615 etp_maybe_start_thread (); 664 etp_maybe_start_thread ();
616 } 665 }
617} 666}
618 667
668static void ecb_cold
619static void etp_set_max_poll_time (double nseconds) 669etp_set_max_poll_time (double nseconds)
620{ 670{
621 if (WORDACCESS_UNSAFE) X_LOCK (reslock); 671 if (WORDACCESS_UNSAFE) X_LOCK (reslock);
622 max_poll_time = nseconds * EIO_TICKS; 672 max_poll_time = nseconds * EIO_TICKS;
623 if (WORDACCESS_UNSAFE) X_UNLOCK (reslock); 673 if (WORDACCESS_UNSAFE) X_UNLOCK (reslock);
624} 674}
625 675
676static void ecb_cold
626static void etp_set_max_poll_reqs (unsigned int maxreqs) 677etp_set_max_poll_reqs (unsigned int maxreqs)
627{ 678{
628 if (WORDACCESS_UNSAFE) X_LOCK (reslock); 679 if (WORDACCESS_UNSAFE) X_LOCK (reslock);
629 max_poll_reqs = maxreqs; 680 max_poll_reqs = maxreqs;
630 if (WORDACCESS_UNSAFE) X_UNLOCK (reslock); 681 if (WORDACCESS_UNSAFE) X_UNLOCK (reslock);
631} 682}
632 683
684static void ecb_cold
633static void etp_set_max_idle (unsigned int nthreads) 685etp_set_max_idle (unsigned int nthreads)
634{ 686{
635 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 687 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
636 max_idle = nthreads; 688 max_idle = nthreads;
637 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock); 689 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
638} 690}
639 691
692static void ecb_cold
640static void etp_set_idle_timeout (unsigned int seconds) 693etp_set_idle_timeout (unsigned int seconds)
641{ 694{
642 if (WORDACCESS_UNSAFE) X_LOCK (reqlock); 695 if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
643 idle_timeout = seconds; 696 idle_timeout = seconds;
644 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock); 697 if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
645} 698}
646 699
700static void ecb_cold
647static void etp_set_min_parallel (unsigned int nthreads) 701etp_set_min_parallel (unsigned int nthreads)
648{ 702{
649 if (wanted < nthreads) 703 if (wanted < nthreads)
650 wanted = nthreads; 704 wanted = nthreads;
651} 705}
652 706
707static void ecb_cold
653static void etp_set_max_parallel (unsigned int nthreads) 708etp_set_max_parallel (unsigned int nthreads)
654{ 709{
655 if (wanted > nthreads) 710 if (wanted > nthreads)
656 wanted = nthreads; 711 wanted = nthreads;
657 712
658 while (started > wanted) 713 while (started > wanted)
659 etp_end_thread (); 714 etp_end_thread ();
660} 715}
661 716
662/*****************************************************************************/ 717/*****************************************************************************/
663 718
719static void
664static void grp_try_feed (eio_req *grp) 720grp_try_feed (eio_req *grp)
665{ 721{
666 while (grp->size < grp->int2 && !EIO_CANCELLED (grp)) 722 while (grp->size < grp->int2 && !EIO_CANCELLED (grp))
667 { 723 {
668 grp->flags &= ~EIO_FLAG_GROUPADD; 724 grp->flags &= ~EIO_FLAG_GROUPADD;
669 725
676 break; 732 break;
677 } 733 }
678 } 734 }
679} 735}
680 736
737static int
681static int grp_dec (eio_req *grp) 738grp_dec (eio_req *grp)
682{ 739{
683 --grp->size; 740 --grp->size;
684 741
685 /* call feeder, if applicable */ 742 /* call feeder, if applicable */
686 grp_try_feed (grp); 743 grp_try_feed (grp);
690 return eio_finish (grp); 747 return eio_finish (grp);
691 else 748 else
692 return 0; 749 return 0;
693} 750}
694 751
752static void
695void eio_destroy (eio_req *req) 753eio_destroy (eio_req *req)
696{ 754{
697 if ((req)->flags & EIO_FLAG_PTR1_FREE) free (req->ptr1); 755 if ((req)->flags & EIO_FLAG_PTR1_FREE) free (req->ptr1);
698 if ((req)->flags & EIO_FLAG_PTR2_FREE) free (req->ptr2); 756 if ((req)->flags & EIO_FLAG_PTR2_FREE) free (req->ptr2);
699 757
700 EIO_DESTROY (req); 758 EIO_DESTROY (req);
701} 759}
702 760
761static int
703static int eio_finish (eio_req *req) 762eio_finish (eio_req *req)
704{ 763{
705 int res = EIO_FINISH (req); 764 int res = EIO_FINISH (req);
706 765
707 if (req->grp) 766 if (req->grp)
708 { 767 {
716 if (grp->grp_first == req) 775 if (grp->grp_first == req)
717 grp->grp_first = req->grp_next; 776 grp->grp_first = req->grp_next;
718 777
719 res2 = grp_dec (grp); 778 res2 = grp_dec (grp);
720 779
721 if (!res && res2) 780 if (!res)
722 res = res2; 781 res = res2;
723 } 782 }
724 783
725 eio_destroy (req); 784 eio_destroy (req);
726 785
727 return res; 786 return res;
728} 787}
729 788
789void
730void eio_grp_cancel (eio_req *grp) 790eio_grp_cancel (eio_req *grp)
731{ 791{
732 for (grp = grp->grp_first; grp; grp = grp->grp_next) 792 for (grp = grp->grp_first; grp; grp = grp->grp_next)
733 eio_cancel (grp); 793 eio_cancel (grp);
734} 794}
735 795
796void
736void eio_cancel (eio_req *req) 797eio_cancel (eio_req *req)
737{ 798{
738 etp_cancel (req); 799 etp_cancel (req);
739} 800}
740 801
802void
741void eio_submit (eio_req *req) 803eio_submit (eio_req *req)
742{ 804{
743 etp_submit (req); 805 etp_submit (req);
744} 806}
745 807
746unsigned int eio_nreqs (void) 808unsigned int
809eio_nreqs (void)
747{ 810{
748 return etp_nreqs (); 811 return etp_nreqs ();
749} 812}
750 813
751unsigned int eio_nready (void) 814unsigned int
815eio_nready (void)
752{ 816{
753 return etp_nready (); 817 return etp_nready ();
754} 818}
755 819
756unsigned int eio_npending (void) 820unsigned int
821eio_npending (void)
757{ 822{
758 return etp_npending (); 823 return etp_npending ();
759} 824}
760 825
761unsigned int eio_nthreads (void) 826unsigned int ecb_cold
827eio_nthreads (void)
762{ 828{
763 return etp_nthreads (); 829 return etp_nthreads ();
764} 830}
765 831
832void ecb_cold
766void eio_set_max_poll_time (double nseconds) 833eio_set_max_poll_time (double nseconds)
767{ 834{
768 etp_set_max_poll_time (nseconds); 835 etp_set_max_poll_time (nseconds);
769} 836}
770 837
838void ecb_cold
771void eio_set_max_poll_reqs (unsigned int maxreqs) 839eio_set_max_poll_reqs (unsigned int maxreqs)
772{ 840{
773 etp_set_max_poll_reqs (maxreqs); 841 etp_set_max_poll_reqs (maxreqs);
774} 842}
775 843
844void ecb_cold
776void eio_set_max_idle (unsigned int nthreads) 845eio_set_max_idle (unsigned int nthreads)
777{ 846{
778 etp_set_max_idle (nthreads); 847 etp_set_max_idle (nthreads);
779} 848}
780 849
850void ecb_cold
781void eio_set_idle_timeout (unsigned int seconds) 851eio_set_idle_timeout (unsigned int seconds)
782{ 852{
783 etp_set_idle_timeout (seconds); 853 etp_set_idle_timeout (seconds);
784} 854}
785 855
856void ecb_cold
786void eio_set_min_parallel (unsigned int nthreads) 857eio_set_min_parallel (unsigned int nthreads)
787{ 858{
788 etp_set_min_parallel (nthreads); 859 etp_set_min_parallel (nthreads);
789} 860}
790 861
862void ecb_cold
791void eio_set_max_parallel (unsigned int nthreads) 863eio_set_max_parallel (unsigned int nthreads)
792{ 864{
793 etp_set_max_parallel (nthreads); 865 etp_set_max_parallel (nthreads);
794} 866}
795 867
796int eio_poll (void) 868int eio_poll (void)
805# undef pread 877# undef pread
806# undef pwrite 878# undef pwrite
807# define pread eio__pread 879# define pread eio__pread
808# define pwrite eio__pwrite 880# define pwrite eio__pwrite
809 881
810static ssize_t 882static eio_ssize_t
811eio__pread (int fd, void *buf, size_t count, off_t offset) 883eio__pread (int fd, void *buf, size_t count, off_t offset)
812{ 884{
813 ssize_t res; 885 eio_ssize_t res;
814 off_t ooffset; 886 off_t ooffset;
815 887
816 X_LOCK (preadwritelock); 888 X_LOCK (preadwritelock);
817 ooffset = lseek (fd, 0, SEEK_CUR); 889 ooffset = lseek (fd, 0, SEEK_CUR);
818 lseek (fd, offset, SEEK_SET); 890 lseek (fd, offset, SEEK_SET);
821 X_UNLOCK (preadwritelock); 893 X_UNLOCK (preadwritelock);
822 894
823 return res; 895 return res;
824} 896}
825 897
826static ssize_t 898static eio_ssize_t
827eio__pwrite (int fd, void *buf, size_t count, off_t offset) 899eio__pwrite (int fd, void *buf, size_t count, off_t offset)
828{ 900{
829 ssize_t res; 901 eio_ssize_t res;
830 off_t ooffset; 902 off_t ooffset;
831 903
832 X_LOCK (preadwritelock); 904 X_LOCK (preadwritelock);
833 ooffset = lseek (fd, 0, SEEK_CUR); 905 ooffset = lseek (fd, 0, SEEK_CUR);
834 lseek (fd, offset, SEEK_SET); 906 lseek (fd, offset, SEEK_SET);
866#ifndef HAVE_FUTIMES 938#ifndef HAVE_FUTIMES
867 939
868# undef futimes 940# undef futimes
869# define futimes(fd,times) eio__futimes (fd, times) 941# define futimes(fd,times) eio__futimes (fd, times)
870 942
943static int
871static int eio__futimes (int fd, const struct timeval tv[2]) 944eio__futimes (int fd, const struct timeval tv[2])
872{ 945{
873 errno = ENOSYS; 946 errno = ENOSYS;
874 return -1; 947 return -1;
875} 948}
876 949
880# undef fdatasync 953# undef fdatasync
881# define fdatasync(fd) fsync (fd) 954# define fdatasync(fd) fsync (fd)
882#endif 955#endif
883 956
884/* sync_file_range always needs emulation */ 957/* sync_file_range always needs emulation */
885int 958static int
886eio__sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags) 959eio__sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags)
887{ 960{
888#if HAVE_SYNC_FILE_RANGE 961#if HAVE_SYNC_FILE_RANGE
889 int res; 962 int res;
890 963
907 /* even though we could play tricks with the flags, it's better to always 980 /* even though we could play tricks with the flags, it's better to always
908 * call fdatasync, as that matches the expectation of its users best */ 981 * call fdatasync, as that matches the expectation of its users best */
909 return fdatasync (fd); 982 return fdatasync (fd);
910} 983}
911 984
985static int
986eio__fallocate (int fd, int mode, off_t offset, size_t len)
987{
988#if HAVE_FALLOCATE
989 return fallocate (fd, mode, offset, len);
990#else
991 errno = ENOSYS;
992 return -1;
993#endif
994}
995
912#if !HAVE_READAHEAD 996#if !HAVE_READAHEAD
913# undef readahead 997# undef readahead
914# define readahead(fd,offset,count) eio__readahead (fd, offset, count, self) 998# define readahead(fd,offset,count) eio__readahead (fd, offset, count, self)
915 999
916static ssize_t 1000static eio_ssize_t
917eio__readahead (int fd, off_t offset, size_t count, etp_worker *self) 1001eio__readahead (int fd, off_t offset, size_t count, etp_worker *self)
918{ 1002{
919 size_t todo = count; 1003 size_t todo = count;
920 dBUF; 1004 dBUF;
921 1005
933} 1017}
934 1018
935#endif 1019#endif
936 1020
937/* sendfile always needs emulation */ 1021/* sendfile always needs emulation */
938static ssize_t 1022static eio_ssize_t
939eio__sendfile (int ofd, int ifd, off_t offset, size_t count, etp_worker *self) 1023eio__sendfile (int ofd, int ifd, off_t offset, size_t count, etp_worker *self)
940{ 1024{
1025 eio_ssize_t written = 0;
941 ssize_t res; 1026 eio_ssize_t res;
942 1027
943 if (!count) 1028 if (!count)
944 return 0; 1029 return 0;
945 1030
1031 for (;;)
1032 {
1033#ifdef __APPLE__
1034# undef HAVE_SENDFILE /* broken, as everything on os x */
1035#endif
946#if HAVE_SENDFILE 1036#if HAVE_SENDFILE
947# if __linux 1037# if __linux
1038 off_t soffset = offset;
948 res = sendfile (ofd, ifd, &offset, count); 1039 res = sendfile (ofd, ifd, &soffset, count);
949 1040
950# elif __FreeBSD__ 1041# elif __FreeBSD__
951 /* 1042 /*
952 * Of course, the freebsd sendfile is a dire hack with no thoughts 1043 * Of course, the freebsd sendfile is a dire hack with no thoughts
953 * wasted on making it similar to other I/O functions. 1044 * wasted on making it similar to other I/O functions.
954 */ 1045 */
955 {
956 off_t sbytes; 1046 off_t sbytes;
957 res = sendfile (ifd, ofd, offset, count, 0, &sbytes, 0); 1047 res = sendfile (ifd, ofd, offset, count, 0, &sbytes, 0);
958 1048
959 #if 0 /* according to the manpage, this is correct, but broken behaviour */ 1049 #if 0 /* according to the manpage, this is correct, but broken behaviour */
960 /* freebsd' sendfile will return 0 on success */ 1050 /* freebsd' sendfile will return 0 on success */
961 /* freebsd 8 documents it as only setting *sbytes on EINTR and EAGAIN, but */ 1051 /* freebsd 8 documents it as only setting *sbytes on EINTR and EAGAIN, but */
962 /* not on e.g. EIO or EPIPE - sounds broken */ 1052 /* not on e.g. EIO or EPIPE - sounds broken */
963 if ((res < 0 && (errno == EAGAIN || errno == EINTR) && sbytes) || res == 0) 1053 if ((res < 0 && (errno == EAGAIN || errno == EINTR) && sbytes) || res == 0)
964 res = sbytes; 1054 res = sbytes;
965 #endif 1055 #endif
966 1056
967 /* according to source inspection, this is correct, and useful behaviour */ 1057 /* according to source inspection, this is correct, and useful behaviour */
968 if (sbytes) 1058 if (sbytes)
969 res = sbytes; 1059 res = sbytes;
970 }
971 1060
972# elif defined (__APPLE__) 1061# elif defined (__APPLE__)
973
974 {
975 off_t sbytes = count; 1062 off_t sbytes = count;
976 res = sendfile (ifd, ofd, offset, &sbytes, 0, 0); 1063 res = sendfile (ifd, ofd, offset, &sbytes, 0, 0);
977 1064
978 /* according to the manpage, sbytes is always valid */ 1065 /* according to the manpage, sbytes is always valid */
979 if (sbytes) 1066 if (sbytes)
980 res = sbytes; 1067 res = sbytes;
981 }
982 1068
983# elif __hpux 1069# elif __hpux
984 res = sendfile (ofd, ifd, offset, count, 0, 0); 1070 res = sendfile (ofd, ifd, offset, count, 0, 0);
985 1071
986# elif __solaris 1072# elif __solaris
987 {
988 struct sendfilevec vec; 1073 struct sendfilevec vec;
989 size_t sbytes; 1074 size_t sbytes;
990 1075
991 vec.sfv_fd = ifd; 1076 vec.sfv_fd = ifd;
992 vec.sfv_flag = 0; 1077 vec.sfv_flag = 0;
993 vec.sfv_off = offset; 1078 vec.sfv_off = offset;
994 vec.sfv_len = count; 1079 vec.sfv_len = count;
995 1080
996 res = sendfilev (ofd, &vec, 1, &sbytes); 1081 res = sendfilev (ofd, &vec, 1, &sbytes);
997 1082
998 if (res < 0 && sbytes) 1083 if (res < 0 && sbytes)
999 res = sbytes; 1084 res = sbytes;
1000 }
1001 1085
1002# endif 1086# endif
1003 1087
1004#elif defined (_WIN32) 1088#elif defined (_WIN32) && 0
1005
1006 /* does not work, just for documentation of what would need to be done */ 1089 /* does not work, just for documentation of what would need to be done */
1007 { 1090 /* actually, cannot be done like this, as TransmitFile changes the file offset, */
1091 /* libeio guarantees that the file offset does not change, and windows */
1092 /* has no way to get an independent handle to the same file description */
1008 HANDLE h = TO_SOCKET (ifd); 1093 HANDLE h = TO_SOCKET (ifd);
1009 SetFilePointer (h, offset, 0, FILE_BEGIN); 1094 SetFilePointer (h, offset, 0, FILE_BEGIN);
1010 res = TransmitFile (TO_SOCKET (ofd), h, count, 0, 0, 0, 0); 1095 res = TransmitFile (TO_SOCKET (ofd), h, count, 0, 0, 0, 0);
1011 }
1012 1096
1013#else 1097#else
1014 res = -1; 1098 res = -1;
1015 errno = ENOSYS; 1099 errno = ENOSYS;
1016#endif 1100#endif
1017 1101
1102 /* we assume sendfile can copy at least 128mb in one go */
1103 if (res <= 128 * 1024 * 1024)
1104 {
1105 if (res > 0)
1106 written += res;
1107
1108 if (written)
1109 return written;
1110
1111 break;
1112 }
1113 else
1114 {
1115 /* if we requested more, then probably the kernel was lazy */
1116 written += res;
1117 offset += res;
1118 count -= res;
1119
1120 if (!count)
1121 return written;
1122 }
1123 }
1124
1018 if (res < 0 1125 if (res < 0
1019 && (errno == ENOSYS || errno == EINVAL || errno == ENOTSOCK 1126 && (errno == ENOSYS || errno == EINVAL || errno == ENOTSOCK
1020 /* BSDs */ 1127 /* BSDs */
1021#ifdef ENOTSUP /* sigh, if the steenking pile called openbsd would only try to at least compile posix code... */ 1128#ifdef ENOTSUP /* sigh, if the steenking pile called openbsd would only try to at least compile posix code... */
1022 || errno == ENOTSUP 1129 || errno == ENOTSUP
1023#endif 1130#endif
1033 1140
1034 res = 0; 1141 res = 0;
1035 1142
1036 while (count) 1143 while (count)
1037 { 1144 {
1038 ssize_t cnt; 1145 eio_ssize_t cnt;
1039 1146
1040 cnt = pread (ifd, eio_buf, count > EIO_BUFSIZE ? EIO_BUFSIZE : count, offset); 1147 cnt = pread (ifd, eio_buf, count > EIO_BUFSIZE ? EIO_BUFSIZE : count, offset);
1041 1148
1042 if (cnt <= 0) 1149 if (cnt <= 0)
1043 { 1150 {
1058 count -= cnt; 1165 count -= cnt;
1059 } 1166 }
1060 } 1167 }
1061 1168
1062 return res; 1169 return res;
1170}
1171
1172#ifdef PAGESIZE
1173# define eio_pagesize() PAGESIZE
1174#else
1175static intptr_t
1176eio_pagesize (void)
1177{
1178 static intptr_t page;
1179
1180 if (!page)
1181 page = sysconf (_SC_PAGESIZE);
1182
1183 return page;
1184}
1185#endif
1186
1187static void
1188eio_page_align (void **addr, size_t *length)
1189{
1190 intptr_t mask = eio_pagesize () - 1;
1191
1192 /* round down addr */
1193 intptr_t adj = mask & (intptr_t)*addr;
1194
1195 *addr = (void *)((intptr_t)*addr - adj);
1196 *length += adj;
1197
1198 /* round up length */
1199 *length = (*length + mask) & ~mask;
1200}
1201
1202#if !_POSIX_MEMLOCK
1203# define eio__mlockall(a) EIO_ENOSYS ()
1204#else
1205
1206static int
1207eio__mlockall (int flags)
1208{
1209 #if __GLIBC__ == 2 && __GLIBC_MINOR__ <= 7
1210 extern int mallopt (int, int);
1211 mallopt (-6, 238); /* http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=473812 */
1212 #endif
1213
1214 if (EIO_MCL_CURRENT != MCL_CURRENT
1215 || EIO_MCL_FUTURE != MCL_FUTURE)
1216 {
1217 flags = 0
1218 | (flags & EIO_MCL_CURRENT ? MCL_CURRENT : 0)
1219 | (flags & EIO_MCL_FUTURE ? MCL_FUTURE : 0);
1220 }
1221
1222 return mlockall (flags);
1223}
1224#endif
1225
1226#if !_POSIX_MEMLOCK_RANGE
1227# define eio__mlock(a,b) EIO_ENOSYS ()
1228#else
1229
1230static int
1231eio__mlock (void *addr, size_t length)
1232{
1233 eio_page_align (&addr, &length);
1234
1235 return mlock (addr, length);
1236}
1237
1238#endif
1239
1240#if !(_POSIX_MAPPED_FILES && _POSIX_SYNCHRONIZED_IO)
1241# define eio__msync(a,b,c) EIO_ENOSYS ()
1242#else
1243
1244static int
1245eio__msync (void *mem, size_t len, int flags)
1246{
1247 eio_page_align (&mem, &len);
1248
1249 if (EIO_MS_ASYNC != MS_SYNC
1250 || EIO_MS_INVALIDATE != MS_INVALIDATE
1251 || EIO_MS_SYNC != MS_SYNC)
1252 {
1253 flags = 0
1254 | (flags & EIO_MS_ASYNC ? MS_ASYNC : 0)
1255 | (flags & EIO_MS_INVALIDATE ? MS_INVALIDATE : 0)
1256 | (flags & EIO_MS_SYNC ? MS_SYNC : 0);
1257 }
1258
1259 return msync (mem, len, flags);
1260}
1261
1262#endif
1263
1264static int
1265eio__mtouch (eio_req *req)
1266{
1267 void *mem = req->ptr2;
1268 size_t len = req->size;
1269 int flags = req->int1;
1270
1271 eio_page_align (&mem, &len);
1272
1273 {
1274 intptr_t addr = (intptr_t)mem;
1275 intptr_t end = addr + len;
1276 intptr_t page = eio_pagesize ();
1277
1278 if (addr < end)
1279 if (flags & EIO_MT_MODIFY) /* modify */
1280 do { *((volatile sig_atomic_t *)addr) |= 0; } while ((addr += page) < len && !EIO_CANCELLED (req));
1281 else
1282 do { *((volatile sig_atomic_t *)addr) ; } while ((addr += page) < len && !EIO_CANCELLED (req));
1283 }
1284
1285 return 0;
1286}
1287
1288/*****************************************************************************/
1289/* requests implemented outside eio_execute, because they are so large */
1290
1291static void
1292eio__realpath (eio_req *req, etp_worker *self)
1293{
1294 char *rel = req->ptr1;
1295 char *res;
1296 char *tmp1, *tmp2;
1297#if SYMLOOP_MAX > 32
1298 int symlinks = SYMLOOP_MAX;
1299#else
1300 int symlinks = 32;
1301#endif
1302
1303 req->result = -1;
1304
1305 errno = EINVAL;
1306 if (!rel)
1307 return;
1308
1309 errno = ENOENT;
1310 if (!*rel)
1311 return;
1312
1313 if (!req->ptr2)
1314 {
1315 X_LOCK (wrklock);
1316 req->flags |= EIO_FLAG_PTR2_FREE;
1317 X_UNLOCK (wrklock);
1318 req->ptr2 = malloc (PATH_MAX * 3);
1319
1320 errno = ENOMEM;
1321 if (!req->ptr2)
1322 return;
1323 }
1324
1325 res = req->ptr2;
1326 tmp1 = res + PATH_MAX;
1327 tmp2 = tmp1 + PATH_MAX;
1328
1329#if 0 /* disabled, the musl way to do things is just too racy */
1330#if __linux && defined(O_NONBLOCK) && defined(O_NOATIME)
1331 /* on linux we may be able to ask the kernel */
1332 {
1333 int fd = open (rel, O_RDONLY | O_NONBLOCK | O_NOCTTY | O_NOATIME);
1334
1335 if (fd >= 0)
1336 {
1337 sprintf (tmp1, "/proc/self/fd/%d", fd);
1338 req->result = readlink (tmp1, res, PATH_MAX);
1339 close (fd);
1340
1341 /* here we should probably stat the open file and the disk file, to make sure they still match */
1342
1343 if (req->result > 0)
1344 goto done;
1345 }
1346 else if (errno == ELOOP || errno == ENAMETOOLONG || errno == ENOENT || errno == ENOTDIR || errno == EIO)
1347 return;
1348 }
1349#endif
1350#endif
1351
1352 if (*rel != '/')
1353 {
1354 if (!getcwd (res, PATH_MAX))
1355 return;
1356
1357 if (res [1]) /* only use if not / */
1358 res += strlen (res);
1359 }
1360
1361 while (*rel)
1362 {
1363 eio_ssize_t len, linklen;
1364 char *beg = rel;
1365
1366 while (*rel && *rel != '/')
1367 ++rel;
1368
1369 len = rel - beg;
1370
1371 if (!len) /* skip slashes */
1372 {
1373 ++rel;
1374 continue;
1375 }
1376
1377 if (beg [0] == '.')
1378 {
1379 if (len == 1)
1380 continue; /* . - nop */
1381
1382 if (beg [1] == '.' && len == 2)
1383 {
1384 /* .. - back up one component, if possible */
1385
1386 while (res != req->ptr2)
1387 if (*--res == '/')
1388 break;
1389
1390 continue;
1391 }
1392 }
1393
1394 errno = ENAMETOOLONG;
1395 if (res + 1 + len + 1 >= tmp1)
1396 return;
1397
1398 /* copy one component */
1399 *res = '/';
1400 memcpy (res + 1, beg, len);
1401
1402 /* zero-terminate, for readlink */
1403 res [len + 1] = 0;
1404
1405 /* now check if it's a symlink */
1406 linklen = readlink (req->ptr2, tmp1, PATH_MAX);
1407
1408 if (linklen < 0)
1409 {
1410 if (errno != EINVAL)
1411 return;
1412
1413 /* it's a normal directory. hopefully */
1414 res += len + 1;
1415 }
1416 else
1417 {
1418 /* yay, it was a symlink - build new path in tmp2 */
1419 int rellen = strlen (rel);
1420
1421 errno = ENAMETOOLONG;
1422 if (linklen + 1 + rellen >= PATH_MAX)
1423 return;
1424
1425 errno = ELOOP;
1426 if (!--symlinks)
1427 return;
1428
1429 if (*tmp1 == '/')
1430 res = req->ptr2; /* symlink resolves to an absolute path */
1431
1432 /* we need to be careful, as rel might point into tmp2 already */
1433 memmove (tmp2 + linklen + 1, rel, rellen + 1);
1434 tmp2 [linklen] = '/';
1435 memcpy (tmp2, tmp1, linklen);
1436
1437 rel = tmp2;
1438 }
1439 }
1440
1441 /* special case for the lone root path */
1442 if (res == req->ptr2)
1443 *res++ = '/';
1444
1445 req->result = res - (char *)req->ptr2;
1446
1447done:
1448 req->ptr2 = realloc (req->ptr2, req->result); /* trade time for space savings */
1063} 1449}
1064 1450
1065static signed char 1451static signed char
1066eio_dent_cmp (const eio_dirent *a, const eio_dirent *b) 1452eio_dent_cmp (const eio_dirent *a, const eio_dirent *b)
1067{ 1453{
1249 1635
1250 X_LOCK (wrklock); 1636 X_LOCK (wrklock);
1251 /* the corresponding closedir is in ETP_WORKER_CLEAR */ 1637 /* the corresponding closedir is in ETP_WORKER_CLEAR */
1252 self->dirp = dirp = opendir (req->ptr1); 1638 self->dirp = dirp = opendir (req->ptr1);
1253 1639
1640 if (req->flags & EIO_FLAG_PTR1_FREE)
1641 free (req->ptr1);
1642
1254 req->flags |= EIO_FLAG_PTR1_FREE | EIO_FLAG_PTR2_FREE; 1643 req->flags |= EIO_FLAG_PTR1_FREE | EIO_FLAG_PTR2_FREE;
1255 req->ptr1 = dents = flags ? malloc (dentalloc * sizeof (eio_dirent)) : 0; 1644 req->ptr1 = dents = flags ? malloc (dentalloc * sizeof (eio_dirent)) : 0;
1256 req->ptr2 = names = malloc (namesalloc); 1645 req->ptr2 = names = malloc (namesalloc);
1257 X_UNLOCK (wrklock); 1646 X_UNLOCK (wrklock);
1258 1647
1309 /* skip . and .. entries */ 1698 /* skip . and .. entries */
1310 if (name [0] != '.' || (name [1] && (name [1] != '.' || name [2]))) 1699 if (name [0] != '.' || (name [1] && (name [1] != '.' || name [2])))
1311 { 1700 {
1312 int len = D_NAMLEN (entp) + 1; 1701 int len = D_NAMLEN (entp) + 1;
1313 1702
1314 while (expect_false (namesoffs + len > namesalloc)) 1703 while (ecb_expect_false (namesoffs + len > namesalloc))
1315 { 1704 {
1316 namesalloc *= 2; 1705 namesalloc *= 2;
1317 X_LOCK (wrklock); 1706 X_LOCK (wrklock);
1318 req->ptr2 = names = realloc (names, namesalloc); 1707 req->ptr2 = names = realloc (names, namesalloc);
1319 X_UNLOCK (wrklock); 1708 X_UNLOCK (wrklock);
1326 1715
1327 if (dents) 1716 if (dents)
1328 { 1717 {
1329 struct eio_dirent *ent; 1718 struct eio_dirent *ent;
1330 1719
1331 if (expect_false (dentoffs == dentalloc)) 1720 if (ecb_expect_false (dentoffs == dentalloc))
1332 { 1721 {
1333 dentalloc *= 2; 1722 dentalloc *= 2;
1334 X_LOCK (wrklock); 1723 X_LOCK (wrklock);
1335 req->ptr1 = dents = realloc (dents, dentalloc * sizeof (eio_dirent)); 1724 req->ptr1 = dents = realloc (dents, dentalloc * sizeof (eio_dirent));
1336 X_UNLOCK (wrklock); 1725 X_UNLOCK (wrklock);
1424 break; 1813 break;
1425 } 1814 }
1426 } 1815 }
1427} 1816}
1428 1817
1429#ifdef PAGESIZE
1430# define eio_pagesize() PAGESIZE
1431#else
1432static intptr_t
1433eio_pagesize (void)
1434{
1435 static intptr_t page;
1436
1437 if (!page)
1438 page = sysconf (_SC_PAGESIZE);
1439
1440 return page;
1441}
1442#endif
1443
1444static void
1445eio_page_align (void **addr, size_t *length)
1446{
1447 intptr_t mask = eio_pagesize () - 1;
1448
1449 /* round down addr */
1450 intptr_t adj = mask & (intptr_t)*addr;
1451
1452 *addr = (void *)((intptr_t)*addr - adj);
1453 *length += adj;
1454
1455 /* round up length */
1456 *length = (*length + mask) & ~mask;
1457}
1458
1459#if !_POSIX_MEMLOCK
1460# define eio__mlockall(a) ((errno = ENOSYS), -1)
1461#else
1462
1463static int
1464eio__mlockall (int flags)
1465{
1466 #if __GLIBC__ == 2 && __GLIBC_MINOR__ <= 7
1467 extern int mallopt (int, int);
1468 mallopt (-6, 238); /* http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=473812 */
1469 #endif
1470
1471 if (EIO_MCL_CURRENT != MCL_CURRENT
1472 || EIO_MCL_FUTURE != MCL_FUTURE)
1473 {
1474 flags = 0
1475 | (flags & EIO_MCL_CURRENT ? MCL_CURRENT : 0)
1476 | (flags & EIO_MCL_FUTURE ? MCL_FUTURE : 0);
1477 }
1478
1479 return mlockall (flags);
1480}
1481#endif
1482
1483#if !_POSIX_MEMLOCK_RANGE
1484# define eio__mlock(a,b) ((errno = ENOSYS), -1)
1485#else
1486
1487static int
1488eio__mlock (void *addr, size_t length)
1489{
1490 eio_page_align (&addr, &length);
1491
1492 return mlock (addr, length);
1493}
1494
1495#endif
1496
1497#if !(_POSIX_MAPPED_FILES && _POSIX_SYNCHRONIZED_IO)
1498# define eio__msync(a,b,c) ((errno = ENOSYS), -1)
1499#else
1500
1501int
1502eio__msync (void *mem, size_t len, int flags)
1503{
1504 eio_page_align (&mem, &len);
1505
1506 if (EIO_MS_ASYNC != MS_SYNC
1507 || EIO_MS_INVALIDATE != MS_INVALIDATE
1508 || EIO_MS_SYNC != MS_SYNC)
1509 {
1510 flags = 0
1511 | (flags & EIO_MS_ASYNC ? MS_ASYNC : 0)
1512 | (flags & EIO_MS_INVALIDATE ? MS_INVALIDATE : 0)
1513 | (flags & EIO_MS_SYNC ? MS_SYNC : 0);
1514 }
1515
1516 return msync (mem, len, flags);
1517}
1518
1519#endif
1520
1521int
1522eio__mtouch (void *mem, size_t len, int flags)
1523{
1524 eio_page_align (&mem, &len);
1525
1526 {
1527 intptr_t addr = (intptr_t)mem;
1528 intptr_t end = addr + len;
1529 intptr_t page = eio_pagesize ();
1530
1531 if (addr < end)
1532 if (flags & EIO_MT_MODIFY) /* modify */
1533 do { *((volatile sig_atomic_t *)addr) |= 0; } while ((addr += page) < len);
1534 else
1535 do { *((volatile sig_atomic_t *)addr) ; } while ((addr += page) < len);
1536 }
1537
1538 return 0;
1539}
1540
1541/*****************************************************************************/ 1818/*****************************************************************************/
1542 1819
1543#define ALLOC(len) \ 1820#define ALLOC(len) \
1544 if (!req->ptr2) \ 1821 if (!req->ptr2) \
1545 { \ 1822 { \
1558X_THREAD_PROC (etp_proc) 1835X_THREAD_PROC (etp_proc)
1559{ 1836{
1560 ETP_REQ *req; 1837 ETP_REQ *req;
1561 struct timespec ts; 1838 struct timespec ts;
1562 etp_worker *self = (etp_worker *)thr_arg; 1839 etp_worker *self = (etp_worker *)thr_arg;
1840 int timeout;
1563 1841
1564 /* try to distribute timeouts somewhat randomly */ 1842 /* try to distribute timeouts somewhat evenly */
1565 ts.tv_nsec = ((unsigned long)self & 1023UL) * (1000000000UL / 1024UL); 1843 ts.tv_nsec = ((unsigned long)self & 1023UL) * (1000000000UL / 1024UL);
1566 1844
1567 for (;;) 1845 for (;;)
1568 { 1846 {
1847 ts.tv_sec = 0;
1848
1569 X_LOCK (reqlock); 1849 X_LOCK (reqlock);
1570 1850
1571 for (;;) 1851 for (;;)
1572 { 1852 {
1573 self->req = req = reqq_shift (&req_queue); 1853 self->req = req = reqq_shift (&req_queue);
1574 1854
1575 if (req) 1855 if (req)
1576 break; 1856 break;
1577 1857
1858 if (ts.tv_sec == 1) /* no request, but timeout detected, let's quit */
1859 {
1860 X_UNLOCK (reqlock);
1861 X_LOCK (wrklock);
1862 --started;
1863 X_UNLOCK (wrklock);
1864 goto quit;
1865 }
1866
1578 ++idle; 1867 ++idle;
1579 1868
1580 ts.tv_sec = time (0) + idle_timeout; 1869 if (idle <= max_idle)
1581 if (X_COND_TIMEDWAIT (reqwait, reqlock, ts) == ETIMEDOUT) 1870 /* we are allowed to idle, so do so without any timeout */
1871 X_COND_WAIT (reqwait, reqlock);
1872 else
1582 { 1873 {
1583 if (idle > max_idle) 1874 /* initialise timeout once */
1584 { 1875 if (!ts.tv_sec)
1585 --idle; 1876 ts.tv_sec = time (0) + idle_timeout;
1586 X_UNLOCK (reqlock);
1587 X_LOCK (wrklock);
1588 --started;
1589 X_UNLOCK (wrklock);
1590 goto quit;
1591 }
1592 1877
1593 /* we are allowed to idle, so do so without any timeout */
1594 X_COND_WAIT (reqwait, reqlock); 1878 if (X_COND_TIMEDWAIT (reqwait, reqlock, ts) == ETIMEDOUT)
1879 ts.tv_sec = 1; /* assuming this is not a value computed above.,.. */
1595 } 1880 }
1596 1881
1597 --idle; 1882 --idle;
1598 } 1883 }
1599 1884
1602 X_UNLOCK (reqlock); 1887 X_UNLOCK (reqlock);
1603 1888
1604 if (req->type < 0) 1889 if (req->type < 0)
1605 goto quit; 1890 goto quit;
1606 1891
1607 if (!EIO_CANCELLED (req))
1608 ETP_EXECUTE (self, req); 1892 ETP_EXECUTE (self, req);
1609 1893
1610 X_LOCK (reslock); 1894 X_LOCK (reslock);
1611 1895
1612 ++npending; 1896 ++npending;
1613 1897
1628 return 0; 1912 return 0;
1629} 1913}
1630 1914
1631/*****************************************************************************/ 1915/*****************************************************************************/
1632 1916
1917int ecb_cold
1633int eio_init (void (*want_poll)(void), void (*done_poll)(void)) 1918eio_init (void (*want_poll)(void), void (*done_poll)(void))
1634{ 1919{
1635 return etp_init (want_poll, done_poll); 1920 return etp_init (want_poll, done_poll);
1636} 1921}
1637 1922
1923ecb_inline void
1638static void eio_api_destroy (eio_req *req) 1924eio_api_destroy (eio_req *req)
1639{ 1925{
1640 free (req); 1926 free (req);
1641} 1927}
1642 1928
1643#define REQ(rtype) \ 1929#define REQ(rtype) \
1662 { \ 1948 { \
1663 eio_api_destroy (req); \ 1949 eio_api_destroy (req); \
1664 return 0; \ 1950 return 0; \
1665 } 1951 }
1666 1952
1953static void
1667static void eio_execute (etp_worker *self, eio_req *req) 1954eio_execute (etp_worker *self, eio_req *req)
1668{ 1955{
1956 if (ecb_expect_false (EIO_CANCELLED (req)))
1957 {
1958 req->result = -1;
1959 req->errorno = ECANCELED;
1960 return;
1961 }
1962
1669 switch (req->type) 1963 switch (req->type)
1670 { 1964 {
1671 case EIO_READ: ALLOC (req->size); 1965 case EIO_READ: ALLOC (req->size);
1672 req->result = req->offs >= 0 1966 req->result = req->offs >= 0
1673 ? pread (req->int1, req->ptr2, req->size, req->offs) 1967 ? pread (req->int1, req->ptr2, req->size, req->offs)
1707 case EIO_RENAME: req->result = rename (req->ptr1, req->ptr2); break; 2001 case EIO_RENAME: req->result = rename (req->ptr1, req->ptr2); break;
1708 case EIO_LINK: req->result = link (req->ptr1, req->ptr2); break; 2002 case EIO_LINK: req->result = link (req->ptr1, req->ptr2); break;
1709 case EIO_SYMLINK: req->result = symlink (req->ptr1, req->ptr2); break; 2003 case EIO_SYMLINK: req->result = symlink (req->ptr1, req->ptr2); break;
1710 case EIO_MKNOD: req->result = mknod (req->ptr1, (mode_t)req->int2, (dev_t)req->offs); break; 2004 case EIO_MKNOD: req->result = mknod (req->ptr1, (mode_t)req->int2, (dev_t)req->offs); break;
1711 2005
2006 case EIO_REALPATH: eio__realpath (req, self); break;
2007
1712 case EIO_READLINK: ALLOC (PATH_MAX); 2008 case EIO_READLINK: ALLOC (PATH_MAX);
1713 req->result = readlink (req->ptr1, req->ptr2, PATH_MAX); break; 2009 req->result = readlink (req->ptr1, req->ptr2, PATH_MAX); break;
1714 2010
1715 case EIO_SYNC: req->result = 0; sync (); break; 2011 case EIO_SYNC: req->result = 0; sync (); break;
1716 case EIO_FSYNC: req->result = fsync (req->int1); break; 2012 case EIO_FSYNC: req->result = fsync (req->int1); break;
1717 case EIO_FDATASYNC: req->result = fdatasync (req->int1); break; 2013 case EIO_FDATASYNC: req->result = fdatasync (req->int1); break;
1718 case EIO_MSYNC: req->result = eio__msync (req->ptr2, req->size, req->int1); break; 2014 case EIO_MSYNC: req->result = eio__msync (req->ptr2, req->size, req->int1); break;
1719 case EIO_MTOUCH: req->result = eio__mtouch (req->ptr2, req->size, req->int1); break; 2015 case EIO_MTOUCH: req->result = eio__mtouch (req); break;
1720 case EIO_MLOCK: req->result = eio__mlock (req->ptr2, req->size); break; 2016 case EIO_MLOCK: req->result = eio__mlock (req->ptr2, req->size); break;
1721 case EIO_MLOCKALL: req->result = eio__mlockall (req->int1); break; 2017 case EIO_MLOCKALL: req->result = eio__mlockall (req->int1); break;
1722 case EIO_SYNC_FILE_RANGE: req->result = eio__sync_file_range (req->int1, req->offs, req->size, req->int2); break; 2018 case EIO_SYNC_FILE_RANGE: req->result = eio__sync_file_range (req->int1, req->offs, req->size, req->int2); break;
2019 case EIO_FALLOCATE: req->result = eio__fallocate (req->int1, req->int2, req->offs, req->size); break;
1723 2020
1724 case EIO_READDIR: eio__scandir (req, self); break; 2021 case EIO_READDIR: eio__scandir (req, self); break;
1725 2022
1726 case EIO_BUSY: 2023 case EIO_BUSY:
1727#ifdef _WIN32 2024#ifdef _WIN32
1768 case EIO_NOP: 2065 case EIO_NOP:
1769 req->result = 0; 2066 req->result = 0;
1770 break; 2067 break;
1771 2068
1772 case EIO_CUSTOM: 2069 case EIO_CUSTOM:
1773 ((void (*)(eio_req *))req->feed) (req); 2070 req->feed (req);
1774 break; 2071 break;
1775 2072
1776 default: 2073 default:
1777 errno = ENOSYS; 2074 errno = ENOSYS;
1778 req->result = -1; 2075 req->result = -1;
1827eio_req *eio_sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags, int pri, eio_cb cb, void *data) 2124eio_req *eio_sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags, int pri, eio_cb cb, void *data)
1828{ 2125{
1829 REQ (EIO_SYNC_FILE_RANGE); req->int1 = fd; req->offs = offset; req->size = nbytes; req->int2 = flags; SEND; 2126 REQ (EIO_SYNC_FILE_RANGE); req->int1 = fd; req->offs = offset; req->size = nbytes; req->int2 = flags; SEND;
1830} 2127}
1831 2128
2129eio_req *eio_fallocate (int fd, int mode, off_t offset, size_t len, int pri, eio_cb cb, void *data)
2130{
2131 REQ (EIO_FALLOCATE); req->int1 = fd; req->int2 = mode; req->offs = offset; req->size = len; SEND;
2132}
2133
1832eio_req *eio_fdatasync (int fd, int pri, eio_cb cb, void *data) 2134eio_req *eio_fdatasync (int fd, int pri, eio_cb cb, void *data)
1833{ 2135{
1834 REQ (EIO_FDATASYNC); req->int1 = fd; SEND; 2136 REQ (EIO_FDATASYNC); req->int1 = fd; SEND;
1835} 2137}
1836 2138
1877eio_req *eio_fchmod (int fd, mode_t mode, int pri, eio_cb cb, void *data) 2179eio_req *eio_fchmod (int fd, mode_t mode, int pri, eio_cb cb, void *data)
1878{ 2180{
1879 REQ (EIO_FCHMOD); req->int1 = fd; req->int2 = (long)mode; SEND; 2181 REQ (EIO_FCHMOD); req->int1 = fd; req->int2 = (long)mode; SEND;
1880} 2182}
1881 2183
1882eio_req *eio_fchown (int fd, uid_t uid, gid_t gid, int pri, eio_cb cb, void *data) 2184eio_req *eio_fchown (int fd, eio_uid_t uid, eio_gid_t gid, int pri, eio_cb cb, void *data)
1883{ 2185{
1884 REQ (EIO_FCHOWN); req->int1 = fd; req->int2 = (long)uid; req->int3 = (long)gid; SEND; 2186 REQ (EIO_FCHOWN); req->int1 = fd; req->int2 = (long)uid; req->int3 = (long)gid; SEND;
1885} 2187}
1886 2188
1887eio_req *eio_dup2 (int fd, int fd2, int pri, eio_cb cb, void *data) 2189eio_req *eio_dup2 (int fd, int fd2, int pri, eio_cb cb, void *data)
1907eio_req *eio_truncate (const char *path, off_t offset, int pri, eio_cb cb, void *data) 2209eio_req *eio_truncate (const char *path, off_t offset, int pri, eio_cb cb, void *data)
1908{ 2210{
1909 REQ (EIO_TRUNCATE); PATH; req->offs = offset; SEND; 2211 REQ (EIO_TRUNCATE); PATH; req->offs = offset; SEND;
1910} 2212}
1911 2213
1912eio_req *eio_chown (const char *path, uid_t uid, gid_t gid, int pri, eio_cb cb, void *data) 2214eio_req *eio_chown (const char *path, eio_uid_t uid, eio_gid_t gid, int pri, eio_cb cb, void *data)
1913{ 2215{
1914 REQ (EIO_CHOWN); PATH; req->int2 = (long)uid; req->int3 = (long)gid; SEND; 2216 REQ (EIO_CHOWN); PATH; req->int2 = (long)uid; req->int3 = (long)gid; SEND;
1915} 2217}
1916 2218
1917eio_req *eio_chmod (const char *path, mode_t mode, int pri, eio_cb cb, void *data) 2219eio_req *eio_chmod (const char *path, mode_t mode, int pri, eio_cb cb, void *data)
1931} 2233}
1932 2234
1933eio_req *eio_readlink (const char *path, int pri, eio_cb cb, void *data) 2235eio_req *eio_readlink (const char *path, int pri, eio_cb cb, void *data)
1934{ 2236{
1935 return eio__1path (EIO_READLINK, path, pri, cb, data); 2237 return eio__1path (EIO_READLINK, path, pri, cb, data);
2238}
2239
2240eio_req *eio_realpath (const char *path, int pri, eio_cb cb, void *data)
2241{
2242 return eio__1path (EIO_REALPATH, path, pri, cb, data);
1936} 2243}
1937 2244
1938eio_req *eio_stat (const char *path, int pri, eio_cb cb, void *data) 2245eio_req *eio_stat (const char *path, int pri, eio_cb cb, void *data)
1939{ 2246{
1940 return eio__1path (EIO_STAT, path, pri, cb, data); 2247 return eio__1path (EIO_STAT, path, pri, cb, data);
1999eio_req *eio_rename (const char *path, const char *new_path, int pri, eio_cb cb, void *data) 2306eio_req *eio_rename (const char *path, const char *new_path, int pri, eio_cb cb, void *data)
2000{ 2307{
2001 return eio__2path (EIO_RENAME, path, new_path, pri, cb, data); 2308 return eio__2path (EIO_RENAME, path, new_path, pri, cb, data);
2002} 2309}
2003 2310
2004eio_req *eio_custom (eio_cb execute, int pri, eio_cb cb, void *data) 2311eio_req *eio_custom (void (*execute)(eio_req *), int pri, eio_cb cb, void *data)
2005{ 2312{
2006 REQ (EIO_CUSTOM); req->feed = (void (*)(eio_req *))execute; SEND; 2313 REQ (EIO_CUSTOM); req->feed = execute; SEND;
2007} 2314}
2008 2315
2009#endif 2316#endif
2010 2317
2011eio_req *eio_grp (eio_cb cb, void *data) 2318eio_req *eio_grp (eio_cb cb, void *data)
2020#undef SEND 2327#undef SEND
2021 2328
2022/*****************************************************************************/ 2329/*****************************************************************************/
2023/* grp functions */ 2330/* grp functions */
2024 2331
2332void
2025void eio_grp_feed (eio_req *grp, void (*feed)(eio_req *req), int limit) 2333eio_grp_feed (eio_req *grp, void (*feed)(eio_req *req), int limit)
2026{ 2334{
2027 grp->int2 = limit; 2335 grp->int2 = limit;
2028 grp->feed = feed; 2336 grp->feed = feed;
2029 2337
2030 grp_try_feed (grp); 2338 grp_try_feed (grp);
2031} 2339}
2032 2340
2341void
2033void eio_grp_limit (eio_req *grp, int limit) 2342eio_grp_limit (eio_req *grp, int limit)
2034{ 2343{
2035 grp->int2 = limit; 2344 grp->int2 = limit;
2036 2345
2037 grp_try_feed (grp); 2346 grp_try_feed (grp);
2038} 2347}
2039 2348
2349void
2040void eio_grp_add (eio_req *grp, eio_req *req) 2350eio_grp_add (eio_req *grp, eio_req *req)
2041{ 2351{
2042 assert (("cannot add requests to IO::AIO::GRP after the group finished", grp->int1 != 2)); 2352 assert (("cannot add requests to IO::AIO::GRP after the group finished", grp->int1 != 2));
2043 2353
2044 grp->flags |= EIO_FLAG_GROUPADD; 2354 grp->flags |= EIO_FLAG_GROUPADD;
2045 2355
2056} 2366}
2057 2367
2058/*****************************************************************************/ 2368/*****************************************************************************/
2059/* misc garbage */ 2369/* misc garbage */
2060 2370
2371eio_ssize_t
2061ssize_t eio_sendfile_sync (int ofd, int ifd, off_t offset, size_t count) 2372eio_sendfile_sync (int ofd, int ifd, off_t offset, size_t count)
2062{ 2373{
2063 etp_worker wrk; 2374 etp_worker wrk;
2064 ssize_t ret; 2375 eio_ssize_t ret;
2065 2376
2066 wrk.dbuf = 0; 2377 wrk.dbuf = 0;
2067 2378
2068 ret = eio__sendfile (ofd, ifd, offset, count, &wrk); 2379 ret = eio__sendfile (ofd, ifd, offset, count, &wrk);
2069 2380

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines