ViewVC Help

View File | Revision Log | Show Annotations | Download File

/cvs/cvsroot/libeio/eio.c

Comparing cvsroot/libeio/eio.c (file contents):
Revision 1.61 by root, Fri Feb 11 00:53:24 2011 UTC vs.
Revision 1.105 by root, Mon Sep 26 18:23:18 2011 UTC

…		…
35	* and other provisions required by the GPL. If you do not delete the	35	* and other provisions required by the GPL. If you do not delete the
36	* provisions above, a recipient may use your version of this file under	36	* provisions above, a recipient may use your version of this file under
37	* either the BSD or the GPL.	37	* either the BSD or the GPL.
38	*/	38	*/
39		39
		40	#ifndef _WIN32
		41	# include "config.h"
		42	#endif
		43
40	#include "eio.h"	44	#include "eio.h"
		45	#include "ecb.h"
41		46
42	#ifdef EIO_STACKSIZE	47	#ifdef EIO_STACKSIZE
43	# define XTHREAD_STACKSIZE EIO_STACKSIZE	48	# define X_STACKSIZE EIO_STACKSIZE
44	#endif	49	#endif
45	#include "xthread.h"	50	#include "xthread.h"
46		51
47	#include <errno.h>	52	#include <errno.h>
48	#include <stddef.h>	53	#include <stddef.h>
49	#include <stdlib.h>	54	#include <stdlib.h>
50	#include <string.h>	55	#include <string.h>
51	#include <errno.h>	56	#include <errno.h>
52	#include <sys/types.h>	57	#include <sys/types.h>
53	#include <sys/stat.h>	58	#include <sys/stat.h>
54	#include <sys/statvfs.h>
55	#include <limits.h>	59	#include <limits.h>
56	#include <fcntl.h>	60	#include <fcntl.h>
57	#include <assert.h>	61	#include <assert.h>
58		62
		63	/* intptr_t comes from unistd.h, says POSIX/UNIX/tradition */
		64	/* intptr_t only comes from stdint.h, says idiot openbsd coder */
		65	#if HAVE_STDINT_H
		66	# include <stdint.h>
		67	#endif
		68
		69	#ifndef ECANCELED
		70	# define ECANCELED EDOM
		71	#endif
		72	#ifndef ELOOP
		73	# define ELOOP EDOM
		74	#endif
		75
		76	#if !defined(ENOTSOCK) && defined(WSAENOTSOCK)
		77	# define ENOTSOCK WSAENOTSOCK
		78	#endif
		79
		80	static void eio_destroy (eio_req *req);
		81
59	#ifndef EIO_FINISH	82	#ifndef EIO_FINISH
60	# define EIO_FINISH(req) ((req)->finish) && !EIO_CANCELLED (req) ? (req)->finish (req) : 0	83	# define EIO_FINISH(req) ((req)->finish) && !EIO_CANCELLED (req) ? (req)->finish (req) : 0
61	#endif	84	#endif
62		85
63	#ifndef EIO_DESTROY	86	#ifndef EIO_DESTROY
…		…
66		89
67	#ifndef EIO_FEED	90	#ifndef EIO_FEED
68	# define EIO_FEED(req) do { if ((req)->feed ) (req)->feed (req); } while (0)	91	# define EIO_FEED(req) do { if ((req)->feed ) (req)->feed (req); } while (0)
69	#endif	92	#endif
70		93
		94	#ifndef EIO_FD_TO_WIN32_HANDLE
		95	# define EIO_FD_TO_WIN32_HANDLE(fd) _get_osfhandle (fd)
		96	#endif
		97	#ifndef EIO_WIN32_HANDLE_TO_FD
		98	# define EIO_WIN32_HANDLE_TO_FD(handle) _open_osfhandle (handle, 0)
		99	#endif
		100
		101	#define EIO_ERRNO(errval,retval) ((errno = errval), retval)
		102
		103	#define EIO_ENOSYS() EIO_ERRNO (ENOSYS, -1)
		104
71	#ifdef _WIN32	105	#ifdef _WIN32
72		106
73	/*doh*/	107	#undef PAGESIZE
		108	#define PAGESIZE 4096 /* GetSystemInfo? */
		109
		110	/* TODO: look at how perl does stat (non-sloppy), unlink (ro-files), utime, link */
		111
		112	#ifdef EIO_STRUCT_STATI64
		113	/* look at perl's non-sloppy stat */
		114	#define stat(path,buf) _stati64 (path,buf)
		115	#define fstat(fd,buf) _fstati64 (fd,buf)
		116	#endif
		117	#define lstat(path,buf) stat (path,buf)
		118	#define fsync(fd) (FlushFileBuffers ((HANDLE)EIO_FD_TO_WIN32_HANDLE (fd)) ? 0 : EIO_ERRNO (EBADF, -1))
		119	#define mkdir(path,mode) _mkdir (path)
		120	#define link(old,neu) (CreateHardLink (neu, old, 0) ? 0 : EIO_ERRNO (ENOENT, -1))
		121
		122	#define chmod(path,mode) _chmod (path, mode)
		123	#define dup(fd) _dup (fd)
		124	#define dup2(fd1,fd2) _dup2 (fd1, fd2)
		125
		126	#define fchmod(fd,mode) EIO_ENOSYS ()
		127	#define chown(path,uid,gid) EIO_ENOSYS ()
		128	#define fchown(fd,uid,gid) EIO_ENOSYS ()
		129	#define truncate(path,offs) EIO_ENOSYS () /* far-miss: SetEndOfFile */
		130	#define ftruncate(fd,offs) EIO_ENOSYS () /* near-miss: SetEndOfFile */
		131	#define mknod(path,mode,dev) EIO_ENOSYS ()
		132	#define sync() EIO_ENOSYS ()
		133	#define readlink(path,buf,s) EIO_ENOSYS ()
		134	#define statvfs(path,buf) EIO_ENOSYS ()
		135	#define fstatvfs(fd,buf) EIO_ENOSYS ()
		136
		137	/* rename() uses MoveFile, which fails to overwrite */
		138	#define rename(old,neu) eio__rename (old, neu)
		139
		140	static int
		141	eio__rename (const char *old, const char *neu)
		142	{
		143	if (MoveFileEx (old, neu, MOVEFILE_REPLACE_EXISTING))
		144	return 0;
		145
		146	/* should steal _dosmaperr */
		147	switch (GetLastError ())
		148	{
		149	case ERROR_FILE_NOT_FOUND:
		150	case ERROR_PATH_NOT_FOUND:
		151	case ERROR_INVALID_DRIVE:
		152	case ERROR_NO_MORE_FILES:
		153	case ERROR_BAD_NETPATH:
		154	case ERROR_BAD_NET_NAME:
		155	case ERROR_BAD_PATHNAME:
		156	case ERROR_FILENAME_EXCED_RANGE:
		157	errno = ENOENT;
		158	break;
		159
		160	default:
		161	errno = EACCES;
		162	break;
		163	}
		164
		165	return -1;
		166	}
		167
		168	/* we could even stat and see if it exists */
		169	static int
		170	symlink (const char *old, const char *neu)
		171	{
		172	#if WINVER >= 0x0600
		173	if (CreateSymbolicLink (neu, old, 1))
		174	return 0;
		175
		176	if (CreateSymbolicLink (neu, old, 0))
		177	return 0;
		178	#endif
		179
		180	return EIO_ERRNO (ENOENT, -1);
		181	}
		182
		183	/* POSIX API only */
		184	#define CreateHardLink(neu,old,flags) 0
		185	#define CreateSymbolicLink(neu,old,flags) 0
		186
		187	struct statvfs
		188	{
		189	int dummy;
		190	};
		191
		192	#define DT_DIR EIO_DT_DIR
		193	#define DT_REG EIO_DT_REG
		194	#define D_NAME(entp) entp.cFileName
		195	#define D_TYPE(entp) (entp.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ? DT_DIR : DT_REG)
		196
74	#else	197	#else
75		198
76	# include "config.h"
77	# include <sys/time.h>	199	#include <sys/time.h>
78	# include <sys/select.h>	200	#include <sys/select.h>
		201	#include <sys/statvfs.h>
79	# include <unistd.h>	202	#include <unistd.h>
		203	#include <signal.h>
		204	#include <dirent.h>
		205
		206	#if _POSIX_MEMLOCK || _POSIX_MEMLOCK_RANGE || _POSIX_MAPPED_FILES
		207	#include <sys/mman.h>
		208	#endif
		209
		210	#define D_NAME(entp) entp->d_name
		211
		212	/* POSIX_SOURCE is useless on bsd's, and XOPEN_SOURCE is unreliable there, too */
		213	#if __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__
		214	#define _DIRENT_HAVE_D_TYPE /* sigh */
		215	#define D_INO(de) (de)->d_fileno
		216	#define D_NAMLEN(de) (de)->d_namlen
		217	#elif __linux || defined d_ino || _XOPEN_SOURCE >= 600
		218	#define D_INO(de) (de)->d_ino
		219	#endif
		220
		221	#ifdef _D_EXACT_NAMLEN
		222	#undef D_NAMLEN
		223	#define D_NAMLEN(de) _D_EXACT_NAMLEN (de)
		224	#endif
		225
		226	#ifdef _DIRENT_HAVE_D_TYPE
		227	#define D_TYPE(de) (de)->d_type
		228	#endif
		229
		230	#ifndef EIO_STRUCT_DIRENT
		231	#define EIO_STRUCT_DIRENT struct dirent
		232	#endif
		233
		234	#endif
		235
		236	#if HAVE_UTIMES
80	# include <utime.h>	237	# include <utime.h>
81	# include <signal.h>
82	# include <dirent.h>
83
84	#if _POSIX_MEMLOCK || _POSIX_MEMLOCK_RANGE || _POSIX_MAPPED_FILES
85	# include <sys/mman.h>
86	#endif
87
88	/* POSIX_SOURCE is useless on bsd's, and XOPEN_SOURCE is unreliable there, too */
89	# if __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__
90	# define _DIRENT_HAVE_D_TYPE /* sigh */
91	# define D_INO(de) (de)->d_fileno
92	# define D_NAMLEN(de) (de)->d_namlen
93	# elif __linux || defined d_ino || _XOPEN_SOURCE >= 600
94	# define D_INO(de) (de)->d_ino
95	# endif	238	#endif
96		239
97	#ifdef _D_EXACT_NAMLEN	240	#if HAVE_SYS_SYSCALL_H
98	# undef D_NAMLEN	241	# include <sys/syscall.h>
99	# define D_NAMLEN(de) _D_EXACT_NAMLEN (de)
100	#endif
101
102	# ifdef _DIRENT_HAVE_D_TYPE
103	# define D_TYPE(de) (de)->d_type
104	# endif	242	#endif
105		243
106	# ifndef EIO_STRUCT_DIRENT	244	#if HAVE_SYS_PRCTL_H
107	# define EIO_STRUCT_DIRENT struct dirent	245	# include <sys/prctl.h>
108	# endif
109
110	#endif	246	#endif
111		247
112	#if HAVE_SENDFILE	248	#if HAVE_SENDFILE
113	# if __linux	249	# if __linux
114	# include <sys/sendfile.h>	250	# include <sys/sendfile.h>
…		…
129	#endif	265	#endif
130	#ifndef D_INO	266	#ifndef D_INO
131	# define D_INO(de) 0	267	# define D_INO(de) 0
132	#endif	268	#endif
133	#ifndef D_NAMLEN	269	#ifndef D_NAMLEN
134	# define D_NAMLEN(de) strlen ((de)->d_name)	270	# define D_NAMLEN(entp) strlen (D_NAME (entp))
135	#endif	271	#endif
136
137	/* number of seconds after which an idle threads exit */
138	#define IDLE_TIMEOUT 10
139		272
140	/* used for struct dirent, AIX doesn't provide it */	273	/* used for struct dirent, AIX doesn't provide it */
141	#ifndef NAME_MAX	274	#ifndef NAME_MAX
142	# define NAME_MAX 4096	275	# define NAME_MAX 4096
143	#endif	276	#endif
…		…
149		282
150	/* buffer size for various temporary buffers */	283	/* buffer size for various temporary buffers */
151	#define EIO_BUFSIZE 65536	284	#define EIO_BUFSIZE 65536
152		285
153	#define dBUF \	286	#define dBUF \
154	char *eio_buf; \
155	ETP_WORKER_LOCK (self); \
156	self->dbuf = eio_buf = malloc (EIO_BUFSIZE); \	287	char *eio_buf = malloc (EIO_BUFSIZE); \
157	ETP_WORKER_UNLOCK (self); \
158	errno = ENOMEM; \	288	errno = ENOMEM; \
159	if (!eio_buf) \	289	if (!eio_buf) \
160	return -1;	290	return -1
		291
		292	#define FUBd \
		293	free (eio_buf)
161		294
162	#define EIO_TICKS ((1000000 + 1023) >> 10)	295	#define EIO_TICKS ((1000000 + 1023) >> 10)
163		296
164	/*****************************************************************************/	297	/*****************************************************************************/
165		298
166	#if __GNUC__ >= 3	299	struct tmpbuf
167	# define expect(expr,value) __builtin_expect ((expr),(value))	300	{
168	#else	301	void *ptr;
169	# define expect(expr,value) (expr)	302	int len;
170	#endif	303	};
171		304
172	#define expect_false(expr) expect ((expr) != 0, 0)	305	static void *
173	#define expect_true(expr) expect ((expr) != 0, 1)	306	tmpbuf_get (struct tmpbuf *buf, int len)
		307	{
		308	if (buf->len < len)
		309	{
		310	free (buf->ptr);
		311	buf->ptr = malloc (buf->len = len);
		312	}
		313
		314	return buf->ptr;
		315	}
174		316
175	/*****************************************************************************/	317	/*****************************************************************************/
176		318
177	#define ETP_PRI_MIN EIO_PRI_MIN	319	#define ETP_PRI_MIN EIO_PRI_MIN
178	#define ETP_PRI_MAX EIO_PRI_MAX	320	#define ETP_PRI_MAX EIO_PRI_MAX
…		…
184	static int eio_finish (eio_req *req);	326	static int eio_finish (eio_req *req);
185	#define ETP_FINISH(req) eio_finish (req)	327	#define ETP_FINISH(req) eio_finish (req)
186	static void eio_execute (struct etp_worker *self, eio_req *req);	328	static void eio_execute (struct etp_worker *self, eio_req *req);
187	#define ETP_EXECUTE(wrk,req) eio_execute (wrk,req)	329	#define ETP_EXECUTE(wrk,req) eio_execute (wrk,req)
188		330
189	#define ETP_WORKER_CLEAR(req) \
190	if (wrk->dbuf) \
191	{ \
192	free (wrk->dbuf); \
193	wrk->dbuf = 0; \
194	} \
195	\
196	if (wrk->dirp) \
197	{ \
198	closedir (wrk->dirp); \
199	wrk->dirp = 0; \
200	}
201
202	#define ETP_WORKER_COMMON \
203	void *dbuf; \
204	DIR *dirp;
205
206	/*****************************************************************************/	331	/*****************************************************************************/
207		332
208	#define ETP_NUM_PRI (ETP_PRI_MAX - ETP_PRI_MIN + 1)	333	#define ETP_NUM_PRI (ETP_PRI_MAX - ETP_PRI_MIN + 1)
209		334
210	/* calculate time difference in ~1/EIO_TICKS of a second */	335	/* calculate time difference in ~1/EIO_TICKS of a second */
		336	ecb_inline int
211	static int tvdiff (struct timeval *tv1, struct timeval *tv2)	337	tvdiff (struct timeval *tv1, struct timeval *tv2)
212	{	338	{
213	return (tv2->tv_sec - tv1->tv_sec ) * EIO_TICKS	339	return (tv2->tv_sec - tv1->tv_sec ) * EIO_TICKS
214	+ ((tv2->tv_usec - tv1->tv_usec) >> 10);	340	+ ((tv2->tv_usec - tv1->tv_usec) >> 10);
215	}	341	}
216		342
…		…
220	static void (*done_poll_cb) (void);	346	static void (*done_poll_cb) (void);
221		347
222	static unsigned int max_poll_time; /* reslock */	348	static unsigned int max_poll_time; /* reslock */
223	static unsigned int max_poll_reqs; /* reslock */	349	static unsigned int max_poll_reqs; /* reslock */
224		350
225	static volatile unsigned int nreqs; /* reqlock */	351	static unsigned int nreqs; /* reqlock */
226	static volatile unsigned int nready; /* reqlock */	352	static unsigned int nready; /* reqlock */
227	static volatile unsigned int npending; /* reqlock */	353	static unsigned int npending; /* reqlock */
228	static volatile unsigned int max_idle = 4;	354	static unsigned int max_idle = 4; /* maximum number of threads that can idle indefinitely */
		355	static unsigned int idle_timeout = 10; /* number of seconds after which an idle threads exit */
229		356
230	static xmutex_t wrklock = X_MUTEX_INIT;	357	static xmutex_t wrklock;
231	static xmutex_t reslock = X_MUTEX_INIT;	358	static xmutex_t reslock;
232	static xmutex_t reqlock = X_MUTEX_INIT;	359	static xmutex_t reqlock;
233	static xcond_t reqwait = X_COND_INIT;	360	static xcond_t reqwait;
234		361
235	#if !HAVE_PREADWRITE	362	#if !HAVE_PREADWRITE
236	/*	363	/*
237	* make our pread/pwrite emulation safe against themselves, but not against	364	* make our pread/pwrite emulation safe against themselves, but not against
238	* normal read/write by using a mutex. slows down execution a lot,	365	* normal read/write by using a mutex. slows down execution a lot,
239	* but that's your problem, not mine.	366	* but that's your problem, not mine.
240	*/	367	*/
241	static xmutex_t preadwritelock = X_MUTEX_INIT;	368	static xmutex_t preadwritelock;
242	#endif	369	#endif
243		370
244	typedef struct etp_worker	371	typedef struct etp_worker
245	{	372	{
		373	struct tmpbuf tmpbuf;
		374
246	/* locked by wrklock */	375	/* locked by wrklock */
247	struct etp_worker *prev, *next;	376	struct etp_worker *prev, *next;
248		377
249	xthread_t tid;	378	xthread_t tid;
250		379
251	/* locked by reslock, reqlock or wrklock */	380	#ifdef ETP_WORKER_COMMON
252	ETP_REQ *req; /* currently processed request */
253
254	ETP_WORKER_COMMON	381	ETP_WORKER_COMMON
		382	#endif
255	} etp_worker;	383	} etp_worker;
256		384
257	static etp_worker wrk_first = { &wrk_first, &wrk_first, 0 }; /* NOT etp */	385	static etp_worker wrk_first; /* NOT etp */
258		386
259	#define ETP_WORKER_LOCK(wrk) X_LOCK (wrklock)	387	#define ETP_WORKER_LOCK(wrk) X_LOCK (wrklock)
260	#define ETP_WORKER_UNLOCK(wrk) X_UNLOCK (wrklock)	388	#define ETP_WORKER_UNLOCK(wrk) X_UNLOCK (wrklock)
261		389
262	/* worker threads management */	390	/* worker threads management */
263		391
		392	static void ecb_cold
264	static void etp_worker_clear (etp_worker *wrk)	393	etp_worker_clear (etp_worker *wrk)
265	{	394	{
266	ETP_WORKER_CLEAR (wrk);
267	}	395	}
268		396
		397	static void ecb_cold
269	static void etp_worker_free (etp_worker *wrk)	398	etp_worker_free (etp_worker *wrk)
270	{	399	{
		400	free (wrk->tmpbuf.ptr);
		401
271	wrk->next->prev = wrk->prev;	402	wrk->next->prev = wrk->prev;
272	wrk->prev->next = wrk->next;	403	wrk->prev->next = wrk->next;
273		404
274	free (wrk);	405	free (wrk);
275	}	406	}
276		407
277	static unsigned int etp_nreqs (void)	408	static unsigned int
		409	etp_nreqs (void)
278	{	410	{
279	int retval;	411	int retval;
280	if (WORDACCESS_UNSAFE) X_LOCK (reqlock);	412	if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
281	retval = nreqs;	413	retval = nreqs;
282	if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);	414	if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
283	return retval;	415	return retval;
284	}	416	}
285		417
286	static unsigned int etp_nready (void)	418	static unsigned int
		419	etp_nready (void)
287	{	420	{
288	unsigned int retval;	421	unsigned int retval;
289		422
290	if (WORDACCESS_UNSAFE) X_LOCK (reqlock);	423	if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
291	retval = nready;	424	retval = nready;
292	if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);	425	if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
293		426
294	return retval;	427	return retval;
295	}	428	}
296		429
297	static unsigned int etp_npending (void)	430	static unsigned int
		431	etp_npending (void)
298	{	432	{
299	unsigned int retval;	433	unsigned int retval;
300		434
301	if (WORDACCESS_UNSAFE) X_LOCK (reqlock);	435	if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
302	retval = npending;	436	retval = npending;
303	if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);	437	if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
304		438
305	return retval;	439	return retval;
306	}	440	}
307		441
308	static unsigned int etp_nthreads (void)	442	static unsigned int
		443	etp_nthreads (void)
309	{	444	{
310	unsigned int retval;	445	unsigned int retval;
311		446
312	if (WORDACCESS_UNSAFE) X_LOCK (reqlock);	447	if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
313	retval = started;	448	retval = started;
…		…
327	} etp_reqq;	462	} etp_reqq;
328		463
329	static etp_reqq req_queue;	464	static etp_reqq req_queue;
330	static etp_reqq res_queue;	465	static etp_reqq res_queue;
331		466
		467	static void ecb_noinline ecb_cold
		468	reqq_init (etp_reqq *q)
		469	{
		470	int pri;
		471
		472	for (pri = 0; pri < ETP_NUM_PRI; ++pri)
		473	q->qs[pri] = q->qe[pri] = 0;
		474
		475	q->size = 0;
		476	}
		477
		478	static int ecb_noinline
332	static int reqq_push (etp_reqq *q, ETP_REQ *req)	479	reqq_push (etp_reqq *q, ETP_REQ *req)
333	{	480	{
334	int pri = req->pri;	481	int pri = req->pri;
335	req->next = 0;	482	req->next = 0;
336		483
337	if (q->qe[pri])	484	if (q->qe[pri])
…		…
343	q->qe[pri] = q->qs[pri] = req;	490	q->qe[pri] = q->qs[pri] = req;
344		491
345	return q->size++;	492	return q->size++;
346	}	493	}
347		494
		495	static ETP_REQ * ecb_noinline
348	static ETP_REQ *reqq_shift (etp_reqq *q)	496	reqq_shift (etp_reqq *q)
349	{	497	{
350	int pri;	498	int pri;
351		499
352	if (!q->size)	500	if (!q->size)
353	return 0;	501	return 0;
…		…
368	}	516	}
369		517
370	abort ();	518	abort ();
371	}	519	}
372		520
373	static void etp_atfork_prepare (void)	521	static int ecb_cold
		522	etp_init (void (*want_poll)(void), void (*done_poll)(void))
374	{	523	{
375	X_LOCK (wrklock);	524	X_MUTEX_CREATE (wrklock);
376	X_LOCK (reqlock);	525	X_MUTEX_CREATE (reslock);
377	X_LOCK (reslock);	526	X_MUTEX_CREATE (reqlock);
378	#if !HAVE_PREADWRITE	527	X_COND_CREATE (reqwait);
379	X_LOCK (preadwritelock);
380	#endif
381	}
382		528
383	static void etp_atfork_parent (void)	529	reqq_init (&req_queue);
384	{	530	reqq_init (&res_queue);
385	#if !HAVE_PREADWRITE
386	X_UNLOCK (preadwritelock);
387	#endif
388	X_UNLOCK (reslock);
389	X_UNLOCK (reqlock);
390	X_UNLOCK (wrklock);
391	}
392		531
393	static void etp_atfork_child (void)	532	wrk_first.next =
394	{	533	wrk_first.prev = &wrk_first;
395	ETP_REQ *prv;
396
397	while ((prv = reqq_shift (&req_queue)))
398	ETP_DESTROY (prv);
399
400	while ((prv = reqq_shift (&res_queue)))
401	ETP_DESTROY (prv);
402
403	while (wrk_first.next != &wrk_first)
404	{
405	etp_worker *wrk = wrk_first.next;
406
407	if (wrk->req)
408	ETP_DESTROY (wrk->req);
409
410	etp_worker_clear (wrk);
411	etp_worker_free (wrk);
412	}
413		534
414	started = 0;	535	started = 0;
415	idle = 0;	536	idle = 0;
416	nreqs = 0;	537	nreqs = 0;
417	nready = 0;	538	nready = 0;
418	npending = 0;	539	npending = 0;
419		540
420	etp_atfork_parent ();
421	}
422
423	static void
424	etp_once_init (void)
425	{
426	X_THREAD_ATFORK (etp_atfork_prepare, etp_atfork_parent, etp_atfork_child);
427	}
428
429	static int
430	etp_init (void (*want_poll)(void), void (*done_poll)(void))
431	{
432	static pthread_once_t doinit = PTHREAD_ONCE_INIT;
433
434	pthread_once (&doinit, etp_once_init);
435
436	want_poll_cb = want_poll;	541	want_poll_cb = want_poll;
437	done_poll_cb = done_poll;	542	done_poll_cb = done_poll;
438		543
439	return 0;	544	return 0;
440	}	545	}
441		546
442	X_THREAD_PROC (etp_proc);	547	X_THREAD_PROC (etp_proc);
443		548
		549	static void ecb_cold
444	static void etp_start_thread (void)	550	etp_start_thread (void)
445	{	551	{
446	etp_worker *wrk = calloc (1, sizeof (etp_worker));	552	etp_worker *wrk = calloc (1, sizeof (etp_worker));
447		553
448	/*TODO*/	554	/*TODO*/
449	assert (("unable to allocate worker thread data", wrk));	555	assert (("unable to allocate worker thread data", wrk));
…		…
462	free (wrk);	568	free (wrk);
463		569
464	X_UNLOCK (wrklock);	570	X_UNLOCK (wrklock);
465	}	571	}
466		572
		573	static void
467	static void etp_maybe_start_thread (void)	574	etp_maybe_start_thread (void)
468	{	575	{
469	if (expect_true (etp_nthreads () >= wanted))	576	if (ecb_expect_true (etp_nthreads () >= wanted))
470	return;	577	return;
471		578
472	/* todo: maybe use idle here, but might be less exact */	579	/* todo: maybe use idle here, but might be less exact */
473	if (expect_true (0 <= (int)etp_nthreads () + (int)etp_npending () - (int)etp_nreqs ()))	580	if (ecb_expect_true (0 <= (int)etp_nthreads () + (int)etp_npending () - (int)etp_nreqs ()))
474	return;	581	return;
475		582
476	etp_start_thread ();	583	etp_start_thread ();
477	}	584	}
478		585
		586	static void ecb_cold
479	static void etp_end_thread (void)	587	etp_end_thread (void)
480	{	588	{
481	eio_req *req = calloc (1, sizeof (eio_req));	589	eio_req *req = calloc (1, sizeof (eio_req)); /* will be freed by worker */
482		590
483	req->type = -1;	591	req->type = -1;
484	req->pri = ETP_PRI_MAX - ETP_PRI_MIN;	592	req->pri = ETP_PRI_MAX - ETP_PRI_MIN;
485		593
486	X_LOCK (reqlock);	594	X_LOCK (reqlock);
…		…
491	X_LOCK (wrklock);	599	X_LOCK (wrklock);
492	--started;	600	--started;
493	X_UNLOCK (wrklock);	601	X_UNLOCK (wrklock);
494	}	602	}
495		603
496	static int etp_poll (void)	604	static int
		605	etp_poll (void)
497	{	606	{
498	unsigned int maxreqs;	607	unsigned int maxreqs;
499	unsigned int maxtime;	608	unsigned int maxtime;
500	struct timeval tv_start, tv_now;	609	struct timeval tv_start, tv_now;
501		610
…		…
531		640
532	X_LOCK (reqlock);	641	X_LOCK (reqlock);
533	--nreqs;	642	--nreqs;
534	X_UNLOCK (reqlock);	643	X_UNLOCK (reqlock);
535		644
536	if (expect_false (req->type == EIO_GROUP && req->size))	645	if (ecb_expect_false (req->type == EIO_GROUP && req->size))
537	{	646	{
538	req->int1 = 1; /* mark request as delayed */	647	req->int1 = 1; /* mark request as delayed */
539	continue;	648	continue;
540	}	649	}
541	else	650	else
542	{	651	{
543	int res = ETP_FINISH (req);	652	int res = ETP_FINISH (req);
544	if (expect_false (res))	653	if (ecb_expect_false (res))
545	return res;	654	return res;
546	}	655	}
547		656
548	if (expect_false (maxreqs && !--maxreqs))	657	if (ecb_expect_false (maxreqs && !--maxreqs))
549	break;	658	break;
550		659
551	if (maxtime)	660	if (maxtime)
552	{	661	{
553	gettimeofday (&tv_now, 0);	662	gettimeofday (&tv_now, 0);
…		…
559		668
560	errno = EAGAIN;	669	errno = EAGAIN;
561	return -1;	670	return -1;
562	}	671	}
563		672
		673	static void
564	static void etp_cancel (ETP_REQ *req)	674	etp_cancel (ETP_REQ *req)
565	{	675	{
566	X_LOCK (wrklock);	676	req->cancelled = 1;
567	req->flags |= EIO_FLAG_CANCELLED;
568	X_UNLOCK (wrklock);
569		677
570	eio_grp_cancel (req);	678	eio_grp_cancel (req);
571	}	679	}
572		680
		681	static void
573	static void etp_submit (ETP_REQ *req)	682	etp_submit (ETP_REQ *req)
574	{	683	{
575	req->pri -= ETP_PRI_MIN;	684	req->pri -= ETP_PRI_MIN;
576		685
577	if (expect_false (req->pri < ETP_PRI_MIN - ETP_PRI_MIN)) req->pri = ETP_PRI_MIN - ETP_PRI_MIN;	686	if (ecb_expect_false (req->pri < ETP_PRI_MIN - ETP_PRI_MIN)) req->pri = ETP_PRI_MIN - ETP_PRI_MIN;
578	if (expect_false (req->pri > ETP_PRI_MAX - ETP_PRI_MIN)) req->pri = ETP_PRI_MAX - ETP_PRI_MIN;	687	if (ecb_expect_false (req->pri > ETP_PRI_MAX - ETP_PRI_MIN)) req->pri = ETP_PRI_MAX - ETP_PRI_MIN;
579		688
580	if (expect_false (req->type == EIO_GROUP))	689	if (ecb_expect_false (req->type == EIO_GROUP))
581	{	690	{
582	/* I hope this is worth it :/ */	691	/* I hope this is worth it :/ */
583	X_LOCK (reqlock);	692	X_LOCK (reqlock);
584	++nreqs;	693	++nreqs;
585	X_UNLOCK (reqlock);	694	X_UNLOCK (reqlock);
…		…
604		713
605	etp_maybe_start_thread ();	714	etp_maybe_start_thread ();
606	}	715	}
607	}	716	}
608		717
		718	static void ecb_cold
609	static void etp_set_max_poll_time (double nseconds)	719	etp_set_max_poll_time (double nseconds)
610	{	720	{
611	if (WORDACCESS_UNSAFE) X_LOCK (reslock);	721	if (WORDACCESS_UNSAFE) X_LOCK (reslock);
612	max_poll_time = nseconds * EIO_TICKS;	722	max_poll_time = nseconds * EIO_TICKS;
613	if (WORDACCESS_UNSAFE) X_UNLOCK (reslock);	723	if (WORDACCESS_UNSAFE) X_UNLOCK (reslock);
614	}	724	}
615		725
		726	static void ecb_cold
616	static void etp_set_max_poll_reqs (unsigned int maxreqs)	727	etp_set_max_poll_reqs (unsigned int maxreqs)
617	{	728	{
618	if (WORDACCESS_UNSAFE) X_LOCK (reslock);	729	if (WORDACCESS_UNSAFE) X_LOCK (reslock);
619	max_poll_reqs = maxreqs;	730	max_poll_reqs = maxreqs;
620	if (WORDACCESS_UNSAFE) X_UNLOCK (reslock);	731	if (WORDACCESS_UNSAFE) X_UNLOCK (reslock);
621	}	732	}
622		733
		734	static void ecb_cold
623	static void etp_set_max_idle (unsigned int nthreads)	735	etp_set_max_idle (unsigned int nthreads)
624	{	736	{
625	if (WORDACCESS_UNSAFE) X_LOCK (reqlock);	737	if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
626	max_idle = nthreads <= 0 ? 1 : nthreads;	738	max_idle = nthreads;
627	if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);	739	if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
628	}	740	}
629		741
		742	static void ecb_cold
		743	etp_set_idle_timeout (unsigned int seconds)
		744	{
		745	if (WORDACCESS_UNSAFE) X_LOCK (reqlock);
		746	idle_timeout = seconds;
		747	if (WORDACCESS_UNSAFE) X_UNLOCK (reqlock);
		748	}
		749
		750	static void ecb_cold
630	static void etp_set_min_parallel (unsigned int nthreads)	751	etp_set_min_parallel (unsigned int nthreads)
631	{	752	{
632	if (wanted < nthreads)	753	if (wanted < nthreads)
633	wanted = nthreads;	754	wanted = nthreads;
634	}	755	}
635		756
		757	static void ecb_cold
636	static void etp_set_max_parallel (unsigned int nthreads)	758	etp_set_max_parallel (unsigned int nthreads)
637	{	759	{
638	if (wanted > nthreads)	760	if (wanted > nthreads)
639	wanted = nthreads;	761	wanted = nthreads;
640		762
641	while (started > wanted)	763	while (started > wanted)
642	etp_end_thread ();	764	etp_end_thread ();
643	}	765	}
644		766
645	/*****************************************************************************/	767	/*****************************************************************************/
646		768
		769	static void
647	static void grp_try_feed (eio_req *grp)	770	grp_try_feed (eio_req *grp)
648	{	771	{
649	while (grp->size < grp->int2 && !EIO_CANCELLED (grp))	772	while (grp->size < grp->int2 && !EIO_CANCELLED (grp))
650	{	773	{
651	grp->flags &= ~EIO_FLAG_GROUPADD;	774	grp->flags &= ~EIO_FLAG_GROUPADD;
652		775
…		…
659	break;	782	break;
660	}	783	}
661	}	784	}
662	}	785	}
663		786
		787	static int
664	static int grp_dec (eio_req *grp)	788	grp_dec (eio_req *grp)
665	{	789	{
666	--grp->size;	790	--grp->size;
667		791
668	/* call feeder, if applicable */	792	/* call feeder, if applicable */
669	grp_try_feed (grp);	793	grp_try_feed (grp);
…		…
673	return eio_finish (grp);	797	return eio_finish (grp);
674	else	798	else
675	return 0;	799	return 0;
676	}	800	}
677		801
		802	static void
678	void eio_destroy (eio_req *req)	803	eio_destroy (eio_req *req)
679	{	804	{
680	if ((req)->flags & EIO_FLAG_PTR1_FREE) free (req->ptr1);	805	if ((req)->flags & EIO_FLAG_PTR1_FREE) free (req->ptr1);
681	if ((req)->flags & EIO_FLAG_PTR2_FREE) free (req->ptr2);	806	if ((req)->flags & EIO_FLAG_PTR2_FREE) free (req->ptr2);
682		807
683	EIO_DESTROY (req);	808	EIO_DESTROY (req);
684	}	809	}
685		810
		811	static int
686	static int eio_finish (eio_req *req)	812	eio_finish (eio_req *req)
687	{	813	{
688	int res = EIO_FINISH (req);	814	int res = EIO_FINISH (req);
689		815
690	if (req->grp)	816	if (req->grp)
691	{	817	{
…		…
699	if (grp->grp_first == req)	825	if (grp->grp_first == req)
700	grp->grp_first = req->grp_next;	826	grp->grp_first = req->grp_next;
701		827
702	res2 = grp_dec (grp);	828	res2 = grp_dec (grp);
703		829
704	if (!res && res2)	830	if (!res)
705	res = res2;	831	res = res2;
706	}	832	}
707		833
708	eio_destroy (req);	834	eio_destroy (req);
709		835
710	return res;	836	return res;
711	}	837	}
712		838
		839	void
713	void eio_grp_cancel (eio_req *grp)	840	eio_grp_cancel (eio_req *grp)
714	{	841	{
715	for (grp = grp->grp_first; grp; grp = grp->grp_next)	842	for (grp = grp->grp_first; grp; grp = grp->grp_next)
716	eio_cancel (grp);	843	eio_cancel (grp);
717	}	844	}
718		845
		846	void
719	void eio_cancel (eio_req *req)	847	eio_cancel (eio_req *req)
720	{	848	{
721	etp_cancel (req);	849	etp_cancel (req);
722	}	850	}
723		851
		852	void
724	void eio_submit (eio_req *req)	853	eio_submit (eio_req *req)
725	{	854	{
726	etp_submit (req);	855	etp_submit (req);
727	}	856	}
728		857
729	unsigned int eio_nreqs (void)	858	unsigned int
		859	eio_nreqs (void)
730	{	860	{
731	return etp_nreqs ();	861	return etp_nreqs ();
732	}	862	}
733		863
734	unsigned int eio_nready (void)	864	unsigned int
		865	eio_nready (void)
735	{	866	{
736	return etp_nready ();	867	return etp_nready ();
737	}	868	}
738		869
739	unsigned int eio_npending (void)	870	unsigned int
		871	eio_npending (void)
740	{	872	{
741	return etp_npending ();	873	return etp_npending ();
742	}	874	}
743		875
744	unsigned int eio_nthreads (void)	876	unsigned int ecb_cold
		877	eio_nthreads (void)
745	{	878	{
746	return etp_nthreads ();	879	return etp_nthreads ();
747	}	880	}
748		881
		882	void ecb_cold
749	void eio_set_max_poll_time (double nseconds)	883	eio_set_max_poll_time (double nseconds)
750	{	884	{
751	etp_set_max_poll_time (nseconds);	885	etp_set_max_poll_time (nseconds);
752	}	886	}
753		887
		888	void ecb_cold
754	void eio_set_max_poll_reqs (unsigned int maxreqs)	889	eio_set_max_poll_reqs (unsigned int maxreqs)
755	{	890	{
756	etp_set_max_poll_reqs (maxreqs);	891	etp_set_max_poll_reqs (maxreqs);
757	}	892	}
758		893
		894	void ecb_cold
759	void eio_set_max_idle (unsigned int nthreads)	895	eio_set_max_idle (unsigned int nthreads)
760	{	896	{
761	etp_set_max_idle (nthreads);	897	etp_set_max_idle (nthreads);
762	}	898	}
763		899
		900	void ecb_cold
		901	eio_set_idle_timeout (unsigned int seconds)
		902	{
		903	etp_set_idle_timeout (seconds);
		904	}
		905
		906	void ecb_cold
764	void eio_set_min_parallel (unsigned int nthreads)	907	eio_set_min_parallel (unsigned int nthreads)
765	{	908	{
766	etp_set_min_parallel (nthreads);	909	etp_set_min_parallel (nthreads);
767	}	910	}
768		911
		912	void ecb_cold
769	void eio_set_max_parallel (unsigned int nthreads)	913	eio_set_max_parallel (unsigned int nthreads)
770	{	914	{
771	etp_set_max_parallel (nthreads);	915	etp_set_max_parallel (nthreads);
772	}	916	}
773		917
774	int eio_poll (void)	918	int eio_poll (void)
…		…
783	# undef pread	927	# undef pread
784	# undef pwrite	928	# undef pwrite
785	# define pread eio__pread	929	# define pread eio__pread
786	# define pwrite eio__pwrite	930	# define pwrite eio__pwrite
787		931
788	static ssize_t	932	static eio_ssize_t
789	eio__pread (int fd, void *buf, size_t count, off_t offset)	933	eio__pread (int fd, void *buf, size_t count, off_t offset)
790	{	934	{
791	ssize_t res;	935	eio_ssize_t res;
792	off_t ooffset;	936	off_t ooffset;
793		937
794	X_LOCK (preadwritelock);	938	X_LOCK (preadwritelock);
795	ooffset = lseek (fd, 0, SEEK_CUR);	939	ooffset = lseek (fd, 0, SEEK_CUR);
796	lseek (fd, offset, SEEK_SET);	940	lseek (fd, offset, SEEK_SET);
…		…
799	X_UNLOCK (preadwritelock);	943	X_UNLOCK (preadwritelock);
800		944
801	return res;	945	return res;
802	}	946	}
803		947
804	static ssize_t	948	static eio_ssize_t
805	eio__pwrite (int fd, void *buf, size_t count, off_t offset)	949	eio__pwrite (int fd, void *buf, size_t count, off_t offset)
806	{	950	{
807	ssize_t res;	951	eio_ssize_t res;
808	off_t ooffset;	952	off_t ooffset;
809		953
810	X_LOCK (preadwritelock);	954	X_LOCK (preadwritelock);
811	ooffset = lseek (fd, 0, SEEK_CUR);	955	ooffset = lseek (fd, 0, SEEK_CUR);
812	lseek (fd, offset, SEEK_SET);	956	lseek (fd, offset, SEEK_SET);
…		…
844	#ifndef HAVE_FUTIMES	988	#ifndef HAVE_FUTIMES
845		989
846	# undef futimes	990	# undef futimes
847	# define futimes(fd,times) eio__futimes (fd, times)	991	# define futimes(fd,times) eio__futimes (fd, times)
848		992
		993	static int
849	static int eio__futimes (int fd, const struct timeval tv[2])	994	eio__futimes (int fd, const struct timeval tv[2])
850	{	995	{
851	errno = ENOSYS;	996	errno = ENOSYS;
852	return -1;	997	return -1;
853	}	998	}
854		999
…		…
857	#if !HAVE_FDATASYNC	1002	#if !HAVE_FDATASYNC
858	# undef fdatasync	1003	# undef fdatasync
859	# define fdatasync(fd) fsync (fd)	1004	# define fdatasync(fd) fsync (fd)
860	#endif	1005	#endif
861		1006
		1007	static int
		1008	eio__syncfs (int fd)
		1009	{
		1010	int res;
		1011
		1012	#if HAVE_SYS_SYNCFS
		1013	res = (int)syscall (__NR_syncfs, (int)(fd));
		1014	#else
		1015	res = -1;
		1016	errno = ENOSYS;
		1017	#endif
		1018
		1019	if (res < 0 && errno == ENOSYS && fd >= 0)
		1020	sync ();
		1021
		1022	return res;
		1023	}
		1024
862	/* sync_file_range always needs emulation */	1025	/* sync_file_range always needs emulation */
863	int	1026	static int
864	eio__sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags)	1027	eio__sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags)
865	{	1028	{
866	#if HAVE_SYNC_FILE_RANGE	1029	#if HAVE_SYNC_FILE_RANGE
867	int res;	1030	int res;
868		1031
…		…
885	/* even though we could play tricks with the flags, it's better to always	1048	/* even though we could play tricks with the flags, it's better to always
886	* call fdatasync, as that matches the expectation of its users best */	1049	* call fdatasync, as that matches the expectation of its users best */
887	return fdatasync (fd);	1050	return fdatasync (fd);
888	}	1051	}
889		1052
		1053	static int
		1054	eio__fallocate (int fd, int mode, off_t offset, size_t len)
		1055	{
		1056	#if HAVE_FALLOCATE
		1057	return fallocate (fd, mode, offset, len);
		1058	#else
		1059	errno = ENOSYS;
		1060	return -1;
		1061	#endif
		1062	}
		1063
890	#if !HAVE_READAHEAD	1064	#if !HAVE_READAHEAD
891	# undef readahead	1065	# undef readahead
892	# define readahead(fd,offset,count) eio__readahead (fd, offset, count, self)	1066	# define readahead(fd,offset,count) eio__readahead (fd, offset, count, self)
893		1067
894	static ssize_t	1068	static eio_ssize_t
895	eio__readahead (int fd, off_t offset, size_t count, etp_worker *self)	1069	eio__readahead (int fd, off_t offset, size_t count, etp_worker *self)
896	{	1070	{
897	size_t todo = count;	1071	size_t todo = count;
898	dBUF;	1072	dBUF;
899		1073
…		…
904	pread (fd, eio_buf, len, offset);	1078	pread (fd, eio_buf, len, offset);
905	offset += len;	1079	offset += len;
906	todo -= len;	1080	todo -= len;
907	}	1081	}
908		1082
		1083	FUBd;
		1084
909	errno = 0;	1085	errno = 0;
910	return count;	1086	return count;
911	}	1087	}
912		1088
913	#endif	1089	#endif
914		1090
915	/* sendfile always needs emulation */	1091	/* sendfile always needs emulation */
916	static ssize_t	1092	static eio_ssize_t
917	eio__sendfile (int ofd, int ifd, off_t offset, size_t count, etp_worker *self)	1093	eio__sendfile (int ofd, int ifd, off_t offset, size_t count)
918	{	1094	{
		1095	eio_ssize_t written = 0;
919	ssize_t res;	1096	eio_ssize_t res;
920		1097
921	if (!count)	1098	if (!count)
922	return 0;	1099	return 0;
923		1100
		1101	for (;;)
		1102	{
		1103	#ifdef __APPLE__
		1104	# undef HAVE_SENDFILE /* broken, as everything on os x */
		1105	#endif
924	#if HAVE_SENDFILE	1106	#if HAVE_SENDFILE
925	# if __linux	1107	# if __linux
		1108	off_t soffset = offset;
926	res = sendfile (ofd, ifd, &offset, count);	1109	res = sendfile (ofd, ifd, &soffset, count);
927		1110
928	# elif __FreeBSD__	1111	# elif __FreeBSD__
929	/*	1112	/*
930	* Of course, the freebsd sendfile is a dire hack with no thoughts	1113	* Of course, the freebsd sendfile is a dire hack with no thoughts
931	* wasted on making it similar to other I/O functions.	1114	* wasted on making it similar to other I/O functions.
932	*/	1115	*/
933	{
934	off_t sbytes;	1116	off_t sbytes;
935	res = sendfile (ifd, ofd, offset, count, 0, &sbytes, 0);	1117	res = sendfile (ifd, ofd, offset, count, 0, &sbytes, 0);
936		1118
937	#if 0 /* according to the manpage, this is correct, but broken behaviour */	1119	#if 0 /* according to the manpage, this is correct, but broken behaviour */
938	/* freebsd' sendfile will return 0 on success */	1120	/* freebsd' sendfile will return 0 on success */
939	/* freebsd 8 documents it as only setting *sbytes on EINTR and EAGAIN, but */	1121	/* freebsd 8 documents it as only setting *sbytes on EINTR and EAGAIN, but */
940	/* not on e.g. EIO or EPIPE - sounds broken */	1122	/* not on e.g. EIO or EPIPE - sounds broken */
941	if ((res < 0 && (errno == EAGAIN || errno == EINTR) && sbytes) || res == 0)	1123	if ((res < 0 && (errno == EAGAIN || errno == EINTR) && sbytes) || res == 0)
942	res = sbytes;	1124	res = sbytes;
943	#endif	1125	#endif
944		1126
945	/* according to source inspection, this is correct, and useful behaviour */	1127	/* according to source inspection, this is correct, and useful behaviour */
946	if (sbytes)	1128	if (sbytes)
947	res = sbytes;	1129	res = sbytes;
948	}
949		1130
950	# elif defined (__APPLE__)	1131	# elif defined (__APPLE__)
951
952	{
953	off_t sbytes = count;	1132	off_t sbytes = count;
954	res = sendfile (ifd, ofd, offset, &sbytes, 0, 0);	1133	res = sendfile (ifd, ofd, offset, &sbytes, 0, 0);
955		1134
956	/* according to the manpage, sbytes is always valid */	1135	/* according to the manpage, sbytes is always valid */
957	if (sbytes)	1136	if (sbytes)
958	res = sbytes;	1137	res = sbytes;
959	}
960		1138
961	# elif __hpux	1139	# elif __hpux
962	res = sendfile (ofd, ifd, offset, count, 0, 0);	1140	res = sendfile (ofd, ifd, offset, count, 0, 0);
963		1141
964	# elif __solaris	1142	# elif __solaris
965	{
966	struct sendfilevec vec;	1143	struct sendfilevec vec;
967	size_t sbytes;	1144	size_t sbytes;
968		1145
969	vec.sfv_fd = ifd;	1146	vec.sfv_fd = ifd;
970	vec.sfv_flag = 0;	1147	vec.sfv_flag = 0;
971	vec.sfv_off = offset;	1148	vec.sfv_off = offset;
972	vec.sfv_len = count;	1149	vec.sfv_len = count;
973		1150
974	res = sendfilev (ofd, &vec, 1, &sbytes);	1151	res = sendfilev (ofd, &vec, 1, &sbytes);
975		1152
976	if (res < 0 && sbytes)	1153	if (res < 0 && sbytes)
977	res = sbytes;	1154	res = sbytes;
978	}
979		1155
980	# endif	1156	# endif
981		1157
982	#elif defined (_WIN32)	1158	#elif defined (_WIN32) && 0
983
984	/* does not work, just for documentation of what would need to be done */	1159	/* does not work, just for documentation of what would need to be done */
985	{	1160	/* actually, cannot be done like this, as TransmitFile changes the file offset, */
		1161	/* libeio guarantees that the file offset does not change, and windows */
		1162	/* has no way to get an independent handle to the same file description */
986	HANDLE h = TO_SOCKET (ifd);	1163	HANDLE h = TO_SOCKET (ifd);
987	SetFilePointer (h, offset, 0, FILE_BEGIN);	1164	SetFilePointer (h, offset, 0, FILE_BEGIN);
988	res = TransmitFile (TO_SOCKET (ofd), h, count, 0, 0, 0, 0);	1165	res = TransmitFile (TO_SOCKET (ofd), h, count, 0, 0, 0, 0);
989	}
990		1166
991	#else	1167	#else
992	res = -1;	1168	res = -1;
993	errno = ENOSYS;	1169	errno = ENOSYS;
994	#endif	1170	#endif
995		1171
		1172	/* we assume sendfile can copy at least 128mb in one go */
		1173	if (res <= 128 * 1024 * 1024)
		1174	{
		1175	if (res > 0)
		1176	written += res;
		1177
		1178	if (written)
		1179	return written;
		1180
		1181	break;
		1182	}
		1183	else
		1184	{
		1185	/* if we requested more, then probably the kernel was lazy */
		1186	written += res;
		1187	offset += res;
		1188	count -= res;
		1189
		1190	if (!count)
		1191	return written;
		1192	}
		1193	}
		1194
996	if (res < 0	1195	if (res < 0
997	&& (errno == ENOSYS || errno == EINVAL || errno == ENOTSOCK	1196	&& (errno == ENOSYS || errno == EINVAL || errno == ENOTSOCK
998	/* BSDs */	1197	/* BSDs */
999	#ifdef ENOTSUP /* sigh, if the steenking pile called openbsd would only try to at least compile posix code... */	1198	#ifdef ENOTSUP /* sigh, if the steenking pile called openbsd would only try to at least compile posix code... */
1000	|| errno == ENOTSUP	1199	|| errno == ENOTSUP
1001	#endif	1200	#endif
		1201	#ifdef EOPNOTSUPP /* windows */
1002	|| errno == EOPNOTSUPP /* BSDs */	1202	|| errno == EOPNOTSUPP /* BSDs */
		1203	#endif
1003	#if __solaris	1204	#if __solaris
1004	|| errno == EAFNOSUPPORT || errno == EPROTOTYPE	1205	|| errno == EAFNOSUPPORT || errno == EPROTOTYPE
1005	#endif	1206	#endif
1006	)	1207	)
1007	)	1208	)
…		…
1011		1212
1012	res = 0;	1213	res = 0;
1013		1214
1014	while (count)	1215	while (count)
1015	{	1216	{
1016	ssize_t cnt;	1217	eio_ssize_t cnt;
1017		1218
1018	cnt = pread (ifd, eio_buf, count > EIO_BUFSIZE ? EIO_BUFSIZE : count, offset);	1219	cnt = pread (ifd, eio_buf, count > EIO_BUFSIZE ? EIO_BUFSIZE : count, offset);
1019		1220
1020	if (cnt <= 0)	1221	if (cnt <= 0)
1021	{	1222	{
…		…
1033		1234
1034	offset += cnt;	1235	offset += cnt;
1035	res += cnt;	1236	res += cnt;
1036	count -= cnt;	1237	count -= cnt;
1037	}	1238	}
		1239
		1240	FUBd;
1038	}	1241	}
1039		1242
1040	return res;	1243	return res;
		1244	}
		1245
		1246	#ifdef PAGESIZE
		1247	# define eio_pagesize() PAGESIZE
		1248	#else
		1249	static intptr_t
		1250	eio_pagesize (void)
		1251	{
		1252	static intptr_t page;
		1253
		1254	if (!page)
		1255	page = sysconf (_SC_PAGESIZE);
		1256
		1257	return page;
		1258	}
		1259	#endif
		1260
		1261	static void
		1262	eio_page_align (void **addr, size_t *length)
		1263	{
		1264	intptr_t mask = eio_pagesize () - 1;
		1265
		1266	/* round down addr */
		1267	intptr_t adj = mask & (intptr_t)*addr;
		1268
		1269	*addr = (void *)((intptr_t)*addr - adj);
		1270	*length += adj;
		1271
		1272	/* round up length */
		1273	*length = (*length + mask) & ~mask;
		1274	}
		1275
		1276	#if !_POSIX_MEMLOCK
		1277	# define eio__mlockall(a) EIO_ENOSYS ()
		1278	#else
		1279
		1280	static int
		1281	eio__mlockall (int flags)
		1282	{
		1283	#if __GLIBC__ == 2 && __GLIBC_MINOR__ <= 7
		1284	extern int mallopt (int, int);
		1285	mallopt (-6, 238); /* http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=473812 */
		1286	#endif
		1287
		1288	if (EIO_MCL_CURRENT != MCL_CURRENT
		1289	|| EIO_MCL_FUTURE != MCL_FUTURE)
		1290	{
		1291	flags = 0
		1292	| (flags & EIO_MCL_CURRENT ? MCL_CURRENT : 0)
		1293	| (flags & EIO_MCL_FUTURE ? MCL_FUTURE : 0);
		1294	}
		1295
		1296	return mlockall (flags);
		1297	}
		1298	#endif
		1299
		1300	#if !_POSIX_MEMLOCK_RANGE
		1301	# define eio__mlock(a,b) EIO_ENOSYS ()
		1302	#else
		1303
		1304	static int
		1305	eio__mlock (void *addr, size_t length)
		1306	{
		1307	eio_page_align (&addr, &length);
		1308
		1309	return mlock (addr, length);
		1310	}
		1311
		1312	#endif
		1313
		1314	#if !(_POSIX_MAPPED_FILES && _POSIX_SYNCHRONIZED_IO)
		1315	# define eio__msync(a,b,c) EIO_ENOSYS ()
		1316	#else
		1317
		1318	static int
		1319	eio__msync (void *mem, size_t len, int flags)
		1320	{
		1321	eio_page_align (&mem, &len);
		1322
		1323	if (EIO_MS_ASYNC != MS_SYNC
		1324	|| EIO_MS_INVALIDATE != MS_INVALIDATE
		1325	|| EIO_MS_SYNC != MS_SYNC)
		1326	{
		1327	flags = 0
		1328	| (flags & EIO_MS_ASYNC ? MS_ASYNC : 0)
		1329	| (flags & EIO_MS_INVALIDATE ? MS_INVALIDATE : 0)
		1330	| (flags & EIO_MS_SYNC ? MS_SYNC : 0);
		1331	}
		1332
		1333	return msync (mem, len, flags);
		1334	}
		1335
		1336	#endif
		1337
		1338	static int
		1339	eio__mtouch (eio_req *req)
		1340	{
		1341	void *mem = req->ptr2;
		1342	size_t len = req->size;
		1343	int flags = req->int1;
		1344
		1345	eio_page_align (&mem, &len);
		1346
		1347	{
		1348	intptr_t addr = (intptr_t)mem;
		1349	intptr_t end = addr + len;
		1350	intptr_t page = eio_pagesize ();
		1351
		1352	if (addr < end)
		1353	if (flags & EIO_MT_MODIFY) /* modify */
		1354	do { *((volatile sig_atomic_t *)addr) |= 0; } while ((addr += page) < len && !EIO_CANCELLED (req));
		1355	else
		1356	do { *((volatile sig_atomic_t *)addr) ; } while ((addr += page) < len && !EIO_CANCELLED (req));
		1357	}
		1358
		1359	return 0;
		1360	}
		1361
		1362	/*****************************************************************************/
		1363	/* requests implemented outside eio_execute, because they are so large */
		1364
		1365	/* copies some absolute path to tmpbuf */
		1366	static char *
		1367	eio__getwd (struct tmpbuf *tmpbuf, eio_wd wd)
		1368	{
		1369	if (wd == EIO_CWD)
		1370	return getcwd (tmpbuf->ptr, PATH_MAX);
		1371
		1372	#if HAVE_AT
		1373	abort (); /*TODO*/
		1374	#else
		1375	strcpy (tmpbuf->ptr, wd);
		1376	#endif
		1377	return tmpbuf->ptr;
		1378	}
		1379
		1380	/* result will always end up in tmpbuf, there is always space for adding a 0-byte */
		1381	static int
		1382	eio__realpath (struct tmpbuf *tmpbuf, eio_wd wd, const char *path)
		1383	{
		1384	const char *rel = path;
		1385	char *res;
		1386	char *tmp1, *tmp2;
		1387	#if SYMLOOP_MAX > 32
		1388	int symlinks = SYMLOOP_MAX;
		1389	#else
		1390	int symlinks = 32;
		1391	#endif
		1392
		1393	/*D*/ /*TODO: wd ignored */
		1394
		1395	errno = EINVAL;
		1396	if (!rel)
		1397	return -1;
		1398
		1399	errno = ENOENT;
		1400	if (!*rel)
		1401	return -1;
		1402
		1403	res = tmpbuf_get (tmpbuf, PATH_MAX * 3);
		1404	tmp1 = res + PATH_MAX;
		1405	tmp2 = tmp1 + PATH_MAX;
		1406
		1407	#if 0 /* disabled, the musl way to do things is just too racy */
		1408	#if __linux && defined(O_NONBLOCK) && defined(O_NOATIME)
		1409	/* on linux we may be able to ask the kernel */
		1410	{
		1411	int fd = open (rel, O_RDONLY | O_NONBLOCK | O_NOCTTY | O_NOATIME);
		1412
		1413	if (fd >= 0)
		1414	{
		1415	sprintf (tmp1, "/proc/self/fd/%d", fd);
		1416	req->result = readlink (tmp1, res, PATH_MAX);
		1417	close (fd);
		1418
		1419	/* here we should probably stat the open file and the disk file, to make sure they still match */
		1420
		1421	if (req->result > 0)
		1422	goto done;
		1423	}
		1424	else if (errno == ELOOP || errno == ENAMETOOLONG || errno == ENOENT || errno == ENOTDIR || errno == EIO)
		1425	return;
		1426	}
		1427	#endif
		1428	#endif
		1429
		1430	if (*rel != '/')
		1431	{
		1432	if (!eio__getwd (tmpbuf, wd))
		1433	return -1;
		1434
		1435	if (res [1]) /* only use if not / */
		1436	res += strlen (res);
		1437	}
		1438
		1439	while (*rel)
		1440	{
		1441	eio_ssize_t len, linklen;
		1442	const char *beg = rel;
		1443
		1444	while (*rel && *rel != '/')
		1445	++rel;
		1446
		1447	len = rel - beg;
		1448
		1449	if (!len) /* skip slashes */
		1450	{
		1451	++rel;
		1452	continue;
		1453	}
		1454
		1455	if (beg [0] == '.')
		1456	{
		1457	if (len == 1)
		1458	continue; /* . - nop */
		1459
		1460	if (beg [1] == '.' && len == 2)
		1461	{
		1462	/* .. - back up one component, if possible */
		1463
		1464	while (res != tmpbuf->ptr)
		1465	if (*--res == '/')
		1466	break;
		1467
		1468	continue;
		1469	}
		1470	}
		1471
		1472	errno = ENAMETOOLONG;
		1473	if (res + 1 + len + 1 >= tmp1)
		1474	return;
		1475
		1476	/* copy one component */
		1477	*res = '/';
		1478	memcpy (res + 1, beg, len);
		1479
		1480	/* zero-terminate, for readlink */
		1481	res [len + 1] = 0;
		1482
		1483	/* now check if it's a symlink */
		1484	linklen = readlink (tmpbuf->ptr, tmp1, PATH_MAX);
		1485
		1486	if (linklen < 0)
		1487	{
		1488	if (errno != EINVAL)
		1489	return -1;
		1490
		1491	/* it's a normal directory. hopefully */
		1492	res += len + 1;
		1493	}
		1494	else
		1495	{
		1496	/* yay, it was a symlink - build new path in tmp2 */
		1497	int rellen = strlen (rel);
		1498
		1499	errno = ENAMETOOLONG;
		1500	if (linklen + 1 + rellen >= PATH_MAX)
		1501	return -1;
		1502
		1503	errno = ELOOP;
		1504	if (!--symlinks)
		1505	return -1;
		1506
		1507	if (*tmp1 == '/')
		1508	res = tmpbuf->ptr; /* symlink resolves to an absolute path */
		1509
		1510	/* we need to be careful, as rel might point into tmp2 already */
		1511	memmove (tmp2 + linklen + 1, rel, rellen + 1);
		1512	tmp2 [linklen] = '/';
		1513	memcpy (tmp2, tmp1, linklen);
		1514
		1515	rel = tmp2;
		1516	}
		1517	}
		1518
		1519	/* special case for the lone root path */
		1520	if (res == tmpbuf->ptr)
		1521	*res++ = '/';
		1522
		1523	return res - (char *)tmpbuf->ptr;
1041	}	1524	}
1042		1525
1043	static signed char	1526	static signed char
1044	eio_dent_cmp (const eio_dirent *a, const eio_dirent *b)	1527	eio_dent_cmp (const eio_dirent *a, const eio_dirent *b)
1045	{	1528	{
1046	return a->score - b->score ? a->score - b->score /* works because our signed char is always 0..100 */	1529	return a->score - b->score ? a->score - b->score /* works because our signed char is always 0..100 */
1047	: a->inode < b->inode ? -1 : a->inode > b->inode ? 1 : 0;	1530	: a->inode < b->inode ? -1
		1531	: a->inode > b->inode ? 1
		1532	: 0;
1048	}	1533	}
1049		1534
1050	#define EIO_DENT_CMP(i,op,j) eio_dent_cmp (&i, &j) op 0	1535	#define EIO_DENT_CMP(i,op,j) eio_dent_cmp (&i, &j) op 0
1051		1536
1052	#define EIO_SORT_CUTOFF 30 /* quite high, but performs well on many filesystems */	1537	#define EIO_SORT_CUTOFF 30 /* quite high, but performs well on many filesystems */
1053	#define EIO_SORT_FAST 60 /* when to only use insertion sort */	1538	#define EIO_SORT_FAST 60 /* when to only use insertion sort */
1054		1539
1055	static void	1540	static void
1056	eio_dent_radix_sort (eio_dirent *dents, int size, signed char score_bits, ino_t inode_bits)	1541	eio_dent_radix_sort (eio_dirent *dents, int size, signed char score_bits, eio_ino_t inode_bits)
1057	{	1542	{
1058	unsigned char bits [9 + sizeof (ino_t) * 8];	1543	unsigned char bits [9 + sizeof (eio_ino_t) * 8];
1059	unsigned char *bit = bits;	1544	unsigned char *bit = bits;
1060		1545
1061	assert (CHAR_BIT == 8);	1546	assert (CHAR_BIT == 8);
1062	assert (sizeof (eio_dirent) * 8 < 256);	1547	assert (sizeof (eio_dirent) * 8 < 256);
1063	assert (offsetof (eio_dirent, inode)); /* we use 0 as sentinel */	1548	assert (offsetof (eio_dirent, inode)); /* we use bit #0 as sentinel */
1064	assert (offsetof (eio_dirent, score)); /* we use 0 as sentinel */	1549	assert (offsetof (eio_dirent, score)); /* we use bit #0 as sentinel */
1065		1550
1066	if (size <= EIO_SORT_FAST)	1551	if (size <= EIO_SORT_FAST)
1067	return;	1552	return;
1068		1553
1069	/* first prepare an array of bits to test in our radix sort */	1554	/* first prepare an array of bits to test in our radix sort */
1070	/* try to take endianness into account, as well as differences in ino_t sizes */	1555	/* try to take endianness into account, as well as differences in eio_ino_t sizes */
1071	/* inode_bits must contain all inodes ORed together */	1556	/* inode_bits must contain all inodes ORed together */
1072	/* which is used to skip bits that are 0 everywhere, which is very common */	1557	/* which is used to skip bits that are 0 everywhere, which is very common */
1073	{	1558	{
1074	ino_t endianness;	1559	eio_ino_t endianness;
1075	int i, j;	1560	int i, j;
1076		1561
1077	/* we store the byte offset of byte n into byte n of "endianness" */	1562	/* we store the byte offset of byte n into byte n of "endianness" */
1078	for (i = 0; i < sizeof (ino_t); ++i)	1563	for (i = 0; i < sizeof (eio_ino_t); ++i)
1079	((unsigned char *)&endianness)[i] = i;	1564	((unsigned char *)&endianness)[i] = i;
1080		1565
1081	*bit++ = 0;	1566	*bit++ = 0;
1082		1567
1083	for (i = 0; i < sizeof (ino_t); ++i)	1568	for (i = 0; i < sizeof (eio_ino_t); ++i)
1084	{	1569	{
1085	/* shifting off the byte offsets out of "endianness" */	1570	/* shifting off the byte offsets out of "endianness" */
1086	int offs = (offsetof (eio_dirent, inode) + (endianness & 0xff)) * 8;	1571	int offs = (offsetof (eio_dirent, inode) + (endianness & 0xff)) * 8;
1087	endianness >>= 8;	1572	endianness >>= 8;
1088		1573
1089	for (j = 0; j < 8; ++j)	1574	for (j = 0; j < 8; ++j)
1090	if (inode_bits & (((ino_t)1) << (i * 8 + j)))	1575	if (inode_bits & (((eio_ino_t)1) << (i * 8 + j)))
1091	*bit++ = offs + j;	1576	*bit++ = offs + j;
1092	}	1577	}
1093		1578
1094	for (j = 0; j < 8; ++j)	1579	for (j = 0; j < 8; ++j)
1095	if (score_bits & (1 << j))	1580	if (score_bits & (1 << j))
1096	*bit++ = offsetof (eio_dirent, score) * 8 + j;	1581	*bit++ = offsetof (eio_dirent, score) * 8 + j;
1097	}	1582	}
1098		1583
1099	/* now actually do the sorting (a variant of MSD radix sort) */	1584	/* now actually do the sorting (a variant of MSD radix sort) */
1100	{	1585	{
1101	eio_dirent *base_stk [9 + sizeof (ino_t) * 8], *base;	1586	eio_dirent *base_stk [9 + sizeof (eio_ino_t) * 8], *base;
1102	eio_dirent *end_stk [9 + sizeof (ino_t) * 8], *end;	1587	eio_dirent *end_stk [9 + sizeof (eio_ino_t) * 8], *end;
1103	unsigned char *bit_stk [9 + sizeof (ino_t) * 8];	1588	unsigned char *bit_stk [9 + sizeof (eio_ino_t) * 8];
1104	int stk_idx = 0;	1589	int stk_idx = 0;
1105		1590
1106	base_stk [stk_idx] = dents;	1591	base_stk [stk_idx] = dents;
1107	end_stk [stk_idx] = dents + size;	1592	end_stk [stk_idx] = dents + size;
1108	bit_stk [stk_idx] = bit - 1;	1593	bit_stk [stk_idx] = bit - 1;
…		…
1187	}	1672	}
1188	}	1673	}
1189	}	1674	}
1190		1675
1191	static void	1676	static void
1192	eio_dent_sort (eio_dirent *dents, int size, signed char score_bits, ino_t inode_bits)	1677	eio_dent_sort (eio_dirent *dents, int size, signed char score_bits, eio_ino_t inode_bits)
1193	{	1678	{
1194	if (size <= 1)	1679	if (size <= 1)
1195	return; /* our insertion sort relies on size > 0 */	1680	return; /* our insertion sort relies on size > 0 */
1196		1681
1197	/* first we use a radix sort, but only for dirs >= EIO_SORT_FAST */	1682	/* first we use a radix sort, but only for dirs >= EIO_SORT_FAST */
…		…
1203	eio_dent_insertion_sort (dents, size);	1688	eio_dent_insertion_sort (dents, size);
1204	}	1689	}
1205		1690
1206	/* read a full directory */	1691	/* read a full directory */
1207	static void	1692	static void
1208	eio__scandir (eio_req *req, etp_worker *self)	1693	eio__scandir (eio_req *req)
1209	{	1694	{
1210	DIR *dirp;
1211	EIO_STRUCT_DIRENT *entp;
1212	char *name, *names;	1695	char *name, *names;
1213	int namesalloc = 4096;	1696	int namesalloc = 4096 - sizeof (void *) * 4;
1214	int namesoffs = 0;	1697	int namesoffs = 0;
1215	int flags = req->int1;	1698	int flags = req->int1;
1216	eio_dirent *dents = 0;	1699	eio_dirent *dents = 0;
1217	int dentalloc = 128;	1700	int dentalloc = 128;
1218	int dentoffs = 0;	1701	int dentoffs = 0;
1219	ino_t inode_bits = 0;	1702	eio_ino_t inode_bits = 0;
		1703	#ifdef _WIN32
		1704	HANDLE dirp;
		1705	WIN32_FIND_DATA entp;
		1706	#else
		1707	DIR *dirp;
		1708	EIO_STRUCT_DIRENT *entp;
		1709	#endif
1220		1710
1221	req->result = -1;	1711	req->result = -1;
1222		1712
1223	if (!(flags & EIO_READDIR_DENTS))	1713	if (!(flags & EIO_READDIR_DENTS))
1224	flags &= ~(EIO_READDIR_DIRS_FIRST | EIO_READDIR_STAT_ORDER);	1714	flags &= ~(EIO_READDIR_DIRS_FIRST | EIO_READDIR_STAT_ORDER);
1225		1715
1226	X_LOCK (wrklock);	1716	#ifdef _WIN32
1227	/* the corresponding closedir is in ETP_WORKER_CLEAR */	1717	{
		1718	int len = strlen ((const char *)req->ptr1);
		1719	char *path = malloc (MAX_PATH);
		1720	const char *fmt;
		1721
		1722	if (!len)
		1723	fmt = "./*";
		1724	else if (((const char *)req->ptr1)[len - 1] == '/' || ((const char *)req->ptr1)[len - 1] == '\\')
		1725	fmt = "%s*";
		1726	else
		1727	fmt = "%s/*";
		1728
		1729	_snprintf (path, MAX_PATH, fmt, (const char *)req->ptr1);
		1730	dirp = FindFirstFile (path, &entp);
		1731	free (path);
		1732
		1733	if (dirp == INVALID_HANDLE_VALUE)
		1734	{
		1735	/* should steal _dosmaperr */
		1736	switch (GetLastError ())
		1737	{
		1738	case ERROR_FILE_NOT_FOUND:
		1739	req->result = 0;
		1740	break;
		1741
		1742	case ERROR_INVALID_NAME:
		1743	case ERROR_PATH_NOT_FOUND:
		1744	case ERROR_NO_MORE_FILES:
		1745	errno = ENOENT;
		1746	break;
		1747
		1748	case ERROR_NOT_ENOUGH_MEMORY:
		1749	errno = ENOMEM;
		1750	break;
		1751
		1752	default:
		1753	errno = EINVAL;
		1754	break;
		1755	}
		1756
		1757	return;
		1758	}
		1759	}
		1760	#else
1228	self->dirp = dirp = opendir (req->ptr1);	1761	dirp = opendir (req->ptr1);
		1762
		1763	if (!dirp)
		1764	return;
		1765	#endif
		1766
		1767	if (req->flags & EIO_FLAG_PTR1_FREE)
		1768	free (req->ptr1);
		1769
1229	req->flags |= EIO_FLAG_PTR1_FREE | EIO_FLAG_PTR2_FREE;	1770	req->flags |= EIO_FLAG_PTR1_FREE | EIO_FLAG_PTR2_FREE;
1230	req->ptr1 = dents = flags ? malloc (dentalloc * sizeof (eio_dirent)) : 0;	1771	req->ptr1 = dents = flags ? malloc (dentalloc * sizeof (eio_dirent)) : 0;
1231	req->ptr2 = names = malloc (namesalloc);	1772	req->ptr2 = names = malloc (namesalloc);
1232	X_UNLOCK (wrklock);
1233		1773
1234	if (dirp && names && (!flags || dents))	1774	if (!names || (flags && !dents))
		1775	return;
		1776
1235	for (;;)	1777	for (;;)
1236	{	1778	{
		1779	int done;
		1780
		1781	#ifdef _WIN32
		1782	done = !dirp;
		1783	#else
1237	errno = 0;	1784	errno = 0;
1238	entp = readdir (dirp);	1785	entp = readdir (dirp);
		1786	done = !entp;
		1787	#endif
1239		1788
1240	if (!entp)	1789	if (done)
1241	{	1790	{
		1791	#ifndef _WIN32
		1792	int old_errno = errno;
		1793	closedir (dirp);
		1794	errno = old_errno;
		1795
1242	if (errno)	1796	if (errno)
1243	break;	1797	break;
		1798	#endif
1244		1799
1245	/* sort etc. */	1800	/* sort etc. */
1246	req->int1 = flags;	1801	req->int1 = flags;
1247	req->result = dentoffs;	1802	req->result = dentoffs;
1248		1803
1249	if (flags & EIO_READDIR_STAT_ORDER)	1804	if (flags & EIO_READDIR_STAT_ORDER)
1250	eio_dent_sort (dents, dentoffs, 0, inode_bits); /* sort by inode exclusively */	1805	eio_dent_sort (dents, dentoffs, flags & EIO_READDIR_DIRS_FIRST ? 7 : 0, inode_bits);
1251	else if (flags & EIO_READDIR_DIRS_FIRST)	1806	else if (flags & EIO_READDIR_DIRS_FIRST)
1252	if (flags & EIO_READDIR_FOUND_UNKNOWN)	1807	if (flags & EIO_READDIR_FOUND_UNKNOWN)
1253	eio_dent_sort (dents, dentoffs, 7, inode_bits); /* sort by score and inode */	1808	eio_dent_sort (dents, dentoffs, 7, inode_bits); /* sort by score and inode */
1254	else	1809	else
		1810	{
		1811	/* in this case, all is known, and we just put dirs first and sort them */
		1812	eio_dirent *oth = dents + dentoffs;
		1813	eio_dirent *dir = dents;
		1814
		1815	/* now partition dirs to the front, and non-dirs to the back */
		1816	/* by walking from both sides and swapping if necessary */
		1817	while (oth > dir)
		1818	{
		1819	if (dir->type == EIO_DT_DIR)
		1820	++dir;
		1821	else if ((--oth)->type == EIO_DT_DIR)
		1822	{
		1823	eio_dirent tmp = *dir; *dir = *oth; *oth = tmp;
		1824
		1825	++dir;
		1826	}
		1827	}
		1828
		1829	/* now sort the dirs only (dirs all have the same score) */
		1830	eio_dent_sort (dents, dir - dents, 0, inode_bits);
		1831	}
		1832
		1833	break;
		1834	}
		1835
		1836	/* now add the entry to our list(s) */
		1837	name = D_NAME (entp);
		1838
		1839	/* skip . and .. entries */
		1840	if (name [0] != '.' || (name [1] && (name [1] != '.' || name [2])))
		1841	{
		1842	int len = D_NAMLEN (entp) + 1;
		1843
		1844	while (ecb_expect_false (namesoffs + len > namesalloc))
		1845	{
		1846	namesalloc *= 2;
		1847	req->ptr2 = names = realloc (names, namesalloc);
		1848
		1849	if (!names)
		1850	break;
		1851	}
		1852
		1853	memcpy (names + namesoffs, name, len);
		1854
		1855	if (dents)
		1856	{
		1857	struct eio_dirent *ent;
		1858
		1859	if (ecb_expect_false (dentoffs == dentalloc))
1255	{	1860	{
1256	/* in this case, all is known, and we just put dirs first and sort them */	1861	dentalloc *= 2;
		1862	req->ptr1 = dents = realloc (dents, dentalloc * sizeof (eio_dirent));
		1863
		1864	if (!dents)
		1865	break;
		1866	}
		1867
1257	eio_dirent *oth = dents + dentoffs;	1868	ent = dents + dentoffs;
1258	eio_dirent *dir = dents;
1259		1869
1260	/* now partition dirs to the front, and non-dirs to the back */	1870	ent->nameofs = namesoffs; /* rather dirtily we store the offset in the pointer */
1261	/* by walking from both sides and swapping if necessary */	1871	ent->namelen = len - 1;
1262	/* also clear score, so it doesn't influence sorting */	1872	ent->inode = D_INO (entp);
1263	while (oth > dir)	1873
		1874	inode_bits |= ent->inode;
		1875
		1876	switch (D_TYPE (entp))
		1877	{
		1878	default:
		1879	ent->type = EIO_DT_UNKNOWN;
		1880	flags |= EIO_READDIR_FOUND_UNKNOWN;
		1881	break;
		1882
		1883	#ifdef DT_FIFO
		1884	case DT_FIFO: ent->type = EIO_DT_FIFO; break;
		1885	#endif
		1886	#ifdef DT_CHR
		1887	case DT_CHR: ent->type = EIO_DT_CHR; break;
		1888	#endif
		1889	#ifdef DT_MPC
		1890	case DT_MPC: ent->type = EIO_DT_MPC; break;
		1891	#endif
		1892	#ifdef DT_DIR
		1893	case DT_DIR: ent->type = EIO_DT_DIR; break;
		1894	#endif
		1895	#ifdef DT_NAM
		1896	case DT_NAM: ent->type = EIO_DT_NAM; break;
		1897	#endif
		1898	#ifdef DT_BLK
		1899	case DT_BLK: ent->type = EIO_DT_BLK; break;
		1900	#endif
		1901	#ifdef DT_MPB
		1902	case DT_MPB: ent->type = EIO_DT_MPB; break;
		1903	#endif
		1904	#ifdef DT_REG
		1905	case DT_REG: ent->type = EIO_DT_REG; break;
		1906	#endif
		1907	#ifdef DT_NWK
		1908	case DT_NWK: ent->type = EIO_DT_NWK; break;
		1909	#endif
		1910	#ifdef DT_CMP
		1911	case DT_CMP: ent->type = EIO_DT_CMP; break;
		1912	#endif
		1913	#ifdef DT_LNK
		1914	case DT_LNK: ent->type = EIO_DT_LNK; break;
		1915	#endif
		1916	#ifdef DT_SOCK
		1917	case DT_SOCK: ent->type = EIO_DT_SOCK; break;
		1918	#endif
		1919	#ifdef DT_DOOR
		1920	case DT_DOOR: ent->type = EIO_DT_DOOR; break;
		1921	#endif
		1922	#ifdef DT_WHT
		1923	case DT_WHT: ent->type = EIO_DT_WHT; break;
		1924	#endif
		1925	}
		1926
		1927	ent->score = 7;
		1928
		1929	if (flags & EIO_READDIR_DIRS_FIRST)
		1930	{
		1931	if (ent->type == EIO_DT_UNKNOWN)
1264	{	1932	{
1265	if (dir->type == EIO_DT_DIR)	1933	if (*name == '.') /* leading dots are likely directories, and, in any case, rare */
1266	++dir;	1934	ent->score = 1;
1267	else if ((--oth)->type == EIO_DT_DIR)	1935	else if (!strchr (name, '.')) /* absense of dots indicate likely dirs */
1268	{	1936	ent->score = len <= 2 ? 4 - len : len <= 4 ? 4 : len <= 7 ? 5 : 6; /* shorter == more likely dir, but avoid too many classes */
1269	eio_dirent tmp = *dir; *dir = *oth; *oth = tmp;
1270
1271	++dir;
1272	}
1273	}	1937	}
1274		1938	else if (ent->type == EIO_DT_DIR)
1275	/* now sort the dirs only */	1939	ent->score = 0;
1276	eio_dent_sort (dents, dir - dents, 0, inode_bits);
1277	}	1940	}
1278
1279	break;
1280	}
1281
1282	/* now add the entry to our list(s) */
1283	name = entp->d_name;
1284
1285	/* skip . and .. entries */
1286	if (name [0] != '.' || (name [1] && (name [1] != '.' || name [2])))
1287	{
1288	int len = D_NAMLEN (entp) + 1;
1289
1290	while (expect_false (namesoffs + len > namesalloc))
1291	{
1292	namesalloc *= 2;
1293	X_LOCK (wrklock);
1294	req->ptr2 = names = realloc (names, namesalloc);
1295	X_UNLOCK (wrklock);
1296
1297	if (!names)
1298	break;
1299	}	1941	}
1300		1942
1301	memcpy (names + namesoffs, name, len);
1302
1303	if (dents)
1304	{
1305	struct eio_dirent *ent;
1306
1307	if (expect_false (dentoffs == dentalloc))
1308	{
1309	dentalloc *= 2;
1310	X_LOCK (wrklock);
1311	req->ptr1 = dents = realloc (dents, dentalloc * sizeof (eio_dirent));
1312	X_UNLOCK (wrklock);
1313
1314	if (!dents)
1315	break;
1316	}
1317
1318	ent = dents + dentoffs;
1319
1320	ent->nameofs = namesoffs; /* rather dirtily we store the offset in the pointer */
1321	ent->namelen = len - 1;
1322	ent->inode = D_INO (entp);
1323
1324	inode_bits |= ent->inode;
1325
1326	switch (D_TYPE (entp))
1327	{
1328	default:
1329	ent->type = EIO_DT_UNKNOWN;
1330	flags |= EIO_READDIR_FOUND_UNKNOWN;
1331	break;
1332
1333	#ifdef DT_FIFO
1334	case DT_FIFO: ent->type = EIO_DT_FIFO; break;
1335	#endif
1336	#ifdef DT_CHR
1337	case DT_CHR: ent->type = EIO_DT_CHR; break;
1338	#endif
1339	#ifdef DT_MPC
1340	case DT_MPC: ent->type = EIO_DT_MPC; break;
1341	#endif
1342	#ifdef DT_DIR
1343	case DT_DIR: ent->type = EIO_DT_DIR; break;
1344	#endif
1345	#ifdef DT_NAM
1346	case DT_NAM: ent->type = EIO_DT_NAM; break;
1347	#endif
1348	#ifdef DT_BLK
1349	case DT_BLK: ent->type = EIO_DT_BLK; break;
1350	#endif
1351	#ifdef DT_MPB
1352	case DT_MPB: ent->type = EIO_DT_MPB; break;
1353	#endif
1354	#ifdef DT_REG
1355	case DT_REG: ent->type = EIO_DT_REG; break;
1356	#endif
1357	#ifdef DT_NWK
1358	case DT_NWK: ent->type = EIO_DT_NWK; break;
1359	#endif
1360	#ifdef DT_CMP
1361	case DT_CMP: ent->type = EIO_DT_CMP; break;
1362	#endif
1363	#ifdef DT_LNK
1364	case DT_LNK: ent->type = EIO_DT_LNK; break;
1365	#endif
1366	#ifdef DT_SOCK
1367	case DT_SOCK: ent->type = EIO_DT_SOCK; break;
1368	#endif
1369	#ifdef DT_DOOR
1370	case DT_DOOR: ent->type = EIO_DT_DOOR; break;
1371	#endif
1372	#ifdef DT_WHT
1373	case DT_WHT: ent->type = EIO_DT_WHT; break;
1374	#endif
1375	}
1376
1377	ent->score = 7;
1378
1379	if (flags & EIO_READDIR_DIRS_FIRST)
1380	{
1381	if (ent->type == EIO_DT_UNKNOWN)
1382	{
1383	if (*name == '.') /* leading dots are likely directories, and, in any case, rare */
1384	ent->score = 1;
1385	else if (!strchr (name, '.')) /* absense of dots indicate likely dirs */
1386	ent->score = len <= 2 ? 4 - len : len <= 4 ? 4 : len <= 7 ? 5 : 6; /* shorter == more likely dir, but avoid too many classes */
1387	}
1388	else if (ent->type == EIO_DT_DIR)
1389	ent->score = 0;
1390	}
1391	}
1392
1393	namesoffs += len;	1943	namesoffs += len;
1394	++dentoffs;	1944	++dentoffs;
1395	}	1945	}
1396		1946
1397	if (EIO_CANCELLED (req))	1947	if (EIO_CANCELLED (req))
1398	{	1948	{
1399	errno = ECANCELED;	1949	errno = ECANCELED;
1400	break;	1950	break;
1401	}	1951	}
		1952
		1953	#ifdef _WIN32
		1954	if (!FindNextFile (dirp, &entp))
		1955	{
		1956	FindClose (dirp);
		1957	dirp = 0;
1402	}	1958	}
		1959	#endif
		1960	}
1403	}	1961	}
1404		1962
1405	#ifdef PAGESIZE	1963	/*****************************************************************************/
1406	# define eio_pagesize() PAGESIZE	1964	/* working directory stuff */
		1965
		1966	#if HAVE_AT
		1967
		1968	#define WD2FD(wd) ((wd) ? ((int)(long)(wd)) - 1 : AT_FDCWD)
		1969
		1970	#ifndef O_SEARCH
		1971	# define O_SEARCH O_RDONLY
		1972	#endif
		1973
		1974	eio_wd
		1975	eio_wd_open_sync (eio_wd wd, const char *path)
		1976	{
		1977	int fd = openat (WD2FD (wd), path, O_CLOEXEC | O_SEARCH | O_DIRECTORY);
		1978
		1979	return fd >= 0 ? (eio_wd)(long)(fd + 1) : EIO_INVALID_WD;
		1980	}
		1981
		1982	static eio_wd
		1983	eio__wd_open_sync (struct tmpbuf *tmpbuf, eio_wd wd, const char *path)
		1984	{
		1985	return eio_wd_open_sync (wd, path);
		1986	}
		1987
		1988	void
		1989	eio_wd_close_sync (eio_wd wd)
		1990	{
		1991	int fd = WD2FD (wd);
		1992
		1993	if (fd >= 0)
		1994	close (fd);
		1995	}
		1996
		1997	static int
		1998	eio__truncateat (int dirfd, const char *path, off_t length)
		1999	{
		2000	int fd = openat (dirfd, path, O_WRONLY | O_CLOEXEC);
		2001	int res;
		2002
		2003	if (fd < 0)
		2004	return fd;
		2005
		2006	res = ftruncate (fd, length);
		2007	close (fd);
		2008	return res;
		2009	}
		2010
		2011	static int
		2012	eio__statvfsat (int dirfd, const char *path, struct statvfs *buf)
		2013	{
		2014	int fd = openat (dirfd, path, O_SEARCH | O_CLOEXEC);
		2015	int res;
		2016
		2017	if (fd < 0)
		2018	return fd;
		2019
		2020	res = fstatvfs (fd, buf);
		2021	close (fd);
		2022	return res;
		2023
		2024	}
		2025
1407	#else	2026	#else
1408	static intptr_t
1409	eio_pagesize (void)
1410	{
1411	static intptr_t page;
1412		2027
1413	if (!page)	2028	/* on legacy systems, we represent the working directories simply by their path strings */
1414	page = sysconf (_SC_PAGESIZE);
1415		2029
		2030	static const char *
		2031	wd_expand (struct tmpbuf *tmpbuf, eio_wd wd, const char *path)
		2032	{
		2033	if (!wd || *path == '/')
1416	return page;	2034	return path;
1417	}
1418	#endif
1419		2035
		2036	{
		2037	int l1 = strlen ((const char *)wd);
		2038	int l2 = strlen (path);
		2039
		2040	char *res = tmpbuf_get (tmpbuf, l1 + l2 + 2);
		2041
		2042	memcpy (res, wd, l1);
		2043	res [l1] = '/';
		2044	memcpy (res + l1 + 1, path, l2 + 1);
		2045
		2046	return res;
		2047	}
		2048	}
		2049
1420	static void	2050	static eio_wd
1421	eio_page_align (void **addr, size_t *length)	2051	eio__wd_open_sync (struct tmpbuf *tmpbuf, eio_wd wd, const char *path)
1422	{	2052	{
1423	intptr_t mask = eio_pagesize () - 1;	2053	if (*path == '/') /* absolute paths ignore wd */
1424		2054	path = strdup (path);
1425	/* round down addr */	2055	else if (path [0] == '.' && !path [1]) /* special case '.', as it is common */
1426	intptr_t adj = mask & (intptr_t)*addr;	2056	return wd;
1427		2057	else
1428	*addr = (void *)((intptr_t)*addr - adj);
1429	*length += adj;
1430
1431	/* round up length */
1432	*length = (*length + mask) & ~mask;
1433	}
1434
1435	#if !_POSIX_MEMLOCK
1436	# define eio__mlockall(a) ((errno = ENOSYS), -1)
1437	#else
1438
1439	static int
1440	eio__mlockall (int flags)
1441	{
1442	#if __GLIBC__ == 2 && __GLIBC_MINOR__ <= 7
1443	extern int mallopt (int, int);
1444	mallopt (-6, 238); /* http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=473812 */
1445	#endif
1446
1447	if (EIO_MCL_CURRENT != MCL_CURRENT
1448	|| EIO_MCL_FUTURE != MCL_FUTURE)
1449	{	2058	{
1450	flags = 0	2059	int len = eio__realpath (tmpbuf, wd, path);
1451	| (flags & EIO_MCL_CURRENT ? MCL_CURRENT : 0)
1452	| (flags & EIO_MCL_FUTURE ? MCL_FUTURE : 0);
1453	}
1454		2060
1455	return mlockall (flags);	2061	path = EIO_INVALID_WD;
1456	}
1457	#endif
1458		2062
1459	#if !_POSIX_MEMLOCK_RANGE	2063	if (len >= 0)
1460	# define eio__mlock(a,b) ((errno = ENOSYS), -1)	2064	{
1461	#else	2065	((char *)tmpbuf->ptr)[len] = 0;
1462		2066	path = strdup (tmpbuf->ptr);
1463	static int	2067	}
1464	eio__mlock (void *addr, size_t length)
1465	{
1466	eio_page_align (&addr, &length);
1467
1468	return mlock (addr, length);
1469	}
1470
1471	#endif
1472
1473	#if !(_POSIX_MAPPED_FILES && _POSIX_SYNCHRONIZED_IO)
1474	# define eio__msync(a,b,c) ((errno = ENOSYS), -1)
1475	#else
1476
1477	int
1478	eio__msync (void *mem, size_t len, int flags)
1479	{
1480	eio_page_align (&mem, &len);
1481
1482	if (EIO_MS_ASYNC != MS_SYNC
1483	|| EIO_MS_INVALIDATE != MS_INVALIDATE
1484	|| EIO_MS_SYNC != MS_SYNC)
1485	{	2068	}
1486	flags = 0
1487	| (flags & EIO_MS_ASYNC ? MS_ASYNC : 0)
1488	| (flags & EIO_MS_INVALIDATE ? MS_INVALIDATE : 0)
1489	| (flags & EIO_MS_SYNC ? MS_SYNC : 0);
1490	}
1491		2069
1492	return msync (mem, len, flags);	2070	if (!path)
1493	}	2071	path = EIO_INVALID_WD;
1494		2072
1495	#endif	2073	return (eio_wd)path;
		2074	}
1496		2075
1497	int	2076	eio_wd
1498	eio__mtouch (void *mem, size_t len, int flags)	2077	eio_wd_open_sync (eio_wd wd, const char *path)
1499	{	2078	{
1500	eio_page_align (&mem, &len);	2079	struct tmpbuf tmpbuf = { 0 };
		2080	wd = eio__wd_open_sync (&tmpbuf, wd, path);
		2081	free (tmpbuf.ptr);
1501		2082
1502	{
1503	intptr_t addr = (intptr_t)mem;
1504	intptr_t end = addr + len;
1505	intptr_t page = eio_pagesize ();
1506
1507	if (addr < end)
1508	if (flags & EIO_MT_MODIFY) /* modify */
1509	do { *((volatile sig_atomic_t *)addr) |= 0; } while ((addr += page) < len);
1510	else
1511	do { *((volatile sig_atomic_t *)addr) ; } while ((addr += page) < len);
1512	}
1513
1514	return 0;	2083	return wd;
1515	}	2084	}
		2085
		2086	void
		2087	eio_wd_close_sync (eio_wd wd)
		2088	{
		2089	if (wd != EIO_INVALID_WD)
		2090	free (wd);
		2091	}
		2092
		2093	#endif
1516		2094
1517	/*****************************************************************************/	2095	/*****************************************************************************/
1518		2096
1519	#define ALLOC(len) \	2097	#define ALLOC(len) \
1520	if (!req->ptr2) \	2098	if (!req->ptr2) \
…		…
1535	{	2113	{
1536	ETP_REQ *req;	2114	ETP_REQ *req;
1537	struct timespec ts;	2115	struct timespec ts;
1538	etp_worker *self = (etp_worker *)thr_arg;	2116	etp_worker *self = (etp_worker *)thr_arg;
1539		2117
		2118	#if HAVE_PRCTL_SET_NAME
		2119	prctl (PR_SET_NAME, (unsigned long)"eio_thread", 0, 0, 0);
		2120	#endif
		2121
1540	/* try to distribute timeouts somewhat randomly */	2122	/* try to distribute timeouts somewhat evenly */
1541	ts.tv_nsec = ((unsigned long)self & 1023UL) * (1000000000UL / 1024UL);	2123	ts.tv_nsec = ((unsigned long)self & 1023UL) * (1000000000UL / 1024UL);
1542		2124
1543	for (;;)	2125	for (;;)
1544	{	2126	{
		2127	ts.tv_sec = 0;
		2128
1545	X_LOCK (reqlock);	2129	X_LOCK (reqlock);
1546		2130
1547	for (;;)	2131	for (;;)
1548	{	2132	{
1549	self->req = req = reqq_shift (&req_queue);	2133	req = reqq_shift (&req_queue);
1550		2134
1551	if (req)	2135	if (req)
1552	break;	2136	break;
1553		2137
		2138	if (ts.tv_sec == 1) /* no request, but timeout detected, let's quit */
		2139	{
		2140	X_UNLOCK (reqlock);
		2141	X_LOCK (wrklock);
		2142	--started;
		2143	X_UNLOCK (wrklock);
		2144	goto quit;
		2145	}
		2146
1554	++idle;	2147	++idle;
1555		2148
1556	ts.tv_sec = time (0) + IDLE_TIMEOUT;	2149	if (idle <= max_idle)
1557	if (X_COND_TIMEDWAIT (reqwait, reqlock, ts) == ETIMEDOUT)	2150	/* we are allowed to idle, so do so without any timeout */
		2151	X_COND_WAIT (reqwait, reqlock);
		2152	else
1558	{	2153	{
1559	if (idle > max_idle)	2154	/* initialise timeout once */
1560	{	2155	if (!ts.tv_sec)
1561	--idle;	2156	ts.tv_sec = time (0) + idle_timeout;
1562	X_UNLOCK (reqlock);
1563	X_LOCK (wrklock);
1564	--started;
1565	X_UNLOCK (wrklock);
1566	goto quit;
1567	}
1568		2157
1569	/* we are allowed to idle, so do so without any timeout */
1570	X_COND_WAIT (reqwait, reqlock);	2158	if (X_COND_TIMEDWAIT (reqwait, reqlock, ts) == ETIMEDOUT)
		2159	ts.tv_sec = 1; /* assuming this is not a value computed above.,.. */
1571	}	2160	}
1572		2161
1573	--idle;	2162	--idle;
1574	}	2163	}
1575		2164
…		…
1578	X_UNLOCK (reqlock);	2167	X_UNLOCK (reqlock);
1579		2168
1580	if (req->type < 0)	2169	if (req->type < 0)
1581	goto quit;	2170	goto quit;
1582		2171
1583	if (!EIO_CANCELLED (req))
1584	ETP_EXECUTE (self, req);	2172	ETP_EXECUTE (self, req);
1585		2173
1586	X_LOCK (reslock);	2174	X_LOCK (reslock);
1587		2175
1588	++npending;	2176	++npending;
1589		2177
1590	if (!reqq_push (&res_queue, req) && want_poll_cb)	2178	if (!reqq_push (&res_queue, req) && want_poll_cb)
1591	want_poll_cb ();	2179	want_poll_cb ();
1592		2180
1593	self->req = 0;
1594	etp_worker_clear (self);	2181	etp_worker_clear (self);
1595		2182
1596	X_UNLOCK (reslock);	2183	X_UNLOCK (reslock);
1597	}	2184	}
1598		2185
1599	quit:	2186	quit:
		2187	free (req);
		2188
1600	X_LOCK (wrklock);	2189	X_LOCK (wrklock);
1601	etp_worker_free (self);	2190	etp_worker_free (self);
1602	X_UNLOCK (wrklock);	2191	X_UNLOCK (wrklock);
1603		2192
1604	return 0;	2193	return 0;
1605	}	2194	}
1606		2195
1607	/*****************************************************************************/	2196	/*****************************************************************************/
1608		2197
		2198	int ecb_cold
1609	int eio_init (void (*want_poll)(void), void (*done_poll)(void))	2199	eio_init (void (*want_poll)(void), void (*done_poll)(void))
1610	{	2200	{
		2201	#if !HAVE_PREADWRITE
		2202	X_MUTEX_CREATE (preadwritelock);
		2203	#endif
		2204
1611	return etp_init (want_poll, done_poll);	2205	return etp_init (want_poll, done_poll);
1612	}	2206	}
1613		2207
		2208	ecb_inline void
1614	static void eio_api_destroy (eio_req *req)	2209	eio_api_destroy (eio_req *req)
1615	{	2210	{
1616	free (req);	2211	free (req);
1617	}	2212	}
1618		2213
1619	#define REQ(rtype) \	2214	#define REQ(rtype) \
…		…
1638	{ \	2233	{ \
1639	eio_api_destroy (req); \	2234	eio_api_destroy (req); \
1640	return 0; \	2235	return 0; \
1641	}	2236	}
1642		2237
		2238	static void
1643	static void eio_execute (etp_worker *self, eio_req *req)	2239	eio_execute (etp_worker *self, eio_req *req)
1644	{	2240	{
		2241	#if HAVE_AT
		2242	int dirfd;
		2243	#else
		2244	const char *path;
		2245	#endif
		2246
		2247	if (ecb_expect_false (EIO_CANCELLED (req)))
		2248	{
		2249	req->result = -1;
		2250	req->errorno = ECANCELED;
		2251	return;
		2252	}
		2253
		2254	if (ecb_expect_false (req->wd == EIO_INVALID_WD))
		2255	{
		2256	req->result = -1;
		2257	req->errorno = ENOENT;
		2258	return;
		2259	}
		2260
		2261	if (req->type >= EIO_OPEN)
		2262	{
		2263	#if HAVE_AT
		2264	dirfd = WD2FD (req->wd);
		2265	#else
		2266	path = wd_expand (&self->tmpbuf, req->wd, req->ptr1);
		2267	#endif
		2268	}
		2269
1645	switch (req->type)	2270	switch (req->type)
1646	{	2271	{
		2272	case EIO_WD_OPEN: req->wd = eio__wd_open_sync (&self->tmpbuf, req->wd, req->ptr1);
		2273	req->result = req->wd == EIO_INVALID_WD ? -1 : 0;
		2274	break;
		2275	case EIO_WD_CLOSE: eio_wd_close_sync (req->wd); break;
		2276
1647	case EIO_READ: ALLOC (req->size);	2277	case EIO_READ: ALLOC (req->size);
1648	req->result = req->offs >= 0	2278	req->result = req->offs >= 0
1649	? pread (req->int1, req->ptr2, req->size, req->offs)	2279	? pread (req->int1, req->ptr2, req->size, req->offs)
1650	: read (req->int1, req->ptr2, req->size); break;	2280	: read (req->int1, req->ptr2, req->size); break;
1651	case EIO_WRITE: req->result = req->offs >= 0	2281	case EIO_WRITE: req->result = req->offs >= 0
1652	? pwrite (req->int1, req->ptr2, req->size, req->offs)	2282	? pwrite (req->int1, req->ptr2, req->size, req->offs)
1653	: write (req->int1, req->ptr2, req->size); break;	2283	: write (req->int1, req->ptr2, req->size); break;
1654		2284
1655	case EIO_READAHEAD: req->result = readahead (req->int1, req->offs, req->size); break;	2285	case EIO_READAHEAD: req->result = readahead (req->int1, req->offs, req->size); break;
1656	case EIO_SENDFILE: req->result = eio__sendfile (req->int1, req->int2, req->offs, req->size, self); break;	2286	case EIO_SENDFILE: req->result = eio__sendfile (req->int1, req->int2, req->offs, req->size); break;
1657		2287
		2288	#if HAVE_AT
1658	case EIO_STAT: ALLOC (sizeof (EIO_STRUCT_STAT));	2289	case EIO_STAT: ALLOC (sizeof (EIO_STRUCT_STAT));
1659	req->result = stat (req->ptr1, (EIO_STRUCT_STAT *)req->ptr2); break;	2290	req->result = fstatat (dirfd, req->ptr1, (EIO_STRUCT_STAT *)req->ptr2, 0); break;
1660	case EIO_LSTAT: ALLOC (sizeof (EIO_STRUCT_STAT));	2291	case EIO_LSTAT: ALLOC (sizeof (EIO_STRUCT_STAT));
		2292	req->result = fstatat (dirfd, req->ptr1, (EIO_STRUCT_STAT *)req->ptr2, AT_SYMLINK_NOFOLLOW); break;
		2293	case EIO_CHOWN: req->result = fchownat (dirfd, req->ptr1, req->int2, req->int3, 0); break;
		2294	case EIO_CHMOD: req->result = fchmodat (dirfd, req->ptr1, (mode_t)req->int2, 0); break;
		2295	case EIO_TRUNCATE: req->result = eio__truncateat (dirfd, req->ptr1, req->offs); break;
		2296	case EIO_OPEN: req->result = openat (dirfd, req->ptr1, req->int1, (mode_t)req->int2); break;
		2297
		2298	case EIO_UNLINK: req->result = unlinkat (dirfd, req->ptr1, 0); break;
		2299	case EIO_RMDIR: req->result = unlinkat (dirfd, req->ptr1, AT_REMOVEDIR); break;
		2300	case EIO_MKDIR: req->result = mkdirat (dirfd, req->ptr1, (mode_t)req->int2); break;
		2301	case EIO_RENAME: req->result = renameat (dirfd, req->ptr1, WD2FD (req->int3), req->ptr2); break;
		2302	case EIO_LINK: req->result = linkat (dirfd, req->ptr1, WD2FD (req->int3), req->ptr2, 0); break;
		2303	case EIO_SYMLINK: req->result = symlinkat (req->ptr1, dirfd, req->ptr2); break;
		2304	case EIO_MKNOD: req->result = mknodat (dirfd, req->ptr1, (mode_t)req->int2, (dev_t)req->offs); break;
		2305	case EIO_READLINK: ALLOC (PATH_MAX);
		2306	req->result = readlinkat (dirfd, req->ptr1, req->ptr2, PATH_MAX); break;
		2307	case EIO_STATVFS: ALLOC (sizeof (EIO_STRUCT_STATVFS));
		2308	req->result = eio__statvfsat (dirfd, req->ptr1, (EIO_STRUCT_STATVFS *)req->ptr2); break;
		2309	#else
		2310	case EIO_STAT: ALLOC (sizeof (EIO_STRUCT_STAT));
		2311	req->result = stat (path , (EIO_STRUCT_STAT *)req->ptr2); break;
		2312	case EIO_LSTAT: ALLOC (sizeof (EIO_STRUCT_STAT));
1661	req->result = lstat (req->ptr1, (EIO_STRUCT_STAT *)req->ptr2); break;	2313	req->result = lstat (path , (EIO_STRUCT_STAT *)req->ptr2); break;
		2314	case EIO_CHOWN: req->result = chown (path , req->int2, req->int3); break;
		2315	case EIO_CHMOD: req->result = chmod (path , (mode_t)req->int2); break;
		2316	case EIO_TRUNCATE: req->result = truncate (path , req->offs); break;
		2317	case EIO_OPEN: req->result = open (path , req->int1, (mode_t)req->int2); break;
		2318
		2319	case EIO_UNLINK: req->result = unlink (path ); break;
		2320	case EIO_RMDIR: req->result = rmdir (path ); break;
		2321	case EIO_MKDIR: req->result = mkdir (path , (mode_t)req->int2); break;
		2322	case EIO_RENAME: req->result = rename (path , req->ptr2); break;
		2323	case EIO_LINK: req->result = link (path , req->ptr2); break;
		2324	case EIO_SYMLINK: req->result = symlink (path , req->ptr2); break;
		2325	case EIO_MKNOD: req->result = mknod (path , (mode_t)req->int2, (dev_t)req->offs); break;
		2326	case EIO_READLINK: ALLOC (PATH_MAX);
		2327	req->result = readlink (path, req->ptr2, PATH_MAX); break;
		2328	case EIO_STATVFS: ALLOC (sizeof (EIO_STRUCT_STATVFS));
		2329	req->result = statvfs (path , (EIO_STRUCT_STATVFS *)req->ptr2); break;
		2330	#endif
		2331
		2332	case EIO_REALPATH: if (0 <= (req->result = eio__realpath (&self->tmpbuf, req->wd, req->ptr1)))
		2333	{
		2334	ALLOC (req->result);
		2335	memcpy (req->ptr2, self->tmpbuf.ptr, req->result);
		2336	}
		2337	break;
		2338
1662	case EIO_FSTAT: ALLOC (sizeof (EIO_STRUCT_STAT));	2339	case EIO_FSTAT: ALLOC (sizeof (EIO_STRUCT_STAT));
1663	req->result = fstat (req->int1, (EIO_STRUCT_STAT *)req->ptr2); break;	2340	req->result = fstat (req->int1, (EIO_STRUCT_STAT *)req->ptr2); break;
1664		2341
1665	case EIO_STATVFS: ALLOC (sizeof (EIO_STRUCT_STATVFS));
1666	req->result = statvfs (req->ptr1, (EIO_STRUCT_STATVFS *)req->ptr2); break;
1667	case EIO_FSTATVFS: ALLOC (sizeof (EIO_STRUCT_STATVFS));	2342	case EIO_FSTATVFS: ALLOC (sizeof (EIO_STRUCT_STATVFS));
1668	req->result = fstatvfs (req->int1, (EIO_STRUCT_STATVFS *)req->ptr2); break;	2343	req->result = fstatvfs (req->int1, (EIO_STRUCT_STATVFS *)req->ptr2); break;
1669		2344
1670	case EIO_CHOWN: req->result = chown (req->ptr1, req->int2, req->int3); break;
1671	case EIO_FCHOWN: req->result = fchown (req->int1, req->int2, req->int3); break;	2345	case EIO_FCHOWN: req->result = fchown (req->int1, req->int2, req->int3); break;
1672	case EIO_CHMOD: req->result = chmod (req->ptr1, (mode_t)req->int2); break;
1673	case EIO_FCHMOD: req->result = fchmod (req->int1, (mode_t)req->int2); break;	2346	case EIO_FCHMOD: req->result = fchmod (req->int1, (mode_t)req->int2); break;
1674	case EIO_TRUNCATE: req->result = truncate (req->ptr1, req->offs); break;
1675	case EIO_FTRUNCATE: req->result = ftruncate (req->int1, req->offs); break;	2347	case EIO_FTRUNCATE: req->result = ftruncate (req->int1, req->offs); break;
1676		2348
1677	case EIO_OPEN: req->result = open (req->ptr1, req->int1, (mode_t)req->int2); break;
1678	case EIO_CLOSE: req->result = close (req->int1); break;	2349	case EIO_CLOSE: req->result = close (req->int1); break;
1679	case EIO_DUP2: req->result = dup2 (req->int1, req->int2); break;	2350	case EIO_DUP2: req->result = dup2 (req->int1, req->int2); break;
1680	case EIO_UNLINK: req->result = unlink (req->ptr1); break;
1681	case EIO_RMDIR: req->result = rmdir (req->ptr1); break;
1682	case EIO_MKDIR: req->result = mkdir (req->ptr1, (mode_t)req->int2); break;
1683	case EIO_RENAME: req->result = rename (req->ptr1, req->ptr2); break;
1684	case EIO_LINK: req->result = link (req->ptr1, req->ptr2); break;
1685	case EIO_SYMLINK: req->result = symlink (req->ptr1, req->ptr2); break;
1686	case EIO_MKNOD: req->result = mknod (req->ptr1, (mode_t)req->int2, (dev_t)req->offs); break;
1687
1688	case EIO_READLINK: ALLOC (PATH_MAX);
1689	req->result = readlink (req->ptr1, req->ptr2, PATH_MAX); break;
1690
1691	case EIO_SYNC: req->result = 0; sync (); break;	2351	case EIO_SYNC: req->result = 0; sync (); break;
1692	case EIO_FSYNC: req->result = fsync (req->int1); break;	2352	case EIO_FSYNC: req->result = fsync (req->int1); break;
1693	case EIO_FDATASYNC: req->result = fdatasync (req->int1); break;	2353	case EIO_FDATASYNC: req->result = fdatasync (req->int1); break;
		2354	case EIO_SYNCFS: req->result = eio__syncfs (req->int1); break;
		2355	case EIO_SYNC_FILE_RANGE: req->result = eio__sync_file_range (req->int1, req->offs, req->size, req->int2); break;
1694	case EIO_MSYNC: req->result = eio__msync (req->ptr2, req->size, req->int1); break;	2356	case EIO_MSYNC: req->result = eio__msync (req->ptr2, req->size, req->int1); break;
1695	case EIO_MTOUCH: req->result = eio__mtouch (req->ptr2, req->size, req->int1); break;	2357	case EIO_MTOUCH: req->result = eio__mtouch (req); break;
1696	case EIO_MLOCK: req->result = eio__mlock (req->ptr2, req->size); break;	2358	case EIO_MLOCK: req->result = eio__mlock (req->ptr2, req->size); break;
1697	case EIO_MLOCKALL: req->result = eio__mlockall (req->int1); break;	2359	case EIO_MLOCKALL: req->result = eio__mlockall (req->int1); break;
1698	case EIO_SYNC_FILE_RANGE: req->result = eio__sync_file_range (req->int1, req->offs, req->size, req->int2); break;	2360	case EIO_FALLOCATE: req->result = eio__fallocate (req->int1, req->int2, req->offs, req->size); break;
1699		2361
1700	case EIO_READDIR: eio__scandir (req, self); break;	2362	case EIO_READDIR: eio__scandir (req); break;
1701		2363
1702	case EIO_BUSY:	2364	case EIO_BUSY:
1703	#ifdef _WIN32	2365	#ifdef _WIN32
1704	Sleep (req->nv1 * 1e3);	2366	Sleep (req->nv1 * 1e3);
1705	#else	2367	#else
…		…
1744	case EIO_NOP:	2406	case EIO_NOP:
1745	req->result = 0;	2407	req->result = 0;
1746	break;	2408	break;
1747		2409
1748	case EIO_CUSTOM:	2410	case EIO_CUSTOM:
1749	((void (*)(eio_req *))req->feed) (req);	2411	req->feed (req);
1750	break;	2412	break;
1751		2413
1752	default:	2414	default:
1753	errno = ENOSYS;	2415	errno = ENOSYS;
1754	req->result = -1;	2416	req->result = -1;
…		…
1783	eio_req *eio_msync (void *addr, size_t length, int flags, int pri, eio_cb cb, void *data)	2445	eio_req *eio_msync (void *addr, size_t length, int flags, int pri, eio_cb cb, void *data)
1784	{	2446	{
1785	REQ (EIO_MSYNC); req->ptr2 = addr; req->size = length; req->int1 = flags; SEND;	2447	REQ (EIO_MSYNC); req->ptr2 = addr; req->size = length; req->int1 = flags; SEND;
1786	}	2448	}
1787		2449
		2450	eio_req *eio_fdatasync (int fd, int pri, eio_cb cb, void *data)
		2451	{
		2452	REQ (EIO_FDATASYNC); req->int1 = fd; SEND;
		2453	}
		2454
		2455	eio_req *eio_syncfs (int fd, int pri, eio_cb cb, void *data)
		2456	{
		2457	REQ (EIO_SYNCFS); req->int1 = fd; SEND;
		2458	}
		2459
		2460	eio_req *eio_sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags, int pri, eio_cb cb, void *data)
		2461	{
		2462	REQ (EIO_SYNC_FILE_RANGE); req->int1 = fd; req->offs = offset; req->size = nbytes; req->int2 = flags; SEND;
		2463	}
		2464
1788	eio_req *eio_mtouch (void *addr, size_t length, int flags, int pri, eio_cb cb, void *data)	2465	eio_req *eio_mtouch (void *addr, size_t length, int flags, int pri, eio_cb cb, void *data)
1789	{	2466	{
1790	REQ (EIO_MTOUCH); req->ptr2 = addr; req->size = length; req->int1 = flags; SEND;	2467	REQ (EIO_MTOUCH); req->ptr2 = addr; req->size = length; req->int1 = flags; SEND;
1791	}	2468	}
1792		2469
…		…
1798	eio_req *eio_mlockall (int flags, int pri, eio_cb cb, void *data)	2475	eio_req *eio_mlockall (int flags, int pri, eio_cb cb, void *data)
1799	{	2476	{
1800	REQ (EIO_MLOCKALL); req->int1 = flags; SEND;	2477	REQ (EIO_MLOCKALL); req->int1 = flags; SEND;
1801	}	2478	}
1802		2479
1803	eio_req *eio_sync_file_range (int fd, off_t offset, size_t nbytes, unsigned int flags, int pri, eio_cb cb, void *data)	2480	eio_req *eio_fallocate (int fd, int mode, off_t offset, size_t len, int pri, eio_cb cb, void *data)
1804	{	2481	{
1805	REQ (EIO_SYNC_FILE_RANGE); req->int1 = fd; req->offs = offset; req->size = nbytes; req->int2 = flags; SEND;	2482	REQ (EIO_FALLOCATE); req->int1 = fd; req->int2 = mode; req->offs = offset; req->size = len; SEND;
1806	}
1807
1808	eio_req *eio_fdatasync (int fd, int pri, eio_cb cb, void *data)
1809	{
1810	REQ (EIO_FDATASYNC); req->int1 = fd; SEND;
1811	}	2483	}
1812		2484
1813	eio_req *eio_close (int fd, int pri, eio_cb cb, void *data)	2485	eio_req *eio_close (int fd, int pri, eio_cb cb, void *data)
1814	{	2486	{
1815	REQ (EIO_CLOSE); req->int1 = fd; SEND;	2487	REQ (EIO_CLOSE); req->int1 = fd; SEND;
…		…
1853	eio_req *eio_fchmod (int fd, mode_t mode, int pri, eio_cb cb, void *data)	2525	eio_req *eio_fchmod (int fd, mode_t mode, int pri, eio_cb cb, void *data)
1854	{	2526	{
1855	REQ (EIO_FCHMOD); req->int1 = fd; req->int2 = (long)mode; SEND;	2527	REQ (EIO_FCHMOD); req->int1 = fd; req->int2 = (long)mode; SEND;
1856	}	2528	}
1857		2529
1858	eio_req *eio_fchown (int fd, uid_t uid, gid_t gid, int pri, eio_cb cb, void *data)	2530	eio_req *eio_fchown (int fd, eio_uid_t uid, eio_gid_t gid, int pri, eio_cb cb, void *data)
1859	{	2531	{
1860	REQ (EIO_FCHOWN); req->int1 = fd; req->int2 = (long)uid; req->int3 = (long)gid; SEND;	2532	REQ (EIO_FCHOWN); req->int1 = fd; req->int2 = (long)uid; req->int3 = (long)gid; SEND;
1861	}	2533	}
1862		2534
1863	eio_req *eio_dup2 (int fd, int fd2, int pri, eio_cb cb, void *data)	2535	eio_req *eio_dup2 (int fd, int fd2, int pri, eio_cb cb, void *data)
…		…
1883	eio_req *eio_truncate (const char *path, off_t offset, int pri, eio_cb cb, void *data)	2555	eio_req *eio_truncate (const char *path, off_t offset, int pri, eio_cb cb, void *data)
1884	{	2556	{
1885	REQ (EIO_TRUNCATE); PATH; req->offs = offset; SEND;	2557	REQ (EIO_TRUNCATE); PATH; req->offs = offset; SEND;
1886	}	2558	}
1887		2559
1888	eio_req *eio_chown (const char *path, uid_t uid, gid_t gid, int pri, eio_cb cb, void *data)	2560	eio_req *eio_chown (const char *path, eio_uid_t uid, eio_gid_t gid, int pri, eio_cb cb, void *data)
1889	{	2561	{
1890	REQ (EIO_CHOWN); PATH; req->int2 = (long)uid; req->int3 = (long)gid; SEND;	2562	REQ (EIO_CHOWN); PATH; req->int2 = (long)uid; req->int3 = (long)gid; SEND;
1891	}	2563	}
1892		2564
1893	eio_req *eio_chmod (const char *path, mode_t mode, int pri, eio_cb cb, void *data)	2565	eio_req *eio_chmod (const char *path, mode_t mode, int pri, eio_cb cb, void *data)
…		…
1907	}	2579	}
1908		2580
1909	eio_req *eio_readlink (const char *path, int pri, eio_cb cb, void *data)	2581	eio_req *eio_readlink (const char *path, int pri, eio_cb cb, void *data)
1910	{	2582	{
1911	return eio__1path (EIO_READLINK, path, pri, cb, data);	2583	return eio__1path (EIO_READLINK, path, pri, cb, data);
		2584	}
		2585
		2586	eio_req *eio_realpath (const char *path, int pri, eio_cb cb, void *data)
		2587	{
		2588	return eio__1path (EIO_REALPATH, path, pri, cb, data);
1912	}	2589	}
1913		2590
1914	eio_req *eio_stat (const char *path, int pri, eio_cb cb, void *data)	2591	eio_req *eio_stat (const char *path, int pri, eio_cb cb, void *data)
1915	{	2592	{
1916	return eio__1path (EIO_STAT, path, pri, cb, data);	2593	return eio__1path (EIO_STAT, path, pri, cb, data);
…		…
1975	eio_req *eio_rename (const char *path, const char *new_path, int pri, eio_cb cb, void *data)	2652	eio_req *eio_rename (const char *path, const char *new_path, int pri, eio_cb cb, void *data)
1976	{	2653	{
1977	return eio__2path (EIO_RENAME, path, new_path, pri, cb, data);	2654	return eio__2path (EIO_RENAME, path, new_path, pri, cb, data);
1978	}	2655	}
1979		2656
1980	eio_req *eio_custom (eio_cb execute, int pri, eio_cb cb, void *data)	2657	eio_req *eio_custom (void (*execute)(eio_req *), int pri, eio_cb cb, void *data)
1981	{	2658	{
1982	REQ (EIO_CUSTOM); req->feed = (void (*)(eio_req *))execute; SEND;	2659	REQ (EIO_CUSTOM); req->feed = execute; SEND;
1983	}	2660	}
1984		2661
1985	#endif	2662	#endif
1986		2663
1987	eio_req *eio_grp (eio_cb cb, void *data)	2664	eio_req *eio_grp (eio_cb cb, void *data)
…		…
1996	#undef SEND	2673	#undef SEND
1997		2674
1998	/*****************************************************************************/	2675	/*****************************************************************************/
1999	/* grp functions */	2676	/* grp functions */
2000		2677
		2678	void
2001	void eio_grp_feed (eio_req *grp, void (*feed)(eio_req *req), int limit)	2679	eio_grp_feed (eio_req *grp, void (*feed)(eio_req *req), int limit)
2002	{	2680	{
2003	grp->int2 = limit;	2681	grp->int2 = limit;
2004	grp->feed = feed;	2682	grp->feed = feed;
2005		2683
2006	grp_try_feed (grp);	2684	grp_try_feed (grp);
2007	}	2685	}
2008		2686
		2687	void
2009	void eio_grp_limit (eio_req *grp, int limit)	2688	eio_grp_limit (eio_req *grp, int limit)
2010	{	2689	{
2011	grp->int2 = limit;	2690	grp->int2 = limit;
2012		2691
2013	grp_try_feed (grp);	2692	grp_try_feed (grp);
2014	}	2693	}
2015		2694
		2695	void
2016	void eio_grp_add (eio_req *grp, eio_req *req)	2696	eio_grp_add (eio_req *grp, eio_req *req)
2017	{	2697	{
2018	assert (("cannot add requests to IO::AIO::GRP after the group finished", grp->int1 != 2));	2698	assert (("cannot add requests to IO::AIO::GRP after the group finished", grp->int1 != 2));
2019		2699
2020	grp->flags |= EIO_FLAG_GROUPADD;	2700	grp->flags |= EIO_FLAG_GROUPADD;
2021		2701
…		…
2032	}	2712	}
2033		2713
2034	/*****************************************************************************/	2714	/*****************************************************************************/
2035	/* misc garbage */	2715	/* misc garbage */
2036		2716
		2717	eio_ssize_t
2037	ssize_t eio_sendfile_sync (int ofd, int ifd, off_t offset, size_t count)	2718	eio_sendfile_sync (int ofd, int ifd, off_t offset, size_t count)
2038	{	2719	{
2039	etp_worker wrk;
2040	ssize_t ret;
2041
2042	wrk.dbuf = 0;
2043
2044	ret = eio__sendfile (ofd, ifd, offset, count, &wrk);	2720	return eio__sendfile (ofd, ifd, offset, count);
2045
2046	if (wrk.dbuf)
2047	free (wrk.dbuf);
2048
2049	return ret;
2050	}	2721	}
2051		2722

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines