ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/IO-AIO/AIO.pm
Revision: 1.39
Committed: Sun Aug 28 11:05:50 2005 UTC (18 years, 8 months ago) by root
Branch: MAIN
Changes since 1.38: +2 -2 lines
Log Message:
*** empty log message ***

File Contents

# User Rev Content
1 root 1.1 =head1 NAME
2    
3     IO::AIO - Asynchronous Input/Output
4    
5     =head1 SYNOPSIS
6    
7     use IO::AIO;
8    
9 root 1.6 aio_open "/etc/passwd", O_RDONLY, 0, sub {
10     my ($fh) = @_;
11     ...
12     };
13    
14     aio_unlink "/tmp/file", sub { };
15    
16     aio_read $fh, 30000, 1024, $buffer, 0, sub {
17 root 1.8 $_[0] > 0 or die "read error: $!";
18 root 1.6 };
19    
20     # Event
21     Event->io (fd => IO::AIO::poll_fileno,
22 root 1.7 poll => 'r',
23 root 1.6 cb => \&IO::AIO::poll_cb);
24    
25     # Glib/Gtk2
26     add_watch Glib::IO IO::AIO::poll_fileno,
27 root 1.22 in => sub { IO::AIO::poll_cb; 1 };
28 root 1.6
29     # Tk
30     Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "",
31     readable => \&IO::AIO::poll_cb);
32    
33 root 1.11 # Danga::Socket
34     Danga::Socket->AddOtherFds (IO::AIO::poll_fileno =>
35     \&IO::AIO::poll_cb);
36    
37    
38 root 1.1 =head1 DESCRIPTION
39    
40     This module implements asynchronous I/O using whatever means your
41 root 1.2 operating system supports.
42 root 1.1
43 root 1.2 Currently, a number of threads are started that execute your read/writes
44     and signal their completion. You don't need thread support in your libc or
45     perl, and the threads created by this module will not be visible to the
46     pthreads library. In the future, this module might make use of the native
47     aio functions available on many operating systems. However, they are often
48     not well-supported (Linux doesn't allow them on normal files currently,
49     for example), and they would only support aio_read and aio_write, so the
50     remaining functionality would have to be implemented using threads anyway.
51 root 1.1
52     Although the module will work with in the presence of other threads, it is
53 root 1.22 currently not reentrant, so use appropriate locking yourself, always call
54     C<poll_cb> from within the same thread, or never call C<poll_cb> (or other
55     C<aio_> functions) recursively.
56 root 1.1
57     =cut
58    
59     package IO::AIO;
60    
61 root 1.23 no warnings;
62    
63 root 1.1 use base 'Exporter';
64    
65 root 1.2 use Fcntl ();
66    
67 root 1.1 BEGIN {
68 root 1.36 $VERSION = 1.6;
69 root 1.1
70 root 1.39 @EXPORT = qw(aio_sendfile aio_read aio_write aio_open aio_close aio_stat
71     aio_lstat aio_unlink aio_rmdir aio_readdir aio_symlink
72 root 1.38 aio_fsync aio_fdatasync aio_readahead);
73     @EXPORT_OK = qw(poll_fileno poll_cb min_parallel max_parallel
74     max_outstanding nreqs);
75 root 1.1
76     require XSLoader;
77     XSLoader::load IO::AIO, $VERSION;
78     }
79    
80 root 1.5 =head1 FUNCTIONS
81 root 1.1
82 root 1.5 =head2 AIO FUNCTIONS
83 root 1.1
84 root 1.5 All the C<aio_*> calls are more or less thin wrappers around the syscall
85     with the same name (sans C<aio_>). The arguments are similar or identical,
86 root 1.14 and they all accept an additional (and optional) C<$callback> argument
87     which must be a code reference. This code reference will get called with
88     the syscall return code (e.g. most syscalls return C<-1> on error, unlike
89     perl, which usually delivers "false") as it's sole argument when the given
90     syscall has been executed asynchronously.
91 root 1.1
92 root 1.23 All functions expecting a filehandle keep a copy of the filehandle
93     internally until the request has finished.
94 root 1.1
95 root 1.28 The pathnames you pass to these routines I<must> be absolute and
96     encoded in byte form. The reason for the former is that at the time the
97     request is being executed, the current working directory could have
98     changed. Alternatively, you can make sure that you never change the
99     current working directory.
100    
101     To encode pathnames to byte form, either make sure you either: a)
102     always pass in filenames you got from outside (command line, readdir
103     etc.), b) are ASCII or ISO 8859-1, c) use the Encode module and encode
104     your pathnames to the locale (or other) encoding in effect in the user
105     environment, d) use Glib::filename_from_unicode on unicode filenames or e)
106     use something else.
107 root 1.1
108 root 1.5 =over 4
109 root 1.1
110     =item aio_open $pathname, $flags, $mode, $callback
111    
112 root 1.2 Asynchronously open or create a file and call the callback with a newly
113     created filehandle for the file.
114 root 1.1
115     The pathname passed to C<aio_open> must be absolute. See API NOTES, above,
116     for an explanation.
117    
118 root 1.20 The C<$flags> argument is a bitmask. See the C<Fcntl> module for a
119     list. They are the same as used by C<sysopen>.
120    
121     Likewise, C<$mode> specifies the mode of the newly created file, if it
122     didn't exist and C<O_CREAT> has been given, just like perl's C<sysopen>,
123     except that it is mandatory (i.e. use C<0> if you don't create new files,
124     and C<0666> or C<0777> if you do).
125 root 1.1
126     Example:
127    
128     aio_open "/etc/passwd", O_RDONLY, 0, sub {
129 root 1.2 if ($_[0]) {
130     print "open successful, fh is $_[0]\n";
131 root 1.1 ...
132     } else {
133     die "open failed: $!\n";
134     }
135     };
136    
137     =item aio_close $fh, $callback
138    
139 root 1.2 Asynchronously close a file and call the callback with the result
140     code. I<WARNING:> although accepted, you should not pass in a perl
141 root 1.20 filehandle here, as perl will likely close the file descriptor another
142     time when the filehandle is destroyed. Normally, you can safely call perls
143     C<close> or just let filehandles go out of scope.
144    
145     This is supposed to be a bug in the API, so that might change. It's
146     therefore best to avoid this function.
147 root 1.1
148     =item aio_read $fh,$offset,$length, $data,$dataoffset,$callback
149    
150     =item aio_write $fh,$offset,$length, $data,$dataoffset,$callback
151    
152     Reads or writes C<length> bytes from the specified C<fh> and C<offset>
153     into the scalar given by C<data> and offset C<dataoffset> and calls the
154     callback without the actual number of bytes read (or -1 on error, just
155     like the syscall).
156    
157 root 1.31 The C<$data> scalar I<MUST NOT> be modified in any way while the request
158     is outstanding. Modifying it can result in segfaults or WW3 (if the
159     necessary/optional hardware is installed).
160    
161 root 1.17 Example: Read 15 bytes at offset 7 into scalar C<$buffer>, starting at
162 root 1.1 offset C<0> within the scalar:
163    
164     aio_read $fh, 7, 15, $buffer, 0, sub {
165 root 1.9 $_[0] > 0 or die "read error: $!";
166     print "read $_[0] bytes: <$buffer>\n";
167 root 1.1 };
168    
169 root 1.35 =item aio_sendfile $out_fh, $in_fh, $in_offset, $length, $callback
170    
171     Tries to copy C<$length> bytes from C<$in_fh> to C<$out_fh>. It starts
172     reading at byte offset C<$in_offset>, and starts writing at the current
173     file offset of C<$out_fh>. Because of that, it is not safe to issue more
174     than one C<aio_sendfile> per C<$out_fh>, as they will interfere with each
175     other.
176    
177     This call tries to make use of a native C<sendfile> syscall to provide
178     zero-copy operation. For this to work, C<$out_fh> should refer to a
179     socket, and C<$in_fh> should refer to mmap'able file.
180    
181     If the native sendfile call fails or is not implemented, it will be
182 root 1.36 emulated, so you can call C<aio_sendfile> on any type of filehandle
183     regardless of the limitations of the operating system.
184 root 1.35
185     Please note, however, that C<aio_sendfile> can read more bytes from
186     C<$in_fh> than are written, and there is no way to find out how many
187 root 1.36 bytes have been read from C<aio_sendfile> alone, as C<aio_sendfile> only
188     provides the number of bytes written to C<$out_fh>. Only if the result
189     value equals C<$length> one can assume that C<$length> bytes have been
190     read.
191 root 1.35
192 root 1.1 =item aio_readahead $fh,$offset,$length, $callback
193    
194 root 1.20 C<aio_readahead> populates the page cache with data from a file so that
195 root 1.1 subsequent reads from that file will not block on disk I/O. The C<$offset>
196     argument specifies the starting point from which data is to be read and
197     C<$length> specifies the number of bytes to be read. I/O is performed in
198     whole pages, so that offset is effectively rounded down to a page boundary
199     and bytes are read up to the next page boundary greater than or equal to
200 root 1.20 (off-set+length). C<aio_readahead> does not read beyond the end of the
201 root 1.1 file. The current file offset of the file is left unchanged.
202    
203 root 1.26 If that syscall doesn't exist (likely if your OS isn't Linux) it will be
204     emulated by simply reading the data, which would have a similar effect.
205    
206 root 1.1 =item aio_stat $fh_or_path, $callback
207    
208     =item aio_lstat $fh, $callback
209    
210     Works like perl's C<stat> or C<lstat> in void context. The callback will
211     be called after the stat and the results will be available using C<stat _>
212     or C<-s _> etc...
213    
214     The pathname passed to C<aio_stat> must be absolute. See API NOTES, above,
215     for an explanation.
216    
217     Currently, the stats are always 64-bit-stats, i.e. instead of returning an
218     error when stat'ing a large file, the results will be silently truncated
219     unless perl itself is compiled with large file support.
220    
221     Example: Print the length of F</etc/passwd>:
222    
223     aio_stat "/etc/passwd", sub {
224     $_[0] and die "stat failed: $!";
225     print "size is ", -s _, "\n";
226     };
227    
228     =item aio_unlink $pathname, $callback
229    
230     Asynchronously unlink (delete) a file and call the callback with the
231     result code.
232    
233 root 1.27 =item aio_rmdir $pathname, $callback
234    
235     Asynchronously rmdir (delete) a directory and call the callback with the
236     result code.
237    
238 root 1.37 =item aio_readdir $pathname $callback
239    
240     Unlike the POSIX call of the same name, C<aio_readdir> reads an entire
241     directory (i.e. opendir + readdir + closedir). The entries will not be
242     sorted, and will B<NOT> include the C<.> and C<..> entries.
243    
244     The callback a single argument which is either C<undef> or an array-ref
245     with the filenames.
246    
247 root 1.1 =item aio_fsync $fh, $callback
248    
249     Asynchronously call fsync on the given filehandle and call the callback
250     with the fsync result code.
251    
252     =item aio_fdatasync $fh, $callback
253    
254     Asynchronously call fdatasync on the given filehandle and call the
255 root 1.26 callback with the fdatasync result code.
256    
257     If this call isn't available because your OS lacks it or it couldn't be
258     detected, it will be emulated by calling C<fsync> instead.
259 root 1.1
260 root 1.5 =back
261    
262     =head2 SUPPORT FUNCTIONS
263    
264     =over 4
265    
266     =item $fileno = IO::AIO::poll_fileno
267    
268 root 1.20 Return the I<request result pipe file descriptor>. This filehandle must be
269     polled for reading by some mechanism outside this module (e.g. Event or
270     select, see below or the SYNOPSIS). If the pipe becomes readable you have
271     to call C<poll_cb> to check the results.
272 root 1.5
273     See C<poll_cb> for an example.
274    
275     =item IO::AIO::poll_cb
276    
277     Process all outstanding events on the result pipe. You have to call this
278     regularly. Returns the number of events processed. Returns immediately
279     when no events are outstanding.
280    
281 root 1.20 Example: Install an Event watcher that automatically calls
282     IO::AIO::poll_cb with high priority:
283 root 1.5
284     Event->io (fd => IO::AIO::poll_fileno,
285     poll => 'r', async => 1,
286     cb => \&IO::AIO::poll_cb);
287    
288     =item IO::AIO::poll_wait
289    
290     Wait till the result filehandle becomes ready for reading (simply does a
291 root 1.20 C<select> on the filehandle. This is useful if you want to synchronously wait
292 root 1.5 for some requests to finish).
293    
294     See C<nreqs> for an example.
295    
296     =item IO::AIO::nreqs
297    
298 root 1.20 Returns the number of requests currently outstanding (i.e. for which their
299     callback has not been invoked yet).
300 root 1.5
301     Example: wait till there are no outstanding requests anymore:
302    
303     IO::AIO::poll_wait, IO::AIO::poll_cb
304     while IO::AIO::nreqs;
305    
306 root 1.12 =item IO::AIO::flush
307    
308     Wait till all outstanding AIO requests have been handled.
309    
310 root 1.13 Strictly equivalent to:
311    
312     IO::AIO::poll_wait, IO::AIO::poll_cb
313     while IO::AIO::nreqs;
314    
315     =item IO::AIO::poll
316    
317     Waits until some requests have been handled.
318    
319     Strictly equivalent to:
320    
321     IO::AIO::poll_wait, IO::AIO::poll_cb
322     if IO::AIO::nreqs;
323    
324 root 1.5 =item IO::AIO::min_parallel $nthreads
325    
326 root 1.34 Set the minimum number of AIO threads to C<$nthreads>. The current default
327     is C<4>, which means four asynchronous operations can be done at one time
328 root 1.5 (the number of outstanding operations, however, is unlimited).
329    
330 root 1.34 IO::AIO starts threads only on demand, when an AIO request is queued and
331     no free thread exists.
332    
333 root 1.5 It is recommended to keep the number of threads low, as some Linux
334     kernel versions will scale negatively with the number of threads (higher
335     parallelity => MUCH higher latency). With current Linux 2.6 versions, 4-32
336     threads should be fine.
337    
338 root 1.34 Under most circumstances you don't need to call this function, as the
339     module selects a default that is suitable for low to moderate load.
340 root 1.5
341     =item IO::AIO::max_parallel $nthreads
342    
343 root 1.34 Sets the maximum number of AIO threads to C<$nthreads>. If more than the
344     specified number of threads are currently running, this function kills
345     them. This function blocks until the limit is reached.
346    
347     While C<$nthreads> are zero, aio requests get queued but not executed
348     until the number of threads has been increased again.
349 root 1.5
350     This module automatically runs C<max_parallel 0> at program end, to ensure
351     that all threads are killed and that there are no outstanding requests.
352    
353     Under normal circumstances you don't need to call this function.
354    
355     =item $oldnreqs = IO::AIO::max_outstanding $nreqs
356    
357     Sets the maximum number of outstanding requests to C<$nreqs>. If you
358     try to queue up more than this number of requests, the caller will block until
359     some requests have been handled.
360    
361     The default is very large, so normally there is no practical limit. If you
362 root 1.34 queue up many requests in a loop it often improves speed if you set
363 root 1.5 this to a relatively low number, such as C<100>.
364    
365     Under normal circumstances you don't need to call this function.
366    
367     =back
368    
369 root 1.1 =cut
370    
371 root 1.2 # support function to convert a fd into a perl filehandle
372     sub _fd2fh {
373     return undef if $_[0] < 0;
374    
375 root 1.23 # try to generate nice filehandles
376     my $sym = "IO::AIO::fd#$_[0]";
377     local *$sym;
378 root 1.25
379 root 1.27 open *$sym, "+<&=$_[0]" # usually works under any unix
380     or open *$sym, "<&=$_[0]" # cygwin needs this
381     or open *$sym, ">&=$_[0]" # or this
382 root 1.2 or return undef;
383    
384 root 1.23 *$sym
385 root 1.2 }
386    
387 root 1.1 min_parallel 4;
388    
389     END {
390     max_parallel 0;
391     }
392    
393     1;
394    
395 root 1.27 =head2 FORK BEHAVIOUR
396    
397 root 1.34 Before the fork, IO::AIO enters a quiescent state where no requests
398     can be added in other threads and no results will be processed. After
399     the fork the parent simply leaves the quiescent state and continues
400     request/result processing, while the child clears the request/result
401     queue (so the requests started before the fork will only be handled in
402     the parent). Threats will be started on demand until the limit ste in the
403     parent process has been reached again.
404 root 1.27
405 root 1.1 =head1 SEE ALSO
406    
407     L<Coro>, L<Linux::AIO>.
408    
409     =head1 AUTHOR
410    
411     Marc Lehmann <schmorp@schmorp.de>
412     http://home.schmorp.de/
413    
414     =cut
415