ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/IO-AIO/AIO.pm
(Generate patch)

Comparing IO-AIO/AIO.pm (file contents):
Revision 1.42 by root, Mon Dec 26 18:15:23 2005 UTC vs.
Revision 1.58 by root, Sun Oct 22 10:33:19 2006 UTC

15 15
16 aio_read $fh, 30000, 1024, $buffer, 0, sub { 16 aio_read $fh, 30000, 1024, $buffer, 0, sub {
17 $_[0] > 0 or die "read error: $!"; 17 $_[0] > 0 or die "read error: $!";
18 }; 18 };
19 19
20 # AnyEvent 20 # version 2+ has request and group objects
21 use IO::AIO 2;
22
23 my $req = aio_unlink "/tmp/file", sub { };
24 $req->cancel; # cancel request if still in queue
25
26 my $grp = aio_group sub { print "all stats done\n" };
27 add $grp aio_stat "..." for ...;
28
29 # AnyEvent integration
21 open my $fh, "<&=" . IO::AIO::poll_fileno or die "$!"; 30 open my $fh, "<&=" . IO::AIO::poll_fileno or die "$!";
22 my $w = AnyEvent->io (fh => $fh, poll => 'r', cb => sub { IO::AIO::poll_cb }); 31 my $w = AnyEvent->io (fh => $fh, poll => 'r', cb => sub { IO::AIO::poll_cb });
23 32
24 # Event 33 # Event integration
25 Event->io (fd => IO::AIO::poll_fileno, 34 Event->io (fd => IO::AIO::poll_fileno,
26 poll => 'r', 35 poll => 'r',
27 cb => \&IO::AIO::poll_cb); 36 cb => \&IO::AIO::poll_cb);
28 37
29 # Glib/Gtk2 38 # Glib/Gtk2 integration
30 add_watch Glib::IO IO::AIO::poll_fileno, 39 add_watch Glib::IO IO::AIO::poll_fileno,
31 in => sub { IO::AIO::poll_cb; 1 }; 40 in => sub { IO::AIO::poll_cb; 1 };
32 41
33 # Tk 42 # Tk integration
34 Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "", 43 Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "",
35 readable => \&IO::AIO::poll_cb); 44 readable => \&IO::AIO::poll_cb);
36 45
37 # Danga::Socket 46 # Danga::Socket integration
38 Danga::Socket->AddOtherFds (IO::AIO::poll_fileno => 47 Danga::Socket->AddOtherFds (IO::AIO::poll_fileno =>
39 \&IO::AIO::poll_cb); 48 \&IO::AIO::poll_cb);
40
41 49
42=head1 DESCRIPTION 50=head1 DESCRIPTION
43 51
44This module implements asynchronous I/O using whatever means your 52This module implements asynchronous I/O using whatever means your
45operating system supports. 53operating system supports.
61=cut 69=cut
62 70
63package IO::AIO; 71package IO::AIO;
64 72
65no warnings; 73no warnings;
74use strict 'vars';
66 75
67use base 'Exporter'; 76use base 'Exporter';
68 77
69use Fcntl ();
70
71BEGIN { 78BEGIN {
72 $VERSION = '1.61'; 79 our $VERSION = '2.0';
73 80
74 @EXPORT = qw(aio_sendfile aio_read aio_write aio_open aio_close aio_stat 81 our @EXPORT = qw(aio_sendfile aio_read aio_write aio_open aio_close aio_stat
75 aio_lstat aio_unlink aio_rmdir aio_readdir aio_scandir aio_symlink 82 aio_lstat aio_unlink aio_rmdir aio_readdir aio_scandir aio_symlink
76 aio_fsync aio_fdatasync aio_readahead); 83 aio_fsync aio_fdatasync aio_readahead aio_rename aio_link aio_move
84 aio_group);
77 @EXPORT_OK = qw(poll_fileno poll_cb min_parallel max_parallel 85 our @EXPORT_OK = qw(poll_fileno poll_cb min_parallel max_parallel max_outstanding nreqs);
78 max_outstanding nreqs); 86
87 @IO::AIO::GRP::ISA = 'IO::AIO::REQ';
79 88
80 require XSLoader; 89 require XSLoader;
81 XSLoader::load IO::AIO, $VERSION; 90 XSLoader::load ("IO::AIO", $VERSION);
82} 91}
83 92
84=head1 FUNCTIONS 93=head1 FUNCTIONS
85 94
86=head2 AIO FUNCTIONS 95=head2 AIO FUNCTIONS
93perl, which usually delivers "false") as it's sole argument when the given 102perl, which usually delivers "false") as it's sole argument when the given
94syscall has been executed asynchronously. 103syscall has been executed asynchronously.
95 104
96All functions expecting a filehandle keep a copy of the filehandle 105All functions expecting a filehandle keep a copy of the filehandle
97internally until the request has finished. 106internally until the request has finished.
107
108All requests return objects of type L<IO::AIO::REQ> that allow further
109manipulation of those requests while they are in-flight.
98 110
99The pathnames you pass to these routines I<must> be absolute and 111The pathnames you pass to these routines I<must> be absolute and
100encoded in byte form. The reason for the former is that at the time the 112encoded in byte form. The reason for the former is that at the time the
101request is being executed, the current working directory could have 113request is being executed, the current working directory could have
102changed. Alternatively, you can make sure that you never change the 114changed. Alternatively, you can make sure that you never change the
168 aio_read $fh, 7, 15, $buffer, 0, sub { 180 aio_read $fh, 7, 15, $buffer, 0, sub {
169 $_[0] > 0 or die "read error: $!"; 181 $_[0] > 0 or die "read error: $!";
170 print "read $_[0] bytes: <$buffer>\n"; 182 print "read $_[0] bytes: <$buffer>\n";
171 }; 183 };
172 184
185=item aio_move $srcpath, $dstpath, $callback->($status)
186
187[EXPERIMENTAL due to internal aio_group use]
188
189Try to move the I<file> (directories not supported as either source or
190destination) from C<$srcpath> to C<$dstpath> and call the callback with
191the C<0> (error) or C<-1> ok.
192
193This is a composite request that tries to rename(2) the file first. If
194rename files with C<EXDEV>, it creates the destination file with mode 0200
195and copies the contents of the source file into it using C<aio_sendfile>,
196followed by restoring atime, mtime, access mode and uid/gid, in that
197order, and unlinking the C<$srcpath>.
198
199If an error occurs, the partial destination file will be unlinked, if
200possible, except when setting atime, mtime, access mode and uid/gid, where
201errors are being ignored.
202
203=cut
204
205sub aio_move($$$) {
206 my ($src, $dst, $cb) = @_;
207
208 my $grp = aio_group $cb;
209
210 add $grp aio_rename $src, $dst, sub {
211 if ($_[0] && $! == EXDEV) {
212 add $grp aio_open $src, O_RDONLY, 0, sub {
213 if (my $src_fh = $_[0]) {
214 my @stat = stat $src_fh;
215
216 add $grp aio_open $dst, O_WRONLY, 0200, sub {
217 if (my $dst_fh = $_[0]) {
218 add $grp aio_sendfile $dst_fh, $src_fh, 0, $stat[7], sub {
219 close $src_fh;
220
221 if ($_[0] == $stat[7]) {
222 utime $stat[8], $stat[9], $dst;
223 chmod $stat[2] & 07777, $dst_fh;
224 chown $stat[4], $stat[5], $dst_fh;
225 close $dst_fh;
226
227 add $grp aio_unlink $src, sub {
228 $grp->result ($_[0]);
229 };
230 } else {
231 my $errno = $!;
232 add $grp aio_unlink $dst, sub {
233 $! = $errno;
234 $grp->result (-1);
235 };
236 }
237 };
238 } else {
239 $grp->result (-1);
240 }
241 },
242
243 } else {
244 $grp->result (-1);
245 }
246 };
247 } else {
248 $grp->result ($_[0]);
249 }
250 };
251
252 $grp
253}
254
173=item aio_sendfile $out_fh, $in_fh, $in_offset, $length, $callback->($retval) 255=item aio_sendfile $out_fh, $in_fh, $in_offset, $length, $callback->($retval)
174 256
175Tries to copy C<$length> bytes from C<$in_fh> to C<$out_fh>. It starts 257Tries to copy C<$length> bytes from C<$in_fh> to C<$out_fh>. It starts
176reading at byte offset C<$in_offset>, and starts writing at the current 258reading at byte offset C<$in_offset>, and starts writing at the current
177file offset of C<$out_fh>. Because of that, it is not safe to issue more 259file offset of C<$out_fh>. Because of that, it is not safe to issue more
232=item aio_unlink $pathname, $callback->($status) 314=item aio_unlink $pathname, $callback->($status)
233 315
234Asynchronously unlink (delete) a file and call the callback with the 316Asynchronously unlink (delete) a file and call the callback with the
235result code. 317result code.
236 318
319=item aio_link $srcpath, $dstpath, $callback->($status)
320
321Asynchronously create a new link to the existing object at C<$srcpath> at
322the path C<$dstpath> and call the callback with the result code.
323
324=item aio_symlink $srcpath, $dstpath, $callback->($status)
325
326Asynchronously create a new symbolic link to the existing object at C<$srcpath> at
327the path C<$dstpath> and call the callback with the result code.
328
329=item aio_rename $srcpath, $dstpath, $callback->($status)
330
331Asynchronously rename the object at C<$srcpath> to C<$dstpath>, just as
332rename(2) and call the callback with the result code.
333
237=item aio_rmdir $pathname, $callback->($status) 334=item aio_rmdir $pathname, $callback->($status)
238 335
239Asynchronously rmdir (delete) a directory and call the callback with the 336Asynchronously rmdir (delete) a directory and call the callback with the
240result code. 337result code.
241 338
242=item aio_readdir $pathname $callback->($entries) 339=item aio_readdir $pathname, $callback->($entries)
243 340
244Unlike the POSIX call of the same name, C<aio_readdir> reads an entire 341Unlike the POSIX call of the same name, C<aio_readdir> reads an entire
245directory (i.e. opendir + readdir + closedir). The entries will not be 342directory (i.e. opendir + readdir + closedir). The entries will not be
246sorted, and will B<NOT> include the C<.> and C<..> entries. 343sorted, and will B<NOT> include the C<.> and C<..> entries.
247 344
248The callback a single argument which is either C<undef> or an array-ref 345The callback a single argument which is either C<undef> or an array-ref
249with the filenames. 346with the filenames.
250 347
251=item aio_scandir $path, $maxreq, $callback->($dirs, $nondirs) 348=item aio_scandir $path, $maxreq, $callback->($dirs, $nondirs)
252 349
350[EXPERIMENTAL due to internal aio_group use]
351
253Scans a directory (similar to C<aio_readdir>) and tries to separate the 352Scans a directory (similar to C<aio_readdir>) but additionally tries to
254entries of directory C<$path> into two sets of names, ones you can recurse 353separate the entries of directory C<$path> into two sets of names, ones
255into (directories), and ones you cannot recurse into (everything else). 354you can recurse into (directories or links to them), and ones you cannot
355recurse into (everything else).
256 356
257C<aio_scandir> is a composite request that consists of many 357C<aio_scandir> is a composite request that consists of many sub
258aio-primitives. C<$maxreq> specifies the maximum number of outstanding 358requests. C<$maxreq> specifies the maximum number of outstanding aio
259aio requests that this function generates. If it is C<< <= 0 >>, then a 359requests that this function generates. If it is C<< <= 0 >>, then a
260suitable default will be chosen (currently 8). 360suitable default will be chosen (currently 8).
261 361
262On error, the callback is called without arguments, otherwise it receives 362On error, the callback is called without arguments, otherwise it receives
263two array-refs with path-relative entry names. 363two array-refs with path-relative entry names.
264 364
273Implementation notes. 373Implementation notes.
274 374
275The C<aio_readdir> cannot be avoided, but C<stat()>'ing every entry can. 375The C<aio_readdir> cannot be avoided, but C<stat()>'ing every entry can.
276 376
277After reading the directory, the modification time, size etc. of the 377After reading the directory, the modification time, size etc. of the
278directory before and after the readdir is checked, and if they match, the 378directory before and after the readdir is checked, and if they match (and
279link count will be used to decide how many entries are directories (if 379isn't the current time), the link count will be used to decide how many
280>= 2). Otherwise, no knowledge of the number of subdirectories will be 380entries are directories (if >= 2). Otherwise, no knowledge of the number
281assumed. 381of subdirectories will be assumed.
282 382
283Then entires will be sorted into likely directories (everything without a 383Then entries will be sorted into likely directories (everything without
284non-initial dot) and likely non-directories (everything else). Then every 384a non-initial dot currently) and likely non-directories (everything
285entry + C</.> will be C<stat>'ed, likely directories first. This is often 385else). Then every entry plus an appended C</.> will be C<stat>'ed,
386likely directories first. If that succeeds, it assumes that the entry
387is a directory or a symlink to directory (which will be checked
388seperately). This is often faster than stat'ing the entry itself because
286faster because filesystems might detect the type of the entry without 389filesystems might detect the type of the entry without reading the inode
287reading the inode data (e.g. ext2s filetype feature). If that succeeds, 390data (e.g. ext2fs filetype feature).
288it assumes that the entry is a directory or a symlink to directory (which
289will be checked seperately).
290 391
291If the known number of directories has been reached, the rest of the 392If the known number of directories (link count - 2) has been reached, the
292entries is assumed to be non-directories. 393rest of the entries is assumed to be non-directories.
394
395This only works with certainty on POSIX (= UNIX) filesystems, which
396fortunately are the vast majority of filesystems around.
397
398It will also likely work on non-POSIX filesystems with reduced efficiency
399as those tend to return 0 or 1 as link counts, which disables the
400directory counting heuristic.
293 401
294=cut 402=cut
295 403
296sub aio_scandir($$$) { 404sub aio_scandir($$$) {
297 my ($path, $maxreq, $cb) = @_; 405 my ($path, $maxreq, $cb) = @_;
298 406
407 my $grp = aio_group $cb;
408
299 $maxreq = 8 if $maxreq <= 0; 409 $maxreq = 8 if $maxreq <= 0;
300 410
301 # stat once 411 # stat once
302 aio_stat $path, sub { 412 add $grp aio_stat $path, sub {
303 $cb->() if $_[0]; 413 return $grp->result () if $_[0];
414 my $now = time;
304 my $hash1 = join ":", (stat _)[0,1,3,7,9]; 415 my $hash1 = join ":", (stat _)[0,1,3,7,9];
305 416
306 # read the directory entries 417 # read the directory entries
307 aio_readdir $path, sub { 418 add $grp aio_readdir $path, sub {
308 my $entries = shift 419 my $entries = shift
309 or return $cb->(); 420 or return $grp->result ();
310 421
311 # stat the dir another time 422 # stat the dir another time
312 aio_stat $path, sub { 423 add $grp aio_stat $path, sub {
313 my $hash2 = join ":", (stat _)[0,1,3,7,9]; 424 my $hash2 = join ":", (stat _)[0,1,3,7,9];
314 425
315 my $ndirs; 426 my $ndirs;
316 427
317 # take the slow route if anything looks fishy 428 # take the slow route if anything looks fishy
318 if ($hash1 ne $hash2) { 429 if ($hash1 ne $hash2 or (stat _)[9] == $now) {
319 $ndirs = -1; 430 $ndirs = -1;
320 } else { 431 } else {
321 # if nlink == 2, we are finished 432 # if nlink == 2, we are finished
322 # on non-posix-fs's, we rely on nlink < 2 433 # on non-posix-fs's, we rely on nlink < 2
323 $ndirs = (stat _)[3] - 2 434 $ndirs = (stat _)[3] - 2
324 or $cb->([], $entries); 435 or return $grp->result ([], $entries);
325 } 436 }
326 437
327 # sort into likely dirs and likely nondirs 438 # sort into likely dirs and likely nondirs
328 # dirs == files without ".", short entries first 439 # dirs == files without ".", short entries first
329 $entries = [map $_->[0], 440 $entries = [map $_->[0],
339 $schedcb = sub { 450 $schedcb = sub {
340 if (@$entries) { 451 if (@$entries) {
341 if ($nreq < $maxreq) { 452 if ($nreq < $maxreq) {
342 my $ent = pop @$entries; 453 my $ent = pop @$entries;
343 $nreq++; 454 $nreq++;
344 aio_stat "$path/$ent/.", sub { $statcb->($_[0], $ent) }; 455 add $grp aio_stat "$path/$ent/.", sub { $statcb->($_[0], $ent) };
345 } 456 }
346 } elsif (!$nreq) { 457 } elsif (!$nreq) {
347 # finished 458 # finished
348 undef $statcb; 459 undef $statcb;
349 undef $schedcb; 460 undef $schedcb;
350 $cb->(\@dirs, \@nondirs); 461 $grp->result (\@dirs, \@nondirs) if $cb;
351 undef $cb; 462 undef $cb;
352 } 463 }
353 }; 464 };
354 $statcb = sub { 465 $statcb = sub {
355 my ($status, $entry) = @_; 466 my ($status, $entry) = @_;
358 $nreq--; 469 $nreq--;
359 push @nondirs, $entry; 470 push @nondirs, $entry;
360 &$schedcb; 471 &$schedcb;
361 } else { 472 } else {
362 # need to check for real directory 473 # need to check for real directory
363 aio_lstat "$path/$entry", sub { 474 add $grp aio_lstat "$path/$entry", sub {
364 $nreq--; 475 $nreq--;
365 476
366 if (-d _) { 477 if (-d _) {
367 push @dirs, $entry; 478 push @dirs, $entry;
368 479
381 492
382 &$schedcb while @$entries && $nreq < $maxreq; 493 &$schedcb while @$entries && $nreq < $maxreq;
383 }; 494 };
384 }; 495 };
385 }; 496 };
497
498 $grp
386} 499}
387 500
388=item aio_fsync $fh, $callback->($status) 501=item aio_fsync $fh, $callback->($status)
389 502
390Asynchronously call fsync on the given filehandle and call the callback 503Asynchronously call fsync on the given filehandle and call the callback
395Asynchronously call fdatasync on the given filehandle and call the 508Asynchronously call fdatasync on the given filehandle and call the
396callback with the fdatasync result code. 509callback with the fdatasync result code.
397 510
398If this call isn't available because your OS lacks it or it couldn't be 511If this call isn't available because your OS lacks it or it couldn't be
399detected, it will be emulated by calling C<fsync> instead. 512detected, it will be emulated by calling C<fsync> instead.
513
514=item aio_group $callback->(...)
515
516[EXPERIMENTAL]
517
518This is a very special aio request: Instead of doing something, it is a
519container for other aio requests, which is useful if you want to bundle
520many requests into a single, composite, request.
521
522Returns an object of class L<IO::AIO::GRP>. See its documentation below
523for more info.
524
525Example:
526
527 my $grp = aio_group sub {
528 print "all stats done\n";
529 };
530
531 add $grp
532 (aio_stat ...),
533 (aio_stat ...),
534 ...;
535
536=item IO::AIO::aio_sleep $fractional_seconds, $callback->() *NOT EXPORTED*
537
538Mainly used for debugging and benchmarking, this aio request puts one of
539the request workers to sleep for the given time.
540
541While it is theoretically handy to have simple I/O scheduling requests
542like sleep and file handle readable/writable, the overhead this creates
543is immense, so do not use this function except to put your application
544under artificial I/O pressure.
545
546=back
547
548=head2 IO::AIO::REQ CLASS
549
550All non-aggregate C<aio_*> functions return an object of this class when
551called in non-void context.
552
553A request always moves through the following five states in its lifetime,
554in order: B<ready> (request has been created, but has not been executed
555yet), B<execute> (request is currently being executed), B<pending>
556(request has been executed but callback has not been called yet),
557B<result> (results are being processed synchronously, includes calling the
558callback) and B<done> (request has reached the end of its lifetime and
559holds no resources anymore).
560
561=over 4
562
563=item $req->cancel
564
565Cancels the request, if possible. Has the effect of skipping execution
566when entering the B<execute> state and skipping calling the callback when
567entering the the B<result> state, but will leave the request otherwise
568untouched. That means that requests that currently execute will not be
569stopped and resources held by the request will not be freed prematurely.
570
571=back
572
573=head2 IO::AIO::GRP CLASS
574
575This class is a subclass of L<IO::AIO::REQ>, so all its methods apply to
576objects of this class, too.
577
578A IO::AIO::GRP object is a special request that can contain multiple other
579aio requests.
580
581You create one by calling the C<aio_group> constructing function with a
582callback that will be called when all contained requests have entered the
583C<done> state:
584
585 my $grp = aio_group sub {
586 print "all requests are done\n";
587 };
588
589You add requests by calling the C<add> method with one or more
590C<IO::AIO::REQ> objects:
591
592 $grp->add (aio_unlink "...");
593
594 add $grp aio_stat "...", sub {
595 $_[0] or return $grp->result ("error");
596
597 # add another request dynamically, if first succeeded
598 add $grp aio_open "...", sub {
599 $grp->result ("ok");
600 };
601 };
602
603This makes it very easy to create composite requests (see the source of
604C<aio_move> for an application) that work and feel like simple requests.
605
606The IO::AIO::GRP objects will be cleaned up during calls to
607C<IO::AIO::poll_cb>, just like any other request.
608
609They can be canceled like any other request. Canceling will cancel not
610just the request itself, but also all requests it contains.
611
612They can also can also be added to other IO::AIO::GRP objects.
613
614Their lifetime, simplified, looks like this: when they are empty, they
615will finish very quickly. If they contain only requests that are in the
616C<done> state, they will also finish. Otherwise they will continue to
617exist.
618
619That means after creating a group you have some time to add requests. And
620in the callbacks of those requests, you can add further requests to the
621group. And only when all those requests have finished will the the group
622itself finish.
623
624=over 4
625
626=item $grp->add (...)
627
628=item add $grp ...
629
630Add one or more requests to the group. Any type of L<IO::AIO::REQ> can
631be added, including other groups, as long as you do not create circular
632dependencies.
633
634Returns all its arguments.
635
636=item $grp->result (...)
637
638Set the result value(s) that will be passed to the group callback when all
639subrequests have finished. By default, no argument will be passed.
400 640
401=back 641=back
402 642
403=head2 SUPPORT FUNCTIONS 643=head2 SUPPORT FUNCTIONS
404 644
532} 772}
533 773
5341; 7741;
535 775
536=head2 FORK BEHAVIOUR 776=head2 FORK BEHAVIOUR
777
778This module should do "the right thing" when the process using it forks:
537 779
538Before the fork, IO::AIO enters a quiescent state where no requests 780Before the fork, IO::AIO enters a quiescent state where no requests
539can be added in other threads and no results will be processed. After 781can be added in other threads and no results will be processed. After
540the fork the parent simply leaves the quiescent state and continues 782the fork the parent simply leaves the quiescent state and continues
541request/result processing, while the child clears the request/result 783request/result processing, while the child clears the request/result
542queue (so the requests started before the fork will only be handled in 784queue (so the requests started before the fork will only be handled in
543the parent). Threats will be started on demand until the limit ste in the 785the parent). Threads will be started on demand until the limit ste in the
544parent process has been reached again. 786parent process has been reached again.
545 787
788In short: the parent will, after a short pause, continue as if fork had
789not been called, while the child will act as if IO::AIO has not been used
790yet.
791
546=head1 SEE ALSO 792=head1 SEE ALSO
547 793
548L<Coro>, L<Linux::AIO>. 794L<Coro>, L<Linux::AIO> (obsolete).
549 795
550=head1 AUTHOR 796=head1 AUTHOR
551 797
552 Marc Lehmann <schmorp@schmorp.de> 798 Marc Lehmann <schmorp@schmorp.de>
553 http://home.schmorp.de/ 799 http://home.schmorp.de/

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines