ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/IO-AIO/AIO.pm
(Generate patch)

Comparing IO-AIO/AIO.pm (file contents):
Revision 1.50 by root, Sat Jun 24 16:27:02 2006 UTC vs.
Revision 1.59 by root, Sun Oct 22 10:33:26 2006 UTC

15 15
16 aio_read $fh, 30000, 1024, $buffer, 0, sub { 16 aio_read $fh, 30000, 1024, $buffer, 0, sub {
17 $_[0] > 0 or die "read error: $!"; 17 $_[0] > 0 or die "read error: $!";
18 }; 18 };
19 19
20 # AnyEvent 20 # version 2+ has request and group objects
21 use IO::AIO 2;
22
23 my $req = aio_unlink "/tmp/file", sub { };
24 $req->cancel; # cancel request if still in queue
25
26 my $grp = aio_group sub { print "all stats done\n" };
27 add $grp aio_stat "..." for ...;
28
29 # AnyEvent integration
21 open my $fh, "<&=" . IO::AIO::poll_fileno or die "$!"; 30 open my $fh, "<&=" . IO::AIO::poll_fileno or die "$!";
22 my $w = AnyEvent->io (fh => $fh, poll => 'r', cb => sub { IO::AIO::poll_cb }); 31 my $w = AnyEvent->io (fh => $fh, poll => 'r', cb => sub { IO::AIO::poll_cb });
23 32
24 # Event 33 # Event integration
25 Event->io (fd => IO::AIO::poll_fileno, 34 Event->io (fd => IO::AIO::poll_fileno,
26 poll => 'r', 35 poll => 'r',
27 cb => \&IO::AIO::poll_cb); 36 cb => \&IO::AIO::poll_cb);
28 37
29 # Glib/Gtk2 38 # Glib/Gtk2 integration
30 add_watch Glib::IO IO::AIO::poll_fileno, 39 add_watch Glib::IO IO::AIO::poll_fileno,
31 in => sub { IO::AIO::poll_cb; 1 }; 40 in => sub { IO::AIO::poll_cb; 1 };
32 41
33 # Tk 42 # Tk integration
34 Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "", 43 Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "",
35 readable => \&IO::AIO::poll_cb); 44 readable => \&IO::AIO::poll_cb);
36 45
37 # Danga::Socket 46 # Danga::Socket integration
38 Danga::Socket->AddOtherFds (IO::AIO::poll_fileno => 47 Danga::Socket->AddOtherFds (IO::AIO::poll_fileno =>
39 \&IO::AIO::poll_cb); 48 \&IO::AIO::poll_cb);
40
41 49
42=head1 DESCRIPTION 50=head1 DESCRIPTION
43 51
44This module implements asynchronous I/O using whatever means your 52This module implements asynchronous I/O using whatever means your
45operating system supports. 53operating system supports.
61=cut 69=cut
62 70
63package IO::AIO; 71package IO::AIO;
64 72
65no warnings; 73no warnings;
74use strict 'vars';
66 75
67use base 'Exporter'; 76use base 'Exporter';
68 77
69use Fcntl ();
70
71BEGIN { 78BEGIN {
72 $VERSION = '1.8'; 79 our $VERSION = '2.0';
73 80
74 @EXPORT = qw(aio_sendfile aio_read aio_write aio_open aio_close aio_stat 81 our @EXPORT = qw(aio_sendfile aio_read aio_write aio_open aio_close aio_stat
75 aio_lstat aio_unlink aio_rmdir aio_readdir aio_scandir aio_symlink 82 aio_lstat aio_unlink aio_rmdir aio_readdir aio_scandir aio_symlink
76 aio_fsync aio_fdatasync aio_readahead aio_rename aio_link aio_move); 83 aio_fsync aio_fdatasync aio_readahead aio_rename aio_link aio_move
84 aio_group);
77 @EXPORT_OK = qw(poll_fileno poll_cb min_parallel max_parallel 85 our @EXPORT_OK = qw(poll_fileno poll_cb min_parallel max_parallel max_outstanding nreqs);
78 max_outstanding nreqs); 86
87 @IO::AIO::GRP::ISA = 'IO::AIO::REQ';
79 88
80 require XSLoader; 89 require XSLoader;
81 XSLoader::load IO::AIO, $VERSION; 90 XSLoader::load ("IO::AIO", $VERSION);
82} 91}
83 92
84=head1 FUNCTIONS 93=head1 FUNCTIONS
85 94
86=head2 AIO FUNCTIONS 95=head2 AIO FUNCTIONS
93perl, which usually delivers "false") as it's sole argument when the given 102perl, which usually delivers "false") as it's sole argument when the given
94syscall has been executed asynchronously. 103syscall has been executed asynchronously.
95 104
96All functions expecting a filehandle keep a copy of the filehandle 105All functions expecting a filehandle keep a copy of the filehandle
97internally until the request has finished. 106internally until the request has finished.
107
108All requests return objects of type L<IO::AIO::REQ> that allow further
109manipulation of those requests while they are in-flight.
98 110
99The pathnames you pass to these routines I<must> be absolute and 111The pathnames you pass to these routines I<must> be absolute and
100encoded in byte form. The reason for the former is that at the time the 112encoded in byte form. The reason for the former is that at the time the
101request is being executed, the current working directory could have 113request is being executed, the current working directory could have
102changed. Alternatively, you can make sure that you never change the 114changed. Alternatively, you can make sure that you never change the
170 print "read $_[0] bytes: <$buffer>\n"; 182 print "read $_[0] bytes: <$buffer>\n";
171 }; 183 };
172 184
173=item aio_move $srcpath, $dstpath, $callback->($status) 185=item aio_move $srcpath, $dstpath, $callback->($status)
174 186
175[EXPERIMENTAL] 187[EXPERIMENTAL due to internal aio_group use]
176 188
177Try to move the I<file> (directories not supported as either source or destination) 189Try to move the I<file> (directories not supported as either source or
178from C<$srcpath> to C<$dstpath> and call the callback with the C<0> (error) or C<-1> ok. 190destination) from C<$srcpath> to C<$dstpath> and call the callback with
191the C<0> (error) or C<-1> ok.
179 192
180This is a composite request that tries to rename(2) the file first. If 193This is a composite request that tries to rename(2) the file first. If
181rename files with C<EXDEV>, it creates the destination file with mode 0200 194rename files with C<EXDEV>, it creates the destination file with mode 0200
182and copies the contents of the source file into it using C<aio_sendfile>, 195and copies the contents of the source file into it using C<aio_sendfile>,
183followed by restoring atime, mtime, access mode and uid/gid, in that 196followed by restoring atime, mtime, access mode and uid/gid, in that
190=cut 203=cut
191 204
192sub aio_move($$$) { 205sub aio_move($$$) {
193 my ($src, $dst, $cb) = @_; 206 my ($src, $dst, $cb) = @_;
194 207
208 my $grp = aio_group $cb;
209
195 aio_rename $src, $dst, sub { 210 add $grp aio_rename $src, $dst, sub {
196 if ($_[0] && $! == Errno::EXDEV) { 211 if ($_[0] && $! == EXDEV) {
197 aio_open $src, O_RDONLY, 0, sub { 212 add $grp aio_open $src, O_RDONLY, 0, sub {
198 if (my $src_fh = $_[0]) { 213 if (my $src_fh = $_[0]) {
199 my @stat = stat $src_fh; 214 my @stat = stat $src_fh;
200 215
201 aio_open $dst, O_WRONLY, 0200, sub { 216 add $grp aio_open $dst, O_WRONLY, 0200, sub {
202 if (my $dst_fh = $_[0]) { 217 if (my $dst_fh = $_[0]) {
203 aio_sendfile $dst_fh, $src_fh, 0, $stat[7], sub { 218 add $grp aio_sendfile $dst_fh, $src_fh, 0, $stat[7], sub {
204 close $src_fh; 219 close $src_fh;
205 220
206 if ($_[0] == $stat[7]) { 221 if ($_[0] == $stat[7]) {
207 utime $stat[8], $stat[9], $dst; 222 utime $stat[8], $stat[9], $dst;
208 chmod $stat[2] & 07777, $dst_fh; 223 chmod $stat[2] & 07777, $dst_fh;
209 chown $stat[4], $stat[5], $dst_fh; 224 chown $stat[4], $stat[5], $dst_fh;
210 close $dst_fh; 225 close $dst_fh;
211 226
212 aio_unlink $src, sub { 227 add $grp aio_unlink $src, sub {
213 $cb->($_[0]); 228 $grp->result ($_[0]);
214 }; 229 };
215 } else { 230 } else {
216 my $errno = $!; 231 my $errno = $!;
217 aio_unlink $dst, sub { 232 add $grp aio_unlink $dst, sub {
218 $! = $errno; 233 $! = $errno;
219 $cb->(-1); 234 $grp->result (-1);
220 }; 235 };
221 } 236 }
222 }; 237 };
223 } else { 238 } else {
224 $cb->(-1); 239 $grp->result (-1);
225 } 240 }
226 }, 241 },
227 242
228 } else { 243 } else {
229 $cb->(-1); 244 $grp->result (-1);
230 } 245 }
231 }; 246 };
232 } else { 247 } else {
233 $cb->($_[0]); 248 $grp->result ($_[0]);
234 } 249 }
235 }; 250 };
251
252 $grp
236} 253}
237 254
238=item aio_sendfile $out_fh, $in_fh, $in_offset, $length, $callback->($retval) 255=item aio_sendfile $out_fh, $in_fh, $in_offset, $length, $callback->($retval)
239 256
240Tries to copy C<$length> bytes from C<$in_fh> to C<$out_fh>. It starts 257Tries to copy C<$length> bytes from C<$in_fh> to C<$out_fh>. It starts
328The callback a single argument which is either C<undef> or an array-ref 345The callback a single argument which is either C<undef> or an array-ref
329with the filenames. 346with the filenames.
330 347
331=item aio_scandir $path, $maxreq, $callback->($dirs, $nondirs) 348=item aio_scandir $path, $maxreq, $callback->($dirs, $nondirs)
332 349
350[EXPERIMENTAL due to internal aio_group use]
351
333Scans a directory (similar to C<aio_readdir>) and tries to separate the 352Scans a directory (similar to C<aio_readdir>) but additionally tries to
334entries of directory C<$path> into two sets of names, ones you can recurse 353separate the entries of directory C<$path> into two sets of names, ones
335into (directories), and ones you cannot recurse into (everything else). 354you can recurse into (directories or links to them), and ones you cannot
355recurse into (everything else).
336 356
337C<aio_scandir> is a composite request that consists of many 357C<aio_scandir> is a composite request that consists of many sub
338aio-primitives. C<$maxreq> specifies the maximum number of outstanding 358requests. C<$maxreq> specifies the maximum number of outstanding aio
339aio requests that this function generates. If it is C<< <= 0 >>, then a 359requests that this function generates. If it is C<< <= 0 >>, then a
340suitable default will be chosen (currently 8). 360suitable default will be chosen (currently 8).
341 361
342On error, the callback is called without arguments, otherwise it receives 362On error, the callback is called without arguments, otherwise it receives
343two array-refs with path-relative entry names. 363two array-refs with path-relative entry names.
344 364
353Implementation notes. 373Implementation notes.
354 374
355The C<aio_readdir> cannot be avoided, but C<stat()>'ing every entry can. 375The C<aio_readdir> cannot be avoided, but C<stat()>'ing every entry can.
356 376
357After reading the directory, the modification time, size etc. of the 377After reading the directory, the modification time, size etc. of the
358directory before and after the readdir is checked, and if they match, the 378directory before and after the readdir is checked, and if they match (and
359link count will be used to decide how many entries are directories (if 379isn't the current time), the link count will be used to decide how many
360>= 2). Otherwise, no knowledge of the number of subdirectories will be 380entries are directories (if >= 2). Otherwise, no knowledge of the number
361assumed. 381of subdirectories will be assumed.
362 382
363Then entires will be sorted into likely directories (everything without a 383Then entries will be sorted into likely directories (everything without
364non-initial dot) and likely non-directories (everything else). Then every 384a non-initial dot currently) and likely non-directories (everything
365entry + C</.> will be C<stat>'ed, likely directories first. This is often 385else). Then every entry plus an appended C</.> will be C<stat>'ed,
386likely directories first. If that succeeds, it assumes that the entry
387is a directory or a symlink to directory (which will be checked
388seperately). This is often faster than stat'ing the entry itself because
366faster because filesystems might detect the type of the entry without 389filesystems might detect the type of the entry without reading the inode
367reading the inode data (e.g. ext2fs filetype feature). If that succeeds, 390data (e.g. ext2fs filetype feature).
368it assumes that the entry is a directory or a symlink to directory (which
369will be checked seperately).
370 391
371If the known number of directories has been reached, the rest of the 392If the known number of directories (link count - 2) has been reached, the
372entries is assumed to be non-directories. 393rest of the entries is assumed to be non-directories.
394
395This only works with certainty on POSIX (= UNIX) filesystems, which
396fortunately are the vast majority of filesystems around.
397
398It will also likely work on non-POSIX filesystems with reduced efficiency
399as those tend to return 0 or 1 as link counts, which disables the
400directory counting heuristic.
373 401
374=cut 402=cut
375 403
376sub aio_scandir($$$) { 404sub aio_scandir($$$) {
377 my ($path, $maxreq, $cb) = @_; 405 my ($path, $maxreq, $cb) = @_;
378 406
407 my $grp = aio_group $cb;
408
379 $maxreq = 8 if $maxreq <= 0; 409 $maxreq = 8 if $maxreq <= 0;
380 410
381 # stat once 411 # stat once
382 aio_stat $path, sub { 412 add $grp aio_stat $path, sub {
383 return $cb->() if $_[0]; 413 return $grp->result () if $_[0];
414 my $now = time;
384 my $hash1 = join ":", (stat _)[0,1,3,7,9]; 415 my $hash1 = join ":", (stat _)[0,1,3,7,9];
385 416
386 # read the directory entries 417 # read the directory entries
387 aio_readdir $path, sub { 418 add $grp aio_readdir $path, sub {
388 my $entries = shift 419 my $entries = shift
389 or return $cb->(); 420 or return $grp->result ();
390 421
391 # stat the dir another time 422 # stat the dir another time
392 aio_stat $path, sub { 423 add $grp aio_stat $path, sub {
393 my $hash2 = join ":", (stat _)[0,1,3,7,9]; 424 my $hash2 = join ":", (stat _)[0,1,3,7,9];
394 425
395 my $ndirs; 426 my $ndirs;
396 427
397 # take the slow route if anything looks fishy 428 # take the slow route if anything looks fishy
398 if ($hash1 ne $hash2) { 429 if ($hash1 ne $hash2 or (stat _)[9] == $now) {
399 $ndirs = -1; 430 $ndirs = -1;
400 } else { 431 } else {
401 # if nlink == 2, we are finished 432 # if nlink == 2, we are finished
402 # on non-posix-fs's, we rely on nlink < 2 433 # on non-posix-fs's, we rely on nlink < 2
403 $ndirs = (stat _)[3] - 2 434 $ndirs = (stat _)[3] - 2
404 or return $cb->([], $entries); 435 or return $grp->result ([], $entries);
405 } 436 }
406 437
407 # sort into likely dirs and likely nondirs 438 # sort into likely dirs and likely nondirs
408 # dirs == files without ".", short entries first 439 # dirs == files without ".", short entries first
409 $entries = [map $_->[0], 440 $entries = [map $_->[0],
419 $schedcb = sub { 450 $schedcb = sub {
420 if (@$entries) { 451 if (@$entries) {
421 if ($nreq < $maxreq) { 452 if ($nreq < $maxreq) {
422 my $ent = pop @$entries; 453 my $ent = pop @$entries;
423 $nreq++; 454 $nreq++;
424 aio_stat "$path/$ent/.", sub { $statcb->($_[0], $ent) }; 455 add $grp aio_stat "$path/$ent/.", sub { $statcb->($_[0], $ent) };
425 } 456 }
426 } elsif (!$nreq) { 457 } elsif (!$nreq) {
427 # finished 458 # finished
428 undef $statcb; 459 undef $statcb;
429 undef $schedcb; 460 undef $schedcb;
430 $cb->(\@dirs, \@nondirs) if $cb; 461 $grp->result (\@dirs, \@nondirs) if $cb;
431 undef $cb; 462 undef $cb;
432 } 463 }
433 }; 464 };
434 $statcb = sub { 465 $statcb = sub {
435 my ($status, $entry) = @_; 466 my ($status, $entry) = @_;
438 $nreq--; 469 $nreq--;
439 push @nondirs, $entry; 470 push @nondirs, $entry;
440 &$schedcb; 471 &$schedcb;
441 } else { 472 } else {
442 # need to check for real directory 473 # need to check for real directory
443 aio_lstat "$path/$entry", sub { 474 add $grp aio_lstat "$path/$entry", sub {
444 $nreq--; 475 $nreq--;
445 476
446 if (-d _) { 477 if (-d _) {
447 push @dirs, $entry; 478 push @dirs, $entry;
448 479
461 492
462 &$schedcb while @$entries && $nreq < $maxreq; 493 &$schedcb while @$entries && $nreq < $maxreq;
463 }; 494 };
464 }; 495 };
465 }; 496 };
497
498 $grp
466} 499}
467 500
468=item aio_fsync $fh, $callback->($status) 501=item aio_fsync $fh, $callback->($status)
469 502
470Asynchronously call fsync on the given filehandle and call the callback 503Asynchronously call fsync on the given filehandle and call the callback
475Asynchronously call fdatasync on the given filehandle and call the 508Asynchronously call fdatasync on the given filehandle and call the
476callback with the fdatasync result code. 509callback with the fdatasync result code.
477 510
478If this call isn't available because your OS lacks it or it couldn't be 511If this call isn't available because your OS lacks it or it couldn't be
479detected, it will be emulated by calling C<fsync> instead. 512detected, it will be emulated by calling C<fsync> instead.
513
514=item aio_group $callback->(...)
515
516[EXPERIMENTAL]
517
518This is a very special aio request: Instead of doing something, it is a
519container for other aio requests, which is useful if you want to bundle
520many requests into a single, composite, request.
521
522Returns an object of class L<IO::AIO::GRP>. See its documentation below
523for more info.
524
525Example:
526
527 my $grp = aio_group sub {
528 print "all stats done\n";
529 };
530
531 add $grp
532 (aio_stat ...),
533 (aio_stat ...),
534 ...;
535
536=item IO::AIO::aio_sleep $fractional_seconds, $callback->() *NOT EXPORTED*
537
538Mainly used for debugging and benchmarking, this aio request puts one of
539the request workers to sleep for the given time.
540
541While it is theoretically handy to have simple I/O scheduling requests
542like sleep and file handle readable/writable, the overhead this creates
543is immense, so do not use this function except to put your application
544under artificial I/O pressure.
545
546=back
547
548=head2 IO::AIO::REQ CLASS
549
550All non-aggregate C<aio_*> functions return an object of this class when
551called in non-void context.
552
553A request always moves through the following five states in its lifetime,
554in order: B<ready> (request has been created, but has not been executed
555yet), B<execute> (request is currently being executed), B<pending>
556(request has been executed but callback has not been called yet),
557B<result> (results are being processed synchronously, includes calling the
558callback) and B<done> (request has reached the end of its lifetime and
559holds no resources anymore).
560
561=over 4
562
563=item $req->cancel
564
565Cancels the request, if possible. Has the effect of skipping execution
566when entering the B<execute> state and skipping calling the callback when
567entering the the B<result> state, but will leave the request otherwise
568untouched. That means that requests that currently execute will not be
569stopped and resources held by the request will not be freed prematurely.
570
571=back
572
573=head2 IO::AIO::GRP CLASS
574
575This class is a subclass of L<IO::AIO::REQ>, so all its methods apply to
576objects of this class, too.
577
578A IO::AIO::GRP object is a special request that can contain multiple other
579aio requests.
580
581You create one by calling the C<aio_group> constructing function with a
582callback that will be called when all contained requests have entered the
583C<done> state:
584
585 my $grp = aio_group sub {
586 print "all requests are done\n";
587 };
588
589You add requests by calling the C<add> method with one or more
590C<IO::AIO::REQ> objects:
591
592 $grp->add (aio_unlink "...");
593
594 add $grp aio_stat "...", sub {
595 $_[0] or return $grp->result ("error");
596
597 # add another request dynamically, if first succeeded
598 add $grp aio_open "...", sub {
599 $grp->result ("ok");
600 };
601 };
602
603This makes it very easy to create composite requests (see the source of
604C<aio_move> for an application) that work and feel like simple requests.
605
606The IO::AIO::GRP objects will be cleaned up during calls to
607C<IO::AIO::poll_cb>, just like any other request.
608
609They can be canceled like any other request. Canceling will cancel not
610only the request itself, but also all requests it contains.
611
612They can also can also be added to other IO::AIO::GRP objects.
613
614Their lifetime, simplified, looks like this: when they are empty, they
615will finish very quickly. If they contain only requests that are in the
616C<done> state, they will also finish. Otherwise they will continue to
617exist.
618
619That means after creating a group you have some time to add requests. And
620in the callbacks of those requests, you can add further requests to the
621group. And only when all those requests have finished will the the group
622itself finish.
623
624=over 4
625
626=item $grp->add (...)
627
628=item add $grp ...
629
630Add one or more requests to the group. Any type of L<IO::AIO::REQ> can
631be added, including other groups, as long as you do not create circular
632dependencies.
633
634Returns all its arguments.
635
636=item $grp->result (...)
637
638Set the result value(s) that will be passed to the group callback when all
639subrequests have finished. By default, no argument will be passed.
480 640
481=back 641=back
482 642
483=head2 SUPPORT FUNCTIONS 643=head2 SUPPORT FUNCTIONS
484 644
612} 772}
613 773
6141; 7741;
615 775
616=head2 FORK BEHAVIOUR 776=head2 FORK BEHAVIOUR
777
778This module should do "the right thing" when the process using it forks:
617 779
618Before the fork, IO::AIO enters a quiescent state where no requests 780Before the fork, IO::AIO enters a quiescent state where no requests
619can be added in other threads and no results will be processed. After 781can be added in other threads and no results will be processed. After
620the fork the parent simply leaves the quiescent state and continues 782the fork the parent simply leaves the quiescent state and continues
621request/result processing, while the child clears the request/result 783request/result processing, while the child clears the request/result
622queue (so the requests started before the fork will only be handled in 784queue (so the requests started before the fork will only be handled in
623the parent). Threats will be started on demand until the limit ste in the 785the parent). Threads will be started on demand until the limit ste in the
624parent process has been reached again. 786parent process has been reached again.
625 787
788In short: the parent will, after a short pause, continue as if fork had
789not been called, while the child will act as if IO::AIO has not been used
790yet.
791
626=head1 SEE ALSO 792=head1 SEE ALSO
627 793
628L<Coro>, L<Linux::AIO>. 794L<Coro>, L<Linux::AIO> (obsolete).
629 795
630=head1 AUTHOR 796=head1 AUTHOR
631 797
632 Marc Lehmann <schmorp@schmorp.de> 798 Marc Lehmann <schmorp@schmorp.de>
633 http://home.schmorp.de/ 799 http://home.schmorp.de/

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines