--- IO-AIO/AIO.pm 2007/01/23 22:57:34 1.103 +++ IO-AIO/AIO.pm 2008/06/17 23:38:10 1.129 @@ -28,9 +28,11 @@ my $grp = aio_group sub { print "all stats done\n" }; add $grp aio_stat "..." for ...; - # AnyEvent integration - open my $fh, "<&=" . IO::AIO::poll_fileno or die "$!"; - my $w = AnyEvent->io (fh => $fh, poll => 'r', cb => sub { IO::AIO::poll_cb }); + # AnyEvent integration (EV, Event, Glib, Tk, POE, urxvt, pureperl...) + use AnyEvent::AIO; + + # EV integration + my $w = EV::io IO::AIO::poll_fileno, EV::READ, \&IO::AIO::poll_cb; # Event integration Event->io (fd => IO::AIO::poll_fileno, @@ -64,12 +66,11 @@ on a RAID volume or over NFS when you do a number of stat operations concurrently. -While most of this works on all types of file descriptors (for example -sockets), using these functions on file descriptors that support -nonblocking operation (again, sockets, pipes etc.) is very inefficient or -might not work (aio_read fails on sockets/pipes/fifos). Use an event loop -for that (such as the L module): IO::AIO will naturally fit -into such an event loop itself. +While most of this works on all types of file descriptors (for +example sockets), using these functions on file descriptors that +support nonblocking operation (again, sockets, pipes etc.) is very +inefficient. Use an event loop for that (such as the L +module): IO::AIO will naturally fit into such an event loop itself. In this version, a number of threads are started that execute your requests and signal their completion. You don't need thread support @@ -81,10 +82,10 @@ aio_write, so the remaining functionality would have to be implemented using threads anyway. -Although the module will work with in the presence of other (Perl-) -threads, it is currently not reentrant in any way, so use appropriate -locking yourself, always call C from within the same thread, or -never call C (or other C functions) recursively. +Although the module will work in the presence of other (Perl-) threads, +it is currently not reentrant in any way, so use appropriate locking +yourself, always call C from within the same thread, or never +call C (or other C functions) recursively. =head2 EXAMPLE @@ -186,19 +187,25 @@ package IO::AIO; +use Carp (); + no warnings; use strict 'vars'; use base 'Exporter'; BEGIN { - our $VERSION = '2.33'; + our $VERSION = '3.04'; - our @AIO_REQ = qw(aio_sendfile aio_read aio_write aio_open aio_close aio_stat - aio_lstat aio_unlink aio_rmdir aio_readdir aio_scandir aio_symlink - aio_readlink aio_fsync aio_fdatasync aio_readahead aio_rename aio_link - aio_move aio_copy aio_group aio_nop aio_mknod aio_load aio_rmtree aio_mkdir); - our @EXPORT = (@AIO_REQ, qw(aioreq_pri aioreq_nice aio_block)); + our @AIO_REQ = qw(aio_sendfile aio_read aio_write aio_open aio_close + aio_stat aio_lstat aio_unlink aio_rmdir aio_readdir + aio_scandir aio_symlink aio_readlink aio_sync aio_fsync + aio_fdatasync aio_pathsync aio_readahead + aio_rename aio_link aio_move aio_copy aio_group + aio_nop aio_mknod aio_load aio_rmtree aio_mkdir aio_chown + aio_chmod aio_utime aio_truncate); + + our @EXPORT = (@AIO_REQ, qw(aioreq_pri aioreq_nice)); our @EXPORT_OK = qw(poll_fileno poll_cb poll_wait flush min_parallel max_parallel max_idle nreqs nready npending nthreads @@ -273,11 +280,13 @@ }; }; + =item aioreq_nice $pri_adjust Similar to C, but subtracts the given value from the current priority, so the effect is cumulative. + =item aio_open $pathname, $flags, $mode, $callback->($fh) Asynchronously open or create a file and call the callback with a newly @@ -307,29 +316,45 @@ } }; + =item aio_close $fh, $callback->($status) Asynchronously close a file and call the callback with the result -code. I although accepted, you should not pass in a perl -filehandle here, as perl will likely close the file descriptor another -time when the filehandle is destroyed. Normally, you can safely call perls -C or just let filehandles go out of scope. +code. + +Unfortunately, you can't do this to perl. Perl I very strongly on +closing the file descriptor associated with the filehandle itself. -This is supposed to be a bug in the API, so that might change. It's -therefore best to avoid this function. +Therefore, C will not close the filehandle - instead it will +use dup2 to overwrite the file descriptor with the write-end of a pipe +(the pipe fd will be created on demand and will be cached). + +Or in other words: the file descriptor will be closed, but it will not be +free for reuse until the perl filehandle is closed. + +=cut =item aio_read $fh,$offset,$length, $data,$dataoffset, $callback->($retval) =item aio_write $fh,$offset,$length, $data,$dataoffset, $callback->($retval) -Reads or writes C bytes from the specified C and C -into the scalar given by C and offset C and calls the +Reads or writes C<$length> bytes from the specified C<$fh> and C<$offset> +into the scalar given by C<$data> and offset C<$dataoffset> and calls the callback without the actual number of bytes read (or -1 on error, just like the syscall). +If C<$offset> is undefined, then the current file descriptor offset will +be used (and updated), otherwise the file descriptor offset will not be +changed by these calls. + +If C<$length> is undefined in C, use the remaining length of C<$data>. + +If C<$dataoffset> is less than zero, it will be counted from the end of +C<$data>. + The C<$data> scalar I be modified in any way while the request -is outstanding. Modifying it can result in segfaults or WW3 (if the -necessary/optional hardware is installed). +is outstanding. Modifying it can result in segfaults or World War III (if +the necessary/optional hardware is installed). Example: Read 15 bytes at offset 7 into scalar C<$buffer>, starting at offset C<0> within the scalar: @@ -339,6 +364,7 @@ print "read $_[0] bytes: <$buffer>\n"; }; + =item aio_sendfile $out_fh, $in_fh, $in_offset, $length, $callback->($retval) Tries to copy C<$length> bytes from C<$in_fh> to C<$out_fh>. It starts @@ -362,6 +388,7 @@ value equals C<$length> one can assume that C<$length> bytes have been read. + =item aio_readahead $fh,$offset,$length, $callback->($retval) C populates the page cache with data from a file so that @@ -376,6 +403,7 @@ If that syscall doesn't exist (likely if your OS isn't Linux) it will be emulated by simply reading the data, which would have a similar effect. + =item aio_stat $fh_or_path, $callback->($status) =item aio_lstat $fh, $callback->($status) @@ -398,11 +426,54 @@ print "size is ", -s _, "\n"; }; + +=item aio_utime $fh_or_path, $atime, $mtime, $callback->($status) + +Works like perl's C function (including the special case of $atime +and $mtime being undef). Fractional times are supported if the underlying +syscalls support them. + +When called with a pathname, uses utimes(2) if available, otherwise +utime(2). If called on a file descriptor, uses futimes(2) if available, +otherwise returns ENOSYS, so this is not portable. + +Examples: + + # set atime and mtime to current time (basically touch(1)): + aio_utime "path", undef, undef; + # set atime to current time and mtime to beginning of the epoch: + aio_utime "path", time, undef; # undef==0 + + +=item aio_chown $fh_or_path, $uid, $gid, $callback->($status) + +Works like perl's C function, except that C for either $uid +or $gid is being interpreted as "do not change" (but -1 can also be used). + +Examples: + + # same as "chown root path" in the shell: + aio_chown "path", 0, -1; + # same as above: + aio_chown "path", 0, undef; + + +=item aio_truncate $fh_or_path, $offset, $callback->($status) + +Works like truncate(2) or ftruncate(2). + + +=item aio_chmod $fh_or_path, $mode, $callback->($status) + +Works like perl's C function. + + =item aio_unlink $pathname, $callback->($status) Asynchronously unlink (delete) a file and call the callback with the result code. + =item aio_mknod $path, $mode, $dev, $callback->($status) [EXPERIMENTAL] @@ -413,38 +484,45 @@ aio_mknod $path, IO::AIO::S_IFIFO | $mode, 0, sub { ... + =item aio_link $srcpath, $dstpath, $callback->($status) Asynchronously create a new link to the existing object at C<$srcpath> at the path C<$dstpath> and call the callback with the result code. + =item aio_symlink $srcpath, $dstpath, $callback->($status) Asynchronously create a new symbolic link to the existing object at C<$srcpath> at the path C<$dstpath> and call the callback with the result code. + =item aio_readlink $path, $callback->($link) Asynchronously read the symlink specified by C<$path> and pass it to the callback. If an error occurs, nothing or undef gets passed to the callback. + =item aio_rename $srcpath, $dstpath, $callback->($status) Asynchronously rename the object at C<$srcpath> to C<$dstpath>, just as rename(2) and call the callback with the result code. + =item aio_mkdir $pathname, $mode, $callback->($status) Asynchronously mkdir (create) a directory and call the callback with the result code. C<$mode> will be modified by the umask at the time the request is executed, so do not change your umask. + =item aio_rmdir $pathname, $callback->($status) Asynchronously rmdir (delete) a directory and call the callback with the result code. + =item aio_readdir $pathname, $callback->($entries) Unlike the POSIX call of the same name, C reads an entire @@ -454,6 +532,7 @@ The callback a single argument which is either C or an array-ref with the filenames. + =item aio_load $path, $data, $callback->($status) This is a composite request that tries to fully load the given file into @@ -462,26 +541,24 @@ =cut sub aio_load($$;$) { - aio_block { - my ($path, undef, $cb) = @_; - my $data = \$_[1]; + my ($path, undef, $cb) = @_; + my $data = \$_[1]; - my $pri = aioreq_pri; - my $grp = aio_group $cb; + my $pri = aioreq_pri; + my $grp = aio_group $cb; - aioreq_pri $pri; - add $grp aio_open $path, O_RDONLY, 0, sub { - my $fh = shift - or return $grp->result (-1); + aioreq_pri $pri; + add $grp aio_open $path, O_RDONLY, 0, sub { + my $fh = shift + or return $grp->result (-1); - aioreq_pri $pri; - add $grp aio_read $fh, 0, (-s $fh), $$data, 0, sub { - $grp->result ($_[0]); - }; + aioreq_pri $pri; + add $grp aio_read $fh, 0, (-s $fh), $$data, 0, sub { + $grp->result ($_[0]); }; + }; - $grp - } + $grp } =item aio_copy $srcpath, $dstpath, $callback->($status) @@ -502,52 +579,52 @@ =cut sub aio_copy($$;$) { - aio_block { - my ($src, $dst, $cb) = @_; + my ($src, $dst, $cb) = @_; - my $pri = aioreq_pri; - my $grp = aio_group $cb; + my $pri = aioreq_pri; + my $grp = aio_group $cb; - aioreq_pri $pri; - add $grp aio_open $src, O_RDONLY, 0, sub { - if (my $src_fh = $_[0]) { - my @stat = stat $src_fh; + aioreq_pri $pri; + add $grp aio_open $src, O_RDONLY, 0, sub { + if (my $src_fh = $_[0]) { + my @stat = stat $src_fh; - aioreq_pri $pri; - add $grp aio_open $dst, O_CREAT | O_WRONLY | O_TRUNC, 0200, sub { - if (my $dst_fh = $_[0]) { - aioreq_pri $pri; - add $grp aio_sendfile $dst_fh, $src_fh, 0, $stat[7], sub { - if ($_[0] == $stat[7]) { - $grp->result (0); - close $src_fh; - - # those should not normally block. should. should. - utime $stat[8], $stat[9], $dst; - chmod $stat[2] & 07777, $dst_fh; - chown $stat[4], $stat[5], $dst_fh; - close $dst_fh; - } else { - $grp->result (-1); - close $src_fh; - close $dst_fh; + aioreq_pri $pri; + add $grp aio_open $dst, O_CREAT | O_WRONLY | O_TRUNC, 0200, sub { + if (my $dst_fh = $_[0]) { + aioreq_pri $pri; + add $grp aio_sendfile $dst_fh, $src_fh, 0, $stat[7], sub { + if ($_[0] == $stat[7]) { + $grp->result (0); + close $src_fh; + + # those should not normally block. should. should. + utime $stat[8], $stat[9], $dst; + chmod $stat[2] & 07777, $dst_fh; + chown $stat[4], $stat[5], $dst_fh; + + aioreq_pri $pri; + add $grp aio_close $dst_fh; + } else { + $grp->result (-1); + close $src_fh; + close $dst_fh; + + aioreq $pri; + add $grp aio_unlink $dst; + } + }; + } else { + $grp->result (-1); + } + }, - aioreq $pri; - add $grp aio_unlink $dst; - } - }; - } else { - $grp->result (-1); - } - }, - - } else { - $grp->result (-1); - } - }; + } else { + $grp->result (-1); + } + }; - $grp - } + $grp } =item aio_move $srcpath, $dstpath, $callback->($status) @@ -563,31 +640,29 @@ =cut sub aio_move($$;$) { - aio_block { - my ($src, $dst, $cb) = @_; + my ($src, $dst, $cb) = @_; - my $pri = aioreq_pri; - my $grp = aio_group $cb; + my $pri = aioreq_pri; + my $grp = aio_group $cb; - aioreq_pri $pri; - add $grp aio_rename $src, $dst, sub { - if ($_[0] && $! == EXDEV) { - aioreq_pri $pri; - add $grp aio_copy $src, $dst, sub { - $grp->result ($_[0]); - - if (!$_[0]) { - aioreq_pri $pri; - add $grp aio_unlink $src; - } - }; - } else { + aioreq_pri $pri; + add $grp aio_rename $src, $dst, sub { + if ($_[0] && $! == EXDEV) { + aioreq_pri $pri; + add $grp aio_copy $src, $dst, sub { $grp->result ($_[0]); - } - }; - $grp - } + if (!$_[0]) { + aioreq_pri $pri; + add $grp aio_unlink $src; + } + }; + } else { + $grp->result ($_[0]); + } + }; + + $grp } =item aio_scandir $path, $maxreq, $callback->($dirs, $nondirs) @@ -645,91 +720,89 @@ =cut sub aio_scandir($$;$) { - aio_block { - my ($path, $maxreq, $cb) = @_; + my ($path, $maxreq, $cb) = @_; - my $pri = aioreq_pri; + my $pri = aioreq_pri; - my $grp = aio_group $cb; + my $grp = aio_group $cb; - $maxreq = 4 if $maxreq <= 0; + $maxreq = 4 if $maxreq <= 0; - # stat once + # stat once + aioreq_pri $pri; + add $grp aio_stat $path, sub { + return $grp->result () if $_[0]; + my $now = time; + my $hash1 = join ":", (stat _)[0,1,3,7,9]; + + # read the directory entries aioreq_pri $pri; - add $grp aio_stat $path, sub { - return $grp->result () if $_[0]; - my $now = time; - my $hash1 = join ":", (stat _)[0,1,3,7,9]; + add $grp aio_readdir $path, sub { + my $entries = shift + or return $grp->result (); - # read the directory entries + # stat the dir another time aioreq_pri $pri; - add $grp aio_readdir $path, sub { - my $entries = shift - or return $grp->result (); - - # stat the dir another time - aioreq_pri $pri; - add $grp aio_stat $path, sub { - my $hash2 = join ":", (stat _)[0,1,3,7,9]; + add $grp aio_stat $path, sub { + my $hash2 = join ":", (stat _)[0,1,3,7,9]; - my $ndirs; + my $ndirs; - # take the slow route if anything looks fishy - if ($hash1 ne $hash2 or (stat _)[9] == $now) { - $ndirs = -1; - } else { - # if nlink == 2, we are finished - # on non-posix-fs's, we rely on nlink < 2 - $ndirs = (stat _)[3] - 2 - or return $grp->result ([], $entries); - } - - # sort into likely dirs and likely nondirs - # dirs == files without ".", short entries first - $entries = [map $_->[0], - sort { $b->[1] cmp $a->[1] } - map [$_, sprintf "%s%04d", (/.\./ ? "1" : "0"), length], - @$entries]; + # take the slow route if anything looks fishy + if ($hash1 ne $hash2 or (stat _)[9] == $now) { + $ndirs = -1; + } else { + # if nlink == 2, we are finished + # on non-posix-fs's, we rely on nlink < 2 + $ndirs = (stat _)[3] - 2 + or return $grp->result ([], $entries); + } + + # sort into likely dirs and likely nondirs + # dirs == files without ".", short entries first + $entries = [map $_->[0], + sort { $b->[1] cmp $a->[1] } + map [$_, sprintf "%s%04d", (/.\./ ? "1" : "0"), length], + @$entries]; - my (@dirs, @nondirs); + my (@dirs, @nondirs); - my $statgrp = add $grp aio_group sub { - $grp->result (\@dirs, \@nondirs); - }; + my $statgrp = add $grp aio_group sub { + $grp->result (\@dirs, \@nondirs); + }; - limit $statgrp $maxreq; - feed $statgrp sub { - return unless @$entries; - my $entry = pop @$entries; - - aioreq_pri $pri; - add $statgrp aio_stat "$path/$entry/.", sub { - if ($_[0] < 0) { - push @nondirs, $entry; - } else { - # need to check for real directory - aioreq_pri $pri; - add $statgrp aio_lstat "$path/$entry", sub { - if (-d _) { - push @dirs, $entry; - - unless (--$ndirs) { - push @nondirs, @$entries; - feed $statgrp; - } - } else { - push @nondirs, $entry; + limit $statgrp $maxreq; + feed $statgrp sub { + return unless @$entries; + my $entry = pop @$entries; + + aioreq_pri $pri; + add $statgrp aio_stat "$path/$entry/.", sub { + if ($_[0] < 0) { + push @nondirs, $entry; + } else { + # need to check for real directory + aioreq_pri $pri; + add $statgrp aio_lstat "$path/$entry", sub { + if (-d _) { + push @dirs, $entry; + + unless (--$ndirs) { + push @nondirs, @$entries; + feed $statgrp; } + } else { + push @nondirs, $entry; } } - }; + } }; }; }; }; + }; - $grp - } + $grp } =item aio_rmtree $path, $callback->($status) @@ -743,32 +816,34 @@ sub aio_rmtree; sub aio_rmtree($;$) { - aio_block { - my ($path, $cb) = @_; + my ($path, $cb) = @_; - my $pri = aioreq_pri; - my $grp = aio_group $cb; + my $pri = aioreq_pri; + my $grp = aio_group $cb; - aioreq_pri $pri; - add $grp aio_scandir $path, 0, sub { - my ($dirs, $nondirs) = @_; + aioreq_pri $pri; + add $grp aio_scandir $path, 0, sub { + my ($dirs, $nondirs) = @_; - my $dirgrp = aio_group sub { - add $grp aio_rmdir $path, sub { - $grp->result ($_[0]); - }; + my $dirgrp = aio_group sub { + add $grp aio_rmdir $path, sub { + $grp->result ($_[0]); }; + }; - (aioreq_pri $pri), add $dirgrp aio_rmtree "$path/$_" for @$dirs; - (aioreq_pri $pri), add $dirgrp aio_unlink "$path/$_" for @$nondirs; + (aioreq_pri $pri), add $dirgrp aio_rmtree "$path/$_" for @$dirs; + (aioreq_pri $pri), add $dirgrp aio_unlink "$path/$_" for @$nondirs; - add $grp $dirgrp; - }; + add $grp $dirgrp; + }; - $grp - } + $grp } +=item aio_sync $callback->($status) + +Asynchronously call sync and call the callback when finished. + =item aio_fsync $fh, $callback->($status) Asynchronously call fsync on the given filehandle and call the callback @@ -782,6 +857,44 @@ If this call isn't available because your OS lacks it or it couldn't be detected, it will be emulated by calling C instead. +=item aio_pathsync $path, $callback->($status) + +This request tries to open, fsync and close the given path. This is a +composite request intended tosync directories after directory operations +(E.g. rename). This might not work on all operating systems or have any +specific effect, but usually it makes sure that directory changes get +written to disc. It works for anything that can be opened for read-only, +not just directories. + +Passes C<0> when everything went ok, and C<-1> on error. + +=cut + +sub aio_pathsync($;$) { + my ($path, $cb) = @_; + + my $pri = aioreq_pri; + my $grp = aio_group $cb; + + aioreq_pri $pri; + add $grp aio_open $path, O_RDONLY, 0, sub { + my ($fh) = @_; + if ($fh) { + aioreq_pri $pri; + add $grp aio_fsync $fh, sub { + $grp->result ($_[0]); + + aioreq_pri $pri; + add $grp aio_close $fh; + }; + } else { + $grp->result (-1); + } + }; + + $grp +} + =item aio_group $callback->(...) This is a very special aio request: Instead of doing something, it is a @@ -927,7 +1040,7 @@ =item $grp->result (...) Set the result value(s) that will be passed to the group callback when all -subrequests have finished and set thre groups errno to the current value +subrequests have finished and set the groups errno to the current value of errno (just like calling C without an error number). By default, no argument will be passed and errno is zero. @@ -1006,12 +1119,14 @@ =item IO::AIO::poll_cb Process some outstanding events on the result pipe. You have to call this -regularly. Returns the number of events processed. Returns immediately -when no events are outstanding. The amount of events processed depends on -the settings of C and C. +regularly. Returns C<0> if all events could be processed, or C<-1> if it +returned earlier for whatever reason. Returns immediately when no events +are outstanding. The amount of events processed depends on the settings of +C and C. If not all requests were processed for whatever reason, the filehandle -will still be ready when C returns. +will still be ready when C returns, so normally you don't have to +do anything special to have it called later. Example: Install an Event watcher that automatically calls IO::AIO::poll_cb with high priority: @@ -1080,8 +1195,12 @@ IO::AIO::poll_wait, IO::AIO::poll_cb while IO::AIO::nreqs; +=back + =head3 CONTROLLING THE NUMBER OF THREADS +=over + =item IO::AIO::min_parallel $nthreads Set the minimum number of AIO threads to C<$nthreads>. The current @@ -1131,14 +1250,14 @@ creation is fast. If thread creation is very slow on your system you might want to use larger values. -=item $oldmaxreqs = IO::AIO::max_outstanding $maxreqs +=item IO::AIO::max_outstanding $maxreqs This is a very bad function to use in interactive programs because it blocks, and a bad way to reduce concurrency because it is inexact: Better use an C together with a feed callback. Sets the maximum number of outstanding requests to C<$nreqs>. If you -to queue up more than this number of requests, the next call to the +do queue up more than this number of requests, the next call to the C (and C and other functions calling C) function will block until the limit is no longer exceeded. @@ -1146,11 +1265,15 @@ number of outstanding requests. You can still queue as many requests as you want. Therefore, -C is mainly useful in simple scripts (with low values) or +C is mainly useful in simple scripts (with low values) or as a stop gap to shield against fatal memory overflow (with large values). +=back + =head3 STATISTICAL INFORMATION +=over + =item IO::AIO::nreqs Returns the number of requests currently in the ready, execute or pending @@ -1175,22 +1298,6 @@ =cut -# support function to convert a fd into a perl filehandle -sub _fd2fh { - return undef if $_[0] < 0; - - # try to generate nice filehandles - my $sym = "IO::AIO::fd#$_[0]"; - local *$sym; - - open *$sym, "+<&=$_[0]" # usually works under any unix - or open *$sym, "<&=$_[0]" # cygwin needs this - or open *$sym, ">&=$_[0]" # or this - or return undef; - - *$sym -} - min_parallel 8; END { flush } @@ -1223,7 +1330,7 @@ scalars and other data passed into aio requests will also be locked and will consume memory till the request has entered the done state. -This is now awfully much, so queuing lots of requests is not usually a +This is not awfully much, so queuing lots of requests is not usually a problem. Per-thread usage: @@ -1238,7 +1345,8 @@ =head1 SEE ALSO -L. +L for easy integration into event loops, L for a +more natural syntax. =head1 AUTHOR