--- IO-AIO/AIO.pm 2009/04/21 20:06:05 1.146 +++ IO-AIO/AIO.pm 2009/06/30 07:33:58 1.159 @@ -32,7 +32,7 @@ use AnyEvent::AIO; # EV integration - my $w = EV::io IO::AIO::poll_fileno, EV::READ, \&IO::AIO::poll_cb; + my $aio_w = EV::io IO::AIO::poll_fileno, EV::READ, \&IO::AIO::poll_cb; # Event integration Event->io (fd => IO::AIO::poll_fileno, @@ -54,7 +54,8 @@ =head1 DESCRIPTION This module implements asynchronous I/O using whatever means your -operating system supports. +operating system supports. It is implemented as an interface to C +(L). Asynchronous means that operations that can normally block your program (e.g. reading from disk) will be done asynchronously: the operation @@ -68,8 +69,8 @@ While most of this works on all types of file descriptors (for example sockets), using these functions on file descriptors that -support nonblocking operation (again, sockets, pipes etc.) is very -inefficient. Use an event loop for that (such as the L +support nonblocking operation (again, sockets, pipes etc.) is +very inefficient. Use an event loop for that (such as the L module): IO::AIO will naturally fit into such an event loop itself. In this version, a number of threads are started that execute your @@ -89,17 +90,15 @@ =head2 EXAMPLE -This is a simple example that uses the Event module and loads +This is a simple example that uses the EV module and loads F asynchronously: use Fcntl; - use Event; + use EV; use IO::AIO; - # register the IO::AIO callback with Event - Event->io (fd => IO::AIO::poll_fileno, - poll => 'r', - cb => \&IO::AIO::poll_cb); + # register the IO::AIO callback with EV + my $aio_w = EV::io IO::AIO::poll_fileno, EV::READ, \&IO::AIO::poll_cb; # queue the request to open /etc/passwd aio_open "/etc/passwd", O_RDONLY, 0, sub { @@ -121,7 +120,7 @@ print $contents; # exit event loop and program - Event::unloop; + EV::unloop; }; }; @@ -129,7 +128,7 @@ # check for sockets etc. etc. # process events as long as there are some: - Event::loop; + EV::loop; =head1 REQUEST ANATOMY AND LIFETIME @@ -195,10 +194,10 @@ use base 'Exporter'; BEGIN { - our $VERSION = '3.19'; + our $VERSION = '3.26'; our @AIO_REQ = qw(aio_sendfile aio_read aio_write aio_open aio_close - aio_stat aio_lstat aio_unlink aio_rmdir aio_readdir + aio_stat aio_lstat aio_unlink aio_rmdir aio_readdir aio_readdirx aio_scandir aio_symlink aio_readlink aio_sync aio_fsync aio_fdatasync aio_sync_file_range aio_pathsync aio_readahead aio_rename aio_link aio_move aio_copy aio_group @@ -209,7 +208,8 @@ our @EXPORT_OK = qw(poll_fileno poll_cb poll_wait flush min_parallel max_parallel max_idle nreqs nready npending nthreads - max_poll_time max_poll_reqs); + max_poll_time max_poll_reqs + sendfile fadvise); push @AIO_REQ, qw(aio_busy); # not exported @@ -535,8 +535,74 @@ directory (i.e. opendir + readdir + closedir). The entries will not be sorted, and will B include the C<.> and C<..> entries. -The callback a single argument which is either C or an array-ref -with the filenames. +The callback is passed a single argument which is either C or an +array-ref with the filenames. + + +=item aio_readdirx $pathname, $flags, $callback->($entries, $flags) + +Quite similar to C, but the C<$flags> argument allows to tune +behaviour and output format. In case of an error, C<$entries> will be +C. + +The flags are a combination of the following constants, ORed together (the +flags will also be passed to the callback, possibly modified): + +=over 4 + +=item IO::AIO::READDIR_DENTS + +When this flag is off, then the callback gets an arrayref with of names +only (as with C), otherwise it gets an arrayref with +C<[$name, $type, $inode]> arrayrefs, each describing a single directory +entry in more detail. + +C<$name> is the name of the entry. + +C<$type> is one of the C constants: + +C, C, C, C, +C, C, C, C, +C. + +C means just that: readdir does not know. If you need to +know, you have to run stat yourself. Also, for speed reasons, the C<$type> +scalars are read-only: you can not modify them. + +C<$inode> is the inode number (which might not be exact on systems with 64 +bit inode numbers and 32 bit perls). This field has unspecified content on +systems that do not deliver the inode information. + +=item IO::AIO::READDIR_DIRS_FIRST + +When this flag is set, then the names will be returned in an order where +likely directories come first. This is useful when you need to quickly +find directories, or you want to find all directories while avoiding to +stat() each entry. + +If the system returns type information in readdir, then this is used +to find directories directly. Otherwise, likely directories are files +beginning with ".", or otherwise files with no dots, of which files with +short names are tried first. + +=item IO::AIO::READDIR_STAT_ORDER + +When this flag is set, then the names will be returned in an order +suitable for stat()'ing each one. That is, when you plan to stat() +all files in the given directory, then the returned order will likely +be fastest. + +If both this flag and C are specified, then +the likely dirs come first, resulting in a less optimal stat order. + +=item IO::AIO::READDIR_FOUND_UNKNOWN + +This flag should not be set when calling C. Instead, it +is being set by C, when any of the C<$type>'s found were +C. The absense of this flag therefore indicates that all +C<$type>'s are known, which can be used to speed up some algorithms. + +=back =item aio_load $path, $data, $callback->($status) @@ -593,7 +659,7 @@ aioreq_pri $pri; add $grp aio_open $src, O_RDONLY, 0, sub { if (my $src_fh = $_[0]) { - my @stat = stat $src_fh; + my @stat = stat $src_fh; # hmm, might bock over nfs? aioreq_pri $pri; add $grp aio_open $dst, O_CREAT | O_WRONLY | O_TRUNC, 0200, sub { @@ -604,13 +670,26 @@ $grp->result (0); close $src_fh; - # those should not normally block. should. should. - utime $stat[8], $stat[9], $dst; - chmod $stat[2] & 07777, $dst_fh; - chown $stat[4], $stat[5], $dst_fh; + my $ch = sub { + aioreq_pri $pri; + add $grp aio_chmod $dst_fh, $stat[2] & 07777, sub { + aioreq_pri $pri; + add $grp aio_chown $dst_fh, $stat[4], $stat[5], sub { + aioreq_pri $pri; + add $grp aio_close $dst_fh; + } + }; + }; aioreq_pri $pri; - add $grp aio_close $dst_fh; + add $grp aio_utime $dst_fh, $stat[8], $stat[9], sub { + if ($_[0] < 0 && $! == ENOSYS) { + aioreq_pri $pri; + add $grp aio_utime $dst, $stat[8], $stat[9], $ch; + } else { + $ch->(); + } + }; } else { $grp->result (-1); close $src_fh; @@ -698,20 +777,24 @@ The C cannot be avoided, but C'ing every entry can. -After reading the directory, the modification time, size etc. of the -directory before and after the readdir is checked, and if they match (and -isn't the current time), the link count will be used to decide how many -entries are directories (if >= 2). Otherwise, no knowledge of the number -of subdirectories will be assumed. - -Then entries will be sorted into likely directories (everything without -a non-initial dot currently) and likely non-directories (everything -else). Then every entry plus an appended C will be C'ed, -likely directories first. If that succeeds, it assumes that the entry -is a directory or a symlink to directory (which will be checked +If readdir returns file type information, then this is used directly to +find directories. + +Otherwise, after reading the directory, the modification time, size etc. +of the directory before and after the readdir is checked, and if they +match (and isn't the current time), the link count will be used to decide +how many entries are directories (if >= 2). Otherwise, no knowledge of the +number of subdirectories will be assumed. + +Then entries will be sorted into likely directories a non-initial dot +currently) and likely non-directories (see C). Then every +entry plus an appended C will be C'ed, likely directories first, +in order of their inode numbers. If that succeeds, it assumes that the +entry is a directory or a symlink to directory (which will be checked seperately). This is often faster than stat'ing the entry itself because filesystems might detect the type of the entry without reading the inode -data (e.g. ext2fs filetype feature). +data (e.g. ext2fs filetype feature), even on systems that cannot return +the filetype information on readdir. If the known number of directories (link count - 2) has been reached, the rest of the entries is assumed to be non-directories. @@ -743,7 +826,7 @@ # read the directory entries aioreq_pri $pri; - add $grp aio_readdir $path, sub { + add $grp aio_readdirx $path, READDIR_DIRS_FIRST, sub { my $entries = shift or return $grp->result (); @@ -759,18 +842,11 @@ $ndirs = -1; } else { # if nlink == 2, we are finished - # on non-posix-fs's, we rely on nlink < 2 + # for non-posix-fs's, we rely on nlink < 2 $ndirs = (stat _)[3] - 2 or return $grp->result ([], $entries); } - # sort into likely dirs and likely nondirs - # dirs == files without ".", short entries first - $entries = [map $_->[0], - sort { $b->[1] cmp $a->[1] } - map [$_, sprintf "%s%04d", (/.\./ ? "1" : "0"), length], - @$entries]; - my (@dirs, @nondirs); my $statgrp = add $grp aio_group sub { @@ -780,7 +856,7 @@ limit $statgrp $maxreq; feed $statgrp sub { return unless @$entries; - my $entry = pop @$entries; + my $entry = shift @$entries; aioreq_pri $pri; add $statgrp aio_stat "$path/$entry/.", sub { @@ -971,8 +1047,9 @@ Cancels the request, if possible. Has the effect of skipping execution when entering the B state and skipping calling the callback when entering the the B state, but will leave the request otherwise -untouched. That means that requests that currently execute will not be -stopped and resources held by the request will not be freed prematurely. +untouched (with the exception of readdir). That means that requests that +currently execute will not be stopped and resources held by the request +will not be freed prematurely. =item cb $req $callback->(...) @@ -1133,9 +1210,9 @@ =item $fileno = IO::AIO::poll_fileno Return the I. This filehandle must be -polled for reading by some mechanism outside this module (e.g. Event or -select, see below or the SYNOPSIS). If the pipe becomes readable you have -to call C to check the results. +polled for reading by some mechanism outside this module (e.g. EV, Glib, +select and so on, see below or the SYNOPSIS). If the pipe becomes readable +you have to call C to check the results. See C for an example. @@ -1152,7 +1229,8 @@ do anything special to have it called later. Example: Install an Event watcher that automatically calls -IO::AIO::poll_cb with high priority: +IO::AIO::poll_cb with high priority (more examples can be found in the +SYNOPSIS section, at the top of this document): Event->io (fd => IO::AIO::poll_fileno, poll => 'r', async => 1, @@ -1319,6 +1397,35 @@ =back +=head3 MISCELLANEOUS FUNCTIONS + +IO::AIO implements some functions that might be useful, but are not +asynchronous. + +=over 4 + +=item IO::AIO::sendfile $ofh, $ifh, $offset, $count + +Calls the C function, which is like C, +but is blocking (this makes most sense if you know the input data is +likely cached already and the output filehandle is set to non-blocking +operations). + +Returns the number of bytes copied, or C<-1> on error. + +=item IO::AIO::fadvise $fh, $offset, $len, $advice + +Simply calls the C function (see it's +manpage for details). The following advice constants are +avaiable: C, C, +C, C, +C, C. + +On systems that do not implement C, this function returns +ENOSYS, otherwise the return value of C. + +=back + =cut min_parallel 8;