--- IO-AIO/AIO.pm 2008/10/22 16:30:49 1.141 +++ IO-AIO/AIO.pm 2011/09/26 20:19:08 1.208 @@ -6,7 +6,7 @@ use IO::AIO; - aio_open "/etc/passwd", O_RDONLY, 0, sub { + aio_open "/etc/passwd", IO::AIO::O_RDONLY, 0, sub { my $fh = shift or die "/etc/passwd: $!"; ... @@ -28,33 +28,11 @@ my $grp = aio_group sub { print "all stats done\n" }; add $grp aio_stat "..." for ...; - # AnyEvent integration (EV, Event, Glib, Tk, POE, urxvt, pureperl...) - use AnyEvent::AIO; - - # EV integration - my $w = EV::io IO::AIO::poll_fileno, EV::READ, \&IO::AIO::poll_cb; - - # Event integration - Event->io (fd => IO::AIO::poll_fileno, - poll => 'r', - cb => \&IO::AIO::poll_cb); - - # Glib/Gtk2 integration - add_watch Glib::IO IO::AIO::poll_fileno, - in => sub { IO::AIO::poll_cb; 1 }; - - # Tk integration - Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "", - readable => \&IO::AIO::poll_cb); - - # Danga::Socket integration - Danga::Socket->AddOtherFds (IO::AIO::poll_fileno => - \&IO::AIO::poll_cb); - =head1 DESCRIPTION This module implements asynchronous I/O using whatever means your -operating system supports. +operating system supports. It is implemented as an interface to C +(L). Asynchronous means that operations that can normally block your program (e.g. reading from disk) will be done asynchronously: the operation @@ -68,8 +46,8 @@ While most of this works on all types of file descriptors (for example sockets), using these functions on file descriptors that -support nonblocking operation (again, sockets, pipes etc.) is very -inefficient. Use an event loop for that (such as the L +support nonblocking operation (again, sockets, pipes etc.) is +very inefficient. Use an event loop for that (such as the L module): IO::AIO will naturally fit into such an event loop itself. In this version, a number of threads are started that execute your @@ -89,20 +67,18 @@ =head2 EXAMPLE -This is a simple example that uses the Event module and loads +This is a simple example that uses the EV module and loads F asynchronously: use Fcntl; - use Event; + use EV; use IO::AIO; - # register the IO::AIO callback with Event - Event->io (fd => IO::AIO::poll_fileno, - poll => 'r', - cb => \&IO::AIO::poll_cb); + # register the IO::AIO callback with EV + my $aio_w = EV::io IO::AIO::poll_fileno, EV::READ, \&IO::AIO::poll_cb; # queue the request to open /etc/passwd - aio_open "/etc/passwd", O_RDONLY, 0, sub { + aio_open "/etc/passwd", IO::AIO::O_RDONLY, 0, sub { my $fh = shift or die "error while opening: $!"; @@ -121,7 +97,7 @@ print $contents; # exit event loop and program - Event::unloop; + EV::unloop; }; }; @@ -129,7 +105,7 @@ # check for sockets etc. etc. # process events as long as there are some: - Event::loop; + EV::loop; =head1 REQUEST ANATOMY AND LIFETIME @@ -189,27 +165,34 @@ use Carp (); -no warnings; -use strict 'vars'; +use common::sense; use base 'Exporter'; BEGIN { - our $VERSION = '3.16'; + our $VERSION = '4.0'; our @AIO_REQ = qw(aio_sendfile aio_read aio_write aio_open aio_close - aio_stat aio_lstat aio_unlink aio_rmdir aio_readdir - aio_scandir aio_symlink aio_readlink aio_sync aio_fsync - aio_fdatasync aio_pathsync aio_readahead + aio_stat aio_lstat aio_unlink aio_rmdir aio_readdir aio_readdirx + aio_scandir aio_symlink aio_readlink aio_realpath aio_sync + aio_fsync aio_syncfs aio_fdatasync aio_sync_file_range aio_fallocate + aio_pathsync aio_readahead aio_rename aio_link aio_move aio_copy aio_group aio_nop aio_mknod aio_load aio_rmtree aio_mkdir aio_chown - aio_chmod aio_utime aio_truncate); + aio_chmod aio_utime aio_truncate + aio_msync aio_mtouch aio_mlock aio_mlockall + aio_statvfs + aio_wd); our @EXPORT = (@AIO_REQ, qw(aioreq_pri aioreq_nice)); our @EXPORT_OK = qw(poll_fileno poll_cb poll_wait flush - min_parallel max_parallel max_idle + min_parallel max_parallel max_idle idle_timeout nreqs nready npending nthreads - max_poll_time max_poll_reqs); + max_poll_time max_poll_reqs + sendfile fadvise madvise + mmap munmap munlock munlockall); + + push @AIO_REQ, qw(aio_busy); # not exported @IO::AIO::GRP::ISA = 'IO::AIO::REQ'; @@ -219,6 +202,82 @@ =head1 FUNCTIONS +=head2 QUICK OVERVIEW + +This section simply lists the prototypes of the most important functions +for quick reference. See the following sections for function-by-function +documentation. + + aio_wd $pathname, $callback->($wd) + aio_open $pathname, $flags, $mode, $callback->($fh) + aio_close $fh, $callback->($status) + aio_read $fh,$offset,$length, $data,$dataoffset, $callback->($retval) + aio_write $fh,$offset,$length, $data,$dataoffset, $callback->($retval) + aio_sendfile $out_fh, $in_fh, $in_offset, $length, $callback->($retval) + aio_readahead $fh,$offset,$length, $callback->($retval) + aio_stat $fh_or_path, $callback->($status) + aio_lstat $fh, $callback->($status) + aio_statvfs $fh_or_path, $callback->($statvfs) + aio_utime $fh_or_path, $atime, $mtime, $callback->($status) + aio_chown $fh_or_path, $uid, $gid, $callback->($status) + aio_truncate $fh_or_path, $offset, $callback->($status) + aio_chmod $fh_or_path, $mode, $callback->($status) + aio_unlink $pathname, $callback->($status) + aio_mknod $path, $mode, $dev, $callback->($status) + aio_link $srcpath, $dstpath, $callback->($status) + aio_symlink $srcpath, $dstpath, $callback->($status) + aio_readlink $path, $callback->($link) + aio_realpath $path, $callback->($link) + aio_rename $srcpath, $dstpath, $callback->($status) + aio_mkdir $pathname, $mode, $callback->($status) + aio_rmdir $pathname, $callback->($status) + aio_readdir $pathname, $callback->($entries) + aio_readdirx $pathname, $flags, $callback->($entries, $flags) + IO::AIO::READDIR_DENTS IO::AIO::READDIR_DIRS_FIRST + IO::AIO::READDIR_STAT_ORDER IO::AIO::READDIR_FOUND_UNKNOWN + aio_load $path, $data, $callback->($status) + aio_copy $srcpath, $dstpath, $callback->($status) + aio_move $srcpath, $dstpath, $callback->($status) + aio_scandir $path, $maxreq, $callback->($dirs, $nondirs) + aio_rmtree $path, $callback->($status) + aio_sync $callback->($status) + aio_syncfs $fh, $callback->($status) + aio_fsync $fh, $callback->($status) + aio_fdatasync $fh, $callback->($status) + aio_sync_file_range $fh, $offset, $nbytes, $flags, $callback->($status) + aio_pathsync $path, $callback->($status) + aio_msync $scalar, $offset = 0, $length = undef, flags = 0, $callback->($status) + aio_mtouch $scalar, $offset = 0, $length = undef, flags = 0, $callback->($status) + aio_mlock $scalar, $offset = 0, $length = undef, $callback->($status) + aio_mlockall $flags, $callback->($status) + aio_group $callback->(...) + aio_nop $callback->() + + $prev_pri = aioreq_pri [$pri] + aioreq_nice $pri_adjust + + IO::AIO::poll_wait + IO::AIO::poll_cb + IO::AIO::poll + IO::AIO::flush + IO::AIO::max_poll_reqs $nreqs + IO::AIO::max_poll_time $seconds + IO::AIO::min_parallel $nthreads + IO::AIO::max_parallel $nthreads + IO::AIO::max_idle $nthreads + IO::AIO::idle_timeout $seconds + IO::AIO::max_outstanding $maxreqs + IO::AIO::nreqs + IO::AIO::nready + IO::AIO::npending + + IO::AIO::sendfile $ofh, $ifh, $offset, $count + IO::AIO::fadvise $fh, $offset, $len, $advice + IO::AIO::madvise $scalar, $offset, $length, $advice + IO::AIO::mprotect $scalar, $offset, $length, $protect + IO::AIO::munlock $scalar, $offset = 0, $length = undef + IO::AIO::munlockall + =head2 AIO REQUEST FUNCTIONS All the C calls are more or less thin wrappers around the syscall @@ -307,7 +366,7 @@ Example: - aio_open "/etc/passwd", O_RDONLY, 0, sub { + aio_open "/etc/passwd", IO::AIO::O_RDONLY, 0, sub { if ($_[0]) { print "open successful, fh is $_[0]\n"; ... @@ -316,6 +375,15 @@ } }; +In addition to all the common open modes/flags (C, C, +C, C, C, C and C), the +following POSIX and non-POSIX constants are available (missing ones on +your system are, as usual, C<0>): + +C, C, C, C, C, C, +C, C, C, C, C, +C, C and C. + =item aio_close $fh, $callback->($status) @@ -338,16 +406,20 @@ =item aio_write $fh,$offset,$length, $data,$dataoffset, $callback->($retval) -Reads or writes C<$length> bytes from the specified C<$fh> and C<$offset> -into the scalar given by C<$data> and offset C<$dataoffset> and calls the -callback without the actual number of bytes read (or -1 on error, just -like the syscall). +Reads or writes C<$length> bytes from or to the specified C<$fh> and +C<$offset> into the scalar given by C<$data> and offset C<$dataoffset> +and calls the callback without the actual number of bytes read (or -1 on +error, just like the syscall). + +C will, like C, shrink or grow the C<$data> scalar to +offset plus the actual number of bytes read. If C<$offset> is undefined, then the current file descriptor offset will be used (and updated), otherwise the file descriptor offset will not be changed by these calls. -If C<$length> is undefined in C, use the remaining length of C<$data>. +If C<$length> is undefined in C, use the remaining length of +C<$data>. If C<$dataoffset> is less than zero, it will be counted from the end of C<$data>. @@ -371,22 +443,40 @@ reading at byte offset C<$in_offset>, and starts writing at the current file offset of C<$out_fh>. Because of that, it is not safe to issue more than one C per C<$out_fh>, as they will interfere with each -other. +other. The same C<$in_fh> works fine though, as this function does not +move or use the file offset of C<$in_fh>. -This call tries to make use of a native C syscall to provide -zero-copy operation. For this to work, C<$out_fh> should refer to a -socket, and C<$in_fh> should refer to mmap'able file. - -If the native sendfile call fails or is not implemented, it will be -emulated, so you can call C on any type of filehandle -regardless of the limitations of the operating system. - -Please note, however, that C can read more bytes from -C<$in_fh> than are written, and there is no way to find out how many -bytes have been read from C alone, as C only -provides the number of bytes written to C<$out_fh>. Only if the result -value equals C<$length> one can assume that C<$length> bytes have been -read. +Please note that C can read more bytes from C<$in_fh> than +are written, and there is no way to find out how many more bytes have been +read from C alone, as C only provides the +number of bytes written to C<$out_fh>. Only if the result value equals +C<$length> one can assume that C<$length> bytes have been read. + +Unlike with other C functions, it makes a lot of sense to use +C on non-blocking sockets, as long as one end (typically +the C<$in_fh>) is a file - the file I/O will then be asynchronous, while +the socket I/O will be non-blocking. Note, however, that you can run +into a trap where C reads some data with readahead, then +fails to write all data, and when the socket is ready the next time, the +data in the cache is already lost, forcing C to again hit +the disk. Explicit C + C let's you better control +resource usage. + +This call tries to make use of a native C-like syscall to +provide zero-copy operation. For this to work, C<$out_fh> should refer to +a socket, and C<$in_fh> should refer to an mmap'able file. + +If a native sendfile cannot be found or it fails with C, +C, C, C, C, C or +C, it will be emulated, so you can call C on any +type of filehandle regardless of the limitations of the operating system. + +As native sendfile syscalls (as practically any non-POSIX interface hacked +together in a hurry to improve benchmark numbers) tend to be rather buggy +on many systems, this implementation tries to work around some known bugs +in Linux and FreeBSD kernels (probably others, too), but that might fail, +so you really really should check the return value of C - +fewre bytes than expected might have been transferred. =item aio_readahead $fh,$offset,$length, $callback->($retval) @@ -419,6 +509,15 @@ error when stat'ing a large file, the results will be silently truncated unless perl itself is compiled with large file support. +To help interpret the mode and dev/rdev stat values, IO::AIO offers the +following constants and functions (if not implemented, the constants will +be C<0> and the functions will either C or fall back on traditional +behaviour). + +C, C, C, C, C, C, +C, C, C, C, +C, C. + Example: Print the length of F: aio_stat "/etc/passwd", sub { @@ -427,6 +526,51 @@ }; +=item aio_statvfs $fh_or_path, $callback->($statvfs) + +Works like the POSIX C or C syscalls, depending on +whether a file handle or path was passed. + +On success, the callback is passed a hash reference with the following +members: C, C, C, C, C, C, +C, C, C, C and C. On failure, C +is passed. + +The following POSIX IO::AIO::ST_* constants are defined: C and +C. + +The following non-POSIX IO::AIO::ST_* flag masks are defined to +their correct value when available, or to C<0> on systems that do +not support them: C, C, C, +C, C, C, C, C, +C and C. + +Example: stat C and dump out the data if successful. + + aio_statvfs "/wd", sub { + my $f = $_[0] + or die "statvfs: $!"; + + use Data::Dumper; + say Dumper $f; + }; + + # result: + { + bsize => 1024, + bfree => 4333064312, + blocks => 10253828096, + files => 2050765568, + flag => 4096, + favail => 2042092649, + bavail => 4333064312, + ffree => 2042092649, + namemax => 255, + frsize => 1024, + fsid => 1810 + } + + =item aio_utime $fh_or_path, $atime, $mtime, $callback->($status) Works like perl's C function (including the special case of $atime @@ -484,6 +628,8 @@ aio_mknod $path, IO::AIO::S_IFIFO | $mode, 0, sub { ... +See C for info about some potentially helpful extra constants +and functions. =item aio_link $srcpath, $dstpath, $callback->($status) @@ -504,6 +650,16 @@ callback. +=item aio_realpath $path, $callback->($path) + +Asynchronously make the path absolute and resolve any symlinks in +C<$path>. The resulting path only consists of directories (Same as +L). + +This request can be used to get the absolute path of the current working +directory by passing it a path of F<.> (a single dot). + + =item aio_rename $srcpath, $dstpath, $callback->($status) Asynchronously rename the object at C<$srcpath> to C<$dstpath>, just as @@ -529,8 +685,74 @@ directory (i.e. opendir + readdir + closedir). The entries will not be sorted, and will B include the C<.> and C<..> entries. -The callback a single argument which is either C or an array-ref -with the filenames. +The callback is passed a single argument which is either C or an +array-ref with the filenames. + + +=item aio_readdirx $pathname, $flags, $callback->($entries, $flags) + +Quite similar to C, but the C<$flags> argument allows one to +tune behaviour and output format. In case of an error, C<$entries> will be +C. + +The flags are a combination of the following constants, ORed together (the +flags will also be passed to the callback, possibly modified): + +=over 4 + +=item IO::AIO::READDIR_DENTS + +When this flag is off, then the callback gets an arrayref consisting of +names only (as with C), otherwise it gets an arrayref with +C<[$name, $type, $inode]> arrayrefs, each describing a single directory +entry in more detail. + +C<$name> is the name of the entry. + +C<$type> is one of the C constants: + +C, C, C, C, +C, C, C, C, +C. + +C means just that: readdir does not know. If you need to +know, you have to run stat yourself. Also, for speed reasons, the C<$type> +scalars are read-only: you can not modify them. + +C<$inode> is the inode number (which might not be exact on systems with 64 +bit inode numbers and 32 bit perls). This field has unspecified content on +systems that do not deliver the inode information. + +=item IO::AIO::READDIR_DIRS_FIRST + +When this flag is set, then the names will be returned in an order where +likely directories come first, in optimal stat order. This is useful when +you need to quickly find directories, or you want to find all directories +while avoiding to stat() each entry. + +If the system returns type information in readdir, then this is used +to find directories directly. Otherwise, likely directories are names +beginning with ".", or otherwise names with no dots, of which names with +short names are tried first. + +=item IO::AIO::READDIR_STAT_ORDER + +When this flag is set, then the names will be returned in an order +suitable for stat()'ing each one. That is, when you plan to stat() +all files in the given directory, then the returned order will likely +be fastest. + +If both this flag and C are specified, then +the likely dirs come first, resulting in a less optimal stat order. + +=item IO::AIO::READDIR_FOUND_UNKNOWN + +This flag should not be set when calling C. Instead, it +is being set by C, when any of the C<$type>'s found were +C. The absence of this flag therefore indicates that all +C<$type>'s are known, which can be used to speed up some algorithms. + +=back =item aio_load $path, $data, $callback->($status) @@ -565,7 +787,7 @@ Try to copy the I (directories not supported as either source or destination) from C<$srcpath> to C<$dstpath> and call the callback with -the C<0> (error) or C<-1> ok. +a status of C<0> (ok) or C<-1> (error, see C<$!>). This is a composite request that creates the destination file with mode 0200 and copies the contents of the source file into it using @@ -587,7 +809,7 @@ aioreq_pri $pri; add $grp aio_open $src, O_RDONLY, 0, sub { if (my $src_fh = $_[0]) { - my @stat = stat $src_fh; + my @stat = stat $src_fh; # hmm, might block over nfs? aioreq_pri $pri; add $grp aio_open $dst, O_CREAT | O_WRONLY | O_TRUNC, 0200, sub { @@ -598,13 +820,26 @@ $grp->result (0); close $src_fh; - # those should not normally block. should. should. - utime $stat[8], $stat[9], $dst; - chmod $stat[2] & 07777, $dst_fh; - chown $stat[4], $stat[5], $dst_fh; + my $ch = sub { + aioreq_pri $pri; + add $grp aio_chmod $dst_fh, $stat[2] & 07777, sub { + aioreq_pri $pri; + add $grp aio_chown $dst_fh, $stat[4], $stat[5], sub { + aioreq_pri $pri; + add $grp aio_close $dst_fh; + } + }; + }; aioreq_pri $pri; - add $grp aio_close $dst_fh; + add $grp aio_utime $dst_fh, $stat[8], $stat[9], sub { + if ($_[0] < 0 && $! == ENOSYS) { + aioreq_pri $pri; + add $grp aio_utime $dst, $stat[8], $stat[9], $ch; + } else { + $ch->(); + } + }; } else { $grp->result (-1); close $src_fh; @@ -631,7 +866,7 @@ Try to move the I (directories not supported as either source or destination) from C<$srcpath> to C<$dstpath> and call the callback with -the C<0> (error) or C<-1> ok. +a status of C<0> (ok) or C<-1> (error, see C<$!>). This is a composite request that tries to rename(2) the file first; if rename fails with C, it copies the file with C and, if @@ -652,7 +887,7 @@ add $grp aio_copy $src, $dst, sub { $grp->result ($_[0]); - if (!$_[0]) { + unless ($_[0]) { aioreq_pri $pri; add $grp aio_unlink $src; } @@ -692,20 +927,24 @@ The C cannot be avoided, but C'ing every entry can. -After reading the directory, the modification time, size etc. of the -directory before and after the readdir is checked, and if they match (and -isn't the current time), the link count will be used to decide how many -entries are directories (if >= 2). Otherwise, no knowledge of the number -of subdirectories will be assumed. - -Then entries will be sorted into likely directories (everything without -a non-initial dot currently) and likely non-directories (everything -else). Then every entry plus an appended C will be C'ed, -likely directories first. If that succeeds, it assumes that the entry -is a directory or a symlink to directory (which will be checked -seperately). This is often faster than stat'ing the entry itself because +If readdir returns file type information, then this is used directly to +find directories. + +Otherwise, after reading the directory, the modification time, size etc. +of the directory before and after the readdir is checked, and if they +match (and isn't the current time), the link count will be used to decide +how many entries are directories (if >= 2). Otherwise, no knowledge of the +number of subdirectories will be assumed. + +Then entries will be sorted into likely directories a non-initial dot +currently) and likely non-directories (see C). Then every +entry plus an appended C will be C'ed, likely directories first, +in order of their inode numbers. If that succeeds, it assumes that the +entry is a directory or a symlink to directory (which will be checked +separately). This is often faster than stat'ing the entry itself because filesystems might detect the type of the entry without reading the inode -data (e.g. ext2fs filetype feature). +data (e.g. ext2fs filetype feature), even on systems that cannot return +the filetype information on readdir. If the known number of directories (link count - 2) has been reached, the rest of the entries is assumed to be non-directories. @@ -737,7 +976,7 @@ # read the directory entries aioreq_pri $pri; - add $grp aio_readdir $path, sub { + add $grp aio_readdirx $path, READDIR_DIRS_FIRST, sub { my $entries = shift or return $grp->result (); @@ -753,18 +992,11 @@ $ndirs = -1; } else { # if nlink == 2, we are finished - # on non-posix-fs's, we rely on nlink < 2 + # for non-posix-fs's, we rely on nlink < 2 $ndirs = (stat _)[3] - 2 or return $grp->result ([], $entries); } - # sort into likely dirs and likely nondirs - # dirs == files without ".", short entries first - $entries = [map $_->[0], - sort { $b->[1] cmp $a->[1] } - map [$_, sprintf "%s%04d", (/.\./ ? "1" : "0"), length], - @$entries]; - my (@dirs, @nondirs); my $statgrp = add $grp aio_group sub { @@ -774,7 +1006,7 @@ limit $statgrp $maxreq; feed $statgrp sub { return unless @$entries; - my $entry = pop @$entries; + my $entry = shift @$entries; aioreq_pri $pri; add $statgrp aio_stat "$path/$entry/.", sub { @@ -857,6 +1089,25 @@ If this call isn't available because your OS lacks it or it couldn't be detected, it will be emulated by calling C instead. +=item aio_syncfs $fh, $callback->($status) + +Asynchronously call the syncfs syscall to sync the filesystem associated +to the given filehandle and call the callback with the syncfs result +code. If syncfs is not available, calls sync(), but returns C<-1> and sets +errno to C nevertheless. + +=item aio_sync_file_range $fh, $offset, $nbytes, $flags, $callback->($status) + +Sync the data portion of the file specified by C<$offset> and C<$length> +to disk (but NOT the metadata), by calling the Linux-specific +sync_file_range call. If sync_file_range is not available or it returns +ENOSYS, then fdatasync or fsync is being substituted. + +C<$flags> can be a combination of C, +C and +C: refer to the sync_file_range +manpage for details. + =item aio_pathsync $path, $callback->($status) This request tries to open, fsync and close the given path. This is a @@ -866,6 +1117,9 @@ written to disc. It works for anything that can be opened for read-only, not just directories. +Future versions of this function might fall back to other methods when +C on the directory fails (such as calling C). + Passes C<0> when everything went ok, and C<-1> on error. =cut @@ -895,6 +1149,72 @@ $grp } +=item aio_msync $scalar, $offset = 0, $length = undef, flags = 0, $callback->($status) + +This is a rather advanced IO::AIO call, which only works on mmap(2)ed +scalars (see the C function, although it also works on data +scalars managed by the L or L modules, note that the +scalar must only be modified in-place while an aio operation is pending on +it). + +It calls the C function of your OS, if available, with the memory +area starting at C<$offset> in the string and ending C<$length> bytes +later. If C<$length> is negative, counts from the end, and if C<$length> +is C, then it goes till the end of the string. The flags can be +a combination of C, C and +C. + +=item aio_mtouch $scalar, $offset = 0, $length = undef, flags = 0, $callback->($status) + +This is a rather advanced IO::AIO call, which works best on mmap(2)ed +scalars. + +It touches (reads or writes) all memory pages in the specified +range inside the scalar. All caveats and parameters are the same +as for C, above, except for flags, which must be either +C<0> (which reads all pages and ensures they are instantiated) or +C, which modifies the memory page s(by reading and +writing an octet from it, which dirties the page). + +=item aio_mlock $scalar, $offset = 0, $length = undef, $callback->($status) + +This is a rather advanced IO::AIO call, which works best on mmap(2)ed +scalars. + +It reads in all the pages of the underlying storage into memory (if any) +and locks them, so they are not getting swapped/paged out or removed. + +If C<$length> is undefined, then the scalar will be locked till the end. + +On systems that do not implement C, this function returns C<-1> +and sets errno to C. + +Note that the corresponding C is synchronous and is +documented under L. + +Example: open a file, mmap and mlock it - both will be undone when +C<$data> gets destroyed. + + open my $fh, "<", $path or die "$path: $!"; + my $data; + IO::AIO::mmap $data, -s $fh, IO::AIO::PROT_READ, IO::AIO::MAP_SHARED, $fh; + aio_mlock $data; # mlock in background + +=item aio_mlockall $flags, $callback->($status) + +Calls the C function with the given C<$flags> (a combination of +C and C). + +On systems that do not implement C, this function returns C<-1> +and sets errno to C. + +Note that the corresponding C is synchronous and is +documented under L. + +Example: asynchronously lock all current and future pages into memory. + + aio_mlockall IO::AIO::MCL_FUTURE; + =item aio_group $callback->(...) This is a very special aio request: Instead of doing something, it is a @@ -953,8 +1273,9 @@ Cancels the request, if possible. Has the effect of skipping execution when entering the B state and skipping calling the callback when entering the the B state, but will leave the request otherwise -untouched. That means that requests that currently execute will not be -stopped and resources held by the request will not be freed prematurely. +untouched (with the exception of readdir). That means that requests that +currently execute will not be stopped and resources held by the request +will not be freed prematurely. =item cb $req $callback->(...) @@ -1038,6 +1359,9 @@ Cancel all subrequests and clears any feeder, but not the group request itself. Useful when you queued a lot of events but got a result early. +The group request will finish normally (you cannot add requests to the +group). + =item $grp->result (...) Set the result value(s) that will be passed to the group callback when all @@ -1115,31 +1439,67 @@ =item $fileno = IO::AIO::poll_fileno Return the I. This filehandle must be -polled for reading by some mechanism outside this module (e.g. Event or -select, see below or the SYNOPSIS). If the pipe becomes readable you have -to call C to check the results. +polled for reading by some mechanism outside this module (e.g. EV, Glib, +select and so on, see below or the SYNOPSIS). If the pipe becomes readable +you have to call C to check the results. See C for an example. =item IO::AIO::poll_cb -Process some outstanding events on the result pipe. You have to call this -regularly. Returns C<0> if all events could be processed, or C<-1> if it -returned earlier for whatever reason. Returns immediately when no events -are outstanding. The amount of events processed depends on the settings of -C and C. +Process some outstanding events on the result pipe. You have to call +this regularly. Returns C<0> if all events could be processed (or there +were no events to process), or C<-1> if it returned earlier for whatever +reason. Returns immediately when no events are outstanding. The amount of +events processed depends on the settings of C and +C. If not all requests were processed for whatever reason, the filehandle will still be ready when C returns, so normally you don't have to do anything special to have it called later. +Apart from calling C when the event filehandle becomes +ready, it can be beneficial to call this function from loops which submit +a lot of requests, to make sure the results get processed when they become +available and not just when the loop is finished and the event loop takes +over again. This function returns very fast when there are no outstanding +requests. + Example: Install an Event watcher that automatically calls -IO::AIO::poll_cb with high priority: +IO::AIO::poll_cb with high priority (more examples can be found in the +SYNOPSIS section, at the top of this document): Event->io (fd => IO::AIO::poll_fileno, poll => 'r', async => 1, cb => \&IO::AIO::poll_cb); +=item IO::AIO::poll_wait + +If there are any outstanding requests and none of them in the result +phase, wait till the result filehandle becomes ready for reading (simply +does a C on the filehandle. This is useful if you want to -synchronously wait for some requests to finish). - -See C for an example. - -=item IO::AIO::poll - -Waits until some requests have been handled. - -Returns the number of requests processed, but is otherwise strictly -equivalent to: - - IO::AIO::poll_wait, IO::AIO::poll_cb - -=item IO::AIO::flush - -Wait till all outstanding AIO requests have been handled. - -Strictly equivalent to: - - IO::AIO::poll_wait, IO::AIO::poll_cb - while IO::AIO::nreqs; - =back =head3 CONTROLLING THE NUMBER OF THREADS @@ -1242,10 +1575,11 @@ =item IO::AIO::max_idle $nthreads -Limit the number of threads (default: 4) that are allowed to idle (i.e., -threads that did not get a request to process within 10 seconds). That -means if a thread becomes idle while C<$nthreads> other threads are also -idle, it will free its resources and exit. +Limit the number of threads (default: 4) that are allowed to idle +(i.e., threads that did not get a request to process within the idle +timeout (default: 10 seconds). That means if a thread becomes idle while +C<$nthreads> other threads are also idle, it will free its resources and +exit. This is useful when you allow a large number of threads (e.g. 100 or 1000) to allow for extremely high load situations, but want to free resources @@ -1255,23 +1589,45 @@ creation is fast. If thread creation is very slow on your system you might want to use larger values. +=item IO::AIO::idle_timeout $seconds + +Sets the minimum idle timeout (default 10) after which worker threads are +allowed to exit. SEe C. + =item IO::AIO::max_outstanding $maxreqs +Sets the maximum number of outstanding requests to C<$nreqs>. If +you do queue up more than this number of requests, the next call to +C (and other functions calling C, such as +C or C) will block until the limit is no +longer exceeded. + +In other words, this setting does not enforce a queue limit, but can be +used to make poll functions block if the limit is exceeded. + This is a very bad function to use in interactive programs because it blocks, and a bad way to reduce concurrency because it is inexact: Better use an C together with a feed callback. -Sets the maximum number of outstanding requests to C<$nreqs>. If you -do queue up more than this number of requests, the next call to the -C (and C and other functions calling C) -function will block until the limit is no longer exceeded. - -The default value is very large, so there is no practical limit on the -number of outstanding requests. - -You can still queue as many requests as you want. Therefore, -C is mainly useful in simple scripts (with low values) or -as a stop gap to shield against fatal memory overflow (with large values). +It's main use is in scripts without an event loop - when you want to stat +a lot of files, you can write somehting like this: + + IO::AIO::max_outstanding 32; + + for my $path (...) { + aio_stat $path , ...; + IO::AIO::poll_cb; + } + + IO::AIO::flush; + +The call to C inside the loop will normally return instantly, but +as soon as more thna C<32> reqeusts are in-flight, it will block until +some requests have been handled. This keeps the loop from pushing a large +number of C requests onto the queue. + +The default value for C is very large, so there is no +practical limit on the number of outstanding requests. =back @@ -1301,6 +1657,121 @@ =back +=head3 MISCELLANEOUS FUNCTIONS + +IO::AIO implements some functions that might be useful, but are not +asynchronous. + +=over 4 + +=item IO::AIO::sendfile $ofh, $ifh, $offset, $count + +Calls the C function, which is like C, +but is blocking (this makes most sense if you know the input data is +likely cached already and the output filehandle is set to non-blocking +operations). + +Returns the number of bytes copied, or C<-1> on error. + +=item IO::AIO::fadvise $fh, $offset, $len, $advice + +Simply calls the C function (see its +manpage for details). The following advice constants are +available: C, C, +C, C, +C, C. + +On systems that do not implement C, this function returns +ENOSYS, otherwise the return value of C. + +=item IO::AIO::madvise $scalar, $offset, $len, $advice + +Simply calls the C function (see its +manpage for details). The following advice constants are +available: C, C, +C, C, C. + +On systems that do not implement C, this function returns +ENOSYS, otherwise the return value of C. + +=item IO::AIO::mprotect $scalar, $offset, $len, $protect + +Simply calls the C function on the preferably AIO::mmap'ed +$scalar (see its manpage for details). The following protect +constants are available: C, C, +C, C. + +On systems that do not implement C, this function returns +ENOSYS, otherwise the return value of C. + +=item IO::AIO::mmap $scalar, $length, $prot, $flags, $fh[, $offset] + +Memory-maps a file (or anonymous memory range) and attaches it to the +given C<$scalar>, which will act like a string scalar. + +The only operations allowed on the scalar are C/C that don't +change the string length, and most read-only operations such as copying it +or searching it with regexes and so on. + +Anything else is unsafe and will, at best, result in memory leaks. + +The memory map associated with the C<$scalar> is automatically removed +when the C<$scalar> is destroyed, or when the C or +C functions are called. + +This calls the C(2) function internally. See your system's manual +page for details on the C<$length>, C<$prot> and C<$flags> parameters. + +The C<$length> must be larger than zero and smaller than the actual +filesize. + +C<$prot> is a combination of C, C, +C and/or C, + +C<$flags> can be a combination of C or +C, or a number of system-specific flags (when +not available, the are defined as 0): C +(which is set to C if your system only provides this +constant), C, C, +C, C or +C + +If C<$fh> is C, then a file descriptor of C<-1> is passed. + +C<$offset> is the offset from the start of the file - it generally must be +a multiple of C and defaults to C<0>. + +Example: + + use Digest::MD5; + use IO::AIO; + + open my $fh, ". + +=item IO::AIO::munlock $scalar, $offset = 0, $length = undef + +Calls the C function, undoing the effects of a previous +C call (see its description for details). + +=item IO::AIO::munlockall + +Calls the C function. + +On systems that do not implement C, this function returns +ENOSYS, otherwise the return value of C. + +=back + =cut min_parallel 8; @@ -1309,21 +1780,69 @@ 1; +=head1 EVENT LOOP INTEGRATION + +It is recommended to use L to integrate IO::AIO +automatically into many event loops: + + # AnyEvent integration (EV, Event, Glib, Tk, POE, urxvt, pureperl...) + use AnyEvent::AIO; + +You can also integrate IO::AIO manually into many event loops, here are +some examples of how to do this: + + # EV integration + my $aio_w = EV::io IO::AIO::poll_fileno, EV::READ, \&IO::AIO::poll_cb; + + # Event integration + Event->io (fd => IO::AIO::poll_fileno, + poll => 'r', + cb => \&IO::AIO::poll_cb); + + # Glib/Gtk2 integration + add_watch Glib::IO IO::AIO::poll_fileno, + in => sub { IO::AIO::poll_cb; 1 }; + + # Tk integration + Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "", + readable => \&IO::AIO::poll_cb); + + # Danga::Socket integration + Danga::Socket->AddOtherFds (IO::AIO::poll_fileno => + \&IO::AIO::poll_cb); + =head2 FORK BEHAVIOUR -This module should do "the right thing" when the process using it forks: +Usage of pthreads in a program changes the semantics of fork +considerably. Specifically, only async-safe functions can be called after +fork. Perl doesn't know about this, so in general, you cannot call fork +with defined behaviour in perl if pthreads are involved. IO::AIO uses +pthreads, so this applies, but many other extensions and (for inexplicable +reasons) perl itself often is linked against pthreads, so this limitation +applies to quite a lot of perls. + +This module no longer tries to fight your OS, or POSIX. That means IO::AIO +only works in the process that loaded it. Forking is fully supported, but +using IO::AIO in the child is not. + +You might get around by not I IO::AIO before (or after) +forking. You could also try to call the L function in the +child: + +=over 4 -Before the fork, IO::AIO enters a quiescent state where no requests -can be added in other threads and no results will be processed. After -the fork the parent simply leaves the quiescent state and continues -request/result processing, while the child frees the request/result queue -(so that the requests started before the fork will only be handled in the -parent). Threads will be started on demand until the limit set in the -parent process has been reached again. - -In short: the parent will, after a short pause, continue as if fork had -not been called, while the child will act as if IO::AIO has not been used -yet. +=item IO::AIO::reinit + +Abandons all current requests and I/O threads and simply reinitialises all +data structures. This is not an operation supported by any standards, but +happens to work on GNU/Linux and some newer BSD systems. + +The only reasonable use for this function is to call it after forking, if +C was used in the parent. Calling it while IO::AIO is active in +the process will result in undefined behaviour. Calling it at any time +will also result in any undefined (by POSIX) behaviour. + +=back =head2 MEMORY USAGE