--- IO-AIO/README 2009/06/07 18:31:18 1.36 +++ IO-AIO/README 2010/01/10 23:44:02 1.43 @@ -26,32 +26,10 @@ my $grp = aio_group sub { print "all stats done\n" }; add $grp aio_stat "..." for ...; - # AnyEvent integration (EV, Event, Glib, Tk, POE, urxvt, pureperl...) - use AnyEvent::AIO; - - # EV integration - my $w = EV::io IO::AIO::poll_fileno, EV::READ, \&IO::AIO::poll_cb; - - # Event integration - Event->io (fd => IO::AIO::poll_fileno, - poll => 'r', - cb => \&IO::AIO::poll_cb); - - # Glib/Gtk2 integration - add_watch Glib::IO IO::AIO::poll_fileno, - in => sub { IO::AIO::poll_cb; 1 }; - - # Tk integration - Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "", - readable => \&IO::AIO::poll_cb); - - # Danga::Socket integration - Danga::Socket->AddOtherFds (IO::AIO::poll_fileno => - \&IO::AIO::poll_cb); - DESCRIPTION This module implements asynchronous I/O using whatever means your - operating system supports. + operating system supports. It is implemented as an interface to "libeio" + (). Asynchronous means that operations that can normally block your program (e.g. reading from disk) will be done asynchronously: the operation will @@ -66,7 +44,7 @@ While most of this works on all types of file descriptors (for example sockets), using these functions on file descriptors that support nonblocking operation (again, sockets, pipes etc.) is very inefficient. - Use an event loop for that (such as the Event module): IO::AIO will + Use an event loop for that (such as the EV module): IO::AIO will naturally fit into such an event loop itself. In this version, a number of threads are started that execute your @@ -85,17 +63,15 @@ call "poll_cb" (or other "aio_" functions) recursively. EXAMPLE - This is a simple example that uses the Event module and loads - /etc/passwd asynchronously: + This is a simple example that uses the EV module and loads /etc/passwd + asynchronously: use Fcntl; - use Event; + use EV; use IO::AIO; - # register the IO::AIO callback with Event - Event->io (fd => IO::AIO::poll_fileno, - poll => 'r', - cb => \&IO::AIO::poll_cb); + # register the IO::AIO callback with EV + my $aio_w = EV::io IO::AIO::poll_fileno, EV::READ, \&IO::AIO::poll_cb; # queue the request to open /etc/passwd aio_open "/etc/passwd", O_RDONLY, 0, sub { @@ -117,7 +93,7 @@ print $contents; # exit event loop and program - Event::unloop; + EV::unloop; }; }; @@ -125,7 +101,7 @@ # check for sockets etc. etc. # process events as long as there are some: - Event::loop; + EV::loop; REQUEST ANATOMY AND LIFETIME Every "aio_*" function creates a request. which is a C data structure @@ -171,6 +147,73 @@ either do nothing or result in a runtime error). FUNCTIONS + QUICK OVERVIEW + This section simply lists the prototypes of the most important functions + for quick reference. See the following sections for function-by-function + documentation. + + aio_open $pathname, $flags, $mode, $callback->($fh) + aio_close $fh, $callback->($status) + aio_read $fh,$offset,$length, $data,$dataoffset, $callback->($retval) + aio_write $fh,$offset,$length, $data,$dataoffset, $callback->($retval) + aio_sendfile $out_fh, $in_fh, $in_offset, $length, $callback->($retval) + aio_readahead $fh,$offset,$length, $callback->($retval) + aio_stat $fh_or_path, $callback->($status) + aio_lstat $fh, $callback->($status) + aio_statvfs $fh_or_path, $callback->($statvfs) + aio_utime $fh_or_path, $atime, $mtime, $callback->($status) + aio_chown $fh_or_path, $uid, $gid, $callback->($status) + aio_truncate $fh_or_path, $offset, $callback->($status) + aio_chmod $fh_or_path, $mode, $callback->($status) + aio_unlink $pathname, $callback->($status) + aio_mknod $path, $mode, $dev, $callback->($status) + aio_link $srcpath, $dstpath, $callback->($status) + aio_symlink $srcpath, $dstpath, $callback->($status) + aio_readlink $path, $callback->($link) + aio_rename $srcpath, $dstpath, $callback->($status) + aio_mkdir $pathname, $mode, $callback->($status) + aio_rmdir $pathname, $callback->($status) + aio_readdir $pathname, $callback->($entries) + aio_readdirx $pathname, $flags, $callback->($entries, $flags) + IO::AIO::READDIR_DENTS IO::AIO::READDIR_DIRS_FIRST + IO::AIO::READDIR_STAT_ORDER IO::AIO::READDIR_FOUND_UNKNOWN + aio_load $path, $data, $callback->($status) + aio_copy $srcpath, $dstpath, $callback->($status) + aio_move $srcpath, $dstpath, $callback->($status) + aio_scandir $path, $maxreq, $callback->($dirs, $nondirs) + aio_rmtree $path, $callback->($status) + aio_sync $callback->($status) + aio_fsync $fh, $callback->($status) + aio_fdatasync $fh, $callback->($status) + aio_sync_file_range $fh, $offset, $nbytes, $flags, $callback->($status) + aio_pathsync $path, $callback->($status) + aio_msync $scalar, $offset = 0, $length = undef, flags = 0, $callback->($status) + aio_mtouch $scalar, $offset = 0, $length = undef, flags = 0, $callback->($status) + aio_group $callback->(...) + aio_nop $callback->() + + $prev_pri = aioreq_pri [$pri] + aioreq_nice $pri_adjust + + IO::AIO::poll_wait + IO::AIO::poll_cb + IO::AIO::poll + IO::AIO::flush + IO::AIO::max_poll_reqs $nreqs + IO::AIO::max_poll_time $seconds + IO::AIO::min_parallel $nthreads + IO::AIO::max_parallel $nthreads + IO::AIO::max_idle $nthreads + IO::AIO::max_outstanding $maxreqs + IO::AIO::nreqs + IO::AIO::nready + IO::AIO::npending + + IO::AIO::sendfile $ofh, $ifh, $offset, $count + IO::AIO::fadvise $fh, $offset, $len, $advice + IO::AIO::mlockall $flags + IO::AIO::munlockall + AIO REQUEST FUNCTIONS All the "aio_*" calls are more or less thin wrappers around the syscall with the same name (sans "aio_"). The arguments are similar or @@ -316,11 +359,12 @@ This call tries to make use of a native "sendfile" syscall to provide zero-copy operation. For this to work, $out_fh should refer - to a socket, and $in_fh should refer to mmap'able file. + to a socket, and $in_fh should refer to an mmap'able file. - If the native sendfile call fails or is not implemented, it will be - emulated, so you can call "aio_sendfile" on any type of filehandle - regardless of the limitations of the operating system. + If a native sendfile cannot be found or it fails with "ENOSYS", + "ENOTSUP", "EOPNOTSUPP", "EAFNOSUPPORT", "EPROTOTYPE" or "ENOTSOCK", + it will be emulated, so you can call "aio_sendfile" on any type of + filehandle regardless of the limitations of the operating system. Please note, however, that "aio_sendfile" can read more bytes from $in_fh than are written, and there is no way to find out how many @@ -365,6 +409,49 @@ print "size is ", -s _, "\n"; }; + aio_statvfs $fh_or_path, $callback->($statvfs) + Works like the POSIX "statvfs" or "fstatvfs" syscalls, depending on + whether a file handle or path was passed. + + On success, the callback is passed a hash reference with the + following members: "bsize", "frsize", "blocks", "bfree", "bavail", + "files", "ffree", "favail", "fsid", "flag" and "namemax". On + failure, "undef" is passed. + + The following POSIX IO::AIO::ST_* constants are defined: "ST_RDONLY" + and "ST_NOSUID". + + The following non-POSIX IO::AIO::ST_* flag masks are defined to + their correct value when available, or to 0 on systems that do not + support them: "ST_NODEV", "ST_NOEXEC", "ST_SYNCHRONOUS", + "ST_MANDLOCK", "ST_WRITE", "ST_APPEND", "ST_IMMUTABLE", + "ST_NOATIME", "ST_NODIRATIME" and "ST_RELATIME". + + Example: stat "/wd" and dump out the data if successful. + + aio_statvfs "/wd", sub { + my $f = $_[0] + or die "statvfs: $!"; + + use Data::Dumper; + say Dumper $f; + }; + + # result: + { + bsize => 1024, + bfree => 4333064312, + blocks => 10253828096, + files => 2050765568, + flag => 4096, + favail => 2042092649, + bavail => 4333064312, + ffree => 2042092649, + namemax => 255, + frsize => 1024, + fsid => 1810 + } + aio_utime $fh_or_path, $atime, $mtime, $callback->($status) Works like perl's "utime" function (including the special case of $atime and $mtime being undef). Fractional times are supported if @@ -476,8 +563,9 @@ them. $inode is the inode number (which might not be exact on systems - with 64 bit inode numbers and 32 bit perls). On systems that do - not deliver the inode information, this will always be zero. + with 64 bit inode numbers and 32 bit perls). This field has + unspecified content on systems that do not deliver the inode + information. IO::AIO::READDIR_DIRS_FIRST When this flag is set, then the names will be returned in an @@ -514,7 +602,7 @@ aio_copy $srcpath, $dstpath, $callback->($status) Try to copy the *file* (directories not supported as either source or destination) from $srcpath to $dstpath and call the callback with - the 0 (error) or -1 ok. + a status of 0 (ok) or -1 (error, see $!). This is a composite request that creates the destination file with mode 0200 and copies the contents of the source file into it using @@ -528,7 +616,7 @@ aio_move $srcpath, $dstpath, $callback->($status) Try to move the *file* (directories not supported as either source or destination) from $srcpath to $dstpath and call the callback with - the 0 (error) or -1 ok. + a status of 0 (ok) or -1 (error, see $!). This is a composite request that tries to rename(2) the file first; if rename fails with "EXDEV", it copies the file with "aio_copy" @@ -633,8 +721,38 @@ directory changes get written to disc. It works for anything that can be opened for read-only, not just directories. + Future versions of this function might fall back to other methods + when "fsync" on the directory fails (such as calling "sync"). + Passes 0 when everything went ok, and -1 on error. + aio_msync $scalar, $offset = 0, $length = undef, flags = 0, + $callback->($status) + This is a rather advanced IO::AIO call, which only works on + mmap(2)ed scalars (see the "IO::AIO::mmap" function, although it + also works on data scalars managed by the Sys::Mmap or Mmap modules, + note that the scalar must only be modified in-place while an aio + operation is pending on it). + + It calls the "msync" function of your OS, if available, with the + memory area starting at $offset in the string and ending $length + bytes later. If $length is negative, counts from the end, and if + $length is "undef", then it goes till the end of the string. The + flags can be a combination of "IO::AIO::MS_ASYNC", + "IO::AIO::MS_INVALIDATE" and "IO::AIO::MS_SYNC". + + aio_mtouch $scalar, $offset = 0, $length = undef, flags = 0, + $callback->($status) + This is a rather advanced IO::AIO call, which works best on + mmap(2)ed scalars. + + It touches (reads or writes) all memory pages in the specified range + inside the scalar. All caveats and parameters are the same as for + "aio_msync", above, except for flags, which must be either 0 (which + reads all pages and ensures they are instantiated) or + "IO::AIO::MT_MODIFY", which modifies the memory page s(by reading + and writing an octet from it, which dirties the page). + aio_group $callback->(...) This is a very special aio request: Instead of doing something, it is a container for other aio requests, which is useful if you want @@ -686,9 +804,9 @@ Cancels the request, if possible. Has the effect of skipping execution when entering the execute state and skipping calling the callback when entering the the result state, but will leave the - request otherwise untouched. That means that requests that currently - execute will not be stopped and resources held by the request will - not be freed prematurely. + request otherwise untouched (with the exception of readdir). That + means that requests that currently execute will not be stopped and + resources held by the request will not be freed prematurely. cb $req $callback->(...) Replace (or simply set) the callback registered to the request. @@ -760,6 +878,9 @@ request itself. Useful when you queued a lot of events but got a result early. + The group request will finish normally (you cannot add requests to + the group). + $grp->result (...) Set the result value(s) that will be passed to the group callback when all subrequests have finished and set the groups errno to the @@ -828,8 +949,9 @@ $fileno = IO::AIO::poll_fileno Return the *request result pipe file descriptor*. This filehandle must be polled for reading by some mechanism outside this module - (e.g. Event or select, see below or the SYNOPSIS). If the pipe - becomes readable you have to call "poll_cb" to check the results. + (e.g. EV, Glib, select and so on, see below or the SYNOPSIS). If the + pipe becomes readable you have to call "poll_cb" to check the + results. See "poll_cb" for an example. @@ -846,12 +968,37 @@ you don't have to do anything special to have it called later. Example: Install an Event watcher that automatically calls - IO::AIO::poll_cb with high priority: + IO::AIO::poll_cb with high priority (more examples can be found in + the SYNOPSIS section, at the top of this document): Event->io (fd => IO::AIO::poll_fileno, poll => 'r', async => 1, cb => \&IO::AIO::poll_cb); + IO::AIO::poll_wait + If there are any outstanding requests and none of them in the result + phase, wait till the result filehandle becomes ready for reading + (simply does a "select" on the filehandle. This is useful if you + want to synchronously wait for some requests to finish). + + See "nreqs" for an example. + + IO::AIO::poll + Waits until some requests have been handled. + + Returns the number of requests processed, but is otherwise strictly + equivalent to: + + IO::AIO::poll_wait, IO::AIO::poll_cb + + IO::AIO::flush + Wait till all outstanding AIO requests have been handled. + + Strictly equivalent to: + + IO::AIO::poll_wait, IO::AIO::poll_cb + while IO::AIO::nreqs; + IO::AIO::max_poll_reqs $nreqs IO::AIO::max_poll_time $seconds These set the maximum number of requests (default 0, meaning @@ -884,30 +1031,6 @@ poll => 'r', nice => 1, cb => &IO::AIO::poll_cb); - IO::AIO::poll_wait - If there are any outstanding requests and none of them in the result - phase, wait till the result filehandle becomes ready for reading - (simply does a "select" on the filehandle. This is useful if you - want to synchronously wait for some requests to finish). - - See "nreqs" for an example. - - IO::AIO::poll - Waits until some requests have been handled. - - Returns the number of requests processed, but is otherwise strictly - equivalent to: - - IO::AIO::poll_wait, IO::AIO::poll_cb - - IO::AIO::flush - Wait till all outstanding AIO requests have been handled. - - Strictly equivalent to: - - IO::AIO::poll_wait, IO::AIO::poll_cb - while IO::AIO::nreqs; - CONTROLLING THE NUMBER OF THREADS IO::AIO::min_parallel $nthreads Set the minimum number of AIO threads to $nthreads. The current @@ -996,6 +1119,124 @@ Returns the number of requests currently in the pending state (executed, but not yet processed by poll_cb). + MISCELLANEOUS FUNCTIONS + IO::AIO implements some functions that might be useful, but are not + asynchronous. + + IO::AIO::sendfile $ofh, $ifh, $offset, $count + Calls the "eio_sendfile_sync" function, which is like + "aio_sendfile", but is blocking (this makes most sense if you know + the input data is likely cached already and the output filehandle is + set to non-blocking operations). + + Returns the number of bytes copied, or -1 on error. + + IO::AIO::fadvise $fh, $offset, $len, $advice + Simply calls the "posix_fadvise" function (see it's manpage for + details). The following advice constants are avaiable: + "IO::AIO::FADV_NORMAL", "IO::AIO::FADV_SEQUENTIAL", + "IO::AIO::FADV_RANDOM", "IO::AIO::FADV_NOREUSE", + "IO::AIO::FADV_WILLNEED", "IO::AIO::FADV_DONTNEED". + + On systems that do not implement "posix_fadvise", this function + returns ENOSYS, otherwise the return value of "posix_fadvise". + + IO::AIO::mmap $scalar, $length, $prot, $flags, $fh[, $offset] + Memory-maps a file (or anonymous memory range) and attaches it to + the given $scalar, which will act like a string scalar. + + The only operations allowed on the scalar are "substr"/"vec" that + don't change the string length, and most read-only operations such + as copying it or searching it with regexes and so on. + + Anything else is unsafe and will, at best, result in memory leaks. + + The memory map associated with the $scalar is automatically removed + when the $scalar is destroyed, or when the "IO::AIO::mmap" or + "IO::AIO::munmap" functions are called. + + This calls the "mmap"(2) function internally. See your system's + manual page for details on the $length, $prot and $flags parameters. + + The $length must be larger than zero and smaller than the actual + filesize. + + $prot is a combination of "IO::AIO::PROT_NONE", + "IO::AIO::PROT_EXEC", "IO::AIO::PROT_READ" and/or + "IO::AIO::PROT_WRITE", + + $flags can be a combination of "IO::AIO::MAP_SHARED" or + "IO::AIO::MAP_PRIVATE", or a number of system-specific flags (when + not available, the are defined as 0): "IO::AIO::MAP_ANONYMOUS" + (which is set to "MAP_ANON" if your system only provides this + constant), "IO::AIO::MAP_HUGETLB", "IO::AIO::MAP_LOCKED", + "IO::AIO::MAP_NORESERVE", "IO::AIO::MAP_POPULATE" or + "IO::AIO::MAP_NONBLOCK" + + If $fh is "undef", then a file descriptor of -1 is passed. + + $offset is the offset from the start of the file - it generally must + be a multiple of "IO::AIO::PAGESIZE" and defaults to 0. + + Example: + + use Digest::MD5; + use IO::AIO; + + open my $fh, "io (fd => IO::AIO::poll_fileno, + poll => 'r', + cb => \&IO::AIO::poll_cb); + + # Glib/Gtk2 integration + add_watch Glib::IO IO::AIO::poll_fileno, + in => sub { IO::AIO::poll_cb; 1 }; + + # Tk integration + Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "", + readable => \&IO::AIO::poll_cb); + + # Danga::Socket integration + Danga::Socket->AddOtherFds (IO::AIO::poll_fileno => + \&IO::AIO::poll_cb); + FORK BEHAVIOUR This module should do "the right thing" when the process using it forks: