--- IO-AIO/AIO.pm 2010/01/10 20:37:33 1.175 +++ IO-AIO/AIO.pm 2011/05/19 23:57:12 1.191 @@ -6,7 +6,7 @@ use IO::AIO; - aio_open "/etc/passwd", O_RDONLY, 0, sub { + aio_open "/etc/passwd", IO::AIO::O_RDONLY, 0, sub { my $fh = shift or die "/etc/passwd: $!"; ... @@ -78,7 +78,7 @@ my $aio_w = EV::io IO::AIO::poll_fileno, EV::READ, \&IO::AIO::poll_cb; # queue the request to open /etc/passwd - aio_open "/etc/passwd", O_RDONLY, 0, sub { + aio_open "/etc/passwd", IO::AIO::O_RDONLY, 0, sub { my $fh = shift or die "error while opening: $!"; @@ -170,7 +170,7 @@ use base 'Exporter'; BEGIN { - our $VERSION = '3.5'; + our $VERSION = '3.8'; our @AIO_REQ = qw(aio_sendfile aio_read aio_write aio_open aio_close aio_stat aio_lstat aio_unlink aio_rmdir aio_readdir aio_readdirx @@ -179,14 +179,16 @@ aio_rename aio_link aio_move aio_copy aio_group aio_nop aio_mknod aio_load aio_rmtree aio_mkdir aio_chown aio_chmod aio_utime aio_truncate - aio_msync aio_mtouch aio_statvfs); + aio_msync aio_mtouch aio_mlock aio_mlockall + aio_statvfs); our @EXPORT = (@AIO_REQ, qw(aioreq_pri aioreq_nice)); our @EXPORT_OK = qw(poll_fileno poll_cb poll_wait flush - min_parallel max_parallel max_idle + min_parallel max_parallel max_idle idle_timeout nreqs nready npending nthreads max_poll_time max_poll_reqs - sendfile fadvise); + sendfile fadvise madvise + mmap munmap munlock munlockall); push @AIO_REQ, qw(aio_busy); # not exported @@ -241,6 +243,8 @@ aio_pathsync $path, $callback->($status) aio_msync $scalar, $offset = 0, $length = undef, flags = 0, $callback->($status) aio_mtouch $scalar, $offset = 0, $length = undef, flags = 0, $callback->($status) + aio_mlock $scalar, $offset = 0, $length = undef, $callback->($status) + aio_mlockall $flags, $callback->($status) aio_group $callback->(...) aio_nop $callback->() @@ -256,6 +260,7 @@ IO::AIO::min_parallel $nthreads IO::AIO::max_parallel $nthreads IO::AIO::max_idle $nthreads + IO::AIO::idle_timeout $seconds IO::AIO::max_outstanding $maxreqs IO::AIO::nreqs IO::AIO::nready @@ -263,7 +268,9 @@ IO::AIO::sendfile $ofh, $ifh, $offset, $count IO::AIO::fadvise $fh, $offset, $len, $advice - IO::AIO::mlockall $flags + IO::AIO::madvise $scalar, $offset, $length, $advice + IO::AIO::mprotect $scalar, $offset, $length, $protect + IO::AIO::munlock $scalar, $offset = 0, $length = undef IO::AIO::munlockall =head2 AIO REQUEST FUNCTIONS @@ -354,7 +361,7 @@ Example: - aio_open "/etc/passwd", O_RDONLY, 0, sub { + aio_open "/etc/passwd", IO::AIO::O_RDONLY, 0, sub { if ($_[0]) { print "open successful, fh is $_[0]\n"; ... @@ -424,22 +431,31 @@ than one C per C<$out_fh>, as they will interfere with each other. +Please note that C can read more bytes from C<$in_fh> than +are written, and there is no way to find out how many bytes have been read +from C alone, as C only provides the number of +bytes written to C<$out_fh>. Only if the result value equals C<$length> +one can assume that C<$length> bytes have been read. + +Unlike with other C functions, it makes a lot of sense to use +C on non-blocking sockets, as long as one end (typically +the C<$in_fh>) is a file - the file I/O will then be asynchronous, while +the socket I/O will be non-blocking. Note, however, that you can run into +a trap where C reads some data with readahead, then fails +to write all data, and when the socket is ready the next time, the data +in the cache is already lost, forcing C to again hit the +disk. Explicit C + C let's you control resource usage +much better. + This call tries to make use of a native C syscall to provide zero-copy operation. For this to work, C<$out_fh> should refer to a -socket, and C<$in_fh> should refer to mmap'able file. +socket, and C<$in_fh> should refer to an mmap'able file. If a native sendfile cannot be found or it fails with C, C, C, C, C or C, it will be emulated, so you can call C on any type of filehandle regardless of the limitations of the operating system. -Please note, however, that C can read more bytes from -C<$in_fh> than are written, and there is no way to find out how many -bytes have been read from C alone, as C only -provides the number of bytes written to C<$out_fh>. Only if the result -value equals C<$length> one can assume that C<$length> bytes have been -read. - =item aio_readahead $fh,$offset,$length, $callback->($retval) @@ -471,6 +487,15 @@ error when stat'ing a large file, the results will be silently truncated unless perl itself is compiled with large file support. +To help interpret the mode and dev/rdev stat values, IO::AIO offers the +following constants and functions (if not implemented, the constants will +be C<0> and the functions will either C or fall back on traditional +behaviour). + +C, C, C, C, C, C, +C, C, C, C, +C, C. + Example: Print the length of F: aio_stat "/etc/passwd", sub { @@ -581,6 +606,8 @@ aio_mknod $path, IO::AIO::S_IFIFO | $mode, 0, sub { ... +See C for info about some potentially helpful extra constants +and functions. =item aio_link $srcpath, $dstpath, $callback->($status) @@ -643,8 +670,8 @@ =item IO::AIO::READDIR_DENTS -When this flag is off, then the callback gets an arrayref with of names -only (as with C), otherwise it gets an arrayref with +When this flag is off, then the callback gets an arrayref consisting of +names only (as with C), otherwise it gets an arrayref with C<[$name, $type, $inode]> arrayrefs, each describing a single directory entry in more detail. @@ -1086,9 +1113,10 @@ =item aio_msync $scalar, $offset = 0, $length = undef, flags = 0, $callback->($status) This is a rather advanced IO::AIO call, which only works on mmap(2)ed -scalars (see the L or L modules for details on this, note -that the scalar must only be modified in-place while an aio operation is -pending on it). +scalars (see the C function, although it also works on data +scalars managed by the L or L modules, note that the +scalar must only be modified in-place while an aio operation is pending on +it). It calls the C function of your OS, if available, with the memory area starting at C<$offset> in the string and ending C<$length> bytes @@ -1109,6 +1137,45 @@ C, which modifies the memory page s(by reading and writing an octet from it, which dirties the page). +=item aio_mlock $scalar, $offset = 0, $length = undef, $callback->($status) + +This is a rather advanced IO::AIO call, which works best on mmap(2)ed +scalars. + +It reads in all the pages of the underlying storage into memory (if any) +and locks them, so they are not getting swapped/paged out or removed. + +If C<$length> is undefined, then the scalar will be locked till the end. + +On systems that do not implement C, this function returns C<-1> +and sets errno to C. + +Note that the corresponding C is synchronous and is +documented under L. + +Example: open a file, mmap and mlock it - both will be undone when +C<$data> gets destroyed. + + open my $fh, "<", $path or die "$path: $!"; + my $data; + IO::AIO::mmap $data, -s $fh, IO::AIO::PROT_READ, IO::AIO::MAP_SHARED, $fh; + aio_mlock $data; # mlock in background + +=item aio_mlockall $flags, $callback->($status) + +Calls the C function with the given C<$flags> (a combination of +C and C). + +On systems that do not implement C, this function returns C<-1> +and sets errno to C. + +Note that the corresponding C is synchronous and is +documented under L. + +Example: asynchronously lock all current and future pages into memory. + + aio_mlockall IO::AIO::MCL_FUTURE; + =item aio_group $callback->(...) This is a very special aio request: Instead of doing something, it is a @@ -1341,11 +1408,12 @@ =item IO::AIO::poll_cb -Process some outstanding events on the result pipe. You have to call this -regularly. Returns C<0> if all events could be processed, or C<-1> if it -returned earlier for whatever reason. Returns immediately when no events -are outstanding. The amount of events processed depends on the settings of -C and C. +Process some outstanding events on the result pipe. You have to call +this regularly. Returns C<0> if all events could be processed (or there +were no events to process), or C<-1> if it returned earlier for whatever +reason. Returns immediately when no events are outstanding. The amount of +events processed depends on the settings of C and +C. If not all requests were processed for whatever reason, the filehandle will still be ready when C returns, so normally you don't have to @@ -1461,10 +1529,11 @@ =item IO::AIO::max_idle $nthreads -Limit the number of threads (default: 4) that are allowed to idle (i.e., -threads that did not get a request to process within 10 seconds). That -means if a thread becomes idle while C<$nthreads> other threads are also -idle, it will free its resources and exit. +Limit the number of threads (default: 4) that are allowed to idle +(i.e., threads that did not get a request to process within the idle +timeout (default: 10 seconds). That means if a thread becomes idle while +C<$nthreads> other threads are also idle, it will free its resources and +exit. This is useful when you allow a large number of threads (e.g. 100 or 1000) to allow for extremely high load situations, but want to free resources @@ -1474,6 +1543,11 @@ creation is fast. If thread creation is very slow on your system you might want to use larger values. +=item IO::AIO::idle_timeout $seconds + +Sets the minimum idle timeout (default 10) after which worker threads are +allowed to exit. SEe C. + =item IO::AIO::max_outstanding $maxreqs This is a very bad function to use in interactive programs because it @@ -1538,7 +1612,7 @@ =item IO::AIO::fadvise $fh, $offset, $len, $advice -Simply calls the C function (see it's +Simply calls the C function (see its manpage for details). The following advice constants are avaiable: C, C, C, C, @@ -1547,13 +1621,84 @@ On systems that do not implement C, this function returns ENOSYS, otherwise the return value of C. -=item IO::AIO::mlockall $flags +=item IO::AIO::madvise $scalar, $offset, $len, $advice -Calls the C function with the given C<$flags> (a combination of -C and C). +Simply calls the C function (see its +manpage for details). The following advice constants are +avaiable: C, C, +C, C, C. + +On systems that do not implement C, this function returns +ENOSYS, otherwise the return value of C. + +=item IO::AIO::mprotect $scalar, $offset, $len, $protect + +Simply calls the C function on the preferably AIO::mmap'ed +$scalar (see its manpage for details). The following protect +constants are avaiable: C, C, +C, C. + +On systems that do not implement C, this function returns +ENOSYS, otherwise the return value of C. + +=item IO::AIO::mmap $scalar, $length, $prot, $flags, $fh[, $offset] + +Memory-maps a file (or anonymous memory range) and attaches it to the +given C<$scalar>, which will act like a string scalar. + +The only operations allowed on the scalar are C/C that don't +change the string length, and most read-only operations such as copying it +or searching it with regexes and so on. + +Anything else is unsafe and will, at best, result in memory leaks. + +The memory map associated with the C<$scalar> is automatically removed +when the C<$scalar> is destroyed, or when the C or +C functions are called. + +This calls the C(2) function internally. See your system's manual +page for details on the C<$length>, C<$prot> and C<$flags> parameters. + +The C<$length> must be larger than zero and smaller than the actual +filesize. + +C<$prot> is a combination of C, C, +C and/or C, + +C<$flags> can be a combination of C or +C, or a number of system-specific flags (when +not available, the are defined as 0): C +(which is set to C if your system only provides this +constant), C, C, +C, C or +C + +If C<$fh> is C, then a file descriptor of C<-1> is passed. + +C<$offset> is the offset from the start of the file - it generally must be +a multiple of C and defaults to C<0>. + +Example: + + use Digest::MD5; + use IO::AIO; + + open my $fh, ". + +=item IO::AIO::munlock $scalar, $offset = 0, $length = undef -On systems that do not implement C, this function returns -ENOSYS, otherwise the return value of C. +Calls the C function, undoing the effects of a previous +C call (see its description for details). =item IO::AIO::munlockall