--- IO-AIO/AIO.pm 2018/02/20 04:32:59 1.281 +++ IO-AIO/AIO.pm 2019/03/18 23:52:09 1.301 @@ -173,7 +173,7 @@ use base 'Exporter'; BEGIN { - our $VERSION = 4.4; + our $VERSION = 4.71; our @AIO_REQ = qw(aio_sendfile aio_seek aio_read aio_write aio_open aio_close aio_stat aio_lstat aio_unlink aio_rmdir aio_readdir aio_readdirx @@ -194,7 +194,7 @@ nreqs nready npending nthreads max_poll_time max_poll_reqs sendfile fadvise madvise - mmap munmap munlock munlockall); + mmap munmap mremap munlock munlockall); push @AIO_REQ, qw(aio_busy); # not exported @@ -287,6 +287,7 @@ IO::AIO::fadvise $fh, $offset, $len, $advice IO::AIO::mmap $scalar, $length, $prot, $flags[, $fh[, $offset]] IO::AIO::munmap $scalar + IO::AIO::mremap $scalar, $new_length, $flags[, $new_address] IO::AIO::madvise $scalar, $offset, $length, $advice IO::AIO::mprotect $scalar, $offset, $length, $protect IO::AIO::munlock $scalar, $offset = 0, $length = undef @@ -406,7 +407,7 @@ C, C, C, C, C, C, C, C, C, C, C, -C, C, C, C, and C. +C, C, C, C, C and C. =item aio_close $fh, $callback->($status) @@ -542,9 +543,10 @@ =item aio_lstat $fh, $callback->($status) -Works like perl's C or C in void context. The callback will -be called after the stat and the results will be available using C -or C<-s _> etc... +Works almost exactly like perl's C or C in void context. The +callback will be called after the stat and the results will be available +using C or C<-s _> and other tests (with the exception of C<-B> +and C<-T>). The pathname passed to C must be absolute. See API NOTES, above, for an explanation. @@ -562,6 +564,9 @@ C, C, C, C, C, C. +To access higher resolution stat timestamps, see L. + Example: Print the length of F: aio_stat "/etc/passwd", sub { @@ -620,9 +625,10 @@ and $mtime being undef). Fractional times are supported if the underlying syscalls support them. -When called with a pathname, uses utimes(2) if available, otherwise -utime(2). If called on a file descriptor, uses futimes(2) if available, -otherwise returns ENOSYS, so this is not portable. +When called with a pathname, uses utimensat(2) or utimes(2) if available, +otherwise utime(2). If called on a file descriptor, uses futimens(2) +or futimes(2) if available, otherwise returns ENOSYS, so this is not +portable. Examples: @@ -791,10 +797,10 @@ =item IO::AIO::READDIR_DENTS -When this flag is off, then the callback gets an arrayref consisting of -names only (as with C), otherwise it gets an arrayref with -C<[$name, $type, $inode]> arrayrefs, each describing a single directory -entry in more detail. +Normally the callback gets an arrayref consisting of names only (as +with C). If this flag is set, then the callback gets an +arrayref with C<[$name, $type, $inode]> arrayrefs, each describing a +single directory entry in more detail: C<$name> is the name of the entry. @@ -804,9 +810,9 @@ C, C, C, C, C. -C means just that: readdir does not know. If you need to -know, you have to run stat yourself. Also, for speed reasons, the C<$type> -scalars are read-only: you can not modify them. +C means just that: readdir does not know. If you need +to know, you have to run stat yourself. Also, for speed/memory reasons, +the C<$type> scalars are read-only: you must not modify them. C<$inode> is the inode number (which might not be exact on systems with 64 bit inode numbers and 32 bit perls). This field has unspecified content on @@ -827,12 +833,14 @@ =item IO::AIO::READDIR_STAT_ORDER When this flag is set, then the names will be returned in an order -suitable for stat()'ing each one. That is, when you plan to stat() -all files in the given directory, then the returned order will likely -be fastest. - -If both this flag and C are specified, then -the likely dirs come first, resulting in a less optimal stat order. +suitable for stat()'ing each one. That is, when you plan to stat() most or +all files in the given directory, then the returned order will likely be +faster. + +If both this flag and C are specified, +then the likely dirs come first, resulting in a less optimal stat order +for stat'ing all entries, but likely a more optimal order for finding +subdirectories. =item IO::AIO::READDIR_FOUND_UNKNOWN @@ -1103,13 +1111,69 @@ return $grp->result () if $_[0]; my $now = time; my $hash1 = join ":", (stat _)[0,1,3,7,9]; + my $rdxflags = READDIR_DIRS_FIRST; + + if ((stat _)[3] < 2) { + # at least one non-POSIX filesystem exists + # that returns useful DT_type values: btrfs, + # so optimise for this here by requesting dents + $rdxflags |= READDIR_DENTS; + } # read the directory entries aioreq_pri $pri; - add $grp aio_readdirx $wd, READDIR_DIRS_FIRST, sub { - my $entries = shift + add $grp aio_readdirx $wd, $rdxflags, sub { + my ($entries, $flags) = @_ or return $grp->result (); + if ($rdxflags & READDIR_DENTS) { + # if we requested type values, see if we can use them directly. + + # if there were any DT_UNKNOWN entries then we assume we + # don't know. alternatively, we could assume that if we get + # one DT_DIR, then all directories are indeed marked with + # DT_DIR, but this seems not required for btrfs, and this + # is basically the "btrfs can't get it's act together" code + # branch. + unless ($flags & READDIR_FOUND_UNKNOWN) { + # now we have valid DT_ information for all entries, + # so use it as an optimisation without further stat's. + # they must also all be at the beginning of @$entries + # by now. + + my $dirs; + + if (@$entries) { + for (0 .. $#$entries) { + if ($entries->[$_][1] != DT_DIR) { + # splice out directories + $dirs = [splice @$entries, 0, $_]; + last; + } + } + + # if we didn't find any non-dir, then all entries are dirs + unless ($dirs) { + ($dirs, $entries) = ($entries, []); + } + } else { + # directory is empty, so there are no sbdirs + $dirs = []; + } + + # either splice'd the directories out or the dir was empty. + # convert dents to filenames + $_ = $_->[0] for @$dirs; + $_ = $_->[0] for @$entries; + + return $grp->result ($dirs, $entries); + } + + # cannot use, so return to our old ways + # by pretending we only scanned for names. + $_ = $_->[0] for @$entries; + } + # stat the dir another time aioreq_pri $pri; add $grp aio_stat $wd, sub { @@ -1379,11 +1443,14 @@ =item aio_mlockall $flags, $callback->($status) -Calls the C function with the given C<$flags> (a combination of -C and C). +Calls the C function with the given C<$flags> (a +combination of C, C and +C). On systems that do not implement C, this function returns C<-1> -and sets errno to C. +and sets errno to C. Similarly, flag combinations not supported +by the system result in a return value of C<-1> with errno being set to +C. Note that the corresponding C is synchronous and is documented under L. @@ -1778,6 +1845,7 @@ =back + =head2 SUPPORT FUNCTIONS =head3 EVENT PROCESSING AND EVENT LOOP INTEGRATION @@ -1852,6 +1920,16 @@ IO::AIO::poll_wait, IO::AIO::poll_cb while IO::AIO::nreqs; +This function can be useful at program aborts, to make sure outstanding +I/O has been done (C uses an C block which already calls +this function on normal exits), or when you are merely using C +for its more advanced functions, rather than for async I/O, e.g.: + + my ($dirs, $nondirs); + IO::AIO::aio_scandir "/tmp", 0, sub { ($dirs, $nondirs) = @_ }; + IO::AIO::flush; + # $dirs, $nondirs are now set + =item IO::AIO::max_poll_reqs $nreqs =item IO::AIO::max_poll_time $seconds @@ -1887,6 +1965,7 @@ =back + =head3 CONTROLLING THE NUMBER OF THREADS =over @@ -1983,6 +2062,7 @@ =back + =head3 STATISTICAL INFORMATION =over @@ -2009,6 +2089,101 @@ =back + +=head3 SUBSECOND STAT TIME ACCESS + +Both C/C and perl's C/C functions can +generally find access/modification and change times with subsecond time +accuracy of the system supports it, but perl's built-in functions only +return the integer part. + +The following functions return the timestamps of the most recent +stat with subsecond precision on most systems and work both after +C/C and perl's C/C calls. Their return +value is only meaningful after a successful C/C call, or +during/after a successful C/C callback. + +This is similar to the L C functions, but can return +full resolution without rounding and work with standard perl C, +alleviating the need to call the special C functions, which +do not act like their perl counterparts. + +On operating systems or file systems where subsecond time resolution is +not supported or could not be detected, a fractional part of C<0> is +returned, so it is always safe to call these functions. + +=over 4 + +=item $seconds = IO::AIO::st_atime, IO::AIO::st_mtime, IO::AIO::st_ctime, IO::AIO::st_btime + +Return the access, modication, change or birth time, respectively, +including fractional part. Due to the limited precision of floating point, +the accuracy on most platforms is only a bit better than milliseconds +for times around now - see the I function family, below, for full +accuracy. + +File birth time is only available when the OS and perl support it (on +FreeBSD and NetBSD at the time of this writing, although support is +adaptive, so if your OS/perl gains support, IO::AIO can take advantage of +it). On systems where it isn't available, C<0> is currently returned, but +this might change to C in a future version. + +=item ($atime, $mtime, $ctime, $btime, ...) = IO::AIO::st_xtime + +Returns access, modification, change and birth time all in one go, and +maybe more times in the future version. + +=item $nanoseconds = IO::AIO::st_atimensec, IO::AIO::st_mtimensec, IO::AIO::st_ctimensec, IO::AIO::st_btimensec + +Return the fractional access, modifcation, change or birth time, in nanoseconds, +as an integer in the range C<0> to C<999999999>. + +Note that no accessors are provided for access, modification and +change times - you need to get those from C if required (C and so on will I generally give you the correct +value). + +=item $seconds = IO::AIO::st_btimesec + +The (integral) seconds part of the file birth time, if available. + +=item ($atime, $mtime, $ctime, $btime, ...) = IO::AIO::st_xtimensec + +Like the functions above, but returns all four times in one go (and maybe +more in future versions). + +=item $counter = IO::AIO::st_gen + +Returns the generation counter (in practice this is just a random number) +of the file. This is only available on platforms which have this member in +their C (most BSDs at the time of this writing) and generally +only to the root usert. If unsupported, C<0> is returned, but this might +change to C in a future version. + +=back + +Example: print the high resolution modification time of F, using +C, and C. + + if (stat "/etc") { + printf "stat(/etc) mtime: %f\n", IO::AIO::st_mtime; + } + + IO::AIO::aio_stat "/etc", sub { + $_[0] + and return; + + printf "aio_stat(/etc) mtime: %d.%09d\n", (stat _)[9], IO::AIO::st_mtimensec; + }; + + IO::AIO::flush; + +Output of the awbove on my system, showing reduced and full accuracy: + + stat(/etc) mtime: 1534043702.020808 + aio_stat(/etc) mtime: 1534043702.020807792 + + =head3 MISCELLANEOUS FUNCTIONS IO::AIO implements some functions that are useful when you want to use @@ -2158,6 +2333,34 @@ Removes a previous mmap and undefines the C<$scalar>. +=item IO::AIO::mremap $scalar, $new_length, $flags = MREMAP_MAYMOVE[, $new_address = 0] + +Calls the Linux-specific mremap(2) system call. The C<$scalar> must have +been mapped by C, and C<$flags> must currently either be +C<0> or C. + +Returns true if successful, and false otherwise. If the underlying mmapped +region has changed address, then the true value has the numerical value +C<1>, otherwise it has the numerical value C<0>: + + my $success = IO::AIO::mremap $mmapped, 8192, IO::AIO::MREMAP_MAYMOVE + or die "mremap: $!"; + + if ($success*1) { + warn "scalar has chanegd address in memory\n"; + } + +C and the C<$new_address> argument are currently +implemented, but not supported and might go away in a future version. + +On systems where this call is not supported or is not emulated, this call +returns falls and sets C<$!> to C. + +=item IO::AIO::mlockall $flags + +Calls the C function, which is like C, +but is blocking. + =item IO::AIO::munlock $scalar, $offset = 0, $length = undef Calls the C function, undoing the effects of a previous @@ -2222,7 +2425,7 @@ my ($rfh, $wfh) = IO::AIO::pipe2 IO::AIO::O_CLOEXEC or die "pipe2: $!\n"; -=item $fh = IO::AIO::eventfd ([$initval, [$flags]]) +=item $fh = IO::AIO::eventfd [$initval, [$flags]] This is a direct interface to the Linux L system call. The (unhelpful) defaults for C<$initval> and C<$flags> are C<0> for both. @@ -2235,6 +2438,73 @@ The following symbol flag values are available: C, C and C (Linux 2.6.30). +Example: create a new eventfd filehandle: + + $fh = IO::AIO::eventfd 0, IO::AIO::O_CLOEXEC + or die "eventfd: $!\n"; + +=item $fh = IO::AIO::timerfd_create $clockid[, $flags] + +This is a direct interface to the Linux L system call. The +(unhelpful) default for C<$flags> is C<0>. + +On success, the new timerfd filehandle is returned, otherwise returns +C. If the eventfd syscall is missing, fails with C. + +Please refer to L for more info on this call. + +The following C<$clockid> values are +available: C, C +C (Linux 3.15) +C (Linux 3.11) and +C (Linux 3.11). + +The following C<$flags> values are available (Linux +2.6.27): C and C. + +Example: create a new timerfd and set it to one-second repeated alarms, +then wait for two alarms: + + my $fh = IO::AIO::timerfd_create IO::AIO::CLOCK_BOOTTIME, IO::AIO::TFD_CLOEXEC + or die "timerfd_create: $!\n"; + + defined IO::AIO::timerfd_settime $fh, 0, 1, 1 + or die "timerfd_settime: $!\n"; + + for (1..2) { + 8 == sysread $fh, my $buf, 8 + or die "timerfd read failure\n"; + + printf "number of expirations (likely 1): %d\n", + unpack "Q", $buf; + } + +=item ($cur_interval, $cur_value) = IO::AIO::timerfd_settime $fh, $flags, $new_interval, $nbw_value + +This is a direct interface to the Linux L system +call. Please refer to its manpage for more info on this call. + +The new itimerspec is specified using two (possibly fractional) second +values, C<$new_interval> and C<$new_value>). + +On success, the current interval and value are returned (as per +C). On failure, the empty list is returned. + +The following C<$flags> values are +available: C and +C. + +See C for a full example. + +=item ($cur_interval, $cur_value) = IO::AIO::timerfd_gettime $fh + +This is a direct interface to the Linux L system +call. Please refer to its manpage for more info on this call. + +On success, returns the current values of interval and value for the given +timerfd (as potentially fractional second values). On failure, the empty +list is returned. + =back =cut @@ -2309,6 +2579,15 @@ =back +=head2 LINUX-SPECIFIC CALLS + +When a call is documented as "linux-specific" then this means it +originated on GNU/Linux. C will usually try to autodetect the +availability and compatibility of such calls regardless of the platform +it is compiled on, so platforms such as FreeBSD which often implement +these calls will work. When in doubt, call them and see if they fail wth +C. + =head2 MEMORY USAGE Per-request usage: @@ -2330,7 +2609,18 @@ =head1 KNOWN BUGS -Known bugs will be fixed in the next release. +Known bugs will be fixed in the next release :) + +=head1 KNOWN ISSUES + +Calls that try to "import" foreign memory areas (such as C +or C) do not work with generic lvalues, such as +non-created hash slots or other scalars I didn't think of. It's best to +avoid such and either use scalar variables or making sure that the scalar +exists (e.g. by storing C) and isn't "funny" (e.g. tied). + +I am not sure anything can be done about this, so this is considered a +known issue, rather than a bug. =head1 SEE ALSO