--- IO-AIO/AIO.pm 2018/07/24 04:58:59 1.287 +++ IO-AIO/AIO.pm 2022/09/05 08:43:58 1.316 @@ -173,7 +173,7 @@ use base 'Exporter'; BEGIN { - our $VERSION = 4.42; + our $VERSION = 4.78; our @AIO_REQ = qw(aio_sendfile aio_seek aio_read aio_write aio_open aio_close aio_stat aio_lstat aio_unlink aio_rmdir aio_readdir aio_readdirx @@ -194,7 +194,12 @@ nreqs nready npending nthreads max_poll_time max_poll_reqs sendfile fadvise madvise - mmap munmap mremap munlock munlockall); + mmap munmap mremap munlock munlockall + + accept4 tee splice pipe2 pipesize + fexecve mount umount memfd_create eventfd + timerfd_create timerfd_settime timerfd_gettime + pidfd_open pidfd_send_signal pidfd_getfd); push @AIO_REQ, qw(aio_busy); # not exported @@ -280,11 +285,15 @@ IO::AIO::nreqs IO::AIO::nready IO::AIO::npending - $nfd = IO::AIO::get_fdlimit [EXPERIMENTAL] - IO::AIO::min_fdlimit $nfd [EXPERIMENTAL] + IO::AIO::reinit + + $nfd = IO::AIO::get_fdlimit + IO::AIO::min_fdlimit $nfd IO::AIO::sendfile $ofh, $ifh, $offset, $count IO::AIO::fadvise $fh, $offset, $len, $advice + IO::AIO::fexecve $fh, $argv, $envp + IO::AIO::mmap $scalar, $length, $prot, $flags[, $fh[, $offset]] IO::AIO::munmap $scalar IO::AIO::mremap $scalar, $new_length, $flags[, $new_address] @@ -293,6 +302,36 @@ IO::AIO::munlock $scalar, $offset = 0, $length = undef IO::AIO::munlockall + # stat extensions + $counter = IO::AIO::st_gen + $seconds = IO::AIO::st_atime, IO::AIO::st_mtime, IO::AIO::st_ctime, IO::AIO::st_btime + ($atime, $mtime, $ctime, $btime, ...) = IO::AIO::st_xtime + $nanoseconds = IO::AIO::st_atimensec, IO::AIO::st_mtimensec, IO::AIO::st_ctimensec, IO::AIO::st_btimensec + $seconds = IO::AIO::st_btimesec + ($atime, $mtime, $ctime, $btime, ...) = IO::AIO::st_xtimensec + + # very much unportable syscalls + IO::AIO::accept4 $r_fh, $sockaddr, $sockaddr_len, $flags + IO::AIO::splice $r_fh, $r_off, $w_fh, $w_off, $length, $flags + IO::AIO::tee $r_fh, $w_fh, $length, $flags + + $actual_size = IO::AIO::pipesize $r_fh[, $new_size] + ($rfh, $wfh) = IO::AIO::pipe2 [$flags] + + $fh = IO::AIO::eventfd [$initval, [$flags]] + $fh = IO::AIO::memfd_create $pathname[, $flags] + + $fh = IO::AIO::timerfd_create $clockid[, $flags] + ($cur_interval, $cur_value) = IO::AIO::timerfd_settime $fh, $flags, $new_interval, $nbw_value + ($cur_interval, $cur_value) = IO::AIO::timerfd_gettime $fh + + $fh = IO::AIO::pidfd_open $pid[, $flags] + $status = IO::AIO::pidfd_send_signal $pidfh, $signal[, $siginfo[, $flags]] + $fh = IO::AIO::pidfd_getfd $pidfh, $targetfd[, $flags] + + $retval = IO::AIO::mount $special, $path, $fstype, $flags = 0, $data = undef + $retval = IO::AIO::umount $path, $flags = 0 + =head2 API NOTES All the C calls are more or less thin wrappers around the syscall @@ -543,9 +582,10 @@ =item aio_lstat $fh, $callback->($status) -Works like perl's C or C in void context. The callback will -be called after the stat and the results will be available using C -or C<-s _> etc... +Works almost exactly like perl's C or C in void context. The +callback will be called after the stat and the results will be available +using C or C<-s _> and other tests (with the exception of C<-B> +and C<-T>). The pathname passed to C must be absolute. See API NOTES, above, for an explanation. @@ -563,6 +603,9 @@ C, C, C, C, C, C. +To access higher resolution stat timestamps, see L. + Example: Print the length of F: aio_stat "/etc/passwd", sub { @@ -621,9 +664,10 @@ and $mtime being undef). Fractional times are supported if the underlying syscalls support them. -When called with a pathname, uses utimes(2) if available, otherwise -utime(2). If called on a file descriptor, uses futimes(2) if available, -otherwise returns ENOSYS, so this is not portable. +When called with a pathname, uses utimensat(2) or utimes(2) if available, +otherwise utime(2). If called on a file descriptor, uses futimens(2) +or futimes(2) if available, otherwise returns ENOSYS, so this is not +portable. Examples: @@ -1106,13 +1150,69 @@ return $grp->result () if $_[0]; my $now = time; my $hash1 = join ":", (stat _)[0,1,3,7,9]; + my $rdxflags = READDIR_DIRS_FIRST; + + if ((stat _)[3] < 2) { + # at least one non-POSIX filesystem exists + # that returns useful DT_type values: btrfs, + # so optimise for this here by requesting dents + $rdxflags |= READDIR_DENTS; + } # read the directory entries aioreq_pri $pri; - add $grp aio_readdirx $wd, READDIR_DIRS_FIRST, sub { - my $entries = shift + add $grp aio_readdirx $wd, $rdxflags, sub { + my ($entries, $flags) = @_ or return $grp->result (); + if ($rdxflags & READDIR_DENTS) { + # if we requested type values, see if we can use them directly. + + # if there were any DT_UNKNOWN entries then we assume we + # don't know. alternatively, we could assume that if we get + # one DT_DIR, then all directories are indeed marked with + # DT_DIR, but this seems not required for btrfs, and this + # is basically the "btrfs can't get it's act together" code + # branch. + unless ($flags & READDIR_FOUND_UNKNOWN) { + # now we have valid DT_ information for all entries, + # so use it as an optimisation without further stat's. + # they must also all be at the beginning of @$entries + # by now. + + my $dirs; + + if (@$entries) { + for (0 .. $#$entries) { + if ($entries->[$_][1] != DT_DIR) { + # splice out directories + $dirs = [splice @$entries, 0, $_]; + last; + } + } + + # if we didn't find any non-dir, then all entries are dirs + unless ($dirs) { + ($dirs, $entries) = ($entries, []); + } + } else { + # directory is empty, so there are no sbdirs + $dirs = []; + } + + # either splice'd the directories out or the dir was empty. + # convert dents to filenames + $_ = $_->[0] for @$dirs; + $_ = $_->[0] for @$entries; + + return $grp->result ($dirs, $entries); + } + + # cannot use, so return to our old ways + # by pretending we only scanned for names. + $_ = $_->[0] for @$entries; + } + # stat the dir another time aioreq_pri $pri; add $grp aio_stat $wd, sub { @@ -1227,7 +1327,8 @@ other processes), although if you are careful and know what you are doing, you still can. -The following constants are available (missing ones are, as usual C<0>): +The following constants are available and can be used for normal C +and C as well (missing ones are, as usual C<0>): C, @@ -1235,6 +1336,9 @@ C, C, C, C, C, C. +C, C, C, C, C and +C. + C, C, C, C, C. @@ -1252,6 +1356,11 @@ C, C, C, C, C, C, C, C, +C, C, C, C, C, C, +C, C, C, C, C, C, +C, C, C, + + =item aio_sync $callback->($status) Asynchronously call sync and call the callback when finished. @@ -1382,11 +1491,14 @@ =item aio_mlockall $flags, $callback->($status) -Calls the C function with the given C<$flags> (a combination of -C and C). +Calls the C function with the given C<$flags> (a +combination of C, C and +C). On systems that do not implement C, this function returns C<-1> -and sets errno to C. +and sets errno to C. Similarly, flag combinations not supported +by the system result in a return value of C<-1> with errno being set to +C. Note that the corresponding C is synchronous and is documented under L. @@ -1590,7 +1702,7 @@ =item IO::AIO::CWD -This is a compiletime constant (object) that represents the process +This is a compile time constant (object) that represents the process current working directory. Specifying this object as working directory object for a pathname is as if @@ -1781,6 +1893,7 @@ =back + =head2 SUPPORT FUNCTIONS =head3 EVENT PROCESSING AND EVENT LOOP INTEGRATION @@ -1855,6 +1968,16 @@ IO::AIO::poll_wait, IO::AIO::poll_cb while IO::AIO::nreqs; +This function can be useful at program aborts, to make sure outstanding +I/O has been done (C uses an C block which already calls +this function on normal exits), or when you are merely using C +for its more advanced functions, rather than for async I/O, e.g.: + + my ($dirs, $nondirs); + IO::AIO::aio_scandir "/tmp", 0, sub { ($dirs, $nondirs) = @_ }; + IO::AIO::flush; + # $dirs, $nondirs are now set + =item IO::AIO::max_poll_reqs $nreqs =item IO::AIO::max_poll_time $seconds @@ -1890,6 +2013,7 @@ =back + =head3 CONTROLLING THE NUMBER OF THREADS =over @@ -1960,12 +2084,13 @@ In other words, this setting does not enforce a queue limit, but can be used to make poll functions block if the limit is exceeded. -This is a very bad function to use in interactive programs because it -blocks, and a bad way to reduce concurrency because it is inexact: Better -use an C together with a feed callback. +This is a bad function to use in interactive programs because it blocks, +and a bad way to reduce concurrency because it is inexact. If you need to +issue many requests without being able to call a poll function on demand, +it is better to use an C together with a feed callback. -Its main use is in scripts without an event loop - when you want to stat -a lot of files, you can write something like this: +Its main use is in scripts without an event loop - when you want to stat a +lot of files, you can write something like this: IO::AIO::max_outstanding 32; @@ -1976,16 +2101,18 @@ IO::AIO::flush; -The call to C inside the loop will normally return instantly, but -as soon as more thna C<32> reqeusts are in-flight, it will block until -some requests have been handled. This keeps the loop from pushing a large -number of C requests onto the queue. +The call to C inside the loop will normally return instantly, +allowing the loop to progress, but as soon as more than C<32> requests +are in-flight, it will block until some requests have been handled. This +keeps the loop from pushing a large number of C requests onto +the queue (which, with many paths to stat, can use up a lot of memory). The default value for C is very large, so there is no practical limit on the number of outstanding requests. =back + =head3 STATISTICAL INFORMATION =over @@ -2012,6 +2139,101 @@ =back + +=head3 SUBSECOND STAT TIME ACCESS + +Both C/C and perl's C/C functions can +generally find access/modification and change times with subsecond time +accuracy of the system supports it, but perl's built-in functions only +return the integer part. + +The following functions return the timestamps of the most recent +stat with subsecond precision on most systems and work both after +C/C and perl's C/C calls. Their return +value is only meaningful after a successful C/C call, or +during/after a successful C/C callback. + +This is similar to the L C functions, but can return +full resolution without rounding and work with standard perl C, +alleviating the need to call the special C functions, which +do not act like their perl counterparts. + +On operating systems or file systems where subsecond time resolution is +not supported or could not be detected, a fractional part of C<0> is +returned, so it is always safe to call these functions. + +=over 4 + +=item $seconds = IO::AIO::st_atime, IO::AIO::st_mtime, IO::AIO::st_ctime, IO::AIO::st_btime + +Return the access, modication, change or birth time, respectively, +including fractional part. Due to the limited precision of floating point, +the accuracy on most platforms is only a bit better than milliseconds +for times around now - see the I function family, below, for full +accuracy. + +File birth time is only available when the OS and perl support it (on +FreeBSD and NetBSD at the time of this writing, although support is +adaptive, so if your OS/perl gains support, IO::AIO can take advantage of +it). On systems where it isn't available, C<0> is currently returned, but +this might change to C in a future version. + +=item ($atime, $mtime, $ctime, $btime, ...) = IO::AIO::st_xtime + +Returns access, modification, change and birth time all in one go, and +maybe more times in the future version. + +=item $nanoseconds = IO::AIO::st_atimensec, IO::AIO::st_mtimensec, IO::AIO::st_ctimensec, IO::AIO::st_btimensec + +Return the fractional access, modifcation, change or birth time, in nanoseconds, +as an integer in the range C<0> to C<999999999>. + +Note that no accessors are provided for access, modification and +change times - you need to get those from C if required (C and so on will I generally give you the correct +value). + +=item $seconds = IO::AIO::st_btimesec + +The (integral) seconds part of the file birth time, if available. + +=item ($atime, $mtime, $ctime, $btime, ...) = IO::AIO::st_xtimensec + +Like the functions above, but returns all four times in one go (and maybe +more in future versions). + +=item $counter = IO::AIO::st_gen + +Returns the generation counter (in practice this is just a random number) +of the file. This is only available on platforms which have this member in +their C (most BSDs at the time of this writing) and generally +only to the root usert. If unsupported, C<0> is returned, but this might +change to C in a future version. + +=back + +Example: print the high resolution modification time of F, using +C, and C. + + if (stat "/etc") { + printf "stat(/etc) mtime: %f\n", IO::AIO::st_mtime; + } + + IO::AIO::aio_stat "/etc", sub { + $_[0] + and return; + + printf "aio_stat(/etc) mtime: %d.%09d\n", (stat _)[9], IO::AIO::st_mtimensec; + }; + + IO::AIO::flush; + +Output of the awbove on my system, showing reduced and full accuracy: + + stat(/etc) mtime: 1534043702.020808 + aio_stat(/etc) mtime: 1534043702.020807792 + + =head3 MISCELLANEOUS FUNCTIONS IO::AIO implements some functions that are useful when you want to use @@ -2021,9 +2243,39 @@ =over 4 -=item $numfd = IO::AIO::get_fdlimit +=item $retval = IO::AIO::fexecve $fh, $argv, $envp + +A more-or-less direct equivalent to the POSIX C functions, which +allows you to specify the program to be executed via a file descriptor (or +handle). Returns C<-1> and sets errno to C if not available. + +=item $retval = IO::AIO::mount $special, $path, $fstype, $flags = 0, $data = undef -This function is I and subject to change. +Calls the GNU/Linux mount syscall with the given arguments. All except +C<$flags> are strings, and if C<$data> is C, a C will be +passed. + +The following values for C<$flags> are available: + +C, C, C, C, C, +C, C, C, C, +C, C, C, C, C, +C, C, C, C, C, +C, C, C, C, +C, C, C, C, C and +C. + +=item $retval = IO::AIO::umount $path, $flags = 0 + +Invokes the GNU/Linux C or C syscalls. Always calls +C if C<$flags> is C<0>, otherwqise always tries to call +C. + +The following C<$flags> are available: + +C, C, C and C. + +=item $numfd = IO::AIO::get_fdlimit Tries to find the current file descriptor limit and returns it, or C and sets C<$!> in case of an error. The limit is one larger than @@ -2031,8 +2283,6 @@ =item IO::AIO::min_fdlimit [$numfd] -This function is I and subject to change. - Try to increase the current file descriptor limit(s) to at least C<$numfd> by changing the soft or hard file descriptor resource limit. If C<$numfd> is missing, it will try to set a very high limit, although this is not @@ -2136,8 +2386,12 @@ C, C, C, -C or -C. +C, +C, +C, +C, +C or +C. If C<$fh> is C, then a file descriptor of C<-1> is passed. @@ -2184,6 +2438,11 @@ On systems where this call is not supported or is not emulated, this call returns falls and sets C<$!> to C. +=item IO::AIO::mlockall $flags + +Calls the C function, which is like C, +but is blocking. + =item IO::AIO::munlock $scalar, $offset = 0, $length = undef Calls the C function, undoing the effects of a previous @@ -2196,6 +2455,27 @@ On systems that do not implement C, this function returns ENOSYS, otherwise the return value of C. +=item $fh = IO::AIO::accept4 $r_fh, $sockaddr, $sockaddr_maxlen, $flags + +Uses the GNU/Linux C syscall, if available, to accept a socket +and return the new file handle on success, or sets C<$!> and returns +C on error. + +The remote name of the new socket will be stored in C<$sockaddr>, which +will be extended to allow for at least C<$sockaddr_maxlen> octets. If the +socket name does not fit into C<$sockaddr_maxlen> octets, this is signaled +by returning a longer string in C<$sockaddr>, which might or might not be +truncated. + +To accept name-less sockets, use C for C<$sockaddr> and C<0> for +C<$sockaddr_maxlen>. + +The main reasons to use this syscall rather than portable C +are that you can specify C and/or C +flags and you can accept name-less sockets by specifying C<0> for +C<$sockaddr_maxlen>, which is sadly not possible with perl's interface to +C. + =item IO::AIO::splice $r_fh, $r_off, $w_fh, $w_off, $length, $flags Calls the GNU/Linux C syscall, if available. If C<$r_off> or @@ -2248,6 +2528,90 @@ my ($rfh, $wfh) = IO::AIO::pipe2 IO::AIO::O_CLOEXEC or die "pipe2: $!\n"; +=item $fh = IO::AIO::memfd_create $pathname[, $flags] + +This is a direct interface to the Linux L system +call. The (unhelpful) default for C<$flags> is C<0>, but your default +should be C. + +On success, the new memfd filehandle is returned, otherwise returns +C. If the memfd_create syscall is missing, fails with C. + +Please refer to L for more info on this call. + +The following C<$flags> values are available: C, +C, C, +C and C. + +Example: create a new memfd. + + my $fh = IO::AIO::memfd_create "somenameforprocfd", IO::AIO::MFD_CLOEXEC + or die "memfd_create: $!\n"; + +=item $fh = IO::AIO::pidfd_open $pid[, $flags] + +This is an interface to the Linux L system call. The +default for C<$flags> is C<0>. + +On success, a new pidfd filehandle is returned (that is already set to +close-on-exec), otherwise returns C. If the syscall is missing, +fails with C. + +Example: open pid 6341 as pidfd. + + my $fh = IO::AIO::pidfd_open 6341 + or die "pidfd_open: $!\n"; + +=item $status = IO::AIO::pidfd_send_signal $pidfh, $signal[, $siginfo[, $flags]] + +This is an interface to the Linux L system call. The +default for C<$siginfo> is C and the default for C<$flags> is C<0>. + +Returns the system call status. If the syscall is missing, fails with +C. + +When specified, C<$siginfo> must be a reference to a hash with one or more +of the following members: + +=over + +=item code - the C member + +=item pid - the C member + +=item uid - the C member + +=item value_int - the C member + +=item value_ptr - the C member, specified as an integer + +=back + +Example: send a SIGKILL to the specified process. + + my $status = IO::AIO::pidfd_send_signal $pidfh, 9, undef + and die "pidfd_send_signal: $!\n"; + +Example: send a SIGKILL to the specified process with extra data. + + my $status = IO::AIO::pidfd_send_signal $pidfh, 9, { code => -1, value_int => 7 } + and die "pidfd_send_signal: $!\n"; + +=item $fh = IO::AIO::pidfd_getfd $pidfh, $targetfd[, $flags] + +This is an interface to the Linux L system call. The default +for C<$flags> is C<0>. + +On success, returns a dup'ed copy of the target file descriptor (specified +as an integer) returned (that is already set to close-on-exec), otherwise +returns C. If the syscall is missing, fails with C. + +Example: get a copy of standard error of another process and print soemthing to it. + + my $errfh = IO::AIO::pidfd_getfd $pidfh, 2 + or die "pidfd_getfd: $!\n"; + print $errfh "stderr\n"; + =item $fh = IO::AIO::eventfd [$initval, [$flags]] This is a direct interface to the Linux L system call. The @@ -2263,16 +2627,17 @@ Example: create a new eventfd filehandle: - $fh = IO::AIO::eventfd 0, IO::AIO::O_CLOEXEC + $fh = IO::AIO::eventfd 0, IO::AIO::EFD_CLOEXEC or die "eventfd: $!\n"; =item $fh = IO::AIO::timerfd_create $clockid[, $flags] -This is a direct interface to the Linux L system call. The -(unhelpful) default for C<$flags> is C<0>. +This is a direct interface to the Linux L system +call. The (unhelpful) default for C<$flags> is C<0>, but your default +should be C. On success, the new timerfd filehandle is returned, otherwise returns -C. If the eventfd syscall is missing, fails with C. +C. If the timerfd_create syscall is missing, fails with C. Please refer to L for more info on this call. @@ -2448,7 +2813,7 @@ =head1 SEE ALSO L for easy integration into event loops, L for a -more natural syntax. +more natural syntax and L for file descriptor passing. =head1 AUTHOR