… | |
… | |
134 | aio_lstat aio_unlink aio_rmdir aio_readdir aio_scandir aio_symlink |
134 | aio_lstat aio_unlink aio_rmdir aio_readdir aio_scandir aio_symlink |
135 | aio_fsync aio_fdatasync aio_readahead aio_rename aio_link aio_move |
135 | aio_fsync aio_fdatasync aio_readahead aio_rename aio_link aio_move |
136 | aio_group aio_nop); |
136 | aio_group aio_nop); |
137 | our @EXPORT = (@AIO_REQ, qw(aioreq_pri aioreq_nice)); |
137 | our @EXPORT = (@AIO_REQ, qw(aioreq_pri aioreq_nice)); |
138 | our @EXPORT_OK = qw(poll_fileno poll_cb poll_wait flush |
138 | our @EXPORT_OK = qw(poll_fileno poll_cb poll_wait flush |
139 | min_parallel max_parallel max_outstanding nreqs); |
139 | min_parallel max_parallel nreqs); |
140 | |
140 | |
141 | @IO::AIO::GRP::ISA = 'IO::AIO::REQ'; |
141 | @IO::AIO::GRP::ISA = 'IO::AIO::REQ'; |
142 | |
142 | |
143 | require XSLoader; |
143 | require XSLoader; |
144 | XSLoader::load ("IO::AIO", $VERSION); |
144 | XSLoader::load ("IO::AIO", $VERSION); |
… | |
… | |
426 | with the filenames. |
426 | with the filenames. |
427 | |
427 | |
428 | =item aio_scandir $path, $maxreq, $callback->($dirs, $nondirs) |
428 | =item aio_scandir $path, $maxreq, $callback->($dirs, $nondirs) |
429 | |
429 | |
430 | Scans a directory (similar to C<aio_readdir>) but additionally tries to |
430 | Scans a directory (similar to C<aio_readdir>) but additionally tries to |
431 | separate the entries of directory C<$path> into two sets of names, ones |
431 | efficiently separate the entries of directory C<$path> into two sets of |
432 | you can recurse into (directories or links to them), and ones you cannot |
432 | names, directories you can recurse into (directories), and ones you cannot |
433 | recurse into (everything else). |
433 | recurse into (everything else, including symlinks to directories). |
434 | |
434 | |
435 | C<aio_scandir> is a composite request that creates of many sub requests_ |
435 | C<aio_scandir> is a composite request that creates of many sub requests_ |
436 | C<$maxreq> specifies the maximum number of outstanding aio requests that |
436 | C<$maxreq> specifies the maximum number of outstanding aio requests that |
437 | this function generates. If it is C<< <= 0 >>, then a suitable default |
437 | this function generates. If it is C<< <= 0 >>, then a suitable default |
438 | will be chosen (currently 6). |
438 | will be chosen (currently 6). |
… | |
… | |
520 | map [$_, sprintf "%s%04d", (/.\./ ? "1" : "0"), length], |
520 | map [$_, sprintf "%s%04d", (/.\./ ? "1" : "0"), length], |
521 | @$entries]; |
521 | @$entries]; |
522 | |
522 | |
523 | my (@dirs, @nondirs); |
523 | my (@dirs, @nondirs); |
524 | |
524 | |
525 | my ($statcb, $schedcb); |
|
|
526 | my $nreq = 0; |
|
|
527 | |
|
|
528 | my $statgrp = add $grp aio_group; |
525 | my $statgrp = add $grp aio_group sub { |
|
|
526 | $grp->result (\@dirs, \@nondirs); |
|
|
527 | }; |
529 | |
528 | |
530 | $schedcb = sub { |
529 | limit $statgrp $maxreq; |
531 | if (@$entries) { |
530 | feed $statgrp sub { |
532 | if ($nreq < $maxreq) { |
531 | return unless @$entries; |
533 | my $ent = pop @$entries; |
532 | my $entry = pop @$entries; |
|
|
533 | |
|
|
534 | add $statgrp aio_stat "$path/$entry/.", sub { |
|
|
535 | if ($_[0] < 0) { |
|
|
536 | push @nondirs, $entry; |
|
|
537 | } else { |
|
|
538 | # need to check for real directory |
|
|
539 | add $statgrp aio_lstat "$path/$entry", sub { |
|
|
540 | if (-d _) { |
|
|
541 | push @dirs, $entry; |
|
|
542 | |
|
|
543 | unless (--$ndirs) { |
|
|
544 | push @nondirs, @$entries; |
|
|
545 | feed $statgrp; |
|
|
546 | } |
|
|
547 | } else { |
|
|
548 | push @nondirs, $entry; |
|
|
549 | } |
534 | $nreq++; |
550 | } |
535 | add $statgrp aio_stat "$path/$ent/.", sub { $statcb->($_[0], $ent) }; |
|
|
536 | } |
551 | } |
537 | } elsif (!$nreq) { |
|
|
538 | # finished |
|
|
539 | $statgrp->cancel; |
|
|
540 | undef $statcb; |
|
|
541 | undef $schedcb; |
|
|
542 | $grp->result (\@dirs, \@nondirs); |
|
|
543 | } |
552 | }; |
544 | }; |
553 | }; |
545 | $statcb = sub { |
|
|
546 | my ($status, $entry) = @_; |
|
|
547 | |
|
|
548 | if ($status < 0) { |
|
|
549 | $nreq--; |
|
|
550 | push @nondirs, $entry; |
|
|
551 | &$schedcb; |
|
|
552 | } else { |
|
|
553 | # need to check for real directory |
|
|
554 | add $grp aio_lstat "$path/$entry", sub { |
|
|
555 | $nreq--; |
|
|
556 | |
|
|
557 | if (-d _) { |
|
|
558 | push @dirs, $entry; |
|
|
559 | |
|
|
560 | if (!--$ndirs) { |
|
|
561 | push @nondirs, @$entries; |
|
|
562 | $entries = []; |
|
|
563 | } |
|
|
564 | } else { |
|
|
565 | push @nondirs, $entry; |
|
|
566 | } |
|
|
567 | |
|
|
568 | &$schedcb; |
|
|
569 | } |
|
|
570 | } |
|
|
571 | }; |
|
|
572 | |
|
|
573 | &$schedcb while @$entries && $nreq < $maxreq; |
|
|
574 | }; |
554 | }; |
575 | }; |
555 | }; |
576 | }; |
556 | }; |
577 | |
557 | |
578 | $grp |
558 | $grp |
… | |
… | |
590 | |
570 | |
591 | If this call isn't available because your OS lacks it or it couldn't be |
571 | If this call isn't available because your OS lacks it or it couldn't be |
592 | detected, it will be emulated by calling C<fsync> instead. |
572 | detected, it will be emulated by calling C<fsync> instead. |
593 | |
573 | |
594 | =item aio_group $callback->(...) |
574 | =item aio_group $callback->(...) |
595 | |
|
|
596 | [EXPERIMENTAL] |
|
|
597 | |
575 | |
598 | This is a very special aio request: Instead of doing something, it is a |
576 | This is a very special aio request: Instead of doing something, it is a |
599 | container for other aio requests, which is useful if you want to bundle |
577 | container for other aio requests, which is useful if you want to bundle |
600 | many requests into a single, composite, request with a definite callback |
578 | many requests into a single, composite, request with a definite callback |
601 | and the ability to cancel the whole request with its subrequests. |
579 | and the ability to cancel the whole request with its subrequests. |
… | |
… | |
704 | =item * They can also can also be added to other IO::AIO::GRP objects. |
682 | =item * They can also can also be added to other IO::AIO::GRP objects. |
705 | |
683 | |
706 | =item * You must not add requests to a group from within the group callback (or |
684 | =item * You must not add requests to a group from within the group callback (or |
707 | any later time). |
685 | any later time). |
708 | |
686 | |
709 | =item * This does not harmonise well with C<max_outstanding>, so best do |
|
|
710 | not combine C<aio_group> with it. Groups and feeders are recommended for |
|
|
711 | this kind of concurrency-limiting. |
|
|
712 | |
|
|
713 | =back |
687 | =back |
714 | |
688 | |
715 | Their lifetime, simplified, looks like this: when they are empty, they |
689 | Their lifetime, simplified, looks like this: when they are empty, they |
716 | will finish very quickly. If they contain only requests that are in the |
690 | will finish very quickly. If they contain only requests that are in the |
717 | C<done> state, they will also finish. Otherwise they will continue to |
691 | C<done> state, they will also finish. Otherwise they will continue to |
… | |
… | |
732 | be added, including other groups, as long as you do not create circular |
706 | be added, including other groups, as long as you do not create circular |
733 | dependencies. |
707 | dependencies. |
734 | |
708 | |
735 | Returns all its arguments. |
709 | Returns all its arguments. |
736 | |
710 | |
|
|
711 | =item $grp->cancel_subs |
|
|
712 | |
|
|
713 | Cancel all subrequests and clears any feeder, but not the group request |
|
|
714 | itself. Useful when you queued a lot of events but got a result early. |
|
|
715 | |
737 | =item $grp->result (...) |
716 | =item $grp->result (...) |
738 | |
717 | |
739 | Set the result value(s) that will be passed to the group callback when all |
718 | Set the result value(s) that will be passed to the group callback when all |
740 | subrequests have finished. By default, no argument will be passed. |
719 | subrequests have finished. By default, no argument will be passed. |
741 | |
720 | |
742 | =item feed $grp $callback->($grp) |
721 | =item feed $grp $callback->($grp) |
743 | |
|
|
744 | [VERY EXPERIMENTAL] |
|
|
745 | |
722 | |
746 | Sets a feeder/generator on this group: every group can have an attached |
723 | Sets a feeder/generator on this group: every group can have an attached |
747 | generator that generates requests if idle. The idea behind this is that, |
724 | generator that generates requests if idle. The idea behind this is that, |
748 | although you could just queue as many requests as you want in a group, |
725 | although you could just queue as many requests as you want in a group, |
749 | this might starve other requests for a potentially long time. For |
726 | this might starve other requests for a potentially long time. For |
… | |
… | |
803 | |
780 | |
804 | Process all outstanding events on the result pipe. You have to call this |
781 | Process all outstanding events on the result pipe. You have to call this |
805 | regularly. Returns the number of events processed. Returns immediately |
782 | regularly. Returns the number of events processed. Returns immediately |
806 | when no events are outstanding. |
783 | when no events are outstanding. |
807 | |
784 | |
|
|
785 | If not all requests were processed for whatever reason, the filehandle |
|
|
786 | will still be ready when C<poll_cb> returns. |
|
|
787 | |
808 | Example: Install an Event watcher that automatically calls |
788 | Example: Install an Event watcher that automatically calls |
809 | IO::AIO::poll_cb with high priority: |
789 | IO::AIO::poll_cb with high priority: |
810 | |
790 | |
811 | Event->io (fd => IO::AIO::poll_fileno, |
791 | Event->io (fd => IO::AIO::poll_fileno, |
812 | poll => 'r', async => 1, |
792 | poll => 'r', async => 1, |
813 | cb => \&IO::AIO::poll_cb); |
793 | cb => \&IO::AIO::poll_cb); |
|
|
794 | |
|
|
795 | =item IO::AIO::poll_some $max_requests |
|
|
796 | |
|
|
797 | Similar to C<poll_cb>, but only processes up to C<$max_requests> requests |
|
|
798 | at a time. |
|
|
799 | |
|
|
800 | Useful if you want to ensure some level of interactiveness when perl is |
|
|
801 | not fast enough to process all requests in time. |
|
|
802 | |
|
|
803 | Example: Install an Event watcher that automatically calls |
|
|
804 | IO::AIO::poll_some with low priority, to ensure that other parts of the |
|
|
805 | program get the CPU sometimes even under high AIO load. |
|
|
806 | |
|
|
807 | Event->io (fd => IO::AIO::poll_fileno, |
|
|
808 | poll => 'r', nice => 1, |
|
|
809 | cb => sub { IO::AIO::poll_some 256 }); |
814 | |
810 | |
815 | =item IO::AIO::poll_wait |
811 | =item IO::AIO::poll_wait |
816 | |
812 | |
817 | Wait till the result filehandle becomes ready for reading (simply does a |
813 | Wait till the result filehandle becomes ready for reading (simply does a |
818 | C<select> on the filehandle. This is useful if you want to synchronously wait |
814 | C<select> on the filehandle. This is useful if you want to synchronously wait |
… | |
… | |
878 | This module automatically runs C<max_parallel 0> at program end, to ensure |
874 | This module automatically runs C<max_parallel 0> at program end, to ensure |
879 | that all threads are killed and that there are no outstanding requests. |
875 | that all threads are killed and that there are no outstanding requests. |
880 | |
876 | |
881 | Under normal circumstances you don't need to call this function. |
877 | Under normal circumstances you don't need to call this function. |
882 | |
878 | |
883 | =item $oldnreqs = IO::AIO::max_outstanding $nreqs |
879 | =item $oldmaxreqs = IO::AIO::max_outstanding $maxreqs |
884 | |
880 | |
885 | [DEPRECATED] |
881 | This is a very bad function to use in interactive programs because it |
|
|
882 | blocks, and a bad way to reduce concurrency because it is inexact: Better |
|
|
883 | use an C<aio_group> together with a feed callback. |
886 | |
884 | |
887 | Sets the maximum number of outstanding requests to C<$nreqs>. If you |
885 | Sets the maximum number of outstanding requests to C<$nreqs>. If you |
888 | try to queue up more than this number of requests, the caller will block until |
886 | to queue up more than this number of requests, the next call to the |
889 | some requests have been handled. |
887 | C<poll_cb> (and C<poll_some> and other functions calling C<poll_cb>) |
|
|
888 | function will block until the limit is no longer exceeded. |
890 | |
889 | |
891 | The default is very large, so normally there is no practical limit. If you |
890 | The default value is very large, so there is no practical limit on the |
892 | queue up many requests in a loop it often improves speed if you set |
891 | number of outstanding requests. |
893 | this to a relatively low number, such as C<100>. |
|
|
894 | |
892 | |
895 | This function does not work well together with C<aio_group>'s, and their |
893 | You can still queue as many requests as you want. Therefore, |
896 | feeder interface is better suited to limiting concurrency, so do not use |
894 | C<max_oustsanding> is mainly useful in simple scripts (with low values) or |
897 | this function. |
895 | as a stop gap to shield against fatal memory overflow (with large values). |
898 | |
|
|
899 | Under normal circumstances you don't need to call this function. |
|
|
900 | |
896 | |
901 | =back |
897 | =back |
902 | |
898 | |
903 | =cut |
899 | =cut |
904 | |
900 | |
… | |
… | |
936 | request/result processing, while the child frees the request/result queue |
932 | request/result processing, while the child frees the request/result queue |
937 | (so that the requests started before the fork will only be handled in the |
933 | (so that the requests started before the fork will only be handled in the |
938 | parent). Threads will be started on demand until the limit set in the |
934 | parent). Threads will be started on demand until the limit set in the |
939 | parent process has been reached again. |
935 | parent process has been reached again. |
940 | |
936 | |
941 | Temporary memory that was allocated for request processing is not |
|
|
942 | reclaimed in the child, however. While this is possible in some cases, it |
|
|
943 | is almost impossible in others (threads are evil you know), so you will |
|
|
944 | have to live with it. This is around 64k buffer (for sendfile, readahead |
|
|
945 | emulation) + the size of the directory being scanned (readdir). |
|
|
946 | |
|
|
947 | In short: the parent will, after a short pause, continue as if fork had |
937 | In short: the parent will, after a short pause, continue as if fork had |
948 | not been called, while the child will act as if IO::AIO has not been used |
938 | not been called, while the child will act as if IO::AIO has not been used |
949 | yet. |
939 | yet. |
950 | |
940 | |
951 | =head2 MEMORY USAGE |
941 | =head2 MEMORY USAGE |
… | |
… | |
967 | temporary buffers, and each thread requires a stack and other data |
957 | temporary buffers, and each thread requires a stack and other data |
968 | structures (usually around 16k-128k, depending on the OS). |
958 | structures (usually around 16k-128k, depending on the OS). |
969 | |
959 | |
970 | =head1 KNOWN BUGS |
960 | =head1 KNOWN BUGS |
971 | |
961 | |
972 | See FORK BEHAVIOUR, above. |
962 | Known bugs will be fixed in the next release. |
973 | |
963 | |
974 | =head1 SEE ALSO |
964 | =head1 SEE ALSO |
975 | |
965 | |
976 | L<Coro::AIO>. |
966 | L<Coro::AIO>. |
977 | |
967 | |