… | |
… | |
27 | |
27 | |
28 | Special care has been taken to make this module useful from other modules, |
28 | Special care has been taken to make this module useful from other modules, |
29 | while still supporting specialised environments such as L<App::Staticperl> |
29 | while still supporting specialised environments such as L<App::Staticperl> |
30 | or L<PAR::Packer>. |
30 | or L<PAR::Packer>. |
31 | |
31 | |
32 | =head1 WHAT THIS MODULE IS NOT |
32 | =head2 WHAT THIS MODULE IS NOT |
33 | |
33 | |
34 | This module only creates processes and lets you pass file handles and |
34 | This module only creates processes and lets you pass file handles and |
35 | strings to it, and run perl code. It does not implement any kind of RPC - |
35 | strings to it, and run perl code. It does not implement any kind of RPC - |
36 | there is no back channel from the process back to you, and there is no RPC |
36 | there is no back channel from the process back to you, and there is no RPC |
37 | or message passing going on. |
37 | or message passing going on. |
… | |
… | |
40 | in whatever way you like, use some message-passing module such |
40 | in whatever way you like, use some message-passing module such |
41 | as L<AnyEvent::MP>, some pipe such as L<AnyEvent::ZeroMQ>, use |
41 | as L<AnyEvent::MP>, some pipe such as L<AnyEvent::ZeroMQ>, use |
42 | L<AnyEvent::Handle> on both sides to send e.g. JSON or Storable messages, |
42 | L<AnyEvent::Handle> on both sides to send e.g. JSON or Storable messages, |
43 | and so on. |
43 | and so on. |
44 | |
44 | |
|
|
45 | =head2 COMPARISON TO OTHER MODULES |
|
|
46 | |
|
|
47 | There is an abundance of modules on CPAN that do "something fork", such as |
|
|
48 | L<Parallel::ForkManager>, L<AnyEvent::ForkManager>, L<AnyEvent::Worker> |
|
|
49 | or L<AnyEvent::Subprocess>. There are modules that implement their own |
|
|
50 | process management, such as L<AnyEvent::DBI>. |
|
|
51 | |
|
|
52 | The problems that all these modules try to solve are real, however, none |
|
|
53 | of them (from what I have seen) tackle the very real problems of unwanted |
|
|
54 | memory sharing, efficiency, not being able to use event processing or |
|
|
55 | similar modules in the processes they create. |
|
|
56 | |
|
|
57 | This module doesn't try to replace any of them - instead it tries to solve |
|
|
58 | the problem of creating processes with a minimum of fuss and overhead (and |
|
|
59 | also luxury). Ideally, most of these would use AnyEvent::Fork internally, |
|
|
60 | except they were written before AnyEvent:Fork was available, so obviously |
|
|
61 | had to roll their own. |
|
|
62 | |
45 | =head1 PROBLEM STATEMENT |
63 | =head2 PROBLEM STATEMENT |
46 | |
64 | |
47 | There are two traditional ways to implement parallel processing on UNIX |
65 | There are two traditional ways to implement parallel processing on UNIX |
48 | like operating systems - fork and process, and fork+exec and process. They |
66 | like operating systems - fork and process, and fork+exec and process. They |
49 | have different advantages and disadvantages that I describe below, |
67 | have different advantages and disadvantages that I describe below, |
50 | together with how this module tries to mitigate the disadvantages. |
68 | together with how this module tries to mitigate the disadvantages. |
… | |
… | |
212 | open my $output, ">/tmp/log" or die "$!"; |
230 | open my $output, ">/tmp/log" or die "$!"; |
213 | |
231 | |
214 | AnyEvent::Fork |
232 | AnyEvent::Fork |
215 | ->new |
233 | ->new |
216 | ->eval (' |
234 | ->eval (' |
|
|
235 | # compile a helper function for later use |
217 | sub run { |
236 | sub run { |
218 | my ($fh, $output, @cmd) = @_; |
237 | my ($fh, $output, @cmd) = @_; |
219 | |
238 | |
220 | # perl will clear close-on-exec on STDOUT/STDERR |
239 | # perl will clear close-on-exec on STDOUT/STDERR |
221 | open STDOUT, ">&", $output or die; |
240 | open STDOUT, ">&", $output or die; |
… | |
… | |
351 | use AnyEvent; |
370 | use AnyEvent; |
352 | use AnyEvent::Util (); |
371 | use AnyEvent::Util (); |
353 | |
372 | |
354 | use IO::FDPass; |
373 | use IO::FDPass; |
355 | |
374 | |
356 | our $VERSION = 0.5; |
375 | our $VERSION = 0.6; |
357 | |
|
|
358 | our $PERL; # the path to the perl interpreter, deduces with various forms of magic |
|
|
359 | |
376 | |
360 | =over 4 |
377 | =over 4 |
361 | |
378 | |
362 | =back |
379 | =back |
363 | |
380 | |
… | |
… | |
366 | # the early fork template process |
383 | # the early fork template process |
367 | our $EARLY; |
384 | our $EARLY; |
368 | |
385 | |
369 | # the empty template process |
386 | # the empty template process |
370 | our $TEMPLATE; |
387 | our $TEMPLATE; |
|
|
388 | |
|
|
389 | sub QUEUE() { 0 } |
|
|
390 | sub FH() { 1 } |
|
|
391 | sub WW() { 2 } |
|
|
392 | sub PID() { 3 } |
|
|
393 | sub CB() { 4 } |
|
|
394 | |
|
|
395 | sub _new { |
|
|
396 | my ($self, $fh, $pid) = @_; |
|
|
397 | |
|
|
398 | AnyEvent::Util::fh_nonblocking $fh, 1; |
|
|
399 | |
|
|
400 | $self = bless [ |
|
|
401 | [], # write queue - strings or fd's |
|
|
402 | $fh, |
|
|
403 | undef, # AE watcher |
|
|
404 | $pid, |
|
|
405 | ], $self; |
|
|
406 | |
|
|
407 | $self |
|
|
408 | } |
371 | |
409 | |
372 | sub _cmd { |
410 | sub _cmd { |
373 | my $self = shift; |
411 | my $self = shift; |
374 | |
412 | |
375 | # ideally, we would want to use "a (w/a)*" as format string, but perl |
413 | # ideally, we would want to use "a (w/a)*" as format string, but perl |
376 | # versions from at least 5.8.9 to 5.16.3 are all buggy and can't unpack |
414 | # versions from at least 5.8.9 to 5.16.3 are all buggy and can't unpack |
377 | # it. |
415 | # it. |
378 | push @{ $self->[2] }, pack "a L/a*", $_[0], $_[1]; |
416 | push @{ $self->[QUEUE] }, pack "a L/a*", $_[0], $_[1]; |
379 | |
417 | |
380 | $self->[3] ||= AE::io $self->[1], 1, sub { |
418 | $self->[WW] ||= AE::io $self->[FH], 1, sub { |
381 | do { |
419 | do { |
382 | # send the next "thing" in the queue - either a reference to an fh, |
420 | # send the next "thing" in the queue - either a reference to an fh, |
383 | # or a plain string. |
421 | # or a plain string. |
384 | |
422 | |
385 | if (ref $self->[2][0]) { |
423 | if (ref $self->[QUEUE][0]) { |
386 | # send fh |
424 | # send fh |
387 | unless (IO::FDPass::send fileno $self->[1], fileno ${ $self->[2][0] }) { |
425 | unless (IO::FDPass::send fileno $self->[FH], fileno ${ $self->[QUEUE][0] }) { |
388 | return if $! == Errno::EAGAIN || $! == Errno::EWOULDBLOCK; |
426 | return if $! == Errno::EAGAIN || $! == Errno::EWOULDBLOCK; |
389 | undef $self->[3]; |
427 | undef $self->[WW]; |
390 | die "AnyEvent::Fork: file descriptor send failure: $!"; |
428 | die "AnyEvent::Fork: file descriptor send failure: $!"; |
391 | } |
429 | } |
392 | |
430 | |
393 | shift @{ $self->[2] }; |
431 | shift @{ $self->[QUEUE] }; |
394 | |
432 | |
395 | } else { |
433 | } else { |
396 | # send string |
434 | # send string |
397 | my $len = syswrite $self->[1], $self->[2][0]; |
435 | my $len = syswrite $self->[FH], $self->[QUEUE][0]; |
398 | |
436 | |
399 | unless ($len) { |
437 | unless ($len) { |
400 | return if $! == Errno::EAGAIN || $! == Errno::EWOULDBLOCK; |
438 | return if $! == Errno::EAGAIN || $! == Errno::EWOULDBLOCK; |
401 | undef $self->[3]; |
439 | undef $self->[3]; |
402 | die "AnyEvent::Fork: command write failure: $!"; |
440 | die "AnyEvent::Fork: command write failure: $!"; |
403 | } |
441 | } |
404 | |
442 | |
405 | substr $self->[2][0], 0, $len, ""; |
443 | substr $self->[QUEUE][0], 0, $len, ""; |
406 | shift @{ $self->[2] } unless length $self->[2][0]; |
444 | shift @{ $self->[QUEUE] } unless length $self->[QUEUE][0]; |
407 | } |
445 | } |
408 | } while @{ $self->[2] }; |
446 | } while @{ $self->[QUEUE] }; |
409 | |
447 | |
410 | # everything written |
448 | # everything written |
411 | undef $self->[3]; |
449 | undef $self->[WW]; |
412 | |
450 | |
413 | # invoke run callback, if any |
451 | # invoke run callback, if any |
414 | $self->[4]->($self->[1]) if $self->[4]; |
452 | $self->[CB]->($self->[FH]) if $self->[CB]; |
415 | }; |
453 | }; |
416 | |
454 | |
417 | () # make sure we don't leak the watcher |
455 | () # make sure we don't leak the watcher |
418 | } |
|
|
419 | |
|
|
420 | sub _new { |
|
|
421 | my ($self, $fh, $pid) = @_; |
|
|
422 | |
|
|
423 | AnyEvent::Util::fh_nonblocking $fh, 1; |
|
|
424 | |
|
|
425 | $self = bless [ |
|
|
426 | $pid, |
|
|
427 | $fh, |
|
|
428 | [], # write queue - strings or fd's |
|
|
429 | undef, # AE watcher |
|
|
430 | ], $self; |
|
|
431 | |
|
|
432 | $self |
|
|
433 | } |
456 | } |
434 | |
457 | |
435 | # fork template from current process, used by AnyEvent::Fork::Early/Template |
458 | # fork template from current process, used by AnyEvent::Fork::Early/Template |
436 | sub _new_fork { |
459 | sub _new_fork { |
437 | my ($fh, $slave) = AnyEvent::Util::portable_socketpair; |
460 | my ($fh, $slave) = AnyEvent::Util::portable_socketpair; |
… | |
… | |
442 | if ($pid eq 0) { |
465 | if ($pid eq 0) { |
443 | require AnyEvent::Fork::Serve; |
466 | require AnyEvent::Fork::Serve; |
444 | $AnyEvent::Fork::Serve::OWNER = $parent; |
467 | $AnyEvent::Fork::Serve::OWNER = $parent; |
445 | close $fh; |
468 | close $fh; |
446 | $0 = "$_[1] of $parent"; |
469 | $0 = "$_[1] of $parent"; |
447 | $SIG{CHLD} = 'IGNORE'; |
|
|
448 | AnyEvent::Fork::Serve::serve ($slave); |
470 | AnyEvent::Fork::Serve::serve ($slave); |
449 | exit 0; |
471 | exit 0; |
450 | } elsif (!$pid) { |
472 | } elsif (!$pid) { |
451 | die "AnyEvent::Fork::Early/Template: unable to fork template process: $!"; |
473 | die "AnyEvent::Fork::Early/Template: unable to fork template process: $!"; |
452 | } |
474 | } |
… | |
… | |
571 | AnyEvent::Fork itself. |
593 | AnyEvent::Fork itself. |
572 | |
594 | |
573 | =cut |
595 | =cut |
574 | |
596 | |
575 | sub pid { |
597 | sub pid { |
576 | $_[0][0] |
598 | $_[0][PID] |
577 | } |
599 | } |
578 | |
600 | |
579 | =item $proc = $proc->eval ($perlcode, @args) |
601 | =item $proc = $proc->eval ($perlcode, @args) |
580 | |
602 | |
581 | Evaluates the given C<$perlcode> as ... perl code, while setting C<@_> to |
603 | Evaluates the given C<$perlcode> as ... perl code, while setting C<@_> to |
… | |
… | |
648 | sub send_fh { |
670 | sub send_fh { |
649 | my ($self, @fh) = @_; |
671 | my ($self, @fh) = @_; |
650 | |
672 | |
651 | for my $fh (@fh) { |
673 | for my $fh (@fh) { |
652 | $self->_cmd ("h"); |
674 | $self->_cmd ("h"); |
653 | push @{ $self->[2] }, \$fh; |
675 | push @{ $self->[QUEUE] }, \$fh; |
654 | } |
676 | } |
655 | |
677 | |
656 | $self |
678 | $self |
657 | } |
679 | } |
658 | |
680 | |
… | |
… | |
744 | =cut |
766 | =cut |
745 | |
767 | |
746 | sub run { |
768 | sub run { |
747 | my ($self, $func, $cb) = @_; |
769 | my ($self, $func, $cb) = @_; |
748 | |
770 | |
749 | $self->[4] = $cb; |
771 | $self->[CB] = $cb; |
750 | $self->_cmd (r => $func); |
772 | $self->_cmd (r => $func); |
751 | } |
773 | } |
752 | |
774 | |
753 | =back |
775 | =back |
754 | |
776 | |
… | |
… | |
851 | L<AnyEvent::Fork::Early> or L<AnyEvent::Fork::Template>, or to delay |
873 | L<AnyEvent::Fork::Early> or L<AnyEvent::Fork::Template>, or to delay |
852 | initialising them, for example, by calling C<init Gtk2> manually. |
874 | initialising them, for example, by calling C<init Gtk2> manually. |
853 | |
875 | |
854 | =item exiting calls object destructors |
876 | =item exiting calls object destructors |
855 | |
877 | |
856 | This only applies to users of Lc<AnyEvent::Fork:Early> and |
878 | This only applies to users of L<AnyEvent::Fork:Early> and |
857 | L<AnyEvent::Fork::Template>. |
879 | L<AnyEvent::Fork::Template>, or when initialiasing code creates objects |
|
|
880 | that reference external resources. |
858 | |
881 | |
859 | When a process created by AnyEvent::Fork exits, it might do so by calling |
882 | When a process created by AnyEvent::Fork exits, it might do so by calling |
860 | exit, or simply letting perl reach the end of the program. At which point |
883 | exit, or simply letting perl reach the end of the program. At which point |
861 | Perl runs all destructors. |
884 | Perl runs all destructors. |
862 | |
885 | |