--- AnyEvent-FCP/FCP.pm 2010/06/18 16:59:13 1.8 +++ AnyEvent-FCP/FCP.pm 2015/08/07 01:54:00 1.11 @@ -37,12 +37,13 @@ my $fcp = new AnyEvent::FCP; - $fcp->watch_global_sync (1, 0); - my $req = $fcp->list_persistent_requests_sync; + $fcp->watch_global (1, 0); + my $req = $fcp->list_persistent_requests; +TODO for my $req (values %$req) { if ($req->{filename} =~ /a/) { - $fcp->modify_persistent_request_sync (1, $req->{identifier}, undef, 0); + $fcp->modify_persistent_request (1, $req->{identifier}, undef, 0); } } @@ -50,7 +51,7 @@ Nothing much can be "imported" from this module right now. -=head2 THE AnyEvent::FCP CLASS +=head1 THE AnyEvent::FCP CLASS =over 4 @@ -68,23 +69,24 @@ use AnyEvent; use AnyEvent::Handle; +use AnyEvent::Util (); sub touc($) { local $_ = shift; - 1 while s/((?:^|_)(?:svk|chk|uri|fcp|ds|mime)(?:_|$))/\U$1/; + 1 while s/((?:^|_)(?:svk|chk|uri|fcp|ds|mime|dda)(?:_|$))/\U$1/; s/(?:^|_)(.)/\U$1/g; $_ } sub tolc($) { local $_ = shift; - 1 while s/(SVK|CHK|URI|FCP|DS|MIME)([^_])/$1\_$2/i; - 1 while s/([^_])(SVK|CHK|URI|FCP|DS|MIME)/$1\_$2/i; + 1 while s/(SVK|CHK|URI|FCP|DS|MIME|DDA)([^_])/$1\_$2/; + 1 while s/([^_])(SVK|CHK|URI|FCP|DS|MIME|DDA)/$1\_$2/; s/(?<=[a-z])(?=[A-Z])/_/g; lc } -=item $fcp = new AnyEvent::FCP [host => $host][, port => $port][, progress => \&cb][, name => $name] +=item $fcp = new AnyEvent::FCP [host => $host][, port => $port][, name => $name] Create a new FCP connection to the given host and port (default 127.0.0.1:9481, or the environment variables C and C). @@ -92,33 +94,20 @@ If no C was specified, then AnyEvent::FCP will generate a (hopefully) unique client name for you. -You can install a progress callback that is being called with the AnyEvent::FCP -object, the type, a hashref with key-value pairs and a reference to any received data, -for all unsolicited messages. - -Example: - - sub progress_cb { - my ($self, $type, $kv, $rdata) = @_; - - if ($type eq "simple_progress") { - warn "$kv->{identifier} $kv->{succeeded}/$kv->{required}\n"; - } - } - =cut sub new { my $class = shift; - my $self = bless { @_ }, $class; - - $self->{host} ||= $ENV{FREDHOST} || "127.0.0.1"; - $self->{port} ||= $ENV{FREDPORT} || 9481; - $self->{name} ||= time.rand.rand.rand; # lame - $self->{timeout} ||= 3600*2; - $self->{progress} ||= sub { }; - - $self->{id} = "a0"; + my $self = bless { + host => $ENV{FREDHOST} || "127.0.0.1", + port => $ENV{FREDPORT} || 9481, + timeout => 3600 * 2, + name => time.rand.rand.rand, # lame + @_, + queue => [], + req => {}, + id => "a0", + }, $class; { Scalar::Util::weaken (my $self = $self); @@ -136,9 +125,8 @@ Scalar::Util::weaken ($self->{hdl}{fcp} = $self); } - $self->send_msg ( - client_hello => - name => $self->{name}, + $self->send_msg (client_hello => + name => $self->{name}, expected_version => "2.0", ); @@ -151,9 +139,8 @@ my $data = delete $kv{data}; if (exists $kv{id_cb}) { - my $id = $kv{identifier} || ++$self->{id}; + my $id = $kv{identifier} ||= ++$self->{id}; $self->{id}{$id} = delete $kv{id_cb}; - $kv{identifier} = $id; } my $msg = (touc $type) . "\012" @@ -175,6 +162,89 @@ $self->{hdl}->push_write ($msg); } +sub on { + my ($self, $cb) = @_; + + # cb return undef - message eaten, remove cb + # cb return 0 - message eaten + # cb return 1 - pass to next + + push @{ $self->{on} }, $cb; +} + +sub _push_queue { + my ($self, $queue) = @_; + + shift @$queue; + $queue->[0]($self, AnyEvent::Util::guard { $self->_push_queue ($queue) }) + if @$queue; +} + +# lock so only one $type (arbitrary string) is in flight, +# to work around horribly misdesigned protocol. +sub serialise { + my ($self, $type, $cb) = @_; + + my $queue = $self->{serialise}{$type} ||= []; + push @$queue, $cb; + $cb->($self, AnyEvent::Util::guard { $self->_push_queue ($queue) }) + unless $#$queue; +} + +# how to merge these types into $self->{persistent} +our %PERSISTENT_TYPE = ( + persistent_get => sub { %{ $_[1] } = (type => "persistent_get" , %{ $_[2] }) }, + persistent_put => sub { %{ $_[1] } = (type => "persistent_put" , %{ $_[2] }) }, + persistent_put_dir => sub { %{ $_[1] } = (type => "persistent_put_dir", %{ $_[2] }) }, + persistent_request_modified => sub { %{ $_[1] } = (%{ $_[1] }, %{ $_[2] }) }, + persistent_request_removed => sub { delete $_[0]{req}{$_[2]{identifier}} }, + + simple_progress => sub { $_[1]{simple_progress} = $_[2] }, # get/put + + uri_generated => sub { $_[1]{uri_generated} = $_[2] }, # put + generated_metadata => sub { $_[1]{generated_metadata} = $_[2] }, # put + started_compression => sub { $_[1]{started_compression} = $_[2] }, # put + finished_compression => sub { $_[1]{finished_compression} = $_[2] }, # put + put_fetchable => sub { $_[1]{put_fetchable} = $_[2] }, # put + put_failed => sub { $_[1]{put_failed} = $_[2] }, # put + put_successful => sub { $_[1]{put_successful} = $_[2] }, # put + + sending_to_network => sub { $_[1]{sending_to_network} = $_[2] }, # get + compatibility_mode => sub { $_[1]{compatibility_mode} = $_[2] }, # get + expected_hashes => sub { $_[1]{expected_hashes} = $_[2] }, # get + expected_mime => sub { $_[1]{expected_mime} = $_[2] }, # get + expected_data_length => sub { $_[1]{expected_data_length} = $_[2] }, # get + get_failed => sub { $_[1]{get_failed} = $_[2] }, # get + data_found => sub { $_[1]{data_found} = $_[2] }, # get + enter_finite_cooldown => sub { $_[1]{enter_finite_cooldown} = $_[2] }, # get +); + +sub recv { + my ($self, $type, $kv, @extra) = @_; + + if (my $cb = $PERSISTENT_TYPE{$type}) { + my $id = $kv->{identifier}; + my $req = $_[0]{req}{$id} ||= {}; + $cb->($self, $req, $kv); + $self->recv (request_change => $kv, $type, @extra); + } + + my $on = $self->{on}; + for (0 .. $#$on) { + unless (my $res = $on->[$_]($self, $type, $kv, @extra)) { + splice @$on, $_, 1 unless defined $res; + return; + } + } + + if (my $cb = $self->{queue}[0]) { + $cb->($self, $type, $kv, @extra) + and shift @{ $self->{queue} }; + } else { + $self->default_recv ($type, $kv, @extra); + } +} + sub on_read { my ($self) = @_; @@ -182,17 +252,6 @@ my %kv; my $rdata; - my $done_cb = sub { - $kv{pkt_type} = $type; - - if (my $cb = $self->{queue}[0]) { - $cb->($self, $type, \%kv, $rdata) - and shift @{ $self->{queue} }; - } else { - $self->default_recv ($type, \%kv, $rdata); - } - }; - my $hdr_cb; $hdr_cb = sub { if ($_[1] =~ /^([^=]+)=(.*)$/) { my ($k, $v) = ($1, $2); @@ -214,10 +273,10 @@ } elsif ($_[1] eq "Data") { $_[0]->push_read (chunk => delete $kv{data_length}, sub { $rdata = \$_[1]; - $done_cb->(); + $self->recv ($type, \%kv, $rdata); }); } elsif ($_[1] eq "EndMessage") { - $done_cb->(); + $self->recv ($type, \%kv); } else { die "protocol error, expected message end, got $_[1]\n";#d# } @@ -237,30 +296,27 @@ } elsif (exists $self->{id}{$kv->{identifier}}) { $self->{id}{$kv->{identifier}}($self, $type, $kv, $rdata) and delete $self->{id}{$kv->{identifier}}; - } else { - &{ $self->{progress} }; } } +our $NOP_CB = sub { }; + sub _txn { my ($name, $sub) = @_; *{$name} = sub { splice @_, 1, 0, (my $cv = AnyEvent->condvar); &$sub; - $cv + $cv->recv }; - *{"$name\_sync"} = sub { - splice @_, 1, 0, (my $cv = AnyEvent->condvar); + *{"$name\_"} = sub { + splice @_, 1, 0, pop || $NOP_CB; &$sub; - $cv->recv }; } -=item $cv = $fcp->list_peers ([$with_metdata[, $with_volatile]]) - -=item $peers = $fcp->list_peers_sync ([$with_metdata[, $with_volatile]]) +=item $peers = $fcp->list_peers ([$with_metdata[, $with_volatile]]) =cut @@ -286,9 +342,7 @@ ); }; -=item $cv = $fcp->list_peer_notes ($node_identifier) - -=item $notes = $fcp->list_peer_notes_sync ($node_identifier) +=item $notes = $fcp->list_peer_notes ($node_identifier) =cut @@ -306,9 +360,7 @@ ); }; -=item $cv = $fcp->watch_global ($enabled[, $verbosity_mask]) - -=item $fcp->watch_global_sync ($enabled[, $verbosity_mask]) +=item $fcp->watch_global ($enabled[, $verbosity_mask]) =cut @@ -323,48 +375,42 @@ $cv->(); }; -=item $cv = $fcp->list_persistent_requests - -=item $reqs = $fcp->list_persistent_requests_sync +=item $reqs = $fcp->list_persistent_requests =cut _txn list_persistent_requests => sub { my ($self, $cv) = @_; - my %res; + $self->serialise (list_persistent_requests => sub { + my ($self, $guard) = @_; - $self->send_msg ("list_persistent_requests"); + my @res; - push @{ $self->{queue} }, sub { - my ($self, $type, $kv, $rdata) = @_; + $self->send_msg ("list_persistent_requests"); - if ($type eq "end_list_persistent_requests") { - $cv->(\%res); - 1 - } else { - my $id = $kv->{identifier}; + $self->on (sub { + my ($self, $type, $kv, $rdata) = @_; - if ($type =~ /^persistent_(get|put|put_dir)$/) { - $res{$id} = { - type => $1, - %{ $res{$id} }, - %$kv, - }; - } elsif ($type eq "simple_progress") { - delete $kv->{pkt_type}; # save memory - push @{ $res{delete $kv->{identifier}}{simple_progress} }, $kv; + $guard if 0; + + if ($type eq "end_list_persistent_requests") { + $cv->(\@res); + return; } else { - $res{delete $kv->{identifier}}{delete $kv->{pkt_type}} = $kv; + my $id = $kv->{identifier}; + + if ($type =~ /^persistent_(get|put|put_dir)$/) { + push @res, [$type, $kv]; + } } - 0 - } - }; -}; -=item $cv = $fcp->remove_request ($global, $identifier) + 1 + }); + }); +}; -=item $status = $fcp->remove_request_sync ($global, $identifier) +=item $status = $fcp->remove_request ($global, $identifier) =cut @@ -383,9 +429,7 @@ ); }; -=item $cv = $fcp->modify_persistent_request ($global, $identifier[, $client_token[, $priority_class]]) - -=item $sync = $fcp->modify_persistent_request_sync ($global, $identifier[, $client_token[, $priority_class]]) +=item $sync = $fcp->modify_persistent_request ($global, $identifier[, $client_token[, $priority_class]]) =cut @@ -406,9 +450,7 @@ ); }; -=item $cv = $fcp->get_plugin_info ($name, $detailed) - -=item $info = $fcp->get_plugin_info_sync ($name, $detailed) +=item $info = $fcp->get_plugin_info ($name, $detailed) =cut @@ -427,9 +469,7 @@ ); }; -=item $cv = $fcp->client_get ($uri, $identifier, %kv) - -=item $status = $fcp->client_get_sync ($uri, $identifier, %kv) +=item $status = $fcp->client_get ($uri, $identifier, %kv) %kv can contain (L). @@ -446,17 +486,272 @@ %kv, uri => $uri, identifier => $identifier, - id_cb => sub { - my ($self, $type, $kv, $rdata) = @_; + ); +}; + +=item $status = $fcp->remove_request ($identifier[, $global]) + +Remove the request with the given isdentifier. Returns true if successful, +false on error. + +=cut + +_txn remove_request => sub { + my ($self, $cv, $identifier, $global) = @_; + + $self->serialise ($identifier => sub { + my ($self, $guard) = @_; + + $self->send_msg (remove_request => + identifier => $identifier, + global => $global ? "true" : "false", + ); + $self->on (sub { + my ($self, $type, $kv, @extra) = @_; + + if ($kv->{identifier} eq $identifier) { + if ($type eq "persistent_request_removed") { + $cv->(1); + return; + } elsif ($type eq "protocol_error") { + $cv->(undef); + return; + } + } - $cv->($kv); 1 - }, - ); + }); + }); +}; + +=item ($can_read, $can_write) = $fcp->test_dda ($local_directory, $remote_directory, $want_read, $want_write)) + +The DDA test in FCP is probably the single most broken protocol - only +one directory test can be outstanding at any time, and some guessing and +heuristics are involved in mangling the paths. + +This function combines C and C in one +request, handling file reading and writing as well, and tries very hard to +do the right thing. + +Both C<$local_directory> and C<$remote_directory> must specify the same +directory - C<$local_directory> is the directory path on the client (where +L runs) and C<$remote_directory> is the directory path on +the server (where the freenet node runs). When both are running on the +same node, the paths are generally identical. + +C<$want_read> and C<$want_write> should be set to a true value when you +want to read (get) files or write (put) files, respectively. + +On error, an exception is thrown. Otherwise, C<$can_read> and +C<$can_write> indicate whether you can reaqd or write to freenet via the +directory. + +=cut + +_txn test_dda => sub { + my ($self, $cv, $local, $remote, $want_read, $want_write) = @_; + + $self->serialise (test_dda => sub { + my ($self, $guard) = @_; + + $self->send_msg (test_dda_request => + directory => $remote, + want_read_directory => $want_read ? "true" : "false", + want_write_directory => $want_write ? "true" : "false", + ); + $self->on (sub { + my ($self, $type, $kv) = @_; + + if ($type eq "test_dda_reply") { + # the filenames are all relative to the server-side directory, + # which might or might not match $remote anymore, so we + # need to rewrite the paths to be relative to $local + for my $k (qw(read_filename write_filename)) { + my $f = $kv->{$k}; + for my $dir ($kv->{directory}, $remote) { + if ($dir eq substr $f, 0, length $dir) { + substr $f, 0, 1 + length $dir, ""; + $kv->{$k} = $f; + last; + } + } + } + + my %response = (directory => $remote); + + if (length $kv->{read_filename}) { + if (open my $fh, "<:raw", "$local/$kv->{read_filename}") { + sysread $fh, my $buf, -s $fh; + $response{read_content} = $buf; + } + } + + if (length $kv->{write_filename}) { + if (open my $fh, ">:raw", "$local/$kv->{write_filename}") { + syswrite $fh, $kv->{content_to_write}; + } + } + + $self->send_msg (test_dda_response => %response); + + $self->on (sub { + my ($self, $type, $kv) = @_; + + $guard if 0; # reference + + if ($type eq "test_dda_complete") { + $cv->( + $kv->{read_directory_allowed} eq "true", + $kv->{write_directory_allowed} eq "true", + ); + } elsif ($type eq "protocol_error" && $kv->{identifier} eq $remote) { + $cv->croak ($kv->{extra_description}); + return; + } + + 1 + }); + + return; + } elsif ($type eq "protocol_error" && $kv->{identifier} eq $remote) { + $cv->croak ($kv->{extra_description}); + return; + } + + 1 + }); + }); }; =back +=head2 REQUEST CACHE + +The C class keeps a request cache, where it caches all +information from requests. + +For these messages, it will store a copy of the key-value pairs, together with a C slot, +in C<< $fcp->{req}{$identifier} >>: + + persistent_get + persistent_put + persistent_put_dir + +This message updates the stored data: + + persistent_request_modified + +This message will remove this entry: + + persistent_request_removed + +These messages get merged into the cache entry, under their +type, i.e. a C message will be stored in C<< +$fcp->{req}{$identifier}{simple_progress} >>: + + simple_progress # get/put + + uri_generated # put + generated_metadata # put + started_compression # put + finished_compression # put + put_failed # put + put_fetchable # put + put_successful # put + + sending_to_network # get + compatibility_mode # get + expected_hashes # get + expected_mime # get + expected_data_length # get + get_failed # get + data_found # get + enter_finite_cooldown # get + +In addition, an event (basically a fake message) of type C is generated +on every change, which will be called as C<< $cb->($fcp, $kv, $type) >>, where C<$type> +is the type of the original message triggering the change, + +To fill this cache with the global queue and keep it updated, +call C to subscribe to updates, followed by +C. + + $fcp->watch_global_sync_; # do not wait + $fcp->list_persistent_requests; # wait + +To get a better idea of what is stored in the cache, here is an example of +what might be stored in C<< $fcp->{req}{"Frost-gpl.txt"} >>: + + { + identifier => "Frost-gpl.txt", + uri => 'CHK@Fnx5kzdrfE,EImdzaVyEWl,AAIC--8/gpl.txt', + binary_blob => "false", + global => "true", + max_retries => -1, + max_size => 9223372036854775807, + persistence => "forever", + priority_class => 3, + real_time => "false", + return_type => "direct", + started => "true", + type => "persistent_get", + verbosity => 2147483647, + sending_to_network => { + identifier => "Frost-gpl.txt", + global => "true", + }, + compatibility_mode => { + identifier => "Frost-gpl.txt", + definitive => "true", + dont_compress => "false", + global => "true", + max => "COMPAT_1255", + min => "COMPAT_1255", + }, + expected_hashes => { + identifier => "Frost-gpl.txt", + global => "true", + hashes => { + ed2k => "d83596f5ee3b7...", + md5 => "e0894e4a2a6...", + sha1 => "...", + sha256 => "...", + sha512 => "...", + tth => "...", + }, + }, + expected_mime => { + identifier => "Frost-gpl.txt", + global => "true", + metadata => { content_type => "application/rar" }, + }, + expected_data_length => { + identifier => "Frost-gpl.txt", + data_length => 37576, + global => "true", + }, + simple_progress => { + identifier => "Frost-gpl.txt", + failed => 0, + fatally_failed => 0, + finalized_total => "true", + global => "true", + last_progress => 1438639282628, + required => 372, + succeeded => 102, + total => 747, + }, + data_found => { + identifier => "Frost-gpl.txt", + completion_time => 1438663354026, + data_length => 37576, + global => "true", + metadata => { content_type => "image/jpeg" }, + startup_time => 1438657196167, + }, + } + =head1 EXAMPLE PROGRAM use AnyEvent::FCP; @@ -464,12 +759,13 @@ my $fcp = new AnyEvent::FCP; # let us look at the global request list - $fcp->watch_global (1, 0); + $fcp->watch_global_ (1); # list them, synchronously - my $req = $fcp->list_persistent_requests_sync; + my $req = $fcp->list_persistent_requests; # go through all requests +TODO for my $req (values %$req) { # skip jobs not directly-to-disk next unless $req->{return_type} eq "disk";