--- AnyEvent-FCP/FCP.pm 2015/08/04 00:50:25 1.10 +++ AnyEvent-FCP/FCP.pm 2015/08/07 01:54:00 1.11 @@ -37,12 +37,13 @@ my $fcp = new AnyEvent::FCP; - $fcp->watch_global_sync (1, 0); - my $req = $fcp->list_persistent_requests_sync; + $fcp->watch_global (1, 0); + my $req = $fcp->list_persistent_requests; +TODO for my $req (values %$req) { if ($req->{filename} =~ /a/) { - $fcp->modify_persistent_request_sync (1, $req->{identifier}, undef, 0); + $fcp->modify_persistent_request (1, $req->{identifier}, undef, 0); } } @@ -50,7 +51,7 @@ Nothing much can be "imported" from this module right now. -=head2 THE AnyEvent::FCP CLASS +=head1 THE AnyEvent::FCP CLASS =over 4 @@ -79,13 +80,13 @@ sub tolc($) { local $_ = shift; - 1 while s/(SVK|CHK|URI|FCP|DS|MIME|DDA)([^_])/$1\_$2/i; - 1 while s/([^_])(SVK|CHK|URI|FCP|DS|MIME|DDA)/$1\_$2/i; + 1 while s/(SVK|CHK|URI|FCP|DS|MIME|DDA)([^_])/$1\_$2/; + 1 while s/([^_])(SVK|CHK|URI|FCP|DS|MIME|DDA)/$1\_$2/; s/(?<=[a-z])(?=[A-Z])/_/g; lc } -=item $fcp = new AnyEvent::FCP [host => $host][, port => $port][, progress => \&cb][, name => $name] +=item $fcp = new AnyEvent::FCP [host => $host][, port => $port][, name => $name] Create a new FCP connection to the given host and port (default 127.0.0.1:9481, or the environment variables C and C). @@ -93,33 +94,20 @@ If no C was specified, then AnyEvent::FCP will generate a (hopefully) unique client name for you. -You can install a progress callback that is being called with the AnyEvent::FCP -object, the type, a hashref with key-value pairs and a reference to any received data, -for all unsolicited messages. - -Example: - - sub progress_cb { - my ($self, $type, $kv, $rdata) = @_; - - if ($type eq "simple_progress") { - warn "$kv->{identifier} $kv->{succeeded}/$kv->{required}\n"; - } - } - =cut sub new { my $class = shift; - my $self = bless { @_ }, $class; - - $self->{host} ||= $ENV{FREDHOST} || "127.0.0.1"; - $self->{port} ||= $ENV{FREDPORT} || 9481; - $self->{name} ||= time.rand.rand.rand; # lame - $self->{timeout} ||= 3600*2; - $self->{progress} ||= sub { }; - - $self->{id} = "a0"; + my $self = bless { + host => $ENV{FREDHOST} || "127.0.0.1", + port => $ENV{FREDPORT} || 9481, + timeout => 3600 * 2, + name => time.rand.rand.rand, # lame + @_, + queue => [], + req => {}, + id => "a0", + }, $class; { Scalar::Util::weaken (my $self = $self); @@ -137,9 +125,8 @@ Scalar::Util::weaken ($self->{hdl}{fcp} = $self); } - $self->send_msg ( - client_hello => - name => $self->{name}, + $self->send_msg (client_hello => + name => $self->{name}, expected_version => "2.0", ); @@ -204,6 +191,60 @@ unless $#$queue; } +# how to merge these types into $self->{persistent} +our %PERSISTENT_TYPE = ( + persistent_get => sub { %{ $_[1] } = (type => "persistent_get" , %{ $_[2] }) }, + persistent_put => sub { %{ $_[1] } = (type => "persistent_put" , %{ $_[2] }) }, + persistent_put_dir => sub { %{ $_[1] } = (type => "persistent_put_dir", %{ $_[2] }) }, + persistent_request_modified => sub { %{ $_[1] } = (%{ $_[1] }, %{ $_[2] }) }, + persistent_request_removed => sub { delete $_[0]{req}{$_[2]{identifier}} }, + + simple_progress => sub { $_[1]{simple_progress} = $_[2] }, # get/put + + uri_generated => sub { $_[1]{uri_generated} = $_[2] }, # put + generated_metadata => sub { $_[1]{generated_metadata} = $_[2] }, # put + started_compression => sub { $_[1]{started_compression} = $_[2] }, # put + finished_compression => sub { $_[1]{finished_compression} = $_[2] }, # put + put_fetchable => sub { $_[1]{put_fetchable} = $_[2] }, # put + put_failed => sub { $_[1]{put_failed} = $_[2] }, # put + put_successful => sub { $_[1]{put_successful} = $_[2] }, # put + + sending_to_network => sub { $_[1]{sending_to_network} = $_[2] }, # get + compatibility_mode => sub { $_[1]{compatibility_mode} = $_[2] }, # get + expected_hashes => sub { $_[1]{expected_hashes} = $_[2] }, # get + expected_mime => sub { $_[1]{expected_mime} = $_[2] }, # get + expected_data_length => sub { $_[1]{expected_data_length} = $_[2] }, # get + get_failed => sub { $_[1]{get_failed} = $_[2] }, # get + data_found => sub { $_[1]{data_found} = $_[2] }, # get + enter_finite_cooldown => sub { $_[1]{enter_finite_cooldown} = $_[2] }, # get +); + +sub recv { + my ($self, $type, $kv, @extra) = @_; + + if (my $cb = $PERSISTENT_TYPE{$type}) { + my $id = $kv->{identifier}; + my $req = $_[0]{req}{$id} ||= {}; + $cb->($self, $req, $kv); + $self->recv (request_change => $kv, $type, @extra); + } + + my $on = $self->{on}; + for (0 .. $#$on) { + unless (my $res = $on->[$_]($self, $type, $kv, @extra)) { + splice @$on, $_, 1 unless defined $res; + return; + } + } + + if (my $cb = $self->{queue}[0]) { + $cb->($self, $type, $kv, @extra) + and shift @{ $self->{queue} }; + } else { + $self->default_recv ($type, $kv, @extra); + } +} + sub on_read { my ($self) = @_; @@ -211,25 +252,6 @@ my %kv; my $rdata; - my $done_cb = sub { - $kv{pkt_type} = $type; - - my $on = $self->{on}; - for (0 .. $#$on) { - unless (my $res = $on->[$_]($self, $type, \%kv, $rdata)) { - splice @$on, $_, 1 unless defined $res; - return; - } - } - - if (my $cb = $self->{queue}[0]) { - $cb->($self, $type, \%kv, $rdata) - and shift @{ $self->{queue} }; - } else { - $self->default_recv ($type, \%kv, $rdata); - } - }; - my $hdr_cb; $hdr_cb = sub { if ($_[1] =~ /^([^=]+)=(.*)$/) { my ($k, $v) = ($1, $2); @@ -251,10 +273,10 @@ } elsif ($_[1] eq "Data") { $_[0]->push_read (chunk => delete $kv{data_length}, sub { $rdata = \$_[1]; - $done_cb->(); + $self->recv ($type, \%kv, $rdata); }); } elsif ($_[1] eq "EndMessage") { - $done_cb->(); + $self->recv ($type, \%kv); } else { die "protocol error, expected message end, got $_[1]\n";#d# } @@ -274,30 +296,27 @@ } elsif (exists $self->{id}{$kv->{identifier}}) { $self->{id}{$kv->{identifier}}($self, $type, $kv, $rdata) and delete $self->{id}{$kv->{identifier}}; - } else { - &{ $self->{progress} }; } } +our $NOP_CB = sub { }; + sub _txn { my ($name, $sub) = @_; *{$name} = sub { splice @_, 1, 0, (my $cv = AnyEvent->condvar); &$sub; - $cv + $cv->recv }; - *{"$name\_sync"} = sub { - splice @_, 1, 0, (my $cv = AnyEvent->condvar); + *{"$name\_"} = sub { + splice @_, 1, 0, pop || $NOP_CB; &$sub; - $cv->recv }; } -=item $cv = $fcp->list_peers ([$with_metdata[, $with_volatile]]) - -=item $peers = $fcp->list_peers_sync ([$with_metdata[, $with_volatile]]) +=item $peers = $fcp->list_peers ([$with_metdata[, $with_volatile]]) =cut @@ -323,9 +342,7 @@ ); }; -=item $cv = $fcp->list_peer_notes ($node_identifier) - -=item $notes = $fcp->list_peer_notes_sync ($node_identifier) +=item $notes = $fcp->list_peer_notes ($node_identifier) =cut @@ -343,9 +360,7 @@ ); }; -=item $cv = $fcp->watch_global ($enabled[, $verbosity_mask]) - -=item $fcp->watch_global_sync ($enabled[, $verbosity_mask]) +=item $fcp->watch_global ($enabled[, $verbosity_mask]) =cut @@ -360,9 +375,7 @@ $cv->(); }; -=item $cv = $fcp->list_persistent_requests - -=item $reqs = $fcp->list_persistent_requests_sync +=item $reqs = $fcp->list_persistent_requests =cut @@ -372,7 +385,7 @@ $self->serialise (list_persistent_requests => sub { my ($self, $guard) = @_; - my %res; + my @res; $self->send_msg ("list_persistent_requests"); @@ -382,22 +395,13 @@ $guard if 0; if ($type eq "end_list_persistent_requests") { - $cv->(\%res); + $cv->(\@res); return; } else { my $id = $kv->{identifier}; if ($type =~ /^persistent_(get|put|put_dir)$/) { - $res{$id} = { - type => $1, - %{ $res{$id} }, - %$kv, - }; - } elsif ($type eq "simple_progress") { - delete $kv->{pkt_type}; # save memory - push @{ $res{delete $kv->{identifier}}{simple_progress} }, $kv; - } else { - $res{delete $kv->{identifier}}{delete $kv->{pkt_type}} = $kv; + push @res, [$type, $kv]; } } @@ -406,9 +410,7 @@ }); }; -=item $cv = $fcp->remove_request ($global, $identifier) - -=item $status = $fcp->remove_request_sync ($global, $identifier) +=item $status = $fcp->remove_request ($global, $identifier) =cut @@ -427,9 +429,7 @@ ); }; -=item $cv = $fcp->modify_persistent_request ($global, $identifier[, $client_token[, $priority_class]]) - -=item $sync = $fcp->modify_persistent_request_sync ($global, $identifier[, $client_token[, $priority_class]]) +=item $sync = $fcp->modify_persistent_request ($global, $identifier[, $client_token[, $priority_class]]) =cut @@ -450,9 +450,7 @@ ); }; -=item $cv = $fcp->get_plugin_info ($name, $detailed) - -=item $info = $fcp->get_plugin_info_sync ($name, $detailed) +=item $info = $fcp->get_plugin_info ($name, $detailed) =cut @@ -471,9 +469,7 @@ ); }; -=item $cv = $fcp->client_get ($uri, $identifier, %kv) - -=item $status = $fcp->client_get_sync ($uri, $identifier, %kv) +=item $status = $fcp->client_get ($uri, $identifier, %kv) %kv can contain (L). @@ -490,25 +486,66 @@ %kv, uri => $uri, identifier => $identifier, - id_cb => sub { - my ($self, $type, $kv, $rdata) = @_; - - $cv->($kv); - 1 - }, ); }; -=item $cv = $fcp->test_dda_sync ($local_directory, $remote_directory, $want_read, $want_write) +=item $status = $fcp->remove_request ($identifier[, $global]) -=item ($can_read, $can_write) = $fcp->test_dda_sync ($local_directory, $remote_directory, $want_read, $want_write)) +Remove the request with the given isdentifier. Returns true if successful, +false on error. + +=cut + +_txn remove_request => sub { + my ($self, $cv, $identifier, $global) = @_; + + $self->serialise ($identifier => sub { + my ($self, $guard) = @_; + + $self->send_msg (remove_request => + identifier => $identifier, + global => $global ? "true" : "false", + ); + $self->on (sub { + my ($self, $type, $kv, @extra) = @_; + + if ($kv->{identifier} eq $identifier) { + if ($type eq "persistent_request_removed") { + $cv->(1); + return; + } elsif ($type eq "protocol_error") { + $cv->(undef); + return; + } + } + + 1 + }); + }); +}; + +=item ($can_read, $can_write) = $fcp->test_dda ($local_directory, $remote_directory, $want_read, $want_write)) The DDA test in FCP is probably the single most broken protocol - only one directory test can be outstanding at any time, and some guessing and heuristics are involved in mangling the paths. This function combines C and C in one -request, handling file reading and writing as well. +request, handling file reading and writing as well, and tries very hard to +do the right thing. + +Both C<$local_directory> and C<$remote_directory> must specify the same +directory - C<$local_directory> is the directory path on the client (where +L runs) and C<$remote_directory> is the directory path on +the server (where the freenet node runs). When both are running on the +same node, the paths are generally identical. + +C<$want_read> and C<$want_write> should be set to a true value when you +want to read (get) files or write (put) files, respectively. + +On error, an exception is thrown. Otherwise, C<$can_read> and +C<$can_write> indicate whether you can reaqd or write to freenet via the +directory. =cut @@ -544,7 +581,6 @@ my %response = (directory => $remote); if (length $kv->{read_filename}) { - warn "$local/$kv->{read_filename}";#d# if (open my $fh, "<:raw", "$local/$kv->{read_filename}") { sysread $fh, my $buf, -s $fh; $response{read_content} = $buf; @@ -590,6 +626,132 @@ =back +=head2 REQUEST CACHE + +The C class keeps a request cache, where it caches all +information from requests. + +For these messages, it will store a copy of the key-value pairs, together with a C slot, +in C<< $fcp->{req}{$identifier} >>: + + persistent_get + persistent_put + persistent_put_dir + +This message updates the stored data: + + persistent_request_modified + +This message will remove this entry: + + persistent_request_removed + +These messages get merged into the cache entry, under their +type, i.e. a C message will be stored in C<< +$fcp->{req}{$identifier}{simple_progress} >>: + + simple_progress # get/put + + uri_generated # put + generated_metadata # put + started_compression # put + finished_compression # put + put_failed # put + put_fetchable # put + put_successful # put + + sending_to_network # get + compatibility_mode # get + expected_hashes # get + expected_mime # get + expected_data_length # get + get_failed # get + data_found # get + enter_finite_cooldown # get + +In addition, an event (basically a fake message) of type C is generated +on every change, which will be called as C<< $cb->($fcp, $kv, $type) >>, where C<$type> +is the type of the original message triggering the change, + +To fill this cache with the global queue and keep it updated, +call C to subscribe to updates, followed by +C. + + $fcp->watch_global_sync_; # do not wait + $fcp->list_persistent_requests; # wait + +To get a better idea of what is stored in the cache, here is an example of +what might be stored in C<< $fcp->{req}{"Frost-gpl.txt"} >>: + + { + identifier => "Frost-gpl.txt", + uri => 'CHK@Fnx5kzdrfE,EImdzaVyEWl,AAIC--8/gpl.txt', + binary_blob => "false", + global => "true", + max_retries => -1, + max_size => 9223372036854775807, + persistence => "forever", + priority_class => 3, + real_time => "false", + return_type => "direct", + started => "true", + type => "persistent_get", + verbosity => 2147483647, + sending_to_network => { + identifier => "Frost-gpl.txt", + global => "true", + }, + compatibility_mode => { + identifier => "Frost-gpl.txt", + definitive => "true", + dont_compress => "false", + global => "true", + max => "COMPAT_1255", + min => "COMPAT_1255", + }, + expected_hashes => { + identifier => "Frost-gpl.txt", + global => "true", + hashes => { + ed2k => "d83596f5ee3b7...", + md5 => "e0894e4a2a6...", + sha1 => "...", + sha256 => "...", + sha512 => "...", + tth => "...", + }, + }, + expected_mime => { + identifier => "Frost-gpl.txt", + global => "true", + metadata => { content_type => "application/rar" }, + }, + expected_data_length => { + identifier => "Frost-gpl.txt", + data_length => 37576, + global => "true", + }, + simple_progress => { + identifier => "Frost-gpl.txt", + failed => 0, + fatally_failed => 0, + finalized_total => "true", + global => "true", + last_progress => 1438639282628, + required => 372, + succeeded => 102, + total => 747, + }, + data_found => { + identifier => "Frost-gpl.txt", + completion_time => 1438663354026, + data_length => 37576, + global => "true", + metadata => { content_type => "image/jpeg" }, + startup_time => 1438657196167, + }, + } + =head1 EXAMPLE PROGRAM use AnyEvent::FCP; @@ -597,12 +759,13 @@ my $fcp = new AnyEvent::FCP; # let us look at the global request list - $fcp->watch_global (1, 0); + $fcp->watch_global_ (1); # list them, synchronously - my $req = $fcp->list_persistent_requests_sync; + my $req = $fcp->list_persistent_requests; # go through all requests +TODO for my $req (values %$req) { # skip jobs not directly-to-disk next unless $req->{return_type} eq "disk";