ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/AnyEvent-MP/MP/Transport.pm
(Generate patch)

Comparing AnyEvent-MP/MP/Transport.pm (file contents):
Revision 1.11 by root, Mon Aug 3 15:40:53 2009 UTC vs.
Revision 1.27 by root, Sun Aug 9 00:31:40 2009 UTC

1=head1 NAME 1=head1 NAME
2 2
3AnyEvent::MP::Transport - actual transport protocol 3AnyEvent::MP::Transport - actual transport protocol handler
4 4
5=head1 SYNOPSIS 5=head1 SYNOPSIS
6 6
7 use AnyEvent::MP::Transport; 7 use AnyEvent::MP::Transport;
8 8
9=head1 DESCRIPTION 9=head1 DESCRIPTION
10 10
11This is the superclass for MP transports, most of which is considered an 11This implements the actual transport protocol for MP (it represents a
12implementation detail. 12single link), most of which is considered an implementation detail.
13 13
14See the "PROTOCOL" section below if you want to write another client for 14See the "PROTOCOL" section below if you want to write another client for
15this protocol. 15this protocol.
16 16
17=head1 FUNCTIONS/METHODS 17=head1 FUNCTIONS/METHODS
22 22
23package AnyEvent::MP::Transport; 23package AnyEvent::MP::Transport;
24 24
25use common::sense; 25use common::sense;
26 26
27use Scalar::Util; 27use Scalar::Util ();
28use List::Util ();
28use MIME::Base64 (); 29use MIME::Base64 ();
29use Storable (); 30use Storable ();
30use JSON::XS (); 31use JSON::XS ();
31 32
33use Digest::MD6 ();
34use Digest::HMAC_MD6 ();
35
32use AE (); 36use AE ();
33use AnyEvent::Socket (); 37use AnyEvent::Socket ();
34use AnyEvent::Handle (); 38use AnyEvent::Handle 4.92 ();
35 39
36use base Exporter::; 40use base Exporter::;
37 41
38our $VERSION = '0.0'; 42our $VERSION = '0.0';
39our $PROTOCOL_VERSION = 0; 43our $PROTOCOL_VERSION = 0;
109 peername => $peername, # for verification 113 peername => $peername, # for verification
110 ; 114 ;
111 115
112=cut 116=cut
113 117
118sub LATENCY() { 3 } # assumed max. network latency
119
114our @FRAMINGS = qw(json storable); # the framing types we accept and send, in order of preference 120our @FRAMINGS = qw(json storable); # the framing types we accept and send, in order of preference
115our @AUTH_SND = qw(hmac_md6_64_256); # auth types we send 121our @AUTH_SND = qw(hmac_md6_64_256); # auth types we send
116our @AUTH_RCV = (@AUTH_SND, qw(hex_secret)); # auth types we accept 122our @AUTH_RCV = (@AUTH_SND, qw(cleartext)); # auth types we accept
117 123
118#AnyEvent::Handle::register_write_type mp_record => sub { 124#AnyEvent::Handle::register_write_type mp_record => sub {
119#}; 125#};
120 126
121sub new { 127sub new {
126 $self->{queue} = []; 132 $self->{queue} = [];
127 133
128 { 134 {
129 Scalar::Util::weaken (my $self = $self); 135 Scalar::Util::weaken (my $self = $self);
130 136
131 $arg{tls_ctx_disabled} ||= {
132 sslv2 => 0,
133 sslv3 => 0,
134 tlsv1 => 1,
135 verify => 1,
136 cert_file => "secret.pem",
137 ca_file => "secret.pem",
138 verify_require_client_cert => 1,
139 };
140
141 $arg{secret} = AnyEvent::MP::Base::default_secret () 137 $arg{secret} = AnyEvent::MP::Base::default_secret ()
142 unless exists $arg{secret}; 138 unless exists $arg{secret};
143 139
140 $arg{timeout} = 30
141 unless exists $arg{timeout};
142
143 $arg{timeout} = 1 + LATENCY
144 if $arg{timeout} < 1 + LATENCY;
145
146 my $secret = $arg{secret};
147
148 if ($secret =~ /-----BEGIN RSA PRIVATE KEY-----.*-----END RSA PRIVATE KEY-----.*-----BEGIN CERTIFICATE-----.*-----END CERTIFICATE-----/s) {
149 # assume TLS mode
150 $arg{tls_ctx} = {
151 sslv2 => 0,
152 sslv3 => 0,
153 tlsv1 => 1,
154 verify => 1,
155 cert => $secret,
156 ca_cert => $secret,
157 verify_require_client_cert => 1,
158 };
159 }
160
144 $self->{hdl} = new AnyEvent::Handle 161 $self->{hdl} = new AnyEvent::Handle
145 fh => delete $arg{fh}, 162 fh => delete $arg{fh},
146 rbuf_max => 64 * 1024,
147 autocork => 1, 163 autocork => 1,
148 no_delay => 1, 164 no_delay => 1,
149 on_error => sub { 165 on_error => sub {
150 $self->error ($_[2]); 166 $self->error ($_[2]);
151 }, 167 },
168 rtimeout => $AnyEvent::MP::Base::CONNECT_TIMEOUT,
152 peername => delete $arg{peername}, 169 peername => delete $arg{peername},
153 ; 170 ;
154 171
155 my $secret = $arg{secret};
156 my $greeting_kv = $self->{greeting} ||= {}; 172 my $greeting_kv = $self->{greeting} ||= {};
173
174 $self->{local_node} = $AnyEvent::MP::Base::NODE;
175
157 $greeting_kv->{"tls"} = "1.0" 176 $greeting_kv->{"tls"} = "1.0" if $arg{tls_ctx};
158 if $arg{tls_ctx};
159 $greeting_kv->{provider} = "AE-$VERSION"; 177 $greeting_kv->{provider} = "AE-$VERSION";
160 $greeting_kv->{peeraddr} = AnyEvent::Socket::format_hostport $self->{peerhost}, $self->{peerport}; 178 $greeting_kv->{peeraddr} = AnyEvent::Socket::format_hostport $self->{peerhost}, $self->{peerport};
179 $greeting_kv->{timeout} = $arg{timeout};
161 180
162 # send greeting 181 # send greeting
163 my $lgreeting1 = "aemp;$PROTOCOL_VERSION;$PROTOCOL_VERSION" # version, min 182 my $lgreeting1 = "aemp;$PROTOCOL_VERSION"
164 . ";$AnyEvent::MP::Base::UNIQ" 183 . ";$self->{local_node}"
165 . ";$AnyEvent::MP::Base::NODE"
166 . ";" . (join ",", @AUTH_RCV) 184 . ";" . (join ",", @AUTH_RCV)
167 . ";" . (join ",", @FRAMINGS) 185 . ";" . (join ",", @FRAMINGS)
168 . (join "", map ";$_=$greeting_kv->{$_}", keys %$greeting_kv); 186 . (join "", map ";$_=$greeting_kv->{$_}", keys %$greeting_kv);
187
169 my $lgreeting2 = MIME::Base64::encode_base64 AnyEvent::MP::Base::nonce (33), ""; 188 my $lgreeting2 = MIME::Base64::encode_base64 AnyEvent::MP::Base::nonce (33), "";
170 189
171 $self->{hdl}->push_write ("$lgreeting1\012$lgreeting2\012"); 190 $self->{hdl}->push_write ("$lgreeting1\012$lgreeting2\012");
172 191
173 # expect greeting 192 # expect greeting
193 $self->{hdl}->rbuf_max (4 * 1024);
174 $self->{hdl}->push_read (line => sub { 194 $self->{hdl}->push_read (line => sub {
175 my $rgreeting1 = $_[1]; 195 my $rgreeting1 = $_[1];
176 196
177 my ($aemp, $version, $version_min, $uniq, $rnode, $auths, $framings, @kv) = split /;/, $rgreeting1; 197 my ($aemp, $version, $rnode, $auths, $framings, @kv) = split /;/, $rgreeting1;
178 198
179 if ($aemp ne "aemp") { 199 if ($aemp ne "aemp") {
180 return $self->error ("unparsable greeting"); 200 return $self->error ("unparsable greeting");
181 } elsif ($version_min > $PROTOCOL_VERSION) { 201 } elsif ($version != $PROTOCOL_VERSION) {
182 return $self->error ("version mismatch (we: $PROTOCOL_VERSION, they: $version_min .. $version)"); 202 return $self->error ("version mismatch (we: $PROTOCOL_VERSION, they: $version)");
183 } 203 }
184 204
185 my $s_auth; 205 my $s_auth;
186 for my $auth_ (split /,/, $auths) { 206 for my $auth_ (split /,/, $auths) {
187 if (grep $auth_ eq $_, @AUTH_SND) { 207 if (grep $auth_ eq $_, @AUTH_SND) {
204 } 224 }
205 225
206 defined $s_framing 226 defined $s_framing
207 or return $self->error ("$framings: no common framing method supported"); 227 or return $self->error ("$framings: no common framing method supported");
208 228
209 $self->{remote_uniq} = $uniq;
210 $self->{remote_node} = $rnode; 229 $self->{remote_node} = $rnode;
211 230
212 $self->{remote_greeting} = { 231 $self->{remote_greeting} = {
213 map /^([^=]+)(?:=(.*))?/ ? ($1 => $2) : (), 232 map /^([^=]+)(?:=(.*))?/ ? ($1 => $2) : (),
214 @kv 233 @kv
216 235
217 # read nonce 236 # read nonce
218 $self->{hdl}->push_read (line => sub { 237 $self->{hdl}->push_read (line => sub {
219 my $rgreeting2 = $_[1]; 238 my $rgreeting2 = $_[1];
220 239
240 "$lgreeting1\012$lgreeting2" ne "$rgreeting1\012$rgreeting2" # echo attack?
241 or return $self->error ("authentication error, echo attack?");
242
243 my $key = Digest::MD6::md6 $secret;
244 my $lauth;
245
221 if ($self->{tls_ctx} and 1 == int $self->{remote_greeting}{tls}) { 246 if ($self->{tls_ctx} and 1 == int $self->{remote_greeting}{tls}) {
222 $self->{tls} = $lgreeting2 lt $rgreeting2 ? "connect" : "accept"; 247 $self->{tls} = $lgreeting2 lt $rgreeting2 ? "connect" : "accept";
223 $self->{hdl}->starttls ($self->{tls}, $self->{tls_ctx}); 248 $self->{hdl}->starttls ($self->{tls}, $self->{tls_ctx});
249 $s_auth = "tls";
250 $lauth = "";
251 } else {
252 # we currently only support hmac_md6_64_256
253 $lauth = Digest::HMAC_MD6::hmac_md6_hex $key, "$lgreeting1\012$lgreeting2\012$rgreeting1\012$rgreeting2\012", 64, 256;
224 } 254 }
225
226 # auth
227 require Digest::MD6;
228 require Digest::HMAC_MD6;
229
230 my $key = Digest::MD6::md6 ($secret);
231 my $lauth = Digest::HMAC_MD6::hmac_md6_base64 ($key, "$lgreeting1\012$lgreeting2\012$rgreeting1\012$rgreeting2\012", 64, 256);
232
233 my $rauth =
234 $s_auth eq "hmac_md6_64_256" ? Digest::HMAC_MD6::hmac_md6_base64 ($key, "$rgreeting1\012$rgreeting2\012$lgreeting1\012$lgreeting2\012", 64, 256)
235 : $s_auth eq "hex_secret" ? unpack "H*", $secret
236 : die;
237
238 $lauth ne $rauth # echo attack?
239 or return $self->error ("authentication error");
240 255
241 $self->{hdl}->push_write ("$s_auth;$lauth;$s_framing\012"); 256 $self->{hdl}->push_write ("$s_auth;$lauth;$s_framing\012");
242 257
243 $self->{hdl}->rbuf_max (64); # enough for 44 reply bytes or so 258 # read the authentication response
244 $self->{hdl}->push_read (line => sub { 259 $self->{hdl}->push_read (line => sub {
245 my ($hdl, $rline) = @_; 260 my ($hdl, $rline) = @_;
246 261
247 my ($auth_method, $rauth2, $r_framing) = split /;/, $rline; 262 my ($auth_method, $rauth2, $r_framing) = split /;/, $rline;
263
264 my $rauth =
265 $auth_method eq "hmac_md6_64_256" ? Digest::HMAC_MD6::hmac_md6_hex $key, "$rgreeting1\012$rgreeting2\012$lgreeting1\012$lgreeting2\012", 64, 256
266 : $auth_method eq "cleartext" ? unpack "H*", $secret
267 : $auth_method eq "tls" ? ($self->{tls} ? "" : "\012\012") # \012\012 never matches
268 : return $self->error ("$auth_method: fatal, selected unsupported auth method");
248 269
249 if ($rauth2 ne $rauth) { 270 if ($rauth2 ne $rauth) {
250 return $self->error ("authentication failure/shared secret mismatch"); 271 return $self->error ("authentication failure/shared secret mismatch");
251 } 272 }
252 273
253 $self->{s_framing} = $s_framing; 274 $self->{s_framing} = $s_framing;
254 275
255 $hdl->rbuf_max (undef); 276 $hdl->rbuf_max (undef);
256 my $queue = delete $self->{queue}; # we are connected 277 my $queue = delete $self->{queue}; # we are connected
257 278
279 $self->{hdl}->rtimeout ($self->{remote_greeting}{timeout});
280 $self->{hdl}->wtimeout ($arg{timeout} - LATENCY);
281 $self->{hdl}->on_wtimeout (sub { $self->send (["", "devnull"]) });
282
258 $self->connected; 283 $self->connected;
259 284
260 $hdl->push_write ($self->{s_framing} => $_) 285 # send queued messages
286 $self->send ($_)
261 for @$queue; 287 for @$queue;
262 288
289 # receive handling
290 my $src_node = $self->{node};
291
263 my $rmsg; $rmsg = sub { 292 my $rmsg; $rmsg = sub {
264 $_[0]->push_read ($r_framing => $rmsg); 293 $_[0]->push_read ($r_framing => $rmsg);
265 294
295 local $AnyEvent::MP::Base::SRCNODE = $src_node;
266 AnyEvent::MP::Base::_inject ($_[1]); 296 AnyEvent::MP::Base::_inject (@{ $_[1] });
267 }; 297 };
268 $hdl->push_read ($r_framing => $rmsg); 298 $hdl->push_read ($r_framing => $rmsg);
269 }); 299 });
270 }); 300 });
271 }); 301 });
276 306
277sub error { 307sub error {
278 my ($self, $msg) = @_; 308 my ($self, $msg) = @_;
279 309
280 if ($self->{node} && $self->{node}{transport} == $self) { 310 if ($self->{node} && $self->{node}{transport} == $self) {
311 #TODO: store error, but do not instantly fail
312 $self->{node}->fail (transport_error => $self->{node}{noderef}, $msg);
281 $self->{node}->clr_transport; 313 $self->{node}->clr_transport;
282 } 314 }
283 $AnyEvent::MP::Base::WARN->("$self->{peerhost}:$self->{peerport}: $msg"); 315 $AnyEvent::MP::Base::WARN->("$self->{peerhost}:$self->{peerport}: $msg");
284 $self->destroy; 316 $self->destroy;
285} 317}
286 318
287sub connected { 319sub connected {
288 my ($self) = @_; 320 my ($self) = @_;
289 321
322 if (ref $AnyEvent::MP::Base::SLAVE) {
323 # first connect with a master node
324 my $via = $self->{remote_node};
325 $via =~ s/,/!/g;
326 $AnyEvent::MP::Base::NODE .= "\@$via";
327 $AnyEvent::MP::Base::NODE{$AnyEvent::MP::Base::NODE} = $AnyEvent::MP::Base::NODE{""};
328 $AnyEvent::MP::Base::SLAVE->();
329 }
330
331 if ($self->{local_node} ne $AnyEvent::MP::Base::NODE) {
332 # node changed its name since first greeting
333 $self->send (["", iam => $AnyEvent::MP::Base::NODE]);
334 }
335
290 my $node = AnyEvent::MP::Base::add_node ($self->{remote_node}); 336 my $node = AnyEvent::MP::Base::add_node ($self->{remote_node});
291 Scalar::Util::weaken ($self->{node} = $node); 337 Scalar::Util::weaken ($self->{node} = $node);
292 $node->set_transport ($self); 338 $node->set_transport ($self);
293} 339}
294 340
322The greeting consists of two text lines that are ended by either an ASCII 368The greeting consists of two text lines that are ended by either an ASCII
323CR LF pair, or a single ASCII LF (recommended). 369CR LF pair, or a single ASCII LF (recommended).
324 370
325=head2 GREETING 371=head2 GREETING
326 372
373All the lines until after authentication must not exceed 4kb in length,
374including delimiter. Afterwards there is no limit on the packet size that
375can be received.
376
377=head3 First Greeting Line
378
379Example:
380
381 aemp;0;fec.4a7720fc;127.0.0.1:1235,[::1]:1235;hmac_md6_64_256;json,storable;provider=AE-0.0
382
327The first line contains strings separated (not ended) by C<;> 383The first line contains strings separated (not ended) by C<;>
328characters. The first seven strings are fixed by the protocol, the 384characters. The first even ixtrings are fixed by the protocol, the
329remaining strings are C<KEY=VALUE> pairs. None of them may contain C<;> 385remaining strings are C<KEY=VALUE> pairs. None of them may contain C<;>
330characters themselves. 386characters themselves.
331 387
332The seven fixed strings are: 388The fixed strings are:
333 389
334=over 4 390=over 4
335 391
336=item C<aemp> 392=item protocol identification
337 393
338The constant C<aemp> to identify the protocol. 394The constant C<aemp> to identify the protocol.
339 395
340=item protocol version 396=item protocol version
341 397
342The (maximum) protocol version supported by this end, currently C<0>.
343
344=item minimum protocol version
345
346The minimum protocol version supported by this end, currently C<0>. 398The protocol version supported by this end, currently C<0>. If the
347 399versions don't match then no communication is possible. Minor extensions
348=item a token uniquely identifying the current node instance 400are supposed to be handled through additional key-value pairs.
349
350This is a string that must change between restarts. It usually contains
351things like the current time, the (OS) process id or similar values, but
352no meaning of the contents are assumed.
353 401
354=item the node endpoint descriptors 402=item the node endpoint descriptors
355 403
356for public nodes, this is a comma-separated list of protocol endpoints, 404for public nodes, this is a comma-separated list of protocol endpoints,
357i.e., the noderef. For slave nodes, this is a unique identifier. 405i.e., the noderef. For slave nodes, this is a unique identifier of the
406form C<slave/nonce>.
358 407
359=item the acceptable authentication methods 408=item the acceptable authentication methods
360 409
361A comma-separated list of authentication methods supported by the 410A comma-separated list of authentication methods supported by the
362node. Note that AnyEvent::MP supports a C<hex_secret> authentication 411node. Note that AnyEvent::MP supports a C<hex_secret> authentication
391=item tls=<major>.<minor> 440=item tls=<major>.<minor>
392 441
393Indicates that the other side supports TLS (version should be 1.0) and 442Indicates that the other side supports TLS (version should be 1.0) and
394wishes to do a TLS handshake. 443wishes to do a TLS handshake.
395 444
445=item timeout=<seconds>
446
447The amount of time after which this node should be detected as dead unless
448some data has been received. The node is responsible to send traffic
449reasonably more often than this interval (such as every timeout minus five
450seconds).
451
396=back 452=back
453
454=head3 Second Greeting Line
397 455
398After this greeting line there will be a second line containing a 456After this greeting line there will be a second line containing a
399cryptographic nonce, i.e. random data of high quality. To keep the 457cryptographic nonce, i.e. random data of high quality. To keep the
400protocol text-only, these are usually 32 base64-encoded octets, but 458protocol text-only, these are usually 32 base64-encoded octets, but
401it could be anything that doesn't contain any ASCII CR or ASCII LF 459it could be anything that doesn't contain any ASCII CR or ASCII LF
402characters. 460characters.
403 461
404Example of the two lines of greeting: 462I<< The two nonces B<must> be different, and an aemp implementation
463B<must> check and fail when they are identical >>.
405 464
406 aemp;0;0;e7d.4a76f48f;10.0.0.1:4040;hmac_md6_64_256,hex_secret;json,storable;provider=AE-0.0;peeraddr=127.0.0.1:1235 465Example of a nonce line:
407 XntegV2Guvss0qNn7phCPnoU87xqxV+4Mqm/5y4iQm6a 466
467 p/I122ql7kJR8lumW3lXlXCeBnyDAvz8NQo3x5IFowE4
408 468
409=head2 TLS handshake 469=head2 TLS handshake
410 470
411If, after the handshake, both sides indicate interest in TLS, then the 471I<< If, after the handshake, both sides indicate interest in TLS, then the
412connection I<must> use TLS, or fail. 472connection B<must> use TLS, or fail. >>
413 473
414Both sides compare their nonces, and the side who sent the lower nonce 474Both sides compare their nonces, and the side who sent the lower nonce
415value ("string" comparison on the raw octet values) becomes the client, 475value ("string" comparison on the raw octet values) becomes the client,
416and the one with the higher nonce the server. 476and the one with the higher nonce the server.
417 477
428 488
429=item the authentication method chosen 489=item the authentication method chosen
430 490
431This must be one of the methods offered by the other side in the greeting. 491This must be one of the methods offered by the other side in the greeting.
432 492
493The currently supported authentication methods are:
494
495=over 4
496
497=item cleartext
498
499This is simply the shared secret, lowercase-hex-encoded. This method is of
500course very insecure, unless TLS is used, which is why this module will
501accept, but not generate, cleartext auth replies.
502
503=item hmac_md6_64_256
504
505This method uses an MD6 HMAC with 64 bit blocksize and 256 bit hash. First, the shared secret
506is hashed with MD6:
507
508 key = MD6 (secret)
509
510This secret is then used to generate the "local auth reply", by taking
511the two local greeting lines and the two remote greeting lines (without
512line endings), appending \012 to all of them, concatenating them and
513calculating the MD6 HMAC with the key.
514
515 lauth = HMAC_MD6 key, "lgreeting1\012lgreeting2\012rgreeting1\012rgreeting2\012"
516
517This authentication token is then lowercase-hex-encoded and sent to the
518other side.
519
520Then the remote auth reply is generated using the same method, but local
521and remote greeting lines swapped:
522
523 rauth = HMAC_MD6 key, "rgreeting1\012rgreeting2\012lgreeting1\012lgreeting2\012"
524
525This is the token that is expected from the other side.
526
527=item tls
528
529This type is only valid iff TLS was enabled and the TLS handshake
530was successful. It has no authentication data, as the server/client
531certificate was successfully verified.
532
533Implementations supporting TLS I<must> accept this authentication type.
534
535=back
536
433=item the authentication data 537=item the authentication data
434 538
435The authentication data itself, usually base64 or hex-encoded data. 539The authentication data itself, usually base64 or hex-encoded data, see
540above.
436 541
437=item the framing protocol chosen 542=item the framing protocol chosen
438 543
439This must be one of the framing protocols offered by the other side in the 544This must be one of the framing protocols offered by the other side in the
440greeting. Each side must accept the choice of the other side. 545greeting. Each side must accept the choice of the other side.
441 546
442=back 547=back
443 548
444Example (the actual reply matching the previous example): 549Example of an authentication reply:
445 550
446 hmac_md6_64_256;wIlLedBY956UCGSISG9mBZRDTG8xUi73/sVse2DSQp0;json 551 hmac_md6_64_256;363d5175df38bd9eaddd3f6ca18aa1c0c4aa22f0da245ac638d048398c26b8d3;json
447 552
448=head2 DATA PHASE 553=head2 DATA PHASE
449 554
450After this, packets get exchanged using the chosen framing protocol. It is 555After this, packets get exchanged using the chosen framing protocol. It is
451quite possible that both sides use a different framing protocol. 556quite possible that both sides use a different framing protocol.
452 557
558=head2 FULL EXAMPLE
559
560This is an actual protocol dump of a handshake, followed by a single data
561packet. The greater than/less than lines indicate the direction of the
562transfer only.
563
564 > aemp;0;nndKd+gn;10.0.0.1:4040;hmac_md6_64_256,cleartext;json,storable;provider=AE-0.0;peeraddr=127.0.0.1:1235
565 > sRG8bbc4TDbkpvH8FTP4HBs87OhepH6VuApoZqXXskuG
566 < aemp;0;nmpKd+gh;127.0.0.1:1235,[::1]:1235;hmac_md6_64_256,cleartext;json,storable;provider=AE-0.0;peeraddr=127.0.0.1:58760
567 < dCEUcL/LJVSTJcx8byEsOzrwhzJYOq+L3YcopA5T6EAo
568 > hmac_md6_64_256;9513d4b258975accfcb2ab7532b83690e9c119a502c612203332a591c7237788;json
569 < hmac_md6_64_256;0298d6ba2240faabb2b2e881cf86b97d70a113ca74a87dc006f9f1e9d3010f90;json
570 > ["","lookup","pinger","10.0.0.1:4040#nndKd+gn.a","resolved"]
571
453=head1 SEE ALSO 572=head1 SEE ALSO
454 573
455L<AnyEvent>. 574L<AnyEvent>.
456 575
457=head1 AUTHOR 576=head1 AUTHOR

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines