--- AnyEvent/lib/AnyEvent/Handle.pm 2010/08/31 01:03:37 1.198 +++ AnyEvent/lib/AnyEvent/Handle.pm 2012/03/27 23:47:57 1.231 @@ -13,7 +13,7 @@ fh => \*STDIN, on_error => sub { my ($hdl, $fatal, $msg) = @_; - warn "got error $msg\n"; + AE::log error => "Got error $msg!"; $hdl->destroy; $cv->send; }; @@ -24,7 +24,7 @@ # read the response line $hdl->push_read (line => sub { my ($hdl, $line) = @_; - warn "got line <$line>\n"; + say "got line <$line>"; $cv->send; }); @@ -77,6 +77,8 @@ \&$func } +sub MAX_READ_SIZE() { 131072 } + =head1 METHODS =over 4 @@ -114,10 +116,10 @@ =item on_prepare => $cb->($handle) This (rarely used) callback is called before a new connection is -attempted, but after the file handle has been created. It could be used to -prepare the file handle with parameters required for the actual connect -(as opposed to settings that can be changed when the connection is already -established). +attempted, but after the file handle has been created (you can access that +file handle via C<< $handle->{fh} >>). It could be used to prepare the +file handle with parameters required for the actual connect (as opposed to +settings that can be changed when the connection is already established). The return value of this callback should be the connect timeout value in seconds (or C<0>, or C, or the empty list, to indicate that the @@ -128,13 +130,19 @@ This callback is called when a connection has been successfully established. The peer's numeric host and port (the socket peername) are passed as -parameters, together with a retry callback. - -If, for some reason, the handle is not acceptable, calling C<$retry> -will continue with the next connection target (in case of multi-homed -hosts or SRV records there can be multiple connection endpoints). At the -time it is called the read and write queues, eof status, tls status and -similar properties of the handle will have been reset. +parameters, together with a retry callback. At the time it is called the +read and write queues, EOF status, TLS status and similar properties of +the handle will have been reset. + +It is not allowed to use the read or write queues while the handle object +is connecting. + +If, for some reason, the handle is not acceptable, calling C<$retry> will +continue with the next connection target (in case of multi-homed hosts or +SRV records there can be multiple connection endpoints). The C<$retry> +callback can be invoked after the connect callback returns, i.e. one can +start a handshake and then decide to retry with the next host if the +handshake fails. In most cases, you should ignore the C<$retry> parameter. @@ -159,7 +167,7 @@ fatal errors the handle object will be destroyed (by a call to C<< -> destroy >>) after invoking the error callback (which means you are free to examine the handle object). Examples of fatal errors are an EOF condition -with active (but unsatisifable) read watchers (C) or I/O errors. In +with active (but unsatisfiable) read watchers (C) or I/O errors. In cases where the other side can close the connection at will, it is often easiest to not report C errors in this callback. @@ -224,8 +232,8 @@ =item on_drain => $cb->($handle) -This sets the callback that is called when the write buffer becomes empty -(or immediately if the buffer is empty already). +This sets the callback that is called once when the write buffer becomes +empty (and immediately when the handle object is created). To append to the write buffer, use the C<< ->push_write >> method. @@ -247,22 +255,27 @@ will be invoked (and if that one is missing, a non-fatal C error will be raised). -There are three variants of the timeouts that work independently -of each other, for both read and write, just read, and just write: +There are three variants of the timeouts that work independently of each +other, for both read and write (triggered when nothing was read I +written), just read (triggered when nothing was read), and just write: C, C and C, with corresponding callbacks C, C and C, and reset functions C, C, and C. -Note that timeout processing is active even when you do not have -any outstanding read or write requests: If you plan to keep the connection -idle then you should disable the timeout temporarily or ignore the timeout -in the C callback, in which case AnyEvent::Handle will simply -restart the timeout. +Note that timeout processing is active even when you do not have any +outstanding read or write requests: If you plan to keep the connection +idle then you should disable the timeout temporarily or ignore the +timeout in the corresponding C callback, in which case +AnyEvent::Handle will simply restart the timeout. -Zero (the default) disables this timeout. +Zero (the default) disables the corresponding timeout. =item on_timeout => $cb->($handle) +=item on_rtimeout => $cb->($handle) + +=item on_wtimeout => $cb->($handle) + Called whenever the inactivity timeout passes. If you return from this callback, then the timeout will be reset as if some activity had happened, so this condition is not fatal in any way. @@ -279,6 +292,21 @@ amount of data without a callback ever being called as long as the line isn't finished). +=item wbuf_max => + +If defined, then a fatal error will be raised (with C<$!> set to C) +when the write buffer ever (strictly) exceeds this size. This is useful to +avoid some forms of denial-of-service attacks. + +Although the units of this parameter is bytes, this is the I number +of bytes not yet accepted by the kernel. This can make a difference when +you e.g. use TLS, as TLS typically makes your write data larger (but it +can also make it smaller due to compression). + +As an example of when this limit is useful, take a chat server that sends +chat messages to a client. If the client does not read those in a timely +manner then the send buffer in the server would grow unbounded. + =item autocork => When disabled (the default), C will try to immediately @@ -339,9 +367,18 @@ =item read_size => -The default read block size (the number of bytes this module will -try to read during each loop iteration, which affects memory -requirements). Default: C<8192>. +The initial read block size, the number of bytes this module will try +to read during each loop iteration. Each handle object will consume +at least this amount of memory for the read buffer as well, so when +handling many connections watch out for memory requirements). See also +C. Default: C<2048>. + +=item max_read_size => + +The maximum read buffer size used by the dynamic adjustment +algorithm: Each time AnyEvent::Handle can read C bytes in +one go it will double C up to the maximum given by this +option. Default: C<131072> or C, whichever is higher. =item low_water_mark => @@ -414,8 +451,9 @@ =item tls_ctx => $anyevent_tls Use the given C object to create the new TLS connection -(unless a connection object was specified directly). If this parameter is -missing, then AnyEvent::Handle will use C. +(unless a connection object was specified directly). If this +parameter is missing (or C), then AnyEvent::Handle will use +C. Instead of an object, you can also specify a hash reference with C<< key => value >> pairs. Those will be passed to L to create a @@ -494,6 +532,8 @@ sub { my ($fh, $host, $port, $retry) = @_; + delete $self->{_connect}; # no longer needed + if ($fh) { $self->{fh} = $fh; @@ -510,7 +550,7 @@ } else { if ($self->{on_connect_error}) { $self->{on_connect_error}($self, "$!"); - $self->destroy; + $self->destroy if $self; } else { $self->_error ($!, 1); } @@ -520,7 +560,7 @@ local $self->{fh} = $_[0]; $self->{on_prepare} - ? $self->{on_prepare}->($self) + ? $self->{on_prepare}->($self) : () } ); @@ -548,6 +588,10 @@ $self->{_ractivity} = $self->{_wactivity} = AE::now; + $self->{read_size} ||= 2048; + $self->{max_read_size} = $self->{read_size} + if $self->{read_size} > ($self->{max_read_size} || MAX_READ_SIZE); + $self->timeout (delete $self->{timeout} ) if $self->{timeout}; $self->rtimeout (delete $self->{rtimeout} ) if $self->{rtimeout}; $self->wtimeout (delete $self->{wtimeout} ) if $self->{wtimeout}; @@ -560,7 +604,7 @@ $self->starttls (delete $self->{tls}, delete $self->{tls_ctx}) if $self->{tls}; - $self->on_drain (delete $self->{on_drain}) if $self->{on_drain}; + $self->on_drain (delete $self->{on_drain} ) if $self->{on_drain}; $self->start_read if $self->{on_read} || @{ $self->{_queue} }; @@ -646,11 +690,8 @@ sub no_delay { $_[0]{no_delay} = $_[1]; - eval { - local $SIG{__DIE__}; - setsockopt $_[0]{fh}, Socket::IPPROTO_TCP (), Socket::TCP_NODELAY (), int $_[1] - if $_[0]{fh}; - }; + setsockopt $_[0]{fh}, Socket::IPPROTO_TCP (), Socket::TCP_NODELAY (), int $_[1] + if $_[0]{fh}; } =item $handle->keepalive ($boolean) @@ -728,12 +769,20 @@ Configures the C setting (C disables it). +=item $handle->wbuf_max ($max_octets) + +Configures the C setting (C disables it). + =cut sub rbuf_max { $_[0]{rbuf_max} = $_[1]; } +sub wbuf_max { + $_[0]{wbuf_max} = $_[1]; +} + ############################################################################# =item $handle->timeout ($seconds) @@ -744,6 +793,9 @@ Configures (or disables) the inactivity timeout. +The timeout will be checked instantly, so this method might destroy the +handle before it returns. + =item $handle->timeout_reset =item $handle->rtimeout_reset @@ -770,6 +822,9 @@ *$timeout = sub { my ($self, $new_value) = @_; + $new_value >= 0 + or Carp::croak "AnyEvent::Handle->$timeout called with negative timeout ($new_value), caught"; + $self->{$timeout} = $new_value; delete $self->{$tw}; &$cb; }; @@ -833,7 +888,7 @@ AnyEvent::Handle will automatically try to get rid of it for you. When data could be written and the write buffer is shorter then the low -water mark, the C callback will be invoked. +water mark, the C callback will be invoked once. =over 4 @@ -858,9 +913,9 @@ =item $handle->push_write ($data) -Queues the given scalar to be written. You can push as much data as you -want (only limited by the available memory), as C -buffers it independently of the kernel. +Queues the given scalar to be written. You can push as much data as +you want (only limited by the available memory and C), as +C buffers it independently of the kernel. This method may invoke callbacks (and therefore the handle might be destroyed after it returns). @@ -898,6 +953,13 @@ # if still data left in wbuf, we need to poll $self->{_ww} = AE::io $self->{fh}, 1, $cb if length $self->{wbuf}; + + if ( + defined $self->{wbuf_max} + && $self->{wbuf_max} < length $self->{wbuf} + ) { + $self->_error (Errno::ENOSPC, 1), return; + } }; } @@ -1027,7 +1089,7 @@ register_write_type storable => sub { my ($self, $ref) = @_; - require Storable; + require Storable unless $Storable::VERSION; pack "w/a*", Storable::nfreeze ($ref) }; @@ -1042,7 +1104,7 @@ C to C<0>). This method is a shorthand for just that, and replaces the C callback with: - sub { shutdown $_[0]{fh}, 1 } # for push_shutdown + sub { shutdown $_[0]{fh}, 1 } This simply shuts down the write side and signals an EOF condition to the the peer. @@ -1074,7 +1136,7 @@ the handle object and the remaining arguments. The function is supposed to return a single octet string that will be -appended to the write buffer, so you cna mentally treat this function as a +appended to the write buffer, so you can mentally treat this function as a "arguments to on-the-wire-format" converter. Example: implement a custom write type C that joins the remaining @@ -1273,17 +1335,18 @@ =item $handle->rbuf -Returns the read buffer (as a modifiable lvalue). - -You can access the read buffer directly as the C<< ->{rbuf} >> -member, if you want. However, the only operation allowed on the -read buffer (apart from looking at it) is removing data from its -beginning. Otherwise modifying or appending to it is not allowed and will -lead to hard-to-track-down bugs. - -NOTE: The read buffer should only be used or modified if the C, -C or C methods are used. The other read methods -automatically manage the read buffer. +Returns the read buffer (as a modifiable lvalue). You can also access the +read buffer directly as the C<< ->{rbuf} >> member, if you want (this is +much faster, and no less clean). + +The only operation allowed on the read buffer (apart from looking at it) +is removing data from its beginning. Otherwise modifying or appending to +it is not allowed and will lead to hard-to-track-down bugs. + +NOTE: The read buffer should only be used or modified in the C +callback or when C or C are used with a single +callback (i.e. untyped). Typed C and C methods +will manage the read buffer on their own. =cut @@ -1344,7 +1407,8 @@ if (@_) { my $type = shift; - $cb = ($RH{$type} or Carp::croak "unsupported type passed to AnyEvent::Handle::unshift_read") + $cb = ($RH{$type} ||= _load_func "$type\::anyevent_read_type" + or Carp::croak "unsupported/unloadable type '$type' passed to AnyEvent::Handle::unshift_read") ->($self, $cb, @_); } @@ -1376,7 +1440,7 @@ Example: read 2 bytes. $handle->push_read (chunk => 2, sub { - warn "yay ", unpack "H*", $_[1]; + say "yay " . unpack "H*", $_[1]; }); =cut @@ -1420,7 +1484,7 @@ sub { $_[0]{rbuf} =~ s/^([^\015\012]*)(\015?\012)// or return; - $cb->($_[0], $1, $2); + $cb->($_[0], "$1", "$2"); 1 } } else { @@ -1430,7 +1494,7 @@ sub { $_[0]{rbuf} =~ s/$eol// or return; - $cb->($_[0], $1, $2); + $cb->($_[0], "$1", "$2"); 1 } } @@ -1488,13 +1552,13 @@ # accept if ($$rbuf =~ $accept) { $data .= substr $$rbuf, 0, $+[0], ""; - $cb->($self, $data); + $cb->($_[0], $data); return 1; } # reject if ($reject && $$rbuf =~ $reject) { - $self->_error (Errno::EBADMSG); + $_[0]->_error (Errno::EBADMSG); } # skip @@ -1520,20 +1584,20 @@ sub { unless ($_[0]{rbuf} =~ s/^(0|[1-9][0-9]*)://) { if ($_[0]{rbuf} =~ /[^0-9]/) { - $self->_error (Errno::EBADMSG); + $_[0]->_error (Errno::EBADMSG); } return; } my $len = $1; - $self->unshift_read (chunk => $len, sub { + $_[0]->unshift_read (chunk => $len, sub { my $string = $_[1]; $_[0]->unshift_read (chunk => 1, sub { if ($_[1] eq ",") { $cb->($_[0], $string); } else { - $self->_error (Errno::EBADMSG); + $_[0]->_error (Errno::EBADMSG); } }); }); @@ -1616,26 +1680,26 @@ my $rbuf = \$self->{rbuf}; sub { - my $ref = eval { $json->incr_parse ($self->{rbuf}) }; + my $ref = eval { $json->incr_parse ($_[0]{rbuf}) }; if ($ref) { - $self->{rbuf} = $json->incr_text; + $_[0]{rbuf} = $json->incr_text; $json->incr_text = ""; - $cb->($self, $ref); + $cb->($_[0], $ref); 1 } elsif ($@) { # error case $json->incr_skip; - $self->{rbuf} = $json->incr_text; + $_[0]{rbuf} = $json->incr_text; $json->incr_text = ""; - $self->_error (Errno::EBADMSG); + $_[0]->_error (Errno::EBADMSG); () } else { - $self->{rbuf} = ""; + $_[0]{rbuf} = ""; () } @@ -1655,7 +1719,7 @@ register_read_type storable => sub { my ($self, $cb) = @_; - require Storable; + require Storable unless $Storable::VERSION; sub { # when we can use 5.10 we can use ".", but for 5.8 we use the re-pack method @@ -1678,7 +1742,7 @@ if (my $ref = eval { Storable::thaw ($_[1]) }) { $cb->($_[0], $ref); } else { - $self->_error (Errno::EBADMSG); + $_[0]->_error (Errno::EBADMSG); } }); } @@ -1726,15 +1790,24 @@ will automatically C for you when neither C is set nor there are any read requests in the queue. -These methods will have no effect when in TLS mode (as TLS doesn't support -half-duplex connections). +In older versions of this module (<= 5.3), these methods had no effect, +as TLS does not support half-duplex connections. In current versions they +work as expected, as this behaviour is required to avoid certain resource +attacks, where the program would be forced to read (and buffer) arbitrary +amounts of data before being able to send some data. The drawback is that +some readings of the the SSL/TLS specifications basically require this +attack to be working, as SSL/TLS implementations might stall sending data +during a rehandshake. + +As a guideline, during the initial handshake, you should not stop reading, +and as a client, it might cause problems, depending on your application. =cut sub stop_read { my ($self) = @_; - delete $self->{_rw} unless $self->{tls}; + delete $self->{_rw}; } sub start_read { @@ -1745,7 +1818,7 @@ $self->{_rw} = AE::io $self->{fh}, 0, sub { my $rbuf = \($self->{tls} ? my $buf : $self->{rbuf}); - my $len = sysread $self->{fh}, $$rbuf, $self->{read_size} || 8192, length $$rbuf; + my $len = sysread $self->{fh}, $$rbuf, $self->{read_size}, length $$rbuf; if ($len > 0) { $self->{_activity} = $self->{_ractivity} = AE::now; @@ -1758,6 +1831,12 @@ $self->_drain_rbuf; } + if ($len == $self->{read_size}) { + $self->{read_size} *= 2; + $self->{read_size} = $self->{max_read_size} || MAX_READ_SIZE + if $self->{read_size} > ($self->{max_read_size} || MAX_READ_SIZE); + } + } elsif (defined $len) { delete $self->{_rw}; $self->{_eof} = 1; @@ -1940,7 +2019,8 @@ $self->{_rbio} = Net::SSLeay::BIO_new (Net::SSLeay::BIO_s_mem ()); $self->{_wbio} = Net::SSLeay::BIO_new (Net::SSLeay::BIO_s_mem ()); - Net::SSLeay::BIO_write ($self->{_rbio}, delete $self->{rbuf}); + Net::SSLeay::BIO_write ($self->{_rbio}, $self->{rbuf}); + $self->{rbuf} = ""; Net::SSLeay::set_bio ($tls, $self->{_rbio}, $self->{_wbio}); @@ -1988,6 +2068,19 @@ delete @$self{qw(_rbio _wbio _tls_wbuf _on_starttls)}; } +=item $handle->resettls + +This rarely-used method simply resets and TLS state on the handle, usually +causing data loss. + +One case where it may be useful is when you want to skip over the data in +the stream but you are not interested in interpreting it, so data loss is +no concern. + +=cut + +*resettls = \&_freetls; + sub DESTROY { my ($self) = @_; @@ -2006,7 +2099,7 @@ if ($len > 0) { substr $wbuf, 0, $len, ""; - } else { + } elsif (defined $len || ($! != EAGAIN && $! != EINTR && $! != WSAEWOULDBLOCK)) { @linger = (); # end } }; @@ -2112,6 +2205,86 @@ from within all other callbacks, you need to explicitly call the C<< ->destroy >> method. +=item Why is my C callback never called? + +Probably because your C callback is being called instead: When +you have outstanding requests in your read queue, then an EOF is +considered an error as you clearly expected some data. + +To avoid this, make sure you have an empty read queue whenever your handle +is supposed to be "idle" (i.e. connection closes are O.K.). You can set +an C handler that simply pushes the first read requests in the +queue. + +See also the next question, which explains this in a bit more detail. + +=item How can I serve requests in a loop? + +Most protocols consist of some setup phase (authentication for example) +followed by a request handling phase, where the server waits for requests +and handles them, in a loop. + +There are two important variants: The first (traditional, better) variant +handles requests until the server gets some QUIT command, causing it to +close the connection first (highly desirable for a busy TCP server). A +client dropping the connection is an error, which means this variant can +detect an unexpected detection close. + +To handle this case, always make sure you have a on-empty read queue, by +pushing the "read request start" handler on it: + + # we assume a request starts with a single line + my @start_request; @start_request = (line => sub { + my ($hdl, $line) = @_; + + ... handle request + + # push next request read, possibly from a nested callback + $hdl->push_read (@start_request); + }); + + # auth done, now go into request handling loop + # now push the first @start_request + $hdl->push_read (@start_request); + +By always having an outstanding C, the handle always expects +some data and raises the C error when the connction is dropped +unexpectedly. + +The second variant is a protocol where the client can drop the connection +at any time. For TCP, this means that the server machine may run out of +sockets easier, and in general, it means you cannot distinguish a protocl +failure/client crash from a normal connection close. Nevertheless, these +kinds of protocols are common (and sometimes even the best solution to the +problem). + +Having an outstanding read request at all times is possible if you ignore +C errors, but this doesn't help with when the client drops the +connection during a request, which would still be an error. + +A better solution is to push the initial request read in an C +callback. This avoids an error, as when the server doesn't expect data +(i.e. is idly waiting for the next request, an EOF will not raise an +error, but simply result in an C callback. It is also a bit slower +and simpler: + + # auth done, now go into request handling loop + $hdl->on_read (sub { + my ($hdl) = @_; + + # called each time we receive data but the read queue is empty + # simply start read the request + + $hdl->push_read (line => sub { + my ($hdl, $line) = @_; + + ... handle request + + # do nothing special when the request has been handled, just + # let the request queue go empty. + }); + }); + =item I get different callback invocations in TLS mode/Why can't I pause reading? @@ -2142,6 +2315,10 @@ my $data = delete $_[0]{rbuf}; }); +Note that this example removes the C member from the handle object, +which is not normally allowed by the API. It is expressly permitted in +this case only, as the handle object needs to be destroyed afterwards. + The reason to use C is that TCP connections, due to latencies and packets loss, might get closed quite violently with an error, when in fact all data has been received. @@ -2161,7 +2338,7 @@ $handle->push_write (...); $handle->on_drain (sub { - warn "all data submitted to the kernel\n"; + AE::log debug => "All data submitted to the kernel."; undef $handle; }); @@ -2255,7 +2432,6 @@ =back - =head1 SUBCLASSING AnyEvent::Handle In many cases, you might want to subclass AnyEvent::Handle. @@ -2291,4 +2467,5 @@ =cut -1; # End of AnyEvent::Handle +1 +