--- AnyEvent-HTTP/HTTP.pm 2008/06/05 16:33:02 1.12 +++ AnyEvent-HTTP/HTTP.pm 2010/09/06 06:31:32 1.57 @@ -6,6 +6,10 @@ use AnyEvent::HTTP; + http_get "http://www.nethype.de/", sub { print $_[1] }; + + # ... do something else here + =head1 DESCRIPTION This module is an L user, you need to make sure that you use and @@ -37,30 +41,31 @@ use strict; no warnings; -use Carp; +use Errno (); -use AnyEvent (); +use AnyEvent 5.0 (); use AnyEvent::Util (); use AnyEvent::Socket (); use AnyEvent::Handle (); use base Exporter::; -our $VERSION = '1.0'; +our $VERSION = '1.46'; -our @EXPORT = qw(http_get http_request); +our @EXPORT = qw(http_get http_post http_head http_request); -our $USERAGENT = "Mozilla/5.0 (compatible; AnyEvent::HTTP/$VERSION; +http://software.schmorp.de/pkg/AnyEvent)"; +our $USERAGENT = "Mozilla/5.0 (compatible; U; AnyEvent-HTTP/$VERSION; +http://software.schmorp.de/pkg/AnyEvent)"; our $MAX_RECURSE = 10; our $MAX_PERSISTENT = 8; our $PERSISTENT_TIMEOUT = 2; our $TIMEOUT = 300; # changing these is evil -our $MAX_PERSISTENT_PER_HOST = 2; +our $MAX_PERSISTENT_PER_HOST = 0; our $MAX_PER_HOST = 4; our $PROXY; +our $ACTIVE = 0; my %KA_COUNT; # number of open keep-alive connections per host my %CO_SLOT; # number of open connections, and wait queue, per host @@ -68,36 +73,58 @@ =item http_get $url, key => value..., $cb->($data, $headers) Executes an HTTP-GET request. See the http_request function for details on -additional parameters. +additional parameters and the return value. =item http_head $url, key => value..., $cb->($data, $headers) -Executes an HTTP-HEAD request. See the http_request function for details on -additional parameters. +Executes an HTTP-HEAD request. See the http_request function for details +on additional parameters and the return value. =item http_post $url, $body, key => value..., $cb->($data, $headers) -Executes an HTTP-POST request with a request body of C<$bod>. See the -http_request function for details on additional parameters. +Executes an HTTP-POST request with a request body of C<$body>. See the +http_request function for details on additional parameters and the return +value. =item http_request $method => $url, key => value..., $cb->($data, $headers) Executes a HTTP request of type C<$method> (e.g. C, C). The URL must be an absolute http or https URL. -The callback will be called with the response data as first argument -(or C if it wasn't available due to errors), and a hash-ref with -response headers as second argument. +When called in void context, nothing is returned. In other contexts, +C returns a "cancellation guard" - you have to keep the +object at least alive until the callback get called. If the object gets +destroyed before the callbakc is called, the request will be cancelled. + +The callback will be called with the response body data as first argument +(or C if an error occured), and a hash-ref with response headers as +second argument. All the headers in that hash are lowercased. In addition to the response -headers, the three "pseudo-headers" C, C and -C contain the three parts of the HTTP Status-Line of the same -name. If the server sends a header multiple lines, then their contents -will be joined together with C<\x00>. +headers, the "pseudo-headers" (uppercase to avoid clashing with possible +response headers) C, C and C contain the +three parts of the HTTP Status-Line of the same name. + +The pseudo-header C contains the actual URL (which can differ from +the requested URL when following redirects - for example, you might get +an error that your URL scheme is not supported even though your URL is a +valid http URL because it redirected to an ftp URL, in which case you can +look at the URL pseudo header). + +The pseudo-header C only exists when the request was a result +of an internal redirect. In that case it is an array reference with +the C<($data, $headers)> from the redirect response. Note that this +response could in turn be the result of a redirect itself, and C<< +$headers->{Redirect}[1]{Redirect} >> will then contain the original +response, and so on. + +If the server sends a header multiple times, then their contents will be +joined together with a comma (C<,>), as per the HTTP spec. If an internal error occurs, such as not being able to resolve a hostname, -then C<$data> will be C, C<< $headers->{Status} >> will be C<599> -and the C pseudo-header will contain an error message. +then C<$data> will be C, C<< $headers->{Status} >> will be C<59x> +(usually C<599>) and the C pseudo-header will contain an error +message. A typical callback might look like this: @@ -125,19 +152,24 @@ The request headers to use. Currently, C may provide its own C, C, C and C headers -and will provide defaults for C and C. +and will provide defaults for C and C (this can be +suppressed by using C for these headers in which case they won't be +sent at all). =item timeout => $seconds The time-out to use for various stages - each connect attempt will reset -the timeout, as will read or write activity. Default timeout is 5 minutes. +the timeout, as will read or write activity, i.e. this is not an overall +timeout. + +Default timeout is 5 minutes. =item proxy => [$host, $port[, $scheme]] or undef Use the given http proxy for all requests. If not specified, then the default proxy (as specified by C<$ENV{http_proxy}>) is used. -C<$scheme> must be either missing or C for HTTP, or C for +C<$scheme> must be either missing, C for HTTP or C for HTTPS. =item body => $string @@ -153,7 +185,7 @@ The C<$hash_ref> must be an (initially empty) hash reference which will get updated automatically. It is possible to save the cookie_jar to persistent storage with something like JSON or Storable, but this is not -recommended, as expire times are currently being ignored. +recommended, as expiry times are currently being ignored. Note that this cookie implementation is not of very high quality, nor meant to be complete. If you want complete cookie management you have to @@ -161,6 +193,89 @@ cookie-using sites working. Cookies are a privacy disaster, do not use them unless required to. +=item tls_ctx => $scheme | $tls_ctx + +Specifies the AnyEvent::TLS context to be used for https connections. This +parameter follows the same rules as the C parameter to +L, but additionally, the two strings C or +C can be specified, which give you a predefined low-security (no +verification, highest compatibility) and high-security (CA and common-name +verification) TLS context. + +The default for this option is C, which could be interpreted as "give +me the page, no matter what". + +=item on_prepare => $callback->($fh) + +In rare cases you need to "tune" the socket before it is used to +connect (for exmaple, to bind it on a given IP address). This parameter +overrides the prepare callback passed to C +and behaves exactly the same way (e.g. it has to provide a +timeout). See the description for the C<$prepare_cb> argument of +C for details. + +=item on_header => $callback->($headers) + +When specified, this callback will be called with the header hash as soon +as headers have been successfully received from the remote server (not on +locally-generated errors). + +It has to return either true (in which case AnyEvent::HTTP will continue), +or false, in which case AnyEvent::HTTP will cancel the download (and call +the finish callback with an error code of C<598>). + +This callback is useful, among other things, to quickly reject unwanted +content, which, if it is supposed to be rare, can be faster than first +doing a C request. + +Example: cancel the request unless the content-type is "text/html". + + on_header => sub { + $_[0]{"content-type"} =~ /^text\/html\s*(?:;|$)/ + }, + +=item on_body => $callback->($partial_body, $headers) + +When specified, all body data will be passed to this callback instead of +to the completion callback. The completion callback will get the empty +string instead of the body data. + +It has to return either true (in which case AnyEvent::HTTP will continue), +or false, in which case AnyEvent::HTTP will cancel the download (and call +the completion callback with an error code of C<598>). + +This callback is useful when the data is too large to be held in memory +(so the callback writes it to a file) or when only some information should +be extracted, or when the body should be processed incrementally. + +It is usually preferred over doing your own body handling via +C, but in case of streaming APIs, where HTTP is +only used to create a connection, C is the better +alternative, as it allows you to install your own event handler, reducing +resource usage. + +=item want_body_handle => $enable + +When enabled (default is disabled), the behaviour of AnyEvent::HTTP +changes considerably: after parsing the headers, and instead of +downloading the body (if any), the completion callback will be +called. Instead of the C<$body> argument containing the body data, the +callback will receive the L object associated with the +connection. In error cases, C will be passed. When there is no body +(e.g. status C<304>), the empty string will be passed. + +The handle object might or might not be in TLS mode, might be connected to +a proxy, be a persistent connection etc., and configured in unspecified +ways. The user is responsible for this handle (it will not be used by this +module anymore). + +This is useful with some push-type services, where, after the initial +headers, an interactive protocol is used (typical example would be the +push-style twitter API which starts a JSON/XML stream). + +If you think you need this, first have a look at C, to see if +that doesn't solve your problem in a better way. + =back Example: make a simple HTTP GET request for http://www.nethype.de/ @@ -183,6 +298,16 @@ } ; +Example: make another simple HTTP GET request, but immediately try to +cancel it. + + my $request = http_request GET => "http://www.nethype.de/", sub { + my ($body, $hdr) = @_; + print "$body\n"; + }; + + undef $request; + =cut sub _slot_schedule; @@ -193,8 +318,10 @@ if (my $cb = shift @{ $CO_SLOT{$host}[1] }) { # somebody wants that slot ++$CO_SLOT{$host}[0]; + ++$ACTIVE; $cb->(AnyEvent::Util::guard { + --$ACTIVE; --$CO_SLOT{$host}[0]; _slot_schedule $host; }); @@ -213,12 +340,20 @@ _slot_schedule $_[0]; } -sub http_request($$$;@) { +our $qr_nlnl = qr{(? 1, sslv2 => 1 }; +our $TLS_CTX_HIGH = { cache => 1, verify => 1, verify_peername => "https" }; + +sub http_request($$@) { my $cb = pop; my ($method, $url, %arg) = @_; my %hdr; + $arg{tls_ctx} = $TLS_CTX_LOW if $arg{tls_ctx} eq "low" || !exists $arg{tls_ctx}; + $arg{tls_ctx} = $TLS_CTX_HIGH if $arg{tls_ctx} eq "high"; + $method = uc $method; if (my $hdr = $arg{headers}) { @@ -227,54 +362,64 @@ } } - my $recurse = exists $arg{recurse} ? $arg{recurse} : $MAX_RECURSE; + # pseudo headers for all subsequent responses + my @pseudo = (URL => $url); + push @pseudo, Redirect => delete $arg{Redirect} if exists $arg{Redirect}; + + my $recurse = exists $arg{recurse} ? delete $arg{recurse} : $MAX_RECURSE; - return $cb->(undef, { Status => 599, Reason => "recursion limit reached" }) + return $cb->(undef, { Status => 599, Reason => "Too many redirections", @pseudo }) if $recurse < 0; my $proxy = $arg{proxy} || $PROXY; my $timeout = $arg{timeout} || $TIMEOUT; - $hdr{"user-agent"} ||= $USERAGENT; - - my ($scheme, $authority, $upath, $query, $fragment) = - $url =~ m|(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*)(?:\?([^#]*))?(?:#(.*))?|; + my ($uscheme, $uauthority, $upath, $query, $fragment) = + $url =~ m|(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*)(?:(\?[^#]*))?(?:#(.*))?|; - $scheme = lc $scheme; + $uscheme = lc $uscheme; - my $uport = $scheme eq "http" ? 80 - : $scheme eq "https" ? 443 - : return $cb->(undef, { Status => 599, Reason => "only http and https URL schemes supported" }); + my $uport = $uscheme eq "http" ? 80 + : $uscheme eq "https" ? 443 + : return $cb->(undef, { Status => 599, Reason => "Only http and https URL schemes supported", @pseudo }); - $authority =~ /^(?: .*\@ )? ([^\@:]+) (?: : (\d+) )?$/x - or return $cb->(undef, { Status => 599, Reason => "unparsable URL" }); + $uauthority =~ /^(?: .*\@ )? ([^\@:]+) (?: : (\d+) )?$/x + or return $cb->(undef, { Status => 599, Reason => "Unparsable URL", @pseudo }); my $uhost = $1; $uport = $2 if defined $2; + $hdr{host} = defined $2 ? "$uhost:$2" : "$uhost" + unless exists $hdr{host}; + $uhost =~ s/^\[(.*)\]$/$1/; - $upath .= "?$query" if length $query; + $upath .= $query if length $query; $upath =~ s%^/?%/%; - $hdr{referer} ||= "$scheme://$authority$upath"; - # cookie processing if (my $jar = $arg{cookie_jar}) { - %$jar = () if $jar->{version} < 1; + %$jar = () if $jar->{version} != 1; my @cookie; while (my ($chost, $v) = each %$jar) { - next unless $chost eq substr $uhost, -length $chost; - next unless $chost =~ /^\./; + if ($chost =~ /^\./) { + next unless $chost eq substr $uhost, -length $chost; + } elsif ($chost =~ /\./) { + next unless $chost eq $uhost; + } else { + next; + } while (my ($cpath, $v) = each %$v) { next unless $cpath eq substr $upath, 0, length $cpath; while (my ($k, $v) = each %$v) { - next if $scheme ne "https" && exists $v->{secure}; - push @cookie, "$k=$v->{value}"; + next if $uscheme ne "https" && exists $v->{secure}; + my $value = $v->{value}; + $value =~ s/([\\"])/\\$1/g; + push @cookie, "$k=\"$value\""; } } } @@ -283,17 +428,26 @@ if @cookie; } - my ($rhost, $rport, $rpath); # request host, port, path + my ($rhost, $rport, $rscheme, $rpath); # request host, port, path if ($proxy) { - ($rhost, $rport, $scheme) = @$proxy; - $rpath = $url; + ($rpath, $rhost, $rport, $rscheme) = ($url, @$proxy); + + $rscheme = "http" unless defined $rscheme; + + # don't support https requests over https-proxy transport, + # can't be done with tls as spec'ed, unless you double-encrypt. + $rscheme = "http" if $uscheme eq "https" && $rscheme eq "https"; } else { - ($rhost, $rport, $rpath) = ($uhost, $uport, $upath); - $hdr{host} = $uhost; + ($rhost, $rport, $rscheme, $rpath) = ($uhost, $uport, $uscheme, $upath); } - $hdr{"content-length"} = length $arg{body}; + # leave out fragment and query string, just a heuristic + $hdr{referer} ||= "$uscheme://$uauthority$upath" unless exists $hdr{referer}; + $hdr{"user-agent"} ||= $USERAGENT unless exists $hdr{"user-agent"}; + + $hdr{"content-length"} = length $arg{body} + if length $arg{body} || $method ne "GET"; my %state = (connect_guard => 1); @@ -304,164 +458,325 @@ $state{connect_guard} = AnyEvent::Socket::tcp_connect $rhost, $rport, sub { $state{fh} = shift - or return $cb->(undef, { Status => 599, Reason => "$!" }); + or do { + my $err = "$!"; + %state = (); + return $cb->(undef, { Status => 599, Reason => $err, @pseudo }); + }; - delete $state{connect_guard}; # reduce memory usage, save a tree + pop; # free memory, save a tree + + return unless delete $state{connect_guard}; # get handle $state{handle} = new AnyEvent::Handle - fh => $state{fh}, - ($scheme eq "https" ? (tls => "connect") : ()); + fh => $state{fh}, + peername => $rhost, + tls_ctx => $arg{tls_ctx}, + # these need to be reconfigured on keepalive handles + timeout => $timeout, + on_error => sub { + %state = (); + $cb->(undef, { Status => 599, Reason => $_[2], @pseudo }); + }, + on_eof => sub { + %state = (); + $cb->(undef, { Status => 599, Reason => "Unexpected end-of-file", @pseudo }); + }, + ; # limit the number of persistent connections - if ($KA_COUNT{$_[1]} < $MAX_PERSISTENT_PER_HOST) { - ++$KA_COUNT{$_[1]}; - $state{handle}{ka_count_guard} = AnyEvent::Util::guard { --$KA_COUNT{$_[1]} }; - $hdr{connection} = "keep-alive"; - delete $hdr{connection}; # keep-alive not yet supported - } else { + # keepalive not yet supported +# if ($KA_COUNT{$_[1]} < $MAX_PERSISTENT_PER_HOST) { +# ++$KA_COUNT{$_[1]}; +# $state{handle}{ka_count_guard} = AnyEvent::Util::guard { +# --$KA_COUNT{$_[1]} +# }; +# $hdr{connection} = "keep-alive"; +# } else { delete $hdr{connection}; - } +# } - # (re-)configure handle - $state{handle}->timeout ($timeout); - $state{handle}->on_error (sub { - %state = (); - $cb->(undef, { Status => 599, Reason => "$!" }); - }); - $state{handle}->on_eof (sub { - %state = (); - $cb->(undef, { Status => 599, Reason => "unexpected end-of-file" }); - }); + $state{handle}->starttls ("connect") if $rscheme eq "https"; - # send request - $state{handle}->push_write ( - "$method $rpath HTTP/1.0\015\012" - . (join "", map "$_: $hdr{$_}\015\012", keys %hdr) - . "\015\012" - . (delete $arg{body}) - ); - - %hdr = (); # reduce memory usage, save a kitten - - # status line - $state{handle}->push_read (line => qr/\015?\012/, sub { - $_[1] =~ /^HTTP\/([0-9\.]+) \s+ ([0-9]{3}) \s+ ([^\015\012]+)/ix - or return (%state = (), $cb->(undef, { Status => 599, Reason => "invalid server response ($_[1])" })); - - my %hdr = ( # response headers - HTTPVersion => "\x00$1", - Status => "\x00$2", - Reason => "\x00$3", + # handle actual, non-tunneled, request + my $handle_actual_request = sub { + $state{handle}->starttls ("connect") if $uscheme eq "https" && !exists $state{handle}{tls}; + + # send request + $state{handle}->push_write ( + "$method $rpath HTTP/1.0\015\012" + . (join "", map "\u$_: $hdr{$_}\015\012", grep defined $hdr{$_}, keys %hdr) + . "\015\012" + . (delete $arg{body}) ); - # headers, could be optimized a bit - $state{handle}->unshift_read (line => qr/\015?\012\015?\012/, sub { - for ("$_[1]\012") { - # we support spaces in field names, as lotus domino - # creates them. - $hdr{lc $1} .= "\x00$2" + # return if error occured during push_write() + return unless %state; + + %hdr = (); # reduce memory usage, save a kitten, also make it possible to re-use + + # status line and headers + $state{handle}->push_read (line => $qr_nlnl, sub { + for ("$_[1]") { + y/\015//d; # weed out any \015, as they show up in the weirdest of places. + + /^HTTP\/([0-9\.]+) \s+ ([0-9]{3}) (?: \s+ ([^\015\012]*) )? \015?\012/igxc + or return (%state = (), $cb->(undef, { Status => 599, Reason => "Invalid server response", @pseudo })); + + push @pseudo, + HTTPVersion => $1, + Status => $2, + Reason => $3, + ; + + # things seen, not parsed: + # p3pP="NON CUR OTPi OUR NOR UNI" + + $hdr{lc $1} .= ",$2" while /\G - ([^:\000-\037]+): + ([^:\000-\037]*): [\011\040]* - ((?: [^\015\012]+ | \015?\012[\011\040] )*) - \015?\012 + ((?: [^\012]+ | \012[\011\040] )*) + \012 /gxc; /\G$/ - or return (%state = (), $cb->(undef, { Status => 599, Reason => "garbled response headers" })); + or return (%state = (), $cb->(undef, { Status => 599, Reason => "Garbled response headers", @pseudo })); } + # remove the "," prefix we added to all headers above substr $_, 0, 1, "" for values %hdr; + # patch in all pseudo headers + %hdr = (%hdr, @pseudo); + + # redirect handling + # microsoft and other shitheads don't give a shit for following standards, + # try to support some common forms of broken Location headers. + if ($hdr{location} !~ /^(?: $ | [^:\/?\#]+ : )/x) { + $hdr{location} =~ s/^\.\/+//; + + my $url = "$rscheme://$uhost:$uport"; + + unless ($hdr{location} =~ s/^\///) { + $url .= $upath; + $url =~ s/\/[^\/]*$//; + } + + $hdr{location} = "$url/$hdr{location}"; + } + + my $redirect; + + if ($recurse) { + my $status = $hdr{Status}; + + # industry standard is to redirect POST as GET for + # 301, 302 and 303, in contrast to http/1.0 and 1.1. + # also, the UA should ask the user for 301 and 307 and POST, + # industry standard seems to be to simply follow. + # we go with the industry standard. + if ($status == 301 or $status == 302 or $status == 303) { + # HTTP/1.1 is unclear on how to mutate the method + $method = "GET" unless $method eq "HEAD"; + $redirect = 1; + } elsif ($status == 307) { + $redirect = 1; + } + } + my $finish = sub { + $state{handle}->destroy if $state{handle}; %state = (); # set-cookie processing - if ($arg{cookie_jar} && exists $hdr{"set-cookie"}) { - for (split /\x00/, $hdr{"set-cookie"}) { - my ($cookie, @arg) = split /;\s*/; - my ($name, $value) = split /=/, $cookie, 2; - my %kv = (value => $value, map { split /=/, $_, 2 } @arg); - - my $cdom = (delete $kv{domain}) || $uhost; - my $cpath = (delete $kv{path}) || "/"; - - $cdom =~ s/^.?/./; # make sure it starts with a "." + if ($arg{cookie_jar}) { + for ($_[1]{"set-cookie"}) { + # parse NAME=VALUE + my @kv; + + while (/\G\s* ([^=;,[:space:]]+) \s*=\s* (?: "((?:[^\\"]+|\\.)*)" | ([^=;,[:space:]]*) )/gcxs) { + my $name = $1; + my $value = $3; + + unless ($value) { + $value = $2; + $value =~ s/\\(.)/$1/gs; + } - next if $cdom =~ /\.$/; + push @kv, $name => $value; + + last unless /\G\s*;/gc; + } + + last unless @kv; + + my $name = shift @kv; + my %kv = (value => shift @kv, @kv); + + my $cdom; + my $cpath = (delete $kv{path}) || "/"; + + if (exists $kv{domain}) { + $cdom = delete $kv{domain}; - # this is not rfc-like and not netscape-like. go figure. - my $ndots = $cdom =~ y/.//; - next if $ndots < ($cdom =~ /\.[^.][^.]\.[^.][^.]$/ ? 3 : 2); + $cdom =~ s/^\.?/./; # make sure it starts with a "." + + next if $cdom =~ /\.$/; + + # this is not rfc-like and not netscape-like. go figure. + my $ndots = $cdom =~ y/.//; + next if $ndots < ($cdom =~ /\.[^.][^.]\.[^.][^.]$/ ? 3 : 2); + } else { + $cdom = $uhost; + } # store it $arg{cookie_jar}{version} = 1; $arg{cookie_jar}{$cdom}{$cpath}{$name} = \%kv; + + redo if /\G\s*,/gc; } } - if ($_[1]{Status} =~ /^x30[12]$/ && $recurse) { - # microsoft and other assholes don't give a shit for following standards, - # try to support a common form of broken Location header. - $_[1]{location} =~ s%^/%$scheme://$uhost:$uport/%; - - http_request ($method, $_[1]{location}, %arg, recurse => $recurse - 1, $cb); + if ($redirect && exists $hdr{location}) { + # we ignore any errors, as it is very common to receive + # Content-Length != 0 but no actual body + # we also access %hdr, as $_[1] might be an erro + http_request ( + $method => $hdr{location}, + %arg, + recurse => $recurse - 1, + Redirect => \@_, + $cb); } else { $cb->($_[0], $_[1]); } }; - if ($hdr{Status} =~ /^(?:1..|204|304)$/ or $method eq "HEAD") { - $finish->(undef, \%hdr); - } else { - if (exists $hdr{"content-length"}) { - $_[0]->unshift_read (chunk => $hdr{"content-length"}, sub { - # could cache persistent connection now - if ($hdr{connection} =~ /\bkeep-alive\b/i) { - # but we don't, due to misdesigns, this is annoyingly complex - }; + my $len = $hdr{"content-length"}; - $finish->($_[1], \%hdr); - }); + if (!$redirect && $arg{on_header} && !$arg{on_header}(\%hdr)) { + $finish->(undef, { Status => 598, Reason => "Request cancelled by on_header", @pseudo }); + } elsif ( + $hdr{Status} =~ /^(?:1..|[23]04)$/ + or $method eq "HEAD" + or (defined $len && !$len) + ) { + # no body + $finish->("", \%hdr); + } else { + # body handling, four different code paths + # for want_body_handle, on_body (2x), normal (2x) + # we might read too much here, but it does not matter yet (no pers. connections) + if (!$redirect && $arg{want_body_handle}) { + $_[0]->on_eof (undef); + $_[0]->on_error (undef); + $_[0]->on_read (undef); + + $finish->(delete $state{handle}, \%hdr); + + } elsif ($arg{on_body}) { + $_[0]->on_error (sub { $finish->(undef, { Status => 599, Reason => $_[2], @pseudo }) }); + if ($len) { + $_[0]->on_eof (undef); + $_[0]->on_read (sub { + $len -= length $_[0]{rbuf}; + + $arg{on_body}(delete $_[0]{rbuf}, \%hdr) + or $finish->(undef, { Status => 598, Reason => "Request cancelled by on_body", @pseudo }); + + $len > 0 + or $finish->("", \%hdr); + }); + } else { + $_[0]->on_eof (sub { + $finish->("", \%hdr); + }); + $_[0]->on_read (sub { + $arg{on_body}(delete $_[0]{rbuf}, \%hdr) + or $finish->(undef, { Status => 598, Reason => "Request cancelled by on_body", @pseudo }); + }); + } } else { - # too bad, need to read until we get an error or EOF, - # no way to detect winged data. - $_[0]->on_error (sub { - $finish->($_[0]{rbuf}, \%hdr); - }); $_[0]->on_eof (undef); - $_[0]->on_read (sub { }); + + if ($len) { + $_[0]->on_error (sub { $finish->(undef, { Status => 599, Reason => $_[2], @pseudo }) }); + $_[0]->on_read (sub { + $finish->((substr delete $_[0]{rbuf}, 0, $len, ""), \%hdr) + if $len <= length $_[0]{rbuf}; + }); + } else { + $_[0]->on_error (sub { + ($! == Errno::EPIPE || !$!) + ? $finish->(delete $_[0]{rbuf}, \%hdr) + : $finish->(undef, { Status => 599, Reason => $_[2], @pseudo }); + }); + $_[0]->on_read (sub { }); + } } } }); - }); - }, sub { - $timeout - }; + }; + + # now handle proxy-CONNECT method + if ($proxy && $uscheme eq "https") { + # oh dear, we have to wrap it into a connect request + + # maybe re-use $uauthority with patched port? + $state{handle}->push_write ("CONNECT $uhost:$uport HTTP/1.0\015\012Host: $uhost\015\012\015\012"); + $state{handle}->push_read (line => $qr_nlnl, sub { + $_[1] =~ /^HTTP\/([0-9\.]+) \s+ ([0-9]{3}) (?: \s+ ([^\015\012]*) )?/ix + or return (%state = (), $cb->(undef, { Status => 599, Reason => "Invalid proxy connect response ($_[1])", @pseudo })); + + if ($2 == 200) { + $rpath = $upath; + &$handle_actual_request; + } else { + %state = (); + $cb->(undef, { Status => $2, Reason => $3, @pseudo }); + } + }); + } else { + &$handle_actual_request; + } + + }, $arg{on_prepare} || sub { $timeout }; }; defined wantarray && AnyEvent::Util::guard { %state = () } } -sub http_get($$;@) { +sub http_get($@) { unshift @_, "GET"; &http_request } -sub http_head($$;@) { +sub http_head($@) { unshift @_, "HEAD"; &http_request } -sub http_post($$$;@) { - unshift @_, "POST", "body"; +sub http_post($$@) { + my $url = shift; + unshift @_, "POST", $url, "body"; &http_request } =back +=head2 DNS CACHING + +AnyEvent::HTTP uses the AnyEvent::Socket::tcp_connect function for +the actual connection, which in turn uses AnyEvent::DNS to resolve +hostnames. The latter is a simple stub resolver and does no caching +on its own. If you want DNS caching, you currently have to provide +your own default resolver (by storing a suitable resolver object in +C<$AnyEvent::DNS::RESOLVER>). + =head2 GLOBAL FUNCTIONS AND VARIABLES =over 4 @@ -469,7 +784,10 @@ =item AnyEvent::HTTP::set_proxy "proxy-url" Sets the default proxy server to use. The proxy-url must begin with a -string of the form C (optionally C). +string of the form C (optionally C), croaks +otherwise. + +To clear an already-set proxy, use C. =item $AnyEvent::HTTP::MAX_RECURSE @@ -478,30 +796,41 @@ =item $AnyEvent::HTTP::USERAGENT The default value for the C header (the default is -C). +C). -=item $AnyEvent::HTTP::MAX_PERSISTENT +=item $AnyEvent::HTTP::MAX_PER_HOST -The maximum number of persistent connections to keep open (default: 8). +The maximum number of concurrent connections to the same host (identified +by the hostname). If the limit is exceeded, then the additional requests +are queued until previous connections are closed. -Not implemented currently. +The default value for this is C<4>, and it is highly advisable to not +increase it. -=item $AnyEvent::HTTP::PERSISTENT_TIMEOUT +=item $AnyEvent::HTTP::ACTIVE -The maximum time to cache a persistent connection, in seconds (default: 2). - -Not implemented currently. +The number of active connections. This is not the number of currently +running requests, but the number of currently open and non-idle TCP +connections. This number of can be useful for load-leveling. =back =cut sub set_proxy($) { - $PROXY = [$2, $3 || 3128, $1] if $_[0] =~ m%^(https?):// ([^:/]+) (?: : (\d*) )?%ix; + if (length $_[0]) { + $_[0] =~ m%^(https?):// ([^:/]+) (?: : (\d*) )?%ix + or Carp::croak "$_[0]: invalid proxy URL"; + $PROXY = [$2, $3 || 3128, $1] + } else { + undef $PROXY; + } } # initialise proxy from environment -set_proxy $ENV{http_proxy}; +eval { + set_proxy $ENV{http_proxy}; +}; =head1 SEE ALSO @@ -509,8 +838,11 @@ =head1 AUTHOR - Marc Lehmann - http://home.schmorp.de/ + Marc Lehmann + http://home.schmorp.de/ + +With many thanks to Дмитрий Шалашов, who provided countless +testcases and bugreports. =cut