ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/AnyEvent-HTTP/HTTP.pm
(Generate patch)

Comparing AnyEvent-HTTP/HTTP.pm (file contents):
Revision 1.6 by root, Wed Jun 4 12:05:45 2008 UTC vs.
Revision 1.68 by root, Fri Dec 31 19:22:18 2010 UTC

3AnyEvent::HTTP - simple but non-blocking HTTP/HTTPS client 3AnyEvent::HTTP - simple but non-blocking HTTP/HTTPS client
4 4
5=head1 SYNOPSIS 5=head1 SYNOPSIS
6 6
7 use AnyEvent::HTTP; 7 use AnyEvent::HTTP;
8
9 http_get "http://www.nethype.de/", sub { print $_[1] };
10
11 # ... do something else here
8 12
9=head1 DESCRIPTION 13=head1 DESCRIPTION
10 14
11This module is an L<AnyEvent> user, you need to make sure that you use and 15This module is an L<AnyEvent> user, you need to make sure that you use and
12run a supported event loop. 16run a supported event loop.
13 17
18This module implements a simple, stateless and non-blocking HTTP
19client. It supports GET, POST and other request methods, cookies and more,
20all on a very low level. It can follow redirects supports proxies and
21automatically limits the number of connections to the values specified in
22the RFC.
23
24It should generally be a "good client" that is enough for most HTTP
25tasks. Simple tasks should be simple, but complex tasks should still be
26possible as the user retains control over request and response headers.
27
28The caller is responsible for authentication management, cookies (if
29the simplistic implementation in this module doesn't suffice), referer
30and other high-level protocol details for which this module offers only
31limited support.
32
14=head2 METHODS 33=head2 METHODS
15 34
16=over 4 35=over 4
17 36
18=cut 37=cut
20package AnyEvent::HTTP; 39package AnyEvent::HTTP;
21 40
22use strict; 41use strict;
23no warnings; 42no warnings;
24 43
25use Carp; 44use Errno ();
26 45
27use AnyEvent (); 46use AnyEvent 5.0 ();
28use AnyEvent::Util (); 47use AnyEvent::Util ();
29use AnyEvent::Socket ();
30use AnyEvent::Handle (); 48use AnyEvent::Handle ();
31 49
32use base Exporter::; 50use base Exporter::;
33 51
34our $VERSION = '1.0'; 52our $VERSION = '1.5';
35 53
36our @EXPORT = qw(http_get http_request); 54our @EXPORT = qw(http_get http_post http_head http_request);
37 55
38our $USERAGENT = "Mozilla/5.0 (compatible; AnyEvent::HTTP/$VERSION; +http://software.schmorp.de/pkg/AnyEvent)"; 56our $USERAGENT = "Mozilla/5.0 (compatible; U; AnyEvent-HTTP/$VERSION; +http://software.schmorp.de/pkg/AnyEvent)";
39our $MAX_RECURSE = 10; 57our $MAX_RECURSE = 10;
40our $MAX_PERSISTENT = 8; 58our $MAX_PERSISTENT = 8;
41our $PERSISTENT_TIMEOUT = 2; 59our $PERSISTENT_TIMEOUT = 2;
42our $TIMEOUT = 300; 60our $TIMEOUT = 300;
43 61
44# changing these is evil 62# changing these is evil
45our $MAX_PERSISTENT_PER_HOST = 2; 63our $MAX_PERSISTENT_PER_HOST = 0;
46our $MAX_PER_HOST = 4; # not respected yet :( 64our $MAX_PER_HOST = 4;
47 65
48our $PROXY; 66our $PROXY;
67our $ACTIVE = 0;
49 68
50my %KA_COUNT; # number of open keep-alive connections per host 69my %KA_COUNT; # number of open keep-alive connections per host
70my %CO_SLOT; # number of open connections, and wait queue, per host
51 71
52=item http_get $url, key => value..., $cb->($data, $headers) 72=item http_get $url, key => value..., $cb->($data, $headers)
53 73
54Executes an HTTP-GET request. See the http_request function for details on 74Executes an HTTP-GET request. See the http_request function for details on
55additional parameters. 75additional parameters and the return value.
56 76
57=item http_head $url, key => value..., $cb->($data, $headers) 77=item http_head $url, key => value..., $cb->($data, $headers)
58 78
59Executes an HTTP-HEAD request. See the http_request function for details on 79Executes an HTTP-HEAD request. See the http_request function for details
60additional parameters. 80on additional parameters and the return value.
61 81
62=item http_post $url, $body, key => value..., $cb->($data, $headers) 82=item http_post $url, $body, key => value..., $cb->($data, $headers)
63 83
64Executes an HTTP-POST request with a requets body of C<$bod>. See the 84Executes an HTTP-POST request with a request body of C<$body>. See the
65http_request function for details on additional parameters. 85http_request function for details on additional parameters and the return
86value.
66 87
67=item http_request $method => $url, key => value..., $cb->($data, $headers) 88=item http_request $method => $url, key => value..., $cb->($data, $headers)
68 89
69Executes a HTTP request of type C<$method> (e.g. C<GET>, C<POST>). The URL 90Executes a HTTP request of type C<$method> (e.g. C<GET>, C<POST>). The URL
70must be an absolute http or https URL. 91must be an absolute http or https URL.
71 92
93When called in void context, nothing is returned. In other contexts,
94C<http_request> returns a "cancellation guard" - you have to keep the
95object at least alive until the callback get called. If the object gets
96destroyed before the callback is called, the request will be cancelled.
97
72The callback will be called with the response data as first argument 98The callback will be called with the response body data as first argument
73(or C<undef> if it wasn't available due to errors), and a hash-ref with 99(or C<undef> if an error occured), and a hash-ref with response headers
74response headers as second argument. 100(and trailers) as second argument.
75 101
76All the headers in that has are lowercased. In addition to the response 102All the headers in that hash are lowercased. In addition to the response
77headers, the three "pseudo-headers" C<HTTPVersion>, C<Status> and 103headers, the "pseudo-headers" (uppercase to avoid clashing with possible
78C<Reason> contain the three parts of the HTTP Status-Line of the same 104response headers) C<HTTPVersion>, C<Status> and C<Reason> contain the
79name. 105three parts of the HTTP Status-Line of the same name. If an error occurs
106during the body phase of a request, then the original C<Status> and
107C<Reason> values from the header are available as C<OrigStatus> and
108C<OrigReason>.
109
110The pseudo-header C<URL> contains the actual URL (which can differ from
111the requested URL when following redirects - for example, you might get
112an error that your URL scheme is not supported even though your URL is a
113valid http URL because it redirected to an ftp URL, in which case you can
114look at the URL pseudo header).
115
116The pseudo-header C<Redirect> only exists when the request was a result
117of an internal redirect. In that case it is an array reference with
118the C<($data, $headers)> from the redirect response. Note that this
119response could in turn be the result of a redirect itself, and C<<
120$headers->{Redirect}[1]{Redirect} >> will then contain the original
121response, and so on.
122
123If the server sends a header multiple times, then their contents will be
124joined together with a comma (C<,>), as per the HTTP spec.
80 125
81If an internal error occurs, such as not being able to resolve a hostname, 126If an internal error occurs, such as not being able to resolve a hostname,
82then C<$data> will be C<undef>, C<< $headers->{Status} >> will be C<599> 127then C<$data> will be C<undef>, C<< $headers->{Status} >> will be C<59x>
83and the C<Reason> pseudo-header will contain an error message. 128(usually C<599>) and the C<Reason> pseudo-header will contain an error
129message.
84 130
85A typical callback might look like this: 131A typical callback might look like this:
86 132
87 sub { 133 sub {
88 my ($body, $hdr) = @_; 134 my ($body, $hdr) = @_;
104Whether to recurse requests or not, e.g. on redirects, authentication 150Whether to recurse requests or not, e.g. on redirects, authentication
105retries and so on, and how often to do so. 151retries and so on, and how often to do so.
106 152
107=item headers => hashref 153=item headers => hashref
108 154
109The request headers to use. 155The request headers to use. Currently, C<http_request> may provide its own
156C<Host:>, C<Content-Length:>, C<Connection:> and C<Cookie:> headers and
157will provide defaults for C<TE:>, C<Referer:> and C<User-Agent:> (this can
158be suppressed by using C<undef> for these headers in which case they won't
159be sent at all).
110 160
111=item timeout => $seconds 161=item timeout => $seconds
112 162
113The time-out to use for various stages - each connect attempt will reset 163The time-out to use for various stages - each connect attempt will reset
114the timeout, as will read or write activity. Default timeout is 5 minutes. 164the timeout, as will read or write activity, i.e. this is not an overall
165timeout.
166
167Default timeout is 5 minutes.
115 168
116=item proxy => [$host, $port[, $scheme]] or undef 169=item proxy => [$host, $port[, $scheme]] or undef
117 170
118Use the given http proxy for all requests. If not specified, then the 171Use the given http proxy for all requests. If not specified, then the
119default proxy (as specified by C<$ENV{http_proxy}>) is used. 172default proxy (as specified by C<$ENV{http_proxy}>) is used.
120 173
121C<$scheme> must be either missing or C<http> for HTTP, or C<https> for 174C<$scheme> must be either missing, C<http> for HTTP or C<https> for
122HTTPS. 175HTTPS.
123 176
124=item body => $string 177=item body => $string
125 178
126The request body, usually empty. Will be-sent as-is (future versions of 179The request body, usually empty. Will be sent as-is (future versions of
127this module might offer more options). 180this module might offer more options).
128 181
182=item cookie_jar => $hash_ref
183
184Passing this parameter enables (simplified) cookie-processing, loosely
185based on the original netscape specification.
186
187The C<$hash_ref> must be an (initially empty) hash reference which will
188get updated automatically. It is possible to save the cookie_jar to
189persistent storage with something like JSON or Storable, but this is not
190recommended, as expiry times are currently being ignored.
191
192Note that this cookie implementation is not of very high quality, nor
193meant to be complete. If you want complete cookie management you have to
194do that on your own. C<cookie_jar> is meant as a quick fix to get some
195cookie-using sites working. Cookies are a privacy disaster, do not use
196them unless required to.
197
198=item tls_ctx => $scheme | $tls_ctx
199
200Specifies the AnyEvent::TLS context to be used for https connections. This
201parameter follows the same rules as the C<tls_ctx> parameter to
202L<AnyEvent::Handle>, but additionally, the two strings C<low> or
203C<high> can be specified, which give you a predefined low-security (no
204verification, highest compatibility) and high-security (CA and common-name
205verification) TLS context.
206
207The default for this option is C<low>, which could be interpreted as "give
208me the page, no matter what".
209
210=item on_prepare => $callback->($fh)
211
212In rare cases you need to "tune" the socket before it is used to
213connect (for exmaple, to bind it on a given IP address). This parameter
214overrides the prepare callback passed to C<AnyEvent::Socket::tcp_connect>
215and behaves exactly the same way (e.g. it has to provide a
216timeout). See the description for the C<$prepare_cb> argument of
217C<AnyEvent::Socket::tcp_connect> for details.
218
219=item tcp_connect => $callback->($host, $service, $connect_cb, $prepare_cb)
220
221In even rarer cases you want total control over how AnyEvent::HTTP
222establishes connections. Normally it uses L<AnyEvent::Socket::tcp_connect>
223to do this, but you can provide your own C<tcp_connect> function -
224obviously, it has to follow the same calling conventions, except that it
225may always return a connection guard object.
226
227There are probably lots of weird uses for this function, starting from
228tracing the hosts C<http_request> actually tries to connect, to (inexact
229but fast) host => IP address caching or even socks protocol support.
230
231=item on_header => $callback->($headers)
232
233When specified, this callback will be called with the header hash as soon
234as headers have been successfully received from the remote server (not on
235locally-generated errors).
236
237It has to return either true (in which case AnyEvent::HTTP will continue),
238or false, in which case AnyEvent::HTTP will cancel the download (and call
239the finish callback with an error code of C<598>).
240
241This callback is useful, among other things, to quickly reject unwanted
242content, which, if it is supposed to be rare, can be faster than first
243doing a C<HEAD> request.
244
245The downside is that cancelling the request makes it impossible to re-use
246the connection. Also, the C<on_header> callback will not receive any
247trailer (headers sent after the response body).
248
249Example: cancel the request unless the content-type is "text/html".
250
251 on_header => sub {
252 $_[0]{"content-type"} =~ /^text\/html\s*(?:;|$)/
253 },
254
255=item on_body => $callback->($partial_body, $headers)
256
257When specified, all body data will be passed to this callback instead of
258to the completion callback. The completion callback will get the empty
259string instead of the body data.
260
261It has to return either true (in which case AnyEvent::HTTP will continue),
262or false, in which case AnyEvent::HTTP will cancel the download (and call
263the completion callback with an error code of C<598>).
264
265The downside to cancelling the request is that it makes it impossible to
266re-use the connection.
267
268This callback is useful when the data is too large to be held in memory
269(so the callback writes it to a file) or when only some information should
270be extracted, or when the body should be processed incrementally.
271
272It is usually preferred over doing your own body handling via
273C<want_body_handle>, but in case of streaming APIs, where HTTP is
274only used to create a connection, C<want_body_handle> is the better
275alternative, as it allows you to install your own event handler, reducing
276resource usage.
277
278=item want_body_handle => $enable
279
280When enabled (default is disabled), the behaviour of AnyEvent::HTTP
281changes considerably: after parsing the headers, and instead of
282downloading the body (if any), the completion callback will be
283called. Instead of the C<$body> argument containing the body data, the
284callback will receive the L<AnyEvent::Handle> object associated with the
285connection. In error cases, C<undef> will be passed. When there is no body
286(e.g. status C<304>), the empty string will be passed.
287
288The handle object might or might not be in TLS mode, might be connected to
289a proxy, be a persistent connection etc., and configured in unspecified
290ways. The user is responsible for this handle (it will not be used by this
291module anymore).
292
293This is useful with some push-type services, where, after the initial
294headers, an interactive protocol is used (typical example would be the
295push-style twitter API which starts a JSON/XML stream).
296
297If you think you need this, first have a look at C<on_body>, to see if
298that doesn't solve your problem in a better way.
299
129=back 300=back
130 301
131=back 302Example: do a simple HTTP GET request for http://www.nethype.de/ and print
303the response body.
304
305 http_request GET => "http://www.nethype.de/", sub {
306 my ($body, $hdr) = @_;
307 print "$body\n";
308 };
309
310Example: do a HTTP HEAD request on https://www.google.com/, use a
311timeout of 30 seconds.
312
313 http_request
314 GET => "https://www.google.com",
315 timeout => 30,
316 sub {
317 my ($body, $hdr) = @_;
318 use Data::Dumper;
319 print Dumper $hdr;
320 }
321 ;
322
323Example: do another simple HTTP GET request, but immediately try to
324cancel it.
325
326 my $request = http_request GET => "http://www.nethype.de/", sub {
327 my ($body, $hdr) = @_;
328 print "$body\n";
329 };
330
331 undef $request;
132 332
133=cut 333=cut
134 334
335sub _slot_schedule;
336sub _slot_schedule($) {
337 my $host = shift;
338
339 while ($CO_SLOT{$host}[0] < $MAX_PER_HOST) {
340 if (my $cb = shift @{ $CO_SLOT{$host}[1] }) {
341 # somebody wants that slot
342 ++$CO_SLOT{$host}[0];
343 ++$ACTIVE;
344
345 $cb->(AnyEvent::Util::guard {
346 --$ACTIVE;
347 --$CO_SLOT{$host}[0];
348 _slot_schedule $host;
349 });
350 } else {
351 # nobody wants the slot, maybe we can forget about it
352 delete $CO_SLOT{$host} unless $CO_SLOT{$host}[0];
353 last;
354 }
355 }
356}
357
358# wait for a free slot on host, call callback
359sub _get_slot($$) {
360 push @{ $CO_SLOT{$_[0]}[1] }, $_[1];
361
362 _slot_schedule $_[0];
363}
364
365# continue to parse $_ for headers and place them into the arg
366sub parse_hdr() {
367 my %hdr;
368
369 # things seen, not parsed:
370 # p3pP="NON CUR OTPi OUR NOR UNI"
371
372 $hdr{lc $1} .= ",$2"
373 while /\G
374 ([^:\000-\037]*):
375 [\011\040]*
376 ((?: [^\012]+ | \012[\011\040] )*)
377 \012
378 /gxc;
379
380 /\G$/
381 or return;
382
383 # remove the "," prefix we added to all headers above
384 substr $_, 0, 1, ""
385 for values %hdr;
386
387 \%hdr
388}
389
390our $qr_nlnl = qr{(?<![^\012])\015?\012};
391
392our $TLS_CTX_LOW = { cache => 1, sslv2 => 1 };
393our $TLS_CTX_HIGH = { cache => 1, verify => 1, verify_peername => "https" };
394
135sub http_request($$$;@) { 395sub http_request($$@) {
136 my $cb = pop; 396 my $cb = pop;
137 my ($method, $url, %arg) = @_; 397 my ($method, $url, %arg) = @_;
138 398
139 my %hdr; 399 my %hdr;
140 400
401 $arg{tls_ctx} = $TLS_CTX_LOW if $arg{tls_ctx} eq "low" || !exists $arg{tls_ctx};
402 $arg{tls_ctx} = $TLS_CTX_HIGH if $arg{tls_ctx} eq "high";
403
141 $method = uc $method; 404 $method = uc $method;
142 405
143 if (my $hdr = delete $arg{headers}) { 406 if (my $hdr = $arg{headers}) {
144 while (my ($k, $v) = each %$hdr) { 407 while (my ($k, $v) = each %$hdr) {
145 $hdr{lc $k} = $v; 408 $hdr{lc $k} = $v;
146 } 409 }
147 } 410 }
148 411
412 # pseudo headers for all subsequent responses
413 my @pseudo = (URL => $url);
414 push @pseudo, Redirect => delete $arg{Redirect} if exists $arg{Redirect};
415
416 my $recurse = exists $arg{recurse} ? delete $arg{recurse} : $MAX_RECURSE;
417
418 return $cb->(undef, { @pseudo, Status => 599, Reason => "Too many redirections" })
419 if $recurse < 0;
420
149 my $proxy = $arg{proxy} || $PROXY; 421 my $proxy = $arg{proxy} || $PROXY;
150 my $timeout = $arg{timeout} || $TIMEOUT; 422 my $timeout = $arg{timeout} || $TIMEOUT;
151 my $recurse = exists $arg{recurse} ? $arg{recurse} : $MAX_RECURSE;
152 423
153 $hdr{"user-agent"} ||= $USERAGENT; 424 my ($uscheme, $uauthority, $upath, $query, $fragment) =
425 $url =~ m|(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*)(?:(\?[^#]*))?(?:#(.*))?|;
154 426
155 my ($host, $port, $path, $scheme); 427 $uscheme = lc $uscheme;
428
429 my $uport = $uscheme eq "http" ? 80
430 : $uscheme eq "https" ? 443
431 : return $cb->(undef, { @pseudo, Status => 599, Reason => "Only http and https URL schemes supported" });
432
433 $uauthority =~ /^(?: .*\@ )? ([^\@:]+) (?: : (\d+) )?$/x
434 or return $cb->(undef, { @pseudo, Status => 599, Reason => "Unparsable URL" });
435
436 my $uhost = $1;
437 $uport = $2 if defined $2;
438
439 $hdr{host} = defined $2 ? "$uhost:$2" : "$uhost"
440 unless exists $hdr{host};
441
442 $uhost =~ s/^\[(.*)\]$/$1/;
443 $upath .= $query if length $query;
444
445 $upath =~ s%^/?%/%;
446
447 # cookie processing
448 if (my $jar = $arg{cookie_jar}) {
449 %$jar = () if $jar->{version} != 1;
450
451 my @cookie;
452
453 while (my ($chost, $v) = each %$jar) {
454 if ($chost =~ /^\./) {
455 next unless $chost eq substr $uhost, -length $chost;
456 } elsif ($chost =~ /\./) {
457 next unless $chost eq $uhost;
458 } else {
459 next;
460 }
461
462 while (my ($cpath, $v) = each %$v) {
463 next unless $cpath eq substr $upath, 0, length $cpath;
464
465 while (my ($k, $v) = each %$v) {
466 next if $uscheme ne "https" && exists $v->{secure};
467 my $value = $v->{value};
468 $value =~ s/([\\"])/\\$1/g;
469 push @cookie, "$k=\"$value\"";
470 }
471 }
472 }
473
474 $hdr{cookie} = join "; ", @cookie
475 if @cookie;
476 }
477
478 my ($rhost, $rport, $rscheme, $rpath); # request host, port, path
156 479
157 if ($proxy) { 480 if ($proxy) {
158 ($host, $port, $scheme) = @$proxy; 481 ($rpath, $rhost, $rport, $rscheme) = ($url, @$proxy);
159 $path = $url; 482
483 $rscheme = "http" unless defined $rscheme;
484
485 # don't support https requests over https-proxy transport,
486 # can't be done with tls as spec'ed, unless you double-encrypt.
487 $rscheme = "http" if $uscheme eq "https" && $rscheme eq "https";
160 } else { 488 } else {
161 ($scheme, my $authority, $path, my $query, my $fragment) = 489 ($rhost, $rport, $rscheme, $rpath) = ($uhost, $uport, $uscheme, $upath);
162 $url =~ m|(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*)(?:\?([^#]*))?(?:#(.*))?|;
163
164 $port = $scheme eq "http" ? 80
165 : $scheme eq "https" ? 443
166 : croak "$url: only http and https URLs supported";
167
168 $authority =~ /^(?: .*\@ )? ([^\@:]+) (?: : (\d+) )?$/x
169 or croak "$authority: unparsable URL";
170
171 $host = $1;
172 $port = $2 if defined $2;
173
174 $host =~ s/^\[(.*)\]$/$1/;
175 $path .= "?$query" if length $query;
176
177 $path = "/" unless $path;
178
179 $hdr{host} = $host = lc $host;
180 } 490 }
181 491
182 $scheme = lc $scheme; 492 # leave out fragment and query string, just a heuristic
493 $hdr{referer} = "$uscheme://$uauthority$upath" unless exists $hdr{referer};
494 $hdr{"user-agent"} = $USERAGENT unless exists $hdr{"user-agent"};
183 495
184 my %state;
185
186 $state{body} = delete $arg{body};
187
188 $hdr{"content-length"} = length $state{body}; 496 $hdr{"content-length"} = length $arg{body}
497 if length $arg{body} || $method ne "GET";
189 498
190 $state{connect_guard} = AnyEvent::Socket::tcp_connect $host, $port, sub { 499 $hdr{connection} = "close TE"; #1.1
500 $hdr{te} = "trailers" unless exists $hdr{te}; #1.1
501
502 my %state = (connect_guard => 1);
503
504 _get_slot $uhost, sub {
505 $state{slot_guard} = shift;
506
507 return unless $state{connect_guard};
508
509 my $connect_cb = sub {
191 $state{fh} = shift 510 $state{fh} = shift
192 or return $cb->(undef, { Status => 599, Reason => "$!" });
193
194 delete $state{connect_guard}; # reduce memory usage, save a tree
195
196 # get handle
197 $state{handle} = new AnyEvent::Handle
198 fh => $state{fh},
199 ($scheme eq "https" ? (tls => "connect") : ());
200
201 # limit the number of persistent connections
202 if ($KA_COUNT{$_[1]} < $MAX_PERSISTENT_PER_HOST) {
203 ++$KA_COUNT{$_[1]};
204 $state{handle}{ka_count_guard} = AnyEvent::Util::guard { --$KA_COUNT{$_[1]} };
205 $hdr{connection} = "keep-alive";
206 delete $hdr{connection}; # keep-alive not yet supported
207 } else {
208 delete $hdr{connection};
209 }
210
211 # (re-)configure handle
212 $state{handle}->timeout ($timeout);
213 $state{handle}->on_error (sub {
214 %state = ();
215 $cb->(undef, { Status => 599, Reason => "$!" });
216 });
217 $state{handle}->on_eof (sub {
218 %state = ();
219 $cb->(undef, { Status => 599, Reason => "unexpected end-of-file" });
220 });
221
222 # send request
223 $state{handle}->push_write (
224 "$method $path HTTP/1.0\015\012"
225 . (join "", map "$_: $hdr{$_}\015\012", keys %hdr)
226 . "\015\012"
227 . (delete $state{body})
228 );
229
230 %hdr = (); # reduce memory usage, save a kitten
231
232 # status line
233 $state{handle}->push_read (line => qr/\015?\012/, sub {
234 $_[1] =~ /^HTTP\/([0-9\.]+) \s+ ([0-9]{3}) \s+ ([^\015\012]+)/ix
235 or return (%state = (), $cb->(undef, { Status => 599, Reason => "invalid server response ($_[1])" }));
236
237 my %hdr = ( # response headers
238 HTTPVersion => ",$1",
239 Status => ",$2",
240 Reason => ",$3",
241 );
242
243 # headers, could be optimized a bit
244 $state{handle}->unshift_read (line => qr/\015?\012\015?\012/, sub {
245 for ("$_[1]\012") {
246 # we support spaces in field names, as lotus domino
247 # creates them.
248 $hdr{lc $1} .= ",$2"
249 while /\G
250 ([^:\000-\037]+):
251 [\011\040]*
252 ((?: [^\015\012]+ | \015?\012[\011\040] )*)
253 \015?\012
254 /gxc;
255
256 /\G$/
257 or return $cb->(undef, { Status => 599, Reason => "garbled response headers" });
258 } 511 or do {
259 512 my $err = "$!";
260 substr $_, 0, 1, ""
261 for values %hdr;
262
263 if ($method eq "HEAD") {
264 %state = (); 513 %state = ();
514 return $cb->(undef, { @pseudo, Status => 599, Reason => $err });
515 };
516
517 pop; # free memory, save a tree
518
519 return unless delete $state{connect_guard};
520
521 # get handle
522 $state{handle} = new AnyEvent::Handle
523 fh => $state{fh},
524 peername => $rhost,
525 tls_ctx => $arg{tls_ctx},
526 # these need to be reconfigured on keepalive handles
527 timeout => $timeout,
528 on_error => sub {
529 %state = ();
530 $cb->(undef, { @pseudo, Status => 599, Reason => $_[2] });
531 },
532 on_eof => sub {
533 %state = ();
534 $cb->(undef, { @pseudo, Status => 599, Reason => "Unexpected end-of-file" });
535 },
536 ;
537
538 # limit the number of persistent connections
539 # keepalive not yet supported
540# if ($KA_COUNT{$_[1]} < $MAX_PERSISTENT_PER_HOST) {
541# ++$KA_COUNT{$_[1]};
542# $state{handle}{ka_count_guard} = AnyEvent::Util::guard {
543# --$KA_COUNT{$_[1]}
544# };
545# $hdr{connection} = "keep-alive";
546# }
547
548 $state{handle}->starttls ("connect") if $rscheme eq "https";
549
550 # handle actual, non-tunneled, request
551 my $handle_actual_request = sub {
552 $state{handle}->starttls ("connect") if $uscheme eq "https" && !exists $state{handle}{tls};
553
554 # send request
555 $state{handle}->push_write (
556 "$method $rpath HTTP/1.1\015\012"
557 . (join "", map "\u$_: $hdr{$_}\015\012", grep defined $hdr{$_}, keys %hdr)
558 . "\015\012"
559 . (delete $arg{body})
560 );
561
562 # return if error occured during push_write()
563 return unless %state;
564
565 %hdr = (); # reduce memory usage, save a kitten, also make it possible to re-use
566
567 # status line and headers
568 $state{read_response} = sub {
569 for ("$_[1]") {
570 y/\015//d; # weed out any \015, as they show up in the weirdest of places.
571
572 /^HTTP\/([0-9\.]+) \s+ ([0-9]{3}) (?: \s+ ([^\012]*) )? \012/igxc
573 or return (%state = (), $cb->(undef, { @pseudo, Status => 599, Reason => "Invalid server response" }));
574
575 # 100 Continue handling
576 # should not happen as we don't send expect: 100-continue,
577 # but we handle it just in case.
578 # since we send the request body regardless, if we get an error
579 # we are out of-sync, which we currently do NOT handle correctly.
580 return $state{handle}->push_read (line => $qr_nlnl, $state{read_response})
581 if $2 eq 100;
582
583 push @pseudo,
584 HTTPVersion => $1,
585 Status => $2,
586 Reason => $3,
587 ;
588
589 my $hdr = parse_hdr
590 or return (%state = (), $cb->(undef, { @pseudo, Status => 599, Reason => "Garbled response headers" }));
591
592 %hdr = (%$hdr, @pseudo);
593 }
594
595 # redirect handling
596 # microsoft and other shitheads don't give a shit for following standards,
597 # try to support some common forms of broken Location headers.
598 if ($hdr{location} !~ /^(?: $ | [^:\/?\#]+ : )/x) {
599 $hdr{location} =~ s/^\.\/+//;
600
601 my $url = "$rscheme://$uhost:$uport";
602
603 unless ($hdr{location} =~ s/^\///) {
604 $url .= $upath;
605 $url =~ s/\/[^\/]*$//;
606 }
607
608 $hdr{location} = "$url/$hdr{location}";
609 }
610
611 my $redirect;
612
613 if ($recurse) {
614 my $status = $hdr{Status};
615
616 # industry standard is to redirect POST as GET for
617 # 301, 302 and 303, in contrast to http/1.0 and 1.1.
618 # also, the UA should ask the user for 301 and 307 and POST,
619 # industry standard seems to be to simply follow.
620 # we go with the industry standard.
621 if ($status == 301 or $status == 302 or $status == 303) {
622 # HTTP/1.1 is unclear on how to mutate the method
623 $method = "GET" unless $method eq "HEAD";
624 $redirect = 1;
625 } elsif ($status == 307) {
626 $redirect = 1;
627 }
628 }
629
630 my $finish = sub { # ($data, $err_status, $err_reason[, $keepalive])
631 my $keepalive = pop;
632
633 $state{handle}->destroy if $state{handle};
634 %state = ();
635
636 if (defined $_[1]) {
637 $hdr{OrigStatus} = $hdr{Status}; $hdr{Status} = $_[1];
638 $hdr{OrigReason} = $hdr{Reason}; $hdr{Reason} = $_[2];
639 }
640
641 # set-cookie processing
642 if ($arg{cookie_jar}) {
643 for ($hdr{"set-cookie"}) {
644 # parse NAME=VALUE
645 my @kv;
646
647 while (/\G\s* ([^=;,[:space:]]+) \s*=\s* (?: "((?:[^\\"]+|\\.)*)" | ([^=;,[:space:]]*) )/gcxs) {
648 my $name = $1;
649 my $value = $3;
650
651 unless ($value) {
652 $value = $2;
653 $value =~ s/\\(.)/$1/gs;
654 }
655
656 push @kv, $name => $value;
657
658 last unless /\G\s*;/gc;
659 }
660
661 last unless @kv;
662
663 my $name = shift @kv;
664 my %kv = (value => shift @kv, @kv);
665
666 my $cdom;
667 my $cpath = (delete $kv{path}) || "/";
668
669 if (exists $kv{domain}) {
670 $cdom = delete $kv{domain};
671
672 $cdom =~ s/^\.?/./; # make sure it starts with a "."
673
674 next if $cdom =~ /\.$/;
675
676 # this is not rfc-like and not netscape-like. go figure.
677 my $ndots = $cdom =~ y/.//;
678 next if $ndots < ($cdom =~ /\.[^.][^.]\.[^.][^.]$/ ? 3 : 2);
679 } else {
680 $cdom = $uhost;
681 }
682
683 # store it
684 $arg{cookie_jar}{version} = 1;
685 $arg{cookie_jar}{$cdom}{$cpath}{$name} = \%kv;
686
687 redo if /\G\s*,/gc;
688 }
689 }
690
691 if ($redirect && exists $hdr{location}) {
692 # we ignore any errors, as it is very common to receive
693 # Content-Length != 0 but no actual body
694 # we also access %hdr, as $_[1] might be an erro
695 http_request (
696 $method => $hdr{location},
697 %arg,
698 recurse => $recurse - 1,
699 Redirect => [$_[0], \%hdr],
700 $cb);
701 } else {
265 $cb->(undef, \%hdr); 702 $cb->($_[0], \%hdr);
703 }
704 };
705
706 my $len = $hdr{"content-length"};
707
708 if (!$redirect && $arg{on_header} && !$arg{on_header}(\%hdr)) {
709 $finish->(undef, 598 => "Request cancelled by on_header");
710 } elsif (
711 $hdr{Status} =~ /^(?:1..|204|205|304)$/
712 or $method eq "HEAD"
713 or (defined $len && !$len)
714 ) {
715 # no body
716 $finish->("", undef, undef, 1);
266 } else { 717 } else {
267 if (exists $hdr{"content-length"}) { 718 # body handling, many different code paths
719 # - no body expected
720 # - want_body_handle
721 # - te chunked
722 # - 2x length known (with or without on_body)
723 # - 2x length not known (with or without on_body)
724 if (!$redirect && $arg{want_body_handle}) {
725 $_[0]->on_eof (undef);
726 $_[0]->on_error (undef);
727 $_[0]->on_read (undef);
728
729 $finish->(delete $state{handle});
730
731 } elsif ($hdr{"transfer-encoding"} =~ /\bchunked\b/i) {
732 my $cl = 0;
733 my $body = undef;
734 my $on_body = $arg{on_body} || sub { $body .= shift; 1 };
735
736 $_[0]->on_error (sub { $finish->(undef, 599 => $_[2]) });
737
738 my $read_chunk; $read_chunk = sub {
739 $_[1] =~ /^([0-9a-fA-F]+)/
740 or $finish->(undef, 599 => "Garbled chunked transfer encoding");
741
742 my $len = hex $1;
743
744 if ($len) {
745 $cl += $len;
746
268 $_[0]->unshift_read (chunk => $hdr{"content-length"}, sub { 747 $_[0]->push_read (chunk => $len, sub {
269 # could cache persistent connection now 748 $on_body->($_[1], \%hdr)
270 if ($hdr{connection} =~ /\bkeep-alive\b/i) { 749 or return $finish->(undef, 598 => "Request cancelled by on_body");
271 # but we don't, due to misdesigns, this is annoyingly complex 750
751 $_[0]->push_read (line => sub {
752 length $_[1]
753 and return $finish->(undef, 599 => "Garbled chunked transfer encoding");
754 $_[0]->push_read (line => $read_chunk);
755 });
756 });
757 } else {
758 $hdr{"content-length"} ||= $cl;
759
760 $_[0]->push_read (line => $qr_nlnl, sub {
761 if (length $_[1]) {
762 for ("$_[1]") {
763 y/\015//d; # weed out any \015, as they show up in the weirdest of places.
764
765 my $hdr = parse_hdr
766 or return $finish->(undef, 599 => "Garbled response trailers");
767
768 %hdr = (%hdr, %$hdr);
769 }
770 }
771
772 $finish->($body, undef, undef, 1);
773 });
774 }
272 }; 775 };
273 776
777 $_[0]->push_read (line => $read_chunk);
778
779 } elsif ($arg{on_body}) {
780 $_[0]->on_error (sub { $finish->(undef, 599 => $_[2]) });
781
782 if ($len) {
783 $_[0]->on_read (sub {
784 $len -= length $_[0]{rbuf};
785
786 $arg{on_body}(delete $_[0]{rbuf}, \%hdr)
787 or return $finish->(undef, 598 => "Request cancelled by on_body");
788
789 $len > 0
790 or $finish->("", undef, undef, 1);
791 });
274 %state = (); 792 } else {
275 $cb->($_[1], \%hdr); 793 $_[0]->on_eof (sub {
794 $finish->("");
795 });
796 $_[0]->on_read (sub {
797 $arg{on_body}(delete $_[0]{rbuf}, \%hdr)
798 or $finish->(undef, 598 => "Request cancelled by on_body");
799 });
800 }
801 } else {
802 $_[0]->on_eof (undef);
803
804 if ($len) {
805 $_[0]->on_error (sub { $finish->(undef, 599 => $_[2]) });
806 $_[0]->on_read (sub {
807 $finish->((substr delete $_[0]{rbuf}, 0, $len, ""), undef, undef, 1)
808 if $len <= length $_[0]{rbuf};
809 });
810 } else {
811 $_[0]->on_error (sub {
812 ($! == Errno::EPIPE || !$!)
813 ? $finish->(delete $_[0]{rbuf})
814 : $finish->(undef, 599 => $_[2]);
815 });
816 $_[0]->on_read (sub { });
817 }
276 }); 818 }
819 }
820 };
821
822 $state{handle}->push_read (line => $qr_nlnl, $state{read_response});
823 };
824
825 # now handle proxy-CONNECT method
826 if ($proxy && $uscheme eq "https") {
827 # oh dear, we have to wrap it into a connect request
828
829 # maybe re-use $uauthority with patched port?
830 $state{handle}->push_write ("CONNECT $uhost:$uport HTTP/1.0\015\012Host: $uhost\015\012\015\012");
831 $state{handle}->push_read (line => $qr_nlnl, sub {
832 $_[1] =~ /^HTTP\/([0-9\.]+) \s+ ([0-9]{3}) (?: \s+ ([^\015\012]*) )?/ix
833 or return (%state = (), $cb->(undef, { @pseudo, Status => 599, Reason => "Invalid proxy connect response ($_[1])" }));
834
835 if ($2 == 200) {
836 $rpath = $upath;
837 &$handle_actual_request;
277 } else { 838 } else {
278 # too bad, need to read until we get an error or EOF,
279 # no way to detect winged data.
280 $_[0]->on_error (sub {
281 %state = (); 839 %state = ();
282 $cb->($_[0]{rbuf}, \%hdr); 840 $cb->(undef, { @pseudo, Status => $2, Reason => $3 });
283 });
284 $_[0]->on_eof (undef);
285 $_[0]->on_read (sub { });
286 } 841 }
287 } 842 });
843 } else {
844 &$handle_actual_request;
288 }); 845 }
289 }); 846 };
290 }, sub { 847
291 $timeout 848 my $tcp_connect = $arg{tcp_connect}
849 || do { require AnyEvent::Socket; \&AnyEvent::Socket::tcp_connect };
850
851 $state{connect_guard} = $tcp_connect->($rhost, $rport, $connect_cb, $arg{on_prepare} || sub { $timeout });
852
292 }; 853 };
293 854
294 defined wantarray && AnyEvent::Util::guard { %state = () } 855 defined wantarray && AnyEvent::Util::guard { %state = () }
295} 856}
296 857
297sub http_get($$;@) { 858sub http_get($@) {
298 unshift @_, "GET"; 859 unshift @_, "GET";
299 &http_request 860 &http_request
300} 861}
301 862
302sub http_head($$;@) { 863sub http_head($@) {
303 unshift @_, "HEAD"; 864 unshift @_, "HEAD";
304 &http_request 865 &http_request
305} 866}
306 867
307sub http_post($$$;@) { 868sub http_post($$@) {
869 my $url = shift;
308 unshift @_, "POST", "body"; 870 unshift @_, "POST", $url, "body";
309 &http_request 871 &http_request
310} 872}
311 873
874=back
875
876=head2 DNS CACHING
877
878AnyEvent::HTTP uses the AnyEvent::Socket::tcp_connect function for
879the actual connection, which in turn uses AnyEvent::DNS to resolve
880hostnames. The latter is a simple stub resolver and does no caching
881on its own. If you want DNS caching, you currently have to provide
882your own default resolver (by storing a suitable resolver object in
883C<$AnyEvent::DNS::RESOLVER>).
884
312=head2 GLOBAL FUNCTIONS AND VARIABLES 885=head2 GLOBAL FUNCTIONS AND VARIABLES
313 886
314=over 4 887=over 4
315 888
316=item AnyEvent::HTTP::set_proxy "proxy-url" 889=item AnyEvent::HTTP::set_proxy "proxy-url"
317 890
318Sets the default proxy server to use. The proxy-url must begin with a 891Sets the default proxy server to use. The proxy-url must begin with a
319string of the form C<http://host:port> (optionally C<https:...>). 892string of the form C<http://host:port> (optionally C<https:...>), croaks
893otherwise.
894
895To clear an already-set proxy, use C<undef>.
896
897=item $date = AnyEvent::HTTP::format_date $timestamp
898
899Takes a POSIX timestamp (seconds since the epoch) and formats it as a HTTP
900Date (RFC 2616).
901
902=item $timestamp = AnyEvent::HTTP::parse_date $date
903
904Takes a HTTP Date (RFC 2616) and returns the corresponding POSIX
905timestamp, or C<undef> if the date cannot be parsed.
320 906
321=item $AnyEvent::HTTP::MAX_RECURSE 907=item $AnyEvent::HTTP::MAX_RECURSE
322 908
323The default value for the C<recurse> request parameter (default: C<10>). 909The default value for the C<recurse> request parameter (default: C<10>).
324 910
325=item $AnyEvent::HTTP::USERAGENT 911=item $AnyEvent::HTTP::USERAGENT
326 912
327The default value for the C<User-Agent> header (the default is 913The default value for the C<User-Agent> header (the default is
328C<Mozilla/5.0 (compatible; AnyEvent::HTTP/$VERSION; +http://software.schmorp.de/pkg/AnyEvent)>). 914C<Mozilla/5.0 (compatible; U; AnyEvent-HTTP/$VERSION; +http://software.schmorp.de/pkg/AnyEvent)>).
329 915
330=item $AnyEvent::HTTP::MAX_PERSISTENT 916=item $AnyEvent::HTTP::MAX_PER_HOST
331 917
332The maximum number of persistent connections to keep open (default: 8). 918The maximum number of concurrent connections to the same host (identified
919by the hostname). If the limit is exceeded, then the additional requests
920are queued until previous connections are closed.
333 921
334Not implemented currently. 922The default value for this is C<4>, and it is highly advisable to not
923increase it.
335 924
336=item $AnyEvent::HTTP::PERSISTENT_TIMEOUT 925=item $AnyEvent::HTTP::ACTIVE
337 926
338The maximum time to cache a persistent connection, in seconds (default: 2). 927The number of active connections. This is not the number of currently
339 928running requests, but the number of currently open and non-idle TCP
340Not implemented currently. 929connections. This number of can be useful for load-leveling.
341 930
342=back 931=back
343 932
344=cut 933=cut
345 934
935our @month = qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec);
936our @weekday = qw(Sun Mon Tue Wed Thu Fri Sat);
937
938sub format_date($) {
939 my ($time) = @_;
940
941 # RFC 822/1123 format
942 my ($S, $M, $H, $mday, $mon, $year, $wday, $yday, undef) = gmtime $time;
943
944 sprintf "%s, %02d %s %04d %02d:%02d:%02d GMT",
945 $weekday[$wday], $mday, $month[$mon], $year + 1900,
946 $H, $M, $S;
947}
948
949sub parse_date($) {
950 my ($date) = @_;
951
952 my ($d, $m, $y, $H, $M, $S);
953
954 if ($date =~ /^[A-Z][a-z][a-z], ([0-9][0-9]) ([A-Z][a-z][a-z]) ([0-9][0-9][0-9][0-9]) ([0-9][0-9]):([0-9][0-9]):([0-9][0-9]) GMT$/) {
955 # RFC 822/1123, required by RFC 2616
956 ($d, $m, $y, $H, $M, $S) = ($1, $2, $3, $4, $5, $6);
957
958 } elsif ($date =~ /^[A-Z][a-z]+, ([0-9][0-9])-([A-Z][a-z][a-z])-([0-9][0-9]) ([0-9][0-9]):([0-9][0-9]):([0-9][0-9]) GMT$/) {
959 # RFC 850
960 ($d, $m, $y, $H, $M, $S) = ($1, $2, $3 < 69 ? $3 + 2000 : $3 + 1900, $4, $5, $6);
961
962 } elsif ($date =~ /^[A-Z][a-z][a-z] ([A-Z][a-z][a-z]) ([0-9 ][0-9]) ([0-9][0-9]):([0-9][0-9]):([0-9][0-9]) ([0-9][0-9][0-9][0-9])$/) {
963 # ISO C's asctime
964 ($d, $m, $y, $H, $M, $S) = ($2, $1, $6, $3, $4, $5);
965 }
966 # other formats fail in the loop below
967
968 for (0..11) {
969 if ($m eq $month[$_]) {
970 require Time::Local;
971 return Time::Local::timegm ($S, $M, $H, $d, $_, $y);
972 }
973 }
974
975 undef
976}
977
346sub set_proxy($) { 978sub set_proxy($) {
979 if (length $_[0]) {
347 $PROXY = [$2, $3 || 3128, $1] if $_[0] =~ m%^(https?):// ([^:/]+) (?: : (\d*) )?%ix; 980 $_[0] =~ m%^(https?):// ([^:/]+) (?: : (\d*) )?%ix
981 or Carp::croak "$_[0]: invalid proxy URL";
982 $PROXY = [$2, $3 || 3128, $1]
983 } else {
984 undef $PROXY;
985 }
348} 986}
349 987
350# initialise proxy from environment 988# initialise proxy from environment
989eval {
351set_proxy $ENV{http_proxy}; 990 set_proxy $ENV{http_proxy};
991};
992
993=head2 SOCKS PROXIES
994
995Socks proxies are not directly supported by AnyEvent::HTTP. You can
996compile your perl to support socks, or use an external program such as
997F<socksify> (dante) or F<tsocks> to make your program use a socks proxy
998transparently.
999
1000Alternatively, for AnyEvent::HTTP only, you can use your own
1001C<tcp_connect> function that does the proxy handshake - here is an example
1002that works with socks4a proxies:
1003
1004 use Errno;
1005 use AnyEvent::Util;
1006 use AnyEvent::Socket;
1007 use AnyEvent::Handle;
1008
1009 # host, port and username of/for your socks4a proxy
1010 my $socks_host = "10.0.0.23";
1011 my $socks_port = 9050;
1012 my $socks_user = "";
1013
1014 sub socks4a_connect {
1015 my ($host, $port, $connect_cb, $prepare_cb) = @_;
1016
1017 my $hdl = new AnyEvent::Handle
1018 connect => [$socks_host, $socks_port],
1019 on_prepare => sub { $prepare_cb->($_[0]{fh}) },
1020 on_error => sub { $connect_cb->() },
1021 ;
1022
1023 $hdl->push_write (pack "CCnNZ*Z*", 4, 1, $port, 1, $socks_user, $host);
1024
1025 $hdl->push_read (chunk => 8, sub {
1026 my ($hdl, $chunk) = @_;
1027 my ($status, $port, $ipn) = unpack "xCna4", $chunk;
1028
1029 if ($status == 0x5a) {
1030 $connect_cb->($hdl->{fh}, (format_address $ipn) . ":$port");
1031 } else {
1032 $! = Errno::ENXIO; $connect_cb->();
1033 }
1034 });
1035
1036 $hdl
1037 }
1038
1039Use C<socks4a_connect> instead of C<tcp_connect> when doing C<http_request>s,
1040possibly after switching off other proxy types:
1041
1042 AnyEvent::HTTP::set_proxy undef; # usually you do not want other proxies
1043
1044 http_get 'http://www.google.com', tcp_connect => \&socks4a_connect, sub {
1045 my ($data, $headers) = @_;
1046 ...
1047 };
352 1048
353=head1 SEE ALSO 1049=head1 SEE ALSO
354 1050
355L<AnyEvent>. 1051L<AnyEvent>.
356 1052
357=head1 AUTHOR 1053=head1 AUTHOR
358 1054
359 Marc Lehmann <schmorp@schmorp.de> 1055 Marc Lehmann <schmorp@schmorp.de>
360 http://home.schmorp.de/ 1056 http://home.schmorp.de/
1057
1058With many thanks to Дмитрий Шалашов, who provided countless
1059testcases and bugreports.
361 1060
362=cut 1061=cut
363 1062
3641 10631
365 1064

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines