ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/AnyEvent-HTTP/HTTP.pm
(Generate patch)

Comparing AnyEvent-HTTP/HTTP.pm (file contents):
Revision 1.16 by root, Fri Jun 6 12:57:48 2008 UTC vs.
Revision 1.125 by root, Thu May 14 02:04:35 2015 UTC

3AnyEvent::HTTP - simple but non-blocking HTTP/HTTPS client 3AnyEvent::HTTP - simple but non-blocking HTTP/HTTPS client
4 4
5=head1 SYNOPSIS 5=head1 SYNOPSIS
6 6
7 use AnyEvent::HTTP; 7 use AnyEvent::HTTP;
8
9 http_get "http://www.nethype.de/", sub { print $_[1] };
10
11 # ... do something else here
8 12
9=head1 DESCRIPTION 13=head1 DESCRIPTION
10 14
11This module is an L<AnyEvent> user, you need to make sure that you use and 15This module is an L<AnyEvent> user, you need to make sure that you use and
12run a supported event loop. 16run a supported event loop.
13 17
14This module implements a simple, stateless and non-blocking HTTP 18This module implements a simple, stateless and non-blocking HTTP
15client. It supports GET, POST and other request methods, cookies and more, 19client. It supports GET, POST and other request methods, cookies and more,
16all on a very low level. It can follow redirects supports proxies and 20all on a very low level. It can follow redirects, supports proxies, and
17automatically limits the number of connections to the values specified in 21automatically limits the number of connections to the values specified in
18the RFC. 22the RFC.
19 23
20It should generally be a "good client" that is enough for most HTTP 24It should generally be a "good client" that is enough for most HTTP
21tasks. Simple tasks should be simple, but complex tasks should still be 25tasks. Simple tasks should be simple, but complex tasks should still be
32 36
33=cut 37=cut
34 38
35package AnyEvent::HTTP; 39package AnyEvent::HTTP;
36 40
37use strict; 41use common::sense;
38no warnings;
39 42
40use Carp; 43use Errno ();
41 44
42use AnyEvent (); 45use AnyEvent 5.0 ();
43use AnyEvent::Util (); 46use AnyEvent::Util ();
44use AnyEvent::Socket ();
45use AnyEvent::Handle (); 47use AnyEvent::Handle ();
46 48
47use base Exporter::; 49use base Exporter::;
48 50
49our $VERSION = '1.01'; 51our $VERSION = 2.22;
50 52
51our @EXPORT = qw(http_get http_request); 53our @EXPORT = qw(http_get http_post http_head http_request);
52 54
53our $USERAGENT = "Mozilla/5.0 (compatible; AnyEvent::HTTP/$VERSION; +http://software.schmorp.de/pkg/AnyEvent)"; 55our $USERAGENT = "Mozilla/5.0 (compatible; U; AnyEvent-HTTP/$VERSION; +http://software.schmorp.de/pkg/AnyEvent)";
54our $MAX_RECURSE = 10; 56our $MAX_RECURSE = 10;
55our $MAX_PERSISTENT = 8;
56our $PERSISTENT_TIMEOUT = 2; 57our $PERSISTENT_TIMEOUT = 3;
57our $TIMEOUT = 300; 58our $TIMEOUT = 300;
58 59our $MAX_PER_HOST = 4; # changing this is evil
59# changing these is evil
60our $MAX_PERSISTENT_PER_HOST = 2;
61our $MAX_PER_HOST = 4;
62 60
63our $PROXY; 61our $PROXY;
64our $ACTIVE = 0; 62our $ACTIVE = 0;
65 63
66my %KA_COUNT; # number of open keep-alive connections per host 64my %KA_CACHE; # indexed by uhost currently, points to [$handle...] array
67my %CO_SLOT; # number of open connections, and wait queue, per host 65my %CO_SLOT; # number of open connections, and wait queue, per host
68 66
69=item http_get $url, key => value..., $cb->($data, $headers) 67=item http_get $url, key => value..., $cb->($data, $headers)
70 68
71Executes an HTTP-GET request. See the http_request function for details on 69Executes an HTTP-GET request. See the http_request function for details on
72additional parameters. 70additional parameters and the return value.
73 71
74=item http_head $url, key => value..., $cb->($data, $headers) 72=item http_head $url, key => value..., $cb->($data, $headers)
75 73
76Executes an HTTP-HEAD request. See the http_request function for details on 74Executes an HTTP-HEAD request. See the http_request function for details
77additional parameters. 75on additional parameters and the return value.
78 76
79=item http_post $url, $body, key => value..., $cb->($data, $headers) 77=item http_post $url, $body, key => value..., $cb->($data, $headers)
80 78
81Executes an HTTP-POST request with a request body of C<$bod>. See the 79Executes an HTTP-POST request with a request body of C<$body>. See the
82http_request function for details on additional parameters. 80http_request function for details on additional parameters and the return
81value.
83 82
84=item http_request $method => $url, key => value..., $cb->($data, $headers) 83=item http_request $method => $url, key => value..., $cb->($data, $headers)
85 84
86Executes a HTTP request of type C<$method> (e.g. C<GET>, C<POST>). The URL 85Executes a HTTP request of type C<$method> (e.g. C<GET>, C<POST>). The URL
87must be an absolute http or https URL. 86must be an absolute http or https URL.
88 87
88When called in void context, nothing is returned. In other contexts,
89C<http_request> returns a "cancellation guard" - you have to keep the
90object at least alive until the callback get called. If the object gets
91destroyed before the callback is called, the request will be cancelled.
92
89The callback will be called with the response data as first argument 93The callback will be called with the response body data as first argument
90(or C<undef> if it wasn't available due to errors), and a hash-ref with 94(or C<undef> if an error occurred), and a hash-ref with response headers
91response headers as second argument. 95(and trailers) as second argument.
92 96
93All the headers in that hash are lowercased. In addition to the response 97All the headers in that hash are lowercased. In addition to the response
94headers, the three "pseudo-headers" C<HTTPVersion>, C<Status> and 98headers, the "pseudo-headers" (uppercase to avoid clashing with possible
95C<Reason> contain the three parts of the HTTP Status-Line of the same 99response headers) C<HTTPVersion>, C<Status> and C<Reason> contain the
100three parts of the HTTP Status-Line of the same name. If an error occurs
101during the body phase of a request, then the original C<Status> and
102C<Reason> values from the header are available as C<OrigStatus> and
103C<OrigReason>.
104
105The pseudo-header C<URL> contains the actual URL (which can differ from
106the requested URL when following redirects - for example, you might get
107an error that your URL scheme is not supported even though your URL is a
108valid http URL because it redirected to an ftp URL, in which case you can
109look at the URL pseudo header).
110
111The pseudo-header C<Redirect> only exists when the request was a result
112of an internal redirect. In that case it is an array reference with
113the C<($data, $headers)> from the redirect response. Note that this
114response could in turn be the result of a redirect itself, and C<<
115$headers->{Redirect}[1]{Redirect} >> will then contain the original
116response, and so on.
117
96name. If the server sends a header multiple lines, then their contents 118If the server sends a header multiple times, then their contents will be
97will be joined together with C<\x00>. 119joined together with a comma (C<,>), as per the HTTP spec.
98 120
99If an internal error occurs, such as not being able to resolve a hostname, 121If an internal error occurs, such as not being able to resolve a hostname,
100then C<$data> will be C<undef>, C<< $headers->{Status} >> will be C<599> 122then C<$data> will be C<undef>, C<< $headers->{Status} >> will be
101and the C<Reason> pseudo-header will contain an error message. 123C<590>-C<599> and the C<Reason> pseudo-header will contain an error
124message. Currently the following status codes are used:
125
126=over 4
127
128=item 595 - errors during connection establishment, proxy handshake.
129
130=item 596 - errors during TLS negotiation, request sending and header processing.
131
132=item 597 - errors during body receiving or processing.
133
134=item 598 - user aborted request via C<on_header> or C<on_body>.
135
136=item 599 - other, usually nonretryable, errors (garbled URL etc.).
137
138=back
102 139
103A typical callback might look like this: 140A typical callback might look like this:
104 141
105 sub { 142 sub {
106 my ($body, $hdr) = @_; 143 my ($body, $hdr) = @_;
117 154
118=over 4 155=over 4
119 156
120=item recurse => $count (default: $MAX_RECURSE) 157=item recurse => $count (default: $MAX_RECURSE)
121 158
122Whether to recurse requests or not, e.g. on redirects, authentication 159Whether to recurse requests or not, e.g. on redirects, authentication and
123retries and so on, and how often to do so. 160other retries and so on, and how often to do so.
161
162Only redirects to http and https URLs are supported. While most common
163redirection forms are handled entirely within this module, some require
164the use of the optional L<URI> module. If it is required but missing, then
165the request will fail with an error.
124 166
125=item headers => hashref 167=item headers => hashref
126 168
127The request headers to use. Currently, C<http_request> may provide its 169The request headers to use. Currently, C<http_request> may provide its own
128own C<Host:>, C<Content-Length:>, C<Connection:> and C<Cookie:> headers 170C<Host:>, C<Content-Length:>, C<Connection:> and C<Cookie:> headers and
129and will provide defaults for C<User-Agent:> and C<Referer:>. 171will provide defaults at least for C<TE:>, C<Referer:> and C<User-Agent:>
172(this can be suppressed by using C<undef> for these headers in which case
173they won't be sent at all).
174
175You really should provide your own C<User-Agent:> header value that is
176appropriate for your program - I wouldn't be surprised if the default
177AnyEvent string gets blocked by webservers sooner or later.
178
179Also, make sure that your headers names and values do not contain any
180embedded newlines.
130 181
131=item timeout => $seconds 182=item timeout => $seconds
132 183
133The time-out to use for various stages - each connect attempt will reset 184The time-out to use for various stages - each connect attempt will reset
134the timeout, as will read or write activity. Default timeout is 5 minutes. 185the timeout, as will read or write activity, i.e. this is not an overall
186timeout.
187
188Default timeout is 5 minutes.
135 189
136=item proxy => [$host, $port[, $scheme]] or undef 190=item proxy => [$host, $port[, $scheme]] or undef
137 191
138Use the given http proxy for all requests. If not specified, then the 192Use the given http proxy for all requests, or no proxy if C<undef> is
139default proxy (as specified by C<$ENV{http_proxy}>) is used. 193used.
140 194
141C<$scheme> must be either missing or C<http> for HTTP, or C<https> for 195C<$scheme> must be either missing or must be C<http> for HTTP.
142HTTPS. 196
197If not specified, then the default proxy is used (see
198C<AnyEvent::HTTP::set_proxy>).
199
200Currently, if your proxy requires authorization, you have to specify an
201appropriate "Proxy-Authorization" header in every request.
143 202
144=item body => $string 203=item body => $string
145 204
146The request body, usually empty. Will be-sent as-is (future versions of 205The request body, usually empty. Will be sent as-is (future versions of
147this module might offer more options). 206this module might offer more options).
148 207
149=item cookie_jar => $hash_ref 208=item cookie_jar => $hash_ref
150 209
151Passing this parameter enables (simplified) cookie-processing, loosely 210Passing this parameter enables (simplified) cookie-processing, loosely
152based on the original netscape specification. 211based on the original netscape specification.
153 212
154The C<$hash_ref> must be an (initially empty) hash reference which will 213The C<$hash_ref> must be an (initially empty) hash reference which
155get updated automatically. It is possible to save the cookie_jar to 214will get updated automatically. It is possible to save the cookie jar
156persistent storage with something like JSON or Storable, but this is not 215to persistent storage with something like JSON or Storable - see the
157recommended, as expire times are currently being ignored. 216C<AnyEvent::HTTP::cookie_jar_expire> function if you wish to remove
217expired or session-only cookies, and also for documentation on the format
218of the cookie jar.
158 219
159Note that this cookie implementation is not of very high quality, nor 220Note that this cookie implementation is not meant to be complete. If
160meant to be complete. If you want complete cookie management you have to 221you want complete cookie management you have to do that on your
161do that on your own. C<cookie_jar> is meant as a quick fix to get some 222own. C<cookie_jar> is meant as a quick fix to get most cookie-using sites
162cookie-using sites working. Cookies are a privacy disaster, do not use 223working. Cookies are a privacy disaster, do not use them unless required
163them unless required to. 224to.
225
226When cookie processing is enabled, the C<Cookie:> and C<Set-Cookie:>
227headers will be set and handled by this module, otherwise they will be
228left untouched.
229
230=item tls_ctx => $scheme | $tls_ctx
231
232Specifies the AnyEvent::TLS context to be used for https connections. This
233parameter follows the same rules as the C<tls_ctx> parameter to
234L<AnyEvent::Handle>, but additionally, the two strings C<low> or
235C<high> can be specified, which give you a predefined low-security (no
236verification, highest compatibility) and high-security (CA and common-name
237verification) TLS context.
238
239The default for this option is C<low>, which could be interpreted as "give
240me the page, no matter what".
241
242See also the C<sessionid> parameter.
243
244=item session => $string
245
246The module might reuse connections to the same host internally. Sometimes
247(e.g. when using TLS), you do not want to reuse connections from other
248sessions. This can be achieved by setting this parameter to some unique
249ID (such as the address of an object storing your state data, or the TLS
250context) - only connections using the same unique ID will be reused.
251
252=item on_prepare => $callback->($fh)
253
254In rare cases you need to "tune" the socket before it is used to
255connect (for example, to bind it on a given IP address). This parameter
256overrides the prepare callback passed to C<AnyEvent::Socket::tcp_connect>
257and behaves exactly the same way (e.g. it has to provide a
258timeout). See the description for the C<$prepare_cb> argument of
259C<AnyEvent::Socket::tcp_connect> for details.
260
261=item tcp_connect => $callback->($host, $service, $connect_cb, $prepare_cb)
262
263In even rarer cases you want total control over how AnyEvent::HTTP
264establishes connections. Normally it uses L<AnyEvent::Socket::tcp_connect>
265to do this, but you can provide your own C<tcp_connect> function -
266obviously, it has to follow the same calling conventions, except that it
267may always return a connection guard object.
268
269There are probably lots of weird uses for this function, starting from
270tracing the hosts C<http_request> actually tries to connect, to (inexact
271but fast) host => IP address caching or even socks protocol support.
272
273=item on_header => $callback->($headers)
274
275When specified, this callback will be called with the header hash as soon
276as headers have been successfully received from the remote server (not on
277locally-generated errors).
278
279It has to return either true (in which case AnyEvent::HTTP will continue),
280or false, in which case AnyEvent::HTTP will cancel the download (and call
281the finish callback with an error code of C<598>).
282
283This callback is useful, among other things, to quickly reject unwanted
284content, which, if it is supposed to be rare, can be faster than first
285doing a C<HEAD> request.
286
287The downside is that cancelling the request makes it impossible to re-use
288the connection. Also, the C<on_header> callback will not receive any
289trailer (headers sent after the response body).
290
291Example: cancel the request unless the content-type is "text/html".
292
293 on_header => sub {
294 $_[0]{"content-type"} =~ /^text\/html\s*(?:;|$)/
295 },
296
297=item on_body => $callback->($partial_body, $headers)
298
299When specified, all body data will be passed to this callback instead of
300to the completion callback. The completion callback will get the empty
301string instead of the body data.
302
303It has to return either true (in which case AnyEvent::HTTP will continue),
304or false, in which case AnyEvent::HTTP will cancel the download (and call
305the completion callback with an error code of C<598>).
306
307The downside to cancelling the request is that it makes it impossible to
308re-use the connection.
309
310This callback is useful when the data is too large to be held in memory
311(so the callback writes it to a file) or when only some information should
312be extracted, or when the body should be processed incrementally.
313
314It is usually preferred over doing your own body handling via
315C<want_body_handle>, but in case of streaming APIs, where HTTP is
316only used to create a connection, C<want_body_handle> is the better
317alternative, as it allows you to install your own event handler, reducing
318resource usage.
319
320=item want_body_handle => $enable
321
322When enabled (default is disabled), the behaviour of AnyEvent::HTTP
323changes considerably: after parsing the headers, and instead of
324downloading the body (if any), the completion callback will be
325called. Instead of the C<$body> argument containing the body data, the
326callback will receive the L<AnyEvent::Handle> object associated with the
327connection. In error cases, C<undef> will be passed. When there is no body
328(e.g. status C<304>), the empty string will be passed.
329
330The handle object might or might not be in TLS mode, might be connected
331to a proxy, be a persistent connection, use chunked transfer encoding
332etc., and configured in unspecified ways. The user is responsible for this
333handle (it will not be used by this module anymore).
334
335This is useful with some push-type services, where, after the initial
336headers, an interactive protocol is used (typical example would be the
337push-style twitter API which starts a JSON/XML stream).
338
339If you think you need this, first have a look at C<on_body>, to see if
340that doesn't solve your problem in a better way.
341
342=item persistent => $boolean
343
344Try to create/reuse a persistent connection. When this flag is set
345(default: true for idempotent requests, false for all others), then
346C<http_request> tries to re-use an existing (previously-created)
347persistent connection to the host and, failing that, tries to create a new
348one.
349
350Requests failing in certain ways will be automatically retried once, which
351is dangerous for non-idempotent requests, which is why it defaults to off
352for them. The reason for this is because the bozos who designed HTTP/1.1
353made it impossible to distinguish between a fatal error and a normal
354connection timeout, so you never know whether there was a problem with
355your request or not.
356
357When reusing an existent connection, many parameters (such as TLS context)
358will be ignored. See the C<session> parameter for a workaround.
359
360=item keepalive => $boolean
361
362Only used when C<persistent> is also true. This parameter decides whether
363C<http_request> tries to handshake a HTTP/1.0-style keep-alive connection
364(as opposed to only a HTTP/1.1 persistent connection).
365
366The default is true, except when using a proxy, in which case it defaults
367to false, as HTTP/1.0 proxies cannot support this in a meaningful way.
368
369=item handle_params => { key => value ... }
370
371The key-value pairs in this hash will be passed to any L<AnyEvent::Handle>
372constructor that is called - not all requests will create a handle, and
373sometimes more than one is created, so this parameter is only good for
374setting hints.
375
376Example: set the maximum read size to 4096, to potentially conserve memory
377at the cost of speed.
378
379 handle_params => {
380 max_read_size => 4096,
381 },
164 382
165=back 383=back
166 384
167Example: make a simple HTTP GET request for http://www.nethype.de/ 385Example: do a simple HTTP GET request for http://www.nethype.de/ and print
386the response body.
168 387
169 http_request GET => "http://www.nethype.de/", sub { 388 http_request GET => "http://www.nethype.de/", sub {
170 my ($body, $hdr) = @_; 389 my ($body, $hdr) = @_;
171 print "$body\n"; 390 print "$body\n";
172 }; 391 };
173 392
174Example: make a HTTP HEAD request on https://www.google.com/, use a 393Example: do a HTTP HEAD request on https://www.google.com/, use a
175timeout of 30 seconds. 394timeout of 30 seconds.
176 395
177 http_request 396 http_request
178 GET => "https://www.google.com", 397 HEAD => "https://www.google.com",
398 headers => { "user-agent" => "MySearchClient 1.0" },
179 timeout => 30, 399 timeout => 30,
180 sub { 400 sub {
181 my ($body, $hdr) = @_; 401 my ($body, $hdr) = @_;
182 use Data::Dumper; 402 use Data::Dumper;
183 print Dumper $hdr; 403 print Dumper $hdr;
184 } 404 }
185 ; 405 ;
186 406
407Example: do another simple HTTP GET request, but immediately try to
408cancel it.
409
410 my $request = http_request GET => "http://www.nethype.de/", sub {
411 my ($body, $hdr) = @_;
412 print "$body\n";
413 };
414
415 undef $request;
416
187=cut 417=cut
418
419#############################################################################
420# wait queue/slots
188 421
189sub _slot_schedule; 422sub _slot_schedule;
190sub _slot_schedule($) { 423sub _slot_schedule($) {
191 my $host = shift; 424 my $host = shift;
192 425
214 push @{ $CO_SLOT{$_[0]}[1] }, $_[1]; 447 push @{ $CO_SLOT{$_[0]}[1] }, $_[1];
215 448
216 _slot_schedule $_[0]; 449 _slot_schedule $_[0];
217} 450}
218 451
452#############################################################################
453# cookie handling
454
455# expire cookies
456sub cookie_jar_expire($;$) {
457 my ($jar, $session_end) = @_;
458
459 %$jar = () if $jar->{version} != 1;
460
461 my $anow = AE::now;
462
463 while (my ($chost, $paths) = each %$jar) {
464 next unless ref $paths;
465
466 while (my ($cpath, $cookies) = each %$paths) {
467 while (my ($cookie, $kv) = each %$cookies) {
468 if (exists $kv->{_expires}) {
469 delete $cookies->{$cookie}
470 if $anow > $kv->{_expires};
471 } elsif ($session_end) {
472 delete $cookies->{$cookie};
473 }
474 }
475
476 delete $paths->{$cpath}
477 unless %$cookies;
478 }
479
480 delete $jar->{$chost}
481 unless %$paths;
482 }
483}
484
485# extract cookies from jar
486sub cookie_jar_extract($$$$) {
487 my ($jar, $scheme, $host, $path) = @_;
488
489 %$jar = () if $jar->{version} != 1;
490
491 my @cookies;
492
493 while (my ($chost, $paths) = each %$jar) {
494 next unless ref $paths;
495
496 if ($chost =~ /^\./) {
497 next unless $chost eq substr $host, -length $chost;
498 } elsif ($chost =~ /\./) {
499 next unless $chost eq $host;
500 } else {
501 next;
502 }
503
504 while (my ($cpath, $cookies) = each %$paths) {
505 next unless $cpath eq substr $path, 0, length $cpath;
506
507 while (my ($cookie, $kv) = each %$cookies) {
508 next if $scheme ne "https" && exists $kv->{secure};
509
510 if (exists $kv->{_expires} and AE::now > $kv->{_expires}) {
511 delete $cookies->{$cookie};
512 next;
513 }
514
515 my $value = $kv->{value};
516
517 if ($value =~ /[=;,[:space:]]/) {
518 $value =~ s/([\\"])/\\$1/g;
519 $value = "\"$value\"";
520 }
521
522 push @cookies, "$cookie=$value";
523 }
524 }
525 }
526
527 \@cookies
528}
529
530# parse set_cookie header into jar
531sub cookie_jar_set_cookie($$$$) {
532 my ($jar, $set_cookie, $host, $date) = @_;
533
534 my $anow = int AE::now;
535 my $snow; # server-now
536
537 for ($set_cookie) {
538 # parse NAME=VALUE
539 my @kv;
540
541 # expires is not http-compliant in the original cookie-spec,
542 # we support the official date format and some extensions
543 while (
544 m{
545 \G\s*
546 (?:
547 expires \s*=\s* ([A-Z][a-z][a-z]+,\ [^,;]+)
548 | ([^=;,[:space:]]+) (?: \s*=\s* (?: "((?:[^\\"]+|\\.)*)" | ([^;,[:space:]]*) ) )?
549 )
550 }gcxsi
551 ) {
552 my $name = $2;
553 my $value = $4;
554
555 if (defined $1) {
556 # expires
557 $name = "expires";
558 $value = $1;
559 } elsif (defined $3) {
560 # quoted
561 $value = $3;
562 $value =~ s/\\(.)/$1/gs;
563 }
564
565 push @kv, @kv ? lc $name : $name, $value;
566
567 last unless /\G\s*;/gc;
568 }
569
570 last unless @kv;
571
572 my $name = shift @kv;
573 my %kv = (value => shift @kv, @kv);
574
575 if (exists $kv{"max-age"}) {
576 $kv{_expires} = $anow + delete $kv{"max-age"};
577 } elsif (exists $kv{expires}) {
578 $snow ||= parse_date ($date) || $anow;
579 $kv{_expires} = $anow + (parse_date (delete $kv{expires}) - $snow);
580 } else {
581 delete $kv{_expires};
582 }
583
584 my $cdom;
585 my $cpath = (delete $kv{path}) || "/";
586
587 if (exists $kv{domain}) {
588 $cdom = delete $kv{domain};
589
590 $cdom =~ s/^\.?/./; # make sure it starts with a "."
591
592 next if $cdom =~ /\.$/;
593
594 # this is not rfc-like and not netscape-like. go figure.
595 my $ndots = $cdom =~ y/.//;
596 next if $ndots < ($cdom =~ /\.[^.][^.]\.[^.][^.]$/ ? 3 : 2);
597 } else {
598 $cdom = $host;
599 }
600
601 # store it
602 $jar->{version} = 1;
603 $jar->{lc $cdom}{$cpath}{$name} = \%kv;
604
605 redo if /\G\s*,/gc;
606 }
607}
608
609#############################################################################
610# keepalive/persistent connection cache
611
612# fetch a connection from the keepalive cache
613sub ka_fetch($) {
614 my $ka_key = shift;
615
616 my $hdl = pop @{ $KA_CACHE{$ka_key} }; # currently we reuse the MOST RECENTLY USED connection
617 delete $KA_CACHE{$ka_key}
618 unless @{ $KA_CACHE{$ka_key} };
619
620 $hdl
621}
622
623sub ka_store($$) {
624 my ($ka_key, $hdl) = @_;
625
626 my $kaa = $KA_CACHE{$ka_key} ||= [];
627
628 my $destroy = sub {
629 my @ka = grep $_ != $hdl, @{ $KA_CACHE{$ka_key} };
630
631 $hdl->destroy;
632
633 @ka
634 ? $KA_CACHE{$ka_key} = \@ka
635 : delete $KA_CACHE{$ka_key};
636 };
637
638 # on error etc., destroy
639 $hdl->on_error ($destroy);
640 $hdl->on_eof ($destroy);
641 $hdl->on_read ($destroy);
642 $hdl->timeout ($PERSISTENT_TIMEOUT);
643
644 push @$kaa, $hdl;
645 shift @$kaa while @$kaa > $MAX_PER_HOST;
646}
647
648#############################################################################
649# utilities
650
651# continue to parse $_ for headers and place them into the arg
652sub _parse_hdr() {
653 my %hdr;
654
655 # things seen, not parsed:
656 # p3pP="NON CUR OTPi OUR NOR UNI"
657
658 $hdr{lc $1} .= ",$2"
659 while /\G
660 ([^:\000-\037]*):
661 [\011\040]*
662 ((?: [^\012]+ | \012[\011\040] )*)
663 \012
664 /gxc;
665
666 /\G$/
667 or return;
668
669 # remove the "," prefix we added to all headers above
670 substr $_, 0, 1, ""
671 for values %hdr;
672
673 \%hdr
674}
675
676#############################################################################
677# http_get
678
679our $qr_nlnl = qr{(?<![^\012])\015?\012};
680
681our $TLS_CTX_LOW = { cache => 1, sslv2 => 1 };
682our $TLS_CTX_HIGH = { cache => 1, verify => 1, verify_peername => "https" };
683
684# maybe it should just become a normal object :/
685
686sub _destroy_state(\%) {
687 my ($state) = @_;
688
689 $state->{handle}->destroy if $state->{handle};
690 %$state = ();
691}
692
693sub _error(\%$$) {
694 my ($state, $cb, $hdr) = @_;
695
696 &_destroy_state ($state);
697
698 $cb->(undef, $hdr);
699 ()
700}
701
702our %IDEMPOTENT = (
703 DELETE => 1,
704 GET => 1,
705 HEAD => 1,
706 OPTIONS => 1,
707 PUT => 1,
708 TRACE => 1,
709
710 ACL => 1,
711 "BASELINE-CONTROL" => 1,
712 BIND => 1,
713 CHECKIN => 1,
714 CHECKOUT => 1,
715 COPY => 1,
716 LABEL => 1,
717 LINK => 1,
718 MERGE => 1,
719 MKACTIVITY => 1,
720 MKCALENDAR => 1,
721 MKCOL => 1,
722 MKREDIRECTREF => 1,
723 MKWORKSPACE => 1,
724 MOVE => 1,
725 ORDERPATCH => 1,
726 PROPFIND => 1,
727 PROPPATCH => 1,
728 REBIND => 1,
729 REPORT => 1,
730 SEARCH => 1,
731 UNBIND => 1,
732 UNCHECKOUT => 1,
733 UNLINK => 1,
734 UNLOCK => 1,
735 UPDATE => 1,
736 UPDATEREDIRECTREF => 1,
737 "VERSION-CONTROL" => 1,
738);
739
219sub http_request($$@) { 740sub http_request($$@) {
220 my $cb = pop; 741 my $cb = pop;
221 my ($method, $url, %arg) = @_; 742 my ($method, $url, %arg) = @_;
222 743
223 my %hdr; 744 my %hdr;
745
746 $arg{tls_ctx} = $TLS_CTX_LOW if $arg{tls_ctx} eq "low" || !exists $arg{tls_ctx};
747 $arg{tls_ctx} = $TLS_CTX_HIGH if $arg{tls_ctx} eq "high";
224 748
225 $method = uc $method; 749 $method = uc $method;
226 750
227 if (my $hdr = $arg{headers}) { 751 if (my $hdr = $arg{headers}) {
228 while (my ($k, $v) = each %$hdr) { 752 while (my ($k, $v) = each %$hdr) {
229 $hdr{lc $k} = $v; 753 $hdr{lc $k} = $v;
230 } 754 }
231 } 755 }
232 756
757 # pseudo headers for all subsequent responses
758 my @pseudo = (URL => $url);
759 push @pseudo, Redirect => delete $arg{Redirect} if exists $arg{Redirect};
760
233 my $recurse = exists $arg{recurse} ? $arg{recurse} : $MAX_RECURSE; 761 my $recurse = exists $arg{recurse} ? delete $arg{recurse} : $MAX_RECURSE;
234 762
235 return $cb->(undef, { Status => 599, Reason => "recursion limit reached" }) 763 return $cb->(undef, { @pseudo, Status => 599, Reason => "Too many redirections" })
236 if $recurse < 0; 764 if $recurse < 0;
237 765
238 my $proxy = $arg{proxy} || $PROXY; 766 my $proxy = exists $arg{proxy} ? $arg{proxy} : $PROXY;
239 my $timeout = $arg{timeout} || $TIMEOUT; 767 my $timeout = $arg{timeout} || $TIMEOUT;
240 768
241 $hdr{"user-agent"} ||= $USERAGENT;
242
243 my ($scheme, $authority, $upath, $query, $fragment) = 769 my ($uscheme, $uauthority, $upath, $query, undef) = # ignore fragment
244 $url =~ m|(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*)(?:\?([^#]*))?(?:#(.*))?|; 770 $url =~ m|^([^:]+):(?://([^/?#]*))?([^?#]*)(?:(\?[^#]*))?(?:#(.*))?$|;
245 771
246 $scheme = lc $scheme; 772 $uscheme = lc $uscheme;
247 773
248 my $uport = $scheme eq "http" ? 80 774 my $uport = $uscheme eq "http" ? 80
249 : $scheme eq "https" ? 443 775 : $uscheme eq "https" ? 443
250 : return $cb->(undef, { Status => 599, Reason => "only http and https URL schemes supported" }); 776 : return $cb->(undef, { @pseudo, Status => 599, Reason => "Only http and https URL schemes supported" });
251 777
252 $hdr{referer} ||= "$scheme://$authority$upath"; # leave out fragment and query string, just a heuristic
253
254 $authority =~ /^(?: .*\@ )? ([^\@:]+) (?: : (\d+) )?$/x 778 $uauthority =~ /^(?: .*\@ )? ([^\@]+?) (?: : (\d+) )?$/x
255 or return $cb->(undef, { Status => 599, Reason => "unparsable URL" }); 779 or return $cb->(undef, { @pseudo, Status => 599, Reason => "Unparsable URL" });
256 780
257 my $uhost = $1; 781 my $uhost = lc $1;
258 $uport = $2 if defined $2; 782 $uport = $2 if defined $2;
259 783
784 $hdr{host} = defined $2 ? "$uhost:$2" : "$uhost"
785 unless exists $hdr{host};
786
260 $uhost =~ s/^\[(.*)\]$/$1/; 787 $uhost =~ s/^\[(.*)\]$/$1/;
261 $upath .= "?$query" if length $query; 788 $upath .= $query if length $query;
262 789
263 $upath =~ s%^/?%/%; 790 $upath =~ s%^/?%/%;
264 791
265 # cookie processing 792 # cookie processing
266 if (my $jar = $arg{cookie_jar}) { 793 if (my $jar = $arg{cookie_jar}) {
267 %$jar = () if $jar->{version} < 1; 794 my $cookies = cookie_jar_extract $jar, $uscheme, $uhost, $upath;
268 795
796 $hdr{cookie} = join "; ", @$cookies
269 my @cookie; 797 if @$cookies;
270 798 }
271 while (my ($chost, $v) = each %$jar) { 799
272 next unless $chost eq substr $uhost, -length $chost; 800 my ($rhost, $rport, $rscheme, $rpath); # request host, port, path
273 next unless $chost =~ /^\./; 801
274 802 if ($proxy) {
275 while (my ($cpath, $v) = each %$v) { 803 ($rpath, $rhost, $rport, $rscheme) = ($url, @$proxy);
276 next unless $cpath eq substr $upath, 0, length $cpath; 804
277 805 $rscheme = "http" unless defined $rscheme;
278 while (my ($k, $v) = each %$v) { 806
279 next if $scheme ne "https" && exists $v->{secure}; 807 # don't support https requests over https-proxy transport,
280 push @cookie, "$k=$v->{value}"; 808 # can't be done with tls as spec'ed, unless you double-encrypt.
809 $rscheme = "http" if $uscheme eq "https" && $rscheme eq "https";
810
811 $rhost = lc $rhost;
812 $rscheme = lc $rscheme;
813 } else {
814 ($rhost, $rport, $rscheme, $rpath) = ($uhost, $uport, $uscheme, $upath);
815 }
816
817 # leave out fragment and query string, just a heuristic
818 $hdr{referer} = "$uscheme://$uauthority$upath" unless exists $hdr{referer};
819 $hdr{"user-agent"} = $USERAGENT unless exists $hdr{"user-agent"};
820
821 $hdr{"content-length"} = length $arg{body}
822 if length $arg{body} || $method ne "GET";
823
824 my $idempotent = $IDEMPOTENT{$method};
825
826 # default value for keepalive is true iff the request is for an idempotent method
827 my $persistent = exists $arg{persistent} ? !!$arg{persistent} : $idempotent;
828 my $keepalive = exists $arg{keepalive} ? !!$arg{keepalive} : !$proxy;
829 my $was_persistent; # true if this is actually a recycled connection
830
831 # the key to use in the keepalive cache
832 my $ka_key = "$uscheme\x00$uhost\x00$uport\x00$arg{sessionid}";
833
834 $hdr{connection} = ($persistent ? $keepalive ? "keep-alive, " : "" : "close, ") . "Te"; #1.1
835 $hdr{te} = "trailers" unless exists $hdr{te}; #1.1
836
837 my %state = (connect_guard => 1);
838
839 my $ae_error = 595; # connecting
840
841 # handle actual, non-tunneled, request
842 my $handle_actual_request = sub {
843 $ae_error = 596; # request phase
844
845 my $hdl = $state{handle};
846
847 $hdl->starttls ("connect") if $uscheme eq "https" && !exists $hdl->{tls};
848
849 # send request
850 $hdl->push_write (
851 "$method $rpath HTTP/1.1\015\012"
852 . (join "", map "\u$_: $hdr{$_}\015\012", grep defined $hdr{$_}, keys %hdr)
853 . "\015\012"
854 . $arg{body}
855 );
856
857 # return if error occurred during push_write()
858 return unless %state;
859
860 # reduce memory usage, save a kitten, also re-use it for the response headers.
861 %hdr = ();
862
863 # status line and headers
864 $state{read_response} = sub {
865 return unless %state;
866
867 for ("$_[1]") {
868 y/\015//d; # weed out any \015, as they show up in the weirdest of places.
869
870 /^HTTP\/0*([0-9\.]+) \s+ ([0-9]{3}) (?: \s+ ([^\012]*) )? \012/gxci
871 or return _error %state, $cb, { @pseudo, Status => 599, Reason => "Invalid server response" };
872
873 # 100 Continue handling
874 # should not happen as we don't send expect: 100-continue,
875 # but we handle it just in case.
876 # since we send the request body regardless, if we get an error
877 # we are out of-sync, which we currently do NOT handle correctly.
878 return $state{handle}->push_read (line => $qr_nlnl, $state{read_response})
879 if $2 eq 100;
880
881 push @pseudo,
882 HTTPVersion => $1,
883 Status => $2,
884 Reason => $3,
885 ;
886
887 my $hdr = _parse_hdr
888 or return _error %state, $cb, { @pseudo, Status => 599, Reason => "Garbled response headers" };
889
890 %hdr = (%$hdr, @pseudo);
891 }
892
893 # redirect handling
894 # relative uri handling forced by microsoft and other shitheads.
895 # we give our best and fall back to URI if available.
896 if (exists $hdr{location}) {
897 my $loc = $hdr{location};
898
899 if ($loc =~ m%^//%) { # //
900 $loc = "$rscheme:$loc";
901
902 } elsif ($loc eq "") {
903 $loc = $url;
904
905 } elsif ($loc !~ /^(?: $ | [^:\/?\#]+ : )/x) { # anything "simple"
906 $loc =~ s/^\.\/+//;
907
908 if ($loc !~ m%^[.?#]%) {
909 my $prefix = "$rscheme://$uhost:$uport";
910
911 unless ($loc =~ s/^\///) {
912 $prefix .= $upath;
913 $prefix =~ s/\/[^\/]*$//;
914 }
915
916 $loc = "$prefix/$loc";
917
918 } elsif (eval { require URI }) { # uri
919 $loc = URI->new_abs ($loc, $url)->as_string;
920
921 } else {
922 return _error %state, $cb, { @pseudo, Status => 599, Reason => "Cannot parse Location (URI module missing)" };
923 #$hdr{Status} = 599;
924 #$hdr{Reason} = "Unparsable Redirect (URI module missing)";
925 #$recurse = 0;
926 }
927 }
928
929 $hdr{location} = $loc;
930 }
931
932 my $redirect;
933
934 if ($recurse) {
935 my $status = $hdr{Status};
936
937 # industry standard is to redirect POST as GET for
938 # 301, 302 and 303, in contrast to HTTP/1.0 and 1.1.
939 # also, the UA should ask the user for 301 and 307 and POST,
940 # industry standard seems to be to simply follow.
941 # we go with the industry standard. 308 is defined
942 # by rfc7538
943 if ($status == 301 or $status == 302 or $status == 303) {
944 $redirect = 1;
945 # HTTP/1.1 is unclear on how to mutate the method
946 unless ($method eq "HEAD") {
947 $method = "GET";
948 delete $arg{body};
949 }
950 } elsif ($status == 307 or $status == 308) {
951 $redirect = 1;
281 } 952 }
282 } 953 }
954
955 my $finish = sub { # ($data, $err_status, $err_reason[, $persistent])
956 if ($state{handle}) {
957 # handle keepalive
958 if (
959 $persistent
960 && $_[3]
961 && ($hdr{HTTPVersion} < 1.1
962 ? $hdr{connection} =~ /\bkeep-?alive\b/i
963 : $hdr{connection} !~ /\bclose\b/i)
964 ) {
965 ka_store $ka_key, delete $state{handle};
966 } else {
967 # no keepalive, destroy the handle
968 $state{handle}->destroy;
969 }
970 }
971
972 %state = ();
973
974 if (defined $_[1]) {
975 $hdr{OrigStatus} = $hdr{Status}; $hdr{Status} = $_[1];
976 $hdr{OrigReason} = $hdr{Reason}; $hdr{Reason} = $_[2];
977 }
978
979 # set-cookie processing
980 if ($arg{cookie_jar}) {
981 cookie_jar_set_cookie $arg{cookie_jar}, $hdr{"set-cookie"}, $uhost, $hdr{date};
982 }
983
984 if ($redirect && exists $hdr{location}) {
985 # we ignore any errors, as it is very common to receive
986 # Content-Length != 0 but no actual body
987 # we also access %hdr, as $_[1] might be an erro
988 $state{recurse} =
989 http_request (
990 $method => $hdr{location},
991 %arg,
992 recurse => $recurse - 1,
993 Redirect => [$_[0], \%hdr],
994 sub {
995 %state = ();
996 &$cb
997 },
998 );
999 } else {
1000 $cb->($_[0], \%hdr);
1001 }
1002 };
1003
1004 $ae_error = 597; # body phase
1005
1006 my $chunked = $hdr{"transfer-encoding"} =~ /\bchunked\b/i; # not quite correct...
1007
1008 my $len = $chunked ? undef : $hdr{"content-length"};
1009
1010 # body handling, many different code paths
1011 # - no body expected
1012 # - want_body_handle
1013 # - te chunked
1014 # - 2x length known (with or without on_body)
1015 # - 2x length not known (with or without on_body)
1016 if (!$redirect && $arg{on_header} && !$arg{on_header}(\%hdr)) {
1017 $finish->(undef, 598 => "Request cancelled by on_header");
1018 } elsif (
1019 $hdr{Status} =~ /^(?:1..|204|205|304)$/
1020 or $method eq "HEAD"
1021 or (defined $len && $len == 0) # == 0, not !, because "0 " is true
1022 ) {
1023 # no body
1024 $finish->("", undef, undef, 1);
1025
1026 } elsif (!$redirect && $arg{want_body_handle}) {
1027 $_[0]->on_eof (undef);
1028 $_[0]->on_error (undef);
1029 $_[0]->on_read (undef);
1030
1031 $finish->(delete $state{handle});
1032
1033 } elsif ($chunked) {
1034 my $cl = 0;
1035 my $body = "";
1036 my $on_body = $arg{on_body} || sub { $body .= shift; 1 };
1037
1038 $state{read_chunk} = sub {
1039 $_[1] =~ /^([0-9a-fA-F]+)/
1040 or return $finish->(undef, $ae_error => "Garbled chunked transfer encoding");
1041
1042 my $len = hex $1;
1043
1044 if ($len) {
1045 $cl += $len;
1046
1047 $_[0]->push_read (chunk => $len, sub {
1048 $on_body->($_[1], \%hdr)
1049 or return $finish->(undef, 598 => "Request cancelled by on_body");
1050
1051 $_[0]->push_read (line => sub {
1052 length $_[1]
1053 and return $finish->(undef, $ae_error => "Garbled chunked transfer encoding");
1054 $_[0]->push_read (line => $state{read_chunk});
1055 });
1056 });
1057 } else {
1058 $hdr{"content-length"} ||= $cl;
1059
1060 $_[0]->push_read (line => $qr_nlnl, sub {
1061 if (length $_[1]) {
1062 for ("$_[1]") {
1063 y/\015//d; # weed out any \015, as they show up in the weirdest of places.
1064
1065 my $hdr = _parse_hdr
1066 or return $finish->(undef, $ae_error => "Garbled response trailers");
1067
1068 %hdr = (%hdr, %$hdr);
1069 }
1070 }
1071
1072 $finish->($body, undef, undef, 1);
1073 });
1074 }
1075 };
1076
1077 $_[0]->push_read (line => $state{read_chunk});
1078
1079 } elsif ($arg{on_body}) {
1080 if (defined $len) {
1081 $_[0]->on_read (sub {
1082 $len -= length $_[0]{rbuf};
1083
1084 $arg{on_body}(delete $_[0]{rbuf}, \%hdr)
1085 or return $finish->(undef, 598 => "Request cancelled by on_body");
1086
1087 $len > 0
1088 or $finish->("", undef, undef, 1);
1089 });
1090 } else {
1091 $_[0]->on_eof (sub {
1092 $finish->("");
1093 });
1094 $_[0]->on_read (sub {
1095 $arg{on_body}(delete $_[0]{rbuf}, \%hdr)
1096 or $finish->(undef, 598 => "Request cancelled by on_body");
1097 });
1098 }
1099 } else {
1100 $_[0]->on_eof (undef);
1101
1102 if (defined $len) {
1103 $_[0]->on_read (sub {
1104 $finish->((substr delete $_[0]{rbuf}, 0, $len, ""), undef, undef, 1)
1105 if $len <= length $_[0]{rbuf};
1106 });
1107 } else {
1108 $_[0]->on_error (sub {
1109 ($! == Errno::EPIPE || !$!)
1110 ? $finish->(delete $_[0]{rbuf})
1111 : $finish->(undef, $ae_error => $_[2]);
1112 });
1113 $_[0]->on_read (sub { });
1114 }
1115 }
1116 };
1117
1118 # if keepalive is enabled, then the server closing the connection
1119 # before a response can happen legally - we retry on idempotent methods.
1120 if ($was_persistent && $idempotent) {
1121 my $old_eof = $hdl->{on_eof};
1122 $hdl->{on_eof} = sub {
1123 _destroy_state %state;
1124
1125 %state = ();
1126 $state{recurse} =
1127 http_request (
1128 $method => $url,
1129 %arg,
1130 recurse => $recurse - 1,
1131 persistent => 0,
1132 sub {
1133 %state = ();
1134 &$cb
1135 }
1136 );
1137 };
1138 $hdl->on_read (sub {
1139 return unless %state;
1140
1141 # as soon as we receive something, a connection close
1142 # once more becomes a hard error
1143 $hdl->{on_eof} = $old_eof;
1144 $hdl->push_read (line => $qr_nlnl, $state{read_response});
1145 });
1146 } else {
1147 $hdl->push_read (line => $qr_nlnl, $state{read_response});
283 } 1148 }
284
285 $hdr{cookie} = join "; ", @cookie
286 if @cookie;
287 } 1149 };
288 1150
289 my ($rhost, $rport, $rpath); # request host, port, path 1151 my $prepare_handle = sub {
1152 my ($hdl) = $state{handle};
290 1153
291 if ($proxy) { 1154 $hdl->on_error (sub {
292 ($rhost, $rport, $scheme) = @$proxy; 1155 _error %state, $cb, { @pseudo, Status => $ae_error, Reason => $_[2] };
293 $rpath = $url; 1156 });
294 } else { 1157 $hdl->on_eof (sub {
295 ($rhost, $rport, $rpath) = ($uhost, $uport, $upath); 1158 _error %state, $cb, { @pseudo, Status => $ae_error, Reason => "Unexpected end-of-file" };
296 $hdr{host} = $uhost; 1159 });
1160 $hdl->timeout_reset;
1161 $hdl->timeout ($timeout);
297 } 1162 };
298 1163
299 $hdr{"content-length"} = length $arg{body}; 1164 # connected to proxy (or origin server)
1165 my $connect_cb = sub {
1166 my $fh = shift
1167 or return _error %state, $cb, { @pseudo, Status => $ae_error, Reason => "$!" };
300 1168
301 my %state = (connect_guard => 1); 1169 return unless delete $state{connect_guard};
1170
1171 # get handle
1172 $state{handle} = new AnyEvent::Handle
1173 %{ $arg{handle_params} },
1174 fh => $fh,
1175 peername => $uhost,
1176 tls_ctx => $arg{tls_ctx},
1177 ;
1178
1179 $prepare_handle->();
1180
1181 #$state{handle}->starttls ("connect") if $rscheme eq "https";
1182
1183 # now handle proxy-CONNECT method
1184 if ($proxy && $uscheme eq "https") {
1185 # oh dear, we have to wrap it into a connect request
1186
1187 my $auth = exists $hdr{"proxy-authorization"}
1188 ? "proxy-authorization: " . (delete $hdr{"proxy-authorization"}) . "\015\012"
1189 : "";
1190
1191 # maybe re-use $uauthority with patched port?
1192 $state{handle}->push_write ("CONNECT $uhost:$uport HTTP/1.0\015\012$auth\015\012");
1193 $state{handle}->push_read (line => $qr_nlnl, sub {
1194 $_[1] =~ /^HTTP\/([0-9\.]+) \s+ ([0-9]{3}) (?: \s+ ([^\015\012]*) )?/ix
1195 or return _error %state, $cb, { @pseudo, Status => 599, Reason => "Invalid proxy connect response ($_[1])" };
1196
1197 if ($2 == 200) {
1198 $rpath = $upath;
1199 $handle_actual_request->();
1200 } else {
1201 _error %state, $cb, { @pseudo, Status => $2, Reason => $3 };
1202 }
1203 });
1204 } else {
1205 delete $hdr{"proxy-authorization"} unless $proxy;
1206
1207 $handle_actual_request->();
1208 }
1209 };
302 1210
303 _get_slot $uhost, sub { 1211 _get_slot $uhost, sub {
304 $state{slot_guard} = shift; 1212 $state{slot_guard} = shift;
305 1213
306 return unless $state{connect_guard}; 1214 return unless $state{connect_guard};
307 1215
308 $state{connect_guard} = AnyEvent::Socket::tcp_connect $rhost, $rport, sub { 1216 # try to use an existing keepalive connection, but only if we, ourselves, plan
309 $state{fh} = shift 1217 # on a keepalive request (in theory, this should be a separate config option).
310 or return $cb->(undef, { Status => 599, Reason => "$!" }); 1218 if ($persistent && $KA_CACHE{$ka_key}) {
1219 $was_persistent = 1;
311 1220
312 delete $state{connect_guard}; # reduce memory usage, save a tree 1221 $state{handle} = ka_fetch $ka_key;
1222 $state{handle}->destroyed
1223 and die "AnyEvent::HTTP: unexpectedly got a destructed handle (1), please report.";#d#
1224 $prepare_handle->();
1225 $state{handle}->destroyed
1226 and die "AnyEvent::HTTP: unexpectedly got a destructed handle (2), please report.";#d#
1227 $handle_actual_request->();
313 1228
314 # get handle
315 $state{handle} = new AnyEvent::Handle
316 fh => $state{fh},
317 ($scheme eq "https" ? (tls => "connect") : ());
318
319 # limit the number of persistent connections
320 if ($KA_COUNT{$_[1]} < $MAX_PERSISTENT_PER_HOST) {
321 ++$KA_COUNT{$_[1]};
322 $state{handle}{ka_count_guard} = AnyEvent::Util::guard { --$KA_COUNT{$_[1]} };
323 $hdr{connection} = "keep-alive";
324 delete $hdr{connection}; # keep-alive not yet supported
325 } else { 1229 } else {
326 delete $hdr{connection}; 1230 my $tcp_connect = $arg{tcp_connect}
327 } 1231 || do { require AnyEvent::Socket; \&AnyEvent::Socket::tcp_connect };
328 1232
329 # (re-)configure handle 1233 $state{connect_guard} = $tcp_connect->($rhost, $rport, $connect_cb, $arg{on_prepare} || sub { $timeout });
330 $state{handle}->timeout ($timeout);
331 $state{handle}->on_error (sub {
332 my $errno = "$!";
333 %state = ();
334 $cb->(undef, { Status => 599, Reason => $errno });
335 });
336 $state{handle}->on_eof (sub {
337 %state = ();
338 $cb->(undef, { Status => 599, Reason => "unexpected end-of-file" });
339 });
340
341 # send request
342 $state{handle}->push_write (
343 "$method $rpath HTTP/1.0\015\012"
344 . (join "", map "$_: $hdr{$_}\015\012", keys %hdr)
345 . "\015\012"
346 . (delete $arg{body})
347 );
348
349 %hdr = (); # reduce memory usage, save a kitten
350
351 # status line
352 $state{handle}->push_read (line => qr/\015?\012/, sub {
353 $_[1] =~ /^HTTP\/([0-9\.]+) \s+ ([0-9]{3}) \s+ ([^\015\012]+)/ix
354 or return (%state = (), $cb->(undef, { Status => 599, Reason => "invalid server response ($_[1])" }));
355
356 my %hdr = ( # response headers
357 HTTPVersion => "\x00$1",
358 Status => "\x00$2",
359 Reason => "\x00$3",
360 );
361
362 # headers, could be optimized a bit
363 $state{handle}->unshift_read (line => qr/\015?\012\015?\012/, sub {
364 for ("$_[1]\012") {
365 # we support spaces in field names, as lotus domino
366 # creates them.
367 $hdr{lc $1} .= "\x00$2"
368 while /\G
369 ([^:\000-\037]+):
370 [\011\040]*
371 ((?: [^\015\012]+ | \015?\012[\011\040] )*)
372 \015?\012
373 /gxc;
374
375 /\G$/
376 or return (%state = (), $cb->(undef, { Status => 599, Reason => "garbled response headers" }));
377 }
378
379 substr $_, 0, 1, ""
380 for values %hdr;
381
382 my $finish = sub {
383 %state = ();
384
385 # set-cookie processing
386 if ($arg{cookie_jar} && exists $hdr{"set-cookie"}) {
387 for (split /\x00/, $hdr{"set-cookie"}) {
388 my ($cookie, @arg) = split /;\s*/;
389 my ($name, $value) = split /=/, $cookie, 2;
390 my %kv = (value => $value, map { split /=/, $_, 2 } @arg);
391
392 my $cdom = (delete $kv{domain}) || $uhost;
393 my $cpath = (delete $kv{path}) || "/";
394
395 $cdom =~ s/^.?/./; # make sure it starts with a "."
396
397 next if $cdom =~ /\.$/;
398
399 # this is not rfc-like and not netscape-like. go figure.
400 my $ndots = $cdom =~ y/.//;
401 next if $ndots < ($cdom =~ /\.[^.][^.]\.[^.][^.]$/ ? 3 : 2);
402
403 # store it
404 $arg{cookie_jar}{version} = 1;
405 $arg{cookie_jar}{$cdom}{$cpath}{$name} = \%kv;
406 }
407 }
408
409 if ($_[1]{Status} =~ /^30[12]$/ && $recurse) {
410 # microsoft and other assholes don't give a shit for following standards,
411 # try to support a common form of broken Location header.
412 $_[1]{location} =~ s%^/%$scheme://$uhost:$uport/%;
413
414 http_request ($method, $_[1]{location}, %arg, recurse => $recurse - 1, $cb);
415 } else {
416 $cb->($_[0], $_[1]);
417 }
418 };
419
420 if ($hdr{Status} =~ /^(?:1..|204|304)$/ or $method eq "HEAD") {
421 $finish->(undef, \%hdr);
422 } else {
423 if (exists $hdr{"content-length"}) {
424 $_[0]->unshift_read (chunk => $hdr{"content-length"}, sub {
425 # could cache persistent connection now
426 if ($hdr{connection} =~ /\bkeep-alive\b/i) {
427 # but we don't, due to misdesigns, this is annoyingly complex
428 };
429
430 $finish->($_[1], \%hdr);
431 });
432 } else {
433 # too bad, need to read until we get an error or EOF,
434 # no way to detect winged data.
435 $_[0]->on_error (sub {
436 $finish->($_[0]{rbuf}, \%hdr);
437 });
438 $_[0]->on_eof (undef);
439 $_[0]->on_read (sub { });
440 }
441 }
442 });
443 });
444 }, sub {
445 $timeout
446 }; 1234 }
447 }; 1235 };
448 1236
449 defined wantarray && AnyEvent::Util::guard { %state = () } 1237 defined wantarray && AnyEvent::Util::guard { _destroy_state %state }
450} 1238}
451 1239
452sub http_get($@) { 1240sub http_get($@) {
453 unshift @_, "GET"; 1241 unshift @_, "GET";
454 &http_request 1242 &http_request
458 unshift @_, "HEAD"; 1246 unshift @_, "HEAD";
459 &http_request 1247 &http_request
460} 1248}
461 1249
462sub http_post($$@) { 1250sub http_post($$@) {
1251 my $url = shift;
463 unshift @_, "POST", "body"; 1252 unshift @_, "POST", $url, "body";
464 &http_request 1253 &http_request
465} 1254}
466 1255
467=back 1256=back
468 1257
1258=head2 DNS CACHING
1259
1260AnyEvent::HTTP uses the AnyEvent::Socket::tcp_connect function for
1261the actual connection, which in turn uses AnyEvent::DNS to resolve
1262hostnames. The latter is a simple stub resolver and does no caching
1263on its own. If you want DNS caching, you currently have to provide
1264your own default resolver (by storing a suitable resolver object in
1265C<$AnyEvent::DNS::RESOLVER>) or your own C<tcp_connect> callback.
1266
469=head2 GLOBAL FUNCTIONS AND VARIABLES 1267=head2 GLOBAL FUNCTIONS AND VARIABLES
470 1268
471=over 4 1269=over 4
472 1270
473=item AnyEvent::HTTP::set_proxy "proxy-url" 1271=item AnyEvent::HTTP::set_proxy "proxy-url"
474 1272
475Sets the default proxy server to use. The proxy-url must begin with a 1273Sets the default proxy server to use. The proxy-url must begin with a
476string of the form C<http://host:port> (optionally C<https:...>). 1274string of the form C<http://host:port>, croaks otherwise.
1275
1276To clear an already-set proxy, use C<undef>.
1277
1278When AnyEvent::HTTP is loaded for the first time it will query the
1279default proxy from the operating system, currently by looking at
1280C<$ENV{http_proxy>}.
1281
1282=item AnyEvent::HTTP::cookie_jar_expire $jar[, $session_end]
1283
1284Remove all cookies from the cookie jar that have been expired. If
1285C<$session_end> is given and true, then additionally remove all session
1286cookies.
1287
1288You should call this function (with a true C<$session_end>) before you
1289save cookies to disk, and you should call this function after loading them
1290again. If you have a long-running program you can additionally call this
1291function from time to time.
1292
1293A cookie jar is initially an empty hash-reference that is managed by this
1294module. Its format is subject to change, but currently it is as follows:
1295
1296The key C<version> has to contain C<1>, otherwise the hash gets
1297emptied. All other keys are hostnames or IP addresses pointing to
1298hash-references. The key for these inner hash references is the
1299server path for which this cookie is meant, and the values are again
1300hash-references. Each key of those hash-references is a cookie name, and
1301the value, you guessed it, is another hash-reference, this time with the
1302key-value pairs from the cookie, except for C<expires> and C<max-age>,
1303which have been replaced by a C<_expires> key that contains the cookie
1304expiry timestamp. Session cookies are indicated by not having an
1305C<_expires> key.
1306
1307Here is an example of a cookie jar with a single cookie, so you have a
1308chance of understanding the above paragraph:
1309
1310 {
1311 version => 1,
1312 "10.0.0.1" => {
1313 "/" => {
1314 "mythweb_id" => {
1315 _expires => 1293917923,
1316 value => "ooRung9dThee3ooyXooM1Ohm",
1317 },
1318 },
1319 },
1320 }
1321
1322=item $date = AnyEvent::HTTP::format_date $timestamp
1323
1324Takes a POSIX timestamp (seconds since the epoch) and formats it as a HTTP
1325Date (RFC 2616).
1326
1327=item $timestamp = AnyEvent::HTTP::parse_date $date
1328
1329Takes a HTTP Date (RFC 2616) or a Cookie date (netscape cookie spec) or a
1330bunch of minor variations of those, and returns the corresponding POSIX
1331timestamp, or C<undef> if the date cannot be parsed.
477 1332
478=item $AnyEvent::HTTP::MAX_RECURSE 1333=item $AnyEvent::HTTP::MAX_RECURSE
479 1334
480The default value for the C<recurse> request parameter (default: C<10>). 1335The default value for the C<recurse> request parameter (default: C<10>).
481 1336
1337=item $AnyEvent::HTTP::TIMEOUT
1338
1339The default timeout for connection operations (default: C<300>).
1340
482=item $AnyEvent::HTTP::USERAGENT 1341=item $AnyEvent::HTTP::USERAGENT
483 1342
484The default value for the C<User-Agent> header (the default is 1343The default value for the C<User-Agent> header (the default is
485C<Mozilla/5.0 (compatible; AnyEvent::HTTP/$VERSION; +http://software.schmorp.de/pkg/AnyEvent)>). 1344C<Mozilla/5.0 (compatible; U; AnyEvent-HTTP/$VERSION; +http://software.schmorp.de/pkg/AnyEvent)>).
486 1345
487=item $AnyEvent::HTTP::MAX_PERSISTENT 1346=item $AnyEvent::HTTP::MAX_PER_HOST
488 1347
489The maximum number of persistent connections to keep open (default: 8). 1348The maximum number of concurrent connections to the same host (identified
1349by the hostname). If the limit is exceeded, then additional requests
1350are queued until previous connections are closed. Both persistent and
1351non-persistent connections are counted in this limit.
490 1352
491Not implemented currently. 1353The default value for this is C<4>, and it is highly advisable to not
1354increase it much.
1355
1356For comparison: the RFC's recommend 4 non-persistent or 2 persistent
1357connections, older browsers used 2, newer ones (such as firefox 3)
1358typically use 6, and Opera uses 8 because like, they have the fastest
1359browser and give a shit for everybody else on the planet.
492 1360
493=item $AnyEvent::HTTP::PERSISTENT_TIMEOUT 1361=item $AnyEvent::HTTP::PERSISTENT_TIMEOUT
494 1362
495The maximum time to cache a persistent connection, in seconds (default: 2). 1363The time after which idle persistent connections get closed by
496 1364AnyEvent::HTTP (default: C<3>).
497Not implemented currently.
498 1365
499=item $AnyEvent::HTTP::ACTIVE 1366=item $AnyEvent::HTTP::ACTIVE
500 1367
501The number of active connections. This is not the number of currently 1368The number of active connections. This is not the number of currently
502running requests, but the number of currently open and non-idle TCP 1369running requests, but the number of currently open and non-idle TCP
503connections. This number of can be useful for load-leveling. 1370connections. This number can be useful for load-leveling.
504 1371
505=back 1372=back
506 1373
507=cut 1374=cut
508 1375
1376our @month = qw(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec);
1377our @weekday = qw(Sun Mon Tue Wed Thu Fri Sat);
1378
1379sub format_date($) {
1380 my ($time) = @_;
1381
1382 # RFC 822/1123 format
1383 my ($S, $M, $H, $mday, $mon, $year, $wday, $yday, undef) = gmtime $time;
1384
1385 sprintf "%s, %02d %s %04d %02d:%02d:%02d GMT",
1386 $weekday[$wday], $mday, $month[$mon], $year + 1900,
1387 $H, $M, $S;
1388}
1389
1390sub parse_date($) {
1391 my ($date) = @_;
1392
1393 my ($d, $m, $y, $H, $M, $S);
1394
1395 if ($date =~ /^[A-Z][a-z][a-z]+, ([0-9][0-9]?)[\- ]([A-Z][a-z][a-z])[\- ]([0-9][0-9][0-9][0-9]) ([0-9][0-9]?):([0-9][0-9]?):([0-9][0-9]?) GMT$/) {
1396 # RFC 822/1123, required by RFC 2616 (with " ")
1397 # cookie dates (with "-")
1398
1399 ($d, $m, $y, $H, $M, $S) = ($1, $2, $3, $4, $5, $6);
1400
1401 } elsif ($date =~ /^[A-Z][a-z][a-z]+, ([0-9][0-9]?)-([A-Z][a-z][a-z])-([0-9][0-9]) ([0-9][0-9]?):([0-9][0-9]?):([0-9][0-9]?) GMT$/) {
1402 # RFC 850
1403 ($d, $m, $y, $H, $M, $S) = ($1, $2, $3 < 69 ? $3 + 2000 : $3 + 1900, $4, $5, $6);
1404
1405 } elsif ($date =~ /^[A-Z][a-z][a-z]+ ([A-Z][a-z][a-z]) ([0-9 ]?[0-9]) ([0-9][0-9]?):([0-9][0-9]?):([0-9][0-9]?) ([0-9][0-9][0-9][0-9])$/) {
1406 # ISO C's asctime
1407 ($d, $m, $y, $H, $M, $S) = ($2, $1, $6, $3, $4, $5);
1408 }
1409 # other formats fail in the loop below
1410
1411 for (0..11) {
1412 if ($m eq $month[$_]) {
1413 require Time::Local;
1414 return eval { Time::Local::timegm ($S, $M, $H, $d, $_, $y) };
1415 }
1416 }
1417
1418 undef
1419}
1420
509sub set_proxy($) { 1421sub set_proxy($) {
510 $PROXY = [$2, $3 || 3128, $1] if $_[0] =~ m%^(https?):// ([^:/]+) (?: : (\d*) )?%ix; 1422 if (length $_[0]) {
1423 $_[0] =~ m%^(http):// ([^:/]+) (?: : (\d*) )?%ix
1424 or Carp::croak "$_[0]: invalid proxy URL";
1425 $PROXY = [$2, $3 || 3128, $1]
1426 } else {
1427 undef $PROXY;
1428 }
511} 1429}
512 1430
513# initialise proxy from environment 1431# initialise proxy from environment
1432eval {
514set_proxy $ENV{http_proxy}; 1433 set_proxy $ENV{http_proxy};
1434};
1435
1436=head2 SHOWCASE
1437
1438This section contains some more elaborate "real-world" examples or code
1439snippets.
1440
1441=head2 HTTP/1.1 FILE DOWNLOAD
1442
1443Downloading files with HTTP can be quite tricky, especially when something
1444goes wrong and you want to resume.
1445
1446Here is a function that initiates and resumes a download. It uses the
1447last modified time to check for file content changes, and works with many
1448HTTP/1.0 servers as well, and usually falls back to a complete re-download
1449on older servers.
1450
1451It calls the completion callback with either C<undef>, which means a
1452nonretryable error occurred, C<0> when the download was partial and should
1453be retried, and C<1> if it was successful.
1454
1455 use AnyEvent::HTTP;
1456
1457 sub download($$$) {
1458 my ($url, $file, $cb) = @_;
1459
1460 open my $fh, "+<", $file
1461 or die "$file: $!";
1462
1463 my %hdr;
1464 my $ofs = 0;
1465
1466 warn stat $fh;
1467 warn -s _;
1468 if (stat $fh and -s _) {
1469 $ofs = -s _;
1470 warn "-s is ", $ofs;
1471 $hdr{"if-unmodified-since"} = AnyEvent::HTTP::format_date +(stat _)[9];
1472 $hdr{"range"} = "bytes=$ofs-";
1473 }
1474
1475 http_get $url,
1476 headers => \%hdr,
1477 on_header => sub {
1478 my ($hdr) = @_;
1479
1480 if ($hdr->{Status} == 200 && $ofs) {
1481 # resume failed
1482 truncate $fh, $ofs = 0;
1483 }
1484
1485 sysseek $fh, $ofs, 0;
1486
1487 1
1488 },
1489 on_body => sub {
1490 my ($data, $hdr) = @_;
1491
1492 if ($hdr->{Status} =~ /^2/) {
1493 length $data == syswrite $fh, $data
1494 or return; # abort on write errors
1495 }
1496
1497 1
1498 },
1499 sub {
1500 my (undef, $hdr) = @_;
1501
1502 my $status = $hdr->{Status};
1503
1504 if (my $time = AnyEvent::HTTP::parse_date $hdr->{"last-modified"}) {
1505 utime $fh, $time, $time;
1506 }
1507
1508 if ($status == 200 || $status == 206 || $status == 416) {
1509 # download ok || resume ok || file already fully downloaded
1510 $cb->(1, $hdr);
1511
1512 } elsif ($status == 412) {
1513 # file has changed while resuming, delete and retry
1514 unlink $file;
1515 $cb->(0, $hdr);
1516
1517 } elsif ($status == 500 or $status == 503 or $status =~ /^59/) {
1518 # retry later
1519 $cb->(0, $hdr);
1520
1521 } else {
1522 $cb->(undef, $hdr);
1523 }
1524 }
1525 ;
1526 }
1527
1528 download "http://server/somelargefile", "/tmp/somelargefile", sub {
1529 if ($_[0]) {
1530 print "OK!\n";
1531 } elsif (defined $_[0]) {
1532 print "please retry later\n";
1533 } else {
1534 print "ERROR\n";
1535 }
1536 };
1537
1538=head3 SOCKS PROXIES
1539
1540Socks proxies are not directly supported by AnyEvent::HTTP. You can
1541compile your perl to support socks, or use an external program such as
1542F<socksify> (dante) or F<tsocks> to make your program use a socks proxy
1543transparently.
1544
1545Alternatively, for AnyEvent::HTTP only, you can use your own
1546C<tcp_connect> function that does the proxy handshake - here is an example
1547that works with socks4a proxies:
1548
1549 use Errno;
1550 use AnyEvent::Util;
1551 use AnyEvent::Socket;
1552 use AnyEvent::Handle;
1553
1554 # host, port and username of/for your socks4a proxy
1555 my $socks_host = "10.0.0.23";
1556 my $socks_port = 9050;
1557 my $socks_user = "";
1558
1559 sub socks4a_connect {
1560 my ($host, $port, $connect_cb, $prepare_cb) = @_;
1561
1562 my $hdl = new AnyEvent::Handle
1563 connect => [$socks_host, $socks_port],
1564 on_prepare => sub { $prepare_cb->($_[0]{fh}) },
1565 on_error => sub { $connect_cb->() },
1566 ;
1567
1568 $hdl->push_write (pack "CCnNZ*Z*", 4, 1, $port, 1, $socks_user, $host);
1569
1570 $hdl->push_read (chunk => 8, sub {
1571 my ($hdl, $chunk) = @_;
1572 my ($status, $port, $ipn) = unpack "xCna4", $chunk;
1573
1574 if ($status == 0x5a) {
1575 $connect_cb->($hdl->{fh}, (format_address $ipn) . ":$port");
1576 } else {
1577 $! = Errno::ENXIO; $connect_cb->();
1578 }
1579 });
1580
1581 $hdl
1582 }
1583
1584Use C<socks4a_connect> instead of C<tcp_connect> when doing C<http_request>s,
1585possibly after switching off other proxy types:
1586
1587 AnyEvent::HTTP::set_proxy undef; # usually you do not want other proxies
1588
1589 http_get 'http://www.google.com', tcp_connect => \&socks4a_connect, sub {
1590 my ($data, $headers) = @_;
1591 ...
1592 };
515 1593
516=head1 SEE ALSO 1594=head1 SEE ALSO
517 1595
518L<AnyEvent>. 1596L<AnyEvent>.
519 1597
520=head1 AUTHOR 1598=head1 AUTHOR
521 1599
522 Marc Lehmann <schmorp@schmorp.de> 1600 Marc Lehmann <schmorp@schmorp.de>
523 http://home.schmorp.de/ 1601 http://home.schmorp.de/
1602
1603With many thanks to Дмитрий Шалашов, who provided countless
1604testcases and bugreports.
524 1605
525=cut 1606=cut
526 1607
5271 16081
528 1609

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines