[ViewVC] Diff of: cvs/AnyEvent-HTTP/HTTP.pm

Comparing AnyEvent-HTTP/HTTP.pm (file contents):
Revision 1.71 by root, Fri Dec 31 20:50:58 2010 UTC vs.
Revision 1.89 by root, Mon Jan 3 00:23:25 2011 UTC

…		…
36		36
37	=cut	37	=cut
38		38
39	package AnyEvent::HTTP;	39	package AnyEvent::HTTP;
40		40
41	use strict;	41	use common::sense;
42	no warnings;
43		42
44	use Errno ();	43	use Errno ();
45		44
46	use AnyEvent 5.0 ();	45	use AnyEvent 5.0 ();
47	use AnyEvent::Util ();	46	use AnyEvent::Util ();
…		…
58	our $MAX_PERSISTENT = 8;	57	our $MAX_PERSISTENT = 8;
59	our $PERSISTENT_TIMEOUT = 2;	58	our $PERSISTENT_TIMEOUT = 2;
60	our $TIMEOUT = 300;	59	our $TIMEOUT = 300;
61		60
62	# changing these is evil	61	# changing these is evil
63	our $MAX_PERSISTENT_PER_HOST = 0;	62	our $MAX_PERSISTENT_PER_HOST = 2;
64	our $MAX_PER_HOST = 4;	63	our $MAX_PER_HOST = 4;
65		64
66	our $PROXY;	65	our $PROXY;
67	our $ACTIVE = 0;	66	our $ACTIVE = 0;
68		67
…		…
122		121
123	If the server sends a header multiple times, then their contents will be	122	If the server sends a header multiple times, then their contents will be
124	joined together with a comma (C<,>), as per the HTTP spec.	123	joined together with a comma (C<,>), as per the HTTP spec.
125		124
126	If an internal error occurs, such as not being able to resolve a hostname,	125	If an internal error occurs, such as not being able to resolve a hostname,
127	then C<$data> will be C<undef>, C<< $headers->{Status} >> will be C<59x>	126	then C<$data> will be C<undef>, C<< $headers->{Status} >> will be
128	(usually C<599>) and the C<Reason> pseudo-header will contain an error	127	C<590>-C<599> and the C<Reason> pseudo-header will contain an error
129	message.	128	message. Currently the following status codes are used:
		129
		130	=over 4
		131
		132	=item 595 - errors during connection etsbalishment, proxy handshake.
		133
		134	=item 596 - errors during TLS negotiation, request sending and header processing.
		135
		136	=item 597 - errors during body receiving or processing.
		137
		138	=item 598 - user aborted request via C<on_header> or C<on_body>.
		139
		140	=item 599 - other, usually nonretryable, errors (garbled URL etc.).
		141
		142	=back
130		143
131	A typical callback might look like this:	144	A typical callback might look like this:
132		145
133	sub {	146	sub {
134	my ($body, $hdr) = @_;	147	my ($body, $hdr) = @_;
…		…
182	=item cookie_jar => $hash_ref	195	=item cookie_jar => $hash_ref
183		196
184	Passing this parameter enables (simplified) cookie-processing, loosely	197	Passing this parameter enables (simplified) cookie-processing, loosely
185	based on the original netscape specification.	198	based on the original netscape specification.
186		199
187	The C<$hash_ref> must be an (initially empty) hash reference which will	200	The C<$hash_ref> must be an (initially empty) hash reference which
188	get updated automatically. It is possible to save the cookie jar to	201	will get updated automatically. It is possible to save the cookie jar
189	persistent storage with something like JSON or Storable, but this is not	202	to persistent storage with something like JSON or Storable - see the
190	recommended, as session-only cookies might survive longer than expected.	203	C<AnyEvent::HTTP::cookie_jar_expire> function if you wish to remove
		204	expired or session-only cookies, and also for documentation on the format
		205	of the cookie jar.
191		206
192	Note that this cookie implementation is not meant to be complete. If	207	Note that this cookie implementation is not meant to be complete. If
193	you want complete cookie management you have to do that on your	208	you want complete cookie management you have to do that on your
194	own. C<cookie_jar> is meant as a quick fix to get some cookie-using sites	209	own. C<cookie_jar> is meant as a quick fix to get most cookie-using sites
195	working. Cookies are a privacy disaster, do not use them unless required	210	working. Cookies are a privacy disaster, do not use them unless required
196	to.	211	to.
197		212
198	When cookie processing is enabled, the C<Cookie:> and C<Set-Cookie:>	213	When cookie processing is enabled, the C<Cookie:> and C<Set-Cookie:>
199	headers will be set and handled by this module, otherwise they will be	214	headers will be set and handled by this module, otherwise they will be
…		…
364	push @{ $CO_SLOT{$_[0]}[1] }, $_[1];	379	push @{ $CO_SLOT{$_[0]}[1] }, $_[1];
365		380
366	_slot_schedule $_[0];	381	_slot_schedule $_[0];
367	}	382	}
368		383
		384	#############################################################################
		385
		386	# expire cookies
		387	sub cookie_jar_expire($;$) {
		388	my ($jar, $session_end) = @_;
		389
		390	%$jar = () if $jar->{version} != 1;
		391
		392	my $anow = AE::now;
		393
		394	while (my ($chost, $paths) = each %$jar) {
		395	next unless ref $paths;
		396
		397	while (my ($cpath, $cookies) = each %$paths) {
		398	while (my ($cookie, $kv) = each %$cookies) {
		399	if (exists $kv->{_expires}) {
		400	delete $cookies->{$cookie}
		401	if $anow > $kv->{_expires};
		402	} elsif ($session_end) {
		403	delete $cookies->{$cookie};
		404	}
		405	}
		406
		407	delete $paths->{$cpath}
		408	unless %$cookies;
		409	}
		410
		411	delete $jar->{$chost}
		412	unless %$paths;
		413	}
		414	}
		415
		416	# extract cookies from jar
369	sub cookie_jar_extract($$$$) {	417	sub cookie_jar_extract($$$$) {
370	my ($jar, $uscheme, $uhost, $upath) = @_;	418	my ($jar, $uscheme, $uhost, $upath) = @_;
371		419
372	%$jar = () if $jar->{version} != 1;	420	%$jar = () if $jar->{version} != 1;
373		421
…		…
388	next unless $cpath eq substr $upath, 0, length $cpath;	436	next unless $cpath eq substr $upath, 0, length $cpath;
389		437
390	while (my ($cookie, $kv) = each %$cookies) {	438	while (my ($cookie, $kv) = each %$cookies) {
391	next if $uscheme ne "https" && exists $kv->{secure};	439	next if $uscheme ne "https" && exists $kv->{secure};
392		440
393	if (exists $kv->{expires}) {	441	if (exists $kv->{_expires} and AE::now > $kv->{_expires}) {
394	if (AE::now > parse_date ($kv->{expires})) {
395	delete $cookies->{$cookie};	442	delete $cookies->{$cookie};
396	next;	443	next;
397	}
398	}	444	}
399		445
400	my $value = $kv->{value};	446	my $value = $kv->{value};
401		447
402	if ($value =~ /[=;,[:space:]]/) {	448	if ($value =~ /[=;,[:space:]]/) {
…		…
410	}	456	}
411		457
412	\@cookies	458	\@cookies
413	}	459	}
414		460
		461	# parse set_cookie header into jar
		462	sub cookie_jar_set_cookie($$$$) {
		463	my ($jar, $set_cookie, $uhost, $date) = @_;
		464
		465	my $anow = int AE::now;
		466	my $snow; # server-now
		467
		468	for ($set_cookie) {
		469	# parse NAME=VALUE
		470	my @kv;
		471
		472	# expires is not http-compliant in the original cookie-spec,
		473	# we support the official date format and some extensions
		474	while (
		475	m{
		476	\G\s*
		477	(?:
		478	expires \s=\s ([A-Z][a-z][a-z]+,\ [^,;]+)
		479	\| ([^=;,[:space:]]+) (?: \s=\s (?: "((?:[^\\"]+\|\\.))" \| ([^=;,[:space:]]) ) )?
		480	)
		481	}gcxsi
		482	) {
		483	my $name = $2;
		484	my $value = $4;
		485
		486	if (defined $1) {
		487	# expires
		488	$name = "expires";
		489	$value = $1;
		490	} elsif (defined $3) {
		491	# quoted
		492	$value = $3;
		493	$value =~ s/\\(.)/$1/gs;
		494	}
		495
		496	push @kv, lc $name, $value;
		497
		498	last unless /\G\s*;/gc;
		499	}
		500
		501	last unless @kv;
		502
		503	my $name = shift @kv;
		504	my %kv = (value => shift @kv, @kv);
		505
		506	if (exists $kv{"max-age"}) {
		507	$kv{_expires} = $anow + delete $kv{"max-age"};
		508	} elsif (exists $kv{expires}) {
		509	$snow \|\|= parse_date ($date) \|\| $anow;
		510	$kv{_expires} = $anow + (parse_date (delete $kv{expires}) - $snow);
		511	} else {
		512	delete $kv{_expires};
		513	}
		514
		515	my $cdom;
		516	my $cpath = (delete $kv{path}) \|\| "/";
		517
		518	if (exists $kv{domain}) {
		519	$cdom = delete $kv{domain};
		520
		521	$cdom =~ s/^\.?/./; # make sure it starts with a "."
		522
		523	next if $cdom =~ /\.$/;
		524
		525	# this is not rfc-like and not netscape-like. go figure.
		526	my $ndots = $cdom =~ y/.//;
		527	next if $ndots < ($cdom =~ /\.[^.][^.]\.[^.][^.]$/ ? 3 : 2);
		528	} else {
		529	$cdom = $uhost;
		530	}
		531
		532	# store it
		533	$jar->{version} = 1;
		534	$jar->{lc $cdom}{$cpath}{$name} = \%kv;
		535
		536	redo if /\G\s*,/gc;
		537	}
		538	}
		539
415	# continue to parse $_ for headers and place them into the arg	540	# continue to parse $_ for headers and place them into the arg
416	sub parse_hdr() {	541	sub parse_hdr() {
417	my %hdr;	542	my %hdr;
418		543
419	# things seen, not parsed:	544	# things seen, not parsed:
…		…
481	: return $cb->(undef, { @pseudo, Status => 599, Reason => "Only http and https URL schemes supported" });	606	: return $cb->(undef, { @pseudo, Status => 599, Reason => "Only http and https URL schemes supported" });
482		607
483	$uauthority =~ /^(?: .*\@ )? ([^\@:]+) (?: : (\d+) )?$/x	608	$uauthority =~ /^(?: .*\@ )? ([^\@:]+) (?: : (\d+) )?$/x
484	or return $cb->(undef, { @pseudo, Status => 599, Reason => "Unparsable URL" });	609	or return $cb->(undef, { @pseudo, Status => 599, Reason => "Unparsable URL" });
485		610
486	my $uhost = $1;	611	my $uhost = lc $1;
487	$uport = $2 if defined $2;	612	$uport = $2 if defined $2;
488		613
489	$hdr{host} = defined $2 ? "$uhost:$2" : "$uhost"	614	$hdr{host} = defined $2 ? "$uhost:$2" : "$uhost"
490	unless exists $hdr{host};	615	unless exists $hdr{host};
491		616
…		…
510	$rscheme = "http" unless defined $rscheme;	635	$rscheme = "http" unless defined $rscheme;
511		636
512	# don't support https requests over https-proxy transport,	637	# don't support https requests over https-proxy transport,
513	# can't be done with tls as spec'ed, unless you double-encrypt.	638	# can't be done with tls as spec'ed, unless you double-encrypt.
514	$rscheme = "http" if $uscheme eq "https" && $rscheme eq "https";	639	$rscheme = "http" if $uscheme eq "https" && $rscheme eq "https";
		640
		641	$rhost = lc $rhost;
		642	$rscheme = lc $rscheme;
515	} else {	643	} else {
516	($rhost, $rport, $rscheme, $rpath) = ($uhost, $uport, $uscheme, $upath);	644	($rhost, $rport, $rscheme, $rpath) = ($uhost, $uport, $uscheme, $upath);
517	}	645	}
518		646
519	# leave out fragment and query string, just a heuristic	647	# leave out fragment and query string, just a heuristic
…		…
521	$hdr{"user-agent"} = $USERAGENT unless exists $hdr{"user-agent"};	649	$hdr{"user-agent"} = $USERAGENT unless exists $hdr{"user-agent"};
522		650
523	$hdr{"content-length"} = length $arg{body}	651	$hdr{"content-length"} = length $arg{body}
524	if length $arg{body} \|\| $method ne "GET";	652	if length $arg{body} \|\| $method ne "GET";
525		653
526	$hdr{connection} = "close TE"; #1.1	654	my $idempotent = $method =~ /^(?:GET\|HEAD\|PUT\|DELETE\|OPTIONS\|TRACE)$/;
		655
		656	# default value for keepalive is true iff the request is for an idempotent method
		657	my $keepalive = exists $arg{keepalive}
		658	? $arg{keepalive}*1
		659	: $idempotent ? $PERSISTENT_TIMEOUT : 0;
		660
		661	$hdr{connection} = ($keepalive ? "" : "close ") . "Te"; #1.1
527	$hdr{te} = "trailers" unless exists $hdr{te}; #1.1	662	$hdr{te} = "trailers" unless exists $hdr{te}; #1.1
528		663
529	my %state = (connect_guard => 1);	664	my %state = (connect_guard => 1);
530		665
531	_get_slot $uhost, sub {	666	my $ae_error = 595; # connecting
532	$state{slot_guard} = shift;
533		667
		668	# handle actual, non-tunneled, request
		669	my $handle_actual_request = sub {
		670	$ae_error = 596; # request phase
		671
		672	$state{handle}->starttls ("connect") if $uscheme eq "https" && !exists $state{handle}{tls};
		673
		674	# send request
		675	$state{handle}->push_write (
		676	"$method $rpath HTTP/1.1\015\012"
		677	. (join "", map "\u$_: $hdr{$_}\015\012", grep defined $hdr{$_}, keys %hdr)
		678	. "\015\012"
		679	. (delete $arg{body})
		680	);
		681
		682	# return if error occured during push_write()
534	return unless $state{connect_guard};	683	return unless %state;
535		684
536	my $connect_cb = sub {	685	# reduce memory usage, save a kitten, also re-use it for the response headers.
537	$state{fh} = shift	686	%hdr = ();
		687
		688	# status line and headers
		689	$state{read_response} = sub {
		690	for ("$_[1]") {
		691	y/\015//d; # weed out any \015, as they show up in the weirdest of places.
		692
		693	/^HTTP\/0([0-9\.]+) \s+ ([0-9]{3}) (?: \s+ ([^\012]) )? \012/gxci
		694	or return (%state = (), $cb->(undef, { @pseudo, Status => 599, Reason => "Invalid server response" }));
		695
		696	# 100 Continue handling
		697	# should not happen as we don't send expect: 100-continue,
		698	# but we handle it just in case.
		699	# since we send the request body regardless, if we get an error
		700	# we are out of-sync, which we currently do NOT handle correctly.
		701	return $state{handle}->push_read (line => $qr_nlnl, $state{read_response})
		702	if $2 eq 100;
		703
		704	push @pseudo,
		705	HTTPVersion => $1,
		706	Status => $2,
		707	Reason => $3,
538	or do {	708	;
539	my $err = "$!";	709
		710	my $hdr = parse_hdr
		711	or return (%state = (), $cb->(undef, { @pseudo, Status => 599, Reason => "Garbled response headers" }));
		712
		713	%hdr = (%$hdr, @pseudo);
		714	}
		715
		716	# redirect handling
		717	# microsoft and other shitheads don't give a shit for following standards,
		718	# try to support some common forms of broken Location headers.
		719	if ($hdr{location} !~ /^(?: $ \| [^:\/?\#]+ : )/x) {
		720	$hdr{location} =~ s/^\.\/+//;
		721
		722	my $url = "$rscheme://$uhost:$uport";
		723
		724	unless ($hdr{location} =~ s/^\///) {
		725	$url .= $upath;
		726	$url =~ s/\/[^\/]*$//;
		727	}
		728
		729	$hdr{location} = "$url/$hdr{location}";
		730	}
		731
		732	my $redirect;
		733
		734	if ($recurse) {
		735	my $status = $hdr{Status};
		736
		737	# industry standard is to redirect POST as GET for
		738	# 301, 302 and 303, in contrast to HTTP/1.0 and 1.1.
		739	# also, the UA should ask the user for 301 and 307 and POST,
		740	# industry standard seems to be to simply follow.
		741	# we go with the industry standard.
		742	if ($status == 301 or $status == 302 or $status == 303) {
		743	# HTTP/1.1 is unclear on how to mutate the method
		744	$method = "GET" unless $method eq "HEAD";
		745	$redirect = 1;
		746	} elsif ($status == 307) {
		747	$redirect = 1;
		748	}
		749	}
		750
		751	my $finish = sub { # ($data, $err_status, $err_reason[, $keepalive])
		752	my $may_keep_alive = $_[3];
		753
		754	$state{handle}->destroy if $state{handle};
540	%state = ();	755	%state = ();
541	return $cb->(undef, { @pseudo, Status => 599, Reason => $err });	756
		757	if (defined $_[1]) {
		758	$hdr{OrigStatus} = $hdr{Status}; $hdr{Status} = $_[1];
		759	$hdr{OrigReason} = $hdr{Reason}; $hdr{Reason} = $_[2];
		760	}
		761
		762	# set-cookie processing
		763	if ($arg{cookie_jar}) {
		764	cookie_jar_set_cookie $arg{cookie_jar}, $hdr{"set-cookie"}, $uhost, $hdr{date};
		765	}
		766
		767	if ($redirect && exists $hdr{location}) {
		768	# we ignore any errors, as it is very common to receive
		769	# Content-Length != 0 but no actual body
		770	# we also access %hdr, as $_[1] might be an erro
		771	http_request (
		772	$method => $hdr{location},
		773	%arg,
		774	recurse => $recurse - 1,
		775	Redirect => [$_[0], \%hdr],
		776	$cb);
		777	} else {
		778	$cb->($_[0], \%hdr);
		779	}
		780	};
		781
		782	$ae_error = 597; # body phase
		783
		784	my $len = $hdr{"content-length"};
		785
		786	# body handling, many different code paths
		787	# - no body expected
		788	# - want_body_handle
		789	# - te chunked
		790	# - 2x length known (with or without on_body)
		791	# - 2x length not known (with or without on_body)
		792	if (!$redirect && $arg{on_header} && !$arg{on_header}(\%hdr)) {
		793	$finish->(undef, 598 => "Request cancelled by on_header");
		794	} elsif (
		795	$hdr{Status} =~ /^(?:1..\|204\|205\|304)$/
		796	or $method eq "HEAD"
		797	or (defined $len && $len == 0) # == 0, not !, because "0 " is true
		798	) {
		799	# no body
		800	$finish->("", undef, undef, 1);
		801
		802	} elsif (!$redirect && $arg{want_body_handle}) {
		803	$_[0]->on_eof (undef);
		804	$_[0]->on_error (undef);
		805	$_[0]->on_read (undef);
		806
		807	$finish->(delete $state{handle});
		808
		809	} elsif ($hdr{"transfer-encoding"} =~ /\bchunked\b/i) {
		810	my $cl = 0;
		811	my $body = undef;
		812	my $on_body = $arg{on_body} \|\| sub { $body .= shift; 1 };
		813
		814	$state{read_chunk} = sub {
		815	$_[1] =~ /^([0-9a-fA-F]+)/
		816	or $finish->(undef, $ae_error => "Garbled chunked transfer encoding");
		817
		818	my $len = hex $1;
		819
		820	if ($len) {
		821	$cl += $len;
		822
		823	$_[0]->push_read (chunk => $len, sub {
		824	$on_body->($_[1], \%hdr)
		825	or return $finish->(undef, 598 => "Request cancelled by on_body");
		826
		827	$_[0]->push_read (line => sub {
		828	length $_[1]
		829	and return $finish->(undef, $ae_error => "Garbled chunked transfer encoding");
		830	$_[0]->push_read (line => $state{read_chunk});
		831	});
		832	});
		833	} else {
		834	$hdr{"content-length"} \|\|= $cl;
		835
		836	$_[0]->push_read (line => $qr_nlnl, sub {
		837	if (length $_[1]) {
		838	for ("$_[1]") {
		839	y/\015//d; # weed out any \015, as they show up in the weirdest of places.
		840
		841	my $hdr = parse_hdr
		842	or return $finish->(undef, $ae_error => "Garbled response trailers");
		843
		844	%hdr = (%hdr, %$hdr);
		845	}
		846	}
		847
		848	$finish->($body, undef, undef, 1);
		849	});
		850	}
542	};	851	};
543		852
544	pop; # free memory, save a tree	853	$_[0]->push_read (line => $state{read_chunk});
545		854
		855	} elsif ($arg{on_body}) {
		856	if (defined $len) {
		857	$_[0]->on_read (sub {
		858	$len -= length $_[0]{rbuf};
		859
		860	$arg{on_body}(delete $_[0]{rbuf}, \%hdr)
		861	or return $finish->(undef, 598 => "Request cancelled by on_body");
		862
		863	$len > 0
		864	or $finish->("", undef, undef, 1);
		865	});
		866	} else {
		867	$_[0]->on_eof (sub {
		868	$finish->("");
		869	});
		870	$_[0]->on_read (sub {
		871	$arg{on_body}(delete $_[0]{rbuf}, \%hdr)
		872	or $finish->(undef, 598 => "Request cancelled by on_body");
		873	});
		874	}
		875	} else {
		876	$_[0]->on_eof (undef);
		877
		878	if (defined $len) {
		879	$_[0]->on_read (sub {
		880	$finish->((substr delete $_[0]{rbuf}, 0, $len, ""), undef, undef, 1)
		881	if $len <= length $_[0]{rbuf};
		882	});
		883	} else {
		884	$_[0]->on_error (sub {
		885	($! == Errno::EPIPE \|\| !$!)
		886	? $finish->(delete $_[0]{rbuf})
		887	: $finish->(undef, $ae_error => $_[2]);
		888	});
		889	$_[0]->on_read (sub { });
		890	}
		891	}
		892	};
		893
		894	$state{handle}->push_read (line => $qr_nlnl, $state{read_response});
		895	};
		896
		897	my $connect_cb = sub {
		898	$state{fh} = shift
		899	or do {
		900	my $err = "$!";
		901	%state = ();
		902	return $cb->(undef, { @pseudo, Status => $ae_error, Reason => $err });
		903	};
		904
546	return unless delete $state{connect_guard};	905	return unless delete $state{connect_guard};
547		906
548	# get handle	907	# get handle
549	$state{handle} = new AnyEvent::Handle	908	$state{handle} = new AnyEvent::Handle
550	fh => $state{fh},	909	fh => $state{fh},
551	peername => $rhost,	910	peername => $rhost,
552	tls_ctx => $arg{tls_ctx},	911	tls_ctx => $arg{tls_ctx},
553	# these need to be reconfigured on keepalive handles	912	# these need to be reconfigured on keepalive handles
554	timeout => $timeout,	913	timeout => $timeout,
555	on_error => sub {	914	on_error => sub {
556	%state = ();	915	%state = ();
557	$cb->(undef, { @pseudo, Status => 599, Reason => $_[2] });	916	$cb->(undef, { @pseudo, Status => $ae_error, Reason => $_[2] });
558	},	917	},
559	on_eof => sub {	918	on_eof => sub {
560	%state = ();	919	%state = ();
561	$cb->(undef, { @pseudo, Status => 599, Reason => "Unexpected end-of-file" });	920	$cb->(undef, { @pseudo, Status => $ae_error, Reason => "Unexpected end-of-file" });
562	},	921	},
563	;	922	;
564		923
565	# limit the number of persistent connections	924	# limit the number of persistent connections
566	# keepalive not yet supported	925	# keepalive not yet supported
567	# if ($KA_COUNT{$_[1]} < $MAX_PERSISTENT_PER_HOST) {	926	# if ($KA_COUNT{$_[1]} < $MAX_PERSISTENT_PER_HOST) {
568	# ++$KA_COUNT{$_[1]};	927	# ++$KA_COUNT{$_[1]};
569	# $state{handle}{ka_count_guard} = AnyEvent::Util::guard {	928	# $state{handle}{ka_count_guard} = AnyEvent::Util::guard {
570	# --$KA_COUNT{$_[1]}	929	# --$KA_COUNT{$_[1]}
571	# };	930	# };
572	# $hdr{connection} = "keep-alive";	931	# $hdr{connection} = "keep-alive";
573	# }	932	# }
574		933
575	$state{handle}->starttls ("connect") if $rscheme eq "https";	934	$state{handle}->starttls ("connect") if $rscheme eq "https";
576		935
577	# handle actual, non-tunneled, request
578	my $handle_actual_request = sub {
579	$state{handle}->starttls ("connect") if $uscheme eq "https" && !exists $state{handle}{tls};
580
581	# send request
582	$state{handle}->push_write (
583	"$method $rpath HTTP/1.1\015\012"
584	. (join "", map "\u$_: $hdr{$_}\015\012", grep defined $hdr{$_}, keys %hdr)
585	. "\015\012"
586	. (delete $arg{body})
587	);
588
589	# return if error occured during push_write()
590	return unless %state;
591
592	%hdr = (); # reduce memory usage, save a kitten, also make it possible to re-use
593
594	# status line and headers
595	$state{read_response} = sub {
596	for ("$_[1]") {
597	y/\015//d; # weed out any \015, as they show up in the weirdest of places.
598
599	/^HTTP\/([0-9\.]+) \s+ ([0-9]{3}) (?: \s+ ([^\012]*) )? \012/igxc
600	or return (%state = (), $cb->(undef, { @pseudo, Status => 599, Reason => "Invalid server response" }));
601
602	# 100 Continue handling
603	# should not happen as we don't send expect: 100-continue,
604	# but we handle it just in case.
605	# since we send the request body regardless, if we get an error
606	# we are out of-sync, which we currently do NOT handle correctly.
607	return $state{handle}->push_read (line => $qr_nlnl, $state{read_response})
608	if $2 eq 100;
609
610	push @pseudo,
611	HTTPVersion => $1,
612	Status => $2,
613	Reason => $3,
614	;
615
616	my $hdr = parse_hdr
617	or return (%state = (), $cb->(undef, { @pseudo, Status => 599, Reason => "Garbled response headers" }));
618
619	%hdr = (%$hdr, @pseudo);
620	}
621
622	# redirect handling
623	# microsoft and other shitheads don't give a shit for following standards,
624	# try to support some common forms of broken Location headers.
625	if ($hdr{location} !~ /^(?: $ \| [^:\/?\#]+ : )/x) {
626	$hdr{location} =~ s/^\.\/+//;
627
628	my $url = "$rscheme://$uhost:$uport";
629
630	unless ($hdr{location} =~ s/^\///) {
631	$url .= $upath;
632	$url =~ s/\/[^\/]*$//;
633	}
634
635	$hdr{location} = "$url/$hdr{location}";
636	}
637
638	my $redirect;
639
640	if ($recurse) {
641	my $status = $hdr{Status};
642
643	# industry standard is to redirect POST as GET for
644	# 301, 302 and 303, in contrast to http/1.0 and 1.1.
645	# also, the UA should ask the user for 301 and 307 and POST,
646	# industry standard seems to be to simply follow.
647	# we go with the industry standard.
648	if ($status == 301 or $status == 302 or $status == 303) {
649	# HTTP/1.1 is unclear on how to mutate the method
650	$method = "GET" unless $method eq "HEAD";
651	$redirect = 1;
652	} elsif ($status == 307) {
653	$redirect = 1;
654	}
655	}
656
657	my $finish = sub { # ($data, $err_status, $err_reason[, $keepalive])
658	my $keepalive = pop;
659
660	$state{handle}->destroy if $state{handle};
661	%state = ();
662
663	if (defined $_[1]) {
664	$hdr{OrigStatus} = $hdr{Status}; $hdr{Status} = $_[1];
665	$hdr{OrigReason} = $hdr{Reason}; $hdr{Reason} = $_[2];
666	}
667
668	# set-cookie processing
669	if ($arg{cookie_jar}) {
670	for ($hdr{"set-cookie"}) {
671	# parse NAME=VALUE
672	my @kv;
673
674	while (
675	m{
676	\G\s*
677	(?:
678	expires \s=\s ([A-Z][a-z][a-z],\ [^,;]+)
679	\| ([^=;,[:space:]]+) \s=\s (?: "((?:[^\\"]+\|\\.))" \| ([^=;,[:space:]]) )
680	)
681	}gcxsi
682	) {
683	my $name = $2;
684	my $value = $4;
685
686	unless (defined $name) {
687	# expires
688	$name = "expires";
689	$value = $1;
690	} elsif (!defined $value) {
691	# quoted
692	$value = $3;
693	$value =~ s/\\(.)/$1/gs;
694	}
695
696	push @kv, lc $name, $value;
697
698	last unless /\G\s*;/gc;
699	}
700
701	last unless @kv;
702
703	my $name = shift @kv;
704	my %kv = (value => shift @kv, @kv);
705
706	$kv{expires} \|\|= format_date (AE::now + $kv{"max-age"})
707	if exists $kv{"max-age"};
708
709	my $cdom;
710	my $cpath = (delete $kv{path}) \|\| "/";
711
712	if (exists $kv{domain}) {
713	$cdom = delete $kv{domain};
714
715	$cdom =~ s/^\.?/./; # make sure it starts with a "."
716
717	next if $cdom =~ /\.$/;
718
719	# this is not rfc-like and not netscape-like. go figure.
720	my $ndots = $cdom =~ y/.//;
721	next if $ndots < ($cdom =~ /\.[^.][^.]\.[^.][^.]$/ ? 3 : 2);
722	} else {
723	$cdom = $uhost;
724	}
725
726	# store it
727	$arg{cookie_jar}{version} = 1;
728	$arg{cookie_jar}{$cdom}{$cpath}{$name} = \%kv;
729
730	redo if /\G\s*,/gc;
731	}
732	}
733
734	if ($redirect && exists $hdr{location}) {
735	# we ignore any errors, as it is very common to receive
736	# Content-Length != 0 but no actual body
737	# we also access %hdr, as $_[1] might be an erro
738	http_request (
739	$method => $hdr{location},
740	%arg,
741	recurse => $recurse - 1,
742	Redirect => [$_[0], \%hdr],
743	$cb);
744	} else {
745	$cb->($_[0], \%hdr);
746	}
747	};
748
749	my $len = $hdr{"content-length"};
750
751	if (!$redirect && $arg{on_header} && !$arg{on_header}(\%hdr)) {
752	$finish->(undef, 598 => "Request cancelled by on_header");
753	} elsif (
754	$hdr{Status} =~ /^(?:1..\|204\|205\|304)$/
755	or $method eq "HEAD"
756	or (defined $len && !$len)
757	) {
758	# no body
759	$finish->("", undef, undef, 1);
760	} else {
761	# body handling, many different code paths
762	# - no body expected
763	# - want_body_handle
764	# - te chunked
765	# - 2x length known (with or without on_body)
766	# - 2x length not known (with or without on_body)
767	if (!$redirect && $arg{want_body_handle}) {
768	$_[0]->on_eof (undef);
769	$_[0]->on_error (undef);
770	$_[0]->on_read (undef);
771
772	$finish->(delete $state{handle});
773
774	} elsif ($hdr{"transfer-encoding"} =~ /\bchunked\b/i) {
775	my $cl = 0;
776	my $body = undef;
777	my $on_body = $arg{on_body} \|\| sub { $body .= shift; 1 };
778
779	$_[0]->on_error (sub { $finish->(undef, 599 => $_[2]) });
780
781	my $read_chunk; $read_chunk = sub {
782	$_[1] =~ /^([0-9a-fA-F]+)/
783	or $finish->(undef, 599 => "Garbled chunked transfer encoding");
784
785	my $len = hex $1;
786
787	if ($len) {
788	$cl += $len;
789
790	$_[0]->push_read (chunk => $len, sub {
791	$on_body->($_[1], \%hdr)
792	or return $finish->(undef, 598 => "Request cancelled by on_body");
793
794	$_[0]->push_read (line => sub {
795	length $_[1]
796	and return $finish->(undef, 599 => "Garbled chunked transfer encoding");
797	$_[0]->push_read (line => $read_chunk);
798	});
799	});
800	} else {
801	$hdr{"content-length"} \|\|= $cl;
802
803	$_[0]->push_read (line => $qr_nlnl, sub {
804	if (length $_[1]) {
805	for ("$_[1]") {
806	y/\015//d; # weed out any \015, as they show up in the weirdest of places.
807
808	my $hdr = parse_hdr
809	or return $finish->(undef, 599 => "Garbled response trailers");
810
811	%hdr = (%hdr, %$hdr);
812	}
813	}
814
815	$finish->($body, undef, undef, 1);
816	});
817	}
818	};
819
820	$_[0]->push_read (line => $read_chunk);
821
822	} elsif ($arg{on_body}) {
823	$_[0]->on_error (sub { $finish->(undef, 599 => $_[2]) });
824
825	if ($len) {
826	$_[0]->on_read (sub {
827	$len -= length $_[0]{rbuf};
828
829	$arg{on_body}(delete $_[0]{rbuf}, \%hdr)
830	or return $finish->(undef, 598 => "Request cancelled by on_body");
831
832	$len > 0
833	or $finish->("", undef, undef, 1);
834	});
835	} else {
836	$_[0]->on_eof (sub {
837	$finish->("");
838	});
839	$_[0]->on_read (sub {
840	$arg{on_body}(delete $_[0]{rbuf}, \%hdr)
841	or $finish->(undef, 598 => "Request cancelled by on_body");
842	});
843	}
844	} else {
845	$_[0]->on_eof (undef);
846
847	if ($len) {
848	$_[0]->on_error (sub { $finish->(undef, 599 => $_[2]) });
849	$_[0]->on_read (sub {
850	$finish->((substr delete $_[0]{rbuf}, 0, $len, ""), undef, undef, 1)
851	if $len <= length $_[0]{rbuf};
852	});
853	} else {
854	$_[0]->on_error (sub {
855	($! == Errno::EPIPE \|\| !$!)
856	? $finish->(delete $_[0]{rbuf})
857	: $finish->(undef, 599 => $_[2]);
858	});
859	$_[0]->on_read (sub { });
860	}
861	}
862	}
863	};
864
865	$state{handle}->push_read (line => $qr_nlnl, $state{read_response});
866	};
867
868	# now handle proxy-CONNECT method	936	# now handle proxy-CONNECT method
869	if ($proxy && $uscheme eq "https") {	937	if ($proxy && $uscheme eq "https") {
870	# oh dear, we have to wrap it into a connect request	938	# oh dear, we have to wrap it into a connect request
871		939
872	# maybe re-use $uauthority with patched port?	940	# maybe re-use $uauthority with patched port?
873	$state{handle}->push_write ("CONNECT $uhost:$uport HTTP/1.0\015\012Host: $uhost\015\012\015\012");	941	$state{handle}->push_write ("CONNECT $uhost:$uport HTTP/1.0\015\012\015\012");
874	$state{handle}->push_read (line => $qr_nlnl, sub {	942	$state{handle}->push_read (line => $qr_nlnl, sub {
875	$_[1] =~ /^HTTP\/([0-9\.]+) \s+ ([0-9]{3}) (?: \s+ ([^\015\012]*) )?/ix	943	$_[1] =~ /^HTTP\/([0-9\.]+) \s+ ([0-9]{3}) (?: \s+ ([^\015\012]*) )?/ix
876	or return (%state = (), $cb->(undef, { @pseudo, Status => 599, Reason => "Invalid proxy connect response ($_[1])" }));	944	or return (%state = (), $cb->(undef, { @pseudo, Status => 599, Reason => "Invalid proxy connect response ($_[1])" }));
877		945
878	if ($2 == 200) {	946	if ($2 == 200) {
879	$rpath = $upath;	947	$rpath = $upath;
880	&$handle_actual_request;	948	$handle_actual_request->();
881	} else {	949	} else {
882	%state = ();	950	%state = ();
883	$cb->(undef, { @pseudo, Status => $2, Reason => $3 });	951	$cb->(undef, { @pseudo, Status => $2, Reason => $3 });
884	}
885	});	952	}
886	} else {
887	&$handle_actual_request;
888	}	953	});
		954	} else {
		955	$handle_actual_request->();
889	};	956	}
		957	};
		958
		959	_get_slot $uhost, sub {
		960	$state{slot_guard} = shift;
		961
		962	return unless $state{connect_guard};
890		963
891	my $tcp_connect = $arg{tcp_connect}	964	my $tcp_connect = $arg{tcp_connect}
892	\|\| do { require AnyEvent::Socket; \&AnyEvent::Socket::tcp_connect };	965	\|\| do { require AnyEvent::Socket; \&AnyEvent::Socket::tcp_connect };
893		966
894	$state{connect_guard} = $tcp_connect->($rhost, $rport, $connect_cb, $arg{on_prepare} \|\| sub { $timeout });	967	$state{connect_guard} = $tcp_connect->($rhost, $rport, $connect_cb, $arg{on_prepare} \|\| sub { $timeout });
895
896	};	968	};
897		969
898	defined wantarray && AnyEvent::Util::guard { %state = () }	970	defined wantarray && AnyEvent::Util::guard { %state = () }
899	}	971	}
900		972
…		…
935	string of the form C<http://host:port> (optionally C<https:...>), croaks	1007	string of the form C<http://host:port> (optionally C<https:...>), croaks
936	otherwise.	1008	otherwise.
937		1009
938	To clear an already-set proxy, use C<undef>.	1010	To clear an already-set proxy, use C<undef>.
939		1011
		1012	=item AnyEvent::HTTP::cookie_jar_expire $jar[, $session_end]
		1013
		1014	Remove all cookies from the cookie jar that have been expired. If
		1015	C<$session_end> is given and true, then additionally remove all session
		1016	cookies.
		1017
		1018	You should call this function (with a true C<$session_end>) before you
		1019	save cookies to disk, and you should call this function after loading them
		1020	again. If you have a long-running program you can additonally call this
		1021	function from time to time.
		1022
		1023	A cookie jar is initially an empty hash-reference that is managed by this
		1024	module. It's format is subject to change, but currently it is like this:
		1025
		1026	The key C<version> has to contain C<1>, otherwise the hash gets
		1027	emptied. All other keys are hostnames or IP addresses pointing to
		1028	hash-references. The key for these inner hash references is the
		1029	server path for which this cookie is meant, and the values are again
		1030	hash-references. The keys of those hash-references is the cookie name, and
		1031	the value, you guessed it, is another hash-reference, this time with the
		1032	key-value pairs from the cookie, except for C<expires> and C<max-age>,
		1033	which have been replaced by a C<_expires> key that contains the cookie
		1034	expiry timestamp.
		1035
		1036	Here is an example of a cookie jar with a single cookie, so you have a
		1037	chance of understanding the above paragraph:
		1038
		1039	{
		1040	version => 1,
		1041	"10.0.0.1" => {
		1042	"/" => {
		1043	"mythweb_id" => {
		1044	_expires => 1293917923,
		1045	value => "ooRung9dThee3ooyXooM1Ohm",
		1046	},
		1047	},
		1048	},
		1049	}
		1050
940	=item $date = AnyEvent::HTTP::format_date $timestamp	1051	=item $date = AnyEvent::HTTP::format_date $timestamp
941		1052
942	Takes a POSIX timestamp (seconds since the epoch) and formats it as a HTTP	1053	Takes a POSIX timestamp (seconds since the epoch) and formats it as a HTTP
943	Date (RFC 2616).	1054	Date (RFC 2616).
944		1055
945	=item $timestamp = AnyEvent::HTTP::parse_date $date	1056	=item $timestamp = AnyEvent::HTTP::parse_date $date
946		1057
947	Takes a HTTP Date (RFC 2616) or a Cookie date (netscape cookie spec) and	1058	Takes a HTTP Date (RFC 2616) or a Cookie date (netscape cookie spec) or a
948	returns the corresponding POSIX timestamp, or C<undef> if the date cannot	1059	bunch of minor variations of those, and returns the corresponding POSIX
949	be parsed.	1060	timestamp, or C<undef> if the date cannot be parsed.
950		1061
951	=item $AnyEvent::HTTP::MAX_RECURSE	1062	=item $AnyEvent::HTTP::MAX_RECURSE
952		1063
953	The default value for the C<recurse> request parameter (default: C<10>).	1064	The default value for the C<recurse> request parameter (default: C<10>).
954		1065
…		…
993	sub parse_date($) {	1104	sub parse_date($) {
994	my ($date) = @_;	1105	my ($date) = @_;
995		1106
996	my ($d, $m, $y, $H, $M, $S);	1107	my ($d, $m, $y, $H, $M, $S);
997		1108
998	if ($date =~ /^[A-Z][a-z][a-z], ([0-9][0-9])[\- ]([A-Z][a-z][a-z])[\- ]([0-9][0-9][0-9][0-9]) ([0-9][0-9]):([0-9][0-9]):([0-9][0-9]) GMT$/) {	1109	if ($date =~ /^[A-Z][a-z][a-z]+, ([0-9][0-9]?)[\- ]([A-Z][a-z][a-z])[\- ]([0-9][0-9][0-9][0-9]) ([0-9][0-9]?):([0-9][0-9]?):([0-9][0-9]?) GMT$/) {
999	# RFC 822/1123, required by RFC 2616 (with " ")	1110	# RFC 822/1123, required by RFC 2616 (with " ")
1000	# cookie dates (with "-")	1111	# cookie dates (with "-")
1001		1112
1002	($d, $m, $y, $H, $M, $S) = ($1, $2, $3, $4, $5, $6);	1113	($d, $m, $y, $H, $M, $S) = ($1, $2, $3, $4, $5, $6);
1003		1114
1004	} elsif ($date =~ /^[A-Z][a-z]+, ([0-9][0-9])-([A-Z][a-z][a-z])-([0-9][0-9]) ([0-9][0-9]):([0-9][0-9]):([0-9][0-9]) GMT$/) {	1115	} elsif ($date =~ /^[A-Z][a-z][a-z]+, ([0-9][0-9]?)-([A-Z][a-z][a-z])-([0-9][0-9]) ([0-9][0-9]?):([0-9][0-9]?):([0-9][0-9]?) GMT$/) {
1005	# RFC 850	1116	# RFC 850
1006	($d, $m, $y, $H, $M, $S) = ($1, $2, $3 < 69 ? $3 + 2000 : $3 + 1900, $4, $5, $6);	1117	($d, $m, $y, $H, $M, $S) = ($1, $2, $3 < 69 ? $3 + 2000 : $3 + 1900, $4, $5, $6);
1007		1118
1008	} elsif ($date =~ /^[A-Z][a-z][a-z] ([A-Z][a-z][a-z]) ([0-9 ][0-9]) ([0-9][0-9]):([0-9][0-9]):([0-9][0-9]) ([0-9][0-9][0-9][0-9])$/) {	1119	} elsif ($date =~ /^[A-Z][a-z][a-z]+ ([A-Z][a-z][a-z]) ([0-9 ]?[0-9]) ([0-9][0-9]?):([0-9][0-9]?):([0-9][0-9]?) ([0-9][0-9][0-9][0-9])$/) {
1009	# ISO C's asctime	1120	# ISO C's asctime
1010	($d, $m, $y, $H, $M, $S) = ($2, $1, $6, $3, $4, $5);	1121	($d, $m, $y, $H, $M, $S) = ($2, $1, $6, $3, $4, $5);
1011	}	1122	}
1012	# other formats fail in the loop below	1123	# other formats fail in the loop below
1013		1124

Diff Legend

-–
+Removed lines
-+
+Added lines
-<
+Changed lines
->
+Changed lines

Comparing AnyEvent-HTTP/HTTP.pm (file contents): Revision 1.71 by root, Fri Dec 31 20:50:58 2010 UTC vs. Revision 1.89 by root, Mon Jan 3 00:23:25 2011 UTC

Diff Legend

Comparing AnyEvent-HTTP/HTTP.pm (file contents):
Revision 1.71 by root, Fri Dec 31 20:50:58 2010 UTC vs.
Revision 1.89 by root, Mon Jan 3 00:23:25 2011 UTC