[ViewVC] Diff of: cvs/AnyEvent-HTTP/HTTP.pm

Comparing AnyEvent-HTTP/HTTP.pm (file contents):
Revision 1.92 by root, Tue Jan 4 08:17:59 2011 UTC vs.
Revision 1.104 by root, Thu Feb 24 15:09:03 2011 UTC

 This module is an L<AnyEvent> user, you need to make sure that you use and
 run a supported event loop.
 This module implements a simple, stateless and non-blocking HTTP
 client. It supports GET, POST and other request methods, cookies and more,
-all on a very low level. It can follow redirects supports proxies and
+all on a very low level. It can follow redirects, supports proxies, and
 automatically limits the number of connections to the values specified in
 the RFC.
 It should generally be a "good client" that is enough for most HTTP
 tasks. Simple tasks should be simple, but complex tasks should still be
 use AnyEvent::Util ();
 use AnyEvent::Handle ();
 use base Exporter::;
-our $VERSION = '2.0';
+our $VERSION = '2.1';
 our @EXPORT = qw(http_get http_post http_head http_request);
 our $USERAGENT          = "Mozilla/5.0 (compatible; U; AnyEvent-HTTP/$VERSION; +http://software.schmorp.de/pkg/AnyEvent)";
 our $MAX_RECURSE        =  10;
 You really should provide your own C<User-Agent:> header value that is
 appropriate for your program - I wouldn't be surprised if the default
 AnyEvent string gets blocked by webservers sooner or later.
+Also, make sure that your headers names and values do not contain any
+embedded newlines.
 =item timeout => $seconds
 The time-out to use for various stages - each connect attempt will reset
 the timeout, as will read or write activity, i.e. this is not an overall
 timeout.
 Default timeout is 5 minutes.
 =item proxy => [$host, $port[, $scheme]] or undef
-Use the given http proxy for all requests. If not specified, then the
+Use the given http proxy for all requests, or no proxy if C<undef> is
-default proxy (as specified by C<$ENV{http_proxy}>) is used.
+used.
 C<$scheme> must be either missing or must be C<http> for HTTP.
+If not specified, then the default proxy is used (see
+C<AnyEvent::HTTP::set_proxy>).
 =item body => $string
 The request body, usually empty. Will be sent as-is (future versions of
 this module might offer more options).
       while (
          m{
             \G\s*
             (?:
                expires \s*=\s* ([A-Z][a-z][a-z]+,\ [^,;]+)
-               | ([^=;,[:space:]]+) (?: \s*=\s* (?: "((?:[^\\"]+|\\.)*)" | ([^=;,[:space:]]*) ) )?
+               | ([^=;,[:space:]]+) (?: \s*=\s* (?: "((?:[^\\"]+|\\.)*)" | ([^;,[:space:]]*) ) )?
             )
          }gcxsi
       ) {
          my $name = $2;
          my $value = $4;
             # quoted
             $value = $3;
             $value =~ s/\\(.)/$1/gs;
          }
-         push @kv, lc $name, $value;
+         push @kv, @kv ? lc $name : $name, $value;
          last unless /\G\s*;/gc;
       }
       last unless @kv;
    my $recurse = exists $arg{recurse} ? delete $arg{recurse} : $MAX_RECURSE;
    return $cb->(undef, { @pseudo, Status => 599, Reason => "Too many redirections" })
       if $recurse < 0;
-   my $proxy   = $arg{proxy}   || $PROXY;
+   my $proxy   = exists $arg{proxy} ? $arg{proxy} : $PROXY;
    my $timeout = $arg{timeout} || $TIMEOUT;
    my ($uscheme, $uauthority, $upath, $query, undef) = # ignore fragment
-      $url =~ m|(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*)(?:(\?[^#]*))?(?:#(.*))?|;
+      $url =~ m|^([^:]+):(?://([^/?#]*))?([^?#]*)(?:(\?[^#]*))?(?:#(.*))?$|;
    $uscheme = lc $uscheme;
    my $uport = $uscheme eq "http"  ?  80
              : $uscheme eq "https" ? 443
       if length $arg{body} || $method ne "GET";
    my $idempotent = $method =~ /^(?:GET|HEAD|PUT|DELETE|OPTIONS|TRACE)$/;
    # default value for keepalive is true iff the request is for an idempotent method
-   my $keepalive = exists $arg{keepalive} ? !!$arg{keepalive} : $idempotent;
+   my $persistent = exists $arg{persistent} ? !!$arg{persistent} : $idempotent;
-   my $keepalive10 = exists $arg{keepalive10} ? $arg{keepalive10} : !$proxy;
+   my $keepalive  = exists $arg{keepalive}  ? !!$arg{keepalive}  : !$proxy;
-   my $keptalive; # true if this is actually a recycled connection
+   my $was_persistent; # true if this is actually a recycled connection
    # the key to use in the keepalive cache
    my $ka_key = "$uhost\x00$arg{sessionid}";
-   $hdr{connection} = ($keepalive ? $keepalive10 ? "keep-alive " : "" : "close ") . "Te"; #1.1
+   $hdr{connection} = ($persistent ? $keepalive ? "keep-alive " : "" : "close ") . "Te"; #1.1
    $hdr{te}         = "trailers" unless exists $hdr{te}; #1.1
    my %state = (connect_guard => 1);
    my $ae_error = 595; # connecting
             } elsif ($status == 307) {
                $redirect = 1;
             }
          }
-         my $finish = sub { # ($data, $err_status, $err_reason[, $keepalive])
+         my $finish = sub { # ($data, $err_status, $err_reason[, $persistent])
             if ($state{handle}) {
                # handle keepalive
                if (
-                  $keepalive
+                  $persistent
                   && $_[3]
                   && ($hdr{HTTPVersion} < 1.1
                       ? $hdr{connection} =~ /\bkeep-?alive\b/i
                       : $hdr{connection} !~ /\bclose\b/i)
                ) {
             if ($redirect && exists $hdr{location}) {
                # we ignore any errors, as it is very common to receive
                # Content-Length != 0 but no actual body
                # we also access %hdr, as $_[1] might be an erro
+               $state{recurse} =
-               http_request (
+                  http_request (
-                  $method  => $hdr{location},
+                     $method  => $hdr{location},
-                  %arg,
+                     %arg,
-                  recurse  => $recurse - 1,
+                     recurse  => $recurse - 1,
-                  Redirect => [$_[0], \%hdr],
+                     Redirect => [$_[0], \%hdr],
+                     sub {
+                        %state = ();
-                  $cb
+                        &$cb
+                     },
-               );
+                  );
             } else {
                $cb->($_[0], \%hdr);
             }
          };
          }
       };
       # if keepalive is enabled, then the server closing the connection
       # before a response can happen legally - we retry on idempotent methods.
-      if ($keptalive && $idempotent) {
+      if ($was_persistent && $idempotent) {
          my $old_eof = $hdl->{on_eof};
          $hdl->{on_eof} = sub {
             _destroy_state %state;
+            %state = ();
+            $state{recurse} =
-            http_request (
+               http_request (
-               $method => $url,
+                  $method => $url,
-               %arg,
+                  %arg,
-               keepalive => 0,
+                  keepalive => 0,
+                  sub {
+                     %state = ();
-               $cb
+                     &$cb
+                  }
-            );
+               );
          };
          $hdl->on_read (sub {
             return unless %state;
             # as soon as we receive something, a connection close
    };
    my $prepare_handle = sub {
       my ($hdl) = $state{handle};
-      $hdl->timeout ($timeout);
       $hdl->on_error (sub {
          _error %state, $cb, { @pseudo, Status => $ae_error, Reason => $_[2] };
       });
       $hdl->on_eof (sub {
          _error %state, $cb, { @pseudo, Status => $ae_error, Reason => "Unexpected end-of-file" };
       });
+      $hdl->timeout_reset;
+      $hdl->timeout ($timeout);
    };
    # connected to proxy (or origin server)
    my $connect_cb = sub {
       my $fh = shift
       return unless $state{connect_guard};
       # try to use an existing keepalive connection, but only if we, ourselves, plan
       # on a keepalive request (in theory, this should be a separate config option).
-      if ($keepalive && $KA_CACHE{$ka_key}) {
+      if ($persistent && $KA_CACHE{$ka_key}) {
-         $keptalive = 1;
+         $was_persistent = 1;
          $state{handle} = ka_fetch $ka_key;
+         $state{handle}->destroyed
+            and die "got a destructed habndle. pah\n";#d#
          $prepare_handle->();
+         $state{handle}->destroyed
+            and die "got a destructed habndle. pa2\n";#d#
          $handle_actual_request->();
+         $state{handle}->destroyed
+            and die "got a destructed habndle. pa3\n";#d#
       } else {
          my $tcp_connect = $arg{tcp_connect}
                            || do { require AnyEvent::Socket; \&AnyEvent::Socket::tcp_connect };
 Sets the default proxy server to use. The proxy-url must begin with a
 string of the form C<http://host:port>, croaks otherwise.
 To clear an already-set proxy, use C<undef>.
+When AnyEvent::HTTP is laoded for the first time it will query the
+default proxy from the operating system, currently by looking at
+C<$ENV{http_proxy>}.
 =item AnyEvent::HTTP::cookie_jar_expire $jar[, $session_end]
 Remove all cookies from the cookie jar that have been expired. If
 C<$session_end> is given and true, then additionally remove all session
 # initialise proxy from environment
 eval {
    set_proxy $ENV{http_proxy};
 };
+=head2 SHOWCASE
+This section contaisn some more elaborate "real-world" examples or code
+snippets.
+=head2 HTTP/1.1 FILE DOWNLOAD
+Downloading files with HTTP can be quite tricky, especially when something
+goes wrong and you want to resume.
+Here is a function that initiates and resumes a download. It uses the
+last modified time to check for file content changes, and works with many
+HTTP/1.0 servers as well, and usually falls back to a complete re-download
+on older servers.
+It calls the completion callback with either C<undef>, which means a
+nonretryable error occured, C<0> when the download was partial and should
+be retried, and C<1> if it was successful.
+   use AnyEvent::HTTP;
+   sub download($$$) {
+      my ($url, $file, $cb) = @_;
+      open my $fh, "+<", $file
+         or die "$file: $!";
+      my %hdr;
+      my $ofs = 0;
+      warn stat $fh;
+      warn -s _;
+      if (stat $fh and -s _) {
+         $ofs = -s _;
+         warn "-s is ", $ofs;#d#
+         $hdr{"if-unmodified-since"} = AnyEvent::HTTP::format_date +(stat _)[9];
+         $hdr{"range"} = "bytes=$ofs-";
+      }
+      http_get $url,
+         headers   => \%hdr,
+         on_header => sub {
+            my ($hdr) = @_;
+            if ($hdr->{Status} == 200 && $ofs) {
+               # resume failed
+               truncate $fh, $ofs = 0;
+            }
+            sysseek $fh, $ofs, 0;
+            1
+         },
+         on_body   => sub {
+            my ($data, $hdr) = @_;
+            if ($hdr->{Status} =~ /^2/) {
+               length $data == syswrite $fh, $data
+                  or return; # abort on write errors
+            }
+            1
+         },
+         sub {
+            my (undef, $hdr) = @_;
+            my $status = $hdr->{Status};
+            if (my $time = AnyEvent::HTTP::parse_date $hdr->{"last-modified"}) {
+               utime $fh, $time, $time;
+            }
+            if ($status == 200 || $status == 206 || $status == 416) {
+               # download ok || resume ok || file already fully downloaded
+               $cb->(1, $hdr);
+            } elsif ($status == 412) {
+               # file has changed while resuming, delete and retry
+               unlink $file;
+               $cb->(0, $hdr);
+            } elsif ($status == 500 or $status == 503 or $status =~ /^59/) {
+               # retry later
+               $cb->(0, $hdr);
+            } else {
+               $cb->(undef, $hdr);
+            }
+         }
+      ;
+   }
+   download "http://server/somelargefile", "/tmp/somelargefile", sub {
+      if ($_[0]) {
+         print "OK!\n";
+      } elsif (defined $_[0]) {
+         print "please retry later\n";
+      } else {
+         print "ERROR\n";
+      }
+   };
-=head2 SOCKS PROXIES
+=head3 SOCKS PROXIES
 Socks proxies are not directly supported by AnyEvent::HTTP. You can
 compile your perl to support socks, or use an external program such as
 F<socksify> (dante) or F<tsocks> to make your program use a socks proxy
 transparently.

Diff Legend

-–
+Removed lines
-+
+Added lines
-<
+Changed lines
->
+Changed lines

Comparing AnyEvent-HTTP/HTTP.pm (file contents): Revision 1.92 by root, Tue Jan 4 08:17:59 2011 UTC vs. Revision 1.104 by root, Thu Feb 24 15:09:03 2011 UTC

Diff Legend

Comparing AnyEvent-HTTP/HTTP.pm (file contents):
Revision 1.92 by root, Tue Jan 4 08:17:59 2011 UTC vs.
Revision 1.104 by root, Thu Feb 24 15:09:03 2011 UTC