| 1 |
#!/opt/bin/perl |
| 2 |
|
| 3 |
BEGIN { require "common.pl" } |
| 4 |
|
| 5 |
our $NUM_DOWNLOADERS = 5; |
| 6 |
|
| 7 |
our $db_http = table "http"; |
| 8 |
our $db_torrent = table "torrent"; |
| 9 |
|
| 10 |
############################################################################# |
| 11 |
|
| 12 |
my $finish = new Coro::Channel 25; |
| 13 |
my @finishers = async { |
| 14 |
while (my $info = $finish->get) { |
| 15 |
my ($i, $data) = @$info; |
| 16 |
|
| 17 |
$stat{data} += length $data; |
| 18 |
|
| 19 |
db_put $db_torrent, undef, $i, $data, 0; |
| 20 |
} |
| 21 |
}; |
| 22 |
|
| 23 |
############################################################################# |
| 24 |
|
| 25 |
my $download = new Coro::Channel 200; |
| 26 |
my @downloaders = map { |
| 27 |
async { |
| 28 |
while (my $info = $download->get) { |
| 29 |
my ($i, $url) = @$info; |
| 30 |
my ($data, $hdr) = GET $url; |
| 31 |
|
| 32 |
if ($hdr->{Status} == 200) { |
| 33 |
$finish->put ([$i, $data]); |
| 34 |
} elsif ($hdr->{Status} == 404) { |
| 35 |
warn "$url: status $hdr->{Status}\n"; |
| 36 |
#$finish->put ([$i, undef]); |
| 37 |
} else { |
| 38 |
warn "$url: status $hdr->{Status}\n"; |
| 39 |
} |
| 40 |
} |
| 41 |
} |
| 42 |
} 1..$NUM_DOWNLOADERS; |
| 43 |
|
| 44 |
############################################################################# |
| 45 |
|
| 46 |
async { |
| 47 |
my $c = $db_http->cursor; |
| 48 |
|
| 49 |
while () { |
| 50 |
db_c_get $c, my $i, my $data, BDB::NEXT; |
| 51 |
last if $!; |
| 52 |
$stat{stat} = $i; |
| 53 |
$data = decompress $data; |
| 54 |
$data =~ m%<a href="([^"]+)" title="Torrent File"% or die "$i $data"; |
| 55 |
my $url = $1; |
| 56 |
BDB::db_get $db_torrent, undef, $i, my $data, 0; |
| 57 |
next unless $!; |
| 58 |
$download->put ([$i, $url]); |
| 59 |
}; |
| 60 |
undef $c; |
| 61 |
|
| 62 |
$download->shutdown; |
| 63 |
$_->join |
| 64 |
for @downloaders; |
| 65 |
|
| 66 |
$finish->shutdown; |
| 67 |
$_->join for @finishers; |
| 68 |
|
| 69 |
EV::unloop; |
| 70 |
}; |
| 71 |
|
| 72 |
EV::loop; |
| 73 |
|
