| … | |
… | |
| 3 | BEGIN { require "common.pl" } |
3 | BEGIN { require "common.pl" } |
| 4 | our ($TXN, %stat, $db_env, $TPB, $db_info, $TODAY); |
4 | our ($TXN, %stat, $db_env, $TPB, $db_info, $TODAY); |
| 5 | |
5 | |
| 6 | use common::sense; |
6 | use common::sense; |
| 7 | |
7 | |
| 8 | our $NUM_DOWNLOADERS = 2; # minimum 2 |
8 | our $NUM_DOWNLOADERS = $ENV{DOWNLOADERS} || 4; # minimum 2 |
| 9 | |
9 | |
| 10 | our $db_http = table "http"; |
10 | our $db_http = table "http"; |
| 11 | our $db_torrent = table "torrent"; |
11 | our $db_torrent = table "torrent"; |
| 12 | |
12 | |
| 13 | our $TODO; |
13 | our $TODO; |
| … | |
… | |
| 27 | sub pool(&@) { |
27 | sub pool(&@) { |
| 28 | $pool->put ([@_]); |
28 | $pool->put ([@_]); |
| 29 | } |
29 | } |
| 30 | |
30 | |
| 31 | sub http_extract($) { |
31 | sub http_extract($) { |
| 32 | $_[0] =~ / id='NumComments'>(\d+)</ |
32 | $_[0] =~ / id=['"]NumComments['"]>(\d+)</ |
| 33 | or die "no numcomments in <$_[0]>\n"; |
33 | or die "no numcomments in <$_[0]>\n"; |
| 34 | $1 |
34 | $1 |
| 35 | } |
35 | } |
| 36 | |
36 | |
| 37 | my $finish = new Coro::Channel 100; |
37 | my $finish = new Coro::Channel 100; |
| … | |
… | |
| 73 | $stat{torrent} += length $torrent; |
73 | $stat{torrent} += length $torrent; |
| 74 | |
74 | |
| 75 | put $db_torrent, $i, $torrent if defined $torrent; |
75 | put $db_torrent, $i, $torrent if defined $torrent; |
| 76 | put $db_http, $i, $html; |
76 | put $db_http, $i, $html; |
| 77 | iput $i, [$TODAY, $info->[1]]; # check every day |
77 | iput $i, [$TODAY, $info->[1]]; # check every day |
|
|
78 | |
|
|
79 | $stat{cur} = $i; |
| 78 | |
80 | |
| 79 | # db_txn_finish $TXN; |
81 | # db_txn_finish $TXN; |
| 80 | # die "transaction failed" if $!; |
82 | # die "transaction failed" if $!; |
| 81 | |
83 | |
| 82 | remove $TODO $i; |
84 | remove $TODO $i; |
