#!/opt/bin/perl

BEGIN { require "common.pl" }
our ($TXN, %stat, $db_env, $TPB, $db_info, $TODAY);

use common::sense;

our $NUM_DOWNLOADERS = 2; # minimum 2

our $db_http    = table "http";
our $db_torrent = table "torrent";

our $TODO;

#############################################################################

my $pool = new Coro::Channel 10000;
my @pool = map {
   async {
      while (my $args = $pool->get) {
         my $cb = pop @$args;
         $cb->(@$args);
      }
   }
} 1..1;

sub pool(&@) {
   $pool->put ([@_]);
}

sub http_extract($) {
   $_[0] =~ / id='NumComments'>(\d+)</
      or die "no numcomments in <$_[0]>\n";
   $1
}

my $finish = new Coro::Channel 100;
my @finishers = async {
   {
      my $txn;
      $Coro::current->swap_sv (\$TXN, \$txn);
   }

   while (my $job = $finish->get) {
#      $TXN = $db_env->txn_begin;

      my ($i, $html, $torrent) = @$job;

      $html =~ s/^.*?Details for this torrent//s;
      $html =~ s/<div id="foot".*$//s;

      my $info = iget $i;

      my $old = get $db_http, $i;

      unless ($!) {
         my $old_html = dic_decompress 0, $old;

         if ((http_extract $html) ne eval { http_extract $old_html }) {
            ++$stat{change};
            $info->[1] = ($info->[1] >> 1) || 1;
         } else {
            ++$stat{same};
            $info->[1] *= 2;
            $info->[1] = 365 if $info->[1] > 365;
         }
      }

      $stat{data} += length $html;
      $html = dic_compress 0, $html if length $html;
      $stat{cmpr} += length $html;

      $stat{torrent} += length $torrent;

      put $db_torrent, $i, $torrent if defined $torrent;
      put $db_http, $i, $html;
      iput $i, [$TODAY, $info->[1]]; # check every day

#      db_txn_finish $TXN;
#      die "transaction failed" if $!;

      remove $TODO $i;
   }
};

#############################################################################

my $download = new Coro::Channel 200;
my @downloaders = map {
   async {
      while (my $i = $download->get) {
         my ($html, $hdr) = GET "https://$TPB/torrent/$i";

         if ($hdr->{Status} == 200) {
            BDB::db_exists $db_torrent, undef, $i, 0;
            if ($!) {
               if ($html =~ m%<a href="([^"]+)" title="Torrent File"%) {
                  my $url = $1;
                  $url =~ s/ /%20/g;
                  $url =~ s%^//%https://%;
                  $url =~ s%^/+%https://$TPB/%;
                  my ($torrent, $hdr) = GET $url;

                  if ($hdr->{Status} == 200) {
                     $finish->put ([$i, $html, $torrent]);
                  } elsif ($hdr->{Status} == 404) {
                     ++$stat{notorrent};
                     $finish->put ([$i, $html, undef]);

                     #warn "https://$TPB/torrent/$i $url\n";
                     #remove $TODO $i;
                  } else {
                     warn "$url https://$TPB/torrent/$i: status $hdr->{Status}\n";
                  }
               } else {
                  ++$stat{notorrentlink};
                  $finish->put ([$i, $html, undef]);
               }
            } else {
               $finish->put ([$i, $html, undef]);
            }
         } elsif ($hdr->{Status} == 404) {
            remove $TODO $i;
            del $db_info, $i;
         } else {
            warn "https://$TPB/torrent/$i: status $hdr->{Status}\n";
         }
      }
   }
} 1..$NUM_DOWNLOADERS;

#############################################################################

if (0) {
   #sput "last_recent", 7100000;
   sput "last_recent", 3210000;
}

if (0) {
   db_env_txn_checkpoint $db_env;
   db_compact $db_info;
   db_env_txn_checkpoint $db_env;
   return;
}

async {
   $TODO = new Set::IntSpan sget "todo";

   if (1) {
      my $last = sget "last_recent";

      my ($html, $hdr) = GET "https://$TPB/recent";

      $html =~ m%<div class="detName">\s*<a href="/torrent/(\d+)/%
         or die "https://$TPB/recent: unable to parse recent torrent ($html)";

      my $next = $1;
      #$next = 7043798;#d#

      $TODO->U ([[$last, $next]]);

      sput todo => "$TODO";
      sput last_recent => $next;
   }

   my $sputter = AE::timer 10, 10, sub {
      async_pool {
         sput todo => "$TODO";
      };
   };

   # new torrents
   if (1) {
      for (spans $TODO) {
         for my $i ($_->[0] .. $_->[1]) {
            $stat{stat} = $i;
            BDB::db_exists $db_http, undef, $i, 0;
            unless ($!) {
               BDB::db_exists $db_torrent, undef, $i, 0;
               unless ($!) {
                  # already have both
                  remove $TODO $i;
                  next;
               }
            }

            iput $i, [0, 1];
#            $download->put ($i);#d#
         }
      }
   }

   # refresh
   if (1) {
      my $c = $db_info->cursor (undef, 0);
      my ($k, $v);
      db_c_get $c, $k, $v, BDB::SET_RANGE;
      my @i;

      until ($!) {
         my $info = decode_cbor $v;
         push @i, $k
#          $download->put ($k)
            if $info->[0] + $info->[1] <= $TODAY;
         db_c_get $c, $k, $v, BDB::NEXT;
#d#         last if @i > 200000;#d#
      }

      undef $c;

      $download->put ($_)
         for @i;
   }

   $download->shutdown;
   $_->join
      for @downloaders;

   $finish->shutdown;
   $_->join for @finishers;

   $pool->shutdown;
   $_->join for @pool;

   sput todo => "$TODO";

   EV::unloop;
};

EV::loop;