#!/opt/bin/perl

BEGIN { require "common.pl" }
our ($TXN, %stat, $db_env, $TPB, $db_info, $TODAY);

use common::sense;
use Compress::LZF;
use BDB;
use JSON::XS;
use HTML::Entities;
use Date::Parse;

#############################################################################

our $db_http    = table "http";
our $db_torrent = table "torrent";

#############################################################################

#db_compact $db_http, undef, undef, undef, undef, 0, undef;

my $c = $db_http->cursor;
my @del;

#open my $out, "| mbuffer -s4M -m64M | lzma -9 >tpb.json.lzma"
open my $out, "| mbuffer -s4M -m64M >tpb.json"
   or die "tpb.json: $!";

sub ashtml($) {
   local $_ = shift;

   s/^\s+//;
   s/\s+$//;
   s/\015//g;

   utf8::decode $_;

   $_
}

sub astext($) {
   local $_ = ashtml shift;

   #s%<a href="([^"]+).*?>.*?</a>%URL<$1>%g;
   #s%<br\s*/?>%\n%g;

   HTML::Entities::decode_entities $_;
   utf8::decode $_;

   $_
}

sub astime($) {
   str2time $_[0]
}

while () {
   db_c_get $c, my $key, my $data, BDB::NEXT;
   last if $!;

   syswrite STDOUT, "\r$key ";

   eval {
      for (dic_decompress 0, $data) {
         my @info;

         m%title="More from this category">(.*?)<%s or die "$_: no category";
         $info[0] = [split / &gt; /, $1];

         $info[1] = $key+0;

         m%<div id="title">(.*?)</div>%s or die "$_: no title";
         $info[2] = astext $1;

         m% href="magnet:\?xt=urn:btih:([0-9a-f]*)&dn% or die "$_: no magnet";
         $info[3] = $1;

         $info[4] = [m%<a href="/tag/([^"]+)%g];

         m%<dt>By:</dt>.*?<dd>.*?<a href="/user/([^/]+)/%s
            and $info[5] = astext $1;

         m%<dt>Uploaded:</dt>.*?<dd>(.*?)</dd>%s or die "$_: no uploaded";
         $info[6] = astime $1;

         m%<div class="nfo">\s*<pre>(.*?)</pre>%s or die "$_: no nfo";
         $info[7] = astext $1;

         my @comments;

         while (
            m%
               <div\sid="comment-\d+">
               <p\sclass="byline">
               \s*
               <a\shref="/user/([^/]+)/".*?\sat\s(....-..-..\s..:..\sCET):\s*</p>
               <div\sclass="comment">(.*?)</div>\s*</div>
            %gsx
         ) {
            my ($user, $date, $comment) = ($1, $2, $3);
            push @comments, [$user, astime $date, astext $comment];
         }

         $info[8] = \@comments;

         print $out JSON::XS::encode_json \@info, "\n";
      }
   };

   if ($@) {
      print STDOUT "$@";
      push @del, $key;
   }
}

undef $c;

if (@del) {
   for (@del) {
      iput $_, [0, 1];
      db_del $db_torrent, undef, $_;
      db_del $db_http   , undef, $_;
   }
}