|
|
1 | package transferqueue; |
| 1 | |
2 | |
|
|
3 | my @reserve = ( |
|
|
4 | [ 1_200_000, 1], |
|
|
5 | [ 3_000_000, 1], |
|
|
6 | [ 75_000_000, 1], |
|
|
7 | ); |
|
|
8 | |
|
|
9 | sub new { |
|
|
10 | my $class = shift; |
|
|
11 | bless { |
|
|
12 | slots => $_[0], |
|
|
13 | lastspb => 0, |
|
|
14 | avgspb => 0, |
|
|
15 | }, $class; |
|
|
16 | } |
|
|
17 | |
|
|
18 | sub start_transfer { |
|
|
19 | my $self = shift; |
|
|
20 | my $size = $_[0]; |
|
|
21 | |
|
|
22 | my $transfer = bless { |
|
|
23 | queue => $self, |
|
|
24 | time => $::NOW, |
|
|
25 | size => $size, |
|
|
26 | coro => $Coro::current, |
|
|
27 | started => 0, |
|
|
28 | }, transfer::; |
|
|
29 | |
|
|
30 | push @{$self->{wait}}, $transfer; |
|
|
31 | |
|
|
32 | $self->wake_next; |
|
|
33 | |
|
|
34 | $transfer; |
|
|
35 | } |
|
|
36 | |
|
|
37 | sub sort { |
|
|
38 | my @queue = grep $_, @{$_[0]{wait}}; |
|
|
39 | |
|
|
40 | $_->{spb} = ($::NOW-$_->{time}) / ($_->{size} || 1) for @queue; |
|
|
41 | |
|
|
42 | $_[0]{wait} = [sort { $b->{spb} <=> $a->{spb} } @queue]; |
|
|
43 | |
|
|
44 | Scalar::Util::weaken $_ for @{$_[0]{wait}}; |
|
|
45 | } |
|
|
46 | |
|
|
47 | sub wake_next { |
|
|
48 | my $self = shift; |
|
|
49 | |
|
|
50 | $self->sort; |
|
|
51 | |
|
|
52 | while (@{$self->{wait}}) { |
|
|
53 | my $size = $self->{wait}[0]{size}; |
|
|
54 | my $min = 0; |
|
|
55 | for (@reserve) { |
|
|
56 | last if $size <= $_->[0]; |
|
|
57 | $min += $_->[1]; |
|
|
58 | } |
|
|
59 | last unless $self->{slots} > $min; |
|
|
60 | my $transfer = shift @{$self->{wait}}; |
|
|
61 | $self->{lastspb} = $transfer->{spb}; |
|
|
62 | $self->{avgspb} ||= $transfer->{spb}; |
|
|
63 | $self->{avgspb} = $self->{avgspb} * 0.95 + $transfer->{spb} * 0.05; |
|
|
64 | $self->{started}++; |
|
|
65 | $transfer->wake; |
|
|
66 | last; |
|
|
67 | } |
|
|
68 | } |
|
|
69 | |
|
|
70 | sub waiters { |
|
|
71 | $_[0]->sort; |
|
|
72 | @{$_[0]{wait}}; |
|
|
73 | } |
|
|
74 | |
|
|
75 | package transfer; |
|
|
76 | |
|
|
77 | use Coro::Timer (); |
|
|
78 | |
|
|
79 | sub wake { |
|
|
80 | my $self = shift; |
|
|
81 | |
|
|
82 | $self->{alloc} = 1; |
|
|
83 | $self->{queue}{slots}--; |
|
|
84 | $self->{wake} and $self->{wake}->ready; |
|
|
85 | } |
|
|
86 | |
|
|
87 | sub try { |
|
|
88 | my $self = shift; |
|
|
89 | |
|
|
90 | $self->{alloc} || do { |
|
|
91 | my $timeout = Coro::Timer::timeout $_[0]; |
|
|
92 | local $self->{wake} = $self->{coro}; |
|
|
93 | |
|
|
94 | Coro::schedule; |
|
|
95 | |
|
|
96 | $self->{alloc}; |
|
|
97 | } |
|
|
98 | } |
|
|
99 | |
|
|
100 | sub DESTROY { |
|
|
101 | my $self = shift; |
|
|
102 | |
|
|
103 | if ($self->{alloc}) { |
|
|
104 | $self->{queue}{slots}++; |
|
|
105 | $self->{queue}->wake_next; |
|
|
106 | } |
|
|
107 | } |
|
|
108 | |
|
|
109 | package conn; |
|
|
110 | |
| 2 | our @blockuri; |
111 | our %blockuri; |
| 3 | our @blockref; |
112 | our $blockref; |
| 4 | |
113 | |
| 5 | sub read_blockuri { |
114 | sub read_blockuri { |
| 6 | local *B; |
115 | local *B; |
| 7 | my %group; |
116 | my %group; |
| 8 | @blockuri = (); |
117 | %blockuri = (); |
| 9 | if (open B, "<blockuri") { |
118 | if (open B, "<blockuri") { |
| 10 | while (<B>) { |
119 | while (<B>) { |
| 11 | chomp; |
120 | chomp; |
| 12 | if (/^group\s+(\S+)\s+(.*)/i) { |
121 | if (/^group\s+(\S+)\s+(.*)/i) { |
| 13 | $group{$1} = [split /\s+/, $2]; |
122 | $group{$1} = [split /\s+/, $2]; |
| … | |
… | |
| 16 | my @r; |
125 | my @r; |
| 17 | for (split /\s+/, $2) { |
126 | for (split /\s+/, $2) { |
| 18 | push @r, $group{$_} ? @{$group{$_}} : $_; |
127 | push @r, $group{$_} ? @{$group{$_}} : $_; |
| 19 | } |
128 | } |
| 20 | print "not($g) => (@r)\n"; |
129 | print "not($g) => (@r)\n"; |
|
|
130 | push @{$blockuri{$_}}, $g for @r; |
| 21 | push @blockuri, [qr/$g/i, \@r]; |
131 | push @blockuri, [qr/$g/i, \@r]; |
| 22 | } elsif (/\S/) { |
132 | } elsif (/\S/) { |
| 23 | print "blockuri: unparsable line: $_\n"; |
133 | print "blockuri: unparsable line: $_\n"; |
| 24 | } |
134 | } |
|
|
135 | } |
|
|
136 | for (keys %blockuri) { |
|
|
137 | my $qr = join ")|(?:", @{$blockuri{$_}}; |
|
|
138 | $blockuri{$_} = qr{(?:$qr)}i; |
| 25 | } |
139 | } |
| 26 | } else { |
140 | } else { |
| 27 | print "no blockuri\n"; |
141 | print "no blockuri\n"; |
| 28 | } |
142 | } |
| 29 | } |
143 | } |
| 30 | |
144 | |
| 31 | sub read_blockref { |
145 | sub read_blockref { |
| 32 | local *B; |
146 | local *B; |
| 33 | @blockref = (); |
147 | my @blockref; |
| 34 | if (open B, "<blockreferer") { |
148 | if (open B, "<blockreferer") { |
| 35 | while (<B>) { |
149 | while (<B>) { |
| 36 | chomp; |
150 | chomp; |
| 37 | if (/^([^\t]*)\t\s*(.*)/) { |
151 | if (/^([^\t]*)\t\s*(.*)/) { |
| 38 | push @blockref, qr/^$1/i; |
152 | push @blockref, $1; |
| 39 | } elsif (/\S/) { |
153 | } elsif (/\S/) { |
| 40 | print "blockref: unparsable line: $_\n"; |
154 | print "blockref: unparsable line: $_\n"; |
| 41 | } |
155 | } |
| 42 | } |
156 | } |
|
|
157 | $blockref = join ")|(?:", @blockref; |
|
|
158 | $blockref = qr{^(?:$blockref)}i; |
| 43 | } else { |
159 | } else { |
| 44 | print "no blockref\n"; |
160 | print "no blockref\n"; |
|
|
161 | $blockref = qr{^x^}; |
| 45 | } |
162 | } |
| 46 | } |
163 | } |
| 47 | |
164 | |
| 48 | read_blockuri; |
165 | read_blockuri; |
| 49 | read_blockref; |
166 | read_blockref; |
| 50 | |
167 | |
| 51 | use Tie::Cache; |
168 | use Tie::Cache; |
| 52 | tie %whois_cache, Tie::Cache::, $MAX_CONNECTS * 1.5; |
169 | tie %whois_cache, Tie::Cache::, 32; |
| 53 | |
170 | |
| 54 | sub conn::err_block_country { |
|
|
| 55 | my $self = shift; |
|
|
| 56 | my $whois = shift; |
|
|
| 57 | |
|
|
| 58 | $whois =~ s/&/&/g; |
|
|
| 59 | $whois =~ s/</</g; |
|
|
| 60 | $self->err(403, "forbidden", { "Content-Type" => "text/html", Connection => "close" }, <<EOF); |
|
|
| 61 | <html> |
|
|
| 62 | <head> |
|
|
| 63 | <title>This material is licensed in your country!</title> |
|
|
| 64 | </head> |
|
|
| 65 | <body bgcolor="#ffffff" text="#000000" link="#0000ff" vlink="#000080" alink="#ff0000"> |
|
|
| 66 | |
|
|
| 67 | <h1>This material is licensed in your country!</h1> |
|
|
| 68 | |
|
|
| 69 | <p>My research has shown that your IP address |
|
|
| 70 | (<b>$self->{remote_addr}</b>) most probably is located in this country: |
|
|
| 71 | <b>$self->{country}</b> (ISO-3166-2 code, XX == unknown). The full record is:</p> |
|
|
| 72 | |
|
|
| 73 | <pre> |
|
|
| 74 | $whois |
|
|
| 75 | </pre> |
|
|
| 76 | |
|
|
| 77 | <p>My database says that the material you are trying to access is licensed |
|
|
| 78 | in your country. If I would distribute these files to your country I would |
|
|
| 79 | actively <em>hurt</em> the industry behind it, which includes the artists |
|
|
| 80 | and authors of these videos/mangas. So I hope you understand that I try to |
|
|
| 81 | avoid this.</p> |
|
|
| 82 | |
|
|
| 83 | <p>Please see the <a href="http://www.goof.com/pcg/marc/animefaq.html#licensed">FAQ</a> |
|
|
| 84 | for a more thorough explanation.</p> |
|
|
| 85 | |
|
|
| 86 | <p>If you <em>really</em> think that this is wrong, i.e. the |
|
|
| 87 | material you tried to access is <em>not</em> licensed in your |
|
|
| 88 | country or your ip address was misdetected, you can write to <a |
|
|
| 89 | href="mailto:licensed\@plan9.de">licensed\@plan9.de</a>. Please explain |
|
|
| 90 | what happened and why you think this is wrong in as much detail as |
|
|
| 91 | possible.</p> |
|
|
| 92 | |
|
|
| 93 | <div align="right">Thanks a lot for understanding.</div> |
|
|
| 94 | |
|
|
| 95 | </body> |
|
|
| 96 | </html> |
|
|
| 97 | EOF |
|
|
| 98 | } |
|
|
| 99 | |
|
|
| 100 | sub conn::err_block_referer { |
|
|
| 101 | my $self = shift; |
|
|
| 102 | |
|
|
| 103 | my $uri = $self->{uri}; |
|
|
| 104 | $uri =~ s/\/[^\/]+$/\//; |
|
|
| 105 | |
|
|
| 106 | $self->slog(6, "REFERER($self->{uri},$self->{h}{referer})"); |
|
|
| 107 | |
|
|
| 108 | $whois =~ s/&/&/g; |
|
|
| 109 | $whois =~ s/</</g; |
|
|
| 110 | $self->err(203, "non-authoritative", { "Content-Type" => "text/html" }, <<EOF); |
|
|
| 111 | <html> |
|
|
| 112 | <head> |
|
|
| 113 | <title>Unallowed Referral</title> |
|
|
| 114 | </head> |
|
|
| 115 | <body bgcolor="#ffffff" text="#000000" link="#0000ff" vlink="#000080" alink="#ff0000"> |
|
|
| 116 | |
|
|
| 117 | <h1>The site which referred you has done something bad!</h1> |
|
|
| 118 | |
|
|
| 119 | <p>It seems that you are coming from this URL:</p> |
|
|
| 120 | |
|
|
| 121 | <pre>$self->{h}{referer}</pre> |
|
|
| 122 | |
|
|
| 123 | <p>This site has been blocked, either because it required you to pay |
|
|
| 124 | money, forced you to click on banners, claimed these files were theirs |
|
|
| 125 | or something very similar. Please note that you can download these files |
|
|
| 126 | <em>without</em> having to pay, <em>without</em> clicking banners or jump |
|
|
| 127 | through other hoops.</p> |
|
|
| 128 | |
|
|
| 129 | <p><b>Sites like the one you came from actively hurt the distribution of |
|
|
| 130 | these files and the service quality for you since I can't move or correct |
|
|
| 131 | files and you will likely not be able to see the full archive.</b></p> |
|
|
| 132 | |
|
|
| 133 | <p>Having that this, you can find the original content (if it is still |
|
|
| 134 | there) by <b>following <a href="$uri">this link</a>.</b></p> |
|
|
| 135 | |
|
|
| 136 | <div align="right">Thanks a lot for understanding.</div> |
|
|
| 137 | |
|
|
| 138 | </body> |
|
|
| 139 | </html> |
|
|
| 140 | EOF |
|
|
| 141 | } |
|
|
| 142 | |
|
|
| 143 | sub conn::access_check { |
171 | sub access_check { |
| 144 | my $self = shift; |
172 | my $self = shift; |
| 145 | |
173 | |
| 146 | my $ref = $self->{h}{referer}; |
174 | my $ref = $self->{h}{referer}; |
| 147 | my $uri = $self->{path}; |
175 | my $uri = $self->{path}; |
| 148 | my %disallow; |
176 | my %disallow; |
| 149 | |
177 | |
| 150 | for (@blockref) { |
|
|
| 151 | $self->err_block_referer if $ref =~ $_; |
178 | $self->err_block_referer |
| 152 | } |
179 | if $self->{h}{referer} =~ $blockref; |
| 153 | |
180 | |
| 154 | for (@blockuri) { |
|
|
| 155 | if ($uri =~ $_->[0]) { |
|
|
| 156 | $disallow{$_}++ for @{$_->[1]}; |
|
|
| 157 | } |
|
|
| 158 | } |
|
|
| 159 | |
|
|
| 160 | my $whois = $whois_cache{$self->{remote_addr}} |
181 | my $whois = $whois_cache{$self->{remote_addr}} |
| 161 | ||= ::ip_request($self->{remote_addr}); |
182 | ||= netgeo::ip_request($self->{remote_addr}); |
| 162 | |
183 | |
| 163 | my $country = "XX"; |
184 | my $country = "XX"; |
| 164 | |
185 | |
| 165 | if ($whois =~ /^\*cy: (\S+)/m) { |
186 | if ($whois =~ /^\*cy: (\S+)/m) { |
| 166 | $country = uc $1; |
187 | $country = uc $1; |
| … | |
… | |
| 168 | $self->slog(9, "no country($whois)"); |
189 | $self->slog(9, "no country($whois)"); |
| 169 | } |
190 | } |
| 170 | |
191 | |
| 171 | $self->{country} = $country; |
192 | $self->{country} = $country; |
| 172 | |
193 | |
| 173 | if ($disallow{$country}) { |
|
|
| 174 | $self->err_block_country($whois); |
194 | $self->err_block_country($whois) |
| 175 | } |
195 | if $self->{path} =~ $blockuri{$country}; |
| 176 | } |
196 | } |
| 177 | |
197 | |
| 178 | 1; |
198 | 1; |
