ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/Coro/myhttpd/netgeo.pl
(Generate patch)

Comparing Coro/myhttpd/netgeo.pl (file contents):
Revision 1.4 by root, Tue Aug 28 02:30:49 2001 UTC vs.
Revision 1.15 by root, Wed May 7 19:00:43 2003 UTC

… …
6use Fcntl; 6use Fcntl;
7 7
8use Coro; 8use Coro;
9use Coro::Event; 9use Coro::Event;
10use Coro::Semaphore; 10use Coro::Semaphore;
11use Coro::SemaphoreSet;
11use Coro::Socket; 12use Coro::Socket;
13use Coro::Timer;
14
15use BerkeleyDB;
12 16
13$Event::DIED = sub { 17$Event::DIED = sub {
14 Event::verbose_exception_handler(@_); 18 Event::verbose_exception_handler(@_);
15 #Event::unloop_all(); 19 #Event::unloop_all();
16}; 20};
… …
60 my $timeout = 5; 64 my $timeout = 5;
61 65
62 while () { 66 while () {
63 my $fh = new Coro::Socket 67 my $fh = new Coro::Socket
64 PeerAddr => $self->ip, 68 PeerAddr => $self->ip,
65 PeerPort => 'whois', 69 PeerPort => $self->{port} || "whois",
66 Timeout => 30; 70 Timeout => 30;
67 if ($fh) { 71 if ($fh) {
68 print $fh "$query\n"; 72 print $fh "$query\n";
69 $fh->read($whois, 16*1024); # max 16k. whois stored 73 $fh->read($whois, 16*1024); # max 16k. whois stored
70 close $fh; 74 close $fh;
… …
80 $timeout *= 2; 84 $timeout *= 2;
81 $timeout = 1 if $timeout > 600; 85 $timeout = 1 if $timeout > 600;
82 } else { 86 } else {
83 last; 87 last;
84 } 88 }
89 } else {
90 # only retry once a minute
91 print STDERR "unable to connect to $self->{ip} ($self->{name}), retrying...\n";
92 Coro::Timer::sleep 300;
85 } 93 }
86 } 94 }
87 95
88 $netgeo::whois{$id} = $whois; 96 $netgeo::whois{$id} = $whois;
89 } 97 }
… …
164 $whois; 172 $whois;
165} 173}
166 174
167package Whois::RIPE; 175package Whois::RIPE;
168 176
177use Socket;
169use base Whois; 178use base Whois;
170 179
171sub sanitize { 180sub sanitize {
172 local $_ = $_[1]; 181 local $_ = $_[1];
182
173 s/^%.*\n//gm; 183 s/^%.*\n//gm;
174 s/^\n+//; 184 s/^\n+//;
175 s/\n*$/\n/; 185 s/\n*$/\n/;
186
187 s/^inetnum:\s+/*in: /gm;
188 s/^admin-c:\s+/*ac: /gm;
189 s/^tech-c:\s+/*tc: /gm;
190 s/^owner-c:\s+/*oc: /gm;
191 s/^country:\s+/*cy: /gm;
192 s/^phone:\s+/*ph: /gm;
193 s/^remarks:\s+/*rm: /gm;
194 s/^changed:\s+/*ch: /gm;
195 s/^created:\s+/*cr: /gm;
196 s/^address:\s+/*ad: /gm;
197 s/^status:\s+/*st: /gm;
198 s/^inetrev:\s+/*ir: /gm;
199 s/^nserver:\s+/*ns: /gm;
200
176 $_; 201 $_;
177} 202}
178 203
179sub ip_request { 204sub ip_request {
180 my ($self, $ip) = @_; 205 my ($self, $ip) = @_;
181 206
182 my $whois = $self->whois_request("-FSTin $ip"); 207 my $whois = $self->whois_request("$self->{rflags}$ip");
208
209 $whois =~ s{
210 (2[0-5][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])
211 (?:\.
212 (2[0-5][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])
213 (?:\.
214 (2[0-5][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])
215 (?:\.
216 (2[0-5][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])
217 )?
218 )?
219 )?
220 /
221 ([0-9]+)
222 }{
223 my $ip = inet_aton sprintf "%d.%d.%d.%d", $1, $2, $3, $4;
224 my $net = 1 << (31 - $5);
225 my $mask = inet_aton 2 ** 32 - $net;
226
227 my $ip1 = $ip & $mask;
228 my $ip2 = $ip1 | inet_aton $net * 2 - 1;
229 (inet_ntoa $ip1) . " - " . (inet_ntoa $ip2);
230 }gex;
183 231
184 $whois =~ /^\*in: 0\.0\.0\.0 - 255\.255\.255\.255/ 232 $whois =~ /^\*in: 0\.0\.0\.0 - 255\.255\.255\.255/
185 and return; 233 and return;
186 234
187 $whois =~ /^\*ac: XXX0/m # 192.0.0.0 235 $whois =~ /^\*ac: XXX0/m # 192.0.0.0
236 and return;
237
238 $whois =~ /^\*st: (?:ALLOCATED )?UNSPECIFIED/m
188 and return; 239 and return;
189 240
190 $whois =~ /^%ERROR:/m 241 $whois =~ /^%ERROR:/m
191 and return; 242 and return;
192 243
193 #while ($whois =~ s/^\*(?:ac|tc):\s+(\S+)\n//m) { 244 #while ($whois =~ s/^\*(?:ac|tc):\s+(\S+)\n//m) {
194 # $whois .= $self->whois_request("-FSTpn $1"); 245 # $whois .= $self->whois_request("-FSTpn $1");
195 #} 246 #}
196 247
248 #$whois =~ s/^\*(?:pn|nh|mb|ch|so|rz|ny|st|rm):.*\n//mg;
249
250 $whois =~ s/\n+$//;
251
252 $whois;
253}
254
255package Whois::RWHOIS;
256
257use base Whois;
258
259sub sanitize {
260 local $_ = $_[1];
261 s/^%referral\s+/referral:/gm;
262 s/^network://gm;
263 s/^%.*\n//gm;
264 s/^\n+//m;
265 s/\n*$/\n/m;
266
267 s/^(\S+):\s*/\L$1: /gm;
268 s/^ip-network-block:/*in:/gm;
269 s/^country-code:/*cy:/gm;
270 s/^tech-contact;i:/*tc:/gm;
271 s/^updated:/*ch:/gm;
272 s/^street-address:/*ad:/gm;
273 s/^org-name:/*rm:/gm;
274 s/^created:/*cr:/gm;
275
276 $_;
277}
278
279sub ip_request {
280 my ($self, $ip) = @_;
281
282 my $whois = $self->whois_request("$ip");
283
284 $whois =~ /^\*in: 0\.0\.0\.0 - 255\.255\.255\.255/
285 and return;
286
287 $whois =~ /^\*ac: XXX0/m # 192.0.0.0
288 and return;
289
290 $whois =~ /^%ERROR:/m
291 and return;
292
293 #while ($whois =~ s/^\*(?:ac|tc):\s+(\S+)\n//m) {
294 # $whois .= $self->whois_request("-FSTpn $1");
295 #}
296
197 $whois =~ s/^\*(?:pn|nh|mb|ch|so|rz|ny|st|rm):.*\n//mg; 297 $whois =~ s/^\*(?:pn|nh|mb|ch|so|rz|ny|st|rm):.*\n//mg;
198 298
199 $whois =~ s/\n+$//; 299 $whois =~ s/\n+$//;
200 300
201 $whois; 301 $whois;
202} 302}
203 303
204package netgeo; 304package netgeo;
205 305
306use Socket;
206use BerkeleyDB; 307use BerkeleyDB;
207use Socket;
208 308
209sub ip2int($) { 309sub ip2int($) {
210 unpack "N", inet_aton $_[0]; 310 unpack "N", inet_aton $_[0];
211} 311}
212 312
… …
214 inet_ntoa pack "N", $_[0]; 314 inet_ntoa pack "N", $_[0];
215} 315}
216 316
217our %WHOIS; 317our %WHOIS;
218 318
219$WHOIS{ARIN} = new Whois::ARIN ARIN => "whois.arin.net", maxjobs => 12; 319#$WHOIS{ARIN} = new Whois::ARIN ARIN => "whois.arin.net", port => 43, maxjobs => 12;
320$WHOIS{ARIN} = new Whois::RWHOIS ARIN => "rwhois.arin.net", port => 4321, maxjobs => 12;
220$WHOIS{RIPE} = new Whois::RIPE RIPE => "whois.ripe.net", maxjobs => 20; 321$WHOIS{RIPE} = new Whois::RIPE RIPE => "whois.ripe.net", port => 43, rflags => "-FTin ", maxjobs => 20;
322$WHOIS{APNIC} = new Whois::RIPE APNIC => "whois.apnic.net", port => 43, rflags => "-FTin ", maxjobs => 20;
221$WHOIS{APNIC} = new Whois::RIPE APNIC => "whois.apnic.net", maxjobs => 20; 323$WHOIS{LACNIC} = new Whois::RIPE LACNIC => "whois.lacnic.net", port => 43, maxjobs => 20;
324
325$whoislock = new Coro::SemaphoreSet;
222 326
223sub ip_request { 327sub ip_request {
224 my $ip = $_[0]; 328 my $ip = $_[0];
329
330 my $guard = $whoislock->guard($ip);
225 331
226 my $c = $iprange->db_cursor; 332 my $c = $iprange->db_cursor;
227 my $v; 333 my $v;
228 334
229 if (!$c->c_get((inet_aton $ip), $v, DB_SET_RANGE)) { 335 if (!$c->c_get((inet_aton $ip), $v, DB_SET_RANGE)) {
230 my ($ip0, $ip1, $whois) = split /\x0/, $v; 336 my ($ip0, $ip1, $whois) = split /\x0/, $v;
231 my $_ip = ip2int $ip; 337 my $_ip = ip2int $ip;
232 print "looked for $_ip, found $ip0, $ip1 ", length($v),"\n";
233 if ($ip0 <= $_ip && $_ip <= $ip1) { 338 if ($ip0 <= $_ip && $_ip <= $ip1) {
234 return $whois; 339 return $whois;
235 } 340 }
236 } 341 }
237 342
238 print "looked for $ip, ONLY found $v->[0], $v->[1]\n";
239
240 my ($arin, $ripe, $apnic); 343 my ($arin, $ripe, $apnic);
241 344
242 $whois = $WHOIS{APNIC}->ip_request($ip) 345 $whois = $WHOIS{RIPE}->ip_request($ip)
243 || $WHOIS{RIPE} ->ip_request($ip) 346 || $WHOIS{APNIC} ->ip_request($ip)
244 || $WHOIS{ARIN} ->ip_request($ip); 347 || $WHOIS{ARIN} ->ip_request($ip)
348# || $WHOIS{LACNIC}->ip_request($ip)
349 ;
245 350
246 $whois =~ /^\*in: ([0-9.]+)\s+-\s+([0-9.]+)\s*$/mi 351 $whois =~ /^\*in: ([0-9.]+)\s+-\s+([0-9.]+)\s*$/mi
247 or do { warn "$whois($ip): no addresses found\n", last }; 352 or do { warn "$whois($ip): no addresses found\n", last };
248 353
249 my ($ip0, $ip1) = ($1, $2); 354 my ($ip0, $ip1) = ($1, $2);
… …
256 $_ip = $_ip & 0xffffff00; 361 $_ip = $_ip & 0xffffff00;
257 $_ip0 = $_ip if $_ip0 < $_ip; 362 $_ip0 = $_ip if $_ip0 < $_ip;
258 $_ip1 = $_ip + 255 if $_ip1 > $_ip + 255; 363 $_ip1 = $_ip + 255 if $_ip1 > $_ip + 255;
259 } 364 }
260 365
261 print "setting entry ($_ip0, $_ip, $_ip1)\n";
262 $iprange->db_put((pack "N", $_ip1), (join "\x0", $_ip0, $_ip1, $whois)); 366 $iprange->db_put((pack "N", $_ip1), (join "\x0", $_ip0, $_ip1, $whois));
263 (tied %whois)->db_sync; 367 (tied %whois)->db_sync;
264 $iprange->db_sync; 368 $iprange->db_sync;
265 369
266 $whois; 370 $whois;
267} 371}
268 372
373if (0) {
374 #print ip_request "68.52.164.8"; # goof
375 #print "\n\n";
376 #print ip_request "200.202.220.222"; # lacnic
377 #print "\n\n";
378 #print ip_request "62.116.167.250";
379 #print "\n\n";
380 #print ip_request "133.11.128.254"; # jp
381 #print "\n\n";
382 print ip_request "80.131.153.93";
383 print "\n\n";
384}
269 385
3861;
270 387
388

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines