ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/Coro/myhttpd/netgeo.pl
(Generate patch)

Comparing Coro/myhttpd/netgeo.pl (file contents):
Revision 1.8 by root, Fri Sep 28 16:35:00 2001 UTC vs.
Revision 1.16 by root, Sat Sep 17 20:21:11 2005 UTC

… …
8use Coro; 8use Coro;
9use Coro::Event; 9use Coro::Event;
10use Coro::Semaphore; 10use Coro::Semaphore;
11use Coro::SemaphoreSet; 11use Coro::SemaphoreSet;
12use Coro::Socket; 12use Coro::Socket;
13use Coro::Timer;
13 14
14use BerkeleyDB; 15use BerkeleyDB;
15 16
16$Event::DIED = sub { 17$Event::DIED = sub {
17 Event::verbose_exception_handler(@_); 18 Event::verbose_exception_handler(@_);
… …
63 my $timeout = 5; 64 my $timeout = 5;
64 65
65 while () { 66 while () {
66 my $fh = new Coro::Socket 67 my $fh = new Coro::Socket
67 PeerAddr => $self->ip, 68 PeerAddr => $self->ip,
68 PeerPort => 'whois', 69 PeerPort => $self->{port} || "whois",
69 Timeout => 30; 70 Timeout => 30;
70 if ($fh) { 71 if ($fh) {
71 print $fh "$query\n"; 72 print $fh "$query\n";
72 $fh->read($whois, 16*1024); # max 16k. whois stored 73 $fh->read($whois, 16*1024); # max 16k. whois stored
73 close $fh; 74 close $fh;
… …
83 $timeout *= 2; 84 $timeout *= 2;
84 $timeout = 1 if $timeout > 600; 85 $timeout = 1 if $timeout > 600;
85 } else { 86 } else {
86 last; 87 last;
87 } 88 }
89 } else {
90 # only retry once a minute
91 print STDERR "unable to connect to $self->{ip} ($self->{name}), retrying...\n";
92 Coro::Timer::sleep 300;
88 } 93 }
89 } 94 }
90 95
91 $netgeo::whois{$id} = $whois; 96 $netgeo::whois{$id} = $whois;
92 } 97 }
… …
115sub ip_request { 120sub ip_request {
116 my ($self, $ip) = @_; 121 my ($self, $ip) = @_;
117 122
118 my $whois = $self->whois_request($ip); 123 my $whois = $self->whois_request($ip);
119 124
120 return () if $whois =~ /^No match/; 125 return if $whois =~ /^No match/;
126 return if $whois =~ /^\*de: This network range is not allocated to /; # APINIC e.g. 24.0.0.0
121 127
122 if ($whois =~ /^To single out one record/m) { 128 if ($whois =~ /^To single out one record/m) {
123 my $handle; 129 my $handle;
124 while ($whois =~ /\G\S.*\(([A-Z0-9\-]+)\).*\n/mg) { 130 while ($whois =~ /\G\S.*\(([A-Z0-9\-]+)\).*\n/mg) {
125 $handle = $1; 131 $handle = $1;
126 #return if $handle =~ /-(RIPE|APNIC)/; # heuristic, bbut bad because ripe might not have better info 132 #return if $handle =~ /-(RIPE|APNIC)/; # heuristic, but bad because ripe might not have better info
127 } 133 }
128 $handle or die "$whois ($ip): unparseable multimatch\n"; 134 $handle or die "$whois ($ip): unparseable multimatch\n";
129 $whois = $self->whois_request("!$handle"); 135 $whois = $self->whois_request("!$handle");
130 } 136 }
131 137
… …
167 $whois; 173 $whois;
168} 174}
169 175
170package Whois::RIPE; 176package Whois::RIPE;
171 177
178use Socket;
172use base Whois; 179use base Whois;
173 180
174sub sanitize { 181sub sanitize {
175 local $_ = $_[1]; 182 local $_ = $_[1];
183
176 s/^%.*\n//gm; 184 s/^%.*\n//gm;
177 s/^\n+//; 185 s/^\n+//;
178 s/\n*$/\n/; 186 s/\n*$/\n/;
187
188 s/^inetnum:\s+/*in: /gm;
189 s/^admin-c:\s+/*ac: /gm;
190 s/^tech-c:\s+/*tc: /gm;
191 s/^owner-c:\s+/*oc: /gm;
192 s/^country:\s+/*cy: /gm;
193 s/^phone:\s+/*ph: /gm;
194 s/^remarks:\s+/*rm: /gm;
195 s/^changed:\s+/*ch: /gm;
196 s/^created:\s+/*cr: /gm;
197 s/^address:\s+/*ad: /gm;
198 s/^status:\s+/*st: /gm;
199 s/^inetrev:\s+/*ir: /gm;
200 s/^nserver:\s+/*ns: /gm;
201
179 $_; 202 $_;
180} 203}
181 204
182sub ip_request { 205sub ip_request {
183 my ($self, $ip) = @_; 206 my ($self, $ip) = @_;
184 207
185 my $whois = $self->whois_request("-FSTin $ip"); 208 my $whois = $self->whois_request("$self->{rflags}$ip");
209
210 $whois =~ s{
211 (2[0-5][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])
212 (?:\.
213 (2[0-5][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])
214 (?:\.
215 (2[0-5][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])
216 (?:\.
217 (2[0-5][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])
218 )?
219 )?
220 )?
221 /
222 ([0-9]+)
223 }{
224 my $ip = inet_aton sprintf "%d.%d.%d.%d", $1, $2, $3, $4;
225 my $net = 1 << (31 - $5);
226 my $mask = inet_aton 2 ** 32 - $net;
227
228 my $ip1 = $ip & $mask;
229 my $ip2 = $ip1 | inet_aton $net * 2 - 1;
230 (inet_ntoa $ip1) . " - " . (inet_ntoa $ip2);
231 }gex;
186 232
187 $whois =~ /^\*in: 0\.0\.0\.0 - 255\.255\.255\.255/ 233 $whois =~ /^\*in: 0\.0\.0\.0 - 255\.255\.255\.255/
188 and return; 234 and return;
189 235
190 $whois =~ /^\*ac: XXX0/m # 192.0.0.0 236 $whois =~ /^\*ac: XXX0/m # 192.0.0.0
237 and return;
238
239 $whois =~ /^\*st: (?:ALLOCATED )?UNSPECIFIED/m
191 and return; 240 and return;
192 241
193 $whois =~ /^%ERROR:/m 242 $whois =~ /^%ERROR:/m
194 and return; 243 and return;
195 244
196 #while ($whois =~ s/^\*(?:ac|tc):\s+(\S+)\n//m) { 245 #while ($whois =~ s/^\*(?:ac|tc):\s+(\S+)\n//m) {
197 # $whois .= $self->whois_request("-FSTpn $1"); 246 # $whois .= $self->whois_request("-FSTpn $1");
198 #} 247 #}
199 248
249 #$whois =~ s/^\*(?:pn|nh|mb|ch|so|rz|ny|st|rm):.*\n//mg;
250
251 $whois =~ s/\n+$//;
252
253 $whois;
254}
255
256package Whois::RWHOIS;
257
258use base Whois;
259
260sub sanitize {
261 local $_ = $_[1];
262 s/^%referral\s+/referral:/gm;
263 s/^network://gm;
264 s/^%.*\n//gm;
265 s/^\n+//m;
266 s/\n*$/\n/m;
267
268 s/^(\S+):\s*/\L$1: /gm;
269 s/^ip-network-block:/*in:/gm;
270 s/^country-code:/*cy:/gm;
271 s/^tech-contact;i:/*tc:/gm;
272 s/^updated:/*ch:/gm;
273 s/^street-address:/*ad:/gm;
274 s/^org-name:/*rm:/gm;
275 s/^created:/*cr:/gm;
276
277 $_;
278}
279
280sub ip_request {
281 my ($self, $ip) = @_;
282
283 my $whois = $self->whois_request("$ip");
284
285 $whois =~ /^\*in: 0\.0\.0\.0 - 255\.255\.255\.255/
286 and return;
287
288 $whois =~ /^\*ac: XXX0/m # 192.0.0.0
289 and return;
290
291 $whois =~ /^%ERROR:/m
292 and return;
293
294 #while ($whois =~ s/^\*(?:ac|tc):\s+(\S+)\n//m) {
295 # $whois .= $self->whois_request("-FSTpn $1");
296 #}
297
200 $whois =~ s/^\*(?:pn|nh|mb|ch|so|rz|ny|st|rm):.*\n//mg; 298 $whois =~ s/^\*(?:pn|nh|mb|ch|so|rz|ny|st|rm):.*\n//mg;
201 299
202 $whois =~ s/\n+$//; 300 $whois =~ s/\n+$//;
203 301
204 $whois; 302 $whois;
… …
217 inet_ntoa pack "N", $_[0]; 315 inet_ntoa pack "N", $_[0];
218} 316}
219 317
220our %WHOIS; 318our %WHOIS;
221 319
222$WHOIS{ARIN} = new Whois::ARIN ARIN => "whois.arin.net", maxjobs => 12; 320#$WHOIS{ARIN} = new Whois::ARIN ARIN => "whois.arin.net", port => 43, maxjobs => 12;
321$WHOIS{ARIN} = new Whois::RWHOIS ARIN => "rwhois.arin.net", port => 4321, maxjobs => 12;
223$WHOIS{RIPE} = new Whois::RIPE RIPE => "whois.ripe.net", maxjobs => 20; 322$WHOIS{RIPE} = new Whois::RIPE RIPE => "whois.ripe.net", port => 43, rflags => "-FTin ", maxjobs => 20;
323$WHOIS{APNIC} = new Whois::RIPE APNIC => "whois.apnic.net", port => 43, rflags => "-FTin ", maxjobs => 20;
224$WHOIS{APNIC} = new Whois::RIPE APNIC => "whois.apnic.net", maxjobs => 20; 324$WHOIS{LACNIC} = new Whois::RIPE LACNIC => "whois.lacnic.net", port => 43, maxjobs => 20;
225 325
226$whoislock = new Coro::SemaphoreSet; 326$whoislock = new Coro::SemaphoreSet;
227 327
228sub ip_request { 328sub ip_request {
229 my $ip = $_[0]; 329 my $ip = $_[0];
… …
241 } 341 }
242 } 342 }
243 343
244 my ($arin, $ripe, $apnic); 344 my ($arin, $ripe, $apnic);
245 345
246 $whois = $WHOIS{APNIC}->ip_request($ip) 346 $whois = $WHOIS{RIPE}->ip_request($ip)
247 || $WHOIS{RIPE} ->ip_request($ip) 347 || $WHOIS{APNIC} ->ip_request($ip)
248 || $WHOIS{ARIN} ->ip_request($ip); 348 || $WHOIS{ARIN} ->ip_request($ip)
349# || $WHOIS{LACNIC}->ip_request($ip)
350 ;
249 351
250 $whois =~ /^\*in: ([0-9.]+)\s+-\s+([0-9.]+)\s*$/mi 352 $whois =~ /^\*in: ([0-9.]+)\s+-\s+([0-9.]+)\s*$/mi
251 or do { warn "$whois($ip): no addresses found\n", last }; 353 or do { warn "$whois($ip): no addresses found\n", last };
252 354
253 my ($ip0, $ip1) = ($1, $2); 355 my ($ip0, $ip1) = ($1, $2);
… …
267 $iprange->db_sync; 369 $iprange->db_sync;
268 370
269 $whois; 371 $whois;
270} 372}
271 373
374if (0) {
375 #print ip_request "68.52.164.8"; # goof
376 #print "\n\n";
377 #print ip_request "200.202.220.222"; # lacnic
378 #print "\n\n";
379 #print ip_request "62.116.167.250";
380 #print "\n\n";
381 #print ip_request "133.11.128.254"; # jp
382 #print "\n\n";
383 print ip_request "80.131.153.93";
384 print "\n\n";
385}
272 386
3871;
273 388
389

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines