Coro/myhttpd/netgeo.pl

#!/usr/bin/perl

# APNIC refer: KRNIC (for 211.104.0.0)

use Socket;
use Fcntl;

use Coro;
use Coro::Event;
use Coro::Semaphore;
use Coro::SemaphoreSet;
use Coro::Socket;

use BerkeleyDB;

$Event::DIED = sub {
   Event::verbose_exception_handler(@_);
   #Event::unloop_all();
};

tie %netgeo::whois, BerkeleyDB::Btree,
    -Env => $db_env,
    -Filename => "whois",
    -Flags => DB_CREATE,
       or die "unable to create/open whois table";
$netgeo::iprange = new BerkeleyDB::Btree
    -Env => $db_env,
    -Filename => "iprange",
    -Flags => DB_CREATE,
       or die "unable to create/open iprange table";

package Whois;

use Coro::Event;

sub new {
   my $class = shift;
   my $name = shift;
   my $ip = shift;
   my $self = bless { name => $name, ip => $ip, @_ }, $class;
   $self->{maxjobs} = new Coro::Semaphore $self->{maxjobs} || 1;
   $self;
}

sub ip {
   $_[0]{ip};
}

sub sanitize {
   $_[1];
}

sub whois_request {
   my ($self, $query) = @_;

   my $id = "$self->{name}\x0$query";
   my $whois = $netgeo::whois{$id};

   unless (defined $whois) {
      print "WHOIS($self->{name},$query)\n";

      my $guard = $self->{maxjobs}->guard;
      my $timeout = 5;

      while () {
         my $fh = new Coro::Socket
                         PeerAddr => $self->ip,
                         PeerPort => 'whois',
                         Timeout  => 30;
         if ($fh) {
            print $fh "$query\n";
            $fh->read($whois, 16*1024); # max 16k. whois stored
            close $fh;
            $whois =~ s/\015?\012/\n/g;
            $whois = $self->sanitize($whois);
            if ($whois eq ""
                or ($whois =~ /query limit/i && $whois =~ /exceeded/i) # ARIN
                or ($whois =~ /wait a while and try again/i) # ARIN
                or ($whois =~ /^%ERROR:202:/) # RIPE/APNIC
            ) {
               print "retrying in $timeout seconds\n";#d#
               do_timer(desc => "timer2", after => $timeout);
               $timeout *= 2;
               $timeout = 1 if $timeout > 600;
            } else {
               last;
            }
         }
      }

      $netgeo::whois{$id} = $whois;
   }

   $whois;
}

package Whois::ARIN;

use Date::Parse;

use base Whois;

sub sanitize {
   local $_ = $_[1];
   s/\n[\t ]{6,}([0-9.]+ - [0-9.]+)/ $1/g;
   $_;
}

# there are only two problems with arin's whois database:
# a) the data cannot be trusted and often is old or even wrong
# b) the database format is nonparsable
#    (no spaces between netname/ip and netnames can end in digits ;)
# of course, the only source to find out about global
# address distribution is... arin.
sub ip_request {
   my ($self, $ip) = @_;

   my $whois = $self->whois_request($ip);
   
   return () if $whois =~ /^No match/;

   if ($whois =~ /^To single out one record/m) {
      my $handle;
      while ($whois =~ /\G\S.*\(([A-Z0-9\-]+)\).*\n/mg) {
         $handle = $1;
         #return if $handle =~ /-(RIPE|APNIC)/; # heuristic, bbut bad because ripe might not have better info
      }
      $handle or die "$whois ($ip): unparseable multimatch\n";
      $whois = $self->whois_request("!$handle");
   }

   my ($address, $info, $coordinator, undef) = split /\n\n/, $whois;

   $info =~ /^\s+Netname: (\S+)$/mi
      or die "$whois($ip): no netname\n";
   my $netname = $1;

   $info =~ /^\s+Netblock: ([0-9.]+\s+-\s+[0-9.]+)\s*$/mi
      or die "$whois($ip): no netblock\n";
   my $netblock = $1;

   my $maintainer;

   if ($info =~ /^\s+Maintainer: (\S+)\s*$/mi) {
      $maintainer = "*ma: $1\n";
      return if $1 =~ /^(?:AP|RIPE)$/;
   }

   $coordinator =~ s/^\s+Coordinator:\s*//si
      or $coordinator = "";

   $address =~ s/\n\s*(\S+)$//
      or die "$whois($ip): no parseable country ($address)\n";
   my $country = $1;

   $address     =~ s/^\s*/*de: /mg;
   $coordinator =~ s/^\s*/*ad: /mg;

   $whois = <<EOF;
*in: $netblock
*na: $netname
*cy: $country
$maintainer$address
$coordinator
EOF
   $whois =~ s/\n+$//;
   $whois;
}

package Whois::RIPE;

use base Whois;

sub sanitize {
   local $_ = $_[1];
   s/^%.*\n//gm;
   s/^\n+//;
   s/\n*$/\n/;
   $_;
}

sub ip_request {
   my ($self, $ip) = @_;

   my $whois = $self->whois_request("-FSTin $ip");

   $whois =~ /^\*in: 0\.0\.0\.0 - 255\.255\.255\.255/
      and return;

   $whois =~  /^\*ac: XXX0/m # 192.0.0.0
      and return;

   $whois =~ /^%ERROR:/m
      and return;

   #while ($whois =~ s/^\*(?:ac|tc):\s+(\S+)\n//m) {
   #   $whois .= $self->whois_request("-FSTpn $1");
   #}

   $whois =~ s/^\*(?:pn|nh|mb|ch|so|rz|ny|st|rm):.*\n//mg;

   $whois =~ s/\n+$//;

   $whois;
}

package netgeo;

use Socket;
use BerkeleyDB;

sub ip2int($) {
   unpack "N", inet_aton $_[0];
}

sub int2ip($) {
   inet_ntoa pack "N", $_[0];
}

our %WHOIS;

$WHOIS{ARIN}  = new Whois::ARIN ARIN  => "whois.arin.net",  maxjobs => 12;
$WHOIS{RIPE}  = new Whois::RIPE RIPE  => "whois.ripe.net",  maxjobs => 20;
$WHOIS{APNIC} = new Whois::RIPE APNIC => "whois.apnic.net", maxjobs => 20;

$whoislock = new Coro::SemaphoreSet;

sub ip_request {
   my $ip = $_[0];

   my $guard = $whoislock->guard($ip);

   my $c = $iprange->db_cursor;
   my $v;

   if (!$c->c_get((inet_aton $ip), $v, DB_SET_RANGE)) {
      my ($ip0, $ip1, $whois) = split /\x0/, $v;
      my $_ip = ip2int $ip;
      if ($ip0 <= $_ip && $_ip <= $ip1) {
         return $whois;
      }
   }

   my ($arin, $ripe, $apnic);

   $whois = $WHOIS{RIPE}->ip_request($ip)
         || $WHOIS{ARIN} ->ip_request($ip)
         || $WHOIS{APNIC} ->ip_request($ip);

   $whois =~ /^\*in: ([0-9.]+)\s+-\s+([0-9.]+)\s*$/mi
      or do { warn "$whois($ip): no addresses found\n", last };

   my ($ip0, $ip1) = ($1, $2);

   my $_ip  = ip2int($ip);
   my $_ip0 = ip2int($ip0);
   my $_ip1 = ip2int($ip1);

   if ($_ip0 + 256 < $_ip1) {
      $_ip  = $_ip & 0xffffff00;
      $_ip0 = $_ip       if $_ip0 < $_ip;
      $_ip1 = $_ip + 255 if $_ip1 > $_ip + 255;
   }

   $iprange->db_put((pack "N", $_ip1), (join "\x0", $_ip0, $_ip1, $whois));
   (tied %whois)->db_sync;
   $iprange->db_sync;

   $whois;
}


Revision:	1.9
Committed:	Sun May 19 21:00:48 2002 UTC (24 years, 4 months ago) by root
Content type:	text/plain
Branch:	MAIN
Changes since 1.8:	+3 -3 lines
Log Message:	* empty log message *
#	User	Rev	Content
1	root	1.1	#!/usr/bin/perl
2
3			# APNIC refer: KRNIC (for 211.104.0.0)
4
5			use Socket;
6			use Fcntl;
7
8			use Coro;
9			use Coro::Event;
10			use Coro::Semaphore;
11	root	1.6	use Coro::SemaphoreSet;
12	root	1.1	use Coro::Socket;
13
14	root	1.7	use BerkeleyDB;
15
16	root	1.1	$Event::DIED = sub {
17			Event::verbose_exception_handler(@_);
18			#Event::unloop_all();
19			};
20
21	root	1.4	tie %netgeo::whois, BerkeleyDB::Btree,
22			-Env => $db_env,
23			-Filename => "whois",
24			-Flags => DB_CREATE,
25			or die "unable to create/open whois table";
26			$netgeo::iprange = new BerkeleyDB::Btree
27			-Env => $db_env,
28			-Filename => "iprange",
29			-Flags => DB_CREATE,
30			or die "unable to create/open iprange table";
31
32	root	1.1	package Whois;
33
34			use Coro::Event;
35
36			sub new {
37			my $class = shift;
38			my $name = shift;
39			my $ip = shift;
40			my $self = bless { name => $name, ip => $ip, @_ }, $class;
41			$self->{maxjobs} = new Coro::Semaphore $self->{maxjobs} \|\| 1;
42			$self;
43			}
44
45			sub ip {
46			$_[0]{ip};
47			}
48
49			sub sanitize {
50			$_[1];
51			}
52
53			sub whois_request {
54			my ($self, $query) = @_;
55
56	root	1.4	my $id = "$self->{name}\x0$query";
57			my $whois = $netgeo::whois{$id};
58
59			unless (defined $whois) {
60			print "WHOIS($self->{name},$query)\n";
61	root	1.1
62			my $guard = $self->{maxjobs}->guard;
63			my $timeout = 5;
64
65			while () {
66			my $fh = new Coro::Socket
67			PeerAddr => $self->ip,
68			PeerPort => 'whois',
69			Timeout => 30;
70			if ($fh) {
71			print $fh "$query\n";
72			$fh->read($whois, 16*1024); # max 16k. whois stored
73			close $fh;
74			$whois =~ s/\015?\012/\n/g;
75			$whois = $self->sanitize($whois);
76			if ($whois eq ""
77			or ($whois =~ /query limit/i && $whois =~ /exceeded/i) # ARIN
78			or ($whois =~ /wait a while and try again/i) # ARIN
79			or ($whois =~ /^%ERROR:202:/) # RIPE/APNIC
80			) {
81			print "retrying in $timeout seconds\n";#d#
82			do_timer(desc => "timer2", after => $timeout);
83			$timeout *= 2;
84			$timeout = 1 if $timeout > 600;
85			} else {
86			last;
87			}
88			}
89			}
90
91	root	1.4	$netgeo::whois{$id} = $whois;
92	root	1.1	}
93
94			$whois;
95			}
96
97			package Whois::ARIN;
98
99			use Date::Parse;
100
101			use base Whois;
102
103			sub sanitize {
104			local $_ = $_[1];
105			s/\n[\t ]{6,}([0-9.]+ - [0-9.]+)/ $1/g;
106			$_;
107			}
108
109			# there are only two problems with arin's whois database:
110			# a) the data cannot be trusted and often is old or even wrong
111			# b) the database format is nonparsable
112			# (no spaces between netname/ip and netnames can end in digits ;)
113			# of course, the only source to find out about global
114			# address distribution is... arin.
115			sub ip_request {
116			my ($self, $ip) = @_;
117
118			my $whois = $self->whois_request($ip);
119
120			return () if $whois =~ /^No match/;
121
122			if ($whois =~ /^To single out one record/m) {
123			my $handle;
124			while ($whois =~ /\G\S.\(([A-Z0-9\-]+)\).\n/mg) {
125			$handle = $1;
126			#return if $handle =~ /-(RIPE\|APNIC)/; # heuristic, bbut bad because ripe might not have better info
127			}
128			$handle or die "$whois ($ip): unparseable multimatch\n";
129			$whois = $self->whois_request("!$handle");
130			}
131
132			my ($address, $info, $coordinator, undef) = split /\n\n/, $whois;
133
134			$info =~ /^\s+Netname: (\S+)$/mi
135			or die "$whois($ip): no netname\n";
136			my $netname = $1;
137
138			$info =~ /^\s+Netblock: ([0-9.]+\s+-\s+[0-9.]+)\s*$/mi
139			or die "$whois($ip): no netblock\n";
140			my $netblock = $1;
141
142			my $maintainer;
143
144			if ($info =~ /^\s+Maintainer: (\S+)\s*$/mi) {
145			$maintainer = "*ma: $1\n";
146			return if $1 =~ /^(?:AP\|RIPE)$/;
147			}
148
149			$coordinator =~ s/^\s+Coordinator:\s*//si
150			or $coordinator = "";
151
152			$address =~ s/\n\s*(\S+)$//
153			or die "$whois($ip): no parseable country ($address)\n";
154			my $country = $1;
155
156			$address =~ s/^\s/de: /mg;
157			$coordinator =~ s/^\s/ad: /mg;
158
159			$whois = <<EOF;
160			*in: $netblock
161			*na: $netname
162			*cy: $country
163			$maintainer$address
164			$coordinator
165			EOF
166			$whois =~ s/\n+$//;
167			$whois;
168			}
169
170			package Whois::RIPE;
171
172			use base Whois;
173
174			sub sanitize {
175			local $_ = $_[1];
176			s/^%.*\n//gm;
177			s/^\n+//;
178			s/\n*$/\n/;
179			$_;
180			}
181
182			sub ip_request {
183			my ($self, $ip) = @_;
184
185			my $whois = $self->whois_request("-FSTin $ip");
186
187			$whois =~ /^\*in: 0\.0\.0\.0 - 255\.255\.255\.255/
188			and return;
189
190			$whois =~ /^\*ac: XXX0/m # 192.0.0.0
191			and return;
192
193			$whois =~ /^%ERROR:/m
194			and return;
195
196			#while ($whois =~ s/^\*(?:ac\|tc):\s+(\S+)\n//m) {
197			# $whois .= $self->whois_request("-FSTpn $1");
198			#}
199
200			$whois =~ s/^\(?:pn\|nh\|mb\|ch\|so\|rz\|ny\|st\|rm):.\n//mg;
201
202			$whois =~ s/\n+$//;
203
204			$whois;
205			}
206
207	root	1.4	package netgeo;
208
209			use Socket;
210	root	1.8	use BerkeleyDB;
211	root	1.1
212			sub ip2int($) {
213			unpack "N", inet_aton $_[0];
214			}
215
216			sub int2ip($) {
217			inet_ntoa pack "N", $_[0];
218			}
219
220			our %WHOIS;
221
222			$WHOIS{ARIN} = new Whois::ARIN ARIN => "whois.arin.net", maxjobs => 12;
223			$WHOIS{RIPE} = new Whois::RIPE RIPE => "whois.ripe.net", maxjobs => 20;
224			$WHOIS{APNIC} = new Whois::RIPE APNIC => "whois.apnic.net", maxjobs => 20;
225
226	root	1.6	$whoislock = new Coro::SemaphoreSet;
227
228	root	1.1	sub ip_request {
229			my $ip = $_[0];
230	root	1.6
231			my $guard = $whoislock->guard($ip);
232	root	1.1
233	root	1.4	my $c = $iprange->db_cursor;
234			my $v;
235
236			if (!$c->c_get((inet_aton $ip), $v, DB_SET_RANGE)) {
237			my ($ip0, $ip1, $whois) = split /\x0/, $v;
238			my $_ip = ip2int $ip;
239			if ($ip0 <= $_ip && $_ip <= $ip1) {
240			return $whois;
241	root	1.1	}
242	root	1.4	}
243
244			my ($arin, $ripe, $apnic);
245
246	root	1.9	$whois = $WHOIS{RIPE}->ip_request($ip)
247			\|\| $WHOIS{ARIN} ->ip_request($ip)
248			\|\| $WHOIS{APNIC} ->ip_request($ip);
249	root	1.1
250	root	1.4	$whois =~ /^\in: ([0-9.]+)\s+-\s+([0-9.]+)\s$/mi
251			or do { warn "$whois($ip): no addresses found\n", last };
252	root	1.1
253	root	1.4	my ($ip0, $ip1) = ($1, $2);
254
255			my $_ip = ip2int($ip);
256			my $_ip0 = ip2int($ip0);
257			my $_ip1 = ip2int($ip1);
258
259			if ($_ip0 + 256 < $_ip1) {
260			$_ip = $_ip & 0xffffff00;
261			$_ip0 = $_ip if $_ip0 < $_ip;
262			$_ip1 = $_ip + 255 if $_ip1 > $_ip + 255;
263	root	1.1	}
264	root	1.4
265			$iprange->db_put((pack "N", $_ip1), (join "\x0", $_ip0, $_ip1, $whois));
266			(tied %whois)->db_sync;
267			$iprange->db_sync;
268	root	1.1
269			$whois;
270			}
271
272
273