ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/deliantra/Deliantra-Client/DC/DB.pm
(Generate patch)

Comparing deliantra/Deliantra-Client/DC/DB.pm (file contents):
Revision 1.3 by root, Fri Apr 6 21:53:56 2007 UTC vs.
Revision 1.23 by root, Fri Nov 16 04:54:25 2007 UTC

1=head1 NAME 1=head1 NAME
2 2
3CFPlus::DB - async. database access for cfplus 3CFPlus::DB - async. database and filesystem access for cfplus
4 4
5=head1 SYNOPSIS 5=head1 SYNOPSIS
6 6
7 use CFPlus::DB; 7 use CFPlus::DB;
8 8
… …
16 16
17use strict; 17use strict;
18use utf8; 18use utf8;
19 19
20use Carp (); 20use Carp ();
21use AnyEvent ();
22use Storable (); # finally 21use Storable ();
22use Config;
23 23
24use CFPlus; 24use CFPlus;
25
26our $DB_HOME = "$Crossfire::VARDIR/cfplus-$BerkeleyDB::db_version-$Config{archname}";
27
28sub path_of_res($) {
29 utf8::downgrade $_[0]; # bug in unpack "H*"
30 "$DB_HOME/res-data-" . unpack "H*", $_[0]
31}
25 32
26sub sync { 33sub sync {
27 # for debugging 34 # for debugging
28 #CFPlus::DB::Server::req (sync => sub { }); 35 #CFPlus::DB::Server::req (sync => sub { });
29 CFPlus::DB::Server::sync (); 36 CFPlus::DB::Server::sync ();
30} 37}
31 38
39sub exists($$$) {
40 CFPlus::DB::Server::req (exists => @_);
41}
42
32sub get($$$) { 43sub get($$$) {
33 CFPlus::DB::Server::req (get => @_); 44 CFPlus::DB::Server::req (get => @_);
34} 45}
35 46
36sub put($$$$) { 47sub put($$$$) {
37 CFPlus::DB::Server::req (put => @_); 48 CFPlus::DB::Server::req (put => @_);
38} 49}
39 50
51sub unlink($$) {
52 CFPlus::DB::Server::req (unlink => @_);
53}
54
55sub read_file($$) {
56 CFPlus::DB::Server::req (read_file => @_);
57}
58
59sub write_file($$$) {
60 CFPlus::DB::Server::req (write_file => @_);
61}
62
63sub prefetch_file($$$) {
64 CFPlus::DB::Server::req (prefetch_file => @_);
65}
66
67sub logprint($$$) {
68 CFPlus::DB::Server::req (logprint => @_);
69}
70
40our $tilemap; 71our $tilemap;
41 72
42sub get_tile_id_sync($) { 73sub get_tile_id_sync($) {
43 my ($hash) = @_; 74 my ($name) = @_;
44 75
45 # fetch the full face table first 76 # fetch the full face table first
46 unless ($tilemap) { 77 unless ($tilemap) {
47 CFPlus::DB::Server::req (table => facemap => sub { $tilemap = $_[0] }); 78 CFPlus::DB::Server::req (table => facemap => sub {
79 $tilemap = $_[0];
80 delete $tilemap->{id};
81 my %maptile = reverse %$tilemap;#d#
82 if ((scalar keys %$tilemap) != (scalar keys %maptile)) {#d#
83 $tilemap = { };#d#
84 CFPlus::error "FATAL: facemap is not a 1:1 mapping, please report this and delete your $DB_HOME directory!\n";#d#
85 }#d#
86 });
48 sync; 87 sync;
49 } 88 }
50 89
51 $tilemap->{$hash} ||= do { 90 $tilemap->{$name} ||= do {
52 my $id; 91 my $id;
53 CFPlus::DB::Server::req (get_tile_id => $hash, sub { $id = $_[0] }); 92 CFPlus::DB::Server::req (get_tile_id => $name, sub { $id = $_[0] });
54 sync; 93 sync;
55 $id 94 $id
56 } 95 }
57} 96}
58 97
59package CFPlus::DB::Server; 98package CFPlus::DB::Server;
60 99
61use strict; 100use strict;
62 101
102use EV ();
63use Fcntl; 103use Fcntl;
64use BerkeleyDB; 104use BerkeleyDB;
65 105
66our $DB_HOME = "$Crossfire::VARDIR/cfplus";
67our $DB_ENV; 106our $DB_ENV;
68our $DB_STATE; 107our $DB_STATE;
69our %DB_TABLE; 108our %DB_TABLE;
70 109
71sub open_db { 110sub open_db {
… …
79 -Cachesize => 8_000_000, 118 -Cachesize => 8_000_000,
80 -ErrFile => "$DB_HOME/errorlog.txt", 119 -ErrFile => "$DB_HOME/errorlog.txt",
81# -ErrPrefix => "DATABASE", 120# -ErrPrefix => "DATABASE",
82 -Verbose => 1, 121 -Verbose => 1,
83 -Flags => DB_CREATE | DB_RECOVER | DB_INIT_MPOOL | DB_INIT_LOCK | DB_INIT_TXN | $recover, 122 -Flags => DB_CREATE | DB_RECOVER | DB_INIT_MPOOL | DB_INIT_LOCK | DB_INIT_TXN | $recover,
84 -SetFlags => DB_AUTO_COMMIT | DB_LOG_AUTOREMOVE, 123 -SetFlags => DB_AUTO_COMMIT | DB_LOG_AUTOREMOVE | DB_TXN_WRITE_NOSYNC,
85 or die "unable to create/open database home $DB_HOME: $BerkeleyDB::Error"; 124 or die "unable to create/open database home $DB_HOME: $BerkeleyDB::Error";
86 125
87 1 126 1
88} 127}
89 128
… …
97 -Env => $DB_ENV, 136 -Env => $DB_ENV,
98 -Filename => $table, 137 -Filename => $table,
99# -Filename => "database", 138# -Filename => "database",
100# -Subname => $table, 139# -Subname => $table,
101 -Property => DB_CHKSUM, 140 -Property => DB_CHKSUM,
102 -Flags => DB_CREATE | DB_UPGRADE, 141 -Flags => DB_AUTO_COMMIT | DB_CREATE | DB_UPGRADE,
103 or die "unable to create/open database table $_[0]: $BerkeleyDB::Error" 142 or die "unable to create/open database table $_[0]: $BerkeleyDB::Error"
104 } 143 }
105} 144}
106
107our $SYNC_INTERVAL = 6;
108 145
109our %CB; 146our %CB;
110our $FH; 147our $FH;
111our $ID = "aaa0"; 148our $ID = "aaa0";
112our ($fh_r_watcher, $fh_w_watcher); 149our ($fh_r_watcher, $fh_w_watcher);
113our $sync_timer; 150our $sync_timer;
114our $write_buf; 151our $write_buf;
115our $read_buf; 152our $read_buf;
116 153
154our $SYNC = EV::timer_ns 0, 60, sub {
155 $_[0]->stop;
156 CFPlus::DB::Server::req (sync => sub { });
157};
158
117sub fh_write { 159sub fh_write {
118 my $len = syswrite $FH, $write_buf; 160 my $len = syswrite $FH, $write_buf;
119 161
120 substr $write_buf, 0, $len, ""; 162 substr $write_buf, 0, $len, "";
121 163
122 undef $fh_w_watcher 164 $fh_w_watcher->stop
123 unless length $write_buf; 165 unless length $write_buf;
124} 166}
125 167
126sub fh_read { 168sub fh_read {
127 my $status = sysread $FH, $read_buf, 16384, length $read_buf; 169 my $status = sysread $FH, $read_buf, 16384, length $read_buf;
… …
163 205
164 my $id = ++$ID; 206 my $id = ++$ID;
165 $write_buf .= pack "N/a*", Storable::freeze [$id, $type, @args]; 207 $write_buf .= pack "N/a*", Storable::freeze [$id, $type, @args];
166 $CB{$id} = $cb; 208 $CB{$id} = $cb;
167 209
168 $fh_w_watcher = AnyEvent->io (fh => $FH, poll => 'w', cb => \&fh_write); 210 $fh_w_watcher->start;
169} 211 $SYNC->again unless $SYNC->is_active;
170
171sub sync_tick {
172 req "sync", sub { };
173 $sync_timer = AnyEvent->timer (after => $SYNC_INTERVAL, cb => \&sync_tick);
174} 212}
175 213
176sub do_sync { 214sub do_sync {
177 $DB_ENV->txn_checkpoint (0, 0, 0); 215 $DB_ENV->txn_checkpoint (0, 0, 0);
178 () 216 ()
179} 217}
180 218
219sub do_exists {
220 my ($db, $key) = @_;
221
222 utf8::downgrade $key;
223 my $data;
224 (table $db)->db_get ($key, $data) == 0
225 ? length $data
226 : ()
227}
228
181sub do_get { 229sub do_get {
182 my ($db, $key) = @_; 230 my ($db, $key) = @_;
183 231
232 utf8::downgrade $key;
184 my $data; 233 my $data;
185 (table $db)->db_get ($key, $data) == 0 234 (table $db)->db_get ($key, $data) == 0
186 ? $data 235 ? $data
187 : () 236 : ()
188} 237}
189 238
190sub do_put { 239sub do_put {
191 my ($db, $key, $data) = @_; 240 my ($db, $key, $data) = @_;
192 241
242 utf8::downgrade $key;
243 utf8::downgrade $data;
193 (table $db)->db_put ($key => $data) 244 (table $db)->db_put ($key => $data)
194} 245}
195 246
196sub do_table { 247sub do_table {
197 my ($db) = @_; 248 my ($db) = @_;
… …
205 256
206 \%kv 257 \%kv
207} 258}
208 259
209sub do_get_tile_id { 260sub do_get_tile_id {
210 my ($hash) = @_; 261 my ($name) = @_;
211 262
212 my $id; 263 my $id;
213 my $table = table "facemap"; 264 my $table = table "facemap";
214 265
215 return $id 266 return $id
216 if $table->db_get ($hash, $id) == 0; 267 if $table->db_get ($name, $id) == 0;
217 268
218 for (1..100) { 269 for (1..100) {
219 my $txn = $DB_ENV->txn_begin; 270 my $txn = $DB_ENV->txn_begin;
220 my $status = $table->db_get (id => $id); 271 my $status = $table->db_get (id => $id);
221 if ($status == 0 || $status == BerkeleyDB::DB_NOTFOUND) { 272 if ($status == 0 || $status == BerkeleyDB::DB_NOTFOUND) {
222 $id = ($id || 64) + 1; 273 $id = ($id || 64) + 1;
223 if ($table->db_put (id => $id) == 0 274 if ($table->db_put (id => $id) == 0
224 && $table->db_put ($hash => $id) == 0) { 275 && $table->db_put ($name => $id) == 0) {
225 $txn->txn_commit; 276 $txn->txn_commit;
226 277
227 return $id; 278 return $id;
228 } 279 }
229 } 280 }
230 $txn->txn_abort; 281 $txn->txn_abort;
282 select undef, undef, undef, 0.01 * rand;
231 } 283 }
232 284
233 die "maximum number of transaction retries reached - database problems?"; 285 die "maximum number of transaction retries reached - database problems?";
286}
287
288sub do_unlink {
289 unlink $_[0];
290}
291
292sub do_read_file {
293 my ($path) = @_;
294
295 utf8::downgrade $path;
296 open my $fh, "<:raw", $path
297 or return;
298 sysread $fh, my $buf, -s $fh;
299
300 $buf
301}
302
303sub do_write_file {
304 my ($path, $data) = @_;
305
306 utf8::downgrade $path;
307 utf8::downgrade $data;
308 open my $fh, ">:raw", $path
309 or return;
310 syswrite $fh, $data;
311 close $fh;
312
313 1
314}
315
316sub do_prefetch_file {
317 my ($path, $size) = @_;
318
319 utf8::downgrade $path;
320 open my $fh, "<:raw", $path
321 or return;
322 sysread $fh, my $buf, $size;
323
324 1
325}
326
327our %LOG_FH;
328
329sub do_logprint {
330 my ($path, $line) = @_;
331
332 $LOG_FH{$path} ||= do {
333 open my $fh, ">>:utf8", $path
334 or warn "Couldn't open logfile $path: $!";
335
336 $fh->autoflush (1);
337
338 $fh
339 };
340
341 my ($sec, $min, $hour, $mday, $mon, $year) = localtime time;
342
343 my $ts = sprintf "%04d-%02d-%02d %02d:%02d:%02d",
344 $year + 1900, $mon + 1, $mday, $hour, $min, $sec;
345
346 print { $LOG_FH{$path} } "$ts $line\n"
234} 347}
235 348
236sub run { 349sub run {
237 ($FH, my $fh) = CFPlus::socketpipe; 350 ($FH, my $fh) = CFPlus::socketpipe;
238 351
… …
240 my $oldfh = select $fh; $| = 1; select $oldfh; 353 my $oldfh = select $fh; $| = 1; select $oldfh;
241 354
242 my $pid = fork; 355 my $pid = fork;
243 356
244 if (defined $pid && !$pid) { 357 if (defined $pid && !$pid) {
358 local $SIG{QUIT};
245 local $SIG{__DIE__}; 359 local $SIG{__DIE__};
360 local $SIG{__WARN__};
246 eval { 361 eval {
247 close $FH; 362 close $FH;
248 363
249 unless (eval { open_db }) { 364 unless (eval { open_db }) {
250 File::Path::rmtree $DB_HOME; 365 eval { File::Path::rmtree $DB_HOME };
251 open_db; 366 open_db;
252 } 367 }
253 368
254 while () { 369 while () {
255 4 == read $fh, my $len, 4 370 4 == read $fh, my $len, 4
… …
263 my ($id, $type, @args) = @$req; 378 my ($id, $type, @args) = @$req;
264 my $cb = CFPlus::DB::Server->can ("do_$type") 379 my $cb = CFPlus::DB::Server->can ("do_$type")
265 or die "$type: unknown database request type\n"; 380 or die "$type: unknown database request type\n";
266 my $res = pack "N/a*", Storable::freeze [$id, $cb->(@args)]; 381 my $res = pack "N/a*", Storable::freeze [$id, $cb->(@args)];
267 (syswrite $fh, $res) == length $res 382 (syswrite $fh, $res) == length $res
268 or die; 383 or die "DB::write: $!";
269 } 384 }
270 }; 385 };
271 386
272 my $error = $@; 387 my $error = $@;
273 388
274 eval { 389 eval {
390 $DB_ENV->txn_checkpoint (0, 0, 0);
391
275 undef %DB_TABLE; 392 undef %DB_TABLE;
276 undef $DB_ENV; 393 undef $DB_ENV;
277 394
278 Storable::store_fd [die => $error], $fh; 395 Storable::store_fd [die => $error], $fh;
279 }; 396 };
… …
284 close $fh; 401 close $fh;
285 CFPlus::fh_nonblocking $FH, 1; 402 CFPlus::fh_nonblocking $FH, 1;
286 403
287 $CB{die} = sub { die shift }; 404 $CB{die} = sub { die shift };
288 405
289 $fh_r_watcher = AnyEvent->io (fh => $FH, poll => 'r', cb => \&fh_read); 406 $fh_r_watcher = EV::io $FH, EV::READ , \&fh_read;
407 $fh_w_watcher = EV::io $FH, EV::WRITE, \&fh_write;
408 $SYNC->again unless $SYNC->is_active;
409}
290 410
291 sync_tick; 411sub stop {
412 close $FH;
292} 413}
293 414
2941; 4151;
295 416
296=back 417=back

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines