--- BDB/BDB.pm 2007/12/10 04:57:14 1.24 +++ BDB/BDB.pm 2011/12/26 01:46:33 1.64 @@ -26,9 +26,11 @@ }; db_sync $db; - # automatic result processing with AnyEvent: - our $FH; open $FH, "<&=" . BDB::poll_fileno; - our $WATCHER = AnyEvent->io (fh => $FH, poll => 'r', cb => \&BDB::poll_cb); + # when you also use Coro, management is easy: + use Coro::BDB; + + # automatic event loop integration with AnyEvent: + use AnyEvent::BDB; # automatic result processing with EV: my $WATCHER = EV::io BDB::poll_fileno, EV::READ, \&BDB::poll_cb; @@ -103,18 +105,21 @@ package BDB; -no warnings; -use strict 'vars'; +use common::sense; use base 'Exporter'; +our $VERSION; + BEGIN { - our $VERSION = '1.3'; + $VERSION = '1.9'; our @BDB_REQ = qw( db_env_open db_env_close db_env_txn_checkpoint db_env_lock_detect - db_env_memp_sync db_env_memp_trickle - db_open db_close db_compact db_sync db_put db_get db_pget db_del db_key_range + db_env_memp_sync db_env_memp_trickle db_env_dbrename db_env_dbremove + db_env_log_archive + db_open db_close db_compact db_sync db_verify db_upgrade + db_put db_exists db_get db_pget db_del db_key_range db_txn_commit db_txn_abort db_txn_finish db_c_close db_c_count db_c_put db_c_get db_c_pget db_c_del db_sequence_open db_sequence_close @@ -132,90 +137,118 @@ XSLoader::load ("BDB", $VERSION); } -=head2 BERKELEYDB FUNCTIONS +=head1 BERKELEYDB FUNCTIONS All of these are functions. The create functions simply return a new -object and never block. All the remaining functions all take an optional +object and never block. All the remaining functions take an optional callback as last argument. If it is missing, then the function will be executed synchronously. In both cases, C<$!> will reflect the return value of the function. BDB functions that cannot block (mostly functions that manipulate settings) are method calls on the relevant objects, so the rule of thumb -is: if its a method, its not blocking, if its a function, it takes a +is: if it's a method, it's not blocking, if it's a function, it takes a callback as last argument. In the following, C<$int> signifies an integer return value, -C is a "binary string" (i.e. a perl string with no character -indices >255), C is an unsigned 32 bit integer, C is some -integer, C is a floating point value. - -The C types are generic perl scalars (for input and output of data -values), and the C is the optional callback function to call -when the request is completed. +C is a "filename" (octets on unix, madness on windows), +C is an unsigned 32 bit integer, C is some integer, C is a +floating point value. + +Most C types are generic perl scalars (for input and output of data +values). The various C etc. arguments are handles return by C, C, C and so on. If they have an appended C<_ornull> this means they are optional and you can pass C for them, resulting a NULL pointer on the C level. -=head3 BDB functions +The C is the optional callback function to call when the +request is completed. This last callback argument is special: the callback +is simply the last argument passed. If there are "optional" arguments +before the callback they can be left out. The callback itself can be left +out or specified as C, in which case the function will be executed +synchronously. + +For example, C usually is called with all integer +arguments zero. These can be left out, so all of these specify a call +to C<< DB_ENV->txn_checkpoint >>, to be executed asynchronously with a +callback to be called: + + db_env_txn_checkpoint $db_env, 0, 0, 0, sub { }; + db_env_txn_checkpoint $db_env, 0, 0, sub { }; + db_env_txn_checkpoint $db_env, sub { }; + +While these all specify a call to C<< DB_ENV->txn_checkpoint >> to be +executed synchronously: + + db_env_txn_checkpoint $db_env, 0, 0, 0, undef; + db_env_txn_checkpoint $db_env, 0, 0, 0; + db_env_txn_checkpoint $db_env, 0; + +=head2 BDB functions Functions in the BDB namespace, exported by default: $env = db_env_create (U32 env_flags = 0) flags: RPCCLIENT - db_env_open (DB_ENV *env, octetstring db_home, U32 open_flags, int mode, SV *callback = &PL_sv_undef) + db_env_open (DB_ENV *env, bdb_filename db_home, U32 open_flags, int mode, SV *callback = 0) open_flags: INIT_CDB INIT_LOCK INIT_LOG INIT_MPOOL INIT_REP INIT_TXN RECOVER RECOVER_FATAL USE_ENVIRON USE_ENVIRON_ROOT CREATE LOCKDOWN PRIVATE REGISTER SYSTEM_MEM - db_env_close (DB_ENV *env, U32 flags = 0, SV *callback = &PL_sv_undef) - db_env_txn_checkpoint (DB_ENV *env, U32 kbyte = 0, U32 min = 0, U32 flags = 0, SV *callback = &PL_sv_undef) + db_env_close (DB_ENV *env, U32 flags = 0, SV *callback = 0) + db_env_txn_checkpoint (DB_ENV *env, U32 kbyte = 0, U32 min = 0, U32 flags = 0, SV *callback = 0) flags: FORCE - db_env_lock_detect (DB_ENV *env, U32 flags = 0, U32 atype = DB_LOCK_DEFAULT, SV *dummy = 0, SV *callback = &PL_sv_undef) + db_env_lock_detect (DB_ENV *env, U32 flags = 0, U32 atype = DB_LOCK_DEFAULT, SV *dummy = 0, SV *callback = 0) atype: LOCK_DEFAULT LOCK_EXPIRE LOCK_MAXLOCKS LOCK_MAXWRITE LOCK_MINLOCKS LOCK_MINWRITE LOCK_OLDEST LOCK_RANDOM LOCK_YOUNGEST - db_env_memp_sync (DB_ENV *env, SV *dummy = 0, SV *callback = &PL_sv_undef) - db_env_memp_trickle (DB_ENV *env, int percent, SV *dummy = 0, SV *callback = &PL_sv_undef) + db_env_memp_sync (DB_ENV *env, SV *dummy = 0, SV *callback = 0) + db_env_memp_trickle (DB_ENV *env, int percent, SV *dummy = 0, SV *callback = 0) + db_env_dbremove (DB_ENV *env, DB_TXN_ornull *txnid, bdb_filename file, bdb_filename database, U32 flags = 0, SV *callback = 0) + db_env_dbrename (DB_ENV *env, DB_TXN_ornull *txnid, bdb_filename file, bdb_filename database, bdb_filename newname, U32 flags = 0, SV *callback = 0) + db_env_log_archive (DB_ENV *env, SV *listp, U32 flags = 0, SV *callback = 0) $db = db_create (DB_ENV *env = 0, U32 flags = 0) flags: XA_CREATE - db_open (DB *db, DB_TXN_ornull *txnid, octetstring file, octetstring database, int type, U32 flags, int mode, SV *callback = &PL_sv_undef) + db_open (DB *db, DB_TXN_ornull *txnid, bdb_filename file, bdb_filename database, int type, U32 flags, int mode, SV *callback = 0) flags: AUTO_COMMIT CREATE EXCL MULTIVERSION NOMMAP RDONLY READ_UNCOMMITTED THREAD TRUNCATE - db_close (DB *db, U32 flags = 0, SV *callback = &PL_sv_undef) + db_close (DB *db, U32 flags = 0, SV *callback = 0) flags: DB_NOSYNC - db_compact (DB *db, DB_TXN_ornull *txn = 0, SV *start = 0, SV *stop = 0, SV *unused1 = 0, U32 flags = DB_FREE_SPACE, SV *unused2 = 0, SV *callback = &PL_sv_undef) + db_verify (DB *db, bdb_filename file, bdb_filename database = 0, SV *dummy = 0, U32 flags = 0, SV *callback = 0) + db_upgrade (DB *db, bdb_filename file, U32 flags = 0, SV *callback = 0) + db_compact (DB *db, DB_TXN_ornull *txn = 0, SV *start = 0, SV *stop = 0, SV *unused1 = 0, U32 flags = DB_FREE_SPACE, SV *unused2 = 0, SV *callback = 0) flags: FREELIST_ONLY FREE_SPACE - db_sync (DB *db, U32 flags = 0, SV *callback = &PL_sv_undef) - db_key_range (DB *db, DB_TXN_ornull *txn, SV *key, SV *key_range, U32 flags = 0, SV *callback = &PL_sv_undef) - db_put (DB *db, DB_TXN_ornull *txn, SV *key, SV *data, U32 flags = 0, SV *callback = &PL_sv_undef) + db_sync (DB *db, U32 flags = 0, SV *callback = 0) + db_key_range (DB *db, DB_TXN_ornull *txn, SV *key, SV *key_range, U32 flags = 0, SV *callback = 0) + db_put (DB *db, DB_TXN_ornull *txn, SV *key, SV *data, U32 flags = 0, SV *callback = 0) flags: APPEND NODUPDATA NOOVERWRITE - db_get (DB *db, DB_TXN_ornull *txn, SV *key, SV *data, U32 flags = 0, SV *callback = &PL_sv_undef) + db_exists (DB *db, DB_TXN_ornull *txn, SV *key, U32 flags = 0, SV *callback = 0) (v4.6) + db_get (DB *db, DB_TXN_ornull *txn, SV *key, SV *data, U32 flags = 0, SV *callback = 0) flags: CONSUME CONSUME_WAIT GET_BOTH SET_RECNO MULTIPLE READ_COMMITTED READ_UNCOMMITTED RMW - db_pget (DB *db, DB_TXN_ornull *txn, SV *key, SV *pkey, SV *data, U32 flags = 0, SV *callback = &PL_sv_undef) + db_pget (DB *db, DB_TXN_ornull *txn, SV *key, SV *pkey, SV *data, U32 flags = 0, SV *callback = 0) flags: CONSUME CONSUME_WAIT GET_BOTH SET_RECNO MULTIPLE READ_COMMITTED READ_UNCOMMITTED RMW - db_del (DB *db, DB_TXN_ornull *txn, SV *key, U32 flags = 0, SV *callback = &PL_sv_undef) - db_txn_commit (DB_TXN *txn, U32 flags = 0, SV *callback = &PL_sv_undef) + db_del (DB *db, DB_TXN_ornull *txn, SV *key, U32 flags = 0, SV *callback = 0) + db_txn_commit (DB_TXN *txn, U32 flags = 0, SV *callback = 0) flags: TXN_NOSYNC TXN_SYNC - db_txn_abort (DB_TXN *txn, SV *callback = &PL_sv_undef) + db_txn_abort (DB_TXN *txn, SV *callback = 0) - db_c_close (DBC *dbc, SV *callback = &PL_sv_undef) - db_c_count (DBC *dbc, SV *count, U32 flags = 0, SV *callback = &PL_sv_undef) - db_c_put (DBC *dbc, SV *key, SV *data, U32 flags = 0, SV *callback = &PL_sv_undef) + db_c_close (DBC *dbc, SV *callback = 0) + db_c_count (DBC *dbc, SV *count, U32 flags = 0, SV *callback = 0) + db_c_put (DBC *dbc, SV *key, SV *data, U32 flags = 0, SV *callback = 0) flags: AFTER BEFORE CURRENT KEYFIRST KEYLAST NODUPDATA - db_c_get (DBC *dbc, SV *key, SV *data, U32 flags = 0, SV *callback = &PL_sv_undef) + db_c_get (DBC *dbc, SV *key, SV *data, U32 flags = 0, SV *callback = 0) flags: CURRENT FIRST GET_BOTH GET_BOTH_RANGE GET_RECNO JOIN_ITEM LAST NEXT NEXT_DUP NEXT_NODUP PREV PREV_DUP PREV_NODUP SET SET_RANGE SET_RECNO READ_UNCOMMITTED MULTIPLE MULTIPLE_KEY RMW - db_c_pget (DBC *dbc, SV *key, SV *pkey, SV *data, U32 flags = 0, SV *callback = &PL_sv_undef) - db_c_del (DBC *dbc, U32 flags = 0, SV *callback = &PL_sv_undef) + db_c_pget (DBC *dbc, SV *key, SV *pkey, SV *data, U32 flags = 0, SV *callback = 0) + db_c_del (DBC *dbc, U32 flags = 0, SV *callback = 0) - db_sequence_open (DB_SEQUENCE *seq, DB_TXN_ornull *txnid, SV *key, U32 flags = 0, SV *callback = &PL_sv_undef) + db_sequence_open (DB_SEQUENCE *seq, DB_TXN_ornull *txnid, SV *key, U32 flags = 0, SV *callback = 0) flags: CREATE EXCL - db_sequence_close (DB_SEQUENCE *seq, U32 flags = 0, SV *callback = &PL_sv_undef) - db_sequence_get (DB_SEQUENCE *seq, DB_TXN_ornull *txnid, int delta, SV *seq_value, U32 flags = DB_TXN_NOSYNC, SV *callback = &PL_sv_undef) + db_sequence_close (DB_SEQUENCE *seq, U32 flags = 0, SV *callback = 0) + db_sequence_get (DB_SEQUENCE *seq, DB_TXN_ornull *txnid, int delta, SV *seq_value, U32 flags = DB_TXN_NOSYNC, SV *callback = 0) flags: TXN_NOSYNC - db_sequence_remove (DB_SEQUENCE *seq, DB_TXN_ornull *txnid = 0, U32 flags = 0, SV *callback = &PL_sv_undef) + db_sequence_remove (DB_SEQUENCE *seq, DB_TXN_ornull *txnid = 0, U32 flags = 0, SV *callback = 0) flags: TXN_NOSYNC -=head4 db_txn_finish (DB_TXN *txn, U32 flags = 0, SV *callback = &PL_sv_undef) +=head3 db_txn_finish (DB_TXN *txn, U32 flags = 0, SV *callback = 0) This is not actually a Berkeley DB function but a BDB module extension. The background for this exytension is: It is very annoying to @@ -251,7 +284,8 @@ has failed in this way and abort further processing (excluding C). -=head3 DB_ENV/database environment methods + +=head2 DB_ENV/database environment methods Methods available on DB_ENV/$env handles: @@ -265,7 +299,9 @@ $int = $env->set_lg_dir (const char *dir) $int = $env->set_shm_key (long shm_key) $int = $env->set_cachesize (U32 gbytes, U32 bytes, int ncache = 0) - $int = $env->set_flags (U32 flags, int onoff) + $int = $env->set_flags (U32 flags, int onoff = 1) + $int = $env->log_set_config (U32 flags, int onoff = 1) (v4.7) + $int = $env->set_intermediate_dir_mode (const char *modestring) (v4.7) $env->set_errfile (FILE *errfile = 0) $env->set_msgfile (FILE *msgfile = 0) $int = $env->set_verbose (U32 which, int onoff = 1) @@ -287,8 +323,9 @@ $txn = $env->txn_begin (DB_TXN_ornull *parent = 0, U32 flags = 0) flags: READ_COMMITTED READ_UNCOMMITTED TXN_NOSYNC TXN_NOWAIT TXN_SNAPSHOT TXN_SYNC TXN_WAIT TXN_WRITE_NOSYNC + $txn = $env->cdsgroup_begin; (v4.5) -=head4 Example: +=head3 Example: use AnyEvent; use BDB; @@ -310,7 +347,7 @@ $env->set_flags (BDB::AUTO_COMMIT | BDB::TXN_NOSYNC, 1); -=head3 DB/database methods +=head2 DB/database methods Methods available on DB/$db handles: @@ -346,7 +383,7 @@ flags: READ_COMMITTED READ_UNCOMMITTED WRITECURSOR TXN_SNAPSHOT $seq = $db->sequence (U32 flags = 0) -=head4 Example: +=head3 Example: my $db = db_create $env; db_open $db, undef, "table", undef, BDB::BTREE, BDB::AUTO_COMMIT | BDB::CREATE | BDB::READ_UNCOMMITTED, 0600; @@ -363,7 +400,7 @@ db_sync $db; -=head3 DB_TXN/transaction methods +=head2 DB_TXN/transaction methods Methods available on DB_TXN/$txn handles: @@ -379,7 +416,7 @@ # see db_txn_finish documentation, above -=head3 DBC/cursor methods +=head2 DBC/cursor methods Methods available on DBC/$dbc handles: @@ -388,7 +425,9 @@ if (dbc) dbc->c_close (dbc); -=head4 Example: + $int = $cursor->set_priority ($priority = PRIORITY_*) (v4.6) + +=head3 Example: my $c = $db->cursor; @@ -401,7 +440,7 @@ db_c_close $c; -=head3 DB_SEQUENCE/sequence methods +=head2 DB_SEQUENCE/sequence methods Methods available on DB_SEQUENCE/$seq handles: @@ -416,7 +455,7 @@ flags: SEQ_DEC SEQ_INC SEQ_WRAP $int = $seq->set_range (db_seq_t min, db_seq_t max) -=head4 Example: +=head3 Example: my $seq = $db->sequence; @@ -424,9 +463,9 @@ db_sequence_get $seq, undef, 1, my $value; -=head2 SUPPORT FUNCTIONS +=head1 SUPPORT FUNCTIONS -=head3 EVENT PROCESSING AND EVENT LOOP INTEGRATION +=head2 EVENT PROCESSING AND EVENT LOOP INTEGRATION =over 4 @@ -435,6 +474,9 @@ Returns the string corresponding to the given errno value. If no argument is given, use C<$!>. +Note that the BDB module also patches the C<$!> variable directly, so you +should be able to get a bdb error string by simply stringifying C<$!>. + =item $fileno = BDB::poll_fileno Return the I. This filehandle must be @@ -520,7 +562,73 @@ =back -=head3 CONTROLLING THE NUMBER OF THREADS +=head2 VERSION CHECKING + +BerkeleyDB comes in various versions, many of them have minor +incompatibilities. This means that traditional "at least version x.x" +checks are often not sufficient. + +Example: set the log_autoremove option in a way compatible with set_flags (&BDB::LOG_AUTOREMOVE ) if BDB::VERSION v0, v4.7; + $DB_ENV->log_set_config (&BDB::LOG_AUTO_REMOVE) if BDB::VERSION v4.7; + +=over 4 + +=item BDB::VERSION + +The C function, when called without arguments, returns the +Berkeley DB version as a v-string (usually with 3 components). You should +use C and C operators exclusively to make comparisons. + +Example: check for at least version 4.7. + + BDB::VERSION ge v4.7 or die; + +=item BDB::VERSION min-version + +Returns true if the BDB version is at least the given version (specified +as a v-string), false otherwise. + +Example: check for at least version 4.5. + + BDB::VERSION v4.7 or die; + +=item BDB::VERSION min-version, max-version + +Returns true of the BDB version is at least version C (specify C or C for any minimum version) +and less then C. + +Example: check wether version is strictly less then v4.7. + + BDB::VERSION v0, v4.7 + or die "version 4.7 is not yet supported"; + +=back + +=cut + +sub VERSION { + # I was dumb enough to override the VERSION method here, so let's try + # to fix it up. + + if ($_[0] eq __PACKAGE__) { + $VERSION + } else { + if (@_ > 0) { + return undef if VERSION_v lt $_[0]; + if (@_ > 1) { + return undef if VERSION_v ge $_[1]; + } + } + + VERSION_v + } +} + +=head2 CONTROLLING THE NUMBER OF THREADS =over 4 @@ -591,12 +699,19 @@ C is mainly useful in simple scripts (with low values) or as a stop gap to shield against fatal memory overflow (with large values). -=item BDB::set_sync_prepare $cb +=item $old_cb = BDB::set_sync_prepare $cb Sets a callback that is called whenever a request is created without an explicit callback. It has to return two code references. The first is used -as the request callback, and the second is called to wait until the first -callback has been called. The default implementation works like this: +as the request callback (it should save the return status), and the second +is called to wait until the first callback has been called (it must set +C<$!> to the return status). + +This mechanism can be used to include BDB into other event mechanisms, +such as L. + +To allow other, callback-based, events to be executed while callback-less +ones are run, you could use this sync prepare function: sub { my $status; @@ -606,9 +721,17 @@ ) } +It works by polling for results till the request has finished and then +sets C<$!> to the return value. This means that if you don't use a +callback, BDB would simply fall back to synchronous operations. + +By default, or if the sync prepare function is set to C, is to +execute callback-less BDB requests in the foreground thread, setting C<$!> +to the return value, without polling for other events. + =back -=head3 STATISTICAL INFORMATION +=head2 STATISTICAL INFORMATION =over 4 @@ -636,18 +759,7 @@ =cut -set_sync_prepare { - my $status; - ( - sub { - $status = $!; - }, - sub { - BDB::poll while !defined $status; - $! = $status; - }, - ) -}; +set_sync_prepare (undef); min_parallel 8; @@ -655,7 +767,30 @@ 1; -=head2 FORK BEHAVIOUR +=head1 COMMON PITFALLS + +=head2 Unexpected Crashes + +Remember that, by default, BDB will execute requests in parallel, in +somewhat random order. That means that it is easy to run a C +request on thesa me database as a concurrent C request, leading +to a crash, silent data corruption, eventually the next world war on +terrorism. + +If you only ever use foreground requests (without a callback), this will +not be an issue. + +=head2 Unexpected Freezes or Deadlocks + +Remember that, by default, BDB will execute requests in parallel, which +easily leads to deadlocks (even concurrent put's on the same database can +deadlock). + +You either need to run deadlock detection (and handle the resulting +errors), or make sure only one process ever updates the database, ine one +thread, e.g. by using only foreground requests (without a callback). + +=head1 FORK BEHAVIOUR This module should do "the right thing" when the process using it forks: @@ -671,7 +806,11 @@ not been called, while the child will act as if BDB has not been used yet. -=head2 MEMORY USAGE +Win32 note: there is no fork on win32, and perls emulation of it is too +broken to be supported, so do not use BDB in a windows pseudo-fork, better +yet, switch to a more capable platform. + +=head1 MEMORY USAGE Per-request usage: @@ -690,6 +829,15 @@ temporary buffers, and each thread requires a stack and other data structures (usually around 16k-128k, depending on the OS). +=head1 WIN32 FILENAMES/DATABASE NAME MESS + +Perl on Win32 supports only ASCII filenames (the reason is that it abuses +an internal flag to store wether a filename is Unicode or ANSI, but that +flag is used for somethign else in the perl core, so there is no way to +detect wether a filename is ANSI or Unicode-encoded). The BDB module +tries to work around this issue by assuming that the filename is an ANSI +filename and BDB was built for unicode support. + =head1 KNOWN BUGS Known bugs will be fixed in the next release, except: @@ -701,7 +849,8 @@ =head1 SEE ALSO -L, L. +L (event loop integration), L (more natural +syntax), L (nice to have). =head1 AUTHOR