ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/cvsroot/TDB_FileX/TDB_FileX.pm
Revision: 1.2
Committed: Mon Apr 28 18:31:58 2025 UTC (16 months, 3 weeks ago) by root
Branch: MAIN
Changes since 1.1: +152 -9 lines
Log Message:
*** empty log message ***

File Contents

# User Rev Content
1 root 1.1 package TDB_FileX;
2    
3     use common::sense;
4    
5     use Exporter ();
6     use XSLoader ();
7    
8     our @ISA = qw(Exporter);
9    
10     # Items to export into callers namespace by default. Note: do not export
11     # names by default without a very good reason. Use EXPORT_OK instead.
12     # Do not simply export all your public functions/methods/constants.
13    
14     our %EXPORT_TAGS = (
15     flags => [qw(
16     ALLOW_NESTING
17     BIGENDIAN
18     CLEAR_IF_FIRST
19     CONVERT
20     DEFAULT
21     DISALLOW_NESTING
22     INCOMPATIBLE_HASH
23     INTERNAL
24     MUTEX_LOCKING
25     NOLOCK
26     NOMMAP
27     NOSYNC
28     SEQNUM
29     VOLATILE
30     )],
31     insert => [qw(
32     INSERT
33     MODIFY
34     REPLACE
35     )],
36     error => [qw(
37     SUCCESS
38     ERR_CORRUPT
39     ERR_EXISTS
40     ERR_IO
41     ERR_LOCK
42     ERR_LOCK_TIMEOUT
43     ERR_NOEXIST
44     ERR_NOLOCK
45     ERR_OOM
46     ERR_EINVAL
47     ERR_RDONLY
48     )],
49     debug => [qw(
50     DEBUG_FATAL
51     DEBUG_ERROR
52     DEBUG_WARNING
53     DEBUG_TRACE}
54     )],
55     );
56    
57     our @EXPORT_OK;
58    
59     Exporter::export_ok_tags qw(flags insert error debug);
60    
61     $EXPORT_TAGS{all} = \@EXPORT_OK;
62    
63     our $VERSION = '0.97';
64    
65     XSLoader::load __PACKAGE__, $VERSION;
66    
67     1;
68     __END__
69    
70     =head1 NAME
71    
72     TDB_FileX - Perl access to the trivial database library
73    
74     =head1 SYNOPSIS
75    
76     use TDB_FileX;
77    
78     # tie interface
79     tie %hash, TDB_FileX => $filename,
80     hash_size => 8000,
81     mutex => 1,
82     ;
83     $hash{key} = 'value';
84     while (my ($k, $v) = each %hash) { print "$k -> $v\n" }
85    
86     # OO interface
87     my $tdb = TDB_FileX->open ($filename, flags => TDB_FileX::CLEAR_IF_FIRST)
88     or die $!;
89    
90     $tdb->store (key => 'value') or die $tdb->errorstr;
91     $tdb->traverse (sub { print "$_[0] -> $_[1]\n" });
92    
93     =head1 DESCRIPTION
94    
95     TDB is a simple database similar to GDBM, but allows multiple simultaneous
96     writers. It's main drawback is the need to manually configure a hash table
97     size in advance - see the C<hash_size> option for C<open>.
98    
99     TDB_FileX provides a simple C<tie> interface, similar to DB_File and
100     friends; and an object-oriented interface, which provides access to all of
101     the functions in the tdb library.
102    
103     =head2 FUNCTIONS
104    
105     =over 4
106    
107     =item TDB_FileX->open ($path[, key => value...])
108    
109     TDB_FileX constructor. Opens $path and returns a TDB_FileX object. The
110     same arguments may be passed to the C<tie> function. On error, C<$!>
111     is set and C<undef> is returned.
112    
113     A number of key-value pairs are accepted, with hopefully sensible defaults
114     for most of these (but you should consider at least setting C<hash_size>
115     if you want to store more than a thousand or so pairs, And C<mutex> if you
116     want to take advantage of much faster locking.
117    
118     =over
119    
120     =item tdb_flags => $flags (default: C<TDB_FileX::DEFAULT>)
121    
122     A set of flags that influence the behaviour and format of the database.
123    
124     See tdb_open(3) for the meanings and possible values (note that the
125     C<TDB_> prefix has to be removed, see the L<EXPORTS> section for a list.
126    
127     =item open_flags => $flags (default: C<Fcntl::O_RDWR | Fcntl::O_CREAT>)
128    
129     Standard open flags, as used in C<sysopen>.
130    
131     =item mode => $mode (default: C<0666>)
132    
133     Standard file open mode, as use din C<sysopen>. Only used when creating
134     the database file.
135    
136     =item hash_size => $int (default: internal to libtdb, but normally C<131>)
137    
138     The size of the internal hash table - only used when creating the
139     database. The default is usually very low and only good for a few thousand
140     keys. As a rule of thumb, it should be at least one percent of the number
141     of keys you plan to store, e.g. for C<800000> keys you should use around a
142     size of C<8000>.
143    
144 root 1.2 Every has entry is only 4 octets, os usually it isn't an issue to make the
145     hash table too large.
146    
147     To give you an idea of the performance, I inserted about 600000 records,
148     3GB of data, into a TDB file, using different hash sizes.
149 root 1.1
150     size time
151     131 160s
152     256 85s
153     1024 12s
154     4096 5s
155     8192 4s
156    
157     As you can see, the default hash table size for this case caused it to use
158     40 times the time to insert than a larger hash table, the optimum being
159 root 1.2 around 75 keys per hash entry).
160 root 1.1
161     =item log_cb => $cb->($level, $msg)
162    
163 root 1.2 Sets a code reference that is called with a message and a log level
164 root 1.1 (lower means more important, there are C<DEBUG_FATAL>, C<DEBUG_ERROR>,
165     C<DEBUG_WARNING> and C<DEBUG_TRACE}>). Unlike the "debug" in the name
166     might indicate, if you want to find out why, for instance, you could not
167     open a database, you need to use a logging callback.
168    
169     =item hash => $hash (default: C<undef>)
170    
171     Selects a hash function to use.
172    
173     =over
174    
175     =item C<"default"> or C<undef>
176    
177     Use autodetection and use either C<"jenkins"> or the original tdb hash function.
178    
179     =item C<"jenkins">
180    
181     The jenkins hash function. Relatively fast, and recommended for modern tdb databases
182     that need to be interoperable between implementations.
183    
184     =item C<"fnv1ax">
185    
186     The FNV1-A hash with 32 bit post-mixing. Pretty good and verxy fast for
187     keys up to 10-20 octets.
188    
189     =item C<"xxh3">
190    
191     The XXH3 hash. Very good and fast especially for long keys.
192    
193     =item C<1> .. C<4>
194    
195     Specify one of up to four custom hash functions (see L<set_hash_function>).
196    
197     =back
198    
199     =item mutex => $bool (default: C<0>)
200    
201     TDB can take advantage of fast interprocess mutexes, which
202     can be orders of magnitude faster than the syscall-based
203     locking used by default. Normally, you need to call
204     C<TDB_FileX::runtime_check_for_robust_mutexes> and set the
205     C<MUTEX_LOCKING> flag if support is indicated.
206    
207     This option, when enabled, enables C<MUTEX_LOCKING> if it is supported
208     by the platform - which involves a fork when opening the database. When
209     disabled, it will remove the flag.
210    
211     It is recommended to keep this one, but enabling this changes the format
212     of the database, so might not be an option if interoperability with other
213     programs is required.
214    
215     =back
216    
217     There is no explicit close function. The database is closed implicitly
218     when there are no remaining references.
219    
220     =item $tdb->store ($key, $value[, $flag=REPLACE])
221    
222     Store $value in the database with key w$key. The $flag defaults to
223     C<REPLACE>, but can also be C<INSERT> or C<MODIFY>, see tdb_store(3) for
224     details.
225    
226     On failure, a perl false is returned. See L</$tdb->error> and
227     L</$tdb->errorstr> for the reason.
228    
229     =item $tdb->fetch ($key)
230    
231     Fetch the value associated with $key, or C<undef> if it is not found.
232    
233     =item $tdb->delete ($key)
234    
235     Delete the value associated with $key.
236    
237     On failure, a perl false is returned. See L</$tdb->error> and
238     L</$tdb->errorstr> for the reason.
239    
240     =item $tdb->exists ($key)
241    
242     Return true if the $key is found, false otherwise.
243    
244     =item $tdb->firstkey
245    
246     Return the key of the first value in the database. Returns C<undef> on
247     failure or if there are no keys in the database. See tdb_firstkey(3)
248     for details.
249    
250     =item $tdb->nextkey ($lastkey)
251    
252     Return the next key in the database after $lastkey. Returns C<undef> on
253     failure or if there are no more keys in the database. See tdb_nextkey(3)
254     for details.
255    
256     =item $tdb->error
257    
258     Returns the current error state of the C<$tdb> object. See the list of
259     error codes given in L<EXPORTS>.
260    
261     =item $tdb->errorstr
262    
263     Returns a printable string that describes the error state of the
264     database.
265    
266     =item $tdb->reopen
267    
268     Closes and reopens the database. Required after a
269     L<fork|perlfunc/fork>, if both processes wish to use the database.
270    
271     B<NB:> If C<reopen> fails (returns false), then it is unsafe to call
272     any further methods on C<$tdb>. Thus, the only way to find out I<why>
273     C<reopen> failed is to use a logging function.
274    
275     =item TDB_FileX::reopen_all
276    
277     Closes and reopens all open databases. See L</$tdb->reopen>.
278    
279     B<NB:> If C<reopen_all> fails (returns false), there is no indication
280     of I<which> C<$tdb> objects failed or why. If you have to survive
281     failures, you may wish to do your own C<reopen> loop instead.
282    
283     =item $tdb->traverse ($cb->()) #TODO
284    
285     Call $cb for each entry in the database.
286    
287     The callback is called with the key and value as arguments and should return a
288     true value if you wish to continue traversal.
289    
290     C<traverse> returns the number of elements traversed or C<undef> on
291     error. If SUB is C<undef>, then this function simply counts the number
292     of elements.
293    
294     =item $tdb->set_logging_function(SUB)
295    
296     Set the logging function to use when this database object encounters
297     errors. SUB should be a coderef or a string giving the name of a
298     function.
299    
300     SUB is called with the severity level (an integer) and the message (a
301     string).
302    
303     =item $tdb->lockall
304    
305 root 1.2 Lock an entire database with an exclusive write lock, returning false on
306     error. The purpose of this call is to avoid locking overhead for many
307     operations, but the database has to be unlocked manually when done.
308 root 1.1
309     =item $tdb->unlockall
310    
311     Unlock an entire database previously locked with
312     L</$tdb->lockall>.
313    
314 root 1.2 =item $tdb->lockall_read
315    
316     Same as L</$tdb->lockall>, but uses a shared read lock instead of a writer lock.
317    
318     =item $tdb->unlockall_read
319    
320     Opposite of L</$tdb->lockall_read>.
321    
322     =item $tdb->lockall_mark
323    
324     =item $tdb->lockall_unmark
325    
326     Please tlel me exactly what these do.
327    
328     =item $tdb->lockall_nonblock
329    
330     =item $tdb->lockall_read_nonblock
331    
332     Please tell me what exactly these do.
333    
334     =item $tdb->transaction_start
335    
336     Starts a transaction - all operations will be queued, but not applied
337     to the database, until the transaction is either committed L</$tdb->transaction_commit>
338     or aborted/thrown away, with L</$tdb->transaction_cancel>.
339    
340     =item $tdb->transaction_start_nonblock
341    
342     Please tell me what this does.
343    
344     =item $tdb->transaction_commit
345    
346     Applies all changes in the transaction.
347    
348     =item $tdb->transaction_cancel
349    
350     Throws away all changes in the transaction.
351    
352     =item $tdb->transaction_prepare_commit
353    
354     Instead of calling L</$tdb->transaction_commit> you can do the commit in
355     two phases by calling this method before commit, which does the expensive
356     steps first.
357    
358     =item $bool = $tdb->transaction_active
359    
360     Returns true if a transaction is currently active.
361    
362     =item $tdb->enable_seqnum
363    
364     Enables sequence number generatiuon supporet for the database.
365    
366     =item $seq = $tdb->get_seqnum
367    
368     Returns the current sequence number. Internally, this is a 32 bit unsigned
369     integer, but the API converts it into a native integer, so the same
370     internal sequence number might be represented differently on different
371     machines.
372    
373     =item $tdb->increment_seqnum_nonblock
374    
375     Increments the sequence number. Note that the sequence number is also
376     incremented by TDB itself oon many operations.
377    
378     =item $size = $tdb->hash_size
379    
380     Returns the size of the hash table. The size cannot be changed other than
381     by recreating the database.
382    
383     =item $octets = $tdb->map_size
384    
385     Returns the current mmap size for the database.
386    
387     =item $flags = $tdb->get_flags
388    
389     Returns the flags for the database (the same as the C<tdb_flags> in C<open>).
390    
391     =item $tdb->add_flags ($flag)
392    
393     Tried to add flags to the database - yes, the parameter says C<$flag> (singular)
394     but the documentation and the code say I<flags> (plural).
395    
396     =item $tdb->remove_flags ($flag)
397    
398     Attempts to remove flags from the database.
399    
400     =item $fileno = $tdb->fd
401    
402     Returns the file descriptor (not file handle) for the underlying database file.
403    
404     =item $path = $tdb->name
405    
406     Returns the path used to open the database.
407    
408     =item $tdb->wipe_all
409    
410     Efficientlly deletes all entries in the database. This does not shrink the
411     file itself.
412    
413     =item $tdb->repack
414    
415     Tries to improve layout of the database by copying all items into a
416     temporary in-memory database, wiping the database, and copying all items
417     back. Yes, the database must fit into memory.
418    
419     =item $bool = TDB_FileX::runtime_check_for_robust_mutexes
420    
421     Tests whether robust mutexes are available for locking. This involves
422     forking the process, so it can be costly and problematic. This function
423     needs to be called before using the C<MUTEX_LOCKING> flag. But see the
424     C<mutex> parametrer to C<open> for an alternative.
425    
426 root 1.1 =item $tdb->dump_all
427    
428     Dump the records and freelist to STDOUT in an almost human readable
429     form.
430    
431     =item $tdb->printfreelist
432    
433     Dump the freelist to STDOUT.
434    
435     =back
436    
437     =head2 EXPORTS
438    
439     Nothing constants are exported by default.
440    
441     The tag C<:all> exports allpo of the constants.
442    
443     Individually or with the tag C<:flags>:
444    
445     DEFAULT
446     CLEAR_IF_FIRST
447     INTERNAL
448     NOLOCK
449     NOMMAP
450     CONVERT
451     BIGENDIAN
452     NOSYNC
453     SEQNUM
454     VOLATILE
455     ALLOW_NESTING
456     DISALLOW_NESTING
457     INCOMPATIBLE_HASH
458     MUTEX_LOCKING
459    
460     Individually or with the tag C<:insert>:
461    
462     REPLACE
463     INSERT
464     MODIFY
465    
466     Individually or with the tag C<:error>:
467    
468     SUCCESS
469     ERR_CORRUPT
470     ERR_IO
471     ERR_LOCK
472     ERR_OOM
473     ERR_EXISTS
474     ERR_NOLOCK
475     ERR_LOCK_TIMEOUT
476     ERR_NOEXIST
477     ERR_EINVAL
478     ERR_RDONLY
479    
480     Individually or with the tag C<:debug>:
481    
482     DEBUG_FATAL
483     DEBUG_ERROR
484     DEBUG_WARNING
485     DEBUG_TRACE
486    
487 root 1.2 =head1 UNICODE HANDLING
488    
489     TDB databases can only store octet strings. Unlike most other database
490     interfaces, TDB_FileX will safely handle Perl strings by downgrading
491     them. Perl will warn about strings that cnanot be downgraded.
492    
493     =head1 DISK USAGE
494    
495     TDB databses use 24 octets for every key-value pair, plus the octet size
496     of the key and sata, e.g. the pair "key" => "value" takes up 24+3+5 octets
497     on disk.
498    
499     The database header is 168 octets (if I haven't miscounted).
500    
501     Each hashtable entry is 4 octets, and one more than the hash size is
502     allocated, so the hahs table size is (hash_size + 1) * 4.
503    
504     =head1 LIMITATIONS
505    
506     =head2 Database Size
507    
508     As far as I can see, TDB dastabases are limited to 4 GB.
509    
510     =head2 Hash Functions
511    
512     Hash functioons need to be set globally - they are limited to a maximum of
513     4, but this can be easily extended, but requires source code editing. This
514     is due to a limitation of the TDB C API.
515    
516     =head2 No recoivery after failed C<reopoen_all>
517 root 1.1
518     There is no way to survive an error during C<reopen_all>.
519     Unfortunately this is a limitation in the TDB C API.
520    
521     =head1 SEE ALSO
522    
523     tdb(3), L<perltie>.
524    
525     =head1 AUTHOR
526    
527     Angus Lees, E<lt>gus@inodes.org>
528    
529     Currently maintained by Marc A. Lehmann <schmorp@schmorp.de>
530     http://home.schmorp.de/
531    
532     =cut