ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/IO-AIO/AIO.pm
(Generate patch)

Comparing IO-AIO/AIO.pm (file contents):
Revision 1.41 by root, Wed Sep 7 17:41:17 2005 UTC vs.
Revision 1.56 by root, Sun Oct 22 00:53:47 2006 UTC

15 15
16 aio_read $fh, 30000, 1024, $buffer, 0, sub { 16 aio_read $fh, 30000, 1024, $buffer, 0, sub {
17 $_[0] > 0 or die "read error: $!"; 17 $_[0] > 0 or die "read error: $!";
18 }; 18 };
19 19
20 # Event 20 # version 2+ has request and group objects
21 use IO::AIO 2;
22
23 my $req = aio_unlink "/tmp/file", sub { };
24 $req->cancel; # cancel request if still in queue
25
26 my $grp = aio_group sub { print "all stats done\n" };
27 add $grp aio_stat "..." for ...;
28
29 # AnyEvent integration
30 open my $fh, "<&=" . IO::AIO::poll_fileno or die "$!";
31 my $w = AnyEvent->io (fh => $fh, poll => 'r', cb => sub { IO::AIO::poll_cb });
32
33 # Event integration
21 Event->io (fd => IO::AIO::poll_fileno, 34 Event->io (fd => IO::AIO::poll_fileno,
22 poll => 'r', 35 poll => 'r',
23 cb => \&IO::AIO::poll_cb); 36 cb => \&IO::AIO::poll_cb);
24 37
25 # Glib/Gtk2 38 # Glib/Gtk2 integration
26 add_watch Glib::IO IO::AIO::poll_fileno, 39 add_watch Glib::IO IO::AIO::poll_fileno,
27 in => sub { IO::AIO::poll_cb; 1 }; 40 in => sub { IO::AIO::poll_cb; 1 };
28 41
29 # Tk 42 # Tk integration
30 Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "", 43 Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "",
31 readable => \&IO::AIO::poll_cb); 44 readable => \&IO::AIO::poll_cb);
32 45
33 # Danga::Socket 46 # Danga::Socket integration
34 Danga::Socket->AddOtherFds (IO::AIO::poll_fileno => 47 Danga::Socket->AddOtherFds (IO::AIO::poll_fileno =>
35 \&IO::AIO::poll_cb); 48 \&IO::AIO::poll_cb);
36
37 49
38=head1 DESCRIPTION 50=head1 DESCRIPTION
39 51
40This module implements asynchronous I/O using whatever means your 52This module implements asynchronous I/O using whatever means your
41operating system supports. 53operating system supports.
57=cut 69=cut
58 70
59package IO::AIO; 71package IO::AIO;
60 72
61no warnings; 73no warnings;
74use strict 'vars';
62 75
63use base 'Exporter'; 76use base 'Exporter';
64 77
65use Fcntl ();
66
67BEGIN { 78BEGIN {
68 $VERSION = '1.61'; 79 our $VERSION = '2.0';
69 80
70 @EXPORT = qw(aio_sendfile aio_read aio_write aio_open aio_close aio_stat 81 our @EXPORT = qw(aio_sendfile aio_read aio_write aio_open aio_close aio_stat
71 aio_lstat aio_unlink aio_rmdir aio_readdir aio_scandir aio_symlink 82 aio_lstat aio_unlink aio_rmdir aio_readdir aio_scandir aio_symlink
72 aio_fsync aio_fdatasync aio_readahead); 83 aio_fsync aio_fdatasync aio_readahead aio_rename aio_link aio_move
84 aio_group);
73 @EXPORT_OK = qw(poll_fileno poll_cb min_parallel max_parallel 85 our @EXPORT_OK = qw(poll_fileno poll_cb min_parallel max_parallel max_outstanding nreqs);
74 max_outstanding nreqs); 86
87 @IO::AIO::GRP::ISA = 'IO::AIO::REQ';
75 88
76 require XSLoader; 89 require XSLoader;
77 XSLoader::load IO::AIO, $VERSION; 90 XSLoader::load ("IO::AIO", $VERSION);
78} 91}
79 92
80=head1 FUNCTIONS 93=head1 FUNCTIONS
81 94
82=head2 AIO FUNCTIONS 95=head2 AIO FUNCTIONS
89perl, which usually delivers "false") as it's sole argument when the given 102perl, which usually delivers "false") as it's sole argument when the given
90syscall has been executed asynchronously. 103syscall has been executed asynchronously.
91 104
92All functions expecting a filehandle keep a copy of the filehandle 105All functions expecting a filehandle keep a copy of the filehandle
93internally until the request has finished. 106internally until the request has finished.
107
108All requests return objects of type L<IO::AIO::REQ> that allow further
109manipulation of those requests while they are in-flight.
94 110
95The pathnames you pass to these routines I<must> be absolute and 111The pathnames you pass to these routines I<must> be absolute and
96encoded in byte form. The reason for the former is that at the time the 112encoded in byte form. The reason for the former is that at the time the
97request is being executed, the current working directory could have 113request is being executed, the current working directory could have
98changed. Alternatively, you can make sure that you never change the 114changed. Alternatively, you can make sure that you never change the
164 aio_read $fh, 7, 15, $buffer, 0, sub { 180 aio_read $fh, 7, 15, $buffer, 0, sub {
165 $_[0] > 0 or die "read error: $!"; 181 $_[0] > 0 or die "read error: $!";
166 print "read $_[0] bytes: <$buffer>\n"; 182 print "read $_[0] bytes: <$buffer>\n";
167 }; 183 };
168 184
185=item aio_move $srcpath, $dstpath, $callback->($status)
186
187Try to move the I<file> (directories not supported as either source or
188destination) from C<$srcpath> to C<$dstpath> and call the callback with
189the C<0> (error) or C<-1> ok.
190
191This is a composite request that tries to rename(2) the file first. If
192rename files with C<EXDEV>, it creates the destination file with mode 0200
193and copies the contents of the source file into it using C<aio_sendfile>,
194followed by restoring atime, mtime, access mode and uid/gid, in that
195order, and unlinking the C<$srcpath>.
196
197If an error occurs, the partial destination file will be unlinked, if
198possible, except when setting atime, mtime, access mode and uid/gid, where
199errors are being ignored.
200
201=cut
202
203sub aio_move($$$) {
204 my ($src, $dst, $cb) = @_;
205
206 my $grp = aio_group;
207
208 add $grp aio_rename $src, $dst, sub {
209 if ($_[0] && $! == EXDEV) {
210 add $grp aio_open $src, O_RDONLY, 0, sub {
211 if (my $src_fh = $_[0]) {
212 my @stat = stat $src_fh;
213
214 add $grp aio_open $dst, O_WRONLY, 0200, sub {
215 if (my $dst_fh = $_[0]) {
216 add $grp aio_sendfile $dst_fh, $src_fh, 0, $stat[7], sub {
217 close $src_fh;
218
219 if ($_[0] == $stat[7]) {
220 utime $stat[8], $stat[9], $dst;
221 chmod $stat[2] & 07777, $dst_fh;
222 chown $stat[4], $stat[5], $dst_fh;
223 close $dst_fh;
224
225 add $grp aio_unlink $src, sub {
226 $cb->($_[0]);
227 };
228 } else {
229 my $errno = $!;
230 add $grp aio_unlink $dst, sub {
231 $! = $errno;
232 $cb->(-1);
233 };
234 }
235 };
236 } else {
237 $cb->(-1);
238 }
239 },
240
241 } else {
242 $cb->(-1);
243 }
244 };
245 } else {
246 $cb->($_[0]);
247 }
248 };
249
250 $grp
251}
252
169=item aio_sendfile $out_fh, $in_fh, $in_offset, $length, $callback->($retval) 253=item aio_sendfile $out_fh, $in_fh, $in_offset, $length, $callback->($retval)
170 254
171Tries to copy C<$length> bytes from C<$in_fh> to C<$out_fh>. It starts 255Tries to copy C<$length> bytes from C<$in_fh> to C<$out_fh>. It starts
172reading at byte offset C<$in_offset>, and starts writing at the current 256reading at byte offset C<$in_offset>, and starts writing at the current
173file offset of C<$out_fh>. Because of that, it is not safe to issue more 257file offset of C<$out_fh>. Because of that, it is not safe to issue more
228=item aio_unlink $pathname, $callback->($status) 312=item aio_unlink $pathname, $callback->($status)
229 313
230Asynchronously unlink (delete) a file and call the callback with the 314Asynchronously unlink (delete) a file and call the callback with the
231result code. 315result code.
232 316
317=item aio_link $srcpath, $dstpath, $callback->($status)
318
319Asynchronously create a new link to the existing object at C<$srcpath> at
320the path C<$dstpath> and call the callback with the result code.
321
322=item aio_symlink $srcpath, $dstpath, $callback->($status)
323
324Asynchronously create a new symbolic link to the existing object at C<$srcpath> at
325the path C<$dstpath> and call the callback with the result code.
326
327=item aio_rename $srcpath, $dstpath, $callback->($status)
328
329Asynchronously rename the object at C<$srcpath> to C<$dstpath>, just as
330rename(2) and call the callback with the result code.
331
233=item aio_rmdir $pathname, $callback->($status) 332=item aio_rmdir $pathname, $callback->($status)
234 333
235Asynchronously rmdir (delete) a directory and call the callback with the 334Asynchronously rmdir (delete) a directory and call the callback with the
236result code. 335result code.
237 336
238=item aio_readdir $pathname $callback->($entries) 337=item aio_readdir $pathname, $callback->($entries)
239 338
240Unlike the POSIX call of the same name, C<aio_readdir> reads an entire 339Unlike the POSIX call of the same name, C<aio_readdir> reads an entire
241directory (i.e. opendir + readdir + closedir). The entries will not be 340directory (i.e. opendir + readdir + closedir). The entries will not be
242sorted, and will B<NOT> include the C<.> and C<..> entries. 341sorted, and will B<NOT> include the C<.> and C<..> entries.
243 342
244The callback a single argument which is either C<undef> or an array-ref 343The callback a single argument which is either C<undef> or an array-ref
245with the filenames. 344with the filenames.
246 345
247=item aio_scandir $path, $maxreq, $callback->($dirs, $nondirs) 346=item aio_scandir $path, $maxreq, $callback->($dirs, $nondirs)
248 347
249Scans a directory (similar to C<aio_readdir>) and tries to separate the 348Scans a directory (similar to C<aio_readdir>) but additionally tries to
250entries of directory C<$path> into two sets of names, ones you can recurse 349separate the entries of directory C<$path> into two sets of names, ones
251into (directories), and ones you cannot recurse into (everything else). 350you can recurse into (directories or links to them), and ones you cannot
351recurse into (everything else).
252 352
253C<aio_scandir> is a composite request that consists of many 353C<aio_scandir> is a composite request that consists of many sub
254aio-primitives. C<$maxreq> specifies the maximum number of outstanding 354requests. C<$maxreq> specifies the maximum number of outstanding aio
255aio requests that this function generates. If it is C<< <= 0 >>, then a 355requests that this function generates. If it is C<< <= 0 >>, then a
256suitable default will be chosen (currently 8). 356suitable default will be chosen (currently 8).
257 357
258On error, the callback is called without arguments, otherwise it receives 358On error, the callback is called without arguments, otherwise it receives
259two array-refs with path-relative entry names. 359two array-refs with path-relative entry names.
260 360
269Implementation notes. 369Implementation notes.
270 370
271The C<aio_readdir> cannot be avoided, but C<stat()>'ing every entry can. 371The C<aio_readdir> cannot be avoided, but C<stat()>'ing every entry can.
272 372
273After reading the directory, the modification time, size etc. of the 373After reading the directory, the modification time, size etc. of the
274directory before and after the readdir is checked, and if they match, the 374directory before and after the readdir is checked, and if they match (and
275link count will be used to decide how many entries are directories (if 375isn't the current time), the link count will be used to decide how many
276>= 2). Otherwise, no knowledge of the number of subdirectories will be 376entries are directories (if >= 2). Otherwise, no knowledge of the number
277assumed. 377of subdirectories will be assumed.
278 378
279Then entires will be sorted into likely directories (everything without a 379Then entries will be sorted into likely directories (everything without
280non-initial dot) and likely non-directories (everything else). Then every 380a non-initial dot currently) and likely non-directories (everything
281entry + C</.> will be C<stat>'ed, likely directories first. This is often 381else). Then every entry plus an appended C</.> will be C<stat>'ed,
382likely directories first. If that succeeds, it assumes that the entry
383is a directory or a symlink to directory (which will be checked
384seperately). This is often faster than stat'ing the entry itself because
282faster because filesystems might detect the type of the entry without 385filesystems might detect the type of the entry without reading the inode
283reading the inode data (e.g. ext2s filetype feature). If that succeeds, 386data (e.g. ext2fs filetype feature).
284it assumes that the entry is a directory or a symlink to directory (which
285will be checked seperately).
286 387
287If the known number of directories has been reached, the rest of the 388If the known number of directories (link count - 2) has been reached, the
288entries is assumed to be non-directories. 389rest of the entries is assumed to be non-directories.
390
391This only works with certainty on POSIX (= UNIX) filesystems, which
392fortunately are the vast majority of filesystems around.
393
394It will also likely work on non-POSIX filesystems with reduced efficiency
395as those tend to return 0 or 1 as link counts, which disables the
396directory counting heuristic.
289 397
290=cut 398=cut
291 399
292sub aio_scandir($$$) { 400sub aio_scandir($$$) {
293 my ($path, $maxreq, $cb) = @_; 401 my ($path, $maxreq, $cb) = @_;
294 402
403 my $grp = aio_group;
404
295 $maxreq = 8 if $maxreq <= 0; 405 $maxreq = 8 if $maxreq <= 0;
296 406
297 # stat once 407 # stat once
298 aio_stat $path, sub { 408 add $grp aio_stat $path, sub {
299 $cb->() if $_[0]; 409 return $cb->() if $_[0];
410 my $now = time;
300 my $hash1 = join ":", (stat _)[0,1,3,7,9]; 411 my $hash1 = join ":", (stat _)[0,1,3,7,9];
301 412
302 # read the directory entries 413 # read the directory entries
303 aio_readdir $path, sub { 414 add $grp aio_readdir $path, sub {
304 my $entries = shift 415 my $entries = shift
305 or return $cb->(); 416 or return $cb->();
306 417
307 # stat the dir another time 418 # stat the dir another time
308 aio_stat $path, sub { 419 add $grp aio_stat $path, sub {
309 my $hash2 = join ":", (stat _)[0,1,3,7,9]; 420 my $hash2 = join ":", (stat _)[0,1,3,7,9];
310 421
311 my $ndirs; 422 my $ndirs;
312 423
313 # take the slow route if anything looks fishy 424 # take the slow route if anything looks fishy
314 if ($hash1 ne $hash2) { 425 if ($hash1 ne $hash2 or (stat _)[9] == $now) {
315 $ndirs = -1; 426 $ndirs = -1;
316 } else { 427 } else {
317 # if nlink == 2, we are finished 428 # if nlink == 2, we are finished
318 # on non-posix-fs's, we rely on nlink < 2 429 # on non-posix-fs's, we rely on nlink < 2
319 $ndirs = (stat _)[3] - 2 430 $ndirs = (stat _)[3] - 2
320 or $cb->([], $entries); 431 or return $cb->([], $entries);
321 } 432 }
322 433
323 # sort into likely dirs and likely nondirs 434 # sort into likely dirs and likely nondirs
324 # dirs == files without ".", short entries first 435 # dirs == files without ".", short entries first
325 $entries = [map $_->[0], 436 $entries = [map $_->[0],
335 $schedcb = sub { 446 $schedcb = sub {
336 if (@$entries) { 447 if (@$entries) {
337 if ($nreq < $maxreq) { 448 if ($nreq < $maxreq) {
338 my $ent = pop @$entries; 449 my $ent = pop @$entries;
339 $nreq++; 450 $nreq++;
340 aio_stat "$path/$ent/.", sub { $statcb->($_[0], $ent) }; 451 add $grp aio_stat "$path/$ent/.", sub { $statcb->($_[0], $ent) };
341 } 452 }
342 } elsif (!$nreq) { 453 } elsif (!$nreq) {
343 # finished 454 # finished
344 undef $statcb; 455 undef $statcb;
345 undef $schedcb; 456 undef $schedcb;
346 $cb->(\@dirs, \@nondirs); 457 $cb->(\@dirs, \@nondirs) if $cb;
347 undef $cb; 458 undef $cb;
348 } 459 }
349 }; 460 };
350 $statcb = sub { 461 $statcb = sub {
351 my ($status, $entry) = @_; 462 my ($status, $entry) = @_;
354 $nreq--; 465 $nreq--;
355 push @nondirs, $entry; 466 push @nondirs, $entry;
356 &$schedcb; 467 &$schedcb;
357 } else { 468 } else {
358 # need to check for real directory 469 # need to check for real directory
359 aio_lstat "$path/$entry", sub { 470 add $grp aio_lstat "$path/$entry", sub {
360 $nreq--; 471 $nreq--;
361 472
362 if (-d _) { 473 if (-d _) {
363 push @dirs, $entry; 474 push @dirs, $entry;
364 475
377 488
378 &$schedcb while @$entries && $nreq < $maxreq; 489 &$schedcb while @$entries && $nreq < $maxreq;
379 }; 490 };
380 }; 491 };
381 }; 492 };
493
494 $grp
382} 495}
383 496
384=item aio_fsync $fh, $callback->($status) 497=item aio_fsync $fh, $callback->($status)
385 498
386Asynchronously call fsync on the given filehandle and call the callback 499Asynchronously call fsync on the given filehandle and call the callback
391Asynchronously call fdatasync on the given filehandle and call the 504Asynchronously call fdatasync on the given filehandle and call the
392callback with the fdatasync result code. 505callback with the fdatasync result code.
393 506
394If this call isn't available because your OS lacks it or it couldn't be 507If this call isn't available because your OS lacks it or it couldn't be
395detected, it will be emulated by calling C<fsync> instead. 508detected, it will be emulated by calling C<fsync> instead.
509
510=item aio_group $callback->()
511
512[EXPERIMENTAL]
513
514This is a very special aio request: Instead of doing something, it is a
515container for other aio requests, which is useful if you want to bundle
516many requests into a single, composite, request.
517
518Returns an object of class L<IO::AIO::GRP>. See its documentation below
519for more info.
520
521Example:
522
523 my $grp = aio_group sub {
524 print "all stats done\n";
525 };
526
527 add $grp
528 (aio_stat ...),
529 (aio_stat ...),
530 ...;
531
532=item IO::AIO::aio_sleep $fractional_seconds, $callback->() *NOT EXPORTED*
533
534Mainly used for debugging and benchmarking, this aio request puts one of
535the request workers to sleep for the given time.
536
537While it is theoretically handy to have simple I/O scheduling requests
538like sleep and file handle readable/writable, the overhead this creates
539is immense, so do not use this function except to put your application
540under artificial I/O pressure.
541
542=back
543
544=head2 IO::AIO::REQ CLASS
545
546All non-aggregate C<aio_*> functions return an object of this class when
547called in non-void context.
548
549A request always moves through the following five states in its lifetime,
550in order: B<ready> (request has been created, but has not been executed
551yet), B<execute> (request is currently being executed), B<pending>
552(request has been executed but callback has not been called yet),
553B<result> (results are being processed synchronously, includes calling the
554callback) and B<done> (request has reached the end of its lifetime and
555holds no resources anymore).
556
557=over 4
558
559=item $req->cancel
560
561Cancels the request, if possible. Has the effect of skipping execution
562when entering the B<execute> state and skipping calling the callback when
563entering the the B<result> state, but will leave the request otherwise
564untouched. That means that requests that currently execute will not be
565stopped and resources held by the request will not be freed prematurely.
566
567=back
568
569=head2 IO::AIO::GRP CLASS
570
571This class is a subclass of L<IO::AIO::REQ>, so all its methods apply to
572objects of this class, too.
573
574A IO::AIO::GRP object is a special request that can contain multiple other
575aio requests.
576
577You create one by calling the C<aio_group> constructing function with a
578callback that will be called when all contained requests have entered the
579C<done> state:
580
581 my $grp = aio_group sub {
582 print "all requests are done\n";
583 };
584
585You add requests by calling the C<add> method with one or more
586C<IO::AIO::REQ> objects:
587
588 $grp->add (aio_unlink "...");
589
590 add $grp aio_stat "...", sub { ... };
591
592This makes it very easy to create composite requests (see the source of
593C<aio_move> for an application) that work and feel like simple requests.
594
595The IO::AIO::GRP objects will be cleaned up during calls to
596C<IO::AIO::poll_cb>, just like any other request.
597
598They can be canceled like any other request. Canceling will cancel not
599just the request itself, but also all requests it contains.
600
601They can also can also be added to other IO::AIO::GRP objects.
602
603Their lifetime, simplified, looks like this: when they are empty, they
604will finish very quickly. If they contain only requests that are in the
605C<done> state, they will also finish. Otherwise they will continue to
606exist.
607
608=over 4
609
610=item $grp->add (...)
611
612=item add $grp ...
613
614Add one or more
615Cancels the request, if possible. Has the effect of skipping execution
616when entering the B<execute> state and skipping calling the callback when
617entering the the B<result> state, but will leave the request otherwise
618untouched. That means that requests that currently execute will not be
619stopped and resources held by the request will not be freed prematurely.
396 620
397=back 621=back
398 622
399=head2 SUPPORT FUNCTIONS 623=head2 SUPPORT FUNCTIONS
400 624
528} 752}
529 753
5301; 7541;
531 755
532=head2 FORK BEHAVIOUR 756=head2 FORK BEHAVIOUR
757
758This module should do "the right thing" when the process using it forks:
533 759
534Before the fork, IO::AIO enters a quiescent state where no requests 760Before the fork, IO::AIO enters a quiescent state where no requests
535can be added in other threads and no results will be processed. After 761can be added in other threads and no results will be processed. After
536the fork the parent simply leaves the quiescent state and continues 762the fork the parent simply leaves the quiescent state and continues
537request/result processing, while the child clears the request/result 763request/result processing, while the child clears the request/result
538queue (so the requests started before the fork will only be handled in 764queue (so the requests started before the fork will only be handled in
539the parent). Threats will be started on demand until the limit ste in the 765the parent). Threads will be started on demand until the limit ste in the
540parent process has been reached again. 766parent process has been reached again.
541 767
768In short: the parent will, after a short pause, continue as if fork had
769not been called, while the child will act as if IO::AIO has not been used
770yet.
771
542=head1 SEE ALSO 772=head1 SEE ALSO
543 773
544L<Coro>, L<Linux::AIO>. 774L<Coro>, L<Linux::AIO> (obsolete).
545 775
546=head1 AUTHOR 776=head1 AUTHOR
547 777
548 Marc Lehmann <schmorp@schmorp.de> 778 Marc Lehmann <schmorp@schmorp.de>
549 http://home.schmorp.de/ 779 http://home.schmorp.de/

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines