ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/IO-AIO/AIO.pm
(Generate patch)

Comparing IO-AIO/AIO.pm (file contents):
Revision 1.41 by root, Wed Sep 7 17:41:17 2005 UTC vs.
Revision 1.59 by root, Sun Oct 22 10:33:26 2006 UTC

15 15
16 aio_read $fh, 30000, 1024, $buffer, 0, sub { 16 aio_read $fh, 30000, 1024, $buffer, 0, sub {
17 $_[0] > 0 or die "read error: $!"; 17 $_[0] > 0 or die "read error: $!";
18 }; 18 };
19 19
20 # Event 20 # version 2+ has request and group objects
21 use IO::AIO 2;
22
23 my $req = aio_unlink "/tmp/file", sub { };
24 $req->cancel; # cancel request if still in queue
25
26 my $grp = aio_group sub { print "all stats done\n" };
27 add $grp aio_stat "..." for ...;
28
29 # AnyEvent integration
30 open my $fh, "<&=" . IO::AIO::poll_fileno or die "$!";
31 my $w = AnyEvent->io (fh => $fh, poll => 'r', cb => sub { IO::AIO::poll_cb });
32
33 # Event integration
21 Event->io (fd => IO::AIO::poll_fileno, 34 Event->io (fd => IO::AIO::poll_fileno,
22 poll => 'r', 35 poll => 'r',
23 cb => \&IO::AIO::poll_cb); 36 cb => \&IO::AIO::poll_cb);
24 37
25 # Glib/Gtk2 38 # Glib/Gtk2 integration
26 add_watch Glib::IO IO::AIO::poll_fileno, 39 add_watch Glib::IO IO::AIO::poll_fileno,
27 in => sub { IO::AIO::poll_cb; 1 }; 40 in => sub { IO::AIO::poll_cb; 1 };
28 41
29 # Tk 42 # Tk integration
30 Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "", 43 Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "",
31 readable => \&IO::AIO::poll_cb); 44 readable => \&IO::AIO::poll_cb);
32 45
33 # Danga::Socket 46 # Danga::Socket integration
34 Danga::Socket->AddOtherFds (IO::AIO::poll_fileno => 47 Danga::Socket->AddOtherFds (IO::AIO::poll_fileno =>
35 \&IO::AIO::poll_cb); 48 \&IO::AIO::poll_cb);
36
37 49
38=head1 DESCRIPTION 50=head1 DESCRIPTION
39 51
40This module implements asynchronous I/O using whatever means your 52This module implements asynchronous I/O using whatever means your
41operating system supports. 53operating system supports.
57=cut 69=cut
58 70
59package IO::AIO; 71package IO::AIO;
60 72
61no warnings; 73no warnings;
74use strict 'vars';
62 75
63use base 'Exporter'; 76use base 'Exporter';
64 77
65use Fcntl ();
66
67BEGIN { 78BEGIN {
68 $VERSION = '1.61'; 79 our $VERSION = '2.0';
69 80
70 @EXPORT = qw(aio_sendfile aio_read aio_write aio_open aio_close aio_stat 81 our @EXPORT = qw(aio_sendfile aio_read aio_write aio_open aio_close aio_stat
71 aio_lstat aio_unlink aio_rmdir aio_readdir aio_scandir aio_symlink 82 aio_lstat aio_unlink aio_rmdir aio_readdir aio_scandir aio_symlink
72 aio_fsync aio_fdatasync aio_readahead); 83 aio_fsync aio_fdatasync aio_readahead aio_rename aio_link aio_move
84 aio_group);
73 @EXPORT_OK = qw(poll_fileno poll_cb min_parallel max_parallel 85 our @EXPORT_OK = qw(poll_fileno poll_cb min_parallel max_parallel max_outstanding nreqs);
74 max_outstanding nreqs); 86
87 @IO::AIO::GRP::ISA = 'IO::AIO::REQ';
75 88
76 require XSLoader; 89 require XSLoader;
77 XSLoader::load IO::AIO, $VERSION; 90 XSLoader::load ("IO::AIO", $VERSION);
78} 91}
79 92
80=head1 FUNCTIONS 93=head1 FUNCTIONS
81 94
82=head2 AIO FUNCTIONS 95=head2 AIO FUNCTIONS
89perl, which usually delivers "false") as it's sole argument when the given 102perl, which usually delivers "false") as it's sole argument when the given
90syscall has been executed asynchronously. 103syscall has been executed asynchronously.
91 104
92All functions expecting a filehandle keep a copy of the filehandle 105All functions expecting a filehandle keep a copy of the filehandle
93internally until the request has finished. 106internally until the request has finished.
107
108All requests return objects of type L<IO::AIO::REQ> that allow further
109manipulation of those requests while they are in-flight.
94 110
95The pathnames you pass to these routines I<must> be absolute and 111The pathnames you pass to these routines I<must> be absolute and
96encoded in byte form. The reason for the former is that at the time the 112encoded in byte form. The reason for the former is that at the time the
97request is being executed, the current working directory could have 113request is being executed, the current working directory could have
98changed. Alternatively, you can make sure that you never change the 114changed. Alternatively, you can make sure that you never change the
164 aio_read $fh, 7, 15, $buffer, 0, sub { 180 aio_read $fh, 7, 15, $buffer, 0, sub {
165 $_[0] > 0 or die "read error: $!"; 181 $_[0] > 0 or die "read error: $!";
166 print "read $_[0] bytes: <$buffer>\n"; 182 print "read $_[0] bytes: <$buffer>\n";
167 }; 183 };
168 184
185=item aio_move $srcpath, $dstpath, $callback->($status)
186
187[EXPERIMENTAL due to internal aio_group use]
188
189Try to move the I<file> (directories not supported as either source or
190destination) from C<$srcpath> to C<$dstpath> and call the callback with
191the C<0> (error) or C<-1> ok.
192
193This is a composite request that tries to rename(2) the file first. If
194rename files with C<EXDEV>, it creates the destination file with mode 0200
195and copies the contents of the source file into it using C<aio_sendfile>,
196followed by restoring atime, mtime, access mode and uid/gid, in that
197order, and unlinking the C<$srcpath>.
198
199If an error occurs, the partial destination file will be unlinked, if
200possible, except when setting atime, mtime, access mode and uid/gid, where
201errors are being ignored.
202
203=cut
204
205sub aio_move($$$) {
206 my ($src, $dst, $cb) = @_;
207
208 my $grp = aio_group $cb;
209
210 add $grp aio_rename $src, $dst, sub {
211 if ($_[0] && $! == EXDEV) {
212 add $grp aio_open $src, O_RDONLY, 0, sub {
213 if (my $src_fh = $_[0]) {
214 my @stat = stat $src_fh;
215
216 add $grp aio_open $dst, O_WRONLY, 0200, sub {
217 if (my $dst_fh = $_[0]) {
218 add $grp aio_sendfile $dst_fh, $src_fh, 0, $stat[7], sub {
219 close $src_fh;
220
221 if ($_[0] == $stat[7]) {
222 utime $stat[8], $stat[9], $dst;
223 chmod $stat[2] & 07777, $dst_fh;
224 chown $stat[4], $stat[5], $dst_fh;
225 close $dst_fh;
226
227 add $grp aio_unlink $src, sub {
228 $grp->result ($_[0]);
229 };
230 } else {
231 my $errno = $!;
232 add $grp aio_unlink $dst, sub {
233 $! = $errno;
234 $grp->result (-1);
235 };
236 }
237 };
238 } else {
239 $grp->result (-1);
240 }
241 },
242
243 } else {
244 $grp->result (-1);
245 }
246 };
247 } else {
248 $grp->result ($_[0]);
249 }
250 };
251
252 $grp
253}
254
169=item aio_sendfile $out_fh, $in_fh, $in_offset, $length, $callback->($retval) 255=item aio_sendfile $out_fh, $in_fh, $in_offset, $length, $callback->($retval)
170 256
171Tries to copy C<$length> bytes from C<$in_fh> to C<$out_fh>. It starts 257Tries to copy C<$length> bytes from C<$in_fh> to C<$out_fh>. It starts
172reading at byte offset C<$in_offset>, and starts writing at the current 258reading at byte offset C<$in_offset>, and starts writing at the current
173file offset of C<$out_fh>. Because of that, it is not safe to issue more 259file offset of C<$out_fh>. Because of that, it is not safe to issue more
228=item aio_unlink $pathname, $callback->($status) 314=item aio_unlink $pathname, $callback->($status)
229 315
230Asynchronously unlink (delete) a file and call the callback with the 316Asynchronously unlink (delete) a file and call the callback with the
231result code. 317result code.
232 318
319=item aio_link $srcpath, $dstpath, $callback->($status)
320
321Asynchronously create a new link to the existing object at C<$srcpath> at
322the path C<$dstpath> and call the callback with the result code.
323
324=item aio_symlink $srcpath, $dstpath, $callback->($status)
325
326Asynchronously create a new symbolic link to the existing object at C<$srcpath> at
327the path C<$dstpath> and call the callback with the result code.
328
329=item aio_rename $srcpath, $dstpath, $callback->($status)
330
331Asynchronously rename the object at C<$srcpath> to C<$dstpath>, just as
332rename(2) and call the callback with the result code.
333
233=item aio_rmdir $pathname, $callback->($status) 334=item aio_rmdir $pathname, $callback->($status)
234 335
235Asynchronously rmdir (delete) a directory and call the callback with the 336Asynchronously rmdir (delete) a directory and call the callback with the
236result code. 337result code.
237 338
238=item aio_readdir $pathname $callback->($entries) 339=item aio_readdir $pathname, $callback->($entries)
239 340
240Unlike the POSIX call of the same name, C<aio_readdir> reads an entire 341Unlike the POSIX call of the same name, C<aio_readdir> reads an entire
241directory (i.e. opendir + readdir + closedir). The entries will not be 342directory (i.e. opendir + readdir + closedir). The entries will not be
242sorted, and will B<NOT> include the C<.> and C<..> entries. 343sorted, and will B<NOT> include the C<.> and C<..> entries.
243 344
244The callback a single argument which is either C<undef> or an array-ref 345The callback a single argument which is either C<undef> or an array-ref
245with the filenames. 346with the filenames.
246 347
247=item aio_scandir $path, $maxreq, $callback->($dirs, $nondirs) 348=item aio_scandir $path, $maxreq, $callback->($dirs, $nondirs)
248 349
350[EXPERIMENTAL due to internal aio_group use]
351
249Scans a directory (similar to C<aio_readdir>) and tries to separate the 352Scans a directory (similar to C<aio_readdir>) but additionally tries to
250entries of directory C<$path> into two sets of names, ones you can recurse 353separate the entries of directory C<$path> into two sets of names, ones
251into (directories), and ones you cannot recurse into (everything else). 354you can recurse into (directories or links to them), and ones you cannot
355recurse into (everything else).
252 356
253C<aio_scandir> is a composite request that consists of many 357C<aio_scandir> is a composite request that consists of many sub
254aio-primitives. C<$maxreq> specifies the maximum number of outstanding 358requests. C<$maxreq> specifies the maximum number of outstanding aio
255aio requests that this function generates. If it is C<< <= 0 >>, then a 359requests that this function generates. If it is C<< <= 0 >>, then a
256suitable default will be chosen (currently 8). 360suitable default will be chosen (currently 8).
257 361
258On error, the callback is called without arguments, otherwise it receives 362On error, the callback is called without arguments, otherwise it receives
259two array-refs with path-relative entry names. 363two array-refs with path-relative entry names.
260 364
269Implementation notes. 373Implementation notes.
270 374
271The C<aio_readdir> cannot be avoided, but C<stat()>'ing every entry can. 375The C<aio_readdir> cannot be avoided, but C<stat()>'ing every entry can.
272 376
273After reading the directory, the modification time, size etc. of the 377After reading the directory, the modification time, size etc. of the
274directory before and after the readdir is checked, and if they match, the 378directory before and after the readdir is checked, and if they match (and
275link count will be used to decide how many entries are directories (if 379isn't the current time), the link count will be used to decide how many
276>= 2). Otherwise, no knowledge of the number of subdirectories will be 380entries are directories (if >= 2). Otherwise, no knowledge of the number
277assumed. 381of subdirectories will be assumed.
278 382
279Then entires will be sorted into likely directories (everything without a 383Then entries will be sorted into likely directories (everything without
280non-initial dot) and likely non-directories (everything else). Then every 384a non-initial dot currently) and likely non-directories (everything
281entry + C</.> will be C<stat>'ed, likely directories first. This is often 385else). Then every entry plus an appended C</.> will be C<stat>'ed,
386likely directories first. If that succeeds, it assumes that the entry
387is a directory or a symlink to directory (which will be checked
388seperately). This is often faster than stat'ing the entry itself because
282faster because filesystems might detect the type of the entry without 389filesystems might detect the type of the entry without reading the inode
283reading the inode data (e.g. ext2s filetype feature). If that succeeds, 390data (e.g. ext2fs filetype feature).
284it assumes that the entry is a directory or a symlink to directory (which
285will be checked seperately).
286 391
287If the known number of directories has been reached, the rest of the 392If the known number of directories (link count - 2) has been reached, the
288entries is assumed to be non-directories. 393rest of the entries is assumed to be non-directories.
394
395This only works with certainty on POSIX (= UNIX) filesystems, which
396fortunately are the vast majority of filesystems around.
397
398It will also likely work on non-POSIX filesystems with reduced efficiency
399as those tend to return 0 or 1 as link counts, which disables the
400directory counting heuristic.
289 401
290=cut 402=cut
291 403
292sub aio_scandir($$$) { 404sub aio_scandir($$$) {
293 my ($path, $maxreq, $cb) = @_; 405 my ($path, $maxreq, $cb) = @_;
294 406
407 my $grp = aio_group $cb;
408
295 $maxreq = 8 if $maxreq <= 0; 409 $maxreq = 8 if $maxreq <= 0;
296 410
297 # stat once 411 # stat once
298 aio_stat $path, sub { 412 add $grp aio_stat $path, sub {
299 $cb->() if $_[0]; 413 return $grp->result () if $_[0];
414 my $now = time;
300 my $hash1 = join ":", (stat _)[0,1,3,7,9]; 415 my $hash1 = join ":", (stat _)[0,1,3,7,9];
301 416
302 # read the directory entries 417 # read the directory entries
303 aio_readdir $path, sub { 418 add $grp aio_readdir $path, sub {
304 my $entries = shift 419 my $entries = shift
305 or return $cb->(); 420 or return $grp->result ();
306 421
307 # stat the dir another time 422 # stat the dir another time
308 aio_stat $path, sub { 423 add $grp aio_stat $path, sub {
309 my $hash2 = join ":", (stat _)[0,1,3,7,9]; 424 my $hash2 = join ":", (stat _)[0,1,3,7,9];
310 425
311 my $ndirs; 426 my $ndirs;
312 427
313 # take the slow route if anything looks fishy 428 # take the slow route if anything looks fishy
314 if ($hash1 ne $hash2) { 429 if ($hash1 ne $hash2 or (stat _)[9] == $now) {
315 $ndirs = -1; 430 $ndirs = -1;
316 } else { 431 } else {
317 # if nlink == 2, we are finished 432 # if nlink == 2, we are finished
318 # on non-posix-fs's, we rely on nlink < 2 433 # on non-posix-fs's, we rely on nlink < 2
319 $ndirs = (stat _)[3] - 2 434 $ndirs = (stat _)[3] - 2
320 or $cb->([], $entries); 435 or return $grp->result ([], $entries);
321 } 436 }
322 437
323 # sort into likely dirs and likely nondirs 438 # sort into likely dirs and likely nondirs
324 # dirs == files without ".", short entries first 439 # dirs == files without ".", short entries first
325 $entries = [map $_->[0], 440 $entries = [map $_->[0],
335 $schedcb = sub { 450 $schedcb = sub {
336 if (@$entries) { 451 if (@$entries) {
337 if ($nreq < $maxreq) { 452 if ($nreq < $maxreq) {
338 my $ent = pop @$entries; 453 my $ent = pop @$entries;
339 $nreq++; 454 $nreq++;
340 aio_stat "$path/$ent/.", sub { $statcb->($_[0], $ent) }; 455 add $grp aio_stat "$path/$ent/.", sub { $statcb->($_[0], $ent) };
341 } 456 }
342 } elsif (!$nreq) { 457 } elsif (!$nreq) {
343 # finished 458 # finished
344 undef $statcb; 459 undef $statcb;
345 undef $schedcb; 460 undef $schedcb;
346 $cb->(\@dirs, \@nondirs); 461 $grp->result (\@dirs, \@nondirs) if $cb;
347 undef $cb; 462 undef $cb;
348 } 463 }
349 }; 464 };
350 $statcb = sub { 465 $statcb = sub {
351 my ($status, $entry) = @_; 466 my ($status, $entry) = @_;
354 $nreq--; 469 $nreq--;
355 push @nondirs, $entry; 470 push @nondirs, $entry;
356 &$schedcb; 471 &$schedcb;
357 } else { 472 } else {
358 # need to check for real directory 473 # need to check for real directory
359 aio_lstat "$path/$entry", sub { 474 add $grp aio_lstat "$path/$entry", sub {
360 $nreq--; 475 $nreq--;
361 476
362 if (-d _) { 477 if (-d _) {
363 push @dirs, $entry; 478 push @dirs, $entry;
364 479
377 492
378 &$schedcb while @$entries && $nreq < $maxreq; 493 &$schedcb while @$entries && $nreq < $maxreq;
379 }; 494 };
380 }; 495 };
381 }; 496 };
497
498 $grp
382} 499}
383 500
384=item aio_fsync $fh, $callback->($status) 501=item aio_fsync $fh, $callback->($status)
385 502
386Asynchronously call fsync on the given filehandle and call the callback 503Asynchronously call fsync on the given filehandle and call the callback
391Asynchronously call fdatasync on the given filehandle and call the 508Asynchronously call fdatasync on the given filehandle and call the
392callback with the fdatasync result code. 509callback with the fdatasync result code.
393 510
394If this call isn't available because your OS lacks it or it couldn't be 511If this call isn't available because your OS lacks it or it couldn't be
395detected, it will be emulated by calling C<fsync> instead. 512detected, it will be emulated by calling C<fsync> instead.
513
514=item aio_group $callback->(...)
515
516[EXPERIMENTAL]
517
518This is a very special aio request: Instead of doing something, it is a
519container for other aio requests, which is useful if you want to bundle
520many requests into a single, composite, request.
521
522Returns an object of class L<IO::AIO::GRP>. See its documentation below
523for more info.
524
525Example:
526
527 my $grp = aio_group sub {
528 print "all stats done\n";
529 };
530
531 add $grp
532 (aio_stat ...),
533 (aio_stat ...),
534 ...;
535
536=item IO::AIO::aio_sleep $fractional_seconds, $callback->() *NOT EXPORTED*
537
538Mainly used for debugging and benchmarking, this aio request puts one of
539the request workers to sleep for the given time.
540
541While it is theoretically handy to have simple I/O scheduling requests
542like sleep and file handle readable/writable, the overhead this creates
543is immense, so do not use this function except to put your application
544under artificial I/O pressure.
545
546=back
547
548=head2 IO::AIO::REQ CLASS
549
550All non-aggregate C<aio_*> functions return an object of this class when
551called in non-void context.
552
553A request always moves through the following five states in its lifetime,
554in order: B<ready> (request has been created, but has not been executed
555yet), B<execute> (request is currently being executed), B<pending>
556(request has been executed but callback has not been called yet),
557B<result> (results are being processed synchronously, includes calling the
558callback) and B<done> (request has reached the end of its lifetime and
559holds no resources anymore).
560
561=over 4
562
563=item $req->cancel
564
565Cancels the request, if possible. Has the effect of skipping execution
566when entering the B<execute> state and skipping calling the callback when
567entering the the B<result> state, but will leave the request otherwise
568untouched. That means that requests that currently execute will not be
569stopped and resources held by the request will not be freed prematurely.
570
571=back
572
573=head2 IO::AIO::GRP CLASS
574
575This class is a subclass of L<IO::AIO::REQ>, so all its methods apply to
576objects of this class, too.
577
578A IO::AIO::GRP object is a special request that can contain multiple other
579aio requests.
580
581You create one by calling the C<aio_group> constructing function with a
582callback that will be called when all contained requests have entered the
583C<done> state:
584
585 my $grp = aio_group sub {
586 print "all requests are done\n";
587 };
588
589You add requests by calling the C<add> method with one or more
590C<IO::AIO::REQ> objects:
591
592 $grp->add (aio_unlink "...");
593
594 add $grp aio_stat "...", sub {
595 $_[0] or return $grp->result ("error");
596
597 # add another request dynamically, if first succeeded
598 add $grp aio_open "...", sub {
599 $grp->result ("ok");
600 };
601 };
602
603This makes it very easy to create composite requests (see the source of
604C<aio_move> for an application) that work and feel like simple requests.
605
606The IO::AIO::GRP objects will be cleaned up during calls to
607C<IO::AIO::poll_cb>, just like any other request.
608
609They can be canceled like any other request. Canceling will cancel not
610only the request itself, but also all requests it contains.
611
612They can also can also be added to other IO::AIO::GRP objects.
613
614Their lifetime, simplified, looks like this: when they are empty, they
615will finish very quickly. If they contain only requests that are in the
616C<done> state, they will also finish. Otherwise they will continue to
617exist.
618
619That means after creating a group you have some time to add requests. And
620in the callbacks of those requests, you can add further requests to the
621group. And only when all those requests have finished will the the group
622itself finish.
623
624=over 4
625
626=item $grp->add (...)
627
628=item add $grp ...
629
630Add one or more requests to the group. Any type of L<IO::AIO::REQ> can
631be added, including other groups, as long as you do not create circular
632dependencies.
633
634Returns all its arguments.
635
636=item $grp->result (...)
637
638Set the result value(s) that will be passed to the group callback when all
639subrequests have finished. By default, no argument will be passed.
396 640
397=back 641=back
398 642
399=head2 SUPPORT FUNCTIONS 643=head2 SUPPORT FUNCTIONS
400 644
528} 772}
529 773
5301; 7741;
531 775
532=head2 FORK BEHAVIOUR 776=head2 FORK BEHAVIOUR
777
778This module should do "the right thing" when the process using it forks:
533 779
534Before the fork, IO::AIO enters a quiescent state where no requests 780Before the fork, IO::AIO enters a quiescent state where no requests
535can be added in other threads and no results will be processed. After 781can be added in other threads and no results will be processed. After
536the fork the parent simply leaves the quiescent state and continues 782the fork the parent simply leaves the quiescent state and continues
537request/result processing, while the child clears the request/result 783request/result processing, while the child clears the request/result
538queue (so the requests started before the fork will only be handled in 784queue (so the requests started before the fork will only be handled in
539the parent). Threats will be started on demand until the limit ste in the 785the parent). Threads will be started on demand until the limit ste in the
540parent process has been reached again. 786parent process has been reached again.
541 787
788In short: the parent will, after a short pause, continue as if fork had
789not been called, while the child will act as if IO::AIO has not been used
790yet.
791
542=head1 SEE ALSO 792=head1 SEE ALSO
543 793
544L<Coro>, L<Linux::AIO>. 794L<Coro>, L<Linux::AIO> (obsolete).
545 795
546=head1 AUTHOR 796=head1 AUTHOR
547 797
548 Marc Lehmann <schmorp@schmorp.de> 798 Marc Lehmann <schmorp@schmorp.de>
549 http://home.schmorp.de/ 799 http://home.schmorp.de/

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines