[ViewVC] Diff of: cvs/IO-AIO/AIO.pm

Comparing IO-AIO/AIO.pm (file contents):
Revision 1.55 by root, Sun Oct 22 00:49:29 2006 UTC vs.
Revision 1.61 by root, Sun Oct 22 13:52:11 2006 UTC

…		…
15		15
16	aio_read $fh, 30000, 1024, $buffer, 0, sub {	16	aio_read $fh, 30000, 1024, $buffer, 0, sub {
17	$_[0] > 0 or die "read error: $!";	17	$_[0] > 0 or die "read error: $!";
18	};	18	};
19		19
20	use IO::AIO 2; # version has aio objects	20	# version 2+ has request and group objects
		21	use IO::AIO 2;
21		22
22	my $req = aio_unlink "/tmp/file", sub { };	23	my $req = aio_unlink "/tmp/file", sub { };
23	$req->cancel; # cancel request if still in queue	24	$req->cancel; # cancel request if still in queue
24		25
25	# AnyEvent	26	my $grp = aio_group sub { print "all stats done\n" };
		27	add $grp aio_stat "..." for ...;
		28
		29	# AnyEvent integration
26	open my $fh, "<&=" . IO::AIO::poll_fileno or die "$!";	30	open my $fh, "<&=" . IO::AIO::poll_fileno or die "$!";
27	my $w = AnyEvent->io (fh => $fh, poll => 'r', cb => sub { IO::AIO::poll_cb });	31	my $w = AnyEvent->io (fh => $fh, poll => 'r', cb => sub { IO::AIO::poll_cb });
28		32
29	# Event	33	# Event integration
30	Event->io (fd => IO::AIO::poll_fileno,	34	Event->io (fd => IO::AIO::poll_fileno,
31	poll => 'r',	35	poll => 'r',
32	cb => \&IO::AIO::poll_cb);	36	cb => \&IO::AIO::poll_cb);
33		37
34	# Glib/Gtk2	38	# Glib/Gtk2 integration
35	add_watch Glib::IO IO::AIO::poll_fileno,	39	add_watch Glib::IO IO::AIO::poll_fileno,
36	in => sub { IO::AIO::poll_cb; 1 };	40	in => sub { IO::AIO::poll_cb; 1 };
37		41
38	# Tk	42	# Tk integration
39	Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "",	43	Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "",
40	readable => \&IO::AIO::poll_cb);	44	readable => \&IO::AIO::poll_cb);
41		45
42	# Danga::Socket	46	# Danga::Socket integration
43	Danga::Socket->AddOtherFds (IO::AIO::poll_fileno =>	47	Danga::Socket->AddOtherFds (IO::AIO::poll_fileno =>
44	\&IO::AIO::poll_cb);	48	\&IO::AIO::poll_cb);
45
46		49
47	=head1 DESCRIPTION	50	=head1 DESCRIPTION
48		51
49	This module implements asynchronous I/O using whatever means your	52	This module implements asynchronous I/O using whatever means your
50	operating system supports.	53	operating system supports.
…		…
179	print "read $_[0] bytes: <$buffer>\n";	182	print "read $_[0] bytes: <$buffer>\n";
180	};	183	};
181		184
182	=item aio_move $srcpath, $dstpath, $callback->($status)	185	=item aio_move $srcpath, $dstpath, $callback->($status)
183		186
		187	[EXPERIMENTAL due to internal aio_group use]
		188
184	Try to move the I<file> (directories not supported as either source or	189	Try to move the I<file> (directories not supported as either source or
185	destination) from C<$srcpath> to C<$dstpath> and call the callback with	190	destination) from C<$srcpath> to C<$dstpath> and call the callback with
186	the C<0> (error) or C<-1> ok.	191	the C<0> (error) or C<-1> ok.
187		192
188	This is a composite request that tries to rename(2) the file first. If	193	This is a composite request that tries to rename(2) the file first. If
…		…
198	=cut	203	=cut
199		204
200	sub aio_move($$$) {	205	sub aio_move($$$) {
201	my ($src, $dst, $cb) = @_;	206	my ($src, $dst, $cb) = @_;
202		207
203	my $grp = aio_group;	208	my $grp = aio_group $cb;
204		209
205	add $grp aio_rename $src, $dst, sub {	210	add $grp aio_rename $src, $dst, sub {
206	if ($_[0] && $! == EXDEV) {	211	if ($_[0] && $! == EXDEV) {
207	add $grp aio_open $src, O_RDONLY, 0, sub {	212	add $grp aio_open $src, O_RDONLY, 0, sub {
208	if (my $src_fh = $_[0]) {	213	if (my $src_fh = $_[0]) {
…		…
218	chmod $stat[2] & 07777, $dst_fh;	223	chmod $stat[2] & 07777, $dst_fh;
219	chown $stat[4], $stat[5], $dst_fh;	224	chown $stat[4], $stat[5], $dst_fh;
220	close $dst_fh;	225	close $dst_fh;
221		226
222	add $grp aio_unlink $src, sub {	227	add $grp aio_unlink $src, sub {
223	$cb->($_[0]);	228	$grp->result ($_[0]);
224	};	229	};
225	} else {	230	} else {
226	my $errno = $!;	231	my $errno = $!;
227	add $grp aio_unlink $dst, sub {	232	add $grp aio_unlink $dst, sub {
228	$! = $errno;	233	$! = $errno;
229	$cb->(-1);	234	$grp->result (-1);
230	};	235	};
231	}	236	}
232	};	237	};
233	} else {	238	} else {
234	$cb->(-1);	239	$grp->result (-1);
235	}	240	}
236	},	241	},
237		242
238	} else {	243	} else {
239	$cb->(-1);	244	$grp->result (-1);
240	}	245	}
241	};	246	};
242	} else {	247	} else {
243	$cb->($_[0]);	248	$grp->result ($_[0]);
244	}	249	}
245	};	250	};
246		251
247	$grp	252	$grp
248	}	253	}
…		…
340	The callback a single argument which is either C<undef> or an array-ref	345	The callback a single argument which is either C<undef> or an array-ref
341	with the filenames.	346	with the filenames.
342		347
343	=item aio_scandir $path, $maxreq, $callback->($dirs, $nondirs)	348	=item aio_scandir $path, $maxreq, $callback->($dirs, $nondirs)
344		349
		350	[EXPERIMENTAL due to internal aio_group use]
		351
345	Scans a directory (similar to C<aio_readdir>) but additionally tries to	352	Scans a directory (similar to C<aio_readdir>) but additionally tries to
346	separate the entries of directory C<$path> into two sets of names, ones	353	separate the entries of directory C<$path> into two sets of names, ones
347	you can recurse into (directories or links to them), and ones you cannot	354	you can recurse into (directories or links to them), and ones you cannot
348	recurse into (everything else).	355	recurse into (everything else).
349		356
350	C<aio_scandir> is a composite request that consists of many sub	357	C<aio_scandir> is a composite request that creates of many sub requests_
351	requests. C<$maxreq> specifies the maximum number of outstanding aio	358	C<$maxreq> specifies the maximum number of outstanding aio requests that
352	requests that this function generates. If it is C<< <= 0 >>, then a	359	this function generates. If it is C<< <= 0 >>, then a suitable default
353	suitable default will be chosen (currently 8).	360	will be chosen (currently 6).
354		361
355	On error, the callback is called without arguments, otherwise it receives	362	On error, the callback is called without arguments, otherwise it receives
356	two array-refs with path-relative entry names.	363	two array-refs with path-relative entry names.
357		364
358	Example:	365	Example:
…		…
395	=cut	402	=cut
396		403
397	sub aio_scandir($$$) {	404	sub aio_scandir($$$) {
398	my ($path, $maxreq, $cb) = @_;	405	my ($path, $maxreq, $cb) = @_;
399		406
400	my $grp = aio_group;	407	my $grp = aio_group $cb;
401		408
402	$maxreq = 8 if $maxreq <= 0;	409	$maxreq = 6 if $maxreq <= 0;
403		410
404	# stat once	411	# stat once
405	add $grp aio_stat $path, sub {	412	add $grp aio_stat $path, sub {
406	return $cb->() if $_[0];	413	return $grp->result () if $_[0];
407	my $now = time;	414	my $now = time;
408	my $hash1 = join ":", (stat _)[0,1,3,7,9];	415	my $hash1 = join ":", (stat _)[0,1,3,7,9];
409		416
410	# read the directory entries	417	# read the directory entries
411	add $grp aio_readdir $path, sub {	418	add $grp aio_readdir $path, sub {
412	my $entries = shift	419	my $entries = shift
413	or return $cb->();	420	or return $grp->result ();
414		421
415	# stat the dir another time	422	# stat the dir another time
416	add $grp aio_stat $path, sub {	423	add $grp aio_stat $path, sub {
417	my $hash2 = join ":", (stat _)[0,1,3,7,9];	424	my $hash2 = join ":", (stat _)[0,1,3,7,9];
418		425
…		…
423	$ndirs = -1;	430	$ndirs = -1;
424	} else {	431	} else {
425	# if nlink == 2, we are finished	432	# if nlink == 2, we are finished
426	# on non-posix-fs's, we rely on nlink < 2	433	# on non-posix-fs's, we rely on nlink < 2
427	$ndirs = (stat _)[3] - 2	434	$ndirs = (stat _)[3] - 2
428	or return $cb->([], $entries);	435	or return $grp->result ([], $entries);
429	}	436	}
430		437
431	# sort into likely dirs and likely nondirs	438	# sort into likely dirs and likely nondirs
432	# dirs == files without ".", short entries first	439	# dirs == files without ".", short entries first
433	$entries = [map $_->[0],	440	$entries = [map $_->[0],
…		…
438	my (@dirs, @nondirs);	445	my (@dirs, @nondirs);
439		446
440	my ($statcb, $schedcb);	447	my ($statcb, $schedcb);
441	my $nreq = 0;	448	my $nreq = 0;
442		449
		450	my $statgrp = add $grp aio_group;
		451
443	$schedcb = sub {	452	$schedcb = sub {
444	if (@$entries) {	453	if (@$entries) {
445	if ($nreq < $maxreq) {	454	if ($nreq < $maxreq) {
446	my $ent = pop @$entries;	455	my $ent = pop @$entries;
447	$nreq++;	456	$nreq++;
448	add $grp aio_stat "$path/$ent/.", sub { $statcb->($_[0], $ent) };	457	add $statgrp aio_stat "$path/$ent/.", sub { $statcb->($_[0], $ent) };
449	}	458	}
450	} elsif (!$nreq) {	459	} elsif (!$nreq) {
451	# finished	460	# finished
		461	$statgrp->cancel;
452	undef $statcb;	462	undef $statcb;
453	undef $schedcb;	463	undef $schedcb;
454	$cb->(\@dirs, \@nondirs) if $cb;	464	$grp->result (\@dirs, \@nondirs);
455	undef $cb;
456	}	465	}
457	};	466	};
458	$statcb = sub {	467	$statcb = sub {
459	my ($status, $entry) = @_;	468	my ($status, $entry) = @_;
460		469
…		…
502	callback with the fdatasync result code.	511	callback with the fdatasync result code.
503		512
504	If this call isn't available because your OS lacks it or it couldn't be	513	If this call isn't available because your OS lacks it or it couldn't be
505	detected, it will be emulated by calling C<fsync> instead.	514	detected, it will be emulated by calling C<fsync> instead.
506		515
507	=item aio_group $callback->()	516	=item aio_group $callback->(...)
508		517
509	[EXPERIMENTAL]	518	[EXPERIMENTAL]
510		519
511	This is a very special aio request: Instead of doing something, it is a	520	This is a very special aio request: Instead of doing something, it is a
512	container for other aio requests, which is useful if you want to bundle	521	container for other aio requests, which is useful if you want to bundle
…		…
524	add $grp	533	add $grp
525	(aio_stat ...),	534	(aio_stat ...),
526	(aio_stat ...),	535	(aio_stat ...),
527	...;	536	...;
528		537
529	=item aio_sleep $fractional_seconds, $callback->() NOT EXPORTED	538	=item IO::AIO::aio_sleep $fractional_seconds, $callback->() NOT EXPORTED
530		539
531	Mainly used for debugging and benchmarking, this aio request puts one of	540	Mainly used for debugging and benchmarking, this aio request puts one of
532	the request workers to sleep for the given time.	541	the request workers to sleep for the given time.
		542
		543	While it is theoretically handy to have simple I/O scheduling requests
		544	like sleep and file handle readable/writable, the overhead this creates
		545	is immense, so do not use this function except to put your application
		546	under artificial I/O pressure.
533		547
534	=back	548	=back
535		549
536	=head2 IO::AIO::REQ CLASS	550	=head2 IO::AIO::REQ CLASS
537		551
…		…
577	You add requests by calling the C<add> method with one or more	591	You add requests by calling the C<add> method with one or more
578	C<IO::AIO::REQ> objects:	592	C<IO::AIO::REQ> objects:
579		593
580	$grp->add (aio_unlink "...");	594	$grp->add (aio_unlink "...");
581		595
582	add $grp aio_stat "...", sub { ... };	596	add $grp aio_stat "...", sub {
		597	$_[0] or return $grp->result ("error");
		598
		599	# add another request dynamically, if first succeeded
		600	add $grp aio_open "...", sub {
		601	$grp->result ("ok");
		602	};
		603	};
583		604
584	This makes it very easy to create composite requests (see the source of	605	This makes it very easy to create composite requests (see the source of
585	C<aio_move> for an application) that work and feel like simple requests.	606	C<aio_move> for an application) that work and feel like simple requests.
586		607
587	The IO::AIO::GRP objects will be cleaned up during calls to	608	The IO::AIO::GRP objects will be cleaned up during calls to
588	C<IO::AIO::poll_cb>, just like any other request.	609	C<IO::AIO::poll_cb>, just like any other request.
589		610
590	They can be canceled like any other request. Canceling will cancel not	611	They can be canceled like any other request. Canceling will cancel not
591	just the request itself, but also all requests it contains.	612	only the request itself, but also all requests it contains.
592		613
593	They can also can also be added to other IO::AIO::GRP objects.	614	They can also can also be added to other IO::AIO::GRP objects.
		615
		616	You must not add requests to a group from within the group callback (or
		617	any later time).
594		618
595	Their lifetime, simplified, looks like this: when they are empty, they	619	Their lifetime, simplified, looks like this: when they are empty, they
596	will finish very quickly. If they contain only requests that are in the	620	will finish very quickly. If they contain only requests that are in the
597	C<done> state, they will also finish. Otherwise they will continue to	621	C<done> state, they will also finish. Otherwise they will continue to
598	exist.	622	exist.
599		623
		624	That means after creating a group you have some time to add requests. And
		625	in the callbacks of those requests, you can add further requests to the
		626	group. And only when all those requests have finished will the the group
		627	itself finish.
		628
600	=over 4	629	=over 4
601		630
602	=item $grp->add (...)	631	=item $grp->add (...)
603		632
604	=item add $grp ...	633	=item add $grp ...
605		634
606	Add one or more	635	Add one or more requests to the group. Any type of L<IO::AIO::REQ> can
607	Cancels the request, if possible. Has the effect of skipping execution	636	be added, including other groups, as long as you do not create circular
608	when entering the B<execute> state and skipping calling the callback when	637	dependencies.
609	entering the the B<result> state, but will leave the request otherwise	638
610	untouched. That means that requests that currently execute will not be	639	Returns all its arguments.
611	stopped and resources held by the request will not be freed prematurely.	640
		641	=item $grp->result (...)
		642
		643	Set the result value(s) that will be passed to the group callback when all
		644	subrequests have finished. By default, no argument will be passed.
		645
		646	=item $grp->set_feeder ($callback->($grp))
		647
		648	[VERY EXPERIMENTAL]
		649
		650	Sets a feeder/generator on this group: every group can have an attached
		651	generator that generates requests if idle. The idea behind this is that,
		652	although you could just queue as many requests as you want in a group,
		653	this might starve other requests for a potentially long time. For
		654	example, C<aio_scandir> might generate hundreds of thousands C<aio_stat>
		655	requests, delaying any later requests for a long time.
		656
		657	To avoid this, and allow incremental generation of requests, you can
		658	instead a group and set a feeder on it that generates those requests. The
		659	feeder will be called whenever there are few enough (see C<feeder_limit>,
		660	below) requests active in the group itself and is expected to queue more
		661	requests.
		662
		663	The feeder can queue as many requests as it likes (i.e. C<add> does not
		664	impose any limits).
		665
		666	If the feeder does not queue more requests when called, it will be
		667	automatically removed from the group.
		668
		669	If the feeder limit is C<0>, it will be set to C<2> automatically.
		670
		671	Example:
		672
		673	# stat all files in @files, but only ever use four aio requests concurrently:
		674
		675	my $grp = aio_group sub { print "finished\n" };
		676	$grp->feeder_limit (4);
		677	$grp->set_feeder (sub {
		678	my $file = pop @files
		679	or return;
		680
		681	add $grp aio_stat $file, sub { ... };
		682	});
		683
		684	=item $grp->feeder_limit ($num)
		685
		686	Sets the feeder limit for the group: The feeder will be called whenever
		687	the group contains less than this many requests.
		688
		689	Setting the limit to C<0> will pause the feeding process.
612		690
613	=back	691	=back
614		692
615	=head2 SUPPORT FUNCTIONS	693	=head2 SUPPORT FUNCTIONS
616		694
…		…
674	IO::AIO::poll_wait, IO::AIO::poll_cb	752	IO::AIO::poll_wait, IO::AIO::poll_cb
675	if IO::AIO::nreqs;	753	if IO::AIO::nreqs;
676		754
677	=item IO::AIO::min_parallel $nthreads	755	=item IO::AIO::min_parallel $nthreads
678		756
679	Set the minimum number of AIO threads to C<$nthreads>. The current default	757	Set the minimum number of AIO threads to C<$nthreads>. The current
680	is C<4>, which means four asynchronous operations can be done at one time	758	default is C<8>, which means eight asynchronous operations can execute
681	(the number of outstanding operations, however, is unlimited).	759	concurrently at any one time (the number of outstanding requests,
		760	however, is unlimited).
682		761
683	IO::AIO starts threads only on demand, when an AIO request is queued and	762	IO::AIO starts threads only on demand, when an AIO request is queued and
684	no free thread exists.	763	no free thread exists.
685		764
686	It is recommended to keep the number of threads low, as some Linux	765	It is recommended to keep the number of threads relatively low, as some
687	kernel versions will scale negatively with the number of threads (higher	766	Linux kernel versions will scale negatively with the number of threads
688	parallelity => MUCH higher latency). With current Linux 2.6 versions, 4-32	767	(higher parallelity => MUCH higher latency). With current Linux 2.6
689	threads should be fine.	768	versions, 4-32 threads should be fine.
690		769
691	Under most circumstances you don't need to call this function, as the	770	Under most circumstances you don't need to call this function, as the
692	module selects a default that is suitable for low to moderate load.	771	module selects a default that is suitable for low to moderate load.
693		772
694	=item IO::AIO::max_parallel $nthreads	773	=item IO::AIO::max_parallel $nthreads
…		…
735	or return undef;	814	or return undef;
736		815
737	*$sym	816	*$sym
738	}	817	}
739		818
740	min_parallel 4;	819	min_parallel 8;
741		820
742	END {	821	END {
743	max_parallel 0;	822	max_parallel 0;
744	}	823	}
745		824
…		…
759		838
760	In short: the parent will, after a short pause, continue as if fork had	839	In short: the parent will, after a short pause, continue as if fork had
761	not been called, while the child will act as if IO::AIO has not been used	840	not been called, while the child will act as if IO::AIO has not been used
762	yet.	841	yet.
763		842
		843	=head2 MEMORY USAGE
		844
		845	Each aio request uses - depending on your architecture - around 128 bytes
		846	of memory. In addition, stat requests need a stat buffer (possibly a few
		847	hundred bytes). Perl scalars and other data passed into aio requests will
		848	also be locked.
		849
		850	This is now awfully much, so queuing lots of requests is not usually a
		851	problem.
		852
		853	Each thread needs a stack area which is usually around 16k, sometimes much
		854	larger, depending on the OS.
		855
764	=head1 SEE ALSO	856	=head1 SEE ALSO
765		857
766	L<Coro>, L<Linux::AIO> (obsolete).	858	L<Coro>, L<Linux::AIO> (obsolete).
767		859
768	=head1 AUTHOR	860	=head1 AUTHOR

Diff Legend

-–
+Removed lines
-+
+Added lines
-<
+Changed lines
->
+Changed lines

Comparing IO-AIO/AIO.pm (file contents): Revision 1.55 by root, Sun Oct 22 00:49:29 2006 UTC vs. Revision 1.61 by root, Sun Oct 22 13:52:11 2006 UTC

Diff Legend

Comparing IO-AIO/AIO.pm (file contents):
Revision 1.55 by root, Sun Oct 22 00:49:29 2006 UTC vs.
Revision 1.61 by root, Sun Oct 22 13:52:11 2006 UTC