[ViewVC] Diff of: cvs/IO-AIO/AIO.pm

Comparing IO-AIO/AIO.pm (file contents):
Revision 1.55 by root, Sun Oct 22 00:49:29 2006 UTC vs.
Revision 1.62 by root, Sun Oct 22 21:13:47 2006 UTC

…		…
15		15
16	aio_read $fh, 30000, 1024, $buffer, 0, sub {	16	aio_read $fh, 30000, 1024, $buffer, 0, sub {
17	$_[0] > 0 or die "read error: $!";	17	$_[0] > 0 or die "read error: $!";
18	};	18	};
19		19
20	use IO::AIO 2; # version has aio objects	20	# version 2+ has request and group objects
		21	use IO::AIO 2;
21		22
22	my $req = aio_unlink "/tmp/file", sub { };	23	my $req = aio_unlink "/tmp/file", sub { };
23	$req->cancel; # cancel request if still in queue	24	$req->cancel; # cancel request if still in queue
24		25
25	# AnyEvent	26	my $grp = aio_group sub { print "all stats done\n" };
		27	add $grp aio_stat "..." for ...;
		28
		29	# AnyEvent integration
26	open my $fh, "<&=" . IO::AIO::poll_fileno or die "$!";	30	open my $fh, "<&=" . IO::AIO::poll_fileno or die "$!";
27	my $w = AnyEvent->io (fh => $fh, poll => 'r', cb => sub { IO::AIO::poll_cb });	31	my $w = AnyEvent->io (fh => $fh, poll => 'r', cb => sub { IO::AIO::poll_cb });
28		32
29	# Event	33	# Event integration
30	Event->io (fd => IO::AIO::poll_fileno,	34	Event->io (fd => IO::AIO::poll_fileno,
31	poll => 'r',	35	poll => 'r',
32	cb => \&IO::AIO::poll_cb);	36	cb => \&IO::AIO::poll_cb);
33		37
34	# Glib/Gtk2	38	# Glib/Gtk2 integration
35	add_watch Glib::IO IO::AIO::poll_fileno,	39	add_watch Glib::IO IO::AIO::poll_fileno,
36	in => sub { IO::AIO::poll_cb; 1 };	40	in => sub { IO::AIO::poll_cb; 1 };
37		41
38	# Tk	42	# Tk integration
39	Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "",	43	Tk::Event::IO->fileevent (IO::AIO::poll_fileno, "",
40	readable => \&IO::AIO::poll_cb);	44	readable => \&IO::AIO::poll_cb);
41		45
42	# Danga::Socket	46	# Danga::Socket integration
43	Danga::Socket->AddOtherFds (IO::AIO::poll_fileno =>	47	Danga::Socket->AddOtherFds (IO::AIO::poll_fileno =>
44	\&IO::AIO::poll_cb);	48	\&IO::AIO::poll_cb);
45
46		49
47	=head1 DESCRIPTION	50	=head1 DESCRIPTION
48		51
49	This module implements asynchronous I/O using whatever means your	52	This module implements asynchronous I/O using whatever means your
50	operating system supports.	53	operating system supports.
…		…
179	print "read $_[0] bytes: <$buffer>\n";	182	print "read $_[0] bytes: <$buffer>\n";
180	};	183	};
181		184
182	=item aio_move $srcpath, $dstpath, $callback->($status)	185	=item aio_move $srcpath, $dstpath, $callback->($status)
183		186
		187	[EXPERIMENTAL due to internal aio_group use]
		188
184	Try to move the I<file> (directories not supported as either source or	189	Try to move the I<file> (directories not supported as either source or
185	destination) from C<$srcpath> to C<$dstpath> and call the callback with	190	destination) from C<$srcpath> to C<$dstpath> and call the callback with
186	the C<0> (error) or C<-1> ok.	191	the C<0> (error) or C<-1> ok.
187		192
188	This is a composite request that tries to rename(2) the file first. If	193	This is a composite request that tries to rename(2) the file first. If
…		…
198	=cut	203	=cut
199		204
200	sub aio_move($$$) {	205	sub aio_move($$$) {
201	my ($src, $dst, $cb) = @_;	206	my ($src, $dst, $cb) = @_;
202		207
203	my $grp = aio_group;	208	my $grp = aio_group $cb;
204		209
205	add $grp aio_rename $src, $dst, sub {	210	add $grp aio_rename $src, $dst, sub {
206	if ($_[0] && $! == EXDEV) {	211	if ($_[0] && $! == EXDEV) {
207	add $grp aio_open $src, O_RDONLY, 0, sub {	212	add $grp aio_open $src, O_RDONLY, 0, sub {
208	if (my $src_fh = $_[0]) {	213	if (my $src_fh = $_[0]) {
…		…
218	chmod $stat[2] & 07777, $dst_fh;	223	chmod $stat[2] & 07777, $dst_fh;
219	chown $stat[4], $stat[5], $dst_fh;	224	chown $stat[4], $stat[5], $dst_fh;
220	close $dst_fh;	225	close $dst_fh;
221		226
222	add $grp aio_unlink $src, sub {	227	add $grp aio_unlink $src, sub {
223	$cb->($_[0]);	228	$grp->result ($_[0]);
224	};	229	};
225	} else {	230	} else {
226	my $errno = $!;	231	my $errno = $!;
227	add $grp aio_unlink $dst, sub {	232	add $grp aio_unlink $dst, sub {
228	$! = $errno;	233	$! = $errno;
229	$cb->(-1);	234	$grp->result (-1);
230	};	235	};
231	}	236	}
232	};	237	};
233	} else {	238	} else {
234	$cb->(-1);	239	$grp->result (-1);
235	}	240	}
236	},	241	},
237		242
238	} else {	243	} else {
239	$cb->(-1);	244	$grp->result (-1);
240	}	245	}
241	};	246	};
242	} else {	247	} else {
243	$cb->($_[0]);	248	$grp->result ($_[0]);
244	}	249	}
245	};	250	};
246		251
247	$grp	252	$grp
248	}	253	}
…		…
340	The callback a single argument which is either C<undef> or an array-ref	345	The callback a single argument which is either C<undef> or an array-ref
341	with the filenames.	346	with the filenames.
342		347
343	=item aio_scandir $path, $maxreq, $callback->($dirs, $nondirs)	348	=item aio_scandir $path, $maxreq, $callback->($dirs, $nondirs)
344		349
		350	[EXPERIMENTAL due to internal aio_group use]
		351
345	Scans a directory (similar to C<aio_readdir>) but additionally tries to	352	Scans a directory (similar to C<aio_readdir>) but additionally tries to
346	separate the entries of directory C<$path> into two sets of names, ones	353	separate the entries of directory C<$path> into two sets of names, ones
347	you can recurse into (directories or links to them), and ones you cannot	354	you can recurse into (directories or links to them), and ones you cannot
348	recurse into (everything else).	355	recurse into (everything else).
349		356
350	C<aio_scandir> is a composite request that consists of many sub	357	C<aio_scandir> is a composite request that creates of many sub requests_
351	requests. C<$maxreq> specifies the maximum number of outstanding aio	358	C<$maxreq> specifies the maximum number of outstanding aio requests that
352	requests that this function generates. If it is C<< <= 0 >>, then a	359	this function generates. If it is C<< <= 0 >>, then a suitable default
353	suitable default will be chosen (currently 8).	360	will be chosen (currently 6).
354		361
355	On error, the callback is called without arguments, otherwise it receives	362	On error, the callback is called without arguments, otherwise it receives
356	two array-refs with path-relative entry names.	363	two array-refs with path-relative entry names.
357		364
358	Example:	365	Example:
…		…
395	=cut	402	=cut
396		403
397	sub aio_scandir($$$) {	404	sub aio_scandir($$$) {
398	my ($path, $maxreq, $cb) = @_;	405	my ($path, $maxreq, $cb) = @_;
399		406
400	my $grp = aio_group;	407	my $grp = aio_group $cb;
401		408
402	$maxreq = 8 if $maxreq <= 0;	409	$maxreq = 6 if $maxreq <= 0;
403		410
404	# stat once	411	# stat once
405	add $grp aio_stat $path, sub {	412	add $grp aio_stat $path, sub {
406	return $cb->() if $_[0];	413	return $grp->result () if $_[0];
407	my $now = time;	414	my $now = time;
408	my $hash1 = join ":", (stat _)[0,1,3,7,9];	415	my $hash1 = join ":", (stat _)[0,1,3,7,9];
409		416
410	# read the directory entries	417	# read the directory entries
411	add $grp aio_readdir $path, sub {	418	add $grp aio_readdir $path, sub {
412	my $entries = shift	419	my $entries = shift
413	or return $cb->();	420	or return $grp->result ();
414		421
415	# stat the dir another time	422	# stat the dir another time
416	add $grp aio_stat $path, sub {	423	add $grp aio_stat $path, sub {
417	my $hash2 = join ":", (stat _)[0,1,3,7,9];	424	my $hash2 = join ":", (stat _)[0,1,3,7,9];
418		425
…		…
423	$ndirs = -1;	430	$ndirs = -1;
424	} else {	431	} else {
425	# if nlink == 2, we are finished	432	# if nlink == 2, we are finished
426	# on non-posix-fs's, we rely on nlink < 2	433	# on non-posix-fs's, we rely on nlink < 2
427	$ndirs = (stat _)[3] - 2	434	$ndirs = (stat _)[3] - 2
428	or return $cb->([], $entries);	435	or return $grp->result ([], $entries);
429	}	436	}
430		437
431	# sort into likely dirs and likely nondirs	438	# sort into likely dirs and likely nondirs
432	# dirs == files without ".", short entries first	439	# dirs == files without ".", short entries first
433	$entries = [map $_->[0],	440	$entries = [map $_->[0],
…		…
438	my (@dirs, @nondirs);	445	my (@dirs, @nondirs);
439		446
440	my ($statcb, $schedcb);	447	my ($statcb, $schedcb);
441	my $nreq = 0;	448	my $nreq = 0;
442		449
		450	my $statgrp = add $grp aio_group;
		451
443	$schedcb = sub {	452	$schedcb = sub {
444	if (@$entries) {	453	if (@$entries) {
445	if ($nreq < $maxreq) {	454	if ($nreq < $maxreq) {
446	my $ent = pop @$entries;	455	my $ent = pop @$entries;
447	$nreq++;	456	$nreq++;
448	add $grp aio_stat "$path/$ent/.", sub { $statcb->($_[0], $ent) };	457	add $statgrp aio_stat "$path/$ent/.", sub { $statcb->($_[0], $ent) };
449	}	458	}
450	} elsif (!$nreq) {	459	} elsif (!$nreq) {
451	# finished	460	# finished
		461	$statgrp->cancel;
452	undef $statcb;	462	undef $statcb;
453	undef $schedcb;	463	undef $schedcb;
454	$cb->(\@dirs, \@nondirs) if $cb;	464	$grp->result (\@dirs, \@nondirs);
455	undef $cb;
456	}	465	}
457	};	466	};
458	$statcb = sub {	467	$statcb = sub {
459	my ($status, $entry) = @_;	468	my ($status, $entry) = @_;
460		469
…		…
502	callback with the fdatasync result code.	511	callback with the fdatasync result code.
503		512
504	If this call isn't available because your OS lacks it or it couldn't be	513	If this call isn't available because your OS lacks it or it couldn't be
505	detected, it will be emulated by calling C<fsync> instead.	514	detected, it will be emulated by calling C<fsync> instead.
506		515
507	=item aio_group $callback->()	516	=item aio_group $callback->(...)
508		517
509	[EXPERIMENTAL]	518	[EXPERIMENTAL]
510		519
511	This is a very special aio request: Instead of doing something, it is a	520	This is a very special aio request: Instead of doing something, it is a
512	container for other aio requests, which is useful if you want to bundle	521	container for other aio requests, which is useful if you want to bundle
…		…
524	add $grp	533	add $grp
525	(aio_stat ...),	534	(aio_stat ...),
526	(aio_stat ...),	535	(aio_stat ...),
527	...;	536	...;
528		537
529	=item aio_sleep $fractional_seconds, $callback->() NOT EXPORTED	538	=item IO::AIO::aio_sleep $fractional_seconds, $callback->() NOT EXPORTED
530		539
531	Mainly used for debugging and benchmarking, this aio request puts one of	540	Mainly used for debugging and benchmarking, this aio request puts one of
532	the request workers to sleep for the given time.	541	the request workers to sleep for the given time.
		542
		543	While it is theoretically handy to have simple I/O scheduling requests
		544	like sleep and file handle readable/writable, the overhead this creates
		545	is immense, so do not use this function except to put your application
		546	under artificial I/O pressure.
533		547
534	=back	548	=back
535		549
536	=head2 IO::AIO::REQ CLASS	550	=head2 IO::AIO::REQ CLASS
537		551
…		…
577	You add requests by calling the C<add> method with one or more	591	You add requests by calling the C<add> method with one or more
578	C<IO::AIO::REQ> objects:	592	C<IO::AIO::REQ> objects:
579		593
580	$grp->add (aio_unlink "...");	594	$grp->add (aio_unlink "...");
581		595
582	add $grp aio_stat "...", sub { ... };	596	add $grp aio_stat "...", sub {
		597	$_[0] or return $grp->result ("error");
		598
		599	# add another request dynamically, if first succeeded
		600	add $grp aio_open "...", sub {
		601	$grp->result ("ok");
		602	};
		603	};
583		604
584	This makes it very easy to create composite requests (see the source of	605	This makes it very easy to create composite requests (see the source of
585	C<aio_move> for an application) that work and feel like simple requests.	606	C<aio_move> for an application) that work and feel like simple requests.
586		607
		608	=over 4
		609
587	The IO::AIO::GRP objects will be cleaned up during calls to	610	=item * The IO::AIO::GRP objects will be cleaned up during calls to
588	C<IO::AIO::poll_cb>, just like any other request.	611	C<IO::AIO::poll_cb>, just like any other request.
589		612
590	They can be canceled like any other request. Canceling will cancel not	613	=item * They can be canceled like any other request. Canceling will cancel not
591	just the request itself, but also all requests it contains.	614	only the request itself, but also all requests it contains.
592		615
593	They can also can also be added to other IO::AIO::GRP objects.	616	=item * They can also can also be added to other IO::AIO::GRP objects.
		617
		618	=item * You must not add requests to a group from within the group callback (or
		619	any later time).
		620
		621	=item * This does not harmonise well with C<max_outstanding>, so best do
		622	not combine C<aio_group> with it. Groups and feeders are recommended for
		623	this kind of concurrency-limiting.
		624
		625	=back
594		626
595	Their lifetime, simplified, looks like this: when they are empty, they	627	Their lifetime, simplified, looks like this: when they are empty, they
596	will finish very quickly. If they contain only requests that are in the	628	will finish very quickly. If they contain only requests that are in the
597	C<done> state, they will also finish. Otherwise they will continue to	629	C<done> state, they will also finish. Otherwise they will continue to
598	exist.	630	exist.
599		631
		632	That means after creating a group you have some time to add requests. And
		633	in the callbacks of those requests, you can add further requests to the
		634	group. And only when all those requests have finished will the the group
		635	itself finish.
		636
600	=over 4	637	=over 4
601		638
602	=item $grp->add (...)	639	=item $grp->add (...)
603		640
604	=item add $grp ...	641	=item add $grp ...
605		642
606	Add one or more	643	Add one or more requests to the group. Any type of L<IO::AIO::REQ> can
607	Cancels the request, if possible. Has the effect of skipping execution	644	be added, including other groups, as long as you do not create circular
608	when entering the B<execute> state and skipping calling the callback when	645	dependencies.
609	entering the the B<result> state, but will leave the request otherwise	646
610	untouched. That means that requests that currently execute will not be	647	Returns all its arguments.
611	stopped and resources held by the request will not be freed prematurely.	648
		649	=item $grp->result (...)
		650
		651	Set the result value(s) that will be passed to the group callback when all
		652	subrequests have finished. By default, no argument will be passed.
		653
		654	=item $grp->set_feeder ($callback->($grp))
		655
		656	[VERY EXPERIMENTAL]
		657
		658	Sets a feeder/generator on this group: every group can have an attached
		659	generator that generates requests if idle. The idea behind this is that,
		660	although you could just queue as many requests as you want in a group,
		661	this might starve other requests for a potentially long time. For
		662	example, C<aio_scandir> might generate hundreds of thousands C<aio_stat>
		663	requests, delaying any later requests for a long time.
		664
		665	To avoid this, and allow incremental generation of requests, you can
		666	instead a group and set a feeder on it that generates those requests. The
		667	feeder will be called whenever there are few enough (see C<feeder_limit>,
		668	below) requests active in the group itself and is expected to queue more
		669	requests.
		670
		671	The feeder can queue as many requests as it likes (i.e. C<add> does not
		672	impose any limits).
		673
		674	If the feeder does not queue more requests when called, it will be
		675	automatically removed from the group.
		676
		677	If the feeder limit is C<0>, it will be set to C<2> automatically.
		678
		679	Example:
		680
		681	# stat all files in @files, but only ever use four aio requests concurrently:
		682
		683	my $grp = aio_group sub { print "finished\n" };
		684	$grp->feeder_limit (4);
		685	$grp->set_feeder (sub {
		686	my $file = pop @files
		687	or return;
		688
		689	add $grp aio_stat $file, sub { ... };
		690	});
		691
		692	=item $grp->feeder_limit ($num)
		693
		694	Sets the feeder limit for the group: The feeder will be called whenever
		695	the group contains less than this many requests.
		696
		697	Setting the limit to C<0> will pause the feeding process.
612		698
613	=back	699	=back
614		700
615	=head2 SUPPORT FUNCTIONS	701	=head2 SUPPORT FUNCTIONS
616		702
…		…
674	IO::AIO::poll_wait, IO::AIO::poll_cb	760	IO::AIO::poll_wait, IO::AIO::poll_cb
675	if IO::AIO::nreqs;	761	if IO::AIO::nreqs;
676		762
677	=item IO::AIO::min_parallel $nthreads	763	=item IO::AIO::min_parallel $nthreads
678		764
679	Set the minimum number of AIO threads to C<$nthreads>. The current default	765	Set the minimum number of AIO threads to C<$nthreads>. The current
680	is C<4>, which means four asynchronous operations can be done at one time	766	default is C<8>, which means eight asynchronous operations can execute
681	(the number of outstanding operations, however, is unlimited).	767	concurrently at any one time (the number of outstanding requests,
		768	however, is unlimited).
682		769
683	IO::AIO starts threads only on demand, when an AIO request is queued and	770	IO::AIO starts threads only on demand, when an AIO request is queued and
684	no free thread exists.	771	no free thread exists.
685		772
686	It is recommended to keep the number of threads low, as some Linux	773	It is recommended to keep the number of threads relatively low, as some
687	kernel versions will scale negatively with the number of threads (higher	774	Linux kernel versions will scale negatively with the number of threads
688	parallelity => MUCH higher latency). With current Linux 2.6 versions, 4-32	775	(higher parallelity => MUCH higher latency). With current Linux 2.6
689	threads should be fine.	776	versions, 4-32 threads should be fine.
690		777
691	Under most circumstances you don't need to call this function, as the	778	Under most circumstances you don't need to call this function, as the
692	module selects a default that is suitable for low to moderate load.	779	module selects a default that is suitable for low to moderate load.
693		780
694	=item IO::AIO::max_parallel $nthreads	781	=item IO::AIO::max_parallel $nthreads
…		…
705		792
706	Under normal circumstances you don't need to call this function.	793	Under normal circumstances you don't need to call this function.
707		794
708	=item $oldnreqs = IO::AIO::max_outstanding $nreqs	795	=item $oldnreqs = IO::AIO::max_outstanding $nreqs
709		796
		797	[DEPRECATED]
		798
710	Sets the maximum number of outstanding requests to C<$nreqs>. If you	799	Sets the maximum number of outstanding requests to C<$nreqs>. If you
711	try to queue up more than this number of requests, the caller will block until	800	try to queue up more than this number of requests, the caller will block until
712	some requests have been handled.	801	some requests have been handled.
713		802
714	The default is very large, so normally there is no practical limit. If you	803	The default is very large, so normally there is no practical limit. If you
715	queue up many requests in a loop it often improves speed if you set	804	queue up many requests in a loop it often improves speed if you set
716	this to a relatively low number, such as C<100>.	805	this to a relatively low number, such as C<100>.
		806
		807	This function does not work well together with C<aio_group>'s, and their
		808	feeder interface is better suited to limiting concurrency, so do not use
		809	this function.
717		810
718	Under normal circumstances you don't need to call this function.	811	Under normal circumstances you don't need to call this function.
719		812
720	=back	813	=back
721		814
…		…
735	or return undef;	828	or return undef;
736		829
737	*$sym	830	*$sym
738	}	831	}
739		832
740	min_parallel 4;	833	min_parallel 8;
741		834
742	END {	835	END {
743	max_parallel 0;	836	max_parallel 0;
744	}	837	}
745		838
…		…
759		852
760	In short: the parent will, after a short pause, continue as if fork had	853	In short: the parent will, after a short pause, continue as if fork had
761	not been called, while the child will act as if IO::AIO has not been used	854	not been called, while the child will act as if IO::AIO has not been used
762	yet.	855	yet.
763		856
		857	=head2 MEMORY USAGE
		858
		859	Each aio request uses - depending on your architecture - around 128 bytes
		860	of memory. In addition, stat requests need a stat buffer (possibly a few
		861	hundred bytes). Perl scalars and other data passed into aio requests will
		862	also be locked.
		863
		864	This is now awfully much, so queuing lots of requests is not usually a
		865	problem.
		866
		867	Each thread needs a stack area which is usually around 16k, sometimes much
		868	larger, depending on the OS.
		869
764	=head1 SEE ALSO	870	=head1 SEE ALSO
765		871
766	L<Coro>, L<Linux::AIO> (obsolete).	872	L<Coro>, L<Linux::AIO> (obsolete).
767		873
768	=head1 AUTHOR	874	=head1 AUTHOR

Diff Legend

-–
+Removed lines
-+
+Added lines
-<
+Changed lines
->
+Changed lines

Comparing IO-AIO/AIO.pm (file contents): Revision 1.55 by root, Sun Oct 22 00:49:29 2006 UTC vs. Revision 1.62 by root, Sun Oct 22 21:13:47 2006 UTC

Diff Legend

Comparing IO-AIO/AIO.pm (file contents):
Revision 1.55 by root, Sun Oct 22 00:49:29 2006 UTC vs.
Revision 1.62 by root, Sun Oct 22 21:13:47 2006 UTC