Coro/Coro/Storable.pm

=head1 NAME

Coro::Storable - offer a more fine-grained Storable interface

=head1 SYNOPSIS

 use Coro::Storable;

=head1 DESCRIPTION

This module implements a few functions from the Storable module in a way
so that it cede's more often. Some applications (such as the Deliantra
game server) sometimes need to load large Storable objects without
blocking the server for a long time.

This is being implemented by using a perlio layer that feeds only small
amounts of data (512 bytes per call) into Storable, and C<Coro::cede>'ing
regularly (at most 100 times per second by default, though).

As it seems that Storable is not reentrant, this module also wraps most
functions of the Storable module so that only one freeze or thaw is done
at any one moment (recursive invocations are not currently supported).

=head1 FUNCTIONS

=over 4

=item $ref = thaw $pst

Retrieve an object from the given $pst, which must have been created with
C<Coro::Storable::freeze> or C<Storable::store_fd>/C<Storable::store>
(sorry, but Storable uses incompatible formats for disk/mem objects).

This works by calling C<Coro::cede> for every 4096 bytes read in.

=item $pst = freeze $ref

Freeze the given scalar into a Storable object. It uses the same format as
C<Storable::store_fd>.

This works by calling C<Coro::cede> for every write that Storable
issues. Unfortunately, Storable often makes many very small writes, so it
is rather inefficient. But it does keep the latency low.

=item $pst = nfreeze $ref

Same as C<freeze> but is compatible to C<Storable::nstore_fd> (note the
C<n>).

=item $pst = blocking_freeze $ref

Same as C<freeze> but is guaranteed to block. This is useful e.g. in
C<Coro::Util::fork_eval> when you want to serialise a data structure
for use with the C<thaw> function for this module. You cannot use
C<Storable::freeze> for this as Storable uses incompatible formats for
memory and file images.

=item $pst = blocking_nfreeze $ref

Same as C<blocking_freeze> but uses C<nfreeze> internally.

=item $guard = guard

Acquire the Storable lock, for when you want to call Storable yourself.

Note that this module already wraps the Storable functions, so there is
rarely the need to do this yourself.

=back

=cut

package Coro::Storable;

use strict qw(subs vars);
no warnings;

use Coro ();
use Coro::Semaphore ();

BEGIN {
   # suppress warnings
   local $^W = 0;
   require Storable;
}

use Storable;
use base "Exporter";

our $VERSION = 4.746;
our @EXPORT = qw(thaw freeze nfreeze blocking_thaw blocking_freeze blocking_nfreeze);

my $lock = new Coro::Semaphore;

sub guard {
   $lock->guard
}

# wrap xs functions
for (qw(net_pstore mstore net_mstore pretrieve mretrieve dclone)) {
   my $orig = \&{"Storable::$_"};
   *{"Storable::$_"} = sub {
      my $guard = $lock->guard;
      &$orig
   };
   die if $@;
}

sub thaw($) {
   open my $fh, "<:via(CoroCede)", \$_[0]
      or die "cannot open pst via CoroCede: $!";
   Storable::fd_retrieve $fh
}

sub freeze($) {
   open my $fh, ">:via(CoroCede)", \my $buf
      or die "cannot open pst via CoroCede: $!";
   Storable::store_fd $_[0], $fh;
   $buf
}

sub nfreeze($) {
   open my $fh, ">:via(CoroCede)", \my $buf
      or die "cannot open pst via CoroCede: $!";
   Storable::nstore_fd $_[0], $fh;
   $buf
}

sub blocking_thaw($) {
   open my $fh, "<", \$_[0]
      or die "cannot open pst: $!";
   Storable::fd_retrieve $fh
}

sub blocking_freeze($) {
   open my $fh, ">", \my $buf
         or die "cannot open pst: $!";
   Storable::store_fd $_[0], $fh;
   close $fh;

   $buf
}

sub blocking_nfreeze($) {
   open my $fh, ">", \my $buf
         or die "cannot open pst: $!";
   Storable::nstore_fd $_[0], $fh;
   close $fh;

   $buf
}

package PerlIO::via::CoroCede;

# generic cede-on-read/write filtering layer

use Time::HiRes ("time");

our $GRANULARITY = 0.01;

my $next_cede;

sub PUSHED {
   __PACKAGE__
}

sub FILL {
   if ($next_cede <= time) {
      $next_cede = time + $GRANULARITY; # calling time() twice usually is a net win
      local *Storable::FILE;
      Coro::cede ();
   }

   read $_[1], my $buf, 512
      or return undef;

   $buf
}

sub WRITE {
   if ($next_cede <= time) {
      $next_cede = time + $GRANULARITY;
      local *Storable::FILE;
      Coro::cede ();
   }

   (print {$_[2]} $_[1]) ? length $_[1] : -1
}

1;

=head1 AUTHOR

 Marc Lehmann <schmorp@schmorp.de>
 http://home.schmorp.de/

=cut


Revision:	1.24
Committed:	Mon Sep 22 05:40:21 2008 UTC (15 years, 8 months ago) by root
Branch:	MAIN
Changes since 1.23:	+2 -0 lines
Log Message:	* empty log message *
#	Content
1	=head1 NAME
2
3	Coro::Storable - offer a more fine-grained Storable interface
4
5	=head1 SYNOPSIS
6
7	use Coro::Storable;
8
9	=head1 DESCRIPTION
10
11	This module implements a few functions from the Storable module in a way
12	so that it cede's more often. Some applications (such as the Deliantra
13	game server) sometimes need to load large Storable objects without
14	blocking the server for a long time.
15
16	This is being implemented by using a perlio layer that feeds only small
17	amounts of data (512 bytes per call) into Storable, and C<Coro::cede>'ing
18	regularly (at most 100 times per second by default, though).
19
20	As it seems that Storable is not reentrant, this module also wraps most
21	functions of the Storable module so that only one freeze or thaw is done
22	at any one moment (recursive invocations are not currently supported).
23
24	=head1 FUNCTIONS
25
26	=over 4
27
28	=item $ref = thaw $pst
29
30	Retrieve an object from the given $pst, which must have been created with
31	C<Coro::Storable::freeze> or C<Storable::store_fd>/C<Storable::store>
32	(sorry, but Storable uses incompatible formats for disk/mem objects).
33
34	This works by calling C<Coro::cede> for every 4096 bytes read in.
35
36	=item $pst = freeze $ref
37
38	Freeze the given scalar into a Storable object. It uses the same format as
39	C<Storable::store_fd>.
40
41	This works by calling C<Coro::cede> for every write that Storable
42	issues. Unfortunately, Storable often makes many very small writes, so it
43	is rather inefficient. But it does keep the latency low.
44
45	=item $pst = nfreeze $ref
46
47	Same as C<freeze> but is compatible to C<Storable::nstore_fd> (note the
48	C<n>).
49
50	=item $pst = blocking_freeze $ref
51
52	Same as C<freeze> but is guaranteed to block. This is useful e.g. in
53	C<Coro::Util::fork_eval> when you want to serialise a data structure
54	for use with the C<thaw> function for this module. You cannot use
55	C<Storable::freeze> for this as Storable uses incompatible formats for
56	memory and file images.
57
58	=item $pst = blocking_nfreeze $ref
59
60	Same as C<blocking_freeze> but uses C<nfreeze> internally.
61
62	=item $guard = guard
63
64	Acquire the Storable lock, for when you want to call Storable yourself.
65
66	Note that this module already wraps the Storable functions, so there is
67	rarely the need to do this yourself.
68
69	=back
70
71	=cut
72
73	package Coro::Storable;
74
75	use strict qw(subs vars);
76	no warnings;
77
78	use Coro ();
79	use Coro::Semaphore ();
80
81	BEGIN {
82	# suppress warnings
83	local $^W = 0;
84	require Storable;
85	}
86
87	use Storable;
88	use base "Exporter";
89
90	our $VERSION = 4.746;
91	our @EXPORT = qw(thaw freeze nfreeze blocking_thaw blocking_freeze blocking_nfreeze);
92
93	my $lock = new Coro::Semaphore;
94
95	sub guard {
96	$lock->guard
97	}
98
99	# wrap xs functions
100	for (qw(net_pstore mstore net_mstore pretrieve mretrieve dclone)) {
101	my $orig = \&{"Storable::$_"};
102	*{"Storable::$_"} = sub {
103	my $guard = $lock->guard;
104	&$orig
105	};
106	die if $@;
107	}
108
109	sub thaw($) {
110	open my $fh, "<:via(CoroCede)", \$_[0]
111	or die "cannot open pst via CoroCede: $!";
112	Storable::fd_retrieve $fh
113	}
114
115	sub freeze($) {
116	open my $fh, ">:via(CoroCede)", \my $buf
117	or die "cannot open pst via CoroCede: $!";
118	Storable::store_fd $_[0], $fh;
119	$buf
120	}
121
122	sub nfreeze($) {
123	open my $fh, ">:via(CoroCede)", \my $buf
124	or die "cannot open pst via CoroCede: $!";
125	Storable::nstore_fd $_[0], $fh;
126	$buf
127	}
128
129	sub blocking_thaw($) {
130	open my $fh, "<", \$_[0]
131	or die "cannot open pst: $!";
132	Storable::fd_retrieve $fh
133	}
134
135	sub blocking_freeze($) {
136	open my $fh, ">", \my $buf
137	or die "cannot open pst: $!";
138	Storable::store_fd $_[0], $fh;
139	close $fh;
140
141	$buf
142	}
143
144	sub blocking_nfreeze($) {
145	open my $fh, ">", \my $buf
146	or die "cannot open pst: $!";
147	Storable::nstore_fd $_[0], $fh;
148	close $fh;
149
150	$buf
151	}
152
153	package PerlIO::via::CoroCede;
154
155	# generic cede-on-read/write filtering layer
156
157	use Time::HiRes ("time");
158
159	our $GRANULARITY = 0.01;
160
161	my $next_cede;
162
163	sub PUSHED {
164	__PACKAGE__
165	}
166
167	sub FILL {
168	if ($next_cede <= time) {
169	$next_cede = time + $GRANULARITY; # calling time() twice usually is a net win
170	local *Storable::FILE;
171	Coro::cede ();
172	}
173
174	read $_[1], my $buf, 512
175	or return undef;
176
177	$buf
178	}
179
180	sub WRITE {
181	if ($next_cede <= time) {
182	$next_cede = time + $GRANULARITY;
183	local *Storable::FILE;
184	Coro::cede ();
185	}
186
187	(print {$_[2]} $_[1]) ? length $_[1] : -1
188	}
189
190	1;
191
192	=head1 AUTHOR
193
194	Marc Lehmann <schmorp@schmorp.de>
195	http://home.schmorp.de/
196
197	=cut
198
199