ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/syncmail/folder.pm
(Generate patch)

Comparing syncmail/folder.pm (file contents):
Revision 1.1 by root, Sat Oct 27 23:53:49 2001 UTC vs.
Revision 1.6 by root, Mon Oct 29 02:36:32 2001 UTC

1package folder; 1package folder;
2 2
3BEGIN { *slog = \&::slog }; 3BEGIN { *slog = \&::slog };
4 4
5use Fcntl;
6use File::Sync ();
7
8use Inline Config => NAME => "syncmail::folder";
9use Inline C;
10
5use constant MDIFVERSION => 1; 11use constant MDIFVERSION => 1;
12
13BEGIN {
14 if (1) {
15 use OpenSSL ();
16 *hash = \&OpenSSL::Digest::sha1_hex;
17 } elsif (0) {
18 # use Digest::SHA1 ();
19 my $digest = new Digest::SHA1;
20 *hash = sub {
21 $digest->reset;
22 $digest->add(@_);
23 $mid = $digest->hexdigest;
24 };
25 }
26}
27
28sub fflush {
29 my $oldfh = select $_[0];
30 $| = 1; $| = 0;
31 select $oldfh;
32}
33
34# rename a file and fsync the directory
35sub replace {
36 my ($src, $dst) = @_;
37 my $self = shift;
38
39 rename $src, $dst;
40
41 $dst =~ s/[^\/]*$/./;
42
43 # now sync the directory
44
45 open my $dir, "<", $dst
46 or die "$dst: $!";
47
48 File::Sync::fsync($dir);
49}
6 50
7sub new { 51sub new {
8 my $class = shift; 52 my $class = shift;
9 my %arg = @_; 53 my %arg = @_;
10 bless { 54 my $self = bless {
11 path => "$::PREFIX/$arg{name}", 55 path => "$::PREFIX/$arg{name}",
12 %arg, 56 %arg,
13 }, $class; 57 }, $class;
58 $self->open(0);
59 $self;
14} 60}
15 61
16sub dirty { 62sub dirty {
17 $_[0]{dirty} = 1; 63 $_[0]{dirty} = 1;
18} 64}
19 65
20sub DESTROY { 66sub DESTROY {
21 $_[0]->write_mdif; 67 #$_[0]->write_mdif; # do NOT!
22} 68}
23 69
24# parse_mbox(mbox-file-path, callback) 70# parse_mbox(mbox-file-path, callback)
25# callback gets called with \$header and \$body, 71# callback gets called with \$header and \$body,
26# $header includes the mbox From_ line without 72# $header includes the mbox From_ line without
27# the leading From_ itself. 73# the leading From_ itself.
28sub parse_mbox { 74sub parse_mbox {
29 my ($path, $cb) = @_; 75 my ($fh, $cb) = @_;
30
31 open my $fh, "<", $path
32 or die "$path: $!";
33 76
34 local $/ = "\n\n"; 77 local $/ = "\n\n";
35 78
36 my ($head, $body, $offs); 79 my ($head, $body, $offs);
37 80
38 5 == read $fh, $head, 5 81 read $fh, $head, 5;
39 or return;
40 82
41 $head eq "From " 83 $head eq "From " or $head eq ""
42 or return; 84 or return;
43 85
44 $offs = 0; 86 $offs = 0;
45 while (defined ($head = <$fh>)) { 87 while (defined ($head = <$fh>)) {
46 $head =~ /^.*? [A-Z][a-z][a-z] [A-Z][a-z][a-z] [ 0-9][0-9] \d\d:\d\d:\d\d(?: [+-]\d\d\d\d)? \d\d(?:\d\d)\n/ 88 $head =~ /^.*? [A-Z][a-z][a-z] [A-Z][a-z][a-z] [ 0-9][0-9] \d\d:\d\d:\d\d(?: [+-]\d\d\d\d)? \d\d(?:\d\d)\n/
55 $body = <$fh>; 97 $body = <$fh>;
56 #} 98 #}
57 chomp $body; 99 chomp $body;
58 $cb->($offs, \$head, \$body); 100 $cb->($offs, \$head, \$body);
59 $offs = (tell $fh) - 5; 101 $offs = (tell $fh) - 5;
60 ::give unless ++$ecnt & 1023; 102 &::give unless ++$ecnt & 255;
61 } 103 }
62 104
63 1; 105 1;
64} 106}
65 107
91 last unless /^([^[].*)=(.*)\n$/; 133 last unless /^([^[].*)=(.*)\n$/;
92 $self->{host}{$1} = $2; 134 $self->{host}{$1} = $2;
93 } 135 }
94 } elsif (/^\[DIFF (\d+)\]\n$/) { 136 } elsif (/^\[DIFF (\d+)\]\n$/) {
95 my $mtime = $1; 137 my $mtime = $1;
96 my @dif; 138 my (@add, @del);
97 while (<$fh>) { 139 while (<$fh>) {
98 last unless /^[+-]/; 140 last unless /^([+-])(.*)\n$/;
99 push @dif, substr $_, 0, -1; 141 if ($1 eq "+") {
142 push @add, $2;
143 } else {
144 push @del, $2;
145 }
100 } 146 }
101 unshift @{$self->{diff}}, [$mtime, \@dif]; 147 unshift @{$self->{diff}}, [$mtime, \@add, \@del];
102 } elsif ($_ eq "[INDEX]\n") { 148 } elsif ($_ eq "[INDEX]\n") {
103 my @idx; 149 my @idx;
104 while (<$fh>) { 150 while (<$fh>) {
105 last unless /^(\d+)=(.*)\n$/; 151 last unless /^(\d+)=(.*)\n$/;
106 push @idx, [$1, $2]; 152 push @idx, [$1, $2];
128 or die "$path~: $!"; 174 or die "$path~: $!";
129 175
130 print $fh "# automatically generated, do NOT edit\n"; 176 print $fh "# automatically generated, do NOT edit\n";
131 177
132 print $fh "[SYNCMAIL]\n"; 178 print $fh "[SYNCMAIL]\n";
133 print $fh "$_=$self->{$_}\n" for (qw(fsize mtime version)); 179 print $fh "$_=$self->{$_}\n" for (qw(fsize mtime version ctime));
134 180
135 print $fh "[HOSTS]\n"; 181 print $fh "[HOSTS]\n";
136 while (my ($k,$v) = each %{$self->{host}}) { 182 while (my ($k,$v) = each %{$self->{host}}) {
137 print $fh "$k=$v\n"; 183 print $fh "$k=$v\n";
138 } 184 }
140 print $fh "[INDEX]\n"; 186 print $fh "[INDEX]\n";
141 print $fh "$_->[0]=$_->[1]\n" for @{$self->{idx}}; 187 print $fh "$_->[0]=$_->[1]\n" for @{$self->{idx}};
142 188
143 for (reverse @{$self->{diff}}) { 189 for (reverse @{$self->{diff}}) {
144 print $fh "[DIFF $_->[0]]\n"; 190 print $fh "[DIFF $_->[0]]\n";
145 print $fh $_, "\n" for @{$_->[1]}; 191 print $fh "+$_\n" for @{$_->[1]};
192 print $fh "-$_\n" for @{$_->[2]};
146 } 193 }
147 194
195 fflush($fh);
196 File::Sync::fsync($fh);
148 close $fh 197 close $fh
149 or die "$path~: unable to create updated .mdif: $!"; 198 or die "$path~: unable to create updated .mdif: $!";
150 199
151 rename "$path~", $path; 200 replace("$path~", $path);
152 201
153 delete $self->{dirty}; 202 delete $self->{dirty};
154} 203}
155 204
156sub gendiff { 205sub gendiff {
157 my ($d1, $d2) = @_; 206 my ($self, $d1, $d2) = @_;
158 207
159 my @d; 208 my (@add, @del);
160 my (%d1, %d2); 209 my (%d1, %d2);
161 210
162 for (@$d2) { 211 for (@$d2) {
163 undef $d2{$_->[1]}; 212 undef $d2{$_->[1]};
164 } 213 }
165 214
166 # delete msgs in d1 but not in d2 215 # delete msgs in d1 but not in d2
167 for (@$d1) { 216 for (@$d1) {
168 undef $d1{$_->[1]}; 217 undef $d1{$_->[1]};
169 push @d, "-$_->[1]" unless exists $d2{$_->[1]}; 218 push @del, $_->[1] unless exists $d2{$_->[1]};
170 } 219 }
171 %d2 = (); # conserve memory 220 %d2 = (); # conserve memory
172 221
173 # add msgs in d2 but not in d1 222 # add msgs in d2 but not in d1
174 for (@$d2) { 223 for (@$d2) {
175 push @d, "+$_->[1]" unless exists $d1{$_->[1]}; 224 push @add, $_->[1] unless exists $d1{$_->[1]};
176 } 225 }
177 226
178 \@d; 227 push @{$self->{diff}}, [
228 $self->{ctime},
229 \@add, \@del,
230 ] if @add || @del;
231}
232
233sub open {
234 my ($self, $rw) = @_;
235
236 if (!$self->{fh} || $self->{rw} != $rw) {
237 $self->close;
238 $self->{rw} = $rw;
239 sysopen $self->{fh}, $self->{path},
240 O_CREAT | ($rw ? O_RDWR : O_RDONLY),
241 0666
242 or die "$self->{path}: $!";
243 0 == setlkw(fileno $self->{fh}, $rw ? 2 : 1)
244 or die "$self->{path}: $!";
245
246 $self->check;
247 }
248}
249
250sub close {
251 my $self = shift;
252
253 if ($self->{rw} && $self->{fh}) {
254 fflush($self->{fh});
255 File::Sync::fsync($self->{fh});
256 }
257
258 $self->write_mdif;
259
260 delete $self->{fh};
179} 261}
180 262
181sub check { 263sub check {
182 my $self = shift; 264 my $self = shift;
183 my $path = $self->{path};
184 my $conf = $self->conf_path; 265 my $conf = $self->conf_path;
185 my $guard = $::lockdisk->guard; 266 my $guard = $::lockdisk->guard;
186 267
187 slog 3, "checking $path\n"; 268 slog 3, "checking $self->{path}\n";
188 269
189 if (stat $path) {
190 my ($fsize, $mtime) = (stat _)[7, 9]; 270 my ($fsize, $mtime) = (stat $self->{fh})[7, 9];
191 271
272 if ($self->{idx}) {
273 return 1 if $fsize == $self->{fsize}
274 && $mtime == $self->{mtime};
275 } else {
192 if (open my $fh, "<", $conf) { 276 if (open my $fh, "<", $conf) {
193 my %conf; 277 my %conf;
194 <$fh>; # skip initial comment 278 <$fh>; # skip initial comment
195 <$fh> eq "[SYNCMAIL]\n" 279 <$fh> eq "[SYNCMAIL]\n"
196 or die "$conf: format error"; 280 or die "$conf: format error";
199 } 283 }
200 return 1 if $fsize == $conf{fsize} 284 return 1 if $fsize == $conf{fsize}
201 && $mtime == $conf{mtime}; 285 && $mtime == $conf{mtime};
202 286
203 $conf{mtime} <= $mtime 287 $conf{mtime} <= $mtime
204 or die "$path: folder older than mdif"; 288 or die "$self->{path}: folder older than mdif";
205 } 289 }
290 }
206 291
207 slog 2, "updating $path\n"; 292 slog 2, "updating $self->{path}\n";
208 293
209 my @idx; 294 my @idx;
210 295
296 seek $self->{fh}, 0, SEEK_SET;
297
211 parse_mbox $path, sub { 298 parse_mbox $self->{fh}, sub {
212 my ($offs, $head, $body) = @_; 299 my ($offs, $head, $body) = @_;
300 push @idx, [$offs, hash($$head, "\n\n", $$body)];
301 } or die "$self->{path}: no valid mbox file";
302
303 $self->read_mdif;
304
305 $self->{version} ||= MDIFVERSION;
306 $self->{ctime} = time;
307
308 $self->gendiff($self->{idx}, \@idx);
309
310 $self->{fsize} = $fsize;
311 $self->{mtime} = $mtime;
312 $self->{idx} = \@idx;
313
314 $self->dirty;
315 $self->write_mdif;#d#
316}
317
318sub inventory {
319 hash sort map { $_->[1] } @{$_[0]{idx}};
320}
321
322sub iidx {
323 my $self = shift;
324
325 $self->{iidx} ||= do {
326 my %iidx;
327 my $idx = $self->{idx};
328
329 push @$idx, [$self->{fsize}];
330 my $ofs = 0;
331 for (0 .. @$idx - 2) {
332 $iidx{$idx->[$_][1]} = [$idx->[$_][0], $idx->[$_+1][0] - $idx->[$_][0]];
333 }
334 pop @$idx, [$self->{fsize}];
335
336 \%iidx;
337 };
338}
339
340sub exists {
341 $_[0]->iidx unless $_[0]{iidx};
342 return $_[0]{iidx}{$_[1]};
343}
344
345sub fetch {
346 my ($self, $hash) = @_;
347
348 $self->iidx unless $self->{iidx};
349
350 my $mail;
351
352 my $msg = $self->{iidx}{$hash}
353 or die "$hash: no such message in $self->{path}";
354
355 print STDERR "$self->{fh}, $msg->[0], SEEK_SET\n";#d#
356
357 seek $self->{fh}, $msg->[0], SEEK_SET
358 or die "$self->{path}: $!";
359
360 print STDERR "$msg->[1] == read $self->{fh}, $msg->[1]\n";#d#
361
362 $msg->[1] == read $self->{fh}, $mail, $msg->[1]
363 or die "$self->{path}: $!";
364
365 $mail =~ /^From \S/
366 or die "$self->{path}: mail folder corrupted";
367
368 $mail;
369}
370
371# begin updating folder
372sub begin_update {
373 my $self = shift;
374
375 $self->{oidx} = $self->{idx};
376}
377
378sub delete {
379 my $self = shift;
380 my $temp = "$self->{path}~";
381
382 $self->iidx unless $self->{iidx};
383
384 for (@_) {
385 if (exists $self->{iidx}{$_}) { # at least one message exists
386 my $guard = $::lockdisk->guard;
387 my %del; @del{@_} = ();
213 my $mid; 388 my @nidx;
214 if ($$head =~ /^Message-Id:\s*(<[^<\n]+>)\s*\n/im) { 389 my $dofs = 0;
215 $mid = $1; 390
391 open my $fh, "+>", $temp
392 or die "$temp: $!";
393
394 eval {
395 0 == setlkw(fileno $fh, 2)
396 or die "$self->{path}~: $!";
397
398 $self->{fsize} = 0; # we virtually truncated the file
399
400 for (@{delete $self->{idx}}) {
401 my $hash = $_->[1];
402 my $buf;
403
404 unless (exists $del{$hash}) {
405 my ($ofs, $len) = @{$self->{iidx}{$hash}};
406
407 $len or die;
408
409 seek $self->{fh}, $ofs, SEEK_SET
410 or die "$self->{path}: $!";
411
412 $len == read $self->{fh}, $buf, $len
413 or die "$self->{path}: $!";
414
415 $buf =~ /^From \S/
416 or die "$self->{path}: corrupted mail folder";
417
418 print $fh $buf
419 or die "$self->{path}: $!";
420
421 push @nidx, [$dofs, $hash];
422 $self->{iidx}{$hash}[0] = $dofs;
423 $dofs += $len;
424
425 &::give unless ++$ecnt & 255;
216 } else { 426 } else {
217 $mid = MD5->hexhash("$$head\0$$body"); 427 delete $self->{iidx}{$hash};
428 slog 0, "skipping/deleting $hash\n";
429 }
430 }
431 };
432
433 if ($@) {
434 close $fh;
435 unlink $temp;
436 die;
218 } 437 }
219 push @idx, [$offs, $mid]; 438
439 fflush($fh);
440 File::Sync::fsync($fh);
441 replace $temp, $self->{path};
442
443 $self->{fh} = $fh;
444 $self->{rw} = 1;
445
446 delete $self->{iidx};
447
448 $self->{idx} = \@nidx;
449 $self->{fsize} = $dofs;
450
220 } or return (); 451 return;
221
222 $self->read_mdif;
223
224 if ($self->{version}) {
225 my $d = gendiff $self->{idx}, \@idx;
226 push @{$self->{diff}}, [
227 $self->{mtime},
228 $d,
229 ] if @$d;
230 } else {
231 slog 2, "$path: previously unknown folder\n";
232 $self->{version} ||= MDIFVERSION;
233 } 452 }
453 }
454}
234 455
456sub append {
457 my ($self, $hash, $mail) = @_;
458
459 if (length $mail) {
460 $self->open(1);
461
462 seek $self->{fh}, $self->{fsize}, SEEK_SET
463 or die "$self->{path}: $!";
464
465 print {$self->{fh}} $mail
466 or die "$self->{path}: $!";
467
468 push @{$self->{idx}}, [$self->{fsize}, $hash];
235 $self->{fsize} = $fsize; 469 $self->{fsize} += length $mail;
236 $self->{mtime} = $mtime; 470 }
237 $self->{idx} = \@idx; 471}
238 472
473sub end_update {
474 my $self = shift;
475
476 $self->gendiff((delete $self->{oidx}), $self->{idx});
477
478 fflush($self->{fh});
479 File::Sync::fsync($self->{fh});
480
481 stat $self->{fh}
482 or die "$self->{path}: $!";
483
484 $self->{fsize} = (stat _)[7];
485 $self->{mtime} = (stat _)[9];
486
239 $self->dirty; 487 $self->dirty;
240
241 return 2;
242 } else {
243 slog 2, "$path: no longer exists\n";
244 unlink $conf;
245
246 return ();
247 }
248} 488}
249 489
2501; 4901;
251 491
492__DATA__
493__C__
494#include <unistd.h>
495#include <fcntl.h>
496
497/* mode0 unlock, mode1 rlock, mode2 rwlock */
498int setlkw(int fd, int mode)
499{
500 struct flock l;
501
502 l.l_type = mode == 0 ? F_UNLCK
503 : mode == 1 ? F_RDLCK
504 : F_WRLCK;
505 l.l_whence = SEEK_SET;
506 l.l_start = 0;
507 l.l_len = 0;
508
509 return fcntl (fd, F_SETLKW, &l);
510}
511

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines