1 | package folder; |
1 | package folder; |
2 | |
2 | |
3 | BEGIN { *slog = \&::slog }; |
3 | BEGIN { *slog = \&::slog }; |
4 | |
4 | |
|
|
5 | use Fcntl; |
|
|
6 | use File::Sync (); |
|
|
7 | |
|
|
8 | use Inline Config => NAME => "syncmail::folder"; |
|
|
9 | use Inline C; |
|
|
10 | |
5 | use constant MDIFVERSION => 1; |
11 | use constant MDIFVERSION => 1; |
|
|
12 | |
|
|
13 | BEGIN { |
|
|
14 | if (1) { |
|
|
15 | use OpenSSL (); |
|
|
16 | *hash = \&OpenSSL::Digest::sha1_hex; |
|
|
17 | } elsif (0) { |
|
|
18 | # use Digest::SHA1 (); |
|
|
19 | my $digest = new Digest::SHA1; |
|
|
20 | *hash = sub { |
|
|
21 | $digest->reset; |
|
|
22 | $digest->add(@_); |
|
|
23 | $mid = $digest->hexdigest; |
|
|
24 | }; |
|
|
25 | } |
|
|
26 | } |
|
|
27 | |
|
|
28 | sub flushfh { |
|
|
29 | my $oldfh = select $_[0]; |
|
|
30 | $| = 1; |
|
|
31 | select $oldfh; |
|
|
32 | } |
|
|
33 | |
|
|
34 | # rename a file and fsync the directory |
|
|
35 | sub replace { |
|
|
36 | my ($src, $dst) = @_; |
|
|
37 | my $self = shift; |
|
|
38 | |
|
|
39 | rename $src, $dst; |
|
|
40 | |
|
|
41 | $dst =~ s/[^\/]*$/./; |
|
|
42 | |
|
|
43 | # now sync the directory |
|
|
44 | |
|
|
45 | open my $dir, "<", $dst |
|
|
46 | or die "$dst: $!"; |
|
|
47 | |
|
|
48 | File::Sync::fsync($dir); |
|
|
49 | } |
6 | |
50 | |
7 | sub new { |
51 | sub new { |
8 | my $class = shift; |
52 | my $class = shift; |
9 | my %arg = @_; |
53 | my %arg = @_; |
10 | bless { |
54 | my $self = bless { |
11 | path => "$::PREFIX/$arg{name}", |
55 | path => "$::PREFIX/$arg{name}", |
12 | %arg, |
56 | %arg, |
13 | }, $class; |
57 | }, $class; |
|
|
58 | $self->open(0); |
|
|
59 | $self; |
14 | } |
60 | } |
15 | |
61 | |
16 | sub dirty { |
62 | sub dirty { |
17 | $_[0]{dirty} = 1; |
63 | $_[0]{dirty} = 1; |
18 | } |
64 | } |
19 | |
65 | |
20 | sub DESTROY { |
66 | sub DESTROY { |
21 | $_[0]->write_mdif; |
67 | #$_[0]->write_mdif; # do NOT! |
22 | } |
68 | } |
23 | |
69 | |
24 | # parse_mbox(mbox-file-path, callback) |
70 | # parse_mbox(mbox-file-path, callback) |
25 | # callback gets called with \$header and \$body, |
71 | # callback gets called with \$header and \$body, |
26 | # $header includes the mbox From_ line without |
72 | # $header includes the mbox From_ line without |
27 | # the leading From_ itself. |
73 | # the leading From_ itself. |
28 | sub parse_mbox { |
74 | sub parse_mbox { |
29 | my ($path, $cb) = @_; |
75 | my ($fh, $cb) = @_; |
30 | |
|
|
31 | open my $fh, "<", $path |
|
|
32 | or die "$path: $!"; |
|
|
33 | |
76 | |
34 | local $/ = "\n\n"; |
77 | local $/ = "\n\n"; |
35 | |
78 | |
36 | my ($head, $body, $offs); |
79 | my ($head, $body, $offs); |
37 | |
80 | |
38 | 5 == read $fh, $head, 5 |
81 | read $fh, $head, 5; |
39 | or return; |
|
|
40 | |
82 | |
41 | $head eq "From " |
83 | $head eq "From " or $head eq "" |
42 | or return; |
84 | or return; |
43 | |
85 | |
44 | $offs = 0; |
86 | $offs = 0; |
45 | while (defined ($head = <$fh>)) { |
87 | while (defined ($head = <$fh>)) { |
46 | $head =~ /^.*? [A-Z][a-z][a-z] [A-Z][a-z][a-z] [ 0-9][0-9] \d\d:\d\d:\d\d(?: [+-]\d\d\d\d)? \d\d(?:\d\d)\n/ |
88 | $head =~ /^.*? [A-Z][a-z][a-z] [A-Z][a-z][a-z] [ 0-9][0-9] \d\d:\d\d:\d\d(?: [+-]\d\d\d\d)? \d\d(?:\d\d)\n/ |
… | |
… | |
55 | $body = <$fh>; |
97 | $body = <$fh>; |
56 | #} |
98 | #} |
57 | chomp $body; |
99 | chomp $body; |
58 | $cb->($offs, \$head, \$body); |
100 | $cb->($offs, \$head, \$body); |
59 | $offs = (tell $fh) - 5; |
101 | $offs = (tell $fh) - 5; |
60 | ::give unless ++$ecnt & 1023; |
102 | &::give unless ++$ecnt & 255; |
61 | } |
103 | } |
62 | |
104 | |
63 | 1; |
105 | 1; |
64 | } |
106 | } |
65 | |
107 | |
… | |
… | |
91 | last unless /^([^[].*)=(.*)\n$/; |
133 | last unless /^([^[].*)=(.*)\n$/; |
92 | $self->{host}{$1} = $2; |
134 | $self->{host}{$1} = $2; |
93 | } |
135 | } |
94 | } elsif (/^\[DIFF (\d+)\]\n$/) { |
136 | } elsif (/^\[DIFF (\d+)\]\n$/) { |
95 | my $mtime = $1; |
137 | my $mtime = $1; |
96 | my @dif; |
138 | my (@add, @del); |
97 | while (<$fh>) { |
139 | while (<$fh>) { |
98 | last unless /^[+-]/; |
140 | last unless /^([+-])(.*)\n$/; |
99 | push @dif, substr $_, 0, -1; |
141 | if ($1 eq "+") { |
|
|
142 | push @add, $2; |
|
|
143 | } else { |
|
|
144 | push @del, $2; |
|
|
145 | } |
100 | } |
146 | } |
101 | unshift @{$self->{diff}}, [$mtime, \@dif]; |
147 | unshift @{$self->{diff}}, [$mtime, \@add, \@del]; |
102 | } elsif ($_ eq "[INDEX]\n") { |
148 | } elsif ($_ eq "[INDEX]\n") { |
103 | my @idx; |
149 | my @idx; |
104 | while (<$fh>) { |
150 | while (<$fh>) { |
105 | last unless /^(\d+)=(.*)\n$/; |
151 | last unless /^(\d+)=(.*)\n$/; |
106 | push @idx, [$1, $2]; |
152 | push @idx, [$1, $2]; |
… | |
… | |
128 | or die "$path~: $!"; |
174 | or die "$path~: $!"; |
129 | |
175 | |
130 | print $fh "# automatically generated, do NOT edit\n"; |
176 | print $fh "# automatically generated, do NOT edit\n"; |
131 | |
177 | |
132 | print $fh "[SYNCMAIL]\n"; |
178 | print $fh "[SYNCMAIL]\n"; |
133 | print $fh "$_=$self->{$_}\n" for (qw(fsize mtime version)); |
179 | print $fh "$_=$self->{$_}\n" for (qw(fsize mtime version ctime)); |
134 | |
180 | |
135 | print $fh "[HOSTS]\n"; |
181 | print $fh "[HOSTS]\n"; |
136 | while (my ($k,$v) = each %{$self->{host}}) { |
182 | while (my ($k,$v) = each %{$self->{host}}) { |
137 | print $fh "$k=$v\n"; |
183 | print $fh "$k=$v\n"; |
138 | } |
184 | } |
… | |
… | |
140 | print $fh "[INDEX]\n"; |
186 | print $fh "[INDEX]\n"; |
141 | print $fh "$_->[0]=$_->[1]\n" for @{$self->{idx}}; |
187 | print $fh "$_->[0]=$_->[1]\n" for @{$self->{idx}}; |
142 | |
188 | |
143 | for (reverse @{$self->{diff}}) { |
189 | for (reverse @{$self->{diff}}) { |
144 | print $fh "[DIFF $_->[0]]\n"; |
190 | print $fh "[DIFF $_->[0]]\n"; |
145 | print $fh $_, "\n" for @{$_->[1]}; |
191 | print $fh "+$_\n" for @{$_->[1]}; |
|
|
192 | print $fh "-$_\n" for @{$_->[2]}; |
146 | } |
193 | } |
147 | |
194 | |
|
|
195 | flushfh($fh); |
|
|
196 | File::Sync::fsync($fh); |
148 | close $fh |
197 | close $fh |
149 | or die "$path~: unable to create updated .mdif: $!"; |
198 | or die "$path~: unable to create updated .mdif: $!"; |
150 | |
199 | |
151 | rename "$path~", $path; |
200 | replace("$path~", $path); |
152 | |
201 | |
153 | delete $self->{dirty}; |
202 | delete $self->{dirty}; |
154 | } |
203 | } |
155 | |
204 | |
156 | sub gendiff { |
205 | sub gendiff { |
157 | my ($d1, $d2) = @_; |
206 | my ($self, $d1, $d2) = @_; |
158 | |
207 | |
159 | my @d; |
208 | my (@add, @del); |
160 | my (%d1, %d2); |
209 | my (%d1, %d2); |
161 | |
210 | |
162 | for (@$d2) { |
211 | for (@$d2) { |
163 | undef $d2{$_->[1]}; |
212 | undef $d2{$_->[1]}; |
164 | } |
213 | } |
165 | |
214 | |
166 | # delete msgs in d1 but not in d2 |
215 | # delete msgs in d1 but not in d2 |
167 | for (@$d1) { |
216 | for (@$d1) { |
168 | undef $d1{$_->[1]}; |
217 | undef $d1{$_->[1]}; |
169 | push @d, "-$_->[1]" unless exists $d2{$_->[1]}; |
218 | push @del, $_->[1] unless exists $d2{$_->[1]}; |
170 | } |
219 | } |
171 | %d2 = (); # conserve memory |
220 | %d2 = (); # conserve memory |
172 | |
221 | |
173 | # add msgs in d2 but not in d1 |
222 | # add msgs in d2 but not in d1 |
174 | for (@$d2) { |
223 | for (@$d2) { |
175 | push @d, "+$_->[1]" unless exists $d1{$_->[1]}; |
224 | push @add, $_->[1] unless exists $d1{$_->[1]}; |
176 | } |
225 | } |
177 | |
226 | |
178 | \@d; |
227 | push @{$self->{diff}}, [ |
|
|
228 | $self->{ctime}, |
|
|
229 | \@add, \@del, |
|
|
230 | ] if @add || @del; |
|
|
231 | } |
|
|
232 | |
|
|
233 | sub open { |
|
|
234 | my ($self, $rw) = @_; |
|
|
235 | |
|
|
236 | if (!$self->{fh} || $self->{rw} != $rw) { |
|
|
237 | $self->close; |
|
|
238 | $self->{rw} = $rw; |
|
|
239 | sysopen $self->{fh}, $self->{path}, |
|
|
240 | O_CREAT | ($rw ? O_RDWR : O_RDONLY), |
|
|
241 | 0666 |
|
|
242 | or die "$self->{path}: $!"; |
|
|
243 | 0 == setlkw(fileno $self->{fh}, $rw ? 2 : 1) |
|
|
244 | or die "$self->{path}: $!"; |
|
|
245 | |
|
|
246 | $self->check; |
|
|
247 | } |
|
|
248 | } |
|
|
249 | |
|
|
250 | sub close { |
|
|
251 | my $self = shift; |
|
|
252 | |
|
|
253 | if ($self->{rw} && $self->{fh}) { |
|
|
254 | flushfh($self->{fh}); |
|
|
255 | File::Sync::fsync($self->{fh}); |
|
|
256 | } |
|
|
257 | |
|
|
258 | $self->write_mdif; |
|
|
259 | |
|
|
260 | delete $self->{fh}; |
179 | } |
261 | } |
180 | |
262 | |
181 | sub check { |
263 | sub check { |
182 | my $self = shift; |
264 | my $self = shift; |
183 | my $path = $self->{path}; |
|
|
184 | my $conf = $self->conf_path; |
265 | my $conf = $self->conf_path; |
185 | my $guard = $::lockdisk->guard; |
266 | my $guard = $::lockdisk->guard; |
186 | |
267 | |
187 | slog 3, "checking $path\n"; |
268 | slog 3, "checking $self->{path}\n"; |
188 | |
269 | |
189 | if (stat $path) { |
|
|
190 | my ($fsize, $mtime) = (stat _)[7, 9]; |
270 | my ($fsize, $mtime) = (stat $self->{fh})[7, 9]; |
191 | |
271 | |
|
|
272 | if ($self->{idx}) { |
|
|
273 | return 1 if $fsize == $self->{fsize} |
|
|
274 | && $mtime == $self->{mtime}; |
|
|
275 | } else { |
192 | if (open my $fh, "<", $conf) { |
276 | if (open my $fh, "<", $conf) { |
193 | my %conf; |
277 | my %conf; |
194 | <$fh>; # skip initial comment |
278 | <$fh>; # skip initial comment |
195 | <$fh> eq "[SYNCMAIL]\n" |
279 | <$fh> eq "[SYNCMAIL]\n" |
196 | or die "$conf: format error"; |
280 | or die "$conf: format error"; |
… | |
… | |
199 | } |
283 | } |
200 | return 1 if $fsize == $conf{fsize} |
284 | return 1 if $fsize == $conf{fsize} |
201 | && $mtime == $conf{mtime}; |
285 | && $mtime == $conf{mtime}; |
202 | |
286 | |
203 | $conf{mtime} <= $mtime |
287 | $conf{mtime} <= $mtime |
204 | or die "$path: folder older than mdif"; |
288 | or die "$self->{path}: folder older than mdif"; |
205 | } |
289 | } |
|
|
290 | } |
206 | |
291 | |
207 | slog 2, "updating $path\n"; |
292 | slog 2, "updating $self->{path}\n"; |
208 | |
293 | |
209 | my @idx; |
294 | my @idx; |
210 | |
295 | |
|
|
296 | seek $self->{fh}, 0, SEEK_SET; |
|
|
297 | |
211 | parse_mbox $path, sub { |
298 | parse_mbox $self->{fh}, sub { |
212 | my ($offs, $head, $body) = @_; |
299 | my ($offs, $head, $body) = @_; |
|
|
300 | push @idx, [$offs, hash($$head, "\n\n", $$body)]; |
|
|
301 | } or die "$self->{path}: no valid mbox file"; |
|
|
302 | |
|
|
303 | $self->read_mdif; |
|
|
304 | |
|
|
305 | $self->{version} ||= MDIFVERSION; |
|
|
306 | $self->{ctime} = time; |
|
|
307 | |
|
|
308 | $self->gendiff($self->{idx}, \@idx); |
|
|
309 | |
|
|
310 | $self->{fsize} = $fsize; |
|
|
311 | $self->{mtime} = $mtime; |
|
|
312 | $self->{idx} = \@idx; |
|
|
313 | |
|
|
314 | $self->dirty; |
|
|
315 | $self->write_mdif;#d# |
|
|
316 | } |
|
|
317 | |
|
|
318 | sub inventory { |
|
|
319 | hash sort map { $_->[1] } @{$_[0]{idx}}; |
|
|
320 | } |
|
|
321 | |
|
|
322 | sub iidx { |
|
|
323 | my $self = shift; |
|
|
324 | |
|
|
325 | $self->{iidx} ||= do { |
|
|
326 | my %iidx; |
|
|
327 | my $idx = $self->{idx}; |
|
|
328 | |
|
|
329 | push @$idx, [$self->{fsize}]; |
|
|
330 | my $ofs = 0; |
|
|
331 | for (0 .. @$idx - 2) { |
|
|
332 | $iidx{$idx->[$_][1]} = [$idx->[$_][0], $idx->[$_+1][0] - $idx->[$_][0]]; |
|
|
333 | } |
|
|
334 | pop @$idx, [$self->{fsize}]; |
|
|
335 | |
|
|
336 | \%iidx; |
|
|
337 | }; |
|
|
338 | } |
|
|
339 | |
|
|
340 | sub exists { |
|
|
341 | $_[0]->iidx unless $_[0]{iidx}; |
|
|
342 | return $_[0]{iidx}{$_[1]}; |
|
|
343 | } |
|
|
344 | |
|
|
345 | sub fetch { |
|
|
346 | my ($self, $hash) = @_; |
|
|
347 | |
|
|
348 | $self->iidx unless $self->{iidx}; |
|
|
349 | |
|
|
350 | my $mail; |
|
|
351 | |
|
|
352 | my $msg = $self->{iidx}{$hash} |
|
|
353 | or die "$hash: no such message in $self->{path}"; |
|
|
354 | |
|
|
355 | seek $self->{fh}, $msg->[0], SEEK_SET |
|
|
356 | or die "$self->{path}: $!"; |
|
|
357 | |
|
|
358 | $msg->[1] == read $self->{fh}, $mail, $msg->[1] |
|
|
359 | or die "$self->{path}: $!"; |
|
|
360 | |
|
|
361 | $mail =~ /^From \S/ |
|
|
362 | or die "$self->{path}: mail folder corrupted"; |
|
|
363 | |
|
|
364 | $mail; |
|
|
365 | } |
|
|
366 | |
|
|
367 | # begin updating folder |
|
|
368 | sub begin_update { |
|
|
369 | my $self = shift; |
|
|
370 | |
|
|
371 | $self->{oidx} = $self->{idx}; |
|
|
372 | } |
|
|
373 | |
|
|
374 | sub delete { |
|
|
375 | my $self = shift; |
|
|
376 | my $temp = "$self->{path}~"; |
|
|
377 | |
|
|
378 | $self->iidx unless $self->{iidx}; |
|
|
379 | |
|
|
380 | for (@_) { |
|
|
381 | if (exists $self->{iidx}{$_}) { # at least one message exists |
|
|
382 | my $guard = $::lockdisk->guard; |
|
|
383 | my %del; @del{@_} = (); |
213 | my $mid; |
384 | my @nidx; |
214 | if ($$head =~ /^Message-Id:\s*(<[^<\n]+>)\s*\n/im) { |
385 | |
|
|
386 | open my $fh, ">", $temp |
|
|
387 | or die "$temp: $!"; |
|
|
388 | |
|
|
389 | eval { |
|
|
390 | 0 == setlkw(fileno $fh, 2) |
|
|
391 | or die "$self->{path}~: $!"; |
|
|
392 | |
|
|
393 | $self->{fsize} = 0; # we virtually truncated the file |
|
|
394 | |
215 | $mid = $1; |
395 | my $dofs = 0; |
|
|
396 | for (@{delete $self->{idx}}) { |
|
|
397 | my $hash = $_->[1]; |
|
|
398 | my $buf; |
|
|
399 | |
|
|
400 | unless (exists $del{$hash}) { |
|
|
401 | my ($ofs, $len) = @{$self->{iidx}{$hash}}; |
|
|
402 | |
|
|
403 | $len or die; |
|
|
404 | |
|
|
405 | seek $self->{fh}, $ofs, SEEK_SET |
|
|
406 | or die "$self->{path}: $!"; |
|
|
407 | |
|
|
408 | $len == read $self->{fh}, $buf, $len |
|
|
409 | or die "$self->{path}: $!"; |
|
|
410 | |
|
|
411 | $buf =~ /^From \S/ |
|
|
412 | or die "$self->{path}: corrupted mail folder"; |
|
|
413 | |
|
|
414 | print $fh $buf |
|
|
415 | or die "$self->{path}: $!"; |
|
|
416 | |
|
|
417 | push @nidx, [$dofs, $hash]; |
|
|
418 | $self->{iidx}{$hash}[0] = $dofs; |
|
|
419 | $dofs += $len; |
|
|
420 | |
|
|
421 | &::give unless ++$ecnt & 255; |
216 | } else { |
422 | } else { |
217 | $mid = MD5->hexhash("$$head\0$$body"); |
423 | delete $self->{iidx}{$hash}; |
|
|
424 | slog 0, "skipping/deleting $hash\n"; |
|
|
425 | } |
|
|
426 | } |
|
|
427 | }; |
|
|
428 | |
|
|
429 | if ($@) { |
|
|
430 | close $fh; |
|
|
431 | unlink $temp; |
|
|
432 | die; |
218 | } |
433 | } |
219 | push @idx, [$offs, $mid]; |
434 | |
|
|
435 | File::Sync::fsync($fh); |
|
|
436 | replace $temp, $self->{path}; |
|
|
437 | |
|
|
438 | $self->{fh} = $fh; |
|
|
439 | $self->{rw} = 1; |
|
|
440 | |
|
|
441 | delete $self->{iidx}; |
|
|
442 | |
|
|
443 | $self->{idx} = \@nidx; |
|
|
444 | $self->{fsize} = $ofs; |
|
|
445 | |
220 | } or return (); |
446 | return; |
221 | |
|
|
222 | $self->read_mdif; |
|
|
223 | |
|
|
224 | if ($self->{version}) { |
|
|
225 | my $d = gendiff $self->{idx}, \@idx; |
|
|
226 | push @{$self->{diff}}, [ |
|
|
227 | $self->{mtime}, |
|
|
228 | $d, |
|
|
229 | ] if @$d; |
|
|
230 | } else { |
|
|
231 | slog 2, "$path: previously unknown folder\n"; |
|
|
232 | $self->{version} ||= MDIFVERSION; |
|
|
233 | } |
447 | } |
|
|
448 | } |
|
|
449 | } |
234 | |
450 | |
|
|
451 | sub append { |
|
|
452 | my ($self, $hash, $mail) = @_; |
|
|
453 | |
|
|
454 | if (length $mail) { |
|
|
455 | $self->open(1); |
|
|
456 | |
|
|
457 | seek $self->{fh}, $self->{fsize}, SEEK_SET |
|
|
458 | or die "$self->{path}: $!"; |
|
|
459 | |
|
|
460 | print {$self->{fh}} $mail |
|
|
461 | or die "$self->{path}: $!"; |
|
|
462 | |
|
|
463 | push @{$self->{idx}}, [$self->{fsize}, $hash]; |
235 | $self->{fsize} = $fsize; |
464 | $self->{fsize} += length $mail; |
236 | $self->{mtime} = $mtime; |
465 | } |
237 | $self->{idx} = \@idx; |
466 | } |
238 | |
467 | |
|
|
468 | sub end_update { |
|
|
469 | my $self = shift; |
|
|
470 | |
|
|
471 | $self->gendiff((delete $self->{oidx}, $self->{idx})); |
|
|
472 | |
|
|
473 | flushfh($self->{fh}); |
|
|
474 | File::Sync::fsync($self->{fh}); |
|
|
475 | |
|
|
476 | stat $self->{fh} |
|
|
477 | or die "$self->{path}: $!"; |
|
|
478 | |
|
|
479 | $self->{fsize} = (stat _)[7]; |
|
|
480 | $self->{mtime} = (stat _)[9]; |
|
|
481 | |
239 | $self->dirty; |
482 | $self->dirty; |
240 | |
|
|
241 | return 2; |
|
|
242 | } else { |
|
|
243 | slog 2, "$path: no longer exists\n"; |
|
|
244 | unlink $conf; |
|
|
245 | |
|
|
246 | return (); |
|
|
247 | } |
|
|
248 | } |
483 | } |
249 | |
484 | |
250 | 1; |
485 | 1; |
251 | |
486 | |
|
|
487 | __DATA__ |
|
|
488 | __C__ |
|
|
489 | #include <unistd.h> |
|
|
490 | #include <fcntl.h> |
|
|
491 | |
|
|
492 | /* mode0 unlock, mode1 rlock, mode2 rwlock */ |
|
|
493 | int setlkw(int fd, int mode) |
|
|
494 | { |
|
|
495 | struct flock l; |
|
|
496 | |
|
|
497 | l.l_type = mode == 0 ? F_UNLCK |
|
|
498 | : mode == 1 ? F_RDLCK |
|
|
499 | : F_WRLCK; |
|
|
500 | l.l_whence = SEEK_SET; |
|
|
501 | l.l_start = 0; |
|
|
502 | l.l_len = 0; |
|
|
503 | |
|
|
504 | return fcntl (fd, F_SETLKW, &l); |
|
|
505 | } |
|
|
506 | |