… | |
… | |
2 | |
2 | |
3 | our $SCHEDULE_INTERVAL = $cf::CFG{extractor_schedule_interval} || 3600; |
3 | our $SCHEDULE_INTERVAL = $cf::CFG{extractor_schedule_interval} || 3600; |
4 | |
4 | |
5 | use JSON::XS; |
5 | use JSON::XS; |
6 | |
6 | |
7 | my $db_mapinfo = cf::sync_job { cf::db_table "tag-mapinfo" }; # info/cache for maps |
7 | our $db_mapinfo = cf::sync_job { cf::db_table "tag-mapinfo" }; # info/cache for maps |
8 | my $db_target = cf::sync_job { cf::db_table "tag-target" }; # tag => maps |
8 | our $db_target = cf::sync_job { cf::db_table "tag-target" }; # tag => maps |
9 | |
9 | |
10 | sub remove_tag_target { |
10 | sub remove_tag_target { |
11 | my ($txn, $tag, $target) = @_; |
11 | my ($txn, $tag, $target) = @_; |
12 | # - U O |
12 | # - U O |
13 | |
13 | |
… | |
… | |
40 | Coro::AIO::aio_stat $file |
40 | Coro::AIO::aio_stat $file |
41 | and next; |
41 | and next; |
42 | |
42 | |
43 | my $hash = join ",", 1, (stat _)[7,9], $file; |
43 | my $hash = join ",", 1, (stat _)[7,9], $file; |
44 | |
44 | |
45 | my $old_tags; |
|
|
46 | |
|
|
47 | my $txn = $cf::DB_ENV->txn_begin; |
45 | my $txn = $cf::DB_ENV->txn_begin; |
48 | |
46 | |
49 | utf8::encode $key; |
47 | utf8::encode $key; |
50 | BDB::db_get $db_mapinfo, $txn, $key, my $data; |
48 | BDB::db_get $db_mapinfo, $txn, $key, my $data; |
51 | |
49 | |
52 | unless ($!) { |
50 | unless ($!) { |
53 | $data = decode_json $data; |
51 | $data = decode_json $data; |
54 | return if $data->{hash} eq $hash; |
52 | return if $data->{hash} eq $hash; |
55 | $old_tags = $data->{tags}; |
53 | |
|
|
54 | # remove all old tags unconditionally |
|
|
55 | remove_tag_target $txn, $_, $key |
|
|
56 | for @{ $data->{tags} }; |
56 | } |
57 | } |
57 | |
|
|
58 | $old_tags ||= []; |
|
|
59 | |
58 | |
60 | my $f = new_from_file cf::object::thawer $file |
59 | my $f = new_from_file cf::object::thawer $file |
61 | or return; |
60 | or return; |
62 | |
61 | |
63 | my @tags = sort $f->extract_tags; |
62 | my @tags = sort $f->extract_tags; |
64 | $data = encode_json { hash => $hash, tags => \@tags }; |
63 | $data = encode_json { hash => $hash, tags => \@tags }; |
65 | |
64 | |
66 | BDB::db_put $db_mapinfo, $txn, $key, $data; |
65 | BDB::db_put $db_mapinfo, $txn, $key, $data; |
67 | |
66 | |
68 | # 1. remove tags no longer existing |
67 | # add all tags |
69 | for my $tag (@$old_tags) { |
|
|
70 | next if grep $_ eq $tag, @tags; |
|
|
71 | remove_tag_target $txn, $tag, $key; |
|
|
72 | } |
|
|
73 | |
|
|
74 | # 2. add tags that are new |
|
|
75 | for my $tag (@tags) { |
|
|
76 | next if grep $_ eq $tag, @$old_tags; |
|
|
77 | add_tag_target $txn, $tag, $key; |
68 | add_tag_target $txn, $_, $key |
78 | } |
69 | for @tags; |
79 | |
70 | |
80 | # we don't actually care if it succeeds or not, as we |
71 | # we don't actually care if it succeeds or not, as we |
81 | # will just retry an hour later |
72 | # will just retry an hour later |
82 | BDB::db_txn_finish $txn; |
73 | BDB::db_txn_finish $txn; |
83 | |
74 | |
84 | warn "tag-updated $file (= $key) <@tags>\n" |
75 | warn "tag-updated $file (= $key) <@tags>\n" |
85 | if @tags; |
76 | if @tags; |
86 | } |
77 | } |
87 | |
78 | |
88 | sub scan_static { |
79 | sub scan_static { |
89 | my ($dir, $map) = @_; |
80 | my $maps = cf::map::static_maps; |
90 | |
81 | |
91 | my ($dirs, $files) = Coro::AIO::aio_scandir $dir, 2 |
82 | scan_map "s$_", "$cf::MAPDIR$_.map" |
92 | or return; |
|
|
93 | |
|
|
94 | for my $file (@$files) { |
|
|
95 | my $name = $file; |
|
|
96 | next unless $name =~ s/\.map$//; |
|
|
97 | utf8::decode $name; |
|
|
98 | |
|
|
99 | scan_map "s$map$name", "$dir/$file"; |
|
|
100 | } |
|
|
101 | |
|
|
102 | &scan_static ("$dir/$_", "$map$_/") |
|
|
103 | for @$dirs; |
83 | for @$maps; |
104 | } |
84 | } |
105 | |
85 | |
106 | sub reload { |
86 | sub reload { |
107 | my $guard = cf::lock_acquire "map-tags::reload"; |
87 | my $guard = cf::lock_acquire "map-tags::reload"; |
108 | |
88 | |
… | |
… | |
196 | grep $_, |
176 | grep $_, |
197 | map { cf::map::find $_ } |
177 | map { cf::map::find $_ } |
198 | grep s/^s//, |
178 | grep s/^s//, |
199 | split /\x00/, $data |
179 | split /\x00/, $data |
200 | } |
180 | } |
|
|
181 | |
|
|
182 | sub unload { |
|
|
183 | my $guard = cf::lock_acquire "map-tags::reload"; |
|
|
184 | |
|
|
185 | db_close $db_target; |
|
|
186 | db_close $db_mapinfo; |
|
|
187 | } |