ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/deliantra/server/utils/cfutil.in
(Generate patch)

Comparing deliantra/server/utils/cfutil.in (file contents):
Revision 1.107 by root, Wed Oct 20 06:36:48 2010 UTC vs.
Revision 1.121 by root, Mon Oct 29 22:50:10 2012 UTC

1#!@PERL@ 1#!@PERL@
2 2
3# 3#
4# This file is part of Deliantra, the Roguelike Realtime MMORPG. 4# This file is part of Deliantra, the Roguelike Realtime MMORPG.
5# 5#
6# Copyright (©) 2007,2008,2009 Marc Alexander Lehmann / Robin Redeker / the Deliantra team 6# Copyright (©) 2007,2008,2009,2010,2011,2012 Marc Alexander Lehmann / Robin Redeker / the Deliantra team
7# 7#
8# Deliantra is free software: you can redistribute it and/or modify it under 8# Deliantra is free software: you can redistribute it and/or modify it under
9# the terms of the Affero GNU General Public License as published by the 9# the terms of the Affero GNU General Public License as published by the
10# Free Software Foundation, either version 3 of the License, or (at your 10# Free Software Foundation, either version 3 of the License, or (at your
11# option) any later version. 11# option) any later version.
12# 12#
13# This program is distributed in the hope that it will be useful, 13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of 14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details. 16# GNU General Public License for more details.
17# 17#
18# You should have received a copy of the Affero GNU General Public License 18# You should have received a copy of the Affero GNU General Public License
19# and the GNU General Public License along with this program. If not, see 19# and the GNU General Public License along with this program. If not, see
20# <http://www.gnu.org/licenses/>. 20# <http://www.gnu.org/licenses/>.
21# 21#
22# The authors can be reached via e-mail to <support@deliantra.net> 22# The authors can be reached via e-mail to <support@deliantra.net>
23# 23#
24 24
25use common::sense; 25use common::sense;
26 26
28my $exec_prefix = "@exec_prefix@"; 28my $exec_prefix = "@exec_prefix@";
29my $datarootdir = "@datarootdir@"; 29my $datarootdir = "@datarootdir@";
30my $DATADIR = "@datadir@/@PACKAGE@"; 30my $DATADIR = "@datadir@/@PACKAGE@";
31 31
32my $CONVERT = "@CONVERT@"; 32my $CONVERT = "@CONVERT@";
33#my $IDENTIFY = "@IDENTIFY@"; 33my $IDENTIFY = "@IDENTIFY@";
34my $OPTIPNG = "@OPTIPNG@"; 34my $OPTIPNG = "@OPTIPNG@";
35my $RSYNC = "@RSYNC@"; 35my $RSYNC = "@RSYNC@";
36my $PNGNQ = "@PNGNQ@"; 36my $PNGNQ = "@PNGNQ@";
37 37
38use Getopt::Long; 38use Getopt::Long;
39use File::Temp; 39use File::Temp;
40use POSIX (); 40use POSIX ();
41use Carp; 41use Carp;
42 42
43use Coro::EV;
44use AnyEvent;
45use YAML::XS (); 43use YAML::XS ();
44use Digest::MD5 ();
45use Storable ();
46use JSON::XS (); 46use JSON::XS ();
47use IO::AIO (); 47use IO::AIO ();
48use Digest::MD5 (); 48use Compress::LZF ();
49 49
50use AnyEvent;
50use Coro 5.12; 51use Coro 5.12;
52use Coro::EV;
51use Coro::AIO; 53use Coro::AIO;
52use Coro::Util; 54use Coro::Util;
53use Coro::Channel; 55use Coro::Channel;
56use Coro::AnyEvent;
54use Coro::Storable; $Storable::canonical = 1; 57use Coro::Storable; $Storable::canonical = 1;
55 58
56use Deliantra; 59use Deliantra;
57 60
58$SIG{QUIT} = sub { Carp::cluck "QUIT" }; 61$SIG{QUIT} = sub { Carp::cluck "QUIT" };
104 khaki => 12, 107 khaki => 12,
105); 108);
106 109
107END { system "rm", "-rf", $TMPDIR } 110END { system "rm", "-rf", $TMPDIR }
108 111
109my $s_INT = EV::signal INT => sub { exit 1 }; 112my $s_INT = AE::signal INT => sub { exit 1 };
110my $s_TERM = EV::signal TERM => sub { exit 1 }; 113my $s_TERM = AE::signal TERM => sub { exit 1 };
111 114
112our %hash; 115our %hash;
113 116
114# here we could try to avoid collisions and reduce chksum size further 117# here we could try to avoid collisions and reduce chksum size further
115sub make_hash($\$\$;$) { 118sub make_hash($\$\$;$) {
116 my ($id, $dataref, $hashref, $clen) = @_; 119 my ($id, $dataref, $hashref, $clen) = @_;
117 120
118 my $hash = substr +(Digest::MD5::md5 $$dataref), 0, $clen || 4; 121 my $hash = substr +(Digest::MD5::md5 $$dataref), 0, $clen || 5;
119 122
120 if (exists $hash{$hash}) { 123 if (exists $hash{$hash}) {
121 # hash collision, but some files are simply identical 124 # hash collision, but some files are simply identical
122 if (${$hash{$hash}[1]} ne $$dataref) { 125 if (${$hash{$hash}[1]} ne $$dataref) {
123 warn "hash collision $hash{$hash}[0] vs. $id\n"; 126 warn "ERROR: hash collision $hash{$hash}[0] vs. $id (increase the default \$clen in make_hash?)\n";
124 exit 1; 127 exit 1;
125 } else { 128 } else {
126 print "$hash{$hash}[0] and $id are identical (which is fine).\n" if $VERBOSE >= 3; 129 print "$hash{$hash}[0] and $id are identical (which is fine).\n" if $VERBOSE >= 3;
127 } 130 }
128 } 131 }
129 $hash{$hash} = [$id, $dataref, $hashref]; 132 $hash{$hash} = [$id, $dataref, $hashref];
130 133
131 $$hashref = $hash; 134 $$hashref = $hash;
132} 135}
133 136
137sub optipng($) {
138 system $OPTIPNG, "-o5", "-i0", "-q", $_[0];
139 die "$_[0] has zero size, aborting." unless -s $_[0];
140}
141
134mkdir $TMPDIR, 0700 142mkdir $TMPDIR, 0700
135 or die "$TMPDIR: $!"; 143 or die "$TMPDIR: $!";
136 144
137sub fork_sub(&) { 145sub fork_exec(&) {
138 my ($cb) = @_; 146 my ($cb) = @_;
139 147
140 if (my $pid = fork) { 148 if (my $pid = fork) {
141 my $current = $Coro::current; 149 my $current = $Coro::current;
142 my $w = AnyEvent->child (pid => $pid, cb => sub { $current->ready }); 150 my $w = AnyEvent->child (pid => $pid, cb => sub { $current->ready });
147 warn $@; 155 warn $@;
148 POSIX::_exit 1; 156 POSIX::_exit 1;
149 } 157 }
150} 158}
151 159
160sub imgsize($) {
161 open my $fh, "-|", $IDENTIFY, qw(-ping -format %w,%h --), $_[0]
162 or die "$IDENTIFY: $!";
163
164 Coro::AnyEvent::readable $fh;
165
166 my ($w, $h) = split /,/, <$fh>;
167
168 ($w+0, $h+0)
169}
170
171# make $dst from $src, if not uptodate, by calling $how
172sub make_file($$$) {
173 my ($src, $dst, $how) = @_;
174
175 my $t = (aio_stat $dst) ? -1 : (stat _)[9];
176
177 for (ref $src ? @$src : $src) {
178 aio_stat $_
179 and die "$_: $!";
180
181 if ((stat _)[9] > $t) {
182 # outdated, redo
183 $how->();
184 last;
185 }
186 }
187}
188
152sub inst_maps($) { 189sub inst_maps($) {
153 my (undef, $path) = @_; 190 my (undef, $path) = @_;
154 191
155 print "\nInstalling '$path' to '$DATADIR/maps'\n\n"; 192 print "\nInstalling '$path' to '$DATADIR/maps'\n\n";
156 193
157 if (!-f "$path/regions") { 194 if (!-f "$path/regions") {
158 warn "'$path' does not look like a maps directory ('regions' file is missing).\n"; 195 warn "ERROR: '$path' does not look like a maps directory ('regions' file is missing).\n";
159 exit 1 unless $FORCE; 196 exit 1 unless $FORCE;
160 } 197 }
161 198
162 system $RSYNC, "-av", "--chmod=u=rwX,go=rX", 199 system $RSYNC, "-av", "--chmod=u=rwX,go=rX",
163 "$path/.", "$DATADIR/maps/.", 200 "$path/.", "$DATADIR/maps/.",
173 our %FACEINFO; 210 our %FACEINFO;
174 our %RESOURCE; 211 our %RESOURCE;
175 our @ARC; 212 our @ARC;
176 our %ARC; 213 our %ARC;
177 our $TRS; 214 our $TRS;
215 our %PNG;
178 our $NFILE; 216 our $NFILE;
179 our $PATH; 217 our $PATH;
180 218
181 our $QUANTIZE = "+dither -colorspace RGB -colors 256"; 219 our $QUANTIZE = "+dither -colorspace RGB -colors 256";
182 220
183 our $c_arc = new Coro::Channel; 221 our $c_arc = new Coro::Channel;
184 our $c_any = new Coro::Channel; 222 our $c_any = new Coro::Channel;
185 223
186 our @c_png; 224 our @c_png;
187 225
226 # texture catalog size, in 64x64 tiles
227 sub TC_W() { 1024 / 64 }
228 sub TC_H() { 1024 / 64 }
229
230 our @TC = ([]); # texture catalogs
231
188 sub commit_png($$$$) { 232 sub commit_png($$$$$$) {
189 my ($stem, $name, $data, $T) = @_; 233 my ($stem, $name, $data, $T, $w, $h) = @_;
190 234
191 $FACEINFO{$name}{"stem"} = substr $stem, 1 + length $PATH; 235 $FACEINFO{$name}{stem} = substr $stem, 1 + length $PATH;
192 $FACEINFO{$name}{"data$T"} = $data; 236 $FACEINFO{$name}{"data$T"} = $data;
237 $FACEINFO{$name}{w} = $w;
238 $FACEINFO{$name}{h} = $h;
193 } 239 }
194 240
195 sub process_png { 241 sub process_png {
196 while (@c_png) { 242 while (@c_png) {
197 my ($path, $delete) = @{pop @c_png}; 243 my ($path, $delete) = @{pop @c_png};
199 my $png; 245 my $png;
200 aio_lstat $path; 246 aio_lstat $path;
201 my ($size, $mtime) = (stat _)[7,9]; 247 my ($size, $mtime) = (stat _)[7,9];
202 248
203 if (0 > aio_load $path, $png) { 249 if (0 > aio_load $path, $png) {
204 warn "$path: $!, skipping.\n"; 250 warn "ERROR: $path: $!, skipping.\n";
205 next; 251 next;
206 } 252 }
207 253
208 my $stem = $path; 254 my $stem = $path;
209 my $T; 255 my $T;
211 if ($stem =~ s/\.32x32\.png~?$//) { 257 if ($stem =~ s/\.32x32\.png~?$//) {
212 $T = 32; 258 $T = 32;
213 } elsif ($stem =~ s/\.64x64\.png~?$//) { 259 } elsif ($stem =~ s/\.64x64\.png~?$//) {
214 $T = 64; 260 $T = 64;
215 } else { 261 } else {
216 warn "$path: weird filename, skipping.\n"; 262 warn "ERROR: $path: weird filename, skipping.\n";
217 next; 263 next;
218 } 264 }
219 265
220 # quickly extract width and height of the (necessarily PNG) image 266 # quickly extract width and height of the (necessarily PNG) image
221 unless ($png =~ /^\x89PNG\x0d\x0a\x1a\x0a....IHDR(........)/s) { 267 unless ($png =~ /^\x89PNG\x0d\x0a\x1a\x0a....IHDR(........)/s) {
222 warn "$path: not a recognized png file, skipping.\n"; 268 warn "ERROR: $path: not a recognized png file, skipping.\n";
223 next; 269 next;
224 } 270 }
225 271
226 my ($w, $h) = unpack "NN", $1; 272 my ($w, $h) = unpack "NN", $1;
227 273
228 if ($w < $T || $h < $T) { 274 if ($w < $T || $h < $T) {
229 warn "$path: too small ($w $h), skipping.\n"; 275 warn "ERROR: $path: too small ($w $h), skipping.\n";
230 next; 276 next;
231 } 277 }
232 278
233 if ($w % $T || $h % $T) { 279 if ($w % $T || $h % $T) {
234 warn "$path: weird png size ($w $h), skipping.\n"; 280 warn "ERROR: $path: weird png size ($w $h), skipping.\n";
235 next; 281 next;
236 } 282 }
237 283
238 (my $base = $stem) =~ s/^.*\///; 284 (my $base = $stem) =~ s/^.*\///;
239 285
248 unless ($path =~ /~$/) { 294 unless ($path =~ /~$/) {
249 # possibly enlarge 295 # possibly enlarge
250 if (0 > aio_stat "$stem.64x64.png") { 296 if (0 > aio_stat "$stem.64x64.png") {
251 my $other = "$stem.64x64.png~"; 297 my $other = "$stem.64x64.png~";
252 298
253 if (0 > aio_lstat $other or (-M _) > (-M $path)) { 299 make_file $path, $other, sub {
254 fork_sub { 300 fork_exec {
255 my $CROP; 301 my $CROP;
256 my $SRC = "png:\Q$path\E"; 302 my $SRC = "png:\Q$path\E";
257 303
258 my $is_floor = $arc->{is_floor}; 304 my $is_floor = $arc->{is_floor};
259 my $is_wall = 0; 305 my $is_wall = 0;
309 $x * ($w - 8) + 4, 355 $x * ($w - 8) + 4,
310 $y * ($h - 8) + 4; 356 $y * ($h - 8) + 4;
311 } 357 }
312 } 358 }
313 359
314 system "convert -depth 8 $SRC rgba:-" 360 system "$CONVERT -depth 8 $SRC rgba:-"
315 . "| $exec_prefix/bin/cfhq2xa $w $h 0" 361 . "| $exec_prefix/bin/cfhq2xa $w $h 0"
316 . "| convert -depth 8 -size ".($w * 2)."x".($h * 2)." rgba:- $CROP $QUANTIZE -quality 00 png32:\Q$other\E~" 362 . "| $CONVERT -depth 8 -size ".($w * 2)."x".($h * 2)." rgba:- $CROP $QUANTIZE -quality 00 png32:\Q$other\E~"
317 and die "convert/cfhq2xa pipeline error: status $? ($!)"; 363 and die "convert/cfhq2xa pipeline error: status $? ($!)";
318 system $OPTIPNG, "-i0", "-q", "$other~"; 364 optipng "$other~";
319 die "$other~ has zero size, aborting." unless -s "$other~";
320 rename "$other~", $other; 365 rename "$other~", $other;
321 }; 366 };
322 } 367 };
323 368
324 push @c_png, [$other, !$CACHE]; 369 push @c_png, [$other, !$CACHE];
325 } 370 }
326 371
327 # possibly scale down 372 # possibly scale down
328 if (0 > aio_stat "$stem.32x32.png") { 373 if (0 > aio_stat "$stem.32x32.png") {
329 my $other = "$stem.32x32.png~"; 374 my $other = "$stem.32x32.png~";
330 375
331 if (0 > aio_lstat $other or (-M _) > (-M $path)) { 376 make_file $path, $other, sub {
332 fork_sub { 377 fork_exec {
333 system "convert png:\Q$path\E -geometry 50% -filter lanczos $QUANTIZE -quality 00 png32:\Q$other\E~"; 378 system "$CONVERT png:\Q$path\E -geometry 50% -filter lanczos $QUANTIZE -quality 00 png32:\Q$other\E~";
334 system $OPTIPNG, "-i0", "-q", "$other~"; 379 optipng "$other~";
335 380
336 # reduce smoothfaces >10000 bytes 381 # reduce smoothfaces >10000 bytes
337 # obsolete, no longer required 382 # obsolete, no longer required
338 if (0 && $stem =~ /_S\./ && (-s "$other~") > 10000) { 383 if (0 && $stem =~ /_S\./ && (-s "$other~") > 10000) {
339 my $ncolor = 256; 384 my $ncolor = 256;
340 while () { 385 while () {
341 system "<\Q$other~\E $PNGNQ -s1 -n$ncolor >\Q$other~~\E"; 386 system "<\Q$other~\E $PNGNQ -s1 -n$ncolor >\Q$other~~\E";
342 system $OPTIPNG, "-i0", "-q", "$other~~"; 387 optipng "$other~~";
343 last if 10000 > -s "$other~~"; 388 last if 10000 > -s "$other~~";
344 $ncolor = int $ncolor * 0.9; 389 $ncolor = int $ncolor * 0.9;
345 $ncolor > 8 or die "cannot reduce filesize to < 10000 bytes"; 390 $ncolor > 8 or die "cannot reduce filesize to < 10000 bytes";
346 } 391 }
347 392
352 } 397 }
353 398
354 die "$other~ has zero size, aborting." unless -s "$other~"; 399 die "$other~ has zero size, aborting." unless -s "$other~";
355 rename "$other~", $other; 400 rename "$other~", $other;
356 }; 401 };
357 } 402 };
358 403
359 #warn "scaled down $path to $other\n";#d# 404 #warn "scaled down $path to $other\n";#d#
360 push @c_png, [$other, !$CACHE]; 405 push @c_png, [$other, !$CACHE];
361 } 406 }
362 } 407 }
363 408
364 (my $face = $stem) =~ s/^.*\///; 409 (my $face = $stem) =~ s/^.*\///;
410
411 for (@{ $fi->{derive} }) {
412 my ($dir, $type) = @$_;
413
414 my $geom = "${T}x${T}";
415 my $dpath = "$dir/$type:$face.$geom.png~";
416
417 if ($type eq "buildmaterial") {
418 my $ov = "$PATH/mapbuilding/buildmaterial.png";
419 make_file [$path, $ov], $dpath, sub {
420 fork_exec {
421 system "$CONVERT -size 64x64 xc:transparent \\( \Q$path\E -geometry 52x52+6+6 \\) -composite \Q$ov\E -composite -geometry $geom png:\Q$dpath\E~";
422 optipng "$dpath~";
423 rename "$dpath~", $dpath;
424 };
425 };
426 } elsif ($type eq "buildmaterial-box") {
427 my $ov = "$PATH/mapbuilding/buildmaterial-box.png";
428 make_file [$path, $ov], $dpath, sub {
429 fork_exec {
430 system "$CONVERT \Q$ov\E "
431 . "\\( \Q$path\E -trim -geometry 34x31\\> "
432 . "-gravity center -background transparent -extent 37x34 +gravity "
433 . "-geometry +22+6 \\) -composite "
434 . "-geometry $geom png:\Q$dpath\E~";
435 optipng "$dpath~";
436 rename "$dpath~", $dpath;
437 };
438 };
439 } elsif ($type eq "buildmaterial-sbox") {
440 my $ov = "$PATH/mapbuilding/buildmaterial-sbox.png";
441 make_file [$path, $ov], $dpath, sub {
442 fork_exec {
443 system "$CONVERT \Q$ov\E "
444 . "\\( \Q$path\E -trim -geometry 16x24\\> "
445 . "-gravity center -background transparent -extent 18x26 +gravity "
446 . "-geometry +26+16 \\) -composite "
447 . "-geometry $geom png:\Q$dpath\E~";
448 optipng "$dpath~";
449 rename "$dpath~", $dpath;
450 };
451 };
452 } else {
453 warn "ERROR: facename $type:$face contains unsupported derivation type.\n";
454 }
455
456 aio_load $dpath, my $png;
457 IO::AIO::aio_unlink $dpath unless $CACHE;
458 commit_png "$dir/$type:$face", "$type:$face", $png, $T, 1, 1;
459 }
365 460
366 # split all bigfaces, but avoid smoothfaces (*_S) 461 # split all bigfaces, but avoid smoothfaces (*_S)
367 if (($w > $T || $h > $T) && $face !~ /_S\./) { 462 if (($w > $T || $h > $T) && $face !~ /_S\./) {
368 # split 463 # split
369 my @tile; 464 my @tile;
374 push @tile, [$x, $y, $file, (stat _)[9]]; 469 push @tile, [$x, $y, $file, (stat _)[9]];
375 } 470 }
376 } 471 }
377 472
378 my $mtime = (lstat $path)[9]; 473 my $mtime = (lstat $path)[9];
379 my @todo = grep { $_->[3] <= $mtime } @tile; 474 my @todo = grep { $_->[3] < $mtime } @tile;
380 if (@todo) { 475 if (@todo) {
381 fork_sub { 476 fork_exec {
382 open my $convert, "|-", $CONVERT, 477 open my $convert, "|-", $CONVERT,
383 "png:-", 478 "png:-",
384 (map { 479 (map {
385 ( 480 (
386 "(", 481 "(",
399 print $convert $png; 494 print $convert $png;
400 close $convert; 495 close $convert;
401 496
402 # pass 2, optimise, and rename 497 # pass 2, optimise, and rename
403 for (@todo) { 498 for (@todo) {
404 system $OPTIPNG, "-o5", "-i0", "-q", "$_->[2]~"; 499 optipng "$_->[2]~";
405 die "$_->[2]~ has zero size, aborting." unless -s "$_->[2]~";
406 rename "$_->[2]~", $_->[2]; 500 rename "$_->[2]~", $_->[2];
407 } 501 }
408 }; 502 };
409 } 503 }
410 504
414 508
415 if (0 > aio_load $file, $tile) { 509 if (0 > aio_load $file, $tile) {
416 die "$path: unable to read tile +$x+$y, aborting.\n"; 510 die "$path: unable to read tile +$x+$y, aborting.\n";
417 } 511 }
418 IO::AIO::aio_unlink $file unless $CACHE; 512 IO::AIO::aio_unlink $file unless $CACHE;
419 commit_png $stem, $x|$y ? "$face+$x+$y" : $face, $tile, $T; 513 commit_png $stem, $x|$y ? "$face+$x+$y" : $face, $tile, $T, 1, 1;
420 } 514 }
421 } else { 515 } else {
422 # use as-is (either small, use smooth) 516 # use as-is (either small, use smooth)
423 commit_png $stem, $face, $png, $T; 517 commit_png $stem, $face, $png, $T, $w / $T, $h / $T;
424 } 518 }
425 519
426 aio_unlink $path if $delete; 520 IO::AIO::aio_unlink $path if $delete;
427 } 521 }
428 } 522 }
429 523
430 sub process_faceinfo { 524 sub process_faceinfo {
431 my ($dir, $file) = @_; 525 my ($dir, $file) = @_;
432 my $path = "$dir/$file"; 526 my $path = "$dir/$file";
433 527
434 my $data; 528 my $data;
435 if (0 > aio_load $path, $data) { 529 if (0 > aio_load $path, $data) {
436 warn "$path: $!, skipping.\n"; 530 warn "ERROR: $path: $!, skipping.\n";
437 return; 531 return;
438 } 532 }
439 533
440 for (split /\n/, $data) { 534 for (split /\n/, $data) {
441 chomp; 535 chomp;
442 my ($face, $visibility, $fg, $bg, $glyph) = split /\s+/; 536 my ($face, $visibility, $fg, $bg, $glyph) = split /\s+/, $_, 5;
443 # bg not used except for text clients 537 # bg not used except for text clients
444 538
445 utf8::decode $glyph; 539 utf8::decode $glyph;
446 $glyph =~ s/^\?(?=.)//; # remove "autoglyph" flag 540 $glyph =~ s/^\?(?=.)//; # remove "autoglyph" flag
541 $glyph =~ s/^"(.+)"$/$1/; # allow for ""-style quoting
447 542
448 $fg = "white" if $fg eq "none"; # lots of faces have no fg colour yet 543 $fg = "white" if $fg eq "none"; # lots of faces have no fg colour yet
449 544
450 (my $fgi = $COLOR{$fg}) 545 (my $fgi = $COLOR{$fg})
451 // warn "WARNING: $path: $face specifies unknown foreground colour '$fg'.\n"; 546 // warn "ERROR: $path: $face specifies unknown foreground colour '$fg'.\n";
452 (my $bgi = $COLOR{$bg}) 547 (my $bgi = $COLOR{$bg})
453 // warn "WARNING: $path: $face specifies unknown background colour '$bg'.\n"; 548 // warn "ERROR: $path: $face specifies unknown background colour '$bg'.\n";
454 549
455 my $fi = $FACEINFO{$face} ||= { }; 550 my $fi = $FACEINFO{$face} ||= { };
456 $fi->{visibility} = $visibility * 1; 551 $fi->{visibility} = $visibility * 1;
457 $fi->{magicmap} = $fgi; # foreground colour becomes magicmap 552 $fi->{magicmap} = $fgi; # foreground colour becomes magicmap
458 553
527 next if /^facings\s/; 622 next if /^facings\s/;
528 623
529 my $face = $_; 624 my $face = $_;
530 $face =~ s/\+\d+\+\d+$//; # remove tile offset coordinates 625 $face =~ s/\+\d+\+\d+$//; # remove tile offset coordinates
531 626
532 my $info = $FACEINFO{$face} ||= { }; 627 # handle derived faces
628 if ($face =~ /^([^:]+):(.*)/) {
629 my ($type, $base) = ($1, $2);
630 push @{ $FACEINFO{$base}{derive} }, [$dir, $type];
631 }
632
533 $info->{arc} = $o; 633 $FACEINFO{$face}{arc} = $o;
534 634
535 next if $face =~ /^blank.x11$|^empty.x11$/; 635 next if $face =~ /^blank.x11$|^empty.x11$/;
536 } 636 }
537 637
538 if (my $smooth = delete $o->{smoothface}) { 638 if (my $smooth = delete $o->{smoothface}) {
554 my ($dir, $file) = @_; 654 my ($dir, $file) = @_;
555 my $path = "$dir/$file"; 655 my $path = "$dir/$file";
556 656
557 my $trs; 657 my $trs;
558 if (0 > aio_load $path, $trs) { 658 if (0 > aio_load $path, $trs) {
559 warn "$path: $!, skipping.\n"; 659 warn "ERROR: $path: $!, skipping.\n";
560 return; 660 return;
561 } 661 }
562 662
563 $TRS .= $trs; 663 $TRS .= $trs;
564 } 664 }
569 unless (exists $FILECACHE{$_[0]}) { 669 unless (exists $FILECACHE{$_[0]}) {
570 my $data; 670 my $data;
571 if (0 < aio_load $_[0], $data) { 671 if (0 < aio_load $_[0], $data) {
572 if ($_[1]) { 672 if ($_[1]) {
573 $data = eval { $_[1]->($data) }; 673 $data = eval { $_[1]->($data) };
574 warn "$_[0]: $@" if $@; 674 warn "ERROR: $_[0]: $@" if $@;
575 } 675 }
576 } 676 }
577 677
578 $FILECACHE{$_[0]} = $data; 678 $FILECACHE{$_[0]} = $data;
579 } 679 }
580 680
581 $FILECACHE{$_[0]} 681 $FILECACHE{$_[0]}
682 }
683
684 # convert an image and a palette to some indexed 2d-matrix structure
685 sub process_plt {
686 my ($base, $plt) = @_;
687
688 my ($w, $h) = imgsize "$base.png";
689
690 $w * $h
691 or die "$base.png: unable to identify correct size\n";
692
693 my @plt;
694 my %map;
695
696 for (split /\n/, $plt) {
697 next unless /\S/;
698 next if /^\s*#/;
699
700 /^([0-9a-fA-F]{3,6})\s*(.*?)\s*$/
701 or die "unparseable palette entry for $base.plt: $_";
702
703 my ($rgb, $name) = ($1, $2);
704
705 $rgb =~ s/^(.)(.)(.)$/$1$1$2$2$3$3/;
706
707 $map{pack "H*", $rgb} = chr @plt;
708 push @plt, $name;
709 }
710
711 make_file ["$base.plt", "$base.png"], "$base.tbl~", sub {
712 warn "building $base\n" if $VERBOSE >= 3;
713
714 fork_exec {
715 open my $png, "-|", $CONVERT, qw(-depth 8 --), "$base.png", "rgb:"
716 or die "$base.png: $!";
717
718 local $/;
719 $png = <$png>;
720
721 $w * $h * 3 == length $png
722 or die "$base.png: failed to read enough data from file\n";
723
724 $png =~ s/(...)/$map{$1}/ge;
725
726 $w * $h == length $png
727 or die "$base.png: failed to map all data - wrong palette?\n";
728
729 {
730 open my $fh, ">:raw", "$base.tbl~~"
731 or die "$base.tbl~~: $!";
732 syswrite $fh, $png;
733 }
734
735 rename "$base.tbl~~", "$base.tbl~";
736 };
737 };
738
739 0 <= aio_load "$base.tbl~", my $tbl
740 or die "$base.tbl~: $!";
741
742 IO::AIO::aio_unlink "$base.tbl~" unless $CACHE;
743
744 Compress::LZF::compress nfreeze {
745 w => $w,
746 h => $h,
747 plt => \@plt,
748 tbl => $tbl,
749 }
582 } 750 }
583 751
584 sub process_res { 752 sub process_res {
585 my ($dir, $file, $type) = @_; 753 my ($dir, $file, $type) = @_;
586 754
587 my $data;
588 aio_load "$dir/$file", $data; 755 0 <= aio_load "$dir/$file", my $data
756 or die "$dir/$file: $!";
589 757
590 my $meta = load_cached "$dir/meta", sub { JSON::XS->new->utf8->relaxed->decode (shift) }; 758 my $meta = load_cached "$dir/meta", sub { JSON::XS->new->utf8->relaxed->decode (shift) };
591 759
592 utf8::decode $dir; 760 utf8::decode $dir;
593 utf8::decode $file; 761 utf8::decode $file;
601 $meta = { 769 $meta = {
602 %{ $meta->{"" } || {} }, 770 %{ $meta->{"" } || {} },
603 %{ $meta->{$file} || {} }, 771 %{ $meta->{$file} || {} },
604 }; 772 };
605 773
606 if ($meta->{license} =~ s/^#//) { 774 if (exists $meta->{license} && $meta->{license} =~ s/^#//) { # exists == avoid autovivification
607 $meta->{license} = ({ 775 $meta->{license} = ({
608 "pd" => "Public Domain", 776 "pd" => "Public Domain",
609 "gpl" => "GNU General Public License, version 3.0 or any later", 777 "gpl" => "GNU General Public License, version 3.0 or any later",
610 "cc/by/2.0" => "Licensed under Creative Commons Attribution 2.0 http://creativecommons.org/licenses/by/2.0/", 778 "cc/by/2.0" => "Licensed under Creative Commons Attribution 2.0 http://creativecommons.org/licenses/by/2.0/",
611 "cc/by/2.1" => "Licensed under Creative Commons Attribution 2.1 http://creativecommons.org/licenses/by/2.1/", 779 "cc/by/2.1" => "Licensed under Creative Commons Attribution 2.1 http://creativecommons.org/licenses/by/2.1/",
612 "cc/by/2.5" => "Licensed under Creative Commons Attribution 2.5 http://creativecommons.org/licenses/by/2.5/", 780 "cc/by/2.5" => "Licensed under Creative Commons Attribution 2.5 http://creativecommons.org/licenses/by/2.5/",
613 "cc/by/3.0" => "Licensed under Creative Commons Attribution 3.0 http://creativecommons.org/licenses/by/3.0/", 781 "cc/by/3.0" => "Licensed under Creative Commons Attribution 3.0 http://creativecommons.org/licenses/by/3.0/",
614 })->{$meta->{license}} 782 })->{$meta->{license}}
615 || warn "$dir/$file: license tag '$meta->{license}' not found."; 783 || warn "ERROR: $dir/$file: license tag '$meta->{license}' not found.";
616 } 784 }
617 785
618 if (!exists $meta->{author} && $meta->{source} =~ m%^http://www.jamendo.com/en/artist/(.*)$%) { 786 if (!exists $meta->{author} && $meta->{source} =~ m%^http://www.jamendo.com/en/artist/(.*)$%) {
619 ($meta->{author} = $1) =~ s/_/ /g; 787 ($meta->{author} = $1) =~ s/_/ /g;
620 } 788 }
621 789
622 $file =~ s/\.res$//; 790 $file =~ s/\.res$//;
623 $file =~ s/\.(ogg|wav|jpg|png)$//; 791 $file =~ s/\.(ogg|wav|jpg|png)$//;
624 792
625 substr $dir, 0, 1 + length $PATH, ""; 793 if ($file =~ s/\.plt$//) {
626 794 $data = process_plt "$dir/$file", $data;
627 if (my $filter = $meta->{cfutil_filter}) { 795 } elsif (my $filter = delete $meta->{cfutil_filter}) {
628 if ($filter eq "yaml2json") { 796 if ($filter eq "yaml2json") {
629 $data = JSON::XS::encode_json YAML::XS::Load $data; 797 $data = JSON::XS::encode_json YAML::XS::Load $data;
630 } elsif ($filter eq "json2json") { 798 } elsif ($filter eq "json2json") {
631 $data = JSON::XS::encode_json JSON::XS->relaxed->utf8->decode ($data); 799 $data = JSON::XS::encode_json JSON::XS->relaxed->utf8->decode ($data);
632 } elsif ($filter eq "perl2json") { 800 } elsif ($filter eq "perl2json") {
633 $data = eval $data; die if $@; 801 $data = eval $data; die if $@;
634 $data = JSON::XS::encode_json $data; 802 $data = JSON::XS::encode_json $data;
635 } else { 803 } else {
636 warn "$dir/$file: unknown filter $filter, skipping\n"; 804 warn "ERROR: $dir/$file: unknown filter $filter, skipping\n";
637 }
638 } 805 }
806 }
807
808 substr $dir, 0, 1 + length $PATH, "";
639 809
640 $RESOURCE{"$dir/$file"} = { 810 $RESOURCE{"$dir/$file"} = {
641 type => (exists $meta->{type} ? delete $meta->{type} : $type), 811 type => (exists $meta->{type} ? delete $meta->{type} : $type),
642 data => $data, 812 data => $data,
643 %$meta ? (meta => $meta) : (), 813 %$meta ? (meta => $meta) : (),
682 if ($file =~ /\.(jpg|png)$/) { 852 if ($file =~ /\.(jpg|png)$/) {
683 $c_any->put ([\&process_res, $path, $file, 0]) # FT_FACE 853 $c_any->put ([\&process_res, $path, $file, 0]) # FT_FACE
684 } elsif ($file =~ /\.(res)$/) { 854 } elsif ($file =~ /\.(res)$/) {
685 $c_any->put ([\&process_res, $path, $file, 6]) # FT_RSRC 855 $c_any->put ([\&process_res, $path, $file, 6]) # FT_RSRC
686 } else { 856 } else {
687 $c_any->put ([\&process_res, $path, $file, undef]); 857 $c_any->put ([\&process_res, $path, $file, 6]); # was type undef, but now meta files are mandatory, so any mentioned file surely has a purpose
688 } 858 }
689 859
690 } elsif ($file =~ /\.png$/) { 860 } elsif ($file =~ /\.png$/) {
691 push @c_png, ["$path/$file", 0]; 861 $PNG{$file} = $path;
692 862
693 } elsif ($file =~ /\.faceinfo$/) { 863 } elsif ($file =~ /\.faceinfo$/) {
694 $c_any->put ([\&process_faceinfo, $path, $file]); 864 $c_any->put ([\&process_faceinfo, $path, $file]);
695 865
696 } elsif ($file =~ /\.trs$/) { 866 } elsif ($file =~ /\.trs$/) {
749 "Unless you run verbosely, all following warning\n", 919 "Unless you run verbosely, all following warning\n",
750 "or error messages indicate serious problems.\n", 920 "or error messages indicate serious problems.\n",
751 "\n"; 921 "\n";
752 922
753 if (!-d "$path/treasures") { 923 if (!-d "$path/treasures") {
754 warn "'$path' does not look like an arch directory ('treasures' directory is missing).\n"; 924 warn "FATAL: '$path' does not look like an arch directory ('treasures' directory is missing).\n";
755 exit 1 unless $FORCE; 925 exit 1 unless $FORCE;
756 } 926 }
757 927
758 print "start file scan, arc, any processing...\n" if $VERBOSE; 928 print "starting file scan, arc, any processing...\n" if $VERBOSE;
759 929
760 my @a_arc = map +(async \&process_arc), 1..2; 930 my @a_arc = map +(async \&process_arc), 1..2;
761 my @a_any = map +(async \&process_any), 1..4; 931 my @a_any = map +(async \&process_any), 1..4;
762 932
763 find_files $path; 933 find_files $path;
764 934
765 $c_arc->shutdown; 935 $c_arc->shutdown;
766 $c_any->shutdown; 936 $c_any->shutdown;
767 937
768 $_->join for @a_arc; # need to parse all archetypes before png processing 938 $_->join for @a_arc; # need to parse all archetypes before png processing
939 print "finished arc processing...\n" if $VERBOSE;
769 940
941 # fill png queue
942 for (keys %FACEINFO) {
943 # we only expect source pngs to exist for non-derived
944 # faces, i.e. no split tiles (+x+x) and no face derivates
945 # (xxx:)
946 /^[^:]+\....$/
947 or next;
948
949 my $seen;
950
951 if (exists $PNG{"$_.32x32.png"}) {
952 push @c_png, ["$PNG{\"$_.32x32.png\"}/$_.32x32.png"];
953 $seen = 1;
954 }
955 if (exists $PNG{"$_.64x64.png"}) {
956 push @c_png, ["$PNG{\"$_.64x64.png\"}/$_.64x64.png"];
957 $seen = 1;
958 }
959
960 warn "ERROR: $_: face not found, expect problems.\n"
961 unless $seen;
962 }
963
770 print "end arc, start png processing...\n" if $VERBOSE; 964 print "starting png processing...\n" if $VERBOSE;
771 965
772 # eight png crunchers work fine for my 4x smp machine 966 # eight png crunchers work fine for my 4x smp machine
773 my @a_png = map +(async \&process_png), 1..8; 967 my @a_png = map +(async \&process_png), 1..8;
774 968
775 print "end any processing...\n" if $VERBOSE;
776 $_->join for @a_any; 969 $_->join for @a_any;
777
778 print "end png processing...\n" if $VERBOSE; 970 print "finished any processing...\n" if $VERBOSE;
971
779 $_->join for @a_png; 972 $_->join for @a_png;
973 print "finished png processing...\n" if $VERBOSE;
780 974
781 print "scanning done, processing results...\n" if $VERBOSE; 975 print "scanning done, processing results...\n" if $VERBOSE;
782 { 976 {
783 # remove path prefix from editor_folder 977 # remove path prefix from editor_folder
784 $_->{editor_folder} =~ /^\x00/ 978 $_->{editor_folder} =~ /^\x00/
802 delete @s{qw(_name more name name_pl)}; 996 delete @s{qw(_name more name name_pl)};
803 %$o = ( %s, %$o ); 997 %$o = ( %s, %$o );
804 ++$progress; 998 ++$progress;
805 } 999 }
806 } else { 1000 } else {
807 warn "WARNING: archetype '$o->{_name}' tries to inherit from undefined archetype '$other', skipping.\n"; 1001 warn "ERROR: archetype '$o->{_name}' tries to inherit from undefined archetype '$other', skipping.\n";
808 delete $ARC{$o->{_name}}; 1002 delete $ARC{$o->{_name}};
809 } 1003 }
810 } 1004 }
811 } 1005 }
812 1006
848 print $fh $TRS; 1042 print $fh $TRS;
849 } 1043 }
850 1044
851 { 1045 {
852 print "processing facedata...\n" if $VERBOSE; 1046 print "processing facedata...\n" if $VERBOSE;
853 while (my ($k, $v) = each %FACEINFO) { 1047
1048 for my $k (sort keys %FACEINFO) {
1049 my $v = $FACEINFO{$k};
1050
854 length $v->{data32} or warn "WARNING: face '$k' has no png32. this will not work (shoddy gcfclient will crash of course).\n"; 1051 length $v->{data32} or warn "ERROR: face '$k' has no png32 - this will not work.\n";
855 length $v->{data64} or warn "WARNING: face '$k' has no png64. this will not work very well.\n"; 1052 length $v->{data64} or warn "ERROR: face '$k' has no png64 - this will not work.\n";
856 1053
857 make_hash $k, $v->{data32}, $v->{hash32}; 1054 make_hash $k, $v->{data32}, $v->{hash32};
858 make_hash $k, $v->{data64}, $v->{hash64}; 1055 make_hash $k, $v->{data64}, $v->{hash64};
859 1056
860 #length $v->{data32} <= 10000 or warn "WARNING: face '$k' has face32 larger than 10000 bytes, will not work with crossfire client.\n"; 1057 #length $v->{data32} <= 10000 or warn "WARNING: face '$k' has face32 larger than 10000 bytes, will not work with crossfire client.\n";
861 #length $v->{data64} <= 10000 or warn "WARNING: face '$k' has face64 larger than 10000 bytes.\n"; 1058 #length $v->{data64} <= 10000 or warn "WARNING: face '$k' has face64 larger than 10000 bytes.\n";
862 1059
863 $v->{glyph} // warn "WARNING: face '$k' has glyph."; 1060 $v->{glyph} // warn "ERROR: face '$k' has no glyph - missing faceinfo entry?\n";
864 $v->{visibility} // warn "WARNING: face '$k' has no visibility info, missing faceinfo entry?\n"; 1061 $v->{visibility} // warn "ERROR: face '$k' has no visibility info - missing faceinfo entry?\n";
865 $v->{magicmap} // warn "WARNING: face '$k' has foreground colour."; 1062 $v->{magicmap} // warn "ERROR: face '$k' has no foreground colour - missing faceinfo entry?\n";
866 1063
1064 if (0 && $v->{w} == 1 && $v->{h} == 1) { # texture catalogs
1065 my $id = @TC;
1066 my $n = @{ $TC[-1] };
1067 my $x = $n % TC_W;
1068 my $y = int $n / TC_W;
1069
1070 push @{ $TC[-1] }, [$v, $x, $y];
1071
1072 $v->{tc} = [$id, $x, $y];
1073
1074 push @TC, [] if $n == TC_W * TC_H - 1; # start new texture if full
1075 }
1076
867 delete @$v{qw(arc stem)}; # not used by the server 1077 delete @$v{qw(w h arc stem derive)}; # not used by the server
1078 }
1079
1080if (0) { #d# texture catalogs
1081 print "creating texture catalogs...\n" if $VERBOSE;
1082
1083 for my $id (0 .. $#TC) {
1084 my $tc = $TC[$id];
1085
1086 my $cmd = "$CONVERT -depth 8 -size " . (TC_W * 64) . "x" . (TC_H * 64) . " xc:transparent";
1087 my $idx = "a";
1088
1089 for (@$tc) {
1090 my $path = "$TMPDIR/tc" . $idx++;
1091
1092 open my $fh, ">:perlio", $path
1093 or die "$path: $!";
1094 syswrite $fh, $_->[0]{data64};
1095
1096 my $x = $_->[1] * 64;
1097 my $y = $_->[2] * 64;
1098
1099 $cmd .= " png:\Q$path\E -geometry +$x+$y -composite";
868 } 1100 }
1101
1102 system "$cmd png:\Q$TMPDIR/tc$id\E";
1103 optipng "$TMPDIR/tc$id";
1104 }
1105}
869 1106
870 print "processing resources...\n" if $VERBOSE; 1107 print "processing resources...\n" if $VERBOSE;
871 my $enc = JSON::XS->new->utf8->canonical->relaxed; 1108 my $enc = JSON::XS->new->utf8->canonical->relaxed;
872 while (my ($k, $v) = each %RESOURCE) { 1109 while (my ($k, $v) = each %RESOURCE) {
873 1110
911 version => 2, 1148 version => 2,
912 faceinfo => \%FACEINFO, 1149 faceinfo => \%FACEINFO,
913 animinfo => \%ANIMINFO, 1150 animinfo => \%ANIMINFO,
914 resource => \%RESOURCE, 1151 resource => \%RESOURCE,
915 }; 1152 };
916
917 } 1153 }
918 1154
919 print "committing files...\n" if $VERBOSE; 1155 print "committing files...\n" if $VERBOSE;
920 1156
921 for (qw(archetypes facedata treasures), @COMMIT) { 1157 for (qw(archetypes facedata treasures), @COMMIT) {

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines