--- deliantra/server/lib/cf.pm	2007/01/02 22:55:05	1.124
+++ deliantra/server/lib/cf.pm	2007/08/28 19:38:40	1.346
@@ -5,76 +5,117 @@
 
 use Symbol;
 use List::Util;
+use Socket;
 use Storable;
+use Event;
 use Opcode;
 use Safe;
 use Safe::Hole;
 
-use Coro 3.3;
+use Coro 3.64 ();
+use Coro::State;
+use Coro::Handle;
 use Coro::Event;
 use Coro::Timer;
 use Coro::Signal;
 use Coro::Semaphore;
 use Coro::AIO;
+use Coro::Storable;
+use Coro::Util ();
 
+use JSON::XS ();
+use BDB ();
+use Data::Dumper;
 use Digest::MD5;
 use Fcntl;
-use IO::AIO 2.31 ();
 use YAML::Syck ();
+use IO::AIO 2.32 ();
 use Time::HiRes;
+use Compress::LZF;
+use Digest::MD5 ();
 
-use Event; $Event::Eval = 1; # no idea why this is required, but it is
+# configure various modules to our taste
+#
+$Storable::canonical = 1; # reduce rsync transfers
+Coro::State::cctx_stacksize 256000; # 1-2MB stack, for deep recursions in maze generator
+Compress::LZF::sfreeze_cr { }; # prime Compress::LZF so it does not use require later
+
+$Event::Eval = 1; # no idea why this is required, but it is
 
 # work around bug in YAML::Syck - bad news for perl6, will it be as broken wrt. unicode?
 $YAML::Syck::ImplicitUnicode = 1;
 
-$Coro::main->prio (2); # run main coroutine ("the server") with very high priority
+$Coro::main->prio (Coro::PRIO_MAX); # run main coroutine ("the server") with very high priority
 
 sub WF_AUTOCANCEL () { 1 } # automatically cancel this watcher on reload
 
 our %COMMAND      = ();
 our %COMMAND_TIME = ();
+
+our @EXTS         = (); # list of extension package names
 our %EXTCMD       = ();
+our %EXTICMD      = ();
+our %EXT_CORO     = (); # coroutines bound to extensions
+our %EXT_MAP      = (); # pluggable maps
 
+our $RELOAD; # number of reloads so far
 our @EVENT;
-our $LIBDIR = datadir . "/ext";
 
-our $TICK = MAX_TIME * 1e-6;
+our $CONFDIR   = confdir;
+our $DATADIR   = datadir;
+our $LIBDIR    = "$DATADIR/ext";
+our $PODDIR    = "$DATADIR/pod";
+our $MAPDIR    = "$DATADIR/" . mapdir;
+our $LOCALDIR  = localdir;
+our $TMPDIR    = "$LOCALDIR/" . tmpdir;
+our $UNIQUEDIR = "$LOCALDIR/" . uniquedir;
+our $PLAYERDIR = "$LOCALDIR/" . playerdir;
+our $RANDOMDIR = "$LOCALDIR/random";
+our $BDBDIR    = "$LOCALDIR/db";
+
+our $TICK = MAX_TIME * 1e-6; # this is a CONSTANT(!)
 our $TICK_WATCHER;
+our $AIO_POLL_WATCHER;
+our $NEXT_RUNTIME_WRITE; # when should the runtime file be written
 our $NEXT_TICK;
 our $NOW;
+our $USE_FSYNC = 1; # use fsync to write maps - default off
+
+our $BDB_POLL_WATCHER;
+our $DB_ENV;
 
 our %CFG;
 
 our $UPTIME; $UPTIME ||= time;
 our $RUNTIME;
 
-our %MAP; # all maps
-our $LINK_MAP; # the special {link} map
-our $FREEZE;
-our $RANDOM_MAPS = cf::localdir . "/random";
-our %EXT_CORO;
+our %PLAYER; # all users
+our %MAP;    # all maps
+our $LINK_MAP; # the special {link} map, which is always available
+
+# used to convert map paths into valid unix filenames by replacing / by ∕
+our $PATH_SEP = "∕"; # U+2215, chosen purely for visual reasons
+
+our $LOAD;       # a number between 0 (idle) and 1 (too many objects)
+our $LOADAVG;    # same thing, but with alpha-smoothing
+our $tick_start; # for load detecting purposes
 
 binmode STDOUT;
 binmode STDERR;
 
 # read virtual server time, if available
-unless ($RUNTIME || !-e cf::localdir . "/runtime") {
-   open my $fh, "<", cf::localdir . "/runtime"
+unless ($RUNTIME || !-e "$LOCALDIR/runtime") {
+   open my $fh, "<", "$LOCALDIR/runtime"
       or die "unable to read runtime file: $!";
    $RUNTIME = <$fh> + 0.;
 }
 
-mkdir cf::localdir;
-mkdir cf::localdir . "/" . cf::playerdir;
-mkdir cf::localdir . "/" . cf::tmpdir;
-mkdir cf::localdir . "/" . cf::uniquedir;
-mkdir $RANDOM_MAPS;
+mkdir $_
+   for $LOCALDIR, $TMPDIR, $UNIQUEDIR, $PLAYERDIR, $RANDOMDIR, $BDBDIR;
 
-# a special map that is always available
-our $LINK_MAP;
+our $EMERGENCY_POSITION;
 
-our $EMERGENCY_POSITION = $cf::CFG{emergency_position} || ["/world/world_105_115", 5, 37];
+sub cf::map::normalise;
 
 #############################################################################
 
@@ -91,10 +132,14 @@
 The time this server has run, starts at 0 and is increased by $cf::TICK on
 every server tick.
 
-=item $cf::LIBDIR
-
-The perl library directory, where extensions and cf-specific modules can
-be found. It will be added to C<@INC> automatically.
+=item $cf::CONFDIR $cf::DATADIR $cf::LIBDIR  $cf::PODDIR   
+$cf::MAPDIR $cf::LOCALDIR $cf::TMPDIR $cf::UNIQUEDIR
+$cf::PLAYERDIR $cf::RANDOMDIR $cf::BDBDIR   
+
+Various directories - "/etc", read-only install directory, perl-library
+directory, pod-directory, read-only maps directory, "/var", "/var/tmp",
+unique-items directory, player file directory, random maps directory and
+database environment.
 
 =item $cf::NOW
 
@@ -104,11 +149,33 @@
 
 The interval between server ticks, in seconds.
 
+=item $cf::LOADAVG
+
+The current CPU load on the server (alpha-smoothed), as a value between 0
+(none) and 1 (overloaded), indicating how much time is spent on processing
+objects per tick. Healthy values are < 0.5.
+
+=item $cf::LOAD
+
+The raw value load value from the last tick.
+
 =item %cf::CFG
 
 Configuration for the server, loaded from C</etc/crossfire/config>, or
 from wherever your confdir points to.
 
+=item cf::wait_for_tick, cf::wait_for_tick_begin
+
+These are functions that inhibit the current coroutine one tick. cf::wait_for_tick_begin only
+returns directly I<after> the tick processing (and consequently, can only wake one process
+per tick), while cf::wait_for_tick wakes up all waiters after tick processing.
+
+=item @cf::INVOKE_RESULTS
+
+This array contains the results of the last C<invoke ()> call.  When
+C<cf::override> is called C<@cf::INVOKE_RESULTS> is set to the parameters of
+that call.
+
 =back
 
 =cut
@@ -116,12 +183,14 @@
 BEGIN {
    *CORE::GLOBAL::warn = sub {
       my $msg = join "", @_;
-      utf8::encode $msg;
 
       $msg .= "\n"
          unless $msg =~ /\n$/;
 
-      LOG llevError, "cfperl: $msg";
+      $msg =~ s/([\x00-\x08\x0b-\x1f])/sprintf "\\x%02x", ord $1/ge;
+
+      utf8::encode $msg;
+      LOG llevError, $msg;
    };
 }
 
@@ -130,7 +199,8 @@
 @safe::cf::player::ISA         = @cf::player::ISA         = 'cf::attachable';
 @safe::cf::client::ISA         = @cf::client::ISA         = 'cf::attachable';
 @safe::cf::map::ISA            = @cf::map::ISA            = 'cf::attachable';
-@safe::cf::object::player::ISA = @cf::object::player::ISA = 'cf::object';
+@safe::cf::arch::ISA           = @cf::arch::ISA           = 'cf::object';
+@safe::cf::object::player::ISA = @cf::object::player::ISA = 'cf::object'; # not really true (yet)
 
 # we bless all objects into (empty) derived classes to force a method lookup
 # within the Safe compartment.
@@ -149,61 +219,49 @@
    warn "error in event callback: @_";
 };
 
-my %ext_pkg;
-my @exts;
-my @hook;
+#############################################################################
 
 =head2 UTILITY FUNCTIONS
 
 =over 4
 
+=item dumpval $ref
+
 =cut
 
-use JSON::Syck (); # TODO# replace by JSON::PC once working
+sub dumpval {
+   eval {
+      local $SIG{__DIE__};
+      my $d;
+      if (1) {
+         $d = new Data::Dumper([$_[0]], ["*var"]);
+         $d->Terse(1);
+         $d->Indent(2);
+         $d->Quotekeys(0);
+         $d->Useqq(1);
+         #$d->Bless(...);
+         $d->Seen($_[1]) if @_ > 1;
+         $d = $d->Dump();
+      }
+      $d =~ s/([\x00-\x07\x09\x0b\x0c\x0e-\x1f])/sprintf "\\x%02x", ord($1)/ge;
+      $d
+   } || "[unable to dump $_[0]: '$@']";
+}
 
 =item $ref = cf::from_json $json
 
 Converts a JSON string into the corresponding perl data structure.
 
-=cut
-
-sub from_json($) {
-   $JSON::Syck::ImplicitUnicode = 1; # work around JSON::Syck bugs
-   JSON::Syck::Load $_[0]
-}
-
 =item $json = cf::to_json $ref
 
 Converts a perl data structure into its JSON representation.
 
 =cut
 
-sub to_json($) {
-   $JSON::Syck::ImplicitUnicode = 0;  # work around JSON::Syck bugs
-   JSON::Syck::Dump $_[0]
-}
-
-=item my $guard = cf::guard { BLOCK }
-
-Run the given callback when the guard object gets destroyed (useful for
-coroutine cancellations).
+our $json_coder = JSON::XS->new->utf8->max_size (1e6); # accept ~1mb max
 
-You can call C<< ->cancel >> on the guard object to stop the block from
-being executed.
-
-=cut
-
-sub guard(&) {
-   bless \(my $cb = $_[0]), cf::guard::;
-}
-
-sub cf::guard::cancel {
-   ${$_[0]} = sub { };
-}
-
-sub cf::guard::DESTROY {
-   ${$_[0]}->();
-}
+sub to_json   ($) { $json_coder->encode ($_[0]) }
+sub from_json ($) { $json_coder->decode ($_[0]) }
 
 =item cf::lock_wait $string
 
@@ -212,12 +270,16 @@
 =item my $lock = cf::lock_acquire $string
 
 Wait until the given lock is available and then acquires it and returns
-a guard object. If the guard object gets destroyed (goes out of scope,
+a Coro::guard object. If the guard object gets destroyed (goes out of scope,
 for example when the coroutine gets canceled), the lock is automatically
 returned.
 
-Lock names should begin with a unique identifier (for example, find_map
-uses map_find and load_map uses map_load).
+Lock names should begin with a unique identifier (for example, cf::map::find
+uses map_find and cf::map::load uses map_load).
+
+=item $locked = cf::lock_active $string
+
+Return true if the lock is currently active, i.e. somebody has locked it.
 
 =cut
 
@@ -241,17 +303,100 @@
 
    $LOCK{$key} = [];
 
-   cf::guard {
+   Coro::guard {
       # wake up all waiters, to be on the safe side
       $_->ready for @{ delete $LOCK{$key} };
    }
 }
 
+sub lock_active($) {
+   my ($key) = @_;
+
+   ! ! $LOCK{$key}
+}
+
+sub freeze_mainloop {
+   return unless $TICK_WATCHER->is_active;
+
+   my $guard = Coro::guard {
+      $TICK_WATCHER->start;
+   };
+   $TICK_WATCHER->stop;
+   $guard
+}
+
+=item cf::get_slot $time[, $priority[, $name]]
+
+Allocate $time seconds of blocking CPU time at priority C<$priority>:
+This call blocks and returns only when you have at least C<$time> seconds
+of cpu time till the next tick. The slot is only valid till the next cede.
+
+The optional C<$name> can be used to identify the job to run. It might be
+used for statistical purposes and should identify the same time-class.
+
+Useful for short background jobs.
+
+=cut
+
+our @SLOT_QUEUE;
+our $SLOT_QUEUE;
+
+$SLOT_QUEUE->cancel if $SLOT_QUEUE;
+$SLOT_QUEUE = Coro::async {
+   my $signal = new Coro::Signal;
+
+   while () {
+      next_job:
+      my $avail = cf::till_tick;
+      if ($avail > 0.01) {
+         for (0 .. $#SLOT_QUEUE) {
+            if ($SLOT_QUEUE[$_][0] < $avail) {
+               my $job = splice @SLOT_QUEUE, $_, 1, ();
+               $job->[2]->send;
+               Coro::cede;
+               goto next_job;
+            }
+         }
+      }
+
+      if (@SLOT_QUEUE) {
+         # we do not use wait_For_tick() as it returns immediately when tick is inactive
+         push @cf::WAIT_FOR_TICK, $signal;
+         $signal->wait;
+      } else {
+         Coro::schedule;
+      }
+   }
+};
+
+sub get_slot($;$$) {
+   my ($time, $pri, $name) = @_;
+
+   $time = $TICK * .6 if $time > $TICK * .6;
+   my $sig = new Coro::Signal;
+
+   push @SLOT_QUEUE, [$time, $pri, $sig, $name];
+   @SLOT_QUEUE = sort { $b->[1] <=> $a->[1] } @SLOT_QUEUE;
+   $SLOT_QUEUE->ready;
+   $sig->wait;
+}
+
+=item cf::async { BLOCK }
+
+Currently the same as Coro::async_pool, meaning you cannot use
+C<on_destroy>, C<join> or other gimmicks on these coroutines. The only
+thing you are allowed to do is call C<prio> on it.
+
+=cut
+
+BEGIN { *async = \&Coro::async_pool }
+
 =item cf::sync_job { BLOCK }
 
-The design of crossfire+ requires that the main coro ($Coro::main) is
-always able to handle events or runnable, as crossfire+ is only partly
-reentrant. Thus "blocking" it by e.g. waiting for I/O is not acceptable.
+The design of Crossfire TRT requires that the main coroutine ($Coro::main)
+is always able to handle events or runnable, as Crossfire TRT is only
+partly reentrant. Thus "blocking" it by e.g. waiting for I/O is not
+acceptable.
 
 If it must be done, put the blocking parts into C<sync_job>. This will run
 the given BLOCK in another coroutine while waiting for the result. The
@@ -264,27 +409,35 @@
    my ($job) = @_;
 
    if ($Coro::current == $Coro::main) {
+      my $time = Event::time;
+
       # this is the main coro, too bad, we have to block
       # till the operation succeeds, freezing the server :/
       
       # TODO: use suspend/resume instead
       # (but this is cancel-safe)
-      local $FREEZE = 1;
+      my $freeze_guard = freeze_mainloop;
 
       my $busy = 1;
       my @res;
 
-      (Coro::async {
+      (async {
          @res = eval { $job->() };
          warn $@ if $@;
          undef $busy;
       })->prio (Coro::PRIO_MAX);
 
       while ($busy) {
-         Coro::cede_notself;
-         Event::one_event unless Coro::nready;
+         Coro::cede or Event::one_event;
       }
 
+      $time = Event::time - $time;
+
+      LOG llevError | logBacktrace, Carp::longmess "long sync job"
+         if $time > $TICK * 0.5 && $TICK_WATCHER->is_active;
+
+      $tick_start += $time; # do not account sync jobs to server load
+
       wantarray ? @res : $res[0]
    } else {
       # we are in another coroutine, how wonderful, everything just works
@@ -293,22 +446,17 @@
    }
 }
 
-=item $coro = cf::coro { BLOCK }
+=item $coro = cf::async_ext { BLOCK }
 
-Creates and returns a new coro. This coro is automcatially being canceled
-when the extension calling this is being unloaded.
+Like async, but this coro is automatically being canceled when the
+extension calling this is being unloaded.
 
 =cut
 
-sub coro(&) {
+sub async_ext(&) {
    my $cb = shift;
 
-   my $coro; $coro = async {
-      eval {
-         $cb->();
-      };
-      warn $@ if $@;
-   };
+   my $coro = &Coro::async ($cb);
 
    $coro->on_destroy (sub {
       delete $EXT_CORO{$coro+0};
@@ -318,151 +466,160 @@
    $coro
 }
 
-sub write_runtime {
-   my $runtime = cf::localdir . "/runtime";
+=item fork_call { }, $args
 
-   my $fh = aio_open "$runtime~", O_WRONLY | O_CREAT, 0644
-      or return;
+Executes the given code block with the given arguments in a seperate
+process, returning the results. Everything must be serialisable with
+Coro::Storable. May, of course, block. Note that the executed sub may
+never block itself or use any form of Event handling.
 
-   my $value = $cf::RUNTIME + 1 + 10; # 10 is the runtime save interval, for a monotonic clock
-   (aio_write $fh, 0, (length $value), $value, 0) <= 0
-      and return;
+=cut
 
-   aio_fsync $fh
-      and return;
+sub fork_call(&@) {
+   my ($cb, @args) = @_;
 
-   close $fh
-      or return;
+   # we seemingly have to make a local copy of the whole thing,
+   # otherwise perl prematurely frees the stuff :/
+   # TODO: investigate and fix (liekly this will be rather laborious)
 
-   aio_rename "$runtime~", $runtime
-      and return;
+   my @res = Coro::Util::fork_eval {
+      reset_signals;
+      &$cb
+   }, @args;
 
-   1
+   wantarray ? @res : $res[-1]
 }
 
-=back
+=item $value = cf::db_get $family => $key
+
+Returns a single value from the environment database.
+
+=item cf::db_put $family => $key => $value
+
+Stores the given C<$value> in the family. It can currently store binary
+data only (use Compress::LZF::sfreeze_cr/sthaw to convert to/from binary).
 
 =cut
 
-#############################################################################
+our $DB;
+
+sub db_init {
+   unless ($DB) {
+      $DB = BDB::db_create $DB_ENV;
+
+      cf::sync_job {
+         eval {
+            $DB->set_flags (BDB::CHKSUM);
+
+            BDB::db_open $DB, undef, "db", undef, BDB::BTREE,
+                         BDB::CREATE | BDB::AUTO_COMMIT, 0666;
+            cf::cleanup "db_open(db): $!" if $!;
+         };
+         cf::cleanup "db_open(db): $@" if $@;
+      };
+   }
+}
 
-package cf::path;
+sub db_get($$) {
+   my $key = "$_[0]/$_[1]";
 
-sub new {
-   my ($class, $path, $base) = @_;
+   cf::sync_job {
+      BDB::db_get $DB, undef, $key, my $data;
 
-   $path = $path->as_string if ref $path;
+      $! ? ()
+         : $data
+   }
+}
 
-   my $self = bless { }, $class;
+sub db_put($$$) {
+   BDB::dbreq_pri 4;
+   BDB::db_put $DB, undef, "$_[0]/$_[1]", $_[2], 0, sub { };
+}
 
-   # {...        are special paths that are not touched
-   # ?xxx/...    are special absolute paths
-   # ?random/... random maps
-   # /!          non-realised random map exit
-   # /...        normal maps
-   # ~/...       per-player maps without a specific player (DO NOT USE)
-   # ~user/...   per-player map of a specific user
+=item cf::cache $id => [$paths...], $processversion => $process
 
-   if ($path =~ /^{/) {
-      # fine as it is
-   } elsif ($path =~ s{^\?random/}{}) {
-      Coro::AIO::aio_load "$cf::RANDOM_MAPS/$path.meta", my $data;
-      $self->{random} = cf::from_json $data;
-   } else {
-      if ($path =~ s{^~([^/]+)?}{}) {
-         $self->{user_rel} = 1;
+Generic caching function that returns the value of the resource $id,
+caching and regenerating as required.
 
-         if (defined $1) {
-            $self->{user} = $1;
-         } elsif ($base =~ m{^~([^/]+)/}) {
-            $self->{user} = $1;
-         } else {
-            warn "cannot resolve user-relative path without user <$path,$base>\n";
-         }
-      } elsif ($path =~ /^\//) {
-         # already absolute
-      } else {
-         $base =~ s{[^/]+/?$}{};
-         return $class->new ("$base/$path");
-      }
+This function can block.
 
-      for ($path) {
-         redo if s{/\.?/}{/};
-         redo if s{/[^/]+/\.\./}{/};
-      }
-   }
+=cut
 
-   $self->{path} = $path;
+sub cache {
+   my ($id, $src, $processversion, $process) = @_;
 
-   $self
-}
+   my $meta =
+      join "\x00",
+         $processversion,
+         map {
+            aio_stat $_
+               and Carp::croak "$_: $!";
 
-# the name / primary key / in-game path
-sub as_string {
-   my ($self) = @_;
+            ($_, (stat _)[7,9])
+         } @$src;
 
-   $self->{user_rel} ? "~$self->{user}$self->{path}"
-   : $self->{random} ? "?random/$self->{path}"
-                     : $self->{path}
-}
+   my $dbmeta = db_get cache => "$id/meta";
+   if ($dbmeta ne $meta) {
+      # changed, we may need to process
 
-# the displayed name, this is a one way mapping
-sub visible_name {
-   my ($self) = @_;
+      my @data;
+      my $md5;
 
-#   if (my $rmp = $self->{random}) {
-#      # todo: be more intelligent about this
-#      "?random/$rmp->{origin_map}+$rmp->{origin_x}+$rmp->{origin_y}/$rmp->{dungeon_level}"
-#   } else {
-      $self->as_string
-#   }
-}
+      for (0 .. $#$src) {
+         0 <= aio_load $src->[$_], $data[$_]
+            or Carp::croak "$src->[$_]: $!";
+      }
 
-# escape the /'s in the path
-sub _escaped_path  {
-   # ∕ is U+2215
-   (my $path = $_[0]{path}) =~ s/\//∕/g;
-   $path
-}
+      # if processing is expensive, check
+      # checksum first
+      if (1) {
+         $md5 =
+            join "\x00",
+               $processversion,
+               map {
+                  cf::cede_to_tick;
+                  ($src->[$_], Digest::MD5::md5_hex $data[$_])
+               } 0.. $#$src;
+           
 
-# the original (read-only) location
-sub load_path {
-   my ($self) = @_;
+         my $dbmd5 = db_get cache => "$id/md5";
+         if ($dbmd5 eq $md5) {
+            db_put cache => "$id/meta", $meta;
 
-   sprintf "%s/%s/%s", cf::datadir, cf::mapdir, $self->{path}
-}
+            return db_get cache => "$id/data";
+         }
+      }
 
-# the temporary/swap location
-sub save_path {
-   my ($self) = @_;
+      my $t1 = Time::HiRes::time;
+      my $data = $process->(\@data);
+      my $t2 = Time::HiRes::time;
 
-   $self->{user_rel} ? sprintf "%s/%s/%s/%s", cf::localdir, cf::playerdir, $self->{user}, $self->_escaped_path
-   : $self->{random} ? sprintf "%s/%s", $RANDOM_MAPS, $self->{path}
-                     : sprintf "%s/%s/%s", cf::localdir, cf::tmpdir, $self->_escaped_path
-}
+      warn "cache: '$id' processed in ", $t2 - $t1, "s\n";
 
-# the unique path, might be eq to save_path
-sub uniq_path {
-   my ($self) = @_;
+      db_put cache => "$id/data", $data;
+      db_put cache => "$id/md5" , $md5;
+      db_put cache => "$id/meta", $meta;
+
+      return $data;
+   }
 
-   $self->{user_rel} || $self->{random}
-      ? undef
-      : sprintf "%s/%s/%s", cf::localdir, cf::uniquedir, $self->_escaped_path
+   db_get cache => "$id/data"
 }
 
-# return random map parameters, or undef
-sub random_map_params {
-   my ($self) = @_;
+=item cf::datalog type => key => value, ...
 
-   $self->{random}
-}
+Log a datalog packet of the given type with the given key-value pairs.
+
+=cut
 
-# this is somewhat ugly, but style maps do need special treatment
-sub is_style_map {
-   $_[0]{path} =~ m{^/styles/}
+sub datalog($@) {
+  my ($type, %kv) = @_;
+  warn "DATALOG ", JSON::XS->new->ascii->encode ({ %kv, type => $type });
 }
 
-package cf;
+=back
+
+=cut
 
 #############################################################################
 
@@ -474,7 +631,7 @@
 
 In the following description, CLASS can be any of C<global>, C<object>
 C<player>, C<client> or C<map> (i.e. the attachable objects in
-crossfire+).
+Crossfire TRT).
 
 =over 4
 
@@ -598,6 +755,12 @@
 
 my %attachment;
 
+sub cf::attachable::thawer_merge {
+   # simply override everything except _meta
+   local $_[0]{_meta};
+   %{$_[0]} = %{$_[1]};
+}
+
 sub _attach_cb($$$$) {
    my ($registry, $event, $prio, $cb) = @_;
 
@@ -641,7 +804,7 @@
       } elsif ($type eq "subtype") {
          defined $object_type or Carp::croak "subtype specified without type";
          my $object_subtype = shift @arg;
-         $registry = $CB_TYPE[$object_type + $object_subtype * NUM_SUBTYPES] ||= [];
+         $registry = $CB_TYPE[$object_type + $object_subtype * NUM_TYPES] ||= [];
 
       } elsif ($type eq "package") {
          my $pkg = shift @arg;
@@ -680,7 +843,7 @@
 
       $obj->{$name} = \%arg;
    } else {
-      warn "object uses attachment '$name' that is not available, postponing.\n";
+      warn "object uses attachment '$name' which is not available, postponing.\n";
    }
 
    $obj->{_attachment}{$name} = undef;
@@ -692,6 +855,7 @@
    } else {
       _attach shift->_attach_registry, @_;
    }
+   _recalc_want;
 };
 
 # all those should be optimised
@@ -704,6 +868,7 @@
    } else {
       Carp::croak "cannot, currently, detach class attachments";
    }
+   _recalc_want;
 };
 
 sub cf::attachable::attached {
@@ -728,18 +893,18 @@
 }
 
 our $override;
-our @invoke_results = (); # referenced from .xs code. TODO: play tricks with reify and mortals?
+our @INVOKE_RESULTS = (); # referenced from .xs code. TODO: play tricks with reify and mortals?
 
 sub override {
    $override = 1;
-   @invoke_results = ();
+   @INVOKE_RESULTS = (@_);
 }
 
 sub do_invoke {
    my $event = shift;
    my $callbacks = shift;
 
-   @invoke_results = ();
+   @INVOKE_RESULTS = ();
 
    local $override;
 
@@ -766,7 +931,7 @@
 
 This API is preliminary (most likely, the EVENT_CLASS_xxx prefix will be
 removed in future versions), and there is no public API to access override
-results (if you must, access C<@cf::invoke_results> directly).
+results (if you must, access C<@cf::INVOKE_RESULTS> directly).
 
 =back
 
@@ -774,11 +939,24 @@
 
 #############################################################################
 # object support
+#
+
+sub _can_merge {
+   my ($ob1, $ob2) = @_;
+
+   local $Storable::canonical = 1;
+   my $fob1 = Storable::freeze $ob1;
+   my $fob2 = Storable::freeze $ob2;
+
+   $fob1 eq $fob2
+}
 
 sub reattach {
    # basically do the same as instantiate, without calling instantiate
    my ($obj) = @_;
 
+   bless $obj, ref $obj; # re-bless in case extensions have been reloaded
+
    my $registry = $obj->registry;
 
    @$registry = ();
@@ -834,7 +1012,7 @@
          if (my $fh = aio_open "$filename~", O_WRONLY | O_CREAT, 0600) {
             chmod SAVE_MODE, $fh;
             aio_write $fh, 0, (length $$rdata), $$rdata, 0;
-            aio_fsync $fh;
+            aio_fsync $fh if $cf::USE_FSYNC;
             close $fh;
 
             if (@$objs) {
@@ -842,7 +1020,7 @@
                   chmod SAVE_MODE, $fh;
                   my $data = Storable::nfreeze { version => 1, objs => $objs };
                   aio_write $fh, 0, (length $data), $data, 0;
-                  aio_fsync $fh;
+                  aio_fsync $fh if $cf::USE_FSYNC;
                   close $fh;
                   aio_rename "$filename.pst~", "$filename.pst";
                }
@@ -884,10 +1062,16 @@
    }
 
    warn sprintf "loading %s (%d)\n",
-      $filename, length $data, scalar @{$av || []};#d#
+      $filename, length $data, scalar @{$av || []};
    return ($data, $av);
 }
 
+=head2 COMMAND CALLBACKS
+
+=over 4
+
+=cut
+
 #############################################################################
 # command handling &c
 
@@ -909,7 +1093,22 @@
 
 =item cf::register_extcmd $name => \&callback($pl,$packet);
 
-Register a callbackf ro execution when the client sends an extcmd packet.
+Register a callback for execution when the client sends an (synchronous)
+extcmd packet. Ext commands will be processed in the order they are
+received by the server, like other user commands. The first argument is
+the logged-in player. Ext commands can only be processed after a player
+has logged in successfully.
+
+If the callback returns something, it is sent back as if reply was being
+called.
+
+=item cf::register_exticmd $name => \&callback($ns,$packet);
+
+Register a callback for execution when the client sends an (asynchronous)
+exticmd packet. Exti commands are processed by the server as soon as they
+are received, i.e. out of order w.r.t. other commands. The first argument
+is a client socket. Exti commands can be received anytime, even before
+log-in.
 
 If the callback returns something, it is sent back as if reply was being
 called.
@@ -919,10 +1118,13 @@
 sub register_extcmd {
    my ($name, $cb) = @_;
 
-   my $caller = caller;
-   #warn "registering extcmd '$name' to '$caller'";
+   $EXTCMD{$name} = $cb;
+}
 
-   $EXTCMD{$name} = [$cb, $caller];
+sub register_exticmd {
+   my ($name, $cb) = @_;
+
+   $EXTICMD{$name} = $cb;
 }
 
 cf::player->attach (
@@ -941,14 +1143,23 @@
    on_extcmd => sub {
       my ($pl, $buf) = @_;
 
-      my $msg = eval { from_json $buf };
+      my $msg = eval { $pl->ns->{json_coder}->decode ($buf) };
 
       if (ref $msg) {
-         if (my $cb = $EXTCMD{$msg->{msgtype}}) {
-            if (my %reply = $cb->[0]->($pl, $msg)) {
-               $pl->ext_reply ($msg->{msgid}, %reply);
-            }
+         my ($type, $reply, @payload) =
+            "ARRAY" eq ref $msg
+               ? @$msg
+               : ($msg->{msgtype}, $msg->{msgid}, %$msg); # TODO: version 1, remove
+
+         my @reply;
+
+         if (my $cb = $EXTCMD{$type}) {
+            @reply = $cb->($pl, @payload);
          }
+
+         $pl->ext_reply ($reply, @reply)
+            if $reply;
+
       } else {
          warn "player " . ($pl->ob->name) . " sent unparseable ext message: <$buf>\n";
       }
@@ -957,165 +1168,393 @@
    },
 );
 
-sub register {
-   my ($base, $pkg) = @_;
+sub load_extensions {
+   cf::sync_job {
+      my %todo;
+
+      for my $path (<$LIBDIR/*.ext>) {
+         next unless -r $path;
+
+         $path =~ /([^\/\\]+)\.ext$/ or die "$path";
+         my $base = $1;
+         my $pkg = $1;
+         $pkg =~ s/[^[:word:]]/_/g;
+         $pkg = "ext::$pkg";
+
+         open my $fh, "<:utf8", $path
+            or die "$path: $!";
+
+         my $source = do { local $/; <$fh> };
+
+         my %ext = (
+            path => $path,
+            base => $base,
+            pkg  => $pkg,
+         );
+
+         $ext{meta} = { map { (split /=/, $_, 2)[0, 1] } split /\s+/, $1 }
+            if $source =~ /\A#!.*?perl.*?#\s*(.*)$/m;
+
+         $ext{source} =
+            "package $pkg; use strict; use utf8;\n"
+            . "#line 1 \"$path\"\n{\n"
+            . $source
+            . "\n};\n1";
+
+         $todo{$base} = \%ext;
+      }
+
+      my %done;
+      while (%todo) {
+         my $progress;
+
+         while (my ($k, $v) = each %todo) {
+            for (split /,\s*/, $v->{meta}{depends}) {
+               goto skip
+                  unless exists $done{$_};
+            }
 
-   #TODO
+            warn "... loading '$k' into '$v->{pkg}'\n";
+
+            unless (eval $v->{source}) {
+               my $msg = $@ ? "$v->{path}: $@\n"
+                            : "$v->{base}: extension inactive.\n";
+
+               if (exists $v->{meta}{mandatory}) {
+                  warn $msg;
+                  warn "mandatory extension failed to load, exiting.\n";
+                  exit 1;
+               }
+               
+               warn $msg;
+            }
+
+            $done{$k} = delete $todo{$k};
+            push @EXTS, $v->{pkg};
+            $progress = 1;
+         }
+
+         skip:
+         die "cannot load " . (join ", ", keys %todo) . ": unable to resolve dependencies\n"
+            unless $progress;
+      }
+   };
 }
 
-sub load_extension {
-   my ($path) = @_;
+#############################################################################
 
-   $path =~ /([^\/\\]+)\.ext$/ or die "$path";
-   my $base = $1;
-   my $pkg = $1;
-   $pkg =~ s/[^[:word:]]/_/g;
-   $pkg = "ext::$pkg";
+=back
 
-   warn "loading '$path' into '$pkg'\n";
+=head2 CORE EXTENSIONS
 
-   open my $fh, "<:utf8", $path
-      or die "$path: $!";
+Functions and methods that extend core crossfire objects.
+
+=cut
+
+package cf::player;
+
+use Coro::AIO;
+
+=head3 cf::player
 
-   my $source =
-      "package $pkg; use strict; use utf8;\n"
-      . "#line 1 \"$path\"\n{\n"
-      . (do { local $/; <$fh> })
-      . "\n};\n1";
+=over 4
 
-   eval $source
-      or die $@ ? "$path: $@\n"
-                : "extension disabled.\n";
+=item cf::player::find $login
 
-   push @exts, $pkg;
-   $ext_pkg{$base} = $pkg;
+Returns the given player object, loading it if necessary (might block).
 
-#   no strict 'refs';
-#   @{"$pkg\::ISA"} = ext::;
+=cut
 
-   register $base, $pkg;
+sub playerdir($) {
+   "$PLAYERDIR/"
+     . (ref $_[0] ? $_[0]->ob->name : $_[0])
 }
 
-sub unload_extension {
-   my ($pkg) = @_;
+sub path($) {
+   my $login = ref $_[0] ? $_[0]->ob->name : $_[0];
 
-   warn "removing extension $pkg\n";
+   (playerdir $login) . "/playerdata"
+}
 
-   # remove hooks
-   #TODO
-#   for my $idx (0 .. $#PLUGIN_EVENT) {
-#      delete $hook[$idx]{$pkg};
-#   }
+sub find_active($) {
+   $cf::PLAYER{$_[0]}
+      and $cf::PLAYER{$_[0]}->active
+      and $cf::PLAYER{$_[0]}
+}
 
-   # remove commands
-   for my $name (keys %COMMAND) {
-      my @cb = grep $_->[0] ne $pkg, @{ $COMMAND{$name} };
+sub exists($) {
+   my ($login) = @_;
 
-      if (@cb) {
-         $COMMAND{$name} = \@cb;
-      } else {
-         delete $COMMAND{$name};
+   $cf::PLAYER{$login}
+      or cf::sync_job { !aio_stat path $login }
+}
+
+sub find($) {
+   return $cf::PLAYER{$_[0]} || do {
+      my $login = $_[0];
+
+      my $guard = cf::lock_acquire "user_find:$login";
+
+      $cf::PLAYER{$_[0]} || do {
+         # rename old playerfiles to new ones
+         #TODO: remove when no longer required
+         aio_link +(playerdir $login) . "/$login.pl.pst", (playerdir $login) . "/playerdata.pst";
+         aio_link +(playerdir $login) . "/$login.pl"    , (playerdir $login) . "/playerdata";
+         aio_unlink +(playerdir $login) . "/$login.pl.pst";
+         aio_unlink +(playerdir $login) . "/$login.pl";
+
+         my $pl = load_pl path $login
+            or return;
+         $cf::PLAYER{$login} = $pl
       }
    }
+}
 
-   # remove extcmds
-   for my $name (grep $EXTCMD{$_}[1] eq $pkg, keys %EXTCMD) {
-      delete $EXTCMD{$name};
-   }
+sub save($) {
+   my ($pl) = @_;
 
-   if (my $cb = $pkg->can ("unload")) {
-      eval {
-         $cb->($pkg);
-         1
-      } or warn "$pkg unloaded, but with errors: $@";
-   }
+   return if $pl->{deny_save};
+
+   my $path = path $pl;
+   my $guard = cf::lock_acquire "user_save:$path";
 
-   Symbol::delete_package $pkg;
+   return if $pl->{deny_save};
+
+   aio_mkdir playerdir $pl, 0770;
+   $pl->{last_save} = $cf::RUNTIME;
+
+   $pl->save_pl ($path);
+   cf::cede_to_tick;
 }
 
-sub load_extensions {
-   for my $ext (<$LIBDIR/*.ext>) {
-      next unless -r $ext;
-      eval {
-         load_extension $ext;
-         1
-      } or warn "$ext not loaded: $@";
-   }
+sub new($) {
+   my ($login) = @_;
+
+   my $self = create;
+
+   $self->ob->name ($login);
+   $self->{deny_save} = 1;
+
+   $cf::PLAYER{$login} = $self;
+
+   $self
 }
 
-#############################################################################
-# load/save/clean perl data associated with a map
+=item $player->send_msg ($channel, $msg, $color, [extra...])
 
-*cf::mapsupport::on_clean = sub {
-   my ($map) = @_;
+=cut
+
+sub send_msg {
+   my $ns = shift->ns
+      or return;
+   $ns->send_msg (@_);
+}
 
-   my $path = $map->tmpname;
-   defined $path or return;
+=item $pl->quit_character
 
-   unlink "$path.pst";
-};
+Nukes the player without looking back. If logged in, the connection will
+be destroyed. May block for a long time.
 
-cf::map->attach (prio => -10000, package => cf::mapsupport::);
+=cut
 
-#############################################################################
-# load/save perl data associated with player->ob objects
+sub quit_character {
+   my ($pl) = @_;
+
+   my $name = $pl->ob->name;
+
+   $pl->{deny_save} = 1;
+   $pl->password ("*"); # this should lock out the player until we nuked the dir
+
+   $pl->invoke (cf::EVENT_PLAYER_LOGOUT, 1) if $pl->active;
+   $pl->deactivate;
+   $pl->invoke (cf::EVENT_PLAYER_QUIT);
+   $pl->ns->destroy if $pl->ns;
 
-sub all_objects(@) {
-   @_, map all_objects ($_->inv), @_
+   my $path = playerdir $pl;
+   my $temp = "$path~$cf::RUNTIME~deleting~";
+   aio_rename $path, $temp;
+   delete $cf::PLAYER{$pl->ob->name};
+   $pl->destroy;
+
+   my $prefix = qr<^~\Q$name\E/>;
+
+   # nuke player maps
+   $cf::MAP{$_}->nuke for grep /$prefix/, keys %cf::MAP;
+
+   IO::AIO::aio_rmtree $temp;
 }
 
-# TODO: compatibility cruft, remove when no longer needed
-cf::player->attach (
-   on_load => sub {
-      my ($pl, $path) = @_;
+=item $pl->kick
 
-      for my $o (all_objects $pl->ob) {
-         if (my $value = $o->get_ob_key_value ("_perl_data")) {
-            $o->set_ob_key_value ("_perl_data");
+Kicks a player out of the game. This destroys the connection.
 
-            %$o = %{ Storable::thaw pack "H*", $value };
-         }
-      }
-   },
-);
+=cut
 
-#############################################################################
+sub kick {
+   my ($pl, $kicker) = @_;
 
-=head2 CORE EXTENSIONS
+   $pl->invoke (cf::EVENT_PLAYER_KICK, $kicker);
+   $pl->killer ("kicked");
+   $pl->ns->destroy;
+}
 
-Functions and methods that extend core crossfire objects.
+=item cf::player::list_logins
 
-=head3 cf::player
+Returns am arrayref of all valid playernames in the system, can take a
+while and may block, so not sync_job-capable, ever.
 
-=over 4
+=cut
+
+sub list_logins {
+   my $dirs = aio_readdir $PLAYERDIR
+      or return [];
+
+   my @logins;
+
+   for my $login (@$dirs) {
+      my $fh = aio_open path $login, Fcntl::O_RDONLY, 0 or next;
+      aio_read $fh, 0, 512, my $buf, 0                  or next;
+      $buf !~ /^password -------------$/m               or next; # official not-valid tag
+
+      utf8::decode $login;
+      push @logins, $login;
+   }
+
+   \@logins
+}
+
+=item $player->maps
+
+Returns an arrayref of map paths that are private for this
+player. May block.
+
+=cut
+
+sub maps($) {
+   my ($pl) = @_;
+
+   $pl = ref $pl ? $pl->ob->name : $pl;
+
+   my $files = aio_readdir playerdir $pl
+      or return;
+
+   my @paths;
 
-=item cf::player::exists $login
+   for (@$files) {
+      utf8::decode $_;
+      next if /\.(?:pl|pst)$/;
+      next unless /^$PATH_SEP/o;
 
-Returns true when the given account exists.
+      push @paths, cf::map::normalise "~$pl/$_";
+   }
+
+   \@paths
+}
+
+=item $protocol_xml = $player->expand_cfpod ($crossfire_pod)
+
+Expand crossfire pod fragments into protocol xml.
 
 =cut
 
-sub cf::player::exists($) {
-   cf::player::find $_[0]
-      or -f sprintf "%s/%s/%s/%s.pl", cf::localdir, cf::playerdir, ($_[0]) x 2;
+sub expand_cfpod {
+   ((my $self), (local $_)) = @_;
+
+   # escape & and <
+   s/&/&amp;/g;
+   s/(?<![BIUGH])</&lt;/g;
+
+   # this is buggy, it needs to properly take care of nested <'s
+
+   1 while
+      # replace B<>, I<>, U<> etc.
+      s/B<([^\>]*)>/<b>$1<\/b>/
+      || s/I<([^\>]*)>/<i>$1<\/i>/
+      || s/U<([^\>]*)>/<u>$1<\/u>/
+      # replace G<male|female> tags
+      || s{G<([^>|]*)\|([^>]*)>}{
+         $self->gender ? $2 : $1
+      }ge
+      # replace H<hint text>
+      || s{H<([^\>]*)>}
+          {
+             ("<fg name=\"lightblue\">[$1 (Use hintmode to suppress hints)]</fg>",
+              "<fg name=\"lightblue\">[Hint suppressed, see hintmode]</fg>",
+              "")
+                [$self->{hintmode}]
+          }ge;
+
+   # create single paragraphs (very hackish)
+   s/(?<=\S)\n(?=\w)/ /g;
+
+   # compress some whitespace
+   s/\s+\n/\n/g; # ws line-ends
+   s/\n\n+/\n/g; # double lines
+   s/^\n+//;     # beginning lines
+   s/\n+$//;     # ending lines
+
+   $_
+}
+
+sub hintmode {
+   $_[0]{hintmode} = $_[1] if @_ > 1;
+   $_[0]{hintmode}
 }
 
-=item $player->ext_reply ($msgid, $msgtype, %msg)
+=item $player->ext_reply ($msgid, @msg)
 
 Sends an ext reply to the player.
 
 =cut
 
-sub cf::player::ext_reply($$$%) {
-   my ($self, $id, %msg) = @_;
+sub ext_reply($$@) {
+   my ($self, $id, @msg) = @_;
 
-   $msg{msgid} = $id;
+   $self->ns->ext_reply ($id, @msg)
+}
+
+=item $player->ext_msg ($type, @msg)
 
-   $self->send ("ext " . to_json \%msg);
+Sends an ext event to the client.
+
+=cut
+
+sub ext_msg($$@) {
+   my ($self, $type, @msg) = @_;
+
+   $self->ns->ext_msg ($type, @msg);
 }
 
-=back
+=head3 cf::region
+
+=over 4
+
+=cut
+
+package cf::region;
+
+=item cf::region::find_by_path $path
 
+Tries to decuce the likely region for a map knowing only its path.
+
+=cut
+
+sub find_by_path($) {
+   my ($path) = @_;
+
+   my ($match, $specificity);
+
+   for my $region (list) {
+      if ($region->{match} && $path =~ $region->{match}) {
+         ($match, $specificity) = ($region, $region->specificity)
+            if $region->specificity > $specificity;
+      }
+   }
+
+   $match
+}
+
+=back
 
 =head3 cf::map
 
@@ -1128,15 +1567,17 @@
 use Fcntl;
 use Coro::AIO;
 
-our $MAX_RESET     = 7200;
-our $DEFAULT_RESET = 3600;
+use overload
+   '""'     => \&as_string,
+   fallback => 1;
 
-sub generate_random_map {
-   my ($path, $rmp) = @_;
+our $MAX_RESET     = 3600;
+our $DEFAULT_RESET = 3000;
 
+sub generate_random_map {
+   my ($self, $rmp) = @_;
    # mit "rum" bekleckern, nicht
-   cf::map::_create_random_map
-      $path,
+   $self->_create_random_map (
       $rmp->{wallstyle}, $rmp->{wall_name}, $rmp->{floorstyle}, $rmp->{monsterstyle},
       $rmp->{treasurestyle}, $rmp->{layoutstyle}, $rmp->{doorstyle}, $rmp->{decorstyle},
       $rmp->{origin_map}, $rmp->{final_map}, $rmp->{exitstyle}, $rmp->{this_map},
@@ -1147,7 +1588,135 @@
       $rmp->{dungeon_level}, $rmp->{dungeon_depth}, $rmp->{decoroptions}, $rmp->{orientation},
       $rmp->{origin_y}, $rmp->{origin_x}, $rmp->{random_seed}, $rmp->{total_map_hp},
       $rmp->{map_layout_style}, $rmp->{treasureoptions}, $rmp->{symmetry_used},
-      (cf::region::find $rmp->{region})
+      (cf::region::find $rmp->{region}), $rmp->{custom}
+   )
+}
+
+=item cf::map->register ($regex, $prio)
+
+Register a handler for the map path matching the given regex at the
+givne priority (higher is better, built-in handlers have priority 0, the
+default).
+
+=cut
+
+sub register {
+   my (undef, $regex, $prio) = @_;
+   my $pkg = caller;
+
+   no strict;
+   push @{"$pkg\::ISA"}, __PACKAGE__;
+
+   $EXT_MAP{$pkg} = [$prio, qr<$regex>];
+}
+
+# also paths starting with '/'
+$EXT_MAP{"cf::map"} = [0, qr{^(?=/)}];
+
+sub thawer_merge {
+   my ($self, $merge) = @_;
+
+   # we have to keep some variables in memory intact
+   local $self->{path};
+   local $self->{load_path};
+
+   $self->SUPER::thawer_merge ($merge);
+}
+
+sub normalise {
+   my ($path, $base) = @_;
+
+   $path = "$path"; # make sure its a string
+
+   $path =~ s/\.map$//;
+
+   # map plan:
+   #
+   # /!          non-realised random map exit (special hack!)
+   # {...        are special paths that are not being touched
+   # ?xxx/...    are special absolute paths
+   # ?random/... random maps
+   # /...        normal maps
+   # ~user/...   per-player map of a specific user
+
+   $path =~ s/$PATH_SEP/\//go;
+
+   # treat it as relative path if it starts with
+   # something that looks reasonable
+   if ($path =~ m{^(?:\./|\.\./|\w)}) {
+      $base or Carp::carp "normalise called with relative path and no base: '$path'";
+
+      $base =~ s{[^/]+/?$}{};
+      $path = "$base/$path";
+   }
+
+   for ($path) {
+      redo if s{//}{/};
+      redo if s{/\.?/}{/};
+      redo if s{/[^/]+/\.\./}{/};
+   }
+
+   $path
+}
+
+sub new_from_path {
+   my (undef, $path, $base) = @_;
+
+   return $path if UNIVERSAL::isa $path, "cf::map"; # already a map object
+
+   $path = normalise $path, $base;
+
+   for my $pkg (sort { $EXT_MAP{$b}[0] <=> $EXT_MAP{$a}[0] } keys %EXT_MAP) {
+      if ($path =~ $EXT_MAP{$pkg}[1]) {
+         my $self = bless cf::map::new, $pkg;
+         $self->{path} = $path; $self->path ($path);
+         $self->init; # pass $1 etc.
+         return $self;
+      }
+   }
+
+   Carp::cluck "unable to resolve path '$path' (base '$base').";
+   ()
+}
+
+sub init {
+   my ($self) = @_;
+
+   $self
+}
+
+sub as_string {
+   my ($self) = @_;
+
+   "$self->{path}"
+}
+
+# the displayed name, this is a one way mapping
+sub visible_name {
+   &as_string
+}
+
+# the original (read-only) location
+sub load_path {
+   my ($self) = @_;
+
+   "$MAPDIR/$self->{path}.map"
+}
+
+# the temporary/swap location
+sub save_path {
+   my ($self) = @_;
+
+   (my $path = $_[0]{path}) =~ s/\//$PATH_SEP/g;
+   "$TMPDIR/$path.map"
+}
+
+# the unique path, undef == no special unique path
+sub uniq_path {
+   my ($self) = @_;
+
+   (my $path = $_[0]{path}) =~ s/\//$PATH_SEP/g;
+   "$UNIQUEDIR/$path"
 }
 
 # and all this just because we cannot iterate over
@@ -1159,164 +1728,307 @@
       for grep $_->outdoor, values %cf::MAP;
 }
 
-sub try_load_header($) {
-   my ($path) = @_;
+sub decay_objects {
+   my ($self) = @_;
+
+   return if $self->{deny_reset};
+
+   $self->do_decay_objects;
+}
+
+sub unlink_save {
+   my ($self) = @_;
+
+   utf8::encode (my $save = $self->save_path);
+   IO::AIO::aioreq_pri 4; Coro::AIO::aio_unlink $save;
+   IO::AIO::aioreq_pri 4; Coro::AIO::aio_unlink "$save.pst";
+}
+
+sub load_header_from($) {
+   my ($self, $path) = @_;
 
    utf8::encode $path;
-   aio_open $path, O_RDONLY, 0
-      or return;
+   #aio_open $path, O_RDONLY, 0
+   #   or return;
 
-   my $map = cf::map::new
+   $self->_load_header ($path)
       or return;
 
-   $map->load_header ($path)
-     or return;
+   $self->{load_path} = $path;
 
-   $map->{load_path} = $path;
+   1
+}
 
-   $map
+sub load_header_orig {
+   my ($self) = @_;
+
+   $self->load_header_from ($self->load_path)
 }
 
-sub find_map {
-   my ($path, $origin) = @_;
+sub load_header_temp {
+   my ($self) = @_;
 
-   #warn "find_map<$path,$origin>\n";#d#
+   $self->load_header_from ($self->save_path)
+}
 
-   $path = new cf::path $path, $origin && $origin->path;
-   my $key = $path->as_string;
+sub prepare_temp {
+   my ($self) = @_;
 
-   cf::lock_wait "map_find:$key";
+   $self->last_access ((delete $self->{last_access})
+                       || $cf::RUNTIME); #d#
+   # safety
+   $self->{instantiate_time} = $cf::RUNTIME
+      if $self->{instantiate_time} > $cf::RUNTIME;
+}
 
-   $cf::MAP{$key} || do {
-      my $guard = cf::lock_acquire "map_find:$key";
+sub prepare_orig {
+   my ($self) = @_;
 
-      # do it the slow way
-      my $map = try_load_header $path->save_path;
+   $self->{load_original} = 1;
+   $self->{instantiate_time} = $cf::RUNTIME;
+   $self->last_access ($cf::RUNTIME);
+   $self->instantiate;
+}
 
-      if ($map) {
-         # safety
-         $map->{instantiate_time} = $cf::RUNTIME
-            if $map->{instantiate_time} > $cf::RUNTIME;
-      } else {
-         if (my $rmp = $path->random_map_params) {
-            $map = generate_random_map $key, $rmp;
-         } else {
-            $map = try_load_header $path->load_path;
-         }
+sub load_header {
+   my ($self) = @_;
+
+   if ($self->load_header_temp) {
+      $self->prepare_temp;
+   } else {
+      $self->load_header_orig
+         or return;
+      $self->prepare_orig;
+   }
 
-         $map or return;
+   $self->{deny_reset} = 1
+      if $self->no_reset;
 
-         $map->{load_original} = 1;
-         $map->{instantiate_time} = $cf::RUNTIME;
-         $map->instantiate;
+   $self->default_region (cf::region::find_by_path $self->{path})
+      unless $self->default_region;
 
-         # per-player maps become, after loading, normal maps
-         $map->per_player (0) if $path->{user_rel};
-      }
+   1
+}
+
+sub find;
+sub find {
+   my ($path, $origin) = @_;
+
+   $path = normalise $path, $origin && $origin->path;
+
+   cf::lock_wait "map_find:$path";
+
+   $cf::MAP{$path} || do {
+      my $guard = cf::lock_acquire "map_find:$path";
+      my $map = new_from_path cf::map $path
+         or return;
 
-      $map->path ($key);
-      $map->{path} = $path;
       $map->{last_save} = $cf::RUNTIME;
-      $map->last_access ($cf::RUNTIME);
 
-      if ($map->should_reset) {
+      $map->load_header
+         or return;
+
+      if ($map->should_reset) {#d#TODO# disabled, crashy (locking issue?)
+         # doing this can freeze the server in a sync job, obviously
+         #$cf::WAIT_FOR_TICK->wait;
          $map->reset;
          undef $guard;
-         $map = cf::map::find_map ($path)
-            or return;
+         return find $path;
       }
 
-      $cf::MAP{$key} = $map
+      $cf::MAP{$path} = $map
    }
 }
 
+sub pre_load  { }
+sub post_load { }
+
 sub load {
    my ($self) = @_;
 
+   local $self->{deny_reset} = 1; # loading can take a long time
+
    my $path = $self->{path};
-   my $guard = cf::lock_acquire "map_load:" . $path->as_string;
 
-   return if $self->in_memory != cf::MAP_SWAPPED;
+   {
+      my $guard1 = cf::lock_acquire "map_data:$path";
+      my $guard2 = cf::lock_acquire "map_load:$path";
 
-   $self->in_memory (cf::MAP_LOADING);
+      return if $self->in_memory != cf::MAP_SWAPPED;
 
-   $self->alloc;
-   $self->load_objects ($self->{load_path}, 1)
-      or return;
+      $self->in_memory (cf::MAP_LOADING);
 
-   $self->set_object_flag (cf::FLAG_OBJ_ORIGINAL, 1)
-      if delete $self->{load_original};
+      $self->alloc;
 
-   if (my $uniq = $path->uniq_path) {
-      utf8::encode $uniq;
-      if (aio_open $uniq, O_RDONLY, 0) {
-         $self->clear_unique_items;
-         $self->load_objects ($uniq, 0);
-      }
-   }
+      $self->pre_load;
+      cf::cede_to_tick;
 
-   # now do the right thing for maps
-   $self->link_multipart_objects;
+      $self->_load_objects ($self->{load_path}, 1)
+         or return;
 
-   if ($self->{path}->is_style_map) {
-      $self->{deny_save}  = 1;
-      $self->{deny_reset} = 1;
-   } else {
-      $self->fix_auto_apply;
-      $self->decay_objects;
-      $self->update_buttons;
-      $self->set_darkness_map;
+      $self->set_object_flag (cf::FLAG_OBJ_ORIGINAL, 1)
+         if delete $self->{load_original};
+
+      if (my $uniq = $self->uniq_path) {
+         utf8::encode $uniq;
+         if (aio_open $uniq, O_RDONLY, 0) {
+            $self->clear_unique_items;
+            $self->_load_objects ($uniq, 0);
+         }
+      }
+
+      cf::cede_to_tick;
+      # now do the right thing for maps
+      $self->link_multipart_objects;
       $self->difficulty ($self->estimate_difficulty)
          unless $self->difficulty;
-      $self->activate;
+      cf::cede_to_tick;
+
+      unless ($self->{deny_activate}) {
+         $self->decay_objects;
+         $self->fix_auto_apply;
+         $self->update_buttons;
+         cf::cede_to_tick;
+         $self->set_darkness_map;
+         cf::cede_to_tick;
+         $self->activate;
+      }
+
+      $self->{last_save} = $cf::RUNTIME;
+      $self->last_access ($cf::RUNTIME);
+
+      $self->in_memory (cf::MAP_IN_MEMORY);
+   }
+
+   $self->post_load;
+}
+
+sub customise_for {
+   my ($self, $ob) = @_;
+
+   return find "~" . $ob->name . "/" . $self->{path}
+      if $self->per_player;
+
+#   return find "?party/" . $ob->name . "/" . $self->{path}
+#      if $self->per_party;
+
+   $self
+}
+
+# find and load all maps in the 3x3 area around a map
+sub load_neighbours {
+   my ($map) = @_;
+
+   my @neigh; # diagonal neighbours
+
+   for (0 .. 3) {
+      my $neigh = $map->tile_path ($_)
+         or next;
+      $neigh = find $neigh, $map
+         or next;
+      $neigh->load;
+
+      push @neigh,
+         [$neigh->tile_path (($_ + 3) % 4), $neigh],
+         [$neigh->tile_path (($_ + 1) % 4), $neigh];
    }
 
-   $self->in_memory (cf::MAP_IN_MEMORY);
+   for (grep defined $_->[0], @neigh) {
+      my ($path, $origin) = @$_;
+      my $neigh = find $path, $origin
+         or next;
+      $neigh->load;
+   }
 }
 
-sub load_map_sync {
+sub find_sync {
    my ($path, $origin) = @_;
 
-   #warn "load_map_sync<$path, $origin>\n";#d#
+   cf::sync_job { find $path, $origin }
+}
 
-   cf::sync_job {
-      my $map = cf::map::find_map $path, $origin
-         or return;
-      $map->load;
-      $map
+sub do_load_sync {
+   my ($map) = @_;
+
+   cf::LOG cf::llevDebug | cf::logBacktrace, "do_load_sync"
+      if $Coro::current == $Coro::main;
+
+   cf::sync_job { $map->load };
+}
+
+our %MAP_PREFETCH;
+our $MAP_PREFETCHER = undef;
+
+sub find_async {
+   my ($path, $origin, $load) = @_;
+
+   $path = normalise $path, $origin && $origin->{path};
+
+   if (my $map = $cf::MAP{$path}) {
+      return $map if !$load || $map->in_memory == cf::MAP_IN_MEMORY;
    }
+
+   $MAP_PREFETCH{$path} |= $load;
+
+   $MAP_PREFETCHER ||= cf::async {
+      while (%MAP_PREFETCH) {
+         while (my ($k, $v) = each %MAP_PREFETCH) {
+            if (my $map = find $k) {
+               $map->load if $v;
+            }
+
+            delete $MAP_PREFETCH{$k};
+         }
+      }
+      undef $MAP_PREFETCHER;
+   };
+   $MAP_PREFETCHER->prio (6);
+
+   ()
 }
 
 sub save {
    my ($self) = @_;
 
+   my $lock = cf::lock_acquire "map_data:$self->{path}";
+
    $self->{last_save} = $cf::RUNTIME;
 
    return unless $self->dirty;
 
-   my $save = $self->{path}->save_path; utf8::encode $save;
-   my $uniq = $self->{path}->uniq_path; utf8::encode $uniq;
+   my $save = $self->save_path; utf8::encode $save;
+   my $uniq = $self->uniq_path; utf8::encode $uniq;
 
    $self->{load_path} = $save;
 
    return if $self->{deny_save};
 
+   local $self->{last_access} = $self->last_access;#d#
+
+   cf::async {
+      $_->contr->save for $self->players;
+   };
+
    if ($uniq) {
-      $self->save_objects ($save, cf::IO_HEADER | cf::IO_OBJECTS);
-      $self->save_objects ($uniq, cf::IO_UNIQUES);
+      $self->_save_objects ($save, cf::IO_HEADER | cf::IO_OBJECTS);
+      $self->_save_objects ($uniq, cf::IO_UNIQUES);
    } else {
-      $self->save_objects ($save, cf::IO_HEADER | cf::IO_OBJECTS | cf::IO_UNIQUES);
+      $self->_save_objects ($save, cf::IO_HEADER | cf::IO_OBJECTS | cf::IO_UNIQUES);
    }
 }
 
 sub swap_out {
    my ($self) = @_;
 
+   # save first because save cedes
+   $self->save;
+
+   my $lock = cf::lock_acquire "map_data:$self->{path}";
+
    return if $self->players;
    return if $self->in_memory != cf::MAP_IN_MEMORY;
    return if $self->{deny_save};
 
-   $self->save;
    $self->clear;
    $self->in_memory (cf::MAP_SWAPPED);
 }
@@ -1325,7 +2037,6 @@
    my ($self) = @_;
 
    # TODO: safety, remove and allow resettable per-player maps
-   return 1e99 if $self->{path}{user_rel};
    return 1e99 if $self->{deny_reset};
 
    my $time = $self->fixed_resettime ? $self->{instantiate_time} : $self->last_access;
@@ -1340,37 +2051,29 @@
    $self->reset_at <= $cf::RUNTIME
 }
 
-sub unlink_save {
+sub reset {
    my ($self) = @_;
 
-   utf8::encode (my $save = $self->{path}->save_path);
-   aioreq_pri 3; IO::AIO::aio_unlink $save;
-   aioreq_pri 3; IO::AIO::aio_unlink "$save.pst";
-}
-
-sub rename {
-   my ($self, $new_path) = @_;
-
-   $self->unlink_save;
+   my $lock = cf::lock_acquire "map_data:$self->{path}";
 
-   delete $cf::MAP{$self->path};
-   $self->{path} = new cf::path $new_path;
-   $self->path ($self->{path}->as_string);
-   $cf::MAP{$self->path} = $self;
+   return if $self->players;
 
-   $self->save;
-}
+   warn "resetting map ", $self->path;
 
-sub reset {
-   my ($self) = @_;
+   $self->in_memory (cf::MAP_SWAPPED);
 
-   return if $self->players;
-   return if $self->{path}{user_rel};#d#
+   # need to save uniques path
+   unless ($self->{deny_save}) {
+      my $uniq = $self->uniq_path; utf8::encode $uniq;
 
-   warn "resetting map ", $self->path;#d#
+      $self->_save_objects ($uniq, cf::IO_UNIQUES)
+         if $uniq;
+   }
 
    delete $cf::MAP{$self->path};
 
+   $self->clear;
+
    $_->clear_links_to ($self) for values %cf::MAP;
 
    $self->unlink_save;
@@ -1382,39 +2085,105 @@
 sub nuke {
    my ($self) = @_;
 
+   delete $cf::MAP{$self->path};
+
+   $self->unlink_save;
+
+   bless $self, "cf::map";
+   delete $self->{deny_reset};
    $self->{deny_save} = 1;
    $self->reset_timeout (1);
-   $self->rename ("{nuke}/" . ($nuke_counter++));
+   $self->path ($self->{path} = "{nuke}/" . ($nuke_counter++));
+
+   $cf::MAP{$self->path} = $self;
+
    $self->reset; # polite request, might not happen
 }
 
-sub customise_for {
-   my ($map, $ob) = @_;
+=item $maps = cf::map::tmp_maps
 
-   if ($map->per_player) {
-      return cf::map::find_map "~" . $ob->name . "/" . $map->{path}{path};
-   }
+Returns an arrayref with all map paths of currently instantiated and saved
+maps. May block.
+
+=cut
 
-   $map
+sub tmp_maps() {
+   [
+      map {
+         utf8::decode $_;
+         /\.map$/
+            ? normalise $_
+            : ()
+      } @{ aio_readdir $TMPDIR or [] }
+   ]
 }
 
-sub emergency_save {
-   local $cf::FREEZE = 1;
+=item $maps = cf::map::random_maps
 
-   warn "enter emergency map save\n";
+Returns an arrayref with all map paths of currently instantiated and saved
+random maps. May block.
 
-   cf::sync_job {
-      warn "begin emergency map save\n";
-      $_->save for values %cf::MAP;
-   };
+=cut
+
+sub random_maps() {
+   [
+      map {
+         utf8::decode $_;
+         /\.map$/
+            ? normalise "?random/$_"
+            : ()
+      } @{ aio_readdir $RANDOMDIR or [] }
+   ]
+}
+
+=item cf::map::unique_maps
+
+Returns an arrayref of paths of all shared maps that have
+instantiated unique items. May block.
+
+=cut
 
-   warn "end emergency map save\n";
+sub unique_maps() {
+   [
+      map {
+         utf8::decode $_;
+         /\.map$/
+            ? normalise $_
+            : ()
+      } @{ aio_readdir $UNIQUEDIR or [] }
+   ]
 }
 
 package cf;
 
 =back
 
+=head3 cf::object
+
+=cut
+
+package cf::object;
+
+=over 4
+
+=item $ob->inv_recursive
+
+Returns the inventory of the object _and_ their inventories, recursively.
+
+=cut
+
+sub inv_recursive_;
+sub inv_recursive_ {
+   map { $_, inv_recursive_ $_->inv } @_
+}
+
+sub inv_recursive {
+   inv_recursive_ inv $_[0]
+}
+
+package cf;
+
+=back
 
 =head3 cf::object::player
 
@@ -1438,12 +2207,31 @@
 
    if ($self->{record_replies}) {
       push @{ $self->{record_replies} }, [$npc, $msg, $flags];
+
    } else {
-      $msg = $npc->name . " says: $msg" if $npc;
-      $self->message ($msg, $flags);
+      my $pl = $self->contr;
+
+      if ($pl->{npc_dialog} && $pl->{npc_dialog}->{id}) {
+         my $dialog = $pl->{npc_dialog};
+         $dialog->{pl}->ext_msg ($dialog->{id}, update => msg => $dialog->{pl}->expand_cfpod ($msg));
+
+      } else {
+         $msg = $npc->name . " says: $msg" if $npc;
+         $self->message ($msg, $flags);
+      }
    }
 }
 
+=item $object->send_msg ($channel, $msg, $color, [extra...])
+
+=cut
+
+sub cf::object::send_msg {
+   my $pl = shift->contr
+      or return;
+   $pl->send_msg (@_);
+}
+
 =item $player_object->may ("access")
 
 Returns wether the given player is authorized to access resource "access"
@@ -1464,46 +2252,60 @@
 
 Freezes the player and moves him/her to a special map (C<{link}>).
 
-The player should be reaosnably safe there for short amounts of time. You
+The player should be reasonably safe there for short amounts of time. You
 I<MUST> call C<leave_link> as soon as possible, though.
 
+Will never block.
+
 =item $player_object->leave_link ($map, $x, $y)
 
-Moves the player out of the specila link map onto the given map. If the
-map is not valid (or omitted), the player will be moved back to the
-location he/she was before the call to C<enter_link>, or, if that fails,
-to the emergency map position.
+Moves the player out of the special C<{link}> map onto the specified
+map. If the map is not valid (or omitted), the player will be moved back
+to the location he/she was before the call to C<enter_link>, or, if that
+fails, to the emergency map position.
 
 Might block.
 
 =cut
 
+sub link_map {
+   unless ($LINK_MAP) {
+      $LINK_MAP = cf::map::find "{link}"
+         or cf::cleanup "FATAL: unable to provide {link} map, exiting.";
+      $LINK_MAP->load;
+   }
+
+   $LINK_MAP
+}
+
 sub cf::object::player::enter_link {
    my ($self) = @_;
 
    $self->deactivate_recursive;
 
-   return if $self->map == $LINK_MAP;
+   return if UNIVERSAL::isa $self->map, "ext::map_link";
 
    $self->{_link_pos} ||= [$self->map->{path}, $self->x, $self->y]
-      if $self->map;
+      if $self->map && $self->map->{path} ne "{link}";
 
-   $self->enter_map ($LINK_MAP, 20, 20);
+   $self->enter_map ($LINK_MAP || link_map, 10, 10);
 }
 
 sub cf::object::player::leave_link {
    my ($self, $map, $x, $y) = @_;
 
+   return unless $self->contr->active;
+
    my $link_pos = delete $self->{_link_pos};
 
    unless ($map) {
       # restore original map position
       ($map, $x, $y) = @{ $link_pos || [] };
-      $map = cf::map::find_map $map;
+      $map = cf::map::find $map;
 
       unless ($map) {
          ($map, $x, $y) = @$EMERGENCY_POSITION;
-         $map = cf::map::find_map $map
+         $map = cf::map::find $map
             or die "FATAL: cannot load emergency map\n";
       }
    }
@@ -1516,60 +2318,60 @@
       if $x <=0 && $y <= 0;
 
    $map->load;
+   $map->load_neighbours;
 
+   return unless $self->contr->active;
    $self->activate_recursive;
+
+   local $self->{_prev_pos} = $link_pos; # ugly hack for rent.ext
    $self->enter_map ($map, $x, $y);
 }
 
-cf::player->attach (
-   on_logout => sub {
-      my ($pl) = @_;
+=item $player_object->goto ($path, $x, $y[, $check->($map)[, $done->()]])
 
-      # abort map switching before logout
-      if ($pl->ob->{_link_pos}) {
-         cf::sync_job {
-            $pl->ob->leave_link
-         };
-      }
-   },
-   on_login => sub {
-      my ($pl) = @_;
+Moves the player to the given map-path and coordinates by first freezing
+her, loading and preparing them map, calling the provided $check callback
+that has to return the map if sucecssful, and then unfreezes the player on
+the new (success) or old (failed) map position. In either case, $done will
+be called at the end of this process.
 
-      # try to abort aborted map switching on player login :)
-      # should happen only on crashes
-      if ($pl->ob->{_link_pos}) {
-         $pl->ob->enter_link;
-         Coro::async {
-            # we need this sleep as the login has a concurrent enter_exit running
-            # and this sleep increases chances of the player not ending up in scorn
-            Coro::Timer::sleep 1;
-            $pl->ob->leave_link;
-         };
-      }
-   },
-);
+=cut
 
-=item $player_object->goto_map ($path, $x, $y)
+our $GOTOGEN;
 
-=cut
+sub cf::object::player::goto {
+   my ($self, $path, $x, $y, $check, $done) = @_;
 
-sub cf::object::player::goto_map {
-   my ($self, $path, $x, $y) = @_;
+   # do generation counting so two concurrent goto's will be executed in-order
+   my $gen = $self->{_goto_generation} = ++$GOTOGEN;
 
    $self->enter_link;
 
-   (Coro::async {
-      $path = new cf::path $path;
+   (async {
+      my $map = eval {
+         my $map = cf::map::find $path;
+
+         if ($map) {
+            $map = $map->customise_for ($self);
+            $map = $check->($map) if $check && $map;
+         } else {
+            $self->message ("The exit to '$path' is closed", cf::NDI_UNIQUE | cf::NDI_RED);
+         }
 
-      my $map = cf::map::find_map $path->as_string;
-      $map = $map->customise_for ($self) if $map;
+         $map
+      };
 
-#      warn "entering ", $map->path, " at ($x, $y)\n"
-#         if $map;
+      if ($@) {
+         $self->message ("Something went wrong within the server, please report this incident!", cf::NDI_UNIQUE | cf::NDI_RED);
+         LOG llevError | logBacktrace, Carp::longmess $@;
+      }
 
-      $map or $self->message ("The exit is closed", cf::NDI_UNIQUE | cf::NDI_RED);
+      if ($gen == $self->{_goto_generation}) {
+         delete $self->{_goto_generation};
+         $self->leave_link ($map, $x, $y);
+      }
 
-      $self->leave_link ($map, $x, $y);
+      $done->() if $done;
    })->prio (1);
 }
 
@@ -1581,8 +2383,10 @@
    my ($spec) = @_;
 
    my $rmp = { # defaults
-      xsize => 10,
-      ysize => 10,
+      xsize    => (cf::rndm 15, 40),
+      ysize    => (cf::rndm 15, 40),
+      symmetry => (cf::rndm 1, cf::SYMMETRY_XY),
+      #layout   => string,
    };
 
    for (split /\n/, $spec) {
@@ -1597,6 +2401,8 @@
 sub prepare_random_map {
    my ($exit) = @_;
 
+   my $guard = cf::lock_acquire "exit_prepare:$exit";
+
    # all this does is basically replace the /! path by
    # a new random map path (?random/...) with a seed
    # that depends on the exit object
@@ -1604,7 +2410,7 @@
    my $rmp = parse_random_map_params $exit->msg;
 
    if ($exit->map) {
-      $rmp->{region}     = $exit->map->region_name;
+      $rmp->{region}     = $exit->region->name;
       $rmp->{origin_map} = $exit->map->path;
       $rmp->{origin_x}   = $exit->x;
       $rmp->{origin_y}   = $exit->y;
@@ -1614,9 +2420,12 @@
 
    my $data = cf::to_json $rmp;
    my $md5 = Digest::MD5::md5_hex $data;
+   my $meta = "$RANDOMDIR/$md5.meta";
 
-   if (my $fh = aio_open "$cf::RANDOM_MAPS/$md5.meta", O_WRONLY | O_CREAT, 0666) {
+   if (my $fh = aio_open "$meta~", O_WRONLY | O_CREAT, 0666) {
       aio_write $fh, 0, (length $data), $data, 0;
+      undef $fh;
+      aio_rename "$meta~", $meta;
 
       $exit->slaying ("?random/$md5");
       $exit->msg (undef);
@@ -1628,23 +2437,34 @@
 
    return unless $self->type == cf::PLAYER;
 
-   $self->enter_link;
+   if ($exit->slaying eq "/!") {
+      #TODO: this should de-fi-ni-te-ly not be a sync-job
+      # the problem is that $exit might not survive long enough
+      # so it needs to be done right now, right here
+      cf::sync_job { prepare_random_map $exit };
+   }
 
-   (Coro::async {
-      unless (eval {
+   my $slaying = cf::map::normalise $exit->slaying, $exit->map && $exit->map->path;
+   my $hp      = $exit->stats->hp;
+   my $sp      = $exit->stats->sp;
 
-         prepare_random_map $exit
-            if $exit->slaying eq "/!";
+   $self->enter_link;
 
-         my $path = new cf::path $exit->slaying, $exit->map && $exit->map->path;
-         $self->goto_map ($path, $exit->stats->hp, $exit->stats->sp);
+   # if exit is damned, update players death & WoR home-position
+   $self->contr->savebed ($slaying, $hp, $sp)
+      if $exit->flag (FLAG_DAMNED);
+
+   (async {
+      $self->deactivate_recursive; # just to be sure
+      unless (eval {
+         $self->goto ($slaying, $hp, $sp);
 
          1;
       }) {
          $self->message ("Something went wrong deep within the crossfire server. "
-                     . "I'll try to bring you back to the map you were before. "
-                     . "Please report this to the dungeon master",
-                       cf::NDI_UNIQUE | cf::NDI_RED);
+                       . "I'll try to bring you back to the map you were before. "
+                       . "Please report this to the dungeon master!",
+                         cf::NDI_UNIQUE | cf::NDI_RED);
 
          warn "ERROR in enter_exit: $@";
          $self->leave_link;
@@ -1667,9 +2487,105 @@
    my ($self, $text, $flags) = @_;
 
    utf8::encode $text;
-   $self->send_packet (sprintf "drawinfo %d %s", $flags, $text);
+   $self->send_packet (sprintf "drawinfo %d %s", $flags || cf::NDI_BLACK, $text);
+}
+
+=item $client->send_msg ($channel, $msg, $color, [extra...])
+
+Send a drawinfo or msg packet to the client, formatting the msg for the
+client if neccessary. C<$type> should be a string identifying the type of
+the message, with C<log> being the default. If C<$color> is negative, suppress
+the message unless the client supports the msg packet.
+
+=cut
+
+sub cf::client::send_msg {
+   my ($self, $channel, $msg, $color, @extra) = @_;
+
+   $msg = $self->pl->expand_cfpod ($msg);
+
+   $color &= cf::NDI_CLIENT_MASK; # just in case...
+
+   if (ref $channel) {
+      # send meta info to client, if not yet sent
+      unless (exists $self->{channel}{$channel->{id}}) {
+         $self->{channel}{$channel->{id}} = $channel;
+         $self->ext_msg (channel_info => $channel);
+      }
+
+      $channel = $channel->{id};
+   }
+
+   return unless @extra || length $msg;
+
+   if ($self->can_msg) {
+      # default colour, mask it out
+      $color &= ~(cf::NDI_COLOR_MASK | cf::NDI_DEF)
+         if $color & cf::NDI_DEF;
+
+      $self->send_packet ("msg " . $self->{json_coder}->encode (
+         [$color & cf::NDI_CLIENT_MASK, $channel, $msg, @extra]));
+   } else {
+      if ($color >= 0) {
+         # replace some tags by gcfclient-compatible ones
+         for ($msg) {
+            1 while
+               s/<b>([^<]*)<\/b>/[b]${1}[\/b]/
+               || s/<i>([^<]*)<\/i>/[i]${1}[\/i]/
+               || s/<u>([^<]*)<\/u>/[ul]${1}[\/ul]/
+               || s/<tt>([^<]*)<\/tt>/[fixed]${1}[\/fixed]/
+               || s/<fg name=\"([^"]+)\">([^<]*)<\/fg>/[color=$1]${2}[\/color]/;
+         }
+
+         $color &= cf::NDI_COLOR_MASK;
+
+         utf8::encode $msg;
+
+         if (0 && $msg =~ /\[/) {
+            # COMMAND/INFO
+            $self->send_packet ("drawextinfo $color 10 8 $msg")
+         } else {
+            $msg =~ s/\[\/?(?:b|i|u|fixed|color)[^\]]*\]//g;
+            $self->send_packet ("drawinfo $color $msg")
+         }
+      }
+   }
 }
 
+=item $client->ext_msg ($type, @msg)
+
+Sends an ext event to the client.
+
+=cut
+
+sub cf::client::ext_msg($$@) {
+   my ($self, $type, @msg) = @_;
+
+   if ($self->extcmd == 2) {
+      $self->send_packet ("ext " . $self->{json_coder}->encode ([$type, @msg]));
+   } elsif ($self->extcmd == 1) { # TODO: remove
+      push @msg, msgtype => "event_$type";
+      $self->send_packet ("ext " . $self->{json_coder}->encode ({@msg}));
+   }
+}
+
+=item $client->ext_reply ($msgid, @msg)
+
+Sends an ext reply to the client.
+
+=cut
+
+sub cf::client::ext_reply($$@) {
+   my ($self, $id, @msg) = @_;
+
+   if ($self->extcmd == 2) {
+      $self->send_packet ("ext " . $self->{json_coder}->encode (["reply-$id", @msg]));
+   } elsif ($self->extcmd == 1) {
+      #TODO: version 1, remove
+      unshift @msg, msgtype => "reply", msgid => $id;
+      $self->send_packet ("ext " . $self->{json_coder}->encode ({@msg}));
+   }
+}
 
 =item $success = $client->query ($flags, "text", \&cb)
 
@@ -1677,8 +2593,8 @@
 the reply text on a reply. flags can be C<cf::CS_QUERY_YESNO>,
 C<cf::CS_QUERY_SINGLECHAR> or C<cf::CS_QUERY_HIDEINPUT> or C<0>.
 
-Queries can fail, so check the return code. Or don't, as queries will become
-reliable at some point in the future.
+Queries can fail, so check the return code. Or don't, as queries will
+become reliable at some point in the future.
 
 =cut
 
@@ -1696,17 +2612,26 @@
 
    $self->send_packet ($self->{query_queue}[0][0])
       if @{ $self->{query_queue} } == 1;
+
+   1
 }
 
 cf::client->attach (
+   on_connect => sub {
+      my ($ns) = @_;
+
+      $ns->{json_coder} = JSON::XS->new->utf8->max_size (1e6)->convert_blessed;
+   },
    on_reply => sub {
       my ($ns, $msg) = @_;
 
       # this weird shuffling is so that direct followup queries
       # get handled first
-      my $queue = delete $ns->{query_queue};
+      my $queue = delete $ns->{query_queue}
+         or return; # be conservative, not sure how that can happen, but we saw a crash here
 
       (shift @$queue)->[1]->($msg);
+      return unless $ns->valid; # temporary(?) workaround for callback destroying socket
 
       push @{ $ns->{query_queue} }, @$queue;
 
@@ -1718,9 +2643,35 @@
          }
       }
    },
+   on_exticmd => sub {
+      my ($ns, $buf) = @_;
+
+      my $msg = eval { $ns->{json_coder}->decode ($buf) };
+
+      if (ref $msg) {
+         my ($type, $reply, @payload) =
+            "ARRAY" eq ref $msg
+               ? @$msg
+               : ($msg->{msgtype}, $msg->{msgid}, %$msg); # TODO: version 1, remove
+
+         my @reply;
+
+         if (my $cb = $EXTICMD{$type}) {
+            @reply = $cb->($ns, @payload);
+         }
+
+         $ns->ext_reply ($reply, @reply)
+            if $reply;
+
+      } else {
+         warn "client " . ($ns->pl ? $ns->pl->ob->name : $ns->host) . " sent unparseable exti message: <$buf>\n";
+      }
+
+      cf::override;
+   },
 );
 
-=item $client->coro (\&cb)
+=item $client->async (\&cb)
 
 Create a new coroutine, running the specified callback. The coroutine will
 be automatically cancelled when the client gets destroyed (e.g. on logout,
@@ -1728,15 +2679,10 @@
 
 =cut
 
-sub cf::client::coro {
+sub cf::client::async {
    my ($self, $cb) = @_;
 
-   my $coro; $coro = async {
-      eval {
-         $cb->();
-      };
-      warn $@ if $@;
-   };
+   my $coro = &Coro::async ($cb);
 
    $coro->on_destroy (sub {
       delete $self->{_coro}{$coro+0};
@@ -1773,23 +2719,37 @@
 
 $SIG{FPE} = 'IGNORE';
 
-$safe->permit_only (Opcode::opset qw(:base_core :base_mem :base_orig :base_math sort time));
+$safe->permit_only (Opcode::opset qw(
+   :base_core :base_mem :base_orig :base_math
+   grepstart grepwhile mapstart mapwhile
+   sort time
+));
 
 # here we export the classes and methods available to script code
 
 =pod
 
-The following fucntions and emthods are available within a safe environment:
+The following functions and methods are available within a safe environment:
+
+  cf::object
+     contr pay_amount pay_player map x y force_find force_add
+     insert remove name archname title slaying race decrease_ob_nr
+
+  cf::object::player
+     player
+
+  cf::player
+     peaceful
 
-  cf::object contr pay_amount pay_player map
-  cf::object::player player
-  cf::player peaceful
-  cf::map trigger
+  cf::map
+     trigger
 
 =cut
 
 for (
-   ["cf::object"         => qw(contr pay_amount pay_player map)],
+   ["cf::object"         => qw(contr pay_amount pay_player map force_find force_add x y
+                               insert remove inv name archname title slaying race
+                               decrease_ob_nr)],
    ["cf::object::player" => qw(player)],
    ["cf::player"         => qw(peaceful)],
    ["cf::map"            => qw(trigger)],
@@ -1870,172 +2830,378 @@
 =cut
 
 #############################################################################
+# the server's init and main functions
 
-=head2 EXTENSION DATABASE SUPPORT
+sub load_facedata($) {
+   my ($path) = @_;
 
-Crossfire maintains a very simple database for extension use. It can
-currently store anything that can be serialised using Storable, which
-excludes objects.
+   my $enc = JSON::XS->new->utf8->canonical->relaxed;
 
-The parameter C<$family> should best start with the name of the extension
-using it, it should be unique.
+   warn "loading facedata from $path\n";
 
-=over 4
+   my $facedata;
+   0 < aio_load $path, $facedata
+      or die "$path: $!";
 
-=item $hashref = cf::db_get $family
+   $facedata = Coro::Storable::thaw $facedata;
 
-Return a hashref for use by the extension C<$family>, which can be
-modified. After modifications, you have to call C<cf::db_dirty> or
-C<cf::db_sync>.
+   $facedata->{version} == 2
+      or cf::cleanup "$path: version mismatch, cannot proceed.";
 
-=item $value = cf::db_get $family => $key
+   # patch in the exptable
+   $facedata->{resource}{"res/exp_table"} = {
+      type => FT_RSRC,
+      data => $enc->encode ([map cf::level_to_min_exp $_, 1 .. cf::settings->max_level]),
+   };
+   cf::cede_to_tick;
+
+   {
+      my $faces = $facedata->{faceinfo};
+
+      while (my ($face, $info) = each %$faces) {
+         my $idx = (cf::face::find $face) || cf::face::alloc $face;
+         cf::face::set_visibility $idx, $info->{visibility};
+         cf::face::set_magicmap   $idx, $info->{magicmap};
+         cf::face::set_data $idx, 0, $info->{data32}, Digest::MD5::md5 $info->{data32};
+         cf::face::set_data $idx, 1, $info->{data64}, Digest::MD5::md5 $info->{data64};
+
+         cf::cede_to_tick;
+      }
+
+      while (my ($face, $info) = each %$faces) {
+         next unless $info->{smooth};
+         my $idx = cf::face::find $face
+            or next;
+         if (my $smooth = cf::face::find $info->{smooth}) {
+            cf::face::set_smooth      $idx, $smooth;
+            cf::face::set_smoothlevel $idx, $info->{smoothlevel};
+         } else {
+            warn "smooth face '$info->{smooth}' not found for face '$face'";
+         }
 
-Returns a single value from the database
+         cf::cede_to_tick;
+      }
+   }
 
-=item cf::db_put $family => $hashref
+   {
+      my $anims = $facedata->{animinfo};
 
-Stores the given family hashref into the database. Updates are delayed, if
-you want the data to be synced to disk immediately, use C<cf::db_sync>.
+      while (my ($anim, $info) = each %$anims) {
+         cf::anim::set $anim, $info->{frames}, $info->{facings};
+         cf::cede_to_tick;
+      }
 
-=item cf::db_put $family => $key => $value
+      cf::anim::invalidate_all; # d'oh
+   }
 
-Stores the given C<$value> in the family hash. Updates are delayed, if you
-want the data to be synced to disk immediately, use C<cf::db_sync>.
+   {
+      # TODO: for gcfclient pleasure, we should give resources
+      # that gcfclient doesn't grok a >10000 face index.
+      my $res = $facedata->{resource};
 
-=item cf::db_dirty
+      my $soundconf = delete $res->{"res/sound.conf"};
 
-Marks the database as dirty, to be updated at a later time.
+      while (my ($name, $info) = each %$res) {
+         my $idx = (cf::face::find $name) || cf::face::alloc $name;
+         my $data;
 
-=item cf::db_sync
+         if ($info->{type} & 1) {
+            # prepend meta info
 
-Immediately write the database to disk I<if it is dirty>.
+            my $meta = $enc->encode ({
+               name => $name,
+               %{ $info->{meta} || {} },
+            });
 
-=cut
+            $data = pack "(w/a*)*", $meta, $info->{data};
+         } else {
+            $data = $info->{data};
+         }
 
-our $DB;
+         cf::face::set_data $idx, 0, $data, Digest::MD5::md5 $data;
+         cf::face::set_type $idx, $info->{type};
 
-{
-   my $path = cf::localdir . "/database.pst";
+         cf::cede_to_tick;
+      }
 
-   sub db_load() {
-      $DB = stat $path ? Storable::retrieve $path : { };
-   }
+      if ($soundconf) {
+         $soundconf = $enc->decode (delete $soundconf->{data});
 
-   my $pid;
+         for (0 .. SOUND_CAST_SPELL_0 - 1) {
+            my $sound = $soundconf->{compat}[$_]
+               or next;
 
-   sub db_save() {
-      waitpid $pid, 0 if $pid;
-      if (0 == ($pid = fork)) {
-         $DB->{_meta}{version} = 1;
-         Storable::nstore $DB, "$path~";
-         rename "$path~", $path;
-         cf::_exit 0 if defined $pid;
+            my $face = cf::face::find "sound/$sound->[1]";
+            cf::sound::set $sound->[0] => $face;
+            cf::sound::old_sound_index $_, $face; # gcfclient-compat
+         }
+
+         while (my ($k, $v) = each %{$soundconf->{event}}) {
+            my $face = cf::face::find "sound/$v";
+            cf::sound::set $k => $face;
+         }
       }
    }
 
-   my $dirty;
+   1
+}
 
-   sub db_sync() {
-      db_save if $dirty;
-      undef $dirty;
+register_exticmd fx_want => sub {
+   my ($ns, $want) = @_;
+
+   while (my ($k, $v) = each %$want) {
+      $ns->fx_want ($k, $v);
    }
+};
 
-   my $idle = Event->idle (min => $TICK * 2.8, max => 10, repeat => 0, data => WF_AUTOCANCEL, cb => sub {
-      db_sync;
-   });
+sub reload_regions {
+   load_resource_file "$MAPDIR/regions"
+      or die "unable to load regions file\n";
 
-   sub db_dirty() {
-      $dirty = 1;
-      $idle->start;
+   for (cf::region::list) {
+      $_->{match} = qr/$_->{match}/
+         if exists $_->{match};
    }
+}
 
-   sub db_get($;$) {
-      @_ >= 2
-         ? $DB->{$_[0]}{$_[1]}
-         : ($DB->{$_[0]} ||= { })
-   }
+sub reload_facedata {
+   load_facedata "$DATADIR/facedata"
+      or die "unable to load facedata\n";
+}
 
-   sub db_put($$;$) {
-      if (@_ >= 3) {
-         $DB->{$_[0]}{$_[1]} = $_[2];
-      } else {
-         $DB->{$_[0]} = $_[1];
-      }
-      db_dirty;
-   }
+sub reload_archetypes {
+   load_resource_file "$DATADIR/archetypes"
+      or die "unable to load archetypes\n";
+   #d# NEED to laod twice to resolve forward references
+   # this really needs to be done in an extra post-pass
+   # (which needs to be synchronous, so solve it differently)
+   load_resource_file "$DATADIR/archetypes"
+      or die "unable to load archetypes\n";
+}
 
-   cf::global->attach (
-      prio       => 10000,
-      on_cleanup => sub {
-         db_sync;
-      },
-   );
+sub reload_treasures {
+   load_resource_file "$DATADIR/treasures"
+      or die "unable to load treasurelists\n";
 }
 
-#############################################################################
-# the server's main()
+sub reload_resources {
+   warn "reloading resource files...\n";
+
+   reload_regions;
+   reload_facedata;
+   #reload_archetypes;#d#
+   reload_archetypes;
+   reload_treasures;
+
+   warn "finished reloading resource files\n";
+}
 
-sub cfg_load {
-   open my $fh, "<:utf8", cf::confdir . "/config"
+sub init {
+   reload_resources;
+}
+
+sub reload_config {
+   open my $fh, "<:utf8", "$CONFDIR/config"
       or return;
 
    local $/;
    *CFG = YAML::Syck::Load <$fh>;
+
+   $EMERGENCY_POSITION = $CFG{emergency_position} || ["/world/world_105_115", 5, 37];
+
+   $cf::map::MAX_RESET     = $CFG{map_max_reset}     if exists $CFG{map_max_reset};
+   $cf::map::DEFAULT_RESET = $CFG{map_default_reset} if exists $CFG{map_default_reset};
+
+   if (exists $CFG{mlockall}) {
+      eval {
+         $CFG{mlockall} ? eval "mlockall()" : eval "munlockall()"
+            and die "WARNING: m(un)lockall failed: $!\n";
+      };
+      warn $@ if $@;
+   }
 }
 
 sub main {
    # we must not ever block the main coroutine
    local $Coro::idle = sub {
       Carp::cluck "FATAL: Coro::idle was called, major BUG, use cf::sync_job!\n";#d#
-      (Coro::unblock_sub {
+      (async {
          Event::one_event;
-      })->();
+      })->prio (Coro::PRIO_MAX);
    };
 
-   cfg_load;
-   db_load;
+   reload_config;
+   db_init;
    load_extensions;
+
+   $TICK_WATCHER->start;
    Event::loop;
 }
 
 #############################################################################
-# initialisation
+# initialisation and cleanup
+
+# install some emergency cleanup handlers
+BEGIN {
+   for my $signal (qw(INT HUP TERM)) {
+      Event->signal (
+         reentrant => 0,
+         data      => WF_AUTOCANCEL,
+         signal    => $signal,
+         prio      => 0,
+         cb        => sub {
+            cf::cleanup "SIG$signal";
+         },
+      );
+   }
+}
+
+sub write_runtime {
+   my $runtime = "$LOCALDIR/runtime";
+
+   # first touch the runtime file to show we are still running:
+   # the fsync below can take a very very long time.
+
+   IO::AIO::aio_utime $runtime, undef, undef;
 
-sub reload() {
+   my $guard = cf::lock_acquire "write_runtime";
+
+   my $fh = aio_open "$runtime~", O_WRONLY | O_CREAT, 0644
+      or return;
+
+   my $value = $cf::RUNTIME + 90 + 10;
+   # 10 is the runtime save interval, for a monotonic clock
+   # 60 allows for the watchdog to kill the server.
+
+   (aio_write $fh, 0, (length $value), $value, 0) <= 0
+      and return;
+
+   # always fsync - this file is important
+   aio_fsync $fh
+      and return;
+
+   # touch it again to show we are up-to-date
+   aio_utime $fh, undef, undef;
+
+   close $fh
+      or return;
+
+   aio_rename "$runtime~", $runtime
+      and return;
+
+   warn "runtime file written.\n";
+
+   1
+}
+
+sub emergency_save() {
+   my $freeze_guard = cf::freeze_mainloop;
+
+   warn "enter emergency perl save\n";
+
+   cf::sync_job {
+      # use a peculiar iteration method to avoid tripping on perl
+      # refcount bugs in for. also avoids problems with players
+      # and maps saved/destroyed asynchronously.
+      warn "begin emergency player save\n";
+      for my $login (keys %cf::PLAYER) {
+         my $pl = $cf::PLAYER{$login} or next;
+         $pl->valid or next;
+         $pl->save;
+      }
+      warn "end emergency player save\n";
+
+      warn "begin emergency map save\n";
+      for my $path (keys %cf::MAP) {
+         my $map = $cf::MAP{$path} or next;
+         $map->valid or next;
+         $map->save;
+      }
+      warn "end emergency map save\n";
+
+      warn "begin emergency database checkpoint\n";
+      BDB::db_env_txn_checkpoint $DB_ENV;
+      warn "end emergency database checkpoint\n";
+   };
+
+   warn "leave emergency perl save\n";
+}
+
+sub post_cleanup {
+   my ($make_core) = @_;
+
+   warn Carp::longmess "post_cleanup backtrace"
+      if $make_core;
+}
+
+sub do_reload_perl() {
    # can/must only be called in main
    if ($Coro::current != $Coro::main) {
-      warn "can only reload from main coroutine\n";
+      warn "can only reload from main coroutine";
       return;
    }
 
    warn "reloading...";
 
-   local $FREEZE = 1;
-   cf::emergency_save;
+   warn "entering sync_job";
+
+   cf::sync_job {
+      cf::write_runtime; # external watchdog should not bark
+      cf::emergency_save;
+      cf::write_runtime; # external watchdog should not bark
+
+      warn "syncing database to disk";
+      BDB::db_env_txn_checkpoint $DB_ENV;
 
-   eval {
       # if anything goes wrong in here, we should simply crash as we already saved
 
-      # cancel all watchers
+      warn "cancelling all WF_AUTOCANCEL watchers";
       for (Event::all_watchers) {
          $_->cancel if $_->data & WF_AUTOCANCEL;
       }
 
-      # cancel all extension coros
+      warn "flushing outstanding aio requests";
+      for (;;) {
+         BDB::flush;
+         IO::AIO::flush;
+         Coro::cede;
+         last unless IO::AIO::nreqs || BDB::nreqs;
+         warn "iterate...";
+      }
+
+      ++$RELOAD;
+
+      warn "cancelling all extension coros";
       $_->cancel for values %EXT_CORO;
       %EXT_CORO = ();
 
-      # unload all extensions
-      for (@exts) {
-         warn "unloading <$_>";
-         unload_extension $_;
+      warn "removing commands";
+      %COMMAND = ();
+
+      warn "removing ext/exti commands";
+      %EXTCMD  = ();
+      %EXTICMD = ();
+
+      warn "unloading/nuking all extensions";
+      for my $pkg (@EXTS) {
+         warn "... unloading $pkg";
+
+         if (my $cb = $pkg->can ("unload")) {
+            eval {
+               $cb->($pkg);
+               1
+            } or warn "$pkg unloaded, but with errors: $@";
+         }
+
+         warn "... nuking $pkg";
+         Symbol::delete_package $pkg;
       }
 
-      # unload all modules loaded from $LIBDIR
+      warn "unloading all perl modules loaded from $LIBDIR";
       while (my ($k, $v) = each %INC) {
          next unless $v =~ /^\Q$LIBDIR\E\/.*\.pm$/;
 
-         warn "removing <$k>";
+         warn "... unloading $k";
          delete $INC{$k};
 
          $k =~ s/\.pm$//;
@@ -2048,150 +3214,287 @@
          Symbol::delete_package $k;
       }
 
-      # sync database to disk
-      cf::db_sync;
-      IO::AIO::flush;
-
-      # get rid of safe::, as good as possible
+      warn "getting rid of safe::, as good as possible";
       Symbol::delete_package "safe::$_"
          for qw(cf::attachable cf::object cf::object::player cf::client cf::player cf::map cf::party cf::region);
 
-      # remove register_script_function callbacks
-      # TODO
-      
-      # unload cf.pm "a bit"
+      warn "unloading cf.pm \"a bit\"";
       delete $INC{"cf.pm"};
+      delete $INC{"cf/pod.pm"};
 
       # don't, removes xs symbols, too,
       # and global variables created in xs
       #Symbol::delete_package __PACKAGE__;
 
-      # reload cf.pm
+      warn "unload completed, starting to reload now";
+
       warn "reloading cf.pm";
       require cf;
       cf::_connect_to_perl; # nominally unnecessary, but cannot hurt
 
-      # load config and database again
-      cf::cfg_load;
-      cf::db_load;
+      warn "loading config and database again";
+      cf::reload_config;
 
-      # load extensions
-      warn "load extensions";
+      warn "loading extensions";
       cf::load_extensions;
 
-      # reattach attachments to objects
-      warn "reattach";
-      _global_reattach;
-   };
+      warn "reattaching attachments to objects/players";
+      _global_reattach; # objects, sockets
+      warn "reattaching attachments to maps";
+      reattach $_ for values %MAP;
+      warn "reattaching attachments to players";
+      reattach $_ for values %PLAYER;
 
-   if ($@) {
+      warn "leaving sync_job";
+
+      1
+   } or do {
       warn $@;
       warn "error while reloading, exiting.";
       exit 1;
-   }
+   };
 
-   warn "reloaded successfully";
+   warn "reloaded";
 };
 
-#############################################################################
-
-unless ($LINK_MAP) {
-   $LINK_MAP = cf::map::new;
-
-   $LINK_MAP->width  (41);
-   $LINK_MAP->height (41);
-   $LINK_MAP->alloc;
-   $LINK_MAP->path ("{link}");
-   $LINK_MAP->{path} = bless { path => "{link}" }, "cf::path";
-   $LINK_MAP->in_memory (MAP_IN_MEMORY);
-
-   # dirty hack because... archetypes are not yet loaded
-   Event->timer (
-      after => 2,
-      cb    => sub {
-         $_[0]->w->cancel;
-
-         # provide some exits "home"
-         my $exit = cf::object::new "exit";
-
-         $exit->slaying   ($EMERGENCY_POSITION->[0]);
-         $exit->stats->hp ($EMERGENCY_POSITION->[1]);
-         $exit->stats->sp ($EMERGENCY_POSITION->[2]);
-
-         $LINK_MAP->insert ($exit->clone, 19, 19);
-         $LINK_MAP->insert ($exit->clone, 19, 20);
-         $LINK_MAP->insert ($exit->clone, 19, 21);
-         $LINK_MAP->insert ($exit->clone, 20, 19);
-         $LINK_MAP->insert ($exit->clone, 20, 21);
-         $LINK_MAP->insert ($exit->clone, 21, 19);
-         $LINK_MAP->insert ($exit->clone, 21, 20);
-         $LINK_MAP->insert ($exit->clone, 21, 21);
-
-         $exit->destroy;
-      });
+our $RELOAD_WATCHER; # used only during reload
 
-   $LINK_MAP->{deny_save}  = 1;
-   $LINK_MAP->{deny_reset} = 1;
-
-   $cf::MAP{$LINK_MAP->path} = $LINK_MAP;
+sub reload_perl() {
+   # doing reload synchronously and two reloads happen back-to-back,
+   # coro crashes during coro_state_free->destroy here.
+
+   $RELOAD_WATCHER ||= Event->timer (
+      reentrant => 0,
+      after     => 0,
+      data      => WF_AUTOCANCEL,
+      cb => sub {
+         do_reload_perl;
+         undef $RELOAD_WATCHER;
+      },
+   );
 }
 
-register "<global>", __PACKAGE__;
-
 register_command "reload" => sub {
    my ($who, $arg) = @_;
 
    if ($who->flag (FLAG_WIZ)) {
-      $who->message ("start of reload.");
-      reload;
-      $who->message ("end of reload.");
+      $who->message ("reloading server.");
+      async { reload_perl };
    }
 };
 
 unshift @INC, $LIBDIR;
 
+my $bug_warning = 0;
+
+our @WAIT_FOR_TICK;
+our @WAIT_FOR_TICK_BEGIN;
+
+sub wait_for_tick {
+   return unless $TICK_WATCHER->is_active;
+   return if $Coro::current == $Coro::main;
+
+   my $signal = new Coro::Signal;
+   push @WAIT_FOR_TICK, $signal;
+   $signal->wait;
+}
+
+sub wait_for_tick_begin {
+   return unless $TICK_WATCHER->is_active;
+   return if $Coro::current == $Coro::main;
+
+   my $signal = new Coro::Signal;
+   push @WAIT_FOR_TICK_BEGIN, $signal;
+   $signal->wait;
+}
+
+      my $min = 1e6;#d#
+      my $avg = 10;
 $TICK_WATCHER = Event->timer (
    reentrant => 0,
+   parked    => 1,
    prio      => 0,
    at        => $NEXT_TICK || $TICK,
    data      => WF_AUTOCANCEL,
    cb        => sub {
-      unless ($FREEZE) {
-         cf::server_tick; # one server iteration
-         $RUNTIME += $TICK;
-      }
+      if ($Coro::current != $Coro::main) {
+         Carp::cluck "major BUG: server tick called outside of main coro, skipping it"
+            unless ++$bug_warning > 10;
+         return;
+      }
+
+      $NOW = $tick_start = Event::time;
+
+      cf::server_tick; # one server iteration
+
+      0 && sync_job {#d#
+         for(1..10)  {
+            my $t = Event::time;
+            my $map = my $map = new_from_path cf::map "/tmp/x.map"
+                     or die;
+
+            $map->width (50);
+            $map->height (50);
+            $map->alloc;
+            $map->_load_objects ("/tmp/x.map", 1);
+            my $t = Event::time - $t;
+
+            #next unless $t < 0.0013;#d#
+            if ($t < $min) {
+               $min = $t;
+            }
+            $avg = $avg * 0.99 + $t * 0.01;
+         }
+         warn "XXXXXXXXXXXXXXXXXX min $min avg $avg\n";#d#
+         exit 0;
+         # 2007-05-22 02:33:04.569 min 0.00112509727478027 avg 0.0012259249572477
+      };
 
+      $RUNTIME   += $TICK;
       $NEXT_TICK += $TICK;
 
+      if ($NOW >= $NEXT_RUNTIME_WRITE) {
+         $NEXT_RUNTIME_WRITE = $NOW + 10;
+         Coro::async_pool {
+            write_runtime
+               or warn "ERROR: unable to write runtime file: $!";
+         };
+      }
+
+#      my $AFTER = Event::time;
+#      warn $AFTER - $NOW;#d#
+
+      if (my $sig = shift @WAIT_FOR_TICK_BEGIN) {
+         $sig->send;
+      }
+      while (my $sig = shift @WAIT_FOR_TICK) {
+         $sig->send;
+      }
+
+      $NOW = Event::time;
+
       # if we are delayed by four ticks or more, skip them all
-      $NEXT_TICK = Event::time if Event::time >= $NEXT_TICK + $TICK * 4;
+      $NEXT_TICK = $NOW if $NOW >= $NEXT_TICK + $TICK * 4;
 
       $TICK_WATCHER->at ($NEXT_TICK);
       $TICK_WATCHER->start;
-   },
-);
 
-IO::AIO::max_poll_time $TICK * 0.2;
+      $LOAD = ($NOW - $tick_start) / $TICK;
+      $LOADAVG = $LOADAVG * 0.75 + $LOAD * 0.25;
+
+      _post_tick;
 
-Event->io (
-   fd   => IO::AIO::poll_fileno,
-   poll => 'r',
-   prio => 5,
-   data => WF_AUTOCANCEL,
-   cb   => \&IO::AIO::poll_cb,
-);
 
-Event->timer (
-   data     => WF_AUTOCANCEL,
-   after    => 0,
-   interval => 10,
-   cb       => sub {
-      (Coro::unblock_sub {
-         write_runtime
-            or warn "ERROR: unable to write runtime file: $!";
-      })->();
    },
 );
 
+{
+   BDB::max_poll_time $TICK * 0.1;
+   $BDB_POLL_WATCHER = Event->io (
+      reentrant => 0,
+      fd        => BDB::poll_fileno,
+      poll      => 'r',
+      prio      => 0,
+      data      => WF_AUTOCANCEL,
+      cb        => \&BDB::poll_cb,
+   );
+   BDB::min_parallel 8;
+
+   BDB::set_sync_prepare {
+      my $status;
+      my $current = $Coro::current;
+      (
+         sub {
+            $status = $!;
+            $current->ready; undef $current;
+         },
+         sub {
+            Coro::schedule while defined $current;
+            $! = $status;
+         },
+      )
+   };
+
+   unless ($DB_ENV) {
+      $DB_ENV = BDB::db_env_create;
+
+      cf::sync_job {
+         eval {
+            BDB::db_env_open
+               $DB_ENV,
+               $BDBDIR,
+               BDB::INIT_LOCK | BDB::INIT_LOG | BDB::INIT_MPOOL | BDB::INIT_TXN
+               | BDB::RECOVER | BDB::REGISTER | BDB::USE_ENVIRON | BDB::CREATE,
+               0666;
+
+            cf::cleanup "db_env_open($BDBDIR): $!" if $!;
+
+            $DB_ENV->set_flags (BDB::AUTO_COMMIT | BDB::REGION_INIT | BDB::TXN_NOSYNC, 1);
+            $DB_ENV->set_lk_detect;
+         };
+
+         cf::cleanup "db_env_open(db): $@" if $@;
+      };
+   }
+}
+
+{
+   IO::AIO::min_parallel 8;
+
+   undef $Coro::AIO::WATCHER;
+   IO::AIO::max_poll_time $TICK * 0.1;
+   $AIO_POLL_WATCHER = Event->io (
+      reentrant => 0,
+      data      => WF_AUTOCANCEL,
+      fd        => IO::AIO::poll_fileno,
+      poll      => 'r',
+      prio      => 6,
+      cb        => \&IO::AIO::poll_cb,
+   );
+}
+
+my $_log_backtrace;
+
+sub _log_backtrace {
+   my ($msg, @addr) = @_;
+
+   $msg =~ s/\n//;
+
+   # limit the # of concurrent backtraces
+   if ($_log_backtrace < 2) {
+      ++$_log_backtrace;
+      async {
+         my @bt = fork_call {
+            @addr = map { sprintf "%x", $_ } @addr;
+            my $self = (-f "/proc/$$/exe") ? "/proc/$$/exe" : $^X;
+            open my $fh, "exec addr2line -C -f -i -e \Q$self\E @addr 2>&1 |"
+               or die "addr2line: $!";
+
+            my @funcs;
+            my @res = <$fh>;
+            chomp for @res;
+            while (@res) {
+               my ($func, $line) = splice @res, 0, 2, ();
+               push @funcs, "[$func] $line";
+            }
+
+            @funcs
+         };
+
+         LOG llevInfo, "[ABT] $msg\n";
+         LOG llevInfo, "[ABT] $_\n" for @bt;
+         --$_log_backtrace;
+      };
+   } else {
+      LOG llevInfo, "[ABT] $msg\n";
+      LOG llevInfo, "[ABT] [suppressed]\n";
+   }
+}
+
+# load additional modules
+use cf::pod;
+
+END { cf::emergency_save }
+
 1