ViewVC Help
View File | Revision Log | Show Annotations | Download File
/cvs/deliantra/server/socket/loop.C
Revision: 1.65
Committed: Sun Jul 29 02:24:34 2007 UTC (16 years, 10 months ago) by root
Content type: text/plain
Branch: MAIN
Changes since 1.64: +78 -94 lines
Log Message:
implement the most elaborate rate limit system to date, from the errors before:

- output-rate is now an upper bound only. its purpose is to give the user
  some control over bandwith use. it should be set too high rather than too low.
- the server will (on linux only, or systems that support tcpi), guess
  how much the kernel is willing to send without delay (this is imperfect as
  we do not know the remote receive window, but we assume its "large enough").
- we accuretyl measure mss and ensure that preferably mss-sized packets
  leave the server, by sending less in some cases and more in others
  to reacht eh desired bandwidth goal
  (e.g. 15000 == http://ue.tst.eu/545350740128735b13aaf541c88bfaf2.txt)

the net effect is that the server will never send (much) more data than the kernel
thinks the network is able to handle. that is, when the connection was idle for a time
and the congestion window is small, we will only start sending small amounts of
data, prompting the kernel to accuratly model the bandwidth.

in essence, this creates a tcp stream that never has more data buffered
than neccessary for in-flight data, ensuring that we can get low-latency
map updates through to the client whole using all excess bandwidth the
network can handle.

I mostly tested with netem, e.g.

   ifconfig lo mtu 1500
   tc qdisc change dev lo root netem delay 190ms 10ms drop 0.1

gave me roughtly 20kb/s throughput even though output-rate was 100kb/s,
without stalling the conenction even when downloading backgorund music and
other large chunks of data.

File Contents

# User Rev Content
1 elmex 1.1 /*
2 root 1.55 * This file is part of Crossfire TRT, the Roguelike Realtime MORPG.
3 pippijn 1.39 *
4 root 1.49 * Copyright (©) 2005,2006,2007 Marc Alexander Lehmann / Robin Redeker / the Crossfire TRT team
5     * Copyright (©) 2002-2003,2007 Mark Wedel & The Crossfire Development Team
6     * Copyright (©) 1992,2007 Frank Tore Johansen
7     *
8 root 1.55 * Crossfire TRT is free software: you can redistribute it and/or modify
9     * it under the terms of the GNU General Public License as published by
10     * the Free Software Foundation, either version 3 of the License, or
11     * (at your option) any later version.
12     *
13     * This program is distributed in the hope that it will be useful,
14     * but WITHOUT ANY WARRANTY; without even the implied warranty of
15     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16     * GNU General Public License for more details.
17     *
18     * You should have received a copy of the GNU General Public License
19     * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 root 1.49 *
21     * The authors can be reached via e-mail to <crossfire@schmorp.de>
22 pippijn 1.39 */
23 elmex 1.1
24     /**
25     * \file
26     * Main client/server loops.
27     *
28     * \date 2003-12-02
29     *
30     * loop.c mainly deals with initialization and higher level socket
31 root 1.26 * maintanance (checking for lost connections and if data has arrived.)
32 elmex 1.1 */
33    
34     #include <global.h>
35 root 1.10 #include <sproto.h>
36     #include <sockproto.h>
37 elmex 1.1
38 pippijn 1.8 #include <sys/types.h>
39     #include <sys/time.h>
40     #include <sys/socket.h>
41     #include <netinet/in.h>
42     #include <netdb.h>
43 elmex 1.1
44 root 1.16 #include <unistd.h>
45     #include <arpa/inet.h>
46 elmex 1.1
47     #include <loader.h>
48    
49 root 1.51 #define BG_SCRUB_RATE 4 // how often to send a face in the background
50    
51 root 1.65 #define MAX_QUEUE_DEPTH 50
52 root 1.51 #define MAX_QUEUE_BACKLOG 3.
53 elmex 1.1
54 root 1.21 void
55     client::reset_state ()
56     {
57     if (!pl)
58     return;
59    
60     pl->run_on = 0;
61     pl->fire_on = 0;
62     }
63 elmex 1.1
64 root 1.20 void
65     client::queue_command (packet_type *handler, char *data, int datalen)
66 root 1.6 {
67 root 1.51 tstamp stamp = NOW;
68 elmex 1.1
69 root 1.20 if (cmd_queue.size () >= MAX_QUEUE_DEPTH)
70 root 1.21 {
71     //TODO: just disconnect here?
72     reset_state ();
73     send_packet_printf ("drawinfo %d command queue overflow, ignoring.", NDI_RED);
74     }
75 root 1.20 else
76 root 1.24 {
77     cmd_queue.push_back (command ());
78     command &cmd = cmd_queue.back ();
79     cmd.stamp = stamp;
80     cmd.handler = handler;
81 root 1.25 cmd.data = salloc<char> (datalen + 1, data);
82 root 1.24 cmd.datalen = datalen;
83     }
84 root 1.20 }
85 elmex 1.1
86 root 1.20 bool
87     client::handle_command ()
88 elmex 1.1 {
89 root 1.48 if (!cmd_queue.empty ()
90     && state == ST_PLAYING
91     && pl->ob->speed_left > 0.f)
92 root 1.26 {
93     command &cmd = cmd_queue.front ();
94 root 1.6
95 root 1.51 if (cmd.stamp + MAX_QUEUE_BACKLOG < NOW)
96 root 1.26 {
97 root 1.48 reset_state ();
98     send_packet_printf ("drawinfo %d ignoring delayed commands.", NDI_RED);
99 root 1.26 }
100     else
101 root 1.48 execute (cmd.handler, cmd.data, cmd.datalen);
102    
103     cmd_queue.pop_front ();
104     return true;
105 root 1.21 }
106 root 1.48 else
107     return false;
108 elmex 1.1 }
109    
110 root 1.6 void
111     flush_sockets (void)
112 elmex 1.1 {
113 root 1.19 for (sockvec::iterator i = clients.begin (); i != clients.end (); ++i)
114 root 1.31 (*i)->flush ();
115 elmex 1.1 }
116    
117     /**
118 root 1.10 * This checks the sockets for input, does the right thing.
119 elmex 1.1 *
120     * A bit of this code is grabbed out of socket.c
121     * There are 2 lists we need to look through - init_sockets is a list
122     *
123     */
124 root 1.6 void
125     doeric_server (void)
126 elmex 1.1 {
127 root 1.33 //TODO: should not be done here, either
128 pippijn 1.40 for (unsigned i = 0; i < clients.size (); ++i)
129 root 1.6 {
130 root 1.42 client *ns = clients [i];
131 elmex 1.1
132 root 1.42 ns->tick ();
133     ns->refcnt_chk ();
134     }
135     }
136    
137     void
138     client::tick ()
139     {
140     if (!pl || destroyed ())
141     return;
142 root 1.37
143 root 1.42 /* Update the players stats once per tick. More efficient than
144     * sending them whenever they change, and probably just as useful
145     */
146     esrv_update_stats (pl);
147 root 1.20
148 root 1.42 if (last_weight != -1 && last_weight != WEIGHT (pl->ob))
149     {
150     esrv_update_item (UPD_WEIGHT, pl->ob, pl->ob);
151     if (last_weight != WEIGHT (pl->ob))
152     LOG (llevError, "esrv_update_item(UPD_WEIGHT) did not set player weight: is %lu, should be %lu\n",
153     (unsigned long) last_weight, WEIGHT (pl->ob));
154     }
155 root 1.6
156 root 1.50 draw_client_map (pl);
157 root 1.20
158 root 1.42 if (update_look)
159 root 1.50 esrv_draw_look (pl);
160 root 1.41
161 root 1.65 if (ixface.empty ())
162     {
163     // regularly send a new face when queue is empty
164     if (bg_scrub && !--bg_scrub && enable_bg_scrub)
165     while (scrub_idx < faces.size () - 1)
166     {
167     ++scrub_idx;
168    
169     if (!faces_sent [scrub_idx])
170     if (faceinfo *f = face_info (scrub_idx))
171     if (f->type == FT_FACE) // only scrub faces for now
172     {
173     send_face (scrub_idx, -120);
174     flush_fx ();
175    
176     bg_scrub = 1; // send up to one face per tick, unless an image was requested
177     break;
178     }
179     }
180    
181     rate_avail = max_rate - outputbuffer_len ();
182     }
183     else
184     {
185     int ol = outputbuffer_len ();
186    
187     rate_avail = min (max_rate + mss, rate_avail + max_rate);
188    
189     int avail = rate_avail;
190    
191     // if we can split images, round to next-lowest mss
192     if (fxix) avail -= avail % mss;
193    
194     rate_avail -= ol;
195     avail -= ol;
196 root 1.64
197     #if HAVE_TCP_INFO
198 root 1.65 // further restrict the available bandwidth by the excess bandwidth available
199     avail = max (0, min (avail, (tcpi.tcpi_snd_cwnd - tcpi.tcpi_unacked + tcpi.tcpi_sacked) * mss));
200 root 1.64 #endif
201 root 1.54
202 root 1.65 bg_scrub = BG_SCRUB_RATE;
203    
204     while (avail > 0)
205 root 1.64 {
206 root 1.65 ixsend &ix = ixface.back ();
207 root 1.44
208 root 1.65 if (facedata *d = face_data (ix.idx, faceset))
209     {
210     if (fxix)
211     {
212     // estimate the packet header overhead "ix " + idx + (new)ofs
213     int pktlen = 3 + ber32::encoded_size (ix.idx) + ber32::encoded_size (ix.ofs);
214     int chunk = min (avail - packet::hdrlen, MAXSOCKBUF) - pktlen;
215    
216     // only transfer something if the amount of data transferred
217     // has a healthy relation to the header overhead
218     if (chunk < 64)
219     break;
220 root 1.46
221 root 1.65 chunk = min (chunk, (int)ix.ofs);
222 root 1.46
223 root 1.65 ix.ofs -= chunk;
224 root 1.46
225 root 1.65 //fprintf (stderr, "i%dx %6d: %5d+%4d (%4d)\n", fxix, ix.idx,ix.ofs,chunk, ixface.size());//D
226 root 1.59
227 root 1.65 packet sl ("ix");
228 root 1.46
229 root 1.65 sl << ber32 (ix.idx)
230     << ber32 (ix.ofs)
231     << data (d->data.data () + ix.ofs, chunk);
232 root 1.46
233 root 1.65 send_packet (sl);
234     }
235     else
236 root 1.63 {
237 root 1.65 send_image (ix.idx);
238     ix.ofs = 0;
239 root 1.46 }
240 root 1.65 }
241     else
242     ix.ofs = 0;
243 root 1.44
244 root 1.65 int consumed = outputbuffer_len () - ol;
245 root 1.54
246 root 1.65 avail -= consumed;
247     rate_avail -= consumed;
248 root 1.54
249 root 1.65 ol = outputbuffer_len ();
250 root 1.54
251 root 1.65 if (!ix.ofs)
252     {
253     ixface.pop_back ();
254 root 1.56
255 root 1.65 if (ixface.empty ())
256     break;
257 root 1.54 }
258 root 1.44 }
259     }
260 elmex 1.1 }
261 root 1.43