[ViewVC] Diff of: cvs/libev/ev.c

Comparing libev/ev.c (file contents):
Revision 1.229 by root, Fri May 2 08:08:45 2008 UTC vs.
Revision 1.236 by root, Wed May 7 14:46:22 2008 UTC

     }
 }
 /*****************************************************************************/
+#define MALLOC_ROUND 4096 /* prefer to allocate in chunks of this size, must be 2**n and >> 4 longs */
 int inline_size
 array_nextsize (int elem, int cur, int cnt)
 {
   int ncur = cur + 1;
   do
     ncur <<= 1;
   while (cnt > ncur);
-  /* if size > 4096, round to 4096 - 4 * longs to accomodate malloc overhead */
+  /* if size is large, round to MALLOC_ROUND - 4 * longs to accomodate malloc overhead */
-  if (elem * ncur > 4096)
+  if (elem * ncur > MALLOC_ROUND - sizeof (void *) * 4)
     {
       ncur *= elem;
-      ncur = (ncur + elem + 4095 + sizeof (void *) * 4) & ~4095;
+      ncur = (ncur + elem + (MALLOC_ROUND - 1) + sizeof (void *) * 4) & ~(MALLOC_ROUND - 1);
       ncur = ncur - sizeof (void *) * 4;
       ncur /= elem;
     }
   return ncur;
       }
 }
 /*****************************************************************************/
+/*
+ * at the moment we allow libev the luxury of two heaps,
+ * a small-code-size 2-heap one and a ~1.5kb larger 4-heap
+ * which is more cache-efficient.
+ * the difference is about 5% with 50000+ watchers.
+ */
+#define USE_4HEAP !EV_MINIMAL
+#if USE_4HEAP
+#define HEAP0 3 /* index of first element in heap */
 /* towards the root */
 void inline_speed
 upheap (WT *heap, int k)
 {
   WT w = heap [k];
   for (;;)
     {
+      int p = ((k - HEAP0 - 1) / 4) + HEAP0;
+      if (p >= HEAP0 || heap [p]->at <= w->at)
+        break;
+      heap [k] = heap [p];
+      ev_active (heap [k]) = k;
+      k = p;
+    }
+  heap [k] = w;
+  ev_active (heap [k]) = k;
+}
+/* away from the root */
+void inline_speed
+downheap (WT *heap, int N, int k)
+{
+  WT w = heap [k];
+  WT *E = heap + N + HEAP0;
+  for (;;)
+    {
+      ev_tstamp minat;
+      WT *minpos;
+      WT *pos = heap + 4 * (k - HEAP0) + HEAP0;
+      // find minimum child
+      if (expect_true (pos +3 < E))
+        {
+          /* fast path */
+                                   (minpos = pos + 0), (minat = (*minpos)->at);
+          if (pos [1]->at < minat) (minpos = pos + 1), (minat = (*minpos)->at);
+          if (pos [2]->at < minat) (minpos = pos + 2), (minat = (*minpos)->at);
+          if (pos [3]->at < minat) (minpos = pos + 3), (minat = (*minpos)->at);
+        }
+      else
+        {
+          /* slow path */
+          if (pos >= E)
+            break;
+                                                  (minpos = pos + 0), (minat = (*minpos)->at);
+          if (pos + 1 < E && pos [1]->at < minat) (minpos = pos + 1), (minat = (*minpos)->at);
+          if (pos + 2 < E && pos [2]->at < minat) (minpos = pos + 2), (minat = (*minpos)->at);
+          if (pos + 3 < E && pos [3]->at < minat) (minpos = pos + 3), (minat = (*minpos)->at);
+        }
+      if (w->at <= minat)
+        break;
+      ev_active (*minpos) = k;
+      heap [k] = *minpos;
+      k = minpos - heap;
+    }
+  heap [k] = w;
+  ev_active (heap [k]) = k;
+}
+#else // 4HEAP
+#define HEAP0 1
+/* towards the root */
+void inline_speed
+upheap (WT *heap, int k)
+{
+  WT w = heap [k];
+  for (;;)
+    {
       int p = k >> 1;
       /* maybe we could use a dummy element at heap [0]? */
       if (!p || heap [p]->at <= w->at)
         break;
       heap [k] = heap [p];
-      ((W)heap [k])->active = k;
+      ev_active (heap [k]) = k;
       k = p;
     }
   heap [k] = w;
-  ((W)heap [k])->active = k;
+  ev_active (heap [k]) = k;
 }
 /* away from the root */
 void inline_speed
 downheap (WT *heap, int N, int k)
       int c = k << 1;
       if (c > N)
         break;
-      c += c < N && heap [c]->at > heap [c + 1]->at
+      c += c + 1 < N && heap [c]->at > heap [c + 1]->at
            ? 1 : 0;
       if (w->at <= heap [c]->at)
         break;
       heap [k] = heap [c];
       ((W)heap [k])->active = k;
       k = c;
     }
   heap [k] = w;
-  ((W)heap [k])->active = k;
+  ev_active (heap [k]) = k;
 }
+#endif
 void inline_size
 adjustheap (WT *heap, int N, int k)
 {
   upheap (heap, k);
 pipecb (EV_P_ ev_io *iow, int revents)
 {
 #if EV_USE_EVENTFD
   if (evfd >= 0)
     {
-      uint64_t counter = 1;
+      uint64_t counter;
       read (evfd, &counter, sizeof (uint64_t));
     }
   else
 #endif
     {
 void
 ev_loop_fork (EV_P)
 {
   postfork = 1; /* must be in line with ev_default_fork */
 }
 #endif
 #if EV_MULTIPLICITY
 struct ev_loop *
 ev_default_loop_init (unsigned int flags)
             EV_CB_INVOKE (p->w, p->events);
           }
       }
 }
+#if EV_IDLE_ENABLE
+void inline_size
+idle_reify (EV_P)
+{
+  if (expect_false (idleall))
+    {
+      int pri;
+      for (pri = NUMPRI; pri--; )
+        {
+          if (pendingcnt [pri])
+            break;
+          if (idlecnt [pri])
+            {
+              queue_events (EV_A_ (W *)idles [pri], idlecnt [pri], EV_IDLE);
+              break;
+            }
+        }
+    }
+}
+#endif
 void inline_size
 timers_reify (EV_P)
 {
-  while (timercnt && ev_at (timers [1]) <= mn_now)
+  while (timercnt && ev_at (timers [HEAP0]) <= mn_now)
     {
-      ev_timer *w = (ev_timer *)timers [1];
+      ev_timer *w = (ev_timer *)timers [HEAP0];
       /*assert (("inactive timer on timer heap detected", ev_is_active (w)));*/
       /* first reschedule or stop timer */
       if (w->repeat)
           ev_at (w) += w->repeat;
           if (ev_at (w) < mn_now)
             ev_at (w) = mn_now;
-          downheap (timers, timercnt, 1);
+          downheap (timers, timercnt, HEAP0);
         }
       else
         ev_timer_stop (EV_A_ w); /* nonrepeating: stop timer */
       ev_feed_event (EV_A_ (W)w, EV_TIMEOUT);
 #if EV_PERIODIC_ENABLE
 void inline_size
 periodics_reify (EV_P)
 {
-  while (periodiccnt && ev_at (periodics [1]) <= ev_rt_now)
+  while (periodiccnt && ev_at (periodics [HEAP0]) <= ev_rt_now)
     {
-      ev_periodic *w = (ev_periodic *)periodics [1];
+      ev_periodic *w = (ev_periodic *)periodics [HEAP0];
       /*assert (("inactive timer on periodic heap detected", ev_is_active (w)));*/
       /* first reschedule or stop timer */
       if (w->reschedule_cb)
       else if (w->interval)
         {
           ev_at (w) = w->offset + ceil ((ev_rt_now - w->offset) / w->interval) * w->interval;
           if (ev_at (w) - ev_rt_now <= TIME_EPSILON) ev_at (w) += w->interval;
           assert (("ev_periodic timeout in the past detected while processing timers, negative interval?", ev_at (w) > ev_rt_now));
-          downheap (periodics, periodiccnt, 1);
+          downheap (periodics, periodiccnt, HEAP0);
         }
       else
         ev_periodic_stop (EV_A_ w); /* nonrepeating: stop timer */
       ev_feed_event (EV_A_ (W)w, EV_PERIODIC);
 periodics_reschedule (EV_P)
 {
   int i;
   /* adjust periodics after time jump */
-  for (i = 0; i < periodiccnt; ++i)
+  for (i = 1; i <= periodiccnt; ++i)
     {
       ev_periodic *w = (ev_periodic *)periodics [i];
       if (w->reschedule_cb)
         ev_at (w) = w->reschedule_cb (w, ev_rt_now);
       else if (w->interval)
         ev_at (w) = w->offset + ceil ((ev_rt_now - w->offset) / w->interval) * w->interval;
     }
   /* now rebuild the heap */
-  for (i = periodiccnt >> 1; i--; )
+  for (i = periodiccnt >> 1; --i; )
-    downheap (periodics, periodiccnt, i);
+    downheap (periodics, periodiccnt, i + HEAP0);
-}
-#endif
-#if EV_IDLE_ENABLE
-void inline_size
-idle_reify (EV_P)
-{
-  if (expect_false (idleall))
-    {
-      int pri;
-      for (pri = NUMPRI; pri--; )
-        {
-          if (pendingcnt [pri])
-            break;
-          if (idlecnt [pri])
-            {
-              queue_events (EV_A_ (W *)idles [pri], idlecnt [pri], EV_IDLE);
-              break;
-            }
-        }
-    }
 }
 #endif
 void inline_speed
 time_update (EV_P_ ev_tstamp max_block)
        */
       for (i = 4; --i; )
         {
           rtmn_diff = ev_rt_now - mn_now;
-          if (fabs (odiff - rtmn_diff) < MIN_TIMEJUMP)
+          if (expect_true (fabs (odiff - rtmn_diff) < MIN_TIMEJUMP))
             return; /* all is well */
           ev_rt_now = ev_time ();
           mn_now    = get_clock ();
           now_floor = mn_now;
             waittime = MAX_BLOCKTIME;
             if (timercnt)
               {
-                ev_tstamp to = ev_at (timers [1]) - mn_now + backend_fudge;
+                ev_tstamp to = ev_at (timers [HEAP0]) - mn_now + backend_fudge;
                 if (waittime > to) waittime = to;
               }
 #if EV_PERIODIC_ENABLE
             if (periodiccnt)
               {
-                ev_tstamp to = ev_at (periodics [1]) - ev_rt_now + backend_fudge;
+                ev_tstamp to = ev_at (periodics [HEAP0]) - ev_rt_now + backend_fudge;
                 if (waittime > to) waittime = to;
               }
 #endif
             if (expect_false (waittime < timeout_blocktime))
   ev_at (w) += mn_now;
   assert (("ev_timer_start called with negative timer repeat value", w->repeat >= 0.));
-  ev_start (EV_A_ (W)w, ++timercnt);
+  ev_start (EV_A_ (W)w, ++timercnt + HEAP0 - 1);
-  array_needsize (WT, timers, timermax, timercnt + 1, EMPTY2);
+  array_needsize (WT, timers, timermax, timercnt + HEAP0, EMPTY2);
-  timers [timercnt] = (WT)w;
+  timers [ev_active (w)] = (WT)w;
-  upheap (timers, timercnt);
+  upheap (timers, ev_active (w));
-  /*assert (("internal timer heap corruption", timers [((W)w)->active] == w));*/
+  /*assert (("internal timer heap corruption", timers [ev_active (w)] == w));*/
 }
 void noinline
 ev_timer_stop (EV_P_ ev_timer *w)
 {
   clear_pending (EV_A_ (W)w);
   if (expect_false (!ev_is_active (w)))
     return;
-  assert (("internal timer heap corruption", timers [((W)w)->active] == (WT)w));
   {
-    int active = ((W)w)->active;
+    int active = ev_active (w);
+    assert (("internal timer heap corruption", timers [active] == (WT)w));
-    if (expect_true (active < timercnt))
+    if (expect_true (active < timercnt + HEAP0 - 1))
       {
-        timers [active] = timers [timercnt];
+        timers [active] = timers [timercnt + HEAP0 - 1];
         adjustheap (timers, timercnt, active);
       }
     --timercnt;
   }
   if (ev_is_active (w))
     {
       if (w->repeat)
         {
           ev_at (w) = mn_now + w->repeat;
-          adjustheap (timers, timercnt, ((W)w)->active);
+          adjustheap (timers, timercnt, ev_active (w));
         }
       else
         ev_timer_stop (EV_A_ w);
     }
   else if (w->repeat)
       ev_at (w) = w->offset + ceil ((ev_rt_now - w->offset) / w->interval) * w->interval;
     }
   else
     ev_at (w) = w->offset;
-  ev_start (EV_A_ (W)w, ++periodiccnt);
+  ev_start (EV_A_ (W)w, ++periodiccnt + HEAP0 - 1);
-  array_needsize (WT, periodics, periodicmax, periodiccnt + 1, EMPTY2);
+  array_needsize (WT, periodics, periodicmax, periodiccnt + HEAP0, EMPTY2);
-  periodics [periodiccnt] = (WT)w;
+  periodics [ev_active (w)] = (WT)w;
-  upheap (periodics, periodiccnt);
+  upheap (periodics, ev_active (w));
-  /*assert (("internal periodic heap corruption", periodics [((W)w)->active - 1] == w));*/
+  /*assert (("internal periodic heap corruption", periodics [ev_active (w)] == w));*/
 }
 void noinline
 ev_periodic_stop (EV_P_ ev_periodic *w)
 {
   clear_pending (EV_A_ (W)w);
   if (expect_false (!ev_is_active (w)))
     return;
-  assert (("internal periodic heap corruption", periodics [((W)w)->active] == (WT)w));
   {
-    int active = ((W)w)->active;
+    int active = ev_active (w);
+    assert (("internal periodic heap corruption", periodics [active] == (WT)w));
-    if (expect_true (active < periodiccnt))
+    if (expect_true (active < periodiccnt + HEAP0 - 1))
       {
-        periodics [active] = periodics [periodiccnt];
+        periodics [active] = periodics [periodiccnt + HEAP0 - 1];
         adjustheap (periodics, periodiccnt, active);
       }
     --periodiccnt;
   }
   if (w->wd < 0)
     {
       ev_timer_start (EV_A_ &w->timer); /* this is not race-free, so we still need to recheck periodically */
       /* monitor some parent directory for speedup hints */
+      /* note that exceeding the hardcoded limit is not a correctness issue, */
+      /* but an efficiency issue only */
       if ((errno == ENOENT || errno == EACCES) && strlen (w->path) < 4096)
         {
           char path [4096];
           strcpy (path, w->path);
   clear_pending (EV_A_ (W)w);
   if (expect_false (!ev_is_active (w)))
     return;
   {
-    int active = ((W)w)->active;
+    int active = ev_active (w);
     idles [ABSPRI (w)][active - 1] = idles [ABSPRI (w)][--idlecnt [ABSPRI (w)]];
-    ((W)idles [ABSPRI (w)][active - 1])->active = active;
+    ev_active (idles [ABSPRI (w)][active - 1]) = active;
     ev_stop (EV_A_ (W)w);
     --idleall;
   }
 }
   clear_pending (EV_A_ (W)w);
   if (expect_false (!ev_is_active (w)))
     return;
   {
-    int active = ((W)w)->active;
+    int active = ev_active (w);
     prepares [active - 1] = prepares [--preparecnt];
-    ((W)prepares [active - 1])->active = active;
+    ev_active (prepares [active - 1]) = active;
   }
   ev_stop (EV_A_ (W)w);
 }
   clear_pending (EV_A_ (W)w);
   if (expect_false (!ev_is_active (w)))
     return;
   {
-    int active = ((W)w)->active;
+    int active = ev_active (w);
     checks [active - 1] = checks [--checkcnt];
-    ((W)checks [active - 1])->active = active;
+    ev_active (checks [active - 1]) = active;
   }
   ev_stop (EV_A_ (W)w);
 }
   clear_pending (EV_A_ (W)w);
   if (expect_false (!ev_is_active (w)))
     return;
   {
-    int active = ((W)w)->active;
+    int active = ev_active (w);
     forks [active - 1] = forks [--forkcnt];
-    ((W)forks [active - 1])->active = active;
+    ev_active (forks [active - 1]) = active;
   }
   ev_stop (EV_A_ (W)w);
 }
 #endif
   clear_pending (EV_A_ (W)w);
   if (expect_false (!ev_is_active (w)))
     return;
   {
-    int active = ((W)w)->active;
+    int active = ev_active (w);
     asyncs [active - 1] = asyncs [--asynccnt];
-    ((W)asyncs [active - 1])->active = active;
+    ev_active (asyncs [active - 1]) = active;
   }
   ev_stop (EV_A_ (W)w);
 }

Diff Legend

-–
+Removed lines
-+
+Added lines
-<
+Changed lines
->
+Changed lines

Comparing libev/ev.c (file contents): Revision 1.229 by root, Fri May 2 08:08:45 2008 UTC vs. Revision 1.236 by root, Wed May 7 14:46:22 2008 UTC

Diff Legend

Comparing libev/ev.c (file contents):
Revision 1.229 by root, Fri May 2 08:08:45 2008 UTC vs.
Revision 1.236 by root, Wed May 7 14:46:22 2008 UTC