Migrate vmm_callout -> OSX libdispatch (#112)

* Migrate vmm_callout to libdispatch

* Some cleanup in dispatch code

* Callout precision with libdispatch code

* Make callout_drain cleanup correctly dispatch_sources

* use dispatch_groups instead of busy-waiting for completion

* Place dispatch_group_leave after c->queued=0 to prevent possible concurrency bug. The bug may occur in callout_stop_safe when queued is still not 0 and we call dispatch_suspend on a released object, can happen only during draining.

Author: bonifaido

include/xhyve/vmm/vmm_callout.h

@@ -1,9 +1,9 @@
 #pragma once
 
 #include <stdint.h>
-#include <pthread.h>
 #include <time.h>
 #include <sys/time.h>
+#include <dispatch/dispatch.h>
 
 #define SBT_1S  ((sbintime_t)1 << 32)
 #define SBT_1M  (SBT_1S * 60)
@@ -70,6 +70,7 @@ static inline void bintime_sub(struct bintime *_bt, const struct bintime *_bt2)
     ((a)->frac cmp (b)->frac) : \
     ((a)->sec cmp (b)->sec))
 
+extern int tc_precexp;
 
 void binuptime(struct bintime *bt);
 void getmicrotime(struct timeval *tv);
@@ -82,10 +83,10 @@ static inline sbintime_t sbinuptime(void) {
 }
 
 struct callout {
-  pthread_cond_t wait;
-  struct callout *prev;
-  struct callout *next;
+  dispatch_group_t group;
+  dispatch_source_t timer;
   uint64_t timeout;
+  uint64_t precision;
   void *argument;
   void (*callout)(void *);
   int flags;
@@ -95,7 +96,6 @@ struct callout {
 #define C_ABSOLUTE 0x0200 /* event time is absolute */
 #define CALLOUT_ACTIVE 0x0002 /* callout is currently active */
 #define CALLOUT_PENDING 0x0004 /* callout is waiting for timeout */
-#define CALLOUT_MPSAFE 0x0008 /* callout handler is mp safe */
 #define CALLOUT_RETURNUNLOCKED 0x0010 /* handler returns with mtx unlocked */
 #define CALLOUT_COMPLETED 0x0020 /* callout thread finished */
 #define CALLOUT_WAITING 0x0040 /* thread waiting for callout to finish */

src/vmm/io/vatpit.c

@@ -155,11 +155,12 @@ static void
 pit_timer_start_cntr0(struct vatpit *vatpit)
 {
 	struct channel *c;
-	sbintime_t now, delta;
+	sbintime_t now, delta, precision;
 
 	c = &vatpit->channel[0];
 	if (c->initial != 0) {
 		delta = c->initial * vatpit->freq_sbt;
+		precision = delta >> tc_precexp;
 		c->callout_sbt = c->callout_sbt + delta;
 
 		/*
@@ -172,7 +173,7 @@ pit_timer_start_cntr0(struct vatpit *vatpit)
 			c->callout_sbt = now + delta;
 
 		callout_reset_sbt(&c->callout, c->callout_sbt,
-		    0, vatpit_callout_handler, &c->callout_arg,
+		    precision, vatpit_callout_handler, &c->callout_arg,
 		    C_ABSOLUTE);
 	}
 }
@@ -429,7 +430,7 @@ vatpit_init(struct vm *vm)
 	vatpit->freq_sbt = bttosbt(bt);
 
 	for (i = 0; i < 3; i++) {
-		callout_init(&vatpit->channel[i].callout, true);
+		callout_init(&vatpit->channel[i].callout, 1);
 		arg = &vatpit->channel[i].callout_arg;
 		arg->vatpit = vatpit;
 		arg->channel_num = i;

src/vmm/io/vhpet.c

@@ -325,7 +325,7 @@ vhpet_stop_timer(struct vhpet *vhpet, int n, sbintime_t now)
 static void
 vhpet_start_timer(struct vhpet *vhpet, int n, uint32_t counter, sbintime_t now)
 {
-	sbintime_t delta;
+	sbintime_t delta, precision;
 
 	if (vhpet->timer[n].comprate != 0)
 		vhpet_adjust_compval(vhpet, n, counter);
@@ -339,9 +339,10 @@ vhpet_start_timer(struct vhpet *vhpet, int n, uint32_t counter, sbintime_t now)
 	}
 
 	delta = (vhpet->timer[n].compval - counter) * vhpet->freq_sbt;
+	precision = delta >> tc_precexp;
 	vhpet->timer[n].callout_sbt = now + delta;
 	callout_reset_sbt(&vhpet->timer[n].callout, vhpet->timer[n].callout_sbt,
-	    0, vhpet_handler, &vhpet->timer[n].arg, C_ABSOLUTE);
+	    precision, vhpet_handler, &vhpet->timer[n].arg, C_ABSOLUTE);
 }
 
 static void