hyperv: Implement "enlightened" time counter, which is rdtsc based.

Reviewed by:	kib
MFC after:	1 week
Sponsored by:	Microsoft
Differential Revision:	https://reviews.freebsd.org/D8763

diff --git a/sys/dev/hyperv/include/hyperv.h b/sys/dev/hyperv/include/hyperv.h
index c5e7dec1f6806dc3538bd22ea820fd9d23ac1047..44851444d26f785d188557a63251c3a1562ef70c 100644 (file)
--- a/sys/dev/hyperv/include/hyperv.h
+++ b/sys/dev/hyperv/include/hyperv.h
@@ -45,6 +45,7 @@
 #define CPUID_HV_MSR_HYPERCALL         0x0020  /* MSR_HV_GUEST_OS_ID
                                                 * MSR_HV_HYPERCALL */
 #define CPUID_HV_MSR_VP_INDEX          0x0040  /* MSR_HV_VP_INDEX */
+#define CPUID_HV_MSR_REFERENCE_TSC     0x0200  /* MSR_HV_REFERENCE_TSC */
 #define CPUID_HV_MSR_GUEST_IDLE                0x0400  /* MSR_HV_GUEST_IDLE */
 
 #ifndef NANOSEC

diff --git a/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c b/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c
index 5b5f205f333eb865fd52c51133985c2feb0330de..712a6e22994bf2cd7b005192c67b99d0e9a0ce40 100644 (file)
--- a/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c
+++ b/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c
@@ -28,7 +28,37 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/timetc.h>
+
+#include <machine/cpufunc.h>
+#include <machine/cputypes.h>
+#include <machine/md_var.h>
+
+#include <dev/hyperv/include/hyperv.h>
+#include <dev/hyperv/include/hyperv_busdma.h>
 #include <dev/hyperv/vmbus/hyperv_machdep.h>
+#include <dev/hyperv/vmbus/hyperv_reg.h>
+#include <dev/hyperv/vmbus/hyperv_var.h>
+
+struct hyperv_reftsc_ctx {
+       struct hyperv_reftsc    *tsc_ref;
+       struct hyperv_dma       tsc_ref_dma;
+};
+
+static struct timecounter      hyperv_tsc_timecounter = {
+       .tc_get_timecount       = NULL, /* based on CPU vendor. */
+       .tc_poll_pps            = NULL,
+       .tc_counter_mask        = 0xffffffff,
+       .tc_frequency           = HYPERV_TIMER_FREQ,
+       .tc_name                = "Hyper-V-TSC",
+       .tc_quality             = 3000,
+       .tc_flags               = 0,
+       .tc_priv                = NULL
+};
+
+static struct hyperv_reftsc_ctx        hyperv_ref_tsc;
 
 uint64_t
 hypercall_md(volatile void *hc_addr, uint64_t in_val,
@@ -41,3 +71,85 @@ hypercall_md(volatile void *hc_addr, uint64_t in_val,
            "c" (in_val), "d" (in_paddr), "m" (hc_addr));
        return (status);
 }
+
+#define HYPERV_TSC_TIMECOUNT(fence)                                    \
+static u_int                                                           \
+hyperv_tsc_timecount_##fence(struct timecounter *tc)                   \
+{                                                                      \
+       struct hyperv_reftsc *tsc_ref = hyperv_ref_tsc.tsc_ref;         \
+       uint32_t seq;                                                   \
+                                                                       \
+       while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) {   \
+               uint64_t disc, ret, tsc;                                \
+               uint64_t scale = tsc_ref->tsc_scale;                    \
+               int64_t ofs = tsc_ref->tsc_ofs;                         \
+                                                                       \
+               fence();                                                \
+               tsc = rdtsc();                                          \
+                                                                       \
+               /* ret = ((tsc * scale) >> 64) + ofs */                 \
+               __asm__ __volatile__ ("mulq %3" :                       \
+                   "=d" (ret), "=a" (disc) :                           \
+                   "a" (tsc), "r" (scale));                            \
+               ret += ofs;                                             \
+                                                                       \
+               atomic_thread_fence_acq();                              \
+               if (tsc_ref->tsc_seq == seq)                            \
+                       return (ret);                                   \
+                                                                       \
+               /* Sequence changed; re-sync. */                        \
+       }                                                               \
+       /* Fallback to the generic timecounter, i.e. rdmsr. */          \
+       return (rdmsr(MSR_HV_TIME_REF_COUNT));                          \
+}                                                                      \
+struct __hack
+
+HYPERV_TSC_TIMECOUNT(lfence);
+HYPERV_TSC_TIMECOUNT(mfence);
+
+static void
+hyperv_tsc_tcinit(void *dummy __unused)
+{
+       uint64_t val, orig;
+
+       if ((hyperv_features &
+            (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC)) !=
+           (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC) ||
+           (cpu_feature & CPUID_SSE2) == 0)    /* SSE2 for mfence/lfence */
+               return;
+
+       switch (cpu_vendor_id) {
+       case CPU_VENDOR_AMD:
+               hyperv_tsc_timecounter.tc_get_timecount =
+                   hyperv_tsc_timecount_mfence;
+               break;
+
+       case CPU_VENDOR_INTEL:
+               hyperv_tsc_timecounter.tc_get_timecount =
+                   hyperv_tsc_timecount_lfence;
+               break;
+
+       default:
+               /* Unsupport CPU vendors. */
+               return;
+       }
+
+       hyperv_ref_tsc.tsc_ref = hyperv_dmamem_alloc(NULL, PAGE_SIZE, 0,
+           sizeof(struct hyperv_reftsc), &hyperv_ref_tsc.tsc_ref_dma,
+           BUS_DMA_WAITOK | BUS_DMA_ZERO);
+       if (hyperv_ref_tsc.tsc_ref == NULL) {
+               printf("hyperv: reftsc page allocation failed\n");
+               return;
+       }
+
+       orig = rdmsr(MSR_HV_REFERENCE_TSC);
+       val = MSR_HV_REFTSC_ENABLE | (orig & MSR_HV_REFTSC_RSVD_MASK) |
+           ((hyperv_ref_tsc.tsc_ref_dma.hv_paddr >> PAGE_SHIFT) <<
+            MSR_HV_REFTSC_PGSHIFT);
+       wrmsr(MSR_HV_REFERENCE_TSC, val);
+
+       /* Register "enlightened" timecounter. */
+       tc_init(&hyperv_tsc_timecounter);
+}
+SYSINIT(hyperv_tsc_init, SI_SUB_DRIVERS, SI_ORDER_FIRST, hyperv_tsc_tcinit,
+    NULL);

diff --git a/sys/dev/hyperv/vmbus/hyperv_reg.h b/sys/dev/hyperv/vmbus/hyperv_reg.h
index 4fb12286be263594aacdcdbbc37840d25140526e..263a191fec235c120288696193dbbb06e7d63461 100644 (file)
--- a/sys/dev/hyperv/vmbus/hyperv_reg.h
+++ b/sys/dev/hyperv/vmbus/hyperv_reg.h
@@ -57,6 +57,11 @@
 
 #define MSR_HV_VP_INDEX                        0x40000002
 
+#define MSR_HV_REFERENCE_TSC           0x40000021
+#define MSR_HV_REFTSC_ENABLE           0x0001ULL
+#define MSR_HV_REFTSC_RSVD_MASK                0x0ffeULL
+#define MSR_HV_REFTSC_PGSHIFT          12
+
 #define MSR_HV_SCONTROL                        0x40000080
 #define MSR_HV_SCTRL_ENABLE            0x0001ULL
 #define MSR_HV_SCTRL_RSVD_MASK         0xfffffffffffffffeULL
@@ -123,6 +128,17 @@
 #define CPUID_LEAF_HV_LIMITS           0x40000005
 #define CPUID_LEAF_HV_HWFEATURES       0x40000006
 
+/*
+ * Hyper-V Reference TSC
+ */
+struct hyperv_reftsc {
+       volatile uint32_t       tsc_seq;
+       volatile uint32_t       tsc_rsvd1;
+       volatile uint64_t       tsc_scale;
+       volatile int64_t        tsc_ofs;
+} __packed __aligned(PAGE_SIZE);
+CTASSERT(sizeof(struct hyperv_reftsc) == PAGE_SIZE);
+
 /*
  * Hyper-V Monitor Notification Facility
  */