libxl,hvmloader: Don't relocate memory for MMIO hole

At the moment, qemu-xen can't handle memory being relocated by
hvmloader.  This may happen if a device with a large enough memory
region is passed through to the guest.  At the moment, if this
happens, then at some point in the future qemu will crash and the
domain will hang.  (qemu-traditional is fine.)

It's too late in the release to do a proper fix, so we try to do
damage control.

hvmloader already has mechanisms to relocate memory to 64-bit space if
it can't make a big enough MMIO hole.  By default this is 2GiB; if we
just refuse to make the hole bigger if it will overlap with guest
memory, then the relocation will happen by default.

v5:
 - Update comment to not refer to "this series".
v4:
 - Wrap long line in libxl_dm.c
 - Fix comment
v3:
 - Fix polarity of comparison
 - Move diagnostic messages to another patch
 - Tested with xen platform pci device hacked to have different BAR sizes
   {256MiB, 1GiB} x {qemu-xen, qemu-traditional} x various memory
   configurations
 - Add comment explaining why we default to "allow"
 - Remove cast to bool
v2:
 - style fixes
 - fix and expand comment on the MMIO hole loop
 - use "%d" rather than "%s" -> (...)?"1":"0"
 - use bool instead of uint8_t
 - Move 64-bit bar relocate detection to another patch
 - Add more diagnostic messages

Signed-off-by: George Dunlap <george.dunlap@eu.citrix.com>
Acked-by: Ian Jackson <ian.jackson@eu.citrix.com>
CC: Ian Campbell <ian.campbell@citrix.com>
CC: Stefano Stabellini <stefano.stabellini@citrix.com>
CC: Hanweidong <hanweidong@huawei.com>
CC: Keir Fraser <keir@xen.org>
CC: Keir Fraser <keir@xen.org>

diff --git a/tools/firmware/hvmloader/pci.c b/tools/firmware/hvmloader/pci.c
index 9253b0b4c6e0fed2bbd12921bf536a5d8526bbf0..627e8cbcb6ead108ab714c747bec64e973391fbd 100644 (file)
--- a/tools/firmware/hvmloader/pci.c
+++ b/tools/firmware/hvmloader/pci.c
@@ -27,6 +27,8 @@
 
 #include <xen/memory.h>
 #include <xen/hvm/ioreq.h>
+#include <xen/hvm/hvm_xs_strings.h>
+#include <stdbool.h>
 
 unsigned long pci_mem_start = PCI_MEM_START;
 unsigned long pci_mem_end = PCI_MEM_END;
@@ -57,6 +59,32 @@ void pci_setup(void)
     } *bars = (struct bars *)scratch_start;
     unsigned int i, nr_bars = 0;
 
+    const char *s;
+    /*
+     * Do we allow hvmloader to relocate guest memory in order to
+     * increase the size of the lowmem MMIO hole?  Defaulting to 1
+     * here will mean that non-libxl toolstacks (including xend and
+     * home-grown ones) means that those using qemu-xen will still
+     * experience the memory relocation bug described below; but it
+     * also means that those using qemu-traditional will *not*
+     * experience any change; and it also means that there is a
+     * work-around for those using qemu-xen, namely switching to
+     * qemu-traditional.
+     *
+     * If we defaulted to 0, and failing to resize the hole caused any
+     * problems with qemu-traditional, then there is no work-around.
+     *
+     * Since xend can only use qemu-traditional, I think this is the
+     * option that will have the least impact.
+     */
+    bool allow_memory_relocate = 1;
+
+    s = xenstore_read(HVM_XS_ALLOW_MEMORY_RELOCATE, NULL);
+    if ( s )
+        allow_memory_relocate = strtoll(s, NULL, 0);
+    printf("Relocating guest memory for lowmem MMIO space %s\n",
+           allow_memory_relocate?"enabled":"disabled");
+
     /* Program PCI-ISA bridge with appropriate link routes. */
     isa_irq = 0;
     for ( link = 0; link < 4; link++ )
@@ -208,8 +236,25 @@ void pci_setup(void)
         pci_writew(devfn, PCI_COMMAND, cmd);
     }
 
-    while ( (mmio_total > (pci_mem_end - pci_mem_start)) &&
-            ((pci_mem_start << 1) != 0) )
+    /*
+     * At the moment qemu-xen can't deal with relocated memory regions.
+     * It's too close to the release to make a proper fix; for now,
+     * only allow the MMIO hole to grow large enough to move guest memory
+     * if we're running qemu-traditional.  Items that don't fit will be
+     * relocated into the 64-bit address space.
+     *
+     * This loop now does the following:
+     * - If allow_memory_relocate, increase the MMIO hole until it's
+     *   big enough, or until it's 2GiB
+     * - If !allow_memory_relocate, increase the MMIO hole until it's
+     *   big enough, or until it's 2GiB, or until it overlaps guest
+     *   memory
+     */
+    while ( (mmio_total > (pci_mem_end - pci_mem_start)) 
+            && ((pci_mem_start << 1) != 0)
+            && (allow_memory_relocate
+                || (((pci_mem_start << 1) >> PAGE_SHIFT)
+                    >= hvm_info->low_mem_pgend)) )
         pci_mem_start <<= 1;
 
     if ( mmio_total > (pci_mem_end - pci_mem_start) )

diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c
index ac1f90e53147d5815743fa37e8a3358fc18bc720..7e54c02a216e57125bfe0bd1decf6ec5466adb79 100644 (file)
--- a/tools/libxl/libxl_dm.c
+++ b/tools/libxl/libxl_dm.c
@@ -1154,6 +1154,14 @@ void libxl__spawn_local_dm(libxl__egc *egc, libxl__dm_spawn_state *dmss)
         libxl__xs_write(gc, XBT_NULL,
                         libxl__sprintf(gc, "%s/hvmloader/bios", path),
                         "%s", libxl_bios_type_to_string(b_info->u.hvm.bios));
+        /* Disable relocating memory to make the MMIO hole larger
+         * unless we're running qemu-traditional */
+        libxl__xs_write(gc, XBT_NULL,
+                        libxl__sprintf(gc,
+                                       "%s/hvmloader/allow-memory-relocate",
+                                       path),
+                        "%d",
+                        b_info->device_model_version==LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL);
         free(path);
     }
 

diff --git a/xen/include/public/hvm/hvm_xs_strings.h b/xen/include/public/hvm/hvm_xs_strings.h
index 9042303b7b92486e8bd0944e02db238ba0bd0f04..4de58816977025e6270b62b2ef3007d723e2cb3f 100644 (file)
--- a/xen/include/public/hvm/hvm_xs_strings.h
+++ b/xen/include/public/hvm/hvm_xs_strings.h
@@ -28,6 +28,7 @@
 #define HVM_XS_HVMLOADER               "hvmloader"
 #define HVM_XS_BIOS                    "hvmloader/bios"
 #define HVM_XS_GENERATION_ID_ADDRESS   "hvmloader/generation-id-address"
+#define HVM_XS_ALLOW_MEMORY_RELOCATE   "hvmloader/allow-memory-relocate"
 
 /* The following values allow additional ACPI tables to be added to the
  * virtual ACPI BIOS that hvmloader constructs. The values specify the guest