xen/balloon: make use of generic balloon driver

Let generic balloon driver handle high memory and movable memory
allocation. Those pages are subject to page migration, so that we can
have balloon page compaction thread de-fragment all those pages for us.

The page migration function is not yet implemented. A stub is created to
always return -EAGAIN, indicating failure in page migration, which
prevents balloon compaction thread from doing any work. This patch is
arranged like this to ease review.

Signed-off-by: Wei Liu <wei.liu2@citrix.com>

diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
index 4f46545b390559803dbdaf0718418de09a613f37..24efdf6996fbee9213d1e1c1fe716e8d61356127 100644 (file)
--- a/drivers/xen/balloon.c
+++ b/drivers/xen/balloon.c
@@ -95,11 +95,6 @@ static DEFINE_PER_CPU(struct page *, balloon_scratch_page);
 static void balloon_process(struct work_struct *work);
 static DECLARE_DELAYED_WORK(balloon_worker, balloon_process);
 
-/* When ballooning out (allocating memory to return to Xen) we don't really
-   want the kernel to try too hard since that can trigger the oom killer. */
-#define GFP_BALLOON \
-       (GFP_HIGHUSER | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC)
-
 static void scrub_page(struct page *page)
 {
 #ifdef CONFIG_XEN_SCRUB_PAGES
@@ -115,21 +110,27 @@ static inline void update_balloon_stats(struct page *page, int count)
                xen_balloon.balloon_stats.balloon_low += count;
 }
 
-/* balloon_append: add the given page to the balloon. */
-static void __balloon_append(struct page *page)
+/* balloon_append: add the given page to the balloon. "managed"
+ * indicates whether we should add the page to our own list.
+ */
+static void __balloon_append(struct page *page, bool managed)
 {
-       /* Lowmem is re-populated first, so highmem pages go at list tail. */
-       if (PageHighMem(page))
-               list_add_tail(&page->lru, &xen_balloon.ballooned_pages);
-       else
-               list_add(&page->lru, &xen_balloon.ballooned_pages);
+       if (managed) {
+               /* Lowmem is re-populated first, so highmem pages go
+                * at list tail. */
+               if (PageHighMem(page))
+                       list_add_tail(&page->lru,
+                                     &xen_balloon.ballooned_pages);
+               else
+                       list_add(&page->lru, &xen_balloon.ballooned_pages);
+       }
 
        update_balloon_stats(page, 1);
 }
 
-static void balloon_append(struct page *page)
+static void balloon_append(struct page *page, bool managed)
 {
-       __balloon_append(page);
+       __balloon_append(page, managed);
        adjust_managed_page_count(page, -1);
 }
 
@@ -244,7 +245,7 @@ static void xen_online_page(struct page *page)
 
        mutex_lock(&xen_balloon.balloon_mutex);
 
-       __balloon_append(page);
+       __balloon_append(page, true);
 
        if (xen_balloon.balloon_stats.hotplug_pages)
                --xen_balloon.balloon_stats.hotplug_pages;
@@ -296,12 +297,31 @@ static enum bp_state reserve_additional_memory(long credit)
 }
 #endif /* CONFIG_XEN_BALLOON_MEMORY_HOTPLUG */
 
+/* This is used to reinsert pages back to generic balloon driver. We
+ * need to use balloon_page_insert. list_splice is not sufficient.
+ */
+static inline void __reinsert_balloon_pages(struct list_head *head)
+{
+       unsigned long flags;
+       struct page *page, *n;
+
+       spin_lock_irqsave(&xen_balloon.xb_dev_info->pages_lock, flags);
+       list_for_each_entry_safe_reverse(page, n, head, lru)
+               balloon_page_insert(page, xen_balloon.xb_dev_info->mapping,
+                                   &xen_balloon.xb_dev_info->pages);
+       spin_unlock_irqrestore(&xen_balloon.xb_dev_info->pages_lock, flags);
+}
+
+/* This function will always try to fill in pages managed by Xen
+ * balloon driver, then pages managed by generic balloon driver.
+ */
 static enum bp_state increase_reservation(unsigned long nr_pages)
 {
        int rc;
        unsigned long  pfn, i;
        struct page   *page;
        LIST_HEAD(queue);
+       bool xen_pages;
        struct xen_memory_reservation reservation = {
                .address_bits = 0,
                .extent_order = 0,
@@ -322,9 +342,19 @@ static enum bp_state increase_reservation(unsigned long nr_pages)
        if (nr_pages > ARRAY_SIZE(frame_list))
                nr_pages = ARRAY_SIZE(frame_list);
 
+       /* We always try to fill in pages managed by Xen balloon
+        * driver first. And we separate the process to fill in Xen
+        * balloon driver pages v.s. generic balloon driver in
+        * different rounds for the sake of simplicity.
+        */
+       xen_pages = !list_empty(&xen_balloon.ballooned_pages);
+
        /* First step: grab all pages we need to balloon in */
        for (i = 0; i < nr_pages; i++) {
-               page = balloon_retrieve(false);
+               if (xen_pages)
+                       page = balloon_retrieve(false);
+               else
+                       page = balloon_page_dequeue(xen_balloon.xb_dev_info, false);
                if (!page) {
                        nr_pages = i;
                        break;
@@ -369,7 +399,10 @@ static enum bp_state increase_reservation(unsigned long nr_pages)
 #endif
 
                /* Relinquish the page back to the allocator. */
-               __free_reserved_page(page);
+               if (xen_pages)
+                       __free_reserved_page(page);
+               else
+                       balloon_page_free(page);
 
                /* We only account for those pages that have been populated. */
                update_balloon_stats(page, -1);
@@ -381,11 +414,26 @@ static enum bp_state increase_reservation(unsigned long nr_pages)
 
 move_pages_back:
        /* Final step: move back any unpopulated pages to balloon driver */
-       list_splice_init(&queue, &xen_balloon.ballooned_pages);
+       if (!list_empty(&queue)) {
+               if (xen_pages)
+                       list_splice(&queue, &xen_balloon.ballooned_pages);
+               else
+                       __reinsert_balloon_pages(&queue);
+       }
        return rc;
 }
 
-static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
+/* decrease_reservation:
+ * core_driver == true:
+ *   Page is not movable and managed by Xen balloon driver. Honor gfp
+ *   when allocating pages.
+ * core_driver == false:
+ *   Page is movable and managed by generic balloon driver, subject to
+ *   migration. Gfp is ignored as page is allocated by generic balloon
+ *   driver.
+ */
+static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp,
+                                         bool core_driver)
 {
        enum bp_state state = BP_DONE;
        unsigned long  pfn, i;
@@ -411,7 +459,11 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
                nr_pages = ARRAY_SIZE(frame_list);
 
        for (i = 0; i < nr_pages; i++) {
-               page = alloc_page(gfp);
+               if (core_driver)
+                       page = alloc_page(gfp);
+               else
+                       page = balloon_page_enqueue(xen_balloon.xb_dev_info);
+
                if (page == NULL) {
                        nr_pages = i;
                        state = BP_EAGAIN;
@@ -459,7 +511,7 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
                }
 #endif
 
-               balloon_append(page);
+               balloon_append(page, core_driver);
        }
 
        flush_tlb_all();
@@ -498,7 +550,7 @@ static void balloon_process(struct work_struct *work)
                }
 
                if (credit < 0)
-                       state = decrease_reservation(-credit, GFP_BALLOON);
+                       state = decrease_reservation(-credit, 0, false);
 
                state = update_schedule(state);
 
@@ -557,10 +609,10 @@ int alloc_xenballooned_pages(int nr_pages, struct page **pages, bool highmem)
                        adjust_managed_page_count(page, 1);
                } else {
                        enum bp_state st;
+                       gfp_t gfp = highmem ? GFP_HIGHUSER : GFP_USER;
                        if (page)
-                               balloon_append(page);
-                       st = decrease_reservation(nr_pages - pgno,
-                                       highmem ? GFP_HIGHUSER : GFP_USER);
+                               balloon_append(page, true);
+                       st = decrease_reservation(nr_pages - pgno, gfp, true);
                        if (st != BP_DONE)
                                goto out_undo;
                }
@@ -569,7 +621,7 @@ int alloc_xenballooned_pages(int nr_pages, struct page **pages, bool highmem)
        return 0;
  out_undo:
        while (pgno)
-               balloon_append(pages[--pgno]);
+               balloon_append(pages[--pgno], true);
        /* Free the memory back to the kernel soon */
        schedule_delayed_work(&balloon_worker, 0);
        mutex_unlock(&xen_balloon.balloon_mutex);
@@ -591,7 +643,7 @@ void free_xenballooned_pages(int nr_pages, struct page **pages)
 
        for (i = 0; i < nr_pages; i++) {
                if (pages[i])
-                       balloon_append(pages[i]);
+                       balloon_append(pages[i], true);
        }
 
        /* The balloon may be too large now. Shrink it if needed. */
@@ -620,7 +672,7 @@ static void __init balloon_add_region(unsigned long start_pfn,
                /* totalram_pages and totalhigh_pages do not
                   include the boot-time balloon extension, so
                   don't subtract from it. */
-               __balloon_append(page);
+               __balloon_append(page, true);
        }
 }
 
@@ -658,9 +710,25 @@ static struct notifier_block balloon_cpu_notifier = {
        .notifier_call  = balloon_cpu_notify,
 };
 
+static const struct address_space_operations xen_balloon_aops;
+#ifdef CONFIG_BALLOON_COMPACTION
+static int xen_balloon_migratepage(struct address_space *mapping,
+                                  struct page *newpage, struct page *page,
+                                  enum migrate_mode mode)
+{
+       return -EAGAIN;
+}
+
+static const struct address_space_operations xen_balloon_aops = {
+       .migratepage = xen_balloon_migratepage,
+};
+#endif /* CONFIG_BALLOON_COMPACTION */
+
 static int __init balloon_init(void)
 {
        int i, cpu;
+       struct address_space *mapping;
+       int err = 0;
 
        if (!xen_domain())
                return -ENODEV;
@@ -683,6 +751,27 @@ static int __init balloon_init(void)
 
        memset(&xen_balloon, 0, sizeof(xen_balloon));
 
+       /* Allocate core balloon driver which is in charge of movable
+        * pages.
+        */
+       xen_balloon.xb_dev_info = balloon_devinfo_alloc(&xen_balloon);
+
+       if (IS_ERR(xen_balloon.xb_dev_info)) {
+               err = PTR_ERR(xen_balloon.xb_dev_info);
+               goto out_err;
+       }
+
+       /* Check CONFIG_BALLOON_COMPACTION */
+       if (balloon_compaction_check()) {
+               mapping = balloon_mapping_alloc(xen_balloon.xb_dev_info,
+                                               &xen_balloon_aops);
+               if (IS_ERR(mapping)) {
+                       err = PTR_ERR(mapping);
+                       goto out_free_xb_dev_info;
+               }
+               pr_info("balloon page compaction enabled\n");
+       }
+
        mutex_init(&xen_balloon.balloon_mutex);
 
        INIT_LIST_HEAD(&xen_balloon.ballooned_pages);
@@ -718,6 +807,11 @@ static int __init balloon_init(void)
                                           PFN_DOWN(xen_extra_mem[i].size));
 
        return 0;
+
+out_free_xb_dev_info:
+       balloon_devinfo_free(xen_balloon.xb_dev_info);
+out_err:
+       return err;
 }
 
 subsys_initcall(balloon_init);

diff --git a/include/xen/balloon.h b/include/xen/balloon.h
index 1d7efae2434246bdef7caff66e70dd09cc27c58e..f35b712025e9966b789344734626eed87b12db0a 100644 (file)
--- a/include/xen/balloon.h
+++ b/include/xen/balloon.h
@@ -1,6 +1,7 @@
 /******************************************************************************
  * Xen balloon functionality
  */
+#include <linux/balloon_compaction.h>
 
 #define RETRY_UNLIMITED        0
 
@@ -32,6 +33,9 @@ struct xen_balloon {
 
        /* Memory statistic */
        struct balloon_stats balloon_stats;
+
+       /* Core balloon driver - in charge of movable pages */
+       struct balloon_dev_info *xb_dev_info;
 };
 
 extern struct xen_balloon xen_balloon;