xl: Implement XENMEM_claim_pages support via 'claim_mode' global config

The XENMEM_claim_pages hypercall operates per domain and it should be
used system wide. As such this patch introduces a global configuration
option 'claim_mode' that by default is disabled.

If this option is enabled then when a guest is created there will be an
guarantee that there is memory available for the guest. This is an
particularly acute problem on hosts with memory over-provisioned guests
that use tmem and have self-balloon enabled (which is the default option
for them). The self-balloon mechanism can deflate/inflate the balloon
quickly and the amount of free memory (which 'xl info' can show) is stale
the moment it is printed. When claim is enabled a reservation for the
amount of memory ('memory' in guest config) is set, which is then reduced
as the domain's memory is populated and eventually reaches zero.

If the reservation cannot be meet the guest creation fails immediately
instead of taking seconds/minutes (depending on the size of the guest)
while the guest is populated.

Note that to enable tmem type guests, one needs to provide 'tmem' on the
Xen hypervisor argument and as well on the Linux kernel command line.

There are two boolean options:

(0) No claim is made. Memory population during guest creation will be
attempted as normal and may fail due to memory exhaustion.

(1) Normal memory and freeable pool of ephemeral pages (tmem) is used when
calculating whether there is enough memory free to launch a guest.
This guarantees immediate feedback whether the guest can be launched due
to memory exhaustion (which can take a long time to find out if launching
massively huge guests) and in parallel.

[v1: Removed own claim_mode type, using just bool, improved docs, all per
Ian's suggestion]
[v2: Updated the comments]
[v3: Rebase on top 733b9c524dbc2bec318bfc3588ed1652455d30ec (xl: add vif.default.script)]
[v4: Fixed up comments]
[v5: s/global_claim_mode/claim_mode/]
[v6: Ian Jackson's feedback: use libxl_defbool, better comments, etc]
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Acked-by: Ian Jackson <ian.jackson@eu.citrix.com>

diff --git a/docs/man/xl.conf.pod.5 b/docs/man/xl.conf.pod.5
index aaf8da159841251507a9beb112b7d3261d0273b7..c4072aa7e475e9a6a110680e4347bd4b3e3d0819 100644 (file)
--- a/docs/man/xl.conf.pod.5
+++ b/docs/man/xl.conf.pod.5
@@ -115,6 +115,47 @@ Configures the name of the first block device to be used for temporary
 block device allocations by the toolstack.
 The default choice is "xvda".
 
+=item B<claim_mode=BOOLEAN>
+
+If this option is enabled then when a guest is created there will be an
+guarantee that there is memory available for the guest. This is an
+particularly acute problem on hosts with memory over-provisioned guests
+that use tmem and have self-balloon enabled (which is the default
+option). The self-balloon mechanism can deflate/inflate the balloon
+quickly and the amount of free memory (which C<xl info> can show) is
+stale the moment it is printed. When claim is enabled a reservation for
+the amount of memory (see 'memory' in xl.conf(5)) is set, which is then
+reduced as the domain's memory is populated and eventually reaches zero.
+
+If the reservation cannot be meet the guest creation fails immediately
+instead of taking seconds/minutes (depending on the size of the guest)
+while the guest is populated.
+
+Note that to enable tmem type guests, one needs to provide C<tmem> on the
+Xen hypervisor argument and as well on the Linux kernel command line.
+
+Note that the claim call is not attempted if C<superpages> option is
+used in the guest config (see xl.cfg(5)).
+
+Default: C<0>
+
+=over 4
+
+=item C<0>
+
+No claim is made. Memory population during guest creation will be
+attempted as normal and may fail due to memory exhaustion.
+
+=item C<1>
+
+Normal memory and freeable pool of ephemeral pages (tmem) is used when
+calculating whether there is enough memory free to launch a guest.
+This guarantees immediate feedback whether the guest can be launched due
+to memory exhaustion (which can take a long time to find out if launching
+massively huge guests).
+
+=back
+
 =back
 
 =head1 SEE ALSO

diff --git a/tools/examples/xl.conf b/tools/examples/xl.conf
index 50cba2bf37c79da873e33350a8be1ee5b9c729eb..9402c3f8bc4e3cfb4dcaa65948443d050a431384 100644 (file)
--- a/tools/examples/xl.conf
+++ b/tools/examples/xl.conf
@@ -27,3 +27,9 @@
 
 # default bridge device to use with vif-bridge hotplug scripts
 #vif.default.bridge="xenbr0"
+
+# Reserve a claim of memory when launching a guest. This guarantees immediate
+# feedback whether the guest can be launched due to memory exhaustion
+# (which can take a long time to find out if launching huge guests).
+# see xl.conf(5) for details.
+#claim_mode=0

diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index d18d22c0c181fe963a9b43dd1f93cf6075464bbf..e4a4ab2c7fe687b6bbd938cb3e8305513ebb9fd8 100644 (file)
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -595,7 +595,6 @@ int libxl_wait_for_free_memory(libxl_ctx *ctx, uint32_t domid, uint32_t memory_k
 /* wait for the memory target of a domain to be reached */
 int libxl_wait_for_memory_target(libxl_ctx *ctx, uint32_t domid, int wait_secs);
 
-
 int libxl_vncviewer_exec(libxl_ctx *ctx, uint32_t domid, int autopass);
 int libxl_console_exec(libxl_ctx *ctx, uint32_t domid, int cons_num, libxl_console_type type);
 /* libxl_primary_console_exec finds the domid and console number

diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 30a4507dfc012406407488446622fbb82eb2545b..ae72f213f8c99ce53e1bf2021eac9d018b17c68f 100644 (file)
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -196,6 +196,8 @@ int libxl__domain_build_info_setdefault(libxl__gc *gc,
     if (b_info->target_memkb == LIBXL_MEMKB_DEFAULT)
         b_info->target_memkb = b_info->max_memkb;
 
+    libxl_defbool_setdefault(&b_info->claim_mode, false);
+
     libxl_defbool_setdefault(&b_info->localtime, false);
 
     libxl_defbool_setdefault(&b_info->disable_migrate, false);

diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 2dd429f6390e006e0ca6cbb785683baeb5d14b3e..92a662833e4fc143c69cc5772ac8e01907b2d66b 100644 (file)
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -371,6 +371,7 @@ int libxl__build_pv(libxl__gc *gc, uint32_t domid,
     dom->console_domid = state->console_domid;
     dom->xenstore_evtchn = state->store_port;
     dom->xenstore_domid = state->store_domid;
+    dom->claim_enabled = libxl_defbool_val(info->claim_mode);
 
     if ( (ret = xc_dom_boot_xen_init(dom, ctx->xch, domid)) != 0 ) {
         LOGE(ERROR, "xc_dom_boot_xen_init failed");
@@ -605,7 +606,7 @@ int libxl__build_hvm(libxl__gc *gc, uint32_t domid,
      */
     args.mem_size = (uint64_t)(info->max_memkb - info->video_memkb) << 10;
     args.mem_target = (uint64_t)(info->target_memkb - info->video_memkb) << 10;
-
+    args.claim_enabled = libxl_defbool_val(info->claim_mode);
     if (libxl__domain_firmware(gc, info, &args)) {
         LOG(ERROR, "initializing domain firmware failed");
         goto out;

diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 6cb6de613cbe4525fc9b399c6c80f934aa7cca47..4d8f7cd18e752c47e1d1b47e96d13f53e074b4d0 100644 (file)
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -293,7 +293,7 @@ libxl_domain_build_info = Struct("domain_build_info",[
     ("ioports",          Array(libxl_ioport_range, "num_ioports")),
     ("irqs",             Array(uint32, "num_irqs")),
     ("iomem",            Array(libxl_iomem_range, "num_iomem")),
-
+    ("claim_mode",          libxl_defbool),
     ("u", KeyedUnion(None, libxl_domain_type, "type",
                 [("hvm", Struct(None, [("firmware",         string),
                                        ("bios",             libxl_bios_type),

diff --git a/tools/libxl/xl.c b/tools/libxl/xl.c
index 16cd3f30027d8c5157cf3ae1afc32d5a1d514842..3c141bfdaaf61ed05ff7bbaa6c3a51ba5ef43b9b 100644 (file)
--- a/tools/libxl/xl.c
+++ b/tools/libxl/xl.c
@@ -46,6 +46,7 @@ char *default_vifscript = NULL;
 char *default_bridge = NULL;
 char *default_gatewaydev = NULL;
 enum output_format default_output_format = OUTPUT_FORMAT_JSON;
+libxl_defbool claim_mode;
 
 static xentoollog_level minmsglevel = XTL_PROGRESS;
 
@@ -168,6 +169,10 @@ static void parse_global_config(const char *configfile,
     }
     if (!xlu_cfg_get_string (config, "blkdev_start", &buf, 0))
         blkdev_start = strdup(buf);
+
+    libxl_defbool_setdefault(&claim_mode, false);
+    (void)xlu_cfg_get_defbool (config, "claim_mode", &claim_mode, 0);
+
     xlu_cfg_destroy(config);
 }
 

diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h
index b881f92269e7ef83336a277bed1ed436beff5fcd..4c5e5d19a9b9750657eaf38a02345d92098a88f0 100644 (file)
--- a/tools/libxl/xl.h
+++ b/tools/libxl/xl.h
@@ -145,6 +145,7 @@ int xl_child_pid(xlchildnum); /* returns 0 if child struct is not in use */
 extern int autoballoon;
 extern int run_hotplug_scripts;
 extern int dryrun_only;
+extern libxl_defbool claim_mode;
 extern char *lockfile;
 extern char *default_vifscript;
 extern char *default_bridge;

diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 61f7b968243b81dd367709f0895acacba8ed38bf..5a0506f1e4e50644f286b7c5c4a8407096434a67 100644 (file)
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -757,6 +757,8 @@ static void parse_config_data(const char *config_source,
     if (!xlu_cfg_get_long (config, "maxmem", &l, 0))
         b_info->max_memkb = l * 1024;
 
+    b_info->claim_mode = claim_mode;
+
     if (xlu_cfg_get_string (config, "on_poweroff", &buf, 0))
         buf = "destroy";
     if (!parse_action_on_shutdown(buf, &d_config->on_poweroff)) {