tools & docs: add tools and docs support for Intel CDP

This is the xl/xc changes to support Intel Code/Data Prioritization.
CAT xl commands to set/get CBMs are extended to support CDP.
Add new CDP options with CAT commands in xl interface man page.
Add description of CDP in xl-psr.markdown.

Signed-off-by: He Chen <he.chen@linux.intel.com>
Reviewed-by: Chao Peng <chao.p.peng@linux.intel.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
Reviewed-by: Wei Liu <wei.liu2@citrix.com>

diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index d0cd6128e750166feac56fdc3d87e3d075be7db8..4279c7c9e20b8feadb1276d03841ba29eb4216ea 100644 (file)
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -1518,6 +1518,13 @@ applications. In the Xen implementation, CAT is used to control cache allocation
 on VM basis. To enforce cache on a specific domain, just set capacity bitmasks
 (CBM) for the domain.
 
+Intel Broadwell and later server platforms also offer Code/Data Prioritization
+(CDP) for cache allocations, which support specifying code or data cache for
+applications. CDP is used on a per VM basis in the Xen implementation. To
+specify code or data CBM for the domain, CDP feature must be enabled and CBM
+type options need to be specified when setting CBM, and the type options (code
+and data) are mutually exclusive.
+
 =over 4
 
 =item B<psr-cat-cbm-set> [I<OPTIONS>] I<domain-id> I<cbm>
@@ -1533,6 +1540,14 @@ B<OPTIONS>
 
 Specify the socket to process, otherwise all sockets are processed.
 
+=item B<-c>, B<--code>
+
+Set code CBM when CDP is enabled.
+
+=item B<-d>, B<--data>
+
+Set data CBM when CDP is enabled.
+
 =back
 
 =item B<psr-cat-show> [I<domain-id>]

diff --git a/docs/misc/xl-psr.markdown b/docs/misc/xl-psr.markdown
index c32e25cb8e16a87439ab0caf2773e542be848cf7..c3c1e8eebbbcb7fbd9c6aae9b0c7143a5567d727 100644 (file)
--- a/docs/misc/xl-psr.markdown
+++ b/docs/misc/xl-psr.markdown
@@ -127,6 +127,59 @@ Per domain CBM settings can be shown by:
 
 `xl psr-cat-show`
 
+## Code and Data Prioritization (CDP)
+
+Code and Data Prioritization (CDP) Technology is an extension of CAT, which
+is available on Intel Broadwell and later server platforms. CDP enables
+isolation and separate prioritization of code and data fetches to the L3
+cache in a software configurable manner, which can enable workload
+prioritization and tuning of cache capacity to the characteristics of the
+workload. CDP extends Cache Allocation Technology (CAT) by providing
+separate code and data masks per Class of Service (COS).
+
+CDP can be enabled by adding `psr=cdp` to Xen command line.
+
+When CDP is enabled,
+
+ * the CAT masks are re-mapped into interleaved pairs of masks for data or
+   code fetches.
+
+ * the range of COS for CAT is re-indexed, with the lower-half of the COS
+   range available for CDP.
+
+CDP allows the OS or Hypervisor to partition cache allocation in a more
+fine-grained manner. Code cache and data cache can be specified independently.
+With CDP enabled, one COS corresponds to two CBMs (code CBM & data CBM),
+since the sum of CBMs is fixed, that means the number of available COSes
+will reduce by half when CDP is on.
+
+For more detailed information please refer to Intel SDM chapter
+"Platform Shared Resource Control: Cache Allocation Technology".
+
+The xl interfaces are the same with that of CAT. The difference is that
+CBM type can be passed as option to set code CBM or data CBM.
+
+When CDP is enabled, `-c` or `--code` option is available to set code CBM
+for the domain.
+
+When CDP is enabled, `-d` or `--data` option is available to set data CBM
+for the domain.
+
+If neither `-c` nor `-d` option is specified when CDP is on, the same code
+CBM and data CBM will be set for the domain. Passing both `-c` and `-d`
+options is invalid.
+
+Example:
+
+Setting code CBM for a domain:
+`xl psr-cat-cbm-set -c <domid> <cbm>`
+
+Setting data CBM for a domain:
+`xl psr-cat-cbm-set -d <domid> <cbm>`
+
+Setting the same code and data CBM for a domain:
+`xl psr-cat-cbm-set <domid> <cbm>`
+
 ## Reference
 
 [1] Intel SDM

diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h
index 3bfa00b5540df1aaa5a4f3a642290f01dc4f8f70..2fec1fb91e0401833b204155a3115d4e0024d46f 100644 (file)
--- a/tools/libxc/include/xenctrl.h
+++ b/tools/libxc/include/xenctrl.h
@@ -2827,7 +2827,9 @@ enum xc_psr_cmt_type {
 typedef enum xc_psr_cmt_type xc_psr_cmt_type;
 
 enum xc_psr_cat_type {
-    XC_PSR_CAT_L3_CBM = 1,
+    XC_PSR_CAT_L3_CBM      = 1,
+    XC_PSR_CAT_L3_CBM_CODE = 2,
+    XC_PSR_CAT_L3_CBM_DATA = 3,
 };
 typedef enum xc_psr_cat_type xc_psr_cat_type;
 
@@ -2853,7 +2855,8 @@ int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t domid,
                                xc_psr_cat_type type, uint32_t target,
                                uint64_t *data);
 int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
-                           uint32_t *cos_max, uint32_t *cbm_len);
+                           uint32_t *cos_max, uint32_t *cbm_len,
+                           bool *cdp_enabled);
 #endif
 
 #endif /* XENCTRL_H */

diff --git a/tools/libxc/xc_psr.c b/tools/libxc/xc_psr.c
index d8b3a5125cd29ecd897684ec9f9c42e1773b1e59..43b3286b784d13f665529bada2d0b8ff7225eb0f 100644 (file)
--- a/tools/libxc/xc_psr.c
+++ b/tools/libxc/xc_psr.c
@@ -260,6 +260,12 @@ int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t domid,
     case XC_PSR_CAT_L3_CBM:
         cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM;
         break;
+    case XC_PSR_CAT_L3_CBM_CODE:
+        cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_CODE;
+        break;
+    case XC_PSR_CAT_L3_CBM_DATA:
+        cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_DATA;
+        break;
     default:
         errno = EINVAL;
         return -1;
@@ -287,6 +293,12 @@ int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t domid,
     case XC_PSR_CAT_L3_CBM:
         cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM;
         break;
+    case XC_PSR_CAT_L3_CBM_CODE:
+        cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_CODE;
+        break;
+    case XC_PSR_CAT_L3_CBM_DATA:
+        cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_DATA;
+        break;
     default:
         errno = EINVAL;
         return -1;
@@ -306,7 +318,8 @@ int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t domid,
 }
 
 int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
-                           uint32_t *cos_max, uint32_t *cbm_len)
+                           uint32_t *cos_max, uint32_t *cbm_len,
+                           bool *cdp_enabled)
 {
     int rc;
     DECLARE_SYSCTL;
@@ -320,6 +333,8 @@ int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
     {
         *cos_max = sysctl.u.psr_cat_op.u.l3_info.cos_max;
         *cbm_len = sysctl.u.psr_cat_op.u.l3_info.cbm_len;
+        *cdp_enabled = sysctl.u.psr_cat_op.u.l3_info.flags &
+                       XEN_SYSCTL_PSR_CAT_L3_CDP;
     }
 
     return rc;

diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 615b1ded9d5b4cb47e7b832c6e250799be983f06..168fedd167185138336cd62701b7a8f82cc45466 100644 (file)
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -803,6 +803,13 @@ void libxl_mac_copy(libxl_ctx *ctx, libxl_mac *dst, libxl_mac *src);
  * If this is defined, the Cache Allocation Technology feature is supported.
  */
 #define LIBXL_HAVE_PSR_CAT 1
+
+/*
+ * LIBXL_HAVE_PSR_CDP
+ *
+ * If this is defined, the Code and Data Prioritization feature is supported.
+ */
+#define LIBXL_HAVE_PSR_CDP 1
 #endif
 
 /*

diff --git a/tools/libxl/libxl_psr.c b/tools/libxl/libxl_psr.c
index 000d748f1f450c3491e0669733be4aa9f372ad18..3d0dc61e92b1c373f68aa4a95d9660a4d6e2431a 100644 (file)
--- a/tools/libxl/libxl_psr.c
+++ b/tools/libxl/libxl_psr.c
@@ -87,6 +87,9 @@ static void libxl__psr_cat_log_err_msg(libxl__gc *gc, int err)
     case EEXIST:
         msg = "The same CBM is already set to this domain";
         break;
+    case ENXIO:
+        msg = "Unable to set code or data CBM when CDP is disabled";
+        break;
 
     default:
         libxl__psr_log_err_msg(gc, err);
@@ -363,7 +366,7 @@ int libxl_psr_cat_get_l3_info(libxl_ctx *ctx, libxl_psr_cat_info **info,
     libxl_for_each_set_bit(socketid, socketmap) {
         ptr[i].id = socketid;
         if (xc_psr_cat_get_l3_info(ctx->xch, socketid, &ptr[i].cos_max,
-                                   &ptr[i].cbm_len)) {
+                                   &ptr[i].cbm_len, &ptr[i].cdp_enabled)) {
             libxl__psr_cat_log_err_msg(gc, errno);
             rc = ERROR_FAIL;
             free(ptr);

diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index d6ef9a29cf6c31597c155665c0c4cc9848f0e6bf..4d78f86233fd2e32c8f2e9c300d10e6f481e62a4 100644 (file)
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -797,10 +797,13 @@ libxl_psr_cmt_type = Enumeration("psr_cmt_type", [
 libxl_psr_cbm_type = Enumeration("psr_cbm_type", [
     (0, "UNKNOWN"),
     (1, "L3_CBM"),
+    (2, "L3_CBM_CODE"),
+    (3, "L3_CBM_DATA"),
     ])
 
 libxl_psr_cat_info = Struct("psr_cat_info", [
     ("id", uint32),
     ("cos_max", uint32),
     ("cbm_len", uint32),
+    ("cdp_enabled", bool),
     ])

diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 365798ba1f8799b97584d2df66f356fe5b1924fb..646b2817d22fed2e3bd8c430da8773e724e734d9 100644 (file)
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -8434,6 +8434,8 @@ static int psr_cat_hwinfo(void)
         }
         printf("%-16s: %u\n", "Socket ID", info[i].id);
         printf("%-16s: %uKB\n", "L3 Cache", l3_cache_size);
+        printf("%-16s: %s\n", "CDP Status",
+               info[i].cdp_enabled ? "Enabled" : "Disabled");
         printf("%-16s: %u\n", "Maximum COS", info[i].cos_max);
         printf("%-16s: %u\n", "CBM length", info[i].cbm_len);
         printf("%-16s: %#llx\n", "Default CBM",
@@ -8445,29 +8447,47 @@ out:
     return rc;
 }
 
-static void psr_cat_print_one_domain_cbm(uint32_t domid, uint32_t socketid)
+static void psr_cat_print_one_domain_cbm_type(uint32_t domid, uint32_t socketid,
+                                              libxl_psr_cbm_type type)
 {
-    char *domain_name;
     uint64_t cbm;
 
+    if (!libxl_psr_cat_get_cbm(ctx, domid, type, socketid, &cbm))
+        printf("%#16"PRIx64, cbm);
+    else
+        printf("%16s", "error");
+}
+
+static void psr_cat_print_one_domain_cbm(uint32_t domid, uint32_t socketid,
+                                         bool cdp_enabled)
+{
+    char *domain_name;
+
     domain_name = libxl_domid_to_name(ctx, domid);
     printf("%5d%25s", domid, domain_name);
     free(domain_name);
 
-    if (!libxl_psr_cat_get_cbm(ctx, domid, LIBXL_PSR_CBM_TYPE_L3_CBM,
-                               socketid, &cbm))
-         printf("%#16"PRIx64, cbm);
+    if (!cdp_enabled) {
+        psr_cat_print_one_domain_cbm_type(domid, socketid,
+                                          LIBXL_PSR_CBM_TYPE_L3_CBM);
+    } else {
+        psr_cat_print_one_domain_cbm_type(domid, socketid,
+                                          LIBXL_PSR_CBM_TYPE_L3_CBM_CODE);
+        psr_cat_print_one_domain_cbm_type(domid, socketid,
+                                          LIBXL_PSR_CBM_TYPE_L3_CBM_DATA);
+    }
 
     printf("\n");
 }
 
-static int psr_cat_print_domain_cbm(uint32_t domid, uint32_t socketid)
+static int psr_cat_print_domain_cbm(uint32_t domid, uint32_t socketid,
+                                    bool cdp_enabled)
 {
     int i, nr_domains;
     libxl_dominfo *list;
 
     if (domid != INVALID_DOMID) {
-        psr_cat_print_one_domain_cbm(domid, socketid);
+        psr_cat_print_one_domain_cbm(domid, socketid, cdp_enabled);
         return 0;
     }
 
@@ -8477,7 +8497,7 @@ static int psr_cat_print_domain_cbm(uint32_t domid, uint32_t socketid)
     }
 
     for (i = 0; i < nr_domains; i++)
-        psr_cat_print_one_domain_cbm(list[i].domid, socketid);
+        psr_cat_print_one_domain_cbm(list[i].domid, socketid, cdp_enabled);
     libxl_dominfo_list_free(list, nr_domains);
 
     return 0;
@@ -8498,9 +8518,12 @@ static int psr_cat_print_socket(uint32_t domid, libxl_psr_cat_info *info)
     printf("%-16s: %u\n", "Socket ID", info->id);
     printf("%-16s: %uKB\n", "L3 Cache", l3_cache_size);
     printf("%-16s: %#llx\n", "Default CBM", (1ull << info->cbm_len) - 1);
-    printf("%5s%25s%16s\n", "ID", "NAME", "CBM");
+    if (info->cdp_enabled)
+        printf("%5s%25s%16s%16s\n", "ID", "NAME", "CBM (code)", "CBM (data)");
+    else
+        printf("%5s%25s%16s\n", "ID", "NAME", "CBM");
 
-    return psr_cat_print_domain_cbm(domid, info->id);
+    return psr_cat_print_domain_cbm(domid, info->id, info->cdp_enabled);
 }
 
 static int psr_cat_show(uint32_t domid)
@@ -8529,9 +8552,10 @@ out:
 int main_psr_cat_cbm_set(int argc, char **argv)
 {
     uint32_t domid;
-    libxl_psr_cbm_type type = LIBXL_PSR_CBM_TYPE_L3_CBM;
+    libxl_psr_cbm_type type;
     uint64_t cbm;
     int ret, opt = 0;
+    int opt_data = 0, opt_code = 0;
     libxl_bitmap target_map;
     char *value;
     libxl_string_list socket_list;
@@ -8540,13 +8564,15 @@ int main_psr_cat_cbm_set(int argc, char **argv)
 
     static struct option opts[] = {
         {"socket", 1, 0, 's'},
+        {"data", 0, 0, 'd'},
+        {"code", 0, 0, 'c'},
         COMMON_LONG_OPTS
     };
 
     libxl_socket_bitmap_alloc(ctx, &target_map, 0);
     libxl_bitmap_set_none(&target_map);
 
-    SWITCH_FOREACH_OPT(opt, "s:", opts, "psr-cat-cbm-set", 2) {
+    SWITCH_FOREACH_OPT(opt, "s:cd", opts, "psr-cat-cbm-set", 2) {
     case 's':
         trim(isspace, optarg, &value);
         split_string_into_string_list(value, ",", &socket_list);
@@ -8560,6 +8586,23 @@ int main_psr_cat_cbm_set(int argc, char **argv)
         libxl_string_list_dispose(&socket_list);
         free(value);
         break;
+    case 'd':
+        opt_data = 1;
+        break;
+    case 'c':
+        opt_code = 1;
+        break;
+    }
+
+    if (opt_data && opt_code) {
+        fprintf(stderr, "Cannot handle -c and -d at the same time\n");
+        return -1;
+    } else if (opt_data) {
+        type = LIBXL_PSR_CBM_TYPE_L3_CBM_DATA;
+    } else if (opt_code) {
+        type = LIBXL_PSR_CBM_TYPE_L3_CBM_CODE;
+    } else {
+        type = LIBXL_PSR_CBM_TYPE_L3_CBM;
     }
 
     if (libxl_bitmap_is_empty(&target_map))

diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c
index 0071f127ea2f76a3c38f76e0f5eed1a190c90516..fdc1ac6156a76e1e5fd7272aac6ae72ef972d3d6 100644 (file)
--- a/tools/libxl/xl_cmdtable.c
+++ b/tools/libxl/xl_cmdtable.c
@@ -543,6 +543,8 @@ struct cmd_spec cmd_table[] = {
       "Set cache capacity bitmasks(CBM) for a domain",
       "[options] <Domain> <CBM>",
       "-s <socket>       Specify the socket to process, otherwise all sockets are processed\n"
+      "-c                Set code CBM if CDP is supported\n"
+      "-d                Set data CBM if CDP is supported\n"
     },
     { "psr-cat-show",
       &main_psr_cat_show, 0, 1,