=back
-=head1 CACHE MONITORING TECHNOLOGY
+=head1 PLATFORM SHARED RESOURCE MONITORING/CONTROL
+
+Intel Haswell and later server platforms offer shared resource monitoring
+and control technologies. The availability of these technologies and the
+hardware capabilities can be shown with B<psr-hwinfo>.
+
+=over 4
+
+=item B<psr-hwinfo> [I<OPTIONS>]
+
+Show Platform Shared Resource (PSR) hardware information.
+
+B<OPTIONS>
+
+=over 4
+
+=item B<-m>, B<--cmt>
+
+Show Cache Monitoring Technology (CMT) hardware information.
+
+=item B<-a>, B<--cat>
+
+Show Cache Allocation Technology (CAT) hardware information.
+
+=back
+
+=back
+
+=head2 CACHE MONITORING TECHNOLOGY
Intel Haswell and later server platforms offer monitoring capability in each
logical processor to measure specific platform shared resource metric, for
-example, L3 cache occupancy. In Xen implementation, the monitoring granularity
-is domain level. To monitor a specific domain, just attach the domain id with
-the monitoring service. When the domain doesn't need to be monitored any more,
-detach the domain id from the monitoring service.
+example, L3 cache occupancy. In the Xen implementation, the monitoring
+granularity is domain level. To monitor a specific domain, just attach the
+domain id with the monitoring service. When the domain doesn't need to be
+monitored any more, detach the domain id from the monitoring service.
Intel Broadwell and later server platforms also offer total/local memory
bandwidth monitoring. Xen supports per-domain monitoring for these two
additional monitoring types. Both memory bandwidth monitoring and L3 cache
occupancy monitoring share the same set of underlying monitoring service. Once
-a domain is attached to the monitoring service, monitoring data can be showed
+a domain is attached to the monitoring service, monitoring data can be shown
for any of these monitoring types.
=over 4
-=item B<psr-hwinfo>
-
-Show CMT hardware information.
-
=item B<psr-cmt-attach> [I<domain-id>]
attach: Attach the platform shared resource monitoring service to a domain.
=back
+=head2 CACHE ALLOCATION TECHNOLOGY
+
+Intel Broadwell and later server platforms offer capabilities to configure and
+make use of the Cache Allocation Technology (CAT) mechanisms, which enable more
+cache resources (i.e. L3 cache) to be made available for high priority
+applications. In the Xen implementation, CAT is used to control cache allocation
+on VM basis. To enforce cache on a specific domain, just set capacity bitmasks
+(CBM) for the domain.
+
+=over 4
+
+=item B<psr-cat-cbm-set> [I<OPTIONS>] I<domain-id> I<cbm>
+
+Set cache capacity bitmasks(CBM) for a domain.
+
+B<OPTIONS>
+
+=over 4
+
+=item B<-s SOCKET>, B<--socket=SOCKET>
+
+Specify the socket to process, otherwise all sockets are processed.
+
+=back
+
+=item B<psr-cat-show> [I<domain-id>]
+
+Show CAT settings for a certain domain or all domains.
+
+=back
+
=head1 TO BE DOCUMENTED
We need better documentation for:
XC_PSR_CMT_LOCAL_MEM_COUNT,
};
typedef enum xc_psr_cmt_type xc_psr_cmt_type;
+
+enum xc_psr_cat_type {
+ XC_PSR_CAT_L3_CBM = 1,
+};
+typedef enum xc_psr_cat_type xc_psr_cat_type;
+
int xc_psr_cmt_attach(xc_interface *xch, uint32_t domid);
int xc_psr_cmt_detach(xc_interface *xch, uint32_t domid);
int xc_psr_cmt_get_domain_rmid(xc_interface *xch, uint32_t domid,
uint32_t psr_cmt_type, uint64_t *monitor_data,
uint64_t *tsc);
int xc_psr_cmt_enabled(xc_interface *xch);
+
+int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t domid,
+ xc_psr_cat_type type, uint32_t target,
+ uint64_t data);
+int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t domid,
+ xc_psr_cat_type type, uint32_t target,
+ uint64_t *data);
+int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
+ uint32_t *cos_max, uint32_t *cbm_len);
#endif
#endif /* XENCTRL_H */
return 0;
}
+int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t domid,
+ xc_psr_cat_type type, uint32_t target,
+ uint64_t data)
+{
+ DECLARE_DOMCTL;
+ uint32_t cmd;
+
+ switch ( type )
+ {
+ case XC_PSR_CAT_L3_CBM:
+ cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM;
+ break;
+ default:
+ errno = EINVAL;
+ return -1;
+ }
+
+ domctl.cmd = XEN_DOMCTL_psr_cat_op;
+ domctl.domain = (domid_t)domid;
+ domctl.u.psr_cat_op.cmd = cmd;
+ domctl.u.psr_cat_op.target = target;
+ domctl.u.psr_cat_op.data = data;
+
+ return do_domctl(xch, &domctl);
+}
+
+int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t domid,
+ xc_psr_cat_type type, uint32_t target,
+ uint64_t *data)
+{
+ int rc;
+ DECLARE_DOMCTL;
+ uint32_t cmd;
+
+ switch ( type )
+ {
+ case XC_PSR_CAT_L3_CBM:
+ cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM;
+ break;
+ default:
+ errno = EINVAL;
+ return -1;
+ }
+
+ domctl.cmd = XEN_DOMCTL_psr_cat_op;
+ domctl.domain = (domid_t)domid;
+ domctl.u.psr_cat_op.cmd = cmd;
+ domctl.u.psr_cat_op.target = target;
+
+ rc = do_domctl(xch, &domctl);
+
+ if ( !rc )
+ *data = domctl.u.psr_cat_op.data;
+
+ return rc;
+}
+
+int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
+ uint32_t *cos_max, uint32_t *cbm_len)
+{
+ int rc;
+ DECLARE_SYSCTL;
+
+ sysctl.cmd = XEN_SYSCTL_psr_cat_op;
+ sysctl.u.psr_cat_op.cmd = XEN_SYSCTL_PSR_CAT_get_l3_info;
+ sysctl.u.psr_cat_op.target = socket;
+
+ rc = xc_sysctl(xch, &sysctl);
+ if ( !rc )
+ {
+ *cos_max = sysctl.u.psr_cat_op.u.l3_info.cos_max;
+ *cbm_len = sysctl.u.psr_cat_op.u.l3_info.cbm_len;
+ }
+
+ return rc;
+}
/*
* Local variables:
* If this is defined, the Memory Bandwidth Monitoring feature is supported.
*/
#define LIBXL_HAVE_PSR_MBM 1
+
+/*
+ * LIBXL_HAVE_PSR_CAT
+ *
+ * If this is defined, the Cache Allocation Technology feature is supported.
+ */
+#define LIBXL_HAVE_PSR_CAT 1
#endif
/*
uint64_t *tsc_r);
#endif
+#ifdef LIBXL_HAVE_PSR_CAT
+/*
+ * Function to set a domain's cbm. It operates on a single or multiple
+ * target(s) defined in 'target_map'. The definition of 'target_map' is
+ * related to 'type':
+ * 'L3_CBM': 'target_map' specifies all the sockets to be operated on.
+ */
+int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid,
+ libxl_psr_cbm_type type, libxl_bitmap *target_map,
+ uint64_t cbm);
+/*
+ * Function to get a domain's cbm. It operates on a single 'target'.
+ * The definition of 'target' is related to 'type':
+ * 'L3_CBM': 'target' specifies which socket to be operated on.
+ */
+int libxl_psr_cat_get_cbm(libxl_ctx *ctx, uint32_t domid,
+ libxl_psr_cbm_type type, uint32_t target,
+ uint64_t *cbm_r);
+
+/*
+ * On success, the function returns an array of elements in 'info',
+ * and the length in 'nr'.
+ */
+int libxl_psr_cat_get_l3_info(libxl_ctx *ctx, libxl_psr_cat_info **info,
+ int *nr);
+void libxl_psr_cat_info_list_free(libxl_psr_cat_info *list, int nr);
+#endif
+
/* misc */
/* Each of these sets or clears the flag according to whether the
#define IA32_QM_CTR_ERROR_MASK (0x3ul << 62)
-static void libxl__psr_cmt_log_err_msg(libxl__gc *gc, int err)
+static void libxl__psr_log_err_msg(libxl__gc *gc, int err)
{
char *msg;
switch (err) {
case ENOSYS:
+ case EOPNOTSUPP:
msg = "unsupported operation";
break;
+ case ESRCH:
+ msg = "invalid domain ID";
+ break;
+ case EBADSLT:
+ msg = "socket is not supported";
+ break;
+ case EFAULT:
+ msg = "failed to exchange data with Xen";
+ break;
+ default:
+ msg = "unknown error";
+ break;
+ }
+
+ LOGE(ERROR, "%s", msg);
+}
+
+static void libxl__psr_cmt_log_err_msg(libxl__gc *gc, int err)
+{
+ char *msg;
+
+ switch (err) {
case ENODEV:
msg = "CMT is not supported in this system";
break;
case EUSERS:
msg = "no free RMID available";
break;
- case ESRCH:
- msg = "invalid domain ID";
+ default:
+ libxl__psr_log_err_msg(gc, err);
+ return;
+ }
+
+ LOGE(ERROR, "%s", msg);
+}
+
+static void libxl__psr_cat_log_err_msg(libxl__gc *gc, int err)
+{
+ char *msg;
+
+ switch (err) {
+ case ENODEV:
+ msg = "CAT is not supported in this system";
break;
- case EFAULT:
- msg = "failed to exchange data with Xen";
+ case ENOENT:
+ msg = "CAT is not enabled on the socket";
break;
- default:
- msg = "unknown error";
+ case EUSERS:
+ msg = "no free COS available";
break;
+ case EEXIST:
+ msg = "The same CBM is already set to this domain";
+ break;
+
+ default:
+ libxl__psr_log_err_msg(gc, err);
+ return;
}
LOGE(ERROR, "%s", msg);
return rc;
}
+int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid,
+ libxl_psr_cbm_type type, libxl_bitmap *target_map,
+ uint64_t cbm)
+{
+ GC_INIT(ctx);
+ int rc;
+ int socketid, nr_sockets;
+
+ rc = libxl__count_physical_sockets(gc, &nr_sockets);
+ if (rc) {
+ LOGE(ERROR, "failed to get system socket count");
+ goto out;
+ }
+
+ libxl_for_each_set_bit(socketid, *target_map) {
+ if (socketid >= nr_sockets)
+ break;
+ if (xc_psr_cat_set_domain_data(ctx->xch, domid, type, socketid, cbm)) {
+ libxl__psr_cat_log_err_msg(gc, errno);
+ rc = ERROR_FAIL;
+ }
+ }
+
+out:
+ GC_FREE;
+ return rc;
+}
+
+int libxl_psr_cat_get_cbm(libxl_ctx *ctx, uint32_t domid,
+ libxl_psr_cbm_type type, uint32_t target,
+ uint64_t *cbm_r)
+{
+ GC_INIT(ctx);
+ int rc = 0;
+
+ if (xc_psr_cat_get_domain_data(ctx->xch, domid, type, target, cbm_r)) {
+ libxl__psr_cat_log_err_msg(gc, errno);
+ rc = ERROR_FAIL;
+ }
+
+ GC_FREE;
+ return rc;
+}
+
+int libxl_psr_cat_get_l3_info(libxl_ctx *ctx, libxl_psr_cat_info **info,
+ int *nr)
+{
+ GC_INIT(ctx);
+ int rc;
+ int i, nr_sockets;
+ libxl_psr_cat_info *ptr;
+
+ rc = libxl__count_physical_sockets(gc, &nr_sockets);
+ if (rc) {
+ LOGE(ERROR, "failed to get system socket count");
+ goto out;
+ }
+
+ ptr = libxl__malloc(NOGC, nr_sockets * sizeof(libxl_psr_cat_info));
+
+ for (i = 0; i < nr_sockets; i++) {
+ if (xc_psr_cat_get_l3_info(ctx->xch, i, &ptr[i].cos_max,
+ &ptr[i].cbm_len)) {
+ libxl__psr_cat_log_err_msg(gc, errno);
+ rc = ERROR_FAIL;
+ free(ptr);
+ goto out;
+ }
+ }
+
+ *info = ptr;
+ *nr = nr_sockets;
+out:
+ GC_FREE;
+ return rc;
+}
+
+void libxl_psr_cat_info_list_free(libxl_psr_cat_info *list, int nr)
+{
+ int i;
+
+ for (i = 0; i < nr; i++)
+ libxl_psr_cat_info_dispose(&list[i]);
+ free(list);
+}
+
/*
* Local variables:
* mode: C
(2, "TOTAL_MEM_COUNT"),
(3, "LOCAL_MEM_COUNT"),
])
+
+libxl_psr_cbm_type = Enumeration("psr_cbm_type", [
+ (0, "UNKNOWN"),
+ (1, "L3_CBM"),
+ ])
+
+libxl_psr_cat_info = Struct("psr_cat_info", [
+ ("cos_max", uint32),
+ ("cbm_len", uint32),
+ ])
int main_psr_cmt_detach(int argc, char **argv);
int main_psr_cmt_show(int argc, char **argv);
#endif
+#ifdef LIBXL_HAVE_PSR_CAT
+int main_psr_cat_cbm_set(int argc, char **argv);
+int main_psr_cat_show(int argc, char **argv);
+#endif
void help(const char *command);
return 0;
}
-int main_psr_hwinfo(int argc, char **argv)
-{
- int opt;
-
- SWITCH_FOREACH_OPT(opt, "", NULL, "psr-hwinfo", 0) {
- /* No options */
- }
-
- return psr_cmt_hwinfo();
-}
-
int main_psr_cmt_attach(int argc, char **argv)
{
uint32_t domid;
}
#endif
+#ifdef LIBXL_HAVE_PSR_CAT
+static int psr_cat_hwinfo(void)
+{
+ int rc;
+ int socketid, nr_sockets;
+ uint32_t l3_cache_size;
+ libxl_psr_cat_info *info;
+
+ printf("Cache Allocation Technology (CAT):\n");
+
+ rc = libxl_psr_cat_get_l3_info(ctx, &info, &nr_sockets);
+ if (rc) {
+ fprintf(stderr, "Failed to get cat info\n");
+ return rc;
+ }
+
+ for (socketid = 0; socketid < nr_sockets; socketid++) {
+ rc = libxl_psr_cmt_get_l3_cache_size(ctx, socketid, &l3_cache_size);
+ if (rc) {
+ fprintf(stderr, "Failed to get l3 cache size for socket:%d\n",
+ socketid);
+ goto out;
+ }
+ printf("%-16s: %u\n", "Socket ID", socketid);
+ printf("%-16s: %uKB\n", "L3 Cache", l3_cache_size);
+ printf("%-16s: %u\n", "Maximum COS", info->cos_max);
+ printf("%-16s: %u\n", "CBM length", info->cbm_len);
+ printf("%-16s: %#llx\n", "Default CBM",
+ (1ull << info->cbm_len) - 1);
+ }
+
+out:
+ libxl_psr_cat_info_list_free(info, nr_sockets);
+ return rc;
+}
+
+static void psr_cat_print_one_domain_cbm(uint32_t domid, uint32_t socketid)
+{
+ char *domain_name;
+ uint64_t cbm;
+
+ domain_name = libxl_domid_to_name(ctx, domid);
+ printf("%5d%25s", domid, domain_name);
+ free(domain_name);
+
+ if (!libxl_psr_cat_get_cbm(ctx, domid, LIBXL_PSR_CBM_TYPE_L3_CBM,
+ socketid, &cbm))
+ printf("%#16"PRIx64, cbm);
+
+ printf("\n");
+}
+
+static int psr_cat_print_domain_cbm(uint32_t domid, uint32_t socketid)
+{
+ int i, nr_domains;
+ libxl_dominfo *list;
+
+ if (domid != INVALID_DOMID) {
+ psr_cat_print_one_domain_cbm(domid, socketid);
+ return 0;
+ }
+
+ if (!(list = libxl_list_domain(ctx, &nr_domains))) {
+ fprintf(stderr, "Failed to get domain list for cbm display\n");
+ return -1;
+ }
+
+ for (i = 0; i < nr_domains; i++)
+ psr_cat_print_one_domain_cbm(list[i].domid, socketid);
+ libxl_dominfo_list_free(list, nr_domains);
+
+ return 0;
+}
+
+static int psr_cat_print_socket(uint32_t domid, uint32_t socketid,
+ libxl_psr_cat_info *info)
+{
+ int rc;
+ uint32_t l3_cache_size;
+
+ rc = libxl_psr_cmt_get_l3_cache_size(ctx, socketid, &l3_cache_size);
+ if (rc) {
+ fprintf(stderr, "Failed to get l3 cache size for socket:%d\n",
+ socketid);
+ return -1;
+ }
+
+ printf("%-16s: %u\n", "Socket ID", socketid);
+ printf("%-16s: %uKB\n", "L3 Cache", l3_cache_size);
+ printf("%-16s: %#llx\n", "Default CBM", (1ull << info->cbm_len) - 1);
+ printf("%5s%25s%16s\n", "ID", "NAME", "CBM");
+
+ return psr_cat_print_domain_cbm(domid, socketid);
+}
+
+static int psr_cat_show(uint32_t domid)
+{
+ int socketid, nr_sockets;
+ int rc;
+ libxl_psr_cat_info *info;
+
+ rc = libxl_psr_cat_get_l3_info(ctx, &info, &nr_sockets);
+ if (rc) {
+ fprintf(stderr, "Failed to get cat info\n");
+ return rc;
+ }
+
+ for (socketid = 0; socketid < nr_sockets; socketid++) {
+ rc = psr_cat_print_socket(domid, socketid, info + socketid);
+ if (rc)
+ goto out;
+ }
+
+out:
+ libxl_psr_cat_info_list_free(info, nr_sockets);
+ return rc;
+}
+
+int main_psr_cat_cbm_set(int argc, char **argv)
+{
+ uint32_t domid;
+ libxl_psr_cbm_type type = LIBXL_PSR_CBM_TYPE_L3_CBM;
+ uint64_t cbm;
+ int ret, opt = 0;
+ libxl_bitmap target_map;
+ char *value;
+ libxl_string_list socket_list;
+ unsigned long start, end;
+ int i, j, len;
+
+ static struct option opts[] = {
+ {"socket", 0, 0, 's'},
+ COMMON_LONG_OPTS,
+ {0, 0, 0, 0}
+ };
+
+ libxl_socket_bitmap_alloc(ctx, &target_map, 0);
+ libxl_bitmap_set_none(&target_map);
+
+ SWITCH_FOREACH_OPT(opt, "s", opts, "psr-cat-cbm-set", 1) {
+ case 's':
+ trim(isspace, optarg, &value);
+ split_string_into_string_list(value, ",", &socket_list);
+ len = libxl_string_list_length(&socket_list);
+ for (i = 0; i < len; i++) {
+ parse_range(socket_list[i], &start, &end);
+ for (j = start; j < end; j++)
+ libxl_bitmap_set(&target_map, j);
+ }
+
+ libxl_string_list_dispose(&socket_list);
+ free(value);
+ break;
+ }
+
+ if (libxl_bitmap_is_empty(&target_map))
+ libxl_bitmap_set_any(&target_map);
+
+ domid = find_domain(argv[optind]);
+ cbm = strtoll(argv[optind + 1], NULL , 0);
+
+ ret = libxl_psr_cat_set_cbm(ctx, domid, type, &target_map, cbm);
+
+ libxl_bitmap_dispose(&target_map);
+ return ret;
+}
+
+int main_psr_cat_show(int argc, char **argv)
+{
+ int opt;
+ uint32_t domid;
+
+ SWITCH_FOREACH_OPT(opt, "", NULL, "psr-cat-show", 0) {
+ /* No options */
+ }
+
+ if (optind >= argc)
+ domid = INVALID_DOMID;
+ else if (optind == argc - 1)
+ domid = find_domain(argv[optind]);
+ else {
+ help("psr-cat-show");
+ return 2;
+ }
+
+ return psr_cat_show(domid);
+}
+
+int main_psr_hwinfo(int argc, char **argv)
+{
+ int opt, ret;
+ int cmt = 0, cat = 0;
+ static struct option opts[] = {
+ {"cmt", 0, 0, 'm'},
+ {"cat", 0, 0, 'a'},
+ COMMON_LONG_OPTS,
+ {0, 0, 0, 0}
+ };
+
+ SWITCH_FOREACH_OPT(opt, "ma", opts, "psr-hwinfo", 0) {
+ case 'm':
+ cmt = 1;
+ break;
+ case 'a':
+ cat = 1;
+ break;
+ }
+
+ if (!(cmt | cat)) {
+ cmt = 1;
+ cat = 1;
+ }
+
+ if (cmt)
+ ret = psr_cmt_hwinfo();
+
+ if (ret)
+ return ret;
+
+ if (cat)
+ ret = psr_cat_hwinfo();
+
+ return ret;
+}
+
+#endif
+
/*
* Local variables:
* mode: C
{ "psr-hwinfo",
&main_psr_hwinfo, 0, 1,
"Show hardware information for Platform Shared Resource",
- "",
+ "[options]",
+ "-m, --cmt Show Cache Monitoring Technology (CMT) hardware info\n"
+ "-a, --cat Show Cache Allocation Technology (CAT) hardware info\n"
},
{ "psr-cmt-attach",
&main_psr_cmt_attach, 0, 1,
"\"local-mem-bandwidth\": Show local memory bandwidth(KB/s)\n",
},
#endif
+#ifdef LIBXL_HAVE_PSR_CAT
+ { "psr-cat-cbm-set",
+ &main_psr_cat_cbm_set, 0, 1,
+ "Set cache capacity bitmasks(CBM) for a domain",
+ "[options] <Domain> <CBM>",
+ "-s <socket> Specify the socket to process, otherwise all sockets are processed\n"
+ },
+ { "psr-cat-show",
+ &main_psr_cat_show, 0, 1,
+ "Show Cache Allocation Technology information",
+ "<Domain>",
+ },
+
+#endif
};
int cmdtable_len = sizeof(cmd_table)/sizeof(struct cmd_spec);
projects / people / liuw / libxenctrl-split / xen.git / commitdiff