TXTSRC-y := $(sort $(shell find misc -name '*.txt' -print))
-PANDOCSRC-y := $(sort $(shell find process/ features/ misc/ specs/ -name '*.pandoc' -print))
+PANDOCSRC-y := $(sort $(shell find designs/ features/ misc/ process/ specs/ -name '*.pandoc' -print))
# Documentation targets
DOC_MAN1 := $(patsubst man/%.pod.1,man1/%.1,$(MAN1SRC-y)) \
+++ /dev/null
-DMOP
-====
-
-Introduction
-------------
-
-The aim of DMOP is to prevent a compromised device model from compromising
-domains other than the one it is providing emulation for (which is therefore
-likely already compromised).
-
-The problem occurs when you a device model issues an hypercall that
-includes references to user memory other than the operation structure
-itself, such as with Track dirty VRAM (as used in VGA emulation).
-Is this case, the address of this other user memory needs to be vetted,
-to ensure it is not within restricted address ranges, such as kernel
-memory. The real problem comes down to how you would vet this address -
-the idea place to do this is within the privcmd driver, without privcmd
-having to have specific knowledge of the hypercall's semantics.
-
-The Design
-----------
-
-The privcmd driver implements a new restriction ioctl, which takes a domid
-parameter. After that restriction ioctl is issued, all unaudited operations
-on the privcmd driver will cease to function, including regular hypercalls.
-DMOP hypercalls will continue to function as they can be audited.
-
-A DMOP hypercall consists of a domid (which is audited to verify that it
-matches any restriction in place) and an array of buffers and lengths,
-with the first one containing the specific DMOP parameters. These can
-then reference further buffers from within in the array. Since the only
-user buffers passed are that found with that array, they can all can be
-audited by privcmd.
-
-The following code illustrates this idea:
-
-struct xen_dm_op {
- uint32_t op;
-};
-
-struct xen_dm_op_buf {
- XEN_GUEST_HANDLE(void) h;
- unsigned long size;
-};
-typedef struct xen_dm_op_buf xen_dm_op_buf_t;
-
-enum neg_errnoval
-HYPERVISOR_dm_op(domid_t domid,
- xen_dm_op_buf_t bufs[],
- unsigned int nr_bufs)
-
-@domid is the domain the hypercall operates on.
-@bufs points to an array of buffers where @bufs[0] contains a struct
-dm_op, describing the specific device model operation and its parameters.
-@bufs[1..] may be referenced in the parameters for the purposes of
-passing extra information to or from the domain.
-@nr_bufs is the number of buffers in the @bufs array.
-
-It is forbidden for the above struct (xen_dm_op) to contain any guest
-handles. If they are needed, they should instead be in
-HYPERVISOR_dm_op->bufs.
-
-Validation by privcmd driver
-----------------------------
-
-If the privcmd driver has been restricted to specific domain (using a
- new ioctl), when it received an op, it will:
-
-1. Check hypercall is DMOP.
-
-2. Check domid == restricted domid.
-
-3. For each @nr_bufs in @bufs: Check @h and @size give a buffer
- wholly in the user space part of the virtual address space. (e.g.
- Linux will use access_ok()).
-
-
-Xen Implementation
-------------------
-
-Since a DMOP buffers need to be copied from or to the guest, functions for
-doing this would be written as below. Note that care is taken to prevent
-damage from buffer under- or over-run situations. If the DMOP is called
-with incorrectly sized buffers, zeros will be read, while extra is ignored.
-
-static bool copy_buf_from_guest(xen_dm_op_buf_t bufs[],
- unsigned int nr_bufs, void *dst,
- unsigned int idx, size_t dst_size)
-{
- size_t size;
-
- if ( idx >= nr_bufs )
- return false;
-
- memset(dst, 0, dst_size);
-
- size = min_t(size_t, dst_size, bufs[idx].size);
-
- return !copy_from_guest(dst, bufs[idx].h, size);
-}
-
-static bool copy_buf_to_guest(xen_dm_op_buf_t bufs[],
- unsigned int nr_bufs, unsigned int idx,
- void *src, size_t src_size)
-{
- size_t size;
-
- if ( idx >= nr_bufs )
- return false;
-
- size = min_t(size_t, bufs[idx].size, src_size);
-
- return !copy_to_guest(bufs[idx].h, src, size);
-}
-
-This leaves do_dm_op easy to implement as below:
-
-static int dm_op(domid_t domid,
- unsigned int nr_bufs,
- xen_dm_op_buf_t bufs[])
-{
- struct domain *d;
- struct xen_dm_op op;
- bool const_op = true;
- long rc;
-
- rc = rcu_lock_remote_domain_by_id(domid, &d);
- if ( rc )
- return rc;
-
- if ( !is_hvm_domain(d) )
- goto out;
-
- rc = xsm_dm_op(XSM_DM_PRIV, d);
- if ( rc )
- goto out;
-
- if ( !copy_buf_from_guest(bufs, nr_bufs, &op, 0, sizeof(op)) )
- {
- rc = -EFAULT;
- goto out;
- }
-
- switch ( op.op )
- {
- default:
- rc = -EOPNOTSUPP;
- break;
- }
-
- if ( !rc &&
- !const_op &&
- !copy_buf_to_guest(bufs, nr_bufs, 0, &op, sizeof(op)) )
- rc = -EFAULT;
-
- out:
- rcu_unlock_domain(d);
-
- return rc;
-}
-
-long do_dm_op(domid_t domid,
- unsigned int nr_bufs,
- XEN_GUEST_HANDLE_PARAM(xen_dm_op_buf_t) bufs)
-{
- struct xen_dm_op_buf nat[MAX_NR_BUFS];
-
- if ( nr_bufs > MAX_NR_BUFS )
- return -EINVAL;
-
- if ( copy_from_guest_offset(nat, bufs, 0, nr_bufs) )
- return -EFAULT;
-
- return dm_op(domid, nr_bufs, nat);
-}
--- /dev/null
+DMOP
+====
+
+Introduction
+------------
+
+The aim of DMOP is to prevent a compromised device model from compromising
+domains other than the one it is providing emulation for (which is therefore
+likely already compromised).
+
+The problem occurs when you a device model issues an hypercall that
+includes references to user memory other than the operation structure
+itself, such as with Track dirty VRAM (as used in VGA emulation).
+Is this case, the address of this other user memory needs to be vetted,
+to ensure it is not within restricted address ranges, such as kernel
+memory. The real problem comes down to how you would vet this address -
+the idea place to do this is within the privcmd driver, without privcmd
+having to have specific knowledge of the hypercall's semantics.
+
+The Design
+----------
+
+The privcmd driver implements a new restriction ioctl, which takes a domid
+parameter. After that restriction ioctl is issued, all unaudited operations
+on the privcmd driver will cease to function, including regular hypercalls.
+DMOP hypercalls will continue to function as they can be audited.
+
+A DMOP hypercall consists of a domid (which is audited to verify that it
+matches any restriction in place) and an array of buffers and lengths,
+with the first one containing the specific DMOP parameters. These can
+then reference further buffers from within in the array. Since the only
+user buffers passed are that found with that array, they can all can be
+audited by privcmd.
+
+The following code illustrates this idea:
+
+struct xen_dm_op {
+ uint32_t op;
+};
+
+struct xen_dm_op_buf {
+ XEN_GUEST_HANDLE(void) h;
+ unsigned long size;
+};
+typedef struct xen_dm_op_buf xen_dm_op_buf_t;
+
+enum neg_errnoval
+HYPERVISOR_dm_op(domid_t domid,
+ xen_dm_op_buf_t bufs[],
+ unsigned int nr_bufs)
+
+@domid is the domain the hypercall operates on.
+@bufs points to an array of buffers where @bufs[0] contains a struct
+dm_op, describing the specific device model operation and its parameters.
+@bufs[1..] may be referenced in the parameters for the purposes of
+passing extra information to or from the domain.
+@nr_bufs is the number of buffers in the @bufs array.
+
+It is forbidden for the above struct (xen_dm_op) to contain any guest
+handles. If they are needed, they should instead be in
+HYPERVISOR_dm_op->bufs.
+
+Validation by privcmd driver
+----------------------------
+
+If the privcmd driver has been restricted to specific domain (using a
+ new ioctl), when it received an op, it will:
+
+1. Check hypercall is DMOP.
+
+2. Check domid == restricted domid.
+
+3. For each @nr_bufs in @bufs: Check @h and @size give a buffer
+ wholly in the user space part of the virtual address space. (e.g.
+ Linux will use access_ok()).
+
+
+Xen Implementation
+------------------
+
+Since a DMOP buffers need to be copied from or to the guest, functions for
+doing this would be written as below. Note that care is taken to prevent
+damage from buffer under- or over-run situations. If the DMOP is called
+with incorrectly sized buffers, zeros will be read, while extra is ignored.
+
+static bool copy_buf_from_guest(xen_dm_op_buf_t bufs[],
+ unsigned int nr_bufs, void *dst,
+ unsigned int idx, size_t dst_size)
+{
+ size_t size;
+
+ if ( idx >= nr_bufs )
+ return false;
+
+ memset(dst, 0, dst_size);
+
+ size = min_t(size_t, dst_size, bufs[idx].size);
+
+ return !copy_from_guest(dst, bufs[idx].h, size);
+}
+
+static bool copy_buf_to_guest(xen_dm_op_buf_t bufs[],
+ unsigned int nr_bufs, unsigned int idx,
+ void *src, size_t src_size)
+{
+ size_t size;
+
+ if ( idx >= nr_bufs )
+ return false;
+
+ size = min_t(size_t, bufs[idx].size, src_size);
+
+ return !copy_to_guest(bufs[idx].h, src, size);
+}
+
+This leaves do_dm_op easy to implement as below:
+
+static int dm_op(domid_t domid,
+ unsigned int nr_bufs,
+ xen_dm_op_buf_t bufs[])
+{
+ struct domain *d;
+ struct xen_dm_op op;
+ bool const_op = true;
+ long rc;
+
+ rc = rcu_lock_remote_domain_by_id(domid, &d);
+ if ( rc )
+ return rc;
+
+ if ( !is_hvm_domain(d) )
+ goto out;
+
+ rc = xsm_dm_op(XSM_DM_PRIV, d);
+ if ( rc )
+ goto out;
+
+ if ( !copy_buf_from_guest(bufs, nr_bufs, &op, 0, sizeof(op)) )
+ {
+ rc = -EFAULT;
+ goto out;
+ }
+
+ switch ( op.op )
+ {
+ default:
+ rc = -EOPNOTSUPP;
+ break;
+ }
+
+ if ( !rc &&
+ !const_op &&
+ !copy_buf_to_guest(bufs, nr_bufs, 0, &op, sizeof(op)) )
+ rc = -EFAULT;
+
+ out:
+ rcu_unlock_domain(d);
+
+ return rc;
+}
+
+long do_dm_op(domid_t domid,
+ unsigned int nr_bufs,
+ XEN_GUEST_HANDLE_PARAM(xen_dm_op_buf_t) bufs)
+{
+ struct xen_dm_op_buf nat[MAX_NR_BUFS];
+
+ if ( nr_bufs > MAX_NR_BUFS )
+ return -EINVAL;
+
+ if ( copy_from_guest_offset(nat, bufs, 0, nr_bufs) )
+ return -EFAULT;
+
+ return dm_op(domid, nr_bufs, nat);
+}
+++ /dev/null
-Dom0less
-========
-
-"Dom0less" is a set of Xen features that enable the deployment of a Xen
-system without an control domain (often referred to as "dom0"). Each
-feature can be used independently from the others, unless otherwise
-stated.
-
-Booting Multiple Domains from Device Tree
------------------------------------------
-
-This feature enables Xen to create a set of DomUs at boot time.
-Information about the DomUs to be created by Xen is passed to the
-hypervisor via Device Tree. Specifically, the existing Device Tree based
-Multiboot specification has been extended to allow for multiple domains
-to be passed to Xen. See docs/misc/arm/device-tree/booting.txt for more
-information about the Multiboot specification and how to use it.
-
-Currently, a control domain ("dom0") is still required, but in the
-future it will become unnecessary when all domains are created
-directly from Xen. Instead of waiting for the control domain to be fully
-booted and the Xen tools to become available, domains created by Xen
-this way are started right away in parallel. Hence, their boot time is
-typically much shorter.
-
-Domains started by Xen at boot time currently have the following
-limitations:
-
-- They cannot be properly shutdown or rebooted using xl. If one of them
- crashes, the whole platform should be rebooted.
-
-- Some xl operations might not work as expected. xl is meant to be used
- with domains that have been created by it. Using xl with domains
- started by Xen at boot might not work as expected.
-
-- The GIC version is the native version. In absence of other
- information, the GIC version exposed to the domains started by Xen at
- boot is the same as the native GIC version.
-
-- No PV drivers. There is no support for PV devices at the moment. All
- devices need to be statically assigned to guests.
-
-- Pinning vCPUs of domains started by Xen at boot can be
- done from the control domain, using `xl vcpu-pin` as usual. It is not
- currently possible to configure vCPU pinning without a control domain.
- However, the NULL scheduler can be selected by passing `sched=null` to
- the Xen command line. The NULL scheduler automatically assigns and
- pins vCPUs to pCPUs, but the vCPU-pCPU assignments cannot be
- configured.
--- /dev/null
+Dom0less
+========
+
+"Dom0less" is a set of Xen features that enable the deployment of a Xen
+system without an control domain (often referred to as "dom0"). Each
+feature can be used independently from the others, unless otherwise
+stated.
+
+Booting Multiple Domains from Device Tree
+-----------------------------------------
+
+This feature enables Xen to create a set of DomUs at boot time.
+Information about the DomUs to be created by Xen is passed to the
+hypervisor via Device Tree. Specifically, the existing Device Tree based
+Multiboot specification has been extended to allow for multiple domains
+to be passed to Xen. See docs/misc/arm/device-tree/booting.txt for more
+information about the Multiboot specification and how to use it.
+
+Currently, a control domain ("dom0") is still required, but in the
+future it will become unnecessary when all domains are created
+directly from Xen. Instead of waiting for the control domain to be fully
+booted and the Xen tools to become available, domains created by Xen
+this way are started right away in parallel. Hence, their boot time is
+typically much shorter.
+
+Domains started by Xen at boot time currently have the following
+limitations:
+
+- They cannot be properly shutdown or rebooted using xl. If one of them
+ crashes, the whole platform should be rebooted.
+
+- Some xl operations might not work as expected. xl is meant to be used
+ with domains that have been created by it. Using xl with domains
+ started by Xen at boot might not work as expected.
+
+- The GIC version is the native version. In absence of other
+ information, the GIC version exposed to the domains started by Xen at
+ boot is the same as the native GIC version.
+
+- No PV drivers. There is no support for PV devices at the moment. All
+ devices need to be statically assigned to guests.
+
+- Pinning vCPUs of domains started by Xen at boot can be
+ done from the control domain, using `xl vcpu-pin` as usual. It is not
+ currently possible to configure vCPU pinning without a control domain.
+ However, the NULL scheduler can be selected by passing `sched=null` to
+ the Xen command line. The NULL scheduler automatically assigns and
+ pins vCPUs to pCPUs, but the vCPU-pCPU assignments cannot be
+ configured.
+++ /dev/null
-# Xen transport for 9pfs version 1
-
-## Background
-
-9pfs is a network filesystem protocol developed for Plan 9. 9pfs is very
-simple and describes a series of commands and responses. It is
-completely independent from the communication channels, in fact many
-clients and servers support multiple channels, usually called
-"transports". For example the Linux client supports tcp and unix
-sockets, fds, virtio and rdma.
-
-
-### 9pfs protocol
-
-This document won't cover the full 9pfs specification. Please refer to
-this [paper] and this [website] for a detailed description of it.
-However it is useful to know that each 9pfs request and response has the
-following header:
-
- struct header {
- uint32_t size;
- uint8_t id;
- uint16_t tag;
- } __attribute__((packed));
-
- 0 4 5 7
- +---------+--+----+
- | size |id|tag |
- +---------+--+----+
-
-- *size*
-The size of the request or response.
-
-- *id*
-The 9pfs request or response operation.
-
-- *tag*
-Unique id that identifies a specific request/response pair. It is used
-to multiplex operations on a single channel.
-
-It is possible to have multiple requests in-flight at any given time.
-
-
-## Rationale
-
-This document describes a Xen based transport for 9pfs, in the
-traditional PV frontend and backend format. The PV frontend is used by
-the client to send commands to the server. The PV backend is used by the
-9pfs server to receive commands from clients and send back responses.
-
-The transport protocol supports multiple rings up to the maximum
-supported by the backend. The size of every ring is also configurable
-and can span multiple pages, up to the maximum supported by the backend
-(although it cannot be more than 2MB). The design is to exploit
-parallelism at the vCPU level and support multiple outstanding requests
-simultaneously.
-
-This document does not cover the 9pfs client/server design or
-implementation, only the transport for it.
-
-
-## Xenstore
-
-The frontend and the backend connect via xenstore to exchange
-information. The toolstack creates front and back nodes with state
-[XenbusStateInitialising]. The protocol node name is **9pfs**.
-
-Multiple rings are supported for each frontend and backend connection.
-
-### Backend XenBus Nodes
-
-Backend specific properties, written by the backend, read by the
-frontend:
-
- versions
- Values: <string>
-
- List of comma separated protocol versions supported by the backend.
- For example "1,2,3". Currently the value is just "1", as there is
- only one version. N.B.: this is the version of the Xen trasport
- protocol, not the version of 9pfs supported by the server.
-
- max-rings
- Values: <uint32_t>
-
- The maximum supported number of rings per frontend.
-
- max-ring-page-order
- Values: <uint32_t>
-
- The maximum supported size of a memory allocation in units of
- log2n(machine pages), e.g. 1 = 2 pages, 2 == 4 pages, etc. It
- must be at least 1.
-
-Backend configuration nodes, written by the toolstack, read by the
-backend:
-
- path
- Values: <string>
-
- Host filesystem path to share.
-
- tag
- Values: <string>
-
- Alphanumeric tag that identifies the 9pfs share. The client needs
- to know the tag to be able to mount it.
-
- security-model
- Values: "none"
-
- *none*: files are stored using the same credentials as they are
- created on the guest (no user ownership squash or remap)
- Only "none" is supported in this version of the protocol.
-
-### Frontend XenBus Nodes
-
- version
- Values: <string>
-
- Protocol version, chosen among the ones supported by the backend
- (see **versions** under [Backend XenBus Nodes]). Currently the
- value must be "1".
-
- num-rings
- Values: <uint32_t>
-
- Number of rings. It needs to be lower or equal to max-rings.
-
- event-channel-<num> (event-channel-0, event-channel-1, etc)
- Values: <uint32_t>
-
- The identifier of the Xen event channel used to signal activity
- in the ring buffer. One for each ring.
-
- ring-ref<num> (ring-ref0, ring-ref1, etc)
- Values: <uint32_t>
-
- The Xen grant reference granting permission for the backend to
- map a page with information to setup a share ring. One for each
- ring.
-
-### State Machine
-
-Initialization:
-
- *Front* *Back*
- XenbusStateInitialising XenbusStateInitialising
- - Query virtual device - Query backend device
- properties. identification data.
- - Setup OS device instance. - Publish backend features
- - Allocate and initialize the and transport parameters
- request ring. |
- - Publish transport parameters |
- that will be in effect during V
- this connection. XenbusStateInitWait
- |
- |
- V
- XenbusStateInitialised
-
- - Query frontend transport parameters.
- - Connect to the request ring and
- event channel.
- |
- |
- V
- XenbusStateConnected
-
- - Query backend device properties.
- - Finalize OS virtual device
- instance.
- |
- |
- V
- XenbusStateConnected
-
-Once frontend and backend are connected, they have a shared page per
-ring, which are used to setup the rings, and an event channel per ring,
-which are used to send notifications.
-
-Shutdown:
-
- *Front* *Back*
- XenbusStateConnected XenbusStateConnected
- |
- |
- V
- XenbusStateClosing
-
- - Unmap grants
- - Unbind evtchns
- |
- |
- V
- XenbusStateClosing
-
- - Unbind evtchns
- - Free rings
- - Free data structures
- |
- |
- V
- XenbusStateClosed
-
- - Free remaining data structures
- |
- |
- V
- XenbusStateClosed
-
-
-## Ring Setup
-
-The shared page has the following layout:
-
- typedef uint32_t XEN_9PFS_RING_IDX;
-
- struct xen_9pfs_intf {
- XEN_9PFS_RING_IDX in_cons, in_prod;
- uint8_t pad[56];
- XEN_9PFS_RING_IDX out_cons, out_prod;
- uint8_t pad[56];
-
- uint32_t ring_order;
- /* this is an array of (1 << ring_order) elements */
- grant_ref_t ref[1];
- };
-
- /* not actually C compliant (ring_order changes from ring to ring) */
- struct ring_data {
- char in[((1 << ring_order) << PAGE_SHIFT) / 2];
- char out[((1 << ring_order) << PAGE_SHIFT) / 2];
- };
-
-- **ring_order**
- It represents the order of the data ring. The following list of grant
- references is of `(1 << ring_order)` elements. It cannot be greater than
- **max-ring-page-order**, as specified by the backend on XenBus.
-- **ref[]**
- The list of grant references which will contain the actual data. They are
- mapped contiguosly in virtual memory. The first half of the pages is the
- **in** array, the second half is the **out** array. The array must
- have a power of two number of elements.
-- **out** is an array used as circular buffer
- It contains client requests. The producer is the frontend, the
- consumer is the backend.
-- **in** is an array used as circular buffer
- It contains server responses. The producer is the backend, the
- consumer is the frontend.
-- **out_cons**, **out_prod**
- Consumer and producer indices for client requests. They keep track of
- how much data has been written by the frontend to **out** and how much
- data has already been consumed by the backend. **out_prod** is
- increased by the frontend, after writing data to **out**. **out_cons**
- is increased by the backend, after reading data from **out**.
-- **in_cons** and **in_prod**
- Consumer and producer indices for responses. They keep track of how
- much data has already been consumed by the frontend from the **in**
- array. **in_prod** is increased by the backend, after writing data to
- **in**. **in_cons** is increased by the frontend, after reading data
- from **in**.
-
-The binary layout of `struct xen_9pfs_intf` follows:
-
- 0 4 8 64 68 72 76
- +---------+---------+-----//-----+---------+---------+---------+
- | in_cons | in_prod | padding |out_cons |out_prod |ring_orde|
- +---------+---------+-----//-----+---------+---------+---------+
-
- 76 80 84 4092 4096
- +---------+---------+----//---+---------+
- | ref[0] | ref[1] | | ref[N] |
- +---------+---------+----//---+---------+
-
-**N.B** For one page, N is maximum 991 (4096-132)/4, but given that N
-needs to be a power of two, actually max N is 512. As 512 == (1 << 9),
-the maximum possible max-ring-page-order value is 9.
-
-The binary layout of the ring buffers follow:
-
- 0 ((1<<ring_order)<<PAGE_SHIFT)/2 ((1<<ring_order)<<PAGE_SHIFT)
- +------------//-------------+------------//-------------+
- | in | out |
- +------------//-------------+------------//-------------+
-
-## Why ring.h is not needed
-
-Many Xen PV protocols use the macros provided by [ring.h] to manage
-their shared ring for communication. This procotol does not, because it
-actually comes with two rings: the **in** ring and the **out** ring.
-Each of them is mono-directional, and there is no static request size:
-the producer writes opaque data to the ring. On the other end, in
-[ring.h] they are combined, and the request size is static and
-well-known. In this protocol:
-
- in -> backend to frontend only
- out-> frontend to backend only
-
-In the case of the **in** ring, the frontend is the consumer, and the
-backend is the producer. Everything is the same but mirrored for the
-**out** ring.
-
-The producer, the backend in this case, never reads from the **in**
-ring. In fact, the producer doesn't need any notifications unless the
-ring is full. This version of the protocol doesn't take advantage of it,
-leaving room for optimizations.
-
-On the other end, the consumer always requires notifications, unless it
-is already actively reading from the ring. The producer can figure it
-out, without any additional fields in the protocol, by comparing the
-indexes at the beginning and the end of the function. This is similar to
-what [ring.h] does.
-
-## Ring Usage
-
-The **in** and **out** arrays are used as circular buffers:
-
- 0 sizeof(array) == ((1<<ring_order)<<PAGE_SHIFT)/2
- +-----------------------------------+
- |to consume| free |to consume |
- +-----------------------------------+
- ^ ^
- prod cons
-
- 0 sizeof(array)
- +-----------------------------------+
- | free | to consume | free |
- +-----------------------------------+
- ^ ^
- cons prod
-
-The following functions are provided to read and write to an array:
-
- #define MASK_XEN_9PFS_IDX(idx) ((idx) & (XEN_9PFS_RING_SIZE - 1))
-
- static inline void xen_9pfs_read(char *buf,
- XEN_9PFS_RING_IDX *masked_prod, XEN_9PFS_RING_IDX *masked_cons,
- uint8_t *h, size_t len) {
- if (*masked_cons < *masked_prod) {
- memcpy(h, buf + *masked_cons, len);
- } else {
- if (len > XEN_9PFS_RING_SIZE - *masked_cons) {
- memcpy(h, buf + *masked_cons, XEN_9PFS_RING_SIZE - *masked_cons);
- memcpy((char *)h + XEN_9PFS_RING_SIZE - *masked_cons, buf, len - (XEN_9PFS_RING_SIZE - *masked_cons));
- } else {
- memcpy(h, buf + *masked_cons, len);
- }
- }
- *masked_cons = _MASK_XEN_9PFS_IDX(*masked_cons + len);
- }
-
- static inline void xen_9pfs_write(char *buf,
- XEN_9PFS_RING_IDX *masked_prod, XEN_9PFS_RING_IDX *masked_cons,
- uint8_t *opaque, size_t len) {
- if (*masked_prod < *masked_cons) {
- memcpy(buf + *masked_prod, opaque, len);
- } else {
- if (len > XEN_9PFS_RING_SIZE - *masked_prod) {
- memcpy(buf + *masked_prod, opaque, XEN_9PFS_RING_SIZE - *masked_prod);
- memcpy(buf, opaque + (XEN_9PFS_RING_SIZE - *masked_prod), len - (XEN_9PFS_RING_SIZE - *masked_prod));
- } else {
- memcpy(buf + *masked_prod, opaque, len);
- }
- }
- *masked_prod = _MASK_XEN_9PFS_IDX(*masked_prod + len);
- }
-
-The producer (the backend for **in**, the frontend for **out**) writes to the
-array in the following way:
-
-- read *cons*, *prod* from shared memory
-- general memory barrier
-- verify *prod* against local copy (consumer shouldn't change it)
-- write to array at position *prod* up to *cons*, wrapping around the circular
- buffer when necessary
-- write memory barrier
-- increase *prod*
-- notify the other end via event channel
-
-The consumer (the backend for **out**, the frontend for **in**) reads from the
-array in the following way:
-
-- read *prod*, *cons* from shared memory
-- read memory barrier
-- verify *cons* against local copy (producer shouldn't change it)
-- read from array at position *cons* up to *prod*, wrapping around the circular
- buffer when necessary
-- general memory barrier
-- increase *cons*
-- notify the other end via event channel
-
-The producer takes care of writing only as many bytes as available in the buffer
-up to *cons*. The consumer takes care of reading only as many bytes as available
-in the buffer up to *prod*.
-
-
-## Request/Response Workflow
-
-The client chooses one of the available rings, then it sends a request
-to the other end on the *out* array, following the producer workflow
-described in [Ring Usage].
-
-The server receives the notification and reads the request, following
-the consumer workflow described in [Ring Usage]. The server knows how
-much to read because it is specified in the *size* field of the 9pfs
-header. The server processes the request and sends back a response on
-the *in* array of the same ring, following the producer workflow as
-usual. Thus, every request/response pair is on one ring.
-
-The client receives a notification and reads the response from the *in*
-array. The client knows how much data to read because it is specified in
-the *size* field of the 9pfs header.
-
-
-[paper]: https://www.usenix.org/legacy/event/usenix05/tech/freenix/full_papers/hensbergen/hensbergen.pdf
-[website]: https://github.com/chaos/diod/blob/master/protocol.md
-[XenbusStateInitialising]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,xenbus.h.html
-[ring.h]: http://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=xen/include/public/io/ring.h;hb=HEAD
--- /dev/null
+# Xen transport for 9pfs version 1
+
+## Background
+
+9pfs is a network filesystem protocol developed for Plan 9. 9pfs is very
+simple and describes a series of commands and responses. It is
+completely independent from the communication channels, in fact many
+clients and servers support multiple channels, usually called
+"transports". For example the Linux client supports tcp and unix
+sockets, fds, virtio and rdma.
+
+
+### 9pfs protocol
+
+This document won't cover the full 9pfs specification. Please refer to
+this [paper] and this [website] for a detailed description of it.
+However it is useful to know that each 9pfs request and response has the
+following header:
+
+ struct header {
+ uint32_t size;
+ uint8_t id;
+ uint16_t tag;
+ } __attribute__((packed));
+
+ 0 4 5 7
+ +---------+--+----+
+ | size |id|tag |
+ +---------+--+----+
+
+- *size*
+The size of the request or response.
+
+- *id*
+The 9pfs request or response operation.
+
+- *tag*
+Unique id that identifies a specific request/response pair. It is used
+to multiplex operations on a single channel.
+
+It is possible to have multiple requests in-flight at any given time.
+
+
+## Rationale
+
+This document describes a Xen based transport for 9pfs, in the
+traditional PV frontend and backend format. The PV frontend is used by
+the client to send commands to the server. The PV backend is used by the
+9pfs server to receive commands from clients and send back responses.
+
+The transport protocol supports multiple rings up to the maximum
+supported by the backend. The size of every ring is also configurable
+and can span multiple pages, up to the maximum supported by the backend
+(although it cannot be more than 2MB). The design is to exploit
+parallelism at the vCPU level and support multiple outstanding requests
+simultaneously.
+
+This document does not cover the 9pfs client/server design or
+implementation, only the transport for it.
+
+
+## Xenstore
+
+The frontend and the backend connect via xenstore to exchange
+information. The toolstack creates front and back nodes with state
+[XenbusStateInitialising]. The protocol node name is **9pfs**.
+
+Multiple rings are supported for each frontend and backend connection.
+
+### Backend XenBus Nodes
+
+Backend specific properties, written by the backend, read by the
+frontend:
+
+ versions
+ Values: <string>
+
+ List of comma separated protocol versions supported by the backend.
+ For example "1,2,3". Currently the value is just "1", as there is
+ only one version. N.B.: this is the version of the Xen trasport
+ protocol, not the version of 9pfs supported by the server.
+
+ max-rings
+ Values: <uint32_t>
+
+ The maximum supported number of rings per frontend.
+
+ max-ring-page-order
+ Values: <uint32_t>
+
+ The maximum supported size of a memory allocation in units of
+ log2n(machine pages), e.g. 1 = 2 pages, 2 == 4 pages, etc. It
+ must be at least 1.
+
+Backend configuration nodes, written by the toolstack, read by the
+backend:
+
+ path
+ Values: <string>
+
+ Host filesystem path to share.
+
+ tag
+ Values: <string>
+
+ Alphanumeric tag that identifies the 9pfs share. The client needs
+ to know the tag to be able to mount it.
+
+ security-model
+ Values: "none"
+
+ *none*: files are stored using the same credentials as they are
+ created on the guest (no user ownership squash or remap)
+ Only "none" is supported in this version of the protocol.
+
+### Frontend XenBus Nodes
+
+ version
+ Values: <string>
+
+ Protocol version, chosen among the ones supported by the backend
+ (see **versions** under [Backend XenBus Nodes]). Currently the
+ value must be "1".
+
+ num-rings
+ Values: <uint32_t>
+
+ Number of rings. It needs to be lower or equal to max-rings.
+
+ event-channel-<num> (event-channel-0, event-channel-1, etc)
+ Values: <uint32_t>
+
+ The identifier of the Xen event channel used to signal activity
+ in the ring buffer. One for each ring.
+
+ ring-ref<num> (ring-ref0, ring-ref1, etc)
+ Values: <uint32_t>
+
+ The Xen grant reference granting permission for the backend to
+ map a page with information to setup a share ring. One for each
+ ring.
+
+### State Machine
+
+Initialization:
+
+ *Front* *Back*
+ XenbusStateInitialising XenbusStateInitialising
+ - Query virtual device - Query backend device
+ properties. identification data.
+ - Setup OS device instance. - Publish backend features
+ - Allocate and initialize the and transport parameters
+ request ring. |
+ - Publish transport parameters |
+ that will be in effect during V
+ this connection. XenbusStateInitWait
+ |
+ |
+ V
+ XenbusStateInitialised
+
+ - Query frontend transport parameters.
+ - Connect to the request ring and
+ event channel.
+ |
+ |
+ V
+ XenbusStateConnected
+
+ - Query backend device properties.
+ - Finalize OS virtual device
+ instance.
+ |
+ |
+ V
+ XenbusStateConnected
+
+Once frontend and backend are connected, they have a shared page per
+ring, which are used to setup the rings, and an event channel per ring,
+which are used to send notifications.
+
+Shutdown:
+
+ *Front* *Back*
+ XenbusStateConnected XenbusStateConnected
+ |
+ |
+ V
+ XenbusStateClosing
+
+ - Unmap grants
+ - Unbind evtchns
+ |
+ |
+ V
+ XenbusStateClosing
+
+ - Unbind evtchns
+ - Free rings
+ - Free data structures
+ |
+ |
+ V
+ XenbusStateClosed
+
+ - Free remaining data structures
+ |
+ |
+ V
+ XenbusStateClosed
+
+
+## Ring Setup
+
+The shared page has the following layout:
+
+ typedef uint32_t XEN_9PFS_RING_IDX;
+
+ struct xen_9pfs_intf {
+ XEN_9PFS_RING_IDX in_cons, in_prod;
+ uint8_t pad[56];
+ XEN_9PFS_RING_IDX out_cons, out_prod;
+ uint8_t pad[56];
+
+ uint32_t ring_order;
+ /* this is an array of (1 << ring_order) elements */
+ grant_ref_t ref[1];
+ };
+
+ /* not actually C compliant (ring_order changes from ring to ring) */
+ struct ring_data {
+ char in[((1 << ring_order) << PAGE_SHIFT) / 2];
+ char out[((1 << ring_order) << PAGE_SHIFT) / 2];
+ };
+
+- **ring_order**
+ It represents the order of the data ring. The following list of grant
+ references is of `(1 << ring_order)` elements. It cannot be greater than
+ **max-ring-page-order**, as specified by the backend on XenBus.
+- **ref[]**
+ The list of grant references which will contain the actual data. They are
+ mapped contiguosly in virtual memory. The first half of the pages is the
+ **in** array, the second half is the **out** array. The array must
+ have a power of two number of elements.
+- **out** is an array used as circular buffer
+ It contains client requests. The producer is the frontend, the
+ consumer is the backend.
+- **in** is an array used as circular buffer
+ It contains server responses. The producer is the backend, the
+ consumer is the frontend.
+- **out_cons**, **out_prod**
+ Consumer and producer indices for client requests. They keep track of
+ how much data has been written by the frontend to **out** and how much
+ data has already been consumed by the backend. **out_prod** is
+ increased by the frontend, after writing data to **out**. **out_cons**
+ is increased by the backend, after reading data from **out**.
+- **in_cons** and **in_prod**
+ Consumer and producer indices for responses. They keep track of how
+ much data has already been consumed by the frontend from the **in**
+ array. **in_prod** is increased by the backend, after writing data to
+ **in**. **in_cons** is increased by the frontend, after reading data
+ from **in**.
+
+The binary layout of `struct xen_9pfs_intf` follows:
+
+ 0 4 8 64 68 72 76
+ +---------+---------+-----//-----+---------+---------+---------+
+ | in_cons | in_prod | padding |out_cons |out_prod |ring_orde|
+ +---------+---------+-----//-----+---------+---------+---------+
+
+ 76 80 84 4092 4096
+ +---------+---------+----//---+---------+
+ | ref[0] | ref[1] | | ref[N] |
+ +---------+---------+----//---+---------+
+
+**N.B** For one page, N is maximum 991 (4096-132)/4, but given that N
+needs to be a power of two, actually max N is 512. As 512 == (1 << 9),
+the maximum possible max-ring-page-order value is 9.
+
+The binary layout of the ring buffers follow:
+
+ 0 ((1<<ring_order)<<PAGE_SHIFT)/2 ((1<<ring_order)<<PAGE_SHIFT)
+ +------------//-------------+------------//-------------+
+ | in | out |
+ +------------//-------------+------------//-------------+
+
+## Why ring.h is not needed
+
+Many Xen PV protocols use the macros provided by [ring.h] to manage
+their shared ring for communication. This procotol does not, because it
+actually comes with two rings: the **in** ring and the **out** ring.
+Each of them is mono-directional, and there is no static request size:
+the producer writes opaque data to the ring. On the other end, in
+[ring.h] they are combined, and the request size is static and
+well-known. In this protocol:
+
+ in -> backend to frontend only
+ out-> frontend to backend only
+
+In the case of the **in** ring, the frontend is the consumer, and the
+backend is the producer. Everything is the same but mirrored for the
+**out** ring.
+
+The producer, the backend in this case, never reads from the **in**
+ring. In fact, the producer doesn't need any notifications unless the
+ring is full. This version of the protocol doesn't take advantage of it,
+leaving room for optimizations.
+
+On the other end, the consumer always requires notifications, unless it
+is already actively reading from the ring. The producer can figure it
+out, without any additional fields in the protocol, by comparing the
+indexes at the beginning and the end of the function. This is similar to
+what [ring.h] does.
+
+## Ring Usage
+
+The **in** and **out** arrays are used as circular buffers:
+
+ 0 sizeof(array) == ((1<<ring_order)<<PAGE_SHIFT)/2
+ +-----------------------------------+
+ |to consume| free |to consume |
+ +-----------------------------------+
+ ^ ^
+ prod cons
+
+ 0 sizeof(array)
+ +-----------------------------------+
+ | free | to consume | free |
+ +-----------------------------------+
+ ^ ^
+ cons prod
+
+The following functions are provided to read and write to an array:
+
+ #define MASK_XEN_9PFS_IDX(idx) ((idx) & (XEN_9PFS_RING_SIZE - 1))
+
+ static inline void xen_9pfs_read(char *buf,
+ XEN_9PFS_RING_IDX *masked_prod, XEN_9PFS_RING_IDX *masked_cons,
+ uint8_t *h, size_t len) {
+ if (*masked_cons < *masked_prod) {
+ memcpy(h, buf + *masked_cons, len);
+ } else {
+ if (len > XEN_9PFS_RING_SIZE - *masked_cons) {
+ memcpy(h, buf + *masked_cons, XEN_9PFS_RING_SIZE - *masked_cons);
+ memcpy((char *)h + XEN_9PFS_RING_SIZE - *masked_cons, buf, len - (XEN_9PFS_RING_SIZE - *masked_cons));
+ } else {
+ memcpy(h, buf + *masked_cons, len);
+ }
+ }
+ *masked_cons = _MASK_XEN_9PFS_IDX(*masked_cons + len);
+ }
+
+ static inline void xen_9pfs_write(char *buf,
+ XEN_9PFS_RING_IDX *masked_prod, XEN_9PFS_RING_IDX *masked_cons,
+ uint8_t *opaque, size_t len) {
+ if (*masked_prod < *masked_cons) {
+ memcpy(buf + *masked_prod, opaque, len);
+ } else {
+ if (len > XEN_9PFS_RING_SIZE - *masked_prod) {
+ memcpy(buf + *masked_prod, opaque, XEN_9PFS_RING_SIZE - *masked_prod);
+ memcpy(buf, opaque + (XEN_9PFS_RING_SIZE - *masked_prod), len - (XEN_9PFS_RING_SIZE - *masked_prod));
+ } else {
+ memcpy(buf + *masked_prod, opaque, len);
+ }
+ }
+ *masked_prod = _MASK_XEN_9PFS_IDX(*masked_prod + len);
+ }
+
+The producer (the backend for **in**, the frontend for **out**) writes to the
+array in the following way:
+
+- read *cons*, *prod* from shared memory
+- general memory barrier
+- verify *prod* against local copy (consumer shouldn't change it)
+- write to array at position *prod* up to *cons*, wrapping around the circular
+ buffer when necessary
+- write memory barrier
+- increase *prod*
+- notify the other end via event channel
+
+The consumer (the backend for **out**, the frontend for **in**) reads from the
+array in the following way:
+
+- read *prod*, *cons* from shared memory
+- read memory barrier
+- verify *cons* against local copy (producer shouldn't change it)
+- read from array at position *cons* up to *prod*, wrapping around the circular
+ buffer when necessary
+- general memory barrier
+- increase *cons*
+- notify the other end via event channel
+
+The producer takes care of writing only as many bytes as available in the buffer
+up to *cons*. The consumer takes care of reading only as many bytes as available
+in the buffer up to *prod*.
+
+
+## Request/Response Workflow
+
+The client chooses one of the available rings, then it sends a request
+to the other end on the *out* array, following the producer workflow
+described in [Ring Usage].
+
+The server receives the notification and reads the request, following
+the consumer workflow described in [Ring Usage]. The server knows how
+much to read because it is specified in the *size* field of the 9pfs
+header. The server processes the request and sends back a response on
+the *in* array of the same ring, following the producer workflow as
+usual. Thus, every request/response pair is on one ring.
+
+The client receives a notification and reads the response from the *in*
+array. The client knows how much data to read because it is specified in
+the *size* field of the 9pfs header.
+
+
+[paper]: https://www.usenix.org/legacy/event/usenix05/tech/freenix/full_papers/hensbergen/hensbergen.pdf
+[website]: https://github.com/chaos/diod/blob/master/protocol.md
+[XenbusStateInitialising]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,xenbus.h.html
+[ring.h]: http://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=xen/include/public/io/ring.h;hb=HEAD
+++ /dev/null
-# Coverage support for Xen
-
-Coverage support allows you to get coverage information from Xen execution.
-You can see how many times a line is executed.
-
-Some compilers have specific options that enable the collection of this
-information. Every basic block in the code will be instrumented by the compiler
-to compute these statistics. It should not be used in production as it slows
-down your hypervisor.
-
-# GCOV (GCC coverage)
-
-## Enable coverage
-
-Test coverage support can be turned on compiling Xen with the `CONFIG_COVERAGE`
-option set to `y`.
-
-Change your `.config` or run `make -C xen menuconfig`.
-
-## Extract coverage data
-
-To extract data you use a simple utility called `xencov`.
-It allows you to do 2 operations:
-
-* `xencov read` extract data
-* `xencov reset` reset all coverage counters
-
-Another utility (`xencov_split`) is used to split extracted data file into
-files needed by userspace tools.
-
-## Split coverage data
-
-Once you extracted data from Xen, it is time to create files which the coverage
-tools can understand. To do it you need to run `xencov_split` utility.
-
-The utility just takes an input file and splits the blob into gcc .gcda files
-in the same directory that you execute the script. As file names are generated
-relative to the current directory, it could be a good idea to run the script
-from `/` on your build machine.
-
-Code for splitting the blob is put in another utility for some reason:
-* It is simpler to maintain a high level script than a C program;
-* You don't need to execute on the Xen host so you just need to copy the file to
- your development box (you usually need development files anyway).
-
-## Possible use
-
-**This section is just an example on how to use these tools!**
-
-This example assumes you compiled Xen from `~/xen-unstable` and installed into
-the host. **Consider that if you even recompile Xen you are not able to use
-blob extracted from xencov!**
-
-* Ensure the `lcov` package is installed
-* From the Xen host machine extract the coverage blob
-
- cd /root
- xencov read coverage.dat
-
-* Copy the extracted blob to your dev machine
-
- cd ~
- scp root@myhost:coverage.dat
-
-* Extract the coverage information
-
- (cd / && xencov_split ~/coverage.dat)
-
-* Produce coverage html output
-
- cd ~/xen-unstable
- rm -rf cov.info cov
- geninfo -o cov.info xen
- mkdir cov
- genhtml -o cov cov.info
-
-* See output in a browser
-
- firefox cov/index.html
-
-# LLVM coverage
-
-## Enable coverage
-
-Coverage can be enabled using a Kconfig option, from the top-level directory
-use the following command to display the Kconfig menu:
-
- make -C xen menuconfig clang=y
-
-The code coverage option can be found inside of the "Debugging Options"
-section. After enabling it just compile Xen as you would normally do:
-
- make xen clang=y
-
-## Extract coverage data
-
-LLVM coverage can be extracted from the hypervisor using the `xencov` tool.
-The following actions are available:
-
-* `xencov read` extract data
-* `xencov reset` reset all coverage counters
-* `xencov read-reset` extract data and reset counters at the same time.
-
-## Possible use
-
-**This section is just an example on how to use these tools!**
-
-This example assumes you compiled Xen and copied the xen-syms file from
-xen/xen-syms into your current directory.
-
-* Extract the coverage data from Xen:
-
- xencov read xen.profraw
-
-* Convert the data into a profile. Note that you can merge more than one
- profraw file into a single profdata file.
-
- llvm-profdata merge xen.profraw -o xen.profdata
-
-* Generate a HTML report of the code coverage:
-
- llvm-cov show -format=html -output-dir=cov/ xen-syms -instr-profile=xen.profdata
-
-* Open cov/index.html with your browser in order to display the profile.
--- /dev/null
+# Coverage support for Xen
+
+Coverage support allows you to get coverage information from Xen execution.
+You can see how many times a line is executed.
+
+Some compilers have specific options that enable the collection of this
+information. Every basic block in the code will be instrumented by the compiler
+to compute these statistics. It should not be used in production as it slows
+down your hypervisor.
+
+# GCOV (GCC coverage)
+
+## Enable coverage
+
+Test coverage support can be turned on compiling Xen with the `CONFIG_COVERAGE`
+option set to `y`.
+
+Change your `.config` or run `make -C xen menuconfig`.
+
+## Extract coverage data
+
+To extract data you use a simple utility called `xencov`.
+It allows you to do 2 operations:
+
+* `xencov read` extract data
+* `xencov reset` reset all coverage counters
+
+Another utility (`xencov_split`) is used to split extracted data file into
+files needed by userspace tools.
+
+## Split coverage data
+
+Once you extracted data from Xen, it is time to create files which the coverage
+tools can understand. To do it you need to run `xencov_split` utility.
+
+The utility just takes an input file and splits the blob into gcc .gcda files
+in the same directory that you execute the script. As file names are generated
+relative to the current directory, it could be a good idea to run the script
+from `/` on your build machine.
+
+Code for splitting the blob is put in another utility for some reason:
+* It is simpler to maintain a high level script than a C program;
+* You don't need to execute on the Xen host so you just need to copy the file to
+ your development box (you usually need development files anyway).
+
+## Possible use
+
+**This section is just an example on how to use these tools!**
+
+This example assumes you compiled Xen from `~/xen-unstable` and installed into
+the host. **Consider that if you even recompile Xen you are not able to use
+blob extracted from xencov!**
+
+* Ensure the `lcov` package is installed
+* From the Xen host machine extract the coverage blob
+
+ cd /root
+ xencov read coverage.dat
+
+* Copy the extracted blob to your dev machine
+
+ cd ~
+ scp root@myhost:coverage.dat
+
+* Extract the coverage information
+
+ (cd / && xencov_split ~/coverage.dat)
+
+* Produce coverage html output
+
+ cd ~/xen-unstable
+ rm -rf cov.info cov
+ geninfo -o cov.info xen
+ mkdir cov
+ genhtml -o cov cov.info
+
+* See output in a browser
+
+ firefox cov/index.html
+
+# LLVM coverage
+
+## Enable coverage
+
+Coverage can be enabled using a Kconfig option, from the top-level directory
+use the following command to display the Kconfig menu:
+
+ make -C xen menuconfig clang=y
+
+The code coverage option can be found inside of the "Debugging Options"
+section. After enabling it just compile Xen as you would normally do:
+
+ make xen clang=y
+
+## Extract coverage data
+
+LLVM coverage can be extracted from the hypervisor using the `xencov` tool.
+The following actions are available:
+
+* `xencov read` extract data
+* `xencov reset` reset all coverage counters
+* `xencov read-reset` extract data and reset counters at the same time.
+
+## Possible use
+
+**This section is just an example on how to use these tools!**
+
+This example assumes you compiled Xen and copied the xen-syms file from
+xen/xen-syms into your current directory.
+
+* Extract the coverage data from Xen:
+
+ xencov read xen.profraw
+
+* Convert the data into a profile. Note that you can merge more than one
+ profraw file into a single profdata file.
+
+ llvm-profdata merge xen.profraw -o xen.profdata
+
+* Generate a HTML report of the code coverage:
+
+ llvm-cov show -format=html -output-dir=cov/ xen-syms -instr-profile=xen.profdata
+
+* Open cov/index.html with your browser in order to display the profile.
+++ /dev/null
-For x86, building xen.efi requires gcc 4.5.x or above (4.6.x or newer
-recommended, as 4.5.x was probably never really tested for this purpose) and
-binutils 2.22 or newer. Additionally, the binutils build must be configured to
-include support for the x86_64-pep emulation (i.e.
-`--enable-targets=x86_64-pep` or an option of equivalent effect should be
-passed to the configure script).
-
-For arm64, the PE/COFF header is open-coded in assembly, so no toolchain
-support for PE/COFF is required. Also, the PE/COFF header co-exists with the
-normal Image format, so a single binary may be booted as an Image file or as an
-EFI application. When booted as an EFI application, Xen requires a
-configuration file as described below unless a bootloader, such as GRUB, has
-loaded the modules and describes them in the device tree provided to Xen. If a
-bootloader provides a device tree containing modules then any configuration
-files are ignored, and the bootloader is responsible for populating all
-relevant device tree nodes.
-
-Once built, `make install-xen` will place the resulting binary directly into
-the EFI boot partition, provided `EFI_VENDOR` is set in the environment (and
-`EFI_MOUNTPOINT` is overridden as needed, should the default of `/boot/efi` not
-match your system). The xen.efi binary will also be installed in
-`/usr/lib64/efi/`, unless `EFI_DIR` is set in the environment to override this
-default.
-
-The binary itself will require a configuration file (names with the `.efi`
-extension of the binary's name replaced by `.cfg`, and - until an existing
-file is found - trailing name components dropped at `.`, `-`, and `_`
-separators will be tried) to be present in the same directory as the binary.
-(To illustrate the name handling, a binary named `xen-4.2-unstable.efi` would
-try `xen-4.2-unstable.cfg`, `xen-4.2.cfg`, `xen-4.cfg`, and `xen.cfg` in
-order.) One can override this with a command line option (`-cfg=<filename>`).
-This configuration file and EFI commandline are only used for booting directly
-from EFI firmware, or when using an EFI loader that does not support
-the multiboot2 protocol. When booting using GRUB or another multiboot aware
-loader the EFI commandline is ignored and all information is passed from
-the loader to Xen using the multiboot protocol.
-
-The configuration file consists of one or more sections headed by a section
-name enclosed in square brackets, with individual values specified in each
-section. A section named `[global]` is treated specially to allow certain
-settings to apply to all other sections (or to provide defaults for certain
-settings in case individual sections don't specify them). This file (for now)
-needs to be of ASCII type and not e.g. UTF-8 or UTF-16. A typical file would
-thus look like this (`#` serving as comment character):
-
- **************************example begin******************************
-
- [global]
- default=sle11sp2
-
- [sle11sp2]
- options=console=vga,com1 com1=57600 loglvl=all noreboot
- kernel=vmlinuz-3.0.31-0.4-xen [domain 0 command line options]
- ramdisk=initrd-3.0.31-0.4-xen
-
- **************************example end********************************
-
-The individual values used here are:
-
-###`default=<name>`
-
-Specifies the section to use for booting, if none was specified on the command
-line; only meaningful in the `[global]` section. This isn't required; if
-absent, section headers will be ignored and for each value looked for the
-first instance within the file will be used.
-
-###`options=<text>`
-
-Specifies the options passed to the hypervisor, see [Xen Hypervisor Command
-Line Options](xen-command-line.html).
-
-###`kernel=<filename>[ <options>]`
-
-Specifies the Dom0 kernel binary and the options to pass to it.
-
-The options should in general be the same as is used when booting
-natively, e.g. including `root=...` etc.
-
-Check your bootloader (e.g. grub) configuration or `/proc/cmdline` for
-the native configuration.
-
-###`ramdisk=<filename>`
-
-Specifies a Linux-style initial RAM disk image to load.
-
-Other values to specify are:
-
-###`video=gfx-<xres>[x<yres>[x<depth>]]`
-
-Specifies a video mode to select if available. In case of problems, the
-`-basevideo` command line option can be used to skip altering video modes.
-
-###`xsm=<filename>`
-
-Specifies an XSM module to load.
-
-###`ucode=<filename>`
-
-Specifies a CPU microcode blob to load. (x86 only)
-
-###`dtb=<filename>`
-
-Specifies a device tree file to load. The platform firmware may provide a
-DTB in an EFI configuration table, so this field is optional in that
-case. A dtb specified in the configuration file will override a device tree
-provided in the EFI configuration table. (ARM only)
-
-###`chain=<filename>`
-
-Specifies an alternate configuration file to use in case the specified section
-(and in particular its `kernel=` setting) can't be found in the default (or
-specified) configuration file. This is only meaningful in the [global] section
-and really not meant to be used together with the `-cfg=` command line option.
-
-Filenames must be specified relative to the location of the EFI binary.
-
-Extra options to be passed to Xen can also be specified on the command line,
-following a `--` separator option.
--- /dev/null
+For x86, building xen.efi requires gcc 4.5.x or above (4.6.x or newer
+recommended, as 4.5.x was probably never really tested for this purpose) and
+binutils 2.22 or newer. Additionally, the binutils build must be configured to
+include support for the x86_64-pep emulation (i.e.
+`--enable-targets=x86_64-pep` or an option of equivalent effect should be
+passed to the configure script).
+
+For arm64, the PE/COFF header is open-coded in assembly, so no toolchain
+support for PE/COFF is required. Also, the PE/COFF header co-exists with the
+normal Image format, so a single binary may be booted as an Image file or as an
+EFI application. When booted as an EFI application, Xen requires a
+configuration file as described below unless a bootloader, such as GRUB, has
+loaded the modules and describes them in the device tree provided to Xen. If a
+bootloader provides a device tree containing modules then any configuration
+files are ignored, and the bootloader is responsible for populating all
+relevant device tree nodes.
+
+Once built, `make install-xen` will place the resulting binary directly into
+the EFI boot partition, provided `EFI_VENDOR` is set in the environment (and
+`EFI_MOUNTPOINT` is overridden as needed, should the default of `/boot/efi` not
+match your system). The xen.efi binary will also be installed in
+`/usr/lib64/efi/`, unless `EFI_DIR` is set in the environment to override this
+default.
+
+The binary itself will require a configuration file (names with the `.efi`
+extension of the binary's name replaced by `.cfg`, and - until an existing
+file is found - trailing name components dropped at `.`, `-`, and `_`
+separators will be tried) to be present in the same directory as the binary.
+(To illustrate the name handling, a binary named `xen-4.2-unstable.efi` would
+try `xen-4.2-unstable.cfg`, `xen-4.2.cfg`, `xen-4.cfg`, and `xen.cfg` in
+order.) One can override this with a command line option (`-cfg=<filename>`).
+This configuration file and EFI commandline are only used for booting directly
+from EFI firmware, or when using an EFI loader that does not support
+the multiboot2 protocol. When booting using GRUB or another multiboot aware
+loader the EFI commandline is ignored and all information is passed from
+the loader to Xen using the multiboot protocol.
+
+The configuration file consists of one or more sections headed by a section
+name enclosed in square brackets, with individual values specified in each
+section. A section named `[global]` is treated specially to allow certain
+settings to apply to all other sections (or to provide defaults for certain
+settings in case individual sections don't specify them). This file (for now)
+needs to be of ASCII type and not e.g. UTF-8 or UTF-16. A typical file would
+thus look like this (`#` serving as comment character):
+
+ **************************example begin******************************
+
+ [global]
+ default=sle11sp2
+
+ [sle11sp2]
+ options=console=vga,com1 com1=57600 loglvl=all noreboot
+ kernel=vmlinuz-3.0.31-0.4-xen [domain 0 command line options]
+ ramdisk=initrd-3.0.31-0.4-xen
+
+ **************************example end********************************
+
+The individual values used here are:
+
+###`default=<name>`
+
+Specifies the section to use for booting, if none was specified on the command
+line; only meaningful in the `[global]` section. This isn't required; if
+absent, section headers will be ignored and for each value looked for the
+first instance within the file will be used.
+
+###`options=<text>`
+
+Specifies the options passed to the hypervisor, see [Xen Hypervisor Command
+Line Options](xen-command-line.html).
+
+###`kernel=<filename>[ <options>]`
+
+Specifies the Dom0 kernel binary and the options to pass to it.
+
+The options should in general be the same as is used when booting
+natively, e.g. including `root=...` etc.
+
+Check your bootloader (e.g. grub) configuration or `/proc/cmdline` for
+the native configuration.
+
+###`ramdisk=<filename>`
+
+Specifies a Linux-style initial RAM disk image to load.
+
+Other values to specify are:
+
+###`video=gfx-<xres>[x<yres>[x<depth>]]`
+
+Specifies a video mode to select if available. In case of problems, the
+`-basevideo` command line option can be used to skip altering video modes.
+
+###`xsm=<filename>`
+
+Specifies an XSM module to load.
+
+###`ucode=<filename>`
+
+Specifies a CPU microcode blob to load. (x86 only)
+
+###`dtb=<filename>`
+
+Specifies a device tree file to load. The platform firmware may provide a
+DTB in an EFI configuration table, so this field is optional in that
+case. A dtb specified in the configuration file will override a device tree
+provided in the EFI configuration table. (ARM only)
+
+###`chain=<filename>`
+
+Specifies an alternate configuration file to use in case the specified section
+(and in particular its `kernel=` setting) can't be found in the default (or
+specified) configuration file. This is only meaningful in the [global] section
+and really not meant to be used together with the `-cfg=` command line option.
+
+Filenames must be specified relative to the location of the EFI binary.
+
+Extra options to be passed to Xen can also be specified on the command line,
+following a `--` separator option.
+++ /dev/null
-#Xen HVM emulated device unplug protocol
-
-The protocol covers three basic things:
-
- * Disconnecting emulated devices.
- * Getting log messages out of the drivers and into dom0.
- * Allowing dom0 to block the loading of specific drivers. This is
- intended as a backwards-compatibility thing: if we discover a bug
- in some old version of the drivers, then rather than working around
- it in Xen, we have the option of just making those drivers fall
- back to emulated mode.
-
-The current protocol works like this (from the point of view of
-drivers):
-
-1. When the drivers first come up, they check whether the unplug logic
- is available by reading a two-byte magic number from IO port `0x10`.
- These should be `0x49d2`. If the magic number doesn't match, the
- drivers don't do anything.
-
-2. The drivers read a one-byte protocol version from IO port `0x12`. If
- this is 0, skip to 6.
-
-3. The drivers write a two-byte product number to IO port `0x12`. At
- the moment, the only drivers using this protocol are our
- closed-source ones, which use product number 1.
-
-4. The drivers write a four-byte build number to IO port `0x10`.
-
-5. The drivers check the magic number by reading two bytes from `0x10`
- again. If it's changed from `0x49d2` to `0xd249`, the drivers are
- blacklisted and should not load.
-
-6. The drivers write a two-byte bitmask of devices to unplug to IO
- port `0x10`. The defined bits are:
-
- * `0` -- All emulated IDE and SCSI disks (not including CD drives).
- * `1` -- All emulated NICs.
- * `2` -- All IDE disks except for the primary master (not including CD
- drives). This is overridden by bit 0.
- * `3` -- All emulated NVMe disks.
-
- The relevant emulated devices then disappear from the relevant
- buses. For most guest operating systems, you want to do this
- before device enumeration happens.
-
-Once the drivers have checked the magic number, they can send log
-messages to qemu which will be logged to wherever qemu's logs go
-(`/var/log/xen/qemu-dm.log` on normal Xen, dom0 syslog on XenServer).
-These messages are written to IO port `0x12` a byte at a time, and are
-terminated by newlines. There's a fairly aggressive rate limiter on
-these messages, so they shouldn't be used for anything even vaguely
-high-volume, but they're rather useful for debugging and support.
-
-It is still permitted for a driver to use this logging feature if it
-is blacklisted, but *ONLY* if it has checked the magic number and found
-it to be `0x49d2` or `0xd249`.
-
-This isn't exactly a pretty protocol, but it does solve the problem.
-
-The blacklist is, from qemu's point of view, handled mostly through
-xenstore. A driver version is considered to be blacklisted if
-`/mh/driver-blacklist/{product_name}/{build_number}` exists and is
-readable, where `{build_number}` is the build number from step 4 as a
-decimal number. `{product_name}` is a string corresponding to the
-product number in step 3.
-
-The master registry of product names and numbers is in
-xen/include/public/hvm/pvdrivers.h.
-
-NOTE: The IO ports implementing the unplug protocol are implemented
-as part of the Xen Platform PCI Device, so if that device is not
-present in the system then this protocol will not work.
-
-
-Unplug protocol for old SUSE PVonHVM
-
-During xen-3.0.4 timeframe an unofficial unplug protocol was added to
-the xen-platform-pci kernel module. The value 0x1 was written to offset
-0x4 in the memory region of the Xen Platform PCI Device. This was done
-unconditionally. The corresponding code in qemu-xen-traditional did an
-unplug of all NIC, IDE and SCSI devices. This was used in all SUSE
-releases up to openSUSE 12.3, SLES11SP3. Starting with openSUSE 13.1 and
-SLES11SP4/SLE12 the official protocol was used.
-
-Unplug protocol for old Novell VMDP
-
-During Xen-3.0 timeframe an unofficial unplug protocol was used in
-Novells VMDP. Depending on how VMDP was configured it would control all
-devices, or either NIC or storage. To control all devices the value 0x1
-was written to offset 0x4 in the memory region of the Xen Platform PCI
-Device. This was supposed to unplug NIC, IDE and SCSI devices. If VMDP
-was configured to control just NIC devices it would write the value 0x2
-to offset 0x8. If VMDP was configured to control just storage devices it
-would write the value 0x1 to offset 0x8. Starting with VMDP version 1.7
-(released 2011) the official protocol was used.
-
--- /dev/null
+#Xen HVM emulated device unplug protocol
+
+The protocol covers three basic things:
+
+ * Disconnecting emulated devices.
+ * Getting log messages out of the drivers and into dom0.
+ * Allowing dom0 to block the loading of specific drivers. This is
+ intended as a backwards-compatibility thing: if we discover a bug
+ in some old version of the drivers, then rather than working around
+ it in Xen, we have the option of just making those drivers fall
+ back to emulated mode.
+
+The current protocol works like this (from the point of view of
+drivers):
+
+1. When the drivers first come up, they check whether the unplug logic
+ is available by reading a two-byte magic number from IO port `0x10`.
+ These should be `0x49d2`. If the magic number doesn't match, the
+ drivers don't do anything.
+
+2. The drivers read a one-byte protocol version from IO port `0x12`. If
+ this is 0, skip to 6.
+
+3. The drivers write a two-byte product number to IO port `0x12`. At
+ the moment, the only drivers using this protocol are our
+ closed-source ones, which use product number 1.
+
+4. The drivers write a four-byte build number to IO port `0x10`.
+
+5. The drivers check the magic number by reading two bytes from `0x10`
+ again. If it's changed from `0x49d2` to `0xd249`, the drivers are
+ blacklisted and should not load.
+
+6. The drivers write a two-byte bitmask of devices to unplug to IO
+ port `0x10`. The defined bits are:
+
+ * `0` -- All emulated IDE and SCSI disks (not including CD drives).
+ * `1` -- All emulated NICs.
+ * `2` -- All IDE disks except for the primary master (not including CD
+ drives). This is overridden by bit 0.
+ * `3` -- All emulated NVMe disks.
+
+ The relevant emulated devices then disappear from the relevant
+ buses. For most guest operating systems, you want to do this
+ before device enumeration happens.
+
+Once the drivers have checked the magic number, they can send log
+messages to qemu which will be logged to wherever qemu's logs go
+(`/var/log/xen/qemu-dm.log` on normal Xen, dom0 syslog on XenServer).
+These messages are written to IO port `0x12` a byte at a time, and are
+terminated by newlines. There's a fairly aggressive rate limiter on
+these messages, so they shouldn't be used for anything even vaguely
+high-volume, but they're rather useful for debugging and support.
+
+It is still permitted for a driver to use this logging feature if it
+is blacklisted, but *ONLY* if it has checked the magic number and found
+it to be `0x49d2` or `0xd249`.
+
+This isn't exactly a pretty protocol, but it does solve the problem.
+
+The blacklist is, from qemu's point of view, handled mostly through
+xenstore. A driver version is considered to be blacklisted if
+`/mh/driver-blacklist/{product_name}/{build_number}` exists and is
+readable, where `{build_number}` is the build number from step 4 as a
+decimal number. `{product_name}` is a string corresponding to the
+product number in step 3.
+
+The master registry of product names and numbers is in
+xen/include/public/hvm/pvdrivers.h.
+
+NOTE: The IO ports implementing the unplug protocol are implemented
+as part of the Xen Platform PCI Device, so if that device is not
+present in the system then this protocol will not work.
+
+
+Unplug protocol for old SUSE PVonHVM
+
+During xen-3.0.4 timeframe an unofficial unplug protocol was added to
+the xen-platform-pci kernel module. The value 0x1 was written to offset
+0x4 in the memory region of the Xen Platform PCI Device. This was done
+unconditionally. The corresponding code in qemu-xen-traditional did an
+unplug of all NIC, IDE and SCSI devices. This was used in all SUSE
+releases up to openSUSE 12.3, SLES11SP3. Starting with openSUSE 13.1 and
+SLES11SP4/SLE12 the official protocol was used.
+
+Unplug protocol for old Novell VMDP
+
+During Xen-3.0 timeframe an unofficial unplug protocol was used in
+Novells VMDP. Depending on how VMDP was configured it would control all
+devices, or either NIC or storage. To control all devices the value 0x1
+was written to offset 0x4 in the memory region of the Xen Platform PCI
+Device. This was supposed to unplug NIC, IDE and SCSI devices. If VMDP
+was configured to control just NIC devices it would write the value 0x2
+to offset 0x8. If VMDP was configured to control just storage devices it
+would write the value 0x1 to offset 0x8. Starting with VMDP version 1.7
+(released 2011) the official protocol was used.
+
+++ /dev/null
-# Xen Live Patching Design v1
-
-## Rationale
-
-A mechanism is required to binarily patch the running hypervisor with new
-opcodes that have come about due to primarily security updates.
-
-This document describes the design of the API that would allow us to
-upload to the hypervisor binary patches.
-
-The document is split in four sections:
-
- * Detailed descriptions of the problem statement.
- * Design of the data structures.
- * Design of the hypercalls.
- * Implementation notes that should be taken into consideration.
-
-
-## Glossary
-
- * splice - patch in the binary code with new opcodes
- * trampoline - a jump to a new instruction.
- * payload - telemetries of the old code along with binary blob of the new
- function (if needed).
- * reloc - telemetries contained in the payload to construct proper trampoline.
-
-## History
-
-The document has gone under various reviews and only covers v1 design.
-
-The end of the document has a section titled `Not Yet Done` which
-outlines ideas and design for the future version of this work.
-
-## Multiple ways to patch
-
-The mechanism needs to be flexible to patch the hypervisor in multiple ways
-and be as simple as possible. The compiled code is contiguous in memory with
-no gaps - so we have no luxury of 'moving' existing code and must either
-insert a trampoline to the new code to be executed - or only modify in-place
-the code if there is sufficient space. The placement of new code has to be done
-by hypervisor and the virtual address for the new code is allocated dynamically.
-
-This implies that the hypervisor must compute the new offsets when splicing
-in the new trampoline code. Where the trampoline is added (inside
-the function we are patching or just the callers?) is also important.
-
-To lessen the amount of code in hypervisor, the consumer of the API
-is responsible for identifying which mechanism to employ and how many locations
-to patch. Combinations of modifying in-place code, adding trampoline, etc
-has to be supported. The API should allow read/write any memory within
-the hypervisor virtual address space.
-
-We must also have a mechanism to query what has been applied and a mechanism
-to revert it if needed.
-
-## Workflow
-
-The expected workflows of higher-level tools that manage multiple patches
-on production machines would be:
-
- * The first obvious task is loading all available / suggested
- hotpatches when they are available.
- * Whenever new hotpatches are installed, they should be loaded too.
- * One wants to query which modules have been loaded at runtime.
- * If unloading is deemed safe (see unloading below), one may want to
- support a workflow where a specific hotpatch is marked as bad and
- unloaded.
-
-## Patching code
-
-The first mechanism to patch that comes in mind is in-place replacement.
-That is replace the affected code with new code. Unfortunately the x86
-ISA is variable size which places limits on how much space we have available
-to replace the instructions. That is not a problem if the change is smaller
-than the original opcode and we can fill it with nops. Problems will
-appear if the replacement code is longer.
-
-The second mechanism is by ti replace the call or jump to the
-old function with the address of the new function.
-
-A third mechanism is to add a jump to the new function at the
-start of the old function. N.B. The Xen hypervisor implements the third
-mechanism. See `Trampoline (e9 opcode)` section for more details.
-
-### Example of trampoline and in-place splicing
-
-As example we will assume the hypervisor does not have XSA-132 (see
-[domctl/sysctl: don't leak hypervisor stack to toolstacks](http://xenbits.xen.org/gitweb/?p=xen.git;a=commitdiff;h=4ff3449f0e9d175ceb9551d3f2aecb59273f639d))
-and we would like to binary patch the hypervisor with it. The original code
-looks as so:
-
- 48 89 e0 mov %rsp,%rax
- 48 25 00 80 ff ff and $0xffffffffffff8000,%rax
-
-while the new patched hypervisor would be:
-
- 48 c7 45 b8 00 00 00 00 movq $0x0,-0x48(%rbp)
- 48 c7 45 c0 00 00 00 00 movq $0x0,-0x40(%rbp)
- 48 c7 45 c8 00 00 00 00 movq $0x0,-0x38(%rbp)
- 48 89 e0 mov %rsp,%rax
- 48 25 00 80 ff ff and $0xffffffffffff8000,%rax
-
-This is inside the arch\_do\_domctl. This new change adds 21 extra
-bytes of code which alters all the offsets inside the function. To alter
-these offsets and add the extra 21 bytes of code we might not have enough
-space in .text to squeeze this in.
-
-As such we could simplify this problem by only patching the site
-which calls arch\_do\_domctl:
-
- do_domctl:
- e8 4b b1 05 00 callq ffff82d08015fbb9 <arch_do_domctl>
-
-with a new address for where the new `arch_do_domctl` would be (this
-area would be allocated dynamically).
-
-Astute readers will wonder what we need to do if we were to patch `do_domctl`
-- which is not called directly by hypervisor but on behalf of the guests via
-the `compat_hypercall_table` and `hypercall_table`. Patching the offset in
-`hypercall_table` for `do_domctl`:
-
- ffff82d08024d490: 79 30
- ffff82d08024d492: 10 80 d0 82 ff ff
-
-with the new address where the new `do_domctl` is possible. The other
-place where it is used is in `hvm_hypercall64_table` which would need
-to be patched in a similar way. This would require an in-place splicing
-of the new virtual address of `arch_do_domctl`.
-
-In summary this example patched the callee of the affected function by
-
- * Allocating memory for the new code to live in,
- * Changing the virtual address in all the functions which called the old
- code (computing the new offset, patching the callq with a new callq).
- * Changing the function pointer tables with the new virtual address of
- the function (splicing in the new virtual address). Since this table
- resides in the .rodata section we would need to temporarily change the
- page table permissions during this part.
-
-However it has drawbacks - the safety checks which have to make sure
-the function is not on the stack - must also check every caller. For some
-patches this could mean - if there were an sufficient large amount of
-callers - that we would never be able to apply the update.
-
-Having the patching done at predetermined instances where the stacks
-are not deep mostly solves this problem.
-
-### Example of different trampoline patching.
-
-An alternative mechanism exists where we can insert a trampoline in the
-existing function to be patched to jump directly to the new code. This
-lessens the locations to be patched to one but it puts pressure on the
-CPU branching logic (I-cache, but it is just one unconditional jump).
-
-For this example we will assume that the hypervisor has not been compiled with
-XSA-125 (see
-[pre-fill structures for certain HYPERVISOR\_xen\_version sub-ops](http://xenbits.xen.org/gitweb/?p=xen.git;a=commitdiff;h=fe2e079f642effb3d24a6e1a7096ef26e691d93e))
-which mem-sets an structure in `xen_version` hypercall. This function is not
-called **anywhere** in the hypervisor (it is called by the guest) but
-referenced in the `compat_hypercall_table` and `hypercall_table` (and
-indirectly called from that). Patching the offset in `hypercall_table` for the
-old `do_xen_version`:
-
- ffff82d08024b270 <hypercall_table>:
- ...
- ffff82d08024b2f8: 9e 2f 11 80 d0 82 ff ff
-
-with the new address where the new `do_xen_version` is possible. The other
-place where it is used is in `hvm_hypercall64_table` which would need
-to be patched in a similar way. This would require an in-place splicing
-of the new virtual address of `do_xen_version`.
-
-An alternative solution would be to patch insert a trampoline in the
-old `do_xen_version` function to directly jump to the new `do_xen_version`:
-
- ffff82d080112f9e do_xen_version:
- ffff82d080112f9e: 48 c7 c0 da ff ff ff mov $0xffffffffffffffda,%rax
- ffff82d080112fa5: 83 ff 09 cmp $0x9,%edi
- ffff82d080112fa8: 0f 87 24 05 00 00 ja ffff82d0801134d2 ; do_xen_version+0x534
-
-with:
-
- ffff82d080112f9e do_xen_version:
- ffff82d080112f9e: e9 XX YY ZZ QQ jmpq [new do_xen_version]
-
-which would lessen the amount of patching to just one location.
-
-In summary this example patched the affected function to jump to the
-new replacement function which required:
-
- * Allocating memory for the new code to live in,
- * Inserting trampoline with new offset in the old function to point to the
- new function.
- * Optionally we can insert in the old function a trampoline jump to an function
- providing an BUG\_ON to catch errant code.
-
-The disadvantage of this are that the unconditional jump will consume a small
-I-cache penalty. However the simplicity of the patching and higher chance
-of passing safety checks make this a worthwhile option.
-
-This patching has a similar drawback as inline patching - the safety
-checks have to make sure the function is not on the stack. However
-since we are replacing at a higher level (a full function as opposed
-to various offsets within functions) the checks are simpler.
-
-Having the patching done at predetermined instances where the stacks
-are not deep mostly solves this problem as well.
-
-### Security
-
-With this method we can re-write the hypervisor - and as such we **MUST** be
-diligent in only allowing certain guests to perform this operation.
-
-Furthermore with SecureBoot or tboot, we **MUST** also verify the signature
-of the payload to be certain it came from a trusted source and integrity
-was intact.
-
-As such the hypercall **MUST** support an XSM policy to limit what the guest
-is allowed to invoke. If the system is booted with signature checking the
-signature checking will be enforced.
-
-## Design of payload format
-
-The payload **MUST** contain enough data to allow us to apply the update
-and also safely reverse it. As such we **MUST** know:
-
- * The locations in memory to be patched. This can be determined dynamically
- via symbols or via virtual addresses.
- * The new code that will be patched in.
-
-This binary format can be constructed using an custom binary format but
-there are severe disadvantages of it:
-
- * The format might need to be changed and we need an mechanism to accommodate
- that.
- * It has to be platform agnostic.
- * Easily constructed using existing tools.
-
-As such having the payload in an ELF file is the sensible way. We would be
-carrying the various sets of structures (and data) in the ELF sections under
-different names and with definitions.
-
-Note that every structure has padding. This is added so that the hypervisor
-can re-use those fields as it sees fit.
-
-Earlier design attempted to ineptly explain the relations of the ELF sections
-to each other without using proper ELF mechanism (sh\_info, sh\_link, data
-structures using Elf types, etc). This design will explain the structures
-and how they are used together and not dig in the ELF format - except mention
-that the section names should match the structure names.
-
-The Xen Live Patch payload is a relocatable ELF binary. A typical binary would have:
-
- * One or more .text sections.
- * Zero or more read-only data sections.
- * Zero or more data sections.
- * Relocations for each of these sections.
-
-It may also have some architecture-specific sections. For example:
-
- * Alternatives instructions.
- * Bug frames.
- * Exception tables.
- * Relocations for each of these sections.
-
-The Xen Live Patch core code loads the payload as a standard ELF binary, relocates it
-and handles the architecture-specifc sections as needed. This process is much
-like what the Linux kernel module loader does.
-
-The payload contains at least three sections:
-
- * `.livepatch.funcs` - which is an array of livepatch\_func structures.
- * `.livepatch.depends` - which is an ELF Note that describes what the payload
- depends on. **MUST** have one.
- * `.note.gnu.build-id` - the build-id of this payload. **MUST** have one.
-
-### .livepatch.funcs
-
-The `.livepatch.funcs` contains an array of livepatch\_func structures
-which describe the functions to be patched:
-
- struct livepatch_func {
- const char *name;
- void *new_addr;
- void *old_addr;
- uint32_t new_size;
- uint32_t old_size;
- uint8_t version;
- uint8_t opaque[31];
- };
-
-The size of the structure is 64 bytes on 64-bit hypervisors. It will be
-52 on 32-bit hypervisors.
-
- * `name` is the symbol name of the old function. Only used if `old_addr` is
- zero, otherwise will be used during dynamic linking (when hypervisor loads
- the payload).
- * `old_addr` is the address of the function to be patched and is filled in at
- payload generation time if hypervisor function address is known. If unknown,
- the value *MUST* be zero and the hypervisor will attempt to resolve the
- address.
- * `new_addr` can either have a non-zero value or be zero.
- * If there is a non-zero value, then it is the address of the function that
- is replacing the old function and the address is recomputed during
- relocation. The value **MUST** be the address of the new function in the
- payload file.
- * If the value is zero, then we NOPing out at the `old_addr` location
- `new_size` bytes.
- * `old_size` contains the sizes of the respective `old_addr` function in
- bytes. The value of `old_size` **MUST** not be zero.
- * `new_size` depends on what `new_addr` contains:
- * If `new_addr` contains an non-zero value, then `new_size` has the size of
- the new function (which will replace the one at `old_addr`) in bytes.
- * If the value of `new_addr` is zero then `new_size` determines how many
- instruction bytes to NOP (up to opaque size modulo smallest platform
- instruction - 1 byte x86 and 4 bytes on ARM).
- * `version` is to be one.
- * `opaque` **MUST** be zero.
-
-The size of the `livepatch_func` array is determined from the ELF section
-size.
-
-When applying the patch the hypervisor iterates over each `livepatch_func`
-structure and the core code inserts a trampoline at `old_addr` to `new_addr`.
-The `new_addr` is altered when the ELF payload is loaded.
-
-When reverting a patch, the hypervisor iterates over each `livepatch_func`
-and the core code copies the data from the undo buffer (private internal copy)
-to `old_addr`.
-
-It optionally may contain the address of functions to be called right before
-being applied and after being reverted:
-
- * `.livepatch.hooks.load` - an array of function pointers.
- * `.livepatch.hooks.unload` - an array of function pointers.
-
-
-### Example of .livepatch.funcs
-
-A simple example of what a payload file can be:
-
- /* MUST be in sync with hypervisor. */
- struct livepatch_func {
- const char *name;
- void *new_addr;
- void *old_addr;
- uint32_t new_size;
- uint32_t old_size;
- uint8_t version;
- uint8_t pad[31];
- };
-
- /* Our replacement function for xen_extra_version. */
- const char *xen_hello_world(void)
- {
- return "Hello World";
- }
-
- static unsigned char patch_this_fnc[] = "xen_extra_version";
-
- struct livepatch_func livepatch_hello_world = {
- .version = LIVEPATCH_PAYLOAD_VERSION,
- .name = patch_this_fnc,
- .new_addr = xen_hello_world,
- .old_addr = (void *)0xffff82d08013963c, /* Extracted from xen-syms. */
- .new_size = 13, /* To be be computed by scripts. */
- .old_size = 13, /* -----------""--------------- */
- } __attribute__((__section__(".livepatch.funcs")));
-
-Code must be compiled with `-fPIC`.
-
-### .livepatch.hooks.load and .livepatch.hooks.unload
-
-This section contains an array of function pointers to be executed
-before payload is being applied (.livepatch.funcs) or after reverting
-the payload. This is useful to prepare data structures that need to
-be modified patching.
-
-Each entry in this array is eight bytes.
-
-The type definition of the function are as follow:
-
- typedef void (*livepatch_loadcall_t)(void);
- typedef void (*livepatch_unloadcall_t)(void);
-
-### .livepatch.depends and .note.gnu.build-id
-
-To support dependencies checking and safe loading (to load the
-appropiate payload against the right hypervisor) there is a need
-to embbed an build-id dependency.
-
-This is done by the payload containing an section `.livepatch.depends`
-which follows the format of an ELF Note. The contents of this
-(name, and description) are specific to the linker utilized to
-build the hypevisor and payload.
-
-If GNU linker is used then the name is `GNU` and the description
-is a NT\_GNU\_BUILD\_ID type ID. The description can be an SHA1
-checksum, MD5 checksum or any unique value.
-
-The size of these structures varies with the `--build-id` linker option.
-
-## Hypercalls
-
-We will employ the sub operations of the system management hypercall (sysctl).
-There are to be four sub-operations:
-
- * upload the payloads.
- * listing of payloads summary uploaded and their state.
- * getting an particular payload summary and its state.
- * command to apply, delete, or revert the payload.
-
-Most of the actions are asynchronous therefore the caller is responsible
-to verify that it has been applied properly by retrieving the summary of it
-and verifying that there are no error codes associated with the payload.
-
-We **MUST** make some of them asynchronous due to the nature of patching
-it requires every physical CPU to be lock-step with each other.
-The patching mechanism while an implementation detail, is not an short
-operation and as such the design **MUST** assume it will be an long-running
-operation.
-
-The sub-operations will spell out how preemption is to be handled (if at all).
-
-Furthermore it is possible to have multiple different payloads for the same
-function. As such an unique name per payload has to be visible to allow proper manipulation.
-
-The hypercall is part of the `xen_sysctl`. The top level structure contains
-one uint32\_t to determine the sub-operations and one padding field which
-*MUST* always be zero.
-
- struct xen_sysctl_livepatch_op {
- uint32_t cmd; /* IN: XEN_SYSCTL_LIVEPATCH_*. */
- uint32_t pad; /* IN: Always zero. */
- union {
- ... see below ...
- } u;
- };
-
-while the rest of hypercall specific structures are part of the this structure.
-
-### Basic type: struct xen\_livepatch\_name
-
-Most of the hypercalls employ an shared structure called `struct xen_livepatch_name`
-which contains:
-
- * `name` - pointer where the string for the name is located.
- * `size` - the size of the string
- * `pad` - padding - to be zero.
-
-The structure is as follow:
-
- /*
- * Uniquely identifies the payload. Should be human readable.
- * Includes the NUL terminator
- */
- #define XEN_LIVEPATCH_NAME_SIZE 128
- struct xen_livepatch_name {
- XEN_GUEST_HANDLE_64(char) name; /* IN, pointer to name. */
- uint16_t size; /* IN, size of name. May be upto
- XEN_LIVEPATCH_NAME_SIZE. */
- uint16_t pad[3]; /* IN: MUST be zero. */
- };
-
-### XEN\_SYSCTL\_LIVEPATCH\_UPLOAD (0)
-
-Upload a payload to the hypervisor. The payload is verified
-against basic checks and if there are any issues the proper return code
-will be returned. The payload is not applied at this time - that is
-controlled by *XEN\_SYSCTL\_LIVEPATCH\_ACTION*.
-
-The caller provides:
-
- * A `struct xen_livepatch_name` called `name` which has the unique name.
- * `size` the size of the ELF payload (in bytes).
- * `payload` the virtual address of where the ELF payload is.
-
-The `name` could be an UUID that stays fixed forever for a given
-payload. It can be embedded into the ELF payload at creation time
-and extracted by tools.
-
-The return value is zero if the payload was succesfully uploaded.
-Otherwise an -XEN\_EXX return value is provided. Duplicate `name` are not supported.
-
-The `payload` is the ELF payload as mentioned in the `Payload format` section.
-
-The structure is as follow:
-
- struct xen_sysctl_livepatch_upload {
- xen_livepatch_name_t name; /* IN, name of the patch. */
- uint64_t size; /* IN, size of the ELF file. */
- XEN_GUEST_HANDLE_64(uint8) payload; /* IN: ELF file. */
- };
-
-### XEN\_SYSCTL\_LIVEPATCH\_GET (1)
-
-Retrieve an status of an specific payload. This caller provides:
-
- * A `struct xen_livepatch_name` called `name` which has the unique name.
- * A `struct xen_livepatch_status` structure. The member values will
- be over-written upon completion.
-
-Upon completion the `struct xen_livepatch_status` is updated.
-
- * `status` - indicates the current status of the payload:
- * *LIVEPATCH\_STATUS\_CHECKED* (1) loaded and the ELF payload safety checks passed.
- * *LIVEPATCH\_STATUS\_APPLIED* (2) loaded, checked, and applied.
- * No other value is possible.
- * `rc` - -XEN\_EXX type errors encountered while performing the last
- LIVEPATCH\_ACTION\_\* operation. The normal values can be zero or -XEN\_EAGAIN which
- respectively mean: success or operation in progress. Other values
- imply an error occurred. If there is an error in `rc`, `status` will **NOT**
- have changed.
-
-The return value of the hypercall is zero on success and -XEN\_EXX on failure.
-(Note that the `rc` value can be different from the return value, as in
-rc=-XEN\_EAGAIN and return value can be 0).
-
-For example, supposing there is an payload:
-
- status: LIVEPATCH_STATUS_CHECKED
- rc: 0
-
-We apply an action - LIVEPATCH\_ACTION\_REVERT - to revert it (which won't work
-as we have not even applied it. Afterwards we will have:
-
- status: LIVEPATCH_STATUS_CHECKED
- rc: -XEN_EINVAL
-
-It has failed but it remains loaded.
-
-This operation is synchronous and does not require preemption.
-
-The structure is as follow:
-
- struct xen_livepatch_status {
- #define LIVEPATCH_STATUS_CHECKED 1
- #define LIVEPATCH_STATUS_APPLIED 2
- uint32_t state; /* OUT: LIVEPATCH_STATE_*. */
- int32_t rc; /* OUT: 0 if no error, otherwise -XEN_EXX. */
- };
-
- struct xen_sysctl_livepatch_get {
- xen_livepatch_name_t name; /* IN, the name of the payload. */
- xen_livepatch_status_t status; /* IN/OUT: status of the payload. */
- };
-
-### XEN\_SYSCTL\_LIVEPATCH\_LIST (2)
-
-Retrieve an array of abbreviated status and names of payloads that are loaded in the
-hypervisor.
-
-The caller provides:
-
- * `version`. Version of the payload. Caller should re-use the field provided by
- the hypervisor. If the value differs the data is stale.
- * `idx` Index iterator. The index into the hypervisor's payload count. It is
- recommended that on first invocation zero be used so that `nr` (which the
- hypervisor will update with the remaining payload count) be provided.
- Also the hypervisor will provide `version` with the most current value.
- * `nr` The max number of entries to populate. Can be zero which will result
- in the hypercall being a probing one and return the number of payloads
- (and update the `version`).
- * `pad` - *MUST* be zero.
- * `status` Virtual address of where to write `struct xen_livepatch_status`
- structures. Caller *MUST* allocate up to `nr` of them.
- * `name` - Virtual address of where to write the unique name of the payload.
- Caller *MUST* allocate up to `nr` of them. Each *MUST* be of
- **XEN\_LIVEPATCH\_NAME\_SIZE** size. Note that **XEN\_LIVEPATCH\_NAME\_SIZE** includes
- the NUL terminator.
- * `len` - Virtual address of where to write the length of each unique name
- of the payload. Caller *MUST* allocate up to `nr` of them. Each *MUST* be
- of sizeof(uint32\_t) (4 bytes).
-
-If the hypercall returns an positive number, it is the number (upto `nr`
-provided to the hypercall) of the payloads returned, along with `nr` updated
-with the number of remaining payloads, `version` updated (it may be the same
-across hypercalls - if it varies the data is stale and further calls could
-fail). The `status`, `name`, and `len` are updated at their designed index
-value (`idx`) with the returned value of data.
-
-If the hypercall returns -XEN\_E2BIG the `nr` is too big and should be
-lowered.
-
-If the hypercall returns an zero value there are no more payloads.
-
-Note that due to the asynchronous nature of hypercalls the control domain might
-have added or removed a number of payloads making this information stale. It is
-the responsibility of the toolstack to use the `version` field to check
-between each invocation. if the version differs it should discard the stale
-data and start from scratch. It is OK for the toolstack to use the new
-`version` field.
-
-The `struct xen_livepatch_status` structure contains an status of payload which includes:
-
- * `status` - indicates the current status of the payload:
- * *LIVEPATCH\_STATUS\_CHECKED* (1) loaded and the ELF payload safety checks passed.
- * *LIVEPATCH\_STATUS\_APPLIED* (2) loaded, checked, and applied.
- * No other value is possible.
- * `rc` - -XEN\_EXX type errors encountered while performing the last
- LIVEPATCH\_ACTION\_\* operation. The normal values can be zero or -XEN\_EAGAIN which
- respectively mean: success or operation in progress. Other values
- imply an error occurred. If there is an error in `rc`, `status` will **NOT**
- have changed.
-
-The structure is as follow:
-
- struct xen_sysctl_livepatch_list {
- uint32_t version; /* OUT: Hypervisor stamps value.
- If varies between calls, we are
- getting stale data. */
- uint32_t idx; /* IN: Index into hypervisor list. */
- uint32_t nr; /* IN: How many status, names, and len
- should be filled out. Can be zero to get
- amount of payloads and version.
- OUT: How many payloads left. */
- uint32_t pad; /* IN: Must be zero. */
- XEN_GUEST_HANDLE_64(xen_livepatch_status_t) status; /* OUT. Must have enough
- space allocate for nr of them. */
- XEN_GUEST_HANDLE_64(char) id; /* OUT: Array of names. Each member
- MUST XEN_LIVEPATCH_NAME_SIZE in size.
- Must have nr of them. */
- XEN_GUEST_HANDLE_64(uint32) len; /* OUT: Array of lengths of name's.
- Must have nr of them. */
- };
-
-### XEN\_SYSCTL\_LIVEPATCH\_ACTION (3)
-
-Perform an operation on the payload structure referenced by the `name` field.
-The operation request is asynchronous and the status should be retrieved
-by using either **XEN\_SYSCTL\_LIVEPATCH\_GET** or **XEN\_SYSCTL\_LIVEPATCH\_LIST** hypercall.
-
-The caller provides:
-
- * A `struct xen_livepatch_name` `name` containing the unique name.
- * `cmd` The command requested:
- * *LIVEPATCH\_ACTION\_UNLOAD* (1) Unload the payload.
- Any further hypercalls against the `name` will result in failure unless
- **XEN\_SYSCTL\_LIVEPATCH\_UPLOAD** hypercall is perfomed with same `name`.
- * *LIVEPATCH\_ACTION\_REVERT* (2) Revert the payload. If the operation takes
- more time than the upper bound of time the `rc` in `xen_livepatch_status`
- retrieved via **XEN\_SYSCTL\_LIVEPATCH\_GET** will be -XEN\_EBUSY.
- * *LIVEPATCH\_ACTION\_APPLY* (3) Apply the payload. If the operation takes
- more time than the upper bound of time the `rc` in `xen_livepatch_status`
- retrieved via **XEN\_SYSCTL\_LIVEPATCH\_GET** will be -XEN\_EBUSY.
- * *LIVEPATCH\_ACTION\_REPLACE* (4) Revert all applied payloads and apply this
- payload. If the operation takes more time than the upper bound of time
- the `rc` in `xen_livepatch_status` retrieved via **XEN\_SYSCTL\_LIVEPATCH\_GET**
- will be -XEN\_EBUSY.
- * `time` The upper bound of time (ns) the cmd should take. Zero means to use
- the hypervisor default. If within the time the operation does not succeed
- the operation would go in error state.
- * `pad` - *MUST* be zero.
-
-The return value will be zero unless the provided fields are incorrect.
-
-The structure is as follow:
-
- #define LIVEPATCH_ACTION_UNLOAD 1
- #define LIVEPATCH_ACTION_REVERT 2
- #define LIVEPATCH_ACTION_APPLY 3
- #define LIVEPATCH_ACTION_REPLACE 4
- struct xen_sysctl_livepatch_action {
- xen_livepatch_name_t name; /* IN, name of the patch. */
- uint32_t cmd; /* IN: LIVEPATCH_ACTION_* */
- uint32_t time; /* IN: If zero then uses */
- /* hypervisor default. */
- /* Or upper bound of time (ns) */
- /* for operation to take. */
- };
-
-
-## State diagrams of LIVEPATCH\_ACTION commands.
-
-There is a strict ordering state of what the commands can be.
-The LIVEPATCH\_ACTION prefix has been dropped to easy reading and
-does not include the LIVEPATCH\_STATES:
-
- /->\
- \ /
- UNLOAD <--- CHECK ---> REPLACE|APPLY --> REVERT --\
- \ |
- \-------------------<-------------/
-
-## State transition table of LIVEPATCH\_ACTION commands and LIVEPATCH\_STATUS.
-
-Note that:
-
- - The CHECKED state is the starting one achieved with *XEN\_SYSCTL\_LIVEPATCH\_UPLOAD* hypercall.
- - The REVERT operation on success will automatically move to the CHECKED state.
- - There are two STATES: CHECKED and APPLIED.
- - There are four actions (aka commands): APPLY, REPLACE, REVERT, and UNLOAD.
-
-The state transition table of valid states and action states:
-
- +---------+---------+--------------------------------+-------+--------+
- | ACTION | Current | Result | Next STATE: |
- | ACTION | STATE | |CHECKED|APPLIED |
- +---------+----------+-------------------------------+-------+--------+
- | UNLOAD | CHECKED | Unload payload. Always works. | | |
- | | | No next states. | | |
- +---------+---------+--------------------------------+-------+--------+
- | APPLY | CHECKED | Apply payload (success). | | x |
- +---------+---------+--------------------------------+-------+--------+
- | APPLY | CHECKED | Apply payload (error|timeout) | x | |
- +---------+---------+--------------------------------+-------+--------+
- | REPLACE | CHECKED | Revert payloads and apply new | | x |
- | | | payload with success. | | |
- +---------+---------+--------------------------------+-------+--------+
- | REPLACE | CHECKED | Revert payloads and apply new | x | |
- | | | payload with error. | | |
- +---------+---------+--------------------------------+-------+--------+
- | REVERT | APPLIED | Revert payload (success). | x | |
- +---------+---------+--------------------------------+-------+--------+
- | REVERT | APPLIED | Revert payload (error|timeout) | | x |
- +---------+---------+--------------------------------+-------+--------+
-
-All the other state transitions are invalid.
-
-## Sequence of events.
-
-The normal sequence of events is to:
-
- 1. *XEN\_SYSCTL\_LIVEPATCH\_UPLOAD* to upload the payload. If there are errors *STOP* here.
- 2. *XEN\_SYSCTL\_LIVEPATCH\_GET* to check the `->rc`. If *-XEN\_EAGAIN* spin. If zero go to next step.
- 3. *XEN\_SYSCTL\_LIVEPATCH\_ACTION* with *LIVEPATCH\_ACTION\_APPLY* to apply the patch.
- 4. *XEN\_SYSCTL\_LIVEPATCH\_GET* to check the `->rc`. If in *-XEN\_EAGAIN* spin. If zero exit with success.
-
-
-## Addendum
-
-Implementation quirks should not be discussed in a design document.
-
-However these observations can provide aid when developing against this
-document.
-
-
-### Alternative assembler
-
-Alternative assembler is a mechanism to use different instructions depending
-on what the CPU supports. This is done by providing multiple streams of code
-that can be patched in - or if the CPU does not support it - padded with
-`nop` operations. The alternative assembler macros cause the compiler to
-expand the code to place a most generic code in place - emit a special
-ELF .section header to tag this location. During run-time the hypervisor
-can leave the areas alone or patch them with an better suited opcodes.
-
-Note that patching functions that copy to or from guest memory requires
-to support alternative support. For example this can be due to SMAP
-(specifically *stac* and *clac* operations) which is enabled on Broadwell
-and later architectures. It may be related to other alternative instructions.
-
-### When to patch
-
-During the discussion on the design two candidates bubbled where
-the call stack for each CPU would be deterministic. This would
-minimize the chance of the patch not being applied due to safety
-checks failing. Safety checks such as not patching code which
-is on the stack - which can lead to corruption.
-
-#### Rendezvous code instead of stop\_machine for patching
-
-The hypervisor's time rendezvous code runs synchronously across all CPUs
-every second. Using the `stop_machine` to patch can stall the time rendezvous
-code and result in NMI. As such having the patching be done at the tail
-of rendezvous code should avoid this problem.
-
-However the entrance point for that code is `do_softirq ->
-timer_softirq_action -> time_calibration` which ends up calling
-`on_selected_cpus` on remote CPUs.
-
-The remote CPUs receive CALL\_FUNCTION\_VECTOR IPI and execute the
-desired function.
-
-#### Before entering the guest code.
-
-Before we call VMXResume we check whether any soft IRQs need to be executed.
-This is a good spot because all Xen stacks are effectively empty at
-that point.
-
-To randezvous all the CPUs an barrier with an maximum timeout (which
-could be adjusted), combined with forcing all other CPUs through the
-hypervisor with IPIs, can be utilized to execute lockstep instructions
-on all CPUs.
-
-The approach is similar in concept to `stop_machine` and the time rendezvous
-but is time-bound. However the local CPU stack is much shorter and
-a lot more deterministic.
-
-This is implemented in the Xen hypervisor.
-
-### Compiling the hypervisor code
-
-Hotpatch generation often requires support for compiling the target
-with `-ffunction-sections` / `-fdata-sections`. Changes would have to
-be done to the linker scripts to support this.
-
-### Generation of Live Patch ELF payloads
-
-The design of that is not discussed in this design.
-
-This is implemented in a seperate tool which lives in a seperate
-GIT repo.
-
-Currently it resides at git://xenbits.xen.org/livepatch-build-tools.git
-
-### Exception tables and symbol tables growth
-
-We may need support for adapting or augmenting exception tables if
-patching such code. Hotpatches may need to bring their own small
-exception tables (similar to how Linux modules support this).
-
-If supporting hotpatches that introduce additional exception-locations
-is not important, one could also change the exception table in-place
-and reorder it afterwards.
-
-As found almost every patch (XSA) to a non-trivial function requires
-additional entries in the exception table and/or the bug frames.
-
-This is implemented in the Xen hypervisor.
-
-### .rodata sections
-
-The patching might require strings to be updated as well. As such we must be
-also able to patch the strings as needed. This sounds simple - but the compiler
-has a habit of coalescing strings that are the same - which means if we in-place
-alter the strings - other users will be inadvertently affected as well.
-
-This is also where pointers to functions live - and we may need to patch this
-as well. And switch-style jump tables.
-
-To guard against that we must be prepared to do patching similar to
-trampoline patching or in-line depending on the flavour. If we can
-do in-line patching we would need to:
-
- * Alter `.rodata` to be writeable.
- * Inline patch.
- * Alter `.rodata` to be read-only.
-
-If are doing trampoline patching we would need to:
-
- * Allocate a new memory location for the string.
- * All locations which use this string will have to be updated to use the
- offset to the string.
- * Mark the region RO when we are done.
-
-The trampoline patching is implemented in the Xen hypervisor.
-
-### .bss and .data sections.
-
-In place patching writable data is not suitable as it is unclear what should be done
-depending on the current state of data. As such it should not be attempted.
-
-However, functions which are being patched can bring in changes to strings
-(.data or .rodata section changes), or even to .bss sections.
-
-As such the ELF payload can introduce new .rodata, .bss, and .data sections.
-Patching in the new function will end up also patching in the new .rodata
-section and the new function will reference the new string in the new
-.rodata section.
-
-This is implemented in the Xen hypervisor.
-
-### Security
-
-Only the privileged domain should be allowed to do this operation.
-
-### Live patch interdependencies
-
-Live patch patches interdependencies are tricky.
-
-There are the ways this can be addressed:
- * A single large patch that subsumes and replaces all previous ones.
- Over the life-time of patching the hypervisor this large patch
- grows to accumulate all the code changes.
- * Hotpatch stack - where an mechanism exists that loads the hotpatches
- in the same order they were built in. We would need an build-id
- of the hypevisor to make sure the hot-patches are build against the
- correct build.
- * Payload containing the old code to check against that. That allows
- the hotpatches to be loaded indepedently (if they don't overlap) - or
- if the old code also containst previously patched code - even if they
- overlap.
-
-The disadvantage of the first large patch is that it can grow over
-time and not provide an bisection mechanism to identify faulty patches.
-
-The hot-patch stack puts stricts requirements on the order of the patches
-being loaded and requires an hypervisor build-id to match against.
-
-The old code allows much more flexibility and an additional guard,
-but is more complex to implement.
-
-The second option which requires an build-id of the hypervisor
-is implemented in the Xen hypervisor.
-
-Specifically each payload has two build-id ELF notes:
- * The build-id of the payload itself (generated via --build-id).
- * The build-id of the payload it depends on (extracted from the
- the previous payload or hypervisor during build time).
-
-This means that the very first payload depends on the hypervisor
-build-id.
-
-# Not Yet Done
-
-This is for further development of live patching.
-
-## TODO Goals
-
-The implementation must also have a mechanism for (in no particular order):
-
- * Be able to lookup in the Xen hypervisor the symbol names of functions from the
- ELF payload. (Either as `symbol` or `symbol`+`offset`).
- * Be able to patch .rodata, .bss, and .data sections.
- * Deal with NMI/MCE checks during patching instead of ignoring them.
- * Further safety checks (blacklist of which functions cannot be patched, check
- the stack, make sure the payload is built with same compiler as hypervisor).
- Specifically we want to make sure that live patching codepaths cannot be patched.
- * NOP out the code sequence if `new_size` is zero.
- * Deal with other relocation types: R\_X86\_64\_[8,16,32,32S], R\_X86\_64\_PC[8,16,64]
- in payload file.
-
-### Handle inlined \_\_LINE\_\_
-
-This problem is related to hotpatch construction
-and potentially has influence on the design of the hotpatching
-infrastructure in Xen.
-
-For example:
-
-We have file1.c with functions f1 and f2 (in that order). f2 contains a
-BUG() (or WARN()) macro and at that point embeds the source line number
-into the generated code for f2.
-
-Now we want to hotpatch f1 and the hotpatch source-code patch adds 2
-lines to f1 and as a consequence shifts out f2 by two lines. The newly
-constructed file1.o will now contain differences in both binary
-functions f1 (because we actually changed it with the applied patch) and
-f2 (because the contained BUG macro embeds the new line number).
-
-Without additional information, an algorithm comparing file1.o before
-and after hotpatch application will determine both functions to be
-changed and will have to include both into the binary hotpatch.
-
-Options:
-
-1. Transform source code patches for hotpatches to be line-neutral for
- each chunk. This can be done in almost all cases with either
- reformatting of the source code or by introducing artificial
- preprocessor "#line n" directives to adjust for the introduced
- differences.
-
- This approach is low-tech and simple. Potentially generated
- backtraces and existing debug information refers to the original
- build and does not reflect hotpatching state except for actually
- hotpatched functions but should be mostly correct.
-
-2. Ignoring the problem and living with artificially large hotpatches
- that unnecessarily patch many functions.
-
- This approach might lead to some very large hotpatches depending on
- content of specific source file. It may also trigger pulling in
- functions into the hotpatch that cannot reasonable be hotpatched due
- to limitations of a hotpatching framework (init-sections, parts of
- the hotpatching framework itself, ...) and may thereby prevent us
- from patching a specific problem.
-
- The decision between 1. and 2. can be made on a patch--by-patch
- basis.
-
-3. Introducing an indirection table for storing line numbers and
- treating that specially for binary diffing. Linux may follow
- this approach.
-
- We might either use this indirection table for runtime use and patch
- that with each hotpatch (similarly to exception tables) or we might
- purely use it when building hotpatches to ignore functions that only
- differ at exactly the location where a line-number is embedded.
-
-For BUG(), WARN(), etc., the line number is embedded into the bug frame, not
-the function itself.
-
-Similar considerations are true to a lesser extent for \_\_FILE\_\_, but it
-could be argued that file renaming should be done outside of hotpatches.
-
-## Signature checking requirements.
-
-The signature checking requires that the layout of the data in memory
-**MUST** be same for signature to be verified. This means that the payload
-data layout in ELF format **MUST** match what the hypervisor would be
-expecting such that it can properly do signature verification.
-
-The signature is based on the all of the payloads continuously laid out
-in memory. The signature is to be appended at the end of the ELF payload
-prefixed with the string '`~Module signature appended~\n`', followed by
-an signature header then followed by the signature, key identifier, and signers
-name.
-
-Specifically the signature header would be:
-
- #define PKEY_ALGO_DSA 0
- #define PKEY_ALGO_RSA 1
-
- #define PKEY_ID_PGP 0 /* OpenPGP generated key ID */
- #define PKEY_ID_X509 1 /* X.509 arbitrary subjectKeyIdentifier */
-
- #define HASH_ALGO_MD4 0
- #define HASH_ALGO_MD5 1
- #define HASH_ALGO_SHA1 2
- #define HASH_ALGO_RIPE_MD_160 3
- #define HASH_ALGO_SHA256 4
- #define HASH_ALGO_SHA384 5
- #define HASH_ALGO_SHA512 6
- #define HASH_ALGO_SHA224 7
- #define HASH_ALGO_RIPE_MD_128 8
- #define HASH_ALGO_RIPE_MD_256 9
- #define HASH_ALGO_RIPE_MD_320 10
- #define HASH_ALGO_WP_256 11
- #define HASH_ALGO_WP_384 12
- #define HASH_ALGO_WP_512 13
- #define HASH_ALGO_TGR_128 14
- #define HASH_ALGO_TGR_160 15
- #define HASH_ALGO_TGR_192 16
-
- struct elf_payload_signature {
- u8 algo; /* Public-key crypto algorithm PKEY_ALGO_*. */
- u8 hash; /* Digest algorithm: HASH_ALGO_*. */
- u8 id_type; /* Key identifier type PKEY_ID*. */
- u8 signer_len; /* Length of signer's name */
- u8 key_id_len; /* Length of key identifier */
- u8 __pad[3];
- __be32 sig_len; /* Length of signature data */
- };
-
-(Note that this has been borrowed from Linux module signature code.).
-
-
-### .bss and .data sections.
-
-In place patching writable data is not suitable as it is unclear what should be done
-depending on the current state of data. As such it should not be attempted.
-
-That said we should provide hook functions so that the existing data
-can be changed during payload application.
-
-To guarantee safety we disallow re-applying an payload after it has been
-reverted. This is because we cannot guarantee that the state of .bss
-and .data to be exactly as it was during loading. Hence the administrator
-MUST unload the payload and upload it again to apply it.
-
-There is an exception to this: if the payload only has .livepatch.funcs;
-and the .data or .bss sections are of zero length.
-
-### Inline patching
-
-The hypervisor should verify that the in-place patching would fit within
-the code or data.
-
-### Trampoline (e9 opcode), x86
-
-The e9 opcode used for jmpq uses a 32-bit signed displacement. That means
-we are limited to up to 2GB of virtual address to place the new code
-from the old code. That should not be a problem since Xen hypervisor has
-a very small footprint.
-
-However if we need - we can always add two trampolines. One at the 2GB
-limit that calls the next trampoline.
-
-Please note there is a small limitation for trampolines in
-function entries: The target function (+ trailing padding) must be able
-to accomodate the trampoline. On x86 with +-2 GB relative jumps,
-this means 5 bytes are required which means that `old_size` **MUST** be
-at least five bytes if patching in trampoline.
-
-Depending on compiler settings, there are several functions in Xen that
-are smaller (without inter-function padding).
-
- readelf -sW xen-syms | grep " FUNC " | \
- awk '{ if ($3 < 5) print $3, $4, $5, $8 }'
-
- ...
- 3 FUNC LOCAL wbinvd_ipi
- 3 FUNC LOCAL shadow_l1_index
- ...
-
-A compile-time check for, e.g., a minimum alignment of functions or a
-runtime check that verifies symbol size (+ padding to next symbols) for
-that in the hypervisor is advised.
-
-The tool for generating payloads currently does perform a compile-time
-check to ensure that the function to be replaced is large enough.
-
-#### Trampoline, ARM
-
-The unconditional branch instruction (for the encoding see the
-DDI 0406C.c and DDI 0487A.j Architecture Reference Manual's).
-with proper offset is used for an unconditional branch to the new code.
-This means that that `old_size` **MUST** be at least four bytes if patching
-in trampoline.
-
-The instruction offset is limited on ARM32 to +/- 32MB to displacement
-and on ARM64 to +/- 128MB displacement.
-
-The new code is placed in the 8M - 10M virtual address space while the
-Xen code is in 2M - 4M. That gives us enough space.
-
-The hypervisor also checks the displacement during loading of the payload.
--- /dev/null
+# Xen Live Patching Design v1
+
+## Rationale
+
+A mechanism is required to binarily patch the running hypervisor with new
+opcodes that have come about due to primarily security updates.
+
+This document describes the design of the API that would allow us to
+upload to the hypervisor binary patches.
+
+The document is split in four sections:
+
+ * Detailed descriptions of the problem statement.
+ * Design of the data structures.
+ * Design of the hypercalls.
+ * Implementation notes that should be taken into consideration.
+
+
+## Glossary
+
+ * splice - patch in the binary code with new opcodes
+ * trampoline - a jump to a new instruction.
+ * payload - telemetries of the old code along with binary blob of the new
+ function (if needed).
+ * reloc - telemetries contained in the payload to construct proper trampoline.
+
+## History
+
+The document has gone under various reviews and only covers v1 design.
+
+The end of the document has a section titled `Not Yet Done` which
+outlines ideas and design for the future version of this work.
+
+## Multiple ways to patch
+
+The mechanism needs to be flexible to patch the hypervisor in multiple ways
+and be as simple as possible. The compiled code is contiguous in memory with
+no gaps - so we have no luxury of 'moving' existing code and must either
+insert a trampoline to the new code to be executed - or only modify in-place
+the code if there is sufficient space. The placement of new code has to be done
+by hypervisor and the virtual address for the new code is allocated dynamically.
+
+This implies that the hypervisor must compute the new offsets when splicing
+in the new trampoline code. Where the trampoline is added (inside
+the function we are patching or just the callers?) is also important.
+
+To lessen the amount of code in hypervisor, the consumer of the API
+is responsible for identifying which mechanism to employ and how many locations
+to patch. Combinations of modifying in-place code, adding trampoline, etc
+has to be supported. The API should allow read/write any memory within
+the hypervisor virtual address space.
+
+We must also have a mechanism to query what has been applied and a mechanism
+to revert it if needed.
+
+## Workflow
+
+The expected workflows of higher-level tools that manage multiple patches
+on production machines would be:
+
+ * The first obvious task is loading all available / suggested
+ hotpatches when they are available.
+ * Whenever new hotpatches are installed, they should be loaded too.
+ * One wants to query which modules have been loaded at runtime.
+ * If unloading is deemed safe (see unloading below), one may want to
+ support a workflow where a specific hotpatch is marked as bad and
+ unloaded.
+
+## Patching code
+
+The first mechanism to patch that comes in mind is in-place replacement.
+That is replace the affected code with new code. Unfortunately the x86
+ISA is variable size which places limits on how much space we have available
+to replace the instructions. That is not a problem if the change is smaller
+than the original opcode and we can fill it with nops. Problems will
+appear if the replacement code is longer.
+
+The second mechanism is by ti replace the call or jump to the
+old function with the address of the new function.
+
+A third mechanism is to add a jump to the new function at the
+start of the old function. N.B. The Xen hypervisor implements the third
+mechanism. See `Trampoline (e9 opcode)` section for more details.
+
+### Example of trampoline and in-place splicing
+
+As example we will assume the hypervisor does not have XSA-132 (see
+[domctl/sysctl: don't leak hypervisor stack to toolstacks](http://xenbits.xen.org/gitweb/?p=xen.git;a=commitdiff;h=4ff3449f0e9d175ceb9551d3f2aecb59273f639d))
+and we would like to binary patch the hypervisor with it. The original code
+looks as so:
+
+ 48 89 e0 mov %rsp,%rax
+ 48 25 00 80 ff ff and $0xffffffffffff8000,%rax
+
+while the new patched hypervisor would be:
+
+ 48 c7 45 b8 00 00 00 00 movq $0x0,-0x48(%rbp)
+ 48 c7 45 c0 00 00 00 00 movq $0x0,-0x40(%rbp)
+ 48 c7 45 c8 00 00 00 00 movq $0x0,-0x38(%rbp)
+ 48 89 e0 mov %rsp,%rax
+ 48 25 00 80 ff ff and $0xffffffffffff8000,%rax
+
+This is inside the `arch_do_domctl`. This new change adds 21 extra
+bytes of code which alters all the offsets inside the function. To alter
+these offsets and add the extra 21 bytes of code we might not have enough
+space in .text to squeeze this in.
+
+As such we could simplify this problem by only patching the site
+which calls `arch_do_domctl`:
+
+ do_domctl:
+ e8 4b b1 05 00 callq ffff82d08015fbb9 <arch_do_domctl>
+
+with a new address for where the new `arch_do_domctl` would be (this
+area would be allocated dynamically).
+
+Astute readers will wonder what we need to do if we were to patch `do_domctl`
+- which is not called directly by hypervisor but on behalf of the guests via
+the `compat_hypercall_table` and `hypercall_table`. Patching the offset in
+`hypercall_table` for `do_domctl`:
+
+ ffff82d08024d490: 79 30
+ ffff82d08024d492: 10 80 d0 82 ff ff
+
+with the new address where the new `do_domctl` is possible. The other
+place where it is used is in `hvm_hypercall64_table` which would need
+to be patched in a similar way. This would require an in-place splicing
+of the new virtual address of `arch_do_domctl`.
+
+In summary this example patched the callee of the affected function by
+
+ * Allocating memory for the new code to live in,
+ * Changing the virtual address in all the functions which called the old
+ code (computing the new offset, patching the callq with a new callq).
+ * Changing the function pointer tables with the new virtual address of
+ the function (splicing in the new virtual address). Since this table
+ resides in the .rodata section we would need to temporarily change the
+ page table permissions during this part.
+
+However it has drawbacks - the safety checks which have to make sure
+the function is not on the stack - must also check every caller. For some
+patches this could mean - if there were an sufficient large amount of
+callers - that we would never be able to apply the update.
+
+Having the patching done at predetermined instances where the stacks
+are not deep mostly solves this problem.
+
+### Example of different trampoline patching.
+
+An alternative mechanism exists where we can insert a trampoline in the
+existing function to be patched to jump directly to the new code. This
+lessens the locations to be patched to one but it puts pressure on the
+CPU branching logic (I-cache, but it is just one unconditional jump).
+
+For this example we will assume that the hypervisor has not been compiled with
+XSA-125 (see
+[pre-fill structures for certain HYPERVISOR_xen_version sub-ops](http://xenbits.xen.org/gitweb/?p=xen.git;a=commitdiff;h=fe2e079f642effb3d24a6e1a7096ef26e691d93e))
+which mem-sets an structure in `xen_version` hypercall. This function is not
+called **anywhere** in the hypervisor (it is called by the guest) but
+referenced in the `compat_hypercall_table` and `hypercall_table` (and
+indirectly called from that). Patching the offset in `hypercall_table` for the
+old `do_xen_version`:
+
+ ffff82d08024b270 <hypercall_table>:
+ ...
+ ffff82d08024b2f8: 9e 2f 11 80 d0 82 ff ff
+
+with the new address where the new `do_xen_version` is possible. The other
+place where it is used is in `hvm_hypercall64_table` which would need
+to be patched in a similar way. This would require an in-place splicing
+of the new virtual address of `do_xen_version`.
+
+An alternative solution would be to patch insert a trampoline in the
+old `do_xen_version` function to directly jump to the new `do_xen_version`:
+
+ ffff82d080112f9e do_xen_version:
+ ffff82d080112f9e: 48 c7 c0 da ff ff ff mov $0xffffffffffffffda,%rax
+ ffff82d080112fa5: 83 ff 09 cmp $0x9,%edi
+ ffff82d080112fa8: 0f 87 24 05 00 00 ja ffff82d0801134d2 ; do_xen_version+0x534
+
+with:
+
+ ffff82d080112f9e do_xen_version:
+ ffff82d080112f9e: e9 XX YY ZZ QQ jmpq [new do_xen_version]
+
+which would lessen the amount of patching to just one location.
+
+In summary this example patched the affected function to jump to the
+new replacement function which required:
+
+ * Allocating memory for the new code to live in,
+ * Inserting trampoline with new offset in the old function to point to the
+ new function.
+ * Optionally we can insert in the old function a trampoline jump to an function
+ providing an BUG_ON to catch errant code.
+
+The disadvantage of this are that the unconditional jump will consume a small
+I-cache penalty. However the simplicity of the patching and higher chance
+of passing safety checks make this a worthwhile option.
+
+This patching has a similar drawback as inline patching - the safety
+checks have to make sure the function is not on the stack. However
+since we are replacing at a higher level (a full function as opposed
+to various offsets within functions) the checks are simpler.
+
+Having the patching done at predetermined instances where the stacks
+are not deep mostly solves this problem as well.
+
+### Security
+
+With this method we can re-write the hypervisor - and as such we **MUST** be
+diligent in only allowing certain guests to perform this operation.
+
+Furthermore with SecureBoot or tboot, we **MUST** also verify the signature
+of the payload to be certain it came from a trusted source and integrity
+was intact.
+
+As such the hypercall **MUST** support an XSM policy to limit what the guest
+is allowed to invoke. If the system is booted with signature checking the
+signature checking will be enforced.
+
+## Design of payload format
+
+The payload **MUST** contain enough data to allow us to apply the update
+and also safely reverse it. As such we **MUST** know:
+
+ * The locations in memory to be patched. This can be determined dynamically
+ via symbols or via virtual addresses.
+ * The new code that will be patched in.
+
+This binary format can be constructed using an custom binary format but
+there are severe disadvantages of it:
+
+ * The format might need to be changed and we need an mechanism to accommodate
+ that.
+ * It has to be platform agnostic.
+ * Easily constructed using existing tools.
+
+As such having the payload in an ELF file is the sensible way. We would be
+carrying the various sets of structures (and data) in the ELF sections under
+different names and with definitions.
+
+Note that every structure has padding. This is added so that the hypervisor
+can re-use those fields as it sees fit.
+
+Earlier design attempted to ineptly explain the relations of the ELF sections
+to each other without using proper ELF mechanism (sh_info, sh_link, data
+structures using Elf types, etc). This design will explain the structures
+and how they are used together and not dig in the ELF format - except mention
+that the section names should match the structure names.
+
+The Xen Live Patch payload is a relocatable ELF binary. A typical binary would have:
+
+ * One or more .text sections.
+ * Zero or more read-only data sections.
+ * Zero or more data sections.
+ * Relocations for each of these sections.
+
+It may also have some architecture-specific sections. For example:
+
+ * Alternatives instructions.
+ * Bug frames.
+ * Exception tables.
+ * Relocations for each of these sections.
+
+The Xen Live Patch core code loads the payload as a standard ELF binary, relocates it
+and handles the architecture-specifc sections as needed. This process is much
+like what the Linux kernel module loader does.
+
+The payload contains at least three sections:
+
+ * `.livepatch.funcs` - which is an array of livepatch_func structures.
+ * `.livepatch.depends` - which is an ELF Note that describes what the payload
+ depends on. **MUST** have one.
+ * `.note.gnu.build-id` - the build-id of this payload. **MUST** have one.
+
+### .livepatch.funcs
+
+The `.livepatch.funcs` contains an array of livepatch_func structures
+which describe the functions to be patched:
+
+ struct livepatch_func {
+ const char *name;
+ void *new_addr;
+ void *old_addr;
+ uint32_t new_size;
+ uint32_t old_size;
+ uint8_t version;
+ uint8_t opaque[31];
+ };
+
+The size of the structure is 64 bytes on 64-bit hypervisors. It will be
+52 on 32-bit hypervisors.
+
+ * `name` is the symbol name of the old function. Only used if `old_addr` is
+ zero, otherwise will be used during dynamic linking (when hypervisor loads
+ the payload).
+ * `old_addr` is the address of the function to be patched and is filled in at
+ payload generation time if hypervisor function address is known. If unknown,
+ the value *MUST* be zero and the hypervisor will attempt to resolve the
+ address.
+ * `new_addr` can either have a non-zero value or be zero.
+ * If there is a non-zero value, then it is the address of the function that
+ is replacing the old function and the address is recomputed during
+ relocation. The value **MUST** be the address of the new function in the
+ payload file.
+ * If the value is zero, then we NOPing out at the `old_addr` location
+ `new_size` bytes.
+ * `old_size` contains the sizes of the respective `old_addr` function in
+ bytes. The value of `old_size` **MUST** not be zero.
+ * `new_size` depends on what `new_addr` contains:
+ * If `new_addr` contains an non-zero value, then `new_size` has the size of
+ the new function (which will replace the one at `old_addr`) in bytes.
+ * If the value of `new_addr` is zero then `new_size` determines how many
+ instruction bytes to NOP (up to opaque size modulo smallest platform
+ instruction - 1 byte x86 and 4 bytes on ARM).
+ * `version` is to be one.
+ * `opaque` **MUST** be zero.
+
+The size of the `livepatch_func` array is determined from the ELF section
+size.
+
+When applying the patch the hypervisor iterates over each `livepatch_func`
+structure and the core code inserts a trampoline at `old_addr` to `new_addr`.
+The `new_addr` is altered when the ELF payload is loaded.
+
+When reverting a patch, the hypervisor iterates over each `livepatch_func`
+and the core code copies the data from the undo buffer (private internal copy)
+to `old_addr`.
+
+It optionally may contain the address of functions to be called right before
+being applied and after being reverted:
+
+ * `.livepatch.hooks.load` - an array of function pointers.
+ * `.livepatch.hooks.unload` - an array of function pointers.
+
+
+### Example of .livepatch.funcs
+
+A simple example of what a payload file can be:
+
+ /* MUST be in sync with hypervisor. */
+ struct livepatch_func {
+ const char *name;
+ void *new_addr;
+ void *old_addr;
+ uint32_t new_size;
+ uint32_t old_size;
+ uint8_t version;
+ uint8_t pad[31];
+ };
+
+ /* Our replacement function for xen_extra_version. */
+ const char *xen_hello_world(void)
+ {
+ return "Hello World";
+ }
+
+ static unsigned char patch_this_fnc[] = "xen_extra_version";
+
+ struct livepatch_func livepatch_hello_world = {
+ .version = LIVEPATCH_PAYLOAD_VERSION,
+ .name = patch_this_fnc,
+ .new_addr = xen_hello_world,
+ .old_addr = (void *)0xffff82d08013963c, /* Extracted from xen-syms. */
+ .new_size = 13, /* To be be computed by scripts. */
+ .old_size = 13, /* -----------""--------------- */
+ } __attribute__((__section__(".livepatch.funcs")));
+
+Code must be compiled with `-fPIC`.
+
+### .livepatch.hooks.load and .livepatch.hooks.unload
+
+This section contains an array of function pointers to be executed
+before payload is being applied (.livepatch.funcs) or after reverting
+the payload. This is useful to prepare data structures that need to
+be modified patching.
+
+Each entry in this array is eight bytes.
+
+The type definition of the function are as follow:
+
+ typedef void (*livepatch_loadcall_t)(void);
+ typedef void (*livepatch_unloadcall_t)(void);
+
+### .livepatch.depends and .note.gnu.build-id
+
+To support dependencies checking and safe loading (to load the
+appropiate payload against the right hypervisor) there is a need
+to embbed an build-id dependency.
+
+This is done by the payload containing an section `.livepatch.depends`
+which follows the format of an ELF Note. The contents of this
+(name, and description) are specific to the linker utilized to
+build the hypevisor and payload.
+
+If GNU linker is used then the name is `GNU` and the description
+is a NT_GNU_BUILD_ID type ID. The description can be an SHA1
+checksum, MD5 checksum or any unique value.
+
+The size of these structures varies with the `--build-id` linker option.
+
+## Hypercalls
+
+We will employ the sub operations of the system management hypercall (sysctl).
+There are to be four sub-operations:
+
+ * upload the payloads.
+ * listing of payloads summary uploaded and their state.
+ * getting an particular payload summary and its state.
+ * command to apply, delete, or revert the payload.
+
+Most of the actions are asynchronous therefore the caller is responsible
+to verify that it has been applied properly by retrieving the summary of it
+and verifying that there are no error codes associated with the payload.
+
+We **MUST** make some of them asynchronous due to the nature of patching
+it requires every physical CPU to be lock-step with each other.
+The patching mechanism while an implementation detail, is not an short
+operation and as such the design **MUST** assume it will be an long-running
+operation.
+
+The sub-operations will spell out how preemption is to be handled (if at all).
+
+Furthermore it is possible to have multiple different payloads for the same
+function. As such an unique name per payload has to be visible to allow proper manipulation.
+
+The hypercall is part of the `xen_sysctl`. The top level structure contains
+one uint32_t to determine the sub-operations and one padding field which
+*MUST* always be zero.
+
+ struct xen_sysctl_livepatch_op {
+ uint32_t cmd; /* IN: XEN_SYSCTL_LIVEPATCH_*. */
+ uint32_t pad; /* IN: Always zero. */
+ union {
+ ... see below ...
+ } u;
+ };
+
+while the rest of hypercall specific structures are part of the this structure.
+
+### Basic type: struct xen_livepatch_name
+
+Most of the hypercalls employ an shared structure called `struct xen_livepatch_name`
+which contains:
+
+ * `name` - pointer where the string for the name is located.
+ * `size` - the size of the string
+ * `pad` - padding - to be zero.
+
+The structure is as follow:
+
+ /*
+ * Uniquely identifies the payload. Should be human readable.
+ * Includes the NUL terminator
+ */
+ #define XEN_LIVEPATCH_NAME_SIZE 128
+ struct xen_livepatch_name {
+ XEN_GUEST_HANDLE_64(char) name; /* IN, pointer to name. */
+ uint16_t size; /* IN, size of name. May be upto
+ XEN_LIVEPATCH_NAME_SIZE. */
+ uint16_t pad[3]; /* IN: MUST be zero. */
+ };
+
+### XEN_SYSCTL_LIVEPATCH_UPLOAD (0)
+
+Upload a payload to the hypervisor. The payload is verified
+against basic checks and if there are any issues the proper return code
+will be returned. The payload is not applied at this time - that is
+controlled by *XEN_SYSCTL_LIVEPATCH_ACTION*.
+
+The caller provides:
+
+ * A `struct xen_livepatch_name` called `name` which has the unique name.
+ * `size` the size of the ELF payload (in bytes).
+ * `payload` the virtual address of where the ELF payload is.
+
+The `name` could be an UUID that stays fixed forever for a given
+payload. It can be embedded into the ELF payload at creation time
+and extracted by tools.
+
+The return value is zero if the payload was succesfully uploaded.
+Otherwise an -XEN_EXX return value is provided. Duplicate `name` are not supported.
+
+The `payload` is the ELF payload as mentioned in the `Payload format` section.
+
+The structure is as follow:
+
+ struct xen_sysctl_livepatch_upload {
+ xen_livepatch_name_t name; /* IN, name of the patch. */
+ uint64_t size; /* IN, size of the ELF file. */
+ XEN_GUEST_HANDLE_64(uint8) payload; /* IN: ELF file. */
+ };
+
+### XEN_SYSCTL_LIVEPATCH_GET (1)
+
+Retrieve an status of an specific payload. This caller provides:
+
+ * A `struct xen_livepatch_name` called `name` which has the unique name.
+ * A `struct xen_livepatch_status` structure. The member values will
+ be over-written upon completion.
+
+Upon completion the `struct xen_livepatch_status` is updated.
+
+ * `status` - indicates the current status of the payload:
+ * *LIVEPATCH_STATUS_CHECKED* (1) loaded and the ELF payload safety checks passed.
+ * *LIVEPATCH_STATUS_APPLIED* (2) loaded, checked, and applied.
+ * No other value is possible.
+ * `rc` - -XEN_EXX type errors encountered while performing the last
+ LIVEPATCH_ACTION_* operation. The normal values can be zero or -XEN_EAGAIN which
+ respectively mean: success or operation in progress. Other values
+ imply an error occurred. If there is an error in `rc`, `status` will **NOT**
+ have changed.
+
+The return value of the hypercall is zero on success and -XEN_EXX on failure.
+(Note that the `rc` value can be different from the return value, as in
+rc = -XEN_EAGAIN and return value can be 0).
+
+For example, supposing there is an payload:
+
+ status: LIVEPATCH_STATUS_CHECKED
+ rc: 0
+
+We apply an action - LIVEPATCH_ACTION_REVERT - to revert it (which won't work
+as we have not even applied it. Afterwards we will have:
+
+ status: LIVEPATCH_STATUS_CHECKED
+ rc: -XEN_EINVAL
+
+It has failed but it remains loaded.
+
+This operation is synchronous and does not require preemption.
+
+The structure is as follow:
+
+ struct xen_livepatch_status {
+ #define LIVEPATCH_STATUS_CHECKED 1
+ #define LIVEPATCH_STATUS_APPLIED 2
+ uint32_t state; /* OUT: LIVEPATCH_STATE_*. */
+ int32_t rc; /* OUT: 0 if no error, otherwise -XEN_EXX. */
+ };
+
+ struct xen_sysctl_livepatch_get {
+ xen_livepatch_name_t name; /* IN, the name of the payload. */
+ xen_livepatch_status_t status; /* IN/OUT: status of the payload. */
+ };
+
+### XEN_SYSCTL_LIVEPATCH_LIST (2)
+
+Retrieve an array of abbreviated status and names of payloads that are loaded in the
+hypervisor.
+
+The caller provides:
+
+ * `version`. Version of the payload. Caller should re-use the field provided by
+ the hypervisor. If the value differs the data is stale.
+ * `idx` Index iterator. The index into the hypervisor's payload count. It is
+ recommended that on first invocation zero be used so that `nr` (which the
+ hypervisor will update with the remaining payload count) be provided.
+ Also the hypervisor will provide `version` with the most current value.
+ * `nr` The max number of entries to populate. Can be zero which will result
+ in the hypercall being a probing one and return the number of payloads
+ (and update the `version`).
+ * `pad` - *MUST* be zero.
+ * `status` Virtual address of where to write `struct xen_livepatch_status`
+ structures. Caller *MUST* allocate up to `nr` of them.
+ * `name` - Virtual address of where to write the unique name of the payload.
+ Caller *MUST* allocate up to `nr` of them. Each *MUST* be of
+ **XEN_LIVEPATCH_NAME_SIZE** size. Note that **XEN_LIVEPATCH_NAME_SIZE** includes
+ the NUL terminator.
+ * `len` - Virtual address of where to write the length of each unique name
+ of the payload. Caller *MUST* allocate up to `nr` of them. Each *MUST* be
+ of sizeof(uint32_t) (4 bytes).
+
+If the hypercall returns an positive number, it is the number (upto `nr`
+provided to the hypercall) of the payloads returned, along with `nr` updated
+with the number of remaining payloads, `version` updated (it may be the same
+across hypercalls - if it varies the data is stale and further calls could
+fail). The `status`, `name`, and `len` are updated at their designed index
+value (`idx`) with the returned value of data.
+
+If the hypercall returns -XEN_E2BIG the `nr` is too big and should be
+lowered.
+
+If the hypercall returns an zero value there are no more payloads.
+
+Note that due to the asynchronous nature of hypercalls the control domain might
+have added or removed a number of payloads making this information stale. It is
+the responsibility of the toolstack to use the `version` field to check
+between each invocation. if the version differs it should discard the stale
+data and start from scratch. It is OK for the toolstack to use the new
+`version` field.
+
+The `struct xen_livepatch_status` structure contains an status of payload which includes:
+
+ * `status` - indicates the current status of the payload:
+ * *LIVEPATCH_STATUS_CHECKED* (1) loaded and the ELF payload safety checks passed.
+ * *LIVEPATCH_STATUS_APPLIED* (2) loaded, checked, and applied.
+ * No other value is possible.
+ * `rc` - -XEN_EXX type errors encountered while performing the last
+ LIVEPATCH_ACTION_* operation. The normal values can be zero or -XEN_EAGAIN which
+ respectively mean: success or operation in progress. Other values
+ imply an error occurred. If there is an error in `rc`, `status` will **NOT**
+ have changed.
+
+The structure is as follow:
+
+ struct xen_sysctl_livepatch_list {
+ uint32_t version; /* OUT: Hypervisor stamps value.
+ If varies between calls, we are
+ getting stale data. */
+ uint32_t idx; /* IN: Index into hypervisor list. */
+ uint32_t nr; /* IN: How many status, names, and len
+ should be filled out. Can be zero to get
+ amount of payloads and version.
+ OUT: How many payloads left. */
+ uint32_t pad; /* IN: Must be zero. */
+ XEN_GUEST_HANDLE_64(xen_livepatch_status_t) status; /* OUT. Must have enough
+ space allocate for nr of them. */
+ XEN_GUEST_HANDLE_64(char) id; /* OUT: Array of names. Each member
+ MUST XEN_LIVEPATCH_NAME_SIZE in size.
+ Must have nr of them. */
+ XEN_GUEST_HANDLE_64(uint32) len; /* OUT: Array of lengths of name's.
+ Must have nr of them. */
+ };
+
+### XEN_SYSCTL_LIVEPATCH_ACTION (3)
+
+Perform an operation on the payload structure referenced by the `name` field.
+The operation request is asynchronous and the status should be retrieved
+by using either **XEN_SYSCTL_LIVEPATCH_GET** or **XEN_SYSCTL_LIVEPATCH_LIST** hypercall.
+
+The caller provides:
+
+ * A `struct xen_livepatch_name` `name` containing the unique name.
+ * `cmd` The command requested:
+ * *LIVEPATCH_ACTION_UNLOAD* (1) Unload the payload.
+ Any further hypercalls against the `name` will result in failure unless
+ **XEN_SYSCTL_LIVEPATCH_UPLOAD** hypercall is perfomed with same `name`.
+ * *LIVEPATCH_ACTION_REVERT* (2) Revert the payload. If the operation takes
+ more time than the upper bound of time the `rc` in `xen_livepatch_status`
+ retrieved via **XEN_SYSCTL_LIVEPATCH_GET** will be -XEN_EBUSY.
+ * *LIVEPATCH_ACTION_APPLY* (3) Apply the payload. If the operation takes
+ more time than the upper bound of time the `rc` in `xen_livepatch_status`
+ retrieved via **XEN_SYSCTL_LIVEPATCH_GET** will be -XEN_EBUSY.
+ * *LIVEPATCH_ACTION_REPLACE* (4) Revert all applied payloads and apply this
+ payload. If the operation takes more time than the upper bound of time
+ the `rc` in `xen_livepatch_status` retrieved via **XEN_SYSCTL_LIVEPATCH_GET**
+ will be -XEN_EBUSY.
+ * `time` The upper bound of time (ns) the cmd should take. Zero means to use
+ the hypervisor default. If within the time the operation does not succeed
+ the operation would go in error state.
+ * `pad` - *MUST* be zero.
+
+The return value will be zero unless the provided fields are incorrect.
+
+The structure is as follow:
+
+ #define LIVEPATCH_ACTION_UNLOAD 1
+ #define LIVEPATCH_ACTION_REVERT 2
+ #define LIVEPATCH_ACTION_APPLY 3
+ #define LIVEPATCH_ACTION_REPLACE 4
+ struct xen_sysctl_livepatch_action {
+ xen_livepatch_name_t name; /* IN, name of the patch. */
+ uint32_t cmd; /* IN: LIVEPATCH_ACTION_* */
+ uint32_t time; /* IN: If zero then uses */
+ /* hypervisor default. */
+ /* Or upper bound of time (ns) */
+ /* for operation to take. */
+ };
+
+
+## State diagrams of LIVEPATCH_ACTION commands.
+
+There is a strict ordering state of what the commands can be.
+The LIVEPATCH_ACTION prefix has been dropped to easy reading and
+does not include the LIVEPATCH_STATES:
+
+ /->\
+ \ /
+ UNLOAD <--- CHECK ---> REPLACE|APPLY --> REVERT --\
+ \ |
+ \-------------------<-------------/
+
+## State transition table of LIVEPATCH_ACTION commands and LIVEPATCH_STATUS.
+
+Note that:
+
+ - The CHECKED state is the starting one achieved with *XEN_SYSCTL_LIVEPATCH_UPLOAD* hypercall.
+ - The REVERT operation on success will automatically move to the CHECKED state.
+ - There are two STATES: CHECKED and APPLIED.
+ - There are four actions (aka commands): APPLY, REPLACE, REVERT, and UNLOAD.
+
+The state transition table of valid states and action states:
+
+ +---------+---------+--------------------------------+-------+--------+
+ | ACTION | Current | Result | Next STATE: |
+ | ACTION | STATE | |CHECKED|APPLIED |
+ +---------+----------+-------------------------------+-------+--------+
+ | UNLOAD | CHECKED | Unload payload. Always works. | | |
+ | | | No next states. | | |
+ +---------+---------+--------------------------------+-------+--------+
+ | APPLY | CHECKED | Apply payload (success). | | x |
+ +---------+---------+--------------------------------+-------+--------+
+ | APPLY | CHECKED | Apply payload (error|timeout) | x | |
+ +---------+---------+--------------------------------+-------+--------+
+ | REPLACE | CHECKED | Revert payloads and apply new | | x |
+ | | | payload with success. | | |
+ +---------+---------+--------------------------------+-------+--------+
+ | REPLACE | CHECKED | Revert payloads and apply new | x | |
+ | | | payload with error. | | |
+ +---------+---------+--------------------------------+-------+--------+
+ | REVERT | APPLIED | Revert payload (success). | x | |
+ +---------+---------+--------------------------------+-------+--------+
+ | REVERT | APPLIED | Revert payload (error|timeout) | | x |
+ +---------+---------+--------------------------------+-------+--------+
+
+All the other state transitions are invalid.
+
+## Sequence of events.
+
+The normal sequence of events is to:
+
+ 1. *XEN_SYSCTL_LIVEPATCH_UPLOAD* to upload the payload. If there are errors *STOP* here.
+ 2. *XEN_SYSCTL_LIVEPATCH_GET* to check the `->rc`. If *-XEN_EAGAIN* spin. If zero go to next step.
+ 3. *XEN_SYSCTL_LIVEPATCH_ACTION* with *LIVEPATCH_ACTION_APPLY* to apply the patch.
+ 4. *XEN_SYSCTL_LIVEPATCH_GET* to check the `->rc`. If in *-XEN_EAGAIN* spin. If zero exit with success.
+
+
+## Addendum
+
+Implementation quirks should not be discussed in a design document.
+
+However these observations can provide aid when developing against this
+document.
+
+
+### Alternative assembler
+
+Alternative assembler is a mechanism to use different instructions depending
+on what the CPU supports. This is done by providing multiple streams of code
+that can be patched in - or if the CPU does not support it - padded with
+`nop` operations. The alternative assembler macros cause the compiler to
+expand the code to place a most generic code in place - emit a special
+ELF .section header to tag this location. During run-time the hypervisor
+can leave the areas alone or patch them with an better suited opcodes.
+
+Note that patching functions that copy to or from guest memory requires
+to support alternative support. For example this can be due to SMAP
+(specifically *stac* and *clac* operations) which is enabled on Broadwell
+and later architectures. It may be related to other alternative instructions.
+
+### When to patch
+
+During the discussion on the design two candidates bubbled where
+the call stack for each CPU would be deterministic. This would
+minimize the chance of the patch not being applied due to safety
+checks failing. Safety checks such as not patching code which
+is on the stack - which can lead to corruption.
+
+#### Rendezvous code instead of stop_machine for patching
+
+The hypervisor's time rendezvous code runs synchronously across all CPUs
+every second. Using the `stop_machine` to patch can stall the time rendezvous
+code and result in NMI. As such having the patching be done at the tail
+of rendezvous code should avoid this problem.
+
+However the entrance point for that code is `do_softirq ->
+timer_softirq_action -> time_calibration` which ends up calling
+`on_selected_cpus` on remote CPUs.
+
+The remote CPUs receive CALL_FUNCTION_VECTOR IPI and execute the
+desired function.
+
+#### Before entering the guest code.
+
+Before we call VMXResume we check whether any soft IRQs need to be executed.
+This is a good spot because all Xen stacks are effectively empty at
+that point.
+
+To randezvous all the CPUs an barrier with an maximum timeout (which
+could be adjusted), combined with forcing all other CPUs through the
+hypervisor with IPIs, can be utilized to execute lockstep instructions
+on all CPUs.
+
+The approach is similar in concept to `stop_machine` and the time rendezvous
+but is time-bound. However the local CPU stack is much shorter and
+a lot more deterministic.
+
+This is implemented in the Xen hypervisor.
+
+### Compiling the hypervisor code
+
+Hotpatch generation often requires support for compiling the target
+with `-ffunction-sections` / `-fdata-sections`. Changes would have to
+be done to the linker scripts to support this.
+
+### Generation of Live Patch ELF payloads
+
+The design of that is not discussed in this design.
+
+This is implemented in a seperate tool which lives in a seperate
+GIT repo.
+
+Currently it resides at git://xenbits.xen.org/livepatch-build-tools.git
+
+### Exception tables and symbol tables growth
+
+We may need support for adapting or augmenting exception tables if
+patching such code. Hotpatches may need to bring their own small
+exception tables (similar to how Linux modules support this).
+
+If supporting hotpatches that introduce additional exception-locations
+is not important, one could also change the exception table in-place
+and reorder it afterwards.
+
+As found almost every patch (XSA) to a non-trivial function requires
+additional entries in the exception table and/or the bug frames.
+
+This is implemented in the Xen hypervisor.
+
+### .rodata sections
+
+The patching might require strings to be updated as well. As such we must be
+also able to patch the strings as needed. This sounds simple - but the compiler
+has a habit of coalescing strings that are the same - which means if we in-place
+alter the strings - other users will be inadvertently affected as well.
+
+This is also where pointers to functions live - and we may need to patch this
+as well. And switch-style jump tables.
+
+To guard against that we must be prepared to do patching similar to
+trampoline patching or in-line depending on the flavour. If we can
+do in-line patching we would need to:
+
+ * Alter `.rodata` to be writeable.
+ * Inline patch.
+ * Alter `.rodata` to be read-only.
+
+If are doing trampoline patching we would need to:
+
+ * Allocate a new memory location for the string.
+ * All locations which use this string will have to be updated to use the
+ offset to the string.
+ * Mark the region RO when we are done.
+
+The trampoline patching is implemented in the Xen hypervisor.
+
+### .bss and .data sections.
+
+In place patching writable data is not suitable as it is unclear what should be done
+depending on the current state of data. As such it should not be attempted.
+
+However, functions which are being patched can bring in changes to strings
+(.data or .rodata section changes), or even to .bss sections.
+
+As such the ELF payload can introduce new .rodata, .bss, and .data sections.
+Patching in the new function will end up also patching in the new .rodata
+section and the new function will reference the new string in the new
+.rodata section.
+
+This is implemented in the Xen hypervisor.
+
+### Security
+
+Only the privileged domain should be allowed to do this operation.
+
+### Live patch interdependencies
+
+Live patch patches interdependencies are tricky.
+
+There are the ways this can be addressed:
+ * A single large patch that subsumes and replaces all previous ones.
+ Over the life-time of patching the hypervisor this large patch
+ grows to accumulate all the code changes.
+ * Hotpatch stack - where an mechanism exists that loads the hotpatches
+ in the same order they were built in. We would need an build-id
+ of the hypevisor to make sure the hot-patches are build against the
+ correct build.
+ * Payload containing the old code to check against that. That allows
+ the hotpatches to be loaded indepedently (if they don't overlap) - or
+ if the old code also containst previously patched code - even if they
+ overlap.
+
+The disadvantage of the first large patch is that it can grow over
+time and not provide an bisection mechanism to identify faulty patches.
+
+The hot-patch stack puts stricts requirements on the order of the patches
+being loaded and requires an hypervisor build-id to match against.
+
+The old code allows much more flexibility and an additional guard,
+but is more complex to implement.
+
+The second option which requires an build-id of the hypervisor
+is implemented in the Xen hypervisor.
+
+Specifically each payload has two build-id ELF notes:
+ * The build-id of the payload itself (generated via --build-id).
+ * The build-id of the payload it depends on (extracted from the
+ the previous payload or hypervisor during build time).
+
+This means that the very first payload depends on the hypervisor
+build-id.
+
+# Not Yet Done
+
+This is for further development of live patching.
+
+## TODO Goals
+
+The implementation must also have a mechanism for (in no particular order):
+
+ * Be able to lookup in the Xen hypervisor the symbol names of functions from the
+ ELF payload. (Either as `symbol` or `symbol`+`offset`).
+ * Be able to patch .rodata, .bss, and .data sections.
+ * Deal with NMI/MCE checks during patching instead of ignoring them.
+ * Further safety checks (blacklist of which functions cannot be patched, check
+ the stack, make sure the payload is built with same compiler as hypervisor).
+ Specifically we want to make sure that live patching codepaths cannot be patched.
+ * NOP out the code sequence if `new_size` is zero.
+ * Deal with other relocation types: `R_X86_64_[8,16,32,32S]`, `R_X86_64_PC[8,16,64]`
+ in payload file.
+
+### Handle inlined \__LINE__
+
+This problem is related to hotpatch construction
+and potentially has influence on the design of the hotpatching
+infrastructure in Xen.
+
+For example:
+
+We have file1.c with functions f1 and f2 (in that order). f2 contains a
+BUG() (or WARN()) macro and at that point embeds the source line number
+into the generated code for f2.
+
+Now we want to hotpatch f1 and the hotpatch source-code patch adds 2
+lines to f1 and as a consequence shifts out f2 by two lines. The newly
+constructed file1.o will now contain differences in both binary
+functions f1 (because we actually changed it with the applied patch) and
+f2 (because the contained BUG macro embeds the new line number).
+
+Without additional information, an algorithm comparing file1.o before
+and after hotpatch application will determine both functions to be
+changed and will have to include both into the binary hotpatch.
+
+Options:
+
+1. Transform source code patches for hotpatches to be line-neutral for
+ each chunk. This can be done in almost all cases with either
+ reformatting of the source code or by introducing artificial
+ preprocessor "#line n" directives to adjust for the introduced
+ differences.
+
+ This approach is low-tech and simple. Potentially generated
+ backtraces and existing debug information refers to the original
+ build and does not reflect hotpatching state except for actually
+ hotpatched functions but should be mostly correct.
+
+2. Ignoring the problem and living with artificially large hotpatches
+ that unnecessarily patch many functions.
+
+ This approach might lead to some very large hotpatches depending on
+ content of specific source file. It may also trigger pulling in
+ functions into the hotpatch that cannot reasonable be hotpatched due
+ to limitations of a hotpatching framework (init-sections, parts of
+ the hotpatching framework itself, ...) and may thereby prevent us
+ from patching a specific problem.
+
+ The decision between 1. and 2. can be made on a patch--by-patch
+ basis.
+
+3. Introducing an indirection table for storing line numbers and
+ treating that specially for binary diffing. Linux may follow
+ this approach.
+
+ We might either use this indirection table for runtime use and patch
+ that with each hotpatch (similarly to exception tables) or we might
+ purely use it when building hotpatches to ignore functions that only
+ differ at exactly the location where a line-number is embedded.
+
+For BUG(), WARN(), etc., the line number is embedded into the bug frame, not
+the function itself.
+
+Similar considerations are true to a lesser extent for \__FILE__, but it
+could be argued that file renaming should be done outside of hotpatches.
+
+## Signature checking requirements.
+
+The signature checking requires that the layout of the data in memory
+**MUST** be same for signature to be verified. This means that the payload
+data layout in ELF format **MUST** match what the hypervisor would be
+expecting such that it can properly do signature verification.
+
+The signature is based on the all of the payloads continuously laid out
+in memory. The signature is to be appended at the end of the ELF payload
+prefixed with the string '`~Module signature appended~\n`', followed by
+an signature header then followed by the signature, key identifier, and signers
+name.
+
+Specifically the signature header would be:
+
+ #define PKEY_ALGO_DSA 0
+ #define PKEY_ALGO_RSA 1
+
+ #define PKEY_ID_PGP 0 /* OpenPGP generated key ID */
+ #define PKEY_ID_X509 1 /* X.509 arbitrary subjectKeyIdentifier */
+
+ #define HASH_ALGO_MD4 0
+ #define HASH_ALGO_MD5 1
+ #define HASH_ALGO_SHA1 2
+ #define HASH_ALGO_RIPE_MD_160 3
+ #define HASH_ALGO_SHA256 4
+ #define HASH_ALGO_SHA384 5
+ #define HASH_ALGO_SHA512 6
+ #define HASH_ALGO_SHA224 7
+ #define HASH_ALGO_RIPE_MD_128 8
+ #define HASH_ALGO_RIPE_MD_256 9
+ #define HASH_ALGO_RIPE_MD_320 10
+ #define HASH_ALGO_WP_256 11
+ #define HASH_ALGO_WP_384 12
+ #define HASH_ALGO_WP_512 13
+ #define HASH_ALGO_TGR_128 14
+ #define HASH_ALGO_TGR_160 15
+ #define HASH_ALGO_TGR_192 16
+
+ struct elf_payload_signature {
+ u8 algo; /* Public-key crypto algorithm PKEY_ALGO_*. */
+ u8 hash; /* Digest algorithm: HASH_ALGO_*. */
+ u8 id_type; /* Key identifier type PKEY_ID*. */
+ u8 signer_len; /* Length of signer's name */
+ u8 key_id_len; /* Length of key identifier */
+ u8 __pad[3];
+ __be32 sig_len; /* Length of signature data */
+ };
+
+(Note that this has been borrowed from Linux module signature code.).
+
+
+### .bss and .data sections.
+
+In place patching writable data is not suitable as it is unclear what should be done
+depending on the current state of data. As such it should not be attempted.
+
+That said we should provide hook functions so that the existing data
+can be changed during payload application.
+
+To guarantee safety we disallow re-applying an payload after it has been
+reverted. This is because we cannot guarantee that the state of .bss
+and .data to be exactly as it was during loading. Hence the administrator
+MUST unload the payload and upload it again to apply it.
+
+There is an exception to this: if the payload only has .livepatch.funcs;
+and the .data or .bss sections are of zero length.
+
+### Inline patching
+
+The hypervisor should verify that the in-place patching would fit within
+the code or data.
+
+### Trampoline (e9 opcode), x86
+
+The e9 opcode used for jmpq uses a 32-bit signed displacement. That means
+we are limited to up to 2GB of virtual address to place the new code
+from the old code. That should not be a problem since Xen hypervisor has
+a very small footprint.
+
+However if we need - we can always add two trampolines. One at the 2GB
+limit that calls the next trampoline.
+
+Please note there is a small limitation for trampolines in
+function entries: The target function (+ trailing padding) must be able
+to accomodate the trampoline. On x86 with +-2 GB relative jumps,
+this means 5 bytes are required which means that `old_size` **MUST** be
+at least five bytes if patching in trampoline.
+
+Depending on compiler settings, there are several functions in Xen that
+are smaller (without inter-function padding).
+
+ readelf -sW xen-syms | grep " FUNC " | \
+ awk '{ if ($3 < 5) print $3, $4, $5, $8 }'
+
+ ...
+ 3 FUNC LOCAL wbinvd_ipi
+ 3 FUNC LOCAL shadow_l1_index
+ ...
+
+A compile-time check for, e.g., a minimum alignment of functions or a
+runtime check that verifies symbol size (+ padding to next symbols) for
+that in the hypervisor is advised.
+
+The tool for generating payloads currently does perform a compile-time
+check to ensure that the function to be replaced is large enough.
+
+#### Trampoline, ARM
+
+The unconditional branch instruction (for the encoding see the
+DDI 0406C.c and DDI 0487A.j Architecture Reference Manual's).
+with proper offset is used for an unconditional branch to the new code.
+This means that that `old_size` **MUST** be at least four bytes if patching
+in trampoline.
+
+The instruction offset is limited on ARM32 to +/- 32MB to displacement
+and on ARM64 to +/- 128MB displacement.
+
+The new code is placed in the 8M - 10M virtual address space while the
+Xen code is in 2M - 4M. That gives us enough space.
+
+The hypervisor also checks the displacement during loading of the payload.
+++ /dev/null
-# Xen PV Drivers lifecycle
-
-## Purpose
-
-Getting new PV drivers accepted in Xen, upstream code bases, and ABI
-stable in the quickest and most efficient way possible.
-
-
-## Design Phase
-
-The first step toward acceptance of a new PV protocol is to write a
-design document and send it to xen-devel. It should cover the xenstore
-handshake mechanism, the ABI, how the protocol works and anything else
-which is required to write an implementation of it. The usage of C-like
-structs to describe language and platform agnostic protocols is
-discouraged.
-
-An attempt should be made to design the ABI such that it will be OS
-agnostic, that future versions will not need to introduce
-backward-incompatible changes, and so on; but these are not yet hard
-requirements.
-
-After the high level design of the protocol has been discussed and
-agreed, the document is committed to xen.git.
-
-
-## Prototype Stage
-
-The contributor sends patches to implement the PV drivers for the new
-protocol to the relevant open source mailing lists, such as LKML,
-qemu-devel and xen-devel.
-
-The code is expected to work, be good quality and faithfully implement
-the spec. However, there are no promises about ABI and cross-platform
-compatibility yet.
-
-After careful review by the relevant maintainers, the code is committed
-to the upstream code bases. The drivers are considered experimental.
-
-
-## Production Stage
-
-The quality of the drivers and the spec is improved. Bugs are fixed.
-The protocol version is likely bumped. More testing leads to confidence
-that the spec and the drivers are ready for production usage. Promises
-about backward compatibility and cross-platform compatibility are
-clearly spelled out.
-
-
-## How to move forward from a stage to the next
-
-The PV protocols Czar is responsible for determining the transitions
-between stages. Our governance principles specify "lazy consensus" for
-most things. It applies to this case too. New PV protocols should move
-from one stage to the next within a reasonable time frame unless someone
-has specific technical objections and voices them in a responsive
-manner.
--- /dev/null
+# Xen PV Drivers lifecycle
+
+## Purpose
+
+Getting new PV drivers accepted in Xen, upstream code bases, and ABI
+stable in the quickest and most efficient way possible.
+
+
+## Design Phase
+
+The first step toward acceptance of a new PV protocol is to write a
+design document and send it to xen-devel. It should cover the xenstore
+handshake mechanism, the ABI, how the protocol works and anything else
+which is required to write an implementation of it. The usage of C-like
+structs to describe language and platform agnostic protocols is
+discouraged.
+
+An attempt should be made to design the ABI such that it will be OS
+agnostic, that future versions will not need to introduce
+backward-incompatible changes, and so on; but these are not yet hard
+requirements.
+
+After the high level design of the protocol has been discussed and
+agreed, the document is committed to xen.git.
+
+
+## Prototype Stage
+
+The contributor sends patches to implement the PV drivers for the new
+protocol to the relevant open source mailing lists, such as LKML,
+qemu-devel and xen-devel.
+
+The code is expected to work, be good quality and faithfully implement
+the spec. However, there are no promises about ABI and cross-platform
+compatibility yet.
+
+After careful review by the relevant maintainers, the code is committed
+to the upstream code bases. The drivers are considered experimental.
+
+
+## Production Stage
+
+The quality of the drivers and the spec is improved. Bugs are fixed.
+The protocol version is likely bumped. More testing leads to confidence
+that the spec and the drivers are ready for production usage. Promises
+about backward compatibility and cross-platform compatibility are
+clearly spelled out.
+
+
+## How to move forward from a stage to the next
+
+The PV protocols Czar is responsible for determining the transitions
+between stages. Our governance principles specify "lazy consensus" for
+most things. It applies to this case too. New PV protocols should move
+from one stage to the next within a reasonable time frame unless someone
+has specific technical objections and voices them in a responsive
+manner.
+++ /dev/null
-# PV Calls Protocol version 1
-
-## Glossary
-
-The following is a list of terms and definitions used in the Xen
-community. If you are a Xen contributor you can skip this section.
-
-* PV
-
- Short for paravirtualized.
-
-* Dom0
-
- First virtual machine that boots. In most configurations Dom0 is
- privileged and has control over hardware devices, such as network
- cards, graphic cards, etc.
-
-* DomU
-
- Regular unprivileged Xen virtual machine.
-
-* Domain
-
- A Xen virtual machine. Dom0 and all DomUs are all separate Xen
- domains.
-
-* Guest
-
- Same as domain: a Xen virtual machine.
-
-* Frontend
-
- Each DomU has one or more paravirtualized frontend drivers to access
- disks, network, console, graphics, etc. The presence of PV devices is
- advertized on XenStore, a cross domain key-value database. Frontends
- are similar in intent to the virtio drivers in Linux.
-
-* Backend
-
- A Xen paravirtualized backend typically runs in Dom0 and it is used to
- export disks, network, console, graphics, etcs, to DomUs. Backends can
- live both in kernel space and in userspace. For example xen-blkback
- lives under drivers/block in the Linux kernel and xen_disk lives under
- hw/block in QEMU. Paravirtualized backends are similar in intent to
- virtio device emulators.
-
-* VMX and SVM
-
- On Intel processors, VMX is the CPU flag for VT-x, hardware
- virtualization support. It corresponds to SVM on AMD processors.
-
-
-
-## Rationale
-
-PV Calls is a paravirtualized protocol that allows the implementation of
-a set of POSIX functions in a different domain. The PV Calls frontend
-sends POSIX function calls to the backend, which implements them and
-returns a value to the frontend and acts on the function call.
-
-This version of the document covers networking function calls, such as
-connect, accept, bind, release, listen, poll, recvmsg and sendmsg; but
-the protocol is meant to be easily extended to cover different sets of
-calls. Unimplemented commands return ENOTSUP.
-
-PV Calls provide the following benefits:
-* full visibility of the guest behavior on the backend domain, allowing
- for inexpensive filtering and manipulation of any guest calls
-* excellent performance
-
-Specifically, PV Calls for networking offer these advantages:
-* guest networking works out of the box with VPNs, wireless networks and
- any other complex configurations on the host
-* guest services listen on ports bound directly to the backend domain IP
- addresses
-* localhost becomes a secure host wide network for inter-VMs
- communications
-
-
-## Design
-
-### Why Xen?
-
-PV Calls are part of an effort to create a secure runtime environment
-for containers (Open Containers Initiative images to be precise). PV
-Calls are based on Xen, although porting them to other hypervisors is
-possible. Xen was chosen because of its security and isolation
-properties and because it supports PV guests, a type of virtual machines
-that does not require hardware virtualization extensions (VMX on Intel
-processors and SVM on AMD processors). This is important because PV
-Calls is meant for containers and containers are often run on top of
-public cloud instances, which do not support nested VMX (or SVM) as of
-today (early 2017). Xen PV guests are lightweight, minimalist, and do
-not require machine emulation: all properties that make them a good fit
-for this project.
-
-### Xenstore
-
-The frontend and the backend connect via [xenstore] to
-exchange information. The toolstack creates front and back nodes with
-state of [XenbusStateInitialising]. The protocol node name
-is **pvcalls**. There can only be one PV Calls frontend per domain.
-
-#### Frontend XenBus Nodes
-
-version
- Values: <string>
-
- Protocol version, chosen among the ones supported by the backend
- (see **versions** under [Backend XenBus Nodes]). Currently the
- value must be "1".
-
-port
- Values: <uint32_t>
-
- The identifier of the Xen event channel used to signal activity
- in the command ring.
-
-ring-ref
- Values: <uint32_t>
-
- The Xen grant reference granting permission for the backend to map
- the sole page in a single page sized command ring.
-
-#### Backend XenBus Nodes
-
-versions
- Values: <string>
-
- List of comma separated protocol versions supported by the backend.
- For example "1,2,3". Currently the value is just "1", as there is
- only one version.
-
-max-page-order
- Values: <uint32_t>
-
- The maximum supported size of a memory allocation in units of
- log2n(machine pages), e.g. 1 = 2 pages, 2 == 4 pages, etc. It must
- be 1 or more.
-
-function-calls
- Values: <uint32_t>
-
- Value "0" means that no calls are supported.
- Value "1" means that socket, connect, release, bind, listen, accept
- and poll are supported.
-
-#### State Machine
-
-Initialization:
-
- *Front* *Back*
- XenbusStateInitialising XenbusStateInitialising
- - Query virtual device - Query backend device
- properties. identification data.
- - Setup OS device instance. - Publish backend features
- - Allocate and initialize the and transport parameters
- request ring. |
- - Publish transport parameters |
- that will be in effect during V
- this connection. XenbusStateInitWait
- |
- |
- V
- XenbusStateInitialised
-
- - Query frontend transport parameters.
- - Connect to the request ring and
- event channel.
- |
- |
- V
- XenbusStateConnected
-
- - Query backend device properties.
- - Finalize OS virtual device
- instance.
- |
- |
- V
- XenbusStateConnected
-
-Once frontend and backend are connected, they have a shared page, which
-will is used to exchange messages over a ring, and an event channel,
-which is used to send notifications.
-
-Shutdown:
-
- *Front* *Back*
- XenbusStateConnected XenbusStateConnected
- |
- |
- V
- XenbusStateClosing
-
- - Unmap grants
- - Unbind event channels
- |
- |
- V
- XenbusStateClosing
-
- - Unbind event channels
- - Free rings
- - Free data structures
- |
- |
- V
- XenbusStateClosed
-
- - Free remaining data structures
- |
- |
- V
- XenbusStateClosed
-
-
-### Commands Ring
-
-The shared ring is used by the frontend to forward POSIX function calls
-to the backend. We shall refer to this ring as **commands ring** to
-distinguish it from other rings which can be created later in the
-lifecycle of the protocol (see [Indexes Page and Data ring]). The grant
-reference for shared page for this ring is shared on xenstore (see
-[Frontend XenBus Nodes]). The ring format is defined using the familiar
-`DEFINE_RING_TYPES` macro (`xen/include/public/io/ring.h`). Frontend
-requests are allocated on the ring using the `RING_GET_REQUEST` macro.
-The list of commands below is in calling order.
-
-The format is defined as follows:
-
- #define PVCALLS_SOCKET 0
- #define PVCALLS_CONNECT 1
- #define PVCALLS_RELEASE 2
- #define PVCALLS_BIND 3
- #define PVCALLS_LISTEN 4
- #define PVCALLS_ACCEPT 5
- #define PVCALLS_POLL 6
-
- struct xen_pvcalls_request {
- uint32_t req_id; /* private to guest, echoed in response */
- uint32_t cmd; /* command to execute */
- union {
- struct xen_pvcalls_socket {
- uint64_t id;
- uint32_t domain;
- uint32_t type;
- uint32_t protocol;
- #ifdef CONFIG_X86_32
- uint8_t pad[4];
- #endif
- } socket;
- struct xen_pvcalls_connect {
- uint64_t id;
- uint8_t addr[28];
- uint32_t len;
- uint32_t flags;
- grant_ref_t ref;
- uint32_t evtchn;
- #ifdef CONFIG_X86_32
- uint8_t pad[4];
- #endif
- } connect;
- struct xen_pvcalls_release {
- uint64_t id;
- uint8_t reuse;
- #ifdef CONFIG_X86_32
- uint8_t pad[7];
- #endif
- } release;
- struct xen_pvcalls_bind {
- uint64_t id;
- uint8_t addr[28];
- uint32_t len;
- } bind;
- struct xen_pvcalls_listen {
- uint64_t id;
- uint32_t backlog;
- #ifdef CONFIG_X86_32
- uint8_t pad[4];
- #endif
- } listen;
- struct xen_pvcalls_accept {
- uint64_t id;
- uint64_t id_new;
- grant_ref_t ref;
- uint32_t evtchn;
- } accept;
- struct xen_pvcalls_poll {
- uint64_t id;
- } poll;
- /* dummy member to force sizeof(struct xen_pvcalls_request) to match across archs */
- struct xen_pvcalls_dummy {
- uint8_t dummy[56];
- } dummy;
- } u;
- };
-
-The first two fields are common for every command. Their binary layout
-is:
-
- 0 4 8
- +-------+-------+
- |req_id | cmd |
- +-------+-------+
-
-- **req_id** is generated by the frontend and is a cookie used to
- identify one specific request/response pair of commands. Not to be
- confused with any command **id** which are used to identify a socket
- across multiple commands, see [Socket].
-- **cmd** is the command requested by the frontend:
-
- - `PVCALLS_SOCKET`: 0
- - `PVCALLS_CONNECT`: 1
- - `PVCALLS_RELEASE`: 2
- - `PVCALLS_BIND`: 3
- - `PVCALLS_LISTEN`: 4
- - `PVCALLS_ACCEPT`: 5
- - `PVCALLS_POLL`: 6
-
-Both fields are echoed back by the backend. See [Socket families and
-address format] for the format of the **addr** field of connect and
-bind. The maximum size of command specific arguments is 56 bytes. Any
-future command that requires more than that will need a bump the
-**version** of the protocol.
-
-Similarly to other Xen ring based protocols, after writing a request to
-the ring, the frontend calls `RING_PUSH_REQUESTS_AND_CHECK_NOTIFY` and
-issues an event channel notification when a notification is required.
-
-Backend responses are allocated on the ring using the `RING_GET_RESPONSE` macro.
-The format is the following:
-
- struct xen_pvcalls_response {
- uint32_t req_id;
- uint32_t cmd;
- int32_t ret;
- uint32_t pad;
- union {
- struct _xen_pvcalls_socket {
- uint64_t id;
- } socket;
- struct _xen_pvcalls_connect {
- uint64_t id;
- } connect;
- struct _xen_pvcalls_release {
- uint64_t id;
- } release;
- struct _xen_pvcalls_bind {
- uint64_t id;
- } bind;
- struct _xen_pvcalls_listen {
- uint64_t id;
- } listen;
- struct _xen_pvcalls_accept {
- uint64_t id;
- } accept;
- struct _xen_pvcalls_poll {
- uint64_t id;
- } poll;
- struct _xen_pvcalls_dummy {
- uint8_t dummy[8];
- } dummy;
- } u;
- };
-
-The first four fields are common for every response. Their binary layout
-is:
-
- 0 4 8 12 16
- +-------+-------+-------+-------+
- |req_id | cmd | ret | pad |
- +-------+-------+-------+-------+
-
-- **req_id**: echoed back from request
-- **cmd**: echoed back from request
-- **ret**: return value, identifies success (0) or failure (see [Error
- numbers] in further sections). If the **cmd** is not supported by the
- backend, ret is ENOTSUP.
-- **pad**: padding
-
-After calling `RING_PUSH_RESPONSES_AND_CHECK_NOTIFY`, the backend checks whether
-it needs to notify the frontend and does so via event channel.
-
-A description of each command, their additional request and response
-fields follow.
-
-
-#### Socket
-
-The **socket** operation corresponds to the POSIX [socket][socket]
-function. It creates a new socket of the specified family, type and
-protocol. **id** is freely chosen by the frontend and references this
-specific socket from this point forward. See [Socket families and
-address format] to see which ones are supported by different versions of
-the protocol.
-
-Request fields:
-
-- **cmd** value: 0
-- additional fields:
- - **id**: generated by the frontend, it identifies the new socket
- - **domain**: the communication domain
- - **type**: the socket type
- - **protocol**: the particular protocol to be used with the socket, usually 0
-
-Request binary layout:
-
- 8 12 16 20 24 28
- +-------+-------+-------+-------+-------+
- | id |domain | type |protoco|
- +-------+-------+-------+-------+-------+
-
-Response additional fields:
-
-- **id**: echoed back from request
-
-Response binary layout:
-
- 16 20 24
- +-------+--------+
- | id |
- +-------+--------+
-
-Return value:
-
- - 0 on success
- - See the [POSIX socket function][connect] for error names; see
- [Error numbers] in further sections.
-
-#### Connect
-
-The **connect** operation corresponds to the POSIX [connect][connect]
-function. It connects a previously created socket (identified by **id**)
-to the specified address.
-
-The connect operation creates a new shared ring, which we'll call **data
-ring**. The data ring is used to send and receive data from the
-socket. The connect operation passes two additional parameters:
-**evtchn** and **ref**. **evtchn** is the port number of a new event
-channel which will be used for notifications of activity on the data
-ring. **ref** is the grant reference of the **indexes page**: a page
-which contains shared indexes that point to the write and read locations
-in the **data ring**. The **indexes page** also contains the full array
-of grant references for the **data ring**. When the frontend issues a
-**connect** command, the backend:
-
-- finds its own internal socket corresponding to **id**
-- connects the socket to **addr**
-- maps the grant reference **ref**, the indexes page, see struct
- pvcalls_data_intf
-- maps all the grant references listed in `struct pvcalls_data_intf` and
- uses them as shared memory for the **data ring**
-- bind the **evtchn**
-- replies to the frontend
-
-The [Indexes Page and Data ring] format will be described in the
-following section. The **data ring** is unmapped and freed upon issuing
-a **release** command on the active socket identified by **id**. A
-frontend state change can also cause data rings to be unmapped.
-
-Request fields:
-
-- **cmd** value: 0
-- additional fields:
- - **id**: identifies the socket
- - **addr**: address to connect to, see [Socket families and address format]
- - **len**: address length up to 28 octets
- - **flags**: flags for the connection, reserved for future usage
- - **ref**: grant reference of the indexes page
- - **evtchn**: port number of the evtchn to signal activity on the **data ring**
-
-Request binary layout:
-
- 8 12 16 20 24 28 32 36 40 44
- +-------+-------+-------+-------+-------+-------+-------+-------+-------+
- | id | addr |
- +-------+-------+-------+-------+-------+-------+-------+-------+-------+
- | len | flags | ref |evtchn |
- +-------+-------+-------+-------+
-
-Response additional fields:
-
-- **id**: echoed back from request
-
-Response binary layout:
-
- 16 20 24
- +-------+-------+
- | id |
- +-------+-------+
-
-Return value:
-
- - 0 on success
- - See the [POSIX connect function][connect] for error names; see
- [Error numbers] in further sections.
-
-#### Release
-
-The **release** operation closes an existing active or a passive socket.
-
-When a release command is issued on a passive socket, the backend
-releases it and frees its internal mappings. When a release command is
-issued for an active socket, the data ring and indexes page are also
-unmapped and freed:
-
-- frontend sends release command for an active socket
-- backend releases the socket
-- backend unmaps the data ring
-- backend unmaps the indexes page
-- backend unbinds the event channel
-- backend replies to frontend with an **ret** value
-- frontend frees data ring, indexes page and unbinds event channel
-
-Request fields:
-
-- **cmd** value: 1
-- additional fields:
- - **id**: identifies the socket
- - **reuse**: an optimization hint for the backend. The field is
- ignored for passive sockets. When set to 1, the frontend lets the
- backend know that it will reuse exactly the same set of grant pages
- (indexes page and data ring) and event channel when creating one of
- the next active sockets. The backend can take advantage of it by
- delaying unmapping grants and unbinding the event channel. The
- backend is free to ignore the hint. Reused data rings are found by
- **ref**, the grant reference of the page containing the indexes.
-
-Request binary layout:
-
- 8 12 16 17
- +-------+-------+-----+
- | id |reuse|
- +-------+-------+-----+
-
-Response additional fields:
-
-- **id**: echoed back from request
-
-Response binary layout:
-
- 16 20 24
- +-------+-------+
- | id |
- +-------+-------+
-
-Return value:
-
- - 0 on success
- - See the [POSIX shutdown function][shutdown] for error names; see
- [Error numbers] in further sections.
-
-#### Bind
-
-The **bind** operation corresponds to the POSIX [bind][bind] function.
-It assigns the address passed as parameter to a previously created
-socket, identified by **id**. **Bind**, **listen** and **accept** are
-the three operations required to have fully working passive sockets and
-should be issued in that order.
-
-Request fields:
-
-- **cmd** value: 2
-- additional fields:
- - **id**: identifies the socket
- - **addr**: address to connect to, see [Socket families and address
- format]
- - **len**: address length up to 28 octets
-
-Request binary layout:
-
- 8 12 16 20 24 28 32 36 40 44
- +-------+-------+-------+-------+-------+-------+-------+-------+-------+
- | id | addr |
- +-------+-------+-------+-------+-------+-------+-------+-------+-------+
- | len |
- +-------+
-
-Response additional fields:
-
-- **id**: echoed back from request
-
-Response binary layout:
-
- 16 20 24
- +-------+-------+
- | id |
- +-------+-------+
-
-Return value:
-
- - 0 on success
- - See the [POSIX bind function][bind] for error names; see
- [Error numbers] in further sections.
-
-
-#### Listen
-
-The **listen** operation marks the socket as a passive socket. It corresponds to
-the [POSIX listen function][listen].
-
-Reuqest fields:
-
-- **cmd** value: 3
-- additional fields:
- - **id**: identifies the socket
- - **backlog**: the maximum length to which the queue of pending
- connections may grow in number of elements
-
-Request binary layout:
-
- 8 12 16 20
- +-------+-------+-------+
- | id |backlog|
- +-------+-------+-------+
-
-Response additional fields:
-
-- **id**: echoed back from request
-
-Response binary layout:
-
- 16 20 24
- +-------+-------+
- | id |
- +-------+-------+
-
-Return value:
- - 0 on success
- - See the [POSIX listen function][listen] for error names; see
- [Error numbers] in further sections.
-
-
-#### Accept
-
-The **accept** operation extracts the first connection request on the
-queue of pending connections for the listening socket identified by
-**id** and creates a new connected socket. The id of the new socket is
-also chosen by the frontend and passed as an additional field of the
-accept request struct (**id_new**). See the [POSIX accept function][accept]
-as reference.
-
-Similarly to the **connect** operation, **accept** creates new [Indexes
-Page and Data ring]. The **data ring** is used to send and receive data from
-the socket. The **accept** operation passes two additional parameters:
-**evtchn** and **ref**. **evtchn** is the port number of a new event
-channel which will be used for notifications of activity on the data
-ring. **ref** is the grant reference of the **indexes page**: a page
-which contains shared indexes that point to the write and read locations
-in the **data ring**. The **indexes page** also contains the full array of
-grant references for the **data ring**.
-
-The backend will reply to the request only when a new connection is
-successfully accepted, i.e. the backend does not return EAGAIN or
-EWOULDBLOCK.
-
-Example workflow:
-
-- frontend issues an **accept** request
-- backend waits for a connection to be available on the socket
-- a new connection becomes available
-- backend accepts the new connection
-- backend creates an internal mapping from **id_new** to the new socket
-- backend maps the grant reference **ref**, the indexes page, see struct
- pvcalls_data_intf
-- backend maps all the grant references listed in `struct
- pvcalls_data_intf` and uses them as shared memory for the new data
- ring **in** and **out** arrays
-- backend binds to the **evtchn**
-- backend replies to the frontend with a **ret** value
-
-Request fields:
-
-- **cmd** value: 4
-- additional fields:
- - **id**: id of listening socket
- - **id_new**: id of the new socket
- - **ref**: grant reference of the indexes page
- - **evtchn**: port number of the evtchn to signal activity on the data ring
-
-Request binary layout:
-
- 8 12 16 20 24 28 32
- +-------+-------+-------+-------+-------+-------+
- | id | id_new | ref |evtchn |
- +-------+-------+-------+-------+-------+-------+
-
-Response additional fields:
-
-- **id**: id of the listening socket, echoed back from request
-
-Response binary layout:
-
- 16 20 24
- +-------+-------+
- | id |
- +-------+-------+
-
-Return value:
-
- - 0 on success
- - See the [POSIX accept function][accept] for error names; see
- [Error numbers] in further sections.
-
-
-#### Poll
-
-In this version of the protocol, the **poll** operation is only valid
-for passive sockets. For active sockets, the frontend should look at the
-indexes on the **indexes page**. When a new connection is available in
-the queue of the passive socket, the backend generates a response and
-notifies the frontend.
-
-Request fields:
-
-- **cmd** value: 5
-- additional fields:
- - **id**: identifies the listening socket
-
-Request binary layout:
-
- 8 12 16
- +-------+-------+
- | id |
- +-------+-------+
-
-
-Response additional fields:
-
-- **id**: echoed back from request
-
-Response binary layout:
-
- 16 20 24
- +--------+--------+
- | id |
- +--------+--------+
-
-Return value:
-
- - 0 on success
- - See the [POSIX poll function][poll] for error names; see
- [Error numbers] in further sections.
-
-#### Expanding the protocol
-
-It is possible to introduce new commands without changing the protocol
-ABI. Naturally, a feature flag among the backend xenstore nodes should
-advertise the availability of a new set of commands.
-
-If a new command requires parameters in struct xen_pvcalls_request
-larger than 56 bytes, which is the current size of the request, then the
-protocol version should be increased. One way to implement the large
-request structure without disrupting the current ABI is to introduce a
-new command, such as PVCALLS_CONNECT_EXTENDED, and a flag to specify
-that the request uses two request slots, for a total of 112 bytes.
-
-#### Error numbers
-
-The numbers corresponding to the error names specified by POSIX are:
-
- [EPERM] -1
- [ENOENT] -2
- [ESRCH] -3
- [EINTR] -4
- [EIO] -5
- [ENXIO] -6
- [E2BIG] -7
- [ENOEXEC] -8
- [EBADF] -9
- [ECHILD] -10
- [EAGAIN] -11
- [EWOULDBLOCK] -11
- [ENOMEM] -12
- [EACCES] -13
- [EFAULT] -14
- [EBUSY] -16
- [EEXIST] -17
- [EXDEV] -18
- [ENODEV] -19
- [EISDIR] -21
- [EINVAL] -22
- [ENFILE] -23
- [EMFILE] -24
- [ENOSPC] -28
- [EROFS] -30
- [EMLINK] -31
- [EDOM] -33
- [ERANGE] -34
- [EDEADLK] -35
- [EDEADLOCK] -35
- [ENAMETOOLONG] -36
- [ENOLCK] -37
- [ENOTEMPTY] -39
- [ENOSYS] -38
- [ENODATA] -61
- [ETIME] -62
- [EBADMSG] -74
- [EOVERFLOW] -75
- [EILSEQ] -84
- [ERESTART] -85
- [ENOTSOCK] -88
- [EOPNOTSUPP] -95
- [EAFNOSUPPORT] -97
- [EADDRINUSE] -98
- [EADDRNOTAVAIL] -99
- [ENOBUFS] -105
- [EISCONN] -106
- [ENOTCONN] -107
- [ETIMEDOUT] -110
- [ENOTSUP] -524
-
-#### Socket families and address format
-
-The following definitions and explicit sizes, together with POSIX
-[sys/socket.h][address] and [netinet/in.h][in] define socket families and
-address format. Please be aware that only the **domain** `AF_INET`, **type**
-`SOCK_STREAM` and **protocol** `0` are supported by this version of the
-specification, others return ENOTSUP.
-
- #define AF_UNSPEC 0
- #define AF_UNIX 1 /* Unix domain sockets */
- #define AF_LOCAL 1 /* POSIX name for AF_UNIX */
- #define AF_INET 2 /* Internet IP Protocol */
- #define AF_INET6 10 /* IP version 6 */
-
- #define SOCK_STREAM 1
- #define SOCK_DGRAM 2
- #define SOCK_RAW 3
-
- /* generic address format */
- struct sockaddr {
- uint16_t sa_family_t;
- char sa_data[26];
- };
-
- struct in_addr {
- uint32_t s_addr;
- };
-
- /* AF_INET address format */
- struct sockaddr_in {
- uint16_t sa_family_t;
- uint16_t sin_port;
- struct in_addr sin_addr;
- char sin_zero[20];
- };
-
-
-### Indexes Page and Data ring
-
-Data rings are used for sending and receiving data over a connected socket. They
-are created upon a successful **accept** or **connect** command.
-The **sendmsg** and **recvmsg** calls are implemented by sending data and
-receiving data from a data ring, and updating the corresponding indexes
-on the **indexes page**.
-
-Firstly, the **indexes page** is shared by a **connect** or **accept**
-command, see **ref** parameter in their sections. The content of the
-**indexes page** is represented by `struct pvcalls_ring_intf`, see
-below. The structure contains the list of grant references which
-constitute the **in** and **out** buffers of the data ring, see ref[]
-below. The backend maps the grant references contiguously. Of the
-resulting shared memory, the first half is dedicated to the **in** array
-and the second half to the **out** array. They are used as circular
-buffers for transferring data, and, together, they are the data ring.
-
-
- +---------------------------+ Indexes page
- | Command ring: | +----------------------+
- | @0: xen_pvcalls_connect: | |@0 pvcalls_data_intf: |
- | @44: ref +-------------------------------->+@76: ring_order = 1 |
- | | |@80: ref[0]+ |
- +---------------------------+ |@84: ref[1]+ |
- | | |
- | | |
- +----------------------+
- |
- v (data ring)
- +-------+-----------+
- | @0->4098: in |
- | ref[0] |
- |-------------------|
- | @4099->8196: out |
- | ref[1] |
- +-------------------+
-
-
-#### Indexes Page Structure
-
- typedef uint32_t PVCALLS_RING_IDX;
-
- struct pvcalls_data_intf {
- PVCALLS_RING_IDX in_cons, in_prod;
- int32_t in_error;
-
- uint8_t pad[52];
-
- PVCALLS_RING_IDX out_cons, out_prod;
- int32_t out_error;
-
- uint8_t pad[52];
-
- uint32_t ring_order;
- grant_ref_t ref[];
- };
-
- /* not actually C compliant (ring_order changes from socket to socket) */
- struct pvcalls_data {
- char in[((1<<ring_order)<<PAGE_SHIFT)/2];
- char out[((1<<ring_order)<<PAGE_SHIFT)/2];
- };
-
-- **ring_order**
- It represents the order of the data ring. The following list of grant
- references is of `(1 << ring_order)` elements. It cannot be greater than
- **max-page-order**, as specified by the backend on XenBus. It has to
- be one at minimum.
-- **ref[]**
- The list of grant references which will contain the actual data. They are
- mapped contiguosly in virtual memory. The first half of the pages is the
- **in** array, the second half is the **out** array. The arrays must
- have a power of two size. Together, their size is `(1 << ring_order) *
- PAGE_SIZE`.
-- **in** is an array used as circular buffer
- It contains data read from the socket. The producer is the backend, the
- consumer is the frontend.
-- **out** is an array used as circular buffer
- It contains data to be written to the socket. The producer is the frontend,
- the consumer is the backend.
-- **in_cons** and **in_prod**
- Consumer and producer indexes for data read from the socket. They keep track
- of how much data has already been consumed by the frontend from the **in**
- array. **in_prod** is increased by the backend, after writing data to **in**.
- **in_cons** is increased by the frontend, after reading data from **in**.
-- **out_cons**, **out_prod**
- Consumer and producer indexes for the data to be written to the socket. They
- keep track of how much data has been written by the frontend to **out** and
- how much data has already been consumed by the backend. **out_prod** is
- increased by the frontend, after writing data to **out**. **out_cons** is
- increased by the backend, after reading data from **out**.
-- **in_error** and **out_error** They signal errors when reading from the socket
- (**in_error**) or when writing to the socket (**out_error**). 0 means no
- errors. When an error occurs, no further reads or writes operations are
- performed on the socket. In the case of an orderly socket shutdown (i.e. read
- returns 0) **in_error** is set to ENOTCONN. **in_error** and **out_error**
- are never set to EAGAIN or EWOULDBLOCK (the data is written to the
- ring as soon as it is available).
-
-The binary layout of `struct pvcalls_data_intf` follows:
-
- 0 4 8 12 64 68 72 76
- +---------+---------+---------+-----//-----+---------+---------+---------+
- | in_cons | in_prod |in_error | padding |out_cons |out_prod |out_error|
- +---------+---------+---------+-----//-----+---------+---------+---------+
-
- 76 80 84 88 4092 4096
- +---------+---------+---------+----//---+---------+
- |ring_orde| ref[0] | ref[1] | | ref[N] |
- +---------+---------+---------+----//---+---------+
-
-**N.B** For one page, N is maximum 991 ((4096-132)/4), but given that N needs
-to be a power of two, actually max N is 512 (ring_order = 9).
-
-#### Data Ring Structure
-
-The binary layout of the data ring follow:
-
- 0 ((1<<ring_order)<<PAGE_SHIFT)/2 ((1<<ring_order)<<PAGE_SHIFT)
- +------------//-------------+------------//-------------+
- | in | out |
- +------------//-------------+------------//-------------+
-
-#### Why ring.h is not needed
-
-Many Xen PV protocols use the macros provided by [ring.h] to manage
-their shared ring for communication. PVCalls does not, because the [Data
-Ring Structure] actually comes with two rings: the **in** ring and the
-**out** ring. Each of them is mono-directional, and there is no static
-request size: the producer writes opaque data to the ring. On the other
-end, in [ring.h] they are combined, and the request size is static and
-well-known. In PVCalls:
-
- in -> backend to frontend only
- out-> frontend to backend only
-
-In the case of the **in** ring, the frontend is the consumer, and the
-backend is the producer. Everything is the same but mirrored for the
-**out** ring.
-
-The producer, the backend in this case, never reads from the **in**
-ring. In fact, the producer doesn't need any notifications unless the
-ring is full. This version of the protocol doesn't take advantage of it,
-leaving room for optimizations.
-
-On the other end, the consumer always requires notifications, unless it
-is already actively reading from the ring. The producer can figure it
-out, without any additional fields in the protocol, by comparing the
-indexes at the beginning and the end of the function. This is similar to
-what [ring.h] does.
-
-#### Workflow
-
-The **in** and **out** arrays are used as circular buffers:
-
- 0 sizeof(array) == ((1<<ring_order)<<PAGE_SHIFT)/2
- +-----------------------------------+
- |to consume| free |to consume |
- +-----------------------------------+
- ^ ^
- prod cons
-
- 0 sizeof(array)
- +-----------------------------------+
- | free | to consume | free |
- +-----------------------------------+
- ^ ^
- cons prod
-
-The following function is provided to calculate how many bytes are currently
-left unconsumed in an array:
-
- #define _MASK_PVCALLS_IDX(idx, ring_size) ((idx) & (ring_size-1))
-
- static inline PVCALLS_RING_IDX pvcalls_ring_unconsumed(PVCALLS_RING_IDX prod,
- PVCALLS_RING_IDX cons,
- PVCALLS_RING_IDX ring_size)
- {
- PVCALLS_RING_IDX size;
-
- if (prod == cons)
- return 0;
-
- prod = _MASK_PVCALLS_IDX(prod, ring_size);
- cons = _MASK_PVCALLS_IDX(cons, ring_size);
-
- if (prod == cons)
- return ring_size;
-
- if (prod > cons)
- size = prod - cons;
- else {
- size = ring_size - cons;
- size += prod;
- }
- return size;
- }
-
-The producer (the backend for **in**, the frontend for **out**) writes to the
-array in the following way:
-
-- read *[in|out]_cons*, *[in|out]_prod*, *[in|out]_error* from shared memory
-- general memory barrier
-- return on *[in|out]_error*
-- write to array at position *[in|out]_prod* up to *[in|out]_cons*,
- wrapping around the circular buffer when necessary
-- write memory barrier
-- increase *[in|out]_prod*
-- notify the other end via evtchn
-
-The consumer (the backend for **out**, the frontend for **in**) reads from the
-array in the following way:
-
-- read *[in|out]_prod*, *[in|out]_cons*, *[in|out]_error* from shared memory
-- read memory barrier
-- return on *[in|out]_error*
-- read from array at position *[in|out]_cons* up to *[in|out]_prod*,
- wrapping around the circular buffer when necessary
-- general memory barrier
-- increase *[in|out]_cons*
-- notify the other end via evtchn
-
-The producer takes care of writing only as many bytes as available in
-the buffer up to *[in|out]_cons*. The consumer takes care of reading
-only as many bytes as available in the buffer up to *[in|out]_prod*.
-*[in|out]_error* is set by the backend when an error occurs writing or
-reading from the socket.
-
-
-[xenstore]: http://xenbits.xen.org/docs/unstable/misc/xenstore.txt
-[XenbusStateInitialising]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,xenbus.h.html
-[address]: http://pubs.opengroup.org/onlinepubs/7908799/xns/syssocket.h.html
-[in]: http://pubs.opengroup.org/onlinepubs/000095399/basedefs/netinet/in.h.html
-[socket]: http://pubs.opengroup.org/onlinepubs/009695399/functions/socket.html
-[connect]: http://pubs.opengroup.org/onlinepubs/7908799/xns/connect.html
-[shutdown]: http://pubs.opengroup.org/onlinepubs/7908799/xns/shutdown.html
-[bind]: http://pubs.opengroup.org/onlinepubs/7908799/xns/bind.html
-[listen]: http://pubs.opengroup.org/onlinepubs/7908799/xns/listen.html
-[accept]: http://pubs.opengroup.org/onlinepubs/7908799/xns/accept.html
-[poll]: http://pubs.opengroup.org/onlinepubs/7908799/xsh/poll.html
-[ring.h]: http://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=xen/include/public/io/ring.h;hb=HEAD
--- /dev/null
+# PV Calls Protocol version 1
+
+## Glossary
+
+The following is a list of terms and definitions used in the Xen
+community. If you are a Xen contributor you can skip this section.
+
+* PV
+
+ Short for paravirtualized.
+
+* Dom0
+
+ First virtual machine that boots. In most configurations Dom0 is
+ privileged and has control over hardware devices, such as network
+ cards, graphic cards, etc.
+
+* DomU
+
+ Regular unprivileged Xen virtual machine.
+
+* Domain
+
+ A Xen virtual machine. Dom0 and all DomUs are all separate Xen
+ domains.
+
+* Guest
+
+ Same as domain: a Xen virtual machine.
+
+* Frontend
+
+ Each DomU has one or more paravirtualized frontend drivers to access
+ disks, network, console, graphics, etc. The presence of PV devices is
+ advertized on XenStore, a cross domain key-value database. Frontends
+ are similar in intent to the virtio drivers in Linux.
+
+* Backend
+
+ A Xen paravirtualized backend typically runs in Dom0 and it is used to
+ export disks, network, console, graphics, etcs, to DomUs. Backends can
+ live both in kernel space and in userspace. For example xen-blkback
+ lives under drivers/block in the Linux kernel and xen_disk lives under
+ hw/block in QEMU. Paravirtualized backends are similar in intent to
+ virtio device emulators.
+
+* VMX and SVM
+
+ On Intel processors, VMX is the CPU flag for VT-x, hardware
+ virtualization support. It corresponds to SVM on AMD processors.
+
+
+
+## Rationale
+
+PV Calls is a paravirtualized protocol that allows the implementation of
+a set of POSIX functions in a different domain. The PV Calls frontend
+sends POSIX function calls to the backend, which implements them and
+returns a value to the frontend and acts on the function call.
+
+This version of the document covers networking function calls, such as
+connect, accept, bind, release, listen, poll, recvmsg and sendmsg; but
+the protocol is meant to be easily extended to cover different sets of
+calls. Unimplemented commands return ENOTSUP.
+
+PV Calls provide the following benefits:
+* full visibility of the guest behavior on the backend domain, allowing
+ for inexpensive filtering and manipulation of any guest calls
+* excellent performance
+
+Specifically, PV Calls for networking offer these advantages:
+* guest networking works out of the box with VPNs, wireless networks and
+ any other complex configurations on the host
+* guest services listen on ports bound directly to the backend domain IP
+ addresses
+* localhost becomes a secure host wide network for inter-VMs
+ communications
+
+
+## Design
+
+### Why Xen?
+
+PV Calls are part of an effort to create a secure runtime environment
+for containers (Open Containers Initiative images to be precise). PV
+Calls are based on Xen, although porting them to other hypervisors is
+possible. Xen was chosen because of its security and isolation
+properties and because it supports PV guests, a type of virtual machines
+that does not require hardware virtualization extensions (VMX on Intel
+processors and SVM on AMD processors). This is important because PV
+Calls is meant for containers and containers are often run on top of
+public cloud instances, which do not support nested VMX (or SVM) as of
+today (early 2017). Xen PV guests are lightweight, minimalist, and do
+not require machine emulation: all properties that make them a good fit
+for this project.
+
+### Xenstore
+
+The frontend and the backend connect via [xenstore] to
+exchange information. The toolstack creates front and back nodes with
+state of [XenbusStateInitialising]. The protocol node name
+is **pvcalls**. There can only be one PV Calls frontend per domain.
+
+#### Frontend XenBus Nodes
+
+version
+ Values: <string>
+
+ Protocol version, chosen among the ones supported by the backend
+ (see **versions** under [Backend XenBus Nodes]). Currently the
+ value must be "1".
+
+port
+ Values: <uint32_t>
+
+ The identifier of the Xen event channel used to signal activity
+ in the command ring.
+
+ring-ref
+ Values: <uint32_t>
+
+ The Xen grant reference granting permission for the backend to map
+ the sole page in a single page sized command ring.
+
+#### Backend XenBus Nodes
+
+versions
+ Values: <string>
+
+ List of comma separated protocol versions supported by the backend.
+ For example "1,2,3". Currently the value is just "1", as there is
+ only one version.
+
+max-page-order
+ Values: <uint32_t>
+
+ The maximum supported size of a memory allocation in units of
+ log2n(machine pages), e.g. 1 = 2 pages, 2 == 4 pages, etc. It must
+ be 1 or more.
+
+function-calls
+ Values: <uint32_t>
+
+ Value "0" means that no calls are supported.
+ Value "1" means that socket, connect, release, bind, listen, accept
+ and poll are supported.
+
+#### State Machine
+
+Initialization:
+
+ *Front* *Back*
+ XenbusStateInitialising XenbusStateInitialising
+ - Query virtual device - Query backend device
+ properties. identification data.
+ - Setup OS device instance. - Publish backend features
+ - Allocate and initialize the and transport parameters
+ request ring. |
+ - Publish transport parameters |
+ that will be in effect during V
+ this connection. XenbusStateInitWait
+ |
+ |
+ V
+ XenbusStateInitialised
+
+ - Query frontend transport parameters.
+ - Connect to the request ring and
+ event channel.
+ |
+ |
+ V
+ XenbusStateConnected
+
+ - Query backend device properties.
+ - Finalize OS virtual device
+ instance.
+ |
+ |
+ V
+ XenbusStateConnected
+
+Once frontend and backend are connected, they have a shared page, which
+will is used to exchange messages over a ring, and an event channel,
+which is used to send notifications.
+
+Shutdown:
+
+ *Front* *Back*
+ XenbusStateConnected XenbusStateConnected
+ |
+ |
+ V
+ XenbusStateClosing
+
+ - Unmap grants
+ - Unbind event channels
+ |
+ |
+ V
+ XenbusStateClosing
+
+ - Unbind event channels
+ - Free rings
+ - Free data structures
+ |
+ |
+ V
+ XenbusStateClosed
+
+ - Free remaining data structures
+ |
+ |
+ V
+ XenbusStateClosed
+
+
+### Commands Ring
+
+The shared ring is used by the frontend to forward POSIX function calls
+to the backend. We shall refer to this ring as **commands ring** to
+distinguish it from other rings which can be created later in the
+lifecycle of the protocol (see [Indexes Page and Data ring]). The grant
+reference for shared page for this ring is shared on xenstore (see
+[Frontend XenBus Nodes]). The ring format is defined using the familiar
+`DEFINE_RING_TYPES` macro (`xen/include/public/io/ring.h`). Frontend
+requests are allocated on the ring using the `RING_GET_REQUEST` macro.
+The list of commands below is in calling order.
+
+The format is defined as follows:
+
+ #define PVCALLS_SOCKET 0
+ #define PVCALLS_CONNECT 1
+ #define PVCALLS_RELEASE 2
+ #define PVCALLS_BIND 3
+ #define PVCALLS_LISTEN 4
+ #define PVCALLS_ACCEPT 5
+ #define PVCALLS_POLL 6
+
+ struct xen_pvcalls_request {
+ uint32_t req_id; /* private to guest, echoed in response */
+ uint32_t cmd; /* command to execute */
+ union {
+ struct xen_pvcalls_socket {
+ uint64_t id;
+ uint32_t domain;
+ uint32_t type;
+ uint32_t protocol;
+ #ifdef CONFIG_X86_32
+ uint8_t pad[4];
+ #endif
+ } socket;
+ struct xen_pvcalls_connect {
+ uint64_t id;
+ uint8_t addr[28];
+ uint32_t len;
+ uint32_t flags;
+ grant_ref_t ref;
+ uint32_t evtchn;
+ #ifdef CONFIG_X86_32
+ uint8_t pad[4];
+ #endif
+ } connect;
+ struct xen_pvcalls_release {
+ uint64_t id;
+ uint8_t reuse;
+ #ifdef CONFIG_X86_32
+ uint8_t pad[7];
+ #endif
+ } release;
+ struct xen_pvcalls_bind {
+ uint64_t id;
+ uint8_t addr[28];
+ uint32_t len;
+ } bind;
+ struct xen_pvcalls_listen {
+ uint64_t id;
+ uint32_t backlog;
+ #ifdef CONFIG_X86_32
+ uint8_t pad[4];
+ #endif
+ } listen;
+ struct xen_pvcalls_accept {
+ uint64_t id;
+ uint64_t id_new;
+ grant_ref_t ref;
+ uint32_t evtchn;
+ } accept;
+ struct xen_pvcalls_poll {
+ uint64_t id;
+ } poll;
+ /* dummy member to force sizeof(struct xen_pvcalls_request) to match across archs */
+ struct xen_pvcalls_dummy {
+ uint8_t dummy[56];
+ } dummy;
+ } u;
+ };
+
+The first two fields are common for every command. Their binary layout
+is:
+
+ 0 4 8
+ +-------+-------+
+ |req_id | cmd |
+ +-------+-------+
+
+- **req_id** is generated by the frontend and is a cookie used to
+ identify one specific request/response pair of commands. Not to be
+ confused with any command **id** which are used to identify a socket
+ across multiple commands, see [Socket].
+- **cmd** is the command requested by the frontend:
+
+ - `PVCALLS_SOCKET`: 0
+ - `PVCALLS_CONNECT`: 1
+ - `PVCALLS_RELEASE`: 2
+ - `PVCALLS_BIND`: 3
+ - `PVCALLS_LISTEN`: 4
+ - `PVCALLS_ACCEPT`: 5
+ - `PVCALLS_POLL`: 6
+
+Both fields are echoed back by the backend. See [Socket families and
+address format] for the format of the **addr** field of connect and
+bind. The maximum size of command specific arguments is 56 bytes. Any
+future command that requires more than that will need a bump the
+**version** of the protocol.
+
+Similarly to other Xen ring based protocols, after writing a request to
+the ring, the frontend calls `RING_PUSH_REQUESTS_AND_CHECK_NOTIFY` and
+issues an event channel notification when a notification is required.
+
+Backend responses are allocated on the ring using the `RING_GET_RESPONSE` macro.
+The format is the following:
+
+ struct xen_pvcalls_response {
+ uint32_t req_id;
+ uint32_t cmd;
+ int32_t ret;
+ uint32_t pad;
+ union {
+ struct _xen_pvcalls_socket {
+ uint64_t id;
+ } socket;
+ struct _xen_pvcalls_connect {
+ uint64_t id;
+ } connect;
+ struct _xen_pvcalls_release {
+ uint64_t id;
+ } release;
+ struct _xen_pvcalls_bind {
+ uint64_t id;
+ } bind;
+ struct _xen_pvcalls_listen {
+ uint64_t id;
+ } listen;
+ struct _xen_pvcalls_accept {
+ uint64_t id;
+ } accept;
+ struct _xen_pvcalls_poll {
+ uint64_t id;
+ } poll;
+ struct _xen_pvcalls_dummy {
+ uint8_t dummy[8];
+ } dummy;
+ } u;
+ };
+
+The first four fields are common for every response. Their binary layout
+is:
+
+ 0 4 8 12 16
+ +-------+-------+-------+-------+
+ |req_id | cmd | ret | pad |
+ +-------+-------+-------+-------+
+
+- **req_id**: echoed back from request
+- **cmd**: echoed back from request
+- **ret**: return value, identifies success (0) or failure (see [Error
+ numbers] in further sections). If the **cmd** is not supported by the
+ backend, ret is ENOTSUP.
+- **pad**: padding
+
+After calling `RING_PUSH_RESPONSES_AND_CHECK_NOTIFY`, the backend checks whether
+it needs to notify the frontend and does so via event channel.
+
+A description of each command, their additional request and response
+fields follow.
+
+
+#### Socket
+
+The **socket** operation corresponds to the POSIX [socket][socket]
+function. It creates a new socket of the specified family, type and
+protocol. **id** is freely chosen by the frontend and references this
+specific socket from this point forward. See [Socket families and
+address format] to see which ones are supported by different versions of
+the protocol.
+
+Request fields:
+
+- **cmd** value: 0
+- additional fields:
+ - **id**: generated by the frontend, it identifies the new socket
+ - **domain**: the communication domain
+ - **type**: the socket type
+ - **protocol**: the particular protocol to be used with the socket, usually 0
+
+Request binary layout:
+
+ 8 12 16 20 24 28
+ +-------+-------+-------+-------+-------+
+ | id |domain | type |protoco|
+ +-------+-------+-------+-------+-------+
+
+Response additional fields:
+
+- **id**: echoed back from request
+
+Response binary layout:
+
+ 16 20 24
+ +-------+--------+
+ | id |
+ +-------+--------+
+
+Return value:
+
+ - 0 on success
+ - See the [POSIX socket function][connect] for error names; see
+ [Error numbers] in further sections.
+
+#### Connect
+
+The **connect** operation corresponds to the POSIX [connect][connect]
+function. It connects a previously created socket (identified by **id**)
+to the specified address.
+
+The connect operation creates a new shared ring, which we'll call **data
+ring**. The data ring is used to send and receive data from the
+socket. The connect operation passes two additional parameters:
+**evtchn** and **ref**. **evtchn** is the port number of a new event
+channel which will be used for notifications of activity on the data
+ring. **ref** is the grant reference of the **indexes page**: a page
+which contains shared indexes that point to the write and read locations
+in the **data ring**. The **indexes page** also contains the full array
+of grant references for the **data ring**. When the frontend issues a
+**connect** command, the backend:
+
+- finds its own internal socket corresponding to **id**
+- connects the socket to **addr**
+- maps the grant reference **ref**, the indexes page, see struct
+ pvcalls_data_intf
+- maps all the grant references listed in `struct pvcalls_data_intf` and
+ uses them as shared memory for the **data ring**
+- bind the **evtchn**
+- replies to the frontend
+
+The [Indexes Page and Data ring] format will be described in the
+following section. The **data ring** is unmapped and freed upon issuing
+a **release** command on the active socket identified by **id**. A
+frontend state change can also cause data rings to be unmapped.
+
+Request fields:
+
+- **cmd** value: 0
+- additional fields:
+ - **id**: identifies the socket
+ - **addr**: address to connect to, see [Socket families and address format]
+ - **len**: address length up to 28 octets
+ - **flags**: flags for the connection, reserved for future usage
+ - **ref**: grant reference of the indexes page
+ - **evtchn**: port number of the evtchn to signal activity on the **data ring**
+
+Request binary layout:
+
+ 8 12 16 20 24 28 32 36 40 44
+ +-------+-------+-------+-------+-------+-------+-------+-------+-------+
+ | id | addr |
+ +-------+-------+-------+-------+-------+-------+-------+-------+-------+
+ | len | flags | ref |evtchn |
+ +-------+-------+-------+-------+
+
+Response additional fields:
+
+- **id**: echoed back from request
+
+Response binary layout:
+
+ 16 20 24
+ +-------+-------+
+ | id |
+ +-------+-------+
+
+Return value:
+
+ - 0 on success
+ - See the [POSIX connect function][connect] for error names; see
+ [Error numbers] in further sections.
+
+#### Release
+
+The **release** operation closes an existing active or a passive socket.
+
+When a release command is issued on a passive socket, the backend
+releases it and frees its internal mappings. When a release command is
+issued for an active socket, the data ring and indexes page are also
+unmapped and freed:
+
+- frontend sends release command for an active socket
+- backend releases the socket
+- backend unmaps the data ring
+- backend unmaps the indexes page
+- backend unbinds the event channel
+- backend replies to frontend with an **ret** value
+- frontend frees data ring, indexes page and unbinds event channel
+
+Request fields:
+
+- **cmd** value: 1
+- additional fields:
+ - **id**: identifies the socket
+ - **reuse**: an optimization hint for the backend. The field is
+ ignored for passive sockets. When set to 1, the frontend lets the
+ backend know that it will reuse exactly the same set of grant pages
+ (indexes page and data ring) and event channel when creating one of
+ the next active sockets. The backend can take advantage of it by
+ delaying unmapping grants and unbinding the event channel. The
+ backend is free to ignore the hint. Reused data rings are found by
+ **ref**, the grant reference of the page containing the indexes.
+
+Request binary layout:
+
+ 8 12 16 17
+ +-------+-------+-----+
+ | id |reuse|
+ +-------+-------+-----+
+
+Response additional fields:
+
+- **id**: echoed back from request
+
+Response binary layout:
+
+ 16 20 24
+ +-------+-------+
+ | id |
+ +-------+-------+
+
+Return value:
+
+ - 0 on success
+ - See the [POSIX shutdown function][shutdown] for error names; see
+ [Error numbers] in further sections.
+
+#### Bind
+
+The **bind** operation corresponds to the POSIX [bind][bind] function.
+It assigns the address passed as parameter to a previously created
+socket, identified by **id**. **Bind**, **listen** and **accept** are
+the three operations required to have fully working passive sockets and
+should be issued in that order.
+
+Request fields:
+
+- **cmd** value: 2
+- additional fields:
+ - **id**: identifies the socket
+ - **addr**: address to connect to, see [Socket families and address
+ format]
+ - **len**: address length up to 28 octets
+
+Request binary layout:
+
+ 8 12 16 20 24 28 32 36 40 44
+ +-------+-------+-------+-------+-------+-------+-------+-------+-------+
+ | id | addr |
+ +-------+-------+-------+-------+-------+-------+-------+-------+-------+
+ | len |
+ +-------+
+
+Response additional fields:
+
+- **id**: echoed back from request
+
+Response binary layout:
+
+ 16 20 24
+ +-------+-------+
+ | id |
+ +-------+-------+
+
+Return value:
+
+ - 0 on success
+ - See the [POSIX bind function][bind] for error names; see
+ [Error numbers] in further sections.
+
+
+#### Listen
+
+The **listen** operation marks the socket as a passive socket. It corresponds to
+the [POSIX listen function][listen].
+
+Reuqest fields:
+
+- **cmd** value: 3
+- additional fields:
+ - **id**: identifies the socket
+ - **backlog**: the maximum length to which the queue of pending
+ connections may grow in number of elements
+
+Request binary layout:
+
+ 8 12 16 20
+ +-------+-------+-------+
+ | id |backlog|
+ +-------+-------+-------+
+
+Response additional fields:
+
+- **id**: echoed back from request
+
+Response binary layout:
+
+ 16 20 24
+ +-------+-------+
+ | id |
+ +-------+-------+
+
+Return value:
+ - 0 on success
+ - See the [POSIX listen function][listen] for error names; see
+ [Error numbers] in further sections.
+
+
+#### Accept
+
+The **accept** operation extracts the first connection request on the
+queue of pending connections for the listening socket identified by
+**id** and creates a new connected socket. The id of the new socket is
+also chosen by the frontend and passed as an additional field of the
+accept request struct (**id_new**). See the [POSIX accept function][accept]
+as reference.
+
+Similarly to the **connect** operation, **accept** creates new [Indexes
+Page and Data ring]. The **data ring** is used to send and receive data from
+the socket. The **accept** operation passes two additional parameters:
+**evtchn** and **ref**. **evtchn** is the port number of a new event
+channel which will be used for notifications of activity on the data
+ring. **ref** is the grant reference of the **indexes page**: a page
+which contains shared indexes that point to the write and read locations
+in the **data ring**. The **indexes page** also contains the full array of
+grant references for the **data ring**.
+
+The backend will reply to the request only when a new connection is
+successfully accepted, i.e. the backend does not return EAGAIN or
+EWOULDBLOCK.
+
+Example workflow:
+
+- frontend issues an **accept** request
+- backend waits for a connection to be available on the socket
+- a new connection becomes available
+- backend accepts the new connection
+- backend creates an internal mapping from **id_new** to the new socket
+- backend maps the grant reference **ref**, the indexes page, see struct
+ pvcalls_data_intf
+- backend maps all the grant references listed in `struct
+ pvcalls_data_intf` and uses them as shared memory for the new data
+ ring **in** and **out** arrays
+- backend binds to the **evtchn**
+- backend replies to the frontend with a **ret** value
+
+Request fields:
+
+- **cmd** value: 4
+- additional fields:
+ - **id**: id of listening socket
+ - **id_new**: id of the new socket
+ - **ref**: grant reference of the indexes page
+ - **evtchn**: port number of the evtchn to signal activity on the data ring
+
+Request binary layout:
+
+ 8 12 16 20 24 28 32
+ +-------+-------+-------+-------+-------+-------+
+ | id | id_new | ref |evtchn |
+ +-------+-------+-------+-------+-------+-------+
+
+Response additional fields:
+
+- **id**: id of the listening socket, echoed back from request
+
+Response binary layout:
+
+ 16 20 24
+ +-------+-------+
+ | id |
+ +-------+-------+
+
+Return value:
+
+ - 0 on success
+ - See the [POSIX accept function][accept] for error names; see
+ [Error numbers] in further sections.
+
+
+#### Poll
+
+In this version of the protocol, the **poll** operation is only valid
+for passive sockets. For active sockets, the frontend should look at the
+indexes on the **indexes page**. When a new connection is available in
+the queue of the passive socket, the backend generates a response and
+notifies the frontend.
+
+Request fields:
+
+- **cmd** value: 5
+- additional fields:
+ - **id**: identifies the listening socket
+
+Request binary layout:
+
+ 8 12 16
+ +-------+-------+
+ | id |
+ +-------+-------+
+
+
+Response additional fields:
+
+- **id**: echoed back from request
+
+Response binary layout:
+
+ 16 20 24
+ +--------+--------+
+ | id |
+ +--------+--------+
+
+Return value:
+
+ - 0 on success
+ - See the [POSIX poll function][poll] for error names; see
+ [Error numbers] in further sections.
+
+#### Expanding the protocol
+
+It is possible to introduce new commands without changing the protocol
+ABI. Naturally, a feature flag among the backend xenstore nodes should
+advertise the availability of a new set of commands.
+
+If a new command requires parameters in struct xen_pvcalls_request
+larger than 56 bytes, which is the current size of the request, then the
+protocol version should be increased. One way to implement the large
+request structure without disrupting the current ABI is to introduce a
+new command, such as PVCALLS_CONNECT_EXTENDED, and a flag to specify
+that the request uses two request slots, for a total of 112 bytes.
+
+#### Error numbers
+
+The numbers corresponding to the error names specified by POSIX are:
+
+ [EPERM] -1
+ [ENOENT] -2
+ [ESRCH] -3
+ [EINTR] -4
+ [EIO] -5
+ [ENXIO] -6
+ [E2BIG] -7
+ [ENOEXEC] -8
+ [EBADF] -9
+ [ECHILD] -10
+ [EAGAIN] -11
+ [EWOULDBLOCK] -11
+ [ENOMEM] -12
+ [EACCES] -13
+ [EFAULT] -14
+ [EBUSY] -16
+ [EEXIST] -17
+ [EXDEV] -18
+ [ENODEV] -19
+ [EISDIR] -21
+ [EINVAL] -22
+ [ENFILE] -23
+ [EMFILE] -24
+ [ENOSPC] -28
+ [EROFS] -30
+ [EMLINK] -31
+ [EDOM] -33
+ [ERANGE] -34
+ [EDEADLK] -35
+ [EDEADLOCK] -35
+ [ENAMETOOLONG] -36
+ [ENOLCK] -37
+ [ENOTEMPTY] -39
+ [ENOSYS] -38
+ [ENODATA] -61
+ [ETIME] -62
+ [EBADMSG] -74
+ [EOVERFLOW] -75
+ [EILSEQ] -84
+ [ERESTART] -85
+ [ENOTSOCK] -88
+ [EOPNOTSUPP] -95
+ [EAFNOSUPPORT] -97
+ [EADDRINUSE] -98
+ [EADDRNOTAVAIL] -99
+ [ENOBUFS] -105
+ [EISCONN] -106
+ [ENOTCONN] -107
+ [ETIMEDOUT] -110
+ [ENOTSUP] -524
+
+#### Socket families and address format
+
+The following definitions and explicit sizes, together with POSIX
+[sys/socket.h][address] and [netinet/in.h][in] define socket families and
+address format. Please be aware that only the **domain** `AF_INET`, **type**
+`SOCK_STREAM` and **protocol** `0` are supported by this version of the
+specification, others return ENOTSUP.
+
+ #define AF_UNSPEC 0
+ #define AF_UNIX 1 /* Unix domain sockets */
+ #define AF_LOCAL 1 /* POSIX name for AF_UNIX */
+ #define AF_INET 2 /* Internet IP Protocol */
+ #define AF_INET6 10 /* IP version 6 */
+
+ #define SOCK_STREAM 1
+ #define SOCK_DGRAM 2
+ #define SOCK_RAW 3
+
+ /* generic address format */
+ struct sockaddr {
+ uint16_t sa_family_t;
+ char sa_data[26];
+ };
+
+ struct in_addr {
+ uint32_t s_addr;
+ };
+
+ /* AF_INET address format */
+ struct sockaddr_in {
+ uint16_t sa_family_t;
+ uint16_t sin_port;
+ struct in_addr sin_addr;
+ char sin_zero[20];
+ };
+
+
+### Indexes Page and Data ring
+
+Data rings are used for sending and receiving data over a connected socket. They
+are created upon a successful **accept** or **connect** command.
+The **sendmsg** and **recvmsg** calls are implemented by sending data and
+receiving data from a data ring, and updating the corresponding indexes
+on the **indexes page**.
+
+Firstly, the **indexes page** is shared by a **connect** or **accept**
+command, see **ref** parameter in their sections. The content of the
+**indexes page** is represented by `struct pvcalls_ring_intf`, see
+below. The structure contains the list of grant references which
+constitute the **in** and **out** buffers of the data ring, see ref[]
+below. The backend maps the grant references contiguously. Of the
+resulting shared memory, the first half is dedicated to the **in** array
+and the second half to the **out** array. They are used as circular
+buffers for transferring data, and, together, they are the data ring.
+
+
+ +---------------------------+ Indexes page
+ | Command ring: | +----------------------+
+ | @0: xen_pvcalls_connect: | |@0 pvcalls_data_intf: |
+ | @44: ref +-------------------------------->+@76: ring_order = 1 |
+ | | |@80: ref[0]+ |
+ +---------------------------+ |@84: ref[1]+ |
+ | | |
+ | | |
+ +----------------------+
+ |
+ v (data ring)
+ +-------+-----------+
+ | @0->4098: in |
+ | ref[0] |
+ |-------------------|
+ | @4099->8196: out |
+ | ref[1] |
+ +-------------------+
+
+
+#### Indexes Page Structure
+
+ typedef uint32_t PVCALLS_RING_IDX;
+
+ struct pvcalls_data_intf {
+ PVCALLS_RING_IDX in_cons, in_prod;
+ int32_t in_error;
+
+ uint8_t pad[52];
+
+ PVCALLS_RING_IDX out_cons, out_prod;
+ int32_t out_error;
+
+ uint8_t pad[52];
+
+ uint32_t ring_order;
+ grant_ref_t ref[];
+ };
+
+ /* not actually C compliant (ring_order changes from socket to socket) */
+ struct pvcalls_data {
+ char in[((1<<ring_order)<<PAGE_SHIFT)/2];
+ char out[((1<<ring_order)<<PAGE_SHIFT)/2];
+ };
+
+- **ring_order**
+ It represents the order of the data ring. The following list of grant
+ references is of `(1 << ring_order)` elements. It cannot be greater than
+ **max-page-order**, as specified by the backend on XenBus. It has to
+ be one at minimum.
+- **ref[]**
+ The list of grant references which will contain the actual data. They are
+ mapped contiguosly in virtual memory. The first half of the pages is the
+ **in** array, the second half is the **out** array. The arrays must
+ have a power of two size. Together, their size is `(1 << ring_order) *
+ PAGE_SIZE`.
+- **in** is an array used as circular buffer
+ It contains data read from the socket. The producer is the backend, the
+ consumer is the frontend.
+- **out** is an array used as circular buffer
+ It contains data to be written to the socket. The producer is the frontend,
+ the consumer is the backend.
+- **in_cons** and **in_prod**
+ Consumer and producer indexes for data read from the socket. They keep track
+ of how much data has already been consumed by the frontend from the **in**
+ array. **in_prod** is increased by the backend, after writing data to **in**.
+ **in_cons** is increased by the frontend, after reading data from **in**.
+- **out_cons**, **out_prod**
+ Consumer and producer indexes for the data to be written to the socket. They
+ keep track of how much data has been written by the frontend to **out** and
+ how much data has already been consumed by the backend. **out_prod** is
+ increased by the frontend, after writing data to **out**. **out_cons** is
+ increased by the backend, after reading data from **out**.
+- **in_error** and **out_error** They signal errors when reading from the socket
+ (**in_error**) or when writing to the socket (**out_error**). 0 means no
+ errors. When an error occurs, no further reads or writes operations are
+ performed on the socket. In the case of an orderly socket shutdown (i.e. read
+ returns 0) **in_error** is set to ENOTCONN. **in_error** and **out_error**
+ are never set to EAGAIN or EWOULDBLOCK (the data is written to the
+ ring as soon as it is available).
+
+The binary layout of `struct pvcalls_data_intf` follows:
+
+ 0 4 8 12 64 68 72 76
+ +---------+---------+---------+-----//-----+---------+---------+---------+
+ | in_cons | in_prod |in_error | padding |out_cons |out_prod |out_error|
+ +---------+---------+---------+-----//-----+---------+---------+---------+
+
+ 76 80 84 88 4092 4096
+ +---------+---------+---------+----//---+---------+
+ |ring_orde| ref[0] | ref[1] | | ref[N] |
+ +---------+---------+---------+----//---+---------+
+
+**N.B** For one page, N is maximum 991 ((4096-132)/4), but given that N needs
+to be a power of two, actually max N is 512 (ring_order = 9).
+
+#### Data Ring Structure
+
+The binary layout of the data ring follow:
+
+ 0 ((1<<ring_order)<<PAGE_SHIFT)/2 ((1<<ring_order)<<PAGE_SHIFT)
+ +------------//-------------+------------//-------------+
+ | in | out |
+ +------------//-------------+------------//-------------+
+
+#### Why ring.h is not needed
+
+Many Xen PV protocols use the macros provided by [ring.h] to manage
+their shared ring for communication. PVCalls does not, because the [Data
+Ring Structure] actually comes with two rings: the **in** ring and the
+**out** ring. Each of them is mono-directional, and there is no static
+request size: the producer writes opaque data to the ring. On the other
+end, in [ring.h] they are combined, and the request size is static and
+well-known. In PVCalls:
+
+ in -> backend to frontend only
+ out-> frontend to backend only
+
+In the case of the **in** ring, the frontend is the consumer, and the
+backend is the producer. Everything is the same but mirrored for the
+**out** ring.
+
+The producer, the backend in this case, never reads from the **in**
+ring. In fact, the producer doesn't need any notifications unless the
+ring is full. This version of the protocol doesn't take advantage of it,
+leaving room for optimizations.
+
+On the other end, the consumer always requires notifications, unless it
+is already actively reading from the ring. The producer can figure it
+out, without any additional fields in the protocol, by comparing the
+indexes at the beginning and the end of the function. This is similar to
+what [ring.h] does.
+
+#### Workflow
+
+The **in** and **out** arrays are used as circular buffers:
+
+ 0 sizeof(array) == ((1<<ring_order)<<PAGE_SHIFT)/2
+ +-----------------------------------+
+ |to consume| free |to consume |
+ +-----------------------------------+
+ ^ ^
+ prod cons
+
+ 0 sizeof(array)
+ +-----------------------------------+
+ | free | to consume | free |
+ +-----------------------------------+
+ ^ ^
+ cons prod
+
+The following function is provided to calculate how many bytes are currently
+left unconsumed in an array:
+
+ #define _MASK_PVCALLS_IDX(idx, ring_size) ((idx) & (ring_size-1))
+
+ static inline PVCALLS_RING_IDX pvcalls_ring_unconsumed(PVCALLS_RING_IDX prod,
+ PVCALLS_RING_IDX cons,
+ PVCALLS_RING_IDX ring_size)
+ {
+ PVCALLS_RING_IDX size;
+
+ if (prod == cons)
+ return 0;
+
+ prod = _MASK_PVCALLS_IDX(prod, ring_size);
+ cons = _MASK_PVCALLS_IDX(cons, ring_size);
+
+ if (prod == cons)
+ return ring_size;
+
+ if (prod > cons)
+ size = prod - cons;
+ else {
+ size = ring_size - cons;
+ size += prod;
+ }
+ return size;
+ }
+
+The producer (the backend for **in**, the frontend for **out**) writes to the
+array in the following way:
+
+- read *[in|out]_cons*, *[in|out]_prod*, *[in|out]_error* from shared memory
+- general memory barrier
+- return on *[in|out]_error*
+- write to array at position *[in|out]_prod* up to *[in|out]_cons*,
+ wrapping around the circular buffer when necessary
+- write memory barrier
+- increase *[in|out]_prod*
+- notify the other end via evtchn
+
+The consumer (the backend for **out**, the frontend for **in**) reads from the
+array in the following way:
+
+- read *[in|out]_prod*, *[in|out]_cons*, *[in|out]_error* from shared memory
+- read memory barrier
+- return on *[in|out]_error*
+- read from array at position *[in|out]_cons* up to *[in|out]_prod*,
+ wrapping around the circular buffer when necessary
+- general memory barrier
+- increase *[in|out]_cons*
+- notify the other end via evtchn
+
+The producer takes care of writing only as many bytes as available in
+the buffer up to *[in|out]_cons*. The consumer takes care of reading
+only as many bytes as available in the buffer up to *[in|out]_prod*.
+*[in|out]_error* is set by the backend when an error occurs writing or
+reading from the socket.
+
+
+[xenstore]: http://xenbits.xen.org/docs/unstable/misc/xenstore.txt
+[XenbusStateInitialising]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,xenbus.h.html
+[address]: http://pubs.opengroup.org/onlinepubs/7908799/xns/syssocket.h.html
+[in]: http://pubs.opengroup.org/onlinepubs/000095399/basedefs/netinet/in.h.html
+[socket]: http://pubs.opengroup.org/onlinepubs/009695399/functions/socket.html
+[connect]: http://pubs.opengroup.org/onlinepubs/7908799/xns/connect.html
+[shutdown]: http://pubs.opengroup.org/onlinepubs/7908799/xns/shutdown.html
+[bind]: http://pubs.opengroup.org/onlinepubs/7908799/xns/bind.html
+[listen]: http://pubs.opengroup.org/onlinepubs/7908799/xns/listen.html
+[accept]: http://pubs.opengroup.org/onlinepubs/7908799/xns/accept.html
+[poll]: http://pubs.opengroup.org/onlinepubs/7908799/xsh/poll.html
+[ring.h]: http://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=xen/include/public/io/ring.h;hb=HEAD
+++ /dev/null
-# x86/HVM direct boot ABI #
-
-Since the Xen entry point into the kernel can be different from the
-native entry point, a `ELFNOTE` is used in order to tell the domain
-builder how to load and jump into the kernel entry point:
-
- ELFNOTE(Xen, XEN_ELFNOTE_PHYS32_ENTRY, .long, xen_start32)
-
-The presence of the `XEN_ELFNOTE_PHYS32_ENTRY` note indicates that the
-kernel supports the boot ABI described in this document.
-
-The domain builder must load the kernel into the guest memory space and
-jump into the entry point defined at `XEN_ELFNOTE_PHYS32_ENTRY` with the
-following machine state:
-
- * `ebx`: contains the physical memory address where the loader has placed
- the boot start info structure.
-
- * `cr0`: bit 0 (PE) must be set. All the other writeable bits are cleared.
-
- * `cr4`: all bits are cleared.
-
- * `cs`: must be a 32-bit read/execute code segment with a base of ‘0’
- and a limit of ‘0xFFFFFFFF’. The selector value is unspecified.
-
- * `ds`, `es`: must be a 32-bit read/write data segment with a base of
- ‘0’ and a limit of ‘0xFFFFFFFF’. The selector values are all unspecified.
-
- * `tr`: must be a 32-bit TSS (active) with a base of '0' and a limit of '0x67'.
-
- * `eflags`: bit 17 (VM) must be cleared. Bit 9 (IF) must be cleared.
- Bit 8 (TF) must be cleared. Other bits are all unspecified.
-
-All other processor registers and flag bits are unspecified. The OS is in
-charge of setting up it's own stack, GDT and IDT.
-
-The format of the boot start info structure (pointed to by %ebx) can be found
-in xen/include/public/arch-x86/hvm/start\_info.h
-
-Other relevant information needed in order to boot a guest kernel
-(console page address, xenstore event channel...) can be obtained
-using HVMPARAMS, just like it's done on HVM guests.
-
-The setup of the hypercall page is also performed in the same way
-as HVM guests, using the hypervisor cpuid leaves and msr ranges.
-
-## AP startup ##
-
-AP startup can be performed using hypercalls or the local APIC if present.
-The following VCPU hypercalls can be used in order to bring up secondary vCPUs:
-
- * `VCPUOP_initialise` is used to set the initial state of the vCPU. The
- argument passed to the hypercall must be of the type vcpu\_hvm\_context.
- See `public/hvm/hvm_vcpu.h` for the layout of the structure. Note that
- this hypercall allows starting the vCPU in several modes (16/32/64bits),
- regardless of the mode the BSP is currently running on.
-
- * `VCPUOP_up` is used to launch the vCPU once the initial state has been
- set using `VCPUOP_initialise`.
-
- * `VCPUOP_down` is used to bring down a vCPU.
-
- * `VCPUOP_is_up` is used to scan the number of available vCPUs.
-
-## Hardware description ##
-
-PVHv2 guests that have access to hardware (either emulated or real) will also
-have ACPI tables with the description of the hardware that's available to the
-guest. This applies to both privileged and unprivileged guests. A pointer to
-the position of the RSDP in memory (if present) can be fetched from the start
-info structure that's passed at boot time (field rsdp\_paddr).
-
-Description of paravirtualized devices will come from XenStore, just as it's
-done for HVM guests.
-
-## Interrupts ##
-
-### Interrupts from physical devices ###
-
-Interrupts from physical devices are delivered using native methods, this is
-done in order to take advantage of new hardware assisted virtualization
-functions, like posted interrupts. This implies that PVHv2 guests with physical
-devices will also have the necessary interrupt controllers in order to manage
-the delivery of interrupts from those devices, using the same interfaces that
-are available on native hardware.
-
-### Interrupts from paravirtualized devices ###
-
-Interrupts from paravirtualized devices are delivered using event channels, see
-[Event Channel Internals][event_channels] for more detailed information about
-event channels. Delivery of those interrupts can be configured in the same way
-as HVM guests, check xen/include/public/hvm/params.h and
-xen/include/public/hvm/hvm\_op.h for more information about available delivery
-methods.
-
-## MTRR ##
-
-### Unprivileged guests ###
-
-PVH guests are currently booted with the default MTRR type set to write-back
-and MTRR enabled. This allows DomUs to start with a sane MTRR state. Note that
-this will have to be revisited when pci-passthrough is added to PVH in order to
-set MMIO regions as UC.
-
-Xen guarantees that RAM regions will always have the WB cache type set in the
-initial MTRR state, either set by the default MTRR type or by other means.
-
-### Hardware domain ###
-
-A PVH hardware domain is booted with the same MTRR state as the one found on
-the host. This is done because the hardware domain memory map is already a
-modified copy of the host memory map, so the same MTRR setup should work.
--- /dev/null
+# x86/HVM direct boot ABI #
+
+Since the Xen entry point into the kernel can be different from the
+native entry point, a `ELFNOTE` is used in order to tell the domain
+builder how to load and jump into the kernel entry point:
+
+ ELFNOTE(Xen, XEN_ELFNOTE_PHYS32_ENTRY, .long, xen_start32)
+
+The presence of the `XEN_ELFNOTE_PHYS32_ENTRY` note indicates that the
+kernel supports the boot ABI described in this document.
+
+The domain builder must load the kernel into the guest memory space and
+jump into the entry point defined at `XEN_ELFNOTE_PHYS32_ENTRY` with the
+following machine state:
+
+ * `ebx`: contains the physical memory address where the loader has placed
+ the boot start info structure.
+
+ * `cr0`: bit 0 (PE) must be set. All the other writeable bits are cleared.
+
+ * `cr4`: all bits are cleared.
+
+ * `cs`: must be a 32-bit read/execute code segment with a base of ‘0’
+ and a limit of ‘0xFFFFFFFF’. The selector value is unspecified.
+
+ * `ds`, `es`: must be a 32-bit read/write data segment with a base of
+ ‘0’ and a limit of ‘0xFFFFFFFF’. The selector values are all unspecified.
+
+ * `tr`: must be a 32-bit TSS (active) with a base of '0' and a limit of '0x67'.
+
+ * `eflags`: bit 17 (VM) must be cleared. Bit 9 (IF) must be cleared.
+ Bit 8 (TF) must be cleared. Other bits are all unspecified.
+
+All other processor registers and flag bits are unspecified. The OS is in
+charge of setting up it's own stack, GDT and IDT.
+
+The format of the boot start info structure (pointed to by %ebx) can be found
+in `xen/include/public/arch-x86/hvm/start_info.h`
+
+Other relevant information needed in order to boot a guest kernel
+(console page address, xenstore event channel...) can be obtained
+using HVMPARAMS, just like it's done on HVM guests.
+
+The setup of the hypercall page is also performed in the same way
+as HVM guests, using the hypervisor cpuid leaves and msr ranges.
+
+## AP startup ##
+
+AP startup can be performed using hypercalls or the local APIC if present.
+The following VCPU hypercalls can be used in order to bring up secondary vCPUs:
+
+ * `VCPUOP_initialise` is used to set the initial state of the vCPU. The
+ argument passed to the hypercall must be of the type vcpu_hvm_context.
+ See `public/hvm/hvm_vcpu.h` for the layout of the structure. Note that
+ this hypercall allows starting the vCPU in several modes (16/32/64bits),
+ regardless of the mode the BSP is currently running on.
+
+ * `VCPUOP_up` is used to launch the vCPU once the initial state has been
+ set using `VCPUOP_initialise`.
+
+ * `VCPUOP_down` is used to bring down a vCPU.
+
+ * `VCPUOP_is_up` is used to scan the number of available vCPUs.
+
+## Hardware description ##
+
+PVHv2 guests that have access to hardware (either emulated or real) will also
+have ACPI tables with the description of the hardware that's available to the
+guest. This applies to both privileged and unprivileged guests. A pointer to
+the position of the RSDP in memory (if present) can be fetched from the start
+info structure that's passed at boot time (field `rsdp_paddr`).
+
+Description of paravirtualized devices will come from XenStore, just as it's
+done for HVM guests.
+
+## Interrupts ##
+
+### Interrupts from physical devices ###
+
+Interrupts from physical devices are delivered using native methods, this is
+done in order to take advantage of new hardware assisted virtualization
+functions, like posted interrupts. This implies that PVHv2 guests with physical
+devices will also have the necessary interrupt controllers in order to manage
+the delivery of interrupts from those devices, using the same interfaces that
+are available on native hardware.
+
+### Interrupts from paravirtualized devices ###
+
+Interrupts from paravirtualized devices are delivered using event channels, see
+[Event Channel Internals][event_channels] for more detailed information about
+event channels. Delivery of those interrupts can be configured in the same way
+as HVM guests, check `xen/include/public/hvm/params.h` and
+`xen/include/public/hvm/hvm_op.h` for more information about available delivery
+methods.
+
+## MTRR ##
+
+### Unprivileged guests ###
+
+PVH guests are currently booted with the default MTRR type set to write-back
+and MTRR enabled. This allows DomUs to start with a sane MTRR state. Note that
+this will have to be revisited when pci-passthrough is added to PVH in order to
+set MMIO regions as UC.
+
+Xen guarantees that RAM regions will always have the WB cache type set in the
+initial MTRR state, either set by the default MTRR type or by other means.
+
+### Hardware domain ###
+
+A PVH hardware domain is booted with the same MTRR state as the one found on
+the host. This is done because the hardware domain memory map is already a
+modified copy of the host memory map, so the same MTRR setup should work.
+++ /dev/null
-# Xen x86 PV Bootloader Protocol
-
-## Introduction
-
-One method for booting an x86 Xen PV guest is to use a PV bootloader,
-that is, a bootloader which is itself a PV kernel but which behaves as
-a bootloader (examples include the pvgrub-legacy and grub2 targeting
-Xen).
-
-In many cases the user wishes to manage this PV bootloader from within
-the guest, and therefore wishes to chainload something from the guest
-filesystem, most likely via a stage 1 PV bootloader provided by dom0.
-
-The purpose of this document is to define the paths within the guest
-filesystem where a stage 1 bootloader should look for the in-guest PV
-bootloader to load and the protocol/format expected from the
-to-be-chainloaded bootloader.
-
-## Protocol
-
-The bootloader binary should be an ELF file of the appropriate type
-(32- or 64-bit). It should contain the standard Xen ELF notes allowing
-it to be loaded by the Xen toolstack domain builder (TBD: Reference).
-
-## Path
-
-The second stage bootloader should be installed into the guest
-filesystem as:
-
- * `/boot/xen/pvboot-<ARCH>.elf`
-
-Where `<ARCH>` is the first element of the GNU triplet e.g. one of:
-
- * i386 (nb only i386, not i686 etc), corresponding to the Xen
- x86\_32(p) arch;
- * x86\_64, corresponding to the Xen x86\_64 arch;
-
-It is allowable for `/boot` to be a separate filesystem from `/` and
-therefore stage 1 bootloaders should search
-`/boot/xen/pvboot-<ARCH>.elf` and `/xen/pvboot-<ARCH>.elf` (in that
-order). The `xen` directory should be on the same filesystem as /boot
-and therefore it is not necessary to search for /pvboot-<ARCH>.elf.
-
-It is not in general possible under Xen for a bootloader to boot a
-kernel of a different width from itself, and this extends to
-chainloading from a stage one. Therefore it is permissible to have
-both `/boot/xen/pvboot-i386.elf` and `/boot/xen/pvboot-x86\_64.elf`
-present in a guest to be used by the appropriate stage 1 (e.g. for
-systems with 32-bit userspace and an optional 64-bit kernel).
--- /dev/null
+# Xen x86 PV Bootloader Protocol
+
+## Introduction
+
+One method for booting an x86 Xen PV guest is to use a PV bootloader,
+that is, a bootloader which is itself a PV kernel but which behaves as
+a bootloader (examples include the pvgrub-legacy and grub2 targeting
+Xen).
+
+In many cases the user wishes to manage this PV bootloader from within
+the guest, and therefore wishes to chainload something from the guest
+filesystem, most likely via a stage 1 PV bootloader provided by dom0.
+
+The purpose of this document is to define the paths within the guest
+filesystem where a stage 1 bootloader should look for the in-guest PV
+bootloader to load and the protocol/format expected from the
+to-be-chainloaded bootloader.
+
+## Protocol
+
+The bootloader binary should be an ELF file of the appropriate type
+(32- or 64-bit). It should contain the standard Xen ELF notes allowing
+it to be loaded by the Xen toolstack domain builder (TBD: Reference).
+
+## Path
+
+The second stage bootloader should be installed into the guest
+filesystem as:
+
+ * `/boot/xen/pvboot-<ARCH>.elf`
+
+Where `<ARCH>` is the first element of the GNU triplet e.g. one of:
+
+ * `i386` (nb only `i386`, not `i686` etc), corresponding to the Xen
+ `x86_32(p)` arch;
+ * `x86_64`, corresponding to the Xen `x86_64` arch;
+
+It is allowable for `/boot` to be a separate filesystem from `/` and
+therefore stage 1 bootloaders should search
+`/boot/xen/pvboot-<ARCH>.elf` and `/xen/pvboot-<ARCH>.elf` (in that
+order). The `xen` directory should be on the same filesystem as /boot
+and therefore it is not necessary to search for /pvboot-<ARCH>.elf.
+
+It is not in general possible under Xen for a bootloader to boot a
+kernel of a different width from itself, and this extends to
+chainloading from a stage one. Therefore it is permissible to have
+both `/boot/xen/pvboot-i386.elf` and `/boot/xen/pvboot-x86_64.elf`
+present in a guest to be used by the appropriate stage 1 (e.g. for
+systems with 32-bit userspace and an optional 64-bit kernel).
+++ /dev/null
-# Xen Hypervisor Command Line Options
-
-This document covers the command line options which the Xen
-Hypervisor.
-
-## Types of parameter
-
-Most parameters take the form `option=value`. Different options on
-the command line should be space delimited. All options are case
-sensitive, as are all values unless explicitly noted.
-
-### Boolean (`<boolean>`)
-
-All boolean option may be explicitly enabled using a `value` of
-> `yes`, `on`, `true`, `enable` or `1`
-
-They may be explicitly disabled using a `value` of
-> `no`, `off`, `false`, `disable` or `0`
-
-In addition, a boolean option may be enabled by simply stating its
-name, and may be disabled by prefixing its name with `no-`.
-
-####Examples
-
-Enable noreboot mode
-> `noreboot=true`
-
-Disable x2apic support (if present)
-> `x2apic=off`
-
-Enable synchronous console mode
-> `sync_console`
-
-Explicitly specifying any value other than those listed above is
-undefined, as is stacking a `no-` prefix with an explicit value.
-
-### Integer (`<integer>`)
-
-An integer parameter will default to decimal and may be prefixed with
-a `-` for negative numbers. Alternatively, a hexadecimal number may be
-used by prefixing the number with `0x`, or an octal number may be used
-if a leading `0` is present.
-
-Providing a string which does not validly convert to an integer is
-undefined.
-
-### Size (`<size>`)
-
-A size parameter may be any integer, with a single size suffix
-
-* `T` or `t`: TiB (2^40)
-* `G` or `g`: GiB (2^30)
-* `M` or `m`: MiB (2^20)
-* `K` or `k`: KiB (2^10)
-* `B` or `b`: Bytes
-
-Without a size suffix, the default will be kilo. Providing a suffix
-other than those listed above is undefined.
-
-### String
-
-Many parameters are more complicated and require more intricate
-configuration. The detailed description of each individual parameter
-specify which values are valid.
-
-### List
-
-Some options take a comma separated list of values.
-
-### Combination
-
-Some parameters act as combinations of the above, most commonly a mix
-of Boolean and String. These are noted in the relevant sections.
-
-## Parameter details
-
-### acpi
-> `= force | ht | noirq | <boolean>`
-
-**String**, or **Boolean** to disable.
-
-The **acpi** option is used to control a set of four related boolean
-flags; `acpi_force`, `acpi_ht`, `acpi_noirq` and `acpi_disabled`.
-
-By default, Xen will scan the DMI data and blacklist certain systems
-which are known to have broken ACPI setups. Providing `acpi=force`
-will cause Xen to ignore the blacklist and attempt to use all ACPI
-features.
-
-Using `acpi=ht` causes Xen to parse the ACPI tables enough to
-enumerate all CPUs, but will not use other ACPI features. This is not
-common, and only has an effect if your system is blacklisted.
-
-The `acpi=noirq` option causes Xen to not parse the ACPI MADT table
-looking for IO-APIC entries. This is also not common, and any system
-which requires this option to function should be blacklisted.
-Additionally, this will not prevent Xen from finding IO-APIC entries
-from the MP tables.
-
-Finally, any of the boolean false options can be used to disable ACPI
-usage entirely.
-
-Because responsibility for ACPI processing is shared between Xen and
-the domain 0 kernel this option is automatically propagated to the
-domain 0 command line
-
-### acpi\_apic\_instance
-> `= <integer>`
-
-Specify which ACPI MADT table to parse for APIC information, if more
-than one is present.
-
-### acpi\_pstate\_strict (x86)
-> `= <boolean>`
-
-> Default: `false`
-
-Enforce checking that P-state transitions by the ACPI cpufreq driver
-actually result in the nominated frequency to be established. A warning
-message will be logged if that isn't the case.
-
-### acpi\_skip\_timer\_override (x86)
-> `= <boolean>`
-
-Instruct Xen to ignore timer-interrupt override.
-
-### acpi\_sleep (x86)
-> `= s3_bios | s3_mode`
-
-`s3_bios` instructs Xen to invoke video BIOS initialization during S3
-resume.
-
-`s3_mode` instructs Xen to set up the boot time (option `vga=`) video
-mode during S3 resume.
-
-### allow\_unsafe (x86)
-> `= <boolean>`
-
-> Default: `false`
-
-Force boot on potentially unsafe systems. By default Xen will refuse
-to boot on systems with the following errata:
-
-* AMD Erratum 121. Processors with this erratum are subject to a guest
- triggerable Denial of Service. Override only if you trust all of
- your PV guests.
-
-### altp2m (Intel)
-> `= <boolean>`
-
-> Default: `false`
-
-Permit multiple copies of host p2m.
-
-### apic (x86)
-> `= bigsmp | default`
-
-Override Xen's logic for choosing the APIC driver. By default, if
-there are more than 8 CPUs, Xen will switch to `bigsmp` over
-`default`.
-
-### apicv (Intel)
-> `= <boolean>`
-
-> Default: `true`
-
-Permit Xen to use APIC Virtualisation Extensions. This is an optimisation
-available as part of VT-x, and allows hardware to take care of the guests APIC
-handling, rather than requiring emulation in Xen.
-
-### apic\_verbosity (x86)
-> `= verbose | debug`
-
-Increase the verbosity of the APIC code from the default value.
-
-### arat (x86)
-> `= <boolean>`
-
-> Default: `true`
-
-Permit Xen to use "Always Running APIC Timer" support on compatible hardware
-in combination with cpuidle. This option is only expected to be useful for
-developers wishing Xen to fall back to older timing methods on newer hardware.
-
-### asid (x86)
-> `= <boolean>`
-
-> Default: `true`
-
-Permit Xen to use Address Space Identifiers. This is an optimisation which
-tags the TLB entries with an ID per vcpu. This allows for guest TLB flushes
-to be performed without the overhead of a complete TLB flush.
-
-### async-show-all (x86)
-> `= <boolean>`
-
-> Default: `false`
-
-Forces all CPUs' full state to be logged upon certain fatal asynchronous
-exceptions (watchdog NMIs and unexpected MCEs).
-
-### ats (x86)
-> `= <boolean>`
-
-> Default: `false`
-
-Permits Xen to set up and use PCI Address Translation Services. This is a
-performance optimisation for PCI Passthrough.
-
-**WARNING: Xen cannot currently safely use ATS because of its synchronous wait
-loops for Queued Invalidation completions.**
-
-### availmem
-> `= <size>`
-
-> Default: `0` (no limit)
-
-Specify a maximum amount of available memory, to which Xen will clamp
-the e820 table.
-
-### badpage
-> `= List of [ <integer> | <integer>-<integer> ]`
-
-Specify that certain pages, or certain ranges of pages contain bad
-bytes and should not be used. For example, if your memory tester says
-that byte `0x12345678` is bad, you would place `badpage=0x12345` on
-Xen's command line.
-
-### bootscrub
-> `= idle | <boolean>`
-
-> Default: `idle`
-
-Scrub free RAM during boot. This is a safety feature to prevent
-accidentally leaking sensitive VM data into other VMs if Xen crashes
-and reboots.
-
-In `idle` mode, RAM is scrubbed in background on all CPUs during idle-loop
-with a guarantee that memory allocations always provide scrubbed pages.
-This option reduces boot time on machines with a large amount of RAM while
-still providing security benefits.
-
-### bootscrub\_chunk
-> `= <size>`
-
-> Default: `128M`
-
-Maximum RAM block size chunks to be scrubbed whilst holding the page heap lock
-and not running softirqs. Reduce this if softirqs are not being run frequently
-enough. Setting this to a high value may cause boot failure, particularly if
-the NMI watchdog is also enabled.
-
-### clocksource (x86)
-> `= pit | hpet | acpi | tsc`
-
-If set, override Xen's default choice for the platform timer.
-Having TSC as platform timer requires being explicitly set. This is because
-TSC can only be safely used if CPU hotplug isn't performed on the system. On
-some platforms, the "maxcpus" option may need to be used to further adjust
-the number of allowed CPUs. When running on platforms that can guarantee a
-monotonic TSC across sockets you may want to adjust the "tsc" command line
-parameter to "stable:socket".
-
-### cmci-threshold (Intel)
-> `= <integer>`
-
-> Default: `2`
-
-Specify the event count threshold for raising Corrected Machine Check
-Interrupts. Specifying zero disables CMCI handling.
-
-### cmos-rtc-probe (x86)
-> `= <boolean>`
-
-> Default: `false`
-
-Flag to indicate whether to probe for a CMOS Real Time Clock irrespective of
-ACPI indicating none to be there.
-
-### com1
-### com2
-> `= <baud>[/<base-baud>][,[DPS][,[<io-base>|pci|amt][,[<irq>|msi][,[<port-bdf>][,[<bridge-bdf>]]]]]]`
-
-Both option `com1` and `com2` follow the same format.
-
-* `<baud>` may be either an integer baud rate, or the string `auto` if
- the bootloader or other earlier firmware has already set it up.
-* Optionally, the base baud rate (usually the highest baud rate the
- device can communicate at) can be specified.
-* `DPS` represents the number of data bits, the parity, and the number
- of stop bits.
- * `D` is an integer between 5 and 8 for the number of data bits.
- * `P` is a single character representing the type of parity:
- * `n` No
- * `o` Odd
- * `e` Even
- * `m` Mark
- * `s` Space
- * `S` is an integer 1 or 2 for the number of stop bits.
-* `<io-base>` is an integer which specifies the IO base port for UART
- registers.
-* `<irq>` is the IRQ number to use, or `0` to use the UART in poll
- mode only, or `msi` to set up a Message Signaled Interrupt.
-* `<port-bdf>` is the PCI location of the UART, in
- `<bus>:<device>.<function>` notation.
-* `<bridge-bdf>` is the PCI bridge behind which is the UART, in
- `<bus>:<device>.<function>` notation.
-* `pci` indicates that Xen should scan the PCI bus for the UART,
- avoiding Intel AMT devices.
-* `amt` indicated that Xen should scan the PCI bus for the UART,
- including Intel AMT devices if present.
-
-A typical setup for most situations might be `com1=115200,8n1`
-
-In addition to the above positional specification for UART parameters,
-name=value pair specfications are also supported. This is used to add
-flexibility for UART devices which require additional UART parameter
-configurations.
-
-The comma separation still delineates positional parameters. Hence,
-unless the parameter is explicitly specified with name=value option, it
-will be considered a positional parameter.
-
-The syntax consists of
-com1=(comma-separated positional parameters),(comma separated name-value pairs)
-
-The accepted name keywords for name=value pairs are:
-
-* `baud` - accepts integer baud rate (eg. 115200) or `auto`
-* `bridge`- Similar to bridge-bdf in positional parameters.
- Used to determine the PCI bridge to access the UART device.
- Notation is xx:xx.x `<bus>:<device>.<function>`
-* `clock-hz`- accepts large integers to setup UART clock frequencies.
- Do note - these values are multiplied by 16.
-* `data-bits` - integer between 5 and 8
-* `dev` - accepted values are `pci` OR `amt`. If this option
- is used to specify if the serial device is pci-based. The io\_base
- cannot be specified when `dev=pci` or `dev=amt` is used.
-* `io-base` - accepts integer which specified IO base port for UART registers
-* `irq` - IRQ number to use
-* `parity` - accepted values are same as positional parameters
-* `port` - Used to specify which port the PCI serial device is located on
- Notation is xx:xx.x `<bus>:<device>.<function>`
-* `reg-shift` - register shifts required to set UART registers
-* `reg-width` - register width required to set UART registers
- (only accepts 1 and 4)
-* `stop-bits` - only accepts 1 or 2 for the number of stop bits
-
-The following are examples of correct specifications:
-
- com1=115200,8n1,0x3f8,4
- com1=115200,8n1,0x3f8,4,reg_width=4,reg_shift=2
- com1=baud=115200,parity=n,stop_bits=1,io_base=0x3f8,reg_width=4
-
-### conring\_size
-> `= <size>`
-
-> Default: `conring_size=16k`
-
-Specify the size of the console ring buffer.
-
-### console
-> `= List of [ vga | com1[H,L] | com2[H,L] | pv | dbgp | none ]`
-
-> Default: `console=com1,vga`
-
-Specify which console(s) Xen should use.
-
-`vga` indicates that Xen should try and use the vga graphics adapter.
-
-`com1` and `com2` indicates that Xen should use serial ports 1 and 2
-respectively. Optionally, these arguments may be followed by an `H` or
-`L`. `H` indicates that transmitted characters will have their MSB
-set, while received characters must have their MSB set. `L` indicates
-the converse; transmitted and received characters will have their MSB
-cleared. This allows a single port to be shared by two subsystems
-(e.g. console and debugger).
-
-`pv` indicates that Xen should use Xen's PV console. This option is
-only available when used together with `pv-in-pvh`.
-
-`dbgp` indicates that Xen should use a USB debug port.
-
-`none` indicates that Xen should not use a console. This option only
-makes sense on its own.
-
-### console\_timestamps
-> `= none | date | datems | boot | raw`
-
-> Default: `none`
-
-> Can be modified at runtime
-
-Specify which timestamp format Xen should use for each console line.
-
-* `none`: No timestamps
-* `date`: Date and time information
- * `[YYYY-MM-DD HH:MM:SS]`
-* `datems`: Date and time, with milliseconds
- * `[YYYY-MM-DD HH:MM:SS.mmm]`
-* `boot`: Seconds and microseconds since boot
- * `[SSSSSS.uuuuuu]`
-+ `raw`: Raw platform ticks, architecture and implementation dependent
- * `[XXXXXXXXXXXXXXXX]`
-
-For compatibility with the older boolean parameter, specifying
-`console_timestamps` alone will enable the `date` option.
-
-### console\_to\_ring
-> `= <boolean>`
-
-> Default: `false`
-
-Flag to indicate whether all guest console output should be copied
-into the console ring buffer.
-
-### conswitch
-> `= <switch char>[x]`
-
-> Default: `conswitch=a`
-
-> Can be modified at runtime
-
-Specify which character should be used to switch serial input between
-Xen and dom0. The required sequence is CTRL-<switch char> three
-times.
-
-The optional trailing `x` indicates that Xen should not automatically
-switch the console input to dom0 during boot. Any other value,
-including omission, causes Xen to automatically switch to the dom0
-console during dom0 boot. Use `conswitch=ax` to keep the default switch
-character, but for xen to keep the console.
-
-### core\_parking
-> `= power | performance`
-
-> Default: `power`
-
-### cpu\_type (x86)
-> `= arch_perfmon`
-
-If set, force use of the performance counters for oprofile, rather than detecting
-available support.
-
-### cpufreq
-> `= none | {{ <boolean> | xen } [:[powersave|performance|ondemand|userspace][,<maxfreq>][,[<minfreq>][,[verbose]]]]} | dom0-kernel`
-
-> Default: `xen`
-
-Indicate where the responsibility for driving power states lies. Note that the
-choice of `dom0-kernel` is deprecated and not supported by all Dom0 kernels.
-
-* Default governor policy is ondemand.
-* `<maxfreq>` and `<minfreq>` are integers which represent max and min processor frequencies
- respectively.
-* `verbose` option can be included as a string or also as `verbose=<integer>`
-
-### cpuid (x86)
-> `= List of comma separated booleans`
-
-This option allows for fine tuning of the facilities Xen will use, after
-accounting for hardware capabilities as enumerated via CPUID.
-
-Currently accepted:
-
-The Speculation Control hardware features `ibrsb`, `stibp`, `ibpb`,
-`l1d-flush` and `ssbd` are used by default if available and applicable. They can
-be ignored, e.g. `no-ibrsb`, at which point Xen won't use them itself, and
-won't offer them to guests.
-
-### cpuid_mask_cpu
-> `= fam_0f_rev_[cdefg] | fam_10_rev_[bc] | fam_11_rev_b`
-
-> Applicability: AMD
-
-If none of the other **cpuid\_mask\_\*** options are given, Xen has a set of
-pre-configured masks to make the current processor appear to be
-family/revision specified.
-
-See below for general information on masking.
-
-**Warning: This option is not fully effective on Family 15h processors or
-later.**
-
-### cpuid_mask_ecx
-### cpuid_mask_edx
-### cpuid_mask_ext_ecx
-### cpuid_mask_ext_edx
-### cpuid_mask_l7s0_eax
-### cpuid_mask_l7s0_ebx
-### cpuid_mask_thermal_ecx
-### cpuid_mask_xsave_eax
-> `= <integer>`
-
-> Applicability: x86. Default: `~0` (all bits set)
-
-The availability of these options are model specific. Some processors don't
-support any of them, and no processor supports all of them. Xen will ignore
-options on processors which are lacking support.
-
-These options can be used to alter the features visible via the `CPUID`
-instruction. Settings applied here take effect globally, including for Xen
-and all guests.
-
-Note: Since Xen 4.7, it is no longer necessary to mask a host to create
-migration safety in heterogeneous scenarios. All necessary CPUID settings
-should be provided in the VM configuration file. Furthermore, it is
-recommended not to use this option, as doing so causes an unnecessary
-reduction of features at Xen's disposal to manage guests.
-
-### cpuidle (x86)
-> `= <boolean>`
-
-### cpuinfo (x86)
-> `= <boolean>`
-
-### crashinfo\_maxaddr
-> `= <size>`
-
-> Default: `4G`
-
-Specify the maximum address to allocate certain structures, if used in
-combination with the `low_crashinfo` command line option.
-
-### crashkernel
-> `= <ramsize-range>:<size>[,...][{@,<}<offset>]`
-> `= <size>[{@,<}<offset>]`
-> `= <size>,below=offset`
-
-Specify sizes and optionally placement of the crash kernel reservation
-area. The `<ramsize-range>:<size>` pairs indicate how much memory to
-set aside for a crash kernel (`<size>`) for a given range of installed
-RAM (`<ramsize-range>`). Each `<ramsize-range>` is of the form
-`<start>-[<end>]`.
-
-A trailing `@<offset>` specifies the exact address this area should be
-placed at, whereas `<` in place of `@` just specifies an upper bound of
-the address range the area should fall into.
-
-< and below are synonyomous, the latter being useful for grub2 systems
-which would otherwise require escaping of the < option
-
-
-### credit2\_balance\_over
-> `= <integer>`
-
-### credit2\_balance\_under
-> `= <integer>`
-
-### credit2\_cap\_period\_ms
-> `= <integer>`
-
-> Default: `10`
-
-Domains subject to a cap receive a replenishment of their runtime budget
-once every cap period interval. Default is 10 ms. The amount of budget
-they receive depends on their cap. For instance, a domain with a 50% cap
-will receive 50% of 10 ms, so 5 ms.
-
-### credit2\_load\_precision\_shift
-> `= <integer>`
-
-> Default: `18`
-
-Specify the number of bits to use for the fractional part of the
-values involved in Credit2 load tracking and load balancing math.
-
-### credit2\_load\_window\_shift
-> `= <integer>`
-
-> Default: `30`
-
-Specify the number of bits to use to represent the length of the
-window (in nanoseconds) we use for load tracking inside Credit2.
-This means that, with the default value (30), we use
-2^30 nsec ~= 1 sec long window.
-
-Load tracking is done by means of a variation of exponentially
-weighted moving average (EWMA). The window length defined here
-is what tells for how long we give value to previous history
-of the load itself. In fact, after a full window has passed,
-what happens is that we discard all previous history entirely.
-
-A short window will make the load balancer quick at reacting
-to load changes, but also short-sighted about previous history
-(and hence, e.g., long term load trends). A long window will
-make the load balancer thoughtful of previous history (and
-hence capable of capturing, e.g., long term load trends), but
-also slow in responding to load changes.
-
-The default value of `1 sec` is rather long.
-
-### credit2\_runqueue
-> `= cpu | core | socket | node | all`
-
-> Default: `socket`
-
-Specify how host CPUs are arranged in runqueues. Runqueues are kept
-balanced with respect to the load generated by the vCPUs running on
-them. Smaller runqueues (as in with `core`) means more accurate load
-balancing (for instance, it will deal better with hyperthreading),
-but also more overhead.
-
-Available alternatives, with their meaning, are:
-* `cpu`: one runqueue per each logical pCPUs of the host;
-* `core`: one runqueue per each physical core of the host;
-* `socket`: one runqueue per each physical socket (which often,
- but not always, matches a NUMA node) of the host;
-* `node`: one runqueue per each NUMA node of the host;
-* `all`: just one runqueue shared by all the logical pCPUs of
- the host
-
-### dbgp
-> `= ehci[ <integer> | @pci<bus>:<slot>.<func> ]`
-
-Specify the USB controller to use, either by instance number (when going
-over the PCI busses sequentially) or by PCI device (must be on segment 0).
-
-### debug\_stack\_lines
-> `= <integer>`
-
-> Default: `20`
-
-Limits the number lines printed in Xen stack traces.
-
-### debugtrace
-> `= <integer>`
-
-> Default: `128`
-
-Specify the size of the console debug trace buffer in KiB. The debug
-trace feature is only enabled in debugging builds of Xen.
-
-### dma\_bits
-> `= <integer>`
-
-Specify the bit width of the DMA heap.
-
-### dom0 (x86)
-> `= List of [ pvh | shadow ]`
-
-> Sub-options:
-
-> `pvh`
-
-> Default: `false`
-
-Flag that makes a dom0 boot in PVHv2 mode.
-
-> `shadow`
-
-> Default: `false`
-
-Flag that makes a dom0 use shadow paging. Only works when "pvh" is
-enabled.
-
-### dom0-iommu
-> `= List of [ passthrough | strict | map-inclusive ]`
-
-This list of booleans controls the iommu usage by Dom0:
-
-* `passthrough`: disables DMA remapping for Dom0. Default is `false`. Note that
- this option is hard coded to `false` for a PVH Dom0 and any attempt to
- overwrite it from the command line is ignored.
-
-* `strict`: sets up DMA remapping only for the RAM Dom0 actually got assigned.
- Default is `false` which means Dom0 will get mappings for all the host
- RAM except regions in use by Xen. Note that this option is hard coded to
- `true` for a PVH Dom0 and any attempt to overwrite it from the command line
- is ignored.
-
-* `map-inclusive`: sets up DMA remapping for all the non-RAM regions below 4GB
- except for unusable ranges. Use this to work around firmware issues providing
- incorrect RMRR/IVMD entries. Rather than only mapping RAM pages for IOMMU
- accesses for Dom0, with this option all pages up to 4GB, not marked as
- unusable in the E820 table, will get a mapping established. Note that this
- option is only applicable to a PV Dom0 and is enabled by default on Intel
- hardware.
-
-* `map-reserved`: sets up DMA remapping for all the reserved regions in the
- memory map for Dom0. Use this to work around firmware issues providing
- incorrect RMRR/IVMD entries. Rather than only mapping RAM pages for IOMMU
- accesses for Dom0, all memory regions marked as reserved in the memory map
- that don't overlap with any MMIO region from emulated devices will be
- identity mapped. This option maps a subset of the memory that would be
- mapped when using the `map-inclusive` option. This option is available to all
- Dom0 modes and is enabled by default on Intel hardware.
-
-### dom0\_ioports\_disable (x86)
-> `= List of <hex>-<hex>`
-
-Specify a list of IO ports to be excluded from dom0 access.
-
-### dom0\_max\_vcpus
-
-Either:
-
-> `= <integer>`.
-
-The number of VCPUs to give to dom0. This number of VCPUs can be more
-than the number of PCPUs on the host. The default is the number of
-PCPUs.
-
-Or:
-
-> `= <min>-<max>` where `<min>` and `<max>` are integers.
-
-Gives dom0 a number of VCPUs equal to the number of PCPUs, but always
-at least `<min>` and no more than `<max>`. Using `<min>` may give
-more VCPUs than PCPUs. `<min>` or `<max>` may be omitted and the
-defaults of 1 and unlimited respectively are used instead.
-
-For example, with `dom0_max_vcpus=4-8`:
-
-> Number of
-> PCPUs | Dom0 VCPUs
-> 2 | 4
-> 4 | 4
-> 6 | 6
-> 8 | 8
-> 10 | 8
-
-### dom0\_mem (ARM)
-> `= <size>`
-
-Set the amount of memory for the initial domain (dom0). It must be
-greater than zero. This parameter is required.
-
-### dom0\_mem (x86)
-> `= List of ( min:<sz> | max:<sz> | <sz> )`
-
-Set the amount of memory for the initial domain (dom0). If a size is
-positive, it represents an absolute value. If a size is negative, it
-is subtracted from the total available memory.
-
-* `<sz>` specifies the exact amount of memory.
-* `min:<sz>` specifies the minimum amount of memory.
-* `max:<sz>` specifies the maximum amount of memory.
-
-If `<sz>` is not specified, the default is all the available memory
-minus some reserve. The reserve is 1/16 of the available memory or
-128 MB (whichever is smaller).
-
-The amount of memory will be at least the minimum but never more than
-the maximum (i.e., `max` overrides the `min` option). If there isn't
-enough memory then as much as possible is allocated.
-
-`max:<sz>` also sets the maximum reservation (the maximum amount of
-memory dom0 can balloon up to). If this is omitted then the maximum
-reservation is unlimited.
-
-For example, to set dom0's initial memory allocation to 512MB but
-allow it to balloon up as far as 1GB use `dom0_mem=512M,max:1G`
-
-> `<sz>` is: `<size> | [<size>+]<frac>%`
-> `<frac>` is an integer < 100
-
-* `<frac>` specifies a fraction of host memory size in percent.
-
-So `<sz>` being `1G+25%` on a 256 GB host would result in 65 GB.
-
-If you use this option then it is highly recommended that you disable
-any dom0 autoballooning feature present in your toolstack. See the
-_xl.conf(5)_ man page or [Xen Best
-Practices](http://wiki.xen.org/wiki/Xen_Best_Practices#Xen_dom0_dedicated_memory_and_preventing_dom0_memory_ballooning).
-
-This option doesn't have effect if pv-shim mode is enabled.
-
-### dom0\_nodes (x86)
-
-> `= List of [ <integer> | relaxed | strict ]`
-
-> Default: `strict`
-
-Specify the NUMA nodes to place Dom0 on. Defaults for vCPU-s created
-and memory assigned to Dom0 will be adjusted to match the node
-restrictions set up here. Note that the values to be specified here are
-ACPI PXM ones, not Xen internal node numbers. `relaxed` sets up vCPU
-affinities to prefer but be not limited to the specified node(s).
-
-### dom0\_vcpus\_pin
-> `= <boolean>`
-
-> Default: `false`
-
-Pin dom0 vcpus to their respective pcpus
-
-### dtuart (ARM)
-> `= path [:options]`
-
-> Default: `""`
-
-Specify the full path in the device tree for the UART. If the path doesn't
-start with `/`, it is assumed to be an alias. The options are device specific.
-
-### e820-mtrr-clip (x86)
-> `= <boolean>`
-
-Flag that specifies if RAM should be clipped to the highest cacheable
-MTRR.
-
-> Default: `true` on Intel CPUs, otherwise `false`
-
-### e820-verbose (x86)
-> `= <boolean>`
-
-> Default: `false`
-
-Flag that enables verbose output when processing e820 information and
-applying clipping.
-
-### edd (x86)
-> `= off | on | skipmbr`
-
-Control retrieval of Extended Disc Data (EDD) from the BIOS during
-boot.
-
-### edid (x86)
-> `= no | force`
-
-Either force retrieval of monitor EDID information via VESA DDC, or
-disable it (edid=no). This option should not normally be required
-except for debugging purposes.
-
-### efi
-> `= List of [ rs | attr ]`
-
-All options are of boolean kind and can be prefixed with `no-` to
-effect the inverse meaning.
-
-> `rs`
-
-> Default: `true`
-
->> Force or disable use of EFI runtime services.
-
-> `attr=uc`
-
-> Default: `off`
-
->> Allows mapping of RuntimeServices which have no cachability attribute
->> set as UC.
-
-### ept
-> `= List of [ ad=<bool>, pml=<bool> ]`
-
-> Applicability: Intel
-
-Extended Page Tables are a feature of Intel's VT-x technology, whereby
-hardware manages the virtualisation of HVM guest pagetables. EPT was
-introduced with the Nehalem architecture.
-
-* The `ad` boolean controls hardware tracking of Access and Dirty bits in the
- EPT pagetables, and was first introduced in Broadwell Server.
-
- By default, Xen will use A/D tracking when available in hardware, except
- on Avoton processors affected by erratum AVR41. Explicitly choosing
- `ad=0` will disable the use of A/D tracking on capable hardware, whereas
- choosing `ad=1` will cause tracking to be used even on AVR41-affected
- hardware.
-
-* The `pml` boolean controls the use of Page Modification Logging, which is
- also introduced in Broadwell Server.
-
- PML is a feature whereby the processor generates a list of pages which
- have been dirtied. This is necessary information for operations such as
- live migration, and having the processor maintain the list of dirtied
- pages is more efficient than traditional software implementations where
- all guest writes trap into Xen so the dirty bitmap can be maintained.
-
- By default, Xen will use PML when it is available in hardware. PML
- functionally depends on A/D tracking, so choosing `ad=0` will implicitly
- disable PML. `pml=0` can be used to prevent the use of PML on otherwise
- capable hardware.
-
-### extra\_guest\_irqs
-> `= [<domU number>][,<dom0 number>]`
-
-> Default: `32,<variable>`
-
-Change the number of PIRQs available for guests. The optional first number is
-common for all domUs, while the optional second number (preceded by a comma)
-is for dom0. Changing the setting for domU has no impact on dom0 and vice
-versa. For example to change dom0 without changing domU, use
-`extra_guest_irqs=,512`. The default value for Dom0 and an eventual separate
-hardware domain is architecture dependent.
-Note that specifying zero as domU value means zero, while for dom0 it means
-to use the default.
-
-### flask
-> `= permissive | enforcing | late | disabled`
-
-> Default: `enforcing`
-
-Specify how the FLASK security server should be configured. This option is only
-available if the hypervisor was compiled with FLASK support. This can be
-enabled by running either:
-- make -C xen config and enabling XSM and FLASK.
-- make -C xen menuconfig and enabling 'FLux Advanced Security Kernel support' and 'Xen Security Modules support'
-
-* `permissive`: This is intended for development and is not suitable for use
- with untrusted guests. If a policy is provided by the bootloader, it will be
- loaded; errors will be reported to the ring buffer but will not prevent
- booting. The policy can be changed to enforcing mode using "xl setenforce".
-* `enforcing`: This will cause the security server to enter enforcing mode prior
- to the creation of domain 0. If an valid policy is not provided by the
- bootloader and no built-in policy is present, the hypervisor will not continue
- booting.
-* `late`: This disables loading of the built-in security policy or the policy
- provided by the bootloader. FLASK will be enabled but will not enforce access
- controls until a policy is loaded by a domain using "xl loadpolicy". Once a
- policy is loaded, FLASK will run in enforcing mode unless "xl setenforce" has
- changed that setting.
-* `disabled`: This causes the XSM framework to revert to the dummy module. The
- dummy module provides the same security policy as is used when compiling the
- hypervisor without support for XSM. The xsm\_op hypercall can also be used to
- switch to this mode after boot, but there is no way to re-enable FLASK once
- the dummy module is loaded.
-
-### font
-> `= <height>` where height is `8x8 | 8x14 | 8x16`
-
-Specify the font size when using the VESA console driver.
-
-### force-ept (Intel)
-> `= <boolean>`
-
-> Default: `false`
-
-Allow EPT to be enabled when VMX feature VM\_ENTRY\_LOAD\_GUEST\_PAT is not
-present.
-
-*Warning:*
-Due to CVE-2013-2212, VMX feature VM\_ENTRY\_LOAD\_GUEST\_PAT is by default
-required as a prerequisite for using EPT. If you are not using PCI Passthrough,
-or trust the guest administrator who would be using passthrough, then the
-requirement can be relaxed. This option is particularly useful for nested
-virtualization, to allow the L1 hypervisor to use EPT even if the L0 hypervisor
-does not provide VM\_ENTRY\_LOAD\_GUEST\_PAT.
-
-### gdb
-> `= com1[H,L] | com2[H,L] | dbgp`
-
-> Default: ``
-
-Specify which console gdbstub should use. See **console**.
-
-### gnttab
-> `= List of [ max-ver:<integer>, transitive=<bool> ]`
-
-> Default: `gnttab=max-ver:2,transitive`
-
-Control various aspects of the grant table behaviour available to guests.
-
-* `max-ver` Select the maximum grant table version to offer to guests. Valid
-version are 1 and 2.
-* `transitive` Permit or disallow the use of transitive grants. Note that the
-use of grant table v2 without transitive grants is an ABI breakage from the
-guests point of view.
-
-The usage of gnttab v2 is not security supported on ARM platforms.
-
-### gnttab\_max\_frames
-> `= <integer>`
-
-> Default: `64`
-
-> Can be modified at runtime
-
-Specify the maximum number of frames which any domain may use as part
-of its grant table. This value is an upper boundary of the per-domain
-value settable via Xen tools.
-
-Dom0 is using this value for sizing its grant table.
-
-### gnttab\_max\_maptrack\_frames
-> `= <integer>`
-
-> Default: `1024`
-
-> Can be modified at runtime
-
-Specify the maximum number of frames to use as part of a domains
-maptrack array. This value is an upper boundary of the per-domain
-value settable via Xen tools.
-
-Dom0 is using this value for sizing its maptrack table.
-
-### guest\_loglvl
-> `= <level>[/<rate-limited level>]` where level is `none | error | warning | info | debug | all`
-
-> Default: `guest_loglvl=none/warning`
-
-> Can be modified at runtime
-
-Set the logging level for Xen guests. Any log message with equal more
-more importance will be printed.
-
-The optional `<rate-limited level>` option instructs which severities
-should be rate limited.
-
-### hap (x86)
-> `= <boolean>`
-
-> Default: `true`
-
-Flag to globally enable or disable support for Hardware Assisted
-Paging (HAP)
-
-### hap\_1gb (x86)
-> `= <boolean>`
-
-> Default: `true`
-
-Flag to enable 1 GB host page table support for Hardware Assisted
-Paging (HAP).
-
-### hap\_2mb (x86)
-> `= <boolean>`
-
-> Default: `true`
-
-Flag to enable 2 MB host page table support for Hardware Assisted
-Paging (HAP).
-
-### hardware\_dom
-> `= <domid>`
-
-> Default: `0`
-
-Enable late hardware domain creation using the specified domain ID. This is
-intended to be used when domain 0 is a stub domain which builds a disaggregated
-system including a hardware domain with the specified domain ID. This option is
-supported only when compiled with XSM on x86.
-
-### hest\_disable
-> ` = <boolean>`
-
-> Default: `false`
-
-Control Xens use of the APEI Hardware Error Source Table, should one be found.
-
-### highmem-start (x86)
-> `= <size>`
-
-Specify the memory boundary past which memory will be treated as highmem (x86
-debug hypervisor only).
-
-### hmp-unsafe (arm)
-> `= <boolean>`
-
-> Default : `false`
-
-Say yes at your own risk if you want to enable heterogenous computing
-(such as big.LITTLE). This may result to an unstable and insecure
-platform, unless you manually specify the cpu affinity of all domains so
-that all vcpus are scheduled on the same class of pcpus (big or LITTLE
-but not both). vcpu migration between big cores and LITTLE cores is not
-supported. See docs/misc/arm/big.LITTLE.txt for more information.
-
-When the hmp-unsafe option is disabled (default), CPUs that are not
-identical to the boot CPU will be parked and not used by Xen.
-
-### hpetbroadcast (x86)
-> `= <boolean>`
-
-### hvm\_debug (x86)
-> `= <integer>`
-
-The specified value is a bit mask with the individual bits having the
-following meaning:
-
-> Bit 0 - debug level 0 (unused at present)
-> Bit 1 - debug level 1 (Control Register logging)
-> Bit 2 - debug level 2 (VMX logging of MSR restores when context switching)
-> Bit 3 - debug level 3 (unused at present)
-> Bit 4 - I/O operation logging
-> Bit 5 - vMMU logging
-> Bit 6 - vLAPIC general logging
-> Bit 7 - vLAPIC timer logging
-> Bit 8 - vLAPIC interrupt logging
-> Bit 9 - vIOAPIC logging
-> Bit 10 - hypercall logging
-> Bit 11 - MSR operation logging
-
-Recognized in debug builds of the hypervisor only.
-
-### hvm\_fep (x86)
-> `= <boolean>`
-
-> Default: `false`
-
-Allow use of the Forced Emulation Prefix in HVM guests, to allow emulation of
-arbitrary instructions.
-
-This option is intended for development and testing purposes.
-
-*Warning*
-As this feature opens up the instruction emulator to arbitrary
-instruction from an HVM guest, don't use this in production system. No
-security support is provided when this flag is set.
-
-### hvm\_port80 (x86)
-> `= <boolean>`
-
-> Default: `true`
-
-Specify whether guests are to be given access to physical port 80
-(often used for debugging purposes), to override the DMI based
-detection of systems known to misbehave upon accesses to that port.
-
-### idle\_latency\_factor (x86)
-> `= <integer>`
-
-### ioapic\_ack (x86)
-> `= old | new`
-
-> Default: `new` unless directed-EOI is supported
-
-### iommu
-> `= List of [ <boolean> | force | required | intremap | intpost | qinval | snoop | sharept | dom0-passthrough | dom0-strict | amd-iommu-perdev-intremap | workaround_bios_bug | igfx | verbose | debug ]`
-
-> Sub-options:
-
-> `<boolean>`
-
-> Default: `on`
-
->> Control the use of IOMMU(s) in the system.
-
-> All other sub-options are of boolean kind and can be prefixed with `no-` to
-> effect the inverse meaning.
-
-> `force` or `required`
-
-> Default: `false`
-
->> Don't continue booting unless IOMMU support is found and can be initialized
->> successfully.
-
-> `intremap`
-
-> Default: `true`
-
->> Control the use of interrupt remapping (DMA remapping will always be enabled
->> if IOMMU functionality is enabled).
-
-> `intpost`
-
-> Default: `false`
-
->> Control the use of interrupt posting, which depends on the availability of
->> interrupt remapping.
-
-> `qinval` (VT-d)
-
-> Default: `true`
-
->> Control the use of Queued Invalidation.
-
-> `snoop` (Intel)
-
-> Default: `true`
-
->> Control the use of Snoop Control.
-
-> `sharept`
-
-> Default: `true`
-
->> Control whether CPU and IOMMU page tables should be shared.
-
-> `dom0-passthrough`
-
-> **WARNING: This command line option is deprecated, and superseded by
-> _dom0-iommu=passthrough_ - using both options in combination is undefined.**
-
-> Default: `false`
-
->> Control whether to disable DMA remapping for Dom0.
-
-> `dom0-strict`
-
-> **WARNING: This command line option is deprecated, and superseded by
-> _dom0-iommu=strict_ - using both options in combination is undefined.**
-
-> Default: `false`
-
->> Control whether to set up DMA remapping only for the memory Dom0 actually
->> got assigned. Implies `no-dom0-passthrough`.
-
-> `amd-iommu-perdev-intremap`
-
-> Default: `true`
-
->> Control whether to set up interrupt remapping data structures per device
->> rather that once for the entire system. Turning this off is making PCI
->> device pass-through insecure and hence unsupported.
-
-> `workaround_bios_bug` (VT-d)
-
-> Default: `false`
-
->> Causes DRHD entries without any PCI discoverable devices under them to be
->> ignored (normally IOMMU setup fails if any of the devices listed by a DRHD
->> entry aren't PCI discoverable).
-
-> `igfx` (VT-d)
-
-> Default: `true`
-
->> Enable IOMMU for Intel graphics devices. The intended usage of this option
->> is `no-igfx`, which is similar to Linux `intel_iommu=igfx_off` option used
->> to workaround graphics issues. If adding `no-igfx` fixes anything, you
->> should file a bug reporting the problem.
-
-> `verbose`
-
-> Default: `false`
-
->> Increase IOMMU code's verbosity.
-
-> `debug`
-
-> Default: `false`
-
->> Enable IOMMU debugging code (implies `verbose`).
-
-### iommu\_dev\_iotlb\_timeout
-> `= <integer>`
-
-> Default: `1000`
-
-Specify the timeout of the device IOTLB invalidation in milliseconds.
-By default, the timeout is 1000 ms. When you see error 'Queue invalidate
-wait descriptor timed out', try increasing this value.
-
-### iommu\_inclusive\_mapping (VT-d)
-> `= <boolean>`
-
-**WARNING: This command line option is deprecated, and superseded by
-_dom0-iommu=map-inclusive_ - using both options in combination is undefined.**
-
-> Default: `true`
-
-Use this to work around firmware issues providing incorrect RMRR entries.
-Rather than only mapping RAM pages for IOMMU accesses for Dom0, with this
-option all pages up to 4GB, not marked as unusable in the E820 table, will
-get a mapping established. Note that this option is only applicable to a
-PV dom0. Also note that if `dom0-strict` mode is enabled then conventional
-RAM pages not assigned to dom0 will not be mapped.
-
-### irq\_ratelimit (x86)
-> `= <integer>`
-
-### irq\_vector\_map (x86)
-### ivrs\_hpet[`<hpet>`] (AMD)
-> `=[<seg>:]<bus>:<device>.<func>`
-
-Force the use of `[<seg>:]<bus>:<device>.<func>` as device ID of HPET
-`<hpet>` instead of the one specified by the IVHD sub-tables of the IVRS
-ACPI table.
-
-### ivrs\_ioapic[`<ioapic>`] (AMD)
-> `=[<seg>:]<bus>:<device>.<func>`
-
-Force the use of `[<seg>:]<bus>:<device>.<func>` as device ID of IO-APIC
-`<ioapic>` instead of the one specified by the IVHD sub-tables of the IVRS
-ACPI table.
-
-### lapic (x86)
-> `= <boolean>`
-
-Force the use of use of the local APIC on a uniprocessor system, even
-if left disabled by the BIOS.
-
-### lapic\_timer\_c2\_ok (x86)
-> `= <boolean>`
-
-### ler (x86)
-> `= <boolean>`
-
-> Default: false
-
-This option is intended for debugging purposes only. Enable MSR_DEBUGCTL.LBR
-in hypervisor context to be able to dump the Last Interrupt/Exception To/From
-record with other registers.
-
-### loglvl
-> `= <level>[/<rate-limited level>]` where level is `none | error | warning | info | debug | all`
-
-> Default: `loglvl=warning`
-
-> Can be modified at runtime
-
-Set the logging level for Xen. Any log message with equal more more
-importance will be printed.
-
-The optional `<rate-limited level>` option instructs which severities
-should be rate limited.
-
-### low\_crashinfo
-> `= none | min | all`
-
-> Default: `none` if not specified at all, or to `min` if **low\_crashinfo** is present without qualification.
-
-This option is only useful for hosts with a 32bit dom0 kernel, wishing
-to use kexec functionality in the case of a crash. It represents
-which data structures should be deliberately allocated in low memory,
-so the crash kernel may find find them. Should be used in combination
-with **crashinfo\_maxaddr**.
-
-### low\_mem\_virq\_limit
-> `= <size>`
-
-> Default: `64M`
-
-Specify the threshold below which Xen will inform dom0 that the quantity of
-free memory is getting low. Specifying `0` will disable this notification.
-
-### maxcpus (x86)
-> `= <integer>`
-
-Specify the maximum number of CPUs that should be brought up.
-
-This option is ignored in **pv-shim** mode.
-
-### max\_cstate (x86)
-> `= <integer>`
-
-### max\_gsi\_irqs (x86)
-> `= <integer>`
-
-Specifies the number of interrupts to be use for pin (IO-APIC or legacy PIC)
-based interrupts. Any higher IRQs will be available for use via PCI MSI.
-
-### max\_lpi\_bits (arm)
-> `= <integer>`
-
-Specifies the number of ARM GICv3 LPI interrupts to allocate on the host,
-presented as the number of bits needed to encode it. This must be at least
-14 and not exceed 32, and each LPI requires one byte (configuration) and
-one pending bit to be allocated.
-Defaults to 20 bits (to cover at most 1048576 interrupts).
-
-### mce (x86)
-> `= <integer>`
-
-### mce\_fb (Intel)
-> `= <integer>`
-
-### mce\_verbosity (x86)
-> `= verbose`
-
-Specify verbose machine check output.
-
-### mem (x86)
-> `= <size>`
-
-Specify the maximum address of physical RAM. Any RAM beyond this
-limit is ignored by Xen.
-
-### memop-max-order
-> `= [<domU>][,[<ctldom>][,[<hwdom>][,<ptdom>]]]`
-
-> x86 default: `9,18,12,12`
-> ARM default: `9,18,10,10`
-
-Change the maximum order permitted for allocation (or allocation-like)
-requests issued by the various kinds of domains (in this order:
-ordinary DomU, control domain, hardware domain, and - when supported
-by the platform - DomU with pass-through device assigned).
-
-### mmcfg (x86)
-> `= <boolean>[,amd-fam10]`
-
-> Default: `1`
-
-Specify if the MMConfig space should be enabled.
-
-### mmio-relax (x86)
-> `= <boolean> | all`
-
-> Default: `false`
-
-By default, domains may not create cached mappings to MMIO regions.
-This option relaxes the check for Domain 0 (or when using `all`, all PV
-domains), to permit the use of cacheable MMIO mappings.
-
-### msi (x86)
-> `= <boolean>`
-
-> Default: `true`
-
-Force Xen to (not) use PCI-MSI, even if ACPI FADT says otherwise.
-
-### mtrr.show (x86)
-> `= <boolean>`
-
-> Default: `false`
-
-Print boot time MTRR state.
-
-### mwait-idle (x86)
-> `= <boolean>`
-
-> Default: `true`
-
-Use the MWAIT idle driver (with model specific C-state knowledge) instead
-of the ACPI based one.
-
-### nmi (x86)
-> `= ignore | dom0 | fatal`
-
-> Default: `fatal` for a debug build, or `dom0` for a non-debug build
-
-Specify what Xen should do in the event of an NMI parity or I/O error.
-`ignore` discards the error; `dom0` causes Xen to report the error to
-dom0, while 'fatal' causes Xen to print diagnostics and then hang.
-
-### noapic (x86)
-
-Instruct Xen to ignore any IOAPICs that are present in the system, and
-instead continue to use the legacy PIC. This is _not_ recommended with
-pvops type kernels.
-
-Because responsibility for APIC setup is shared between Xen and the
-domain 0 kernel this option is automatically propagated to the domain
-0 command line.
-
-### invpcid (x86)
-> `= <boolean>`
-
-> Default: `true`
-
-By default, Xen will use the INVPCID instruction for TLB management if
-it is available. This option can be used to cause Xen to fall back to
-older mechanisms, which are generally slower.
-
-### noirqbalance (x86)
-> `= <boolean>`
-
-Disable software IRQ balancing and affinity. This can be used on
-systems such as Dell 1850/2850 that have workarounds in hardware for
-IRQ routing issues.
-
-### nolapic (x86)
-> `= <boolean>`
-
-> Default: `false`
-
-Ignore the local APIC on a uniprocessor system, even if enabled by the
-BIOS.
-
-### no-real-mode (x86)
-> `= <boolean>`
-
-Do not execute real-mode bootstrap code when booting Xen. This option
-should not be used except for debugging. It will effectively disable
-the **vga** option, which relies on real mode to set the video mode.
-
-### noreboot
-> `= <boolean>`
-
-Do not automatically reboot after an error. This is useful for
-catching debug output. Defaults to automatically reboot after 5
-seconds.
-
-### nosmp (x86)
-> `= <boolean>`
-
-Disable SMP support. No secondary processors will be booted.
-Defaults to booting secondary processors.
-
-This option is ignored in **pv-shim** mode.
-
-### nr\_irqs (x86)
-> `= <integer>`
-
-### numa (x86)
-> `= on | off | fake=<integer> | noacpi`
-
-> Default: `on`
-
-### pci
-> `= {no-}serr | {no-}perr`
-
-> Default: Signaling left as set by firmware.
-
-Disable signaling of SERR (system errors) and/or PERR (parity errors)
-on all PCI devices.
-
-
-### pci-phantom
-> `=[<seg>:]<bus>:<device>,<stride>`
-
-Mark a group of PCI devices as using phantom functions without actually
-advertising so, so the IOMMU can create translation contexts for them.
-
-All numbers specified must be hexadecimal ones.
-
-This option can be specified more than once (up to 8 times at present).
-
-### pcid (x86)
-> `= <boolean> | xpti=<bool>`
-
-> Default: `xpti`
-
-> Can be modified at runtime (change takes effect only for domains created
- afterwards)
-
-If available, control usage of the PCID feature of the processor for
-64-bit pv-domains. PCID can be used either for no domain at all (`false`),
-for all of them (`true`), only for those subject to XPTI (`xpti`) or for
-those not subject to XPTI (`no-xpti`). The feature is used only in case
-INVPCID is supported and not disabled via `invpcid=false`.
-
-### pku (x86)
-> `= <boolean>`
-
-> Default: `true`
-
-Flag to enable Memory Protection Keys.
-
-The protection-key feature provides an additional mechanism by which IA-32e
-paging controls access to usermode addresses.
-
-### ple\_gap
-> `= <integer>`
-
-### ple\_window (Intel)
-> `= <integer>`
-
-### psr (Intel)
-> `= List of ( cmt:<boolean> | rmid_max:<integer> | cat:<boolean> | cos_max:<integer> | cdp:<boolean> )`
-
-> Default: `psr=cmt:0,rmid_max:255,cat:0,cos_max:255,cdp:0`
-
-Platform Shared Resource(PSR) Services. Intel Haswell and later server
-platforms offer information about the sharing of resources.
-
-To use the PSR monitoring service for a certain domain, a Resource
-Monitoring ID(RMID) is used to bind the domain to corresponding shared
-resource. RMID is a hardware-provided layer of abstraction between software
-and logical processors.
-
-To use the PSR cache allocation service for a certain domain, a capacity
-bitmasks(CBM) is used to bind the domain to corresponding shared resource.
-CBM represents cache capacity and indicates the degree of overlap and isolation
-between domains. In hypervisor a Class of Service(COS) ID is allocated for each
-unique CBM.
-
-The following resources are available:
-
-* Cache Monitoring Technology (Haswell and later). Information regarding the
- L3 cache occupancy.
- * `cmt` instructs Xen to enable/disable Cache Monitoring Technology.
- * `rmid_max` indicates the max value for rmid.
-* Memory Bandwidth Monitoring (Broadwell and later). Information regarding the
- total/local memory bandwidth. Follow the same options with Cache Monitoring
- Technology.
-
-* Cache Allocation Technology (Broadwell and later). Information regarding
- the cache allocation.
- * `cat` instructs Xen to enable/disable Cache Allocation Technology.
- * `cos_max` indicates the max value for COS ID.
-* Code and Data Prioritization Technology (Broadwell and later). Information
- regarding the code cache and the data cache allocation. CDP is based on CAT.
- * `cdp` instructs Xen to enable/disable Code and Data Prioritization. Note
- that `cos_max` of CDP is a little different from `cos_max` of CAT. With
- CDP, one COS will corespond two CBMs other than one with CAT, due to the
- sum of CBMs is fixed, that means actual `cos_max` in use will automatically
- reduce to half when CDP is enabled.
-
-### pv-linear-pt (x86)
-> `= <boolean>`
-
-> Default: `true`
-
-Only available if Xen is compiled with CONFIG\_PV\_LINEAR\_PT support
-enabled.
-
-Allow PV guests to have pagetable entries pointing to other pagetables
-of the same level (i.e., allowing L2 PTEs to point to other L2 pages).
-This technique is often called "linear pagetables", and is sometimes
-used to allow operating systems a simple way to consistently map the
-current process's pagetables into its own virtual address space.
-
-Linux and MiniOS don't use this technique. NetBSD and Novell Netware
-do; there may be other custom operating systems which do. If you're
-certain you don't plan on having PV guests which use this feature,
-turning it off can reduce the attack surface.
-
-### pv-l1tf (x86)
-> `= List of [ <bool>, dom0=<bool>, domu=<bool> ]`
-
-> Default: `false` on believed-unaffected hardware, or in pv-shim mode.
-> `domu` on believed-affected hardware.
-
-Mitigations for L1TF / XSA-273 / CVE-2018-3620 for PV guests.
-
-For backwards compatibility, we may not alter an architecturally-legitimate
-pagetable entry a PV guest chooses to write. We can however force such a
-guest into shadow mode so that Xen controls the PTEs which are reachable by
-the CPU pagewalk.
-
-Shadowing is performed at the point where a PV guest first tries to write an
-L1TF-vulnerable PTE. Therefore, a PV guest kernel which has been updated with
-its own L1TF mitigations will not trigger shadow mode if it is well behaved.
-
-If CONFIG\_SHADOW\_PAGING is not compiled in, this mitigation instead crashes
-the guest when an L1TF-vulnerable PTE is written, which still allows updated,
-well-behaved PV guests to run, despite Shadow being compiled out.
-
-In the pv-shim case, Shadow is expected to be compiled out, and a malicious
-guest kernel can only leak data from the shim Xen, rather than the host Xen.
-
-### pv-shim (x86)
-> `= <boolean>`
-
-> Default: `false`
-
-This option is intended for use by a toolstack, when choosing to run a PV
-guest compatibly inside an HVM container.
-
-In this mode, the kernel and initrd passed as modules to the hypervisor are
-constructed into a plain unprivileged PV domain.
-
-### rcu-idle-timer-period-ms
-> `= <integer>`
-
-> Default: `10`
-
-How frequently a CPU which has gone idle, but with pending RCU callbacks,
-should be woken up to check if the grace period has completed, and the
-callbacks are safe to be executed. Expressed in milliseconds; maximum is
-100, and it can't be 0.
-
-### reboot (x86)
-> `= t[riple] | k[bd] | a[cpi] | p[ci] | P[ower] | e[fi] | n[o] [, [w]arm | [c]old]`
-
-> Default: `0`
-
-Specify the host reboot method.
-
-`warm` instructs Xen to not set the cold reboot flag.
-
-`cold` instructs Xen to set the cold reboot flag.
-
-`no` instructs Xen to not automatically reboot after panics or crashes.
-
-`triple` instructs Xen to reboot the host by causing a triple fault.
-
-`kbd` instructs Xen to reboot the host via the keyboard controller.
-
-`acpi` instructs Xen to reboot the host using RESET\_REG in the ACPI FADT.
-
-`pci` instructs Xen to reboot the host using PCI reset register (port CF9).
-
-`Power` instructs Xen to power-cycle the host using PCI reset register (port CF9).
-
-'efi' instructs Xen to reboot using the EFI reboot call (in EFI mode by
- default it will use that method first).
-
-`xen` instructs Xen to reboot using Xen's SCHEDOP hypercall (this is the default
-when running nested Xen)
-
-### rmrr
-> `= start<-end>=[s1]bdf1[,[s1]bdf2[,...]];start<-end>=[s2]bdf1[,[s2]bdf2[,...]]`
-
-Define RMRR units that are missing from ACPI table along with device they
-belong to and use them for 1:1 mapping. End addresses can be omitted and one
-page will be mapped. The ranges are inclusive when start and end are specified.
-If segment of the first device is not specified, segment zero will be used.
-If other segments are not specified, first device segment will be used.
-If a segment is specified for other than the first device and it does not match
-the one specified for the first one, an error will be reported.
-
-'start' and 'end' values are page numbers (not full physical addresses),
-in hexadecimal format (can optionally be preceded by "0x").
-
-Usage example: If device 0:0:1d.0 requires one page (0xd5d45) to be
-reserved, and device 0:0:1a.0 requires three pages (0xd5d46 thru 0xd5d48)
-to be reserved, one usage would be:
-
-rmrr=d5d45=0:0:1d.0;0xd5d46-0xd5d48=0:0:1a.0
-
-Note: grub2 requires to escape or use quotations if special characters are used,
-namely ';', refer to the grub2 documentation if multiple ranges are specified.
-
-### ro-hpet (x86)
-> `= <boolean>`
-
-> Default: `true`
-
-Map the HPET page as read only in Dom0. If disabled the page will be mapped
-with read and write permissions.
-
-### sched
-> `= credit | credit2 | arinc653 | rtds | null`
-
-> Default: `sched=credit`
-
-Choose the default scheduler.
-
-### sched\_credit2\_migrate\_resist
-> `= <integer>`
-
-### sched\_credit\_tslice\_ms
-> `= <integer>`
-
-Set the timeslice of the credit1 scheduler, in milliseconds. The
-default is 30ms. Reasonable values may include 10, 5, or even 1 for
-very latency-sensitive workloads.
-
-### sched\_ratelimit\_us
-> `= <integer>`
-
-In order to limit the rate of context switching, set the minimum
-amount of time that a vcpu can be scheduled for before preempting it,
-in microseconds. The default is 1000us (1ms). Setting this to 0
-disables it altogether.
-
-### sched\_smt\_power\_savings
-> `= <boolean>`
-
-Normally Xen will try to maximize performance and cache utilization by
-spreading out vcpus across as many different divisions as possible
-(i.e, numa nodes, sockets, cores threads, &c). This often maximizes
-throughput, but also maximizes energy usage, since it reduces the
-depth to which a processor can sleep.
-
-This option inverts the logic, so that the scheduler in effect tries
-to keep the vcpus on the smallest amount of silicon possible; i.e.,
-first fill up sibling threads, then sibling cores, then sibling
-sockets, &c. This will reduce performance somewhat, particularly on
-systems with hyperthreading enabled, but should reduce power by
-enabling more sockets and cores to go into deeper sleep states.
-
-### serial\_tx\_buffer
-> `= <size>`
-
-> Default: `16kB`
-
-Set the serial transmit buffer size.
-
-### serrors (ARM)
-> `= diverse | forward | panic`
-
-> Default: `diverse`
-
-This parameter is provided to administrators to determine how the
-hypervisors handle SErrors.
-
-In order to distinguish guest-generated SErrors from hypervisor-generated
-SErrors we have to place SError checking code in every EL1 <-> EL2 paths.
-That will cause overhead on entries and exits due to dsb/isb. However, not all
-platforms need to categorize SErrors. For example, a host that is running with
-trusted guests. The administrator can confirm that all guests that are running
-on the host will not trigger such SErrors. In this case, the administrator can
-use this parameter to skip categorizing SErrors and reduce the overhead of
-dsb/isb.
-
-We provided the following 3 options to administrators to determine how the
-hypervisors handle SErrors:
-
-* `diverse`:
- The hypervisor will distinguish guest SErrors from hypervisor SErrors.
- The guest generated SErrors will be forwarded to guests, the hypervisor
- generated SErrors will cause the whole system to crash.
- It requires:
- 1. dsb/isb on all EL1 -> EL2 trap entries to categorize SErrors correctly.
- 2. dsb/isb on EL2 -> EL1 return paths to prevent slipping hypervisor
- SErrors to guests.
- 3. dsb/isb in context switch to isolate SErrors between 2 vCPUs.
-
-* `forward`:
- The hypervisor will not distinguish guest SErrors from hypervisor SErrors.
- All SErrors will be forwarded to guests, except the SErrors generated when
- the idle vCPU is running. The idle domain doesn't have the ability to handle
- SErrors, so we have to crash the whole system when we get SErros with the
- idle vCPU. This option will avoid most overhead of the dsb/isb, except the
- dsb/isb in context switch which is used to isolate the SErrors between 2
- vCPUs.
-
-* `panic`:
- The hypervisor will not distinguish guest SErrors from hypervisor SErrors.
- All SErrors will crash the whole system. This option will avoid all overhead
- of the dsb/isb pairs.
-
-### shim\_mem (x86)
-> `= List of ( min:<size> | max:<size> | <size> )`
-
-Set the amount of memory that xen-shim uses. Only has effect if pv-shim mode is
-enabled. Note that this value accounts for the memory used by the shim itself
-plus the free memory slack given to the shim for runtime allocations.
-
-* `min:<size>` specifies the minimum amount of memory. Ignored if greater
- than max.
-* `max:<size>` specifies the maximum amount of memory.
-* `<size>` specifies the exact amount of memory. Overrides both min and max.
-
-By default, the amount of free memory slack given to the shim for runtime usage
-is 1MB.
-
-### smap (x86)
-> `= <boolean> | hvm`
-
-> Default: `true`
-
-Flag to enable Supervisor Mode Access Prevention
-Use `smap=hvm` to allow SMAP use by HVM guests only.
-
-### smep (x86)
-> `= <boolean> | hvm`
-
-> Default: `true`
-
-Flag to enable Supervisor Mode Execution Protection
-Use `smep=hvm` to allow SMEP use by HVM guests only.
-
-### smt (x86)
-> `= <boolean>`
-
-Default: `true`
-
-Control bring up of multiple hyper-threads per CPU core.
-
-### snb\_igd\_quirk
-> `= <boolean> | cap | <integer>`
-
-A true boolean value enables legacy behavior (1s timeout), while `cap`
-enforces the maximum theoretically necessary timeout of 670ms. Any number
-is being interpreted as a custom timeout in milliseconds. Zero or boolean
-false disable the quirk workaround, which is also the default.
-
-### spec-ctrl (Arm)
-> `= List of [ ssbd=force-disable|runtime|force-enable ]`
-
-Controls for speculative execution sidechannel mitigations.
-
-The option `ssbd=` is used to control the state of Speculative Store
-Bypass Disable (SSBD) mitigation.
-
-* `ssbd=force-disable` will keep the mitigation permanently off. The guest
-will not be able to control the state of the mitigation.
-* `ssbd=runtime` will always turn on the mitigation when running in the
-hypervisor context. The guest will be to turn on/off the mitigation for
-itself by using the firmware interface ARCH\_WORKAROUND\_2.
-* `ssbd=force-enable` will keep the mitigation permanently on. The guest will
-not be able to control the state of the mitigation.
-
-By default SSBD will be mitigated at runtime (i.e `ssbd=runtime`).
-
-### spec-ctrl (x86)
-> `= List of [ <bool>, xen=<bool>, {pv,hvm,msr-sc,rsb}=<bool>,
-> bti-thunk=retpoline|lfence|jmp, {ibrs,ibpb,ssbd,eager-fpu,
-> l1d-flush}=<bool> ]`
-
-Controls for speculative execution sidechannel mitigations. By default, Xen
-will pick the most appropriate mitigations based on compiled in support,
-loaded microcode, and hardware details, and will virtualise appropriate
-mitigations for guests to use.
-
-**WARNING: Any use of this option may interfere with heuristics. Use with
-extreme care.**
-
-An overall boolean value, `spec-ctrl=no`, can be specified to turn off all
-mitigations, including pieces of infrastructure used to virtualise certain
-mitigation features for guests. This also includes settings which `xpti`,
-`smt`, `pv-l1tf` control, unless the respective option(s) have been
-specified earlier on the command line.
-
-Alternatively, a slightly more restricted `spec-ctrl=no-xen` can be used to
-turn off all of Xen's mitigations, while leaving the virtualisation support
-in place for guests to use.
-
-Use of a positive boolean value for either of these options is invalid.
-
-The booleans `pv=`, `hvm=`, `msr-sc=` and `rsb=` offer fine grained control
-over the alternative blocks used by Xen. These impact Xen's ability to
-protect itself, and Xen's ability to virtualise support for guests to use.
-
-* `pv=` and `hvm=` offer control over all suboptions for PV and HVM guests
- respectively.
-* `msr-sc=` offers control over Xen's support for manipulating MSR\_SPEC\_CTRL
- on entry and exit. These blocks are necessary to virtualise support for
- guests and if disabled, guests will be unable to use IBRS/STIBP/SSBD/etc.
-* `rsb=` offers control over whether to overwrite the Return Stack Buffer /
- Return Address Stack on entry to Xen.
-
-If Xen was compiled with INDIRECT\_THUNK support, `bti-thunk=` can be used to
-select which of the thunks gets patched into the `__x86_indirect_thunk_%reg`
-locations. The default thunk is `retpoline` (generally preferred for Intel
-hardware), with the alternatives being `jmp` (a `jmp *%reg` gadget, minimal
-overhead), and `lfence` (an `lfence; jmp *%reg` gadget, preferred for AMD).
-
-On hardware supporting IBRS (Indirect Branch Restricted Speculation), the
-`ibrs=` option can be used to force or prevent Xen using the feature itself.
-If Xen is not using IBRS itself, functionality is still set up so IBRS can be
-virtualised for guests.
-
-On hardware supporting IBPB (Indirect Branch Prediction Barrier), the `ibpb=`
-option can be used to force (the default) or prevent Xen from issuing branch
-prediction barriers on vcpu context switches.
-
-On hardware supporting SSBD (Speculative Store Bypass Disable), the `ssbd=`
-option can be used to force or prevent Xen using the feature itself. On AMD
-hardware, this is a global option applied at boot, and not virtualised for
-guest use. On Intel hardware, the feature is virtualised for guests,
-independently of Xen's choice of setting.
-
-On all hardware, the `eager-fpu=` option can be used to force or prevent Xen
-from using fully eager FPU context switches. This is currently implemented as
-a global control. By default, Xen will choose to use fully eager context
-switches on hardware believed to speculate past #NM exceptions.
-
-On hardware supporting L1D_FLUSH, the `l1d-flush=` option can be used to force
-or prevent Xen from issuing an L1 data cache flush on each VMEntry.
-Irrespective of Xen's setting, the feature is virtualised for HVM guests to
-use. By default, Xen will enable this mitigation on hardware believed to be
-vulnerable to L1TF.
-
-### sync\_console
-> `= <boolean>`
-
-> Default: `false`
-
-Flag to force synchronous console output. Useful for debugging, but
-not suitable for production environments due to incurred overhead.
-
-### tboot (x86)
-> `= 0x<phys_addr>`
-
-Specify the physical address of the trusted boot shared page.
-
-### tbuf\_size
-> `= <integer>`
-
-Specify the per-cpu trace buffer size in pages.
-
-### tdt (x86)
-> `= <boolean>`
-
-> Default: `true`
-
-Flag to enable TSC deadline as the APIC timer mode.
-
-### tevt\_mask
-> `= <integer>`
-
-Specify a mask for Xen event tracing. This allows Xen tracing to be
-enabled at boot. Refer to the xentrace(8) documentation for a list of
-valid event mask values. In order to enable tracing, a buffer size (in
-pages) must also be specified via the tbuf\_size parameter.
-
-### tickle\_one\_idle\_cpu
-> `= <boolean>`
-
-### timer\_slop
-> `= <integer>`
-
-### tmem
-> `= <boolean>`
-
-### tmem\_compress
-> `= <boolean>`
-
-### tsc (x86)
-> `= unstable | skewed | stable:socket`
-
-### ucode (x86)
-> `= [<integer> | scan]`
-
-Specify how and where to find CPU microcode update blob.
-
-'integer' specifies the CPU microcode update blob module index. When positive,
-this specifies the n-th module (in the GrUB entry, zero based) to be used
-for updating CPU micrcode. When negative, counting starts at the end of
-the modules in the GrUB entry (so with the blob commonly being last,
-one could specify `ucode=-1`). Note that the value of zero is not valid
-here (entry zero, i.e. the first module, is always the Dom0 kernel
-image). Note further that use of this option has an unspecified effect
-when used with xen.efi (there the concept of modules doesn't exist, and
-the blob gets specified via the `ucode=<filename>` config file/section
-entry; see [EFI configuration file description](efi.html)).
-
-'scan' instructs the hypervisor to scan the multiboot images for an cpio
-image that contains microcode. Depending on the platform the blob with the
-microcode in the cpio name space must be:
- - on Intel: kernel/x86/microcode/GenuineIntel.bin
- - on AMD : kernel/x86/microcode/AuthenticAMD.bin
-
-### unrestricted\_guest (Intel)
-> `= <boolean>`
-
-### vcpu\_migration\_delay
-> `= <integer>`
-
-> Default: `0`
-
-Specify a delay, in microseconds, between migrations of a VCPU between
-PCPUs when using the credit1 scheduler. This prevents rapid fluttering
-of a VCPU between CPUs, and reduces the implicit overheads such as
-cache-warming. 1ms (1000) has been measured as a good value.
-
-### vesa-map
-> `= <integer>`
-
-### vesa-mtrr
-> `= <integer>`
-
-### vesa-ram
-> `= <integer>`
-
-### vga
-> `= ( ask | current | text-80x<rows> | gfx-<width>x<height>x<depth> | mode-<mode> )[,keep]`
-
-`ask` causes Xen to display a menu of available modes and request the
-user to choose one of them.
-
-`current` causes Xen to use the graphics adapter in its current state,
-without further setup.
-
-`text-80x<rows>` instructs Xen to set up text mode. Valid values for
-`<rows>` are `25, 28, 30, 34, 43, 50, 80`
-
-`gfx-<width>x<height>x<depth>` instructs Xen to set up graphics mode
-with the specified width, height and depth.
-
-`mode-<mode>` instructs Xen to use a specific mode, as shown with the
-`ask` option. (N.B menu modes are displayed in hex, so `<mode>`
-should be a hexadecimal number)
-
-The optional `keep` parameter causes Xen to continue using the vga
-console even after dom0 has been started. The default behaviour is to
-relinquish control to dom0.
-
-### viridian-spinlock-retry-count (x86)
-> `= <integer>`
-
-> Default: `2047`
-
-Specify the maximum number of retries before an enlightened Windows
-guest will notify Xen that it has failed to acquire a spinlock.
-
-### viridian-version (x86)
-> `= [<major>],[<minor>],[<build>]`
-
-> Default: `6,0,0x1772`
-
-<major>, <minor> and <build> must be integers. The values will be
-encoded in guest CPUID 0x40000002 if viridian enlightenments are enabled.
-
-### vpid (Intel)
-> `= <boolean>`
-
-> Default: `true`
-
-Use Virtual Processor ID support if available. This prevents the need for TLB
-flushes on VM entry and exit, increasing performance.
-
-### vpmu (x86)
-> `= ( <boolean> | { bts | ipc | arch [, ...] } )`
-
-> Default: `off`
-
-Switch on the virtualized performance monitoring unit for HVM guests.
-
-If the current cpu isn't supported a message like
-'VPMU: Initialization failed. ...'
-is printed on the hypervisor serial log.
-
-For some Intel Nehalem processors a quirk handling exist for an unknown
-wrong behaviour (see handle\_pmc\_quirk()).
-
-If 'vpmu=bts' is specified the virtualisation of the Branch Trace Store (BTS)
-feature is switched on on Intel processors supporting this feature.
-
-vpmu=ipc enables performance monitoring, but restricts the counters to the
-most minimum set possible: instructions, cycles, and reference cycles. These
-can be used to calculate instructions per cycle (IPC).
-
-vpmu=arch enables performance monitoring, but restricts the counters to the
-pre-defined architectural events only. These are exposed by cpuid, and listed
-in the Pre-Defined Architectural Performance Events table from the Intel 64
-and IA-32 Architectures Software Developer's Manual, Volume 3B, System
-Programming Guide, Part 2.
-
-If a boolean is not used, combinations of flags are allowed, comma separated.
-For example, vpmu=arch,bts.
-
-Note that if **watchdog** option is also specified vpmu will be turned off.
-
-*Warning:*
-As the virtualisation is not 100% safe, don't use the vpmu flag on
-production systems (see http://xenbits.xen.org/xsa/advisory-163.html)!
-
-### vwfi (arm)
-> `= trap | native`
-
-> Default: `trap`
-
-WFI is the ARM instruction to "wait for interrupt". WFE is similar and
-means "wait for event". This option, which is ARM specific, changes the
-way guest WFI and WFE are implemented in Xen. By default, Xen traps both
-instructions. In the case of WFI, Xen blocks the guest vcpu; in the case
-of WFE, Xen yield the guest vcpu. When setting vwfi to `native`, Xen
-doesn't trap either instruction, running them in guest context. Setting
-vwfi to `native` reduces irq latency significantly. It can also lead to
-suboptimal scheduling decisions, but only when the system is
-oversubscribed (i.e., in total there are more vCPUs than pCPUs).
-
-### watchdog (x86)
-> `= force | <boolean>`
-
-> Default: `false`
-
-Run an NMI watchdog on each processor. If a processor is stuck for
-longer than the **watchdog\_timeout**, a panic occurs. When `force` is
-specified, in addition to running an NMI watchdog on each processor,
-unknown NMIs will still be processed.
-
-### watchdog\_timeout (x86)
-> `= <integer>`
-
-> Default: `5`
-
-Set the NMI watchdog timeout in seconds. Specifying `0` will turn off
-the watchdog.
-
-### x2apic (x86)
-> `= <boolean>`
-
-> Default: `true`
-
-Permit use of x2apic setup for SMP environments.
-
-### x2apic\_phys (x86)
-> `= <boolean>`
-
-> Default: `true` if **FADT** mandates physical mode, `false` otherwise.
-
-In the case that x2apic is in use, this option switches between physical and
-clustered mode. The default, given no hint from the **FADT**, is cluster
-mode.
-
-### xenheap\_megabytes (arm32)
-> `= <size>`
-
-> Default: `0` (1/32 of RAM)
-
-Amount of RAM to set aside for the Xenheap. Must be an integer multiple of 32.
-
-By default will use 1/32 of the RAM up to a maximum of 1GB and with a
-minimum of 32M, subject to a suitably aligned and sized contiguous
-region of memory being available.
-
-### xpti (x86)
-> `= List of [ default | <boolean> | dom0=<bool> | domu=<bool> ]`
-
-> Default: `false` on hardware known not to be vulnerable to Meltdown (e.g. AMD)
-> Default: `true` everywhere else
-
-Override default selection of whether to isolate 64-bit PV guest page
-tables.
-
-`true` activates page table isolation even on hardware not vulnerable by
-Meltdown for all domains.
-
-`false` deactivates page table isolation on all systems for all domains.
-
-`default` sets the default behaviour.
-
-With `dom0` and `domu` it is possible to control page table isolation
-for dom0 or guest domains only.
-
-### xsave (x86)
-> `= <boolean>`
-
-> Default: `true`
-
-Permit use of the `xsave/xrstor` instructions.
-
-### xsm
-> `= dummy | flask | silo`
-
-> Default: selectable via Kconfig. Depends on enabled XSM modules.
-
-Specify which XSM module should be enabled. This option is only available if
-the hypervisor was compiled with CONFIG\_XSM enabled.
-
-* `dummy`: this is the default choice. Basic restriction for common deployment
- (the dummy module) will be applied. It's also used when XSM is compiled out.
-* `flask`: this is the policy based access control. To choose this, the
- separated option in kconfig must also be enabled.
-* `silo`: this will deny any unmediated communication channels between
- unprivileged VMs. To choose this, the separated option in kconfig must also
- be enabled.
--- /dev/null
+# Xen Hypervisor Command Line Options
+
+This document covers the command line options which the Xen
+Hypervisor.
+
+## Types of parameter
+
+Most parameters take the form `option=value`. Different options on
+the command line should be space delimited. All options are case
+sensitive, as are all values unless explicitly noted.
+
+### Boolean (`<boolean>`)
+
+All boolean option may be explicitly enabled using a `value` of
+> `yes`, `on`, `true`, `enable` or `1`
+
+They may be explicitly disabled using a `value` of
+> `no`, `off`, `false`, `disable` or `0`
+
+In addition, a boolean option may be enabled by simply stating its
+name, and may be disabled by prefixing its name with `no-`.
+
+####Examples
+
+Enable noreboot mode
+> `noreboot=true`
+
+Disable x2apic support (if present)
+> `x2apic=off`
+
+Enable synchronous console mode
+> `sync_console`
+
+Explicitly specifying any value other than those listed above is
+undefined, as is stacking a `no-` prefix with an explicit value.
+
+### Integer (`<integer>`)
+
+An integer parameter will default to decimal and may be prefixed with
+a `-` for negative numbers. Alternatively, a hexadecimal number may be
+used by prefixing the number with `0x`, or an octal number may be used
+if a leading `0` is present.
+
+Providing a string which does not validly convert to an integer is
+undefined.
+
+### Size (`<size>`)
+
+A size parameter may be any integer, with a single size suffix
+
+* `T` or `t`: TiB (2^40)
+* `G` or `g`: GiB (2^30)
+* `M` or `m`: MiB (2^20)
+* `K` or `k`: KiB (2^10)
+* `B` or `b`: Bytes
+
+Without a size suffix, the default will be kilo. Providing a suffix
+other than those listed above is undefined.
+
+### String
+
+Many parameters are more complicated and require more intricate
+configuration. The detailed description of each individual parameter
+specify which values are valid.
+
+### List
+
+Some options take a comma separated list of values.
+
+### Combination
+
+Some parameters act as combinations of the above, most commonly a mix
+of Boolean and String. These are noted in the relevant sections.
+
+## Parameter details
+
+### acpi
+> `= force | ht | noirq | <boolean>`
+
+**String**, or **Boolean** to disable.
+
+The **acpi** option is used to control a set of four related boolean
+flags; `acpi_force`, `acpi_ht`, `acpi_noirq` and `acpi_disabled`.
+
+By default, Xen will scan the DMI data and blacklist certain systems
+which are known to have broken ACPI setups. Providing `acpi=force`
+will cause Xen to ignore the blacklist and attempt to use all ACPI
+features.
+
+Using `acpi=ht` causes Xen to parse the ACPI tables enough to
+enumerate all CPUs, but will not use other ACPI features. This is not
+common, and only has an effect if your system is blacklisted.
+
+The `acpi=noirq` option causes Xen to not parse the ACPI MADT table
+looking for IO-APIC entries. This is also not common, and any system
+which requires this option to function should be blacklisted.
+Additionally, this will not prevent Xen from finding IO-APIC entries
+from the MP tables.
+
+Finally, any of the boolean false options can be used to disable ACPI
+usage entirely.
+
+Because responsibility for ACPI processing is shared between Xen and
+the domain 0 kernel this option is automatically propagated to the
+domain 0 command line
+
+### acpi_apic_instance
+> `= <integer>`
+
+Specify which ACPI MADT table to parse for APIC information, if more
+than one is present.
+
+### acpi_pstate_strict (x86)
+> `= <boolean>`
+
+> Default: `false`
+
+Enforce checking that P-state transitions by the ACPI cpufreq driver
+actually result in the nominated frequency to be established. A warning
+message will be logged if that isn't the case.
+
+### acpi_skip_timer_override (x86)
+> `= <boolean>`
+
+Instruct Xen to ignore timer-interrupt override.
+
+### acpi_sleep (x86)
+> `= s3_bios | s3_mode`
+
+`s3_bios` instructs Xen to invoke video BIOS initialization during S3
+resume.
+
+`s3_mode` instructs Xen to set up the boot time (option `vga=`) video
+mode during S3 resume.
+
+### allow_unsafe (x86)
+> `= <boolean>`
+
+> Default: `false`
+
+Force boot on potentially unsafe systems. By default Xen will refuse
+to boot on systems with the following errata:
+
+* AMD Erratum 121. Processors with this erratum are subject to a guest
+ triggerable Denial of Service. Override only if you trust all of
+ your PV guests.
+
+### altp2m (Intel)
+> `= <boolean>`
+
+> Default: `false`
+
+Permit multiple copies of host p2m.
+
+### apic (x86)
+> `= bigsmp | default`
+
+Override Xen's logic for choosing the APIC driver. By default, if
+there are more than 8 CPUs, Xen will switch to `bigsmp` over
+`default`.
+
+### apicv (Intel)
+> `= <boolean>`
+
+> Default: `true`
+
+Permit Xen to use APIC Virtualisation Extensions. This is an optimisation
+available as part of VT-x, and allows hardware to take care of the guests APIC
+handling, rather than requiring emulation in Xen.
+
+### apic_verbosity (x86)
+> `= verbose | debug`
+
+Increase the verbosity of the APIC code from the default value.
+
+### arat (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+Permit Xen to use "Always Running APIC Timer" support on compatible hardware
+in combination with cpuidle. This option is only expected to be useful for
+developers wishing Xen to fall back to older timing methods on newer hardware.
+
+### asid (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+Permit Xen to use Address Space Identifiers. This is an optimisation which
+tags the TLB entries with an ID per vcpu. This allows for guest TLB flushes
+to be performed without the overhead of a complete TLB flush.
+
+### async-show-all (x86)
+> `= <boolean>`
+
+> Default: `false`
+
+Forces all CPUs' full state to be logged upon certain fatal asynchronous
+exceptions (watchdog NMIs and unexpected MCEs).
+
+### ats (x86)
+> `= <boolean>`
+
+> Default: `false`
+
+Permits Xen to set up and use PCI Address Translation Services. This is a
+performance optimisation for PCI Passthrough.
+
+**WARNING: Xen cannot currently safely use ATS because of its synchronous wait
+loops for Queued Invalidation completions.**
+
+### availmem
+> `= <size>`
+
+> Default: `0` (no limit)
+
+Specify a maximum amount of available memory, to which Xen will clamp
+the e820 table.
+
+### badpage
+> `= List of [ <integer> | <integer>-<integer> ]`
+
+Specify that certain pages, or certain ranges of pages contain bad
+bytes and should not be used. For example, if your memory tester says
+that byte `0x12345678` is bad, you would place `badpage=0x12345` on
+Xen's command line.
+
+### bootscrub
+> `= idle | <boolean>`
+
+> Default: `idle`
+
+Scrub free RAM during boot. This is a safety feature to prevent
+accidentally leaking sensitive VM data into other VMs if Xen crashes
+and reboots.
+
+In `idle` mode, RAM is scrubbed in background on all CPUs during idle-loop
+with a guarantee that memory allocations always provide scrubbed pages.
+This option reduces boot time on machines with a large amount of RAM while
+still providing security benefits.
+
+### bootscrub_chunk
+> `= <size>`
+
+> Default: `128M`
+
+Maximum RAM block size chunks to be scrubbed whilst holding the page heap lock
+and not running softirqs. Reduce this if softirqs are not being run frequently
+enough. Setting this to a high value may cause boot failure, particularly if
+the NMI watchdog is also enabled.
+
+### clocksource (x86)
+> `= pit | hpet | acpi | tsc`
+
+If set, override Xen's default choice for the platform timer.
+Having TSC as platform timer requires being explicitly set. This is because
+TSC can only be safely used if CPU hotplug isn't performed on the system. On
+some platforms, the "maxcpus" option may need to be used to further adjust
+the number of allowed CPUs. When running on platforms that can guarantee a
+monotonic TSC across sockets you may want to adjust the "tsc" command line
+parameter to "stable:socket".
+
+### cmci-threshold (Intel)
+> `= <integer>`
+
+> Default: `2`
+
+Specify the event count threshold for raising Corrected Machine Check
+Interrupts. Specifying zero disables CMCI handling.
+
+### cmos-rtc-probe (x86)
+> `= <boolean>`
+
+> Default: `false`
+
+Flag to indicate whether to probe for a CMOS Real Time Clock irrespective of
+ACPI indicating none to be there.
+
+### com1
+### com2
+> `= <baud>[/<base-baud>][,[DPS][,[<io-base>|pci|amt][,[<irq>|msi][,[<port-bdf>][,[<bridge-bdf>]]]]]]`
+
+Both option `com1` and `com2` follow the same format.
+
+* `<baud>` may be either an integer baud rate, or the string `auto` if
+ the bootloader or other earlier firmware has already set it up.
+* Optionally, the base baud rate (usually the highest baud rate the
+ device can communicate at) can be specified.
+* `DPS` represents the number of data bits, the parity, and the number
+ of stop bits.
+ * `D` is an integer between 5 and 8 for the number of data bits.
+ * `P` is a single character representing the type of parity:
+ * `n` No
+ * `o` Odd
+ * `e` Even
+ * `m` Mark
+ * `s` Space
+ * `S` is an integer 1 or 2 for the number of stop bits.
+* `<io-base>` is an integer which specifies the IO base port for UART
+ registers.
+* `<irq>` is the IRQ number to use, or `0` to use the UART in poll
+ mode only, or `msi` to set up a Message Signaled Interrupt.
+* `<port-bdf>` is the PCI location of the UART, in
+ `<bus>:<device>.<function>` notation.
+* `<bridge-bdf>` is the PCI bridge behind which is the UART, in
+ `<bus>:<device>.<function>` notation.
+* `pci` indicates that Xen should scan the PCI bus for the UART,
+ avoiding Intel AMT devices.
+* `amt` indicated that Xen should scan the PCI bus for the UART,
+ including Intel AMT devices if present.
+
+A typical setup for most situations might be `com1=115200,8n1`
+
+In addition to the above positional specification for UART parameters,
+name=value pair specfications are also supported. This is used to add
+flexibility for UART devices which require additional UART parameter
+configurations.
+
+The comma separation still delineates positional parameters. Hence,
+unless the parameter is explicitly specified with name=value option, it
+will be considered a positional parameter.
+
+The syntax consists of
+com1=(comma-separated positional parameters),(comma separated name-value pairs)
+
+The accepted name keywords for name=value pairs are:
+
+* `baud` - accepts integer baud rate (eg. 115200) or `auto`
+* `bridge`- Similar to bridge-bdf in positional parameters.
+ Used to determine the PCI bridge to access the UART device.
+ Notation is xx:xx.x `<bus>:<device>.<function>`
+* `clock-hz`- accepts large integers to setup UART clock frequencies.
+ Do note - these values are multiplied by 16.
+* `data-bits` - integer between 5 and 8
+* `dev` - accepted values are `pci` OR `amt`. If this option
+ is used to specify if the serial device is pci-based. The io_base
+ cannot be specified when `dev=pci` or `dev=amt` is used.
+* `io-base` - accepts integer which specified IO base port for UART registers
+* `irq` - IRQ number to use
+* `parity` - accepted values are same as positional parameters
+* `port` - Used to specify which port the PCI serial device is located on
+ Notation is xx:xx.x `<bus>:<device>.<function>`
+* `reg-shift` - register shifts required to set UART registers
+* `reg-width` - register width required to set UART registers
+ (only accepts 1 and 4)
+* `stop-bits` - only accepts 1 or 2 for the number of stop bits
+
+The following are examples of correct specifications:
+
+ com1=115200,8n1,0x3f8,4
+ com1=115200,8n1,0x3f8,4,reg_width=4,reg_shift=2
+ com1=baud=115200,parity=n,stop_bits=1,io_base=0x3f8,reg_width=4
+
+### conring_size
+> `= <size>`
+
+> Default: `conring_size=16k`
+
+Specify the size of the console ring buffer.
+
+### console
+> `= List of [ vga | com1[H,L] | com2[H,L] | pv | dbgp | none ]`
+
+> Default: `console=com1,vga`
+
+Specify which console(s) Xen should use.
+
+`vga` indicates that Xen should try and use the vga graphics adapter.
+
+`com1` and `com2` indicates that Xen should use serial ports 1 and 2
+respectively. Optionally, these arguments may be followed by an `H` or
+`L`. `H` indicates that transmitted characters will have their MSB
+set, while received characters must have their MSB set. `L` indicates
+the converse; transmitted and received characters will have their MSB
+cleared. This allows a single port to be shared by two subsystems
+(e.g. console and debugger).
+
+`pv` indicates that Xen should use Xen's PV console. This option is
+only available when used together with `pv-in-pvh`.
+
+`dbgp` indicates that Xen should use a USB debug port.
+
+`none` indicates that Xen should not use a console. This option only
+makes sense on its own.
+
+### console_timestamps
+> `= none | date | datems | boot | raw`
+
+> Default: `none`
+
+> Can be modified at runtime
+
+Specify which timestamp format Xen should use for each console line.
+
+* `none`: No timestamps
+* `date`: Date and time information
+ * `[YYYY-MM-DD HH:MM:SS]`
+* `datems`: Date and time, with milliseconds
+ * `[YYYY-MM-DD HH:MM:SS.mmm]`
+* `boot`: Seconds and microseconds since boot
+ * `[SSSSSS.uuuuuu]`
++ `raw`: Raw platform ticks, architecture and implementation dependent
+ * `[XXXXXXXXXXXXXXXX]`
+
+For compatibility with the older boolean parameter, specifying
+`console_timestamps` alone will enable the `date` option.
+
+### console_to_ring
+> `= <boolean>`
+
+> Default: `false`
+
+Flag to indicate whether all guest console output should be copied
+into the console ring buffer.
+
+### conswitch
+> `= <switch char>[x]`
+
+> Default: `conswitch=a`
+
+> Can be modified at runtime
+
+Specify which character should be used to switch serial input between
+Xen and dom0. The required sequence is CTRL-<switch char> three
+times.
+
+The optional trailing `x` indicates that Xen should not automatically
+switch the console input to dom0 during boot. Any other value,
+including omission, causes Xen to automatically switch to the dom0
+console during dom0 boot. Use `conswitch=ax` to keep the default switch
+character, but for xen to keep the console.
+
+### core_parking
+> `= power | performance`
+
+> Default: `power`
+
+### cpu_type (x86)
+> `= arch_perfmon`
+
+If set, force use of the performance counters for oprofile, rather than detecting
+available support.
+
+### cpufreq
+> `= none | {{ <boolean> | xen } [:[powersave|performance|ondemand|userspace][,<maxfreq>][,[<minfreq>][,[verbose]]]]} | dom0-kernel`
+
+> Default: `xen`
+
+Indicate where the responsibility for driving power states lies. Note that the
+choice of `dom0-kernel` is deprecated and not supported by all Dom0 kernels.
+
+* Default governor policy is ondemand.
+* `<maxfreq>` and `<minfreq>` are integers which represent max and min processor frequencies
+ respectively.
+* `verbose` option can be included as a string or also as `verbose=<integer>`
+
+### cpuid (x86)
+> `= List of comma separated booleans`
+
+This option allows for fine tuning of the facilities Xen will use, after
+accounting for hardware capabilities as enumerated via CPUID.
+
+Currently accepted:
+
+The Speculation Control hardware features `ibrsb`, `stibp`, `ibpb`,
+`l1d-flush` and `ssbd` are used by default if available and applicable. They can
+be ignored, e.g. `no-ibrsb`, at which point Xen won't use them itself, and
+won't offer them to guests.
+
+### cpuid_mask_cpu
+> `= fam_0f_rev_[cdefg] | fam_10_rev_[bc] | fam_11_rev_b`
+
+> Applicability: AMD
+
+If none of the other **cpuid_mask_\*** options are given, Xen has a set of
+pre-configured masks to make the current processor appear to be
+family/revision specified.
+
+See below for general information on masking.
+
+**Warning: This option is not fully effective on Family 15h processors or
+later.**
+
+### cpuid_mask_ecx
+### cpuid_mask_edx
+### cpuid_mask_ext_ecx
+### cpuid_mask_ext_edx
+### cpuid_mask_l7s0_eax
+### cpuid_mask_l7s0_ebx
+### cpuid_mask_thermal_ecx
+### cpuid_mask_xsave_eax
+> `= <integer>`
+
+> Applicability: x86. Default: `~0` (all bits set)
+
+The availability of these options are model specific. Some processors don't
+support any of them, and no processor supports all of them. Xen will ignore
+options on processors which are lacking support.
+
+These options can be used to alter the features visible via the `CPUID`
+instruction. Settings applied here take effect globally, including for Xen
+and all guests.
+
+Note: Since Xen 4.7, it is no longer necessary to mask a host to create
+migration safety in heterogeneous scenarios. All necessary CPUID settings
+should be provided in the VM configuration file. Furthermore, it is
+recommended not to use this option, as doing so causes an unnecessary
+reduction of features at Xen's disposal to manage guests.
+
+### cpuidle (x86)
+> `= <boolean>`
+
+### cpuinfo (x86)
+> `= <boolean>`
+
+### crashinfo_maxaddr
+> `= <size>`
+
+> Default: `4G`
+
+Specify the maximum address to allocate certain structures, if used in
+combination with the **low_crashinfo** command line option.
+
+### crashkernel
+> `= <ramsize-range>:<size>[,...][{@,<}<offset>]`
+> `= <size>[{@,<}<offset>]`
+> `= <size>,below=offset`
+
+Specify sizes and optionally placement of the crash kernel reservation
+area. The `<ramsize-range>:<size>` pairs indicate how much memory to
+set aside for a crash kernel (`<size>`) for a given range of installed
+RAM (`<ramsize-range>`). Each `<ramsize-range>` is of the form
+`<start>-[<end>]`.
+
+A trailing `@<offset>` specifies the exact address this area should be
+placed at, whereas `<` in place of `@` just specifies an upper bound of
+the address range the area should fall into.
+
+< and below are synonyomous, the latter being useful for grub2 systems
+which would otherwise require escaping of the < option
+
+
+### credit2_balance_over
+> `= <integer>`
+
+### credit2_balance_under
+> `= <integer>`
+
+### credit2_cap_period_ms
+> `= <integer>`
+
+> Default: `10`
+
+Domains subject to a cap receive a replenishment of their runtime budget
+once every cap period interval. Default is 10 ms. The amount of budget
+they receive depends on their cap. For instance, a domain with a 50% cap
+will receive 50% of 10 ms, so 5 ms.
+
+### credit2_load_precision_shift
+> `= <integer>`
+
+> Default: `18`
+
+Specify the number of bits to use for the fractional part of the
+values involved in Credit2 load tracking and load balancing math.
+
+### credit2_load_window_shift
+> `= <integer>`
+
+> Default: `30`
+
+Specify the number of bits to use to represent the length of the
+window (in nanoseconds) we use for load tracking inside Credit2.
+This means that, with the default value (30), we use
+2^30 nsec ~= 1 sec long window.
+
+Load tracking is done by means of a variation of exponentially
+weighted moving average (EWMA). The window length defined here
+is what tells for how long we give value to previous history
+of the load itself. In fact, after a full window has passed,
+what happens is that we discard all previous history entirely.
+
+A short window will make the load balancer quick at reacting
+to load changes, but also short-sighted about previous history
+(and hence, e.g., long term load trends). A long window will
+make the load balancer thoughtful of previous history (and
+hence capable of capturing, e.g., long term load trends), but
+also slow in responding to load changes.
+
+The default value of `1 sec` is rather long.
+
+### credit2_runqueue
+> `= cpu | core | socket | node | all`
+
+> Default: `socket`
+
+Specify how host CPUs are arranged in runqueues. Runqueues are kept
+balanced with respect to the load generated by the vCPUs running on
+them. Smaller runqueues (as in with `core`) means more accurate load
+balancing (for instance, it will deal better with hyperthreading),
+but also more overhead.
+
+Available alternatives, with their meaning, are:
+* `cpu`: one runqueue per each logical pCPUs of the host;
+* `core`: one runqueue per each physical core of the host;
+* `socket`: one runqueue per each physical socket (which often,
+ but not always, matches a NUMA node) of the host;
+* `node`: one runqueue per each NUMA node of the host;
+* `all`: just one runqueue shared by all the logical pCPUs of
+ the host
+
+### dbgp
+> `= ehci[ <integer> | @pci<bus>:<slot>.<func> ]`
+
+Specify the USB controller to use, either by instance number (when going
+over the PCI busses sequentially) or by PCI device (must be on segment 0).
+
+### debug_stack_lines
+> `= <integer>`
+
+> Default: `20`
+
+Limits the number lines printed in Xen stack traces.
+
+### debugtrace
+> `= <integer>`
+
+> Default: `128`
+
+Specify the size of the console debug trace buffer in KiB. The debug
+trace feature is only enabled in debugging builds of Xen.
+
+### dma_bits
+> `= <integer>`
+
+Specify the bit width of the DMA heap.
+
+### dom0 (x86)
+> `= List of [ pvh | shadow ]`
+
+> Sub-options:
+
+> `pvh`
+
+> Default: `false`
+
+Flag that makes a dom0 boot in PVHv2 mode.
+
+> `shadow`
+
+> Default: `false`
+
+Flag that makes a dom0 use shadow paging. Only works when "pvh" is
+enabled.
+
+### dom0-iommu
+> `= List of [ passthrough | strict | map-inclusive ]`
+
+This list of booleans controls the iommu usage by Dom0:
+
+* `passthrough`: disables DMA remapping for Dom0. Default is `false`. Note that
+ this option is hard coded to `false` for a PVH Dom0 and any attempt to
+ overwrite it from the command line is ignored.
+
+* `strict`: sets up DMA remapping only for the RAM Dom0 actually got assigned.
+ Default is `false` which means Dom0 will get mappings for all the host
+ RAM except regions in use by Xen. Note that this option is hard coded to
+ `true` for a PVH Dom0 and any attempt to overwrite it from the command line
+ is ignored.
+
+* `map-inclusive`: sets up DMA remapping for all the non-RAM regions below 4GB
+ except for unusable ranges. Use this to work around firmware issues providing
+ incorrect RMRR/IVMD entries. Rather than only mapping RAM pages for IOMMU
+ accesses for Dom0, with this option all pages up to 4GB, not marked as
+ unusable in the E820 table, will get a mapping established. Note that this
+ option is only applicable to a PV Dom0 and is enabled by default on Intel
+ hardware.
+
+* `map-reserved`: sets up DMA remapping for all the reserved regions in the
+ memory map for Dom0. Use this to work around firmware issues providing
+ incorrect RMRR/IVMD entries. Rather than only mapping RAM pages for IOMMU
+ accesses for Dom0, all memory regions marked as reserved in the memory map
+ that don't overlap with any MMIO region from emulated devices will be
+ identity mapped. This option maps a subset of the memory that would be
+ mapped when using the `map-inclusive` option. This option is available to all
+ Dom0 modes and is enabled by default on Intel hardware.
+
+### dom0_ioports_disable (x86)
+> `= List of <hex>-<hex>`
+
+Specify a list of IO ports to be excluded from dom0 access.
+
+### dom0_max_vcpus
+
+Either:
+
+> `= <integer>`.
+
+The number of VCPUs to give to dom0. This number of VCPUs can be more
+than the number of PCPUs on the host. The default is the number of
+PCPUs.
+
+Or:
+
+> `= <min>-<max>` where `<min>` and `<max>` are integers.
+
+Gives dom0 a number of VCPUs equal to the number of PCPUs, but always
+at least `<min>` and no more than `<max>`. Using `<min>` may give
+more VCPUs than PCPUs. `<min>` or `<max>` may be omitted and the
+defaults of 1 and unlimited respectively are used instead.
+
+For example, with `dom0_max_vcpus=4-8`:
+
+> Number of
+> PCPUs | Dom0 VCPUs
+> 2 | 4
+> 4 | 4
+> 6 | 6
+> 8 | 8
+> 10 | 8
+
+### dom0_mem (ARM)
+> `= <size>`
+
+Set the amount of memory for the initial domain (dom0). It must be
+greater than zero. This parameter is required.
+
+### dom0_mem (x86)
+> `= List of ( min:<sz> | max:<sz> | <sz> )`
+
+Set the amount of memory for the initial domain (dom0). If a size is
+positive, it represents an absolute value. If a size is negative, it
+is subtracted from the total available memory.
+
+* `<sz>` specifies the exact amount of memory.
+* `min:<sz>` specifies the minimum amount of memory.
+* `max:<sz>` specifies the maximum amount of memory.
+
+If `<sz>` is not specified, the default is all the available memory
+minus some reserve. The reserve is 1/16 of the available memory or
+128 MB (whichever is smaller).
+
+The amount of memory will be at least the minimum but never more than
+the maximum (i.e., `max` overrides the `min` option). If there isn't
+enough memory then as much as possible is allocated.
+
+`max:<sz>` also sets the maximum reservation (the maximum amount of
+memory dom0 can balloon up to). If this is omitted then the maximum
+reservation is unlimited.
+
+For example, to set dom0's initial memory allocation to 512MB but
+allow it to balloon up as far as 1GB use `dom0_mem=512M,max:1G`
+
+> `<sz>` is: `<size> | [<size>+]<frac>%`
+> `<frac>` is an integer < 100
+
+* `<frac>` specifies a fraction of host memory size in percent.
+
+So `<sz>` being `1G+25%` on a 256 GB host would result in 65 GB.
+
+If you use this option then it is highly recommended that you disable
+any dom0 autoballooning feature present in your toolstack. See the
+_xl.conf(5)_ man page or [Xen Best
+Practices](http://wiki.xen.org/wiki/Xen_Best_Practices#Xen_dom0_dedicated_memory_and_preventing_dom0_memory_ballooning).
+
+This option doesn't have effect if pv-shim mode is enabled.
+
+### dom0_nodes (x86)
+
+> `= List of [ <integer> | relaxed | strict ]`
+
+> Default: `strict`
+
+Specify the NUMA nodes to place Dom0 on. Defaults for vCPU-s created
+and memory assigned to Dom0 will be adjusted to match the node
+restrictions set up here. Note that the values to be specified here are
+ACPI PXM ones, not Xen internal node numbers. `relaxed` sets up vCPU
+affinities to prefer but be not limited to the specified node(s).
+
+### dom0_vcpus_pin
+> `= <boolean>`
+
+> Default: `false`
+
+Pin dom0 vcpus to their respective pcpus
+
+### dtuart (ARM)
+> `= path [:options]`
+
+> Default: `""`
+
+Specify the full path in the device tree for the UART. If the path doesn't
+start with `/`, it is assumed to be an alias. The options are device specific.
+
+### e820-mtrr-clip (x86)
+> `= <boolean>`
+
+Flag that specifies if RAM should be clipped to the highest cacheable
+MTRR.
+
+> Default: `true` on Intel CPUs, otherwise `false`
+
+### e820-verbose (x86)
+> `= <boolean>`
+
+> Default: `false`
+
+Flag that enables verbose output when processing e820 information and
+applying clipping.
+
+### edd (x86)
+> `= off | on | skipmbr`
+
+Control retrieval of Extended Disc Data (EDD) from the BIOS during
+boot.
+
+### edid (x86)
+> `= no | force`
+
+Either force retrieval of monitor EDID information via VESA DDC, or
+disable it (edid=no). This option should not normally be required
+except for debugging purposes.
+
+### efi
+> `= List of [ rs | attr ]`
+
+All options are of boolean kind and can be prefixed with `no-` to
+effect the inverse meaning.
+
+> `rs`
+
+> Default: `true`
+
+>> Force or disable use of EFI runtime services.
+
+> `attr=uc`
+
+> Default: `off`
+
+>> Allows mapping of RuntimeServices which have no cachability attribute
+>> set as UC.
+
+### ept
+> `= List of [ ad=<bool>, pml=<bool> ]`
+
+> Applicability: Intel
+
+Extended Page Tables are a feature of Intel's VT-x technology, whereby
+hardware manages the virtualisation of HVM guest pagetables. EPT was
+introduced with the Nehalem architecture.
+
+* The `ad` boolean controls hardware tracking of Access and Dirty bits in the
+ EPT pagetables, and was first introduced in Broadwell Server.
+
+ By default, Xen will use A/D tracking when available in hardware, except
+ on Avoton processors affected by erratum AVR41. Explicitly choosing
+ `ad=0` will disable the use of A/D tracking on capable hardware, whereas
+ choosing `ad=1` will cause tracking to be used even on AVR41-affected
+ hardware.
+
+* The `pml` boolean controls the use of Page Modification Logging, which is
+ also introduced in Broadwell Server.
+
+ PML is a feature whereby the processor generates a list of pages which
+ have been dirtied. This is necessary information for operations such as
+ live migration, and having the processor maintain the list of dirtied
+ pages is more efficient than traditional software implementations where
+ all guest writes trap into Xen so the dirty bitmap can be maintained.
+
+ By default, Xen will use PML when it is available in hardware. PML
+ functionally depends on A/D tracking, so choosing `ad=0` will implicitly
+ disable PML. `pml=0` can be used to prevent the use of PML on otherwise
+ capable hardware.
+
+### extra_guest_irqs
+> `= [<domU number>][,<dom0 number>]`
+
+> Default: `32,<variable>`
+
+Change the number of PIRQs available for guests. The optional first number is
+common for all domUs, while the optional second number (preceded by a comma)
+is for dom0. Changing the setting for domU has no impact on dom0 and vice
+versa. For example to change dom0 without changing domU, use
+`extra_guest_irqs=,512`. The default value for Dom0 and an eventual separate
+hardware domain is architecture dependent.
+Note that specifying zero as domU value means zero, while for dom0 it means
+to use the default.
+
+### flask
+> `= permissive | enforcing | late | disabled`
+
+> Default: `enforcing`
+
+Specify how the FLASK security server should be configured. This option is only
+available if the hypervisor was compiled with FLASK support. This can be
+enabled by running either:
+- make -C xen config and enabling XSM and FLASK.
+- make -C xen menuconfig and enabling 'FLux Advanced Security Kernel support' and 'Xen Security Modules support'
+
+* `permissive`: This is intended for development and is not suitable for use
+ with untrusted guests. If a policy is provided by the bootloader, it will be
+ loaded; errors will be reported to the ring buffer but will not prevent
+ booting. The policy can be changed to enforcing mode using "xl setenforce".
+* `enforcing`: This will cause the security server to enter enforcing mode prior
+ to the creation of domain 0. If an valid policy is not provided by the
+ bootloader and no built-in policy is present, the hypervisor will not continue
+ booting.
+* `late`: This disables loading of the built-in security policy or the policy
+ provided by the bootloader. FLASK will be enabled but will not enforce access
+ controls until a policy is loaded by a domain using "xl loadpolicy". Once a
+ policy is loaded, FLASK will run in enforcing mode unless "xl setenforce" has
+ changed that setting.
+* `disabled`: This causes the XSM framework to revert to the dummy module. The
+ dummy module provides the same security policy as is used when compiling the
+ hypervisor without support for XSM. The xsm_op hypercall can also be used to
+ switch to this mode after boot, but there is no way to re-enable FLASK once
+ the dummy module is loaded.
+
+### font
+> `= <height>` where height is `8x8 | 8x14 | 8x16`
+
+Specify the font size when using the VESA console driver.
+
+### force-ept (Intel)
+> `= <boolean>`
+
+> Default: `false`
+
+Allow EPT to be enabled when VMX feature `VM_ENTRY_LOAD_GUEST_PAT` is not
+present.
+
+*Warning:*
+Due to CVE-2013-2212, VMX feature `VM_ENTRY_LOAD_GUEST_PAT` is by default
+required as a prerequisite for using EPT. If you are not using PCI Passthrough,
+or trust the guest administrator who would be using passthrough, then the
+requirement can be relaxed. This option is particularly useful for nested
+virtualization, to allow the L1 hypervisor to use EPT even if the L0 hypervisor
+does not provide `VM_ENTRY_LOAD_GUEST_PAT`.
+
+### gdb
+> `= com1[H,L] | com2[H,L] | dbgp`
+
+> Default: ``
+
+Specify which console gdbstub should use. See **console**.
+
+### gnttab
+> `= List of [ max-ver:<integer>, transitive=<bool> ]`
+
+> Default: `gnttab=max-ver:2,transitive`
+
+Control various aspects of the grant table behaviour available to guests.
+
+* `max-ver` Select the maximum grant table version to offer to guests. Valid
+version are 1 and 2.
+* `transitive` Permit or disallow the use of transitive grants. Note that the
+use of grant table v2 without transitive grants is an ABI breakage from the
+guests point of view.
+
+The usage of gnttab v2 is not security supported on ARM platforms.
+
+### gnttab_max_frames
+> `= <integer>`
+
+> Default: `64`
+
+> Can be modified at runtime
+
+Specify the maximum number of frames which any domain may use as part
+of its grant table. This value is an upper boundary of the per-domain
+value settable via Xen tools.
+
+Dom0 is using this value for sizing its grant table.
+
+### gnttab_max_maptrack_frames
+> `= <integer>`
+
+> Default: `1024`
+
+> Can be modified at runtime
+
+Specify the maximum number of frames to use as part of a domains
+maptrack array. This value is an upper boundary of the per-domain
+value settable via Xen tools.
+
+Dom0 is using this value for sizing its maptrack table.
+
+### guest_loglvl
+> `= <level>[/<rate-limited level>]` where level is `none | error | warning | info | debug | all`
+
+> Default: `guest_loglvl=none/warning`
+
+> Can be modified at runtime
+
+Set the logging level for Xen guests. Any log message with equal more
+more importance will be printed.
+
+The optional `<rate-limited level>` option instructs which severities
+should be rate limited.
+
+### hap (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+Flag to globally enable or disable support for Hardware Assisted
+Paging (HAP)
+
+### hap_1gb (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+Flag to enable 1 GB host page table support for Hardware Assisted
+Paging (HAP).
+
+### hap_2mb (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+Flag to enable 2 MB host page table support for Hardware Assisted
+Paging (HAP).
+
+### hardware_dom
+> `= <domid>`
+
+> Default: `0`
+
+Enable late hardware domain creation using the specified domain ID. This is
+intended to be used when domain 0 is a stub domain which builds a disaggregated
+system including a hardware domain with the specified domain ID. This option is
+supported only when compiled with XSM on x86.
+
+### hest_disable
+> ` = <boolean>`
+
+> Default: `false`
+
+Control Xens use of the APEI Hardware Error Source Table, should one be found.
+
+### highmem-start (x86)
+> `= <size>`
+
+Specify the memory boundary past which memory will be treated as highmem (x86
+debug hypervisor only).
+
+### hmp-unsafe (arm)
+> `= <boolean>`
+
+> Default : `false`
+
+Say yes at your own risk if you want to enable heterogenous computing
+(such as big.LITTLE). This may result to an unstable and insecure
+platform, unless you manually specify the cpu affinity of all domains so
+that all vcpus are scheduled on the same class of pcpus (big or LITTLE
+but not both). vcpu migration between big cores and LITTLE cores is not
+supported. See docs/misc/arm/big.LITTLE.txt for more information.
+
+When the hmp-unsafe option is disabled (default), CPUs that are not
+identical to the boot CPU will be parked and not used by Xen.
+
+### hpetbroadcast (x86)
+> `= <boolean>`
+
+### hvm_debug (x86)
+> `= <integer>`
+
+The specified value is a bit mask with the individual bits having the
+following meaning:
+
+> Bit 0 - debug level 0 (unused at present)
+> Bit 1 - debug level 1 (Control Register logging)
+> Bit 2 - debug level 2 (VMX logging of MSR restores when context switching)
+> Bit 3 - debug level 3 (unused at present)
+> Bit 4 - I/O operation logging
+> Bit 5 - vMMU logging
+> Bit 6 - vLAPIC general logging
+> Bit 7 - vLAPIC timer logging
+> Bit 8 - vLAPIC interrupt logging
+> Bit 9 - vIOAPIC logging
+> Bit 10 - hypercall logging
+> Bit 11 - MSR operation logging
+
+Recognized in debug builds of the hypervisor only.
+
+### hvm_fep (x86)
+> `= <boolean>`
+
+> Default: `false`
+
+Allow use of the Forced Emulation Prefix in HVM guests, to allow emulation of
+arbitrary instructions.
+
+This option is intended for development and testing purposes.
+
+*Warning*
+As this feature opens up the instruction emulator to arbitrary
+instruction from an HVM guest, don't use this in production system. No
+security support is provided when this flag is set.
+
+### hvm_port80 (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+Specify whether guests are to be given access to physical port 80
+(often used for debugging purposes), to override the DMI based
+detection of systems known to misbehave upon accesses to that port.
+
+### idle_latency_factor (x86)
+> `= <integer>`
+
+### ioapic_ack (x86)
+> `= old | new`
+
+> Default: `new` unless directed-EOI is supported
+
+### iommu
+> `= List of [ <boolean> | force | required | intremap | intpost | qinval | snoop | sharept | dom0-passthrough | dom0-strict | amd-iommu-perdev-intremap | workaround_bios_bug | igfx | verbose | debug ]`
+
+> Sub-options:
+
+> `<boolean>`
+
+> Default: `on`
+
+>> Control the use of IOMMU(s) in the system.
+
+> All other sub-options are of boolean kind and can be prefixed with `no-` to
+> effect the inverse meaning.
+
+> `force` or `required`
+
+> Default: `false`
+
+>> Don't continue booting unless IOMMU support is found and can be initialized
+>> successfully.
+
+> `intremap`
+
+> Default: `true`
+
+>> Control the use of interrupt remapping (DMA remapping will always be enabled
+>> if IOMMU functionality is enabled).
+
+> `intpost`
+
+> Default: `false`
+
+>> Control the use of interrupt posting, which depends on the availability of
+>> interrupt remapping.
+
+> `qinval` (VT-d)
+
+> Default: `true`
+
+>> Control the use of Queued Invalidation.
+
+> `snoop` (Intel)
+
+> Default: `true`
+
+>> Control the use of Snoop Control.
+
+> `sharept`
+
+> Default: `true`
+
+>> Control whether CPU and IOMMU page tables should be shared.
+
+> `dom0-passthrough`
+
+> **WARNING: This command line option is deprecated, and superseded by
+> _dom0-iommu=passthrough_ - using both options in combination is undefined.**
+
+> Default: `false`
+
+>> Control whether to disable DMA remapping for Dom0.
+
+> `dom0-strict`
+
+> **WARNING: This command line option is deprecated, and superseded by
+> _dom0-iommu=strict_ - using both options in combination is undefined.**
+
+> Default: `false`
+
+>> Control whether to set up DMA remapping only for the memory Dom0 actually
+>> got assigned. Implies `no-dom0-passthrough`.
+
+> `amd-iommu-perdev-intremap`
+
+> Default: `true`
+
+>> Control whether to set up interrupt remapping data structures per device
+>> rather that once for the entire system. Turning this off is making PCI
+>> device pass-through insecure and hence unsupported.
+
+> `workaround_bios_bug` (VT-d)
+
+> Default: `false`
+
+>> Causes DRHD entries without any PCI discoverable devices under them to be
+>> ignored (normally IOMMU setup fails if any of the devices listed by a DRHD
+>> entry aren't PCI discoverable).
+
+> `igfx` (VT-d)
+
+> Default: `true`
+
+>> Enable IOMMU for Intel graphics devices. The intended usage of this option
+>> is `no-igfx`, which is similar to Linux `intel_iommu=igfx_off` option used
+>> to workaround graphics issues. If adding `no-igfx` fixes anything, you
+>> should file a bug reporting the problem.
+
+> `verbose`
+
+> Default: `false`
+
+>> Increase IOMMU code's verbosity.
+
+> `debug`
+
+> Default: `false`
+
+>> Enable IOMMU debugging code (implies `verbose`).
+
+### iommu_dev_iotlb_timeout
+> `= <integer>`
+
+> Default: `1000`
+
+Specify the timeout of the device IOTLB invalidation in milliseconds.
+By default, the timeout is 1000 ms. When you see error 'Queue invalidate
+wait descriptor timed out', try increasing this value.
+
+### iommu_inclusive_mapping (VT-d)
+> `= <boolean>`
+
+**WARNING: This command line option is deprecated, and superseded by
+_dom0-iommu=map-inclusive_ - using both options in combination is undefined.**
+
+> Default: `true`
+
+Use this to work around firmware issues providing incorrect RMRR entries.
+Rather than only mapping RAM pages for IOMMU accesses for Dom0, with this
+option all pages up to 4GB, not marked as unusable in the E820 table, will
+get a mapping established. Note that this option is only applicable to a
+PV dom0. Also note that if `dom0-strict` mode is enabled then conventional
+RAM pages not assigned to dom0 will not be mapped.
+
+### irq_ratelimit (x86)
+> `= <integer>`
+
+### irq_vector_map (x86)
+### ivrs_hpet[`<hpet>`] (AMD)
+> `=[<seg>:]<bus>:<device>.<func>`
+
+Force the use of `[<seg>:]<bus>:<device>.<func>` as device ID of HPET
+`<hpet>` instead of the one specified by the IVHD sub-tables of the IVRS
+ACPI table.
+
+### ivrs_ioapic[`<ioapic>`] (AMD)
+> `=[<seg>:]<bus>:<device>.<func>`
+
+Force the use of `[<seg>:]<bus>:<device>.<func>` as device ID of IO-APIC
+`<ioapic>` instead of the one specified by the IVHD sub-tables of the IVRS
+ACPI table.
+
+### lapic (x86)
+> `= <boolean>`
+
+Force the use of use of the local APIC on a uniprocessor system, even
+if left disabled by the BIOS.
+
+### lapic_timer_c2_ok (x86)
+> `= <boolean>`
+
+### ler (x86)
+> `= <boolean>`
+
+> Default: false
+
+This option is intended for debugging purposes only. Enable MSR_DEBUGCTL.LBR
+in hypervisor context to be able to dump the Last Interrupt/Exception To/From
+record with other registers.
+
+### loglvl
+> `= <level>[/<rate-limited level>]` where level is `none | error | warning | info | debug | all`
+
+> Default: `loglvl=warning`
+
+> Can be modified at runtime
+
+Set the logging level for Xen. Any log message with equal more more
+importance will be printed.
+
+The optional `<rate-limited level>` option instructs which severities
+should be rate limited.
+
+### low_crashinfo
+> `= none | min | all`
+
+> Default: `none` if not specified at all, or to `min` if **low_crashinfo** is present without qualification.
+
+This option is only useful for hosts with a 32bit dom0 kernel, wishing
+to use kexec functionality in the case of a crash. It represents
+which data structures should be deliberately allocated in low memory,
+so the crash kernel may find find them. Should be used in combination
+with **crashinfo_maxaddr**.
+
+### low_mem_virq_limit
+> `= <size>`
+
+> Default: `64M`
+
+Specify the threshold below which Xen will inform dom0 that the quantity of
+free memory is getting low. Specifying `0` will disable this notification.
+
+### maxcpus (x86)
+> `= <integer>`
+
+Specify the maximum number of CPUs that should be brought up.
+
+This option is ignored in **pv-shim** mode.
+
+### max_cstate (x86)
+> `= <integer>`
+
+### max_gsi_irqs (x86)
+> `= <integer>`
+
+Specifies the number of interrupts to be use for pin (IO-APIC or legacy PIC)
+based interrupts. Any higher IRQs will be available for use via PCI MSI.
+
+### max_lpi_bits (arm)
+> `= <integer>`
+
+Specifies the number of ARM GICv3 LPI interrupts to allocate on the host,
+presented as the number of bits needed to encode it. This must be at least
+14 and not exceed 32, and each LPI requires one byte (configuration) and
+one pending bit to be allocated.
+Defaults to 20 bits (to cover at most 1048576 interrupts).
+
+### mce (x86)
+> `= <integer>`
+
+### mce_fb (Intel)
+> `= <integer>`
+
+### mce_verbosity (x86)
+> `= verbose`
+
+Specify verbose machine check output.
+
+### mem (x86)
+> `= <size>`
+
+Specify the maximum address of physical RAM. Any RAM beyond this
+limit is ignored by Xen.
+
+### memop-max-order
+> `= [<domU>][,[<ctldom>][,[<hwdom>][,<ptdom>]]]`
+
+> x86 default: `9,18,12,12`
+> ARM default: `9,18,10,10`
+
+Change the maximum order permitted for allocation (or allocation-like)
+requests issued by the various kinds of domains (in this order:
+ordinary DomU, control domain, hardware domain, and - when supported
+by the platform - DomU with pass-through device assigned).
+
+### mmcfg (x86)
+> `= <boolean>[,amd-fam10]`
+
+> Default: `1`
+
+Specify if the MMConfig space should be enabled.
+
+### mmio-relax (x86)
+> `= <boolean> | all`
+
+> Default: `false`
+
+By default, domains may not create cached mappings to MMIO regions.
+This option relaxes the check for Domain 0 (or when using `all`, all PV
+domains), to permit the use of cacheable MMIO mappings.
+
+### msi (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+Force Xen to (not) use PCI-MSI, even if ACPI FADT says otherwise.
+
+### mtrr.show (x86)
+> `= <boolean>`
+
+> Default: `false`
+
+Print boot time MTRR state.
+
+### mwait-idle (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+Use the MWAIT idle driver (with model specific C-state knowledge) instead
+of the ACPI based one.
+
+### nmi (x86)
+> `= ignore | dom0 | fatal`
+
+> Default: `fatal` for a debug build, or `dom0` for a non-debug build
+
+Specify what Xen should do in the event of an NMI parity or I/O error.
+`ignore` discards the error; `dom0` causes Xen to report the error to
+dom0, while 'fatal' causes Xen to print diagnostics and then hang.
+
+### noapic (x86)
+
+Instruct Xen to ignore any IOAPICs that are present in the system, and
+instead continue to use the legacy PIC. This is _not_ recommended with
+pvops type kernels.
+
+Because responsibility for APIC setup is shared between Xen and the
+domain 0 kernel this option is automatically propagated to the domain
+0 command line.
+
+### invpcid (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+By default, Xen will use the INVPCID instruction for TLB management if
+it is available. This option can be used to cause Xen to fall back to
+older mechanisms, which are generally slower.
+
+### noirqbalance (x86)
+> `= <boolean>`
+
+Disable software IRQ balancing and affinity. This can be used on
+systems such as Dell 1850/2850 that have workarounds in hardware for
+IRQ routing issues.
+
+### nolapic (x86)
+> `= <boolean>`
+
+> Default: `false`
+
+Ignore the local APIC on a uniprocessor system, even if enabled by the
+BIOS.
+
+### no-real-mode (x86)
+> `= <boolean>`
+
+Do not execute real-mode bootstrap code when booting Xen. This option
+should not be used except for debugging. It will effectively disable
+the **vga** option, which relies on real mode to set the video mode.
+
+### noreboot
+> `= <boolean>`
+
+Do not automatically reboot after an error. This is useful for
+catching debug output. Defaults to automatically reboot after 5
+seconds.
+
+### nosmp (x86)
+> `= <boolean>`
+
+Disable SMP support. No secondary processors will be booted.
+Defaults to booting secondary processors.
+
+This option is ignored in **pv-shim** mode.
+
+### nr_irqs (x86)
+> `= <integer>`
+
+### numa (x86)
+> `= on | off | fake=<integer> | noacpi`
+
+> Default: `on`
+
+### pci
+> `= {no-}serr | {no-}perr`
+
+> Default: Signaling left as set by firmware.
+
+Disable signaling of SERR (system errors) and/or PERR (parity errors)
+on all PCI devices.
+
+
+### pci-phantom
+> `=[<seg>:]<bus>:<device>,<stride>`
+
+Mark a group of PCI devices as using phantom functions without actually
+advertising so, so the IOMMU can create translation contexts for them.
+
+All numbers specified must be hexadecimal ones.
+
+This option can be specified more than once (up to 8 times at present).
+
+### pcid (x86)
+> `= <boolean> | xpti=<bool>`
+
+> Default: `xpti`
+
+> Can be modified at runtime (change takes effect only for domains created
+ afterwards)
+
+If available, control usage of the PCID feature of the processor for
+64-bit pv-domains. PCID can be used either for no domain at all (`false`),
+for all of them (`true`), only for those subject to XPTI (`xpti`) or for
+those not subject to XPTI (`no-xpti`). The feature is used only in case
+INVPCID is supported and not disabled via `invpcid=false`.
+
+### pku (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+Flag to enable Memory Protection Keys.
+
+The protection-key feature provides an additional mechanism by which IA-32e
+paging controls access to usermode addresses.
+
+### ple_gap
+> `= <integer>`
+
+### ple_window (Intel)
+> `= <integer>`
+
+### psr (Intel)
+> `= List of ( cmt:<boolean> | rmid_max:<integer> | cat:<boolean> | cos_max:<integer> | cdp:<boolean> )`
+
+> Default: `psr=cmt:0,rmid_max:255,cat:0,cos_max:255,cdp:0`
+
+Platform Shared Resource(PSR) Services. Intel Haswell and later server
+platforms offer information about the sharing of resources.
+
+To use the PSR monitoring service for a certain domain, a Resource
+Monitoring ID(RMID) is used to bind the domain to corresponding shared
+resource. RMID is a hardware-provided layer of abstraction between software
+and logical processors.
+
+To use the PSR cache allocation service for a certain domain, a capacity
+bitmasks(CBM) is used to bind the domain to corresponding shared resource.
+CBM represents cache capacity and indicates the degree of overlap and isolation
+between domains. In hypervisor a Class of Service(COS) ID is allocated for each
+unique CBM.
+
+The following resources are available:
+
+* Cache Monitoring Technology (Haswell and later). Information regarding the
+ L3 cache occupancy.
+ * `cmt` instructs Xen to enable/disable Cache Monitoring Technology.
+ * `rmid_max` indicates the max value for rmid.
+* Memory Bandwidth Monitoring (Broadwell and later). Information regarding the
+ total/local memory bandwidth. Follow the same options with Cache Monitoring
+ Technology.
+
+* Cache Allocation Technology (Broadwell and later). Information regarding
+ the cache allocation.
+ * `cat` instructs Xen to enable/disable Cache Allocation Technology.
+ * `cos_max` indicates the max value for COS ID.
+* Code and Data Prioritization Technology (Broadwell and later). Information
+ regarding the code cache and the data cache allocation. CDP is based on CAT.
+ * `cdp` instructs Xen to enable/disable Code and Data Prioritization. Note
+ that `cos_max` of CDP is a little different from `cos_max` of CAT. With
+ CDP, one COS will corespond two CBMs other than one with CAT, due to the
+ sum of CBMs is fixed, that means actual `cos_max` in use will automatically
+ reduce to half when CDP is enabled.
+
+### pv-linear-pt (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+Only available if Xen is compiled with `CONFIG_PV_LINEAR_PT` support
+enabled.
+
+Allow PV guests to have pagetable entries pointing to other pagetables
+of the same level (i.e., allowing L2 PTEs to point to other L2 pages).
+This technique is often called "linear pagetables", and is sometimes
+used to allow operating systems a simple way to consistently map the
+current process's pagetables into its own virtual address space.
+
+Linux and MiniOS don't use this technique. NetBSD and Novell Netware
+do; there may be other custom operating systems which do. If you're
+certain you don't plan on having PV guests which use this feature,
+turning it off can reduce the attack surface.
+
+### pv-l1tf (x86)
+> `= List of [ <bool>, dom0=<bool>, domu=<bool> ]`
+
+> Default: `false` on believed-unaffected hardware, or in pv-shim mode.
+> `domu` on believed-affected hardware.
+
+Mitigations for L1TF / XSA-273 / CVE-2018-3620 for PV guests.
+
+For backwards compatibility, we may not alter an architecturally-legitimate
+pagetable entry a PV guest chooses to write. We can however force such a
+guest into shadow mode so that Xen controls the PTEs which are reachable by
+the CPU pagewalk.
+
+Shadowing is performed at the point where a PV guest first tries to write an
+L1TF-vulnerable PTE. Therefore, a PV guest kernel which has been updated with
+its own L1TF mitigations will not trigger shadow mode if it is well behaved.
+
+If `CONFIG_SHADOW_PAGING` is not compiled in, this mitigation instead crashes
+the guest when an L1TF-vulnerable PTE is written, which still allows updated,
+well-behaved PV guests to run, despite Shadow being compiled out.
+
+In the pv-shim case, Shadow is expected to be compiled out, and a malicious
+guest kernel can only leak data from the shim Xen, rather than the host Xen.
+
+### pv-shim (x86)
+> `= <boolean>`
+
+> Default: `false`
+
+This option is intended for use by a toolstack, when choosing to run a PV
+guest compatibly inside an HVM container.
+
+In this mode, the kernel and initrd passed as modules to the hypervisor are
+constructed into a plain unprivileged PV domain.
+
+### rcu-idle-timer-period-ms
+> `= <integer>`
+
+> Default: `10`
+
+How frequently a CPU which has gone idle, but with pending RCU callbacks,
+should be woken up to check if the grace period has completed, and the
+callbacks are safe to be executed. Expressed in milliseconds; maximum is
+100, and it can't be 0.
+
+### reboot (x86)
+> `= t[riple] | k[bd] | a[cpi] | p[ci] | P[ower] | e[fi] | n[o] [, [w]arm | [c]old]`
+
+> Default: `0`
+
+Specify the host reboot method.
+
+`warm` instructs Xen to not set the cold reboot flag.
+
+`cold` instructs Xen to set the cold reboot flag.
+
+`no` instructs Xen to not automatically reboot after panics or crashes.
+
+`triple` instructs Xen to reboot the host by causing a triple fault.
+
+`kbd` instructs Xen to reboot the host via the keyboard controller.
+
+`acpi` instructs Xen to reboot the host using RESET_REG in the ACPI FADT.
+
+`pci` instructs Xen to reboot the host using PCI reset register (port CF9).
+
+`Power` instructs Xen to power-cycle the host using PCI reset register (port CF9).
+
+'efi' instructs Xen to reboot using the EFI reboot call (in EFI mode by
+ default it will use that method first).
+
+`xen` instructs Xen to reboot using Xen's SCHEDOP hypercall (this is the default
+when running nested Xen)
+
+### rmrr
+> `= start<-end>=[s1]bdf1[,[s1]bdf2[,...]];start<-end>=[s2]bdf1[,[s2]bdf2[,...]]`
+
+Define RMRR units that are missing from ACPI table along with device they
+belong to and use them for 1:1 mapping. End addresses can be omitted and one
+page will be mapped. The ranges are inclusive when start and end are specified.
+If segment of the first device is not specified, segment zero will be used.
+If other segments are not specified, first device segment will be used.
+If a segment is specified for other than the first device and it does not match
+the one specified for the first one, an error will be reported.
+
+'start' and 'end' values are page numbers (not full physical addresses),
+in hexadecimal format (can optionally be preceded by "0x").
+
+Usage example: If device 0:0:1d.0 requires one page (0xd5d45) to be
+reserved, and device 0:0:1a.0 requires three pages (0xd5d46 thru 0xd5d48)
+to be reserved, one usage would be:
+
+rmrr=d5d45=0:0:1d.0;0xd5d46-0xd5d48=0:0:1a.0
+
+Note: grub2 requires to escape or use quotations if special characters are used,
+namely ';', refer to the grub2 documentation if multiple ranges are specified.
+
+### ro-hpet (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+Map the HPET page as read only in Dom0. If disabled the page will be mapped
+with read and write permissions.
+
+### sched
+> `= credit | credit2 | arinc653 | rtds | null`
+
+> Default: `sched=credit`
+
+Choose the default scheduler.
+
+### sched_credit2_migrate_resist
+> `= <integer>`
+
+### sched_credit_tslice_ms
+> `= <integer>`
+
+Set the timeslice of the credit1 scheduler, in milliseconds. The
+default is 30ms. Reasonable values may include 10, 5, or even 1 for
+very latency-sensitive workloads.
+
+### sched_ratelimit_us
+> `= <integer>`
+
+In order to limit the rate of context switching, set the minimum
+amount of time that a vcpu can be scheduled for before preempting it,
+in microseconds. The default is 1000us (1ms). Setting this to 0
+disables it altogether.
+
+### sched_smt_power_savings
+> `= <boolean>`
+
+Normally Xen will try to maximize performance and cache utilization by
+spreading out vcpus across as many different divisions as possible
+(i.e, numa nodes, sockets, cores threads, &c). This often maximizes
+throughput, but also maximizes energy usage, since it reduces the
+depth to which a processor can sleep.
+
+This option inverts the logic, so that the scheduler in effect tries
+to keep the vcpus on the smallest amount of silicon possible; i.e.,
+first fill up sibling threads, then sibling cores, then sibling
+sockets, &c. This will reduce performance somewhat, particularly on
+systems with hyperthreading enabled, but should reduce power by
+enabling more sockets and cores to go into deeper sleep states.
+
+### serial_tx_buffer
+> `= <size>`
+
+> Default: `16kB`
+
+Set the serial transmit buffer size.
+
+### serrors (ARM)
+> `= diverse | forward | panic`
+
+> Default: `diverse`
+
+This parameter is provided to administrators to determine how the
+hypervisors handle SErrors.
+
+In order to distinguish guest-generated SErrors from hypervisor-generated
+SErrors we have to place SError checking code in every EL1 <-> EL2 paths.
+That will cause overhead on entries and exits due to dsb/isb. However, not all
+platforms need to categorize SErrors. For example, a host that is running with
+trusted guests. The administrator can confirm that all guests that are running
+on the host will not trigger such SErrors. In this case, the administrator can
+use this parameter to skip categorizing SErrors and reduce the overhead of
+dsb/isb.
+
+We provided the following 3 options to administrators to determine how the
+hypervisors handle SErrors:
+
+* `diverse`:
+ The hypervisor will distinguish guest SErrors from hypervisor SErrors.
+ The guest generated SErrors will be forwarded to guests, the hypervisor
+ generated SErrors will cause the whole system to crash.
+ It requires:
+ 1. dsb/isb on all EL1 -> EL2 trap entries to categorize SErrors correctly.
+ 2. dsb/isb on EL2 -> EL1 return paths to prevent slipping hypervisor
+ SErrors to guests.
+ 3. dsb/isb in context switch to isolate SErrors between 2 vCPUs.
+
+* `forward`:
+ The hypervisor will not distinguish guest SErrors from hypervisor SErrors.
+ All SErrors will be forwarded to guests, except the SErrors generated when
+ the idle vCPU is running. The idle domain doesn't have the ability to handle
+ SErrors, so we have to crash the whole system when we get SErros with the
+ idle vCPU. This option will avoid most overhead of the dsb/isb, except the
+ dsb/isb in context switch which is used to isolate the SErrors between 2
+ vCPUs.
+
+* `panic`:
+ The hypervisor will not distinguish guest SErrors from hypervisor SErrors.
+ All SErrors will crash the whole system. This option will avoid all overhead
+ of the dsb/isb pairs.
+
+### shim_mem (x86)
+> `= List of ( min:<size> | max:<size> | <size> )`
+
+Set the amount of memory that xen-shim uses. Only has effect if pv-shim mode is
+enabled. Note that this value accounts for the memory used by the shim itself
+plus the free memory slack given to the shim for runtime allocations.
+
+* `min:<size>` specifies the minimum amount of memory. Ignored if greater
+ than max.
+* `max:<size>` specifies the maximum amount of memory.
+* `<size>` specifies the exact amount of memory. Overrides both min and max.
+
+By default, the amount of free memory slack given to the shim for runtime usage
+is 1MB.
+
+### smap (x86)
+> `= <boolean> | hvm`
+
+> Default: `true`
+
+Flag to enable Supervisor Mode Access Prevention
+Use `smap=hvm` to allow SMAP use by HVM guests only.
+
+### smep (x86)
+> `= <boolean> | hvm`
+
+> Default: `true`
+
+Flag to enable Supervisor Mode Execution Protection
+Use `smep=hvm` to allow SMEP use by HVM guests only.
+
+### smt (x86)
+> `= <boolean>`
+
+Default: `true`
+
+Control bring up of multiple hyper-threads per CPU core.
+
+### snb_igd_quirk
+> `= <boolean> | cap | <integer>`
+
+A true boolean value enables legacy behavior (1s timeout), while `cap`
+enforces the maximum theoretically necessary timeout of 670ms. Any number
+is being interpreted as a custom timeout in milliseconds. Zero or boolean
+false disable the quirk workaround, which is also the default.
+
+### spec-ctrl (Arm)
+> `= List of [ ssbd=force-disable|runtime|force-enable ]`
+
+Controls for speculative execution sidechannel mitigations.
+
+The option `ssbd=` is used to control the state of Speculative Store
+Bypass Disable (SSBD) mitigation.
+
+* `ssbd=force-disable` will keep the mitigation permanently off. The guest
+will not be able to control the state of the mitigation.
+* `ssbd=runtime` will always turn on the mitigation when running in the
+hypervisor context. The guest will be to turn on/off the mitigation for
+itself by using the firmware interface `ARCH_WORKAROUND_2`.
+* `ssbd=force-enable` will keep the mitigation permanently on. The guest will
+not be able to control the state of the mitigation.
+
+By default SSBD will be mitigated at runtime (i.e `ssbd=runtime`).
+
+### spec-ctrl (x86)
+> `= List of [ <bool>, xen=<bool>, {pv,hvm,msr-sc,rsb}=<bool>,
+> bti-thunk=retpoline|lfence|jmp, {ibrs,ibpb,ssbd,eager-fpu,
+> l1d-flush}=<bool> ]`
+
+Controls for speculative execution sidechannel mitigations. By default, Xen
+will pick the most appropriate mitigations based on compiled in support,
+loaded microcode, and hardware details, and will virtualise appropriate
+mitigations for guests to use.
+
+**WARNING: Any use of this option may interfere with heuristics. Use with
+extreme care.**
+
+An overall boolean value, `spec-ctrl=no`, can be specified to turn off all
+mitigations, including pieces of infrastructure used to virtualise certain
+mitigation features for guests. This also includes settings which `xpti`,
+`smt`, `pv-l1tf` control, unless the respective option(s) have been
+specified earlier on the command line.
+
+Alternatively, a slightly more restricted `spec-ctrl=no-xen` can be used to
+turn off all of Xen's mitigations, while leaving the virtualisation support
+in place for guests to use.
+
+Use of a positive boolean value for either of these options is invalid.
+
+The booleans `pv=`, `hvm=`, `msr-sc=` and `rsb=` offer fine grained control
+over the alternative blocks used by Xen. These impact Xen's ability to
+protect itself, and Xen's ability to virtualise support for guests to use.
+
+* `pv=` and `hvm=` offer control over all suboptions for PV and HVM guests
+ respectively.
+* `msr-sc=` offers control over Xen's support for manipulating `MSR_SPEC_CTRL`
+ on entry and exit. These blocks are necessary to virtualise support for
+ guests and if disabled, guests will be unable to use IBRS/STIBP/SSBD/etc.
+* `rsb=` offers control over whether to overwrite the Return Stack Buffer /
+ Return Address Stack on entry to Xen.
+
+If Xen was compiled with INDIRECT_THUNK support, `bti-thunk=` can be used to
+select which of the thunks gets patched into the `__x86_indirect_thunk_%reg`
+locations. The default thunk is `retpoline` (generally preferred for Intel
+hardware), with the alternatives being `jmp` (a `jmp *%reg` gadget, minimal
+overhead), and `lfence` (an `lfence; jmp *%reg` gadget, preferred for AMD).
+
+On hardware supporting IBRS (Indirect Branch Restricted Speculation), the
+`ibrs=` option can be used to force or prevent Xen using the feature itself.
+If Xen is not using IBRS itself, functionality is still set up so IBRS can be
+virtualised for guests.
+
+On hardware supporting IBPB (Indirect Branch Prediction Barrier), the `ibpb=`
+option can be used to force (the default) or prevent Xen from issuing branch
+prediction barriers on vcpu context switches.
+
+On hardware supporting SSBD (Speculative Store Bypass Disable), the `ssbd=`
+option can be used to force or prevent Xen using the feature itself. On AMD
+hardware, this is a global option applied at boot, and not virtualised for
+guest use. On Intel hardware, the feature is virtualised for guests,
+independently of Xen's choice of setting.
+
+On all hardware, the `eager-fpu=` option can be used to force or prevent Xen
+from using fully eager FPU context switches. This is currently implemented as
+a global control. By default, Xen will choose to use fully eager context
+switches on hardware believed to speculate past #NM exceptions.
+
+On hardware supporting L1D_FLUSH, the `l1d-flush=` option can be used to force
+or prevent Xen from issuing an L1 data cache flush on each VMEntry.
+Irrespective of Xen's setting, the feature is virtualised for HVM guests to
+use. By default, Xen will enable this mitigation on hardware believed to be
+vulnerable to L1TF.
+
+### sync_console
+> `= <boolean>`
+
+> Default: `false`
+
+Flag to force synchronous console output. Useful for debugging, but
+not suitable for production environments due to incurred overhead.
+
+### tboot (x86)
+> `= 0x<phys_addr>`
+
+Specify the physical address of the trusted boot shared page.
+
+### tbuf_size
+> `= <integer>`
+
+Specify the per-cpu trace buffer size in pages.
+
+### tdt (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+Flag to enable TSC deadline as the APIC timer mode.
+
+### tevt_mask
+> `= <integer>`
+
+Specify a mask for Xen event tracing. This allows Xen tracing to be
+enabled at boot. Refer to the xentrace(8) documentation for a list of
+valid event mask values. In order to enable tracing, a buffer size (in
+pages) must also be specified via the tbuf_size parameter.
+
+### tickle_one_idle_cpu
+> `= <boolean>`
+
+### timer_slop
+> `= <integer>`
+
+### tmem
+> `= <boolean>`
+
+### tmem_compress
+> `= <boolean>`
+
+### tsc (x86)
+> `= unstable | skewed | stable:socket`
+
+### ucode (x86)
+> `= [<integer> | scan]`
+
+Specify how and where to find CPU microcode update blob.
+
+'integer' specifies the CPU microcode update blob module index. When positive,
+this specifies the n-th module (in the GrUB entry, zero based) to be used
+for updating CPU micrcode. When negative, counting starts at the end of
+the modules in the GrUB entry (so with the blob commonly being last,
+one could specify `ucode=-1`). Note that the value of zero is not valid
+here (entry zero, i.e. the first module, is always the Dom0 kernel
+image). Note further that use of this option has an unspecified effect
+when used with xen.efi (there the concept of modules doesn't exist, and
+the blob gets specified via the `ucode=<filename>` config file/section
+entry; see [EFI configuration file description](efi.html)).
+
+'scan' instructs the hypervisor to scan the multiboot images for an cpio
+image that contains microcode. Depending on the platform the blob with the
+microcode in the cpio name space must be:
+ - on Intel: kernel/x86/microcode/GenuineIntel.bin
+ - on AMD : kernel/x86/microcode/AuthenticAMD.bin
+
+### unrestricted_guest (Intel)
+> `= <boolean>`
+
+### vcpu_migration_delay
+> `= <integer>`
+
+> Default: `0`
+
+Specify a delay, in microseconds, between migrations of a VCPU between
+PCPUs when using the credit1 scheduler. This prevents rapid fluttering
+of a VCPU between CPUs, and reduces the implicit overheads such as
+cache-warming. 1ms (1000) has been measured as a good value.
+
+### vesa-map
+> `= <integer>`
+
+### vesa-mtrr
+> `= <integer>`
+
+### vesa-ram
+> `= <integer>`
+
+### vga
+> `= ( ask | current | text-80x<rows> | gfx-<width>x<height>x<depth> | mode-<mode> )[,keep]`
+
+`ask` causes Xen to display a menu of available modes and request the
+user to choose one of them.
+
+`current` causes Xen to use the graphics adapter in its current state,
+without further setup.
+
+`text-80x<rows>` instructs Xen to set up text mode. Valid values for
+`<rows>` are `25, 28, 30, 34, 43, 50, 80`
+
+`gfx-<width>x<height>x<depth>` instructs Xen to set up graphics mode
+with the specified width, height and depth.
+
+`mode-<mode>` instructs Xen to use a specific mode, as shown with the
+`ask` option. (N.B menu modes are displayed in hex, so `<mode>`
+should be a hexadecimal number)
+
+The optional `keep` parameter causes Xen to continue using the vga
+console even after dom0 has been started. The default behaviour is to
+relinquish control to dom0.
+
+### viridian-spinlock-retry-count (x86)
+> `= <integer>`
+
+> Default: `2047`
+
+Specify the maximum number of retries before an enlightened Windows
+guest will notify Xen that it has failed to acquire a spinlock.
+
+### viridian-version (x86)
+> `= [<major>],[<minor>],[<build>]`
+
+> Default: `6,0,0x1772`
+
+<major>, <minor> and <build> must be integers. The values will be
+encoded in guest CPUID 0x40000002 if viridian enlightenments are enabled.
+
+### vpid (Intel)
+> `= <boolean>`
+
+> Default: `true`
+
+Use Virtual Processor ID support if available. This prevents the need for TLB
+flushes on VM entry and exit, increasing performance.
+
+### vpmu (x86)
+> `= ( <boolean> | { bts | ipc | arch [, ...] } )`
+
+> Default: `off`
+
+Switch on the virtualized performance monitoring unit for HVM guests.
+
+If the current cpu isn't supported a message like
+'VPMU: Initialization failed. ...'
+is printed on the hypervisor serial log.
+
+For some Intel Nehalem processors a quirk handling exist for an unknown
+wrong behaviour (see `handle_pmc_quirk()`).
+
+If 'vpmu=bts' is specified the virtualisation of the Branch Trace Store (BTS)
+feature is switched on on Intel processors supporting this feature.
+
+vpmu=ipc enables performance monitoring, but restricts the counters to the
+most minimum set possible: instructions, cycles, and reference cycles. These
+can be used to calculate instructions per cycle (IPC).
+
+vpmu=arch enables performance monitoring, but restricts the counters to the
+pre-defined architectural events only. These are exposed by cpuid, and listed
+in the Pre-Defined Architectural Performance Events table from the Intel 64
+and IA-32 Architectures Software Developer's Manual, Volume 3B, System
+Programming Guide, Part 2.
+
+If a boolean is not used, combinations of flags are allowed, comma separated.
+For example, vpmu=arch,bts.
+
+Note that if **watchdog** option is also specified vpmu will be turned off.
+
+*Warning:*
+As the virtualisation is not 100% safe, don't use the vpmu flag on
+production systems (see http://xenbits.xen.org/xsa/advisory-163.html)!
+
+### vwfi (arm)
+> `= trap | native`
+
+> Default: `trap`
+
+WFI is the ARM instruction to "wait for interrupt". WFE is similar and
+means "wait for event". This option, which is ARM specific, changes the
+way guest WFI and WFE are implemented in Xen. By default, Xen traps both
+instructions. In the case of WFI, Xen blocks the guest vcpu; in the case
+of WFE, Xen yield the guest vcpu. When setting vwfi to `native`, Xen
+doesn't trap either instruction, running them in guest context. Setting
+vwfi to `native` reduces irq latency significantly. It can also lead to
+suboptimal scheduling decisions, but only when the system is
+oversubscribed (i.e., in total there are more vCPUs than pCPUs).
+
+### watchdog (x86)
+> `= force | <boolean>`
+
+> Default: `false`
+
+Run an NMI watchdog on each processor. If a processor is stuck for
+longer than the **watchdog_timeout**, a panic occurs. When `force` is
+specified, in addition to running an NMI watchdog on each processor,
+unknown NMIs will still be processed.
+
+### watchdog_timeout (x86)
+> `= <integer>`
+
+> Default: `5`
+
+Set the NMI watchdog timeout in seconds. Specifying `0` will turn off
+the watchdog.
+
+### x2apic (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+Permit use of x2apic setup for SMP environments.
+
+### x2apic_phys (x86)
+> `= <boolean>`
+
+> Default: `true` if **FADT** mandates physical mode, `false` otherwise.
+
+In the case that x2apic is in use, this option switches between physical and
+clustered mode. The default, given no hint from the **FADT**, is cluster
+mode.
+
+### xenheap_megabytes (arm32)
+> `= <size>`
+
+> Default: `0` (1/32 of RAM)
+
+Amount of RAM to set aside for the Xenheap. Must be an integer multiple of 32.
+
+By default will use 1/32 of the RAM up to a maximum of 1GB and with a
+minimum of 32M, subject to a suitably aligned and sized contiguous
+region of memory being available.
+
+### xpti (x86)
+> `= List of [ default | <boolean> | dom0=<bool> | domu=<bool> ]`
+
+> Default: `false` on hardware known not to be vulnerable to Meltdown (e.g. AMD)
+> Default: `true` everywhere else
+
+Override default selection of whether to isolate 64-bit PV guest page
+tables.
+
+`true` activates page table isolation even on hardware not vulnerable by
+Meltdown for all domains.
+
+`false` deactivates page table isolation on all systems for all domains.
+
+`default` sets the default behaviour.
+
+With `dom0` and `domu` it is possible to control page table isolation
+for dom0 or guest domains only.
+
+### xsave (x86)
+> `= <boolean>`
+
+> Default: `true`
+
+Permit use of the `xsave/xrstor` instructions.
+
+### xsm
+> `= dummy | flask | silo`
+
+> Default: selectable via Kconfig. Depends on enabled XSM modules.
+
+Specify which XSM module should be enabled. This option is only available if
+the hypervisor was compiled with `CONFIG_XSM` enabled.
+
+* `dummy`: this is the default choice. Basic restriction for common deployment
+ (the dummy module) will be applied. It's also used when XSM is compiled out.
+* `flask`: this is the policy based access control. To choose this, the
+ separated option in kconfig must also be enabled.
+* `silo`: this will deny any unmediated communication channels between
+ unprivileged VMs. To choose this, the separated option in kconfig must also
+ be enabled.
+++ /dev/null
-# XenStore Paths
-
-This document attempts to defines all the paths which are in common
-use by either guests, front-/back-end drivers, toolstacks etc.
-
-The XenStore wire protocol itself is described in
-[xenstore.txt](xenstore.txt).
-
-## Notation
-
-This document is intended to be partially machine readable, such that
-test system etc can use it to validate whether the xenstore paths used
-by a test are allowable etc.
-
-Therefore the following notation conventions apply:
-
-A xenstore path is generically defined as:
-
- PATH = VALUES [TAGS]
-
- PATH/* [TAGS]
-
-The first syntax defines a simple path with a single value. The second
-syntax defines an aggregated set of paths which are usually described
-externally to this document. The text will give a pointer to the
-appropriate external documentation.
-
-PATH can contain simple regex constructs following the Perl compatible
-regexp syntax described in pcre(3) or perlre(1). In addition the
-following additional wild card names are defined and are evaluated
-before regexp expansion:
-
-* ~ -- expands to an arbitrary a domain's home path (described below).
- Only valid at the begining of a path.
-* $DEVID -- a per-device type device identifier. Typically an integer.
-* $DOMID -- a domain identifier, an integer. Typically this refers to
- the "other" domain. i.e. ~ refers to the domain providing a service
- while $DOMID is the consumer of that service.
-* $UUID -- a UUID in the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
-* $INDEX -- an integer used as part of a path when listing a set of
- values. Typically these integers are contiguous.
-
-VALUES are strings and can take the following forms:
-
-* PATH -- a XenStore path.
-* STRING -- an arbitrary string.
-* INTEGER -- An integer, in decimal representation unless otherwise
- noted.
- * MEMKB -- the decimal representation of a number of kilobytes.
- * EVTCHN -- the decimal representation of an event channel.
- * GNTREF -- the decimal representation of a grant reference.
-* "a literal string" -- literal strings are contained within quotes.
-* (VALUE | VALUE | ... ) -- a set of alternatives. Alternatives are
- separated by a "|" and all the alternatives are enclosed in "(" and
- ")".
-* DISTRIBUTION -- information about a software distribution, comprised
- of 3 or 4 space separated UTF-8 strings as follows:
-
- VENDOR -- Commonly used vendor short name,
- e.g "Citrix" rather than "Citrix Systems
- Inc."
-
- PRODUCT -- Commonly used product (e.g. driver) name
- without version information.
-
- If the toolstack needs to match on either of the above
- values it should support Unix glob style matching.
-
- VERSION -- A version number that will sort properly
- under coreutils version sorting (sort -V)
- rules.
-
- ATTRIBUTES -- Optional human readable text to denote
- attributes of the software, e.g. "debug".
- This text is freeform and no meaning
- should be inferred. It is intended for
- display purposes only.
-
-* MAC_ADDRESS -- 6 integers, in hexadecimal form, separated by ':',
- specifying an IEEE 802.3 ethernet MAC address.
-* IPV4_ADDRESS -- 4 integers, in decimal form, separated by '.',
- specifying an IP version 4 address as described
- IETF RFC 791.
-* IPV6_ADDRESS -- Up to 8 integers, in hexadecimal form, separated
- by ':', specifying an IP version 6 address as
- described in IETF RFC 4291.
-
-Additional TAGS may follow as a comma separated set of the following
-tags enclosed in square brackets.
-
-* w -- Path is writable by the containing domain, that is the domain
- whose home path ~ this key is under or which /vm/$UUID refers to. By
- default paths under both of these locations are read only for the
- domain.
-* n -- Path is neither readable nor writeable for guest domains.
-* HVM -- Path is valid for HVM domains only
-* PV -- Path is valid for PV domains only
-* BACKEND -- Path is valid for a backend domain (AKA driver domain)
-* INTERNAL -- Although a path is visible to the domain its use is
- reserved for the virtual firmware or Xen platform code. Guest
- Operating Systems must not read this key or otherwise rely on its
- presence or contents.
-* DEPRECATED -- This path is deprecated and may be removed in its
- current form in the future. Guests should not add new dependencies
- on such paths.
-
-Owning domain means the domain whose home path this tag is found
-under.
-
-Lack of either a __HVM__ or __PV__ tag indicates that the path is
-valid for either type of domain (including PVonHVM and similar mixed
-domain types).
-
-## Domain Home Path
-
-Every domain has a home path within the xenstore hierarchy. This is
-the path where the majority of the domain-visible information about
-each domain is stored.
-
-This path is:
-
- /local/domain/$DOMID
-
-All non-absolute paths are relative to this path.
-
-Although this path could be considered a "Home Directory" for the
-domain it would not usually be writable by the domain. The tools will
-create writable subdirectories as necessary.
-
-## Per Domain Paths
-
-## General Paths
-
-#### ~/vm = PATH []
-
-A pointer back to the domain's /vm/$UUID path (described below).
-
-#### ~/name = STRING []
-
-The guests name.
-
-#### ~/domid = INTEGER []
-
-The domain's own ID.
-
-#### ~/image/device-model-pid = INTEGER [INTERNAL]
-
-The process ID of the device model associated with this domain, if it
-has one.
-
-#### ~/cpu/[0-9]+/availability = ("online"|"offline") [PV]
-
-One node for each virtual CPU up to the guest's configured
-maximum. Valid values are "online" and "offline". The guest is expected to react to changes in this path by bringing the appropriate VCPU on or offline using the VCPUOP interface described in [xen/include/public/vcpu.h][VCPU]
-
-This protocol is not currently well documented.
-
-#### ~/memory/static-max = MEMKB []
-
-Specifies a static maximum amount memory which this domain should
-expect to be given. In the absence of in-guest memory hotplug support
-this set on domain boot and is usually the maximum amount of RAM which
-a guest can make use of. See [docs/misc/libxl_memory.txt][LIBXLMEM]
-for a description of how memory is accounted for in toolstacks using
-the libxl library.
-
-#### ~/memory/target = MEMKB []
-
-The current balloon target for the domain. The balloon driver within
-the guest is expected to make every effort to every effort use no more
-than this amount of RAM.
-
-#### ~/memory/videoram = MEMKB [HVM,INTERNAL]
-
-The size of the video RAM this domain is configured with.
-
-#### ~/device/suspend/event-channel = ""|EVTCHN [w]
-
-The domain's suspend event channel. The toolstack will create this
-path with an empty value which the guest may choose to overwrite.
-
-If the guest overwrites this, it will be with the number of an unbound
-event channel port it has acquired. The toolstack is expected to use
-an interdomain bind, and then, when it wishes to ask the guest to
-suspend, to signal the event channel.
-
-The guest does not need to explicitly acknowledge the request; indeed,
-there is no explicit signalling by the guest in the reverse direction.
-The guest, when it is ready, simply shuts down (`SCHEDOP_shutdown`)
-with reason code `SHUTDOWN_suspend`. The toolstack is expected to use
-`XEN_DOMCTL_subscribe` to be alerted to guest state changes, and
-`XEN_SYSCTL_getdomaininfolist` to verify that the domain has
-suspended.
-
-Note that the use of this event channel suspend protocol is optional
-for both sides. By writing a non-empty string to the node, the guest
-is advertising its support. However, the toolstack is at liberty to
-use the xenstore-based protocol instead (see ~/control/shutdown,
-below) even if the guest has advertised support for the event channel
-protocol.
-
-#### ~/hvmloader/allow-memory-relocate = ("1"|"0") [HVM,INTERNAL]
-
-If the default low MMIO hole (below 4GiB) is not big enough for all
-the devices, this indicates if hvmloader should relocate guest memory
-into the high memory region (above 4GiB). If "1", hvmloader will
-relocate memory as needed, until 2GiB is reached; if "0", hvmloader
-will not relocate guest memory.
-
-#### ~/hvmloader/bios = ("rombios"|"seabios"|"OVMF") [HVM,INTERNAL]
-
-The BIOS used by this domain.
-
-#### ~/bios-strings/bios-vendor = STRING [HVM,INTERNAL]
-#### ~/bios-strings/bios-version = STRING [HVM,INTERNAL]
-#### ~/bios-strings/system-manufacturer = STRING [HVM,INTERNAL]
-#### ~/bios-strings/system-product-name = STRING [HVM,INTERNAL]
-#### ~/bios-strings/system-version = STRING [HVM,INTERNAL]
-#### ~/bios-strings/system-serial-number = STRING [HVM,INTERNAL]
-#### ~/bios-strings/enclosure-manufacturer = STRING [HVM,INTERNAL]
-#### ~/bios-strings/enclosure-serial-number = STRING [HVM,INTERNAL]
-#### ~/bios-strings/enclosure-asset-tag = STRING [HVM,INTERNAL]
-#### ~/bios-strings/battery-manufacturer = STRING [HVM,INTERNAL]
-#### ~/bios-strings/battery-device-name = STRING [HVM,INTERNAL]
-
-These xenstore values are used to override some of the default string
-values in the SMBIOS table constructed in hvmloader. See the SMBIOS
-table specification at http://www.dmtf.org/standards/smbios/
-
-#### ~/bios-strings/oem-* = STRING [HVM,INTERNAL]
-
-1 to 99 OEM strings can be set in xenstore using values of the form
-'~/bios-strings/oem-1' to '~/bios-strings/oem-99'. These strings will be
-loaded into the SMBIOS type 11 structure.
-
-#### ~/platform/* = ("0"|"1") [HVM,INTERNAL]
-
-Various boolean platform properties.
-
-* acpi -- is ACPI enabled for this domain
-* acpi_s3 -- is ACPI S3 support enabled for this domain
-* acpi_s4 -- is ACPI S4 support enabled for this domain
-* acpi_laptop_slate -- is the ACPI laptop/slate device present in
- this domain
-
-#### ~/platform/generation-id = INTEGER ":" INTEGER [HVM,INTERNAL]
-
-The lower and upper 64-bit words of the 128-bit VM Generation ID.
-
-This key is used by hvmloader to create the ACPI VM Generation ID
-device. It initialises a 16 octet region of guest memory with this
-value. The guest physical address of this region is saved in the
-HVM_PARAM_VM_GENERATION_ID_ADDR HVM parameter.
-
-If this key is not present, is empty, or is all-zeros ("0:0") then the
-ACPI device is not created.
-
-When restoring a guest, the toolstack may (in certain circumstances)
-need generate a new random generation ID and write it to guest memory
-at the guest physical address in HVM_PARAM_VM_GENERATION_ID_ADDR.
-
-See Microsoft's "Virtual Machine Generation ID" specification for the
-circumstances where the generation ID needs to be changed.
-
-### Frontend device paths
-
-Paravirtual device frontends are generally specified by their own
-directory within the XenStore hierarchy. Usually this is under
-~/device/$TYPE/$DEVID although there are exceptions, e.g. ~/console
-for the first PV console.
-
-#### ~/device/vbd/$DEVID/* []
-
-A virtual block device frontend. Described by
-[xen/include/public/io/blkif.h][BLKIF]
-
-#### ~/device/vfb/$DEVID/* []
-
-A virtual framebuffer frontend. Described by
-[xen/include/public/io/fbif.h][FBIF]
-
-#### ~/device/vkbd/$DEVID/* []
-
-A virtual keyboard device frontend. Described by
-[xen/include/public/io/kbdif.h][KBDIF]
-
-#### ~/device/vif/$DEVID/* []
-
-A virtual network device frontend. Described by
-[xen/include/public/io/netif.h][NETIF]
-
-#### ~/device/vscsi/$DEVID/* []
-
-A virtual scsi device frontend. Described by
-[xen/include/public/io/vscsiif.h][SCSIIF]
-
-#### ~/device/vusb/$DEVID/* []
-
-A virtual usb device frontend. Described by
-[xen/include/public/io/usbif.h][USBIF]
-
-#### ~/device/pvcalls/$DEVID/* []
-
-Paravirtualized POSIX function calls frontend. Described by
-[docs/misc/pvcalls.markdown][PVCALLS]
-
-#### ~/console/* []
-
-The primary PV console device. Described in [console.txt](console.txt)
-
-#### ~/device/console/$DEVID/* []
-
-A secondary PV console device. Described in [console.txt](console.txt)
-
-#### ~/serial/$DEVID/* [HVM]
-
-An emulated serial device. Described in [console.txt](console.txt)
-
-#### ~/store/port = EVTCHN [DEPRECATED]
-
-The event channel used by the domain's connection to XenStore.
-
-This path is deprecated since the same information is provided via the
-[start_info][SI] for PV guests and as an [HVM param][HVMPARAMS] for
-HVM guests. There is an obvious chicken and egg problem with
-extracting this value from xenstore in order to setup the xenstore
-communication ring.
-
-#### ~/store/ring-ref = GNTREF [DEPRECATED]
-
-The grant reference of the domain's XenStore ring.
-
-As with ~/store/port this path is deprecated.
-
-### Backend Device Paths
-
-Paravirtual device backends are generally specified by their own
-directory within the XenStore hierarchy. Usually this is under
-~/backend/$TYPE/$DOMID/$DEVID.
-
-#### ~/backend/vbd/$DOMID/$DEVID/* []
-
-A virtual block device backend. Described by
-[xen/include/public/io/blkif.h][BLKIF]
-
-Uses the in-kernel blkback driver.
-
-#### ~/backend/qdisk/$DOMID/$DEVID/* []
-
-A virtual block device backend. Described by
-[xen/include/public/io/blkif.h][BLKIF]
-
-Uses the qemu based disk backend.
-
-#### ~/backend/tap/$DOMID/$DEVID/* []
-
-A virtual block device backend. Described by
-[xen/include/public/io/blkif.h][BLKIF]
-
-Uses the in-kernel blktap (v1) disk backend (deprecated).
-
-#### ~/backend/vfb/$DOMID/$DEVID/* []
-
-A virtual framebuffer backend. Described by
-[xen/include/public/io/fbif.h][FBIF]
-
-#### ~/backend/vkbd/$DOMID/$DEVID/* []
-
-A virtual keyboard device backend. Described by
-[xen/include/public/io/kbdif.h][KBDIF]
-
-#### ~/backend/vif/$DOMID/$DEVID/* []
-
-A virtual network device backend. Described by
-[xen/include/public/io/netif.h][NETIF]
-
-#### ~/backend/vscsi/$DOMID/$DEVID/* []
-
-A PV SCSI backend.
-
-#### ~/backend/vusb/$DOMID/$DEVID/* []
-
-A PV USB backend. Described by
-[xen/include/public/io/usbif.h][USBIF]
-
-#### ~/backend/pvcalls/$DOMID/$DEVID/* []
-
-A PVCalls backend. Described in [docs/misc/pvcalls.markdown][PVCALLS].
-
-#### ~/backend/console/$DOMID/$DEVID/* []
-
-A PV console backend. Described in [console.txt](console.txt)
-
-#### ~/backend/qusb/$DOMID/$DEVID/* []
-
-A PV USB device backend. Described by
-[xen/include/public/io/usbif.h][USBIF]
-
-Uses the qemu based USB backend.
-
-#### ~/device-model/$DOMID/* [INTERNAL]
-
-Information relating to device models running in the domain. $DOMID is
-the target domain of the device model.
-
-#### ~/libxl/disable_udev = ("1"|"0") []
-
-Indicates whether device hotplug scripts in this domain should be run
-by udev ("0") or will be run by the toolstack directly ("1").
-
-### Platform Feature and Control Paths
-
-#### ~/control/sysrq = (""|COMMAND) [w]
-
-This is the PV SysRq control node. A toolstack can write a single character
-representing a magic SysRq key understood by the Linux kernel. The guest
-acknowledges a request by writing the empty string back to the command node.
-
-This protocol is Linux only.
-
-#### ~/control/shutdown = (""|COMMAND) [w]
-
-This is the PV shutdown control node. A toolstack can write various
-commands here to cause various guest shutdown, reboot or suspend
-activities. The guest acknowledges a request by writing the empty
-string back to the command node.
-
-The precise protocol is not yet documented.
-
-#### ~/control/feature-poweroff = (""|"0"|"1") [w]
-#### ~/control/feature-reboot = (""|"0"|"1") [w]
-#### ~/control/feature-suspend = (""|"0"|"1") [w]
-
-These may be initialized to "" by the toolstack and may then be set
-to 0 or 1 by a guest to indicate whether it is capable or incapable,
-respectively, of responding to the corresponding command when written
-to ~/control/shutdown.
-A toolstack may then sample the feature- value at the point of issuing
-a PV control command and respond accordingly:
-
-"0" -> the frontend should not be expected to respond, so fail the
- control operation
-"1" -> the frontend should be expected to respond, so wait for it to
- do so and maybe fail the control operation after some reasonable
- timeout.
-"" -> the frontend may or may not respond, so wait for it to do so and
- then maybe try an alternative control mechanism after some
- reasonable timeout.
-
-Since a toolstack may not initialize these paths, and the parent
-~/control path is read-only to a guest, a guest should not expect a
-write to succeed. If it fails the guest may log the failure but should
-continue to process the corresponding command when written to
-~/control/shutdown regardless.
-
-#### ~/control/feature-s3 = (""|"0"|"1") [w,HVM]
-#### ~/control/feature-s4 = (""|"0"|"1") [w,HVM]
-
-These purpose of these feature flags is identical to feature-poweroff,
-feature-reboot and feature-suspend above but concern triggering the
-S3 or S4 power states of HVM guests.
-A toolstack may create these values, but should not sample them unless
-the corresponding acpi_ feature flag is set in ~/platform.
-
-#### ~/control/platform-feature-multiprocessor-suspend = (0|1) []
-
-Indicates to the guest that this platform supports the multiprocessor
-suspend feature.
-
-#### ~/control/platform-feature-xs\_reset\_watches = (0|1) []
-
-Indicates to the guest that this platform supports the
-XS_RESET_WATCHES xenstore message. See
-[xen/include/public/io/xs\_wire.h][XSWIRE] for the XenStore wire
-protocol definition.
-
-#### ~/control/laptop-slate-mode = (""|"laptop"|"slate") [w]
-
-This is the PV laptop/slate mode control node. If the toolstack has
-provisioned a guest with the ACPI laptop/slate mode device then it
-can write the desired mode here to cause the guest to switch modes if
-necessary. The guest acknowledges a request by writing the empty
-string back to the control node.
-
-#### ~/control/feature-laptop-slate-mode = (""|"0"|"1") [w]
-
-This may be initialized to "" by the toolstack and may then be set
-to 0 or 1 by a guest to indicate whether it is capable or incapable,
-respectively, of responding to a mode value written to
-~/control/laptop-slate-mode.
-
-### Domain Controlled Paths
-
-#### ~/data/* [w]
-
-A domain writable path. Available for arbitrary domain use.
-
-#### ~/drivers/$INDEX = DISTRIBUTION [w]
-
-A domain may write information about installed PV drivers using
-paths of this form.
-
-#### ~/feature/hotplug/vif = ("0"|"1") [w]
-#### ~/feature/hotplug/vbd = ("0"|"1") [w]
-
-By setting these paths to "1" a guest can indicate to a toolstack
-that it is capable of responding immediately to instantiation of,
-respectively, new vif by bringing online a new PV network device or
-a new vbd by bringing online a new PV block device.
-If the guest sets this path to "0" then it is indicating that it is
-definitely unable to respond immediately and hence the toolstack should
-defer instantiaton to the next VM start. However, if the path is absent
-then the toolstack may attempt the operation.
-
-#### ~/attr/vif/$DEVID/name = STRING [w]
-
-A domain may write its internal 'friendly' name for a network device
-using this path using UTF-8 encoding. A toolstack or UI may use this
-for display purposes. No particular meaning should be inferred from the
-name.
-
-#### ~/attr/vif/$DEVID/mac/$INDEX = MAC_ADDRESS [w]
-
-Paths of this form may be written by the guest to indicate MAC addresses
-it is currently using. These may be multicast or unicast addresses. For
-any of the paths the value of $INDEX is arbitrary.
-The values written are primarily for display purposes and must not be used
-for packet filtering or routing purposes.
-
-#### ~/attr/vif/$DEVID/ipv4/$INDEX = IPV4_ADDRESS [w]
-#### ~/attr/vif/$DEVID/ipv6/$INDEX = IPV6_ADDRESS [w]
-
-Paths of this form may be written by the guest to indicate IP addresses
-in use by the stack bound to the network frontend. For any of the paths
-the value of $INDEX is arbitrary.
-The values written are primarily for display purposes and must not be used
-for packet filtering or routing purposes. A toolstack may attempt to use an
-address written in one of these paths to, for example, establish a VNC
-session to the guest (although clearly some level of trust is placed
-in the value supplied by the guest in this case).
-
-### Paths private to the toolstack
-
-#### ~/device-model/$DOMID/state [w]
-
-Contains the status of the device models running on the domain.
-
-#### ~/device-model/$DOMID/backends/* [w]
-
-Backend types the device model is supporting. Each entry below backends
-is a directory which may contain further nodes specific to the backend
-type. The name of each backend directory is the same as the backend type
-(e.g. "qdisk").
-
-#### ~/libxl/$DOMID/qdisk-backend-pid [w]
-
-Contains the PIDs of the device models running on the domain.
-
-## Virtual Machine Paths
-
-The /vm/$UUID namespace is used by toolstacks to store various
-information relating to the domain which is not intended to be guest
-visible (hence they are all tagged [n,INTERNAL]).
-
-Several of the keys here are not well defined and/or not well located
-and are liable to be replaced with more fully defined paths in the
-future.
-
-### /vm/$UUID/uuid = UUID [n,INTERNAL]
-
-Value is the same UUID as the path.
-
-### /vm/$UUID/name = STRING [n,INTERNAL]
-
-The domain's name.
-
-### /vm/$UUID/image/* [n,INTERNAL]
-
-Various information relating to the domain builder used for this guest.
-
-### /vm/$UUID/start_time = INTEGER "." INTEGER [n,INTERNAL]
-
-The time which the guest was started in SECONDS.MICROSECONDS format
-
-### /vm/$UUID/rtc/timeoffset = ""|INTEGER [n,HVM,INTERNAL]
-
-The guest's virtual time offset from UTC in seconds.
-
-## Platform-Level paths
-
-### libxl Specific Paths
-
-#### /libxl/$DOMID/device/$KIND/$DEVID
-
-Created by libxl for every frontend/backend pair created for $DOMID.
-Used by libxl for enumeration and management of the device.
-
-#### /libxl/$DOMID/device/$KIND/$DEVID/frontend
-
-Path in xenstore to the frontend, normally
-/local/domain/$DOMID/device/$KIND/$DEVID
-
-#### /libxl/$DOMID/device/$KIND/$DEVID/backend
-
-Path in xenstore to the backend, normally
-/local/domain/$BACKEND_DOMID/backend/$KIND/$DOMID/$DEVID
-
-#### /libxl/$DOMID/device/$KIND/$DEVID/$NODE
-
-Trustworthy copy of /local/domain/$DOMID/backend/$KIND/$DEVID/$NODE.
-
-#### /libxl/$DOMID/dm-version ("qemu\_xen"|"qemu\_xen\_traditional") = [n,INTERNAL]
-
-The device model version for a domain.
-
-#### /libxl/$DOMID/remus/netbuf/$DEVID/ifb = STRING [n,INTERNAL]
-
-ifb device used by Remus to buffer network output from the associated vif.
-
-### xenstored specific paths
-
-The /tool/xenstored namespace is created by the xenstore daemon or domain
-for the toolstack to obtain e.g. the domain id of a xenstore domain.
-
-#### /tool/xenstored/domid = INTEGER [n,INTERNAL]
-
-Domain Id of the xenstore domain in case xenstore is provided via a
-domain instead of a daemon in dom0.
-
-[BLKIF]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,blkif.h.html
-[FBIF]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,fbif.h.html
-[HVMPARAMS]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,hvm,params.h.html
-[KBDIF]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,kbdif.h.html
-[LIBXLMEM]: http://xenbits.xen.org/docs/unstable/misc/libxl_memory.txt
-[NETIF]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,netif.h.html
-[SCSIIF]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,vscsiif.h.html
-[SI]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,xen.h.html#Struct_start_info
-[USBIF]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,usbif.h.html
-[VCPU]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,vcpu.h.html
-[XSWIRE]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,xs_wire.h.html
--- /dev/null
+# XenStore Paths
+
+This document attempts to defines all the paths which are in common
+use by either guests, front-/back-end drivers, toolstacks etc.
+
+The XenStore wire protocol itself is described in
+[xenstore.txt](xenstore.txt).
+
+## Notation
+
+This document is intended to be partially machine readable, such that
+test system etc can use it to validate whether the xenstore paths used
+by a test are allowable etc.
+
+Therefore the following notation conventions apply:
+
+A xenstore path is generically defined as:
+
+ PATH = VALUES [TAGS]
+
+ PATH/* [TAGS]
+
+The first syntax defines a simple path with a single value. The second
+syntax defines an aggregated set of paths which are usually described
+externally to this document. The text will give a pointer to the
+appropriate external documentation.
+
+PATH can contain simple regex constructs following the Perl compatible
+regexp syntax described in pcre(3) or perlre(1). In addition the
+following additional wild card names are defined and are evaluated
+before regexp expansion:
+
+* ~ -- expands to an arbitrary a domain's home path (described below).
+ Only valid at the begining of a path.
+* $DEVID -- a per-device type device identifier. Typically an integer.
+* $DOMID -- a domain identifier, an integer. Typically this refers to
+ the "other" domain. i.e. ~ refers to the domain providing a service
+ while $DOMID is the consumer of that service.
+* $UUID -- a UUID in the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+* $INDEX -- an integer used as part of a path when listing a set of
+ values. Typically these integers are contiguous.
+
+VALUES are strings and can take the following forms:
+
+* PATH -- a XenStore path.
+* STRING -- an arbitrary string.
+* INTEGER -- An integer, in decimal representation unless otherwise
+ noted.
+ * MEMKB -- the decimal representation of a number of kilobytes.
+ * EVTCHN -- the decimal representation of an event channel.
+ * GNTREF -- the decimal representation of a grant reference.
+* "a literal string" -- literal strings are contained within quotes.
+* (VALUE | VALUE | ... ) -- a set of alternatives. Alternatives are
+ separated by a "|" and all the alternatives are enclosed in "(" and
+ ")".
+* DISTRIBUTION -- information about a software distribution, comprised
+ of 3 or 4 space separated UTF-8 strings as follows:
+
+ VENDOR -- Commonly used vendor short name,
+ e.g "Citrix" rather than "Citrix Systems
+ Inc."
+
+ PRODUCT -- Commonly used product (e.g. driver) name
+ without version information.
+
+ If the toolstack needs to match on either of the above
+ values it should support Unix glob style matching.
+
+ VERSION -- A version number that will sort properly
+ under coreutils version sorting (sort -V)
+ rules.
+
+ ATTRIBUTES -- Optional human readable text to denote
+ attributes of the software, e.g. "debug".
+ This text is freeform and no meaning
+ should be inferred. It is intended for
+ display purposes only.
+
+* MAC_ADDRESS -- 6 integers, in hexadecimal form, separated by ':',
+ specifying an IEEE 802.3 ethernet MAC address.
+* IPV4_ADDRESS -- 4 integers, in decimal form, separated by '.',
+ specifying an IP version 4 address as described
+ IETF RFC 791.
+* IPV6_ADDRESS -- Up to 8 integers, in hexadecimal form, separated
+ by ':', specifying an IP version 6 address as
+ described in IETF RFC 4291.
+
+Additional TAGS may follow as a comma separated set of the following
+tags enclosed in square brackets.
+
+* w -- Path is writable by the containing domain, that is the domain
+ whose home path ~ this key is under or which /vm/$UUID refers to. By
+ default paths under both of these locations are read only for the
+ domain.
+* n -- Path is neither readable nor writeable for guest domains.
+* HVM -- Path is valid for HVM domains only
+* PV -- Path is valid for PV domains only
+* BACKEND -- Path is valid for a backend domain (AKA driver domain)
+* INTERNAL -- Although a path is visible to the domain its use is
+ reserved for the virtual firmware or Xen platform code. Guest
+ Operating Systems must not read this key or otherwise rely on its
+ presence or contents.
+* DEPRECATED -- This path is deprecated and may be removed in its
+ current form in the future. Guests should not add new dependencies
+ on such paths.
+
+Owning domain means the domain whose home path this tag is found
+under.
+
+Lack of either a __HVM__ or __PV__ tag indicates that the path is
+valid for either type of domain (including PVonHVM and similar mixed
+domain types).
+
+## Domain Home Path
+
+Every domain has a home path within the xenstore hierarchy. This is
+the path where the majority of the domain-visible information about
+each domain is stored.
+
+This path is:
+
+ /local/domain/$DOMID
+
+All non-absolute paths are relative to this path.
+
+Although this path could be considered a "Home Directory" for the
+domain it would not usually be writable by the domain. The tools will
+create writable subdirectories as necessary.
+
+## Per Domain Paths
+
+## General Paths
+
+#### ~/vm = PATH []
+
+A pointer back to the domain's /vm/$UUID path (described below).
+
+#### ~/name = STRING []
+
+The guests name.
+
+#### ~/domid = INTEGER []
+
+The domain's own ID.
+
+#### ~/image/device-model-pid = INTEGER [INTERNAL]
+
+The process ID of the device model associated with this domain, if it
+has one.
+
+#### ~/cpu/[0-9]+/availability = ("online"|"offline") [PV]
+
+One node for each virtual CPU up to the guest's configured
+maximum. Valid values are "online" and "offline". The guest is expected to react to changes in this path by bringing the appropriate VCPU on or offline using the VCPUOP interface described in [xen/include/public/vcpu.h][VCPU]
+
+This protocol is not currently well documented.
+
+#### ~/memory/static-max = MEMKB []
+
+Specifies a static maximum amount memory which this domain should
+expect to be given. In the absence of in-guest memory hotplug support
+this set on domain boot and is usually the maximum amount of RAM which
+a guest can make use of. See [docs/misc/libxl_memory.txt][LIBXLMEM]
+for a description of how memory is accounted for in toolstacks using
+the libxl library.
+
+#### ~/memory/target = MEMKB []
+
+The current balloon target for the domain. The balloon driver within
+the guest is expected to make every effort to every effort use no more
+than this amount of RAM.
+
+#### ~/memory/videoram = MEMKB [HVM,INTERNAL]
+
+The size of the video RAM this domain is configured with.
+
+#### ~/device/suspend/event-channel = ""|EVTCHN [w]
+
+The domain's suspend event channel. The toolstack will create this
+path with an empty value which the guest may choose to overwrite.
+
+If the guest overwrites this, it will be with the number of an unbound
+event channel port it has acquired. The toolstack is expected to use
+an interdomain bind, and then, when it wishes to ask the guest to
+suspend, to signal the event channel.
+
+The guest does not need to explicitly acknowledge the request; indeed,
+there is no explicit signalling by the guest in the reverse direction.
+The guest, when it is ready, simply shuts down (`SCHEDOP_shutdown`)
+with reason code `SHUTDOWN_suspend`. The toolstack is expected to use
+`XEN_DOMCTL_subscribe` to be alerted to guest state changes, and
+`XEN_SYSCTL_getdomaininfolist` to verify that the domain has
+suspended.
+
+Note that the use of this event channel suspend protocol is optional
+for both sides. By writing a non-empty string to the node, the guest
+is advertising its support. However, the toolstack is at liberty to
+use the xenstore-based protocol instead (see ~/control/shutdown,
+below) even if the guest has advertised support for the event channel
+protocol.
+
+#### ~/hvmloader/allow-memory-relocate = ("1"|"0") [HVM,INTERNAL]
+
+If the default low MMIO hole (below 4GiB) is not big enough for all
+the devices, this indicates if hvmloader should relocate guest memory
+into the high memory region (above 4GiB). If "1", hvmloader will
+relocate memory as needed, until 2GiB is reached; if "0", hvmloader
+will not relocate guest memory.
+
+#### ~/hvmloader/bios = ("rombios"|"seabios"|"OVMF") [HVM,INTERNAL]
+
+The BIOS used by this domain.
+
+#### ~/bios-strings/bios-vendor = STRING [HVM,INTERNAL]
+#### ~/bios-strings/bios-version = STRING [HVM,INTERNAL]
+#### ~/bios-strings/system-manufacturer = STRING [HVM,INTERNAL]
+#### ~/bios-strings/system-product-name = STRING [HVM,INTERNAL]
+#### ~/bios-strings/system-version = STRING [HVM,INTERNAL]
+#### ~/bios-strings/system-serial-number = STRING [HVM,INTERNAL]
+#### ~/bios-strings/enclosure-manufacturer = STRING [HVM,INTERNAL]
+#### ~/bios-strings/enclosure-serial-number = STRING [HVM,INTERNAL]
+#### ~/bios-strings/enclosure-asset-tag = STRING [HVM,INTERNAL]
+#### ~/bios-strings/battery-manufacturer = STRING [HVM,INTERNAL]
+#### ~/bios-strings/battery-device-name = STRING [HVM,INTERNAL]
+
+These xenstore values are used to override some of the default string
+values in the SMBIOS table constructed in hvmloader. See the SMBIOS
+table specification at http://www.dmtf.org/standards/smbios/
+
+#### ~/bios-strings/oem-* = STRING [HVM,INTERNAL]
+
+1 to 99 OEM strings can be set in xenstore using values of the form
+'~/bios-strings/oem-1' to '~/bios-strings/oem-99'. These strings will be
+loaded into the SMBIOS type 11 structure.
+
+#### ~/platform/* = ("0"|"1") [HVM,INTERNAL]
+
+Various boolean platform properties.
+
+* acpi -- is ACPI enabled for this domain
+* acpi_s3 -- is ACPI S3 support enabled for this domain
+* acpi_s4 -- is ACPI S4 support enabled for this domain
+* acpi_laptop_slate -- is the ACPI laptop/slate device present in
+ this domain
+
+#### ~/platform/generation-id = INTEGER ":" INTEGER [HVM,INTERNAL]
+
+The lower and upper 64-bit words of the 128-bit VM Generation ID.
+
+This key is used by hvmloader to create the ACPI VM Generation ID
+device. It initialises a 16 octet region of guest memory with this
+value. The guest physical address of this region is saved in the
+HVM_PARAM_VM_GENERATION_ID_ADDR HVM parameter.
+
+If this key is not present, is empty, or is all-zeros ("0:0") then the
+ACPI device is not created.
+
+When restoring a guest, the toolstack may (in certain circumstances)
+need generate a new random generation ID and write it to guest memory
+at the guest physical address in HVM_PARAM_VM_GENERATION_ID_ADDR.
+
+See Microsoft's "Virtual Machine Generation ID" specification for the
+circumstances where the generation ID needs to be changed.
+
+### Frontend device paths
+
+Paravirtual device frontends are generally specified by their own
+directory within the XenStore hierarchy. Usually this is under
+~/device/$TYPE/$DEVID although there are exceptions, e.g. ~/console
+for the first PV console.
+
+#### ~/device/vbd/$DEVID/* []
+
+A virtual block device frontend. Described by
+[xen/include/public/io/blkif.h][BLKIF]
+
+#### ~/device/vfb/$DEVID/* []
+
+A virtual framebuffer frontend. Described by
+[xen/include/public/io/fbif.h][FBIF]
+
+#### ~/device/vkbd/$DEVID/* []
+
+A virtual keyboard device frontend. Described by
+[xen/include/public/io/kbdif.h][KBDIF]
+
+#### ~/device/vif/$DEVID/* []
+
+A virtual network device frontend. Described by
+[xen/include/public/io/netif.h][NETIF]
+
+#### ~/device/vscsi/$DEVID/* []
+
+A virtual scsi device frontend. Described by
+[xen/include/public/io/vscsiif.h][SCSIIF]
+
+#### ~/device/vusb/$DEVID/* []
+
+A virtual usb device frontend. Described by
+[xen/include/public/io/usbif.h][USBIF]
+
+#### ~/device/pvcalls/$DEVID/* []
+
+Paravirtualized POSIX function calls frontend. Described by
+[docs/misc/pvcalls.markdown][PVCALLS]
+
+#### ~/console/* []
+
+The primary PV console device. Described in [console.txt](console.txt)
+
+#### ~/device/console/$DEVID/* []
+
+A secondary PV console device. Described in [console.txt](console.txt)
+
+#### ~/serial/$DEVID/* [HVM]
+
+An emulated serial device. Described in [console.txt](console.txt)
+
+#### ~/store/port = EVTCHN [DEPRECATED]
+
+The event channel used by the domain's connection to XenStore.
+
+This path is deprecated since the same information is provided via the
+[start_info][SI] for PV guests and as an [HVM param][HVMPARAMS] for
+HVM guests. There is an obvious chicken and egg problem with
+extracting this value from xenstore in order to setup the xenstore
+communication ring.
+
+#### ~/store/ring-ref = GNTREF [DEPRECATED]
+
+The grant reference of the domain's XenStore ring.
+
+As with ~/store/port this path is deprecated.
+
+### Backend Device Paths
+
+Paravirtual device backends are generally specified by their own
+directory within the XenStore hierarchy. Usually this is under
+~/backend/$TYPE/$DOMID/$DEVID.
+
+#### ~/backend/vbd/$DOMID/$DEVID/* []
+
+A virtual block device backend. Described by
+[xen/include/public/io/blkif.h][BLKIF]
+
+Uses the in-kernel blkback driver.
+
+#### ~/backend/qdisk/$DOMID/$DEVID/* []
+
+A virtual block device backend. Described by
+[xen/include/public/io/blkif.h][BLKIF]
+
+Uses the qemu based disk backend.
+
+#### ~/backend/tap/$DOMID/$DEVID/* []
+
+A virtual block device backend. Described by
+[xen/include/public/io/blkif.h][BLKIF]
+
+Uses the in-kernel blktap (v1) disk backend (deprecated).
+
+#### ~/backend/vfb/$DOMID/$DEVID/* []
+
+A virtual framebuffer backend. Described by
+[xen/include/public/io/fbif.h][FBIF]
+
+#### ~/backend/vkbd/$DOMID/$DEVID/* []
+
+A virtual keyboard device backend. Described by
+[xen/include/public/io/kbdif.h][KBDIF]
+
+#### ~/backend/vif/$DOMID/$DEVID/* []
+
+A virtual network device backend. Described by
+[xen/include/public/io/netif.h][NETIF]
+
+#### ~/backend/vscsi/$DOMID/$DEVID/* []
+
+A PV SCSI backend.
+
+#### ~/backend/vusb/$DOMID/$DEVID/* []
+
+A PV USB backend. Described by
+[xen/include/public/io/usbif.h][USBIF]
+
+#### ~/backend/pvcalls/$DOMID/$DEVID/* []
+
+A PVCalls backend. Described in [docs/misc/pvcalls.markdown][PVCALLS].
+
+#### ~/backend/console/$DOMID/$DEVID/* []
+
+A PV console backend. Described in [console.txt](console.txt)
+
+#### ~/backend/qusb/$DOMID/$DEVID/* []
+
+A PV USB device backend. Described by
+[xen/include/public/io/usbif.h][USBIF]
+
+Uses the qemu based USB backend.
+
+#### ~/device-model/$DOMID/* [INTERNAL]
+
+Information relating to device models running in the domain. $DOMID is
+the target domain of the device model.
+
+#### ~/libxl/disable_udev = ("1"|"0") []
+
+Indicates whether device hotplug scripts in this domain should be run
+by udev ("0") or will be run by the toolstack directly ("1").
+
+### Platform Feature and Control Paths
+
+#### ~/control/sysrq = (""|COMMAND) [w]
+
+This is the PV SysRq control node. A toolstack can write a single character
+representing a magic SysRq key understood by the Linux kernel. The guest
+acknowledges a request by writing the empty string back to the command node.
+
+This protocol is Linux only.
+
+#### ~/control/shutdown = (""|COMMAND) [w]
+
+This is the PV shutdown control node. A toolstack can write various
+commands here to cause various guest shutdown, reboot or suspend
+activities. The guest acknowledges a request by writing the empty
+string back to the command node.
+
+The precise protocol is not yet documented.
+
+#### ~/control/feature-poweroff = (""|"0"|"1") [w]
+#### ~/control/feature-reboot = (""|"0"|"1") [w]
+#### ~/control/feature-suspend = (""|"0"|"1") [w]
+
+These may be initialized to "" by the toolstack and may then be set
+to 0 or 1 by a guest to indicate whether it is capable or incapable,
+respectively, of responding to the corresponding command when written
+to ~/control/shutdown.
+A toolstack may then sample the feature- value at the point of issuing
+a PV control command and respond accordingly:
+
+"0" -> the frontend should not be expected to respond, so fail the
+ control operation
+"1" -> the frontend should be expected to respond, so wait for it to
+ do so and maybe fail the control operation after some reasonable
+ timeout.
+"" -> the frontend may or may not respond, so wait for it to do so and
+ then maybe try an alternative control mechanism after some
+ reasonable timeout.
+
+Since a toolstack may not initialize these paths, and the parent
+~/control path is read-only to a guest, a guest should not expect a
+write to succeed. If it fails the guest may log the failure but should
+continue to process the corresponding command when written to
+~/control/shutdown regardless.
+
+#### ~/control/feature-s3 = (""|"0"|"1") [w,HVM]
+#### ~/control/feature-s4 = (""|"0"|"1") [w,HVM]
+
+These purpose of these feature flags is identical to feature-poweroff,
+feature-reboot and feature-suspend above but concern triggering the
+S3 or S4 power states of HVM guests.
+A toolstack may create these values, but should not sample them unless
+the corresponding acpi_ feature flag is set in ~/platform.
+
+#### ~/control/platform-feature-multiprocessor-suspend = (0|1) []
+
+Indicates to the guest that this platform supports the multiprocessor
+suspend feature.
+
+#### ~/control/platform-feature-xs_reset_watches = (0|1) []
+
+Indicates to the guest that this platform supports the
+XS_RESET_WATCHES xenstore message. See
+[xen/include/public/io/xs_wire.h][XSWIRE] for the XenStore wire
+protocol definition.
+
+#### ~/control/laptop-slate-mode = (""|"laptop"|"slate") [w]
+
+This is the PV laptop/slate mode control node. If the toolstack has
+provisioned a guest with the ACPI laptop/slate mode device then it
+can write the desired mode here to cause the guest to switch modes if
+necessary. The guest acknowledges a request by writing the empty
+string back to the control node.
+
+#### ~/control/feature-laptop-slate-mode = (""|"0"|"1") [w]
+
+This may be initialized to "" by the toolstack and may then be set
+to 0 or 1 by a guest to indicate whether it is capable or incapable,
+respectively, of responding to a mode value written to
+~/control/laptop-slate-mode.
+
+### Domain Controlled Paths
+
+#### ~/data/* [w]
+
+A domain writable path. Available for arbitrary domain use.
+
+#### ~/drivers/$INDEX = DISTRIBUTION [w]
+
+A domain may write information about installed PV drivers using
+paths of this form.
+
+#### ~/feature/hotplug/vif = ("0"|"1") [w]
+#### ~/feature/hotplug/vbd = ("0"|"1") [w]
+
+By setting these paths to "1" a guest can indicate to a toolstack
+that it is capable of responding immediately to instantiation of,
+respectively, new vif by bringing online a new PV network device or
+a new vbd by bringing online a new PV block device.
+If the guest sets this path to "0" then it is indicating that it is
+definitely unable to respond immediately and hence the toolstack should
+defer instantiaton to the next VM start. However, if the path is absent
+then the toolstack may attempt the operation.
+
+#### ~/attr/vif/$DEVID/name = STRING [w]
+
+A domain may write its internal 'friendly' name for a network device
+using this path using UTF-8 encoding. A toolstack or UI may use this
+for display purposes. No particular meaning should be inferred from the
+name.
+
+#### ~/attr/vif/$DEVID/mac/$INDEX = MAC_ADDRESS [w]
+
+Paths of this form may be written by the guest to indicate MAC addresses
+it is currently using. These may be multicast or unicast addresses. For
+any of the paths the value of $INDEX is arbitrary.
+The values written are primarily for display purposes and must not be used
+for packet filtering or routing purposes.
+
+#### ~/attr/vif/$DEVID/ipv4/$INDEX = IPV4_ADDRESS [w]
+#### ~/attr/vif/$DEVID/ipv6/$INDEX = IPV6_ADDRESS [w]
+
+Paths of this form may be written by the guest to indicate IP addresses
+in use by the stack bound to the network frontend. For any of the paths
+the value of $INDEX is arbitrary.
+The values written are primarily for display purposes and must not be used
+for packet filtering or routing purposes. A toolstack may attempt to use an
+address written in one of these paths to, for example, establish a VNC
+session to the guest (although clearly some level of trust is placed
+in the value supplied by the guest in this case).
+
+### Paths private to the toolstack
+
+#### ~/device-model/$DOMID/state [w]
+
+Contains the status of the device models running on the domain.
+
+#### ~/device-model/$DOMID/backends/* [w]
+
+Backend types the device model is supporting. Each entry below backends
+is a directory which may contain further nodes specific to the backend
+type. The name of each backend directory is the same as the backend type
+(e.g. "qdisk").
+
+#### ~/libxl/$DOMID/qdisk-backend-pid [w]
+
+Contains the PIDs of the device models running on the domain.
+
+## Virtual Machine Paths
+
+The /vm/$UUID namespace is used by toolstacks to store various
+information relating to the domain which is not intended to be guest
+visible (hence they are all tagged [n,INTERNAL]).
+
+Several of the keys here are not well defined and/or not well located
+and are liable to be replaced with more fully defined paths in the
+future.
+
+### /vm/$UUID/uuid = UUID [n,INTERNAL]
+
+Value is the same UUID as the path.
+
+### /vm/$UUID/name = STRING [n,INTERNAL]
+
+The domain's name.
+
+### /vm/$UUID/image/* [n,INTERNAL]
+
+Various information relating to the domain builder used for this guest.
+
+### /vm/$UUID/start_time = INTEGER "." INTEGER [n,INTERNAL]
+
+The time which the guest was started in SECONDS.MICROSECONDS format
+
+### /vm/$UUID/rtc/timeoffset = ""|INTEGER [n,HVM,INTERNAL]
+
+The guest's virtual time offset from UTC in seconds.
+
+## Platform-Level paths
+
+### libxl Specific Paths
+
+#### /libxl/$DOMID/device/$KIND/$DEVID
+
+Created by libxl for every frontend/backend pair created for $DOMID.
+Used by libxl for enumeration and management of the device.
+
+#### /libxl/$DOMID/device/$KIND/$DEVID/frontend
+
+Path in xenstore to the frontend, normally
+/local/domain/$DOMID/device/$KIND/$DEVID
+
+#### /libxl/$DOMID/device/$KIND/$DEVID/backend
+
+Path in xenstore to the backend, normally
+/local/domain/$BACKEND_DOMID/backend/$KIND/$DOMID/$DEVID
+
+#### /libxl/$DOMID/device/$KIND/$DEVID/$NODE
+
+Trustworthy copy of /local/domain/$DOMID/backend/$KIND/$DEVID/$NODE.
+
+#### /libxl/$DOMID/dm-version ("qemu_xen"|"qemu_xen_traditional") = [n,INTERNAL]
+
+The device model version for a domain.
+
+#### /libxl/$DOMID/remus/netbuf/$DEVID/ifb = STRING [n,INTERNAL]
+
+ifb device used by Remus to buffer network output from the associated vif.
+
+### xenstored specific paths
+
+The /tool/xenstored namespace is created by the xenstore daemon or domain
+for the toolstack to obtain e.g. the domain id of a xenstore domain.
+
+#### /tool/xenstored/domid = INTEGER [n,INTERNAL]
+
+Domain Id of the xenstore domain in case xenstore is provided via a
+domain instead of a daemon in dom0.
+
+[BLKIF]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,blkif.h.html
+[FBIF]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,fbif.h.html
+[HVMPARAMS]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,hvm,params.h.html
+[KBDIF]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,kbdif.h.html
+[LIBXLMEM]: http://xenbits.xen.org/docs/unstable/misc/libxl_memory.txt
+[NETIF]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,netif.h.html
+[SCSIIF]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,vscsiif.h.html
+[SI]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,xen.h.html#Struct_start_info
+[USBIF]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,usbif.h.html
+[VCPU]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,vcpu.h.html
+[XSWIRE]: http://xenbits.xen.org/docs/unstable/hypercall/x86_64/include,public,io,xs_wire.h.html
+++ /dev/null
-# Intel Platform Shared Resource Monitoring/Control in xl
-
-This document introduces Intel Platform Shared Resource Monitoring/Control
-technologies, their basic concepts and the xl interfaces.
-
-## Cache Monitoring Technology (CMT)
-
-Cache Monitoring Technology (CMT) is a new feature available on Intel Haswell
-and later server platforms that allows an OS or Hypervisor/VMM to determine
-the usage of cache (currently only L3 cache supported) by applications running
-on the platform. A Resource Monitoring ID (RMID) is the abstraction of the
-application(s) that will be monitored for its cache usage. The CMT hardware
-tracks cache utilization of memory accesses according to the RMID and reports
-monitored data via a counter register.
-
-For more detailed information please refer to Intel SDM chapter
-"Platform Shared Resource Monitoring: Cache Monitoring Technology".
-
-In Xen's implementation, each domain in the system can be assigned a RMID
-independently, while RMID=0 is reserved for monitoring domains that don't
-have CMT service attached. RMID is opaque for xl/libxl and is only used in
-hypervisor.
-
-### xl interfaces
-
-A domain is assigned a RMID implicitly by attaching it to CMT service:
-
-`xl psr-cmt-attach <domid>`
-
-After that, cache usage for the domain can be shown by:
-
-`xl psr-cmt-show cache-occupancy <domid>`
-
-Once monitoring is not needed any more, the domain can be detached from the
-CMT service by:
-
-`xl psr-cmt-detach <domid>`
-
-An attach may fail because of no free RMID available. In such case unused
-RMID(s) can be freed by detaching corresponding domains from CMT service.
-
-Maximum RMID and supported monitor types in the system can be obtained by:
-
-`xl psr-hwinfo --cmt`
-
-## Memory Bandwidth Monitoring (MBM)
-
-Memory Bandwidth Monitoring(MBM) is a new hardware feature available on Intel
-Broadwell and later server platforms which builds on the CMT infrastructure to
-allow monitoring of system memory bandwidth. It introduces two new monitoring
-event type to monitor system total/local memory bandwidth. The same RMID can
-be used to monitor both cache usage and memory bandwidth at the same time.
-
-For more detailed information please refer to Intel SDM chapter
-"Overview of Cache Monitoring Technology and Memory Bandwidth Monitoring".
-
-In Xen's implementation, MBM shares the same set of underlying monitoring
-service with CMT and can be used to monitor memory bandwidth on a per domain
-basis.
-
-The xl interfaces are the same with that of CMT. The difference is the
-monitor type is corresponding memory monitoring type (local-mem-bandwidth/
-total-mem-bandwidth instead of cache-occupancy). E.g. after a `xl psr-cmt-attach`:
-
-`xl psr-cmt-show local-mem-bandwidth <domid>`
-
-`xl psr-cmt-show total-mem-bandwidth <domid>`
-
-## Cache Allocation Technology (CAT)
-
-Cache Allocation Technology (CAT) is a new feature available on Intel
-Broadwell and later server platforms that allows an OS or Hypervisor/VMM to
-partition cache allocation (i.e. L3/L2 cache) based on application priority or
-Class of Service (COS). Each COS is configured using capacity bitmasks (CBM)
-which represent cache capacity and indicate the degree of overlap and
-isolation between classes. System cache resource is divided into numbers of
-minimum portions which is then made up into subset for cache partition. Each
-portion corresponds to a bit in CBM and the set bit represents the
-corresponding cache portion is available.
-
-For example, assuming a system with 8 portions and 3 domains:
-
- * A CBM of 0xff for every domain means each domain can access the whole cache.
- This is the default.
-
- * Giving one domain a CBM of 0x0f and the other two domain's 0xf0 means that
- the first domain gets exclusive access to half of the cache (half of the
- portions) and the other two will share the other half.
-
- * Giving one domain a CBM of 0x0f, one 0x30 and the last 0xc0 would give the
- first domain exclusive access to half the cache, and the other two exclusive
- access to one quarter each.
-
-For more detailed information please refer to Intel SDM chapter
-"Platform Shared Resource Control: Cache Allocation Technology".
-
-In Xen's implementation, CBM can be configured with libxl/xl interfaces but
-COS is maintained in hypervisor only. The cache partition granularity is per
-domain, each domain has COS=0 assigned by default, the corresponding CBM is
-all-ones, which means all the cache resource can be used by default.
-
-### xl interfaces
-
-System CAT information such as maximum COS and CBM length can be obtained by:
-
-`xl psr-hwinfo --cat`
-
-The simplest way to change a domain's CBM from its default is running:
-
-`xl psr-cat-set [OPTIONS] <domid> <cbm>`
-
-where cbm is a number to represent the corresponding cache subset can be used.
-A cbm is valid only when:
-
- * Set bits only exist in the range of [0, cbm_len), where cbm_len can be
- obtained with `xl psr-hwinfo --cat`.
- * All the set bits are contiguous.
-
-In a multi-socket system, the same cbm will be set on each socket by default.
-Per socket cbm can be specified with the `--socket SOCKET` option.
-
-In different systems, the different cache level is supported, e.g. L3 cache or
-L2 cache. Per cache level cbm can be specified with the `--level LEVEL` option.
-
-Setting the CBM may not be successful if insufficient COS is available. In
-such case unused COS(es) may be freed by setting CBM of all related domains to
-its default value(all-ones).
-
-Per domain CBM settings can be shown by:
-
-`xl psr-cat-show [OPTIONS] <domid>`
-
-In different systems, the different cache level is supported, e.g. L3 cache or
-L2 cache. Per cache level cbm can be specified with the `--level LEVEL` option.
-
-## Code and Data Prioritization (CDP)
-
-Code and Data Prioritization (CDP) Technology is an extension of CAT, which
-is available on Intel Broadwell and later server platforms. CDP enables
-isolation and separate prioritization of code and data fetches to the L3
-cache in a software configurable manner, which can enable workload
-prioritization and tuning of cache capacity to the characteristics of the
-workload. CDP extends Cache Allocation Technology (CAT) by providing
-separate code and data masks per Class of Service (COS).
-
-CDP can be enabled by adding `psr=cdp` to Xen command line.
-
-When CDP is enabled,
-
- * the CAT masks are re-mapped into interleaved pairs of masks for data or
- code fetches.
-
- * the range of COS for CAT is re-indexed, with the lower-half of the COS
- range available for CDP.
-
-CDP allows the OS or Hypervisor to partition cache allocation in a more
-fine-grained manner. Code cache and data cache can be specified independently.
-With CDP enabled, one COS corresponds to two CBMs (code CBM & data CBM),
-since the sum of CBMs is fixed, that means the number of available COSes
-will reduce by half when CDP is on.
-
-For more detailed information please refer to Intel SDM chapter
-"Platform Shared Resource Control: Cache Allocation Technology".
-
-The xl interfaces are the same with that of CAT. The difference is that
-CBM type can be passed as option to set code CBM or data CBM.
-
-When CDP is enabled, `-c` or `--code` option is available to set code CBM
-for the domain.
-
-When CDP is enabled, `-d` or `--data` option is available to set data CBM
-for the domain.
-
-If neither `-c` nor `-d` option is specified when CDP is on, the same code
-CBM and data CBM will be set for the domain. Passing both `-c` and `-d`
-options is invalid.
-
-Example:
-
-Setting code CBM for a domain:
-`xl psr-cat-set -c <domid> <cbm>`
-
-Setting data CBM for a domain:
-`xl psr-cat-set -d <domid> <cbm>`
-
-Setting the same code and data CBM for a domain:
-`xl psr-cat-set <domid> <cbm>`
-
-## Memory Bandwidth Allocation (MBA)
-
-Memory Bandwidth Allocation (MBA) is a new feature available on Intel
-Skylake and later server platforms that allows an OS or Hypervisor/VMM to
-slow misbehaving apps/VMs by using a credit-based throttling mechanism. To
-enforce bandwidth on a specific domain, just set throttling value (THRTL)
-into Class of Service (COS). MBA provides two THRTL mode. One is linear mode
-and the other is non-linear mode.
-
-In the linear mode the input precision is defined as 100-(THRTL_MAX). Values
-not an even multiple of the precision (e.g., 12%) will be rounded down (e.g.,
-to 10% delay by the hardware).
-
-If linear values are not supported then input delay values are powers-of-two
-from zero to the THRTL_MAX value from CPUID. In this case any values not a power
-of two will be rounded down the next nearest power of two.
-
-For example, assuming a system with 2 domains:
-
- * A THRTL of 0x0 for every domain means each domain can access the whole cache
- without any delay. This is the default.
-
- * Linear mode: Giving one domain a THRTL of 0xC and the other domain's 0 means
- that the first domain gets 10% delay to access the cache and the other one
- without any delay.
-
- * Non-linear mode: Giving one domain a THRTL of 0xC and the other domain's 0
- means that the first domain gets 8% delay to access the cache and the other
- one without any delay.
-
-For more detailed information please refer to Intel SDM chapter
-"Introduction to Memory Bandwidth Allocation".
-
-In Xen's implementation, THRTL can be configured with libxl/xl interfaces but
-COS is maintained in hypervisor only. The cache partition granularity is per
-domain, each domain has COS=0 assigned by default, the corresponding THRTL is
-0, which means all the cache resource can be accessed without delay.
-
-### xl interfaces
-
-System MBA information such as maximum COS and maximum THRTL can be obtained by:
-
-`xl psr-hwinfo --mba`
-
-The simplest way to change a domain's THRTL from its default is running:
-
-`xl psr-mba-set [OPTIONS] <domid> <thrtl>`
-
-In a multi-socket system, the same thrtl will be set on each socket by default.
-Per socket thrtl can be specified with the `--socket SOCKET` option.
-
-Setting the THRTL may not be successful if insufficient COS is available. In
-such case unused COS(es) may be freed by setting THRTL of all related domains to
-its default value(0).
-
-Per domain THRTL settings can be shown by:
-
-`xl psr-mba-show [OPTIONS] <domid>`
-
-For linear mode, it shows the decimal value. For non-linear mode, it shows
-hexadecimal value.
-
-## Reference
-
-[1] Intel SDM
-(http://www.intel.com/content/www/us/en/processors/architectures-software-developer-manuals.html).
--- /dev/null
+# Intel Platform Shared Resource Monitoring/Control in xl
+
+This document introduces Intel Platform Shared Resource Monitoring/Control
+technologies, their basic concepts and the xl interfaces.
+
+## Cache Monitoring Technology (CMT)
+
+Cache Monitoring Technology (CMT) is a new feature available on Intel Haswell
+and later server platforms that allows an OS or Hypervisor/VMM to determine
+the usage of cache (currently only L3 cache supported) by applications running
+on the platform. A Resource Monitoring ID (RMID) is the abstraction of the
+application(s) that will be monitored for its cache usage. The CMT hardware
+tracks cache utilization of memory accesses according to the RMID and reports
+monitored data via a counter register.
+
+For more detailed information please refer to Intel SDM chapter
+"Platform Shared Resource Monitoring: Cache Monitoring Technology".
+
+In Xen's implementation, each domain in the system can be assigned a RMID
+independently, while RMID=0 is reserved for monitoring domains that don't
+have CMT service attached. RMID is opaque for xl/libxl and is only used in
+hypervisor.
+
+### xl interfaces
+
+A domain is assigned a RMID implicitly by attaching it to CMT service:
+
+`xl psr-cmt-attach <domid>`
+
+After that, cache usage for the domain can be shown by:
+
+`xl psr-cmt-show cache-occupancy <domid>`
+
+Once monitoring is not needed any more, the domain can be detached from the
+CMT service by:
+
+`xl psr-cmt-detach <domid>`
+
+An attach may fail because of no free RMID available. In such case unused
+RMID(s) can be freed by detaching corresponding domains from CMT service.
+
+Maximum RMID and supported monitor types in the system can be obtained by:
+
+`xl psr-hwinfo --cmt`
+
+## Memory Bandwidth Monitoring (MBM)
+
+Memory Bandwidth Monitoring(MBM) is a new hardware feature available on Intel
+Broadwell and later server platforms which builds on the CMT infrastructure to
+allow monitoring of system memory bandwidth. It introduces two new monitoring
+event type to monitor system total/local memory bandwidth. The same RMID can
+be used to monitor both cache usage and memory bandwidth at the same time.
+
+For more detailed information please refer to Intel SDM chapter
+"Overview of Cache Monitoring Technology and Memory Bandwidth Monitoring".
+
+In Xen's implementation, MBM shares the same set of underlying monitoring
+service with CMT and can be used to monitor memory bandwidth on a per domain
+basis.
+
+The xl interfaces are the same with that of CMT. The difference is the
+monitor type is corresponding memory monitoring type (local-mem-bandwidth/
+total-mem-bandwidth instead of cache-occupancy). E.g. after a `xl psr-cmt-attach`:
+
+`xl psr-cmt-show local-mem-bandwidth <domid>`
+
+`xl psr-cmt-show total-mem-bandwidth <domid>`
+
+## Cache Allocation Technology (CAT)
+
+Cache Allocation Technology (CAT) is a new feature available on Intel
+Broadwell and later server platforms that allows an OS or Hypervisor/VMM to
+partition cache allocation (i.e. L3/L2 cache) based on application priority or
+Class of Service (COS). Each COS is configured using capacity bitmasks (CBM)
+which represent cache capacity and indicate the degree of overlap and
+isolation between classes. System cache resource is divided into numbers of
+minimum portions which is then made up into subset for cache partition. Each
+portion corresponds to a bit in CBM and the set bit represents the
+corresponding cache portion is available.
+
+For example, assuming a system with 8 portions and 3 domains:
+
+ * A CBM of 0xff for every domain means each domain can access the whole cache.
+ This is the default.
+
+ * Giving one domain a CBM of 0x0f and the other two domain's 0xf0 means that
+ the first domain gets exclusive access to half of the cache (half of the
+ portions) and the other two will share the other half.
+
+ * Giving one domain a CBM of 0x0f, one 0x30 and the last 0xc0 would give the
+ first domain exclusive access to half the cache, and the other two exclusive
+ access to one quarter each.
+
+For more detailed information please refer to Intel SDM chapter
+"Platform Shared Resource Control: Cache Allocation Technology".
+
+In Xen's implementation, CBM can be configured with libxl/xl interfaces but
+COS is maintained in hypervisor only. The cache partition granularity is per
+domain, each domain has COS=0 assigned by default, the corresponding CBM is
+all-ones, which means all the cache resource can be used by default.
+
+### xl interfaces
+
+System CAT information such as maximum COS and CBM length can be obtained by:
+
+`xl psr-hwinfo --cat`
+
+The simplest way to change a domain's CBM from its default is running:
+
+`xl psr-cat-set [OPTIONS] <domid> <cbm>`
+
+where cbm is a number to represent the corresponding cache subset can be used.
+A cbm is valid only when:
+
+ * Set bits only exist in the range of [0, cbm_len), where cbm_len can be
+ obtained with `xl psr-hwinfo --cat`.
+ * All the set bits are contiguous.
+
+In a multi-socket system, the same cbm will be set on each socket by default.
+Per socket cbm can be specified with the `--socket SOCKET` option.
+
+In different systems, the different cache level is supported, e.g. L3 cache or
+L2 cache. Per cache level cbm can be specified with the `--level LEVEL` option.
+
+Setting the CBM may not be successful if insufficient COS is available. In
+such case unused COS(es) may be freed by setting CBM of all related domains to
+its default value(all-ones).
+
+Per domain CBM settings can be shown by:
+
+`xl psr-cat-show [OPTIONS] <domid>`
+
+In different systems, the different cache level is supported, e.g. L3 cache or
+L2 cache. Per cache level cbm can be specified with the `--level LEVEL` option.
+
+## Code and Data Prioritization (CDP)
+
+Code and Data Prioritization (CDP) Technology is an extension of CAT, which
+is available on Intel Broadwell and later server platforms. CDP enables
+isolation and separate prioritization of code and data fetches to the L3
+cache in a software configurable manner, which can enable workload
+prioritization and tuning of cache capacity to the characteristics of the
+workload. CDP extends Cache Allocation Technology (CAT) by providing
+separate code and data masks per Class of Service (COS).
+
+CDP can be enabled by adding `psr=cdp` to Xen command line.
+
+When CDP is enabled,
+
+ * the CAT masks are re-mapped into interleaved pairs of masks for data or
+ code fetches.
+
+ * the range of COS for CAT is re-indexed, with the lower-half of the COS
+ range available for CDP.
+
+CDP allows the OS or Hypervisor to partition cache allocation in a more
+fine-grained manner. Code cache and data cache can be specified independently.
+With CDP enabled, one COS corresponds to two CBMs (code CBM & data CBM),
+since the sum of CBMs is fixed, that means the number of available COSes
+will reduce by half when CDP is on.
+
+For more detailed information please refer to Intel SDM chapter
+"Platform Shared Resource Control: Cache Allocation Technology".
+
+The xl interfaces are the same with that of CAT. The difference is that
+CBM type can be passed as option to set code CBM or data CBM.
+
+When CDP is enabled, `-c` or `--code` option is available to set code CBM
+for the domain.
+
+When CDP is enabled, `-d` or `--data` option is available to set data CBM
+for the domain.
+
+If neither `-c` nor `-d` option is specified when CDP is on, the same code
+CBM and data CBM will be set for the domain. Passing both `-c` and `-d`
+options is invalid.
+
+Example:
+
+Setting code CBM for a domain:
+`xl psr-cat-set -c <domid> <cbm>`
+
+Setting data CBM for a domain:
+`xl psr-cat-set -d <domid> <cbm>`
+
+Setting the same code and data CBM for a domain:
+`xl psr-cat-set <domid> <cbm>`
+
+## Memory Bandwidth Allocation (MBA)
+
+Memory Bandwidth Allocation (MBA) is a new feature available on Intel
+Skylake and later server platforms that allows an OS or Hypervisor/VMM to
+slow misbehaving apps/VMs by using a credit-based throttling mechanism. To
+enforce bandwidth on a specific domain, just set throttling value (THRTL)
+into Class of Service (COS). MBA provides two THRTL mode. One is linear mode
+and the other is non-linear mode.
+
+In the linear mode the input precision is defined as 100-(THRTL_MAX). Values
+not an even multiple of the precision (e.g., 12%) will be rounded down (e.g.,
+to 10% delay by the hardware).
+
+If linear values are not supported then input delay values are powers-of-two
+from zero to the THRTL_MAX value from CPUID. In this case any values not a power
+of two will be rounded down the next nearest power of two.
+
+For example, assuming a system with 2 domains:
+
+ * A THRTL of 0x0 for every domain means each domain can access the whole cache
+ without any delay. This is the default.
+
+ * Linear mode: Giving one domain a THRTL of 0xC and the other domain's 0 means
+ that the first domain gets 10% delay to access the cache and the other one
+ without any delay.
+
+ * Non-linear mode: Giving one domain a THRTL of 0xC and the other domain's 0
+ means that the first domain gets 8% delay to access the cache and the other
+ one without any delay.
+
+For more detailed information please refer to Intel SDM chapter
+"Introduction to Memory Bandwidth Allocation".
+
+In Xen's implementation, THRTL can be configured with libxl/xl interfaces but
+COS is maintained in hypervisor only. The cache partition granularity is per
+domain, each domain has COS=0 assigned by default, the corresponding THRTL is
+0, which means all the cache resource can be accessed without delay.
+
+### xl interfaces
+
+System MBA information such as maximum COS and maximum THRTL can be obtained by:
+
+`xl psr-hwinfo --mba`
+
+The simplest way to change a domain's THRTL from its default is running:
+
+`xl psr-mba-set [OPTIONS] <domid> <thrtl>`
+
+In a multi-socket system, the same thrtl will be set on each socket by default.
+Per socket thrtl can be specified with the `--socket SOCKET` option.
+
+Setting the THRTL may not be successful if insufficient COS is available. In
+such case unused COS(es) may be freed by setting THRTL of all related domains to
+its default value(0).
+
+Per domain THRTL settings can be shown by:
+
+`xl psr-mba-show [OPTIONS] <domid>`
+
+For linear mode, it shows the decimal value. For non-linear mode, it shows
+hexadecimal value.
+
+## Reference
+
+[1] Intel SDM
+(http://www.intel.com/content/www/us/en/processors/architectures-software-developer-manuals.html).
projects / people / liuw / xen.git / commitdiff