KVM: x86: Untag addresses for LAM emulation where applicable

Stub in vmx_get_untagged_addr() and wire up calls from the emulator (via
get_untagged_addr()) and "direct" calls from various VM-Exit handlers in
VMX where LAM untagging is supposed to be applied.  Defer implementing
the guts of vmx_get_untagged_addr() to future patches purely to make the
changes easier to consume.

LAM is active only for 64-bit linear addresses and several types of
accesses are exempted.

- Cases need to untag address (handled in get_vmx_mem_address())
  Operand(s) of VMX instructions and INVPCID.
  Operand(s) of SGX ENCLS.

- Cases LAM doesn't apply to (no change needed)
  Operand of INVLPG.
  Linear address in INVPCID descriptor.
  Linear address in INVVPID descriptor.
  BASEADDR specified in SECS of ECREATE.

Note:
  - LAM doesn't apply to write to control registers or MSRs
  - LAM masking is applied before walking page tables, i.e. the faulting
    linear address in CR2 doesn't contain the metadata.
  - The guest linear address saved in VMCS doesn't contain metadata.

Signed-off-by: Binbin Wu <binbin.wu@linux.intel.com>
Reviewed-by: Chao Gao <chao.gao@intel.com>
Tested-by: Xuelian Guo <xuelian.guo@intel.com>
Link: https://lore.kernel.org/r/20230913124227.12574-10-binbin.wu@linux.intel.com
[sean: massage changelog]
Signed-off-by: Sean Christopherson <seanjc@google.com>

diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 51622878d6e4450e95099f967973595ca317ed92..4ba46e1b29d2b23f13adf8773cef07dc9e74e696 100644 (file)
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -4980,6 +4980,7 @@ int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification,
                else
                        *ret = off;
 
+               *ret = vmx_get_untagged_addr(vcpu, *ret, 0);
                /* Long mode: #GP(0)/#SS(0) if the memory address is in a
                 * non-canonical form. This is the only check on the memory
                 * destination for long mode!
@@ -5797,6 +5798,10 @@ static int handle_invvpid(struct kvm_vcpu *vcpu)
        vpid02 = nested_get_vpid02(vcpu);
        switch (type) {
        case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
+               /*
+                * LAM doesn't apply to addresses that are inputs to TLB
+                * invalidation.
+                */
                if (!operand.vpid ||
                    is_noncanonical_address(operand.gla, vcpu))
                        return nested_vmx_fail(vcpu,

diff --git a/arch/x86/kvm/vmx/sgx.c b/arch/x86/kvm/vmx/sgx.c
index 3e822e58249753c5ff267b5ce4f69e2ede9a00c9..6fef01e0536e5079c34e6efc6b64716119c10c4e 100644 (file)
--- a/arch/x86/kvm/vmx/sgx.c
+++ b/arch/x86/kvm/vmx/sgx.c
@@ -37,6 +37,7 @@ static int sgx_get_encls_gva(struct kvm_vcpu *vcpu, unsigned long offset,
        if (!IS_ALIGNED(*gva, alignment)) {
                fault = true;
        } else if (likely(is_64_bit_mode(vcpu))) {
+               *gva = vmx_get_untagged_addr(vcpu, *gva, 0);
                fault = is_noncanonical_address(*gva, vcpu);
        } else {
                *gva &= 0xffffffff;

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 69072e418c73daab3e7f961d9ed725ac73af8e83..d7ff06d7772022b0950b9b44a393979a29a4b9a0 100644 (file)
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -8205,6 +8205,11 @@ static void vmx_vm_destroy(struct kvm *kvm)
        free_pages((unsigned long)kvm_vmx->pid_table, vmx_get_pid_table_order(kvm));
 }
 
+gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags)
+{
+       return gva;
+}
+
 static struct kvm_x86_ops vmx_x86_ops __initdata = {
        .name = KBUILD_MODNAME,
 
@@ -8345,6 +8350,8 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = {
        .complete_emulated_msr = kvm_complete_insn_gp,
 
        .vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector,
+
+       .get_untagged_addr = vmx_get_untagged_addr,
 };
 
 static unsigned int vmx_handle_intel_pt_intr(void)

diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h
index c2130d2c8e24bb5ff3a529a4bde67f875376adda..45cee1a8bc0a413bd95f01bb24caa036ae8ada64 100644 (file)
--- a/arch/x86/kvm/vmx/vmx.h
+++ b/arch/x86/kvm/vmx/vmx.h
@@ -420,6 +420,8 @@ void vmx_enable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type);
 u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu);
 u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu);
 
+gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags);
+
 static inline void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr,
                                             int type, bool value)
 {

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 557872c0331a14e4d9ccfc481d1befd01ffa4d35..ecfe97aa35c2a6c4a414542cabd19f06068628a2 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -13563,6 +13563,10 @@ int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)
 
        switch (type) {
        case INVPCID_TYPE_INDIV_ADDR:
+               /*
+                * LAM doesn't apply to addresses that are inputs to TLB
+                * invalidation.
+                */
                if ((!pcid_enabled && (operand.pcid != 0)) ||
                    is_noncanonical_address(operand.gla, vcpu)) {
                        kvm_inject_gp(vcpu, 0);