iommu/amd: Add 5 level guest page table support

Newer AMD IOMMU supports 5 level guest page table (v2 page table). If both
processor and IOMMU supports 5 level page table then enable it. Otherwise
fall back to 4 level page table.

Co-developed-by: Wei Huang <wei.huang2@amd.com>
Signed-off-by: Wei Huang <wei.huang2@amd.com>
Reviewed-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
Signed-off-by: Vasant Hegde <vasant.hegde@amd.com>
Link: https://lore.kernel.org/r/20230310090000.1117786-1-vasant.hegde@amd.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>

diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 20a142b544985026c8bea015a63a121fbb1de5ac..e98f20a9bdd82e6be58e4d9938094f18979482b6 100644 (file)
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -34,6 +34,7 @@ extern int amd_iommu_reenable(int);
 extern int amd_iommu_enable_faulting(void);
 extern int amd_iommu_guest_ir;
 extern enum io_pgtable_fmt amd_iommu_pgtable;
+extern int amd_iommu_gpt_level;
 
 /* IOMMUv2 specific functions */
 struct iommu_domain;

diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index e5cfdeaaad48a4ecf3f3eea719e458006540fe24..3cda781b6c7f240d1c3e8a715f4bf9ef9e5e0672 100644 (file)
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -93,6 +93,8 @@
 #define FEATURE_GA             (1ULL<<7)
 #define FEATURE_HE             (1ULL<<8)
 #define FEATURE_PC             (1ULL<<9)
+#define FEATURE_GATS_SHIFT     (12)
+#define FEATURE_GATS_MASK      (3ULL)
 #define FEATURE_GAM_VAPIC      (1ULL<<21)
 #define FEATURE_GIOSUP         (1ULL<<48)
 #define FEATURE_EPHSUP         (1ULL<<50)
@@ -305,6 +307,9 @@
 #define PAGE_MODE_6_LEVEL 0x06
 #define PAGE_MODE_7_LEVEL 0x07
 
+#define GUEST_PGTABLE_4_LEVEL  0x00
+#define GUEST_PGTABLE_5_LEVEL  0x01
+
 #define PM_LEVEL_SHIFT(x)      (12 + ((x) * 9))
 #define PM_LEVEL_SIZE(x)       (((x) < 6) ? \
                                  ((1ULL << PM_LEVEL_SHIFT((x))) - 1): \
@@ -398,6 +403,8 @@
 #define DTE_GCR3_SHIFT_B       16
 #define DTE_GCR3_SHIFT_C       43
 
+#define DTE_GPT_LEVEL_SHIFT    54
+
 #define GCR3_VALID             0x01ULL
 
 #define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL)

diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index 141eca29c951155e4b63b730ae08647f51dc9361..c46ee218d387dd0566123534a01abad34b42c33f 100644 (file)
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -153,6 +153,8 @@ bool amd_iommu_dump;
 bool amd_iommu_irq_remap __read_mostly;
 
 enum io_pgtable_fmt amd_iommu_pgtable = AMD_IOMMU_V1;
+/* Guest page table level */
+int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL;
 
 int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
 static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
@@ -306,6 +308,11 @@ static bool check_feature_on_all_iommus(u64 mask)
        return !!(amd_iommu_efr & mask);
 }
 
+static inline int check_feature_gpt_level(void)
+{
+       return ((amd_iommu_efr >> FEATURE_GATS_SHIFT) & FEATURE_GATS_MASK);
+}
+
 /*
  * For IVHD type 0x11/0x40, EFR is also available via IVHD.
  * Default to IVHD EFR since it is available sooner
@@ -2155,8 +2162,10 @@ static void print_iommu_info(void)
                if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
                        pr_info("X2APIC enabled\n");
        }
-       if (amd_iommu_pgtable == AMD_IOMMU_V2)
-               pr_info("V2 page table enabled\n");
+       if (amd_iommu_pgtable == AMD_IOMMU_V2) {
+               pr_info("V2 page table enabled (Paging mode : %d level)\n",
+                       amd_iommu_gpt_level);
+       }
 }
 
 static int __init amd_iommu_init_pci(void)
@@ -3026,6 +3035,11 @@ static int __init early_amd_iommu_init(void)
        if (ret)
                goto out;
 
+       /* 5 level guest page table */
+       if (cpu_feature_enabled(X86_FEATURE_LA57) &&
+           check_feature_gpt_level() == GUEST_PGTABLE_5_LEVEL)
+               amd_iommu_gpt_level = PAGE_MODE_5_LEVEL;
+
        /* Disable any previously enabled IOMMUs */
        if (!is_kdump_kernel() || amd_iommu_disabled)
                disable_iommus();
@@ -3557,6 +3571,11 @@ __setup("ivrs_acpihid",          parse_ivrs_acpihid);
 
 bool amd_iommu_v2_supported(void)
 {
+       /* CPU page table size should match IOMMU guest page table size */
+       if (cpu_feature_enabled(X86_FEATURE_LA57) &&
+           amd_iommu_gpt_level != PAGE_MODE_5_LEVEL)
+               return false;
+
        /*
         * Since DTE[Mode]=0 is prohibited on SNP-enabled system
         * (i.e. EFR[SNPSup]=1), IOMMUv2 page table cannot be used without

diff --git a/drivers/iommu/amd/io_pgtable_v2.c b/drivers/iommu/amd/io_pgtable_v2.c
index 9f8b423f7630d25890a5b104cb56b5fba041328a..27c3015947e6fdc2fec5da7bc92614014af9d76f 100644 (file)
--- a/drivers/iommu/amd/io_pgtable_v2.c
+++ b/drivers/iommu/amd/io_pgtable_v2.c
@@ -37,8 +37,7 @@
 
 static inline int get_pgtable_level(void)
 {
-       /* 5 level page table is not supported */
-       return PAGE_MODE_4_LEVEL;
+       return amd_iommu_gpt_level;
 }
 
 static inline bool is_large_pte(u64 pte)
@@ -379,6 +378,7 @@ static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo
        struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg);
        struct protection_domain *pdom = (struct protection_domain *)cookie;
        int ret;
+       int ias = IOMMU_IN_ADDR_BIT_SIZE;
 
        pgtable->pgd = alloc_pgtable_page(pdom->nid, GFP_ATOMIC);
        if (!pgtable->pgd)
@@ -388,12 +388,15 @@ static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo
        if (ret)
                goto err_free_pgd;
 
+       if (get_pgtable_level() == PAGE_MODE_5_LEVEL)
+               ias = 57;
+
        pgtable->iop.ops.map_pages    = iommu_v2_map_pages;
        pgtable->iop.ops.unmap_pages  = iommu_v2_unmap_pages;
        pgtable->iop.ops.iova_to_phys = iommu_v2_iova_to_phys;
 
        cfg->pgsize_bitmap = AMD_IOMMU_PGSIZES_V2,
-       cfg->ias           = IOMMU_IN_ADDR_BIT_SIZE,
+       cfg->ias           = ias,
        cfg->oas           = IOMMU_OUT_ADDR_BIT_SIZE,
        cfg->tlb           = &v2_flush_ops;
 

diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 5452d0dd65691e08d52ba54108ba862c59cd70c8..e290f2034252906883627d32b0ebbf832e880d8b 100644 (file)
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -1611,6 +1611,11 @@ static void set_dte_entry(struct amd_iommu *iommu, u16 devid,
                tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
                flags    |= tmp;
 
+               if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL) {
+                       dev_table[devid].data[2] |=
+                               ((u64)GUEST_PGTABLE_5_LEVEL << DTE_GPT_LEVEL_SHIFT);
+               }
+
                if (domain->flags & PD_GIOV_MASK)
                        pte_root |= DTE_FLAG_GIOV;
        }