ia64/xen-unstable

changeset 19673:f3bed18decfc

[VTD] laying the ground work for ATS

These changes lay the ground work for ATS enabling in Xen. It will be
followed by patch which enables PCI MMCFG which is needed for actual
enabling of ATS functionality.

Signed-off-by: Allen Kay <allen.m.kay@intel.com>
author Keir Fraser <keir.fraser@citrix.com>
date Fri May 29 09:19:30 2009 +0100 (2009-05-29)
parents fe84a14aacd1
children 527b628b8e83
files xen/drivers/passthrough/vtd/dmar.c xen/drivers/passthrough/vtd/extern.h xen/drivers/passthrough/vtd/ia64/Makefile xen/drivers/passthrough/vtd/ia64/ats.c xen/drivers/passthrough/vtd/iommu.c xen/drivers/passthrough/vtd/iommu.h xen/drivers/passthrough/vtd/qinval.c xen/drivers/passthrough/vtd/x86/Makefile xen/drivers/passthrough/vtd/x86/ats.c xen/include/xen/pci_regs.h
line diff
     1.1 --- a/xen/drivers/passthrough/vtd/dmar.c	Thu May 28 11:07:19 2009 +0100
     1.2 +++ b/xen/drivers/passthrough/vtd/dmar.c	Fri May 29 09:19:30 2009 +0100
     1.3 @@ -156,7 +156,7 @@ struct acpi_drhd_unit * acpi_find_matche
     1.4  {
     1.5      u8 bus, devfn;
     1.6      struct acpi_drhd_unit *drhd;
     1.7 -    struct acpi_drhd_unit *found = NULL, *include_all = NULL;
     1.8 +    struct acpi_drhd_unit *include_all = NULL;
     1.9      int i;
    1.10  
    1.11      if (pdev->info.is_extfn) {
    1.12 @@ -177,35 +177,28 @@ struct acpi_drhd_unit * acpi_find_matche
    1.13                  return drhd;
    1.14  
    1.15          if ( test_bit(bus, drhd->scope.buses) )
    1.16 -            found = drhd;
    1.17 +            return drhd;
    1.18  
    1.19          if ( drhd->include_all )
    1.20              include_all = drhd;
    1.21      }
    1.22 -
    1.23 -    return found ? found : include_all;
    1.24 +    return include_all;
    1.25  }
    1.26  
    1.27  struct acpi_atsr_unit * acpi_find_matched_atsr_unit(u8 bus, u8 devfn)
    1.28  {
    1.29      struct acpi_atsr_unit *atsr;
    1.30 -    struct acpi_atsr_unit *found = NULL, *include_all = NULL;
    1.31 -    int i;
    1.32 +    struct acpi_atsr_unit *all_ports = NULL;
    1.33  
    1.34      list_for_each_entry ( atsr, &acpi_atsr_units, list )
    1.35      {
    1.36 -        for (i = 0; i < atsr->scope.devices_cnt; i++)
    1.37 -            if ( atsr->scope.devices[i] == PCI_BDF2(bus, devfn) )
    1.38 -                return atsr;
    1.39 -
    1.40          if ( test_bit(bus, atsr->scope.buses) )
    1.41 -            found = atsr;
    1.42 +            return atsr;
    1.43  
    1.44          if ( atsr->all_ports )
    1.45 -            include_all = atsr;
    1.46 +            all_ports = atsr;
    1.47      }
    1.48 -
    1.49 -    return found ? found : include_all;
    1.50 +    return all_ports;
    1.51  }
    1.52  
    1.53  /*
    1.54 @@ -227,7 +220,8 @@ static int scope_device_count(void *star
    1.55              return -EINVAL;
    1.56          }
    1.57  
    1.58 -        if ( scope->dev_type == ACPI_DEV_ENDPOINT ||
    1.59 +        if ( scope->dev_type == ACPI_DEV_P2PBRIDGE ||
    1.60 +             scope->dev_type == ACPI_DEV_ENDPOINT ||
    1.61               scope->dev_type == ACPI_DEV_IOAPIC ||
    1.62               scope->dev_type == ACPI_DEV_MSI_HPET )
    1.63              count++;
    1.64 @@ -286,19 +280,18 @@ static int __init acpi_parse_dev_scope(v
    1.65                      "found bridge: bdf = %x:%x.%x  sec = %x  sub = %x\n",
    1.66                      bus, path->dev, path->fn, sec_bus, sub_bus);
    1.67  
    1.68 +            dmar_scope_add_buses(scope, acpi_scope->start_bus, acpi_scope->start_bus);
    1.69              dmar_scope_add_buses(scope, sec_bus, sub_bus);
    1.70              break;
    1.71  
    1.72          case ACPI_DEV_MSI_HPET:
    1.73              dprintk(XENLOG_INFO VTDPREFIX, "found MSI HPET: bdf = %x:%x.%x\n",
    1.74                      bus, path->dev, path->fn);
    1.75 -            scope->devices[didx++] = PCI_BDF(bus, path->dev, path->fn);
    1.76              break;
    1.77  
    1.78          case ACPI_DEV_ENDPOINT:
    1.79              dprintk(XENLOG_INFO VTDPREFIX, "found endpoint: bdf = %x:%x.%x\n",
    1.80                      bus, path->dev, path->fn);
    1.81 -            scope->devices[didx++] = PCI_BDF(bus, path->dev, path->fn);
    1.82              break;
    1.83  
    1.84          case ACPI_DEV_IOAPIC:
    1.85 @@ -318,10 +311,9 @@ static int __init acpi_parse_dev_scope(v
    1.86                  list_add(&acpi_ioapic_unit->list, &drhd->ioapic_list);
    1.87              }
    1.88  
    1.89 -            scope->devices[didx++] = PCI_BDF(bus, path->dev, path->fn);
    1.90              break;
    1.91          }
    1.92 -
    1.93 +        scope->devices[didx++] = PCI_BDF(bus, path->dev, path->fn);
    1.94          start += acpi_scope->length;
    1.95     }
    1.96  
     2.1 --- a/xen/drivers/passthrough/vtd/extern.h	Thu May 28 11:07:19 2009 +0100
     2.2 +++ b/xen/drivers/passthrough/vtd/extern.h	Fri May 29 09:19:30 2009 +0100
     2.3 @@ -23,6 +23,8 @@
     2.4  
     2.5  #include "dmar.h"
     2.6  
     2.7 +extern int qinval_enabled;
     2.8 +extern int ats_enabled;
     2.9  extern struct qi_ctrl *qi_ctrl;
    2.10  extern struct ir_ctrl *ir_ctrl;
    2.11  
    2.12 @@ -46,5 +48,8 @@ int iommu_flush_iec_index(struct iommu *
    2.13  struct iommu * ioapic_to_iommu(unsigned int apic_id);
    2.14  struct acpi_drhd_unit * ioapic_to_drhd(unsigned int apic_id);
    2.15  void clear_fault_bits(struct iommu *iommu);
    2.16 +int qinval_device_iotlb(struct iommu *iommu,
    2.17 +                        u32 max_invs_pend, u16 sid, u16 size, u64 addr);
    2.18 +struct acpi_drhd_unit * find_ats_dev_drhd(struct iommu *iommu);
    2.19  
    2.20  #endif // _VTD_EXTERN_H_
     3.1 --- a/xen/drivers/passthrough/vtd/ia64/Makefile	Thu May 28 11:07:19 2009 +0100
     3.2 +++ b/xen/drivers/passthrough/vtd/ia64/Makefile	Fri May 29 09:19:30 2009 +0100
     3.3 @@ -1,1 +1,2 @@
     3.4  obj-y += vtd.o
     3.5 +obj-y += ats.o
     4.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     4.2 +++ b/xen/drivers/passthrough/vtd/ia64/ats.c	Fri May 29 09:19:30 2009 +0100
     4.3 @@ -0,0 +1,66 @@
     4.4 +/*
     4.5 + * Copyright (c) 2006, Intel Corporation.
     4.6 + *
     4.7 + * This program is free software; you can redistribute it and/or modify it
     4.8 + * under the terms and conditions of the GNU General Public License,
     4.9 + * version 2, as published by the Free Software Foundation.
    4.10 + *
    4.11 + * This program is distributed in the hope it will be useful, but WITHOUT
    4.12 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    4.13 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
    4.14 + * more details.
    4.15 + *
    4.16 + * You should have received a copy of the GNU General Public License along with
    4.17 + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
    4.18 + * Place - Suite 330, Boston, MA 02111-1307 USA.
    4.19 + *
    4.20 + * Author: Allen Kay <allen.m.kay@intel.com>
    4.21 + */
    4.22 +
    4.23 +#include <xen/sched.h>
    4.24 +#include <xen/iommu.h>
    4.25 +#include <xen/time.h>
    4.26 +#include <xen/pci.h>
    4.27 +#include <xen/pci_regs.h>
    4.28 +#include <asm/msi.h>
    4.29 +#include "../iommu.h"
    4.30 +#include "../dmar.h"
    4.31 +#include "../vtd.h"
    4.32 +#include "../extern.h"
    4.33 +
    4.34 +int ats_enabled = 0;
    4.35 +
    4.36 +struct acpi_drhd_unit * find_ats_dev_drhd(struct iommu *iommu)
    4.37 +{
    4.38 +    return NULL;
    4.39 +}
    4.40 +
    4.41 +/*
    4.42 + * BUGBUG: return 0 until pcimmcfg is checked in.
    4.43 + */
    4.44 +int pci_find_ext_capability(int seg, int bus, int devfn, int cap)
    4.45 +{
    4.46 +    return 0;
    4.47 +}
    4.48 +
    4.49 +int ats_device(int seg, int bus, int devfn)
    4.50 +{
    4.51 +    return 0;
    4.52 +}
    4.53 +
    4.54 +int enable_ats_device(int seg, int bus, int devfn)
    4.55 +{
    4.56 +    return 0;
    4.57 +}        
    4.58 +
    4.59 +static int device_in_domain(struct iommu *iommu,
    4.60 +                            struct pci_ats_dev *pdev, u16 did)
    4.61 +{
    4.62 +    return 0;
    4.63 +}
    4.64 +
    4.65 +int dev_invalidate_iotlb(struct iommu *iommu, u16 did,
    4.66 +    u64 addr, unsigned int size_order, u64 type)
    4.67 +{
    4.68 +    return 0;
    4.69 +}
     5.1 --- a/xen/drivers/passthrough/vtd/iommu.c	Thu May 28 11:07:19 2009 +0100
     5.2 +++ b/xen/drivers/passthrough/vtd/iommu.c	Fri May 29 09:19:30 2009 +0100
     5.3 @@ -258,7 +258,7 @@ static void iommu_flush_write_buffer(str
     5.4  static int flush_context_reg(
     5.5      void *_iommu,
     5.6      u16 did, u16 source_id, u8 function_mask, u64 type,
     5.7 -    int non_present_entry_flush)
     5.8 +    int flush_non_present_entry)
     5.9  {
    5.10      struct iommu *iommu = (struct iommu *) _iommu;
    5.11      u64 val = 0;
    5.12 @@ -271,7 +271,7 @@ static int flush_context_reg(
    5.13       * entry, we flush entries of domain 0 (the domain id is used to cache
    5.14       * any non-present entries)
    5.15       */
    5.16 -    if ( non_present_entry_flush )
    5.17 +    if ( flush_non_present_entry )
    5.18      {
    5.19          if ( !cap_caching_mode(iommu->cap) )
    5.20              return 1;
    5.21 @@ -318,35 +318,35 @@ static int flush_context_reg(
    5.22  }
    5.23  
    5.24  static int inline iommu_flush_context_global(
    5.25 -    struct iommu *iommu, int non_present_entry_flush)
    5.26 +    struct iommu *iommu, int flush_non_present_entry)
    5.27  {
    5.28      struct iommu_flush *flush = iommu_get_flush(iommu);
    5.29      return flush->context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
    5.30 -                                 non_present_entry_flush);
    5.31 +                                 flush_non_present_entry);
    5.32  }
    5.33  
    5.34  static int inline iommu_flush_context_domain(
    5.35 -    struct iommu *iommu, u16 did, int non_present_entry_flush)
    5.36 +    struct iommu *iommu, u16 did, int flush_non_present_entry)
    5.37  {
    5.38      struct iommu_flush *flush = iommu_get_flush(iommu);
    5.39      return flush->context(iommu, did, 0, 0, DMA_CCMD_DOMAIN_INVL,
    5.40 -                                 non_present_entry_flush);
    5.41 +                                 flush_non_present_entry);
    5.42  }
    5.43  
    5.44  static int inline iommu_flush_context_device(
    5.45      struct iommu *iommu, u16 did, u16 source_id,
    5.46 -    u8 function_mask, int non_present_entry_flush)
    5.47 +    u8 function_mask, int flush_non_present_entry)
    5.48  {
    5.49      struct iommu_flush *flush = iommu_get_flush(iommu);
    5.50      return flush->context(iommu, did, source_id, function_mask,
    5.51                                   DMA_CCMD_DEVICE_INVL,
    5.52 -                                 non_present_entry_flush);
    5.53 +                                 flush_non_present_entry);
    5.54  }
    5.55  
    5.56  /* return value determine if we need a write buffer flush */
    5.57  static int flush_iotlb_reg(void *_iommu, u16 did,
    5.58 -                               u64 addr, unsigned int size_order, u64 type,
    5.59 -                               int non_present_entry_flush)
    5.60 +                           u64 addr, unsigned int size_order, u64 type,
    5.61 +                           int flush_non_present_entry, int flush_dev_iotlb)
    5.62  {
    5.63      struct iommu *iommu = (struct iommu *) _iommu;
    5.64      int tlb_offset = ecap_iotlb_offset(iommu->ecap);
    5.65 @@ -360,7 +360,7 @@ static int flush_iotlb_reg(void *_iommu,
    5.66       * entry, we flush entries of domain 0 (the domain id is used to cache
    5.67       * any non-present entries)
    5.68       */
    5.69 -    if ( non_present_entry_flush )
    5.70 +    if ( flush_non_present_entry )
    5.71      {
    5.72          if ( !cap_caching_mode(iommu->cap) )
    5.73              return 1;
    5.74 @@ -421,19 +421,19 @@ static int flush_iotlb_reg(void *_iommu,
    5.75  }
    5.76  
    5.77  static int inline iommu_flush_iotlb_global(struct iommu *iommu,
    5.78 -                                           int non_present_entry_flush)
    5.79 +    int flush_non_present_entry, int flush_dev_iotlb)
    5.80  {
    5.81      struct iommu_flush *flush = iommu_get_flush(iommu);
    5.82      return flush->iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
    5.83 -                               non_present_entry_flush);
    5.84 +                        flush_non_present_entry, flush_dev_iotlb);
    5.85  }
    5.86  
    5.87  static int inline iommu_flush_iotlb_dsi(struct iommu *iommu, u16 did,
    5.88 -                                        int non_present_entry_flush)
    5.89 +    int flush_non_present_entry, int flush_dev_iotlb)
    5.90  {
    5.91      struct iommu_flush *flush = iommu_get_flush(iommu);
    5.92      return flush->iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH,
    5.93 -                               non_present_entry_flush);
    5.94 +                        flush_non_present_entry, flush_dev_iotlb);
    5.95  }
    5.96  
    5.97  static int inline get_alignment(u64 base, unsigned int size)
    5.98 @@ -452,8 +452,8 @@ static int inline get_alignment(u64 base
    5.99  }
   5.100  
   5.101  static int inline iommu_flush_iotlb_psi(
   5.102 -    struct iommu *iommu, u16 did,
   5.103 -    u64 addr, unsigned int pages, int non_present_entry_flush)
   5.104 +    struct iommu *iommu, u16 did, u64 addr, unsigned int pages,
   5.105 +    int flush_non_present_entry, int flush_dev_iotlb)
   5.106  {
   5.107      unsigned int align;
   5.108      struct iommu_flush *flush = iommu_get_flush(iommu);
   5.109 @@ -463,8 +463,7 @@ static int inline iommu_flush_iotlb_psi(
   5.110  
   5.111      /* Fallback to domain selective flush if no PSI support */
   5.112      if ( !cap_pgsel_inv(iommu->cap) )
   5.113 -        return iommu_flush_iotlb_dsi(iommu, did,
   5.114 -                                     non_present_entry_flush);
   5.115 +        return iommu_flush_iotlb_dsi(iommu, did, flush_non_present_entry, flush_dev_iotlb);
   5.116  
   5.117      /*
   5.118       * PSI requires page size is 2 ^ x, and the base address is naturally
   5.119 @@ -473,27 +472,28 @@ static int inline iommu_flush_iotlb_psi(
   5.120      align = get_alignment(addr >> PAGE_SHIFT_4K, pages);
   5.121      /* Fallback to domain selective flush if size is too big */
   5.122      if ( align > cap_max_amask_val(iommu->cap) )
   5.123 -        return iommu_flush_iotlb_dsi(iommu, did,
   5.124 -                                     non_present_entry_flush);
   5.125 +        return iommu_flush_iotlb_dsi(iommu, did, flush_non_present_entry, flush_dev_iotlb);
   5.126  
   5.127      addr >>= PAGE_SHIFT_4K + align;
   5.128      addr <<= PAGE_SHIFT_4K + align;
   5.129  
   5.130 -    return flush->iotlb(iommu, did, addr, align,
   5.131 -                               DMA_TLB_PSI_FLUSH, non_present_entry_flush);
   5.132 +    return flush->iotlb(iommu, did, addr, align, DMA_TLB_PSI_FLUSH,
   5.133 +                        flush_non_present_entry, flush_dev_iotlb);
   5.134  }
   5.135  
   5.136  void iommu_flush_all(void)
   5.137  {
   5.138      struct acpi_drhd_unit *drhd;
   5.139      struct iommu *iommu;
   5.140 +    int flush_dev_iotlb;
   5.141  
   5.142      flush_all_cache();
   5.143      for_each_drhd_unit ( drhd )
   5.144      {
   5.145          iommu = drhd->iommu;
   5.146          iommu_flush_context_global(iommu, 0);
   5.147 -        iommu_flush_iotlb_global(iommu, 0);
   5.148 +        flush_dev_iotlb = find_ats_dev_drhd(iommu) ? 1 : 0;
   5.149 +        iommu_flush_iotlb_global(iommu, 0, flush_dev_iotlb);
   5.150      }
   5.151  }
   5.152  
   5.153 @@ -505,6 +505,7 @@ static void dma_pte_clear_one(struct dom
   5.154      struct iommu *iommu;
   5.155      struct dma_pte *page = NULL, *pte = NULL;
   5.156      u64 pg_maddr;
   5.157 +    int flush_dev_iotlb;
   5.158  
   5.159      spin_lock(&hd->mapping_lock);
   5.160      /* get last level pte */
   5.161 @@ -534,9 +535,12 @@ static void dma_pte_clear_one(struct dom
   5.162      {
   5.163          iommu = drhd->iommu;
   5.164          if ( test_bit(iommu->index, &hd->iommu_bitmap) )
   5.165 +        {
   5.166 +            flush_dev_iotlb = find_ats_dev_drhd(iommu) ? 1 : 0;
   5.167              if ( iommu_flush_iotlb_psi(iommu, domain_iommu_domid(domain),
   5.168 -                                       addr, 1, 0))
   5.169 +                                       addr, 1, 0, flush_dev_iotlb) )
   5.170                  iommu_flush_write_buffer(iommu);
   5.171 +        }
   5.172      }
   5.173  
   5.174      unmap_vtd_domain_page(page);
   5.175 @@ -926,6 +930,10 @@ static int iommu_alloc(struct acpi_drhd_
   5.176      iommu->cap = dmar_readq(iommu->reg, DMAR_CAP_REG);
   5.177      iommu->ecap = dmar_readq(iommu->reg, DMAR_ECAP_REG);
   5.178  
   5.179 +    gdprintk(XENLOG_INFO VTDPREFIX,
   5.180 +             "drhd->address = %"PRIx64"\n", drhd->address);
   5.181 +    gdprintk(XENLOG_INFO VTDPREFIX, "iommu->reg = %p\n", iommu->reg);
   5.182 +
   5.183      /* Calculate number of pagetable levels: between 2 and 4. */
   5.184      sagaw = cap_sagaw(iommu->cap);
   5.185      for ( agaw = level_to_agaw(4); agaw >= 0; agaw-- )
   5.186 @@ -1079,7 +1087,11 @@ static int domain_context_mapping_one(
   5.187          }
   5.188  
   5.189          context_set_address_root(*context, pgd_maddr);
   5.190 -        context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
   5.191 +        if ( ats_enabled && ecap_dev_iotlb(iommu->ecap) )
   5.192 +            context_set_translation_type(*context, CONTEXT_TT_DEV_IOTLB);
   5.193 +        else
   5.194 +            context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
   5.195 +
   5.196          spin_unlock(&hd->mapping_lock);
   5.197      }
   5.198  
   5.199 @@ -1099,7 +1111,10 @@ static int domain_context_mapping_one(
   5.200                                      DMA_CCMD_MASK_NOBIT, 1) )
   5.201          iommu_flush_write_buffer(iommu);
   5.202      else
   5.203 -        iommu_flush_iotlb_dsi(iommu, 0, 1);
   5.204 +    {
   5.205 +        int flush_dev_iotlb = find_ats_dev_drhd(iommu) ? 1 : 0;
   5.206 +        iommu_flush_iotlb_dsi(iommu, 0, 1, flush_dev_iotlb);
   5.207 +    }
   5.208  
   5.209      set_bit(iommu->index, &hd->iommu_bitmap);
   5.210  
   5.211 @@ -1322,7 +1337,10 @@ static int domain_context_unmap_one(
   5.212                                      DMA_CCMD_MASK_NOBIT, 0) )
   5.213          iommu_flush_write_buffer(iommu);
   5.214      else
   5.215 -        iommu_flush_iotlb_dsi(iommu, domain_iommu_domid(domain), 0);
   5.216 +    {
   5.217 +        int flush_dev_iotlb = find_ats_dev_drhd(iommu) ? 1 : 0;
   5.218 +        iommu_flush_iotlb_dsi(iommu, domain_iommu_domid(domain), 0, flush_dev_iotlb);
   5.219 +    }
   5.220  
   5.221      spin_unlock(&iommu->lock);
   5.222      unmap_vtd_domain_page(context_entries);
   5.223 @@ -1463,6 +1481,7 @@ int intel_iommu_map_page(
   5.224      struct dma_pte *page = NULL, *pte = NULL;
   5.225      u64 pg_maddr;
   5.226      int pte_present;
   5.227 +    int flush_dev_iotlb;
   5.228  
   5.229      drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
   5.230      iommu = drhd->iommu;
   5.231 @@ -1504,9 +1523,10 @@ int intel_iommu_map_page(
   5.232          if ( !test_bit(iommu->index, &hd->iommu_bitmap) )
   5.233              continue;
   5.234  
   5.235 +        flush_dev_iotlb = find_ats_dev_drhd(iommu) ? 1 : 0;
   5.236          if ( iommu_flush_iotlb_psi(iommu, domain_iommu_domid(d),
   5.237                                     (paddr_t)gfn << PAGE_SHIFT_4K, 1,
   5.238 -                                   !pte_present) )
   5.239 +                                   !pte_present, flush_dev_iotlb) )
   5.240              iommu_flush_write_buffer(iommu);
   5.241      }
   5.242  
   5.243 @@ -1643,6 +1663,10 @@ static void setup_dom0_devices(struct do
   5.244                  pdev->domain = d;
   5.245                  list_add(&pdev->domain_list, &d->arch.pdev_list);
   5.246                  domain_context_mapping(d, pdev->bus, pdev->devfn);
   5.247 +#if defined(NOT_YET)
   5.248 +                if ( ats_device(0, pdev->bus, pdev->devfn) )
   5.249 +                    enable_ats_device(0, pdev->bus, pdev->devfn);
   5.250 +#endif
   5.251              }
   5.252          }
   5.253      }
     6.1 --- a/xen/drivers/passthrough/vtd/iommu.h	Thu May 28 11:07:19 2009 +0100
     6.2 +++ b/xen/drivers/passthrough/vtd/iommu.h	Fri May 29 09:19:30 2009 +0100
     6.3 @@ -455,7 +455,7 @@ struct iommu_flush {
     6.4      int (*context)(void *iommu, u16 did, u16 source_id,
     6.5                     u8 function_mask, u64 type, int non_present_entry_flush);
     6.6      int (*iotlb)(void *iommu, u16 did, u64 addr, unsigned int size_order,
     6.7 -                 u64 type, int non_present_entry_flush);
     6.8 +                 u64 type, int flush_non_present_entry, int flush_dev_iotlb);
     6.9  };
    6.10  
    6.11  struct intel_iommu {
     7.1 --- a/xen/drivers/passthrough/vtd/qinval.c	Thu May 28 11:07:19 2009 +0100
     7.2 +++ b/xen/drivers/passthrough/vtd/qinval.c	Fri May 29 09:19:30 2009 +0100
     7.3 @@ -29,6 +29,8 @@
     7.4  #include "vtd.h"
     7.5  #include "extern.h"
     7.6  
     7.7 +int qinval_enabled;
     7.8 +
     7.9  static void print_qi_regs(struct iommu *iommu)
    7.10  {
    7.11      u64 val;
    7.12 @@ -343,7 +345,7 @@ int iommu_flush_iec_index(struct iommu *
    7.13  
    7.14  static int flush_context_qi(
    7.15      void *_iommu, u16 did, u16 sid, u8 fm, u64 type,
    7.16 -    int non_present_entry_flush)
    7.17 +    int flush_non_present_entry)
    7.18  {
    7.19      int ret = 0;
    7.20      struct iommu *iommu = (struct iommu *)_iommu;
    7.21 @@ -355,7 +357,7 @@ static int flush_context_qi(
    7.22       * entry, we flush entries of domain 0 (the domain id is used to cache
    7.23       * any non-present entries)
    7.24       */
    7.25 -    if ( non_present_entry_flush )
    7.26 +    if ( flush_non_present_entry )
    7.27      {
    7.28          if ( !cap_caching_mode(iommu->cap) )
    7.29              return 1;
    7.30 @@ -375,7 +377,7 @@ static int flush_context_qi(
    7.31  static int flush_iotlb_qi(
    7.32      void *_iommu, u16 did,
    7.33      u64 addr, unsigned int size_order, u64 type,
    7.34 -    int non_present_entry_flush)
    7.35 +    int flush_non_present_entry, int flush_dev_iotlb)
    7.36  {
    7.37      u8 dr = 0, dw = 0;
    7.38      int ret = 0;
    7.39 @@ -388,7 +390,7 @@ static int flush_iotlb_qi(
    7.40       * entry, we flush entries of domain 0 (the domain id is used to cache
    7.41       * any non-present entries)
    7.42       */
    7.43 -    if ( non_present_entry_flush )
    7.44 +    if ( flush_non_present_entry )
    7.45      {
    7.46          if ( !cap_caching_mode(iommu->cap) )
    7.47              return 1;
    7.48 @@ -407,6 +409,10 @@ static int flush_iotlb_qi(
    7.49          ret = queue_invalidate_iotlb(iommu,
    7.50                    (type >> DMA_TLB_FLUSH_GRANU_OFFSET), dr,
    7.51                    dw, did, (u8)size_order, 0, addr);
    7.52 +#if defined(NOT_YET)
    7.53 +        if ( flush_dev_iotlb )
    7.54 +            ret |= dev_invalidate_iotlb(iommu, did, addr, size_order, type);
    7.55 +#endif
    7.56          ret |= invalidate_sync(iommu);
    7.57      }
    7.58      return ret;
    7.59 @@ -462,6 +468,7 @@ int enable_qinval(struct iommu *iommu)
    7.60          cpu_relax();
    7.61      }
    7.62  
    7.63 +    qinval_enabled = 1;
    7.64      return 0;
    7.65  }
    7.66  
     8.1 --- a/xen/drivers/passthrough/vtd/x86/Makefile	Thu May 28 11:07:19 2009 +0100
     8.2 +++ b/xen/drivers/passthrough/vtd/x86/Makefile	Fri May 29 09:19:30 2009 +0100
     8.3 @@ -1,1 +1,2 @@
     8.4  obj-y += vtd.o
     8.5 +obj-y += ats.o
     9.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     9.2 +++ b/xen/drivers/passthrough/vtd/x86/ats.c	Fri May 29 09:19:30 2009 +0100
     9.3 @@ -0,0 +1,254 @@
     9.4 +/*
     9.5 + * Copyright (c) 2006, Intel Corporation.
     9.6 + *
     9.7 + * This program is free software; you can redistribute it and/or modify it
     9.8 + * under the terms and conditions of the GNU General Public License,
     9.9 + * version 2, as published by the Free Software Foundation.
    9.10 + *
    9.11 + * This program is distributed in the hope it will be useful, but WITHOUT
    9.12 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    9.13 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
    9.14 + * more details.
    9.15 + *
    9.16 + * You should have received a copy of the GNU General Public License along with
    9.17 + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
    9.18 + * Place - Suite 330, Boston, MA 02111-1307 USA.
    9.19 + *
    9.20 + * Author: Allen Kay <allen.m.kay@intel.com>
    9.21 + */
    9.22 +
    9.23 +#include <xen/sched.h>
    9.24 +#include <xen/iommu.h>
    9.25 +#include <xen/time.h>
    9.26 +#include <xen/pci.h>
    9.27 +#include <xen/pci_regs.h>
    9.28 +#include <asm/msi.h>
    9.29 +#include "../iommu.h"
    9.30 +#include "../dmar.h"
    9.31 +#include "../vtd.h"
    9.32 +#include "../extern.h"
    9.33 +
    9.34 +LIST_HEAD(ats_dev_drhd_units);
    9.35 +
    9.36 +#define ATS_REG_CAP    4
    9.37 +#define ATS_REG_CTL    6
    9.38 +#define ATS_QUEUE_DEPTH_MASK     0xF
    9.39 +#define ATS_ENABLE               (1<<15)
    9.40 +
    9.41 +struct pci_ats_dev {
    9.42 +    struct list_head list;
    9.43 +    u8 bus;
    9.44 +    u8 devfn;
    9.45 +    u16 ats_queue_depth;    /* ATS device invalidation queue depth */
    9.46 +    spinlock_t lock;
    9.47 +};
    9.48 +static LIST_HEAD(ats_devices);
    9.49 +
    9.50 +static void parse_ats_param(char *s);
    9.51 +custom_param("ats", parse_ats_param);
    9.52 +
    9.53 +int ats_enabled = 1;
    9.54 +
    9.55 +static void parse_ats_param(char *s)
    9.56 +{
    9.57 +    char *ss;
    9.58 +
    9.59 +    do {
    9.60 +        ss = strchr(s, ',');
    9.61 +        if ( ss )
    9.62 +            *ss = '\0';
    9.63 +
    9.64 +        if ( !strcmp(s, "off") || !strcmp(s, "no") || !strcmp(s, "false") ||
    9.65 +             !strcmp(s, "0") || !strcmp(s, "disable") )
    9.66 +            ats_enabled = 0;
    9.67 +
    9.68 +        if ( !strcmp(s, "on") || !strcmp(s, "yes") || !strcmp(s, "true") ||
    9.69 +             !strcmp(s, "1") || !strcmp(s, "enable") )
    9.70 +            ats_enabled = 1;
    9.71 +
    9.72 +        s = ss + 1;
    9.73 +    } while ( ss );
    9.74 +}
    9.75 +
    9.76 +struct acpi_drhd_unit * find_ats_dev_drhd(struct iommu *iommu)
    9.77 +{
    9.78 +    struct acpi_drhd_unit *drhd;
    9.79 +    list_for_each_entry ( drhd, &ats_dev_drhd_units, list )
    9.80 +    {
    9.81 +        if ( drhd->iommu == iommu )
    9.82 +            return drhd;
    9.83 +    }
    9.84 +    return NULL;
    9.85 +}
    9.86 +
    9.87 +/*
    9.88 + * BUGBUG: return 0 until pcimmcfg is checked in.
    9.89 + */
    9.90 +int pci_find_ext_capability(int seg, int bus, int devfn, int cap)
    9.91 +{
    9.92 +    return 0;
    9.93 +}
    9.94 +
    9.95 +int ats_device(int seg, int bus, int devfn)
    9.96 +{
    9.97 +    struct acpi_drhd_unit *drhd, *ats_drhd, *new_drhd;
    9.98 +    struct pci_dev *pdev;
    9.99 +    int pos = 0;
   9.100 +
   9.101 +    if ( !ats_enabled )
   9.102 +        return 0;
   9.103 +
   9.104 +    if ( !qinval_enabled )
   9.105 +        return 0;
   9.106 +
   9.107 +    pdev = pci_get_pdev(bus, devfn);
   9.108 +    drhd = acpi_find_matched_drhd_unit(pdev);
   9.109 +    if ( !ecap_dev_iotlb(drhd->iommu->ecap) )
   9.110 +        return 0;
   9.111 +
   9.112 +    if ( !acpi_find_matched_atsr_unit(bus, devfn) )
   9.113 +        return 0;
   9.114 +
   9.115 +    ats_drhd = find_ats_dev_drhd(drhd->iommu);
   9.116 +    pos = pci_find_ext_capability(seg, bus, devfn, PCI_EXT_CAP_ID_ATS);
   9.117 +
   9.118 +    if ( pos && (ats_drhd == NULL) )
   9.119 +    {
   9.120 +        new_drhd = xmalloc(struct acpi_drhd_unit);
   9.121 +        memcpy(new_drhd, drhd, sizeof(struct acpi_drhd_unit));
   9.122 +        list_add_tail(&new_drhd->list, &ats_dev_drhd_units);
   9.123 +    }
   9.124 +    return pos;
   9.125 +}
   9.126 +
   9.127 +int enable_ats_device(int seg, int bus, int devfn)
   9.128 +{
   9.129 +    struct pci_ats_dev *pdev;
   9.130 +    u32 value;
   9.131 +    u16 queue_depth;
   9.132 +    int pos;
   9.133 +
   9.134 +    pos = pci_find_ext_capability(seg, bus, devfn, PCI_EXT_CAP_ID_ATS);
   9.135 +
   9.136 +    if ( !pos )
   9.137 +    {
   9.138 +        dprintk(XENLOG_ERR VTDPREFIX, "ats capability not found %x:%x:%x\n",
   9.139 +                bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
   9.140 +        return 0;
   9.141 +    }
   9.142 +    else
   9.143 +        dprintk(XENLOG_ERR VTDPREFIX, "ats capability found %x:%x:%x\n",
   9.144 +                bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
   9.145 +
   9.146 +    /* BUGBUG: add back seg when multi-seg platform support is enabled */
   9.147 +    value = pci_conf_read16(bus, PCI_SLOT(devfn),
   9.148 +                            PCI_FUNC(devfn), pos + ATS_REG_CAP);
   9.149 +    queue_depth = value & ATS_QUEUE_DEPTH_MASK;
   9.150 +
   9.151 +    /* BUGBUG: add back seg when multi-seg platform support is enabled */
   9.152 +    value = pci_conf_read16(bus, PCI_SLOT(devfn), PCI_FUNC(devfn), pos + ATS_REG_CTL);
   9.153 +    value |= ATS_ENABLE;
   9.154 +
   9.155 +    /* BUGBUG: add back seg when multi-seg platform support is enabled */
   9.156 +    pci_conf_write16(bus, PCI_SLOT(devfn), PCI_FUNC(devfn), pos + ATS_REG_CTL, value);
   9.157 +
   9.158 +    if ( acpi_find_matched_atsr_unit(bus, devfn) )
   9.159 +    {
   9.160 +        pdev = xmalloc(struct pci_ats_dev);
   9.161 +        pdev->bus = bus;
   9.162 +        pdev->devfn = devfn;
   9.163 +        pdev->ats_queue_depth = queue_depth;
   9.164 +        list_add(&(pdev->list), &ats_devices);
   9.165 +    }
   9.166 +    return pos;
   9.167 +}        
   9.168 +
   9.169 +static int device_in_domain(struct iommu *iommu, struct pci_ats_dev *pdev, u16 did)
   9.170 +{
   9.171 +    struct root_entry *root_entry = NULL;
   9.172 +    struct context_entry *ctxt_entry = NULL;
   9.173 +    int tt, found = 0;
   9.174 +
   9.175 +    root_entry = (struct root_entry *) map_vtd_domain_page(iommu->root_maddr);
   9.176 +    if ( !root_entry || !root_present(root_entry[pdev->bus]) )
   9.177 +        goto out;
   9.178 +
   9.179 +    ctxt_entry = (struct context_entry *)
   9.180 +                 map_vtd_domain_page(root_entry[pdev->bus].val);
   9.181 +
   9.182 +    if ( ctxt_entry == NULL )
   9.183 +        goto out;
   9.184 +
   9.185 +    if ( context_domain_id(ctxt_entry[pdev->devfn]) != did )
   9.186 +        goto out;
   9.187 +
   9.188 +    tt = context_translation_type(ctxt_entry[pdev->devfn]);
   9.189 +    if ( tt != CONTEXT_TT_DEV_IOTLB )
   9.190 +        goto out;
   9.191 +
   9.192 +    found = 1;
   9.193 +out:
   9.194 +    if ( root_entry )
   9.195 +        unmap_vtd_domain_page(root_entry);
   9.196 +
   9.197 +    if ( ctxt_entry )
   9.198 +        unmap_vtd_domain_page(ctxt_entry);
   9.199 +
   9.200 +    if ( found )
   9.201 +        return 1;
   9.202 +
   9.203 +    return 0;
   9.204 +}
   9.205 +
   9.206 +int dev_invalidate_iotlb(struct iommu *iommu, u16 did,
   9.207 +    u64 addr, unsigned int size_order, u64 type)
   9.208 +{
   9.209 +    struct pci_ats_dev *pdev;
   9.210 +    int sbit, ret = 0;
   9.211 +    u16 sid;
   9.212 +
   9.213 +    if ( !ecap_dev_iotlb(iommu->ecap) )
   9.214 +        return ret;
   9.215 +
   9.216 +    list_for_each_entry( pdev, &ats_devices, list )
   9.217 +    {
   9.218 +        sid = (pdev->bus << 8) | pdev->devfn;
   9.219 +
   9.220 +        switch ( type ) {
   9.221 +        case DMA_TLB_DSI_FLUSH:
   9.222 +            if ( !device_in_domain(iommu, pdev, did) )
   9.223 +                break;
   9.224 +            /* fall through if DSI condition met */
   9.225 +        case DMA_TLB_GLOBAL_FLUSH:
   9.226 +            /* invalidate all translations: sbit=1,bit_63=0,bit[62:12]=1 */
   9.227 +            sbit = 1;
   9.228 +            addr = (~0 << PAGE_SHIFT_4K) & 0x7FFFFFFFFFFFFFFF;
   9.229 +            ret |= qinval_device_iotlb(iommu, pdev->ats_queue_depth,
   9.230 +                                       sid, sbit, addr);
   9.231 +            break;
   9.232 +        case DMA_TLB_PSI_FLUSH:
   9.233 +            if ( !device_in_domain(iommu, pdev, did) )
   9.234 +                break;
   9.235 +
   9.236 +            addr &= ~0 << (PAGE_SHIFT + size_order);
   9.237 +
   9.238 +            /* if size <= 4K, set sbit = 0, else set sbit = 1 */
   9.239 +            sbit = size_order ? 1 : 0; 
   9.240 + 
   9.241 +            /* clear lower bits */
   9.242 +            addr &= (~0 << (PAGE_SHIFT + size_order));
   9.243 +
   9.244 +            /* if sbit == 1, zero out size_order bit and set lower bits to 1 */
   9.245 +            if ( sbit )
   9.246 +                addr &= (~0  & ~(1 << (PAGE_SHIFT + size_order)));
   9.247 +
   9.248 +            ret |= qinval_device_iotlb(iommu, pdev->ats_queue_depth,
   9.249 +                                       sid, sbit, addr);
   9.250 +            break;
   9.251 +        default:
   9.252 +            dprintk(XENLOG_WARNING VTDPREFIX, "invalid vt-d flush type\n");
   9.253 +            break; 
   9.254 +        }
   9.255 +    }
   9.256 +    return ret;
   9.257 +}
    10.1 --- a/xen/include/xen/pci_regs.h	Thu May 28 11:07:19 2009 +0100
    10.2 +++ b/xen/include/xen/pci_regs.h	Fri May 29 09:19:30 2009 +0100
    10.3 @@ -419,6 +419,9 @@
    10.4  #define PCI_EXT_CAP_ID_VC	2
    10.5  #define PCI_EXT_CAP_ID_DSN	3
    10.6  #define PCI_EXT_CAP_ID_PWR	4
    10.7 +#define PCI_EXT_CAP_ID_ARI	0xE
    10.8 +#define PCI_EXT_CAP_ID_ATS	0xF
    10.9 +#define PCI_EXT_CAP_ID_IOV	0x10
   10.10  
   10.11  /* Advanced Error Reporting */
   10.12  #define PCI_ERR_UNCOR_STATUS	4	/* Uncorrectable Error Status */