ia64/xen-unstable

changeset 16066:48d42d659a04

Fix TLB flush on grant unmap.
Signed-off-by: Kieran Mansley <kmansley@solarflare.com>
author Keir Fraser <keir@xensource.com>
date Thu Oct 04 10:25:35 2007 +0100 (2007-10-04)
parents 671dcd08e806
children 9f9f9b68cd08
files xen/common/grant_table.c xen/include/xen/iocap.h
line diff
     1.1 --- a/xen/common/grant_table.c	Thu Oct 04 10:03:04 2007 +0100
     1.2 +++ b/xen/common/grant_table.c	Thu Oct 04 10:25:35 2007 +0100
     1.3 @@ -59,14 +59,26 @@ union grant_combo {
     1.4  
     1.5  /* Used to share code between unmap_grant_ref and unmap_and_replace. */
     1.6  struct gnttab_unmap_common {
     1.7 +    /* Input */
     1.8      uint64_t host_addr;
     1.9      uint64_t dev_bus_addr;
    1.10      uint64_t new_addr;
    1.11      grant_handle_t handle;
    1.12  
    1.13 +    /* Return */
    1.14      int16_t status;
    1.15 +
    1.16 +    /* Shared state beteen *_unmap and *_unmap_complete */
    1.17 +    u16 flags;
    1.18 +    unsigned long frame;
    1.19 +    struct grant_mapping *map;
    1.20 +    struct domain *rd;
    1.21  };
    1.22  
    1.23 +/* Number of unmap operations that are done between each tlb flush */
    1.24 +#define GNTTAB_UNMAP_BATCH_SIZE 32
    1.25 +
    1.26 +
    1.27  #define PIN_FAIL(_lbl, _rc, _f, _a...)          \
    1.28      do {                                        \
    1.29          gdprintk(XENLOG_WARNING, _f, ## _a );   \
    1.30 @@ -412,18 +424,14 @@ static void
    1.31      struct gnttab_unmap_common *op)
    1.32  {
    1.33      domid_t          dom;
    1.34 -    grant_ref_t      ref;
    1.35      struct domain   *ld, *rd;
    1.36      struct active_grant_entry *act;
    1.37      grant_entry_t   *sha;
    1.38 -    struct grant_mapping *map;
    1.39 -    u16              flags;
    1.40      s16              rc = 0;
    1.41 -    unsigned long    frame;
    1.42  
    1.43      ld = current->domain;
    1.44  
    1.45 -    frame = (unsigned long)(op->dev_bus_addr >> PAGE_SHIFT);
    1.46 +    op->frame = (unsigned long)(op->dev_bus_addr >> PAGE_SHIFT);
    1.47  
    1.48      if ( unlikely(op->handle >= ld->grant_table->maptrack_limit) )
    1.49      {
    1.50 @@ -432,20 +440,19 @@ static void
    1.51          return;
    1.52      }
    1.53  
    1.54 -    map = &maptrack_entry(ld->grant_table, op->handle);
    1.55 +    op->map = &maptrack_entry(ld->grant_table, op->handle);
    1.56  
    1.57 -    if ( unlikely(!map->flags) )
    1.58 +    if ( unlikely(!op->map->flags) )
    1.59      {
    1.60          gdprintk(XENLOG_INFO, "Zero flags for handle (%d).\n", op->handle);
    1.61          op->status = GNTST_bad_handle;
    1.62          return;
    1.63      }
    1.64  
    1.65 -    dom   = map->domid;
    1.66 -    ref   = map->ref;
    1.67 -    flags = map->flags;
    1.68 +    dom   = op->map->domid;
    1.69 +    op->flags = op->map->flags;
    1.70  
    1.71 -    if ( unlikely((rd = rcu_lock_domain_by_id(dom)) == NULL) )
    1.72 +    if ( unlikely((op->rd = rd = rcu_lock_domain_by_id(dom)) == NULL) )
    1.73      {
    1.74          /* This can happen when a grant is implicitly unmapped. */
    1.75          gdprintk(XENLOG_INFO, "Could not find domain %d\n", dom);
    1.76 @@ -465,71 +472,47 @@ static void
    1.77  
    1.78      spin_lock(&rd->grant_table->lock);
    1.79  
    1.80 -    act = &active_entry(rd->grant_table, ref);
    1.81 -    sha = &shared_entry(rd->grant_table, ref);
    1.82 +    act = &active_entry(rd->grant_table, op->map->ref);
    1.83 +    sha = &shared_entry(rd->grant_table, op->map->ref);
    1.84  
    1.85 -    if ( frame == 0 )
    1.86 +    if ( op->frame == 0 )
    1.87      {
    1.88 -        frame = act->frame;
    1.89 +        op->frame = act->frame;
    1.90      }
    1.91      else
    1.92      {
    1.93 -        if ( unlikely(frame != act->frame) )
    1.94 +        if ( unlikely(op->frame != act->frame) )
    1.95              PIN_FAIL(unmap_out, GNTST_general_error,
    1.96                       "Bad frame number doesn't match gntref.\n");
    1.97 -        if ( flags & GNTMAP_device_map )
    1.98 +        if ( op->flags & GNTMAP_device_map )
    1.99          {
   1.100              ASSERT(act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask));
   1.101 -            map->flags &= ~GNTMAP_device_map;
   1.102 -            if ( flags & GNTMAP_readonly )
   1.103 -            {
   1.104 +            op->map->flags &= ~GNTMAP_device_map;
   1.105 +            if ( op->flags & GNTMAP_readonly )
   1.106                  act->pin -= GNTPIN_devr_inc;
   1.107 -                put_page(mfn_to_page(frame));
   1.108 -            }
   1.109              else
   1.110 -            {
   1.111                  act->pin -= GNTPIN_devw_inc;
   1.112 -                put_page_and_type(mfn_to_page(frame));
   1.113 -            }
   1.114          }
   1.115      }
   1.116  
   1.117 -    if ( (op->host_addr != 0) && (flags & GNTMAP_host_map) )
   1.118 +    if ( (op->host_addr != 0) && (op->flags & GNTMAP_host_map) )
   1.119      {
   1.120          if ( (rc = replace_grant_host_mapping(op->host_addr,
   1.121 -                                              frame, op->new_addr, flags)) < 0 )
   1.122 +                                              op->frame, op->new_addr, 
   1.123 +                                              op->flags)) < 0 )
   1.124              goto unmap_out;
   1.125  
   1.126          ASSERT(act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask));
   1.127 -        map->flags &= ~GNTMAP_host_map;
   1.128 -        if ( flags & GNTMAP_readonly )
   1.129 -        {
   1.130 +        op->map->flags &= ~GNTMAP_host_map;
   1.131 +        if ( op->flags & GNTMAP_readonly )
   1.132              act->pin -= GNTPIN_hstr_inc;
   1.133 -            put_page(mfn_to_page(frame));
   1.134 -        }
   1.135          else
   1.136 -        {
   1.137              act->pin -= GNTPIN_hstw_inc;
   1.138 -            put_page_and_type(mfn_to_page(frame));
   1.139 -        }
   1.140 -    }
   1.141 -
   1.142 -    if ( (map->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0 )
   1.143 -    {
   1.144 -        map->flags = 0;
   1.145 -        put_maptrack_handle(ld->grant_table, op->handle);
   1.146      }
   1.147  
   1.148      /* If just unmapped a writable mapping, mark as dirtied */
   1.149 -    if ( !(flags & GNTMAP_readonly) )
   1.150 -         gnttab_mark_dirty(rd, frame);
   1.151 -
   1.152 -    if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0) &&
   1.153 -         !(flags & GNTMAP_readonly) )
   1.154 -        gnttab_clear_flag(_GTF_writing, &sha->flags);
   1.155 -
   1.156 -    if ( act->pin == 0 )
   1.157 -        gnttab_clear_flag(_GTF_reading, &sha->flags);
   1.158 +    if ( !(op->flags & GNTMAP_readonly) )
   1.159 +         gnttab_mark_dirty(rd, op->frame);
   1.160  
   1.161   unmap_out:
   1.162      op->status = rc;
   1.163 @@ -538,78 +521,205 @@ static void
   1.164  }
   1.165  
   1.166  static void
   1.167 -__gnttab_unmap_grant_ref(
   1.168 -    struct gnttab_unmap_grant_ref *op)
   1.169 +__gnttab_unmap_common_complete(struct gnttab_unmap_common *op)
   1.170  {
   1.171 -    struct gnttab_unmap_common common = {
   1.172 -        .host_addr = op->host_addr,
   1.173 -        .dev_bus_addr = op->dev_bus_addr,
   1.174 -        .handle = op->handle,
   1.175 -    };
   1.176 +    struct domain   *ld, *rd;
   1.177 +    struct active_grant_entry *act;
   1.178 +    grant_entry_t   *sha;
   1.179 +
   1.180 +    rd = op->rd;
   1.181 +
   1.182 +    if ( rd == NULL ) { 
   1.183 +        /*
   1.184 +         * Suggests that __gntab_unmap_common failed in
   1.185 +         * rcu_lock_domain_by_id() or earlier, and so we have nothing
   1.186 +         * to complete
   1.187 +         */
   1.188 +        return;
   1.189 +    }
   1.190 +
   1.191 +    ld = current->domain;
   1.192 +
   1.193 +    rcu_lock_domain(rd);
   1.194 +    spin_lock(&rd->grant_table->lock);
   1.195 +
   1.196 +    act = &active_entry(rd->grant_table, op->map->ref);
   1.197 +    sha = &shared_entry(rd->grant_table, op->map->ref);
   1.198 +
   1.199 +    if ( unlikely(op->frame != act->frame) ) 
   1.200 +    {
   1.201 +        /*
   1.202 +         * Suggests that __gntab_unmap_common failed early and so
   1.203 +         * nothing further to do
   1.204 +         */
   1.205 +        goto unmap_out;
   1.206 +    }
   1.207 +
   1.208 +    if ( op->flags & GNTMAP_device_map ) 
   1.209 +    {
   1.210 +        if ( op->flags & GNTMAP_readonly )
   1.211 +            put_page(mfn_to_page(op->frame));
   1.212 +        else
   1.213 +            put_page_and_type(mfn_to_page(op->frame));
   1.214 +    }
   1.215  
   1.216 -    __gnttab_unmap_common(&common);
   1.217 -    op->status = common.status;
   1.218 +    if ( (op->host_addr != 0) && (op->flags & GNTMAP_host_map) )
   1.219 +    {
   1.220 +        if ( op->status != 0 ) 
   1.221 +        {
   1.222 +            /*
   1.223 +             * Suggests that __gntab_unmap_common failed in
   1.224 +             * replace_grant_host_mapping() so nothing further to do
   1.225 +             */
   1.226 +            goto unmap_out;
   1.227 +        }
   1.228 +
   1.229 +        if ( op->flags & GNTMAP_readonly )
   1.230 +            put_page(mfn_to_page(op->frame));
   1.231 +        else
   1.232 +            put_page_and_type(mfn_to_page(op->frame));
   1.233 +    }
   1.234 +
   1.235 +    if ( (op->map->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0 )
   1.236 +    {
   1.237 +        op->map->flags = 0;
   1.238 +        put_maptrack_handle(ld->grant_table, op->handle);
   1.239 +    }
   1.240 +
   1.241 +    if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0) &&
   1.242 +         !(op->flags & GNTMAP_readonly) )
   1.243 +        gnttab_clear_flag(_GTF_writing, &sha->flags);
   1.244 +
   1.245 +    if ( act->pin == 0 )
   1.246 +        gnttab_clear_flag(_GTF_reading, &sha->flags);
   1.247 +
   1.248 + unmap_out:
   1.249 +    spin_unlock(&rd->grant_table->lock);
   1.250 +    rcu_unlock_domain(rd);
   1.251  }
   1.252  
   1.253 +static void
   1.254 +__gnttab_unmap_grant_ref(
   1.255 +    struct gnttab_unmap_grant_ref *op,
   1.256 +    struct gnttab_unmap_common *common)
   1.257 +{
   1.258 +	common->host_addr = op->host_addr;
   1.259 +    common->dev_bus_addr = op->dev_bus_addr;
   1.260 +    common->handle = op->handle;
   1.261 +
   1.262 +    /* Intialise these in case common contains old state */
   1.263 +    common->new_addr = 0;
   1.264 +    common->rd = NULL;
   1.265 +
   1.266 +    __gnttab_unmap_common(common);
   1.267 +    op->status = common->status;
   1.268 +}
   1.269 +
   1.270 +
   1.271  static long
   1.272  gnttab_unmap_grant_ref(
   1.273      XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) uop, unsigned int count)
   1.274  {
   1.275 -    int i;
   1.276 +    int i, c, partial_done, done = 0;
   1.277      struct gnttab_unmap_grant_ref op;
   1.278 +    struct gnttab_unmap_common common[GNTTAB_UNMAP_BATCH_SIZE];
   1.279 +
   1.280 +    while (count != 0) {
   1.281 +        c = min(count, (unsigned int)GNTTAB_UNMAP_BATCH_SIZE);
   1.282 +        partial_done = 0;
   1.283  
   1.284 -    for ( i = 0; i < count; i++ )
   1.285 -    {
   1.286 -        if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
   1.287 -            goto fault;
   1.288 -        __gnttab_unmap_grant_ref(&op);
   1.289 -        if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
   1.290 -            goto fault;
   1.291 +        for ( i = 0; i < c; i++ )
   1.292 +        {
   1.293 +            if ( unlikely(__copy_from_guest_offset(&op, uop, done+i, 1)) )
   1.294 +                goto fault;
   1.295 +            __gnttab_unmap_grant_ref(&op, &(common[i]));
   1.296 +            ++partial_done;
   1.297 +            if ( unlikely(__copy_to_guest_offset(uop, done+i, &op, 1)) )
   1.298 +                goto fault;
   1.299 +        }
   1.300 +
   1.301 +        flush_tlb_mask(current->domain->domain_dirty_cpumask);
   1.302 +
   1.303 +        for ( i = 0; i < partial_done; i++ )
   1.304 +        {
   1.305 +            __gnttab_unmap_common_complete(&(common[i]));
   1.306 +        }
   1.307 +
   1.308 +        count -= c;
   1.309 +        done += c;
   1.310      }
   1.311 -
   1.312 -    flush_tlb_mask(current->domain->domain_dirty_cpumask);
   1.313 +     
   1.314      return 0;
   1.315  
   1.316  fault:
   1.317      flush_tlb_mask(current->domain->domain_dirty_cpumask);
   1.318 -    return -EFAULT;    
   1.319 +
   1.320 +    for ( i = 0; i < partial_done; i++ )
   1.321 +    {
   1.322 +        __gnttab_unmap_common_complete(&(common[i]));
   1.323 +    }
   1.324 +    return -EFAULT;
   1.325  }
   1.326  
   1.327  static void
   1.328  __gnttab_unmap_and_replace(
   1.329 -    struct gnttab_unmap_and_replace *op)
   1.330 +    struct gnttab_unmap_and_replace *op,
   1.331 +    struct gnttab_unmap_common *common)
   1.332  {
   1.333 -    struct gnttab_unmap_common common = {
   1.334 -        .host_addr = op->host_addr,
   1.335 -        .new_addr = op->new_addr,
   1.336 -        .handle = op->handle,
   1.337 -    };
   1.338 +	common->host_addr = op->host_addr;
   1.339 +	common->new_addr = op->new_addr;
   1.340 +	common->handle = op->handle;
   1.341 +    
   1.342 +    /* Intialise these in case common contains old state */
   1.343 +    common->dev_bus_addr = 0;
   1.344 +    common->rd = NULL;
   1.345  
   1.346 -    __gnttab_unmap_common(&common);
   1.347 -    op->status = common.status;
   1.348 +    __gnttab_unmap_common(common);
   1.349 +    op->status = common->status;
   1.350  }
   1.351  
   1.352  static long
   1.353  gnttab_unmap_and_replace(
   1.354      XEN_GUEST_HANDLE(gnttab_unmap_and_replace_t) uop, unsigned int count)
   1.355  {
   1.356 -    int i;
   1.357 +    int i, c, partial_done, done = 0;
   1.358      struct gnttab_unmap_and_replace op;
   1.359 +    struct gnttab_unmap_common common[GNTTAB_UNMAP_BATCH_SIZE];
   1.360  
   1.361 -    for ( i = 0; i < count; i++ )
   1.362 -    {
   1.363 -        if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
   1.364 -            goto fault;
   1.365 -        __gnttab_unmap_and_replace(&op);
   1.366 -        if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
   1.367 -            goto fault;
   1.368 +    while (count != 0) {
   1.369 +        c = min(count, (unsigned int)GNTTAB_UNMAP_BATCH_SIZE);
   1.370 +        partial_done = 0;
   1.371 +        
   1.372 +        for ( i = 0; i < c; i++ )
   1.373 +        {
   1.374 +            if ( unlikely(__copy_from_guest_offset(&op, uop, done+i, 1)) )
   1.375 +                goto fault;
   1.376 +            __gnttab_unmap_and_replace(&op, &(common[i]));
   1.377 +            ++partial_done;
   1.378 +            if ( unlikely(__copy_to_guest_offset(uop, done+i, &op, 1)) )
   1.379 +                goto fault;
   1.380 +        }
   1.381 +        
   1.382 +        flush_tlb_mask(current->domain->domain_dirty_cpumask);
   1.383 +        
   1.384 +        for ( i = 0; i < partial_done; i++ )
   1.385 +        {
   1.386 +            __gnttab_unmap_common_complete(&(common[i]));
   1.387 +        }
   1.388 +
   1.389 +        count -= c;
   1.390 +        done += c;
   1.391      }
   1.392  
   1.393 -    flush_tlb_mask(current->domain->domain_dirty_cpumask);
   1.394      return 0;
   1.395  
   1.396  fault:
   1.397      flush_tlb_mask(current->domain->domain_dirty_cpumask);
   1.398 +
   1.399 +    for ( i = 0; i < partial_done; i++ )
   1.400 +    {
   1.401 +        __gnttab_unmap_common_complete(&(common[i]));
   1.402 +    }
   1.403      return -EFAULT;    
   1.404  }
   1.405  
     2.1 --- a/xen/include/xen/iocap.h	Thu Oct 04 10:03:04 2007 +0100
     2.2 +++ b/xen/include/xen/iocap.h	Thu Oct 04 10:25:35 2007 +0100
     2.3 @@ -31,12 +31,6 @@
     2.4  #define multipage_allocation_permitted(d)               \
     2.5      (!rangeset_is_empty((d)->iomem_caps))
     2.6  
     2.7 -/*
     2.8 - * Until TLB flushing issues are sorted out we consider it unsafe for
     2.9 - * domains with no hardware-access privileges to perform grant map/transfer
    2.10 - * operations.
    2.11 - */
    2.12 -#define grant_operation_permitted(d)                    \
    2.13 -    (!rangeset_is_empty((d)->iomem_caps))
    2.14 +#define grant_operation_permitted(d) (1)
    2.15  
    2.16  #endif /* __XEN_IOCAP_H__ */