direct-io.hg

changeset 11429:66dd34f2f439

[XEN] Clean up grant-table pinning logic.
Signed-off-by: Keir Fraser <keir@xensource.com>
author kfraser@ubuntu.eng.hq.xensource.com
date Tue Sep 05 12:20:31 2006 -0700 (2006-09-05)
parents 4fc1688d9a16
children 698eb277331c
files xen/common/grant_table.c
line diff
     1.1 --- a/xen/common/grant_table.c	Tue Sep 05 17:26:20 2006 +0100
     1.2 +++ b/xen/common/grant_table.c	Tue Sep 05 12:20:31 2006 -0700
     1.3 @@ -90,11 +90,8 @@ static void
     1.4      unsigned long  frame = 0;
     1.5      int            rc = GNTST_okay;
     1.6      struct active_grant_entry *act;
     1.7 -
     1.8 -    /* Entry details from @rd's shared grant table. */
     1.9      grant_entry_t *sha;
    1.10 -    domid_t        sdom;
    1.11 -    u16            sflags;
    1.12 +    union grant_combo scombo, prev_scombo, new_scombo;
    1.13  
    1.14      /*
    1.15       * We bound the number of times we retry CMPXCHG on memory locations that
    1.16 @@ -175,13 +172,20 @@ static void
    1.17  
    1.18      spin_lock(&rd->grant_table->lock);
    1.19  
    1.20 -    sdom = sha->domid;
    1.21 +    /* If already pinned, check the active domid and avoid refcnt overflow. */
    1.22 +    if ( act->pin &&
    1.23 +         ((act->domid != ld->domain_id) ||
    1.24 +          (act->pin & 0x80808080U) != 0) )
    1.25 +        PIN_FAIL(unlock_out, GNTST_general_error,
    1.26 +                 "Bad domain (%d != %d), or risk of counter overflow %08x\n",
    1.27 +                 act->domid, ld->domain_id, act->pin);
    1.28  
    1.29      if ( !act->pin ||
    1.30           (!(op->flags & GNTMAP_readonly) &&
    1.31            !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask))) )
    1.32      {
    1.33 -        sflags = sha->flags;
    1.34 +        scombo.shorts.flags = sha->flags;
    1.35 +        scombo.shorts.domid = sha->domid;
    1.36  
    1.37          /*
    1.38           * This loop attempts to set the access (reading/writing) flags
    1.39 @@ -191,33 +195,29 @@ static void
    1.40           */
    1.41          for ( ; ; )
    1.42          {
    1.43 -            union grant_combo scombo, prev_scombo, new_scombo;
    1.44 +            /* If not already pinned, check the grant domid and type. */
    1.45 +            if ( !act->pin &&
    1.46 +                 (((scombo.shorts.flags & GTF_type_mask) !=
    1.47 +                   GTF_permit_access) ||
    1.48 +                  (scombo.shorts.domid != ld->domain_id)) )
    1.49 +                 PIN_FAIL(unlock_out, GNTST_general_error,
    1.50 +                          "Bad flags (%x) or dom (%d). (expected dom %d)\n",
    1.51 +                          scombo.shorts.flags, scombo.shorts.domid,
    1.52 +                          ld->domain_id);
    1.53  
    1.54 -            if ( unlikely((sflags & GTF_type_mask) != GTF_permit_access) ||
    1.55 -                 unlikely(sdom != led->domain->domain_id) )
    1.56 -                PIN_FAIL(unlock_out, GNTST_general_error,
    1.57 -                         "Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
    1.58 -                         sflags, sdom, led->domain->domain_id);
    1.59 -
    1.60 -            /* Merge two 16-bit values into a 32-bit combined update. */
    1.61 -            scombo.shorts.flags = sflags;
    1.62 -            scombo.shorts.domid = sdom;
    1.63 -            
    1.64              new_scombo = scombo;
    1.65              new_scombo.shorts.flags |= GTF_reading;
    1.66  
    1.67              if ( !(op->flags & GNTMAP_readonly) )
    1.68              {
    1.69                  new_scombo.shorts.flags |= GTF_writing;
    1.70 -                if ( unlikely(sflags & GTF_readonly) )
    1.71 +                if ( unlikely(scombo.shorts.flags & GTF_readonly) )
    1.72                      PIN_FAIL(unlock_out, GNTST_general_error,
    1.73                               "Attempt to write-pin a r/o grant entry.\n");
    1.74              }
    1.75  
    1.76              prev_scombo.word = cmpxchg((u32 *)&sha->flags,
    1.77                                         scombo.word, new_scombo.word);
    1.78 -
    1.79 -            /* Did the combined update work (did we see what we expected?). */
    1.80              if ( likely(prev_scombo.word == scombo.word) )
    1.81                  break;
    1.82  
    1.83 @@ -225,24 +225,15 @@ static void
    1.84                  PIN_FAIL(unlock_out, GNTST_general_error,
    1.85                           "Shared grant entry is unstable.\n");
    1.86  
    1.87 -            /* Didn't see what we expected. Split out the seen flags & dom. */
    1.88 -            sflags = prev_scombo.shorts.flags;
    1.89 -            sdom   = prev_scombo.shorts.domid;
    1.90 +            scombo = prev_scombo;
    1.91          }
    1.92  
    1.93          if ( !act->pin )
    1.94          {
    1.95 -            act->domid = sdom;
    1.96 +            act->domid = scombo.shorts.domid;
    1.97              act->frame = gmfn_to_mfn(rd, sha->frame);
    1.98          }
    1.99      }
   1.100 -    else if ( unlikely(sdom != ld->domain_id) )
   1.101 -        PIN_FAIL(unlock_out, GNTST_general_error,
   1.102 -                 "Bad domain (%d). (NB. expected dom %d)\n",
   1.103 -                 sdom, ld->domain_id);
   1.104 -    else if ( (act->pin & 0x80808080U) != 0 )
   1.105 -        PIN_FAIL(unlock_out, ENOSPC,
   1.106 -                 "Risk of counter overflow %08x\n", act->pin);
   1.107  
   1.108      if ( op->flags & GNTMAP_device_map )
   1.109          act->pin += (op->flags & GNTMAP_readonly) ?
   1.110 @@ -550,9 +541,7 @@ gnttab_prepare_for_transfer(
   1.111  {
   1.112      struct grant_table *rgt;
   1.113      struct grant_entry *sha;
   1.114 -    domid_t             sdom;
   1.115 -    u16                 sflags;
   1.116 -    union grant_combo   scombo, prev_scombo, tmp_scombo;
   1.117 +    union grant_combo   scombo, prev_scombo, new_scombo;
   1.118      int                 retries = 0;
   1.119  
   1.120      if ( unlikely((rgt = rd->grant_table) == NULL) ||
   1.121 @@ -567,29 +556,25 @@ gnttab_prepare_for_transfer(
   1.122  
   1.123      sha = &rgt->shared[ref];
   1.124      
   1.125 -    sflags = sha->flags;
   1.126 -    sdom   = sha->domid;
   1.127 +    scombo.shorts.flags = sha->flags;
   1.128 +    scombo.shorts.domid = sha->domid;
   1.129  
   1.130      for ( ; ; )
   1.131      {
   1.132 -        if ( unlikely(sflags != GTF_accept_transfer) ||
   1.133 -             unlikely(sdom != ld->domain_id) )
   1.134 +        if ( unlikely(scombo.shorts.flags != GTF_accept_transfer) ||
   1.135 +             unlikely(scombo.shorts.domid != ld->domain_id) )
   1.136          {
   1.137              DPRINTK("Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
   1.138 -                    sflags, sdom, ld->domain_id);
   1.139 +                    scombo.shorts.flags, scombo.shorts.domid,
   1.140 +                    ld->domain_id);
   1.141              goto fail;
   1.142          }
   1.143  
   1.144 -        /* Merge two 16-bit values into a 32-bit combined update. */
   1.145 -        scombo.shorts.flags = sflags;
   1.146 -        scombo.shorts.domid = sdom;
   1.147 +        new_scombo = scombo;
   1.148 +        new_scombo.shorts.flags |= GTF_transfer_committed;
   1.149  
   1.150 -        tmp_scombo = scombo;
   1.151 -        tmp_scombo.shorts.flags |= GTF_transfer_committed;
   1.152          prev_scombo.word = cmpxchg((u32 *)&sha->flags,
   1.153 -                                   scombo.word, tmp_scombo.word);
   1.154 -
   1.155 -        /* Did the combined update work (did we see what we expected?). */
   1.156 +                                   scombo.word, new_scombo.word);
   1.157          if ( likely(prev_scombo.word == scombo.word) )
   1.158              break;
   1.159  
   1.160 @@ -599,9 +584,7 @@ gnttab_prepare_for_transfer(
   1.161              goto fail;
   1.162          }
   1.163  
   1.164 -        /* Didn't see what we expected. Split out the seen flags & dom. */
   1.165 -        sflags = prev_scombo.shorts.flags;
   1.166 -        sdom   = prev_scombo.shorts.domid;
   1.167 +        scombo = prev_scombo;
   1.168      }
   1.169  
   1.170      spin_unlock(&rgt->lock);
   1.171 @@ -739,16 +722,21 @@ static void
   1.172          gnttab_mark_dirty(rd, r_frame);
   1.173  
   1.174      spin_lock(&rd->grant_table->lock);
   1.175 +
   1.176      if ( readonly )
   1.177 +    {
   1.178          act->pin -= GNTPIN_hstr_inc;
   1.179 +    }
   1.180      else
   1.181 +    {
   1.182          act->pin -= GNTPIN_hstw_inc;
   1.183 -
   1.184 -    if ( !(act->pin & GNTPIN_hstw_mask) && !readonly )
   1.185 -        gnttab_clear_flag(_GTF_writing, &sha->flags);
   1.186 +        if ( !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) )
   1.187 +            gnttab_clear_flag(_GTF_writing, &sha->flags);
   1.188 +    }
   1.189  
   1.190      if ( !act->pin )
   1.191          gnttab_clear_flag(_GTF_reading, &sha->flags);
   1.192 +
   1.193      spin_unlock(&rd->grant_table->lock);
   1.194  }
   1.195  
   1.196 @@ -764,8 +752,7 @@ static int
   1.197      struct active_grant_entry *act;
   1.198      s16 rc = GNTST_okay;
   1.199      int retries = 0;
   1.200 -    u16 sflags;
   1.201 -    domid_t sdom;
   1.202 +    union grant_combo scombo, prev_scombo, new_scombo;
   1.203  
   1.204      if ( unlikely(gref >= NR_GRANT_ENTRIES) )
   1.205          PIN_FAIL(error_out, GNTST_bad_gntref,
   1.206 @@ -776,37 +763,43 @@ static int
   1.207  
   1.208      spin_lock(&rd->grant_table->lock);
   1.209      
   1.210 -    sdom = sha->domid;
   1.211 +    /* If already pinned, check the active domid and avoid refcnt overflow. */
   1.212 +    if ( act->pin &&
   1.213 +         ((act->domid != current->domain->domain_id) ||
   1.214 +          (act->pin & 0x80808080U) != 0) )
   1.215 +        PIN_FAIL(unlock_out, GNTST_general_error,
   1.216 +                 "Bad domain (%d != %d), or risk of counter overflow %08x\n",
   1.217 +                 act->domid, current->domain->domain_id, act->pin);
   1.218  
   1.219      if ( !act->pin ||
   1.220 -         (!readonly && !(act->pin & GNTPIN_hstw_mask)) )
   1.221 +         (!readonly && !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask))) )
   1.222      {
   1.223 -        sflags = sha->flags;
   1.224 +        scombo.shorts.flags = sha->flags;
   1.225 +        scombo.shorts.domid = sha->domid;
   1.226  
   1.227          for ( ; ; )
   1.228          {
   1.229 -            union grant_combo scombo, prev_scombo, new_scombo;
   1.230 +            /* If not already pinned, check the grant domid and type. */
   1.231 +            if ( !act->pin &&
   1.232 +                 (((scombo.shorts.flags & GTF_type_mask) !=
   1.233 +                   GTF_permit_access) ||
   1.234 +                  (scombo.shorts.domid != current->domain->domain_id)) )
   1.235 +                 PIN_FAIL(unlock_out, GNTST_general_error,
   1.236 +                          "Bad flags (%x) or dom (%d). (expected dom %d)\n",
   1.237 +                          scombo.shorts.flags, scombo.shorts.domid,
   1.238 +                          current->domain->domain_id);
   1.239  
   1.240 -            if ( unlikely((sflags & GTF_type_mask) != GTF_permit_access ||
   1.241 -                          sdom != current->domain->domain_id ) )
   1.242 -                PIN_FAIL(unlock_out, GNTST_general_error,
   1.243 -                         "Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
   1.244 -                         sflags, sdom, current->domain->domain_id);
   1.245 -
   1.246 -            /* Merge two 16-bit values into a 32-bit combined update. */
   1.247 -            scombo.shorts.flags = sflags;
   1.248 -            scombo.shorts.domid = sdom;
   1.249 -            
   1.250              new_scombo = scombo;
   1.251              new_scombo.shorts.flags |= GTF_reading;
   1.252  
   1.253              if ( !readonly )
   1.254              {
   1.255                  new_scombo.shorts.flags |= GTF_writing;
   1.256 -                if ( unlikely(sflags & GTF_readonly) )
   1.257 +                if ( unlikely(scombo.shorts.flags & GTF_readonly) )
   1.258                      PIN_FAIL(unlock_out, GNTST_general_error,
   1.259                               "Attempt to write-pin a r/o grant entry.\n");
   1.260              }
   1.261 +
   1.262              prev_scombo.word = cmpxchg((u32 *)&sha->flags,
   1.263                                         scombo.word, new_scombo.word);
   1.264              if ( likely(prev_scombo.word == scombo.word) )
   1.265 @@ -815,23 +808,16 @@ static int
   1.266              if ( retries++ == 4 )
   1.267                  PIN_FAIL(unlock_out, GNTST_general_error,
   1.268                           "Shared grant entry is unstable.\n");
   1.269 -            sflags = prev_scombo.shorts.flags;
   1.270 -            sdom = prev_scombo.shorts.flags;
   1.271 +
   1.272 +            scombo = prev_scombo;
   1.273          }
   1.274  
   1.275          if ( !act->pin )
   1.276          {
   1.277 -            act->domid = sdom;
   1.278 +            act->domid = scombo.shorts.domid;
   1.279              act->frame = gmfn_to_mfn(rd, sha->frame);
   1.280          }
   1.281      }
   1.282 -    else if ( unlikely(sdom != current->domain->domain_id) )
   1.283 -        PIN_FAIL(unlock_out, GNTST_general_error,
   1.284 -                 "Bad domain (%d). (NB. expected dom %d)\n",
   1.285 -                 sdom, current->domain->domain_id);
   1.286 -    else if ( (act->pin & 0x80808080U) != 0 )
   1.287 -        PIN_FAIL(unlock_out, ENOSPC,
   1.288 -                 "Risk of counter overflow %08x\n", act->pin);
   1.289  
   1.290      act->pin += readonly ? GNTPIN_hstr_inc : GNTPIN_hstw_inc;
   1.291