ia64/xen-unstable

annotate xen/common/grant_table.c @ 19835:edfdeb150f27

Fix buildsystem to detect udev > version 124

udev removed the udevinfo symlink from versions higher than 123 and
xen's build-system could not detect if udev is in place and has the
required version.

Signed-off-by: Marc-A. Dahlhaus <mad@wol.de>
author Keir Fraser <keir.fraser@citrix.com>
date Thu Jun 25 13:02:37 2009 +0100 (2009-06-25)
parents 2e83c670f680
children
rev   line source
kaf24@2322 1 /******************************************************************************
kaf24@2322 2 * common/grant_table.c
kaf24@2322 3 *
kaf24@2322 4 * Mechanism for granting foreign access to page frames, and receiving
kaf24@2322 5 * page-ownership transfers.
kaf24@2322 6 *
kfraser@13970 7 * Copyright (c) 2005-2006 Christopher Clark
kaf24@2322 8 * Copyright (c) 2004 K A Fraser
akw27@6145 9 * Copyright (c) 2005 Andrew Warfield
akw27@6145 10 * Modifications by Geoffrey Lefebvre are (c) Intel Research Cambridge
kaf24@2322 11 *
kaf24@2322 12 * This program is free software; you can redistribute it and/or modify
kaf24@2322 13 * it under the terms of the GNU General Public License as published by
kaf24@2322 14 * the Free Software Foundation; either version 2 of the License, or
kaf24@2322 15 * (at your option) any later version.
kaf24@2322 16 *
kaf24@2322 17 * This program is distributed in the hope that it will be useful,
kaf24@2322 18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
kaf24@2322 19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
kaf24@2322 20 * GNU General Public License for more details.
kaf24@2322 21 *
kaf24@2322 22 * You should have received a copy of the GNU General Public License
kaf24@2322 23 * along with this program; if not, write to the Free Software
kaf24@2322 24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
kaf24@2322 25 */
kaf24@2322 26
kfraser@12507 27 #include <xen/config.h>
kfraser@12507 28 #include <xen/iocap.h>
cl349@5247 29 #include <xen/lib.h>
kaf24@2322 30 #include <xen/sched.h>
kaf24@4484 31 #include <xen/mm.h>
kaf24@9183 32 #include <xen/trace.h>
kaf24@9183 33 #include <xen/guest_access.h>
kaf24@11094 34 #include <xen/domain_page.h>
keir@17732 35 #include <xen/iommu.h>
keir@17732 36 #include <xen/paging.h>
kfraser@15815 37 #include <xsm/xsm.h>
kaf24@2322 38
hollisb@14233 39 #ifndef max_nr_grant_frames
kfraser@13970 40 unsigned int max_nr_grant_frames = DEFAULT_MAX_NR_GRANT_FRAMES;
kfraser@13970 41 integer_param("gnttab_max_nr_frames", max_nr_grant_frames);
hollisb@14233 42 #endif
kfraser@13970 43
kfraser@13970 44 /* The maximum number of grant mappings is defined as a multiplier of the
kfraser@13970 45 * maximum number of grant table entries. This defines the multiplier used.
kfraser@13970 46 * Pretty arbitrary. [POLICY]
kfraser@13970 47 */
kfraser@13970 48 #define MAX_MAPTRACK_TO_GRANTS_RATIO 8
kfraser@13970 49
kfraser@11106 50 /*
kfraser@11106 51 * The first two members of a grant entry are updated as a combined pair.
kfraser@11106 52 * The following union allows that to happen in an endian-neutral fashion.
kfraser@11106 53 */
kfraser@11106 54 union grant_combo {
kfraser@11106 55 uint32_t word;
kfraser@11106 56 struct {
kfraser@11106 57 uint16_t flags;
kfraser@11106 58 domid_t domid;
kfraser@11106 59 } shorts;
kfraser@11106 60 };
kfraser@11106 61
kfraser@15235 62 /* Used to share code between unmap_grant_ref and unmap_and_replace. */
kfraser@15235 63 struct gnttab_unmap_common {
keir@16066 64 /* Input */
kfraser@15235 65 uint64_t host_addr;
kfraser@15235 66 uint64_t dev_bus_addr;
kfraser@15235 67 uint64_t new_addr;
kfraser@15235 68 grant_handle_t handle;
kfraser@15235 69
keir@16066 70 /* Return */
kfraser@15235 71 int16_t status;
keir@16066 72
keir@16066 73 /* Shared state beteen *_unmap and *_unmap_complete */
keir@16066 74 u16 flags;
keir@16066 75 unsigned long frame;
keir@16066 76 struct grant_mapping *map;
keir@16066 77 struct domain *rd;
kfraser@15235 78 };
kfraser@15235 79
keir@16066 80 /* Number of unmap operations that are done between each tlb flush */
keir@16066 81 #define GNTTAB_UNMAP_BATCH_SIZE 32
keir@16066 82
keir@16066 83
kaf24@7984 84 #define PIN_FAIL(_lbl, _rc, _f, _a...) \
kaf24@7984 85 do { \
kaf24@12038 86 gdprintk(XENLOG_WARNING, _f, ## _a ); \
kaf24@7984 87 rc = (_rc); \
kaf24@7984 88 goto _lbl; \
kaf24@2344 89 } while ( 0 )
kaf24@2344 90
kfraser@13970 91 #define MAPTRACK_PER_PAGE (PAGE_SIZE / sizeof(struct grant_mapping))
kfraser@13970 92 #define maptrack_entry(t, e) \
kfraser@13970 93 ((t)->maptrack[(e)/MAPTRACK_PER_PAGE][(e)%MAPTRACK_PER_PAGE])
kfraser@13970 94
kfraser@13970 95 static inline unsigned int
kfraser@13970 96 nr_maptrack_frames(struct grant_table *t)
kfraser@13970 97 {
kfraser@13970 98 return t->maptrack_limit / MAPTRACK_PER_PAGE;
kfraser@13970 99 }
kfraser@13970 100
kfraser@13970 101 static unsigned inline int max_nr_maptrack_frames(void)
kfraser@13970 102 {
kfraser@13970 103 return (max_nr_grant_frames * MAX_MAPTRACK_TO_GRANTS_RATIO);
kfraser@13970 104 }
kfraser@13970 105
kfraser@13970 106
kfraser@13970 107 #define SHGNT_PER_PAGE (PAGE_SIZE / sizeof(grant_entry_t))
kfraser@13970 108 #define shared_entry(t, e) \
kfraser@13970 109 ((t)->shared[(e)/SHGNT_PER_PAGE][(e)%SHGNT_PER_PAGE])
kfraser@13970 110 #define ACGNT_PER_PAGE (PAGE_SIZE / sizeof(struct active_grant_entry))
kfraser@13970 111 #define active_entry(t, e) \
kfraser@13970 112 ((t)->active[(e)/ACGNT_PER_PAGE][(e)%ACGNT_PER_PAGE])
kfraser@13970 113
kaf24@2617 114 static inline int
kfraser@13970 115 __get_maptrack_handle(
kaf24@9723 116 struct grant_table *t)
kaf24@2617 117 {
kaf24@2617 118 unsigned int h;
akw27@6145 119 if ( unlikely((h = t->maptrack_head) == (t->maptrack_limit - 1)) )
kaf24@2617 120 return -1;
kfraser@13970 121 t->maptrack_head = maptrack_entry(t, h).ref;
kaf24@2617 122 return h;
kaf24@2617 123 }
kaf24@2617 124
kaf24@2340 125 static inline void
kaf24@2617 126 put_maptrack_handle(
kaf24@9723 127 struct grant_table *t, int handle)
kaf24@2340 128 {
kfraser@13970 129 maptrack_entry(t, handle).ref = t->maptrack_head;
kaf24@2617 130 t->maptrack_head = handle;
kaf24@2340 131 }
kaf24@2340 132
kfraser@13970 133 static inline int
kfraser@13970 134 get_maptrack_handle(
kfraser@13970 135 struct grant_table *lgt)
kfraser@13970 136 {
kfraser@13970 137 int i;
kfraser@13970 138 grant_handle_t handle;
kfraser@13970 139 struct grant_mapping *new_mt;
kfraser@13970 140 unsigned int new_mt_limit, nr_frames;
kfraser@13970 141
kfraser@13970 142 if ( unlikely((handle = __get_maptrack_handle(lgt)) == -1) )
kfraser@13970 143 {
keir@19694 144 spin_lock(&lgt->lock);
kfraser@13970 145
keir@19694 146 if ( unlikely((handle = __get_maptrack_handle(lgt)) == -1) )
keir@19694 147 {
keir@19694 148 nr_frames = nr_maptrack_frames(lgt);
keir@19694 149 if ( nr_frames >= max_nr_maptrack_frames() )
keir@19694 150 {
keir@19694 151 spin_unlock(&lgt->lock);
keir@19694 152 return -1;
keir@19694 153 }
kfraser@13970 154
keir@19694 155 new_mt = alloc_xenheap_page();
keir@19694 156 if ( new_mt == NULL )
keir@19694 157 {
keir@19694 158 spin_unlock(&lgt->lock);
keir@19694 159 return -1;
keir@19694 160 }
keir@19694 161
keir@19694 162 clear_page(new_mt);
keir@19694 163
keir@19694 164 new_mt_limit = lgt->maptrack_limit + MAPTRACK_PER_PAGE;
keir@19694 165
keir@19694 166 for ( i = lgt->maptrack_limit; i < new_mt_limit; i++ )
keir@19694 167 {
keir@19694 168 new_mt[i % MAPTRACK_PER_PAGE].ref = i+1;
keir@19694 169 new_mt[i % MAPTRACK_PER_PAGE].flags = 0;
keir@19694 170 }
keir@19694 171
keir@19694 172 lgt->maptrack[nr_frames] = new_mt;
keir@19694 173 lgt->maptrack_limit = new_mt_limit;
keir@19694 174
keir@19694 175 gdprintk(XENLOG_INFO,
keir@19694 176 "Increased maptrack size to %u frames.\n", nr_frames + 1);
keir@19694 177 handle = __get_maptrack_handle(lgt);
kfraser@13970 178 }
kfraser@13970 179
keir@19694 180 spin_unlock(&lgt->lock);
kfraser@13970 181 }
kfraser@13970 182 return handle;
kfraser@13970 183 }
kfraser@13970 184
kaf24@7042 185 /*
kaf24@7042 186 * Returns 0 if TLB flush / invalidate required by caller.
kaf24@7042 187 * va will indicate the address to be invalidated.
kaf24@7042 188 *
kaf24@7042 189 * addr is _either_ a host virtual address, or the address of the pte to
kaf24@7042 190 * update, as indicated by the GNTMAP_contains_pte flag.
kaf24@7042 191 */
kaf24@8859 192 static void
kaf24@7042 193 __gnttab_map_grant_ref(
kaf24@8859 194 struct gnttab_map_grant_ref *op)
kaf24@2337 195 {
keir@19270 196 struct domain *ld, *rd, *owner;
kaf24@7042 197 struct vcpu *led;
kaf24@7042 198 int handle;
keir@17727 199 unsigned long frame = 0, nr_gets = 0;
kaf24@7055 200 int rc = GNTST_okay;
keir@17727 201 u32 old_pin;
keir@19626 202 u32 act_pin;
keir@16407 203 unsigned int cache_flags;
kaf24@9723 204 struct active_grant_entry *act;
kfraser@13970 205 struct grant_mapping *mt;
kaf24@7042 206 grant_entry_t *sha;
kfraser@11437 207 union grant_combo scombo, prev_scombo, new_scombo;
kaf24@2322 208
kaf24@2347 209 /*
kaf24@2617 210 * We bound the number of times we retry CMPXCHG on memory locations that
kaf24@2617 211 * we share with a guest OS. The reason is that the guest can modify that
kaf24@2617 212 * location at a higher rate than we can read-modify-CMPXCHG, so the guest
kaf24@2617 213 * could cause us to livelock. There are a few cases where it is valid for
kaf24@2617 214 * the guest to race our updates (e.g., to change the GTF_readonly flag),
kaf24@2617 215 * so we allow a few retries before failing.
kaf24@2347 216 */
kaf24@7042 217 int retries = 0;
kaf24@2337 218
kaf24@7042 219 led = current;
kaf24@7042 220 ld = led->domain;
kaf24@2617 221
kfraser@13970 222 if ( unlikely((op->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0) )
kaf24@7042 223 {
kfraser@13970 224 gdprintk(XENLOG_INFO, "Bad flags in grant map op (%x).\n", op->flags);
kaf24@8859 225 op->status = GNTST_bad_gntref;
kaf24@8859 226 return;
kaf24@7042 227 }
kaf24@7042 228
kfraser@14192 229 if ( unlikely((rd = rcu_lock_domain_by_id(op->dom)) == NULL) )
kaf24@7042 230 {
kaf24@12038 231 gdprintk(XENLOG_INFO, "Could not find domain %d\n", op->dom);
kaf24@8859 232 op->status = GNTST_bad_domain;
kaf24@8859 233 return;
kaf24@7042 234 }
kaf24@7042 235
kfraser@15815 236 rc = xsm_grant_mapref(ld, rd, op->flags);
kfraser@15815 237 if ( rc )
kfraser@15815 238 {
kfraser@15815 239 rcu_unlock_domain(rd);
kfraser@15815 240 op->status = GNTST_permission_denied;
kfraser@15815 241 return;
kfraser@15815 242 }
kfraser@15815 243
kaf24@7042 244 if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
kaf24@7042 245 {
kfraser@14192 246 rcu_unlock_domain(rd);
kfraser@13970 247 gdprintk(XENLOG_INFO, "Failed to obtain maptrack handle.\n");
kfraser@13970 248 op->status = GNTST_no_device_space;
kfraser@13970 249 return;
kaf24@7042 250 }
kaf24@7042 251
kfraser@13970 252 spin_lock(&rd->grant_table->lock);
kaf24@7042 253
kfraser@13970 254 /* Bounds check on the grant ref */
kfraser@13970 255 if ( unlikely(op->ref >= nr_grant_entries(rd->grant_table)))
kfraser@13970 256 PIN_FAIL(unlock_out, GNTST_bad_gntref, "Bad ref (%d).\n", op->ref);
kfraser@13970 257
kfraser@13970 258 act = &active_entry(rd->grant_table, op->ref);
kfraser@13970 259 sha = &shared_entry(rd->grant_table, op->ref);
kaf24@2337 260
kfraser@11437 261 /* If already pinned, check the active domid and avoid refcnt overflow. */
kfraser@11437 262 if ( act->pin &&
kfraser@11437 263 ((act->domid != ld->domain_id) ||
kfraser@11437 264 (act->pin & 0x80808080U) != 0) )
kfraser@11437 265 PIN_FAIL(unlock_out, GNTST_general_error,
kfraser@11437 266 "Bad domain (%d != %d), or risk of counter overflow %08x\n",
kfraser@11437 267 act->domid, ld->domain_id, act->pin);
ssmith@11427 268
kaf24@8427 269 if ( !act->pin ||
kaf24@8859 270 (!(op->flags & GNTMAP_readonly) &&
kaf24@8427 271 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask))) )
kaf24@2337 272 {
kfraser@11439 273 scombo.word = *(u32 *)&sha->flags;
kaf24@2337 274
kaf24@8427 275 /*
kaf24@8427 276 * This loop attempts to set the access (reading/writing) flags
akw27@6145 277 * in the grant table entry. It tries a cmpxchg on the field
akw27@6145 278 * up to five times, and then fails under the assumption that
kaf24@8427 279 * the guest is misbehaving.
kaf24@8427 280 */
kaf24@2340 281 for ( ; ; )
kaf24@2340 282 {
kfraser@11437 283 /* If not already pinned, check the grant domid and type. */
kfraser@11437 284 if ( !act->pin &&
kfraser@11437 285 (((scombo.shorts.flags & GTF_type_mask) !=
kfraser@11437 286 GTF_permit_access) ||
kfraser@11437 287 (scombo.shorts.domid != ld->domain_id)) )
kfraser@11437 288 PIN_FAIL(unlock_out, GNTST_general_error,
kfraser@11437 289 "Bad flags (%x) or dom (%d). (expected dom %d)\n",
kfraser@11437 290 scombo.shorts.flags, scombo.shorts.domid,
kfraser@11437 291 ld->domain_id);
kaf24@2340 292
kfraser@11106 293 new_scombo = scombo;
kfraser@11106 294 new_scombo.shorts.flags |= GTF_reading;
kaf24@2347 295
kaf24@8859 296 if ( !(op->flags & GNTMAP_readonly) )
kaf24@2337 297 {
kfraser@11106 298 new_scombo.shorts.flags |= GTF_writing;
kfraser@11437 299 if ( unlikely(scombo.shorts.flags & GTF_readonly) )
cwc22@4461 300 PIN_FAIL(unlock_out, GNTST_general_error,
kaf24@2344 301 "Attempt to write-pin a r/o grant entry.\n");
kaf24@2337 302 }
kaf24@2340 303
kfraser@11106 304 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
kfraser@11106 305 scombo.word, new_scombo.word);
kfraser@11106 306 if ( likely(prev_scombo.word == scombo.word) )
kaf24@2340 307 break;
kaf24@2340 308
kaf24@2347 309 if ( retries++ == 4 )
cwc22@4461 310 PIN_FAIL(unlock_out, GNTST_general_error,
kaf24@2347 311 "Shared grant entry is unstable.\n");
kaf24@2347 312
kfraser@11437 313 scombo = prev_scombo;
kaf24@2337 314 }
kaf24@2340 315
kaf24@8427 316 if ( !act->pin )
kaf24@2340 317 {
kfraser@11437 318 act->domid = scombo.shorts.domid;
keir@18973 319 act->gfn = sha->frame;
kaf24@8726 320 act->frame = gmfn_to_mfn(rd, sha->frame);
kaf24@8427 321 }
kaf24@8427 322 }
kaf24@2347 323
keir@17727 324 old_pin = act->pin;
kaf24@8859 325 if ( op->flags & GNTMAP_device_map )
kaf24@8859 326 act->pin += (op->flags & GNTMAP_readonly) ?
kaf24@8427 327 GNTPIN_devr_inc : GNTPIN_devw_inc;
kaf24@8859 328 if ( op->flags & GNTMAP_host_map )
kaf24@8859 329 act->pin += (op->flags & GNTMAP_readonly) ?
kaf24@8427 330 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
kaf24@2337 331
kfraser@13970 332 frame = act->frame;
keir@19626 333 act_pin = act->pin;
kfraser@13970 334
keir@16407 335 cache_flags = (sha->flags & (GTF_PAT | GTF_PWT | GTF_PCD) );
keir@16407 336
kaf24@7042 337 spin_unlock(&rd->grant_table->lock);
kaf24@2617 338
keir@19270 339 if ( !mfn_valid(frame) ||
keir@19270 340 (owner = page_get_owner_and_reference(mfn_to_page(frame))) == dom_io )
kfraser@12321 341 {
keir@19270 342 /* Only needed the reference to confirm dom_io ownership. */
keir@19270 343 if ( mfn_valid(frame) )
keir@19270 344 put_page(mfn_to_page(frame));
keir@19270 345
keir@16407 346 if ( !iomem_access_permitted(rd, frame, frame) )
keir@16068 347 {
keir@16407 348 gdprintk(XENLOG_WARNING,
keir@16407 349 "Iomem mapping not permitted %lx (domain %d)\n",
keir@16407 350 frame, rd->domain_id);
keir@16407 351 rc = GNTST_general_error;
kaf24@8427 352 goto undo_out;
kaf24@8427 353 }
keir@16372 354
keir@16407 355 rc = create_grant_host_mapping(
keir@16407 356 op->host_addr, frame, op->flags, cache_flags);
keir@16407 357 if ( rc != GNTST_okay )
keir@16407 358 goto undo_out;
keir@16407 359 }
keir@19270 360 else if ( owner == rd )
keir@16407 361 {
keir@19270 362 if ( gnttab_host_mapping_get_page_type(op, ld, rd) &&
keir@19270 363 !get_page_type(mfn_to_page(frame), PGT_writable_page) )
keir@19270 364 goto could_not_pin;
keir@17727 365
keir@17727 366 nr_gets++;
keir@16407 367 if ( op->flags & GNTMAP_host_map )
keir@16407 368 {
keir@16407 369 rc = create_grant_host_mapping(op->host_addr, frame, op->flags, 0);
keir@16407 370 if ( rc != GNTST_okay )
keir@16407 371 goto undo_out;
keir@16407 372
keir@16407 373 if ( op->flags & GNTMAP_device_map )
keir@16407 374 {
keir@17727 375 nr_gets++;
keir@16407 376 (void)get_page(mfn_to_page(frame), rd);
keir@16407 377 if ( !(op->flags & GNTMAP_readonly) )
keir@16407 378 get_page_type(mfn_to_page(frame), PGT_writable_page);
keir@16407 379 }
cwc22@4461 380 }
cwc22@4461 381 }
keir@19270 382 else
keir@19270 383 {
keir@19270 384 could_not_pin:
keir@19270 385 if ( !rd->is_dying )
keir@19270 386 gdprintk(XENLOG_WARNING, "Could not pin grant frame %lx\n",
keir@19270 387 frame);
keir@19270 388 if ( owner != NULL )
keir@19270 389 put_page(mfn_to_page(frame));
keir@19270 390 rc = GNTST_general_error;
keir@19270 391 goto undo_out;
keir@19270 392 }
kaf24@4571 393
keir@17727 394 if ( need_iommu(ld) &&
keir@17727 395 !(old_pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) &&
keir@19626 396 (act_pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
keir@17727 397 {
keir@17727 398 if ( iommu_map_page(ld, mfn_to_gmfn(ld, frame), frame) )
keir@17727 399 {
keir@17727 400 rc = GNTST_general_error;
keir@17727 401 goto undo_out;
keir@17727 402 }
keir@17727 403 }
keir@17727 404
kaf24@8859 405 TRACE_1D(TRC_MEM_PAGE_GRANT_MAP, op->dom);
kaf24@7818 406
kfraser@13970 407 mt = &maptrack_entry(ld->grant_table, handle);
kfraser@13970 408 mt->domid = op->dom;
kfraser@13970 409 mt->ref = op->ref;
kfraser@13970 410 mt->flags = op->flags;
kaf24@7042 411
kaf24@8859 412 op->dev_bus_addr = (u64)frame << PAGE_SHIFT;
kaf24@8859 413 op->handle = handle;
kaf24@8859 414 op->status = GNTST_okay;
kaf24@7042 415
kfraser@14192 416 rcu_unlock_domain(rd);
kaf24@8859 417 return;
cwc22@4461 418
kaf24@8427 419 undo_out:
keir@17727 420 if ( nr_gets > 1 )
keir@17727 421 {
keir@17727 422 if ( !(op->flags & GNTMAP_readonly) )
keir@17727 423 put_page_type(mfn_to_page(frame));
keir@17727 424 put_page(mfn_to_page(frame));
keir@17727 425 }
keir@17727 426 if ( nr_gets > 0 )
keir@17727 427 {
keir@17727 428 if ( gnttab_host_mapping_get_page_type(op, ld, rd) )
keir@17727 429 put_page_type(mfn_to_page(frame));
keir@17727 430 put_page(mfn_to_page(frame));
keir@17727 431 }
keir@17727 432
kaf24@8427 433 spin_lock(&rd->grant_table->lock);
kaf24@8427 434
kfraser@13970 435 act = &active_entry(rd->grant_table, op->ref);
kfraser@13970 436 sha = &shared_entry(rd->grant_table, op->ref);
kfraser@13970 437
kaf24@8859 438 if ( op->flags & GNTMAP_device_map )
kaf24@8859 439 act->pin -= (op->flags & GNTMAP_readonly) ?
kaf24@8427 440 GNTPIN_devr_inc : GNTPIN_devw_inc;
kaf24@8859 441 if ( op->flags & GNTMAP_host_map )
kaf24@8859 442 act->pin -= (op->flags & GNTMAP_readonly) ?
kaf24@8427 443 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
kaf24@8427 444
kaf24@8859 445 if ( !(op->flags & GNTMAP_readonly) &&
kaf24@8427 446 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
kaf24@10614 447 gnttab_clear_flag(_GTF_writing, &sha->flags);
kaf24@8427 448
kaf24@8427 449 if ( !act->pin )
kaf24@10614 450 gnttab_clear_flag(_GTF_reading, &sha->flags);
kaf24@8427 451
cwc22@4461 452 unlock_out:
kaf24@7042 453 spin_unlock(&rd->grant_table->lock);
kaf24@8859 454 op->status = rc;
kaf24@7042 455 put_maptrack_handle(ld->grant_table, handle);
kfraser@14192 456 rcu_unlock_domain(rd);
kaf24@2617 457 }
kaf24@2617 458
kaf24@2617 459 static long
kaf24@2617 460 gnttab_map_grant_ref(
kaf24@9873 461 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) uop, unsigned int count)
kaf24@2617 462 {
kaf24@7042 463 int i;
kaf24@8859 464 struct gnttab_map_grant_ref op;
cwc22@4016 465
kaf24@2617 466 for ( i = 0; i < count; i++ )
kaf24@8859 467 {
kaf24@9183 468 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
kaf24@8859 469 return -EFAULT;
kaf24@8859 470 __gnttab_map_grant_ref(&op);
kaf24@9183 471 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
kaf24@8859 472 return -EFAULT;
kaf24@8859 473 }
cwc22@4063 474
kaf24@2617 475 return 0;
kaf24@2617 476 }
kaf24@2617 477
kaf24@8859 478 static void
kfraser@15235 479 __gnttab_unmap_common(
kfraser@15235 480 struct gnttab_unmap_common *op)
kaf24@2617 481 {
kaf24@6322 482 domid_t dom;
kaf24@6322 483 struct domain *ld, *rd;
kaf24@9723 484 struct active_grant_entry *act;
kaf24@6322 485 grant_entry_t *sha;
kaf24@7042 486 s16 rc = 0;
keir@17727 487 u32 old_pin;
kaf24@2617 488
cl349@2919 489 ld = current->domain;
kaf24@2617 490
keir@16066 491 op->frame = (unsigned long)(op->dev_bus_addr >> PAGE_SHIFT);
kaf24@2617 492
kfraser@13970 493 if ( unlikely(op->handle >= ld->grant_table->maptrack_limit) )
kaf24@2617 494 {
kaf24@12038 495 gdprintk(XENLOG_INFO, "Bad handle (%d).\n", op->handle);
kaf24@8859 496 op->status = GNTST_bad_handle;
kaf24@8859 497 return;
kaf24@2617 498 }
kaf24@2617 499
keir@16066 500 op->map = &maptrack_entry(ld->grant_table, op->handle);
kfraser@13970 501
keir@16066 502 if ( unlikely(!op->map->flags) )
kfraser@13970 503 {
kfraser@13970 504 gdprintk(XENLOG_INFO, "Zero flags for handle (%d).\n", op->handle);
kfraser@13970 505 op->status = GNTST_bad_handle;
kfraser@13970 506 return;
kfraser@13970 507 }
kfraser@13970 508
keir@16066 509 dom = op->map->domid;
keir@16066 510 op->flags = op->map->flags;
kaf24@2617 511
keir@16066 512 if ( unlikely((op->rd = rd = rcu_lock_domain_by_id(dom)) == NULL) )
kaf24@2617 513 {
kfraser@12533 514 /* This can happen when a grant is implicitly unmapped. */
kaf24@12038 515 gdprintk(XENLOG_INFO, "Could not find domain %d\n", dom);
kfraser@12533 516 domain_crash(ld); /* naughty... */
kaf24@8859 517 return;
kaf24@2617 518 }
kaf24@4571 519
kfraser@15815 520 rc = xsm_grant_unmapref(ld, rd);
kfraser@15815 521 if ( rc )
kfraser@15815 522 {
kfraser@15815 523 rcu_unlock_domain(rd);
kfraser@15815 524 op->status = GNTST_permission_denied;
kfraser@15815 525 return;
kfraser@15815 526 }
kfraser@15815 527
kaf24@7818 528 TRACE_1D(TRC_MEM_PAGE_GRANT_UNMAP, dom);
kaf24@7818 529
kfraser@13970 530 spin_lock(&rd->grant_table->lock);
kaf24@2617 531
keir@16066 532 act = &active_entry(rd->grant_table, op->map->ref);
keir@16066 533 sha = &shared_entry(rd->grant_table, op->map->ref);
keir@17727 534 old_pin = act->pin;
kaf24@2617 535
keir@16066 536 if ( op->frame == 0 )
kaf24@4571 537 {
keir@16066 538 op->frame = act->frame;
kaf24@4571 539 }
kaf24@2617 540 else
kaf24@2617 541 {
keir@16066 542 if ( unlikely(op->frame != act->frame) )
cwc22@4461 543 PIN_FAIL(unmap_out, GNTST_general_error,
keir@16067 544 "Bad frame number doesn't match gntref. (%lx != %lx)\n",
keir@16067 545 op->frame, act->frame);
keir@16066 546 if ( op->flags & GNTMAP_device_map )
kaf24@8427 547 {
kaf24@8427 548 ASSERT(act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask));
keir@16066 549 op->map->flags &= ~GNTMAP_device_map;
keir@16066 550 if ( op->flags & GNTMAP_readonly )
kaf24@8427 551 act->pin -= GNTPIN_devr_inc;
kaf24@8427 552 else
kaf24@8427 553 act->pin -= GNTPIN_devw_inc;
kaf24@8427 554 }
kaf24@2617 555 }
kaf24@2617 556
keir@16372 557 if ( (op->host_addr != 0) && (op->flags & GNTMAP_host_map) )
kaf24@2617 558 {
keir@16372 559 if ( (rc = replace_grant_host_mapping(op->host_addr,
keir@16372 560 op->frame, op->new_addr,
keir@16372 561 op->flags)) < 0 )
keir@16372 562 goto unmap_out;
cwc22@4063 563
kaf24@8427 564 ASSERT(act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask));
keir@16066 565 op->map->flags &= ~GNTMAP_host_map;
keir@16066 566 if ( op->flags & GNTMAP_readonly )
kaf24@8427 567 act->pin -= GNTPIN_hstr_inc;
kaf24@8427 568 else
kaf24@8427 569 act->pin -= GNTPIN_hstw_inc;
keir@16372 570 }
cwc22@4461 571
keir@17727 572 if ( need_iommu(ld) &&
keir@17727 573 (old_pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) &&
keir@17727 574 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
keir@17727 575 {
keir@17727 576 if ( iommu_unmap_page(ld, mfn_to_gmfn(ld, op->frame)) )
keir@17727 577 {
keir@17727 578 rc = GNTST_general_error;
keir@17727 579 goto unmap_out;
keir@17727 580 }
keir@17727 581 }
keir@17727 582
cwc22@4461 583 /* If just unmapped a writable mapping, mark as dirtied */
keir@16066 584 if ( !(op->flags & GNTMAP_readonly) )
keir@16066 585 gnttab_mark_dirty(rd, op->frame);
kaf24@2617 586
cwc22@4461 587 unmap_out:
kaf24@8859 588 op->status = rc;
kaf24@2617 589 spin_unlock(&rd->grant_table->lock);
kfraser@14192 590 rcu_unlock_domain(rd);
kaf24@2617 591 }
kaf24@2617 592
kfraser@15235 593 static void
keir@16066 594 __gnttab_unmap_common_complete(struct gnttab_unmap_common *op)
kfraser@15235 595 {
keir@16066 596 struct domain *ld, *rd;
keir@16066 597 struct active_grant_entry *act;
keir@16066 598 grant_entry_t *sha;
kfraser@15235 599
keir@16066 600 rd = op->rd;
keir@16066 601
keir@16068 602 if ( rd == NULL )
keir@16068 603 {
keir@16066 604 /*
keir@16066 605 * Suggests that __gntab_unmap_common failed in
keir@16066 606 * rcu_lock_domain_by_id() or earlier, and so we have nothing
keir@16066 607 * to complete
keir@16066 608 */
keir@16066 609 return;
keir@16066 610 }
keir@16066 611
keir@16066 612 ld = current->domain;
keir@16066 613
keir@16066 614 rcu_lock_domain(rd);
keir@16066 615 spin_lock(&rd->grant_table->lock);
keir@16066 616
keir@16066 617 act = &active_entry(rd->grant_table, op->map->ref);
keir@16066 618 sha = &shared_entry(rd->grant_table, op->map->ref);
keir@16066 619
keir@16066 620 if ( unlikely(op->frame != act->frame) )
keir@16066 621 {
keir@16066 622 /*
keir@16066 623 * Suggests that __gntab_unmap_common failed early and so
keir@16066 624 * nothing further to do
keir@16066 625 */
keir@16066 626 goto unmap_out;
keir@16066 627 }
keir@16066 628
keir@16066 629 if ( op->flags & GNTMAP_device_map )
keir@16066 630 {
keir@16407 631 if ( !is_iomem_page(act->frame) )
keir@16407 632 {
keir@16407 633 if ( op->flags & GNTMAP_readonly )
keir@16407 634 put_page(mfn_to_page(op->frame));
keir@16407 635 else
keir@16407 636 put_page_and_type(mfn_to_page(op->frame));
keir@16407 637 }
keir@16066 638 }
keir@16066 639
keir@16066 640 if ( (op->host_addr != 0) && (op->flags & GNTMAP_host_map) )
keir@16066 641 {
keir@16066 642 if ( op->status != 0 )
keir@16066 643 {
keir@16066 644 /*
keir@16066 645 * Suggests that __gntab_unmap_common failed in
keir@16066 646 * replace_grant_host_mapping() so nothing further to do
keir@16066 647 */
keir@16066 648 goto unmap_out;
keir@16066 649 }
keir@16066 650
keir@16407 651 if ( !is_iomem_page(op->frame) )
keir@16407 652 {
keir@17114 653 if ( gnttab_host_mapping_get_page_type(op, ld, rd) )
keir@16407 654 put_page_type(mfn_to_page(op->frame));
keir@16066 655 put_page(mfn_to_page(op->frame));
keir@16407 656 }
keir@16066 657 }
keir@16066 658
keir@16066 659 if ( (op->map->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0 )
keir@16066 660 {
keir@16066 661 op->map->flags = 0;
keir@16066 662 put_maptrack_handle(ld->grant_table, op->handle);
keir@16066 663 }
keir@16066 664
keir@16066 665 if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0) &&
keir@16066 666 !(op->flags & GNTMAP_readonly) )
keir@16066 667 gnttab_clear_flag(_GTF_writing, &sha->flags);
keir@16066 668
keir@16066 669 if ( act->pin == 0 )
keir@16066 670 gnttab_clear_flag(_GTF_reading, &sha->flags);
keir@16066 671
keir@16066 672 unmap_out:
keir@16066 673 spin_unlock(&rd->grant_table->lock);
keir@16066 674 rcu_unlock_domain(rd);
kfraser@15235 675 }
kfraser@15235 676
keir@16066 677 static void
keir@16066 678 __gnttab_unmap_grant_ref(
keir@16066 679 struct gnttab_unmap_grant_ref *op,
keir@16066 680 struct gnttab_unmap_common *common)
keir@16066 681 {
keir@16066 682 common->host_addr = op->host_addr;
keir@16066 683 common->dev_bus_addr = op->dev_bus_addr;
keir@16066 684 common->handle = op->handle;
keir@16066 685
keir@16066 686 /* Intialise these in case common contains old state */
keir@16066 687 common->new_addr = 0;
keir@16066 688 common->rd = NULL;
keir@16066 689
keir@16066 690 __gnttab_unmap_common(common);
keir@16066 691 op->status = common->status;
keir@16066 692 }
keir@16066 693
keir@16066 694
kaf24@2617 695 static long
kaf24@2617 696 gnttab_unmap_grant_ref(
kaf24@9873 697 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) uop, unsigned int count)
kaf24@2617 698 {
keir@16066 699 int i, c, partial_done, done = 0;
kaf24@8859 700 struct gnttab_unmap_grant_ref op;
keir@16066 701 struct gnttab_unmap_common common[GNTTAB_UNMAP_BATCH_SIZE];
cwc22@4016 702
keir@16068 703 while ( count != 0 )
keir@16068 704 {
keir@16066 705 c = min(count, (unsigned int)GNTTAB_UNMAP_BATCH_SIZE);
keir@16066 706 partial_done = 0;
keir@16066 707
keir@16066 708 for ( i = 0; i < c; i++ )
keir@16066 709 {
keir@16066 710 if ( unlikely(__copy_from_guest_offset(&op, uop, done+i, 1)) )
keir@16066 711 goto fault;
keir@16066 712 __gnttab_unmap_grant_ref(&op, &(common[i]));
keir@16066 713 ++partial_done;
keir@16066 714 if ( unlikely(__copy_to_guest_offset(uop, done+i, &op, 1)) )
keir@16066 715 goto fault;
keir@16066 716 }
keir@16066 717
keir@19651 718 flush_tlb_mask(&current->domain->domain_dirty_cpumask);
keir@16066 719
keir@16066 720 for ( i = 0; i < partial_done; i++ )
keir@16066 721 __gnttab_unmap_common_complete(&(common[i]));
keir@16066 722
keir@16066 723 count -= c;
keir@16066 724 done += c;
kaf24@8859 725 }
keir@16066 726
kaf24@8859 727 return 0;
cwc22@4063 728
kaf24@8859 729 fault:
keir@19651 730 flush_tlb_mask(&current->domain->domain_dirty_cpumask);
keir@16066 731
keir@16066 732 for ( i = 0; i < partial_done; i++ )
keir@16066 733 __gnttab_unmap_common_complete(&(common[i]));
keir@16066 734 return -EFAULT;
kaf24@2337 735 }
kaf24@2337 736
kfraser@15235 737 static void
kfraser@15235 738 __gnttab_unmap_and_replace(
keir@16066 739 struct gnttab_unmap_and_replace *op,
keir@16066 740 struct gnttab_unmap_common *common)
kfraser@15235 741 {
keir@16066 742 common->host_addr = op->host_addr;
keir@16066 743 common->new_addr = op->new_addr;
keir@16066 744 common->handle = op->handle;
keir@16066 745
keir@16066 746 /* Intialise these in case common contains old state */
keir@16066 747 common->dev_bus_addr = 0;
keir@16066 748 common->rd = NULL;
kfraser@15235 749
keir@16066 750 __gnttab_unmap_common(common);
keir@16066 751 op->status = common->status;
kfraser@15235 752 }
kfraser@15235 753
kfraser@15235 754 static long
kfraser@15235 755 gnttab_unmap_and_replace(
kfraser@15235 756 XEN_GUEST_HANDLE(gnttab_unmap_and_replace_t) uop, unsigned int count)
kfraser@15235 757 {
keir@16066 758 int i, c, partial_done, done = 0;
kfraser@15235 759 struct gnttab_unmap_and_replace op;
keir@16066 760 struct gnttab_unmap_common common[GNTTAB_UNMAP_BATCH_SIZE];
kfraser@15235 761
keir@16068 762 while ( count != 0 )
keir@16068 763 {
keir@16066 764 c = min(count, (unsigned int)GNTTAB_UNMAP_BATCH_SIZE);
keir@16066 765 partial_done = 0;
keir@16066 766
keir@16066 767 for ( i = 0; i < c; i++ )
keir@16066 768 {
keir@16066 769 if ( unlikely(__copy_from_guest_offset(&op, uop, done+i, 1)) )
keir@16066 770 goto fault;
keir@16066 771 __gnttab_unmap_and_replace(&op, &(common[i]));
keir@16066 772 ++partial_done;
keir@16066 773 if ( unlikely(__copy_to_guest_offset(uop, done+i, &op, 1)) )
keir@16066 774 goto fault;
keir@16066 775 }
keir@16066 776
keir@19651 777 flush_tlb_mask(&current->domain->domain_dirty_cpumask);
keir@16066 778
keir@16066 779 for ( i = 0; i < partial_done; i++ )
keir@16066 780 __gnttab_unmap_common_complete(&(common[i]));
keir@16066 781
keir@16066 782 count -= c;
keir@16066 783 done += c;
kfraser@15235 784 }
kfraser@15235 785
kfraser@15235 786 return 0;
kfraser@15235 787
kfraser@15235 788 fault:
keir@19651 789 flush_tlb_mask(&current->domain->domain_dirty_cpumask);
keir@16066 790
keir@16066 791 for ( i = 0; i < partial_done; i++ )
keir@16066 792 __gnttab_unmap_common_complete(&(common[i]));
kfraser@15235 793 return -EFAULT;
kfraser@15235 794 }
kfraser@15235 795
kfraser@13970 796 int
kfraser@13970 797 gnttab_grow_table(struct domain *d, unsigned int req_nr_frames)
kfraser@13970 798 {
kfraser@13970 799 /* d's grant table lock must be held by the caller */
kfraser@13970 800
kfraser@13970 801 struct grant_table *gt = d->grant_table;
kfraser@13970 802 unsigned int i;
kfraser@13970 803
kfraser@13970 804 ASSERT(req_nr_frames <= max_nr_grant_frames);
kfraser@13970 805
kfraser@13970 806 gdprintk(XENLOG_INFO,
kfraser@13970 807 "Expanding dom (%d) grant table from (%d) to (%d) frames.\n",
kfraser@13970 808 d->domain_id, nr_grant_frames(gt), req_nr_frames);
kfraser@13970 809
kfraser@13970 810 /* Active */
kfraser@13970 811 for ( i = nr_active_grant_frames(gt);
kfraser@13970 812 i < num_act_frames_from_sha_frames(req_nr_frames); i++ )
kfraser@13970 813 {
kfraser@13970 814 if ( (gt->active[i] = alloc_xenheap_page()) == NULL )
kfraser@13970 815 goto active_alloc_failed;
kfraser@15405 816 clear_page(gt->active[i]);
kfraser@13970 817 }
kfraser@13970 818
kfraser@13970 819 /* Shared */
kfraser@13970 820 for ( i = nr_grant_frames(gt); i < req_nr_frames; i++ )
kfraser@13970 821 {
kfraser@13970 822 if ( (gt->shared[i] = alloc_xenheap_page()) == NULL )
kfraser@13970 823 goto shared_alloc_failed;
kfraser@15405 824 clear_page(gt->shared[i]);
kfraser@13970 825 }
kfraser@13970 826
kfraser@13970 827 /* Share the new shared frames with the recipient domain */
kfraser@13970 828 for ( i = nr_grant_frames(gt); i < req_nr_frames; i++ )
kfraser@13970 829 gnttab_create_shared_page(d, gt, i);
kfraser@13970 830
kfraser@13970 831 gt->nr_grant_frames = req_nr_frames;
kfraser@13970 832
kfraser@13970 833 return 1;
kfraser@13970 834
kfraser@13970 835 shared_alloc_failed:
kfraser@13970 836 for ( i = nr_grant_frames(gt); i < req_nr_frames; i++ )
kfraser@13970 837 {
kfraser@13970 838 free_xenheap_page(gt->shared[i]);
kfraser@13970 839 gt->shared[i] = NULL;
kfraser@13970 840 }
kfraser@13970 841 active_alloc_failed:
kfraser@13970 842 for ( i = nr_active_grant_frames(gt);
kfraser@13970 843 i < num_act_frames_from_sha_frames(req_nr_frames); i++ )
kfraser@13970 844 {
kfraser@13970 845 free_xenheap_page(gt->active[i]);
kfraser@13970 846 gt->active[i] = NULL;
kfraser@13970 847 }
kfraser@13970 848 gdprintk(XENLOG_INFO, "Allocation failure when expanding grant table.\n");
kfraser@13970 849 return 0;
kfraser@13970 850 }
kfraser@13970 851
kaf24@2347 852 static long
kaf24@2347 853 gnttab_setup_table(
kaf24@9873 854 XEN_GUEST_HANDLE(gnttab_setup_table_t) uop, unsigned int count)
kaf24@2347 855 {
kaf24@8859 856 struct gnttab_setup_table op;
kaf24@8859 857 struct domain *d;
kaf24@8859 858 int i;
kaf24@8859 859 unsigned long gmfn;
kaf24@8859 860 domid_t dom;
kaf24@2347 861
kaf24@2617 862 if ( count != 1 )
kaf24@2617 863 return -EINVAL;
kaf24@2617 864
kaf24@9183 865 if ( unlikely(copy_from_guest(&op, uop, 1) != 0) )
kaf24@2347 866 {
kaf24@12038 867 gdprintk(XENLOG_INFO, "Fault while reading gnttab_setup_table_t.\n");
kaf24@2347 868 return -EFAULT;
kaf24@2347 869 }
kaf24@2347 870
kfraser@13970 871 if ( unlikely(op.nr_frames > max_nr_grant_frames) )
kaf24@2347 872 {
kaf24@12038 873 gdprintk(XENLOG_INFO, "Xen only supports up to %d grant-table frames"
kaf24@12032 874 " per domain.\n",
kfraser@13970 875 max_nr_grant_frames);
kaf24@8859 876 op.status = GNTST_general_error;
keir@17349 877 goto out1;
kaf24@2347 878 }
kaf24@2347 879
kaf24@8859 880 dom = op.dom;
kaf24@8859 881 if ( dom == DOMID_SELF )
kaf24@2617 882 {
keir@17349 883 d = rcu_lock_current_domain();
kaf24@2617 884 }
keir@17349 885 else
keir@17349 886 {
keir@16856 887 if ( unlikely((d = rcu_lock_domain_by_id(dom)) == NULL) )
keir@16856 888 {
keir@16856 889 gdprintk(XENLOG_INFO, "Bad domid %d.\n", dom);
keir@16856 890 op.status = GNTST_bad_domain;
keir@17349 891 goto out1;
keir@16856 892 }
keir@17349 893
keir@17349 894 if ( unlikely(!IS_PRIV_FOR(current->domain, d)) )
keir@17349 895 {
keir@16856 896 op.status = GNTST_permission_denied;
keir@17349 897 goto out2;
keir@16856 898 }
kaf24@2347 899 }
kaf24@2347 900
kfraser@15815 901 if ( xsm_grant_setup(current->domain, d) )
kfraser@15815 902 {
kfraser@15815 903 op.status = GNTST_permission_denied;
keir@17349 904 goto out2;
kfraser@15815 905 }
kfraser@15815 906
kfraser@13970 907 spin_lock(&d->grant_table->lock);
kfraser@13970 908
kfraser@13970 909 if ( (op.nr_frames > nr_grant_frames(d->grant_table)) &&
kfraser@13970 910 !gnttab_grow_table(d, op.nr_frames) )
kfraser@13970 911 {
kfraser@13970 912 gdprintk(XENLOG_INFO,
kfraser@13970 913 "Expand grant table to %d failed. Current: %d Max: %d.\n",
kfraser@13970 914 op.nr_frames,
kfraser@13970 915 nr_grant_frames(d->grant_table),
kfraser@13970 916 max_nr_grant_frames);
kfraser@13970 917 op.status = GNTST_general_error;
keir@17349 918 goto out3;
kfraser@13970 919 }
kfraser@13970 920
kaf24@10180 921 op.status = GNTST_okay;
kaf24@10180 922 for ( i = 0; i < op.nr_frames; i++ )
kaf24@2347 923 {
kaf24@10180 924 gmfn = gnttab_shared_gmfn(d, d->grant_table, i);
kaf24@10180 925 (void)copy_to_guest_offset(op.frame_list, i, &gmfn, 1);
kaf24@2347 926 }
kaf24@2347 927
keir@17349 928 out3:
kfraser@13970 929 spin_unlock(&d->grant_table->lock);
keir@17349 930 out2:
kfraser@14192 931 rcu_unlock_domain(d);
keir@17349 932 out1:
kaf24@9183 933 if ( unlikely(copy_to_guest(uop, &op, 1)) )
kaf24@8859 934 return -EFAULT;
cwc22@4461 935
cwc22@3985 936 return 0;
cwc22@3985 937 }
cwc22@3985 938
kfraser@13970 939 static long
kfraser@13970 940 gnttab_query_size(
kfraser@13970 941 XEN_GUEST_HANDLE(gnttab_query_size_t) uop, unsigned int count)
kfraser@13970 942 {
kfraser@13970 943 struct gnttab_query_size op;
kfraser@13970 944 struct domain *d;
kfraser@13970 945 domid_t dom;
kfraser@15815 946 int rc;
kfraser@13970 947
kfraser@13970 948 if ( count != 1 )
kfraser@13970 949 return -EINVAL;
kfraser@13970 950
kfraser@13970 951 if ( unlikely(copy_from_guest(&op, uop, 1) != 0) )
kfraser@13970 952 {
kfraser@13970 953 gdprintk(XENLOG_INFO, "Fault while reading gnttab_query_size_t.\n");
kfraser@13970 954 return -EFAULT;
kfraser@13970 955 }
kfraser@13970 956
kfraser@13970 957 dom = op.dom;
kfraser@13970 958 if ( dom == DOMID_SELF )
kfraser@13970 959 {
keir@17349 960 d = rcu_lock_current_domain();
kfraser@13970 961 }
keir@17349 962 else
keir@17349 963 {
keir@16856 964 if ( unlikely((d = rcu_lock_domain_by_id(dom)) == NULL) )
keir@16856 965 {
keir@16856 966 gdprintk(XENLOG_INFO, "Bad domid %d.\n", dom);
keir@16856 967 op.status = GNTST_bad_domain;
keir@16856 968 goto query_out;
keir@16856 969 }
keir@17349 970
keir@17349 971 if ( unlikely(!IS_PRIV_FOR(current->domain, d)) )
keir@17349 972 {
keir@16856 973 op.status = GNTST_permission_denied;
keir@16856 974 goto query_out_unlock;
keir@16856 975 }
kfraser@13970 976 }
kfraser@13970 977
kfraser@15815 978 rc = xsm_grant_query_size(current->domain, d);
kfraser@15815 979 if ( rc )
kfraser@15815 980 {
kfraser@15815 981 op.status = GNTST_permission_denied;
keir@16856 982 goto query_out_unlock;
kfraser@15815 983 }
kfraser@15815 984
kfraser@13970 985 spin_lock(&d->grant_table->lock);
kfraser@13970 986
kfraser@13970 987 op.nr_frames = nr_grant_frames(d->grant_table);
kfraser@13970 988 op.max_nr_frames = max_nr_grant_frames;
kfraser@13970 989 op.status = GNTST_okay;
kfraser@13970 990
kfraser@13970 991 spin_unlock(&d->grant_table->lock);
kfraser@13970 992
keir@16856 993
keir@16856 994 query_out_unlock:
kfraser@14192 995 rcu_unlock_domain(d);
kfraser@13970 996
kfraser@13970 997 query_out:
kfraser@13970 998 if ( unlikely(copy_to_guest(uop, &op, 1)) )
kfraser@13970 999 return -EFAULT;
kfraser@13970 1000
kfraser@13970 1001 return 0;
kfraser@13970 1002 }
kfraser@13970 1003
kaf24@8428 1004 /*
kaf24@8428 1005 * Check that the given grant reference (rd,ref) allows 'ld' to transfer
kaf24@8428 1006 * ownership of a page frame. If so, lock down the grant entry.
kaf24@8428 1007 */
kaf24@8428 1008 static int
kaf24@8428 1009 gnttab_prepare_for_transfer(
kaf24@8428 1010 struct domain *rd, struct domain *ld, grant_ref_t ref)
kaf24@8428 1011 {
kaf24@9723 1012 struct grant_table *rgt;
kaf24@9723 1013 struct grant_entry *sha;
kfraser@11437 1014 union grant_combo scombo, prev_scombo, new_scombo;
kaf24@9723 1015 int retries = 0;
kaf24@8428 1016
kfraser@13970 1017 if ( unlikely((rgt = rd->grant_table) == NULL) )
kaf24@8428 1018 {
kfraser@13970 1019 gdprintk(XENLOG_INFO, "Dom %d has no grant table.\n", rd->domain_id);
kaf24@8428 1020 return 0;
kaf24@8428 1021 }
kaf24@8428 1022
kaf24@8428 1023 spin_lock(&rgt->lock);
kaf24@8428 1024
kfraser@13970 1025 if ( unlikely(ref >= nr_grant_entries(rd->grant_table)) )
kfraser@13970 1026 {
kfraser@13970 1027 gdprintk(XENLOG_INFO,
kfraser@13970 1028 "Bad grant reference (%d) for transfer to domain(%d).\n",
kfraser@13970 1029 ref, rd->domain_id);
kfraser@13970 1030 goto fail;
kfraser@13970 1031 }
kfraser@13970 1032
kfraser@13970 1033 sha = &shared_entry(rgt, ref);
kaf24@8428 1034
kfraser@11439 1035 scombo.word = *(u32 *)&sha->flags;
kaf24@8428 1036
kaf24@8428 1037 for ( ; ; )
kaf24@8428 1038 {
kfraser@11437 1039 if ( unlikely(scombo.shorts.flags != GTF_accept_transfer) ||
kfraser@11437 1040 unlikely(scombo.shorts.domid != ld->domain_id) )
kaf24@8428 1041 {
kaf24@12038 1042 gdprintk(XENLOG_INFO, "Bad flags (%x) or dom (%d). "
kaf24@12032 1043 "(NB. expected dom %d)\n",
kfraser@11437 1044 scombo.shorts.flags, scombo.shorts.domid,
kfraser@11437 1045 ld->domain_id);
kaf24@8428 1046 goto fail;
kaf24@8428 1047 }
kaf24@8428 1048
kfraser@11437 1049 new_scombo = scombo;
kfraser@11437 1050 new_scombo.shorts.flags |= GTF_transfer_committed;
kaf24@8428 1051
kfraser@11106 1052 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
kfraser@11437 1053 scombo.word, new_scombo.word);
kfraser@11106 1054 if ( likely(prev_scombo.word == scombo.word) )
kaf24@8428 1055 break;
kaf24@8428 1056
kaf24@8428 1057 if ( retries++ == 4 )
kaf24@8428 1058 {
kaf24@12038 1059 gdprintk(XENLOG_WARNING, "Shared grant entry is unstable.\n");
kaf24@8428 1060 goto fail;
kaf24@8428 1061 }
kaf24@8428 1062
kfraser@11437 1063 scombo = prev_scombo;
kaf24@8428 1064 }
kaf24@8428 1065
kaf24@8428 1066 spin_unlock(&rgt->lock);
kaf24@8428 1067 return 1;
kaf24@8428 1068
kaf24@8428 1069 fail:
kaf24@8428 1070 spin_unlock(&rgt->lock);
kaf24@8428 1071 return 0;
kaf24@8428 1072 }
kaf24@8428 1073
vh249@5844 1074 static long
kaf24@7042 1075 gnttab_transfer(
kaf24@9873 1076 XEN_GUEST_HANDLE(gnttab_transfer_t) uop, unsigned int count)
vh249@5844 1077 {
vh249@5844 1078 struct domain *d = current->domain;
vh249@5844 1079 struct domain *e;
kaf24@8726 1080 struct page_info *page;
vh249@5844 1081 int i;
kaf24@6900 1082 grant_entry_t *sha;
kaf24@8859 1083 struct gnttab_transfer gop;
kaf24@8694 1084 unsigned long mfn;
keir@16549 1085 unsigned int max_bitsize;
vh249@5844 1086
kaf24@6703 1087 for ( i = 0; i < count; i++ )
kaf24@6703 1088 {
kaf24@7953 1089 /* Read from caller address space. */
kaf24@9183 1090 if ( unlikely(__copy_from_guest_offset(&gop, uop, i, 1)) )
kaf24@6704 1091 {
kaf24@12038 1092 gdprintk(XENLOG_INFO, "gnttab_transfer: error reading req %d/%d\n",
kaf24@12032 1093 i, count);
kaf24@8859 1094 return -EFAULT;
vh249@5844 1095 }
kaf24@6704 1096
sos22@9581 1097 mfn = gmfn_to_mfn(d, gop.mfn);
sos22@9581 1098
kaf24@7953 1099 /* Check the passed page frame for basic validity. */
sos22@9581 1100 if ( unlikely(!mfn_valid(mfn)) )
vhanquez@8657 1101 {
kaf24@12038 1102 gdprintk(XENLOG_INFO, "gnttab_transfer: out-of-range %lx\n",
vhanquez@8657 1103 (unsigned long)gop.mfn);
kaf24@8859 1104 gop.status = GNTST_bad_page;
kaf24@8859 1105 goto copyback;
vhanquez@8657 1106 }
vhanquez@8657 1107
kaf24@8726 1108 page = mfn_to_page(mfn);
keir@16353 1109 if ( unlikely(is_xen_heap_page(page)) )
kaf24@7953 1110 {
kaf24@12038 1111 gdprintk(XENLOG_INFO, "gnttab_transfer: xen frame %lx\n",
kaf24@7953 1112 (unsigned long)gop.mfn);
kaf24@8859 1113 gop.status = GNTST_bad_page;
kaf24@8859 1114 goto copyback;
vh249@5844 1115 }
vh249@5844 1116
kfraser@10418 1117 if ( steal_page(d, page, 0) < 0 )
kaf24@8001 1118 {
kaf24@8859 1119 gop.status = GNTST_bad_page;
kaf24@8859 1120 goto copyback;
kaf24@8001 1121 }
vh249@5844 1122
keir@17756 1123 #ifndef __ia64__ /* IA64 implicitly replaces the old page in steal_page(). */
keir@17723 1124 guest_physmap_remove_page(d, gop.mfn, mfn, 0);
keir@17756 1125 #endif
keir@19651 1126 flush_tlb_mask(&d->domain_dirty_cpumask);
keir@17723 1127
kaf24@7967 1128 /* Find the target domain. */
kfraser@14192 1129 if ( unlikely((e = rcu_lock_domain_by_id(gop.domid)) == NULL) )
kaf24@7967 1130 {
kaf24@12038 1131 gdprintk(XENLOG_INFO, "gnttab_transfer: can't find domain %d\n",
kaf24@12032 1132 gop.domid);
kaf24@7967 1133 page->count_info &= ~(PGC_count_mask|PGC_allocated);
kaf24@7967 1134 free_domheap_page(page);
kaf24@8859 1135 gop.status = GNTST_bad_domain;
kaf24@8859 1136 goto copyback;
kaf24@7967 1137 }
kaf24@7967 1138
kfraser@15815 1139 if ( xsm_grant_transfer(d, e) )
kfraser@15815 1140 {
keir@16549 1141 gop.status = GNTST_permission_denied;
keir@16548 1142 unlock_and_copyback:
kfraser@15815 1143 rcu_unlock_domain(e);
keir@16548 1144 page->count_info &= ~(PGC_count_mask|PGC_allocated);
keir@16548 1145 free_domheap_page(page);
kfraser@15815 1146 goto copyback;
kfraser@15815 1147 }
kfraser@15815 1148
keir@16549 1149 max_bitsize = domain_clamp_alloc_bitsize(
keir@16549 1150 e, BITS_PER_LONG+PAGE_SHIFT-1);
keir@16549 1151 if ( (1UL << (max_bitsize - PAGE_SHIFT)) <= mfn )
keir@16548 1152 {
keir@16548 1153 struct page_info *new_page;
keir@16548 1154 void *sp, *dp;
keir@16548 1155
keir@17385 1156 new_page = alloc_domheap_page(NULL, MEMF_bits(max_bitsize));
keir@16548 1157 if ( new_page == NULL )
keir@16549 1158 {
keir@16549 1159 gop.status = GNTST_address_too_big;
keir@16548 1160 goto unlock_and_copyback;
keir@16549 1161 }
keir@16548 1162
keir@16548 1163 sp = map_domain_page(mfn);
keir@16548 1164 dp = map_domain_page(page_to_mfn(new_page));
keir@16548 1165 memcpy(dp, sp, PAGE_SIZE);
keir@16548 1166 unmap_domain_page(dp);
keir@16548 1167 unmap_domain_page(sp);
keir@16548 1168
keir@16548 1169 page->count_info &= ~(PGC_count_mask|PGC_allocated);
keir@16548 1170 free_domheap_page(page);
keir@16548 1171 page = new_page;
keir@16548 1172 }
keir@16548 1173
vh249@5844 1174 spin_lock(&e->page_alloc_lock);
vh249@5844 1175
vh249@5844 1176 /*
vh249@5844 1177 * Check that 'e' will accept the page and has reservation
vh249@5844 1178 * headroom. Also, a domain mustn't have PGC_allocated
vh249@5844 1179 * pages when it is dying.
vh249@5844 1180 */
kfraser@14642 1181 if ( unlikely(e->is_dying) ||
kaf24@6704 1182 unlikely(e->tot_pages >= e->max_pages) ||
kaf24@7953 1183 unlikely(!gnttab_prepare_for_transfer(e, d, gop.ref)) )
kaf24@6704 1184 {
kfraser@14642 1185 if ( !e->is_dying )
kaf24@12038 1186 gdprintk(XENLOG_INFO, "gnttab_transfer: "
kaf24@12032 1187 "Transferee has no reservation "
kaf24@8041 1188 "headroom (%d,%d) or provided a bad grant ref (%08x) "
kfraser@14642 1189 "or is dying (%d)\n",
kfraser@14642 1190 e->tot_pages, e->max_pages, gop.ref, e->is_dying);
vh249@5844 1191 spin_unlock(&e->page_alloc_lock);
kfraser@14192 1192 rcu_unlock_domain(e);
kaf24@7967 1193 page->count_info &= ~(PGC_count_mask|PGC_allocated);
kaf24@7967 1194 free_domheap_page(page);
kaf24@8859 1195 gop.status = GNTST_general_error;
kaf24@8859 1196 goto copyback;
vh249@5844 1197 }
kaf24@6703 1198
vh249@5844 1199 /* Okay, add the page to 'e'. */
kaf24@6704 1200 if ( unlikely(e->tot_pages++ == 0) )
vh249@5844 1201 get_knownalive_domain(e);
keir@19134 1202 page_list_add_tail(page, &e->page_list);
vh249@5844 1203 page_set_owner(page, e);
kaf24@7953 1204
vh249@5844 1205 spin_unlock(&e->page_alloc_lock);
kaf24@7818 1206
kaf24@7818 1207 TRACE_1D(TRC_MEM_PAGE_GRANT_TRANSFER, e->domain_id);
kaf24@7953 1208
kaf24@6900 1209 /* Tell the guest about its new page frame. */
kfraser@13970 1210 spin_lock(&e->grant_table->lock);
kfraser@13970 1211
kfraser@13970 1212 sha = &shared_entry(e->grant_table, gop.ref);
keir@17664 1213 guest_physmap_add_page(e, sha->frame, mfn, 0);
kaf24@8694 1214 sha->frame = mfn;
kaf24@6900 1215 wmb();
kaf24@6900 1216 sha->flags |= GTF_transfer_completed;
kaf24@7953 1217
kfraser@13970 1218 spin_unlock(&e->grant_table->lock);
kfraser@13970 1219
kfraser@14192 1220 rcu_unlock_domain(e);
kaf24@7953 1221
kaf24@8859 1222 gop.status = GNTST_okay;
kaf24@8859 1223
kaf24@8859 1224 copyback:
kaf24@9183 1225 if ( unlikely(__copy_to_guest_offset(uop, i, &gop, 1)) )
kaf24@8859 1226 {
kfraser@13970 1227 gdprintk(XENLOG_INFO, "gnttab_transfer: error writing resp "
kfraser@13970 1228 "%d/%d\n", i, count);
kaf24@8859 1229 return -EFAULT;
kaf24@8859 1230 }
vh249@5844 1231 }
kaf24@6704 1232
kaf24@7953 1233 return 0;
vh249@5844 1234 }
vh249@5844 1235
kfraser@11079 1236 /* Undo __acquire_grant_for_copy. Again, this has no effect on page
kfraser@11079 1237 type and reference counts. */
kfraser@11079 1238 static void
kfraser@11079 1239 __release_grant_for_copy(
kfraser@11079 1240 struct domain *rd, unsigned long gref, int readonly)
kfraser@11079 1241 {
kfraser@13970 1242 grant_entry_t *sha;
kfraser@13970 1243 struct active_grant_entry *act;
kfraser@13970 1244 unsigned long r_frame;
kfraser@11079 1245
kfraser@11079 1246 spin_lock(&rd->grant_table->lock);
kfraser@11437 1247
kfraser@13970 1248 act = &active_entry(rd->grant_table, gref);
kfraser@13970 1249 sha = &shared_entry(rd->grant_table, gref);
kfraser@13970 1250 r_frame = act->frame;
kfraser@13970 1251
kfraser@11079 1252 if ( readonly )
kfraser@11437 1253 {
kfraser@11079 1254 act->pin -= GNTPIN_hstr_inc;
kfraser@11437 1255 }
kfraser@11079 1256 else
kfraser@11437 1257 {
kfraser@13970 1258 gnttab_mark_dirty(rd, r_frame);
kfraser@13970 1259
kfraser@11079 1260 act->pin -= GNTPIN_hstw_inc;
kfraser@11437 1261 if ( !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) )
kfraser@11437 1262 gnttab_clear_flag(_GTF_writing, &sha->flags);
kfraser@11437 1263 }
kfraser@11079 1264
kfraser@11079 1265 if ( !act->pin )
kaf24@11093 1266 gnttab_clear_flag(_GTF_reading, &sha->flags);
kfraser@11437 1267
kfraser@11079 1268 spin_unlock(&rd->grant_table->lock);
kfraser@11079 1269 }
kfraser@11079 1270
kfraser@11079 1271 /* Grab a frame number from a grant entry and update the flags and pin
kfraser@11079 1272 count as appropriate. Note that this does *not* update the page
ssmith@12223 1273 type or reference counts, and does not check that the mfn is
ssmith@12223 1274 actually valid. */
kfraser@11079 1275 static int
kfraser@11079 1276 __acquire_grant_for_copy(
kfraser@11079 1277 struct domain *rd, unsigned long gref, int readonly,
kfraser@11079 1278 unsigned long *frame)
kfraser@11079 1279 {
kfraser@11079 1280 grant_entry_t *sha;
kfraser@11079 1281 struct active_grant_entry *act;
kfraser@11079 1282 s16 rc = GNTST_okay;
kfraser@11079 1283 int retries = 0;
kfraser@11437 1284 union grant_combo scombo, prev_scombo, new_scombo;
kfraser@11079 1285
kfraser@13970 1286 spin_lock(&rd->grant_table->lock);
kfraser@13970 1287
kfraser@13970 1288 if ( unlikely(gref >= nr_grant_entries(rd->grant_table)) )
kfraser@13970 1289 PIN_FAIL(unlock_out, GNTST_bad_gntref,
kfraser@11079 1290 "Bad grant reference %ld\n", gref);
kfraser@11079 1291
kfraser@13970 1292 act = &active_entry(rd->grant_table, gref);
kfraser@13970 1293 sha = &shared_entry(rd->grant_table, gref);
kfraser@11079 1294
kfraser@11437 1295 /* If already pinned, check the active domid and avoid refcnt overflow. */
kfraser@11437 1296 if ( act->pin &&
kfraser@11437 1297 ((act->domid != current->domain->domain_id) ||
kfraser@11437 1298 (act->pin & 0x80808080U) != 0) )
kfraser@11437 1299 PIN_FAIL(unlock_out, GNTST_general_error,
kfraser@11437 1300 "Bad domain (%d != %d), or risk of counter overflow %08x\n",
kfraser@11437 1301 act->domid, current->domain->domain_id, act->pin);
ssmith@11427 1302
kfraser@11079 1303 if ( !act->pin ||
kfraser@11437 1304 (!readonly && !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask))) )
kfraser@11079 1305 {
kfraser@11439 1306 scombo.word = *(u32 *)&sha->flags;
kfraser@11079 1307
kfraser@11079 1308 for ( ; ; )
kfraser@11079 1309 {
kfraser@11437 1310 /* If not already pinned, check the grant domid and type. */
kfraser@11437 1311 if ( !act->pin &&
kfraser@11437 1312 (((scombo.shorts.flags & GTF_type_mask) !=
kfraser@11437 1313 GTF_permit_access) ||
kfraser@11437 1314 (scombo.shorts.domid != current->domain->domain_id)) )
kfraser@11437 1315 PIN_FAIL(unlock_out, GNTST_general_error,
kfraser@11437 1316 "Bad flags (%x) or dom (%d). (expected dom %d)\n",
kfraser@11437 1317 scombo.shorts.flags, scombo.shorts.domid,
kfraser@11437 1318 current->domain->domain_id);
kfraser@11079 1319
kfraser@11106 1320 new_scombo = scombo;
kfraser@11106 1321 new_scombo.shorts.flags |= GTF_reading;
kfraser@11106 1322
kfraser@11079 1323 if ( !readonly )
kfraser@11079 1324 {
kfraser@11106 1325 new_scombo.shorts.flags |= GTF_writing;
kfraser@11437 1326 if ( unlikely(scombo.shorts.flags & GTF_readonly) )
kfraser@11079 1327 PIN_FAIL(unlock_out, GNTST_general_error,
kfraser@11079 1328 "Attempt to write-pin a r/o grant entry.\n");
kfraser@11079 1329 }
kfraser@11437 1330
kfraser@11106 1331 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
kfraser@11106 1332 scombo.word, new_scombo.word);
kfraser@11106 1333 if ( likely(prev_scombo.word == scombo.word) )
kfraser@11079 1334 break;
kfraser@11079 1335
kfraser@11079 1336 if ( retries++ == 4 )
kfraser@11079 1337 PIN_FAIL(unlock_out, GNTST_general_error,
kfraser@11079 1338 "Shared grant entry is unstable.\n");
kfraser@11437 1339
kfraser@11437 1340 scombo = prev_scombo;
kfraser@11079 1341 }
kfraser@11079 1342
kfraser@11079 1343 if ( !act->pin )
kfraser@11079 1344 {
kfraser@11437 1345 act->domid = scombo.shorts.domid;
keir@18973 1346 act->gfn = sha->frame;
kfraser@11079 1347 act->frame = gmfn_to_mfn(rd, sha->frame);
kfraser@11079 1348 }
kfraser@11079 1349 }
kfraser@11079 1350
kfraser@11079 1351 act->pin += readonly ? GNTPIN_hstr_inc : GNTPIN_hstw_inc;
kfraser@11079 1352
kfraser@11079 1353 *frame = act->frame;
kfraser@11079 1354
kfraser@11079 1355 unlock_out:
kfraser@11079 1356 spin_unlock(&rd->grant_table->lock);
kfraser@11079 1357 return rc;
kfraser@11079 1358 }
kfraser@11079 1359
kfraser@11079 1360 static void
kfraser@11079 1361 __gnttab_copy(
kfraser@11079 1362 struct gnttab_copy *op)
kfraser@11079 1363 {
kfraser@11079 1364 struct domain *sd = NULL, *dd = NULL;
kfraser@11079 1365 unsigned long s_frame, d_frame;
kaf24@11094 1366 char *sp, *dp;
kfraser@11079 1367 s16 rc = GNTST_okay;
ssmith@11096 1368 int have_d_grant = 0, have_s_grant = 0, have_s_ref = 0;
ssmith@11095 1369 int src_is_gref, dest_is_gref;
kfraser@11079 1370
kfraser@11079 1371 if ( ((op->source.offset + op->len) > PAGE_SIZE) ||
kfraser@11079 1372 ((op->dest.offset + op->len) > PAGE_SIZE) )
kfraser@11079 1373 PIN_FAIL(error_out, GNTST_bad_copy_arg, "copy beyond page area.\n");
kfraser@11079 1374
ssmith@11095 1375 src_is_gref = op->flags & GNTCOPY_source_gref;
ssmith@11095 1376 dest_is_gref = op->flags & GNTCOPY_dest_gref;
ssmith@11095 1377
ssmith@11095 1378 if ( (op->source.domid != DOMID_SELF && !src_is_gref ) ||
ssmith@11095 1379 (op->dest.domid != DOMID_SELF && !dest_is_gref) )
ssmith@11095 1380 PIN_FAIL(error_out, GNTST_permission_denied,
ssmith@11095 1381 "only allow copy-by-mfn for DOMID_SELF.\n");
ssmith@11095 1382
kfraser@11079 1383 if ( op->source.domid == DOMID_SELF )
kfraser@14196 1384 sd = rcu_lock_current_domain();
kfraser@14192 1385 else if ( (sd = rcu_lock_domain_by_id(op->source.domid)) == NULL )
kfraser@11079 1386 PIN_FAIL(error_out, GNTST_bad_domain,
kfraser@11079 1387 "couldn't find %d\n", op->source.domid);
kfraser@11079 1388
kfraser@11079 1389 if ( op->dest.domid == DOMID_SELF )
kfraser@14196 1390 dd = rcu_lock_current_domain();
kfraser@14192 1391 else if ( (dd = rcu_lock_domain_by_id(op->dest.domid)) == NULL )
kfraser@11079 1392 PIN_FAIL(error_out, GNTST_bad_domain,
kfraser@11079 1393 "couldn't find %d\n", op->dest.domid);
kfraser@11079 1394
kfraser@15815 1395 rc = xsm_grant_copy(sd, dd);
kfraser@15815 1396 if ( rc )
kfraser@15815 1397 {
kfraser@15815 1398 rc = GNTST_permission_denied;
kfraser@15815 1399 goto error_out;
kfraser@15815 1400 }
kfraser@15815 1401
ssmith@11095 1402 if ( src_is_gref )
kfraser@11079 1403 {
kfraser@11079 1404 rc = __acquire_grant_for_copy(sd, op->source.u.ref, 1, &s_frame);
kfraser@11079 1405 if ( rc != GNTST_okay )
kfraser@11079 1406 goto error_out;
kfraser@11079 1407 have_s_grant = 1;
kfraser@11079 1408 }
kfraser@11079 1409 else
kfraser@11079 1410 {
kfraser@11079 1411 s_frame = gmfn_to_mfn(sd, op->source.u.gmfn);
kfraser@11079 1412 }
ssmith@12223 1413 if ( unlikely(!mfn_valid(s_frame)) )
ssmith@12223 1414 PIN_FAIL(error_out, GNTST_general_error,
ssmith@12223 1415 "source frame %lx invalid.\n", s_frame);
kfraser@11079 1416 if ( !get_page(mfn_to_page(s_frame), sd) )
kfraser@12321 1417 {
kfraser@14642 1418 if ( !sd->is_dying )
kfraser@12321 1419 gdprintk(XENLOG_WARNING, "Could not get src frame %lx\n", s_frame);
kfraser@12321 1420 rc = GNTST_general_error;
kfraser@12321 1421 goto error_out;
kfraser@12321 1422 }
ssmith@11096 1423 have_s_ref = 1;
kfraser@11079 1424
ssmith@11095 1425 if ( dest_is_gref )
kfraser@11079 1426 {
kfraser@11079 1427 rc = __acquire_grant_for_copy(dd, op->dest.u.ref, 0, &d_frame);
kfraser@11079 1428 if ( rc != GNTST_okay )
kfraser@11079 1429 goto error_out;
kfraser@11079 1430 have_d_grant = 1;
kfraser@11079 1431 }
kfraser@11079 1432 else
kfraser@11079 1433 {
ssmith@12223 1434 d_frame = gmfn_to_mfn(dd, op->dest.u.gmfn);
kfraser@11079 1435 }
ssmith@12223 1436 if ( unlikely(!mfn_valid(d_frame)) )
ssmith@12223 1437 PIN_FAIL(error_out, GNTST_general_error,
ssmith@12223 1438 "destination frame %lx invalid.\n", d_frame);
kfraser@11079 1439 if ( !get_page_and_type(mfn_to_page(d_frame), dd, PGT_writable_page) )
kfraser@12321 1440 {
kfraser@14642 1441 if ( !dd->is_dying )
kfraser@12321 1442 gdprintk(XENLOG_WARNING, "Could not get dst frame %lx\n", d_frame);
kfraser@12321 1443 rc = GNTST_general_error;
kfraser@12321 1444 goto error_out;
kfraser@12321 1445 }
kfraser@11079 1446
kfraser@11079 1447 sp = map_domain_page(s_frame);
kfraser@11079 1448 dp = map_domain_page(d_frame);
kfraser@11079 1449
kfraser@11079 1450 memcpy(dp + op->dest.offset, sp + op->source.offset, op->len);
kfraser@11079 1451
kfraser@11079 1452 unmap_domain_page(dp);
kfraser@11079 1453 unmap_domain_page(sp);
kfraser@11079 1454
ssmith@12223 1455 gnttab_mark_dirty(dd, d_frame);
ssmith@12223 1456
ssmith@11096 1457 put_page_and_type(mfn_to_page(d_frame));
kfraser@11079 1458 error_out:
ssmith@11096 1459 if ( have_s_ref )
ssmith@11096 1460 put_page(mfn_to_page(s_frame));
kfraser@11079 1461 if ( have_s_grant )
kfraser@11079 1462 __release_grant_for_copy(sd, op->source.u.ref, 1);
kfraser@11079 1463 if ( have_d_grant )
kfraser@11079 1464 __release_grant_for_copy(dd, op->dest.u.ref, 0);
kfraser@11079 1465 if ( sd )
kfraser@14192 1466 rcu_unlock_domain(sd);
kfraser@11079 1467 if ( dd )
kfraser@14192 1468 rcu_unlock_domain(dd);
kfraser@11079 1469 op->status = rc;
kfraser@11079 1470 }
kfraser@11079 1471
kfraser@11079 1472 static long
kfraser@11079 1473 gnttab_copy(
kfraser@11079 1474 XEN_GUEST_HANDLE(gnttab_copy_t) uop, unsigned int count)
kfraser@11079 1475 {
kfraser@11079 1476 int i;
kfraser@11079 1477 struct gnttab_copy op;
kfraser@11079 1478
kfraser@11079 1479 for ( i = 0; i < count; i++ )
kfraser@11079 1480 {
kfraser@11079 1481 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
kfraser@11079 1482 return -EFAULT;
kfraser@11079 1483 __gnttab_copy(&op);
kfraser@11079 1484 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
kfraser@11079 1485 return -EFAULT;
kfraser@11079 1486 }
kfraser@11079 1487 return 0;
kfraser@11079 1488 }
kfraser@11079 1489
kaf24@9183 1490 long
kaf24@2340 1491 do_grant_table_op(
kaf24@9873 1492 unsigned int cmd, XEN_GUEST_HANDLE(void) uop, unsigned int count)
kaf24@2337 1493 {
kaf24@2337 1494 long rc;
kaf24@6165 1495 struct domain *d = current->domain;
smh22@6176 1496
kaf24@2617 1497 if ( count > 512 )
kaf24@2617 1498 return -EINVAL;
smh22@6176 1499
keir@17449 1500 domain_lock(d);
smh22@6176 1501
vh249@4523 1502 rc = -EFAULT;
kaf24@2337 1503 switch ( cmd )
kaf24@6703 1504 {
kaf24@6703 1505 case GNTTABOP_map_grant_ref:
kaf24@9183 1506 {
kaf24@9873 1507 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) map =
kaf24@9183 1508 guest_handle_cast(uop, gnttab_map_grant_ref_t);
kaf24@9183 1509 if ( unlikely(!guest_handle_okay(map, count)) )
kaf24@6703 1510 goto out;
kaf24@9183 1511 rc = gnttab_map_grant_ref(map, count);
kaf24@6703 1512 break;
kaf24@9183 1513 }
kaf24@9183 1514 case GNTTABOP_unmap_grant_ref:
kaf24@9183 1515 {
kaf24@9873 1516 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) unmap =
kaf24@9183 1517 guest_handle_cast(uop, gnttab_unmap_grant_ref_t);
kaf24@9183 1518 if ( unlikely(!guest_handle_okay(unmap, count)) )
kaf24@9183 1519 goto out;
kaf24@9183 1520 rc = gnttab_unmap_grant_ref(unmap, count);
kaf24@9183 1521 break;
kaf24@9183 1522 }
kfraser@15235 1523 case GNTTABOP_unmap_and_replace:
kfraser@15235 1524 {
kfraser@15235 1525 XEN_GUEST_HANDLE(gnttab_unmap_and_replace_t) unmap =
kfraser@15235 1526 guest_handle_cast(uop, gnttab_unmap_and_replace_t);
kfraser@15235 1527 if ( unlikely(!guest_handle_okay(unmap, count)) )
kfraser@15235 1528 goto out;
kfraser@15235 1529 rc = -ENOSYS;
kfraser@15235 1530 if ( unlikely(!replace_grant_supported()) )
kfraser@15235 1531 goto out;
kfraser@15235 1532 rc = gnttab_unmap_and_replace(unmap, count);
kfraser@15235 1533 break;
kfraser@15235 1534 }
kaf24@9183 1535 case GNTTABOP_setup_table:
kaf24@9183 1536 {
kaf24@9183 1537 rc = gnttab_setup_table(
kaf24@9183 1538 guest_handle_cast(uop, gnttab_setup_table_t), count);
kaf24@9183 1539 break;
kaf24@9183 1540 }
kaf24@6900 1541 case GNTTABOP_transfer:
kaf24@9183 1542 {
kaf24@9873 1543 XEN_GUEST_HANDLE(gnttab_transfer_t) transfer =
kaf24@9183 1544 guest_handle_cast(uop, gnttab_transfer_t);
kaf24@9183 1545 if ( unlikely(!guest_handle_okay(transfer, count)) )
kaf24@6703 1546 goto out;
kaf24@9183 1547 rc = gnttab_transfer(transfer, count);
kaf24@6703 1548 break;
kaf24@9183 1549 }
kfraser@11079 1550 case GNTTABOP_copy:
kfraser@11079 1551 {
kfraser@11079 1552 XEN_GUEST_HANDLE(gnttab_copy_t) copy =
kfraser@11079 1553 guest_handle_cast(uop, gnttab_copy_t);
kfraser@11079 1554 if ( unlikely(!guest_handle_okay(copy, count)) )
kfraser@11079 1555 goto out;
kfraser@11079 1556 rc = gnttab_copy(copy, count);
kfraser@11079 1557 break;
kfraser@11079 1558 }
kfraser@13970 1559 case GNTTABOP_query_size:
kfraser@13970 1560 {
kfraser@13970 1561 rc = gnttab_query_size(
kfraser@13970 1562 guest_handle_cast(uop, gnttab_query_size_t), count);
kfraser@13970 1563 break;
kfraser@13970 1564 }
kaf24@6703 1565 default:
kaf24@6703 1566 rc = -ENOSYS;
kaf24@6703 1567 break;
kaf24@6703 1568 }
smh22@6176 1569
smh22@6176 1570 out:
keir@17449 1571 domain_unlock(d);
smh22@6176 1572
kaf24@2337 1573 return rc;
kaf24@2337 1574 }
kaf24@2322 1575
ack@13303 1576 #ifdef CONFIG_COMPAT
ack@13303 1577 #include "compat/grant_table.c"
ack@13303 1578 #endif
ack@13303 1579
kfraser@13970 1580 static unsigned int max_nr_active_grant_frames(void)
kfraser@13970 1581 {
kfraser@13970 1582 return (((max_nr_grant_frames * (PAGE_SIZE / sizeof(grant_entry_t))) +
kfraser@13970 1583 ((PAGE_SIZE / sizeof(struct active_grant_entry))-1))
kfraser@13970 1584 / (PAGE_SIZE / sizeof(struct active_grant_entry)));
kfraser@13970 1585 }
kfraser@13970 1586
kaf24@2347 1587 int
kaf24@2340 1588 grant_table_create(
kaf24@2340 1589 struct domain *d)
kaf24@2322 1590 {
kaf24@9723 1591 struct grant_table *t;
kaf24@9723 1592 int i;
kaf24@2322 1593
kfraser@13970 1594 /* If this sizeof assertion fails, fix the function: shared_index */
kfraser@13970 1595 ASSERT(sizeof(grant_entry_t) == 8);
kfraser@13970 1596
kaf24@9723 1597 if ( (t = xmalloc(struct grant_table)) == NULL )
kfraser@13970 1598 goto no_mem_0;
kaf24@2322 1599
kaf24@2322 1600 /* Simple stuff. */
kaf24@2617 1601 memset(t, 0, sizeof(*t));
kaf24@2322 1602 spin_lock_init(&t->lock);
kfraser@13970 1603 t->nr_grant_frames = INITIAL_NR_GRANT_FRAMES;
kaf24@2322 1604
kaf24@2710 1605 /* Active grant table. */
kfraser@13970 1606 if ( (t->active = xmalloc_array(struct active_grant_entry *,
kfraser@13970 1607 max_nr_active_grant_frames())) == NULL )
kfraser@13970 1608 goto no_mem_1;
kfraser@13970 1609 memset(t->active, 0, max_nr_active_grant_frames() * sizeof(t->active[0]));
kfraser@13970 1610 for ( i = 0;
kfraser@13970 1611 i < num_act_frames_from_sha_frames(INITIAL_NR_GRANT_FRAMES); i++ )
kfraser@13970 1612 {
kfraser@13970 1613 if ( (t->active[i] = alloc_xenheap_page()) == NULL )
kfraser@13970 1614 goto no_mem_2;
kfraser@15405 1615 clear_page(t->active[i]);
kfraser@13970 1616 }
kaf24@2322 1617
cwc22@4461 1618 /* Tracking of mapped foreign frames table */
kfraser@13970 1619 if ( (t->maptrack = xmalloc_array(struct grant_mapping *,
kfraser@13970 1620 max_nr_maptrack_frames())) == NULL )
kfraser@13970 1621 goto no_mem_2;
kfraser@13970 1622 memset(t->maptrack, 0, max_nr_maptrack_frames() * sizeof(t->maptrack[0]));
kfraser@13970 1623 if ( (t->maptrack[0] = alloc_xenheap_page()) == NULL )
kfraser@13970 1624 goto no_mem_3;
kfraser@15405 1625 clear_page(t->maptrack[0]);
kaf24@9723 1626 t->maptrack_limit = PAGE_SIZE / sizeof(struct grant_mapping);
cwc22@4463 1627 for ( i = 0; i < t->maptrack_limit; i++ )
kfraser@13970 1628 t->maptrack[0][i].ref = i+1;
kaf24@2617 1629
kaf24@2710 1630 /* Shared grant table. */
kfraser@13970 1631 if ( (t->shared = xmalloc_array(struct grant_entry *,
kfraser@13970 1632 max_nr_grant_frames)) == NULL )
kfraser@13970 1633 goto no_mem_3;
kfraser@13970 1634 memset(t->shared, 0, max_nr_grant_frames * sizeof(t->shared[0]));
kfraser@13970 1635 for ( i = 0; i < INITIAL_NR_GRANT_FRAMES; i++ )
kfraser@13970 1636 {
kfraser@13970 1637 if ( (t->shared[i] = alloc_xenheap_page()) == NULL )
kfraser@13970 1638 goto no_mem_4;
kfraser@15405 1639 clear_page(t->shared[i]);
kfraser@13970 1640 }
cwc22@4461 1641
kfraser@13970 1642 for ( i = 0; i < INITIAL_NR_GRANT_FRAMES; i++ )
kaf24@8726 1643 gnttab_create_shared_page(d, t, i);
kaf24@2322 1644
kaf24@2322 1645 /* Okay, install the structure. */
kaf24@2322 1646 d->grant_table = t;
kaf24@2322 1647 return 0;
kaf24@2322 1648
kfraser@13970 1649 no_mem_4:
kfraser@13970 1650 for ( i = 0; i < INITIAL_NR_GRANT_FRAMES; i++ )
kfraser@13970 1651 free_xenheap_page(t->shared[i]);
kfraser@13970 1652 xfree(t->shared);
kfraser@13970 1653 no_mem_3:
kfraser@13970 1654 free_xenheap_page(t->maptrack[0]);
kfraser@13970 1655 xfree(t->maptrack);
kfraser@13970 1656 no_mem_2:
kfraser@13970 1657 for ( i = 0;
kfraser@13970 1658 i < num_act_frames_from_sha_frames(INITIAL_NR_GRANT_FRAMES); i++ )
kfraser@13970 1659 free_xenheap_page(t->active[i]);
kfraser@13970 1660 xfree(t->active);
kfraser@13970 1661 no_mem_1:
kfraser@13970 1662 xfree(t);
kfraser@13970 1663 no_mem_0:
kaf24@2322 1664 return -ENOMEM;
kaf24@2322 1665 }
kaf24@2322 1666
kaf24@2340 1667 void
kaf24@7984 1668 gnttab_release_mappings(
kaf24@7984 1669 struct domain *d)
cwc22@4100 1670 {
kaf24@9723 1671 struct grant_table *gt = d->grant_table;
kaf24@9723 1672 struct grant_mapping *map;
kaf24@7984 1673 grant_ref_t ref;
kaf24@8137 1674 grant_handle_t handle;
kaf24@7984 1675 struct domain *rd;
kaf24@9723 1676 struct active_grant_entry *act;
kaf24@9723 1677 struct grant_entry *sha;
cwc22@4100 1678
kfraser@14642 1679 BUG_ON(!d->is_dying);
cwc22@4100 1680
cwc22@4463 1681 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
cwc22@4100 1682 {
kfraser@13970 1683 map = &maptrack_entry(gt, handle);
kaf24@9722 1684 if ( !(map->flags & (GNTMAP_device_map|GNTMAP_host_map)) )
kaf24@6703 1685 continue;
cwc22@4100 1686
kaf24@9722 1687 ref = map->ref;
cwc22@4100 1688
kaf24@12038 1689 gdprintk(XENLOG_INFO, "Grant release (%hu) ref:(%hu) "
kaf24@12032 1690 "flags:(%x) dom:(%hu)\n",
kaf24@9722 1691 handle, ref, map->flags, map->domid);
kaf24@6703 1692
kfraser@14192 1693 rd = rcu_lock_domain_by_id(map->domid);
kfraser@12533 1694 if ( rd == NULL )
kfraser@12533 1695 {
kfraser@12533 1696 /* Nothing to clear up... */
kfraser@12533 1697 map->flags = 0;
kfraser@12533 1698 continue;
kfraser@12533 1699 }
kaf24@7984 1700
kaf24@7984 1701 spin_lock(&rd->grant_table->lock);
kaf24@6703 1702
kfraser@13970 1703 act = &active_entry(rd->grant_table, ref);
kfraser@13970 1704 sha = &shared_entry(rd->grant_table, ref);
kaf24@6703 1705
kaf24@9722 1706 if ( map->flags & GNTMAP_readonly )
kaf24@6703 1707 {
kaf24@9722 1708 if ( map->flags & GNTMAP_device_map )
kaf24@7984 1709 {
kaf24@8427 1710 BUG_ON(!(act->pin & GNTPIN_devr_mask));
kaf24@7984 1711 act->pin -= GNTPIN_devr_inc;
keir@16407 1712 if ( !is_iomem_page(act->frame) )
keir@16407 1713 put_page(mfn_to_page(act->frame));
kaf24@7984 1714 }
kaf24@6703 1715
kaf24@9722 1716 if ( map->flags & GNTMAP_host_map )
kaf24@7984 1717 {
kaf24@8427 1718 BUG_ON(!(act->pin & GNTPIN_hstr_mask));
kaf24@7984 1719 act->pin -= GNTPIN_hstr_inc;
keir@17114 1720 if ( gnttab_release_host_mappings &&
keir@17114 1721 !is_iomem_page(act->frame) )
keir@17114 1722 put_page(mfn_to_page(act->frame));
kaf24@7984 1723 }
kaf24@7984 1724 }
kaf24@7984 1725 else
kaf24@7984 1726 {
kaf24@9722 1727 if ( map->flags & GNTMAP_device_map )
kaf24@7984 1728 {
kaf24@8427 1729 BUG_ON(!(act->pin & GNTPIN_devw_mask));
kaf24@7984 1730 act->pin -= GNTPIN_devw_inc;
keir@16407 1731 if ( !is_iomem_page(act->frame) )
keir@16407 1732 put_page_and_type(mfn_to_page(act->frame));
kaf24@7984 1733 }
kaf24@7984 1734
kaf24@9722 1735 if ( map->flags & GNTMAP_host_map )
kaf24@7984 1736 {
kaf24@8427 1737 BUG_ON(!(act->pin & GNTPIN_hstw_mask));
kaf24@7984 1738 act->pin -= GNTPIN_hstw_inc;
keir@17114 1739 if ( gnttab_release_host_mappings &&
keir@17114 1740 !is_iomem_page(act->frame) )
keir@17114 1741 {
keir@17114 1742 if ( gnttab_host_mapping_get_page_type(map, d, rd) )
keir@17114 1743 put_page_type(mfn_to_page(act->frame));
keir@17114 1744 put_page(mfn_to_page(act->frame));
keir@17114 1745 }
kaf24@7984 1746 }
kaf24@7984 1747
kaf24@7984 1748 if ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0 )
kaf24@10614 1749 gnttab_clear_flag(_GTF_writing, &sha->flags);
kaf24@7984 1750 }
cwc22@4100 1751
kaf24@7984 1752 if ( act->pin == 0 )
kaf24@10614 1753 gnttab_clear_flag(_GTF_reading, &sha->flags);
cwc22@4100 1754
kaf24@6703 1755 spin_unlock(&rd->grant_table->lock);
cwc22@4100 1756
kfraser@14192 1757 rcu_unlock_domain(rd);
kaf24@7984 1758
kaf24@9722 1759 map->flags = 0;
cwc22@4100 1760 }
cwc22@4100 1761 }
cwc22@4100 1762
cwc22@4100 1763
cwc22@4100 1764 void
kaf24@2340 1765 grant_table_destroy(
kaf24@2340 1766 struct domain *d)
kaf24@2322 1767 {
kaf24@9723 1768 struct grant_table *t = d->grant_table;
kfraser@13970 1769 int i;
kaf24@2322 1770
kaf24@8428 1771 if ( t == NULL )
kaf24@8428 1772 return;
kaf24@8428 1773
kfraser@13970 1774 for ( i = 0; i < nr_grant_frames(t); i++ )
kfraser@13970 1775 free_xenheap_page(t->shared[i]);
kfraser@13970 1776 xfree(t->shared);
kfraser@13970 1777
kfraser@13970 1778 for ( i = 0; i < nr_maptrack_frames(t); i++ )
kfraser@13970 1779 free_xenheap_page(t->maptrack[i]);
kfraser@13970 1780 xfree(t->maptrack);
kfraser@13970 1781
kfraser@13970 1782 for ( i = 0; i < nr_active_grant_frames(t); i++ )
kfraser@13970 1783 free_xenheap_page(t->active[i]);
kaf24@8428 1784 xfree(t->active);
kfraser@13970 1785
kaf24@8428 1786 xfree(t);
kaf24@8428 1787 d->grant_table = NULL;
kaf24@2322 1788 }
kaf24@3914 1789
kaf24@3914 1790 /*
kaf24@3914 1791 * Local variables:
kaf24@3914 1792 * mode: C
kaf24@3914 1793 * c-set-style: "BSD"
kaf24@3914 1794 * c-basic-offset: 4
kaf24@3914 1795 * tab-width: 4
kaf24@3914 1796 * indent-tabs-mode: nil
kaf24@3988 1797 * End:
kaf24@3914 1798 */