ia64/xen-unstable

view xen/common/grant_table.c @ 8609:85d693e6f61a

Arch-specific per-vcpu info should be initialised to zero
when allocating a new vcpu structure, not copied from
CPU0's idle VCPU. Especially now that the idle VCPU itself
is dynamically allocated.

This should fix assertions people have been seeing in
getdomain_info_ctxt() relation to IOPL in eflags.

Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Sat Jan 14 21:26:40 2006 +0100 (2006-01-14)
parents 8af1199488d3
children aad2b2da3c8b
line source
1 /******************************************************************************
2 * common/grant_table.c
3 *
4 * Mechanism for granting foreign access to page frames, and receiving
5 * page-ownership transfers.
6 *
7 * Copyright (c) 2005 Christopher Clark
8 * Copyright (c) 2004 K A Fraser
9 * Copyright (c) 2005 Andrew Warfield
10 * Modifications by Geoffrey Lefebvre are (c) Intel Research Cambridge
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software
24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 */
27 #include <xen/lib.h>
28 #include <xen/sched.h>
29 #include <xen/shadow.h>
30 #include <xen/mm.h>
31 #include <acm/acm_hooks.h>
32 #include <xen/trace.h>
34 #define PIN_FAIL(_lbl, _rc, _f, _a...) \
35 do { \
36 DPRINTK( _f, ## _a ); \
37 rc = (_rc); \
38 goto _lbl; \
39 } while ( 0 )
41 static inline int
42 get_maptrack_handle(
43 grant_table_t *t)
44 {
45 unsigned int h;
46 if ( unlikely((h = t->maptrack_head) == (t->maptrack_limit - 1)) )
47 return -1;
48 t->maptrack_head = t->maptrack[h].ref_and_flags >> MAPTRACK_REF_SHIFT;
49 t->map_count++;
50 return h;
51 }
53 static inline void
54 put_maptrack_handle(
55 grant_table_t *t, int handle)
56 {
57 t->maptrack[handle].ref_and_flags = t->maptrack_head << MAPTRACK_REF_SHIFT;
58 t->maptrack_head = handle;
59 t->map_count--;
60 }
62 /*
63 * Returns 0 if TLB flush / invalidate required by caller.
64 * va will indicate the address to be invalidated.
65 *
66 * addr is _either_ a host virtual address, or the address of the pte to
67 * update, as indicated by the GNTMAP_contains_pte flag.
68 */
69 static int
70 __gnttab_map_grant_ref(
71 gnttab_map_grant_ref_t *uop)
72 {
73 domid_t dom;
74 grant_ref_t ref;
75 struct domain *ld, *rd;
76 struct vcpu *led;
77 u32 dev_hst_ro_flags;
78 int handle;
79 u64 addr;
80 unsigned long frame = 0;
81 int rc = GNTST_okay;
82 active_grant_entry_t *act;
84 /* Entry details from @rd's shared grant table. */
85 grant_entry_t *sha;
86 domid_t sdom;
87 u16 sflags;
89 /*
90 * We bound the number of times we retry CMPXCHG on memory locations that
91 * we share with a guest OS. The reason is that the guest can modify that
92 * location at a higher rate than we can read-modify-CMPXCHG, so the guest
93 * could cause us to livelock. There are a few cases where it is valid for
94 * the guest to race our updates (e.g., to change the GTF_readonly flag),
95 * so we allow a few retries before failing.
96 */
97 int retries = 0;
99 led = current;
100 ld = led->domain;
102 /* Bitwise-OR avoids short-circuiting which screws control flow. */
103 if ( unlikely(__get_user(dom, &uop->dom) |
104 __get_user(ref, &uop->ref) |
105 __get_user(addr, &uop->host_addr) |
106 __get_user(dev_hst_ro_flags, &uop->flags)) )
107 {
108 DPRINTK("Fault while reading gnttab_map_grant_ref_t.\n");
109 return -EFAULT; /* don't set status */
110 }
112 if ( unlikely(ref >= NR_GRANT_ENTRIES) ||
113 unlikely((dev_hst_ro_flags &
114 (GNTMAP_device_map|GNTMAP_host_map)) == 0) )
115 {
116 DPRINTK("Bad ref (%d) or flags (%x).\n", ref, dev_hst_ro_flags);
117 (void)__put_user(GNTST_bad_gntref, &uop->status);
118 return GNTST_bad_gntref;
119 }
121 if ( acm_pre_grant_map_ref(dom) )
122 {
123 (void)__put_user(GNTST_permission_denied, &uop->status);
124 return GNTST_permission_denied;
125 }
127 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
128 unlikely(ld == rd) )
129 {
130 if ( rd != NULL )
131 put_domain(rd);
132 DPRINTK("Could not find domain %d\n", dom);
133 (void)__put_user(GNTST_bad_domain, &uop->status);
134 return GNTST_bad_domain;
135 }
137 /* Get a maptrack handle. */
138 if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
139 {
140 int i;
141 grant_mapping_t *new_mt;
142 grant_table_t *lgt = ld->grant_table;
144 if ( (lgt->maptrack_limit << 1) > MAPTRACK_MAX_ENTRIES )
145 {
146 put_domain(rd);
147 DPRINTK("Maptrack table is at maximum size.\n");
148 (void)__put_user(GNTST_no_device_space, &uop->status);
149 return GNTST_no_device_space;
150 }
152 /* Grow the maptrack table. */
153 new_mt = alloc_xenheap_pages(lgt->maptrack_order + 1);
154 if ( new_mt == NULL )
155 {
156 put_domain(rd);
157 DPRINTK("No more map handles available.\n");
158 (void)__put_user(GNTST_no_device_space, &uop->status);
159 return GNTST_no_device_space;
160 }
162 memcpy(new_mt, lgt->maptrack, PAGE_SIZE << lgt->maptrack_order);
163 for ( i = lgt->maptrack_limit; i < (lgt->maptrack_limit << 1); i++ )
164 new_mt[i].ref_and_flags = (i+1) << MAPTRACK_REF_SHIFT;
166 free_xenheap_pages(lgt->maptrack, lgt->maptrack_order);
167 lgt->maptrack = new_mt;
168 lgt->maptrack_order += 1;
169 lgt->maptrack_limit <<= 1;
171 DPRINTK("Doubled maptrack size\n");
172 handle = get_maptrack_handle(ld->grant_table);
173 }
175 act = &rd->grant_table->active[ref];
176 sha = &rd->grant_table->shared[ref];
178 spin_lock(&rd->grant_table->lock);
180 if ( !act->pin ||
181 (!(dev_hst_ro_flags & GNTMAP_readonly) &&
182 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask))) )
183 {
184 sflags = sha->flags;
185 sdom = sha->domid;
187 /*
188 * This loop attempts to set the access (reading/writing) flags
189 * in the grant table entry. It tries a cmpxchg on the field
190 * up to five times, and then fails under the assumption that
191 * the guest is misbehaving.
192 */
193 for ( ; ; )
194 {
195 u32 scombo, prev_scombo, new_scombo;
197 if ( unlikely((sflags & GTF_type_mask) != GTF_permit_access) ||
198 unlikely(sdom != led->domain->domain_id) )
199 PIN_FAIL(unlock_out, GNTST_general_error,
200 "Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
201 sflags, sdom, led->domain->domain_id);
203 /* Merge two 16-bit values into a 32-bit combined update. */
204 /* NB. Endianness! */
205 prev_scombo = scombo = ((u32)sdom << 16) | (u32)sflags;
207 new_scombo = scombo | GTF_reading;
208 if ( !(dev_hst_ro_flags & GNTMAP_readonly) )
209 {
210 new_scombo |= GTF_writing;
211 if ( unlikely(sflags & GTF_readonly) )
212 PIN_FAIL(unlock_out, GNTST_general_error,
213 "Attempt to write-pin a r/o grant entry.\n");
214 }
216 /* NB. prev_scombo is updated in place to seen value. */
217 if ( unlikely(cmpxchg_user((u32 *)&sha->flags,
218 prev_scombo,
219 new_scombo)) )
220 PIN_FAIL(unlock_out, GNTST_general_error,
221 "Fault while modifying shared flags and domid.\n");
223 /* Did the combined update work (did we see what we expected?). */
224 if ( likely(prev_scombo == scombo) )
225 break;
227 if ( retries++ == 4 )
228 PIN_FAIL(unlock_out, GNTST_general_error,
229 "Shared grant entry is unstable.\n");
231 /* Didn't see what we expected. Split out the seen flags & dom. */
232 /* NB. Endianness! */
233 sflags = (u16)prev_scombo;
234 sdom = (u16)(prev_scombo >> 16);
235 }
237 if ( !act->pin )
238 {
239 act->domid = sdom;
240 act->frame = __gpfn_to_mfn(rd, sha->frame);
241 }
242 }
243 else if ( (act->pin & 0x80808080U) != 0 )
244 PIN_FAIL(unlock_out, ENOSPC,
245 "Risk of counter overflow %08x\n", act->pin);
247 if ( dev_hst_ro_flags & GNTMAP_device_map )
248 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
249 GNTPIN_devr_inc : GNTPIN_devw_inc;
250 if ( dev_hst_ro_flags & GNTMAP_host_map )
251 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
252 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
254 spin_unlock(&rd->grant_table->lock);
256 frame = act->frame;
257 if ( unlikely(!pfn_valid(frame)) ||
258 unlikely(!((dev_hst_ro_flags & GNTMAP_readonly) ?
259 get_page(pfn_to_page(frame), rd) :
260 get_page_and_type(pfn_to_page(frame), rd,
261 PGT_writable_page))) )
262 PIN_FAIL(undo_out, GNTST_general_error,
263 "Could not pin the granted frame (%lx)!\n", frame);
265 if ( dev_hst_ro_flags & GNTMAP_host_map )
266 {
267 rc = create_grant_host_mapping(addr, frame, dev_hst_ro_flags);
268 if ( rc != GNTST_okay )
269 {
270 if ( !(dev_hst_ro_flags & GNTMAP_readonly) )
271 put_page_type(pfn_to_page(frame));
272 put_page(pfn_to_page(frame));
273 goto undo_out;
274 }
276 if ( dev_hst_ro_flags & GNTMAP_device_map )
277 {
278 (void)get_page(pfn_to_page(frame), rd);
279 if ( !(dev_hst_ro_flags & GNTMAP_readonly) )
280 get_page_type(pfn_to_page(frame), PGT_writable_page);
281 }
282 }
284 TRACE_1D(TRC_MEM_PAGE_GRANT_MAP, dom);
286 ld->grant_table->maptrack[handle].domid = dom;
287 ld->grant_table->maptrack[handle].ref_and_flags =
288 (ref << MAPTRACK_REF_SHIFT) |
289 (dev_hst_ro_flags & MAPTRACK_GNTMAP_MASK);
291 (void)__put_user((u64)frame << PAGE_SHIFT, &uop->dev_bus_addr);
292 (void)__put_user(handle, &uop->handle);
293 (void)__put_user(GNTST_okay, &uop->status);
295 put_domain(rd);
296 return rc;
298 undo_out:
299 spin_lock(&rd->grant_table->lock);
301 if ( dev_hst_ro_flags & GNTMAP_device_map )
302 act->pin -= (dev_hst_ro_flags & GNTMAP_readonly) ?
303 GNTPIN_devr_inc : GNTPIN_devw_inc;
304 if ( dev_hst_ro_flags & GNTMAP_host_map )
305 act->pin -= (dev_hst_ro_flags & GNTMAP_readonly) ?
306 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
308 if ( !(dev_hst_ro_flags & GNTMAP_readonly) &&
309 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
310 clear_bit(_GTF_writing, &sha->flags);
312 if ( !act->pin )
313 clear_bit(_GTF_reading, &sha->flags);
315 unlock_out:
316 spin_unlock(&rd->grant_table->lock);
317 (void)__put_user(rc, &uop->status);
318 put_maptrack_handle(ld->grant_table, handle);
319 return rc;
320 }
322 static long
323 gnttab_map_grant_ref(
324 gnttab_map_grant_ref_t *uop, unsigned int count)
325 {
326 int i;
328 for ( i = 0; i < count; i++ )
329 (void)__gnttab_map_grant_ref(&uop[i]);
331 return 0;
332 }
334 static int
335 __gnttab_unmap_grant_ref(
336 gnttab_unmap_grant_ref_t *uop)
337 {
338 domid_t dom;
339 grant_ref_t ref;
340 grant_handle_t handle;
341 struct domain *ld, *rd;
342 active_grant_entry_t *act;
343 grant_entry_t *sha;
344 grant_mapping_t *map;
345 u16 flags;
346 s16 rc = 0;
347 u64 addr, dev_bus_addr;
348 unsigned long frame;
350 ld = current->domain;
352 /* Bitwise-OR avoids short-circuiting which screws control flow. */
353 if ( unlikely(__get_user(addr, &uop->host_addr) |
354 __get_user(dev_bus_addr, &uop->dev_bus_addr) |
355 __get_user(handle, &uop->handle)) )
356 {
357 DPRINTK("Fault while reading gnttab_unmap_grant_ref_t.\n");
358 return -EFAULT; /* don't set status */
359 }
361 frame = (unsigned long)(dev_bus_addr >> PAGE_SHIFT);
363 map = &ld->grant_table->maptrack[handle];
365 if ( unlikely(handle >= ld->grant_table->maptrack_limit) ||
366 unlikely(!(map->ref_and_flags & MAPTRACK_GNTMAP_MASK)) )
367 {
368 DPRINTK("Bad handle (%d).\n", handle);
369 (void)__put_user(GNTST_bad_handle, &uop->status);
370 return GNTST_bad_handle;
371 }
373 dom = map->domid;
374 ref = map->ref_and_flags >> MAPTRACK_REF_SHIFT;
375 flags = map->ref_and_flags & MAPTRACK_GNTMAP_MASK;
377 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
378 unlikely(ld == rd) )
379 {
380 if ( rd != NULL )
381 put_domain(rd);
382 DPRINTK("Could not find domain %d\n", dom);
383 (void)__put_user(GNTST_bad_domain, &uop->status);
384 return GNTST_bad_domain;
385 }
387 TRACE_1D(TRC_MEM_PAGE_GRANT_UNMAP, dom);
389 act = &rd->grant_table->active[ref];
390 sha = &rd->grant_table->shared[ref];
392 spin_lock(&rd->grant_table->lock);
394 if ( frame == 0 )
395 {
396 frame = act->frame;
397 }
398 else
399 {
400 if ( unlikely(frame != act->frame) )
401 PIN_FAIL(unmap_out, GNTST_general_error,
402 "Bad frame number doesn't match gntref.\n");
403 if ( flags & GNTMAP_device_map )
404 {
405 ASSERT(act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask));
406 map->ref_and_flags &= ~GNTMAP_device_map;
407 if ( flags & GNTMAP_readonly )
408 {
409 act->pin -= GNTPIN_devr_inc;
410 put_page(pfn_to_page(frame));
411 }
412 else
413 {
414 act->pin -= GNTPIN_devw_inc;
415 put_page_and_type(pfn_to_page(frame));
416 }
417 }
418 }
420 if ( (addr != 0) && (flags & GNTMAP_host_map) )
421 {
422 if ( (rc = destroy_grant_host_mapping(addr, frame, flags)) < 0 )
423 goto unmap_out;
425 ASSERT(act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask));
426 map->ref_and_flags &= ~GNTMAP_host_map;
427 if ( flags & GNTMAP_readonly )
428 {
429 act->pin -= GNTPIN_hstr_inc;
430 put_page(pfn_to_page(frame));
431 }
432 else
433 {
434 act->pin -= GNTPIN_hstw_inc;
435 put_page_and_type(pfn_to_page(frame));
436 }
437 }
439 if ( (map->ref_and_flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0 )
440 {
441 map->ref_and_flags = 0;
442 put_maptrack_handle(ld->grant_table, handle);
443 }
445 /* If just unmapped a writable mapping, mark as dirtied */
446 if ( !(flags & GNTMAP_readonly) )
447 gnttab_log_dirty(rd, frame);
449 if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0) &&
450 !(flags & GNTMAP_readonly) )
451 clear_bit(_GTF_writing, &sha->flags);
453 if ( act->pin == 0 )
454 clear_bit(_GTF_reading, &sha->flags);
456 unmap_out:
457 (void)__put_user(rc, &uop->status);
458 spin_unlock(&rd->grant_table->lock);
459 put_domain(rd);
460 return rc;
461 }
463 static long
464 gnttab_unmap_grant_ref(
465 gnttab_unmap_grant_ref_t *uop, unsigned int count)
466 {
467 int i;
469 for ( i = 0; i < count; i++ )
470 (void)__gnttab_unmap_grant_ref(&uop[i]);
472 flush_tlb_mask(current->domain->domain_dirty_cpumask);
474 return 0;
475 }
477 static long
478 gnttab_setup_table(
479 gnttab_setup_table_t *uop, unsigned int count)
480 {
481 gnttab_setup_table_t op;
482 struct domain *d;
483 int i;
485 if ( count != 1 )
486 return -EINVAL;
488 if ( unlikely(copy_from_user(&op, uop, sizeof(op)) != 0) )
489 {
490 DPRINTK("Fault while reading gnttab_setup_table_t.\n");
491 return -EFAULT;
492 }
494 if ( unlikely(op.nr_frames > NR_GRANT_FRAMES) )
495 {
496 DPRINTK("Xen only supports up to %d grant-table frames per domain.\n",
497 NR_GRANT_FRAMES);
498 (void)put_user(GNTST_general_error, &uop->status);
499 return 0;
500 }
502 if ( op.dom == DOMID_SELF )
503 {
504 op.dom = current->domain->domain_id;
505 }
506 else if ( unlikely(!IS_PRIV(current->domain)) )
507 {
508 (void)put_user(GNTST_permission_denied, &uop->status);
509 return 0;
510 }
512 if ( unlikely((d = find_domain_by_id(op.dom)) == NULL) )
513 {
514 DPRINTK("Bad domid %d.\n", op.dom);
515 (void)put_user(GNTST_bad_domain, &uop->status);
516 return 0;
517 }
519 if ( op.nr_frames <= NR_GRANT_FRAMES )
520 {
521 ASSERT(d->grant_table != NULL);
522 (void)put_user(GNTST_okay, &uop->status);
523 for ( i = 0; i < op.nr_frames; i++ )
524 (void)put_user(gnttab_shared_mfn(d, d->grant_table, i),
525 &op.frame_list[i]);
526 }
528 put_domain(d);
529 return 0;
530 }
532 static int
533 gnttab_dump_table(
534 gnttab_dump_table_t *uop)
535 {
536 grant_table_t *gt;
537 gnttab_dump_table_t op;
538 struct domain *d;
539 u32 shared_mfn;
540 active_grant_entry_t *act;
541 grant_entry_t sha_copy;
542 grant_mapping_t *maptrack;
543 int i;
545 if ( !IS_PRIV(current->domain) )
546 return -EPERM;
548 if ( unlikely(copy_from_user(&op, uop, sizeof(op)) != 0) )
549 {
550 DPRINTK("Fault while reading gnttab_dump_table_t.\n");
551 return -EFAULT;
552 }
554 if ( op.dom == DOMID_SELF )
555 op.dom = current->domain->domain_id;
557 if ( unlikely((d = find_domain_by_id(op.dom)) == NULL) )
558 {
559 DPRINTK("Bad domid %d.\n", op.dom);
560 (void)put_user(GNTST_bad_domain, &uop->status);
561 return 0;
562 }
564 ASSERT(d->grant_table != NULL);
565 gt = d->grant_table;
566 (void)put_user(GNTST_okay, &uop->status);
568 shared_mfn = virt_to_phys(d->grant_table->shared);
570 DPRINTK("Grant table for dom (%hu) MFN (%x)\n",
571 op.dom, shared_mfn);
573 ASSERT(d->grant_table->active != NULL);
574 ASSERT(d->grant_table->shared != NULL);
575 ASSERT(d->grant_table->maptrack != NULL);
577 for ( i = 0; i < NR_GRANT_ENTRIES; i++ )
578 {
579 sha_copy = gt->shared[i];
580 if ( sha_copy.flags )
581 DPRINTK("Grant: dom (%hu) SHARED (%d) flags:(%hx) "
582 "dom:(%hu) frame:(%x)\n",
583 op.dom, i, sha_copy.flags, sha_copy.domid, sha_copy.frame);
584 }
586 spin_lock(&gt->lock);
588 for ( i = 0; i < NR_GRANT_ENTRIES; i++ )
589 {
590 act = &gt->active[i];
591 if ( act->pin )
592 DPRINTK("Grant: dom (%hu) ACTIVE (%d) pin:(%x) "
593 "dom:(%hu) frame:(%lx)\n",
594 op.dom, i, act->pin, act->domid, act->frame);
595 }
597 for ( i = 0; i < gt->maptrack_limit; i++ )
598 {
599 maptrack = &gt->maptrack[i];
600 if ( maptrack->ref_and_flags & MAPTRACK_GNTMAP_MASK )
601 DPRINTK("Grant: dom (%hu) MAP (%d) ref:(%hu) flags:(%x) "
602 "dom:(%hu)\n",
603 op.dom, i,
604 maptrack->ref_and_flags >> MAPTRACK_REF_SHIFT,
605 maptrack->ref_and_flags & MAPTRACK_GNTMAP_MASK,
606 maptrack->domid);
607 }
609 spin_unlock(&gt->lock);
611 put_domain(d);
612 return 0;
613 }
615 /*
616 * Check that the given grant reference (rd,ref) allows 'ld' to transfer
617 * ownership of a page frame. If so, lock down the grant entry.
618 */
619 static int
620 gnttab_prepare_for_transfer(
621 struct domain *rd, struct domain *ld, grant_ref_t ref)
622 {
623 grant_table_t *rgt;
624 grant_entry_t *sha;
625 domid_t sdom;
626 u16 sflags;
627 u32 scombo, prev_scombo;
628 int retries = 0;
629 unsigned long target_pfn;
631 if ( unlikely((rgt = rd->grant_table) == NULL) ||
632 unlikely(ref >= NR_GRANT_ENTRIES) )
633 {
634 DPRINTK("Dom %d has no g.t., or ref is bad (%d).\n",
635 rd->domain_id, ref);
636 return 0;
637 }
639 spin_lock(&rgt->lock);
641 sha = &rgt->shared[ref];
643 sflags = sha->flags;
644 sdom = sha->domid;
646 for ( ; ; )
647 {
648 target_pfn = sha->frame;
650 if ( unlikely(target_pfn >= max_page ) )
651 {
652 DPRINTK("Bad pfn (%lx)\n", target_pfn);
653 goto fail;
654 }
656 if ( unlikely(sflags != GTF_accept_transfer) ||
657 unlikely(sdom != ld->domain_id) )
658 {
659 DPRINTK("Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
660 sflags, sdom, ld->domain_id);
661 goto fail;
662 }
664 /* Merge two 16-bit values into a 32-bit combined update. */
665 /* NB. Endianness! */
666 prev_scombo = scombo = ((u32)sdom << 16) | (u32)sflags;
668 /* NB. prev_scombo is updated in place to seen value. */
669 if ( unlikely(cmpxchg_user((u32 *)&sha->flags, prev_scombo,
670 prev_scombo | GTF_transfer_committed)) )
671 {
672 DPRINTK("Fault while modifying shared flags and domid.\n");
673 goto fail;
674 }
676 /* Did the combined update work (did we see what we expected?). */
677 if ( likely(prev_scombo == scombo) )
678 break;
680 if ( retries++ == 4 )
681 {
682 DPRINTK("Shared grant entry is unstable.\n");
683 goto fail;
684 }
686 /* Didn't see what we expected. Split out the seen flags & dom. */
687 /* NB. Endianness! */
688 sflags = (u16)prev_scombo;
689 sdom = (u16)(prev_scombo >> 16);
690 }
692 spin_unlock(&rgt->lock);
693 return 1;
695 fail:
696 spin_unlock(&rgt->lock);
697 return 0;
698 }
700 static long
701 gnttab_transfer(
702 gnttab_transfer_t *uop, unsigned int count)
703 {
704 struct domain *d = current->domain;
705 struct domain *e;
706 struct pfn_info *page;
707 int i;
708 grant_entry_t *sha;
709 gnttab_transfer_t gop;
711 for ( i = 0; i < count; i++ )
712 {
713 /* Read from caller address space. */
714 if ( unlikely(__copy_from_user(&gop, &uop[i], sizeof(gop))) )
715 {
716 DPRINTK("gnttab_transfer: error reading req %d/%d\n", i, count);
717 (void)__put_user(GNTST_bad_page, &uop[i].status);
718 return -EFAULT; /* This is *very* fatal. */
719 }
721 /* Check the passed page frame for basic validity. */
722 page = pfn_to_page(gop.mfn);
723 if ( unlikely(!pfn_valid(gop.mfn) || IS_XEN_HEAP_FRAME(page)) )
724 {
725 DPRINTK("gnttab_transfer: out-of-range or xen frame %lx\n",
726 (unsigned long)gop.mfn);
727 (void)__put_user(GNTST_bad_page, &uop[i].status);
728 continue;
729 }
731 if ( steal_page_for_grant_transfer(d, page) < 0 )
732 {
733 (void)__put_user(GNTST_bad_page, &uop[i].status);
734 continue;
735 }
737 /* Find the target domain. */
738 if ( unlikely((e = find_domain_by_id(gop.domid)) == NULL) )
739 {
740 DPRINTK("gnttab_transfer: can't find domain %d\n", gop.domid);
741 (void)__put_user(GNTST_bad_domain, &uop[i].status);
742 page->count_info &= ~(PGC_count_mask|PGC_allocated);
743 free_domheap_page(page);
744 continue;
745 }
747 spin_lock(&e->page_alloc_lock);
749 /*
750 * Check that 'e' will accept the page and has reservation
751 * headroom. Also, a domain mustn't have PGC_allocated
752 * pages when it is dying.
753 */
754 if ( unlikely(test_bit(_DOMF_dying, &e->domain_flags)) ||
755 unlikely(e->tot_pages >= e->max_pages) ||
756 unlikely(!gnttab_prepare_for_transfer(e, d, gop.ref)) )
757 {
758 if ( !test_bit(_DOMF_dying, &e->domain_flags) )
759 DPRINTK("gnttab_transfer: Transferee has no reservation "
760 "headroom (%d,%d) or provided a bad grant ref (%08x) "
761 "or is dying (%lx)\n",
762 e->tot_pages, e->max_pages, gop.ref, e->domain_flags);
763 spin_unlock(&e->page_alloc_lock);
764 put_domain(e);
765 (void)__put_user(GNTST_general_error, &uop[i].status);
766 page->count_info &= ~(PGC_count_mask|PGC_allocated);
767 free_domheap_page(page);
768 continue;
769 }
771 /* Okay, add the page to 'e'. */
772 if ( unlikely(e->tot_pages++ == 0) )
773 get_knownalive_domain(e);
774 list_add_tail(&page->list, &e->page_list);
775 page_set_owner(page, e);
777 spin_unlock(&e->page_alloc_lock);
779 TRACE_1D(TRC_MEM_PAGE_GRANT_TRANSFER, e->domain_id);
781 /* Tell the guest about its new page frame. */
782 sha = &e->grant_table->shared[gop.ref];
783 sha->frame = gop.mfn;
784 wmb();
785 sha->flags |= GTF_transfer_completed;
787 put_domain(e);
789 (void)__put_user(GNTST_okay, &uop[i].status);
790 }
792 return 0;
793 }
795 long
796 do_grant_table_op(
797 unsigned int cmd, void *uop, unsigned int count)
798 {
799 long rc;
800 struct domain *d = current->domain;
802 if ( count > 512 )
803 return -EINVAL;
805 LOCK_BIGLOCK(d);
807 sync_pagetable_state(d);
809 rc = -EFAULT;
810 switch ( cmd )
811 {
812 case GNTTABOP_map_grant_ref:
813 if ( unlikely(!array_access_ok(
814 uop, count, sizeof(gnttab_map_grant_ref_t))) )
815 goto out;
816 rc = gnttab_map_grant_ref((gnttab_map_grant_ref_t *)uop, count);
817 break;
818 case GNTTABOP_unmap_grant_ref:
819 if ( unlikely(!array_access_ok(
820 uop, count, sizeof(gnttab_unmap_grant_ref_t))) )
821 goto out;
822 rc = gnttab_unmap_grant_ref(
823 (gnttab_unmap_grant_ref_t *)uop, count);
824 break;
825 case GNTTABOP_setup_table:
826 rc = gnttab_setup_table((gnttab_setup_table_t *)uop, count);
827 break;
828 case GNTTABOP_dump_table:
829 rc = gnttab_dump_table((gnttab_dump_table_t *)uop);
830 break;
831 case GNTTABOP_transfer:
832 if (unlikely(!array_access_ok(
833 uop, count, sizeof(gnttab_transfer_t))))
834 goto out;
835 rc = gnttab_transfer(uop, count);
836 break;
837 default:
838 rc = -ENOSYS;
839 break;
840 }
842 out:
843 UNLOCK_BIGLOCK(d);
845 return rc;
846 }
848 int
849 grant_table_create(
850 struct domain *d)
851 {
852 grant_table_t *t;
853 int i;
855 if ( (t = xmalloc(grant_table_t)) == NULL )
856 goto no_mem;
858 /* Simple stuff. */
859 memset(t, 0, sizeof(*t));
860 spin_lock_init(&t->lock);
862 /* Active grant table. */
863 if ( (t->active = xmalloc_array(active_grant_entry_t, NR_GRANT_ENTRIES))
864 == NULL )
865 goto no_mem;
866 memset(t->active, 0, sizeof(active_grant_entry_t) * NR_GRANT_ENTRIES);
868 /* Tracking of mapped foreign frames table */
869 if ( (t->maptrack = alloc_xenheap_page()) == NULL )
870 goto no_mem;
871 t->maptrack_order = 0;
872 t->maptrack_limit = PAGE_SIZE / sizeof(grant_mapping_t);
873 memset(t->maptrack, 0, PAGE_SIZE);
874 for ( i = 0; i < t->maptrack_limit; i++ )
875 t->maptrack[i].ref_and_flags = (i+1) << MAPTRACK_REF_SHIFT;
877 /* Shared grant table. */
878 t->shared = alloc_xenheap_pages(ORDER_GRANT_FRAMES);
879 if ( t->shared == NULL )
880 goto no_mem;
881 memset(t->shared, 0, NR_GRANT_FRAMES * PAGE_SIZE);
883 for ( i = 0; i < NR_GRANT_FRAMES; i++ )
884 gnttab_create_shared_mfn(d, t, i);
886 /* Okay, install the structure. */
887 wmb(); /* avoid races with lock-free access to d->grant_table */
888 d->grant_table = t;
889 return 0;
891 no_mem:
892 if ( t != NULL )
893 {
894 xfree(t->active);
895 free_xenheap_page(t->maptrack);
896 xfree(t);
897 }
898 return -ENOMEM;
899 }
901 void
902 gnttab_release_mappings(
903 struct domain *d)
904 {
905 grant_table_t *gt = d->grant_table;
906 grant_mapping_t *map;
907 grant_ref_t ref;
908 grant_handle_t handle;
909 struct domain *rd;
910 active_grant_entry_t *act;
911 grant_entry_t *sha;
913 BUG_ON(!test_bit(_DOMF_dying, &d->domain_flags));
915 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
916 {
917 map = &gt->maptrack[handle];
918 if ( !(map->ref_and_flags & (GNTMAP_device_map|GNTMAP_host_map)) )
919 continue;
921 ref = map->ref_and_flags >> MAPTRACK_REF_SHIFT;
923 DPRINTK("Grant release (%hu) ref:(%hu) flags:(%x) dom:(%hu)\n",
924 handle, ref, map->ref_and_flags & MAPTRACK_GNTMAP_MASK,
925 map->domid);
927 rd = find_domain_by_id(map->domid);
928 BUG_ON(rd == NULL);
930 spin_lock(&rd->grant_table->lock);
932 act = &rd->grant_table->active[ref];
933 sha = &rd->grant_table->shared[ref];
935 if ( map->ref_and_flags & GNTMAP_readonly )
936 {
937 if ( map->ref_and_flags & GNTMAP_device_map )
938 {
939 BUG_ON(!(act->pin & GNTPIN_devr_mask));
940 act->pin -= GNTPIN_devr_inc;
941 put_page(pfn_to_page(act->frame));
942 }
944 if ( map->ref_and_flags & GNTMAP_host_map )
945 {
946 BUG_ON(!(act->pin & GNTPIN_hstr_mask));
947 act->pin -= GNTPIN_hstr_inc;
948 /* Done implicitly when page tables are destroyed. */
949 /* put_page(pfn_to_page(act->frame)); */
950 }
951 }
952 else
953 {
954 if ( map->ref_and_flags & GNTMAP_device_map )
955 {
956 BUG_ON(!(act->pin & GNTPIN_devw_mask));
957 act->pin -= GNTPIN_devw_inc;
958 put_page_and_type(pfn_to_page(act->frame));
959 }
961 if ( map->ref_and_flags & GNTMAP_host_map )
962 {
963 BUG_ON(!(act->pin & GNTPIN_hstw_mask));
964 act->pin -= GNTPIN_hstw_inc;
965 /* Done implicitly when page tables are destroyed. */
966 /* put_page_and_type(pfn_to_page(act->frame)); */
967 }
969 if ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0 )
970 clear_bit(_GTF_writing, &sha->flags);
971 }
973 if ( act->pin == 0 )
974 clear_bit(_GTF_reading, &sha->flags);
976 spin_unlock(&rd->grant_table->lock);
978 put_domain(rd);
980 map->ref_and_flags = 0;
981 }
982 }
985 void
986 grant_table_destroy(
987 struct domain *d)
988 {
989 grant_table_t *t = d->grant_table;
991 if ( t == NULL )
992 return;
994 free_xenheap_pages(t->shared, ORDER_GRANT_FRAMES);
995 free_xenheap_page(t->maptrack);
996 xfree(t->active);
997 xfree(t);
999 d->grant_table = NULL;
1002 /*
1003 * Local variables:
1004 * mode: C
1005 * c-set-style: "BSD"
1006 * c-basic-offset: 4
1007 * tab-width: 4
1008 * indent-tabs-mode: nil
1009 * End:
1010 */