ia64/xen-unstable

view xen/common/grant_table.c @ 11096:eb66b68db7b1

[XEN] Add some missing put_page()s to __gnttab_copy.

Signed-off-by: Steven Smith <sos22@cam.ac.uk>
author ssmith@weybridge.uk.xensource.com
date Mon Aug 14 10:47:59 2006 +0100 (2006-08-14)
parents c0a3f670d0d6
children f328519053f5
line source
1 /******************************************************************************
2 * common/grant_table.c
3 *
4 * Mechanism for granting foreign access to page frames, and receiving
5 * page-ownership transfers.
6 *
7 * Copyright (c) 2005 Christopher Clark
8 * Copyright (c) 2004 K A Fraser
9 * Copyright (c) 2005 Andrew Warfield
10 * Modifications by Geoffrey Lefebvre are (c) Intel Research Cambridge
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software
24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 */
27 #include <xen/lib.h>
28 #include <xen/sched.h>
29 #include <xen/shadow.h>
30 #include <xen/mm.h>
31 #include <xen/trace.h>
32 #include <xen/guest_access.h>
33 #include <xen/domain_page.h>
34 #include <acm/acm_hooks.h>
36 #define PIN_FAIL(_lbl, _rc, _f, _a...) \
37 do { \
38 DPRINTK( _f, ## _a ); \
39 rc = (_rc); \
40 goto _lbl; \
41 } while ( 0 )
43 static inline int
44 get_maptrack_handle(
45 struct grant_table *t)
46 {
47 unsigned int h;
48 if ( unlikely((h = t->maptrack_head) == (t->maptrack_limit - 1)) )
49 return -1;
50 t->maptrack_head = t->maptrack[h].ref;
51 t->map_count++;
52 return h;
53 }
55 static inline void
56 put_maptrack_handle(
57 struct grant_table *t, int handle)
58 {
59 t->maptrack[handle].ref = t->maptrack_head;
60 t->maptrack_head = handle;
61 t->map_count--;
62 }
64 /*
65 * Returns 0 if TLB flush / invalidate required by caller.
66 * va will indicate the address to be invalidated.
67 *
68 * addr is _either_ a host virtual address, or the address of the pte to
69 * update, as indicated by the GNTMAP_contains_pte flag.
70 */
71 static void
72 __gnttab_map_grant_ref(
73 struct gnttab_map_grant_ref *op)
74 {
75 struct domain *ld, *rd;
76 struct vcpu *led;
77 int handle;
78 unsigned long frame = 0;
79 int rc = GNTST_okay;
80 struct active_grant_entry *act;
82 /* Entry details from @rd's shared grant table. */
83 grant_entry_t *sha;
84 domid_t sdom;
85 u16 sflags;
87 /*
88 * We bound the number of times we retry CMPXCHG on memory locations that
89 * we share with a guest OS. The reason is that the guest can modify that
90 * location at a higher rate than we can read-modify-CMPXCHG, so the guest
91 * could cause us to livelock. There are a few cases where it is valid for
92 * the guest to race our updates (e.g., to change the GTF_readonly flag),
93 * so we allow a few retries before failing.
94 */
95 int retries = 0;
97 led = current;
98 ld = led->domain;
100 if ( unlikely(op->ref >= NR_GRANT_ENTRIES) ||
101 unlikely((op->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0) )
102 {
103 DPRINTK("Bad ref (%d) or flags (%x).\n", op->ref, op->flags);
104 op->status = GNTST_bad_gntref;
105 return;
106 }
108 if ( acm_pre_grant_map_ref(op->dom) )
109 {
110 op->status = GNTST_permission_denied;
111 return;
112 }
114 if ( unlikely((rd = find_domain_by_id(op->dom)) == NULL) )
115 {
116 if ( rd != NULL )
117 put_domain(rd);
118 DPRINTK("Could not find domain %d\n", op->dom);
119 op->status = GNTST_bad_domain;
120 return;
121 }
123 /* Get a maptrack handle. */
124 if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
125 {
126 int i;
127 struct grant_mapping *new_mt;
128 struct grant_table *lgt = ld->grant_table;
130 if ( (lgt->maptrack_limit << 1) > MAPTRACK_MAX_ENTRIES )
131 {
132 put_domain(rd);
133 DPRINTK("Maptrack table is at maximum size.\n");
134 op->status = GNTST_no_device_space;
135 return;
136 }
138 /* Grow the maptrack table. */
139 new_mt = alloc_xenheap_pages(lgt->maptrack_order + 1);
140 if ( new_mt == NULL )
141 {
142 put_domain(rd);
143 DPRINTK("No more map handles available.\n");
144 op->status = GNTST_no_device_space;
145 return;
146 }
148 memcpy(new_mt, lgt->maptrack, PAGE_SIZE << lgt->maptrack_order);
149 for ( i = lgt->maptrack_limit; i < (lgt->maptrack_limit << 1); i++ )
150 new_mt[i].ref = i+1;
152 free_xenheap_pages(lgt->maptrack, lgt->maptrack_order);
153 lgt->maptrack = new_mt;
154 lgt->maptrack_order += 1;
155 lgt->maptrack_limit <<= 1;
157 DPRINTK("Doubled maptrack size\n");
158 handle = get_maptrack_handle(ld->grant_table);
159 }
161 act = &rd->grant_table->active[op->ref];
162 sha = &rd->grant_table->shared[op->ref];
164 spin_lock(&rd->grant_table->lock);
166 if ( !act->pin ||
167 (!(op->flags & GNTMAP_readonly) &&
168 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask))) )
169 {
170 sflags = sha->flags;
171 sdom = sha->domid;
173 /*
174 * This loop attempts to set the access (reading/writing) flags
175 * in the grant table entry. It tries a cmpxchg on the field
176 * up to five times, and then fails under the assumption that
177 * the guest is misbehaving.
178 */
179 for ( ; ; )
180 {
181 u32 scombo, prev_scombo, new_scombo;
183 if ( unlikely((sflags & GTF_type_mask) != GTF_permit_access) ||
184 unlikely(sdom != led->domain->domain_id) )
185 PIN_FAIL(unlock_out, GNTST_general_error,
186 "Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
187 sflags, sdom, led->domain->domain_id);
189 /* Merge two 16-bit values into a 32-bit combined update. */
190 /* NB. Endianness! */
191 scombo = ((u32)sdom << 16) | (u32)sflags;
193 new_scombo = scombo | GTF_reading;
194 if ( !(op->flags & GNTMAP_readonly) )
195 {
196 new_scombo |= GTF_writing;
197 if ( unlikely(sflags & GTF_readonly) )
198 PIN_FAIL(unlock_out, GNTST_general_error,
199 "Attempt to write-pin a r/o grant entry.\n");
200 }
202 prev_scombo = cmpxchg((u32 *)&sha->flags, scombo, new_scombo);
204 /* Did the combined update work (did we see what we expected?). */
205 if ( likely(prev_scombo == scombo) )
206 break;
208 if ( retries++ == 4 )
209 PIN_FAIL(unlock_out, GNTST_general_error,
210 "Shared grant entry is unstable.\n");
212 /* Didn't see what we expected. Split out the seen flags & dom. */
213 /* NB. Endianness! */
214 sflags = (u16)prev_scombo;
215 sdom = (u16)(prev_scombo >> 16);
216 }
218 if ( !act->pin )
219 {
220 act->domid = sdom;
221 act->frame = gmfn_to_mfn(rd, sha->frame);
222 }
223 }
224 else if ( (act->pin & 0x80808080U) != 0 )
225 PIN_FAIL(unlock_out, ENOSPC,
226 "Risk of counter overflow %08x\n", act->pin);
228 if ( op->flags & GNTMAP_device_map )
229 act->pin += (op->flags & GNTMAP_readonly) ?
230 GNTPIN_devr_inc : GNTPIN_devw_inc;
231 if ( op->flags & GNTMAP_host_map )
232 act->pin += (op->flags & GNTMAP_readonly) ?
233 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
235 spin_unlock(&rd->grant_table->lock);
237 frame = act->frame;
238 if ( unlikely(!mfn_valid(frame)) ||
239 unlikely(!((op->flags & GNTMAP_readonly) ?
240 get_page(mfn_to_page(frame), rd) :
241 get_page_and_type(mfn_to_page(frame), rd,
242 PGT_writable_page))) )
243 PIN_FAIL(undo_out, GNTST_general_error,
244 "Could not pin the granted frame (%lx)!\n", frame);
246 if ( op->flags & GNTMAP_host_map )
247 {
248 rc = create_grant_host_mapping(op->host_addr, frame, op->flags);
249 if ( rc != GNTST_okay )
250 {
251 if ( !(op->flags & GNTMAP_readonly) )
252 put_page_type(mfn_to_page(frame));
253 put_page(mfn_to_page(frame));
254 goto undo_out;
255 }
257 if ( op->flags & GNTMAP_device_map )
258 {
259 (void)get_page(mfn_to_page(frame), rd);
260 if ( !(op->flags & GNTMAP_readonly) )
261 get_page_type(mfn_to_page(frame), PGT_writable_page);
262 }
263 }
265 TRACE_1D(TRC_MEM_PAGE_GRANT_MAP, op->dom);
267 ld->grant_table->maptrack[handle].domid = op->dom;
268 ld->grant_table->maptrack[handle].ref = op->ref;
269 ld->grant_table->maptrack[handle].flags = op->flags;
271 op->dev_bus_addr = (u64)frame << PAGE_SHIFT;
272 op->handle = handle;
273 op->status = GNTST_okay;
275 put_domain(rd);
276 return;
278 undo_out:
279 spin_lock(&rd->grant_table->lock);
281 if ( op->flags & GNTMAP_device_map )
282 act->pin -= (op->flags & GNTMAP_readonly) ?
283 GNTPIN_devr_inc : GNTPIN_devw_inc;
284 if ( op->flags & GNTMAP_host_map )
285 act->pin -= (op->flags & GNTMAP_readonly) ?
286 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
288 if ( !(op->flags & GNTMAP_readonly) &&
289 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
290 gnttab_clear_flag(_GTF_writing, &sha->flags);
292 if ( !act->pin )
293 gnttab_clear_flag(_GTF_reading, &sha->flags);
295 unlock_out:
296 spin_unlock(&rd->grant_table->lock);
297 op->status = rc;
298 put_maptrack_handle(ld->grant_table, handle);
299 put_domain(rd);
300 }
302 static long
303 gnttab_map_grant_ref(
304 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) uop, unsigned int count)
305 {
306 int i;
307 struct gnttab_map_grant_ref op;
309 for ( i = 0; i < count; i++ )
310 {
311 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
312 return -EFAULT;
313 __gnttab_map_grant_ref(&op);
314 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
315 return -EFAULT;
316 }
318 return 0;
319 }
321 static void
322 __gnttab_unmap_grant_ref(
323 struct gnttab_unmap_grant_ref *op)
324 {
325 domid_t dom;
326 grant_ref_t ref;
327 struct domain *ld, *rd;
328 struct active_grant_entry *act;
329 grant_entry_t *sha;
330 struct grant_mapping *map;
331 u16 flags;
332 s16 rc = 0;
333 unsigned long frame;
335 ld = current->domain;
337 frame = (unsigned long)(op->dev_bus_addr >> PAGE_SHIFT);
339 map = &ld->grant_table->maptrack[op->handle];
341 if ( unlikely(op->handle >= ld->grant_table->maptrack_limit) ||
342 unlikely(!map->flags) )
343 {
344 DPRINTK("Bad handle (%d).\n", op->handle);
345 op->status = GNTST_bad_handle;
346 return;
347 }
349 dom = map->domid;
350 ref = map->ref;
351 flags = map->flags;
353 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) )
354 {
355 if ( rd != NULL )
356 put_domain(rd);
357 DPRINTK("Could not find domain %d\n", dom);
358 op->status = GNTST_bad_domain;
359 return;
360 }
362 TRACE_1D(TRC_MEM_PAGE_GRANT_UNMAP, dom);
364 act = &rd->grant_table->active[ref];
365 sha = &rd->grant_table->shared[ref];
367 spin_lock(&rd->grant_table->lock);
369 if ( frame == 0 )
370 {
371 frame = act->frame;
372 }
373 else
374 {
375 if ( unlikely(frame != act->frame) )
376 PIN_FAIL(unmap_out, GNTST_general_error,
377 "Bad frame number doesn't match gntref.\n");
378 if ( flags & GNTMAP_device_map )
379 {
380 ASSERT(act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask));
381 map->flags &= ~GNTMAP_device_map;
382 if ( flags & GNTMAP_readonly )
383 {
384 act->pin -= GNTPIN_devr_inc;
385 put_page(mfn_to_page(frame));
386 }
387 else
388 {
389 act->pin -= GNTPIN_devw_inc;
390 put_page_and_type(mfn_to_page(frame));
391 }
392 }
393 }
395 if ( (op->host_addr != 0) && (flags & GNTMAP_host_map) )
396 {
397 if ( (rc = destroy_grant_host_mapping(op->host_addr,
398 frame, flags)) < 0 )
399 goto unmap_out;
401 ASSERT(act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask));
402 map->flags &= ~GNTMAP_host_map;
403 if ( flags & GNTMAP_readonly )
404 {
405 act->pin -= GNTPIN_hstr_inc;
406 put_page(mfn_to_page(frame));
407 }
408 else
409 {
410 act->pin -= GNTPIN_hstw_inc;
411 put_page_and_type(mfn_to_page(frame));
412 }
413 }
415 if ( (map->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0 )
416 {
417 map->flags = 0;
418 put_maptrack_handle(ld->grant_table, op->handle);
419 }
421 /* If just unmapped a writable mapping, mark as dirtied */
422 if ( !(flags & GNTMAP_readonly) )
423 gnttab_log_dirty(rd, frame);
425 if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0) &&
426 !(flags & GNTMAP_readonly) )
427 gnttab_clear_flag(_GTF_writing, &sha->flags);
429 if ( act->pin == 0 )
430 gnttab_clear_flag(_GTF_reading, &sha->flags);
432 unmap_out:
433 op->status = rc;
434 spin_unlock(&rd->grant_table->lock);
435 put_domain(rd);
436 }
438 static long
439 gnttab_unmap_grant_ref(
440 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) uop, unsigned int count)
441 {
442 int i;
443 struct gnttab_unmap_grant_ref op;
445 for ( i = 0; i < count; i++ )
446 {
447 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
448 goto fault;
449 __gnttab_unmap_grant_ref(&op);
450 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
451 goto fault;
452 }
454 flush_tlb_mask(current->domain->domain_dirty_cpumask);
455 return 0;
457 fault:
458 flush_tlb_mask(current->domain->domain_dirty_cpumask);
459 return -EFAULT;
460 }
462 static long
463 gnttab_setup_table(
464 XEN_GUEST_HANDLE(gnttab_setup_table_t) uop, unsigned int count)
465 {
466 struct gnttab_setup_table op;
467 struct domain *d;
468 int i;
469 unsigned long gmfn;
470 domid_t dom;
472 if ( count != 1 )
473 return -EINVAL;
475 if ( unlikely(copy_from_guest(&op, uop, 1) != 0) )
476 {
477 DPRINTK("Fault while reading gnttab_setup_table_t.\n");
478 return -EFAULT;
479 }
481 if ( unlikely(op.nr_frames > NR_GRANT_FRAMES) )
482 {
483 DPRINTK("Xen only supports up to %d grant-table frames per domain.\n",
484 NR_GRANT_FRAMES);
485 op.status = GNTST_general_error;
486 goto out;
487 }
489 dom = op.dom;
490 if ( dom == DOMID_SELF )
491 {
492 dom = current->domain->domain_id;
493 }
494 else if ( unlikely(!IS_PRIV(current->domain)) )
495 {
496 op.status = GNTST_permission_denied;
497 goto out;
498 }
500 if ( unlikely((d = find_domain_by_id(dom)) == NULL) )
501 {
502 DPRINTK("Bad domid %d.\n", dom);
503 op.status = GNTST_bad_domain;
504 goto out;
505 }
507 ASSERT(d->grant_table != NULL);
508 op.status = GNTST_okay;
509 for ( i = 0; i < op.nr_frames; i++ )
510 {
511 gmfn = gnttab_shared_gmfn(d, d->grant_table, i);
512 (void)copy_to_guest_offset(op.frame_list, i, &gmfn, 1);
513 }
515 put_domain(d);
517 out:
518 if ( unlikely(copy_to_guest(uop, &op, 1)) )
519 return -EFAULT;
521 return 0;
522 }
524 /*
525 * Check that the given grant reference (rd,ref) allows 'ld' to transfer
526 * ownership of a page frame. If so, lock down the grant entry.
527 */
528 static int
529 gnttab_prepare_for_transfer(
530 struct domain *rd, struct domain *ld, grant_ref_t ref)
531 {
532 struct grant_table *rgt;
533 struct grant_entry *sha;
534 domid_t sdom;
535 u16 sflags;
536 u32 scombo, prev_scombo;
537 int retries = 0;
539 if ( unlikely((rgt = rd->grant_table) == NULL) ||
540 unlikely(ref >= NR_GRANT_ENTRIES) )
541 {
542 DPRINTK("Dom %d has no g.t., or ref is bad (%d).\n",
543 rd->domain_id, ref);
544 return 0;
545 }
547 spin_lock(&rgt->lock);
549 sha = &rgt->shared[ref];
551 sflags = sha->flags;
552 sdom = sha->domid;
554 for ( ; ; )
555 {
556 if ( unlikely(sflags != GTF_accept_transfer) ||
557 unlikely(sdom != ld->domain_id) )
558 {
559 DPRINTK("Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
560 sflags, sdom, ld->domain_id);
561 goto fail;
562 }
564 /* Merge two 16-bit values into a 32-bit combined update. */
565 /* NB. Endianness! */
566 scombo = ((u32)sdom << 16) | (u32)sflags;
568 prev_scombo = cmpxchg((u32 *)&sha->flags, scombo,
569 scombo | GTF_transfer_committed);
571 /* Did the combined update work (did we see what we expected?). */
572 if ( likely(prev_scombo == scombo) )
573 break;
575 if ( retries++ == 4 )
576 {
577 DPRINTK("Shared grant entry is unstable.\n");
578 goto fail;
579 }
581 /* Didn't see what we expected. Split out the seen flags & dom. */
582 /* NB. Endianness! */
583 sflags = (u16)prev_scombo;
584 sdom = (u16)(prev_scombo >> 16);
585 }
587 spin_unlock(&rgt->lock);
588 return 1;
590 fail:
591 spin_unlock(&rgt->lock);
592 return 0;
593 }
595 static long
596 gnttab_transfer(
597 XEN_GUEST_HANDLE(gnttab_transfer_t) uop, unsigned int count)
598 {
599 struct domain *d = current->domain;
600 struct domain *e;
601 struct page_info *page;
602 int i;
603 grant_entry_t *sha;
604 struct gnttab_transfer gop;
605 unsigned long mfn;
607 for ( i = 0; i < count; i++ )
608 {
609 /* Read from caller address space. */
610 if ( unlikely(__copy_from_guest_offset(&gop, uop, i, 1)) )
611 {
612 DPRINTK("gnttab_transfer: error reading req %d/%d\n", i, count);
613 return -EFAULT;
614 }
616 mfn = gmfn_to_mfn(d, gop.mfn);
618 /* Check the passed page frame for basic validity. */
619 if ( unlikely(!mfn_valid(mfn)) )
620 {
621 DPRINTK("gnttab_transfer: out-of-range %lx\n",
622 (unsigned long)gop.mfn);
623 gop.status = GNTST_bad_page;
624 goto copyback;
625 }
627 page = mfn_to_page(mfn);
628 if ( unlikely(IS_XEN_HEAP_FRAME(page)) )
629 {
630 DPRINTK("gnttab_transfer: xen frame %lx\n",
631 (unsigned long)gop.mfn);
632 gop.status = GNTST_bad_page;
633 goto copyback;
634 }
636 if ( steal_page(d, page, 0) < 0 )
637 {
638 gop.status = GNTST_bad_page;
639 goto copyback;
640 }
642 /* Find the target domain. */
643 if ( unlikely((e = find_domain_by_id(gop.domid)) == NULL) )
644 {
645 DPRINTK("gnttab_transfer: can't find domain %d\n", gop.domid);
646 page->count_info &= ~(PGC_count_mask|PGC_allocated);
647 free_domheap_page(page);
648 gop.status = GNTST_bad_domain;
649 goto copyback;
650 }
652 spin_lock(&e->page_alloc_lock);
654 /*
655 * Check that 'e' will accept the page and has reservation
656 * headroom. Also, a domain mustn't have PGC_allocated
657 * pages when it is dying.
658 */
659 if ( unlikely(test_bit(_DOMF_dying, &e->domain_flags)) ||
660 unlikely(e->tot_pages >= e->max_pages) ||
661 unlikely(!gnttab_prepare_for_transfer(e, d, gop.ref)) )
662 {
663 if ( !test_bit(_DOMF_dying, &e->domain_flags) )
664 DPRINTK("gnttab_transfer: Transferee has no reservation "
665 "headroom (%d,%d) or provided a bad grant ref (%08x) "
666 "or is dying (%lx)\n",
667 e->tot_pages, e->max_pages, gop.ref, e->domain_flags);
668 spin_unlock(&e->page_alloc_lock);
669 put_domain(e);
670 page->count_info &= ~(PGC_count_mask|PGC_allocated);
671 free_domheap_page(page);
672 gop.status = GNTST_general_error;
673 goto copyback;
674 }
676 /* Okay, add the page to 'e'. */
677 if ( unlikely(e->tot_pages++ == 0) )
678 get_knownalive_domain(e);
679 list_add_tail(&page->list, &e->page_list);
680 page_set_owner(page, e);
682 spin_unlock(&e->page_alloc_lock);
684 TRACE_1D(TRC_MEM_PAGE_GRANT_TRANSFER, e->domain_id);
686 /* Tell the guest about its new page frame. */
687 sha = &e->grant_table->shared[gop.ref];
688 guest_physmap_add_page(e, sha->frame, mfn);
689 sha->frame = mfn;
690 wmb();
691 sha->flags |= GTF_transfer_completed;
693 put_domain(e);
695 gop.status = GNTST_okay;
697 copyback:
698 if ( unlikely(__copy_to_guest_offset(uop, i, &gop, 1)) )
699 {
700 DPRINTK("gnttab_transfer: error writing resp %d/%d\n", i, count);
701 return -EFAULT;
702 }
703 }
705 return 0;
706 }
708 /* Undo __acquire_grant_for_copy. Again, this has no effect on page
709 type and reference counts. */
710 static void
711 __release_grant_for_copy(
712 struct domain *rd, unsigned long gref, int readonly)
713 {
714 grant_entry_t *const sha = &rd->grant_table->shared[gref];
715 struct active_grant_entry *const act = &rd->grant_table->active[gref];
716 const unsigned long r_frame = act->frame;
718 if ( !readonly )
719 gnttab_log_dirty(rd, r_frame);
721 spin_lock(&rd->grant_table->lock);
722 if ( readonly )
723 act->pin -= GNTPIN_hstr_inc;
724 else
725 act->pin -= GNTPIN_hstw_inc;
727 if ( !(act->pin & GNTPIN_hstw_mask) && !readonly )
728 gnttab_clear_flag(_GTF_writing, &sha->flags);
730 if ( !act->pin )
731 gnttab_clear_flag(_GTF_reading, &sha->flags);
732 spin_unlock(&rd->grant_table->lock);
733 }
735 /* Grab a frame number from a grant entry and update the flags and pin
736 count as appropriate. Note that this does *not* update the page
737 type or reference counts. */
738 static int
739 __acquire_grant_for_copy(
740 struct domain *rd, unsigned long gref, int readonly,
741 unsigned long *frame)
742 {
743 grant_entry_t *sha;
744 struct active_grant_entry *act;
745 s16 rc = GNTST_okay;
746 int retries = 0;
747 u16 sflags;
748 domid_t sdom;
750 if ( unlikely(gref >= NR_GRANT_ENTRIES) )
751 PIN_FAIL(error_out, GNTST_bad_gntref,
752 "Bad grant reference %ld\n", gref);
754 act = &rd->grant_table->active[gref];
755 sha = &rd->grant_table->shared[gref];
757 spin_lock(&rd->grant_table->lock);
759 if ( !act->pin ||
760 (!readonly && !(act->pin & GNTPIN_hstw_mask)) )
761 {
762 sflags = sha->flags;
763 sdom = sha->domid;
765 for ( ; ; )
766 {
767 u32 scombo;
768 u32 prev_scombo;
769 u32 new_scombo;
771 if ( unlikely((sflags & GTF_type_mask) != GTF_permit_access ||
772 sdom != current->domain->domain_id ) )
773 PIN_FAIL(unlock_out, GNTST_general_error,
774 "Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
775 sflags, sdom, current->domain->domain_id);
776 scombo = ((u32)sdom << 16) | (u32)sflags;
777 new_scombo = scombo | GTF_reading;
778 if ( !readonly )
779 {
780 new_scombo |= GTF_writing;
781 if ( unlikely(sflags & GTF_readonly) )
782 PIN_FAIL(unlock_out, GNTST_general_error,
783 "Attempt to write-pin a r/o grant entry.\n");
784 }
785 prev_scombo = cmpxchg((u32 *)&sha->flags, scombo, new_scombo);
786 if ( likely(prev_scombo == scombo) )
787 break;
789 if ( retries++ == 4 )
790 PIN_FAIL(unlock_out, GNTST_general_error,
791 "Shared grant entry is unstable.\n");
792 sflags = (u16)prev_scombo;
793 sdom = (u16)(prev_scombo >> 16);
794 }
796 if ( !act->pin )
797 {
798 act->domid = sdom;
799 act->frame = gmfn_to_mfn(rd, sha->frame);
800 }
801 }
802 else if ( (act->pin & 0x80808080U) != 0 )
803 PIN_FAIL(unlock_out, ENOSPC,
804 "Risk of counter overflow %08x\n", act->pin);
806 act->pin += readonly ? GNTPIN_hstr_inc : GNTPIN_hstw_inc;
808 *frame = act->frame;
810 unlock_out:
811 spin_unlock(&rd->grant_table->lock);
812 error_out:
813 return rc;
814 }
816 static void
817 __gnttab_copy(
818 struct gnttab_copy *op)
819 {
820 struct domain *sd = NULL, *dd = NULL;
821 unsigned long s_frame, d_frame;
822 char *sp, *dp;
823 s16 rc = GNTST_okay;
824 int have_d_grant = 0, have_s_grant = 0, have_s_ref = 0;
825 int src_is_gref, dest_is_gref;
827 if ( ((op->source.offset + op->len) > PAGE_SIZE) ||
828 ((op->dest.offset + op->len) > PAGE_SIZE) )
829 PIN_FAIL(error_out, GNTST_bad_copy_arg, "copy beyond page area.\n");
831 src_is_gref = op->flags & GNTCOPY_source_gref;
832 dest_is_gref = op->flags & GNTCOPY_dest_gref;
834 if ( (op->source.domid != DOMID_SELF && !src_is_gref ) ||
835 (op->dest.domid != DOMID_SELF && !dest_is_gref) )
836 PIN_FAIL(error_out, GNTST_permission_denied,
837 "only allow copy-by-mfn for DOMID_SELF.\n");
839 if ( op->source.domid == DOMID_SELF )
840 {
841 sd = current->domain;
842 get_knownalive_domain(sd);
843 }
844 else if ( (sd = find_domain_by_id(op->source.domid)) == NULL )
845 {
846 PIN_FAIL(error_out, GNTST_bad_domain,
847 "couldn't find %d\n", op->source.domid);
848 }
850 if ( op->dest.domid == DOMID_SELF )
851 {
852 dd = current->domain;
853 get_knownalive_domain(dd);
854 }
855 else if ( (dd = find_domain_by_id(op->dest.domid)) == NULL )
856 {
857 PIN_FAIL(error_out, GNTST_bad_domain,
858 "couldn't find %d\n", op->dest.domid);
859 }
861 if ( src_is_gref )
862 {
863 rc = __acquire_grant_for_copy(sd, op->source.u.ref, 1, &s_frame);
864 if ( rc != GNTST_okay )
865 goto error_out;
866 have_s_grant = 1;
867 }
868 else
869 {
870 s_frame = gmfn_to_mfn(sd, op->source.u.gmfn);
871 }
872 if ( !get_page(mfn_to_page(s_frame), sd) )
873 PIN_FAIL(error_out, GNTST_general_error,
874 "could not get source frame %lx.\n", s_frame);
875 have_s_ref = 1;
877 if ( dest_is_gref )
878 {
879 rc = __acquire_grant_for_copy(dd, op->dest.u.ref, 0, &d_frame);
880 if ( rc != GNTST_okay )
881 goto error_out;
882 have_d_grant = 1;
883 }
884 else
885 {
886 d_frame = gmfn_to_mfn(sd, op->dest.u.gmfn);
887 }
888 if ( !get_page_and_type(mfn_to_page(d_frame), dd, PGT_writable_page) )
889 PIN_FAIL(error_out, GNTST_general_error,
890 "could not get source frame %lx.\n", d_frame);
892 sp = map_domain_page(s_frame);
893 dp = map_domain_page(d_frame);
895 memcpy(dp + op->dest.offset, sp + op->source.offset, op->len);
897 unmap_domain_page(dp);
898 unmap_domain_page(sp);
900 put_page_and_type(mfn_to_page(d_frame));
901 error_out:
902 if ( have_s_ref )
903 put_page(mfn_to_page(s_frame));
904 if ( have_s_grant )
905 __release_grant_for_copy(sd, op->source.u.ref, 1);
906 if ( have_d_grant )
907 __release_grant_for_copy(dd, op->dest.u.ref, 0);
908 if ( sd )
909 put_domain(sd);
910 if ( dd )
911 put_domain(dd);
912 op->status = rc;
913 }
915 static long
916 gnttab_copy(
917 XEN_GUEST_HANDLE(gnttab_copy_t) uop, unsigned int count)
918 {
919 int i;
920 struct gnttab_copy op;
922 for ( i = 0; i < count; i++ )
923 {
924 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
925 return -EFAULT;
926 __gnttab_copy(&op);
927 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
928 return -EFAULT;
929 }
930 return 0;
931 }
933 long
934 do_grant_table_op(
935 unsigned int cmd, XEN_GUEST_HANDLE(void) uop, unsigned int count)
936 {
937 long rc;
938 struct domain *d = current->domain;
940 if ( count > 512 )
941 return -EINVAL;
943 LOCK_BIGLOCK(d);
945 sync_pagetable_state(d);
947 rc = -EFAULT;
948 switch ( cmd )
949 {
950 case GNTTABOP_map_grant_ref:
951 {
952 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) map =
953 guest_handle_cast(uop, gnttab_map_grant_ref_t);
954 if ( unlikely(!guest_handle_okay(map, count)) )
955 goto out;
956 rc = gnttab_map_grant_ref(map, count);
957 break;
958 }
959 case GNTTABOP_unmap_grant_ref:
960 {
961 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) unmap =
962 guest_handle_cast(uop, gnttab_unmap_grant_ref_t);
963 if ( unlikely(!guest_handle_okay(unmap, count)) )
964 goto out;
965 rc = gnttab_unmap_grant_ref(unmap, count);
966 break;
967 }
968 case GNTTABOP_setup_table:
969 {
970 rc = gnttab_setup_table(
971 guest_handle_cast(uop, gnttab_setup_table_t), count);
972 break;
973 }
974 case GNTTABOP_transfer:
975 {
976 XEN_GUEST_HANDLE(gnttab_transfer_t) transfer =
977 guest_handle_cast(uop, gnttab_transfer_t);
978 if ( unlikely(!guest_handle_okay(transfer, count)) )
979 goto out;
980 rc = gnttab_transfer(transfer, count);
981 break;
982 }
983 case GNTTABOP_copy:
984 {
985 XEN_GUEST_HANDLE(gnttab_copy_t) copy =
986 guest_handle_cast(uop, gnttab_copy_t);
987 if ( unlikely(!guest_handle_okay(copy, count)) )
988 goto out;
989 rc = gnttab_copy(copy, count);
990 break;
991 }
992 default:
993 rc = -ENOSYS;
994 break;
995 }
997 out:
998 UNLOCK_BIGLOCK(d);
1000 return rc;
1003 int
1004 grant_table_create(
1005 struct domain *d)
1007 struct grant_table *t;
1008 int i;
1010 BUG_ON(MAPTRACK_MAX_ENTRIES < NR_GRANT_ENTRIES);
1011 if ( (t = xmalloc(struct grant_table)) == NULL )
1012 goto no_mem;
1014 /* Simple stuff. */
1015 memset(t, 0, sizeof(*t));
1016 spin_lock_init(&t->lock);
1018 /* Active grant table. */
1019 t->active = xmalloc_array(struct active_grant_entry, NR_GRANT_ENTRIES);
1020 if ( t->active == NULL )
1021 goto no_mem;
1022 memset(t->active, 0, sizeof(struct active_grant_entry) * NR_GRANT_ENTRIES);
1024 /* Tracking of mapped foreign frames table */
1025 if ( (t->maptrack = alloc_xenheap_page()) == NULL )
1026 goto no_mem;
1027 t->maptrack_order = 0;
1028 t->maptrack_limit = PAGE_SIZE / sizeof(struct grant_mapping);
1029 memset(t->maptrack, 0, PAGE_SIZE);
1030 for ( i = 0; i < t->maptrack_limit; i++ )
1031 t->maptrack[i].ref = i+1;
1033 /* Shared grant table. */
1034 t->shared = alloc_xenheap_pages(ORDER_GRANT_FRAMES);
1035 if ( t->shared == NULL )
1036 goto no_mem;
1037 memset(t->shared, 0, NR_GRANT_FRAMES * PAGE_SIZE);
1039 for ( i = 0; i < NR_GRANT_FRAMES; i++ )
1040 gnttab_create_shared_page(d, t, i);
1042 /* Okay, install the structure. */
1043 wmb(); /* avoid races with lock-free access to d->grant_table */
1044 d->grant_table = t;
1045 return 0;
1047 no_mem:
1048 if ( t != NULL )
1050 xfree(t->active);
1051 free_xenheap_page(t->maptrack);
1052 xfree(t);
1054 return -ENOMEM;
1057 void
1058 gnttab_release_mappings(
1059 struct domain *d)
1061 struct grant_table *gt = d->grant_table;
1062 struct grant_mapping *map;
1063 grant_ref_t ref;
1064 grant_handle_t handle;
1065 struct domain *rd;
1066 struct active_grant_entry *act;
1067 struct grant_entry *sha;
1069 BUG_ON(!test_bit(_DOMF_dying, &d->domain_flags));
1071 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
1073 map = &gt->maptrack[handle];
1074 if ( !(map->flags & (GNTMAP_device_map|GNTMAP_host_map)) )
1075 continue;
1077 ref = map->ref;
1079 DPRINTK("Grant release (%hu) ref:(%hu) flags:(%x) dom:(%hu)\n",
1080 handle, ref, map->flags, map->domid);
1082 rd = find_domain_by_id(map->domid);
1083 BUG_ON(rd == NULL);
1085 spin_lock(&rd->grant_table->lock);
1087 act = &rd->grant_table->active[ref];
1088 sha = &rd->grant_table->shared[ref];
1090 if ( map->flags & GNTMAP_readonly )
1092 if ( map->flags & GNTMAP_device_map )
1094 BUG_ON(!(act->pin & GNTPIN_devr_mask));
1095 act->pin -= GNTPIN_devr_inc;
1096 put_page(mfn_to_page(act->frame));
1099 if ( map->flags & GNTMAP_host_map )
1101 BUG_ON(!(act->pin & GNTPIN_hstr_mask));
1102 act->pin -= GNTPIN_hstr_inc;
1103 /* Done implicitly when page tables are destroyed. */
1104 /* put_page(mfn_to_page(act->frame)); */
1107 else
1109 if ( map->flags & GNTMAP_device_map )
1111 BUG_ON(!(act->pin & GNTPIN_devw_mask));
1112 act->pin -= GNTPIN_devw_inc;
1113 put_page_and_type(mfn_to_page(act->frame));
1116 if ( map->flags & GNTMAP_host_map )
1118 BUG_ON(!(act->pin & GNTPIN_hstw_mask));
1119 act->pin -= GNTPIN_hstw_inc;
1120 /* Done implicitly when page tables are destroyed. */
1121 /* put_page_and_type(mfn_to_page(act->frame)); */
1124 if ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0 )
1125 gnttab_clear_flag(_GTF_writing, &sha->flags);
1128 if ( act->pin == 0 )
1129 gnttab_clear_flag(_GTF_reading, &sha->flags);
1131 spin_unlock(&rd->grant_table->lock);
1133 put_domain(rd);
1135 map->flags = 0;
1140 void
1141 grant_table_destroy(
1142 struct domain *d)
1144 struct grant_table *t = d->grant_table;
1146 if ( t == NULL )
1147 return;
1149 free_xenheap_pages(t->shared, ORDER_GRANT_FRAMES);
1150 free_xenheap_page(t->maptrack);
1151 xfree(t->active);
1152 xfree(t);
1154 d->grant_table = NULL;
1157 /*
1158 * Local variables:
1159 * mode: C
1160 * c-set-style: "BSD"
1161 * c-basic-offset: 4
1162 * tab-width: 4
1163 * indent-tabs-mode: nil
1164 * End:
1165 */