ia64/xen-unstable

view xen/common/grant_table.c @ 12390:e28beea6d228

[IA64] Fix time services of EFI emulation

This patch serializes the execution of following efi.runtimes.
- GetTime
- SetTime
- GetWakeTime
- SetWakeTime

Linux/ia64 uses similar spinlocks in the EFI RTC driver.

Signed-off-by: Masaki Kanno <kanno.masaki@jp.fujitsu.com>
author awilliam@xenbuild.aw
date Fri Nov 10 12:03:19 2006 -0700 (2006-11-10)
parents 5e0fb830a53c
children 781ea5017f18
line source
1 /******************************************************************************
2 * common/grant_table.c
3 *
4 * Mechanism for granting foreign access to page frames, and receiving
5 * page-ownership transfers.
6 *
7 * Copyright (c) 2005 Christopher Clark
8 * Copyright (c) 2004 K A Fraser
9 * Copyright (c) 2005 Andrew Warfield
10 * Modifications by Geoffrey Lefebvre are (c) Intel Research Cambridge
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software
24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 */
27 #include <xen/lib.h>
28 #include <xen/sched.h>
29 #include <xen/shadow.h>
30 #include <xen/mm.h>
31 #include <xen/trace.h>
32 #include <xen/guest_access.h>
33 #include <xen/domain_page.h>
34 #include <acm/acm_hooks.h>
36 /*
37 * The first two members of a grant entry are updated as a combined pair.
38 * The following union allows that to happen in an endian-neutral fashion.
39 */
40 union grant_combo {
41 uint32_t word;
42 struct {
43 uint16_t flags;
44 domid_t domid;
45 } shorts;
46 };
48 #define PIN_FAIL(_lbl, _rc, _f, _a...) \
49 do { \
50 gdprintk(XENLOG_WARNING, _f, ## _a ); \
51 rc = (_rc); \
52 goto _lbl; \
53 } while ( 0 )
55 static inline int
56 get_maptrack_handle(
57 struct grant_table *t)
58 {
59 unsigned int h;
60 if ( unlikely((h = t->maptrack_head) == (t->maptrack_limit - 1)) )
61 return -1;
62 t->maptrack_head = t->maptrack[h].ref;
63 t->map_count++;
64 return h;
65 }
67 static inline void
68 put_maptrack_handle(
69 struct grant_table *t, int handle)
70 {
71 t->maptrack[handle].ref = t->maptrack_head;
72 t->maptrack_head = handle;
73 t->map_count--;
74 }
76 /*
77 * Returns 0 if TLB flush / invalidate required by caller.
78 * va will indicate the address to be invalidated.
79 *
80 * addr is _either_ a host virtual address, or the address of the pte to
81 * update, as indicated by the GNTMAP_contains_pte flag.
82 */
83 static void
84 __gnttab_map_grant_ref(
85 struct gnttab_map_grant_ref *op)
86 {
87 struct domain *ld, *rd;
88 struct vcpu *led;
89 int handle;
90 unsigned long frame = 0;
91 int rc = GNTST_okay;
92 struct active_grant_entry *act;
93 grant_entry_t *sha;
94 union grant_combo scombo, prev_scombo, new_scombo;
96 /*
97 * We bound the number of times we retry CMPXCHG on memory locations that
98 * we share with a guest OS. The reason is that the guest can modify that
99 * location at a higher rate than we can read-modify-CMPXCHG, so the guest
100 * could cause us to livelock. There are a few cases where it is valid for
101 * the guest to race our updates (e.g., to change the GTF_readonly flag),
102 * so we allow a few retries before failing.
103 */
104 int retries = 0;
106 led = current;
107 ld = led->domain;
109 if ( unlikely(op->ref >= NR_GRANT_ENTRIES) ||
110 unlikely((op->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0) )
111 {
112 gdprintk(XENLOG_INFO, "Bad ref (%d) or flags (%x).\n",
113 op->ref, op->flags);
114 op->status = GNTST_bad_gntref;
115 return;
116 }
118 if ( acm_pre_grant_map_ref(op->dom) )
119 {
120 op->status = GNTST_permission_denied;
121 return;
122 }
124 if ( unlikely((rd = find_domain_by_id(op->dom)) == NULL) )
125 {
126 if ( rd != NULL )
127 put_domain(rd);
128 gdprintk(XENLOG_INFO, "Could not find domain %d\n", op->dom);
129 op->status = GNTST_bad_domain;
130 return;
131 }
133 /* Get a maptrack handle. */
134 if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
135 {
136 int i;
137 struct grant_mapping *new_mt;
138 struct grant_table *lgt = ld->grant_table;
140 if ( (lgt->maptrack_limit << 1) > MAPTRACK_MAX_ENTRIES )
141 {
142 put_domain(rd);
143 gdprintk(XENLOG_INFO, "Maptrack table is at maximum size.\n");
144 op->status = GNTST_no_device_space;
145 return;
146 }
148 /* Grow the maptrack table. */
149 new_mt = alloc_xenheap_pages(lgt->maptrack_order + 1);
150 if ( new_mt == NULL )
151 {
152 put_domain(rd);
153 gdprintk(XENLOG_INFO, "No more map handles available.\n");
154 op->status = GNTST_no_device_space;
155 return;
156 }
158 memcpy(new_mt, lgt->maptrack, PAGE_SIZE << lgt->maptrack_order);
159 for ( i = lgt->maptrack_limit; i < (lgt->maptrack_limit << 1); i++ )
160 {
161 new_mt[i].ref = i+1;
162 new_mt[i].flags = 0;
163 }
165 free_xenheap_pages(lgt->maptrack, lgt->maptrack_order);
166 lgt->maptrack = new_mt;
167 lgt->maptrack_order += 1;
168 lgt->maptrack_limit <<= 1;
170 gdprintk(XENLOG_INFO, "Doubled maptrack size\n");
171 handle = get_maptrack_handle(ld->grant_table);
172 }
174 act = &rd->grant_table->active[op->ref];
175 sha = &rd->grant_table->shared[op->ref];
177 spin_lock(&rd->grant_table->lock);
179 /* If already pinned, check the active domid and avoid refcnt overflow. */
180 if ( act->pin &&
181 ((act->domid != ld->domain_id) ||
182 (act->pin & 0x80808080U) != 0) )
183 PIN_FAIL(unlock_out, GNTST_general_error,
184 "Bad domain (%d != %d), or risk of counter overflow %08x\n",
185 act->domid, ld->domain_id, act->pin);
187 if ( !act->pin ||
188 (!(op->flags & GNTMAP_readonly) &&
189 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask))) )
190 {
191 scombo.word = *(u32 *)&sha->flags;
193 /*
194 * This loop attempts to set the access (reading/writing) flags
195 * in the grant table entry. It tries a cmpxchg on the field
196 * up to five times, and then fails under the assumption that
197 * the guest is misbehaving.
198 */
199 for ( ; ; )
200 {
201 /* If not already pinned, check the grant domid and type. */
202 if ( !act->pin &&
203 (((scombo.shorts.flags & GTF_type_mask) !=
204 GTF_permit_access) ||
205 (scombo.shorts.domid != ld->domain_id)) )
206 PIN_FAIL(unlock_out, GNTST_general_error,
207 "Bad flags (%x) or dom (%d). (expected dom %d)\n",
208 scombo.shorts.flags, scombo.shorts.domid,
209 ld->domain_id);
211 new_scombo = scombo;
212 new_scombo.shorts.flags |= GTF_reading;
214 if ( !(op->flags & GNTMAP_readonly) )
215 {
216 new_scombo.shorts.flags |= GTF_writing;
217 if ( unlikely(scombo.shorts.flags & GTF_readonly) )
218 PIN_FAIL(unlock_out, GNTST_general_error,
219 "Attempt to write-pin a r/o grant entry.\n");
220 }
222 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
223 scombo.word, new_scombo.word);
224 if ( likely(prev_scombo.word == scombo.word) )
225 break;
227 if ( retries++ == 4 )
228 PIN_FAIL(unlock_out, GNTST_general_error,
229 "Shared grant entry is unstable.\n");
231 scombo = prev_scombo;
232 }
234 if ( !act->pin )
235 {
236 act->domid = scombo.shorts.domid;
237 act->frame = gmfn_to_mfn(rd, sha->frame);
238 }
239 }
241 if ( op->flags & GNTMAP_device_map )
242 act->pin += (op->flags & GNTMAP_readonly) ?
243 GNTPIN_devr_inc : GNTPIN_devw_inc;
244 if ( op->flags & GNTMAP_host_map )
245 act->pin += (op->flags & GNTMAP_readonly) ?
246 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
248 spin_unlock(&rd->grant_table->lock);
250 frame = act->frame;
251 if ( unlikely(!mfn_valid(frame)) ||
252 unlikely(!((op->flags & GNTMAP_readonly) ?
253 get_page(mfn_to_page(frame), rd) :
254 get_page_and_type(mfn_to_page(frame), rd,
255 PGT_writable_page))) )
256 {
257 if ( !test_bit(_DOMF_dying, &rd->domain_flags) )
258 gdprintk(XENLOG_WARNING, "Could not pin grant frame %lx\n", frame);
259 rc = GNTST_general_error;
260 goto undo_out;
261 }
263 if ( op->flags & GNTMAP_host_map )
264 {
265 rc = create_grant_host_mapping(op->host_addr, frame, op->flags);
266 if ( rc != GNTST_okay )
267 {
268 if ( !(op->flags & GNTMAP_readonly) )
269 put_page_type(mfn_to_page(frame));
270 put_page(mfn_to_page(frame));
271 goto undo_out;
272 }
274 if ( op->flags & GNTMAP_device_map )
275 {
276 (void)get_page(mfn_to_page(frame), rd);
277 if ( !(op->flags & GNTMAP_readonly) )
278 get_page_type(mfn_to_page(frame), PGT_writable_page);
279 }
280 }
282 TRACE_1D(TRC_MEM_PAGE_GRANT_MAP, op->dom);
284 ld->grant_table->maptrack[handle].domid = op->dom;
285 ld->grant_table->maptrack[handle].ref = op->ref;
286 ld->grant_table->maptrack[handle].flags = op->flags;
288 op->dev_bus_addr = (u64)frame << PAGE_SHIFT;
289 op->handle = handle;
290 op->status = GNTST_okay;
292 put_domain(rd);
293 return;
295 undo_out:
296 spin_lock(&rd->grant_table->lock);
298 if ( op->flags & GNTMAP_device_map )
299 act->pin -= (op->flags & GNTMAP_readonly) ?
300 GNTPIN_devr_inc : GNTPIN_devw_inc;
301 if ( op->flags & GNTMAP_host_map )
302 act->pin -= (op->flags & GNTMAP_readonly) ?
303 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
305 if ( !(op->flags & GNTMAP_readonly) &&
306 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
307 gnttab_clear_flag(_GTF_writing, &sha->flags);
309 if ( !act->pin )
310 gnttab_clear_flag(_GTF_reading, &sha->flags);
312 unlock_out:
313 spin_unlock(&rd->grant_table->lock);
314 op->status = rc;
315 put_maptrack_handle(ld->grant_table, handle);
316 put_domain(rd);
317 }
319 static long
320 gnttab_map_grant_ref(
321 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) uop, unsigned int count)
322 {
323 int i;
324 struct gnttab_map_grant_ref op;
326 for ( i = 0; i < count; i++ )
327 {
328 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
329 return -EFAULT;
330 __gnttab_map_grant_ref(&op);
331 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
332 return -EFAULT;
333 }
335 return 0;
336 }
338 static void
339 __gnttab_unmap_grant_ref(
340 struct gnttab_unmap_grant_ref *op)
341 {
342 domid_t dom;
343 grant_ref_t ref;
344 struct domain *ld, *rd;
345 struct active_grant_entry *act;
346 grant_entry_t *sha;
347 struct grant_mapping *map;
348 u16 flags;
349 s16 rc = 0;
350 unsigned long frame;
352 ld = current->domain;
354 frame = (unsigned long)(op->dev_bus_addr >> PAGE_SHIFT);
356 map = &ld->grant_table->maptrack[op->handle];
358 if ( unlikely(op->handle >= ld->grant_table->maptrack_limit) ||
359 unlikely(!map->flags) )
360 {
361 gdprintk(XENLOG_INFO, "Bad handle (%d).\n", op->handle);
362 op->status = GNTST_bad_handle;
363 return;
364 }
366 dom = map->domid;
367 ref = map->ref;
368 flags = map->flags;
370 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) )
371 {
372 if ( rd != NULL )
373 put_domain(rd);
374 gdprintk(XENLOG_INFO, "Could not find domain %d\n", dom);
375 op->status = GNTST_bad_domain;
376 return;
377 }
379 TRACE_1D(TRC_MEM_PAGE_GRANT_UNMAP, dom);
381 act = &rd->grant_table->active[ref];
382 sha = &rd->grant_table->shared[ref];
384 spin_lock(&rd->grant_table->lock);
386 if ( frame == 0 )
387 {
388 frame = act->frame;
389 }
390 else
391 {
392 if ( unlikely(frame != act->frame) )
393 PIN_FAIL(unmap_out, GNTST_general_error,
394 "Bad frame number doesn't match gntref.\n");
395 if ( flags & GNTMAP_device_map )
396 {
397 ASSERT(act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask));
398 map->flags &= ~GNTMAP_device_map;
399 if ( flags & GNTMAP_readonly )
400 {
401 act->pin -= GNTPIN_devr_inc;
402 put_page(mfn_to_page(frame));
403 }
404 else
405 {
406 act->pin -= GNTPIN_devw_inc;
407 put_page_and_type(mfn_to_page(frame));
408 }
409 }
410 }
412 if ( (op->host_addr != 0) && (flags & GNTMAP_host_map) )
413 {
414 if ( (rc = destroy_grant_host_mapping(op->host_addr,
415 frame, flags)) < 0 )
416 goto unmap_out;
418 ASSERT(act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask));
419 map->flags &= ~GNTMAP_host_map;
420 if ( flags & GNTMAP_readonly )
421 {
422 act->pin -= GNTPIN_hstr_inc;
423 put_page(mfn_to_page(frame));
424 }
425 else
426 {
427 act->pin -= GNTPIN_hstw_inc;
428 put_page_and_type(mfn_to_page(frame));
429 }
430 }
432 if ( (map->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0 )
433 {
434 map->flags = 0;
435 put_maptrack_handle(ld->grant_table, op->handle);
436 }
438 /* If just unmapped a writable mapping, mark as dirtied */
439 if ( !(flags & GNTMAP_readonly) )
440 gnttab_mark_dirty(rd, frame);
442 if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0) &&
443 !(flags & GNTMAP_readonly) )
444 gnttab_clear_flag(_GTF_writing, &sha->flags);
446 if ( act->pin == 0 )
447 gnttab_clear_flag(_GTF_reading, &sha->flags);
449 unmap_out:
450 op->status = rc;
451 spin_unlock(&rd->grant_table->lock);
452 put_domain(rd);
453 }
455 static long
456 gnttab_unmap_grant_ref(
457 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) uop, unsigned int count)
458 {
459 int i;
460 struct gnttab_unmap_grant_ref op;
462 for ( i = 0; i < count; i++ )
463 {
464 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
465 goto fault;
466 __gnttab_unmap_grant_ref(&op);
467 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
468 goto fault;
469 }
471 flush_tlb_mask(current->domain->domain_dirty_cpumask);
472 return 0;
474 fault:
475 flush_tlb_mask(current->domain->domain_dirty_cpumask);
476 return -EFAULT;
477 }
479 static long
480 gnttab_setup_table(
481 XEN_GUEST_HANDLE(gnttab_setup_table_t) uop, unsigned int count)
482 {
483 struct gnttab_setup_table op;
484 struct domain *d;
485 int i;
486 unsigned long gmfn;
487 domid_t dom;
489 if ( count != 1 )
490 return -EINVAL;
492 if ( unlikely(copy_from_guest(&op, uop, 1) != 0) )
493 {
494 gdprintk(XENLOG_INFO, "Fault while reading gnttab_setup_table_t.\n");
495 return -EFAULT;
496 }
498 if ( unlikely(op.nr_frames > NR_GRANT_FRAMES) )
499 {
500 gdprintk(XENLOG_INFO, "Xen only supports up to %d grant-table frames"
501 " per domain.\n",
502 NR_GRANT_FRAMES);
503 op.status = GNTST_general_error;
504 goto out;
505 }
507 dom = op.dom;
508 if ( dom == DOMID_SELF )
509 {
510 dom = current->domain->domain_id;
511 }
512 else if ( unlikely(!IS_PRIV(current->domain)) )
513 {
514 op.status = GNTST_permission_denied;
515 goto out;
516 }
518 if ( unlikely((d = find_domain_by_id(dom)) == NULL) )
519 {
520 gdprintk(XENLOG_INFO, "Bad domid %d.\n", dom);
521 op.status = GNTST_bad_domain;
522 goto out;
523 }
525 ASSERT(d->grant_table != NULL);
526 op.status = GNTST_okay;
527 for ( i = 0; i < op.nr_frames; i++ )
528 {
529 gmfn = gnttab_shared_gmfn(d, d->grant_table, i);
530 (void)copy_to_guest_offset(op.frame_list, i, &gmfn, 1);
531 }
533 put_domain(d);
535 out:
536 if ( unlikely(copy_to_guest(uop, &op, 1)) )
537 return -EFAULT;
539 return 0;
540 }
542 /*
543 * Check that the given grant reference (rd,ref) allows 'ld' to transfer
544 * ownership of a page frame. If so, lock down the grant entry.
545 */
546 static int
547 gnttab_prepare_for_transfer(
548 struct domain *rd, struct domain *ld, grant_ref_t ref)
549 {
550 struct grant_table *rgt;
551 struct grant_entry *sha;
552 union grant_combo scombo, prev_scombo, new_scombo;
553 int retries = 0;
555 if ( unlikely((rgt = rd->grant_table) == NULL) ||
556 unlikely(ref >= NR_GRANT_ENTRIES) )
557 {
558 gdprintk(XENLOG_INFO, "Dom %d has no g.t., or ref is bad (%d).\n",
559 rd->domain_id, ref);
560 return 0;
561 }
563 spin_lock(&rgt->lock);
565 sha = &rgt->shared[ref];
567 scombo.word = *(u32 *)&sha->flags;
569 for ( ; ; )
570 {
571 if ( unlikely(scombo.shorts.flags != GTF_accept_transfer) ||
572 unlikely(scombo.shorts.domid != ld->domain_id) )
573 {
574 gdprintk(XENLOG_INFO, "Bad flags (%x) or dom (%d). "
575 "(NB. expected dom %d)\n",
576 scombo.shorts.flags, scombo.shorts.domid,
577 ld->domain_id);
578 goto fail;
579 }
581 new_scombo = scombo;
582 new_scombo.shorts.flags |= GTF_transfer_committed;
584 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
585 scombo.word, new_scombo.word);
586 if ( likely(prev_scombo.word == scombo.word) )
587 break;
589 if ( retries++ == 4 )
590 {
591 gdprintk(XENLOG_WARNING, "Shared grant entry is unstable.\n");
592 goto fail;
593 }
595 scombo = prev_scombo;
596 }
598 spin_unlock(&rgt->lock);
599 return 1;
601 fail:
602 spin_unlock(&rgt->lock);
603 return 0;
604 }
606 static long
607 gnttab_transfer(
608 XEN_GUEST_HANDLE(gnttab_transfer_t) uop, unsigned int count)
609 {
610 struct domain *d = current->domain;
611 struct domain *e;
612 struct page_info *page;
613 int i;
614 grant_entry_t *sha;
615 struct gnttab_transfer gop;
616 unsigned long mfn;
618 for ( i = 0; i < count; i++ )
619 {
620 /* Read from caller address space. */
621 if ( unlikely(__copy_from_guest_offset(&gop, uop, i, 1)) )
622 {
623 gdprintk(XENLOG_INFO, "gnttab_transfer: error reading req %d/%d\n",
624 i, count);
625 return -EFAULT;
626 }
628 mfn = gmfn_to_mfn(d, gop.mfn);
630 /* Check the passed page frame for basic validity. */
631 if ( unlikely(!mfn_valid(mfn)) )
632 {
633 gdprintk(XENLOG_INFO, "gnttab_transfer: out-of-range %lx\n",
634 (unsigned long)gop.mfn);
635 gop.status = GNTST_bad_page;
636 goto copyback;
637 }
639 page = mfn_to_page(mfn);
640 if ( unlikely(IS_XEN_HEAP_FRAME(page)) )
641 {
642 gdprintk(XENLOG_INFO, "gnttab_transfer: xen frame %lx\n",
643 (unsigned long)gop.mfn);
644 gop.status = GNTST_bad_page;
645 goto copyback;
646 }
648 if ( steal_page(d, page, 0) < 0 )
649 {
650 gop.status = GNTST_bad_page;
651 goto copyback;
652 }
654 /* Find the target domain. */
655 if ( unlikely((e = find_domain_by_id(gop.domid)) == NULL) )
656 {
657 gdprintk(XENLOG_INFO, "gnttab_transfer: can't find domain %d\n",
658 gop.domid);
659 page->count_info &= ~(PGC_count_mask|PGC_allocated);
660 free_domheap_page(page);
661 gop.status = GNTST_bad_domain;
662 goto copyback;
663 }
665 spin_lock(&e->page_alloc_lock);
667 /*
668 * Check that 'e' will accept the page and has reservation
669 * headroom. Also, a domain mustn't have PGC_allocated
670 * pages when it is dying.
671 */
672 if ( unlikely(test_bit(_DOMF_dying, &e->domain_flags)) ||
673 unlikely(e->tot_pages >= e->max_pages) ||
674 unlikely(!gnttab_prepare_for_transfer(e, d, gop.ref)) )
675 {
676 if ( !test_bit(_DOMF_dying, &e->domain_flags) )
677 gdprintk(XENLOG_INFO, "gnttab_transfer: "
678 "Transferee has no reservation "
679 "headroom (%d,%d) or provided a bad grant ref (%08x) "
680 "or is dying (%lx)\n",
681 e->tot_pages, e->max_pages, gop.ref, e->domain_flags);
682 spin_unlock(&e->page_alloc_lock);
683 put_domain(e);
684 page->count_info &= ~(PGC_count_mask|PGC_allocated);
685 free_domheap_page(page);
686 gop.status = GNTST_general_error;
687 goto copyback;
688 }
690 /* Okay, add the page to 'e'. */
691 if ( unlikely(e->tot_pages++ == 0) )
692 get_knownalive_domain(e);
693 list_add_tail(&page->list, &e->page_list);
694 page_set_owner(page, e);
696 spin_unlock(&e->page_alloc_lock);
698 TRACE_1D(TRC_MEM_PAGE_GRANT_TRANSFER, e->domain_id);
700 /* Tell the guest about its new page frame. */
701 sha = &e->grant_table->shared[gop.ref];
702 guest_physmap_add_page(e, sha->frame, mfn);
703 sha->frame = mfn;
704 wmb();
705 sha->flags |= GTF_transfer_completed;
707 put_domain(e);
709 gop.status = GNTST_okay;
711 copyback:
712 if ( unlikely(__copy_to_guest_offset(uop, i, &gop, 1)) )
713 {
714 gdprintk(XENLOG_INFO, "gnttab_transfer: error writing resp %d/%d\n",
715 i, count);
716 return -EFAULT;
717 }
718 }
720 return 0;
721 }
723 /* Undo __acquire_grant_for_copy. Again, this has no effect on page
724 type and reference counts. */
725 static void
726 __release_grant_for_copy(
727 struct domain *rd, unsigned long gref, int readonly)
728 {
729 grant_entry_t *const sha = &rd->grant_table->shared[gref];
730 struct active_grant_entry *const act = &rd->grant_table->active[gref];
732 spin_lock(&rd->grant_table->lock);
734 if ( readonly )
735 {
736 act->pin -= GNTPIN_hstr_inc;
737 }
738 else
739 {
740 act->pin -= GNTPIN_hstw_inc;
741 if ( !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) )
742 gnttab_clear_flag(_GTF_writing, &sha->flags);
743 }
745 if ( !act->pin )
746 gnttab_clear_flag(_GTF_reading, &sha->flags);
748 spin_unlock(&rd->grant_table->lock);
749 }
751 /* Grab a frame number from a grant entry and update the flags and pin
752 count as appropriate. Note that this does *not* update the page
753 type or reference counts, and does not check that the mfn is
754 actually valid. */
755 static int
756 __acquire_grant_for_copy(
757 struct domain *rd, unsigned long gref, int readonly,
758 unsigned long *frame)
759 {
760 grant_entry_t *sha;
761 struct active_grant_entry *act;
762 s16 rc = GNTST_okay;
763 int retries = 0;
764 union grant_combo scombo, prev_scombo, new_scombo;
766 if ( unlikely(gref >= NR_GRANT_ENTRIES) )
767 PIN_FAIL(error_out, GNTST_bad_gntref,
768 "Bad grant reference %ld\n", gref);
770 act = &rd->grant_table->active[gref];
771 sha = &rd->grant_table->shared[gref];
773 spin_lock(&rd->grant_table->lock);
775 /* If already pinned, check the active domid and avoid refcnt overflow. */
776 if ( act->pin &&
777 ((act->domid != current->domain->domain_id) ||
778 (act->pin & 0x80808080U) != 0) )
779 PIN_FAIL(unlock_out, GNTST_general_error,
780 "Bad domain (%d != %d), or risk of counter overflow %08x\n",
781 act->domid, current->domain->domain_id, act->pin);
783 if ( !act->pin ||
784 (!readonly && !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask))) )
785 {
786 scombo.word = *(u32 *)&sha->flags;
788 for ( ; ; )
789 {
790 /* If not already pinned, check the grant domid and type. */
791 if ( !act->pin &&
792 (((scombo.shorts.flags & GTF_type_mask) !=
793 GTF_permit_access) ||
794 (scombo.shorts.domid != current->domain->domain_id)) )
795 PIN_FAIL(unlock_out, GNTST_general_error,
796 "Bad flags (%x) or dom (%d). (expected dom %d)\n",
797 scombo.shorts.flags, scombo.shorts.domid,
798 current->domain->domain_id);
800 new_scombo = scombo;
801 new_scombo.shorts.flags |= GTF_reading;
803 if ( !readonly )
804 {
805 new_scombo.shorts.flags |= GTF_writing;
806 if ( unlikely(scombo.shorts.flags & GTF_readonly) )
807 PIN_FAIL(unlock_out, GNTST_general_error,
808 "Attempt to write-pin a r/o grant entry.\n");
809 }
811 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
812 scombo.word, new_scombo.word);
813 if ( likely(prev_scombo.word == scombo.word) )
814 break;
816 if ( retries++ == 4 )
817 PIN_FAIL(unlock_out, GNTST_general_error,
818 "Shared grant entry is unstable.\n");
820 scombo = prev_scombo;
821 }
823 if ( !act->pin )
824 {
825 act->domid = scombo.shorts.domid;
826 act->frame = gmfn_to_mfn(rd, sha->frame);
827 }
828 }
830 act->pin += readonly ? GNTPIN_hstr_inc : GNTPIN_hstw_inc;
832 *frame = act->frame;
834 unlock_out:
835 spin_unlock(&rd->grant_table->lock);
836 error_out:
837 return rc;
838 }
840 static void
841 __gnttab_copy(
842 struct gnttab_copy *op)
843 {
844 struct domain *sd = NULL, *dd = NULL;
845 unsigned long s_frame, d_frame;
846 char *sp, *dp;
847 s16 rc = GNTST_okay;
848 int have_d_grant = 0, have_s_grant = 0, have_s_ref = 0;
849 int src_is_gref, dest_is_gref;
851 if ( ((op->source.offset + op->len) > PAGE_SIZE) ||
852 ((op->dest.offset + op->len) > PAGE_SIZE) )
853 PIN_FAIL(error_out, GNTST_bad_copy_arg, "copy beyond page area.\n");
855 src_is_gref = op->flags & GNTCOPY_source_gref;
856 dest_is_gref = op->flags & GNTCOPY_dest_gref;
858 if ( (op->source.domid != DOMID_SELF && !src_is_gref ) ||
859 (op->dest.domid != DOMID_SELF && !dest_is_gref) )
860 PIN_FAIL(error_out, GNTST_permission_denied,
861 "only allow copy-by-mfn for DOMID_SELF.\n");
863 if ( op->source.domid == DOMID_SELF )
864 {
865 sd = current->domain;
866 get_knownalive_domain(sd);
867 }
868 else if ( (sd = find_domain_by_id(op->source.domid)) == NULL )
869 {
870 PIN_FAIL(error_out, GNTST_bad_domain,
871 "couldn't find %d\n", op->source.domid);
872 }
874 if ( op->dest.domid == DOMID_SELF )
875 {
876 dd = current->domain;
877 get_knownalive_domain(dd);
878 }
879 else if ( (dd = find_domain_by_id(op->dest.domid)) == NULL )
880 {
881 PIN_FAIL(error_out, GNTST_bad_domain,
882 "couldn't find %d\n", op->dest.domid);
883 }
885 if ( src_is_gref )
886 {
887 rc = __acquire_grant_for_copy(sd, op->source.u.ref, 1, &s_frame);
888 if ( rc != GNTST_okay )
889 goto error_out;
890 have_s_grant = 1;
891 }
892 else
893 {
894 s_frame = gmfn_to_mfn(sd, op->source.u.gmfn);
895 }
896 if ( unlikely(!mfn_valid(s_frame)) )
897 PIN_FAIL(error_out, GNTST_general_error,
898 "source frame %lx invalid.\n", s_frame);
899 if ( !get_page(mfn_to_page(s_frame), sd) )
900 {
901 if ( !test_bit(_DOMF_dying, &sd->domain_flags) )
902 gdprintk(XENLOG_WARNING, "Could not get src frame %lx\n", s_frame);
903 rc = GNTST_general_error;
904 goto error_out;
905 }
906 have_s_ref = 1;
908 if ( dest_is_gref )
909 {
910 rc = __acquire_grant_for_copy(dd, op->dest.u.ref, 0, &d_frame);
911 if ( rc != GNTST_okay )
912 goto error_out;
913 have_d_grant = 1;
914 }
915 else
916 {
917 d_frame = gmfn_to_mfn(dd, op->dest.u.gmfn);
918 }
919 if ( unlikely(!mfn_valid(d_frame)) )
920 PIN_FAIL(error_out, GNTST_general_error,
921 "destination frame %lx invalid.\n", d_frame);
922 if ( !get_page_and_type(mfn_to_page(d_frame), dd, PGT_writable_page) )
923 {
924 if ( !test_bit(_DOMF_dying, &dd->domain_flags) )
925 gdprintk(XENLOG_WARNING, "Could not get dst frame %lx\n", d_frame);
926 rc = GNTST_general_error;
927 goto error_out;
928 }
930 sp = map_domain_page(s_frame);
931 dp = map_domain_page(d_frame);
933 memcpy(dp + op->dest.offset, sp + op->source.offset, op->len);
935 unmap_domain_page(dp);
936 unmap_domain_page(sp);
938 gnttab_mark_dirty(dd, d_frame);
940 put_page_and_type(mfn_to_page(d_frame));
941 error_out:
942 if ( have_s_ref )
943 put_page(mfn_to_page(s_frame));
944 if ( have_s_grant )
945 __release_grant_for_copy(sd, op->source.u.ref, 1);
946 if ( have_d_grant )
947 __release_grant_for_copy(dd, op->dest.u.ref, 0);
948 if ( sd )
949 put_domain(sd);
950 if ( dd )
951 put_domain(dd);
952 op->status = rc;
953 }
955 static long
956 gnttab_copy(
957 XEN_GUEST_HANDLE(gnttab_copy_t) uop, unsigned int count)
958 {
959 int i;
960 struct gnttab_copy op;
962 for ( i = 0; i < count; i++ )
963 {
964 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
965 return -EFAULT;
966 __gnttab_copy(&op);
967 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
968 return -EFAULT;
969 }
970 return 0;
971 }
973 long
974 do_grant_table_op(
975 unsigned int cmd, XEN_GUEST_HANDLE(void) uop, unsigned int count)
976 {
977 long rc;
978 struct domain *d = current->domain;
980 if ( count > 512 )
981 return -EINVAL;
983 LOCK_BIGLOCK(d);
985 rc = -EFAULT;
986 switch ( cmd )
987 {
988 case GNTTABOP_map_grant_ref:
989 {
990 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) map =
991 guest_handle_cast(uop, gnttab_map_grant_ref_t);
992 if ( unlikely(!guest_handle_okay(map, count)) )
993 goto out;
994 rc = gnttab_map_grant_ref(map, count);
995 break;
996 }
997 case GNTTABOP_unmap_grant_ref:
998 {
999 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) unmap =
1000 guest_handle_cast(uop, gnttab_unmap_grant_ref_t);
1001 if ( unlikely(!guest_handle_okay(unmap, count)) )
1002 goto out;
1003 rc = gnttab_unmap_grant_ref(unmap, count);
1004 break;
1006 case GNTTABOP_setup_table:
1008 rc = gnttab_setup_table(
1009 guest_handle_cast(uop, gnttab_setup_table_t), count);
1010 break;
1012 case GNTTABOP_transfer:
1014 XEN_GUEST_HANDLE(gnttab_transfer_t) transfer =
1015 guest_handle_cast(uop, gnttab_transfer_t);
1016 if ( unlikely(!guest_handle_okay(transfer, count)) )
1017 goto out;
1018 rc = gnttab_transfer(transfer, count);
1019 break;
1021 case GNTTABOP_copy:
1023 XEN_GUEST_HANDLE(gnttab_copy_t) copy =
1024 guest_handle_cast(uop, gnttab_copy_t);
1025 if ( unlikely(!guest_handle_okay(copy, count)) )
1026 goto out;
1027 rc = gnttab_copy(copy, count);
1028 break;
1030 default:
1031 rc = -ENOSYS;
1032 break;
1035 out:
1036 UNLOCK_BIGLOCK(d);
1038 return rc;
1041 int
1042 grant_table_create(
1043 struct domain *d)
1045 struct grant_table *t;
1046 int i;
1048 BUG_ON(MAPTRACK_MAX_ENTRIES < NR_GRANT_ENTRIES);
1049 if ( (t = xmalloc(struct grant_table)) == NULL )
1050 goto no_mem;
1052 /* Simple stuff. */
1053 memset(t, 0, sizeof(*t));
1054 spin_lock_init(&t->lock);
1056 /* Active grant table. */
1057 t->active = xmalloc_array(struct active_grant_entry, NR_GRANT_ENTRIES);
1058 if ( t->active == NULL )
1059 goto no_mem;
1060 memset(t->active, 0, sizeof(struct active_grant_entry) * NR_GRANT_ENTRIES);
1062 /* Tracking of mapped foreign frames table */
1063 if ( (t->maptrack = alloc_xenheap_page()) == NULL )
1064 goto no_mem;
1065 t->maptrack_order = 0;
1066 t->maptrack_limit = PAGE_SIZE / sizeof(struct grant_mapping);
1067 memset(t->maptrack, 0, PAGE_SIZE);
1068 for ( i = 0; i < t->maptrack_limit; i++ )
1069 t->maptrack[i].ref = i+1;
1071 /* Shared grant table. */
1072 t->shared = alloc_xenheap_pages(ORDER_GRANT_FRAMES);
1073 if ( t->shared == NULL )
1074 goto no_mem;
1075 memset(t->shared, 0, NR_GRANT_FRAMES * PAGE_SIZE);
1077 for ( i = 0; i < NR_GRANT_FRAMES; i++ )
1078 gnttab_create_shared_page(d, t, i);
1080 /* Okay, install the structure. */
1081 wmb(); /* avoid races with lock-free access to d->grant_table */
1082 d->grant_table = t;
1083 return 0;
1085 no_mem:
1086 if ( t != NULL )
1088 xfree(t->active);
1089 free_xenheap_page(t->maptrack);
1090 xfree(t);
1092 return -ENOMEM;
1095 void
1096 gnttab_release_mappings(
1097 struct domain *d)
1099 struct grant_table *gt = d->grant_table;
1100 struct grant_mapping *map;
1101 grant_ref_t ref;
1102 grant_handle_t handle;
1103 struct domain *rd;
1104 struct active_grant_entry *act;
1105 struct grant_entry *sha;
1107 BUG_ON(!test_bit(_DOMF_dying, &d->domain_flags));
1109 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
1111 map = &gt->maptrack[handle];
1112 if ( !(map->flags & (GNTMAP_device_map|GNTMAP_host_map)) )
1113 continue;
1115 ref = map->ref;
1117 gdprintk(XENLOG_INFO, "Grant release (%hu) ref:(%hu) "
1118 "flags:(%x) dom:(%hu)\n",
1119 handle, ref, map->flags, map->domid);
1121 rd = find_domain_by_id(map->domid);
1122 BUG_ON(rd == NULL);
1124 spin_lock(&rd->grant_table->lock);
1126 act = &rd->grant_table->active[ref];
1127 sha = &rd->grant_table->shared[ref];
1129 if ( map->flags & GNTMAP_readonly )
1131 if ( map->flags & GNTMAP_device_map )
1133 BUG_ON(!(act->pin & GNTPIN_devr_mask));
1134 act->pin -= GNTPIN_devr_inc;
1135 put_page(mfn_to_page(act->frame));
1138 if ( map->flags & GNTMAP_host_map )
1140 BUG_ON(!(act->pin & GNTPIN_hstr_mask));
1141 act->pin -= GNTPIN_hstr_inc;
1142 /* Done implicitly when page tables are destroyed. */
1143 /* put_page(mfn_to_page(act->frame)); */
1146 else
1148 if ( map->flags & GNTMAP_device_map )
1150 BUG_ON(!(act->pin & GNTPIN_devw_mask));
1151 act->pin -= GNTPIN_devw_inc;
1152 put_page_and_type(mfn_to_page(act->frame));
1155 if ( map->flags & GNTMAP_host_map )
1157 BUG_ON(!(act->pin & GNTPIN_hstw_mask));
1158 act->pin -= GNTPIN_hstw_inc;
1159 /* Done implicitly when page tables are destroyed. */
1160 /* put_page_and_type(mfn_to_page(act->frame)); */
1163 if ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0 )
1164 gnttab_clear_flag(_GTF_writing, &sha->flags);
1167 if ( act->pin == 0 )
1168 gnttab_clear_flag(_GTF_reading, &sha->flags);
1170 spin_unlock(&rd->grant_table->lock);
1172 put_domain(rd);
1174 map->flags = 0;
1179 void
1180 grant_table_destroy(
1181 struct domain *d)
1183 struct grant_table *t = d->grant_table;
1185 if ( t == NULL )
1186 return;
1188 free_xenheap_pages(t->shared, ORDER_GRANT_FRAMES);
1189 free_xenheap_pages(t->maptrack, t->maptrack_order);
1190 xfree(t->active);
1191 xfree(t);
1193 d->grant_table = NULL;
1196 /*
1197 * Local variables:
1198 * mode: C
1199 * c-set-style: "BSD"
1200 * c-basic-offset: 4
1201 * tab-width: 4
1202 * indent-tabs-mode: nil
1203 * End:
1204 */