direct-io.hg

view xen/common/grant_table.c @ 4644:c7f1dee0ac34

bitkeeper revision 1.1372 (426a291er6fDcjQLyYtGSP3XQORoWQ)

grant_table.c:
gcc4 tweak.
author kaf24@firebug.cl.cam.ac.uk
date Sat Apr 23 10:53:18 2005 +0000 (2005-04-23)
parents 1803018b3b05
children ccc4ee412321
line source
1 /******************************************************************************
2 * common/grant_table.c
3 *
4 * Mechanism for granting foreign access to page frames, and receiving
5 * page-ownership transfers.
6 *
7 * Copyright (c) 2005 Christopher Clark
8 * Copyright (c) 2004 K A Fraser
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 */
25 #define GRANT_DEBUG 0
26 #define GRANT_DEBUG_VERBOSE 0
28 #include <xen/config.h>
29 #include <xen/sched.h>
30 #include <xen/shadow.h>
31 #include <xen/mm.h>
33 #define PIN_FAIL(_lbl, _rc, _f, _a...) \
34 do { \
35 DPRINTK( _f, ## _a ); \
36 rc = (_rc); \
37 goto _lbl; \
38 } while ( 0 )
40 static inline int
41 get_maptrack_handle(
42 grant_table_t *t)
43 {
44 unsigned int h;
45 if ( unlikely((h = t->maptrack_head) == t->maptrack_limit) )
46 return -1;
47 t->maptrack_head = t->maptrack[h].ref_and_flags >> MAPTRACK_REF_SHIFT;
48 t->map_count++;
49 return h;
50 }
52 static inline void
53 put_maptrack_handle(
54 grant_table_t *t, int handle)
55 {
56 t->maptrack[handle].ref_and_flags = t->maptrack_head << MAPTRACK_REF_SHIFT;
57 t->maptrack_head = handle;
58 t->map_count--;
59 }
61 static int
62 __gnttab_activate_grant_ref(
63 struct domain *mapping_d, /* IN */
64 struct exec_domain *mapping_ed,
65 struct domain *granting_d,
66 grant_ref_t ref,
67 u16 dev_hst_ro_flags,
68 unsigned long host_virt_addr,
69 unsigned long *pframe ) /* OUT */
70 {
71 domid_t sdom;
72 u16 sflags;
73 active_grant_entry_t *act;
74 grant_entry_t *sha;
75 s16 rc = 1;
76 unsigned long frame = 0;
77 int retries = 0;
79 /*
80 * Objectives of this function:
81 * . Make the record ( granting_d, ref ) active, if not already.
82 * . Update shared grant entry of owner, indicating frame is mapped.
83 * . Increment the owner act->pin reference counts.
84 * . get_page on shared frame if new mapping.
85 * . get_page_type if this is first RW mapping of frame.
86 * . Add PTE to virtual address space of mapping_d, if necessary.
87 * Returns:
88 * . -ve: error
89 * . 1: ok
90 * . 0: ok and TLB invalidate of host_virt_addr needed.
91 *
92 * On success, *pframe contains mfn.
93 */
95 /*
96 * We bound the number of times we retry CMPXCHG on memory locations that
97 * we share with a guest OS. The reason is that the guest can modify that
98 * location at a higher rate than we can read-modify-CMPXCHG, so the guest
99 * could cause us to livelock. There are a few cases where it is valid for
100 * the guest to race our updates (e.g., to change the GTF_readonly flag),
101 * so we allow a few retries before failing.
102 */
104 act = &granting_d->grant_table->active[ref];
105 sha = &granting_d->grant_table->shared[ref];
107 spin_lock(&granting_d->grant_table->lock);
109 if ( act->pin == 0 )
110 {
111 /* CASE 1: Activating a previously inactive entry. */
113 sflags = sha->flags;
114 sdom = sha->domid;
116 for ( ; ; )
117 {
118 u32 scombo, prev_scombo, new_scombo;
120 if ( unlikely((sflags & GTF_type_mask) != GTF_permit_access) ||
121 unlikely(sdom != mapping_d->id) )
122 PIN_FAIL(unlock_out, GNTST_general_error,
123 "Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
124 sflags, sdom, mapping_d->id);
126 /* Merge two 16-bit values into a 32-bit combined update. */
127 /* NB. Endianness! */
128 prev_scombo = scombo = ((u32)sdom << 16) | (u32)sflags;
130 new_scombo = scombo | GTF_reading;
131 if ( !(dev_hst_ro_flags & GNTMAP_readonly) )
132 {
133 new_scombo |= GTF_writing;
134 if ( unlikely(sflags & GTF_readonly) )
135 PIN_FAIL(unlock_out, GNTST_general_error,
136 "Attempt to write-pin a r/o grant entry.\n");
137 }
139 /* NB. prev_scombo is updated in place to seen value. */
140 if ( unlikely(cmpxchg_user((u32 *)&sha->flags,
141 prev_scombo,
142 new_scombo)) )
143 PIN_FAIL(unlock_out, GNTST_general_error,
144 "Fault while modifying shared flags and domid.\n");
146 /* Did the combined update work (did we see what we expected?). */
147 if ( likely(prev_scombo == scombo) )
148 break;
150 if ( retries++ == 4 )
151 PIN_FAIL(unlock_out, GNTST_general_error,
152 "Shared grant entry is unstable.\n");
154 /* Didn't see what we expected. Split out the seen flags & dom. */
155 /* NB. Endianness! */
156 sflags = (u16)prev_scombo;
157 sdom = (u16)(prev_scombo >> 16);
158 }
160 /* rmb(); */ /* not on x86 */
162 frame = __gpfn_to_mfn_foreign(granting_d, sha->frame);
164 if ( unlikely(!pfn_valid(frame)) ||
165 unlikely(!((dev_hst_ro_flags & GNTMAP_readonly) ?
166 get_page(&frame_table[frame], granting_d) :
167 get_page_and_type(&frame_table[frame], granting_d,
168 PGT_writable_page))) )
169 {
170 clear_bit(_GTF_writing, &sha->flags);
171 clear_bit(_GTF_reading, &sha->flags);
172 PIN_FAIL(unlock_out, GNTST_general_error,
173 "Could not pin the granted frame (%lx)!\n", frame);
174 }
176 if ( dev_hst_ro_flags & GNTMAP_device_map )
177 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
178 GNTPIN_devr_inc : GNTPIN_devw_inc;
179 if ( dev_hst_ro_flags & GNTMAP_host_map )
180 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
181 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
182 act->domid = sdom;
183 act->frame = frame;
184 }
185 else
186 {
187 /* CASE 2: Active modications to an already active entry. */
189 /*
190 * A cheesy check for possible pin-count overflow.
191 * A more accurate check cannot be done with a single comparison.
192 */
193 if ( (act->pin & 0x80808080U) != 0 )
194 PIN_FAIL(unlock_out, ENOSPC,
195 "Risk of counter overflow %08x\n", act->pin);
197 frame = act->frame;
199 if ( !(dev_hst_ro_flags & GNTMAP_readonly) &&
200 !((sflags = sha->flags) & GTF_writing) )
201 {
202 for ( ; ; )
203 {
204 u16 prev_sflags;
206 if ( unlikely(sflags & GTF_readonly) )
207 PIN_FAIL(unlock_out, GNTST_general_error,
208 "Attempt to write-pin a r/o grant entry.\n");
210 prev_sflags = sflags;
212 /* NB. prev_sflags is updated in place to seen value. */
213 if ( unlikely(cmpxchg_user(&sha->flags, prev_sflags,
214 prev_sflags | GTF_writing)) )
215 PIN_FAIL(unlock_out, GNTST_general_error,
216 "Fault while modifying shared flags.\n");
218 if ( likely(prev_sflags == sflags) )
219 break;
221 if ( retries++ == 4 )
222 PIN_FAIL(unlock_out, GNTST_general_error,
223 "Shared grant entry is unstable.\n");
225 sflags = prev_sflags;
226 }
228 if ( unlikely(!get_page_type(&frame_table[frame],
229 PGT_writable_page)) )
230 {
231 clear_bit(_GTF_writing, &sha->flags);
232 PIN_FAIL(unlock_out, GNTST_general_error,
233 "Attempt to write-pin a unwritable page.\n");
234 }
235 }
237 if ( dev_hst_ro_flags & GNTMAP_device_map )
238 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
239 GNTPIN_devr_inc : GNTPIN_devw_inc;
241 if ( dev_hst_ro_flags & GNTMAP_host_map )
242 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
243 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
244 }
246 /*
247 * At this point:
248 * act->pin updated to reflect mapping.
249 * sha->flags updated to indicate to granting domain mapping done.
250 * frame contains the mfn.
251 */
253 spin_unlock(&granting_d->grant_table->lock);
255 if ( (host_virt_addr != 0) && (dev_hst_ro_flags & GNTMAP_host_map) )
256 {
257 /* Write update into the pagetable. */
258 l1_pgentry_t pte;
259 pte = l1e_create_pfn(frame, _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_DIRTY);
260 if ( !(dev_hst_ro_flags & GNTMAP_readonly) )
261 l1e_add_flags(&pte,_PAGE_RW);
262 rc = update_grant_va_mapping( host_virt_addr, pte,
263 mapping_d, mapping_ed );
265 /*
266 * IMPORTANT: (rc == 0) => must flush / invalidate entry in TLB.
267 * This is done in the outer gnttab_map_grant_ref.
268 */
270 if ( rc < 0 )
271 {
272 /* Failure: undo and abort. */
274 spin_lock(&granting_d->grant_table->lock);
276 if ( dev_hst_ro_flags & GNTMAP_readonly )
277 {
278 act->pin -= GNTPIN_hstr_inc;
279 }
280 else
281 {
282 act->pin -= GNTPIN_hstw_inc;
283 if ( (act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) == 0 )
284 {
285 clear_bit(_GTF_writing, &sha->flags);
286 put_page_type(&frame_table[frame]);
287 }
288 }
290 if ( act->pin == 0 )
291 {
292 clear_bit(_GTF_reading, &sha->flags);
293 put_page(&frame_table[frame]);
294 }
296 spin_unlock(&granting_d->grant_table->lock);
297 }
299 }
301 *pframe = frame;
302 return rc;
304 unlock_out:
305 spin_unlock(&granting_d->grant_table->lock);
306 return rc;
307 }
309 /*
310 * Returns 0 if TLB flush / invalidate required by caller.
311 * va will indicate the address to be invalidated.
312 */
313 static int
314 __gnttab_map_grant_ref(
315 gnttab_map_grant_ref_t *uop,
316 unsigned long *va)
317 {
318 domid_t dom;
319 grant_ref_t ref;
320 struct domain *ld, *rd;
321 struct exec_domain *led;
322 u16 dev_hst_ro_flags;
323 int handle;
324 unsigned long frame = 0, host_virt_addr;
325 int rc;
327 led = current;
328 ld = led->domain;
330 /* Bitwise-OR avoids short-circuiting which screws control flow. */
331 if ( unlikely(__get_user(dom, &uop->dom) |
332 __get_user(ref, &uop->ref) |
333 __get_user(host_virt_addr, &uop->host_virt_addr) |
334 __get_user(dev_hst_ro_flags, &uop->flags)) )
335 {
336 DPRINTK("Fault while reading gnttab_map_grant_ref_t.\n");
337 return -EFAULT; /* don't set status */
338 }
341 if ( ((host_virt_addr != 0) || (dev_hst_ro_flags & GNTMAP_host_map)) &&
342 unlikely(!__addr_ok(host_virt_addr)))
343 {
344 DPRINTK("Bad virtual address (%x) or flags (%x).\n",
345 host_virt_addr, dev_hst_ro_flags);
346 (void)__put_user(GNTST_bad_virt_addr, &uop->handle);
347 return GNTST_bad_gntref;
348 }
350 if ( unlikely(ref >= NR_GRANT_ENTRIES) ||
351 unlikely((dev_hst_ro_flags &
352 (GNTMAP_device_map|GNTMAP_host_map)) == 0) )
353 {
354 DPRINTK("Bad ref (%d) or flags (%x).\n", ref, dev_hst_ro_flags);
355 (void)__put_user(GNTST_bad_gntref, &uop->handle);
356 return GNTST_bad_gntref;
357 }
359 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
360 unlikely(ld == rd) )
361 {
362 if ( rd != NULL )
363 put_domain(rd);
364 DPRINTK("Could not find domain %d\n", dom);
365 (void)__put_user(GNTST_bad_domain, &uop->handle);
366 return GNTST_bad_domain;
367 }
369 /* Get a maptrack handle. */
370 if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
371 {
372 int i;
373 grant_mapping_t *new_mt;
374 grant_table_t *lgt = ld->grant_table;
376 /* Grow the maptrack table. */
377 new_mt = (void *)alloc_xenheap_pages(lgt->maptrack_order + 1);
378 if ( new_mt == NULL )
379 {
380 put_domain(rd);
381 DPRINTK("No more map handles available\n");
382 (void)__put_user(GNTST_no_device_space, &uop->handle);
383 return GNTST_no_device_space;
384 }
386 memcpy(new_mt, lgt->maptrack, PAGE_SIZE << lgt->maptrack_order);
387 for ( i = lgt->maptrack_limit; i < (lgt->maptrack_limit << 1); i++ )
388 new_mt[i].ref_and_flags = (i+1) << MAPTRACK_REF_SHIFT;
390 free_xenheap_pages((unsigned long)lgt->maptrack, lgt->maptrack_order);
391 lgt->maptrack = new_mt;
392 lgt->maptrack_order += 1;
393 lgt->maptrack_limit <<= 1;
395 printk("Doubled maptrack size\n");
396 handle = get_maptrack_handle(ld->grant_table);
397 }
399 #if GRANT_DEBUG_VERBOSE
400 DPRINTK("Mapping grant ref (%hu) for domain (%hu) with flags (%x)\n",
401 ref, dom, dev_hst_ro_flags);
402 #endif
404 if ( 0 <= ( rc = __gnttab_activate_grant_ref( ld, led, rd, ref,
405 dev_hst_ro_flags,
406 host_virt_addr, &frame)))
407 {
408 /*
409 * Only make the maptrack live _after_ writing the pte, in case we
410 * overwrite the same frame number, causing a maptrack walk to find it
411 */
412 ld->grant_table->maptrack[handle].domid = dom;
414 ld->grant_table->maptrack[handle].ref_and_flags
415 = (ref << MAPTRACK_REF_SHIFT) |
416 (dev_hst_ro_flags & MAPTRACK_GNTMAP_MASK);
418 (void)__put_user(frame, &uop->dev_bus_addr);
420 if ( dev_hst_ro_flags & GNTMAP_host_map )
421 *va = host_virt_addr;
423 (void)__put_user(handle, &uop->handle);
424 }
425 else
426 {
427 (void)__put_user(rc, &uop->handle);
428 put_maptrack_handle(ld->grant_table, handle);
429 }
431 put_domain(rd);
432 return rc;
433 }
435 static long
436 gnttab_map_grant_ref(
437 gnttab_map_grant_ref_t *uop, unsigned int count)
438 {
439 int i, flush = 0;
440 unsigned long va = 0;
442 for ( i = 0; i < count; i++ )
443 if ( __gnttab_map_grant_ref(&uop[i], &va) == 0 )
444 flush++;
446 if ( flush == 1 )
447 flush_tlb_one_mask(current->domain->cpuset, va);
448 else if ( flush != 0 )
449 flush_tlb_mask(current->domain->cpuset);
451 return 0;
452 }
454 static int
455 __gnttab_unmap_grant_ref(
456 gnttab_unmap_grant_ref_t *uop,
457 unsigned long *va)
458 {
459 domid_t dom;
460 grant_ref_t ref;
461 u16 handle;
462 struct domain *ld, *rd;
464 active_grant_entry_t *act;
465 grant_entry_t *sha;
466 grant_mapping_t *map;
467 u16 flags;
468 s16 rc = 1;
469 unsigned long frame, virt;
471 ld = current->domain;
473 /* Bitwise-OR avoids short-circuiting which screws control flow. */
474 if ( unlikely(__get_user(virt, &uop->host_virt_addr) |
475 __get_user(frame, &uop->dev_bus_addr) |
476 __get_user(handle, &uop->handle)) )
477 {
478 DPRINTK("Fault while reading gnttab_unmap_grant_ref_t.\n");
479 return -EFAULT; /* don't set status */
480 }
482 map = &ld->grant_table->maptrack[handle];
484 if ( unlikely(handle >= ld->grant_table->maptrack_limit) ||
485 unlikely(!(map->ref_and_flags & MAPTRACK_GNTMAP_MASK)) )
486 {
487 DPRINTK("Bad handle (%d).\n", handle);
488 (void)__put_user(GNTST_bad_handle, &uop->status);
489 return GNTST_bad_handle;
490 }
492 dom = map->domid;
493 ref = map->ref_and_flags >> MAPTRACK_REF_SHIFT;
494 flags = map->ref_and_flags & MAPTRACK_GNTMAP_MASK;
496 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
497 unlikely(ld == rd) )
498 {
499 if ( rd != NULL )
500 put_domain(rd);
501 DPRINTK("Could not find domain %d\n", dom);
502 (void)__put_user(GNTST_bad_domain, &uop->status);
503 return GNTST_bad_domain;
504 }
506 #if GRANT_DEBUG_VERBOSE
507 DPRINTK("Unmapping grant ref (%hu) for domain (%hu) with handle (%hu)\n",
508 ref, dom, handle);
509 #endif
511 act = &rd->grant_table->active[ref];
512 sha = &rd->grant_table->shared[ref];
514 spin_lock(&rd->grant_table->lock);
516 if ( frame == 0 )
517 {
518 frame = act->frame;
519 }
520 else if ( frame == GNTUNMAP_DEV_FROM_VIRT )
521 {
522 if ( !( flags & GNTMAP_device_map ) )
523 PIN_FAIL(unmap_out, GNTST_bad_dev_addr,
524 "Bad frame number: frame not mapped for dev access.\n");
525 frame = act->frame;
527 /* Frame will be unmapped for device access below if virt addr okay. */
528 }
529 else
530 {
531 if ( unlikely(frame != act->frame) )
532 PIN_FAIL(unmap_out, GNTST_general_error,
533 "Bad frame number doesn't match gntref.\n");
534 if ( flags & GNTMAP_device_map )
535 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_devr_inc
536 : GNTPIN_devw_inc;
538 map->ref_and_flags &= ~GNTMAP_device_map;
539 (void)__put_user(0, &uop->dev_bus_addr);
541 /* Frame is now unmapped for device access. */
542 }
544 if ( (virt != 0) &&
545 (flags & GNTMAP_host_map) &&
546 ((act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask)) > 0))
547 {
548 l1_pgentry_t *pl1e;
549 unsigned long _ol1e;
551 pl1e = &linear_pg_table[l1_linear_offset(virt)];
553 if ( unlikely(__get_user(_ol1e, (unsigned long *)pl1e) != 0) )
554 {
555 DPRINTK("Could not find PTE entry for address %x\n", virt);
556 rc = -EINVAL;
557 goto unmap_out;
558 }
560 /*
561 * Check that the virtual address supplied is actually mapped to
562 * act->frame.
563 */
564 if ( unlikely((_ol1e >> PAGE_SHIFT) != frame ))
565 {
566 DPRINTK("PTE entry %x for address %x doesn't match frame %x\n",
567 _ol1e, virt, frame);
568 rc = -EINVAL;
569 goto unmap_out;
570 }
572 /* Delete pagetable entry. */
573 if ( unlikely(__put_user(0, (unsigned long *)pl1e)))
574 {
575 DPRINTK("Cannot delete PTE entry at %x for virtual address %x\n",
576 pl1e, virt);
577 rc = -EINVAL;
578 goto unmap_out;
579 }
581 map->ref_and_flags &= ~GNTMAP_host_map;
583 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_hstr_inc
584 : GNTPIN_hstw_inc;
586 if ( frame == GNTUNMAP_DEV_FROM_VIRT )
587 {
588 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_devr_inc
589 : GNTPIN_devw_inc;
591 map->ref_and_flags &= ~GNTMAP_device_map;
592 (void)__put_user(0, &uop->dev_bus_addr);
593 }
595 rc = 0;
596 *va = virt;
597 }
599 if ( (map->ref_and_flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0)
600 {
601 map->ref_and_flags = 0;
602 put_maptrack_handle(ld->grant_table, handle);
603 }
605 /* If just unmapped a writable mapping, mark as dirtied */
606 if ( unlikely(shadow_mode_log_dirty(rd)) &&
607 !( flags & GNTMAP_readonly ) )
608 mark_dirty(rd, frame);
610 /* If the last writable mapping has been removed, put_page_type */
611 if ( ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask) ) == 0) &&
612 ( !( flags & GNTMAP_readonly ) ) )
613 {
614 clear_bit(_GTF_writing, &sha->flags);
615 put_page_type(&frame_table[frame]);
616 }
618 if ( act->pin == 0 )
619 {
620 clear_bit(_GTF_reading, &sha->flags);
621 put_page(&frame_table[frame]);
622 }
624 unmap_out:
625 (void)__put_user(rc, &uop->status);
626 spin_unlock(&rd->grant_table->lock);
627 put_domain(rd);
628 return rc;
629 }
631 static long
632 gnttab_unmap_grant_ref(
633 gnttab_unmap_grant_ref_t *uop, unsigned int count)
634 {
635 int i, flush = 0;
636 unsigned long va = 0;
638 for ( i = 0; i < count; i++ )
639 if ( __gnttab_unmap_grant_ref(&uop[i], &va) == 0 )
640 flush++;
642 if ( flush == 1 )
643 flush_tlb_one_mask(current->domain->cpuset, va);
644 else if ( flush != 0 )
645 flush_tlb_mask(current->domain->cpuset);
647 return 0;
648 }
650 static long
651 gnttab_setup_table(
652 gnttab_setup_table_t *uop, unsigned int count)
653 {
654 gnttab_setup_table_t op;
655 struct domain *d;
656 int i;
658 if ( count != 1 )
659 return -EINVAL;
661 if ( unlikely(copy_from_user(&op, uop, sizeof(op)) != 0) )
662 {
663 DPRINTK("Fault while reading gnttab_setup_table_t.\n");
664 return -EFAULT;
665 }
667 if ( unlikely(op.nr_frames > NR_GRANT_FRAMES) )
668 {
669 DPRINTK("Xen only supports up to %d grant-table frames per domain.\n",
670 NR_GRANT_FRAMES);
671 (void)put_user(GNTST_general_error, &uop->status);
672 return 0;
673 }
675 if ( op.dom == DOMID_SELF )
676 {
677 op.dom = current->domain->id;
678 }
679 else if ( unlikely(!IS_PRIV(current->domain)) )
680 {
681 (void)put_user(GNTST_permission_denied, &uop->status);
682 return 0;
683 }
685 if ( unlikely((d = find_domain_by_id(op.dom)) == NULL) )
686 {
687 DPRINTK("Bad domid %d.\n", op.dom);
688 (void)put_user(GNTST_bad_domain, &uop->status);
689 return 0;
690 }
692 if ( op.nr_frames <= NR_GRANT_FRAMES )
693 {
694 ASSERT(d->grant_table != NULL);
695 (void)put_user(GNTST_okay, &uop->status);
696 for ( i = 0; i < op.nr_frames; i++ )
697 (void)put_user(
698 (virt_to_phys(d->grant_table->shared) >> PAGE_SHIFT) + i,
699 &uop->frame_list[i]);
700 }
702 put_domain(d);
703 return 0;
704 }
706 #if GRANT_DEBUG
707 static int
708 gnttab_dump_table(gnttab_dump_table_t *uop)
709 {
710 grant_table_t *gt;
711 gnttab_dump_table_t op;
712 struct domain *d;
713 u32 shared_mfn;
714 active_grant_entry_t *act;
715 grant_entry_t sha_copy;
716 grant_mapping_t *maptrack;
717 int i;
720 if ( unlikely(copy_from_user(&op, uop, sizeof(op)) != 0) )
721 {
722 DPRINTK("Fault while reading gnttab_dump_table_t.\n");
723 return -EFAULT;
724 }
726 if ( op.dom == DOMID_SELF )
727 {
728 op.dom = current->domain->id;
729 }
731 if ( unlikely((d = find_domain_by_id(op.dom)) == NULL) )
732 {
733 DPRINTK("Bad domid %d.\n", op.dom);
734 (void)put_user(GNTST_bad_domain, &uop->status);
735 return 0;
736 }
738 ASSERT(d->grant_table != NULL);
739 gt = d->grant_table;
740 (void)put_user(GNTST_okay, &uop->status);
742 shared_mfn = virt_to_phys(d->grant_table->shared);
744 DPRINTK("Grant table for dom (%hu) MFN (%x)\n",
745 op.dom, shared_mfn);
747 ASSERT(d->grant_table->active != NULL);
748 ASSERT(d->grant_table->shared != NULL);
749 ASSERT(d->grant_table->maptrack != NULL);
751 for ( i = 0; i < NR_GRANT_ENTRIES; i++ )
752 {
753 sha_copy = gt->shared[i];
755 if ( sha_copy.flags )
756 {
757 DPRINTK("Grant: dom (%hu) SHARED (%d) flags:(%hx) "
758 "dom:(%hu) frame:(%lx)\n",
759 op.dom, i, sha_copy.flags, sha_copy.domid, sha_copy.frame);
760 }
761 }
763 spin_lock(&gt->lock);
765 for ( i = 0; i < NR_GRANT_ENTRIES; i++ )
766 {
767 act = &gt->active[i];
769 if ( act->pin )
770 {
771 DPRINTK("Grant: dom (%hu) ACTIVE (%d) pin:(%x) "
772 "dom:(%hu) frame:(%lx)\n",
773 op.dom, i, act->pin, act->domid, act->frame);
774 }
775 }
777 for ( i = 0; i < gt->maptrack_limit; i++ )
778 {
779 maptrack = &gt->maptrack[i];
781 if ( maptrack->ref_and_flags & MAPTRACK_GNTMAP_MASK )
782 {
783 DPRINTK("Grant: dom (%hu) MAP (%d) ref:(%hu) flags:(%x) "
784 "dom:(%hu)\n",
785 op.dom, i,
786 maptrack->ref_and_flags >> MAPTRACK_REF_SHIFT,
787 maptrack->ref_and_flags & MAPTRACK_GNTMAP_MASK,
788 maptrack->domid);
789 }
790 }
792 spin_unlock(&gt->lock);
794 put_domain(d);
795 return 0;
796 }
797 #endif
799 long
800 do_grant_table_op(
801 unsigned int cmd, void *uop, unsigned int count)
802 {
803 long rc;
805 if ( count > 512 )
806 return -EINVAL;
808 LOCK_BIGLOCK(current->domain);
810 rc = -EFAULT;
811 switch ( cmd )
812 {
813 case GNTTABOP_map_grant_ref:
814 if ( unlikely(!array_access_ok(
815 uop, count, sizeof(gnttab_map_grant_ref_t))) )
816 goto out;
817 rc = gnttab_map_grant_ref((gnttab_map_grant_ref_t *)uop, count);
818 break;
819 case GNTTABOP_unmap_grant_ref:
820 if ( unlikely(!array_access_ok(
821 uop, count, sizeof(gnttab_unmap_grant_ref_t))) )
822 goto out;
823 rc = gnttab_unmap_grant_ref((gnttab_unmap_grant_ref_t *)uop, count);
824 break;
825 case GNTTABOP_setup_table:
826 rc = gnttab_setup_table((gnttab_setup_table_t *)uop, count);
827 break;
828 #if GRANT_DEBUG
829 case GNTTABOP_dump_table:
830 rc = gnttab_dump_table((gnttab_dump_table_t *)uop);
831 break;
832 #endif
833 default:
834 rc = -ENOSYS;
835 break;
836 }
838 out:
839 UNLOCK_BIGLOCK(current->domain);
841 return rc;
842 }
844 int
845 gnttab_check_unmap(
846 struct domain *rd, struct domain *ld, unsigned long frame, int readonly)
847 {
848 /* Called when put_page is invoked on a page belonging to a foreign domain.
849 * Instead of decrementing the frame table ref count, locate the grant
850 * table entry, if any, and if found, decrement that count.
851 * Called a _lot_ at domain creation because pages mapped by priv domains
852 * also traverse this.
853 */
855 /* Note: If the same frame is mapped multiple times, and then one of
856 * the ptes is overwritten, which maptrack handle gets invalidated?
857 * Advice: Don't do it. Explicitly unmap.
858 */
860 unsigned int handle, ref, refcount;
861 grant_table_t *lgt, *rgt;
862 active_grant_entry_t *act;
863 grant_mapping_t *map;
864 int found = 0;
866 lgt = ld->grant_table;
868 #if GRANT_DEBUG_VERBOSE
869 if ( ld->id != 0 )
870 {
871 DPRINTK("Foreign unref rd(%d) ld(%d) frm(%x) flgs(%x).\n",
872 rd->id, ld->id, frame, readonly);
873 }
874 #endif
876 /* Fast exit if we're not mapping anything using grant tables */
877 if ( lgt->map_count == 0 )
878 return 0;
880 if ( get_domain(rd) == 0 )
881 {
882 DPRINTK("gnttab_check_unmap: couldn't get_domain rd(%d)\n", rd->id);
883 return 0;
884 }
886 rgt = rd->grant_table;
888 for ( handle = 0; handle < lgt->maptrack_limit; handle++ )
889 {
890 map = &lgt->maptrack[handle];
892 if ( ( map->ref_and_flags & MAPTRACK_GNTMAP_MASK ) &&
893 ( readonly ? 1 : (!(map->ref_and_flags & GNTMAP_readonly))))
894 {
895 ref = (map->ref_and_flags >> MAPTRACK_REF_SHIFT);
896 act = &rgt->active[ref];
898 spin_lock(&rgt->lock);
900 if ( act->frame != frame )
901 {
902 spin_unlock(&rgt->lock);
903 continue;
904 }
906 refcount = act->pin & ( readonly ? GNTPIN_hstr_mask
907 : GNTPIN_hstw_mask );
908 if ( refcount == 0 )
909 {
910 spin_unlock(&rgt->lock);
911 continue;
912 }
914 /* gotcha */
915 DPRINTK("Grant unref rd(%d) ld(%d) frm(%x) flgs(%x).\n",
916 rd->id, ld->id, frame, readonly);
918 if ( readonly )
919 act->pin -= GNTPIN_hstr_inc;
920 else
921 {
922 act->pin -= GNTPIN_hstw_inc;
924 /* any more granted writable mappings? */
925 if ( (act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) == 0 )
926 {
927 clear_bit(_GTF_writing, &rgt->shared[ref].flags);
928 put_page_type(&frame_table[frame]);
929 }
930 }
932 if ( act->pin == 0 )
933 {
934 clear_bit(_GTF_reading, &rgt->shared[ref].flags);
935 put_page(&frame_table[frame]);
936 }
937 spin_unlock(&rgt->lock);
939 clear_bit(GNTMAP_host_map, &map->ref_and_flags);
941 if ( !(map->ref_and_flags & GNTMAP_device_map) )
942 put_maptrack_handle(lgt, handle);
944 found = 1;
945 break;
946 }
947 }
948 put_domain(rd);
950 return found;
951 }
953 int
954 gnttab_prepare_for_transfer(
955 struct domain *rd, struct domain *ld, grant_ref_t ref)
956 {
957 grant_table_t *rgt;
958 grant_entry_t *sha;
959 domid_t sdom;
960 u16 sflags;
961 u32 scombo, prev_scombo;
962 int retries = 0;
963 unsigned long target_pfn;
965 DPRINTK("gnttab_prepare_for_transfer rd(%hu) ld(%hu) ref(%hu).\n",
966 rd->id, ld->id, ref);
968 if ( unlikely((rgt = rd->grant_table) == NULL) ||
969 unlikely(ref >= NR_GRANT_ENTRIES) )
970 {
971 DPRINTK("Dom %d has no g.t., or ref is bad (%d).\n", rd->id, ref);
972 return 0;
973 }
975 spin_lock(&rgt->lock);
977 sha = &rgt->shared[ref];
979 sflags = sha->flags;
980 sdom = sha->domid;
982 for ( ; ; )
983 {
984 target_pfn = sha->frame;
986 if ( unlikely(target_pfn >= max_page ) )
987 {
988 DPRINTK("Bad pfn (%x)\n", target_pfn);
989 goto fail;
990 }
992 if ( unlikely(sflags != GTF_accept_transfer) ||
993 unlikely(sdom != ld->id) )
994 {
995 DPRINTK("Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
996 sflags, sdom, ld->id);
997 goto fail;
998 }
1000 /* Merge two 16-bit values into a 32-bit combined update. */
1001 /* NB. Endianness! */
1002 prev_scombo = scombo = ((u32)sdom << 16) | (u32)sflags;
1004 /* NB. prev_scombo is updated in place to seen value. */
1005 if ( unlikely(cmpxchg_user((u32 *)&sha->flags, prev_scombo,
1006 prev_scombo | GTF_transfer_committed)) )
1008 DPRINTK("Fault while modifying shared flags and domid.\n");
1009 goto fail;
1012 /* Did the combined update work (did we see what we expected?). */
1013 if ( likely(prev_scombo == scombo) )
1014 break;
1016 if ( retries++ == 4 )
1018 DPRINTK("Shared grant entry is unstable.\n");
1019 goto fail;
1022 /* Didn't see what we expected. Split out the seen flags & dom. */
1023 /* NB. Endianness! */
1024 sflags = (u16)prev_scombo;
1025 sdom = (u16)(prev_scombo >> 16);
1028 spin_unlock(&rgt->lock);
1029 return 1;
1031 fail:
1032 spin_unlock(&rgt->lock);
1033 return 0;
1036 void
1037 gnttab_notify_transfer(
1038 struct domain *rd, struct domain *ld, grant_ref_t ref, unsigned long frame)
1040 grant_entry_t *sha;
1041 unsigned long pfn;
1043 DPRINTK("gnttab_notify_transfer rd(%hu) ld(%hu) ref(%hu).\n",
1044 rd->id, ld->id, ref);
1046 sha = &rd->grant_table->shared[ref];
1048 spin_lock(&rd->grant_table->lock);
1050 pfn = sha->frame;
1052 if ( unlikely(pfn >= max_page ) )
1053 DPRINTK("Bad pfn (%x)\n", pfn);
1054 else
1056 machine_to_phys_mapping[frame] = pfn;
1058 if ( unlikely(shadow_mode_log_dirty(ld)))
1059 mark_dirty(ld, frame);
1061 if (shadow_mode_translate(ld))
1062 __phys_to_machine_mapping[pfn] = frame;
1064 sha->frame = __mfn_to_gpfn(rd, frame);
1065 sha->domid = rd->id;
1066 wmb();
1067 sha->flags = ( GTF_accept_transfer | GTF_transfer_completed );
1069 spin_unlock(&rd->grant_table->lock);
1071 return;
1074 int
1075 grant_table_create(
1076 struct domain *d)
1078 grant_table_t *t;
1079 int i;
1081 if ( (t = xmalloc(grant_table_t)) == NULL )
1082 goto no_mem;
1084 /* Simple stuff. */
1085 memset(t, 0, sizeof(*t));
1086 spin_lock_init(&t->lock);
1088 /* Active grant table. */
1089 if ( (t->active = xmalloc_array(active_grant_entry_t, NR_GRANT_ENTRIES))
1090 == NULL )
1091 goto no_mem;
1092 memset(t->active, 0, sizeof(active_grant_entry_t) * NR_GRANT_ENTRIES);
1094 /* Tracking of mapped foreign frames table */
1095 if ( (t->maptrack = (void *)alloc_xenheap_page()) == NULL )
1096 goto no_mem;
1097 t->maptrack_order = 0;
1098 t->maptrack_limit = PAGE_SIZE / sizeof(grant_mapping_t);
1099 memset(t->maptrack, 0, PAGE_SIZE);
1100 for ( i = 0; i < t->maptrack_limit; i++ )
1101 t->maptrack[i].ref_and_flags = (i+1) << MAPTRACK_REF_SHIFT;
1103 /* Shared grant table. */
1104 t->shared = (void *)alloc_xenheap_pages(ORDER_GRANT_FRAMES);
1105 if ( t->shared == NULL )
1106 goto no_mem;
1107 memset(t->shared, 0, NR_GRANT_FRAMES * PAGE_SIZE);
1109 for ( i = 0; i < NR_GRANT_FRAMES; i++ )
1111 SHARE_PFN_WITH_DOMAIN(
1112 virt_to_page((char *)(t->shared)+(i*PAGE_SIZE)), d);
1113 machine_to_phys_mapping[(virt_to_phys(t->shared) >> PAGE_SHIFT) + i] =
1114 INVALID_M2P_ENTRY;
1117 /* Okay, install the structure. */
1118 wmb(); /* avoid races with lock-free access to d->grant_table */
1119 d->grant_table = t;
1120 return 0;
1122 no_mem:
1123 if ( t != NULL )
1125 if ( t->active != NULL )
1126 xfree(t->active);
1127 if ( t->maptrack != NULL )
1128 free_xenheap_page((unsigned long)t->maptrack);
1129 xfree(t);
1131 return -ENOMEM;
1134 void
1135 gnttab_release_dev_mappings(grant_table_t *gt)
1137 grant_mapping_t *map;
1138 domid_t dom;
1139 grant_ref_t ref;
1140 u16 handle;
1141 struct domain *ld, *rd;
1142 unsigned long frame;
1143 active_grant_entry_t *act;
1144 grant_entry_t *sha;
1146 ld = current->domain;
1148 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
1150 map = &gt->maptrack[handle];
1152 if ( map->ref_and_flags & GNTMAP_device_map )
1154 dom = map->domid;
1155 ref = map->ref_and_flags >> MAPTRACK_REF_SHIFT;
1157 DPRINTK("Grant release (%hu) ref:(%hu) flags:(%x) dom:(%hu)\n",
1158 handle, ref,
1159 map->ref_and_flags & MAPTRACK_GNTMAP_MASK, dom);
1161 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
1162 unlikely(ld == rd) )
1164 if ( rd != NULL )
1165 put_domain(rd);
1167 printk(KERN_WARNING "Grant release: No dom%d\n", dom);
1168 continue;
1171 act = &rd->grant_table->active[ref];
1172 sha = &rd->grant_table->shared[ref];
1174 spin_lock(&rd->grant_table->lock);
1176 if ( act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask) )
1178 frame = act->frame;
1180 if ( ( (act->pin & GNTPIN_hstw_mask) == 0 ) &&
1181 ( (act->pin & GNTPIN_devw_mask) > 0 ) )
1183 clear_bit(_GTF_writing, &sha->flags);
1184 put_page_type(&frame_table[frame]);
1187 act->pin &= ~(GNTPIN_devw_mask | GNTPIN_devr_mask);
1189 if ( act->pin == 0 )
1191 clear_bit(_GTF_reading, &sha->flags);
1192 map->ref_and_flags = 0;
1193 put_page(&frame_table[frame]);
1195 else
1196 map->ref_and_flags &= ~GNTMAP_device_map;
1199 spin_unlock(&rd->grant_table->lock);
1201 put_domain(rd);
1207 void
1208 grant_table_destroy(
1209 struct domain *d)
1211 grant_table_t *t;
1213 if ( (t = d->grant_table) != NULL )
1215 /* Free memory relating to this grant table. */
1216 d->grant_table = NULL;
1217 free_xenheap_pages((unsigned long)t->shared, ORDER_GRANT_FRAMES);
1218 free_xenheap_page((unsigned long)t->maptrack);
1219 xfree(t->active);
1220 xfree(t);
1224 void
1225 grant_table_init(
1226 void)
1228 /* Nothing. */
1229 DPRINTK("Grant table init\n");
1232 /*
1233 * Local variables:
1234 * mode: C
1235 * c-set-style: "BSD"
1236 * c-basic-offset: 4
1237 * tab-width: 4
1238 * indent-tabs-mode: nil
1239 * End:
1240 */