direct-io.hg

view xen/common/grant_table.c @ 4523:b0422f2ab81d

bitkeeper revision 1.1295 (425eab1cIikVxIVyd1yoZpti7rBCLA)

fix return with the lock held

Signed-off-by: Reiner Sailer <sailer@us.ibm.com>
Signed-off-by: Vincent Hanquez <vincent@xensource.com>
author vh249@arcadians.cl.cam.ac.uk
date Thu Apr 14 17:40:44 2005 +0000 (2005-04-14)
parents 0af94d69263f
children 5517afc58cf9
line source
1 /******************************************************************************
2 * common/grant_table.c
3 *
4 * Mechanism for granting foreign access to page frames, and receiving
5 * page-ownership transfers.
6 *
7 * Copyright (c) 2005 Christopher Clark
8 * Copyright (c) 2004 K A Fraser
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 */
25 #define GRANT_DEBUG 0
26 #define GRANT_DEBUG_VERBOSE 0
28 #include <xen/config.h>
29 #include <xen/sched.h>
30 #include <xen/shadow.h>
31 #include <xen/mm.h>
33 #define PIN_FAIL(_lbl, _rc, _f, _a...) \
34 do { \
35 DPRINTK( _f, ## _a ); \
36 rc = (_rc); \
37 goto _lbl; \
38 } while ( 0 )
40 static inline int
41 get_maptrack_handle(
42 grant_table_t *t)
43 {
44 unsigned int h;
45 if ( unlikely((h = t->maptrack_head) == t->maptrack_limit) )
46 return -1;
47 t->maptrack_head = t->maptrack[h].ref_and_flags >> MAPTRACK_REF_SHIFT;
48 t->map_count++;
49 return h;
50 }
52 static inline void
53 put_maptrack_handle(
54 grant_table_t *t, int handle)
55 {
56 t->maptrack[handle].ref_and_flags = t->maptrack_head << MAPTRACK_REF_SHIFT;
57 t->maptrack_head = handle;
58 t->map_count--;
59 }
61 static int
62 __gnttab_activate_grant_ref(
63 struct domain *mapping_d, /* IN */
64 struct exec_domain *mapping_ed,
65 struct domain *granting_d,
66 grant_ref_t ref,
67 u16 dev_hst_ro_flags,
68 unsigned long host_virt_addr,
69 unsigned long *pframe ) /* OUT */
70 {
71 domid_t sdom;
72 u16 sflags;
73 active_grant_entry_t *act;
74 grant_entry_t *sha;
75 s16 rc = 1;
76 unsigned long frame = 0;
77 int retries = 0;
79 /*
80 * Objectives of this function:
81 * . Make the record ( granting_d, ref ) active, if not already.
82 * . Update shared grant entry of owner, indicating frame is mapped.
83 * . Increment the owner act->pin reference counts.
84 * . get_page on shared frame if new mapping.
85 * . get_page_type if this is first RW mapping of frame.
86 * . Add PTE to virtual address space of mapping_d, if necessary.
87 * Returns:
88 * . -ve: error
89 * . 1: ok
90 * . 0: ok and TLB invalidate of host_virt_addr needed.
91 *
92 * On success, *pframe contains mfn.
93 */
95 /*
96 * We bound the number of times we retry CMPXCHG on memory locations that
97 * we share with a guest OS. The reason is that the guest can modify that
98 * location at a higher rate than we can read-modify-CMPXCHG, so the guest
99 * could cause us to livelock. There are a few cases where it is valid for
100 * the guest to race our updates (e.g., to change the GTF_readonly flag),
101 * so we allow a few retries before failing.
102 */
104 act = &granting_d->grant_table->active[ref];
105 sha = &granting_d->grant_table->shared[ref];
107 spin_lock(&granting_d->grant_table->lock);
109 if ( act->pin == 0 )
110 {
111 /* CASE 1: Activating a previously inactive entry. */
113 sflags = sha->flags;
114 sdom = sha->domid;
116 for ( ; ; )
117 {
118 u32 scombo, prev_scombo, new_scombo;
120 if ( unlikely((sflags & GTF_type_mask) != GTF_permit_access) ||
121 unlikely(sdom != mapping_d->id) )
122 PIN_FAIL(unlock_out, GNTST_general_error,
123 "Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
124 sflags, sdom, mapping_d->id);
126 /* Merge two 16-bit values into a 32-bit combined update. */
127 /* NB. Endianness! */
128 prev_scombo = scombo = ((u32)sdom << 16) | (u32)sflags;
130 new_scombo = scombo | GTF_reading;
131 if ( !(dev_hst_ro_flags & GNTMAP_readonly) )
132 {
133 new_scombo |= GTF_writing;
134 if ( unlikely(sflags & GTF_readonly) )
135 PIN_FAIL(unlock_out, GNTST_general_error,
136 "Attempt to write-pin a r/o grant entry.\n");
137 }
139 /* NB. prev_scombo is updated in place to seen value. */
140 if ( unlikely(cmpxchg_user((u32 *)&sha->flags,
141 prev_scombo,
142 new_scombo)) )
143 PIN_FAIL(unlock_out, GNTST_general_error,
144 "Fault while modifying shared flags and domid.\n");
146 /* Did the combined update work (did we see what we expected?). */
147 if ( likely(prev_scombo == scombo) )
148 break;
150 if ( retries++ == 4 )
151 PIN_FAIL(unlock_out, GNTST_general_error,
152 "Shared grant entry is unstable.\n");
154 /* Didn't see what we expected. Split out the seen flags & dom. */
155 /* NB. Endianness! */
156 sflags = (u16)prev_scombo;
157 sdom = (u16)(prev_scombo >> 16);
158 }
160 /* rmb(); */ /* not on x86 */
162 frame = __gpfn_to_mfn_foreign(granting_d, sha->frame);
164 if ( unlikely(!pfn_is_ram(frame)) ||
165 unlikely(!((dev_hst_ro_flags & GNTMAP_readonly) ?
166 get_page(&frame_table[frame], granting_d) :
167 get_page_and_type(&frame_table[frame], granting_d,
168 PGT_writable_page))) )
169 {
170 clear_bit(_GTF_writing, &sha->flags);
171 clear_bit(_GTF_reading, &sha->flags);
172 PIN_FAIL(unlock_out, GNTST_general_error,
173 "Could not pin the granted frame (%lx)!\n", frame);
174 }
176 if ( dev_hst_ro_flags & GNTMAP_device_map )
177 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
178 GNTPIN_devr_inc : GNTPIN_devw_inc;
179 if ( dev_hst_ro_flags & GNTMAP_host_map )
180 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
181 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
182 act->domid = sdom;
183 act->frame = frame;
184 }
185 else
186 {
187 /* CASE 2: Active modications to an already active entry. */
189 /*
190 * A cheesy check for possible pin-count overflow.
191 * A more accurate check cannot be done with a single comparison.
192 */
193 if ( (act->pin & 0x80808080U) != 0 )
194 PIN_FAIL(unlock_out, ENOSPC, "Risk of counter overflow %08x\n", act->pin);
196 frame = act->frame;
198 if ( !(dev_hst_ro_flags & GNTMAP_readonly) &&
199 !((sflags = sha->flags) & GTF_writing) )
200 {
201 for ( ; ; )
202 {
203 u16 prev_sflags;
205 if ( unlikely(sflags & GTF_readonly) )
206 PIN_FAIL(unlock_out, GNTST_general_error,
207 "Attempt to write-pin a r/o grant entry.\n");
209 prev_sflags = sflags;
211 /* NB. prev_sflags is updated in place to seen value. */
212 if ( unlikely(cmpxchg_user(&sha->flags, prev_sflags,
213 prev_sflags | GTF_writing)) )
214 PIN_FAIL(unlock_out, GNTST_general_error,
215 "Fault while modifying shared flags.\n");
217 if ( likely(prev_sflags == sflags) )
218 break;
220 if ( retries++ == 4 )
221 PIN_FAIL(unlock_out, GNTST_general_error,
222 "Shared grant entry is unstable.\n");
224 sflags = prev_sflags;
225 }
227 if ( unlikely(!get_page_type(&frame_table[frame],
228 PGT_writable_page)) )
229 {
230 clear_bit(_GTF_writing, &sha->flags);
231 PIN_FAIL(unlock_out, GNTST_general_error,
232 "Attempt to write-pin a unwritable page.\n");
233 }
234 }
236 if ( dev_hst_ro_flags & GNTMAP_device_map )
237 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
238 GNTPIN_devr_inc : GNTPIN_devw_inc;
239 if ( dev_hst_ro_flags & GNTMAP_host_map )
240 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
241 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
242 }
244 /* At this point:
245 * act->pin updated to reflect mapping.
246 * sha->flags updated to indicate to granting domain mapping done.
247 * frame contains the mfn.
248 */
250 spin_unlock(&granting_d->grant_table->lock);
252 if ( (host_virt_addr != 0) && (dev_hst_ro_flags & GNTMAP_host_map) )
253 {
254 /* Write update into the pagetable
255 */
257 rc = update_grant_va_mapping( host_virt_addr,
258 (frame << PAGE_SHIFT) | _PAGE_PRESENT |
259 _PAGE_ACCESSED |
260 _PAGE_DIRTY |
261 ((dev_hst_ro_flags & GNTMAP_readonly) ? 0 : _PAGE_RW),
262 mapping_d, mapping_ed );
264 /* IMPORTANT: (rc == 0) => must flush / invalidate entry in TLB.
265 * This is done in the outer gnttab_map_grant_ref.
266 */
268 if ( 0 > rc )
269 {
270 /* Abort. */
272 spin_lock(&granting_d->grant_table->lock);
274 if ( dev_hst_ro_flags & GNTMAP_readonly )
275 act->pin -= GNTPIN_hstr_inc;
276 else
277 {
278 act->pin -= GNTPIN_hstw_inc;
279 if ( (act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) == 0 )
280 {
281 clear_bit(_GTF_writing, &sha->flags);
282 put_page_type(&frame_table[frame]);
283 }
284 }
285 if ( act->pin == 0 )
286 {
287 clear_bit(_GTF_reading, &sha->flags);
288 put_page(&frame_table[frame]);
289 }
291 spin_unlock(&granting_d->grant_table->lock);
292 }
294 }
295 *pframe = frame;
296 return rc;
298 unlock_out:
299 spin_unlock(&granting_d->grant_table->lock);
300 return rc;
301 }
303 static int
304 __gnttab_map_grant_ref(
305 gnttab_map_grant_ref_t *uop,
306 unsigned long *va)
307 {
308 domid_t dom;
309 grant_ref_t ref;
310 struct domain *ld, *rd;
311 struct exec_domain *led;
312 u16 dev_hst_ro_flags;
313 int handle;
314 unsigned long frame, host_virt_addr;
315 int rc;
317 /* Returns 0 if TLB flush / invalidate required by caller.
318 * va will indicate the address to be invalidated. */
320 led = current;
321 ld = led->domain;
323 /* Bitwise-OR avoids short-circuiting which screws control flow. */
324 if ( unlikely(__get_user(dom, &uop->dom) |
325 __get_user(ref, &uop->ref) |
326 __get_user(host_virt_addr, &uop->host_virt_addr) |
327 __get_user(dev_hst_ro_flags, &uop->flags)) )
328 {
329 DPRINTK("Fault while reading gnttab_map_grant_ref_t.\n");
330 return -EFAULT; /* don't set status */
331 }
334 if ( ((host_virt_addr != 0) || (dev_hst_ro_flags & GNTMAP_host_map) ) &&
335 unlikely(!__addr_ok(host_virt_addr)))
336 {
337 DPRINTK("Bad virtual address (%x) or flags (%x).\n",
338 host_virt_addr, dev_hst_ro_flags);
339 (void)__put_user(GNTST_bad_virt_addr, &uop->handle);
340 return GNTST_bad_gntref;
341 }
343 if ( unlikely(ref >= NR_GRANT_ENTRIES) ||
344 unlikely((dev_hst_ro_flags & (GNTMAP_device_map|GNTMAP_host_map)) ==
345 0) )
346 {
347 DPRINTK("Bad ref (%d) or flags (%x).\n", ref, dev_hst_ro_flags);
348 (void)__put_user(GNTST_bad_gntref, &uop->handle);
349 return GNTST_bad_gntref;
350 }
352 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
353 unlikely(ld == rd) )
354 {
355 if ( rd != NULL )
356 put_domain(rd);
357 DPRINTK("Could not find domain %d\n", dom);
358 (void)__put_user(GNTST_bad_domain, &uop->handle);
359 return GNTST_bad_domain;
360 }
362 /* get a maptrack handle */
363 if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
364 {
365 int i;
366 grant_mapping_t *new_mt;
367 grant_table_t *lgt = ld->grant_table;
369 /* grow the maptrack table */
370 if ( (new_mt = (void *)alloc_xenheap_pages(lgt->maptrack_order + 1)) == NULL )
371 {
372 put_domain(rd);
373 DPRINTK("No more map handles available\n");
374 (void)__put_user(GNTST_no_device_space, &uop->handle);
375 return GNTST_no_device_space;
376 }
378 memcpy(new_mt, lgt->maptrack, PAGE_SIZE << lgt->maptrack_order);
379 for ( i = lgt->maptrack_limit; i < (lgt->maptrack_limit << 1); i++ )
380 new_mt[i].ref_and_flags = (i+1) << MAPTRACK_REF_SHIFT;
382 free_xenheap_pages((unsigned long)lgt->maptrack, lgt->maptrack_order);
383 lgt->maptrack = new_mt;
384 lgt->maptrack_order += 1;
385 lgt->maptrack_limit <<= 1;
387 printk("Doubled maptrack size\n");
388 handle = get_maptrack_handle(ld->grant_table);
389 }
391 #if GRANT_DEBUG_VERBOSE
392 DPRINTK("Mapping grant ref (%hu) for domain (%hu) with flags (%x)\n",
393 ref, dom, dev_hst_ro_flags);
394 #endif
396 if ( 0 <= ( rc = __gnttab_activate_grant_ref( ld, led, rd, ref,
397 dev_hst_ro_flags,
398 host_virt_addr, &frame)))
399 {
400 /* Only make the maptrack live _after_ writing the pte,
401 * in case we overwrite the same frame number, causing a
402 * maptrack walk to find it
403 */
404 ld->grant_table->maptrack[handle].domid = dom;
406 ld->grant_table->maptrack[handle].ref_and_flags
407 = (ref << MAPTRACK_REF_SHIFT) |
408 (dev_hst_ro_flags & MAPTRACK_GNTMAP_MASK);
410 (void)__put_user(frame, &uop->dev_bus_addr);
412 if ( dev_hst_ro_flags & GNTMAP_host_map )
413 *va = host_virt_addr;
415 (void)__put_user(handle, &uop->handle);
416 }
417 else
418 {
419 (void)__put_user(rc, &uop->handle);
420 put_maptrack_handle(ld->grant_table, handle);
421 }
423 put_domain(rd);
424 return rc;
425 }
427 static long
428 gnttab_map_grant_ref(
429 gnttab_map_grant_ref_t *uop, unsigned int count)
430 {
431 int i, flush = 0;
432 unsigned long va[8];
434 for ( i = 0; i < count; i++ )
435 if ( __gnttab_map_grant_ref(&uop[i],
436 &va[ (flush < 8 ? flush : 0) ] ) == 0)
437 flush++;
439 if ( flush != 0 )
440 {
441 if ( flush <= 8 )
442 for ( i = 0; i < flush; i++ )
443 flush_tlb_one_mask(current->domain->cpuset, va[i]);
444 else
445 local_flush_tlb();
446 }
448 return 0;
449 }
451 static int
452 __gnttab_unmap_grant_ref(
453 gnttab_unmap_grant_ref_t *uop,
454 unsigned long *va)
455 {
456 domid_t dom;
457 grant_ref_t ref;
458 u16 handle;
459 struct domain *ld, *rd;
461 active_grant_entry_t *act;
462 grant_entry_t *sha;
463 grant_mapping_t *map;
464 u16 flags;
465 s16 rc = 1;
466 unsigned long frame, virt;
468 ld = current->domain;
470 /* Bitwise-OR avoids short-circuiting which screws control flow. */
471 if ( unlikely(__get_user(virt, &uop->host_virt_addr) |
472 __get_user(frame, &uop->dev_bus_addr) |
473 __get_user(handle, &uop->handle)) )
474 {
475 DPRINTK("Fault while reading gnttab_unmap_grant_ref_t.\n");
476 return -EFAULT; /* don't set status */
477 }
479 map = &ld->grant_table->maptrack[handle];
481 if ( unlikely(handle >= ld->grant_table->maptrack_limit) ||
482 unlikely(!(map->ref_and_flags & MAPTRACK_GNTMAP_MASK)) )
483 {
484 DPRINTK("Bad handle (%d).\n", handle);
485 (void)__put_user(GNTST_bad_handle, &uop->status);
486 return GNTST_bad_handle;
487 }
489 dom = map->domid;
490 ref = map->ref_and_flags >> MAPTRACK_REF_SHIFT;
491 flags = map->ref_and_flags & MAPTRACK_GNTMAP_MASK;
493 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
494 unlikely(ld == rd) )
495 {
496 if ( rd != NULL )
497 put_domain(rd);
498 DPRINTK("Could not find domain %d\n", dom);
499 (void)__put_user(GNTST_bad_domain, &uop->status);
500 return GNTST_bad_domain;
501 }
502 #if GRANT_DEBUG_VERBOSE
503 DPRINTK("Unmapping grant ref (%hu) for domain (%hu) with handle (%hu)\n",
504 ref, dom, handle);
505 #endif
507 act = &rd->grant_table->active[ref];
508 sha = &rd->grant_table->shared[ref];
510 spin_lock(&rd->grant_table->lock);
512 if ( frame == 0 )
513 frame = act->frame;
514 else if ( frame == GNTUNMAP_DEV_FROM_VIRT )
515 {
516 if ( !( flags & GNTMAP_device_map ) )
517 PIN_FAIL(unmap_out, GNTST_bad_dev_addr,
518 "Bad frame number: frame not mapped for device access.\n");
519 frame = act->frame;
521 /* frame will be unmapped for device access below if virt addr ok */
522 }
523 else
524 {
525 if ( unlikely(frame != act->frame) )
526 PIN_FAIL(unmap_out, GNTST_general_error,
527 "Bad frame number doesn't match gntref.\n");
528 if ( flags & GNTMAP_device_map )
529 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_devr_inc
530 : GNTPIN_devw_inc;
532 map->ref_and_flags &= ~GNTMAP_device_map;
533 (void)__put_user(0, &uop->dev_bus_addr);
535 /* frame is now unmapped for device access */
536 }
538 if ( (virt != 0) &&
539 (flags & GNTMAP_host_map) &&
540 ((act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask)) > 0))
541 {
542 l1_pgentry_t *pl1e;
543 unsigned long _ol1e;
545 pl1e = &linear_pg_table[l1_linear_offset(virt)];
547 if ( unlikely(__get_user(_ol1e, (unsigned long *)pl1e) != 0) )
548 {
549 DPRINTK("Could not find PTE entry for address %x\n", virt);
550 rc = -EINVAL;
551 goto unmap_out;
552 }
554 /* check that the virtual address supplied is actually
555 * mapped to act->frame.
556 */
557 if ( unlikely((_ol1e >> PAGE_SHIFT) != frame ))
558 {
559 DPRINTK("PTE entry %x for address %x doesn't match frame %x\n",
560 _ol1e, virt, frame);
561 rc = -EINVAL;
562 goto unmap_out;
563 }
565 /* Delete pagetable entry
566 */
567 if ( unlikely(__put_user(0, (unsigned long *)pl1e)))
568 {
569 DPRINTK("Cannot delete PTE entry at %x for virtual address %x\n",
570 pl1e, virt);
571 rc = -EINVAL;
572 goto unmap_out;
573 }
575 map->ref_and_flags &= ~GNTMAP_host_map;
577 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_hstr_inc
578 : GNTPIN_hstw_inc;
580 if ( frame == GNTUNMAP_DEV_FROM_VIRT )
581 {
582 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_devr_inc
583 : GNTPIN_devw_inc;
585 map->ref_and_flags &= ~GNTMAP_device_map;
586 (void)__put_user(0, &uop->dev_bus_addr);
587 }
589 rc = 0;
590 *va = virt;
591 }
593 if ( (map->ref_and_flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0)
594 {
595 map->ref_and_flags = 0;
596 put_maptrack_handle(ld->grant_table, handle);
597 }
599 /* If just unmapped a writable mapping, mark as dirtied */
600 if ( unlikely(shadow_mode_log_dirty(rd)) &&
601 !( flags & GNTMAP_readonly ) )
602 mark_dirty(rd, frame);
604 /* If the last writable mapping has been removed, put_page_type */
605 if ( ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask) ) == 0) &&
606 ( !( flags & GNTMAP_readonly ) ) )
607 {
608 clear_bit(_GTF_writing, &sha->flags);
609 put_page_type(&frame_table[frame]);
610 }
612 if ( act->pin == 0 )
613 {
614 clear_bit(_GTF_reading, &sha->flags);
615 put_page(&frame_table[frame]);
616 }
618 unmap_out:
619 (void)__put_user(rc, &uop->status);
620 spin_unlock(&rd->grant_table->lock);
621 put_domain(rd);
622 return rc;
623 }
625 static long
626 gnttab_unmap_grant_ref(
627 gnttab_unmap_grant_ref_t *uop, unsigned int count)
628 {
629 int i, flush = 0;
630 unsigned long va[8];
632 for ( i = 0; i < count; i++ )
633 if ( __gnttab_unmap_grant_ref(&uop[i],
634 &va[ (flush < 8 ? flush : 0) ] ) == 0)
635 flush++;
637 if ( flush != 0 )
638 {
639 if ( flush <= 8 )
640 for ( i = 0; i < flush; i++ )
641 flush_tlb_one_mask(current->domain->cpuset, va[i]);
642 else
643 local_flush_tlb();
644 }
646 return 0;
647 }
649 static long
650 gnttab_setup_table(
651 gnttab_setup_table_t *uop, unsigned int count)
652 {
653 gnttab_setup_table_t op;
654 struct domain *d;
655 int i;
657 if ( count != 1 )
658 return -EINVAL;
660 if ( unlikely(copy_from_user(&op, uop, sizeof(op)) != 0) )
661 {
662 DPRINTK("Fault while reading gnttab_setup_table_t.\n");
663 return -EFAULT;
664 }
666 if ( unlikely(op.nr_frames > NR_GRANT_FRAMES) )
667 {
668 DPRINTK("Xen only supports at most %d grant-table frames per domain.\n",
669 NR_GRANT_FRAMES);
670 (void)put_user(GNTST_general_error, &uop->status);
671 return 0;
672 }
674 if ( op.dom == DOMID_SELF )
675 {
676 op.dom = current->domain->id;
677 }
678 else if ( unlikely(!IS_PRIV(current->domain)) )
679 {
680 (void)put_user(GNTST_permission_denied, &uop->status);
681 return 0;
682 }
684 if ( unlikely((d = find_domain_by_id(op.dom)) == NULL) )
685 {
686 DPRINTK("Bad domid %d.\n", op.dom);
687 (void)put_user(GNTST_bad_domain, &uop->status);
688 return 0;
689 }
691 if ( op.nr_frames <= NR_GRANT_FRAMES )
692 {
693 ASSERT(d->grant_table != NULL);
694 (void)put_user(GNTST_okay, &uop->status);
696 for ( i = 0; i < op.nr_frames; i++ )
697 (void)put_user( (
698 virt_to_phys( (char*)(d->grant_table->shared)+(i*PAGE_SIZE) )
699 >> PAGE_SHIFT ), &uop->frame_list[i]);
700 }
702 put_domain(d);
703 return 0;
704 }
706 #if GRANT_DEBUG
707 static int
708 gnttab_dump_table(gnttab_dump_table_t *uop)
709 {
710 grant_table_t *gt;
711 gnttab_dump_table_t op;
712 struct domain *d;
713 u32 shared_mfn;
714 active_grant_entry_t *act;
715 grant_entry_t sha_copy;
716 grant_mapping_t *maptrack;
717 int i;
720 if ( unlikely(copy_from_user(&op, uop, sizeof(op)) != 0) )
721 {
722 DPRINTK("Fault while reading gnttab_dump_table_t.\n");
723 return -EFAULT;
724 }
726 if ( op.dom == DOMID_SELF )
727 {
728 op.dom = current->domain->id;
729 }
731 if ( unlikely((d = find_domain_by_id(op.dom)) == NULL) )
732 {
733 DPRINTK("Bad domid %d.\n", op.dom);
734 (void)put_user(GNTST_bad_domain, &uop->status);
735 return 0;
736 }
738 ASSERT(d->grant_table != NULL);
739 gt = d->grant_table;
740 (void)put_user(GNTST_okay, &uop->status);
742 shared_mfn = virt_to_phys(d->grant_table->shared);
744 DPRINTK("Grant table for dom (%hu) MFN (%x)\n",
745 op.dom, shared_mfn);
747 ASSERT(d->grant_table->active != NULL);
748 ASSERT(d->grant_table->shared != NULL);
749 ASSERT(d->grant_table->maptrack != NULL);
751 for ( i = 0; i < NR_GRANT_ENTRIES; i++ )
752 {
753 sha_copy = gt->shared[i];
755 if ( sha_copy.flags )
756 {
757 DPRINTK("Grant: dom (%hu) SHARED (%d) flags:(%hx) dom:(%hu) frame:(%lx)\n",
758 op.dom, i, sha_copy.flags, sha_copy.domid, sha_copy.frame);
759 }
760 }
762 spin_lock(&gt->lock);
764 for ( i = 0; i < NR_GRANT_ENTRIES; i++ )
765 {
766 act = &gt->active[i];
768 if ( act->pin )
769 {
770 DPRINTK("Grant: dom (%hu) ACTIVE (%d) pin:(%x) dom:(%hu) frame:(%lx)\n",
771 op.dom, i, act->pin, act->domid, act->frame);
772 }
773 }
775 for ( i = 0; i < gt->maptrack_limit; i++ )
776 {
777 maptrack = &gt->maptrack[i];
779 if ( maptrack->ref_and_flags & MAPTRACK_GNTMAP_MASK )
780 {
781 DPRINTK("Grant: dom (%hu) MAP (%d) ref:(%hu) flags:(%x) dom:(%hu)\n",
782 op.dom, i,
783 maptrack->ref_and_flags >> MAPTRACK_REF_SHIFT,
784 maptrack->ref_and_flags & MAPTRACK_GNTMAP_MASK,
785 maptrack->domid);
786 }
787 }
789 spin_unlock(&gt->lock);
791 put_domain(d);
792 return 0;
793 }
794 #endif
796 long
797 do_grant_table_op(
798 unsigned int cmd, void *uop, unsigned int count)
799 {
800 long rc;
802 if ( count > 512 )
803 return -EINVAL;
805 LOCK_BIGLOCK(current->domain);
807 rc = -EFAULT;
808 switch ( cmd )
809 {
810 case GNTTABOP_map_grant_ref:
811 if ( unlikely(!array_access_ok(
812 VERIFY_WRITE, uop, count, sizeof(gnttab_map_grant_ref_t))) )
813 goto out;
814 rc = gnttab_map_grant_ref((gnttab_map_grant_ref_t *)uop, count);
815 break;
816 case GNTTABOP_unmap_grant_ref:
817 if ( unlikely(!array_access_ok(
818 VERIFY_WRITE, uop, count, sizeof(gnttab_unmap_grant_ref_t))) )
819 goto out;
820 rc = gnttab_unmap_grant_ref((gnttab_unmap_grant_ref_t *)uop, count);
821 break;
822 case GNTTABOP_setup_table:
823 rc = gnttab_setup_table((gnttab_setup_table_t *)uop, count);
824 break;
825 #if GRANT_DEBUG
826 case GNTTABOP_dump_table:
827 rc = gnttab_dump_table((gnttab_dump_table_t *)uop);
828 break;
829 #endif
830 default:
831 rc = -ENOSYS;
832 break;
833 }
835 out:
836 UNLOCK_BIGLOCK(current->domain);
838 return rc;
839 }
841 int
842 gnttab_check_unmap(
843 struct domain *rd, struct domain *ld, unsigned long frame, int readonly)
844 {
845 /* Called when put_page is invoked on a page belonging to a foreign domain.
846 * Instead of decrementing the frame table ref count, locate the grant
847 * table entry, if any, and if found, decrement that count.
848 * Called a _lot_ at domain creation because pages mapped by priv domains
849 * also traverse this.
850 */
852 /* Note: if the same frame is mapped multiple times, and then one of
853 * the ptes is overwritten, which maptrack handle gets invalidated?
854 * Advice: don't do it.
855 */
857 unsigned int handle, ref, refcount;
858 grant_table_t *lgt, *rgt;
859 active_grant_entry_t *act;
860 grant_mapping_t *map;
861 int found = 0;
863 lgt = ld->grant_table;
865 #if GRANT_DEBUG_VERBOSE
866 if ( ld->id != 0 )
867 {
868 DPRINTK("Foreign unref rd(%d) ld(%d) frm(%x) flgs(%x).\n",
869 rd->id, ld->id, frame, readonly);
870 }
871 #endif
873 /* Fast exit if we're not mapping anything using grant tables */
874 if ( lgt->map_count == 0 )
875 return 0;
877 if ( get_domain(rd) == 0 )
878 {
879 DPRINTK("gnttab_check_unmap: couldn't get_domain rd(%d)\n", rd->id);
880 return 0;
881 }
883 rgt = rd->grant_table;
885 for ( handle = 0; handle < lgt->maptrack_limit; handle++ )
886 {
887 map = &lgt->maptrack[handle];
890 /* cwc22: if multiple grants of the same frame are disallowed,
891 * then the readonly check here can be changed to cause an early abort
892 * if we've matched on frame, but not on write permission.
893 */
894 if ( ( map->ref_and_flags & MAPTRACK_GNTMAP_MASK ) &&
895 ( readonly ? 1 : (!(map->ref_and_flags & GNTMAP_readonly))))
896 {
897 ref = (map->ref_and_flags >> MAPTRACK_REF_SHIFT);
898 act = &rgt->active[ref];
900 spin_lock(&rgt->lock);
902 if ( act->frame != frame )
903 {
904 spin_unlock(&rgt->lock);
905 continue;
906 }
908 refcount = act->pin & ( readonly ? GNTPIN_hstr_mask
909 : GNTPIN_hstw_mask );
910 if ( refcount == 0 )
911 {
912 spin_unlock(&rgt->lock);
913 continue;
914 }
916 /* gotcha */
917 DPRINTK("Grant unref rd(%d) ld(%d) frm(%x) flgs(%x).\n",
918 rd->id, ld->id, frame, readonly);
920 if ( readonly )
921 act->pin -= GNTPIN_hstr_inc;
922 else
923 {
924 act->pin -= GNTPIN_hstw_inc;
926 /* any more granted writable mappings? */
927 if ( (act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) == 0 )
928 {
929 clear_bit(_GTF_writing, &rgt->shared[ref].flags);
930 put_page_type(&frame_table[frame]);
931 }
932 }
934 if ( act->pin == 0 )
935 {
936 clear_bit(_GTF_reading, &rgt->shared[ref].flags);
937 put_page(&frame_table[frame]);
938 }
939 spin_unlock(&rgt->lock);
941 clear_bit(GNTMAP_host_map, &map->ref_and_flags);
943 if ( !(map->ref_and_flags & GNTMAP_device_map) )
944 put_maptrack_handle(lgt, handle);
946 found = 1;
947 break;
948 }
949 }
950 put_domain(rd);
952 return found;
953 }
955 int
956 gnttab_prepare_for_transfer(
957 struct domain *rd, struct domain *ld, grant_ref_t ref)
958 {
959 grant_table_t *rgt;
960 grant_entry_t *sha;
961 domid_t sdom;
962 u16 sflags;
963 u32 scombo, prev_scombo;
964 int retries = 0;
965 unsigned long target_pfn;
967 DPRINTK("gnttab_prepare_for_transfer rd(%hu) ld(%hu) ref(%hu).\n",
968 rd->id, ld->id, ref);
970 if ( unlikely((rgt = rd->grant_table) == NULL) ||
971 unlikely(ref >= NR_GRANT_ENTRIES) )
972 {
973 DPRINTK("Dom %d has no g.t., or ref is bad (%d).\n", rd->id, ref);
974 return 0;
975 }
977 spin_lock(&rgt->lock);
979 sha = &rgt->shared[ref];
981 sflags = sha->flags;
982 sdom = sha->domid;
984 for ( ; ; )
985 {
986 target_pfn = sha->frame;
988 if ( unlikely(target_pfn >= max_page ) )
989 {
990 DPRINTK("Bad pfn (%x)\n", target_pfn);
991 goto fail;
992 }
994 if ( unlikely(sflags != GTF_accept_transfer) ||
995 unlikely(sdom != ld->id) )
996 {
997 DPRINTK("Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
998 sflags, sdom, ld->id);
999 goto fail;
1002 /* Merge two 16-bit values into a 32-bit combined update. */
1003 /* NB. Endianness! */
1004 prev_scombo = scombo = ((u32)sdom << 16) | (u32)sflags;
1006 /* NB. prev_scombo is updated in place to seen value. */
1007 if ( unlikely(cmpxchg_user((u32 *)&sha->flags, prev_scombo,
1008 prev_scombo | GTF_transfer_committed)) )
1010 DPRINTK("Fault while modifying shared flags and domid.\n");
1011 goto fail;
1014 /* Did the combined update work (did we see what we expected?). */
1015 if ( likely(prev_scombo == scombo) )
1016 break;
1018 if ( retries++ == 4 )
1020 DPRINTK("Shared grant entry is unstable.\n");
1021 goto fail;
1024 /* Didn't see what we expected. Split out the seen flags & dom. */
1025 /* NB. Endianness! */
1026 sflags = (u16)prev_scombo;
1027 sdom = (u16)(prev_scombo >> 16);
1030 spin_unlock(&rgt->lock);
1031 return 1;
1033 fail:
1034 spin_unlock(&rgt->lock);
1035 return 0;
1038 void
1039 gnttab_notify_transfer(
1040 struct domain *rd, struct domain *ld, grant_ref_t ref, unsigned long frame)
1042 grant_entry_t *sha;
1043 unsigned long pfn;
1045 DPRINTK("gnttab_notify_transfer rd(%hu) ld(%hu) ref(%hu).\n",
1046 rd->id, ld->id, ref);
1048 sha = &rd->grant_table->shared[ref];
1050 spin_lock(&rd->grant_table->lock);
1052 pfn = sha->frame;
1054 if ( unlikely(pfn >= max_page ) )
1055 DPRINTK("Bad pfn (%x)\n", pfn);
1056 else
1058 machine_to_phys_mapping[frame] = pfn;
1060 if ( unlikely(shadow_mode_log_dirty(ld)))
1061 mark_dirty(ld, frame);
1063 if (shadow_mode_translate(ld))
1064 __phys_to_machine_mapping[pfn] = frame;
1066 sha->frame = __mfn_to_gpfn(rd, frame);
1067 sha->domid = rd->id;
1068 wmb();
1069 sha->flags = ( GTF_accept_transfer | GTF_transfer_completed );
1071 spin_unlock(&rd->grant_table->lock);
1073 return;
1076 int
1077 grant_table_create(
1078 struct domain *d)
1080 grant_table_t *t;
1081 int i;
1083 if ( (t = xmalloc(grant_table_t)) == NULL )
1084 goto no_mem;
1086 /* Simple stuff. */
1087 memset(t, 0, sizeof(*t));
1088 spin_lock_init(&t->lock);
1090 /* Active grant table. */
1091 if ( (t->active = xmalloc_array(active_grant_entry_t, NR_GRANT_ENTRIES))
1092 == NULL )
1093 goto no_mem;
1094 memset(t->active, 0, sizeof(active_grant_entry_t) * NR_GRANT_ENTRIES);
1096 /* Tracking of mapped foreign frames table */
1097 if ( (t->maptrack = (void *)alloc_xenheap_page()) == NULL )
1098 goto no_mem;
1099 t->maptrack_order = 0;
1100 t->maptrack_limit = PAGE_SIZE / sizeof(grant_mapping_t);
1101 memset(t->maptrack, 0, PAGE_SIZE);
1102 for ( i = 0; i < t->maptrack_limit; i++ )
1103 t->maptrack[i].ref_and_flags = (i+1) << MAPTRACK_REF_SHIFT;
1105 /* Shared grant table. */
1106 if ( (t->shared = (void *)alloc_xenheap_pages(ORDER_GRANT_FRAMES)) == NULL )
1107 goto no_mem;
1108 memset(t->shared, 0, NR_GRANT_FRAMES * PAGE_SIZE);
1110 for ( i = 0; i < NR_GRANT_FRAMES; i++ )
1112 SHARE_PFN_WITH_DOMAIN(virt_to_page((char *)(t->shared)+(i*PAGE_SIZE)), d);
1113 machine_to_phys_mapping[ (virt_to_phys((char*)(t->shared)+(i*PAGE_SIZE))
1114 >> PAGE_SHIFT) ] = INVALID_M2P_ENTRY;
1117 /* Okay, install the structure. */
1118 wmb(); /* avoid races with lock-free access to d->grant_table */
1119 d->grant_table = t;
1120 return 0;
1122 no_mem:
1123 if ( t != NULL )
1125 if ( t->active != NULL )
1126 xfree(t->active);
1127 if ( t->maptrack != NULL )
1128 free_xenheap_page((unsigned long)t->maptrack);
1129 xfree(t);
1131 return -ENOMEM;
1134 void
1135 gnttab_release_dev_mappings(grant_table_t *gt)
1137 grant_mapping_t *map;
1138 domid_t dom;
1139 grant_ref_t ref;
1140 u16 handle;
1141 struct domain *ld, *rd;
1142 unsigned long frame;
1143 active_grant_entry_t *act;
1144 grant_entry_t *sha;
1146 ld = current->domain;
1148 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
1150 map = &gt->maptrack[handle];
1152 if ( map->ref_and_flags & GNTMAP_device_map )
1154 dom = map->domid;
1155 ref = map->ref_and_flags >> MAPTRACK_REF_SHIFT;
1157 DPRINTK("Grant release (%hu) ref:(%hu) flags:(%x) dom:(%hu)\n",
1158 handle, ref,
1159 map->ref_and_flags & MAPTRACK_GNTMAP_MASK, dom);
1161 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
1162 unlikely(ld == rd) )
1164 if ( rd != NULL )
1165 put_domain(rd);
1167 printk(KERN_WARNING "Grant release: Could not find domain %d\n", dom);
1168 continue;
1171 act = &rd->grant_table->active[ref];
1172 sha = &rd->grant_table->shared[ref];
1174 spin_lock(&rd->grant_table->lock);
1176 if ( act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask) )
1178 frame = act->frame;
1180 if ( ( (act->pin & GNTPIN_hstw_mask) == 0 ) &&
1181 ( (act->pin & GNTPIN_devw_mask) > 0 ) )
1183 clear_bit(_GTF_writing, &sha->flags);
1184 put_page_type(&frame_table[frame]);
1187 act->pin &= ~(GNTPIN_devw_mask | GNTPIN_devr_mask);
1189 if ( act->pin == 0 )
1191 clear_bit(_GTF_reading, &sha->flags);
1192 map->ref_and_flags = 0;
1193 put_page(&frame_table[frame]);
1195 else
1196 map->ref_and_flags &= ~GNTMAP_device_map;
1199 spin_unlock(&rd->grant_table->lock);
1201 put_domain(rd);
1207 void
1208 grant_table_destroy(
1209 struct domain *d)
1211 grant_table_t *t;
1213 if ( (t = d->grant_table) != NULL )
1215 /* Free memory relating to this grant table. */
1216 d->grant_table = NULL;
1217 free_xenheap_pages((unsigned long)t->shared, ORDER_GRANT_FRAMES);
1218 free_xenheap_page((unsigned long)t->maptrack); //cwc22
1219 xfree(t->active);
1220 xfree(t);
1224 void
1225 grant_table_init(
1226 void)
1228 /* Nothing. */
1229 DPRINTK("Grant table init\n");
1232 /*
1233 * Local variables:
1234 * mode: C
1235 * c-set-style: "BSD"
1236 * c-basic-offset: 4
1237 * tab-width: 4
1238 * indent-tabs-mode: nil
1239 * End:
1240 */