ia64/xen-unstable

view xen/common/grant_table.c @ 19658:28a197617286

Fix up the synchronisation around grant table map track handles.
At present, we're not doing any at all, so if a domain e.g. tries to
do two map operations at the same time from different vcpus then you
could end up with both operations getting back the same maptrack
handle.

Fix this problem by just shoving an enormous lock around grant table
operations. This is unlikely to be heavily contended, because netback
and blkback both restrict themselves to mapping on a single vcpu at a
time (globally for netback, and per-device for blkback), and most of
the interesting bits are already protected by the remote domain's
grant table lock anyway.

The unconteded acquisition cost might be significant for some
workloads. If that were the case, it might be worth only acquiring
the lock only for multi-vcpu domains, since we only manipulate the
maptrack table in the context of one of the domain's vcpus. I've not
done that optimisation here, because I didn't want to think about what
would happen if e.g. a cpu got hot-unplugged from a domain while it
was performing a map operation.

Signed-off-by: Steven Smith <steven.smith@citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Wed May 27 11:29:38 2009 +0100 (2009-05-27)
parents 822ea2bf0c54
children 2e83c670f680
line source
1 /******************************************************************************
2 * common/grant_table.c
3 *
4 * Mechanism for granting foreign access to page frames, and receiving
5 * page-ownership transfers.
6 *
7 * Copyright (c) 2005-2006 Christopher Clark
8 * Copyright (c) 2004 K A Fraser
9 * Copyright (c) 2005 Andrew Warfield
10 * Modifications by Geoffrey Lefebvre are (c) Intel Research Cambridge
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software
24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 */
27 #include <xen/config.h>
28 #include <xen/iocap.h>
29 #include <xen/lib.h>
30 #include <xen/sched.h>
31 #include <xen/mm.h>
32 #include <xen/trace.h>
33 #include <xen/guest_access.h>
34 #include <xen/domain_page.h>
35 #include <xen/iommu.h>
36 #include <xen/paging.h>
37 #include <xsm/xsm.h>
39 #ifndef max_nr_grant_frames
40 unsigned int max_nr_grant_frames = DEFAULT_MAX_NR_GRANT_FRAMES;
41 integer_param("gnttab_max_nr_frames", max_nr_grant_frames);
42 #endif
44 /* The maximum number of grant mappings is defined as a multiplier of the
45 * maximum number of grant table entries. This defines the multiplier used.
46 * Pretty arbitrary. [POLICY]
47 */
48 #define MAX_MAPTRACK_TO_GRANTS_RATIO 8
50 /*
51 * The first two members of a grant entry are updated as a combined pair.
52 * The following union allows that to happen in an endian-neutral fashion.
53 */
54 union grant_combo {
55 uint32_t word;
56 struct {
57 uint16_t flags;
58 domid_t domid;
59 } shorts;
60 };
62 /* Used to share code between unmap_grant_ref and unmap_and_replace. */
63 struct gnttab_unmap_common {
64 /* Input */
65 uint64_t host_addr;
66 uint64_t dev_bus_addr;
67 uint64_t new_addr;
68 grant_handle_t handle;
70 /* Return */
71 int16_t status;
73 /* Shared state beteen *_unmap and *_unmap_complete */
74 u16 flags;
75 unsigned long frame;
76 struct grant_mapping *map;
77 struct domain *rd;
78 };
80 /* Number of unmap operations that are done between each tlb flush */
81 #define GNTTAB_UNMAP_BATCH_SIZE 32
84 #define PIN_FAIL(_lbl, _rc, _f, _a...) \
85 do { \
86 gdprintk(XENLOG_WARNING, _f, ## _a ); \
87 rc = (_rc); \
88 goto _lbl; \
89 } while ( 0 )
91 #define MAPTRACK_PER_PAGE (PAGE_SIZE / sizeof(struct grant_mapping))
92 #define maptrack_entry(t, e) \
93 ((t)->maptrack[(e)/MAPTRACK_PER_PAGE][(e)%MAPTRACK_PER_PAGE])
95 static inline unsigned int
96 nr_maptrack_frames(struct grant_table *t)
97 {
98 return t->maptrack_limit / MAPTRACK_PER_PAGE;
99 }
101 static unsigned inline int max_nr_maptrack_frames(void)
102 {
103 return (max_nr_grant_frames * MAX_MAPTRACK_TO_GRANTS_RATIO);
104 }
107 #define SHGNT_PER_PAGE (PAGE_SIZE / sizeof(grant_entry_t))
108 #define shared_entry(t, e) \
109 ((t)->shared[(e)/SHGNT_PER_PAGE][(e)%SHGNT_PER_PAGE])
110 #define ACGNT_PER_PAGE (PAGE_SIZE / sizeof(struct active_grant_entry))
111 #define active_entry(t, e) \
112 ((t)->active[(e)/ACGNT_PER_PAGE][(e)%ACGNT_PER_PAGE])
114 /* Technically, we only really need to acquire the lock for SMP
115 guests, because you only ever touch the maptrack tables from the
116 context of the guest which owns them, so if it's uniproc then the
117 lock can't be contended, and is therefore pointless. Don't bother
118 with that optimisation for now, though, because it's scary and
119 confusing. */
120 /* The maptrack lock is top-level: you're not allowed to be holding
121 any other locks when you acquire it. */
122 static void
123 maptrack_lock(struct grant_table *lgt)
124 {
125 spin_lock(&lgt->maptrack_lock);
126 }
128 static void
129 maptrack_unlock(struct grant_table *lgt)
130 {
131 spin_unlock(&lgt->maptrack_lock);
132 }
134 static inline int
135 __get_maptrack_handle(
136 struct grant_table *t)
137 {
138 unsigned int h;
139 if ( unlikely((h = t->maptrack_head) == (t->maptrack_limit - 1)) )
140 return -1;
141 t->maptrack_head = maptrack_entry(t, h).ref;
142 return h;
143 }
145 static inline void
146 put_maptrack_handle(
147 struct grant_table *t, int handle)
148 {
149 maptrack_entry(t, handle).ref = t->maptrack_head;
150 t->maptrack_head = handle;
151 }
153 static inline int
154 get_maptrack_handle(
155 struct grant_table *lgt)
156 {
157 int i;
158 grant_handle_t handle;
159 struct grant_mapping *new_mt;
160 unsigned int new_mt_limit, nr_frames;
162 if ( unlikely((handle = __get_maptrack_handle(lgt)) == -1) )
163 {
164 nr_frames = nr_maptrack_frames(lgt);
165 if ( nr_frames >= max_nr_maptrack_frames() )
166 return -1;
168 new_mt = alloc_xenheap_page();
169 if ( new_mt == NULL )
170 return -1;
172 clear_page(new_mt);
174 new_mt_limit = lgt->maptrack_limit + MAPTRACK_PER_PAGE;
176 for ( i = lgt->maptrack_limit; i < new_mt_limit; i++ )
177 {
178 new_mt[i % MAPTRACK_PER_PAGE].ref = i+1;
179 new_mt[i % MAPTRACK_PER_PAGE].flags = 0;
180 }
182 lgt->maptrack[nr_frames] = new_mt;
183 lgt->maptrack_limit = new_mt_limit;
185 gdprintk(XENLOG_INFO,
186 "Increased maptrack size to %u frames.\n", nr_frames + 1);
187 handle = __get_maptrack_handle(lgt);
188 }
189 return handle;
190 }
192 /*
193 * Returns 0 if TLB flush / invalidate required by caller.
194 * va will indicate the address to be invalidated.
195 *
196 * addr is _either_ a host virtual address, or the address of the pte to
197 * update, as indicated by the GNTMAP_contains_pte flag.
198 */
199 static void
200 __gnttab_map_grant_ref(
201 struct gnttab_map_grant_ref *op)
202 {
203 struct domain *ld, *rd, *owner;
204 struct vcpu *led;
205 int handle;
206 unsigned long frame = 0, nr_gets = 0;
207 int rc = GNTST_okay;
208 u32 old_pin;
209 u32 act_pin;
210 unsigned int cache_flags;
211 struct active_grant_entry *act;
212 struct grant_mapping *mt;
213 grant_entry_t *sha;
214 union grant_combo scombo, prev_scombo, new_scombo;
216 /*
217 * We bound the number of times we retry CMPXCHG on memory locations that
218 * we share with a guest OS. The reason is that the guest can modify that
219 * location at a higher rate than we can read-modify-CMPXCHG, so the guest
220 * could cause us to livelock. There are a few cases where it is valid for
221 * the guest to race our updates (e.g., to change the GTF_readonly flag),
222 * so we allow a few retries before failing.
223 */
224 int retries = 0;
226 led = current;
227 ld = led->domain;
229 if ( unlikely((op->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0) )
230 {
231 gdprintk(XENLOG_INFO, "Bad flags in grant map op (%x).\n", op->flags);
232 op->status = GNTST_bad_gntref;
233 return;
234 }
236 if ( unlikely((rd = rcu_lock_domain_by_id(op->dom)) == NULL) )
237 {
238 gdprintk(XENLOG_INFO, "Could not find domain %d\n", op->dom);
239 op->status = GNTST_bad_domain;
240 return;
241 }
243 rc = xsm_grant_mapref(ld, rd, op->flags);
244 if ( rc )
245 {
246 rcu_unlock_domain(rd);
247 op->status = GNTST_permission_denied;
248 return;
249 }
251 if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
252 {
253 rcu_unlock_domain(rd);
254 gdprintk(XENLOG_INFO, "Failed to obtain maptrack handle.\n");
255 op->status = GNTST_no_device_space;
256 return;
257 }
259 spin_lock(&rd->grant_table->lock);
261 /* Bounds check on the grant ref */
262 if ( unlikely(op->ref >= nr_grant_entries(rd->grant_table)))
263 PIN_FAIL(unlock_out, GNTST_bad_gntref, "Bad ref (%d).\n", op->ref);
265 act = &active_entry(rd->grant_table, op->ref);
266 sha = &shared_entry(rd->grant_table, op->ref);
268 /* If already pinned, check the active domid and avoid refcnt overflow. */
269 if ( act->pin &&
270 ((act->domid != ld->domain_id) ||
271 (act->pin & 0x80808080U) != 0) )
272 PIN_FAIL(unlock_out, GNTST_general_error,
273 "Bad domain (%d != %d), or risk of counter overflow %08x\n",
274 act->domid, ld->domain_id, act->pin);
276 if ( !act->pin ||
277 (!(op->flags & GNTMAP_readonly) &&
278 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask))) )
279 {
280 scombo.word = *(u32 *)&sha->flags;
282 /*
283 * This loop attempts to set the access (reading/writing) flags
284 * in the grant table entry. It tries a cmpxchg on the field
285 * up to five times, and then fails under the assumption that
286 * the guest is misbehaving.
287 */
288 for ( ; ; )
289 {
290 /* If not already pinned, check the grant domid and type. */
291 if ( !act->pin &&
292 (((scombo.shorts.flags & GTF_type_mask) !=
293 GTF_permit_access) ||
294 (scombo.shorts.domid != ld->domain_id)) )
295 PIN_FAIL(unlock_out, GNTST_general_error,
296 "Bad flags (%x) or dom (%d). (expected dom %d)\n",
297 scombo.shorts.flags, scombo.shorts.domid,
298 ld->domain_id);
300 new_scombo = scombo;
301 new_scombo.shorts.flags |= GTF_reading;
303 if ( !(op->flags & GNTMAP_readonly) )
304 {
305 new_scombo.shorts.flags |= GTF_writing;
306 if ( unlikely(scombo.shorts.flags & GTF_readonly) )
307 PIN_FAIL(unlock_out, GNTST_general_error,
308 "Attempt to write-pin a r/o grant entry.\n");
309 }
311 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
312 scombo.word, new_scombo.word);
313 if ( likely(prev_scombo.word == scombo.word) )
314 break;
316 if ( retries++ == 4 )
317 PIN_FAIL(unlock_out, GNTST_general_error,
318 "Shared grant entry is unstable.\n");
320 scombo = prev_scombo;
321 }
323 if ( !act->pin )
324 {
325 act->domid = scombo.shorts.domid;
326 act->gfn = sha->frame;
327 act->frame = gmfn_to_mfn(rd, sha->frame);
328 }
329 }
331 old_pin = act->pin;
332 if ( op->flags & GNTMAP_device_map )
333 act->pin += (op->flags & GNTMAP_readonly) ?
334 GNTPIN_devr_inc : GNTPIN_devw_inc;
335 if ( op->flags & GNTMAP_host_map )
336 act->pin += (op->flags & GNTMAP_readonly) ?
337 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
339 frame = act->frame;
340 act_pin = act->pin;
342 cache_flags = (sha->flags & (GTF_PAT | GTF_PWT | GTF_PCD) );
344 spin_unlock(&rd->grant_table->lock);
346 if ( !mfn_valid(frame) ||
347 (owner = page_get_owner_and_reference(mfn_to_page(frame))) == dom_io )
348 {
349 /* Only needed the reference to confirm dom_io ownership. */
350 if ( mfn_valid(frame) )
351 put_page(mfn_to_page(frame));
353 if ( !iomem_access_permitted(rd, frame, frame) )
354 {
355 gdprintk(XENLOG_WARNING,
356 "Iomem mapping not permitted %lx (domain %d)\n",
357 frame, rd->domain_id);
358 rc = GNTST_general_error;
359 goto undo_out;
360 }
362 rc = create_grant_host_mapping(
363 op->host_addr, frame, op->flags, cache_flags);
364 if ( rc != GNTST_okay )
365 goto undo_out;
366 }
367 else if ( owner == rd )
368 {
369 if ( gnttab_host_mapping_get_page_type(op, ld, rd) &&
370 !get_page_type(mfn_to_page(frame), PGT_writable_page) )
371 goto could_not_pin;
373 nr_gets++;
374 if ( op->flags & GNTMAP_host_map )
375 {
376 rc = create_grant_host_mapping(op->host_addr, frame, op->flags, 0);
377 if ( rc != GNTST_okay )
378 goto undo_out;
380 if ( op->flags & GNTMAP_device_map )
381 {
382 nr_gets++;
383 (void)get_page(mfn_to_page(frame), rd);
384 if ( !(op->flags & GNTMAP_readonly) )
385 get_page_type(mfn_to_page(frame), PGT_writable_page);
386 }
387 }
388 }
389 else
390 {
391 could_not_pin:
392 if ( !rd->is_dying )
393 gdprintk(XENLOG_WARNING, "Could not pin grant frame %lx\n",
394 frame);
395 if ( owner != NULL )
396 put_page(mfn_to_page(frame));
397 rc = GNTST_general_error;
398 goto undo_out;
399 }
401 if ( need_iommu(ld) &&
402 !(old_pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) &&
403 (act_pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
404 {
405 if ( iommu_map_page(ld, mfn_to_gmfn(ld, frame), frame) )
406 {
407 rc = GNTST_general_error;
408 goto undo_out;
409 }
410 }
412 TRACE_1D(TRC_MEM_PAGE_GRANT_MAP, op->dom);
414 mt = &maptrack_entry(ld->grant_table, handle);
415 mt->domid = op->dom;
416 mt->ref = op->ref;
417 mt->flags = op->flags;
419 op->dev_bus_addr = (u64)frame << PAGE_SHIFT;
420 op->handle = handle;
421 op->status = GNTST_okay;
423 rcu_unlock_domain(rd);
424 return;
426 undo_out:
427 if ( nr_gets > 1 )
428 {
429 if ( !(op->flags & GNTMAP_readonly) )
430 put_page_type(mfn_to_page(frame));
431 put_page(mfn_to_page(frame));
432 }
433 if ( nr_gets > 0 )
434 {
435 if ( gnttab_host_mapping_get_page_type(op, ld, rd) )
436 put_page_type(mfn_to_page(frame));
437 put_page(mfn_to_page(frame));
438 }
440 spin_lock(&rd->grant_table->lock);
442 act = &active_entry(rd->grant_table, op->ref);
443 sha = &shared_entry(rd->grant_table, op->ref);
445 if ( op->flags & GNTMAP_device_map )
446 act->pin -= (op->flags & GNTMAP_readonly) ?
447 GNTPIN_devr_inc : GNTPIN_devw_inc;
448 if ( op->flags & GNTMAP_host_map )
449 act->pin -= (op->flags & GNTMAP_readonly) ?
450 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
452 if ( !(op->flags & GNTMAP_readonly) &&
453 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
454 gnttab_clear_flag(_GTF_writing, &sha->flags);
456 if ( !act->pin )
457 gnttab_clear_flag(_GTF_reading, &sha->flags);
459 unlock_out:
460 spin_unlock(&rd->grant_table->lock);
461 op->status = rc;
462 put_maptrack_handle(ld->grant_table, handle);
463 rcu_unlock_domain(rd);
464 }
466 static long
467 gnttab_map_grant_ref(
468 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) uop, unsigned int count)
469 {
470 int i;
471 struct gnttab_map_grant_ref op;
473 for ( i = 0; i < count; i++ )
474 {
475 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
476 return -EFAULT;
477 __gnttab_map_grant_ref(&op);
478 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
479 return -EFAULT;
480 }
482 return 0;
483 }
485 static void
486 __gnttab_unmap_common(
487 struct gnttab_unmap_common *op)
488 {
489 domid_t dom;
490 struct domain *ld, *rd;
491 struct active_grant_entry *act;
492 grant_entry_t *sha;
493 s16 rc = 0;
494 u32 old_pin;
496 ld = current->domain;
498 op->frame = (unsigned long)(op->dev_bus_addr >> PAGE_SHIFT);
500 if ( unlikely(op->handle >= ld->grant_table->maptrack_limit) )
501 {
502 gdprintk(XENLOG_INFO, "Bad handle (%d).\n", op->handle);
503 op->status = GNTST_bad_handle;
504 return;
505 }
507 op->map = &maptrack_entry(ld->grant_table, op->handle);
509 if ( unlikely(!op->map->flags) )
510 {
511 gdprintk(XENLOG_INFO, "Zero flags for handle (%d).\n", op->handle);
512 op->status = GNTST_bad_handle;
513 return;
514 }
516 dom = op->map->domid;
517 op->flags = op->map->flags;
519 if ( unlikely((op->rd = rd = rcu_lock_domain_by_id(dom)) == NULL) )
520 {
521 /* This can happen when a grant is implicitly unmapped. */
522 gdprintk(XENLOG_INFO, "Could not find domain %d\n", dom);
523 domain_crash(ld); /* naughty... */
524 return;
525 }
527 rc = xsm_grant_unmapref(ld, rd);
528 if ( rc )
529 {
530 rcu_unlock_domain(rd);
531 op->status = GNTST_permission_denied;
532 return;
533 }
535 TRACE_1D(TRC_MEM_PAGE_GRANT_UNMAP, dom);
537 spin_lock(&rd->grant_table->lock);
539 act = &active_entry(rd->grant_table, op->map->ref);
540 sha = &shared_entry(rd->grant_table, op->map->ref);
541 old_pin = act->pin;
543 if ( op->frame == 0 )
544 {
545 op->frame = act->frame;
546 }
547 else
548 {
549 if ( unlikely(op->frame != act->frame) )
550 PIN_FAIL(unmap_out, GNTST_general_error,
551 "Bad frame number doesn't match gntref. (%lx != %lx)\n",
552 op->frame, act->frame);
553 if ( op->flags & GNTMAP_device_map )
554 {
555 ASSERT(act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask));
556 op->map->flags &= ~GNTMAP_device_map;
557 if ( op->flags & GNTMAP_readonly )
558 act->pin -= GNTPIN_devr_inc;
559 else
560 act->pin -= GNTPIN_devw_inc;
561 }
562 }
564 if ( (op->host_addr != 0) && (op->flags & GNTMAP_host_map) )
565 {
566 if ( (rc = replace_grant_host_mapping(op->host_addr,
567 op->frame, op->new_addr,
568 op->flags)) < 0 )
569 goto unmap_out;
571 ASSERT(act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask));
572 op->map->flags &= ~GNTMAP_host_map;
573 if ( op->flags & GNTMAP_readonly )
574 act->pin -= GNTPIN_hstr_inc;
575 else
576 act->pin -= GNTPIN_hstw_inc;
577 }
579 if ( need_iommu(ld) &&
580 (old_pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) &&
581 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
582 {
583 if ( iommu_unmap_page(ld, mfn_to_gmfn(ld, op->frame)) )
584 {
585 rc = GNTST_general_error;
586 goto unmap_out;
587 }
588 }
590 /* If just unmapped a writable mapping, mark as dirtied */
591 if ( !(op->flags & GNTMAP_readonly) )
592 gnttab_mark_dirty(rd, op->frame);
594 unmap_out:
595 op->status = rc;
596 spin_unlock(&rd->grant_table->lock);
597 rcu_unlock_domain(rd);
598 }
600 static void
601 __gnttab_unmap_common_complete(struct gnttab_unmap_common *op)
602 {
603 struct domain *ld, *rd;
604 struct active_grant_entry *act;
605 grant_entry_t *sha;
607 rd = op->rd;
609 if ( rd == NULL )
610 {
611 /*
612 * Suggests that __gntab_unmap_common failed in
613 * rcu_lock_domain_by_id() or earlier, and so we have nothing
614 * to complete
615 */
616 return;
617 }
619 ld = current->domain;
621 rcu_lock_domain(rd);
622 spin_lock(&rd->grant_table->lock);
624 act = &active_entry(rd->grant_table, op->map->ref);
625 sha = &shared_entry(rd->grant_table, op->map->ref);
627 if ( unlikely(op->frame != act->frame) )
628 {
629 /*
630 * Suggests that __gntab_unmap_common failed early and so
631 * nothing further to do
632 */
633 goto unmap_out;
634 }
636 if ( op->flags & GNTMAP_device_map )
637 {
638 if ( !is_iomem_page(act->frame) )
639 {
640 if ( op->flags & GNTMAP_readonly )
641 put_page(mfn_to_page(op->frame));
642 else
643 put_page_and_type(mfn_to_page(op->frame));
644 }
645 }
647 if ( (op->host_addr != 0) && (op->flags & GNTMAP_host_map) )
648 {
649 if ( op->status != 0 )
650 {
651 /*
652 * Suggests that __gntab_unmap_common failed in
653 * replace_grant_host_mapping() so nothing further to do
654 */
655 goto unmap_out;
656 }
658 if ( !is_iomem_page(op->frame) )
659 {
660 if ( gnttab_host_mapping_get_page_type(op, ld, rd) )
661 put_page_type(mfn_to_page(op->frame));
662 put_page(mfn_to_page(op->frame));
663 }
664 }
666 if ( (op->map->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0 )
667 {
668 op->map->flags = 0;
669 put_maptrack_handle(ld->grant_table, op->handle);
670 }
672 if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0) &&
673 !(op->flags & GNTMAP_readonly) )
674 gnttab_clear_flag(_GTF_writing, &sha->flags);
676 if ( act->pin == 0 )
677 gnttab_clear_flag(_GTF_reading, &sha->flags);
679 unmap_out:
680 spin_unlock(&rd->grant_table->lock);
681 rcu_unlock_domain(rd);
682 }
684 static void
685 __gnttab_unmap_grant_ref(
686 struct gnttab_unmap_grant_ref *op,
687 struct gnttab_unmap_common *common)
688 {
689 common->host_addr = op->host_addr;
690 common->dev_bus_addr = op->dev_bus_addr;
691 common->handle = op->handle;
693 /* Intialise these in case common contains old state */
694 common->new_addr = 0;
695 common->rd = NULL;
697 __gnttab_unmap_common(common);
698 op->status = common->status;
699 }
702 static long
703 gnttab_unmap_grant_ref(
704 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) uop, unsigned int count)
705 {
706 int i, c, partial_done, done = 0;
707 struct gnttab_unmap_grant_ref op;
708 struct gnttab_unmap_common common[GNTTAB_UNMAP_BATCH_SIZE];
710 while ( count != 0 )
711 {
712 c = min(count, (unsigned int)GNTTAB_UNMAP_BATCH_SIZE);
713 partial_done = 0;
715 for ( i = 0; i < c; i++ )
716 {
717 if ( unlikely(__copy_from_guest_offset(&op, uop, done+i, 1)) )
718 goto fault;
719 __gnttab_unmap_grant_ref(&op, &(common[i]));
720 ++partial_done;
721 if ( unlikely(__copy_to_guest_offset(uop, done+i, &op, 1)) )
722 goto fault;
723 }
725 flush_tlb_mask(&current->domain->domain_dirty_cpumask);
727 for ( i = 0; i < partial_done; i++ )
728 __gnttab_unmap_common_complete(&(common[i]));
730 count -= c;
731 done += c;
732 }
734 return 0;
736 fault:
737 flush_tlb_mask(&current->domain->domain_dirty_cpumask);
739 for ( i = 0; i < partial_done; i++ )
740 __gnttab_unmap_common_complete(&(common[i]));
741 return -EFAULT;
742 }
744 static void
745 __gnttab_unmap_and_replace(
746 struct gnttab_unmap_and_replace *op,
747 struct gnttab_unmap_common *common)
748 {
749 common->host_addr = op->host_addr;
750 common->new_addr = op->new_addr;
751 common->handle = op->handle;
753 /* Intialise these in case common contains old state */
754 common->dev_bus_addr = 0;
755 common->rd = NULL;
757 __gnttab_unmap_common(common);
758 op->status = common->status;
759 }
761 static long
762 gnttab_unmap_and_replace(
763 XEN_GUEST_HANDLE(gnttab_unmap_and_replace_t) uop, unsigned int count)
764 {
765 int i, c, partial_done, done = 0;
766 struct gnttab_unmap_and_replace op;
767 struct gnttab_unmap_common common[GNTTAB_UNMAP_BATCH_SIZE];
769 while ( count != 0 )
770 {
771 c = min(count, (unsigned int)GNTTAB_UNMAP_BATCH_SIZE);
772 partial_done = 0;
774 for ( i = 0; i < c; i++ )
775 {
776 if ( unlikely(__copy_from_guest_offset(&op, uop, done+i, 1)) )
777 goto fault;
778 __gnttab_unmap_and_replace(&op, &(common[i]));
779 ++partial_done;
780 if ( unlikely(__copy_to_guest_offset(uop, done+i, &op, 1)) )
781 goto fault;
782 }
784 flush_tlb_mask(&current->domain->domain_dirty_cpumask);
786 for ( i = 0; i < partial_done; i++ )
787 __gnttab_unmap_common_complete(&(common[i]));
789 count -= c;
790 done += c;
791 }
793 return 0;
795 fault:
796 flush_tlb_mask(&current->domain->domain_dirty_cpumask);
798 for ( i = 0; i < partial_done; i++ )
799 __gnttab_unmap_common_complete(&(common[i]));
800 return -EFAULT;
801 }
803 int
804 gnttab_grow_table(struct domain *d, unsigned int req_nr_frames)
805 {
806 /* d's grant table lock must be held by the caller */
808 struct grant_table *gt = d->grant_table;
809 unsigned int i;
811 ASSERT(req_nr_frames <= max_nr_grant_frames);
813 gdprintk(XENLOG_INFO,
814 "Expanding dom (%d) grant table from (%d) to (%d) frames.\n",
815 d->domain_id, nr_grant_frames(gt), req_nr_frames);
817 /* Active */
818 for ( i = nr_active_grant_frames(gt);
819 i < num_act_frames_from_sha_frames(req_nr_frames); i++ )
820 {
821 if ( (gt->active[i] = alloc_xenheap_page()) == NULL )
822 goto active_alloc_failed;
823 clear_page(gt->active[i]);
824 }
826 /* Shared */
827 for ( i = nr_grant_frames(gt); i < req_nr_frames; i++ )
828 {
829 if ( (gt->shared[i] = alloc_xenheap_page()) == NULL )
830 goto shared_alloc_failed;
831 clear_page(gt->shared[i]);
832 }
834 /* Share the new shared frames with the recipient domain */
835 for ( i = nr_grant_frames(gt); i < req_nr_frames; i++ )
836 gnttab_create_shared_page(d, gt, i);
838 gt->nr_grant_frames = req_nr_frames;
840 return 1;
842 shared_alloc_failed:
843 for ( i = nr_grant_frames(gt); i < req_nr_frames; i++ )
844 {
845 free_xenheap_page(gt->shared[i]);
846 gt->shared[i] = NULL;
847 }
848 active_alloc_failed:
849 for ( i = nr_active_grant_frames(gt);
850 i < num_act_frames_from_sha_frames(req_nr_frames); i++ )
851 {
852 free_xenheap_page(gt->active[i]);
853 gt->active[i] = NULL;
854 }
855 gdprintk(XENLOG_INFO, "Allocation failure when expanding grant table.\n");
856 return 0;
857 }
859 static long
860 gnttab_setup_table(
861 XEN_GUEST_HANDLE(gnttab_setup_table_t) uop, unsigned int count)
862 {
863 struct gnttab_setup_table op;
864 struct domain *d;
865 int i;
866 unsigned long gmfn;
867 domid_t dom;
869 if ( count != 1 )
870 return -EINVAL;
872 if ( unlikely(copy_from_guest(&op, uop, 1) != 0) )
873 {
874 gdprintk(XENLOG_INFO, "Fault while reading gnttab_setup_table_t.\n");
875 return -EFAULT;
876 }
878 if ( unlikely(op.nr_frames > max_nr_grant_frames) )
879 {
880 gdprintk(XENLOG_INFO, "Xen only supports up to %d grant-table frames"
881 " per domain.\n",
882 max_nr_grant_frames);
883 op.status = GNTST_general_error;
884 goto out1;
885 }
887 dom = op.dom;
888 if ( dom == DOMID_SELF )
889 {
890 d = rcu_lock_current_domain();
891 }
892 else
893 {
894 if ( unlikely((d = rcu_lock_domain_by_id(dom)) == NULL) )
895 {
896 gdprintk(XENLOG_INFO, "Bad domid %d.\n", dom);
897 op.status = GNTST_bad_domain;
898 goto out1;
899 }
901 if ( unlikely(!IS_PRIV_FOR(current->domain, d)) )
902 {
903 op.status = GNTST_permission_denied;
904 goto out2;
905 }
906 }
908 if ( xsm_grant_setup(current->domain, d) )
909 {
910 op.status = GNTST_permission_denied;
911 goto out2;
912 }
914 spin_lock(&d->grant_table->lock);
916 if ( (op.nr_frames > nr_grant_frames(d->grant_table)) &&
917 !gnttab_grow_table(d, op.nr_frames) )
918 {
919 gdprintk(XENLOG_INFO,
920 "Expand grant table to %d failed. Current: %d Max: %d.\n",
921 op.nr_frames,
922 nr_grant_frames(d->grant_table),
923 max_nr_grant_frames);
924 op.status = GNTST_general_error;
925 goto out3;
926 }
928 op.status = GNTST_okay;
929 for ( i = 0; i < op.nr_frames; i++ )
930 {
931 gmfn = gnttab_shared_gmfn(d, d->grant_table, i);
932 (void)copy_to_guest_offset(op.frame_list, i, &gmfn, 1);
933 }
935 out3:
936 spin_unlock(&d->grant_table->lock);
937 out2:
938 rcu_unlock_domain(d);
939 out1:
940 if ( unlikely(copy_to_guest(uop, &op, 1)) )
941 return -EFAULT;
943 return 0;
944 }
946 static long
947 gnttab_query_size(
948 XEN_GUEST_HANDLE(gnttab_query_size_t) uop, unsigned int count)
949 {
950 struct gnttab_query_size op;
951 struct domain *d;
952 domid_t dom;
953 int rc;
955 if ( count != 1 )
956 return -EINVAL;
958 if ( unlikely(copy_from_guest(&op, uop, 1) != 0) )
959 {
960 gdprintk(XENLOG_INFO, "Fault while reading gnttab_query_size_t.\n");
961 return -EFAULT;
962 }
964 dom = op.dom;
965 if ( dom == DOMID_SELF )
966 {
967 d = rcu_lock_current_domain();
968 }
969 else
970 {
971 if ( unlikely((d = rcu_lock_domain_by_id(dom)) == NULL) )
972 {
973 gdprintk(XENLOG_INFO, "Bad domid %d.\n", dom);
974 op.status = GNTST_bad_domain;
975 goto query_out;
976 }
978 if ( unlikely(!IS_PRIV_FOR(current->domain, d)) )
979 {
980 op.status = GNTST_permission_denied;
981 goto query_out_unlock;
982 }
983 }
985 rc = xsm_grant_query_size(current->domain, d);
986 if ( rc )
987 {
988 op.status = GNTST_permission_denied;
989 goto query_out_unlock;
990 }
992 spin_lock(&d->grant_table->lock);
994 op.nr_frames = nr_grant_frames(d->grant_table);
995 op.max_nr_frames = max_nr_grant_frames;
996 op.status = GNTST_okay;
998 spin_unlock(&d->grant_table->lock);
1001 query_out_unlock:
1002 rcu_unlock_domain(d);
1004 query_out:
1005 if ( unlikely(copy_to_guest(uop, &op, 1)) )
1006 return -EFAULT;
1008 return 0;
1011 /*
1012 * Check that the given grant reference (rd,ref) allows 'ld' to transfer
1013 * ownership of a page frame. If so, lock down the grant entry.
1014 */
1015 static int
1016 gnttab_prepare_for_transfer(
1017 struct domain *rd, struct domain *ld, grant_ref_t ref)
1019 struct grant_table *rgt;
1020 struct grant_entry *sha;
1021 union grant_combo scombo, prev_scombo, new_scombo;
1022 int retries = 0;
1024 if ( unlikely((rgt = rd->grant_table) == NULL) )
1026 gdprintk(XENLOG_INFO, "Dom %d has no grant table.\n", rd->domain_id);
1027 return 0;
1030 spin_lock(&rgt->lock);
1032 if ( unlikely(ref >= nr_grant_entries(rd->grant_table)) )
1034 gdprintk(XENLOG_INFO,
1035 "Bad grant reference (%d) for transfer to domain(%d).\n",
1036 ref, rd->domain_id);
1037 goto fail;
1040 sha = &shared_entry(rgt, ref);
1042 scombo.word = *(u32 *)&sha->flags;
1044 for ( ; ; )
1046 if ( unlikely(scombo.shorts.flags != GTF_accept_transfer) ||
1047 unlikely(scombo.shorts.domid != ld->domain_id) )
1049 gdprintk(XENLOG_INFO, "Bad flags (%x) or dom (%d). "
1050 "(NB. expected dom %d)\n",
1051 scombo.shorts.flags, scombo.shorts.domid,
1052 ld->domain_id);
1053 goto fail;
1056 new_scombo = scombo;
1057 new_scombo.shorts.flags |= GTF_transfer_committed;
1059 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
1060 scombo.word, new_scombo.word);
1061 if ( likely(prev_scombo.word == scombo.word) )
1062 break;
1064 if ( retries++ == 4 )
1066 gdprintk(XENLOG_WARNING, "Shared grant entry is unstable.\n");
1067 goto fail;
1070 scombo = prev_scombo;
1073 spin_unlock(&rgt->lock);
1074 return 1;
1076 fail:
1077 spin_unlock(&rgt->lock);
1078 return 0;
1081 static long
1082 gnttab_transfer(
1083 XEN_GUEST_HANDLE(gnttab_transfer_t) uop, unsigned int count)
1085 struct domain *d = current->domain;
1086 struct domain *e;
1087 struct page_info *page;
1088 int i;
1089 grant_entry_t *sha;
1090 struct gnttab_transfer gop;
1091 unsigned long mfn;
1092 unsigned int max_bitsize;
1094 for ( i = 0; i < count; i++ )
1096 /* Read from caller address space. */
1097 if ( unlikely(__copy_from_guest_offset(&gop, uop, i, 1)) )
1099 gdprintk(XENLOG_INFO, "gnttab_transfer: error reading req %d/%d\n",
1100 i, count);
1101 return -EFAULT;
1104 mfn = gmfn_to_mfn(d, gop.mfn);
1106 /* Check the passed page frame for basic validity. */
1107 if ( unlikely(!mfn_valid(mfn)) )
1109 gdprintk(XENLOG_INFO, "gnttab_transfer: out-of-range %lx\n",
1110 (unsigned long)gop.mfn);
1111 gop.status = GNTST_bad_page;
1112 goto copyback;
1115 page = mfn_to_page(mfn);
1116 if ( unlikely(is_xen_heap_page(page)) )
1118 gdprintk(XENLOG_INFO, "gnttab_transfer: xen frame %lx\n",
1119 (unsigned long)gop.mfn);
1120 gop.status = GNTST_bad_page;
1121 goto copyback;
1124 if ( steal_page(d, page, 0) < 0 )
1126 gop.status = GNTST_bad_page;
1127 goto copyback;
1130 #ifndef __ia64__ /* IA64 implicitly replaces the old page in steal_page(). */
1131 guest_physmap_remove_page(d, gop.mfn, mfn, 0);
1132 #endif
1133 flush_tlb_mask(&d->domain_dirty_cpumask);
1135 /* Find the target domain. */
1136 if ( unlikely((e = rcu_lock_domain_by_id(gop.domid)) == NULL) )
1138 gdprintk(XENLOG_INFO, "gnttab_transfer: can't find domain %d\n",
1139 gop.domid);
1140 page->count_info &= ~(PGC_count_mask|PGC_allocated);
1141 free_domheap_page(page);
1142 gop.status = GNTST_bad_domain;
1143 goto copyback;
1146 if ( xsm_grant_transfer(d, e) )
1148 gop.status = GNTST_permission_denied;
1149 unlock_and_copyback:
1150 rcu_unlock_domain(e);
1151 page->count_info &= ~(PGC_count_mask|PGC_allocated);
1152 free_domheap_page(page);
1153 goto copyback;
1156 max_bitsize = domain_clamp_alloc_bitsize(
1157 e, BITS_PER_LONG+PAGE_SHIFT-1);
1158 if ( (1UL << (max_bitsize - PAGE_SHIFT)) <= mfn )
1160 struct page_info *new_page;
1161 void *sp, *dp;
1163 new_page = alloc_domheap_page(NULL, MEMF_bits(max_bitsize));
1164 if ( new_page == NULL )
1166 gop.status = GNTST_address_too_big;
1167 goto unlock_and_copyback;
1170 sp = map_domain_page(mfn);
1171 dp = map_domain_page(page_to_mfn(new_page));
1172 memcpy(dp, sp, PAGE_SIZE);
1173 unmap_domain_page(dp);
1174 unmap_domain_page(sp);
1176 page->count_info &= ~(PGC_count_mask|PGC_allocated);
1177 free_domheap_page(page);
1178 page = new_page;
1181 spin_lock(&e->page_alloc_lock);
1183 /*
1184 * Check that 'e' will accept the page and has reservation
1185 * headroom. Also, a domain mustn't have PGC_allocated
1186 * pages when it is dying.
1187 */
1188 if ( unlikely(e->is_dying) ||
1189 unlikely(e->tot_pages >= e->max_pages) ||
1190 unlikely(!gnttab_prepare_for_transfer(e, d, gop.ref)) )
1192 if ( !e->is_dying )
1193 gdprintk(XENLOG_INFO, "gnttab_transfer: "
1194 "Transferee has no reservation "
1195 "headroom (%d,%d) or provided a bad grant ref (%08x) "
1196 "or is dying (%d)\n",
1197 e->tot_pages, e->max_pages, gop.ref, e->is_dying);
1198 spin_unlock(&e->page_alloc_lock);
1199 rcu_unlock_domain(e);
1200 page->count_info &= ~(PGC_count_mask|PGC_allocated);
1201 free_domheap_page(page);
1202 gop.status = GNTST_general_error;
1203 goto copyback;
1206 /* Okay, add the page to 'e'. */
1207 if ( unlikely(e->tot_pages++ == 0) )
1208 get_knownalive_domain(e);
1209 page_list_add_tail(page, &e->page_list);
1210 page_set_owner(page, e);
1212 spin_unlock(&e->page_alloc_lock);
1214 TRACE_1D(TRC_MEM_PAGE_GRANT_TRANSFER, e->domain_id);
1216 /* Tell the guest about its new page frame. */
1217 spin_lock(&e->grant_table->lock);
1219 sha = &shared_entry(e->grant_table, gop.ref);
1220 guest_physmap_add_page(e, sha->frame, mfn, 0);
1221 sha->frame = mfn;
1222 wmb();
1223 sha->flags |= GTF_transfer_completed;
1225 spin_unlock(&e->grant_table->lock);
1227 rcu_unlock_domain(e);
1229 gop.status = GNTST_okay;
1231 copyback:
1232 if ( unlikely(__copy_to_guest_offset(uop, i, &gop, 1)) )
1234 gdprintk(XENLOG_INFO, "gnttab_transfer: error writing resp "
1235 "%d/%d\n", i, count);
1236 return -EFAULT;
1240 return 0;
1243 /* Undo __acquire_grant_for_copy. Again, this has no effect on page
1244 type and reference counts. */
1245 static void
1246 __release_grant_for_copy(
1247 struct domain *rd, unsigned long gref, int readonly)
1249 grant_entry_t *sha;
1250 struct active_grant_entry *act;
1251 unsigned long r_frame;
1253 spin_lock(&rd->grant_table->lock);
1255 act = &active_entry(rd->grant_table, gref);
1256 sha = &shared_entry(rd->grant_table, gref);
1257 r_frame = act->frame;
1259 if ( readonly )
1261 act->pin -= GNTPIN_hstr_inc;
1263 else
1265 gnttab_mark_dirty(rd, r_frame);
1267 act->pin -= GNTPIN_hstw_inc;
1268 if ( !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) )
1269 gnttab_clear_flag(_GTF_writing, &sha->flags);
1272 if ( !act->pin )
1273 gnttab_clear_flag(_GTF_reading, &sha->flags);
1275 spin_unlock(&rd->grant_table->lock);
1278 /* Grab a frame number from a grant entry and update the flags and pin
1279 count as appropriate. Note that this does *not* update the page
1280 type or reference counts, and does not check that the mfn is
1281 actually valid. */
1282 static int
1283 __acquire_grant_for_copy(
1284 struct domain *rd, unsigned long gref, int readonly,
1285 unsigned long *frame)
1287 grant_entry_t *sha;
1288 struct active_grant_entry *act;
1289 s16 rc = GNTST_okay;
1290 int retries = 0;
1291 union grant_combo scombo, prev_scombo, new_scombo;
1293 spin_lock(&rd->grant_table->lock);
1295 if ( unlikely(gref >= nr_grant_entries(rd->grant_table)) )
1296 PIN_FAIL(unlock_out, GNTST_bad_gntref,
1297 "Bad grant reference %ld\n", gref);
1299 act = &active_entry(rd->grant_table, gref);
1300 sha = &shared_entry(rd->grant_table, gref);
1302 /* If already pinned, check the active domid and avoid refcnt overflow. */
1303 if ( act->pin &&
1304 ((act->domid != current->domain->domain_id) ||
1305 (act->pin & 0x80808080U) != 0) )
1306 PIN_FAIL(unlock_out, GNTST_general_error,
1307 "Bad domain (%d != %d), or risk of counter overflow %08x\n",
1308 act->domid, current->domain->domain_id, act->pin);
1310 if ( !act->pin ||
1311 (!readonly && !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask))) )
1313 scombo.word = *(u32 *)&sha->flags;
1315 for ( ; ; )
1317 /* If not already pinned, check the grant domid and type. */
1318 if ( !act->pin &&
1319 (((scombo.shorts.flags & GTF_type_mask) !=
1320 GTF_permit_access) ||
1321 (scombo.shorts.domid != current->domain->domain_id)) )
1322 PIN_FAIL(unlock_out, GNTST_general_error,
1323 "Bad flags (%x) or dom (%d). (expected dom %d)\n",
1324 scombo.shorts.flags, scombo.shorts.domid,
1325 current->domain->domain_id);
1327 new_scombo = scombo;
1328 new_scombo.shorts.flags |= GTF_reading;
1330 if ( !readonly )
1332 new_scombo.shorts.flags |= GTF_writing;
1333 if ( unlikely(scombo.shorts.flags & GTF_readonly) )
1334 PIN_FAIL(unlock_out, GNTST_general_error,
1335 "Attempt to write-pin a r/o grant entry.\n");
1338 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
1339 scombo.word, new_scombo.word);
1340 if ( likely(prev_scombo.word == scombo.word) )
1341 break;
1343 if ( retries++ == 4 )
1344 PIN_FAIL(unlock_out, GNTST_general_error,
1345 "Shared grant entry is unstable.\n");
1347 scombo = prev_scombo;
1350 if ( !act->pin )
1352 act->domid = scombo.shorts.domid;
1353 act->gfn = sha->frame;
1354 act->frame = gmfn_to_mfn(rd, sha->frame);
1358 act->pin += readonly ? GNTPIN_hstr_inc : GNTPIN_hstw_inc;
1360 *frame = act->frame;
1362 unlock_out:
1363 spin_unlock(&rd->grant_table->lock);
1364 return rc;
1367 static void
1368 __gnttab_copy(
1369 struct gnttab_copy *op)
1371 struct domain *sd = NULL, *dd = NULL;
1372 unsigned long s_frame, d_frame;
1373 char *sp, *dp;
1374 s16 rc = GNTST_okay;
1375 int have_d_grant = 0, have_s_grant = 0, have_s_ref = 0;
1376 int src_is_gref, dest_is_gref;
1378 if ( ((op->source.offset + op->len) > PAGE_SIZE) ||
1379 ((op->dest.offset + op->len) > PAGE_SIZE) )
1380 PIN_FAIL(error_out, GNTST_bad_copy_arg, "copy beyond page area.\n");
1382 src_is_gref = op->flags & GNTCOPY_source_gref;
1383 dest_is_gref = op->flags & GNTCOPY_dest_gref;
1385 if ( (op->source.domid != DOMID_SELF && !src_is_gref ) ||
1386 (op->dest.domid != DOMID_SELF && !dest_is_gref) )
1387 PIN_FAIL(error_out, GNTST_permission_denied,
1388 "only allow copy-by-mfn for DOMID_SELF.\n");
1390 if ( op->source.domid == DOMID_SELF )
1391 sd = rcu_lock_current_domain();
1392 else if ( (sd = rcu_lock_domain_by_id(op->source.domid)) == NULL )
1393 PIN_FAIL(error_out, GNTST_bad_domain,
1394 "couldn't find %d\n", op->source.domid);
1396 if ( op->dest.domid == DOMID_SELF )
1397 dd = rcu_lock_current_domain();
1398 else if ( (dd = rcu_lock_domain_by_id(op->dest.domid)) == NULL )
1399 PIN_FAIL(error_out, GNTST_bad_domain,
1400 "couldn't find %d\n", op->dest.domid);
1402 rc = xsm_grant_copy(sd, dd);
1403 if ( rc )
1405 rc = GNTST_permission_denied;
1406 goto error_out;
1409 if ( src_is_gref )
1411 rc = __acquire_grant_for_copy(sd, op->source.u.ref, 1, &s_frame);
1412 if ( rc != GNTST_okay )
1413 goto error_out;
1414 have_s_grant = 1;
1416 else
1418 s_frame = gmfn_to_mfn(sd, op->source.u.gmfn);
1420 if ( unlikely(!mfn_valid(s_frame)) )
1421 PIN_FAIL(error_out, GNTST_general_error,
1422 "source frame %lx invalid.\n", s_frame);
1423 if ( !get_page(mfn_to_page(s_frame), sd) )
1425 if ( !sd->is_dying )
1426 gdprintk(XENLOG_WARNING, "Could not get src frame %lx\n", s_frame);
1427 rc = GNTST_general_error;
1428 goto error_out;
1430 have_s_ref = 1;
1432 if ( dest_is_gref )
1434 rc = __acquire_grant_for_copy(dd, op->dest.u.ref, 0, &d_frame);
1435 if ( rc != GNTST_okay )
1436 goto error_out;
1437 have_d_grant = 1;
1439 else
1441 d_frame = gmfn_to_mfn(dd, op->dest.u.gmfn);
1443 if ( unlikely(!mfn_valid(d_frame)) )
1444 PIN_FAIL(error_out, GNTST_general_error,
1445 "destination frame %lx invalid.\n", d_frame);
1446 if ( !get_page_and_type(mfn_to_page(d_frame), dd, PGT_writable_page) )
1448 if ( !dd->is_dying )
1449 gdprintk(XENLOG_WARNING, "Could not get dst frame %lx\n", d_frame);
1450 rc = GNTST_general_error;
1451 goto error_out;
1454 sp = map_domain_page(s_frame);
1455 dp = map_domain_page(d_frame);
1457 memcpy(dp + op->dest.offset, sp + op->source.offset, op->len);
1459 unmap_domain_page(dp);
1460 unmap_domain_page(sp);
1462 gnttab_mark_dirty(dd, d_frame);
1464 put_page_and_type(mfn_to_page(d_frame));
1465 error_out:
1466 if ( have_s_ref )
1467 put_page(mfn_to_page(s_frame));
1468 if ( have_s_grant )
1469 __release_grant_for_copy(sd, op->source.u.ref, 1);
1470 if ( have_d_grant )
1471 __release_grant_for_copy(dd, op->dest.u.ref, 0);
1472 if ( sd )
1473 rcu_unlock_domain(sd);
1474 if ( dd )
1475 rcu_unlock_domain(dd);
1476 op->status = rc;
1479 static long
1480 gnttab_copy(
1481 XEN_GUEST_HANDLE(gnttab_copy_t) uop, unsigned int count)
1483 int i;
1484 struct gnttab_copy op;
1486 for ( i = 0; i < count; i++ )
1488 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
1489 return -EFAULT;
1490 __gnttab_copy(&op);
1491 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
1492 return -EFAULT;
1494 return 0;
1497 long
1498 do_grant_table_op(
1499 unsigned int cmd, XEN_GUEST_HANDLE(void) uop, unsigned int count)
1501 long rc;
1502 struct domain *d = current->domain;
1504 if ( count > 512 )
1505 return -EINVAL;
1507 domain_lock(d);
1509 rc = -EFAULT;
1510 switch ( cmd )
1512 case GNTTABOP_map_grant_ref:
1514 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) map =
1515 guest_handle_cast(uop, gnttab_map_grant_ref_t);
1516 if ( unlikely(!guest_handle_okay(map, count)) )
1517 goto out;
1518 maptrack_lock(current->domain->grant_table);
1519 rc = gnttab_map_grant_ref(map, count);
1520 maptrack_unlock(current->domain->grant_table);
1521 break;
1523 case GNTTABOP_unmap_grant_ref:
1525 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) unmap =
1526 guest_handle_cast(uop, gnttab_unmap_grant_ref_t);
1527 if ( unlikely(!guest_handle_okay(unmap, count)) )
1528 goto out;
1529 maptrack_lock(current->domain->grant_table);
1530 rc = gnttab_unmap_grant_ref(unmap, count);
1531 maptrack_unlock(current->domain->grant_table);
1532 break;
1534 case GNTTABOP_unmap_and_replace:
1536 XEN_GUEST_HANDLE(gnttab_unmap_and_replace_t) unmap =
1537 guest_handle_cast(uop, gnttab_unmap_and_replace_t);
1538 if ( unlikely(!guest_handle_okay(unmap, count)) )
1539 goto out;
1540 rc = -ENOSYS;
1541 if ( unlikely(!replace_grant_supported()) )
1542 goto out;
1543 maptrack_lock(current->domain->grant_table);
1544 rc = gnttab_unmap_and_replace(unmap, count);
1545 maptrack_unlock(current->domain->grant_table);
1546 break;
1548 case GNTTABOP_setup_table:
1550 rc = gnttab_setup_table(
1551 guest_handle_cast(uop, gnttab_setup_table_t), count);
1552 break;
1554 case GNTTABOP_transfer:
1556 XEN_GUEST_HANDLE(gnttab_transfer_t) transfer =
1557 guest_handle_cast(uop, gnttab_transfer_t);
1558 if ( unlikely(!guest_handle_okay(transfer, count)) )
1559 goto out;
1560 rc = gnttab_transfer(transfer, count);
1561 break;
1563 case GNTTABOP_copy:
1565 XEN_GUEST_HANDLE(gnttab_copy_t) copy =
1566 guest_handle_cast(uop, gnttab_copy_t);
1567 if ( unlikely(!guest_handle_okay(copy, count)) )
1568 goto out;
1569 rc = gnttab_copy(copy, count);
1570 break;
1572 case GNTTABOP_query_size:
1574 rc = gnttab_query_size(
1575 guest_handle_cast(uop, gnttab_query_size_t), count);
1576 break;
1578 default:
1579 rc = -ENOSYS;
1580 break;
1583 out:
1584 domain_unlock(d);
1586 return rc;
1589 #ifdef CONFIG_COMPAT
1590 #include "compat/grant_table.c"
1591 #endif
1593 static unsigned int max_nr_active_grant_frames(void)
1595 return (((max_nr_grant_frames * (PAGE_SIZE / sizeof(grant_entry_t))) +
1596 ((PAGE_SIZE / sizeof(struct active_grant_entry))-1))
1597 / (PAGE_SIZE / sizeof(struct active_grant_entry)));
1600 int
1601 grant_table_create(
1602 struct domain *d)
1604 struct grant_table *t;
1605 int i;
1607 /* If this sizeof assertion fails, fix the function: shared_index */
1608 ASSERT(sizeof(grant_entry_t) == 8);
1610 if ( (t = xmalloc(struct grant_table)) == NULL )
1611 goto no_mem_0;
1613 /* Simple stuff. */
1614 memset(t, 0, sizeof(*t));
1615 spin_lock_init(&t->lock);
1616 spin_lock_init(&t->maptrack_lock);
1617 t->nr_grant_frames = INITIAL_NR_GRANT_FRAMES;
1619 /* Active grant table. */
1620 if ( (t->active = xmalloc_array(struct active_grant_entry *,
1621 max_nr_active_grant_frames())) == NULL )
1622 goto no_mem_1;
1623 memset(t->active, 0, max_nr_active_grant_frames() * sizeof(t->active[0]));
1624 for ( i = 0;
1625 i < num_act_frames_from_sha_frames(INITIAL_NR_GRANT_FRAMES); i++ )
1627 if ( (t->active[i] = alloc_xenheap_page()) == NULL )
1628 goto no_mem_2;
1629 clear_page(t->active[i]);
1632 /* Tracking of mapped foreign frames table */
1633 if ( (t->maptrack = xmalloc_array(struct grant_mapping *,
1634 max_nr_maptrack_frames())) == NULL )
1635 goto no_mem_2;
1636 memset(t->maptrack, 0, max_nr_maptrack_frames() * sizeof(t->maptrack[0]));
1637 if ( (t->maptrack[0] = alloc_xenheap_page()) == NULL )
1638 goto no_mem_3;
1639 clear_page(t->maptrack[0]);
1640 t->maptrack_limit = PAGE_SIZE / sizeof(struct grant_mapping);
1641 for ( i = 0; i < t->maptrack_limit; i++ )
1642 t->maptrack[0][i].ref = i+1;
1644 /* Shared grant table. */
1645 if ( (t->shared = xmalloc_array(struct grant_entry *,
1646 max_nr_grant_frames)) == NULL )
1647 goto no_mem_3;
1648 memset(t->shared, 0, max_nr_grant_frames * sizeof(t->shared[0]));
1649 for ( i = 0; i < INITIAL_NR_GRANT_FRAMES; i++ )
1651 if ( (t->shared[i] = alloc_xenheap_page()) == NULL )
1652 goto no_mem_4;
1653 clear_page(t->shared[i]);
1656 for ( i = 0; i < INITIAL_NR_GRANT_FRAMES; i++ )
1657 gnttab_create_shared_page(d, t, i);
1659 /* Okay, install the structure. */
1660 d->grant_table = t;
1661 return 0;
1663 no_mem_4:
1664 for ( i = 0; i < INITIAL_NR_GRANT_FRAMES; i++ )
1665 free_xenheap_page(t->shared[i]);
1666 xfree(t->shared);
1667 no_mem_3:
1668 free_xenheap_page(t->maptrack[0]);
1669 xfree(t->maptrack);
1670 no_mem_2:
1671 for ( i = 0;
1672 i < num_act_frames_from_sha_frames(INITIAL_NR_GRANT_FRAMES); i++ )
1673 free_xenheap_page(t->active[i]);
1674 xfree(t->active);
1675 no_mem_1:
1676 xfree(t);
1677 no_mem_0:
1678 return -ENOMEM;
1681 void
1682 gnttab_release_mappings(
1683 struct domain *d)
1685 struct grant_table *gt = d->grant_table;
1686 struct grant_mapping *map;
1687 grant_ref_t ref;
1688 grant_handle_t handle;
1689 struct domain *rd;
1690 struct active_grant_entry *act;
1691 struct grant_entry *sha;
1693 BUG_ON(!d->is_dying);
1695 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
1697 /* Domain is dying, so don't need maptrack lock */
1698 map = &maptrack_entry(gt, handle);
1699 if ( !(map->flags & (GNTMAP_device_map|GNTMAP_host_map)) )
1700 continue;
1702 ref = map->ref;
1704 gdprintk(XENLOG_INFO, "Grant release (%hu) ref:(%hu) "
1705 "flags:(%x) dom:(%hu)\n",
1706 handle, ref, map->flags, map->domid);
1708 rd = rcu_lock_domain_by_id(map->domid);
1709 if ( rd == NULL )
1711 /* Nothing to clear up... */
1712 map->flags = 0;
1713 continue;
1716 spin_lock(&rd->grant_table->lock);
1718 act = &active_entry(rd->grant_table, ref);
1719 sha = &shared_entry(rd->grant_table, ref);
1721 if ( map->flags & GNTMAP_readonly )
1723 if ( map->flags & GNTMAP_device_map )
1725 BUG_ON(!(act->pin & GNTPIN_devr_mask));
1726 act->pin -= GNTPIN_devr_inc;
1727 if ( !is_iomem_page(act->frame) )
1728 put_page(mfn_to_page(act->frame));
1731 if ( map->flags & GNTMAP_host_map )
1733 BUG_ON(!(act->pin & GNTPIN_hstr_mask));
1734 act->pin -= GNTPIN_hstr_inc;
1735 if ( gnttab_release_host_mappings &&
1736 !is_iomem_page(act->frame) )
1737 put_page(mfn_to_page(act->frame));
1740 else
1742 if ( map->flags & GNTMAP_device_map )
1744 BUG_ON(!(act->pin & GNTPIN_devw_mask));
1745 act->pin -= GNTPIN_devw_inc;
1746 if ( !is_iomem_page(act->frame) )
1747 put_page_and_type(mfn_to_page(act->frame));
1750 if ( map->flags & GNTMAP_host_map )
1752 BUG_ON(!(act->pin & GNTPIN_hstw_mask));
1753 act->pin -= GNTPIN_hstw_inc;
1754 if ( gnttab_release_host_mappings &&
1755 !is_iomem_page(act->frame) )
1757 if ( gnttab_host_mapping_get_page_type(map, d, rd) )
1758 put_page_type(mfn_to_page(act->frame));
1759 put_page(mfn_to_page(act->frame));
1763 if ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0 )
1764 gnttab_clear_flag(_GTF_writing, &sha->flags);
1767 if ( act->pin == 0 )
1768 gnttab_clear_flag(_GTF_reading, &sha->flags);
1770 spin_unlock(&rd->grant_table->lock);
1772 rcu_unlock_domain(rd);
1774 map->flags = 0;
1779 void
1780 grant_table_destroy(
1781 struct domain *d)
1783 struct grant_table *t = d->grant_table;
1784 int i;
1786 if ( t == NULL )
1787 return;
1789 for ( i = 0; i < nr_grant_frames(t); i++ )
1790 free_xenheap_page(t->shared[i]);
1791 xfree(t->shared);
1793 for ( i = 0; i < nr_maptrack_frames(t); i++ )
1794 free_xenheap_page(t->maptrack[i]);
1795 xfree(t->maptrack);
1797 for ( i = 0; i < nr_active_grant_frames(t); i++ )
1798 free_xenheap_page(t->active[i]);
1799 xfree(t->active);
1801 xfree(t);
1802 d->grant_table = NULL;
1805 /*
1806 * Local variables:
1807 * mode: C
1808 * c-set-style: "BSD"
1809 * c-basic-offset: 4
1810 * tab-width: 4
1811 * indent-tabs-mode: nil
1812 * End:
1813 */