ia64/xen-unstable

annotate xen/common/memory.c @ 18975:2090917489c5

PoD memory 7/9: Xen interface

Implement Xen interface to PoD functionality.
* Increase the number of MEMOP bits from 4 to 6 (increasing the number
of available memory operations from 16 to 64).
* Introduce XENMEMF_populate_on_demand, which will cause
populate_physmap() to fill a range with PoD entries rather than
backing it with ram
* Introduce XENMEM_[sg]et_pod_target operation to the memory
hypercall, to get and set PoD cache size. set_pod_target() should be
called during domain creation, as well as after modifying the memory
target of any domain which may have outstanding PoD entries.

Signed-off-by: George Dunlap <george.dunlap@eu.citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Mon Jan 05 10:45:48 2009 +0000 (2009-01-05)
parents bd33ff263e2c
children 2737293c761e
rev   line source
kaf24@6486 1 /******************************************************************************
kaf24@6486 2 * memory.c
kaf24@6486 3 *
kaf24@6486 4 * Code to handle memory-related requests.
kaf24@6486 5 *
kaf24@6486 6 * Copyright (c) 2003-2004, B Dragovic
kaf24@6486 7 * Copyright (c) 2003-2005, K A Fraser
kaf24@6486 8 */
kaf24@6486 9
kaf24@6486 10 #include <xen/config.h>
kaf24@6486 11 #include <xen/types.h>
kaf24@6486 12 #include <xen/lib.h>
kaf24@6486 13 #include <xen/mm.h>
kaf24@6486 14 #include <xen/perfc.h>
kaf24@6486 15 #include <xen/sched.h>
kaf24@6486 16 #include <xen/event.h>
Tim@15635 17 #include <xen/paging.h>
kaf24@8468 18 #include <xen/iocap.h>
kaf24@9068 19 #include <xen/guest_access.h>
ack@13295 20 #include <xen/hypercall.h>
kaf24@11219 21 #include <xen/errno.h>
kaf24@6486 22 #include <asm/current.h>
kaf24@6486 23 #include <asm/hardirq.h>
keir@17385 24 #include <xen/numa.h>
kaf24@6486 25 #include <public/memory.h>
kfraser@15815 26 #include <xsm/xsm.h>
kaf24@6486 27
kfraser@12374 28 struct memop_args {
kfraser@12374 29 /* INPUT */
kfraser@12374 30 struct domain *domain; /* Domain to be affected. */
kfraser@12374 31 XEN_GUEST_HANDLE(xen_pfn_t) extent_list; /* List of extent base addrs. */
kfraser@12374 32 unsigned int nr_extents; /* Number of extents to allocate or free. */
kfraser@12374 33 unsigned int extent_order; /* Size of each extent. */
kfraser@12374 34 unsigned int memflags; /* Allocation flags. */
kfraser@12374 35
kfraser@12374 36 /* INPUT/OUTPUT */
kfraser@12374 37 unsigned int nr_done; /* Number of extents processed so far. */
kfraser@12374 38 int preempted; /* Was the hypercall preempted? */
kfraser@12374 39 };
kfraser@12374 40
kfraser@12374 41 static void increase_reservation(struct memop_args *a)
kaf24@6486 42 {
kaf24@8726 43 struct page_info *page;
kaf24@10314 44 unsigned long i;
kaf24@10314 45 xen_pfn_t mfn;
kfraser@12374 46 struct domain *d = a->domain;
kaf24@6486 47
kfraser@12374 48 if ( !guest_handle_is_null(a->extent_list) &&
keir@17860 49 !guest_handle_subrange_okay(a->extent_list, a->nr_done,
keir@17860 50 a->nr_extents-1) )
kfraser@12374 51 return;
kaf24@6486 52
kfraser@12374 53 if ( (a->extent_order != 0) &&
kaf24@8468 54 !multipage_allocation_permitted(current->domain) )
kfraser@12374 55 return;
kaf24@6486 56
kfraser@12374 57 for ( i = a->nr_done; i < a->nr_extents; i++ )
kaf24@6486 58 {
kaf24@6486 59 if ( hypercall_preempt_check() )
kaf24@6607 60 {
kfraser@12374 61 a->preempted = 1;
kfraser@12374 62 goto out;
kaf24@6607 63 }
kaf24@6486 64
keir@17986 65 page = alloc_domheap_pages(d, a->extent_order, a->memflags);
kfraser@12374 66 if ( unlikely(page == NULL) )
kaf24@6486 67 {
kaf24@12038 68 gdprintk(XENLOG_INFO, "Could not allocate order=%d extent: "
kfraser@10418 69 "id=%d memflags=%x (%ld of %d)\n",
kfraser@12374 70 a->extent_order, d->domain_id, a->memflags,
kfraser@12374 71 i, a->nr_extents);
kfraser@12374 72 goto out;
kaf24@6486 73 }
kaf24@6486 74
kaf24@6486 75 /* Inform the domain of the new page's machine address. */
kfraser@12374 76 if ( !guest_handle_is_null(a->extent_list) )
kaf24@8859 77 {
kaf24@8859 78 mfn = page_to_mfn(page);
kfraser@12374 79 if ( unlikely(__copy_to_guest_offset(a->extent_list, i, &mfn, 1)) )
kfraser@12374 80 goto out;
kaf24@8859 81 }
kaf24@6486 82 }
kaf24@6486 83
kfraser@12374 84 out:
kfraser@12374 85 a->nr_done = i;
kaf24@6486 86 }
sos22@8688 87
kfraser@12374 88 static void populate_physmap(struct memop_args *a)
kaf24@8673 89 {
kaf24@8726 90 struct page_info *page;
kaf24@10314 91 unsigned long i, j;
kfraser@12374 92 xen_pfn_t gpfn, mfn;
kfraser@12374 93 struct domain *d = a->domain;
kaf24@8673 94
keir@17860 95 if ( !guest_handle_subrange_okay(a->extent_list, a->nr_done,
keir@17860 96 a->nr_extents-1) )
kfraser@12374 97 return;
kaf24@8673 98
kfraser@12374 99 if ( (a->extent_order != 0) &&
kaf24@8673 100 !multipage_allocation_permitted(current->domain) )
kfraser@12374 101 return;
kaf24@8673 102
kfraser@12374 103 for ( i = a->nr_done; i < a->nr_extents; i++ )
kaf24@8673 104 {
kaf24@8673 105 if ( hypercall_preempt_check() )
kaf24@8673 106 {
kfraser@12374 107 a->preempted = 1;
sos22@8688 108 goto out;
kaf24@8673 109 }
kaf24@8673 110
kfraser@12374 111 if ( unlikely(__copy_from_guest_offset(&gpfn, a->extent_list, i, 1)) )
kaf24@8859 112 goto out;
kaf24@8859 113
keir@18975 114 if ( a->memflags & MEMF_populate_on_demand )
kaf24@8673 115 {
keir@18975 116 if ( guest_physmap_mark_populate_on_demand(d, gpfn,
keir@18975 117 a->extent_order) < 0 )
keir@18975 118 goto out;
kaf24@8673 119 }
keir@18975 120 else
keir@18975 121 {
keir@18975 122 page = alloc_domheap_pages(d, a->extent_order, a->memflags);
keir@18975 123 if ( unlikely(page == NULL) )
keir@18975 124 {
keir@18975 125 gdprintk(XENLOG_INFO, "Could not allocate order=%d extent: "
keir@18975 126 "id=%d memflags=%x (%ld of %d)\n",
keir@18975 127 a->extent_order, d->domain_id, a->memflags,
keir@18975 128 i, a->nr_extents);
keir@18975 129 goto out;
keir@18975 130 }
kaf24@8673 131
keir@18975 132 mfn = page_to_mfn(page);
keir@18975 133 guest_physmap_add_page(d, gpfn, mfn, a->extent_order);
kaf24@8673 134
keir@18975 135 if ( !paging_mode_translate(d) )
keir@18975 136 {
keir@18975 137 for ( j = 0; j < (1 << a->extent_order); j++ )
keir@18975 138 set_gpfn_from_mfn(mfn + j, gpfn + j);
kaf24@8673 139
keir@18975 140 /* Inform the domain of the new page's machine address. */
keir@18975 141 if ( unlikely(__copy_to_guest_offset(a->extent_list, i, &mfn, 1)) )
keir@18975 142 goto out;
keir@18975 143 }
sos22@8688 144 }
kaf24@8673 145 }
kaf24@8673 146
keir@18975 147 out:
kfraser@12374 148 a->nr_done = i;
kaf24@8673 149 }
cl349@9211 150
kfraser@12374 151 int guest_remove_page(struct domain *d, unsigned long gmfn)
cl349@9211 152 {
cl349@9211 153 struct page_info *page;
cl349@9211 154 unsigned long mfn;
cl349@9211 155
cl349@9211 156 mfn = gmfn_to_mfn(d, gmfn);
cl349@9211 157 if ( unlikely(!mfn_valid(mfn)) )
cl349@9211 158 {
kaf24@12038 159 gdprintk(XENLOG_INFO, "Domain %u page number %lx invalid\n",
tdeegan@11172 160 d->domain_id, gmfn);
cl349@9211 161 return 0;
cl349@9211 162 }
cl349@9211 163
cl349@9211 164 page = mfn_to_page(mfn);
cl349@9211 165 if ( unlikely(!get_page(page, d)) )
cl349@9211 166 {
kaf24@12038 167 gdprintk(XENLOG_INFO, "Bad page free for domain %u\n", d->domain_id);
cl349@9211 168 return 0;
cl349@9211 169 }
cl349@9211 170
cl349@9211 171 if ( test_and_clear_bit(_PGT_pinned, &page->u.inuse.type_info) )
cl349@9211 172 put_page_and_type(page);
cl349@9211 173
cl349@9211 174 if ( test_and_clear_bit(_PGC_allocated, &page->count_info) )
cl349@9211 175 put_page(page);
cl349@9211 176
keir@17664 177 guest_physmap_remove_page(d, gmfn, mfn, 0);
cl349@9211 178
cl349@9211 179 put_page(page);
cl349@9211 180
cl349@9211 181 return 1;
cl349@9211 182 }
cl349@9211 183
kfraser@12374 184 static void decrease_reservation(struct memop_args *a)
kaf24@6486 185 {
kaf24@10314 186 unsigned long i, j;
kaf24@10314 187 xen_pfn_t gmfn;
kaf24@6486 188
keir@17860 189 if ( !guest_handle_subrange_okay(a->extent_list, a->nr_done,
keir@17860 190 a->nr_extents-1) )
kfraser@12374 191 return;
kaf24@6486 192
kfraser@12374 193 for ( i = a->nr_done; i < a->nr_extents; i++ )
kaf24@6486 194 {
kaf24@6486 195 if ( hypercall_preempt_check() )
kaf24@6607 196 {
kfraser@12374 197 a->preempted = 1;
kfraser@12374 198 goto out;
kaf24@6607 199 }
kaf24@6486 200
kfraser@12374 201 if ( unlikely(__copy_from_guest_offset(&gmfn, a->extent_list, i, 1)) )
kfraser@12374 202 goto out;
kaf24@6486 203
keir@18972 204 /* See if populate-on-demand wants to handle this */
keir@18972 205 if ( is_hvm_domain(a->domain)
keir@18972 206 && p2m_pod_decrease_reservation(a->domain, gmfn, a->extent_order) )
keir@18972 207 continue;
keir@18972 208
kfraser@12374 209 for ( j = 0; j < (1 << a->extent_order); j++ )
kfraser@12374 210 if ( !guest_remove_page(a->domain, gmfn + j) )
kfraser@12374 211 goto out;
kaf24@6486 212 }
kaf24@6486 213
kfraser@12374 214 out:
kfraser@12374 215 a->nr_done = i;
kaf24@6486 216 }
kaf24@6486 217
kfraser@12374 218 static long translate_gpfn_list(
kaf24@9873 219 XEN_GUEST_HANDLE(xen_translate_gpfn_list_t) uop, unsigned long *progress)
kaf24@8871 220 {
kaf24@8871 221 struct xen_translate_gpfn_list op;
kaf24@10314 222 unsigned long i;
kaf24@10314 223 xen_pfn_t gpfn;
kaf24@10314 224 xen_pfn_t mfn;
kaf24@8871 225 struct domain *d;
kfraser@15815 226 int rc;
kaf24@6486 227
kaf24@9068 228 if ( copy_from_guest(&op, uop, 1) )
kaf24@8871 229 return -EFAULT;
kaf24@8871 230
kaf24@8871 231 /* Is size too large for us to encode a continuation? */
ack@13295 232 if ( op.nr_gpfns > (ULONG_MAX >> MEMOP_EXTENT_SHIFT) )
kaf24@8871 233 return -EINVAL;
kaf24@8871 234
keir@17860 235 if ( !guest_handle_subrange_okay(op.gpfn_list, *progress, op.nr_gpfns-1) ||
keir@17860 236 !guest_handle_subrange_okay(op.mfn_list, *progress, op.nr_gpfns-1) )
kaf24@8871 237 return -EFAULT;
kaf24@8871 238
keir@18574 239 rc = rcu_lock_target_domain_by_id(op.domid, &d);
keir@18574 240 if ( rc )
keir@18574 241 return rc;
kaf24@8871 242
Tim@15635 243 if ( !paging_mode_translate(d) )
kaf24@8871 244 {
kfraser@14192 245 rcu_unlock_domain(d);
kaf24@8871 246 return -EINVAL;
kaf24@8871 247 }
kaf24@8871 248
kaf24@8871 249 for ( i = *progress; i < op.nr_gpfns; i++ )
kaf24@8871 250 {
kaf24@8871 251 if ( hypercall_preempt_check() )
kaf24@8871 252 {
kfraser@14192 253 rcu_unlock_domain(d);
kaf24@8871 254 *progress = i;
kaf24@8871 255 return -EAGAIN;
kaf24@8871 256 }
kaf24@8871 257
kaf24@9068 258 if ( unlikely(__copy_from_guest_offset(&gpfn, op.gpfn_list, i, 1)) )
kaf24@8871 259 {
kfraser@14192 260 rcu_unlock_domain(d);
kaf24@8871 261 return -EFAULT;
kaf24@8871 262 }
kaf24@8871 263
kaf24@8871 264 mfn = gmfn_to_mfn(d, gpfn);
kaf24@8871 265
kfraser@15815 266 rc = xsm_translate_gpfn_list(current->domain, mfn);
kfraser@15815 267 if ( rc )
kfraser@15815 268 {
kfraser@15815 269 rcu_unlock_domain(d);
kfraser@15815 270 return rc;
kfraser@15815 271 }
kfraser@15815 272
kaf24@9068 273 if ( unlikely(__copy_to_guest_offset(op.mfn_list, i, &mfn, 1)) )
kaf24@8871 274 {
kfraser@14192 275 rcu_unlock_domain(d);
kaf24@8871 276 return -EFAULT;
kaf24@8871 277 }
kaf24@8871 278 }
kaf24@8871 279
kfraser@14192 280 rcu_unlock_domain(d);
kaf24@8871 281 return 0;
kaf24@8871 282 }
kaf24@8871 283
kfraser@12374 284 static long memory_exchange(XEN_GUEST_HANDLE(xen_memory_exchange_t) arg)
kfraser@10418 285 {
kfraser@10418 286 struct xen_memory_exchange exch;
kfraser@10418 287 LIST_HEAD(in_chunk_list);
kfraser@10418 288 LIST_HEAD(out_chunk_list);
kfraser@10418 289 unsigned long in_chunk_order, out_chunk_order;
kaf24@10459 290 xen_pfn_t gpfn, gmfn, mfn;
kfraser@10418 291 unsigned long i, j, k;
keir@17988 292 unsigned int node, memflags = 0;
kfraser@10418 293 long rc = 0;
kfraser@10418 294 struct domain *d;
kfraser@10418 295 struct page_info *page;
kfraser@10418 296
kfraser@10418 297 if ( copy_from_guest(&exch, arg, 1) )
kfraser@10418 298 return -EFAULT;
kfraser@10418 299
kfraser@10418 300 /* Various sanity checks. */
kfraser@10418 301 if ( (exch.nr_exchanged > exch.in.nr_extents) ||
kfraser@10418 302 /* Input and output domain identifiers match? */
kfraser@10418 303 (exch.in.domid != exch.out.domid) ||
kfraser@10418 304 /* Sizes of input and output lists do not overflow a long? */
kfraser@10418 305 ((~0UL >> exch.in.extent_order) < exch.in.nr_extents) ||
kfraser@10418 306 ((~0UL >> exch.out.extent_order) < exch.out.nr_extents) ||
kfraser@10418 307 /* Sizes of input and output lists match? */
kfraser@10418 308 ((exch.in.nr_extents << exch.in.extent_order) !=
kfraser@10418 309 (exch.out.nr_extents << exch.out.extent_order)) )
kfraser@10418 310 {
kfraser@10418 311 rc = -EINVAL;
kfraser@10418 312 goto fail_early;
kfraser@10418 313 }
kfraser@10418 314
kfraser@10418 315 /* Only privileged guests can allocate multi-page contiguous extents. */
kfraser@10418 316 if ( ((exch.in.extent_order != 0) || (exch.out.extent_order != 0)) &&
kfraser@10418 317 !multipage_allocation_permitted(current->domain) )
kfraser@10418 318 {
kfraser@10418 319 rc = -EPERM;
kfraser@10418 320 goto fail_early;
kfraser@10418 321 }
kfraser@10418 322
kfraser@10418 323 if ( exch.in.extent_order <= exch.out.extent_order )
kfraser@10418 324 {
kfraser@10418 325 in_chunk_order = exch.out.extent_order - exch.in.extent_order;
kfraser@10418 326 out_chunk_order = 0;
kfraser@10418 327 }
kfraser@10418 328 else
kfraser@10418 329 {
kfraser@10418 330 in_chunk_order = 0;
kfraser@10418 331 out_chunk_order = exch.in.extent_order - exch.out.extent_order;
kfraser@10418 332 }
kfraser@10418 333
kfraser@10418 334 /*
kfraser@10418 335 * Only support exchange on calling domain right now. Otherwise there are
kfraser@14642 336 * tricky corner cases to consider (e.g., dying domain).
kfraser@10418 337 */
kfraser@10418 338 if ( unlikely(exch.in.domid != DOMID_SELF) )
kfraser@10418 339 {
kfraser@10418 340 rc = IS_PRIV(current->domain) ? -EINVAL : -EPERM;
kfraser@10418 341 goto fail_early;
kfraser@10418 342 }
kfraser@10418 343 d = current->domain;
kfraser@10418 344
keir@16548 345 memflags |= MEMF_bits(domain_clamp_alloc_bitsize(
keir@17986 346 d,
keir@17986 347 XENMEMF_get_address_bits(exch.out.mem_flags) ? :
keir@17986 348 (BITS_PER_LONG+PAGE_SHIFT)));
keir@17988 349 node = XENMEMF_get_node(exch.out.mem_flags);
keir@17988 350 if ( node == NUMA_NO_NODE )
keir@17988 351 node = domain_to_node(d);
keir@17988 352 memflags |= MEMF_node(node);
kfraser@11973 353
kfraser@12374 354 for ( i = (exch.nr_exchanged >> in_chunk_order);
kfraser@12374 355 i < (exch.in.nr_extents >> in_chunk_order);
kfraser@12374 356 i++ )
kfraser@10418 357 {
kfraser@10418 358 if ( hypercall_preempt_check() )
kfraser@10418 359 {
kfraser@12374 360 exch.nr_exchanged = i << in_chunk_order;
kfraser@10418 361 if ( copy_field_to_guest(arg, &exch, nr_exchanged) )
kfraser@10418 362 return -EFAULT;
kfraser@10418 363 return hypercall_create_continuation(
kfraser@10418 364 __HYPERVISOR_memory_op, "lh", XENMEM_exchange, arg);
kfraser@10418 365 }
kfraser@10418 366
kfraser@10418 367 /* Steal a chunk's worth of input pages from the domain. */
kfraser@10418 368 for ( j = 0; j < (1UL << in_chunk_order); j++ )
kfraser@10418 369 {
kfraser@10418 370 if ( unlikely(__copy_from_guest_offset(
kfraser@10418 371 &gmfn, exch.in.extent_start, (i<<in_chunk_order)+j, 1)) )
kfraser@10418 372 {
kfraser@10418 373 rc = -EFAULT;
kfraser@10418 374 goto fail;
kfraser@10418 375 }
kfraser@10418 376
kfraser@10418 377 for ( k = 0; k < (1UL << exch.in.extent_order); k++ )
kfraser@10418 378 {
kfraser@10418 379 mfn = gmfn_to_mfn(d, gmfn + k);
kfraser@10418 380 if ( unlikely(!mfn_valid(mfn)) )
kfraser@10418 381 {
kfraser@10418 382 rc = -EINVAL;
kfraser@10418 383 goto fail;
kfraser@10418 384 }
kfraser@10418 385
kfraser@10418 386 page = mfn_to_page(mfn);
kfraser@10418 387
kfraser@10418 388 if ( unlikely(steal_page(d, page, MEMF_no_refcount)) )
kfraser@10418 389 {
kfraser@10418 390 rc = -EINVAL;
kfraser@10418 391 goto fail;
kfraser@10418 392 }
kfraser@10418 393
kfraser@10418 394 list_add(&page->list, &in_chunk_list);
kfraser@10418 395 }
kfraser@10418 396 }
kfraser@10418 397
kfraser@10418 398 /* Allocate a chunk's worth of anonymous output pages. */
kfraser@10418 399 for ( j = 0; j < (1UL << out_chunk_order); j++ )
kfraser@10418 400 {
keir@17385 401 page = alloc_domheap_pages(NULL, exch.out.extent_order, memflags);
kfraser@10418 402 if ( unlikely(page == NULL) )
kfraser@10418 403 {
kfraser@10418 404 rc = -ENOMEM;
kfraser@10418 405 goto fail;
kfraser@10418 406 }
kfraser@10418 407
kfraser@10418 408 list_add(&page->list, &out_chunk_list);
kfraser@10418 409 }
kfraser@10418 410
kfraser@10418 411 /*
kfraser@10418 412 * Success! Beyond this point we cannot fail for this chunk.
kfraser@10418 413 */
kfraser@10418 414
kfraser@10418 415 /* Destroy final reference to each input page. */
kfraser@10418 416 while ( !list_empty(&in_chunk_list) )
kfraser@10418 417 {
kfraser@10418 418 page = list_entry(in_chunk_list.next, struct page_info, list);
kfraser@10418 419 list_del(&page->list);
kfraser@10418 420 if ( !test_and_clear_bit(_PGC_allocated, &page->count_info) )
kfraser@10418 421 BUG();
kfraser@10418 422 mfn = page_to_mfn(page);
keir@17664 423 guest_physmap_remove_page(d, mfn_to_gmfn(d, mfn), mfn, 0);
kfraser@10418 424 put_page(page);
kfraser@10418 425 }
kfraser@10418 426
kfraser@10418 427 /* Assign each output page to the domain. */
kfraser@10418 428 j = 0;
kfraser@10418 429 while ( !list_empty(&out_chunk_list) )
kfraser@10418 430 {
kfraser@10418 431 page = list_entry(out_chunk_list.next, struct page_info, list);
kfraser@10418 432 list_del(&page->list);
kfraser@10418 433 if ( assign_pages(d, page, exch.out.extent_order,
kfraser@10418 434 MEMF_no_refcount) )
kfraser@10418 435 BUG();
kfraser@10418 436
kfraser@10418 437 /* Note that we ignore errors accessing the output extent list. */
kfraser@10418 438 (void)__copy_from_guest_offset(
kfraser@10418 439 &gpfn, exch.out.extent_start, (i<<out_chunk_order)+j, 1);
kfraser@10418 440
kfraser@10418 441 mfn = page_to_mfn(page);
keir@17727 442 guest_physmap_add_page(d, gpfn, mfn, exch.out.extent_order);
keir@17727 443
keir@17727 444 if ( !paging_mode_translate(d) )
kfraser@10418 445 {
kfraser@10418 446 for ( k = 0; k < (1UL << exch.out.extent_order); k++ )
kfraser@10418 447 set_gpfn_from_mfn(mfn + k, gpfn + k);
kfraser@10418 448 (void)__copy_to_guest_offset(
kfraser@10418 449 exch.out.extent_start, (i<<out_chunk_order)+j, &mfn, 1);
kfraser@10418 450 }
kfraser@10418 451
kfraser@10418 452 j++;
kfraser@10418 453 }
kfraser@10418 454 BUG_ON(j != (1UL << out_chunk_order));
kfraser@10418 455 }
kfraser@10418 456
kfraser@12374 457 exch.nr_exchanged = exch.in.nr_extents;
kfraser@10418 458 if ( copy_field_to_guest(arg, &exch, nr_exchanged) )
kfraser@10418 459 rc = -EFAULT;
kfraser@10418 460 return rc;
kfraser@10418 461
kfraser@10418 462 /*
kfraser@10418 463 * Failed a chunk! Free any partial chunk work. Tell caller how many
kfraser@10418 464 * chunks succeeded.
kfraser@10418 465 */
kfraser@10418 466 fail:
kfraser@10418 467 /* Reassign any input pages we managed to steal. */
kfraser@10418 468 while ( !list_empty(&in_chunk_list) )
kfraser@10418 469 {
kfraser@10418 470 page = list_entry(in_chunk_list.next, struct page_info, list);
kfraser@10418 471 list_del(&page->list);
kfraser@10418 472 if ( assign_pages(d, page, 0, MEMF_no_refcount) )
kfraser@10418 473 BUG();
kfraser@10418 474 }
kfraser@10418 475
kfraser@10418 476 /* Free any output pages we managed to allocate. */
kfraser@10418 477 while ( !list_empty(&out_chunk_list) )
kfraser@10418 478 {
kfraser@10418 479 page = list_entry(out_chunk_list.next, struct page_info, list);
kfraser@10418 480 list_del(&page->list);
kfraser@10418 481 free_domheap_pages(page, exch.out.extent_order);
kfraser@10418 482 }
kfraser@10418 483
kfraser@12374 484 exch.nr_exchanged = i << in_chunk_order;
kfraser@10418 485
kfraser@10418 486 fail_early:
kfraser@10418 487 if ( copy_field_to_guest(arg, &exch, nr_exchanged) )
kfraser@10418 488 rc = -EFAULT;
kfraser@10418 489 return rc;
kfraser@10418 490 }
kfraser@10418 491
kaf24@9873 492 long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE(void) arg)
kaf24@6486 493 {
kaf24@6486 494 struct domain *d;
kfraser@12374 495 int rc, op;
keir@17986 496 unsigned int address_bits;
kaf24@8871 497 unsigned long start_extent, progress;
kaf24@6486 498 struct xen_memory_reservation reservation;
kfraser@12374 499 struct memop_args args;
kaf24@7959 500 domid_t domid;
kaf24@6486 501
ack@13295 502 op = cmd & MEMOP_CMD_MASK;
kaf24@6486 503
kaf24@6486 504 switch ( op )
kaf24@6486 505 {
kaf24@6486 506 case XENMEM_increase_reservation:
kaf24@6486 507 case XENMEM_decrease_reservation:
kaf24@8673 508 case XENMEM_populate_physmap:
ack@13295 509 start_extent = cmd >> MEMOP_EXTENT_SHIFT;
kfraser@10418 510
kaf24@9068 511 if ( copy_from_guest(&reservation, arg, 1) )
kfraser@10418 512 return start_extent;
kaf24@6486 513
kaf24@8871 514 /* Is size too large for us to encode a continuation? */
ack@13295 515 if ( reservation.nr_extents > (ULONG_MAX >> MEMOP_EXTENT_SHIFT) )
kfraser@10418 516 return start_extent;
kaf24@8871 517
kaf24@6486 518 if ( unlikely(start_extent > reservation.nr_extents) )
kfraser@10418 519 return start_extent;
kaf24@9068 520
kfraser@12374 521 args.extent_list = reservation.extent_start;
kfraser@12374 522 args.nr_extents = reservation.nr_extents;
kfraser@12374 523 args.extent_order = reservation.extent_order;
kfraser@12374 524 args.nr_done = start_extent;
kfraser@12374 525 args.preempted = 0;
kfraser@12374 526 args.memflags = 0;
kaf24@6486 527
keir@17986 528 address_bits = XENMEMF_get_address_bits(reservation.mem_flags);
keir@17986 529 if ( (address_bits != 0) &&
keir@17986 530 (address_bits < (get_order_from_pages(max_page) + PAGE_SHIFT)) )
kaf24@6486 531 {
keir@17986 532 if ( address_bits <= PAGE_SHIFT )
kfraser@10418 533 return start_extent;
keir@17986 534 args.memflags = MEMF_bits(address_bits);
kaf24@6486 535 }
kaf24@6486 536
keir@17986 537 args.memflags |= MEMF_node(XENMEMF_get_node(reservation.mem_flags));
keir@17986 538
keir@18975 539 if ( op == XENMEM_populate_physmap
keir@18975 540 && (reservation.mem_flags & XENMEMF_populate_on_demand) )
keir@18975 541 args.memflags |= MEMF_populate_on_demand;
keir@18975 542
kaf24@6486 543 if ( likely(reservation.domid == DOMID_SELF) )
keir@17349 544 {
keir@17349 545 d = rcu_lock_current_domain();
keir@17349 546 }
keir@17349 547 else
keir@17349 548 {
keir@17349 549 if ( (d = rcu_lock_domain_by_id(reservation.domid)) == NULL )
keir@16856 550 return start_extent;
keir@17349 551 if ( !IS_PRIV_FOR(current->domain, d) )
keir@17349 552 {
keir@16856 553 rcu_unlock_domain(d);
keir@16856 554 return start_extent;
keir@16856 555 }
keir@16856 556 }
kfraser@12374 557 args.domain = d;
kaf24@6486 558
kfraser@15815 559 rc = xsm_memory_adjust_reservation(current->domain, d);
kfraser@15815 560 if ( rc )
kfraser@15815 561 {
keir@17349 562 rcu_unlock_domain(d);
kfraser@15815 563 return rc;
kfraser@15815 564 }
kfraser@15815 565
kaf24@8673 566 switch ( op )
kaf24@8673 567 {
kaf24@8673 568 case XENMEM_increase_reservation:
kfraser@12374 569 increase_reservation(&args);
kaf24@8673 570 break;
kaf24@8673 571 case XENMEM_decrease_reservation:
kfraser@12374 572 decrease_reservation(&args);
kaf24@8673 573 break;
kfraser@12374 574 default: /* XENMEM_populate_physmap */
kfraser@12374 575 populate_physmap(&args);
kaf24@8673 576 break;
kaf24@8673 577 }
kaf24@6486 578
keir@17349 579 rcu_unlock_domain(d);
kaf24@6486 580
kfraser@12374 581 rc = args.nr_done;
kaf24@6486 582
kfraser@12374 583 if ( args.preempted )
kaf24@9068 584 return hypercall_create_continuation(
kaf24@9068 585 __HYPERVISOR_memory_op, "lh",
ack@13295 586 op | (rc << MEMOP_EXTENT_SHIFT), arg);
kaf24@6607 587
kaf24@6486 588 break;
kaf24@6486 589
kfraser@10418 590 case XENMEM_exchange:
kfraser@10418 591 rc = memory_exchange(guest_handle_cast(arg, xen_memory_exchange_t));
kfraser@10418 592 break;
kfraser@10418 593
kaf24@6486 594 case XENMEM_maximum_ram_page:
kaf24@7959 595 rc = max_page;
kaf24@7959 596 break;
kaf24@7959 597
kaf24@7959 598 case XENMEM_current_reservation:
kaf24@7959 599 case XENMEM_maximum_reservation:
kfraser@14471 600 case XENMEM_maximum_gpfn:
kaf24@9068 601 if ( copy_from_guest(&domid, arg, 1) )
kaf24@6486 602 return -EFAULT;
kaf24@7959 603
keir@18574 604 rc = rcu_lock_target_domain_by_id(domid, &d);
keir@18574 605 if ( rc )
keir@18574 606 return rc;
kaf24@7959 607
kfraser@15815 608 rc = xsm_memory_stat_reservation(current->domain, d);
kfraser@15815 609 if ( rc )
kfraser@15815 610 {
keir@17349 611 rcu_unlock_domain(d);
kfraser@15815 612 return rc;
kfraser@15815 613 }
kfraser@15815 614
kfraser@14471 615 switch ( op )
kfraser@14471 616 {
kfraser@14471 617 case XENMEM_current_reservation:
kfraser@14471 618 rc = d->tot_pages;
kfraser@14471 619 break;
kfraser@14471 620 case XENMEM_maximum_reservation:
kfraser@14471 621 rc = d->max_pages;
kfraser@14471 622 break;
kfraser@14471 623 default:
kfraser@14471 624 ASSERT(op == XENMEM_maximum_gpfn);
kfraser@14471 625 rc = domain_get_maximum_gpfn(d);
kfraser@14471 626 break;
kfraser@14471 627 }
kaf24@7959 628
keir@17349 629 rcu_unlock_domain(d);
kaf24@7959 630
kaf24@6486 631 break;
kaf24@6486 632
kaf24@8871 633 case XENMEM_translate_gpfn_list:
ack@13295 634 progress = cmd >> MEMOP_EXTENT_SHIFT;
kaf24@9068 635 rc = translate_gpfn_list(
kaf24@9068 636 guest_handle_cast(arg, xen_translate_gpfn_list_t),
kaf24@9068 637 &progress);
kaf24@8871 638 if ( rc == -EAGAIN )
kaf24@9068 639 return hypercall_create_continuation(
kaf24@9068 640 __HYPERVISOR_memory_op, "lh",
ack@13295 641 op | (progress << MEMOP_EXTENT_SHIFT), arg);
kaf24@8871 642 break;
kaf24@8871 643
kaf24@6486 644 default:
kaf24@8059 645 rc = arch_memory_op(op, arg);
kaf24@6486 646 break;
kaf24@6486 647 }
kaf24@6486 648
kaf24@6486 649 return rc;
kaf24@6486 650 }
kaf24@6486 651
kaf24@6486 652 /*
kaf24@6486 653 * Local variables:
kaf24@6486 654 * mode: C
kaf24@6486 655 * c-set-style: "BSD"
kaf24@6486 656 * c-basic-offset: 4
kaf24@6486 657 * tab-width: 4
kaf24@6486 658 * indent-tabs-mode: nil
kaf24@6486 659 * End:
kaf24@6486 660 */