ia64/linux-2.6.18-xen.hg

view mm/sparse.c @ 563:3b3701ad4eec

pvSCSI frontend driver

Signed-off-by: Tomonari Horikoshi <t.horikoshi@jp.fujitsu.com>
Signed-off-by: Jun Kamada <kama@jp.fujitsu.com>
author Keir Fraser <keir.fraser@citrix.com>
date Mon Jun 02 09:58:56 2008 +0100 (2008-06-02)
parents 831230e53067
children
line source
1 /*
2 * sparse memory mappings.
3 */
4 #include <linux/mm.h>
5 #include <linux/mmzone.h>
6 #include <linux/bootmem.h>
7 #include <linux/highmem.h>
8 #include <linux/module.h>
9 #include <linux/spinlock.h>
10 #include <linux/vmalloc.h>
11 #include <asm/dma.h>
13 /*
14 * Permanent SPARSEMEM data:
15 *
16 * 1) mem_section - memory sections, mem_map's for valid memory
17 */
18 #ifdef CONFIG_SPARSEMEM_EXTREME
19 struct mem_section *mem_section[NR_SECTION_ROOTS]
20 ____cacheline_internodealigned_in_smp;
21 #else
22 struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]
23 ____cacheline_internodealigned_in_smp;
24 #endif
25 EXPORT_SYMBOL(mem_section);
27 #ifdef CONFIG_SPARSEMEM_EXTREME
28 static struct mem_section *sparse_index_alloc(int nid)
29 {
30 struct mem_section *section = NULL;
31 unsigned long array_size = SECTIONS_PER_ROOT *
32 sizeof(struct mem_section);
34 if (slab_is_available())
35 section = kmalloc_node(array_size, GFP_KERNEL, nid);
36 else
37 section = alloc_bootmem_node(NODE_DATA(nid), array_size);
39 if (section)
40 memset(section, 0, array_size);
42 return section;
43 }
45 static int sparse_index_init(unsigned long section_nr, int nid)
46 {
47 static DEFINE_SPINLOCK(index_init_lock);
48 unsigned long root = SECTION_NR_TO_ROOT(section_nr);
49 struct mem_section *section;
50 int ret = 0;
52 if (mem_section[root])
53 return -EEXIST;
55 section = sparse_index_alloc(nid);
56 /*
57 * This lock keeps two different sections from
58 * reallocating for the same index
59 */
60 spin_lock(&index_init_lock);
62 if (mem_section[root]) {
63 ret = -EEXIST;
64 goto out;
65 }
67 mem_section[root] = section;
68 out:
69 spin_unlock(&index_init_lock);
70 return ret;
71 }
72 #else /* !SPARSEMEM_EXTREME */
73 static inline int sparse_index_init(unsigned long section_nr, int nid)
74 {
75 return 0;
76 }
77 #endif
79 /*
80 * Although written for the SPARSEMEM_EXTREME case, this happens
81 * to also work for the flat array case becase
82 * NR_SECTION_ROOTS==NR_MEM_SECTIONS.
83 */
84 int __section_nr(struct mem_section* ms)
85 {
86 unsigned long root_nr;
87 struct mem_section* root;
89 for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) {
90 root = __nr_to_section(root_nr * SECTIONS_PER_ROOT);
91 if (!root)
92 continue;
94 if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT)))
95 break;
96 }
98 return (root_nr * SECTIONS_PER_ROOT) + (ms - root);
99 }
101 /*
102 * During early boot, before section_mem_map is used for an actual
103 * mem_map, we use section_mem_map to store the section's NUMA
104 * node. This keeps us from having to use another data structure. The
105 * node information is cleared just before we store the real mem_map.
106 */
107 static inline unsigned long sparse_encode_early_nid(int nid)
108 {
109 return (nid << SECTION_NID_SHIFT);
110 }
112 static inline int sparse_early_nid(struct mem_section *section)
113 {
114 return (section->section_mem_map >> SECTION_NID_SHIFT);
115 }
117 /* Record a memory area against a node. */
118 void memory_present(int nid, unsigned long start, unsigned long end)
119 {
120 unsigned long pfn;
122 start &= PAGE_SECTION_MASK;
123 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {
124 unsigned long section = pfn_to_section_nr(pfn);
125 struct mem_section *ms;
127 sparse_index_init(section, nid);
129 ms = __nr_to_section(section);
130 if (!ms->section_mem_map)
131 ms->section_mem_map = sparse_encode_early_nid(nid) |
132 SECTION_MARKED_PRESENT;
133 }
134 }
136 /*
137 * Only used by the i386 NUMA architecures, but relatively
138 * generic code.
139 */
140 unsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn,
141 unsigned long end_pfn)
142 {
143 unsigned long pfn;
144 unsigned long nr_pages = 0;
146 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
147 if (nid != early_pfn_to_nid(pfn))
148 continue;
150 if (pfn_valid(pfn))
151 nr_pages += PAGES_PER_SECTION;
152 }
154 return nr_pages * sizeof(struct page);
155 }
157 /*
158 * Subtle, we encode the real pfn into the mem_map such that
159 * the identity pfn - section_mem_map will return the actual
160 * physical page frame number.
161 */
162 static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum)
163 {
164 return (unsigned long)(mem_map - (section_nr_to_pfn(pnum)));
165 }
167 /*
168 * We need this if we ever free the mem_maps. While not implemented yet,
169 * this function is included for parity with its sibling.
170 */
171 static __attribute((unused))
172 struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum)
173 {
174 return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum);
175 }
177 static int sparse_init_one_section(struct mem_section *ms,
178 unsigned long pnum, struct page *mem_map)
179 {
180 if (!valid_section(ms))
181 return -EINVAL;
183 ms->section_mem_map &= ~SECTION_MAP_MASK;
184 ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum);
186 return 1;
187 }
189 static struct page *sparse_early_mem_map_alloc(unsigned long pnum)
190 {
191 struct page *map;
192 struct mem_section *ms = __nr_to_section(pnum);
193 int nid = sparse_early_nid(ms);
195 map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION);
196 if (map)
197 return map;
199 map = alloc_bootmem_node(NODE_DATA(nid),
200 sizeof(struct page) * PAGES_PER_SECTION);
201 if (map)
202 return map;
204 printk(KERN_WARNING "%s: allocation failed\n", __FUNCTION__);
205 ms->section_mem_map = 0;
206 return NULL;
207 }
209 static struct page *__kmalloc_section_memmap(unsigned long nr_pages)
210 {
211 struct page *page, *ret;
212 unsigned long memmap_size = sizeof(struct page) * nr_pages;
214 page = alloc_pages(GFP_KERNEL, get_order(memmap_size));
215 if (page)
216 goto got_map_page;
218 ret = vmalloc(memmap_size);
219 if (ret)
220 goto got_map_ptr;
222 return NULL;
223 got_map_page:
224 ret = (struct page *)pfn_to_kaddr(page_to_pfn(page));
225 got_map_ptr:
226 memset(ret, 0, memmap_size);
228 return ret;
229 }
231 static int vaddr_in_vmalloc_area(void *addr)
232 {
233 if (addr >= (void *)VMALLOC_START &&
234 addr < (void *)VMALLOC_END)
235 return 1;
236 return 0;
237 }
239 static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages)
240 {
241 if (vaddr_in_vmalloc_area(memmap))
242 vfree(memmap);
243 else
244 free_pages((unsigned long)memmap,
245 get_order(sizeof(struct page) * nr_pages));
246 }
248 /*
249 * Allocate the accumulated non-linear sections, allocate a mem_map
250 * for each and record the physical to section mapping.
251 */
252 void sparse_init(void)
253 {
254 unsigned long pnum;
255 struct page *map;
257 for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
258 if (!valid_section_nr(pnum))
259 continue;
261 map = sparse_early_mem_map_alloc(pnum);
262 if (!map)
263 continue;
264 sparse_init_one_section(__nr_to_section(pnum), pnum, map);
265 }
266 }
268 /*
269 * returns the number of sections whose mem_maps were properly
270 * set. If this is <=0, then that means that the passed-in
271 * map was not consumed and must be freed.
272 */
273 int sparse_add_one_section(struct zone *zone, unsigned long start_pfn,
274 int nr_pages)
275 {
276 unsigned long section_nr = pfn_to_section_nr(start_pfn);
277 struct pglist_data *pgdat = zone->zone_pgdat;
278 struct mem_section *ms;
279 struct page *memmap;
280 unsigned long flags;
281 int ret;
283 /*
284 * no locking for this, because it does its own
285 * plus, it does a kmalloc
286 */
287 sparse_index_init(section_nr, pgdat->node_id);
288 memmap = __kmalloc_section_memmap(nr_pages);
290 pgdat_resize_lock(pgdat, &flags);
292 ms = __pfn_to_section(start_pfn);
293 if (ms->section_mem_map & SECTION_MARKED_PRESENT) {
294 ret = -EEXIST;
295 goto out;
296 }
297 ms->section_mem_map |= SECTION_MARKED_PRESENT;
299 ret = sparse_init_one_section(ms, section_nr, memmap);
301 out:
302 pgdat_resize_unlock(pgdat, &flags);
303 if (ret <= 0)
304 __kfree_section_memmap(memmap, nr_pages);
305 return ret;
306 }