ia64/xen-unstable

view linux-2.6-xen-sparse/include/asm-xen/asm-i386/system.h @ 6780:e17161930711

synch_bitops.h is an arch-specific header file.

Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Tue Sep 13 10:40:29 2005 +0000 (2005-09-13)
parents dd668f7527cb
children 4d899a738d59 8ca0f98ba8e2
line source
1 #ifndef __ASM_SYSTEM_H
2 #define __ASM_SYSTEM_H
4 #include <linux/config.h>
5 #include <linux/kernel.h>
6 #include <linux/bitops.h>
7 #include <asm/synch_bitops.h>
8 #include <asm/segment.h>
9 #include <asm/cpufeature.h>
10 #include <asm-xen/hypervisor.h>
11 #include <asm/smp_alt.h>
13 #ifdef __KERNEL__
15 struct task_struct; /* one of the stranger aspects of C forward declarations.. */
16 extern struct task_struct * FASTCALL(__switch_to(struct task_struct *prev, struct task_struct *next));
18 #define switch_to(prev,next,last) do { \
19 unsigned long esi,edi; \
20 asm volatile("pushfl\n\t" \
21 "pushl %%ebp\n\t" \
22 "movl %%esp,%0\n\t" /* save ESP */ \
23 "movl %5,%%esp\n\t" /* restore ESP */ \
24 "movl $1f,%1\n\t" /* save EIP */ \
25 "pushl %6\n\t" /* restore EIP */ \
26 "jmp __switch_to\n" \
27 "1:\t" \
28 "popl %%ebp\n\t" \
29 "popfl" \
30 :"=m" (prev->thread.esp),"=m" (prev->thread.eip), \
31 "=a" (last),"=S" (esi),"=D" (edi) \
32 :"m" (next->thread.esp),"m" (next->thread.eip), \
33 "2" (prev), "d" (next)); \
34 } while (0)
36 #define _set_base(addr,base) do { unsigned long __pr; \
37 __asm__ __volatile__ ("movw %%dx,%1\n\t" \
38 "rorl $16,%%edx\n\t" \
39 "movb %%dl,%2\n\t" \
40 "movb %%dh,%3" \
41 :"=&d" (__pr) \
42 :"m" (*((addr)+2)), \
43 "m" (*((addr)+4)), \
44 "m" (*((addr)+7)), \
45 "0" (base) \
46 ); } while(0)
48 #define _set_limit(addr,limit) do { unsigned long __lr; \
49 __asm__ __volatile__ ("movw %%dx,%1\n\t" \
50 "rorl $16,%%edx\n\t" \
51 "movb %2,%%dh\n\t" \
52 "andb $0xf0,%%dh\n\t" \
53 "orb %%dh,%%dl\n\t" \
54 "movb %%dl,%2" \
55 :"=&d" (__lr) \
56 :"m" (*(addr)), \
57 "m" (*((addr)+6)), \
58 "0" (limit) \
59 ); } while(0)
61 #define set_base(ldt,base) _set_base( ((char *)&(ldt)) , (base) )
62 #define set_limit(ldt,limit) _set_limit( ((char *)&(ldt)) , ((limit)-1)>>12 )
64 static inline unsigned long _get_base(char * addr)
65 {
66 unsigned long __base;
67 __asm__("movb %3,%%dh\n\t"
68 "movb %2,%%dl\n\t"
69 "shll $16,%%edx\n\t"
70 "movw %1,%%dx"
71 :"=&d" (__base)
72 :"m" (*((addr)+2)),
73 "m" (*((addr)+4)),
74 "m" (*((addr)+7)));
75 return __base;
76 }
78 #define get_base(ldt) _get_base( ((char *)&(ldt)) )
80 /*
81 * Load a segment. Fall back on loading the zero
82 * segment if something goes wrong..
83 */
84 #define loadsegment(seg,value) \
85 asm volatile("\n" \
86 "1:\t" \
87 "mov %0,%%" #seg "\n" \
88 "2:\n" \
89 ".section .fixup,\"ax\"\n" \
90 "3:\t" \
91 "pushl $0\n\t" \
92 "popl %%" #seg "\n\t" \
93 "jmp 2b\n" \
94 ".previous\n" \
95 ".section __ex_table,\"a\"\n\t" \
96 ".align 4\n\t" \
97 ".long 1b,3b\n" \
98 ".previous" \
99 : :"m" (value))
101 /*
102 * Save a segment register away
103 */
104 #define savesegment(seg, value) \
105 asm volatile("mov %%" #seg ",%0":"=m" (value))
107 /*
108 * Clear and set 'TS' bit respectively
109 */
110 #define clts() (HYPERVISOR_fpu_taskswitch(0))
111 #define read_cr0() ({ \
112 unsigned int __dummy; \
113 __asm__( \
114 "movl %%cr0,%0\n\t" \
115 :"=r" (__dummy)); \
116 __dummy; \
117 })
118 #define write_cr0(x) \
119 __asm__("movl %0,%%cr0": :"r" (x));
121 #define read_cr4() ({ \
122 unsigned int __dummy; \
123 __asm__( \
124 "movl %%cr4,%0\n\t" \
125 :"=r" (__dummy)); \
126 __dummy; \
127 })
128 #define write_cr4(x) \
129 __asm__("movl %0,%%cr4": :"r" (x));
130 #define stts() (HYPERVISOR_fpu_taskswitch(1))
132 #endif /* __KERNEL__ */
134 #define wbinvd() \
135 __asm__ __volatile__ ("wbinvd": : :"memory");
137 static inline unsigned long get_limit(unsigned long segment)
138 {
139 unsigned long __limit;
140 __asm__("lsll %1,%0"
141 :"=r" (__limit):"r" (segment));
142 return __limit+1;
143 }
145 #define nop() __asm__ __volatile__ ("nop")
147 #define xchg(ptr,v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v),(ptr),sizeof(*(ptr))))
149 #define tas(ptr) (xchg((ptr),1))
151 struct __xchg_dummy { unsigned long a[100]; };
152 #define __xg(x) ((struct __xchg_dummy *)(x))
155 /*
156 * The semantics of XCHGCMP8B are a bit strange, this is why
157 * there is a loop and the loading of %%eax and %%edx has to
158 * be inside. This inlines well in most cases, the cached
159 * cost is around ~38 cycles. (in the future we might want
160 * to do an SIMD/3DNOW!/MMX/FPU 64-bit store here, but that
161 * might have an implicit FPU-save as a cost, so it's not
162 * clear which path to go.)
163 *
164 * cmpxchg8b must be used with the lock prefix here to allow
165 * the instruction to be executed atomically, see page 3-102
166 * of the instruction set reference 24319102.pdf. We need
167 * the reader side to see the coherent 64bit value.
168 */
169 static inline void __set_64bit (unsigned long long * ptr,
170 unsigned int low, unsigned int high)
171 {
172 __asm__ __volatile__ (
173 "\n1:\t"
174 "movl (%0), %%eax\n\t"
175 "movl 4(%0), %%edx\n\t"
176 "lock cmpxchg8b (%0)\n\t"
177 "jnz 1b"
178 : /* no outputs */
179 : "D"(ptr),
180 "b"(low),
181 "c"(high)
182 : "ax","dx","memory");
183 }
185 static inline void __set_64bit_constant (unsigned long long *ptr,
186 unsigned long long value)
187 {
188 __set_64bit(ptr,(unsigned int)(value), (unsigned int)((value)>>32ULL));
189 }
190 #define ll_low(x) *(((unsigned int*)&(x))+0)
191 #define ll_high(x) *(((unsigned int*)&(x))+1)
193 static inline void __set_64bit_var (unsigned long long *ptr,
194 unsigned long long value)
195 {
196 __set_64bit(ptr,ll_low(value), ll_high(value));
197 }
199 #define set_64bit(ptr,value) \
200 (__builtin_constant_p(value) ? \
201 __set_64bit_constant(ptr, value) : \
202 __set_64bit_var(ptr, value) )
204 #define _set_64bit(ptr,value) \
205 (__builtin_constant_p(value) ? \
206 __set_64bit(ptr, (unsigned int)(value), (unsigned int)((value)>>32ULL) ) : \
207 __set_64bit(ptr, ll_low(value), ll_high(value)) )
209 /*
210 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway
211 * Note 2: xchg has side effect, so that attribute volatile is necessary,
212 * but generally the primitive is invalid, *ptr is output argument. --ANK
213 */
214 static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size)
215 {
216 switch (size) {
217 case 1:
218 __asm__ __volatile__("xchgb %b0,%1"
219 :"=q" (x)
220 :"m" (*__xg(ptr)), "0" (x)
221 :"memory");
222 break;
223 case 2:
224 __asm__ __volatile__("xchgw %w0,%1"
225 :"=r" (x)
226 :"m" (*__xg(ptr)), "0" (x)
227 :"memory");
228 break;
229 case 4:
230 __asm__ __volatile__("xchgl %0,%1"
231 :"=r" (x)
232 :"m" (*__xg(ptr)), "0" (x)
233 :"memory");
234 break;
235 }
236 return x;
237 }
239 /*
240 * Atomic compare and exchange. Compare OLD with MEM, if identical,
241 * store NEW in MEM. Return the initial value in MEM. Success is
242 * indicated by comparing RETURN with OLD.
243 */
245 #ifdef CONFIG_X86_CMPXCHG
246 #define __HAVE_ARCH_CMPXCHG 1
247 #endif
249 static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
250 unsigned long new, int size)
251 {
252 unsigned long prev;
253 switch (size) {
254 case 1:
255 __asm__ __volatile__(LOCK "cmpxchgb %b1,%2"
256 : "=a"(prev)
257 : "q"(new), "m"(*__xg(ptr)), "0"(old)
258 : "memory");
259 return prev;
260 case 2:
261 __asm__ __volatile__(LOCK "cmpxchgw %w1,%2"
262 : "=a"(prev)
263 : "q"(new), "m"(*__xg(ptr)), "0"(old)
264 : "memory");
265 return prev;
266 case 4:
267 __asm__ __volatile__(LOCK "cmpxchgl %1,%2"
268 : "=a"(prev)
269 : "q"(new), "m"(*__xg(ptr)), "0"(old)
270 : "memory");
271 return prev;
272 }
273 return old;
274 }
276 #define cmpxchg(ptr,o,n)\
277 ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
278 (unsigned long)(n),sizeof(*(ptr))))
280 #ifdef __KERNEL__
281 struct alt_instr {
282 __u8 *instr; /* original instruction */
283 __u8 *replacement;
284 __u8 cpuid; /* cpuid bit set for replacement */
285 __u8 instrlen; /* length of original instruction */
286 __u8 replacementlen; /* length of new instruction, <= instrlen */
287 __u8 pad;
288 };
289 #endif
291 /*
292 * Alternative instructions for different CPU types or capabilities.
293 *
294 * This allows to use optimized instructions even on generic binary
295 * kernels.
296 *
297 * length of oldinstr must be longer or equal the length of newinstr
298 * It can be padded with nops as needed.
299 *
300 * For non barrier like inlines please define new variants
301 * without volatile and memory clobber.
302 */
303 #define alternative(oldinstr, newinstr, feature) \
304 asm volatile ("661:\n\t" oldinstr "\n662:\n" \
305 ".section .altinstructions,\"a\"\n" \
306 " .align 4\n" \
307 " .long 661b\n" /* label */ \
308 " .long 663f\n" /* new instruction */ \
309 " .byte %c0\n" /* feature bit */ \
310 " .byte 662b-661b\n" /* sourcelen */ \
311 " .byte 664f-663f\n" /* replacementlen */ \
312 ".previous\n" \
313 ".section .altinstr_replacement,\"ax\"\n" \
314 "663:\n\t" newinstr "\n664:\n" /* replacement */ \
315 ".previous" :: "i" (feature) : "memory")
317 /*
318 * Alternative inline assembly with input.
319 *
320 * Pecularities:
321 * No memory clobber here.
322 * Argument numbers start with 1.
323 * Best is to use constraints that are fixed size (like (%1) ... "r")
324 * If you use variable sized constraints like "m" or "g" in the
325 * replacement maake sure to pad to the worst case length.
326 */
327 #define alternative_input(oldinstr, newinstr, feature, input...) \
328 asm volatile ("661:\n\t" oldinstr "\n662:\n" \
329 ".section .altinstructions,\"a\"\n" \
330 " .align 4\n" \
331 " .long 661b\n" /* label */ \
332 " .long 663f\n" /* new instruction */ \
333 " .byte %c0\n" /* feature bit */ \
334 " .byte 662b-661b\n" /* sourcelen */ \
335 " .byte 664f-663f\n" /* replacementlen */ \
336 ".previous\n" \
337 ".section .altinstr_replacement,\"ax\"\n" \
338 "663:\n\t" newinstr "\n664:\n" /* replacement */ \
339 ".previous" :: "i" (feature), ##input)
341 /*
342 * Force strict CPU ordering.
343 * And yes, this is required on UP too when we're talking
344 * to devices.
345 *
346 * For now, "wmb()" doesn't actually do anything, as all
347 * Intel CPU's follow what Intel calls a *Processor Order*,
348 * in which all writes are seen in the program order even
349 * outside the CPU.
350 *
351 * I expect future Intel CPU's to have a weaker ordering,
352 * but I'd also expect them to finally get their act together
353 * and add some real memory barriers if so.
354 *
355 * Some non intel clones support out of order store. wmb() ceases to be a
356 * nop for these.
357 */
360 /*
361 * Actually only lfence would be needed for mb() because all stores done
362 * by the kernel should be already ordered. But keep a full barrier for now.
363 */
365 #define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2)
366 #define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2)
368 /**
369 * read_barrier_depends - Flush all pending reads that subsequents reads
370 * depend on.
371 *
372 * No data-dependent reads from memory-like regions are ever reordered
373 * over this barrier. All reads preceding this primitive are guaranteed
374 * to access memory (but not necessarily other CPUs' caches) before any
375 * reads following this primitive that depend on the data return by
376 * any of the preceding reads. This primitive is much lighter weight than
377 * rmb() on most CPUs, and is never heavier weight than is
378 * rmb().
379 *
380 * These ordering constraints are respected by both the local CPU
381 * and the compiler.
382 *
383 * Ordering is not guaranteed by anything other than these primitives,
384 * not even by data dependencies. See the documentation for
385 * memory_barrier() for examples and URLs to more information.
386 *
387 * For example, the following code would force ordering (the initial
388 * value of "a" is zero, "b" is one, and "p" is "&a"):
389 *
390 * <programlisting>
391 * CPU 0 CPU 1
392 *
393 * b = 2;
394 * memory_barrier();
395 * p = &b; q = p;
396 * read_barrier_depends();
397 * d = *q;
398 * </programlisting>
399 *
400 * because the read of "*q" depends on the read of "p" and these
401 * two reads are separated by a read_barrier_depends(). However,
402 * the following code, with the same initial values for "a" and "b":
403 *
404 * <programlisting>
405 * CPU 0 CPU 1
406 *
407 * a = 2;
408 * memory_barrier();
409 * b = 3; y = b;
410 * read_barrier_depends();
411 * x = a;
412 * </programlisting>
413 *
414 * does not enforce ordering, since there is no data dependency between
415 * the read of "a" and the read of "b". Therefore, on some CPUs, such
416 * as Alpha, "y" could be set to 3 and "x" to 0. Use rmb()
417 * in cases like thiswhere there are no data dependencies.
418 **/
420 #define read_barrier_depends() do { } while(0)
422 #ifdef CONFIG_X86_OOSTORE
423 /* Actually there are no OOO store capable CPUs for now that do SSE,
424 but make it already an possibility. */
425 #define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM)
426 #else
427 #define wmb() __asm__ __volatile__ ("": : :"memory")
428 #endif
430 #ifdef CONFIG_SMP
431 #define smp_wmb() wmb()
432 #if defined(CONFIG_SMP_ALTERNATIVES) && !defined(MODULE)
433 #define smp_alt_mb(instr) \
434 __asm__ __volatile__("6667:\nnop\nnop\nnop\nnop\nnop\nnop\n6668:\n" \
435 ".section __smp_alternatives,\"a\"\n" \
436 ".long 6667b\n" \
437 ".long 6673f\n" \
438 ".previous\n" \
439 ".section __smp_replacements,\"a\"\n" \
440 "6673:.byte 6668b-6667b\n" \
441 ".byte 6670f-6669f\n" \
442 ".byte 6671f-6670f\n" \
443 ".byte 0\n" \
444 ".byte %c0\n" \
445 "6669:lock;addl $0,0(%%esp)\n" \
446 "6670:" instr "\n" \
447 "6671:\n" \
448 ".previous\n" \
449 : \
450 : "i" (X86_FEATURE_XMM2) \
451 : "memory")
452 #define smp_rmb() smp_alt_mb("lfence")
453 #define smp_mb() smp_alt_mb("mfence")
454 #define set_mb(var, value) do { \
455 unsigned long __set_mb_temp; \
456 __asm__ __volatile__("6667:movl %1, %0\n6668:\n" \
457 ".section __smp_alternatives,\"a\"\n" \
458 ".long 6667b\n" \
459 ".long 6673f\n" \
460 ".previous\n" \
461 ".section __smp_replacements,\"a\"\n" \
462 "6673: .byte 6668b-6667b\n" \
463 ".byte 6670f-6669f\n" \
464 ".byte 0\n" \
465 ".byte 6671f-6670f\n" \
466 ".byte -1\n" \
467 "6669: xchg %1, %0\n" \
468 "6670:movl %1, %0\n" \
469 "6671:\n" \
470 ".previous\n" \
471 : "=m" (var), "=r" (__set_mb_temp) \
472 : "1" (value) \
473 : "memory"); } while (0)
474 #else
475 #define smp_rmb() rmb()
476 #define smp_mb() mb()
477 #define set_mb(var, value) do { xchg(&var, value); } while (0)
478 #endif
479 #define smp_read_barrier_depends() read_barrier_depends()
480 #else
481 #define smp_mb() barrier()
482 #define smp_rmb() barrier()
483 #define smp_wmb() barrier()
484 #define smp_read_barrier_depends() do { } while(0)
485 #define set_mb(var, value) do { var = value; barrier(); } while (0)
486 #endif
488 #define set_wmb(var, value) do { var = value; wmb(); } while (0)
490 /* interrupt control.. */
492 /*
493 * The use of 'barrier' in the following reflects their use as local-lock
494 * operations. Reentrancy must be prevented (e.g., __cli()) /before/ following
495 * critical operations are executed. All critical operations must complete
496 * /before/ reentrancy is permitted (e.g., __sti()). Alpha architecture also
497 * includes these barriers, for example.
498 */
500 #define __cli() \
501 do { \
502 vcpu_info_t *_vcpu; \
503 preempt_disable(); \
504 _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \
505 _vcpu->evtchn_upcall_mask = 1; \
506 preempt_enable_no_resched(); \
507 barrier(); \
508 } while (0)
510 #define __sti() \
511 do { \
512 vcpu_info_t *_vcpu; \
513 barrier(); \
514 preempt_disable(); \
515 _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \
516 _vcpu->evtchn_upcall_mask = 0; \
517 barrier(); /* unmask then check (avoid races) */ \
518 if ( unlikely(_vcpu->evtchn_upcall_pending) ) \
519 force_evtchn_callback(); \
520 preempt_enable(); \
521 } while (0)
523 #define __save_flags(x) \
524 do { \
525 vcpu_info_t *_vcpu; \
526 _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \
527 (x) = _vcpu->evtchn_upcall_mask; \
528 } while (0)
530 #define __restore_flags(x) \
531 do { \
532 vcpu_info_t *_vcpu; \
533 barrier(); \
534 preempt_disable(); \
535 _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \
536 if ((_vcpu->evtchn_upcall_mask = (x)) == 0) { \
537 barrier(); /* unmask then check (avoid races) */ \
538 if ( unlikely(_vcpu->evtchn_upcall_pending) ) \
539 force_evtchn_callback(); \
540 preempt_enable(); \
541 } else \
542 preempt_enable_no_resched(); \
543 } while (0)
545 #define safe_halt() ((void)0)
547 #define __save_and_cli(x) \
548 do { \
549 vcpu_info_t *_vcpu; \
550 preempt_disable(); \
551 _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \
552 (x) = _vcpu->evtchn_upcall_mask; \
553 _vcpu->evtchn_upcall_mask = 1; \
554 preempt_enable_no_resched(); \
555 barrier(); \
556 } while (0)
558 #define local_irq_save(x) __save_and_cli(x)
559 #define local_irq_restore(x) __restore_flags(x)
560 #define local_save_flags(x) __save_flags(x)
561 #define local_irq_disable() __cli()
562 #define local_irq_enable() __sti()
564 /* Don't use smp_processor_id: this is called in debug versions of that fn. */
565 #ifdef CONFIG_SMP
566 #define irqs_disabled() \
567 HYPERVISOR_shared_info->vcpu_data[__smp_processor_id()].evtchn_upcall_mask
568 #else
569 #define irqs_disabled() \
570 HYPERVISOR_shared_info->vcpu_data[0].evtchn_upcall_mask
571 #endif
573 /*
574 * disable hlt during certain critical i/o operations
575 */
576 #define HAVE_DISABLE_HLT
577 void disable_hlt(void);
578 void enable_hlt(void);
580 extern int es7000_plat;
581 void cpu_idle_wait(void);
583 extern unsigned long arch_align_stack(unsigned long sp);
585 #endif