/drivers/include/asm/agp.h |
---|
0,0 → 1,31 |
#ifndef _ASM_X86_AGP_H |
#define _ASM_X86_AGP_H |
#include <asm/pgtable.h> |
#include <asm/cacheflush.h> |
/* |
* Functions to keep the agpgart mappings coherent with the MMU. The |
* GART gives the CPU a physical alias of pages in memory. The alias |
* region is mapped uncacheable. Make sure there are no conflicting |
* mappings with different cachability attributes for the same |
* page. This avoids data corruption on some CPUs. |
*/ |
#define map_page_into_agp(page) set_pages_uc(page, 1) |
#define unmap_page_from_agp(page) set_pages_wb(page, 1) |
/* |
* Could use CLFLUSH here if the cpu supports it. But then it would |
* need to be called for each cacheline of the whole page so it may |
* not be worth it. Would need a page for it. |
*/ |
#define flush_agp_cache() wbinvd() |
/* GATT allocation. Returns/accepts GATT kernel virtual address. */ |
#define alloc_gatt_pages(order) \ |
((char *)__get_free_pages(GFP_KERNEL, (order))) |
#define free_gatt_pages(table, order) \ |
free_pages((unsigned long)(table), (order)) |
#endif /* _ASM_X86_AGP_H */ |
/drivers/include/asm/alternative.h |
---|
0,0 → 1,243 |
#ifndef _ASM_X86_ALTERNATIVE_H |
#define _ASM_X86_ALTERNATIVE_H |
#include <linux/types.h> |
#include <linux/stddef.h> |
#include <linux/stringify.h> |
#include <asm/asm.h> |
/* |
* Alternative inline assembly for SMP. |
* |
* The LOCK_PREFIX macro defined here replaces the LOCK and |
* LOCK_PREFIX macros used everywhere in the source tree. |
* |
* SMP alternatives use the same data structures as the other |
* alternatives and the X86_FEATURE_UP flag to indicate the case of a |
* UP system running a SMP kernel. The existing apply_alternatives() |
* works fine for patching a SMP kernel for UP. |
* |
* The SMP alternative tables can be kept after boot and contain both |
* UP and SMP versions of the instructions to allow switching back to |
* SMP at runtime, when hotplugging in a new CPU, which is especially |
* useful in virtualized environments. |
* |
* The very common lock prefix is handled as special case in a |
* separate table which is a pure address list without replacement ptr |
* and size information. That keeps the table sizes small. |
*/ |
#ifdef CONFIG_SMP |
#define LOCK_PREFIX_HERE \ |
".pushsection .smp_locks,\"a\"\n" \ |
".balign 4\n" \ |
".long 671f - .\n" /* offset */ \ |
".popsection\n" \ |
"671:" |
#define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; " |
#else /* ! CONFIG_SMP */ |
#define LOCK_PREFIX_HERE "" |
#define LOCK_PREFIX "" |
#endif |
struct alt_instr { |
s32 instr_offset; /* original instruction */ |
s32 repl_offset; /* offset to replacement instruction */ |
u16 cpuid; /* cpuid bit set for replacement */ |
u8 instrlen; /* length of original instruction */ |
u8 replacementlen; /* length of new instruction, <= instrlen */ |
}; |
extern void alternative_instructions(void); |
extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end); |
struct module; |
#ifdef CONFIG_SMP |
extern void alternatives_smp_module_add(struct module *mod, char *name, |
void *locks, void *locks_end, |
void *text, void *text_end); |
extern void alternatives_smp_module_del(struct module *mod); |
extern void alternatives_enable_smp(void); |
extern int alternatives_text_reserved(void *start, void *end); |
extern bool skip_smp_alternatives; |
#else |
static inline void alternatives_smp_module_add(struct module *mod, char *name, |
void *locks, void *locks_end, |
void *text, void *text_end) {} |
static inline void alternatives_smp_module_del(struct module *mod) {} |
static inline void alternatives_enable_smp(void) {} |
static inline int alternatives_text_reserved(void *start, void *end) |
{ |
return 0; |
} |
#endif /* CONFIG_SMP */ |
#define OLDINSTR(oldinstr) "661:\n\t" oldinstr "\n662:\n" |
#define b_replacement(number) "663"#number |
#define e_replacement(number) "664"#number |
#define alt_slen "662b-661b" |
#define alt_rlen(number) e_replacement(number)"f-"b_replacement(number)"f" |
#define ALTINSTR_ENTRY(feature, number) \ |
" .long 661b - .\n" /* label */ \ |
" .long " b_replacement(number)"f - .\n" /* new instruction */ \ |
" .word " __stringify(feature) "\n" /* feature bit */ \ |
" .byte " alt_slen "\n" /* source len */ \ |
" .byte " alt_rlen(number) "\n" /* replacement len */ |
#define DISCARD_ENTRY(number) /* rlen <= slen */ \ |
" .byte 0xff + (" alt_rlen(number) ") - (" alt_slen ")\n" |
#define ALTINSTR_REPLACEMENT(newinstr, feature, number) /* replacement */ \ |
b_replacement(number)":\n\t" newinstr "\n" e_replacement(number) ":\n\t" |
/* alternative assembly primitive: */ |
#define ALTERNATIVE(oldinstr, newinstr, feature) \ |
OLDINSTR(oldinstr) \ |
".pushsection .altinstructions,\"a\"\n" \ |
ALTINSTR_ENTRY(feature, 1) \ |
".popsection\n" \ |
".pushsection .discard,\"aw\",@progbits\n" \ |
DISCARD_ENTRY(1) \ |
".popsection\n" \ |
".pushsection .altinstr_replacement, \"ax\"\n" \ |
ALTINSTR_REPLACEMENT(newinstr, feature, 1) \ |
".popsection" |
#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\ |
OLDINSTR(oldinstr) \ |
".pushsection .altinstructions,\"a\"\n" \ |
ALTINSTR_ENTRY(feature1, 1) \ |
ALTINSTR_ENTRY(feature2, 2) \ |
".popsection\n" \ |
".pushsection .discard,\"aw\",@progbits\n" \ |
DISCARD_ENTRY(1) \ |
DISCARD_ENTRY(2) \ |
".popsection\n" \ |
".pushsection .altinstr_replacement, \"ax\"\n" \ |
ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \ |
ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \ |
".popsection" |
/* |
* This must be included *after* the definition of ALTERNATIVE due to |
* <asm/arch_hweight.h> |
*/ |
#include <asm/cpufeature.h> |
/* |
* Alternative instructions for different CPU types or capabilities. |
* |
* This allows to use optimized instructions even on generic binary |
* kernels. |
* |
* length of oldinstr must be longer or equal the length of newinstr |
* It can be padded with nops as needed. |
* |
* For non barrier like inlines please define new variants |
* without volatile and memory clobber. |
*/ |
#define alternative(oldinstr, newinstr, feature) \ |
asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory") |
/* |
* Alternative inline assembly with input. |
* |
* Pecularities: |
* No memory clobber here. |
* Argument numbers start with 1. |
* Best is to use constraints that are fixed size (like (%1) ... "r") |
* If you use variable sized constraints like "m" or "g" in the |
* replacement make sure to pad to the worst case length. |
* Leaving an unused argument 0 to keep API compatibility. |
*/ |
#define alternative_input(oldinstr, newinstr, feature, input...) \ |
asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ |
: : "i" (0), ## input) |
/* |
* This is similar to alternative_input. But it has two features and |
* respective instructions. |
* |
* If CPU has feature2, newinstr2 is used. |
* Otherwise, if CPU has feature1, newinstr1 is used. |
* Otherwise, oldinstr is used. |
*/ |
#define alternative_input_2(oldinstr, newinstr1, feature1, newinstr2, \ |
feature2, input...) \ |
asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, \ |
newinstr2, feature2) \ |
: : "i" (0), ## input) |
/* Like alternative_input, but with a single output argument */ |
#define alternative_io(oldinstr, newinstr, feature, output, input...) \ |
asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ |
: output : "i" (0), ## input) |
/* Like alternative_io, but for replacing a direct call with another one. */ |
#define alternative_call(oldfunc, newfunc, feature, output, input...) \ |
asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \ |
: output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input) |
/* |
* Like alternative_call, but there are two features and respective functions. |
* If CPU has feature2, function2 is used. |
* Otherwise, if CPU has feature1, function1 is used. |
* Otherwise, old function is used. |
*/ |
#define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2, \ |
output, input...) \ |
asm volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\ |
"call %P[new2]", feature2) \ |
: output : [old] "i" (oldfunc), [new1] "i" (newfunc1), \ |
[new2] "i" (newfunc2), ## input) |
/* |
* use this macro(s) if you need more than one output parameter |
* in alternative_io |
*/ |
#define ASM_OUTPUT2(a...) a |
/* |
* use this macro if you need clobbers but no inputs in |
* alternative_{input,io,call}() |
*/ |
#define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr |
struct paravirt_patch_site; |
#ifdef CONFIG_PARAVIRT |
void apply_paravirt(struct paravirt_patch_site *start, |
struct paravirt_patch_site *end); |
#else |
static inline void apply_paravirt(struct paravirt_patch_site *start, |
struct paravirt_patch_site *end) |
{} |
#define __parainstructions NULL |
#define __parainstructions_end NULL |
#endif |
extern void *text_poke_early(void *addr, const void *opcode, size_t len); |
/* |
* Clear and restore the kernel write-protection flag on the local CPU. |
* Allows the kernel to edit read-only pages. |
* Side-effect: any interrupt handler running between save and restore will have |
* the ability to write to read-only pages. |
* |
* Warning: |
* Code patching in the UP case is safe if NMIs and MCE handlers are stopped and |
* no thread can be preempted in the instructions being modified (no iret to an |
* invalid instruction possible) or if the instructions are changed from a |
* consistent state to another consistent state atomically. |
* On the local CPU you need to be protected again NMI or MCE handlers seeing an |
* inconsistent instruction while you patch. |
*/ |
extern void *text_poke(void *addr, const void *opcode, size_t len); |
extern void *text_poke_bp(void *addr, const void *opcode, size_t len, void *handler); |
#endif /* _ASM_X86_ALTERNATIVE_H */ |
/drivers/include/asm/arch_hweight.h |
---|
0,0 → 1,61 |
#ifndef _ASM_X86_HWEIGHT_H |
#define _ASM_X86_HWEIGHT_H |
#ifdef CONFIG_64BIT |
/* popcnt %edi, %eax -- redundant REX prefix for alignment */ |
#define POPCNT32 ".byte 0xf3,0x40,0x0f,0xb8,0xc7" |
/* popcnt %rdi, %rax */ |
#define POPCNT64 ".byte 0xf3,0x48,0x0f,0xb8,0xc7" |
#define REG_IN "D" |
#define REG_OUT "a" |
#else |
/* popcnt %eax, %eax */ |
#define POPCNT32 ".byte 0xf3,0x0f,0xb8,0xc0" |
#define REG_IN "a" |
#define REG_OUT "a" |
#endif |
/* |
* __sw_hweightXX are called from within the alternatives below |
* and callee-clobbered registers need to be taken care of. See |
* ARCH_HWEIGHT_CFLAGS in <arch/x86/Kconfig> for the respective |
* compiler switches. |
*/ |
static inline unsigned int __arch_hweight32(unsigned int w) |
{ |
unsigned int res = 0; |
asm ("call __sw_hweight32" |
: "="REG_OUT (res) |
: REG_IN (w)); |
return res; |
} |
static inline unsigned int __arch_hweight16(unsigned int w) |
{ |
return __arch_hweight32(w & 0xffff); |
} |
static inline unsigned int __arch_hweight8(unsigned int w) |
{ |
return __arch_hweight32(w & 0xff); |
} |
static inline unsigned long __arch_hweight64(__u64 w) |
{ |
unsigned long res = 0; |
#ifdef CONFIG_X86_32 |
return __arch_hweight32((u32)w) + |
__arch_hweight32((u32)(w >> 32)); |
#else |
asm (ALTERNATIVE("call __sw_hweight64", POPCNT64, X86_FEATURE_POPCNT) |
: "="REG_OUT (res) |
: REG_IN (w)); |
#endif /* CONFIG_X86_32 */ |
return res; |
} |
#endif |
/drivers/include/asm/asm.h |
---|
0,0 → 1,83 |
#ifndef _ASM_X86_ASM_H |
#define _ASM_X86_ASM_H |
#ifdef __ASSEMBLY__ |
# define __ASM_FORM(x) x |
# define __ASM_FORM_RAW(x) x |
# define __ASM_FORM_COMMA(x) x, |
#else |
# define __ASM_FORM(x) " " #x " " |
# define __ASM_FORM_RAW(x) #x |
# define __ASM_FORM_COMMA(x) " " #x "," |
#endif |
#ifdef CONFIG_X86_32 |
# define __ASM_SEL(a,b) __ASM_FORM(a) |
# define __ASM_SEL_RAW(a,b) __ASM_FORM_RAW(a) |
#else |
# define __ASM_SEL(a,b) __ASM_FORM(b) |
# define __ASM_SEL_RAW(a,b) __ASM_FORM_RAW(b) |
#endif |
#define __ASM_SIZE(inst, ...) __ASM_SEL(inst##l##__VA_ARGS__, \ |
inst##q##__VA_ARGS__) |
#define __ASM_REG(reg) __ASM_SEL_RAW(e##reg, r##reg) |
#define _ASM_PTR __ASM_SEL(.long, .quad) |
#define _ASM_ALIGN __ASM_SEL(.balign 4, .balign 8) |
#define _ASM_MOV __ASM_SIZE(mov) |
#define _ASM_INC __ASM_SIZE(inc) |
#define _ASM_DEC __ASM_SIZE(dec) |
#define _ASM_ADD __ASM_SIZE(add) |
#define _ASM_SUB __ASM_SIZE(sub) |
#define _ASM_XADD __ASM_SIZE(xadd) |
#define _ASM_AX __ASM_REG(ax) |
#define _ASM_BX __ASM_REG(bx) |
#define _ASM_CX __ASM_REG(cx) |
#define _ASM_DX __ASM_REG(dx) |
#define _ASM_SP __ASM_REG(sp) |
#define _ASM_BP __ASM_REG(bp) |
#define _ASM_SI __ASM_REG(si) |
#define _ASM_DI __ASM_REG(di) |
/* Exception table entry */ |
#ifdef __ASSEMBLY__ |
# define _ASM_EXTABLE(from,to) \ |
.pushsection "__ex_table","a" ; \ |
.balign 8 ; \ |
.long (from) - . ; \ |
.long (to) - . ; \ |
.popsection |
# define _ASM_EXTABLE_EX(from,to) \ |
.pushsection "__ex_table","a" ; \ |
.balign 8 ; \ |
.long (from) - . ; \ |
.long (to) - . + 0x7ffffff0 ; \ |
.popsection |
# define _ASM_NOKPROBE(entry) \ |
.pushsection "_kprobe_blacklist","aw" ; \ |
_ASM_ALIGN ; \ |
_ASM_PTR (entry); \ |
.popsection |
#else |
# define _ASM_EXTABLE(from,to) \ |
" .pushsection \"__ex_table\",\"a\"\n" \ |
" .balign 8\n" \ |
" .long (" #from ") - .\n" \ |
" .long (" #to ") - .\n" \ |
" .popsection\n" |
# define _ASM_EXTABLE_EX(from,to) \ |
" .pushsection \"__ex_table\",\"a\"\n" \ |
" .balign 8\n" \ |
" .long (" #from ") - .\n" \ |
" .long (" #to ") - . + 0x7ffffff0\n" \ |
" .popsection\n" |
/* For C file, we already have NOKPROBE_SYMBOL macro */ |
#endif |
#endif /* _ASM_X86_ASM_H */ |
/drivers/include/asm/atomic.h |
---|
0,0 → 1,238 |
#ifndef _ASM_X86_ATOMIC_H |
#define _ASM_X86_ATOMIC_H |
#include <linux/compiler.h> |
#include <linux/types.h> |
#include <asm/processor.h> |
#include <asm/alternative.h> |
#include <asm/cmpxchg.h> |
#include <asm/rmwcc.h> |
#include <asm/barrier.h> |
/* |
* Atomic operations that C can't guarantee us. Useful for |
* resource counting etc.. |
*/ |
#define ATOMIC_INIT(i) { (i) } |
/** |
* atomic_read - read atomic variable |
* @v: pointer of type atomic_t |
* |
* Atomically reads the value of @v. |
*/ |
static inline int atomic_read(const atomic_t *v) |
{ |
return ACCESS_ONCE((v)->counter); |
} |
/** |
* atomic_set - set atomic variable |
* @v: pointer of type atomic_t |
* @i: required value |
* |
* Atomically sets the value of @v to @i. |
*/ |
static inline void atomic_set(atomic_t *v, int i) |
{ |
v->counter = i; |
} |
/** |
* atomic_add - add integer to atomic variable |
* @i: integer value to add |
* @v: pointer of type atomic_t |
* |
* Atomically adds @i to @v. |
*/ |
static inline void atomic_add(int i, atomic_t *v) |
{ |
asm volatile(LOCK_PREFIX "addl %1,%0" |
: "+m" (v->counter) |
: "ir" (i)); |
} |
/** |
* atomic_sub - subtract integer from atomic variable |
* @i: integer value to subtract |
* @v: pointer of type atomic_t |
* |
* Atomically subtracts @i from @v. |
*/ |
static inline void atomic_sub(int i, atomic_t *v) |
{ |
asm volatile(LOCK_PREFIX "subl %1,%0" |
: "+m" (v->counter) |
: "ir" (i)); |
} |
/** |
* atomic_sub_and_test - subtract value from variable and test result |
* @i: integer value to subtract |
* @v: pointer of type atomic_t |
* |
* Atomically subtracts @i from @v and returns |
* true if the result is zero, or false for all |
* other cases. |
*/ |
static inline int atomic_sub_and_test(int i, atomic_t *v) |
{ |
GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, "er", i, "%0", "e"); |
} |
/** |
* atomic_inc - increment atomic variable |
* @v: pointer of type atomic_t |
* |
* Atomically increments @v by 1. |
*/ |
static inline void atomic_inc(atomic_t *v) |
{ |
asm volatile(LOCK_PREFIX "incl %0" |
: "+m" (v->counter)); |
} |
/** |
* atomic_dec - decrement atomic variable |
* @v: pointer of type atomic_t |
* |
* Atomically decrements @v by 1. |
*/ |
static inline void atomic_dec(atomic_t *v) |
{ |
asm volatile(LOCK_PREFIX "decl %0" |
: "+m" (v->counter)); |
} |
/** |
* atomic_dec_and_test - decrement and test |
* @v: pointer of type atomic_t |
* |
* Atomically decrements @v by 1 and |
* returns true if the result is 0, or false for all other |
* cases. |
*/ |
static inline int atomic_dec_and_test(atomic_t *v) |
{ |
GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, "%0", "e"); |
} |
/** |
* atomic_inc_and_test - increment and test |
* @v: pointer of type atomic_t |
* |
* Atomically increments @v by 1 |
* and returns true if the result is zero, or false for all |
* other cases. |
*/ |
static inline int atomic_inc_and_test(atomic_t *v) |
{ |
GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, "%0", "e"); |
} |
/** |
* atomic_add_negative - add and test if negative |
* @i: integer value to add |
* @v: pointer of type atomic_t |
* |
* Atomically adds @i to @v and returns true |
* if the result is negative, or false when |
* result is greater than or equal to zero. |
*/ |
static inline int atomic_add_negative(int i, atomic_t *v) |
{ |
GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, "er", i, "%0", "s"); |
} |
/** |
* atomic_add_return - add integer and return |
* @i: integer value to add |
* @v: pointer of type atomic_t |
* |
* Atomically adds @i to @v and returns @i + @v |
*/ |
static inline int atomic_add_return(int i, atomic_t *v) |
{ |
return i + xadd(&v->counter, i); |
} |
/** |
* atomic_sub_return - subtract integer and return |
* @v: pointer of type atomic_t |
* @i: integer value to subtract |
* |
* Atomically subtracts @i from @v and returns @v - @i |
*/ |
static inline int atomic_sub_return(int i, atomic_t *v) |
{ |
return atomic_add_return(-i, v); |
} |
#define atomic_inc_return(v) (atomic_add_return(1, v)) |
#define atomic_dec_return(v) (atomic_sub_return(1, v)) |
static inline int atomic_cmpxchg(atomic_t *v, int old, int new) |
{ |
return cmpxchg(&v->counter, old, new); |
} |
static inline int atomic_xchg(atomic_t *v, int new) |
{ |
return xchg(&v->counter, new); |
} |
/** |
* __atomic_add_unless - add unless the number is already a given value |
* @v: pointer of type atomic_t |
* @a: the amount to add to v... |
* @u: ...unless v is equal to u. |
* |
* Atomically adds @a to @v, so long as @v was not already @u. |
* Returns the old value of @v. |
*/ |
static inline int __atomic_add_unless(atomic_t *v, int a, int u) |
{ |
int c, old; |
c = atomic_read(v); |
for (;;) { |
if (unlikely(c == (u))) |
break; |
old = atomic_cmpxchg((v), c, c + (a)); |
if (likely(old == c)) |
break; |
c = old; |
} |
return c; |
} |
/** |
* atomic_inc_short - increment of a short integer |
* @v: pointer to type int |
* |
* Atomically adds 1 to @v |
* Returns the new value of @u |
*/ |
static inline short int atomic_inc_short(short int *v) |
{ |
asm(LOCK_PREFIX "addw $1, %0" : "+m" (*v)); |
return *v; |
} |
/* These are x86-specific, used by some header files */ |
#define atomic_clear_mask(mask, addr) \ |
asm volatile(LOCK_PREFIX "andl %0,%1" \ |
: : "r" (~(mask)), "m" (*(addr)) : "memory") |
#define atomic_set_mask(mask, addr) \ |
asm volatile(LOCK_PREFIX "orl %0,%1" \ |
: : "r" ((unsigned)(mask)), "m" (*(addr)) \ |
: "memory") |
#ifdef CONFIG_X86_32 |
# include <asm/atomic64_32.h> |
#else |
# include <asm/atomic64_64.h> |
#endif |
#endif /* _ASM_X86_ATOMIC_H */ |
/drivers/include/asm/atomic64_32.h |
---|
0,0 → 1,315 |
#ifndef _ASM_X86_ATOMIC64_32_H |
#define _ASM_X86_ATOMIC64_32_H |
#include <linux/compiler.h> |
#include <linux/types.h> |
#include <asm/processor.h> |
//#include <asm/cmpxchg.h> |
/* An 64bit atomic type */ |
typedef struct { |
u64 __aligned(8) counter; |
} atomic64_t; |
#define ATOMIC64_INIT(val) { (val) } |
#define __ATOMIC64_DECL(sym) void atomic64_##sym(atomic64_t *, ...) |
#ifndef ATOMIC64_EXPORT |
#define ATOMIC64_DECL_ONE __ATOMIC64_DECL |
#else |
#define ATOMIC64_DECL_ONE(sym) __ATOMIC64_DECL(sym); \ |
ATOMIC64_EXPORT(atomic64_##sym) |
#endif |
#ifdef CONFIG_X86_CMPXCHG64 |
#define __alternative_atomic64(f, g, out, in...) \ |
asm volatile("call %P[func]" \ |
: out : [func] "i" (atomic64_##g##_cx8), ## in) |
#define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8) |
#else |
#define __alternative_atomic64(f, g, out, in...) \ |
alternative_call(atomic64_##f##_386, atomic64_##g##_cx8, \ |
X86_FEATURE_CX8, ASM_OUTPUT2(out), ## in) |
#define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8); \ |
ATOMIC64_DECL_ONE(sym##_386) |
ATOMIC64_DECL_ONE(add_386); |
ATOMIC64_DECL_ONE(sub_386); |
ATOMIC64_DECL_ONE(inc_386); |
ATOMIC64_DECL_ONE(dec_386); |
#endif |
#define alternative_atomic64(f, out, in...) \ |
__alternative_atomic64(f, f, ASM_OUTPUT2(out), ## in) |
ATOMIC64_DECL(read); |
ATOMIC64_DECL(set); |
ATOMIC64_DECL(xchg); |
ATOMIC64_DECL(add_return); |
ATOMIC64_DECL(sub_return); |
ATOMIC64_DECL(inc_return); |
ATOMIC64_DECL(dec_return); |
ATOMIC64_DECL(dec_if_positive); |
ATOMIC64_DECL(inc_not_zero); |
ATOMIC64_DECL(add_unless); |
#undef ATOMIC64_DECL |
#undef ATOMIC64_DECL_ONE |
#undef __ATOMIC64_DECL |
#undef ATOMIC64_EXPORT |
/** |
* atomic64_cmpxchg - cmpxchg atomic64 variable |
* @v: pointer to type atomic64_t |
* @o: expected value |
* @n: new value |
* |
* Atomically sets @v to @n if it was equal to @o and returns |
* the old value. |
*/ |
static inline long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n) |
{ |
return cmpxchg64(&v->counter, o, n); |
} |
/** |
* atomic64_xchg - xchg atomic64 variable |
* @v: pointer to type atomic64_t |
* @n: value to assign |
* |
* Atomically xchgs the value of @v to @n and returns |
* the old value. |
*/ |
static inline long long atomic64_xchg(atomic64_t *v, long long n) |
{ |
long long o; |
unsigned high = (unsigned)(n >> 32); |
unsigned low = (unsigned)n; |
asm volatile( |
"1: \n\t" |
"cmpxchg8b (%%esi) \n\t" |
"jnz 1b \n\t" |
:"=&A" (o) |
:"S" (v), "b" (low), "c" (high) |
: "memory", "cc"); |
return o; |
} |
/** |
* atomic64_set - set atomic64 variable |
* @v: pointer to type atomic64_t |
* @i: value to assign |
* |
* Atomically sets the value of @v to @n. |
*/ |
static inline void atomic64_set(atomic64_t *v, long long i) |
{ |
__sync_lock_test_and_set((long long *)&v->counter, i); |
} |
/** |
* atomic64_read - read atomic64 variable |
* @v: pointer to type atomic64_t |
* |
* Atomically reads the value of @v and returns it. |
*/ |
static inline long long atomic64_read(const atomic64_t *v) |
{ |
return __sync_fetch_and_add( (long long *)&v->counter, 0); |
} |
/** |
* atomic64_add_return - add and return |
* @i: integer value to add |
* @v: pointer to type atomic64_t |
* |
* Atomically adds @i to @v and returns @i + *@v |
*/ |
static inline long long atomic64_add_return(long long i, atomic64_t *v) |
{ |
alternative_atomic64(add_return, |
ASM_OUTPUT2("+A" (i), "+c" (v)), |
ASM_NO_INPUT_CLOBBER("memory")); |
return i; |
} |
/* |
* Other variants with different arithmetic operators: |
*/ |
static inline long long atomic64_sub_return(long long i, atomic64_t *v) |
{ |
alternative_atomic64(sub_return, |
ASM_OUTPUT2("+A" (i), "+c" (v)), |
ASM_NO_INPUT_CLOBBER("memory")); |
return i; |
} |
static inline long long atomic64_inc_return(atomic64_t *v) |
{ |
long long a; |
alternative_atomic64(inc_return, "=&A" (a), |
"S" (v) : "memory", "ecx"); |
return a; |
} |
static inline long long atomic64_dec_return(atomic64_t *v) |
{ |
long long a; |
alternative_atomic64(dec_return, "=&A" (a), |
"S" (v) : "memory", "ecx"); |
return a; |
} |
/** |
* atomic64_add - add integer to atomic64 variable |
* @i: integer value to add |
* @v: pointer to type atomic64_t |
* |
* Atomically adds @i to @v. |
*/ |
static inline long long atomic64_add(long long i, atomic64_t *v) |
{ |
__alternative_atomic64(add, add_return, |
ASM_OUTPUT2("+A" (i), "+c" (v)), |
ASM_NO_INPUT_CLOBBER("memory")); |
return i; |
} |
/** |
* atomic64_sub - subtract the atomic64 variable |
* @i: integer value to subtract |
* @v: pointer to type atomic64_t |
* |
* Atomically subtracts @i from @v. |
*/ |
static inline long long atomic64_sub(long long i, atomic64_t *v) |
{ |
__alternative_atomic64(sub, sub_return, |
ASM_OUTPUT2("+A" (i), "+c" (v)), |
ASM_NO_INPUT_CLOBBER("memory")); |
return i; |
} |
/** |
* atomic64_sub_and_test - subtract value from variable and test result |
* @i: integer value to subtract |
* @v: pointer to type atomic64_t |
* |
* Atomically subtracts @i from @v and returns |
* true if the result is zero, or false for all |
* other cases. |
*/ |
static inline int atomic64_sub_and_test(long long i, atomic64_t *v) |
{ |
return atomic64_sub_return(i, v) == 0; |
} |
/** |
* atomic64_inc - increment atomic64 variable |
* @v: pointer to type atomic64_t |
* |
* Atomically increments @v by 1. |
*/ |
static inline void atomic64_inc(atomic64_t *v) |
{ |
__alternative_atomic64(inc, inc_return, /* no output */, |
"S" (v) : "memory", "eax", "ecx", "edx"); |
} |
/** |
* atomic64_dec - decrement atomic64 variable |
* @v: pointer to type atomic64_t |
* |
* Atomically decrements @v by 1. |
*/ |
static inline void atomic64_dec(atomic64_t *v) |
{ |
__alternative_atomic64(dec, dec_return, /* no output */, |
"S" (v) : "memory", "eax", "ecx", "edx"); |
} |
/** |
* atomic64_dec_and_test - decrement and test |
* @v: pointer to type atomic64_t |
* |
* Atomically decrements @v by 1 and |
* returns true if the result is 0, or false for all other |
* cases. |
*/ |
static inline int atomic64_dec_and_test(atomic64_t *v) |
{ |
return atomic64_dec_return(v) == 0; |
} |
/** |
* atomic64_inc_and_test - increment and test |
* @v: pointer to type atomic64_t |
* |
* Atomically increments @v by 1 |
* and returns true if the result is zero, or false for all |
* other cases. |
*/ |
static inline int atomic64_inc_and_test(atomic64_t *v) |
{ |
return atomic64_inc_return(v) == 0; |
} |
/** |
* atomic64_add_negative - add and test if negative |
* @i: integer value to add |
* @v: pointer to type atomic64_t |
* |
* Atomically adds @i to @v and returns true |
* if the result is negative, or false when |
* result is greater than or equal to zero. |
*/ |
static inline int atomic64_add_negative(long long i, atomic64_t *v) |
{ |
return atomic64_add_return(i, v) < 0; |
} |
/** |
* atomic64_add_unless - add unless the number is a given value |
* @v: pointer of type atomic64_t |
* @a: the amount to add to v... |
* @u: ...unless v is equal to u. |
* |
* Atomically adds @a to @v, so long as it was not @u. |
* Returns non-zero if the add was done, zero otherwise. |
*/ |
static inline int atomic64_add_unless(atomic64_t *v, long long a, long long u) |
{ |
unsigned low = (unsigned)u; |
unsigned high = (unsigned)(u >> 32); |
alternative_atomic64(add_unless, |
ASM_OUTPUT2("+A" (a), "+c" (low), "+D" (high)), |
"S" (v) : "memory"); |
return (int)a; |
} |
static inline int atomic64_inc_not_zero(atomic64_t *v) |
{ |
int r; |
alternative_atomic64(inc_not_zero, "=&a" (r), |
"S" (v) : "ecx", "edx", "memory"); |
return r; |
} |
static inline long long atomic64_dec_if_positive(atomic64_t *v) |
{ |
long long r; |
alternative_atomic64(dec_if_positive, "=&A" (r), |
"S" (v) : "ecx", "memory"); |
return r; |
} |
#undef alternative_atomic64 |
#undef __alternative_atomic64 |
#endif /* _ASM_X86_ATOMIC64_32_H */ |
/drivers/include/asm/atomic_32.h |
---|
0,0 → 1,441 |
#ifndef _ASM_X86_ATOMIC_32_H |
#define _ASM_X86_ATOMIC_32_H |
#include <linux/compiler.h> |
#include <linux/types.h> |
#include <asm/processor.h> |
#include <asm/cmpxchg.h> |
/* |
* Atomic operations that C can't guarantee us. Useful for |
* resource counting etc.. |
*/ |
#define ATOMIC_INIT(i) { (i) } |
/** |
* atomic_read - read atomic variable |
* @v: pointer of type atomic_t |
* |
* Atomically reads the value of @v. |
*/ |
static inline int atomic_read(const atomic_t *v) |
{ |
return v->counter; |
} |
/** |
* atomic_set - set atomic variable |
* @v: pointer of type atomic_t |
* @i: required value |
* |
* Atomically sets the value of @v to @i. |
*/ |
static inline void atomic_set(atomic_t *v, int i) |
{ |
v->counter = i; |
} |
/** |
* atomic_add - add integer to atomic variable |
* @i: integer value to add |
* @v: pointer of type atomic_t |
* |
* Atomically adds @i to @v. |
*/ |
static inline void atomic_add(int i, atomic_t *v) |
{ |
asm volatile(LOCK_PREFIX "addl %1,%0" |
: "+m" (v->counter) |
: "ir" (i)); |
} |
/** |
* atomic_sub - subtract integer from atomic variable |
* @i: integer value to subtract |
* @v: pointer of type atomic_t |
* |
* Atomically subtracts @i from @v. |
*/ |
static inline void atomic_sub(int i, atomic_t *v) |
{ |
asm volatile(LOCK_PREFIX "subl %1,%0" |
: "+m" (v->counter) |
: "ir" (i)); |
} |
/** |
* atomic_sub_and_test - subtract value from variable and test result |
* @i: integer value to subtract |
* @v: pointer of type atomic_t |
* |
* Atomically subtracts @i from @v and returns |
* true if the result is zero, or false for all |
* other cases. |
*/ |
static inline int atomic_sub_and_test(int i, atomic_t *v) |
{ |
unsigned char c; |
asm volatile(LOCK_PREFIX "subl %2,%0; sete %1" |
: "+m" (v->counter), "=qm" (c) |
: "ir" (i) : "memory"); |
return c; |
} |
/** |
* atomic_inc - increment atomic variable |
* @v: pointer of type atomic_t |
* |
* Atomically increments @v by 1. |
*/ |
static inline void atomic_inc(atomic_t *v) |
{ |
asm volatile(LOCK_PREFIX "incl %0" |
: "+m" (v->counter)); |
} |
/** |
* atomic_dec - decrement atomic variable |
* @v: pointer of type atomic_t |
* |
* Atomically decrements @v by 1. |
*/ |
static inline void atomic_dec(atomic_t *v) |
{ |
asm volatile(LOCK_PREFIX "decl %0" |
: "+m" (v->counter)); |
} |
/** |
* atomic_dec_and_test - decrement and test |
* @v: pointer of type atomic_t |
* |
* Atomically decrements @v by 1 and |
* returns true if the result is 0, or false for all other |
* cases. |
*/ |
static inline int atomic_dec_and_test(atomic_t *v) |
{ |
unsigned char c; |
asm volatile(LOCK_PREFIX "decl %0; sete %1" |
: "+m" (v->counter), "=qm" (c) |
: : "memory"); |
return c != 0; |
} |
/** |
* atomic_inc_and_test - increment and test |
* @v: pointer of type atomic_t |
* |
* Atomically increments @v by 1 |
* and returns true if the result is zero, or false for all |
* other cases. |
*/ |
static inline int atomic_inc_and_test(atomic_t *v) |
{ |
unsigned char c; |
asm volatile(LOCK_PREFIX "incl %0; sete %1" |
: "+m" (v->counter), "=qm" (c) |
: : "memory"); |
return c != 0; |
} |
/** |
* atomic_add_negative - add and test if negative |
* @v: pointer of type atomic_t |
* @i: integer value to add |
* |
* Atomically adds @i to @v and returns true |
* if the result is negative, or false when |
* result is greater than or equal to zero. |
*/ |
static inline int atomic_add_negative(int i, atomic_t *v) |
{ |
unsigned char c; |
asm volatile(LOCK_PREFIX "addl %2,%0; sets %1" |
: "+m" (v->counter), "=qm" (c) |
: "ir" (i) : "memory"); |
return c; |
} |
/** |
* atomic_add_return - add integer and return |
* @v: pointer of type atomic_t |
* @i: integer value to add |
* |
* Atomically adds @i to @v and returns @i + @v |
*/ |
static inline int atomic_add_return(int i, atomic_t *v) |
{ |
int __i; |
#ifdef CONFIG_M386 |
unsigned long flags; |
if (unlikely(boot_cpu_data.x86 <= 3)) |
goto no_xadd; |
#endif |
/* Modern 486+ processor */ |
__i = i; |
asm volatile(LOCK_PREFIX "xaddl %0, %1" |
: "+r" (i), "+m" (v->counter) |
: : "memory"); |
return i + __i; |
#ifdef CONFIG_M386 |
no_xadd: /* Legacy 386 processor */ |
local_irq_save(flags); |
__i = atomic_read(v); |
atomic_set(v, i + __i); |
local_irq_restore(flags); |
return i + __i; |
#endif |
} |
/** |
* atomic_sub_return - subtract integer and return |
* @v: pointer of type atomic_t |
* @i: integer value to subtract |
* |
* Atomically subtracts @i from @v and returns @v - @i |
*/ |
static inline int atomic_sub_return(int i, atomic_t *v) |
{ |
return atomic_add_return(-i, v); |
} |
static inline int atomic_cmpxchg(atomic_t *v, int old, int new) |
{ |
return cmpxchg(&v->counter, old, new); |
} |
static inline int atomic_xchg(atomic_t *v, int new) |
{ |
return xchg(&v->counter, new); |
} |
/** |
* atomic_add_unless - add unless the number is already a given value |
* @v: pointer of type atomic_t |
* @a: the amount to add to v... |
* @u: ...unless v is equal to u. |
* |
* Atomically adds @a to @v, so long as @v was not already @u. |
* Returns non-zero if @v was not @u, and zero otherwise. |
*/ |
static inline int atomic_add_unless(atomic_t *v, int a, int u) |
{ |
int c, old; |
c = atomic_read(v); |
for (;;) { |
if (unlikely(c == (u))) |
break; |
old = atomic_cmpxchg((v), c, c + (a)); |
if (likely(old == c)) |
break; |
c = old; |
} |
return c != (u); |
} |
#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) |
#define atomic_inc_return(v) (atomic_add_return(1, v)) |
#define atomic_dec_return(v) (atomic_sub_return(1, v)) |
/* These are x86-specific, used by some header files */ |
#define atomic_clear_mask(mask, addr) \ |
asm volatile(LOCK_PREFIX "andl %0,%1" \ |
: : "r" (~(mask)), "m" (*(addr)) : "memory") |
#define atomic_set_mask(mask, addr) \ |
asm volatile(LOCK_PREFIX "orl %0,%1" \ |
: : "r" (mask), "m" (*(addr)) : "memory") |
/* Atomic operations are already serializing on x86 */ |
#define smp_mb__before_atomic_dec() barrier() |
#define smp_mb__after_atomic_dec() barrier() |
#define smp_mb__before_atomic_inc() barrier() |
#define smp_mb__after_atomic_inc() barrier() |
/* An 64bit atomic type */ |
typedef struct { |
u64 __aligned(8) counter; |
} atomic64_t; |
extern u64 atomic64_cmpxchg(atomic64_t *ptr, u64 old_val, u64 new_val); |
/** |
* atomic64_xchg - xchg atomic64 variable |
* @ptr: pointer to type atomic64_t |
* @new_val: value to assign |
* |
* Atomically xchgs the value of @ptr to @new_val and returns |
* the old value. |
*/ |
static inline long long atomic64_xchg(atomic64_t *v, long long n) |
{ |
long long o; |
unsigned high = (unsigned)(n >> 32); |
unsigned low = (unsigned)n; |
asm volatile( |
"1: \n\t" |
"cmpxchg8b (%%esi) \n\t" |
"jnz 1b \n\t" |
:"=&A" (o) |
:"S" (v), "b" (low), "c" (high) |
: "memory", "cc"); |
return o; |
} |
/** |
* atomic64_set - set atomic64 variable |
* @ptr: pointer to type atomic64_t |
* @new_val: value to assign |
* |
* Atomically sets the value of @ptr to @new_val. |
*/ |
static inline void atomic64_set(atomic64_t *v, long long i) |
{ |
unsigned high = (unsigned)(i >> 32); |
unsigned low = (unsigned)i; |
asm volatile ( |
"1: \n\t" |
"cmpxchg8b (%%esi) \n\t" |
"jnz 1b \n\t" |
: |
:"S" (v), "b" (low), "c" (high) |
: "eax", "edx", "memory", "cc"); |
} |
/** |
* atomic64_read - read atomic64 variable |
* @ptr: pointer to type atomic64_t |
* |
* Atomically reads the value of @ptr and returns it. |
*/ |
static inline u64 atomic64_read(atomic64_t *ptr) |
{ |
u64 res; |
/* |
* Note, we inline this atomic64_t primitive because |
* it only clobbers EAX/EDX and leaves the others |
* untouched. We also (somewhat subtly) rely on the |
* fact that cmpxchg8b returns the current 64-bit value |
* of the memory location we are touching: |
*/ |
asm volatile( |
"mov %%ebx, %%eax\n\t" |
"mov %%ecx, %%edx\n\t" |
LOCK_PREFIX "cmpxchg8b %1\n" |
: "=&A" (res) |
: "m" (*ptr) |
); |
return res; |
} |
/** |
* atomic64_add_return - add and return |
* @delta: integer value to add |
* @ptr: pointer to type atomic64_t |
* |
* Atomically adds @delta to @ptr and returns @delta + *@ptr |
*/ |
extern u64 atomic64_add_return(u64 delta, atomic64_t *ptr); |
/* |
* Other variants with different arithmetic operators: |
*/ |
extern u64 atomic64_sub_return(u64 delta, atomic64_t *ptr); |
extern u64 atomic64_inc_return(atomic64_t *ptr); |
extern u64 atomic64_dec_return(atomic64_t *ptr); |
/** |
* atomic64_add - add integer to atomic64 variable |
* @delta: integer value to add |
* @ptr: pointer to type atomic64_t |
* |
* Atomically adds @delta to @ptr. |
*/ |
extern void atomic64_add(u64 delta, atomic64_t *ptr); |
/** |
* atomic64_sub - subtract the atomic64 variable |
* @delta: integer value to subtract |
* @ptr: pointer to type atomic64_t |
* |
* Atomically subtracts @delta from @ptr. |
*/ |
extern void atomic64_sub(u64 delta, atomic64_t *ptr); |
/** |
* atomic64_sub_and_test - subtract value from variable and test result |
* @delta: integer value to subtract |
* @ptr: pointer to type atomic64_t |
* |
* Atomically subtracts @delta from @ptr and returns |
* true if the result is zero, or false for all |
* other cases. |
*/ |
extern int atomic64_sub_and_test(u64 delta, atomic64_t *ptr); |
/** |
* atomic64_inc - increment atomic64 variable |
* @ptr: pointer to type atomic64_t |
* |
* Atomically increments @ptr by 1. |
*/ |
extern void atomic64_inc(atomic64_t *ptr); |
/** |
* atomic64_dec - decrement atomic64 variable |
* @ptr: pointer to type atomic64_t |
* |
* Atomically decrements @ptr by 1. |
*/ |
extern void atomic64_dec(atomic64_t *ptr); |
/** |
* atomic64_dec_and_test - decrement and test |
* @ptr: pointer to type atomic64_t |
* |
* Atomically decrements @ptr by 1 and |
* returns true if the result is 0, or false for all other |
* cases. |
*/ |
extern int atomic64_dec_and_test(atomic64_t *ptr); |
/** |
* atomic64_inc_and_test - increment and test |
* @ptr: pointer to type atomic64_t |
* |
* Atomically increments @ptr by 1 |
* and returns true if the result is zero, or false for all |
* other cases. |
*/ |
extern int atomic64_inc_and_test(atomic64_t *ptr); |
/** |
* atomic64_add_negative - add and test if negative |
* @delta: integer value to add |
* @ptr: pointer to type atomic64_t |
* |
* Atomically adds @delta to @ptr and returns true |
* if the result is negative, or false when |
* result is greater than or equal to zero. |
*/ |
extern int atomic64_add_negative(u64 delta, atomic64_t *ptr); |
#include <asm-generic/atomic-long.h> |
#endif /* _ASM_X86_ATOMIC_32_H */ |
/drivers/include/asm/barrier.h |
---|
0,0 → 1,107 |
#ifndef _ASM_X86_BARRIER_H |
#define _ASM_X86_BARRIER_H |
#include <asm/alternative.h> |
#include <asm/nops.h> |
/* |
* Force strict CPU ordering. |
* And yes, this is required on UP too when we're talking |
* to devices. |
*/ |
#ifdef CONFIG_X86_32 |
/* |
* Some non-Intel clones support out of order store. wmb() ceases to be a |
* nop for these. |
*/ |
#define mb() asm volatile ("lock; addl $0,0(%esp)")/*, "mfence", X86_FEATURE_XMM2) */ |
#define rmb() asm volatile("lock; addl $0,0(%esp)")/*, "lfence", X86_FEATURE_XMM2) */ |
#define wmb() asm volatile("lock; addl $0,0(%esp)")/*, "sfence", X86_FEATURE_XMM) */ |
#else |
#define mb() asm volatile("mfence":::"memory") |
#define rmb() asm volatile("lfence":::"memory") |
#define wmb() asm volatile("sfence" ::: "memory") |
#endif |
#ifdef CONFIG_X86_PPRO_FENCE |
#define dma_rmb() rmb() |
#else |
#define dma_rmb() barrier() |
#endif |
#define dma_wmb() barrier() |
#ifdef CONFIG_SMP |
#define smp_mb() mb() |
#define smp_rmb() dma_rmb() |
#define smp_wmb() barrier() |
#define set_mb(var, value) do { (void)xchg(&var, value); } while (0) |
#else /* !SMP */ |
#define smp_mb() barrier() |
#define smp_rmb() barrier() |
#define smp_wmb() barrier() |
#define set_mb(var, value) do { var = value; barrier(); } while (0) |
#endif /* SMP */ |
#define read_barrier_depends() do { } while (0) |
#define smp_read_barrier_depends() do { } while (0) |
#if defined(CONFIG_X86_PPRO_FENCE) |
/* |
* For this option x86 doesn't have a strong TSO memory |
* model and we should fall back to full barriers. |
*/ |
#define smp_store_release(p, v) \ |
do { \ |
compiletime_assert_atomic_type(*p); \ |
smp_mb(); \ |
ACCESS_ONCE(*p) = (v); \ |
} while (0) |
#define smp_load_acquire(p) \ |
({ \ |
typeof(*p) ___p1 = ACCESS_ONCE(*p); \ |
compiletime_assert_atomic_type(*p); \ |
smp_mb(); \ |
___p1; \ |
}) |
#else /* regular x86 TSO memory ordering */ |
#define smp_store_release(p, v) \ |
do { \ |
compiletime_assert_atomic_type(*p); \ |
barrier(); \ |
ACCESS_ONCE(*p) = (v); \ |
} while (0) |
#define smp_load_acquire(p) \ |
({ \ |
typeof(*p) ___p1 = ACCESS_ONCE(*p); \ |
compiletime_assert_atomic_type(*p); \ |
barrier(); \ |
___p1; \ |
}) |
#endif |
/* Atomic operations are already serializing on x86 */ |
#define smp_mb__before_atomic() barrier() |
#define smp_mb__after_atomic() barrier() |
/* |
* Stop RDTSC speculation. This is needed when you need to use RDTSC |
* (or get_cycles or vread that possibly accesses the TSC) in a defined |
* code region. |
* |
* (Could use an alternative three way for this if there was one.) |
*/ |
static __always_inline void rdtsc_barrier(void) |
{ |
alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC); |
alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC); |
} |
#endif /* _ASM_X86_BARRIER_H */ |
/drivers/include/asm/bitops.h |
---|
0,0 → 1,509 |
#ifndef _ASM_X86_BITOPS_H |
#define _ASM_X86_BITOPS_H |
/* |
* Copyright 1992, Linus Torvalds. |
* |
* Note: inlines with more than a single statement should be marked |
* __always_inline to avoid problems with older gcc's inlining heuristics. |
*/ |
#ifndef _LINUX_BITOPS_H |
#error only <linux/bitops.h> can be included directly |
#endif |
#include <linux/compiler.h> |
#include <asm/alternative.h> |
#include <asm/rmwcc.h> |
#include <asm/barrier.h> |
#if BITS_PER_LONG == 32 |
# define _BITOPS_LONG_SHIFT 5 |
#elif BITS_PER_LONG == 64 |
# define _BITOPS_LONG_SHIFT 6 |
#else |
# error "Unexpected BITS_PER_LONG" |
#endif |
#define BIT_64(n) (U64_C(1) << (n)) |
/* |
* These have to be done with inline assembly: that way the bit-setting |
* is guaranteed to be atomic. All bit operations return 0 if the bit |
* was cleared before the operation and != 0 if it was not. |
* |
* bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1). |
*/ |
#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1) |
/* Technically wrong, but this avoids compilation errors on some gcc |
versions. */ |
#define BITOP_ADDR(x) "=m" (*(volatile long *) (x)) |
#else |
#define BITOP_ADDR(x) "+m" (*(volatile long *) (x)) |
#endif |
#define ADDR BITOP_ADDR(addr) |
/* |
* We do the locked ops that don't return the old value as |
* a mask operation on a byte. |
*/ |
#define IS_IMMEDIATE(nr) (__builtin_constant_p(nr)) |
#define CONST_MASK_ADDR(nr, addr) BITOP_ADDR((void *)(addr) + ((nr)>>3)) |
#define CONST_MASK(nr) (1 << ((nr) & 7)) |
/** |
* set_bit - Atomically set a bit in memory |
* @nr: the bit to set |
* @addr: the address to start counting from |
* |
* This function is atomic and may not be reordered. See __set_bit() |
* if you do not require the atomic guarantees. |
* |
* Note: there are no guarantees that this function will not be reordered |
* on non x86 architectures, so if you are writing portable code, |
* make sure not to rely on its reordering guarantees. |
* |
* Note that @nr may be almost arbitrarily large; this function is not |
* restricted to acting on a single-word quantity. |
*/ |
static __always_inline void |
set_bit(long nr, volatile unsigned long *addr) |
{ |
if (IS_IMMEDIATE(nr)) { |
asm volatile(LOCK_PREFIX "orb %1,%0" |
: CONST_MASK_ADDR(nr, addr) |
: "iq" ((u8)CONST_MASK(nr)) |
: "memory"); |
} else { |
asm volatile(LOCK_PREFIX "bts %1,%0" |
: BITOP_ADDR(addr) : "Ir" (nr) : "memory"); |
} |
} |
/** |
* __set_bit - Set a bit in memory |
* @nr: the bit to set |
* @addr: the address to start counting from |
* |
* Unlike set_bit(), this function is non-atomic and may be reordered. |
* If it's called on the same region of memory simultaneously, the effect |
* may be that only one operation succeeds. |
*/ |
static inline void __set_bit(long nr, volatile unsigned long *addr) |
{ |
asm volatile("bts %1,%0" : ADDR : "Ir" (nr) : "memory"); |
} |
/** |
* clear_bit - Clears a bit in memory |
* @nr: Bit to clear |
* @addr: Address to start counting from |
* |
* clear_bit() is atomic and may not be reordered. However, it does |
* not contain a memory barrier, so if it is used for locking purposes, |
* you should call smp_mb__before_atomic() and/or smp_mb__after_atomic() |
* in order to ensure changes are visible on other processors. |
*/ |
static __always_inline void |
clear_bit(long nr, volatile unsigned long *addr) |
{ |
if (IS_IMMEDIATE(nr)) { |
asm volatile(LOCK_PREFIX "andb %1,%0" |
: CONST_MASK_ADDR(nr, addr) |
: "iq" ((u8)~CONST_MASK(nr))); |
} else { |
asm volatile(LOCK_PREFIX "btr %1,%0" |
: BITOP_ADDR(addr) |
: "Ir" (nr)); |
} |
} |
/* |
* clear_bit_unlock - Clears a bit in memory |
* @nr: Bit to clear |
* @addr: Address to start counting from |
* |
* clear_bit() is atomic and implies release semantics before the memory |
* operation. It can be used for an unlock. |
*/ |
static inline void clear_bit_unlock(long nr, volatile unsigned long *addr) |
{ |
barrier(); |
clear_bit(nr, addr); |
} |
static inline void __clear_bit(long nr, volatile unsigned long *addr) |
{ |
asm volatile("btr %1,%0" : ADDR : "Ir" (nr)); |
} |
/* |
* __clear_bit_unlock - Clears a bit in memory |
* @nr: Bit to clear |
* @addr: Address to start counting from |
* |
* __clear_bit() is non-atomic and implies release semantics before the memory |
* operation. It can be used for an unlock if no other CPUs can concurrently |
* modify other bits in the word. |
* |
* No memory barrier is required here, because x86 cannot reorder stores past |
* older loads. Same principle as spin_unlock. |
*/ |
static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr) |
{ |
barrier(); |
__clear_bit(nr, addr); |
} |
/** |
* __change_bit - Toggle a bit in memory |
* @nr: the bit to change |
* @addr: the address to start counting from |
* |
* Unlike change_bit(), this function is non-atomic and may be reordered. |
* If it's called on the same region of memory simultaneously, the effect |
* may be that only one operation succeeds. |
*/ |
static inline void __change_bit(long nr, volatile unsigned long *addr) |
{ |
asm volatile("btc %1,%0" : ADDR : "Ir" (nr)); |
} |
/** |
* change_bit - Toggle a bit in memory |
* @nr: Bit to change |
* @addr: Address to start counting from |
* |
* change_bit() is atomic and may not be reordered. |
* Note that @nr may be almost arbitrarily large; this function is not |
* restricted to acting on a single-word quantity. |
*/ |
static inline void change_bit(long nr, volatile unsigned long *addr) |
{ |
if (IS_IMMEDIATE(nr)) { |
asm volatile(LOCK_PREFIX "xorb %1,%0" |
: CONST_MASK_ADDR(nr, addr) |
: "iq" ((u8)CONST_MASK(nr))); |
} else { |
asm volatile(LOCK_PREFIX "btc %1,%0" |
: BITOP_ADDR(addr) |
: "Ir" (nr)); |
} |
} |
/** |
* test_and_set_bit - Set a bit and return its old value |
* @nr: Bit to set |
* @addr: Address to count from |
* |
* This operation is atomic and cannot be reordered. |
* It also implies a memory barrier. |
*/ |
static inline int test_and_set_bit(long nr, volatile unsigned long *addr) |
{ |
GEN_BINARY_RMWcc(LOCK_PREFIX "bts", *addr, "Ir", nr, "%0", "c"); |
} |
/** |
* test_and_set_bit_lock - Set a bit and return its old value for lock |
* @nr: Bit to set |
* @addr: Address to count from |
* |
* This is the same as test_and_set_bit on x86. |
*/ |
static __always_inline int |
test_and_set_bit_lock(long nr, volatile unsigned long *addr) |
{ |
return test_and_set_bit(nr, addr); |
} |
/** |
* __test_and_set_bit - Set a bit and return its old value |
* @nr: Bit to set |
* @addr: Address to count from |
* |
* This operation is non-atomic and can be reordered. |
* If two examples of this operation race, one can appear to succeed |
* but actually fail. You must protect multiple accesses with a lock. |
*/ |
static inline int __test_and_set_bit(long nr, volatile unsigned long *addr) |
{ |
int oldbit; |
asm("bts %2,%1\n\t" |
"sbb %0,%0" |
: "=r" (oldbit), ADDR |
: "Ir" (nr)); |
return oldbit; |
} |
/** |
* test_and_clear_bit - Clear a bit and return its old value |
* @nr: Bit to clear |
* @addr: Address to count from |
* |
* This operation is atomic and cannot be reordered. |
* It also implies a memory barrier. |
*/ |
static inline int test_and_clear_bit(long nr, volatile unsigned long *addr) |
{ |
GEN_BINARY_RMWcc(LOCK_PREFIX "btr", *addr, "Ir", nr, "%0", "c"); |
} |
/** |
* __test_and_clear_bit - Clear a bit and return its old value |
* @nr: Bit to clear |
* @addr: Address to count from |
* |
* This operation is non-atomic and can be reordered. |
* If two examples of this operation race, one can appear to succeed |
* but actually fail. You must protect multiple accesses with a lock. |
* |
* Note: the operation is performed atomically with respect to |
* the local CPU, but not other CPUs. Portable code should not |
* rely on this behaviour. |
* KVM relies on this behaviour on x86 for modifying memory that is also |
* accessed from a hypervisor on the same CPU if running in a VM: don't change |
* this without also updating arch/x86/kernel/kvm.c |
*/ |
static inline int __test_and_clear_bit(long nr, volatile unsigned long *addr) |
{ |
int oldbit; |
asm volatile("btr %2,%1\n\t" |
"sbb %0,%0" |
: "=r" (oldbit), ADDR |
: "Ir" (nr)); |
return oldbit; |
} |
/* WARNING: non atomic and it can be reordered! */ |
static inline int __test_and_change_bit(long nr, volatile unsigned long *addr) |
{ |
int oldbit; |
asm volatile("btc %2,%1\n\t" |
"sbb %0,%0" |
: "=r" (oldbit), ADDR |
: "Ir" (nr) : "memory"); |
return oldbit; |
} |
/** |
* test_and_change_bit - Change a bit and return its old value |
* @nr: Bit to change |
* @addr: Address to count from |
* |
* This operation is atomic and cannot be reordered. |
* It also implies a memory barrier. |
*/ |
static inline int test_and_change_bit(long nr, volatile unsigned long *addr) |
{ |
GEN_BINARY_RMWcc(LOCK_PREFIX "btc", *addr, "Ir", nr, "%0", "c"); |
} |
static __always_inline int constant_test_bit(long nr, const volatile unsigned long *addr) |
{ |
return ((1UL << (nr & (BITS_PER_LONG-1))) & |
(addr[nr >> _BITOPS_LONG_SHIFT])) != 0; |
} |
static inline int variable_test_bit(long nr, volatile const unsigned long *addr) |
{ |
int oldbit; |
asm volatile("bt %2,%1\n\t" |
"sbb %0,%0" |
: "=r" (oldbit) |
: "m" (*(unsigned long *)addr), "Ir" (nr)); |
return oldbit; |
} |
#if 0 /* Fool kernel-doc since it doesn't do macros yet */ |
/** |
* test_bit - Determine whether a bit is set |
* @nr: bit number to test |
* @addr: Address to start counting from |
*/ |
static int test_bit(int nr, const volatile unsigned long *addr); |
#endif |
#define test_bit(nr, addr) \ |
(__builtin_constant_p((nr)) \ |
? constant_test_bit((nr), (addr)) \ |
: variable_test_bit((nr), (addr))) |
/** |
* __ffs - find first set bit in word |
* @word: The word to search |
* |
* Undefined if no bit exists, so code should check against 0 first. |
*/ |
static inline unsigned long __ffs(unsigned long word) |
{ |
asm("rep; bsf %1,%0" |
: "=r" (word) |
: "rm" (word)); |
return word; |
} |
/** |
* ffz - find first zero bit in word |
* @word: The word to search |
* |
* Undefined if no zero exists, so code should check against ~0UL first. |
*/ |
static inline unsigned long ffz(unsigned long word) |
{ |
asm("rep; bsf %1,%0" |
: "=r" (word) |
: "r" (~word)); |
return word; |
} |
/* |
* __fls: find last set bit in word |
* @word: The word to search |
* |
* Undefined if no set bit exists, so code should check against 0 first. |
*/ |
static inline unsigned long __fls(unsigned long word) |
{ |
asm("bsr %1,%0" |
: "=r" (word) |
: "rm" (word)); |
return word; |
} |
#undef ADDR |
#ifdef __KERNEL__ |
/** |
* ffs - find first set bit in word |
* @x: the word to search |
* |
* This is defined the same way as the libc and compiler builtin ffs |
* routines, therefore differs in spirit from the other bitops. |
* |
* ffs(value) returns 0 if value is 0 or the position of the first |
* set bit if value is nonzero. The first (least significant) bit |
* is at position 1. |
*/ |
static inline int ffs(int x) |
{ |
int r; |
#ifdef CONFIG_X86_64 |
/* |
* AMD64 says BSFL won't clobber the dest reg if x==0; Intel64 says the |
* dest reg is undefined if x==0, but their CPU architect says its |
* value is written to set it to the same as before, except that the |
* top 32 bits will be cleared. |
* |
* We cannot do this on 32 bits because at the very least some |
* 486 CPUs did not behave this way. |
*/ |
asm("bsfl %1,%0" |
: "=r" (r) |
: "rm" (x), "0" (-1)); |
#elif defined(CONFIG_X86_CMOV) |
asm("bsfl %1,%0\n\t" |
"cmovzl %2,%0" |
: "=&r" (r) : "rm" (x), "r" (-1)); |
#else |
asm("bsfl %1,%0\n\t" |
"jnz 1f\n\t" |
"movl $-1,%0\n" |
"1:" : "=r" (r) : "rm" (x)); |
#endif |
return r + 1; |
} |
/** |
* fls - find last set bit in word |
* @x: the word to search |
* |
* This is defined in a similar way as the libc and compiler builtin |
* ffs, but returns the position of the most significant set bit. |
* |
* fls(value) returns 0 if value is 0 or the position of the last |
* set bit if value is nonzero. The last (most significant) bit is |
* at position 32. |
*/ |
static inline int fls(int x) |
{ |
int r; |
#ifdef CONFIG_X86_64 |
/* |
* AMD64 says BSRL won't clobber the dest reg if x==0; Intel64 says the |
* dest reg is undefined if x==0, but their CPU architect says its |
* value is written to set it to the same as before, except that the |
* top 32 bits will be cleared. |
* |
* We cannot do this on 32 bits because at the very least some |
* 486 CPUs did not behave this way. |
*/ |
asm("bsrl %1,%0" |
: "=r" (r) |
: "rm" (x), "0" (-1)); |
#elif defined(CONFIG_X86_CMOV) |
asm("bsrl %1,%0\n\t" |
"cmovzl %2,%0" |
: "=&r" (r) : "rm" (x), "rm" (-1)); |
#else |
asm("bsrl %1,%0\n\t" |
"jnz 1f\n\t" |
"movl $-1,%0\n" |
"1:" : "=r" (r) : "rm" (x)); |
#endif |
return r + 1; |
} |
/** |
* fls64 - find last set bit in a 64-bit word |
* @x: the word to search |
* |
* This is defined in a similar way as the libc and compiler builtin |
* ffsll, but returns the position of the most significant set bit. |
* |
* fls64(value) returns 0 if value is 0 or the position of the last |
* set bit if value is nonzero. The last (most significant) bit is |
* at position 64. |
*/ |
#ifdef CONFIG_X86_64 |
static __always_inline int fls64(__u64 x) |
{ |
int bitpos = -1; |
/* |
* AMD64 says BSRQ won't clobber the dest reg if x==0; Intel64 says the |
* dest reg is undefined if x==0, but their CPU architect says its |
* value is written to set it to the same as before. |
*/ |
asm("bsrq %1,%q0" |
: "+r" (bitpos) |
: "rm" (x)); |
return bitpos + 1; |
} |
#else |
#include <asm-generic/bitops/fls64.h> |
#endif |
#include <asm-generic/bitops/find.h> |
#include <asm-generic/bitops/sched.h> |
#include <asm/arch_hweight.h> |
#include <asm-generic/bitops/const_hweight.h> |
#include <asm-generic/bitops/le.h> |
#include <asm-generic/bitops/ext2-atomic-setbit.h> |
#endif /* __KERNEL__ */ |
#endif /* _ASM_X86_BITOPS_H */ |
/drivers/include/asm/bitsperlong.h |
---|
0,0 → 1,13 |
#ifndef __ASM_X86_BITSPERLONG_H |
#define __ASM_X86_BITSPERLONG_H |
#ifdef __x86_64__ |
# define __BITS_PER_LONG 64 |
#else |
# define __BITS_PER_LONG 32 |
#endif |
#include <asm-generic/bitsperlong.h> |
#endif /* __ASM_X86_BITSPERLONG_H */ |
/drivers/include/asm/byteorder.h |
---|
0,0 → 1,6 |
#ifndef _ASM_X86_BYTEORDER_H |
#define _ASM_X86_BYTEORDER_H |
#include <linux/byteorder/little_endian.h> |
#endif /* _ASM_X86_BYTEORDER_H */ |
/drivers/include/asm/cache.h |
---|
0,0 → 1,23 |
#ifndef _ASM_X86_CACHE_H |
#define _ASM_X86_CACHE_H |
#include <linux/linkage.h> |
/* L1 cache line size */ |
#define L1_CACHE_SHIFT (CONFIG_X86_L1_CACHE_SHIFT) |
#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) |
#define __read_mostly __attribute__((__section__(".data..read_mostly"))) |
#define INTERNODE_CACHE_SHIFT CONFIG_X86_INTERNODE_CACHE_SHIFT |
#define INTERNODE_CACHE_BYTES (1 << INTERNODE_CACHE_SHIFT) |
#ifdef CONFIG_X86_VSMP |
#ifdef CONFIG_SMP |
#define __cacheline_aligned_in_smp \ |
__attribute__((__aligned__(INTERNODE_CACHE_BYTES))) \ |
__page_aligned_data |
#endif |
#endif |
#endif /* _ASM_X86_CACHE_H */ |
/drivers/include/asm/cacheflush.h |
---|
0,0 → 1,131 |
#ifndef _ASM_X86_CACHEFLUSH_H |
#define _ASM_X86_CACHEFLUSH_H |
/* Caches aren't brain-dead on the intel. */ |
#include <asm-generic/cacheflush.h> |
#include <asm/special_insns.h> |
/* |
* The set_memory_* API can be used to change various attributes of a virtual |
* address range. The attributes include: |
* Cachability : UnCached, WriteCombining, WriteBack |
* Executability : eXeutable, NoteXecutable |
* Read/Write : ReadOnly, ReadWrite |
* Presence : NotPresent |
* |
* Within a category, the attributes are mutually exclusive. |
* |
* The implementation of this API will take care of various aspects that |
* are associated with changing such attributes, such as: |
* - Flushing TLBs |
* - Flushing CPU caches |
* - Making sure aliases of the memory behind the mapping don't violate |
* coherency rules as defined by the CPU in the system. |
* |
* What this API does not do: |
* - Provide exclusion between various callers - including callers that |
* operation on other mappings of the same physical page |
* - Restore default attributes when a page is freed |
* - Guarantee that mappings other than the requested one are |
* in any state, other than that these do not violate rules for |
* the CPU you have. Do not depend on any effects on other mappings, |
* CPUs other than the one you have may have more relaxed rules. |
* The caller is required to take care of these. |
*/ |
int _set_memory_uc(unsigned long addr, int numpages); |
int _set_memory_wc(unsigned long addr, int numpages); |
int _set_memory_wb(unsigned long addr, int numpages); |
int set_memory_uc(unsigned long addr, int numpages); |
int set_memory_wc(unsigned long addr, int numpages); |
int set_memory_wb(unsigned long addr, int numpages); |
int set_memory_x(unsigned long addr, int numpages); |
int set_memory_nx(unsigned long addr, int numpages); |
int set_memory_ro(unsigned long addr, int numpages); |
int set_memory_rw(unsigned long addr, int numpages); |
int set_memory_np(unsigned long addr, int numpages); |
int set_memory_4k(unsigned long addr, int numpages); |
int set_memory_array_uc(unsigned long *addr, int addrinarray); |
int set_memory_array_wc(unsigned long *addr, int addrinarray); |
int set_memory_array_wb(unsigned long *addr, int addrinarray); |
int set_pages_array_uc(struct page **pages, int addrinarray); |
int set_pages_array_wc(struct page **pages, int addrinarray); |
int set_pages_array_wb(struct page **pages, int addrinarray); |
/* |
* For legacy compatibility with the old APIs, a few functions |
* are provided that work on a "struct page". |
* These functions operate ONLY on the 1:1 kernel mapping of the |
* memory that the struct page represents, and internally just |
* call the set_memory_* function. See the description of the |
* set_memory_* function for more details on conventions. |
* |
* These APIs should be considered *deprecated* and are likely going to |
* be removed in the future. |
* The reason for this is the implicit operation on the 1:1 mapping only, |
* making this not a generally useful API. |
* |
* Specifically, many users of the old APIs had a virtual address, |
* called virt_to_page() or vmalloc_to_page() on that address to |
* get a struct page* that the old API required. |
* To convert these cases, use set_memory_*() on the original |
* virtual address, do not use these functions. |
*/ |
static int set_pages_uc(struct page *page, int numpages) |
{ |
return 0; |
}; |
static int set_pages_wb(struct page *page, int numpages) |
{ |
return 0; |
}; |
static int set_pages_x(struct page *page, int numpages) |
{ |
return 0; |
}; |
static int set_pages_nx(struct page *page, int numpages) |
{ |
return 0; |
}; |
static int set_pages_ro(struct page *page, int numpages) |
{ |
return 0; |
}; |
static int set_pages_rw(struct page *page, int numpages) |
{ |
return 0; |
}; |
void clflush_cache_range(void *addr, unsigned int size); |
#ifdef CONFIG_DEBUG_RODATA |
void mark_rodata_ro(void); |
extern const int rodata_test_data; |
extern int kernel_set_to_readonly; |
void set_kernel_text_rw(void); |
void set_kernel_text_ro(void); |
#else |
static inline void set_kernel_text_rw(void) { } |
static inline void set_kernel_text_ro(void) { } |
#endif |
#ifdef CONFIG_DEBUG_RODATA_TEST |
int rodata_test(void); |
#else |
static inline int rodata_test(void) |
{ |
return 0; |
} |
#endif |
#endif /* _ASM_X86_CACHEFLUSH_H */ |
/drivers/include/asm/cmpxchg.h |
---|
0,0 → 1,233 |
#ifndef ASM_X86_CMPXCHG_H |
#define ASM_X86_CMPXCHG_H |
#include <linux/compiler.h> |
#include <asm/alternative.h> /* Provides LOCK_PREFIX */ |
#define __HAVE_ARCH_CMPXCHG 1 |
/* |
* Non-existant functions to indicate usage errors at link time |
* (or compile-time if the compiler implements __compiletime_error(). |
*/ |
extern void __xchg_wrong_size(void) |
__compiletime_error("Bad argument size for xchg"); |
extern void __cmpxchg_wrong_size(void) |
__compiletime_error("Bad argument size for cmpxchg"); |
extern void __xadd_wrong_size(void) |
__compiletime_error("Bad argument size for xadd"); |
extern void __add_wrong_size(void) |
__compiletime_error("Bad argument size for add"); |
/* |
* Constants for operation sizes. On 32-bit, the 64-bit size it set to |
* -1 because sizeof will never return -1, thereby making those switch |
* case statements guaranteeed dead code which the compiler will |
* eliminate, and allowing the "missing symbol in the default case" to |
* indicate a usage error. |
*/ |
#define __X86_CASE_B 1 |
#define __X86_CASE_W 2 |
#define __X86_CASE_L 4 |
#ifdef CONFIG_64BIT |
#define __X86_CASE_Q 8 |
#else |
#define __X86_CASE_Q -1 /* sizeof will never return -1 */ |
#endif |
/* |
* An exchange-type operation, which takes a value and a pointer, and |
* returns the old value. |
*/ |
#define __xchg_op(ptr, arg, op, lock) \ |
({ \ |
__typeof__ (*(ptr)) __ret = (arg); \ |
switch (sizeof(*(ptr))) { \ |
case __X86_CASE_B: \ |
asm volatile (lock #op "b %b0, %1\n" \ |
: "+q" (__ret), "+m" (*(ptr)) \ |
: : "memory", "cc"); \ |
break; \ |
case __X86_CASE_W: \ |
asm volatile (lock #op "w %w0, %1\n" \ |
: "+r" (__ret), "+m" (*(ptr)) \ |
: : "memory", "cc"); \ |
break; \ |
case __X86_CASE_L: \ |
asm volatile (lock #op "l %0, %1\n" \ |
: "+r" (__ret), "+m" (*(ptr)) \ |
: : "memory", "cc"); \ |
break; \ |
case __X86_CASE_Q: \ |
asm volatile (lock #op "q %q0, %1\n" \ |
: "+r" (__ret), "+m" (*(ptr)) \ |
: : "memory", "cc"); \ |
break; \ |
default: \ |
__ ## op ## _wrong_size(); \ |
} \ |
__ret; \ |
}) |
/* |
* Note: no "lock" prefix even on SMP: xchg always implies lock anyway. |
* Since this is generally used to protect other memory information, we |
* use "asm volatile" and "memory" clobbers to prevent gcc from moving |
* information around. |
*/ |
#define xchg(ptr, v) __xchg_op((ptr), (v), xchg, "") |
/* |
* Atomic compare and exchange. Compare OLD with MEM, if identical, |
* store NEW in MEM. Return the initial value in MEM. Success is |
* indicated by comparing RETURN with OLD. |
*/ |
#define __raw_cmpxchg(ptr, old, new, size, lock) \ |
({ \ |
__typeof__(*(ptr)) __ret; \ |
__typeof__(*(ptr)) __old = (old); \ |
__typeof__(*(ptr)) __new = (new); \ |
switch (size) { \ |
case __X86_CASE_B: \ |
{ \ |
volatile u8 *__ptr = (volatile u8 *)(ptr); \ |
asm volatile(lock "cmpxchgb %2,%1" \ |
: "=a" (__ret), "+m" (*__ptr) \ |
: "q" (__new), "0" (__old) \ |
: "memory"); \ |
break; \ |
} \ |
case __X86_CASE_W: \ |
{ \ |
volatile u16 *__ptr = (volatile u16 *)(ptr); \ |
asm volatile(lock "cmpxchgw %2,%1" \ |
: "=a" (__ret), "+m" (*__ptr) \ |
: "r" (__new), "0" (__old) \ |
: "memory"); \ |
break; \ |
} \ |
case __X86_CASE_L: \ |
{ \ |
volatile u32 *__ptr = (volatile u32 *)(ptr); \ |
asm volatile(lock "cmpxchgl %2,%1" \ |
: "=a" (__ret), "+m" (*__ptr) \ |
: "r" (__new), "0" (__old) \ |
: "memory"); \ |
break; \ |
} \ |
case __X86_CASE_Q: \ |
{ \ |
volatile u64 *__ptr = (volatile u64 *)(ptr); \ |
asm volatile(lock "cmpxchgq %2,%1" \ |
: "=a" (__ret), "+m" (*__ptr) \ |
: "r" (__new), "0" (__old) \ |
: "memory"); \ |
break; \ |
} \ |
default: \ |
__cmpxchg_wrong_size(); \ |
} \ |
__ret; \ |
}) |
#define __cmpxchg(ptr, old, new, size) \ |
__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX) |
#define __sync_cmpxchg(ptr, old, new, size) \ |
__raw_cmpxchg((ptr), (old), (new), (size), "lock; ") |
#define __cmpxchg_local(ptr, old, new, size) \ |
__raw_cmpxchg((ptr), (old), (new), (size), "") |
#ifdef CONFIG_X86_32 |
# include <asm/cmpxchg_32.h> |
#else |
# include <asm/cmpxchg_64.h> |
#endif |
#define cmpxchg(ptr, old, new) \ |
__cmpxchg(ptr, old, new, sizeof(*(ptr))) |
#define sync_cmpxchg(ptr, old, new) \ |
__sync_cmpxchg(ptr, old, new, sizeof(*(ptr))) |
#define cmpxchg_local(ptr, old, new) \ |
__cmpxchg_local(ptr, old, new, sizeof(*(ptr))) |
/* |
* xadd() adds "inc" to "*ptr" and atomically returns the previous |
* value of "*ptr". |
* |
* xadd() is locked when multiple CPUs are online |
* xadd_sync() is always locked |
* xadd_local() is never locked |
*/ |
#define __xadd(ptr, inc, lock) __xchg_op((ptr), (inc), xadd, lock) |
#define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX) |
#define xadd_sync(ptr, inc) __xadd((ptr), (inc), "lock; ") |
#define xadd_local(ptr, inc) __xadd((ptr), (inc), "") |
#define __add(ptr, inc, lock) \ |
({ \ |
__typeof__ (*(ptr)) __ret = (inc); \ |
switch (sizeof(*(ptr))) { \ |
case __X86_CASE_B: \ |
asm volatile (lock "addb %b1, %0\n" \ |
: "+m" (*(ptr)) : "qi" (inc) \ |
: "memory", "cc"); \ |
break; \ |
case __X86_CASE_W: \ |
asm volatile (lock "addw %w1, %0\n" \ |
: "+m" (*(ptr)) : "ri" (inc) \ |
: "memory", "cc"); \ |
break; \ |
case __X86_CASE_L: \ |
asm volatile (lock "addl %1, %0\n" \ |
: "+m" (*(ptr)) : "ri" (inc) \ |
: "memory", "cc"); \ |
break; \ |
case __X86_CASE_Q: \ |
asm volatile (lock "addq %1, %0\n" \ |
: "+m" (*(ptr)) : "ri" (inc) \ |
: "memory", "cc"); \ |
break; \ |
default: \ |
__add_wrong_size(); \ |
} \ |
__ret; \ |
}) |
/* |
* add_*() adds "inc" to "*ptr" |
* |
* __add() takes a lock prefix |
* add_smp() is locked when multiple CPUs are online |
* add_sync() is always locked |
*/ |
#define add_smp(ptr, inc) __add((ptr), (inc), LOCK_PREFIX) |
#define add_sync(ptr, inc) __add((ptr), (inc), "lock; ") |
#define __cmpxchg_double(pfx, p1, p2, o1, o2, n1, n2) \ |
({ \ |
bool __ret; \ |
__typeof__(*(p1)) __old1 = (o1), __new1 = (n1); \ |
__typeof__(*(p2)) __old2 = (o2), __new2 = (n2); \ |
BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \ |
BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \ |
VM_BUG_ON((unsigned long)(p1) % (2 * sizeof(long))); \ |
VM_BUG_ON((unsigned long)((p1) + 1) != (unsigned long)(p2)); \ |
asm volatile(pfx "cmpxchg%c4b %2; sete %0" \ |
: "=a" (__ret), "+d" (__old2), \ |
"+m" (*(p1)), "+m" (*(p2)) \ |
: "i" (2 * sizeof(long)), "a" (__old1), \ |
"b" (__new1), "c" (__new2)); \ |
__ret; \ |
}) |
#define cmpxchg_double(p1, p2, o1, o2, n1, n2) \ |
__cmpxchg_double(LOCK_PREFIX, p1, p2, o1, o2, n1, n2) |
#define cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \ |
__cmpxchg_double(, p1, p2, o1, o2, n1, n2) |
#endif /* ASM_X86_CMPXCHG_H */ |
/drivers/include/asm/cmpxchg_32.h |
---|
0,0 → 1,114 |
#ifndef _ASM_X86_CMPXCHG_32_H |
#define _ASM_X86_CMPXCHG_32_H |
/* |
* Note: if you use set64_bit(), __cmpxchg64(), or their variants, you |
* you need to test for the feature in boot_cpu_data. |
*/ |
/* |
* CMPXCHG8B only writes to the target if we had the previous |
* value in registers, otherwise it acts as a read and gives us the |
* "new previous" value. That is why there is a loop. Preloading |
* EDX:EAX is a performance optimization: in the common case it means |
* we need only one locked operation. |
* |
* A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very |
* least an FPU save and/or %cr0.ts manipulation. |
* |
* cmpxchg8b must be used with the lock prefix here to allow the |
* instruction to be executed atomically. We need to have the reader |
* side to see the coherent 64bit value. |
*/ |
static inline void set_64bit(volatile u64 *ptr, u64 value) |
{ |
u32 low = value; |
u32 high = value >> 32; |
u64 prev = *ptr; |
asm volatile("\n1:\t" |
LOCK_PREFIX "cmpxchg8b %0\n\t" |
"jnz 1b" |
: "=m" (*ptr), "+A" (prev) |
: "b" (low), "c" (high) |
: "memory"); |
} |
#ifdef CONFIG_X86_CMPXCHG64 |
#define cmpxchg64(ptr, o, n) \ |
((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \ |
(unsigned long long)(n))) |
#define cmpxchg64_local(ptr, o, n) \ |
((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ |
(unsigned long long)(n))) |
#endif |
static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new) |
{ |
u64 prev; |
asm volatile(LOCK_PREFIX "cmpxchg8b %1" |
: "=A" (prev), |
"+m" (*ptr) |
: "b" ((u32)new), |
"c" ((u32)(new >> 32)), |
"0" (old) |
: "memory"); |
return prev; |
} |
static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) |
{ |
u64 prev; |
asm volatile("cmpxchg8b %1" |
: "=A" (prev), |
"+m" (*ptr) |
: "b" ((u32)new), |
"c" ((u32)(new >> 32)), |
"0" (old) |
: "memory"); |
return prev; |
} |
#ifndef CONFIG_X86_CMPXCHG64 |
/* |
* Building a kernel capable running on 80386 and 80486. It may be necessary |
* to simulate the cmpxchg8b on the 80386 and 80486 CPU. |
*/ |
#define cmpxchg64(ptr, o, n) \ |
({ \ |
__typeof__(*(ptr)) __ret; \ |
__typeof__(*(ptr)) __old = (o); \ |
__typeof__(*(ptr)) __new = (n); \ |
alternative_io(LOCK_PREFIX_HERE \ |
"call cmpxchg8b_emu", \ |
"lock; cmpxchg8b (%%esi)" , \ |
X86_FEATURE_CX8, \ |
"=A" (__ret), \ |
"S" ((ptr)), "0" (__old), \ |
"b" ((unsigned int)__new), \ |
"c" ((unsigned int)(__new>>32)) \ |
: "memory"); \ |
__ret; }) |
#define cmpxchg64_local(ptr, o, n) \ |
({ \ |
__typeof__(*(ptr)) __ret; \ |
__typeof__(*(ptr)) __old = (o); \ |
__typeof__(*(ptr)) __new = (n); \ |
alternative_io("call cmpxchg8b_emu", \ |
"cmpxchg8b (%%esi)" , \ |
X86_FEATURE_CX8, \ |
"=A" (__ret), \ |
"S" ((ptr)), "0" (__old), \ |
"b" ((unsigned int)__new), \ |
"c" ((unsigned int)(__new>>32)) \ |
: "memory"); \ |
__ret; }) |
#endif |
#define system_has_cmpxchg_double() cpu_has_cx8 |
#endif /* _ASM_X86_CMPXCHG_32_H */ |
/drivers/include/asm/cpufeature.h |
---|
0,0 → 1,586 |
/* |
* Defines x86 CPU feature bits |
*/ |
#ifndef _ASM_X86_CPUFEATURE_H |
#define _ASM_X86_CPUFEATURE_H |
#ifndef _ASM_X86_REQUIRED_FEATURES_H |
#include <asm/required-features.h> |
#endif |
#ifndef _ASM_X86_DISABLED_FEATURES_H |
#include <asm/disabled-features.h> |
#endif |
#define NCAPINTS 11 /* N 32-bit words worth of info */ |
#define NBUGINTS 1 /* N 32-bit bug flags */ |
/* |
* Note: If the comment begins with a quoted string, that string is used |
* in /proc/cpuinfo instead of the macro name. If the string is "", |
* this feature bit is not displayed in /proc/cpuinfo at all. |
*/ |
/* Intel-defined CPU features, CPUID level 0x00000001 (edx), word 0 */ |
#define X86_FEATURE_FPU ( 0*32+ 0) /* Onboard FPU */ |
#define X86_FEATURE_VME ( 0*32+ 1) /* Virtual Mode Extensions */ |
#define X86_FEATURE_DE ( 0*32+ 2) /* Debugging Extensions */ |
#define X86_FEATURE_PSE ( 0*32+ 3) /* Page Size Extensions */ |
#define X86_FEATURE_TSC ( 0*32+ 4) /* Time Stamp Counter */ |
#define X86_FEATURE_MSR ( 0*32+ 5) /* Model-Specific Registers */ |
#define X86_FEATURE_PAE ( 0*32+ 6) /* Physical Address Extensions */ |
#define X86_FEATURE_MCE ( 0*32+ 7) /* Machine Check Exception */ |
#define X86_FEATURE_CX8 ( 0*32+ 8) /* CMPXCHG8 instruction */ |
#define X86_FEATURE_APIC ( 0*32+ 9) /* Onboard APIC */ |
#define X86_FEATURE_SEP ( 0*32+11) /* SYSENTER/SYSEXIT */ |
#define X86_FEATURE_MTRR ( 0*32+12) /* Memory Type Range Registers */ |
#define X86_FEATURE_PGE ( 0*32+13) /* Page Global Enable */ |
#define X86_FEATURE_MCA ( 0*32+14) /* Machine Check Architecture */ |
#define X86_FEATURE_CMOV ( 0*32+15) /* CMOV instructions */ |
/* (plus FCMOVcc, FCOMI with FPU) */ |
#define X86_FEATURE_PAT ( 0*32+16) /* Page Attribute Table */ |
#define X86_FEATURE_PSE36 ( 0*32+17) /* 36-bit PSEs */ |
#define X86_FEATURE_PN ( 0*32+18) /* Processor serial number */ |
#define X86_FEATURE_CLFLUSH ( 0*32+19) /* CLFLUSH instruction */ |
#define X86_FEATURE_DS ( 0*32+21) /* "dts" Debug Store */ |
#define X86_FEATURE_ACPI ( 0*32+22) /* ACPI via MSR */ |
#define X86_FEATURE_MMX ( 0*32+23) /* Multimedia Extensions */ |
#define X86_FEATURE_FXSR ( 0*32+24) /* FXSAVE/FXRSTOR, CR4.OSFXSR */ |
#define X86_FEATURE_XMM ( 0*32+25) /* "sse" */ |
#define X86_FEATURE_XMM2 ( 0*32+26) /* "sse2" */ |
#define X86_FEATURE_SELFSNOOP ( 0*32+27) /* "ss" CPU self snoop */ |
#define X86_FEATURE_HT ( 0*32+28) /* Hyper-Threading */ |
#define X86_FEATURE_ACC ( 0*32+29) /* "tm" Automatic clock control */ |
#define X86_FEATURE_IA64 ( 0*32+30) /* IA-64 processor */ |
#define X86_FEATURE_PBE ( 0*32+31) /* Pending Break Enable */ |
/* AMD-defined CPU features, CPUID level 0x80000001, word 1 */ |
/* Don't duplicate feature flags which are redundant with Intel! */ |
#define X86_FEATURE_SYSCALL ( 1*32+11) /* SYSCALL/SYSRET */ |
#define X86_FEATURE_MP ( 1*32+19) /* MP Capable. */ |
#define X86_FEATURE_NX ( 1*32+20) /* Execute Disable */ |
#define X86_FEATURE_MMXEXT ( 1*32+22) /* AMD MMX extensions */ |
#define X86_FEATURE_FXSR_OPT ( 1*32+25) /* FXSAVE/FXRSTOR optimizations */ |
#define X86_FEATURE_GBPAGES ( 1*32+26) /* "pdpe1gb" GB pages */ |
#define X86_FEATURE_RDTSCP ( 1*32+27) /* RDTSCP */ |
#define X86_FEATURE_LM ( 1*32+29) /* Long Mode (x86-64) */ |
#define X86_FEATURE_3DNOWEXT ( 1*32+30) /* AMD 3DNow! extensions */ |
#define X86_FEATURE_3DNOW ( 1*32+31) /* 3DNow! */ |
/* Transmeta-defined CPU features, CPUID level 0x80860001, word 2 */ |
#define X86_FEATURE_RECOVERY ( 2*32+ 0) /* CPU in recovery mode */ |
#define X86_FEATURE_LONGRUN ( 2*32+ 1) /* Longrun power control */ |
#define X86_FEATURE_LRTI ( 2*32+ 3) /* LongRun table interface */ |
/* Other features, Linux-defined mapping, word 3 */ |
/* This range is used for feature bits which conflict or are synthesized */ |
#define X86_FEATURE_CXMMX ( 3*32+ 0) /* Cyrix MMX extensions */ |
#define X86_FEATURE_K6_MTRR ( 3*32+ 1) /* AMD K6 nonstandard MTRRs */ |
#define X86_FEATURE_CYRIX_ARR ( 3*32+ 2) /* Cyrix ARRs (= MTRRs) */ |
#define X86_FEATURE_CENTAUR_MCR ( 3*32+ 3) /* Centaur MCRs (= MTRRs) */ |
/* cpu types for specific tunings: */ |
#define X86_FEATURE_K8 ( 3*32+ 4) /* "" Opteron, Athlon64 */ |
#define X86_FEATURE_K7 ( 3*32+ 5) /* "" Athlon */ |
#define X86_FEATURE_P3 ( 3*32+ 6) /* "" P3 */ |
#define X86_FEATURE_P4 ( 3*32+ 7) /* "" P4 */ |
#define X86_FEATURE_CONSTANT_TSC ( 3*32+ 8) /* TSC ticks at a constant rate */ |
#define X86_FEATURE_UP ( 3*32+ 9) /* smp kernel running on up */ |
/* free, was #define X86_FEATURE_FXSAVE_LEAK ( 3*32+10) * "" FXSAVE leaks FOP/FIP/FOP */ |
#define X86_FEATURE_ARCH_PERFMON ( 3*32+11) /* Intel Architectural PerfMon */ |
#define X86_FEATURE_PEBS ( 3*32+12) /* Precise-Event Based Sampling */ |
#define X86_FEATURE_BTS ( 3*32+13) /* Branch Trace Store */ |
#define X86_FEATURE_SYSCALL32 ( 3*32+14) /* "" syscall in ia32 userspace */ |
#define X86_FEATURE_SYSENTER32 ( 3*32+15) /* "" sysenter in ia32 userspace */ |
#define X86_FEATURE_REP_GOOD ( 3*32+16) /* rep microcode works well */ |
#define X86_FEATURE_MFENCE_RDTSC ( 3*32+17) /* "" Mfence synchronizes RDTSC */ |
#define X86_FEATURE_LFENCE_RDTSC ( 3*32+18) /* "" Lfence synchronizes RDTSC */ |
/* free, was #define X86_FEATURE_11AP ( 3*32+19) * "" Bad local APIC aka 11AP */ |
#define X86_FEATURE_NOPL ( 3*32+20) /* The NOPL (0F 1F) instructions */ |
#define X86_FEATURE_ALWAYS ( 3*32+21) /* "" Always-present feature */ |
#define X86_FEATURE_XTOPOLOGY ( 3*32+22) /* cpu topology enum extensions */ |
#define X86_FEATURE_TSC_RELIABLE ( 3*32+23) /* TSC is known to be reliable */ |
#define X86_FEATURE_NONSTOP_TSC ( 3*32+24) /* TSC does not stop in C states */ |
/* free, was #define X86_FEATURE_CLFLUSH_MONITOR ( 3*32+25) * "" clflush reqd with monitor */ |
#define X86_FEATURE_EXTD_APICID ( 3*32+26) /* has extended APICID (8 bits) */ |
#define X86_FEATURE_AMD_DCM ( 3*32+27) /* multi-node processor */ |
#define X86_FEATURE_APERFMPERF ( 3*32+28) /* APERFMPERF */ |
#define X86_FEATURE_EAGER_FPU ( 3*32+29) /* "eagerfpu" Non lazy FPU restore */ |
#define X86_FEATURE_NONSTOP_TSC_S3 ( 3*32+30) /* TSC doesn't stop in S3 state */ |
/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */ |
#define X86_FEATURE_XMM3 ( 4*32+ 0) /* "pni" SSE-3 */ |
#define X86_FEATURE_PCLMULQDQ ( 4*32+ 1) /* PCLMULQDQ instruction */ |
#define X86_FEATURE_DTES64 ( 4*32+ 2) /* 64-bit Debug Store */ |
#define X86_FEATURE_MWAIT ( 4*32+ 3) /* "monitor" Monitor/Mwait support */ |
#define X86_FEATURE_DSCPL ( 4*32+ 4) /* "ds_cpl" CPL Qual. Debug Store */ |
#define X86_FEATURE_VMX ( 4*32+ 5) /* Hardware virtualization */ |
#define X86_FEATURE_SMX ( 4*32+ 6) /* Safer mode */ |
#define X86_FEATURE_EST ( 4*32+ 7) /* Enhanced SpeedStep */ |
#define X86_FEATURE_TM2 ( 4*32+ 8) /* Thermal Monitor 2 */ |
#define X86_FEATURE_SSSE3 ( 4*32+ 9) /* Supplemental SSE-3 */ |
#define X86_FEATURE_CID ( 4*32+10) /* Context ID */ |
#define X86_FEATURE_FMA ( 4*32+12) /* Fused multiply-add */ |
#define X86_FEATURE_CX16 ( 4*32+13) /* CMPXCHG16B */ |
#define X86_FEATURE_XTPR ( 4*32+14) /* Send Task Priority Messages */ |
#define X86_FEATURE_PDCM ( 4*32+15) /* Performance Capabilities */ |
#define X86_FEATURE_PCID ( 4*32+17) /* Process Context Identifiers */ |
#define X86_FEATURE_DCA ( 4*32+18) /* Direct Cache Access */ |
#define X86_FEATURE_XMM4_1 ( 4*32+19) /* "sse4_1" SSE-4.1 */ |
#define X86_FEATURE_XMM4_2 ( 4*32+20) /* "sse4_2" SSE-4.2 */ |
#define X86_FEATURE_X2APIC ( 4*32+21) /* x2APIC */ |
#define X86_FEATURE_MOVBE ( 4*32+22) /* MOVBE instruction */ |
#define X86_FEATURE_POPCNT ( 4*32+23) /* POPCNT instruction */ |
#define X86_FEATURE_TSC_DEADLINE_TIMER ( 4*32+24) /* Tsc deadline timer */ |
#define X86_FEATURE_AES ( 4*32+25) /* AES instructions */ |
#define X86_FEATURE_XSAVE ( 4*32+26) /* XSAVE/XRSTOR/XSETBV/XGETBV */ |
#define X86_FEATURE_OSXSAVE ( 4*32+27) /* "" XSAVE enabled in the OS */ |
#define X86_FEATURE_AVX ( 4*32+28) /* Advanced Vector Extensions */ |
#define X86_FEATURE_F16C ( 4*32+29) /* 16-bit fp conversions */ |
#define X86_FEATURE_RDRAND ( 4*32+30) /* The RDRAND instruction */ |
#define X86_FEATURE_HYPERVISOR ( 4*32+31) /* Running on a hypervisor */ |
/* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */ |
#define X86_FEATURE_XSTORE ( 5*32+ 2) /* "rng" RNG present (xstore) */ |
#define X86_FEATURE_XSTORE_EN ( 5*32+ 3) /* "rng_en" RNG enabled */ |
#define X86_FEATURE_XCRYPT ( 5*32+ 6) /* "ace" on-CPU crypto (xcrypt) */ |
#define X86_FEATURE_XCRYPT_EN ( 5*32+ 7) /* "ace_en" on-CPU crypto enabled */ |
#define X86_FEATURE_ACE2 ( 5*32+ 8) /* Advanced Cryptography Engine v2 */ |
#define X86_FEATURE_ACE2_EN ( 5*32+ 9) /* ACE v2 enabled */ |
#define X86_FEATURE_PHE ( 5*32+10) /* PadLock Hash Engine */ |
#define X86_FEATURE_PHE_EN ( 5*32+11) /* PHE enabled */ |
#define X86_FEATURE_PMM ( 5*32+12) /* PadLock Montgomery Multiplier */ |
#define X86_FEATURE_PMM_EN ( 5*32+13) /* PMM enabled */ |
/* More extended AMD flags: CPUID level 0x80000001, ecx, word 6 */ |
#define X86_FEATURE_LAHF_LM ( 6*32+ 0) /* LAHF/SAHF in long mode */ |
#define X86_FEATURE_CMP_LEGACY ( 6*32+ 1) /* If yes HyperThreading not valid */ |
#define X86_FEATURE_SVM ( 6*32+ 2) /* Secure virtual machine */ |
#define X86_FEATURE_EXTAPIC ( 6*32+ 3) /* Extended APIC space */ |
#define X86_FEATURE_CR8_LEGACY ( 6*32+ 4) /* CR8 in 32-bit mode */ |
#define X86_FEATURE_ABM ( 6*32+ 5) /* Advanced bit manipulation */ |
#define X86_FEATURE_SSE4A ( 6*32+ 6) /* SSE-4A */ |
#define X86_FEATURE_MISALIGNSSE ( 6*32+ 7) /* Misaligned SSE mode */ |
#define X86_FEATURE_3DNOWPREFETCH ( 6*32+ 8) /* 3DNow prefetch instructions */ |
#define X86_FEATURE_OSVW ( 6*32+ 9) /* OS Visible Workaround */ |
#define X86_FEATURE_IBS ( 6*32+10) /* Instruction Based Sampling */ |
#define X86_FEATURE_XOP ( 6*32+11) /* extended AVX instructions */ |
#define X86_FEATURE_SKINIT ( 6*32+12) /* SKINIT/STGI instructions */ |
#define X86_FEATURE_WDT ( 6*32+13) /* Watchdog timer */ |
#define X86_FEATURE_LWP ( 6*32+15) /* Light Weight Profiling */ |
#define X86_FEATURE_FMA4 ( 6*32+16) /* 4 operands MAC instructions */ |
#define X86_FEATURE_TCE ( 6*32+17) /* translation cache extension */ |
#define X86_FEATURE_NODEID_MSR ( 6*32+19) /* NodeId MSR */ |
#define X86_FEATURE_TBM ( 6*32+21) /* trailing bit manipulations */ |
#define X86_FEATURE_TOPOEXT ( 6*32+22) /* topology extensions CPUID leafs */ |
#define X86_FEATURE_PERFCTR_CORE ( 6*32+23) /* core performance counter extensions */ |
#define X86_FEATURE_PERFCTR_NB ( 6*32+24) /* NB performance counter extensions */ |
#define X86_FEATURE_PERFCTR_L2 ( 6*32+28) /* L2 performance counter extensions */ |
/* |
* Auxiliary flags: Linux defined - For features scattered in various |
* CPUID levels like 0x6, 0xA etc, word 7 |
*/ |
#define X86_FEATURE_IDA ( 7*32+ 0) /* Intel Dynamic Acceleration */ |
#define X86_FEATURE_ARAT ( 7*32+ 1) /* Always Running APIC Timer */ |
#define X86_FEATURE_CPB ( 7*32+ 2) /* AMD Core Performance Boost */ |
#define X86_FEATURE_EPB ( 7*32+ 3) /* IA32_ENERGY_PERF_BIAS support */ |
#define X86_FEATURE_PLN ( 7*32+ 5) /* Intel Power Limit Notification */ |
#define X86_FEATURE_PTS ( 7*32+ 6) /* Intel Package Thermal Status */ |
#define X86_FEATURE_DTHERM ( 7*32+ 7) /* Digital Thermal Sensor */ |
#define X86_FEATURE_HW_PSTATE ( 7*32+ 8) /* AMD HW-PState */ |
#define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */ |
#define X86_FEATURE_HWP ( 7*32+ 10) /* "hwp" Intel HWP */ |
#define X86_FEATURE_HWP_NOITFY ( 7*32+ 11) /* Intel HWP_NOTIFY */ |
#define X86_FEATURE_HWP_ACT_WINDOW ( 7*32+ 12) /* Intel HWP_ACT_WINDOW */ |
#define X86_FEATURE_HWP_EPP ( 7*32+13) /* Intel HWP_EPP */ |
#define X86_FEATURE_HWP_PKG_REQ ( 7*32+14) /* Intel HWP_PKG_REQ */ |
/* Virtualization flags: Linux defined, word 8 */ |
#define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */ |
#define X86_FEATURE_VNMI ( 8*32+ 1) /* Intel Virtual NMI */ |
#define X86_FEATURE_FLEXPRIORITY ( 8*32+ 2) /* Intel FlexPriority */ |
#define X86_FEATURE_EPT ( 8*32+ 3) /* Intel Extended Page Table */ |
#define X86_FEATURE_VPID ( 8*32+ 4) /* Intel Virtual Processor ID */ |
#define X86_FEATURE_NPT ( 8*32+ 5) /* AMD Nested Page Table support */ |
#define X86_FEATURE_LBRV ( 8*32+ 6) /* AMD LBR Virtualization support */ |
#define X86_FEATURE_SVML ( 8*32+ 7) /* "svm_lock" AMD SVM locking MSR */ |
#define X86_FEATURE_NRIPS ( 8*32+ 8) /* "nrip_save" AMD SVM next_rip save */ |
#define X86_FEATURE_TSCRATEMSR ( 8*32+ 9) /* "tsc_scale" AMD TSC scaling support */ |
#define X86_FEATURE_VMCBCLEAN ( 8*32+10) /* "vmcb_clean" AMD VMCB clean bits support */ |
#define X86_FEATURE_FLUSHBYASID ( 8*32+11) /* AMD flush-by-ASID support */ |
#define X86_FEATURE_DECODEASSISTS ( 8*32+12) /* AMD Decode Assists support */ |
#define X86_FEATURE_PAUSEFILTER ( 8*32+13) /* AMD filtered pause intercept */ |
#define X86_FEATURE_PFTHRESHOLD ( 8*32+14) /* AMD pause filter threshold */ |
#define X86_FEATURE_VMMCALL ( 8*32+15) /* Prefer vmmcall to vmcall */ |
/* Intel-defined CPU features, CPUID level 0x00000007:0 (ebx), word 9 */ |
#define X86_FEATURE_FSGSBASE ( 9*32+ 0) /* {RD/WR}{FS/GS}BASE instructions*/ |
#define X86_FEATURE_TSC_ADJUST ( 9*32+ 1) /* TSC adjustment MSR 0x3b */ |
#define X86_FEATURE_BMI1 ( 9*32+ 3) /* 1st group bit manipulation extensions */ |
#define X86_FEATURE_HLE ( 9*32+ 4) /* Hardware Lock Elision */ |
#define X86_FEATURE_AVX2 ( 9*32+ 5) /* AVX2 instructions */ |
#define X86_FEATURE_SMEP ( 9*32+ 7) /* Supervisor Mode Execution Protection */ |
#define X86_FEATURE_BMI2 ( 9*32+ 8) /* 2nd group bit manipulation extensions */ |
#define X86_FEATURE_ERMS ( 9*32+ 9) /* Enhanced REP MOVSB/STOSB */ |
#define X86_FEATURE_INVPCID ( 9*32+10) /* Invalidate Processor Context ID */ |
#define X86_FEATURE_RTM ( 9*32+11) /* Restricted Transactional Memory */ |
#define X86_FEATURE_MPX ( 9*32+14) /* Memory Protection Extension */ |
#define X86_FEATURE_AVX512F ( 9*32+16) /* AVX-512 Foundation */ |
#define X86_FEATURE_RDSEED ( 9*32+18) /* The RDSEED instruction */ |
#define X86_FEATURE_ADX ( 9*32+19) /* The ADCX and ADOX instructions */ |
#define X86_FEATURE_SMAP ( 9*32+20) /* Supervisor Mode Access Prevention */ |
#define X86_FEATURE_CLFLUSHOPT ( 9*32+23) /* CLFLUSHOPT instruction */ |
#define X86_FEATURE_AVX512PF ( 9*32+26) /* AVX-512 Prefetch */ |
#define X86_FEATURE_AVX512ER ( 9*32+27) /* AVX-512 Exponential and Reciprocal */ |
#define X86_FEATURE_AVX512CD ( 9*32+28) /* AVX-512 Conflict Detection */ |
/* Extended state features, CPUID level 0x0000000d:1 (eax), word 10 */ |
#define X86_FEATURE_XSAVEOPT (10*32+ 0) /* XSAVEOPT */ |
#define X86_FEATURE_XSAVEC (10*32+ 1) /* XSAVEC */ |
#define X86_FEATURE_XGETBV1 (10*32+ 2) /* XGETBV with ECX = 1 */ |
#define X86_FEATURE_XSAVES (10*32+ 3) /* XSAVES/XRSTORS */ |
/* |
* BUG word(s) |
*/ |
#define X86_BUG(x) (NCAPINTS*32 + (x)) |
#define X86_BUG_F00F X86_BUG(0) /* Intel F00F */ |
#define X86_BUG_FDIV X86_BUG(1) /* FPU FDIV */ |
#define X86_BUG_COMA X86_BUG(2) /* Cyrix 6x86 coma */ |
#define X86_BUG_AMD_TLB_MMATCH X86_BUG(3) /* "tlb_mmatch" AMD Erratum 383 */ |
#define X86_BUG_AMD_APIC_C1E X86_BUG(4) /* "apic_c1e" AMD Erratum 400 */ |
#define X86_BUG_11AP X86_BUG(5) /* Bad local APIC aka 11AP */ |
#define X86_BUG_FXSAVE_LEAK X86_BUG(6) /* FXSAVE leaks FOP/FIP/FOP */ |
#define X86_BUG_CLFLUSH_MONITOR X86_BUG(7) /* AAI65, CLFLUSH required before MONITOR */ |
#if defined(__KERNEL__) && !defined(__ASSEMBLY__) |
#include <asm/asm.h> |
#include <linux/bitops.h> |
#ifdef CONFIG_X86_FEATURE_NAMES |
extern const char * const x86_cap_flags[NCAPINTS*32]; |
extern const char * const x86_power_flags[32]; |
#define X86_CAP_FMT "%s" |
#define x86_cap_flag(flag) x86_cap_flags[flag] |
#else |
#define X86_CAP_FMT "%d:%d" |
#define x86_cap_flag(flag) ((flag) >> 5), ((flag) & 31) |
#endif |
/* |
* In order to save room, we index into this array by doing |
* X86_BUG_<name> - NCAPINTS*32. |
*/ |
extern const char * const x86_bug_flags[NBUGINTS*32]; |
#define test_cpu_cap(c, bit) \ |
test_bit(bit, (unsigned long *)((c)->x86_capability)) |
#define REQUIRED_MASK_BIT_SET(bit) \ |
( (((bit)>>5)==0 && (1UL<<((bit)&31) & REQUIRED_MASK0)) || \ |
(((bit)>>5)==1 && (1UL<<((bit)&31) & REQUIRED_MASK1)) || \ |
(((bit)>>5)==2 && (1UL<<((bit)&31) & REQUIRED_MASK2)) || \ |
(((bit)>>5)==3 && (1UL<<((bit)&31) & REQUIRED_MASK3)) || \ |
(((bit)>>5)==4 && (1UL<<((bit)&31) & REQUIRED_MASK4)) || \ |
(((bit)>>5)==5 && (1UL<<((bit)&31) & REQUIRED_MASK5)) || \ |
(((bit)>>5)==6 && (1UL<<((bit)&31) & REQUIRED_MASK6)) || \ |
(((bit)>>5)==7 && (1UL<<((bit)&31) & REQUIRED_MASK7)) || \ |
(((bit)>>5)==8 && (1UL<<((bit)&31) & REQUIRED_MASK8)) || \ |
(((bit)>>5)==9 && (1UL<<((bit)&31) & REQUIRED_MASK9)) ) |
#define DISABLED_MASK_BIT_SET(bit) \ |
( (((bit)>>5)==0 && (1UL<<((bit)&31) & DISABLED_MASK0)) || \ |
(((bit)>>5)==1 && (1UL<<((bit)&31) & DISABLED_MASK1)) || \ |
(((bit)>>5)==2 && (1UL<<((bit)&31) & DISABLED_MASK2)) || \ |
(((bit)>>5)==3 && (1UL<<((bit)&31) & DISABLED_MASK3)) || \ |
(((bit)>>5)==4 && (1UL<<((bit)&31) & DISABLED_MASK4)) || \ |
(((bit)>>5)==5 && (1UL<<((bit)&31) & DISABLED_MASK5)) || \ |
(((bit)>>5)==6 && (1UL<<((bit)&31) & DISABLED_MASK6)) || \ |
(((bit)>>5)==7 && (1UL<<((bit)&31) & DISABLED_MASK7)) || \ |
(((bit)>>5)==8 && (1UL<<((bit)&31) & DISABLED_MASK8)) || \ |
(((bit)>>5)==9 && (1UL<<((bit)&31) & DISABLED_MASK9)) ) |
#define cpu_has(c, bit) \ |
(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \ |
test_cpu_cap(c, bit)) |
#define this_cpu_has(bit) \ |
(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \ |
x86_this_cpu_test_bit(bit, (unsigned long *)&cpu_info.x86_capability)) |
/* |
* This macro is for detection of features which need kernel |
* infrastructure to be used. It may *not* directly test the CPU |
* itself. Use the cpu_has() family if you want true runtime |
* testing of CPU features, like in hypervisor code where you are |
* supporting a possible guest feature where host support for it |
* is not relevant. |
*/ |
#define cpu_feature_enabled(bit) \ |
(__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 : \ |
cpu_has(&boot_cpu_data, bit)) |
#define boot_cpu_has(bit) cpu_has(&boot_cpu_data, bit) |
#define set_cpu_cap(c, bit) set_bit(bit, (unsigned long *)((c)->x86_capability)) |
#define clear_cpu_cap(c, bit) clear_bit(bit, (unsigned long *)((c)->x86_capability)) |
#define setup_clear_cpu_cap(bit) do { \ |
clear_cpu_cap(&boot_cpu_data, bit); \ |
set_bit(bit, (unsigned long *)cpu_caps_cleared); \ |
} while (0) |
#define setup_force_cpu_cap(bit) do { \ |
set_cpu_cap(&boot_cpu_data, bit); \ |
set_bit(bit, (unsigned long *)cpu_caps_set); \ |
} while (0) |
#define cpu_has_fpu boot_cpu_has(X86_FEATURE_FPU) |
#define cpu_has_de boot_cpu_has(X86_FEATURE_DE) |
#define cpu_has_pse boot_cpu_has(X86_FEATURE_PSE) |
#define cpu_has_tsc boot_cpu_has(X86_FEATURE_TSC) |
#define cpu_has_pge boot_cpu_has(X86_FEATURE_PGE) |
#define cpu_has_apic boot_cpu_has(X86_FEATURE_APIC) |
#define cpu_has_sep boot_cpu_has(X86_FEATURE_SEP) |
#define cpu_has_mtrr boot_cpu_has(X86_FEATURE_MTRR) |
#define cpu_has_mmx boot_cpu_has(X86_FEATURE_MMX) |
#define cpu_has_fxsr boot_cpu_has(X86_FEATURE_FXSR) |
#define cpu_has_xmm boot_cpu_has(X86_FEATURE_XMM) |
#define cpu_has_xmm2 boot_cpu_has(X86_FEATURE_XMM2) |
#define cpu_has_xmm3 boot_cpu_has(X86_FEATURE_XMM3) |
#define cpu_has_ssse3 boot_cpu_has(X86_FEATURE_SSSE3) |
#define cpu_has_aes boot_cpu_has(X86_FEATURE_AES) |
#define cpu_has_avx boot_cpu_has(X86_FEATURE_AVX) |
#define cpu_has_avx2 boot_cpu_has(X86_FEATURE_AVX2) |
#define cpu_has_ht boot_cpu_has(X86_FEATURE_HT) |
#define cpu_has_nx boot_cpu_has(X86_FEATURE_NX) |
#define cpu_has_xstore boot_cpu_has(X86_FEATURE_XSTORE) |
#define cpu_has_xstore_enabled boot_cpu_has(X86_FEATURE_XSTORE_EN) |
#define cpu_has_xcrypt boot_cpu_has(X86_FEATURE_XCRYPT) |
#define cpu_has_xcrypt_enabled boot_cpu_has(X86_FEATURE_XCRYPT_EN) |
#define cpu_has_ace2 boot_cpu_has(X86_FEATURE_ACE2) |
#define cpu_has_ace2_enabled boot_cpu_has(X86_FEATURE_ACE2_EN) |
#define cpu_has_phe boot_cpu_has(X86_FEATURE_PHE) |
#define cpu_has_phe_enabled boot_cpu_has(X86_FEATURE_PHE_EN) |
#define cpu_has_pmm boot_cpu_has(X86_FEATURE_PMM) |
#define cpu_has_pmm_enabled boot_cpu_has(X86_FEATURE_PMM_EN) |
#define cpu_has_ds boot_cpu_has(X86_FEATURE_DS) |
#define cpu_has_pebs boot_cpu_has(X86_FEATURE_PEBS) |
#define cpu_has_clflush boot_cpu_has(X86_FEATURE_CLFLUSH) |
#define cpu_has_bts boot_cpu_has(X86_FEATURE_BTS) |
#define cpu_has_gbpages boot_cpu_has(X86_FEATURE_GBPAGES) |
#define cpu_has_arch_perfmon boot_cpu_has(X86_FEATURE_ARCH_PERFMON) |
#define cpu_has_pat boot_cpu_has(X86_FEATURE_PAT) |
#define cpu_has_xmm4_1 boot_cpu_has(X86_FEATURE_XMM4_1) |
#define cpu_has_xmm4_2 boot_cpu_has(X86_FEATURE_XMM4_2) |
#define cpu_has_x2apic boot_cpu_has(X86_FEATURE_X2APIC) |
#define cpu_has_xsave boot_cpu_has(X86_FEATURE_XSAVE) |
#define cpu_has_xsaveopt boot_cpu_has(X86_FEATURE_XSAVEOPT) |
#define cpu_has_xsaves boot_cpu_has(X86_FEATURE_XSAVES) |
#define cpu_has_osxsave boot_cpu_has(X86_FEATURE_OSXSAVE) |
#define cpu_has_hypervisor boot_cpu_has(X86_FEATURE_HYPERVISOR) |
#define cpu_has_pclmulqdq boot_cpu_has(X86_FEATURE_PCLMULQDQ) |
#define cpu_has_perfctr_core boot_cpu_has(X86_FEATURE_PERFCTR_CORE) |
#define cpu_has_perfctr_nb boot_cpu_has(X86_FEATURE_PERFCTR_NB) |
#define cpu_has_perfctr_l2 boot_cpu_has(X86_FEATURE_PERFCTR_L2) |
#define cpu_has_cx8 boot_cpu_has(X86_FEATURE_CX8) |
#define cpu_has_cx16 boot_cpu_has(X86_FEATURE_CX16) |
#define cpu_has_eager_fpu boot_cpu_has(X86_FEATURE_EAGER_FPU) |
#define cpu_has_topoext boot_cpu_has(X86_FEATURE_TOPOEXT) |
#if __GNUC__ >= 4 |
extern void warn_pre_alternatives(void); |
extern bool __static_cpu_has_safe(u16 bit); |
/* |
* Static testing of CPU features. Used the same as boot_cpu_has(). |
* These are only valid after alternatives have run, but will statically |
* patch the target code for additional performance. |
*/ |
static __always_inline __pure bool __static_cpu_has(u16 bit) |
{ |
#ifdef CC_HAVE_ASM_GOTO |
#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS |
/* |
* Catch too early usage of this before alternatives |
* have run. |
*/ |
asm_volatile_goto("1: jmp %l[t_warn]\n" |
"2:\n" |
".section .altinstructions,\"a\"\n" |
" .long 1b - .\n" |
" .long 0\n" /* no replacement */ |
" .word %P0\n" /* 1: do replace */ |
" .byte 2b - 1b\n" /* source len */ |
" .byte 0\n" /* replacement len */ |
".previous\n" |
/* skipping size check since replacement size = 0 */ |
: : "i" (X86_FEATURE_ALWAYS) : : t_warn); |
#endif |
asm_volatile_goto("1: jmp %l[t_no]\n" |
"2:\n" |
".section .altinstructions,\"a\"\n" |
" .long 1b - .\n" |
" .long 0\n" /* no replacement */ |
" .word %P0\n" /* feature bit */ |
" .byte 2b - 1b\n" /* source len */ |
" .byte 0\n" /* replacement len */ |
".previous\n" |
/* skipping size check since replacement size = 0 */ |
: : "i" (bit) : : t_no); |
return true; |
t_no: |
return false; |
#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS |
t_warn: |
warn_pre_alternatives(); |
return false; |
#endif |
#else /* CC_HAVE_ASM_GOTO */ |
u8 flag; |
/* Open-coded due to __stringify() in ALTERNATIVE() */ |
asm volatile("1: movb $0,%0\n" |
"2:\n" |
".section .altinstructions,\"a\"\n" |
" .long 1b - .\n" |
" .long 3f - .\n" |
" .word %P1\n" /* feature bit */ |
" .byte 2b - 1b\n" /* source len */ |
" .byte 4f - 3f\n" /* replacement len */ |
".previous\n" |
".section .discard,\"aw\",@progbits\n" |
" .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */ |
".previous\n" |
".section .altinstr_replacement,\"ax\"\n" |
"3: movb $1,%0\n" |
"4:\n" |
".previous\n" |
: "=qm" (flag) : "i" (bit)); |
return flag; |
#endif /* CC_HAVE_ASM_GOTO */ |
} |
#define static_cpu_has(bit) \ |
( \ |
__builtin_constant_p(boot_cpu_has(bit)) ? \ |
boot_cpu_has(bit) : \ |
__builtin_constant_p(bit) ? \ |
__static_cpu_has(bit) : \ |
boot_cpu_has(bit) \ |
) |
static __always_inline __pure bool _static_cpu_has_safe(u16 bit) |
{ |
#ifdef CC_HAVE_ASM_GOTO |
/* |
* We need to spell the jumps to the compiler because, depending on the offset, |
* the replacement jump can be bigger than the original jump, and this we cannot |
* have. Thus, we force the jump to the widest, 4-byte, signed relative |
* offset even though the last would often fit in less bytes. |
*/ |
asm_volatile_goto("1: .byte 0xe9\n .long %l[t_dynamic] - 2f\n" |
"2:\n" |
".section .altinstructions,\"a\"\n" |
" .long 1b - .\n" /* src offset */ |
" .long 3f - .\n" /* repl offset */ |
" .word %P1\n" /* always replace */ |
" .byte 2b - 1b\n" /* src len */ |
" .byte 4f - 3f\n" /* repl len */ |
".previous\n" |
".section .altinstr_replacement,\"ax\"\n" |
"3: .byte 0xe9\n .long %l[t_no] - 2b\n" |
"4:\n" |
".previous\n" |
".section .altinstructions,\"a\"\n" |
" .long 1b - .\n" /* src offset */ |
" .long 0\n" /* no replacement */ |
" .word %P0\n" /* feature bit */ |
" .byte 2b - 1b\n" /* src len */ |
" .byte 0\n" /* repl len */ |
".previous\n" |
: : "i" (bit), "i" (X86_FEATURE_ALWAYS) |
: : t_dynamic, t_no); |
return true; |
t_no: |
return false; |
t_dynamic: |
return __static_cpu_has_safe(bit); |
#else |
u8 flag; |
/* Open-coded due to __stringify() in ALTERNATIVE() */ |
asm volatile("1: movb $2,%0\n" |
"2:\n" |
".section .altinstructions,\"a\"\n" |
" .long 1b - .\n" /* src offset */ |
" .long 3f - .\n" /* repl offset */ |
" .word %P2\n" /* always replace */ |
" .byte 2b - 1b\n" /* source len */ |
" .byte 4f - 3f\n" /* replacement len */ |
".previous\n" |
".section .discard,\"aw\",@progbits\n" |
" .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */ |
".previous\n" |
".section .altinstr_replacement,\"ax\"\n" |
"3: movb $0,%0\n" |
"4:\n" |
".previous\n" |
".section .altinstructions,\"a\"\n" |
" .long 1b - .\n" /* src offset */ |
" .long 5f - .\n" /* repl offset */ |
" .word %P1\n" /* feature bit */ |
" .byte 4b - 3b\n" /* src len */ |
" .byte 6f - 5f\n" /* repl len */ |
".previous\n" |
".section .discard,\"aw\",@progbits\n" |
" .byte 0xff + (6f-5f) - (4b-3b)\n" /* size check */ |
".previous\n" |
".section .altinstr_replacement,\"ax\"\n" |
"5: movb $1,%0\n" |
"6:\n" |
".previous\n" |
: "=qm" (flag) |
: "i" (bit), "i" (X86_FEATURE_ALWAYS)); |
return (flag == 2 ? __static_cpu_has_safe(bit) : flag); |
#endif /* CC_HAVE_ASM_GOTO */ |
} |
#define static_cpu_has_safe(bit) \ |
( \ |
__builtin_constant_p(boot_cpu_has(bit)) ? \ |
boot_cpu_has(bit) : \ |
_static_cpu_has_safe(bit) \ |
) |
#else |
/* |
* gcc 3.x is too stupid to do the static test; fall back to dynamic. |
*/ |
#define static_cpu_has(bit) boot_cpu_has(bit) |
#define static_cpu_has_safe(bit) boot_cpu_has(bit) |
#endif |
#define cpu_has_bug(c, bit) cpu_has(c, (bit)) |
#define set_cpu_bug(c, bit) set_cpu_cap(c, (bit)) |
#define clear_cpu_bug(c, bit) clear_cpu_cap(c, (bit)) |
#define static_cpu_has_bug(bit) static_cpu_has((bit)) |
#define static_cpu_has_bug_safe(bit) static_cpu_has_safe((bit)) |
#define boot_cpu_has_bug(bit) cpu_has_bug(&boot_cpu_data, (bit)) |
#define MAX_CPU_FEATURES (NCAPINTS * 32) |
#define cpu_have_feature boot_cpu_has |
#define CPU_FEATURE_TYPEFMT "x86,ven%04Xfam%04Xmod%04X" |
#define CPU_FEATURE_TYPEVAL boot_cpu_data.x86_vendor, boot_cpu_data.x86, \ |
boot_cpu_data.x86_model |
#endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */ |
#endif /* _ASM_X86_CPUFEATURE_H */ |
/drivers/include/asm/cpumask.h |
---|
0,0 → 1,14 |
#ifndef _ASM_X86_CPUMASK_H |
#define _ASM_X86_CPUMASK_H |
#ifndef __ASSEMBLY__ |
#include <linux/cpumask.h> |
extern cpumask_var_t cpu_callin_mask; |
extern cpumask_var_t cpu_callout_mask; |
extern cpumask_var_t cpu_initialized_mask; |
extern cpumask_var_t cpu_sibling_setup_mask; |
extern void setup_cpu_local_masks(void); |
#endif /* __ASSEMBLY__ */ |
#endif /* _ASM_X86_CPUMASK_H */ |
/drivers/include/asm/current.h |
---|
0,0 → 1,21 |
#ifndef _ASM_X86_CURRENT_H |
#define _ASM_X86_CURRENT_H |
#include <linux/compiler.h> |
#include <asm/percpu.h> |
#ifndef __ASSEMBLY__ |
struct task_struct; |
DECLARE_PER_CPU(struct task_struct *, current_task); |
static __always_inline struct task_struct *get_current(void) |
{ |
return this_cpu_read_stable(current_task); |
} |
#define current (void*)GetPid() |
#endif /* __ASSEMBLY__ */ |
#endif /* _ASM_X86_CURRENT_H */ |
/drivers/include/asm/delay.h |
---|
0,0 → 1,8 |
#ifndef _ASM_X86_DELAY_H |
#define _ASM_X86_DELAY_H |
#include <asm-generic/delay.h> |
void use_tsc_delay(void); |
#endif /* _ASM_X86_DELAY_H */ |
/drivers/include/asm/desc_defs.h |
---|
0,0 → 1,101 |
/* Written 2000 by Andi Kleen */ |
#ifndef _ASM_X86_DESC_DEFS_H |
#define _ASM_X86_DESC_DEFS_H |
/* |
* Segment descriptor structure definitions, usable from both x86_64 and i386 |
* archs. |
*/ |
#ifndef __ASSEMBLY__ |
#include <linux/types.h> |
/* |
* FIXME: Accessing the desc_struct through its fields is more elegant, |
* and should be the one valid thing to do. However, a lot of open code |
* still touches the a and b accessors, and doing this allow us to do it |
* incrementally. We keep the signature as a struct, rather than an union, |
* so we can get rid of it transparently in the future -- glommer |
*/ |
/* 8 byte segment descriptor */ |
struct desc_struct { |
union { |
struct { |
unsigned int a; |
unsigned int b; |
}; |
struct { |
u16 limit0; |
u16 base0; |
unsigned base1: 8, type: 4, s: 1, dpl: 2, p: 1; |
unsigned limit: 4, avl: 1, l: 1, d: 1, g: 1, base2: 8; |
}; |
}; |
} __attribute__((packed)); |
#define GDT_ENTRY_INIT(flags, base, limit) { { { \ |
.a = ((limit) & 0xffff) | (((base) & 0xffff) << 16), \ |
.b = (((base) & 0xff0000) >> 16) | (((flags) & 0xf0ff) << 8) | \ |
((limit) & 0xf0000) | ((base) & 0xff000000), \ |
} } } |
enum { |
GATE_INTERRUPT = 0xE, |
GATE_TRAP = 0xF, |
GATE_CALL = 0xC, |
GATE_TASK = 0x5, |
}; |
/* 16byte gate */ |
struct gate_struct64 { |
u16 offset_low; |
u16 segment; |
unsigned ist : 3, zero0 : 5, type : 5, dpl : 2, p : 1; |
u16 offset_middle; |
u32 offset_high; |
u32 zero1; |
} __attribute__((packed)); |
#define PTR_LOW(x) ((unsigned long long)(x) & 0xFFFF) |
#define PTR_MIDDLE(x) (((unsigned long long)(x) >> 16) & 0xFFFF) |
#define PTR_HIGH(x) ((unsigned long long)(x) >> 32) |
enum { |
DESC_TSS = 0x9, |
DESC_LDT = 0x2, |
DESCTYPE_S = 0x10, /* !system */ |
}; |
/* LDT or TSS descriptor in the GDT. 16 bytes. */ |
struct ldttss_desc64 { |
u16 limit0; |
u16 base0; |
unsigned base1 : 8, type : 5, dpl : 2, p : 1; |
unsigned limit1 : 4, zero0 : 3, g : 1, base2 : 8; |
u32 base3; |
u32 zero1; |
} __attribute__((packed)); |
#ifdef CONFIG_X86_64 |
typedef struct gate_struct64 gate_desc; |
typedef struct ldttss_desc64 ldt_desc; |
typedef struct ldttss_desc64 tss_desc; |
#define gate_offset(g) ((g).offset_low | ((unsigned long)(g).offset_middle << 16) | ((unsigned long)(g).offset_high << 32)) |
#define gate_segment(g) ((g).segment) |
#else |
typedef struct desc_struct gate_desc; |
typedef struct desc_struct ldt_desc; |
typedef struct desc_struct tss_desc; |
#define gate_offset(g) (((g).b & 0xffff0000) | ((g).a & 0x0000ffff)) |
#define gate_segment(g) ((g).a >> 16) |
#endif |
struct desc_ptr { |
unsigned short size; |
unsigned long address; |
} __attribute__((packed)) ; |
#endif /* !__ASSEMBLY__ */ |
#endif /* _ASM_X86_DESC_DEFS_H */ |
/drivers/include/asm/disabled-features.h |
---|
0,0 → 1,45 |
#ifndef _ASM_X86_DISABLED_FEATURES_H |
#define _ASM_X86_DISABLED_FEATURES_H |
/* These features, although they might be available in a CPU |
* will not be used because the compile options to support |
* them are not present. |
* |
* This code allows them to be checked and disabled at |
* compile time without an explicit #ifdef. Use |
* cpu_feature_enabled(). |
*/ |
#ifdef CONFIG_X86_INTEL_MPX |
# define DISABLE_MPX 0 |
#else |
# define DISABLE_MPX (1<<(X86_FEATURE_MPX & 31)) |
#endif |
#ifdef CONFIG_X86_64 |
# define DISABLE_VME (1<<(X86_FEATURE_VME & 31)) |
# define DISABLE_K6_MTRR (1<<(X86_FEATURE_K6_MTRR & 31)) |
# define DISABLE_CYRIX_ARR (1<<(X86_FEATURE_CYRIX_ARR & 31)) |
# define DISABLE_CENTAUR_MCR (1<<(X86_FEATURE_CENTAUR_MCR & 31)) |
#else |
# define DISABLE_VME 0 |
# define DISABLE_K6_MTRR 0 |
# define DISABLE_CYRIX_ARR 0 |
# define DISABLE_CENTAUR_MCR 0 |
#endif /* CONFIG_X86_64 */ |
/* |
* Make sure to add features to the correct mask |
*/ |
#define DISABLED_MASK0 (DISABLE_VME) |
#define DISABLED_MASK1 0 |
#define DISABLED_MASK2 0 |
#define DISABLED_MASK3 (DISABLE_CYRIX_ARR|DISABLE_CENTAUR_MCR|DISABLE_K6_MTRR) |
#define DISABLED_MASK4 0 |
#define DISABLED_MASK5 0 |
#define DISABLED_MASK6 0 |
#define DISABLED_MASK7 0 |
#define DISABLED_MASK8 0 |
#define DISABLED_MASK9 (DISABLE_MPX) |
#endif /* _ASM_X86_DISABLED_FEATURES_H */ |
/drivers/include/asm/div64.h |
---|
0,0 → 1,66 |
#ifndef _ASM_X86_DIV64_H |
#define _ASM_X86_DIV64_H |
#ifdef CONFIG_X86_32 |
#include <linux/types.h> |
#include <linux/log2.h> |
/* |
* do_div() is NOT a C function. It wants to return |
* two values (the quotient and the remainder), but |
* since that doesn't work very well in C, what it |
* does is: |
* |
* - modifies the 64-bit dividend _in_place_ |
* - returns the 32-bit remainder |
* |
* This ends up being the most efficient "calling |
* convention" on x86. |
*/ |
#define do_div(n, base) \ |
({ \ |
unsigned long __upper, __low, __high, __mod, __base; \ |
__base = (base); \ |
if (__builtin_constant_p(__base) && is_power_of_2(__base)) { \ |
__mod = n & (__base - 1); \ |
n >>= ilog2(__base); \ |
} else { \ |
asm("" : "=a" (__low), "=d" (__high) : "A" (n));\ |
__upper = __high; \ |
if (__high) { \ |
__upper = __high % (__base); \ |
__high = __high / (__base); \ |
} \ |
asm("divl %2" : "=a" (__low), "=d" (__mod) \ |
: "rm" (__base), "0" (__low), "1" (__upper)); \ |
asm("" : "=A" (n) : "a" (__low), "d" (__high)); \ |
} \ |
__mod; \ |
}) |
static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) |
{ |
union { |
u64 v64; |
u32 v32[2]; |
} d = { dividend }; |
u32 upper; |
upper = d.v32[1]; |
d.v32[1] = 0; |
if (upper >= divisor) { |
d.v32[1] = upper / divisor; |
upper %= divisor; |
} |
asm ("divl %2" : "=a" (d.v32[0]), "=d" (*remainder) : |
"rm" (divisor), "0" (d.v32[0]), "1" (upper)); |
return d.v64; |
} |
#define div_u64_rem div_u64_rem |
#else |
# include <asm-generic/div64.h> |
#endif /* CONFIG_X86_32 */ |
#endif /* _ASM_X86_DIV64_H */ |
/drivers/include/asm/e820.h |
---|
0,0 → 1,77 |
#ifndef _ASM_X86_E820_H |
#define _ASM_X86_E820_H |
#ifdef CONFIG_EFI |
#include <linux/numa.h> |
#define E820_X_MAX (E820MAX + 3 * MAX_NUMNODES) |
#else /* ! CONFIG_EFI */ |
#define E820_X_MAX E820MAX |
#endif |
#include <uapi/asm/e820.h> |
#ifndef __ASSEMBLY__ |
/* see comment in arch/x86/kernel/e820.c */ |
extern struct e820map e820; |
extern struct e820map e820_saved; |
extern unsigned long pci_mem_start; |
extern int e820_any_mapped(u64 start, u64 end, unsigned type); |
extern int e820_all_mapped(u64 start, u64 end, unsigned type); |
extern void e820_add_region(u64 start, u64 size, int type); |
extern void e820_print_map(char *who); |
extern int |
sanitize_e820_map(struct e820entry *biosmap, int max_nr_map, u32 *pnr_map); |
extern u64 e820_update_range(u64 start, u64 size, unsigned old_type, |
unsigned new_type); |
extern u64 e820_remove_range(u64 start, u64 size, unsigned old_type, |
int checktype); |
extern void update_e820(void); |
extern void e820_setup_gap(void); |
extern int e820_search_gap(unsigned long *gapstart, unsigned long *gapsize, |
unsigned long start_addr, unsigned long long end_addr); |
struct setup_data; |
extern void parse_e820_ext(u64 phys_addr, u32 data_len); |
#if defined(CONFIG_X86_64) || \ |
(defined(CONFIG_X86_32) && defined(CONFIG_HIBERNATION)) |
extern void e820_mark_nosave_regions(unsigned long limit_pfn); |
#else |
static inline void e820_mark_nosave_regions(unsigned long limit_pfn) |
{ |
} |
#endif |
#ifdef CONFIG_MEMTEST |
extern void early_memtest(unsigned long start, unsigned long end); |
#else |
static inline void early_memtest(unsigned long start, unsigned long end) |
{ |
} |
#endif |
extern unsigned long e820_end_of_ram_pfn(void); |
extern unsigned long e820_end_of_low_ram_pfn(void); |
extern u64 early_reserve_e820(u64 sizet, u64 align); |
void memblock_x86_fill(void); |
void memblock_find_dma_reserve(void); |
extern void finish_e820_parsing(void); |
extern void e820_reserve_resources(void); |
extern void e820_reserve_resources_late(void); |
extern void setup_memory_map(void); |
extern char *default_machine_specific_memory_setup(void); |
/* |
* Returns true iff the specified range [s,e) is completely contained inside |
* the ISA region. |
*/ |
static inline bool is_ISA_range(u64 s, u64 e) |
{ |
return s >= ISA_START_ADDRESS && e <= ISA_END_ADDRESS; |
} |
#endif /* __ASSEMBLY__ */ |
#include <linux/ioport.h> |
#define HIGH_MEMORY (1024*1024) |
#endif /* _ASM_X86_E820_H */ |
/drivers/include/asm/irqflags.h |
---|
0,0 → 1,209 |
#ifndef _X86_IRQFLAGS_H_ |
#define _X86_IRQFLAGS_H_ |
#include <asm/processor-flags.h> |
#ifndef __ASSEMBLY__ |
/* |
* Interrupt control: |
*/ |
static inline unsigned long native_save_fl(void) |
{ |
unsigned long flags; |
/* |
* "=rm" is safe here, because "pop" adjusts the stack before |
* it evaluates its effective address -- this is part of the |
* documented behavior of the "pop" instruction. |
*/ |
asm volatile("# __raw_save_flags\n\t" |
"pushf ; pop %0" |
: "=rm" (flags) |
: /* no input */ |
: "memory"); |
return flags; |
} |
static inline void native_restore_fl(unsigned long flags) |
{ |
asm volatile("push %0 ; popf" |
: /* no output */ |
:"g" (flags) |
:"memory", "cc"); |
} |
static inline void native_irq_disable(void) |
{ |
asm volatile("cli": : :"memory"); |
} |
static inline void native_irq_enable(void) |
{ |
asm volatile("sti": : :"memory"); |
} |
static inline void native_safe_halt(void) |
{ |
asm volatile("sti; hlt": : :"memory"); |
} |
static inline void native_halt(void) |
{ |
asm volatile("hlt": : :"memory"); |
} |
#endif |
#ifdef CONFIG_PARAVIRT |
#include <asm/paravirt.h> |
#else |
#ifndef __ASSEMBLY__ |
#include <linux/types.h> |
static inline notrace unsigned long arch_local_save_flags(void) |
{ |
return native_save_fl(); |
} |
static inline notrace void arch_local_irq_restore(unsigned long flags) |
{ |
native_restore_fl(flags); |
} |
static inline notrace void arch_local_irq_disable(void) |
{ |
native_irq_disable(); |
} |
static inline notrace void arch_local_irq_enable(void) |
{ |
native_irq_enable(); |
} |
/* |
* Used in the idle loop; sti takes one instruction cycle |
* to complete: |
*/ |
static inline void arch_safe_halt(void) |
{ |
native_safe_halt(); |
} |
/* |
* Used when interrupts are already enabled or to |
* shutdown the processor: |
*/ |
static inline void halt(void) |
{ |
native_halt(); |
} |
/* |
* For spinlocks, etc: |
*/ |
static inline notrace unsigned long arch_local_irq_save(void) |
{ |
unsigned long flags = arch_local_save_flags(); |
arch_local_irq_disable(); |
return flags; |
} |
#else |
#define ENABLE_INTERRUPTS(x) sti |
#define DISABLE_INTERRUPTS(x) cli |
#ifdef CONFIG_X86_64 |
#define SWAPGS swapgs |
/* |
* Currently paravirt can't handle swapgs nicely when we |
* don't have a stack we can rely on (such as a user space |
* stack). So we either find a way around these or just fault |
* and emulate if a guest tries to call swapgs directly. |
* |
* Either way, this is a good way to document that we don't |
* have a reliable stack. x86_64 only. |
*/ |
#define SWAPGS_UNSAFE_STACK swapgs |
#define PARAVIRT_ADJUST_EXCEPTION_FRAME /* */ |
#define INTERRUPT_RETURN jmp native_iret |
#define USERGS_SYSRET64 \ |
swapgs; \ |
sysretq; |
#define USERGS_SYSRET32 \ |
swapgs; \ |
sysretl |
#define ENABLE_INTERRUPTS_SYSEXIT32 \ |
swapgs; \ |
sti; \ |
sysexit |
#else |
#define INTERRUPT_RETURN iret |
#define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit |
#define GET_CR0_INTO_EAX movl %cr0, %eax |
#endif |
#endif /* __ASSEMBLY__ */ |
#endif /* CONFIG_PARAVIRT */ |
#ifndef __ASSEMBLY__ |
static inline int arch_irqs_disabled_flags(unsigned long flags) |
{ |
return !(flags & X86_EFLAGS_IF); |
} |
static inline int arch_irqs_disabled(void) |
{ |
unsigned long flags = arch_local_save_flags(); |
return arch_irqs_disabled_flags(flags); |
} |
#else |
#ifdef CONFIG_X86_64 |
#define ARCH_LOCKDEP_SYS_EXIT call lockdep_sys_exit_thunk |
#define ARCH_LOCKDEP_SYS_EXIT_IRQ \ |
TRACE_IRQS_ON; \ |
sti; \ |
SAVE_REST; \ |
LOCKDEP_SYS_EXIT; \ |
RESTORE_REST; \ |
cli; \ |
TRACE_IRQS_OFF; |
#else |
#define ARCH_LOCKDEP_SYS_EXIT \ |
pushl %eax; \ |
pushl %ecx; \ |
pushl %edx; \ |
call lockdep_sys_exit; \ |
popl %edx; \ |
popl %ecx; \ |
popl %eax; |
#define ARCH_LOCKDEP_SYS_EXIT_IRQ |
#endif |
#ifdef CONFIG_TRACE_IRQFLAGS |
# define TRACE_IRQS_ON call trace_hardirqs_on_thunk; |
# define TRACE_IRQS_OFF call trace_hardirqs_off_thunk; |
#else |
# define TRACE_IRQS_ON |
# define TRACE_IRQS_OFF |
#endif |
#ifdef CONFIG_DEBUG_LOCK_ALLOC |
# define LOCKDEP_SYS_EXIT ARCH_LOCKDEP_SYS_EXIT |
# define LOCKDEP_SYS_EXIT_IRQ ARCH_LOCKDEP_SYS_EXIT_IRQ |
# else |
# define LOCKDEP_SYS_EXIT |
# define LOCKDEP_SYS_EXIT_IRQ |
# endif |
#endif /* __ASSEMBLY__ */ |
#endif |
/drivers/include/asm/linkage.h |
---|
0,0 → 1,61 |
#ifndef _ASM_X86_LINKAGE_H |
#define _ASM_X86_LINKAGE_H |
#include <linux/stringify.h> |
#undef notrace |
#define notrace __attribute__((no_instrument_function)) |
#ifdef CONFIG_X86_32 |
#define asmlinkage CPP_ASMLINKAGE __attribute__((regparm(0))) |
/* |
* Make sure the compiler doesn't do anything stupid with the |
* arguments on the stack - they are owned by the *caller*, not |
* the callee. This just fools gcc into not spilling into them, |
* and keeps it from doing tailcall recursion and/or using the |
* stack slots for temporaries, since they are live and "used" |
* all the way to the end of the function. |
* |
* NOTE! On x86-64, all the arguments are in registers, so this |
* only matters on a 32-bit kernel. |
*/ |
#define asmlinkage_protect(n, ret, args...) \ |
__asmlinkage_protect##n(ret, ##args) |
#define __asmlinkage_protect_n(ret, args...) \ |
__asm__ __volatile__ ("" : "=r" (ret) : "0" (ret), ##args) |
#define __asmlinkage_protect0(ret) \ |
__asmlinkage_protect_n(ret) |
#define __asmlinkage_protect1(ret, arg1) \ |
__asmlinkage_protect_n(ret, "m" (arg1)) |
#define __asmlinkage_protect2(ret, arg1, arg2) \ |
__asmlinkage_protect_n(ret, "m" (arg1), "m" (arg2)) |
#define __asmlinkage_protect3(ret, arg1, arg2, arg3) \ |
__asmlinkage_protect_n(ret, "m" (arg1), "m" (arg2), "m" (arg3)) |
#define __asmlinkage_protect4(ret, arg1, arg2, arg3, arg4) \ |
__asmlinkage_protect_n(ret, "m" (arg1), "m" (arg2), "m" (arg3), \ |
"m" (arg4)) |
#define __asmlinkage_protect5(ret, arg1, arg2, arg3, arg4, arg5) \ |
__asmlinkage_protect_n(ret, "m" (arg1), "m" (arg2), "m" (arg3), \ |
"m" (arg4), "m" (arg5)) |
#define __asmlinkage_protect6(ret, arg1, arg2, arg3, arg4, arg5, arg6) \ |
__asmlinkage_protect_n(ret, "m" (arg1), "m" (arg2), "m" (arg3), \ |
"m" (arg4), "m" (arg5), "m" (arg6)) |
#endif /* CONFIG_X86_32 */ |
#ifdef __ASSEMBLY__ |
#define GLOBAL(name) \ |
.globl name; \ |
name: |
#if defined(CONFIG_X86_64) || defined(CONFIG_X86_ALIGNMENT_16) |
#define __ALIGN .p2align 4, 0x90 |
#define __ALIGN_STR __stringify(__ALIGN) |
#endif |
#endif /* __ASSEMBLY__ */ |
#endif /* _ASM_X86_LINKAGE_H */ |
/drivers/include/asm/math_emu.h |
---|
0,0 → 1,18 |
#ifndef _ASM_X86_MATH_EMU_H |
#define _ASM_X86_MATH_EMU_H |
#include <asm/ptrace.h> |
#include <asm/vm86.h> |
/* This structure matches the layout of the data saved to the stack |
following a device-not-present interrupt, part of it saved |
automatically by the 80386/80486. |
*/ |
struct math_emu_info { |
long ___orig_eip; |
union { |
struct pt_regs *regs; |
struct kernel_vm86_regs *vm86; |
}; |
}; |
#endif /* _ASM_X86_MATH_EMU_H */ |
/drivers/include/asm/msr.h |
---|
0,0 → 1,291 |
#ifndef _ASM_X86_MSR_H |
#define _ASM_X86_MSR_H |
#include <uapi/asm/msr.h> |
#ifndef __ASSEMBLY__ |
#include <asm/asm.h> |
#include <asm/errno.h> |
#include <asm/cpumask.h> |
struct msr { |
union { |
struct { |
u32 l; |
u32 h; |
}; |
u64 q; |
}; |
}; |
struct msr_info { |
u32 msr_no; |
struct msr reg; |
struct msr *msrs; |
int err; |
}; |
struct msr_regs_info { |
u32 *regs; |
int err; |
}; |
static inline unsigned long long native_read_tscp(unsigned int *aux) |
{ |
unsigned long low, high; |
asm volatile(".byte 0x0f,0x01,0xf9" |
: "=a" (low), "=d" (high), "=c" (*aux)); |
return low | ((u64)high << 32); |
} |
/* |
* both i386 and x86_64 returns 64-bit value in edx:eax, but gcc's "A" |
* constraint has different meanings. For i386, "A" means exactly |
* edx:eax, while for x86_64 it doesn't mean rdx:rax or edx:eax. Instead, |
* it means rax *or* rdx. |
*/ |
#ifdef CONFIG_X86_64 |
#define DECLARE_ARGS(val, low, high) unsigned low, high |
#define EAX_EDX_VAL(val, low, high) ((low) | ((u64)(high) << 32)) |
#define EAX_EDX_ARGS(val, low, high) "a" (low), "d" (high) |
#define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high) |
#else |
#define DECLARE_ARGS(val, low, high) unsigned long long val |
#define EAX_EDX_VAL(val, low, high) (val) |
#define EAX_EDX_ARGS(val, low, high) "A" (val) |
#define EAX_EDX_RET(val, low, high) "=A" (val) |
#endif |
static inline unsigned long long native_read_msr(unsigned int msr) |
{ |
DECLARE_ARGS(val, low, high); |
asm volatile("rdmsr" : EAX_EDX_RET(val, low, high) : "c" (msr)); |
return EAX_EDX_VAL(val, low, high); |
} |
static inline unsigned long long native_read_msr_safe(unsigned int msr, |
int *err) |
{ |
DECLARE_ARGS(val, low, high); |
asm volatile("2: rdmsr ; xor %[err],%[err]\n" |
"1:\n\t" |
".section .fixup,\"ax\"\n\t" |
"3: mov %[fault],%[err] ; jmp 1b\n\t" |
".previous\n\t" |
_ASM_EXTABLE(2b, 3b) |
: [err] "=r" (*err), EAX_EDX_RET(val, low, high) |
: "c" (msr), [fault] "i" (-EIO)); |
return EAX_EDX_VAL(val, low, high); |
} |
static inline void native_write_msr(unsigned int msr, |
unsigned low, unsigned high) |
{ |
asm volatile("wrmsr" : : "c" (msr), "a"(low), "d" (high) : "memory"); |
} |
/* Can be uninlined because referenced by paravirt */ |
notrace static inline int native_write_msr_safe(unsigned int msr, |
unsigned low, unsigned high) |
{ |
int err; |
asm volatile("2: wrmsr ; xor %[err],%[err]\n" |
"1:\n\t" |
".section .fixup,\"ax\"\n\t" |
"3: mov %[fault],%[err] ; jmp 1b\n\t" |
".previous\n\t" |
_ASM_EXTABLE(2b, 3b) |
: [err] "=a" (err) |
: "c" (msr), "0" (low), "d" (high), |
[fault] "i" (-EIO) |
: "memory"); |
return err; |
} |
extern unsigned long long native_read_tsc(void); |
extern int rdmsr_safe_regs(u32 regs[8]); |
extern int wrmsr_safe_regs(u32 regs[8]); |
static __always_inline unsigned long long __native_read_tsc(void) |
{ |
DECLARE_ARGS(val, low, high); |
asm volatile("rdtsc" : EAX_EDX_RET(val, low, high)); |
return EAX_EDX_VAL(val, low, high); |
} |
static inline unsigned long long native_read_pmc(int counter) |
{ |
DECLARE_ARGS(val, low, high); |
asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter)); |
return EAX_EDX_VAL(val, low, high); |
} |
#ifdef CONFIG_PARAVIRT |
#include <asm/paravirt.h> |
#else |
#include <linux/errno.h> |
/* |
* Access to machine-specific registers (available on 586 and better only) |
* Note: the rd* operations modify the parameters directly (without using |
* pointer indirection), this allows gcc to optimize better |
*/ |
#define rdmsr(msr, low, high) \ |
do { \ |
u64 __val = native_read_msr((msr)); \ |
(void)((low) = (u32)__val); \ |
(void)((high) = (u32)(__val >> 32)); \ |
} while (0) |
static inline void wrmsr(unsigned msr, unsigned low, unsigned high) |
{ |
native_write_msr(msr, low, high); |
} |
#define rdmsrl(msr, val) \ |
((val) = native_read_msr((msr))) |
#define wrmsrl(msr, val) \ |
native_write_msr((msr), (u32)((u64)(val)), (u32)((u64)(val) >> 32)) |
/* wrmsr with exception handling */ |
static inline int wrmsr_safe(unsigned msr, unsigned low, unsigned high) |
{ |
return native_write_msr_safe(msr, low, high); |
} |
/* rdmsr with exception handling */ |
#define rdmsr_safe(msr, low, high) \ |
({ \ |
int __err; \ |
u64 __val = native_read_msr_safe((msr), &__err); \ |
(*low) = (u32)__val; \ |
(*high) = (u32)(__val >> 32); \ |
__err; \ |
}) |
static inline int rdmsrl_safe(unsigned msr, unsigned long long *p) |
{ |
int err; |
*p = native_read_msr_safe(msr, &err); |
return err; |
} |
#define rdtscl(low) \ |
((low) = (u32)__native_read_tsc()) |
#define rdtscll(val) \ |
((val) = __native_read_tsc()) |
#define rdpmc(counter, low, high) \ |
do { \ |
u64 _l = native_read_pmc((counter)); \ |
(low) = (u32)_l; \ |
(high) = (u32)(_l >> 32); \ |
} while (0) |
#define rdpmcl(counter, val) ((val) = native_read_pmc(counter)) |
#define rdtscp(low, high, aux) \ |
do { \ |
unsigned long long _val = native_read_tscp(&(aux)); \ |
(low) = (u32)_val; \ |
(high) = (u32)(_val >> 32); \ |
} while (0) |
#define rdtscpll(val, aux) (val) = native_read_tscp(&(aux)) |
#endif /* !CONFIG_PARAVIRT */ |
#define wrmsrl_safe(msr, val) wrmsr_safe((msr), (u32)(val), \ |
(u32)((val) >> 32)) |
#define write_tsc(low, high) wrmsr(MSR_IA32_TSC, (low), (high)) |
#define write_rdtscp_aux(val) wrmsr(MSR_TSC_AUX, (val), 0) |
struct msr *msrs_alloc(void); |
void msrs_free(struct msr *msrs); |
int msr_set_bit(u32 msr, u8 bit); |
int msr_clear_bit(u32 msr, u8 bit); |
#ifdef CONFIG_SMP |
int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); |
int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); |
int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); |
int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q); |
void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs); |
void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs); |
int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); |
int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); |
int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q); |
int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q); |
int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); |
int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); |
#else /* CONFIG_SMP */ |
static inline int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h) |
{ |
rdmsr(msr_no, *l, *h); |
return 0; |
} |
static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) |
{ |
wrmsr(msr_no, l, h); |
return 0; |
} |
static inline int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) |
{ |
rdmsrl(msr_no, *q); |
return 0; |
} |
static inline int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q) |
{ |
wrmsrl(msr_no, q); |
return 0; |
} |
static inline void rdmsr_on_cpus(const struct cpumask *m, u32 msr_no, |
struct msr *msrs) |
{ |
rdmsr_on_cpu(0, msr_no, &(msrs[0].l), &(msrs[0].h)); |
} |
static inline void wrmsr_on_cpus(const struct cpumask *m, u32 msr_no, |
struct msr *msrs) |
{ |
wrmsr_on_cpu(0, msr_no, msrs[0].l, msrs[0].h); |
} |
static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, |
u32 *l, u32 *h) |
{ |
return rdmsr_safe(msr_no, l, h); |
} |
static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h) |
{ |
return wrmsr_safe(msr_no, l, h); |
} |
static inline int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q) |
{ |
return rdmsrl_safe(msr_no, q); |
} |
static inline int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q) |
{ |
return wrmsrl_safe(msr_no, q); |
} |
static inline int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) |
{ |
return rdmsr_safe_regs(regs); |
} |
static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]) |
{ |
return wrmsr_safe_regs(regs); |
} |
#endif /* CONFIG_SMP */ |
#endif /* __ASSEMBLY__ */ |
#endif /* _ASM_X86_MSR_H */ |
/drivers/include/asm/nops.h |
---|
0,0 → 1,146 |
#ifndef _ASM_X86_NOPS_H |
#define _ASM_X86_NOPS_H |
/* |
* Define nops for use with alternative() and for tracing. |
* |
* *_NOP5_ATOMIC must be a single instruction. |
*/ |
#define NOP_DS_PREFIX 0x3e |
/* generic versions from gas |
1: nop |
the following instructions are NOT nops in 64-bit mode, |
for 64-bit mode use K8 or P6 nops instead |
2: movl %esi,%esi |
3: leal 0x00(%esi),%esi |
4: leal 0x00(,%esi,1),%esi |
6: leal 0x00000000(%esi),%esi |
7: leal 0x00000000(,%esi,1),%esi |
*/ |
#define GENERIC_NOP1 0x90 |
#define GENERIC_NOP2 0x89,0xf6 |
#define GENERIC_NOP3 0x8d,0x76,0x00 |
#define GENERIC_NOP4 0x8d,0x74,0x26,0x00 |
#define GENERIC_NOP5 GENERIC_NOP1,GENERIC_NOP4 |
#define GENERIC_NOP6 0x8d,0xb6,0x00,0x00,0x00,0x00 |
#define GENERIC_NOP7 0x8d,0xb4,0x26,0x00,0x00,0x00,0x00 |
#define GENERIC_NOP8 GENERIC_NOP1,GENERIC_NOP7 |
#define GENERIC_NOP5_ATOMIC NOP_DS_PREFIX,GENERIC_NOP4 |
/* Opteron 64bit nops |
1: nop |
2: osp nop |
3: osp osp nop |
4: osp osp osp nop |
*/ |
#define K8_NOP1 GENERIC_NOP1 |
#define K8_NOP2 0x66,K8_NOP1 |
#define K8_NOP3 0x66,K8_NOP2 |
#define K8_NOP4 0x66,K8_NOP3 |
#define K8_NOP5 K8_NOP3,K8_NOP2 |
#define K8_NOP6 K8_NOP3,K8_NOP3 |
#define K8_NOP7 K8_NOP4,K8_NOP3 |
#define K8_NOP8 K8_NOP4,K8_NOP4 |
#define K8_NOP5_ATOMIC 0x66,K8_NOP4 |
/* K7 nops |
uses eax dependencies (arbitrary choice) |
1: nop |
2: movl %eax,%eax |
3: leal (,%eax,1),%eax |
4: leal 0x00(,%eax,1),%eax |
6: leal 0x00000000(%eax),%eax |
7: leal 0x00000000(,%eax,1),%eax |
*/ |
#define K7_NOP1 GENERIC_NOP1 |
#define K7_NOP2 0x8b,0xc0 |
#define K7_NOP3 0x8d,0x04,0x20 |
#define K7_NOP4 0x8d,0x44,0x20,0x00 |
#define K7_NOP5 K7_NOP4,K7_NOP1 |
#define K7_NOP6 0x8d,0x80,0,0,0,0 |
#define K7_NOP7 0x8D,0x04,0x05,0,0,0,0 |
#define K7_NOP8 K7_NOP7,K7_NOP1 |
#define K7_NOP5_ATOMIC NOP_DS_PREFIX,K7_NOP4 |
/* P6 nops |
uses eax dependencies (Intel-recommended choice) |
1: nop |
2: osp nop |
3: nopl (%eax) |
4: nopl 0x00(%eax) |
5: nopl 0x00(%eax,%eax,1) |
6: osp nopl 0x00(%eax,%eax,1) |
7: nopl 0x00000000(%eax) |
8: nopl 0x00000000(%eax,%eax,1) |
Note: All the above are assumed to be a single instruction. |
There is kernel code that depends on this. |
*/ |
#define P6_NOP1 GENERIC_NOP1 |
#define P6_NOP2 0x66,0x90 |
#define P6_NOP3 0x0f,0x1f,0x00 |
#define P6_NOP4 0x0f,0x1f,0x40,0 |
#define P6_NOP5 0x0f,0x1f,0x44,0x00,0 |
#define P6_NOP6 0x66,0x0f,0x1f,0x44,0x00,0 |
#define P6_NOP7 0x0f,0x1f,0x80,0,0,0,0 |
#define P6_NOP8 0x0f,0x1f,0x84,0x00,0,0,0,0 |
#define P6_NOP5_ATOMIC P6_NOP5 |
#ifdef __ASSEMBLY__ |
#define _ASM_MK_NOP(x) .byte x |
#else |
#define _ASM_MK_NOP(x) ".byte " __stringify(x) "\n" |
#endif |
#if defined(CONFIG_MK7) |
#define ASM_NOP1 _ASM_MK_NOP(K7_NOP1) |
#define ASM_NOP2 _ASM_MK_NOP(K7_NOP2) |
#define ASM_NOP3 _ASM_MK_NOP(K7_NOP3) |
#define ASM_NOP4 _ASM_MK_NOP(K7_NOP4) |
#define ASM_NOP5 _ASM_MK_NOP(K7_NOP5) |
#define ASM_NOP6 _ASM_MK_NOP(K7_NOP6) |
#define ASM_NOP7 _ASM_MK_NOP(K7_NOP7) |
#define ASM_NOP8 _ASM_MK_NOP(K7_NOP8) |
#define ASM_NOP5_ATOMIC _ASM_MK_NOP(K7_NOP5_ATOMIC) |
#elif defined(CONFIG_X86_P6_NOP) |
#define ASM_NOP1 _ASM_MK_NOP(P6_NOP1) |
#define ASM_NOP2 _ASM_MK_NOP(P6_NOP2) |
#define ASM_NOP3 _ASM_MK_NOP(P6_NOP3) |
#define ASM_NOP4 _ASM_MK_NOP(P6_NOP4) |
#define ASM_NOP5 _ASM_MK_NOP(P6_NOP5) |
#define ASM_NOP6 _ASM_MK_NOP(P6_NOP6) |
#define ASM_NOP7 _ASM_MK_NOP(P6_NOP7) |
#define ASM_NOP8 _ASM_MK_NOP(P6_NOP8) |
#define ASM_NOP5_ATOMIC _ASM_MK_NOP(P6_NOP5_ATOMIC) |
#elif defined(CONFIG_X86_64) |
#define ASM_NOP1 _ASM_MK_NOP(K8_NOP1) |
#define ASM_NOP2 _ASM_MK_NOP(K8_NOP2) |
#define ASM_NOP3 _ASM_MK_NOP(K8_NOP3) |
#define ASM_NOP4 _ASM_MK_NOP(K8_NOP4) |
#define ASM_NOP5 _ASM_MK_NOP(K8_NOP5) |
#define ASM_NOP6 _ASM_MK_NOP(K8_NOP6) |
#define ASM_NOP7 _ASM_MK_NOP(K8_NOP7) |
#define ASM_NOP8 _ASM_MK_NOP(K8_NOP8) |
#define ASM_NOP5_ATOMIC _ASM_MK_NOP(K8_NOP5_ATOMIC) |
#else |
#define ASM_NOP1 _ASM_MK_NOP(GENERIC_NOP1) |
#define ASM_NOP2 _ASM_MK_NOP(GENERIC_NOP2) |
#define ASM_NOP3 _ASM_MK_NOP(GENERIC_NOP3) |
#define ASM_NOP4 _ASM_MK_NOP(GENERIC_NOP4) |
#define ASM_NOP5 _ASM_MK_NOP(GENERIC_NOP5) |
#define ASM_NOP6 _ASM_MK_NOP(GENERIC_NOP6) |
#define ASM_NOP7 _ASM_MK_NOP(GENERIC_NOP7) |
#define ASM_NOP8 _ASM_MK_NOP(GENERIC_NOP8) |
#define ASM_NOP5_ATOMIC _ASM_MK_NOP(GENERIC_NOP5_ATOMIC) |
#endif |
#define ASM_NOP_MAX 8 |
#define NOP_ATOMIC5 (ASM_NOP_MAX+1) /* Entry for the 5-byte atomic NOP */ |
#ifndef __ASSEMBLY__ |
extern const unsigned char * const *ideal_nops; |
extern void arch_init_ideal_nops(void); |
#endif |
#endif /* _ASM_X86_NOPS_H */ |
/drivers/include/asm/page.h |
---|
0,0 → 1,76 |
#ifndef _ASM_X86_PAGE_H |
#define _ASM_X86_PAGE_H |
#include <linux/types.h> |
#ifdef __KERNEL__ |
#include <asm/page_types.h> |
#ifdef CONFIG_X86_64 |
#include <asm/page_64.h> |
#else |
#include <asm/page_32.h> |
#endif /* CONFIG_X86_64 */ |
#ifndef __ASSEMBLY__ |
struct page; |
#include <linux/range.h> |
extern struct range pfn_mapped[]; |
extern int nr_pfn_mapped; |
static inline void clear_user_page(void *page, unsigned long vaddr, |
struct page *pg) |
{ |
clear_page(page); |
} |
static inline void copy_user_page(void *to, void *from, unsigned long vaddr, |
struct page *topage) |
{ |
copy_page(to, from); |
} |
#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \ |
alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr) |
#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE |
#define __pa(x) __phys_addr((unsigned long)(x)) |
#define __pa_nodebug(x) __phys_addr_nodebug((unsigned long)(x)) |
/* __pa_symbol should be used for C visible symbols. |
This seems to be the official gcc blessed way to do such arithmetic. */ |
/* |
* We need __phys_reloc_hide() here because gcc may assume that there is no |
* overflow during __pa() calculation and can optimize it unexpectedly. |
* Newer versions of gcc provide -fno-strict-overflow switch to handle this |
* case properly. Once all supported versions of gcc understand it, we can |
* remove this Voodoo magic stuff. (i.e. once gcc3.x is deprecated) |
*/ |
#define __pa_symbol(x) \ |
__phys_addr_symbol(__phys_reloc_hide((unsigned long)(x))) |
#define __va(x) ((void *)((unsigned long)(x)+PAGE_OFFSET)) |
#define __boot_va(x) __va(x) |
#define __boot_pa(x) __pa(x) |
/* |
* virt_to_page(kaddr) returns a valid pointer if and only if |
* virt_addr_valid(kaddr) returns true. |
*/ |
#define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) |
#define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT) |
extern bool __virt_addr_valid(unsigned long kaddr); |
#define virt_addr_valid(kaddr) __virt_addr_valid((unsigned long) (kaddr)) |
#endif /* __ASSEMBLY__ */ |
#include <asm-generic/memory_model.h> |
#include <asm-generic/getorder.h> |
#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA |
#endif /* __KERNEL__ */ |
#endif /* _ASM_X86_PAGE_H */ |
/drivers/include/asm/page_32.h |
---|
0,0 → 1,48 |
#ifndef _ASM_X86_PAGE_32_H |
#define _ASM_X86_PAGE_32_H |
#include <asm/page_32_types.h> |
#ifndef __ASSEMBLY__ |
#define __phys_addr_nodebug(x) ((x) - PAGE_OFFSET) |
#ifdef CONFIG_DEBUG_VIRTUAL |
extern unsigned long __phys_addr(unsigned long); |
#else |
#define __phys_addr(x) __phys_addr_nodebug(x) |
#endif |
#define __phys_addr_symbol(x) __phys_addr(x) |
#define __phys_reloc_hide(x) RELOC_HIDE((x), 0) |
#ifdef CONFIG_FLATMEM |
#define pfn_valid(pfn) ((pfn) < max_mapnr) |
#endif /* CONFIG_FLATMEM */ |
#ifdef CONFIG_X86_USE_3DNOW |
#include <asm/mmx.h> |
static inline void clear_page(void *page) |
{ |
mmx_clear_page(page); |
} |
static inline void copy_page(void *to, void *from) |
{ |
mmx_copy_page(to, from); |
} |
#else /* !CONFIG_X86_USE_3DNOW */ |
#include <linux/string.h> |
static inline void clear_page(void *page) |
{ |
memset(page, 0, PAGE_SIZE); |
} |
static inline void copy_page(void *to, void *from) |
{ |
memcpy(to, from, PAGE_SIZE); |
} |
#endif /* CONFIG_X86_3DNOW */ |
#endif /* !__ASSEMBLY__ */ |
#endif /* _ASM_X86_PAGE_32_H */ |
/drivers/include/asm/percpu.h |
---|
0,0 → 1,620 |
#ifndef _ASM_X86_PERCPU_H |
#define _ASM_X86_PERCPU_H |
#ifdef CONFIG_X86_64 |
#define __percpu_seg gs |
#define __percpu_mov_op movq |
#else |
#define __percpu_seg fs |
#define __percpu_mov_op movl |
#endif |
#ifdef __ASSEMBLY__ |
/* |
* PER_CPU finds an address of a per-cpu variable. |
* |
* Args: |
* var - variable name |
* reg - 32bit register |
* |
* The resulting address is stored in the "reg" argument. |
* |
* Example: |
* PER_CPU(cpu_gdt_descr, %ebx) |
*/ |
#ifdef CONFIG_SMP |
#define PER_CPU(var, reg) \ |
__percpu_mov_op %__percpu_seg:this_cpu_off, reg; \ |
lea var(reg), reg |
#define PER_CPU_VAR(var) %__percpu_seg:var |
#else /* ! SMP */ |
#define PER_CPU(var, reg) __percpu_mov_op $var, reg |
#define PER_CPU_VAR(var) var |
#endif /* SMP */ |
#ifdef CONFIG_X86_64_SMP |
#define INIT_PER_CPU_VAR(var) init_per_cpu__##var |
#else |
#define INIT_PER_CPU_VAR(var) var |
#endif |
#else /* ...!ASSEMBLY */ |
#include <linux/kernel.h> |
#include <linux/stringify.h> |
#ifdef CONFIG_SMP |
#define __percpu_prefix "%%"__stringify(__percpu_seg)":" |
#define __my_cpu_offset this_cpu_read(this_cpu_off) |
/* |
* Compared to the generic __my_cpu_offset version, the following |
* saves one instruction and avoids clobbering a temp register. |
*/ |
#define arch_raw_cpu_ptr(ptr) \ |
({ \ |
unsigned long tcp_ptr__; \ |
asm volatile("add " __percpu_arg(1) ", %0" \ |
: "=r" (tcp_ptr__) \ |
: "m" (this_cpu_off), "0" (ptr)); \ |
(typeof(*(ptr)) __kernel __force *)tcp_ptr__; \ |
}) |
#else |
#define __percpu_prefix "" |
#endif |
#define __percpu_arg(x) __percpu_prefix "%" #x |
/* |
* Initialized pointers to per-cpu variables needed for the boot |
* processor need to use these macros to get the proper address |
* offset from __per_cpu_load on SMP. |
* |
* There also must be an entry in vmlinux_64.lds.S |
*/ |
#define DECLARE_INIT_PER_CPU(var) \ |
extern typeof(var) init_per_cpu_var(var) |
#ifdef CONFIG_X86_64_SMP |
#define init_per_cpu_var(var) init_per_cpu__##var |
#else |
#define init_per_cpu_var(var) var |
#endif |
/* For arch-specific code, we can use direct single-insn ops (they |
* don't give an lvalue though). */ |
extern void __bad_percpu_size(void); |
#define percpu_to_op(op, var, val) \ |
do { \ |
typedef typeof(var) pto_T__; \ |
if (0) { \ |
pto_T__ pto_tmp__; \ |
pto_tmp__ = (val); \ |
(void)pto_tmp__; \ |
} \ |
switch (sizeof(var)) { \ |
case 1: \ |
asm(op "b %1,"__percpu_arg(0) \ |
: "+m" (var) \ |
: "qi" ((pto_T__)(val))); \ |
break; \ |
case 2: \ |
asm(op "w %1,"__percpu_arg(0) \ |
: "+m" (var) \ |
: "ri" ((pto_T__)(val))); \ |
break; \ |
case 4: \ |
asm(op "l %1,"__percpu_arg(0) \ |
: "+m" (var) \ |
: "ri" ((pto_T__)(val))); \ |
break; \ |
case 8: \ |
asm(op "q %1,"__percpu_arg(0) \ |
: "+m" (var) \ |
: "re" ((pto_T__)(val))); \ |
break; \ |
default: __bad_percpu_size(); \ |
} \ |
} while (0) |
/* |
* Generate a percpu add to memory instruction and optimize code |
* if one is added or subtracted. |
*/ |
#define percpu_add_op(var, val) \ |
do { \ |
typedef typeof(var) pao_T__; \ |
const int pao_ID__ = (__builtin_constant_p(val) && \ |
((val) == 1 || (val) == -1)) ? \ |
(int)(val) : 0; \ |
if (0) { \ |
pao_T__ pao_tmp__; \ |
pao_tmp__ = (val); \ |
(void)pao_tmp__; \ |
} \ |
switch (sizeof(var)) { \ |
case 1: \ |
if (pao_ID__ == 1) \ |
asm("incb "__percpu_arg(0) : "+m" (var)); \ |
else if (pao_ID__ == -1) \ |
asm("decb "__percpu_arg(0) : "+m" (var)); \ |
else \ |
asm("addb %1, "__percpu_arg(0) \ |
: "+m" (var) \ |
: "qi" ((pao_T__)(val))); \ |
break; \ |
case 2: \ |
if (pao_ID__ == 1) \ |
asm("incw "__percpu_arg(0) : "+m" (var)); \ |
else if (pao_ID__ == -1) \ |
asm("decw "__percpu_arg(0) : "+m" (var)); \ |
else \ |
asm("addw %1, "__percpu_arg(0) \ |
: "+m" (var) \ |
: "ri" ((pao_T__)(val))); \ |
break; \ |
case 4: \ |
if (pao_ID__ == 1) \ |
asm("incl "__percpu_arg(0) : "+m" (var)); \ |
else if (pao_ID__ == -1) \ |
asm("decl "__percpu_arg(0) : "+m" (var)); \ |
else \ |
asm("addl %1, "__percpu_arg(0) \ |
: "+m" (var) \ |
: "ri" ((pao_T__)(val))); \ |
break; \ |
case 8: \ |
if (pao_ID__ == 1) \ |
asm("incq "__percpu_arg(0) : "+m" (var)); \ |
else if (pao_ID__ == -1) \ |
asm("decq "__percpu_arg(0) : "+m" (var)); \ |
else \ |
asm("addq %1, "__percpu_arg(0) \ |
: "+m" (var) \ |
: "re" ((pao_T__)(val))); \ |
break; \ |
default: __bad_percpu_size(); \ |
} \ |
} while (0) |
#define percpu_from_op(op, var) \ |
({ \ |
typeof(var) pfo_ret__; \ |
switch (sizeof(var)) { \ |
case 1: \ |
asm(op "b "__percpu_arg(1)",%0" \ |
: "=q" (pfo_ret__) \ |
: "m" (var)); \ |
break; \ |
case 2: \ |
asm(op "w "__percpu_arg(1)",%0" \ |
: "=r" (pfo_ret__) \ |
: "m" (var)); \ |
break; \ |
case 4: \ |
asm(op "l "__percpu_arg(1)",%0" \ |
: "=r" (pfo_ret__) \ |
: "m" (var)); \ |
break; \ |
case 8: \ |
asm(op "q "__percpu_arg(1)",%0" \ |
: "=r" (pfo_ret__) \ |
: "m" (var)); \ |
break; \ |
default: __bad_percpu_size(); \ |
} \ |
pfo_ret__; \ |
}) |
#define percpu_stable_op(op, var) \ |
({ \ |
typeof(var) pfo_ret__; \ |
switch (sizeof(var)) { \ |
case 1: \ |
asm(op "b "__percpu_arg(P1)",%0" \ |
: "=q" (pfo_ret__) \ |
: "p" (&(var))); \ |
break; \ |
case 2: \ |
asm(op "w "__percpu_arg(P1)",%0" \ |
: "=r" (pfo_ret__) \ |
: "p" (&(var))); \ |
break; \ |
case 4: \ |
asm(op "l "__percpu_arg(P1)",%0" \ |
: "=r" (pfo_ret__) \ |
: "p" (&(var))); \ |
break; \ |
case 8: \ |
asm(op "q "__percpu_arg(P1)",%0" \ |
: "=r" (pfo_ret__) \ |
: "p" (&(var))); \ |
break; \ |
default: __bad_percpu_size(); \ |
} \ |
pfo_ret__; \ |
}) |
#define percpu_unary_op(op, var) \ |
({ \ |
switch (sizeof(var)) { \ |
case 1: \ |
asm(op "b "__percpu_arg(0) \ |
: "+m" (var)); \ |
break; \ |
case 2: \ |
asm(op "w "__percpu_arg(0) \ |
: "+m" (var)); \ |
break; \ |
case 4: \ |
asm(op "l "__percpu_arg(0) \ |
: "+m" (var)); \ |
break; \ |
case 8: \ |
asm(op "q "__percpu_arg(0) \ |
: "+m" (var)); \ |
break; \ |
default: __bad_percpu_size(); \ |
} \ |
}) |
/* |
* Add return operation |
*/ |
#define percpu_add_return_op(var, val) \ |
({ \ |
typeof(var) paro_ret__ = val; \ |
switch (sizeof(var)) { \ |
case 1: \ |
asm("xaddb %0, "__percpu_arg(1) \ |
: "+q" (paro_ret__), "+m" (var) \ |
: : "memory"); \ |
break; \ |
case 2: \ |
asm("xaddw %0, "__percpu_arg(1) \ |
: "+r" (paro_ret__), "+m" (var) \ |
: : "memory"); \ |
break; \ |
case 4: \ |
asm("xaddl %0, "__percpu_arg(1) \ |
: "+r" (paro_ret__), "+m" (var) \ |
: : "memory"); \ |
break; \ |
case 8: \ |
asm("xaddq %0, "__percpu_arg(1) \ |
: "+re" (paro_ret__), "+m" (var) \ |
: : "memory"); \ |
break; \ |
default: __bad_percpu_size(); \ |
} \ |
paro_ret__ += val; \ |
paro_ret__; \ |
}) |
/* |
* xchg is implemented using cmpxchg without a lock prefix. xchg is |
* expensive due to the implied lock prefix. The processor cannot prefetch |
* cachelines if xchg is used. |
*/ |
#define percpu_xchg_op(var, nval) \ |
({ \ |
typeof(var) pxo_ret__; \ |
typeof(var) pxo_new__ = (nval); \ |
switch (sizeof(var)) { \ |
case 1: \ |
asm("\n\tmov "__percpu_arg(1)",%%al" \ |
"\n1:\tcmpxchgb %2, "__percpu_arg(1) \ |
"\n\tjnz 1b" \ |
: "=&a" (pxo_ret__), "+m" (var) \ |
: "q" (pxo_new__) \ |
: "memory"); \ |
break; \ |
case 2: \ |
asm("\n\tmov "__percpu_arg(1)",%%ax" \ |
"\n1:\tcmpxchgw %2, "__percpu_arg(1) \ |
"\n\tjnz 1b" \ |
: "=&a" (pxo_ret__), "+m" (var) \ |
: "r" (pxo_new__) \ |
: "memory"); \ |
break; \ |
case 4: \ |
asm("\n\tmov "__percpu_arg(1)",%%eax" \ |
"\n1:\tcmpxchgl %2, "__percpu_arg(1) \ |
"\n\tjnz 1b" \ |
: "=&a" (pxo_ret__), "+m" (var) \ |
: "r" (pxo_new__) \ |
: "memory"); \ |
break; \ |
case 8: \ |
asm("\n\tmov "__percpu_arg(1)",%%rax" \ |
"\n1:\tcmpxchgq %2, "__percpu_arg(1) \ |
"\n\tjnz 1b" \ |
: "=&a" (pxo_ret__), "+m" (var) \ |
: "r" (pxo_new__) \ |
: "memory"); \ |
break; \ |
default: __bad_percpu_size(); \ |
} \ |
pxo_ret__; \ |
}) |
/* |
* cmpxchg has no such implied lock semantics as a result it is much |
* more efficient for cpu local operations. |
*/ |
#define percpu_cmpxchg_op(var, oval, nval) \ |
({ \ |
typeof(var) pco_ret__; \ |
typeof(var) pco_old__ = (oval); \ |
typeof(var) pco_new__ = (nval); \ |
switch (sizeof(var)) { \ |
case 1: \ |
asm("cmpxchgb %2, "__percpu_arg(1) \ |
: "=a" (pco_ret__), "+m" (var) \ |
: "q" (pco_new__), "0" (pco_old__) \ |
: "memory"); \ |
break; \ |
case 2: \ |
asm("cmpxchgw %2, "__percpu_arg(1) \ |
: "=a" (pco_ret__), "+m" (var) \ |
: "r" (pco_new__), "0" (pco_old__) \ |
: "memory"); \ |
break; \ |
case 4: \ |
asm("cmpxchgl %2, "__percpu_arg(1) \ |
: "=a" (pco_ret__), "+m" (var) \ |
: "r" (pco_new__), "0" (pco_old__) \ |
: "memory"); \ |
break; \ |
case 8: \ |
asm("cmpxchgq %2, "__percpu_arg(1) \ |
: "=a" (pco_ret__), "+m" (var) \ |
: "r" (pco_new__), "0" (pco_old__) \ |
: "memory"); \ |
break; \ |
default: __bad_percpu_size(); \ |
} \ |
pco_ret__; \ |
}) |
/* |
* this_cpu_read() makes gcc load the percpu variable every time it is |
* accessed while this_cpu_read_stable() allows the value to be cached. |
* this_cpu_read_stable() is more efficient and can be used if its value |
* is guaranteed to be valid across cpus. The current users include |
* get_current() and get_thread_info() both of which are actually |
* per-thread variables implemented as per-cpu variables and thus |
* stable for the duration of the respective task. |
*/ |
#define this_cpu_read_stable(var) percpu_stable_op("mov", var) |
#define raw_cpu_read_1(pcp) percpu_from_op("mov", pcp) |
#define raw_cpu_read_2(pcp) percpu_from_op("mov", pcp) |
#define raw_cpu_read_4(pcp) percpu_from_op("mov", pcp) |
#define raw_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) |
#define raw_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) |
#define raw_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) |
#define raw_cpu_add_1(pcp, val) percpu_add_op((pcp), val) |
#define raw_cpu_add_2(pcp, val) percpu_add_op((pcp), val) |
#define raw_cpu_add_4(pcp, val) percpu_add_op((pcp), val) |
#define raw_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) |
#define raw_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) |
#define raw_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) |
#define raw_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val) |
#define raw_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val) |
#define raw_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val) |
#define raw_cpu_xchg_1(pcp, val) percpu_xchg_op(pcp, val) |
#define raw_cpu_xchg_2(pcp, val) percpu_xchg_op(pcp, val) |
#define raw_cpu_xchg_4(pcp, val) percpu_xchg_op(pcp, val) |
#define this_cpu_read_1(pcp) percpu_from_op("mov", pcp) |
#define this_cpu_read_2(pcp) percpu_from_op("mov", pcp) |
#define this_cpu_read_4(pcp) percpu_from_op("mov", pcp) |
#define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val) |
#define this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val) |
#define this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val) |
#define this_cpu_add_1(pcp, val) percpu_add_op((pcp), val) |
#define this_cpu_add_2(pcp, val) percpu_add_op((pcp), val) |
#define this_cpu_add_4(pcp, val) percpu_add_op((pcp), val) |
#define this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val) |
#define this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val) |
#define this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val) |
#define this_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val) |
#define this_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val) |
#define this_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val) |
#define this_cpu_xchg_1(pcp, nval) percpu_xchg_op(pcp, nval) |
#define this_cpu_xchg_2(pcp, nval) percpu_xchg_op(pcp, nval) |
#define this_cpu_xchg_4(pcp, nval) percpu_xchg_op(pcp, nval) |
#define raw_cpu_add_return_1(pcp, val) percpu_add_return_op(pcp, val) |
#define raw_cpu_add_return_2(pcp, val) percpu_add_return_op(pcp, val) |
#define raw_cpu_add_return_4(pcp, val) percpu_add_return_op(pcp, val) |
#define raw_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
#define raw_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
#define raw_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
#define this_cpu_add_return_1(pcp, val) percpu_add_return_op(pcp, val) |
#define this_cpu_add_return_2(pcp, val) percpu_add_return_op(pcp, val) |
#define this_cpu_add_return_4(pcp, val) percpu_add_return_op(pcp, val) |
#define this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
#define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
#define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
#ifdef CONFIG_X86_CMPXCHG64 |
#define percpu_cmpxchg8b_double(pcp1, pcp2, o1, o2, n1, n2) \ |
({ \ |
bool __ret; \ |
typeof(pcp1) __o1 = (o1), __n1 = (n1); \ |
typeof(pcp2) __o2 = (o2), __n2 = (n2); \ |
asm volatile("cmpxchg8b "__percpu_arg(1)"\n\tsetz %0\n\t" \ |
: "=a" (__ret), "+m" (pcp1), "+m" (pcp2), "+d" (__o2) \ |
: "b" (__n1), "c" (__n2), "a" (__o1)); \ |
__ret; \ |
}) |
#define raw_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double |
#define this_cpu_cmpxchg_double_4 percpu_cmpxchg8b_double |
#endif /* CONFIG_X86_CMPXCHG64 */ |
/* |
* Per cpu atomic 64 bit operations are only available under 64 bit. |
* 32 bit must fall back to generic operations. |
*/ |
#ifdef CONFIG_X86_64 |
#define raw_cpu_read_8(pcp) percpu_from_op("mov", pcp) |
#define raw_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) |
#define raw_cpu_add_8(pcp, val) percpu_add_op((pcp), val) |
#define raw_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) |
#define raw_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) |
#define raw_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val) |
#define raw_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval) |
#define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
#define this_cpu_read_8(pcp) percpu_from_op("mov", pcp) |
#define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val) |
#define this_cpu_add_8(pcp, val) percpu_add_op((pcp), val) |
#define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val) |
#define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val) |
#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(pcp, val) |
#define this_cpu_xchg_8(pcp, nval) percpu_xchg_op(pcp, nval) |
#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(pcp, oval, nval) |
/* |
* Pretty complex macro to generate cmpxchg16 instruction. The instruction |
* is not supported on early AMD64 processors so we must be able to emulate |
* it in software. The address used in the cmpxchg16 instruction must be |
* aligned to a 16 byte boundary. |
*/ |
#define percpu_cmpxchg16b_double(pcp1, pcp2, o1, o2, n1, n2) \ |
({ \ |
bool __ret; \ |
typeof(pcp1) __o1 = (o1), __n1 = (n1); \ |
typeof(pcp2) __o2 = (o2), __n2 = (n2); \ |
alternative_io("leaq %P1,%%rsi\n\tcall this_cpu_cmpxchg16b_emu\n\t", \ |
"cmpxchg16b " __percpu_arg(1) "\n\tsetz %0\n\t", \ |
X86_FEATURE_CX16, \ |
ASM_OUTPUT2("=a" (__ret), "+m" (pcp1), \ |
"+m" (pcp2), "+d" (__o2)), \ |
"b" (__n1), "c" (__n2), "a" (__o1) : "rsi"); \ |
__ret; \ |
}) |
#define raw_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double |
#define this_cpu_cmpxchg_double_8 percpu_cmpxchg16b_double |
#endif |
/* This is not atomic against other CPUs -- CPU preemption needs to be off */ |
#define x86_test_and_clear_bit_percpu(bit, var) \ |
({ \ |
int old__; \ |
asm volatile("btr %2,"__percpu_arg(1)"\n\tsbbl %0,%0" \ |
: "=r" (old__), "+m" (var) \ |
: "dIr" (bit)); \ |
old__; \ |
}) |
static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr, |
const unsigned long __percpu *addr) |
{ |
unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG; |
#ifdef CONFIG_X86_64 |
return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_8(*a)) != 0; |
#else |
return ((1UL << (nr % BITS_PER_LONG)) & raw_cpu_read_4(*a)) != 0; |
#endif |
} |
static inline int x86_this_cpu_variable_test_bit(int nr, |
const unsigned long __percpu *addr) |
{ |
int oldbit; |
asm volatile("bt "__percpu_arg(2)",%1\n\t" |
"sbb %0,%0" |
: "=r" (oldbit) |
: "m" (*(unsigned long *)addr), "Ir" (nr)); |
return oldbit; |
} |
#define x86_this_cpu_test_bit(nr, addr) \ |
(__builtin_constant_p((nr)) \ |
? x86_this_cpu_constant_test_bit((nr), (addr)) \ |
: x86_this_cpu_variable_test_bit((nr), (addr))) |
#include <asm-generic/percpu.h> |
/* We can use this directly for local CPU (faster). */ |
DECLARE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off); |
#endif /* !__ASSEMBLY__ */ |
#ifdef CONFIG_SMP |
/* |
* Define the "EARLY_PER_CPU" macros. These are used for some per_cpu |
* variables that are initialized and accessed before there are per_cpu |
* areas allocated. |
*/ |
#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ |
DEFINE_PER_CPU(_type, _name) = _initvalue; \ |
__typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \ |
{ [0 ... NR_CPUS-1] = _initvalue }; \ |
__typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map |
#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \ |
DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue; \ |
__typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \ |
{ [0 ... NR_CPUS-1] = _initvalue }; \ |
__typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map |
#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ |
EXPORT_PER_CPU_SYMBOL(_name) |
#define DECLARE_EARLY_PER_CPU(_type, _name) \ |
DECLARE_PER_CPU(_type, _name); \ |
extern __typeof__(_type) *_name##_early_ptr; \ |
extern __typeof__(_type) _name##_early_map[] |
#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \ |
DECLARE_PER_CPU_READ_MOSTLY(_type, _name); \ |
extern __typeof__(_type) *_name##_early_ptr; \ |
extern __typeof__(_type) _name##_early_map[] |
#define early_per_cpu_ptr(_name) (_name##_early_ptr) |
#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx]) |
#define early_per_cpu(_name, _cpu) \ |
*(early_per_cpu_ptr(_name) ? \ |
&early_per_cpu_ptr(_name)[_cpu] : \ |
&per_cpu(_name, _cpu)) |
#else /* !CONFIG_SMP */ |
#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ |
DEFINE_PER_CPU(_type, _name) = _initvalue |
#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \ |
DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue |
#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ |
EXPORT_PER_CPU_SYMBOL(_name) |
#define DECLARE_EARLY_PER_CPU(_type, _name) \ |
DECLARE_PER_CPU(_type, _name) |
#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \ |
DECLARE_PER_CPU_READ_MOSTLY(_type, _name) |
#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu) |
#define early_per_cpu_ptr(_name) NULL |
/* no early_per_cpu_map() */ |
#endif /* !CONFIG_SMP */ |
#endif /* _ASM_X86_PERCPU_H */ |
/drivers/include/asm/pgtable-2level.h |
---|
0,0 → 1,116 |
#ifndef _ASM_X86_PGTABLE_2LEVEL_H |
#define _ASM_X86_PGTABLE_2LEVEL_H |
#define pte_ERROR(e) \ |
pr_err("%s:%d: bad pte %08lx\n", __FILE__, __LINE__, (e).pte_low) |
#define pgd_ERROR(e) \ |
pr_err("%s:%d: bad pgd %08lx\n", __FILE__, __LINE__, pgd_val(e)) |
/* |
* Certain architectures need to do special things when PTEs |
* within a page table are directly modified. Thus, the following |
* hook is made available. |
*/ |
static inline void native_set_pte(pte_t *ptep , pte_t pte) |
{ |
*ptep = pte; |
} |
static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) |
{ |
*pmdp = pmd; |
} |
static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) |
{ |
native_set_pte(ptep, pte); |
} |
static inline void native_pmd_clear(pmd_t *pmdp) |
{ |
native_set_pmd(pmdp, __pmd(0)); |
} |
static inline void native_pte_clear(struct mm_struct *mm, |
unsigned long addr, pte_t *xp) |
{ |
*xp = native_make_pte(0); |
} |
#ifdef CONFIG_SMP |
static inline pte_t native_ptep_get_and_clear(pte_t *xp) |
{ |
return __pte(xchg(&xp->pte_low, 0)); |
} |
#else |
#define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp) |
#endif |
#ifdef CONFIG_SMP |
static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) |
{ |
return __pmd(xchg((pmdval_t *)xp, 0)); |
} |
#else |
#define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp) |
#endif |
/* Bit manipulation helper on pte/pgoff entry */ |
static inline unsigned long pte_bitop(unsigned long value, unsigned int rightshift, |
unsigned long mask, unsigned int leftshift) |
{ |
return ((value >> rightshift) & mask) << leftshift; |
} |
/* |
* Bits _PAGE_BIT_PRESENT, _PAGE_BIT_FILE and _PAGE_BIT_PROTNONE are taken, |
* split up the 29 bits of offset into this range. |
*/ |
#define PTE_FILE_MAX_BITS 29 |
#define PTE_FILE_SHIFT1 (_PAGE_BIT_PRESENT + 1) |
#define PTE_FILE_SHIFT2 (_PAGE_BIT_FILE + 1) |
#define PTE_FILE_SHIFT3 (_PAGE_BIT_PROTNONE + 1) |
#define PTE_FILE_BITS1 (PTE_FILE_SHIFT2 - PTE_FILE_SHIFT1 - 1) |
#define PTE_FILE_BITS2 (PTE_FILE_SHIFT3 - PTE_FILE_SHIFT2 - 1) |
#define PTE_FILE_MASK1 ((1U << PTE_FILE_BITS1) - 1) |
#define PTE_FILE_MASK2 ((1U << PTE_FILE_BITS2) - 1) |
#define PTE_FILE_LSHIFT2 (PTE_FILE_BITS1) |
#define PTE_FILE_LSHIFT3 (PTE_FILE_BITS1 + PTE_FILE_BITS2) |
static __always_inline pgoff_t pte_to_pgoff(pte_t pte) |
{ |
return (pgoff_t) |
(pte_bitop(pte.pte_low, PTE_FILE_SHIFT1, PTE_FILE_MASK1, 0) + |
pte_bitop(pte.pte_low, PTE_FILE_SHIFT2, PTE_FILE_MASK2, PTE_FILE_LSHIFT2) + |
pte_bitop(pte.pte_low, PTE_FILE_SHIFT3, -1UL, PTE_FILE_LSHIFT3)); |
} |
static __always_inline pte_t pgoff_to_pte(pgoff_t off) |
{ |
return (pte_t){ |
.pte_low = |
pte_bitop(off, 0, PTE_FILE_MASK1, PTE_FILE_SHIFT1) + |
pte_bitop(off, PTE_FILE_LSHIFT2, PTE_FILE_MASK2, PTE_FILE_SHIFT2) + |
pte_bitop(off, PTE_FILE_LSHIFT3, -1UL, PTE_FILE_SHIFT3) + |
_PAGE_FILE, |
}; |
} |
/* Encode and de-code a swap entry */ |
#define SWP_TYPE_BITS (_PAGE_BIT_FILE - _PAGE_BIT_PRESENT - 1) |
#define SWP_OFFSET_SHIFT (_PAGE_BIT_PROTNONE + 1) |
#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS) |
#define __swp_type(x) (((x).val >> (_PAGE_BIT_PRESENT + 1)) \ |
& ((1U << SWP_TYPE_BITS) - 1)) |
#define __swp_offset(x) ((x).val >> SWP_OFFSET_SHIFT) |
#define __swp_entry(type, offset) ((swp_entry_t) { \ |
((type) << (_PAGE_BIT_PRESENT + 1)) \ |
| ((offset) << SWP_OFFSET_SHIFT) }) |
#define __pte_to_swp_entry(pte) ((swp_entry_t) { (pte).pte_low }) |
#define __swp_entry_to_pte(x) ((pte_t) { .pte = (x).val }) |
#endif /* _ASM_X86_PGTABLE_2LEVEL_H */ |
/drivers/include/asm/pgtable-2level_types.h |
---|
0,0 → 1,37 |
#ifndef _ASM_X86_PGTABLE_2LEVEL_DEFS_H |
#define _ASM_X86_PGTABLE_2LEVEL_DEFS_H |
#ifndef __ASSEMBLY__ |
#include <linux/types.h> |
typedef unsigned long pteval_t; |
typedef unsigned long pmdval_t; |
typedef unsigned long pudval_t; |
typedef unsigned long pgdval_t; |
typedef unsigned long pgprotval_t; |
typedef union { |
pteval_t pte; |
pteval_t pte_low; |
} pte_t; |
#endif /* !__ASSEMBLY__ */ |
#define SHARED_KERNEL_PMD 0 |
#define PAGETABLE_LEVELS 2 |
/* |
* traditional i386 two-level paging structure: |
*/ |
#define PGDIR_SHIFT 22 |
#define PTRS_PER_PGD 1024 |
/* |
* the i386 is two-level, so we don't really have any |
* PMD directory physically. |
*/ |
#define PTRS_PER_PTE 1024 |
#endif /* _ASM_X86_PGTABLE_2LEVEL_DEFS_H */ |
/drivers/include/asm/pgtable.h |
---|
0,0 → 1,905 |
#ifndef _ASM_X86_PGTABLE_H |
#define _ASM_X86_PGTABLE_H |
#include <asm/page.h> |
#include <asm/e820.h> |
#include <asm/pgtable_types.h> |
/* |
* Macro to mark a page protection value as UC- |
*/ |
#define pgprot_noncached(prot) \ |
((boot_cpu_data.x86 > 3) \ |
? (__pgprot(pgprot_val(prot) | \ |
cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS))) \ |
: (prot)) |
#ifndef __ASSEMBLY__ |
#include <asm/x86_init.h> |
void ptdump_walk_pgd_level(struct seq_file *m, pgd_t *pgd); |
/* |
* ZERO_PAGE is a global shared page that is always zero: used |
* for zero-mapped memory areas etc.. |
*/ |
extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] |
__visible; |
#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) |
extern spinlock_t pgd_lock; |
extern struct list_head pgd_list; |
extern struct mm_struct *pgd_page_get_mm(struct page *page); |
#ifdef CONFIG_PARAVIRT |
#include <asm/paravirt.h> |
#else /* !CONFIG_PARAVIRT */ |
#define set_pte(ptep, pte) native_set_pte(ptep, pte) |
#define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte) |
#define set_pmd_at(mm, addr, pmdp, pmd) native_set_pmd_at(mm, addr, pmdp, pmd) |
#define set_pte_atomic(ptep, pte) \ |
native_set_pte_atomic(ptep, pte) |
#define set_pmd(pmdp, pmd) native_set_pmd(pmdp, pmd) |
#ifndef __PAGETABLE_PUD_FOLDED |
#define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd) |
#define pgd_clear(pgd) native_pgd_clear(pgd) |
#endif |
#ifndef set_pud |
# define set_pud(pudp, pud) native_set_pud(pudp, pud) |
#endif |
#ifndef __PAGETABLE_PMD_FOLDED |
#define pud_clear(pud) native_pud_clear(pud) |
#endif |
#define pte_clear(mm, addr, ptep) native_pte_clear(mm, addr, ptep) |
#define pmd_clear(pmd) native_pmd_clear(pmd) |
#define pte_update(mm, addr, ptep) do { } while (0) |
#define pte_update_defer(mm, addr, ptep) do { } while (0) |
#define pmd_update(mm, addr, ptep) do { } while (0) |
#define pmd_update_defer(mm, addr, ptep) do { } while (0) |
#define pgd_val(x) native_pgd_val(x) |
#define __pgd(x) native_make_pgd(x) |
#ifndef __PAGETABLE_PUD_FOLDED |
#define pud_val(x) native_pud_val(x) |
#define __pud(x) native_make_pud(x) |
#endif |
#ifndef __PAGETABLE_PMD_FOLDED |
#define pmd_val(x) native_pmd_val(x) |
#define __pmd(x) native_make_pmd(x) |
#endif |
#define pte_val(x) native_pte_val(x) |
#define __pte(x) native_make_pte(x) |
#define arch_end_context_switch(prev) do {} while(0) |
#endif /* CONFIG_PARAVIRT */ |
/* |
* The following only work if pte_present() is true. |
* Undefined behaviour if not.. |
*/ |
static inline int pte_dirty(pte_t pte) |
{ |
return pte_flags(pte) & _PAGE_DIRTY; |
} |
static inline int pte_young(pte_t pte) |
{ |
return pte_flags(pte) & _PAGE_ACCESSED; |
} |
static inline int pmd_dirty(pmd_t pmd) |
{ |
return pmd_flags(pmd) & _PAGE_DIRTY; |
} |
static inline int pmd_young(pmd_t pmd) |
{ |
return pmd_flags(pmd) & _PAGE_ACCESSED; |
} |
static inline int pte_write(pte_t pte) |
{ |
return pte_flags(pte) & _PAGE_RW; |
} |
static inline int pte_file(pte_t pte) |
{ |
return pte_flags(pte) & _PAGE_FILE; |
} |
static inline int pte_huge(pte_t pte) |
{ |
return pte_flags(pte) & _PAGE_PSE; |
} |
static inline int pte_global(pte_t pte) |
{ |
return pte_flags(pte) & _PAGE_GLOBAL; |
} |
static inline int pte_exec(pte_t pte) |
{ |
return !(pte_flags(pte) & _PAGE_NX); |
} |
static inline int pte_special(pte_t pte) |
{ |
/* |
* See CONFIG_NUMA_BALANCING pte_numa in include/asm-generic/pgtable.h. |
* On x86 we have _PAGE_BIT_NUMA == _PAGE_BIT_GLOBAL+1 == |
* __PAGE_BIT_SOFTW1 == _PAGE_BIT_SPECIAL. |
*/ |
return (pte_flags(pte) & _PAGE_SPECIAL) && |
(pte_flags(pte) & (_PAGE_PRESENT|_PAGE_PROTNONE)); |
} |
static inline unsigned long pte_pfn(pte_t pte) |
{ |
return (pte_val(pte) & PTE_PFN_MASK) >> PAGE_SHIFT; |
} |
static inline unsigned long pmd_pfn(pmd_t pmd) |
{ |
return (pmd_val(pmd) & PTE_PFN_MASK) >> PAGE_SHIFT; |
} |
static inline unsigned long pud_pfn(pud_t pud) |
{ |
return (pud_val(pud) & PTE_PFN_MASK) >> PAGE_SHIFT; |
} |
#define pte_page(pte) pfn_to_page(pte_pfn(pte)) |
static inline int pmd_large(pmd_t pte) |
{ |
return pmd_flags(pte) & _PAGE_PSE; |
} |
#ifdef CONFIG_TRANSPARENT_HUGEPAGE |
static inline int pmd_trans_splitting(pmd_t pmd) |
{ |
return pmd_val(pmd) & _PAGE_SPLITTING; |
} |
static inline int pmd_trans_huge(pmd_t pmd) |
{ |
return pmd_val(pmd) & _PAGE_PSE; |
} |
static inline int has_transparent_hugepage(void) |
{ |
return cpu_has_pse; |
} |
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
static inline pte_t pte_set_flags(pte_t pte, pteval_t set) |
{ |
pteval_t v = native_pte_val(pte); |
return native_make_pte(v | set); |
} |
static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear) |
{ |
pteval_t v = native_pte_val(pte); |
return native_make_pte(v & ~clear); |
} |
static inline pte_t pte_mkclean(pte_t pte) |
{ |
return pte_clear_flags(pte, _PAGE_DIRTY); |
} |
static inline pte_t pte_mkold(pte_t pte) |
{ |
return pte_clear_flags(pte, _PAGE_ACCESSED); |
} |
static inline pte_t pte_wrprotect(pte_t pte) |
{ |
return pte_clear_flags(pte, _PAGE_RW); |
} |
static inline pte_t pte_mkexec(pte_t pte) |
{ |
return pte_clear_flags(pte, _PAGE_NX); |
} |
static inline pte_t pte_mkdirty(pte_t pte) |
{ |
return pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY); |
} |
static inline pte_t pte_mkyoung(pte_t pte) |
{ |
return pte_set_flags(pte, _PAGE_ACCESSED); |
} |
static inline pte_t pte_mkwrite(pte_t pte) |
{ |
return pte_set_flags(pte, _PAGE_RW); |
} |
static inline pte_t pte_mkhuge(pte_t pte) |
{ |
return pte_set_flags(pte, _PAGE_PSE); |
} |
static inline pte_t pte_clrhuge(pte_t pte) |
{ |
return pte_clear_flags(pte, _PAGE_PSE); |
} |
static inline pte_t pte_mkglobal(pte_t pte) |
{ |
return pte_set_flags(pte, _PAGE_GLOBAL); |
} |
static inline pte_t pte_clrglobal(pte_t pte) |
{ |
return pte_clear_flags(pte, _PAGE_GLOBAL); |
} |
static inline pte_t pte_mkspecial(pte_t pte) |
{ |
return pte_set_flags(pte, _PAGE_SPECIAL); |
} |
static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set) |
{ |
pmdval_t v = native_pmd_val(pmd); |
return __pmd(v | set); |
} |
static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear) |
{ |
pmdval_t v = native_pmd_val(pmd); |
return __pmd(v & ~clear); |
} |
static inline pmd_t pmd_mkold(pmd_t pmd) |
{ |
return pmd_clear_flags(pmd, _PAGE_ACCESSED); |
} |
static inline pmd_t pmd_wrprotect(pmd_t pmd) |
{ |
return pmd_clear_flags(pmd, _PAGE_RW); |
} |
static inline pmd_t pmd_mkdirty(pmd_t pmd) |
{ |
return pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY); |
} |
static inline pmd_t pmd_mkhuge(pmd_t pmd) |
{ |
return pmd_set_flags(pmd, _PAGE_PSE); |
} |
static inline pmd_t pmd_mkyoung(pmd_t pmd) |
{ |
return pmd_set_flags(pmd, _PAGE_ACCESSED); |
} |
static inline pmd_t pmd_mkwrite(pmd_t pmd) |
{ |
return pmd_set_flags(pmd, _PAGE_RW); |
} |
static inline pmd_t pmd_mknotpresent(pmd_t pmd) |
{ |
return pmd_clear_flags(pmd, _PAGE_PRESENT); |
} |
#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY |
static inline int pte_soft_dirty(pte_t pte) |
{ |
return pte_flags(pte) & _PAGE_SOFT_DIRTY; |
} |
static inline int pmd_soft_dirty(pmd_t pmd) |
{ |
return pmd_flags(pmd) & _PAGE_SOFT_DIRTY; |
} |
static inline pte_t pte_mksoft_dirty(pte_t pte) |
{ |
return pte_set_flags(pte, _PAGE_SOFT_DIRTY); |
} |
static inline pmd_t pmd_mksoft_dirty(pmd_t pmd) |
{ |
return pmd_set_flags(pmd, _PAGE_SOFT_DIRTY); |
} |
static inline pte_t pte_file_clear_soft_dirty(pte_t pte) |
{ |
return pte_clear_flags(pte, _PAGE_SOFT_DIRTY); |
} |
static inline pte_t pte_file_mksoft_dirty(pte_t pte) |
{ |
return pte_set_flags(pte, _PAGE_SOFT_DIRTY); |
} |
static inline int pte_file_soft_dirty(pte_t pte) |
{ |
return pte_flags(pte) & _PAGE_SOFT_DIRTY; |
} |
#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ |
/* |
* Mask out unsupported bits in a present pgprot. Non-present pgprots |
* can use those bits for other purposes, so leave them be. |
*/ |
static inline pgprotval_t massage_pgprot(pgprot_t pgprot) |
{ |
pgprotval_t protval = pgprot_val(pgprot); |
if (protval & _PAGE_PRESENT) |
protval &= __supported_pte_mask; |
return protval; |
} |
static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot) |
{ |
return __pte(((phys_addr_t)page_nr << PAGE_SHIFT) | |
massage_pgprot(pgprot)); |
} |
static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot) |
{ |
return __pmd(((phys_addr_t)page_nr << PAGE_SHIFT) | |
massage_pgprot(pgprot)); |
} |
static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) |
{ |
pteval_t val = pte_val(pte); |
/* |
* Chop off the NX bit (if present), and add the NX portion of |
* the newprot (if present): |
*/ |
val &= _PAGE_CHG_MASK; |
val |= massage_pgprot(newprot) & ~_PAGE_CHG_MASK; |
return __pte(val); |
} |
static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) |
{ |
pmdval_t val = pmd_val(pmd); |
val &= _HPAGE_CHG_MASK; |
val |= massage_pgprot(newprot) & ~_HPAGE_CHG_MASK; |
return __pmd(val); |
} |
/* mprotect needs to preserve PAT bits when updating vm_page_prot */ |
#define pgprot_modify pgprot_modify |
static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) |
{ |
pgprotval_t preservebits = pgprot_val(oldprot) & _PAGE_CHG_MASK; |
pgprotval_t addbits = pgprot_val(newprot); |
return __pgprot(preservebits | addbits); |
} |
#define pte_pgprot(x) __pgprot(pte_flags(x) & PTE_FLAGS_MASK) |
#define canon_pgprot(p) __pgprot(massage_pgprot(p)) |
static inline int is_new_memtype_allowed(u64 paddr, unsigned long size, |
enum page_cache_mode pcm, |
enum page_cache_mode new_pcm) |
{ |
/* |
* PAT type is always WB for untracked ranges, so no need to check. |
*/ |
if (x86_platform.is_untracked_pat_range(paddr, paddr + size)) |
return 1; |
/* |
* Certain new memtypes are not allowed with certain |
* requested memtype: |
* - request is uncached, return cannot be write-back |
* - request is write-combine, return cannot be write-back |
*/ |
if ((pcm == _PAGE_CACHE_MODE_UC_MINUS && |
new_pcm == _PAGE_CACHE_MODE_WB) || |
(pcm == _PAGE_CACHE_MODE_WC && |
new_pcm == _PAGE_CACHE_MODE_WB)) { |
return 0; |
} |
return 1; |
} |
pmd_t *populate_extra_pmd(unsigned long vaddr); |
pte_t *populate_extra_pte(unsigned long vaddr); |
#endif /* __ASSEMBLY__ */ |
#ifdef CONFIG_X86_32 |
# include <asm/pgtable_32.h> |
#else |
# include <asm/pgtable_64.h> |
#endif |
#ifndef __ASSEMBLY__ |
//#include <linux/mm_types.h> |
#include <linux/mmdebug.h> |
#include <linux/log2.h> |
static inline int pte_none(pte_t pte) |
{ |
return !pte.pte; |
} |
#define __HAVE_ARCH_PTE_SAME |
static inline int pte_same(pte_t a, pte_t b) |
{ |
return a.pte == b.pte; |
} |
static inline int pte_present(pte_t a) |
{ |
return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE | |
_PAGE_NUMA); |
} |
#define pte_present_nonuma pte_present_nonuma |
static inline int pte_present_nonuma(pte_t a) |
{ |
return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE); |
} |
#define pte_accessible pte_accessible |
static inline bool pte_accessible(struct mm_struct *mm, pte_t a) |
{ |
if (pte_flags(a) & _PAGE_PRESENT) |
return true; |
if ((pte_flags(a) & (_PAGE_PROTNONE | _PAGE_NUMA)) && |
mm_tlb_flush_pending(mm)) |
return true; |
return false; |
} |
static inline int pte_hidden(pte_t pte) |
{ |
return pte_flags(pte) & _PAGE_HIDDEN; |
} |
static inline int pmd_present(pmd_t pmd) |
{ |
/* |
* Checking for _PAGE_PSE is needed too because |
* split_huge_page will temporarily clear the present bit (but |
* the _PAGE_PSE flag will remain set at all times while the |
* _PAGE_PRESENT bit is clear). |
*/ |
return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE | |
_PAGE_NUMA); |
} |
static inline int pmd_none(pmd_t pmd) |
{ |
/* Only check low word on 32-bit platforms, since it might be |
out of sync with upper half. */ |
return (unsigned long)native_pmd_val(pmd) == 0; |
} |
static inline unsigned long pmd_page_vaddr(pmd_t pmd) |
{ |
return (unsigned long)__va(pmd_val(pmd) & PTE_PFN_MASK); |
} |
/* |
* Currently stuck as a macro due to indirect forward reference to |
* linux/mmzone.h's __section_mem_map_addr() definition: |
*/ |
#define pmd_page(pmd) pfn_to_page((pmd_val(pmd) & PTE_PFN_MASK) >> PAGE_SHIFT) |
/* |
* the pmd page can be thought of an array like this: pmd_t[PTRS_PER_PMD] |
* |
* this macro returns the index of the entry in the pmd page which would |
* control the given virtual address |
*/ |
static inline unsigned long pmd_index(unsigned long address) |
{ |
return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1); |
} |
/* |
* Conversion functions: convert a page and protection to a page entry, |
* and a page entry and page directory to the page they refer to. |
* |
* (Currently stuck as a macro because of indirect forward reference |
* to linux/mm.h:page_to_nid()) |
*/ |
#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) |
/* |
* the pte page can be thought of an array like this: pte_t[PTRS_PER_PTE] |
* |
* this function returns the index of the entry in the pte page which would |
* control the given virtual address |
*/ |
static inline unsigned long pte_index(unsigned long address) |
{ |
return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); |
} |
static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address) |
{ |
return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address); |
} |
static inline int pmd_bad(pmd_t pmd) |
{ |
#ifdef CONFIG_NUMA_BALANCING |
/* pmd_numa check */ |
if ((pmd_flags(pmd) & (_PAGE_NUMA|_PAGE_PRESENT)) == _PAGE_NUMA) |
return 0; |
#endif |
return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE; |
} |
static inline unsigned long pages_to_mb(unsigned long npg) |
{ |
return npg >> (20 - PAGE_SHIFT); |
} |
#if PAGETABLE_LEVELS > 2 |
static inline int pud_none(pud_t pud) |
{ |
return native_pud_val(pud) == 0; |
} |
static inline int pud_present(pud_t pud) |
{ |
return pud_flags(pud) & _PAGE_PRESENT; |
} |
static inline unsigned long pud_page_vaddr(pud_t pud) |
{ |
return (unsigned long)__va((unsigned long)pud_val(pud) & PTE_PFN_MASK); |
} |
/* |
* Currently stuck as a macro due to indirect forward reference to |
* linux/mmzone.h's __section_mem_map_addr() definition: |
*/ |
#define pud_page(pud) pfn_to_page(pud_val(pud) >> PAGE_SHIFT) |
/* Find an entry in the second-level page table.. */ |
static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) |
{ |
return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address); |
} |
static inline int pud_large(pud_t pud) |
{ |
return (pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) == |
(_PAGE_PSE | _PAGE_PRESENT); |
} |
static inline int pud_bad(pud_t pud) |
{ |
return (pud_flags(pud) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0; |
} |
#else |
static inline int pud_large(pud_t pud) |
{ |
return 0; |
} |
#endif /* PAGETABLE_LEVELS > 2 */ |
#if PAGETABLE_LEVELS > 3 |
static inline int pgd_present(pgd_t pgd) |
{ |
return pgd_flags(pgd) & _PAGE_PRESENT; |
} |
static inline unsigned long pgd_page_vaddr(pgd_t pgd) |
{ |
return (unsigned long)__va((unsigned long)pgd_val(pgd) & PTE_PFN_MASK); |
} |
/* |
* Currently stuck as a macro due to indirect forward reference to |
* linux/mmzone.h's __section_mem_map_addr() definition: |
*/ |
#define pgd_page(pgd) pfn_to_page(pgd_val(pgd) >> PAGE_SHIFT) |
/* to find an entry in a page-table-directory. */ |
static inline unsigned long pud_index(unsigned long address) |
{ |
return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1); |
} |
static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address) |
{ |
return (pud_t *)pgd_page_vaddr(*pgd) + pud_index(address); |
} |
static inline int pgd_bad(pgd_t pgd) |
{ |
return (pgd_flags(pgd) & ~_PAGE_USER) != _KERNPG_TABLE; |
} |
static inline int pgd_none(pgd_t pgd) |
{ |
return !native_pgd_val(pgd); |
} |
#endif /* PAGETABLE_LEVELS > 3 */ |
#endif /* __ASSEMBLY__ */ |
/* |
* the pgd page can be thought of an array like this: pgd_t[PTRS_PER_PGD] |
* |
* this macro returns the index of the entry in the pgd page which would |
* control the given virtual address |
*/ |
#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) |
/* |
* pgd_offset() returns a (pgd_t *) |
* pgd_index() is used get the offset into the pgd page's array of pgd_t's; |
*/ |
#define pgd_offset(mm, address) ((mm)->pgd + pgd_index((address))) |
/* |
* a shortcut which implies the use of the kernel's pgd, instead |
* of a process's |
*/ |
#define pgd_offset_k(address) pgd_offset(&init_mm, (address)) |
#define KERNEL_PGD_BOUNDARY pgd_index(PAGE_OFFSET) |
#define KERNEL_PGD_PTRS (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY) |
#ifndef __ASSEMBLY__ |
extern int direct_gbpages; |
void init_mem_mapping(void); |
void early_alloc_pgt_buf(void); |
/* local pte updates need not use xchg for locking */ |
static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep) |
{ |
pte_t res = *ptep; |
/* Pure native function needs no input for mm, addr */ |
native_pte_clear(NULL, 0, ptep); |
return res; |
} |
static inline pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp) |
{ |
pmd_t res = *pmdp; |
native_pmd_clear(pmdp); |
return res; |
} |
static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr, |
pte_t *ptep , pte_t pte) |
{ |
native_set_pte(ptep, pte); |
} |
static inline void native_set_pmd_at(struct mm_struct *mm, unsigned long addr, |
pmd_t *pmdp , pmd_t pmd) |
{ |
native_set_pmd(pmdp, pmd); |
} |
#ifndef CONFIG_PARAVIRT |
/* |
* Rules for using pte_update - it must be called after any PTE update which |
* has not been done using the set_pte / clear_pte interfaces. It is used by |
* shadow mode hypervisors to resynchronize the shadow page tables. Kernel PTE |
* updates should either be sets, clears, or set_pte_atomic for P->P |
* transitions, which means this hook should only be called for user PTEs. |
* This hook implies a P->P protection or access change has taken place, which |
* requires a subsequent TLB flush. The notification can optionally be delayed |
* until the TLB flush event by using the pte_update_defer form of the |
* interface, but care must be taken to assure that the flush happens while |
* still holding the same page table lock so that the shadow and primary pages |
* do not become out of sync on SMP. |
*/ |
#define pte_update(mm, addr, ptep) do { } while (0) |
#define pte_update_defer(mm, addr, ptep) do { } while (0) |
#endif |
/* |
* We only update the dirty/accessed state if we set |
* the dirty bit by hand in the kernel, since the hardware |
* will do the accessed bit for us, and we don't want to |
* race with other CPU's that might be updating the dirty |
* bit at the same time. |
*/ |
struct vm_area_struct; |
#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS |
extern int ptep_set_access_flags(struct vm_area_struct *vma, |
unsigned long address, pte_t *ptep, |
pte_t entry, int dirty); |
#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG |
extern int ptep_test_and_clear_young(struct vm_area_struct *vma, |
unsigned long addr, pte_t *ptep); |
#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH |
extern int ptep_clear_flush_young(struct vm_area_struct *vma, |
unsigned long address, pte_t *ptep); |
#define __HAVE_ARCH_PTEP_GET_AND_CLEAR |
static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, |
pte_t *ptep) |
{ |
pte_t pte = native_ptep_get_and_clear(ptep); |
pte_update(mm, addr, ptep); |
return pte; |
} |
#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL |
static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, |
unsigned long addr, pte_t *ptep, |
int full) |
{ |
pte_t pte; |
if (full) { |
/* |
* Full address destruction in progress; paravirt does not |
* care about updates and native needs no locking |
*/ |
pte = native_local_ptep_get_and_clear(ptep); |
} else { |
pte = ptep_get_and_clear(mm, addr, ptep); |
} |
return pte; |
} |
#define __HAVE_ARCH_PTEP_SET_WRPROTECT |
static inline void ptep_set_wrprotect(struct mm_struct *mm, |
unsigned long addr, pte_t *ptep) |
{ |
clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte); |
pte_update(mm, addr, ptep); |
} |
#define flush_tlb_fix_spurious_fault(vma, address) do { } while (0) |
#define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot)) |
#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS |
extern int pmdp_set_access_flags(struct vm_area_struct *vma, |
unsigned long address, pmd_t *pmdp, |
pmd_t entry, int dirty); |
#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG |
extern int pmdp_test_and_clear_young(struct vm_area_struct *vma, |
unsigned long addr, pmd_t *pmdp); |
#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH |
extern int pmdp_clear_flush_young(struct vm_area_struct *vma, |
unsigned long address, pmd_t *pmdp); |
#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH |
extern void pmdp_splitting_flush(struct vm_area_struct *vma, |
unsigned long addr, pmd_t *pmdp); |
#define __HAVE_ARCH_PMD_WRITE |
static inline int pmd_write(pmd_t pmd) |
{ |
return pmd_flags(pmd) & _PAGE_RW; |
} |
#define __HAVE_ARCH_PMDP_GET_AND_CLEAR |
static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, unsigned long addr, |
pmd_t *pmdp) |
{ |
pmd_t pmd = native_pmdp_get_and_clear(pmdp); |
pmd_update(mm, addr, pmdp); |
return pmd; |
} |
#define __HAVE_ARCH_PMDP_SET_WRPROTECT |
static inline void pmdp_set_wrprotect(struct mm_struct *mm, |
unsigned long addr, pmd_t *pmdp) |
{ |
clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp); |
pmd_update(mm, addr, pmdp); |
} |
/* |
* clone_pgd_range(pgd_t *dst, pgd_t *src, int count); |
* |
* dst - pointer to pgd range anwhere on a pgd page |
* src - "" |
* count - the number of pgds to copy. |
* |
* dst and src can be on the same page, but the range must not overlap, |
* and must not cross a page boundary. |
*/ |
static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count) |
{ |
memcpy(dst, src, count * sizeof(pgd_t)); |
} |
#define PTE_SHIFT ilog2(PTRS_PER_PTE) |
static inline int page_level_shift(enum pg_level level) |
{ |
return (PAGE_SHIFT - PTE_SHIFT) + level * PTE_SHIFT; |
} |
static inline unsigned long page_level_size(enum pg_level level) |
{ |
return 1UL << page_level_shift(level); |
} |
static inline unsigned long page_level_mask(enum pg_level level) |
{ |
return ~(page_level_size(level) - 1); |
} |
/* |
* The x86 doesn't have any external MMU info: the kernel page |
* tables contain all the necessary information. |
*/ |
static inline void update_mmu_cache(struct vm_area_struct *vma, |
unsigned long addr, pte_t *ptep) |
{ |
} |
static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, |
unsigned long addr, pmd_t *pmd) |
{ |
} |
#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY |
static inline pte_t pte_swp_mksoft_dirty(pte_t pte) |
{ |
VM_BUG_ON(pte_present_nonuma(pte)); |
return pte_set_flags(pte, _PAGE_SWP_SOFT_DIRTY); |
} |
static inline int pte_swp_soft_dirty(pte_t pte) |
{ |
VM_BUG_ON(pte_present_nonuma(pte)); |
return pte_flags(pte) & _PAGE_SWP_SOFT_DIRTY; |
} |
static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) |
{ |
VM_BUG_ON(pte_present_nonuma(pte)); |
return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY); |
} |
#endif |
//#include <asm-generic/pgtable.h> |
#endif /* __ASSEMBLY__ */ |
#endif /* _ASM_X86_PGTABLE_H */ |
/drivers/include/asm/pgtable_32.h |
---|
0,0 → 1,76 |
#ifndef _ASM_X86_PGTABLE_32_H |
#define _ASM_X86_PGTABLE_32_H |
#include <asm/pgtable_32_types.h> |
/* |
* The Linux memory management assumes a three-level page table setup. On |
* the i386, we use that, but "fold" the mid level into the top-level page |
* table, so that we physically have the same two-level page table as the |
* i386 mmu expects. |
* |
* This file contains the functions and defines necessary to modify and use |
* the i386 page table tree. |
*/ |
#ifndef __ASSEMBLY__ |
#include <asm/processor.h> |
#include <linux/threads.h> |
#include <linux/bitops.h> |
#include <linux/list.h> |
#include <linux/spinlock.h> |
struct mm_struct; |
struct vm_area_struct; |
extern pgd_t swapper_pg_dir[1024]; |
extern pgd_t initial_page_table[1024]; |
static inline void pgtable_cache_init(void) { } |
static inline void check_pgt_cache(void) { } |
void paging_init(void); |
/* |
* Define this if things work differently on an i386 and an i486: |
* it will (on an i486) warn about kernel memory accesses that are |
* done without a 'access_ok(VERIFY_WRITE,..)' |
*/ |
#undef TEST_ACCESS_OK |
#ifdef CONFIG_X86_PAE |
# include <asm/pgtable-3level.h> |
#else |
# include <asm/pgtable-2level.h> |
#endif |
#if defined(CONFIG_HIGHPTE) |
#define pte_offset_map(dir, address) \ |
((pte_t *)kmap_atomic(pmd_page(*(dir))) + \ |
pte_index((address))) |
#define pte_unmap(pte) kunmap_atomic((pte)) |
#else |
#define pte_offset_map(dir, address) \ |
((pte_t *)page_address(pmd_page(*(dir))) + pte_index((address))) |
#define pte_unmap(pte) do { } while (0) |
#endif |
/* Clear a kernel PTE and flush it from the TLB */ |
#define kpte_clear_flush(ptep, vaddr) \ |
do { \ |
pte_clear(&init_mm, (vaddr), (ptep)); \ |
__flush_tlb_one((vaddr)); \ |
} while (0) |
#endif /* !__ASSEMBLY__ */ |
/* |
* kern_addr_valid() is (1) for FLATMEM and (0) for |
* SPARSEMEM and DISCONTIGMEM |
*/ |
#ifdef CONFIG_FLATMEM |
#define kern_addr_valid(addr) (1) |
#else |
#define kern_addr_valid(kaddr) (0) |
#endif |
#endif /* _ASM_X86_PGTABLE_32_H */ |
/drivers/include/asm/pgtable_32_types.h |
---|
0,0 → 1,55 |
#ifndef _ASM_X86_PGTABLE_32_DEFS_H |
#define _ASM_X86_PGTABLE_32_DEFS_H |
/* |
* The Linux x86 paging architecture is 'compile-time dual-mode', it |
* implements both the traditional 2-level x86 page tables and the |
* newer 3-level PAE-mode page tables. |
*/ |
#ifdef CONFIG_X86_PAE |
# include <asm/pgtable-3level_types.h> |
# define PMD_SIZE (1UL << PMD_SHIFT) |
# define PMD_MASK (~(PMD_SIZE - 1)) |
#else |
# include <asm/pgtable-2level_types.h> |
#endif |
#define PGDIR_SIZE (1UL << PGDIR_SHIFT) |
#define PGDIR_MASK (~(PGDIR_SIZE - 1)) |
/* Just any arbitrary offset to the start of the vmalloc VM area: the |
* current 8MB value just means that there will be a 8MB "hole" after the |
* physical memory until the kernel virtual memory starts. That means that |
* any out-of-bounds memory accesses will hopefully be caught. |
* The vmalloc() routines leaves a hole of 4kB between each vmalloced |
* area for the same reason. ;) |
*/ |
#define VMALLOC_OFFSET (8 * 1024 * 1024) |
#ifndef __ASSEMBLY__ |
extern bool __vmalloc_start_set; /* set once high_memory is set */ |
#endif |
#define VMALLOC_START ((unsigned long)high_memory + VMALLOC_OFFSET) |
#ifdef CONFIG_X86_PAE |
#define LAST_PKMAP 512 |
#else |
#define LAST_PKMAP 1024 |
#endif |
#define PKMAP_BASE ((FIXADDR_START - PAGE_SIZE * (LAST_PKMAP + 1)) \ |
& PMD_MASK) |
#ifdef CONFIG_HIGHMEM |
# define VMALLOC_END (PKMAP_BASE - 2 * PAGE_SIZE) |
#else |
# define VMALLOC_END (FIXADDR_START - 2 * PAGE_SIZE) |
#endif |
#define MODULES_VADDR VMALLOC_START |
#define MODULES_END VMALLOC_END |
#define MODULES_LEN (MODULES_VADDR - MODULES_END) |
#define MAXMEM (VMALLOC_END - PAGE_OFFSET - __VMALLOC_RESERVE) |
#endif /* _ASM_X86_PGTABLE_32_DEFS_H */ |
/drivers/include/asm/pgtable_types.h |
---|
0,0 → 1,463 |
#ifndef _ASM_X86_PGTABLE_DEFS_H |
#define _ASM_X86_PGTABLE_DEFS_H |
#include <linux/const.h> |
#include <asm/page_types.h> |
#define FIRST_USER_ADDRESS 0 |
#define _PAGE_BIT_PRESENT 0 /* is present */ |
#define _PAGE_BIT_RW 1 /* writeable */ |
#define _PAGE_BIT_USER 2 /* userspace addressable */ |
#define _PAGE_BIT_PWT 3 /* page write through */ |
#define _PAGE_BIT_PCD 4 /* page cache disabled */ |
#define _PAGE_BIT_ACCESSED 5 /* was accessed (raised by CPU) */ |
#define _PAGE_BIT_DIRTY 6 /* was written to (raised by CPU) */ |
#define _PAGE_BIT_PSE 7 /* 4 MB (or 2MB) page */ |
#define _PAGE_BIT_PAT 7 /* on 4KB pages */ |
#define _PAGE_BIT_GLOBAL 8 /* Global TLB entry PPro+ */ |
#define _PAGE_BIT_SOFTW1 9 /* available for programmer */ |
#define _PAGE_BIT_SOFTW2 10 /* " */ |
#define _PAGE_BIT_SOFTW3 11 /* " */ |
#define _PAGE_BIT_PAT_LARGE 12 /* On 2MB or 1GB pages */ |
#define _PAGE_BIT_SPECIAL _PAGE_BIT_SOFTW1 |
#define _PAGE_BIT_CPA_TEST _PAGE_BIT_SOFTW1 |
#define _PAGE_BIT_SPLITTING _PAGE_BIT_SOFTW2 /* only valid on a PSE pmd */ |
#define _PAGE_BIT_HIDDEN _PAGE_BIT_SOFTW3 /* hidden by kmemcheck */ |
#define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */ |
#define _PAGE_BIT_NX 63 /* No execute: only valid after cpuid check */ |
/* |
* Swap offsets on configurations that allow automatic NUMA balancing use the |
* bits after _PAGE_BIT_GLOBAL. To uniquely distinguish NUMA hinting PTEs from |
* swap entries, we use the first bit after _PAGE_BIT_GLOBAL and shrink the |
* maximum possible swap space from 16TB to 8TB. |
*/ |
#define _PAGE_BIT_NUMA (_PAGE_BIT_GLOBAL+1) |
/* If _PAGE_BIT_PRESENT is clear, we use these: */ |
/* - if the user mapped it with PROT_NONE; pte_present gives true */ |
#define _PAGE_BIT_PROTNONE _PAGE_BIT_GLOBAL |
/* - set: nonlinear file mapping, saved PTE; unset:swap */ |
#define _PAGE_BIT_FILE _PAGE_BIT_DIRTY |
#define _PAGE_PRESENT (_AT(pteval_t, 1) << _PAGE_BIT_PRESENT) |
#define _PAGE_RW (_AT(pteval_t, 1) << _PAGE_BIT_RW) |
#define _PAGE_USER (_AT(pteval_t, 1) << _PAGE_BIT_USER) |
#define _PAGE_PWT (_AT(pteval_t, 1) << _PAGE_BIT_PWT) |
#define _PAGE_PCD (_AT(pteval_t, 1) << _PAGE_BIT_PCD) |
#define _PAGE_ACCESSED (_AT(pteval_t, 1) << _PAGE_BIT_ACCESSED) |
#define _PAGE_DIRTY (_AT(pteval_t, 1) << _PAGE_BIT_DIRTY) |
#define _PAGE_PSE (_AT(pteval_t, 1) << _PAGE_BIT_PSE) |
#define _PAGE_GLOBAL (_AT(pteval_t, 1) << _PAGE_BIT_GLOBAL) |
#define _PAGE_SOFTW1 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW1) |
#define _PAGE_SOFTW2 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW2) |
#define _PAGE_PAT (_AT(pteval_t, 1) << _PAGE_BIT_PAT) |
#define _PAGE_PAT_LARGE (_AT(pteval_t, 1) << _PAGE_BIT_PAT_LARGE) |
#define _PAGE_SPECIAL (_AT(pteval_t, 1) << _PAGE_BIT_SPECIAL) |
#define _PAGE_CPA_TEST (_AT(pteval_t, 1) << _PAGE_BIT_CPA_TEST) |
#define _PAGE_SPLITTING (_AT(pteval_t, 1) << _PAGE_BIT_SPLITTING) |
#define __HAVE_ARCH_PTE_SPECIAL |
#ifdef CONFIG_KMEMCHECK |
#define _PAGE_HIDDEN (_AT(pteval_t, 1) << _PAGE_BIT_HIDDEN) |
#else |
#define _PAGE_HIDDEN (_AT(pteval_t, 0)) |
#endif |
/* |
* The same hidden bit is used by kmemcheck, but since kmemcheck |
* works on kernel pages while soft-dirty engine on user space, |
* they do not conflict with each other. |
*/ |
#ifdef CONFIG_MEM_SOFT_DIRTY |
#define _PAGE_SOFT_DIRTY (_AT(pteval_t, 1) << _PAGE_BIT_SOFT_DIRTY) |
#else |
#define _PAGE_SOFT_DIRTY (_AT(pteval_t, 0)) |
#endif |
/* |
* _PAGE_NUMA distinguishes between a numa hinting minor fault and a page |
* that is not present. The hinting fault gathers numa placement statistics |
* (see pte_numa()). The bit is always zero when the PTE is not present. |
* |
* The bit picked must be always zero when the pmd is present and not |
* present, so that we don't lose information when we set it while |
* atomically clearing the present bit. |
*/ |
#ifdef CONFIG_NUMA_BALANCING |
#define _PAGE_NUMA (_AT(pteval_t, 1) << _PAGE_BIT_NUMA) |
#else |
#define _PAGE_NUMA (_AT(pteval_t, 0)) |
#endif |
/* |
* Tracking soft dirty bit when a page goes to a swap is tricky. |
* We need a bit which can be stored in pte _and_ not conflict |
* with swap entry format. On x86 bits 6 and 7 are *not* involved |
* into swap entry computation, but bit 6 is used for nonlinear |
* file mapping, so we borrow bit 7 for soft dirty tracking. |
* |
* Please note that this bit must be treated as swap dirty page |
* mark if and only if the PTE has present bit clear! |
*/ |
#ifdef CONFIG_MEM_SOFT_DIRTY |
#define _PAGE_SWP_SOFT_DIRTY _PAGE_PSE |
#else |
#define _PAGE_SWP_SOFT_DIRTY (_AT(pteval_t, 0)) |
#endif |
#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) |
#define _PAGE_NX (_AT(pteval_t, 1) << _PAGE_BIT_NX) |
#else |
#define _PAGE_NX (_AT(pteval_t, 0)) |
#endif |
#define _PAGE_FILE (_AT(pteval_t, 1) << _PAGE_BIT_FILE) |
#define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE) |
#define _PAGE_TABLE (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | \ |
_PAGE_ACCESSED | _PAGE_DIRTY) |
#define _KERNPG_TABLE (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | \ |
_PAGE_DIRTY) |
/* Set of bits not changed in pte_modify */ |
#define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ |
_PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY | \ |
_PAGE_SOFT_DIRTY | _PAGE_NUMA) |
#define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_NUMA) |
/* |
* The cache modes defined here are used to translate between pure SW usage |
* and the HW defined cache mode bits and/or PAT entries. |
* |
* The resulting bits for PWT, PCD and PAT should be chosen in a way |
* to have the WB mode at index 0 (all bits clear). This is the default |
* right now and likely would break too much if changed. |
*/ |
#ifndef __ASSEMBLY__ |
enum page_cache_mode { |
_PAGE_CACHE_MODE_WB = 0, |
_PAGE_CACHE_MODE_WC = 1, |
_PAGE_CACHE_MODE_UC_MINUS = 2, |
_PAGE_CACHE_MODE_UC = 3, |
_PAGE_CACHE_MODE_WT = 4, |
_PAGE_CACHE_MODE_WP = 5, |
_PAGE_CACHE_MODE_NUM = 8 |
}; |
#endif |
#define _PAGE_CACHE_MASK (_PAGE_PAT | _PAGE_PCD | _PAGE_PWT) |
#define _PAGE_NOCACHE (cachemode2protval(_PAGE_CACHE_MODE_UC)) |
#define PAGE_NONE __pgprot(_PAGE_PROTNONE | _PAGE_ACCESSED) |
#define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | \ |
_PAGE_ACCESSED | _PAGE_NX) |
#define PAGE_SHARED_EXEC __pgprot(_PAGE_PRESENT | _PAGE_RW | \ |
_PAGE_USER | _PAGE_ACCESSED) |
#define PAGE_COPY_NOEXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | \ |
_PAGE_ACCESSED | _PAGE_NX) |
#define PAGE_COPY_EXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | \ |
_PAGE_ACCESSED) |
#define PAGE_COPY PAGE_COPY_NOEXEC |
#define PAGE_READONLY __pgprot(_PAGE_PRESENT | _PAGE_USER | \ |
_PAGE_ACCESSED | _PAGE_NX) |
#define PAGE_READONLY_EXEC __pgprot(_PAGE_PRESENT | _PAGE_USER | \ |
_PAGE_ACCESSED) |
#define __PAGE_KERNEL_EXEC \ |
(_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_GLOBAL) |
#define __PAGE_KERNEL (__PAGE_KERNEL_EXEC | _PAGE_NX) |
#define __PAGE_KERNEL_RO (__PAGE_KERNEL & ~_PAGE_RW) |
#define __PAGE_KERNEL_RX (__PAGE_KERNEL_EXEC & ~_PAGE_RW) |
#define __PAGE_KERNEL_NOCACHE (__PAGE_KERNEL | _PAGE_NOCACHE) |
#define __PAGE_KERNEL_VSYSCALL (__PAGE_KERNEL_RX | _PAGE_USER) |
#define __PAGE_KERNEL_VVAR (__PAGE_KERNEL_RO | _PAGE_USER) |
#define __PAGE_KERNEL_LARGE (__PAGE_KERNEL | _PAGE_PSE) |
#define __PAGE_KERNEL_LARGE_EXEC (__PAGE_KERNEL_EXEC | _PAGE_PSE) |
#define __PAGE_KERNEL_IO (__PAGE_KERNEL) |
#define __PAGE_KERNEL_IO_NOCACHE (__PAGE_KERNEL_NOCACHE) |
#define PAGE_KERNEL __pgprot(__PAGE_KERNEL) |
#define PAGE_KERNEL_RO __pgprot(__PAGE_KERNEL_RO) |
#define PAGE_KERNEL_EXEC __pgprot(__PAGE_KERNEL_EXEC) |
#define PAGE_KERNEL_RX __pgprot(__PAGE_KERNEL_RX) |
#define PAGE_KERNEL_NOCACHE __pgprot(__PAGE_KERNEL_NOCACHE) |
#define PAGE_KERNEL_LARGE __pgprot(__PAGE_KERNEL_LARGE) |
#define PAGE_KERNEL_LARGE_EXEC __pgprot(__PAGE_KERNEL_LARGE_EXEC) |
#define PAGE_KERNEL_VSYSCALL __pgprot(__PAGE_KERNEL_VSYSCALL) |
#define PAGE_KERNEL_VVAR __pgprot(__PAGE_KERNEL_VVAR) |
#define PAGE_KERNEL_IO __pgprot(__PAGE_KERNEL_IO) |
#define PAGE_KERNEL_IO_NOCACHE __pgprot(__PAGE_KERNEL_IO_NOCACHE) |
/* xwr */ |
#define __P000 PAGE_NONE |
#define __P001 PAGE_READONLY |
#define __P010 PAGE_COPY |
#define __P011 PAGE_COPY |
#define __P100 PAGE_READONLY_EXEC |
#define __P101 PAGE_READONLY_EXEC |
#define __P110 PAGE_COPY_EXEC |
#define __P111 PAGE_COPY_EXEC |
#define __S000 PAGE_NONE |
#define __S001 PAGE_READONLY |
#define __S010 PAGE_SHARED |
#define __S011 PAGE_SHARED |
#define __S100 PAGE_READONLY_EXEC |
#define __S101 PAGE_READONLY_EXEC |
#define __S110 PAGE_SHARED_EXEC |
#define __S111 PAGE_SHARED_EXEC |
/* |
* early identity mapping pte attrib macros. |
*/ |
#ifdef CONFIG_X86_64 |
#define __PAGE_KERNEL_IDENT_LARGE_EXEC __PAGE_KERNEL_LARGE_EXEC |
#else |
#define PTE_IDENT_ATTR 0x003 /* PRESENT+RW */ |
#define PDE_IDENT_ATTR 0x063 /* PRESENT+RW+DIRTY+ACCESSED */ |
#define PGD_IDENT_ATTR 0x001 /* PRESENT (no other attributes) */ |
#endif |
#ifdef CONFIG_X86_32 |
# include <asm/pgtable_32_types.h> |
#else |
# include <asm/pgtable_64_types.h> |
#endif |
#ifndef __ASSEMBLY__ |
#include <linux/types.h> |
/* PTE_PFN_MASK extracts the PFN from a (pte|pmd|pud|pgd)val_t */ |
#define PTE_PFN_MASK ((pteval_t)PHYSICAL_PAGE_MASK) |
/* PTE_FLAGS_MASK extracts the flags from a (pte|pmd|pud|pgd)val_t */ |
#define PTE_FLAGS_MASK (~PTE_PFN_MASK) |
typedef struct pgprot { pgprotval_t pgprot; } pgprot_t; |
typedef struct { pgdval_t pgd; } pgd_t; |
static inline pgd_t native_make_pgd(pgdval_t val) |
{ |
return (pgd_t) { val }; |
} |
static inline pgdval_t native_pgd_val(pgd_t pgd) |
{ |
return pgd.pgd; |
} |
static inline pgdval_t pgd_flags(pgd_t pgd) |
{ |
return native_pgd_val(pgd) & PTE_FLAGS_MASK; |
} |
#if PAGETABLE_LEVELS > 3 |
typedef struct { pudval_t pud; } pud_t; |
static inline pud_t native_make_pud(pmdval_t val) |
{ |
return (pud_t) { val }; |
} |
static inline pudval_t native_pud_val(pud_t pud) |
{ |
return pud.pud; |
} |
#else |
#include <asm-generic/pgtable-nopud.h> |
static inline pudval_t native_pud_val(pud_t pud) |
{ |
return native_pgd_val(pud.pgd); |
} |
#endif |
#if PAGETABLE_LEVELS > 2 |
typedef struct { pmdval_t pmd; } pmd_t; |
static inline pmd_t native_make_pmd(pmdval_t val) |
{ |
return (pmd_t) { val }; |
} |
static inline pmdval_t native_pmd_val(pmd_t pmd) |
{ |
return pmd.pmd; |
} |
#else |
#include <asm-generic/pgtable-nopmd.h> |
static inline pmdval_t native_pmd_val(pmd_t pmd) |
{ |
return native_pgd_val(pmd.pud.pgd); |
} |
#endif |
static inline pudval_t pud_flags(pud_t pud) |
{ |
return native_pud_val(pud) & PTE_FLAGS_MASK; |
} |
static inline pmdval_t pmd_flags(pmd_t pmd) |
{ |
return native_pmd_val(pmd) & PTE_FLAGS_MASK; |
} |
static inline pte_t native_make_pte(pteval_t val) |
{ |
return (pte_t) { .pte = val }; |
} |
static inline pteval_t native_pte_val(pte_t pte) |
{ |
return pte.pte; |
} |
static inline pteval_t pte_flags(pte_t pte) |
{ |
return native_pte_val(pte) & PTE_FLAGS_MASK; |
} |
#ifdef CONFIG_NUMA_BALANCING |
/* Set of bits that distinguishes present, prot_none and numa ptes */ |
#define _PAGE_NUMA_MASK (_PAGE_NUMA|_PAGE_PROTNONE|_PAGE_PRESENT) |
static inline pteval_t ptenuma_flags(pte_t pte) |
{ |
return pte_flags(pte) & _PAGE_NUMA_MASK; |
} |
static inline pmdval_t pmdnuma_flags(pmd_t pmd) |
{ |
return pmd_flags(pmd) & _PAGE_NUMA_MASK; |
} |
#endif /* CONFIG_NUMA_BALANCING */ |
#define pgprot_val(x) ((x).pgprot) |
#define __pgprot(x) ((pgprot_t) { (x) } ) |
extern uint16_t __cachemode2pte_tbl[_PAGE_CACHE_MODE_NUM]; |
extern uint8_t __pte2cachemode_tbl[8]; |
#define __pte2cm_idx(cb) \ |
((((cb) >> (_PAGE_BIT_PAT - 2)) & 4) | \ |
(((cb) >> (_PAGE_BIT_PCD - 1)) & 2) | \ |
(((cb) >> _PAGE_BIT_PWT) & 1)) |
#define __cm_idx2pte(i) \ |
((((i) & 4) << (_PAGE_BIT_PAT - 2)) | \ |
(((i) & 2) << (_PAGE_BIT_PCD - 1)) | \ |
(((i) & 1) << _PAGE_BIT_PWT)) |
static inline unsigned long cachemode2protval(enum page_cache_mode pcm) |
{ |
if (likely(pcm == 0)) |
return 0; |
return __cachemode2pte_tbl[pcm]; |
} |
static inline pgprot_t cachemode2pgprot(enum page_cache_mode pcm) |
{ |
return __pgprot(cachemode2protval(pcm)); |
} |
static inline enum page_cache_mode pgprot2cachemode(pgprot_t pgprot) |
{ |
unsigned long masked; |
masked = pgprot_val(pgprot) & _PAGE_CACHE_MASK; |
if (likely(masked == 0)) |
return 0; |
return __pte2cachemode_tbl[__pte2cm_idx(masked)]; |
} |
static inline pgprot_t pgprot_4k_2_large(pgprot_t pgprot) |
{ |
pgprot_t new; |
unsigned long val; |
val = pgprot_val(pgprot); |
pgprot_val(new) = (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) | |
((val & _PAGE_PAT) << (_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT)); |
return new; |
} |
static inline pgprot_t pgprot_large_2_4k(pgprot_t pgprot) |
{ |
pgprot_t new; |
unsigned long val; |
val = pgprot_val(pgprot); |
pgprot_val(new) = (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) | |
((val & _PAGE_PAT_LARGE) >> |
(_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT)); |
return new; |
} |
typedef struct page *pgtable_t; |
extern pteval_t __supported_pte_mask; |
extern void set_nx(void); |
extern int nx_enabled; |
#define pgprot_writecombine pgprot_writecombine |
extern pgprot_t pgprot_writecombine(pgprot_t prot); |
/* Indicate that x86 has its own track and untrack pfn vma functions */ |
#define __HAVE_PFNMAP_TRACKING |
#define __HAVE_PHYS_MEM_ACCESS_PROT |
struct file; |
pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, |
unsigned long size, pgprot_t vma_prot); |
int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, |
unsigned long size, pgprot_t *vma_prot); |
/* Install a pte for a particular vaddr in kernel space. */ |
void set_pte_vaddr(unsigned long vaddr, pte_t pte); |
#ifdef CONFIG_X86_32 |
extern void native_pagetable_init(void); |
#else |
#define native_pagetable_init paging_init |
#endif |
struct seq_file; |
extern void arch_report_meminfo(struct seq_file *m); |
enum pg_level { |
PG_LEVEL_NONE, |
PG_LEVEL_4K, |
PG_LEVEL_2M, |
PG_LEVEL_1G, |
PG_LEVEL_NUM |
}; |
#ifdef CONFIG_PROC_FS |
extern void update_page_count(int level, unsigned long pages); |
#else |
static inline void update_page_count(int level, unsigned long pages) { } |
#endif |
/* |
* Helper function that returns the kernel pagetable entry controlling |
* the virtual address 'address'. NULL means no pagetable entry present. |
* NOTE: the return type is pte_t but if the pmd is PSE then we return it |
* as a pte too. |
*/ |
extern pte_t *lookup_address(unsigned long address, unsigned int *level); |
extern pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address, |
unsigned int *level); |
extern pmd_t *lookup_pmd_address(unsigned long address); |
extern phys_addr_t slow_virt_to_phys(void *__address); |
extern int kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn, unsigned long address, |
unsigned numpages, unsigned long page_flags); |
void kernel_unmap_pages_in_pgd(pgd_t *root, unsigned long address, |
unsigned numpages); |
#endif /* !__ASSEMBLY__ */ |
#endif /* _ASM_X86_PGTABLE_DEFS_H */ |
/drivers/include/asm/posix_types.h |
---|
0,0 → 1,5 |
# ifdef CONFIG_X86_32 |
# include <asm/posix_types_32.h> |
# else |
# include <asm/posix_types_64.h> |
# endif |
/drivers/include/asm/posix_types_32.h |
---|
0,0 → 1,85 |
#ifndef _ASM_X86_POSIX_TYPES_32_H |
#define _ASM_X86_POSIX_TYPES_32_H |
/* |
* This file is generally used by user-level software, so you need to |
* be a little careful about namespace pollution etc. Also, we cannot |
* assume GCC is being used. |
*/ |
typedef unsigned long __kernel_ino_t; |
typedef unsigned short __kernel_mode_t; |
typedef unsigned short __kernel_nlink_t; |
typedef long __kernel_off_t; |
typedef int __kernel_pid_t; |
typedef unsigned short __kernel_ipc_pid_t; |
typedef unsigned short __kernel_uid_t; |
typedef unsigned short __kernel_gid_t; |
typedef unsigned int __kernel_size_t; |
typedef int __kernel_ssize_t; |
typedef int __kernel_ptrdiff_t; |
typedef long __kernel_time_t; |
typedef long __kernel_suseconds_t; |
typedef long __kernel_clock_t; |
typedef int __kernel_timer_t; |
typedef int __kernel_clockid_t; |
typedef int __kernel_daddr_t; |
typedef char * __kernel_caddr_t; |
typedef unsigned short __kernel_uid16_t; |
typedef unsigned short __kernel_gid16_t; |
typedef unsigned int __kernel_uid32_t; |
typedef unsigned int __kernel_gid32_t; |
typedef unsigned short __kernel_old_uid_t; |
typedef unsigned short __kernel_old_gid_t; |
typedef unsigned short __kernel_old_dev_t; |
#ifdef __GNUC__ |
typedef long long __kernel_loff_t; |
#endif |
typedef struct { |
int val[2]; |
} __kernel_fsid_t; |
#if defined(__KERNEL__) |
#undef __FD_SET |
#define __FD_SET(fd,fdsetp) \ |
asm volatile("btsl %1,%0": \ |
"+m" (*(__kernel_fd_set *)(fdsetp)) \ |
: "r" ((int)(fd))) |
#undef __FD_CLR |
#define __FD_CLR(fd,fdsetp) \ |
asm volatile("btrl %1,%0": \ |
"+m" (*(__kernel_fd_set *)(fdsetp)) \ |
: "r" ((int) (fd))) |
#undef __FD_ISSET |
#define __FD_ISSET(fd,fdsetp) \ |
(__extension__ \ |
({ \ |
unsigned char __result; \ |
asm volatile("btl %1,%2 ; setb %0" \ |
: "=q" (__result) \ |
: "r" ((int)(fd)), \ |
"m" (*(__kernel_fd_set *)(fdsetp))); \ |
__result; \ |
})) |
#undef __FD_ZERO |
#define __FD_ZERO(fdsetp) \ |
do { \ |
int __d0, __d1; \ |
asm volatile("cld ; rep ; stosl" \ |
: "=m" (*(__kernel_fd_set *)(fdsetp)), \ |
"=&c" (__d0), "=&D" (__d1) \ |
: "a" (0), "1" (__FDSET_LONGS), \ |
"2" ((__kernel_fd_set *)(fdsetp)) \ |
: "memory"); \ |
} while (0) |
#endif /* defined(__KERNEL__) */ |
#endif /* _ASM_X86_POSIX_TYPES_32_H */ |
/drivers/include/asm/processor-flags.h |
---|
0,0 → 1,11 |
#ifndef _ASM_X86_PROCESSOR_FLAGS_H |
#define _ASM_X86_PROCESSOR_FLAGS_H |
#include <uapi/asm/processor-flags.h> |
#ifdef CONFIG_VM86 |
#define X86_VM_MASK X86_EFLAGS_VM |
#else |
#define X86_VM_MASK 0 /* No VM86 support */ |
#endif |
#endif /* _ASM_X86_PROCESSOR_FLAGS_H */ |
/drivers/include/asm/processor.h |
---|
0,0 → 1,1010 |
#ifndef _ASM_X86_PROCESSOR_H |
#define _ASM_X86_PROCESSOR_H |
#include <asm/processor-flags.h> |
/* Forward declaration, a strange C thing */ |
struct task_struct; |
struct mm_struct; |
#include <asm/vm86.h> |
#include <asm/math_emu.h> |
#include <asm/segment.h> |
#include <asm/types.h> |
#include <asm/sigcontext.h> |
#include <asm/current.h> |
#include <asm/cpufeature.h> |
#include <asm/page.h> |
#include <asm/pgtable_types.h> |
#include <asm/percpu.h> |
#include <asm/msr.h> |
#include <asm/desc_defs.h> |
#include <asm/nops.h> |
#include <asm/special_insns.h> |
#include <linux/personality.h> |
#include <linux/cpumask.h> |
#include <linux/cache.h> |
#include <linux/threads.h> |
#include <linux/math64.h> |
#include <linux/err.h> |
#include <linux/irqflags.h> |
/* |
* We handle most unaligned accesses in hardware. On the other hand |
* unaligned DMA can be quite expensive on some Nehalem processors. |
* |
* Based on this we disable the IP header alignment in network drivers. |
*/ |
#define NET_IP_ALIGN 0 |
#define HBP_NUM 4 |
/* |
* Default implementation of macro that returns current |
* instruction pointer ("program counter"). |
*/ |
static inline void *current_text_addr(void) |
{ |
void *pc; |
asm volatile("mov $1f, %0; 1:":"=r" (pc)); |
return pc; |
} |
#ifdef CONFIG_X86_VSMP |
# define ARCH_MIN_TASKALIGN (1 << INTERNODE_CACHE_SHIFT) |
# define ARCH_MIN_MMSTRUCT_ALIGN (1 << INTERNODE_CACHE_SHIFT) |
#else |
# define ARCH_MIN_TASKALIGN 16 |
# define ARCH_MIN_MMSTRUCT_ALIGN 0 |
#endif |
enum tlb_infos { |
ENTRIES, |
NR_INFO |
}; |
extern u16 __read_mostly tlb_lli_4k[NR_INFO]; |
extern u16 __read_mostly tlb_lli_2m[NR_INFO]; |
extern u16 __read_mostly tlb_lli_4m[NR_INFO]; |
extern u16 __read_mostly tlb_lld_4k[NR_INFO]; |
extern u16 __read_mostly tlb_lld_2m[NR_INFO]; |
extern u16 __read_mostly tlb_lld_4m[NR_INFO]; |
extern u16 __read_mostly tlb_lld_1g[NR_INFO]; |
/* |
* CPU type and hardware bug flags. Kept separately for each CPU. |
* Members of this structure are referenced in head.S, so think twice |
* before touching them. [mj] |
*/ |
struct cpuinfo_x86 { |
__u8 x86; /* CPU family */ |
__u8 x86_vendor; /* CPU vendor */ |
__u8 x86_model; |
__u8 x86_mask; |
#ifdef CONFIG_X86_32 |
char wp_works_ok; /* It doesn't on 386's */ |
/* Problems on some 486Dx4's and old 386's: */ |
char rfu; |
char pad0; |
char pad1; |
#else |
/* Number of 4K pages in DTLB/ITLB combined(in pages): */ |
int x86_tlbsize; |
#endif |
__u8 x86_virt_bits; |
__u8 x86_phys_bits; |
/* CPUID returned core id bits: */ |
__u8 x86_coreid_bits; |
/* Max extended CPUID function supported: */ |
__u32 extended_cpuid_level; |
/* Maximum supported CPUID level, -1=no CPUID: */ |
int cpuid_level; |
__u32 x86_capability[NCAPINTS + NBUGINTS]; |
char x86_vendor_id[16]; |
char x86_model_id[64]; |
/* in KB - valid for CPUS which support this call: */ |
int x86_cache_size; |
int x86_cache_alignment; /* In bytes */ |
int x86_power; |
unsigned long loops_per_jiffy; |
/* cpuid returned max cores value: */ |
u16 x86_max_cores; |
u16 apicid; |
u16 initial_apicid; |
u16 x86_clflush_size; |
/* number of cores as seen by the OS: */ |
u16 booted_cores; |
/* Physical processor id: */ |
u16 phys_proc_id; |
/* Core id: */ |
u16 cpu_core_id; |
/* Compute unit id */ |
u8 compute_unit_id; |
/* Index into per_cpu list: */ |
u16 cpu_index; |
u32 microcode; |
}; |
#define X86_VENDOR_INTEL 0 |
#define X86_VENDOR_CYRIX 1 |
#define X86_VENDOR_AMD 2 |
#define X86_VENDOR_UMC 3 |
#define X86_VENDOR_CENTAUR 5 |
#define X86_VENDOR_TRANSMETA 7 |
#define X86_VENDOR_NSC 8 |
#define X86_VENDOR_NUM 9 |
#define X86_VENDOR_UNKNOWN 0xff |
/* |
* capabilities of CPUs |
*/ |
extern struct cpuinfo_x86 boot_cpu_data; |
extern struct cpuinfo_x86 new_cpu_data; |
extern struct tss_struct doublefault_tss; |
extern __u32 cpu_caps_cleared[NCAPINTS]; |
extern __u32 cpu_caps_set[NCAPINTS]; |
#ifdef CONFIG_SMP |
DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info); |
#define cpu_data(cpu) per_cpu(cpu_info, cpu) |
#else |
#define cpu_info boot_cpu_data |
#define cpu_data(cpu) boot_cpu_data |
#endif |
extern const struct seq_operations cpuinfo_op; |
#define cache_line_size() (x86_cache_alignment) |
extern void cpu_detect(struct cpuinfo_x86 *c); |
extern void fpu_detect(struct cpuinfo_x86 *c); |
extern void early_cpu_init(void); |
extern void identify_boot_cpu(void); |
extern void identify_secondary_cpu(struct cpuinfo_x86 *); |
extern void print_cpu_info(struct cpuinfo_x86 *); |
void print_cpu_msr(struct cpuinfo_x86 *); |
extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c); |
extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c); |
extern void init_amd_cacheinfo(struct cpuinfo_x86 *c); |
extern void detect_extended_topology(struct cpuinfo_x86 *c); |
extern void detect_ht(struct cpuinfo_x86 *c); |
#ifdef CONFIG_X86_32 |
extern int have_cpuid_p(void); |
#else |
static inline int have_cpuid_p(void) |
{ |
return 1; |
} |
#endif |
static inline void native_cpuid(unsigned int *eax, unsigned int *ebx, |
unsigned int *ecx, unsigned int *edx) |
{ |
/* ecx is often an input as well as an output. */ |
asm volatile("cpuid" |
: "=a" (*eax), |
"=b" (*ebx), |
"=c" (*ecx), |
"=d" (*edx) |
: "0" (*eax), "2" (*ecx) |
: "memory"); |
} |
static inline void load_cr3(pgd_t *pgdir) |
{ |
write_cr3(__pa(pgdir)); |
} |
#ifdef CONFIG_X86_32 |
/* This is the TSS defined by the hardware. */ |
struct x86_hw_tss { |
unsigned short back_link, __blh; |
unsigned long sp0; |
unsigned short ss0, __ss0h; |
unsigned long sp1; |
/* ss1 caches MSR_IA32_SYSENTER_CS: */ |
unsigned short ss1, __ss1h; |
unsigned long sp2; |
unsigned short ss2, __ss2h; |
unsigned long __cr3; |
unsigned long ip; |
unsigned long flags; |
unsigned long ax; |
unsigned long cx; |
unsigned long dx; |
unsigned long bx; |
unsigned long sp; |
unsigned long bp; |
unsigned long si; |
unsigned long di; |
unsigned short es, __esh; |
unsigned short cs, __csh; |
unsigned short ss, __ssh; |
unsigned short ds, __dsh; |
unsigned short fs, __fsh; |
unsigned short gs, __gsh; |
unsigned short ldt, __ldth; |
unsigned short trace; |
unsigned short io_bitmap_base; |
} __attribute__((packed)); |
#else |
struct x86_hw_tss { |
u32 reserved1; |
u64 sp0; |
u64 sp1; |
u64 sp2; |
u64 reserved2; |
u64 ist[7]; |
u32 reserved3; |
u32 reserved4; |
u16 reserved5; |
u16 io_bitmap_base; |
} __attribute__((packed)) ____cacheline_aligned; |
#endif |
/* |
* IO-bitmap sizes: |
*/ |
#define IO_BITMAP_BITS 65536 |
#define IO_BITMAP_BYTES (IO_BITMAP_BITS/8) |
#define IO_BITMAP_LONGS (IO_BITMAP_BYTES/sizeof(long)) |
#define IO_BITMAP_OFFSET offsetof(struct tss_struct, io_bitmap) |
#define INVALID_IO_BITMAP_OFFSET 0x8000 |
struct tss_struct { |
/* |
* The hardware state: |
*/ |
struct x86_hw_tss x86_tss; |
/* |
* The extra 1 is there because the CPU will access an |
* additional byte beyond the end of the IO permission |
* bitmap. The extra byte must be all 1 bits, and must |
* be within the limit. |
*/ |
unsigned long io_bitmap[IO_BITMAP_LONGS + 1]; |
/* |
* .. and then another 0x100 bytes for the emergency kernel stack: |
*/ |
unsigned long stack[64]; |
} ____cacheline_aligned; |
DECLARE_PER_CPU_SHARED_ALIGNED(struct tss_struct, init_tss); |
/* |
* Save the original ist values for checking stack pointers during debugging |
*/ |
struct orig_ist { |
unsigned long ist[7]; |
}; |
#define MXCSR_DEFAULT 0x1f80 |
struct i387_fsave_struct { |
u32 cwd; /* FPU Control Word */ |
u32 swd; /* FPU Status Word */ |
u32 twd; /* FPU Tag Word */ |
u32 fip; /* FPU IP Offset */ |
u32 fcs; /* FPU IP Selector */ |
u32 foo; /* FPU Operand Pointer Offset */ |
u32 fos; /* FPU Operand Pointer Selector */ |
/* 8*10 bytes for each FP-reg = 80 bytes: */ |
u32 st_space[20]; |
/* Software status information [not touched by FSAVE ]: */ |
u32 status; |
}; |
struct i387_fxsave_struct { |
u16 cwd; /* Control Word */ |
u16 swd; /* Status Word */ |
u16 twd; /* Tag Word */ |
u16 fop; /* Last Instruction Opcode */ |
union { |
struct { |
u64 rip; /* Instruction Pointer */ |
u64 rdp; /* Data Pointer */ |
}; |
struct { |
u32 fip; /* FPU IP Offset */ |
u32 fcs; /* FPU IP Selector */ |
u32 foo; /* FPU Operand Offset */ |
u32 fos; /* FPU Operand Selector */ |
}; |
}; |
u32 mxcsr; /* MXCSR Register State */ |
u32 mxcsr_mask; /* MXCSR Mask */ |
/* 8*16 bytes for each FP-reg = 128 bytes: */ |
u32 st_space[32]; |
/* 16*16 bytes for each XMM-reg = 256 bytes: */ |
u32 xmm_space[64]; |
u32 padding[12]; |
union { |
u32 padding1[12]; |
u32 sw_reserved[12]; |
}; |
} __attribute__((aligned(16))); |
struct i387_soft_struct { |
u32 cwd; |
u32 swd; |
u32 twd; |
u32 fip; |
u32 fcs; |
u32 foo; |
u32 fos; |
/* 8*10 bytes for each FP-reg = 80 bytes: */ |
u32 st_space[20]; |
u8 ftop; |
u8 changed; |
u8 lookahead; |
u8 no_update; |
u8 rm; |
u8 alimit; |
struct math_emu_info *info; |
u32 entry_eip; |
}; |
struct ymmh_struct { |
/* 16 * 16 bytes for each YMMH-reg = 256 bytes */ |
u32 ymmh_space[64]; |
}; |
/* We don't support LWP yet: */ |
struct lwp_struct { |
u8 reserved[128]; |
}; |
struct bndreg { |
u64 lower_bound; |
u64 upper_bound; |
} __packed; |
struct bndcsr { |
u64 bndcfgu; |
u64 bndstatus; |
} __packed; |
struct xsave_hdr_struct { |
u64 xstate_bv; |
u64 xcomp_bv; |
u64 reserved[6]; |
} __attribute__((packed)); |
struct xsave_struct { |
struct i387_fxsave_struct i387; |
struct xsave_hdr_struct xsave_hdr; |
struct ymmh_struct ymmh; |
struct lwp_struct lwp; |
struct bndreg bndreg[4]; |
struct bndcsr bndcsr; |
/* new processor state extensions will go here */ |
} __attribute__ ((packed, aligned (64))); |
union thread_xstate { |
struct i387_fsave_struct fsave; |
struct i387_fxsave_struct fxsave; |
struct i387_soft_struct soft; |
struct xsave_struct xsave; |
}; |
struct fpu { |
unsigned int last_cpu; |
unsigned int has_fpu; |
union thread_xstate *state; |
}; |
#ifdef CONFIG_X86_64 |
DECLARE_PER_CPU(struct orig_ist, orig_ist); |
union irq_stack_union { |
char irq_stack[IRQ_STACK_SIZE]; |
/* |
* GCC hardcodes the stack canary as %gs:40. Since the |
* irq_stack is the object at %gs:0, we reserve the bottom |
* 48 bytes of the irq stack for the canary. |
*/ |
struct { |
char gs_base[40]; |
unsigned long stack_canary; |
}; |
}; |
DECLARE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union) __visible; |
DECLARE_INIT_PER_CPU(irq_stack_union); |
DECLARE_PER_CPU(char *, irq_stack_ptr); |
DECLARE_PER_CPU(unsigned int, irq_count); |
extern asmlinkage void ignore_sysret(void); |
#else /* X86_64 */ |
#ifdef CONFIG_CC_STACKPROTECTOR |
/* |
* Make sure stack canary segment base is cached-aligned: |
* "For Intel Atom processors, avoid non zero segment base address |
* that is not aligned to cache line boundary at all cost." |
* (Optim Ref Manual Assembly/Compiler Coding Rule 15.) |
*/ |
struct stack_canary { |
char __pad[20]; /* canary at %gs:20 */ |
unsigned long canary; |
}; |
DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary); |
#endif |
/* |
* per-CPU IRQ handling stacks |
*/ |
struct irq_stack { |
u32 stack[THREAD_SIZE/sizeof(u32)]; |
} __aligned(THREAD_SIZE); |
DECLARE_PER_CPU(struct irq_stack *, hardirq_stack); |
DECLARE_PER_CPU(struct irq_stack *, softirq_stack); |
#endif /* X86_64 */ |
extern unsigned int xstate_size; |
extern void free_thread_xstate(struct task_struct *); |
extern struct kmem_cache *task_xstate_cachep; |
struct perf_event; |
struct thread_struct { |
/* Cached TLS descriptors: */ |
struct desc_struct tls_array[GDT_ENTRY_TLS_ENTRIES]; |
unsigned long sp0; |
unsigned long sp; |
#ifdef CONFIG_X86_32 |
unsigned long sysenter_cs; |
#else |
unsigned long usersp; /* Copy from PDA */ |
unsigned short es; |
unsigned short ds; |
unsigned short fsindex; |
unsigned short gsindex; |
#endif |
#ifdef CONFIG_X86_32 |
unsigned long ip; |
#endif |
#ifdef CONFIG_X86_64 |
unsigned long fs; |
#endif |
unsigned long gs; |
/* Save middle states of ptrace breakpoints */ |
struct perf_event *ptrace_bps[HBP_NUM]; |
/* Debug status used for traps, single steps, etc... */ |
unsigned long debugreg6; |
/* Keep track of the exact dr7 value set by the user */ |
unsigned long ptrace_dr7; |
/* Fault info: */ |
unsigned long cr2; |
unsigned long trap_nr; |
unsigned long error_code; |
/* floating point and extended processor state */ |
struct fpu fpu; |
#ifdef CONFIG_X86_32 |
/* Virtual 86 mode info */ |
struct vm86_struct __user *vm86_info; |
unsigned long screen_bitmap; |
unsigned long v86flags; |
unsigned long v86mask; |
unsigned long saved_sp0; |
unsigned int saved_fs; |
unsigned int saved_gs; |
#endif |
/* IO permissions: */ |
unsigned long *io_bitmap_ptr; |
unsigned long iopl; |
/* Max allowed port in the bitmap, in bytes: */ |
unsigned io_bitmap_max; |
/* |
* fpu_counter contains the number of consecutive context switches |
* that the FPU is used. If this is over a threshold, the lazy fpu |
* saving becomes unlazy to save the trap. This is an unsigned char |
* so that after 256 times the counter wraps and the behavior turns |
* lazy again; this to deal with bursty apps that only use FPU for |
* a short time |
*/ |
unsigned char fpu_counter; |
}; |
/* |
* Set IOPL bits in EFLAGS from given mask |
*/ |
static inline void native_set_iopl_mask(unsigned mask) |
{ |
#ifdef CONFIG_X86_32 |
unsigned int reg; |
asm volatile ("pushfl;" |
"popl %0;" |
"andl %1, %0;" |
"orl %2, %0;" |
"pushl %0;" |
"popfl" |
: "=&r" (reg) |
: "i" (~X86_EFLAGS_IOPL), "r" (mask)); |
#endif |
} |
static inline void |
native_load_sp0(struct tss_struct *tss, struct thread_struct *thread) |
{ |
tss->x86_tss.sp0 = thread->sp0; |
#ifdef CONFIG_X86_32 |
/* Only happens when SEP is enabled, no need to test "SEP"arately: */ |
if (unlikely(tss->x86_tss.ss1 != thread->sysenter_cs)) { |
tss->x86_tss.ss1 = thread->sysenter_cs; |
wrmsr(MSR_IA32_SYSENTER_CS, thread->sysenter_cs, 0); |
} |
#endif |
} |
static inline void native_swapgs(void) |
{ |
#ifdef CONFIG_X86_64 |
asm volatile("swapgs" ::: "memory"); |
#endif |
} |
#ifdef CONFIG_PARAVIRT |
#include <asm/paravirt.h> |
#else |
#define __cpuid native_cpuid |
#define paravirt_enabled() 0 |
static inline void load_sp0(struct tss_struct *tss, |
struct thread_struct *thread) |
{ |
native_load_sp0(tss, thread); |
} |
#define set_iopl_mask native_set_iopl_mask |
#endif /* CONFIG_PARAVIRT */ |
/* |
* Save the cr4 feature set we're using (ie |
* Pentium 4MB enable and PPro Global page |
* enable), so that any CPU's that boot up |
* after us can get the correct flags. |
*/ |
extern unsigned long mmu_cr4_features; |
extern u32 *trampoline_cr4_features; |
static inline void set_in_cr4(unsigned long mask) |
{ |
unsigned long cr4; |
mmu_cr4_features |= mask; |
if (trampoline_cr4_features) |
*trampoline_cr4_features = mmu_cr4_features; |
cr4 = read_cr4(); |
cr4 |= mask; |
write_cr4(cr4); |
} |
static inline void clear_in_cr4(unsigned long mask) |
{ |
unsigned long cr4; |
mmu_cr4_features &= ~mask; |
if (trampoline_cr4_features) |
*trampoline_cr4_features = mmu_cr4_features; |
cr4 = read_cr4(); |
cr4 &= ~mask; |
write_cr4(cr4); |
} |
typedef struct { |
unsigned long seg; |
} mm_segment_t; |
/* Free all resources held by a thread. */ |
extern void release_thread(struct task_struct *); |
unsigned long get_wchan(struct task_struct *p); |
/* |
* Generic CPUID function |
* clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx |
* resulting in stale register contents being returned. |
*/ |
static inline void cpuid(unsigned int op, |
unsigned int *eax, unsigned int *ebx, |
unsigned int *ecx, unsigned int *edx) |
{ |
*eax = op; |
*ecx = 0; |
__cpuid(eax, ebx, ecx, edx); |
} |
/* Some CPUID calls want 'count' to be placed in ecx */ |
static inline void cpuid_count(unsigned int op, int count, |
unsigned int *eax, unsigned int *ebx, |
unsigned int *ecx, unsigned int *edx) |
{ |
*eax = op; |
*ecx = count; |
__cpuid(eax, ebx, ecx, edx); |
} |
/* |
* CPUID functions returning a single datum |
*/ |
static inline unsigned int cpuid_eax(unsigned int op) |
{ |
unsigned int eax, ebx, ecx, edx; |
cpuid(op, &eax, &ebx, &ecx, &edx); |
return eax; |
} |
static inline unsigned int cpuid_ebx(unsigned int op) |
{ |
unsigned int eax, ebx, ecx, edx; |
cpuid(op, &eax, &ebx, &ecx, &edx); |
return ebx; |
} |
static inline unsigned int cpuid_ecx(unsigned int op) |
{ |
unsigned int eax, ebx, ecx, edx; |
cpuid(op, &eax, &ebx, &ecx, &edx); |
return ecx; |
} |
static inline unsigned int cpuid_edx(unsigned int op) |
{ |
unsigned int eax, ebx, ecx, edx; |
cpuid(op, &eax, &ebx, &ecx, &edx); |
return edx; |
} |
/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */ |
static inline void rep_nop(void) |
{ |
asm volatile("rep; nop" ::: "memory"); |
} |
static inline void cpu_relax(void) |
{ |
rep_nop(); |
} |
#define cpu_relax_lowlatency() cpu_relax() |
/* Stop speculative execution and prefetching of modified code. */ |
static inline void sync_core(void) |
{ |
int tmp; |
#ifdef CONFIG_M486 |
/* |
* Do a CPUID if available, otherwise do a jump. The jump |
* can conveniently enough be the jump around CPUID. |
*/ |
asm volatile("cmpl %2,%1\n\t" |
"jl 1f\n\t" |
"cpuid\n" |
"1:" |
: "=a" (tmp) |
: "rm" (boot_cpu_data.cpuid_level), "ri" (0), "0" (1) |
: "ebx", "ecx", "edx", "memory"); |
#else |
/* |
* CPUID is a barrier to speculative execution. |
* Prefetched instructions are automatically |
* invalidated when modified. |
*/ |
asm volatile("cpuid" |
: "=a" (tmp) |
: "0" (1) |
: "ebx", "ecx", "edx", "memory"); |
#endif |
} |
extern void select_idle_routine(const struct cpuinfo_x86 *c); |
extern void init_amd_e400_c1e_mask(void); |
extern unsigned long boot_option_idle_override; |
extern bool amd_e400_c1e_detected; |
enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT, |
IDLE_POLL}; |
extern void enable_sep_cpu(void); |
extern int sysenter_setup(void); |
extern void early_trap_init(void); |
void early_trap_pf_init(void); |
/* Defined in head.S */ |
extern struct desc_ptr early_gdt_descr; |
extern void cpu_set_gdt(int); |
extern void switch_to_new_gdt(int); |
extern void load_percpu_segment(int); |
extern void cpu_init(void); |
static inline unsigned long get_debugctlmsr(void) |
{ |
unsigned long debugctlmsr = 0; |
#ifndef CONFIG_X86_DEBUGCTLMSR |
if (boot_cpu_data.x86 < 6) |
return 0; |
#endif |
rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr); |
return debugctlmsr; |
} |
static inline void update_debugctlmsr(unsigned long debugctlmsr) |
{ |
#ifndef CONFIG_X86_DEBUGCTLMSR |
if (boot_cpu_data.x86 < 6) |
return; |
#endif |
wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr); |
} |
extern void set_task_blockstep(struct task_struct *task, bool on); |
/* |
* from system description table in BIOS. Mostly for MCA use, but |
* others may find it useful: |
*/ |
extern unsigned int machine_id; |
extern unsigned int machine_submodel_id; |
extern unsigned int BIOS_revision; |
/* Boot loader type from the setup header: */ |
extern int bootloader_type; |
extern int bootloader_version; |
extern char ignore_fpu_irq; |
#define HAVE_ARCH_PICK_MMAP_LAYOUT 1 |
#define ARCH_HAS_PREFETCHW |
#define ARCH_HAS_SPINLOCK_PREFETCH |
#ifdef CONFIG_X86_32 |
# define BASE_PREFETCH ASM_NOP4 |
# define ARCH_HAS_PREFETCH |
#else |
# define BASE_PREFETCH "prefetcht0 (%1)" |
#endif |
/* |
* Prefetch instructions for Pentium III (+) and AMD Athlon (+) |
* |
* It's not worth to care about 3dnow prefetches for the K6 |
* because they are microcoded there and very slow. |
*/ |
static inline void prefetch(const void *x) |
{ |
alternative_input(BASE_PREFETCH, |
"prefetchnta (%1)", |
X86_FEATURE_XMM, |
"r" (x)); |
} |
/* |
* 3dnow prefetch to get an exclusive cache line. |
* Useful for spinlocks to avoid one state transition in the |
* cache coherency protocol: |
*/ |
static inline void prefetchw(const void *x) |
{ |
alternative_input(BASE_PREFETCH, |
"prefetchw (%1)", |
X86_FEATURE_3DNOW, |
"r" (x)); |
} |
static inline void spin_lock_prefetch(const void *x) |
{ |
prefetchw(x); |
} |
#ifdef CONFIG_X86_32 |
/* |
* User space process size: 3GB (default). |
*/ |
#define TASK_SIZE PAGE_OFFSET |
#define TASK_SIZE_MAX TASK_SIZE |
#define STACK_TOP TASK_SIZE |
#define STACK_TOP_MAX STACK_TOP |
#define INIT_THREAD { \ |
.sp0 = sizeof(init_stack) + (long)&init_stack, \ |
.vm86_info = NULL, \ |
.sysenter_cs = __KERNEL_CS, \ |
.io_bitmap_ptr = NULL, \ |
} |
/* |
* Note that the .io_bitmap member must be extra-big. This is because |
* the CPU will access an additional byte beyond the end of the IO |
* permission bitmap. The extra byte must be all 1 bits, and must |
* be within the limit. |
*/ |
#define INIT_TSS { \ |
.x86_tss = { \ |
.sp0 = sizeof(init_stack) + (long)&init_stack, \ |
.ss0 = __KERNEL_DS, \ |
.ss1 = __KERNEL_CS, \ |
.io_bitmap_base = INVALID_IO_BITMAP_OFFSET, \ |
}, \ |
.io_bitmap = { [0 ... IO_BITMAP_LONGS] = ~0 }, \ |
} |
extern unsigned long thread_saved_pc(struct task_struct *tsk); |
#define THREAD_SIZE_LONGS (THREAD_SIZE/sizeof(unsigned long)) |
#define KSTK_TOP(info) \ |
({ \ |
unsigned long *__ptr = (unsigned long *)(info); \ |
(unsigned long)(&__ptr[THREAD_SIZE_LONGS]); \ |
}) |
/* |
* The below -8 is to reserve 8 bytes on top of the ring0 stack. |
* This is necessary to guarantee that the entire "struct pt_regs" |
* is accessible even if the CPU haven't stored the SS/ESP registers |
* on the stack (interrupt gate does not save these registers |
* when switching to the same priv ring). |
* Therefore beware: accessing the ss/esp fields of the |
* "struct pt_regs" is possible, but they may contain the |
* completely wrong values. |
*/ |
#define task_pt_regs(task) \ |
({ \ |
struct pt_regs *__regs__; \ |
__regs__ = (struct pt_regs *)(KSTK_TOP(task_stack_page(task))-8); \ |
__regs__ - 1; \ |
}) |
#define KSTK_ESP(task) (task_pt_regs(task)->sp) |
#else |
/* |
* User space process size. 47bits minus one guard page. The guard |
* page is necessary on Intel CPUs: if a SYSCALL instruction is at |
* the highest possible canonical userspace address, then that |
* syscall will enter the kernel with a non-canonical return |
* address, and SYSRET will explode dangerously. We avoid this |
* particular problem by preventing anything from being mapped |
* at the maximum canonical address. |
*/ |
#define TASK_SIZE_MAX ((1UL << 47) - PAGE_SIZE) |
/* This decides where the kernel will search for a free chunk of vm |
* space during mmap's. |
*/ |
#define IA32_PAGE_OFFSET ((current->personality & ADDR_LIMIT_3GB) ? \ |
0xc0000000 : 0xFFFFe000) |
#define TASK_SIZE (test_thread_flag(TIF_ADDR32) ? \ |
IA32_PAGE_OFFSET : TASK_SIZE_MAX) |
#define TASK_SIZE_OF(child) ((test_tsk_thread_flag(child, TIF_ADDR32)) ? \ |
IA32_PAGE_OFFSET : TASK_SIZE_MAX) |
#define STACK_TOP TASK_SIZE |
#define STACK_TOP_MAX TASK_SIZE_MAX |
#define INIT_THREAD { \ |
.sp0 = (unsigned long)&init_stack + sizeof(init_stack) \ |
} |
#define INIT_TSS { \ |
.x86_tss.sp0 = (unsigned long)&init_stack + sizeof(init_stack) \ |
} |
/* |
* Return saved PC of a blocked thread. |
* What is this good for? it will be always the scheduler or ret_from_fork. |
*/ |
#define thread_saved_pc(t) (*(unsigned long *)((t)->thread.sp - 8)) |
#define task_pt_regs(tsk) ((struct pt_regs *)(tsk)->thread.sp0 - 1) |
extern unsigned long KSTK_ESP(struct task_struct *task); |
/* |
* User space RSP while inside the SYSCALL fast path |
*/ |
DECLARE_PER_CPU(unsigned long, old_rsp); |
#endif /* CONFIG_X86_64 */ |
extern void start_thread(struct pt_regs *regs, unsigned long new_ip, |
unsigned long new_sp); |
/* |
* This decides where the kernel will search for a free chunk of vm |
* space during mmap's. |
*/ |
#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 3)) |
#define KSTK_EIP(task) (task_pt_regs(task)->ip) |
/* Get/set a process' ability to use the timestamp counter instruction */ |
#define GET_TSC_CTL(adr) get_tsc_mode((adr)) |
#define SET_TSC_CTL(val) set_tsc_mode((val)) |
extern int get_tsc_mode(unsigned long adr); |
extern int set_tsc_mode(unsigned int val); |
/* Register/unregister a process' MPX related resource */ |
#define MPX_ENABLE_MANAGEMENT(tsk) mpx_enable_management((tsk)) |
#define MPX_DISABLE_MANAGEMENT(tsk) mpx_disable_management((tsk)) |
#ifdef CONFIG_X86_INTEL_MPX |
extern int mpx_enable_management(struct task_struct *tsk); |
extern int mpx_disable_management(struct task_struct *tsk); |
#else |
static inline int mpx_enable_management(struct task_struct *tsk) |
{ |
return -EINVAL; |
} |
static inline int mpx_disable_management(struct task_struct *tsk) |
{ |
return -EINVAL; |
} |
#endif /* CONFIG_X86_INTEL_MPX */ |
extern u16 amd_get_nb_id(int cpu); |
static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves) |
{ |
uint32_t base, eax, signature[3]; |
for (base = 0x40000000; base < 0x40010000; base += 0x100) { |
cpuid(base, &eax, &signature[0], &signature[1], &signature[2]); |
if (!memcmp(sig, signature, 12) && |
(leaves == 0 || ((eax - base) >= leaves))) |
return base; |
} |
return 0; |
} |
extern unsigned long arch_align_stack(unsigned long sp); |
extern void free_init_pages(char *what, unsigned long begin, unsigned long end); |
void default_idle(void); |
#ifdef CONFIG_XEN |
bool xen_set_default_idle(void); |
#else |
#define xen_set_default_idle 0 |
#endif |
void stop_this_cpu(void *dummy); |
void df_debug(struct pt_regs *regs, long error_code); |
#endif /* _ASM_X86_PROCESSOR_H */ |
/drivers/include/asm/required-features.h |
---|
0,0 → 1,96 |
#ifndef _ASM_X86_REQUIRED_FEATURES_H |
#define _ASM_X86_REQUIRED_FEATURES_H |
/* Define minimum CPUID feature set for kernel These bits are checked |
really early to actually display a visible error message before the |
kernel dies. Make sure to assign features to the proper mask! |
Some requirements that are not in CPUID yet are also in the |
CONFIG_X86_MINIMUM_CPU_FAMILY which is checked too. |
The real information is in arch/x86/Kconfig.cpu, this just converts |
the CONFIGs into a bitmask */ |
#ifndef CONFIG_MATH_EMULATION |
# define NEED_FPU (1<<(X86_FEATURE_FPU & 31)) |
#else |
# define NEED_FPU 0 |
#endif |
#if defined(CONFIG_X86_PAE) || defined(CONFIG_X86_64) |
# define NEED_PAE (1<<(X86_FEATURE_PAE & 31)) |
#else |
# define NEED_PAE 0 |
#endif |
#ifdef CONFIG_X86_CMPXCHG64 |
# define NEED_CX8 (1<<(X86_FEATURE_CX8 & 31)) |
#else |
# define NEED_CX8 0 |
#endif |
#if defined(CONFIG_X86_CMOV) || defined(CONFIG_X86_64) |
# define NEED_CMOV (1<<(X86_FEATURE_CMOV & 31)) |
#else |
# define NEED_CMOV 0 |
#endif |
#ifdef CONFIG_X86_USE_3DNOW |
# define NEED_3DNOW (1<<(X86_FEATURE_3DNOW & 31)) |
#else |
# define NEED_3DNOW 0 |
#endif |
#if defined(CONFIG_X86_P6_NOP) || defined(CONFIG_X86_64) |
# define NEED_NOPL (1<<(X86_FEATURE_NOPL & 31)) |
#else |
# define NEED_NOPL 0 |
#endif |
#ifdef CONFIG_MATOM |
# define NEED_MOVBE (1<<(X86_FEATURE_MOVBE & 31)) |
#else |
# define NEED_MOVBE 0 |
#endif |
#ifdef CONFIG_X86_64 |
#ifdef CONFIG_PARAVIRT |
/* Paravirtualized systems may not have PSE or PGE available */ |
#define NEED_PSE 0 |
#define NEED_PGE 0 |
#else |
#define NEED_PSE (1<<(X86_FEATURE_PSE) & 31) |
#define NEED_PGE (1<<(X86_FEATURE_PGE) & 31) |
#endif |
#define NEED_MSR (1<<(X86_FEATURE_MSR & 31)) |
#define NEED_FXSR (1<<(X86_FEATURE_FXSR & 31)) |
#define NEED_XMM (1<<(X86_FEATURE_XMM & 31)) |
#define NEED_XMM2 (1<<(X86_FEATURE_XMM2 & 31)) |
#define NEED_LM (1<<(X86_FEATURE_LM & 31)) |
#else |
#define NEED_PSE 0 |
#define NEED_MSR 0 |
#define NEED_PGE 0 |
#define NEED_FXSR 0 |
#define NEED_XMM 0 |
#define NEED_XMM2 0 |
#define NEED_LM 0 |
#endif |
#define REQUIRED_MASK0 (NEED_FPU|NEED_PSE|NEED_MSR|NEED_PAE|\ |
NEED_CX8|NEED_PGE|NEED_FXSR|NEED_CMOV|\ |
NEED_XMM|NEED_XMM2) |
#define SSE_MASK (NEED_XMM|NEED_XMM2) |
#define REQUIRED_MASK1 (NEED_LM|NEED_3DNOW) |
#define REQUIRED_MASK2 0 |
#define REQUIRED_MASK3 (NEED_NOPL) |
#define REQUIRED_MASK4 (NEED_MOVBE) |
#define REQUIRED_MASK5 0 |
#define REQUIRED_MASK6 0 |
#define REQUIRED_MASK7 0 |
#define REQUIRED_MASK8 0 |
#define REQUIRED_MASK9 0 |
#endif /* _ASM_X86_REQUIRED_FEATURES_H */ |
/drivers/include/asm/rmwcc.h |
---|
0,0 → 1,41 |
#ifndef _ASM_X86_RMWcc |
#define _ASM_X86_RMWcc |
#ifdef CC_HAVE_ASM_GOTO |
#define __GEN_RMWcc(fullop, var, cc, ...) \ |
do { \ |
asm_volatile_goto (fullop "; j" cc " %l[cc_label]" \ |
: : "m" (var), ## __VA_ARGS__ \ |
: "memory" : cc_label); \ |
return 0; \ |
cc_label: \ |
return 1; \ |
} while (0) |
#define GEN_UNARY_RMWcc(op, var, arg0, cc) \ |
__GEN_RMWcc(op " " arg0, var, cc) |
#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \ |
__GEN_RMWcc(op " %1, " arg0, var, cc, vcon (val)) |
#else /* !CC_HAVE_ASM_GOTO */ |
#define __GEN_RMWcc(fullop, var, cc, ...) \ |
do { \ |
char c; \ |
asm volatile (fullop "; set" cc " %1" \ |
: "+m" (var), "=qm" (c) \ |
: __VA_ARGS__ : "memory"); \ |
return c != 0; \ |
} while (0) |
#define GEN_UNARY_RMWcc(op, var, arg0, cc) \ |
__GEN_RMWcc(op " " arg0, var, cc) |
#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \ |
__GEN_RMWcc(op " %2, " arg0, var, cc, vcon (val)) |
#endif /* CC_HAVE_ASM_GOTO */ |
#endif /* _ASM_X86_RMWcc */ |
/drivers/include/asm/scatterlist.h |
---|
0,0 → 1,41 |
#ifndef __ASM_GENERIC_SCATTERLIST_H |
#define __ASM_GENERIC_SCATTERLIST_H |
#include <linux/types.h> |
struct scatterlist { |
#ifdef CONFIG_DEBUG_SG |
unsigned long sg_magic; |
#endif |
unsigned long page_link; |
unsigned int offset; |
unsigned int length; |
dma_addr_t dma_address; |
#ifdef CONFIG_NEED_SG_DMA_LENGTH |
unsigned int dma_length; |
#endif |
}; |
/* |
* These macros should be used after a dma_map_sg call has been done |
* to get bus addresses of each of the SG entries and their lengths. |
* You should only work with the number of sg entries pci_map_sg |
* returns, or alternatively stop on the first sg_dma_len(sg) which |
* is 0. |
*/ |
#define sg_dma_address(sg) ((sg)->dma_address) |
#ifdef CONFIG_NEED_SG_DMA_LENGTH |
#define sg_dma_len(sg) ((sg)->dma_length) |
#else |
#define sg_dma_len(sg) ((sg)->length) |
#endif |
#define ARCH_HAS_SG_CHAIN |
int dma_map_sg(struct device *dev, struct scatterlist *sglist, |
int nelems, int dir); |
#define dma_unmap_sg(d, s, n, r) |
#endif /* __ASM_GENERIC_SCATTERLIST_H */ |
/drivers/include/asm/sigcontext.h |
---|
0,0 → 1,79 |
#ifndef _ASM_X86_SIGCONTEXT_H |
#define _ASM_X86_SIGCONTEXT_H |
#include <uapi/asm/sigcontext.h> |
#ifdef __i386__ |
struct sigcontext { |
unsigned short gs, __gsh; |
unsigned short fs, __fsh; |
unsigned short es, __esh; |
unsigned short ds, __dsh; |
unsigned long di; |
unsigned long si; |
unsigned long bp; |
unsigned long sp; |
unsigned long bx; |
unsigned long dx; |
unsigned long cx; |
unsigned long ax; |
unsigned long trapno; |
unsigned long err; |
unsigned long ip; |
unsigned short cs, __csh; |
unsigned long flags; |
unsigned long sp_at_signal; |
unsigned short ss, __ssh; |
/* |
* fpstate is really (struct _fpstate *) or (struct _xstate *) |
* depending on the FP_XSTATE_MAGIC1 encoded in the SW reserved |
* bytes of (struct _fpstate) and FP_XSTATE_MAGIC2 present at the end |
* of extended memory layout. See comments at the definition of |
* (struct _fpx_sw_bytes) |
*/ |
void __user *fpstate; /* zero when no FPU/extended context */ |
unsigned long oldmask; |
unsigned long cr2; |
}; |
#else /* __i386__ */ |
struct sigcontext { |
unsigned long r8; |
unsigned long r9; |
unsigned long r10; |
unsigned long r11; |
unsigned long r12; |
unsigned long r13; |
unsigned long r14; |
unsigned long r15; |
unsigned long di; |
unsigned long si; |
unsigned long bp; |
unsigned long bx; |
unsigned long dx; |
unsigned long ax; |
unsigned long cx; |
unsigned long sp; |
unsigned long ip; |
unsigned long flags; |
unsigned short cs; |
unsigned short gs; |
unsigned short fs; |
unsigned short __pad0; |
unsigned long err; |
unsigned long trapno; |
unsigned long oldmask; |
unsigned long cr2; |
/* |
* fpstate is really (struct _fpstate *) or (struct _xstate *) |
* depending on the FP_XSTATE_MAGIC1 encoded in the SW reserved |
* bytes of (struct _fpstate) and FP_XSTATE_MAGIC2 present at the end |
* of extended memory layout. See comments at the definition of |
* (struct _fpx_sw_bytes) |
*/ |
void __user *fpstate; /* zero when no FPU/extended context */ |
unsigned long reserved1[8]; |
}; |
#endif /* !__i386__ */ |
#endif /* _ASM_X86_SIGCONTEXT_H */ |
/drivers/include/asm/special_insns.h |
---|
0,0 → 1,207 |
#ifndef _ASM_X86_SPECIAL_INSNS_H |
#define _ASM_X86_SPECIAL_INSNS_H |
#ifdef __KERNEL__ |
static inline void native_clts(void) |
{ |
asm volatile("clts"); |
} |
/* |
* Volatile isn't enough to prevent the compiler from reordering the |
* read/write functions for the control registers and messing everything up. |
* A memory clobber would solve the problem, but would prevent reordering of |
* all loads stores around it, which can hurt performance. Solution is to |
* use a variable and mimic reads and writes to it to enforce serialization |
*/ |
extern unsigned long __force_order; |
static inline unsigned long native_read_cr0(void) |
{ |
unsigned long val; |
asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order)); |
return val; |
} |
static inline void native_write_cr0(unsigned long val) |
{ |
asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order)); |
} |
static inline unsigned long native_read_cr2(void) |
{ |
unsigned long val; |
asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order)); |
return val; |
} |
static inline void native_write_cr2(unsigned long val) |
{ |
asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order)); |
} |
static inline unsigned long native_read_cr3(void) |
{ |
unsigned long val; |
asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order)); |
return val; |
} |
static inline void native_write_cr3(unsigned long val) |
{ |
asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order)); |
} |
static inline unsigned long native_read_cr4(void) |
{ |
unsigned long val; |
asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order)); |
return val; |
} |
static inline unsigned long native_read_cr4_safe(void) |
{ |
unsigned long val; |
/* This could fault if %cr4 does not exist. In x86_64, a cr4 always |
* exists, so it will never fail. */ |
#ifdef CONFIG_X86_32 |
asm volatile("1: mov %%cr4, %0\n" |
"2:\n" |
_ASM_EXTABLE(1b, 2b) |
: "=r" (val), "=m" (__force_order) : "0" (0)); |
#else |
val = native_read_cr4(); |
#endif |
return val; |
} |
static inline void native_write_cr4(unsigned long val) |
{ |
asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order)); |
} |
#ifdef CONFIG_X86_64 |
static inline unsigned long native_read_cr8(void) |
{ |
unsigned long cr8; |
asm volatile("movq %%cr8,%0" : "=r" (cr8)); |
return cr8; |
} |
static inline void native_write_cr8(unsigned long val) |
{ |
asm volatile("movq %0,%%cr8" :: "r" (val) : "memory"); |
} |
#endif |
static inline void native_wbinvd(void) |
{ |
asm volatile("wbinvd": : :"memory"); |
} |
extern asmlinkage void native_load_gs_index(unsigned); |
#ifdef CONFIG_PARAVIRT |
#include <asm/paravirt.h> |
#else |
static inline unsigned long read_cr0(void) |
{ |
return native_read_cr0(); |
} |
static inline void write_cr0(unsigned long x) |
{ |
native_write_cr0(x); |
} |
static inline unsigned long read_cr2(void) |
{ |
return native_read_cr2(); |
} |
static inline void write_cr2(unsigned long x) |
{ |
native_write_cr2(x); |
} |
static inline unsigned long read_cr3(void) |
{ |
return native_read_cr3(); |
} |
static inline void write_cr3(unsigned long x) |
{ |
native_write_cr3(x); |
} |
static inline unsigned long read_cr4(void) |
{ |
return native_read_cr4(); |
} |
static inline unsigned long read_cr4_safe(void) |
{ |
return native_read_cr4_safe(); |
} |
static inline void write_cr4(unsigned long x) |
{ |
native_write_cr4(x); |
} |
static inline void wbinvd(void) |
{ |
native_wbinvd(); |
} |
#ifdef CONFIG_X86_64 |
static inline unsigned long read_cr8(void) |
{ |
return native_read_cr8(); |
} |
static inline void write_cr8(unsigned long x) |
{ |
native_write_cr8(x); |
} |
static inline void load_gs_index(unsigned selector) |
{ |
native_load_gs_index(selector); |
} |
#endif |
/* Clear the 'TS' bit */ |
static inline void clts(void) |
{ |
native_clts(); |
} |
#endif/* CONFIG_PARAVIRT */ |
#define stts() write_cr0(read_cr0() | X86_CR0_TS) |
static inline void clflush(volatile void *__p) |
{ |
asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p)); |
} |
static inline void clflushopt(volatile void *__p) |
{ |
alternative_io(".byte " __stringify(NOP_DS_PREFIX) "; clflush %P0", |
".byte 0x66; clflush %P0", |
X86_FEATURE_CLFLUSHOPT, |
"+m" (*(volatile char __force *)__p)); |
} |
#define nop() asm volatile ("nop") |
#endif /* __KERNEL__ */ |
#endif /* _ASM_X86_SPECIAL_INSNS_H */ |
/drivers/include/asm/spinlock_types.h |
---|
0,0 → 1,20 |
#ifndef _ASM_X86_SPINLOCK_TYPES_H |
#define _ASM_X86_SPINLOCK_TYPES_H |
#ifndef __LINUX_SPINLOCK_TYPES_H |
# error "please don't include this file directly" |
#endif |
typedef struct raw_spinlock { |
unsigned int slock; |
} raw_spinlock_t; |
#define __RAW_SPIN_LOCK_UNLOCKED { 0 } |
typedef struct { |
unsigned int lock; |
} raw_rwlock_t; |
#define __RAW_RW_LOCK_UNLOCKED { RW_LOCK_BIAS } |
#endif /* _ASM_X86_SPINLOCK_TYPES_H */ |
/drivers/include/asm/string.h |
---|
0,0 → 1,5 |
#ifdef CONFIG_X86_32 |
# include <asm/string_32.h> |
#else |
# include <asm/string_64.h> |
#endif |
/drivers/include/asm/string_32.h |
---|
0,0 → 1,342 |
#ifndef _ASM_X86_STRING_32_H |
#define _ASM_X86_STRING_32_H |
#ifdef __KERNEL__ |
/* Let gcc decide whether to inline or use the out of line functions */ |
#define __HAVE_ARCH_STRCPY |
extern char *strcpy(char *dest, const char *src); |
#define __HAVE_ARCH_STRNCPY |
extern char *strncpy(char *dest, const char *src, size_t count); |
#define __HAVE_ARCH_STRCAT |
extern char *strcat(char *dest, const char *src); |
#define __HAVE_ARCH_STRNCAT |
extern char *strncat(char *dest, const char *src, size_t count); |
#define __HAVE_ARCH_STRCMP |
extern int strcmp(const char *cs, const char *ct); |
#define __HAVE_ARCH_STRNCMP |
extern int strncmp(const char *cs, const char *ct, size_t count); |
#define __HAVE_ARCH_STRCHR |
extern char *strchr(const char *s, int c); |
#define __HAVE_ARCH_STRLEN |
extern size_t strlen(const char *s); |
static __always_inline void *__memcpy(void *to, const void *from, size_t n) |
{ |
int d0, d1, d2; |
asm volatile("rep ; movsl\n\t" |
"movl %4,%%ecx\n\t" |
"andl $3,%%ecx\n\t" |
"jz 1f\n\t" |
"rep ; movsb\n\t" |
"1:" |
: "=&c" (d0), "=&D" (d1), "=&S" (d2) |
: "0" (n / 4), "g" (n), "1" ((long)to), "2" ((long)from) |
: "memory"); |
return to; |
} |
/* |
* This looks ugly, but the compiler can optimize it totally, |
* as the count is constant. |
*/ |
static __always_inline void *__constant_memcpy(void *to, const void *from, |
size_t n) |
{ |
long esi, edi; |
if (!n) |
return to; |
switch (n) { |
case 1: |
*(char *)to = *(char *)from; |
return to; |
case 2: |
*(short *)to = *(short *)from; |
return to; |
case 4: |
*(int *)to = *(int *)from; |
return to; |
case 3: |
*(short *)to = *(short *)from; |
*((char *)to + 2) = *((char *)from + 2); |
return to; |
case 5: |
*(int *)to = *(int *)from; |
*((char *)to + 4) = *((char *)from + 4); |
return to; |
case 6: |
*(int *)to = *(int *)from; |
*((short *)to + 2) = *((short *)from + 2); |
return to; |
case 8: |
*(int *)to = *(int *)from; |
*((int *)to + 1) = *((int *)from + 1); |
return to; |
} |
esi = (long)from; |
edi = (long)to; |
if (n >= 5 * 4) { |
/* large block: use rep prefix */ |
int ecx; |
asm volatile("rep ; movsl" |
: "=&c" (ecx), "=&D" (edi), "=&S" (esi) |
: "0" (n / 4), "1" (edi), "2" (esi) |
: "memory" |
); |
} else { |
/* small block: don't clobber ecx + smaller code */ |
if (n >= 4 * 4) |
asm volatile("movsl" |
: "=&D"(edi), "=&S"(esi) |
: "0"(edi), "1"(esi) |
: "memory"); |
if (n >= 3 * 4) |
asm volatile("movsl" |
: "=&D"(edi), "=&S"(esi) |
: "0"(edi), "1"(esi) |
: "memory"); |
if (n >= 2 * 4) |
asm volatile("movsl" |
: "=&D"(edi), "=&S"(esi) |
: "0"(edi), "1"(esi) |
: "memory"); |
if (n >= 1 * 4) |
asm volatile("movsl" |
: "=&D"(edi), "=&S"(esi) |
: "0"(edi), "1"(esi) |
: "memory"); |
} |
switch (n % 4) { |
/* tail */ |
case 0: |
return to; |
case 1: |
asm volatile("movsb" |
: "=&D"(edi), "=&S"(esi) |
: "0"(edi), "1"(esi) |
: "memory"); |
return to; |
case 2: |
asm volatile("movsw" |
: "=&D"(edi), "=&S"(esi) |
: "0"(edi), "1"(esi) |
: "memory"); |
return to; |
default: |
asm volatile("movsw\n\tmovsb" |
: "=&D"(edi), "=&S"(esi) |
: "0"(edi), "1"(esi) |
: "memory"); |
return to; |
} |
} |
#define __HAVE_ARCH_MEMCPY |
#ifdef CONFIG_X86_USE_3DNOW |
#include <asm/mmx.h> |
/* |
* This CPU favours 3DNow strongly (eg AMD Athlon) |
*/ |
static inline void *__constant_memcpy3d(void *to, const void *from, size_t len) |
{ |
if (len < 512) |
return __constant_memcpy(to, from, len); |
return _mmx_memcpy(to, from, len); |
} |
static inline void *__memcpy3d(void *to, const void *from, size_t len) |
{ |
if (len < 512) |
return __memcpy(to, from, len); |
return _mmx_memcpy(to, from, len); |
} |
#define memcpy(t, f, n) \ |
(__builtin_constant_p((n)) \ |
? __constant_memcpy3d((t), (f), (n)) \ |
: __memcpy3d((t), (f), (n))) |
#else |
/* |
* No 3D Now! |
*/ |
#ifndef CONFIG_KMEMCHECK |
#if (__GNUC__ >= 4) |
#define memcpy(t, f, n) __builtin_memcpy(t, f, n) |
#else |
#define memcpy(t, f, n) \ |
(__builtin_constant_p((n)) \ |
? __constant_memcpy((t), (f), (n)) \ |
: __memcpy((t), (f), (n))) |
#endif |
#else |
/* |
* kmemcheck becomes very happy if we use the REP instructions unconditionally, |
* because it means that we know both memory operands in advance. |
*/ |
#define memcpy(t, f, n) __memcpy((t), (f), (n)) |
#endif |
#endif |
#define __HAVE_ARCH_MEMMOVE |
void *memmove(void *dest, const void *src, size_t n); |
#define memcmp __builtin_memcmp |
#define __HAVE_ARCH_MEMCHR |
extern void *memchr(const void *cs, int c, size_t count); |
static inline void *__memset_generic(void *s, char c, size_t count) |
{ |
int d0, d1; |
asm volatile("rep\n\t" |
"stosb" |
: "=&c" (d0), "=&D" (d1) |
: "a" (c), "1" (s), "0" (count) |
: "memory"); |
return s; |
} |
/* we might want to write optimized versions of these later */ |
#define __constant_count_memset(s, c, count) __memset_generic((s), (c), (count)) |
/* |
* memset(x, 0, y) is a reasonably common thing to do, so we want to fill |
* things 32 bits at a time even when we don't know the size of the |
* area at compile-time.. |
*/ |
static __always_inline |
void *__constant_c_memset(void *s, unsigned long c, size_t count) |
{ |
int d0, d1; |
asm volatile("rep ; stosl\n\t" |
"testb $2,%b3\n\t" |
"je 1f\n\t" |
"stosw\n" |
"1:\ttestb $1,%b3\n\t" |
"je 2f\n\t" |
"stosb\n" |
"2:" |
: "=&c" (d0), "=&D" (d1) |
: "a" (c), "q" (count), "0" (count/4), "1" ((long)s) |
: "memory"); |
return s; |
} |
/* Added by Gertjan van Wingerde to make minix and sysv module work */ |
#define __HAVE_ARCH_STRNLEN |
extern size_t strnlen(const char *s, size_t count); |
/* end of additional stuff */ |
#define __HAVE_ARCH_STRSTR |
extern char *strstr(const char *cs, const char *ct); |
/* |
* This looks horribly ugly, but the compiler can optimize it totally, |
* as we by now know that both pattern and count is constant.. |
*/ |
static __always_inline |
void *__constant_c_and_count_memset(void *s, unsigned long pattern, |
size_t count) |
{ |
switch (count) { |
case 0: |
return s; |
case 1: |
*(unsigned char *)s = pattern & 0xff; |
return s; |
case 2: |
*(unsigned short *)s = pattern & 0xffff; |
return s; |
case 3: |
*(unsigned short *)s = pattern & 0xffff; |
*((unsigned char *)s + 2) = pattern & 0xff; |
return s; |
case 4: |
*(unsigned long *)s = pattern; |
return s; |
} |
#define COMMON(x) \ |
asm volatile("rep ; stosl" \ |
x \ |
: "=&c" (d0), "=&D" (d1) \ |
: "a" (eax), "0" (count/4), "1" ((long)s) \ |
: "memory") |
{ |
int d0, d1; |
#if __GNUC__ == 4 && __GNUC_MINOR__ == 0 |
/* Workaround for broken gcc 4.0 */ |
register unsigned long eax asm("%eax") = pattern; |
#else |
unsigned long eax = pattern; |
#endif |
switch (count % 4) { |
case 0: |
COMMON(""); |
return s; |
case 1: |
COMMON("\n\tstosb"); |
return s; |
case 2: |
COMMON("\n\tstosw"); |
return s; |
default: |
COMMON("\n\tstosw\n\tstosb"); |
return s; |
} |
} |
#undef COMMON |
} |
#define __constant_c_x_memset(s, c, count) \ |
(__builtin_constant_p(count) \ |
? __constant_c_and_count_memset((s), (c), (count)) \ |
: __constant_c_memset((s), (c), (count))) |
#define __memset(s, c, count) \ |
(__builtin_constant_p(count) \ |
? __constant_count_memset((s), (c), (count)) \ |
: __memset_generic((s), (c), (count))) |
#define __HAVE_ARCH_MEMSET |
#if (__GNUC__ >= 4) |
#define memset(s, c, count) __builtin_memset(s, c, count) |
#else |
#define memset(s, c, count) \ |
(__builtin_constant_p(c) \ |
? __constant_c_x_memset((s), (0x01010101UL * (unsigned char)(c)), \ |
(count)) \ |
: __memset((s), (c), (count))) |
#endif |
/* |
* find the first occurrence of byte 'c', or 1 past the area if none |
*/ |
#define __HAVE_ARCH_MEMSCAN |
extern void *memscan(void *addr, int c, size_t size); |
#endif /* __KERNEL__ */ |
#endif /* _ASM_X86_STRING_32_H */ |
/drivers/include/asm/swab.h |
---|
0,0 → 1,61 |
#ifndef _ASM_X86_SWAB_H |
#define _ASM_X86_SWAB_H |
#include <linux/types.h> |
#include <linux/compiler.h> |
static inline __attribute_const__ __u32 __arch_swab32(__u32 val) |
{ |
#ifdef __i386__ |
# ifdef CONFIG_X86_BSWAP |
asm("bswap %0" : "=r" (val) : "0" (val)); |
# else |
asm("xchgb %b0,%h0\n\t" /* swap lower bytes */ |
"rorl $16,%0\n\t" /* swap words */ |
"xchgb %b0,%h0" /* swap higher bytes */ |
: "=q" (val) |
: "0" (val)); |
# endif |
#else /* __i386__ */ |
asm("bswapl %0" |
: "=r" (val) |
: "0" (val)); |
#endif |
return val; |
} |
#define __arch_swab32 __arch_swab32 |
static inline __attribute_const__ __u64 __arch_swab64(__u64 val) |
{ |
#ifdef __i386__ |
union { |
struct { |
__u32 a; |
__u32 b; |
} s; |
__u64 u; |
} v; |
v.u = val; |
# ifdef CONFIG_X86_BSWAP |
asm("bswapl %0 ; bswapl %1 ; xchgl %0,%1" |
: "=r" (v.s.a), "=r" (v.s.b) |
: "0" (v.s.a), "1" (v.s.b)); |
# else |
v.s.a = __arch_swab32(v.s.a); |
v.s.b = __arch_swab32(v.s.b); |
asm("xchgl %0,%1" |
: "=r" (v.s.a), "=r" (v.s.b) |
: "0" (v.s.a), "1" (v.s.b)); |
# endif |
return v.u; |
#else /* __i386__ */ |
asm("bswapq %0" |
: "=r" (val) |
: "0" (val)); |
return val; |
#endif |
} |
#define __arch_swab64 __arch_swab64 |
#endif /* _ASM_X86_SWAB_H */ |
/drivers/include/asm/types.h |
---|
0,0 → 1,16 |
#ifndef _ASM_X86_TYPES_H |
#define _ASM_X86_TYPES_H |
#define dma_addr_t dma_addr_t |
#include <asm-generic/types.h> |
#ifdef __KERNEL__ |
#ifndef __ASSEMBLY__ |
typedef u64 dma64_addr_t; |
#endif /* __ASSEMBLY__ */ |
#endif /* __KERNEL__ */ |
#endif /* _ASM_X86_TYPES_H */ |
/drivers/include/asm/unaligned.h |
---|
0,0 → 1,14 |
#ifndef _ASM_X86_UNALIGNED_H |
#define _ASM_X86_UNALIGNED_H |
/* |
* The x86 can do unaligned accesses itself. |
*/ |
#include <linux/unaligned/access_ok.h> |
#include <linux/unaligned/generic.h> |
#define get_unaligned __get_unaligned_le |
#define put_unaligned __put_unaligned_le |
#endif /* _ASM_X86_UNALIGNED_H */ |
/drivers/include/asm/x86_init.h |
---|
0,0 → 1,216 |
#ifndef _ASM_X86_PLATFORM_H |
#define _ASM_X86_PLATFORM_H |
#include <asm/pgtable_types.h> |
//#include <asm/bootparam.h> |
struct mpc_bus; |
struct mpc_cpu; |
struct mpc_table; |
struct cpuinfo_x86; |
/** |
* struct x86_init_mpparse - platform specific mpparse ops |
* @mpc_record: platform specific mpc record accounting |
* @setup_ioapic_ids: platform specific ioapic id override |
* @mpc_apic_id: platform specific mpc apic id assignment |
* @smp_read_mpc_oem: platform specific oem mpc table setup |
* @mpc_oem_pci_bus: platform specific pci bus setup (default NULL) |
* @mpc_oem_bus_info: platform specific mpc bus info |
* @find_smp_config: find the smp configuration |
* @get_smp_config: get the smp configuration |
*/ |
struct x86_init_mpparse { |
void (*mpc_record)(unsigned int mode); |
void (*setup_ioapic_ids)(void); |
int (*mpc_apic_id)(struct mpc_cpu *m); |
void (*smp_read_mpc_oem)(struct mpc_table *mpc); |
void (*mpc_oem_pci_bus)(struct mpc_bus *m); |
void (*mpc_oem_bus_info)(struct mpc_bus *m, char *name); |
void (*find_smp_config)(void); |
void (*get_smp_config)(unsigned int early); |
}; |
/** |
* struct x86_init_resources - platform specific resource related ops |
* @probe_roms: probe BIOS roms |
* @reserve_resources: reserve the standard resources for the |
* platform |
* @memory_setup: platform specific memory setup |
* |
*/ |
struct x86_init_resources { |
void (*probe_roms)(void); |
void (*reserve_resources)(void); |
char *(*memory_setup)(void); |
}; |
/** |
* struct x86_init_irqs - platform specific interrupt setup |
* @pre_vector_init: init code to run before interrupt vectors |
* are set up. |
* @intr_init: interrupt init code |
* @trap_init: platform specific trap setup |
*/ |
struct x86_init_irqs { |
void (*pre_vector_init)(void); |
void (*intr_init)(void); |
void (*trap_init)(void); |
}; |
/** |
* struct x86_init_oem - oem platform specific customizing functions |
* @arch_setup: platform specific architecure setup |
* @banner: print a platform specific banner |
*/ |
struct x86_init_oem { |
void (*arch_setup)(void); |
void (*banner)(void); |
}; |
/** |
* struct x86_init_paging - platform specific paging functions |
* @pagetable_init: platform specific paging initialization call to setup |
* the kernel pagetables and prepare accessors functions. |
* Callback must call paging_init(). Called once after the |
* direct mapping for phys memory is available. |
*/ |
struct x86_init_paging { |
void (*pagetable_init)(void); |
}; |
/** |
* struct x86_init_timers - platform specific timer setup |
* @setup_perpcu_clockev: set up the per cpu clock event device for the |
* boot cpu |
* @tsc_pre_init: platform function called before TSC init |
* @timer_init: initialize the platform timer (default PIT/HPET) |
* @wallclock_init: init the wallclock device |
*/ |
struct x86_init_timers { |
void (*setup_percpu_clockev)(void); |
void (*tsc_pre_init)(void); |
void (*timer_init)(void); |
void (*wallclock_init)(void); |
}; |
/** |
* struct x86_init_iommu - platform specific iommu setup |
* @iommu_init: platform specific iommu setup |
*/ |
struct x86_init_iommu { |
int (*iommu_init)(void); |
}; |
/** |
* struct x86_init_pci - platform specific pci init functions |
* @arch_init: platform specific pci arch init call |
* @init: platform specific pci subsystem init |
* @init_irq: platform specific pci irq init |
* @fixup_irqs: platform specific pci irq fixup |
*/ |
struct x86_init_pci { |
int (*arch_init)(void); |
int (*init)(void); |
void (*init_irq)(void); |
void (*fixup_irqs)(void); |
}; |
/** |
* struct x86_init_ops - functions for platform specific setup |
* |
*/ |
struct x86_init_ops { |
struct x86_init_resources resources; |
struct x86_init_mpparse mpparse; |
struct x86_init_irqs irqs; |
struct x86_init_oem oem; |
struct x86_init_paging paging; |
struct x86_init_timers timers; |
struct x86_init_iommu iommu; |
struct x86_init_pci pci; |
}; |
/** |
* struct x86_cpuinit_ops - platform specific cpu hotplug setups |
* @setup_percpu_clockev: set up the per cpu clock event device |
* @early_percpu_clock_init: early init of the per cpu clock event device |
*/ |
struct x86_cpuinit_ops { |
void (*setup_percpu_clockev)(void); |
void (*early_percpu_clock_init)(void); |
void (*fixup_cpu_id)(struct cpuinfo_x86 *c, int node); |
}; |
struct timespec; |
/** |
* struct x86_platform_ops - platform specific runtime functions |
* @calibrate_tsc: calibrate TSC |
* @get_wallclock: get time from HW clock like RTC etc. |
* @set_wallclock: set time back to HW clock |
* @is_untracked_pat_range exclude from PAT logic |
* @nmi_init enable NMI on cpus |
* @i8042_detect pre-detect if i8042 controller exists |
* @save_sched_clock_state: save state for sched_clock() on suspend |
* @restore_sched_clock_state: restore state for sched_clock() on resume |
* @apic_post_init: adjust apic if neeeded |
*/ |
struct x86_platform_ops { |
unsigned long (*calibrate_tsc)(void); |
void (*get_wallclock)(struct timespec *ts); |
int (*set_wallclock)(const struct timespec *ts); |
void (*iommu_shutdown)(void); |
bool (*is_untracked_pat_range)(u64 start, u64 end); |
void (*nmi_init)(void); |
unsigned char (*get_nmi_reason)(void); |
int (*i8042_detect)(void); |
void (*save_sched_clock_state)(void); |
void (*restore_sched_clock_state)(void); |
void (*apic_post_init)(void); |
}; |
struct pci_dev; |
struct msi_msg; |
struct x86_msi_ops { |
int (*setup_msi_irqs)(struct pci_dev *dev, int nvec, int type); |
void (*compose_msi_msg)(struct pci_dev *dev, unsigned int irq, |
unsigned int dest, struct msi_msg *msg, |
u8 hpet_id); |
void (*teardown_msi_irq)(unsigned int irq); |
void (*teardown_msi_irqs)(struct pci_dev *dev); |
void (*restore_msi_irqs)(struct pci_dev *dev); |
int (*setup_hpet_msi)(unsigned int irq, unsigned int id); |
}; |
struct IO_APIC_route_entry; |
struct io_apic_irq_attr; |
struct irq_data; |
struct cpumask; |
struct x86_io_apic_ops { |
void (*init) (void); |
unsigned int (*read) (unsigned int apic, unsigned int reg); |
void (*write) (unsigned int apic, unsigned int reg, unsigned int value); |
void (*modify) (unsigned int apic, unsigned int reg, unsigned int value); |
void (*disable)(void); |
void (*print_entries)(unsigned int apic, unsigned int nr_entries); |
int (*set_affinity)(struct irq_data *data, |
const struct cpumask *mask, |
bool force); |
int (*setup_entry)(int irq, struct IO_APIC_route_entry *entry, |
unsigned int destination, int vector, |
struct io_apic_irq_attr *attr); |
void (*eoi_ioapic_pin)(int apic, int pin, int vector); |
}; |
extern struct x86_init_ops x86_init; |
extern struct x86_cpuinit_ops x86_cpuinit; |
extern struct x86_platform_ops x86_platform; |
extern struct x86_msi_ops x86_msi; |
extern struct x86_io_apic_ops x86_io_apic_ops; |
extern void x86_init_noop(void); |
extern void x86_init_uint_noop(unsigned int unused); |
#endif |