/drivers/include/asm/bitsperlong.h |
---|
File deleted |
/drivers/include/asm/types.h |
---|
File deleted |
/drivers/include/asm/atomic_32.h |
---|
File deleted |
/drivers/include/asm/swab.h |
---|
File deleted |
/drivers/include/asm/scatterlist.h |
---|
File deleted |
/drivers/include/asm/posix_types_32.h |
---|
File deleted |
/drivers/include/asm/byteorder.h |
---|
File deleted |
/drivers/include/asm/alternative.h |
---|
152,12 → 152,6 |
".popsection" |
/* |
* This must be included *after* the definition of ALTERNATIVE due to |
* <asm/arch_hweight.h> |
*/ |
#include <asm/cpufeature.h> |
/* |
* Alternative instructions for different CPU types or capabilities. |
* |
* This allows to use optimized instructions even on generic binary |
/drivers/include/asm/arch_hweight.h |
---|
1,6 → 1,8 |
#ifndef _ASM_X86_HWEIGHT_H |
#define _ASM_X86_HWEIGHT_H |
#include <asm/cpufeatures.h> |
#ifdef CONFIG_64BIT |
/* popcnt %edi, %eax -- redundant REX prefix for alignment */ |
#define POPCNT32 ".byte 0xf3,0x40,0x0f,0xb8,0xc7" |
/drivers/include/asm/asm.h |
---|
44,19 → 44,22 |
/* Exception table entry */ |
#ifdef __ASSEMBLY__ |
# define _ASM_EXTABLE(from,to) \ |
# define _ASM_EXTABLE_HANDLE(from, to, handler) \ |
.pushsection "__ex_table","a" ; \ |
.balign 8 ; \ |
.balign 4 ; \ |
.long (from) - . ; \ |
.long (to) - . ; \ |
.long (handler) - . ; \ |
.popsection |
# define _ASM_EXTABLE(from, to) \ |
_ASM_EXTABLE_HANDLE(from, to, ex_handler_default) |
# define _ASM_EXTABLE_FAULT(from, to) \ |
_ASM_EXTABLE_HANDLE(from, to, ex_handler_fault) |
# define _ASM_EXTABLE_EX(from,to) \ |
.pushsection "__ex_table","a" ; \ |
.balign 8 ; \ |
.long (from) - . ; \ |
.long (to) - . + 0x7ffffff0 ; \ |
.popsection |
_ASM_EXTABLE_HANDLE(from, to, ex_handler_ext) |
# define _ASM_NOKPROBE(entry) \ |
.pushsection "_kprobe_blacklist","aw" ; \ |
89,19 → 92,24 |
.endm |
#else |
# define _ASM_EXTABLE(from,to) \ |
# define _EXPAND_EXTABLE_HANDLE(x) #x |
# define _ASM_EXTABLE_HANDLE(from, to, handler) \ |
" .pushsection \"__ex_table\",\"a\"\n" \ |
" .balign 8\n" \ |
" .balign 4\n" \ |
" .long (" #from ") - .\n" \ |
" .long (" #to ") - .\n" \ |
" .long (" _EXPAND_EXTABLE_HANDLE(handler) ") - .\n" \ |
" .popsection\n" |
# define _ASM_EXTABLE(from, to) \ |
_ASM_EXTABLE_HANDLE(from, to, ex_handler_default) |
# define _ASM_EXTABLE_FAULT(from, to) \ |
_ASM_EXTABLE_HANDLE(from, to, ex_handler_fault) |
# define _ASM_EXTABLE_EX(from,to) \ |
" .pushsection \"__ex_table\",\"a\"\n" \ |
" .balign 8\n" \ |
" .long (" #from ") - .\n" \ |
" .long (" #to ") - . + 0x7ffffff0\n" \ |
" .popsection\n" |
_ASM_EXTABLE_HANDLE(from, to, ex_handler_ext) |
/* For C file, we already have NOKPROBE_SYMBOL macro */ |
#endif |
/drivers/include/asm/barrier.h |
---|
6,7 → 6,7 |
/* |
* Force strict CPU ordering. |
* And yes, this is required on UP too when we're talking |
* And yes, this might be required on UP too when we're talking |
* to devices. |
*/ |
31,21 → 31,11 |
#endif |
#define dma_wmb() barrier() |
#ifdef CONFIG_SMP |
#define smp_mb() mb() |
#define smp_rmb() dma_rmb() |
#define smp_wmb() barrier() |
#define smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0) |
#else /* !SMP */ |
#define smp_mb() barrier() |
#define smp_rmb() barrier() |
#define smp_wmb() barrier() |
#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0) |
#endif /* SMP */ |
#define __smp_mb() mb() |
#define __smp_rmb() dma_rmb() |
#define __smp_wmb() barrier() |
#define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0) |
#define read_barrier_depends() do { } while (0) |
#define smp_read_barrier_depends() do { } while (0) |
#if defined(CONFIG_X86_PPRO_FENCE) |
/* |
/drivers/include/asm/bitops.h |
---|
91,7 → 91,7 |
* If it's called on the same region of memory simultaneously, the effect |
* may be that only one operation succeeds. |
*/ |
static inline void __set_bit(long nr, volatile unsigned long *addr) |
static __always_inline void __set_bit(long nr, volatile unsigned long *addr) |
{ |
asm volatile("bts %1,%0" : ADDR : "Ir" (nr) : "memory"); |
} |
128,13 → 128,13 |
* clear_bit() is atomic and implies release semantics before the memory |
* operation. It can be used for an unlock. |
*/ |
static inline void clear_bit_unlock(long nr, volatile unsigned long *addr) |
static __always_inline void clear_bit_unlock(long nr, volatile unsigned long *addr) |
{ |
barrier(); |
clear_bit(nr, addr); |
} |
static inline void __clear_bit(long nr, volatile unsigned long *addr) |
static __always_inline void __clear_bit(long nr, volatile unsigned long *addr) |
{ |
asm volatile("btr %1,%0" : ADDR : "Ir" (nr)); |
} |
151,7 → 151,7 |
* No memory barrier is required here, because x86 cannot reorder stores past |
* older loads. Same principle as spin_unlock. |
*/ |
static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr) |
static __always_inline void __clear_bit_unlock(long nr, volatile unsigned long *addr) |
{ |
barrier(); |
__clear_bit(nr, addr); |
166,7 → 166,7 |
* If it's called on the same region of memory simultaneously, the effect |
* may be that only one operation succeeds. |
*/ |
static inline void __change_bit(long nr, volatile unsigned long *addr) |
static __always_inline void __change_bit(long nr, volatile unsigned long *addr) |
{ |
asm volatile("btc %1,%0" : ADDR : "Ir" (nr)); |
} |
180,7 → 180,7 |
* Note that @nr may be almost arbitrarily large; this function is not |
* restricted to acting on a single-word quantity. |
*/ |
static inline void change_bit(long nr, volatile unsigned long *addr) |
static __always_inline void change_bit(long nr, volatile unsigned long *addr) |
{ |
if (IS_IMMEDIATE(nr)) { |
asm volatile(LOCK_PREFIX "xorb %1,%0" |
201,7 → 201,7 |
* This operation is atomic and cannot be reordered. |
* It also implies a memory barrier. |
*/ |
static inline int test_and_set_bit(long nr, volatile unsigned long *addr) |
static __always_inline int test_and_set_bit(long nr, volatile unsigned long *addr) |
{ |
GEN_BINARY_RMWcc(LOCK_PREFIX "bts", *addr, "Ir", nr, "%0", "c"); |
} |
228,7 → 228,7 |
* If two examples of this operation race, one can appear to succeed |
* but actually fail. You must protect multiple accesses with a lock. |
*/ |
static inline int __test_and_set_bit(long nr, volatile unsigned long *addr) |
static __always_inline int __test_and_set_bit(long nr, volatile unsigned long *addr) |
{ |
int oldbit; |
247,7 → 247,7 |
* This operation is atomic and cannot be reordered. |
* It also implies a memory barrier. |
*/ |
static inline int test_and_clear_bit(long nr, volatile unsigned long *addr) |
static __always_inline int test_and_clear_bit(long nr, volatile unsigned long *addr) |
{ |
GEN_BINARY_RMWcc(LOCK_PREFIX "btr", *addr, "Ir", nr, "%0", "c"); |
} |
268,7 → 268,7 |
* accessed from a hypervisor on the same CPU if running in a VM: don't change |
* this without also updating arch/x86/kernel/kvm.c |
*/ |
static inline int __test_and_clear_bit(long nr, volatile unsigned long *addr) |
static __always_inline int __test_and_clear_bit(long nr, volatile unsigned long *addr) |
{ |
int oldbit; |
280,7 → 280,7 |
} |
/* WARNING: non atomic and it can be reordered! */ |
static inline int __test_and_change_bit(long nr, volatile unsigned long *addr) |
static __always_inline int __test_and_change_bit(long nr, volatile unsigned long *addr) |
{ |
int oldbit; |
300,7 → 300,7 |
* This operation is atomic and cannot be reordered. |
* It also implies a memory barrier. |
*/ |
static inline int test_and_change_bit(long nr, volatile unsigned long *addr) |
static __always_inline int test_and_change_bit(long nr, volatile unsigned long *addr) |
{ |
GEN_BINARY_RMWcc(LOCK_PREFIX "btc", *addr, "Ir", nr, "%0", "c"); |
} |
311,7 → 311,7 |
(addr[nr >> _BITOPS_LONG_SHIFT])) != 0; |
} |
static inline int variable_test_bit(long nr, volatile const unsigned long *addr) |
static __always_inline int variable_test_bit(long nr, volatile const unsigned long *addr) |
{ |
int oldbit; |
343,7 → 343,7 |
* |
* Undefined if no bit exists, so code should check against 0 first. |
*/ |
static inline unsigned long __ffs(unsigned long word) |
static __always_inline unsigned long __ffs(unsigned long word) |
{ |
asm("rep; bsf %1,%0" |
: "=r" (word) |
357,7 → 357,7 |
* |
* Undefined if no zero exists, so code should check against ~0UL first. |
*/ |
static inline unsigned long ffz(unsigned long word) |
static __always_inline unsigned long ffz(unsigned long word) |
{ |
asm("rep; bsf %1,%0" |
: "=r" (word) |
371,7 → 371,7 |
* |
* Undefined if no set bit exists, so code should check against 0 first. |
*/ |
static inline unsigned long __fls(unsigned long word) |
static __always_inline unsigned long __fls(unsigned long word) |
{ |
asm("bsr %1,%0" |
: "=r" (word) |
393,7 → 393,7 |
* set bit if value is nonzero. The first (least significant) bit |
* is at position 1. |
*/ |
static inline int ffs(int x) |
static __always_inline int ffs(int x) |
{ |
int r; |
434,7 → 434,7 |
* set bit if value is nonzero. The last (most significant) bit is |
* at position 32. |
*/ |
static inline int fls(int x) |
static __always_inline int fls(int x) |
{ |
int r; |
/drivers/include/asm/cacheflush.h |
---|
4,6 → 4,7 |
/* Caches aren't brain-dead on the intel. */ |
#include <asm-generic/cacheflush.h> |
#include <asm/special_insns.h> |
#include <asm/uaccess.h> |
/* |
* The set_memory_* API can be used to change various attributes of a virtual |
113,16 → 114,10 |
#define mmio_flush_range(addr, size) clflush_cache_range(addr, size) |
#ifdef CONFIG_DEBUG_RODATA |
void mark_rodata_ro(void); |
extern const int rodata_test_data; |
extern int kernel_set_to_readonly; |
void set_kernel_text_rw(void); |
void set_kernel_text_ro(void); |
#else |
static inline void set_kernel_text_rw(void) { } |
static inline void set_kernel_text_ro(void) { } |
#endif |
#ifdef CONFIG_DEBUG_RODATA_TEST |
int rodata_test(void); |
/drivers/include/asm/cmpxchg.h |
---|
2,6 → 2,7 |
#define ASM_X86_CMPXCHG_H |
#include <linux/compiler.h> |
#include <asm/cpufeatures.h> |
#include <asm/alternative.h> /* Provides LOCK_PREFIX */ |
#define __HAVE_ARCH_CMPXCHG 1 |
/drivers/include/asm/cpufeature.h |
---|
1,289 → 1,8 |
/* |
* Defines x86 CPU feature bits |
*/ |
#ifndef _ASM_X86_CPUFEATURE_H |
#define _ASM_X86_CPUFEATURE_H |
#ifndef _ASM_X86_REQUIRED_FEATURES_H |
#include <asm/required-features.h> |
#endif |
#include <asm/processor.h> |
#ifndef _ASM_X86_DISABLED_FEATURES_H |
#include <asm/disabled-features.h> |
#endif |
#define NCAPINTS 16 /* N 32-bit words worth of info */ |
#define NBUGINTS 1 /* N 32-bit bug flags */ |
/* |
* Note: If the comment begins with a quoted string, that string is used |
* in /proc/cpuinfo instead of the macro name. If the string is "", |
* this feature bit is not displayed in /proc/cpuinfo at all. |
*/ |
/* Intel-defined CPU features, CPUID level 0x00000001 (edx), word 0 */ |
#define X86_FEATURE_FPU ( 0*32+ 0) /* Onboard FPU */ |
#define X86_FEATURE_VME ( 0*32+ 1) /* Virtual Mode Extensions */ |
#define X86_FEATURE_DE ( 0*32+ 2) /* Debugging Extensions */ |
#define X86_FEATURE_PSE ( 0*32+ 3) /* Page Size Extensions */ |
#define X86_FEATURE_TSC ( 0*32+ 4) /* Time Stamp Counter */ |
#define X86_FEATURE_MSR ( 0*32+ 5) /* Model-Specific Registers */ |
#define X86_FEATURE_PAE ( 0*32+ 6) /* Physical Address Extensions */ |
#define X86_FEATURE_MCE ( 0*32+ 7) /* Machine Check Exception */ |
#define X86_FEATURE_CX8 ( 0*32+ 8) /* CMPXCHG8 instruction */ |
#define X86_FEATURE_APIC ( 0*32+ 9) /* Onboard APIC */ |
#define X86_FEATURE_SEP ( 0*32+11) /* SYSENTER/SYSEXIT */ |
#define X86_FEATURE_MTRR ( 0*32+12) /* Memory Type Range Registers */ |
#define X86_FEATURE_PGE ( 0*32+13) /* Page Global Enable */ |
#define X86_FEATURE_MCA ( 0*32+14) /* Machine Check Architecture */ |
#define X86_FEATURE_CMOV ( 0*32+15) /* CMOV instructions */ |
/* (plus FCMOVcc, FCOMI with FPU) */ |
#define X86_FEATURE_PAT ( 0*32+16) /* Page Attribute Table */ |
#define X86_FEATURE_PSE36 ( 0*32+17) /* 36-bit PSEs */ |
#define X86_FEATURE_PN ( 0*32+18) /* Processor serial number */ |
#define X86_FEATURE_CLFLUSH ( 0*32+19) /* CLFLUSH instruction */ |
#define X86_FEATURE_DS ( 0*32+21) /* "dts" Debug Store */ |
#define X86_FEATURE_ACPI ( 0*32+22) /* ACPI via MSR */ |
#define X86_FEATURE_MMX ( 0*32+23) /* Multimedia Extensions */ |
#define X86_FEATURE_FXSR ( 0*32+24) /* FXSAVE/FXRSTOR, CR4.OSFXSR */ |
#define X86_FEATURE_XMM ( 0*32+25) /* "sse" */ |
#define X86_FEATURE_XMM2 ( 0*32+26) /* "sse2" */ |
#define X86_FEATURE_SELFSNOOP ( 0*32+27) /* "ss" CPU self snoop */ |
#define X86_FEATURE_HT ( 0*32+28) /* Hyper-Threading */ |
#define X86_FEATURE_ACC ( 0*32+29) /* "tm" Automatic clock control */ |
#define X86_FEATURE_IA64 ( 0*32+30) /* IA-64 processor */ |
#define X86_FEATURE_PBE ( 0*32+31) /* Pending Break Enable */ |
/* AMD-defined CPU features, CPUID level 0x80000001, word 1 */ |
/* Don't duplicate feature flags which are redundant with Intel! */ |
#define X86_FEATURE_SYSCALL ( 1*32+11) /* SYSCALL/SYSRET */ |
#define X86_FEATURE_MP ( 1*32+19) /* MP Capable. */ |
#define X86_FEATURE_NX ( 1*32+20) /* Execute Disable */ |
#define X86_FEATURE_MMXEXT ( 1*32+22) /* AMD MMX extensions */ |
#define X86_FEATURE_FXSR_OPT ( 1*32+25) /* FXSAVE/FXRSTOR optimizations */ |
#define X86_FEATURE_GBPAGES ( 1*32+26) /* "pdpe1gb" GB pages */ |
#define X86_FEATURE_RDTSCP ( 1*32+27) /* RDTSCP */ |
#define X86_FEATURE_LM ( 1*32+29) /* Long Mode (x86-64) */ |
#define X86_FEATURE_3DNOWEXT ( 1*32+30) /* AMD 3DNow! extensions */ |
#define X86_FEATURE_3DNOW ( 1*32+31) /* 3DNow! */ |
/* Transmeta-defined CPU features, CPUID level 0x80860001, word 2 */ |
#define X86_FEATURE_RECOVERY ( 2*32+ 0) /* CPU in recovery mode */ |
#define X86_FEATURE_LONGRUN ( 2*32+ 1) /* Longrun power control */ |
#define X86_FEATURE_LRTI ( 2*32+ 3) /* LongRun table interface */ |
/* Other features, Linux-defined mapping, word 3 */ |
/* This range is used for feature bits which conflict or are synthesized */ |
#define X86_FEATURE_CXMMX ( 3*32+ 0) /* Cyrix MMX extensions */ |
#define X86_FEATURE_K6_MTRR ( 3*32+ 1) /* AMD K6 nonstandard MTRRs */ |
#define X86_FEATURE_CYRIX_ARR ( 3*32+ 2) /* Cyrix ARRs (= MTRRs) */ |
#define X86_FEATURE_CENTAUR_MCR ( 3*32+ 3) /* Centaur MCRs (= MTRRs) */ |
/* cpu types for specific tunings: */ |
#define X86_FEATURE_K8 ( 3*32+ 4) /* "" Opteron, Athlon64 */ |
#define X86_FEATURE_K7 ( 3*32+ 5) /* "" Athlon */ |
#define X86_FEATURE_P3 ( 3*32+ 6) /* "" P3 */ |
#define X86_FEATURE_P4 ( 3*32+ 7) /* "" P4 */ |
#define X86_FEATURE_CONSTANT_TSC ( 3*32+ 8) /* TSC ticks at a constant rate */ |
#define X86_FEATURE_UP ( 3*32+ 9) /* smp kernel running on up */ |
/* free, was #define X86_FEATURE_FXSAVE_LEAK ( 3*32+10) * "" FXSAVE leaks FOP/FIP/FOP */ |
#define X86_FEATURE_ARCH_PERFMON ( 3*32+11) /* Intel Architectural PerfMon */ |
#define X86_FEATURE_PEBS ( 3*32+12) /* Precise-Event Based Sampling */ |
#define X86_FEATURE_BTS ( 3*32+13) /* Branch Trace Store */ |
#define X86_FEATURE_SYSCALL32 ( 3*32+14) /* "" syscall in ia32 userspace */ |
#define X86_FEATURE_SYSENTER32 ( 3*32+15) /* "" sysenter in ia32 userspace */ |
#define X86_FEATURE_REP_GOOD ( 3*32+16) /* rep microcode works well */ |
#define X86_FEATURE_MFENCE_RDTSC ( 3*32+17) /* "" Mfence synchronizes RDTSC */ |
#define X86_FEATURE_LFENCE_RDTSC ( 3*32+18) /* "" Lfence synchronizes RDTSC */ |
/* free, was #define X86_FEATURE_11AP ( 3*32+19) * "" Bad local APIC aka 11AP */ |
#define X86_FEATURE_NOPL ( 3*32+20) /* The NOPL (0F 1F) instructions */ |
#define X86_FEATURE_ALWAYS ( 3*32+21) /* "" Always-present feature */ |
#define X86_FEATURE_XTOPOLOGY ( 3*32+22) /* cpu topology enum extensions */ |
#define X86_FEATURE_TSC_RELIABLE ( 3*32+23) /* TSC is known to be reliable */ |
#define X86_FEATURE_NONSTOP_TSC ( 3*32+24) /* TSC does not stop in C states */ |
/* free, was #define X86_FEATURE_CLFLUSH_MONITOR ( 3*32+25) * "" clflush reqd with monitor */ |
#define X86_FEATURE_EXTD_APICID ( 3*32+26) /* has extended APICID (8 bits) */ |
#define X86_FEATURE_AMD_DCM ( 3*32+27) /* multi-node processor */ |
#define X86_FEATURE_APERFMPERF ( 3*32+28) /* APERFMPERF */ |
#define X86_FEATURE_EAGER_FPU ( 3*32+29) /* "eagerfpu" Non lazy FPU restore */ |
#define X86_FEATURE_NONSTOP_TSC_S3 ( 3*32+30) /* TSC doesn't stop in S3 state */ |
/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */ |
#define X86_FEATURE_XMM3 ( 4*32+ 0) /* "pni" SSE-3 */ |
#define X86_FEATURE_PCLMULQDQ ( 4*32+ 1) /* PCLMULQDQ instruction */ |
#define X86_FEATURE_DTES64 ( 4*32+ 2) /* 64-bit Debug Store */ |
#define X86_FEATURE_MWAIT ( 4*32+ 3) /* "monitor" Monitor/Mwait support */ |
#define X86_FEATURE_DSCPL ( 4*32+ 4) /* "ds_cpl" CPL Qual. Debug Store */ |
#define X86_FEATURE_VMX ( 4*32+ 5) /* Hardware virtualization */ |
#define X86_FEATURE_SMX ( 4*32+ 6) /* Safer mode */ |
#define X86_FEATURE_EST ( 4*32+ 7) /* Enhanced SpeedStep */ |
#define X86_FEATURE_TM2 ( 4*32+ 8) /* Thermal Monitor 2 */ |
#define X86_FEATURE_SSSE3 ( 4*32+ 9) /* Supplemental SSE-3 */ |
#define X86_FEATURE_CID ( 4*32+10) /* Context ID */ |
#define X86_FEATURE_SDBG ( 4*32+11) /* Silicon Debug */ |
#define X86_FEATURE_FMA ( 4*32+12) /* Fused multiply-add */ |
#define X86_FEATURE_CX16 ( 4*32+13) /* CMPXCHG16B */ |
#define X86_FEATURE_XTPR ( 4*32+14) /* Send Task Priority Messages */ |
#define X86_FEATURE_PDCM ( 4*32+15) /* Performance Capabilities */ |
#define X86_FEATURE_PCID ( 4*32+17) /* Process Context Identifiers */ |
#define X86_FEATURE_DCA ( 4*32+18) /* Direct Cache Access */ |
#define X86_FEATURE_XMM4_1 ( 4*32+19) /* "sse4_1" SSE-4.1 */ |
#define X86_FEATURE_XMM4_2 ( 4*32+20) /* "sse4_2" SSE-4.2 */ |
#define X86_FEATURE_X2APIC ( 4*32+21) /* x2APIC */ |
#define X86_FEATURE_MOVBE ( 4*32+22) /* MOVBE instruction */ |
#define X86_FEATURE_POPCNT ( 4*32+23) /* POPCNT instruction */ |
#define X86_FEATURE_TSC_DEADLINE_TIMER ( 4*32+24) /* Tsc deadline timer */ |
#define X86_FEATURE_AES ( 4*32+25) /* AES instructions */ |
#define X86_FEATURE_XSAVE ( 4*32+26) /* XSAVE/XRSTOR/XSETBV/XGETBV */ |
#define X86_FEATURE_OSXSAVE ( 4*32+27) /* "" XSAVE enabled in the OS */ |
#define X86_FEATURE_AVX ( 4*32+28) /* Advanced Vector Extensions */ |
#define X86_FEATURE_F16C ( 4*32+29) /* 16-bit fp conversions */ |
#define X86_FEATURE_RDRAND ( 4*32+30) /* The RDRAND instruction */ |
#define X86_FEATURE_HYPERVISOR ( 4*32+31) /* Running on a hypervisor */ |
/* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */ |
#define X86_FEATURE_XSTORE ( 5*32+ 2) /* "rng" RNG present (xstore) */ |
#define X86_FEATURE_XSTORE_EN ( 5*32+ 3) /* "rng_en" RNG enabled */ |
#define X86_FEATURE_XCRYPT ( 5*32+ 6) /* "ace" on-CPU crypto (xcrypt) */ |
#define X86_FEATURE_XCRYPT_EN ( 5*32+ 7) /* "ace_en" on-CPU crypto enabled */ |
#define X86_FEATURE_ACE2 ( 5*32+ 8) /* Advanced Cryptography Engine v2 */ |
#define X86_FEATURE_ACE2_EN ( 5*32+ 9) /* ACE v2 enabled */ |
#define X86_FEATURE_PHE ( 5*32+10) /* PadLock Hash Engine */ |
#define X86_FEATURE_PHE_EN ( 5*32+11) /* PHE enabled */ |
#define X86_FEATURE_PMM ( 5*32+12) /* PadLock Montgomery Multiplier */ |
#define X86_FEATURE_PMM_EN ( 5*32+13) /* PMM enabled */ |
/* More extended AMD flags: CPUID level 0x80000001, ecx, word 6 */ |
#define X86_FEATURE_LAHF_LM ( 6*32+ 0) /* LAHF/SAHF in long mode */ |
#define X86_FEATURE_CMP_LEGACY ( 6*32+ 1) /* If yes HyperThreading not valid */ |
#define X86_FEATURE_SVM ( 6*32+ 2) /* Secure virtual machine */ |
#define X86_FEATURE_EXTAPIC ( 6*32+ 3) /* Extended APIC space */ |
#define X86_FEATURE_CR8_LEGACY ( 6*32+ 4) /* CR8 in 32-bit mode */ |
#define X86_FEATURE_ABM ( 6*32+ 5) /* Advanced bit manipulation */ |
#define X86_FEATURE_SSE4A ( 6*32+ 6) /* SSE-4A */ |
#define X86_FEATURE_MISALIGNSSE ( 6*32+ 7) /* Misaligned SSE mode */ |
#define X86_FEATURE_3DNOWPREFETCH ( 6*32+ 8) /* 3DNow prefetch instructions */ |
#define X86_FEATURE_OSVW ( 6*32+ 9) /* OS Visible Workaround */ |
#define X86_FEATURE_IBS ( 6*32+10) /* Instruction Based Sampling */ |
#define X86_FEATURE_XOP ( 6*32+11) /* extended AVX instructions */ |
#define X86_FEATURE_SKINIT ( 6*32+12) /* SKINIT/STGI instructions */ |
#define X86_FEATURE_WDT ( 6*32+13) /* Watchdog timer */ |
#define X86_FEATURE_LWP ( 6*32+15) /* Light Weight Profiling */ |
#define X86_FEATURE_FMA4 ( 6*32+16) /* 4 operands MAC instructions */ |
#define X86_FEATURE_TCE ( 6*32+17) /* translation cache extension */ |
#define X86_FEATURE_NODEID_MSR ( 6*32+19) /* NodeId MSR */ |
#define X86_FEATURE_TBM ( 6*32+21) /* trailing bit manipulations */ |
#define X86_FEATURE_TOPOEXT ( 6*32+22) /* topology extensions CPUID leafs */ |
#define X86_FEATURE_PERFCTR_CORE ( 6*32+23) /* core performance counter extensions */ |
#define X86_FEATURE_PERFCTR_NB ( 6*32+24) /* NB performance counter extensions */ |
#define X86_FEATURE_BPEXT (6*32+26) /* data breakpoint extension */ |
#define X86_FEATURE_PERFCTR_L2 ( 6*32+28) /* L2 performance counter extensions */ |
#define X86_FEATURE_MWAITX ( 6*32+29) /* MWAIT extension (MONITORX/MWAITX) */ |
/* |
* Auxiliary flags: Linux defined - For features scattered in various |
* CPUID levels like 0x6, 0xA etc, word 7. |
* |
* Reuse free bits when adding new feature flags! |
*/ |
#define X86_FEATURE_CPB ( 7*32+ 2) /* AMD Core Performance Boost */ |
#define X86_FEATURE_EPB ( 7*32+ 3) /* IA32_ENERGY_PERF_BIAS support */ |
#define X86_FEATURE_HW_PSTATE ( 7*32+ 8) /* AMD HW-PState */ |
#define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */ |
#define X86_FEATURE_INTEL_PT ( 7*32+15) /* Intel Processor Trace */ |
/* Virtualization flags: Linux defined, word 8 */ |
#define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */ |
#define X86_FEATURE_VNMI ( 8*32+ 1) /* Intel Virtual NMI */ |
#define X86_FEATURE_FLEXPRIORITY ( 8*32+ 2) /* Intel FlexPriority */ |
#define X86_FEATURE_EPT ( 8*32+ 3) /* Intel Extended Page Table */ |
#define X86_FEATURE_VPID ( 8*32+ 4) /* Intel Virtual Processor ID */ |
#define X86_FEATURE_VMMCALL ( 8*32+15) /* Prefer vmmcall to vmcall */ |
#define X86_FEATURE_XENPV ( 8*32+16) /* "" Xen paravirtual guest */ |
/* Intel-defined CPU features, CPUID level 0x00000007:0 (ebx), word 9 */ |
#define X86_FEATURE_FSGSBASE ( 9*32+ 0) /* {RD/WR}{FS/GS}BASE instructions*/ |
#define X86_FEATURE_TSC_ADJUST ( 9*32+ 1) /* TSC adjustment MSR 0x3b */ |
#define X86_FEATURE_BMI1 ( 9*32+ 3) /* 1st group bit manipulation extensions */ |
#define X86_FEATURE_HLE ( 9*32+ 4) /* Hardware Lock Elision */ |
#define X86_FEATURE_AVX2 ( 9*32+ 5) /* AVX2 instructions */ |
#define X86_FEATURE_SMEP ( 9*32+ 7) /* Supervisor Mode Execution Protection */ |
#define X86_FEATURE_BMI2 ( 9*32+ 8) /* 2nd group bit manipulation extensions */ |
#define X86_FEATURE_ERMS ( 9*32+ 9) /* Enhanced REP MOVSB/STOSB */ |
#define X86_FEATURE_INVPCID ( 9*32+10) /* Invalidate Processor Context ID */ |
#define X86_FEATURE_RTM ( 9*32+11) /* Restricted Transactional Memory */ |
#define X86_FEATURE_CQM ( 9*32+12) /* Cache QoS Monitoring */ |
#define X86_FEATURE_MPX ( 9*32+14) /* Memory Protection Extension */ |
#define X86_FEATURE_AVX512F ( 9*32+16) /* AVX-512 Foundation */ |
#define X86_FEATURE_RDSEED ( 9*32+18) /* The RDSEED instruction */ |
#define X86_FEATURE_ADX ( 9*32+19) /* The ADCX and ADOX instructions */ |
#define X86_FEATURE_SMAP ( 9*32+20) /* Supervisor Mode Access Prevention */ |
#define X86_FEATURE_PCOMMIT ( 9*32+22) /* PCOMMIT instruction */ |
#define X86_FEATURE_CLFLUSHOPT ( 9*32+23) /* CLFLUSHOPT instruction */ |
#define X86_FEATURE_CLWB ( 9*32+24) /* CLWB instruction */ |
#define X86_FEATURE_AVX512PF ( 9*32+26) /* AVX-512 Prefetch */ |
#define X86_FEATURE_AVX512ER ( 9*32+27) /* AVX-512 Exponential and Reciprocal */ |
#define X86_FEATURE_AVX512CD ( 9*32+28) /* AVX-512 Conflict Detection */ |
#define X86_FEATURE_SHA_NI ( 9*32+29) /* SHA1/SHA256 Instruction Extensions */ |
/* Extended state features, CPUID level 0x0000000d:1 (eax), word 10 */ |
#define X86_FEATURE_XSAVEOPT (10*32+ 0) /* XSAVEOPT */ |
#define X86_FEATURE_XSAVEC (10*32+ 1) /* XSAVEC */ |
#define X86_FEATURE_XGETBV1 (10*32+ 2) /* XGETBV with ECX = 1 */ |
#define X86_FEATURE_XSAVES (10*32+ 3) /* XSAVES/XRSTORS */ |
/* Intel-defined CPU QoS Sub-leaf, CPUID level 0x0000000F:0 (edx), word 11 */ |
#define X86_FEATURE_CQM_LLC (11*32+ 1) /* LLC QoS if 1 */ |
/* Intel-defined CPU QoS Sub-leaf, CPUID level 0x0000000F:1 (edx), word 12 */ |
#define X86_FEATURE_CQM_OCCUP_LLC (12*32+ 0) /* LLC occupancy monitoring if 1 */ |
/* AMD-defined CPU features, CPUID level 0x80000008 (ebx), word 13 */ |
#define X86_FEATURE_CLZERO (13*32+0) /* CLZERO instruction */ |
/* Thermal and Power Management Leaf, CPUID level 0x00000006 (eax), word 14 */ |
#define X86_FEATURE_DTHERM (14*32+ 0) /* Digital Thermal Sensor */ |
#define X86_FEATURE_IDA (14*32+ 1) /* Intel Dynamic Acceleration */ |
#define X86_FEATURE_ARAT (14*32+ 2) /* Always Running APIC Timer */ |
#define X86_FEATURE_PLN (14*32+ 4) /* Intel Power Limit Notification */ |
#define X86_FEATURE_PTS (14*32+ 6) /* Intel Package Thermal Status */ |
#define X86_FEATURE_HWP (14*32+ 7) /* Intel Hardware P-states */ |
#define X86_FEATURE_HWP_NOTIFY (14*32+ 8) /* HWP Notification */ |
#define X86_FEATURE_HWP_ACT_WINDOW (14*32+ 9) /* HWP Activity Window */ |
#define X86_FEATURE_HWP_EPP (14*32+10) /* HWP Energy Perf. Preference */ |
#define X86_FEATURE_HWP_PKG_REQ (14*32+11) /* HWP Package Level Request */ |
/* AMD SVM Feature Identification, CPUID level 0x8000000a (edx), word 15 */ |
#define X86_FEATURE_NPT (15*32+ 0) /* Nested Page Table support */ |
#define X86_FEATURE_LBRV (15*32+ 1) /* LBR Virtualization support */ |
#define X86_FEATURE_SVML (15*32+ 2) /* "svm_lock" SVM locking MSR */ |
#define X86_FEATURE_NRIPS (15*32+ 3) /* "nrip_save" SVM next_rip save */ |
#define X86_FEATURE_TSCRATEMSR (15*32+ 4) /* "tsc_scale" TSC scaling support */ |
#define X86_FEATURE_VMCBCLEAN (15*32+ 5) /* "vmcb_clean" VMCB clean bits support */ |
#define X86_FEATURE_FLUSHBYASID (15*32+ 6) /* flush-by-ASID support */ |
#define X86_FEATURE_DECODEASSISTS (15*32+ 7) /* Decode Assists support */ |
#define X86_FEATURE_PAUSEFILTER (15*32+10) /* filtered pause intercept */ |
#define X86_FEATURE_PFTHRESHOLD (15*32+12) /* pause filter threshold */ |
/* |
* BUG word(s) |
*/ |
#define X86_BUG(x) (NCAPINTS*32 + (x)) |
#define X86_BUG_F00F X86_BUG(0) /* Intel F00F */ |
#define X86_BUG_FDIV X86_BUG(1) /* FPU FDIV */ |
#define X86_BUG_COMA X86_BUG(2) /* Cyrix 6x86 coma */ |
#define X86_BUG_AMD_TLB_MMATCH X86_BUG(3) /* "tlb_mmatch" AMD Erratum 383 */ |
#define X86_BUG_AMD_APIC_C1E X86_BUG(4) /* "apic_c1e" AMD Erratum 400 */ |
#define X86_BUG_11AP X86_BUG(5) /* Bad local APIC aka 11AP */ |
#define X86_BUG_FXSAVE_LEAK X86_BUG(6) /* FXSAVE leaks FOP/FIP/FOP */ |
#define X86_BUG_CLFLUSH_MONITOR X86_BUG(7) /* AAI65, CLFLUSH required before MONITOR */ |
#define X86_BUG_SYSRET_SS_ATTRS X86_BUG(8) /* SYSRET doesn't fix up SS attrs */ |
#if defined(__KERNEL__) && !defined(__ASSEMBLY__) |
#include <asm/asm.h> |
307,6 → 26,7 |
CPUID_8000_0008_EBX, |
CPUID_6_EAX, |
CPUID_8000_000A_EDX, |
CPUID_7_ECX, |
}; |
#ifdef CONFIG_X86_FEATURE_NAMES |
338,7 → 58,14 |
(((bit)>>5)==6 && (1UL<<((bit)&31) & REQUIRED_MASK6)) || \ |
(((bit)>>5)==7 && (1UL<<((bit)&31) & REQUIRED_MASK7)) || \ |
(((bit)>>5)==8 && (1UL<<((bit)&31) & REQUIRED_MASK8)) || \ |
(((bit)>>5)==9 && (1UL<<((bit)&31) & REQUIRED_MASK9)) ) |
(((bit)>>5)==9 && (1UL<<((bit)&31) & REQUIRED_MASK9 )) || \ |
(((bit)>>5)==10 && (1UL<<((bit)&31) & REQUIRED_MASK10)) || \ |
(((bit)>>5)==11 && (1UL<<((bit)&31) & REQUIRED_MASK11)) || \ |
(((bit)>>5)==12 && (1UL<<((bit)&31) & REQUIRED_MASK12)) || \ |
(((bit)>>5)==13 && (1UL<<((bit)&31) & REQUIRED_MASK13)) || \ |
(((bit)>>5)==14 && (1UL<<((bit)&31) & REQUIRED_MASK14)) || \ |
(((bit)>>5)==15 && (1UL<<((bit)&31) & REQUIRED_MASK15)) || \ |
(((bit)>>5)==16 && (1UL<<((bit)&31) & REQUIRED_MASK16)) ) |
#define DISABLED_MASK_BIT_SET(bit) \ |
( (((bit)>>5)==0 && (1UL<<((bit)&31) & DISABLED_MASK0)) || \ |
350,7 → 77,14 |
(((bit)>>5)==6 && (1UL<<((bit)&31) & DISABLED_MASK6)) || \ |
(((bit)>>5)==7 && (1UL<<((bit)&31) & DISABLED_MASK7)) || \ |
(((bit)>>5)==8 && (1UL<<((bit)&31) & DISABLED_MASK8)) || \ |
(((bit)>>5)==9 && (1UL<<((bit)&31) & DISABLED_MASK9)) ) |
(((bit)>>5)==9 && (1UL<<((bit)&31) & DISABLED_MASK9 )) || \ |
(((bit)>>5)==10 && (1UL<<((bit)&31) & DISABLED_MASK10)) || \ |
(((bit)>>5)==11 && (1UL<<((bit)&31) & DISABLED_MASK11)) || \ |
(((bit)>>5)==12 && (1UL<<((bit)&31) & DISABLED_MASK12)) || \ |
(((bit)>>5)==13 && (1UL<<((bit)&31) & DISABLED_MASK13)) || \ |
(((bit)>>5)==14 && (1UL<<((bit)&31) & DISABLED_MASK14)) || \ |
(((bit)>>5)==15 && (1UL<<((bit)&31) & DISABLED_MASK15)) || \ |
(((bit)>>5)==16 && (1UL<<((bit)&31) & DISABLED_MASK16)) ) |
#define cpu_has(c, bit) \ |
(__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 : \ |
369,8 → 103,7 |
* is not relevant. |
*/ |
#define cpu_feature_enabled(bit) \ |
(__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 : \ |
cpu_has(&boot_cpu_data, bit)) |
(__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 : static_cpu_has(bit)) |
#define boot_cpu_has(bit) cpu_has(&boot_cpu_data, bit) |
406,107 → 139,20 |
#define cpu_has_osxsave boot_cpu_has(X86_FEATURE_OSXSAVE) |
#define cpu_has_hypervisor boot_cpu_has(X86_FEATURE_HYPERVISOR) |
/* |
* Do not add any more of those clumsy macros - use static_cpu_has_safe() for |
* Do not add any more of those clumsy macros - use static_cpu_has() for |
* fast paths and boot_cpu_has() otherwise! |
*/ |
#if __GNUC__ >= 4 && defined(CONFIG_X86_FAST_FEATURE_TESTS) |
extern void warn_pre_alternatives(void); |
extern bool __static_cpu_has_safe(u16 bit); |
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_X86_FAST_FEATURE_TESTS) |
/* |
* Static testing of CPU features. Used the same as boot_cpu_has(). |
* These are only valid after alternatives have run, but will statically |
* patch the target code for additional performance. |
* These will statically patch the target code for additional |
* performance. |
*/ |
static __always_inline __pure bool __static_cpu_has(u16 bit) |
static __always_inline __pure bool _static_cpu_has(u16 bit) |
{ |
#ifdef CC_HAVE_ASM_GOTO |
#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS |
/* |
* Catch too early usage of this before alternatives |
* have run. |
*/ |
asm_volatile_goto("1: jmp %l[t_warn]\n" |
asm_volatile_goto("1: jmp 6f\n" |
"2:\n" |
".section .altinstructions,\"a\"\n" |
" .long 1b - .\n" |
" .long 0\n" /* no replacement */ |
" .word %P0\n" /* 1: do replace */ |
" .byte 2b - 1b\n" /* source len */ |
" .byte 0\n" /* replacement len */ |
" .byte 0\n" /* pad len */ |
".previous\n" |
/* skipping size check since replacement size = 0 */ |
: : "i" (X86_FEATURE_ALWAYS) : : t_warn); |
#endif |
asm_volatile_goto("1: jmp %l[t_no]\n" |
"2:\n" |
".section .altinstructions,\"a\"\n" |
" .long 1b - .\n" |
" .long 0\n" /* no replacement */ |
" .word %P0\n" /* feature bit */ |
" .byte 2b - 1b\n" /* source len */ |
" .byte 0\n" /* replacement len */ |
" .byte 0\n" /* pad len */ |
".previous\n" |
/* skipping size check since replacement size = 0 */ |
: : "i" (bit) : : t_no); |
return true; |
t_no: |
return false; |
#ifdef CONFIG_X86_DEBUG_STATIC_CPU_HAS |
t_warn: |
warn_pre_alternatives(); |
return false; |
#endif |
#else /* CC_HAVE_ASM_GOTO */ |
u8 flag; |
/* Open-coded due to __stringify() in ALTERNATIVE() */ |
asm volatile("1: movb $0,%0\n" |
"2:\n" |
".section .altinstructions,\"a\"\n" |
" .long 1b - .\n" |
" .long 3f - .\n" |
" .word %P1\n" /* feature bit */ |
" .byte 2b - 1b\n" /* source len */ |
" .byte 4f - 3f\n" /* replacement len */ |
" .byte 0\n" /* pad len */ |
".previous\n" |
".section .discard,\"aw\",@progbits\n" |
" .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */ |
".previous\n" |
".section .altinstr_replacement,\"ax\"\n" |
"3: movb $1,%0\n" |
"4:\n" |
".previous\n" |
: "=qm" (flag) : "i" (bit)); |
return flag; |
#endif /* CC_HAVE_ASM_GOTO */ |
} |
#define static_cpu_has(bit) \ |
( \ |
__builtin_constant_p(boot_cpu_has(bit)) ? \ |
boot_cpu_has(bit) : \ |
__builtin_constant_p(bit) ? \ |
__static_cpu_has(bit) : \ |
boot_cpu_has(bit) \ |
) |
static __always_inline __pure bool _static_cpu_has_safe(u16 bit) |
{ |
#ifdef CC_HAVE_ASM_GOTO |
asm_volatile_goto("1: jmp %l[t_dynamic]\n" |
"2:\n" |
".skip -(((5f-4f) - (2b-1b)) > 0) * " |
"((5f-4f) - (2b-1b)),0x90\n" |
"3:\n" |
530,66 → 176,34 |
" .byte 0\n" /* repl len */ |
" .byte 0\n" /* pad len */ |
".previous\n" |
: : "i" (bit), "i" (X86_FEATURE_ALWAYS) |
: : t_dynamic, t_no); |
".section .altinstr_aux,\"ax\"\n" |
"6:\n" |
" testb %[bitnum],%[cap_byte]\n" |
" jnz %l[t_yes]\n" |
" jmp %l[t_no]\n" |
".previous\n" |
: : "i" (bit), "i" (X86_FEATURE_ALWAYS), |
[bitnum] "i" (1 << (bit & 7)), |
[cap_byte] "m" (((const char *)boot_cpu_data.x86_capability)[bit >> 3]) |
: : t_yes, t_no); |
t_yes: |
return true; |
t_no: |
return false; |
t_dynamic: |
return __static_cpu_has_safe(bit); |
#else |
u8 flag; |
/* Open-coded due to __stringify() in ALTERNATIVE() */ |
asm volatile("1: movb $2,%0\n" |
"2:\n" |
".section .altinstructions,\"a\"\n" |
" .long 1b - .\n" /* src offset */ |
" .long 3f - .\n" /* repl offset */ |
" .word %P2\n" /* always replace */ |
" .byte 2b - 1b\n" /* source len */ |
" .byte 4f - 3f\n" /* replacement len */ |
" .byte 0\n" /* pad len */ |
".previous\n" |
".section .discard,\"aw\",@progbits\n" |
" .byte 0xff + (4f-3f) - (2b-1b)\n" /* size check */ |
".previous\n" |
".section .altinstr_replacement,\"ax\"\n" |
"3: movb $0,%0\n" |
"4:\n" |
".previous\n" |
".section .altinstructions,\"a\"\n" |
" .long 1b - .\n" /* src offset */ |
" .long 5f - .\n" /* repl offset */ |
" .word %P1\n" /* feature bit */ |
" .byte 4b - 3b\n" /* src len */ |
" .byte 6f - 5f\n" /* repl len */ |
" .byte 0\n" /* pad len */ |
".previous\n" |
".section .discard,\"aw\",@progbits\n" |
" .byte 0xff + (6f-5f) - (4b-3b)\n" /* size check */ |
".previous\n" |
".section .altinstr_replacement,\"ax\"\n" |
"5: movb $1,%0\n" |
"6:\n" |
".previous\n" |
: "=qm" (flag) |
: "i" (bit), "i" (X86_FEATURE_ALWAYS)); |
return (flag == 2 ? __static_cpu_has_safe(bit) : flag); |
#endif /* CC_HAVE_ASM_GOTO */ |
} |
#define static_cpu_has_safe(bit) \ |
#define static_cpu_has(bit) \ |
( \ |
__builtin_constant_p(boot_cpu_has(bit)) ? \ |
boot_cpu_has(bit) : \ |
_static_cpu_has_safe(bit) \ |
_static_cpu_has(bit) \ |
) |
#else |
/* |
* gcc 3.x is too stupid to do the static test; fall back to dynamic. |
* Fall back to dynamic for gcc versions which don't support asm goto. Should be |
* a minority now anyway. |
*/ |
#define static_cpu_has(bit) boot_cpu_has(bit) |
#define static_cpu_has_safe(bit) boot_cpu_has(bit) |
#endif |
#define cpu_has_bug(c, bit) cpu_has(c, (bit)) |
597,7 → 211,6 |
#define clear_cpu_bug(c, bit) clear_cpu_cap(c, (bit)) |
#define static_cpu_has_bug(bit) static_cpu_has((bit)) |
#define static_cpu_has_bug_safe(bit) static_cpu_has_safe((bit)) |
#define boot_cpu_has_bug(bit) cpu_has_bug(&boot_cpu_data, (bit)) |
#define MAX_CPU_FEATURES (NCAPINTS * 32) |
/drivers/include/asm/cpufeatures.h |
---|
0,0 → 1,306 |
#ifndef _ASM_X86_CPUFEATURES_H |
#define _ASM_X86_CPUFEATURES_H |
#ifndef _ASM_X86_REQUIRED_FEATURES_H |
#include <asm/required-features.h> |
#endif |
#ifndef _ASM_X86_DISABLED_FEATURES_H |
#include <asm/disabled-features.h> |
#endif |
/* |
* Defines x86 CPU feature bits |
*/ |
#define NCAPINTS 17 /* N 32-bit words worth of info */ |
#define NBUGINTS 1 /* N 32-bit bug flags */ |
/* |
* Note: If the comment begins with a quoted string, that string is used |
* in /proc/cpuinfo instead of the macro name. If the string is "", |
* this feature bit is not displayed in /proc/cpuinfo at all. |
*/ |
/* Intel-defined CPU features, CPUID level 0x00000001 (edx), word 0 */ |
#define X86_FEATURE_FPU ( 0*32+ 0) /* Onboard FPU */ |
#define X86_FEATURE_VME ( 0*32+ 1) /* Virtual Mode Extensions */ |
#define X86_FEATURE_DE ( 0*32+ 2) /* Debugging Extensions */ |
#define X86_FEATURE_PSE ( 0*32+ 3) /* Page Size Extensions */ |
#define X86_FEATURE_TSC ( 0*32+ 4) /* Time Stamp Counter */ |
#define X86_FEATURE_MSR ( 0*32+ 5) /* Model-Specific Registers */ |
#define X86_FEATURE_PAE ( 0*32+ 6) /* Physical Address Extensions */ |
#define X86_FEATURE_MCE ( 0*32+ 7) /* Machine Check Exception */ |
#define X86_FEATURE_CX8 ( 0*32+ 8) /* CMPXCHG8 instruction */ |
#define X86_FEATURE_APIC ( 0*32+ 9) /* Onboard APIC */ |
#define X86_FEATURE_SEP ( 0*32+11) /* SYSENTER/SYSEXIT */ |
#define X86_FEATURE_MTRR ( 0*32+12) /* Memory Type Range Registers */ |
#define X86_FEATURE_PGE ( 0*32+13) /* Page Global Enable */ |
#define X86_FEATURE_MCA ( 0*32+14) /* Machine Check Architecture */ |
#define X86_FEATURE_CMOV ( 0*32+15) /* CMOV instructions */ |
/* (plus FCMOVcc, FCOMI with FPU) */ |
#define X86_FEATURE_PAT ( 0*32+16) /* Page Attribute Table */ |
#define X86_FEATURE_PSE36 ( 0*32+17) /* 36-bit PSEs */ |
#define X86_FEATURE_PN ( 0*32+18) /* Processor serial number */ |
#define X86_FEATURE_CLFLUSH ( 0*32+19) /* CLFLUSH instruction */ |
#define X86_FEATURE_DS ( 0*32+21) /* "dts" Debug Store */ |
#define X86_FEATURE_ACPI ( 0*32+22) /* ACPI via MSR */ |
#define X86_FEATURE_MMX ( 0*32+23) /* Multimedia Extensions */ |
#define X86_FEATURE_FXSR ( 0*32+24) /* FXSAVE/FXRSTOR, CR4.OSFXSR */ |
#define X86_FEATURE_XMM ( 0*32+25) /* "sse" */ |
#define X86_FEATURE_XMM2 ( 0*32+26) /* "sse2" */ |
#define X86_FEATURE_SELFSNOOP ( 0*32+27) /* "ss" CPU self snoop */ |
#define X86_FEATURE_HT ( 0*32+28) /* Hyper-Threading */ |
#define X86_FEATURE_ACC ( 0*32+29) /* "tm" Automatic clock control */ |
#define X86_FEATURE_IA64 ( 0*32+30) /* IA-64 processor */ |
#define X86_FEATURE_PBE ( 0*32+31) /* Pending Break Enable */ |
/* AMD-defined CPU features, CPUID level 0x80000001, word 1 */ |
/* Don't duplicate feature flags which are redundant with Intel! */ |
#define X86_FEATURE_SYSCALL ( 1*32+11) /* SYSCALL/SYSRET */ |
#define X86_FEATURE_MP ( 1*32+19) /* MP Capable. */ |
#define X86_FEATURE_NX ( 1*32+20) /* Execute Disable */ |
#define X86_FEATURE_MMXEXT ( 1*32+22) /* AMD MMX extensions */ |
#define X86_FEATURE_FXSR_OPT ( 1*32+25) /* FXSAVE/FXRSTOR optimizations */ |
#define X86_FEATURE_GBPAGES ( 1*32+26) /* "pdpe1gb" GB pages */ |
#define X86_FEATURE_RDTSCP ( 1*32+27) /* RDTSCP */ |
#define X86_FEATURE_LM ( 1*32+29) /* Long Mode (x86-64) */ |
#define X86_FEATURE_3DNOWEXT ( 1*32+30) /* AMD 3DNow! extensions */ |
#define X86_FEATURE_3DNOW ( 1*32+31) /* 3DNow! */ |
/* Transmeta-defined CPU features, CPUID level 0x80860001, word 2 */ |
#define X86_FEATURE_RECOVERY ( 2*32+ 0) /* CPU in recovery mode */ |
#define X86_FEATURE_LONGRUN ( 2*32+ 1) /* Longrun power control */ |
#define X86_FEATURE_LRTI ( 2*32+ 3) /* LongRun table interface */ |
/* Other features, Linux-defined mapping, word 3 */ |
/* This range is used for feature bits which conflict or are synthesized */ |
#define X86_FEATURE_CXMMX ( 3*32+ 0) /* Cyrix MMX extensions */ |
#define X86_FEATURE_K6_MTRR ( 3*32+ 1) /* AMD K6 nonstandard MTRRs */ |
#define X86_FEATURE_CYRIX_ARR ( 3*32+ 2) /* Cyrix ARRs (= MTRRs) */ |
#define X86_FEATURE_CENTAUR_MCR ( 3*32+ 3) /* Centaur MCRs (= MTRRs) */ |
/* cpu types for specific tunings: */ |
#define X86_FEATURE_K8 ( 3*32+ 4) /* "" Opteron, Athlon64 */ |
#define X86_FEATURE_K7 ( 3*32+ 5) /* "" Athlon */ |
#define X86_FEATURE_P3 ( 3*32+ 6) /* "" P3 */ |
#define X86_FEATURE_P4 ( 3*32+ 7) /* "" P4 */ |
#define X86_FEATURE_CONSTANT_TSC ( 3*32+ 8) /* TSC ticks at a constant rate */ |
#define X86_FEATURE_UP ( 3*32+ 9) /* smp kernel running on up */ |
#define X86_FEATURE_ART ( 3*32+10) /* Platform has always running timer (ART) */ |
#define X86_FEATURE_ARCH_PERFMON ( 3*32+11) /* Intel Architectural PerfMon */ |
#define X86_FEATURE_PEBS ( 3*32+12) /* Precise-Event Based Sampling */ |
#define X86_FEATURE_BTS ( 3*32+13) /* Branch Trace Store */ |
#define X86_FEATURE_SYSCALL32 ( 3*32+14) /* "" syscall in ia32 userspace */ |
#define X86_FEATURE_SYSENTER32 ( 3*32+15) /* "" sysenter in ia32 userspace */ |
#define X86_FEATURE_REP_GOOD ( 3*32+16) /* rep microcode works well */ |
#define X86_FEATURE_MFENCE_RDTSC ( 3*32+17) /* "" Mfence synchronizes RDTSC */ |
#define X86_FEATURE_LFENCE_RDTSC ( 3*32+18) /* "" Lfence synchronizes RDTSC */ |
#define X86_FEATURE_ACC_POWER ( 3*32+19) /* AMD Accumulated Power Mechanism */ |
#define X86_FEATURE_NOPL ( 3*32+20) /* The NOPL (0F 1F) instructions */ |
#define X86_FEATURE_ALWAYS ( 3*32+21) /* "" Always-present feature */ |
#define X86_FEATURE_XTOPOLOGY ( 3*32+22) /* cpu topology enum extensions */ |
#define X86_FEATURE_TSC_RELIABLE ( 3*32+23) /* TSC is known to be reliable */ |
#define X86_FEATURE_NONSTOP_TSC ( 3*32+24) /* TSC does not stop in C states */ |
/* free, was #define X86_FEATURE_CLFLUSH_MONITOR ( 3*32+25) * "" clflush reqd with monitor */ |
#define X86_FEATURE_EXTD_APICID ( 3*32+26) /* has extended APICID (8 bits) */ |
#define X86_FEATURE_AMD_DCM ( 3*32+27) /* multi-node processor */ |
#define X86_FEATURE_APERFMPERF ( 3*32+28) /* APERFMPERF */ |
#define X86_FEATURE_EAGER_FPU ( 3*32+29) /* "eagerfpu" Non lazy FPU restore */ |
#define X86_FEATURE_NONSTOP_TSC_S3 ( 3*32+30) /* TSC doesn't stop in S3 state */ |
#define X86_FEATURE_MCE_RECOVERY ( 3*32+31) /* cpu has recoverable machine checks */ |
/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */ |
#define X86_FEATURE_XMM3 ( 4*32+ 0) /* "pni" SSE-3 */ |
#define X86_FEATURE_PCLMULQDQ ( 4*32+ 1) /* PCLMULQDQ instruction */ |
#define X86_FEATURE_DTES64 ( 4*32+ 2) /* 64-bit Debug Store */ |
#define X86_FEATURE_MWAIT ( 4*32+ 3) /* "monitor" Monitor/Mwait support */ |
#define X86_FEATURE_DSCPL ( 4*32+ 4) /* "ds_cpl" CPL Qual. Debug Store */ |
#define X86_FEATURE_VMX ( 4*32+ 5) /* Hardware virtualization */ |
#define X86_FEATURE_SMX ( 4*32+ 6) /* Safer mode */ |
#define X86_FEATURE_EST ( 4*32+ 7) /* Enhanced SpeedStep */ |
#define X86_FEATURE_TM2 ( 4*32+ 8) /* Thermal Monitor 2 */ |
#define X86_FEATURE_SSSE3 ( 4*32+ 9) /* Supplemental SSE-3 */ |
#define X86_FEATURE_CID ( 4*32+10) /* Context ID */ |
#define X86_FEATURE_SDBG ( 4*32+11) /* Silicon Debug */ |
#define X86_FEATURE_FMA ( 4*32+12) /* Fused multiply-add */ |
#define X86_FEATURE_CX16 ( 4*32+13) /* CMPXCHG16B */ |
#define X86_FEATURE_XTPR ( 4*32+14) /* Send Task Priority Messages */ |
#define X86_FEATURE_PDCM ( 4*32+15) /* Performance Capabilities */ |
#define X86_FEATURE_PCID ( 4*32+17) /* Process Context Identifiers */ |
#define X86_FEATURE_DCA ( 4*32+18) /* Direct Cache Access */ |
#define X86_FEATURE_XMM4_1 ( 4*32+19) /* "sse4_1" SSE-4.1 */ |
#define X86_FEATURE_XMM4_2 ( 4*32+20) /* "sse4_2" SSE-4.2 */ |
#define X86_FEATURE_X2APIC ( 4*32+21) /* x2APIC */ |
#define X86_FEATURE_MOVBE ( 4*32+22) /* MOVBE instruction */ |
#define X86_FEATURE_POPCNT ( 4*32+23) /* POPCNT instruction */ |
#define X86_FEATURE_TSC_DEADLINE_TIMER ( 4*32+24) /* Tsc deadline timer */ |
#define X86_FEATURE_AES ( 4*32+25) /* AES instructions */ |
#define X86_FEATURE_XSAVE ( 4*32+26) /* XSAVE/XRSTOR/XSETBV/XGETBV */ |
#define X86_FEATURE_OSXSAVE ( 4*32+27) /* "" XSAVE enabled in the OS */ |
#define X86_FEATURE_AVX ( 4*32+28) /* Advanced Vector Extensions */ |
#define X86_FEATURE_F16C ( 4*32+29) /* 16-bit fp conversions */ |
#define X86_FEATURE_RDRAND ( 4*32+30) /* The RDRAND instruction */ |
#define X86_FEATURE_HYPERVISOR ( 4*32+31) /* Running on a hypervisor */ |
/* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */ |
#define X86_FEATURE_XSTORE ( 5*32+ 2) /* "rng" RNG present (xstore) */ |
#define X86_FEATURE_XSTORE_EN ( 5*32+ 3) /* "rng_en" RNG enabled */ |
#define X86_FEATURE_XCRYPT ( 5*32+ 6) /* "ace" on-CPU crypto (xcrypt) */ |
#define X86_FEATURE_XCRYPT_EN ( 5*32+ 7) /* "ace_en" on-CPU crypto enabled */ |
#define X86_FEATURE_ACE2 ( 5*32+ 8) /* Advanced Cryptography Engine v2 */ |
#define X86_FEATURE_ACE2_EN ( 5*32+ 9) /* ACE v2 enabled */ |
#define X86_FEATURE_PHE ( 5*32+10) /* PadLock Hash Engine */ |
#define X86_FEATURE_PHE_EN ( 5*32+11) /* PHE enabled */ |
#define X86_FEATURE_PMM ( 5*32+12) /* PadLock Montgomery Multiplier */ |
#define X86_FEATURE_PMM_EN ( 5*32+13) /* PMM enabled */ |
/* More extended AMD flags: CPUID level 0x80000001, ecx, word 6 */ |
#define X86_FEATURE_LAHF_LM ( 6*32+ 0) /* LAHF/SAHF in long mode */ |
#define X86_FEATURE_CMP_LEGACY ( 6*32+ 1) /* If yes HyperThreading not valid */ |
#define X86_FEATURE_SVM ( 6*32+ 2) /* Secure virtual machine */ |
#define X86_FEATURE_EXTAPIC ( 6*32+ 3) /* Extended APIC space */ |
#define X86_FEATURE_CR8_LEGACY ( 6*32+ 4) /* CR8 in 32-bit mode */ |
#define X86_FEATURE_ABM ( 6*32+ 5) /* Advanced bit manipulation */ |
#define X86_FEATURE_SSE4A ( 6*32+ 6) /* SSE-4A */ |
#define X86_FEATURE_MISALIGNSSE ( 6*32+ 7) /* Misaligned SSE mode */ |
#define X86_FEATURE_3DNOWPREFETCH ( 6*32+ 8) /* 3DNow prefetch instructions */ |
#define X86_FEATURE_OSVW ( 6*32+ 9) /* OS Visible Workaround */ |
#define X86_FEATURE_IBS ( 6*32+10) /* Instruction Based Sampling */ |
#define X86_FEATURE_XOP ( 6*32+11) /* extended AVX instructions */ |
#define X86_FEATURE_SKINIT ( 6*32+12) /* SKINIT/STGI instructions */ |
#define X86_FEATURE_WDT ( 6*32+13) /* Watchdog timer */ |
#define X86_FEATURE_LWP ( 6*32+15) /* Light Weight Profiling */ |
#define X86_FEATURE_FMA4 ( 6*32+16) /* 4 operands MAC instructions */ |
#define X86_FEATURE_TCE ( 6*32+17) /* translation cache extension */ |
#define X86_FEATURE_NODEID_MSR ( 6*32+19) /* NodeId MSR */ |
#define X86_FEATURE_TBM ( 6*32+21) /* trailing bit manipulations */ |
#define X86_FEATURE_TOPOEXT ( 6*32+22) /* topology extensions CPUID leafs */ |
#define X86_FEATURE_PERFCTR_CORE ( 6*32+23) /* core performance counter extensions */ |
#define X86_FEATURE_PERFCTR_NB ( 6*32+24) /* NB performance counter extensions */ |
#define X86_FEATURE_BPEXT (6*32+26) /* data breakpoint extension */ |
#define X86_FEATURE_PERFCTR_L2 ( 6*32+28) /* L2 performance counter extensions */ |
#define X86_FEATURE_MWAITX ( 6*32+29) /* MWAIT extension (MONITORX/MWAITX) */ |
/* |
* Auxiliary flags: Linux defined - For features scattered in various |
* CPUID levels like 0x6, 0xA etc, word 7. |
* |
* Reuse free bits when adding new feature flags! |
*/ |
#define X86_FEATURE_CPB ( 7*32+ 2) /* AMD Core Performance Boost */ |
#define X86_FEATURE_EPB ( 7*32+ 3) /* IA32_ENERGY_PERF_BIAS support */ |
#define X86_FEATURE_HW_PSTATE ( 7*32+ 8) /* AMD HW-PState */ |
#define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */ |
#define X86_FEATURE_INTEL_PT ( 7*32+15) /* Intel Processor Trace */ |
/* Virtualization flags: Linux defined, word 8 */ |
#define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */ |
#define X86_FEATURE_VNMI ( 8*32+ 1) /* Intel Virtual NMI */ |
#define X86_FEATURE_FLEXPRIORITY ( 8*32+ 2) /* Intel FlexPriority */ |
#define X86_FEATURE_EPT ( 8*32+ 3) /* Intel Extended Page Table */ |
#define X86_FEATURE_VPID ( 8*32+ 4) /* Intel Virtual Processor ID */ |
#define X86_FEATURE_VMMCALL ( 8*32+15) /* Prefer vmmcall to vmcall */ |
#define X86_FEATURE_XENPV ( 8*32+16) /* "" Xen paravirtual guest */ |
/* Intel-defined CPU features, CPUID level 0x00000007:0 (ebx), word 9 */ |
#define X86_FEATURE_FSGSBASE ( 9*32+ 0) /* {RD/WR}{FS/GS}BASE instructions*/ |
#define X86_FEATURE_TSC_ADJUST ( 9*32+ 1) /* TSC adjustment MSR 0x3b */ |
#define X86_FEATURE_BMI1 ( 9*32+ 3) /* 1st group bit manipulation extensions */ |
#define X86_FEATURE_HLE ( 9*32+ 4) /* Hardware Lock Elision */ |
#define X86_FEATURE_AVX2 ( 9*32+ 5) /* AVX2 instructions */ |
#define X86_FEATURE_SMEP ( 9*32+ 7) /* Supervisor Mode Execution Protection */ |
#define X86_FEATURE_BMI2 ( 9*32+ 8) /* 2nd group bit manipulation extensions */ |
#define X86_FEATURE_ERMS ( 9*32+ 9) /* Enhanced REP MOVSB/STOSB */ |
#define X86_FEATURE_INVPCID ( 9*32+10) /* Invalidate Processor Context ID */ |
#define X86_FEATURE_RTM ( 9*32+11) /* Restricted Transactional Memory */ |
#define X86_FEATURE_CQM ( 9*32+12) /* Cache QoS Monitoring */ |
#define X86_FEATURE_MPX ( 9*32+14) /* Memory Protection Extension */ |
#define X86_FEATURE_AVX512F ( 9*32+16) /* AVX-512 Foundation */ |
#define X86_FEATURE_AVX512DQ ( 9*32+17) /* AVX-512 DQ (Double/Quad granular) Instructions */ |
#define X86_FEATURE_RDSEED ( 9*32+18) /* The RDSEED instruction */ |
#define X86_FEATURE_ADX ( 9*32+19) /* The ADCX and ADOX instructions */ |
#define X86_FEATURE_SMAP ( 9*32+20) /* Supervisor Mode Access Prevention */ |
#define X86_FEATURE_PCOMMIT ( 9*32+22) /* PCOMMIT instruction */ |
#define X86_FEATURE_CLFLUSHOPT ( 9*32+23) /* CLFLUSHOPT instruction */ |
#define X86_FEATURE_CLWB ( 9*32+24) /* CLWB instruction */ |
#define X86_FEATURE_AVX512PF ( 9*32+26) /* AVX-512 Prefetch */ |
#define X86_FEATURE_AVX512ER ( 9*32+27) /* AVX-512 Exponential and Reciprocal */ |
#define X86_FEATURE_AVX512CD ( 9*32+28) /* AVX-512 Conflict Detection */ |
#define X86_FEATURE_SHA_NI ( 9*32+29) /* SHA1/SHA256 Instruction Extensions */ |
#define X86_FEATURE_AVX512BW ( 9*32+30) /* AVX-512 BW (Byte/Word granular) Instructions */ |
#define X86_FEATURE_AVX512VL ( 9*32+31) /* AVX-512 VL (128/256 Vector Length) Extensions */ |
/* Extended state features, CPUID level 0x0000000d:1 (eax), word 10 */ |
#define X86_FEATURE_XSAVEOPT (10*32+ 0) /* XSAVEOPT */ |
#define X86_FEATURE_XSAVEC (10*32+ 1) /* XSAVEC */ |
#define X86_FEATURE_XGETBV1 (10*32+ 2) /* XGETBV with ECX = 1 */ |
#define X86_FEATURE_XSAVES (10*32+ 3) /* XSAVES/XRSTORS */ |
/* Intel-defined CPU QoS Sub-leaf, CPUID level 0x0000000F:0 (edx), word 11 */ |
#define X86_FEATURE_CQM_LLC (11*32+ 1) /* LLC QoS if 1 */ |
/* Intel-defined CPU QoS Sub-leaf, CPUID level 0x0000000F:1 (edx), word 12 */ |
#define X86_FEATURE_CQM_OCCUP_LLC (12*32+ 0) /* LLC occupancy monitoring if 1 */ |
#define X86_FEATURE_CQM_MBM_TOTAL (12*32+ 1) /* LLC Total MBM monitoring */ |
#define X86_FEATURE_CQM_MBM_LOCAL (12*32+ 2) /* LLC Local MBM monitoring */ |
/* AMD-defined CPU features, CPUID level 0x80000008 (ebx), word 13 */ |
#define X86_FEATURE_CLZERO (13*32+0) /* CLZERO instruction */ |
/* Thermal and Power Management Leaf, CPUID level 0x00000006 (eax), word 14 */ |
#define X86_FEATURE_DTHERM (14*32+ 0) /* Digital Thermal Sensor */ |
#define X86_FEATURE_IDA (14*32+ 1) /* Intel Dynamic Acceleration */ |
#define X86_FEATURE_ARAT (14*32+ 2) /* Always Running APIC Timer */ |
#define X86_FEATURE_PLN (14*32+ 4) /* Intel Power Limit Notification */ |
#define X86_FEATURE_PTS (14*32+ 6) /* Intel Package Thermal Status */ |
#define X86_FEATURE_HWP (14*32+ 7) /* Intel Hardware P-states */ |
#define X86_FEATURE_HWP_NOTIFY (14*32+ 8) /* HWP Notification */ |
#define X86_FEATURE_HWP_ACT_WINDOW (14*32+ 9) /* HWP Activity Window */ |
#define X86_FEATURE_HWP_EPP (14*32+10) /* HWP Energy Perf. Preference */ |
#define X86_FEATURE_HWP_PKG_REQ (14*32+11) /* HWP Package Level Request */ |
/* AMD SVM Feature Identification, CPUID level 0x8000000a (edx), word 15 */ |
#define X86_FEATURE_NPT (15*32+ 0) /* Nested Page Table support */ |
#define X86_FEATURE_LBRV (15*32+ 1) /* LBR Virtualization support */ |
#define X86_FEATURE_SVML (15*32+ 2) /* "svm_lock" SVM locking MSR */ |
#define X86_FEATURE_NRIPS (15*32+ 3) /* "nrip_save" SVM next_rip save */ |
#define X86_FEATURE_TSCRATEMSR (15*32+ 4) /* "tsc_scale" TSC scaling support */ |
#define X86_FEATURE_VMCBCLEAN (15*32+ 5) /* "vmcb_clean" VMCB clean bits support */ |
#define X86_FEATURE_FLUSHBYASID (15*32+ 6) /* flush-by-ASID support */ |
#define X86_FEATURE_DECODEASSISTS (15*32+ 7) /* Decode Assists support */ |
#define X86_FEATURE_PAUSEFILTER (15*32+10) /* filtered pause intercept */ |
#define X86_FEATURE_PFTHRESHOLD (15*32+12) /* pause filter threshold */ |
#define X86_FEATURE_AVIC (15*32+13) /* Virtual Interrupt Controller */ |
/* Intel-defined CPU features, CPUID level 0x00000007:0 (ecx), word 16 */ |
#define X86_FEATURE_PKU (16*32+ 3) /* Protection Keys for Userspace */ |
#define X86_FEATURE_OSPKE (16*32+ 4) /* OS Protection Keys Enable */ |
/* |
* BUG word(s) |
*/ |
#define X86_BUG(x) (NCAPINTS*32 + (x)) |
#define X86_BUG_F00F X86_BUG(0) /* Intel F00F */ |
#define X86_BUG_FDIV X86_BUG(1) /* FPU FDIV */ |
#define X86_BUG_COMA X86_BUG(2) /* Cyrix 6x86 coma */ |
#define X86_BUG_AMD_TLB_MMATCH X86_BUG(3) /* "tlb_mmatch" AMD Erratum 383 */ |
#define X86_BUG_AMD_APIC_C1E X86_BUG(4) /* "apic_c1e" AMD Erratum 400 */ |
#define X86_BUG_11AP X86_BUG(5) /* Bad local APIC aka 11AP */ |
#define X86_BUG_FXSAVE_LEAK X86_BUG(6) /* FXSAVE leaks FOP/FIP/FOP */ |
#define X86_BUG_CLFLUSH_MONITOR X86_BUG(7) /* AAI65, CLFLUSH required before MONITOR */ |
#define X86_BUG_SYSRET_SS_ATTRS X86_BUG(8) /* SYSRET doesn't fix up SS attrs */ |
#ifdef CONFIG_X86_32 |
/* |
* 64-bit kernels don't use X86_BUG_ESPFIX. Make the define conditional |
* to avoid confusion. |
*/ |
#define X86_BUG_ESPFIX X86_BUG(9) /* "" IRET to 16-bit SS corrupts ESP/RSP high bits */ |
#endif |
#endif /* _ASM_X86_CPUFEATURES_H */ |
/drivers/include/asm/desc_defs.h |
---|
98,4 → 98,27 |
#endif /* !__ASSEMBLY__ */ |
/* Access rights as returned by LAR */ |
#define AR_TYPE_RODATA (0 * (1 << 9)) |
#define AR_TYPE_RWDATA (1 * (1 << 9)) |
#define AR_TYPE_RODATA_EXPDOWN (2 * (1 << 9)) |
#define AR_TYPE_RWDATA_EXPDOWN (3 * (1 << 9)) |
#define AR_TYPE_XOCODE (4 * (1 << 9)) |
#define AR_TYPE_XRCODE (5 * (1 << 9)) |
#define AR_TYPE_XOCODE_CONF (6 * (1 << 9)) |
#define AR_TYPE_XRCODE_CONF (7 * (1 << 9)) |
#define AR_TYPE_MASK (7 * (1 << 9)) |
#define AR_DPL0 (0 * (1 << 13)) |
#define AR_DPL3 (3 * (1 << 13)) |
#define AR_DPL_MASK (3 * (1 << 13)) |
#define AR_A (1 << 8) /* "Accessed" */ |
#define AR_S (1 << 12) /* If clear, "System" segment */ |
#define AR_P (1 << 15) /* "Present" */ |
#define AR_AVL (1 << 20) /* "AVaiLable" (no HW effect) */ |
#define AR_L (1 << 21) /* "Long mode" for code segments */ |
#define AR_DB (1 << 22) /* D/B, effect depends on type */ |
#define AR_G (1 << 23) /* "Granularity" (limit in pages) */ |
#endif /* _ASM_X86_DESC_DEFS_H */ |
/drivers/include/asm/disabled-features.h |
---|
28,6 → 28,14 |
# define DISABLE_CENTAUR_MCR 0 |
#endif /* CONFIG_X86_64 */ |
#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS |
# define DISABLE_PKU 0 |
# define DISABLE_OSPKE 0 |
#else |
# define DISABLE_PKU (1<<(X86_FEATURE_PKU & 31)) |
# define DISABLE_OSPKE (1<<(X86_FEATURE_OSPKE & 31)) |
#endif /* CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS */ |
/* |
* Make sure to add features to the correct mask |
*/ |
41,5 → 49,12 |
#define DISABLED_MASK7 0 |
#define DISABLED_MASK8 0 |
#define DISABLED_MASK9 (DISABLE_MPX) |
#define DISABLED_MASK10 0 |
#define DISABLED_MASK11 0 |
#define DISABLED_MASK12 0 |
#define DISABLED_MASK13 0 |
#define DISABLED_MASK14 0 |
#define DISABLED_MASK15 0 |
#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE) |
#endif /* _ASM_X86_DISABLED_FEATURES_H */ |
/drivers/include/asm/dma-mapping.h |
---|
46,8 → 46,6 |
#define HAVE_ARCH_DMA_SUPPORTED 1 |
extern int dma_supported(struct device *hwdev, u64 mask); |
#include <asm-generic/dma-mapping-common.h> |
extern void *dma_generic_alloc_coherent(struct device *dev, size_t size, |
dma_addr_t *dma_addr, gfp_t flag, |
struct dma_attrs *attrs); |
/drivers/include/asm/fixmap.h |
---|
138,7 → 138,7 |
extern int fixmaps_set; |
extern pte_t *kmap_pte; |
extern pgprot_t kmap_prot; |
#define kmap_prot PAGE_KERNEL |
extern pte_t *pkmap_page_table; |
void __native_set_fixmap(enum fixed_addresses idx, pte_t pte); |
/drivers/include/asm/fpu/types.h |
---|
108,6 → 108,8 |
XFEATURE_OPMASK, |
XFEATURE_ZMM_Hi256, |
XFEATURE_Hi16_ZMM, |
XFEATURE_PT_UNIMPLEMENTED_SO_FAR, |
XFEATURE_PKRU, |
XFEATURE_MAX, |
}; |
120,6 → 122,7 |
#define XFEATURE_MASK_OPMASK (1 << XFEATURE_OPMASK) |
#define XFEATURE_MASK_ZMM_Hi256 (1 << XFEATURE_ZMM_Hi256) |
#define XFEATURE_MASK_Hi16_ZMM (1 << XFEATURE_Hi16_ZMM) |
#define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU) |
#define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE) |
#define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK \ |
212,6 → 215,15 |
struct reg_512_bit hi16_zmm[16]; |
} __packed; |
/* |
* State component 9: 32-bit PKRU register. The state is |
* 8 bytes long but only 4 bytes is used currently. |
*/ |
struct pkru_state { |
u32 pkru; |
u32 pad; |
} __packed; |
struct xstate_header { |
u64 xfeatures; |
u64 xcomp_bv; |
/drivers/include/asm/io.h |
---|
152,7 → 152,7 |
* If the area you are trying to map is a PCI BAR you should have a |
* look at pci_iomap(). |
*/ |
//extern void __iomem *ioremap_nocache(resource_size_t offset, unsigned long size); |
extern void __iomem *ioremap_nocache(resource_size_t offset, unsigned long size); |
extern void __iomem *ioremap_uc(resource_size_t offset, unsigned long size); |
#define ioremap_uc ioremap_uc |
163,12 → 163,12 |
/* |
* The default ioremap() behavior is non-cached: |
*/ |
//static inline void __iomem *ioremap(resource_size_t offset, unsigned long size) |
//{ |
// return ioremap_nocache(offset, size); |
//} |
static inline void __iomem *ioremap(resource_size_t offset, unsigned long size) |
{ |
return ioremap_nocache(offset, size); |
} |
//extern void iounmap(volatile void __iomem *addr); |
extern void iounmap(volatile void __iomem *addr); |
extern void set_iounmap_nonlazy(void); |
296,7 → 296,7 |
extern int ioremap_change_attr(unsigned long vaddr, unsigned long size, |
enum page_cache_mode pcm); |
//extern void __iomem *ioremap_wc(resource_size_t offset, unsigned long size); |
extern void __iomem *ioremap_wc(resource_size_t offset, unsigned long size); |
extern void __iomem *ioremap_wt(resource_size_t offset, unsigned long size); |
extern bool is_early_ioremap_ptep(pte_t *ptep); |
/drivers/include/asm/iomap.h |
---|
0,0 → 1,40 |
#ifndef _ASM_X86_IOMAP_H |
#define _ASM_X86_IOMAP_H |
/* |
* Copyright © 2008 Ingo Molnar |
* |
* This program is free software; you can redistribute it and/or modify |
* it under the terms of the GNU General Public License as published by |
* the Free Software Foundation; either version 2 of the License, or |
* (at your option) any later version. |
* |
* This program is distributed in the hope that it will be useful, but |
* WITHOUT ANY WARRANTY; without even the implied warranty of |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
* General Public License for more details. |
* |
* You should have received a copy of the GNU General Public License along |
* with this program; if not, write to the Free Software Foundation, Inc., |
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. |
*/ |
#include <linux/fs.h> |
#include <linux/mm.h> |
#include <linux/uaccess.h> |
#include <asm/cacheflush.h> |
#include <asm/pgtable.h> |
void __iomem * |
iomap_atomic_prot_pfn(unsigned long pfn, pgprot_t prot); |
void |
iounmap_atomic(void __iomem *kvaddr); |
int |
iomap_create_wc(resource_size_t base, unsigned long size, pgprot_t *prot); |
void |
iomap_free(resource_size_t base, unsigned long size); |
#endif /* _ASM_X86_IOMAP_H */ |
/drivers/include/asm/msr-index.h |
---|
1,7 → 1,12 |
#ifndef _ASM_X86_MSR_INDEX_H |
#define _ASM_X86_MSR_INDEX_H |
/* CPU model specific register (MSR) numbers */ |
/* |
* CPU model specific register (MSR) numbers. |
* |
* Do not add new entries to this file unless the definitions are shared |
* between multiple compilation units. |
*/ |
/* x86-64 specific MSRs */ |
#define MSR_EFER 0xc0000080 /* extended feature register */ |
162,6 → 167,14 |
#define MSR_PKG_C9_RESIDENCY 0x00000631 |
#define MSR_PKG_C10_RESIDENCY 0x00000632 |
/* Interrupt Response Limit */ |
#define MSR_PKGC3_IRTL 0x0000060a |
#define MSR_PKGC6_IRTL 0x0000060b |
#define MSR_PKGC7_IRTL 0x0000060c |
#define MSR_PKGC8_IRTL 0x00000633 |
#define MSR_PKGC9_IRTL 0x00000634 |
#define MSR_PKGC10_IRTL 0x00000635 |
/* Run Time Average Power Limiting (RAPL) Interface */ |
#define MSR_RAPL_POWER_UNIT 0x00000606 |
185,6 → 198,7 |
#define MSR_PP1_ENERGY_STATUS 0x00000641 |
#define MSR_PP1_POLICY 0x00000642 |
/* Config TDP MSRs */ |
#define MSR_CONFIG_TDP_NOMINAL 0x00000648 |
#define MSR_CONFIG_TDP_LEVEL_1 0x00000649 |
#define MSR_CONFIG_TDP_LEVEL_2 0x0000064A |
205,13 → 219,6 |
#define MSR_GFX_PERF_LIMIT_REASONS 0x000006B0 |
#define MSR_RING_PERF_LIMIT_REASONS 0x000006B1 |
/* Config TDP MSRs */ |
#define MSR_CONFIG_TDP_NOMINAL 0x00000648 |
#define MSR_CONFIG_TDP_LEVEL1 0x00000649 |
#define MSR_CONFIG_TDP_LEVEL2 0x0000064A |
#define MSR_CONFIG_TDP_CONTROL 0x0000064B |
#define MSR_TURBO_ACTIVATION_RATIO 0x0000064C |
/* Hardware P state interface */ |
#define MSR_PPERF 0x0000064e |
#define MSR_PERF_LIMIT_REASONS 0x0000064f |
230,10 → 237,10 |
#define HWP_PACKAGE_LEVEL_REQUEST_BIT (1<<11) |
/* IA32_HWP_CAPABILITIES */ |
#define HWP_HIGHEST_PERF(x) (x & 0xff) |
#define HWP_GUARANTEED_PERF(x) ((x & (0xff << 8)) >>8) |
#define HWP_MOSTEFFICIENT_PERF(x) ((x & (0xff << 16)) >>16) |
#define HWP_LOWEST_PERF(x) ((x & (0xff << 24)) >>24) |
#define HWP_HIGHEST_PERF(x) (((x) >> 0) & 0xff) |
#define HWP_GUARANTEED_PERF(x) (((x) >> 8) & 0xff) |
#define HWP_MOSTEFFICIENT_PERF(x) (((x) >> 16) & 0xff) |
#define HWP_LOWEST_PERF(x) (((x) >> 24) & 0xff) |
/* IA32_HWP_REQUEST */ |
#define HWP_MIN_PERF(x) (x & 0xff) |
/drivers/include/asm/msr.h |
---|
42,14 → 42,6 |
struct saved_msr *array; |
}; |
static inline unsigned long long native_read_tscp(unsigned int *aux) |
{ |
unsigned long low, high; |
asm volatile(".byte 0x0f,0x01,0xf9" |
: "=a" (low), "=d" (high), "=c" (*aux)); |
return low | ((u64)high << 32); |
} |
/* |
* both i386 and x86_64 returns 64-bit value in edx:eax, but gcc's "A" |
* constraint has different meanings. For i386, "A" means exactly |
67,11 → 59,34 |
#define EAX_EDX_RET(val, low, high) "=A" (val) |
#endif |
#ifdef CONFIG_TRACEPOINTS |
/* |
* Be very careful with includes. This header is prone to include loops. |
*/ |
#include <asm/atomic.h> |
#include <linux/tracepoint-defs.h> |
extern struct tracepoint __tracepoint_read_msr; |
extern struct tracepoint __tracepoint_write_msr; |
extern struct tracepoint __tracepoint_rdpmc; |
#define msr_tracepoint_active(t) static_key_false(&(t).key) |
extern void do_trace_write_msr(unsigned msr, u64 val, int failed); |
extern void do_trace_read_msr(unsigned msr, u64 val, int failed); |
extern void do_trace_rdpmc(unsigned msr, u64 val, int failed); |
#else |
#define msr_tracepoint_active(t) false |
static inline void do_trace_write_msr(unsigned msr, u64 val, int failed) {} |
static inline void do_trace_read_msr(unsigned msr, u64 val, int failed) {} |
static inline void do_trace_rdpmc(unsigned msr, u64 val, int failed) {} |
#endif |
static inline unsigned long long native_read_msr(unsigned int msr) |
{ |
DECLARE_ARGS(val, low, high); |
asm volatile("rdmsr" : EAX_EDX_RET(val, low, high) : "c" (msr)); |
if (msr_tracepoint_active(__tracepoint_read_msr)) |
do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), 0); |
return EAX_EDX_VAL(val, low, high); |
} |
88,6 → 103,8 |
_ASM_EXTABLE(2b, 3b) |
: [err] "=r" (*err), EAX_EDX_RET(val, low, high) |
: "c" (msr), [fault] "i" (-EIO)); |
if (msr_tracepoint_active(__tracepoint_read_msr)) |
do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), *err); |
return EAX_EDX_VAL(val, low, high); |
} |
95,6 → 112,8 |
unsigned low, unsigned high) |
{ |
asm volatile("wrmsr" : : "c" (msr), "a"(low), "d" (high) : "memory"); |
if (msr_tracepoint_active(__tracepoint_write_msr)) |
do_trace_write_msr(msr, ((u64)high << 32 | low), 0); |
} |
/* Can be uninlined because referenced by paravirt */ |
112,6 → 131,8 |
: "c" (msr), "0" (low), "d" (high), |
[fault] "i" (-EIO) |
: "memory"); |
if (msr_tracepoint_active(__tracepoint_write_msr)) |
do_trace_write_msr(msr, ((u64)high << 32 | low), err); |
return err; |
} |
136,11 → 157,42 |
return EAX_EDX_VAL(val, low, high); |
} |
/** |
* rdtsc_ordered() - read the current TSC in program order |
* |
* rdtsc_ordered() returns the result of RDTSC as a 64-bit integer. |
* It is ordered like a load to a global in-memory counter. It should |
* be impossible to observe non-monotonic rdtsc_unordered() behavior |
* across multiple CPUs as long as the TSC is synced. |
*/ |
static __always_inline unsigned long long rdtsc_ordered(void) |
{ |
/* |
* The RDTSC instruction is not ordered relative to memory |
* access. The Intel SDM and the AMD APM are both vague on this |
* point, but empirically an RDTSC instruction can be |
* speculatively executed before prior loads. An RDTSC |
* immediately after an appropriate barrier appears to be |
* ordered as a normal load, that is, it provides the same |
* ordering guarantees as reading from a global memory location |
* that some other imaginary CPU is updating continuously with a |
* time stamp. |
*/ |
alternative_2("", "mfence", X86_FEATURE_MFENCE_RDTSC, |
"lfence", X86_FEATURE_LFENCE_RDTSC); |
return rdtsc(); |
} |
/* Deprecated, keep it for a cycle for easier merging: */ |
#define rdtscll(now) do { (now) = rdtsc_ordered(); } while (0) |
static inline unsigned long long native_read_pmc(int counter) |
{ |
DECLARE_ARGS(val, low, high); |
asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter)); |
if (msr_tracepoint_active(__tracepoint_rdpmc)) |
do_trace_rdpmc(counter, EAX_EDX_VAL(val, low, high), 0); |
return EAX_EDX_VAL(val, low, high); |
} |
/drivers/include/asm/pci.h |
---|
20,6 → 20,9 |
#ifdef CONFIG_X86_64 |
void *iommu; /* IOMMU private data */ |
#endif |
#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN |
void *fwnode; /* IRQ domain for MSI assignment */ |
#endif |
}; |
extern int pci_routeirq; |
32,6 → 35,7 |
static inline int pci_domain_nr(struct pci_bus *bus) |
{ |
struct pci_sysdata *sd = bus->sysdata; |
return sd->domain; |
} |
41,6 → 45,17 |
} |
#endif |
#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN |
static inline void *_pci_root_bus_fwnode(struct pci_bus *bus) |
{ |
struct pci_sysdata *sd = bus->sysdata; |
return sd->fwnode; |
} |
#define pci_root_bus_fwnode _pci_root_bus_fwnode |
#endif |
/* Can be used to override the logic in pci_scan_bus for skipping |
already-configured bus numbers - to be used for buggy BIOSes |
or architectures with incomplete PCI setup by the loader */ |
105,9 → 120,6 |
#include <asm/pci_64.h> |
#endif |
/* implement the pci_ DMA API in terms of the generic device dma_ one */ |
#include <asm-generic/pci-dma-compat.h> |
/* generic pci stuff */ |
#include <asm-generic/pci.h> |
/drivers/include/asm/pgtable.h |
---|
487,18 → 487,7 |
#endif |
#define pte_accessible pte_accessible |
static inline bool pte_accessible(struct mm_struct *mm, pte_t a) |
{ |
if (pte_flags(a) & _PAGE_PRESENT) |
return true; |
if ((pte_flags(a) & _PAGE_PROTNONE) && |
mm_tlb_flush_pending(mm)) |
return true; |
return false; |
} |
static inline int pte_hidden(pte_t pte) |
{ |
return pte_flags(pte) & _PAGE_HIDDEN; |
/drivers/include/asm/pgtable_32.h |
---|
14,6 → 14,7 |
*/ |
#ifndef __ASSEMBLY__ |
#include <asm/processor.h> |
#include <asm/fixmap.h> |
#include <linux/threads.h> |
#include <linux/bitops.h> |
/drivers/include/asm/pgtable_types.h |
---|
20,13 → 20,18 |
#define _PAGE_BIT_SOFTW2 10 /* " */ |
#define _PAGE_BIT_SOFTW3 11 /* " */ |
#define _PAGE_BIT_PAT_LARGE 12 /* On 2MB or 1GB pages */ |
#define _PAGE_BIT_SOFTW4 58 /* available for programmer */ |
#define _PAGE_BIT_PKEY_BIT0 59 /* Protection Keys, bit 1/4 */ |
#define _PAGE_BIT_PKEY_BIT1 60 /* Protection Keys, bit 2/4 */ |
#define _PAGE_BIT_PKEY_BIT2 61 /* Protection Keys, bit 3/4 */ |
#define _PAGE_BIT_PKEY_BIT3 62 /* Protection Keys, bit 4/4 */ |
#define _PAGE_BIT_NX 63 /* No execute: only valid after cpuid check */ |
#define _PAGE_BIT_SPECIAL _PAGE_BIT_SOFTW1 |
#define _PAGE_BIT_CPA_TEST _PAGE_BIT_SOFTW1 |
#define _PAGE_BIT_HIDDEN _PAGE_BIT_SOFTW3 /* hidden by kmemcheck */ |
#define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */ |
#define _PAGE_BIT_SOFTW4 58 /* available for programmer */ |
#define _PAGE_BIT_DEVMAP _PAGE_BIT_SOFTW4 |
#define _PAGE_BIT_NX 63 /* No execute: only valid after cpuid check */ |
/* If _PAGE_BIT_PRESENT is clear, we use these: */ |
/* - if the user mapped it with PROT_NONE; pte_present gives true */ |
47,8 → 52,24 |
#define _PAGE_PAT_LARGE (_AT(pteval_t, 1) << _PAGE_BIT_PAT_LARGE) |
#define _PAGE_SPECIAL (_AT(pteval_t, 1) << _PAGE_BIT_SPECIAL) |
#define _PAGE_CPA_TEST (_AT(pteval_t, 1) << _PAGE_BIT_CPA_TEST) |
#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS |
#define _PAGE_PKEY_BIT0 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT0) |
#define _PAGE_PKEY_BIT1 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT1) |
#define _PAGE_PKEY_BIT2 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT2) |
#define _PAGE_PKEY_BIT3 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT3) |
#else |
#define _PAGE_PKEY_BIT0 (_AT(pteval_t, 0)) |
#define _PAGE_PKEY_BIT1 (_AT(pteval_t, 0)) |
#define _PAGE_PKEY_BIT2 (_AT(pteval_t, 0)) |
#define _PAGE_PKEY_BIT3 (_AT(pteval_t, 0)) |
#endif |
#define __HAVE_ARCH_PTE_SPECIAL |
#define _PAGE_PKEY_MASK (_PAGE_PKEY_BIT0 | \ |
_PAGE_PKEY_BIT1 | \ |
_PAGE_PKEY_BIT2 | \ |
_PAGE_PKEY_BIT3) |
#ifdef CONFIG_KMEMCHECK |
#define _PAGE_HIDDEN (_AT(pteval_t, 1) << _PAGE_BIT_HIDDEN) |
#else |
99,7 → 120,12 |
#define _KERNPG_TABLE (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | \ |
_PAGE_DIRTY) |
/* Set of bits not changed in pte_modify */ |
/* |
* Set of bits not changed in pte_modify. The pte's |
* protection key is treated like _PAGE_RW, for |
* instance, and is *not* included in this mask since |
* pte_modify() does modify it. |
*/ |
#define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ |
_PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY | \ |
_PAGE_SOFT_DIRTY) |
215,7 → 241,10 |
/* Extracts the PFN from a (pte|pmd|pud|pgd)val_t of a 4KB page */ |
#define PTE_PFN_MASK ((pteval_t)PHYSICAL_PAGE_MASK) |
/* Extracts the flags from a (pte|pmd|pud|pgd)val_t of a 4KB page */ |
/* |
* Extracts the flags from a (pte|pmd|pud|pgd)val_t |
* This includes the protection key value. |
*/ |
#define PTE_FLAGS_MASK (~PTE_PFN_MASK) |
typedef struct pgprot { pgprotval_t pgprot; } pgprot_t; |
/drivers/include/asm/processor.h |
---|
13,7 → 13,7 |
#include <asm/types.h> |
#include <uapi/asm/sigcontext.h> |
#include <asm/current.h> |
#include <asm/cpufeature.h> |
#include <asm/cpufeatures.h> |
#include <asm/page.h> |
#include <asm/pgtable_types.h> |
#include <asm/percpu.h> |
24,7 → 24,6 |
#include <asm/fpu/types.h> |
#include <linux/personality.h> |
#include <linux/cpumask.h> |
#include <linux/cache.h> |
#include <linux/threads.h> |
#include <linux/math64.h> |
129,10 → 128,10 |
u16 booted_cores; |
/* Physical processor id: */ |
u16 phys_proc_id; |
/* Logical processor id: */ |
u16 logical_proc_id; |
/* Core id: */ |
u16 cpu_core_id; |
/* Compute unit id */ |
u8 compute_unit_id; |
/* Index into per_cpu list: */ |
u16 cpu_index; |
u32 microcode; |
296,10 → 295,13 |
*/ |
unsigned long io_bitmap[IO_BITMAP_LONGS + 1]; |
#ifdef CONFIG_X86_32 |
/* |
* Space for the temporary SYSENTER stack: |
* Space for the temporary SYSENTER stack. |
*/ |
unsigned long SYSENTER_stack_canary; |
unsigned long SYSENTER_stack[64]; |
#endif |
} ____cacheline_aligned; |
660,10 → 662,9 |
*/ |
static inline void prefetch(const void *x) |
{ |
alternative_input(BASE_PREFETCH, |
"prefetchnta (%1)", |
alternative_input(BASE_PREFETCH, "prefetchnta %P1", |
X86_FEATURE_XMM, |
"r" (x)); |
"m" (*(const char *)x)); |
} |
/* |
673,10 → 674,9 |
*/ |
static inline void prefetchw(const void *x) |
{ |
alternative_input(BASE_PREFETCH, |
"prefetchw (%1)", |
X86_FEATURE_3DNOW, |
"r" (x)); |
alternative_input(BASE_PREFETCH, "prefetchw %P1", |
X86_FEATURE_3DNOWPREFETCH, |
"m" (*(const char *)x)); |
} |
static inline void spin_lock_prefetch(const void *x) |
757,7 → 757,7 |
* Return saved PC of a blocked thread. |
* What is this good for? it will be always the scheduler or ret_from_fork. |
*/ |
#define thread_saved_pc(t) (*(unsigned long *)((t)->thread.sp - 8)) |
#define thread_saved_pc(t) READ_ONCE_NOCHECK(*(unsigned long *)((t)->thread.sp - 8)) |
#define task_pt_regs(tsk) ((struct pt_regs *)(tsk)->thread.sp0 - 1) |
extern unsigned long KSTK_ESP(struct task_struct *task); |
/drivers/include/asm/pvclock.h |
---|
65,10 → 65,5 |
} __attribute__((__aligned__(SMP_CACHE_BYTES))); |
#define PVTI_SIZE sizeof(struct pvclock_vsyscall_time_info) |
#define PVCLOCK_VSYSCALL_NR_PAGES (((NR_CPUS-1)/(PAGE_SIZE/PVTI_SIZE))+1) |
int __init pvclock_init_vsyscall(struct pvclock_vsyscall_time_info *i, |
int size); |
struct pvclock_vcpu_time_info *pvclock_get_vsyscall_time_info(int cpu); |
#endif /* _ASM_X86_PVCLOCK_H */ |
/drivers/include/asm/required-features.h |
---|
92,5 → 92,12 |
#define REQUIRED_MASK7 0 |
#define REQUIRED_MASK8 0 |
#define REQUIRED_MASK9 0 |
#define REQUIRED_MASK10 0 |
#define REQUIRED_MASK11 0 |
#define REQUIRED_MASK12 0 |
#define REQUIRED_MASK13 0 |
#define REQUIRED_MASK14 0 |
#define REQUIRED_MASK15 0 |
#define REQUIRED_MASK16 0 |
#endif /* _ASM_X86_REQUIRED_FEATURES_H */ |
/drivers/include/asm/rwsem.h |
---|
25,7 → 25,7 |
* This should be totally fair - if anything is waiting, a process that wants a |
* lock will go to the back of the queue. When the currently active lock is |
* released, if there's a writer at the front of the queue, then that and only |
* that will be woken up; if there's a bunch of consequtive readers at the |
* that will be woken up; if there's a bunch of consecutive readers at the |
* front, then they'll all be woken up, but no other readers will be. |
*/ |
/drivers/include/asm/smap.h |
---|
0,0 → 1,79 |
/* |
* Supervisor Mode Access Prevention support |
* |
* Copyright (C) 2012 Intel Corporation |
* Author: H. Peter Anvin <hpa@linux.intel.com> |
* |
* This program is free software; you can redistribute it and/or |
* modify it under the terms of the GNU General Public License |
* as published by the Free Software Foundation; version 2 |
* of the License. |
*/ |
#ifndef _ASM_X86_SMAP_H |
#define _ASM_X86_SMAP_H |
#include <linux/stringify.h> |
#include <asm/nops.h> |
#include <asm/cpufeatures.h> |
/* "Raw" instruction opcodes */ |
#define __ASM_CLAC .byte 0x0f,0x01,0xca |
#define __ASM_STAC .byte 0x0f,0x01,0xcb |
#ifdef __ASSEMBLY__ |
#include <asm/alternative-asm.h> |
#ifdef CONFIG_X86_SMAP |
#define ASM_CLAC \ |
ALTERNATIVE "", __stringify(__ASM_CLAC), X86_FEATURE_SMAP |
#define ASM_STAC \ |
ALTERNATIVE "", __stringify(__ASM_STAC), X86_FEATURE_SMAP |
#else /* CONFIG_X86_SMAP */ |
#define ASM_CLAC |
#define ASM_STAC |
#endif /* CONFIG_X86_SMAP */ |
#else /* __ASSEMBLY__ */ |
#include <asm/alternative.h> |
#ifdef CONFIG_X86_SMAP |
static __always_inline void clac(void) |
{ |
/* Note: a barrier is implicit in alternative() */ |
alternative("", __stringify(__ASM_CLAC), X86_FEATURE_SMAP); |
} |
static __always_inline void stac(void) |
{ |
/* Note: a barrier is implicit in alternative() */ |
alternative("", __stringify(__ASM_STAC), X86_FEATURE_SMAP); |
} |
/* These macros can be used in asm() statements */ |
#define ASM_CLAC \ |
ALTERNATIVE("", __stringify(__ASM_CLAC), X86_FEATURE_SMAP) |
#define ASM_STAC \ |
ALTERNATIVE("", __stringify(__ASM_STAC), X86_FEATURE_SMAP) |
#else /* CONFIG_X86_SMAP */ |
static inline void clac(void) { } |
static inline void stac(void) { } |
#define ASM_CLAC |
#define ASM_STAC |
#endif /* CONFIG_X86_SMAP */ |
#endif /* __ASSEMBLY__ */ |
#endif /* _ASM_X86_SMAP_H */ |
/drivers/include/asm/special_insns.h |
---|
4,6 → 4,8 |
#ifdef __KERNEL__ |
#include <asm/nops.h> |
static inline void native_clts(void) |
{ |
asm volatile("clts"); |
96,6 → 98,44 |
} |
#endif |
#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS |
static inline u32 __read_pkru(void) |
{ |
u32 ecx = 0; |
u32 edx, pkru; |
/* |
* "rdpkru" instruction. Places PKRU contents in to EAX, |
* clears EDX and requires that ecx=0. |
*/ |
asm volatile(".byte 0x0f,0x01,0xee\n\t" |
: "=a" (pkru), "=d" (edx) |
: "c" (ecx)); |
return pkru; |
} |
static inline void __write_pkru(u32 pkru) |
{ |
u32 ecx = 0, edx = 0; |
/* |
* "wrpkru" instruction. Loads contents in EAX to PKRU, |
* requires that ecx = edx = 0. |
*/ |
asm volatile(".byte 0x0f,0x01,0xef\n\t" |
: : "a" (pkru), "c"(ecx), "d"(edx)); |
} |
#else |
static inline u32 __read_pkru(void) |
{ |
return 0; |
} |
static inline void __write_pkru(u32 pkru) |
{ |
} |
#endif |
static inline void native_wbinvd(void) |
{ |
asm volatile("wbinvd": : :"memory"); |
/drivers/include/asm/timex.h |
---|
1,7 → 1,7 |
#ifndef _ASM_X86_TIMEX_H |
#define _ASM_X86_TIMEX_H |
//#include <asm/processor.h> |
#include <asm/processor.h> |
//#include <asm/tsc.h> |
/* Assume we use the PIT time source for the clock tick */ |
/drivers/include/asm/topology.h |
---|
119,6 → 119,7 |
extern const struct cpumask *cpu_coregroup_mask(int cpu); |
#define topology_logical_package_id(cpu) (cpu_data(cpu).logical_proc_id) |
#define topology_physical_package_id(cpu) (cpu_data(cpu).phys_proc_id) |
#define topology_core_id(cpu) (cpu_data(cpu).cpu_core_id) |
125,6 → 126,16 |
#ifdef ENABLE_TOPO_DEFINES |
#define topology_core_cpumask(cpu) (per_cpu(cpu_core_map, cpu)) |
#define topology_sibling_cpumask(cpu) (per_cpu(cpu_sibling_map, cpu)) |
extern unsigned int __max_logical_packages; |
#define topology_max_packages() (__max_logical_packages) |
int topology_update_package_map(unsigned int apicid, unsigned int cpu); |
extern int topology_phys_to_logical_pkg(unsigned int pkg); |
#else |
#define topology_max_packages() (1) |
static inline int |
topology_update_package_map(unsigned int apicid, unsigned int cpu) { return 0; } |
static inline int topology_phys_to_logical_pkg(unsigned int pkg) { return 0; } |
#endif |
static inline void arch_fix_phys_package_id(int num, u32 slot) |
/drivers/include/asm/uaccess.h |
---|
0,0 → 1,808 |
#ifndef _ASM_X86_UACCESS_H |
#define _ASM_X86_UACCESS_H |
/* |
* User space memory access functions |
*/ |
#include <linux/errno.h> |
#include <linux/compiler.h> |
#include <linux/thread_info.h> |
#include <linux/string.h> |
#include <asm/asm.h> |
#include <asm/page.h> |
#include <asm/smap.h> |
#define VERIFY_READ 0 |
#define VERIFY_WRITE 1 |
/* |
* The fs value determines whether argument validity checking should be |
* performed or not. If get_fs() == USER_DS, checking is performed, with |
* get_fs() == KERNEL_DS, checking is bypassed. |
* |
* For historical reasons, these macros are grossly misnamed. |
*/ |
#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) |
#define KERNEL_DS MAKE_MM_SEG(-1UL) |
#define USER_DS MAKE_MM_SEG(TASK_SIZE_MAX) |
#define get_ds() (KERNEL_DS) |
#define get_fs() (current_thread_info()->addr_limit) |
#define set_fs(x) (current_thread_info()->addr_limit = (x)) |
#define segment_eq(a, b) ((a).seg == (b).seg) |
#define user_addr_max() (current_thread_info()->addr_limit.seg) |
#define __addr_ok(addr) \ |
((unsigned long __force)(addr) < user_addr_max()) |
/* |
* Test whether a block of memory is a valid user space address. |
* Returns 0 if the range is valid, nonzero otherwise. |
*/ |
static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, unsigned long limit) |
{ |
/* |
* If we have used "sizeof()" for the size, |
* we know it won't overflow the limit (but |
* it might overflow the 'addr', so it's |
* important to subtract the size from the |
* limit, not add it to the address). |
*/ |
if (__builtin_constant_p(size)) |
return unlikely(addr > limit - size); |
/* Arbitrary sizes? Be careful about overflow */ |
addr += size; |
if (unlikely(addr < size)) |
return true; |
return unlikely(addr > limit); |
} |
#define __range_not_ok(addr, size, limit) \ |
({ \ |
__chk_user_ptr(addr); \ |
__chk_range_not_ok((unsigned long __force)(addr), size, limit); \ |
}) |
/** |
* access_ok: - Checks if a user space pointer is valid |
* @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that |
* %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe |
* to write to a block, it is always safe to read from it. |
* @addr: User space pointer to start of block to check |
* @size: Size of block to check |
* |
* Context: User context only. This function may sleep if pagefaults are |
* enabled. |
* |
* Checks if a pointer to a block of memory in user space is valid. |
* |
* Returns true (nonzero) if the memory block may be valid, false (zero) |
* if it is definitely invalid. |
* |
* Note that, depending on architecture, this function probably just |
* checks that the pointer is in the user space range - after calling |
* this function, memory access functions may still return -EFAULT. |
*/ |
#define access_ok(type, addr, size) \ |
likely(!__range_not_ok(addr, size, user_addr_max())) |
/* |
* The exception table consists of triples of addresses relative to the |
* exception table entry itself. The first address is of an instruction |
* that is allowed to fault, the second is the target at which the program |
* should continue. The third is a handler function to deal with the fault |
* caused by the instruction in the first field. |
* |
* All the routines below use bits of fixup code that are out of line |
* with the main instruction path. This means when everything is well, |
* we don't even have to jump over them. Further, they do not intrude |
* on our cache or tlb entries. |
*/ |
struct exception_table_entry { |
int insn, fixup, handler; |
}; |
#define ARCH_HAS_RELATIVE_EXTABLE |
#define swap_ex_entry_fixup(a, b, tmp, delta) \ |
do { \ |
(a)->fixup = (b)->fixup + (delta); \ |
(b)->fixup = (tmp).fixup - (delta); \ |
(a)->handler = (b)->handler + (delta); \ |
(b)->handler = (tmp).handler - (delta); \ |
} while (0) |
extern int fixup_exception(struct pt_regs *regs, int trapnr); |
extern bool ex_has_fault_handler(unsigned long ip); |
extern int early_fixup_exception(unsigned long *ip); |
/* |
* These are the main single-value transfer routines. They automatically |
* use the right size if we just have the right pointer type. |
* |
* This gets kind of ugly. We want to return _two_ values in "get_user()" |
* and yet we don't want to do any pointers, because that is too much |
* of a performance impact. Thus we have a few rather ugly macros here, |
* and hide all the ugliness from the user. |
* |
* The "__xxx" versions of the user access functions are versions that |
* do not verify the address space, that must have been done previously |
* with a separate "access_ok()" call (this is used when we do multiple |
* accesses to the same area of user memory). |
*/ |
extern int __get_user_1(void); |
extern int __get_user_2(void); |
extern int __get_user_4(void); |
extern int __get_user_8(void); |
extern int __get_user_bad(void); |
#define __uaccess_begin() stac() |
#define __uaccess_end() clac() |
/* |
* This is a type: either unsigned long, if the argument fits into |
* that type, or otherwise unsigned long long. |
*/ |
#define __inttype(x) \ |
__typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) |
/** |
* get_user: - Get a simple variable from user space. |
* @x: Variable to store result. |
* @ptr: Source address, in user space. |
* |
* Context: User context only. This function may sleep if pagefaults are |
* enabled. |
* |
* This macro copies a single simple variable from user space to kernel |
* space. It supports simple types like char and int, but not larger |
* data types like structures or arrays. |
* |
* @ptr must have pointer-to-simple-variable type, and the result of |
* dereferencing @ptr must be assignable to @x without a cast. |
* |
* Returns zero on success, or -EFAULT on error. |
* On error, the variable @x is set to zero. |
*/ |
/* |
* Careful: we have to cast the result to the type of the pointer |
* for sign reasons. |
* |
* The use of _ASM_DX as the register specifier is a bit of a |
* simplification, as gcc only cares about it as the starting point |
* and not size: for a 64-bit value it will use %ecx:%edx on 32 bits |
* (%ecx being the next register in gcc's x86 register sequence), and |
* %rdx on 64 bits. |
* |
* Clang/LLVM cares about the size of the register, but still wants |
* the base register for something that ends up being a pair. |
*/ |
#define get_user(x, ptr) \ |
({ \ |
int __ret_gu; \ |
register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX); \ |
register void *__sp asm(_ASM_SP); \ |
__chk_user_ptr(ptr); \ |
might_fault(); \ |
asm volatile("call __get_user_%P4" \ |
: "=a" (__ret_gu), "=r" (__val_gu), "+r" (__sp) \ |
: "0" (ptr), "i" (sizeof(*(ptr)))); \ |
(x) = (__force __typeof__(*(ptr))) __val_gu; \ |
__builtin_expect(__ret_gu, 0); \ |
}) |
#define __put_user_x(size, x, ptr, __ret_pu) \ |
asm volatile("call __put_user_" #size : "=a" (__ret_pu) \ |
: "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") |
#ifdef CONFIG_X86_32 |
#define __put_user_asm_u64(x, addr, err, errret) \ |
asm volatile("\n" \ |
"1: movl %%eax,0(%2)\n" \ |
"2: movl %%edx,4(%2)\n" \ |
"3:" \ |
".section .fixup,\"ax\"\n" \ |
"4: movl %3,%0\n" \ |
" jmp 3b\n" \ |
".previous\n" \ |
_ASM_EXTABLE(1b, 4b) \ |
_ASM_EXTABLE(2b, 4b) \ |
: "=r" (err) \ |
: "A" (x), "r" (addr), "i" (errret), "0" (err)) |
#define __put_user_asm_ex_u64(x, addr) \ |
asm volatile("\n" \ |
"1: movl %%eax,0(%1)\n" \ |
"2: movl %%edx,4(%1)\n" \ |
"3:" \ |
_ASM_EXTABLE_EX(1b, 2b) \ |
_ASM_EXTABLE_EX(2b, 3b) \ |
: : "A" (x), "r" (addr)) |
#define __put_user_x8(x, ptr, __ret_pu) \ |
asm volatile("call __put_user_8" : "=a" (__ret_pu) \ |
: "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") |
#else |
#define __put_user_asm_u64(x, ptr, retval, errret) \ |
__put_user_asm(x, ptr, retval, "q", "", "er", errret) |
#define __put_user_asm_ex_u64(x, addr) \ |
__put_user_asm_ex(x, addr, "q", "", "er") |
#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu) |
#endif |
extern void __put_user_bad(void); |
/* |
* Strange magic calling convention: pointer in %ecx, |
* value in %eax(:%edx), return value in %eax. clobbers %rbx |
*/ |
extern void __put_user_1(void); |
extern void __put_user_2(void); |
extern void __put_user_4(void); |
extern void __put_user_8(void); |
/** |
* put_user: - Write a simple value into user space. |
* @x: Value to copy to user space. |
* @ptr: Destination address, in user space. |
* |
* Context: User context only. This function may sleep if pagefaults are |
* enabled. |
* |
* This macro copies a single simple value from kernel space to user |
* space. It supports simple types like char and int, but not larger |
* data types like structures or arrays. |
* |
* @ptr must have pointer-to-simple-variable type, and @x must be assignable |
* to the result of dereferencing @ptr. |
* |
* Returns zero on success, or -EFAULT on error. |
*/ |
#define put_user(x, ptr) \ |
({ \ |
int __ret_pu; \ |
__typeof__(*(ptr)) __pu_val; \ |
__chk_user_ptr(ptr); \ |
might_fault(); \ |
__pu_val = x; \ |
switch (sizeof(*(ptr))) { \ |
case 1: \ |
__put_user_x(1, __pu_val, ptr, __ret_pu); \ |
break; \ |
case 2: \ |
__put_user_x(2, __pu_val, ptr, __ret_pu); \ |
break; \ |
case 4: \ |
__put_user_x(4, __pu_val, ptr, __ret_pu); \ |
break; \ |
case 8: \ |
__put_user_x8(__pu_val, ptr, __ret_pu); \ |
break; \ |
default: \ |
__put_user_x(X, __pu_val, ptr, __ret_pu); \ |
break; \ |
} \ |
__builtin_expect(__ret_pu, 0); \ |
}) |
#define __put_user_size(x, ptr, size, retval, errret) \ |
do { \ |
retval = 0; \ |
__chk_user_ptr(ptr); \ |
switch (size) { \ |
case 1: \ |
__put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \ |
break; \ |
case 2: \ |
__put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \ |
break; \ |
case 4: \ |
__put_user_asm(x, ptr, retval, "l", "k", "ir", errret); \ |
break; \ |
case 8: \ |
__put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval, \ |
errret); \ |
break; \ |
default: \ |
__put_user_bad(); \ |
} \ |
} while (0) |
/* |
* This doesn't do __uaccess_begin/end - the exception handling |
* around it must do that. |
*/ |
#define __put_user_size_ex(x, ptr, size) \ |
do { \ |
__chk_user_ptr(ptr); \ |
switch (size) { \ |
case 1: \ |
__put_user_asm_ex(x, ptr, "b", "b", "iq"); \ |
break; \ |
case 2: \ |
__put_user_asm_ex(x, ptr, "w", "w", "ir"); \ |
break; \ |
case 4: \ |
__put_user_asm_ex(x, ptr, "l", "k", "ir"); \ |
break; \ |
case 8: \ |
__put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr); \ |
break; \ |
default: \ |
__put_user_bad(); \ |
} \ |
} while (0) |
#ifdef CONFIG_X86_32 |
#define __get_user_asm_u64(x, ptr, retval, errret) (x) = __get_user_bad() |
#define __get_user_asm_ex_u64(x, ptr) (x) = __get_user_bad() |
#else |
#define __get_user_asm_u64(x, ptr, retval, errret) \ |
__get_user_asm(x, ptr, retval, "q", "", "=r", errret) |
#define __get_user_asm_ex_u64(x, ptr) \ |
__get_user_asm_ex(x, ptr, "q", "", "=r") |
#endif |
#define __get_user_size(x, ptr, size, retval, errret) \ |
do { \ |
retval = 0; \ |
__chk_user_ptr(ptr); \ |
switch (size) { \ |
case 1: \ |
__get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \ |
break; \ |
case 2: \ |
__get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \ |
break; \ |
case 4: \ |
__get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \ |
break; \ |
case 8: \ |
__get_user_asm_u64(x, ptr, retval, errret); \ |
break; \ |
default: \ |
(x) = __get_user_bad(); \ |
} \ |
} while (0) |
#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \ |
asm volatile("\n" \ |
"1: mov"itype" %2,%"rtype"1\n" \ |
"2:\n" \ |
".section .fixup,\"ax\"\n" \ |
"3: mov %3,%0\n" \ |
" xor"itype" %"rtype"1,%"rtype"1\n" \ |
" jmp 2b\n" \ |
".previous\n" \ |
_ASM_EXTABLE(1b, 3b) \ |
: "=r" (err), ltype(x) \ |
: "m" (__m(addr)), "i" (errret), "0" (err)) |
/* |
* This doesn't do __uaccess_begin/end - the exception handling |
* around it must do that. |
*/ |
#define __get_user_size_ex(x, ptr, size) \ |
do { \ |
__chk_user_ptr(ptr); \ |
switch (size) { \ |
case 1: \ |
__get_user_asm_ex(x, ptr, "b", "b", "=q"); \ |
break; \ |
case 2: \ |
__get_user_asm_ex(x, ptr, "w", "w", "=r"); \ |
break; \ |
case 4: \ |
__get_user_asm_ex(x, ptr, "l", "k", "=r"); \ |
break; \ |
case 8: \ |
__get_user_asm_ex_u64(x, ptr); \ |
break; \ |
default: \ |
(x) = __get_user_bad(); \ |
} \ |
} while (0) |
#define __get_user_asm_ex(x, addr, itype, rtype, ltype) \ |
asm volatile("1: mov"itype" %1,%"rtype"0\n" \ |
"2:\n" \ |
_ASM_EXTABLE_EX(1b, 2b) \ |
: ltype(x) : "m" (__m(addr))) |
#define __put_user_nocheck(x, ptr, size) \ |
({ \ |
int __pu_err; \ |
__uaccess_begin(); \ |
__put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \ |
__uaccess_end(); \ |
__builtin_expect(__pu_err, 0); \ |
}) |
#define __get_user_nocheck(x, ptr, size) \ |
({ \ |
int __gu_err; \ |
unsigned long __gu_val; \ |
__uaccess_begin(); \ |
__get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \ |
__uaccess_end(); \ |
(x) = (__force __typeof__(*(ptr)))__gu_val; \ |
__builtin_expect(__gu_err, 0); \ |
}) |
/* FIXME: this hack is definitely wrong -AK */ |
struct __large_struct { unsigned long buf[100]; }; |
#define __m(x) (*(struct __large_struct __user *)(x)) |
/* |
* Tell gcc we read from memory instead of writing: this is because |
* we do not write to any memory gcc knows about, so there are no |
* aliasing issues. |
*/ |
#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \ |
asm volatile("\n" \ |
"1: mov"itype" %"rtype"1,%2\n" \ |
"2:\n" \ |
".section .fixup,\"ax\"\n" \ |
"3: mov %3,%0\n" \ |
" jmp 2b\n" \ |
".previous\n" \ |
_ASM_EXTABLE(1b, 3b) \ |
: "=r"(err) \ |
: ltype(x), "m" (__m(addr)), "i" (errret), "0" (err)) |
#define __put_user_asm_ex(x, addr, itype, rtype, ltype) \ |
asm volatile("1: mov"itype" %"rtype"0,%1\n" \ |
"2:\n" \ |
_ASM_EXTABLE_EX(1b, 2b) \ |
: : ltype(x), "m" (__m(addr))) |
/* |
* uaccess_try and catch |
*/ |
#define uaccess_try do { \ |
current_thread_info()->uaccess_err = 0; \ |
__uaccess_begin(); \ |
barrier(); |
#define uaccess_catch(err) \ |
__uaccess_end(); \ |
(err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0); \ |
} while (0) |
/** |
* __get_user: - Get a simple variable from user space, with less checking. |
* @x: Variable to store result. |
* @ptr: Source address, in user space. |
* |
* Context: User context only. This function may sleep if pagefaults are |
* enabled. |
* |
* This macro copies a single simple variable from user space to kernel |
* space. It supports simple types like char and int, but not larger |
* data types like structures or arrays. |
* |
* @ptr must have pointer-to-simple-variable type, and the result of |
* dereferencing @ptr must be assignable to @x without a cast. |
* |
* Caller must check the pointer with access_ok() before calling this |
* function. |
* |
* Returns zero on success, or -EFAULT on error. |
* On error, the variable @x is set to zero. |
*/ |
#define __get_user(x, ptr) \ |
__get_user_nocheck((x), (ptr), sizeof(*(ptr))) |
/** |
* __put_user: - Write a simple value into user space, with less checking. |
* @x: Value to copy to user space. |
* @ptr: Destination address, in user space. |
* |
* Context: User context only. This function may sleep if pagefaults are |
* enabled. |
* |
* This macro copies a single simple value from kernel space to user |
* space. It supports simple types like char and int, but not larger |
* data types like structures or arrays. |
* |
* @ptr must have pointer-to-simple-variable type, and @x must be assignable |
* to the result of dereferencing @ptr. |
* |
* Caller must check the pointer with access_ok() before calling this |
* function. |
* |
* Returns zero on success, or -EFAULT on error. |
*/ |
#define __put_user(x, ptr) \ |
__put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) |
#define __get_user_unaligned __get_user |
#define __put_user_unaligned __put_user |
/* |
* {get|put}_user_try and catch |
* |
* get_user_try { |
* get_user_ex(...); |
* } get_user_catch(err) |
*/ |
#define get_user_try uaccess_try |
#define get_user_catch(err) uaccess_catch(err) |
#define get_user_ex(x, ptr) do { \ |
unsigned long __gue_val; \ |
__get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr)))); \ |
(x) = (__force __typeof__(*(ptr)))__gue_val; \ |
} while (0) |
#define put_user_try uaccess_try |
#define put_user_catch(err) uaccess_catch(err) |
#define put_user_ex(x, ptr) \ |
__put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) |
extern unsigned long |
copy_from_user_nmi(void *to, const void __user *from, unsigned long n); |
extern __must_check long |
strncpy_from_user(char *dst, const char __user *src, long count); |
extern __must_check long strlen_user(const char __user *str); |
extern __must_check long strnlen_user(const char __user *str, long n); |
unsigned long __must_check clear_user(void __user *mem, unsigned long len); |
unsigned long __must_check __clear_user(void __user *mem, unsigned long len); |
extern void __cmpxchg_wrong_size(void) |
__compiletime_error("Bad argument size for cmpxchg"); |
#define __user_atomic_cmpxchg_inatomic(uval, ptr, old, new, size) \ |
({ \ |
int __ret = 0; \ |
__typeof__(ptr) __uval = (uval); \ |
__typeof__(*(ptr)) __old = (old); \ |
__typeof__(*(ptr)) __new = (new); \ |
__uaccess_begin(); \ |
switch (size) { \ |
case 1: \ |
{ \ |
asm volatile("\n" \ |
"1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n" \ |
"2:\n" \ |
"\t.section .fixup, \"ax\"\n" \ |
"3:\tmov %3, %0\n" \ |
"\tjmp 2b\n" \ |
"\t.previous\n" \ |
_ASM_EXTABLE(1b, 3b) \ |
: "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ |
: "i" (-EFAULT), "q" (__new), "1" (__old) \ |
: "memory" \ |
); \ |
break; \ |
} \ |
case 2: \ |
{ \ |
asm volatile("\n" \ |
"1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n" \ |
"2:\n" \ |
"\t.section .fixup, \"ax\"\n" \ |
"3:\tmov %3, %0\n" \ |
"\tjmp 2b\n" \ |
"\t.previous\n" \ |
_ASM_EXTABLE(1b, 3b) \ |
: "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ |
: "i" (-EFAULT), "r" (__new), "1" (__old) \ |
: "memory" \ |
); \ |
break; \ |
} \ |
case 4: \ |
{ \ |
asm volatile("\n" \ |
"1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" \ |
"2:\n" \ |
"\t.section .fixup, \"ax\"\n" \ |
"3:\tmov %3, %0\n" \ |
"\tjmp 2b\n" \ |
"\t.previous\n" \ |
_ASM_EXTABLE(1b, 3b) \ |
: "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ |
: "i" (-EFAULT), "r" (__new), "1" (__old) \ |
: "memory" \ |
); \ |
break; \ |
} \ |
case 8: \ |
{ \ |
if (!IS_ENABLED(CONFIG_X86_64)) \ |
__cmpxchg_wrong_size(); \ |
\ |
asm volatile("\n" \ |
"1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n" \ |
"2:\n" \ |
"\t.section .fixup, \"ax\"\n" \ |
"3:\tmov %3, %0\n" \ |
"\tjmp 2b\n" \ |
"\t.previous\n" \ |
_ASM_EXTABLE(1b, 3b) \ |
: "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \ |
: "i" (-EFAULT), "r" (__new), "1" (__old) \ |
: "memory" \ |
); \ |
break; \ |
} \ |
default: \ |
__cmpxchg_wrong_size(); \ |
} \ |
__uaccess_end(); \ |
*__uval = __old; \ |
__ret; \ |
}) |
#define user_atomic_cmpxchg_inatomic(uval, ptr, old, new) \ |
({ \ |
access_ok(VERIFY_WRITE, (ptr), sizeof(*(ptr))) ? \ |
__user_atomic_cmpxchg_inatomic((uval), (ptr), \ |
(old), (new), sizeof(*(ptr))) : \ |
-EFAULT; \ |
}) |
/* |
* movsl can be slow when source and dest are not both 8-byte aligned |
*/ |
#ifdef CONFIG_X86_INTEL_USERCOPY |
extern struct movsl_mask { |
int mask; |
} ____cacheline_aligned_in_smp movsl_mask; |
#endif |
#define ARCH_HAS_NOCACHE_UACCESS 1 |
#ifdef CONFIG_X86_32 |
# include <asm/uaccess_32.h> |
#else |
# include <asm/uaccess_64.h> |
#endif |
unsigned long __must_check _copy_from_user(void *to, const void __user *from, |
unsigned n); |
unsigned long __must_check _copy_to_user(void __user *to, const void *from, |
unsigned n); |
#ifdef CONFIG_DEBUG_STRICT_USER_COPY_CHECKS |
# define copy_user_diag __compiletime_error |
#else |
# define copy_user_diag __compiletime_warning |
#endif |
extern void copy_user_diag("copy_from_user() buffer size is too small") |
copy_from_user_overflow(void); |
extern void copy_user_diag("copy_to_user() buffer size is too small") |
copy_to_user_overflow(void) __asm__("copy_from_user_overflow"); |
#undef copy_user_diag |
#ifdef CONFIG_DEBUG_STRICT_USER_COPY_CHECKS |
extern void |
__compiletime_warning("copy_from_user() buffer size is not provably correct") |
__copy_from_user_overflow(void) __asm__("copy_from_user_overflow"); |
#define __copy_from_user_overflow(size, count) __copy_from_user_overflow() |
extern void |
__compiletime_warning("copy_to_user() buffer size is not provably correct") |
__copy_to_user_overflow(void) __asm__("copy_from_user_overflow"); |
#define __copy_to_user_overflow(size, count) __copy_to_user_overflow() |
#else |
static inline void |
__copy_from_user_overflow(int size, unsigned long count) |
{ |
WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count); |
} |
#define __copy_to_user_overflow __copy_from_user_overflow |
#endif |
static inline unsigned long __must_check |
copy_from_user(void *to, const void __user *from, unsigned long n) |
{ |
int sz = __compiletime_object_size(to); |
might_fault(); |
/* |
* While we would like to have the compiler do the checking for us |
* even in the non-constant size case, any false positives there are |
* a problem (especially when DEBUG_STRICT_USER_COPY_CHECKS, but even |
* without - the [hopefully] dangerous looking nature of the warning |
* would make people go look at the respecitive call sites over and |
* over again just to find that there's no problem). |
* |
* And there are cases where it's just not realistic for the compiler |
* to prove the count to be in range. For example when multiple call |
* sites of a helper function - perhaps in different source files - |
* all doing proper range checking, yet the helper function not doing |
* so again. |
* |
* Therefore limit the compile time checking to the constant size |
* case, and do only runtime checking for non-constant sizes. |
*/ |
if (likely(sz < 0 || sz >= n)) |
n = _copy_from_user(to, from, n); |
else if(__builtin_constant_p(n)) |
copy_from_user_overflow(); |
else |
__copy_from_user_overflow(sz, n); |
return n; |
} |
static inline unsigned long __must_check |
copy_to_user(void __user *to, const void *from, unsigned long n) |
{ |
int sz = __compiletime_object_size(from); |
might_fault(); |
/* See the comment in copy_from_user() above. */ |
if (likely(sz < 0 || sz >= n)) |
n = _copy_to_user(to, from, n); |
else if(__builtin_constant_p(n)) |
copy_to_user_overflow(); |
else |
__copy_to_user_overflow(sz, n); |
return n; |
} |
#undef __copy_from_user_overflow |
#undef __copy_to_user_overflow |
/* |
* We rely on the nested NMI work to allow atomic faults from the NMI path; the |
* nested NMI paths are careful to preserve CR2. |
* |
* Caller must use pagefault_enable/disable, or run in interrupt context, |
* and also do a uaccess_ok() check |
*/ |
#define __copy_from_user_nmi __copy_from_user_inatomic |
/* |
* The "unsafe" user accesses aren't really "unsafe", but the naming |
* is a big fat warning: you have to not only do the access_ok() |
* checking before using them, but you have to surround them with the |
* user_access_begin/end() pair. |
*/ |
#define user_access_begin() __uaccess_begin() |
#define user_access_end() __uaccess_end() |
#define unsafe_put_user(x, ptr) \ |
({ \ |
int __pu_err; \ |
__put_user_size((x), (ptr), sizeof(*(ptr)), __pu_err, -EFAULT); \ |
__builtin_expect(__pu_err, 0); \ |
}) |
#define unsafe_get_user(x, ptr) \ |
({ \ |
int __gu_err; \ |
unsigned long __gu_val; \ |
__get_user_size(__gu_val, (ptr), sizeof(*(ptr)), __gu_err, -EFAULT); \ |
(x) = (__force __typeof__(*(ptr)))__gu_val; \ |
__builtin_expect(__gu_err, 0); \ |
}) |
#endif /* _ASM_X86_UACCESS_H */ |
/drivers/include/asm/uaccess_32.h |
---|
0,0 → 1,208 |
#ifndef _ASM_X86_UACCESS_32_H |
#define _ASM_X86_UACCESS_32_H |
/* |
* User space memory access functions |
*/ |
#include <linux/errno.h> |
#include <linux/thread_info.h> |
#include <linux/string.h> |
#include <asm/asm.h> |
#include <asm/page.h> |
unsigned long __must_check __copy_to_user_ll |
(void __user *to, const void *from, unsigned long n); |
unsigned long __must_check __copy_from_user_ll |
(void *to, const void __user *from, unsigned long n); |
unsigned long __must_check __copy_from_user_ll_nozero |
(void *to, const void __user *from, unsigned long n); |
unsigned long __must_check __copy_from_user_ll_nocache |
(void *to, const void __user *from, unsigned long n); |
unsigned long __must_check __copy_from_user_ll_nocache_nozero |
(void *to, const void __user *from, unsigned long n); |
/** |
* __copy_to_user_inatomic: - Copy a block of data into user space, with less checking. |
* @to: Destination address, in user space. |
* @from: Source address, in kernel space. |
* @n: Number of bytes to copy. |
* |
* Context: User context only. |
* |
* Copy data from kernel space to user space. Caller must check |
* the specified block with access_ok() before calling this function. |
* The caller should also make sure he pins the user space address |
* so that we don't result in page fault and sleep. |
* |
* Here we special-case 1, 2 and 4-byte copy_*_user invocations. On a fault |
* we return the initial request size (1, 2 or 4), as copy_*_user should do. |
* If a store crosses a page boundary and gets a fault, the x86 will not write |
* anything, so this is accurate. |
*/ |
static __always_inline unsigned long __must_check |
__copy_to_user_inatomic(void __user *to, const void *from, unsigned long n) |
{ |
if (__builtin_constant_p(n)) { |
switch(n) { |
case 1: |
*(u8 __force *)to = *(u8 *)from; |
return 0; |
case 2: |
*(u16 __force *)to = *(u16 *)from; |
return 0; |
case 4: |
*(u32 __force *)to = *(u32 *)from; |
return 0; |
case 8: |
*(u64 __force *)to = *(u64 *)from; |
return 0; |
default: |
break; |
} |
} |
__builtin_memcpy((void __force *)to, from, n); |
return 0; |
} |
/** |
* __copy_to_user: - Copy a block of data into user space, with less checking. |
* @to: Destination address, in user space. |
* @from: Source address, in kernel space. |
* @n: Number of bytes to copy. |
* |
* Context: User context only. This function may sleep if pagefaults are |
* enabled. |
* |
* Copy data from kernel space to user space. Caller must check |
* the specified block with access_ok() before calling this function. |
* |
* Returns number of bytes that could not be copied. |
* On success, this will be zero. |
*/ |
static __always_inline unsigned long __must_check |
__copy_to_user(void __user *to, const void *from, unsigned long n) |
{ |
might_fault(); |
return __copy_to_user_inatomic(to, from, n); |
} |
static __always_inline unsigned long |
__copy_from_user_inatomic(void *to, const void __user *from, unsigned long n) |
{ |
/* Avoid zeroing the tail if the copy fails.. |
* If 'n' is constant and 1, 2, or 4, we do still zero on a failure, |
* but as the zeroing behaviour is only significant when n is not |
* constant, that shouldn't be a problem. |
*/ |
if (__builtin_constant_p(n)) { |
switch(n) { |
case 1: |
*(u8 *)to = *(u8 __force *)from; |
return 0; |
case 2: |
*(u16 *)to = *(u16 __force *)from; |
return 0; |
case 4: |
*(u32 *)to = *(u32 __force *)from; |
return 0; |
case 8: |
*(u64 *)to = *(u64 __force *)from; |
return 0; |
default: |
break; |
} |
} |
__builtin_memcpy(to, (const void __force *)from, n); |
return 0; |
} |
/** |
* __copy_from_user: - Copy a block of data from user space, with less checking. |
* @to: Destination address, in kernel space. |
* @from: Source address, in user space. |
* @n: Number of bytes to copy. |
* |
* Context: User context only. This function may sleep if pagefaults are |
* enabled. |
* |
* Copy data from user space to kernel space. Caller must check |
* the specified block with access_ok() before calling this function. |
* |
* Returns number of bytes that could not be copied. |
* On success, this will be zero. |
* |
* If some data could not be copied, this function will pad the copied |
* data to the requested size using zero bytes. |
* |
* An alternate version - __copy_from_user_inatomic() - may be called from |
* atomic context and will fail rather than sleep. In this case the |
* uncopied bytes will *NOT* be padded with zeros. See fs/filemap.h |
* for explanation of why this is needed. |
*/ |
static __always_inline unsigned long |
__copy_from_user(void *to, const void __user *from, unsigned long n) |
{ |
might_fault(); |
if (__builtin_constant_p(n)) { |
switch(n) { |
case 1: |
*(u8 *)to = *(u8 __force *)from; |
return 0; |
case 2: |
*(u16 *)to = *(u16 __force *)from; |
return 0; |
case 4: |
*(u32 *)to = *(u32 __force *)from; |
return 0; |
case 8: |
*(u64 *)to = *(u64 __force *)from; |
return 0; |
default: |
break; |
} |
} |
__builtin_memcpy(to, (const void __force *)from, n); |
return 0; |
} |
static __always_inline unsigned long __copy_from_user_nocache(void *to, |
const void __user *from, unsigned long n) |
{ |
might_fault(); |
if (__builtin_constant_p(n)) { |
switch(n) { |
case 1: |
*(u8 *)to = *(u8 __force *)from; |
return 0; |
case 2: |
*(u16 *)to = *(u16 __force *)from; |
return 0; |
case 4: |
*(u32 *)to = *(u32 __force *)from; |
return 0; |
default: |
break; |
} |
} |
__builtin_memcpy(to, (const void __force *)from, n); |
return 0; |
} |
static __always_inline unsigned long |
__copy_from_user_inatomic_nocache(void *to, const void __user *from, |
unsigned long n) |
{ |
return __copy_from_user_inatomic(to, from, n); |
} |
#endif /* _ASM_X86_UACCESS_32_H */ |