Rev 5270 | Rev 6588 | Go to most recent revision | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 5270 | Rev 6082 | ||
---|---|---|---|
Line 2... | Line 2... | ||
2 | #define _ASM_X86_PGTABLE_DEFS_H |
2 | #define _ASM_X86_PGTABLE_DEFS_H |
Line 3... | Line 3... | ||
3 | 3 | ||
4 | #include |
4 | #include |
Line 5... | Line 5... | ||
5 | #include |
5 | #include |
Line 6... | Line 6... | ||
6 | 6 | ||
7 | #define FIRST_USER_ADDRESS 0 |
7 | #define FIRST_USER_ADDRESS 0UL |
8 | 8 | ||
9 | #define _PAGE_BIT_PRESENT 0 /* is present */ |
9 | #define _PAGE_BIT_PRESENT 0 /* is present */ |
Line 25... | Line 25... | ||
25 | #define _PAGE_BIT_SPLITTING _PAGE_BIT_SOFTW2 /* only valid on a PSE pmd */ |
25 | #define _PAGE_BIT_SPLITTING _PAGE_BIT_SOFTW2 /* only valid on a PSE pmd */ |
26 | #define _PAGE_BIT_HIDDEN _PAGE_BIT_SOFTW3 /* hidden by kmemcheck */ |
26 | #define _PAGE_BIT_HIDDEN _PAGE_BIT_SOFTW3 /* hidden by kmemcheck */ |
27 | #define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */ |
27 | #define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */ |
28 | #define _PAGE_BIT_NX 63 /* No execute: only valid after cpuid check */ |
28 | #define _PAGE_BIT_NX 63 /* No execute: only valid after cpuid check */ |
Line 29... | Line -... | ||
29 | - | ||
30 | /* |
- | |
31 | * Swap offsets on configurations that allow automatic NUMA balancing use the |
- | |
32 | * bits after _PAGE_BIT_GLOBAL. To uniquely distinguish NUMA hinting PTEs from |
- | |
33 | * swap entries, we use the first bit after _PAGE_BIT_GLOBAL and shrink the |
- | |
34 | * maximum possible swap space from 16TB to 8TB. |
- | |
35 | */ |
- | |
36 | #define _PAGE_BIT_NUMA (_PAGE_BIT_GLOBAL+1) |
- | |
37 | 29 | ||
38 | /* If _PAGE_BIT_PRESENT is clear, we use these: */ |
30 | /* If _PAGE_BIT_PRESENT is clear, we use these: */ |
39 | /* - if the user mapped it with PROT_NONE; pte_present gives true */ |
31 | /* - if the user mapped it with PROT_NONE; pte_present gives true */ |
40 | #define _PAGE_BIT_PROTNONE _PAGE_BIT_GLOBAL |
- | |
41 | /* - set: nonlinear file mapping, saved PTE; unset:swap */ |
- | |
Line 42... | Line 32... | ||
42 | #define _PAGE_BIT_FILE _PAGE_BIT_DIRTY |
32 | #define _PAGE_BIT_PROTNONE _PAGE_BIT_GLOBAL |
43 | 33 | ||
44 | #define _PAGE_PRESENT (_AT(pteval_t, 1) << _PAGE_BIT_PRESENT) |
34 | #define _PAGE_PRESENT (_AT(pteval_t, 1) << _PAGE_BIT_PRESENT) |
45 | #define _PAGE_RW (_AT(pteval_t, 1) << _PAGE_BIT_RW) |
35 | #define _PAGE_RW (_AT(pteval_t, 1) << _PAGE_BIT_RW) |
Line 76... | Line 66... | ||
76 | #else |
66 | #else |
77 | #define _PAGE_SOFT_DIRTY (_AT(pteval_t, 0)) |
67 | #define _PAGE_SOFT_DIRTY (_AT(pteval_t, 0)) |
78 | #endif |
68 | #endif |
Line 79... | Line 69... | ||
79 | 69 | ||
80 | /* |
- | |
81 | * _PAGE_NUMA distinguishes between a numa hinting minor fault and a page |
- | |
82 | * that is not present. The hinting fault gathers numa placement statistics |
- | |
83 | * (see pte_numa()). The bit is always zero when the PTE is not present. |
- | |
84 | * |
- | |
85 | * The bit picked must be always zero when the pmd is present and not |
- | |
86 | * present, so that we don't lose information when we set it while |
- | |
87 | * atomically clearing the present bit. |
- | |
88 | */ |
- | |
89 | #ifdef CONFIG_NUMA_BALANCING |
- | |
90 | #define _PAGE_NUMA (_AT(pteval_t, 1) << _PAGE_BIT_NUMA) |
- | |
91 | #else |
- | |
92 | #define _PAGE_NUMA (_AT(pteval_t, 0)) |
- | |
93 | #endif |
- | |
94 | - | ||
95 | /* |
70 | /* |
96 | * Tracking soft dirty bit when a page goes to a swap is tricky. |
71 | * Tracking soft dirty bit when a page goes to a swap is tricky. |
97 | * We need a bit which can be stored in pte _and_ not conflict |
72 | * We need a bit which can be stored in pte _and_ not conflict |
98 | * with swap entry format. On x86 bits 6 and 7 are *not* involved |
73 | * with swap entry format. On x86 bits 6 and 7 are *not* involved |
99 | * into swap entry computation, but bit 6 is used for nonlinear |
74 | * into swap entry computation, but bit 6 is used for nonlinear |
Line 112... | Line 87... | ||
112 | #define _PAGE_NX (_AT(pteval_t, 1) << _PAGE_BIT_NX) |
87 | #define _PAGE_NX (_AT(pteval_t, 1) << _PAGE_BIT_NX) |
113 | #else |
88 | #else |
114 | #define _PAGE_NX (_AT(pteval_t, 0)) |
89 | #define _PAGE_NX (_AT(pteval_t, 0)) |
115 | #endif |
90 | #endif |
Line 116... | Line -... | ||
116 | - | ||
117 | #define _PAGE_FILE (_AT(pteval_t, 1) << _PAGE_BIT_FILE) |
91 | |
Line 118... | Line 92... | ||
118 | #define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE) |
92 | #define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE) |
119 | 93 | ||
120 | #define _PAGE_TABLE (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | \ |
94 | #define _PAGE_TABLE (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | \ |
121 | _PAGE_ACCESSED | _PAGE_DIRTY) |
95 | _PAGE_ACCESSED | _PAGE_DIRTY) |
Line 122... | Line 96... | ||
122 | #define _KERNPG_TABLE (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | \ |
96 | #define _KERNPG_TABLE (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED | \ |
123 | _PAGE_DIRTY) |
97 | _PAGE_DIRTY) |
124 | 98 | ||
125 | /* Set of bits not changed in pte_modify */ |
99 | /* Set of bits not changed in pte_modify */ |
126 | #define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ |
100 | #define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ |
Line 127... | Line 101... | ||
127 | _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY | \ |
101 | _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY | \ |
128 | _PAGE_SOFT_DIRTY | _PAGE_NUMA) |
102 | _PAGE_SOFT_DIRTY) |
129 | #define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_NUMA) |
103 | #define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE) |
130 | 104 | ||
Line 233... | Line 207... | ||
233 | 207 | ||
Line 234... | Line 208... | ||
234 | #ifndef __ASSEMBLY__ |
208 | #ifndef __ASSEMBLY__ |
Line 235... | Line 209... | ||
235 | 209 | ||
236 | #include |
210 | #include |
Line 237... | Line 211... | ||
237 | 211 | ||
238 | /* PTE_PFN_MASK extracts the PFN from a (pte|pmd|pud|pgd)val_t */ |
212 | /* Extracts the PFN from a (pte|pmd|pud|pgd)val_t of a 4KB page */ |
Line 239... | Line 213... | ||
239 | #define PTE_PFN_MASK ((pteval_t)PHYSICAL_PAGE_MASK) |
213 | #define PTE_PFN_MASK ((pteval_t)PHYSICAL_PAGE_MASK) |
Line 240... | Line 214... | ||
240 | 214 | ||
Line 258... | Line 232... | ||
258 | static inline pgdval_t pgd_flags(pgd_t pgd) |
232 | static inline pgdval_t pgd_flags(pgd_t pgd) |
259 | { |
233 | { |
260 | return native_pgd_val(pgd) & PTE_FLAGS_MASK; |
234 | return native_pgd_val(pgd) & PTE_FLAGS_MASK; |
261 | } |
235 | } |
Line 262... | Line 236... | ||
262 | 236 | ||
263 | #if PAGETABLE_LEVELS > 3 |
237 | #if CONFIG_PGTABLE_LEVELS > 3 |
Line 264... | Line 238... | ||
264 | typedef struct { pudval_t pud; } pud_t; |
238 | typedef struct { pudval_t pud; } pud_t; |
265 | 239 | ||
266 | static inline pud_t native_make_pud(pmdval_t val) |
240 | static inline pud_t native_make_pud(pmdval_t val) |
Line 279... | Line 253... | ||
279 | { |
253 | { |
280 | return native_pgd_val(pud.pgd); |
254 | return native_pgd_val(pud.pgd); |
281 | } |
255 | } |
282 | #endif |
256 | #endif |
Line 283... | Line 257... | ||
283 | 257 | ||
284 | #if PAGETABLE_LEVELS > 2 |
258 | #if CONFIG_PGTABLE_LEVELS > 2 |
Line 285... | Line 259... | ||
285 | typedef struct { pmdval_t pmd; } pmd_t; |
259 | typedef struct { pmdval_t pmd; } pmd_t; |
286 | 260 | ||
287 | static inline pmd_t native_make_pmd(pmdval_t val) |
261 | static inline pmd_t native_make_pmd(pmdval_t val) |
Line 300... | Line 274... | ||
300 | { |
274 | { |
301 | return native_pgd_val(pmd.pud.pgd); |
275 | return native_pgd_val(pmd.pud.pgd); |
302 | } |
276 | } |
303 | #endif |
277 | #endif |
Line -... | Line 278... | ||
- | 278 | ||
- | 279 | static inline pudval_t pud_pfn_mask(pud_t pud) |
|
- | 280 | { |
|
- | 281 | if (native_pud_val(pud) & _PAGE_PSE) |
|
- | 282 | return PHYSICAL_PUD_PAGE_MASK; |
|
- | 283 | else |
|
- | 284 | return PTE_PFN_MASK; |
|
- | 285 | } |
|
- | 286 | ||
- | 287 | static inline pudval_t pud_flags_mask(pud_t pud) |
|
- | 288 | { |
|
- | 289 | return ~pud_pfn_mask(pud); |
|
- | 290 | } |
|
304 | 291 | ||
305 | static inline pudval_t pud_flags(pud_t pud) |
292 | static inline pudval_t pud_flags(pud_t pud) |
306 | { |
293 | { |
- | 294 | return native_pud_val(pud) & pud_flags_mask(pud); |
|
- | 295 | } |
|
- | 296 | ||
- | 297 | static inline pmdval_t pmd_pfn_mask(pmd_t pmd) |
|
- | 298 | { |
|
- | 299 | if (native_pmd_val(pmd) & _PAGE_PSE) |
|
- | 300 | return PHYSICAL_PMD_PAGE_MASK; |
|
- | 301 | else |
|
- | 302 | return PTE_PFN_MASK; |
|
- | 303 | } |
|
- | 304 | ||
- | 305 | static inline pmdval_t pmd_flags_mask(pmd_t pmd) |
|
- | 306 | { |
|
307 | return native_pud_val(pud) & PTE_FLAGS_MASK; |
307 | return ~pmd_pfn_mask(pmd); |
Line 308... | Line 308... | ||
308 | } |
308 | } |
309 | 309 | ||
310 | static inline pmdval_t pmd_flags(pmd_t pmd) |
310 | static inline pmdval_t pmd_flags(pmd_t pmd) |
311 | { |
311 | { |
Line 312... | Line 312... | ||
312 | return native_pmd_val(pmd) & PTE_FLAGS_MASK; |
312 | return native_pmd_val(pmd) & pmd_flags_mask(pmd); |
313 | } |
313 | } |
314 | 314 | ||
Line 325... | Line 325... | ||
325 | static inline pteval_t pte_flags(pte_t pte) |
325 | static inline pteval_t pte_flags(pte_t pte) |
326 | { |
326 | { |
327 | return native_pte_val(pte) & PTE_FLAGS_MASK; |
327 | return native_pte_val(pte) & PTE_FLAGS_MASK; |
328 | } |
328 | } |
Line 329... | Line -... | ||
329 | - | ||
330 | #ifdef CONFIG_NUMA_BALANCING |
- | |
331 | /* Set of bits that distinguishes present, prot_none and numa ptes */ |
- | |
332 | #define _PAGE_NUMA_MASK (_PAGE_NUMA|_PAGE_PROTNONE|_PAGE_PRESENT) |
- | |
333 | static inline pteval_t ptenuma_flags(pte_t pte) |
- | |
334 | { |
- | |
335 | return pte_flags(pte) & _PAGE_NUMA_MASK; |
- | |
336 | } |
- | |
337 | - | ||
338 | static inline pmdval_t pmdnuma_flags(pmd_t pmd) |
- | |
339 | { |
- | |
340 | return pmd_flags(pmd) & _PAGE_NUMA_MASK; |
- | |
341 | } |
- | |
342 | #endif /* CONFIG_NUMA_BALANCING */ |
- | |
343 | 329 | ||
344 | #define pgprot_val(x) ((x).pgprot) |
330 | #define pgprot_val(x) ((x).pgprot) |
Line 345... | Line 331... | ||
345 | #define __pgprot(x) ((pgprot_t) { (x) } ) |
331 | #define __pgprot(x) ((pgprot_t) { (x) } ) |
346 | 332 | ||
Line 405... | Line 391... | ||
405 | extern int nx_enabled; |
391 | extern int nx_enabled; |
Line 406... | Line 392... | ||
406 | 392 | ||
407 | #define pgprot_writecombine pgprot_writecombine |
393 | #define pgprot_writecombine pgprot_writecombine |
Line -... | Line 394... | ||
- | 394 | extern pgprot_t pgprot_writecombine(pgprot_t prot); |
|
- | 395 | ||
- | 396 | #define pgprot_writethrough pgprot_writethrough |
|
408 | extern pgprot_t pgprot_writecombine(pgprot_t prot); |
397 | extern pgprot_t pgprot_writethrough(pgprot_t prot); |
409 | 398 | ||
Line 410... | Line 399... | ||
410 | /* Indicate that x86 has its own track and untrack pfn vma functions */ |
399 | /* Indicate that x86 has its own track and untrack pfn vma functions */ |
411 | #define __HAVE_PFNMAP_TRACKING |
400 | #define __HAVE_PFNMAP_TRACKING |