Subversion Repositories Kolibri OS

Rev

Rev 5270 | Rev 6934 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. #ifndef _ASM_X86_PGTABLE_H
  2. #define _ASM_X86_PGTABLE_H
  3.  
  4. #include <asm/page.h>
  5. #include <asm/e820.h>
  6.  
  7. #include <asm/pgtable_types.h>
  8.  
  9. /*
  10.  * Macro to mark a page protection value as UC-
  11.  */
  12. #define pgprot_noncached(prot)                                          \
  13.         ((boot_cpu_data.x86 > 3)                                        \
  14.          ? (__pgprot(pgprot_val(prot) |                                 \
  15.                      cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS)))     \
  16.          : (prot))
  17.  
  18. #ifndef __ASSEMBLY__
  19. #include <asm/x86_init.h>
  20.  
  21. void ptdump_walk_pgd_level(struct seq_file *m, pgd_t *pgd);
  22. void ptdump_walk_pgd_level_checkwx(void);
  23.  
  24. #ifdef CONFIG_DEBUG_WX
  25. #define debug_checkwx() ptdump_walk_pgd_level_checkwx()
  26. #else
  27. #define debug_checkwx() do { } while (0)
  28. #endif
  29.  
  30. /*
  31.  * ZERO_PAGE is a global shared page that is always zero: used
  32.  * for zero-mapped memory areas etc..
  33.  */
  34. extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
  35.         __visible;
  36. #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
  37.  
  38. extern spinlock_t pgd_lock;
  39. extern struct list_head pgd_list;
  40.  
  41. extern struct mm_struct *pgd_page_get_mm(struct page *page);
  42.  
  43. #ifdef CONFIG_PARAVIRT
  44. #include <asm/paravirt.h>
  45. #else  /* !CONFIG_PARAVIRT */
  46. #define set_pte(ptep, pte)              native_set_pte(ptep, pte)
  47. #define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte)
  48. #define set_pmd_at(mm, addr, pmdp, pmd) native_set_pmd_at(mm, addr, pmdp, pmd)
  49.  
  50. #define set_pte_atomic(ptep, pte)                                       \
  51.         native_set_pte_atomic(ptep, pte)
  52.  
  53. #define set_pmd(pmdp, pmd)              native_set_pmd(pmdp, pmd)
  54.  
  55. #ifndef __PAGETABLE_PUD_FOLDED
  56. #define set_pgd(pgdp, pgd)              native_set_pgd(pgdp, pgd)
  57. #define pgd_clear(pgd)                  native_pgd_clear(pgd)
  58. #endif
  59.  
  60. #ifndef set_pud
  61. # define set_pud(pudp, pud)             native_set_pud(pudp, pud)
  62. #endif
  63.  
  64. #ifndef __PAGETABLE_PMD_FOLDED
  65. #define pud_clear(pud)                  native_pud_clear(pud)
  66. #endif
  67.  
  68. #define pte_clear(mm, addr, ptep)       native_pte_clear(mm, addr, ptep)
  69. #define pmd_clear(pmd)                  native_pmd_clear(pmd)
  70.  
  71. #define pte_update(mm, addr, ptep)              do { } while (0)
  72. #define pte_update_defer(mm, addr, ptep)        do { } while (0)
  73. #define pmd_update(mm, addr, ptep)              do { } while (0)
  74. #define pmd_update_defer(mm, addr, ptep)        do { } while (0)
  75.  
  76. #define pgd_val(x)      native_pgd_val(x)
  77. #define __pgd(x)        native_make_pgd(x)
  78.  
  79. #ifndef __PAGETABLE_PUD_FOLDED
  80. #define pud_val(x)      native_pud_val(x)
  81. #define __pud(x)        native_make_pud(x)
  82. #endif
  83.  
  84. #ifndef __PAGETABLE_PMD_FOLDED
  85. #define pmd_val(x)      native_pmd_val(x)
  86. #define __pmd(x)        native_make_pmd(x)
  87. #endif
  88.  
  89. #define pte_val(x)      native_pte_val(x)
  90. #define __pte(x)        native_make_pte(x)
  91.  
  92. #define arch_end_context_switch(prev)   do {} while(0)
  93.  
  94. #endif  /* CONFIG_PARAVIRT */
  95.  
  96. /*
  97.  * The following only work if pte_present() is true.
  98.  * Undefined behaviour if not..
  99.  */
  100. static inline int pte_dirty(pte_t pte)
  101. {
  102.         return pte_flags(pte) & _PAGE_DIRTY;
  103. }
  104.  
  105. static inline int pte_young(pte_t pte)
  106. {
  107.         return pte_flags(pte) & _PAGE_ACCESSED;
  108. }
  109.  
  110. static inline int pmd_dirty(pmd_t pmd)
  111. {
  112.         return pmd_flags(pmd) & _PAGE_DIRTY;
  113. }
  114.  
  115. static inline int pmd_young(pmd_t pmd)
  116. {
  117.         return pmd_flags(pmd) & _PAGE_ACCESSED;
  118. }
  119.  
  120. static inline int pte_write(pte_t pte)
  121. {
  122.         return pte_flags(pte) & _PAGE_RW;
  123. }
  124.  
  125. static inline int pte_huge(pte_t pte)
  126. {
  127.         return pte_flags(pte) & _PAGE_PSE;
  128. }
  129.  
  130. static inline int pte_global(pte_t pte)
  131. {
  132.         return pte_flags(pte) & _PAGE_GLOBAL;
  133. }
  134.  
  135. static inline int pte_exec(pte_t pte)
  136. {
  137.         return !(pte_flags(pte) & _PAGE_NX);
  138. }
  139.  
  140. static inline int pte_special(pte_t pte)
  141. {
  142.         return pte_flags(pte) & _PAGE_SPECIAL;
  143. }
  144.  
  145. static inline unsigned long pte_pfn(pte_t pte)
  146. {
  147.         return (pte_val(pte) & PTE_PFN_MASK) >> PAGE_SHIFT;
  148. }
  149.  
  150. static inline unsigned long pmd_pfn(pmd_t pmd)
  151. {
  152.         return (pmd_val(pmd) & pmd_pfn_mask(pmd)) >> PAGE_SHIFT;
  153. }
  154.  
  155. static inline unsigned long pud_pfn(pud_t pud)
  156. {
  157.         return (pud_val(pud) & pud_pfn_mask(pud)) >> PAGE_SHIFT;
  158. }
  159.  
  160. #define pte_page(pte)   pfn_to_page(pte_pfn(pte))
  161.  
  162. static inline int pmd_large(pmd_t pte)
  163. {
  164.         return pmd_flags(pte) & _PAGE_PSE;
  165. }
  166.  
  167. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  168. static inline int pmd_trans_splitting(pmd_t pmd)
  169. {
  170.         return pmd_val(pmd) & _PAGE_SPLITTING;
  171. }
  172.  
  173. static inline int pmd_trans_huge(pmd_t pmd)
  174. {
  175.         return pmd_val(pmd) & _PAGE_PSE;
  176. }
  177.  
  178. static inline int has_transparent_hugepage(void)
  179. {
  180.         return cpu_has_pse;
  181. }
  182. #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  183.  
  184. static inline pte_t pte_set_flags(pte_t pte, pteval_t set)
  185. {
  186.         pteval_t v = native_pte_val(pte);
  187.  
  188.         return native_make_pte(v | set);
  189. }
  190.  
  191. static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear)
  192. {
  193.         pteval_t v = native_pte_val(pte);
  194.  
  195.         return native_make_pte(v & ~clear);
  196. }
  197.  
  198. static inline pte_t pte_mkclean(pte_t pte)
  199. {
  200.         return pte_clear_flags(pte, _PAGE_DIRTY);
  201. }
  202.  
  203. static inline pte_t pte_mkold(pte_t pte)
  204. {
  205.         return pte_clear_flags(pte, _PAGE_ACCESSED);
  206. }
  207.  
  208. static inline pte_t pte_wrprotect(pte_t pte)
  209. {
  210.         return pte_clear_flags(pte, _PAGE_RW);
  211. }
  212.  
  213. static inline pte_t pte_mkexec(pte_t pte)
  214. {
  215.         return pte_clear_flags(pte, _PAGE_NX);
  216. }
  217.  
  218. static inline pte_t pte_mkdirty(pte_t pte)
  219. {
  220.         return pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
  221. }
  222.  
  223. static inline pte_t pte_mkyoung(pte_t pte)
  224. {
  225.         return pte_set_flags(pte, _PAGE_ACCESSED);
  226. }
  227.  
  228. static inline pte_t pte_mkwrite(pte_t pte)
  229. {
  230.         return pte_set_flags(pte, _PAGE_RW);
  231. }
  232.  
  233. static inline pte_t pte_mkhuge(pte_t pte)
  234. {
  235.         return pte_set_flags(pte, _PAGE_PSE);
  236. }
  237.  
  238. static inline pte_t pte_clrhuge(pte_t pte)
  239. {
  240.         return pte_clear_flags(pte, _PAGE_PSE);
  241. }
  242.  
  243. static inline pte_t pte_mkglobal(pte_t pte)
  244. {
  245.         return pte_set_flags(pte, _PAGE_GLOBAL);
  246. }
  247.  
  248. static inline pte_t pte_clrglobal(pte_t pte)
  249. {
  250.         return pte_clear_flags(pte, _PAGE_GLOBAL);
  251. }
  252.  
  253. static inline pte_t pte_mkspecial(pte_t pte)
  254. {
  255.         return pte_set_flags(pte, _PAGE_SPECIAL);
  256. }
  257.  
  258. static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)
  259. {
  260.         pmdval_t v = native_pmd_val(pmd);
  261.  
  262.         return __pmd(v | set);
  263. }
  264.  
  265. static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)
  266. {
  267.         pmdval_t v = native_pmd_val(pmd);
  268.  
  269.         return __pmd(v & ~clear);
  270. }
  271.  
  272. static inline pmd_t pmd_mkold(pmd_t pmd)
  273. {
  274.         return pmd_clear_flags(pmd, _PAGE_ACCESSED);
  275. }
  276.  
  277. static inline pmd_t pmd_wrprotect(pmd_t pmd)
  278. {
  279.         return pmd_clear_flags(pmd, _PAGE_RW);
  280. }
  281.  
  282. static inline pmd_t pmd_mkdirty(pmd_t pmd)
  283. {
  284.         return pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
  285. }
  286.  
  287. static inline pmd_t pmd_mkhuge(pmd_t pmd)
  288. {
  289.         return pmd_set_flags(pmd, _PAGE_PSE);
  290. }
  291.  
  292. static inline pmd_t pmd_mkyoung(pmd_t pmd)
  293. {
  294.         return pmd_set_flags(pmd, _PAGE_ACCESSED);
  295. }
  296.  
  297. static inline pmd_t pmd_mkwrite(pmd_t pmd)
  298. {
  299.         return pmd_set_flags(pmd, _PAGE_RW);
  300. }
  301.  
  302. static inline pmd_t pmd_mknotpresent(pmd_t pmd)
  303. {
  304.         return pmd_clear_flags(pmd, _PAGE_PRESENT | _PAGE_PROTNONE);
  305. }
  306.  
  307. #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
  308. static inline int pte_soft_dirty(pte_t pte)
  309. {
  310.         return pte_flags(pte) & _PAGE_SOFT_DIRTY;
  311. }
  312.  
  313. static inline int pmd_soft_dirty(pmd_t pmd)
  314. {
  315.         return pmd_flags(pmd) & _PAGE_SOFT_DIRTY;
  316. }
  317.  
  318. static inline pte_t pte_mksoft_dirty(pte_t pte)
  319. {
  320.         return pte_set_flags(pte, _PAGE_SOFT_DIRTY);
  321. }
  322.  
  323. static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
  324. {
  325.         return pmd_set_flags(pmd, _PAGE_SOFT_DIRTY);
  326. }
  327.  
  328. static inline pte_t pte_clear_soft_dirty(pte_t pte)
  329. {
  330.         return pte_clear_flags(pte, _PAGE_SOFT_DIRTY);
  331. }
  332.  
  333. static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
  334. {
  335.         return pmd_clear_flags(pmd, _PAGE_SOFT_DIRTY);
  336. }
  337.  
  338. #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
  339.  
  340. /*
  341.  * Mask out unsupported bits in a present pgprot.  Non-present pgprots
  342.  * can use those bits for other purposes, so leave them be.
  343.  */
  344. static inline pgprotval_t massage_pgprot(pgprot_t pgprot)
  345. {
  346.         pgprotval_t protval = pgprot_val(pgprot);
  347.  
  348.         if (protval & _PAGE_PRESENT)
  349.                 protval &= __supported_pte_mask;
  350.  
  351.         return protval;
  352. }
  353.  
  354. static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
  355. {
  356.         return __pte(((phys_addr_t)page_nr << PAGE_SHIFT) |
  357.                      massage_pgprot(pgprot));
  358. }
  359.  
  360. static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
  361. {
  362.         return __pmd(((phys_addr_t)page_nr << PAGE_SHIFT) |
  363.                      massage_pgprot(pgprot));
  364. }
  365.  
  366. static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
  367. {
  368.         pteval_t val = pte_val(pte);
  369.  
  370.         /*
  371.          * Chop off the NX bit (if present), and add the NX portion of
  372.          * the newprot (if present):
  373.          */
  374.         val &= _PAGE_CHG_MASK;
  375.         val |= massage_pgprot(newprot) & ~_PAGE_CHG_MASK;
  376.  
  377.         return __pte(val);
  378. }
  379.  
  380. static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
  381. {
  382.         pmdval_t val = pmd_val(pmd);
  383.  
  384.         val &= _HPAGE_CHG_MASK;
  385.         val |= massage_pgprot(newprot) & ~_HPAGE_CHG_MASK;
  386.  
  387.         return __pmd(val);
  388. }
  389.  
  390. /* mprotect needs to preserve PAT bits when updating vm_page_prot */
  391. #define pgprot_modify pgprot_modify
  392. static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
  393. {
  394.         pgprotval_t preservebits = pgprot_val(oldprot) & _PAGE_CHG_MASK;
  395.         pgprotval_t addbits = pgprot_val(newprot);
  396.         return __pgprot(preservebits | addbits);
  397. }
  398.  
  399. #define pte_pgprot(x) __pgprot(pte_flags(x))
  400. #define pmd_pgprot(x) __pgprot(pmd_flags(x))
  401. #define pud_pgprot(x) __pgprot(pud_flags(x))
  402.  
  403. #define canon_pgprot(p) __pgprot(massage_pgprot(p))
  404.  
  405. static inline int is_new_memtype_allowed(u64 paddr, unsigned long size,
  406.                                          enum page_cache_mode pcm,
  407.                                          enum page_cache_mode new_pcm)
  408. {
  409.         /*
  410.          * PAT type is always WB for untracked ranges, so no need to check.
  411.          */
  412.         if (x86_platform.is_untracked_pat_range(paddr, paddr + size))
  413.                 return 1;
  414.  
  415.         /*
  416.          * Certain new memtypes are not allowed with certain
  417.          * requested memtype:
  418.          * - request is uncached, return cannot be write-back
  419.          * - request is write-combine, return cannot be write-back
  420.          * - request is write-through, return cannot be write-back
  421.          * - request is write-through, return cannot be write-combine
  422.          */
  423.         if ((pcm == _PAGE_CACHE_MODE_UC_MINUS &&
  424.              new_pcm == _PAGE_CACHE_MODE_WB) ||
  425.             (pcm == _PAGE_CACHE_MODE_WC &&
  426.              new_pcm == _PAGE_CACHE_MODE_WB) ||
  427.             (pcm == _PAGE_CACHE_MODE_WT &&
  428.              new_pcm == _PAGE_CACHE_MODE_WB) ||
  429.             (pcm == _PAGE_CACHE_MODE_WT &&
  430.              new_pcm == _PAGE_CACHE_MODE_WC)) {
  431.                 return 0;
  432.         }
  433.  
  434.         return 1;
  435. }
  436.  
  437. pmd_t *populate_extra_pmd(unsigned long vaddr);
  438. pte_t *populate_extra_pte(unsigned long vaddr);
  439. #endif  /* __ASSEMBLY__ */
  440.  
  441. #ifdef CONFIG_X86_32
  442. # include <asm/pgtable_32.h>
  443. #else
  444. # include <asm/pgtable_64.h>
  445. #endif
  446.  
  447. #ifndef __ASSEMBLY__
  448. //#include <linux/mm_types.h>
  449. #include <linux/mmdebug.h>
  450. #include <linux/log2.h>
  451.  
  452. static inline int pte_none(pte_t pte)
  453. {
  454.         return !pte.pte;
  455. }
  456.  
  457. #define __HAVE_ARCH_PTE_SAME
  458. static inline int pte_same(pte_t a, pte_t b)
  459. {
  460.         return a.pte == b.pte;
  461. }
  462.  
  463. static inline int pte_present(pte_t a)
  464. {
  465.         return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE);
  466. }
  467.  
  468. #define pte_accessible pte_accessible
  469. static inline bool pte_accessible(struct mm_struct *mm, pte_t a)
  470. {
  471.         if (pte_flags(a) & _PAGE_PRESENT)
  472.                 return true;
  473.  
  474.         if ((pte_flags(a) & _PAGE_PROTNONE) &&
  475.                         mm_tlb_flush_pending(mm))
  476.                 return true;
  477.  
  478.         return false;
  479. }
  480.  
  481. static inline int pte_hidden(pte_t pte)
  482. {
  483.         return pte_flags(pte) & _PAGE_HIDDEN;
  484. }
  485.  
  486. static inline int pmd_present(pmd_t pmd)
  487. {
  488.         /*
  489.          * Checking for _PAGE_PSE is needed too because
  490.          * split_huge_page will temporarily clear the present bit (but
  491.          * the _PAGE_PSE flag will remain set at all times while the
  492.          * _PAGE_PRESENT bit is clear).
  493.          */
  494.         return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE);
  495. }
  496.  
  497. #ifdef CONFIG_NUMA_BALANCING
  498. /*
  499.  * These work without NUMA balancing but the kernel does not care. See the
  500.  * comment in include/asm-generic/pgtable.h
  501.  */
  502. static inline int pte_protnone(pte_t pte)
  503. {
  504.         return (pte_flags(pte) & (_PAGE_PROTNONE | _PAGE_PRESENT))
  505.                 == _PAGE_PROTNONE;
  506. }
  507.  
  508. static inline int pmd_protnone(pmd_t pmd)
  509. {
  510.         return (pmd_flags(pmd) & (_PAGE_PROTNONE | _PAGE_PRESENT))
  511.                 == _PAGE_PROTNONE;
  512. }
  513. #endif /* CONFIG_NUMA_BALANCING */
  514.  
  515. static inline int pmd_none(pmd_t pmd)
  516. {
  517.         /* Only check low word on 32-bit platforms, since it might be
  518.            out of sync with upper half. */
  519.         return (unsigned long)native_pmd_val(pmd) == 0;
  520. }
  521.  
  522. static inline unsigned long pmd_page_vaddr(pmd_t pmd)
  523. {
  524.         return (unsigned long)__va(pmd_val(pmd) & pmd_pfn_mask(pmd));
  525. }
  526.  
  527. /*
  528.  * Currently stuck as a macro due to indirect forward reference to
  529.  * linux/mmzone.h's __section_mem_map_addr() definition:
  530.  */
  531. #define pmd_page(pmd)           \
  532.         pfn_to_page((pmd_val(pmd) & pmd_pfn_mask(pmd)) >> PAGE_SHIFT)
  533.  
  534. /*
  535.  * the pmd page can be thought of an array like this: pmd_t[PTRS_PER_PMD]
  536.  *
  537.  * this macro returns the index of the entry in the pmd page which would
  538.  * control the given virtual address
  539.  */
  540. static inline unsigned long pmd_index(unsigned long address)
  541. {
  542.         return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1);
  543. }
  544.  
  545. /*
  546.  * Conversion functions: convert a page and protection to a page entry,
  547.  * and a page entry and page directory to the page they refer to.
  548.  *
  549.  * (Currently stuck as a macro because of indirect forward reference
  550.  * to linux/mm.h:page_to_nid())
  551.  */
  552. #define mk_pte(page, pgprot)   pfn_pte(page_to_pfn(page), (pgprot))
  553.  
  554. /*
  555.  * the pte page can be thought of an array like this: pte_t[PTRS_PER_PTE]
  556.  *
  557.  * this function returns the index of the entry in the pte page which would
  558.  * control the given virtual address
  559.  */
  560. static inline unsigned long pte_index(unsigned long address)
  561. {
  562.         return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
  563. }
  564.  
  565. static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
  566. {
  567.         return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address);
  568. }
  569.  
  570. static inline int pmd_bad(pmd_t pmd)
  571. {
  572.         return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE;
  573. }
  574.  
  575. static inline unsigned long pages_to_mb(unsigned long npg)
  576. {
  577.         return npg >> (20 - PAGE_SHIFT);
  578. }
  579.  
  580. #if CONFIG_PGTABLE_LEVELS > 2
  581. static inline int pud_none(pud_t pud)
  582. {
  583.         return native_pud_val(pud) == 0;
  584. }
  585.  
  586. static inline int pud_present(pud_t pud)
  587. {
  588.         return pud_flags(pud) & _PAGE_PRESENT;
  589. }
  590.  
  591. static inline unsigned long pud_page_vaddr(pud_t pud)
  592. {
  593.         return (unsigned long)__va(pud_val(pud) & pud_pfn_mask(pud));
  594. }
  595.  
  596. /*
  597.  * Currently stuck as a macro due to indirect forward reference to
  598.  * linux/mmzone.h's __section_mem_map_addr() definition:
  599.  */
  600. #define pud_page(pud)           \
  601.         pfn_to_page((pud_val(pud) & pud_pfn_mask(pud)) >> PAGE_SHIFT)
  602.  
  603. /* Find an entry in the second-level page table.. */
  604. static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
  605. {
  606.         return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address);
  607. }
  608.  
  609. static inline int pud_large(pud_t pud)
  610. {
  611.         return (pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) ==
  612.                 (_PAGE_PSE | _PAGE_PRESENT);
  613. }
  614.  
  615. static inline int pud_bad(pud_t pud)
  616. {
  617.         return (pud_flags(pud) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0;
  618. }
  619. #else
  620. static inline int pud_large(pud_t pud)
  621. {
  622.         return 0;
  623. }
  624. #endif  /* CONFIG_PGTABLE_LEVELS > 2 */
  625.  
  626. #if CONFIG_PGTABLE_LEVELS > 3
  627. static inline int pgd_present(pgd_t pgd)
  628. {
  629.         return pgd_flags(pgd) & _PAGE_PRESENT;
  630. }
  631.  
  632. static inline unsigned long pgd_page_vaddr(pgd_t pgd)
  633. {
  634.         return (unsigned long)__va((unsigned long)pgd_val(pgd) & PTE_PFN_MASK);
  635. }
  636.  
  637. /*
  638.  * Currently stuck as a macro due to indirect forward reference to
  639.  * linux/mmzone.h's __section_mem_map_addr() definition:
  640.  */
  641. #define pgd_page(pgd)           pfn_to_page(pgd_val(pgd) >> PAGE_SHIFT)
  642.  
  643. /* to find an entry in a page-table-directory. */
  644. static inline unsigned long pud_index(unsigned long address)
  645. {
  646.         return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);
  647. }
  648.  
  649. static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
  650. {
  651.         return (pud_t *)pgd_page_vaddr(*pgd) + pud_index(address);
  652. }
  653.  
  654. static inline int pgd_bad(pgd_t pgd)
  655. {
  656.         return (pgd_flags(pgd) & ~_PAGE_USER) != _KERNPG_TABLE;
  657. }
  658.  
  659. static inline int pgd_none(pgd_t pgd)
  660. {
  661.         return !native_pgd_val(pgd);
  662. }
  663. #endif  /* CONFIG_PGTABLE_LEVELS > 3 */
  664.  
  665. #endif  /* __ASSEMBLY__ */
  666.  
  667. /*
  668.  * the pgd page can be thought of an array like this: pgd_t[PTRS_PER_PGD]
  669.  *
  670.  * this macro returns the index of the entry in the pgd page which would
  671.  * control the given virtual address
  672.  */
  673. #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
  674.  
  675. /*
  676.  * pgd_offset() returns a (pgd_t *)
  677.  * pgd_index() is used get the offset into the pgd page's array of pgd_t's;
  678.  */
  679. #define pgd_offset(mm, address) ((mm)->pgd + pgd_index((address)))
  680. /*
  681.  * a shortcut which implies the use of the kernel's pgd, instead
  682.  * of a process's
  683.  */
  684. #define pgd_offset_k(address) pgd_offset(&init_mm, (address))
  685.  
  686.  
  687. #define KERNEL_PGD_BOUNDARY     pgd_index(PAGE_OFFSET)
  688. #define KERNEL_PGD_PTRS         (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY)
  689.  
  690. #ifndef __ASSEMBLY__
  691.  
  692. extern int direct_gbpages;
  693. void init_mem_mapping(void);
  694. void early_alloc_pgt_buf(void);
  695.  
  696. /* local pte updates need not use xchg for locking */
  697. static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
  698. {
  699.         pte_t res = *ptep;
  700.  
  701.         /* Pure native function needs no input for mm, addr */
  702.         native_pte_clear(NULL, 0, ptep);
  703.         return res;
  704. }
  705.  
  706. static inline pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp)
  707. {
  708.         pmd_t res = *pmdp;
  709.  
  710.         native_pmd_clear(pmdp);
  711.         return res;
  712. }
  713.  
  714. static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
  715.                                      pte_t *ptep , pte_t pte)
  716. {
  717.         native_set_pte(ptep, pte);
  718. }
  719.  
  720. static inline void native_set_pmd_at(struct mm_struct *mm, unsigned long addr,
  721.                                      pmd_t *pmdp , pmd_t pmd)
  722. {
  723.         native_set_pmd(pmdp, pmd);
  724. }
  725.  
  726. #ifndef CONFIG_PARAVIRT
  727. /*
  728.  * Rules for using pte_update - it must be called after any PTE update which
  729.  * has not been done using the set_pte / clear_pte interfaces.  It is used by
  730.  * shadow mode hypervisors to resynchronize the shadow page tables.  Kernel PTE
  731.  * updates should either be sets, clears, or set_pte_atomic for P->P
  732.  * transitions, which means this hook should only be called for user PTEs.
  733.  * This hook implies a P->P protection or access change has taken place, which
  734.  * requires a subsequent TLB flush.  The notification can optionally be delayed
  735.  * until the TLB flush event by using the pte_update_defer form of the
  736.  * interface, but care must be taken to assure that the flush happens while
  737.  * still holding the same page table lock so that the shadow and primary pages
  738.  * do not become out of sync on SMP.
  739.  */
  740. #define pte_update(mm, addr, ptep)              do { } while (0)
  741. #define pte_update_defer(mm, addr, ptep)        do { } while (0)
  742. #endif
  743.  
  744. /*
  745.  * We only update the dirty/accessed state if we set
  746.  * the dirty bit by hand in the kernel, since the hardware
  747.  * will do the accessed bit for us, and we don't want to
  748.  * race with other CPU's that might be updating the dirty
  749.  * bit at the same time.
  750.  */
  751. struct vm_area_struct;
  752.  
  753. #define  __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
  754. extern int ptep_set_access_flags(struct vm_area_struct *vma,
  755.                                  unsigned long address, pte_t *ptep,
  756.                                  pte_t entry, int dirty);
  757.  
  758. #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
  759. extern int ptep_test_and_clear_young(struct vm_area_struct *vma,
  760.                                      unsigned long addr, pte_t *ptep);
  761.  
  762. #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
  763. extern int ptep_clear_flush_young(struct vm_area_struct *vma,
  764.                                   unsigned long address, pte_t *ptep);
  765.  
  766. #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
  767. static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
  768.                                        pte_t *ptep)
  769. {
  770.         pte_t pte = native_ptep_get_and_clear(ptep);
  771.         pte_update(mm, addr, ptep);
  772.         return pte;
  773. }
  774.  
  775. #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
  776. static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
  777.                                             unsigned long addr, pte_t *ptep,
  778.                                             int full)
  779. {
  780.         pte_t pte;
  781.         if (full) {
  782.                 /*
  783.                  * Full address destruction in progress; paravirt does not
  784.                  * care about updates and native needs no locking
  785.                  */
  786.                 pte = native_local_ptep_get_and_clear(ptep);
  787.         } else {
  788.                 pte = ptep_get_and_clear(mm, addr, ptep);
  789.         }
  790.         return pte;
  791. }
  792.  
  793. #define __HAVE_ARCH_PTEP_SET_WRPROTECT
  794. static inline void ptep_set_wrprotect(struct mm_struct *mm,
  795.                                       unsigned long addr, pte_t *ptep)
  796. {
  797.         clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);
  798.         pte_update(mm, addr, ptep);
  799. }
  800.  
  801. #define flush_tlb_fix_spurious_fault(vma, address) do { } while (0)
  802.  
  803. #define mk_pmd(page, pgprot)   pfn_pmd(page_to_pfn(page), (pgprot))
  804.  
  805. #define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
  806. extern int pmdp_set_access_flags(struct vm_area_struct *vma,
  807.                                  unsigned long address, pmd_t *pmdp,
  808.                                  pmd_t entry, int dirty);
  809.  
  810. #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
  811. extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
  812.                                      unsigned long addr, pmd_t *pmdp);
  813.  
  814. #define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
  815. extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
  816.                                   unsigned long address, pmd_t *pmdp);
  817.  
  818.  
  819. #define __HAVE_ARCH_PMDP_SPLITTING_FLUSH
  820. extern void pmdp_splitting_flush(struct vm_area_struct *vma,
  821.                                  unsigned long addr, pmd_t *pmdp);
  822.  
  823. #define __HAVE_ARCH_PMD_WRITE
  824. static inline int pmd_write(pmd_t pmd)
  825. {
  826.         return pmd_flags(pmd) & _PAGE_RW;
  827. }
  828.  
  829. #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
  830. static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr,
  831.                                        pmd_t *pmdp)
  832. {
  833.         pmd_t pmd = native_pmdp_get_and_clear(pmdp);
  834.         pmd_update(mm, addr, pmdp);
  835.         return pmd;
  836. }
  837.  
  838. #define __HAVE_ARCH_PMDP_SET_WRPROTECT
  839. static inline void pmdp_set_wrprotect(struct mm_struct *mm,
  840.                                       unsigned long addr, pmd_t *pmdp)
  841. {
  842.         clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp);
  843.         pmd_update(mm, addr, pmdp);
  844. }
  845.  
  846. /*
  847.  * clone_pgd_range(pgd_t *dst, pgd_t *src, int count);
  848.  *
  849.  *  dst - pointer to pgd range anwhere on a pgd page
  850.  *  src - ""
  851.  *  count - the number of pgds to copy.
  852.  *
  853.  * dst and src can be on the same page, but the range must not overlap,
  854.  * and must not cross a page boundary.
  855.  */
  856. static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
  857. {
  858.        memcpy(dst, src, count * sizeof(pgd_t));
  859. }
  860.  
  861. #define PTE_SHIFT ilog2(PTRS_PER_PTE)
  862. static inline int page_level_shift(enum pg_level level)
  863. {
  864.         return (PAGE_SHIFT - PTE_SHIFT) + level * PTE_SHIFT;
  865. }
  866. static inline unsigned long page_level_size(enum pg_level level)
  867. {
  868.         return 1UL << page_level_shift(level);
  869. }
  870. static inline unsigned long page_level_mask(enum pg_level level)
  871. {
  872.         return ~(page_level_size(level) - 1);
  873. }
  874.  
  875. /*
  876.  * The x86 doesn't have any external MMU info: the kernel page
  877.  * tables contain all the necessary information.
  878.  */
  879. static inline void update_mmu_cache(struct vm_area_struct *vma,
  880.                 unsigned long addr, pte_t *ptep)
  881. {
  882. }
  883. static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
  884.                 unsigned long addr, pmd_t *pmd)
  885. {
  886. }
  887.  
  888. #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
  889. static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
  890. {
  891.         return pte_set_flags(pte, _PAGE_SWP_SOFT_DIRTY);
  892. }
  893.  
  894. static inline int pte_swp_soft_dirty(pte_t pte)
  895. {
  896.         return pte_flags(pte) & _PAGE_SWP_SOFT_DIRTY;
  897. }
  898.  
  899. static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
  900. {
  901.         return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY);
  902. }
  903. #endif
  904.  
  905. //#include <asm-generic/pgtable.h>
  906. #endif  /* __ASSEMBLY__ */
  907.  
  908. #endif /* _ASM_X86_PGTABLE_H */
  909.