Subversion Repositories Kolibri OS

Rev

Rev 6721 | Blame | Compare with Previous | Last modification | View Log | Download | RSS feed

  1. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  2. ;;                                                              ;;
  3. ;; Copyright (C) KolibriOS team 2004-2015. All rights reserved. ;;
  4. ;; Distributed under terms of the GNU General Public License    ;;
  5. ;;                                                              ;;
  6. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  7.  
  8. $Revision: 7132 $
  9.  
  10. ; Initializes MTRRs.
  11. proc init_mtrr
  12.  
  13.         cmp     [BOOT.mtrr], byte 2
  14.         je      .exit
  15.  
  16.         bt      [cpu_caps], CAPS_MTRR
  17.         jnc     .exit
  18.  
  19.         call    mtrr_reconfigure
  20.         stdcall set_mtrr, [LFBAddress], 0x1000000, MEM_WC
  21.  
  22. .exit:
  23.         ret
  24. endp
  25.  
  26. ; Helper procedure for mtrr_reconfigure and set_mtrr,
  27. ; called before changes in MTRRs.
  28. ; 1. disable and flush caches
  29. ; 2. clear PGE bit in cr4
  30. ; 3. flush TLB
  31. ; 4. disable mtrr
  32.  
  33. proc mtrr_begin_change
  34.         mov     eax, cr0
  35.         or      eax, 0x60000000 ;disable caching
  36.         mov     cr0, eax
  37.         wbinvd                  ;invalidate cache
  38.  
  39.         bt      [cpu_caps], CAPS_PGE
  40.         jnc     .cr3_flush
  41.  
  42.         mov     eax, cr4
  43.         btr     eax, 7          ;clear cr4.PGE
  44.         mov     cr4, eax        ;flush TLB
  45.         jmp     @F              ;skip extra serialization
  46.  
  47. .cr3_flush:
  48.         mov     eax, cr3
  49.         mov     cr3, eax        ;flush TLB
  50. @@:
  51.         mov     ecx, MSR_MTRR_DEF_TYPE
  52.         rdmsr
  53.         btr     eax, 11         ;clear enable flag
  54.         wrmsr                   ;disable mtrr
  55.         ret
  56. endp
  57.  
  58. ; Helper procedure for mtrr_reconfigure and set_mtrr,
  59. ; called after changes in MTRRs.
  60. ; 1. enable mtrr
  61. ; 2. flush all caches
  62. ; 3. flush TLB
  63. ; 4. restore cr4.PGE flag, if required
  64.  
  65. proc mtrr_end_change
  66.         mov     ecx, MSR_MTRR_DEF_TYPE
  67.         rdmsr
  68.         or      ah, 8           ; enable variable-ranges MTRR
  69.         and     al, 0xF0        ; default memtype = UC
  70.         wrmsr
  71.  
  72.         wbinvd                  ;again invalidate
  73.         mov     eax, cr0
  74.         and     eax, not 0x60000000
  75.         mov     cr0, eax        ; enable caching
  76.  
  77.         mov     eax, cr3
  78.         mov     cr3, eax        ;flush tlb
  79.  
  80.         bt      [cpu_caps], CAPS_PGE
  81.         jnc     @F
  82.  
  83.         mov     eax, cr4
  84.         bts     eax, 7          ;set cr4.PGE flag
  85.         mov     cr4, eax
  86. @@:
  87.         ret
  88. endp
  89.  
  90. ; Some limits to number of structures located in the stack.
  91. MAX_USEFUL_MTRRS = 16
  92. MAX_RANGES = 16
  93.  
  94. ; mtrr_reconfigure keeps a list of MEM_WB ranges.
  95. ; This structure describes one item in the list.
  96. struct mtrr_range
  97. next            dd      ?       ; next item
  98. start           dq      ?       ; first byte
  99. length          dq      ?       ; length in bytes
  100. ends
  101.  
  102. uglobal
  103. align 4
  104. num_variable_mtrrs      dd      0       ; number of variable-range MTRRs
  105. endg
  106.  
  107. ; Helper procedure for MTRR initialization.
  108. ; Takes MTRR configured by BIOS and tries to recongifure them
  109. ; in order to allow non-UC data at top of 4G memory.
  110. ; Example: if low part of physical memory is 3.5G = 0xE0000000 bytes wide,
  111. ; BIOS can configure two MTRRs so that the first MTRR describes [0, 4G) as WB
  112. ; and the second MTRR describes [3.5G, 4G) as UC;
  113. ; WB+UC=UC, so the resulting memory map would be as needed,
  114. ; but in this configuration our attempts to map LFB at (say) 0xE8000000 as WC
  115. ; would be ignored, WB+UC+WC is still UC.
  116. ; So we must keep top of 4G memory not covered by MTRRs,
  117. ; using three WB MTRRs [0,2G) + [2G,3G) + [3G,3.5G),
  118. ; this gives the same memory map, but allows to add further entries.
  119. ; See mtrrtest.asm for detailed input/output from real hardware+BIOS.
  120. proc mtrr_reconfigure
  121.         push    ebp     ; we're called from init_LFB, and it feels hurt when ebp is destroyed
  122. ; 1. Prepare local variables.
  123. ; 1a. Create list of MAX_RANGES free (aka not yet allocated) ranges.
  124.         xor     eax, eax
  125.         lea     ecx, [eax+MAX_RANGES]
  126. .init_ranges:
  127.         sub     esp, sizeof.mtrr_range - 4
  128.         push    eax
  129.         mov     eax, esp
  130.         dec     ecx
  131.         jnz     .init_ranges
  132.         mov     eax, esp
  133. ; 1b. Fill individual local variables.
  134.         xor     edx, edx
  135.         sub     esp, MAX_USEFUL_MTRRS * 16      ; .mtrrs
  136.         push    edx             ; .mtrrs_end
  137.         push    edx             ; .num_used_mtrrs
  138.         push    eax             ; .first_free_range
  139.         push    edx             ; .first_range: no ranges yet
  140.         mov     cl, [cpu_phys_addr_width]
  141.         or      eax, -1
  142.         shl     eax, cl ; note: this uses cl&31 = cl-32, not the entire cl
  143.         push    eax     ; .phys_reserved_mask
  144. virtual at esp
  145. .phys_reserved_mask     dd      ?
  146. .first_range            dd      ?
  147. .first_free_range       dd      ?
  148. .num_used_mtrrs         dd      ?
  149. .mtrrs_end              dd      ?
  150. .mtrrs          rq      MAX_USEFUL_MTRRS * 2
  151. .local_vars_size = $ - esp
  152. end virtual
  153.  
  154. ; 2. Get the number of variable-range MTRRs from MTRRCAP register.
  155. ; Abort if zero.
  156.         mov     ecx, 0xFE
  157.         rdmsr
  158.         test    al, al
  159.         jz      .abort
  160.         mov     byte [num_variable_mtrrs], al
  161. ; 3. Validate MTRR_DEF_TYPE register.
  162.         mov     ecx, 0x2FF
  163.         rdmsr
  164. ; If BIOS has not initialized variable-range MTRRs, fallback to step 7.
  165.         test    ah, 8
  166.         jz      .fill_ranges_from_memory_map
  167. ; If the default memory type (not covered by MTRRs) is not UC,
  168. ; then probably BIOS did something strange, so it is better to exit immediately
  169. ; hoping for the best.
  170.         cmp     al, MEM_UC
  171.         jnz     .abort
  172. ; 4. Validate all variable-range MTRRs
  173. ; and copy configured MTRRs to the local array [.mtrrs].
  174. ; 4a. Prepare for the loop over existing variable-range MTRRs.
  175.         mov     ecx, 0x200
  176.         lea     edi, [.mtrrs]
  177. .get_used_mtrrs_loop:
  178. ; 4b. For every MTRR, read PHYSBASEn and PHYSMASKn.
  179. ; In PHYSBASEn, clear upper bits and copy to ebp:ebx.
  180.         rdmsr
  181.         or      edx, [.phys_reserved_mask]
  182.         xor     edx, [.phys_reserved_mask]
  183.         mov     ebp, edx
  184.         mov     ebx, eax
  185.         inc     ecx
  186. ; If PHYSMASKn is not active, ignore this MTRR.
  187.         rdmsr
  188.         inc     ecx
  189.         test    ah, 8
  190.         jz      .get_used_mtrrs_next
  191. ; 4c. For every active MTRR, check that number of local entries is not too large.
  192.         inc     [.num_used_mtrrs]
  193.         cmp     [.num_used_mtrrs], MAX_USEFUL_MTRRS
  194.         ja      .abort
  195. ; 4d. For every active MTRR, store PHYSBASEn with upper bits cleared.
  196. ; This contains the MTRR base and the memory type in low byte.
  197.         mov     [edi], ebx
  198.         mov     [edi+4], ebp
  199. ; 4e. For every active MTRR, check that the range is continuous:
  200. ; PHYSMASKn with upper bits set must be negated power of two, and
  201. ; low bits of PHYSBASEn must be zeroes:
  202. ; PHYSMASKn = 1...10...0,
  203. ; PHYSBASEn = x...x0...0,
  204. ; this defines a continuous range from x...x0...0 to x...x1...1,
  205. ; length = 10...0 = negated PHYSMASKn.
  206. ; Store length in the local array.
  207.         and     eax, not 0xFFF
  208.         or      edx, [.phys_reserved_mask]
  209.         mov     dword [edi+8], 0
  210.         mov     dword [edi+12], 0
  211.         sub     [edi+8], eax
  212.         sbb     [edi+12], edx
  213. ; (x and -x) is the maximum power of two that divides x.
  214. ; Condition for powers of two: (x and -x) equals x.
  215.         and     eax, [edi+8]
  216.         and     edx, [edi+12]
  217.         cmp     eax, [edi+8]
  218.         jnz     .abort
  219.         cmp     edx, [edi+12]
  220.         jnz     .abort
  221.         sub     eax, 1
  222.         sbb     edx, 0
  223.         and     eax, not 0xFFF
  224.         and     eax, ebx
  225.         jnz     .abort
  226.         and     edx, ebp
  227.         jnz     .abort
  228. ; 4f. For every active MTRR, validate memory type: it must be either WB or UC.
  229.         add     edi, 16
  230.         cmp     bl, MEM_UC
  231.         jz      .get_used_mtrrs_next
  232.         cmp     bl, MEM_WB
  233.         jnz     .abort
  234. .get_used_mtrrs_next:
  235. ; 4g. Repeat the loop at 4b-4f for all [num_variable_mtrrs] entries.
  236.         mov     eax, [num_variable_mtrrs]
  237.         lea     eax, [0x200+eax*2]
  238.         cmp     ecx, eax
  239.         jb      .get_used_mtrrs_loop
  240. ; 4h. If no active MTRRs were detected, fallback to step 7.
  241.         cmp     [.num_used_mtrrs], 0
  242.         jz      .fill_ranges_from_memory_map
  243.         mov     [.mtrrs_end], edi
  244. ; 5. Generate sorted list of ranges marked as WB.
  245. ; 5a. Prepare for the loop over configured MTRRs filled at step 4.
  246.         lea     ecx, [.mtrrs]
  247. .fill_wb_ranges:
  248. ; 5b. Ignore non-WB MTRRs.
  249.         mov     ebx, [ecx]
  250.         cmp     bl, MEM_WB
  251.         jnz     .next_wb_range
  252.         mov     ebp, [ecx+4]
  253.         and     ebx, not 0xFFF  ; clear memory type and reserved bits
  254. ; ebp:ebx = start of the range described by the current MTRR.
  255. ; 5c. Find the first existing range containing a point greater than ebp:ebx.
  256.         lea     esi, [.first_range]
  257. .find_range_wb:
  258. ; If there is no next range or start of the next range is greater than ebp:ebx,
  259. ; exit the loop to 5d.
  260.         mov     edi, [esi]
  261.         test    edi, edi
  262.         jz      .found_place_wb
  263.         mov     eax, ebx
  264.         mov     edx, ebp
  265.         sub     eax, dword [edi+mtrr_range.start]
  266.         sbb     edx, dword [edi+mtrr_range.start+4]
  267.         jb      .found_place_wb
  268. ; Otherwise, if end of the next range is greater than or equal to ebp:ebx,
  269. ; exit the loop to 5e.
  270.         mov     esi, edi
  271.         sub     eax, dword [edi+mtrr_range.length]
  272.         sbb     edx, dword [edi+mtrr_range.length+4]
  273.         jb      .expand_wb
  274.         or      eax, edx
  275.         jnz     .find_range_wb
  276.         jmp     .expand_wb
  277. .found_place_wb:
  278. ; 5d. ebp:ebx is not within any existing range.
  279. ; Insert a new range between esi and edi.
  280. ; (Later, during 5e, it can be merged with the following ranges.)
  281.         mov     eax, [.first_free_range]
  282.         test    eax, eax
  283.         jz      .abort
  284.         mov     [esi], eax
  285.         mov     edx, [eax+mtrr_range.next]
  286.         mov     [.first_free_range], edx
  287.         mov     dword [eax+mtrr_range.start], ebx
  288.         mov     dword [eax+mtrr_range.start+4], ebp
  289. ; Don't fill [eax+mtrr_range.next] and [eax+mtrr_range.length] yet,
  290. ; they will be calculated including merges at step 5e.
  291.         mov     esi, edi
  292.         mov     edi, eax
  293. .expand_wb:
  294. ; 5e. The range at edi contains ebp:ebx, and esi points to the first range
  295. ; to be checked for merge: esi=edi if ebp:ebx was found in an existing range,
  296. ; esi is next after edi if a new range with ebp:ebx was created.
  297. ; Merge it with following ranges while start of the next range is not greater
  298. ; than the end of the new range.
  299.         add     ebx, [ecx+8]
  300.         adc     ebp, [ecx+12]
  301. ; ebp:ebx = end of the range described by the current MTRR.
  302. .expand_wb_loop:
  303. ; If there is no next range or start of the next range is greater than ebp:ebx,
  304. ; exit the loop to 5g.
  305.         test    esi, esi
  306.         jz      .expand_wb_done
  307.         mov     eax, ebx
  308.         mov     edx, ebp
  309.         sub     eax, dword [esi+mtrr_range.start]
  310.         sbb     edx, dword [esi+mtrr_range.start+4]
  311.         jb      .expand_wb_done
  312. ; Otherwise, if end of the next range is greater than or equal to ebp:ebx,
  313. ; exit the loop to 5f.
  314.         sub     eax, dword [esi+mtrr_range.length]
  315.         sbb     edx, dword [esi+mtrr_range.length+4]
  316.         jb      .expand_wb_last
  317. ; Otherwise, the current range is completely within the new range.
  318. ; Free it and continue the loop.
  319.         mov     edx, [esi+mtrr_range.next]
  320.         cmp     esi, edi
  321.         jz      @f
  322.         mov     eax, [.first_free_range]
  323.         mov     [esi+mtrr_range.next], eax
  324.         mov     [.first_free_range], esi
  325. @@:
  326.         mov     esi, edx
  327.         jmp     .expand_wb_loop
  328. .expand_wb_last:
  329. ; 5f. Start of the new range is inside range described by esi,
  330. ; end of the new range is inside range described by edi.
  331. ; If esi is equal to edi, the new range is completely within
  332. ; an existing range, so proceed to the next range.
  333.         cmp     esi, edi
  334.         jz      .next_wb_range
  335. ; Otherwise, set end of interval at esi to end of interval at edi
  336. ; and free range described by edi.
  337.         mov     ebx, dword [esi+mtrr_range.start]
  338.         mov     ebp, dword [esi+mtrr_range.start+4]
  339.         add     ebx, dword [esi+mtrr_range.length]
  340.         adc     ebp, dword [esi+mtrr_range.length+4]
  341.         mov     edx, [esi+mtrr_range.next]
  342.         mov     eax, [.first_free_range]
  343.         mov     [esi+mtrr_range.next], eax
  344.         mov     [.first_free_range], esi
  345.         mov     esi, edx
  346. .expand_wb_done:
  347. ; 5g. We have found the next range (maybe 0) after merging and
  348. ; the new end of range (maybe ebp:ebx from the new range
  349. ; or end of another existing interval calculated at step 5f).
  350. ; Write them to range at edi.
  351.         mov     [edi+mtrr_range.next], esi
  352.         sub     ebx, dword [edi+mtrr_range.start]
  353.         sbb     ebp, dword [edi+mtrr_range.start+4]
  354.         mov     dword [edi+mtrr_range.length], ebx
  355.         mov     dword [edi+mtrr_range.length+4], ebp
  356. .next_wb_range:
  357. ; 5h. Continue the loop 5b-5g over all configured MTRRs.
  358.         add     ecx, 16
  359.         cmp     ecx, [.mtrrs_end]
  360.         jb      .fill_wb_ranges
  361. ; 6. Exclude all ranges marked as UC.
  362. ; 6a. Prepare for the loop over configured MTRRs filled at step 4.
  363.         lea     ecx, [.mtrrs]
  364. .fill_uc_ranges:
  365. ; 6b. Ignore non-UC MTRRs.
  366.         mov     ebx, [ecx]
  367.         cmp     bl, MEM_UC
  368.         jnz     .next_uc_range
  369.         mov     ebp, [ecx+4]
  370.         and     ebx, not 0xFFF  ; clear memory type and reserved bits
  371. ; ebp:ebx = start of the range described by the current MTRR.
  372.         lea     esi, [.first_range]
  373. ; 6c. Find the first existing range containing a point greater than ebp:ebx.
  374. .find_range_uc:
  375. ; If there is no next range, ignore this MTRR,
  376. ; exit the loop and continue to next MTRR.
  377.         mov     edi, [esi]
  378.         test    edi, edi
  379.         jz      .next_uc_range
  380. ; If start of the next range is greater than or equal to ebp:ebx,
  381. ; exit the loop to 6e.
  382.         mov     eax, dword [edi+mtrr_range.start]
  383.         mov     edx, dword [edi+mtrr_range.start+4]
  384.         sub     eax, ebx
  385.         sbb     edx, ebp
  386.         jnb     .truncate_uc
  387. ; Otherwise, continue the loop if end of the next range is less than ebp:ebx,
  388. ; exit the loop to 6d otherwise.
  389.         mov     esi, edi
  390.         add     eax, dword [edi+mtrr_range.length]
  391.         adc     edx, dword [edi+mtrr_range.length+4]
  392.         jnb     .find_range_uc
  393. ; 6d. ebp:ebx is inside (or at end of) an existing range.
  394. ; Split the range. (The second range, maybe containing completely within UC-range,
  395. ; maybe of zero length, can be removed at step 6e, if needed.)
  396.         mov     edi, [.first_free_range]
  397.         test    edi, edi
  398.         jz      .abort
  399.         mov     dword [edi+mtrr_range.start], ebx
  400.         mov     dword [edi+mtrr_range.start+4], ebp
  401.         mov     dword [edi+mtrr_range.length], eax
  402.         mov     dword [edi+mtrr_range.length+4], edx
  403.         mov     eax, [edi+mtrr_range.next]
  404.         mov     [.first_free_range], eax
  405.         mov     eax, [esi+mtrr_range.next]
  406.         mov     [edi+mtrr_range.next], eax
  407. ; don't change [esi+mtrr_range.next] yet, it will be filled at step 6e
  408.         mov     eax, ebx
  409.         mov     edx, ebp
  410.         sub     eax, dword [esi+mtrr_range.start]
  411.         sbb     edx, dword [esi+mtrr_range.start+4]
  412.         mov     dword [esi+mtrr_range.length], eax
  413.         mov     dword [esi+mtrr_range.length+4], edx
  414. .truncate_uc:
  415. ; 6e. edi is the first range after ebp:ebx, check it and next ranges
  416. ; for intersection with the new range, truncate heads.
  417.         add     ebx, [ecx+8]
  418.         adc     ebp, [ecx+12]
  419. ; ebp:ebx = end of the range described by the current MTRR.
  420. .truncate_uc_loop:
  421. ; If start of the next range is greater than ebp:ebx,
  422. ; exit the loop to 6g.
  423.         mov     eax, ebx
  424.         mov     edx, ebp
  425.         sub     eax, dword [edi+mtrr_range.start]
  426.         sbb     edx, dword [edi+mtrr_range.start+4]
  427.         jb      .truncate_uc_done
  428. ; Otherwise, if end of the next range is greater than ebp:ebx,
  429. ; exit the loop to 6f.
  430.         sub     eax, dword [edi+mtrr_range.length]
  431.         sbb     edx, dword [edi+mtrr_range.length+4]
  432.         jb      .truncate_uc_last
  433. ; Otherwise, the current range is completely within the new range.
  434. ; Free it and continue the loop if there is a next range.
  435. ; If that was a last range, exit the loop to 6g.
  436.         mov     edx, [edi+mtrr_range.next]
  437.         mov     eax, [.first_free_range]
  438.         mov     [.first_free_range], edi
  439.         mov     [edi+mtrr_range.next], eax
  440.         mov     edi, edx
  441.         test    edi, edi
  442.         jnz     .truncate_uc_loop
  443.         jmp     .truncate_uc_done
  444. .truncate_uc_last:
  445. ; 6f. The range at edi partially intersects with the UC-range described by MTRR.
  446. ; Truncate it from the head.
  447.         mov     dword [edi+mtrr_range.start], ebx
  448.         mov     dword [edi+mtrr_range.start+4], ebp
  449.         neg     eax
  450.         adc     edx, 0
  451.         neg     edx
  452.         mov     dword [edi+mtrr_range.length], eax
  453.         mov     dword [edi+mtrr_range.length+4], edx
  454. .truncate_uc_done:
  455. ; 6g. We have found the next range (maybe 0) after intersection.
  456. ; Write it to [esi+mtrr_range.next].
  457.         mov     [esi+mtrr_range.next], edi
  458. .next_uc_range:
  459. ; 6h. Continue the loop 6b-6g over all configured MTRRs.
  460.         add     ecx, 16
  461.         cmp     ecx, [.mtrrs_end]
  462.         jb      .fill_uc_ranges
  463. ; Sanity check: if there are no ranges after steps 5-6,
  464. ; fallback to step 7. Otherwise, go to 8.
  465.         cmp     [.first_range], 0
  466.         jnz     .ranges_ok
  467. .fill_ranges_from_memory_map:
  468. ; 7. BIOS has not configured variable-range MTRRs.
  469. ; Create one range from 0 to [MEM_AMOUNT].
  470.         mov     eax, [.first_free_range]
  471.         mov     edx, [eax+mtrr_range.next]
  472.         mov     [.first_free_range], edx
  473.         mov     [.first_range], eax
  474.         xor     edx, edx
  475.         mov     [eax+mtrr_range.next], edx
  476.         mov     dword [eax+mtrr_range.start], edx
  477.         mov     dword [eax+mtrr_range.start+4], edx
  478.         mov     ecx, [MEM_AMOUNT]
  479.         mov     dword [eax+mtrr_range.length], ecx
  480.         mov     dword [eax+mtrr_range.length+4], edx
  481. .ranges_ok:
  482. ; 8. We have calculated list of WB-ranges.
  483. ; Now we should calculate a list of MTRRs so that
  484. ; * every MTRR describes a range with length = power of 2 and start that is aligned,
  485. ; * every MTRR can be WB or UC
  486. ; * (sum of all WB ranges) minus (sum of all UC ranges) equals the calculated list
  487. ; * top of 4G memory must not be covered by any ranges
  488. ; Example: range [0,0xBC000000) can be converted to
  489. ; [0,0x80000000)+[0x80000000,0xC0000000)-[0xBC000000,0xC0000000)
  490. ; WB            +WB                     -UC
  491. ; but not to [0,0x100000000)-[0xC0000000,0x100000000)-[0xBC000000,0xC0000000).
  492. ; 8a. Check that list of ranges is [0,something) plus, optionally, [4G,something).
  493. ; This holds in practice (see mtrrtest.asm for real-life examples)
  494. ; and significantly simplifies the code: ranges are independent, start of range
  495. ; is almost always aligned (the only exception >4G upper memory can be easily covered),
  496. ; there is no need to consider adding holes before start of range, only
  497. ; append them to end of range.
  498.         xor     eax, eax
  499.         mov     edi, [.first_range]
  500.         cmp     dword [edi+mtrr_range.start], eax
  501.         jnz     .abort
  502.         cmp     dword [edi+mtrr_range.start+4], eax
  503.         jnz     .abort
  504.         cmp     dword [edi+mtrr_range.length+4], eax
  505.         jnz     .abort
  506.         mov     edx, [edi+mtrr_range.next]
  507.         test    edx, edx
  508.         jz      @f
  509.         cmp     dword [edx+mtrr_range.start], eax
  510.         jnz     .abort
  511.         cmp     dword [edx+mtrr_range.start+4], 1
  512.         jnz     .abort
  513.         cmp     [edx+mtrr_range.next], eax
  514.         jnz     .abort
  515. @@:
  516. ; 8b. Initialize: no MTRRs filled.
  517.         mov     [.num_used_mtrrs], eax
  518.         lea     esi, [.mtrrs]
  519. .range2mtrr_loop:
  520. ; 8c. If we are dealing with upper-memory range (after 4G)
  521. ; with length > start, create one WB MTRR with [start,2*start),
  522. ; reset start to 2*start and return to this step.
  523. ; Example: [4G,24G) -> [4G,8G) {returning} + [8G,16G) {returning}
  524. ; + [16G,24G) {advancing to ?}.
  525.         mov     eax, dword [edi+mtrr_range.length+4]
  526.         test    eax, eax
  527.         jz      .less4G
  528.         mov     edx, dword [edi+mtrr_range.start+4]
  529.         cmp     eax, edx
  530.         jb      .start_aligned
  531.         inc     [.num_used_mtrrs]
  532.         cmp     [.num_used_mtrrs], MAX_USEFUL_MTRRS
  533.         ja      .abort
  534.         mov     dword [esi], MEM_WB
  535.         mov     dword [esi+4], edx
  536.         mov     dword [esi+8], 0
  537.         mov     dword [esi+12], edx
  538.         add     esi, 16
  539.         add     dword [edi+mtrr_range.start+4], edx
  540.         sub     dword [edi+mtrr_range.length+4], edx
  541.         jnz     .range2mtrr_loop
  542.         cmp     dword [edi+mtrr_range.length], 0
  543.         jz      .range2mtrr_next
  544. .less4G:
  545. ; 8d. If we are dealing with low-memory range (before 4G)
  546. ; and appending a maximal-size hole would create a range covering top of 4G,
  547. ; create a maximal-size WB range and return to this step.
  548. ; Example: for [0,0xBC000000) the following steps would consider
  549. ; variants [0,0x80000000)+(another range to be splitted) and
  550. ; [0,0x100000000)-(another range to be splitted); we forbid the last variant,
  551. ; so the first variant must be used.
  552.         bsr     ecx, dword [edi+mtrr_range.length]
  553.         xor     edx, edx
  554.         inc     edx
  555.         shl     edx, cl
  556.         lea     eax, [edx*2]
  557.         add     eax, dword [edi+mtrr_range.start]
  558.         jnz     .start_aligned
  559.         inc     [.num_used_mtrrs]
  560.         cmp     [.num_used_mtrrs], MAX_USEFUL_MTRRS
  561.         ja      .abort
  562.         mov     eax, dword [edi+mtrr_range.start]
  563.         mov     dword [esi], eax
  564.         or      dword [esi], MEM_WB
  565.         mov     dword [esi+4], 0
  566.         mov     dword [esi+8], edx
  567.         mov     dword [esi+12], 0
  568.         add     esi, 16
  569.         add     dword [edi+mtrr_range.start], edx
  570.         sub     dword [edi+mtrr_range.length], edx
  571.         jnz     .less4G
  572.         jmp     .range2mtrr_next
  573. .start_aligned:
  574. ; Start is aligned for any allowed length, maximum-size hole is allowed.
  575. ; Select the best MTRR configuration for one range.
  576. ; length=...101101
  577. ; Without hole at the end, we need one WB MTRR for every 1-bit in length:
  578. ; length=...100000 + ...001000 + ...000100 + ...000001
  579. ; We can also append one hole at the end so that one 0-bit (selected by us)
  580. ; becomes 1 and all lower bits become 0 for WB-range:
  581. ; length=...110000 - (...00010 + ...00001)
  582. ; In this way, we need one WB MTRR for every 1-bit higher than the selected bit,
  583. ; one WB MTRR for the selected bit, one UC MTRR for every 0-bit between
  584. ; the selected bit and lowest 1-bit (they become 1-bits after negation)
  585. ; and one UC MTRR for lowest 1-bit.
  586. ; So we need to select 0-bit with the maximal difference
  587. ; (number of 0-bits) - (number of 1-bits) between selected and lowest 1-bit,
  588. ; this equals the gain from using a hole. If the difference is negative for
  589. ; all 0-bits, don't append hole.
  590. ; Note that lowest 1-bit is not included when counting, but selected 0-bit is.
  591. ; 8e. Find the optimal bit position for hole.
  592. ; eax = current difference, ebx = best difference,
  593. ; ecx = hole bit position, edx = current bit position.
  594.         xor     eax, eax
  595.         xor     ebx, ebx
  596.         xor     ecx, ecx
  597.         bsf     edx, dword [edi+mtrr_range.length]
  598.         jnz     @f
  599.         bsf     edx, dword [edi+mtrr_range.length+4]
  600.         add     edx, 32
  601. @@:
  602.         push    edx     ; save position of lowest 1-bit for step 8f
  603. .calc_stat:
  604.         inc     edx
  605.         cmp     edx, 64
  606.         jae     .stat_done
  607.         inc     eax     ; increment difference in hope for 1-bit
  608. ; Note: bt conveniently works with both .length and .length+4,
  609. ; depending on whether edx>=32.
  610.         bt      dword [edi+mtrr_range.length], edx
  611.         jc      .calc_stat
  612.         dec     eax     ; hope was wrong, decrement difference to correct 'inc'
  613.         dec     eax     ; and again, now getting the real difference
  614.         cmp     eax, ebx
  615.         jle     .calc_stat
  616.         mov     ebx, eax
  617.         mov     ecx, edx
  618.         jmp     .calc_stat
  619. .stat_done:
  620. ; 8f. If we decided to create a hole, flip all bits between lowest and selected.
  621.         pop     edx     ; restore position of lowest 1-bit saved at step 8e
  622.         test    ecx, ecx
  623.         jz      .fill_hi_init
  624. @@:
  625.         inc     edx
  626.         cmp     edx, ecx
  627.         ja      .fill_hi_init
  628.         btc     dword [edi+mtrr_range.length], edx
  629.         jmp     @b
  630. .fill_hi_init:
  631. ; 8g. Create MTRR ranges corresponding to upper 32 bits.
  632.         sub     ecx, 32
  633. .fill_hi_loop:
  634.         bsr     edx, dword [edi+mtrr_range.length+4]
  635.         jz      .fill_hi_done
  636.         inc     [.num_used_mtrrs]
  637.         cmp     [.num_used_mtrrs], MAX_USEFUL_MTRRS
  638.         ja      .abort
  639.         mov     eax, dword [edi+mtrr_range.start]
  640.         mov     [esi], eax
  641.         mov     eax, dword [edi+mtrr_range.start+4]
  642.         mov     [esi+4], eax
  643.         xor     eax, eax
  644.         mov     [esi+8], eax
  645.         bts     eax, edx
  646.         mov     [esi+12], eax
  647.         cmp     edx, ecx
  648.         jl      .fill_hi_uc
  649.         or      dword [esi], MEM_WB
  650.         add     dword [edi+mtrr_range.start+4], eax
  651.         jmp     @f
  652. .fill_hi_uc:
  653.         sub     dword [esi+4], eax
  654.         sub     dword [edi+mtrr_range.start+4], eax
  655. @@:
  656.         add     esi, 16
  657.         sub     dword [edi+mtrr_range.length], eax
  658.         jmp     .fill_hi_loop
  659. .fill_hi_done:
  660. ; 8h. Create MTRR ranges corresponding to lower 32 bits.
  661.         add     ecx, 32
  662. .fill_lo_loop:
  663.         bsr     edx, dword [edi+mtrr_range.length]
  664.         jz      .range2mtrr_next
  665.         inc     [.num_used_mtrrs]
  666.         cmp     [.num_used_mtrrs], MAX_USEFUL_MTRRS
  667.         ja      .abort
  668.         mov     eax, dword [edi+mtrr_range.start]
  669.         mov     [esi], eax
  670.         mov     eax, dword [edi+mtrr_range.start+4]
  671.         mov     [esi+4], eax
  672.         xor     eax, eax
  673.         mov     [esi+12], eax
  674.         bts     eax, edx
  675.         mov     [esi+8], eax
  676.         cmp     edx, ecx
  677.         jl      .fill_lo_uc
  678.         or      dword [esi], MEM_WB
  679.         add     dword [edi+mtrr_range.start], eax
  680.         jmp     @f
  681. .fill_lo_uc:
  682.         sub     dword [esi], eax
  683.         sub     dword [edi+mtrr_range.start], eax
  684. @@:
  685.         add     esi, 16
  686.         sub     dword [edi+mtrr_range.length], eax
  687.         jmp     .fill_lo_loop
  688. .range2mtrr_next:
  689. ; 8i. Repeat the loop at 8c-8h for all ranges.
  690.         mov     edi, [edi+mtrr_range.next]
  691.         test    edi, edi
  692.         jnz     .range2mtrr_loop
  693. ; 9. We have calculated needed MTRRs, now setup them in the CPU.
  694. ; 9a. Abort if number of MTRRs is too large.
  695.         mov     eax, [num_variable_mtrrs]
  696.         cmp     [.num_used_mtrrs], eax
  697.         ja      .abort
  698.  
  699. ; 9b. Prepare for changes.
  700.         call    mtrr_begin_change
  701.  
  702. ; 9c. Prepare for loop over MTRRs.
  703.         lea     esi, [.mtrrs]
  704.         mov     ecx, 0x200
  705. @@:
  706. ; 9d. For every MTRR, copy PHYSBASEn as is: step 8 has configured
  707. ; start value and type bits as needed.
  708.         mov     eax, [esi]
  709.         mov     edx, [esi+4]
  710.         wrmsr
  711.         inc     ecx
  712. ; 9e. For every MTRR, calculate PHYSMASKn = -(length) or 0x800
  713. ; with upper bits cleared, 0x800 = MTRR is valid.
  714.         xor     eax, eax
  715.         xor     edx, edx
  716.         sub     eax, [esi+8]
  717.         sbb     edx, [esi+12]
  718.         or      eax, 0x800
  719.         or      edx, [.phys_reserved_mask]
  720.         xor     edx, [.phys_reserved_mask]
  721.         wrmsr
  722.         inc     ecx
  723. ; 9f. Continue steps 9d and 9e for all MTRRs calculated at step 8.
  724.         add     esi, 16
  725.         dec     [.num_used_mtrrs]
  726.         jnz     @b
  727. ; 9g. Zero other MTRRs.
  728.         xor     eax, eax
  729.         xor     edx, edx
  730.         mov     ebx, [num_variable_mtrrs]
  731.         lea     ebx, [0x200+ebx*2]
  732. @@:
  733.         cmp     ecx, ebx
  734.         jae     @f
  735.         wrmsr
  736.         inc     ecx
  737.         wrmsr
  738.         inc     ecx
  739.         jmp     @b
  740. @@:
  741.  
  742. ; 9i. Check PAT support and reprogram PAT_MASR for write combining memory
  743.         bt      [cpu_caps], CAPS_PAT
  744.         jnc     @F
  745.  
  746.         mov     ecx, MSR_CR_PAT
  747.         mov     eax, PAT_VALUE  ;UC UCM WC WB
  748.         mov     edx, eax
  749.         wrmsr
  750. @@:
  751.  
  752. ; 9j. Changes are done.
  753.         call    mtrr_end_change
  754.  
  755. .abort:
  756.         add     esp, .local_vars_size + MAX_RANGES * sizeof.mtrr_range
  757.         pop     ebp
  758.         ret
  759. endp
  760.  
  761. ; Allocate&set one MTRR for given range.
  762. ; size must be power of 2 that divides base.
  763. proc set_mtrr stdcall, base:dword,size:dword,mem_type:dword
  764. ; find unused register
  765.         mov     ecx, 0x201
  766. .scan:
  767.         mov     eax, [num_variable_mtrrs]
  768.         lea     eax, [0x200+eax*2]
  769.         cmp     ecx, eax
  770.         jae     .ret
  771.         rdmsr
  772.         dec     ecx
  773.         test    ah, 8
  774.         jz      .found
  775.         rdmsr
  776.         test    edx, edx
  777.         jnz     @f
  778.         and     eax, not 0xFFF  ; clear reserved bits
  779.         cmp     eax, [base]
  780.         jz      .ret
  781. @@:
  782.         add     ecx, 3
  783.         jmp     .scan
  784. ; no free registers, ignore the call
  785. .ret:
  786.         ret
  787. .found:
  788. ; found, write values
  789.         push    ecx
  790.         call    mtrr_begin_change
  791.         pop     ecx
  792.         xor     edx, edx
  793.         mov     eax, [base]
  794.         or      eax, [mem_type]
  795.         wrmsr
  796.  
  797.         mov     al, [cpu_phys_addr_width]
  798.         xor     edx, edx
  799.         bts     edx, eax
  800.         xor     eax, eax
  801.         sub     eax, [size]
  802.         sbb     edx, 0
  803.         or      eax, 0x800
  804.         inc     ecx
  805.         wrmsr
  806.         call    mtrr_end_change
  807.         ret
  808. endp
  809.  
  810. ; Helper procedure for mtrr_validate.
  811. ; Calculates memory type for given address according to variable-range MTRRs.
  812. ; Assumes that MTRRs are enabled.
  813. ; in: ebx = 32-bit physical address
  814. ; out: eax = memory type for ebx
  815. proc mtrr_get_real_type
  816. ; 1. Initialize: we have not yet found any MTRRs covering ebx.
  817.         push    0
  818.         mov     ecx, 0x201
  819. .mtrr_loop:
  820. ; 2. For every MTRR, check whether it is valid; if not, continue to the next MTRR.
  821.         rdmsr
  822.         dec     ecx
  823.         test    ah, 8
  824.         jz      .next
  825. ; 3. For every valid MTRR, check whether (ebx and PHYSMASKn) == PHYSBASEn,
  826. ; excluding low 12 bits.
  827.         and     eax, ebx
  828.         push    eax
  829.         rdmsr
  830.         test    edx, edx
  831.         pop     edx
  832.         jnz     .next
  833.         xor     edx, eax
  834.         and     edx, not 0xFFF
  835.         jnz     .next
  836. ; 4. If so, set the bit corresponding to memory type defined by this MTRR.
  837.         and     eax, 7
  838.         bts     [esp], eax
  839. .next:
  840. ; 5. Continue loop at 2-4 for all variable-range MTRRs.
  841.         add     ecx, 3
  842.         mov     eax, [num_variable_mtrrs]
  843.         lea     eax, [0x200+eax*2]
  844.         cmp     ecx, eax
  845.         jb      .mtrr_loop
  846. ; 6. If no MTRRs cover address in ebx, use default MTRR type from MTRR_DEF_CAP.
  847.         pop     edx
  848.         test    edx, edx
  849.         jz      .default
  850. ; 7. Find&clear 1-bit in edx.
  851.         bsf     eax, edx
  852.         btr     edx, eax
  853. ; 8. If there was only one 1-bit, then all MTRRs are consistent, return that bit.
  854.         test    edx, edx
  855.         jz      .nothing
  856. ; Otherwise, return MEM_UC (e.g. WB+UC is UC).
  857.         xor     eax, eax
  858. .nothing:
  859.         ret
  860. .default:
  861.         mov     ecx, 0x2FF
  862.         rdmsr
  863.         movzx   eax, al
  864.         ret
  865. endp
  866.  
  867. ; If MTRRs are configured improperly, this is not obvious to the user;
  868. ; everything works, but the performance can be horrible.
  869. ; Try to detect this and let the user know that the low performance
  870. ; is caused by some problem and is not a global property of the system.
  871. ; Let's hope he would report it to developers...
  872. proc mtrr_validate
  873. ; 1. If MTRRs are not supported, they cannot be configured improperly.
  874. ; Note: VirtualBox claims MTRR support in cpuid, but emulates MTRRCAP=0,
  875. ; which is efficiently equivalent to absent MTRRs.
  876. ; So check [num_variable_mtrrs] instead of CAPS_MTRR in [cpu_caps].
  877.         cmp     [num_variable_mtrrs], 0
  878.         jz      .exit
  879. ; 2. If variable-range MTRRs are not configured, this is a problem.
  880.         mov     ecx, 0x2FF
  881.         rdmsr
  882.         test    ah, 8
  883.         jz      .fail
  884. ; 3. Get the memory type for address somewhere inside working memory.
  885. ; It must be write-back.
  886.         mov     ebx, 0x27FFFF
  887.         call    mtrr_get_real_type
  888.         cmp     al, MEM_WB
  889.         jnz     .fail
  890. ; 4. If we're using a mode with LFB,
  891. ; get the memory type for last pixel of the framebuffer.
  892. ; It must be write-combined.
  893.         test    word [SCR_MODE], 0x4000
  894.         jz      .exit
  895.         mov     eax, [_display.lfb_pitch]
  896.         mul     [_display.height]
  897.         dec     eax
  898. ; LFB is mapped to virtual address LFB_BASE,
  899. ; it uses global pages if supported by CPU.
  900.         mov     ebx, [sys_proc+PROC.pdt_0+(LFB_BASE shr 20)]
  901.         test    ebx, PDE_LARGE
  902.         jnz     @f
  903.         mov     ebx, [page_tabs+(LFB_BASE shr 10)]
  904. @@:
  905.         and     ebx, not 0xFFF
  906.         add     ebx, eax
  907.         call    mtrr_get_real_type
  908.         cmp     al, MEM_WC
  909.         jz      .exit
  910. ; 5. The check at step 4 fails on Bochs:
  911. ; Bochs BIOS configures MTRRs in a strange way not respecting [cpu_phys_addr_width],
  912. ; so mtrr_reconfigure avoids to touch anything.
  913. ; However, Bochs core ignores MTRRs (keeping them only for rdmsr/wrmsr),
  914. ; so we don't care about proper setting for Bochs.
  915. ; Use northbridge PCI id to detect Bochs: it emulates either i440fx or i430fx
  916. ; depending on configuration file.
  917.         mov     eax, [pcidev_list.fd]
  918.         cmp     eax, pcidev_list        ; sanity check: fail if no PCI devices
  919.         jz      .fail
  920.         cmp     [eax+PCIDEV.vendor_device_id], 0x12378086
  921.         jz      .exit
  922.         cmp     [eax+PCIDEV.vendor_device_id], 0x01228086
  923.         jnz     .fail
  924. .exit:
  925.         ret
  926. .fail:
  927.         mov     ebx, mtrr_user_message
  928.         mov     ebp, notifyapp
  929.         call    fs_execute_from_sysdir_param
  930.         ret
  931. endp
  932.