Subversion Repositories Kolibri OS

Rev

Rev 5360 | Rev 6663 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
2288 clevermous 1
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2
;;                                                              ;;
5363 yogev_ezra 3
;; Copyright (C) KolibriOS team 2004-2015. All rights reserved. ;;
2288 clevermous 4
;; Distributed under terms of the GNU General Public License    ;;
5
;;                                                              ;;
6
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
7
 
8
$Revision: 5363 $
9
 
4608 clevermous 10
; Initializes MTRRs.
11
proc init_mtrr
2288 clevermous 12
 
4608 clevermous 13
        cmp     [BOOT_VARS+BOOT_MTRR], byte 2
14
        je      .exit
2288 clevermous 15
 
4608 clevermous 16
        bt      [cpu_caps], CAPS_MTRR
17
        jnc     .exit
2288 clevermous 18
 
4608 clevermous 19
        call    mtrr_reconfigure
20
        stdcall set_mtrr, [LFBAddress], 0x1000000, MEM_WC
2288 clevermous 21
 
4608 clevermous 22
.exit:
2288 clevermous 23
        ret
24
endp
25
 
4608 clevermous 26
; Helper procedure for mtrr_reconfigure and set_mtrr,
27
; called before changes in MTRRs.
5360 serge 28
; 1. disable and flush caches
29
; 2. clear PGE bit in cr4
30
; 3. flush TLB
31
; 4. disable mtrr
32
 
4608 clevermous 33
proc mtrr_begin_change
34
        mov     eax, cr0
35
        or      eax, 0x60000000 ;disable caching
36
        mov     cr0, eax
37
        wbinvd                  ;invalidate cache
5360 serge 38
 
39
        bt      [cpu_caps], CAPS_PGE
40
        jnc     .cr3_flush
41
 
42
        mov     eax, cr4
43
        btr     eax, 7          ;clear cr4.PGE
44
        mov     cr4, eax        ;flush TLB
45
        jmp     @F              ;skip extra serialization
46
 
47
.cr3_flush:
48
        mov     eax, cr3
49
        mov     cr3, eax        ;flush TLB
50
@@:
51
        mov     ecx, MSR_MTRR_DEF_TYPE
52
        rdmsr
53
        btr     eax, 11         ;clear enable flag
54
        wrmsr                   ;disable mtrr
2288 clevermous 55
        ret
56
endp
57
 
4608 clevermous 58
; Helper procedure for mtrr_reconfigure and set_mtrr,
59
; called after changes in MTRRs.
5360 serge 60
; 1. enable mtrr
61
; 2. flush all caches
62
; 3. flush TLB
63
; 4. restore cr4.PGE flag, if required
64
 
4608 clevermous 65
proc mtrr_end_change
5360 serge 66
        mov     ecx, MSR_MTRR_DEF_TYPE
67
        rdmsr
68
        or      ah, 8           ; enable variable-ranges MTRR
69
        and     al, 0xF0        ; default memtype = UC
70
        wrmsr
71
 
4608 clevermous 72
        wbinvd                  ;again invalidate
73
        mov     eax, cr0
74
        and     eax, not 0x60000000
75
        mov     cr0, eax        ; enable caching
5360 serge 76
 
77
        mov     eax, cr3
78
        mov     cr3, eax        ;flush tlb
79
 
80
        bt      [cpu_caps], CAPS_PGE
81
        jnc     @F
82
 
83
        mov     eax, cr4
84
        bts     eax, 7          ;set cr4.PGE flag
85
        mov     cr4, eax
86
@@:
2288 clevermous 87
        ret
88
endp
89
 
4608 clevermous 90
; Some limits to number of structures located in the stack.
91
MAX_USEFUL_MTRRS = 16
92
MAX_RANGES = 16
2288 clevermous 93
 
4608 clevermous 94
; mtrr_reconfigure keeps a list of MEM_WB ranges.
95
; This structure describes one item in the list.
96
struct mtrr_range
97
next            dd      ?       ; next item
98
start           dq      ?       ; first byte
99
length          dq      ?       ; length in bytes
100
ends
2288 clevermous 101
 
4608 clevermous 102
uglobal
2288 clevermous 103
align 4
4608 clevermous 104
num_variable_mtrrs      dd      0       ; number of variable-range MTRRs
105
endg
2288 clevermous 106
 
4608 clevermous 107
; Helper procedure for MTRR initialization.
108
; Takes MTRR configured by BIOS and tries to recongifure them
109
; in order to allow non-UC data at top of 4G memory.
110
; Example: if low part of physical memory is 3.5G = 0xE0000000 bytes wide,
111
; BIOS can configure two MTRRs so that the first MTRR describes [0, 4G) as WB
112
; and the second MTRR describes [3.5G, 4G) as UC;
113
; WB+UC=UC, so the resulting memory map would be as needed,
114
; but in this configuration our attempts to map LFB at (say) 0xE8000000 as WC
115
; would be ignored, WB+UC+WC is still UC.
116
; So we must keep top of 4G memory not covered by MTRRs,
117
; using three WB MTRRs [0,2G) + [2G,3G) + [3G,3.5G),
118
; this gives the same memory map, but allows to add further entries.
119
; See mtrrtest.asm for detailed input/output from real hardware+BIOS.
120
proc mtrr_reconfigure
121
        push    ebp     ; we're called from init_LFB, and it feels hurt when ebp is destroyed
122
; 1. Prepare local variables.
123
; 1a. Create list of MAX_RANGES free (aka not yet allocated) ranges.
2288 clevermous 124
        xor     eax, eax
4608 clevermous 125
        lea     ecx, [eax+MAX_RANGES]
126
.init_ranges:
127
        sub     esp, sizeof.mtrr_range - 4
128
        push    eax
129
        mov     eax, esp
130
        dec     ecx
131
        jnz     .init_ranges
132
        mov     eax, esp
133
; 1b. Fill individual local variables.
134
        xor     edx, edx
135
        sub     esp, MAX_USEFUL_MTRRS * 16      ; .mtrrs
136
        push    edx             ; .mtrrs_end
137
        push    edx             ; .num_used_mtrrs
138
        push    eax             ; .first_free_range
139
        push    edx             ; .first_range: no ranges yet
140
        mov     cl, [cpu_phys_addr_width]
141
        or      eax, -1
142
        shl     eax, cl ; note: this uses cl&31 = cl-32, not the entire cl
143
        push    eax     ; .phys_reserved_mask
144
virtual at esp
145
.phys_reserved_mask     dd      ?
146
.first_range            dd      ?
147
.first_free_range       dd      ?
148
.num_used_mtrrs         dd      ?
149
.mtrrs_end              dd      ?
150
.mtrrs          rq      MAX_USEFUL_MTRRS * 2
151
.local_vars_size = $ - esp
152
end virtual
2288 clevermous 153
 
4608 clevermous 154
; 2. Get the number of variable-range MTRRs from MTRRCAP register.
155
; Abort if zero.
156
        mov     ecx, 0xFE
157
        rdmsr
158
        test    al, al
159
        jz      .abort
160
        mov     byte [num_variable_mtrrs], al
161
; 3. Validate MTRR_DEF_TYPE register.
162
        mov     ecx, 0x2FF
163
        rdmsr
164
; If BIOS has not initialized variable-range MTRRs, fallback to step 7.
165
        test    ah, 8
166
        jz      .fill_ranges_from_memory_map
167
; If the default memory type (not covered by MTRRs) is not UC,
168
; then probably BIOS did something strange, so it is better to exit immediately
169
; hoping for the best.
170
        cmp     al, MEM_UC
171
        jnz     .abort
172
; 4. Validate all variable-range MTRRs
173
; and copy configured MTRRs to the local array [.mtrrs].
174
; 4a. Prepare for the loop over existing variable-range MTRRs.
175
        mov     ecx, 0x200
176
        lea     edi, [.mtrrs]
177
.get_used_mtrrs_loop:
178
; 4b. For every MTRR, read PHYSBASEn and PHYSMASKn.
179
; In PHYSBASEn, clear upper bits and copy to ebp:ebx.
180
        rdmsr
181
        or      edx, [.phys_reserved_mask]
182
        xor     edx, [.phys_reserved_mask]
183
        mov     ebp, edx
2288 clevermous 184
        mov     ebx, eax
4608 clevermous 185
        inc     ecx
186
; If PHYSMASKn is not active, ignore this MTRR.
187
        rdmsr
188
        inc     ecx
189
        test    ah, 8
190
        jz      .get_used_mtrrs_next
191
; 4c. For every active MTRR, check that number of local entries is not too large.
192
        inc     [.num_used_mtrrs]
193
        cmp     [.num_used_mtrrs], MAX_USEFUL_MTRRS
194
        ja      .abort
195
; 4d. For every active MTRR, store PHYSBASEn with upper bits cleared.
196
; This contains the MTRR base and the memory type in low byte.
197
        mov     [edi], ebx
198
        mov     [edi+4], ebp
199
; 4e. For every active MTRR, check that the range is continuous:
200
; PHYSMASKn with upper bits set must be negated power of two, and
201
; low bits of PHYSBASEn must be zeroes:
202
; PHYSMASKn = 1...10...0,
203
; PHYSBASEn = x...x0...0,
204
; this defines a continuous range from x...x0...0 to x...x1...1,
205
; length = 10...0 = negated PHYSMASKn.
206
; Store length in the local array.
207
        and     eax, not 0xFFF
208
        or      edx, [.phys_reserved_mask]
209
        mov     dword [edi+8], 0
210
        mov     dword [edi+12], 0
211
        sub     [edi+8], eax
212
        sbb     [edi+12], edx
213
; (x and -x) is the maximum power of two that divides x.
214
; Condition for powers of two: (x and -x) equals x.
215
        and     eax, [edi+8]
216
        and     edx, [edi+12]
217
        cmp     eax, [edi+8]
218
        jnz     .abort
219
        cmp     edx, [edi+12]
220
        jnz     .abort
221
        sub     eax, 1
222
        sbb     edx, 0
223
        and     eax, not 0xFFF
224
        and     eax, ebx
225
        jnz     .abort
226
        and     edx, ebp
227
        jnz     .abort
228
; 4f. For every active MTRR, validate memory type: it must be either WB or UC.
229
        add     edi, 16
230
        cmp     bl, MEM_UC
231
        jz      .get_used_mtrrs_next
232
        cmp     bl, MEM_WB
233
        jnz     .abort
234
.get_used_mtrrs_next:
235
; 4g. Repeat the loop at 4b-4f for all [num_variable_mtrrs] entries.
236
        mov     eax, [num_variable_mtrrs]
237
        lea     eax, [0x200+eax*2]
238
        cmp     ecx, eax
239
        jb      .get_used_mtrrs_loop
240
; 4h. If no active MTRRs were detected, fallback to step 7.
241
        cmp     [.num_used_mtrrs], 0
242
        jz      .fill_ranges_from_memory_map
243
        mov     [.mtrrs_end], edi
244
; 5. Generate sorted list of ranges marked as WB.
245
; 5a. Prepare for the loop over configured MTRRs filled at step 4.
246
        lea     ecx, [.mtrrs]
247
.fill_wb_ranges:
248
; 5b. Ignore non-WB MTRRs.
249
        mov     ebx, [ecx]
250
        cmp     bl, MEM_WB
251
        jnz     .next_wb_range
252
        mov     ebp, [ecx+4]
253
        and     ebx, not 0xFFF  ; clear memory type and reserved bits
254
; ebp:ebx = start of the range described by the current MTRR.
255
; 5c. Find the first existing range containing a point greater than ebp:ebx.
256
        lea     esi, [.first_range]
257
.find_range_wb:
258
; If there is no next range or start of the next range is greater than ebp:ebx,
259
; exit the loop to 5d.
260
        mov     edi, [esi]
261
        test    edi, edi
262
        jz      .found_place_wb
263
        mov     eax, ebx
264
        mov     edx, ebp
265
        sub     eax, dword [edi+mtrr_range.start]
266
        sbb     edx, dword [edi+mtrr_range.start+4]
267
        jb      .found_place_wb
268
; Otherwise, if end of the next range is greater than or equal to ebp:ebx,
269
; exit the loop to 5e.
270
        mov     esi, edi
271
        sub     eax, dword [edi+mtrr_range.length]
272
        sbb     edx, dword [edi+mtrr_range.length+4]
273
        jb      .expand_wb
274
        or      eax, edx
275
        jnz     .find_range_wb
276
        jmp     .expand_wb
277
.found_place_wb:
278
; 5d. ebp:ebx is not within any existing range.
279
; Insert a new range between esi and edi.
280
; (Later, during 5e, it can be merged with the following ranges.)
281
        mov     eax, [.first_free_range]
282
        test    eax, eax
283
        jz      .abort
284
        mov     [esi], eax
285
        mov     edx, [eax+mtrr_range.next]
286
        mov     [.first_free_range], edx
287
        mov     dword [eax+mtrr_range.start], ebx
288
        mov     dword [eax+mtrr_range.start+4], ebp
289
; Don't fill [eax+mtrr_range.next] and [eax+mtrr_range.length] yet,
290
; they will be calculated including merges at step 5e.
291
        mov     esi, edi
2288 clevermous 292
        mov     edi, eax
4608 clevermous 293
.expand_wb:
294
; 5e. The range at edi contains ebp:ebx, and esi points to the first range
295
; to be checked for merge: esi=edi if ebp:ebx was found in an existing range,
296
; esi is next after edi if a new range with ebp:ebx was created.
297
; Merge it with following ranges while start of the next range is not greater
298
; than the end of the new range.
299
        add     ebx, [ecx+8]
300
        adc     ebp, [ecx+12]
301
; ebp:ebx = end of the range described by the current MTRR.
302
.expand_wb_loop:
303
; If there is no next range or start of the next range is greater than ebp:ebx,
304
; exit the loop to 5g.
305
        test    esi, esi
306
        jz      .expand_wb_done
307
        mov     eax, ebx
308
        mov     edx, ebp
309
        sub     eax, dword [esi+mtrr_range.start]
310
        sbb     edx, dword [esi+mtrr_range.start+4]
311
        jb      .expand_wb_done
312
; Otherwise, if end of the next range is greater than or equal to ebp:ebx,
313
; exit the loop to 5f.
314
        sub     eax, dword [esi+mtrr_range.length]
315
        sbb     edx, dword [esi+mtrr_range.length+4]
316
        jb      .expand_wb_last
317
; Otherwise, the current range is completely within the new range.
318
; Free it and continue the loop.
319
        mov     edx, [esi+mtrr_range.next]
320
        cmp     esi, edi
321
        jz      @f
322
        mov     eax, [.first_free_range]
323
        mov     [esi+mtrr_range.next], eax
324
        mov     [.first_free_range], esi
2288 clevermous 325
@@:
4608 clevermous 326
        mov     esi, edx
327
        jmp     .expand_wb_loop
328
.expand_wb_last:
329
; 5f. Start of the new range is inside range described by esi,
330
; end of the new range is inside range described by edi.
331
; If esi is equal to edi, the new range is completely within
332
; an existing range, so proceed to the next range.
333
        cmp     esi, edi
334
        jz      .next_wb_range
335
; Otherwise, set end of interval at esi to end of interval at edi
336
; and free range described by edi.
337
        mov     ebx, dword [esi+mtrr_range.start]
338
        mov     ebp, dword [esi+mtrr_range.start+4]
339
        add     ebx, dword [esi+mtrr_range.length]
340
        adc     ebp, dword [esi+mtrr_range.length+4]
341
        mov     edx, [esi+mtrr_range.next]
342
        mov     eax, [.first_free_range]
343
        mov     [esi+mtrr_range.next], eax
344
        mov     [.first_free_range], esi
345
        mov     esi, edx
346
.expand_wb_done:
347
; 5g. We have found the next range (maybe 0) after merging and
348
; the new end of range (maybe ebp:ebx from the new range
349
; or end of another existing interval calculated at step 5f).
350
; Write them to range at edi.
351
        mov     [edi+mtrr_range.next], esi
352
        sub     ebx, dword [edi+mtrr_range.start]
353
        sbb     ebp, dword [edi+mtrr_range.start+4]
354
        mov     dword [edi+mtrr_range.length], ebx
355
        mov     dword [edi+mtrr_range.length+4], ebp
356
.next_wb_range:
357
; 5h. Continue the loop 5b-5g over all configured MTRRs.
358
        add     ecx, 16
359
        cmp     ecx, [.mtrrs_end]
360
        jb      .fill_wb_ranges
361
; 6. Exclude all ranges marked as UC.
362
; 6a. Prepare for the loop over configured MTRRs filled at step 4.
363
        lea     ecx, [.mtrrs]
364
.fill_uc_ranges:
365
; 6b. Ignore non-UC MTRRs.
366
        mov     ebx, [ecx]
367
        cmp     bl, MEM_UC
368
        jnz     .next_uc_range
369
        mov     ebp, [ecx+4]
370
        and     ebx, not 0xFFF  ; clear memory type and reserved bits
371
; ebp:ebx = start of the range described by the current MTRR.
372
        lea     esi, [.first_range]
373
; 6c. Find the first existing range containing a point greater than ebp:ebx.
374
.find_range_uc:
375
; If there is no next range, ignore this MTRR,
376
; exit the loop and continue to next MTRR.
377
        mov     edi, [esi]
378
        test    edi, edi
379
        jz      .next_uc_range
380
; If start of the next range is greater than or equal to ebp:ebx,
381
; exit the loop to 6e.
382
        mov     eax, dword [edi+mtrr_range.start]
383
        mov     edx, dword [edi+mtrr_range.start+4]
384
        sub     eax, ebx
385
        sbb     edx, ebp
386
        jnb     .truncate_uc
387
; Otherwise, continue the loop if end of the next range is less than ebp:ebx,
388
; exit the loop to 6d otherwise.
389
        mov     esi, edi
390
        add     eax, dword [edi+mtrr_range.length]
391
        adc     edx, dword [edi+mtrr_range.length+4]
392
        jnb     .find_range_uc
393
; 6d. ebp:ebx is inside (or at end of) an existing range.
394
; Split the range. (The second range, maybe containing completely within UC-range,
395
; maybe of zero length, can be removed at step 6e, if needed.)
396
        mov     edi, [.first_free_range]
397
        test    edi, edi
398
        jz      .abort
399
        mov     dword [edi+mtrr_range.start], ebx
400
        mov     dword [edi+mtrr_range.start+4], ebp
401
        mov     dword [edi+mtrr_range.length], eax
402
        mov     dword [edi+mtrr_range.length+4], edx
403
        mov     eax, [edi+mtrr_range.next]
404
        mov     [.first_free_range], eax
405
        mov     eax, [esi+mtrr_range.next]
406
        mov     [edi+mtrr_range.next], eax
407
; don't change [esi+mtrr_range.next] yet, it will be filled at step 6e
408
        mov     eax, ebx
409
        mov     edx, ebp
410
        sub     eax, dword [esi+mtrr_range.start]
411
        sbb     edx, dword [esi+mtrr_range.start+4]
412
        mov     dword [esi+mtrr_range.length], eax
413
        mov     dword [esi+mtrr_range.length+4], edx
414
.truncate_uc:
415
; 6e. edi is the first range after ebp:ebx, check it and next ranges
416
; for intersection with the new range, truncate heads.
417
        add     ebx, [ecx+8]
418
        adc     ebp, [ecx+12]
419
; ebp:ebx = end of the range described by the current MTRR.
420
.truncate_uc_loop:
421
; If start of the next range is greater than ebp:ebx,
422
; exit the loop to 6g.
423
        mov     eax, ebx
424
        mov     edx, ebp
425
        sub     eax, dword [edi+mtrr_range.start]
426
        sbb     edx, dword [edi+mtrr_range.start+4]
427
        jb      .truncate_uc_done
428
; Otherwise, if end of the next range is greater than ebp:ebx,
429
; exit the loop to 6f.
430
        sub     eax, dword [edi+mtrr_range.length]
431
        sbb     edx, dword [edi+mtrr_range.length+4]
432
        jb      .truncate_uc_last
433
; Otherwise, the current range is completely within the new range.
434
; Free it and continue the loop if there is a next range.
435
; If that was a last range, exit the loop to 6g.
436
        mov     edx, [edi+mtrr_range.next]
437
        mov     eax, [.first_free_range]
438
        mov     [.first_free_range], edi
439
        mov     [edi+mtrr_range.next], eax
440
        mov     edi, edx
441
        test    edi, edi
442
        jnz     .truncate_uc_loop
443
        jmp     .truncate_uc_done
444
.truncate_uc_last:
445
; 6f. The range at edi partially intersects with the UC-range described by MTRR.
446
; Truncate it from the head.
447
        mov     dword [edi+mtrr_range.start], ebx
448
        mov     dword [edi+mtrr_range.start+4], ebp
449
        neg     eax
450
        adc     edx, 0
451
        neg     edx
452
        mov     dword [edi+mtrr_range.length], eax
453
        mov     dword [edi+mtrr_range.length+4], edx
454
.truncate_uc_done:
455
; 6g. We have found the next range (maybe 0) after intersection.
456
; Write it to [esi+mtrr_range.next].
457
        mov     [esi+mtrr_range.next], edi
458
.next_uc_range:
459
; 6h. Continue the loop 6b-6g over all configured MTRRs.
460
        add     ecx, 16
461
        cmp     ecx, [.mtrrs_end]
462
        jb      .fill_uc_ranges
463
; Sanity check: if there are no ranges after steps 5-6,
464
; fallback to step 7. Otherwise, go to 8.
465
        cmp     [.first_range], 0
466
        jnz     .ranges_ok
467
.fill_ranges_from_memory_map:
468
; 7. BIOS has not configured variable-range MTRRs.
469
; Create one range from 0 to [MEM_AMOUNT].
470
        mov     eax, [.first_free_range]
471
        mov     edx, [eax+mtrr_range.next]
472
        mov     [.first_free_range], edx
473
        mov     [.first_range], eax
474
        xor     edx, edx
475
        mov     [eax+mtrr_range.next], edx
476
        mov     dword [eax+mtrr_range.start], edx
477
        mov     dword [eax+mtrr_range.start+4], edx
478
        mov     ecx, [MEM_AMOUNT]
479
        mov     dword [eax+mtrr_range.length], ecx
480
        mov     dword [eax+mtrr_range.length+4], edx
481
.ranges_ok:
482
; 8. We have calculated list of WB-ranges.
483
; Now we should calculate a list of MTRRs so that
484
; * every MTRR describes a range with length = power of 2 and start that is aligned,
485
; * every MTRR can be WB or UC
486
; * (sum of all WB ranges) minus (sum of all UC ranges) equals the calculated list
487
; * top of 4G memory must not be covered by any ranges
488
; Example: range [0,0xBC000000) can be converted to
489
; [0,0x80000000)+[0x80000000,0xC0000000)-[0xBC000000,0xC0000000)
490
; WB            +WB                     -UC
491
; but not to [0,0x100000000)-[0xC0000000,0x100000000)-[0xBC000000,0xC0000000).
492
; 8a. Check that list of ranges is [0,something) plus, optionally, [4G,something).
493
; This holds in practice (see mtrrtest.asm for real-life examples)
494
; and significantly simplifies the code: ranges are independent, start of range
495
; is almost always aligned (the only exception >4G upper memory can be easily covered),
496
; there is no need to consider adding holes before start of range, only
497
; append them to end of range.
2288 clevermous 498
        xor     eax, eax
4608 clevermous 499
        mov     edi, [.first_range]
500
        cmp     dword [edi+mtrr_range.start], eax
501
        jnz     .abort
502
        cmp     dword [edi+mtrr_range.start+4], eax
503
        jnz     .abort
504
        cmp     dword [edi+mtrr_range.length+4], eax
505
        jnz     .abort
506
        mov     edx, [edi+mtrr_range.next]
507
        test    edx, edx
508
        jz      @f
509
        cmp     dword [edx+mtrr_range.start], eax
510
        jnz     .abort
511
        cmp     dword [edx+mtrr_range.start+4], 1
512
        jnz     .abort
513
        cmp     [edx+mtrr_range.next], eax
514
        jnz     .abort
3166 clevermous 515
@@:
4608 clevermous 516
; 8b. Initialize: no MTRRs filled.
517
        mov     [.num_used_mtrrs], eax
518
        lea     esi, [.mtrrs]
519
.range2mtrr_loop:
520
; 8c. If we are dealing with upper-memory range (after 4G)
521
; with length > start, create one WB MTRR with [start,2*start),
522
; reset start to 2*start and return to this step.
523
; Example: [4G,24G) -> [4G,8G) {returning} + [8G,16G) {returning}
524
; + [16G,24G) {advancing to ?}.
525
        mov     eax, dword [edi+mtrr_range.length+4]
2288 clevermous 526
        test    eax, eax
4608 clevermous 527
        jz      .less4G
528
        mov     edx, dword [edi+mtrr_range.start+4]
529
        cmp     eax, edx
530
        jb      .start_aligned
531
        inc     [.num_used_mtrrs]
532
        cmp     [.num_used_mtrrs], MAX_USEFUL_MTRRS
533
        ja      .abort
534
        mov     dword [esi], MEM_WB
535
        mov     dword [esi+4], edx
536
        mov     dword [esi+8], 0
537
        mov     dword [esi+12], edx
538
        add     esi, 16
539
        add     dword [edi+mtrr_range.start+4], edx
540
        sub     dword [edi+mtrr_range.length+4], edx
541
        jnz     .range2mtrr_loop
542
        cmp     dword [edi+mtrr_range.length], 0
543
        jz      .range2mtrr_next
544
.less4G:
545
; 8d. If we are dealing with low-memory range (before 4G)
546
; and appending a maximal-size hole would create a range covering top of 4G,
547
; create a maximal-size WB range and return to this step.
548
; Example: for [0,0xBC000000) the following steps would consider
549
; variants [0,0x80000000)+(another range to be splitted) and
550
; [0,0x100000000)-(another range to be splitted); we forbid the last variant,
551
; so the first variant must be used.
552
        bsr     ecx, dword [edi+mtrr_range.length]
553
        xor     edx, edx
554
        inc     edx
555
        shl     edx, cl
556
        lea     eax, [edx*2]
557
        add     eax, dword [edi+mtrr_range.start]
558
        jnz     .start_aligned
559
        inc     [.num_used_mtrrs]
560
        cmp     [.num_used_mtrrs], MAX_USEFUL_MTRRS
561
        ja      .abort
562
        mov     eax, dword [edi+mtrr_range.start]
563
        mov     dword [esi], eax
564
        or      dword [esi], MEM_WB
565
        mov     dword [esi+4], 0
566
        mov     dword [esi+8], edx
567
        mov     dword [esi+12], 0
568
        add     esi, 16
569
        add     dword [edi+mtrr_range.start], edx
570
        sub     dword [edi+mtrr_range.length], edx
571
        jnz     .less4G
572
        jmp     .range2mtrr_next
573
.start_aligned:
574
; Start is aligned for any allowed length, maximum-size hole is allowed.
575
; Select the best MTRR configuration for one range.
576
; length=...101101
577
; Without hole at the end, we need one WB MTRR for every 1-bit in length:
578
; length=...100000 + ...001000 + ...000100 + ...000001
579
; We can also append one hole at the end so that one 0-bit (selected by us)
580
; becomes 1 and all lower bits become 0 for WB-range:
581
; length=...110000 - (...00010 + ...00001)
582
; In this way, we need one WB MTRR for every 1-bit higher than the selected bit,
583
; one WB MTRR for the selected bit, one UC MTRR for every 0-bit between
584
; the selected bit and lowest 1-bit (they become 1-bits after negation)
585
; and one UC MTRR for lowest 1-bit.
586
; So we need to select 0-bit with the maximal difference
587
; (number of 0-bits) - (number of 1-bits) between selected and lowest 1-bit,
588
; this equals the gain from using a hole. If the difference is negative for
589
; all 0-bits, don't append hole.
590
; Note that lowest 1-bit is not included when counting, but selected 0-bit is.
591
; 8e. Find the optimal bit position for hole.
592
; eax = current difference, ebx = best difference,
593
; ecx = hole bit position, edx = current bit position.
2288 clevermous 594
        xor     eax, eax
4608 clevermous 595
        xor     ebx, ebx
596
        xor     ecx, ecx
597
        bsf     edx, dword [edi+mtrr_range.length]
598
        jnz     @f
599
        bsf     edx, dword [edi+mtrr_range.length+4]
600
        add     edx, 32
2288 clevermous 601
@@:
4608 clevermous 602
        push    edx     ; save position of lowest 1-bit for step 8f
603
.calc_stat:
2288 clevermous 604
        inc     edx
4608 clevermous 605
        cmp     edx, 64
606
        jae     .stat_done
607
        inc     eax     ; increment difference in hope for 1-bit
608
; Note: bt conveniently works with both .length and .length+4,
609
; depending on whether edx>=32.
610
        bt      dword [edi+mtrr_range.length], edx
611
        jc      .calc_stat
612
        dec     eax     ; hope was wrong, decrement difference to correct 'inc'
613
        dec     eax     ; and again, now getting the real difference
614
        cmp     eax, ebx
615
        jle     .calc_stat
616
        mov     ebx, eax
617
        mov     ecx, edx
618
        jmp     .calc_stat
619
.stat_done:
620
; 8f. If we decided to create a hole, flip all bits between lowest and selected.
621
        pop     edx     ; restore position of lowest 1-bit saved at step 8e
622
        test    ecx, ecx
623
        jz      .fill_hi_init
2288 clevermous 624
@@:
625
        inc     edx
4608 clevermous 626
        cmp     edx, ecx
627
        ja      .fill_hi_init
628
        btc     dword [edi+mtrr_range.length], edx
629
        jmp     @b
630
.fill_hi_init:
631
; 8g. Create MTRR ranges corresponding to upper 32 bits.
632
        sub     ecx, 32
633
.fill_hi_loop:
634
        bsr     edx, dword [edi+mtrr_range.length+4]
635
        jz      .fill_hi_done
636
        inc     [.num_used_mtrrs]
637
        cmp     [.num_used_mtrrs], MAX_USEFUL_MTRRS
638
        ja      .abort
639
        mov     eax, dword [edi+mtrr_range.start]
640
        mov     [esi], eax
641
        mov     eax, dword [edi+mtrr_range.start+4]
642
        mov     [esi+4], eax
643
        xor     eax, eax
644
        mov     [esi+8], eax
645
        bts     eax, edx
646
        mov     [esi+12], eax
647
        cmp     edx, ecx
648
        jl      .fill_hi_uc
649
        or      dword [esi], MEM_WB
650
        add     dword [edi+mtrr_range.start+4], eax
651
        jmp     @f
652
.fill_hi_uc:
653
        sub     dword [esi+4], eax
654
        sub     dword [edi+mtrr_range.start+4], eax
2288 clevermous 655
@@:
4608 clevermous 656
        add     esi, 16
657
        sub     dword [edi+mtrr_range.length], eax
658
        jmp     .fill_hi_loop
659
.fill_hi_done:
660
; 8h. Create MTRR ranges corresponding to lower 32 bits.
661
        add     ecx, 32
662
.fill_lo_loop:
663
        bsr     edx, dword [edi+mtrr_range.length]
664
        jz      .range2mtrr_next
665
        inc     [.num_used_mtrrs]
666
        cmp     [.num_used_mtrrs], MAX_USEFUL_MTRRS
667
        ja      .abort
668
        mov     eax, dword [edi+mtrr_range.start]
669
        mov     [esi], eax
670
        mov     eax, dword [edi+mtrr_range.start+4]
671
        mov     [esi+4], eax
672
        xor     eax, eax
673
        mov     [esi+12], eax
674
        bts     eax, edx
675
        mov     [esi+8], eax
676
        cmp     edx, ecx
677
        jl      .fill_lo_uc
678
        or      dword [esi], MEM_WB
679
        add     dword [edi+mtrr_range.start], eax
680
        jmp     @f
681
.fill_lo_uc:
682
        sub     dword [esi], eax
683
        sub     dword [edi+mtrr_range.start], eax
2288 clevermous 684
@@:
4608 clevermous 685
        add     esi, 16
686
        sub     dword [edi+mtrr_range.length], eax
687
        jmp     .fill_lo_loop
688
.range2mtrr_next:
689
; 8i. Repeat the loop at 8c-8h for all ranges.
690
        mov     edi, [edi+mtrr_range.next]
2288 clevermous 691
        test    edi, edi
4608 clevermous 692
        jnz     .range2mtrr_loop
693
; 9. We have calculated needed MTRRs, now setup them in the CPU.
694
; 9a. Abort if number of MTRRs is too large.
695
        mov     eax, [num_variable_mtrrs]
696
        cmp     [.num_used_mtrrs], eax
697
        ja      .abort
2288 clevermous 698
 
4608 clevermous 699
; 9b. Prepare for changes.
700
        call    mtrr_begin_change
2288 clevermous 701
 
4608 clevermous 702
; 9c. Prepare for loop over MTRRs.
703
        lea     esi, [.mtrrs]
704
        mov     ecx, 0x200
2288 clevermous 705
@@:
4608 clevermous 706
; 9d. For every MTRR, copy PHYSBASEn as is: step 8 has configured
707
; start value and type bits as needed.
708
        mov     eax, [esi]
709
        mov     edx, [esi+4]
710
        wrmsr
711
        inc     ecx
712
; 9e. For every MTRR, calculate PHYSMASKn = -(length) or 0x800
713
; with upper bits cleared, 0x800 = MTRR is valid.
2288 clevermous 714
        xor     eax, eax
715
        xor     edx, edx
4608 clevermous 716
        sub     eax, [esi+8]
717
        sbb     edx, [esi+12]
718
        or      eax, 0x800
719
        or      edx, [.phys_reserved_mask]
720
        xor     edx, [.phys_reserved_mask]
2288 clevermous 721
        wrmsr
722
        inc     ecx
4608 clevermous 723
; 9f. Continue steps 9d and 9e for all MTRRs calculated at step 8.
724
        add     esi, 16
725
        dec     [.num_used_mtrrs]
726
        jnz     @b
727
; 9g. Zero other MTRRs.
2288 clevermous 728
        xor     eax, eax
729
        xor     edx, edx
4608 clevermous 730
        mov     ebx, [num_variable_mtrrs]
731
        lea     ebx, [0x200+ebx*2]
2288 clevermous 732
@@:
4608 clevermous 733
        cmp     ecx, ebx
734
        jae     @f
735
        wrmsr
4418 clevermous 736
        inc     ecx
2288 clevermous 737
        wrmsr
4608 clevermous 738
        inc     ecx
739
        jmp     @b
740
@@:
741
 
5360 serge 742
; 9i. Check PAT support and reprogram PAT_MASR for write combining memory
743
        bt      [cpu_caps], CAPS_PAT
744
        jnc     @F
745
 
746
        mov     ecx, MSR_CR_PAT
747
        mov     eax, PAT_VALUE  ;UC UCM WC WB
748
        mov     edx, eax
2288 clevermous 749
        wrmsr
5360 serge 750
@@:
2288 clevermous 751
 
4608 clevermous 752
; 9j. Changes are done.
753
        call    mtrr_end_change
2288 clevermous 754
 
4608 clevermous 755
.abort:
756
        add     esp, .local_vars_size + MAX_RANGES * sizeof.mtrr_range
757
        pop     ebp
2288 clevermous 758
        ret
759
endp
760
 
4608 clevermous 761
; Allocate&set one MTRR for given range.
762
; size must be power of 2 that divides base.
2288 clevermous 763
proc set_mtrr stdcall, base:dword,size:dword,mem_type:dword
764
; find unused register
765
        mov     ecx, 0x201
4608 clevermous 766
.scan:
2288 clevermous 767
        rdmsr
768
        dec     ecx
769
        test    ah, 8
770
        jz      .found
771
        rdmsr
4608 clevermous 772
        test    edx, edx
773
        jnz     @f
774
        and     eax, not 0xFFF  ; clear reserved bits
2288 clevermous 775
        cmp     eax, [base]
776
        jz      .ret
4608 clevermous 777
@@:
2288 clevermous 778
        add     ecx, 3
4608 clevermous 779
        mov     eax, [num_variable_mtrrs]
780
        lea     eax, [0x200+eax*2]
781
        cmp     ecx, eax
782
        jb      .scan
2288 clevermous 783
; no free registers, ignore the call
784
.ret:
785
        ret
786
.found:
787
; found, write values
5360 serge 788
        push    ecx
4608 clevermous 789
        call    mtrr_begin_change
5360 serge 790
        pop     ecx
2288 clevermous 791
        xor     edx, edx
792
        mov     eax, [base]
793
        or      eax, [mem_type]
794
        wrmsr
795
 
4608 clevermous 796
        mov     al, [cpu_phys_addr_width]
797
        xor     edx, edx
798
        bts     edx, eax
799
        xor     eax, eax
800
        sub     eax, [size]
2288 clevermous 801
        sbb     edx, 0
802
        or      eax, 0x800
803
        inc     ecx
804
        wrmsr
4608 clevermous 805
        call    mtrr_end_change
2288 clevermous 806
        ret
807
endp
808
 
4608 clevermous 809
; Helper procedure for mtrr_validate.
810
; Calculates memory type for given address according to variable-range MTRRs.
811
; Assumes that MTRRs are enabled.
812
; in: ebx = 32-bit physical address
813
; out: eax = memory type for ebx
814
proc mtrr_get_real_type
815
; 1. Initialize: we have not yet found any MTRRs covering ebx.
816
        push    0
817
        mov     ecx, 0x201
818
.mtrr_loop:
819
; 2. For every MTRR, check whether it is valid; if not, continue to the next MTRR.
820
        rdmsr
2288 clevermous 821
        dec     ecx
4608 clevermous 822
        test    ah, 8
823
        jz      .next
824
; 3. For every valid MTRR, check whether (ebx and PHYSMASKn) == PHYSBASEn,
825
; excluding low 12 bits.
826
        and     eax, ebx
827
        push    eax
828
        rdmsr
829
        test    edx, edx
830
        pop     edx
831
        jnz     .next
832
        xor     edx, eax
833
        and     edx, not 0xFFF
834
        jnz     .next
835
; 4. If so, set the bit corresponding to memory type defined by this MTRR.
836
        and     eax, 7
837
        bts     [esp], eax
838
.next:
839
; 5. Continue loop at 2-4 for all variable-range MTRRs.
840
        add     ecx, 3
841
        mov     eax, [num_variable_mtrrs]
842
        lea     eax, [0x200+eax*2]
843
        cmp     ecx, eax
844
        jb      .mtrr_loop
845
; 6. If no MTRRs cover address in ebx, use default MTRR type from MTRR_DEF_CAP.
846
        pop     edx
847
        test    edx, edx
848
        jz      .default
849
; 7. Find&clear 1-bit in edx.
850
        bsf     eax, edx
851
        btr     edx, eax
852
; 8. If there was only one 1-bit, then all MTRRs are consistent, return that bit.
853
        test    edx, edx
854
        jz      .nothing
855
; Otherwise, return MEM_UC (e.g. WB+UC is UC).
2288 clevermous 856
        xor     eax, eax
4608 clevermous 857
.nothing:
2288 clevermous 858
        ret
4608 clevermous 859
.default:
860
        mov     ecx, 0x2FF
861
        rdmsr
862
        movzx   eax, al
863
        ret
2288 clevermous 864
endp
2466 Serge 865
 
4608 clevermous 866
; If MTRRs are configured improperly, this is not obvious to the user;
867
; everything works, but the performance can be horrible.
868
; Try to detect this and let the user know that the low performance
869
; is caused by some problem and is not a global property of the system.
870
; Let's hope he would report it to developers...
871
proc mtrr_validate
872
; 1. If MTRRs are not supported, they cannot be configured improperly.
4619 clevermous 873
; Note: VirtualBox claims MTRR support in cpuid, but emulates MTRRCAP=0,
874
; which is efficiently equivalent to absent MTRRs.
875
; So check [num_variable_mtrrs] instead of CAPS_MTRR in [cpu_caps].
876
        cmp     [num_variable_mtrrs], 0
877
        jz      .exit
4608 clevermous 878
; 2. If variable-range MTRRs are not configured, this is a problem.
879
        mov     ecx, 0x2FF
880
        rdmsr
881
        test    ah, 8
882
        jz      .fail
883
; 3. Get the memory type for address somewhere inside working memory.
884
; It must be write-back.
885
        mov     ebx, 0x27FFFF
886
        call    mtrr_get_real_type
887
        cmp     al, MEM_WB
888
        jnz     .fail
889
; 4. If we're using a mode with LFB,
890
; get the memory type for last pixel of the framebuffer.
891
; It must be write-combined.
892
        test    word [SCR_MODE], 0x4000
893
        jz      .exit
5351 serge 894
        mov     eax, [_display.lfb_pitch]
4608 clevermous 895
        mul     [_display.height]
896
        dec     eax
897
; LFB is mapped to virtual address LFB_BASE,
898
; it uses global pages if supported by CPU.
5130 serge 899
        mov     ebx, [sys_proc+PROC.pdt_0+(LFB_BASE shr 20)]
5356 serge 900
        test    ebx, PDE_LARGE
4608 clevermous 901
        jnz     @f
902
        mov     ebx, [page_tabs+(LFB_BASE shr 10)]
2466 Serge 903
@@:
4608 clevermous 904
        and     ebx, not 0xFFF
905
        add     ebx, eax
906
        call    mtrr_get_real_type
907
        cmp     al, MEM_WC
908
        jz      .exit
909
; 5. The check at step 4 fails on Bochs:
910
; Bochs BIOS configures MTRRs in a strange way not respecting [cpu_phys_addr_width],
911
; so mtrr_reconfigure avoids to touch anything.
912
; However, Bochs core ignores MTRRs (keeping them only for rdmsr/wrmsr),
913
; so we don't care about proper setting for Bochs.
914
; Use northbridge PCI id to detect Bochs: it emulates either i440fx or i430fx
915
; depending on configuration file.
916
        mov     eax, [pcidev_list.fd]
917
        cmp     eax, pcidev_list        ; sanity check: fail if no PCI devices
918
        jz      .fail
919
        cmp     [eax+PCIDEV.vendor_device_id], 0x12378086
920
        jz      .exit
921
        cmp     [eax+PCIDEV.vendor_device_id], 0x01228086
922
        jnz     .fail
923
.exit:
2466 Serge 924
        ret
4608 clevermous 925
.fail:
926
        mov     ebx, mtrr_user_message
927
        mov     ebp, notifyapp
928
        call    fs_execute_from_sysdir_param
929
        ret
2466 Serge 930
endp