Rev 4280 | Rev 4560 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
2332 | Serge | 1 | /* |
2 | * Copyright © 2010 Daniel Vetter |
||
3 | * |
||
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
||
5 | * copy of this software and associated documentation files (the "Software"), |
||
6 | * to deal in the Software without restriction, including without limitation |
||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
||
8 | * and/or sell copies of the Software, and to permit persons to whom the |
||
9 | * Software is furnished to do so, subject to the following conditions: |
||
10 | * |
||
11 | * The above copyright notice and this permission notice (including the next |
||
12 | * paragraph) shall be included in all copies or substantial portions of the |
||
13 | * Software. |
||
14 | * |
||
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
||
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
||
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
||
21 | * IN THE SOFTWARE. |
||
22 | * |
||
23 | */ |
||
24 | |||
3243 | Serge | 25 | |
3480 | Serge | 26 | #define AGP_NORMAL_MEMORY 0 |
27 | |||
28 | #define AGP_USER_TYPES (1 << 16) |
||
29 | #define AGP_USER_MEMORY (AGP_USER_TYPES) |
||
30 | #define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1) |
||
31 | |||
3031 | serge | 32 | #include |
33 | #include |
||
2332 | Serge | 34 | #include "i915_drv.h" |
2351 | Serge | 35 | #include "i915_trace.h" |
2332 | Serge | 36 | #include "intel_drv.h" |
37 | |||
4104 | Serge | 38 | #define GEN6_PPGTT_PD_ENTRIES 512 |
39 | #define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t)) |
||
3243 | Serge | 40 | |
41 | /* PPGTT stuff */ |
||
42 | #define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) |
||
4104 | Serge | 43 | #define HSW_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0x7f0)) |
3243 | Serge | 44 | |
45 | #define GEN6_PDE_VALID (1 << 0) |
||
46 | /* gen6+ has bit 11-4 for physical addr bit 39-32 */ |
||
47 | #define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) |
||
48 | |||
49 | #define GEN6_PTE_VALID (1 << 0) |
||
50 | #define GEN6_PTE_UNCACHED (1 << 1) |
||
51 | #define HSW_PTE_UNCACHED (0) |
||
52 | #define GEN6_PTE_CACHE_LLC (2 << 1) |
||
4104 | Serge | 53 | #define GEN7_PTE_CACHE_L3_LLC (3 << 1) |
3243 | Serge | 54 | #define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) |
4104 | Serge | 55 | #define HSW_PTE_ADDR_ENCODE(addr) HSW_GTT_ADDR_ENCODE(addr) |
3243 | Serge | 56 | |
4104 | Serge | 57 | /* Cacheability Control is a 4-bit value. The low three bits are stored in * |
58 | * bits 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE. |
||
59 | */ |
||
60 | #define HSW_CACHEABILITY_CONTROL(bits) ((((bits) & 0x7) << 1) | \ |
||
61 | (((bits) & 0x8) << (11 - 3))) |
||
62 | #define HSW_WB_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x2) |
||
63 | #define HSW_WB_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x3) |
||
64 | #define HSW_WB_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0xb) |
||
65 | #define HSW_WT_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x6) |
||
66 | |||
67 | static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr, |
||
4280 | Serge | 68 | enum i915_cache_level level, |
69 | bool valid) |
||
4104 | Serge | 70 | { |
4280 | Serge | 71 | gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0; |
4104 | Serge | 72 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
73 | |||
74 | switch (level) { |
||
75 | case I915_CACHE_L3_LLC: |
||
76 | case I915_CACHE_LLC: |
||
77 | pte |= GEN6_PTE_CACHE_LLC; |
||
78 | break; |
||
79 | case I915_CACHE_NONE: |
||
80 | pte |= GEN6_PTE_UNCACHED; |
||
81 | break; |
||
82 | default: |
||
83 | WARN_ON(1); |
||
84 | } |
||
85 | |||
86 | return pte; |
||
87 | } |
||
88 | |||
89 | static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr, |
||
4280 | Serge | 90 | enum i915_cache_level level, |
91 | bool valid) |
||
3243 | Serge | 92 | { |
4280 | Serge | 93 | gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0; |
3243 | Serge | 94 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
95 | |||
96 | switch (level) { |
||
4104 | Serge | 97 | case I915_CACHE_L3_LLC: |
98 | pte |= GEN7_PTE_CACHE_L3_LLC; |
||
3243 | Serge | 99 | break; |
100 | case I915_CACHE_LLC: |
||
101 | pte |= GEN6_PTE_CACHE_LLC; |
||
102 | break; |
||
103 | case I915_CACHE_NONE: |
||
104 | pte |= GEN6_PTE_UNCACHED; |
||
105 | break; |
||
106 | default: |
||
4104 | Serge | 107 | WARN_ON(1); |
3243 | Serge | 108 | } |
109 | |||
110 | return pte; |
||
111 | } |
||
112 | |||
4104 | Serge | 113 | #define BYT_PTE_WRITEABLE (1 << 1) |
114 | #define BYT_PTE_SNOOPED_BY_CPU_CACHES (1 << 2) |
||
115 | |||
116 | static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr, |
||
4280 | Serge | 117 | enum i915_cache_level level, |
118 | bool valid) |
||
3746 | Serge | 119 | { |
4280 | Serge | 120 | gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0; |
4104 | Serge | 121 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
122 | |||
123 | /* Mark the page as writeable. Other platforms don't have a |
||
124 | * setting for read-only/writable, so this matches that behavior. |
||
125 | */ |
||
126 | pte |= BYT_PTE_WRITEABLE; |
||
127 | |||
128 | if (level != I915_CACHE_NONE) |
||
129 | pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; |
||
130 | |||
131 | return pte; |
||
132 | } |
||
133 | |||
134 | static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr, |
||
4280 | Serge | 135 | enum i915_cache_level level, |
136 | bool valid) |
||
4104 | Serge | 137 | { |
4280 | Serge | 138 | gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0; |
4104 | Serge | 139 | pte |= HSW_PTE_ADDR_ENCODE(addr); |
140 | |||
141 | if (level != I915_CACHE_NONE) |
||
142 | pte |= HSW_WB_LLC_AGE3; |
||
143 | |||
144 | return pte; |
||
145 | } |
||
146 | |||
147 | static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr, |
||
4280 | Serge | 148 | enum i915_cache_level level, |
149 | bool valid) |
||
4104 | Serge | 150 | { |
4280 | Serge | 151 | gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0; |
4104 | Serge | 152 | pte |= HSW_PTE_ADDR_ENCODE(addr); |
153 | |||
154 | switch (level) { |
||
155 | case I915_CACHE_NONE: |
||
156 | break; |
||
157 | case I915_CACHE_WT: |
||
158 | pte |= HSW_WT_ELLC_LLC_AGE0; |
||
159 | break; |
||
160 | default: |
||
161 | pte |= HSW_WB_ELLC_LLC_AGE0; |
||
162 | break; |
||
163 | } |
||
164 | |||
165 | return pte; |
||
166 | } |
||
167 | |||
168 | static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt) |
||
169 | { |
||
170 | struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private; |
||
3746 | Serge | 171 | gen6_gtt_pte_t __iomem *pd_addr; |
172 | uint32_t pd_entry; |
||
173 | int i; |
||
174 | |||
4104 | Serge | 175 | WARN_ON(ppgtt->pd_offset & 0x3f); |
3746 | Serge | 176 | pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm + |
177 | ppgtt->pd_offset / sizeof(gen6_gtt_pte_t); |
||
178 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
||
179 | dma_addr_t pt_addr; |
||
180 | |||
181 | pt_addr = ppgtt->pt_dma_addr[i]; |
||
182 | pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); |
||
183 | pd_entry |= GEN6_PDE_VALID; |
||
184 | |||
185 | writel(pd_entry, pd_addr + i); |
||
186 | } |
||
187 | readl(pd_addr); |
||
4104 | Serge | 188 | } |
3746 | Serge | 189 | |
4104 | Serge | 190 | static int gen6_ppgtt_enable(struct drm_device *dev) |
191 | { |
||
192 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
193 | uint32_t pd_offset; |
||
194 | struct intel_ring_buffer *ring; |
||
195 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
||
196 | int i; |
||
197 | |||
198 | BUG_ON(ppgtt->pd_offset & 0x3f); |
||
199 | |||
200 | gen6_write_pdes(ppgtt); |
||
201 | |||
3746 | Serge | 202 | pd_offset = ppgtt->pd_offset; |
203 | pd_offset /= 64; /* in cachelines, */ |
||
204 | pd_offset <<= 16; |
||
205 | |||
206 | if (INTEL_INFO(dev)->gen == 6) { |
||
207 | uint32_t ecochk, gab_ctl, ecobits; |
||
208 | |||
209 | ecobits = I915_READ(GAC_ECO_BITS); |
||
210 | I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | |
||
211 | ECOBITS_PPGTT_CACHE64B); |
||
212 | |||
213 | gab_ctl = I915_READ(GAB_CTL); |
||
214 | I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); |
||
215 | |||
216 | ecochk = I915_READ(GAM_ECOCHK); |
||
217 | I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | |
||
218 | ECOCHK_PPGTT_CACHE64B); |
||
219 | I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
||
220 | } else if (INTEL_INFO(dev)->gen >= 7) { |
||
221 | uint32_t ecochk, ecobits; |
||
222 | |||
223 | ecobits = I915_READ(GAC_ECO_BITS); |
||
224 | I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); |
||
225 | |||
226 | ecochk = I915_READ(GAM_ECOCHK); |
||
227 | if (IS_HASWELL(dev)) { |
||
228 | ecochk |= ECOCHK_PPGTT_WB_HSW; |
||
229 | } else { |
||
230 | ecochk |= ECOCHK_PPGTT_LLC_IVB; |
||
231 | ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; |
||
232 | } |
||
233 | I915_WRITE(GAM_ECOCHK, ecochk); |
||
234 | /* GFX_MODE is per-ring on gen7+ */ |
||
235 | } |
||
236 | |||
237 | for_each_ring(ring, dev_priv, i) { |
||
238 | if (INTEL_INFO(dev)->gen >= 7) |
||
239 | I915_WRITE(RING_MODE_GEN7(ring), |
||
240 | _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
||
241 | |||
242 | I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); |
||
243 | I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); |
||
244 | } |
||
245 | return 0; |
||
246 | } |
||
247 | |||
3031 | serge | 248 | /* PPGTT support for Sandybdrige/Gen6 and later */ |
4104 | Serge | 249 | static void gen6_ppgtt_clear_range(struct i915_address_space *vm, |
3031 | serge | 250 | unsigned first_entry, |
4280 | Serge | 251 | unsigned num_entries, |
252 | bool use_scratch) |
||
3031 | serge | 253 | { |
4104 | Serge | 254 | struct i915_hw_ppgtt *ppgtt = |
255 | container_of(vm, struct i915_hw_ppgtt, base); |
||
3746 | Serge | 256 | gen6_gtt_pte_t *pt_vaddr, scratch_pte; |
257 | unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; |
||
3031 | serge | 258 | unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
259 | unsigned last_pte, i; |
||
260 | |||
4280 | Serge | 261 | scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true); |
3031 | serge | 262 | |
263 | pt_vaddr = AllocKernelSpace(4096); |
||
264 | |||
3480 | Serge | 265 | if(pt_vaddr == NULL) |
266 | return; |
||
267 | |||
268 | while (num_entries) { |
||
3031 | serge | 269 | last_pte = first_pte + num_entries; |
270 | if (last_pte > I915_PPGTT_PT_ENTRIES) |
||
271 | last_pte = I915_PPGTT_PT_ENTRIES; |
||
272 | |||
3746 | Serge | 273 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
3031 | serge | 274 | |
275 | for (i = first_pte; i < last_pte; i++) |
||
276 | pt_vaddr[i] = scratch_pte; |
||
277 | |||
278 | num_entries -= last_pte - first_pte; |
||
279 | first_pte = 0; |
||
3746 | Serge | 280 | act_pt++; |
3480 | Serge | 281 | }; |
282 | |||
283 | FreeKernelSpace(pt_vaddr); |
||
284 | } |
||
285 | |||
4104 | Serge | 286 | static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, |
3480 | Serge | 287 | struct sg_table *pages, |
288 | unsigned first_entry, |
||
289 | enum i915_cache_level cache_level) |
||
290 | { |
||
4104 | Serge | 291 | struct i915_hw_ppgtt *ppgtt = |
292 | container_of(vm, struct i915_hw_ppgtt, base); |
||
3746 | Serge | 293 | gen6_gtt_pte_t *pt_vaddr; |
294 | unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; |
||
295 | unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
||
296 | struct sg_page_iter sg_iter; |
||
3480 | Serge | 297 | dma_addr_t page_addr; |
298 | |||
299 | |||
300 | pt_vaddr = AllocKernelSpace(4096); |
||
301 | |||
302 | if(pt_vaddr == NULL) |
||
303 | return; |
||
304 | |||
3746 | Serge | 305 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
306 | for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { |
||
307 | dma_addr_t page_addr; |
||
3480 | Serge | 308 | |
3746 | Serge | 309 | page_addr = sg_page_iter_dma_address(&sg_iter); |
4280 | Serge | 310 | pt_vaddr[act_pte] = vm->pte_encode(page_addr, cache_level, true); |
3746 | Serge | 311 | if (++act_pte == I915_PPGTT_PT_ENTRIES) { |
312 | act_pt++; |
||
313 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
||
314 | act_pte = 0; |
||
3480 | Serge | 315 | |
4104 | Serge | 316 | } |
3480 | Serge | 317 | } |
4104 | Serge | 318 | FreeKernelSpace(pt_vaddr); |
3031 | serge | 319 | } |
320 | |||
4104 | Serge | 321 | static void gen6_ppgtt_cleanup(struct i915_address_space *vm) |
3031 | serge | 322 | { |
4104 | Serge | 323 | struct i915_hw_ppgtt *ppgtt = |
324 | container_of(vm, struct i915_hw_ppgtt, base); |
||
3480 | Serge | 325 | int i; |
326 | |||
4104 | Serge | 327 | drm_mm_takedown(&ppgtt->base.mm); |
328 | |||
3480 | Serge | 329 | if (ppgtt->pt_dma_addr) { |
330 | for (i = 0; i < ppgtt->num_pd_entries; i++) |
||
4104 | Serge | 331 | pci_unmap_page(ppgtt->base.dev->pdev, |
3480 | Serge | 332 | ppgtt->pt_dma_addr[i], |
333 | 4096, PCI_DMA_BIDIRECTIONAL); |
||
334 | } |
||
335 | |||
336 | kfree(ppgtt->pt_dma_addr); |
||
337 | for (i = 0; i < ppgtt->num_pd_entries; i++) |
||
338 | __free_page(ppgtt->pt_pages[i]); |
||
339 | kfree(ppgtt->pt_pages); |
||
340 | kfree(ppgtt); |
||
341 | } |
||
342 | |||
343 | static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) |
||
344 | { |
||
4104 | Serge | 345 | struct drm_device *dev = ppgtt->base.dev; |
3031 | serge | 346 | struct drm_i915_private *dev_priv = dev->dev_private; |
347 | unsigned first_pd_entry_in_global_pt; |
||
348 | int i; |
||
349 | int ret = -ENOMEM; |
||
350 | |||
351 | /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 |
||
352 | * entries. For aliasing ppgtt support we just steal them at the end for |
||
353 | * now. */ |
||
3746 | Serge | 354 | first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt); |
3031 | serge | 355 | |
4104 | Serge | 356 | ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode; |
357 | ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES; |
||
3746 | Serge | 358 | ppgtt->enable = gen6_ppgtt_enable; |
4104 | Serge | 359 | ppgtt->base.clear_range = gen6_ppgtt_clear_range; |
360 | ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; |
||
361 | ppgtt->base.cleanup = gen6_ppgtt_cleanup; |
||
362 | ppgtt->base.scratch = dev_priv->gtt.base.scratch; |
||
3243 | Serge | 363 | ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries, |
3031 | serge | 364 | GFP_KERNEL); |
365 | if (!ppgtt->pt_pages) |
||
3480 | Serge | 366 | return -ENOMEM; |
3031 | serge | 367 | |
368 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
||
3243 | Serge | 369 | ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL); |
3031 | serge | 370 | if (!ppgtt->pt_pages[i]) |
371 | goto err_pt_alloc; |
||
372 | } |
||
373 | |||
3480 | Serge | 374 | ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries, |
3031 | serge | 375 | GFP_KERNEL); |
4104 | Serge | 376 | if (!ppgtt->pt_dma_addr) |
377 | goto err_pt_alloc; |
||
3031 | serge | 378 | |
4104 | Serge | 379 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
380 | dma_addr_t pt_addr; |
||
3031 | serge | 381 | |
3480 | Serge | 382 | pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096, |
3031 | serge | 383 | PCI_DMA_BIDIRECTIONAL); |
384 | |||
4104 | Serge | 385 | ppgtt->pt_dma_addr[i] = pt_addr; |
386 | } |
||
3031 | serge | 387 | |
4104 | Serge | 388 | ppgtt->base.clear_range(&ppgtt->base, 0, |
4280 | Serge | 389 | ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES, true); |
3031 | serge | 390 | |
3746 | Serge | 391 | ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t); |
3031 | serge | 392 | |
393 | return 0; |
||
394 | |||
395 | err_pd_pin: |
||
3480 | Serge | 396 | if (ppgtt->pt_dma_addr) { |
397 | for (i--; i >= 0; i--) |
||
398 | pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], |
||
399 | 4096, PCI_DMA_BIDIRECTIONAL); |
||
400 | } |
||
3031 | serge | 401 | err_pt_alloc: |
3480 | Serge | 402 | kfree(ppgtt->pt_dma_addr); |
3031 | serge | 403 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
404 | if (ppgtt->pt_pages[i]) |
||
3480 | Serge | 405 | __free_page(ppgtt->pt_pages[i]); |
3031 | serge | 406 | } |
407 | kfree(ppgtt->pt_pages); |
||
408 | |||
409 | return ret; |
||
410 | } |
||
411 | |||
3480 | Serge | 412 | static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) |
3031 | serge | 413 | { |
414 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
3480 | Serge | 415 | struct i915_hw_ppgtt *ppgtt; |
416 | int ret; |
||
3031 | serge | 417 | |
3480 | Serge | 418 | ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); |
3031 | serge | 419 | if (!ppgtt) |
3480 | Serge | 420 | return -ENOMEM; |
3031 | serge | 421 | |
4104 | Serge | 422 | ppgtt->base.dev = dev; |
3031 | serge | 423 | |
3746 | Serge | 424 | if (INTEL_INFO(dev)->gen < 8) |
3480 | Serge | 425 | ret = gen6_ppgtt_init(ppgtt); |
3746 | Serge | 426 | else |
427 | BUG(); |
||
428 | |||
3480 | Serge | 429 | if (ret) |
4104 | Serge | 430 | kfree(ppgtt); |
431 | else { |
||
3480 | Serge | 432 | dev_priv->mm.aliasing_ppgtt = ppgtt; |
4104 | Serge | 433 | drm_mm_init(&ppgtt->base.mm, ppgtt->base.start, |
434 | ppgtt->base.total); |
||
435 | } |
||
3480 | Serge | 436 | |
437 | return ret; |
||
3031 | serge | 438 | } |
439 | |||
3480 | Serge | 440 | void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) |
3031 | serge | 441 | { |
3480 | Serge | 442 | struct drm_i915_private *dev_priv = dev->dev_private; |
443 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
||
3031 | serge | 444 | |
3480 | Serge | 445 | if (!ppgtt) |
3243 | Serge | 446 | return; |
3031 | serge | 447 | |
4104 | Serge | 448 | ppgtt->base.cleanup(&ppgtt->base); |
3746 | Serge | 449 | dev_priv->mm.aliasing_ppgtt = NULL; |
3031 | serge | 450 | } |
451 | |||
452 | void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, |
||
453 | struct drm_i915_gem_object *obj, |
||
454 | enum i915_cache_level cache_level) |
||
455 | { |
||
4104 | Serge | 456 | ppgtt->base.insert_entries(&ppgtt->base, obj->pages, |
457 | i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT, |
||
3243 | Serge | 458 | cache_level); |
3031 | serge | 459 | } |
460 | |||
461 | void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, |
||
462 | struct drm_i915_gem_object *obj) |
||
463 | { |
||
4104 | Serge | 464 | ppgtt->base.clear_range(&ppgtt->base, |
465 | i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT, |
||
4280 | Serge | 466 | obj->base.size >> PAGE_SHIFT, |
467 | true); |
||
3031 | serge | 468 | } |
469 | |||
3480 | Serge | 470 | extern int intel_iommu_gfx_mapped; |
471 | /* Certain Gen5 chipsets require require idling the GPU before |
||
472 | * unmapping anything from the GTT when VT-d is enabled. |
||
473 | */ |
||
474 | static inline bool needs_idle_maps(struct drm_device *dev) |
||
475 | { |
||
476 | #ifdef CONFIG_INTEL_IOMMU |
||
477 | /* Query intel_iommu to see if we need the workaround. Presumably that |
||
478 | * was loaded first. |
||
479 | */ |
||
480 | if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped) |
||
481 | return true; |
||
482 | #endif |
||
483 | return false; |
||
484 | } |
||
485 | |||
2344 | Serge | 486 | static bool do_idling(struct drm_i915_private *dev_priv) |
487 | { |
||
488 | bool ret = dev_priv->mm.interruptible; |
||
489 | |||
3480 | Serge | 490 | if (unlikely(dev_priv->gtt.do_idle_maps)) { |
2344 | Serge | 491 | dev_priv->mm.interruptible = false; |
492 | if (i915_gpu_idle(dev_priv->dev)) { |
||
493 | DRM_ERROR("Couldn't idle GPU\n"); |
||
494 | /* Wait a bit, in hopes it avoids the hang */ |
||
495 | udelay(10); |
||
496 | } |
||
497 | } |
||
498 | |||
499 | return ret; |
||
500 | } |
||
501 | |||
502 | static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) |
||
503 | { |
||
3480 | Serge | 504 | if (unlikely(dev_priv->gtt.do_idle_maps)) |
2344 | Serge | 505 | dev_priv->mm.interruptible = interruptible; |
506 | } |
||
507 | |||
4280 | Serge | 508 | void i915_check_and_clear_faults(struct drm_device *dev) |
509 | { |
||
510 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
511 | struct intel_ring_buffer *ring; |
||
512 | int i; |
||
513 | |||
514 | if (INTEL_INFO(dev)->gen < 6) |
||
515 | return; |
||
516 | |||
517 | for_each_ring(ring, dev_priv, i) { |
||
518 | u32 fault_reg; |
||
519 | fault_reg = I915_READ(RING_FAULT_REG(ring)); |
||
520 | if (fault_reg & RING_FAULT_VALID) { |
||
521 | DRM_DEBUG_DRIVER("Unexpected fault\n" |
||
522 | "\tAddr: 0x%08lx\\n" |
||
523 | "\tAddress space: %s\n" |
||
524 | "\tSource ID: %d\n" |
||
525 | "\tType: %d\n", |
||
526 | fault_reg & PAGE_MASK, |
||
527 | fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT", |
||
528 | RING_FAULT_SRCID(fault_reg), |
||
529 | RING_FAULT_FAULT_TYPE(fault_reg)); |
||
530 | I915_WRITE(RING_FAULT_REG(ring), |
||
531 | fault_reg & ~RING_FAULT_VALID); |
||
532 | } |
||
533 | } |
||
534 | POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS])); |
||
535 | } |
||
536 | |||
537 | void i915_gem_suspend_gtt_mappings(struct drm_device *dev) |
||
538 | { |
||
539 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
540 | |||
541 | /* Don't bother messing with faults pre GEN6 as we have little |
||
542 | * documentation supporting that it's a good idea. |
||
543 | */ |
||
544 | if (INTEL_INFO(dev)->gen < 6) |
||
545 | return; |
||
546 | |||
547 | i915_check_and_clear_faults(dev); |
||
548 | |||
549 | dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, |
||
550 | dev_priv->gtt.base.start / PAGE_SIZE, |
||
551 | dev_priv->gtt.base.total / PAGE_SIZE, |
||
552 | false); |
||
553 | } |
||
554 | |||
2332 | Serge | 555 | void i915_gem_restore_gtt_mappings(struct drm_device *dev) |
556 | { |
||
557 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
558 | struct drm_i915_gem_object *obj; |
||
559 | |||
4280 | Serge | 560 | i915_check_and_clear_faults(dev); |
561 | |||
2332 | Serge | 562 | /* First fill our portion of the GTT with scratch pages */ |
4104 | Serge | 563 | dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, |
564 | dev_priv->gtt.base.start / PAGE_SIZE, |
||
4280 | Serge | 565 | dev_priv->gtt.base.total / PAGE_SIZE, |
566 | true); |
||
2332 | Serge | 567 | |
4104 | Serge | 568 | list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { |
569 | i915_gem_clflush_object(obj, obj->pin_display); |
||
3031 | serge | 570 | i915_gem_gtt_bind_object(obj, obj->cache_level); |
2332 | Serge | 571 | } |
572 | |||
3243 | Serge | 573 | i915_gem_chipset_flush(dev); |
2332 | Serge | 574 | } |
575 | |||
3031 | serge | 576 | int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) |
2332 | Serge | 577 | { |
3243 | Serge | 578 | if (obj->has_dma_mapping) |
579 | return 0; |
||
580 | |||
3480 | Serge | 581 | if (!dma_map_sg(&obj->base.dev->pdev->dev, |
582 | obj->pages->sgl, obj->pages->nents, |
||
583 | PCI_DMA_BIDIRECTIONAL)) |
||
584 | return -ENOSPC; |
||
3243 | Serge | 585 | |
2332 | Serge | 586 | return 0; |
587 | } |
||
588 | |||
3243 | Serge | 589 | /* |
590 | * Binds an object into the global gtt with the specified cache level. The object |
||
591 | * will be accessible to the GPU via commands whose operands reference offsets |
||
592 | * within the global GTT as well as accessible by the GPU through the GMADR |
||
593 | * mapped BAR (dev_priv->mm.gtt->gtt). |
||
594 | */ |
||
4104 | Serge | 595 | static void gen6_ggtt_insert_entries(struct i915_address_space *vm, |
3480 | Serge | 596 | struct sg_table *st, |
597 | unsigned int first_entry, |
||
3243 | Serge | 598 | enum i915_cache_level level) |
599 | { |
||
4104 | Serge | 600 | struct drm_i915_private *dev_priv = vm->dev->dev_private; |
3746 | Serge | 601 | gen6_gtt_pte_t __iomem *gtt_entries = |
602 | (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; |
||
603 | int i = 0; |
||
604 | struct sg_page_iter sg_iter; |
||
3243 | Serge | 605 | dma_addr_t addr; |
606 | |||
3746 | Serge | 607 | for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { |
608 | addr = sg_page_iter_dma_address(&sg_iter); |
||
4280 | Serge | 609 | iowrite32(vm->pte_encode(addr, level, true), >t_entries[i]); |
3243 | Serge | 610 | i++; |
611 | } |
||
612 | |||
613 | /* XXX: This serves as a posting read to make sure that the PTE has |
||
614 | * actually been updated. There is some concern that even though |
||
615 | * registers and PTEs are within the same BAR that they are potentially |
||
616 | * of NUMA access patterns. Therefore, even with the way we assume |
||
617 | * hardware should work, we must keep this posting read for paranoia. |
||
618 | */ |
||
619 | if (i != 0) |
||
4104 | Serge | 620 | WARN_ON(readl(>t_entries[i-1]) != |
4280 | Serge | 621 | vm->pte_encode(addr, level, true)); |
3243 | Serge | 622 | |
623 | /* This next bit makes the above posting read even more important. We |
||
624 | * want to flush the TLBs only after we're certain all the PTE updates |
||
625 | * have finished. |
||
626 | */ |
||
627 | I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); |
||
628 | POSTING_READ(GFX_FLSH_CNTL_GEN6); |
||
629 | } |
||
630 | |||
4104 | Serge | 631 | static void gen6_ggtt_clear_range(struct i915_address_space *vm, |
3480 | Serge | 632 | unsigned int first_entry, |
4280 | Serge | 633 | unsigned int num_entries, |
634 | bool use_scratch) |
||
3480 | Serge | 635 | { |
4104 | Serge | 636 | struct drm_i915_private *dev_priv = vm->dev->dev_private; |
3746 | Serge | 637 | gen6_gtt_pte_t scratch_pte, __iomem *gtt_base = |
638 | (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; |
||
3480 | Serge | 639 | const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; |
640 | int i; |
||
641 | |||
4126 | Serge | 642 | if (WARN(num_entries > max_entries, |
643 | "First entry = %d; Num entries = %d (max=%d)\n", |
||
644 | first_entry, num_entries, max_entries)) |
||
4104 | Serge | 645 | num_entries = max_entries; |
3480 | Serge | 646 | |
4280 | Serge | 647 | scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch); |
648 | |||
3480 | Serge | 649 | for (i = 0; i < num_entries; i++) |
650 | iowrite32(scratch_pte, >t_base[i]); |
||
651 | readl(gtt_base); |
||
652 | } |
||
653 | |||
654 | |||
4104 | Serge | 655 | static void i915_ggtt_insert_entries(struct i915_address_space *vm, |
3480 | Serge | 656 | struct sg_table *st, |
657 | unsigned int pg_start, |
||
658 | enum i915_cache_level cache_level) |
||
659 | { |
||
660 | unsigned int flags = (cache_level == I915_CACHE_NONE) ? |
||
661 | AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; |
||
662 | |||
663 | intel_gtt_insert_sg_entries(st, pg_start, flags); |
||
664 | |||
665 | } |
||
666 | |||
4104 | Serge | 667 | static void i915_ggtt_clear_range(struct i915_address_space *vm, |
3480 | Serge | 668 | unsigned int first_entry, |
4280 | Serge | 669 | unsigned int num_entries, |
670 | bool unused) |
||
3480 | Serge | 671 | { |
672 | intel_gtt_clear_range(first_entry, num_entries); |
||
673 | } |
||
674 | |||
675 | |||
3031 | serge | 676 | void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj, |
2332 | Serge | 677 | enum i915_cache_level cache_level) |
678 | { |
||
679 | struct drm_device *dev = obj->base.dev; |
||
3480 | Serge | 680 | struct drm_i915_private *dev_priv = dev->dev_private; |
4104 | Serge | 681 | const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT; |
3480 | Serge | 682 | |
4104 | Serge | 683 | dev_priv->gtt.base.insert_entries(&dev_priv->gtt.base, obj->pages, |
684 | entry, |
||
3480 | Serge | 685 | cache_level); |
2332 | Serge | 686 | |
3031 | serge | 687 | obj->has_global_gtt_mapping = 1; |
2332 | Serge | 688 | } |
689 | |||
690 | void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) |
||
691 | { |
||
3480 | Serge | 692 | struct drm_device *dev = obj->base.dev; |
693 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
4104 | Serge | 694 | const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT; |
3480 | Serge | 695 | |
4104 | Serge | 696 | dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, |
697 | entry, |
||
4280 | Serge | 698 | obj->base.size >> PAGE_SHIFT, |
699 | true); |
||
3031 | serge | 700 | |
701 | obj->has_global_gtt_mapping = 0; |
||
702 | } |
||
703 | |||
704 | void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) |
||
705 | { |
||
2344 | Serge | 706 | struct drm_device *dev = obj->base.dev; |
707 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
708 | bool interruptible; |
||
709 | |||
710 | interruptible = do_idling(dev_priv); |
||
711 | |||
3480 | Serge | 712 | if (!obj->has_dma_mapping) |
713 | dma_unmap_sg(&dev->pdev->dev, |
||
714 | obj->pages->sgl, obj->pages->nents, |
||
715 | PCI_DMA_BIDIRECTIONAL); |
||
2332 | Serge | 716 | |
3031 | serge | 717 | undo_idling(dev_priv, interruptible); |
718 | } |
||
719 | |||
720 | static void i915_gtt_color_adjust(struct drm_mm_node *node, |
||
721 | unsigned long color, |
||
722 | unsigned long *start, |
||
723 | unsigned long *end) |
||
724 | { |
||
725 | if (node->color != color) |
||
726 | *start += 4096; |
||
727 | |||
728 | if (!list_empty(&node->node_list)) { |
||
729 | node = list_entry(node->node_list.next, |
||
730 | struct drm_mm_node, |
||
731 | node_list); |
||
732 | if (node->allocated && node->color != color) |
||
733 | *end -= 4096; |
||
2332 | Serge | 734 | } |
3031 | serge | 735 | } |
3480 | Serge | 736 | void i915_gem_setup_global_gtt(struct drm_device *dev, |
3031 | serge | 737 | unsigned long start, |
738 | unsigned long mappable_end, |
||
739 | unsigned long end) |
||
740 | { |
||
3480 | Serge | 741 | /* Let GEM Manage all of the aperture. |
742 | * |
||
743 | * However, leave one page at the end still bound to the scratch page. |
||
744 | * There are a number of places where the hardware apparently prefetches |
||
745 | * past the end of the object, and we've seen multiple hangs with the |
||
746 | * GPU head pointer stuck in a batchbuffer bound at the last page of the |
||
747 | * aperture. One page should be enough to keep any prefetching inside |
||
748 | * of the aperture. |
||
749 | */ |
||
4104 | Serge | 750 | struct drm_i915_private *dev_priv = dev->dev_private; |
751 | struct i915_address_space *ggtt_vm = &dev_priv->gtt.base; |
||
3480 | Serge | 752 | struct drm_mm_node *entry; |
753 | struct drm_i915_gem_object *obj; |
||
754 | unsigned long hole_start, hole_end; |
||
3031 | serge | 755 | |
3480 | Serge | 756 | BUG_ON(mappable_end > end); |
757 | |||
758 | /* Subtract the guard page ... */ |
||
4104 | Serge | 759 | drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE); |
3031 | serge | 760 | if (!HAS_LLC(dev)) |
4104 | Serge | 761 | dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust; |
3031 | serge | 762 | |
3480 | Serge | 763 | /* Mark any preallocated objects as occupied */ |
4104 | Serge | 764 | list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { |
765 | struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm); |
||
766 | int ret; |
||
767 | DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n", |
||
768 | i915_gem_obj_ggtt_offset(obj), obj->base.size); |
||
3031 | serge | 769 | |
4104 | Serge | 770 | WARN_ON(i915_gem_obj_ggtt_bound(obj)); |
771 | ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node); |
||
772 | if (ret) |
||
773 | DRM_DEBUG_KMS("Reservation failed\n"); |
||
3480 | Serge | 774 | obj->has_global_gtt_mapping = 1; |
4104 | Serge | 775 | list_add(&vma->vma_link, &obj->vma_list); |
3480 | Serge | 776 | } |
777 | |||
4104 | Serge | 778 | dev_priv->gtt.base.start = start; |
779 | dev_priv->gtt.base.total = end - start; |
||
3480 | Serge | 780 | |
781 | /* Clear any non-preallocated blocks */ |
||
4104 | Serge | 782 | drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) { |
783 | const unsigned long count = (hole_end - hole_start) / PAGE_SIZE; |
||
3480 | Serge | 784 | DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n", |
785 | hole_start, hole_end); |
||
4280 | Serge | 786 | ggtt_vm->clear_range(ggtt_vm, hole_start / PAGE_SIZE, count, true); |
3480 | Serge | 787 | } |
788 | |||
789 | /* And finally clear the reserved guard page */ |
||
4280 | Serge | 790 | ggtt_vm->clear_range(ggtt_vm, end / PAGE_SIZE - 1, 1, true); |
2332 | Serge | 791 | } |
3243 | Serge | 792 | |
3480 | Serge | 793 | static bool |
794 | intel_enable_ppgtt(struct drm_device *dev) |
||
795 | { |
||
796 | if (i915_enable_ppgtt >= 0) |
||
797 | return i915_enable_ppgtt; |
||
798 | |||
799 | #ifdef CONFIG_INTEL_IOMMU |
||
800 | /* Disable ppgtt on SNB if VT-d is on. */ |
||
801 | if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) |
||
802 | return false; |
||
803 | #endif |
||
804 | |||
805 | return true; |
||
806 | } |
||
807 | |||
808 | void i915_gem_init_global_gtt(struct drm_device *dev) |
||
809 | { |
||
810 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
811 | unsigned long gtt_size, mappable_size; |
||
812 | |||
4104 | Serge | 813 | gtt_size = dev_priv->gtt.base.total; |
3480 | Serge | 814 | mappable_size = dev_priv->gtt.mappable_end; |
815 | |||
816 | if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { |
||
817 | int ret; |
||
3746 | Serge | 818 | |
819 | if (INTEL_INFO(dev)->gen <= 7) { |
||
3480 | Serge | 820 | /* PPGTT pdes are stolen from global gtt ptes, so shrink the |
821 | * aperture accordingly when using aliasing ppgtt. */ |
||
4104 | Serge | 822 | gtt_size -= GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE; |
3746 | Serge | 823 | } |
3480 | Serge | 824 | |
4280 | Serge | 825 | i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size); |
3480 | Serge | 826 | |
827 | ret = i915_gem_init_aliasing_ppgtt(dev); |
||
828 | if (!ret) |
||
829 | return; |
||
830 | |||
831 | DRM_ERROR("Aliased PPGTT setup failed %d\n", ret); |
||
4104 | Serge | 832 | drm_mm_takedown(&dev_priv->gtt.base.mm); |
833 | gtt_size += GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE; |
||
4539 | Serge | 834 | } |
4280 | Serge | 835 | i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size); |
3480 | Serge | 836 | } |
837 | |||
3243 | Serge | 838 | static int setup_scratch_page(struct drm_device *dev) |
839 | { |
||
840 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
841 | struct page *page; |
||
842 | dma_addr_t dma_addr; |
||
843 | |||
844 | page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO); |
||
845 | if (page == NULL) |
||
846 | return -ENOMEM; |
||
4104 | Serge | 847 | get_page(page); |
3480 | Serge | 848 | set_pages_uc(page, 1); |
3243 | Serge | 849 | |
850 | #ifdef CONFIG_INTEL_IOMMU |
||
851 | dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE, |
||
852 | PCI_DMA_BIDIRECTIONAL); |
||
853 | if (pci_dma_mapping_error(dev->pdev, dma_addr)) |
||
854 | return -EINVAL; |
||
855 | #else |
||
856 | dma_addr = page_to_phys(page); |
||
857 | #endif |
||
4104 | Serge | 858 | dev_priv->gtt.base.scratch.page = page; |
859 | dev_priv->gtt.base.scratch.addr = dma_addr; |
||
3243 | Serge | 860 | |
861 | return 0; |
||
862 | } |
||
863 | |||
3480 | Serge | 864 | static void teardown_scratch_page(struct drm_device *dev) |
865 | { |
||
866 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
4104 | Serge | 867 | struct page *page = dev_priv->gtt.base.scratch.page; |
868 | |||
869 | set_pages_wb(page, 1); |
||
870 | pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr, |
||
3480 | Serge | 871 | PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); |
4104 | Serge | 872 | put_page(page); |
873 | __free_page(page); |
||
3480 | Serge | 874 | } |
3243 | Serge | 875 | |
876 | static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) |
||
877 | { |
||
878 | snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; |
||
879 | snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; |
||
880 | return snb_gmch_ctl << 20; |
||
881 | } |
||
882 | |||
3480 | Serge | 883 | static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl) |
3243 | Serge | 884 | { |
885 | snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; |
||
886 | snb_gmch_ctl &= SNB_GMCH_GMS_MASK; |
||
887 | return snb_gmch_ctl << 25; /* 32 MB units */ |
||
888 | } |
||
889 | |||
3480 | Serge | 890 | static int gen6_gmch_probe(struct drm_device *dev, |
891 | size_t *gtt_total, |
||
892 | size_t *stolen, |
||
893 | phys_addr_t *mappable_base, |
||
894 | unsigned long *mappable_end) |
||
3243 | Serge | 895 | { |
896 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
897 | phys_addr_t gtt_bus_addr; |
||
3480 | Serge | 898 | unsigned int gtt_size; |
3243 | Serge | 899 | u16 snb_gmch_ctl; |
900 | int ret; |
||
901 | |||
3480 | Serge | 902 | *mappable_base = pci_resource_start(dev->pdev, 2); |
903 | *mappable_end = pci_resource_len(dev->pdev, 2); |
||
904 | |||
905 | /* 64/512MB is the current min/max we actually know of, but this is just |
||
906 | * a coarse sanity check. |
||
3243 | Serge | 907 | */ |
3480 | Serge | 908 | if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { |
909 | DRM_ERROR("Unknown GMADR size (%lx)\n", |
||
910 | dev_priv->gtt.mappable_end); |
||
911 | return -ENXIO; |
||
3243 | Serge | 912 | } |
913 | |||
3480 | Serge | 914 | if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40))) |
915 | pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40)); |
||
916 | pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); |
||
917 | gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); |
||
3243 | Serge | 918 | |
4104 | Serge | 919 | *stolen = gen6_get_stolen_size(snb_gmch_ctl); |
3746 | Serge | 920 | *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT; |
3243 | Serge | 921 | |
3746 | Serge | 922 | /* For Modern GENs the PTEs and register space are split in the BAR */ |
923 | gtt_bus_addr = pci_resource_start(dev->pdev, 0) + |
||
924 | (pci_resource_len(dev->pdev, 0) / 2); |
||
3243 | Serge | 925 | |
3480 | Serge | 926 | dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size); |
927 | if (!dev_priv->gtt.gsm) { |
||
928 | DRM_ERROR("Failed to map the gtt page table\n"); |
||
929 | return -ENOMEM; |
||
3243 | Serge | 930 | } |
931 | |||
932 | ret = setup_scratch_page(dev); |
||
3480 | Serge | 933 | if (ret) |
3243 | Serge | 934 | DRM_ERROR("Scratch setup failed\n"); |
935 | |||
4104 | Serge | 936 | dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range; |
937 | dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries; |
||
3480 | Serge | 938 | |
939 | return ret; |
||
940 | } |
||
941 | |||
4104 | Serge | 942 | static void gen6_gmch_remove(struct i915_address_space *vm) |
3480 | Serge | 943 | { |
4104 | Serge | 944 | |
945 | struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base); |
||
946 | iounmap(gtt->gsm); |
||
947 | teardown_scratch_page(vm->dev); |
||
3480 | Serge | 948 | } |
949 | |||
950 | static int i915_gmch_probe(struct drm_device *dev, |
||
951 | size_t *gtt_total, |
||
952 | size_t *stolen, |
||
953 | phys_addr_t *mappable_base, |
||
954 | unsigned long *mappable_end) |
||
955 | { |
||
956 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
957 | int ret; |
||
958 | |||
959 | ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL); |
||
960 | if (!ret) { |
||
961 | DRM_ERROR("failed to set up gmch\n"); |
||
962 | return -EIO; |
||
3243 | Serge | 963 | } |
964 | |||
3480 | Serge | 965 | intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end); |
3243 | Serge | 966 | |
3480 | Serge | 967 | dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev); |
4104 | Serge | 968 | dev_priv->gtt.base.clear_range = i915_ggtt_clear_range; |
969 | dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries; |
||
3480 | Serge | 970 | |
3243 | Serge | 971 | return 0; |
3480 | Serge | 972 | } |
3243 | Serge | 973 | |
4104 | Serge | 974 | static void i915_gmch_remove(struct i915_address_space *vm) |
3480 | Serge | 975 | { |
976 | // intel_gmch_remove(); |
||
977 | } |
||
978 | |||
979 | int i915_gem_gtt_init(struct drm_device *dev) |
||
980 | { |
||
981 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
982 | struct i915_gtt *gtt = &dev_priv->gtt; |
||
983 | int ret; |
||
984 | |||
985 | if (INTEL_INFO(dev)->gen <= 5) { |
||
4104 | Serge | 986 | gtt->gtt_probe = i915_gmch_probe; |
987 | gtt->base.cleanup = i915_gmch_remove; |
||
3480 | Serge | 988 | } else { |
4104 | Serge | 989 | gtt->gtt_probe = gen6_gmch_probe; |
990 | gtt->base.cleanup = gen6_gmch_remove; |
||
991 | if (IS_HASWELL(dev) && dev_priv->ellc_size) |
||
992 | gtt->base.pte_encode = iris_pte_encode; |
||
993 | else if (IS_HASWELL(dev)) |
||
994 | gtt->base.pte_encode = hsw_pte_encode; |
||
995 | else if (IS_VALLEYVIEW(dev)) |
||
996 | gtt->base.pte_encode = byt_pte_encode; |
||
997 | else if (INTEL_INFO(dev)->gen >= 7) |
||
998 | gtt->base.pte_encode = ivb_pte_encode; |
||
999 | else |
||
1000 | gtt->base.pte_encode = snb_pte_encode; |
||
3480 | Serge | 1001 | } |
1002 | |||
4104 | Serge | 1003 | ret = gtt->gtt_probe(dev, >t->base.total, >t->stolen_size, |
1004 | >t->mappable_base, >t->mappable_end); |
||
3480 | Serge | 1005 | if (ret) |
4104 | Serge | 1006 | return ret; |
3480 | Serge | 1007 | |
4104 | Serge | 1008 | gtt->base.dev = dev; |
1009 | |||
3480 | Serge | 1010 | /* GMADR is the PCI mmio aperture into the global GTT. */ |
1011 | DRM_INFO("Memory usable by graphics device = %zdM\n", |
||
4104 | Serge | 1012 | gtt->base.total >> 20); |
1013 | DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20); |
||
1014 | DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20); |
||
3480 | Serge | 1015 | |
1016 | return 0; |
||
3243 | Serge | 1017 | } |
1018 | |||
1019 | struct scatterlist *sg_next(struct scatterlist *sg) |
||
1020 | { |
||
1021 | if (sg_is_last(sg)) |
||
1022 | return NULL; |
||
1023 | |||
1024 | sg++; |
||
1025 | if (unlikely(sg_is_chain(sg))) |
||
1026 | sg = sg_chain_ptr(sg); |
||
1027 | |||
1028 | return sg; |
||
1029 | } |
||
1030 | |||
1031 | |||
1032 | void __sg_free_table(struct sg_table *table, unsigned int max_ents, |
||
1033 | sg_free_fn *free_fn) |
||
1034 | { |
||
1035 | struct scatterlist *sgl, *next; |
||
1036 | |||
1037 | if (unlikely(!table->sgl)) |
||
1038 | return; |
||
1039 | |||
1040 | sgl = table->sgl; |
||
1041 | while (table->orig_nents) { |
||
1042 | unsigned int alloc_size = table->orig_nents; |
||
1043 | unsigned int sg_size; |
||
1044 | |||
1045 | /* |
||
1046 | * If we have more than max_ents segments left, |
||
1047 | * then assign 'next' to the sg table after the current one. |
||
1048 | * sg_size is then one less than alloc size, since the last |
||
1049 | * element is the chain pointer. |
||
1050 | */ |
||
1051 | if (alloc_size > max_ents) { |
||
1052 | next = sg_chain_ptr(&sgl[max_ents - 1]); |
||
1053 | alloc_size = max_ents; |
||
1054 | sg_size = alloc_size - 1; |
||
1055 | } else { |
||
1056 | sg_size = alloc_size; |
||
1057 | next = NULL; |
||
1058 | } |
||
1059 | |||
1060 | table->orig_nents -= sg_size; |
||
1061 | kfree(sgl); |
||
1062 | sgl = next; |
||
1063 | } |
||
1064 | |||
1065 | table->sgl = NULL; |
||
1066 | } |
||
1067 | |||
1068 | void sg_free_table(struct sg_table *table) |
||
1069 | { |
||
1070 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
||
1071 | } |
||
1072 | |||
1073 | int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) |
||
1074 | { |
||
1075 | struct scatterlist *sg, *prv; |
||
1076 | unsigned int left; |
||
1077 | unsigned int max_ents = SG_MAX_SINGLE_ALLOC; |
||
1078 | |||
1079 | #ifndef ARCH_HAS_SG_CHAIN |
||
1080 | BUG_ON(nents > max_ents); |
||
1081 | #endif |
||
1082 | |||
1083 | memset(table, 0, sizeof(*table)); |
||
1084 | |||
1085 | left = nents; |
||
1086 | prv = NULL; |
||
1087 | do { |
||
1088 | unsigned int sg_size, alloc_size = left; |
||
1089 | |||
1090 | if (alloc_size > max_ents) { |
||
1091 | alloc_size = max_ents; |
||
1092 | sg_size = alloc_size - 1; |
||
1093 | } else |
||
1094 | sg_size = alloc_size; |
||
1095 | |||
1096 | left -= sg_size; |
||
1097 | |||
1098 | sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask); |
||
1099 | if (unlikely(!sg)) { |
||
1100 | /* |
||
1101 | * Adjust entry count to reflect that the last |
||
1102 | * entry of the previous table won't be used for |
||
1103 | * linkage. Without this, sg_kfree() may get |
||
1104 | * confused. |
||
1105 | */ |
||
1106 | if (prv) |
||
1107 | table->nents = ++table->orig_nents; |
||
1108 | |||
1109 | goto err; |
||
1110 | } |
||
1111 | |||
1112 | sg_init_table(sg, alloc_size); |
||
1113 | table->nents = table->orig_nents += sg_size; |
||
1114 | |||
1115 | /* |
||
1116 | * If this is the first mapping, assign the sg table header. |
||
1117 | * If this is not the first mapping, chain previous part. |
||
1118 | */ |
||
1119 | if (prv) |
||
1120 | sg_chain(prv, max_ents, sg); |
||
1121 | else |
||
1122 | table->sgl = sg; |
||
1123 | |||
1124 | /* |
||
1125 | * If no more entries after this one, mark the end |
||
1126 | */ |
||
1127 | if (!left) |
||
1128 | sg_mark_end(&sg[sg_size - 1]); |
||
1129 | |||
1130 | prv = sg; |
||
1131 | } while (left); |
||
1132 | |||
1133 | return 0; |
||
1134 | |||
1135 | err: |
||
1136 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
||
1137 | |||
1138 | return -ENOMEM; |
||
1139 | } |
||
1140 | |||
1141 | |||
1142 | void sg_init_table(struct scatterlist *sgl, unsigned int nents) |
||
1143 | { |
||
1144 | memset(sgl, 0, sizeof(*sgl) * nents); |
||
1145 | #ifdef CONFIG_DEBUG_SG |
||
1146 | { |
||
1147 | unsigned int i; |
||
1148 | for (i = 0; i < nents; i++) |
||
1149 | sgl[i].sg_magic = SG_MAGIC; |
||
1150 | } |
||
1151 | #endif |
||
1152 | sg_mark_end(&sgl[nents - 1]); |
||
1153 | } |
||
1154 | |||
3746 | Serge | 1155 | |
1156 | void __sg_page_iter_start(struct sg_page_iter *piter, |
||
1157 | struct scatterlist *sglist, unsigned int nents, |
||
1158 | unsigned long pgoffset) |
||
1159 | { |
||
1160 | piter->__pg_advance = 0; |
||
1161 | piter->__nents = nents; |
||
1162 | |||
1163 | piter->sg = sglist; |
||
1164 | piter->sg_pgoffset = pgoffset; |
||
1165 | } |
||
1166 | |||
1167 | static int sg_page_count(struct scatterlist *sg) |
||
1168 | { |
||
1169 | return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT; |
||
1170 | } |
||
1171 | |||
1172 | bool __sg_page_iter_next(struct sg_page_iter *piter) |
||
1173 | { |
||
1174 | if (!piter->__nents || !piter->sg) |
||
1175 | return false; |
||
1176 | |||
1177 | piter->sg_pgoffset += piter->__pg_advance; |
||
1178 | piter->__pg_advance = 1; |
||
1179 | |||
1180 | while (piter->sg_pgoffset >= sg_page_count(piter->sg)) { |
||
1181 | piter->sg_pgoffset -= sg_page_count(piter->sg); |
||
1182 | piter->sg = sg_next(piter->sg); |
||
1183 | if (!--piter->__nents || !piter->sg) |
||
1184 | return false; |
||
1185 | } |
||
1186 | |||
1187 | return true; |
||
1188 | } |
||
1189 | EXPORT_SYMBOL(__sg_page_iter_next);>=>><>20))))><20))))>20)><20)>>><>><>=>>>>>>>>>>>=><=>>><>><>><>><>><>><>><>><>><>><> |
||
1190 |