Rev 3033 | Rev 3480 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
2332 | Serge | 1 | /* |
2 | * Copyright © 2010 Daniel Vetter |
||
3 | * |
||
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
||
5 | * copy of this software and associated documentation files (the "Software"), |
||
6 | * to deal in the Software without restriction, including without limitation |
||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
||
8 | * and/or sell copies of the Software, and to permit persons to whom the |
||
9 | * Software is furnished to do so, subject to the following conditions: |
||
10 | * |
||
11 | * The above copyright notice and this permission notice (including the next |
||
12 | * paragraph) shall be included in all copies or substantial portions of the |
||
13 | * Software. |
||
14 | * |
||
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
||
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
||
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
||
21 | * IN THE SOFTWARE. |
||
22 | * |
||
23 | */ |
||
24 | |||
3243 | Serge | 25 | #define iowrite32(v, addr) writel((v), (addr)) |
26 | |||
3031 | serge | 27 | #include |
28 | #include |
||
2332 | Serge | 29 | #include "i915_drv.h" |
2351 | Serge | 30 | #include "i915_trace.h" |
2332 | Serge | 31 | #include "intel_drv.h" |
32 | |||
33 | #define AGP_USER_TYPES (1 << 16) |
||
34 | #define AGP_USER_MEMORY (AGP_USER_TYPES) |
||
35 | #define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1) |
||
36 | |||
3243 | Serge | 37 | typedef uint32_t gtt_pte_t; |
38 | |||
39 | /* PPGTT stuff */ |
||
40 | #define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) |
||
41 | |||
42 | #define GEN6_PDE_VALID (1 << 0) |
||
43 | /* gen6+ has bit 11-4 for physical addr bit 39-32 */ |
||
44 | #define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) |
||
45 | |||
46 | #define GEN6_PTE_VALID (1 << 0) |
||
47 | #define GEN6_PTE_UNCACHED (1 << 1) |
||
48 | #define HSW_PTE_UNCACHED (0) |
||
49 | #define GEN6_PTE_CACHE_LLC (2 << 1) |
||
50 | #define GEN6_PTE_CACHE_LLC_MLC (3 << 1) |
||
51 | #define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) |
||
52 | |||
53 | static inline gtt_pte_t pte_encode(struct drm_device *dev, |
||
54 | dma_addr_t addr, |
||
55 | enum i915_cache_level level) |
||
56 | { |
||
57 | gtt_pte_t pte = GEN6_PTE_VALID; |
||
58 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
||
59 | |||
60 | switch (level) { |
||
61 | case I915_CACHE_LLC_MLC: |
||
62 | /* Haswell doesn't set L3 this way */ |
||
63 | if (IS_HASWELL(dev)) |
||
64 | pte |= GEN6_PTE_CACHE_LLC; |
||
65 | else |
||
66 | pte |= GEN6_PTE_CACHE_LLC_MLC; |
||
67 | break; |
||
68 | case I915_CACHE_LLC: |
||
69 | pte |= GEN6_PTE_CACHE_LLC; |
||
70 | break; |
||
71 | case I915_CACHE_NONE: |
||
72 | if (IS_HASWELL(dev)) |
||
73 | pte |= HSW_PTE_UNCACHED; |
||
74 | else |
||
75 | pte |= GEN6_PTE_UNCACHED; |
||
76 | break; |
||
77 | default: |
||
78 | BUG(); |
||
79 | } |
||
80 | |||
81 | |||
82 | return pte; |
||
83 | } |
||
84 | |||
3031 | serge | 85 | /* PPGTT support for Sandybdrige/Gen6 and later */ |
86 | static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt, |
||
87 | unsigned first_entry, |
||
88 | unsigned num_entries) |
||
89 | { |
||
3243 | Serge | 90 | gtt_pte_t *pt_vaddr; |
91 | gtt_pte_t scratch_pte; |
||
3031 | serge | 92 | unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; |
93 | unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
||
94 | unsigned last_pte, i; |
||
95 | |||
3243 | Serge | 96 | scratch_pte = pte_encode(ppgtt->dev, ppgtt->scratch_page_dma_addr, |
97 | I915_CACHE_LLC); |
||
3031 | serge | 98 | |
99 | pt_vaddr = AllocKernelSpace(4096); |
||
100 | |||
101 | if(pt_vaddr != NULL) |
||
102 | { |
||
103 | while (num_entries) |
||
104 | { |
||
105 | last_pte = first_pte + num_entries; |
||
106 | if (last_pte > I915_PPGTT_PT_ENTRIES) |
||
107 | last_pte = I915_PPGTT_PT_ENTRIES; |
||
108 | |||
3243 | Serge | 109 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pd]), 3); |
3031 | serge | 110 | |
111 | for (i = first_pte; i < last_pte; i++) |
||
112 | pt_vaddr[i] = scratch_pte; |
||
113 | |||
114 | num_entries -= last_pte - first_pte; |
||
115 | first_pte = 0; |
||
116 | act_pd++; |
||
117 | } |
||
118 | FreeKernelSpace(pt_vaddr); |
||
119 | }; |
||
120 | } |
||
121 | |||
122 | int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) |
||
123 | { |
||
124 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
125 | struct i915_hw_ppgtt *ppgtt; |
||
126 | unsigned first_pd_entry_in_global_pt; |
||
127 | int i; |
||
128 | int ret = -ENOMEM; |
||
129 | |||
130 | /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 |
||
131 | * entries. For aliasing ppgtt support we just steal them at the end for |
||
132 | * now. */ |
||
133 | first_pd_entry_in_global_pt = dev_priv->mm.gtt->gtt_total_entries - I915_PPGTT_PD_ENTRIES; |
||
134 | |||
135 | ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); |
||
136 | if (!ppgtt) |
||
137 | return ret; |
||
138 | |||
139 | ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES; |
||
3243 | Serge | 140 | ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries, |
3031 | serge | 141 | GFP_KERNEL); |
142 | if (!ppgtt->pt_pages) |
||
143 | goto err_ppgtt; |
||
144 | |||
145 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
||
3243 | Serge | 146 | ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL); |
3031 | serge | 147 | if (!ppgtt->pt_pages[i]) |
148 | goto err_pt_alloc; |
||
149 | } |
||
150 | |||
151 | /* |
||
152 | if (dev_priv->mm.gtt->needs_dmar) { |
||
153 | ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) |
||
154 | *ppgtt->num_pd_entries, |
||
155 | GFP_KERNEL); |
||
156 | if (!ppgtt->pt_dma_addr) |
||
157 | goto err_pt_alloc; |
||
158 | |||
159 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
||
160 | dma_addr_t pt_addr; |
||
161 | |||
162 | pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], |
||
163 | 0, 4096, |
||
164 | PCI_DMA_BIDIRECTIONAL); |
||
165 | |||
166 | if (pci_dma_mapping_error(dev->pdev, |
||
167 | pt_addr)) { |
||
168 | ret = -EIO; |
||
169 | goto err_pd_pin; |
||
170 | |||
171 | } |
||
172 | ppgtt->pt_dma_addr[i] = pt_addr; |
||
173 | } |
||
174 | } |
||
175 | */ |
||
176 | ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma; |
||
177 | |||
178 | i915_ppgtt_clear_range(ppgtt, 0, |
||
179 | ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES); |
||
180 | |||
3243 | Serge | 181 | ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(gtt_pte_t); |
3031 | serge | 182 | |
183 | dev_priv->mm.aliasing_ppgtt = ppgtt; |
||
184 | |||
185 | return 0; |
||
186 | |||
187 | err_pd_pin: |
||
188 | // if (ppgtt->pt_dma_addr) { |
||
189 | // for (i--; i >= 0; i--) |
||
190 | // pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], |
||
191 | // 4096, PCI_DMA_BIDIRECTIONAL); |
||
192 | // } |
||
193 | err_pt_alloc: |
||
194 | // kfree(ppgtt->pt_dma_addr); |
||
195 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
||
196 | if (ppgtt->pt_pages[i]) |
||
3243 | Serge | 197 | FreePage((addr_t)(ppgtt->pt_pages[i])); |
3031 | serge | 198 | } |
199 | kfree(ppgtt->pt_pages); |
||
200 | err_ppgtt: |
||
201 | kfree(ppgtt); |
||
202 | |||
203 | return ret; |
||
204 | } |
||
205 | |||
206 | void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) |
||
207 | { |
||
208 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
209 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
||
210 | int i; |
||
211 | |||
212 | if (!ppgtt) |
||
213 | return; |
||
214 | |||
215 | // if (ppgtt->pt_dma_addr) { |
||
216 | // for (i = 0; i < ppgtt->num_pd_entries; i++) |
||
217 | // pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], |
||
218 | // 4096, PCI_DMA_BIDIRECTIONAL); |
||
219 | // } |
||
220 | |||
221 | // kfree(ppgtt->pt_dma_addr); |
||
222 | for (i = 0; i < ppgtt->num_pd_entries; i++) |
||
3243 | Serge | 223 | FreePage((addr_t)(ppgtt->pt_pages[i])); |
3031 | serge | 224 | kfree(ppgtt->pt_pages); |
225 | kfree(ppgtt); |
||
226 | } |
||
227 | |||
228 | static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt, |
||
3243 | Serge | 229 | const struct sg_table *pages, |
3031 | serge | 230 | unsigned first_entry, |
3243 | Serge | 231 | enum i915_cache_level cache_level) |
3031 | serge | 232 | { |
3243 | Serge | 233 | gtt_pte_t *pt_vaddr; |
3031 | serge | 234 | unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; |
235 | unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
||
3243 | Serge | 236 | unsigned i, j, m, segment_len; |
3031 | serge | 237 | dma_addr_t page_addr; |
3243 | Serge | 238 | struct scatterlist *sg; |
3031 | serge | 239 | |
3243 | Serge | 240 | /* init sg walking */ |
241 | sg = pages->sgl; |
||
3031 | serge | 242 | i = 0; |
3243 | Serge | 243 | segment_len = sg_dma_len(sg) >> PAGE_SHIFT; |
244 | m = 0; |
||
3031 | serge | 245 | |
246 | pt_vaddr = AllocKernelSpace(4096); |
||
3243 | Serge | 247 | if( pt_vaddr == NULL) |
248 | return; |
||
3031 | serge | 249 | |
3243 | Serge | 250 | while (i < pages->nents) { |
251 | MapPage(pt_vaddr,(addr_t)ppgtt->pt_pages[act_pd], 3); |
||
3031 | serge | 252 | |
3243 | Serge | 253 | for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) { |
254 | page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT); |
||
255 | pt_vaddr[j] = pte_encode(ppgtt->dev, page_addr, |
||
256 | cache_level); |
||
3031 | serge | 257 | |
3243 | Serge | 258 | /* grab the next page */ |
259 | if (++m == segment_len) { |
||
260 | if (++i == pages->nents) |
||
261 | break; |
||
262 | |||
263 | sg = sg_next(sg); |
||
264 | segment_len = sg_dma_len(sg) >> PAGE_SHIFT; |
||
265 | m = 0; |
||
266 | } |
||
267 | } |
||
268 | |||
269 | |||
270 | first_pte = 0; |
||
271 | act_pd++; |
||
3031 | serge | 272 | } |
273 | FreeKernelSpace(pt_vaddr); |
||
274 | } |
||
275 | |||
276 | void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, |
||
277 | struct drm_i915_gem_object *obj, |
||
278 | enum i915_cache_level cache_level) |
||
279 | { |
||
280 | i915_ppgtt_insert_sg_entries(ppgtt, |
||
3243 | Serge | 281 | obj->pages, |
3031 | serge | 282 | obj->gtt_space->start >> PAGE_SHIFT, |
3243 | Serge | 283 | cache_level); |
3031 | serge | 284 | } |
285 | |||
286 | void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, |
||
287 | struct drm_i915_gem_object *obj) |
||
288 | { |
||
289 | i915_ppgtt_clear_range(ppgtt, |
||
290 | obj->gtt_space->start >> PAGE_SHIFT, |
||
291 | obj->base.size >> PAGE_SHIFT); |
||
292 | } |
||
293 | |||
3243 | Serge | 294 | void i915_gem_init_ppgtt(struct drm_device *dev) |
2332 | Serge | 295 | { |
3243 | Serge | 296 | drm_i915_private_t *dev_priv = dev->dev_private; |
297 | uint32_t pd_offset; |
||
298 | struct intel_ring_buffer *ring; |
||
299 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
||
300 | uint32_t __iomem *pd_addr; |
||
301 | uint32_t pd_entry; |
||
302 | int i; |
||
303 | |||
304 | if (!dev_priv->mm.aliasing_ppgtt) |
||
305 | return; |
||
306 | |||
307 | |||
308 | pd_addr = dev_priv->mm.gtt->gtt + ppgtt->pd_offset/sizeof(uint32_t); |
||
309 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
||
310 | dma_addr_t pt_addr; |
||
311 | |||
312 | if (dev_priv->mm.gtt->needs_dmar) |
||
313 | pt_addr = ppgtt->pt_dma_addr[i]; |
||
314 | else |
||
315 | pt_addr = page_to_phys(ppgtt->pt_pages[i]); |
||
316 | |||
317 | pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); |
||
318 | pd_entry |= GEN6_PDE_VALID; |
||
319 | |||
320 | writel(pd_entry, pd_addr + i); |
||
2332 | Serge | 321 | } |
3243 | Serge | 322 | readl(pd_addr); |
323 | |||
324 | pd_offset = ppgtt->pd_offset; |
||
325 | pd_offset /= 64; /* in cachelines, */ |
||
326 | pd_offset <<= 16; |
||
327 | |||
328 | if (INTEL_INFO(dev)->gen == 6) { |
||
329 | uint32_t ecochk, gab_ctl, ecobits; |
||
330 | |||
331 | ecobits = I915_READ(GAC_ECO_BITS); |
||
332 | I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); |
||
333 | |||
334 | gab_ctl = I915_READ(GAB_CTL); |
||
335 | I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); |
||
336 | |||
337 | ecochk = I915_READ(GAM_ECOCHK); |
||
338 | I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | |
||
339 | ECOCHK_PPGTT_CACHE64B); |
||
340 | I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
||
341 | } else if (INTEL_INFO(dev)->gen >= 7) { |
||
342 | I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); |
||
343 | /* GFX_MODE is per-ring on gen7+ */ |
||
344 | } |
||
345 | |||
346 | for_each_ring(ring, dev_priv, i) { |
||
347 | if (INTEL_INFO(dev)->gen >= 7) |
||
348 | I915_WRITE(RING_MODE_GEN7(ring), |
||
349 | _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
||
350 | |||
351 | I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); |
||
352 | I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); |
||
353 | } |
||
2332 | Serge | 354 | } |
355 | |||
2344 | Serge | 356 | static bool do_idling(struct drm_i915_private *dev_priv) |
357 | { |
||
358 | bool ret = dev_priv->mm.interruptible; |
||
359 | |||
360 | if (unlikely(dev_priv->mm.gtt->do_idle_maps)) { |
||
361 | dev_priv->mm.interruptible = false; |
||
362 | if (i915_gpu_idle(dev_priv->dev)) { |
||
363 | DRM_ERROR("Couldn't idle GPU\n"); |
||
364 | /* Wait a bit, in hopes it avoids the hang */ |
||
365 | udelay(10); |
||
366 | } |
||
367 | } |
||
368 | |||
369 | return ret; |
||
370 | } |
||
371 | |||
372 | static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) |
||
373 | { |
||
374 | if (unlikely(dev_priv->mm.gtt->do_idle_maps)) |
||
375 | dev_priv->mm.interruptible = interruptible; |
||
376 | } |
||
377 | |||
3243 | Serge | 378 | |
379 | static void i915_ggtt_clear_range(struct drm_device *dev, |
||
380 | unsigned first_entry, |
||
381 | unsigned num_entries) |
||
382 | { |
||
383 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
384 | gtt_pte_t scratch_pte; |
||
385 | gtt_pte_t __iomem *gtt_base = dev_priv->mm.gtt->gtt + first_entry; |
||
386 | const int max_entries = dev_priv->mm.gtt->gtt_total_entries - first_entry; |
||
387 | int i; |
||
388 | |||
389 | if (INTEL_INFO(dev)->gen < 6) { |
||
390 | intel_gtt_clear_range(first_entry, num_entries); |
||
391 | return; |
||
392 | } |
||
393 | |||
394 | if (WARN(num_entries > max_entries, |
||
395 | "First entry = %d; Num entries = %d (max=%d)\n", |
||
396 | first_entry, num_entries, max_entries)) |
||
397 | num_entries = max_entries; |
||
398 | |||
399 | scratch_pte = pte_encode(dev, dev_priv->mm.gtt->scratch_page_dma, I915_CACHE_LLC); |
||
400 | for (i = 0; i < num_entries; i++) |
||
401 | iowrite32(scratch_pte, >t_base[i]); |
||
402 | readl(gtt_base); |
||
403 | } |
||
404 | |||
405 | |||
2332 | Serge | 406 | #if 0 |
407 | void i915_gem_restore_gtt_mappings(struct drm_device *dev) |
||
408 | { |
||
409 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
410 | struct drm_i915_gem_object *obj; |
||
411 | |||
412 | /* First fill our portion of the GTT with scratch pages */ |
||
3243 | Serge | 413 | i915_ggtt_clear_range(dev, dev_priv->mm.gtt_start / PAGE_SIZE, |
2332 | Serge | 414 | (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE); |
415 | |||
3031 | serge | 416 | list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) { |
2332 | Serge | 417 | i915_gem_clflush_object(obj); |
3031 | serge | 418 | i915_gem_gtt_bind_object(obj, obj->cache_level); |
2332 | Serge | 419 | } |
420 | |||
3243 | Serge | 421 | i915_gem_chipset_flush(dev); |
2332 | Serge | 422 | } |
423 | #endif |
||
424 | |||
3031 | serge | 425 | int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) |
2332 | Serge | 426 | { |
3243 | Serge | 427 | struct scatterlist *sg, *s; |
428 | unsigned int nents ; |
||
429 | int i; |
||
430 | |||
431 | if (obj->has_dma_mapping) |
||
432 | return 0; |
||
433 | |||
434 | sg = obj->pages->sgl; |
||
435 | nents = obj->pages->nents; |
||
436 | |||
437 | |||
438 | WARN_ON(nents == 0 || sg[0].length == 0); |
||
439 | |||
440 | for_each_sg(sg, s, nents, i) { |
||
441 | BUG_ON(!sg_page(s)); |
||
442 | s->dma_address = sg_phys(s); |
||
443 | } |
||
444 | |||
445 | asm volatile("lock; addl $0,0(%%esp)": : :"memory"); |
||
446 | |||
2332 | Serge | 447 | return 0; |
448 | } |
||
449 | |||
3243 | Serge | 450 | /* |
451 | * Binds an object into the global gtt with the specified cache level. The object |
||
452 | * will be accessible to the GPU via commands whose operands reference offsets |
||
453 | * within the global GTT as well as accessible by the GPU through the GMADR |
||
454 | * mapped BAR (dev_priv->mm.gtt->gtt). |
||
455 | */ |
||
456 | static void gen6_ggtt_bind_object(struct drm_i915_gem_object *obj, |
||
457 | enum i915_cache_level level) |
||
458 | { |
||
459 | struct drm_device *dev = obj->base.dev; |
||
460 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
461 | struct sg_table *st = obj->pages; |
||
462 | struct scatterlist *sg = st->sgl; |
||
463 | const int first_entry = obj->gtt_space->start >> PAGE_SHIFT; |
||
464 | const int max_entries = dev_priv->mm.gtt->gtt_total_entries - first_entry; |
||
465 | gtt_pte_t __iomem *gtt_entries = dev_priv->mm.gtt->gtt + first_entry; |
||
466 | int unused, i = 0; |
||
467 | unsigned int len, m = 0; |
||
468 | dma_addr_t addr; |
||
469 | |||
470 | for_each_sg(st->sgl, sg, st->nents, unused) { |
||
471 | len = sg_dma_len(sg) >> PAGE_SHIFT; |
||
472 | for (m = 0; m < len; m++) { |
||
473 | addr = sg_dma_address(sg) + (m << PAGE_SHIFT); |
||
474 | iowrite32(pte_encode(dev, addr, level), >t_entries[i]); |
||
475 | i++; |
||
476 | } |
||
477 | } |
||
478 | |||
479 | BUG_ON(i > max_entries); |
||
480 | BUG_ON(i != obj->base.size / PAGE_SIZE); |
||
481 | |||
482 | /* XXX: This serves as a posting read to make sure that the PTE has |
||
483 | * actually been updated. There is some concern that even though |
||
484 | * registers and PTEs are within the same BAR that they are potentially |
||
485 | * of NUMA access patterns. Therefore, even with the way we assume |
||
486 | * hardware should work, we must keep this posting read for paranoia. |
||
487 | */ |
||
488 | if (i != 0) |
||
489 | WARN_ON(readl(>t_entries[i-1]) != pte_encode(dev, addr, level)); |
||
490 | |||
491 | /* This next bit makes the above posting read even more important. We |
||
492 | * want to flush the TLBs only after we're certain all the PTE updates |
||
493 | * have finished. |
||
494 | */ |
||
495 | I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); |
||
496 | POSTING_READ(GFX_FLSH_CNTL_GEN6); |
||
497 | } |
||
498 | |||
3031 | serge | 499 | void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj, |
2332 | Serge | 500 | enum i915_cache_level cache_level) |
501 | { |
||
502 | struct drm_device *dev = obj->base.dev; |
||
3243 | Serge | 503 | if (INTEL_INFO(dev)->gen < 6) { |
504 | unsigned int flags = (cache_level == I915_CACHE_NONE) ? |
||
505 | AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; |
||
506 | intel_gtt_insert_sg_entries(obj->pages, |
||
507 | obj->gtt_space->start >> PAGE_SHIFT, |
||
508 | flags); |
||
509 | } else { |
||
510 | gen6_ggtt_bind_object(obj, cache_level); |
||
511 | } |
||
2332 | Serge | 512 | |
3031 | serge | 513 | obj->has_global_gtt_mapping = 1; |
2332 | Serge | 514 | } |
515 | |||
516 | void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) |
||
517 | { |
||
3243 | Serge | 518 | i915_ggtt_clear_range(obj->base.dev, |
519 | obj->gtt_space->start >> PAGE_SHIFT, |
||
3031 | serge | 520 | obj->base.size >> PAGE_SHIFT); |
521 | |||
522 | obj->has_global_gtt_mapping = 0; |
||
523 | } |
||
524 | |||
525 | void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) |
||
526 | { |
||
2344 | Serge | 527 | struct drm_device *dev = obj->base.dev; |
528 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
529 | bool interruptible; |
||
530 | |||
531 | interruptible = do_idling(dev_priv); |
||
532 | |||
3031 | serge | 533 | // if (!obj->has_dma_mapping) |
534 | // dma_unmap_sg(&dev->pdev->dev, |
||
535 | // obj->pages->sgl, obj->pages->nents, |
||
536 | // PCI_DMA_BIDIRECTIONAL); |
||
2332 | Serge | 537 | |
3031 | serge | 538 | undo_idling(dev_priv, interruptible); |
539 | } |
||
540 | |||
541 | static void i915_gtt_color_adjust(struct drm_mm_node *node, |
||
542 | unsigned long color, |
||
543 | unsigned long *start, |
||
544 | unsigned long *end) |
||
545 | { |
||
546 | if (node->color != color) |
||
547 | *start += 4096; |
||
548 | |||
549 | if (!list_empty(&node->node_list)) { |
||
550 | node = list_entry(node->node_list.next, |
||
551 | struct drm_mm_node, |
||
552 | node_list); |
||
553 | if (node->allocated && node->color != color) |
||
554 | *end -= 4096; |
||
2332 | Serge | 555 | } |
3031 | serge | 556 | } |
2344 | Serge | 557 | |
3031 | serge | 558 | void i915_gem_init_global_gtt(struct drm_device *dev, |
559 | unsigned long start, |
||
560 | unsigned long mappable_end, |
||
561 | unsigned long end) |
||
562 | { |
||
563 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
564 | |||
565 | /* Substract the guard page ... */ |
||
566 | drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE); |
||
567 | if (!HAS_LLC(dev)) |
||
568 | dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust; |
||
569 | |||
570 | dev_priv->mm.gtt_start = start; |
||
571 | dev_priv->mm.gtt_mappable_end = mappable_end; |
||
572 | dev_priv->mm.gtt_end = end; |
||
573 | dev_priv->mm.gtt_total = end - start; |
||
574 | dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start; |
||
575 | |||
576 | /* ... but ensure that we clear the entire range. */ |
||
3243 | Serge | 577 | i915_ggtt_clear_range(dev, start / PAGE_SIZE, (end-start) / PAGE_SIZE); |
2332 | Serge | 578 | } |
3243 | Serge | 579 | |
580 | static int setup_scratch_page(struct drm_device *dev) |
||
581 | { |
||
582 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
583 | struct page *page; |
||
584 | dma_addr_t dma_addr; |
||
585 | |||
586 | page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO); |
||
587 | if (page == NULL) |
||
588 | return -ENOMEM; |
||
589 | |||
590 | #ifdef CONFIG_INTEL_IOMMU |
||
591 | dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE, |
||
592 | PCI_DMA_BIDIRECTIONAL); |
||
593 | if (pci_dma_mapping_error(dev->pdev, dma_addr)) |
||
594 | return -EINVAL; |
||
595 | #else |
||
596 | dma_addr = page_to_phys(page); |
||
597 | #endif |
||
598 | dev_priv->mm.gtt->scratch_page = page; |
||
599 | dev_priv->mm.gtt->scratch_page_dma = dma_addr; |
||
600 | |||
601 | return 0; |
||
602 | } |
||
603 | |||
604 | |||
605 | static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) |
||
606 | { |
||
607 | snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; |
||
608 | snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; |
||
609 | return snb_gmch_ctl << 20; |
||
610 | } |
||
611 | |||
612 | static inline unsigned int gen6_get_stolen_size(u16 snb_gmch_ctl) |
||
613 | { |
||
614 | snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; |
||
615 | snb_gmch_ctl &= SNB_GMCH_GMS_MASK; |
||
616 | return snb_gmch_ctl << 25; /* 32 MB units */ |
||
617 | } |
||
618 | |||
619 | static inline unsigned int gen7_get_stolen_size(u16 snb_gmch_ctl) |
||
620 | { |
||
621 | static const int stolen_decoder[] = { |
||
622 | 0, 0, 0, 0, 0, 32, 48, 64, 128, 256, 96, 160, 224, 352}; |
||
623 | snb_gmch_ctl >>= IVB_GMCH_GMS_SHIFT; |
||
624 | snb_gmch_ctl &= IVB_GMCH_GMS_MASK; |
||
625 | return stolen_decoder[snb_gmch_ctl] << 20; |
||
626 | } |
||
627 | |||
628 | int i915_gem_gtt_init(struct drm_device *dev) |
||
629 | { |
||
630 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
631 | phys_addr_t gtt_bus_addr; |
||
632 | u16 snb_gmch_ctl; |
||
633 | int ret; |
||
634 | |||
635 | /* On modern platforms we need not worry ourself with the legacy |
||
636 | * hostbridge query stuff. Skip it entirely |
||
637 | */ |
||
638 | if (INTEL_INFO(dev)->gen < 6) { |
||
639 | ret = intel_gmch_probe(dev_priv->bridge_dev, dev->pdev, NULL); |
||
640 | if (!ret) { |
||
641 | DRM_ERROR("failed to set up gmch\n"); |
||
642 | return -EIO; |
||
643 | } |
||
644 | |||
645 | dev_priv->mm.gtt = intel_gtt_get(); |
||
646 | if (!dev_priv->mm.gtt) { |
||
647 | DRM_ERROR("Failed to initialize GTT\n"); |
||
648 | return -ENODEV; |
||
649 | } |
||
650 | return 0; |
||
651 | } |
||
652 | |||
653 | dev_priv->mm.gtt = kzalloc(sizeof(*dev_priv->mm.gtt), GFP_KERNEL); |
||
654 | if (!dev_priv->mm.gtt) |
||
655 | return -ENOMEM; |
||
656 | |||
657 | |||
658 | #ifdef CONFIG_INTEL_IOMMU |
||
659 | dev_priv->mm.gtt->needs_dmar = 1; |
||
660 | #endif |
||
661 | |||
662 | /* For GEN6+ the PTEs for the ggtt live at 2MB + BAR0 */ |
||
663 | gtt_bus_addr = pci_resource_start(dev->pdev, 0) + (2<<20); |
||
664 | dev_priv->mm.gtt->gma_bus_addr = pci_resource_start(dev->pdev, 2); |
||
665 | |||
666 | /* i9xx_setup */ |
||
667 | pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); |
||
668 | dev_priv->mm.gtt->gtt_total_entries = |
||
669 | gen6_get_total_gtt_size(snb_gmch_ctl) / sizeof(gtt_pte_t); |
||
670 | if (INTEL_INFO(dev)->gen < 7) |
||
671 | dev_priv->mm.gtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl); |
||
672 | else |
||
673 | dev_priv->mm.gtt->stolen_size = gen7_get_stolen_size(snb_gmch_ctl); |
||
674 | |||
675 | dev_priv->mm.gtt->gtt_mappable_entries = pci_resource_len(dev->pdev, 2) >> PAGE_SHIFT; |
||
676 | /* 64/512MB is the current min/max we actually know of, but this is just a |
||
677 | * coarse sanity check. |
||
678 | */ |
||
679 | if ((dev_priv->mm.gtt->gtt_mappable_entries >> 8) < 64 || |
||
680 | dev_priv->mm.gtt->gtt_mappable_entries > dev_priv->mm.gtt->gtt_total_entries) { |
||
681 | DRM_ERROR("Unknown GMADR entries (%d)\n", |
||
682 | dev_priv->mm.gtt->gtt_mappable_entries); |
||
683 | ret = -ENXIO; |
||
684 | goto err_out; |
||
685 | } |
||
686 | |||
687 | ret = setup_scratch_page(dev); |
||
688 | if (ret) { |
||
689 | DRM_ERROR("Scratch setup failed\n"); |
||
690 | goto err_out; |
||
691 | } |
||
692 | |||
693 | dev_priv->mm.gtt->gtt = ioremap(gtt_bus_addr, |
||
694 | dev_priv->mm.gtt->gtt_total_entries * sizeof(gtt_pte_t)); |
||
695 | if (!dev_priv->mm.gtt->gtt) { |
||
696 | DRM_ERROR("Failed to map the gtt page table\n"); |
||
697 | ret = -ENOMEM; |
||
698 | goto err_out; |
||
699 | } |
||
700 | |||
701 | /* GMADR is the PCI aperture used by SW to access tiled GFX surfaces in a linear fashion. */ |
||
702 | DRM_INFO("Memory usable by graphics device = %dM\n", dev_priv->mm.gtt->gtt_total_entries >> 8); |
||
703 | DRM_DEBUG_DRIVER("GMADR size = %dM\n", dev_priv->mm.gtt->gtt_mappable_entries >> 8); |
||
704 | DRM_DEBUG_DRIVER("GTT stolen size = %dM\n", dev_priv->mm.gtt->stolen_size >> 20); |
||
705 | |||
706 | return 0; |
||
707 | |||
708 | err_out: |
||
709 | kfree(dev_priv->mm.gtt); |
||
710 | return ret; |
||
711 | } |
||
712 | |||
713 | |||
714 | struct scatterlist *sg_next(struct scatterlist *sg) |
||
715 | { |
||
716 | if (sg_is_last(sg)) |
||
717 | return NULL; |
||
718 | |||
719 | sg++; |
||
720 | if (unlikely(sg_is_chain(sg))) |
||
721 | sg = sg_chain_ptr(sg); |
||
722 | |||
723 | return sg; |
||
724 | } |
||
725 | |||
726 | |||
727 | void __sg_free_table(struct sg_table *table, unsigned int max_ents, |
||
728 | sg_free_fn *free_fn) |
||
729 | { |
||
730 | struct scatterlist *sgl, *next; |
||
731 | |||
732 | if (unlikely(!table->sgl)) |
||
733 | return; |
||
734 | |||
735 | sgl = table->sgl; |
||
736 | while (table->orig_nents) { |
||
737 | unsigned int alloc_size = table->orig_nents; |
||
738 | unsigned int sg_size; |
||
739 | |||
740 | /* |
||
741 | * If we have more than max_ents segments left, |
||
742 | * then assign 'next' to the sg table after the current one. |
||
743 | * sg_size is then one less than alloc size, since the last |
||
744 | * element is the chain pointer. |
||
745 | */ |
||
746 | if (alloc_size > max_ents) { |
||
747 | next = sg_chain_ptr(&sgl[max_ents - 1]); |
||
748 | alloc_size = max_ents; |
||
749 | sg_size = alloc_size - 1; |
||
750 | } else { |
||
751 | sg_size = alloc_size; |
||
752 | next = NULL; |
||
753 | } |
||
754 | |||
755 | table->orig_nents -= sg_size; |
||
756 | kfree(sgl); |
||
757 | sgl = next; |
||
758 | } |
||
759 | |||
760 | table->sgl = NULL; |
||
761 | } |
||
762 | |||
763 | void sg_free_table(struct sg_table *table) |
||
764 | { |
||
765 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
||
766 | } |
||
767 | |||
768 | int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) |
||
769 | { |
||
770 | struct scatterlist *sg, *prv; |
||
771 | unsigned int left; |
||
772 | unsigned int max_ents = SG_MAX_SINGLE_ALLOC; |
||
773 | |||
774 | #ifndef ARCH_HAS_SG_CHAIN |
||
775 | BUG_ON(nents > max_ents); |
||
776 | #endif |
||
777 | |||
778 | memset(table, 0, sizeof(*table)); |
||
779 | |||
780 | left = nents; |
||
781 | prv = NULL; |
||
782 | do { |
||
783 | unsigned int sg_size, alloc_size = left; |
||
784 | |||
785 | if (alloc_size > max_ents) { |
||
786 | alloc_size = max_ents; |
||
787 | sg_size = alloc_size - 1; |
||
788 | } else |
||
789 | sg_size = alloc_size; |
||
790 | |||
791 | left -= sg_size; |
||
792 | |||
793 | sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask); |
||
794 | if (unlikely(!sg)) { |
||
795 | /* |
||
796 | * Adjust entry count to reflect that the last |
||
797 | * entry of the previous table won't be used for |
||
798 | * linkage. Without this, sg_kfree() may get |
||
799 | * confused. |
||
800 | */ |
||
801 | if (prv) |
||
802 | table->nents = ++table->orig_nents; |
||
803 | |||
804 | goto err; |
||
805 | } |
||
806 | |||
807 | sg_init_table(sg, alloc_size); |
||
808 | table->nents = table->orig_nents += sg_size; |
||
809 | |||
810 | /* |
||
811 | * If this is the first mapping, assign the sg table header. |
||
812 | * If this is not the first mapping, chain previous part. |
||
813 | */ |
||
814 | if (prv) |
||
815 | sg_chain(prv, max_ents, sg); |
||
816 | else |
||
817 | table->sgl = sg; |
||
818 | |||
819 | /* |
||
820 | * If no more entries after this one, mark the end |
||
821 | */ |
||
822 | if (!left) |
||
823 | sg_mark_end(&sg[sg_size - 1]); |
||
824 | |||
825 | prv = sg; |
||
826 | } while (left); |
||
827 | |||
828 | return 0; |
||
829 | |||
830 | err: |
||
831 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
||
832 | |||
833 | return -ENOMEM; |
||
834 | } |
||
835 | |||
836 | |||
837 | void sg_init_table(struct scatterlist *sgl, unsigned int nents) |
||
838 | { |
||
839 | memset(sgl, 0, sizeof(*sgl) * nents); |
||
840 | #ifdef CONFIG_DEBUG_SG |
||
841 | { |
||
842 | unsigned int i; |
||
843 | for (i = 0; i < nents; i++) |
||
844 | sgl[i].sg_magic = SG_MAGIC; |
||
845 | } |
||
846 | #endif |
||
847 | sg_mark_end(&sgl[nents - 1]); |
||
848 | }>>>20); |
||
849 |