Rev 3764 | Rev 4126 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
2332 | Serge | 1 | /* |
2 | * Copyright © 2010 Daniel Vetter |
||
3 | * |
||
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
||
5 | * copy of this software and associated documentation files (the "Software"), |
||
6 | * to deal in the Software without restriction, including without limitation |
||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
||
8 | * and/or sell copies of the Software, and to permit persons to whom the |
||
9 | * Software is furnished to do so, subject to the following conditions: |
||
10 | * |
||
11 | * The above copyright notice and this permission notice (including the next |
||
12 | * paragraph) shall be included in all copies or substantial portions of the |
||
13 | * Software. |
||
14 | * |
||
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
||
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
||
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
||
21 | * IN THE SOFTWARE. |
||
22 | * |
||
23 | */ |
||
24 | |||
3243 | Serge | 25 | |
3480 | Serge | 26 | #define AGP_NORMAL_MEMORY 0 |
27 | |||
28 | #define AGP_USER_TYPES (1 << 16) |
||
29 | #define AGP_USER_MEMORY (AGP_USER_TYPES) |
||
30 | #define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1) |
||
31 | |||
3031 | serge | 32 | #include |
33 | #include |
||
2332 | Serge | 34 | #include "i915_drv.h" |
2351 | Serge | 35 | #include "i915_trace.h" |
2332 | Serge | 36 | #include "intel_drv.h" |
37 | |||
4104 | Serge | 38 | #define GEN6_PPGTT_PD_ENTRIES 512 |
39 | #define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t)) |
||
3243 | Serge | 40 | |
41 | /* PPGTT stuff */ |
||
42 | #define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) |
||
4104 | Serge | 43 | #define HSW_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0x7f0)) |
3243 | Serge | 44 | |
45 | #define GEN6_PDE_VALID (1 << 0) |
||
46 | /* gen6+ has bit 11-4 for physical addr bit 39-32 */ |
||
47 | #define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) |
||
48 | |||
49 | #define GEN6_PTE_VALID (1 << 0) |
||
50 | #define GEN6_PTE_UNCACHED (1 << 1) |
||
51 | #define HSW_PTE_UNCACHED (0) |
||
52 | #define GEN6_PTE_CACHE_LLC (2 << 1) |
||
4104 | Serge | 53 | #define GEN7_PTE_CACHE_L3_LLC (3 << 1) |
3243 | Serge | 54 | #define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) |
4104 | Serge | 55 | #define HSW_PTE_ADDR_ENCODE(addr) HSW_GTT_ADDR_ENCODE(addr) |
3243 | Serge | 56 | |
4104 | Serge | 57 | /* Cacheability Control is a 4-bit value. The low three bits are stored in * |
58 | * bits 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE. |
||
59 | */ |
||
60 | #define HSW_CACHEABILITY_CONTROL(bits) ((((bits) & 0x7) << 1) | \ |
||
61 | (((bits) & 0x8) << (11 - 3))) |
||
62 | #define HSW_WB_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x2) |
||
63 | #define HSW_WB_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x3) |
||
64 | #define HSW_WB_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0xb) |
||
65 | #define HSW_WT_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x6) |
||
66 | |||
67 | static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr, |
||
68 | enum i915_cache_level level) |
||
69 | { |
||
70 | gen6_gtt_pte_t pte = GEN6_PTE_VALID; |
||
71 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
||
72 | |||
73 | switch (level) { |
||
74 | case I915_CACHE_L3_LLC: |
||
75 | case I915_CACHE_LLC: |
||
76 | pte |= GEN6_PTE_CACHE_LLC; |
||
77 | break; |
||
78 | case I915_CACHE_NONE: |
||
79 | pte |= GEN6_PTE_UNCACHED; |
||
80 | break; |
||
81 | default: |
||
82 | WARN_ON(1); |
||
83 | } |
||
84 | |||
85 | return pte; |
||
86 | } |
||
87 | |||
88 | static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr, |
||
3243 | Serge | 89 | enum i915_cache_level level) |
90 | { |
||
3746 | Serge | 91 | gen6_gtt_pte_t pte = GEN6_PTE_VALID; |
3243 | Serge | 92 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
93 | |||
94 | switch (level) { |
||
4104 | Serge | 95 | case I915_CACHE_L3_LLC: |
96 | pte |= GEN7_PTE_CACHE_L3_LLC; |
||
3243 | Serge | 97 | break; |
98 | case I915_CACHE_LLC: |
||
99 | pte |= GEN6_PTE_CACHE_LLC; |
||
100 | break; |
||
101 | case I915_CACHE_NONE: |
||
102 | pte |= GEN6_PTE_UNCACHED; |
||
103 | break; |
||
104 | default: |
||
4104 | Serge | 105 | WARN_ON(1); |
3243 | Serge | 106 | } |
107 | |||
108 | return pte; |
||
109 | } |
||
110 | |||
4104 | Serge | 111 | #define BYT_PTE_WRITEABLE (1 << 1) |
112 | #define BYT_PTE_SNOOPED_BY_CPU_CACHES (1 << 2) |
||
113 | |||
114 | static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr, |
||
115 | enum i915_cache_level level) |
||
3746 | Serge | 116 | { |
4104 | Serge | 117 | gen6_gtt_pte_t pte = GEN6_PTE_VALID; |
118 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
||
119 | |||
120 | /* Mark the page as writeable. Other platforms don't have a |
||
121 | * setting for read-only/writable, so this matches that behavior. |
||
122 | */ |
||
123 | pte |= BYT_PTE_WRITEABLE; |
||
124 | |||
125 | if (level != I915_CACHE_NONE) |
||
126 | pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; |
||
127 | |||
128 | return pte; |
||
129 | } |
||
130 | |||
131 | static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr, |
||
132 | enum i915_cache_level level) |
||
133 | { |
||
134 | gen6_gtt_pte_t pte = GEN6_PTE_VALID; |
||
135 | pte |= HSW_PTE_ADDR_ENCODE(addr); |
||
136 | |||
137 | if (level != I915_CACHE_NONE) |
||
138 | pte |= HSW_WB_LLC_AGE3; |
||
139 | |||
140 | return pte; |
||
141 | } |
||
142 | |||
143 | static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr, |
||
144 | enum i915_cache_level level) |
||
145 | { |
||
146 | gen6_gtt_pte_t pte = GEN6_PTE_VALID; |
||
147 | pte |= HSW_PTE_ADDR_ENCODE(addr); |
||
148 | |||
149 | switch (level) { |
||
150 | case I915_CACHE_NONE: |
||
151 | break; |
||
152 | case I915_CACHE_WT: |
||
153 | pte |= HSW_WT_ELLC_LLC_AGE0; |
||
154 | break; |
||
155 | default: |
||
156 | pte |= HSW_WB_ELLC_LLC_AGE0; |
||
157 | break; |
||
158 | } |
||
159 | |||
160 | return pte; |
||
161 | } |
||
162 | |||
163 | static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt) |
||
164 | { |
||
165 | struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private; |
||
3746 | Serge | 166 | gen6_gtt_pte_t __iomem *pd_addr; |
167 | uint32_t pd_entry; |
||
168 | int i; |
||
169 | |||
4104 | Serge | 170 | WARN_ON(ppgtt->pd_offset & 0x3f); |
3746 | Serge | 171 | pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm + |
172 | ppgtt->pd_offset / sizeof(gen6_gtt_pte_t); |
||
173 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
||
174 | dma_addr_t pt_addr; |
||
175 | |||
176 | pt_addr = ppgtt->pt_dma_addr[i]; |
||
177 | pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); |
||
178 | pd_entry |= GEN6_PDE_VALID; |
||
179 | |||
180 | writel(pd_entry, pd_addr + i); |
||
181 | } |
||
182 | readl(pd_addr); |
||
4104 | Serge | 183 | } |
3746 | Serge | 184 | |
4104 | Serge | 185 | static int gen6_ppgtt_enable(struct drm_device *dev) |
186 | { |
||
187 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
188 | uint32_t pd_offset; |
||
189 | struct intel_ring_buffer *ring; |
||
190 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
||
191 | int i; |
||
192 | |||
193 | BUG_ON(ppgtt->pd_offset & 0x3f); |
||
194 | |||
195 | gen6_write_pdes(ppgtt); |
||
196 | |||
3746 | Serge | 197 | pd_offset = ppgtt->pd_offset; |
198 | pd_offset /= 64; /* in cachelines, */ |
||
199 | pd_offset <<= 16; |
||
200 | |||
201 | if (INTEL_INFO(dev)->gen == 6) { |
||
202 | uint32_t ecochk, gab_ctl, ecobits; |
||
203 | |||
204 | ecobits = I915_READ(GAC_ECO_BITS); |
||
205 | I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | |
||
206 | ECOBITS_PPGTT_CACHE64B); |
||
207 | |||
208 | gab_ctl = I915_READ(GAB_CTL); |
||
209 | I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); |
||
210 | |||
211 | ecochk = I915_READ(GAM_ECOCHK); |
||
212 | I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | |
||
213 | ECOCHK_PPGTT_CACHE64B); |
||
214 | I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
||
215 | } else if (INTEL_INFO(dev)->gen >= 7) { |
||
216 | uint32_t ecochk, ecobits; |
||
217 | |||
218 | ecobits = I915_READ(GAC_ECO_BITS); |
||
219 | I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); |
||
220 | |||
221 | ecochk = I915_READ(GAM_ECOCHK); |
||
222 | if (IS_HASWELL(dev)) { |
||
223 | ecochk |= ECOCHK_PPGTT_WB_HSW; |
||
224 | } else { |
||
225 | ecochk |= ECOCHK_PPGTT_LLC_IVB; |
||
226 | ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; |
||
227 | } |
||
228 | I915_WRITE(GAM_ECOCHK, ecochk); |
||
229 | /* GFX_MODE is per-ring on gen7+ */ |
||
230 | } |
||
231 | |||
232 | for_each_ring(ring, dev_priv, i) { |
||
233 | if (INTEL_INFO(dev)->gen >= 7) |
||
234 | I915_WRITE(RING_MODE_GEN7(ring), |
||
235 | _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
||
236 | |||
237 | I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); |
||
238 | I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); |
||
239 | } |
||
240 | return 0; |
||
241 | } |
||
242 | |||
3031 | serge | 243 | /* PPGTT support for Sandybdrige/Gen6 and later */ |
4104 | Serge | 244 | static void gen6_ppgtt_clear_range(struct i915_address_space *vm, |
3031 | serge | 245 | unsigned first_entry, |
246 | unsigned num_entries) |
||
247 | { |
||
4104 | Serge | 248 | struct i915_hw_ppgtt *ppgtt = |
249 | container_of(vm, struct i915_hw_ppgtt, base); |
||
3746 | Serge | 250 | gen6_gtt_pte_t *pt_vaddr, scratch_pte; |
251 | unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; |
||
3031 | serge | 252 | unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
253 | unsigned last_pte, i; |
||
254 | |||
4104 | Serge | 255 | scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC); |
3031 | serge | 256 | |
257 | pt_vaddr = AllocKernelSpace(4096); |
||
258 | |||
3480 | Serge | 259 | if(pt_vaddr == NULL) |
260 | return; |
||
261 | |||
262 | while (num_entries) { |
||
3031 | serge | 263 | last_pte = first_pte + num_entries; |
264 | if (last_pte > I915_PPGTT_PT_ENTRIES) |
||
265 | last_pte = I915_PPGTT_PT_ENTRIES; |
||
266 | |||
3746 | Serge | 267 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
3031 | serge | 268 | |
269 | for (i = first_pte; i < last_pte; i++) |
||
270 | pt_vaddr[i] = scratch_pte; |
||
271 | |||
272 | num_entries -= last_pte - first_pte; |
||
273 | first_pte = 0; |
||
3746 | Serge | 274 | act_pt++; |
3480 | Serge | 275 | }; |
276 | |||
277 | FreeKernelSpace(pt_vaddr); |
||
278 | } |
||
279 | |||
4104 | Serge | 280 | static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, |
3480 | Serge | 281 | struct sg_table *pages, |
282 | unsigned first_entry, |
||
283 | enum i915_cache_level cache_level) |
||
284 | { |
||
4104 | Serge | 285 | struct i915_hw_ppgtt *ppgtt = |
286 | container_of(vm, struct i915_hw_ppgtt, base); |
||
3746 | Serge | 287 | gen6_gtt_pte_t *pt_vaddr; |
288 | unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; |
||
289 | unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
||
290 | struct sg_page_iter sg_iter; |
||
3480 | Serge | 291 | dma_addr_t page_addr; |
292 | |||
293 | |||
294 | pt_vaddr = AllocKernelSpace(4096); |
||
295 | |||
296 | if(pt_vaddr == NULL) |
||
297 | return; |
||
298 | |||
3746 | Serge | 299 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
300 | for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { |
||
301 | dma_addr_t page_addr; |
||
3480 | Serge | 302 | |
3746 | Serge | 303 | page_addr = sg_page_iter_dma_address(&sg_iter); |
4104 | Serge | 304 | pt_vaddr[act_pte] = vm->pte_encode(page_addr, cache_level); |
3746 | Serge | 305 | if (++act_pte == I915_PPGTT_PT_ENTRIES) { |
306 | act_pt++; |
||
307 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
||
308 | act_pte = 0; |
||
3480 | Serge | 309 | |
4104 | Serge | 310 | } |
3480 | Serge | 311 | } |
4104 | Serge | 312 | FreeKernelSpace(pt_vaddr); |
3031 | serge | 313 | } |
314 | |||
4104 | Serge | 315 | static void gen6_ppgtt_cleanup(struct i915_address_space *vm) |
3031 | serge | 316 | { |
4104 | Serge | 317 | struct i915_hw_ppgtt *ppgtt = |
318 | container_of(vm, struct i915_hw_ppgtt, base); |
||
3480 | Serge | 319 | int i; |
320 | |||
4104 | Serge | 321 | drm_mm_takedown(&ppgtt->base.mm); |
322 | |||
3480 | Serge | 323 | if (ppgtt->pt_dma_addr) { |
324 | for (i = 0; i < ppgtt->num_pd_entries; i++) |
||
4104 | Serge | 325 | pci_unmap_page(ppgtt->base.dev->pdev, |
3480 | Serge | 326 | ppgtt->pt_dma_addr[i], |
327 | 4096, PCI_DMA_BIDIRECTIONAL); |
||
328 | } |
||
329 | |||
330 | kfree(ppgtt->pt_dma_addr); |
||
331 | for (i = 0; i < ppgtt->num_pd_entries; i++) |
||
332 | __free_page(ppgtt->pt_pages[i]); |
||
333 | kfree(ppgtt->pt_pages); |
||
334 | kfree(ppgtt); |
||
335 | } |
||
336 | |||
337 | static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) |
||
338 | { |
||
4104 | Serge | 339 | struct drm_device *dev = ppgtt->base.dev; |
3031 | serge | 340 | struct drm_i915_private *dev_priv = dev->dev_private; |
341 | unsigned first_pd_entry_in_global_pt; |
||
342 | int i; |
||
343 | int ret = -ENOMEM; |
||
344 | |||
345 | /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 |
||
346 | * entries. For aliasing ppgtt support we just steal them at the end for |
||
347 | * now. */ |
||
3746 | Serge | 348 | first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt); |
3031 | serge | 349 | |
4104 | Serge | 350 | ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode; |
351 | ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES; |
||
3746 | Serge | 352 | ppgtt->enable = gen6_ppgtt_enable; |
4104 | Serge | 353 | ppgtt->base.clear_range = gen6_ppgtt_clear_range; |
354 | ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; |
||
355 | ppgtt->base.cleanup = gen6_ppgtt_cleanup; |
||
356 | ppgtt->base.scratch = dev_priv->gtt.base.scratch; |
||
3243 | Serge | 357 | ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries, |
3031 | serge | 358 | GFP_KERNEL); |
359 | if (!ppgtt->pt_pages) |
||
3480 | Serge | 360 | return -ENOMEM; |
3031 | serge | 361 | |
362 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
||
3243 | Serge | 363 | ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL); |
3031 | serge | 364 | if (!ppgtt->pt_pages[i]) |
365 | goto err_pt_alloc; |
||
366 | } |
||
367 | |||
3480 | Serge | 368 | ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries, |
3031 | serge | 369 | GFP_KERNEL); |
4104 | Serge | 370 | if (!ppgtt->pt_dma_addr) |
371 | goto err_pt_alloc; |
||
3031 | serge | 372 | |
4104 | Serge | 373 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
374 | dma_addr_t pt_addr; |
||
3031 | serge | 375 | |
3480 | Serge | 376 | pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096, |
3031 | serge | 377 | PCI_DMA_BIDIRECTIONAL); |
378 | |||
4104 | Serge | 379 | ppgtt->pt_dma_addr[i] = pt_addr; |
380 | } |
||
3031 | serge | 381 | |
4104 | Serge | 382 | ppgtt->base.clear_range(&ppgtt->base, 0, |
3031 | serge | 383 | ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES); |
384 | |||
3746 | Serge | 385 | ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t); |
3031 | serge | 386 | |
387 | return 0; |
||
388 | |||
389 | err_pd_pin: |
||
3480 | Serge | 390 | if (ppgtt->pt_dma_addr) { |
391 | for (i--; i >= 0; i--) |
||
392 | pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], |
||
393 | 4096, PCI_DMA_BIDIRECTIONAL); |
||
394 | } |
||
3031 | serge | 395 | err_pt_alloc: |
3480 | Serge | 396 | kfree(ppgtt->pt_dma_addr); |
3031 | serge | 397 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
398 | if (ppgtt->pt_pages[i]) |
||
3480 | Serge | 399 | __free_page(ppgtt->pt_pages[i]); |
3031 | serge | 400 | } |
401 | kfree(ppgtt->pt_pages); |
||
402 | |||
403 | return ret; |
||
404 | } |
||
405 | |||
3480 | Serge | 406 | static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) |
3031 | serge | 407 | { |
408 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
3480 | Serge | 409 | struct i915_hw_ppgtt *ppgtt; |
410 | int ret; |
||
3031 | serge | 411 | |
3480 | Serge | 412 | ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); |
3031 | serge | 413 | if (!ppgtt) |
3480 | Serge | 414 | return -ENOMEM; |
3031 | serge | 415 | |
4104 | Serge | 416 | ppgtt->base.dev = dev; |
3031 | serge | 417 | |
3746 | Serge | 418 | if (INTEL_INFO(dev)->gen < 8) |
3480 | Serge | 419 | ret = gen6_ppgtt_init(ppgtt); |
3746 | Serge | 420 | else |
421 | BUG(); |
||
422 | |||
3480 | Serge | 423 | if (ret) |
4104 | Serge | 424 | kfree(ppgtt); |
425 | else { |
||
3480 | Serge | 426 | dev_priv->mm.aliasing_ppgtt = ppgtt; |
4104 | Serge | 427 | drm_mm_init(&ppgtt->base.mm, ppgtt->base.start, |
428 | ppgtt->base.total); |
||
429 | } |
||
3480 | Serge | 430 | |
431 | return ret; |
||
3031 | serge | 432 | } |
433 | |||
3480 | Serge | 434 | void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) |
3031 | serge | 435 | { |
3480 | Serge | 436 | struct drm_i915_private *dev_priv = dev->dev_private; |
437 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
||
3031 | serge | 438 | |
3480 | Serge | 439 | if (!ppgtt) |
3243 | Serge | 440 | return; |
3031 | serge | 441 | |
4104 | Serge | 442 | ppgtt->base.cleanup(&ppgtt->base); |
3746 | Serge | 443 | dev_priv->mm.aliasing_ppgtt = NULL; |
3031 | serge | 444 | } |
445 | |||
446 | void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, |
||
447 | struct drm_i915_gem_object *obj, |
||
448 | enum i915_cache_level cache_level) |
||
449 | { |
||
4104 | Serge | 450 | ppgtt->base.insert_entries(&ppgtt->base, obj->pages, |
451 | i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT, |
||
3243 | Serge | 452 | cache_level); |
3031 | serge | 453 | } |
454 | |||
455 | void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, |
||
456 | struct drm_i915_gem_object *obj) |
||
457 | { |
||
4104 | Serge | 458 | ppgtt->base.clear_range(&ppgtt->base, |
459 | i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT, |
||
3031 | serge | 460 | obj->base.size >> PAGE_SHIFT); |
461 | } |
||
462 | |||
3480 | Serge | 463 | extern int intel_iommu_gfx_mapped; |
464 | /* Certain Gen5 chipsets require require idling the GPU before |
||
465 | * unmapping anything from the GTT when VT-d is enabled. |
||
466 | */ |
||
467 | static inline bool needs_idle_maps(struct drm_device *dev) |
||
468 | { |
||
469 | #ifdef CONFIG_INTEL_IOMMU |
||
470 | /* Query intel_iommu to see if we need the workaround. Presumably that |
||
471 | * was loaded first. |
||
472 | */ |
||
473 | if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped) |
||
474 | return true; |
||
475 | #endif |
||
476 | return false; |
||
477 | } |
||
478 | |||
2344 | Serge | 479 | static bool do_idling(struct drm_i915_private *dev_priv) |
480 | { |
||
481 | bool ret = dev_priv->mm.interruptible; |
||
482 | |||
3480 | Serge | 483 | if (unlikely(dev_priv->gtt.do_idle_maps)) { |
2344 | Serge | 484 | dev_priv->mm.interruptible = false; |
485 | if (i915_gpu_idle(dev_priv->dev)) { |
||
486 | DRM_ERROR("Couldn't idle GPU\n"); |
||
487 | /* Wait a bit, in hopes it avoids the hang */ |
||
488 | udelay(10); |
||
489 | } |
||
490 | } |
||
491 | |||
492 | return ret; |
||
493 | } |
||
494 | |||
495 | static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) |
||
496 | { |
||
3480 | Serge | 497 | if (unlikely(dev_priv->gtt.do_idle_maps)) |
2344 | Serge | 498 | dev_priv->mm.interruptible = interruptible; |
499 | } |
||
500 | |||
2332 | Serge | 501 | void i915_gem_restore_gtt_mappings(struct drm_device *dev) |
502 | { |
||
503 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
504 | struct drm_i915_gem_object *obj; |
||
505 | |||
506 | /* First fill our portion of the GTT with scratch pages */ |
||
4104 | Serge | 507 | dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, |
508 | dev_priv->gtt.base.start / PAGE_SIZE, |
||
509 | dev_priv->gtt.base.total / PAGE_SIZE); |
||
2332 | Serge | 510 | |
4104 | Serge | 511 | list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { |
512 | i915_gem_clflush_object(obj, obj->pin_display); |
||
3031 | serge | 513 | i915_gem_gtt_bind_object(obj, obj->cache_level); |
2332 | Serge | 514 | } |
515 | |||
3243 | Serge | 516 | i915_gem_chipset_flush(dev); |
2332 | Serge | 517 | } |
518 | |||
3031 | serge | 519 | int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) |
2332 | Serge | 520 | { |
3243 | Serge | 521 | if (obj->has_dma_mapping) |
522 | return 0; |
||
523 | |||
3480 | Serge | 524 | if (!dma_map_sg(&obj->base.dev->pdev->dev, |
525 | obj->pages->sgl, obj->pages->nents, |
||
526 | PCI_DMA_BIDIRECTIONAL)) |
||
527 | return -ENOSPC; |
||
3243 | Serge | 528 | |
2332 | Serge | 529 | return 0; |
530 | } |
||
531 | |||
3243 | Serge | 532 | /* |
533 | * Binds an object into the global gtt with the specified cache level. The object |
||
534 | * will be accessible to the GPU via commands whose operands reference offsets |
||
535 | * within the global GTT as well as accessible by the GPU through the GMADR |
||
536 | * mapped BAR (dev_priv->mm.gtt->gtt). |
||
537 | */ |
||
4104 | Serge | 538 | static void gen6_ggtt_insert_entries(struct i915_address_space *vm, |
3480 | Serge | 539 | struct sg_table *st, |
540 | unsigned int first_entry, |
||
3243 | Serge | 541 | enum i915_cache_level level) |
542 | { |
||
4104 | Serge | 543 | struct drm_i915_private *dev_priv = vm->dev->dev_private; |
3746 | Serge | 544 | gen6_gtt_pte_t __iomem *gtt_entries = |
545 | (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; |
||
546 | int i = 0; |
||
547 | struct sg_page_iter sg_iter; |
||
3243 | Serge | 548 | dma_addr_t addr; |
549 | |||
3746 | Serge | 550 | for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { |
551 | addr = sg_page_iter_dma_address(&sg_iter); |
||
4104 | Serge | 552 | iowrite32(vm->pte_encode(addr, level), >t_entries[i]); |
3243 | Serge | 553 | i++; |
554 | } |
||
555 | |||
556 | /* XXX: This serves as a posting read to make sure that the PTE has |
||
557 | * actually been updated. There is some concern that even though |
||
558 | * registers and PTEs are within the same BAR that they are potentially |
||
559 | * of NUMA access patterns. Therefore, even with the way we assume |
||
560 | * hardware should work, we must keep this posting read for paranoia. |
||
561 | */ |
||
562 | if (i != 0) |
||
4104 | Serge | 563 | WARN_ON(readl(>t_entries[i-1]) != |
564 | vm->pte_encode(addr, level)); |
||
3243 | Serge | 565 | |
566 | /* This next bit makes the above posting read even more important. We |
||
567 | * want to flush the TLBs only after we're certain all the PTE updates |
||
568 | * have finished. |
||
569 | */ |
||
570 | I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); |
||
571 | POSTING_READ(GFX_FLSH_CNTL_GEN6); |
||
572 | } |
||
573 | |||
4104 | Serge | 574 | static void gen6_ggtt_clear_range(struct i915_address_space *vm, |
3480 | Serge | 575 | unsigned int first_entry, |
576 | unsigned int num_entries) |
||
577 | { |
||
4104 | Serge | 578 | struct drm_i915_private *dev_priv = vm->dev->dev_private; |
3746 | Serge | 579 | gen6_gtt_pte_t scratch_pte, __iomem *gtt_base = |
580 | (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; |
||
3480 | Serge | 581 | const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; |
582 | int i; |
||
583 | |||
3746 | Serge | 584 | // if (WARN(num_entries > max_entries, |
585 | // "First entry = %d; Num entries = %d (max=%d)\n", |
||
586 | // first_entry, num_entries, max_entries)) |
||
587 | if (num_entries > max_entries) |
||
4104 | Serge | 588 | num_entries = max_entries; |
3480 | Serge | 589 | |
4104 | Serge | 590 | scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC); |
3480 | Serge | 591 | for (i = 0; i < num_entries; i++) |
592 | iowrite32(scratch_pte, >t_base[i]); |
||
593 | readl(gtt_base); |
||
594 | } |
||
595 | |||
596 | |||
4104 | Serge | 597 | static void i915_ggtt_insert_entries(struct i915_address_space *vm, |
3480 | Serge | 598 | struct sg_table *st, |
599 | unsigned int pg_start, |
||
600 | enum i915_cache_level cache_level) |
||
601 | { |
||
602 | unsigned int flags = (cache_level == I915_CACHE_NONE) ? |
||
603 | AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; |
||
604 | |||
605 | intel_gtt_insert_sg_entries(st, pg_start, flags); |
||
606 | |||
607 | } |
||
608 | |||
4104 | Serge | 609 | static void i915_ggtt_clear_range(struct i915_address_space *vm, |
3480 | Serge | 610 | unsigned int first_entry, |
611 | unsigned int num_entries) |
||
612 | { |
||
613 | intel_gtt_clear_range(first_entry, num_entries); |
||
614 | } |
||
615 | |||
616 | |||
3031 | serge | 617 | void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj, |
2332 | Serge | 618 | enum i915_cache_level cache_level) |
619 | { |
||
620 | struct drm_device *dev = obj->base.dev; |
||
3480 | Serge | 621 | struct drm_i915_private *dev_priv = dev->dev_private; |
4104 | Serge | 622 | const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT; |
3480 | Serge | 623 | |
4104 | Serge | 624 | dev_priv->gtt.base.insert_entries(&dev_priv->gtt.base, obj->pages, |
625 | entry, |
||
3480 | Serge | 626 | cache_level); |
2332 | Serge | 627 | |
3031 | serge | 628 | obj->has_global_gtt_mapping = 1; |
2332 | Serge | 629 | } |
630 | |||
631 | void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) |
||
632 | { |
||
3480 | Serge | 633 | struct drm_device *dev = obj->base.dev; |
634 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
4104 | Serge | 635 | const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT; |
3480 | Serge | 636 | |
4104 | Serge | 637 | dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, |
638 | entry, |
||
3031 | serge | 639 | obj->base.size >> PAGE_SHIFT); |
640 | |||
641 | obj->has_global_gtt_mapping = 0; |
||
642 | } |
||
643 | |||
644 | void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) |
||
645 | { |
||
2344 | Serge | 646 | struct drm_device *dev = obj->base.dev; |
647 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
648 | bool interruptible; |
||
649 | |||
650 | interruptible = do_idling(dev_priv); |
||
651 | |||
3480 | Serge | 652 | if (!obj->has_dma_mapping) |
653 | dma_unmap_sg(&dev->pdev->dev, |
||
654 | obj->pages->sgl, obj->pages->nents, |
||
655 | PCI_DMA_BIDIRECTIONAL); |
||
2332 | Serge | 656 | |
3031 | serge | 657 | undo_idling(dev_priv, interruptible); |
658 | } |
||
659 | |||
660 | static void i915_gtt_color_adjust(struct drm_mm_node *node, |
||
661 | unsigned long color, |
||
662 | unsigned long *start, |
||
663 | unsigned long *end) |
||
664 | { |
||
665 | if (node->color != color) |
||
666 | *start += 4096; |
||
667 | |||
668 | if (!list_empty(&node->node_list)) { |
||
669 | node = list_entry(node->node_list.next, |
||
670 | struct drm_mm_node, |
||
671 | node_list); |
||
672 | if (node->allocated && node->color != color) |
||
673 | *end -= 4096; |
||
2332 | Serge | 674 | } |
3031 | serge | 675 | } |
3480 | Serge | 676 | void i915_gem_setup_global_gtt(struct drm_device *dev, |
3031 | serge | 677 | unsigned long start, |
678 | unsigned long mappable_end, |
||
679 | unsigned long end) |
||
680 | { |
||
3480 | Serge | 681 | /* Let GEM Manage all of the aperture. |
682 | * |
||
683 | * However, leave one page at the end still bound to the scratch page. |
||
684 | * There are a number of places where the hardware apparently prefetches |
||
685 | * past the end of the object, and we've seen multiple hangs with the |
||
686 | * GPU head pointer stuck in a batchbuffer bound at the last page of the |
||
687 | * aperture. One page should be enough to keep any prefetching inside |
||
688 | * of the aperture. |
||
689 | */ |
||
4104 | Serge | 690 | struct drm_i915_private *dev_priv = dev->dev_private; |
691 | struct i915_address_space *ggtt_vm = &dev_priv->gtt.base; |
||
3480 | Serge | 692 | struct drm_mm_node *entry; |
693 | struct drm_i915_gem_object *obj; |
||
694 | unsigned long hole_start, hole_end; |
||
3031 | serge | 695 | |
3480 | Serge | 696 | BUG_ON(mappable_end > end); |
697 | |||
698 | /* Subtract the guard page ... */ |
||
4104 | Serge | 699 | drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE); |
3031 | serge | 700 | if (!HAS_LLC(dev)) |
4104 | Serge | 701 | dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust; |
3031 | serge | 702 | |
3480 | Serge | 703 | /* Mark any preallocated objects as occupied */ |
4104 | Serge | 704 | list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { |
705 | struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm); |
||
706 | int ret; |
||
707 | DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n", |
||
708 | i915_gem_obj_ggtt_offset(obj), obj->base.size); |
||
3031 | serge | 709 | |
4104 | Serge | 710 | WARN_ON(i915_gem_obj_ggtt_bound(obj)); |
711 | ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node); |
||
712 | if (ret) |
||
713 | DRM_DEBUG_KMS("Reservation failed\n"); |
||
3480 | Serge | 714 | obj->has_global_gtt_mapping = 1; |
4104 | Serge | 715 | list_add(&vma->vma_link, &obj->vma_list); |
3480 | Serge | 716 | } |
717 | |||
4104 | Serge | 718 | dev_priv->gtt.base.start = start; |
719 | dev_priv->gtt.base.total = end - start; |
||
3480 | Serge | 720 | |
721 | /* Clear any non-preallocated blocks */ |
||
4104 | Serge | 722 | drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) { |
723 | const unsigned long count = (hole_end - hole_start) / PAGE_SIZE; |
||
3480 | Serge | 724 | DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n", |
725 | hole_start, hole_end); |
||
4104 | Serge | 726 | ggtt_vm->clear_range(ggtt_vm, hole_start / PAGE_SIZE, count); |
3480 | Serge | 727 | } |
728 | |||
729 | /* And finally clear the reserved guard page */ |
||
4104 | Serge | 730 | ggtt_vm->clear_range(ggtt_vm, end / PAGE_SIZE - 1, 1); |
2332 | Serge | 731 | } |
3243 | Serge | 732 | |
3480 | Serge | 733 | static bool |
734 | intel_enable_ppgtt(struct drm_device *dev) |
||
735 | { |
||
736 | if (i915_enable_ppgtt >= 0) |
||
737 | return i915_enable_ppgtt; |
||
738 | |||
739 | #ifdef CONFIG_INTEL_IOMMU |
||
740 | /* Disable ppgtt on SNB if VT-d is on. */ |
||
741 | if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) |
||
742 | return false; |
||
743 | #endif |
||
744 | |||
745 | return true; |
||
746 | } |
||
747 | |||
748 | void i915_gem_init_global_gtt(struct drm_device *dev) |
||
749 | { |
||
750 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
751 | unsigned long gtt_size, mappable_size; |
||
752 | |||
4104 | Serge | 753 | gtt_size = dev_priv->gtt.base.total; |
3480 | Serge | 754 | mappable_size = dev_priv->gtt.mappable_end; |
755 | |||
3746 | Serge | 756 | #if 0 |
3480 | Serge | 757 | if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { |
758 | int ret; |
||
3746 | Serge | 759 | |
760 | if (INTEL_INFO(dev)->gen <= 7) { |
||
3480 | Serge | 761 | /* PPGTT pdes are stolen from global gtt ptes, so shrink the |
762 | * aperture accordingly when using aliasing ppgtt. */ |
||
4104 | Serge | 763 | gtt_size -= GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE; |
3746 | Serge | 764 | } |
3480 | Serge | 765 | |
4104 | Serge | 766 | i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size-LFB_SIZE); |
3480 | Serge | 767 | |
768 | ret = i915_gem_init_aliasing_ppgtt(dev); |
||
769 | if (!ret) |
||
770 | return; |
||
771 | |||
772 | DRM_ERROR("Aliased PPGTT setup failed %d\n", ret); |
||
4104 | Serge | 773 | drm_mm_takedown(&dev_priv->gtt.base.mm); |
774 | gtt_size += GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE; |
||
3746 | Serge | 775 | } |
776 | #endif |
||
777 | |||
4104 | Serge | 778 | i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size-LFB_SIZE); |
3480 | Serge | 779 | } |
780 | |||
3243 | Serge | 781 | static int setup_scratch_page(struct drm_device *dev) |
782 | { |
||
783 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
784 | struct page *page; |
||
785 | dma_addr_t dma_addr; |
||
786 | |||
787 | page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO); |
||
788 | if (page == NULL) |
||
789 | return -ENOMEM; |
||
4104 | Serge | 790 | get_page(page); |
3480 | Serge | 791 | set_pages_uc(page, 1); |
3243 | Serge | 792 | |
793 | #ifdef CONFIG_INTEL_IOMMU |
||
794 | dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE, |
||
795 | PCI_DMA_BIDIRECTIONAL); |
||
796 | if (pci_dma_mapping_error(dev->pdev, dma_addr)) |
||
797 | return -EINVAL; |
||
798 | #else |
||
799 | dma_addr = page_to_phys(page); |
||
800 | #endif |
||
4104 | Serge | 801 | dev_priv->gtt.base.scratch.page = page; |
802 | dev_priv->gtt.base.scratch.addr = dma_addr; |
||
3243 | Serge | 803 | |
804 | return 0; |
||
805 | } |
||
806 | |||
3480 | Serge | 807 | static void teardown_scratch_page(struct drm_device *dev) |
808 | { |
||
809 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
4104 | Serge | 810 | struct page *page = dev_priv->gtt.base.scratch.page; |
811 | |||
812 | set_pages_wb(page, 1); |
||
813 | pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr, |
||
3480 | Serge | 814 | PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); |
4104 | Serge | 815 | put_page(page); |
816 | __free_page(page); |
||
3480 | Serge | 817 | } |
3243 | Serge | 818 | |
819 | static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) |
||
820 | { |
||
821 | snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; |
||
822 | snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; |
||
823 | return snb_gmch_ctl << 20; |
||
824 | } |
||
825 | |||
3480 | Serge | 826 | static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl) |
3243 | Serge | 827 | { |
828 | snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; |
||
829 | snb_gmch_ctl &= SNB_GMCH_GMS_MASK; |
||
830 | return snb_gmch_ctl << 25; /* 32 MB units */ |
||
831 | } |
||
832 | |||
3480 | Serge | 833 | static int gen6_gmch_probe(struct drm_device *dev, |
834 | size_t *gtt_total, |
||
835 | size_t *stolen, |
||
836 | phys_addr_t *mappable_base, |
||
837 | unsigned long *mappable_end) |
||
3243 | Serge | 838 | { |
839 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
840 | phys_addr_t gtt_bus_addr; |
||
3480 | Serge | 841 | unsigned int gtt_size; |
3243 | Serge | 842 | u16 snb_gmch_ctl; |
843 | int ret; |
||
844 | |||
3480 | Serge | 845 | *mappable_base = pci_resource_start(dev->pdev, 2); |
846 | *mappable_end = pci_resource_len(dev->pdev, 2); |
||
847 | |||
848 | /* 64/512MB is the current min/max we actually know of, but this is just |
||
849 | * a coarse sanity check. |
||
3243 | Serge | 850 | */ |
3480 | Serge | 851 | if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { |
852 | DRM_ERROR("Unknown GMADR size (%lx)\n", |
||
853 | dev_priv->gtt.mappable_end); |
||
854 | return -ENXIO; |
||
3243 | Serge | 855 | } |
856 | |||
3480 | Serge | 857 | if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40))) |
858 | pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40)); |
||
859 | pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); |
||
860 | gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); |
||
3243 | Serge | 861 | |
4104 | Serge | 862 | *stolen = gen6_get_stolen_size(snb_gmch_ctl); |
3746 | Serge | 863 | *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT; |
3243 | Serge | 864 | |
3746 | Serge | 865 | /* For Modern GENs the PTEs and register space are split in the BAR */ |
866 | gtt_bus_addr = pci_resource_start(dev->pdev, 0) + |
||
867 | (pci_resource_len(dev->pdev, 0) / 2); |
||
3243 | Serge | 868 | |
3480 | Serge | 869 | dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size); |
870 | if (!dev_priv->gtt.gsm) { |
||
871 | DRM_ERROR("Failed to map the gtt page table\n"); |
||
872 | return -ENOMEM; |
||
3243 | Serge | 873 | } |
874 | |||
875 | ret = setup_scratch_page(dev); |
||
3480 | Serge | 876 | if (ret) |
3243 | Serge | 877 | DRM_ERROR("Scratch setup failed\n"); |
878 | |||
4104 | Serge | 879 | dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range; |
880 | dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries; |
||
3480 | Serge | 881 | |
882 | return ret; |
||
883 | } |
||
884 | |||
4104 | Serge | 885 | static void gen6_gmch_remove(struct i915_address_space *vm) |
3480 | Serge | 886 | { |
4104 | Serge | 887 | |
888 | struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base); |
||
889 | iounmap(gtt->gsm); |
||
890 | teardown_scratch_page(vm->dev); |
||
3480 | Serge | 891 | } |
892 | |||
893 | static int i915_gmch_probe(struct drm_device *dev, |
||
894 | size_t *gtt_total, |
||
895 | size_t *stolen, |
||
896 | phys_addr_t *mappable_base, |
||
897 | unsigned long *mappable_end) |
||
898 | { |
||
899 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
900 | int ret; |
||
901 | |||
902 | ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL); |
||
903 | if (!ret) { |
||
904 | DRM_ERROR("failed to set up gmch\n"); |
||
905 | return -EIO; |
||
3243 | Serge | 906 | } |
907 | |||
3480 | Serge | 908 | intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end); |
3243 | Serge | 909 | |
3480 | Serge | 910 | dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev); |
4104 | Serge | 911 | dev_priv->gtt.base.clear_range = i915_ggtt_clear_range; |
912 | dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries; |
||
3480 | Serge | 913 | |
3243 | Serge | 914 | return 0; |
3480 | Serge | 915 | } |
3243 | Serge | 916 | |
4104 | Serge | 917 | static void i915_gmch_remove(struct i915_address_space *vm) |
3480 | Serge | 918 | { |
919 | // intel_gmch_remove(); |
||
920 | } |
||
921 | |||
922 | int i915_gem_gtt_init(struct drm_device *dev) |
||
923 | { |
||
924 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
925 | struct i915_gtt *gtt = &dev_priv->gtt; |
||
926 | int ret; |
||
927 | |||
928 | if (INTEL_INFO(dev)->gen <= 5) { |
||
4104 | Serge | 929 | gtt->gtt_probe = i915_gmch_probe; |
930 | gtt->base.cleanup = i915_gmch_remove; |
||
3480 | Serge | 931 | } else { |
4104 | Serge | 932 | gtt->gtt_probe = gen6_gmch_probe; |
933 | gtt->base.cleanup = gen6_gmch_remove; |
||
934 | if (IS_HASWELL(dev) && dev_priv->ellc_size) |
||
935 | gtt->base.pte_encode = iris_pte_encode; |
||
936 | else if (IS_HASWELL(dev)) |
||
937 | gtt->base.pte_encode = hsw_pte_encode; |
||
938 | else if (IS_VALLEYVIEW(dev)) |
||
939 | gtt->base.pte_encode = byt_pte_encode; |
||
940 | else if (INTEL_INFO(dev)->gen >= 7) |
||
941 | gtt->base.pte_encode = ivb_pte_encode; |
||
942 | else |
||
943 | gtt->base.pte_encode = snb_pte_encode; |
||
3480 | Serge | 944 | } |
945 | |||
4104 | Serge | 946 | ret = gtt->gtt_probe(dev, >t->base.total, >t->stolen_size, |
947 | >t->mappable_base, >t->mappable_end); |
||
3480 | Serge | 948 | if (ret) |
4104 | Serge | 949 | return ret; |
3480 | Serge | 950 | |
4104 | Serge | 951 | gtt->base.dev = dev; |
952 | |||
3480 | Serge | 953 | /* GMADR is the PCI mmio aperture into the global GTT. */ |
954 | DRM_INFO("Memory usable by graphics device = %zdM\n", |
||
4104 | Serge | 955 | gtt->base.total >> 20); |
956 | DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20); |
||
957 | DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20); |
||
3480 | Serge | 958 | |
959 | return 0; |
||
3243 | Serge | 960 | } |
961 | |||
962 | struct scatterlist *sg_next(struct scatterlist *sg) |
||
963 | { |
||
964 | if (sg_is_last(sg)) |
||
965 | return NULL; |
||
966 | |||
967 | sg++; |
||
968 | if (unlikely(sg_is_chain(sg))) |
||
969 | sg = sg_chain_ptr(sg); |
||
970 | |||
971 | return sg; |
||
972 | } |
||
973 | |||
974 | |||
975 | void __sg_free_table(struct sg_table *table, unsigned int max_ents, |
||
976 | sg_free_fn *free_fn) |
||
977 | { |
||
978 | struct scatterlist *sgl, *next; |
||
979 | |||
980 | if (unlikely(!table->sgl)) |
||
981 | return; |
||
982 | |||
983 | sgl = table->sgl; |
||
984 | while (table->orig_nents) { |
||
985 | unsigned int alloc_size = table->orig_nents; |
||
986 | unsigned int sg_size; |
||
987 | |||
988 | /* |
||
989 | * If we have more than max_ents segments left, |
||
990 | * then assign 'next' to the sg table after the current one. |
||
991 | * sg_size is then one less than alloc size, since the last |
||
992 | * element is the chain pointer. |
||
993 | */ |
||
994 | if (alloc_size > max_ents) { |
||
995 | next = sg_chain_ptr(&sgl[max_ents - 1]); |
||
996 | alloc_size = max_ents; |
||
997 | sg_size = alloc_size - 1; |
||
998 | } else { |
||
999 | sg_size = alloc_size; |
||
1000 | next = NULL; |
||
1001 | } |
||
1002 | |||
1003 | table->orig_nents -= sg_size; |
||
1004 | kfree(sgl); |
||
1005 | sgl = next; |
||
1006 | } |
||
1007 | |||
1008 | table->sgl = NULL; |
||
1009 | } |
||
1010 | |||
1011 | void sg_free_table(struct sg_table *table) |
||
1012 | { |
||
1013 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
||
1014 | } |
||
1015 | |||
1016 | int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) |
||
1017 | { |
||
1018 | struct scatterlist *sg, *prv; |
||
1019 | unsigned int left; |
||
1020 | unsigned int max_ents = SG_MAX_SINGLE_ALLOC; |
||
1021 | |||
1022 | #ifndef ARCH_HAS_SG_CHAIN |
||
1023 | BUG_ON(nents > max_ents); |
||
1024 | #endif |
||
1025 | |||
1026 | memset(table, 0, sizeof(*table)); |
||
1027 | |||
1028 | left = nents; |
||
1029 | prv = NULL; |
||
1030 | do { |
||
1031 | unsigned int sg_size, alloc_size = left; |
||
1032 | |||
1033 | if (alloc_size > max_ents) { |
||
1034 | alloc_size = max_ents; |
||
1035 | sg_size = alloc_size - 1; |
||
1036 | } else |
||
1037 | sg_size = alloc_size; |
||
1038 | |||
1039 | left -= sg_size; |
||
1040 | |||
1041 | sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask); |
||
1042 | if (unlikely(!sg)) { |
||
1043 | /* |
||
1044 | * Adjust entry count to reflect that the last |
||
1045 | * entry of the previous table won't be used for |
||
1046 | * linkage. Without this, sg_kfree() may get |
||
1047 | * confused. |
||
1048 | */ |
||
1049 | if (prv) |
||
1050 | table->nents = ++table->orig_nents; |
||
1051 | |||
1052 | goto err; |
||
1053 | } |
||
1054 | |||
1055 | sg_init_table(sg, alloc_size); |
||
1056 | table->nents = table->orig_nents += sg_size; |
||
1057 | |||
1058 | /* |
||
1059 | * If this is the first mapping, assign the sg table header. |
||
1060 | * If this is not the first mapping, chain previous part. |
||
1061 | */ |
||
1062 | if (prv) |
||
1063 | sg_chain(prv, max_ents, sg); |
||
1064 | else |
||
1065 | table->sgl = sg; |
||
1066 | |||
1067 | /* |
||
1068 | * If no more entries after this one, mark the end |
||
1069 | */ |
||
1070 | if (!left) |
||
1071 | sg_mark_end(&sg[sg_size - 1]); |
||
1072 | |||
1073 | prv = sg; |
||
1074 | } while (left); |
||
1075 | |||
1076 | return 0; |
||
1077 | |||
1078 | err: |
||
1079 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
||
1080 | |||
1081 | return -ENOMEM; |
||
1082 | } |
||
1083 | |||
1084 | |||
1085 | void sg_init_table(struct scatterlist *sgl, unsigned int nents) |
||
1086 | { |
||
1087 | memset(sgl, 0, sizeof(*sgl) * nents); |
||
1088 | #ifdef CONFIG_DEBUG_SG |
||
1089 | { |
||
1090 | unsigned int i; |
||
1091 | for (i = 0; i < nents; i++) |
||
1092 | sgl[i].sg_magic = SG_MAGIC; |
||
1093 | } |
||
1094 | #endif |
||
1095 | sg_mark_end(&sgl[nents - 1]); |
||
1096 | } |
||
1097 | |||
3746 | Serge | 1098 | |
1099 | void __sg_page_iter_start(struct sg_page_iter *piter, |
||
1100 | struct scatterlist *sglist, unsigned int nents, |
||
1101 | unsigned long pgoffset) |
||
1102 | { |
||
1103 | piter->__pg_advance = 0; |
||
1104 | piter->__nents = nents; |
||
1105 | |||
1106 | piter->sg = sglist; |
||
1107 | piter->sg_pgoffset = pgoffset; |
||
1108 | } |
||
1109 | |||
1110 | static int sg_page_count(struct scatterlist *sg) |
||
1111 | { |
||
1112 | return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT; |
||
1113 | } |
||
1114 | |||
1115 | bool __sg_page_iter_next(struct sg_page_iter *piter) |
||
1116 | { |
||
1117 | if (!piter->__nents || !piter->sg) |
||
1118 | return false; |
||
1119 | |||
1120 | piter->sg_pgoffset += piter->__pg_advance; |
||
1121 | piter->__pg_advance = 1; |
||
1122 | |||
1123 | while (piter->sg_pgoffset >= sg_page_count(piter->sg)) { |
||
1124 | piter->sg_pgoffset -= sg_page_count(piter->sg); |
||
1125 | piter->sg = sg_next(piter->sg); |
||
1126 | if (!--piter->__nents || !piter->sg) |
||
1127 | return false; |
||
1128 | } |
||
1129 | |||
1130 | return true; |
||
1131 | } |
||
1132 | EXPORT_SYMBOL(__sg_page_iter_next);>=>><>20))))><20))))>20)><20)>>><>><>=>>>>>>>>>=><=>>><>><>><>><>><>><>><>><>><>><> |
||
1133 |