Rev 3243 | Rev 3746 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
2332 | Serge | 1 | /* |
2 | * Copyright © 2010 Daniel Vetter |
||
3 | * |
||
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
||
5 | * copy of this software and associated documentation files (the "Software"), |
||
6 | * to deal in the Software without restriction, including without limitation |
||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
||
8 | * and/or sell copies of the Software, and to permit persons to whom the |
||
9 | * Software is furnished to do so, subject to the following conditions: |
||
10 | * |
||
11 | * The above copyright notice and this permission notice (including the next |
||
12 | * paragraph) shall be included in all copies or substantial portions of the |
||
13 | * Software. |
||
14 | * |
||
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
||
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
||
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
||
21 | * IN THE SOFTWARE. |
||
22 | * |
||
23 | */ |
||
24 | |||
3243 | Serge | 25 | #define iowrite32(v, addr) writel((v), (addr)) |
26 | |||
3480 | Serge | 27 | #define AGP_NORMAL_MEMORY 0 |
28 | |||
29 | #define AGP_USER_TYPES (1 << 16) |
||
30 | #define AGP_USER_MEMORY (AGP_USER_TYPES) |
||
31 | #define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1) |
||
32 | |||
3031 | serge | 33 | #include |
34 | #include |
||
2332 | Serge | 35 | #include "i915_drv.h" |
2351 | Serge | 36 | #include "i915_trace.h" |
2332 | Serge | 37 | #include "intel_drv.h" |
38 | |||
3243 | Serge | 39 | typedef uint32_t gtt_pte_t; |
40 | |||
41 | /* PPGTT stuff */ |
||
42 | #define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) |
||
43 | |||
44 | #define GEN6_PDE_VALID (1 << 0) |
||
45 | /* gen6+ has bit 11-4 for physical addr bit 39-32 */ |
||
46 | #define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) |
||
47 | |||
48 | #define GEN6_PTE_VALID (1 << 0) |
||
49 | #define GEN6_PTE_UNCACHED (1 << 1) |
||
50 | #define HSW_PTE_UNCACHED (0) |
||
51 | #define GEN6_PTE_CACHE_LLC (2 << 1) |
||
52 | #define GEN6_PTE_CACHE_LLC_MLC (3 << 1) |
||
53 | #define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) |
||
54 | |||
3480 | Serge | 55 | static inline gtt_pte_t gen6_pte_encode(struct drm_device *dev, |
3243 | Serge | 56 | dma_addr_t addr, |
57 | enum i915_cache_level level) |
||
58 | { |
||
59 | gtt_pte_t pte = GEN6_PTE_VALID; |
||
60 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
||
61 | |||
62 | switch (level) { |
||
63 | case I915_CACHE_LLC_MLC: |
||
64 | /* Haswell doesn't set L3 this way */ |
||
65 | if (IS_HASWELL(dev)) |
||
66 | pte |= GEN6_PTE_CACHE_LLC; |
||
67 | else |
||
68 | pte |= GEN6_PTE_CACHE_LLC_MLC; |
||
69 | break; |
||
70 | case I915_CACHE_LLC: |
||
71 | pte |= GEN6_PTE_CACHE_LLC; |
||
72 | break; |
||
73 | case I915_CACHE_NONE: |
||
74 | if (IS_HASWELL(dev)) |
||
75 | pte |= HSW_PTE_UNCACHED; |
||
76 | else |
||
77 | pte |= GEN6_PTE_UNCACHED; |
||
78 | break; |
||
79 | default: |
||
80 | BUG(); |
||
81 | } |
||
82 | |||
83 | |||
84 | return pte; |
||
85 | } |
||
86 | |||
3031 | serge | 87 | /* PPGTT support for Sandybdrige/Gen6 and later */ |
3480 | Serge | 88 | static void gen6_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt, |
3031 | serge | 89 | unsigned first_entry, |
90 | unsigned num_entries) |
||
91 | { |
||
3243 | Serge | 92 | gtt_pte_t *pt_vaddr; |
93 | gtt_pte_t scratch_pte; |
||
3031 | serge | 94 | unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; |
95 | unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
||
96 | unsigned last_pte, i; |
||
97 | |||
3480 | Serge | 98 | scratch_pte = gen6_pte_encode(ppgtt->dev, |
99 | ppgtt->scratch_page_dma_addr, |
||
3243 | Serge | 100 | I915_CACHE_LLC); |
3031 | serge | 101 | |
102 | pt_vaddr = AllocKernelSpace(4096); |
||
103 | |||
3480 | Serge | 104 | if(pt_vaddr == NULL) |
105 | return; |
||
106 | |||
107 | while (num_entries) { |
||
3031 | serge | 108 | last_pte = first_pte + num_entries; |
109 | if (last_pte > I915_PPGTT_PT_ENTRIES) |
||
110 | last_pte = I915_PPGTT_PT_ENTRIES; |
||
111 | |||
3243 | Serge | 112 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pd]), 3); |
3031 | serge | 113 | |
114 | for (i = first_pte; i < last_pte; i++) |
||
115 | pt_vaddr[i] = scratch_pte; |
||
116 | |||
117 | num_entries -= last_pte - first_pte; |
||
118 | first_pte = 0; |
||
119 | act_pd++; |
||
3480 | Serge | 120 | }; |
121 | |||
122 | FreeKernelSpace(pt_vaddr); |
||
123 | } |
||
124 | |||
125 | static void gen6_ppgtt_insert_entries(struct i915_hw_ppgtt *ppgtt, |
||
126 | struct sg_table *pages, |
||
127 | unsigned first_entry, |
||
128 | enum i915_cache_level cache_level) |
||
129 | { |
||
130 | gtt_pte_t *pt_vaddr; |
||
131 | unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; |
||
132 | unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
||
133 | unsigned i, j, m, segment_len; |
||
134 | dma_addr_t page_addr; |
||
135 | struct scatterlist *sg; |
||
136 | |||
137 | /* init sg walking */ |
||
138 | sg = pages->sgl; |
||
139 | i = 0; |
||
140 | segment_len = sg_dma_len(sg) >> PAGE_SHIFT; |
||
141 | m = 0; |
||
142 | |||
143 | pt_vaddr = AllocKernelSpace(4096); |
||
144 | |||
145 | if(pt_vaddr == NULL) |
||
146 | return; |
||
147 | |||
148 | while (i < pages->nents) { |
||
149 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pd]), 3); |
||
150 | |||
151 | for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) { |
||
152 | page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT); |
||
153 | pt_vaddr[j] = gen6_pte_encode(ppgtt->dev, page_addr, |
||
154 | cache_level); |
||
155 | |||
156 | /* grab the next page */ |
||
157 | if (++m == segment_len) { |
||
158 | if (++i == pages->nents) |
||
159 | break; |
||
160 | |||
161 | sg = sg_next(sg); |
||
162 | segment_len = sg_dma_len(sg) >> PAGE_SHIFT; |
||
163 | m = 0; |
||
164 | } |
||
165 | } |
||
166 | |||
167 | first_pte = 0; |
||
168 | act_pd++; |
||
3031 | serge | 169 | } |
170 | FreeKernelSpace(pt_vaddr); |
||
171 | } |
||
172 | |||
3480 | Serge | 173 | static void gen6_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt) |
3031 | serge | 174 | { |
3480 | Serge | 175 | int i; |
176 | |||
177 | if (ppgtt->pt_dma_addr) { |
||
178 | for (i = 0; i < ppgtt->num_pd_entries; i++) |
||
179 | pci_unmap_page(ppgtt->dev->pdev, |
||
180 | ppgtt->pt_dma_addr[i], |
||
181 | 4096, PCI_DMA_BIDIRECTIONAL); |
||
182 | } |
||
183 | |||
184 | kfree(ppgtt->pt_dma_addr); |
||
185 | for (i = 0; i < ppgtt->num_pd_entries; i++) |
||
186 | __free_page(ppgtt->pt_pages[i]); |
||
187 | kfree(ppgtt->pt_pages); |
||
188 | kfree(ppgtt); |
||
189 | } |
||
190 | |||
191 | static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) |
||
192 | { |
||
193 | struct drm_device *dev = ppgtt->dev; |
||
3031 | serge | 194 | struct drm_i915_private *dev_priv = dev->dev_private; |
195 | unsigned first_pd_entry_in_global_pt; |
||
196 | int i; |
||
197 | int ret = -ENOMEM; |
||
198 | |||
199 | /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 |
||
200 | * entries. For aliasing ppgtt support we just steal them at the end for |
||
201 | * now. */ |
||
3480 | Serge | 202 | first_pd_entry_in_global_pt = |
203 | gtt_total_entries(dev_priv->gtt) - I915_PPGTT_PD_ENTRIES; |
||
3031 | serge | 204 | |
205 | ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES; |
||
3480 | Serge | 206 | ppgtt->clear_range = gen6_ppgtt_clear_range; |
207 | ppgtt->insert_entries = gen6_ppgtt_insert_entries; |
||
208 | ppgtt->cleanup = gen6_ppgtt_cleanup; |
||
3243 | Serge | 209 | ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries, |
3031 | serge | 210 | GFP_KERNEL); |
211 | if (!ppgtt->pt_pages) |
||
3480 | Serge | 212 | return -ENOMEM; |
3031 | serge | 213 | |
214 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
||
3243 | Serge | 215 | ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL); |
3031 | serge | 216 | if (!ppgtt->pt_pages[i]) |
217 | goto err_pt_alloc; |
||
218 | } |
||
219 | |||
3480 | Serge | 220 | ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries, |
3031 | serge | 221 | GFP_KERNEL); |
222 | if (!ppgtt->pt_dma_addr) |
||
223 | goto err_pt_alloc; |
||
224 | |||
225 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
||
226 | dma_addr_t pt_addr; |
||
227 | |||
3480 | Serge | 228 | pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096, |
3031 | serge | 229 | PCI_DMA_BIDIRECTIONAL); |
230 | |||
231 | ppgtt->pt_dma_addr[i] = pt_addr; |
||
232 | } |
||
233 | |||
3480 | Serge | 234 | ppgtt->scratch_page_dma_addr = dev_priv->gtt.scratch_page_dma; |
235 | |||
236 | ppgtt->clear_range(ppgtt, 0, |
||
3031 | serge | 237 | ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES); |
238 | |||
3243 | Serge | 239 | ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(gtt_pte_t); |
3031 | serge | 240 | |
241 | return 0; |
||
242 | |||
243 | err_pd_pin: |
||
3480 | Serge | 244 | if (ppgtt->pt_dma_addr) { |
245 | for (i--; i >= 0; i--) |
||
246 | pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], |
||
247 | 4096, PCI_DMA_BIDIRECTIONAL); |
||
248 | } |
||
3031 | serge | 249 | err_pt_alloc: |
3480 | Serge | 250 | kfree(ppgtt->pt_dma_addr); |
3031 | serge | 251 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
252 | if (ppgtt->pt_pages[i]) |
||
3480 | Serge | 253 | __free_page(ppgtt->pt_pages[i]); |
3031 | serge | 254 | } |
255 | kfree(ppgtt->pt_pages); |
||
256 | |||
257 | return ret; |
||
258 | } |
||
259 | |||
3480 | Serge | 260 | static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) |
3031 | serge | 261 | { |
262 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
3480 | Serge | 263 | struct i915_hw_ppgtt *ppgtt; |
264 | int ret; |
||
3031 | serge | 265 | |
3480 | Serge | 266 | ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); |
3031 | serge | 267 | if (!ppgtt) |
3480 | Serge | 268 | return -ENOMEM; |
3031 | serge | 269 | |
3480 | Serge | 270 | ppgtt->dev = dev; |
3031 | serge | 271 | |
3480 | Serge | 272 | ret = gen6_ppgtt_init(ppgtt); |
273 | if (ret) |
||
3031 | serge | 274 | kfree(ppgtt); |
3480 | Serge | 275 | else |
276 | dev_priv->mm.aliasing_ppgtt = ppgtt; |
||
277 | |||
278 | return ret; |
||
3031 | serge | 279 | } |
280 | |||
3480 | Serge | 281 | void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) |
3031 | serge | 282 | { |
3480 | Serge | 283 | struct drm_i915_private *dev_priv = dev->dev_private; |
284 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
||
3031 | serge | 285 | |
3480 | Serge | 286 | if (!ppgtt) |
3243 | Serge | 287 | return; |
3031 | serge | 288 | |
3480 | Serge | 289 | ppgtt->cleanup(ppgtt); |
3031 | serge | 290 | } |
291 | |||
292 | void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, |
||
293 | struct drm_i915_gem_object *obj, |
||
294 | enum i915_cache_level cache_level) |
||
295 | { |
||
3480 | Serge | 296 | ppgtt->insert_entries(ppgtt, obj->pages, |
3031 | serge | 297 | obj->gtt_space->start >> PAGE_SHIFT, |
3243 | Serge | 298 | cache_level); |
3031 | serge | 299 | } |
300 | |||
301 | void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, |
||
302 | struct drm_i915_gem_object *obj) |
||
303 | { |
||
3480 | Serge | 304 | ppgtt->clear_range(ppgtt, |
3031 | serge | 305 | obj->gtt_space->start >> PAGE_SHIFT, |
306 | obj->base.size >> PAGE_SHIFT); |
||
307 | } |
||
308 | |||
3243 | Serge | 309 | void i915_gem_init_ppgtt(struct drm_device *dev) |
2332 | Serge | 310 | { |
3243 | Serge | 311 | drm_i915_private_t *dev_priv = dev->dev_private; |
312 | uint32_t pd_offset; |
||
313 | struct intel_ring_buffer *ring; |
||
314 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
||
3480 | Serge | 315 | gtt_pte_t __iomem *pd_addr; |
3243 | Serge | 316 | uint32_t pd_entry; |
317 | int i; |
||
318 | |||
319 | if (!dev_priv->mm.aliasing_ppgtt) |
||
320 | return; |
||
321 | |||
322 | |||
3480 | Serge | 323 | pd_addr = (gtt_pte_t __iomem*)dev_priv->gtt.gsm + ppgtt->pd_offset/sizeof(gtt_pte_t); |
3243 | Serge | 324 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
325 | dma_addr_t pt_addr; |
||
326 | |||
327 | pt_addr = ppgtt->pt_dma_addr[i]; |
||
328 | pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); |
||
329 | pd_entry |= GEN6_PDE_VALID; |
||
330 | |||
331 | writel(pd_entry, pd_addr + i); |
||
2332 | Serge | 332 | } |
3243 | Serge | 333 | readl(pd_addr); |
334 | |||
335 | pd_offset = ppgtt->pd_offset; |
||
336 | pd_offset /= 64; /* in cachelines, */ |
||
337 | pd_offset <<= 16; |
||
338 | |||
339 | if (INTEL_INFO(dev)->gen == 6) { |
||
340 | uint32_t ecochk, gab_ctl, ecobits; |
||
341 | |||
342 | ecobits = I915_READ(GAC_ECO_BITS); |
||
343 | I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); |
||
344 | |||
345 | gab_ctl = I915_READ(GAB_CTL); |
||
346 | I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); |
||
347 | |||
348 | ecochk = I915_READ(GAM_ECOCHK); |
||
349 | I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | |
||
350 | ECOCHK_PPGTT_CACHE64B); |
||
351 | I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
||
352 | } else if (INTEL_INFO(dev)->gen >= 7) { |
||
353 | I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); |
||
354 | /* GFX_MODE is per-ring on gen7+ */ |
||
355 | } |
||
356 | |||
357 | for_each_ring(ring, dev_priv, i) { |
||
358 | if (INTEL_INFO(dev)->gen >= 7) |
||
359 | I915_WRITE(RING_MODE_GEN7(ring), |
||
360 | _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
||
361 | |||
362 | I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); |
||
363 | I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); |
||
364 | } |
||
2332 | Serge | 365 | } |
366 | |||
3480 | Serge | 367 | extern int intel_iommu_gfx_mapped; |
368 | /* Certain Gen5 chipsets require require idling the GPU before |
||
369 | * unmapping anything from the GTT when VT-d is enabled. |
||
370 | */ |
||
371 | static inline bool needs_idle_maps(struct drm_device *dev) |
||
372 | { |
||
373 | #ifdef CONFIG_INTEL_IOMMU |
||
374 | /* Query intel_iommu to see if we need the workaround. Presumably that |
||
375 | * was loaded first. |
||
376 | */ |
||
377 | if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped) |
||
378 | return true; |
||
379 | #endif |
||
380 | return false; |
||
381 | } |
||
382 | |||
2344 | Serge | 383 | static bool do_idling(struct drm_i915_private *dev_priv) |
384 | { |
||
385 | bool ret = dev_priv->mm.interruptible; |
||
386 | |||
3480 | Serge | 387 | if (unlikely(dev_priv->gtt.do_idle_maps)) { |
2344 | Serge | 388 | dev_priv->mm.interruptible = false; |
389 | if (i915_gpu_idle(dev_priv->dev)) { |
||
390 | DRM_ERROR("Couldn't idle GPU\n"); |
||
391 | /* Wait a bit, in hopes it avoids the hang */ |
||
392 | udelay(10); |
||
393 | } |
||
394 | } |
||
395 | |||
396 | return ret; |
||
397 | } |
||
398 | |||
399 | static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) |
||
400 | { |
||
3480 | Serge | 401 | if (unlikely(dev_priv->gtt.do_idle_maps)) |
2344 | Serge | 402 | dev_priv->mm.interruptible = interruptible; |
403 | } |
||
404 | |||
2332 | Serge | 405 | void i915_gem_restore_gtt_mappings(struct drm_device *dev) |
406 | { |
||
407 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
408 | struct drm_i915_gem_object *obj; |
||
409 | |||
410 | /* First fill our portion of the GTT with scratch pages */ |
||
3480 | Serge | 411 | dev_priv->gtt.gtt_clear_range(dev, dev_priv->gtt.start / PAGE_SIZE, |
412 | dev_priv->gtt.total / PAGE_SIZE); |
||
2332 | Serge | 413 | |
3031 | serge | 414 | list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) { |
2332 | Serge | 415 | i915_gem_clflush_object(obj); |
3031 | serge | 416 | i915_gem_gtt_bind_object(obj, obj->cache_level); |
2332 | Serge | 417 | } |
418 | |||
3243 | Serge | 419 | i915_gem_chipset_flush(dev); |
2332 | Serge | 420 | } |
421 | |||
3031 | serge | 422 | int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) |
2332 | Serge | 423 | { |
3243 | Serge | 424 | if (obj->has_dma_mapping) |
425 | return 0; |
||
426 | |||
3480 | Serge | 427 | if (!dma_map_sg(&obj->base.dev->pdev->dev, |
428 | obj->pages->sgl, obj->pages->nents, |
||
429 | PCI_DMA_BIDIRECTIONAL)) |
||
430 | return -ENOSPC; |
||
3243 | Serge | 431 | |
2332 | Serge | 432 | return 0; |
433 | } |
||
434 | |||
3243 | Serge | 435 | /* |
436 | * Binds an object into the global gtt with the specified cache level. The object |
||
437 | * will be accessible to the GPU via commands whose operands reference offsets |
||
438 | * within the global GTT as well as accessible by the GPU through the GMADR |
||
439 | * mapped BAR (dev_priv->mm.gtt->gtt). |
||
440 | */ |
||
3480 | Serge | 441 | static void gen6_ggtt_insert_entries(struct drm_device *dev, |
442 | struct sg_table *st, |
||
443 | unsigned int first_entry, |
||
3243 | Serge | 444 | enum i915_cache_level level) |
445 | { |
||
446 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
447 | struct scatterlist *sg = st->sgl; |
||
3480 | Serge | 448 | gtt_pte_t __iomem *gtt_entries = |
449 | (gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; |
||
3243 | Serge | 450 | int unused, i = 0; |
451 | unsigned int len, m = 0; |
||
452 | dma_addr_t addr; |
||
453 | |||
454 | for_each_sg(st->sgl, sg, st->nents, unused) { |
||
455 | len = sg_dma_len(sg) >> PAGE_SHIFT; |
||
456 | for (m = 0; m < len; m++) { |
||
457 | addr = sg_dma_address(sg) + (m << PAGE_SHIFT); |
||
3480 | Serge | 458 | iowrite32(gen6_pte_encode(dev, addr, level), |
459 | >t_entries[i]); |
||
3243 | Serge | 460 | i++; |
461 | } |
||
462 | } |
||
463 | |||
464 | /* XXX: This serves as a posting read to make sure that the PTE has |
||
465 | * actually been updated. There is some concern that even though |
||
466 | * registers and PTEs are within the same BAR that they are potentially |
||
467 | * of NUMA access patterns. Therefore, even with the way we assume |
||
468 | * hardware should work, we must keep this posting read for paranoia. |
||
469 | */ |
||
470 | if (i != 0) |
||
3480 | Serge | 471 | WARN_ON(readl(>t_entries[i-1]) |
472 | != gen6_pte_encode(dev, addr, level)); |
||
3243 | Serge | 473 | |
474 | /* This next bit makes the above posting read even more important. We |
||
475 | * want to flush the TLBs only after we're certain all the PTE updates |
||
476 | * have finished. |
||
477 | */ |
||
478 | I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); |
||
479 | POSTING_READ(GFX_FLSH_CNTL_GEN6); |
||
480 | } |
||
481 | |||
3480 | Serge | 482 | static void gen6_ggtt_clear_range(struct drm_device *dev, |
483 | unsigned int first_entry, |
||
484 | unsigned int num_entries) |
||
485 | { |
||
486 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
487 | gtt_pte_t scratch_pte; |
||
488 | gtt_pte_t __iomem *gtt_base = (gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; |
||
489 | const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; |
||
490 | int i; |
||
491 | |||
492 | if (WARN(num_entries > max_entries, |
||
493 | "First entry = %d; Num entries = %d (max=%d)\n", |
||
494 | first_entry, num_entries, max_entries)) |
||
495 | num_entries = max_entries; |
||
496 | |||
497 | scratch_pte = gen6_pte_encode(dev, dev_priv->gtt.scratch_page_dma, |
||
498 | I915_CACHE_LLC); |
||
499 | for (i = 0; i < num_entries; i++) |
||
500 | iowrite32(scratch_pte, >t_base[i]); |
||
501 | readl(gtt_base); |
||
502 | } |
||
503 | |||
504 | |||
505 | static void i915_ggtt_insert_entries(struct drm_device *dev, |
||
506 | struct sg_table *st, |
||
507 | unsigned int pg_start, |
||
508 | enum i915_cache_level cache_level) |
||
509 | { |
||
510 | unsigned int flags = (cache_level == I915_CACHE_NONE) ? |
||
511 | AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; |
||
512 | |||
513 | intel_gtt_insert_sg_entries(st, pg_start, flags); |
||
514 | |||
515 | } |
||
516 | |||
517 | static void i915_ggtt_clear_range(struct drm_device *dev, |
||
518 | unsigned int first_entry, |
||
519 | unsigned int num_entries) |
||
520 | { |
||
521 | intel_gtt_clear_range(first_entry, num_entries); |
||
522 | } |
||
523 | |||
524 | |||
3031 | serge | 525 | void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj, |
2332 | Serge | 526 | enum i915_cache_level cache_level) |
527 | { |
||
528 | struct drm_device *dev = obj->base.dev; |
||
3480 | Serge | 529 | struct drm_i915_private *dev_priv = dev->dev_private; |
530 | |||
531 | dev_priv->gtt.gtt_insert_entries(dev, obj->pages, |
||
3243 | Serge | 532 | obj->gtt_space->start >> PAGE_SHIFT, |
3480 | Serge | 533 | cache_level); |
2332 | Serge | 534 | |
3031 | serge | 535 | obj->has_global_gtt_mapping = 1; |
2332 | Serge | 536 | } |
537 | |||
538 | void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) |
||
539 | { |
||
3480 | Serge | 540 | struct drm_device *dev = obj->base.dev; |
541 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
542 | |||
543 | dev_priv->gtt.gtt_clear_range(obj->base.dev, |
||
3243 | Serge | 544 | obj->gtt_space->start >> PAGE_SHIFT, |
3031 | serge | 545 | obj->base.size >> PAGE_SHIFT); |
546 | |||
547 | obj->has_global_gtt_mapping = 0; |
||
548 | } |
||
549 | |||
550 | void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) |
||
551 | { |
||
2344 | Serge | 552 | struct drm_device *dev = obj->base.dev; |
553 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
554 | bool interruptible; |
||
555 | |||
556 | interruptible = do_idling(dev_priv); |
||
557 | |||
3480 | Serge | 558 | if (!obj->has_dma_mapping) |
559 | dma_unmap_sg(&dev->pdev->dev, |
||
560 | obj->pages->sgl, obj->pages->nents, |
||
561 | PCI_DMA_BIDIRECTIONAL); |
||
2332 | Serge | 562 | |
3031 | serge | 563 | undo_idling(dev_priv, interruptible); |
564 | } |
||
565 | |||
566 | static void i915_gtt_color_adjust(struct drm_mm_node *node, |
||
567 | unsigned long color, |
||
568 | unsigned long *start, |
||
569 | unsigned long *end) |
||
570 | { |
||
571 | if (node->color != color) |
||
572 | *start += 4096; |
||
573 | |||
574 | if (!list_empty(&node->node_list)) { |
||
575 | node = list_entry(node->node_list.next, |
||
576 | struct drm_mm_node, |
||
577 | node_list); |
||
578 | if (node->allocated && node->color != color) |
||
579 | *end -= 4096; |
||
2332 | Serge | 580 | } |
3031 | serge | 581 | } |
3480 | Serge | 582 | void i915_gem_setup_global_gtt(struct drm_device *dev, |
3031 | serge | 583 | unsigned long start, |
584 | unsigned long mappable_end, |
||
585 | unsigned long end) |
||
586 | { |
||
3480 | Serge | 587 | /* Let GEM Manage all of the aperture. |
588 | * |
||
589 | * However, leave one page at the end still bound to the scratch page. |
||
590 | * There are a number of places where the hardware apparently prefetches |
||
591 | * past the end of the object, and we've seen multiple hangs with the |
||
592 | * GPU head pointer stuck in a batchbuffer bound at the last page of the |
||
593 | * aperture. One page should be enough to keep any prefetching inside |
||
594 | * of the aperture. |
||
595 | */ |
||
3031 | serge | 596 | drm_i915_private_t *dev_priv = dev->dev_private; |
3480 | Serge | 597 | struct drm_mm_node *entry; |
598 | struct drm_i915_gem_object *obj; |
||
599 | unsigned long hole_start, hole_end; |
||
3031 | serge | 600 | |
3480 | Serge | 601 | BUG_ON(mappable_end > end); |
602 | |||
603 | /* Subtract the guard page ... */ |
||
3031 | serge | 604 | drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE); |
605 | if (!HAS_LLC(dev)) |
||
606 | dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust; |
||
607 | |||
3480 | Serge | 608 | /* Mark any preallocated objects as occupied */ |
609 | list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) { |
||
610 | DRM_DEBUG_KMS("reserving preallocated space: %x + %zx\n", |
||
611 | obj->gtt_offset, obj->base.size); |
||
3031 | serge | 612 | |
3480 | Serge | 613 | BUG_ON(obj->gtt_space != I915_GTT_RESERVED); |
614 | obj->gtt_space = drm_mm_create_block(&dev_priv->mm.gtt_space, |
||
615 | obj->gtt_offset, |
||
616 | obj->base.size, |
||
617 | false); |
||
618 | obj->has_global_gtt_mapping = 1; |
||
619 | } |
||
620 | |||
621 | dev_priv->gtt.start = start; |
||
622 | dev_priv->gtt.total = end - start; |
||
623 | |||
624 | /* Clear any non-preallocated blocks */ |
||
625 | drm_mm_for_each_hole(entry, &dev_priv->mm.gtt_space, |
||
626 | hole_start, hole_end) { |
||
627 | DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n", |
||
628 | hole_start, hole_end); |
||
629 | dev_priv->gtt.gtt_clear_range(dev, hole_start / PAGE_SIZE, |
||
630 | (hole_end-hole_start) / PAGE_SIZE); |
||
631 | } |
||
632 | |||
633 | /* And finally clear the reserved guard page */ |
||
634 | dev_priv->gtt.gtt_clear_range(dev, end / PAGE_SIZE - 1, 1); |
||
2332 | Serge | 635 | } |
3243 | Serge | 636 | |
3480 | Serge | 637 | static bool |
638 | intel_enable_ppgtt(struct drm_device *dev) |
||
639 | { |
||
640 | if (i915_enable_ppgtt >= 0) |
||
641 | return i915_enable_ppgtt; |
||
642 | |||
643 | #ifdef CONFIG_INTEL_IOMMU |
||
644 | /* Disable ppgtt on SNB if VT-d is on. */ |
||
645 | if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) |
||
646 | return false; |
||
647 | #endif |
||
648 | |||
649 | return true; |
||
650 | } |
||
651 | |||
652 | void i915_gem_init_global_gtt(struct drm_device *dev) |
||
653 | { |
||
654 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
655 | unsigned long gtt_size, mappable_size; |
||
656 | |||
657 | gtt_size = dev_priv->gtt.total; |
||
658 | mappable_size = dev_priv->gtt.mappable_end; |
||
659 | |||
660 | if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { |
||
661 | int ret; |
||
662 | /* PPGTT pdes are stolen from global gtt ptes, so shrink the |
||
663 | * aperture accordingly when using aliasing ppgtt. */ |
||
664 | gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; |
||
665 | gtt_size -= LFB_SIZE; |
||
666 | |||
667 | i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size); |
||
668 | |||
669 | ret = i915_gem_init_aliasing_ppgtt(dev); |
||
670 | if (!ret) |
||
671 | return; |
||
672 | |||
673 | DRM_ERROR("Aliased PPGTT setup failed %d\n", ret); |
||
674 | drm_mm_takedown(&dev_priv->mm.gtt_space); |
||
675 | gtt_size += I915_PPGTT_PD_ENTRIES*PAGE_SIZE; |
||
676 | } |
||
677 | i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size); |
||
678 | } |
||
679 | |||
3243 | Serge | 680 | static int setup_scratch_page(struct drm_device *dev) |
681 | { |
||
682 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
683 | struct page *page; |
||
684 | dma_addr_t dma_addr; |
||
685 | |||
686 | page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO); |
||
687 | if (page == NULL) |
||
688 | return -ENOMEM; |
||
3480 | Serge | 689 | get_page(page); |
690 | set_pages_uc(page, 1); |
||
3243 | Serge | 691 | |
692 | #ifdef CONFIG_INTEL_IOMMU |
||
693 | dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE, |
||
694 | PCI_DMA_BIDIRECTIONAL); |
||
695 | if (pci_dma_mapping_error(dev->pdev, dma_addr)) |
||
696 | return -EINVAL; |
||
697 | #else |
||
698 | dma_addr = page_to_phys(page); |
||
699 | #endif |
||
3480 | Serge | 700 | dev_priv->gtt.scratch_page = page; |
701 | dev_priv->gtt.scratch_page_dma = dma_addr; |
||
3243 | Serge | 702 | |
703 | return 0; |
||
704 | } |
||
705 | |||
3480 | Serge | 706 | static void teardown_scratch_page(struct drm_device *dev) |
707 | { |
||
708 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
709 | set_pages_wb(dev_priv->gtt.scratch_page, 1); |
||
710 | pci_unmap_page(dev->pdev, dev_priv->gtt.scratch_page_dma, |
||
711 | PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); |
||
712 | put_page(dev_priv->gtt.scratch_page); |
||
713 | __free_page(dev_priv->gtt.scratch_page); |
||
714 | } |
||
3243 | Serge | 715 | |
716 | static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) |
||
717 | { |
||
718 | snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; |
||
719 | snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; |
||
720 | return snb_gmch_ctl << 20; |
||
721 | } |
||
722 | |||
3480 | Serge | 723 | static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl) |
3243 | Serge | 724 | { |
725 | snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; |
||
726 | snb_gmch_ctl &= SNB_GMCH_GMS_MASK; |
||
727 | return snb_gmch_ctl << 25; /* 32 MB units */ |
||
728 | } |
||
729 | |||
3480 | Serge | 730 | static inline size_t gen7_get_stolen_size(u16 snb_gmch_ctl) |
3243 | Serge | 731 | { |
732 | static const int stolen_decoder[] = { |
||
733 | 0, 0, 0, 0, 0, 32, 48, 64, 128, 256, 96, 160, 224, 352}; |
||
734 | snb_gmch_ctl >>= IVB_GMCH_GMS_SHIFT; |
||
735 | snb_gmch_ctl &= IVB_GMCH_GMS_MASK; |
||
736 | return stolen_decoder[snb_gmch_ctl] << 20; |
||
737 | } |
||
738 | |||
3480 | Serge | 739 | static int gen6_gmch_probe(struct drm_device *dev, |
740 | size_t *gtt_total, |
||
741 | size_t *stolen, |
||
742 | phys_addr_t *mappable_base, |
||
743 | unsigned long *mappable_end) |
||
3243 | Serge | 744 | { |
745 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
746 | phys_addr_t gtt_bus_addr; |
||
3480 | Serge | 747 | unsigned int gtt_size; |
3243 | Serge | 748 | u16 snb_gmch_ctl; |
749 | int ret; |
||
750 | |||
3480 | Serge | 751 | *mappable_base = pci_resource_start(dev->pdev, 2); |
752 | *mappable_end = pci_resource_len(dev->pdev, 2); |
||
753 | |||
754 | /* 64/512MB is the current min/max we actually know of, but this is just |
||
755 | * a coarse sanity check. |
||
3243 | Serge | 756 | */ |
3480 | Serge | 757 | if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { |
758 | DRM_ERROR("Unknown GMADR size (%lx)\n", |
||
759 | dev_priv->gtt.mappable_end); |
||
760 | return -ENXIO; |
||
3243 | Serge | 761 | } |
762 | |||
3480 | Serge | 763 | if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40))) |
764 | pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40)); |
||
765 | pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); |
||
766 | gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); |
||
3243 | Serge | 767 | |
3480 | Serge | 768 | if (IS_GEN7(dev)) |
769 | *stolen = gen7_get_stolen_size(snb_gmch_ctl); |
||
770 | else |
||
771 | *stolen = gen6_get_stolen_size(snb_gmch_ctl); |
||
3243 | Serge | 772 | |
3480 | Serge | 773 | *gtt_total = (gtt_size / sizeof(gtt_pte_t)) << PAGE_SHIFT; |
3243 | Serge | 774 | |
775 | /* For GEN6+ the PTEs for the ggtt live at 2MB + BAR0 */ |
||
776 | gtt_bus_addr = pci_resource_start(dev->pdev, 0) + (2<<20); |
||
3480 | Serge | 777 | dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size); |
778 | if (!dev_priv->gtt.gsm) { |
||
779 | DRM_ERROR("Failed to map the gtt page table\n"); |
||
780 | return -ENOMEM; |
||
3243 | Serge | 781 | } |
782 | |||
783 | ret = setup_scratch_page(dev); |
||
3480 | Serge | 784 | if (ret) |
3243 | Serge | 785 | DRM_ERROR("Scratch setup failed\n"); |
786 | |||
3480 | Serge | 787 | dev_priv->gtt.gtt_clear_range = gen6_ggtt_clear_range; |
788 | dev_priv->gtt.gtt_insert_entries = gen6_ggtt_insert_entries; |
||
789 | |||
790 | return ret; |
||
791 | } |
||
792 | |||
793 | static void gen6_gmch_remove(struct drm_device *dev) |
||
794 | { |
||
795 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
796 | iounmap(dev_priv->gtt.gsm); |
||
797 | teardown_scratch_page(dev_priv->dev); |
||
798 | } |
||
799 | |||
800 | static int i915_gmch_probe(struct drm_device *dev, |
||
801 | size_t *gtt_total, |
||
802 | size_t *stolen, |
||
803 | phys_addr_t *mappable_base, |
||
804 | unsigned long *mappable_end) |
||
805 | { |
||
806 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
807 | int ret; |
||
808 | |||
809 | ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL); |
||
810 | if (!ret) { |
||
811 | DRM_ERROR("failed to set up gmch\n"); |
||
812 | return -EIO; |
||
3243 | Serge | 813 | } |
814 | |||
3480 | Serge | 815 | intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end); |
3243 | Serge | 816 | |
3480 | Serge | 817 | dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev); |
818 | dev_priv->gtt.gtt_clear_range = i915_ggtt_clear_range; |
||
819 | dev_priv->gtt.gtt_insert_entries = i915_ggtt_insert_entries; |
||
820 | |||
3243 | Serge | 821 | return 0; |
3480 | Serge | 822 | } |
3243 | Serge | 823 | |
3480 | Serge | 824 | static void i915_gmch_remove(struct drm_device *dev) |
825 | { |
||
826 | // intel_gmch_remove(); |
||
827 | } |
||
828 | |||
829 | int i915_gem_gtt_init(struct drm_device *dev) |
||
830 | { |
||
831 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
832 | struct i915_gtt *gtt = &dev_priv->gtt; |
||
833 | unsigned long gtt_size; |
||
834 | int ret; |
||
835 | |||
836 | if (INTEL_INFO(dev)->gen <= 5) { |
||
837 | dev_priv->gtt.gtt_probe = i915_gmch_probe; |
||
838 | dev_priv->gtt.gtt_remove = i915_gmch_remove; |
||
839 | } else { |
||
840 | dev_priv->gtt.gtt_probe = gen6_gmch_probe; |
||
841 | dev_priv->gtt.gtt_remove = gen6_gmch_remove; |
||
842 | } |
||
843 | |||
844 | ret = dev_priv->gtt.gtt_probe(dev, &dev_priv->gtt.total, |
||
845 | &dev_priv->gtt.stolen_size, |
||
846 | >t->mappable_base, |
||
847 | >t->mappable_end); |
||
848 | if (ret) |
||
3243 | Serge | 849 | return ret; |
3480 | Serge | 850 | |
851 | gtt_size = (dev_priv->gtt.total >> PAGE_SHIFT) * sizeof(gtt_pte_t); |
||
852 | |||
853 | /* GMADR is the PCI mmio aperture into the global GTT. */ |
||
854 | DRM_INFO("Memory usable by graphics device = %zdM\n", |
||
855 | dev_priv->gtt.total >> 20); |
||
856 | DRM_DEBUG_DRIVER("GMADR size = %ldM\n", |
||
857 | dev_priv->gtt.mappable_end >> 20); |
||
858 | DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", |
||
859 | dev_priv->gtt.stolen_size >> 20); |
||
860 | |||
861 | return 0; |
||
3243 | Serge | 862 | } |
863 | |||
864 | struct scatterlist *sg_next(struct scatterlist *sg) |
||
865 | { |
||
866 | if (sg_is_last(sg)) |
||
867 | return NULL; |
||
868 | |||
869 | sg++; |
||
870 | if (unlikely(sg_is_chain(sg))) |
||
871 | sg = sg_chain_ptr(sg); |
||
872 | |||
873 | return sg; |
||
874 | } |
||
875 | |||
876 | |||
877 | void __sg_free_table(struct sg_table *table, unsigned int max_ents, |
||
878 | sg_free_fn *free_fn) |
||
879 | { |
||
880 | struct scatterlist *sgl, *next; |
||
881 | |||
882 | if (unlikely(!table->sgl)) |
||
883 | return; |
||
884 | |||
885 | sgl = table->sgl; |
||
886 | while (table->orig_nents) { |
||
887 | unsigned int alloc_size = table->orig_nents; |
||
888 | unsigned int sg_size; |
||
889 | |||
890 | /* |
||
891 | * If we have more than max_ents segments left, |
||
892 | * then assign 'next' to the sg table after the current one. |
||
893 | * sg_size is then one less than alloc size, since the last |
||
894 | * element is the chain pointer. |
||
895 | */ |
||
896 | if (alloc_size > max_ents) { |
||
897 | next = sg_chain_ptr(&sgl[max_ents - 1]); |
||
898 | alloc_size = max_ents; |
||
899 | sg_size = alloc_size - 1; |
||
900 | } else { |
||
901 | sg_size = alloc_size; |
||
902 | next = NULL; |
||
903 | } |
||
904 | |||
905 | table->orig_nents -= sg_size; |
||
906 | kfree(sgl); |
||
907 | sgl = next; |
||
908 | } |
||
909 | |||
910 | table->sgl = NULL; |
||
911 | } |
||
912 | |||
913 | void sg_free_table(struct sg_table *table) |
||
914 | { |
||
915 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
||
916 | } |
||
917 | |||
918 | int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) |
||
919 | { |
||
920 | struct scatterlist *sg, *prv; |
||
921 | unsigned int left; |
||
922 | unsigned int max_ents = SG_MAX_SINGLE_ALLOC; |
||
923 | |||
924 | #ifndef ARCH_HAS_SG_CHAIN |
||
925 | BUG_ON(nents > max_ents); |
||
926 | #endif |
||
927 | |||
928 | memset(table, 0, sizeof(*table)); |
||
929 | |||
930 | left = nents; |
||
931 | prv = NULL; |
||
932 | do { |
||
933 | unsigned int sg_size, alloc_size = left; |
||
934 | |||
935 | if (alloc_size > max_ents) { |
||
936 | alloc_size = max_ents; |
||
937 | sg_size = alloc_size - 1; |
||
938 | } else |
||
939 | sg_size = alloc_size; |
||
940 | |||
941 | left -= sg_size; |
||
942 | |||
943 | sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask); |
||
944 | if (unlikely(!sg)) { |
||
945 | /* |
||
946 | * Adjust entry count to reflect that the last |
||
947 | * entry of the previous table won't be used for |
||
948 | * linkage. Without this, sg_kfree() may get |
||
949 | * confused. |
||
950 | */ |
||
951 | if (prv) |
||
952 | table->nents = ++table->orig_nents; |
||
953 | |||
954 | goto err; |
||
955 | } |
||
956 | |||
957 | sg_init_table(sg, alloc_size); |
||
958 | table->nents = table->orig_nents += sg_size; |
||
959 | |||
960 | /* |
||
961 | * If this is the first mapping, assign the sg table header. |
||
962 | * If this is not the first mapping, chain previous part. |
||
963 | */ |
||
964 | if (prv) |
||
965 | sg_chain(prv, max_ents, sg); |
||
966 | else |
||
967 | table->sgl = sg; |
||
968 | |||
969 | /* |
||
970 | * If no more entries after this one, mark the end |
||
971 | */ |
||
972 | if (!left) |
||
973 | sg_mark_end(&sg[sg_size - 1]); |
||
974 | |||
975 | prv = sg; |
||
976 | } while (left); |
||
977 | |||
978 | return 0; |
||
979 | |||
980 | err: |
||
981 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
||
982 | |||
983 | return -ENOMEM; |
||
984 | } |
||
985 | |||
986 | |||
987 | void sg_init_table(struct scatterlist *sgl, unsigned int nents) |
||
988 | { |
||
989 | memset(sgl, 0, sizeof(*sgl) * nents); |
||
990 | #ifdef CONFIG_DEBUG_SG |
||
991 | { |
||
992 | unsigned int i; |
||
993 | for (i = 0; i < nents; i++) |
||
994 | sgl[i].sg_magic = SG_MAGIC; |
||
995 | } |
||
996 | #endif |
||
997 | sg_mark_end(&sgl[nents - 1]); |
||
998 | }>=>20); |
||
999 |