Rev 3480 | Rev 3764 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
2332 | Serge | 1 | /* |
2 | * Copyright © 2010 Daniel Vetter |
||
3 | * |
||
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
||
5 | * copy of this software and associated documentation files (the "Software"), |
||
6 | * to deal in the Software without restriction, including without limitation |
||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
||
8 | * and/or sell copies of the Software, and to permit persons to whom the |
||
9 | * Software is furnished to do so, subject to the following conditions: |
||
10 | * |
||
11 | * The above copyright notice and this permission notice (including the next |
||
12 | * paragraph) shall be included in all copies or substantial portions of the |
||
13 | * Software. |
||
14 | * |
||
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
||
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
||
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
||
21 | * IN THE SOFTWARE. |
||
22 | * |
||
23 | */ |
||
24 | |||
3243 | Serge | 25 | |
3480 | Serge | 26 | #define AGP_NORMAL_MEMORY 0 |
27 | |||
28 | #define AGP_USER_TYPES (1 << 16) |
||
29 | #define AGP_USER_MEMORY (AGP_USER_TYPES) |
||
30 | #define AGP_USER_CACHED_MEMORY (AGP_USER_TYPES + 1) |
||
31 | |||
3031 | serge | 32 | #include |
33 | #include |
||
2332 | Serge | 34 | #include "i915_drv.h" |
2351 | Serge | 35 | #include "i915_trace.h" |
2332 | Serge | 36 | #include "intel_drv.h" |
37 | |||
3746 | Serge | 38 | typedef uint32_t gen6_gtt_pte_t; |
3243 | Serge | 39 | |
40 | /* PPGTT stuff */ |
||
41 | #define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) |
||
42 | |||
43 | #define GEN6_PDE_VALID (1 << 0) |
||
44 | /* gen6+ has bit 11-4 for physical addr bit 39-32 */ |
||
45 | #define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) |
||
46 | |||
47 | #define GEN6_PTE_VALID (1 << 0) |
||
48 | #define GEN6_PTE_UNCACHED (1 << 1) |
||
49 | #define HSW_PTE_UNCACHED (0) |
||
50 | #define GEN6_PTE_CACHE_LLC (2 << 1) |
||
51 | #define GEN6_PTE_CACHE_LLC_MLC (3 << 1) |
||
52 | #define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) |
||
53 | |||
3746 | Serge | 54 | static inline gen6_gtt_pte_t gen6_pte_encode(struct drm_device *dev, |
3243 | Serge | 55 | dma_addr_t addr, |
56 | enum i915_cache_level level) |
||
57 | { |
||
3746 | Serge | 58 | gen6_gtt_pte_t pte = GEN6_PTE_VALID; |
3243 | Serge | 59 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
60 | |||
61 | switch (level) { |
||
62 | case I915_CACHE_LLC_MLC: |
||
63 | /* Haswell doesn't set L3 this way */ |
||
64 | if (IS_HASWELL(dev)) |
||
65 | pte |= GEN6_PTE_CACHE_LLC; |
||
66 | else |
||
67 | pte |= GEN6_PTE_CACHE_LLC_MLC; |
||
68 | break; |
||
69 | case I915_CACHE_LLC: |
||
70 | pte |= GEN6_PTE_CACHE_LLC; |
||
71 | break; |
||
72 | case I915_CACHE_NONE: |
||
73 | if (IS_HASWELL(dev)) |
||
74 | pte |= HSW_PTE_UNCACHED; |
||
75 | else |
||
76 | pte |= GEN6_PTE_UNCACHED; |
||
77 | break; |
||
78 | default: |
||
79 | BUG(); |
||
80 | } |
||
81 | |||
82 | return pte; |
||
83 | } |
||
84 | |||
3746 | Serge | 85 | static int gen6_ppgtt_enable(struct drm_device *dev) |
86 | { |
||
87 | drm_i915_private_t *dev_priv = dev->dev_private; |
||
88 | uint32_t pd_offset; |
||
89 | struct intel_ring_buffer *ring; |
||
90 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
||
91 | gen6_gtt_pte_t __iomem *pd_addr; |
||
92 | uint32_t pd_entry; |
||
93 | int i; |
||
94 | |||
95 | pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm + |
||
96 | ppgtt->pd_offset / sizeof(gen6_gtt_pte_t); |
||
97 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
||
98 | dma_addr_t pt_addr; |
||
99 | |||
100 | pt_addr = ppgtt->pt_dma_addr[i]; |
||
101 | pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); |
||
102 | pd_entry |= GEN6_PDE_VALID; |
||
103 | |||
104 | writel(pd_entry, pd_addr + i); |
||
105 | } |
||
106 | readl(pd_addr); |
||
107 | |||
108 | pd_offset = ppgtt->pd_offset; |
||
109 | pd_offset /= 64; /* in cachelines, */ |
||
110 | pd_offset <<= 16; |
||
111 | |||
112 | if (INTEL_INFO(dev)->gen == 6) { |
||
113 | uint32_t ecochk, gab_ctl, ecobits; |
||
114 | |||
115 | ecobits = I915_READ(GAC_ECO_BITS); |
||
116 | I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT | |
||
117 | ECOBITS_PPGTT_CACHE64B); |
||
118 | |||
119 | gab_ctl = I915_READ(GAB_CTL); |
||
120 | I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); |
||
121 | |||
122 | ecochk = I915_READ(GAM_ECOCHK); |
||
123 | I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | |
||
124 | ECOCHK_PPGTT_CACHE64B); |
||
125 | I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
||
126 | } else if (INTEL_INFO(dev)->gen >= 7) { |
||
127 | uint32_t ecochk, ecobits; |
||
128 | |||
129 | ecobits = I915_READ(GAC_ECO_BITS); |
||
130 | I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); |
||
131 | |||
132 | ecochk = I915_READ(GAM_ECOCHK); |
||
133 | if (IS_HASWELL(dev)) { |
||
134 | ecochk |= ECOCHK_PPGTT_WB_HSW; |
||
135 | } else { |
||
136 | ecochk |= ECOCHK_PPGTT_LLC_IVB; |
||
137 | ecochk &= ~ECOCHK_PPGTT_GFDT_IVB; |
||
138 | } |
||
139 | I915_WRITE(GAM_ECOCHK, ecochk); |
||
140 | /* GFX_MODE is per-ring on gen7+ */ |
||
141 | } |
||
142 | |||
143 | for_each_ring(ring, dev_priv, i) { |
||
144 | if (INTEL_INFO(dev)->gen >= 7) |
||
145 | I915_WRITE(RING_MODE_GEN7(ring), |
||
146 | _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
||
147 | |||
148 | I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); |
||
149 | I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); |
||
150 | } |
||
151 | return 0; |
||
152 | } |
||
153 | |||
3031 | serge | 154 | /* PPGTT support for Sandybdrige/Gen6 and later */ |
3480 | Serge | 155 | static void gen6_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt, |
3031 | serge | 156 | unsigned first_entry, |
157 | unsigned num_entries) |
||
158 | { |
||
3746 | Serge | 159 | gen6_gtt_pte_t *pt_vaddr, scratch_pte; |
160 | unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; |
||
3031 | serge | 161 | unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
162 | unsigned last_pte, i; |
||
163 | |||
3480 | Serge | 164 | scratch_pte = gen6_pte_encode(ppgtt->dev, |
165 | ppgtt->scratch_page_dma_addr, |
||
3243 | Serge | 166 | I915_CACHE_LLC); |
3031 | serge | 167 | |
168 | pt_vaddr = AllocKernelSpace(4096); |
||
169 | |||
3480 | Serge | 170 | if(pt_vaddr == NULL) |
171 | return; |
||
172 | |||
173 | while (num_entries) { |
||
3031 | serge | 174 | last_pte = first_pte + num_entries; |
175 | if (last_pte > I915_PPGTT_PT_ENTRIES) |
||
176 | last_pte = I915_PPGTT_PT_ENTRIES; |
||
177 | |||
3746 | Serge | 178 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
3031 | serge | 179 | |
180 | for (i = first_pte; i < last_pte; i++) |
||
181 | pt_vaddr[i] = scratch_pte; |
||
182 | |||
183 | num_entries -= last_pte - first_pte; |
||
184 | first_pte = 0; |
||
3746 | Serge | 185 | act_pt++; |
3480 | Serge | 186 | }; |
187 | |||
188 | FreeKernelSpace(pt_vaddr); |
||
189 | } |
||
190 | |||
191 | static void gen6_ppgtt_insert_entries(struct i915_hw_ppgtt *ppgtt, |
||
192 | struct sg_table *pages, |
||
193 | unsigned first_entry, |
||
194 | enum i915_cache_level cache_level) |
||
195 | { |
||
3746 | Serge | 196 | gen6_gtt_pte_t *pt_vaddr; |
197 | unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; |
||
198 | unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
||
199 | struct sg_page_iter sg_iter; |
||
3480 | Serge | 200 | dma_addr_t page_addr; |
201 | |||
202 | |||
203 | pt_vaddr = AllocKernelSpace(4096); |
||
204 | |||
205 | if(pt_vaddr == NULL) |
||
206 | return; |
||
207 | |||
3746 | Serge | 208 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
209 | for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { |
||
210 | dma_addr_t page_addr; |
||
3480 | Serge | 211 | |
3746 | Serge | 212 | page_addr = sg_page_iter_dma_address(&sg_iter); |
213 | pt_vaddr[act_pte] = gen6_pte_encode(ppgtt->dev, page_addr, |
||
3480 | Serge | 214 | cache_level); |
3746 | Serge | 215 | if (++act_pte == I915_PPGTT_PT_ENTRIES) { |
216 | act_pt++; |
||
217 | MapPage(pt_vaddr,(addr_t)(ppgtt->pt_pages[act_pt]), 3); |
||
218 | act_pte = 0; |
||
3480 | Serge | 219 | |
220 | } |
||
3031 | serge | 221 | } |
222 | FreeKernelSpace(pt_vaddr); |
||
223 | } |
||
224 | |||
3480 | Serge | 225 | static void gen6_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt) |
3031 | serge | 226 | { |
3480 | Serge | 227 | int i; |
228 | |||
229 | if (ppgtt->pt_dma_addr) { |
||
230 | for (i = 0; i < ppgtt->num_pd_entries; i++) |
||
231 | pci_unmap_page(ppgtt->dev->pdev, |
||
232 | ppgtt->pt_dma_addr[i], |
||
233 | 4096, PCI_DMA_BIDIRECTIONAL); |
||
234 | } |
||
235 | |||
236 | kfree(ppgtt->pt_dma_addr); |
||
237 | for (i = 0; i < ppgtt->num_pd_entries; i++) |
||
238 | __free_page(ppgtt->pt_pages[i]); |
||
239 | kfree(ppgtt->pt_pages); |
||
240 | kfree(ppgtt); |
||
241 | } |
||
242 | |||
243 | static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) |
||
244 | { |
||
245 | struct drm_device *dev = ppgtt->dev; |
||
3031 | serge | 246 | struct drm_i915_private *dev_priv = dev->dev_private; |
247 | unsigned first_pd_entry_in_global_pt; |
||
248 | int i; |
||
249 | int ret = -ENOMEM; |
||
250 | |||
251 | /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 |
||
252 | * entries. For aliasing ppgtt support we just steal them at the end for |
||
253 | * now. */ |
||
3746 | Serge | 254 | first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt); |
3031 | serge | 255 | |
256 | ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES; |
||
3746 | Serge | 257 | ppgtt->enable = gen6_ppgtt_enable; |
3480 | Serge | 258 | ppgtt->clear_range = gen6_ppgtt_clear_range; |
259 | ppgtt->insert_entries = gen6_ppgtt_insert_entries; |
||
260 | ppgtt->cleanup = gen6_ppgtt_cleanup; |
||
3243 | Serge | 261 | ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries, |
3031 | serge | 262 | GFP_KERNEL); |
263 | if (!ppgtt->pt_pages) |
||
3480 | Serge | 264 | return -ENOMEM; |
3031 | serge | 265 | |
266 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
||
3243 | Serge | 267 | ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL); |
3031 | serge | 268 | if (!ppgtt->pt_pages[i]) |
269 | goto err_pt_alloc; |
||
270 | } |
||
271 | |||
3480 | Serge | 272 | ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries, |
3031 | serge | 273 | GFP_KERNEL); |
274 | if (!ppgtt->pt_dma_addr) |
||
275 | goto err_pt_alloc; |
||
276 | |||
277 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
||
278 | dma_addr_t pt_addr; |
||
279 | |||
3480 | Serge | 280 | pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096, |
3031 | serge | 281 | PCI_DMA_BIDIRECTIONAL); |
282 | |||
283 | ppgtt->pt_dma_addr[i] = pt_addr; |
||
284 | } |
||
285 | |||
3480 | Serge | 286 | ppgtt->clear_range(ppgtt, 0, |
3031 | serge | 287 | ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES); |
288 | |||
3746 | Serge | 289 | ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t); |
3031 | serge | 290 | |
291 | return 0; |
||
292 | |||
293 | err_pd_pin: |
||
3480 | Serge | 294 | if (ppgtt->pt_dma_addr) { |
295 | for (i--; i >= 0; i--) |
||
296 | pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], |
||
297 | 4096, PCI_DMA_BIDIRECTIONAL); |
||
298 | } |
||
3031 | serge | 299 | err_pt_alloc: |
3480 | Serge | 300 | kfree(ppgtt->pt_dma_addr); |
3031 | serge | 301 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
302 | if (ppgtt->pt_pages[i]) |
||
3480 | Serge | 303 | __free_page(ppgtt->pt_pages[i]); |
3031 | serge | 304 | } |
305 | kfree(ppgtt->pt_pages); |
||
306 | |||
307 | return ret; |
||
308 | } |
||
309 | |||
3480 | Serge | 310 | static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) |
3031 | serge | 311 | { |
312 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
3480 | Serge | 313 | struct i915_hw_ppgtt *ppgtt; |
314 | int ret; |
||
3031 | serge | 315 | |
3480 | Serge | 316 | ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); |
3031 | serge | 317 | if (!ppgtt) |
3480 | Serge | 318 | return -ENOMEM; |
3031 | serge | 319 | |
3480 | Serge | 320 | ppgtt->dev = dev; |
3746 | Serge | 321 | ppgtt->scratch_page_dma_addr = dev_priv->gtt.scratch_page_dma; |
3031 | serge | 322 | |
3746 | Serge | 323 | if (INTEL_INFO(dev)->gen < 8) |
3480 | Serge | 324 | ret = gen6_ppgtt_init(ppgtt); |
3746 | Serge | 325 | else |
326 | BUG(); |
||
327 | |||
3480 | Serge | 328 | if (ret) |
3031 | serge | 329 | kfree(ppgtt); |
3480 | Serge | 330 | else |
331 | dev_priv->mm.aliasing_ppgtt = ppgtt; |
||
332 | |||
333 | return ret; |
||
3031 | serge | 334 | } |
335 | |||
3480 | Serge | 336 | void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) |
3031 | serge | 337 | { |
3480 | Serge | 338 | struct drm_i915_private *dev_priv = dev->dev_private; |
339 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
||
3031 | serge | 340 | |
3480 | Serge | 341 | if (!ppgtt) |
3243 | Serge | 342 | return; |
3031 | serge | 343 | |
3480 | Serge | 344 | ppgtt->cleanup(ppgtt); |
3746 | Serge | 345 | dev_priv->mm.aliasing_ppgtt = NULL; |
3031 | serge | 346 | } |
347 | |||
348 | void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, |
||
349 | struct drm_i915_gem_object *obj, |
||
350 | enum i915_cache_level cache_level) |
||
351 | { |
||
3480 | Serge | 352 | ppgtt->insert_entries(ppgtt, obj->pages, |
3031 | serge | 353 | obj->gtt_space->start >> PAGE_SHIFT, |
3243 | Serge | 354 | cache_level); |
3031 | serge | 355 | } |
356 | |||
357 | void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, |
||
358 | struct drm_i915_gem_object *obj) |
||
359 | { |
||
3480 | Serge | 360 | ppgtt->clear_range(ppgtt, |
3031 | serge | 361 | obj->gtt_space->start >> PAGE_SHIFT, |
362 | obj->base.size >> PAGE_SHIFT); |
||
363 | } |
||
364 | |||
3480 | Serge | 365 | extern int intel_iommu_gfx_mapped; |
366 | /* Certain Gen5 chipsets require require idling the GPU before |
||
367 | * unmapping anything from the GTT when VT-d is enabled. |
||
368 | */ |
||
369 | static inline bool needs_idle_maps(struct drm_device *dev) |
||
370 | { |
||
371 | #ifdef CONFIG_INTEL_IOMMU |
||
372 | /* Query intel_iommu to see if we need the workaround. Presumably that |
||
373 | * was loaded first. |
||
374 | */ |
||
375 | if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped) |
||
376 | return true; |
||
377 | #endif |
||
378 | return false; |
||
379 | } |
||
380 | |||
2344 | Serge | 381 | static bool do_idling(struct drm_i915_private *dev_priv) |
382 | { |
||
383 | bool ret = dev_priv->mm.interruptible; |
||
384 | |||
3480 | Serge | 385 | if (unlikely(dev_priv->gtt.do_idle_maps)) { |
2344 | Serge | 386 | dev_priv->mm.interruptible = false; |
387 | if (i915_gpu_idle(dev_priv->dev)) { |
||
388 | DRM_ERROR("Couldn't idle GPU\n"); |
||
389 | /* Wait a bit, in hopes it avoids the hang */ |
||
390 | udelay(10); |
||
391 | } |
||
392 | } |
||
393 | |||
394 | return ret; |
||
395 | } |
||
396 | |||
397 | static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) |
||
398 | { |
||
3480 | Serge | 399 | if (unlikely(dev_priv->gtt.do_idle_maps)) |
2344 | Serge | 400 | dev_priv->mm.interruptible = interruptible; |
401 | } |
||
402 | |||
2332 | Serge | 403 | void i915_gem_restore_gtt_mappings(struct drm_device *dev) |
404 | { |
||
405 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
406 | struct drm_i915_gem_object *obj; |
||
407 | |||
408 | /* First fill our portion of the GTT with scratch pages */ |
||
3480 | Serge | 409 | dev_priv->gtt.gtt_clear_range(dev, dev_priv->gtt.start / PAGE_SIZE, |
410 | dev_priv->gtt.total / PAGE_SIZE); |
||
2332 | Serge | 411 | |
3031 | serge | 412 | list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) { |
2332 | Serge | 413 | i915_gem_clflush_object(obj); |
3031 | serge | 414 | i915_gem_gtt_bind_object(obj, obj->cache_level); |
2332 | Serge | 415 | } |
416 | |||
3243 | Serge | 417 | i915_gem_chipset_flush(dev); |
2332 | Serge | 418 | } |
419 | |||
3031 | serge | 420 | int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) |
2332 | Serge | 421 | { |
3243 | Serge | 422 | if (obj->has_dma_mapping) |
423 | return 0; |
||
424 | |||
3480 | Serge | 425 | if (!dma_map_sg(&obj->base.dev->pdev->dev, |
426 | obj->pages->sgl, obj->pages->nents, |
||
427 | PCI_DMA_BIDIRECTIONAL)) |
||
428 | return -ENOSPC; |
||
3243 | Serge | 429 | |
2332 | Serge | 430 | return 0; |
431 | } |
||
432 | |||
3243 | Serge | 433 | /* |
434 | * Binds an object into the global gtt with the specified cache level. The object |
||
435 | * will be accessible to the GPU via commands whose operands reference offsets |
||
436 | * within the global GTT as well as accessible by the GPU through the GMADR |
||
437 | * mapped BAR (dev_priv->mm.gtt->gtt). |
||
438 | */ |
||
3480 | Serge | 439 | static void gen6_ggtt_insert_entries(struct drm_device *dev, |
440 | struct sg_table *st, |
||
441 | unsigned int first_entry, |
||
3243 | Serge | 442 | enum i915_cache_level level) |
443 | { |
||
444 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
3746 | Serge | 445 | gen6_gtt_pte_t __iomem *gtt_entries = |
446 | (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; |
||
447 | int i = 0; |
||
448 | struct sg_page_iter sg_iter; |
||
3243 | Serge | 449 | dma_addr_t addr; |
450 | |||
3746 | Serge | 451 | for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) { |
452 | addr = sg_page_iter_dma_address(&sg_iter); |
||
453 | iowrite32(gen6_pte_encode(dev, addr, level), >t_entries[i]); |
||
3243 | Serge | 454 | i++; |
455 | } |
||
456 | |||
457 | /* XXX: This serves as a posting read to make sure that the PTE has |
||
458 | * actually been updated. There is some concern that even though |
||
459 | * registers and PTEs are within the same BAR that they are potentially |
||
460 | * of NUMA access patterns. Therefore, even with the way we assume |
||
461 | * hardware should work, we must keep this posting read for paranoia. |
||
462 | */ |
||
463 | if (i != 0) |
||
3480 | Serge | 464 | WARN_ON(readl(>t_entries[i-1]) |
465 | != gen6_pte_encode(dev, addr, level)); |
||
3243 | Serge | 466 | |
467 | /* This next bit makes the above posting read even more important. We |
||
468 | * want to flush the TLBs only after we're certain all the PTE updates |
||
469 | * have finished. |
||
470 | */ |
||
471 | I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); |
||
472 | POSTING_READ(GFX_FLSH_CNTL_GEN6); |
||
473 | } |
||
474 | |||
3480 | Serge | 475 | static void gen6_ggtt_clear_range(struct drm_device *dev, |
476 | unsigned int first_entry, |
||
477 | unsigned int num_entries) |
||
478 | { |
||
479 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
3746 | Serge | 480 | gen6_gtt_pte_t scratch_pte, __iomem *gtt_base = |
481 | (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; |
||
3480 | Serge | 482 | const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; |
483 | int i; |
||
484 | |||
3746 | Serge | 485 | // if (WARN(num_entries > max_entries, |
486 | // "First entry = %d; Num entries = %d (max=%d)\n", |
||
487 | // first_entry, num_entries, max_entries)) |
||
488 | if (num_entries > max_entries) |
||
3480 | Serge | 489 | num_entries = max_entries; |
490 | |||
491 | scratch_pte = gen6_pte_encode(dev, dev_priv->gtt.scratch_page_dma, |
||
492 | I915_CACHE_LLC); |
||
493 | for (i = 0; i < num_entries; i++) |
||
494 | iowrite32(scratch_pte, >t_base[i]); |
||
495 | readl(gtt_base); |
||
496 | } |
||
497 | |||
498 | |||
499 | static void i915_ggtt_insert_entries(struct drm_device *dev, |
||
500 | struct sg_table *st, |
||
501 | unsigned int pg_start, |
||
502 | enum i915_cache_level cache_level) |
||
503 | { |
||
504 | unsigned int flags = (cache_level == I915_CACHE_NONE) ? |
||
505 | AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; |
||
506 | |||
507 | intel_gtt_insert_sg_entries(st, pg_start, flags); |
||
508 | |||
509 | } |
||
510 | |||
511 | static void i915_ggtt_clear_range(struct drm_device *dev, |
||
512 | unsigned int first_entry, |
||
513 | unsigned int num_entries) |
||
514 | { |
||
515 | intel_gtt_clear_range(first_entry, num_entries); |
||
516 | } |
||
517 | |||
518 | |||
3031 | serge | 519 | void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj, |
2332 | Serge | 520 | enum i915_cache_level cache_level) |
521 | { |
||
522 | struct drm_device *dev = obj->base.dev; |
||
3480 | Serge | 523 | struct drm_i915_private *dev_priv = dev->dev_private; |
524 | |||
525 | dev_priv->gtt.gtt_insert_entries(dev, obj->pages, |
||
3243 | Serge | 526 | obj->gtt_space->start >> PAGE_SHIFT, |
3480 | Serge | 527 | cache_level); |
2332 | Serge | 528 | |
3031 | serge | 529 | obj->has_global_gtt_mapping = 1; |
2332 | Serge | 530 | } |
531 | |||
532 | void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) |
||
533 | { |
||
3480 | Serge | 534 | struct drm_device *dev = obj->base.dev; |
535 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
536 | |||
537 | dev_priv->gtt.gtt_clear_range(obj->base.dev, |
||
3243 | Serge | 538 | obj->gtt_space->start >> PAGE_SHIFT, |
3031 | serge | 539 | obj->base.size >> PAGE_SHIFT); |
540 | |||
541 | obj->has_global_gtt_mapping = 0; |
||
542 | } |
||
543 | |||
544 | void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) |
||
545 | { |
||
2344 | Serge | 546 | struct drm_device *dev = obj->base.dev; |
547 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
548 | bool interruptible; |
||
549 | |||
550 | interruptible = do_idling(dev_priv); |
||
551 | |||
3480 | Serge | 552 | if (!obj->has_dma_mapping) |
553 | dma_unmap_sg(&dev->pdev->dev, |
||
554 | obj->pages->sgl, obj->pages->nents, |
||
555 | PCI_DMA_BIDIRECTIONAL); |
||
2332 | Serge | 556 | |
3031 | serge | 557 | undo_idling(dev_priv, interruptible); |
558 | } |
||
559 | |||
560 | static void i915_gtt_color_adjust(struct drm_mm_node *node, |
||
561 | unsigned long color, |
||
562 | unsigned long *start, |
||
563 | unsigned long *end) |
||
564 | { |
||
565 | if (node->color != color) |
||
566 | *start += 4096; |
||
567 | |||
568 | if (!list_empty(&node->node_list)) { |
||
569 | node = list_entry(node->node_list.next, |
||
570 | struct drm_mm_node, |
||
571 | node_list); |
||
572 | if (node->allocated && node->color != color) |
||
573 | *end -= 4096; |
||
2332 | Serge | 574 | } |
3031 | serge | 575 | } |
3480 | Serge | 576 | void i915_gem_setup_global_gtt(struct drm_device *dev, |
3031 | serge | 577 | unsigned long start, |
578 | unsigned long mappable_end, |
||
579 | unsigned long end) |
||
580 | { |
||
3480 | Serge | 581 | /* Let GEM Manage all of the aperture. |
582 | * |
||
583 | * However, leave one page at the end still bound to the scratch page. |
||
584 | * There are a number of places where the hardware apparently prefetches |
||
585 | * past the end of the object, and we've seen multiple hangs with the |
||
586 | * GPU head pointer stuck in a batchbuffer bound at the last page of the |
||
587 | * aperture. One page should be enough to keep any prefetching inside |
||
588 | * of the aperture. |
||
589 | */ |
||
3031 | serge | 590 | drm_i915_private_t *dev_priv = dev->dev_private; |
3480 | Serge | 591 | struct drm_mm_node *entry; |
592 | struct drm_i915_gem_object *obj; |
||
593 | unsigned long hole_start, hole_end; |
||
3031 | serge | 594 | |
3480 | Serge | 595 | BUG_ON(mappable_end > end); |
596 | |||
597 | /* Subtract the guard page ... */ |
||
3031 | serge | 598 | drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE); |
599 | if (!HAS_LLC(dev)) |
||
600 | dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust; |
||
601 | |||
3480 | Serge | 602 | /* Mark any preallocated objects as occupied */ |
603 | list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) { |
||
604 | DRM_DEBUG_KMS("reserving preallocated space: %x + %zx\n", |
||
605 | obj->gtt_offset, obj->base.size); |
||
3031 | serge | 606 | |
3480 | Serge | 607 | BUG_ON(obj->gtt_space != I915_GTT_RESERVED); |
608 | obj->gtt_space = drm_mm_create_block(&dev_priv->mm.gtt_space, |
||
609 | obj->gtt_offset, |
||
610 | obj->base.size, |
||
611 | false); |
||
612 | obj->has_global_gtt_mapping = 1; |
||
613 | } |
||
614 | |||
615 | dev_priv->gtt.start = start; |
||
616 | dev_priv->gtt.total = end - start; |
||
617 | |||
618 | /* Clear any non-preallocated blocks */ |
||
619 | drm_mm_for_each_hole(entry, &dev_priv->mm.gtt_space, |
||
620 | hole_start, hole_end) { |
||
621 | DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n", |
||
622 | hole_start, hole_end); |
||
623 | dev_priv->gtt.gtt_clear_range(dev, hole_start / PAGE_SIZE, |
||
624 | (hole_end-hole_start) / PAGE_SIZE); |
||
625 | } |
||
626 | |||
627 | /* And finally clear the reserved guard page */ |
||
628 | dev_priv->gtt.gtt_clear_range(dev, end / PAGE_SIZE - 1, 1); |
||
2332 | Serge | 629 | } |
3243 | Serge | 630 | |
3480 | Serge | 631 | static bool |
632 | intel_enable_ppgtt(struct drm_device *dev) |
||
633 | { |
||
634 | if (i915_enable_ppgtt >= 0) |
||
635 | return i915_enable_ppgtt; |
||
636 | |||
637 | #ifdef CONFIG_INTEL_IOMMU |
||
638 | /* Disable ppgtt on SNB if VT-d is on. */ |
||
639 | if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) |
||
640 | return false; |
||
641 | #endif |
||
642 | |||
643 | return true; |
||
644 | } |
||
645 | |||
646 | void i915_gem_init_global_gtt(struct drm_device *dev) |
||
647 | { |
||
648 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
649 | unsigned long gtt_size, mappable_size; |
||
650 | |||
651 | gtt_size = dev_priv->gtt.total; |
||
652 | mappable_size = dev_priv->gtt.mappable_end; |
||
653 | |||
3746 | Serge | 654 | #if 0 |
3480 | Serge | 655 | if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { |
656 | int ret; |
||
3746 | Serge | 657 | |
658 | if (INTEL_INFO(dev)->gen <= 7) { |
||
3480 | Serge | 659 | /* PPGTT pdes are stolen from global gtt ptes, so shrink the |
660 | * aperture accordingly when using aliasing ppgtt. */ |
||
661 | gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; |
||
3746 | Serge | 662 | } |
3480 | Serge | 663 | |
3746 | Serge | 664 | // gtt_size -= LFB_SIZE; |
665 | |||
3480 | Serge | 666 | i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size); |
667 | |||
668 | ret = i915_gem_init_aliasing_ppgtt(dev); |
||
669 | if (!ret) |
||
670 | return; |
||
671 | |||
672 | DRM_ERROR("Aliased PPGTT setup failed %d\n", ret); |
||
673 | drm_mm_takedown(&dev_priv->mm.gtt_space); |
||
674 | gtt_size += I915_PPGTT_PD_ENTRIES*PAGE_SIZE; |
||
3746 | Serge | 675 | } |
676 | #endif |
||
677 | |||
3480 | Serge | 678 | i915_gem_setup_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size); |
679 | } |
||
680 | |||
3243 | Serge | 681 | static int setup_scratch_page(struct drm_device *dev) |
682 | { |
||
683 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
684 | struct page *page; |
||
685 | dma_addr_t dma_addr; |
||
686 | |||
687 | page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO); |
||
688 | if (page == NULL) |
||
689 | return -ENOMEM; |
||
3480 | Serge | 690 | get_page(page); |
691 | set_pages_uc(page, 1); |
||
3243 | Serge | 692 | |
693 | #ifdef CONFIG_INTEL_IOMMU |
||
694 | dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE, |
||
695 | PCI_DMA_BIDIRECTIONAL); |
||
696 | if (pci_dma_mapping_error(dev->pdev, dma_addr)) |
||
697 | return -EINVAL; |
||
698 | #else |
||
699 | dma_addr = page_to_phys(page); |
||
700 | #endif |
||
3480 | Serge | 701 | dev_priv->gtt.scratch_page = page; |
702 | dev_priv->gtt.scratch_page_dma = dma_addr; |
||
3243 | Serge | 703 | |
704 | return 0; |
||
705 | } |
||
706 | |||
3480 | Serge | 707 | static void teardown_scratch_page(struct drm_device *dev) |
708 | { |
||
709 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
710 | set_pages_wb(dev_priv->gtt.scratch_page, 1); |
||
711 | pci_unmap_page(dev->pdev, dev_priv->gtt.scratch_page_dma, |
||
712 | PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); |
||
713 | put_page(dev_priv->gtt.scratch_page); |
||
714 | __free_page(dev_priv->gtt.scratch_page); |
||
715 | } |
||
3243 | Serge | 716 | |
717 | static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) |
||
718 | { |
||
719 | snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; |
||
720 | snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; |
||
721 | return snb_gmch_ctl << 20; |
||
722 | } |
||
723 | |||
3480 | Serge | 724 | static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl) |
3243 | Serge | 725 | { |
726 | snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; |
||
727 | snb_gmch_ctl &= SNB_GMCH_GMS_MASK; |
||
728 | return snb_gmch_ctl << 25; /* 32 MB units */ |
||
729 | } |
||
730 | |||
3480 | Serge | 731 | static int gen6_gmch_probe(struct drm_device *dev, |
732 | size_t *gtt_total, |
||
733 | size_t *stolen, |
||
734 | phys_addr_t *mappable_base, |
||
735 | unsigned long *mappable_end) |
||
3243 | Serge | 736 | { |
737 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
738 | phys_addr_t gtt_bus_addr; |
||
3480 | Serge | 739 | unsigned int gtt_size; |
3243 | Serge | 740 | u16 snb_gmch_ctl; |
741 | int ret; |
||
742 | |||
3480 | Serge | 743 | *mappable_base = pci_resource_start(dev->pdev, 2); |
744 | *mappable_end = pci_resource_len(dev->pdev, 2); |
||
745 | |||
746 | /* 64/512MB is the current min/max we actually know of, but this is just |
||
747 | * a coarse sanity check. |
||
3243 | Serge | 748 | */ |
3480 | Serge | 749 | if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { |
750 | DRM_ERROR("Unknown GMADR size (%lx)\n", |
||
751 | dev_priv->gtt.mappable_end); |
||
752 | return -ENXIO; |
||
3243 | Serge | 753 | } |
754 | |||
3480 | Serge | 755 | if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40))) |
756 | pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40)); |
||
757 | pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); |
||
758 | gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); |
||
3243 | Serge | 759 | |
3480 | Serge | 760 | *stolen = gen6_get_stolen_size(snb_gmch_ctl); |
3746 | Serge | 761 | *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT; |
3243 | Serge | 762 | |
3746 | Serge | 763 | /* For Modern GENs the PTEs and register space are split in the BAR */ |
764 | gtt_bus_addr = pci_resource_start(dev->pdev, 0) + |
||
765 | (pci_resource_len(dev->pdev, 0) / 2); |
||
3243 | Serge | 766 | |
3480 | Serge | 767 | dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size); |
768 | if (!dev_priv->gtt.gsm) { |
||
769 | DRM_ERROR("Failed to map the gtt page table\n"); |
||
770 | return -ENOMEM; |
||
3243 | Serge | 771 | } |
772 | |||
773 | ret = setup_scratch_page(dev); |
||
3480 | Serge | 774 | if (ret) |
3243 | Serge | 775 | DRM_ERROR("Scratch setup failed\n"); |
776 | |||
3480 | Serge | 777 | dev_priv->gtt.gtt_clear_range = gen6_ggtt_clear_range; |
778 | dev_priv->gtt.gtt_insert_entries = gen6_ggtt_insert_entries; |
||
779 | |||
780 | return ret; |
||
781 | } |
||
782 | |||
783 | static void gen6_gmch_remove(struct drm_device *dev) |
||
784 | { |
||
785 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
786 | iounmap(dev_priv->gtt.gsm); |
||
787 | teardown_scratch_page(dev_priv->dev); |
||
788 | } |
||
789 | |||
790 | static int i915_gmch_probe(struct drm_device *dev, |
||
791 | size_t *gtt_total, |
||
792 | size_t *stolen, |
||
793 | phys_addr_t *mappable_base, |
||
794 | unsigned long *mappable_end) |
||
795 | { |
||
796 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
797 | int ret; |
||
798 | |||
799 | ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL); |
||
800 | if (!ret) { |
||
801 | DRM_ERROR("failed to set up gmch\n"); |
||
802 | return -EIO; |
||
3243 | Serge | 803 | } |
804 | |||
3480 | Serge | 805 | intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end); |
3243 | Serge | 806 | |
3480 | Serge | 807 | dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev); |
808 | dev_priv->gtt.gtt_clear_range = i915_ggtt_clear_range; |
||
809 | dev_priv->gtt.gtt_insert_entries = i915_ggtt_insert_entries; |
||
810 | |||
3243 | Serge | 811 | return 0; |
3480 | Serge | 812 | } |
3243 | Serge | 813 | |
3480 | Serge | 814 | static void i915_gmch_remove(struct drm_device *dev) |
815 | { |
||
816 | // intel_gmch_remove(); |
||
817 | } |
||
818 | |||
819 | int i915_gem_gtt_init(struct drm_device *dev) |
||
820 | { |
||
821 | struct drm_i915_private *dev_priv = dev->dev_private; |
||
822 | struct i915_gtt *gtt = &dev_priv->gtt; |
||
823 | int ret; |
||
824 | |||
825 | if (INTEL_INFO(dev)->gen <= 5) { |
||
826 | dev_priv->gtt.gtt_probe = i915_gmch_probe; |
||
827 | dev_priv->gtt.gtt_remove = i915_gmch_remove; |
||
828 | } else { |
||
829 | dev_priv->gtt.gtt_probe = gen6_gmch_probe; |
||
830 | dev_priv->gtt.gtt_remove = gen6_gmch_remove; |
||
831 | } |
||
832 | |||
833 | ret = dev_priv->gtt.gtt_probe(dev, &dev_priv->gtt.total, |
||
834 | &dev_priv->gtt.stolen_size, |
||
835 | >t->mappable_base, |
||
836 | >t->mappable_end); |
||
837 | if (ret) |
||
3243 | Serge | 838 | return ret; |
3480 | Serge | 839 | |
840 | /* GMADR is the PCI mmio aperture into the global GTT. */ |
||
841 | DRM_INFO("Memory usable by graphics device = %zdM\n", |
||
842 | dev_priv->gtt.total >> 20); |
||
843 | DRM_DEBUG_DRIVER("GMADR size = %ldM\n", |
||
844 | dev_priv->gtt.mappable_end >> 20); |
||
845 | DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", |
||
846 | dev_priv->gtt.stolen_size >> 20); |
||
847 | |||
848 | return 0; |
||
3243 | Serge | 849 | } |
850 | |||
851 | struct scatterlist *sg_next(struct scatterlist *sg) |
||
852 | { |
||
853 | if (sg_is_last(sg)) |
||
854 | return NULL; |
||
855 | |||
856 | sg++; |
||
857 | if (unlikely(sg_is_chain(sg))) |
||
858 | sg = sg_chain_ptr(sg); |
||
859 | |||
860 | return sg; |
||
861 | } |
||
862 | |||
863 | |||
864 | void __sg_free_table(struct sg_table *table, unsigned int max_ents, |
||
865 | sg_free_fn *free_fn) |
||
866 | { |
||
867 | struct scatterlist *sgl, *next; |
||
868 | |||
869 | if (unlikely(!table->sgl)) |
||
870 | return; |
||
871 | |||
872 | sgl = table->sgl; |
||
873 | while (table->orig_nents) { |
||
874 | unsigned int alloc_size = table->orig_nents; |
||
875 | unsigned int sg_size; |
||
876 | |||
877 | /* |
||
878 | * If we have more than max_ents segments left, |
||
879 | * then assign 'next' to the sg table after the current one. |
||
880 | * sg_size is then one less than alloc size, since the last |
||
881 | * element is the chain pointer. |
||
882 | */ |
||
883 | if (alloc_size > max_ents) { |
||
884 | next = sg_chain_ptr(&sgl[max_ents - 1]); |
||
885 | alloc_size = max_ents; |
||
886 | sg_size = alloc_size - 1; |
||
887 | } else { |
||
888 | sg_size = alloc_size; |
||
889 | next = NULL; |
||
890 | } |
||
891 | |||
892 | table->orig_nents -= sg_size; |
||
893 | kfree(sgl); |
||
894 | sgl = next; |
||
895 | } |
||
896 | |||
897 | table->sgl = NULL; |
||
898 | } |
||
899 | |||
900 | void sg_free_table(struct sg_table *table) |
||
901 | { |
||
902 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
||
903 | } |
||
904 | |||
905 | int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) |
||
906 | { |
||
907 | struct scatterlist *sg, *prv; |
||
908 | unsigned int left; |
||
909 | unsigned int max_ents = SG_MAX_SINGLE_ALLOC; |
||
910 | |||
911 | #ifndef ARCH_HAS_SG_CHAIN |
||
912 | BUG_ON(nents > max_ents); |
||
913 | #endif |
||
914 | |||
915 | memset(table, 0, sizeof(*table)); |
||
916 | |||
917 | left = nents; |
||
918 | prv = NULL; |
||
919 | do { |
||
920 | unsigned int sg_size, alloc_size = left; |
||
921 | |||
922 | if (alloc_size > max_ents) { |
||
923 | alloc_size = max_ents; |
||
924 | sg_size = alloc_size - 1; |
||
925 | } else |
||
926 | sg_size = alloc_size; |
||
927 | |||
928 | left -= sg_size; |
||
929 | |||
930 | sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask); |
||
931 | if (unlikely(!sg)) { |
||
932 | /* |
||
933 | * Adjust entry count to reflect that the last |
||
934 | * entry of the previous table won't be used for |
||
935 | * linkage. Without this, sg_kfree() may get |
||
936 | * confused. |
||
937 | */ |
||
938 | if (prv) |
||
939 | table->nents = ++table->orig_nents; |
||
940 | |||
941 | goto err; |
||
942 | } |
||
943 | |||
944 | sg_init_table(sg, alloc_size); |
||
945 | table->nents = table->orig_nents += sg_size; |
||
946 | |||
947 | /* |
||
948 | * If this is the first mapping, assign the sg table header. |
||
949 | * If this is not the first mapping, chain previous part. |
||
950 | */ |
||
951 | if (prv) |
||
952 | sg_chain(prv, max_ents, sg); |
||
953 | else |
||
954 | table->sgl = sg; |
||
955 | |||
956 | /* |
||
957 | * If no more entries after this one, mark the end |
||
958 | */ |
||
959 | if (!left) |
||
960 | sg_mark_end(&sg[sg_size - 1]); |
||
961 | |||
962 | prv = sg; |
||
963 | } while (left); |
||
964 | |||
965 | return 0; |
||
966 | |||
967 | err: |
||
968 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, NULL); |
||
969 | |||
970 | return -ENOMEM; |
||
971 | } |
||
972 | |||
973 | |||
974 | void sg_init_table(struct scatterlist *sgl, unsigned int nents) |
||
975 | { |
||
976 | memset(sgl, 0, sizeof(*sgl) * nents); |
||
977 | #ifdef CONFIG_DEBUG_SG |
||
978 | { |
||
979 | unsigned int i; |
||
980 | for (i = 0; i < nents; i++) |
||
981 | sgl[i].sg_magic = SG_MAGIC; |
||
982 | } |
||
983 | #endif |
||
984 | sg_mark_end(&sgl[nents - 1]); |
||
985 | } |
||
986 | |||
3746 | Serge | 987 | |
988 | void __sg_page_iter_start(struct sg_page_iter *piter, |
||
989 | struct scatterlist *sglist, unsigned int nents, |
||
990 | unsigned long pgoffset) |
||
991 | { |
||
992 | piter->__pg_advance = 0; |
||
993 | piter->__nents = nents; |
||
994 | |||
995 | piter->sg = sglist; |
||
996 | piter->sg_pgoffset = pgoffset; |
||
997 | } |
||
998 | |||
999 | static int sg_page_count(struct scatterlist *sg) |
||
1000 | { |
||
1001 | return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT; |
||
1002 | } |
||
1003 | |||
1004 | bool __sg_page_iter_next(struct sg_page_iter *piter) |
||
1005 | { |
||
1006 | if (!piter->__nents || !piter->sg) |
||
1007 | return false; |
||
1008 | |||
1009 | piter->sg_pgoffset += piter->__pg_advance; |
||
1010 | piter->__pg_advance = 1; |
||
1011 | |||
1012 | while (piter->sg_pgoffset >= sg_page_count(piter->sg)) { |
||
1013 | piter->sg_pgoffset -= sg_page_count(piter->sg); |
||
1014 | piter->sg = sg_next(piter->sg); |
||
1015 | if (!--piter->__nents || !piter->sg) |
||
1016 | return false; |
||
1017 | } |
||
1018 | |||
1019 | return true; |
||
1020 | } |
||
1021 | EXPORT_SYMBOL(__sg_page_iter_next);>=>><>20))))><20))))>20)><20)>>><>><>=>>>>>>>>>=><=>>><>><>><>><>><>><> |
||
1022 |