Rev 5354 | Rev 6660 | Go to most recent revision | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 5354 | Rev 6084 | ||
---|---|---|---|
Line 25... | Line 25... | ||
25 | 25 | ||
26 | #include |
26 | #include |
27 | #include |
27 | #include |
28 | #include |
28 | #include |
- | 29 | #include "i915_drv.h" |
|
29 | #include "i915_drv.h" |
30 | #include "i915_vgpu.h" |
30 | #include "i915_trace.h" |
31 | #include "i915_trace.h" |
Line -... | Line 32... | ||
- | 32 | #include "intel_drv.h" |
|
- | 33 | ||
- | 34 | /** |
|
- | 35 | * DOC: Global GTT views |
|
- | 36 | * |
|
- | 37 | * Background and previous state |
|
- | 38 | * |
|
- | 39 | * Historically objects could exists (be bound) in global GTT space only as |
|
- | 40 | * singular instances with a view representing all of the object's backing pages |
|
- | 41 | * in a linear fashion. This view will be called a normal view. |
|
- | 42 | * |
|
- | 43 | * To support multiple views of the same object, where the number of mapped |
|
- | 44 | * pages is not equal to the backing store, or where the layout of the pages |
|
- | 45 | * is not linear, concept of a GGTT view was added. |
|
- | 46 | * |
|
- | 47 | * One example of an alternative view is a stereo display driven by a single |
|
- | 48 | * image. In this case we would have a framebuffer looking like this |
|
- | 49 | * (2x2 pages): |
|
- | 50 | * |
|
- | 51 | * 12 |
|
- | 52 | * 34 |
|
- | 53 | * |
|
- | 54 | * Above would represent a normal GGTT view as normally mapped for GPU or CPU |
|
- | 55 | * rendering. In contrast, fed to the display engine would be an alternative |
|
- | 56 | * view which could look something like this: |
|
- | 57 | * |
|
- | 58 | * 1212 |
|
- | 59 | * 3434 |
|
- | 60 | * |
|
- | 61 | * In this example both the size and layout of pages in the alternative view is |
|
31 | #include "intel_drv.h" |
62 | * different from the normal view. |
- | 63 | * |
|
- | 64 | * Implementation and usage |
|
- | 65 | * |
|
- | 66 | * GGTT views are implemented using VMAs and are distinguished via enum |
|
- | 67 | * i915_ggtt_view_type and struct i915_ggtt_view. |
|
- | 68 | * |
|
- | 69 | * A new flavour of core GEM functions which work with GGTT bound objects were |
|
- | 70 | * added with the _ggtt_ infix, and sometimes with _view postfix to avoid |
|
- | 71 | * renaming in large amounts of code. They take the struct i915_ggtt_view |
|
- | 72 | * parameter encapsulating all metadata required to implement a view. |
|
- | 73 | * |
|
- | 74 | * As a helper for callers which are only interested in the normal view, |
|
- | 75 | * globally const i915_ggtt_view_normal singleton instance exists. All old core |
|
- | 76 | * GEM API functions, the ones not taking the view parameter, are operating on, |
|
- | 77 | * or with the normal GGTT view. |
|
- | 78 | * |
|
- | 79 | * Code wanting to add or use a new GGTT view needs to: |
|
- | 80 | * |
|
- | 81 | * 1. Add a new enum with a suitable name. |
|
- | 82 | * 2. Extend the metadata in the i915_ggtt_view structure if required. |
|
- | 83 | * 3. Add support to i915_get_vma_pages(). |
|
- | 84 | * |
|
- | 85 | * New views are required to build a scatter-gather table from within the |
|
- | 86 | * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and |
|
- | 87 | * exists for the lifetime of an VMA. |
|
- | 88 | * |
|
- | 89 | * Core API is designed to have copy semantics which means that passed in |
|
- | 90 | * struct i915_ggtt_view does not need to be persistent (left around after |
|
- | 91 | * calling the core API functions). |
|
Line -... | Line 92... | ||
- | 92 | * |
|
32 | 93 | */ |
|
- | 94 | ||
33 | #include |
95 | static int |
- | 96 | i915_get_ggtt_vma_pages(struct i915_vma *vma); |
|
- | 97 | ||
- | 98 | const struct i915_ggtt_view i915_ggtt_view_normal; |
|
Line 34... | Line 99... | ||
34 | 99 | const struct i915_ggtt_view i915_ggtt_view_rotated = { |
|
35 | static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv); |
100 | .type = I915_GGTT_VIEW_ROTATED |
36 | static void chv_setup_private_ppat(struct drm_i915_private *dev_priv); |
101 | }; |
37 | 102 | ||
Line 38... | Line 103... | ||
38 | static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt) |
103 | static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt) |
39 | { |
104 | { |
- | 105 | bool has_aliasing_ppgtt; |
|
40 | bool has_aliasing_ppgtt; |
106 | bool has_full_ppgtt; |
41 | bool has_full_ppgtt; |
107 | |
Line 42... | Line 108... | ||
42 | 108 | has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6; |
|
43 | has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6; |
109 | has_full_ppgtt = INTEL_INFO(dev)->gen >= 7; |
44 | has_full_ppgtt = INTEL_INFO(dev)->gen >= 7; |
110 | |
45 | if (IS_GEN8(dev)) |
111 | if (intel_vgpu_active(dev)) |
Line 72... | Line 138... | ||
72 | dev->pdev->revision < 0xb) { |
138 | dev->pdev->revision < 0xb) { |
73 | DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n"); |
139 | DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n"); |
74 | return 0; |
140 | return 0; |
75 | } |
141 | } |
Line -... | Line 142... | ||
- | 142 | ||
- | 143 | if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists) |
|
- | 144 | return 2; |
|
76 | 145 | else |
|
77 | return has_aliasing_ppgtt ? 1 : 0; |
146 | return has_aliasing_ppgtt ? 1 : 0; |
Line 78... | Line -... | ||
78 | } |
- | |
79 | 147 | } |
|
80 | 148 | ||
81 | static void ppgtt_bind_vma(struct i915_vma *vma, |
149 | static int ppgtt_bind_vma(struct i915_vma *vma, |
- | 150 | enum i915_cache_level cache_level, |
|
- | 151 | u32 unused) |
|
- | 152 | { |
|
82 | enum i915_cache_level cache_level, |
153 | u32 pte_flags = 0; |
- | 154 | ||
- | 155 | /* Currently applicable only to VLV */ |
|
Line -... | Line 156... | ||
- | 156 | if (vma->obj->gt_ro) |
|
- | 157 | pte_flags |= PTE_READ_ONLY; |
|
- | 158 | ||
- | 159 | vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start, |
|
- | 160 | cache_level, pte_flags); |
|
- | 161 | ||
- | 162 | return 0; |
|
- | 163 | } |
|
- | 164 | ||
- | 165 | static void ppgtt_unbind_vma(struct i915_vma *vma) |
|
- | 166 | { |
|
- | 167 | vma->vm->clear_range(vma->vm, |
|
- | 168 | vma->node.start, |
|
- | 169 | vma->obj->base.size, |
|
83 | u32 flags); |
170 | true); |
84 | static void ppgtt_unbind_vma(struct i915_vma *vma); |
171 | } |
85 | 172 | ||
86 | static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr, |
173 | static gen8_pte_t gen8_pte_encode(dma_addr_t addr, |
87 | enum i915_cache_level level, |
174 | enum i915_cache_level level, |
88 | bool valid) |
175 | bool valid) |
Line 89... | Line 176... | ||
89 | { |
176 | { |
90 | gen8_gtt_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0; |
177 | gen8_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0; |
91 | pte |= addr; |
178 | pte |= addr; |
Line 103... | Line 190... | ||
103 | } |
190 | } |
Line 104... | Line 191... | ||
104 | 191 | ||
105 | return pte; |
192 | return pte; |
Line 106... | Line 193... | ||
106 | } |
193 | } |
107 | - | ||
108 | static inline gen8_ppgtt_pde_t gen8_pde_encode(struct drm_device *dev, |
194 | |
109 | dma_addr_t addr, |
195 | static gen8_pde_t gen8_pde_encode(const dma_addr_t addr, |
110 | enum i915_cache_level level) |
196 | const enum i915_cache_level level) |
111 | { |
197 | { |
112 | gen8_ppgtt_pde_t pde = _PAGE_PRESENT | _PAGE_RW; |
198 | gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW; |
113 | pde |= addr; |
199 | pde |= addr; |
114 | if (level != I915_CACHE_NONE) |
200 | if (level != I915_CACHE_NONE) |
115 | pde |= PPAT_CACHED_PDE_INDEX; |
201 | pde |= PPAT_CACHED_PDE_INDEX; |
116 | else |
202 | else |
117 | pde |= PPAT_UNCACHED_INDEX; |
203 | pde |= PPAT_UNCACHED_INDEX; |
Line -... | Line 204... | ||
- | 204 | return pde; |
|
- | 205 | } |
|
- | 206 | ||
118 | return pde; |
207 | #define gen8_pdpe_encode gen8_pde_encode |
119 | } |
208 | #define gen8_pml4e_encode gen8_pde_encode |
120 | 209 | ||
121 | static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr, |
210 | static gen6_pte_t snb_pte_encode(dma_addr_t addr, |
122 | enum i915_cache_level level, |
211 | enum i915_cache_level level, |
123 | bool valid, u32 unused) |
212 | bool valid, u32 unused) |
Line 124... | Line 213... | ||
124 | { |
213 | { |
125 | gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0; |
214 | gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; |
126 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
215 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
Line 132... | Line 221... | ||
132 | break; |
221 | break; |
133 | case I915_CACHE_NONE: |
222 | case I915_CACHE_NONE: |
134 | pte |= GEN6_PTE_UNCACHED; |
223 | pte |= GEN6_PTE_UNCACHED; |
135 | break; |
224 | break; |
136 | default: |
225 | default: |
137 | WARN_ON(1); |
226 | MISSING_CASE(level); |
138 | } |
227 | } |
Line 139... | Line 228... | ||
139 | 228 | ||
140 | return pte; |
229 | return pte; |
Line 141... | Line 230... | ||
141 | } |
230 | } |
142 | 231 | ||
143 | static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr, |
232 | static gen6_pte_t ivb_pte_encode(dma_addr_t addr, |
144 | enum i915_cache_level level, |
233 | enum i915_cache_level level, |
145 | bool valid, u32 unused) |
234 | bool valid, u32 unused) |
146 | { |
235 | { |
Line 147... | Line 236... | ||
147 | gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0; |
236 | gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; |
148 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
237 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
149 | 238 | ||
Line 156... | Line 245... | ||
156 | break; |
245 | break; |
157 | case I915_CACHE_NONE: |
246 | case I915_CACHE_NONE: |
158 | pte |= GEN6_PTE_UNCACHED; |
247 | pte |= GEN6_PTE_UNCACHED; |
159 | break; |
248 | break; |
160 | default: |
249 | default: |
161 | WARN_ON(1); |
250 | MISSING_CASE(level); |
162 | } |
251 | } |
Line 163... | Line 252... | ||
163 | 252 | ||
164 | return pte; |
253 | return pte; |
Line 165... | Line 254... | ||
165 | } |
254 | } |
166 | 255 | ||
167 | static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr, |
256 | static gen6_pte_t byt_pte_encode(dma_addr_t addr, |
168 | enum i915_cache_level level, |
257 | enum i915_cache_level level, |
169 | bool valid, u32 flags) |
258 | bool valid, u32 flags) |
170 | { |
259 | { |
Line 171... | Line 260... | ||
171 | gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0; |
260 | gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; |
172 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
261 | pte |= GEN6_PTE_ADDR_ENCODE(addr); |
Line 178... | Line 267... | ||
178 | pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; |
267 | pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; |
Line 179... | Line 268... | ||
179 | 268 | ||
180 | return pte; |
269 | return pte; |
Line 181... | Line 270... | ||
181 | } |
270 | } |
182 | 271 | ||
183 | static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr, |
272 | static gen6_pte_t hsw_pte_encode(dma_addr_t addr, |
184 | enum i915_cache_level level, |
273 | enum i915_cache_level level, |
185 | bool valid, u32 unused) |
274 | bool valid, u32 unused) |
186 | { |
275 | { |
Line 187... | Line 276... | ||
187 | gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0; |
276 | gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0; |
188 | pte |= HSW_PTE_ADDR_ENCODE(addr); |
277 | pte |= HSW_PTE_ADDR_ENCODE(addr); |
Line 189... | Line 278... | ||
189 | 278 | ||
190 | if (level != I915_CACHE_NONE) |
279 | if (level != I915_CACHE_NONE) |
Line 191... | Line 280... | ||
191 | pte |= HSW_WB_LLC_AGE3; |
280 | pte |= HSW_WB_LLC_AGE3; |
192 | 281 | ||
193 | return pte; |
282 | return pte; |
194 | } |
283 | } |
195 | 284 | ||
196 | static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr, |
285 | static gen6_pte_t iris_pte_encode(dma_addr_t addr, |
Line 197... | Line 286... | ||
197 | enum i915_cache_level level, |
286 | enum i915_cache_level level, |
198 | bool valid, u32 unused) |
287 | bool valid, u32 unused) |
199 | { |
288 | { |
Line 212... | Line 301... | ||
212 | } |
301 | } |
Line 213... | Line 302... | ||
213 | 302 | ||
214 | return pte; |
303 | return pte; |
Line -... | Line 304... | ||
- | 304 | } |
|
- | 305 | ||
- | 306 | static int __setup_page_dma(struct drm_device *dev, |
|
- | 307 | struct i915_page_dma *p, gfp_t flags) |
|
- | 308 | { |
|
- | 309 | struct device *device = &dev->pdev->dev; |
|
- | 310 | ||
- | 311 | p->page = alloc_page(flags); |
|
- | 312 | if (!p->page) |
|
- | 313 | return -ENOMEM; |
|
- | 314 | ||
- | 315 | p->daddr = page_to_phys(p->page); |
|
- | 316 | ||
- | 317 | return 0; |
|
- | 318 | } |
|
- | 319 | ||
- | 320 | static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p) |
|
- | 321 | { |
|
- | 322 | return __setup_page_dma(dev, p, GFP_KERNEL); |
|
- | 323 | } |
|
- | 324 | ||
- | 325 | static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p) |
|
- | 326 | { |
|
- | 327 | if (WARN_ON(!p->page)) |
|
- | 328 | return; |
|
- | 329 | ||
- | 330 | __free_page(p->page); |
|
- | 331 | memset(p, 0, sizeof(*p)); |
|
- | 332 | } |
|
- | 333 | ||
- | 334 | static void *kmap_page_dma(struct i915_page_dma *p) |
|
- | 335 | { |
|
- | 336 | return kmap_atomic(p->page); |
|
- | 337 | } |
|
- | 338 | ||
- | 339 | /* We use the flushing unmap only with ppgtt structures: |
|
- | 340 | * page directories, page tables and scratch pages. |
|
- | 341 | */ |
|
- | 342 | static void kunmap_page_dma(struct drm_device *dev, void *vaddr) |
|
- | 343 | { |
|
- | 344 | /* There are only few exceptions for gen >=6. chv and bxt. |
|
- | 345 | * And we are not sure about the latter so play safe for now. |
|
- | 346 | */ |
|
- | 347 | if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) |
|
- | 348 | drm_clflush_virt_range(vaddr, PAGE_SIZE); |
|
- | 349 | ||
- | 350 | kunmap_atomic(vaddr); |
|
- | 351 | } |
|
- | 352 | ||
- | 353 | #define kmap_px(px) kmap_page_dma(px_base(px)) |
|
- | 354 | #define kunmap_px(ppgtt, vaddr) kunmap_page_dma((ppgtt)->base.dev, (vaddr)) |
|
- | 355 | ||
- | 356 | #define setup_px(dev, px) setup_page_dma((dev), px_base(px)) |
|
- | 357 | #define cleanup_px(dev, px) cleanup_page_dma((dev), px_base(px)) |
|
- | 358 | #define fill_px(dev, px, v) fill_page_dma((dev), px_base(px), (v)) |
|
- | 359 | #define fill32_px(dev, px, v) fill_page_dma_32((dev), px_base(px), (v)) |
|
- | 360 | ||
- | 361 | static void fill_page_dma(struct drm_device *dev, struct i915_page_dma *p, |
|
- | 362 | const uint64_t val) |
|
- | 363 | { |
|
- | 364 | int i; |
|
- | 365 | uint64_t * const vaddr = kmap_page_dma(p); |
|
- | 366 | ||
- | 367 | for (i = 0; i < 512; i++) |
|
- | 368 | vaddr[i] = val; |
|
- | 369 | ||
- | 370 | kunmap_page_dma(dev, vaddr); |
|
- | 371 | } |
|
- | 372 | ||
- | 373 | static void fill_page_dma_32(struct drm_device *dev, struct i915_page_dma *p, |
|
- | 374 | const uint32_t val32) |
|
- | 375 | { |
|
- | 376 | uint64_t v = val32; |
|
- | 377 | ||
- | 378 | v = v << 32 | val32; |
|
- | 379 | ||
- | 380 | fill_page_dma(dev, p, v); |
|
- | 381 | } |
|
- | 382 | ||
- | 383 | static struct i915_page_scratch *alloc_scratch_page(struct drm_device *dev) |
|
- | 384 | { |
|
- | 385 | struct i915_page_scratch *sp; |
|
- | 386 | int ret; |
|
- | 387 | ||
- | 388 | sp = kzalloc(sizeof(*sp), GFP_KERNEL); |
|
- | 389 | if (sp == NULL) |
|
- | 390 | return ERR_PTR(-ENOMEM); |
|
- | 391 | ||
- | 392 | ret = __setup_page_dma(dev, px_base(sp), GFP_DMA32 | __GFP_ZERO); |
|
- | 393 | if (ret) { |
|
- | 394 | kfree(sp); |
|
- | 395 | return ERR_PTR(ret); |
|
- | 396 | } |
|
- | 397 | ||
- | 398 | // set_pages_uc(px_page(sp), 1); |
|
- | 399 | ||
- | 400 | return sp; |
|
- | 401 | } |
|
- | 402 | ||
- | 403 | static void free_scratch_page(struct drm_device *dev, |
|
- | 404 | struct i915_page_scratch *sp) |
|
- | 405 | { |
|
- | 406 | // set_pages_wb(px_page(sp), 1); |
|
- | 407 | ||
- | 408 | cleanup_px(dev, sp); |
|
- | 409 | kfree(sp); |
|
- | 410 | } |
|
- | 411 | ||
- | 412 | static struct i915_page_table *alloc_pt(struct drm_device *dev) |
|
- | 413 | { |
|
- | 414 | struct i915_page_table *pt; |
|
- | 415 | const size_t count = INTEL_INFO(dev)->gen >= 8 ? |
|
- | 416 | GEN8_PTES : GEN6_PTES; |
|
- | 417 | int ret = -ENOMEM; |
|
- | 418 | ||
- | 419 | pt = kzalloc(sizeof(*pt), GFP_KERNEL); |
|
- | 420 | if (!pt) |
|
- | 421 | return ERR_PTR(-ENOMEM); |
|
- | 422 | ||
- | 423 | pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes), |
|
- | 424 | GFP_KERNEL); |
|
- | 425 | ||
- | 426 | if (!pt->used_ptes) |
|
- | 427 | goto fail_bitmap; |
|
- | 428 | ||
- | 429 | ret = setup_px(dev, pt); |
|
- | 430 | if (ret) |
|
- | 431 | goto fail_page_m; |
|
- | 432 | ||
- | 433 | return pt; |
|
- | 434 | ||
- | 435 | fail_page_m: |
|
- | 436 | kfree(pt->used_ptes); |
|
- | 437 | fail_bitmap: |
|
- | 438 | kfree(pt); |
|
- | 439 | ||
- | 440 | return ERR_PTR(ret); |
|
- | 441 | } |
|
- | 442 | ||
- | 443 | static void free_pt(struct drm_device *dev, struct i915_page_table *pt) |
|
- | 444 | { |
|
- | 445 | cleanup_px(dev, pt); |
|
- | 446 | kfree(pt->used_ptes); |
|
- | 447 | kfree(pt); |
|
- | 448 | } |
|
- | 449 | ||
- | 450 | static void gen8_initialize_pt(struct i915_address_space *vm, |
|
- | 451 | struct i915_page_table *pt) |
|
- | 452 | { |
|
- | 453 | gen8_pte_t scratch_pte; |
|
- | 454 | ||
- | 455 | scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), |
|
- | 456 | I915_CACHE_LLC, true); |
|
- | 457 | ||
- | 458 | fill_px(vm->dev, pt, scratch_pte); |
|
- | 459 | } |
|
- | 460 | ||
- | 461 | static void gen6_initialize_pt(struct i915_address_space *vm, |
|
- | 462 | struct i915_page_table *pt) |
|
- | 463 | { |
|
- | 464 | gen6_pte_t scratch_pte; |
|
- | 465 | ||
- | 466 | WARN_ON(px_dma(vm->scratch_page) == 0); |
|
- | 467 | ||
- | 468 | scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), |
|
- | 469 | I915_CACHE_LLC, true, 0); |
|
- | 470 | ||
- | 471 | fill32_px(vm->dev, pt, scratch_pte); |
|
- | 472 | } |
|
- | 473 | ||
- | 474 | static struct i915_page_directory *alloc_pd(struct drm_device *dev) |
|
- | 475 | { |
|
- | 476 | struct i915_page_directory *pd; |
|
- | 477 | int ret = -ENOMEM; |
|
- | 478 | ||
- | 479 | pd = kzalloc(sizeof(*pd), GFP_KERNEL); |
|
- | 480 | if (!pd) |
|
- | 481 | return ERR_PTR(-ENOMEM); |
|
- | 482 | ||
- | 483 | pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES), |
|
- | 484 | sizeof(*pd->used_pdes), GFP_KERNEL); |
|
- | 485 | if (!pd->used_pdes) |
|
- | 486 | goto fail_bitmap; |
|
- | 487 | ||
- | 488 | ret = setup_px(dev, pd); |
|
- | 489 | if (ret) |
|
- | 490 | goto fail_page_m; |
|
- | 491 | ||
- | 492 | return pd; |
|
- | 493 | ||
- | 494 | fail_page_m: |
|
- | 495 | kfree(pd->used_pdes); |
|
- | 496 | fail_bitmap: |
|
- | 497 | kfree(pd); |
|
- | 498 | ||
- | 499 | return ERR_PTR(ret); |
|
- | 500 | } |
|
- | 501 | ||
- | 502 | static void free_pd(struct drm_device *dev, struct i915_page_directory *pd) |
|
- | 503 | { |
|
- | 504 | if (px_page(pd)) { |
|
- | 505 | cleanup_px(dev, pd); |
|
- | 506 | kfree(pd->used_pdes); |
|
- | 507 | kfree(pd); |
|
- | 508 | } |
|
- | 509 | } |
|
- | 510 | ||
- | 511 | static void gen8_initialize_pd(struct i915_address_space *vm, |
|
- | 512 | struct i915_page_directory *pd) |
|
- | 513 | { |
|
- | 514 | gen8_pde_t scratch_pde; |
|
- | 515 | ||
- | 516 | scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC); |
|
- | 517 | ||
- | 518 | fill_px(vm->dev, pd, scratch_pde); |
|
- | 519 | } |
|
- | 520 | ||
- | 521 | static int __pdp_init(struct drm_device *dev, |
|
- | 522 | struct i915_page_directory_pointer *pdp) |
|
- | 523 | { |
|
- | 524 | size_t pdpes = I915_PDPES_PER_PDP(dev); |
|
- | 525 | ||
- | 526 | pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes), |
|
- | 527 | sizeof(unsigned long), |
|
- | 528 | GFP_KERNEL); |
|
- | 529 | if (!pdp->used_pdpes) |
|
- | 530 | return -ENOMEM; |
|
- | 531 | ||
- | 532 | pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory), |
|
- | 533 | GFP_KERNEL); |
|
- | 534 | if (!pdp->page_directory) { |
|
- | 535 | kfree(pdp->used_pdpes); |
|
- | 536 | /* the PDP might be the statically allocated top level. Keep it |
|
- | 537 | * as clean as possible */ |
|
- | 538 | pdp->used_pdpes = NULL; |
|
- | 539 | return -ENOMEM; |
|
- | 540 | } |
|
- | 541 | ||
- | 542 | return 0; |
|
- | 543 | } |
|
- | 544 | ||
- | 545 | static void __pdp_fini(struct i915_page_directory_pointer *pdp) |
|
- | 546 | { |
|
- | 547 | kfree(pdp->used_pdpes); |
|
- | 548 | kfree(pdp->page_directory); |
|
- | 549 | pdp->page_directory = NULL; |
|
- | 550 | } |
|
- | 551 | ||
- | 552 | static struct |
|
- | 553 | i915_page_directory_pointer *alloc_pdp(struct drm_device *dev) |
|
- | 554 | { |
|
- | 555 | struct i915_page_directory_pointer *pdp; |
|
- | 556 | int ret = -ENOMEM; |
|
- | 557 | ||
- | 558 | WARN_ON(!USES_FULL_48BIT_PPGTT(dev)); |
|
- | 559 | ||
- | 560 | pdp = kzalloc(sizeof(*pdp), GFP_KERNEL); |
|
- | 561 | if (!pdp) |
|
- | 562 | return ERR_PTR(-ENOMEM); |
|
- | 563 | ||
- | 564 | ret = __pdp_init(dev, pdp); |
|
- | 565 | if (ret) |
|
- | 566 | goto fail_bitmap; |
|
- | 567 | ||
- | 568 | ret = setup_px(dev, pdp); |
|
- | 569 | if (ret) |
|
- | 570 | goto fail_page_m; |
|
- | 571 | ||
- | 572 | return pdp; |
|
- | 573 | ||
- | 574 | fail_page_m: |
|
- | 575 | __pdp_fini(pdp); |
|
- | 576 | fail_bitmap: |
|
- | 577 | kfree(pdp); |
|
- | 578 | ||
- | 579 | return ERR_PTR(ret); |
|
- | 580 | } |
|
- | 581 | ||
- | 582 | static void free_pdp(struct drm_device *dev, |
|
- | 583 | struct i915_page_directory_pointer *pdp) |
|
- | 584 | { |
|
- | 585 | __pdp_fini(pdp); |
|
- | 586 | if (USES_FULL_48BIT_PPGTT(dev)) { |
|
- | 587 | cleanup_px(dev, pdp); |
|
- | 588 | kfree(pdp); |
|
- | 589 | } |
|
- | 590 | } |
|
- | 591 | ||
- | 592 | static void gen8_initialize_pdp(struct i915_address_space *vm, |
|
- | 593 | struct i915_page_directory_pointer *pdp) |
|
- | 594 | { |
|
- | 595 | gen8_ppgtt_pdpe_t scratch_pdpe; |
|
- | 596 | ||
- | 597 | scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC); |
|
- | 598 | ||
- | 599 | fill_px(vm->dev, pdp, scratch_pdpe); |
|
- | 600 | } |
|
- | 601 | ||
- | 602 | static void gen8_initialize_pml4(struct i915_address_space *vm, |
|
- | 603 | struct i915_pml4 *pml4) |
|
- | 604 | { |
|
- | 605 | gen8_ppgtt_pml4e_t scratch_pml4e; |
|
- | 606 | ||
- | 607 | scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp), |
|
- | 608 | I915_CACHE_LLC); |
|
- | 609 | ||
- | 610 | fill_px(vm->dev, pml4, scratch_pml4e); |
|
- | 611 | } |
|
- | 612 | ||
- | 613 | static void |
|
- | 614 | gen8_setup_page_directory(struct i915_hw_ppgtt *ppgtt, |
|
- | 615 | struct i915_page_directory_pointer *pdp, |
|
- | 616 | struct i915_page_directory *pd, |
|
- | 617 | int index) |
|
- | 618 | { |
|
- | 619 | gen8_ppgtt_pdpe_t *page_directorypo; |
|
- | 620 | ||
- | 621 | if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) |
|
- | 622 | return; |
|
- | 623 | ||
- | 624 | page_directorypo = kmap_px(pdp); |
|
- | 625 | page_directorypo[index] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC); |
|
- | 626 | kunmap_px(ppgtt, page_directorypo); |
|
- | 627 | } |
|
- | 628 | ||
- | 629 | static void |
|
- | 630 | gen8_setup_page_directory_pointer(struct i915_hw_ppgtt *ppgtt, |
|
- | 631 | struct i915_pml4 *pml4, |
|
- | 632 | struct i915_page_directory_pointer *pdp, |
|
- | 633 | int index) |
|
- | 634 | { |
|
- | 635 | gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4); |
|
- | 636 | ||
- | 637 | WARN_ON(!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)); |
|
- | 638 | pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC); |
|
- | 639 | kunmap_px(ppgtt, pagemap); |
|
215 | } |
640 | } |
216 | 641 | ||
- | 642 | /* Broadwell Page Directory Pointer Descriptors */ |
|
217 | /* Broadwell Page Directory Pointer Descriptors */ |
643 | static int gen8_write_pdp(struct drm_i915_gem_request *req, |
218 | static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry, |
644 | unsigned entry, |
- | 645 | dma_addr_t addr) |
|
219 | uint64_t val) |
646 | { |
Line 220... | Line 647... | ||
220 | { |
647 | struct intel_engine_cs *ring = req->ring; |
Line 221... | Line 648... | ||
221 | int ret; |
648 | int ret; |
222 | 649 | ||
223 | BUG_ON(entry >= 4); |
650 | BUG_ON(entry >= 4); |
Line 224... | Line 651... | ||
224 | 651 | ||
225 | ret = intel_ring_begin(ring, 6); |
652 | ret = intel_ring_begin(req, 6); |
226 | if (ret) |
653 | if (ret) |
227 | return ret; |
654 | return ret; |
228 | 655 | ||
229 | intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); |
656 | intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); |
230 | intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry)); |
657 | intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry)); |
Line 231... | Line 658... | ||
231 | intel_ring_emit(ring, (u32)(val >> 32)); |
658 | intel_ring_emit(ring, upper_32_bits(addr)); |
232 | intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); |
659 | intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); |
Line 233... | Line 660... | ||
233 | intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry)); |
660 | intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry)); |
234 | intel_ring_emit(ring, (u32)(val)); |
661 | intel_ring_emit(ring, lower_32_bits(addr)); |
235 | intel_ring_advance(ring); |
662 | intel_ring_advance(ring); |
236 | 663 | ||
Line 237... | Line 664... | ||
237 | return 0; |
664 | return 0; |
238 | } |
665 | } |
Line 239... | Line -... | ||
239 | - | ||
240 | static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt, |
- | |
241 | struct intel_engine_cs *ring) |
666 | |
242 | { |
667 | static int gen8_legacy_mm_switch(struct i915_hw_ppgtt *ppgtt, |
243 | int i, ret; |
668 | struct drm_i915_gem_request *req) |
244 | 669 | { |
|
Line 245... | Line 670... | ||
245 | /* bit of a hack to find the actual last used pd */ |
670 | int i, ret; |
246 | int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE; |
671 | |
Line -... | Line 672... | ||
- | 672 | for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) { |
|
- | 673 | const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); |
|
- | 674 | ||
- | 675 | ret = gen8_write_pdp(req, i, pd_daddr); |
|
- | 676 | if (ret) |
|
- | 677 | return ret; |
|
247 | 678 | } |
|
- | 679 | ||
248 | for (i = used_pd - 1; i >= 0; i--) { |
680 | return 0; |
249 | dma_addr_t addr = ppgtt->pd_dma_addr[i]; |
681 | } |
250 | ret = gen8_write_pdp(ring, i, addr); |
682 | |
251 | if (ret) |
683 | static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt, |
252 | return ret; |
684 | struct drm_i915_gem_request *req) |
253 | } |
685 | { |
254 | 686 | return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4)); |
|
255 | return 0; |
687 | } |
256 | } |
688 | |
257 | 689 | static void gen8_ppgtt_clear_pte_range(struct i915_address_space *vm, |
|
258 | static void gen8_ppgtt_clear_range(struct i915_address_space *vm, |
690 | struct i915_page_directory_pointer *pdp, |
259 | uint64_t start, |
691 | uint64_t start, |
Line 260... | Line 692... | ||
260 | uint64_t length, |
692 | uint64_t length, |
261 | bool use_scratch) |
693 | gen8_pte_t scratch_pte) |
Line 262... | Line 694... | ||
262 | { |
694 | { |
- | 695 | struct i915_hw_ppgtt *ppgtt = |
|
263 | struct i915_hw_ppgtt *ppgtt = |
696 | container_of(vm, struct i915_hw_ppgtt, base); |
- | 697 | gen8_pte_t *pt_vaddr; |
|
- | 698 | unsigned pdpe = gen8_pdpe_index(start); |
|
- | 699 | unsigned pde = gen8_pde_index(start); |
|
- | 700 | unsigned pte = gen8_pte_index(start); |
|
- | 701 | unsigned num_entries = length >> PAGE_SHIFT; |
|
- | 702 | unsigned last_pte, i; |
|
- | 703 | ||
- | 704 | if (WARN_ON(!pdp)) |
|
- | 705 | return; |
|
- | 706 | ||
- | 707 | while (num_entries) { |
|
- | 708 | struct i915_page_directory *pd; |
|
- | 709 | struct i915_page_table *pt; |
|
Line 264... | Line 710... | ||
264 | container_of(vm, struct i915_hw_ppgtt, base); |
710 | |
265 | gen8_gtt_pte_t *pt_vaddr, scratch_pte; |
711 | if (WARN_ON(!pdp->page_directory[pdpe])) |
266 | unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK; |
712 | break; |
Line 267... | Line 713... | ||
267 | unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK; |
713 | |
Line 268... | Line 714... | ||
268 | unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK; |
714 | pd = pdp->page_directory[pdpe]; |
269 | unsigned num_entries = length >> PAGE_SHIFT; |
715 | |
270 | unsigned last_pte, i; |
716 | if (WARN_ON(!pd->page_table[pde])) |
271 | 717 | break; |
|
Line 272... | Line -... | ||
272 | scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr, |
- | |
273 | I915_CACHE_LLC, use_scratch); |
- | |
274 | 718 | ||
Line 275... | Line 719... | ||
275 | while (num_entries) { |
719 | pt = pd->page_table[pde]; |
276 | struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde]; |
720 | |
- | 721 | if (WARN_ON(!px_page(pt))) |
|
277 | 722 | break; |
|
278 | last_pte = pte + num_entries; |
723 | |
279 | if (last_pte > GEN8_PTES_PER_PAGE) |
724 | last_pte = pte + num_entries; |
280 | last_pte = GEN8_PTES_PER_PAGE; |
725 | if (last_pte > GEN8_PTES) |
281 | 726 | last_pte = GEN8_PTES; |
|
Line 282... | Line 727... | ||
282 | pt_vaddr = kmap_atomic(page_table); |
727 | |
283 | - | ||
284 | for (i = pte; i < last_pte; i++) { |
728 | pt_vaddr = kmap_px(pt); |
285 | pt_vaddr[i] = scratch_pte; |
729 | |
- | 730 | for (i = pte; i < last_pte; i++) { |
|
286 | num_entries--; |
731 | pt_vaddr[i] = scratch_pte; |
287 | } |
732 | num_entries--; |
288 | 733 | } |
|
289 | if (!HAS_LLC(ppgtt->base.dev)) |
- | |
290 | drm_clflush_virt_range(pt_vaddr, PAGE_SIZE); |
- | |
291 | kunmap_atomic(pt_vaddr); |
- | |
292 | 734 | ||
293 | pte = 0; |
735 | kunmap_px(ppgtt, pt); |
Line -... | Line 736... | ||
- | 736 | ||
- | 737 | pte = 0; |
|
294 | if (++pde == GEN8_PDES_PER_PAGE) { |
738 | if (++pde == I915_PDES) { |
- | 739 | if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) |
|
- | 740 | break; |
|
- | 741 | pde = 0; |
|
Line 295... | Line 742... | ||
295 | pdpe++; |
742 | } |
296 | pde = 0; |
743 | } |
- | 744 | } |
|
297 | } |
745 | |
- | 746 | static void gen8_ppgtt_clear_range(struct i915_address_space *vm, |
|
- | 747 | uint64_t start, |
|
Line -... | Line 748... | ||
- | 748 | uint64_t length, |
|
- | 749 | bool use_scratch) |
|
- | 750 | { |
|
- | 751 | struct i915_hw_ppgtt *ppgtt = |
|
- | 752 | container_of(vm, struct i915_hw_ppgtt, base); |
|
- | 753 | gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), |
|
- | 754 | I915_CACHE_LLC, use_scratch); |
|
- | 755 | ||
- | 756 | if (!USES_FULL_48BIT_PPGTT(vm->dev)) { |
|
- | 757 | gen8_ppgtt_clear_pte_range(vm, &ppgtt->pdp, start, length, |
|
- | 758 | scratch_pte); |
|
- | 759 | } else { |
|
- | 760 | uint64_t templ4, pml4e; |
|
- | 761 | struct i915_page_directory_pointer *pdp; |
|
- | 762 | ||
- | 763 | gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) { |
|
- | 764 | gen8_ppgtt_clear_pte_range(vm, pdp, start, length, |
|
298 | } |
765 | scratch_pte); |
- | 766 | } |
|
299 | } |
767 | } |
- | 768 | } |
|
- | 769 | ||
Line 300... | Line 770... | ||
300 | 770 | static void |
|
301 | static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, |
771 | gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm, |
302 | struct sg_table *pages, |
772 | struct i915_page_directory_pointer *pdp, |
303 | uint64_t start, |
773 | struct sg_page_iter *sg_iter, |
304 | enum i915_cache_level cache_level, u32 unused) |
- | |
305 | { |
- | |
306 | struct i915_hw_ppgtt *ppgtt = |
774 | uint64_t start, |
307 | container_of(vm, struct i915_hw_ppgtt, base); |
775 | enum i915_cache_level cache_level) |
308 | gen8_gtt_pte_t *pt_vaddr; |
776 | { |
- | 777 | struct i915_hw_ppgtt *ppgtt = |
|
309 | unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK; |
778 | container_of(vm, struct i915_hw_ppgtt, base); |
310 | unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK; |
779 | gen8_pte_t *pt_vaddr; |
311 | unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK; |
780 | unsigned pdpe = gen8_pdpe_index(start); |
312 | struct sg_page_iter sg_iter; |
781 | unsigned pde = gen8_pde_index(start); |
313 | 782 | unsigned pte = gen8_pte_index(start); |
|
314 | pt_vaddr = NULL; |
783 | |
- | 784 | pt_vaddr = NULL; |
|
315 | 785 | ||
316 | for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { |
786 | while (__sg_page_iter_next(sg_iter)) { |
- | 787 | if (pt_vaddr == NULL) { |
|
- | 788 | struct i915_page_directory *pd = pdp->page_directory[pdpe]; |
|
- | 789 | struct i915_page_table *pt = pd->page_table[pde]; |
|
- | 790 | pt_vaddr = kmap_px(pt); |
|
- | 791 | } |
|
- | 792 | ||
- | 793 | pt_vaddr[pte] = |
|
- | 794 | gen8_pte_encode(sg_page_iter_dma_address(sg_iter), |
|
- | 795 | cache_level, true); |
|
- | 796 | if (++pte == GEN8_PTES) { |
|
- | 797 | kunmap_px(ppgtt, pt_vaddr); |
|
- | 798 | pt_vaddr = NULL; |
|
- | 799 | if (++pde == I915_PDES) { |
|
- | 800 | if (++pdpe == I915_PDPES_PER_PDP(vm->dev)) |
|
- | 801 | break; |
|
- | 802 | pde = 0; |
|
- | 803 | } |
|
- | 804 | pte = 0; |
|
317 | if (WARN_ON(pdpe >= GEN8_LEGACY_PDPS)) |
805 | } |
318 | break; |
806 | } |
- | 807 | ||
- | 808 | if (pt_vaddr) |
|
- | 809 | kunmap_px(ppgtt, pt_vaddr); |
|
- | 810 | } |
|
- | 811 | ||
- | 812 | static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, |
|
319 | 813 | struct sg_table *pages, |
|
320 | if (pt_vaddr == NULL) |
814 | uint64_t start, |
Line 321... | Line 815... | ||
321 | pt_vaddr = kmap_atomic(ppgtt->gen8_pt_pages[pdpe][pde]); |
815 | enum i915_cache_level cache_level, |
- | 816 | u32 unused) |
|
322 | 817 | { |
|
323 | pt_vaddr[pte] = |
818 | struct i915_hw_ppgtt *ppgtt = |
Line 324... | Line 819... | ||
324 | gen8_pte_encode(sg_page_iter_dma_address(&sg_iter), |
819 | container_of(vm, struct i915_hw_ppgtt, base); |
325 | cache_level, true); |
820 | struct sg_page_iter sg_iter; |
Line 326... | Line 821... | ||
326 | if (++pte == GEN8_PTES_PER_PAGE) { |
821 | |
327 | if (!HAS_LLC(ppgtt->base.dev)) |
822 | __sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0); |
- | 823 | ||
- | 824 | if (!USES_FULL_48BIT_PPGTT(vm->dev)) { |
|
328 | drm_clflush_virt_range(pt_vaddr, PAGE_SIZE); |
825 | gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start, |
- | 826 | cache_level); |
|
- | 827 | } else { |
|
329 | kunmap_atomic(pt_vaddr); |
828 | struct i915_page_directory_pointer *pdp; |
Line 330... | Line 829... | ||
330 | pt_vaddr = NULL; |
829 | uint64_t templ4, pml4e; |
331 | if (++pde == GEN8_PDES_PER_PAGE) { |
830 | uint64_t length = (uint64_t)pages->orig_nents << PAGE_SHIFT; |
- | 831 | ||
- | 832 | gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) { |
|
- | 833 | gen8_ppgtt_insert_pte_entries(vm, pdp, &sg_iter, |
|
- | 834 | start, cache_level); |
|
- | 835 | } |
|
- | 836 | } |
|
- | 837 | } |
|
- | 838 | ||
- | 839 | static void gen8_free_page_tables(struct drm_device *dev, |
|
- | 840 | struct i915_page_directory *pd) |
|
- | 841 | { |
|
- | 842 | int i; |
|
- | 843 | ||
- | 844 | if (!px_page(pd)) |
|
- | 845 | return; |
|
- | 846 | ||
- | 847 | for_each_set_bit(i, pd->used_pdes, I915_PDES) { |
|
- | 848 | if (WARN_ON(!pd->page_table[i])) |
|
- | 849 | continue; |
|
- | 850 | ||
- | 851 | free_pt(dev, pd->page_table[i]); |
|
- | 852 | pd->page_table[i] = NULL; |
|
- | 853 | } |
|
- | 854 | } |
|
- | 855 | ||
- | 856 | static int gen8_init_scratch(struct i915_address_space *vm) |
|
- | 857 | { |
|
- | 858 | struct drm_device *dev = vm->dev; |
|
- | 859 | ||
- | 860 | vm->scratch_page = alloc_scratch_page(dev); |
|
- | 861 | if (IS_ERR(vm->scratch_page)) |
|
- | 862 | return PTR_ERR(vm->scratch_page); |
|
- | 863 | ||
- | 864 | vm->scratch_pt = alloc_pt(dev); |
|
- | 865 | if (IS_ERR(vm->scratch_pt)) { |
|
- | 866 | free_scratch_page(dev, vm->scratch_page); |
|
- | 867 | return PTR_ERR(vm->scratch_pt); |
|
- | 868 | } |
|
- | 869 | ||
- | 870 | vm->scratch_pd = alloc_pd(dev); |
|
- | 871 | if (IS_ERR(vm->scratch_pd)) { |
|
- | 872 | free_pt(dev, vm->scratch_pt); |
|
- | 873 | free_scratch_page(dev, vm->scratch_page); |
|
332 | pdpe++; |
874 | return PTR_ERR(vm->scratch_pd); |
Line -... | Line 875... | ||
- | 875 | } |
|
- | 876 | ||
- | 877 | if (USES_FULL_48BIT_PPGTT(dev)) { |
|
- | 878 | vm->scratch_pdp = alloc_pdp(dev); |
|
- | 879 | if (IS_ERR(vm->scratch_pdp)) { |
|
- | 880 | free_pd(dev, vm->scratch_pd); |
|
- | 881 | free_pt(dev, vm->scratch_pt); |
|
- | 882 | free_scratch_page(dev, vm->scratch_page); |
|
- | 883 | return PTR_ERR(vm->scratch_pdp); |
|
333 | pde = 0; |
884 | } |
334 | } |
885 | } |
- | 886 | ||
335 | pte = 0; |
887 | gen8_initialize_pt(vm, vm->scratch_pt); |
- | 888 | gen8_initialize_pd(vm, vm->scratch_pd); |
|
- | 889 | if (USES_FULL_48BIT_PPGTT(dev)) |
|
- | 890 | gen8_initialize_pdp(vm, vm->scratch_pdp); |
|
- | 891 | ||
- | 892 | return 0; |
|
- | 893 | } |
|
- | 894 | ||
- | 895 | static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create) |
|
- | 896 | { |
|
336 | } |
897 | enum vgt_g2v_type msg; |
- | 898 | struct drm_device *dev = ppgtt->base.dev; |
|
- | 899 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
337 | } |
900 | unsigned int offset = vgtif_reg(pdp0_lo); |
Line -... | Line 901... | ||
- | 901 | int i; |
|
- | 902 | ||
- | 903 | if (USES_FULL_48BIT_PPGTT(dev)) { |
|
- | 904 | u64 daddr = px_dma(&ppgtt->pml4); |
|
- | 905 | ||
- | 906 | I915_WRITE(offset, lower_32_bits(daddr)); |
|
- | 907 | I915_WRITE(offset + 4, upper_32_bits(daddr)); |
|
- | 908 | ||
338 | if (pt_vaddr) { |
909 | msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE : |
339 | if (!HAS_LLC(ppgtt->base.dev)) |
910 | VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY); |
Line 340... | Line 911... | ||
340 | drm_clflush_virt_range(pt_vaddr, PAGE_SIZE); |
911 | } else { |
- | 912 | for (i = 0; i < GEN8_LEGACY_PDPES; i++) { |
|
341 | kunmap_atomic(pt_vaddr); |
913 | u64 daddr = i915_page_dir_dma_addr(ppgtt, i); |
342 | } |
- | |
343 | } |
914 | |
Line 344... | Line 915... | ||
344 | 915 | I915_WRITE(offset, lower_32_bits(daddr)); |
|
345 | static void gen8_free_page_tables(struct page **pt_pages) |
- | |
346 | { |
- | |
347 | int i; |
916 | I915_WRITE(offset + 4, upper_32_bits(daddr)); |
348 | 917 | ||
Line 349... | Line 918... | ||
349 | if (pt_pages == NULL) |
918 | offset += 8; |
350 | return; |
919 | } |
- | 920 | ||
Line 351... | Line -... | ||
351 | - | ||
352 | // for (i = 0; i < GEN8_PDES_PER_PAGE; i++) |
- | |
353 | // if (pt_pages[i]) |
921 | msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE : |
354 | // __free_pages(pt_pages[i], 0); |
- | |
355 | } |
- | |
356 | 922 | VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY); |
|
- | 923 | } |
|
- | 924 | ||
- | 925 | I915_WRITE(vgtif_reg(g2v_notify), msg); |
|
- | 926 | ||
- | 927 | return 0; |
|
- | 928 | } |
|
- | 929 | ||
- | 930 | static void gen8_free_scratch(struct i915_address_space *vm) |
|
- | 931 | { |
|
- | 932 | struct drm_device *dev = vm->dev; |
|
357 | static void gen8_ppgtt_free(const struct i915_hw_ppgtt *ppgtt) |
933 | |
- | 934 | if (USES_FULL_48BIT_PPGTT(dev)) |
|
- | 935 | free_pdp(dev, vm->scratch_pdp); |
|
358 | { |
936 | free_pd(dev, vm->scratch_pd); |
Line 359... | Line 937... | ||
359 | int i; |
937 | free_pt(dev, vm->scratch_pt); |
360 | 938 | free_scratch_page(dev, vm->scratch_page); |
|
361 | for (i = 0; i < ppgtt->num_pd_pages; i++) { |
939 | } |
362 | gen8_free_page_tables(ppgtt->gen8_pt_pages[i]); |
940 | |
Line -... | Line 941... | ||
- | 941 | static void gen8_ppgtt_cleanup_3lvl(struct drm_device *dev, |
|
363 | kfree(ppgtt->gen8_pt_pages[i]); |
942 | struct i915_page_directory_pointer *pdp) |
- | 943 | { |
|
- | 944 | int i; |
|
- | 945 | ||
- | 946 | for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev)) { |
|
364 | kfree(ppgtt->gen8_pt_dma_addr[i]); |
947 | if (WARN_ON(!pdp->page_directory[i])) |
- | 948 | continue; |
|
- | 949 | ||
365 | } |
950 | gen8_free_page_tables(dev, pdp->page_directory[i]); |
Line -... | Line 951... | ||
- | 951 | free_pd(dev, pdp->page_directory[i]); |
|
- | 952 | } |
|
- | 953 | ||
- | 954 | free_pdp(dev, pdp); |
|
- | 955 | } |
|
- | 956 | ||
- | 957 | static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt) |
|
- | 958 | { |
|
- | 959 | int i; |
|
- | 960 | ||
- | 961 | for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) { |
|
- | 962 | if (WARN_ON(!ppgtt->pml4.pdps[i])) |
|
- | 963 | continue; |
|
- | 964 | ||
- | 965 | gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, ppgtt->pml4.pdps[i]); |
|
- | 966 | } |
|
- | 967 | ||
- | 968 | cleanup_px(ppgtt->base.dev, &ppgtt->pml4); |
|
366 | 969 | } |
|
- | 970 | ||
- | 971 | static void gen8_ppgtt_cleanup(struct i915_address_space *vm) |
|
- | 972 | { |
|
- | 973 | struct i915_hw_ppgtt *ppgtt = |
|
367 | // __free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT)); |
974 | container_of(vm, struct i915_hw_ppgtt, base); |
- | 975 | ||
368 | } |
976 | if (intel_vgpu_active(vm->dev)) |
- | 977 | gen8_ppgtt_notify_vgt(ppgtt, false); |
|
369 | 978 | ||
- | 979 | if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) |
|
- | 980 | gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, &ppgtt->pdp); |
|
- | 981 | else |
|
- | 982 | gen8_ppgtt_cleanup_4lvl(ppgtt); |
|
- | 983 | ||
- | 984 | gen8_free_scratch(vm); |
|
- | 985 | } |
|
- | 986 | ||
Line 370... | Line 987... | ||
370 | static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) |
987 | /** |
371 | { |
988 | * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range. |
372 | struct pci_dev *hwdev = ppgtt->base.dev->pdev; |
989 | * @vm: Master vm structure. |
Line 373... | Line 990... | ||
373 | int i, j; |
990 | * @pd: Page directory for this address range. |
374 | 991 | * @start: Starting virtual address to begin allocations. |
|
375 | for (i = 0; i < ppgtt->num_pd_pages; i++) { |
992 | * @length: Size of the allocations. |
376 | /* TODO: In the future we'll support sparse mappings, so this |
993 | * @new_pts: Bitmap set by function with new allocations. Likely used by the |
377 | * will have to change. */ |
994 | * caller to free on error. |
Line 378... | Line 995... | ||
378 | if (!ppgtt->pd_dma_addr[i]) |
995 | * |
Line 379... | Line 996... | ||
379 | continue; |
996 | * Allocate the required number of page tables. Extremely similar to |
380 | 997 | * gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by |
|
381 | pci_unmap_page(hwdev, ppgtt->pd_dma_addr[i], PAGE_SIZE, |
998 | * the page directory boundary (instead of the page directory pointer). That |
- | 999 | * boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is |
|
382 | PCI_DMA_BIDIRECTIONAL); |
1000 | * possible, and likely that the caller will need to use multiple calls of this |
383 | 1001 | * function to achieve the appropriate allocation. |
|
Line -... | Line 1002... | ||
- | 1002 | * |
|
- | 1003 | * Return: 0 if success; negative error code otherwise. |
|
- | 1004 | */ |
|
- | 1005 | static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm, |
|
- | 1006 | struct i915_page_directory *pd, |
|
- | 1007 | uint64_t start, |
|
- | 1008 | uint64_t length, |
|
- | 1009 | unsigned long *new_pts) |
|
- | 1010 | { |
|
- | 1011 | struct drm_device *dev = vm->dev; |
|
- | 1012 | struct i915_page_table *pt; |
|
- | 1013 | uint64_t temp; |
|
- | 1014 | uint32_t pde; |
|
- | 1015 | ||
- | 1016 | gen8_for_each_pde(pt, pd, start, length, temp, pde) { |
|
- | 1017 | /* Don't reallocate page tables */ |
|
- | 1018 | if (test_bit(pde, pd->used_pdes)) { |
|
- | 1019 | /* Scratch is never allocated this way */ |
|
- | 1020 | WARN_ON(pt == vm->scratch_pt); |
|
- | 1021 | continue; |
|
- | 1022 | } |
|
- | 1023 | ||
- | 1024 | pt = alloc_pt(dev); |
|
- | 1025 | if (IS_ERR(pt)) |
|
384 | for (j = 0; j < GEN8_PDES_PER_PAGE; j++) { |
1026 | goto unwind_out; |
- | 1027 | ||
- | 1028 | gen8_initialize_pt(vm, pt); |
|
- | 1029 | pd->page_table[pde] = pt; |
|
385 | dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j]; |
1030 | __set_bit(pde, new_pts); |
386 | if (addr) |
1031 | trace_i915_page_table_entry_alloc(vm, pde, start, GEN8_PDE_SHIFT); |
- | 1032 | } |
|
387 | pci_unmap_page(hwdev, addr, PAGE_SIZE, |
1033 | |
388 | PCI_DMA_BIDIRECTIONAL); |
1034 | return 0; |
- | 1035 | ||
- | 1036 | unwind_out: |
|
- | 1037 | for_each_set_bit(pde, new_pts, I915_PDES) |
|
- | 1038 | free_pt(dev, pd->page_table[pde]); |
|
Line 389... | Line 1039... | ||
389 | } |
1039 | |
390 | } |
1040 | return -ENOMEM; |
- | 1041 | } |
|
- | 1042 | ||
391 | } |
1043 | /** |
392 | 1044 | * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range. |
|
393 | static void gen8_ppgtt_cleanup(struct i915_address_space *vm) |
1045 | * @vm: Master vm structure. |
394 | { |
- | |
395 | struct i915_hw_ppgtt *ppgtt = |
- | |
Line 396... | Line 1046... | ||
396 | container_of(vm, struct i915_hw_ppgtt, base); |
1046 | * @pdp: Page directory pointer for this address range. |
397 | 1047 | * @start: Starting virtual address to begin allocations. |
|
398 | gen8_ppgtt_unmap_pages(ppgtt); |
- | |
399 | gen8_ppgtt_free(ppgtt); |
1048 | * @length: Size of the allocations. |
400 | } |
1049 | * @new_pds: Bitmap set by function with new allocations. Likely used by the |
- | 1050 | * caller to free on error. |
|
Line 401... | Line 1051... | ||
401 | 1051 | * |
|
Line 402... | Line 1052... | ||
402 | static struct page **__gen8_alloc_page_tables(void) |
1052 | * Allocate the required number of page directories starting at the pde index of |
403 | { |
- | |
404 | struct page **pt_pages; |
1053 | * @start, and ending at the pde index @start + @length. This function will skip |
405 | int i; |
1054 | * over already allocated page directories within the range, and only allocate |
406 | - | ||
Line 407... | Line 1055... | ||
407 | pt_pages = kcalloc(GEN8_PDES_PER_PAGE, sizeof(struct page *), GFP_KERNEL); |
1055 | * new ones, setting the appropriate pointer within the pdp as well as the |
408 | if (!pt_pages) |
1056 | * correct position in the bitmap @new_pds. |
Line -... | Line 1057... | ||
- | 1057 | * |
|
- | 1058 | * The function will only allocate the pages within the range for a give page |
|
- | 1059 | * directory pointer. In other words, if @start + @length straddles a virtually |
|
- | 1060 | * addressed PDP boundary (512GB for 4k pages), there will be more allocations |
|
- | 1061 | * required by the caller, This is not currently possible, and the BUG in the |
|
- | 1062 | * code will prevent it. |
|
- | 1063 | * |
|
- | 1064 | * Return: 0 if success; negative error code otherwise. |
|
- | 1065 | */ |
|
- | 1066 | static int |
|
- | 1067 | gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm, |
|
- | 1068 | struct i915_page_directory_pointer *pdp, |
|
- | 1069 | uint64_t start, |
|
- | 1070 | uint64_t length, |
|
- | 1071 | unsigned long *new_pds) |
|
- | 1072 | { |
|
- | 1073 | struct drm_device *dev = vm->dev; |
|
409 | return ERR_PTR(-ENOMEM); |
1074 | struct i915_page_directory *pd; |
- | 1075 | uint64_t temp; |
|
- | 1076 | uint32_t pdpe; |
|
- | 1077 | uint32_t pdpes = I915_PDPES_PER_PDP(dev); |
|
- | 1078 | ||
410 | 1079 | WARN_ON(!bitmap_empty(new_pds, pdpes)); |
|
- | 1080 | ||
- | 1081 | gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { |
|
411 | for (i = 0; i < GEN8_PDES_PER_PAGE; i++) { |
1082 | if (test_bit(pdpe, pdp->used_pdpes)) |
- | 1083 | continue; |
|
- | 1084 | ||
- | 1085 | pd = alloc_pd(dev); |
|
- | 1086 | if (IS_ERR(pd)) |
|
- | 1087 | goto unwind_out; |
|
- | 1088 | ||
- | 1089 | gen8_initialize_pd(vm, pd); |
|
- | 1090 | pdp->page_directory[pdpe] = pd; |
|
- | 1091 | __set_bit(pdpe, new_pds); |
|
Line -... | Line 1092... | ||
- | 1092 | trace_i915_page_directory_entry_alloc(vm, pdpe, start, GEN8_PDPE_SHIFT); |
|
- | 1093 | } |
|
412 | pt_pages[i] = alloc_page(GFP_KERNEL); |
1094 | |
413 | if (!pt_pages[i]) |
1095 | return 0; |
414 | goto bail; |
1096 | |
415 | } |
1097 | unwind_out: |
416 | 1098 | for_each_set_bit(pdpe, new_pds, pdpes) |
|
417 | return pt_pages; |
1099 | free_pd(dev, pdp->page_directory[pdpe]); |
418 | 1100 | ||
Line 419... | Line 1101... | ||
419 | bail: |
1101 | return -ENOMEM; |
- | 1102 | } |
|
- | 1103 | ||
- | 1104 | /** |
|
- | 1105 | * gen8_ppgtt_alloc_page_dirpointers() - Allocate pdps for VA range. |
|
- | 1106 | * @vm: Master vm structure. |
|
- | 1107 | * @pml4: Page map level 4 for this address range. |
|
420 | gen8_free_page_tables(pt_pages); |
1108 | * @start: Starting virtual address to begin allocations. |
Line -... | Line 1109... | ||
- | 1109 | * @length: Size of the allocations. |
|
- | 1110 | * @new_pdps: Bitmap set by function with new allocations. Likely used by the |
|
- | 1111 | * caller to free on error. |
|
- | 1112 | * |
|
- | 1113 | * Allocate the required number of page directory pointers. Extremely similar to |
|
- | 1114 | * gen8_ppgtt_alloc_page_directories() and gen8_ppgtt_alloc_pagetabs(). |
|
- | 1115 | * The main difference is here we are limited by the pml4 boundary (instead of |
|
- | 1116 | * the page directory pointer). |
|
- | 1117 | * |
|
- | 1118 | * Return: 0 if success; negative error code otherwise. |
|
- | 1119 | */ |
|
421 | kfree(pt_pages); |
1120 | static int |
- | 1121 | gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm, |
|
422 | return ERR_PTR(-ENOMEM); |
1122 | struct i915_pml4 *pml4, |
423 | } |
1123 | uint64_t start, |
- | 1124 | uint64_t length, |
|
- | 1125 | unsigned long *new_pdps) |
|
- | 1126 | { |
|
424 | 1127 | struct drm_device *dev = vm->dev; |
|
425 | static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt, |
1128 | struct i915_page_directory_pointer *pdp; |
426 | const int max_pdp) |
1129 | uint64_t temp; |
Line 427... | Line 1130... | ||
427 | { |
1130 | uint32_t pml4e; |
428 | struct page **pt_pages[GEN8_LEGACY_PDPS]; |
1131 | |
- | 1132 | WARN_ON(!bitmap_empty(new_pdps, GEN8_PML4ES_PER_PML4)); |
|
- | 1133 | ||
- | 1134 | gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) { |
|
- | 1135 | if (!test_bit(pml4e, pml4->used_pml4es)) { |
|
- | 1136 | pdp = alloc_pdp(dev); |
|
Line 429... | Line 1137... | ||
429 | int i, ret; |
1137 | if (IS_ERR(pdp)) |
- | 1138 | goto unwind_out; |
|
- | 1139 | ||
- | 1140 | gen8_initialize_pdp(vm, pdp); |
|
- | 1141 | pml4->pdps[pml4e] = pdp; |
|
- | 1142 | __set_bit(pml4e, new_pdps); |
|
- | 1143 | trace_i915_page_directory_pointer_entry_alloc(vm, |
|
- | 1144 | pml4e, |
|
- | 1145 | start, |
|
- | 1146 | GEN8_PML4E_SHIFT); |
|
- | 1147 | } |
|
- | 1148 | } |
|
- | 1149 | ||
- | 1150 | return 0; |
|
- | 1151 | ||
430 | 1152 | unwind_out: |
|
Line 431... | Line 1153... | ||
431 | for (i = 0; i < max_pdp; i++) { |
1153 | for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) |
- | 1154 | free_pdp(dev, pml4->pdps[pml4e]); |
|
- | 1155 | ||
432 | pt_pages[i] = __gen8_alloc_page_tables(); |
1156 | return -ENOMEM; |
433 | if (IS_ERR(pt_pages[i])) { |
1157 | } |
- | 1158 | ||
- | 1159 | static void |
|
- | 1160 | free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long *new_pts) |
|
- | 1161 | { |
|
- | 1162 | kfree(new_pts); |
|
- | 1163 | kfree(new_pds); |
|
- | 1164 | } |
|
- | 1165 | ||
- | 1166 | /* Fills in the page directory bitmap, and the array of page tables bitmap. Both |
|
- | 1167 | * of these are based on the number of PDPEs in the system. |
|
434 | ret = PTR_ERR(pt_pages[i]); |
1168 | */ |
Line -... | Line 1169... | ||
- | 1169 | static |
|
- | 1170 | int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds, |
|
- | 1171 | unsigned long **new_pts, |
|
- | 1172 | uint32_t pdpes) |
|
- | 1173 | { |
|
- | 1174 | unsigned long *pds; |
|
- | 1175 | unsigned long *pts; |
|
- | 1176 | ||
- | 1177 | pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_TEMPORARY); |
|
435 | goto unwind_out; |
1178 | if (!pds) |
436 | } |
1179 | return -ENOMEM; |
437 | } |
1180 | |
Line -... | Line 1181... | ||
- | 1181 | pts = kcalloc(pdpes, BITS_TO_LONGS(I915_PDES) * sizeof(unsigned long), |
|
438 | 1182 | GFP_TEMPORARY); |
|
- | 1183 | if (!pts) |
|
439 | /* NB: Avoid touching gen8_pt_pages until last to keep the allocation, |
1184 | goto err_out; |
440 | * "atomic" - for cleanup purposes. |
1185 | |
441 | */ |
1186 | *new_pds = pds; |
442 | for (i = 0; i < max_pdp; i++) |
1187 | *new_pts = pts; |
Line -... | Line 1188... | ||
- | 1188 | ||
443 | ppgtt->gen8_pt_pages[i] = pt_pages[i]; |
1189 | return 0; |
444 | - | ||
445 | return 0; |
1190 | |
- | 1191 | err_out: |
|
446 | 1192 | free_gen8_temp_bitmaps(pds, pts); |
|
- | 1193 | return -ENOMEM; |
|
- | 1194 | } |
|
- | 1195 | ||
- | 1196 | /* PDE TLBs are a pain to invalidate on GEN8+. When we modify |
|
- | 1197 | * the page table structures, we mark them dirty so that |
|
- | 1198 | * context switching/execlist queuing code takes extra steps |
|
- | 1199 | * to ensure that tlbs are flushed. |
|
- | 1200 | */ |
|
- | 1201 | static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) |
|
- | 1202 | { |
|
- | 1203 | ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask; |
|
- | 1204 | } |
|
- | 1205 | ||
- | 1206 | static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm, |
|
- | 1207 | struct i915_page_directory_pointer *pdp, |
|
- | 1208 | uint64_t start, |
|
- | 1209 | uint64_t length) |
|
- | 1210 | { |
|
- | 1211 | struct i915_hw_ppgtt *ppgtt = |
|
- | 1212 | container_of(vm, struct i915_hw_ppgtt, base); |
|
- | 1213 | unsigned long *new_page_dirs, *new_page_tables; |
|
- | 1214 | struct drm_device *dev = vm->dev; |
|
- | 1215 | struct i915_page_directory *pd; |
|
- | 1216 | const uint64_t orig_start = start; |
|
- | 1217 | const uint64_t orig_length = length; |
|
- | 1218 | uint64_t temp; |
|
- | 1219 | uint32_t pdpe; |
|
- | 1220 | uint32_t pdpes = I915_PDPES_PER_PDP(dev); |
|
- | 1221 | int ret; |
|
- | 1222 | ||
- | 1223 | /* Wrap is never okay since we can only represent 48b, and we don't |
|
- | 1224 | * actually use the other side of the canonical address space. |
|
- | 1225 | */ |
|
- | 1226 | if (WARN_ON(start + length < start)) |
|
- | 1227 | return -ENODEV; |
|
- | 1228 | ||
- | 1229 | if (WARN_ON(start + length > vm->total)) |
|
- | 1230 | return -ENODEV; |
|
- | 1231 | ||
- | 1232 | ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes); |
|
- | 1233 | if (ret) |
|
- | 1234 | return ret; |
|
- | 1235 | ||
- | 1236 | /* Do the allocations first so we can easily bail out */ |
|
- | 1237 | ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length, |
|
- | 1238 | new_page_dirs); |
|
- | 1239 | if (ret) { |
|
- | 1240 | free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); |
|
- | 1241 | return ret; |
|
- | 1242 | } |
|
447 | unwind_out: |
1243 | |
- | 1244 | /* For every page directory referenced, allocate page tables */ |
|
- | 1245 | gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { |
|
- | 1246 | ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length, |
|
- | 1247 | new_page_tables + pdpe * BITS_TO_LONGS(I915_PDES)); |
|
- | 1248 | if (ret) |
|
- | 1249 | goto err_out; |
|
- | 1250 | } |
|
- | 1251 | ||
Line -... | Line 1252... | ||
- | 1252 | start = orig_start; |
|
- | 1253 | length = orig_length; |
|
- | 1254 | ||
- | 1255 | /* Allocations have completed successfully, so set the bitmaps, and do |
|
- | 1256 | * the mappings. */ |
|
448 | while (i--) { |
1257 | gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { |
449 | gen8_free_page_tables(pt_pages[i]); |
1258 | gen8_pde_t *const page_directory = kmap_px(pd); |
Line 450... | Line 1259... | ||
450 | kfree(pt_pages[i]); |
1259 | struct i915_page_table *pt; |
- | 1260 | uint64_t pd_len = length; |
|
- | 1261 | uint64_t pd_start = start; |
|
451 | } |
1262 | uint32_t pde; |
452 | 1263 | ||
- | 1264 | /* Every pd should be allocated, we just did that above. */ |
|
- | 1265 | WARN_ON(!pd); |
|
- | 1266 | ||
- | 1267 | gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) { |
|
453 | return ret; |
1268 | /* Same reasoning as pd */ |
454 | } |
1269 | WARN_ON(!pt); |
- | 1270 | WARN_ON(!pd_len); |
|
- | 1271 | WARN_ON(!gen8_pte_count(pd_start, pd_len)); |
|
- | 1272 | ||
- | 1273 | /* Set our used ptes within the page table */ |
|
Line -... | Line 1274... | ||
- | 1274 | bitmap_set(pt->used_ptes, |
|
455 | 1275 | gen8_pte_index(pd_start), |
|
- | 1276 | gen8_pte_count(pd_start, pd_len)); |
|
456 | static int gen8_ppgtt_allocate_dma(struct i915_hw_ppgtt *ppgtt) |
1277 | |
- | 1278 | /* Our pde is now pointing to the pagetable, pt */ |
|
- | 1279 | __set_bit(pde, pd->used_pdes); |
|
- | 1280 | ||
- | 1281 | /* Map the PDE to the page table */ |
|
- | 1282 | page_directory[pde] = gen8_pde_encode(px_dma(pt), |
|
- | 1283 | I915_CACHE_LLC); |
|
457 | { |
1284 | trace_i915_page_table_entry_map(&ppgtt->base, pde, pt, |
Line 458... | Line 1285... | ||
458 | int i; |
1285 | gen8_pte_index(start), |
459 | 1286 | gen8_pte_count(start, length), |
|
460 | for (i = 0; i < ppgtt->num_pd_pages; i++) { |
- | |
Line -... | Line 1287... | ||
- | 1287 | GEN8_PTES); |
|
- | 1288 | ||
- | 1289 | /* NB: We haven't yet mapped ptes to pages. At this |
|
- | 1290 | * point we're still relying on insert_entries() */ |
|
- | 1291 | } |
|
- | 1292 | ||
- | 1293 | kunmap_px(ppgtt, page_directory); |
|
461 | ppgtt->gen8_pt_dma_addr[i] = kcalloc(GEN8_PDES_PER_PAGE, |
1294 | __set_bit(pdpe, pdp->used_pdpes); |
- | 1295 | gen8_setup_page_directory(ppgtt, pdp, pd, pdpe); |
|
Line 462... | Line 1296... | ||
462 | sizeof(dma_addr_t), |
1296 | } |
- | 1297 | ||
- | 1298 | free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); |
|
- | 1299 | mark_tlbs_dirty(ppgtt); |
|
- | 1300 | return 0; |
|
- | 1301 | ||
- | 1302 | err_out: |
|
463 | GFP_KERNEL); |
1303 | while (pdpe--) { |
Line 464... | Line 1304... | ||
464 | if (!ppgtt->gen8_pt_dma_addr[i]) |
1304 | for_each_set_bit(temp, new_page_tables + pdpe * |
465 | return -ENOMEM; |
- | |
466 | } |
1305 | BITS_TO_LONGS(I915_PDES), I915_PDES) |
467 | 1306 | free_pt(dev, pdp->page_directory[pdpe]->page_table[temp]); |
|
- | 1307 | } |
|
- | 1308 | ||
- | 1309 | for_each_set_bit(pdpe, new_page_dirs, pdpes) |
|
- | 1310 | free_pd(dev, pdp->page_directory[pdpe]); |
|
- | 1311 | ||
- | 1312 | free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); |
|
- | 1313 | mark_tlbs_dirty(ppgtt); |
|
- | 1314 | return ret; |
|
- | 1315 | } |
|
- | 1316 | ||
- | 1317 | static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm, |
|
- | 1318 | struct i915_pml4 *pml4, |
|
- | 1319 | uint64_t start, |
|
- | 1320 | uint64_t length) |
|
- | 1321 | { |
|
- | 1322 | DECLARE_BITMAP(new_pdps, GEN8_PML4ES_PER_PML4); |
|
468 | return 0; |
1323 | struct i915_hw_ppgtt *ppgtt = |
- | 1324 | container_of(vm, struct i915_hw_ppgtt, base); |
|
- | 1325 | struct i915_page_directory_pointer *pdp; |
|
469 | } |
1326 | uint64_t temp, pml4e; |
- | 1327 | int ret = 0; |
|
- | 1328 | ||
- | 1329 | /* Do the pml4 allocations first, so we don't need to track the newly |
|
- | 1330 | * allocated tables below the pdp */ |
|
- | 1331 | bitmap_zero(new_pdps, GEN8_PML4ES_PER_PML4); |
|
- | 1332 | ||
- | 1333 | /* The pagedirectory and pagetable allocations are done in the shared 3 |
|
- | 1334 | * and 4 level code. Just allocate the pdps. |
|
- | 1335 | */ |
|
- | 1336 | ret = gen8_ppgtt_alloc_page_dirpointers(vm, pml4, start, length, |
|
- | 1337 | new_pdps); |
|
- | 1338 | if (ret) |
|
- | 1339 | return ret; |
|
- | 1340 | ||
- | 1341 | WARN(bitmap_weight(new_pdps, GEN8_PML4ES_PER_PML4) > 2, |
|
- | 1342 | "The allocation has spanned more than 512GB. " |
|
- | 1343 | "It is highly likely this is incorrect."); |
|
- | 1344 | ||
- | 1345 | gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) { |
|
- | 1346 | WARN_ON(!pdp); |
|
- | 1347 | ||
- | 1348 | ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length); |
|
- | 1349 | if (ret) |
|
- | 1350 | goto err_out; |
|
- | 1351 | ||
- | 1352 | gen8_setup_page_directory_pointer(ppgtt, pml4, pdp, pml4e); |
|
- | 1353 | } |
|
- | 1354 | ||
- | 1355 | bitmap_or(pml4->used_pml4es, new_pdps, pml4->used_pml4es, |
|
- | 1356 | GEN8_PML4ES_PER_PML4); |
|
- | 1357 | ||
- | 1358 | return 0; |
|
- | 1359 | ||
- | 1360 | err_out: |
|
- | 1361 | for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4) |
|
- | 1362 | gen8_ppgtt_cleanup_3lvl(vm->dev, pml4->pdps[pml4e]); |
|
- | 1363 | ||
- | 1364 | return ret; |
|
- | 1365 | } |
|
- | 1366 | ||
- | 1367 | static int gen8_alloc_va_range(struct i915_address_space *vm, |
|
- | 1368 | uint64_t start, uint64_t length) |
|
- | 1369 | { |
|
- | 1370 | struct i915_hw_ppgtt *ppgtt = |
|
- | 1371 | container_of(vm, struct i915_hw_ppgtt, base); |
|
- | 1372 | ||
- | 1373 | if (USES_FULL_48BIT_PPGTT(vm->dev)) |
|
- | 1374 | return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length); |
|
- | 1375 | else |
|
- | 1376 | return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length); |
|
- | 1377 | } |
|
- | 1378 | ||
- | 1379 | static void gen8_dump_pdp(struct i915_page_directory_pointer *pdp, |
|
- | 1380 | uint64_t start, uint64_t length, |
|
- | 1381 | gen8_pte_t scratch_pte, |
|
- | 1382 | struct seq_file *m) |
|
- | 1383 | { |
|
- | 1384 | struct i915_page_directory *pd; |
|
- | 1385 | uint64_t temp; |
|
- | 1386 | uint32_t pdpe; |
|
- | 1387 | ||
- | 1388 | gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) { |
|
- | 1389 | struct i915_page_table *pt; |
|
- | 1390 | uint64_t pd_len = length; |
|
- | 1391 | uint64_t pd_start = start; |
|
- | 1392 | uint32_t pde; |
|
- | 1393 | ||
- | 1394 | if (!test_bit(pdpe, pdp->used_pdpes)) |
|
- | 1395 | continue; |
|
- | 1396 | ||
- | 1397 | seq_printf(m, "\tPDPE #%d\n", pdpe); |
|
- | 1398 | gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) { |
|
- | 1399 | uint32_t pte; |
|
- | 1400 | gen8_pte_t *pt_vaddr; |
|
- | 1401 | ||
- | 1402 | if (!test_bit(pde, pd->used_pdes)) |
|
470 | 1403 | continue; |
|
Line 471... | Line -... | ||
471 | static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt, |
- | |
472 | const int max_pdp) |
1404 | |
473 | { |
1405 | pt_vaddr = kmap_px(pt); |
- | 1406 | for (pte = 0; pte < GEN8_PTES; pte += 4) { |
|
474 | // ppgtt->pd_pages = alloc_pages(GFP_KERNEL, get_order(max_pdp << PAGE_SHIFT)); |
1407 | uint64_t va = |
475 | if (!ppgtt->pd_pages) |
1408 | (pdpe << GEN8_PDPE_SHIFT) | |
476 | return -ENOMEM; |
1409 | (pde << GEN8_PDE_SHIFT) | |
- | 1410 | (pte << GEN8_PTE_SHIFT); |
|
- | 1411 | int i; |
|
- | 1412 | bool found = false; |
|
- | 1413 | ||
- | 1414 | for (i = 0; i < 4; i++) |
|
- | 1415 | if (pt_vaddr[pte + i] != scratch_pte) |
|
- | 1416 | found = true; |
|
- | 1417 | if (!found) |
|
- | 1418 | continue; |
|
Line 477... | Line 1419... | ||
477 | 1419 | ||
Line 478... | Line 1420... | ||
478 | // ppgtt->num_pd_pages = 1 << get_order(max_pdp << PAGE_SHIFT); |
1420 | seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte); |
479 | BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPS); |
1421 | for (i = 0; i < 4; i++) { |
Line 480... | Line 1422... | ||
480 | 1422 | if (pt_vaddr[pte + i] != scratch_pte) |
|
481 | return 0; |
1423 | seq_printf(m, " %llx", pt_vaddr[pte + i]); |
482 | } |
1424 | else |
483 | 1425 | seq_puts(m, " SCRATCH "); |
|
484 | static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt, |
1426 | } |
485 | const int max_pdp) |
1427 | seq_puts(m, "\n"); |
486 | { |
- | |
487 | int ret; |
- | |
488 | - | ||
489 | ret = gen8_ppgtt_allocate_page_directories(ppgtt, max_pdp); |
1428 | } |
490 | if (ret) |
1429 | /* don't use kunmap_px, it could trigger |
491 | return ret; |
1430 | * an unnecessary flush. |
492 | - | ||
493 | ret = gen8_ppgtt_allocate_page_tables(ppgtt, max_pdp); |
- | |
494 | if (ret) { |
1431 | */ |
495 | // __free_pages(ppgtt->pd_pages, get_order(max_pdp << PAGE_SHIFT)); |
- | |
496 | return ret; |
- | |
497 | } |
- | |
Line 498... | Line -... | ||
498 | - | ||
499 | ppgtt->num_pd_entries = max_pdp * GEN8_PDES_PER_PAGE; |
1432 | kunmap_atomic(pt_vaddr); |
500 | 1433 | } |
|
501 | ret = gen8_ppgtt_allocate_dma(ppgtt); |
1434 | } |
Line 502... | Line 1435... | ||
502 | if (ret) |
1435 | } |
- | 1436 | ||
- | 1437 | static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) |
|
503 | gen8_ppgtt_free(ppgtt); |
1438 | { |
- | 1439 | struct i915_address_space *vm = &ppgtt->base; |
|
- | 1440 | uint64_t start = ppgtt->base.start; |
|
- | 1441 | uint64_t length = ppgtt->base.total; |
|
- | 1442 | gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), |
|
504 | 1443 | I915_CACHE_LLC, true); |
|
505 | return ret; |
1444 | |
506 | } |
1445 | if (!USES_FULL_48BIT_PPGTT(vm->dev)) { |
507 | 1446 | gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m); |
|
508 | static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt, |
1447 | } else { |
- | 1448 | uint64_t templ4, pml4e; |
|
- | 1449 | struct i915_pml4 *pml4 = &ppgtt->pml4; |
|
Line -... | Line 1450... | ||
- | 1450 | struct i915_page_directory_pointer *pdp; |
|
509 | const int pd) |
1451 | |
- | 1452 | gen8_for_each_pml4e(pdp, pml4, start, length, templ4, pml4e) { |
|
510 | { |
1453 | if (!test_bit(pml4e, pml4->used_pml4es)) |
511 | dma_addr_t pd_addr; |
1454 | continue; |
512 | int ret; |
1455 | |
513 | - | ||
514 | pd_addr = pci_map_page(ppgtt->base.dev->pdev, |
- | |
Line 515... | Line -... | ||
515 | &ppgtt->pd_pages[pd], 0, |
- | |
516 | PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); |
- | |
517 | 1456 | seq_printf(m, " PML4E #%llu\n", pml4e); |
|
518 | // ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr); |
- | |
519 | // if (ret) |
1457 | gen8_dump_pdp(pdp, start, length, scratch_pte, m); |
520 | // return ret; |
1458 | } |
521 | - | ||
522 | ppgtt->pd_dma_addr[pd] = pd_addr; |
- | |
523 | 1459 | } |
|
524 | return 0; |
1460 | } |
- | 1461 | ||
525 | } |
1462 | static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt) |
526 | 1463 | { |
|
527 | static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt, |
1464 | unsigned long *new_page_dirs, *new_page_tables; |
528 | const int pd, |
- | |
529 | const int pt) |
1465 | uint32_t pdpes = I915_PDPES_PER_PDP(dev); |
530 | { |
1466 | int ret; |
531 | dma_addr_t pt_addr; |
- | |
532 | struct page *p; |
- | |
533 | int ret; |
- | |
534 | 1467 | ||
Line 535... | Line -... | ||
535 | p = ppgtt->gen8_pt_pages[pd][pt]; |
- | |
536 | pt_addr = pci_map_page(ppgtt->base.dev->pdev, |
1468 | /* We allocate temp bitmap for page tables for no gain |
537 | p, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); |
- | |
538 | // ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pt_addr); |
1469 | * but as this is for init only, lets keep the things simple |
539 | // if (ret) |
- | |
540 | // return ret; |
- | |
541 | - | ||
542 | ppgtt->gen8_pt_dma_addr[pd][pt] = pt_addr; |
- | |
Line 543... | Line -... | ||
543 | - | ||
544 | return 0; |
- | |
545 | } |
- | |
546 | - | ||
547 | /** |
- | |
548 | * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers |
1470 | */ |
Line 549... | Line 1471... | ||
549 | * with a net effect resembling a 2-level page table in normal x86 terms. Each |
1471 | ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes); |
550 | * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address |
- | |
551 | * space. |
1472 | if (ret) |
552 | * |
1473 | return ret; |
553 | * FIXME: split allocation into smaller pieces. For now we only ever do this |
1474 | |
Line 554... | Line 1475... | ||
554 | * once, but with full PPGTT, the multiple contiguous allocations will be bad. |
1475 | /* Allocate for all pdps regardless of how the ppgtt |
555 | * TODO: Do something with the size parameter |
1476 | * was defined. |
556 | */ |
1477 | */ |
- | 1478 | ret = gen8_ppgtt_alloc_page_directories(&ppgtt->base, &ppgtt->pdp, |
|
557 | static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) |
1479 | 0, 1ULL << 32, |
558 | { |
1480 | new_page_dirs); |
- | 1481 | if (!ret) |
|
- | 1482 | *ppgtt->pdp.used_pdpes = *new_page_dirs; |
|
- | 1483 | ||
- | 1484 | free_gen8_temp_bitmaps(new_page_dirs, new_page_tables); |
|
- | 1485 | ||
- | 1486 | return ret; |
|
- | 1487 | } |
|
- | 1488 | ||
- | 1489 | /* |
|
- | 1490 | * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers |
|
- | 1491 | * with a net effect resembling a 2-level page table in normal x86 terms. Each |
|
- | 1492 | * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address |
|
- | 1493 | * space. |
|
- | 1494 | * |
|
- | 1495 | */ |
|
- | 1496 | static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt) |
|
- | 1497 | { |
|
- | 1498 | int ret; |
|
- | 1499 | ||
- | 1500 | ret = gen8_init_scratch(&ppgtt->base); |
|
- | 1501 | if (ret) |
|
- | 1502 | return ret; |
|
- | 1503 | ||
- | 1504 | ppgtt->base.start = 0; |
|
- | 1505 | ppgtt->base.cleanup = gen8_ppgtt_cleanup; |
|
- | 1506 | ppgtt->base.allocate_va_range = gen8_alloc_va_range; |
|
559 | const int max_pdp = DIV_ROUND_UP(size, 1 << 30); |
1507 | ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; |
- | 1508 | ppgtt->base.clear_range = gen8_ppgtt_clear_range; |
|
- | 1509 | ppgtt->base.unbind_vma = ppgtt_unbind_vma; |
|
- | 1510 | ppgtt->base.bind_vma = ppgtt_bind_vma; |
|
- | 1511 | ppgtt->debug_dump = gen8_dump_ppgtt; |
|
- | 1512 | ||
- | 1513 | if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) { |
|
- | 1514 | ret = setup_px(ppgtt->base.dev, &ppgtt->pml4); |
|
- | 1515 | if (ret) |
|
- | 1516 | goto free_scratch; |
|
- | 1517 | ||
- | 1518 | gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4); |
|
- | 1519 | ||
- | 1520 | ppgtt->base.total = 1ULL << 48; |
|
- | 1521 | ppgtt->switch_mm = gen8_48b_mm_switch; |
|
- | 1522 | } else { |
|
- | 1523 | ret = __pdp_init(ppgtt->base.dev, &ppgtt->pdp); |
|
- | 1524 | if (ret) |
|
- | 1525 | goto free_scratch; |
|
- | 1526 | ||
Line 560... | Line 1527... | ||
560 | const int min_pt_pages = GEN8_PDES_PER_PAGE * max_pdp; |
1527 | ppgtt->base.total = 1ULL << 32; |
561 | int i, j, ret; |
1528 | ppgtt->switch_mm = gen8_legacy_mm_switch; |
562 | 1529 | trace_i915_page_directory_pointer_entry_alloc(&ppgtt->base, |
|
- | 1530 | 0, 0, |
|
- | 1531 | GEN8_PML4E_SHIFT); |
|
- | 1532 | ||
563 | if (size % (1<<30)) |
1533 | if (intel_vgpu_active(ppgtt->base.dev)) { |
564 | DRM_INFO("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n", size); |
1534 | ret = gen8_preallocate_top_level_pdps(ppgtt); |
Line 565... | Line -... | ||
565 | - | ||
566 | /* 1. Do all our allocations for page directories and page tables. */ |
1535 | if (ret) |
567 | ret = gen8_ppgtt_alloc(ppgtt, max_pdp); |
1536 | goto free_scratch; |
Line 568... | Line 1537... | ||
568 | if (ret) |
1537 | } |
569 | return ret; |
1538 | } |
- | 1539 | ||
- | 1540 | if (intel_vgpu_active(ppgtt->base.dev)) |
|
- | 1541 | gen8_ppgtt_notify_vgt(ppgtt, true); |
|
- | 1542 | ||
- | 1543 | return 0; |
|
- | 1544 | ||
- | 1545 | free_scratch: |
|
- | 1546 | gen8_free_scratch(&ppgtt->base); |
|
- | 1547 | return ret; |
|
- | 1548 | } |
|
- | 1549 | ||
- | 1550 | static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) |
|
- | 1551 | { |
|
- | 1552 | struct i915_address_space *vm = &ppgtt->base; |
|
- | 1553 | struct i915_page_table *unused; |
|
570 | 1554 | gen6_pte_t scratch_pte; |
|
571 | /* |
1555 | uint32_t pd_entry; |
Line 572... | Line 1556... | ||
572 | * 2. Create DMA mappings for the page directories and page tables. |
1556 | uint32_t pte, pde, temp; |
573 | */ |
1557 | uint32_t start = ppgtt->base.start, length = ppgtt->base.total; |
574 | for (i = 0; i < max_pdp; i++) { |
1558 | |
Line 575... | Line 1559... | ||
575 | ret = gen8_ppgtt_setup_page_directories(ppgtt, i); |
1559 | scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), |
576 | if (ret) |
1560 | I915_CACHE_LLC, true, 0); |
Line 577... | Line 1561... | ||
577 | goto bail; |
1561 | |
578 | 1562 | gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) { |
|
579 | for (j = 0; j < GEN8_PDES_PER_PAGE; j++) { |
1563 | u32 expected; |
- | 1564 | gen6_pte_t *pt_vaddr; |
|
580 | ret = gen8_ppgtt_setup_page_tables(ppgtt, i, j); |
1565 | const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]); |
Line 581... | Line 1566... | ||
581 | if (ret) |
1566 | pd_entry = readl(ppgtt->pd_addr + pde); |
582 | goto bail; |
1567 | expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID); |
583 | } |
1568 | |
584 | } |
1569 | if (pd_entry != expected) |
Line 585... | Line 1570... | ||
585 | 1570 | seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n", |
|
586 | /* |
1571 | pde, |
587 | * 3. Map all the page directory entires to point to the page tables |
1572 | pd_entry, |
Line 588... | Line 1573... | ||
588 | * we've allocated. |
1573 | expected); |
589 | * |
1574 | seq_printf(m, "\tPDE: %x\n", pd_entry); |
Line 678... | Line 1663... | ||
678 | intel_ring_advance(ring); |
1663 | intel_ring_advance(ring); |
Line 679... | Line 1664... | ||
679 | 1664 | ||
680 | return 0; |
1665 | return 0; |
Line -... | Line 1666... | ||
- | 1666 | } |
|
- | 1667 | ||
- | 1668 | static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt, |
|
- | 1669 | struct drm_i915_gem_request *req) |
|
- | 1670 | { |
|
- | 1671 | struct intel_engine_cs *ring = req->ring; |
|
- | 1672 | struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev); |
|
- | 1673 | ||
- | 1674 | I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); |
|
- | 1675 | I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); |
|
- | 1676 | return 0; |
|
681 | } |
1677 | } |
682 | 1678 | ||
683 | static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, |
1679 | static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, |
- | 1680 | struct drm_i915_gem_request *req) |
|
684 | struct intel_engine_cs *ring) |
1681 | { |
Line 685... | Line 1682... | ||
685 | { |
1682 | struct intel_engine_cs *ring = req->ring; |
686 | int ret; |
1683 | int ret; |
687 | 1684 | ||
688 | /* NB: TLBs must be flushed and invalidated before a switch */ |
1685 | /* NB: TLBs must be flushed and invalidated before a switch */ |
Line 689... | Line 1686... | ||
689 | ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); |
1686 | ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); |
690 | if (ret) |
1687 | if (ret) |
691 | return ret; |
1688 | return ret; |
Line 692... | Line 1689... | ||
692 | 1689 | ||
693 | ret = intel_ring_begin(ring, 6); |
1690 | ret = intel_ring_begin(req, 6); |
Line 702... | Line 1699... | ||
702 | intel_ring_emit(ring, MI_NOOP); |
1699 | intel_ring_emit(ring, MI_NOOP); |
703 | intel_ring_advance(ring); |
1700 | intel_ring_advance(ring); |
Line 704... | Line 1701... | ||
704 | 1701 | ||
705 | /* XXX: RCS is the only one to auto invalidate the TLBs? */ |
1702 | /* XXX: RCS is the only one to auto invalidate the TLBs? */ |
706 | if (ring->id != RCS) { |
1703 | if (ring->id != RCS) { |
707 | ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); |
1704 | ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); |
708 | if (ret) |
1705 | if (ret) |
709 | return ret; |
1706 | return ret; |
Line 710... | Line 1707... | ||
710 | } |
1707 | } |
711 | 1708 | ||
Line 712... | Line 1709... | ||
712 | return 0; |
1709 | return 0; |
713 | } |
1710 | } |
714 | 1711 | ||
- | 1712 | static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, |
|
715 | static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, |
1713 | struct drm_i915_gem_request *req) |
716 | struct intel_engine_cs *ring) |
1714 | { |
Line 717... | Line 1715... | ||
717 | { |
1715 | struct intel_engine_cs *ring = req->ring; |
Line 732... | Line 1730... | ||
732 | struct drm_i915_private *dev_priv = dev->dev_private; |
1730 | struct drm_i915_private *dev_priv = dev->dev_private; |
733 | struct intel_engine_cs *ring; |
1731 | struct intel_engine_cs *ring; |
734 | int j; |
1732 | int j; |
Line 735... | Line 1733... | ||
735 | 1733 | ||
- | 1734 | for_each_ring(ring, dev_priv, j) { |
|
736 | for_each_ring(ring, dev_priv, j) { |
1735 | u32 four_level = USES_FULL_48BIT_PPGTT(dev) ? GEN8_GFX_PPGTT_48B : 0; |
737 | I915_WRITE(RING_MODE_GEN7(ring), |
1736 | I915_WRITE(RING_MODE_GEN7(ring), |
738 | _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
1737 | _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level)); |
739 | } |
1738 | } |
Line 740... | Line 1739... | ||
740 | } |
1739 | } |
741 | 1740 | ||
Line 789... | Line 1788... | ||
789 | uint64_t length, |
1788 | uint64_t length, |
790 | bool use_scratch) |
1789 | bool use_scratch) |
791 | { |
1790 | { |
792 | struct i915_hw_ppgtt *ppgtt = |
1791 | struct i915_hw_ppgtt *ppgtt = |
793 | container_of(vm, struct i915_hw_ppgtt, base); |
1792 | container_of(vm, struct i915_hw_ppgtt, base); |
794 | gen6_gtt_pte_t *pt_vaddr, scratch_pte; |
1793 | gen6_pte_t *pt_vaddr, scratch_pte; |
795 | unsigned first_entry = start >> PAGE_SHIFT; |
1794 | unsigned first_entry = start >> PAGE_SHIFT; |
796 | unsigned num_entries = length >> PAGE_SHIFT; |
1795 | unsigned num_entries = length >> PAGE_SHIFT; |
797 | unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; |
1796 | unsigned act_pt = first_entry / GEN6_PTES; |
798 | unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
1797 | unsigned first_pte = first_entry % GEN6_PTES; |
799 | unsigned last_pte, i; |
1798 | unsigned last_pte, i; |
Line 800... | Line 1799... | ||
800 | 1799 | ||
- | 1800 | scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), |
|
Line 801... | Line 1801... | ||
801 | scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true, 0); |
1801 | I915_CACHE_LLC, true, 0); |
802 | 1802 | ||
803 | while (num_entries) { |
1803 | while (num_entries) { |
804 | last_pte = first_pte + num_entries; |
1804 | last_pte = first_pte + num_entries; |
Line 805... | Line 1805... | ||
805 | if (last_pte > I915_PPGTT_PT_ENTRIES) |
1805 | if (last_pte > GEN6_PTES) |
Line 806... | Line 1806... | ||
806 | last_pte = I915_PPGTT_PT_ENTRIES; |
1806 | last_pte = GEN6_PTES; |
807 | 1807 | ||
Line 808... | Line 1808... | ||
808 | pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]); |
1808 | pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); |
Line 809... | Line 1809... | ||
809 | 1809 | ||
810 | for (i = first_pte; i < last_pte; i++) |
1810 | for (i = first_pte; i < last_pte; i++) |
811 | pt_vaddr[i] = scratch_pte; |
1811 | pt_vaddr[i] = scratch_pte; |
812 | 1812 | ||
Line 823... | Line 1823... | ||
823 | uint64_t start, |
1823 | uint64_t start, |
824 | enum i915_cache_level cache_level, u32 flags) |
1824 | enum i915_cache_level cache_level, u32 flags) |
825 | { |
1825 | { |
826 | struct i915_hw_ppgtt *ppgtt = |
1826 | struct i915_hw_ppgtt *ppgtt = |
827 | container_of(vm, struct i915_hw_ppgtt, base); |
1827 | container_of(vm, struct i915_hw_ppgtt, base); |
828 | gen6_gtt_pte_t *pt_vaddr; |
1828 | gen6_pte_t *pt_vaddr; |
829 | unsigned first_entry = start >> PAGE_SHIFT; |
1829 | unsigned first_entry = start >> PAGE_SHIFT; |
830 | unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES; |
1830 | unsigned act_pt = first_entry / GEN6_PTES; |
831 | unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES; |
1831 | unsigned act_pte = first_entry % GEN6_PTES; |
832 | struct sg_page_iter sg_iter; |
1832 | struct sg_page_iter sg_iter; |
Line 833... | Line 1833... | ||
833 | 1833 | ||
834 | pt_vaddr = NULL; |
1834 | pt_vaddr = NULL; |
835 | for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { |
1835 | for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) { |
836 | if (pt_vaddr == NULL) |
1836 | if (pt_vaddr == NULL) |
Line 837... | Line 1837... | ||
837 | pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]); |
1837 | pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]); |
838 | 1838 | ||
839 | pt_vaddr[act_pte] = |
1839 | pt_vaddr[act_pte] = |
Line 840... | Line 1840... | ||
840 | vm->pte_encode(sg_page_iter_dma_address(&sg_iter), |
1840 | vm->pte_encode(sg_page_iter_dma_address(&sg_iter), |
841 | cache_level, true, flags); |
1841 | cache_level, true, flags); |
842 | 1842 | ||
843 | if (++act_pte == I915_PPGTT_PT_ENTRIES) { |
1843 | if (++act_pte == GEN6_PTES) { |
844 | kunmap_atomic(pt_vaddr); |
1844 | kunmap_px(ppgtt, pt_vaddr); |
845 | pt_vaddr = NULL; |
1845 | pt_vaddr = NULL; |
846 | act_pt++; |
1846 | act_pt++; |
847 | act_pte = 0; |
1847 | act_pte = 0; |
848 | } |
1848 | } |
849 | } |
1849 | } |
Line 850... | Line 1850... | ||
850 | if (pt_vaddr) |
1850 | if (pt_vaddr) |
- | 1851 | kunmap_px(ppgtt, pt_vaddr); |
|
851 | kunmap_atomic(pt_vaddr); |
1852 | } |
- | 1853 | ||
- | 1854 | static int gen6_alloc_va_range(struct i915_address_space *vm, |
|
- | 1855 | uint64_t start_in, uint64_t length_in) |
|
- | 1856 | { |
|
- | 1857 | DECLARE_BITMAP(new_page_tables, I915_PDES); |
|
- | 1858 | struct drm_device *dev = vm->dev; |
|
- | 1859 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
- | 1860 | struct i915_hw_ppgtt *ppgtt = |
|
852 | } |
1861 | container_of(vm, struct i915_hw_ppgtt, base); |
- | 1862 | struct i915_page_table *pt; |
|
- | 1863 | uint32_t start, length, start_save, length_save; |
|
- | 1864 | uint32_t pde, temp; |
|
- | 1865 | int ret; |
|
- | 1866 | ||
- | 1867 | if (WARN_ON(start_in + length_in > ppgtt->base.total)) |
|
- | 1868 | return -ENODEV; |
|
- | 1869 | ||
- | 1870 | start = start_save = start_in; |
|
- | 1871 | length = length_save = length_in; |
|
- | 1872 | ||
- | 1873 | bitmap_zero(new_page_tables, I915_PDES); |
|
- | 1874 | ||
- | 1875 | /* The allocation is done in two stages so that we can bail out with |
|
- | 1876 | * minimal amount of pain. The first stage finds new page tables that |
|
- | 1877 | * need allocation. The second stage marks use ptes within the page |
|
- | 1878 | * tables. |
|
- | 1879 | */ |
|
- | 1880 | gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) { |
|
- | 1881 | if (pt != vm->scratch_pt) { |
|
- | 1882 | WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES)); |
|
- | 1883 | continue; |
|
- | 1884 | } |
|
- | 1885 | ||
- | 1886 | /* We've already allocated a page table */ |
|
- | 1887 | WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES)); |
|
- | 1888 | ||
- | 1889 | pt = alloc_pt(dev); |
|
Line 853... | Line 1890... | ||
853 | 1890 | if (IS_ERR(pt)) { |
|
854 | static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) |
- | |
- | 1891 | ret = PTR_ERR(pt); |
|
855 | { |
1892 | goto unwind_out; |
856 | int i; |
1893 | } |
857 | 1894 | ||
858 | if (ppgtt->pt_dma_addr) { |
1895 | gen6_initialize_pt(vm, pt); |
- | 1896 | ||
- | 1897 | ppgtt->pd.page_table[pde] = pt; |
|
- | 1898 | __set_bit(pde, new_page_tables); |
|
- | 1899 | trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT); |
|
- | 1900 | } |
|
- | 1901 | ||
- | 1902 | start = start_save; |
|
- | 1903 | length = length_save; |
|
- | 1904 | ||
- | 1905 | gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) { |
|
- | 1906 | DECLARE_BITMAP(tmp_bitmap, GEN6_PTES); |
|
- | 1907 | ||
- | 1908 | bitmap_zero(tmp_bitmap, GEN6_PTES); |
|
- | 1909 | bitmap_set(tmp_bitmap, gen6_pte_index(start), |
|
- | 1910 | gen6_pte_count(start, length)); |
|
- | 1911 | ||
- | 1912 | if (__test_and_clear_bit(pde, new_page_tables)) |
|
- | 1913 | gen6_write_pde(&ppgtt->pd, pde, pt); |
|
- | 1914 | ||
- | 1915 | trace_i915_page_table_entry_map(vm, pde, pt, |
|
859 | for (i = 0; i < ppgtt->num_pd_entries; i++) |
1916 | gen6_pte_index(start), |
Line -... | Line 1917... | ||
- | 1917 | gen6_pte_count(start, length), |
|
- | 1918 | GEN6_PTES); |
|
- | 1919 | bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes, |
|
- | 1920 | GEN6_PTES); |
|
- | 1921 | } |
|
- | 1922 | ||
- | 1923 | WARN_ON(!bitmap_empty(new_page_tables, I915_PDES)); |
|
- | 1924 | ||
- | 1925 | /* Make sure write is complete before other code can use this page |
|
- | 1926 | * table. Also require for WC mapped PTEs */ |
|
- | 1927 | readl(dev_priv->gtt.gsm); |
|
- | 1928 | ||
- | 1929 | mark_tlbs_dirty(ppgtt); |
|
- | 1930 | return 0; |
|
- | 1931 | ||
- | 1932 | unwind_out: |
|
- | 1933 | for_each_set_bit(pde, new_page_tables, I915_PDES) { |
|
- | 1934 | struct i915_page_table *pt = ppgtt->pd.page_table[pde]; |
|
- | 1935 | ||
- | 1936 | ppgtt->pd.page_table[pde] = vm->scratch_pt; |
|
- | 1937 | free_pt(vm->dev, pt); |
|
860 | pci_unmap_page(ppgtt->base.dev->pdev, |
1938 | } |
861 | ppgtt->pt_dma_addr[i], |
1939 | |
- | 1940 | mark_tlbs_dirty(ppgtt); |
|
- | 1941 | return ret; |
|
- | 1942 | } |
|
- | 1943 | ||
- | 1944 | static int gen6_init_scratch(struct i915_address_space *vm) |
|
- | 1945 | { |
|
- | 1946 | struct drm_device *dev = vm->dev; |
|
- | 1947 | ||
- | 1948 | vm->scratch_page = alloc_scratch_page(dev); |
|
- | 1949 | if (IS_ERR(vm->scratch_page)) |
|
862 | 4096, PCI_DMA_BIDIRECTIONAL); |
1950 | return PTR_ERR(vm->scratch_page); |
- | 1951 | ||
- | 1952 | vm->scratch_pt = alloc_pt(dev); |
|
Line -... | Line 1953... | ||
- | 1953 | if (IS_ERR(vm->scratch_pt)) { |
|
- | 1954 | free_scratch_page(dev, vm->scratch_page); |
|
- | 1955 | return PTR_ERR(vm->scratch_pt); |
|
863 | } |
1956 | } |
- | 1957 | ||
864 | } |
1958 | gen6_initialize_pt(vm, vm->scratch_pt); |
- | 1959 | ||
865 | 1960 | return 0; |
|
866 | static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt) |
1961 | } |
867 | { |
1962 | |
Line 868... | Line 1963... | ||
868 | int i; |
1963 | static void gen6_free_scratch(struct i915_address_space *vm) |
869 | 1964 | { |
|
870 | kfree(ppgtt->pt_dma_addr); |
1965 | struct drm_device *dev = vm->dev; |
871 | for (i = 0; i < ppgtt->num_pd_entries; i++) |
1966 | |
- | 1967 | free_pt(dev, vm->scratch_pt); |
|
- | 1968 | free_scratch_page(dev, vm->scratch_page); |
|
Line 872... | Line 1969... | ||
872 | __free_page(ppgtt->pt_pages[i]); |
1969 | } |
Line 873... | Line 1970... | ||
873 | kfree(ppgtt->pt_pages); |
1970 | |
- | 1971 | static void gen6_ppgtt_cleanup(struct i915_address_space *vm) |
|
- | 1972 | { |
|
- | 1973 | struct i915_hw_ppgtt *ppgtt = |
|
- | 1974 | container_of(vm, struct i915_hw_ppgtt, base); |
|
874 | } |
1975 | struct i915_page_table *pt; |
875 | 1976 | uint32_t pde; |
|
Line 876... | Line 1977... | ||
876 | static void gen6_ppgtt_cleanup(struct i915_address_space *vm) |
1977 | |
877 | { |
1978 | drm_mm_remove_node(&ppgtt->node); |
- | 1979 | ||
878 | struct i915_hw_ppgtt *ppgtt = |
1980 | gen6_for_all_pdes(pt, ppgtt, pde) { |
879 | container_of(vm, struct i915_hw_ppgtt, base); |
1981 | if (pt != vm->scratch_pt) |
880 | 1982 | free_pt(ppgtt->base.dev, pt); |
|
881 | drm_mm_remove_node(&ppgtt->node); |
1983 | } |
Line 882... | Line 1984... | ||
882 | 1984 | ||
883 | gen6_ppgtt_unmap_pages(ppgtt); |
1985 | gen6_free_scratch(vm); |
884 | gen6_ppgtt_free(ppgtt); |
1986 | } |
885 | } |
1987 | |
886 | 1988 | static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt) |
|
- | 1989 | { |
|
- | 1990 | struct i915_address_space *vm = &ppgtt->base; |
|
- | 1991 | struct drm_device *dev = ppgtt->base.dev; |
|
- | 1992 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
- | 1993 | bool retried = false; |
|
887 | static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt) |
1994 | int ret; |
888 | { |
1995 | |
889 | struct drm_device *dev = ppgtt->base.dev; |
1996 | /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The |
890 | struct drm_i915_private *dev_priv = dev->dev_private; |
1997 | * allocator works in address space sizes, so it's multiplied by page |
891 | bool retried = false; |
1998 | * size. We allocate at the top of the GTT to avoid fragmentation. |
Line 907... | Line 2014... | ||
907 | GEN6_PD_SIZE, GEN6_PD_ALIGN, |
2014 | GEN6_PD_SIZE, GEN6_PD_ALIGN, |
908 | I915_CACHE_NONE, |
2015 | I915_CACHE_NONE, |
909 | 0, dev_priv->gtt.base.total, |
2016 | 0, dev_priv->gtt.base.total, |
910 | 0); |
2017 | 0); |
911 | if (ret) |
2018 | if (ret) |
912 | return ret; |
2019 | goto err_out; |
Line 913... | Line 2020... | ||
913 | 2020 | ||
914 | retried = true; |
2021 | retried = true; |
915 | goto alloc; |
2022 | goto alloc; |
Line 916... | Line -... | ||
916 | } |
- | |
917 | - | ||
918 | if (ppgtt->node.start < dev_priv->gtt.mappable_end) |
- | |
919 | DRM_DEBUG("Forced to use aperture for PDEs\n"); |
- | |
920 | 2023 | } |
|
921 | ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES; |
- | |
922 | return ret; |
- | |
923 | } |
- | |
924 | - | ||
925 | static int gen6_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt) |
- | |
926 | { |
- | |
927 | int i; |
- | |
928 | 2024 | ||
Line 929... | Line -... | ||
929 | ppgtt->pt_pages = kcalloc(ppgtt->num_pd_entries, sizeof(struct page *), |
- | |
930 | GFP_KERNEL); |
- | |
Line 931... | Line 2025... | ||
931 | 2025 | if (ret) |
|
932 | if (!ppgtt->pt_pages) |
2026 | goto err_out; |
933 | return -ENOMEM; |
- | |
934 | - | ||
935 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
- | |
936 | ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL); |
- | |
937 | if (!ppgtt->pt_pages[i]) { |
- | |
Line 938... | Line 2027... | ||
938 | gen6_ppgtt_free(ppgtt); |
2027 | |
939 | return -ENOMEM; |
- | |
Line 940... | Line -... | ||
940 | } |
- | |
941 | } |
- | |
942 | 2028 | ||
943 | return 0; |
- | |
944 | } |
2029 | if (ppgtt->node.start < dev_priv->gtt.mappable_end) |
945 | - | ||
946 | static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt) |
2030 | DRM_DEBUG("Forced to use aperture for PDEs\n"); |
947 | { |
- | |
948 | int ret; |
- | |
949 | - | ||
950 | ret = gen6_ppgtt_allocate_page_directories(ppgtt); |
- | |
951 | if (ret) |
- | |
952 | return ret; |
- | |
953 | - | ||
954 | ret = gen6_ppgtt_allocate_page_tables(ppgtt); |
- | |
955 | if (ret) { |
- | |
956 | drm_mm_remove_node(&ppgtt->node); |
- | |
957 | return ret; |
- | |
958 | } |
- | |
959 | - | ||
960 | ppgtt->pt_dma_addr = kcalloc(ppgtt->num_pd_entries, sizeof(dma_addr_t), |
2031 | |
Line -... | Line 2032... | ||
- | 2032 | return 0; |
|
- | 2033 | ||
961 | GFP_KERNEL); |
2034 | err_out: |
962 | if (!ppgtt->pt_dma_addr) { |
2035 | gen6_free_scratch(vm); |
Line 963... | Line 2036... | ||
963 | drm_mm_remove_node(&ppgtt->node); |
2036 | return ret; |
- | 2037 | } |
|
964 | gen6_ppgtt_free(ppgtt); |
2038 | |
965 | return -ENOMEM; |
2039 | static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt) |
966 | } |
- | |
967 | - | ||
968 | return 0; |
- | |
969 | } |
2040 | { |
Line 970... | Line 2041... | ||
970 | 2041 | return gen6_ppgtt_allocate_page_directories(ppgtt); |
|
971 | static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt) |
- | |
972 | { |
- | |
973 | struct drm_device *dev = ppgtt->base.dev; |
- | |
974 | int i; |
- | |
975 | - | ||
976 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
- | |
977 | dma_addr_t pt_addr; |
- | |
978 | 2042 | } |
|
979 | pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096, |
- | |
980 | PCI_DMA_BIDIRECTIONAL); |
- | |
981 | - | ||
982 | // if (pci_dma_mapping_error(dev->pdev, pt_addr)) { |
2043 | |
Line 983... | Line 2044... | ||
983 | // gen6_ppgtt_unmap_pages(ppgtt); |
2044 | static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt, |
984 | // return -EIO; |
2045 | uint64_t start, uint64_t length) |
985 | // } |
2046 | { |
Line 1004... | Line 2065... | ||
1004 | } else if (IS_GEN7(dev)) { |
2065 | } else if (IS_GEN7(dev)) { |
1005 | ppgtt->switch_mm = gen7_mm_switch; |
2066 | ppgtt->switch_mm = gen7_mm_switch; |
1006 | } else |
2067 | } else |
1007 | BUG(); |
2068 | BUG(); |
Line -... | Line 2069... | ||
- | 2069 | ||
- | 2070 | if (intel_vgpu_active(dev)) |
|
- | 2071 | ppgtt->switch_mm = vgpu_mm_switch; |
|
1008 | 2072 | ||
1009 | ret = gen6_ppgtt_alloc(ppgtt); |
2073 | ret = gen6_ppgtt_alloc(ppgtt); |
1010 | if (ret) |
2074 | if (ret) |
Line 1011... | Line 2075... | ||
1011 | return ret; |
2075 | return ret; |
1012 | - | ||
1013 | ret = gen6_ppgtt_setup_page_tables(ppgtt); |
- | |
1014 | if (ret) { |
- | |
1015 | gen6_ppgtt_free(ppgtt); |
- | |
1016 | return ret; |
- | |
1017 | } |
2076 | |
1018 | 2077 | ppgtt->base.allocate_va_range = gen6_alloc_va_range; |
|
- | 2078 | ppgtt->base.clear_range = gen6_ppgtt_clear_range; |
|
- | 2079 | ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; |
|
1019 | ppgtt->base.clear_range = gen6_ppgtt_clear_range; |
2080 | ppgtt->base.unbind_vma = ppgtt_unbind_vma; |
1020 | ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; |
2081 | ppgtt->base.bind_vma = ppgtt_bind_vma; |
1021 | ppgtt->base.cleanup = gen6_ppgtt_cleanup; |
2082 | ppgtt->base.cleanup = gen6_ppgtt_cleanup; |
1022 | ppgtt->base.start = 0; |
2083 | ppgtt->base.start = 0; |
- | 2084 | ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE; |
|
- | 2085 | ppgtt->debug_dump = gen6_dump_ppgtt; |
|
- | 2086 | ||
Line 1023... | Line 2087... | ||
1023 | ppgtt->base.total = ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES * PAGE_SIZE; |
2087 | ppgtt->pd.base.ggtt_offset = |
1024 | // ppgtt->debug_dump = gen6_dump_ppgtt; |
2088 | ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t); |
Line 1025... | Line 2089... | ||
1025 | 2089 | ||
Line -... | Line 2090... | ||
- | 2090 | ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm + |
|
- | 2091 | ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t); |
|
1026 | ppgtt->pd_offset = |
2092 | |
1027 | ppgtt->node.start / PAGE_SIZE * sizeof(gen6_gtt_pte_t); |
2093 | gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total); |
1028 | 2094 | ||
Line 1029... | Line -... | ||
1029 | ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true); |
- | |
1030 | 2095 | gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total); |
|
1031 | DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n", |
2096 | |
Line 1032... | Line 2097... | ||
1032 | ppgtt->node.size >> 20, |
2097 | DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n", |
1033 | ppgtt->node.start / PAGE_SIZE); |
2098 | ppgtt->node.size >> 20, |
Line 1034... | Line 2099... | ||
1034 | 2099 | ppgtt->node.start / PAGE_SIZE); |
|
1035 | gen6_write_pdes(ppgtt); |
2100 | |
1036 | DRM_DEBUG("Adding PPGTT at offset %x\n", |
- | |
1037 | ppgtt->pd_offset << 10); |
- | |
1038 | 2101 | DRM_DEBUG("Adding PPGTT at offset %x\n", |
|
1039 | return 0; |
- | |
Line 1040... | Line 2102... | ||
1040 | } |
2102 | ppgtt->pd.base.ggtt_offset << 10); |
1041 | 2103 | ||
1042 | static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) |
- | |
1043 | { |
- | |
1044 | struct drm_i915_private *dev_priv = dev->dev_private; |
2104 | return 0; |
1045 | 2105 | } |
|
- | 2106 | ||
- | 2107 | static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) |
|
- | 2108 | { |
|
- | 2109 | ppgtt->base.dev = dev; |
|
- | 2110 | ||
- | 2111 | if (INTEL_INFO(dev)->gen < 8) |
|
- | 2112 | return gen6_ppgtt_init(ppgtt); |
|
- | 2113 | else |
|
- | 2114 | return gen8_ppgtt_init(ppgtt); |
|
- | 2115 | } |
|
1046 | ppgtt->base.dev = dev; |
2116 | |
- | 2117 | static void i915_address_space_init(struct i915_address_space *vm, |
|
1047 | ppgtt->base.scratch = dev_priv->gtt.base.scratch; |
2118 | struct drm_i915_private *dev_priv) |
1048 | 2119 | { |
|
1049 | if (INTEL_INFO(dev)->gen < 8) |
2120 | drm_mm_init(&vm->mm, vm->start, vm->total); |
1050 | return gen6_ppgtt_init(ppgtt); |
2121 | vm->dev = dev_priv->dev; |
Line 1051... | Line 2122... | ||
1051 | else if (IS_GEN8(dev) || IS_GEN9(dev)) |
2122 | INIT_LIST_HEAD(&vm->active_list); |
1052 | return gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total); |
2123 | INIT_LIST_HEAD(&vm->inactive_list); |
1053 | else |
2124 | list_add_tail(&vm->global_link, &dev_priv->vm_list); |
1054 | BUG(); |
- | |
1055 | } |
- | |
1056 | int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) |
2125 | } |
1057 | { |
2126 | |
Line 1058... | Line 2127... | ||
1058 | struct drm_i915_private *dev_priv = dev->dev_private; |
2127 | int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) |
1059 | int ret = 0; |
2128 | { |
Line 1060... | Line 2129... | ||
1060 | 2129 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
1061 | ret = __hw_ppgtt_init(dev, ppgtt); |
2130 | int ret = 0; |
1062 | if (ret == 0) { |
- | |
1063 | kref_init(&ppgtt->ref); |
- | |
1064 | drm_mm_init(&ppgtt->base.mm, ppgtt->base.start, |
- | |
1065 | ppgtt->base.total); |
- | |
1066 | i915_init_vm(dev_priv, &ppgtt->base); |
- | |
1067 | } |
2131 | |
1068 | 2132 | ret = __hw_ppgtt_init(dev, ppgtt); |
|
1069 | return ret; |
2133 | if (ret == 0) { |
1070 | } |
2134 | kref_init(&ppgtt->ref); |
1071 | 2135 | i915_address_space_init(&ppgtt->base, dev_priv); |
|
Line 1090... | Line 2154... | ||
1090 | else if (IS_GEN7(dev)) |
2154 | else if (IS_GEN7(dev)) |
1091 | gen7_ppgtt_enable(dev); |
2155 | gen7_ppgtt_enable(dev); |
1092 | else if (INTEL_INFO(dev)->gen >= 8) |
2156 | else if (INTEL_INFO(dev)->gen >= 8) |
1093 | gen8_ppgtt_enable(dev); |
2157 | gen8_ppgtt_enable(dev); |
1094 | else |
2158 | else |
1095 | WARN_ON(1); |
2159 | MISSING_CASE(INTEL_INFO(dev)->gen); |
Line 1096... | Line -... | ||
1096 | - | ||
1097 | if (ppgtt) { |
- | |
1098 | for_each_ring(ring, dev_priv, i) { |
- | |
1099 | ret = ppgtt->switch_mm(ppgtt, ring); |
- | |
1100 | if (ret != 0) |
2160 | |
1101 | return ret; |
- | |
1102 | } |
2161 | return 0; |
Line -... | Line 2162... | ||
- | 2162 | } |
|
- | 2163 | ||
- | 2164 | int i915_ppgtt_init_ring(struct drm_i915_gem_request *req) |
|
- | 2165 | { |
|
- | 2166 | struct drm_i915_private *dev_priv = req->ring->dev->dev_private; |
|
- | 2167 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
|
1103 | } |
2168 | |
- | 2169 | if (i915.enable_execlists) |
|
- | 2170 | return 0; |
|
- | 2171 | ||
- | 2172 | if (!ppgtt) |
|
- | 2173 | return 0; |
|
1104 | 2174 | ||
- | 2175 | return ppgtt->switch_mm(ppgtt, req); |
|
1105 | return ret; |
2176 | } |
1106 | } |
2177 | |
1107 | struct i915_hw_ppgtt * |
2178 | struct i915_hw_ppgtt * |
1108 | i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv) |
2179 | i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv) |
1109 | { |
2180 | { |
Line 1143... | Line 2214... | ||
1143 | 2214 | ||
1144 | ppgtt->base.cleanup(&ppgtt->base); |
2215 | ppgtt->base.cleanup(&ppgtt->base); |
1145 | kfree(ppgtt); |
2216 | kfree(ppgtt); |
Line 1146... | Line -... | ||
1146 | } |
- | |
1147 | - | ||
1148 | static void |
- | |
1149 | ppgtt_bind_vma(struct i915_vma *vma, |
- | |
1150 | enum i915_cache_level cache_level, |
- | |
1151 | u32 flags) |
- | |
1152 | { |
- | |
1153 | /* Currently applicable only to VLV */ |
- | |
1154 | if (vma->obj->gt_ro) |
- | |
1155 | flags |= PTE_READ_ONLY; |
- | |
1156 | - | ||
1157 | vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start, |
- | |
1158 | cache_level, flags); |
- | |
1159 | } |
- | |
1160 | - | ||
1161 | static void ppgtt_unbind_vma(struct i915_vma *vma) |
- | |
1162 | { |
- | |
1163 | vma->vm->clear_range(vma->vm, |
- | |
1164 | vma->node.start, |
- | |
1165 | vma->obj->base.size, |
- | |
1166 | true); |
- | |
1167 | } |
2217 | } |
1168 | 2218 | ||
1169 | extern int intel_iommu_gfx_mapped; |
2219 | extern int intel_iommu_gfx_mapped; |
1170 | /* Certain Gen5 chipsets require require idling the GPU before |
2220 | /* Certain Gen5 chipsets require require idling the GPU before |
1171 | * unmapping anything from the GTT when VT-d is enabled. |
2221 | * unmapping anything from the GTT when VT-d is enabled. |
1172 | */ |
2222 | */ |
1173 | static inline bool needs_idle_maps(struct drm_device *dev) |
2223 | static bool needs_idle_maps(struct drm_device *dev) |
1174 | { |
2224 | { |
1175 | #ifdef CONFIG_INTEL_IOMMU |
2225 | #ifdef CONFIG_INTEL_IOMMU |
1176 | /* Query intel_iommu to see if we need the workaround. Presumably that |
2226 | /* Query intel_iommu to see if we need the workaround. Presumably that |
Line 1261... | Line 2311... | ||
1261 | true); |
2311 | true); |
Line 1262... | Line 2312... | ||
1262 | 2312 | ||
1263 | i915_ggtt_flush(dev_priv); |
2313 | i915_ggtt_flush(dev_priv); |
Line 1264... | Line -... | ||
1264 | } |
- | |
1265 | - | ||
1266 | void i915_gem_restore_gtt_mappings(struct drm_device *dev) |
- | |
1267 | { |
- | |
1268 | struct drm_i915_private *dev_priv = dev->dev_private; |
- | |
1269 | struct drm_i915_gem_object *obj; |
- | |
1270 | struct i915_address_space *vm; |
- | |
1271 | - | ||
1272 | i915_check_and_clear_faults(dev); |
- | |
1273 | - | ||
1274 | /* First fill our portion of the GTT with scratch pages */ |
- | |
1275 | dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, |
- | |
1276 | dev_priv->gtt.base.start, |
- | |
1277 | dev_priv->gtt.base.total, |
- | |
1278 | true); |
- | |
1279 | - | ||
1280 | list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { |
- | |
1281 | struct i915_vma *vma = i915_gem_obj_to_vma(obj, |
- | |
1282 | &dev_priv->gtt.base); |
- | |
1283 | if (!vma) |
- | |
1284 | continue; |
- | |
1285 | - | ||
1286 | i915_gem_clflush_object(obj, obj->pin_display); |
- | |
1287 | /* The bind_vma code tries to be smart about tracking mappings. |
- | |
1288 | * Unfortunately above, we've just wiped out the mappings |
- | |
1289 | * without telling our object about it. So we need to fake it. |
- | |
1290 | */ |
- | |
1291 | vma->bound &= ~GLOBAL_BIND; |
- | |
1292 | vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND); |
- | |
1293 | } |
- | |
1294 | - | ||
1295 | - | ||
1296 | if (INTEL_INFO(dev)->gen >= 8) { |
- | |
1297 | if (IS_CHERRYVIEW(dev)) |
- | |
1298 | chv_setup_private_ppat(dev_priv); |
- | |
1299 | else |
- | |
1300 | bdw_setup_private_ppat(dev_priv); |
- | |
1301 | - | ||
1302 | return; |
- | |
1303 | } |
- | |
1304 | - | ||
1305 | list_for_each_entry(vm, &dev_priv->vm_list, global_link) { |
- | |
1306 | /* TODO: Perhaps it shouldn't be gen6 specific */ |
- | |
1307 | if (i915_is_ggtt(vm)) { |
- | |
1308 | if (dev_priv->mm.aliasing_ppgtt) |
- | |
1309 | gen6_write_pdes(dev_priv->mm.aliasing_ppgtt); |
- | |
1310 | continue; |
- | |
1311 | } |
- | |
1312 | - | ||
1313 | gen6_write_pdes(container_of(vm, struct i915_hw_ppgtt, base)); |
- | |
1314 | } |
- | |
1315 | - | ||
1316 | i915_ggtt_flush(dev_priv); |
- | |
1317 | } |
2314 | } |
1318 | 2315 | ||
1319 | int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) |
- | |
1320 | { |
- | |
1321 | if (obj->has_dma_mapping) |
- | |
1322 | return 0; |
2316 | int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) |
1323 | 2317 | { |
|
1324 | if (!dma_map_sg(&obj->base.dev->pdev->dev, |
2318 | if (!dma_map_sg(&obj->base.dev->pdev->dev, |
1325 | obj->pages->sgl, obj->pages->nents, |
2319 | obj->pages->sgl, obj->pages->nents, |
Line 1326... | Line 2320... | ||
1326 | PCI_DMA_BIDIRECTIONAL)) |
2320 | PCI_DMA_BIDIRECTIONAL)) |
1327 | return -ENOSPC; |
2321 | return -ENOSPC; |
Line 1328... | Line 2322... | ||
1328 | 2322 | ||
1329 | return 0; |
2323 | return 0; |
1330 | } |
2324 | } |
1331 | 2325 | ||
1332 | static inline void gen8_set_pte(void __iomem *addr, gen8_gtt_pte_t pte) |
2326 | static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) |
1333 | { |
2327 | { |
Line 1344... | Line 2338... | ||
1344 | uint64_t start, |
2338 | uint64_t start, |
1345 | enum i915_cache_level level, u32 unused) |
2339 | enum i915_cache_level level, u32 unused) |
1346 | { |
2340 | { |
1347 | struct drm_i915_private *dev_priv = vm->dev->dev_private; |
2341 | struct drm_i915_private *dev_priv = vm->dev->dev_private; |
1348 | unsigned first_entry = start >> PAGE_SHIFT; |
2342 | unsigned first_entry = start >> PAGE_SHIFT; |
1349 | gen8_gtt_pte_t __iomem *gtt_entries = |
2343 | gen8_pte_t __iomem *gtt_entries = |
1350 | (gen8_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; |
2344 | (gen8_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; |
1351 | int i = 0; |
2345 | int i = 0; |
1352 | struct sg_page_iter sg_iter; |
2346 | struct sg_page_iter sg_iter; |
1353 | dma_addr_t addr = 0; /* shut up gcc */ |
2347 | dma_addr_t addr = 0; /* shut up gcc */ |
Line 1354... | Line 2348... | ||
1354 | 2348 | ||
Line 1390... | Line 2384... | ||
1390 | uint64_t start, |
2384 | uint64_t start, |
1391 | enum i915_cache_level level, u32 flags) |
2385 | enum i915_cache_level level, u32 flags) |
1392 | { |
2386 | { |
1393 | struct drm_i915_private *dev_priv = vm->dev->dev_private; |
2387 | struct drm_i915_private *dev_priv = vm->dev->dev_private; |
1394 | unsigned first_entry = start >> PAGE_SHIFT; |
2388 | unsigned first_entry = start >> PAGE_SHIFT; |
1395 | gen6_gtt_pte_t __iomem *gtt_entries = |
2389 | gen6_pte_t __iomem *gtt_entries = |
1396 | (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; |
2390 | (gen6_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; |
1397 | int i = 0; |
2391 | int i = 0; |
1398 | struct sg_page_iter sg_iter; |
2392 | struct sg_page_iter sg_iter; |
1399 | dma_addr_t addr = 0; |
2393 | dma_addr_t addr = 0; |
Line 1400... | Line 2394... | ||
1400 | 2394 | ||
Line 1429... | Line 2423... | ||
1429 | bool use_scratch) |
2423 | bool use_scratch) |
1430 | { |
2424 | { |
1431 | struct drm_i915_private *dev_priv = vm->dev->dev_private; |
2425 | struct drm_i915_private *dev_priv = vm->dev->dev_private; |
1432 | unsigned first_entry = start >> PAGE_SHIFT; |
2426 | unsigned first_entry = start >> PAGE_SHIFT; |
1433 | unsigned num_entries = length >> PAGE_SHIFT; |
2427 | unsigned num_entries = length >> PAGE_SHIFT; |
1434 | gen8_gtt_pte_t scratch_pte, __iomem *gtt_base = |
2428 | gen8_pte_t scratch_pte, __iomem *gtt_base = |
1435 | (gen8_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; |
2429 | (gen8_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; |
1436 | const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; |
2430 | const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; |
1437 | int i; |
2431 | int i; |
Line 1438... | Line 2432... | ||
1438 | 2432 | ||
1439 | if (WARN(num_entries > max_entries, |
2433 | if (WARN(num_entries > max_entries, |
1440 | "First entry = %d; Num entries = %d (max=%d)\n", |
2434 | "First entry = %d; Num entries = %d (max=%d)\n", |
1441 | first_entry, num_entries, max_entries)) |
2435 | first_entry, num_entries, max_entries)) |
Line 1442... | Line 2436... | ||
1442 | num_entries = max_entries; |
2436 | num_entries = max_entries; |
1443 | 2437 | ||
1444 | scratch_pte = gen8_pte_encode(vm->scratch.addr, |
2438 | scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page), |
1445 | I915_CACHE_LLC, |
2439 | I915_CACHE_LLC, |
1446 | use_scratch); |
2440 | use_scratch); |
1447 | for (i = 0; i < num_entries; i++) |
2441 | for (i = 0; i < num_entries; i++) |
Line 1455... | Line 2449... | ||
1455 | bool use_scratch) |
2449 | bool use_scratch) |
1456 | { |
2450 | { |
1457 | struct drm_i915_private *dev_priv = vm->dev->dev_private; |
2451 | struct drm_i915_private *dev_priv = vm->dev->dev_private; |
1458 | unsigned first_entry = start >> PAGE_SHIFT; |
2452 | unsigned first_entry = start >> PAGE_SHIFT; |
1459 | unsigned num_entries = length >> PAGE_SHIFT; |
2453 | unsigned num_entries = length >> PAGE_SHIFT; |
1460 | gen6_gtt_pte_t scratch_pte, __iomem *gtt_base = |
2454 | gen6_pte_t scratch_pte, __iomem *gtt_base = |
1461 | (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; |
2455 | (gen6_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; |
1462 | const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; |
2456 | const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; |
1463 | int i; |
2457 | int i; |
Line 1464... | Line 2458... | ||
1464 | 2458 | ||
1465 | if (WARN(num_entries > max_entries, |
2459 | if (WARN(num_entries > max_entries, |
1466 | "First entry = %d; Num entries = %d (max=%d)\n", |
2460 | "First entry = %d; Num entries = %d (max=%d)\n", |
1467 | first_entry, num_entries, max_entries)) |
2461 | first_entry, num_entries, max_entries)) |
Line 1468... | Line 2462... | ||
1468 | num_entries = max_entries; |
2462 | num_entries = max_entries; |
- | 2463 | ||
Line 1469... | Line 2464... | ||
1469 | 2464 | scratch_pte = vm->pte_encode(px_dma(vm->scratch_page), |
|
1470 | scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch, 0); |
2465 | I915_CACHE_LLC, use_scratch, 0); |
1471 | 2466 | ||
1472 | for (i = 0; i < num_entries; i++) |
2467 | for (i = 0; i < num_entries; i++) |
Line 1473... | Line -... | ||
1473 | iowrite32(scratch_pte, >t_base[i]); |
- | |
1474 | readl(gtt_base); |
2468 | iowrite32(scratch_pte, >t_base[i]); |
1475 | } |
2469 | readl(gtt_base); |
1476 | 2470 | } |
|
- | 2471 | ||
1477 | 2472 | static void i915_ggtt_insert_entries(struct i915_address_space *vm, |
|
1478 | static void i915_ggtt_bind_vma(struct i915_vma *vma, |
- | |
1479 | enum i915_cache_level cache_level, |
2473 | struct sg_table *pages, |
1480 | u32 unused) |
2474 | uint64_t start, |
Line 1481... | Line -... | ||
1481 | { |
- | |
1482 | const unsigned long entry = vma->node.start >> PAGE_SHIFT; |
2475 | enum i915_cache_level cache_level, u32 unused) |
1483 | unsigned int flags = (cache_level == I915_CACHE_NONE) ? |
- | |
- | 2476 | { |
|
1484 | AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; |
2477 | unsigned int flags = (cache_level == I915_CACHE_NONE) ? |
Line 1485... | Line 2478... | ||
1485 | 2478 | AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; |
|
1486 | BUG_ON(!i915_is_ggtt(vma->vm)); |
2479 | |
1487 | intel_gtt_insert_sg_entries(vma->obj->pages, entry, flags); |
2480 | intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags); |
Line 1496... | Line 2489... | ||
1496 | unsigned first_entry = start >> PAGE_SHIFT; |
2489 | unsigned first_entry = start >> PAGE_SHIFT; |
1497 | unsigned num_entries = length >> PAGE_SHIFT; |
2490 | unsigned num_entries = length >> PAGE_SHIFT; |
1498 | intel_gtt_clear_range(first_entry, num_entries); |
2491 | intel_gtt_clear_range(first_entry, num_entries); |
1499 | } |
2492 | } |
Line 1500... | Line 2493... | ||
1500 | 2493 | ||
- | 2494 | static int ggtt_bind_vma(struct i915_vma *vma, |
|
- | 2495 | enum i915_cache_level cache_level, |
|
1501 | static void i915_ggtt_unbind_vma(struct i915_vma *vma) |
2496 | u32 flags) |
1502 | { |
2497 | { |
1503 | const unsigned int first = vma->node.start >> PAGE_SHIFT; |
2498 | struct drm_i915_gem_object *obj = vma->obj; |
- | 2499 | u32 pte_flags = 0; |
|
Line 1504... | Line 2500... | ||
1504 | const unsigned int size = vma->obj->base.size >> PAGE_SHIFT; |
2500 | int ret; |
- | 2501 | ||
- | 2502 | ret = i915_get_ggtt_vma_pages(vma); |
|
- | 2503 | if (ret) |
|
- | 2504 | return ret; |
|
- | 2505 | ||
- | 2506 | /* Currently applicable only to VLV */ |
|
- | 2507 | if (obj->gt_ro) |
|
- | 2508 | pte_flags |= PTE_READ_ONLY; |
|
1505 | 2509 | ||
1506 | BUG_ON(!i915_is_ggtt(vma->vm)); |
2510 | vma->vm->insert_entries(vma->vm, vma->ggtt_view.pages, |
- | 2511 | vma->node.start, |
|
- | 2512 | cache_level, pte_flags); |
|
- | 2513 | ||
- | 2514 | /* |
|
- | 2515 | * Without aliasing PPGTT there's no difference between |
|
- | 2516 | * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally |
|
- | 2517 | * upgrade to both bound if we bind either to avoid double-binding. |
|
- | 2518 | */ |
|
- | 2519 | vma->bound |= GLOBAL_BIND | LOCAL_BIND; |
|
1507 | vma->bound = 0; |
2520 | |
Line 1508... | Line 2521... | ||
1508 | intel_gtt_clear_range(first, size); |
2521 | return 0; |
1509 | } |
2522 | } |
1510 | 2523 | ||
1511 | static void ggtt_bind_vma(struct i915_vma *vma, |
2524 | static int aliasing_gtt_bind_vma(struct i915_vma *vma, |
1512 | enum i915_cache_level cache_level, |
2525 | enum i915_cache_level cache_level, |
1513 | u32 flags) |
2526 | u32 flags) |
1514 | { |
2527 | { |
- | 2528 | struct drm_device *dev = vma->vm->dev; |
|
- | 2529 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
- | 2530 | struct drm_i915_gem_object *obj = vma->obj; |
|
- | 2531 | struct sg_table *pages = obj->pages; |
|
- | 2532 | u32 pte_flags = 0; |
|
- | 2533 | int ret; |
|
- | 2534 | ||
- | 2535 | ret = i915_get_ggtt_vma_pages(vma); |
|
Line 1515... | Line 2536... | ||
1515 | struct drm_device *dev = vma->vm->dev; |
2536 | if (ret) |
1516 | struct drm_i915_private *dev_priv = dev->dev_private; |
2537 | return ret; |
1517 | struct drm_i915_gem_object *obj = vma->obj; |
2538 | pages = vma->ggtt_view.pages; |
- | 2539 | ||
Line 1518... | Line -... | ||
1518 | - | ||
1519 | /* Currently applicable only to VLV */ |
- | |
1520 | if (obj->gt_ro) |
- | |
1521 | flags |= PTE_READ_ONLY; |
- | |
1522 | - | ||
1523 | /* If there is no aliasing PPGTT, or the caller needs a global mapping, |
- | |
1524 | * or we have a global mapping already but the cacheability flags have |
- | |
1525 | * changed, set the global PTEs. |
- | |
1526 | * |
- | |
1527 | * If there is an aliasing PPGTT it is anecdotally faster, so use that |
- | |
1528 | * instead if none of the above hold true. |
- | |
1529 | * |
- | |
1530 | * NB: A global mapping should only be needed for special regions like |
2540 | /* Currently applicable only to VLV */ |
1531 | * "gtt mappable", SNB errata, or if specified via special execbuf |
- | |
1532 | * flags. At all other times, the GPU will use the aliasing PPGTT. |
2541 | if (obj->gt_ro) |
1533 | */ |
2542 | pte_flags |= PTE_READ_ONLY; |
1534 | if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) { |
2543 | |
1535 | if (!(vma->bound & GLOBAL_BIND) || |
- | |
1536 | (cache_level != obj->cache_level)) { |
- | |
1537 | vma->vm->insert_entries(vma->vm, obj->pages, |
2544 | |
Line 1538... | Line -... | ||
1538 | vma->node.start, |
- | |
1539 | cache_level, flags); |
2545 | if (flags & GLOBAL_BIND) { |
1540 | vma->bound |= GLOBAL_BIND; |
- | |
1541 | } |
2546 | vma->vm->insert_entries(vma->vm, pages, |
1542 | } |
2547 | vma->node.start, |
1543 | - | ||
1544 | if (dev_priv->mm.aliasing_ppgtt && |
2548 | cache_level, pte_flags); |
1545 | (!(vma->bound & LOCAL_BIND) || |
2549 | } |
1546 | (cache_level != obj->cache_level))) { |
- | |
1547 | struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; |
2550 | |
- | 2551 | if (flags & LOCAL_BIND) { |
|
- | 2552 | struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; |
|
1548 | appgtt->base.insert_entries(&appgtt->base, |
2553 | appgtt->base.insert_entries(&appgtt->base, pages, |
Line 1549... | Line 2554... | ||
1549 | vma->obj->pages, |
2554 | vma->node.start, |
1550 | vma->node.start, |
2555 | cache_level, pte_flags); |
1551 | cache_level, flags); |
2556 | } |
1552 | vma->bound |= LOCAL_BIND; |
2557 | |
1553 | } |
2558 | return 0; |
- | 2559 | } |
|
- | 2560 | ||
- | 2561 | static void ggtt_unbind_vma(struct i915_vma *vma) |
|
Line 1554... | Line 2562... | ||
1554 | } |
2562 | { |
1555 | 2563 | struct drm_device *dev = vma->vm->dev; |
|
1556 | static void ggtt_unbind_vma(struct i915_vma *vma) |
2564 | struct drm_i915_private *dev_priv = dev->dev_private; |
1557 | { |
2565 | struct drm_i915_gem_object *obj = vma->obj; |
1558 | struct drm_device *dev = vma->vm->dev; |
2566 | const uint64_t size = min_t(uint64_t, |
1559 | struct drm_i915_private *dev_priv = dev->dev_private; |
- | |
1560 | struct drm_i915_gem_object *obj = vma->obj; |
2567 | obj->base.size, |
Line 1561... | Line 2568... | ||
1561 | 2568 | vma->node.size); |
|
1562 | if (vma->bound & GLOBAL_BIND) { |
2569 | |
- | 2570 | if (vma->bound & GLOBAL_BIND) { |
|
1563 | vma->vm->clear_range(vma->vm, |
2571 | vma->vm->clear_range(vma->vm, |
1564 | vma->node.start, |
2572 | vma->node.start, |
1565 | obj->base.size, |
2573 | size, |
1566 | true); |
2574 | true); |
1567 | vma->bound &= ~GLOBAL_BIND; |
- | |
1568 | } |
2575 | } |
1569 | 2576 | ||
Line 1570... | Line 2577... | ||
1570 | if (vma->bound & LOCAL_BIND) { |
2577 | if (dev_priv->mm.aliasing_ppgtt && vma->bound & LOCAL_BIND) { |
1571 | struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; |
2578 | struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; |
Line 1583... | Line 2590... | ||
1583 | struct drm_i915_private *dev_priv = dev->dev_private; |
2590 | struct drm_i915_private *dev_priv = dev->dev_private; |
1584 | bool interruptible; |
2591 | bool interruptible; |
Line 1585... | Line 2592... | ||
1585 | 2592 | ||
Line 1586... | Line -... | ||
1586 | interruptible = do_idling(dev_priv); |
- | |
1587 | - | ||
1588 | if (!obj->has_dma_mapping) |
2593 | interruptible = do_idling(dev_priv); |
1589 | dma_unmap_sg(&dev->pdev->dev, |
2594 | |
Line 1590... | Line 2595... | ||
1590 | obj->pages->sgl, obj->pages->nents, |
2595 | dma_unmap_sg(&dev->pdev->dev, obj->pages->sgl, obj->pages->nents, |
1591 | PCI_DMA_BIDIRECTIONAL); |
2596 | PCI_DMA_BIDIRECTIONAL); |
Line 1592... | Line 2597... | ||
1592 | 2597 | ||
1593 | undo_idling(dev_priv, interruptible); |
2598 | undo_idling(dev_priv, interruptible); |
1594 | } |
2599 | } |
1595 | 2600 | ||
1596 | static void i915_gtt_color_adjust(struct drm_mm_node *node, |
2601 | static void i915_gtt_color_adjust(struct drm_mm_node *node, |
1597 | unsigned long color, |
2602 | unsigned long color, |
1598 | unsigned long *start, |
2603 | u64 *start, |
Line 1599... | Line 2604... | ||
1599 | unsigned long *end) |
2604 | u64 *end) |
Line 1609... | Line 2614... | ||
1609 | *end -= 4096; |
2614 | *end -= 4096; |
1610 | } |
2615 | } |
1611 | } |
2616 | } |
Line 1612... | Line 2617... | ||
1612 | 2617 | ||
1613 | static int i915_gem_setup_global_gtt(struct drm_device *dev, |
2618 | static int i915_gem_setup_global_gtt(struct drm_device *dev, |
1614 | unsigned long start, |
2619 | u64 start, |
1615 | unsigned long mappable_end, |
2620 | u64 mappable_end, |
1616 | unsigned long end) |
2621 | u64 end) |
1617 | { |
2622 | { |
1618 | /* Let GEM Manage all of the aperture. |
2623 | /* Let GEM Manage all of the aperture. |
1619 | * |
2624 | * |
1620 | * However, leave one page at the end still bound to the scratch page. |
2625 | * However, leave one page at the end still bound to the scratch page. |
Line 1631... | Line 2636... | ||
1631 | unsigned long hole_start, hole_end; |
2636 | unsigned long hole_start, hole_end; |
1632 | int ret; |
2637 | int ret; |
Line 1633... | Line 2638... | ||
1633 | 2638 | ||
Line -... | Line 2639... | ||
- | 2639 | BUG_ON(mappable_end > end); |
|
- | 2640 | ||
- | 2641 | ggtt_vm->start = start; |
|
1634 | BUG_ON(mappable_end > end); |
2642 | |
1635 | 2643 | /* Subtract the guard page before address space initialization to |
|
- | 2644 | * shrink the range used by drm_mm */ |
|
- | 2645 | ggtt_vm->total = end - start - PAGE_SIZE; |
|
- | 2646 | i915_address_space_init(ggtt_vm, dev_priv); |
|
- | 2647 | ggtt_vm->total += PAGE_SIZE; |
|
- | 2648 | ||
- | 2649 | if (intel_vgpu_active(dev)) { |
|
- | 2650 | ret = intel_vgt_balloon(dev); |
|
- | 2651 | if (ret) |
|
- | 2652 | return ret; |
|
1636 | /* Subtract the guard page ... */ |
2653 | } |
1637 | drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE); |
2654 | |
Line 1638... | Line 2655... | ||
1638 | if (!HAS_LLC(dev)) |
2655 | if (!HAS_LLC(dev)) |
1639 | dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust; |
2656 | ggtt_vm->mm.color_adjust = i915_gtt_color_adjust; |
1640 | 2657 | ||
Line 1641... | Line 2658... | ||
1641 | /* Mark any preallocated objects as occupied */ |
2658 | /* Mark any preallocated objects as occupied */ |
1642 | list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { |
2659 | list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { |
Line 1643... | Line 2660... | ||
1643 | struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm); |
2660 | struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm); |
1644 | 2661 | ||
1645 | DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n", |
2662 | DRM_DEBUG_KMS("reserving preallocated space: %llx + %zx\n", |
1646 | i915_gem_obj_ggtt_offset(obj), obj->base.size); |
2663 | i915_gem_obj_ggtt_offset(obj), obj->base.size); |
1647 | 2664 | ||
1648 | WARN_ON(i915_gem_obj_ggtt_bound(obj)); |
2665 | WARN_ON(i915_gem_obj_ggtt_bound(obj)); |
1649 | ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node); |
2666 | ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node); |
- | 2667 | if (ret) { |
|
- | 2668 | DRM_DEBUG_KMS("Reservation failed: %i\n", ret); |
|
1650 | if (ret) { |
2669 | return ret; |
Line 1651... | Line -... | ||
1651 | DRM_DEBUG_KMS("Reservation failed: %i\n", ret); |
- | |
1652 | return ret; |
- | |
1653 | } |
- | |
1654 | vma->bound |= GLOBAL_BIND; |
2670 | } |
1655 | } |
2671 | vma->bound |= GLOBAL_BIND; |
1656 | 2672 | __i915_vma_set_map_and_fenceable(vma); |
|
1657 | dev_priv->gtt.base.start = start; |
2673 | list_add_tail(&vma->mm_list, &ggtt_vm->inactive_list); |
1658 | dev_priv->gtt.base.total = end - start; |
2674 | } |
Line 1674... | Line 2690... | ||
1674 | ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); |
2690 | ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); |
1675 | if (!ppgtt) |
2691 | if (!ppgtt) |
1676 | return -ENOMEM; |
2692 | return -ENOMEM; |
Line 1677... | Line 2693... | ||
1677 | 2693 | ||
1678 | ret = __hw_ppgtt_init(dev, ppgtt); |
2694 | ret = __hw_ppgtt_init(dev, ppgtt); |
- | 2695 | if (ret) { |
|
- | 2696 | ppgtt->base.cleanup(&ppgtt->base); |
|
- | 2697 | kfree(ppgtt); |
|
- | 2698 | return ret; |
|
- | 2699 | } |
|
- | 2700 | ||
- | 2701 | if (ppgtt->base.allocate_va_range) |
|
- | 2702 | ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0, |
|
- | 2703 | ppgtt->base.total); |
|
- | 2704 | if (ret) { |
|
- | 2705 | ppgtt->base.cleanup(&ppgtt->base); |
|
1679 | if (ret != 0) |
2706 | kfree(ppgtt); |
- | 2707 | return ret; |
|
- | 2708 | } |
|
- | 2709 | ||
- | 2710 | ppgtt->base.clear_range(&ppgtt->base, |
|
- | 2711 | ppgtt->base.start, |
|
- | 2712 | ppgtt->base.total, |
|
Line 1680... | Line 2713... | ||
1680 | return ret; |
2713 | true); |
- | 2714 | ||
- | 2715 | dev_priv->mm.aliasing_ppgtt = ppgtt; |
|
1681 | 2716 | WARN_ON(dev_priv->gtt.base.bind_vma != ggtt_bind_vma); |
|
Line 1682... | Line 2717... | ||
1682 | dev_priv->mm.aliasing_ppgtt = ppgtt; |
2717 | dev_priv->gtt.base.bind_vma = aliasing_gtt_bind_vma; |
1683 | } |
2718 | } |
Line 1684... | Line 2719... | ||
1684 | 2719 | ||
1685 | return 0; |
2720 | return 0; |
1686 | } |
2721 | } |
1687 | 2722 | ||
Line 1688... | Line 2723... | ||
1688 | void i915_gem_init_global_gtt(struct drm_device *dev) |
2723 | void i915_gem_init_global_gtt(struct drm_device *dev) |
1689 | { |
2724 | { |
Line 1690... | Line 2725... | ||
1690 | struct drm_i915_private *dev_priv = dev->dev_private; |
2725 | struct drm_i915_private *dev_priv = dev->dev_private; |
Line 1706... | Line 2741... | ||
1706 | 2741 | ||
1707 | ppgtt->base.cleanup(&ppgtt->base); |
2742 | ppgtt->base.cleanup(&ppgtt->base); |
Line 1708... | Line 2743... | ||
1708 | } |
2743 | } |
- | 2744 | ||
- | 2745 | if (drm_mm_initialized(&vm->mm)) { |
|
- | 2746 | if (intel_vgpu_active(dev)) |
|
1709 | 2747 | intel_vgt_deballoon(); |
|
1710 | if (drm_mm_initialized(&vm->mm)) { |
2748 | |
1711 | drm_mm_takedown(&vm->mm); |
2749 | drm_mm_takedown(&vm->mm); |
Line 1712... | Line 2750... | ||
1712 | list_del(&vm->global_link); |
2750 | list_del(&vm->global_link); |
1713 | } |
2751 | } |
Line 1714... | Line -... | ||
1714 | - | ||
1715 | vm->cleanup(vm); |
- | |
1716 | } |
- | |
1717 | - | ||
1718 | static int setup_scratch_page(struct drm_device *dev) |
- | |
1719 | { |
- | |
1720 | struct drm_i915_private *dev_priv = dev->dev_private; |
- | |
1721 | struct page *page; |
- | |
1722 | dma_addr_t dma_addr; |
- | |
1723 | - | ||
1724 | page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO); |
- | |
1725 | if (page == NULL) |
- | |
1726 | return -ENOMEM; |
- | |
1727 | set_pages_uc(page, 1); |
- | |
1728 | - | ||
1729 | #ifdef CONFIG_INTEL_IOMMU |
- | |
1730 | dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE, |
- | |
1731 | PCI_DMA_BIDIRECTIONAL); |
- | |
1732 | if (pci_dma_mapping_error(dev->pdev, dma_addr)) |
- | |
1733 | return -EINVAL; |
- | |
1734 | #else |
- | |
1735 | dma_addr = page_to_phys(page); |
- | |
1736 | #endif |
- | |
1737 | dev_priv->gtt.base.scratch.page = page; |
- | |
1738 | dev_priv->gtt.base.scratch.addr = dma_addr; |
- | |
1739 | - | ||
1740 | return 0; |
- | |
1741 | } |
- | |
1742 | - | ||
1743 | static void teardown_scratch_page(struct drm_device *dev) |
- | |
1744 | { |
- | |
1745 | struct drm_i915_private *dev_priv = dev->dev_private; |
- | |
1746 | struct page *page = dev_priv->gtt.base.scratch.page; |
- | |
1747 | - | ||
1748 | set_pages_wb(page, 1); |
- | |
1749 | pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr, |
- | |
1750 | PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); |
2752 | |
1751 | __free_page(page); |
2753 | vm->cleanup(vm); |
1752 | } |
2754 | } |
1753 | 2755 | ||
1754 | static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) |
2756 | static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) |
1755 | { |
2757 | { |
Line 1756... | Line 2758... | ||
1756 | snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; |
2758 | snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; |
1757 | snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; |
2759 | snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; |
1758 | return snb_gmch_ctl << 20; |
2760 | return snb_gmch_ctl << 20; |
1759 | } |
2761 | } |
1760 | 2762 | ||
1761 | static inline unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) |
2763 | static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) |
Line 1772... | Line 2774... | ||
1772 | #endif |
2774 | #endif |
Line 1773... | Line 2775... | ||
1773 | 2775 | ||
1774 | return bdw_gmch_ctl << 20; |
2776 | return bdw_gmch_ctl << 20; |
Line 1775... | Line 2777... | ||
1775 | } |
2777 | } |
1776 | 2778 | ||
1777 | static inline unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) |
2779 | static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) |
1778 | { |
2780 | { |
Line 1779... | Line 2781... | ||
1779 | gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; |
2781 | gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; |
1780 | gmch_ctrl &= SNB_GMCH_GGMS_MASK; |
2782 | gmch_ctrl &= SNB_GMCH_GGMS_MASK; |
Line 1781... | Line 2783... | ||
1781 | 2783 | ||
1782 | if (gmch_ctrl) |
2784 | if (gmch_ctrl) |
Line 1783... | Line 2785... | ||
1783 | return 1 << (20 + gmch_ctrl); |
2785 | return 1 << (20 + gmch_ctrl); |
1784 | 2786 | ||
1785 | return 0; |
2787 | return 0; |
1786 | } |
2788 | } |
1787 | 2789 | ||
1788 | static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl) |
2790 | static size_t gen6_get_stolen_size(u16 snb_gmch_ctl) |
Line 1789... | Line 2791... | ||
1789 | { |
2791 | { |
1790 | snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; |
2792 | snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; |
1791 | snb_gmch_ctl &= SNB_GMCH_GMS_MASK; |
2793 | snb_gmch_ctl &= SNB_GMCH_GMS_MASK; |
1792 | return snb_gmch_ctl << 25; /* 32 MB units */ |
2794 | return snb_gmch_ctl << 25; /* 32 MB units */ |
1793 | } |
2795 | } |
1794 | 2796 | ||
Line 1831... | Line 2833... | ||
1831 | 2833 | ||
1832 | static int ggtt_probe_common(struct drm_device *dev, |
2834 | static int ggtt_probe_common(struct drm_device *dev, |
1833 | size_t gtt_size) |
2835 | size_t gtt_size) |
1834 | { |
2836 | { |
- | 2837 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
1835 | struct drm_i915_private *dev_priv = dev->dev_private; |
2838 | struct i915_page_scratch *scratch_page; |
1836 | phys_addr_t gtt_phys_addr; |
- | |
Line 1837... | Line 2839... | ||
1837 | int ret; |
2839 | phys_addr_t gtt_phys_addr; |
1838 | 2840 | ||
1839 | /* For Modern GENs the PTEs and register space are split in the BAR */ |
2841 | /* For Modern GENs the PTEs and register space are split in the BAR */ |
Line -... | Line 2842... | ||
- | 2842 | gtt_phys_addr = pci_resource_start(dev->pdev, 0) + |
|
- | 2843 | (pci_resource_len(dev->pdev, 0) / 2); |
|
- | 2844 | ||
- | 2845 | /* |
|
- | 2846 | * On BXT writes larger than 64 bit to the GTT pagetable range will be |
|
- | 2847 | * dropped. For WC mappings in general we have 64 byte burst writes |
|
- | 2848 | * when the WC buffer is flushed, so we can't use it, but have to |
|
- | 2849 | * resort to an uncached mapping. The WC issue is easily caught by the |
|
- | 2850 | * readback check when writing GTT PTE entries. |
|
- | 2851 | */ |
|
1840 | gtt_phys_addr = pci_resource_start(dev->pdev, 0) + |
2852 | if (IS_BROXTON(dev)) |
1841 | (pci_resource_len(dev->pdev, 0) / 2); |
2853 | dev_priv->gtt.gsm = ioremap_nocache(gtt_phys_addr, gtt_size); |
1842 | 2854 | else |
|
1843 | dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size); |
2855 | dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size); |
1844 | if (!dev_priv->gtt.gsm) { |
2856 | if (!dev_priv->gtt.gsm) { |
Line 1845... | Line 2857... | ||
1845 | DRM_ERROR("Failed to map the gtt page table\n"); |
2857 | DRM_ERROR("Failed to map the gtt page table\n"); |
1846 | return -ENOMEM; |
2858 | return -ENOMEM; |
1847 | } |
2859 | } |
1848 | 2860 | ||
1849 | ret = setup_scratch_page(dev); |
2861 | scratch_page = alloc_scratch_page(dev); |
- | 2862 | if (IS_ERR(scratch_page)) { |
|
1850 | if (ret) { |
2863 | DRM_ERROR("Scratch setup failed\n"); |
Line -... | Line 2864... | ||
- | 2864 | /* iounmap will also get called at remove, but meh */ |
|
- | 2865 | iounmap(dev_priv->gtt.gsm); |
|
1851 | DRM_ERROR("Scratch setup failed\n"); |
2866 | return PTR_ERR(scratch_page); |
1852 | /* iounmap will also get called at remove, but meh */ |
2867 | } |
Line 1853... | Line 2868... | ||
1853 | iounmap(dev_priv->gtt.gsm); |
2868 | |
1854 | } |
2869 | dev_priv->gtt.base.scratch_page = scratch_page; |
1855 | 2870 | ||
Line 1888... | Line 2903... | ||
1888 | */ |
2903 | */ |
1889 | pat = GEN8_PPAT(0, GEN8_PPAT_UC); |
2904 | pat = GEN8_PPAT(0, GEN8_PPAT_UC); |
Line 1890... | Line 2905... | ||
1890 | 2905 | ||
1891 | /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b |
2906 | /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b |
1892 | * write would work. */ |
2907 | * write would work. */ |
1893 | I915_WRITE(GEN8_PRIVATE_PAT, pat); |
2908 | I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); |
1894 | I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32); |
2909 | I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); |
Line 1895... | Line 2910... | ||
1895 | } |
2910 | } |
1896 | 2911 | ||
1897 | static void chv_setup_private_ppat(struct drm_i915_private *dev_priv) |
2912 | static void chv_setup_private_ppat(struct drm_i915_private *dev_priv) |
Line 1923... | Line 2938... | ||
1923 | GEN8_PPAT(4, CHV_PPAT_SNOOP) | |
2938 | GEN8_PPAT(4, CHV_PPAT_SNOOP) | |
1924 | GEN8_PPAT(5, CHV_PPAT_SNOOP) | |
2939 | GEN8_PPAT(5, CHV_PPAT_SNOOP) | |
1925 | GEN8_PPAT(6, CHV_PPAT_SNOOP) | |
2940 | GEN8_PPAT(6, CHV_PPAT_SNOOP) | |
1926 | GEN8_PPAT(7, CHV_PPAT_SNOOP); |
2941 | GEN8_PPAT(7, CHV_PPAT_SNOOP); |
Line 1927... | Line 2942... | ||
1927 | 2942 | ||
1928 | I915_WRITE(GEN8_PRIVATE_PAT, pat); |
2943 | I915_WRITE(GEN8_PRIVATE_PAT_LO, pat); |
1929 | I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32); |
2944 | I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); |
Line 1930... | Line 2945... | ||
1930 | } |
2945 | } |
1931 | 2946 | ||
1932 | static int gen8_gmch_probe(struct drm_device *dev, |
2947 | static int gen8_gmch_probe(struct drm_device *dev, |
1933 | size_t *gtt_total, |
2948 | u64 *gtt_total, |
1934 | size_t *stolen, |
2949 | size_t *stolen, |
1935 | phys_addr_t *mappable_base, |
2950 | phys_addr_t *mappable_base, |
1936 | unsigned long *mappable_end) |
2951 | u64 *mappable_end) |
1937 | { |
2952 | { |
1938 | struct drm_i915_private *dev_priv = dev->dev_private; |
2953 | struct drm_i915_private *dev_priv = dev->dev_private; |
1939 | unsigned int gtt_size; |
2954 | u64 gtt_size; |
Line 1940... | Line 2955... | ||
1940 | u16 snb_gmch_ctl; |
2955 | u16 snb_gmch_ctl; |
1941 | int ret; |
2956 | int ret; |
Line 1958... | Line 2973... | ||
1958 | } else { |
2973 | } else { |
1959 | *stolen = gen8_get_stolen_size(snb_gmch_ctl); |
2974 | *stolen = gen8_get_stolen_size(snb_gmch_ctl); |
1960 | gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); |
2975 | gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); |
1961 | } |
2976 | } |
Line 1962... | Line 2977... | ||
1962 | 2977 | ||
Line 1963... | Line 2978... | ||
1963 | *gtt_total = (gtt_size / sizeof(gen8_gtt_pte_t)) << PAGE_SHIFT; |
2978 | *gtt_total = (gtt_size / sizeof(gen8_pte_t)) << PAGE_SHIFT; |
1964 | 2979 | ||
1965 | if (IS_CHERRYVIEW(dev)) |
2980 | if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) |
1966 | chv_setup_private_ppat(dev_priv); |
2981 | chv_setup_private_ppat(dev_priv); |
Line 1967... | Line 2982... | ||
1967 | else |
2982 | else |
Line 1968... | Line 2983... | ||
1968 | bdw_setup_private_ppat(dev_priv); |
2983 | bdw_setup_private_ppat(dev_priv); |
1969 | 2984 | ||
- | 2985 | ret = ggtt_probe_common(dev, gtt_size); |
|
- | 2986 | ||
Line 1970... | Line 2987... | ||
1970 | ret = ggtt_probe_common(dev, gtt_size); |
2987 | dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range; |
1971 | 2988 | dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries; |
|
Line 1972... | Line 2989... | ||
1972 | dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range; |
2989 | dev_priv->gtt.base.bind_vma = ggtt_bind_vma; |
1973 | dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries; |
2990 | dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; |
1974 | 2991 | ||
1975 | return ret; |
2992 | return ret; |
1976 | } |
2993 | } |
1977 | 2994 | ||
1978 | static int gen6_gmch_probe(struct drm_device *dev, |
2995 | static int gen6_gmch_probe(struct drm_device *dev, |
1979 | size_t *gtt_total, |
2996 | u64 *gtt_total, |
1980 | size_t *stolen, |
2997 | size_t *stolen, |
1981 | phys_addr_t *mappable_base, |
2998 | phys_addr_t *mappable_base, |
Line 1991... | Line 3008... | ||
1991 | 3008 | ||
1992 | /* 64/512MB is the current min/max we actually know of, but this is just |
3009 | /* 64/512MB is the current min/max we actually know of, but this is just |
1993 | * a coarse sanity check. |
3010 | * a coarse sanity check. |
1994 | */ |
3011 | */ |
1995 | if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { |
3012 | if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { |
1996 | DRM_ERROR("Unknown GMADR size (%lx)\n", |
3013 | DRM_ERROR("Unknown GMADR size (%llx)\n", |
1997 | dev_priv->gtt.mappable_end); |
3014 | dev_priv->gtt.mappable_end); |
1998 | return -ENXIO; |
3015 | return -ENXIO; |
Line 1999... | Line 3016... | ||
1999 | } |
3016 | } |
Line 2003... | Line 3020... | ||
2003 | pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); |
3020 | pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); |
Line 2004... | Line 3021... | ||
2004 | 3021 | ||
Line 2005... | Line 3022... | ||
2005 | *stolen = gen6_get_stolen_size(snb_gmch_ctl); |
3022 | *stolen = gen6_get_stolen_size(snb_gmch_ctl); |
2006 | 3023 | ||
Line 2007... | Line 3024... | ||
2007 | gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); |
3024 | gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); |
Line 2008... | Line 3025... | ||
2008 | *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT; |
3025 | *gtt_total = (gtt_size / sizeof(gen6_pte_t)) << PAGE_SHIFT; |
2009 | 3026 | ||
- | 3027 | ret = ggtt_probe_common(dev, gtt_size); |
|
- | 3028 | ||
Line 2010... | Line 3029... | ||
2010 | ret = ggtt_probe_common(dev, gtt_size); |
3029 | dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range; |
2011 | 3030 | dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries; |
|
Line 2012... | Line 3031... | ||
2012 | dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range; |
3031 | dev_priv->gtt.base.bind_vma = ggtt_bind_vma; |
2013 | dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries; |
3032 | dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; |
Line 2014... | Line 3033... | ||
2014 | 3033 | ||
Line 2015... | Line 3034... | ||
2015 | return ret; |
3034 | return ret; |
2016 | } |
3035 | } |
2017 | 3036 | ||
Line 2018... | Line 3037... | ||
2018 | static void gen6_gmch_remove(struct i915_address_space *vm) |
3037 | static void gen6_gmch_remove(struct i915_address_space *vm) |
2019 | { |
3038 | { |
2020 | 3039 | ||
2021 | struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base); |
3040 | struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base); |
2022 | 3041 | ||
2023 | iounmap(gtt->gsm); |
3042 | iounmap(gtt->gsm); |
2024 | teardown_scratch_page(vm->dev); |
3043 | free_scratch_page(vm->dev, vm->scratch_page); |
2025 | } |
3044 | } |
Line 2026... | Line 3045... | ||
2026 | 3045 | ||
Line 2040... | Line 3059... | ||
2040 | } |
3059 | } |
Line 2041... | Line 3060... | ||
2041 | 3060 | ||
Line 2042... | Line 3061... | ||
2042 | intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end); |
3061 | intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end); |
- | 3062 | ||
2043 | 3063 | dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev); |
|
- | 3064 | dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries; |
|
- | 3065 | dev_priv->gtt.base.clear_range = i915_ggtt_clear_range; |
|
Line 2044... | Line 3066... | ||
2044 | dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev); |
3066 | dev_priv->gtt.base.bind_vma = ggtt_bind_vma; |
2045 | dev_priv->gtt.base.clear_range = i915_ggtt_clear_range; |
3067 | dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; |
Line 2046... | Line 3068... | ||
2046 | 3068 | ||
Line 2080... | Line 3102... | ||
2080 | } else { |
3102 | } else { |
2081 | dev_priv->gtt.gtt_probe = gen8_gmch_probe; |
3103 | dev_priv->gtt.gtt_probe = gen8_gmch_probe; |
2082 | dev_priv->gtt.base.cleanup = gen6_gmch_remove; |
3104 | dev_priv->gtt.base.cleanup = gen6_gmch_remove; |
2083 | } |
3105 | } |
Line -... | Line 3106... | ||
- | 3106 | ||
- | 3107 | gtt->base.dev = dev; |
|
2084 | 3108 | ||
2085 | ret = gtt->gtt_probe(dev, >t->base.total, >t->stolen_size, |
3109 | ret = gtt->gtt_probe(dev, >t->base.total, >t->stolen_size, |
2086 | >t->mappable_base, >t->mappable_end); |
3110 | >t->mappable_base, >t->mappable_end); |
2087 | if (ret) |
3111 | if (ret) |
Line 2088... | Line -... | ||
2088 | return ret; |
- | |
2089 | - | ||
2090 | gtt->base.dev = dev; |
3112 | return ret; |
2091 | 3113 | ||
2092 | /* GMADR is the PCI mmio aperture into the global GTT. */ |
3114 | /* GMADR is the PCI mmio aperture into the global GTT. */ |
2093 | DRM_INFO("Memory usable by graphics device = %zdM\n", |
3115 | DRM_INFO("Memory usable by graphics device = %lluM\n", |
2094 | gtt->base.total >> 20); |
3116 | gtt->base.total >> 20); |
2095 | DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20); |
3117 | DRM_DEBUG_DRIVER("GMADR size = %lldM\n", gtt->mappable_end >> 20); |
2096 | DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20); |
3118 | DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20); |
2097 | #ifdef CONFIG_INTEL_IOMMU |
3119 | #ifdef CONFIG_INTEL_IOMMU |
2098 | if (intel_iommu_gfx_mapped) |
3120 | if (intel_iommu_gfx_mapped) |
Line 2108... | Line 3130... | ||
2108 | DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt); |
3130 | DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt); |
Line 2109... | Line 3131... | ||
2109 | 3131 | ||
2110 | return 0; |
3132 | return 0; |
Line 2111... | Line 3133... | ||
2111 | } |
3133 | } |
2112 | - | ||
2113 | static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj, |
3134 | |
- | 3135 | void i915_gem_restore_gtt_mappings(struct drm_device *dev) |
|
- | 3136 | { |
|
- | 3137 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
- | 3138 | struct drm_i915_gem_object *obj; |
|
- | 3139 | struct i915_address_space *vm; |
|
- | 3140 | struct i915_vma *vma; |
|
- | 3141 | bool flush; |
|
- | 3142 | ||
- | 3143 | i915_check_and_clear_faults(dev); |
|
- | 3144 | ||
- | 3145 | /* First fill our portion of the GTT with scratch pages */ |
|
- | 3146 | dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, |
|
- | 3147 | dev_priv->gtt.base.start, |
|
- | 3148 | dev_priv->gtt.base.total, |
|
- | 3149 | true); |
|
- | 3150 | ||
- | 3151 | /* Cache flush objects bound into GGTT and rebind them. */ |
|
- | 3152 | vm = &dev_priv->gtt.base; |
|
- | 3153 | list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { |
|
- | 3154 | flush = false; |
|
- | 3155 | list_for_each_entry(vma, &obj->vma_list, vma_link) { |
|
- | 3156 | if (vma->vm != vm) |
|
- | 3157 | continue; |
|
- | 3158 | ||
- | 3159 | WARN_ON(i915_vma_bind(vma, obj->cache_level, |
|
- | 3160 | PIN_UPDATE)); |
|
- | 3161 | ||
- | 3162 | flush = true; |
|
- | 3163 | } |
|
- | 3164 | ||
- | 3165 | if (flush) |
|
- | 3166 | i915_gem_clflush_object(obj, obj->pin_display); |
|
- | 3167 | } |
|
- | 3168 | ||
- | 3169 | if (INTEL_INFO(dev)->gen >= 8) { |
|
- | 3170 | if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) |
|
- | 3171 | chv_setup_private_ppat(dev_priv); |
|
- | 3172 | else |
|
- | 3173 | bdw_setup_private_ppat(dev_priv); |
|
- | 3174 | ||
- | 3175 | return; |
|
- | 3176 | } |
|
- | 3177 | ||
- | 3178 | if (USES_PPGTT(dev)) { |
|
- | 3179 | list_for_each_entry(vm, &dev_priv->vm_list, global_link) { |
|
- | 3180 | /* TODO: Perhaps it shouldn't be gen6 specific */ |
|
- | 3181 | ||
- | 3182 | struct i915_hw_ppgtt *ppgtt = |
|
- | 3183 | container_of(vm, struct i915_hw_ppgtt, |
|
- | 3184 | base); |
|
- | 3185 | ||
- | 3186 | if (i915_is_ggtt(vm)) |
|
- | 3187 | ppgtt = dev_priv->mm.aliasing_ppgtt; |
|
- | 3188 | ||
- | 3189 | gen6_write_page_range(dev_priv, &ppgtt->pd, |
|
- | 3190 | 0, ppgtt->base.total); |
|
- | 3191 | } |
|
- | 3192 | } |
|
- | 3193 | ||
- | 3194 | i915_ggtt_flush(dev_priv); |
|
- | 3195 | } |
|
- | 3196 | ||
- | 3197 | static struct i915_vma * |
|
- | 3198 | __i915_gem_vma_create(struct drm_i915_gem_object *obj, |
|
- | 3199 | struct i915_address_space *vm, |
|
- | 3200 | const struct i915_ggtt_view *ggtt_view) |
|
- | 3201 | { |
|
- | 3202 | struct i915_vma *vma; |
|
- | 3203 | ||
- | 3204 | if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view)) |
|
- | 3205 | return ERR_PTR(-EINVAL); |
|
2114 | struct i915_address_space *vm) |
3206 | |
2115 | { |
3207 | // vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL); |
2116 | struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); |
3208 | vma = kzalloc(sizeof(*vma), GFP_KERNEL); |
Line 2117... | Line 3209... | ||
2117 | if (vma == NULL) |
3209 | if (vma == NULL) |
2118 | return ERR_PTR(-ENOMEM); |
3210 | return ERR_PTR(-ENOMEM); |
2119 | 3211 | ||
2120 | INIT_LIST_HEAD(&vma->vma_link); |
3212 | INIT_LIST_HEAD(&vma->vma_link); |
2121 | INIT_LIST_HEAD(&vma->mm_list); |
3213 | INIT_LIST_HEAD(&vma->mm_list); |
Line 2122... | Line -... | ||
2122 | INIT_LIST_HEAD(&vma->exec_list); |
- | |
2123 | vma->vm = vm; |
- | |
2124 | vma->obj = obj; |
- | |
2125 | - | ||
2126 | switch (INTEL_INFO(vm->dev)->gen) { |
- | |
2127 | case 9: |
- | |
2128 | case 8: |
- | |
2129 | case 7: |
- | |
2130 | case 6: |
- | |
2131 | if (i915_is_ggtt(vm)) { |
- | |
2132 | vma->unbind_vma = ggtt_unbind_vma; |
- | |
2133 | vma->bind_vma = ggtt_bind_vma; |
- | |
2134 | } else { |
- | |
2135 | vma->unbind_vma = ppgtt_unbind_vma; |
- | |
2136 | vma->bind_vma = ppgtt_bind_vma; |
- | |
2137 | } |
- | |
2138 | break; |
- | |
2139 | case 5: |
- | |
2140 | case 4: |
- | |
2141 | case 3: |
- | |
2142 | case 2: |
- | |
2143 | BUG_ON(!i915_is_ggtt(vm)); |
- | |
2144 | vma->unbind_vma = i915_ggtt_unbind_vma; |
- | |
2145 | vma->bind_vma = i915_ggtt_bind_vma; |
- | |
2146 | break; |
- | |
2147 | default: |
- | |
2148 | BUG(); |
3214 | INIT_LIST_HEAD(&vma->exec_list); |
2149 | } |
3215 | vma->vm = vm; |
2150 | 3216 | vma->obj = obj; |
|
2151 | /* Keep GGTT vmas first to make debug easier */ |
3217 | |
- | 3218 | if (i915_is_ggtt(vm)) |
|
2152 | if (i915_is_ggtt(vm)) |
3219 | vma->ggtt_view = *ggtt_view; |
2153 | list_add(&vma->vma_link, &obj->vma_list); |
- | |
Line 2154... | Line 3220... | ||
2154 | else { |
3220 | |
2155 | list_add_tail(&vma->vma_link, &obj->vma_list); |
3221 | list_add_tail(&vma->vma_link, &obj->vma_list); |
Line 2156... | Line 3222... | ||
2156 | i915_ppgtt_get(i915_vm_to_ppgtt(vm)); |
3222 | if (!i915_is_ggtt(vm)) |
Line 2165... | Line 3231... | ||
2165 | { |
3231 | { |
2166 | struct i915_vma *vma; |
3232 | struct i915_vma *vma; |
Line 2167... | Line 3233... | ||
2167 | 3233 | ||
2168 | vma = i915_gem_obj_to_vma(obj, vm); |
3234 | vma = i915_gem_obj_to_vma(obj, vm); |
2169 | if (!vma) |
3235 | if (!vma) |
- | 3236 | vma = __i915_gem_vma_create(obj, vm, |
|
Line 2170... | Line 3237... | ||
2170 | vma = __i915_gem_vma_create(obj, vm); |
3237 | i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL); |
2171 | 3238 | ||
Line -... | Line 3239... | ||
- | 3239 | return vma; |
|
- | 3240 | } |
|
2172 | return vma; |
3241 | |
2173 | } |
3242 | struct i915_vma * |
2174 | 3243 | i915_gem_obj_lookup_or_create_ggtt_vma(struct drm_i915_gem_object *obj, |
|
2175 | struct scatterlist *sg_next(struct scatterlist *sg) |
3244 | const struct i915_ggtt_view *view) |
Line 2176... | Line -... | ||
2176 | { |
- | |
2177 | if (sg_is_last(sg)) |
3245 | { |
2178 | return NULL; |
3246 | struct i915_address_space *ggtt = i915_obj_to_ggtt(obj); |
Line 2179... | Line 3247... | ||
2179 | 3247 | struct i915_vma *vma; |
|
2180 | sg++; |
- | |
Line -... | Line 3248... | ||
- | 3248 | ||
- | 3249 | if (WARN_ON(!view)) |
|
Line 2181... | Line -... | ||
2181 | if (unlikely(sg_is_chain(sg))) |
- | |
2182 | sg = sg_chain_ptr(sg); |
- | |
2183 | 3250 | return ERR_PTR(-EINVAL); |
|
2184 | return sg; |
3251 | |
2185 | } |
- | |
2186 | - | ||
2187 | - | ||
Line 2188... | Line 3252... | ||
2188 | void __sg_free_table(struct sg_table *table, unsigned int max_ents, |
3252 | vma = i915_gem_obj_to_ggtt_view(obj, view); |
2189 | bool skip_first_chunk, sg_free_fn *free_fn) |
- | |
2190 | { |
- | |
2191 | struct scatterlist *sgl, *next; |
- | |
Line 2192... | Line -... | ||
2192 | - | ||
2193 | if (unlikely(!table->sgl)) |
- | |
2194 | return; |
- | |
2195 | - | ||
2196 | sgl = table->sgl; |
- | |
2197 | while (table->orig_nents) { |
- | |
2198 | unsigned int alloc_size = table->orig_nents; |
- | |
2199 | unsigned int sg_size; |
- | |
2200 | - | ||
2201 | /* |
- | |
2202 | * If we have more than max_ents segments left, |
- | |
2203 | * then assign 'next' to the sg table after the current one. |
- | |
2204 | * sg_size is then one less than alloc size, since the last |
- | |
2205 | * element is the chain pointer. |
3253 | |
Line -... | Line 3254... | ||
- | 3254 | if (IS_ERR(vma)) |
|
- | 3255 | return vma; |
|
- | 3256 | ||
2206 | */ |
3257 | if (!vma) |
- | 3258 | vma = __i915_gem_vma_create(obj, ggtt, view); |
|
- | 3259 | ||
- | 3260 | return vma; |
|
- | 3261 | ||
- | 3262 | } |
|
- | 3263 | ||
- | 3264 | static struct scatterlist * |
|
- | 3265 | rotate_pages(dma_addr_t *in, unsigned int offset, |
|
- | 3266 | unsigned int width, unsigned int height, |
|
- | 3267 | struct sg_table *st, struct scatterlist *sg) |
|
- | 3268 | { |
|
2207 | if (alloc_size > max_ents) { |
3269 | unsigned int column, row; |
- | 3270 | unsigned int src_idx; |
|
- | 3271 | ||
- | 3272 | if (!sg) { |
|
- | 3273 | st->nents = 0; |
|
- | 3274 | sg = st->sgl; |
|
- | 3275 | } |
|
- | 3276 | ||
- | 3277 | for (column = 0; column < width; column++) { |
|
2208 | next = sg_chain_ptr(&sgl[max_ents - 1]); |
3278 | src_idx = width * (height - 1) + column; |
2209 | alloc_size = max_ents; |
3279 | for (row = 0; row < height; row++) { |
2210 | sg_size = alloc_size - 1; |
3280 | st->nents++; |
2211 | } else { |
- | |
2212 | sg_size = alloc_size; |
3281 | /* We don't need the pages, but need to initialize |
Line 2213... | Line 3282... | ||
2213 | next = NULL; |
3282 | * the entries so the sg list can be happily traversed. |
2214 | } |
- | |
2215 | - | ||
2216 | table->orig_nents -= sg_size; |
- | |
2217 | if (!skip_first_chunk) { |
- | |
2218 | kfree(sgl); |
3283 | * The only thing we need are DMA addresses. |
Line -... | Line 3284... | ||
- | 3284 | */ |
|
2219 | skip_first_chunk = false; |
3285 | sg_set_page(sg, NULL, PAGE_SIZE, 0); |
- | 3286 | sg_dma_address(sg) = in[offset + src_idx]; |
|
2220 | } |
3287 | sg_dma_len(sg) = PAGE_SIZE; |
- | 3288 | sg = sg_next(sg); |
|
- | 3289 | src_idx -= width; |
|
- | 3290 | } |
|
2221 | sgl = next; |
3291 | } |
2222 | } |
3292 | |
- | 3293 | return sg; |
|
- | 3294 | } |
|
2223 | 3295 | ||
- | 3296 | static struct sg_table * |
|
- | 3297 | intel_rotate_fb_obj_pages(struct i915_ggtt_view *ggtt_view, |
|
2224 | table->sgl = NULL; |
3298 | struct drm_i915_gem_object *obj) |
- | 3299 | { |
|
2225 | } |
3300 | struct intel_rotation_info *rot_info = &ggtt_view->rotation_info; |
2226 | void sg_free_table(struct sg_table *table) |
3301 | unsigned int size_pages = rot_info->size >> PAGE_SHIFT; |
2227 | { |
3302 | unsigned int size_pages_uv; |
- | 3303 | struct sg_page_iter sg_iter; |
|
Line 2228... | Line 3304... | ||
2228 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, NULL); |
3304 | unsigned long i; |
- | 3305 | dma_addr_t *page_addr_list; |
|
- | 3306 | struct sg_table *st; |
|
- | 3307 | unsigned int uv_start_page; |
|
- | 3308 | struct scatterlist *sg; |
|
Line 2229... | Line 3309... | ||
2229 | } |
3309 | int ret = -ENOMEM; |
2230 | - | ||
2231 | int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) |
- | |
2232 | { |
- | |
2233 | struct scatterlist *sg, *prv; |
- | |
2234 | unsigned int left; |
3310 | |
2235 | unsigned int max_ents = SG_MAX_SINGLE_ALLOC; |
- | |
2236 | - | ||
2237 | #ifndef ARCH_HAS_SG_CHAIN |
3311 | /* Allocate a temporary list of source pages for random access. */ |
2238 | BUG_ON(nents > max_ents); |
3312 | page_addr_list = drm_malloc_ab(obj->base.size / PAGE_SIZE, |
Line -... | Line 3313... | ||
- | 3313 | sizeof(dma_addr_t)); |
|
- | 3314 | if (!page_addr_list) |
|
2239 | #endif |
3315 | return ERR_PTR(ret); |
Line 2240... | Line 3316... | ||
2240 | 3316 | ||
2241 | memset(table, 0, sizeof(*table)); |
- | |
2242 | 3317 | /* Account for UV plane with NV12. */ |
|
2243 | left = nents; |
3318 | if (rot_info->pixel_format == DRM_FORMAT_NV12) |
2244 | prv = NULL; |
- | |
2245 | do { |
3319 | size_pages_uv = rot_info->size_uv >> PAGE_SHIFT; |
2246 | unsigned int sg_size, alloc_size = left; |
- | |
2247 | 3320 | else |
|
2248 | if (alloc_size > max_ents) { |
- | |
2249 | alloc_size = max_ents; |
- | |
- | 3321 | size_pages_uv = 0; |
|
Line -... | Line 3322... | ||
- | 3322 | ||
- | 3323 | /* Allocate target SG list. */ |
|
- | 3324 | st = kmalloc(sizeof(*st), GFP_KERNEL); |
|
- | 3325 | if (!st) |
|
- | 3326 | goto err_st_alloc; |
|
- | 3327 | ||
- | 3328 | ret = sg_alloc_table(st, size_pages + size_pages_uv, GFP_KERNEL); |
|
- | 3329 | if (ret) |
|
- | 3330 | goto err_sg_alloc; |
|
- | 3331 | ||
- | 3332 | /* Populate source page list from the object. */ |
|
- | 3333 | i = 0; |
|
- | 3334 | for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) { |
|
- | 3335 | page_addr_list[i] = sg_page_iter_dma_address(&sg_iter); |
|
- | 3336 | i++; |
|
- | 3337 | } |
|
- | 3338 | ||
- | 3339 | /* Rotate the pages. */ |
|
- | 3340 | sg = rotate_pages(page_addr_list, 0, |
|
- | 3341 | rot_info->width_pages, rot_info->height_pages, |
|
- | 3342 | st, NULL); |
|
- | 3343 | ||
- | 3344 | /* Append the UV plane if NV12. */ |
|
- | 3345 | if (rot_info->pixel_format == DRM_FORMAT_NV12) { |
|
- | 3346 | uv_start_page = size_pages; |
|
- | 3347 | ||
- | 3348 | /* Check for tile-row un-alignment. */ |
|
- | 3349 | if (offset_in_page(rot_info->uv_offset)) |
|
- | 3350 | uv_start_page--; |
|
- | 3351 | ||
- | 3352 | rot_info->uv_start_page = uv_start_page; |
|
- | 3353 | ||
- | 3354 | rotate_pages(page_addr_list, uv_start_page, |
|
- | 3355 | rot_info->width_pages_uv, |
|
- | 3356 | rot_info->height_pages_uv, |
|
- | 3357 | st, sg); |
|
- | 3358 | } |
|
- | 3359 | ||
- | 3360 | DRM_DEBUG_KMS( |
|
- | 3361 | "Created rotated page mapping for object size %zu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0)).\n", |
|
- | 3362 | obj->base.size, rot_info->pitch, rot_info->height, |
|
- | 3363 | rot_info->pixel_format, rot_info->width_pages, |
|
2250 | sg_size = alloc_size - 1; |
3364 | rot_info->height_pages, size_pages + size_pages_uv, |
- | 3365 | size_pages); |
|
2251 | } else |
3366 | |
Line -... | Line 3367... | ||
- | 3367 | drm_free_large(page_addr_list); |
|
- | 3368 | ||
- | 3369 | return st; |
|
- | 3370 | ||
- | 3371 | err_sg_alloc: |
|
- | 3372 | kfree(st); |
|
2252 | sg_size = alloc_size; |
3373 | err_st_alloc: |
- | 3374 | drm_free_large(page_addr_list); |
|
- | 3375 | ||
2253 | 3376 | DRM_DEBUG_KMS( |
|
- | 3377 | "Failed to create rotated mapping for object size %zu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0))\n", |
|
- | 3378 | obj->base.size, ret, rot_info->pitch, rot_info->height, |
|
Line 2254... | Line -... | ||
2254 | left -= sg_size; |
- | |
2255 | 3379 | rot_info->pixel_format, rot_info->width_pages, |
|
2256 | sg = kmalloc(alloc_size * sizeof(struct scatterlist), gfp_mask); |
- | |
2257 | if (unlikely(!sg)) { |
- | |
2258 | /* |
3380 | rot_info->height_pages, size_pages + size_pages_uv, |
2259 | * Adjust entry count to reflect that the last |
- | |
2260 | * entry of the previous table won't be used for |
- | |
2261 | * linkage. Without this, sg_kfree() may get |
3381 | size_pages); |
Line 2262... | Line 3382... | ||
2262 | * confused. |
3382 | return ERR_PTR(ret); |
2263 | */ |
- | |
2264 | if (prv) |
3383 | } |
- | 3384 | ||
2265 | table->nents = ++table->orig_nents; |
3385 | static struct sg_table * |
- | 3386 | intel_partial_pages(const struct i915_ggtt_view *view, |
|
2266 | 3387 | struct drm_i915_gem_object *obj) |
|
- | 3388 | { |
|
Line 2267... | Line 3389... | ||
2267 | goto err; |
3389 | struct sg_table *st; |
- | 3390 | struct scatterlist *sg; |
|
2268 | } |
3391 | struct sg_page_iter obj_sg_iter; |
Line 2269... | Line 3392... | ||
2269 | 3392 | int ret = -ENOMEM; |
|
- | 3393 | ||
- | 3394 | st = kmalloc(sizeof(*st), GFP_KERNEL); |
|
Line 2270... | Line 3395... | ||
2270 | sg_init_table(sg, alloc_size); |
3395 | if (!st) |
2271 | table->nents = table->orig_nents += sg_size; |
- | |
Line -... | Line 3396... | ||
- | 3396 | goto err_st_alloc; |
|
- | 3397 | ||
- | 3398 | ret = sg_alloc_table(st, view->params.partial.size, GFP_KERNEL); |
|
2272 | 3399 | if (ret) |
|
2273 | /* |
3400 | goto err_sg_alloc; |
Line 2274... | Line -... | ||
2274 | * If this is the first mapping, assign the sg table header. |
- | |
2275 | * If this is not the first mapping, chain previous part. |
- | |
2276 | */ |
3401 | |
2277 | if (prv) |
3402 | sg = st->sgl; |
2278 | sg_chain(prv, max_ents, sg); |
- | |
2279 | else |
3403 | st->nents = 0; |
2280 | table->sgl = sg; |
- | |
2281 | - | ||
2282 | /* |
- | |
2283 | * If no more entries after this one, mark the end |
3404 | for_each_sg_page(obj->pages->sgl, &obj_sg_iter, obj->pages->nents, |
2284 | */ |
- | |
2285 | if (!left) |
- | |
2286 | sg_mark_end(&sg[sg_size - 1]); |
- | |
Line -... | Line 3405... | ||
- | 3405 | view->params.partial.offset) |
|
- | 3406 | { |
|
Line 2287... | Line 3407... | ||
2287 | 3407 | if (st->nents >= view->params.partial.size) |
|
- | 3408 | break; |
|
- | 3409 | ||
- | 3410 | sg_set_page(sg, NULL, PAGE_SIZE, 0); |
|
2288 | prv = sg; |
3411 | sg_dma_address(sg) = sg_page_iter_dma_address(&obj_sg_iter); |
- | 3412 | sg_dma_len(sg) = PAGE_SIZE; |
|
- | 3413 | ||
2289 | } while (left); |
3414 | sg = sg_next(sg); |
2290 | 3415 | st->nents++; |
|
2291 | return 0; |
3416 | } |
2292 | 3417 | ||
Line -... | Line 3418... | ||
- | 3418 | return st; |
|
- | 3419 | ||
2293 | err: |
3420 | err_sg_alloc: |
- | 3421 | kfree(st); |
|
- | 3422 | err_st_alloc: |
|
- | 3423 | return ERR_PTR(ret); |
|
- | 3424 | } |
|
- | 3425 | ||
2294 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, NULL); |
3426 | static int |
2295 | 3427 | i915_get_ggtt_vma_pages(struct i915_vma *vma) |
|
Line 2296... | Line 3428... | ||
2296 | return -ENOMEM; |
3428 | { |
2297 | } |
- | |
2298 | - | ||
2299 | 3429 | int ret = 0; |
|
Line -... | Line 3430... | ||
- | 3430 | ||
- | 3431 | if (vma->ggtt_view.pages) |
|
- | 3432 | return 0; |
|
- | 3433 | ||
2300 | void sg_init_table(struct scatterlist *sgl, unsigned int nents) |
3434 | if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) |
- | 3435 | vma->ggtt_view.pages = vma->obj->pages; |
|
- | 3436 | else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED) |
|
- | 3437 | vma->ggtt_view.pages = |
|
- | 3438 | intel_rotate_fb_obj_pages(&vma->ggtt_view, vma->obj); |
|
- | 3439 | else if (vma->ggtt_view.type == I915_GGTT_VIEW_PARTIAL) |
|
- | 3440 | vma->ggtt_view.pages = |
|
- | 3441 | intel_partial_pages(&vma->ggtt_view, vma->obj); |
|
2301 | { |
3442 | else |
2302 | memset(sgl, 0, sizeof(*sgl) * nents); |
3443 | WARN_ONCE(1, "GGTT view %u not implemented!\n", |
2303 | #ifdef CONFIG_DEBUG_SG |
3444 | vma->ggtt_view.type); |
Line 2304... | Line 3445... | ||
2304 | { |
3445 | |
2305 | unsigned int i; |
3446 | if (!vma->ggtt_view.pages) { |
Line 2306... | Line 3447... | ||
2306 | for (i = 0; i < nents; i++) |
3447 | DRM_ERROR("Failed to get pages for GGTT view type %u!\n", |
2307 | sgl[i].sg_magic = SG_MAGIC; |
3448 | vma->ggtt_view.type); |
2308 | } |
3449 | ret = -EINVAL; |
2309 | #endif |
3450 | } else if (IS_ERR(vma->ggtt_view.pages)) { |
2310 | sg_mark_end(&sgl[nents - 1]); |
3451 | ret = PTR_ERR(vma->ggtt_view.pages); |
2311 | } |
- | |
Line -... | Line 3452... | ||
- | 3452 | vma->ggtt_view.pages = NULL; |
|
- | 3453 | DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n", |
|
- | 3454 | vma->ggtt_view.type, ret); |
|
- | 3455 | } |
|
- | 3456 | ||
- | 3457 | return ret; |
|
- | 3458 | } |
|
- | 3459 | ||
- | 3460 | /** |
|
- | 3461 | * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. |
|
- | 3462 | * @vma: VMA to map |
|
- | 3463 | * @cache_level: mapping cache level |
|
- | 3464 | * @flags: flags like global or local mapping |
|
- | 3465 | * |
|
- | 3466 | * DMA addresses are taken from the scatter-gather table of this object (or of |
|
- | 3467 | * this VMA in case of non-default GGTT views) and PTE entries set up. |
|
- | 3468 | * Note that DMA addresses are also the only part of the SG table we care about. |
|
- | 3469 | */ |
|
- | 3470 | int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, |
|
- | 3471 | u32 flags) |
|
- | 3472 | { |
|
2312 | 3473 | int ret; |
|
2313 | 3474 | u32 bind_flags; |
|
2314 | void __sg_page_iter_start(struct sg_page_iter *piter, |
- | |
Line -... | Line 3475... | ||
- | 3475 | ||
- | 3476 | if (WARN_ON(flags == 0)) |
|
- | 3477 | return -EINVAL; |
|
- | 3478 | ||
- | 3479 | bind_flags = 0; |
|
- | 3480 | if (flags & PIN_GLOBAL) |
|
- | 3481 | bind_flags |= GLOBAL_BIND; |
|
- | 3482 | if (flags & PIN_USER) |
|
Line -... | Line 3483... | ||
- | 3483 | bind_flags |= LOCAL_BIND; |
|
- | 3484 | ||
- | 3485 | if (flags & PIN_UPDATE) |
|
- | 3486 | bind_flags |= vma->bound; |
|
- | 3487 | else |
|
- | 3488 | bind_flags &= ~vma->bound; |
|
- | 3489 | ||
- | 3490 | if (bind_flags == 0) |
|
- | 3491 | return 0; |
|
- | 3492 | ||
- | 3493 | if (vma->bound == 0 && vma->vm->allocate_va_range) { |
|
- | 3494 | trace_i915_va_alloc(vma->vm, |
|
- | 3495 | vma->node.start, |
|
- | 3496 | vma->node.size, |
|
- | 3497 | VM_TO_TRACE_NAME(vma->vm)); |
|
- | 3498 | ||
- | 3499 | /* XXX: i915_vma_pin() will fix this +- hack */ |
|
- | 3500 | vma->pin_count++; |
|
- | 3501 | ret = vma->vm->allocate_va_range(vma->vm, |
|
- | 3502 | vma->node.start, |
|
- | 3503 | vma->node.size); |
|
- | 3504 | vma->pin_count--; |