Rev 2360 | Rev 3037 | Go to most recent revision | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 2360 | Rev 3031 | ||
---|---|---|---|
Line 23... | Line 23... | ||
23 | * Authors: |
23 | * Authors: |
24 | * Eric Anholt |
24 | * Eric Anholt |
25 | * |
25 | * |
26 | */ |
26 | */ |
Line 27... | Line 27... | ||
27 | 27 | ||
28 | #include "drmP.h" |
- | |
29 | #include "drm.h" |
28 | #include |
30 | #include "i915_drm.h" |
29 | #include |
31 | #include "i915_drv.h" |
30 | #include "i915_drv.h" |
32 | #include "i915_trace.h" |
31 | #include "i915_trace.h" |
33 | #include "intel_drv.h" |
32 | #include "intel_drv.h" |
34 | //#include |
33 | //#include |
Line 105... | Line 104... | ||
105 | #define I915_EXEC_CONSTANTS_MASK (3<<6) |
104 | #define I915_EXEC_CONSTANTS_MASK (3<<6) |
106 | #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */ |
105 | #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */ |
107 | #define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6) |
106 | #define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6) |
108 | #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */ |
107 | #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */ |
Line 109... | Line -... | ||
109 | - | ||
110 | static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj); |
108 | |
111 | static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); |
109 | static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); |
112 | static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); |
- | |
113 | static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, |
- | |
114 | bool write); |
- | |
115 | static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, |
- | |
116 | uint64_t offset, |
- | |
117 | uint64_t size); |
- | |
118 | static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj); |
110 | static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); |
119 | static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, |
111 | static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, |
120 | unsigned alignment, |
112 | unsigned alignment, |
121 | bool map_and_fenceable); |
- | |
122 | static void i915_gem_clear_fence_reg(struct drm_device *dev, |
113 | bool map_and_fenceable, |
123 | struct drm_i915_fence_reg *reg); |
114 | bool nonblocking); |
124 | static int i915_gem_phys_pwrite(struct drm_device *dev, |
115 | static int i915_gem_phys_pwrite(struct drm_device *dev, |
125 | struct drm_i915_gem_object *obj, |
116 | struct drm_i915_gem_object *obj, |
126 | struct drm_i915_gem_pwrite *args, |
117 | struct drm_i915_gem_pwrite *args, |
127 | struct drm_file *file); |
- | |
Line 128... | Line 118... | ||
128 | static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj); |
118 | struct drm_file *file); |
- | 119 | ||
- | 120 | static void i915_gem_write_fence(struct drm_device *dev, int reg, |
|
- | 121 | struct drm_i915_gem_object *obj); |
|
- | 122 | static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, |
|
- | 123 | struct drm_i915_fence_reg *fence, |
|
- | 124 | bool enable); |
|
- | 125 | ||
- | 126 | static long i915_gem_purge(struct drm_i915_private *dev_priv, long target); |
|
- | 127 | static void i915_gem_shrink_all(struct drm_i915_private *dev_priv); |
|
- | 128 | static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); |
|
- | 129 | ||
- | 130 | static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj) |
|
- | 131 | { |
|
- | 132 | if (obj->tiling_mode) |
|
- | 133 | i915_gem_release_mmap(obj); |
|
129 | 134 | ||
- | 135 | /* As we do not have an associated fence register, we will force |
|
- | 136 | * a tiling change if we ever need to acquire one. |
|
- | 137 | */ |
|
- | 138 | obj->fence_dirty = false; |
|
Line 130... | Line 139... | ||
130 | //static int i915_gem_inactive_shrink(struct shrinker *shrinker, |
139 | obj->fence_reg = I915_FENCE_REG_NONE; |
131 | // struct shrink_control *sc); |
140 | } |
132 | 141 | ||
133 | /* some bookkeeping */ |
142 | /* some bookkeeping */ |
Line 156... | Line 165... | ||
156 | int ret; |
165 | int ret; |
Line 157... | Line 166... | ||
157 | 166 | ||
158 | if (!atomic_read(&dev_priv->mm.wedged)) |
167 | if (!atomic_read(&dev_priv->mm.wedged)) |
Line -... | Line 168... | ||
- | 168 | return 0; |
|
- | 169 | ||
- | 170 | /* |
|
- | 171 | * Only wait 10 seconds for the gpu reset to complete to avoid hanging |
|
- | 172 | * userspace. If it takes that long something really bad is going on and |
|
159 | return 0; |
173 | * we should simply try to bail out and fail as gracefully as possible. |
160 | 174 | */ |
|
- | 175 | ret = wait_for_completion_interruptible_timeout(x, 10*HZ); |
|
- | 176 | if (ret == 0) { |
|
- | 177 | DRM_ERROR("Timed out waiting for the gpu reset to complete\n"); |
|
161 | ret = wait_for_completion_interruptible(x); |
178 | return -EIO; |
- | 179 | } else if (ret < 0) { |
|
Line 162... | Line 180... | ||
162 | if (ret) |
180 | return ret; |
163 | return ret; |
181 | } |
164 | 182 | ||
165 | if (atomic_read(&dev_priv->mm.wedged)) { |
183 | if (atomic_read(&dev_priv->mm.wedged)) { |
Line 193... | Line 211... | ||
193 | #endif |
211 | #endif |
Line 194... | Line 212... | ||
194 | 212 | ||
195 | static inline bool |
213 | static inline bool |
196 | i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) |
214 | i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) |
197 | { |
215 | { |
198 | return obj->gtt_space && !obj->active && obj->pin_count == 0; |
216 | return obj->gtt_space && !obj->active; |
Line 199... | Line -... | ||
199 | } |
- | |
200 | - | ||
201 | void i915_gem_do_init(struct drm_device *dev, |
- | |
202 | unsigned long start, |
- | |
203 | unsigned long mappable_end, |
- | |
204 | unsigned long end) |
- | |
205 | { |
- | |
206 | drm_i915_private_t *dev_priv = dev->dev_private; |
- | |
207 | - | ||
208 | drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); |
- | |
209 | - | ||
210 | dev_priv->mm.gtt_start = start; |
- | |
211 | dev_priv->mm.gtt_mappable_end = mappable_end; |
- | |
212 | dev_priv->mm.gtt_end = end; |
- | |
213 | dev_priv->mm.gtt_total = end - start; |
- | |
214 | dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start; |
- | |
215 | - | ||
216 | /* Take over this portion of the GTT */ |
- | |
Line 217... | Line 217... | ||
217 | intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); |
217 | } |
Line 218... | Line 218... | ||
218 | } |
218 | |
219 | 219 | ||
220 | #if 0 |
220 | #if 0 |
221 | 221 | ||
222 | int |
222 | int |
Line -... | Line 223... | ||
- | 223 | i915_gem_init_ioctl(struct drm_device *dev, void *data, |
|
- | 224 | struct drm_file *file) |
|
- | 225 | { |
|
223 | i915_gem_init_ioctl(struct drm_device *dev, void *data, |
226 | struct drm_i915_gem_init *args = data; |
224 | struct drm_file *file) |
227 | |
225 | { |
228 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
Line -... | Line 229... | ||
- | 229 | return -ENODEV; |
|
- | 230 | ||
- | 231 | if (args->gtt_start >= args->gtt_end || |
|
- | 232 | (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) |
|
226 | struct drm_i915_gem_init *args = data; |
233 | return -EINVAL; |
227 | 234 | ||
- | 235 | /* GEM with user mode setting was never supported on ilk and later. */ |
|
228 | if (args->gtt_start >= args->gtt_end || |
236 | if (INTEL_INFO(dev)->gen >= 5) |
Line 229... | Line 237... | ||
229 | (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) |
237 | return -ENODEV; |
230 | return -EINVAL; |
238 | |
231 | 239 | mutex_lock(&dev->struct_mutex); |
|
Line 244... | Line 252... | ||
244 | struct drm_i915_private *dev_priv = dev->dev_private; |
252 | struct drm_i915_private *dev_priv = dev->dev_private; |
245 | struct drm_i915_gem_get_aperture *args = data; |
253 | struct drm_i915_gem_get_aperture *args = data; |
246 | struct drm_i915_gem_object *obj; |
254 | struct drm_i915_gem_object *obj; |
247 | size_t pinned; |
255 | size_t pinned; |
Line 248... | Line -... | ||
248 | - | ||
249 | 256 | ||
250 | pinned = 0; |
257 | pinned = 0; |
251 | mutex_lock(&dev->struct_mutex); |
258 | mutex_lock(&dev->struct_mutex); |
- | 259 | list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) |
|
252 | list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) |
260 | if (obj->pin_count) |
253 | pinned += obj->gtt_space->size; |
261 | pinned += obj->gtt_space->size; |
Line 254... | Line 262... | ||
254 | mutex_unlock(&dev->struct_mutex); |
262 | mutex_unlock(&dev->struct_mutex); |
255 | 263 | ||
Line 256... | Line 264... | ||
256 | args->aper_size = dev_priv->mm.gtt_total; |
264 | args->aper_size = dev_priv->mm.gtt_total; |
257 | args->aper_available_size = args->aper_size - pinned; |
265 | args->aper_available_size = args->aper_size - pinned; |
Line 258... | Line 266... | ||
258 | 266 | ||
259 | return 0; |
267 | return 0; |
260 | } |
268 | } |
261 | 269 | ||
262 | #if 0 |
270 | #if 0 |
263 | 271 | static int |
|
264 | int i915_gem_create(struct drm_file *file, |
272 | i915_gem_create(struct drm_file *file, |
265 | struct drm_device *dev, |
273 | struct drm_device *dev, |
Line 320... | Line 328... | ||
320 | int |
328 | int |
321 | i915_gem_create_ioctl(struct drm_device *dev, void *data, |
329 | i915_gem_create_ioctl(struct drm_device *dev, void *data, |
322 | struct drm_file *file) |
330 | struct drm_file *file) |
323 | { |
331 | { |
324 | struct drm_i915_gem_create *args = data; |
332 | struct drm_i915_gem_create *args = data; |
- | 333 | ||
325 | return i915_gem_create(file, dev, |
334 | return i915_gem_create(file, dev, |
326 | args->size, &args->handle); |
335 | args->size, &args->handle); |
327 | } |
336 | } |
Line 328... | Line 337... | ||
328 | 337 | ||
Line 332... | Line 341... | ||
332 | 341 | ||
333 | return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && |
342 | return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && |
334 | obj->tiling_mode != I915_TILING_NONE; |
343 | obj->tiling_mode != I915_TILING_NONE; |
Line 335... | Line 344... | ||
335 | } |
344 | } |
336 | 345 | ||
337 | static inline void |
346 | static inline int |
338 | slow_shmem_copy(struct page *dst_page, |
- | |
339 | int dst_offset, |
- | |
340 | struct page *src_page, |
347 | __copy_to_user_swizzled(char __user *cpu_vaddr, |
341 | int src_offset, |
348 | const char *gpu_vaddr, int gpu_offset, |
342 | int length) |
349 | int length) |
Line 343... | Line 350... | ||
343 | { |
350 | { |
344 | char *dst_vaddr, *src_vaddr; |
351 | int ret, cpu_offset = 0; |
- | 352 | ||
- | 353 | while (length > 0) { |
|
Line 345... | Line 354... | ||
345 | 354 | int cacheline_end = ALIGN(gpu_offset + 1, 64); |
|
- | 355 | int this_length = min(cacheline_end - gpu_offset, length); |
|
- | 356 | int swizzled_gpu_offset = gpu_offset ^ 64; |
|
- | 357 | ||
- | 358 | ret = __copy_to_user(cpu_vaddr + cpu_offset, |
|
Line -... | Line 359... | ||
- | 359 | gpu_vaddr + swizzled_gpu_offset, |
|
346 | dst_vaddr = kmap(dst_page); |
360 | this_length); |
347 | src_vaddr = kmap(src_page); |
361 | if (ret) |
348 | 362 | return ret + length; |
|
Line 349... | Line -... | ||
349 | memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length); |
- | |
350 | - | ||
351 | kunmap(src_page); |
- | |
352 | kunmap(dst_page); |
- | |
353 | } |
- | |
354 | - | ||
355 | static inline void |
- | |
356 | slow_shmem_bit17_copy(struct page *gpu_page, |
- | |
357 | int gpu_offset, |
- | |
358 | struct page *cpu_page, |
- | |
359 | int cpu_offset, |
- | |
360 | int length, |
- | |
361 | int is_read) |
- | |
362 | { |
- | |
363 | char *gpu_vaddr, *cpu_vaddr; |
- | |
364 | 363 | ||
365 | /* Use the unswizzled path if this page isn't affected. */ |
- | |
366 | if ((page_to_phys(gpu_page) & (1 << 17)) == 0) { |
- | |
367 | if (is_read) |
364 | cpu_offset += this_length; |
Line -... | Line 365... | ||
- | 365 | gpu_offset += this_length; |
|
- | 366 | length -= this_length; |
|
368 | return slow_shmem_copy(cpu_page, cpu_offset, |
367 | } |
- | 368 | ||
- | 369 | return 0; |
|
369 | gpu_page, gpu_offset, length); |
370 | } |
Line 370... | Line -... | ||
370 | else |
- | |
371 | return slow_shmem_copy(gpu_page, gpu_offset, |
- | |
372 | cpu_page, cpu_offset, length); |
- | |
373 | } |
371 | |
374 | 372 | static inline int |
|
375 | gpu_vaddr = kmap(gpu_page); |
373 | __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset, |
376 | cpu_vaddr = kmap(cpu_page); |
374 | const char __user *cpu_vaddr, |
Line 377... | Line -... | ||
377 | - | ||
378 | /* Copy the data, XORing A6 with A17 (1). The user already knows he's |
- | |
379 | * XORing with the other bits (A9 for Y, A9 and A10 for X) |
- | |
380 | */ |
- | |
381 | while (length > 0) { |
- | |
382 | int cacheline_end = ALIGN(gpu_offset + 1, 64); |
375 | int length) |
383 | int this_length = min(cacheline_end - gpu_offset, length); |
376 | { |
384 | int swizzled_gpu_offset = gpu_offset ^ 64; |
377 | int ret, cpu_offset = 0; |
- | 378 | ||
- | 379 | while (length > 0) { |
|
385 | 380 | int cacheline_end = ALIGN(gpu_offset + 1, 64); |
|
386 | if (is_read) { |
381 | int this_length = min(cacheline_end - gpu_offset, length); |
387 | memcpy(cpu_vaddr + cpu_offset, |
382 | int swizzled_gpu_offset = gpu_offset ^ 64; |
388 | gpu_vaddr + swizzled_gpu_offset, |
383 | |
389 | this_length); |
384 | ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset, |
Line -... | Line 385... | ||
- | 385 | cpu_vaddr + cpu_offset, |
|
- | 386 | this_length); |
|
- | 387 | if (ret) |
|
- | 388 | return ret + length; |
|
- | 389 | ||
- | 390 | cpu_offset += this_length; |
|
- | 391 | gpu_offset += this_length; |
|
- | 392 | length -= this_length; |
|
- | 393 | } |
|
- | 394 | ||
- | 395 | return 0; |
|
- | 396 | } |
|
- | 397 | ||
- | 398 | /* Per-page copy function for the shmem pread fastpath. |
|
- | 399 | * Flushes invalid cachelines before reading the target if |
|
- | 400 | * needs_clflush is set. */ |
|
- | 401 | static int |
|
- | 402 | shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length, |
|
- | 403 | char __user *user_data, |
|
- | 404 | bool page_do_bit17_swizzling, bool needs_clflush) |
|
- | 405 | { |
|
- | 406 | char *vaddr; |
|
- | 407 | int ret; |
|
- | 408 | ||
390 | } else { |
409 | if (unlikely(page_do_bit17_swizzling)) |
- | 410 | return -EINVAL; |
|
- | 411 | ||
- | 412 | vaddr = kmap_atomic(page); |
|
- | 413 | if (needs_clflush) |
|
- | 414 | drm_clflush_virt_range(vaddr + shmem_page_offset, |
|
- | 415 | page_length); |
|
- | 416 | ret = __copy_to_user_inatomic(user_data, |
|
- | 417 | vaddr + shmem_page_offset, |
|
- | 418 | page_length); |
|
- | 419 | kunmap_atomic(vaddr); |
|
- | 420 | ||
- | 421 | return ret ? -EFAULT : 0; |
|
- | 422 | } |
|
- | 423 | ||
- | 424 | static void |
|
- | 425 | shmem_clflush_swizzled_range(char *addr, unsigned long length, |
|
- | 426 | bool swizzled) |
|
- | 427 | { |
|
- | 428 | if (unlikely(swizzled)) { |
|
- | 429 | unsigned long start = (unsigned long) addr; |
|
- | 430 | unsigned long end = (unsigned long) addr + length; |
|
- | 431 | ||
- | 432 | /* For swizzling simply ensure that we always flush both |
|
- | 433 | * channels. Lame, but simple and it works. Swizzled |
|
- | 434 | * pwrite/pread is far from a hotpath - current userspace |
|
- | 435 | * doesn't use it at all. */ |
|
- | 436 | start = round_down(start, 128); |
|
- | 437 | end = round_up(end, 128); |
|
- | 438 | ||
- | 439 | drm_clflush_virt_range((void *)start, end - start); |
|
- | 440 | } else { |
|
- | 441 | drm_clflush_virt_range(addr, length); |
|
- | 442 | } |
|
- | 443 | ||
- | 444 | } |
|
- | 445 | ||
- | 446 | /* Only difference to the fast-path function is that this can handle bit17 |
|
- | 447 | * and uses non-atomic copy and kmap functions. */ |
|
- | 448 | static int |
|
- | 449 | shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length, |
|
- | 450 | char __user *user_data, |
|
- | 451 | bool page_do_bit17_swizzling, bool needs_clflush) |
|
- | 452 | { |
|
- | 453 | char *vaddr; |
|
- | 454 | int ret; |
|
- | 455 | ||
- | 456 | vaddr = kmap(page); |
|
- | 457 | if (needs_clflush) |
|
- | 458 | shmem_clflush_swizzled_range(vaddr + shmem_page_offset, |
|
- | 459 | page_length, |
|
391 | memcpy(gpu_vaddr + swizzled_gpu_offset, |
460 | page_do_bit17_swizzling); |
- | 461 | ||
- | 462 | if (page_do_bit17_swizzling) |
|
392 | cpu_vaddr + cpu_offset, |
463 | ret = __copy_to_user_swizzled(user_data, |
Line 393... | Line -... | ||
393 | this_length); |
- | |
394 | } |
- | |
395 | cpu_offset += this_length; |
- | |
396 | gpu_offset += this_length; |
- | |
397 | length -= this_length; |
- | |
398 | } |
464 | vaddr, shmem_page_offset, |
399 | 465 | page_length); |
|
400 | kunmap(cpu_page); |
466 | else |
401 | kunmap(gpu_page); |
467 | ret = __copy_to_user(user_data, |
402 | } |
468 | vaddr + shmem_page_offset, |
403 | 469 | page_length); |
|
404 | /** |
470 | kunmap(page); |
405 | * This is the fast shmem pread path, which attempts to copy_from_user directly |
471 | |
406 | * from the backing pages of the object to the user's address space. On a |
472 | return ret ? - EFAULT : 0; |
- | 473 | } |
|
- | 474 | ||
407 | * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow(). |
475 | static int |
408 | */ |
476 | i915_gem_shmem_pread(struct drm_device *dev, |
- | 477 | struct drm_i915_gem_object *obj, |
|
- | 478 | struct drm_i915_gem_pread *args, |
|
- | 479 | struct drm_file *file) |
|
Line 409... | Line 480... | ||
409 | static int |
480 | { |
410 | i915_gem_shmem_pread_fast(struct drm_device *dev, |
481 | char __user *user_data; |
Line -... | Line 482... | ||
- | 482 | ssize_t remain; |
|
- | 483 | loff_t offset; |
|
- | 484 | int shmem_page_offset, page_length, ret = 0; |
|
- | 485 | int obj_do_bit17_swizzling, page_do_bit17_swizzling; |
|
- | 486 | int hit_slowpath = 0; |
|
- | 487 | int prefaulted = 0; |
|
- | 488 | int needs_clflush = 0; |
|
- | 489 | struct scatterlist *sg; |
|
- | 490 | int i; |
|
- | 491 | ||
- | 492 | user_data = (char __user *) (uintptr_t) args->data_ptr; |
|
- | 493 | remain = args->size; |
|
- | 494 | ||
- | 495 | obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); |
|
- | 496 | ||
- | 497 | if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) { |
|
- | 498 | /* If we're not in the cpu read domain, set ourself into the gtt |
|
- | 499 | * read domain and manually flush cachelines (if required). This |
|
- | 500 | * optimizes for the case when the gpu will dirty the data |
|
- | 501 | * anyway again before the next pread happens. */ |
|
- | 502 | if (obj->cache_level == I915_CACHE_NONE) |
|
- | 503 | needs_clflush = 1; |
|
411 | struct drm_i915_gem_object *obj, |
504 | if (obj->gtt_space) { |
Line 412... | Line 505... | ||
412 | struct drm_i915_gem_pread *args, |
505 | ret = i915_gem_object_set_to_gtt_domain(obj, false); |
413 | struct drm_file *file) |
506 | if (ret) |
- | 507 | return ret; |
|
- | 508 | } |
|
414 | { |
509 | } |
- | 510 | ||
- | 511 | ret = i915_gem_object_get_pages(obj); |
|
415 | struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; |
512 | if (ret) |
Line 416... | Line 513... | ||
416 | ssize_t remain; |
513 | return ret; |
417 | loff_t offset; |
514 | |
418 | char __user *user_data; |
515 | i915_gem_object_pin_pages(obj); |
419 | int page_offset, page_length; |
516 | |
420 | 517 | offset = args->offset; |
|
421 | user_data = (char __user *) (uintptr_t) args->data_ptr; |
518 | |
422 | remain = args->size; |
519 | for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) { |
423 | 520 | struct page *page; |
|
424 | offset = args->offset; |
521 | |
Line -... | Line 522... | ||
- | 522 | if (i < offset >> PAGE_SHIFT) |
|
- | 523 | continue; |
|
- | 524 | ||
- | 525 | if (remain <= 0) |
|
425 | 526 | break; |
|
- | 527 | ||
- | 528 | /* Operation in this page |
|
426 | while (remain > 0) { |
529 | * |
427 | struct page *page; |
530 | * shmem_page_offset = offset within page in shmem file |
Line -... | Line 531... | ||
- | 531 | * page_length = bytes to copy for this page |
|
428 | char *vaddr; |
532 | */ |
- | 533 | shmem_page_offset = offset_in_page(offset); |
|
- | 534 | page_length = remain; |
|
429 | int ret; |
535 | if ((shmem_page_offset + page_length) > PAGE_SIZE) |
- | 536 | page_length = PAGE_SIZE - shmem_page_offset; |
|
430 | 537 | ||
- | 538 | page = sg_page(sg); |
|
431 | /* Operation in this page |
539 | page_do_bit17_swizzling = obj_do_bit17_swizzling && |
- | 540 | (page_to_phys(page) & (1 << 17)) != 0; |
|
432 | * |
541 | |
- | 542 | ret = shmem_pread_fast(page, shmem_page_offset, page_length, |
|
Line -... | Line 543... | ||
- | 543 | user_data, page_do_bit17_swizzling, |
|
- | 544 | needs_clflush); |
|
- | 545 | if (ret == 0) |
|
- | 546 | goto next_page; |
|
- | 547 | ||
- | 548 | hit_slowpath = 1; |
|
- | 549 | mutex_unlock(&dev->struct_mutex); |
|
433 | * page_offset = offset within page |
550 | |
434 | * page_length = bytes to copy for this page |
- | |
- | 551 | if (!prefaulted) { |
|
435 | */ |
552 | ret = fault_in_multipages_writeable(user_data, remain); |
436 | page_offset = offset_in_page(offset); |
553 | /* Userspace is tricking us, but we've already clobbered |
Line 437... | Line 554... | ||
437 | page_length = remain; |
554 | * its pages with the prefault and promised to write the |
438 | if ((page_offset + remain) > PAGE_SIZE) |
555 | * data up to the first fault. Hence ignore any errors |
439 | page_length = PAGE_SIZE - page_offset; |
556 | * and just continue. */ |
440 | 557 | (void)ret; |
|
Line -... | Line 558... | ||
- | 558 | prefaulted = 1; |
|
- | 559 | } |
|
- | 560 | ||
- | 561 | ret = shmem_pread_slow(page, shmem_page_offset, page_length, |
|
- | 562 | user_data, page_do_bit17_swizzling, |
|
- | 563 | needs_clflush); |
|
- | 564 | ||
- | 565 | mutex_lock(&dev->struct_mutex); |
|
- | 566 | ||
- | 567 | next_page: |
|
- | 568 | mark_page_accessed(page); |
|
- | 569 | ||
- | 570 | if (ret) |
|
- | 571 | goto out; |
|
- | 572 | ||
- | 573 | remain -= page_length; |
|
- | 574 | user_data += page_length; |
|
- | 575 | offset += page_length; |
|
- | 576 | } |
|
- | 577 | ||
- | 578 | out: |
|
- | 579 | i915_gem_object_unpin_pages(obj); |
|
- | 580 | ||
- | 581 | if (hit_slowpath) { |
|
- | 582 | /* Fixup: Kill any reinstated backing storage pages */ |
|
- | 583 | if (obj->madv == __I915_MADV_PURGED) |
|
441 | page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT); |
584 | i915_gem_object_truncate(obj); |
- | 585 | } |
|
- | 586 | ||
- | 587 | return ret; |
|
- | 588 | } |
|
- | 589 | ||
- | 590 | /** |
|
- | 591 | * Reads data from the object referenced by handle. |
|
- | 592 | * |
|
- | 593 | * On error, the contents of *data are undefined. |
|
- | 594 | */ |
|
- | 595 | int |
|
- | 596 | i915_gem_pread_ioctl(struct drm_device *dev, void *data, |
|
- | 597 | struct drm_file *file) |
|
- | 598 | { |
|
- | 599 | struct drm_i915_gem_pread *args = data; |
|
- | 600 | struct drm_i915_gem_object *obj; |
|
- | 601 | int ret = 0; |
|
- | 602 | ||
- | 603 | if (args->size == 0) |
|
- | 604 | return 0; |
|
- | 605 | ||
- | 606 | if (!access_ok(VERIFY_WRITE, |
|
- | 607 | (char __user *)(uintptr_t)args->data_ptr, |
|
- | 608 | args->size)) |
|
- | 609 | return -EFAULT; |
|
- | 610 | ||
- | 611 | ret = i915_mutex_lock_interruptible(dev); |
|
- | 612 | if (ret) |
|
- | 613 | return ret; |
|
- | 614 | ||
- | 615 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
|
- | 616 | if (&obj->base == NULL) { |
|
- | 617 | ret = -ENOENT; |
|
- | 618 | goto unlock; |
|
- | 619 | } |
|
- | 620 | ||
- | 621 | /* Bounds check source. */ |
|
- | 622 | if (args->offset > obj->base.size || |
|
- | 623 | args->size > obj->base.size - args->offset) { |
|
- | 624 | ret = -EINVAL; |
|
- | 625 | goto out; |
|
- | 626 | } |
|
- | 627 | ||
- | 628 | /* prime objects have no backing filp to GEM pread/pwrite |
|
- | 629 | * pages from. |
|
- | 630 | */ |
|
- | 631 | if (!obj->base.filp) { |
|
- | 632 | ret = -EINVAL; |
|
- | 633 | goto out; |
|
- | 634 | } |
|
- | 635 | ||
- | 636 | trace_i915_gem_object_pread(obj, args->offset, args->size); |
|
- | 637 | ||
- | 638 | ret = i915_gem_shmem_pread(dev, obj, args, file); |
|
- | 639 | ||
- | 640 | out: |
|
- | 641 | drm_gem_object_unreference(&obj->base); |
|
- | 642 | unlock: |
|
- | 643 | mutex_unlock(&dev->struct_mutex); |
|
- | 644 | return ret; |
|
- | 645 | } |
|
- | 646 | ||
- | 647 | /* This is the fast write path which cannot handle |
|
442 | if (IS_ERR(page)) |
648 | * page faults in the source data |
Line 443... | Line 649... | ||
443 | return PTR_ERR(page); |
649 | */ |
444 | 650 | ||
445 | vaddr = kmap_atomic(page); |
- | |
446 | ret = __copy_to_user_inatomic(user_data, |
- | |
447 | vaddr + page_offset, |
651 | static inline int |
448 | page_length); |
652 | fast_user_write(struct io_mapping *mapping, |
449 | kunmap_atomic(vaddr); |
653 | loff_t page_base, int page_offset, |
450 | 654 | char __user *user_data, |
|
451 | mark_page_accessed(page); |
655 | int length) |
452 | page_cache_release(page); |
656 | { |
453 | if (ret) |
657 | void __iomem *vaddr_atomic; |
454 | return -EFAULT; |
658 | void *vaddr; |
455 | - | ||
456 | remain -= page_length; |
659 | unsigned long unwritten; |
457 | user_data += page_length; |
- | |
458 | offset += page_length; |
660 | |
459 | } |
661 | vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base); |
460 | - | ||
461 | return 0; |
662 | /* We can use the cpu mem copy function because this is X86. */ |
462 | } |
663 | vaddr = (void __force*)vaddr_atomic + page_offset; |
463 | - | ||
464 | /** |
- | |
465 | * This is the fallback shmem pread path, which allocates temporary storage |
- | |
466 | * in kernel space to copy_to_user into outside of the struct_mutex, so we |
- | |
Line -... | Line 664... | ||
- | 664 | unwritten = __copy_from_user_inatomic_nocache(vaddr, |
|
- | 665 | user_data, length); |
|
- | 666 | io_mapping_unmap_atomic(vaddr_atomic); |
|
- | 667 | return unwritten; |
|
- | 668 | } |
|
- | 669 | ||
- | 670 | /** |
|
- | 671 | * This is the fast pwrite path, where we copy the data directly from the |
|
- | 672 | * user into the GTT, uncached. |
|
- | 673 | */ |
|
- | 674 | static int |
|
- | 675 | i915_gem_gtt_pwrite_fast(struct drm_device *dev, |
|
- | 676 | struct drm_i915_gem_object *obj, |
|
467 | * can copy out of the object's backing pages while holding the struct mutex |
677 | struct drm_i915_gem_pwrite *args, |
Line 468... | Line -... | ||
468 | * and not take page faults. |
- | |
469 | */ |
- | |
470 | static int |
- | |
471 | i915_gem_shmem_pread_slow(struct drm_device *dev, |
- | |
472 | struct drm_i915_gem_object *obj, |
- | |
473 | struct drm_i915_gem_pread *args, |
- | |
474 | struct drm_file *file) |
678 | struct drm_file *file) |
Line -... | Line 679... | ||
- | 679 | { |
|
- | 680 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 681 | ssize_t remain; |
|
475 | { |
682 | loff_t offset, page_base; |
- | 683 | char __user *user_data; |
|
- | 684 | int page_offset, page_length, ret; |
|
- | 685 | ||
- | 686 | ret = i915_gem_object_pin(obj, 0, true, true); |
|
476 | struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; |
687 | if (ret) |
477 | struct mm_struct *mm = current->mm; |
688 | goto out; |
- | 689 | ||
- | 690 | ret = i915_gem_object_set_to_gtt_domain(obj, true); |
|
Line 478... | Line -... | ||
478 | struct page **user_pages; |
- | |
479 | ssize_t remain; |
- | |
480 | loff_t offset, pinned_pages, i; |
691 | if (ret) |
481 | loff_t first_data_page, last_data_page, num_pages; |
692 | goto out_unpin; |
482 | int shmem_page_offset; |
693 | |
- | 694 | ret = i915_gem_object_put_fence(obj); |
|
483 | int data_page_index, data_page_offset; |
695 | if (ret) |
484 | int page_length; |
696 | goto out_unpin; |
485 | int ret; |
697 | |
486 | uint64_t data_ptr = args->data_ptr; |
698 | user_data = (char __user *) (uintptr_t) args->data_ptr; |
487 | int do_bit17_swizzling; |
699 | remain = args->size; |
Line -... | Line 700... | ||
- | 700 | ||
- | 701 | offset = obj->gtt_offset + args->offset; |
|
- | 702 | ||
- | 703 | while (remain > 0) { |
|
- | 704 | /* Operation in this page |
|
- | 705 | * |
|
488 | 706 | * page_base = page offset within aperture |
|
- | 707 | * page_offset = offset within page |
|
- | 708 | * page_length = bytes to copy for this page |
|
- | 709 | */ |
|
- | 710 | page_base = offset & PAGE_MASK; |
|
- | 711 | page_offset = offset_in_page(offset); |
|
- | 712 | page_length = remain; |
|
- | 713 | if ((page_offset + remain) > PAGE_SIZE) |
|
- | 714 | page_length = PAGE_SIZE - page_offset; |
|
- | 715 | ||
- | 716 | /* If we get a fault while copying data, then (presumably) our |
|
- | 717 | * source page isn't available. Return the error and we'll |
|
- | 718 | * retry in the slow path. |
|
- | 719 | */ |
|
- | 720 | if (fast_user_write(dev_priv->mm.gtt_mapping, page_base, |
|
- | 721 | page_offset, user_data, page_length)) { |
|
- | 722 | ret = -EFAULT; |
|
- | 723 | goto out_unpin; |
|
- | 724 | } |
|
- | 725 | ||
- | 726 | remain -= page_length; |
|
- | 727 | user_data += page_length; |
|
- | 728 | offset += page_length; |
|
- | 729 | } |
|
- | 730 | ||
- | 731 | out_unpin: |
|
- | 732 | i915_gem_object_unpin(obj); |
|
- | 733 | out: |
|
- | 734 | return ret; |
|
- | 735 | } |
|
- | 736 | ||
- | 737 | /* Per-page copy function for the shmem pwrite fastpath. |
|
- | 738 | * Flushes invalid cachelines before writing to the target if |
|
- | 739 | * needs_clflush_before is set and flushes out any written cachelines after |
|
- | 740 | * writing if needs_clflush is set. */ |
|
- | 741 | static int |
|
- | 742 | shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length, |
|
- | 743 | char __user *user_data, |
|
- | 744 | bool page_do_bit17_swizzling, |
|
- | 745 | bool needs_clflush_before, |
|
- | 746 | bool needs_clflush_after) |
|
- | 747 | { |
|
- | 748 | char *vaddr; |
|
- | 749 | int ret; |
|
- | 750 | ||
- | 751 | if (unlikely(page_do_bit17_swizzling)) |
|
- | 752 | return -EINVAL; |
|
- | 753 | ||
- | 754 | vaddr = kmap_atomic(page); |
|
- | 755 | if (needs_clflush_before) |
|
- | 756 | drm_clflush_virt_range(vaddr + shmem_page_offset, |
|
- | 757 | page_length); |
|
- | 758 | ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset, |
|
- | 759 | user_data, |
|
- | 760 | page_length); |
|
- | 761 | if (needs_clflush_after) |
|
- | 762 | drm_clflush_virt_range(vaddr + shmem_page_offset, |
|
489 | remain = args->size; |
763 | page_length); |
- | 764 | kunmap_atomic(vaddr); |
|
- | 765 | ||
- | 766 | return ret ? -EFAULT : 0; |
|
- | 767 | } |
|
- | 768 | ||
- | 769 | /* Only difference to the fast-path function is that this can handle bit17 |
|
- | 770 | * and uses non-atomic copy and kmap functions. */ |
|
- | 771 | static int |
|
- | 772 | shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, |
|
- | 773 | char __user *user_data, |
|
- | 774 | bool page_do_bit17_swizzling, |
|
- | 775 | bool needs_clflush_before, |
|
- | 776 | bool needs_clflush_after) |
|
- | 777 | { |
|
- | 778 | char *vaddr; |
|
- | 779 | int ret; |
|
- | 780 | ||
- | 781 | vaddr = kmap(page); |
|
- | 782 | if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) |
|
- | 783 | shmem_clflush_swizzled_range(vaddr + shmem_page_offset, |
|
- | 784 | page_length, |
|
- | 785 | page_do_bit17_swizzling); |
|
- | 786 | if (page_do_bit17_swizzling) |
|
- | 787 | ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, |
|
- | 788 | user_data, |
|
- | 789 | page_length); |
|
- | 790 | else |
|
- | 791 | ret = __copy_from_user(vaddr + shmem_page_offset, |
|
- | 792 | user_data, |
|
- | 793 | page_length); |
|
- | 794 | if (needs_clflush_after) |
|
490 | 795 | shmem_clflush_swizzled_range(vaddr + shmem_page_offset, |
|
- | 796 | page_length, |
|
- | 797 | page_do_bit17_swizzling); |
|
- | 798 | kunmap(page); |
|
- | 799 | ||
- | 800 | return ret ? -EFAULT : 0; |
|
- | 801 | } |
|
- | 802 | ||
- | 803 | static int |
|
- | 804 | i915_gem_shmem_pwrite(struct drm_device *dev, |
|
- | 805 | struct drm_i915_gem_object *obj, |
|
- | 806 | struct drm_i915_gem_pwrite *args, |
|
- | 807 | struct drm_file *file) |
|
491 | /* Pin the user pages containing the data. We can't fault while |
808 | { |
- | 809 | ssize_t remain; |
|
- | 810 | loff_t offset; |
|
- | 811 | char __user *user_data; |
|
- | 812 | int shmem_page_offset, page_length, ret = 0; |
|
- | 813 | int obj_do_bit17_swizzling, page_do_bit17_swizzling; |
|
- | 814 | int hit_slowpath = 0; |
|
- | 815 | int needs_clflush_after = 0; |
|
- | 816 | int needs_clflush_before = 0; |
|
- | 817 | int i; |
|
- | 818 | struct scatterlist *sg; |
|
- | 819 | ||
492 | * holding the struct mutex, yet we want to hold it while |
820 | user_data = (char __user *) (uintptr_t) args->data_ptr; |
Line 493... | Line 821... | ||
493 | * dereferencing the user data. |
821 | remain = args->size; |
Line 494... | Line 822... | ||
494 | */ |
822 | |
- | 823 | obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); |
|
Line 495... | Line 824... | ||
495 | first_data_page = data_ptr / PAGE_SIZE; |
824 | |
496 | last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE; |
825 | if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
- | 826 | /* If we're not in the cpu write domain, set ourself into the gtt |
|
- | 827 | * write domain and manually flush cachelines (if required). This |
|
- | 828 | * optimizes for the case when the gpu will use the data |
|
- | 829 | * right away and we therefore have to clflush anyway. */ |
|
- | 830 | if (obj->cache_level == I915_CACHE_NONE) |
|
- | 831 | needs_clflush_after = 1; |
|
- | 832 | if (obj->gtt_space) { |
|
Line 497... | Line 833... | ||
497 | num_pages = last_data_page - first_data_page + 1; |
833 | ret = i915_gem_object_set_to_gtt_domain(obj, true); |
498 | 834 | if (ret) |
|
499 | user_pages = drm_malloc_ab(num_pages, sizeof(struct page *)); |
835 | return ret; |
500 | if (user_pages == NULL) |
- | |
501 | return -ENOMEM; |
- | |
502 | 836 | } |
|
503 | mutex_unlock(&dev->struct_mutex); |
837 | } |
504 | down_read(&mm->mmap_sem); |
838 | /* Same trick applies for invalidate partially written cachelines before |
505 | pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr, |
- | |
506 | num_pages, 1, 0, user_pages, NULL); |
- | |
Line 507... | Line 839... | ||
507 | up_read(&mm->mmap_sem); |
839 | * writing. */ |
508 | mutex_lock(&dev->struct_mutex); |
840 | if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU) |
509 | if (pinned_pages < num_pages) { |
841 | && obj->cache_level == I915_CACHE_NONE) |
510 | ret = -EFAULT; |
- | |
511 | goto out; |
- | |
Line -... | Line 842... | ||
- | 842 | needs_clflush_before = 1; |
|
- | 843 | ||
- | 844 | ret = i915_gem_object_get_pages(obj); |
|
- | 845 | if (ret) |
|
512 | } |
846 | return ret; |
- | 847 | ||
- | 848 | i915_gem_object_pin_pages(obj); |
|
513 | 849 | ||
- | 850 | offset = args->offset; |
|
- | 851 | obj->dirty = 1; |
|
- | 852 | ||
- | 853 | for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) { |
|
- | 854 | struct page *page; |
|
- | 855 | int partial_cacheline_write; |
|
514 | ret = i915_gem_object_set_cpu_read_domain_range(obj, |
856 | |
515 | args->offset, |
857 | if (i < offset >> PAGE_SHIFT) |
516 | args->size); |
858 | continue; |
Line 517... | Line 859... | ||
517 | if (ret) |
859 | |
518 | goto out; |
- | |
519 | - | ||
520 | do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); |
860 | if (remain <= 0) |
521 | - | ||
522 | offset = args->offset; |
- | |
523 | - | ||
524 | while (remain > 0) { |
- | |
525 | struct page *page; |
861 | break; |
526 | 862 | ||
527 | /* Operation in this page |
- | |
528 | * |
863 | /* Operation in this page |
529 | * shmem_page_offset = offset within page in shmem file |
864 | * |
530 | * data_page_index = page number in get_user_pages return |
- | |
Line -... | Line 865... | ||
- | 865 | * shmem_page_offset = offset within page in shmem file |
|
- | 866 | * page_length = bytes to copy for this page |
|
- | 867 | */ |
|
- | 868 | shmem_page_offset = offset_in_page(offset); |
|
531 | * data_page_offset = offset with data_page_index page. |
869 | |
- | 870 | page_length = remain; |
|
- | 871 | if ((shmem_page_offset + page_length) > PAGE_SIZE) |
|
532 | * page_length = bytes to copy for this page |
872 | page_length = PAGE_SIZE - shmem_page_offset; |
Line 533... | Line 873... | ||
533 | */ |
873 | |
534 | shmem_page_offset = offset_in_page(offset); |
874 | /* If we don't overwrite a cacheline completely we need to be |
535 | data_page_index = data_ptr / PAGE_SIZE - first_data_page; |
875 | * careful to have up-to-date data by first clflushing. Don't |
536 | data_page_offset = offset_in_page(data_ptr); |
876 | * overcomplicate things and flush the entire patch. */ |
Line 537... | Line 877... | ||
537 | 877 | partial_cacheline_write = needs_clflush_before && |
|
538 | page_length = remain; |
878 | ((shmem_page_offset | page_length) |
- | 879 | & (boot_cpu_data.x86_clflush_size - 1)); |
|
- | 880 | ||
- | 881 | page = sg_page(sg); |
|
- | 882 | page_do_bit17_swizzling = obj_do_bit17_swizzling && |
|
539 | if ((shmem_page_offset + page_length) > PAGE_SIZE) |
883 | (page_to_phys(page) & (1 << 17)) != 0; |
- | 884 | ||
- | 885 | ret = shmem_pwrite_fast(page, shmem_page_offset, page_length, |
|
- | 886 | user_data, page_do_bit17_swizzling, |
|
540 | page_length = PAGE_SIZE - shmem_page_offset; |
887 | partial_cacheline_write, |
541 | if ((data_page_offset + page_length) > PAGE_SIZE) |
888 | needs_clflush_after); |
- | 889 | if (ret == 0) |
|
542 | page_length = PAGE_SIZE - data_page_offset; |
890 | goto next_page; |
- | 891 | ||
543 | 892 | hit_slowpath = 1; |
|
- | 893 | mutex_unlock(&dev->struct_mutex); |
|
Line 544... | Line 894... | ||
544 | page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT); |
894 | ret = shmem_pwrite_slow(page, shmem_page_offset, page_length, |
545 | if (IS_ERR(page)) { |
895 | user_data, page_do_bit17_swizzling, |
- | 896 | partial_cacheline_write, |
|
- | 897 | needs_clflush_after); |
|
- | 898 | ||
- | 899 | mutex_lock(&dev->struct_mutex); |
|
- | 900 | ||
- | 901 | next_page: |
|
- | 902 | set_page_dirty(page); |
|
- | 903 | mark_page_accessed(page); |
|
- | 904 | ||
- | 905 | if (ret) |
|
- | 906 | goto out; |
|
- | 907 | ||
- | 908 | remain -= page_length; |
|
- | 909 | user_data += page_length; |
|
- | 910 | offset += page_length; |
|
- | 911 | } |
|
- | 912 | ||
- | 913 | out: |
|
- | 914 | i915_gem_object_unpin_pages(obj); |
|
- | 915 | ||
- | 916 | if (hit_slowpath) { |
|
- | 917 | /* Fixup: Kill any reinstated backing storage pages */ |
|
- | 918 | if (obj->madv == __I915_MADV_PURGED) |
|
- | 919 | i915_gem_object_truncate(obj); |
|
- | 920 | /* and flush dirty cachelines in case the object isn't in the cpu write |
|
- | 921 | * domain anymore. */ |
|
- | 922 | if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
|
- | 923 | i915_gem_clflush_object(obj); |
|
- | 924 | intel_gtt_chipset_flush(); |
|
- | 925 | } |
|
- | 926 | } |
|
- | 927 | ||
- | 928 | if (needs_clflush_after) |
|
- | 929 | intel_gtt_chipset_flush(); |
|
- | 930 | ||
- | 931 | return ret; |
|
- | 932 | } |
|
- | 933 | ||
- | 934 | /** |
|
- | 935 | * Writes data to the object referenced by handle. |
|
- | 936 | * |
|
- | 937 | * On error, the contents of the buffer that were to be modified are undefined. |
|
- | 938 | */ |
|
- | 939 | int |
|
- | 940 | i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, |
|
- | 941 | struct drm_file *file) |
|
- | 942 | { |
|
- | 943 | struct drm_i915_gem_pwrite *args = data; |
|
- | 944 | struct drm_i915_gem_object *obj; |
|
- | 945 | int ret; |
|
- | 946 | ||
- | 947 | if (args->size == 0) |
|
- | 948 | return 0; |
|
- | 949 | ||
- | 950 | if (!access_ok(VERIFY_READ, |
|
- | 951 | (char __user *)(uintptr_t)args->data_ptr, |
|
- | 952 | args->size)) |
|
- | 953 | return -EFAULT; |
|
- | 954 | ||
- | 955 | ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr, |
|
- | 956 | args->size); |
|
- | 957 | if (ret) |
|
- | 958 | return -EFAULT; |
|
- | 959 | ||
- | 960 | ret = i915_mutex_lock_interruptible(dev); |
|
- | 961 | if (ret) |
|
- | 962 | return ret; |
|
- | 963 | ||
- | 964 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
|
- | 965 | if (&obj->base == NULL) { |
|
- | 966 | ret = -ENOENT; |
|
- | 967 | goto unlock; |
|
- | 968 | } |
|
- | 969 | ||
- | 970 | /* Bounds check destination. */ |
|
- | 971 | if (args->offset > obj->base.size || |
|
- | 972 | args->size > obj->base.size - args->offset) { |
|
- | 973 | ret = -EINVAL; |
|
- | 974 | goto out; |
|
- | 975 | } |
|
- | 976 | ||
- | 977 | /* prime objects have no backing filp to GEM pread/pwrite |
|
- | 978 | * pages from. |
|
- | 979 | */ |
|
- | 980 | if (!obj->base.filp) { |
|
546 | ret = PTR_ERR(page); |
981 | ret = -EINVAL; |
Line -... | Line 982... | ||
- | 982 | goto out; |
|
- | 983 | } |
|
- | 984 | ||
- | 985 | trace_i915_gem_object_pwrite(obj, args->offset, args->size); |
|
- | 986 | ||
- | 987 | ret = -EFAULT; |
|
- | 988 | /* We can only do the GTT pwrite on untiled buffers, as otherwise |
|
- | 989 | * it would end up going through the fenced access, and we'll get |
|
Line -... | Line 990... | ||
- | 990 | * different detiling behavior between reading and writing. |
|
- | 991 | * pread/pwrite currently are reading and writing from the CPU |
|
- | 992 | * perspective, requiring manual detiling by the client. |
|
- | 993 | */ |
|
Line -... | Line 994... | ||
- | 994 | if (obj->phys_obj) { |
|
- | 995 | ret = i915_gem_phys_pwrite(dev, obj, args, file); |
|
- | 996 | goto out; |
|
- | 997 | } |
|
- | 998 | ||
- | 999 | if (obj->cache_level == I915_CACHE_NONE && |
|
- | 1000 | obj->tiling_mode == I915_TILING_NONE && |
|
- | 1001 | obj->base.write_domain != I915_GEM_DOMAIN_CPU) { |
|
Line -... | Line 1002... | ||
- | 1002 | ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file); |
|
- | 1003 | /* Note that the gtt paths might fail with non-page-backed user |
|
Line -... | Line 1004... | ||
- | 1004 | * pointers (e.g. gtt mappings when moving data between |
|
- | 1005 | * textures). Fallback to the shmem path in that case. */ |
|
Line -... | Line 1006... | ||
- | 1006 | } |
|
- | 1007 | ||
- | 1008 | if (ret == -EFAULT || ret == -ENOSPC) |
|
- | 1009 | ret = i915_gem_shmem_pwrite(dev, obj, args, file); |
|
- | 1010 | ||
- | 1011 | out: |
|
- | 1012 | drm_gem_object_unreference(&obj->base); |
|
- | 1013 | unlock: |
|
Line -... | Line 1014... | ||
- | 1014 | mutex_unlock(&dev->struct_mutex); |
|
Line -... | Line 1015... | ||
- | 1015 | return ret; |
|
- | 1016 | } |
|
- | 1017 | ||
Line -... | Line 1018... | ||
- | 1018 | #endif |
|
- | 1019 | ||
Line -... | Line 1020... | ||
- | 1020 | int |
|
- | 1021 | i915_gem_check_wedge(struct drm_i915_private *dev_priv, |
|
- | 1022 | bool interruptible) |
|
- | 1023 | { |
|
- | 1024 | if (atomic_read(&dev_priv->mm.wedged)) { |
|
- | 1025 | struct completion *x = &dev_priv->error_completion; |
|
- | 1026 | bool recovery_complete; |
|
- | 1027 | unsigned long flags; |
|
- | 1028 | ||
- | 1029 | /* Give the error handler a chance to run. */ |
|
- | 1030 | spin_lock_irqsave(&x->wait.lock, flags); |
|
- | 1031 | recovery_complete = x->done > 0; |
|
- | 1032 | spin_unlock_irqrestore(&x->wait.lock, flags); |
|
- | 1033 | ||
- | 1034 | /* Non-interruptible callers can't handle -EAGAIN, hence return |
|
- | 1035 | * -EIO unconditionally for these. */ |
|
- | 1036 | if (!interruptible) |
|
- | 1037 | return -EIO; |
|
- | 1038 | ||
Line -... | Line 1039... | ||
- | 1039 | /* Recovery complete, but still wedged means reset failure. */ |
|
- | 1040 | if (recovery_complete) |
|
Line -... | Line 1041... | ||
- | 1041 | return -EIO; |
|
Line -... | Line 1042... | ||
- | 1042 | ||
- | 1043 | return -EAGAIN; |
|
- | 1044 | } |
|
- | 1045 | ||
Line -... | Line 1046... | ||
- | 1046 | return 0; |
|
Line -... | Line 1047... | ||
- | 1047 | } |
|
- | 1048 | ||
- | 1049 | /* |
|
Line -... | Line 1050... | ||
- | 1050 | * Compare seqno against outstanding lazy request. Emit a request if they are |
|
- | 1051 | * equal. |
|
Line -... | Line 1052... | ||
- | 1052 | */ |
|
- | 1053 | static int |
|
- | 1054 | i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno) |
|
- | 1055 | { |
|
- | 1056 | int ret; |
|
- | 1057 | ||
Line -... | Line 1058... | ||
- | 1058 | BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex)); |
|
- | 1059 | ||
- | 1060 | ret = 0; |
|
- | 1061 | if (seqno == ring->outstanding_lazy_request) |
|
Line -... | Line 1062... | ||
- | 1062 | ret = i915_add_request(ring, NULL, NULL); |
|
Line -... | Line 1063... | ||
- | 1063 | ||
- | 1064 | return ret; |
|
- | 1065 | } |
|
Line -... | Line 1066... | ||
- | 1066 | ||
- | 1067 | /** |
|
- | 1068 | * __wait_seqno - wait until execution of seqno has finished |
|
- | 1069 | * @ring: the ring expected to report seqno |
|
- | 1070 | * @seqno: duh! |
|
- | 1071 | * @interruptible: do an interruptible wait (normally yes) |
|
- | 1072 | * @timeout: in - how long to wait (NULL forever); out - how much time remaining |
|
- | 1073 | * |
|
- | 1074 | * Returns 0 if the seqno was found within the alloted time. Else returns the |
|
- | 1075 | * errno with remaining time filled in timeout argument. |
|
- | 1076 | */ |
|
- | 1077 | static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno, |
|
- | 1078 | bool interruptible, struct timespec *timeout) |
|
- | 1079 | { |
|
- | 1080 | drm_i915_private_t *dev_priv = ring->dev->dev_private; |
|
- | 1081 | struct timespec before, now, wait_time={1,0}; |
|
- | 1082 | unsigned long timeout_jiffies; |
|
- | 1083 | long end; |
|
- | 1084 | bool wait_forever = true; |
|
Line -... | Line 1085... | ||
- | 1085 | int ret; |
|
- | 1086 | ||
- | 1087 | if (i915_seqno_passed(ring->get_seqno(ring, true), seqno)) |
|
- | 1088 | return 0; |
|
- | 1089 | ||
- | 1090 | trace_i915_gem_request_wait_begin(ring, seqno); |
|
Line -... | Line 1091... | ||
- | 1091 | ||
- | 1092 | if (timeout != NULL) { |
|
Line -... | Line 1093... | ||
- | 1093 | wait_time = *timeout; |
|
- | 1094 | wait_forever = false; |
|
- | 1095 | } |
|
- | 1096 | ||
- | 1097 | // timeout_jiffies = timespec_to_jiffies(&wait_time); |
|
- | 1098 | ||
- | 1099 | if (WARN_ON(!ring->irq_get(ring))) |
|
- | 1100 | return -ENODEV; |
|
- | 1101 | #if 0 |
|
- | 1102 | ||
- | 1103 | /* Record current time in case interrupted by signal, or wedged * */ |
|
Line -... | Line 1104... | ||
- | 1104 | getrawmonotonic(&before); |
|
- | 1105 | ||
Line -... | Line 1106... | ||
- | 1106 | #define EXIT_COND \ |
|
- | 1107 | (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \ |
|
- | 1108 | atomic_read(&dev_priv->mm.wedged)) |
|
Line -... | Line 1109... | ||
- | 1109 | do { |
|
- | 1110 | end = wait_event_timeout(ring->irq_queue, EXIT_COND, |
|
- | 1111 | timeout_jiffies); |
|
Line -... | Line 1112... | ||
- | 1112 | ||
- | 1113 | ret = i915_gem_check_wedge(dev_priv, interruptible); |
|
- | 1114 | if (ret) |
|
- | 1115 | end = ret; |
|
- | 1116 | } while (end == 0 && wait_forever); |
|
- | 1117 | ||
- | 1118 | getrawmonotonic(&now); |
|
- | 1119 | ||
- | 1120 | ring->irq_put(ring); |
|
- | 1121 | trace_i915_gem_request_wait_end(ring, seqno); |
|
- | 1122 | #undef EXIT_COND |
|
- | 1123 | ||
- | 1124 | if (timeout) { |
|
- | 1125 | // struct timespec sleep_time = timespec_sub(now, before); |
|
- | 1126 | // *timeout = timespec_sub(*timeout, sleep_time); |
|
- | 1127 | } |
|
- | 1128 | ||
- | 1129 | switch (end) { |
|
- | 1130 | case -EIO: |
|
- | 1131 | case -EAGAIN: /* Wedged */ |
|
- | 1132 | case -ERESTARTSYS: /* Signal */ |
|
- | 1133 | return (int)end; |
|
- | 1134 | case 0: /* Timeout */ |
|
- | 1135 | // if (timeout) |
|
- | 1136 | // set_normalized_timespec(timeout, 0, 0); |
|
- | 1137 | return -ETIME; |
|
- | 1138 | default: /* Completed */ |
|
- | 1139 | WARN_ON(end < 0); /* We're not aware of other errors */ |
|
- | 1140 | return 0; |
|
- | 1141 | } |
|
- | 1142 | #endif |
|
- | 1143 | ||
- | 1144 | #define EXIT_COND \ |
|
- | 1145 | (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \ |
|
- | 1146 | atomic_read(&dev_priv->mm.wedged)) |
|
- | 1147 | wait_event(ring->irq_queue, EXIT_COND); |
|
Line -... | Line 1251... | ||
- | 1251 | ||
- | 1252 | ||
- | 1253 | ||
- | 1254 | ||
- | 1255 | ||
- | 1256 | /** |
|
- | 1257 | * i915_gem_release_mmap - remove physical page mappings |
|
- | 1258 | * @obj: obj in question |
|
- | 1259 | * |
|
- | 1260 | * Preserve the reservation of the mmapping with the DRM core code, but |
|
- | 1261 | * relinquish ownership of the pages back to the system. |
|
- | 1262 | * |
|
- | 1263 | * It is vital that we remove the page mapping if we have mapped a tiled |
|
- | 1264 | * object through the GTT and then lose the fence register due to |
|
- | 1265 | * resource pressure. Similarly if the object has been moved out of the |
|
- | 1266 | * aperture, than pages mapped into userspace must be revoked. Removing the |
|
- | 1267 | * mapping will then trigger a page fault on the next user access, allowing |
|
- | 1268 | * fixup by i915_gem_fault(). |
|
- | 1269 | */ |
|
- | 1270 | void |
|
- | 1271 | i915_gem_release_mmap(struct drm_i915_gem_object *obj) |
|
- | 1272 | { |
|
- | 1273 | if (!obj->fault_mappable) |
|
- | 1274 | return; |
|
Line -... | Line 1275... | ||
- | 1275 | ||
- | 1276 | if (obj->base.dev->dev_mapping) |
|
Line 650... | Line 1277... | ||
650 | 1277 | // unmap_mapping_range(obj->base.dev->dev_mapping, |
|
651 | 1278 | // (loff_t)obj->base.map_list.hash.key< |
|
652 | 1279 | // obj->base.size, 1); |
|
653 | 1280 | ||
Line 729... | Line 1356... | ||
729 | * the power-of-tile object size. |
1356 | * the power-of-tile object size. |
730 | */ |
1357 | */ |
731 | return i915_gem_get_gtt_size(dev, size, tiling_mode); |
1358 | return i915_gem_get_gtt_size(dev, size, tiling_mode); |
732 | } |
1359 | } |
Line -... | Line 1360... | ||
- | 1360 | ||
- | 1361 | /* Immediately discard the backing storage */ |
|
- | 1362 | static void |
|
- | 1363 | i915_gem_object_truncate(struct drm_i915_gem_object *obj) |
|
- | 1364 | { |
|
- | 1365 | // struct inode *inode; |
|
- | 1366 | ||
- | 1367 | // i915_gem_object_free_mmap_offset(obj); |
|
- | 1368 | ||
- | 1369 | // if (obj->base.filp == NULL) |
|
- | 1370 | // return; |
|
- | 1371 | ||
- | 1372 | /* Our goal here is to return as much of the memory as |
|
- | 1373 | * is possible back to the system as we are called from OOM. |
|
- | 1374 | * To do this we must instruct the shmfs to drop all of its |
|
- | 1375 | * backing pages, *now*. |
|
- | 1376 | */ |
|
- | 1377 | // inode = obj->base.filp->f_path.dentry->d_inode; |
|
- | 1378 | // shmem_truncate_range(inode, 0, (loff_t)-1); |
|
- | 1379 | ||
- | 1380 | obj->madv = __I915_MADV_PURGED; |
|
- | 1381 | } |
|
- | 1382 | ||
- | 1383 | static inline int |
|
- | 1384 | i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) |
|
- | 1385 | { |
|
- | 1386 | return obj->madv == I915_MADV_DONTNEED; |
|
- | 1387 | } |
|
- | 1388 | ||
- | 1389 | static void |
|
- | 1390 | i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) |
|
- | 1391 | { |
|
- | 1392 | int ret, i; |
|
- | 1393 | ||
- | 1394 | BUG_ON(obj->madv == __I915_MADV_PURGED); |
|
- | 1395 | ||
- | 1396 | ret = i915_gem_object_set_to_cpu_domain(obj, true); |
|
- | 1397 | if (ret) { |
|
- | 1398 | /* In the event of a disaster, abandon all caches and |
|
- | 1399 | * hope for the best. |
|
- | 1400 | */ |
|
- | 1401 | WARN_ON(ret != -EIO); |
|
- | 1402 | i915_gem_clflush_object(obj); |
|
- | 1403 | obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
|
- | 1404 | } |
|
- | 1405 | ||
- | 1406 | if (obj->madv == I915_MADV_DONTNEED) |
|
Line -... | Line 1407... | ||
- | 1407 | obj->dirty = 0; |
|
- | 1408 | ||
Line -... | Line 1409... | ||
- | 1409 | for (i = 0; i < obj->pages.nents; i++) |
|
- | 1410 | FreePage(obj->pages.page[i]); |
|
- | 1411 | ||
Line -... | Line 1412... | ||
- | 1412 | obj->dirty = 0; |
|
- | 1413 | kfree(obj->pages.page); |
|
- | 1414 | } |
|
- | 1415 | ||
- | 1416 | static int |
|
- | 1417 | i915_gem_object_put_pages(struct drm_i915_gem_object *obj) |
|
- | 1418 | { |
|
Line -... | Line 1419... | ||
- | 1419 | const struct drm_i915_gem_object_ops *ops = obj->ops; |
|
- | 1420 | ||
- | 1421 | if (obj->pages.page == NULL) |
|
- | 1422 | return 0; |
|
Line -... | Line 1423... | ||
- | 1423 | ||
- | 1424 | BUG_ON(obj->gtt_space); |
|
Line -... | Line 1425... | ||
- | 1425 | ||
- | 1426 | if (obj->pages_pin_count) |
|
- | 1427 | return -EBUSY; |
|
- | 1428 | ||
- | 1429 | ops->put_pages(obj); |
|
- | 1430 | obj->pages.page = NULL; |
|
Line 733... | Line 1431... | ||
733 | 1431 | ||
734 | 1432 | list_del(&obj->gtt_list); |
|
735 | - | ||
736 | 1433 | if (i915_gem_object_is_purgeable(obj)) |
|
- | 1434 | i915_gem_object_truncate(obj); |
|
737 | 1435 | ||
738 | - | ||
Line 739... | Line 1436... | ||
739 | 1436 | return 0; |
|
740 | 1437 | } |
|
741 | 1438 | ||
742 | 1439 | ||
743 | 1440 | ||
744 | 1441 | ||
745 | 1442 | ||
746 | 1443 | ||
Line 747... | Line -... | ||
747 | - | ||
748 | static int |
1444 | |
749 | i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, |
1445 | |
750 | gfp_t gfpmask) |
1446 | static int |
751 | { |
1447 | i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) |
Line 752... | Line 1448... | ||
752 | int page_count, i; |
1448 | { |
753 | struct page *page; |
1449 | dma_addr_t page; |
Line 754... | Line 1450... | ||
754 | 1450 | int page_count, i; |
|
755 | /* Get the list of pages out of our struct file. They'll be pinned |
- | |
Line -... | Line 1451... | ||
- | 1451 | ||
- | 1452 | /* Get the list of pages out of our struct file. They'll be pinned |
|
Line 756... | Line 1453... | ||
756 | * at this point until we release them. |
1453 | * at this point until we release them. |
Line 757... | Line 1454... | ||
757 | */ |
1454 | */ |
758 | page_count = obj->base.size / PAGE_SIZE; |
1455 | page_count = obj->base.size / PAGE_SIZE; |
759 | BUG_ON(obj->pages != NULL); |
1456 | BUG_ON(obj->pages.page != NULL); |
- | 1457 | obj->pages.page = malloc(page_count * sizeof(dma_addr_t)); |
|
- | 1458 | if (obj->pages.page == NULL) |
|
- | 1459 | return -ENOMEM; |
|
- | 1460 | ||
Line 760... | Line 1461... | ||
760 | obj->pages = malloc(page_count * sizeof(struct page *)); |
1461 | for (i = 0; i < page_count; i++) { |
761 | if (obj->pages == NULL) |
- | |
762 | return -ENOMEM; |
- | |
763 | 1462 | page = AllocPage(); // oh-oh |
|
Line -... | Line 1463... | ||
- | 1463 | if ( page == 0 ) |
|
- | 1464 | goto err_pages; |
|
- | 1465 | ||
- | 1466 | obj->pages.page[i] = page; |
|
- | 1467 | }; |
|
- | 1468 | ||
764 | 1469 | obj->pages.nents = page_count; |
|
- | 1470 | ||
765 | for (i = 0; i < page_count; i++) { |
1471 | |
766 | page = (struct page*)AllocPage(); // oh-oh |
1472 | // if (obj->tiling_mode != I915_TILING_NONE) |
767 | if (IS_ERR(page)) |
1473 | // i915_gem_object_do_bit_17_swizzle(obj); |
- | 1474 | ||
768 | goto err_pages; |
1475 | return 0; |
769 | - | ||
770 | obj->pages[i] = page; |
- | |
Line 771... | Line 1476... | ||
771 | } |
1476 | |
772 | 1477 | err_pages: |
|
Line 773... | Line 1478... | ||
773 | // if (obj->tiling_mode != I915_TILING_NONE) |
1478 | while (i--) |
774 | // i915_gem_object_do_bit_17_swizzle(obj); |
- | |
Line 775... | Line 1479... | ||
775 | 1479 | FreePage(obj->pages.page[i]); |
|
776 | - | ||
777 | 1480 | ||
778 | return 0; |
1481 | free(obj->pages.page); |
Line 779... | Line 1482... | ||
779 | 1482 | obj->pages.page = NULL; |
|
780 | err_pages: |
1483 | obj->pages.nents = 0; |
781 | while (i--) |
1484 | |
Line 782... | Line 1485... | ||
782 | FreePage((addr_t)obj->pages[i]); |
1485 | return -ENOMEM; |
783 | 1486 | } |
|
784 | free(obj->pages); |
1487 | |
Line 828... | Line 1531... | ||
828 | 1531 | ||
829 | /* Move from whatever list we were on to the tail of execution. */ |
1532 | /* Move from whatever list we were on to the tail of execution. */ |
830 | list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); |
1533 | list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); |
Line 831... | Line 1534... | ||
831 | list_move_tail(&obj->ring_list, &ring->active_list); |
1534 | list_move_tail(&obj->ring_list, &ring->active_list); |
832 | - | ||
833 | obj->last_rendering_seqno = seqno; |
- | |
834 | if (obj->fenced_gpu_access) { |
- | |
835 | struct drm_i915_fence_reg *reg; |
- | |
Line -... | Line 1535... | ||
- | 1535 | ||
836 | 1536 | obj->last_read_seqno = seqno; |
|
- | 1537 | ||
- | 1538 | if (obj->fenced_gpu_access) { |
|
- | 1539 | obj->last_fenced_seqno = seqno; |
|
837 | BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE); |
1540 | |
Line 838... | Line 1541... | ||
838 | 1541 | /* Bump MRU to take account of the delayed flush */ |
|
839 | obj->last_fenced_seqno = seqno; |
1542 | if (obj->fence_reg != I915_FENCE_REG_NONE) { |
- | 1543 | struct drm_i915_fence_reg *reg; |
|
840 | obj->last_fenced_ring = ring; |
1544 | |
841 | 1545 | reg = &dev_priv->fence_regs[obj->fence_reg]; |
|
842 | reg = &dev_priv->fence_regs[obj->fence_reg]; |
- | |
843 | list_move_tail(®->lru_list, &dev_priv->mm.fence_list); |
- | |
844 | } |
- | |
845 | } |
- | |
846 | - | ||
847 | static void |
- | |
848 | i915_gem_object_move_off_active(struct drm_i915_gem_object *obj) |
1546 | list_move_tail(®->lru_list, |
Line 849... | Line 1547... | ||
849 | { |
1547 | &dev_priv->mm.fence_list); |
850 | list_del_init(&obj->ring_list); |
1548 | } |
851 | obj->last_rendering_seqno = 0; |
1549 | } |
852 | } |
1550 | } |
853 | 1551 | ||
Line -... | Line 1552... | ||
- | 1552 | static void |
|
854 | static void |
1553 | i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) |
855 | i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj) |
- | |
856 | { |
- | |
857 | struct drm_device *dev = obj->base.dev; |
- | |
858 | drm_i915_private_t *dev_priv = dev->dev_private; |
- | |
Line 859... | Line -... | ||
859 | - | ||
860 | BUG_ON(!obj->active); |
1554 | { |
861 | list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list); |
- | |
862 | 1555 | struct drm_device *dev = obj->base.dev; |
|
863 | i915_gem_object_move_off_active(obj); |
- | |
Line 864... | Line -... | ||
864 | } |
- | |
865 | - | ||
866 | static void |
- | |
867 | i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) |
1556 | struct drm_i915_private *dev_priv = dev->dev_private; |
Line 868... | Line 1557... | ||
868 | { |
1557 | |
869 | struct drm_device *dev = obj->base.dev; |
- | |
870 | struct drm_i915_private *dev_priv = dev->dev_private; |
1558 | BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS); |
Line -... | Line 1559... | ||
- | 1559 | BUG_ON(!obj->active); |
|
- | 1560 | ||
- | 1561 | if (obj->pin_count) /* are we a framebuffer? */ |
|
- | 1562 | intel_mark_fb_idle(obj); |
|
871 | 1563 | ||
872 | if (obj->pin_count != 0) |
1564 | list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); |
Line 873... | Line 1565... | ||
873 | list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); |
1565 | |
874 | else |
- | |
875 | list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); |
1566 | list_del_init(&obj->ring_list); |
Line 876... | Line 1567... | ||
876 | 1567 | obj->ring = NULL; |
|
877 | BUG_ON(!list_empty(&obj->gpu_write_list)); |
1568 | |
Line 878... | Line -... | ||
878 | BUG_ON(!obj->active); |
- | |
879 | obj->ring = NULL; |
1569 | obj->last_read_seqno = 0; |
880 | 1570 | obj->last_write_seqno = 0; |
|
881 | i915_gem_object_move_off_active(obj); |
1571 | obj->base.write_domain = 0; |
882 | obj->fenced_gpu_access = false; |
- | |
883 | - | ||
884 | obj->active = 0; |
- | |
885 | obj->pending_gpu_write = false; |
1572 | |
886 | drm_gem_object_unreference(&obj->base); |
- | |
887 | 1573 | obj->last_fenced_seqno = 0; |
|
888 | WARN_ON(i915_verify_lists(dev)); |
- | |
Line 889... | Line 1574... | ||
889 | } |
1574 | obj->fenced_gpu_access = false; |
890 | - | ||
- | 1575 | ||
- | 1576 | obj->active = 0; |
|
Line 891... | Line 1577... | ||
891 | /* Immediately discard the backing storage */ |
1577 | drm_gem_object_unreference(&obj->base); |
892 | static void |
- | |
893 | i915_gem_object_truncate(struct drm_i915_gem_object *obj) |
- | |
894 | { |
- | |
895 | struct inode *inode; |
1578 | |
Line 896... | Line 1579... | ||
896 | 1579 | WARN_ON(i915_verify_lists(dev)); |
|
897 | /* Our goal here is to return as much of the memory as |
1580 | } |
898 | * is possible back to the system as we are called from OOM. |
- | |
899 | * To do this we must instruct the shmfs to drop all of its |
1581 | |
- | 1582 | static u32 |
|
900 | * backing pages, *now*. |
1583 | i915_gem_get_seqno(struct drm_device *dev) |
Line 901... | Line -... | ||
901 | */ |
- | |
902 | 1584 | { |
|
903 | obj->madv = __I915_MADV_PURGED; |
- | |
904 | } |
- | |
905 | - | ||
906 | static inline int |
- | |
907 | i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) |
- | |
908 | { |
- | |
909 | return obj->madv == I915_MADV_DONTNEED; |
- | |
910 | } |
- | |
911 | - | ||
912 | static void |
- | |
913 | i915_gem_process_flushing_list(struct intel_ring_buffer *ring, |
- | |
914 | uint32_t flush_domains) |
- | |
915 | { |
- | |
916 | struct drm_i915_gem_object *obj, *next; |
- | |
917 | 1585 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
Line 918... | Line 1586... | ||
918 | list_for_each_entry_safe(obj, next, |
1586 | u32 seqno = dev_priv->next_seqno; |
919 | &ring->gpu_write_list, |
1587 | |
920 | gpu_write_list) { |
1588 | /* reserve 0 for non-seqno */ |
921 | if (obj->base.write_domain & flush_domains) { |
1589 | if (++dev_priv->next_seqno == 0) |
922 | uint32_t old_write_domain = obj->base.write_domain; |
1590 | dev_priv->next_seqno = 1; |
923 | 1591 | ||
- | 1592 | return seqno; |
|
- | 1593 | } |
|
924 | obj->base.write_domain = 0; |
1594 | |
925 | list_del_init(&obj->gpu_write_list); |
1595 | u32 |
926 | i915_gem_object_move_to_active(obj, ring, |
1596 | i915_gem_next_request_seqno(struct intel_ring_buffer *ring) |
Line -... | Line 1597... | ||
- | 1597 | { |
|
- | 1598 | if (ring->outstanding_lazy_request == 0) |
|
- | 1599 | ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev); |
|
- | 1600 | ||
- | 1601 | return ring->outstanding_lazy_request; |
|
- | 1602 | } |
|
- | 1603 | ||
- | 1604 | int |
|
- | 1605 | i915_add_request(struct intel_ring_buffer *ring, |
|
- | 1606 | struct drm_file *file, |
|
- | 1607 | u32 *out_seqno) |
|
- | 1608 | { |
|
927 | i915_gem_next_request_seqno(ring)); |
1609 | drm_i915_private_t *dev_priv = ring->dev->dev_private; |
- | 1610 | struct drm_i915_gem_request *request; |
|
- | 1611 | u32 request_ring_position; |
|
- | 1612 | u32 seqno; |
|
- | 1613 | int was_empty; |
|
- | 1614 | int ret; |
|
- | 1615 | ||
- | 1616 | /* |
|
- | 1617 | * Emit any outstanding flushes - execbuf can fail to emit the flush |
|
- | 1618 | * after having emitted the batchbuffer command. Hence we need to fix |
|
- | 1619 | * things up similar to emitting the lazy request. The difference here |
|
Line 928... | Line 1620... | ||
928 | 1620 | * is that the flush _must_ happen before the next request, no matter |
|
929 | trace_i915_gem_object_change_domain(obj, |
1621 | * what. |
- | 1622 | */ |
|
930 | obj->base.read_domains, |
1623 | ret = intel_ring_flush_all_caches(ring); |
- | 1624 | if (ret) |
|
Line 931... | Line 1625... | ||
931 | old_write_domain); |
1625 | return ret; |
Line 932... | Line 1626... | ||
932 | } |
1626 | |
933 | } |
1627 | request = kmalloc(sizeof(*request), GFP_KERNEL); |
- | 1628 | if (request == NULL) |
|
934 | } |
1629 | return -ENOMEM; |
935 | 1630 | ||
936 | int |
1631 | seqno = i915_gem_next_request_seqno(ring); |
- | 1632 | ||
Line 937... | Line 1633... | ||
937 | i915_add_request(struct intel_ring_buffer *ring, |
1633 | /* Record the position of the start of the request so that |
Line 938... | Line 1634... | ||
938 | struct drm_file *file, |
1634 | * should we detect the updated seqno part-way through the |
939 | struct drm_i915_gem_request *request) |
1635 | * GPU processing the request, we never over-estimate the |
940 | { |
1636 | * position of the head. |
941 | drm_i915_private_t *dev_priv = ring->dev->dev_private; |
1637 | */ |
942 | uint32_t seqno; |
1638 | request_ring_position = intel_ring_get_tail(ring); |
943 | int was_empty; |
1639 | |
944 | int ret; |
1640 | ret = ring->add_request(ring, &seqno); |
945 | 1641 | if (ret) { |
|
946 | BUG_ON(request == NULL); |
1642 | kfree(request); |
- | 1643 | return ret; |
|
947 | 1644 | } |
|
948 | ret = ring->add_request(ring, &seqno); |
- | |
949 | if (ret) |
1645 | |
Line -... | Line 1646... | ||
- | 1646 | trace_i915_gem_request_add(ring, seqno); |
|
- | 1647 | ||
- | 1648 | request->seqno = seqno; |
|
- | 1649 | request->ring = ring; |
|
Line -... | Line 1650... | ||
- | 1650 | request->tail = request_ring_position; |
|
- | 1651 | request->emitted_jiffies = GetTimerTicks(); |
|
- | 1652 | was_empty = list_empty(&ring->request_list); |
|
- | 1653 | list_add_tail(&request->list, &ring->request_list); |
|
- | 1654 | request->file_priv = NULL; |
|
Line -... | Line 1655... | ||
- | 1655 | ||
- | 1656 | ||
- | 1657 | ring->outstanding_lazy_request = 0; |
|
Line -... | Line 1658... | ||
- | 1658 | ||
- | 1659 | if (!dev_priv->mm.suspended) { |
|
- | 1660 | if (i915_enable_hangcheck) { |
|
- | 1661 | // mod_timer(&dev_priv->hangcheck_timer, |
|
Line -... | Line 1662... | ||
- | 1662 | // jiffies + |
|
- | 1663 | // msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)); |
|
Line -... | Line 1664... | ||
- | 1664 | } |
|
- | 1665 | if (was_empty) { |
|
- | 1666 | queue_delayed_work(dev_priv->wq, |
|
Line -... | Line 1667... | ||
- | 1667 | &dev_priv->mm.retire_work, HZ); |
|
- | 1668 | intel_mark_busy(dev_priv->dev); |
|
- | 1669 | } |
|
Line -... | Line 1670... | ||
- | 1670 | } |
|
- | 1671 | ||
- | 1672 | if (out_seqno) |
|
- | 1673 | *out_seqno = seqno; |
|
Line -... | Line 1674... | ||
- | 1674 | return 0; |
|
- | 1675 | } |
|
Line -... | Line 1676... | ||
- | 1676 | ||
Line -... | Line 1677... | ||
- | 1677 | ||
- | 1678 | ||
Line -... | Line 1679... | ||
- | 1679 | ||
- | 1680 | static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, |
|
- | 1681 | struct intel_ring_buffer *ring) |
|
- | 1682 | { |
|
Line -... | Line 1683... | ||
- | 1683 | while (!list_empty(&ring->request_list)) { |
|
- | 1684 | struct drm_i915_gem_request *request; |
|
Line -... | Line 1685... | ||
- | 1685 | ||
- | 1686 | request = list_first_entry(&ring->request_list, |
|
- | 1687 | struct drm_i915_gem_request, |
|
- | 1688 | list); |
|
- | 1689 | ||
- | 1690 | list_del(&request->list); |
|
Line -... | Line 1691... | ||
- | 1691 | // i915_gem_request_remove_from_client(request); |
|
- | 1692 | kfree(request); |
|
Line -... | Line 1693... | ||
- | 1693 | } |
|
- | 1694 | ||
- | 1695 | while (!list_empty(&ring->active_list)) { |
|
- | 1696 | struct drm_i915_gem_object *obj; |
|
- | 1697 | ||
- | 1698 | obj = list_first_entry(&ring->active_list, |
|
- | 1699 | struct drm_i915_gem_object, |
|
- | 1700 | ring_list); |
|
- | 1701 | ||
Line -... | Line 1702... | ||
- | 1702 | i915_gem_object_move_to_inactive(obj); |
|
- | 1703 | } |
|
950 | return ret; |
1704 | } |
Line 951... | Line 1705... | ||
951 | 1705 | ||
952 | trace_i915_gem_request_add(ring, seqno); |
1706 | static void i915_gem_reset_fences(struct drm_device *dev) |
953 | 1707 | { |
|
954 | request->seqno = seqno; |
1708 | struct drm_i915_private *dev_priv = dev->dev_private; |
955 | request->ring = ring; |
1709 | int i; |
956 | request->emitted_jiffies = jiffies; |
1710 | |
957 | was_empty = list_empty(&ring->request_list); |
1711 | for (i = 0; i < dev_priv->num_fence_regs; i++) { |
958 | list_add_tail(&request->list, &ring->request_list); |
1712 | struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; |
Line 959... | Line 1713... | ||
959 | 1713 | ||
960 | 1714 | i915_gem_write_fence(dev, i, NULL); |
|
Line 961... | Line 1715... | ||
961 | ring->outstanding_lazy_request = false; |
1715 | |
Line 962... | Line 1716... | ||
962 | 1716 | if (reg->obj) |
|
Line 963... | Line 1717... | ||
963 | if (!dev_priv->mm.suspended) { |
1717 | i915_gem_object_fence_lost(reg->obj); |
964 | if (i915_enable_hangcheck) { |
1718 | |
965 | // mod_timer(&dev_priv->hangcheck_timer, |
1719 | reg->pin_count = 0; |
Line 1023... | Line 1777... | ||
1023 | 1777 | ||
1024 | if (!i915_seqno_passed(seqno, request->seqno)) |
1778 | if (!i915_seqno_passed(seqno, request->seqno)) |
Line 1025... | Line 1779... | ||
1025 | break; |
1779 | break; |
- | 1780 | ||
- | 1781 | trace_i915_gem_request_retire(ring, request->seqno); |
|
- | 1782 | /* We know the GPU must have read the request to have |
|
- | 1783 | * sent us the seqno + interrupt, so use the position |
|
- | 1784 | * of tail of the request to update the last known position |
|
- | 1785 | * of the GPU head. |
|
Line 1026... | Line 1786... | ||
1026 | 1786 | */ |
|
1027 | trace_i915_gem_request_retire(ring, request->seqno); |
1787 | ring->last_retired_head = request->tail; |
1028 | 1788 | ||
Line 1038... | Line 1798... | ||
1038 | 1798 | ||
1039 | obj = list_first_entry(&ring->active_list, |
1799 | obj = list_first_entry(&ring->active_list, |
1040 | struct drm_i915_gem_object, |
1800 | struct drm_i915_gem_object, |
Line 1041... | Line 1801... | ||
1041 | ring_list); |
1801 | ring_list); |
1042 | 1802 | ||
Line 1043... | Line -... | ||
1043 | if (!i915_seqno_passed(seqno, obj->last_rendering_seqno)) |
- | |
1044 | break; |
- | |
1045 | - | ||
1046 | if (obj->base.write_domain != 0) |
1803 | if (!i915_seqno_passed(seqno, obj->last_read_seqno)) |
1047 | i915_gem_object_move_to_flushing(obj); |
1804 | break; |
Line 1048... | Line 1805... | ||
1048 | else |
1805 | |
1049 | i915_gem_object_move_to_inactive(obj); |
1806 | i915_gem_object_move_to_inactive(obj); |
Line 1060... | Line 1817... | ||
1060 | 1817 | ||
1061 | void |
1818 | void |
1062 | i915_gem_retire_requests(struct drm_device *dev) |
1819 | i915_gem_retire_requests(struct drm_device *dev) |
1063 | { |
1820 | { |
- | 1821 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
1064 | drm_i915_private_t *dev_priv = dev->dev_private; |
1822 | struct intel_ring_buffer *ring; |
Line 1065... | Line -... | ||
1065 | int i; |
- | |
1066 | - | ||
1067 | if (!list_empty(&dev_priv->mm.deferred_free_list)) { |
- | |
1068 | struct drm_i915_gem_object *obj, *next; |
- | |
1069 | - | ||
1070 | /* We must be careful that during unbind() we do not |
- | |
1071 | * accidentally infinitely recurse into retire requests. |
- | |
1072 | * Currently: |
- | |
1073 | * retire -> free -> unbind -> wait -> retire_ring |
1823 | int i; |
1074 | */ |
- | |
1075 | list_for_each_entry_safe(obj, next, |
- | |
1076 | &dev_priv->mm.deferred_free_list, |
- | |
1077 | mm_list) |
- | |
1078 | i915_gem_free_object_tail(obj); |
- | |
1079 | } |
- | |
1080 | 1824 | ||
1081 | for (i = 0; i < I915_NUM_RINGS; i++) |
1825 | for_each_ring(ring, dev_priv, i) |
Line 1082... | Line 1826... | ||
1082 | i915_gem_retire_requests_ring(&dev_priv->ring[i]); |
1826 | i915_gem_retire_requests_ring(ring); |
1083 | } |
1827 | } |
1084 | 1828 | ||
1085 | static void |
1829 | static void |
1086 | i915_gem_retire_work_handler(struct work_struct *work) |
1830 | i915_gem_retire_work_handler(struct work_struct *work) |
- | 1831 | { |
|
1087 | { |
1832 | drm_i915_private_t *dev_priv; |
1088 | drm_i915_private_t *dev_priv; |
1833 | struct drm_device *dev; |
Line 1089... | Line 1834... | ||
1089 | struct drm_device *dev; |
1834 | struct intel_ring_buffer *ring; |
Line 1107... | Line 1852... | ||
1107 | 1852 | ||
1108 | /* Send a periodic flush down the ring so we don't hold onto GEM |
1853 | /* Send a periodic flush down the ring so we don't hold onto GEM |
1109 | * objects indefinitely. |
1854 | * objects indefinitely. |
1110 | */ |
1855 | */ |
1111 | idle = true; |
- | |
1112 | for (i = 0; i < I915_NUM_RINGS; i++) { |
1856 | idle = true; |
1113 | struct intel_ring_buffer *ring = &dev_priv->ring[i]; |
- | |
1114 | 1857 | for_each_ring(ring, dev_priv, i) { |
|
1115 | if (!list_empty(&ring->gpu_write_list)) { |
- | |
1116 | struct drm_i915_gem_request *request; |
- | |
1117 | int ret; |
- | |
1118 | - | ||
1119 | ret = i915_gem_flush_ring(ring, |
- | |
1120 | 0, I915_GEM_GPU_DOMAINS); |
- | |
1121 | request = kzalloc(sizeof(*request), GFP_KERNEL); |
- | |
1122 | if (ret || request == NULL || |
1858 | if (ring->gpu_caches_dirty) |
1123 | i915_add_request(ring, NULL, request)) |
- | |
1124 | kfree(request); |
- | |
Line 1125... | Line 1859... | ||
1125 | } |
1859 | i915_add_request(ring, NULL, NULL); |
1126 | 1860 | ||
Line 1127... | Line 1861... | ||
1127 | idle &= list_empty(&ring->request_list); |
1861 | idle &= list_empty(&ring->request_list); |
1128 | } |
1862 | } |
- | 1863 | ||
- | 1864 | if (!dev_priv->mm.suspended && !idle) |
|
Line 1129... | Line 1865... | ||
1129 | 1865 | queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); |
|
1130 | if (!dev_priv->mm.suspended && !idle) |
1866 | if (idle) |
1131 | queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); |
1867 | intel_mark_idle(dev); |
Line 1132... | Line 1868... | ||
1132 | 1868 | ||
1133 | mutex_unlock(&dev->struct_mutex); |
1869 | mutex_unlock(&dev->struct_mutex); |
1134 | // LEAVE(); |
1870 | // LEAVE(); |
- | 1871 | } |
|
1135 | } |
1872 | |
1136 | 1873 | /** |
|
1137 | /** |
1874 | * Ensures that an object will eventually get non-busy by flushing any required |
1138 | * Waits for a sequence number to be signaled, and cleans up the |
- | |
1139 | * request and object lists appropriately for that event. |
1875 | * write domains, emitting any outstanding lazy request and retiring and |
1140 | */ |
- | |
1141 | int |
- | |
1142 | i915_wait_request(struct intel_ring_buffer *ring, |
1876 | * completed requests. |
1143 | uint32_t seqno) |
- | |
1144 | { |
- | |
1145 | drm_i915_private_t *dev_priv = ring->dev->dev_private; |
- | |
1146 | u32 ier; |
- | |
1147 | int ret = 0; |
- | |
1148 | - | ||
1149 | BUG_ON(seqno == 0); |
- | |
1150 | - | ||
1151 | // if (atomic_read(&dev_priv->mm.wedged)) { |
- | |
1152 | // struct completion *x = &dev_priv->error_completion; |
- | |
1153 | // bool recovery_complete; |
- | |
1154 | // unsigned long flags; |
- | |
1155 | - | ||
1156 | /* Give the error handler a chance to run. */ |
- | |
1157 | // spin_lock_irqsave(&x->wait.lock, flags); |
- | |
1158 | // recovery_complete = x->done > 0; |
- | |
1159 | // spin_unlock_irqrestore(&x->wait.lock, flags); |
- | |
1160 | // |
- | |
1161 | // return recovery_complete ? -EIO : -EAGAIN; |
- | |
1162 | // } |
- | |
1163 | - | ||
1164 | if (seqno == ring->outstanding_lazy_request) { |
- | |
Line -... | Line 1877... | ||
- | 1877 | */ |
|
1165 | struct drm_i915_gem_request *request; |
1878 | static int |
1166 | 1879 | i915_gem_object_flush_active(struct drm_i915_gem_object *obj) |
|
1167 | request = kzalloc(sizeof(*request), GFP_KERNEL); |
- | |
1168 | if (request == NULL) |
1880 | { |
1169 | return -ENOMEM; |
- | |
Line 1170... | Line 1881... | ||
1170 | 1881 | int ret; |
|
1171 | ret = i915_add_request(ring, NULL, request); |
1882 | |
Line 1172... | Line -... | ||
1172 | if (ret) { |
- | |
1173 | kfree(request); |
- | |
1174 | return ret; |
- | |
1175 | } |
1883 | if (obj->active) { |
1176 | - | ||
1177 | seqno = request->seqno; |
- | |
1178 | } |
- | |
1179 | - | ||
1180 | if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { |
- | |
1181 | if (HAS_PCH_SPLIT(ring->dev)) |
- | |
1182 | ier = I915_READ(DEIER) | I915_READ(GTIER); |
1884 | ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno); |
Line 1183... | Line -... | ||
1183 | else |
- | |
Line 1184... | Line -... | ||
1184 | ier = I915_READ(IER); |
- | |
1185 | if (!ier) { |
- | |
1186 | DRM_ERROR("something (likely vbetool) disabled " |
- | |
1187 | "interrupts, re-enabling\n"); |
- | |
1188 | // ring->dev->driver->irq_preinstall(ring->dev); |
- | |
1189 | // ring->dev->driver->irq_postinstall(ring->dev); |
- | |
Line 1190... | Line -... | ||
1190 | } |
- | |
1191 | - | ||
1192 | trace_i915_gem_request_wait_begin(ring, seqno); |
- | |
1193 | - | ||
1194 | ring->waiting_seqno = seqno; |
- | |
1195 | if (ring->irq_get(ring)) { |
- | |
Line 1196... | Line -... | ||
1196 | // printf("enter wait\n"); |
- | |
1197 | wait_event(ring->irq_queue, |
- | |
1198 | i915_seqno_passed(ring->get_seqno(ring), seqno) |
- | |
1199 | || atomic_read(&dev_priv->mm.wedged)); |
- | |
Line 1200... | Line -... | ||
1200 | - | ||
1201 | ring->irq_put(ring); |
- | |
1202 | } else if (wait_for_atomic(i915_seqno_passed(ring->get_seqno(ring), |
- | |
1203 | seqno) || |
- | |
1204 | atomic_read(&dev_priv->mm.wedged), 3000)) |
- | |
1205 | ret = -EBUSY; |
- | |
1206 | ring->waiting_seqno = 0; |
- | |
1207 | - | ||
1208 | trace_i915_gem_request_wait_end(ring, seqno); |
- | |
1209 | } |
- | |
1210 | if (atomic_read(&dev_priv->mm.wedged)) |
- | |
1211 | ret = -EAGAIN; |
- | |
Line 1212... | Line 1885... | ||
1212 | 1885 | if (ret) |
|
1213 | if (ret && ret != -ERESTARTSYS) |
1886 | return ret; |
- | 1887 | ||
Line 1214... | Line 1888... | ||
1214 | DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n", |
1888 | i915_gem_retire_requests_ring(obj->ring); |
- | 1889 | } |
|
- | 1890 | ||
- | 1891 | return 0; |
|
- | 1892 | } |
|
- | 1893 | ||
1215 | __func__, ret, seqno, ring->get_seqno(ring), |
1894 | |
1216 | dev_priv->next_seqno); |
1895 | |
- | 1896 | ||
- | 1897 | ||
- | 1898 | ||
1217 | 1899 | ||
1218 | /* Directly dispatch request retiring. While we have the work queue |
1900 | |
1219 | * to handle this, the waiter on a request often wants an associated |
1901 | |
- | 1902 | ||
1220 | * buffer to have made it to the inactive list, and we would need |
1903 | /** |
- | 1904 | * i915_gem_object_sync - sync an object to a ring. |
|
- | 1905 | * |
|
1221 | * a separate wait queue to handle that. |
1906 | * @obj: object which may be in use on another ring. |
Line 1222... | Line -... | ||
1222 | */ |
- | |
1223 | if (ret == 0) |
1907 | * @to: ring we wish to use the object on. May be NULL. |
1224 | i915_gem_retire_requests_ring(ring); |
1908 | * |
1225 | - | ||
Line -... | Line 1909... | ||
- | 1909 | * This code is meant to abstract object synchronization with the GPU. |
|
1226 | return ret; |
1910 | * Calling with NULL implies synchronizing the object with the CPU |
1227 | } |
1911 | * rather than a particular GPU ring. |
- | 1912 | * |
|
1228 | 1913 | * Returns 0 if successful, else propagates up the lower layer error. |
|
1229 | /** |
1914 | */ |
- | 1915 | int |
|
- | 1916 | i915_gem_object_sync(struct drm_i915_gem_object *obj, |
|
- | 1917 | struct intel_ring_buffer *to) |
|
1230 | * Ensures that all rendering to the object has completed and the object is |
1918 | { |
1231 | * safe to unbind from the GTT or access from the CPU. |
1919 | struct intel_ring_buffer *from = obj->ring; |
1232 | */ |
1920 | u32 seqno; |
1233 | int |
- | |
Line -... | Line 1921... | ||
- | 1921 | int ret, idx; |
|
- | 1922 | ||
- | 1923 | if (from == NULL || to == from) |
|
- | 1924 | return 0; |
|
1234 | i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) |
1925 | |
1235 | { |
1926 | if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev)) |
Line 1236... | Line 1927... | ||
1236 | int ret; |
1927 | return i915_gem_object_wait_rendering(obj, false); |
1237 | 1928 | ||
1238 | /* This function only exists to support waiting for existing rendering, |
1929 | idx = intel_ring_sync_index(from, to); |
Line 1280... | Line 1971... | ||
1280 | * Unbinds an object from the GTT aperture. |
1971 | * Unbinds an object from the GTT aperture. |
1281 | */ |
1972 | */ |
1282 | int |
1973 | int |
1283 | i915_gem_object_unbind(struct drm_i915_gem_object *obj) |
1974 | i915_gem_object_unbind(struct drm_i915_gem_object *obj) |
1284 | { |
1975 | { |
- | 1976 | drm_i915_private_t *dev_priv = obj->base.dev->dev_private; |
|
1285 | int ret = 0; |
1977 | int ret = 0; |
Line 1286... | Line 1978... | ||
1286 | 1978 | ||
1287 | if (obj->gtt_space == NULL) |
1979 | if (obj->gtt_space == NULL) |
Line 1288... | Line 1980... | ||
1288 | return 0; |
1980 | return 0; |
1289 | - | ||
1290 | if (obj->pin_count != 0) { |
1981 | |
1291 | DRM_ERROR("Attempting to unbind pinned buffer\n"); |
1982 | if (obj->pin_count) |
- | 1983 | return -EBUSY; |
|
Line 1292... | Line 1984... | ||
1292 | return -EINVAL; |
1984 | |
1293 | } |
1985 | BUG_ON(obj->pages.page == NULL); |
1294 | 1986 | ||
1295 | ret = i915_gem_object_finish_gpu(obj); |
1987 | ret = i915_gem_object_finish_gpu(obj); |
1296 | if (ret == -ERESTARTSYS) |
1988 | if (ret) |
1297 | return ret; |
1989 | return ret; |
1298 | /* Continue on if we fail due to EIO, the GPU is hung so we |
1990 | /* Continue on if we fail due to EIO, the GPU is hung so we |
Line 1299... | Line 1991... | ||
1299 | * should be safe and we need to cleanup or else we might |
1991 | * should be safe and we need to cleanup or else we might |
Line 1300... | Line -... | ||
1300 | * cause memory corruption through use-after-free. |
- | |
1301 | */ |
- | |
1302 | - | ||
1303 | i915_gem_object_finish_gtt(obj); |
- | |
1304 | - | ||
1305 | /* Move the object to the CPU domain to ensure that |
- | |
1306 | * any possible CPU writes while it's not in the GTT |
- | |
1307 | * are flushed when we go to remap it. |
- | |
1308 | */ |
- | |
1309 | if (ret == 0) |
- | |
1310 | ret = i915_gem_object_set_to_cpu_domain(obj, 1); |
- | |
1311 | if (ret == -ERESTARTSYS) |
- | |
1312 | return ret; |
- | |
1313 | if (ret) { |
- | |
1314 | /* In the event of a disaster, abandon all caches and |
- | |
1315 | * hope for the best. |
- | |
1316 | */ |
1992 | * cause memory corruption through use-after-free. |
1317 | i915_gem_clflush_object(obj); |
1993 | */ |
1318 | obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
1994 | |
1319 | } |
1995 | i915_gem_object_finish_gtt(obj); |
Line 1320... | Line 1996... | ||
1320 | 1996 | ||
Line -... | Line 1997... | ||
- | 1997 | /* release the fence reg _after_ flushing */ |
|
1321 | /* release the fence reg _after_ flushing */ |
1998 | ret = i915_gem_object_put_fence(obj); |
- | 1999 | if (ret) |
|
- | 2000 | return ret; |
|
- | 2001 | ||
- | 2002 | trace_i915_gem_object_unbind(obj); |
|
1322 | ret = i915_gem_object_put_fence(obj); |
2003 | |
Line 1323... | Line 2004... | ||
1323 | if (ret == -ERESTARTSYS) |
2004 | if (obj->has_global_gtt_mapping) |
1324 | return ret; |
2005 | i915_gem_gtt_unbind_object(obj); |
1325 | 2006 | if (obj->has_aliasing_ppgtt_mapping) { |
|
1326 | trace_i915_gem_object_unbind(obj); |
2007 | i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); |
Line 1327... | Line 2008... | ||
1327 | 2008 | obj->has_aliasing_ppgtt_mapping = 0; |
|
1328 | i915_gem_gtt_unbind_object(obj); |
2009 | } |
1329 | i915_gem_object_put_pages_gtt(obj); |
2010 | i915_gem_gtt_finish_object(obj); |
Line 1330... | Line -... | ||
1330 | - | ||
1331 | list_del_init(&obj->gtt_list); |
- | |
1332 | list_del_init(&obj->mm_list); |
- | |
1333 | /* Avoid an unnecessary call to unbind on rebind. */ |
- | |
1334 | obj->map_and_fenceable = true; |
- | |
1335 | - | ||
1336 | drm_mm_put_block(obj->gtt_space); |
- | |
1337 | obj->gtt_space = NULL; |
- | |
1338 | obj->gtt_offset = 0; |
- | |
1339 | - | ||
1340 | if (i915_gem_object_is_purgeable(obj)) |
- | |
1341 | i915_gem_object_truncate(obj); |
- | |
1342 | - | ||
1343 | return ret; |
- | |
1344 | } |
- | |
1345 | - | ||
1346 | int |
- | |
1347 | i915_gem_flush_ring(struct intel_ring_buffer *ring, |
- | |
1348 | uint32_t invalidate_domains, |
- | |
1349 | uint32_t flush_domains) |
- | |
1350 | { |
- | |
1351 | int ret; |
- | |
1352 | - | ||
1353 | if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) |
- | |
1354 | return 0; |
- | |
1355 | 2011 | ||
1356 | trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains); |
2012 | list_del(&obj->mm_list); |
Line 1357... | Line 2013... | ||
1357 | 2013 | list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list); |
|
1358 | ret = ring->flush(ring, invalidate_domains, flush_domains); |
2014 | /* Avoid an unnecessary call to unbind on rebind. */ |
1359 | if (ret) |
- | |
1360 | return ret; |
- | |
1361 | 2015 | obj->map_and_fenceable = true; |
|
1362 | if (flush_domains & I915_GEM_GPU_DOMAINS) |
2016 | |
Line 1363... | Line -... | ||
1363 | i915_gem_process_flushing_list(ring, flush_domains); |
- | |
1364 | - | ||
1365 | return 0; |
- | |
1366 | } |
- | |
1367 | - | ||
1368 | static int i915_ring_idle(struct intel_ring_buffer *ring) |
- | |
1369 | { |
- | |
1370 | int ret; |
2017 | drm_mm_put_block(obj->gtt_space); |
1371 | 2018 | obj->gtt_space = NULL; |
|
Line 1372... | Line -... | ||
1372 | if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) |
- | |
1373 | return 0; |
2019 | obj->gtt_offset = 0; |
1374 | 2020 | ||
1375 | if (!list_empty(&ring->gpu_write_list)) { |
2021 | return 0; |
- | 2022 | } |
|
1376 | ret = i915_gem_flush_ring(ring, |
2023 | |
Line 1377... | Line 2024... | ||
1377 | I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); |
2024 | static int i915_ring_idle(struct intel_ring_buffer *ring) |
1378 | if (ret) |
2025 | { |
- | 2026 | if (list_empty(&ring->active_list)) |
|
- | 2027 | return 0; |
|
- | 2028 | ||
- | 2029 | return i915_wait_seqno(ring, i915_gem_next_request_seqno(ring)); |
|
1379 | return ret; |
2030 | } |
1380 | } |
2031 | |
1381 | 2032 | int i915_gpu_idle(struct drm_device *dev) |
|
1382 | return i915_wait_request(ring, i915_gem_next_request_seqno(ring)); |
2033 | { |
Line 1383... | Line 2034... | ||
1383 | } |
2034 | drm_i915_private_t *dev_priv = dev->dev_private; |
1384 | 2035 | struct intel_ring_buffer *ring; |
|
Line -... | Line 2036... | ||
- | 2036 | int ret, i; |
|
- | 2037 | ||
- | 2038 | /* Flush everything onto the inactive list. */ |
|
- | 2039 | for_each_ring(ring, dev_priv, i) { |
|
- | 2040 | ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID); |
|
Line -... | Line 2041... | ||
- | 2041 | if (ret) |
|
- | 2042 | return ret; |
|
Line -... | Line 2043... | ||
- | 2043 | ||
- | 2044 | ret = i915_ring_idle(ring); |
|
- | 2045 | if (ret) |
|
- | 2046 | return ret; |
|
- | 2047 | } |
|
- | 2048 | ||
- | 2049 | return 0; |
|
- | 2050 | } |
|
- | 2051 | ||
- | 2052 | static void sandybridge_write_fence_reg(struct drm_device *dev, int reg, |
|
- | 2053 | struct drm_i915_gem_object *obj) |
|
Line -... | Line 2054... | ||
- | 2054 | { |
|
- | 2055 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 2056 | uint64_t val; |
|
Line -... | Line 2057... | ||
- | 2057 | ||
- | 2058 | if (obj) { |
|
- | 2059 | u32 size = obj->gtt_space->size; |
|
- | 2060 | ||
- | 2061 | val = (uint64_t)((obj->gtt_offset + size - 4096) & |
|
Line -... | Line 2062... | ||
- | 2062 | 0xfffff000) << 32; |
|
- | 2063 | val |= obj->gtt_offset & 0xfffff000; |
|
Line -... | Line 2064... | ||
- | 2064 | val |= (uint64_t)((obj->stride / 128) - 1) << |
|
- | 2065 | SANDYBRIDGE_FENCE_PITCH_SHIFT; |
|
- | 2066 | ||
- | 2067 | if (obj->tiling_mode == I915_TILING_Y) |
|
- | 2068 | val |= 1 << I965_FENCE_TILING_Y_SHIFT; |
|
- | 2069 | val |= I965_FENCE_REG_VALID; |
|
- | 2070 | } else |
|
- | 2071 | val = 0; |
|
- | 2072 | ||
Line -... | Line 2073... | ||
- | 2073 | I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val); |
|
- | 2074 | POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8); |
|
- | 2075 | } |
|
Line -... | Line 2076... | ||
- | 2076 | ||
- | 2077 | static void i965_write_fence_reg(struct drm_device *dev, int reg, |
|
- | 2078 | struct drm_i915_gem_object *obj) |
|
- | 2079 | { |
|
- | 2080 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
Line -... | Line 2081... | ||
- | 2081 | uint64_t val; |
|
- | 2082 | ||
- | 2083 | if (obj) { |
|
- | 2084 | u32 size = obj->gtt_space->size; |
|
- | 2085 | ||
- | 2086 | val = (uint64_t)((obj->gtt_offset + size - 4096) & |
|
- | 2087 | 0xfffff000) << 32; |
|
- | 2088 | val |= obj->gtt_offset & 0xfffff000; |
|
- | 2089 | val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; |
|
- | 2090 | if (obj->tiling_mode == I915_TILING_Y) |
|
Line -... | Line 2091... | ||
- | 2091 | val |= 1 << I965_FENCE_TILING_Y_SHIFT; |
|
- | 2092 | val |= I965_FENCE_REG_VALID; |
|
- | 2093 | } else |
|
- | 2094 | val = 0; |
|
Line -... | Line 2095... | ||
- | 2095 | ||
- | 2096 | I915_WRITE64(FENCE_REG_965_0 + reg * 8, val); |
|
- | 2097 | POSTING_READ(FENCE_REG_965_0 + reg * 8); |
|
- | 2098 | } |
|
- | 2099 | ||
- | 2100 | static void i915_write_fence_reg(struct drm_device *dev, int reg, |
|
- | 2101 | struct drm_i915_gem_object *obj) |
|
- | 2102 | { |
|
- | 2103 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 2104 | u32 val; |
|
- | 2105 | ||
- | 2106 | if (obj) { |
|
Line -... | Line 2107... | ||
- | 2107 | u32 size = obj->gtt_space->size; |
|
- | 2108 | int pitch_val; |
|
- | 2109 | int tile_width; |
|
- | 2110 | ||
Line -... | Line 2111... | ||
- | 2111 | WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) || |
|
- | 2112 | (size & -size) != size || |
|
- | 2113 | (obj->gtt_offset & (size - 1)), |
|
Line -... | Line 2114... | ||
- | 2114 | "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", |
|
- | 2115 | obj->gtt_offset, obj->map_and_fenceable, size); |
|
- | 2116 | ||
- | 2117 | if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) |
|
- | 2118 | tile_width = 128; |
|
Line -... | Line 2119... | ||
- | 2119 | else |
|
- | 2120 | tile_width = 512; |
|
- | 2121 | ||
- | 2122 | /* Note: pitch better be a power of two tile widths */ |
|
- | 2123 | pitch_val = obj->stride / tile_width; |
|
- | 2124 | pitch_val = ffs(pitch_val) - 1; |
|
- | 2125 | ||
- | 2126 | val = obj->gtt_offset; |
|
- | 2127 | if (obj->tiling_mode == I915_TILING_Y) |
|
- | 2128 | val |= 1 << I830_FENCE_TILING_Y_SHIFT; |
|
- | 2129 | val |= I915_FENCE_SIZE_BITS(size); |
|
- | 2130 | val |= pitch_val << I830_FENCE_PITCH_SHIFT; |
|
- | 2131 | val |= I830_FENCE_REG_VALID; |
|
- | 2132 | } else |
|
- | 2133 | val = 0; |
|
- | 2134 | ||
- | 2135 | if (reg < 8) |
|
- | 2136 | reg = FENCE_REG_830_0 + reg * 4; |
|
- | 2137 | else |
|
- | 2138 | reg = FENCE_REG_945_8 + (reg - 8) * 4; |
|
- | 2139 | ||
Line -... | Line 2140... | ||
- | 2140 | I915_WRITE(reg, val); |
|
- | 2141 | POSTING_READ(reg); |
|
- | 2142 | } |
|
Line -... | Line 2143... | ||
- | 2143 | ||
- | 2144 | static void i830_write_fence_reg(struct drm_device *dev, int reg, |
|
- | 2145 | struct drm_i915_gem_object *obj) |
|
- | 2146 | { |
|
- | 2147 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 2148 | uint32_t val; |
|
- | 2149 | ||
- | 2150 | if (obj) { |
|
- | 2151 | u32 size = obj->gtt_space->size; |
|
- | 2152 | uint32_t pitch_val; |
|
- | 2153 | ||
- | 2154 | WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) || |
|
- | 2155 | (size & -size) != size || |
|
Line 1385... | Line -... | ||
1385 | int |
- | |
1386 | i915_gpu_idle(struct drm_device *dev) |
- | |
1387 | { |
- | |
1388 | drm_i915_private_t *dev_priv = dev->dev_private; |
2156 | (obj->gtt_offset & (size - 1)), |
- | 2157 | "object 0x%08x not 512K or pot-size 0x%08x aligned\n", |
|
1389 | int ret, i; |
2158 | obj->gtt_offset, size); |
1390 | 2159 | ||
1391 | /* Flush everything onto the inactive list. */ |
2160 | pitch_val = obj->stride / 128; |
Line 1392... | Line -... | ||
1392 | for (i = 0; i < I915_NUM_RINGS; i++) { |
- | |
1393 | ret = i915_ring_idle(&dev_priv->ring[i]); |
2161 | pitch_val = ffs(pitch_val) - 1; |
1394 | if (ret) |
2162 | |
- | 2163 | val = obj->gtt_offset; |
|
1395 | return ret; |
2164 | if (obj->tiling_mode == I915_TILING_Y) |
- | 2165 | val |= 1 << I830_FENCE_TILING_Y_SHIFT; |
|
1396 | } |
2166 | val |= I830_FENCE_SIZE_BITS(size); |
Line 1397... | Line -... | ||
1397 | - | ||
1398 | return 0; |
- | |
1399 | } |
2167 | val |= pitch_val << I830_FENCE_PITCH_SHIFT; |
1400 | - | ||
1401 | - | ||
1402 | - | ||
1403 | - | ||
Line -... | Line 2168... | ||
- | 2168 | val |= I830_FENCE_REG_VALID; |
|
1404 | 2169 | } else |
|
- | 2170 | val = 0; |
|
- | 2171 | ||
- | 2172 | I915_WRITE(FENCE_REG_830_0 + reg * 4, val); |
|
- | 2173 | POSTING_READ(FENCE_REG_830_0 + reg * 4); |
|
- | 2174 | } |
|
- | 2175 | ||
- | 2176 | static void i915_gem_write_fence(struct drm_device *dev, int reg, |
|
1405 | 2177 | struct drm_i915_gem_object *obj) |
|
Line -... | Line 2178... | ||
- | 2178 | { |
|
1406 | 2179 | switch (INTEL_INFO(dev)->gen) { |
|
1407 | - | ||
- | 2180 | case 7: |
|
1408 | 2181 | case 6: sandybridge_write_fence_reg(dev, reg, obj); break; |
|
1409 | 2182 | case 5: |
|
1410 | - | ||
1411 | 2183 | case 4: i965_write_fence_reg(dev, reg, obj); break; |
|
1412 | 2184 | case 3: i915_write_fence_reg(dev, reg, obj); break; |
|
1413 | - | ||
Line 1414... | Line 2185... | ||
1414 | 2185 | case 2: i830_write_fence_reg(dev, reg, obj); break; |
|
1415 | - | ||
1416 | 2186 | default: break; |
|
Line 1417... | Line 2187... | ||
1417 | 2187 | } |
|
1418 | 2188 | } |
|
1419 | 2189 | ||
1420 | 2190 | static inline int fence_number(struct drm_i915_private *dev_priv, |
|
1421 | 2191 | struct drm_i915_fence_reg *fence) |
|
Line -... | Line 2192... | ||
- | 2192 | { |
|
1422 | static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno) |
2193 | return fence - dev_priv->fence_regs; |
1423 | { |
2194 | } |
Line 1424... | Line 2195... | ||
1424 | return i915_seqno_passed(ring->get_seqno(ring), seqno); |
2195 | |
1425 | } |
2196 | static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj, |
1426 | 2197 | struct drm_i915_fence_reg *fence, |
|
- | 2198 | bool enable) |
|
1427 | static int |
2199 | { |
Line 1428... | Line -... | ||
1428 | i915_gem_object_flush_fence(struct drm_i915_gem_object *obj, |
- | |
1429 | struct intel_ring_buffer *pipelined) |
- | |
1430 | { |
- | |
1431 | int ret; |
2200 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
1432 | 2201 | int reg = fence_number(dev_priv, fence); |
|
1433 | if (obj->fenced_gpu_access) { |
2202 | |
Line 1434... | Line 2203... | ||
1434 | if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { |
2203 | i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL); |
1435 | ret = i915_gem_flush_ring(obj->last_fenced_ring, |
- | |
1436 | 0, obj->base.write_domain); |
- | |
1437 | if (ret) |
2204 | |
Line -... | Line 2205... | ||
- | 2205 | if (enable) { |
|
1438 | return ret; |
2206 | obj->fence_reg = reg; |
1439 | } |
2207 | fence->obj = obj; |
- | 2208 | list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list); |
|
Line 1440... | Line 2209... | ||
1440 | 2209 | } else { |
|
1441 | obj->fenced_gpu_access = false; |
2210 | obj->fence_reg = I915_FENCE_REG_NONE; |
Line -... | Line 2211... | ||
- | 2211 | fence->obj = NULL; |
|
- | 2212 | list_del_init(&fence->lru_list); |
|
- | 2213 | } |
|
- | 2214 | } |
|
- | 2215 | ||
- | 2216 | static int |
|
Line -... | Line 2217... | ||
- | 2217 | i915_gem_object_flush_fence(struct drm_i915_gem_object *obj) |
|
- | 2218 | { |
|
- | 2219 | if (obj->last_fenced_seqno) { |
|
- | 2220 | int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno); |
|
- | 2221 | if (ret) |
|
- | 2222 | return ret; |
|
Line -... | Line 2223... | ||
- | 2223 | ||
- | 2224 | obj->last_fenced_seqno = 0; |
|
- | 2225 | } |
|
Line -... | Line 2226... | ||
- | 2226 | ||
- | 2227 | /* Ensure that all CPU reads are completed before installing a fence |
|
Line -... | Line 2228... | ||
- | 2228 | * and all writes before removing the fence. |
|
- | 2229 | */ |
|
- | 2230 | if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) |
|
- | 2231 | mb(); |
|
Line -... | Line 2232... | ||
- | 2232 | ||
- | 2233 | obj->fenced_gpu_access = false; |
|
Line -... | Line 2234... | ||
- | 2234 | return 0; |
|
- | 2235 | } |
|
Line -... | Line 2236... | ||
- | 2236 | ||
- | 2237 | int |
|
- | 2238 | i915_gem_object_put_fence(struct drm_i915_gem_object *obj) |
|
- | 2239 | { |
|
- | 2240 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
|
- | 2241 | int ret; |
|
- | 2242 | ||
- | 2243 | ret = i915_gem_object_flush_fence(obj); |
|
- | 2244 | if (ret) |
|
- | 2245 | return ret; |
|
- | 2246 | ||
- | 2247 | if (obj->fence_reg == I915_FENCE_REG_NONE) |
|
- | 2248 | return 0; |
|
- | 2249 | ||
- | 2250 | i915_gem_object_update_fence(obj, |
|
- | 2251 | &dev_priv->fence_regs[obj->fence_reg], |
|
- | 2252 | false); |
|
- | 2253 | i915_gem_object_fence_lost(obj); |
|
- | 2254 | ||
- | 2255 | return 0; |
|
- | 2256 | } |
|
- | 2257 | ||
Line -... | Line 2258... | ||
- | 2258 | static struct drm_i915_fence_reg * |
|
- | 2259 | i915_find_fence_reg(struct drm_device *dev) |
|
- | 2260 | { |
|
- | 2261 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
- | 2262 | struct drm_i915_fence_reg *reg, *avail; |
|
- | 2263 | int i; |
|
- | 2264 | ||
- | 2265 | /* First try to find a free reg */ |
|
Line -... | Line 2266... | ||
- | 2266 | avail = NULL; |
|
- | 2267 | for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { |
|
- | 2268 | reg = &dev_priv->fence_regs[i]; |
|
- | 2269 | if (!reg->obj) |
|
- | 2270 | return reg; |
|
- | 2271 | ||
- | 2272 | if (!reg->pin_count) |
|
- | 2273 | avail = reg; |
|
- | 2274 | } |
|
- | 2275 | ||
- | 2276 | if (avail == NULL) |
|
- | 2277 | return NULL; |
|
Line -... | Line 2278... | ||
- | 2278 | ||
- | 2279 | /* None available, try to steal one or wait for a user to finish */ |
|
Line -... | Line 2280... | ||
- | 2280 | list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { |
|
- | 2281 | if (reg->pin_count) |
|
- | 2282 | continue; |
|
Line -... | Line 2283... | ||
- | 2283 | ||
- | 2284 | return reg; |
|
- | 2285 | } |
|
- | 2286 | ||
Line -... | Line 2287... | ||
- | 2287 | return NULL; |
|
- | 2288 | } |
|
Line -... | Line 2289... | ||
- | 2289 | ||
- | 2290 | /** |
|
Line -... | Line 2291... | ||
- | 2291 | * i915_gem_object_get_fence - set up fencing for an object |
|
- | 2292 | * @obj: object to map through a fence reg |
|
- | 2293 | * |
|
- | 2294 | * When mapping objects through the GTT, userspace wants to be able to write |
|
- | 2295 | * to them without having to worry about swizzling if the object is tiled. |
|
Line -... | Line 2296... | ||
- | 2296 | * This function walks the fence regs looking for a free one for @obj, |
|
- | 2297 | * stealing one if it can't find any. |
|
- | 2298 | * |
|
- | 2299 | * It then sets up the reg based on the object's properties: address, pitch |
|
- | 2300 | * and tiling format. |
|
- | 2301 | * |
|
Line -... | Line 2302... | ||
- | 2302 | * For an untiled surface, this removes any existing fence. |
|
- | 2303 | */ |
|
Line -... | Line 2304... | ||
- | 2304 | int |
|
- | 2305 | i915_gem_object_get_fence(struct drm_i915_gem_object *obj) |
|
Line -... | Line 2306... | ||
- | 2306 | { |
|
- | 2307 | struct drm_device *dev = obj->base.dev; |
|
- | 2308 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
Line -... | Line 2309... | ||
- | 2309 | bool enable = obj->tiling_mode != I915_TILING_NONE; |
|
- | 2310 | struct drm_i915_fence_reg *reg; |
|
- | 2311 | int ret; |
|
Line -... | Line 2312... | ||
- | 2312 | ||
- | 2313 | /* Have we updated the tiling parameters upon the object and so |
|
Line 1442... | Line -... | ||
1442 | } |
- | |
1443 | - | ||
1444 | if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) { |
- | |
1445 | if (!ring_passed_seqno(obj->last_fenced_ring, |
- | |
1446 | obj->last_fenced_seqno)) { |
- | |
1447 | ret = i915_wait_request(obj->last_fenced_ring, |
- | |
1448 | obj->last_fenced_seqno); |
- | |
1449 | if (ret) |
- | |
1450 | return ret; |
- | |
1451 | } |
- | |
1452 | - | ||
1453 | obj->last_fenced_seqno = 0; |
- | |
1454 | obj->last_fenced_ring = NULL; |
- | |
1455 | } |
- | |
1456 | - | ||
1457 | /* Ensure that all CPU reads are completed before installing a fence |
- | |
1458 | * and all writes before removing the fence. |
2314 | * will need to serialise the write to the associated fence register? |
1459 | */ |
- | |
1460 | if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) |
2315 | */ |
1461 | mb(); |
- | |
1462 | - | ||
1463 | return 0; |
- | |
1464 | } |
- | |
1465 | - | ||
1466 | int |
- | |
1467 | i915_gem_object_put_fence(struct drm_i915_gem_object *obj) |
- | |
1468 | { |
2316 | if (obj->fence_dirty) { |
1469 | int ret; |
- | |
1470 | - | ||
1471 | // if (obj->tiling_mode) |
2317 | ret = i915_gem_object_flush_fence(obj); |
1472 | // i915_gem_release_mmap(obj); |
- | |
1473 | - | ||
1474 | ret = i915_gem_object_flush_fence(obj, NULL); |
2318 | if (ret) |
1475 | if (ret) |
- | |
1476 | return ret; |
2319 | return ret; |
1477 | - | ||
1478 | if (obj->fence_reg != I915_FENCE_REG_NONE) { |
- | |
Line -... | Line 2320... | ||
- | 2320 | } |
|
- | 2321 | ||
- | 2322 | /* Just update our place in the LRU if our fence is getting reused. */ |
|
- | 2323 | if (obj->fence_reg != I915_FENCE_REG_NONE) { |
|
- | 2324 | reg = &dev_priv->fence_regs[obj->fence_reg]; |
|
- | 2325 | if (!obj->fence_dirty) { |
|
- | 2326 | list_move_tail(®->lru_list, |
|
- | 2327 | &dev_priv->mm.fence_list); |
|
- | 2328 | return 0; |
|
- | 2329 | } |
|
- | 2330 | } else if (enable) { |
|
- | 2331 | reg = i915_find_fence_reg(dev); |
|
1479 | struct drm_i915_private *dev_priv = obj->base.dev->dev_private; |
2332 | if (reg == NULL) |
- | 2333 | return -EDEADLK; |
|
- | 2334 | ||
- | 2335 | if (reg->obj) { |
|
- | 2336 | struct drm_i915_gem_object *old = reg->obj; |
|
- | 2337 | ||
- | 2338 | ret = i915_gem_object_flush_fence(old); |
|
- | 2339 | if (ret) |
|
- | 2340 | return ret; |
|
- | 2341 | ||
- | 2342 | i915_gem_object_fence_lost(old); |
|
1480 | i915_gem_clear_fence_reg(obj->base.dev, |
2343 | } |
- | 2344 | } else |
|
- | 2345 | return 0; |
|
- | 2346 | ||
1481 | &dev_priv->fence_regs[obj->fence_reg]); |
2347 | i915_gem_object_update_fence(obj, reg, enable); |
Line 1482... | Line -... | ||
1482 | - | ||
1483 | obj->fence_reg = I915_FENCE_REG_NONE; |
2348 | obj->fence_dirty = false; |
1484 | } |
2349 | |
1485 | 2350 | return 0; |
|
Line 1486... | Line 2351... | ||
1486 | return 0; |
2351 | } |
1487 | } |
2352 | |
1488 | 2353 | static bool i915_gem_valid_gtt_space(struct drm_device *dev, |
|
1489 | 2354 | struct drm_mm_node *gtt_space, |
|
1490 | 2355 | unsigned long cache_level) |
|
1491 | 2356 | { |
|
1492 | 2357 | struct drm_mm_node *other; |
|
- | 2358 | ||
1493 | 2359 | /* On non-LLC machines we have to be careful when putting differing |
|
1494 | 2360 | * types of snoopable memory together to avoid the prefetcher |
|
1495 | 2361 | * crossing memory domains and dieing. |
|
1496 | 2362 | */ |
|
1497 | - | ||
1498 | 2363 | if (HAS_LLC(dev)) |
|
1499 | 2364 | return true; |
|
1500 | 2365 | ||
Line 1501... | Line 2366... | ||
1501 | 2366 | if (gtt_space == NULL) |
|
Line 1604... | Line 2469... | ||
1604 | (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { |
2469 | (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { |
1605 | DRM_ERROR("Attempting to bind an object larger than the aperture\n"); |
2470 | DRM_ERROR("Attempting to bind an object larger than the aperture\n"); |
1606 | return -E2BIG; |
2471 | return -E2BIG; |
1607 | } |
2472 | } |
Line -... | Line 2473... | ||
- | 2473 | ||
- | 2474 | ret = i915_gem_object_get_pages(obj); |
|
- | 2475 | if (ret) |
|
- | 2476 | return ret; |
|
1608 | 2477 | ||
1609 | search_free: |
2478 | search_free: |
1610 | if (map_and_fenceable) |
2479 | if (map_and_fenceable) |
1611 | free_space = |
2480 | free_space = |
1612 | drm_mm_search_free_in_range(&dev_priv->mm.gtt_space, |
2481 | drm_mm_search_free_in_range_color(&dev_priv->mm.gtt_space, |
1613 | size, alignment, 0, |
2482 | size, alignment, obj->cache_level, |
1614 | dev_priv->mm.gtt_mappable_end, |
2483 | 0, dev_priv->mm.gtt_mappable_end, |
1615 | 0); |
2484 | false); |
1616 | else |
2485 | else |
1617 | free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, |
2486 | free_space = drm_mm_search_free_color(&dev_priv->mm.gtt_space, |
- | 2487 | size, alignment, obj->cache_level, |
|
Line 1618... | Line 2488... | ||
1618 | size, alignment, 0); |
2488 | false); |
1619 | 2489 | ||
1620 | if (free_space != NULL) { |
2490 | if (free_space != NULL) { |
1621 | if (map_and_fenceable) |
2491 | if (map_and_fenceable) |
1622 | obj->gtt_space = |
2492 | obj->gtt_space = |
1623 | drm_mm_get_block_range_generic(free_space, |
2493 | drm_mm_get_block_range_generic(free_space, |
1624 | size, alignment, 0, |
2494 | size, alignment, obj->cache_level, |
1625 | dev_priv->mm.gtt_mappable_end, |
2495 | 0, dev_priv->mm.gtt_mappable_end, |
1626 | 0); |
2496 | false); |
1627 | else |
2497 | else |
- | 2498 | obj->gtt_space = |
|
- | 2499 | drm_mm_get_block_generic(free_space, |
|
1628 | obj->gtt_space = |
2500 | size, alignment, obj->cache_level, |
1629 | drm_mm_get_block(free_space, size, alignment); |
2501 | false); |
1630 | } |
- | |
1631 | if (obj->gtt_space == NULL) { |
- | |
1632 | /* If the gtt is empty and we're still having trouble |
- | |
1633 | * fitting our object in, we're out of memory. |
2502 | } |
1634 | */ |
2503 | if (obj->gtt_space == NULL) { |
1635 | ret = 1; //i915_gem_evict_something(dev, size, alignment, |
2504 | ret = 1; //i915_gem_evict_something(dev, size, alignment, |
1636 | // map_and_fenceable); |
2505 | // map_and_fenceable); |
Line 1637... | Line 2506... | ||
1637 | if (ret) |
2506 | if (ret) |
1638 | return ret; |
2507 | return ret; |
1639 | - | ||
1640 | goto search_free; |
2508 | |
- | 2509 | goto search_free; |
|
1641 | } |
2510 | } |
1642 | 2511 | if (WARN_ON(!i915_gem_valid_gtt_space(dev, |
|
1643 | ret = i915_gem_object_get_pages_gtt(obj, gfpmask); |
2512 | obj->gtt_space, |
1644 | if (ret) { |
- | |
1645 | drm_mm_put_block(obj->gtt_space); |
- | |
1646 | obj->gtt_space = NULL; |
- | |
1647 | #if 0 |
- | |
1648 | if (ret == -ENOMEM) { |
- | |
1649 | /* first try to reclaim some memory by clearing the GTT */ |
- | |
1650 | ret = i915_gem_evict_everything(dev, false); |
- | |
1651 | if (ret) { |
- | |
1652 | /* now try to shrink everyone else */ |
- | |
1653 | if (gfpmask) { |
- | |
1654 | gfpmask = 0; |
- | |
1655 | goto search_free; |
2513 | obj->cache_level))) { |
1656 | } |
2514 | drm_mm_put_block(obj->gtt_space); |
Line 1657... | Line -... | ||
1657 | - | ||
1658 | return -ENOMEM; |
- | |
1659 | } |
- | |
1660 | - | ||
1661 | goto search_free; |
- | |
Line 1662... | Line 2515... | ||
1662 | } |
2515 | obj->gtt_space = NULL; |
1663 | #endif |
2516 | return -EINVAL; |
1664 | return ret; |
- | |
1665 | } |
2517 | } |
1666 | 2518 | ||
1667 | ret = i915_gem_gtt_bind_object(obj); |
- | |
1668 | if (ret) { |
- | |
1669 | i915_gem_object_put_pages_gtt(obj); |
2519 | |
1670 | drm_mm_put_block(obj->gtt_space); |
- | |
1671 | obj->gtt_space = NULL; |
- | |
1672 | 2520 | ret = i915_gem_gtt_prepare_object(obj); |
|
Line 1673... | Line 2521... | ||
1673 | // if (i915_gem_evict_everything(dev, false)) |
2521 | if (ret) { |
1674 | return ret; |
2522 | drm_mm_put_block(obj->gtt_space); |
Line 1675... | Line 2523... | ||
1675 | 2523 | obj->gtt_space = NULL; |
|
1676 | // goto search_free; |
- | |
1677 | } |
- | |
1678 | - | ||
1679 | list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); |
- | |
1680 | list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); |
2524 | return ret; |
Line 1681... | Line 2525... | ||
1681 | 2525 | } |
|
Line 1682... | Line 2526... | ||
1682 | /* Assert that the object is not currently in any GPU domain. As it |
2526 | |
1683 | * wasn't in the GTT, there shouldn't be any way it could have been in |
2527 | if (!dev_priv->mm.aliasing_ppgtt) |
Line 1696... | Line 2540... | ||
1696 | obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; |
2540 | obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; |
Line 1697... | Line 2541... | ||
1697 | 2541 | ||
Line 1698... | Line 2542... | ||
1698 | obj->map_and_fenceable = mappable && fenceable; |
2542 | obj->map_and_fenceable = mappable && fenceable; |
- | 2543 | ||
1699 | 2544 | trace_i915_gem_object_bind(obj, map_and_fenceable); |
|
1700 | trace_i915_gem_object_bind(obj, map_and_fenceable); |
2545 | i915_gem_verify_gtt(dev); |
Line 1701... | Line 2546... | ||
1701 | return 0; |
2546 | return 0; |
1702 | } |
2547 | } |
1703 | 2548 | ||
1704 | void |
2549 | void |
1705 | i915_gem_clflush_object(struct drm_i915_gem_object *obj) |
2550 | i915_gem_clflush_object(struct drm_i915_gem_object *obj) |
1706 | { |
2551 | { |
1707 | /* If we don't have a page list set up, then we're not pinned |
2552 | /* If we don't have a page list set up, then we're not pinned |
1708 | * to GPU, and we can ignore the cache flush because it'll happen |
2553 | * to GPU, and we can ignore the cache flush because it'll happen |
1709 | * again at bind time. |
2554 | * again at bind time. |
Line 1710... | Line 2555... | ||
1710 | */ |
2555 | */ |
1711 | if (obj->pages == NULL) |
2556 | if (obj->pages.page == NULL) |
1712 | return; |
2557 | return; |
Line 1738... | Line 2583... | ||
1738 | uint8_t *page_virtual; |
2583 | uint8_t *page_virtual; |
1739 | unsigned int i; |
2584 | unsigned int i; |
1740 | page_virtual = AllocKernelSpace(obj->base.size); |
2585 | page_virtual = AllocKernelSpace(obj->base.size); |
1741 | if(page_virtual != NULL) |
2586 | if(page_virtual != NULL) |
1742 | { |
2587 | { |
1743 | u32_t *src, *dst; |
2588 | dma_addr_t *src, *dst; |
1744 | u32 count; |
2589 | u32 count; |
Line 1745... | Line 2590... | ||
1745 | 2590 | ||
Line 1746... | Line 2591... | ||
1746 | #define page_tabs 0xFDC00000 /* really dirty hack */ |
2591 | #define page_tabs 0xFDC00000 /* really dirty hack */ |
1747 | 2592 | ||
1748 | src = (u32_t*)obj->pages; |
2593 | src = obj->pages.page; |
Line 1749... | Line 2594... | ||
1749 | dst = &((u32_t*)page_tabs)[(u32_t)page_virtual >> 12]; |
2594 | dst = &((dma_addr_t*)page_tabs)[(u32_t)page_virtual >> 12]; |
1750 | count = obj->base.size/4096; |
2595 | count = obj->base.size/4096; |
1751 | 2596 | ||
Line 1768... | Line 2613... | ||
1768 | "mfence"); |
2613 | "mfence"); |
1769 | } |
2614 | } |
1770 | } |
2615 | } |
1771 | } |
2616 | } |
Line 1772... | Line -... | ||
1772 | - | ||
1773 | /** Flushes any GPU write domain for the object if it's dirty. */ |
- | |
1774 | static int |
- | |
1775 | i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) |
- | |
1776 | { |
- | |
1777 | if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) |
- | |
1778 | return 0; |
- | |
1779 | - | ||
1780 | /* Queue the GPU write cache flushing we need. */ |
- | |
1781 | return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); |
- | |
1782 | } |
- | |
1783 | 2617 | ||
1784 | /** Flushes the GTT write domain for the object if it's dirty. */ |
2618 | /** Flushes the GTT write domain for the object if it's dirty. */ |
1785 | static void |
2619 | static void |
1786 | i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) |
2620 | i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) |
1787 | { |
2621 | { |
Line 1834... | Line 2668... | ||
1834 | * flushes to occur. |
2668 | * flushes to occur. |
1835 | */ |
2669 | */ |
1836 | int |
2670 | int |
1837 | i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) |
2671 | i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) |
1838 | { |
2672 | { |
- | 2673 | drm_i915_private_t *dev_priv = obj->base.dev->dev_private; |
|
1839 | uint32_t old_write_domain, old_read_domains; |
2674 | uint32_t old_write_domain, old_read_domains; |
1840 | int ret; |
2675 | int ret; |
Line 1841... | Line 2676... | ||
1841 | 2676 | ||
1842 | /* Not valid to be called on unbound objects. */ |
2677 | /* Not valid to be called on unbound objects. */ |
1843 | if (obj->gtt_space == NULL) |
2678 | if (obj->gtt_space == NULL) |
Line 1844... | Line 2679... | ||
1844 | return -EINVAL; |
2679 | return -EINVAL; |
1845 | 2680 | ||
Line 1846... | Line 2681... | ||
1846 | if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) |
2681 | if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) |
1847 | return 0; |
2682 | return 0; |
1848 | 2683 | ||
Line 1849... | Line -... | ||
1849 | ret = i915_gem_object_flush_gpu_write_domain(obj); |
- | |
1850 | if (ret) |
- | |
1851 | return ret; |
- | |
1852 | - | ||
1853 | if (obj->pending_gpu_write || write) { |
- | |
1854 | ret = i915_gem_object_wait_rendering(obj); |
- | |
1855 | if (ret) |
2684 | ret = i915_gem_object_wait_rendering(obj, !write); |
Line 1856... | Line 2685... | ||
1856 | return ret; |
2685 | if (ret) |
1857 | } |
2686 | return ret; |
Line 1874... | Line 2703... | ||
1874 | 2703 | ||
1875 | trace_i915_gem_object_change_domain(obj, |
2704 | trace_i915_gem_object_change_domain(obj, |
1876 | old_read_domains, |
2705 | old_read_domains, |
Line -... | Line 2706... | ||
- | 2706 | old_write_domain); |
|
- | 2707 | ||
- | 2708 | /* And bump the LRU for this access */ |
|
- | 2709 | if (i915_gem_object_is_inactive(obj)) |
|
1877 | old_write_domain); |
2710 | list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); |
1878 | 2711 | ||
Line 1879... | Line 2712... | ||
1879 | return 0; |
2712 | return 0; |
1880 | } |
2713 | } |
1881 | 2714 | ||
- | 2715 | int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, |
|
- | 2716 | enum i915_cache_level cache_level) |
|
1882 | int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, |
2717 | { |
Line 1883... | Line 2718... | ||
1883 | enum i915_cache_level cache_level) |
2718 | struct drm_device *dev = obj->base.dev; |
1884 | { |
2719 | drm_i915_private_t *dev_priv = dev->dev_private; |
Line 1885... | Line 2720... | ||
1885 | int ret; |
2720 | int ret; |
1886 | 2721 | ||
1887 | if (obj->cache_level == cache_level) |
2722 | if (obj->cache_level == cache_level) |
1888 | return 0; |
2723 | return 0; |
Line -... | Line 2724... | ||
- | 2724 | ||
- | 2725 | if (obj->pin_count) { |
|
- | 2726 | DRM_DEBUG("can not change the cache level of pinned objects\n"); |
|
- | 2727 | return -EBUSY; |
|
- | 2728 | } |
|
- | 2729 | ||
1889 | 2730 | if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) { |
|
1890 | if (obj->pin_count) { |
2731 | ret = i915_gem_object_unbind(obj); |
1891 | DRM_DEBUG("can not change the cache level of pinned objects\n"); |
2732 | if (ret) |
1892 | return -EBUSY; |
2733 | return ret; |
Line 1901... | Line 2742... | ||
1901 | 2742 | ||
1902 | /* Before SandyBridge, you could not use tiling or fence |
2743 | /* Before SandyBridge, you could not use tiling or fence |
1903 | * registers with snooped memory, so relinquish any fences |
2744 | * registers with snooped memory, so relinquish any fences |
1904 | * currently pointing to our region in the aperture. |
2745 | * currently pointing to our region in the aperture. |
1905 | */ |
2746 | */ |
1906 | if (INTEL_INFO(obj->base.dev)->gen < 6) { |
2747 | if (INTEL_INFO(dev)->gen < 6) { |
1907 | ret = i915_gem_object_put_fence(obj); |
2748 | ret = i915_gem_object_put_fence(obj); |
1908 | if (ret) |
2749 | if (ret) |
1909 | return ret; |
2750 | return ret; |
Line -... | Line 2751... | ||
- | 2751 | } |
|
1910 | } |
2752 | |
- | 2753 | if (obj->has_global_gtt_mapping) |
|
- | 2754 | i915_gem_gtt_bind_object(obj, cache_level); |
|
- | 2755 | if (obj->has_aliasing_ppgtt_mapping) |
|
- | 2756 | i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, |
|
- | 2757 | obj, cache_level); |
|
1911 | 2758 | ||
Line 1912... | Line 2759... | ||
1912 | i915_gem_gtt_rebind_object(obj, cache_level); |
2759 | obj->gtt_space->color = cache_level; |
1913 | } |
2760 | } |
Line 1934... | Line 2781... | ||
1934 | old_read_domains, |
2781 | old_read_domains, |
1935 | old_write_domain); |
2782 | old_write_domain); |
1936 | } |
2783 | } |
Line 1937... | Line 2784... | ||
1937 | 2784 | ||
- | 2785 | obj->cache_level = cache_level; |
|
1938 | obj->cache_level = cache_level; |
2786 | i915_gem_verify_gtt(dev); |
1939 | return 0; |
2787 | return 0; |
Line 1940... | Line 2788... | ||
1940 | } |
2788 | } |
1941 | 2789 | ||
1942 | /* |
2790 | /* |
1943 | * Prepare buffer for display plane (scanout, cursors, etc). |
2791 | * Prepare buffer for display plane (scanout, cursors, etc). |
1944 | * Can be called from an uninterruptible phase (modesetting) and allows |
- | |
1945 | * any flushes to be pipelined (for pageflips). |
- | |
1946 | * |
- | |
1947 | * For the display plane, we want to be in the GTT but out of any write |
- | |
1948 | * domains. So in many ways this looks like set_to_gtt_domain() apart from the |
- | |
1949 | * ability to pipeline the waits, pinning and any additional subtleties |
2792 | * Can be called from an uninterruptible phase (modesetting) and allows |
1950 | * that may differentiate the display plane from ordinary buffers. |
2793 | * any flushes to be pipelined (for pageflips). |
1951 | */ |
2794 | */ |
1952 | int |
2795 | int |
1953 | i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, |
2796 | i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, |
1954 | u32 alignment, |
2797 | u32 alignment, |
1955 | struct intel_ring_buffer *pipelined) |
2798 | struct intel_ring_buffer *pipelined) |
1956 | { |
2799 | { |
Line 1957... | Line -... | ||
1957 | u32 old_read_domains, old_write_domain; |
- | |
1958 | int ret; |
- | |
1959 | - | ||
1960 | ret = i915_gem_object_flush_gpu_write_domain(obj); |
- | |
1961 | if (ret) |
2800 | u32 old_read_domains, old_write_domain; |
1962 | return ret; |
2801 | int ret; |
1963 | 2802 | ||
1964 | if (pipelined != obj->ring) { |
2803 | if (pipelined != obj->ring) { |
1965 | ret = i915_gem_object_wait_rendering(obj); |
2804 | ret = i915_gem_object_sync(obj, pipelined); |
Line 1966... | Line 2805... | ||
1966 | if (ret == -ERESTARTSYS) |
2805 | if (ret) |
1967 | return ret; |
2806 | return ret; |
Line 1982... | Line 2821... | ||
1982 | 2821 | ||
1983 | /* As the user may map the buffer once pinned in the display plane |
2822 | /* As the user may map the buffer once pinned in the display plane |
1984 | * (e.g. libkms for the bootup splash), we have to ensure that we |
2823 | * (e.g. libkms for the bootup splash), we have to ensure that we |
1985 | * always use map_and_fenceable for all scanout buffers. |
2824 | * always use map_and_fenceable for all scanout buffers. |
1986 | */ |
2825 | */ |
1987 | ret = i915_gem_object_pin(obj, alignment, true); |
2826 | ret = i915_gem_object_pin(obj, alignment, true, false); |
1988 | if (ret) |
2827 | if (ret) |
Line 1989... | Line 2828... | ||
1989 | return ret; |
2828 | return ret; |
Line 1994... | Line 2833... | ||
1994 | old_read_domains = obj->base.read_domains; |
2833 | old_read_domains = obj->base.read_domains; |
Line 1995... | Line 2834... | ||
1995 | 2834 | ||
1996 | /* It should now be out of any other write domains, and we can update |
2835 | /* It should now be out of any other write domains, and we can update |
1997 | * the domain values for our changes. |
2836 | * the domain values for our changes. |
1998 | */ |
2837 | */ |
1999 | BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); |
2838 | obj->base.write_domain = 0; |
Line 2000... | Line 2839... | ||
2000 | obj->base.read_domains |= I915_GEM_DOMAIN_GTT; |
2839 | obj->base.read_domains |= I915_GEM_DOMAIN_GTT; |
2001 | 2840 | ||
2002 | trace_i915_gem_object_change_domain(obj, |
2841 | trace_i915_gem_object_change_domain(obj, |
Line 2012... | Line 2851... | ||
2012 | int ret; |
2851 | int ret; |
Line 2013... | Line 2852... | ||
2013 | 2852 | ||
2014 | if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) |
2853 | if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) |
Line 2015... | Line -... | ||
2015 | return 0; |
- | |
2016 | 2854 | return 0; |
|
2017 | if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { |
2855 | |
2018 | ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); |
2856 | ret = i915_gem_object_wait_rendering(obj, false); |
2019 | if (ret) |
- | |
Line 2020... | Line 2857... | ||
2020 | return ret; |
2857 | if (ret) |
2021 | } |
2858 | return ret; |
2022 | - | ||
2023 | /* Ensure that we invalidate the GPU's caches and TLBs. */ |
2859 | |
2024 | obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; |
2860 | /* Ensure that we invalidate the GPU's caches and TLBs. */ |
Line 2025... | Line 2861... | ||
2025 | 2861 | obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; |
|
2026 | return i915_gem_object_wait_rendering(obj); |
2862 | return 0; |
2027 | } |
2863 | } |
2028 | 2864 | ||
2029 | /** |
2865 | /** |
2030 | * Moves a single object to the CPU read, and possibly write domain. |
2866 | * Moves a single object to the CPU read, and possibly write domain. |
2031 | * |
2867 | * |
2032 | * This function returns when the move is complete, including waiting on |
2868 | * This function returns when the move is complete, including waiting on |
2033 | * flushes to occur. |
2869 | * flushes to occur. |
2034 | */ |
2870 | */ |
2035 | static int |
2871 | int |
Line 2036... | Line 2872... | ||
2036 | i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) |
2872 | i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) |
2037 | { |
2873 | { |
Line 2038... | Line -... | ||
2038 | uint32_t old_write_domain, old_read_domains; |
- | |
2039 | int ret; |
- | |
2040 | - | ||
2041 | if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) |
- | |
2042 | return 0; |
2874 | uint32_t old_write_domain, old_read_domains; |
2043 | 2875 | int ret; |
|
2044 | ret = i915_gem_object_flush_gpu_write_domain(obj); |
2876 | |
Line 2045... | Line 2877... | ||
2045 | if (ret) |
2877 | if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) |
Line 2046... | Line -... | ||
2046 | return ret; |
- | |
2047 | 2878 | return 0; |
|
2048 | ret = i915_gem_object_wait_rendering(obj); |
2879 | |
Line 2049... | Line 2880... | ||
2049 | if (ret) |
2880 | ret = i915_gem_object_wait_rendering(obj, !write); |
2050 | return ret; |
2881 | if (ret) |
Line 2080... | Line 2911... | ||
2080 | old_write_domain); |
2911 | old_write_domain); |
Line 2081... | Line 2912... | ||
2081 | 2912 | ||
2082 | return 0; |
2913 | return 0; |
Line 2083... | Line 2914... | ||
2083 | } |
2914 | } |
2084 | 2915 | ||
- | 2916 | #if 0 |
|
2085 | /** |
2917 | /* Throttle our rendering by waiting until the ring has completed our requests |
2086 | * Moves the object from a partially CPU read to a full one. |
2918 | * emitted over 20 msec ago. |
2087 | * |
2919 | * |
- | 2920 | * Note that if we were to use the current jiffies each time around the loop, |
|
- | 2921 | * we wouldn't escape the function with any frames outstanding if the time to |
|
- | 2922 | * render a frame was over 20ms. |
|
- | 2923 | * |
|
2088 | * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(), |
2924 | * This should get us reasonable parallelism between CPU and GPU but also |
2089 | * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU). |
2925 | * relatively low latency when blocking on a particular request to finish. |
2090 | */ |
2926 | */ |
2091 | static void |
2927 | static int |
2092 | i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj) |
- | |
2093 | { |
- | |
2094 | if (!obj->page_cpu_valid) |
- | |
2095 | return; |
2928 | i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) |
2096 | - | ||
2097 | /* If we're partially in the CPU read domain, finish moving it in. |
2929 | { |
2098 | */ |
- | |
2099 | if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) { |
- | |
2100 | } |
2930 | struct drm_i915_private *dev_priv = dev->dev_private; |
2101 | 2931 | struct drm_i915_file_private *file_priv = file->driver_priv; |
|
2102 | /* Free the page_cpu_valid mappings which are now stale, whether |
- | |
2103 | * or not we've got I915_GEM_DOMAIN_CPU. |
2932 | unsigned long recent_enough = GetTimerTics() - msecs_to_jiffies(20); |
2104 | */ |
2933 | struct drm_i915_gem_request *request; |
2105 | kfree(obj->page_cpu_valid); |
2934 | struct intel_ring_buffer *ring = NULL; |
2106 | obj->page_cpu_valid = NULL; |
- | |
Line -... | Line 2935... | ||
- | 2935 | u32 seqno = 0; |
|
- | 2936 | int ret; |
|
Line -... | Line 2937... | ||
- | 2937 | ||
- | 2938 | if (atomic_read(&dev_priv->mm.wedged)) |
|
- | 2939 | return -EIO; |
|
- | 2940 | ||
Line 2107... | Line 2941... | ||
2107 | } |
2941 | spin_lock(&file_priv->mm.lock); |
2108 | - | ||
2109 | 2942 | list_for_each_entry(request, &file_priv->mm.request_list, client_list) { |
|
2110 | 2943 | if (time_after_eq(request->emitted_jiffies, recent_enough)) |
|
- | 2944 | break; |
|
Line -... | Line 2945... | ||
- | 2945 | ||
- | 2946 | ring = request->ring; |
|
Line -... | Line 2947... | ||
- | 2947 | seqno = request->seqno; |
|
- | 2948 | } |
|
- | 2949 | spin_unlock(&file_priv->mm.lock); |
|
Line 2111... | Line 2950... | ||
2111 | 2950 | ||
2112 | int gem_object_lock(struct drm_i915_gem_object *obj) |
2951 | if (seqno == 0) |
2113 | { |
- | |
2114 | return i915_gem_object_set_to_cpu_domain(obj, true); |
- | |
2115 | } |
- | |
2116 | - | ||
2117 | - | ||
2118 | - | ||
2119 | - | ||
2120 | - | ||
2121 | - | ||
2122 | 2952 | return 0; |
|
2123 | - | ||
2124 | - | ||
2125 | - | ||
Line 2126... | Line 2953... | ||
2126 | 2953 | ||
2127 | 2954 | ret = __wait_seqno(ring, seqno, true, NULL); |
|
2128 | 2955 | if (ret == 0) |
|
2129 | 2956 | queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); |
|
- | 2957 | ||
2130 | 2958 | return ret; |
|
2131 | - | ||
2132 | - | ||
2133 | 2959 | } |
|
Line 2134... | Line 2960... | ||
2134 | 2960 | #endif |
|
2135 | int |
2961 | |
Line 2136... | Line 2962... | ||
2136 | i915_gem_object_pin(struct drm_i915_gem_object *obj, |
2962 | int |
2137 | uint32_t alignment, |
2963 | i915_gem_object_pin(struct drm_i915_gem_object *obj, |
2138 | bool map_and_fenceable) |
2964 | uint32_t alignment, |
2139 | { |
2965 | bool map_and_fenceable, |
Line 2162... | Line 2988... | ||
2162 | } |
2988 | } |
2163 | #endif |
2989 | #endif |
Line 2164... | Line 2990... | ||
2164 | 2990 | ||
2165 | if (obj->gtt_space == NULL) { |
2991 | if (obj->gtt_space == NULL) { |
2166 | ret = i915_gem_object_bind_to_gtt(obj, alignment, |
2992 | ret = i915_gem_object_bind_to_gtt(obj, alignment, |
- | 2993 | map_and_fenceable, |
|
2167 | map_and_fenceable); |
2994 | nonblocking); |
2168 | if (ret) |
2995 | if (ret) |
2169 | return ret; |
2996 | return ret; |
Line 2170... | Line 2997... | ||
2170 | } |
2997 | } |
2171 | - | ||
2172 | if (obj->pin_count++ == 0) { |
2998 | |
2173 | if (!obj->active) |
- | |
2174 | list_move_tail(&obj->mm_list, |
2999 | if (!obj->has_global_gtt_mapping && map_and_fenceable) |
- | 3000 | i915_gem_gtt_bind_object(obj, obj->cache_level); |
|
2175 | &dev_priv->mm.pinned_list); |
3001 | |
Line 2176... | Line -... | ||
2176 | } |
- | |
2177 | obj->pin_mappable |= map_and_fenceable; |
3002 | obj->pin_count++; |
2178 | 3003 | obj->pin_mappable |= map_and_fenceable; |
|
Line 2179... | Line 3004... | ||
2179 | WARN_ON(i915_verify_lists(dev)); |
3004 | |
2180 | return 0; |
3005 | return 0; |
2181 | } |
3006 | } |
2182 | - | ||
2183 | void |
- | |
2184 | i915_gem_object_unpin(struct drm_i915_gem_object *obj) |
- | |
2185 | { |
- | |
2186 | struct drm_device *dev = obj->base.dev; |
3007 | |
2187 | drm_i915_private_t *dev_priv = dev->dev_private; |
3008 | void |
Line 2188... | Line 3009... | ||
2188 | 3009 | i915_gem_object_unpin(struct drm_i915_gem_object *obj) |
|
2189 | WARN_ON(i915_verify_lists(dev)); |
- | |
2190 | BUG_ON(obj->pin_count == 0); |
- | |
2191 | BUG_ON(obj->gtt_space == NULL); |
- | |
2192 | 3010 | { |
|
2193 | if (--obj->pin_count == 0) { |
3011 | BUG_ON(obj->pin_count == 0); |
- | 3012 | BUG_ON(obj->gtt_space == NULL); |
|
- | 3013 | ||
- | 3014 | if (--obj->pin_count == 0) |
|
- | 3015 | obj->pin_mappable = false; |
|
- | 3016 | } |
|
- | 3017 | ||
2194 | if (!obj->active) |
3018 | #if 0 |
- | 3019 | int |
|
- | 3020 | i915_gem_pin_ioctl(struct drm_device *dev, void *data, |
|
- | 3021 | struct drm_file *file) |
|
- | 3022 | { |
|
- | 3023 | struct drm_i915_gem_pin *args = data; |
|
- | 3024 | struct drm_i915_gem_object *obj; |
|
- | 3025 | int ret; |
|
- | 3026 | ||
- | 3027 | ret = i915_mutex_lock_interruptible(dev); |
|
- | 3028 | if (ret) |
|
- | 3029 | return ret; |
|
2195 | list_move_tail(&obj->mm_list, |
3030 | |
Line -... | Line 3031... | ||
- | 3031 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
|
- | 3032 | if (&obj->base == NULL) { |
|
- | 3033 | ret = -ENOENT; |
|
- | 3034 | goto unlock; |
|
- | 3035 | } |
|
Line -... | Line 3036... | ||
- | 3036 | ||
- | 3037 | if (obj->madv != I915_MADV_WILLNEED) { |
|
- | 3038 | DRM_ERROR("Attempting to pin a purgeable buffer\n"); |
|
- | 3039 | ret = -EINVAL; |
|
- | 3040 | goto out; |
|
- | 3041 | } |
|
Line -... | Line 3042... | ||
- | 3042 | ||
- | 3043 | if (obj->pin_filp != NULL && obj->pin_filp != file) { |
|
- | 3044 | DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", |
|
- | 3045 | args->handle); |
|
- | 3046 | ret = -EINVAL; |
|
- | 3047 | goto out; |
|
- | 3048 | } |
|
Line -... | Line 3049... | ||
- | 3049 | ||
- | 3050 | obj->user_pin_count++; |
|
- | 3051 | obj->pin_filp = file; |
|
- | 3052 | if (obj->user_pin_count == 1) { |
|
- | 3053 | ret = i915_gem_object_pin(obj, args->alignment, true, false); |
|
- | 3054 | if (ret) |
|
- | 3055 | goto out; |
|
- | 3056 | } |
|
- | 3057 | ||
- | 3058 | /* XXX - flush the CPU caches for pinned objects |
|
- | 3059 | * as the X server doesn't manage domains yet |
|
Line -... | Line 3060... | ||
- | 3060 | */ |
|
- | 3061 | i915_gem_object_flush_cpu_write_domain(obj); |
|
- | 3062 | args->offset = obj->gtt_offset; |
|
- | 3063 | out: |
|
- | 3064 | drm_gem_object_unreference(&obj->base); |
|
- | 3065 | unlock: |
|
- | 3066 | mutex_unlock(&dev->struct_mutex); |
|
Line -... | Line 3067... | ||
- | 3067 | return ret; |
|
- | 3068 | } |
|
- | 3069 | ||
Line -... | Line 3070... | ||
- | 3070 | int |
|
- | 3071 | i915_gem_unpin_ioctl(struct drm_device *dev, void *data, |
|
- | 3072 | struct drm_file *file) |
|
- | 3073 | { |
|
- | 3074 | struct drm_i915_gem_pin *args = data; |
|
Line -... | Line 3075... | ||
- | 3075 | struct drm_i915_gem_object *obj; |
|
- | 3076 | int ret; |
|
- | 3077 | ||
- | 3078 | ret = i915_mutex_lock_interruptible(dev); |
|
- | 3079 | if (ret) |
|
- | 3080 | return ret; |
|
- | 3081 | ||
- | 3082 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
|
- | 3083 | if (&obj->base == NULL) { |
|
- | 3084 | ret = -ENOENT; |
|
- | 3085 | goto unlock; |
|
Line -... | Line 3086... | ||
- | 3086 | } |
|
- | 3087 | ||
- | 3088 | if (obj->pin_filp != file) { |
|
- | 3089 | DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", |
|
- | 3090 | args->handle); |
|
- | 3091 | ret = -EINVAL; |
|
- | 3092 | goto out; |
|
- | 3093 | } |
|
- | 3094 | obj->user_pin_count--; |
|
- | 3095 | if (obj->user_pin_count == 0) { |
|
- | 3096 | obj->pin_filp = NULL; |
|
- | 3097 | i915_gem_object_unpin(obj); |
|
- | 3098 | } |
|
- | 3099 | ||
Line -... | Line 3100... | ||
- | 3100 | out: |
|
- | 3101 | drm_gem_object_unreference(&obj->base); |
|
- | 3102 | unlock: |
|
Line -... | Line 3103... | ||
- | 3103 | mutex_unlock(&dev->struct_mutex); |
|
- | 3104 | return ret; |
|
- | 3105 | } |
|
- | 3106 | ||
- | 3107 | int |
|
Line -... | Line 3108... | ||
- | 3108 | i915_gem_busy_ioctl(struct drm_device *dev, void *data, |
|
- | 3109 | struct drm_file *file) |
|
- | 3110 | { |
|
- | 3111 | struct drm_i915_gem_busy *args = data; |
|
- | 3112 | struct drm_i915_gem_object *obj; |
|
- | 3113 | int ret; |
|
Line -... | Line 3114... | ||
- | 3114 | ||
- | 3115 | ret = i915_mutex_lock_interruptible(dev); |
|
- | 3116 | if (ret) |
|
- | 3117 | return ret; |
|
- | 3118 | ||
Line -... | Line 3119... | ||
- | 3119 | obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); |
|
- | 3120 | if (&obj->base == NULL) { |
|
- | 3121 | ret = -ENOENT; |
|
- | 3122 | goto unlock; |
|
- | 3123 | } |
|
Line -... | Line 3124... | ||
- | 3124 | ||
- | 3125 | /* Count all active objects as busy, even if they are currently not used |
|
- | 3126 | * by the gpu. Users of this interface expect objects to eventually |
|
- | 3127 | * become non-busy without any further actions, therefore emit any |
|
- | 3128 | * necessary flushes here. |
|
- | 3129 | */ |
|
Line -... | Line 3130... | ||
- | 3130 | ret = i915_gem_object_flush_active(obj); |
|
- | 3131 | ||
- | 3132 | args->busy = obj->active; |
|
- | 3133 | if (obj->ring) { |
|
- | 3134 | BUILD_BUG_ON(I915_NUM_RINGS > 16); |
|
- | 3135 | args->busy |= intel_ring_flag(obj->ring) << 16; |
|
- | 3136 | } |
|
Line -... | Line 3137... | ||
- | 3137 | ||
- | 3138 | drm_gem_object_unreference(&obj->base); |
|
- | 3139 | unlock: |
|
- | 3140 | mutex_unlock(&dev->struct_mutex); |
|
- | 3141 | return ret; |
|
- | 3142 | } |
|
- | 3143 | ||
Line -... | Line 3144... | ||
- | 3144 | int |
|
- | 3145 | i915_gem_throttle_ioctl(struct drm_device *dev, void *data, |
|
- | 3146 | struct drm_file *file_priv) |
|
Line -... | Line 3147... | ||
- | 3147 | { |
|
- | 3148 | return i915_gem_ring_throttle(dev, file_priv); |
|
- | 3149 | } |
|
- | 3150 | ||
- | 3151 | int |
|
Line -... | Line 3152... | ||
- | 3152 | i915_gem_madvise_ioctl(struct drm_device *dev, void *data, |
|
- | 3153 | struct drm_file *file_priv) |
|
- | 3154 | { |
|
- | 3155 | struct drm_i915_gem_madvise *args = data; |
|
Line -... | Line 3156... | ||
- | 3156 | struct drm_i915_gem_object *obj; |
|
- | 3157 | int ret; |
|
Line -... | Line 3158... | ||
- | 3158 | ||
- | 3159 | switch (args->madv) { |
|
- | 3160 | case I915_MADV_DONTNEED: |
|
Line -... | Line 3161... | ||
- | 3161 | case I915_MADV_WILLNEED: |
|
Line -... | Line 3162... | ||
- | 3162 | break; |
|
- | 3163 | default: |
|
- | 3164 | return -EINVAL; |
|
- | 3165 | } |
|
- | 3166 | ||
- | 3167 | ret = i915_mutex_lock_interruptible(dev); |
|
- | 3168 | if (ret) |
|
Line -... | Line 3169... | ||
- | 3169 | return ret; |
|
- | 3170 | ||
- | 3171 | obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); |
|
- | 3172 | if (&obj->base == NULL) { |
|
- | 3173 | ret = -ENOENT; |
|
- | 3174 | goto unlock; |
|
- | 3175 | } |
|
Line -... | Line 3176... | ||
- | 3176 | ||
Line -... | Line 3177... | ||
- | 3177 | if (obj->pin_count) { |
|
- | 3178 | ret = -EINVAL; |
|
- | 3179 | goto out; |
|
- | 3180 | } |
|
Line -... | Line 3181... | ||
- | 3181 | ||
- | 3182 | if (obj->madv != __I915_MADV_PURGED) |
|
Line -... | Line 3183... | ||
- | 3183 | obj->madv = args->madv; |
|
- | 3184 | ||
- | 3185 | /* if the object is no longer attached, discard its backing storage */ |
|
- | 3186 | if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL) |
|
Line 2196... | Line 3187... | ||
2196 | &dev_priv->mm.inactive_list); |
3187 | i915_gem_object_truncate(obj); |
2197 | obj->pin_mappable = false; |
3188 | |
2198 | } |
3189 | args->retained = obj->madv != __I915_MADV_PURGED; |
2199 | WARN_ON(i915_verify_lists(dev)); |
- | |
2200 | } |
3190 | |
- | 3191 | out: |
|
- | 3192 | drm_gem_object_unreference(&obj->base); |
|
Line 2201... | Line 3193... | ||
2201 | 3193 | unlock: |
|
2202 | 3194 | mutex_unlock(&dev->struct_mutex); |
|
2203 | 3195 | return ret; |
|
Line 2242... | Line 3234... | ||
2242 | kfree(obj); |
3234 | kfree(obj); |
2243 | return NULL; |
3235 | return NULL; |
2244 | } |
3236 | } |
Line 2245... | Line 3237... | ||
2245 | 3237 | ||
Line 2246... | Line 3238... | ||
2246 | 3238 | ||
2247 | i915_gem_info_add_obj(dev_priv, size); |
3239 | i915_gem_object_init(obj, &i915_gem_object_ops); |
Line 2248... | Line 3240... | ||
2248 | 3240 | ||
2249 | obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
3241 | obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
2250 | obj->base.read_domains = I915_GEM_DOMAIN_CPU; |
3242 | obj->base.read_domains = I915_GEM_DOMAIN_CPU; |
2251 | 3243 | ||
2252 | if (IS_GEN6(dev) || IS_GEN7(dev)) { |
3244 | if (HAS_LLC(dev)) { |
2253 | /* On Gen6, we can have the GPU use the LLC (the CPU |
3245 | /* On some devices, we can have the GPU use the LLC (the CPU |
2254 | * cache) for about a 10% performance improvement |
3246 | * cache) for about a 10% performance improvement |
Line 2264... | Line 3256... | ||
2264 | */ |
3256 | */ |
2265 | obj->cache_level = I915_CACHE_LLC; |
3257 | obj->cache_level = I915_CACHE_LLC; |
2266 | } else |
3258 | } else |
2267 | obj->cache_level = I915_CACHE_NONE; |
3259 | obj->cache_level = I915_CACHE_NONE; |
Line 2268... | Line -... | ||
2268 | - | ||
2269 | obj->base.driver_private = NULL; |
- | |
2270 | obj->fence_reg = I915_FENCE_REG_NONE; |
- | |
2271 | INIT_LIST_HEAD(&obj->mm_list); |
- | |
2272 | INIT_LIST_HEAD(&obj->gtt_list); |
- | |
2273 | INIT_LIST_HEAD(&obj->ring_list); |
- | |
2274 | INIT_LIST_HEAD(&obj->exec_list); |
- | |
2275 | INIT_LIST_HEAD(&obj->gpu_write_list); |
- | |
2276 | obj->madv = I915_MADV_WILLNEED; |
- | |
2277 | /* Avoid an unnecessary call to unbind on the first bind. */ |
- | |
2278 | obj->map_and_fenceable = true; |
- | |
2279 | 3260 | ||
2280 | return obj; |
3261 | return obj; |
Line 2281... | Line 3262... | ||
2281 | } |
3262 | } |
2282 | 3263 | ||
2283 | int i915_gem_init_object(struct drm_gem_object *obj) |
3264 | int i915_gem_init_object(struct drm_gem_object *obj) |
Line 2284... | Line 3265... | ||
2284 | { |
3265 | { |
2285 | BUG(); |
3266 | BUG(); |
Line 2286... | Line 3267... | ||
2286 | 3267 | ||
2287 | return 0; |
3268 | return 0; |
- | 3269 | } |
|
2288 | } |
3270 | |
2289 | 3271 | void i915_gem_free_object(struct drm_gem_object *gem_obj) |
|
2290 | static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj) |
- | |
Line 2291... | Line 3272... | ||
2291 | { |
3272 | { |
- | 3273 | struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); |
|
- | 3274 | struct drm_device *dev = obj->base.dev; |
|
- | 3275 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 3276 | ||
- | 3277 | trace_i915_gem_object_destroy(obj); |
|
2292 | struct drm_device *dev = obj->base.dev; |
3278 | |
2293 | drm_i915_private_t *dev_priv = dev->dev_private; |
3279 | // if (obj->phys_obj) |
- | 3280 | // i915_gem_detach_phys_object(dev, obj); |
|
- | 3281 | ||
2294 | int ret; |
3282 | obj->pin_count = 0; |
- | 3283 | if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) { |
|
2295 | 3284 | bool was_interruptible; |
|
- | 3285 | ||
- | 3286 | was_interruptible = dev_priv->mm.interruptible; |
|
2296 | ret = i915_gem_object_unbind(obj); |
3287 | dev_priv->mm.interruptible = false; |
Line -... | Line 3288... | ||
- | 3288 | ||
2297 | if (ret == -ERESTARTSYS) { |
3289 | WARN_ON(i915_gem_object_unbind(obj)); |
- | 3290 | ||
- | 3291 | dev_priv->mm.interruptible = was_interruptible; |
|
- | 3292 | } |
|
Line 2298... | Line 3293... | ||
2298 | list_move(&obj->mm_list, |
3293 | |
2299 | &dev_priv->mm.deferred_free_list); |
3294 | obj->pages_pin_count = 0; |
Line 2300... | Line 3295... | ||
2300 | return; |
3295 | i915_gem_object_put_pages(obj); |
2301 | } |
3296 | // i915_gem_object_free_mmap_offset(obj); |
Line 2302... | Line -... | ||
2302 | - | ||
2303 | trace_i915_gem_object_destroy(obj); |
3297 | |
2304 | 3298 | BUG_ON(obj->pages.page); |
|
2305 | // if (obj->base.map_list.map) |
3299 | |
Line -... | Line 3300... | ||
- | 3300 | // if (obj->base.import_attach) |
|
- | 3301 | // drm_prime_gem_destroy(&obj->base, NULL); |
|
2306 | // drm_gem_free_mmap_offset(&obj->base); |
3302 | |
2307 | 3303 | drm_gem_object_release(&obj->base); |
|
2308 | drm_gem_object_release(&obj->base); |
3304 | i915_gem_info_remove_obj(dev_priv, obj->base.size); |
2309 | i915_gem_info_remove_obj(dev_priv, obj->base.size); |
3305 | |
Line 2310... | Line -... | ||
2310 | - | ||
2311 | kfree(obj->page_cpu_valid); |
3306 | kfree(obj->bit_17); |
Line 2312... | Line 3307... | ||
2312 | kfree(obj->bit_17); |
3307 | kfree(obj); |
2313 | kfree(obj); |
3308 | } |
- | 3309 | ||
- | 3310 | #if 0 |
|
Line 2314... | Line 3311... | ||
2314 | } |
3311 | int |
- | 3312 | i915_gem_idle(struct drm_device *dev) |
|
- | 3313 | { |
|
- | 3314 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
2315 | 3315 | int ret; |
|
- | 3316 | ||
Line -... | Line 3317... | ||
- | 3317 | mutex_lock(&dev->struct_mutex); |
|
Line -... | Line 3318... | ||
- | 3318 | ||
- | 3319 | if (dev_priv->mm.suspended) { |
|
- | 3320 | mutex_unlock(&dev->struct_mutex); |
|
- | 3321 | return 0; |
|
- | 3322 | } |
|
- | 3323 | ||
Line -... | Line 3324... | ||
- | 3324 | ret = i915_gpu_idle(dev); |
|
- | 3325 | if (ret) { |
|
Line -... | Line 3326... | ||
- | 3326 | mutex_unlock(&dev->struct_mutex); |
|
Line -... | Line 3327... | ||
- | 3327 | return ret; |
|
- | 3328 | } |
|
Line -... | Line 3329... | ||
- | 3329 | i915_gem_retire_requests(dev); |
|
- | 3330 | ||
- | 3331 | i915_gem_reset_fences(dev); |
|
Line -... | Line 3332... | ||
- | 3332 | ||
- | 3333 | /* Hack! Don't let anybody do execbuf while we don't control the chip. |
|
- | 3334 | * We need to replace this with a semaphore, or something. |
|
- | 3335 | * And not confound mm.suspended! |
|
- | 3336 | */ |
|
Line -... | Line 3337... | ||
- | 3337 | dev_priv->mm.suspended = 1; |
|
- | 3338 | del_timer_sync(&dev_priv->hangcheck_timer); |
|
Line -... | Line 3339... | ||
- | 3339 | ||
- | 3340 | i915_kernel_lost_context(dev); |
|
Line -... | Line 3341... | ||
- | 3341 | i915_gem_cleanup_ringbuffer(dev); |
|
- | 3342 | ||
- | 3343 | mutex_unlock(&dev->struct_mutex); |
|
Line -... | Line 3344... | ||
- | 3344 | ||
- | 3345 | /* Cancel the retire work handler, which should be idle now. */ |
|
- | 3346 | // cancel_delayed_work_sync(&dev_priv->mm.retire_work); |
|
- | 3347 | ||
- | 3348 | return 0; |
|
- | 3349 | } |
|
- | 3350 | #endif |
|
- | 3351 | ||
- | 3352 | void i915_gem_l3_remap(struct drm_device *dev) |
|
- | 3353 | { |
|
- | 3354 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 3355 | u32 misccpctl; |
|
- | 3356 | int i; |
|
- | 3357 | ||
- | 3358 | if (!IS_IVYBRIDGE(dev)) |
|
- | 3359 | return; |
|
- | 3360 | ||
- | 3361 | if (!dev_priv->mm.l3_remap_info) |
|
- | 3362 | return; |
|
- | 3363 | ||
- | 3364 | misccpctl = I915_READ(GEN7_MISCCPCTL); |
|
- | 3365 | I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE); |
|
- | 3366 | POSTING_READ(GEN7_MISCCPCTL); |
|
- | 3367 | ||
- | 3368 | for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) { |
|
- | 3369 | u32 remap = I915_READ(GEN7_L3LOG_BASE + i); |
|
- | 3370 | if (remap && remap != dev_priv->mm.l3_remap_info[i/4]) |
|
- | 3371 | DRM_DEBUG("0x%x was already programmed to %x\n", |
|
- | 3372 | GEN7_L3LOG_BASE + i, remap); |
|
- | 3373 | if (remap && !dev_priv->mm.l3_remap_info[i/4]) |
|
- | 3374 | DRM_DEBUG_DRIVER("Clearing remapped register\n"); |
|
- | 3375 | I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->mm.l3_remap_info[i/4]); |
|
- | 3376 | } |
|
- | 3377 | ||
- | 3378 | /* Make sure all the writes land before disabling dop clock gating */ |
|
- | 3379 | POSTING_READ(GEN7_L3LOG_BASE); |
|
- | 3380 | ||
- | 3381 | I915_WRITE(GEN7_MISCCPCTL, misccpctl); |
|
- | 3382 | } |
|
- | 3383 | ||
- | 3384 | void i915_gem_init_swizzling(struct drm_device *dev) |
|
- | 3385 | { |
|
- | 3386 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 3387 | ||
- | 3388 | if (INTEL_INFO(dev)->gen < 5 || |
|
- | 3389 | dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) |
|
- | 3390 | return; |
|
- | 3391 | ||
- | 3392 | I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | |
|
- | 3393 | DISP_TILE_SURFACE_SWIZZLING); |
|
- | 3394 | ||
- | 3395 | if (IS_GEN5(dev)) |
|
- | 3396 | return; |
|
- | 3397 | ||
- | 3398 | I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); |
|
- | 3399 | if (IS_GEN6(dev)) |
|
- | 3400 | I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); |
|
- | 3401 | else |
|
- | 3402 | I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); |
|
- | 3403 | } |
|
- | 3404 | ||
- | 3405 | void i915_gem_init_ppgtt(struct drm_device *dev) |
|
- | 3406 | { |
|
- | 3407 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 3408 | uint32_t pd_offset; |
|
- | 3409 | struct intel_ring_buffer *ring; |
|
- | 3410 | struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; |
|
- | 3411 | uint32_t __iomem *pd_addr; |
|
- | 3412 | uint32_t pd_entry; |
|
- | 3413 | int i; |
|
- | 3414 | ||
- | 3415 | if (!dev_priv->mm.aliasing_ppgtt) |
|
- | 3416 | return; |
|
- | 3417 | ||
- | 3418 | ||
- | 3419 | pd_addr = dev_priv->mm.gtt->gtt + ppgtt->pd_offset/sizeof(uint32_t); |
|
- | 3420 | for (i = 0; i < ppgtt->num_pd_entries; i++) { |
|
- | 3421 | dma_addr_t pt_addr; |
|
- | 3422 | ||
- | 3423 | if (dev_priv->mm.gtt->needs_dmar) |
|
- | 3424 | pt_addr = ppgtt->pt_dma_addr[i]; |
|
- | 3425 | else |
|
- | 3426 | pt_addr = ppgtt->pt_pages[i]; |
|
- | 3427 | ||
- | 3428 | pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); |
|
- | 3429 | pd_entry |= GEN6_PDE_VALID; |
|
- | 3430 | ||
- | 3431 | writel(pd_entry, pd_addr + i); |
|
- | 3432 | } |
|
- | 3433 | readl(pd_addr); |
|
- | 3434 | ||
- | 3435 | pd_offset = ppgtt->pd_offset; |
|
- | 3436 | pd_offset /= 64; /* in cachelines, */ |
|
- | 3437 | pd_offset <<= 16; |
|
- | 3438 | ||
- | 3439 | if (INTEL_INFO(dev)->gen == 6) { |
|
- | 3440 | uint32_t ecochk, gab_ctl, ecobits; |
|
- | 3441 | ||
- | 3442 | ecobits = I915_READ(GAC_ECO_BITS); |
|
- | 3443 | I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); |
|
- | 3444 | ||
- | 3445 | gab_ctl = I915_READ(GAB_CTL); |
|
- | 3446 | I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); |
|
- | 3447 | ||
- | 3448 | ecochk = I915_READ(GAM_ECOCHK); |
|
- | 3449 | I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | |
|
- | 3450 | ECOCHK_PPGTT_CACHE64B); |
|
- | 3451 | I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
|
- | 3452 | } else if (INTEL_INFO(dev)->gen >= 7) { |
|
- | 3453 | I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); |
|
- | 3454 | /* GFX_MODE is per-ring on gen7+ */ |
|
- | 3455 | } |
|
- | 3456 | ||
- | 3457 | for_each_ring(ring, dev_priv, i) { |
|
Line 2316... | Line 3458... | ||
2316 | void i915_gem_free_object(struct drm_gem_object *gem_obj) |
3458 | if (INTEL_INFO(dev)->gen >= 7) |
2317 | { |
3459 | I915_WRITE(RING_MODE_GEN7(ring), |
2318 | struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); |
3460 | _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); |
2319 | struct drm_device *dev = obj->base.dev; |
3461 | |
2320 | 3462 | I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); |
|
Line -... | Line 3463... | ||
- | 3463 | I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); |
|
- | 3464 | } |
|
- | 3465 | } |
|
- | 3466 | ||
- | 3467 | static bool |
|
- | 3468 | intel_enable_blt(struct drm_device *dev) |
|
- | 3469 | { |
|
- | 3470 | if (!HAS_BLT(dev)) |
|
- | 3471 | return false; |
|
- | 3472 | ||
2321 | while (obj->pin_count > 0) |
3473 | /* The blitter was dysfunctional on early prototypes */ |
2322 | i915_gem_object_unpin(obj); |
3474 | if (IS_GEN6(dev) && dev->pdev->revision < 8) { |
2323 | 3475 | DRM_INFO("BLT not supported on this pre-production hardware;" |
|
Line 2324... | Line 3476... | ||
2324 | // if (obj->phys_obj) |
3476 | " graphics performance will be degraded.\n"); |
2325 | // i915_gem_detach_phys_object(dev, obj); |
3477 | return false; |
2326 | 3478 | } |
|
2327 | i915_gem_free_object_tail(obj); |
3479 | |
2328 | } |
3480 | return true; |
Line 2329... | Line 3481... | ||
2329 | 3481 | } |
|
2330 | 3482 | ||
2331 | 3483 | int |
|
2332 | 3484 | i915_gem_init_hw(struct drm_device *dev) |
|
2333 | 3485 | { |
|
Line 2334... | Line 3486... | ||
2334 | 3486 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
Line -... | Line 3487... | ||
- | 3487 | int ret; |
|
- | 3488 | ||
- | 3489 | if (!intel_enable_gtt()) |
|
- | 3490 | return -EIO; |
|
- | 3491 | ||
- | 3492 | if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1)) |
|
- | 3493 | I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000); |
|
2335 | 3494 | ||
Line 2336... | Line 3495... | ||
2336 | 3495 | i915_gem_l3_remap(dev); |
|
2337 | 3496 | ||
2338 | 3497 | i915_gem_init_swizzling(dev); |
|
2339 | 3498 | ||
2340 | 3499 | ret = intel_init_render_ring_buffer(dev); |
|
2341 | int |
3500 | if (ret) |
Line -... | Line 3501... | ||
- | 3501 | return ret; |
|
- | 3502 | ||
- | 3503 | if (HAS_BSD(dev)) { |
|
- | 3504 | ret = intel_init_bsd_ring_buffer(dev); |
|
- | 3505 | if (ret) |
|
- | 3506 | goto cleanup_render_ring; |
|
- | 3507 | } |
|
- | 3508 | ||
- | 3509 | if (intel_enable_blt(dev)) { |
|
- | 3510 | ret = intel_init_blt_ring_buffer(dev); |
|
- | 3511 | if (ret) |
|
- | 3512 | goto cleanup_bsd_ring; |
|
- | 3513 | } |
|
- | 3514 | ||
- | 3515 | dev_priv->next_seqno = 1; |
|
- | 3516 | ||
- | 3517 | /* |
|
- | 3518 | * XXX: There was some w/a described somewhere suggesting loading |
|
- | 3519 | * contexts before PPGTT. |
|
- | 3520 | */ |
|
- | 3521 | i915_gem_context_init(dev); |
|
- | 3522 | i915_gem_init_ppgtt(dev); |
|
- | 3523 | ||
- | 3524 | return 0; |
|
- | 3525 | ||
- | 3526 | cleanup_bsd_ring: |
|
- | 3527 | intel_cleanup_ring_buffer(&dev_priv->ring[VCS]); |
|
- | 3528 | cleanup_render_ring: |
|
- | 3529 | intel_cleanup_ring_buffer(&dev_priv->ring[RCS]); |
|
- | 3530 | return ret; |
|
- | 3531 | } |
|
- | 3532 | ||
- | 3533 | static bool |
|
- | 3534 | intel_enable_ppgtt(struct drm_device *dev) |
|
- | 3535 | { |
|
- | 3536 | if (i915_enable_ppgtt >= 0) |
|
- | 3537 | return i915_enable_ppgtt; |
|
- | 3538 | ||
- | 3539 | #ifdef CONFIG_INTEL_IOMMU |
|
- | 3540 | /* Disable ppgtt on SNB if VT-d is on. */ |
|
- | 3541 | if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) |
|
2342 | i915_gem_init_ringbuffer(struct drm_device *dev) |
3542 | return false; |
- | 3543 | #endif |
|
- | 3544 | ||
- | 3545 | return true; |
|
- | 3546 | } |
|
- | 3547 | ||
- | 3548 | #define LFB_SIZE 0xC00000 |
|
- | 3549 | ||
- | 3550 | int i915_gem_init(struct drm_device *dev) |
|
- | 3551 | { |
|
- | 3552 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
- | 3553 | unsigned long gtt_size, mappable_size; |
|
- | 3554 | int ret; |
|
- | 3555 | ||
- | 3556 | gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT; |
|
- | 3557 | mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT; |
|
- | 3558 | ||
- | 3559 | mutex_lock(&dev->struct_mutex); |
|
- | 3560 | if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) { |
|
- | 3561 | /* PPGTT pdes are stolen from global gtt ptes, so shrink the |
|
- | 3562 | * aperture accordingly when using aliasing ppgtt. */ |
|
- | 3563 | gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE; |
|
2343 | { |
3564 | |
2344 | drm_i915_private_t *dev_priv = dev->dev_private; |
3565 | i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE); |
2345 | int ret; |
3566 | |
2346 | 3567 | ret = i915_gem_init_aliasing_ppgtt(dev); |
|
- | 3568 | if (ret) { |
|
2347 | ret = intel_init_render_ring_buffer(dev); |
3569 | mutex_unlock(&dev->struct_mutex); |
Line 2348... | Line 3570... | ||
2348 | if (ret) |
3570 | return ret; |
2349 | return ret; |
3571 | } |
2350 | 3572 | } else { |
|
Line -... | Line 3573... | ||
- | 3573 | /* Let GEM Manage all of the aperture. |
|
- | 3574 | * |
|
2351 | if (HAS_BSD(dev)) { |
3575 | * However, leave one page at the end still bound to the scratch |
2352 | ret = intel_init_bsd_ring_buffer(dev); |
3576 | * page. There are a number of places where the hardware |
2353 | if (ret) |
3577 | * apparently prefetches past the end of the object, and we've |
2354 | goto cleanup_render_ring; |
3578 | * seen multiple hangs with the GPU head pointer stuck in a |
2355 | } |
3579 | * batchbuffer bound at the last page of the aperture. One page |
2356 | 3580 | * should be enough to keep any prefetching inside of the |
|
Line 2357... | Line 3581... | ||
2357 | if (HAS_BLT(dev)) { |
3581 | * aperture. |
2358 | ret = intel_init_blt_ring_buffer(dev); |
3582 | */ |
Line 2359... | Line 3583... | ||
2359 | if (ret) |
3583 | i915_gem_init_global_gtt(dev, LFB_SIZE, mappable_size, gtt_size - LFB_SIZE); |
Line 2398... | Line 3622... | ||
2398 | } |
3622 | } |
Line 2399... | Line 3623... | ||
2399 | 3623 | ||
2400 | mutex_lock(&dev->struct_mutex); |
3624 | mutex_lock(&dev->struct_mutex); |
Line 2401... | Line 3625... | ||
2401 | dev_priv->mm.suspended = 0; |
3625 | dev_priv->mm.suspended = 0; |
2402 | 3626 | ||
2403 | ret = i915_gem_init_ringbuffer(dev); |
3627 | ret = i915_gem_init_hw(dev); |
2404 | if (ret != 0) { |
3628 | if (ret != 0) { |
2405 | mutex_unlock(&dev->struct_mutex); |
3629 | mutex_unlock(&dev->struct_mutex); |
Line 2406... | Line 3630... | ||
2406 | return ret; |
3630 | return ret; |
2407 | } |
- | |
2408 | - | ||
2409 | BUG_ON(!list_empty(&dev_priv->mm.active_list)); |
- | |
2410 | BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); |
- | |
2411 | BUG_ON(!list_empty(&dev_priv->mm.inactive_list)); |
- | |
2412 | for (i = 0; i < I915_NUM_RINGS; i++) { |
- | |
2413 | BUG_ON(!list_empty(&dev_priv->ring[i].active_list)); |
3631 | } |
Line 2414... | Line 3632... | ||
2414 | BUG_ON(!list_empty(&dev_priv->ring[i].request_list)); |
3632 | |
2415 | } |
3633 | BUG_ON(!list_empty(&dev_priv->mm.active_list)); |
2416 | mutex_unlock(&dev->struct_mutex); |
3634 | mutex_unlock(&dev->struct_mutex); |
Line 2458... | Line 3676... | ||
2458 | static void |
3676 | static void |
2459 | init_ring_lists(struct intel_ring_buffer *ring) |
3677 | init_ring_lists(struct intel_ring_buffer *ring) |
2460 | { |
3678 | { |
2461 | INIT_LIST_HEAD(&ring->active_list); |
3679 | INIT_LIST_HEAD(&ring->active_list); |
2462 | INIT_LIST_HEAD(&ring->request_list); |
3680 | INIT_LIST_HEAD(&ring->request_list); |
2463 | INIT_LIST_HEAD(&ring->gpu_write_list); |
- | |
2464 | } |
3681 | } |
Line 2465... | Line 3682... | ||
2465 | 3682 | ||
2466 | void |
3683 | void |
2467 | i915_gem_load(struct drm_device *dev) |
3684 | i915_gem_load(struct drm_device *dev) |
2468 | { |
3685 | { |
2469 | int i; |
3686 | int i; |
Line 2470... | Line 3687... | ||
2470 | drm_i915_private_t *dev_priv = dev->dev_private; |
3687 | drm_i915_private_t *dev_priv = dev->dev_private; |
2471 | - | ||
2472 | INIT_LIST_HEAD(&dev_priv->mm.active_list); |
3688 | |
- | 3689 | INIT_LIST_HEAD(&dev_priv->mm.active_list); |
|
2473 | INIT_LIST_HEAD(&dev_priv->mm.flushing_list); |
3690 | INIT_LIST_HEAD(&dev_priv->mm.inactive_list); |
2474 | INIT_LIST_HEAD(&dev_priv->mm.inactive_list); |
3691 | INIT_LIST_HEAD(&dev_priv->mm.unbound_list); |
2475 | INIT_LIST_HEAD(&dev_priv->mm.pinned_list); |
- | |
2476 | INIT_LIST_HEAD(&dev_priv->mm.fence_list); |
- | |
2477 | INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); |
3692 | INIT_LIST_HEAD(&dev_priv->mm.bound_list); |
2478 | INIT_LIST_HEAD(&dev_priv->mm.gtt_list); |
3693 | INIT_LIST_HEAD(&dev_priv->mm.fence_list); |
2479 | for (i = 0; i < I915_NUM_RINGS; i++) |
3694 | for (i = 0; i < I915_NUM_RINGS; i++) |
2480 | init_ring_lists(&dev_priv->ring[i]); |
3695 | init_ring_lists(&dev_priv->ring[i]); |
2481 | for (i = 0; i < I915_MAX_NUM_FENCES; i++) |
3696 | for (i = 0; i < I915_MAX_NUM_FENCES; i++) |
2482 | INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); |
3697 | INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); |
Line 2483... | Line 3698... | ||
2483 | INIT_DELAYED_WORK(&dev_priv->mm.retire_work, |
3698 | INIT_DELAYED_WORK(&dev_priv->mm.retire_work, |
2484 | i915_gem_retire_work_handler); |
3699 | i915_gem_retire_work_handler); |
2485 | 3700 | ||
2486 | /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ |
3701 | /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ |
2487 | if (IS_GEN3(dev)) { |
- | |
2488 | u32 tmp = I915_READ(MI_ARB_STATE); |
- | |
2489 | if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) { |
- | |
2490 | /* arb state is a masked write, so set bit + bit in mask */ |
- | |
2491 | tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT); |
3702 | if (IS_GEN3(dev)) { |
Line 2492... | Line 3703... | ||
2492 | I915_WRITE(MI_ARB_STATE, tmp); |
3703 | I915_WRITE(MI_ARB_STATE, |
Line 2493... | Line 3704... | ||
2493 | } |
3704 | _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); |
2494 | } |
3705 | } |
2495 | 3706 | ||
2496 | dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; |
3707 | dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; |
Line 2497... | Line 3708... | ||
2497 | 3708 | ||
2498 | if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) |
- | |
2499 | dev_priv->num_fence_regs = 16; |
3709 | if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) |
2500 | else |
- | |
Line 2501... | Line 3710... | ||
2501 | dev_priv->num_fence_regs = 8; |
3710 | dev_priv->num_fence_regs = 16; |
Line 2502... | Line 3711... | ||
2502 | 3711 | else |