Rev 2330 | Rev 2335 | Go to most recent revision | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 2330 | Rev 2332 | ||
---|---|---|---|
Line 34... | Line 34... | ||
34 | //#include |
34 | //#include |
35 | #include |
35 | #include |
36 | //#include |
36 | //#include |
37 | #include |
37 | #include |
Line -... | Line 38... | ||
- | 38 | ||
- | 39 | ||
- | 40 | #define MAX_ERRNO 4095 |
|
- | 41 | ||
- | 42 | #define IS_ERR_VALUE(x) unlikely((x) >= (unsigned long)-MAX_ERRNO) |
|
- | 43 | ||
- | 44 | static inline long IS_ERR(const void *ptr) |
|
- | 45 | { |
|
- | 46 | return IS_ERR_VALUE((unsigned long)ptr); |
|
- | 47 | } |
|
- | 48 | ||
- | 49 | static inline void *ERR_PTR(long error) |
|
- | 50 | { |
|
- | 51 | return (void *) error; |
|
- | 52 | } |
|
- | 53 | ||
- | 54 | static inline long PTR_ERR(const void *ptr) |
|
- | 55 | { |
|
- | 56 | return (long) ptr; |
|
- | 57 | } |
|
- | 58 | ||
- | 59 | ||
- | 60 | /** |
|
- | 61 | * Initialize an already allocated GEM object of the specified size with |
|
- | 62 | * shmfs backing store. |
|
- | 63 | */ |
|
- | 64 | int drm_gem_object_init(struct drm_device *dev, |
|
- | 65 | struct drm_gem_object *obj, size_t size) |
|
- | 66 | { |
|
- | 67 | BUG_ON((size & (PAGE_SIZE - 1)) != 0); |
|
- | 68 | ||
- | 69 | obj->dev = dev; |
|
- | 70 | atomic_set(&obj->handle_count, 0); |
|
- | 71 | obj->size = size; |
|
- | 72 | ||
- | 73 | return 0; |
|
- | 74 | } |
|
- | 75 | ||
- | 76 | ||
- | 77 | ||
38 | 78 | ||
39 | #define I915_EXEC_CONSTANTS_MASK (3<<6) |
79 | #define I915_EXEC_CONSTANTS_MASK (3<<6) |
40 | #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */ |
80 | #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */ |
41 | #define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6) |
81 | #define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6) |
Line -... | Line 82... | ||
- | 82 | #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */ |
|
- | 83 | ||
- | 84 | static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj); |
|
- | 85 | static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); |
|
- | 86 | static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); |
|
- | 87 | static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, |
|
- | 88 | bool write); |
|
- | 89 | static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, |
|
- | 90 | uint64_t offset, |
|
- | 91 | uint64_t size); |
|
- | 92 | static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj); |
|
- | 93 | static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, |
|
- | 94 | unsigned alignment, |
|
- | 95 | bool map_and_fenceable); |
|
- | 96 | static void i915_gem_clear_fence_reg(struct drm_device *dev, |
|
- | 97 | struct drm_i915_fence_reg *reg); |
|
- | 98 | static int i915_gem_phys_pwrite(struct drm_device *dev, |
|
- | 99 | struct drm_i915_gem_object *obj, |
|
- | 100 | struct drm_i915_gem_pwrite *args, |
|
- | 101 | struct drm_file *file); |
|
- | 102 | static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj); |
|
- | 103 | ||
- | 104 | static int i915_gem_inactive_shrink(struct shrinker *shrinker, |
|
- | 105 | struct shrink_control *sc); |
|
- | 106 | ||
- | 107 | /* some bookkeeping */ |
|
- | 108 | static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, |
|
- | 109 | size_t size) |
|
- | 110 | { |
|
- | 111 | dev_priv->mm.object_count++; |
|
- | 112 | dev_priv->mm.object_memory += size; |
|
- | 113 | } |
|
- | 114 | ||
- | 115 | static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, |
|
- | 116 | size_t size) |
|
- | 117 | { |
|
- | 118 | dev_priv->mm.object_count--; |
|
- | 119 | dev_priv->mm.object_memory -= size; |
|
- | 120 | } |
|
- | 121 | ||
- | 122 | #if 0 |
|
- | 123 | ||
- | 124 | static int |
|
- | 125 | i915_gem_wait_for_error(struct drm_device *dev) |
|
- | 126 | { |
|
- | 127 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
- | 128 | struct completion *x = &dev_priv->error_completion; |
|
- | 129 | unsigned long flags; |
|
- | 130 | int ret; |
|
- | 131 | ||
- | 132 | if (!atomic_read(&dev_priv->mm.wedged)) |
|
- | 133 | return 0; |
|
- | 134 | ||
- | 135 | ret = wait_for_completion_interruptible(x); |
|
- | 136 | if (ret) |
|
- | 137 | return ret; |
|
- | 138 | ||
- | 139 | if (atomic_read(&dev_priv->mm.wedged)) { |
|
- | 140 | /* GPU is hung, bump the completion count to account for |
|
- | 141 | * the token we just consumed so that we never hit zero and |
|
- | 142 | * end up waiting upon a subsequent completion event that |
|
- | 143 | * will never happen. |
|
- | 144 | */ |
|
- | 145 | spin_lock_irqsave(&x->wait.lock, flags); |
|
- | 146 | x->done++; |
|
- | 147 | spin_unlock_irqrestore(&x->wait.lock, flags); |
|
- | 148 | } |
|
- | 149 | return 0; |
|
- | 150 | } |
|
- | 151 | ||
- | 152 | int i915_mutex_lock_interruptible(struct drm_device *dev) |
|
- | 153 | { |
|
- | 154 | int ret; |
|
- | 155 | ||
- | 156 | ret = i915_gem_wait_for_error(dev); |
|
- | 157 | if (ret) |
|
- | 158 | return ret; |
|
- | 159 | ||
- | 160 | ret = mutex_lock_interruptible(&dev->struct_mutex); |
|
- | 161 | if (ret) |
|
- | 162 | return ret; |
|
- | 163 | ||
- | 164 | WARN_ON(i915_verify_lists(dev)); |
|
- | 165 | return 0; |
|
- | 166 | } |
|
- | 167 | ||
- | 168 | static inline bool |
|
- | 169 | i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) |
|
- | 170 | { |
|
- | 171 | return obj->gtt_space && !obj->active && obj->pin_count == 0; |
|
- | 172 | } |
|
- | 173 | ||
- | 174 | #endif |
|
- | 175 | ||
- | 176 | void i915_gem_do_init(struct drm_device *dev, |
|
- | 177 | unsigned long start, |
|
- | 178 | unsigned long mappable_end, |
|
- | 179 | unsigned long end) |
|
- | 180 | { |
|
- | 181 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 182 | ||
- | 183 | drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); |
|
- | 184 | ||
- | 185 | dev_priv->mm.gtt_start = start; |
|
- | 186 | dev_priv->mm.gtt_mappable_end = mappable_end; |
|
- | 187 | dev_priv->mm.gtt_end = end; |
|
- | 188 | dev_priv->mm.gtt_total = end - start; |
|
- | 189 | dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start; |
|
- | 190 | ||
- | 191 | /* Take over this portion of the GTT */ |
|
- | 192 | intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); |
|
- | 193 | } |
|
- | 194 | ||
- | 195 | #if 0 |
|
- | 196 | ||
- | 197 | int |
|
- | 198 | i915_gem_init_ioctl(struct drm_device *dev, void *data, |
|
- | 199 | struct drm_file *file) |
|
- | 200 | { |
|
- | 201 | struct drm_i915_gem_init *args = data; |
|
- | 202 | ||
- | 203 | if (args->gtt_start >= args->gtt_end || |
|
- | 204 | (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) |
|
- | 205 | return -EINVAL; |
|
- | 206 | ||
- | 207 | mutex_lock(&dev->struct_mutex); |
|
- | 208 | i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end); |
|
- | 209 | mutex_unlock(&dev->struct_mutex); |
|
- | 210 | ||
- | 211 | return 0; |
|
- | 212 | } |
|
- | 213 | ||
- | 214 | int |
|
- | 215 | i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, |
|
- | 216 | struct drm_file *file) |
|
- | 217 | { |
|
- | 218 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
- | 219 | struct drm_i915_gem_get_aperture *args = data; |
|
- | 220 | struct drm_i915_gem_object *obj; |
|
- | 221 | size_t pinned; |
|
- | 222 | ||
- | 223 | if (!(dev->driver->driver_features & DRIVER_GEM)) |
|
- | 224 | return -ENODEV; |
|
- | 225 | ||
- | 226 | pinned = 0; |
|
- | 227 | mutex_lock(&dev->struct_mutex); |
|
- | 228 | list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) |
|
- | 229 | pinned += obj->gtt_space->size; |
|
- | 230 | mutex_unlock(&dev->struct_mutex); |
|
- | 231 | ||
- | 232 | args->aper_size = dev_priv->mm.gtt_total; |
|
- | 233 | args->aper_available_size = args->aper_size -pinned; |
|
- | 234 | ||
- | 235 | return 0; |
|
- | 236 | } |
|
- | 237 | ||
- | 238 | static int |
|
- | 239 | i915_gem_create(struct drm_file *file, |
|
- | 240 | struct drm_device *dev, |
|
- | 241 | uint64_t size, |
|
- | 242 | uint32_t *handle_p) |
|
- | 243 | { |
|
- | 244 | struct drm_i915_gem_object *obj; |
|
- | 245 | int ret; |
|
- | 246 | u32 handle; |
|
- | 247 | ||
- | 248 | size = roundup(size, PAGE_SIZE); |
|
- | 249 | ||
- | 250 | /* Allocate the new object */ |
|
- | 251 | obj = i915_gem_alloc_object(dev, size); |
|
- | 252 | if (obj == NULL) |
|
- | 253 | return -ENOMEM; |
|
- | 254 | ||
- | 255 | ret = drm_gem_handle_create(file, &obj->base, &handle); |
|
- | 256 | if (ret) { |
|
- | 257 | drm_gem_object_release(&obj->base); |
|
- | 258 | i915_gem_info_remove_obj(dev->dev_private, obj->base.size); |
|
- | 259 | kfree(obj); |
|
- | 260 | return ret; |
|
- | 261 | } |
|
- | 262 | ||
- | 263 | /* drop reference from allocate - handle holds it now */ |
|
- | 264 | drm_gem_object_unreference(&obj->base); |
|
- | 265 | // trace_i915_gem_object_create(obj); |
|
- | 266 | ||
- | 267 | *handle_p = handle; |
|
- | 268 | return 0; |
|
- | 269 | } |
|
- | 270 | ||
- | 271 | int |
|
- | 272 | i915_gem_dumb_create(struct drm_file *file, |
|
- | 273 | struct drm_device *dev, |
|
- | 274 | struct drm_mode_create_dumb *args) |
|
- | 275 | { |
|
- | 276 | /* have to work out size/pitch and return them */ |
|
- | 277 | args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64); |
|
- | 278 | args->size = args->pitch * args->height; |
|
- | 279 | return i915_gem_create(file, dev, |
|
- | 280 | args->size, &args->handle); |
|
- | 281 | } |
|
- | 282 | ||
- | 283 | int i915_gem_dumb_destroy(struct drm_file *file, |
|
- | 284 | struct drm_device *dev, |
|
- | 285 | uint32_t handle) |
|
- | 286 | { |
|
- | 287 | return drm_gem_handle_delete(file, handle); |
|
- | 288 | } |
|
- | 289 | ||
- | 290 | /** |
|
- | 291 | * Creates a new mm object and returns a handle to it. |
|
- | 292 | */ |
|
- | 293 | int |
|
- | 294 | i915_gem_create_ioctl(struct drm_device *dev, void *data, |
|
- | 295 | struct drm_file *file) |
|
- | 296 | { |
|
- | 297 | struct drm_i915_gem_create *args = data; |
|
- | 298 | return i915_gem_create(file, dev, |
|
- | 299 | args->size, &args->handle); |
|
- | 300 | } |
|
- | 301 | ||
- | 302 | static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) |
|
- | 303 | { |
|
- | 304 | drm_i915_private_t *dev_priv = obj->base.dev->dev_private; |
|
- | 305 | ||
- | 306 | return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && |
|
- | 307 | obj->tiling_mode != I915_TILING_NONE; |
|
- | 308 | } |
|
- | 309 | ||
- | 310 | static inline void |
|
- | 311 | slow_shmem_copy(struct page *dst_page, |
|
- | 312 | int dst_offset, |
|
- | 313 | struct page *src_page, |
|
- | 314 | int src_offset, |
|
- | 315 | int length) |
|
- | 316 | { |
|
- | 317 | char *dst_vaddr, *src_vaddr; |
|
- | 318 | ||
- | 319 | dst_vaddr = kmap(dst_page); |
|
- | 320 | src_vaddr = kmap(src_page); |
|
- | 321 | ||
- | 322 | memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length); |
|
- | 323 | ||
- | 324 | kunmap(src_page); |
|
- | 325 | kunmap(dst_page); |
|
- | 326 | } |
|
- | 327 | ||
- | 328 | static inline void |
|
- | 329 | slow_shmem_bit17_copy(struct page *gpu_page, |
|
- | 330 | int gpu_offset, |
|
- | 331 | struct page *cpu_page, |
|
- | 332 | int cpu_offset, |
|
- | 333 | int length, |
|
- | 334 | int is_read) |
|
- | 335 | { |
|
- | 336 | char *gpu_vaddr, *cpu_vaddr; |
|
- | 337 | ||
- | 338 | /* Use the unswizzled path if this page isn't affected. */ |
|
- | 339 | if ((page_to_phys(gpu_page) & (1 << 17)) == 0) { |
|
- | 340 | if (is_read) |
|
- | 341 | return slow_shmem_copy(cpu_page, cpu_offset, |
|
- | 342 | gpu_page, gpu_offset, length); |
|
- | 343 | else |
|
- | 344 | return slow_shmem_copy(gpu_page, gpu_offset, |
|
- | 345 | cpu_page, cpu_offset, length); |
|
- | 346 | } |
|
- | 347 | ||
- | 348 | gpu_vaddr = kmap(gpu_page); |
|
- | 349 | cpu_vaddr = kmap(cpu_page); |
|
- | 350 | ||
- | 351 | /* Copy the data, XORing A6 with A17 (1). The user already knows he's |
|
- | 352 | * XORing with the other bits (A9 for Y, A9 and A10 for X) |
|
- | 353 | */ |
|
- | 354 | while (length > 0) { |
|
- | 355 | int cacheline_end = ALIGN(gpu_offset + 1, 64); |
|
- | 356 | int this_length = min(cacheline_end - gpu_offset, length); |
|
- | 357 | int swizzled_gpu_offset = gpu_offset ^ 64; |
|
- | 358 | ||
- | 359 | if (is_read) { |
|
- | 360 | memcpy(cpu_vaddr + cpu_offset, |
|
- | 361 | gpu_vaddr + swizzled_gpu_offset, |
|
- | 362 | this_length); |
|
- | 363 | } else { |
|
- | 364 | memcpy(gpu_vaddr + swizzled_gpu_offset, |
|
- | 365 | cpu_vaddr + cpu_offset, |
|
- | 366 | this_length); |
|
- | 367 | } |
|
- | 368 | cpu_offset += this_length; |
|
- | 369 | gpu_offset += this_length; |
|
- | 370 | length -= this_length; |
|
- | 371 | } |
|
- | 372 | ||
- | 373 | kunmap(cpu_page); |
|
- | 374 | kunmap(gpu_page); |
|
- | 375 | } |
|
- | 376 | ||
- | 377 | /** |
|
- | 378 | * This is the fast shmem pread path, which attempts to copy_from_user directly |
|
- | 379 | * from the backing pages of the object to the user's address space. On a |
|
- | 380 | * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow(). |
|
- | 381 | */ |
|
- | 382 | static int |
|
- | 383 | i915_gem_shmem_pread_fast(struct drm_device *dev, |
|
- | 384 | struct drm_i915_gem_object *obj, |
|
- | 385 | struct drm_i915_gem_pread *args, |
|
- | 386 | struct drm_file *file) |
|
- | 387 | { |
|
- | 388 | struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; |
|
- | 389 | ssize_t remain; |
|
- | 390 | loff_t offset; |
|
- | 391 | char __user *user_data; |
|
- | 392 | int page_offset, page_length; |
|
- | 393 | ||
- | 394 | user_data = (char __user *) (uintptr_t) args->data_ptr; |
|
- | 395 | remain = args->size; |
|
- | 396 | ||
- | 397 | offset = args->offset; |
|
- | 398 | ||
- | 399 | while (remain > 0) { |
|
- | 400 | struct page *page; |
|
- | 401 | char *vaddr; |
|
- | 402 | int ret; |
|
- | 403 | ||
- | 404 | /* Operation in this page |
|
- | 405 | * |
|
- | 406 | * page_offset = offset within page |
|
- | 407 | * page_length = bytes to copy for this page |
|
- | 408 | */ |
|
- | 409 | page_offset = offset_in_page(offset); |
|
- | 410 | page_length = remain; |
|
- | 411 | if ((page_offset + remain) > PAGE_SIZE) |
|
- | 412 | page_length = PAGE_SIZE - page_offset; |
|
- | 413 | ||
- | 414 | page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT); |
|
- | 415 | if (IS_ERR(page)) |
|
- | 416 | return PTR_ERR(page); |
|
- | 417 | ||
- | 418 | vaddr = kmap_atomic(page); |
|
- | 419 | ret = __copy_to_user_inatomic(user_data, |
|
- | 420 | vaddr + page_offset, |
|
- | 421 | page_length); |
|
- | 422 | kunmap_atomic(vaddr); |
|
- | 423 | ||
- | 424 | mark_page_accessed(page); |
|
- | 425 | page_cache_release(page); |
|
- | 426 | if (ret) |
|
- | 427 | return -EFAULT; |
|
- | 428 | ||
- | 429 | remain -= page_length; |
|
- | 430 | user_data += page_length; |
|
- | 431 | offset += page_length; |
|
- | 432 | } |
|
- | 433 | ||
- | 434 | return 0; |
|
- | 435 | } |
|
- | 436 | ||
- | 437 | /** |
|
- | 438 | * This is the fallback shmem pread path, which allocates temporary storage |
|
- | 439 | * in kernel space to copy_to_user into outside of the struct_mutex, so we |
|
- | 440 | * can copy out of the object's backing pages while holding the struct mutex |
|
- | 441 | * and not take page faults. |
|
- | 442 | */ |
|
- | 443 | static int |
|
- | 444 | i915_gem_shmem_pread_slow(struct drm_device *dev, |
|
- | 445 | struct drm_i915_gem_object *obj, |
|
- | 446 | struct drm_i915_gem_pread *args, |
|
- | 447 | struct drm_file *file) |
|
- | 448 | { |
|
- | 449 | struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; |
|
- | 450 | struct mm_struct *mm = current->mm; |
|
- | 451 | struct page **user_pages; |
|
- | 452 | ssize_t remain; |
|
- | 453 | loff_t offset, pinned_pages, i; |
|
- | 454 | loff_t first_data_page, last_data_page, num_pages; |
|
- | 455 | int shmem_page_offset; |
|
- | 456 | int data_page_index, data_page_offset; |
|
- | 457 | int page_length; |
|
- | 458 | int ret; |
|
- | 459 | uint64_t data_ptr = args->data_ptr; |
|
- | 460 | int do_bit17_swizzling; |
|
- | 461 | ||
- | 462 | remain = args->size; |
|
- | 463 | ||
- | 464 | /* Pin the user pages containing the data. We can't fault while |
|
- | 465 | * holding the struct mutex, yet we want to hold it while |
|
- | 466 | * dereferencing the user data. |
|
- | 467 | */ |
|
- | 468 | first_data_page = data_ptr / PAGE_SIZE; |
|
- | 469 | last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE; |
|
- | 470 | num_pages = last_data_page - first_data_page + 1; |
|
- | 471 | ||
- | 472 | user_pages = drm_malloc_ab(num_pages, sizeof(struct page *)); |
|
- | 473 | if (user_pages == NULL) |
|
- | 474 | return -ENOMEM; |
|
- | 475 | ||
- | 476 | mutex_unlock(&dev->struct_mutex); |
|
- | 477 | down_read(&mm->mmap_sem); |
|
- | 478 | pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr, |
|
- | 479 | num_pages, 1, 0, user_pages, NULL); |
|
- | 480 | up_read(&mm->mmap_sem); |
|
- | 481 | mutex_lock(&dev->struct_mutex); |
|
- | 482 | if (pinned_pages < num_pages) { |
|
- | 483 | ret = -EFAULT; |
|
- | 484 | goto out; |
|
- | 485 | } |
|
- | 486 | ||
- | 487 | ret = i915_gem_object_set_cpu_read_domain_range(obj, |
|
- | 488 | args->offset, |
|
- | 489 | args->size); |
|
- | 490 | if (ret) |
|
- | 491 | goto out; |
|
- | 492 | ||
- | 493 | do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); |
|
- | 494 | ||
- | 495 | offset = args->offset; |
|
- | 496 | ||
- | 497 | while (remain > 0) { |
|
- | 498 | struct page *page; |
|
- | 499 | ||
- | 500 | /* Operation in this page |
|
- | 501 | * |
|
- | 502 | * shmem_page_offset = offset within page in shmem file |
|
- | 503 | * data_page_index = page number in get_user_pages return |
|
- | 504 | * data_page_offset = offset with data_page_index page. |
|
- | 505 | * page_length = bytes to copy for this page |
|
- | 506 | */ |
|
- | 507 | shmem_page_offset = offset_in_page(offset); |
|
- | 508 | data_page_index = data_ptr / PAGE_SIZE - first_data_page; |
|
- | 509 | data_page_offset = offset_in_page(data_ptr); |
|
- | 510 | ||
- | 511 | page_length = remain; |
|
- | 512 | if ((shmem_page_offset + page_length) > PAGE_SIZE) |
|
- | 513 | page_length = PAGE_SIZE - shmem_page_offset; |
|
- | 514 | if ((data_page_offset + page_length) > PAGE_SIZE) |
|
- | 515 | page_length = PAGE_SIZE - data_page_offset; |
|
- | 516 | ||
- | 517 | page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT); |
|
- | 518 | if (IS_ERR(page)) { |
|
- | 519 | ret = PTR_ERR(page); |
|
- | 520 | goto out; |
|
- | 521 | } |
|
- | 522 | ||
- | 523 | if (do_bit17_swizzling) { |
|
- | 524 | slow_shmem_bit17_copy(page, |
|
- | 525 | shmem_page_offset, |
|
- | 526 | user_pages[data_page_index], |
|
- | 527 | data_page_offset, |
|
- | 528 | page_length, |
|
- | 529 | 1); |
|
- | 530 | } else { |
|
- | 531 | slow_shmem_copy(user_pages[data_page_index], |
|
- | 532 | data_page_offset, |
|
- | 533 | page, |
|
- | 534 | shmem_page_offset, |
|
- | 535 | page_length); |
|
- | 536 | } |
|
- | 537 | ||
- | 538 | mark_page_accessed(page); |
|
- | 539 | page_cache_release(page); |
|
- | 540 | ||
- | 541 | remain -= page_length; |
|
- | 542 | data_ptr += page_length; |
|
- | 543 | offset += page_length; |
|
- | 544 | } |
|
- | 545 | ||
- | 546 | out: |
|
- | 547 | for (i = 0; i < pinned_pages; i++) { |
|
- | 548 | SetPageDirty(user_pages[i]); |
|
- | 549 | mark_page_accessed(user_pages[i]); |
|
- | 550 | page_cache_release(user_pages[i]); |
|
- | 551 | } |
|
- | 552 | drm_free_large(user_pages); |
|
- | 553 | ||
- | 554 | return ret; |
|
- | 555 | } |
|
- | 556 | #endif |
|
- | 557 | ||
- | 558 | ||
- | 559 | ||
- | 560 | ||
- | 561 | ||
- | 562 | ||
- | 563 | ||
- | 564 | ||
- | 565 | ||
- | 566 | ||
- | 567 | ||
- | 568 | ||
- | 569 | ||
- | 570 | ||
- | 571 | ||
- | 572 | ||
- | 573 | ||
- | 574 | ||
- | 575 | ||
- | 576 | ||
- | 577 | ||
- | 578 | ||
- | 579 | ||
- | 580 | ||
- | 581 | ||
- | 582 | ||
- | 583 | ||
- | 584 | ||
- | 585 | ||
- | 586 | ||
- | 587 | ||
- | 588 | ||
- | 589 | ||
- | 590 | ||
- | 591 | ||
- | 592 | ||
- | 593 | ||
- | 594 | ||
- | 595 | ||
- | 596 | ||
- | 597 | ||
- | 598 | ||
- | 599 | ||
- | 600 | ||
- | 601 | ||
- | 602 | ||
- | 603 | ||
- | 604 | ||
- | 605 | ||
- | 606 | ||
- | 607 | ||
- | 608 | ||
- | 609 | ||
- | 610 | ||
- | 611 | ||
- | 612 | ||
- | 613 | ||
- | 614 | ||
- | 615 | ||
- | 616 | ||
- | 617 | ||
- | 618 | ||
- | 619 | ||
- | 620 | ||
- | 621 | ||
- | 622 | ||
- | 623 | ||
- | 624 | ||
- | 625 | ||
- | 626 | ||
- | 627 | ||
- | 628 | ||
- | 629 | ||
- | 630 | static uint32_t |
|
- | 631 | i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) |
|
- | 632 | { |
|
- | 633 | uint32_t gtt_size; |
|
- | 634 | ||
- | 635 | if (INTEL_INFO(dev)->gen >= 4 || |
|
- | 636 | tiling_mode == I915_TILING_NONE) |
|
- | 637 | return size; |
|
- | 638 | ||
- | 639 | /* Previous chips need a power-of-two fence region when tiling */ |
|
- | 640 | if (INTEL_INFO(dev)->gen == 3) |
|
- | 641 | gtt_size = 1024*1024; |
|
- | 642 | else |
|
- | 643 | gtt_size = 512*1024; |
|
- | 644 | ||
- | 645 | while (gtt_size < size) |
|
- | 646 | gtt_size <<= 1; |
|
- | 647 | ||
- | 648 | return gtt_size; |
|
- | 649 | } |
|
- | 650 | ||
- | 651 | /** |
|
- | 652 | * i915_gem_get_gtt_alignment - return required GTT alignment for an object |
|
- | 653 | * @obj: object to check |
|
- | 654 | * |
|
- | 655 | * Return the required GTT alignment for an object, taking into account |
|
- | 656 | * potential fence register mapping. |
|
- | 657 | */ |
|
- | 658 | static uint32_t |
|
- | 659 | i915_gem_get_gtt_alignment(struct drm_device *dev, |
|
- | 660 | uint32_t size, |
|
- | 661 | int tiling_mode) |
|
- | 662 | { |
|
- | 663 | /* |
|
- | 664 | * Minimum alignment is 4k (GTT page size), but might be greater |
|
- | 665 | * if a fence register is needed for the object. |
|
- | 666 | */ |
|
- | 667 | if (INTEL_INFO(dev)->gen >= 4 || |
|
- | 668 | tiling_mode == I915_TILING_NONE) |
|
- | 669 | return 4096; |
|
- | 670 | ||
- | 671 | /* |
|
- | 672 | * Previous chips need to be aligned to the size of the smallest |
|
- | 673 | * fence register that can contain the object. |
|
- | 674 | */ |
|
- | 675 | return i915_gem_get_gtt_size(dev, size, tiling_mode); |
|
- | 676 | } |
|
- | 677 | ||
- | 678 | /** |
|
- | 679 | * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an |
|
- | 680 | * unfenced object |
|
- | 681 | * @dev: the device |
|
- | 682 | * @size: size of the object |
|
- | 683 | * @tiling_mode: tiling mode of the object |
|
- | 684 | * |
|
- | 685 | * Return the required GTT alignment for an object, only taking into account |
|
- | 686 | * unfenced tiled surface requirements. |
|
- | 687 | */ |
|
- | 688 | uint32_t |
|
- | 689 | i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, |
|
- | 690 | uint32_t size, |
|
- | 691 | int tiling_mode) |
|
- | 692 | { |
|
- | 693 | /* |
|
- | 694 | * Minimum alignment is 4k (GTT page size) for sane hw. |
|
- | 695 | */ |
|
- | 696 | if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) || |
|
- | 697 | tiling_mode == I915_TILING_NONE) |
|
- | 698 | return 4096; |
|
- | 699 | ||
- | 700 | /* Previous hardware however needs to be aligned to a power-of-two |
|
- | 701 | * tile height. The simplest method for determining this is to reuse |
|
- | 702 | * the power-of-tile object size. |
|
- | 703 | */ |
|
- | 704 | return i915_gem_get_gtt_size(dev, size, tiling_mode); |
|
- | 705 | } |
|
- | 706 | ||
- | 707 | ||
- | 708 | ||
- | 709 | ||
- | 710 | ||
- | 711 | ||
- | 712 | ||
- | 713 | ||
- | 714 | ||
- | 715 | ||
- | 716 | ||
- | 717 | ||
- | 718 | ||
- | 719 | ||
- | 720 | ||
- | 721 | ||
- | 722 | ||
- | 723 | static int |
|
- | 724 | i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, |
|
- | 725 | gfp_t gfpmask) |
|
- | 726 | { |
|
- | 727 | int page_count, i; |
|
- | 728 | struct page *page; |
|
- | 729 | ||
- | 730 | ENTER(); |
|
- | 731 | ||
- | 732 | /* Get the list of pages out of our struct file. They'll be pinned |
|
- | 733 | * at this point until we release them. |
|
- | 734 | */ |
|
- | 735 | page_count = obj->base.size / PAGE_SIZE; |
|
- | 736 | BUG_ON(obj->pages != NULL); |
|
- | 737 | obj->pages = malloc(page_count * sizeof(struct page *)); |
|
- | 738 | if (obj->pages == NULL) |
|
- | 739 | return -ENOMEM; |
|
- | 740 | ||
- | 741 | ||
- | 742 | for (i = 0; i < page_count; i++) { |
|
- | 743 | page = (struct page*)AllocPage(); // oh-oh |
|
- | 744 | if (IS_ERR(page)) |
|
- | 745 | goto err_pages; |
|
- | 746 | ||
- | 747 | obj->pages[i] = page; |
|
- | 748 | } |
|
- | 749 | ||
- | 750 | // if (obj->tiling_mode != I915_TILING_NONE) |
|
- | 751 | // i915_gem_object_do_bit_17_swizzle(obj); |
|
- | 752 | ||
- | 753 | LEAVE(); |
|
- | 754 | ||
- | 755 | return 0; |
|
- | 756 | ||
- | 757 | err_pages: |
|
- | 758 | // while (i--) |
|
- | 759 | // page_cache_release(obj->pages[i]); |
|
- | 760 | ||
- | 761 | free(obj->pages); |
|
- | 762 | obj->pages = NULL; |
|
- | 763 | return PTR_ERR(page); |
|
- | 764 | } |
|
- | 765 | ||
- | 766 | static void |
|
- | 767 | i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) |
|
- | 768 | { |
|
- | 769 | int page_count = obj->base.size / PAGE_SIZE; |
|
- | 770 | int i; |
|
- | 771 | ||
- | 772 | BUG_ON(obj->madv == __I915_MADV_PURGED); |
|
- | 773 | ||
- | 774 | // if (obj->tiling_mode != I915_TILING_NONE) |
|
- | 775 | // i915_gem_object_save_bit_17_swizzle(obj); |
|
- | 776 | ||
- | 777 | if (obj->madv == I915_MADV_DONTNEED) |
|
- | 778 | obj->dirty = 0; |
|
- | 779 | /* It's a swap!!! |
|
- | 780 | for (i = 0; i < page_count; i++) { |
|
- | 781 | if (obj->dirty) |
|
- | 782 | set_page_dirty(obj->pages[i]); |
|
- | 783 | ||
- | 784 | if (obj->madv == I915_MADV_WILLNEED) |
|
- | 785 | mark_page_accessed(obj->pages[i]); |
|
- | 786 | ||
- | 787 | //page_cache_release(obj->pages[i]); |
|
- | 788 | } |
|
- | 789 | obj->dirty = 0; |
|
- | 790 | */ |
|
- | 791 | ||
- | 792 | free(obj->pages); |
|
- | 793 | obj->pages = NULL; |
|
- | 794 | } |
|
- | 795 | ||
- | 796 | void |
|
- | 797 | i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, |
|
- | 798 | struct intel_ring_buffer *ring, |
|
- | 799 | u32 seqno) |
|
- | 800 | { |
|
- | 801 | struct drm_device *dev = obj->base.dev; |
|
- | 802 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
- | 803 | ||
- | 804 | BUG_ON(ring == NULL); |
|
- | 805 | obj->ring = ring; |
|
- | 806 | ||
- | 807 | /* Add a reference if we're newly entering the active list. */ |
|
- | 808 | if (!obj->active) { |
|
- | 809 | // drm_gem_object_reference(&obj->base); |
|
- | 810 | obj->active = 1; |
|
- | 811 | } |
|
- | 812 | ||
- | 813 | /* Move from whatever list we were on to the tail of execution. */ |
|
- | 814 | list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); |
|
- | 815 | list_move_tail(&obj->ring_list, &ring->active_list); |
|
- | 816 | ||
- | 817 | obj->last_rendering_seqno = seqno; |
|
- | 818 | if (obj->fenced_gpu_access) { |
|
- | 819 | struct drm_i915_fence_reg *reg; |
|
- | 820 | ||
- | 821 | BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE); |
|
- | 822 | ||
- | 823 | obj->last_fenced_seqno = seqno; |
|
- | 824 | obj->last_fenced_ring = ring; |
|
- | 825 | ||
- | 826 | reg = &dev_priv->fence_regs[obj->fence_reg]; |
|
- | 827 | list_move_tail(®->lru_list, &dev_priv->mm.fence_list); |
|
- | 828 | } |
|
- | 829 | } |
|
- | 830 | ||
- | 831 | ||
- | 832 | ||
- | 833 | ||
- | 834 | static void |
|
- | 835 | i915_gem_process_flushing_list(struct intel_ring_buffer *ring, |
|
- | 836 | uint32_t flush_domains) |
|
- | 837 | { |
|
- | 838 | struct drm_i915_gem_object *obj, *next; |
|
- | 839 | ||
- | 840 | list_for_each_entry_safe(obj, next, |
|
- | 841 | &ring->gpu_write_list, |
|
- | 842 | gpu_write_list) { |
|
- | 843 | if (obj->base.write_domain & flush_domains) { |
|
- | 844 | uint32_t old_write_domain = obj->base.write_domain; |
|
- | 845 | ||
- | 846 | obj->base.write_domain = 0; |
|
- | 847 | list_del_init(&obj->gpu_write_list); |
|
- | 848 | i915_gem_object_move_to_active(obj, ring, |
|
- | 849 | i915_gem_next_request_seqno(ring)); |
|
- | 850 | ||
- | 851 | // trace_i915_gem_object_change_domain(obj, |
|
- | 852 | // obj->base.read_domains, |
|
- | 853 | // old_write_domain); |
|
- | 854 | } |
|
- | 855 | } |
|
- | 856 | } |
|
- | 857 | ||
- | 858 | ||
- | 859 | ||
- | 860 | ||
- | 861 | ||
- | 862 | ||
- | 863 | ||
- | 864 | ||
- | 865 | ||
- | 866 | ||
- | 867 | ||
- | 868 | ||
- | 869 | ||
- | 870 | ||
- | 871 | ||
- | 872 | ||
- | 873 | ||
- | 874 | ||
- | 875 | ||
- | 876 | ||
- | 877 | ||
- | 878 | ||
- | 879 | ||
- | 880 | ||
- | 881 | ||
- | 882 | ||
- | 883 | ||
- | 884 | ||
- | 885 | ||
- | 886 | ||
- | 887 | ||
- | 888 | ||
- | 889 | ||
- | 890 | ||
- | 891 | ||
- | 892 | ||
- | 893 | ||
- | 894 | ||
- | 895 | ||
- | 896 | ||
- | 897 | ||
- | 898 | ||
- | 899 | ||
- | 900 | ||
- | 901 | ||
- | 902 | ||
- | 903 | ||
- | 904 | ||
- | 905 | ||
- | 906 | ||
- | 907 | ||
- | 908 | ||
- | 909 | ||
- | 910 | ||
- | 911 | ||
- | 912 | ||
- | 913 | ||
- | 914 | ||
- | 915 | ||
- | 916 | ||
- | 917 | ||
- | 918 | ||
- | 919 | ||
- | 920 | ||
- | 921 | ||
- | 922 | ||
- | 923 | ||
- | 924 | ||
- | 925 | ||
- | 926 | ||
- | 927 | ||
- | 928 | ||
- | 929 | ||
- | 930 | ||
- | 931 | ||
- | 932 | ||
- | 933 | ||
- | 934 | ||
- | 935 | ||
- | 936 | ||
- | 937 | ||
- | 938 | ||
- | 939 | ||
- | 940 | ||
- | 941 | ||
- | 942 | ||
- | 943 | ||
- | 944 | ||
- | 945 | ||
- | 946 | ||
- | 947 | ||
- | 948 | ||
- | 949 | /** |
|
- | 950 | * Ensures that all rendering to the object has completed and the object is |
|
- | 951 | * safe to unbind from the GTT or access from the CPU. |
|
- | 952 | */ |
|
- | 953 | int |
|
- | 954 | i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) |
|
- | 955 | { |
|
- | 956 | int ret; |
|
- | 957 | ||
- | 958 | /* This function only exists to support waiting for existing rendering, |
|
- | 959 | * not for emitting required flushes. |
|
- | 960 | */ |
|
- | 961 | BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0); |
|
- | 962 | ||
- | 963 | /* If there is rendering queued on the buffer being evicted, wait for |
|
- | 964 | * it. |
|
- | 965 | */ |
|
- | 966 | // if (obj->active) { |
|
- | 967 | // ret = i915_wait_request(obj->ring, obj->last_rendering_seqno); |
|
- | 968 | // if (ret) |
|
- | 969 | // return ret; |
|
- | 970 | // } |
|
- | 971 | ||
- | 972 | return 0; |
|
- | 973 | } |
|
- | 974 | ||
- | 975 | ||
- | 976 | int |
|
- | 977 | i915_gem_flush_ring(struct intel_ring_buffer *ring, |
|
- | 978 | uint32_t invalidate_domains, |
|
- | 979 | uint32_t flush_domains) |
|
- | 980 | { |
|
- | 981 | int ret; |
|
- | 982 | ||
- | 983 | if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) |
|
- | 984 | return 0; |
|
- | 985 | ||
- | 986 | // trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains); |
|
- | 987 | ||
- | 988 | ret = ring->flush(ring, invalidate_domains, flush_domains); |
|
- | 989 | if (ret) |
|
- | 990 | return ret; |
|
- | 991 | ||
- | 992 | if (flush_domains & I915_GEM_GPU_DOMAINS) |
|
- | 993 | i915_gem_process_flushing_list(ring, flush_domains); |
|
- | 994 | ||
- | 995 | return 0; |
|
- | 996 | } |
|
- | 997 | ||
- | 998 | ||
- | 999 | ||
- | 1000 | ||
- | 1001 | ||
- | 1002 | ||
- | 1003 | ||
- | 1004 | ||
- | 1005 | ||
- | 1006 | ||
- | 1007 | ||
- | 1008 | ||
- | 1009 | ||
- | 1010 | ||
- | 1011 | ||
- | 1012 | ||
- | 1013 | ||
Line 42... | Line 1014... | ||
42 | #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */ |
1014 | |
43 | 1015 | ||
44 | 1016 | ||
45 | /** |
1017 | /** |
Line 79... | Line 1051... | ||
79 | list_del_init(®->lru_list); |
1051 | list_del_init(®->lru_list); |
80 | reg->obj = NULL; |
1052 | reg->obj = NULL; |
81 | reg->setup_seqno = 0; |
1053 | reg->setup_seqno = 0; |
82 | } |
1054 | } |
Line -... | Line 1055... | ||
- | 1055 | ||
- | 1056 | /** |
|
- | 1057 | * Finds free space in the GTT aperture and binds the object there. |
|
- | 1058 | */ |
|
- | 1059 | static int |
|
- | 1060 | i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, |
|
- | 1061 | unsigned alignment, |
|
- | 1062 | bool map_and_fenceable) |
|
- | 1063 | { |
|
- | 1064 | struct drm_device *dev = obj->base.dev; |
|
- | 1065 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 1066 | struct drm_mm_node *free_space; |
|
- | 1067 | gfp_t gfpmask = 0; //__GFP_NORETRY | __GFP_NOWARN; |
|
- | 1068 | u32 size, fence_size, fence_alignment, unfenced_alignment; |
|
- | 1069 | bool mappable, fenceable; |
|
- | 1070 | int ret; |
|
- | 1071 | ||
- | 1072 | ENTER(); |
|
- | 1073 | ||
- | 1074 | if (obj->madv != I915_MADV_WILLNEED) { |
|
- | 1075 | DRM_ERROR("Attempting to bind a purgeable object\n"); |
|
- | 1076 | return -EINVAL; |
|
- | 1077 | } |
|
- | 1078 | ||
- | 1079 | fence_size = i915_gem_get_gtt_size(dev, |
|
- | 1080 | obj->base.size, |
|
- | 1081 | obj->tiling_mode); |
|
- | 1082 | fence_alignment = i915_gem_get_gtt_alignment(dev, |
|
- | 1083 | obj->base.size, |
|
- | 1084 | obj->tiling_mode); |
|
- | 1085 | unfenced_alignment = |
|
- | 1086 | i915_gem_get_unfenced_gtt_alignment(dev, |
|
- | 1087 | obj->base.size, |
|
- | 1088 | obj->tiling_mode); |
|
- | 1089 | ||
- | 1090 | if (alignment == 0) |
|
- | 1091 | alignment = map_and_fenceable ? fence_alignment : |
|
- | 1092 | unfenced_alignment; |
|
- | 1093 | if (map_and_fenceable && alignment & (fence_alignment - 1)) { |
|
- | 1094 | DRM_ERROR("Invalid object alignment requested %u\n", alignment); |
|
- | 1095 | return -EINVAL; |
|
- | 1096 | } |
|
- | 1097 | ||
- | 1098 | size = map_and_fenceable ? fence_size : obj->base.size; |
|
- | 1099 | ||
- | 1100 | /* If the object is bigger than the entire aperture, reject it early |
|
- | 1101 | * before evicting everything in a vain attempt to find space. |
|
- | 1102 | */ |
|
- | 1103 | if (obj->base.size > |
|
- | 1104 | (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { |
|
- | 1105 | DRM_ERROR("Attempting to bind an object larger than the aperture\n"); |
|
- | 1106 | return -E2BIG; |
|
- | 1107 | } |
|
- | 1108 | ||
- | 1109 | search_free: |
|
- | 1110 | if (map_and_fenceable) |
|
- | 1111 | free_space = |
|
- | 1112 | drm_mm_search_free_in_range(&dev_priv->mm.gtt_space, |
|
- | 1113 | size, alignment, 0, |
|
- | 1114 | dev_priv->mm.gtt_mappable_end, |
|
- | 1115 | 0); |
|
- | 1116 | else |
|
- | 1117 | free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, |
|
- | 1118 | size, alignment, 0); |
|
- | 1119 | ||
- | 1120 | if (free_space != NULL) { |
|
- | 1121 | if (map_and_fenceable) |
|
- | 1122 | obj->gtt_space = |
|
- | 1123 | drm_mm_get_block_range_generic(free_space, |
|
- | 1124 | size, alignment, 0, |
|
- | 1125 | dev_priv->mm.gtt_mappable_end, |
|
- | 1126 | 0); |
|
- | 1127 | else |
|
- | 1128 | obj->gtt_space = |
|
- | 1129 | drm_mm_get_block(free_space, size, alignment); |
|
- | 1130 | } |
|
- | 1131 | if (obj->gtt_space == NULL) { |
|
- | 1132 | /* If the gtt is empty and we're still having trouble |
|
- | 1133 | * fitting our object in, we're out of memory. |
|
- | 1134 | */ |
|
- | 1135 | ret = 1; //i915_gem_evict_something(dev, size, alignment, |
|
- | 1136 | // map_and_fenceable); |
|
- | 1137 | if (ret) |
|
- | 1138 | return ret; |
|
- | 1139 | ||
- | 1140 | goto search_free; |
|
- | 1141 | } |
|
- | 1142 | ||
- | 1143 | ret = i915_gem_object_get_pages_gtt(obj, gfpmask); |
|
- | 1144 | if (ret) { |
|
- | 1145 | drm_mm_put_block(obj->gtt_space); |
|
- | 1146 | obj->gtt_space = NULL; |
|
- | 1147 | #if 0 |
|
- | 1148 | if (ret == -ENOMEM) { |
|
- | 1149 | /* first try to reclaim some memory by clearing the GTT */ |
|
- | 1150 | ret = i915_gem_evict_everything(dev, false); |
|
- | 1151 | if (ret) { |
|
- | 1152 | /* now try to shrink everyone else */ |
|
- | 1153 | if (gfpmask) { |
|
- | 1154 | gfpmask = 0; |
|
- | 1155 | goto search_free; |
|
- | 1156 | } |
|
- | 1157 | ||
- | 1158 | return -ENOMEM; |
|
- | 1159 | } |
|
- | 1160 | ||
- | 1161 | goto search_free; |
|
- | 1162 | } |
|
- | 1163 | #endif |
|
- | 1164 | return ret; |
|
- | 1165 | } |
|
- | 1166 | ||
- | 1167 | ret = i915_gem_gtt_bind_object(obj); |
|
- | 1168 | if (ret) { |
|
- | 1169 | // i915_gem_object_put_pages_gtt(obj); |
|
- | 1170 | drm_mm_put_block(obj->gtt_space); |
|
- | 1171 | obj->gtt_space = NULL; |
|
- | 1172 | ||
- | 1173 | // if (i915_gem_evict_everything(dev, false)) |
|
- | 1174 | return ret; |
|
- | 1175 | ||
- | 1176 | // goto search_free; |
|
- | 1177 | } |
|
- | 1178 | ||
- | 1179 | list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); |
|
- | 1180 | list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); |
|
- | 1181 | ||
- | 1182 | /* Assert that the object is not currently in any GPU domain. As it |
|
- | 1183 | * wasn't in the GTT, there shouldn't be any way it could have been in |
|
- | 1184 | * a GPU cache |
|
- | 1185 | */ |
|
- | 1186 | BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); |
|
- | 1187 | BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); |
|
- | 1188 | ||
- | 1189 | obj->gtt_offset = obj->gtt_space->start; |
|
- | 1190 | ||
- | 1191 | fenceable = |
|
- | 1192 | obj->gtt_space->size == fence_size && |
|
- | 1193 | (obj->gtt_space->start & (fence_alignment -1)) == 0; |
|
- | 1194 | ||
- | 1195 | mappable = |
|
- | 1196 | obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; |
|
- | 1197 | ||
- | 1198 | obj->map_and_fenceable = mappable && fenceable; |
|
- | 1199 | ||
- | 1200 | LEAVE(); |
|
- | 1201 | // trace_i915_gem_object_bind(obj, map_and_fenceable); |
|
- | 1202 | return 0; |
|
- | 1203 | } |
|
- | 1204 | ||
- | 1205 | void |
|
- | 1206 | i915_gem_clflush_object(struct drm_i915_gem_object *obj) |
|
- | 1207 | { |
|
- | 1208 | /* If we don't have a page list set up, then we're not pinned |
|
- | 1209 | * to GPU, and we can ignore the cache flush because it'll happen |
|
- | 1210 | * again at bind time. |
|
- | 1211 | */ |
|
- | 1212 | if (obj->pages == NULL) |
|
- | 1213 | return; |
|
- | 1214 | ||
- | 1215 | /* If the GPU is snooping the contents of the CPU cache, |
|
- | 1216 | * we do not need to manually clear the CPU cache lines. However, |
|
- | 1217 | * the caches are only snooped when the render cache is |
|
- | 1218 | * flushed/invalidated. As we always have to emit invalidations |
|
- | 1219 | * and flushes when moving into and out of the RENDER domain, correct |
|
- | 1220 | * snooping behaviour occurs naturally as the result of our domain |
|
- | 1221 | * tracking. |
|
- | 1222 | */ |
|
- | 1223 | if (obj->cache_level != I915_CACHE_NONE) |
|
- | 1224 | return; |
|
- | 1225 | ||
- | 1226 | // trace_i915_gem_object_clflush(obj); |
|
- | 1227 | ||
- | 1228 | // drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); |
|
- | 1229 | mb(); |
|
- | 1230 | __asm__ ("wbinvd"); // this is really ugly |
|
- | 1231 | mb(); |
|
- | 1232 | } |
|
- | 1233 | ||
- | 1234 | /** Flushes any GPU write domain for the object if it's dirty. */ |
|
- | 1235 | static int |
|
- | 1236 | i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) |
|
- | 1237 | { |
|
- | 1238 | if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) |
|
- | 1239 | return 0; |
|
- | 1240 | ||
- | 1241 | /* Queue the GPU write cache flushing we need. */ |
|
- | 1242 | return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); |
|
- | 1243 | } |
|
- | 1244 | ||
- | 1245 | ||
- | 1246 | ||
- | 1247 | ||
- | 1248 | ||
- | 1249 | ||
- | 1250 | /** Flushes the CPU write domain for the object if it's dirty. */ |
|
- | 1251 | static void |
|
- | 1252 | i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) |
|
- | 1253 | { |
|
- | 1254 | uint32_t old_write_domain; |
|
- | 1255 | ||
- | 1256 | if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) |
|
- | 1257 | return; |
|
- | 1258 | ||
- | 1259 | i915_gem_clflush_object(obj); |
|
- | 1260 | intel_gtt_chipset_flush(); |
|
- | 1261 | old_write_domain = obj->base.write_domain; |
|
- | 1262 | obj->base.write_domain = 0; |
|
- | 1263 | ||
- | 1264 | // trace_i915_gem_object_change_domain(obj, |
|
- | 1265 | // obj->base.read_domains, |
|
- | 1266 | // old_write_domain); |
|
- | 1267 | } |
|
- | 1268 | ||
- | 1269 | /** |
|
- | 1270 | * Moves a single object to the GTT read, and possibly write domain. |
|
- | 1271 | * |
|
- | 1272 | * This function returns when the move is complete, including waiting on |
|
- | 1273 | * flushes to occur. |
|
- | 1274 | */ |
|
- | 1275 | int |
|
- | 1276 | i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) |
|
- | 1277 | { |
|
- | 1278 | uint32_t old_write_domain, old_read_domains; |
|
- | 1279 | int ret; |
|
- | 1280 | ||
- | 1281 | /* Not valid to be called on unbound objects. */ |
|
- | 1282 | if (obj->gtt_space == NULL) |
|
- | 1283 | return -EINVAL; |
|
- | 1284 | ||
- | 1285 | if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) |
|
- | 1286 | return 0; |
|
- | 1287 | ||
- | 1288 | ret = i915_gem_object_flush_gpu_write_domain(obj); |
|
- | 1289 | if (ret) |
|
- | 1290 | return ret; |
|
- | 1291 | ||
- | 1292 | if (obj->pending_gpu_write || write) { |
|
- | 1293 | ret = i915_gem_object_wait_rendering(obj); |
|
- | 1294 | if (ret) |
|
- | 1295 | return ret; |
|
- | 1296 | } |
|
- | 1297 | ||
- | 1298 | i915_gem_object_flush_cpu_write_domain(obj); |
|
- | 1299 | ||
- | 1300 | old_write_domain = obj->base.write_domain; |
|
- | 1301 | old_read_domains = obj->base.read_domains; |
|
- | 1302 | ||
- | 1303 | /* It should now be out of any other write domains, and we can update |
|
- | 1304 | * the domain values for our changes. |
|
- | 1305 | */ |
|
- | 1306 | BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); |
|
- | 1307 | obj->base.read_domains |= I915_GEM_DOMAIN_GTT; |
|
- | 1308 | if (write) { |
|
- | 1309 | obj->base.read_domains = I915_GEM_DOMAIN_GTT; |
|
- | 1310 | obj->base.write_domain = I915_GEM_DOMAIN_GTT; |
|
- | 1311 | obj->dirty = 1; |
|
- | 1312 | } |
|
- | 1313 | ||
- | 1314 | return 0; |
|
- | 1315 | } |
|
- | 1316 | ||
- | 1317 | ||
- | 1318 | ||
- | 1319 | ||
- | 1320 | ||
- | 1321 | ||
- | 1322 | ||
- | 1323 | ||
- | 1324 | ||
- | 1325 | ||
- | 1326 | ||
- | 1327 | ||
- | 1328 | ||
- | 1329 | ||
- | 1330 | ||
- | 1331 | ||
- | 1332 | ||
- | 1333 | ||
- | 1334 | ||
- | 1335 | ||
- | 1336 | ||
- | 1337 | ||
- | 1338 | ||
- | 1339 | ||
- | 1340 | ||
- | 1341 | ||
- | 1342 | ||
- | 1343 | ||
- | 1344 | ||
- | 1345 | ||
- | 1346 | ||
- | 1347 | ||
- | 1348 | ||
- | 1349 | ||
- | 1350 | ||
- | 1351 | ||
- | 1352 | ||
- | 1353 | ||
- | 1354 | ||
- | 1355 | ||
- | 1356 | ||
- | 1357 | ||
- | 1358 | ||
- | 1359 | ||
- | 1360 | ||
- | 1361 | ||
- | 1362 | ||
- | 1363 | ||
- | 1364 | ||
- | 1365 | ||
- | 1366 | ||
- | 1367 | ||
- | 1368 | ||
- | 1369 | ||
- | 1370 | ||
- | 1371 | ||
- | 1372 | ||
- | 1373 | ||
- | 1374 | ||
- | 1375 | ||
- | 1376 | ||
- | 1377 | ||
- | 1378 | ||
- | 1379 | ||
- | 1380 | ||
- | 1381 | ||
- | 1382 | ||
- | 1383 | int |
|
- | 1384 | i915_gem_object_pin(struct drm_i915_gem_object *obj, |
|
- | 1385 | uint32_t alignment, |
|
- | 1386 | bool map_and_fenceable) |
|
- | 1387 | { |
|
- | 1388 | struct drm_device *dev = obj->base.dev; |
|
- | 1389 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
- | 1390 | int ret; |
|
- | 1391 | ||
- | 1392 | ENTER(); |
|
- | 1393 | ||
- | 1394 | BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT); |
|
- | 1395 | // WARN_ON(i915_verify_lists(dev)); |
|
- | 1396 | ||
- | 1397 | #if 0 |
|
- | 1398 | if (obj->gtt_space != NULL) { |
|
- | 1399 | if ((alignment && obj->gtt_offset & (alignment - 1)) || |
|
- | 1400 | (map_and_fenceable && !obj->map_and_fenceable)) { |
|
- | 1401 | WARN(obj->pin_count, |
|
- | 1402 | "bo is already pinned with incorrect alignment:" |
|
- | 1403 | " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," |
|
- | 1404 | " obj->map_and_fenceable=%d\n", |
|
- | 1405 | obj->gtt_offset, alignment, |
|
- | 1406 | map_and_fenceable, |
|
- | 1407 | obj->map_and_fenceable); |
|
- | 1408 | ret = i915_gem_object_unbind(obj); |
|
- | 1409 | if (ret) |
|
- | 1410 | return ret; |
|
- | 1411 | } |
|
- | 1412 | } |
|
- | 1413 | #endif |
|
- | 1414 | ||
- | 1415 | if (obj->gtt_space == NULL) { |
|
- | 1416 | ret = i915_gem_object_bind_to_gtt(obj, alignment, |
|
- | 1417 | map_and_fenceable); |
|
- | 1418 | if (ret) |
|
- | 1419 | return ret; |
|
- | 1420 | } |
|
- | 1421 | ||
- | 1422 | if (obj->pin_count++ == 0) { |
|
- | 1423 | if (!obj->active) |
|
- | 1424 | list_move_tail(&obj->mm_list, |
|
- | 1425 | &dev_priv->mm.pinned_list); |
|
- | 1426 | } |
|
- | 1427 | obj->pin_mappable |= map_and_fenceable; |
|
- | 1428 | ||
- | 1429 | LEAVE(); |
|
- | 1430 | ||
- | 1431 | // WARN_ON(i915_verify_lists(dev)); |
|
- | 1432 | return 0; |
|
- | 1433 | } |
|
- | 1434 | ||
- | 1435 | ||
- | 1436 | ||
- | 1437 | ||
- | 1438 | ||
- | 1439 | ||
- | 1440 | ||
- | 1441 | ||
- | 1442 | ||
- | 1443 | ||
- | 1444 | ||
- | 1445 | ||
- | 1446 | ||
- | 1447 | ||
- | 1448 | ||
- | 1449 | ||
- | 1450 | ||
- | 1451 | ||
- | 1452 | ||
- | 1453 | ||
- | 1454 | ||
- | 1455 | ||
- | 1456 | ||
- | 1457 | ||
- | 1458 | ||
- | 1459 | ||
- | 1460 | ||
- | 1461 | ||
- | 1462 | ||
- | 1463 | ||
- | 1464 | ||
- | 1465 | ||
- | 1466 | ||
- | 1467 | ||
- | 1468 | ||
- | 1469 | ||
- | 1470 | ||
- | 1471 | ||
- | 1472 | ||
- | 1473 | ||
- | 1474 | ||
- | 1475 | ||
- | 1476 | struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, |
|
- | 1477 | size_t size) |
|
- | 1478 | { |
|
- | 1479 | struct drm_i915_private *dev_priv = dev->dev_private; |
|
- | 1480 | struct drm_i915_gem_object *obj; |
|
- | 1481 | ENTER(); |
|
- | 1482 | obj = kzalloc(sizeof(*obj), GFP_KERNEL); |
|
- | 1483 | if (obj == NULL) |
|
- | 1484 | return NULL; |
|
- | 1485 | ||
- | 1486 | if (drm_gem_object_init(dev, &obj->base, size) != 0) { |
|
- | 1487 | kfree(obj); |
|
- | 1488 | return NULL; |
|
- | 1489 | } |
|
- | 1490 | ||
- | 1491 | ||
- | 1492 | i915_gem_info_add_obj(dev_priv, size); |
|
- | 1493 | ||
- | 1494 | obj->base.write_domain = I915_GEM_DOMAIN_CPU; |
|
- | 1495 | obj->base.read_domains = I915_GEM_DOMAIN_CPU; |
|
- | 1496 | ||
- | 1497 | if (IS_GEN6(dev)) { |
|
- | 1498 | /* On Gen6, we can have the GPU use the LLC (the CPU |
|
- | 1499 | * cache) for about a 10% performance improvement |
|
- | 1500 | * compared to uncached. Graphics requests other than |
|
- | 1501 | * display scanout are coherent with the CPU in |
|
- | 1502 | * accessing this cache. This means in this mode we |
|
- | 1503 | * don't need to clflush on the CPU side, and on the |
|
- | 1504 | * GPU side we only need to flush internal caches to |
|
- | 1505 | * get data visible to the CPU. |
|
- | 1506 | * |
|
- | 1507 | * However, we maintain the display planes as UC, and so |
|
- | 1508 | * need to rebind when first used as such. |
|
- | 1509 | */ |
|
- | 1510 | obj->cache_level = I915_CACHE_LLC; |
|
- | 1511 | } else |
|
- | 1512 | obj->cache_level = I915_CACHE_NONE; |
|
- | 1513 | ||
- | 1514 | obj->base.driver_private = NULL; |
|
- | 1515 | obj->fence_reg = I915_FENCE_REG_NONE; |
|
- | 1516 | INIT_LIST_HEAD(&obj->mm_list); |
|
- | 1517 | INIT_LIST_HEAD(&obj->gtt_list); |
|
- | 1518 | INIT_LIST_HEAD(&obj->ring_list); |
|
- | 1519 | INIT_LIST_HEAD(&obj->exec_list); |
|
- | 1520 | INIT_LIST_HEAD(&obj->gpu_write_list); |
|
- | 1521 | obj->madv = I915_MADV_WILLNEED; |
|
- | 1522 | /* Avoid an unnecessary call to unbind on the first bind. */ |
|
- | 1523 | obj->map_and_fenceable = true; |
|
- | 1524 | LEAVE(); |
|
- | 1525 | return obj; |
|
- | 1526 | } |
|
- | 1527 | ||
- | 1528 | ||
- | 1529 | ||
- | 1530 | ||
- | 1531 | ||
- | 1532 | ||
- | 1533 | ||
- | 1534 | ||
- | 1535 | ||
- | 1536 | ||
- | 1537 | ||
- | 1538 | ||
- | 1539 | ||
- | 1540 | ||
- | 1541 | ||
- | 1542 | ||
- | 1543 | ||
- | 1544 | ||
- | 1545 | ||
- | 1546 | ||
- | 1547 | ||
- | 1548 | ||
- | 1549 | ||
- | 1550 | ||
- | 1551 | int |
|
- | 1552 | i915_gem_init_ringbuffer(struct drm_device *dev) |
|
- | 1553 | { |
|
- | 1554 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 1555 | int ret; |
|
- | 1556 | ENTER(); |
|
- | 1557 | ret = intel_init_render_ring_buffer(dev); |
|
- | 1558 | if (ret) |
|
- | 1559 | return ret; |
|
- | 1560 | ||
- | 1561 | if (HAS_BSD(dev)) { |
|
- | 1562 | ret = intel_init_bsd_ring_buffer(dev); |
|
- | 1563 | if (ret) |
|
- | 1564 | goto cleanup_render_ring; |
|
- | 1565 | } |
|
- | 1566 | ||
- | 1567 | if (HAS_BLT(dev)) { |
|
- | 1568 | ret = intel_init_blt_ring_buffer(dev); |
|
- | 1569 | if (ret) |
|
- | 1570 | goto cleanup_bsd_ring; |
|
- | 1571 | } |
|
- | 1572 | ||
- | 1573 | dev_priv->next_seqno = 1; |
|
- | 1574 | LEAVE(); |
|
- | 1575 | return 0; |
|
- | 1576 | ||
- | 1577 | cleanup_bsd_ring: |
|
- | 1578 | intel_cleanup_ring_buffer(&dev_priv->ring[VCS]); |
|
- | 1579 | cleanup_render_ring: |
|
- | 1580 | intel_cleanup_ring_buffer(&dev_priv->ring[RCS]); |
|
- | 1581 | return ret; |
|
- | 1582 | } |
|
- | 1583 | ||
- | 1584 | #if 0 |
|
- | 1585 | void |
|
- | 1586 | i915_gem_cleanup_ringbuffer(struct drm_device *dev) |
|
- | 1587 | { |
|
- | 1588 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 1589 | int i; |
|
- | 1590 | ||
- | 1591 | for (i = 0; i < I915_NUM_RINGS; i++) |
|
- | 1592 | intel_cleanup_ring_buffer(&dev_priv->ring[i]); |
|
- | 1593 | } |
|
- | 1594 | ||
- | 1595 | int |
|
- | 1596 | i915_gem_entervt_ioctl(struct drm_device *dev, void *data, |
|
- | 1597 | struct drm_file *file_priv) |
|
- | 1598 | { |
|
- | 1599 | drm_i915_private_t *dev_priv = dev->dev_private; |
|
- | 1600 | int ret, i; |
|
- | 1601 | ||
- | 1602 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
|
- | 1603 | return 0; |
|
- | 1604 | ||
- | 1605 | if (atomic_read(&dev_priv->mm.wedged)) { |
|
- | 1606 | DRM_ERROR("Reenabling wedged hardware, good luck\n"); |
|
- | 1607 | atomic_set(&dev_priv->mm.wedged, 0); |
|
- | 1608 | } |
|
- | 1609 | ||
- | 1610 | mutex_lock(&dev->struct_mutex); |
|
- | 1611 | dev_priv->mm.suspended = 0; |
|
- | 1612 | ||
- | 1613 | ret = i915_gem_init_ringbuffer(dev); |
|
- | 1614 | if (ret != 0) { |
|
- | 1615 | mutex_unlock(&dev->struct_mutex); |
|
- | 1616 | return ret; |
|
- | 1617 | } |
|
- | 1618 | ||
- | 1619 | BUG_ON(!list_empty(&dev_priv->mm.active_list)); |
|
- | 1620 | BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); |
|
- | 1621 | BUG_ON(!list_empty(&dev_priv->mm.inactive_list)); |
|
- | 1622 | for (i = 0; i < I915_NUM_RINGS; i++) { |
|
- | 1623 | BUG_ON(!list_empty(&dev_priv->ring[i].active_list)); |
|
- | 1624 | BUG_ON(!list_empty(&dev_priv->ring[i].request_list)); |
|
- | 1625 | } |
|
- | 1626 | mutex_unlock(&dev->struct_mutex); |
|
- | 1627 | ||
- | 1628 | ret = drm_irq_install(dev); |
|
- | 1629 | if (ret) |
|
- | 1630 | goto cleanup_ringbuffer; |
|
- | 1631 | ||
- | 1632 | return 0; |
|
- | 1633 | ||
- | 1634 | cleanup_ringbuffer: |
|
- | 1635 | mutex_lock(&dev->struct_mutex); |
|
- | 1636 | i915_gem_cleanup_ringbuffer(dev); |
|
- | 1637 | dev_priv->mm.suspended = 1; |
|
- | 1638 | mutex_unlock(&dev->struct_mutex); |
|
- | 1639 | ||
- | 1640 | return ret; |
|
- | 1641 | } |
|
- | 1642 | ||
- | 1643 | int |
|
- | 1644 | i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, |
|
- | 1645 | struct drm_file *file_priv) |
|
- | 1646 | { |
|
- | 1647 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
|
- | 1648 | return 0; |
|
- | 1649 | ||
- | 1650 | drm_irq_uninstall(dev); |
|
- | 1651 | return i915_gem_idle(dev); |
|
- | 1652 | } |
|
- | 1653 | ||
- | 1654 | void |
|
- | 1655 | i915_gem_lastclose(struct drm_device *dev) |
|
- | 1656 | { |
|
- | 1657 | int ret; |
|
- | 1658 | ||
- | 1659 | if (drm_core_check_feature(dev, DRIVER_MODESET)) |
|
- | 1660 | return; |
|
- | 1661 | ||
- | 1662 | ret = i915_gem_idle(dev); |
|
- | 1663 | if (ret) |
|
- | 1664 | DRM_ERROR("failed to idle hardware: %d\n", ret); |
|
- | 1665 | } |
|
Line 83... | Line 1666... | ||
83 | 1666 | #endif |
|
84 | 1667 | ||
85 | static void |
1668 | static void |
86 | init_ring_lists(struct intel_ring_buffer *ring) |
1669 | init_ring_lists(struct intel_ring_buffer *ring) |
87 | { |
1670 | { |
88 | INIT_LIST_HEAD(&ring->active_list); |
1671 | INIT_LIST_HEAD(&ring->active_list); |
89 | INIT_LIST_HEAD(&ring->request_list); |
1672 | INIT_LIST_HEAD(&ring->request_list); |
Line 90... | Line -... | ||
90 | INIT_LIST_HEAD(&ring->gpu_write_list); |
- | |
91 | } |
1673 | INIT_LIST_HEAD(&ring->gpu_write_list); |
92 | 1674 | } |
|
93 | 1675 | ||
94 | void |
1676 | void |
95 | i915_gem_load(struct drm_device *dev) |
1677 | i915_gem_load(struct drm_device *dev) |